├── .coveragerc ├── .cursorrules ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── pull_request_template.md └── workflows │ ├── build_and_test.yml │ ├── build_desktop.yml │ ├── build_docs.yml │ ├── format_and_lint.yml │ ├── test_count.yml │ ├── web_format_lint_build.yml │ └── windows_release_build.yml ├── .gitignore ├── .vscode ├── launch.json └── settings.json ├── CLA.md ├── CONTRIBUTING.md ├── LICENSE.txt ├── README.md ├── app ├── EULA.md ├── LICENSE.txt ├── __init__.py ├── desktop │ ├── .gitignore │ ├── .python-version │ ├── README.md │ ├── WinInnoSetup.iss │ ├── __init__.py │ ├── build_desktop_app.sh │ ├── custom_tray.py │ ├── desktop.py │ ├── desktop_server.py │ ├── dev_server.py │ ├── dmg │ │ ├── README.md │ │ └── appdmg.json │ ├── log_config.py │ ├── mac_entitlements.plist │ ├── mac_icon.png │ ├── mac_taskbar.png │ ├── pyproject.toml │ ├── run_desktop_dev.sh │ ├── studio_server │ │ ├── correlation_calculator.py │ │ ├── data_gen_api.py │ │ ├── eval_api.py │ │ ├── finetune_api.py │ │ ├── prompt_api.py │ │ ├── provider_api.py │ │ ├── repair_api.py │ │ ├── settings_api.py │ │ ├── test_correlation_calculator.py │ │ ├── test_data_gen_api.py │ │ ├── test_eval_api.py │ │ ├── test_finetune_api.py │ │ ├── test_generate_docs.py │ │ ├── test_prompt_api.py │ │ ├── test_provider_api.py │ │ ├── test_repair_api.py │ │ ├── test_server.py │ │ ├── test_settings_api.py │ │ └── webhost.py │ ├── taskbar.png │ ├── test_desktop.py │ ├── win_icon.ico │ ├── win_icon.png │ ├── win_splash.png │ └── win_taskbar.png └── web_ui │ ├── .eslintignore │ ├── .eslintrc.cjs │ ├── .gitignore │ ├── .npmrc │ ├── .prettierignore │ ├── .prettierrc │ ├── package-lock.json │ ├── package.json │ ├── postcss.config.js │ ├── src │ ├── ambient.d.ts │ ├── app.css │ ├── app.d.ts │ ├── app.html │ ├── config.ts │ ├── index.test.ts │ ├── lib │ │ ├── api_client.ts │ │ ├── api_schema.d.ts │ │ ├── generate_schema.sh │ │ ├── stores.test.ts │ │ ├── stores.ts │ │ ├── stores │ │ │ ├── fine_tune_store.ts │ │ │ ├── index_db_store.test.ts │ │ │ ├── index_db_store.ts │ │ │ └── progress_ui_store.ts │ │ ├── types.ts │ │ ├── ui │ │ │ ├── completed.svelte │ │ │ ├── delete_dialog.svelte │ │ │ ├── dialog.svelte │ │ │ ├── edit_dialog.svelte │ │ │ ├── fancy_select.svelte │ │ │ ├── fancy_select_types.ts │ │ │ ├── info_tooltip.svelte │ │ │ ├── option_list.svelte │ │ │ ├── progress_widget.svelte │ │ │ ├── property_list.svelte │ │ │ ├── splits.svelte │ │ │ └── warning.svelte │ │ └── utils │ │ │ ├── error_handlers.ts │ │ │ ├── form_container.svelte │ │ │ ├── form_element.svelte │ │ │ ├── form_list.svelte │ │ │ ├── formatters.ts │ │ │ ├── json_schema_editor │ │ │ ├── json_schema_form_element.svelte │ │ │ ├── json_schema_templates.test.ts │ │ │ └── json_schema_templates.ts │ │ │ ├── link_builder.ts │ │ │ ├── update.test.ts │ │ │ └── update.ts │ └── routes │ │ ├── (app) │ │ ├── +layout.svelte │ │ ├── +page.svelte │ │ ├── +page.ts │ │ ├── app_page.svelte │ │ ├── dataset │ │ │ └── [project_id] │ │ │ │ └── [task_id] │ │ │ │ ├── +page.svelte │ │ │ │ ├── +page.ts │ │ │ │ ├── [run_id] │ │ │ │ └── run │ │ │ │ │ ├── +page.svelte │ │ │ │ │ └── +page.ts │ │ │ │ ├── add_data │ │ │ │ ├── +page.svelte │ │ │ │ └── +page.ts │ │ │ │ ├── empty_into.svelte │ │ │ │ └── upload_dataset_dialog.svelte │ │ ├── evals │ │ │ └── [project_id] │ │ │ │ └── [task_id] │ │ │ │ ├── +page.svelte │ │ │ │ ├── +page.ts │ │ │ │ ├── [eval_id] │ │ │ │ ├── +page.svelte │ │ │ │ ├── +page.ts │ │ │ │ ├── [eval_config_id] │ │ │ │ │ └── [run_config_id] │ │ │ │ │ │ └── run_result │ │ │ │ │ │ ├── +page.svelte │ │ │ │ │ │ └── +page.ts │ │ │ │ ├── compare_run_methods │ │ │ │ │ ├── +page.svelte │ │ │ │ │ └── +page.ts │ │ │ │ ├── create_eval_config │ │ │ │ │ ├── +page.svelte │ │ │ │ │ └── +page.ts │ │ │ │ ├── eval_configs │ │ │ │ │ ├── +page.svelte │ │ │ │ │ ├── +page.ts │ │ │ │ │ └── eval_config_instruction.svelte │ │ │ │ ├── output_type_table_preview.svelte │ │ │ │ └── run_eval.svelte │ │ │ │ ├── create_evaluator │ │ │ │ ├── +page.svelte │ │ │ │ ├── +page.ts │ │ │ │ ├── eval_template.ts │ │ │ │ └── select_eval_template.svelte │ │ │ │ └── empty_eval.svelte │ │ ├── fine_tune │ │ │ └── [project_id] │ │ │ │ └── [task_id] │ │ │ │ ├── +page.svelte │ │ │ │ ├── +page.ts │ │ │ │ ├── create_finetune │ │ │ │ ├── +page.svelte │ │ │ │ ├── +page.ts │ │ │ │ └── select_finetune_dataset.svelte │ │ │ │ ├── empty_finetune.svelte │ │ │ │ └── fine_tune │ │ │ │ └── [finetune_id] │ │ │ │ ├── +page.svelte │ │ │ │ └── +page.ts │ │ ├── generate │ │ │ └── [project_id] │ │ │ │ └── [task_id] │ │ │ │ ├── +page.svelte │ │ │ │ ├── +page.ts │ │ │ │ ├── data_gen_intro.svelte │ │ │ │ ├── gen_model.ts │ │ │ │ ├── generate_samples_modal.svelte │ │ │ │ ├── generated_data_node.svelte │ │ │ │ └── increment_ui.svelte │ │ ├── prompts │ │ │ └── [project_id] │ │ │ │ └── [task_id] │ │ │ │ ├── +page.svelte │ │ │ │ ├── +page.ts │ │ │ │ ├── create │ │ │ │ ├── +page.svelte │ │ │ │ └── +page.ts │ │ │ │ ├── generator_details │ │ │ │ └── [generator_id] │ │ │ │ │ ├── +page.svelte │ │ │ │ │ └── +page.ts │ │ │ │ └── saved │ │ │ │ └── [prompt_id] │ │ │ │ ├── +page.svelte │ │ │ │ └── +page.ts │ │ ├── run │ │ │ ├── +page.svelte │ │ │ ├── available_models_dropdown.svelte │ │ │ ├── output.svelte │ │ │ ├── output_repair_edit_form.svelte │ │ │ ├── prompt_type_selector.svelte │ │ │ ├── rating.svelte │ │ │ ├── run.svelte │ │ │ ├── run_input_form.svelte │ │ │ └── tag_dropdown.svelte │ │ ├── select_tasks_menu.svelte │ │ ├── settings │ │ │ ├── +page.svelte │ │ │ ├── check_for_update │ │ │ │ └── +page.svelte │ │ │ ├── clone_task │ │ │ │ └── [project_id] │ │ │ │ │ └── [task_id] │ │ │ │ │ ├── +page.svelte │ │ │ │ │ └── +page.ts │ │ │ ├── create_project │ │ │ │ └── +page.svelte │ │ │ ├── create_task │ │ │ │ └── [slug] │ │ │ │ │ ├── +page.svelte │ │ │ │ │ └── +page.ts │ │ │ ├── edit_project │ │ │ │ └── [slug] │ │ │ │ │ ├── +page.svelte │ │ │ │ │ └── +page.ts │ │ │ ├── edit_task │ │ │ │ └── [project_id] │ │ │ │ │ └── [task_id] │ │ │ │ │ ├── +page.svelte │ │ │ │ │ ├── +page.ts │ │ │ │ │ └── load_task_editor.svelte │ │ │ ├── intro │ │ │ │ └── +page.svelte │ │ │ ├── manage_projects │ │ │ │ └── +page.svelte │ │ │ └── providers │ │ │ │ ├── +page.svelte │ │ │ │ └── add_models │ │ │ │ └── +page.svelte │ │ └── types.ts │ │ ├── (fullscreen) │ │ ├── +layout.svelte │ │ └── setup │ │ │ ├── (setup) │ │ │ ├── +layout.svelte │ │ │ ├── connect_providers │ │ │ │ ├── +page.svelte │ │ │ │ └── connect_providers.svelte │ │ │ ├── create_project │ │ │ │ ├── +page.svelte │ │ │ │ └── edit_project.svelte │ │ │ ├── create_task │ │ │ │ ├── [slug] │ │ │ │ │ ├── +page.svelte │ │ │ │ │ └── +page.ts │ │ │ │ ├── edit_task.svelte │ │ │ │ └── schema_section.svelte │ │ │ ├── intro │ │ │ │ ├── +page.svelte │ │ │ │ ├── tutorial.svelte │ │ │ │ └── tutorial_section.svelte │ │ │ ├── select_task │ │ │ │ └── +page.svelte │ │ │ └── subscribe │ │ │ │ └── +page.svelte │ │ │ └── +page.svelte │ │ ├── +error.svelte │ │ ├── +layout.svelte │ │ └── +layout.ts │ ├── static │ ├── favicon.ico │ ├── favicon.png │ ├── favicons │ │ ├── apple-touch-icon.png │ │ ├── favicon-16x16.png │ │ ├── favicon-32x32.png │ │ ├── icon_192.png │ │ ├── icon_512.png │ │ └── site.webmanifest │ ├── images │ │ ├── add.svg │ │ ├── animated_logo.svg │ │ ├── anthropic.svg │ │ ├── api.svg │ │ ├── aws.svg │ │ ├── azure_openai.svg │ │ ├── circle-check.svg │ │ ├── collaborate.png │ │ ├── data_driven.png │ │ ├── data_gen_tree.png │ │ ├── delete.svg │ │ ├── developers.png │ │ ├── filter.svg │ │ ├── fireworks.svg │ │ ├── gemini.svg │ │ ├── google_logo.svg │ │ ├── groq.svg │ │ ├── hugging_face.svg │ │ ├── logo_grid.png │ │ ├── next.svg │ │ ├── ollama.svg │ │ ├── openai.svg │ │ ├── openrouter.svg │ │ ├── previous.svg │ │ ├── tag.svg │ │ ├── together_ai.svg │ │ ├── training.png │ │ └── wandb.svg │ ├── logo.svg │ └── styles │ │ └── highlightjs.min.css │ ├── svelte.config.js │ ├── tailwind.config.js │ ├── tsconfig.json │ └── vite.config.ts ├── checks.sh ├── conftest.py ├── guides ├── Collaborating with Kiln.md ├── Fine Tuning LLM Models Guide.md ├── Model Support.md ├── Synthetic Data Generation.md ├── kiln_preview.avif └── kiln_preview.gif ├── libs ├── core │ ├── .python-version │ ├── LICENSE.txt │ ├── README.md │ ├── kiln_ai │ │ ├── __init__.py │ │ ├── adapters │ │ │ ├── __init__.py │ │ │ ├── adapter_registry.py │ │ │ ├── data_gen │ │ │ │ ├── __init__.py │ │ │ │ ├── data_gen_prompts.py │ │ │ │ ├── data_gen_task.py │ │ │ │ └── test_data_gen_task.py │ │ │ ├── eval │ │ │ │ ├── __init__.py │ │ │ │ ├── base_eval.py │ │ │ │ ├── eval_runner.py │ │ │ │ ├── g_eval.py │ │ │ │ ├── registry.py │ │ │ │ ├── test_base_eval.py │ │ │ │ ├── test_eval_runner.py │ │ │ │ ├── test_g_eval.py │ │ │ │ └── test_g_eval_data.py │ │ │ ├── fine_tune │ │ │ │ ├── __init__.py │ │ │ │ ├── base_finetune.py │ │ │ │ ├── dataset_formatter.py │ │ │ │ ├── finetune_registry.py │ │ │ │ ├── fireworks_finetune.py │ │ │ │ ├── openai_finetune.py │ │ │ │ ├── test_base_finetune.py │ │ │ │ ├── test_dataset_formatter.py │ │ │ │ ├── test_fireworks_tinetune.py │ │ │ │ ├── test_openai_finetune.py │ │ │ │ ├── test_together_finetune.py │ │ │ │ ├── test_vertex_finetune.py │ │ │ │ ├── together_finetune.py │ │ │ │ └── vertex_finetune.py │ │ │ ├── ml_model_list.py │ │ │ ├── model_adapters │ │ │ │ ├── __init__.py │ │ │ │ ├── base_adapter.py │ │ │ │ ├── litellm_adapter.py │ │ │ │ ├── litellm_config.py │ │ │ │ ├── test_base_adapter.py │ │ │ │ ├── test_litellm_adapter.py │ │ │ │ ├── test_saving_adapter_results.py │ │ │ │ └── test_structured_output.py │ │ │ ├── ollama_tools.py │ │ │ ├── parsers │ │ │ │ ├── __init__.py │ │ │ │ ├── base_parser.py │ │ │ │ ├── json_parser.py │ │ │ │ ├── parser_registry.py │ │ │ │ ├── r1_parser.py │ │ │ │ ├── request_formatters.py │ │ │ │ ├── test_json_parser.py │ │ │ │ ├── test_parser_registry.py │ │ │ │ ├── test_r1_parser.py │ │ │ │ └── test_request_formatters.py │ │ │ ├── prompt_builders.py │ │ │ ├── provider_tools.py │ │ │ ├── repair │ │ │ │ ├── __init__.py │ │ │ │ ├── repair_task.py │ │ │ │ └── test_repair_task.py │ │ │ ├── run_output.py │ │ │ ├── test_adapter_registry.py │ │ │ ├── test_ollama_tools.py │ │ │ ├── test_prompt_adaptors.py │ │ │ ├── test_prompt_builders.py │ │ │ └── test_provider_tools.py │ │ ├── datamodel │ │ │ ├── __init__.py │ │ │ ├── basemodel.py │ │ │ ├── datamodel_enums.py │ │ │ ├── dataset_filters.py │ │ │ ├── dataset_split.py │ │ │ ├── eval.py │ │ │ ├── finetune.py │ │ │ ├── json_schema.py │ │ │ ├── model_cache.py │ │ │ ├── project.py │ │ │ ├── prompt.py │ │ │ ├── prompt_id.py │ │ │ ├── registry.py │ │ │ ├── strict_mode.py │ │ │ ├── task.py │ │ │ ├── task_output.py │ │ │ ├── task_run.py │ │ │ ├── test_basemodel.py │ │ │ ├── test_dataset_filters.py │ │ │ ├── test_dataset_split.py │ │ │ ├── test_datasource.py │ │ │ ├── test_eval_model.py │ │ │ ├── test_example_models.py │ │ │ ├── test_json_schema.py │ │ │ ├── test_model_cache.py │ │ │ ├── test_model_perf.py │ │ │ ├── test_models.py │ │ │ ├── test_nested_save.py │ │ │ ├── test_output_rating.py │ │ │ ├── test_prompt_id.py │ │ │ ├── test_registry.py │ │ │ └── test_task.py │ │ └── utils │ │ │ ├── __init__.py │ │ │ ├── async_job_runner.py │ │ │ ├── config.py │ │ │ ├── dataset_import.py │ │ │ ├── exhaustive_error.py │ │ │ ├── formatting.py │ │ │ ├── name_generator.py │ │ │ ├── test_async_job_runner.py │ │ │ ├── test_config.py │ │ │ ├── test_dataset_import.py │ │ │ └── test_name_geneator.py │ ├── pyproject.toml │ ├── setup.cfg │ └── uv.lock └── server │ ├── .python-version │ ├── LICENSE.txt │ ├── README.md │ ├── kiln_server │ ├── __init__.py │ ├── custom_errors.py │ ├── generate_openapi.py │ ├── project_api.py │ ├── prompt_api.py │ ├── run_api.py │ ├── server.py │ ├── task_api.py │ ├── test_custom_error.py │ ├── test_project_api.py │ ├── test_prompt_api.py │ ├── test_run_api.py │ └── test_task_api.py │ ├── pyproject.toml │ ├── setup.cfg │ └── uv.lock ├── pyproject.toml ├── pyrightconfig.json ├── pytest.ini ├── start_env.sh └── uv.lock /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = 3 | **/test_*.py 4 | libs/core/kiln_ai/adapters/ml_model_list.py 5 | conftest.py 6 | -------------------------------------------------------------------------------- /.cursorrules: -------------------------------------------------------------------------------- 1 | - call me "boss" 2 | - We always use typescript, not javascript 3 | - Always assume pydantic 2 (not pydantic 1) 4 | - The project supports Python 3.10 and above 5 | - When writing tests: 6 | 1) Always use pytest for tests in python code 7 | 2) assume an appriopirate test file already exists, find it, and suggest tests get appended to that file. If no such file exists, ask me before assuming a new test file is the correct route. 8 | 3) Test brevity is important. Use approaches for re-use and brevity including using fixtures for repeated code, and using pytest parameterize for similar tests 9 | 4) After writing a new test, run it and ensure it works. Fix any issues you find. 10 | - To run web tests `cd app/web_ui` then `npm run test_run` 11 | - Don't include comments in code explaining changes, explain changes in chat instead. 12 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: "[Bug] " 5 | labels: bug 6 | assignees: "" 7 | --- 8 | 9 | **Describe the bug** 10 | Insert a clear and concise description of what the bug is. 11 | 12 | **Checks** 13 | 14 | - [ ] I've read the [troubleshooting guide](https://docs.getkiln.ai/docs/troubleshooting-and-logs) 15 | - [ ] I've tried to reproduce the problem using another model, and confirmed it's not an issue specific to the model I've chosen. 16 | - [ ] I've searched [the docs](https://docs.getkiln.ai) for a solution 17 | - [ ] I've searched for existing Github issues/discussions 18 | 19 | **To Reproduce** 20 | Steps to reproduce the behavior: 21 | 22 | 1. Go to '...' 23 | 2. Click on '....' 24 | 3. Scroll down to '....' 25 | 4. See error 26 | 27 | **Expected behavior** 28 | A clear and concise description of what you expected to happen. 29 | 30 | **Screenshots** 31 | If applicable, add screenshots to help explain your problem. 32 | 33 | **Error Logs** 34 | Please include the logs if the issue shows an error. State that no error is shown if there is no error. 35 | 36 | https://app.gitbook.com/o/63UPvNXRQ4QyV9E5eioW/s/EJ4b8A4QiEQlOGbYkXDX/docs/troubleshooting-and-logs 37 | 38 | **System Information:** 39 | 40 | - OS: [e.g. Windows 11, MacOS] 41 | - Browser [e.g. chrome, safari] 42 | - Kiln app Version [e.g. v0.11.1] 43 | 44 | **Additional context** 45 | Add any other context about the problem here. 46 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: "[Feature Request]" 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Checks** 14 | 15 | - [ ] I've searched [the docs](https://docs.getkiln.ai) for a solution 16 | - [ ] I've searched for existing Github issues 17 | 18 | **Describe the solution you'd like** 19 | A clear and concise description of what you want to happen. 20 | 21 | **Describe alternatives you've considered** 22 | A clear and concise description of any alternative solutions or features you've considered. 23 | 24 | **Additional context** 25 | Add any other context or screenshots about the feature request here. 26 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## What does this PR do? 2 | 3 | 4 | 5 | ## Related Issues 6 | 7 | 8 | 9 | ## Contributor License Agreement 10 | 11 | I, @, confirm that I have read and agree to the [Contributors License Agreement](https://github.com/Kiln-AI/Kiln/blob/main/CLA.md). 12 | 13 | ## Checklists 14 | 15 | - [ ] Tests have been run locally and passed 16 | - [ ] New tests have been added to any work in /lib 17 | -------------------------------------------------------------------------------- /.github/workflows/build_and_test.yml: -------------------------------------------------------------------------------- 1 | name: Build and Test 2 | 3 | on: 4 | push: 5 | 6 | jobs: 7 | build: 8 | name: Build, Typecheck, and Test Python 9 | runs-on: ubuntu-latest 10 | strategy: 11 | matrix: 12 | python-version: 13 | - "3.10" 14 | - "3.11" 15 | - "3.12" 16 | - "3.13" 17 | 18 | steps: 19 | - uses: actions/checkout@v4 20 | 21 | - name: Install uv 22 | uses: astral-sh/setup-uv@v3 23 | with: 24 | enable-cache: true 25 | 26 | - name: Set up Python ${{ matrix.python-version }} 27 | run: uv python install ${{ matrix.python-version }} 28 | 29 | - name: Install the project 30 | run: uv sync --all-extras --dev 31 | 32 | # Remove this later: needed as server expects this dir to exist 33 | - name: Mock Studio Web UI 34 | run: mkdir -p ./app/web_ui/build && echo "test" > ./app/web_ui/build/index.html 35 | 36 | - name: Test All Python 37 | run: uv run python3 -m pytest . 38 | 39 | - name: Check Python Types 40 | run: uv run pyright . 41 | 42 | - name: Build Core 43 | run: uv build 44 | working-directory: ./libs/core 45 | 46 | - name: Build Server 47 | run: uv build 48 | working-directory: ./libs/server 49 | -------------------------------------------------------------------------------- /.github/workflows/build_desktop.yml: -------------------------------------------------------------------------------- 1 | name: Build Desktop Apps 2 | 3 | on: 4 | push: 5 | 6 | jobs: 7 | build: 8 | name: Build Desktop Apps 9 | runs-on: ${{ matrix.os }} 10 | strategy: 11 | fail-fast: false 12 | matrix: 13 | # macos-latest is for arm64 build, macos-13 is for x86_64 14 | include: 15 | - os: windows-latest 16 | - os: macos-latest # arm64 macOS 17 | - os: macos-13 # x86_64 macOS 18 | - os: ubuntu-22.04 # x86_64 linux, oldest available GH runner. Older libc for maximal compatibility 19 | - os: ubuntu-22.04-arm # arm linux build 20 | 21 | steps: 22 | - uses: actions/checkout@v4 23 | 24 | - name: Install uv 25 | uses: astral-sh/setup-uv@v3 26 | with: 27 | enable-cache: true 28 | 29 | - uses: actions/setup-python@v5 30 | with: 31 | python-version: 3.13 32 | 33 | # Use GH python version (includes TK/TCL) 34 | - name: Set up Python using GH python version 35 | run: uv venv --python 3.13 --python-preference only-system 36 | 37 | - name: Install the project 38 | run: uv sync 39 | 40 | # Compress MacOS app param ignored on other platforms 41 | - name: Build Desktop App 42 | run: uv run bash ./app/desktop/build_desktop_app.sh 43 | 44 | - name: Build Windows Installer 45 | if: matrix.os == 'windows-latest' 46 | uses: scosman/Inno-Setup-Action-Back@v1.2.5 47 | with: 48 | path: ./app/desktop/WinInnoSetup.iss 49 | 50 | - name: Copy Windows Installer 51 | if: matrix.os == 'windows-latest' 52 | run: cp ./app/desktop/Output/kilnsetup.exe ./app/desktop/build/dist/Kiln.Windows.Installer.exe 53 | 54 | # Print checksums, so anyone can verify the build 55 | 56 | - name: Checksums (macOS) 57 | if: runner.os == 'macOS' 58 | run: find ./app/desktop/build/dist/Kiln.app -type f -not -path "*/Contents/Resources/botocore/data/*" -exec md5 {} \; 59 | 60 | - name: Checksums (Linux) 61 | if: runner.os == 'Linux' 62 | run: find ./app/desktop/build/dist -type f -exec sha256sum {} \; 63 | 64 | - name: Checksums (Windows) 65 | if: runner.os == 'Windows' 66 | run: | 67 | certutil -hashfile ./app/desktop/build/dist/Kiln.Windows.Installer.exe SHA256 68 | certutil -hashfile ./app/desktop/build/dist/Kiln/Kiln.exe SHA256 69 | 70 | # MacOS apps have symlinks, and GitHub artifact upload zip will break them. Tar instead. 71 | - name: Compress MacOS App 72 | if: runner.os == 'macOS' 73 | working-directory: ./app/desktop/build/dist 74 | run: | 75 | echo "Compressing MacOS app" 76 | tar czpvf Kiln.app.tgz Kiln.app 77 | rm -r Kiln.app 78 | 79 | - name: Upload Build 80 | uses: actions/upload-artifact@v4 81 | with: 82 | name: kiln-desktop-${{ runner.os }}-${{ runner.arch }} 83 | path: ./app/desktop/build/dist/* 84 | -------------------------------------------------------------------------------- /.github/workflows/build_docs.yml: -------------------------------------------------------------------------------- 1 | name: Docs Build 2 | 3 | on: 4 | push: 5 | branches: 6 | - docs 7 | 8 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages 9 | permissions: 10 | contents: read 11 | pages: write 12 | id-token: write 13 | 14 | jobs: 15 | build: 16 | name: Build Docs 17 | runs-on: ubuntu-latest 18 | 19 | steps: 20 | - uses: actions/checkout@v4 21 | 22 | - name: Install uv 23 | uses: astral-sh/setup-uv@v3 24 | with: 25 | enable-cache: true 26 | 27 | - name: Set up Python 28 | run: uv python install 3.13 29 | 30 | - name: Install the project 31 | run: uv sync --all-extras --dev 32 | 33 | - name: Build Python Core Docs 34 | working-directory: ./libs/core 35 | run: uv run pdoc -o ./docs/kiln_core_docs kiln_ai --logo https://github.com/user-attachments/assets/046f44ae-28cf-4c78-85c3-c3e5ad744fd7 --logo-link https://getkiln.ai 36 | 37 | - name: Install Redoc CLI 38 | run: npm install -g @redocly/cli 39 | 40 | - name: Build OpenAPI Docs 41 | run: uv run python -m libs.server.kiln_server.generate_openapi && mkdir ./libs/core/docs/kiln_server_openapi_docs && npx @redocly/cli build-docs openapi.json -o ./libs/core/docs/kiln_server_openapi_docs/index.html 42 | 43 | - name: Setup Pages 44 | id: pages 45 | uses: actions/configure-pages@v5 46 | 47 | - name: Upload to Pages 48 | uses: actions/upload-pages-artifact@v3 49 | with: 50 | path: ./libs/core/docs 51 | 52 | # Deployment job 53 | deploy: 54 | environment: 55 | name: github-pages 56 | url: ${{ steps.deployment.outputs.page_url }} 57 | runs-on: ubuntu-latest 58 | needs: build 59 | steps: 60 | - name: Deploy to GitHub Pages 61 | id: deployment 62 | uses: actions/deploy-pages@v4 63 | -------------------------------------------------------------------------------- /.github/workflows/format_and_lint.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a single version of Python 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | # We use "ruff" for linting and formatting 5 | 6 | name: Format and Lint 7 | 8 | on: 9 | push: 10 | pull_request: 11 | branches: [main] 12 | 13 | jobs: 14 | format_and_lint: 15 | name: Format and Lint Python 16 | runs-on: ubuntu-latest 17 | 18 | steps: 19 | - uses: actions/checkout@v4 20 | - uses: actions/setup-go@v5 21 | with: 22 | go-version: "^1.22" 23 | 24 | - name: Download go misspell 25 | run: go install github.com/client9/misspell/cmd/misspell@latest 26 | - name: Run misspell 27 | run: misspell -error . 28 | 29 | - name: Install uv 30 | uses: astral-sh/setup-uv@v3 31 | 32 | - name: Set up Python 33 | run: uv python install 34 | 35 | - name: Install the project 36 | run: uv tool install ruff 37 | 38 | - name: Lint with ruff 39 | run: | 40 | uvx ruff check --select I . 41 | - name: Format with ruff 42 | run: | 43 | uvx ruff format --check . 44 | -------------------------------------------------------------------------------- /.github/workflows/test_count.yml: -------------------------------------------------------------------------------- 1 | name: Test Count And Coverage 2 | 3 | on: 4 | push: 5 | branches: 6 | - main # Adjust this if your main branch has a different name 7 | 8 | jobs: 9 | count_tests: 10 | name: Count Tests 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v4 15 | 16 | - name: Install uv 17 | uses: astral-sh/setup-uv@v3 18 | with: 19 | enable-cache: true 20 | 21 | - name: Set up Python 22 | run: uv python install 3.13 23 | 24 | - name: Install the project 25 | run: uv sync --all-extras --dev 26 | 27 | - name: Set up Node.js 28 | uses: actions/setup-node@v4 29 | 30 | - name: Install NPM dependencies 31 | run: npm install 32 | working-directory: ./app/web_ui 33 | 34 | - name: Count Python tests 35 | run: | 36 | PYTHON_TEST_COUNT=$(uv run python3 -m pytest --collect-only -q . | grep "tests collected" | awk '{print $1}' | sed 's/[^0-9]//g') 37 | echo "PYTHON_TEST_COUNT=$PYTHON_TEST_COUNT" >> $GITHUB_ENV 38 | 39 | - name: Count NPM tests 40 | run: | 41 | JS_TEST_COUNT=$(npx vitest run --no-color | grep 'Tests *[0-9]* passed ([0-9]*)' | awk '{print $2}' | sed 's/[^0-9]*//g') 42 | echo "JS_TEST_COUNT=$JS_TEST_COUNT" >> $GITHUB_ENV 43 | working-directory: ./app/web_ui 44 | 45 | - name: Calculate total test count 46 | run: | 47 | TOTAL_TEST_COUNT=$((PYTHON_TEST_COUNT + JS_TEST_COUNT)) 48 | echo "TOTAL_TEST_COUNT=$TOTAL_TEST_COUNT" >> $GITHUB_ENV 49 | 50 | - name: Create test count badge 51 | uses: schneegans/dynamic-badges-action@v1.7.0 52 | with: 53 | auth: ${{ secrets.GIST_SECRET }} 54 | gistID: 57742c1b1b60d597a6aba5d5148d728e 55 | filename: test_count_kiln.json 56 | label: Test Count 57 | message: ${{ env.TOTAL_TEST_COUNT }} 58 | color: brightgreen 59 | 60 | - name: Run library coverage 61 | # n0 disables parallel execution 62 | run: | 63 | uv run coverage run -m pytest -q ./libs/ -n0 64 | TOTAL_LIB_COVERAGE=$(uv run coverage report | grep TOTAL | awk {'print $4'}) 65 | echo "TOTAL_LIB_COVERAGE=$TOTAL_LIB_COVERAGE" >> $GITHUB_ENV 66 | 67 | - name: Create coverage badge 68 | uses: schneegans/dynamic-badges-action@v1.7.0 69 | with: 70 | auth: ${{ secrets.GIST_SECRET }} 71 | gistID: 57742c1b1b60d597a6aba5d5148d728e 72 | filename: library_coverage_kiln.json 73 | label: Coverage 74 | message: ${{ env.TOTAL_LIB_COVERAGE }} 75 | color: brightgreen 76 | -------------------------------------------------------------------------------- /.github/workflows/web_format_lint_build.yml: -------------------------------------------------------------------------------- 1 | name: Web UI Checks 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | webui_checks: 7 | name: Web UI Code Format, Lint, Typecheck, Test, and Build 8 | runs-on: ubuntu-latest 9 | defaults: 10 | run: 11 | working-directory: ./app/web_ui 12 | 13 | steps: 14 | - uses: actions/checkout@v4 15 | 16 | - name: NPM install 17 | run: npm install . 18 | 19 | - name: Check Code Format 20 | run: npm run format_check 21 | 22 | - name: Linting 23 | run: npm run lint 24 | 25 | - name: Typechecking 26 | run: npm run check 27 | 28 | - name: Tests 29 | run: npm run test_run 30 | 31 | - name: SvelteKit Build 32 | run: npm run build 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | 3 | .env 4 | .act_secrets 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | .coverage 9 | **/.venv 10 | **/*.egg-info 11 | 12 | libs/core/docs 13 | libs/core/build 14 | libs/server/build 15 | 16 | dist/ 17 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Python Debugger: Current File", 9 | "type": "debugpy", 10 | "request": "launch", 11 | "program": "${file}", 12 | "console": "integratedTerminal" 13 | } 14 | ] 15 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.formatOnSave": true, 3 | "[python]": { 4 | "editor.formatOnSave": true, 5 | "editor.defaultFormatter": "charliermarsh.ruff", 6 | "editor.codeActionsOnSave": { 7 | "source.organizeImports": "explicit" 8 | }, 9 | "editor.tabSize": 4, 10 | }, 11 | "python.testing.pytestArgs": [ 12 | ".", 13 | "-s", 14 | "--runsinglewithoutchecks" 15 | ], 16 | "python.testing.unittestEnabled": false, 17 | "python.testing.pytestEnabled": true, 18 | "python.analysis.typeCheckingMode": "basic", 19 | "python.analysis.autoImportCompletions": true, 20 | "python.analysis.extraPaths": ["./"], 21 | "python.analysis.ignore": ["**/test_*.py"], 22 | "files.exclude": { 23 | "**/__pycache__": true, 24 | "**/.pytest_cache": true, 25 | "**/.ruff_cache": true, 26 | "**/dist": true, 27 | "**/node_modules": true 28 | }, 29 | // Svelte, JS, TS files 30 | "[svelte]": { 31 | "editor.defaultFormatter": "svelte.svelte-vscode" 32 | }, 33 | "[typescript]": { 34 | "editor.defaultFormatter": "esbenp.prettier-vscode" 35 | }, 36 | "[javascript]": { 37 | "editor.defaultFormatter": "esbenp.prettier-vscode" 38 | }, 39 | "eslint.validate": ["javascript", "javascriptreact", "typescript", "svelte"], 40 | } 41 | -------------------------------------------------------------------------------- /CLA.md: -------------------------------------------------------------------------------- 1 | # Contributor License Agreement 2 | 3 | The following terms are used throughout this agreement: 4 | 5 | You - the person or legal entity including its affiliates asked to accept this agreement. An affiliate is any entity that controls or is controlled by the legal entity, or is under common control with it. 6 | 7 | Project - is an umbrella term that refers to any and all Chesterfield Laboratories Inc. open source projects. 8 | 9 | Contribution - any type of work that is submitted to a Project, including any modifications or additions to existing work. 10 | 11 | Submitted - conveyed to a Project via a pull request, commit, issue, or any form of electronic, written, or verbal communication with Chesterfield Laboratories Inc., contributors or maintainers. 12 | 13 | ## 1. Grant of Copyright License. 14 | 15 | Subject to the terms and conditions of this agreement, You grant to the Projects’ maintainers, contributors, users and to Chesterfield Laboratories Inc. a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense, and distribute Your contributions and such derivative works. Except for this license, You reserve all rights, title, and interest in your contributions. 16 | 17 | ## 2. Grant of Patent License. 18 | 19 | Subject to the terms and conditions of this agreement, You grant to the Projects’ maintainers, contributors, users and to Chesterfield Laboratories Inc. a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer your contributions, where such license applies only to those patent claims licensable by you that are necessarily infringed by your contribution or by combination of your contribution with the project to which this contribution was submitted. 20 | 21 | If any entity institutes patent litigation - including cross-claim or counterclaim in a lawsuit - against You alleging that your contribution or any project it was submitted to constitutes or is responsible for direct or contributory patent infringement, then any patent licenses granted to that entity under this agreement shall terminate as of the date such litigation is filed. 22 | 23 | ## 3. Source of Contribution. 24 | 25 | Your contribution is either your original creation, based upon previous work that, to the best of your knowledge, is covered under an appropriate open source license and you have the right under that license to submit that work with modifications, whether created in whole or in part by you, or you have clearly identified the source of the contribution and any license or other restriction (like related patents, trademarks, and license agreements) of which you are personally aware. 26 | 27 | ## References 28 | 29 | Adapted from the Github CLA, under the CC-BY-SA 4.0 license. 30 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Kiln 2 | 3 | ## Issues and Bug Tracking 4 | 5 | We use [GitHub issues](https://github.com/Kiln-AI/Kiln/issues) for tracking issues, bugs, and feature requests. 6 | 7 | ## Contributing 8 | 9 | New contributors must agree to the [contributor license agreement](CLA.md). 10 | 11 | ## Development Environment Setup 12 | 13 | We use [uv](https://github.com/astral-sh/uv) to manage the Python environment and dependencies, and npm to manage the web UI. 14 | 15 | ``` 16 | # First install uv: https://github.com/astral-sh/uv 17 | uv sync 18 | cd app/web_ui 19 | # install Node if you don't have it already 20 | npm install 21 | ``` 22 | 23 | ### Running Development Servers 24 | 25 | Running the web-UI and Python servers separately is useful for development, as both can hot-reload. 26 | 27 | To run the API server, Studio server, and Studio Web UI with auto-reload for development: 28 | 29 | 1. In your first terminal, navigate to the base Kiln directory: 30 | 31 | ```bash 32 | uv run python -m app.desktop.dev_server 33 | ``` 34 | 35 | 2. In a second terminal, navigate to the web UI directory and start the dev server: 36 | 37 | ```bash 38 | cd app/web_ui 39 | npm run dev -- 40 | ``` 41 | 42 | 3. Open the app: http://localhost:5173/run 43 | 44 | ### Running and Building the Desktop App 45 | 46 | See the [desktop README](app/desktop/README.md) instructions for running the desktop app locally. 47 | 48 | ## Tests, Formatting, and Linting 49 | 50 | We have a large test suite, and use [ruff](https://github.com/astral-sh/ruff) for linting and formatting. 51 | 52 | Please ensure any new code has test coverage, and that all code is formatted and linted. CI will block merging if tests fail or your code is not formatted and linted correctly. 53 | 54 | To confirm everything works locally, run: 55 | 56 | ```bash 57 | ./checks.sh 58 | ``` 59 | 60 | ## Optional Setup 61 | 62 | ### IDE Extensions 63 | 64 | We suggest the following extensions for VSCode/Cursor. With them, you'll get compliant formatting and linting in your IDE. 65 | 66 | - Prettier 67 | - Pylance 68 | - Python 69 | - Python Debugger 70 | - Ruff 71 | - Svelte for VS Code 72 | - Vitest 73 | - ESLint 74 | 75 | ### llms.txt 76 | 77 | Vibing? Here are some [llms.txt](https://llmstxt.org) you may want to add. 78 | 79 | Usage: `@docs Svelte` in cursor lets the LLM read the docs of the specified library. Most popular libraries added by Cursor automatically, but here are some to add manually: 80 | 81 | - daisyUI’s: https://daisyui.com/docs/editor/cursor/ 82 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Kiln AI - License Information 2 | Copyright (c) 2024 Chesterfield Laboratories Inc. 3 | 4 | This repository contains components under different licenses: 5 | 6 | 1. Python Libraries (/libs/core and /libs/server directories) 7 | Licensed under the MIT License 8 | See libs/core/LICENSE.txt and libs/server/LICENSE.txt for details 9 | 10 | 2. Desktop Application (/app directory) 11 | Free to download and use under the Kiln AI Desktop EULA 12 | See app/EULA.md and app/LICENSE.txt for details 13 | 14 | 3. Trademarks 15 | The Kiln names and logos are trademarks of Chesterfield Laboratories Inc. 16 | All rights reserved. 17 | 18 | For the full text of each license, please see the respective LICENSE.txt files in each directory. 19 | -------------------------------------------------------------------------------- /app/LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | This license applies to the software in the app/ directory and the desktop apps. 3 | 4 | ======================================================= 5 | 6 | Copyright 2024 - Chesterfield Laboratories Inc. 7 | 8 | All rights reserved. 9 | 10 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 11 | 12 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /app/__init__.py: -------------------------------------------------------------------------------- 1 | # Blank needed for pyinstaller to find module 2 | -------------------------------------------------------------------------------- /app/desktop/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | build/ 3 | 4 | -------------------------------------------------------------------------------- /app/desktop/.python-version: -------------------------------------------------------------------------------- 1 | 3.13 2 | -------------------------------------------------------------------------------- /app/desktop/README.md: -------------------------------------------------------------------------------- 1 | # Desktop Apps 2 | 3 | ## MacOS Environment Setup 4 | 5 | UV python doesn't include TK/TCL [yet](https://github.com/astral-sh/uv/issues/7036). Instead, we install system python including TK/TCL, and tell UV venv to use system python. 6 | 7 | ``` 8 | # Install python 3.13 and python-tk 3.13 with homebrew 9 | brew install python-tk@3.13 10 | brew install python@3.13 11 | 12 | # check uv can see it hoembrew version 13 | uv python list --python-preference only-system 14 | 15 | # setup 3.13 uv-managed venv with system (homebrew) python 3.13 16 | uv venv --python 3.13 --python-preference only-system 17 | 18 | # Check it worked 19 | uv run python --version 20 | 21 | # Run desktop 22 | uv run python -m app.desktop.desktop 23 | ``` 24 | 25 | ## Building the Desktop App 26 | 27 | Typically building desktop apps are done in a CI/CD pipeline, but if you need to build the desktop app locally, you can do so with: 28 | 29 | ```bash 30 | cd app/desktop 31 | uv run ./build_desktop_app.sh 32 | ``` 33 | 34 | ## MacOS Code Signing 35 | 36 | Easy way, but just signs with personal ID for local development: `codesign --force --deep -s - kiln.app` 37 | 38 | Sign with a developer ID (should only be done for official releases by Kiln team): 39 | 40 | 1. Get developer ID name: `security find-identity -v -p codesigning` 41 | 2. Run `codesign --force --deep -s "Developer ID Application: YOUR NAME (XXXXXXXX)" kiln.app` 42 | -------------------------------------------------------------------------------- /app/desktop/WinInnoSetup.iss: -------------------------------------------------------------------------------- 1 | ; Script generated by the Inno Setup Script Wizard. 2 | ; SEE THE DOCUMENTATION FOR DETAILS ON CREATING INNO SETUP SCRIPT FILES! 3 | 4 | #define MyAppPath "build\dist\Kiln" 5 | #define MyAppName "Kiln" 6 | #define MyAppVersion "0.16.0" 7 | #define MyAppPublisher "Chesterfield Laboratories Inc" 8 | #define MyAppURL "https://getkiln.ai" 9 | #define MyAppExeName "Kiln.exe" 10 | 11 | [Setup] 12 | ; NOTE: The value of AppId uniquely identifies this application. Do not use the same AppId value in installers for other applications. 13 | ; (To generate a new GUID, click Tools | Generate GUID inside the IDE.) 14 | AppId={{0D014F6B-4438-45BC-A8D5-DD6AF90DFF1F} 15 | AppName={#MyAppName} 16 | AppVersion={#MyAppVersion} 17 | ;AppVerName={#MyAppName} {#MyAppVersion} 18 | AppPublisher={#MyAppPublisher} 19 | AppPublisherURL={#MyAppURL} 20 | AppSupportURL={#MyAppURL} 21 | AppUpdatesURL={#MyAppURL} 22 | DefaultDirName={autopf}\{#MyAppName} 23 | DisableDirPage=yes 24 | ; "ArchitecturesAllowed=x64compatible" specifies that Setup cannot run 25 | ; on anything but x64 and Windows 11 on Arm. 26 | ArchitecturesAllowed=x64compatible 27 | ; "ArchitecturesInstallIn64BitMode=x64compatible" requests that the 28 | ; install be done in "64-bit mode" on x64 or Windows 11 on Arm, 29 | ; meaning it should use the native 64-bit Program Files directory and 30 | ; the 64-bit view of the registry. 31 | ArchitecturesInstallIn64BitMode=x64compatible 32 | DisableProgramGroupPage=yes 33 | ; Remove the following line to run in administrative install mode (install for all users.) 34 | PrivilegesRequired=lowest 35 | PrivilegesRequiredOverridesAllowed=dialog 36 | OutputBaseFilename=kilnsetup 37 | Compression=lzma 38 | SolidCompression=yes 39 | WizardStyle=modern 40 | 41 | [Languages] 42 | Name: "english"; MessagesFile: "compiler:Default.isl" 43 | 44 | [Tasks] 45 | Name: "desktopicon"; Description: "{cm:CreateDesktopIcon}"; GroupDescription: "{cm:AdditionalIcons}"; Flags: unchecked 46 | 47 | [Files] 48 | Source: "{#MyAppPath}\{#MyAppExeName}"; DestDir: "{app}"; Flags: ignoreversion 49 | Source: "{#MyAppPath}\_internal\*"; DestDir: "{app}\_internal"; Flags: ignoreversion recursesubdirs createallsubdirs 50 | ; NOTE: Don't use "Flags: ignoreversion" on any shared system files 51 | 52 | [Icons] 53 | Name: "{autoprograms}\{#MyAppName}"; Filename: "{app}\{#MyAppExeName}" 54 | Name: "{autodesktop}\{#MyAppName}"; Filename: "{app}\{#MyAppExeName}"; Tasks: desktopicon 55 | 56 | [Run] 57 | Filename: "{app}\{#MyAppExeName}"; Description: "{cm:LaunchProgram,{#StringChange(MyAppName, '&', '&&')}}"; Flags: nowait postinstall skipifsilent 58 | 59 | -------------------------------------------------------------------------------- /app/desktop/__init__.py: -------------------------------------------------------------------------------- 1 | # Blank needed for pyinstaller to find module 2 | -------------------------------------------------------------------------------- /app/desktop/build_desktop_app.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | # move to the /app directory of the project 6 | cd "$(dirname "$0")" 7 | cd .. 8 | APP_DIR=$PWD 9 | 10 | if [[ $* != *--skip-web* ]]; then 11 | # build the web ui 12 | echo "Building web UI" 13 | cd web_ui 14 | npm install 15 | npm run build 16 | cd $APP_DIR 17 | fi 18 | 19 | # Building the bootloader ourselves helps not be falsely detected as malware by antivirus software on windows. 20 | # We clone pyinstaller, build the bootloader, and install it into the pyproject desktop pyproject. 21 | if [[ $* == *--build-bootloader* ]]; then 22 | echo "Building pyinstaller inlucding bootloader" 23 | 24 | mkdir -p desktop/build/bootloader 25 | cd desktop/build/bootloader 26 | curl -L https://github.com/pyinstaller/pyinstaller/archive/refs/tags/v6.11.1.tar.gz -o pyinstaller.tar.gz 27 | tar -xzf pyinstaller.tar.gz 28 | mv pyinstaller-6.11.1 pyinstaller 29 | cd pyinstaller/bootloader 30 | python ./waf all 31 | 32 | # install the pyinstaller we just built into the desktop pyproject 33 | cd $APP_DIR/desktop 34 | uv add build/bootloader/pyinstaller 35 | 36 | # return to the /app directory of the project 37 | cd $APP_DIR 38 | fi 39 | 40 | mkdir -p desktop/build 41 | 42 | echo "Building for $(uname)" 43 | if [ "$(uname)" == "Darwin" ]; then 44 | echo "Building MacOS app" 45 | cp desktop/mac_taskbar.png desktop/build/taskbar.png 46 | # onedir launches faster, and still looks like 1 file with MacOS .app bundles 47 | PLATFORM_OPTS="--onedir --windowed --icon=../mac_icon.png --osx-bundle-identifier=com.kiln-ai.kiln.studio" 48 | 49 | PY_PLAT=$(python -c 'import platform; print(platform.machine())') 50 | echo "Building MacOS app for single platform ($PY_PLAT)" 51 | elif [[ "$(uname)" =~ ^MINGW64_NT-10.0 ]] || [[ "$(uname)" =~ ^MSYS_NT-10.0 ]]; then 52 | echo "Building Windows App" 53 | cp desktop/win_taskbar.png desktop/build/taskbar.png 54 | PLATFORM_OPTS="--windowed --splash=../win_splash.png --icon=../win_icon.ico" 55 | elif [ "$(uname)" == "Linux" ]; then 56 | echo "Building Linux App" 57 | cp desktop/mac_taskbar.png desktop/build/taskbar.png 58 | PLATFORM_OPTS="--windowed --onefile --splash=../win_splash.png --icon=../mac_icon.png" 59 | else 60 | echo "Unsupported operating system: $(uname)" 61 | exit 1 62 | fi 63 | 64 | # Builds the desktop app 65 | # TODO: use a spec instead of long winded command line 66 | pyinstaller $(printf %s "$PLATFORM_OPTS") \ 67 | --add-data "./taskbar.png:." --add-data "../../web_ui/build:./web_ui/build" \ 68 | --noconfirm --distpath=./desktop/build/dist --workpath=./desktop/build/work \ 69 | -n Kiln --specpath=./desktop/build --hidden-import=tiktoken_ext.openai_public --hidden-import=tiktoken_ext \ 70 | --hidden-import=litellm \ 71 | --collect-all=litellm \ 72 | --paths=. ./desktop/desktop.py 73 | -------------------------------------------------------------------------------- /app/desktop/custom_tray.py: -------------------------------------------------------------------------------- 1 | import io 2 | import logging 3 | import sys 4 | 5 | import pystray 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | 10 | class KilnTray(pystray.Icon): 11 | # Special handling for Mac to support dark/light mode and retina icons 12 | # lots of type ignores because we're accessing private attributes of pystray 13 | def _assert_image(self): 14 | if sys.platform != "darwin": 15 | super()._assert_image() # type: ignore 16 | return 17 | 18 | import AppKit 19 | import Foundation 20 | 21 | thickness = self._status_bar.thickness() # type: ignore 22 | logical_size = (int(thickness), int(thickness)) 23 | if self._icon_image and self._icon_image.size() == logical_size: 24 | return 25 | 26 | source = self._icon 27 | 28 | # Convert the PIL image to an NSImage 29 | b = io.BytesIO() 30 | source.save(b, "png") # type: ignore 31 | data = Foundation.NSData(b.getvalue()) # type: ignore 32 | 33 | self._icon_image = AppKit.NSImage.alloc().initWithData_(data) # type: ignore 34 | try: 35 | # template image will respect dark/light mode 36 | self._icon_image.setTemplate_(True) 37 | # set the logical size of the image, which will be scaled for retina 38 | self._icon_image.setSize_(logical_size) 39 | except Exception: 40 | # Continue, this shouldn't be fatal 41 | logger.error("Mac Tray Error", exc_info=True) 42 | self._status_item.button().setImage_(self._icon_image) # type: ignore 43 | -------------------------------------------------------------------------------- /app/desktop/dev_server.py: -------------------------------------------------------------------------------- 1 | # Run a desktop server for development: 2 | # - Auto-reload is enabled 3 | # - Extra logging (level+colors) is enabled 4 | import os 5 | 6 | import uvicorn 7 | 8 | from app.desktop.desktop_server import make_app 9 | 10 | # top level app object, as that's needed by auto-reload 11 | dev_app = make_app() 12 | 13 | os.environ["DEBUG_EVENT_LOOP"] = "true" 14 | 15 | 16 | if __name__ == "__main__": 17 | uvicorn.run( 18 | "app.desktop.dev_server:dev_app", 19 | host="127.0.0.1", 20 | port=8757, 21 | reload=True, 22 | ) 23 | -------------------------------------------------------------------------------- /app/desktop/dmg/README.md: -------------------------------------------------------------------------------- 1 | # Make a MacOS DMG 2 | 3 | Weird node 20 workaround for this: https://github.com/sindresorhus/create-dmg 4 | 5 | Each time it should just be: 6 | 7 | ``` 8 | create-dmg Kiln.app 9 | # then rename the output dmg to Kiln.MacOS.Intel.dmg or Kiln.MacOS.AppleSilicon.M-Processor.dmg 10 | ``` 11 | 12 | Env Setup while node 20 is required: 13 | 14 | - `nvm use 20` 15 | - `npm install -g create-dmg` 16 | - `create-dmg Kiln.app` 17 | - rename the output dmg to Kiln.MacOS.Intel.dmg or Kiln.MacOS.AppleSilicon.M-Processor.dmg 18 | - `nvm use 23` 19 | -------------------------------------------------------------------------------- /app/desktop/dmg/appdmg.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Kiln Installer", 3 | "contents": [ 4 | { "x": 310, "y": 120, "type": "link", "path": "/Applications" }, 5 | { "x": 120, "y": 120, "type": "file", "path": "Kiln.app" } 6 | ], 7 | "window": { 8 | "size": { 9 | "width": 426, 10 | "height": 272 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /app/desktop/mac_entitlements.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | com.apple.security.cs.allow-jit 7 | 8 | com.apple.security.cs.allow-unsigned-executable-memory 9 | 10 | com.apple.security.cs.disable-library-validation 11 | 12 | 13 | -------------------------------------------------------------------------------- /app/desktop/mac_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kiln-AI/Kiln/06b5dbe8560ee8cdfee1bbcc3ddfb0f5840a4508/app/desktop/mac_icon.png -------------------------------------------------------------------------------- /app/desktop/mac_taskbar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kiln-AI/Kiln/06b5dbe8560ee8cdfee1bbcc3ddfb0f5840a4508/app/desktop/mac_taskbar.png -------------------------------------------------------------------------------- /app/desktop/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "kiln-studio-desktop" 3 | version = "0.1.0" 4 | description = "The Kiln Desktop App. Download from https://getkiln.ai" 5 | requires-python = ">=3.10" 6 | dependencies = [ 7 | "kiln-server", 8 | "pillow>=11.0.0", 9 | "pystray>=0.19.5", 10 | "pyinstaller==6.11.1", 11 | "scipy>=1.15.2", 12 | ] 13 | 14 | 15 | [tool.uv.sources] 16 | kiln-server = { workspace = true } 17 | -------------------------------------------------------------------------------- /app/desktop/run_desktop_dev.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Needs to run from the root of the project 4 | cd "$(dirname "$0")" 5 | cd .. 6 | 7 | python -m desktop.desktop -------------------------------------------------------------------------------- /app/desktop/studio_server/prompt_api.py: -------------------------------------------------------------------------------- 1 | from fastapi import FastAPI, HTTPException 2 | from kiln_ai.adapters.prompt_builders import prompt_builder_from_id 3 | from kiln_ai.datamodel import PromptId 4 | from kiln_server.task_api import task_from_id 5 | from pydantic import BaseModel 6 | 7 | 8 | class PromptApiResponse(BaseModel): 9 | prompt: str 10 | prompt_id: PromptId 11 | 12 | 13 | def connect_prompt_api(app: FastAPI): 14 | @app.get("/api/projects/{project_id}/task/{task_id}/gen_prompt/{prompt_id}") 15 | async def generate_prompt( 16 | project_id: str, 17 | task_id: str, 18 | prompt_id: PromptId, 19 | ) -> PromptApiResponse: 20 | task = task_from_id(project_id, task_id) 21 | 22 | try: 23 | prompt_builder = prompt_builder_from_id(prompt_id, task) 24 | prompt = prompt_builder.build_prompt_for_ui() 25 | except Exception as e: 26 | raise HTTPException(status_code=400, detail=str(e)) 27 | 28 | return PromptApiResponse( 29 | prompt=prompt, 30 | prompt_id=prompt_id, 31 | ) 32 | -------------------------------------------------------------------------------- /app/desktop/studio_server/settings_api.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | from fastapi import FastAPI 4 | from kiln_ai.utils.config import Config 5 | 6 | 7 | def connect_settings(app: FastAPI): 8 | @app.post("/api/settings") 9 | def update_settings( 10 | new_settings: dict[str, int | float | str | bool | list | None], 11 | ): 12 | Config.shared().update_settings(new_settings) 13 | return Config.shared().settings(hide_sensitive=True) 14 | 15 | @app.get("/api/settings") 16 | def read_settings() -> dict[str, Any]: 17 | settings = Config.shared().settings(hide_sensitive=True) 18 | return settings 19 | 20 | @app.get("/api/settings/{item_id}") 21 | def read_setting_item(item_id: str): 22 | settings = Config.shared().settings(hide_sensitive=True) 23 | return {item_id: settings.get(item_id, None)} 24 | -------------------------------------------------------------------------------- /app/desktop/studio_server/test_prompt_api.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import MagicMock, patch 2 | 3 | import pytest 4 | 5 | # Create a FastAPI app and connect the prompt_api 6 | from fastapi import FastAPI 7 | from fastapi.testclient import TestClient 8 | from kiln_ai.adapters.prompt_builders import BasePromptBuilder 9 | from kiln_ai.datamodel import Task 10 | 11 | from app.desktop.studio_server.prompt_api import connect_prompt_api 12 | 13 | 14 | @pytest.fixture 15 | def client(): 16 | app = FastAPI() 17 | connect_prompt_api(app) 18 | return TestClient(app) 19 | 20 | 21 | # Mock prompt builder class 22 | class MockPromptBuilder(BasePromptBuilder): 23 | def build_base_prompt(self): 24 | return "Mock prompt" 25 | 26 | def build_prompt_for_ui(self): 27 | return "Mock prompt for UI" 28 | 29 | 30 | @pytest.fixture 31 | def mock_task(): 32 | return MagicMock(spec=Task) 33 | 34 | 35 | @pytest.fixture 36 | def mock_prompt_builder_from_id(mock_task): 37 | with patch("app.desktop.studio_server.prompt_api.prompt_builder_from_id") as mock: 38 | mock.return_value = MockPromptBuilder(mock_task) 39 | yield mock 40 | 41 | 42 | @pytest.fixture 43 | def mock_task_from_id(mock_task): 44 | with patch("app.desktop.studio_server.prompt_api.task_from_id") as mock: 45 | mock.return_value = mock_task 46 | yield mock 47 | 48 | 49 | def test_generate_prompt_success( 50 | client, mock_task, mock_prompt_builder_from_id, mock_task_from_id 51 | ): 52 | response = client.get( 53 | "/api/projects/project123/task/task456/gen_prompt/simple_prompt_builder" 54 | ) 55 | 56 | assert response.status_code == 200 57 | data = response.json() 58 | assert data == { 59 | "prompt": "Mock prompt for UI", 60 | "prompt_id": "simple_prompt_builder", 61 | } 62 | 63 | mock_task_from_id.assert_called_once_with("project123", "task456") 64 | mock_prompt_builder_from_id.assert_called_once_with( 65 | "simple_prompt_builder", mock_task 66 | ) 67 | 68 | 69 | def test_generate_prompt_exception( 70 | client, mock_task, mock_prompt_builder_from_id, mock_task_from_id 71 | ): 72 | mock_prompt_builder_from_id.side_effect = ValueError("Invalid prompt generator") 73 | 74 | response = client.get( 75 | "/api/projects/project123/task/task456/gen_prompt/simple_prompt_builder" 76 | ) 77 | 78 | assert response.status_code == 400 79 | assert "Invalid prompt generator" in response.text 80 | 81 | 82 | def test_generate_prompt_id_format(client, mock_task, mock_task_from_id): 83 | response = client.get( 84 | "/api/projects/project123/task/task456/gen_prompt/invalid_generator_id" 85 | ) 86 | 87 | assert response.status_code == 422 88 | assert "Value error, Invalid prompt ID: invalid_generator_id" in response.text 89 | -------------------------------------------------------------------------------- /app/desktop/studio_server/webhost.py: -------------------------------------------------------------------------------- 1 | import mimetypes 2 | import os 3 | import sys 4 | 5 | from fastapi import FastAPI, Response 6 | from fastapi.responses import FileResponse 7 | from fastapi.staticfiles import StaticFiles 8 | 9 | # Explicitly add MIME types for most common file types. Several users have reported issues on windows 11, where these should be loaded from the registry, but aren't working. 10 | mimetypes.add_type("text/css", ".css") 11 | mimetypes.add_type("text/javascript", ".js") 12 | mimetypes.add_type("text/html", ".html") 13 | mimetypes.add_type("image/svg+xml", ".svg") 14 | mimetypes.add_type("image/png", ".png") 15 | mimetypes.add_type("image/jpeg", ".jpg") 16 | 17 | 18 | def studio_path(): 19 | try: 20 | # pyinstaller path 21 | base_path = sys._MEIPASS # type: ignore 22 | return os.path.join(base_path, "./web_ui/build") 23 | except Exception: 24 | base_path = os.path.join(os.path.dirname(__file__), "..") 25 | return os.path.join(base_path, "../../app/web_ui/build") 26 | 27 | 28 | def add_no_cache_headers(response: Response): 29 | # This is already local, disable browser caching to prevent issues of old web-app trying to load old APIs and out of date web-ui 30 | response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate, max-age=0" 31 | response.headers["Pragma"] = "no-cache" 32 | response.headers["Expires"] = "0" 33 | 34 | 35 | # File server that maps /foo/bar to /foo/bar.html (Starlette StaticFiles only does index.html) 36 | class HTMLStaticFiles(StaticFiles): 37 | async def get_response(self, path: str, scope): 38 | try: 39 | response = await super().get_response(path, scope) 40 | if response.status_code != 404: 41 | add_no_cache_headers(response) 42 | return response 43 | except Exception as e: 44 | # catching HTTPException explicitly not working for some reason 45 | if getattr(e, "status_code", None) != 404: 46 | # Don't raise on 404, fall through to return the .html version 47 | raise e 48 | # Try the .html version of the file if the .html version exists, for 404s 49 | response = await super().get_response(f"{path}.html", scope) 50 | add_no_cache_headers(response) 51 | return response 52 | 53 | 54 | def connect_webhost(app: FastAPI): 55 | # Ensure studio_path exists (test servers don't necessarily create it) 56 | os.makedirs(studio_path(), exist_ok=True) 57 | # Serves the web UI at root 58 | app.mount("/", HTMLStaticFiles(directory=studio_path(), html=True), name="studio") 59 | 60 | # add pretty 404s 61 | @app.exception_handler(404) 62 | def not_found_exception_handler(request, exc): 63 | # don't handle /api routes, which return JSON errors 64 | if request.url.path.startswith("/api"): 65 | raise exc 66 | return FileResponse(os.path.join(studio_path(), "404.html"), status_code=404) 67 | -------------------------------------------------------------------------------- /app/desktop/taskbar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kiln-AI/Kiln/06b5dbe8560ee8cdfee1bbcc3ddfb0f5840a4508/app/desktop/taskbar.png -------------------------------------------------------------------------------- /app/desktop/test_desktop.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | import requests 4 | 5 | import app.desktop.desktop_server as desktop_server 6 | 7 | 8 | def test_desktop_app_server(): 9 | # random port between 9000 and 12000 10 | port = random.randint(9000, 12000) 11 | config = desktop_server.server_config(port=port) 12 | uni_server = desktop_server.ThreadedServer(config=config) 13 | with uni_server.run_in_thread(): 14 | r = requests.get("http://127.0.0.1:{}/ping".format(port)) 15 | assert r.status_code == 200 16 | -------------------------------------------------------------------------------- /app/desktop/win_icon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kiln-AI/Kiln/06b5dbe8560ee8cdfee1bbcc3ddfb0f5840a4508/app/desktop/win_icon.ico -------------------------------------------------------------------------------- /app/desktop/win_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kiln-AI/Kiln/06b5dbe8560ee8cdfee1bbcc3ddfb0f5840a4508/app/desktop/win_icon.png -------------------------------------------------------------------------------- /app/desktop/win_splash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kiln-AI/Kiln/06b5dbe8560ee8cdfee1bbcc3ddfb0f5840a4508/app/desktop/win_splash.png -------------------------------------------------------------------------------- /app/desktop/win_taskbar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kiln-AI/Kiln/06b5dbe8560ee8cdfee1bbcc3ddfb0f5840a4508/app/desktop/win_taskbar.png -------------------------------------------------------------------------------- /app/web_ui/.eslintignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules 3 | /build 4 | /.svelte-kit 5 | /package 6 | .env 7 | .env.* 8 | !.env.example 9 | 10 | # Ignore files for PNPM, NPM and YARN 11 | pnpm-lock.yaml 12 | package-lock.json 13 | yarn.lock 14 | -------------------------------------------------------------------------------- /app/web_ui/.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | extends: [ 4 | "eslint:recommended", 5 | "plugin:@typescript-eslint/recommended", 6 | "plugin:svelte/recommended", 7 | "prettier", 8 | ], 9 | parser: "@typescript-eslint/parser", 10 | plugins: ["@typescript-eslint"], 11 | parserOptions: { 12 | sourceType: "module", 13 | ecmaVersion: 2020, 14 | extraFileExtensions: [".svelte"], 15 | }, 16 | overrides: [ 17 | { 18 | files: ["*.svelte"], 19 | parser: "svelte-eslint-parser", 20 | parserOptions: { 21 | parser: { 22 | // Specify a parser for each lang. 23 | ts: "@typescript-eslint/parser", 24 | js: "espree", 25 | typescript: "@typescript-eslint/parser", 26 | }, 27 | }, 28 | }, 29 | ], 30 | env: { 31 | browser: true, 32 | es2017: true, 33 | node: true, 34 | }, 35 | rules: { 36 | // no-undef has been turned off because of this: 37 | // basically, it causes issues and TS does those checks so it's redundant 38 | // https://typescript-eslint.io/linting/troubleshooting#i-get-errors-from-the-no-undef-rule-about-global-variables-not-being-defined-even-though-there-are-no-typescript-errors 39 | "no-undef": "off", 40 | 41 | // Default eslint no-unused-vars modified to ignore underscore vars which we use for known unused vars 42 | "no-unused-vars": "off", 43 | "@typescript-eslint/no-unused-vars": [ 44 | "error", 45 | { 46 | argsIgnorePattern: "^_$", 47 | varsIgnorePattern: "^_$", 48 | caughtErrorsIgnorePattern: "^_$", 49 | }, 50 | ], 51 | }, 52 | } 53 | -------------------------------------------------------------------------------- /app/web_ui/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules 3 | /build 4 | /.svelte-kit 5 | /package 6 | .env 7 | .env.* 8 | !.env.example 9 | vite.config.js.timestamp-* 10 | vite.config.ts.timestamp-* 11 | -------------------------------------------------------------------------------- /app/web_ui/.npmrc: -------------------------------------------------------------------------------- 1 | engine-strict=true 2 | resolution-mode=highest 3 | -------------------------------------------------------------------------------- /app/web_ui/.prettierignore: -------------------------------------------------------------------------------- 1 | **/.svelte-kit/ 2 | **/node_modules/ 3 | **/package-lock.json 4 | 5 | src/lib/api_schema.d.ts 6 | -------------------------------------------------------------------------------- /app/web_ui/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "plugins": ["prettier-plugin-svelte"], 3 | "useTabs": false, 4 | "tabWidth": 2, 5 | "semi": false, 6 | "overrides": [{ "files": "*.svelte", "options": { "parser": "svelte" } }] 7 | } 8 | -------------------------------------------------------------------------------- /app/web_ui/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kiln-web-ui", 3 | "version": "0.0.2", 4 | "private": true, 5 | "scripts": { 6 | "dev": "vite dev", 7 | "build": "vite build", 8 | "preview": "vite preview", 9 | "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", 10 | "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch", 11 | "test": "vitest", 12 | "test_run": "vitest run", 13 | "lint": "eslint .", 14 | "format": "prettier --write --plugin prettier-plugin-svelte ./", 15 | "format_check": "prettier --check --plugin prettier-plugin-svelte ./" 16 | }, 17 | "devDependencies": { 18 | "@redocly/cli": "^1.34.1", 19 | "@sveltejs/adapter-static": "^3.0.2", 20 | "@sveltejs/kit": "^2.20.6", 21 | "@sveltejs/vite-plugin-svelte": "^3.1.1", 22 | "@tailwindcss/typography": "^0.5.13", 23 | "@typescript-eslint/eslint-plugin": "^6.20.0", 24 | "@typescript-eslint/parser": "^6.19.0", 25 | "autoprefixer": "^10.4.15", 26 | "daisyui": "^4.7.3", 27 | "eslint": "^8.28.0", 28 | "eslint-config-prettier": "^9.1.0", 29 | "eslint-plugin-svelte": "^2.30.0", 30 | "openapi-typescript": "^7.4.1", 31 | "postcss": "^8.4.31", 32 | "prettier": "^3.0.3", 33 | "prettier-plugin-svelte": "^3.0.3", 34 | "svelte": "^4.2.19", 35 | "svelte-check": "^3.4.3", 36 | "tailwindcss": "^3.4.1", 37 | "typescript": "^5.6.3", 38 | "vite": "^5.4.19", 39 | "vitest": "^1.6.1" 40 | }, 41 | "type": "module", 42 | "dependencies": { 43 | "highlight.js": "^11.10.0", 44 | "openapi-fetch": "^0.12.2", 45 | "posthog-js": "^1.184.2" 46 | }, 47 | "overrides": { 48 | "@sveltejs/kit": { 49 | "cookie": "^0.7.0" 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /app/web_ui/postcss.config.js: -------------------------------------------------------------------------------- 1 | export default { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | } 7 | -------------------------------------------------------------------------------- /app/web_ui/src/ambient.d.ts: -------------------------------------------------------------------------------- 1 | declare global {} 2 | 3 | export {} 4 | -------------------------------------------------------------------------------- /app/web_ui/src/app.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | a { 6 | @apply cursor-pointer; 7 | } 8 | 9 | /* Work around https://github.com/saadeghi/daisyui/issues/2570 */ 10 | .loading-spinner { 11 | mask-image: url("data:image/svg+xml,%3Csvg width='24' height='24' stroke='black' viewBox='0 0 24 24' xmlns='http://www.w3.org/2000/svg'%3E%3Cg transform-origin='center'%3E%3Ccircle cx='12' cy='12' r='9.5' fill='none' stroke-width='3' stroke-linecap='round'%3E%3CanimateTransform attributeName='transform' type='rotate' from='0 12 12' to='360 12 12' dur='2s' repeatCount='indefinite'/%3E%3Canimate attributeName='stroke-dasharray' values='0,150;42,150;42,150' keyTimes='0;0.475;1' dur='1.5s' repeatCount='indefinite'/%3E%3Canimate attributeName='stroke-dashoffset' values='0;-16;-59' keyTimes='0;0.475;1' dur='1.5s' repeatCount='indefinite'/%3E%3C/circle%3E%3C/g%3E%3C/svg%3E"); 12 | } 13 | 14 | .btn-mid { 15 | @apply btn-sm; 16 | @apply h-10; 17 | @apply px-6; 18 | } 19 | -------------------------------------------------------------------------------- /app/web_ui/src/app.d.ts: -------------------------------------------------------------------------------- 1 | // See https://kit.svelte.dev/docs/types#app 2 | // for information about these interfaces 3 | declare global { 4 | namespace App { 5 | interface Locals {} 6 | interface PageData { 7 | session: Session | null 8 | } 9 | // interface Error {} 10 | // interface Platform {} 11 | } 12 | } 13 | 14 | export {} 15 | -------------------------------------------------------------------------------- /app/web_ui/src/app.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %sveltekit.head% 8 | 9 | 14 |
%sveltekit.body%
15 | 16 | 17 | -------------------------------------------------------------------------------- /app/web_ui/src/config.ts: -------------------------------------------------------------------------------- 1 | export const WebsiteName: string = "Kiln Studio" 2 | export const WebsiteBaseUrl: string = "https://getkiln.ai" 3 | export const WebsiteDescription: string = "Build ML Products with Kiln Studio" 4 | -------------------------------------------------------------------------------- /app/web_ui/src/index.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect } from "vitest" 2 | 3 | describe("sum test", () => { 4 | it("adds 1 + 2 to equal 3", () => { 5 | expect(1 + 2).toBe(3) 6 | }) 7 | }) 8 | -------------------------------------------------------------------------------- /app/web_ui/src/lib/api_client.ts: -------------------------------------------------------------------------------- 1 | import createClient from "openapi-fetch" 2 | import type { paths } from "./api_schema" 3 | 4 | export const base_url = "http://localhost:8757" 5 | 6 | export const client = createClient({ 7 | baseUrl: base_url, 8 | }) 9 | -------------------------------------------------------------------------------- /app/web_ui/src/lib/generate_schema.sh: -------------------------------------------------------------------------------- 1 | 2 | npx openapi-typescript http://localhost:8757/openapi.json -o api_schema.d.ts 3 | -------------------------------------------------------------------------------- /app/web_ui/src/lib/stores/fine_tune_store.ts: -------------------------------------------------------------------------------- 1 | import type { FinetuneProvider } from "$lib/types" 2 | import { writable, get } from "svelte/store" 3 | import { client } from "$lib/api_client" 4 | import { createKilnError, KilnError } from "$lib/utils/error_handlers" 5 | 6 | export const available_tuning_models = writable(null) 7 | export const available_models_error = writable(null) 8 | export const available_models_loading = writable(false) 9 | 10 | export async function get_available_models() { 11 | try { 12 | if (get(available_tuning_models)) { 13 | // Already loaded 14 | return 15 | } 16 | available_models_loading.set(true) 17 | available_models_error.set(null) 18 | const { data: available_models_response, error: get_error } = 19 | await client.GET("/api/finetune_providers", {}) 20 | if (get_error) { 21 | throw get_error 22 | } 23 | if (!available_models_response) { 24 | throw new Error("Invalid response from server") 25 | } 26 | available_tuning_models.set(available_models_response) 27 | } catch (e) { 28 | if (e instanceof Error && e.message.includes("Load failed")) { 29 | available_models_error.set( 30 | new KilnError("Could not load available models for fine-tuning.", null), 31 | ) 32 | } else { 33 | available_models_error.set(createKilnError(e)) 34 | } 35 | } finally { 36 | available_models_loading.set(false) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /app/web_ui/src/lib/stores/progress_ui_store.ts: -------------------------------------------------------------------------------- 1 | import type { Writable } from "svelte/store" 2 | import { localStorageStore } from "../stores" 3 | 4 | export type ProgressUIState = { 5 | title: string 6 | body: string 7 | cta: string | null 8 | link: string 9 | progress: number | null 10 | step_count: number | null 11 | current_step: number | null 12 | } 13 | 14 | export const progress_ui_state: Writable = 15 | localStorageStore("progress_ui_state", null) 16 | -------------------------------------------------------------------------------- /app/web_ui/src/lib/types.ts: -------------------------------------------------------------------------------- 1 | import type { components } from "./api_schema" 2 | 3 | // Project-Input is a variant with path 4 | export type Project = components["schemas"]["Project-Input"] 5 | export type Task = components["schemas"]["Task"] 6 | export type TaskRun = components["schemas"]["TaskRun-Input"] 7 | export type TaskRequirement = components["schemas"]["TaskRequirement"] 8 | export type TaskOutputRating = components["schemas"]["TaskOutputRating-Output"] 9 | export type TaskOutputRatingType = components["schemas"]["TaskOutputRatingType"] 10 | export type RequirementRating = components["schemas"]["RequirementRating"] 11 | export type RatingType = components["schemas"]["TaskOutputRatingType"] 12 | export type AvailableModels = components["schemas"]["AvailableModels"] 13 | export type ProviderModels = components["schemas"]["ProviderModels"] 14 | export type ProviderModel = components["schemas"]["ProviderModel"] 15 | export type ModelDetails = components["schemas"]["ModelDetails"] 16 | export type DatasetSplit = components["schemas"]["DatasetSplit"] 17 | export type Finetune = components["schemas"]["Finetune"] 18 | export type FinetuneProvider = components["schemas"]["FinetuneProvider"] 19 | export type FineTuneParameter = components["schemas"]["FineTuneParameter"] 20 | export type FinetuneWithStatus = components["schemas"]["FinetuneWithStatus"] 21 | export type OllamaConnection = components["schemas"]["OllamaConnection"] 22 | export type RunSummary = components["schemas"]["RunSummary"] 23 | export type PromptResponse = components["schemas"]["PromptResponse"] 24 | export type FinetuneDataStrategy = components["schemas"]["FinetuneDataStrategy"] 25 | export type EvalOutputScore = components["schemas"]["EvalOutputScore"] 26 | export type EvalTemplateId = components["schemas"]["EvalTemplateId"] 27 | export type Eval = components["schemas"]["Eval"] 28 | export type EvalConfigType = components["schemas"]["EvalConfigType"] 29 | export type EvalConfig = components["schemas"]["EvalConfig"] 30 | export type TaskRunConfig = components["schemas"]["TaskRunConfig"] 31 | export type EvalResultSummary = components["schemas"]["EvalResultSummary"] 32 | export type EvalRunResult = components["schemas"]["EvalRunResult"] 33 | export type EvalConfigCompareSummary = 34 | components["schemas"]["EvalConfigCompareSummary"] 35 | export type EvalRun = components["schemas"]["EvalRun"] 36 | export type EvalProgress = components["schemas"]["EvalProgress"] 37 | export type RatingOption = components["schemas"]["RatingOption"] 38 | export type RatingOptionResponse = components["schemas"]["RatingOptionResponse"] 39 | export type FinetuneDatasetInfo = components["schemas"]["FinetuneDatasetInfo"] 40 | -------------------------------------------------------------------------------- /app/web_ui/src/lib/ui/completed.svelte: -------------------------------------------------------------------------------- 1 | 7 | 8 |
11 | 12 | 21 |
{title}
22 |
23 | {subtitle} 24 |
25 | 26 | {button_text} 27 | 28 |
29 |
30 |
31 | -------------------------------------------------------------------------------- /app/web_ui/src/lib/ui/delete_dialog.svelte: -------------------------------------------------------------------------------- 1 | 41 | 42 | 53 |
54 | {#if delete_message} 55 |

{delete_message}

56 | {:else} 57 |

58 | Are you sure you want to delete this {name}? 59 |

60 | {/if} 61 | 64 |
65 | 66 | -------------------------------------------------------------------------------- /app/web_ui/src/lib/ui/fancy_select_types.ts: -------------------------------------------------------------------------------- 1 | export type OptionGroup = { 2 | label: string 3 | options: Option[] 4 | } 5 | export type Option = { 6 | label: string 7 | value: unknown 8 | description?: string 9 | } 10 | -------------------------------------------------------------------------------- /app/web_ui/src/lib/ui/info_tooltip.svelte: -------------------------------------------------------------------------------- 1 | 19 | 20 | 37 | -------------------------------------------------------------------------------- /app/web_ui/src/lib/ui/option_list.svelte: -------------------------------------------------------------------------------- 1 | 12 | 13 |
14 | {#each options as option} 15 | 43 | {/each} 44 |
45 | -------------------------------------------------------------------------------- /app/web_ui/src/lib/ui/progress_widget.svelte: -------------------------------------------------------------------------------- 1 | 34 | 35 | {#if $state} 36 | 45 |
46 | {$state?.title} 47 |
48 |
49 | {$state?.body} 50 | {$state?.cta}. 51 |
52 | {#if $state?.progress !== null} 53 | 58 | {:else if $state?.step_count !== null && $state?.current_step !== null} 59 |
60 |
61 |
    62 | {#each Array($state.step_count) as _, index} 63 |
  • 69 |   70 |
    • 71 | {/each} 72 |
73 |
74 |
75 | {:else} 76 |
In Progress
77 | {/if} 78 | 79 | {/if} 80 | -------------------------------------------------------------------------------- /app/web_ui/src/lib/ui/property_list.svelte: -------------------------------------------------------------------------------- 1 | 14 | 15 |
{title}
16 |
17 | {#each properties || [] as property} 18 |
19 | {property.name} 20 | {#if property.tooltip} 21 | 22 | {/if} 23 |
24 |
25 | {#if property.link} 26 | {property.value} 27 | {:else} 28 | {property.value} 29 | {/if} 30 |
31 | {/each} 32 |
33 | -------------------------------------------------------------------------------- /app/web_ui/src/lib/ui/splits.svelte: -------------------------------------------------------------------------------- 1 | 63 | -------------------------------------------------------------------------------- /app/web_ui/src/lib/utils/error_handlers.ts: -------------------------------------------------------------------------------- 1 | export class KilnError extends Error { 2 | private error_messages: string[] | null 3 | 4 | constructor(message: string | null, error_messages: string[] | null = null) { 5 | super(message || "Unknown error") 6 | this.name = "KilnError" 7 | this.error_messages = error_messages 8 | } 9 | 10 | getMessage(): string { 11 | if (this.error_messages && this.error_messages.length > 0) { 12 | return this.error_messages.join(".\n") 13 | } 14 | return this.message 15 | } 16 | 17 | getErrorMessages(): string[] { 18 | if (this.error_messages && this.error_messages.length > 0) { 19 | return this.error_messages 20 | } 21 | return [this.getMessage()] 22 | } 23 | } 24 | 25 | export function createKilnError(e: unknown): KilnError { 26 | if (e instanceof KilnError) { 27 | return e 28 | } 29 | if ( 30 | e && 31 | typeof e === "object" && 32 | "message" in e && 33 | typeof e.message === "string" 34 | ) { 35 | return new KilnError("Unexpected error: " + e.message, null) 36 | } 37 | if ( 38 | e && 39 | typeof e === "object" && 40 | "details" in e && 41 | typeof e.details === "string" 42 | ) { 43 | return new KilnError("Unexpected error: " + e.details, null) 44 | } 45 | if ( 46 | e && 47 | typeof e === "object" && 48 | "message" in e && 49 | typeof e.message === "string" 50 | ) { 51 | return new KilnError("Unexpected error: " + e.message, null) 52 | } 53 | 54 | return new KilnError("Unknown error", null) 55 | } 56 | -------------------------------------------------------------------------------- /app/web_ui/src/lib/utils/formatters.ts: -------------------------------------------------------------------------------- 1 | import { type EvalConfigType } from "$lib/types" 2 | 3 | export function formatDate(dateString: string | undefined): string { 4 | if (!dateString) { 5 | return "Unknown" 6 | } 7 | const date = new Date(dateString) 8 | const time_ago = Date.now() - date.getTime() 9 | 10 | if (time_ago < 1000 * 60) { 11 | return "just now" 12 | } 13 | if (time_ago < 1000 * 60 * 2) { 14 | return "1 minute ago" 15 | } 16 | if (time_ago < 1000 * 60 * 60) { 17 | return `${Math.floor(time_ago / (1000 * 60))} minutes ago` 18 | } 19 | if (date.toDateString() === new Date().toDateString()) { 20 | return ( 21 | date.toLocaleString(undefined, { 22 | hour: "numeric", 23 | minute: "2-digit", 24 | hour12: true, 25 | }) + " today" 26 | ) 27 | } 28 | 29 | const options: Intl.DateTimeFormatOptions = { 30 | year: "numeric", 31 | month: "2-digit", 32 | day: "2-digit", 33 | hour: "numeric", 34 | minute: "2-digit", 35 | hour12: true, 36 | } 37 | 38 | const formattedDate = date.toLocaleString(undefined, options) 39 | // Helps on line breaks with CA/US locales 40 | return formattedDate 41 | .replace(" AM", "am") 42 | .replace(" PM", "pm") 43 | .replace(",", "") 44 | } 45 | 46 | export function eval_config_to_ui_name( 47 | eval_config_type: EvalConfigType, 48 | ): string { 49 | return ( 50 | { 51 | g_eval: "G-Eval", 52 | llm_as_judge: "LLM as Judge", 53 | }[eval_config_type] || eval_config_type 54 | ) 55 | } 56 | 57 | export function data_strategy_name(data_strategy: string): string { 58 | switch (data_strategy) { 59 | case "final_only": 60 | return "Standard" 61 | case "final_and_intermediate": 62 | return "Reasoning" 63 | default: 64 | return data_strategy 65 | } 66 | } 67 | 68 | export function rating_name(rating_type: string): string { 69 | switch (rating_type) { 70 | case "five_star": 71 | return "5 star" 72 | case "pass_fail": 73 | return "Pass/Fail" 74 | case "pass_fail_critical": 75 | return "Pass/Fail/Critical" 76 | default: 77 | return rating_type 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /app/web_ui/src/lib/utils/link_builder.ts: -------------------------------------------------------------------------------- 1 | export function prompt_link( 2 | project_id: string, 3 | task_id: string, 4 | prompt_id: string, 5 | ): string | undefined { 6 | if (!project_id || !task_id || !prompt_id) { 7 | return undefined 8 | } 9 | if (!prompt_id.includes("::")) { 10 | // not an ID style prompt, link to static 11 | return `/prompts/${project_id}/${task_id}/generator_details/${encodeURIComponent(prompt_id)}` 12 | } 13 | // ID style prompt, link to saved 14 | return `/prompts/${project_id}/${task_id}/saved/${encodeURIComponent(prompt_id)}` 15 | } 16 | -------------------------------------------------------------------------------- /app/web_ui/src/lib/utils/update.ts: -------------------------------------------------------------------------------- 1 | import { createKilnError, KilnError } from "$lib/utils/error_handlers" 2 | import { writable } from "svelte/store" 3 | 4 | export const app_version = "0.16.0" 5 | 6 | export type UpdateCheckResult = { 7 | has_update: boolean 8 | latest_version: string 9 | link: string 10 | } 11 | 12 | export type UpdateState = { 13 | update_result: UpdateCheckResult | null 14 | update_loading: boolean 15 | update_error: KilnError | null 16 | } 17 | 18 | export const default_update_state: UpdateState = { 19 | update_result: null, 20 | update_loading: false, 21 | update_error: null, 22 | } 23 | 24 | export const update_info = writable(default_update_state) 25 | 26 | export async function update_update_store() { 27 | let update_result: UpdateCheckResult | null = null 28 | let update_error: KilnError | null = null 29 | try { 30 | update_info.update(() => default_update_state) 31 | const update = await check_for_update() 32 | if (update instanceof KilnError) { 33 | update_error = update 34 | } else { 35 | update_result = update 36 | } 37 | } catch (e) { 38 | update_error = createKilnError(e) 39 | } finally { 40 | update_info.update(() => ({ 41 | update_loading: false, 42 | update_result, 43 | update_error, 44 | })) 45 | } 46 | } 47 | 48 | export async function check_for_update(): Promise< 49 | UpdateCheckResult | KilnError 50 | > { 51 | try { 52 | const response = await fetch( 53 | "https://api.github.com/repos/Kiln-AI/Kiln/releases/latest", 54 | ) 55 | const data = await response.json() 56 | const html_url = data.html_url 57 | const full_version = data.tag_name 58 | if (!html_url || !full_version) { 59 | return new KilnError("Failed to fetch update data", []) 60 | } 61 | const [version] = full_version.split("-") 62 | return { 63 | has_update: semantic_version_compare(version, app_version), 64 | latest_version: full_version, 65 | link: html_url, 66 | } 67 | } catch (e) { 68 | return createKilnError(e) 69 | } 70 | } 71 | 72 | export function semantic_version_compare(a: string, b: string): boolean { 73 | // Strip leading 'v' if present 74 | const clean_a = a.replace(/^v/, "") 75 | const clean_b = b.replace(/^v/, "") 76 | 77 | const a_parts = clean_a.split(".").map(Number) 78 | const b_parts = clean_b.split(".").map(Number) 79 | 80 | // Pad shorter array with zeros to match length 81 | const max_length = Math.max(a_parts.length, b_parts.length) 82 | while (a_parts.length < max_length) a_parts.push(0) 83 | while (b_parts.length < max_length) b_parts.push(0) 84 | 85 | // Compare each part from left to right 86 | for (let i = 0; i < max_length; i++) { 87 | if (a_parts[i] > b_parts[i]) return true 88 | if (a_parts[i] < b_parts[i]) return false 89 | } 90 | 91 | return false // versions are equal 92 | } 93 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/+page.svelte: -------------------------------------------------------------------------------- 1 | 6 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/+page.ts: -------------------------------------------------------------------------------- 1 | export const ssr = false 2 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/dataset/[project_id]/[task_id]/+page.ts: -------------------------------------------------------------------------------- 1 | export const prerender = false 2 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/dataset/[project_id]/[task_id]/[run_id]/run/+page.ts: -------------------------------------------------------------------------------- 1 | export const prerender = false 2 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/dataset/[project_id]/[task_id]/add_data/+page.ts: -------------------------------------------------------------------------------- 1 | export const prerender = false 2 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/dataset/[project_id]/[task_id]/empty_into.svelte: -------------------------------------------------------------------------------- 1 | 5 | 6 |
7 |
8 |
9 | 10 | 16 | 22 | 28 | 33 | 38 | 43 | 44 |
45 |
Your dataset for this task is empty.
46 |
Adding data will allow the model to improve at it's task.
47 |
To get started, generate some synthetic data or add data manually
48 | 49 | Manually Add Data 50 | 51 | Generate Synthetic Data 52 | 53 |
54 |
55 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/evals/[project_id]/[task_id]/+page.ts: -------------------------------------------------------------------------------- 1 | export const prerender = false 2 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/evals/[project_id]/[task_id]/[eval_id]/+page.ts: -------------------------------------------------------------------------------- 1 | export const prerender = false 2 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/evals/[project_id]/[task_id]/[eval_id]/[eval_config_id]/[run_config_id]/run_result/+page.ts: -------------------------------------------------------------------------------- 1 | export const prerender = false 2 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/evals/[project_id]/[task_id]/[eval_id]/compare_run_methods/+page.ts: -------------------------------------------------------------------------------- 1 | export const prerender = false 2 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/evals/[project_id]/[task_id]/[eval_id]/create_eval_config/+page.ts: -------------------------------------------------------------------------------- 1 | export const prerender = false 2 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/evals/[project_id]/[task_id]/[eval_id]/eval_configs/+page.ts: -------------------------------------------------------------------------------- 1 | export const prerender = false 2 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/evals/[project_id]/[task_id]/[eval_id]/eval_configs/eval_config_instruction.svelte: -------------------------------------------------------------------------------- 1 | 15 | 16 | {#if eval_config} 17 | {@const eval_steps = get_eval_steps(eval_config)} 18 |
19 |
Task Description:
20 | {eval_config.properties["task_description"] || "No description provided."} 21 |
22 | {#if eval_steps} 23 |
24 |
Evaluation Steps:
25 |
    26 | {#each eval_steps as step} 27 |
  1. 28 | 29 | {step} 30 | 31 |
    2. 32 | {/each} 33 |
34 |
35 | {/if} 36 | {/if} 37 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/evals/[project_id]/[task_id]/[eval_id]/output_type_table_preview.svelte: -------------------------------------------------------------------------------- 1 | 7 | 8 |
9 | {#if output_score_type === "five_star"} 10 | 1 to 5 11 | 12 | 13 | 14 | {:else if output_score_type === "pass_fail"} 15 | pass/fail 16 | 17 | 18 | 19 | {:else if output_score_type === "pass_fail_critical"} 20 | pass/fail/critical 21 | 25 | {:else} 26 | {output_score_type} 27 | {/if} 28 |
29 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/evals/[project_id]/[task_id]/create_evaluator/+page.ts: -------------------------------------------------------------------------------- 1 | export const prerender = false 2 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/evals/[project_id]/[task_id]/create_evaluator/eval_template.ts: -------------------------------------------------------------------------------- 1 | import type { EvalOutputScore, EvalTemplateId } from "$lib/types" 2 | 3 | export type EvalTemplateResult = { 4 | // Server IDs are EvalTemplateId. We have a custom "none" value for the UI. 5 | template_id: EvalTemplateId | "none" 6 | name: string 7 | description: string 8 | output_scores: EvalOutputScore[] 9 | } 10 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/evals/[project_id]/[task_id]/empty_eval.svelte: -------------------------------------------------------------------------------- 1 | 5 | 6 |
7 |
8 |
9 | 10 | 16 | 21 | 28 | 35 | 41 | 47 | 48 |
49 |
50 | Improve Quality and Move Faster with Evaluations 51 |
52 |
Create powerful evaluators using LLMs to judge performance.
53 |
54 | Quickly compare many approaches to find what works best for your task. 55 |
56 |
57 | Ensure quality over time, back testing prior bugs and benchmarking new 58 | approaches. 59 |
60 | 64 | Create an Evaluator 65 | 66 | 71 | Evals Guide 72 | 73 |
74 |
75 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/fine_tune/[project_id]/[task_id]/+page.ts: -------------------------------------------------------------------------------- 1 | export const prerender = false 2 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/fine_tune/[project_id]/[task_id]/create_finetune/+page.ts: -------------------------------------------------------------------------------- 1 | export const prerender = false 2 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/fine_tune/[project_id]/[task_id]/empty_finetune.svelte: -------------------------------------------------------------------------------- 1 | 5 | 6 |
7 |
8 |
9 | 10 | 16 | 24 | 32 | 40 | 46 | 52 | 58 | 64 | 70 | 76 | 77 |
78 |
79 | Fine-Tuning Learns from Your Dataset to Create Custom Models 80 |
81 |
82 | Fine-tuned models can be faster, cheaper and more accurate than standard 83 | models. 84 |
85 | 89 | Create a Fine-Tune 90 | 91 | 96 | Fine Tuning Guide 97 | 98 |
99 |
100 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/fine_tune/[project_id]/[task_id]/fine_tune/[finetune_id]/+page.ts: -------------------------------------------------------------------------------- 1 | export const prerender = false 2 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/generate/[project_id]/[task_id]/+page.ts: -------------------------------------------------------------------------------- 1 | export const prerender = false 2 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/generate/[project_id]/[task_id]/gen_model.ts: -------------------------------------------------------------------------------- 1 | export type SampleData = { 2 | input: string 3 | saved_id: string | null 4 | model_name: string 5 | model_provider: string 6 | // Optional. The tree path to the topic that the sample belongs to. 7 | // The actual node tree has this, but it can also be stored here for convenience. 8 | topic_path?: string[] 9 | } 10 | 11 | export type SampleDataNode = { 12 | topic: string 13 | sub_topics: SampleDataNode[] 14 | samples: SampleData[] 15 | } 16 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/generate/[project_id]/[task_id]/increment_ui.svelte: -------------------------------------------------------------------------------- 1 | 6 | 7 |
8 | 14 | {value} 15 | 21 |
22 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/prompts/[project_id]/[task_id]/+page.ts: -------------------------------------------------------------------------------- 1 | export const prerender = false 2 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/prompts/[project_id]/[task_id]/create/+page.ts: -------------------------------------------------------------------------------- 1 | export const prerender = false 2 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/prompts/[project_id]/[task_id]/generator_details/[generator_id]/+page.ts: -------------------------------------------------------------------------------- 1 | export const prerender = false 2 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/prompts/[project_id]/[task_id]/saved/[prompt_id]/+page.ts: -------------------------------------------------------------------------------- 1 | export const prerender = false 2 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/run/output.svelte: -------------------------------------------------------------------------------- 1 | 27 | 28 | 29 | 30 | 31 | 32 |
33 | 34 | 
{#if formatted_json_html}{@html formatted_json_html}{:else}{raw_output}{/if}
36 | 37 |
38 | 57 |
58 |
59 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/run/rating.svelte: -------------------------------------------------------------------------------- 1 | 23 | 24 |
25 | {#if type === "five_star"} 26 | 27 | 28 | 36 | {#each [1, 2, 3, 4, 5] as r} 37 | (hover_rating = r)} 43 | on:focus={() => (hover_rating = r)} 44 | on:mouseleave={() => (hover_rating = null)} 45 | on:blur={() => (hover_rating = null)} 46 | on:click={() => rating_clicked(r)} 47 | value={r} 48 | bind:group={rating} 49 | /> 50 | {/each} 51 | {:else if type === "custom"} 52 |
53 | Custom type not supported in UI 54 |
55 | {:else if type === "pass_fail_critical" || type === "pass_fail"} 56 |
57 | 63 | 69 | {#if type === "pass_fail_critical"} 70 | 76 | {/if} 77 |
78 | {/if} 79 |
80 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/run/tag_dropdown.svelte: -------------------------------------------------------------------------------- 1 | 32 | 33 |
34 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | {#if error} 50 |
{error}
51 | {/if} 52 |
53 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/settings/+page.svelte: -------------------------------------------------------------------------------- 1 | 53 | 54 | 55 |
56 | {#each sections as section} 57 |
58 |
59 |

{section.name}

60 |

{section.description}

61 |
62 | 68 | {section.button_text} 69 | 70 |
71 | {/each} 72 |
73 | 74 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/settings/check_for_update/+page.svelte: -------------------------------------------------------------------------------- 1 | 14 | 15 | 19 |
20 | {#if $update_info.update_loading} 21 |
22 | 23 |
24 | {:else if $update_info.update_result && $update_info.update_result.has_update} 25 |
Update Available
26 |
27 | Version {$update_info.update_result.latest_version} is available. 28 |
29 | Download Update 34 | {:else if $update_info.update_result && !$update_info.update_result.has_update} 35 |
No Update Available
36 |
You are using the latest version of Kiln.
37 | {:else} 38 |
Error Checking for Update
39 |
{$update_info.update_error?.message}
40 | {/if} 41 |
42 | 43 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/settings/clone_task/[project_id]/[task_id]/+page.svelte: -------------------------------------------------------------------------------- 1 | 5 | 6 |
7 | 12 | 13 | 14 |
15 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/settings/clone_task/[project_id]/[task_id]/+page.ts: -------------------------------------------------------------------------------- 1 | export const prerender = false 2 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/settings/create_project/+page.svelte: -------------------------------------------------------------------------------- 1 | 5 | 6 |
7 | 11 | 12 | 13 |
14 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/settings/create_task/[slug]/+page.svelte: -------------------------------------------------------------------------------- 1 | 9 | 10 |
11 | 16 | 17 | 18 |
19 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/settings/create_task/[slug]/+page.ts: -------------------------------------------------------------------------------- 1 | export const prerender = false 2 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/settings/edit_project/[slug]/+page.svelte: -------------------------------------------------------------------------------- 1 | 11 | 12 |
13 | 14 | 15 | 16 |
17 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/settings/edit_project/[slug]/+page.ts: -------------------------------------------------------------------------------- 1 | export const prerender = false 2 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/settings/edit_task/[project_id]/[task_id]/+page.svelte: -------------------------------------------------------------------------------- 1 | 28 | 29 |
30 | delete_dialog?.show(), 37 | }, 38 | { 39 | label: "Clone Task", 40 | handler: () => { 41 | goto(`/settings/clone_task/${project_id}/${task_id}`) 42 | }, 43 | }, 44 | ]} 45 | > 46 | 47 | 48 |
49 | 50 | 56 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/settings/edit_task/[project_id]/[task_id]/+page.ts: -------------------------------------------------------------------------------- 1 | export const prerender = false 2 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/settings/edit_task/[project_id]/[task_id]/load_task_editor.svelte: -------------------------------------------------------------------------------- 1 | 65 | 66 | {#if loading} 67 |
68 | 69 |
70 | {:else if error} 71 |
Error loading task: {error.getMessage()}
72 | {:else if task} 73 | 80 | {/if} 81 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/settings/intro/+page.svelte: -------------------------------------------------------------------------------- 1 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/settings/providers/+page.svelte: -------------------------------------------------------------------------------- 1 | 6 | 7 | 19 | 22 | 23 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(app)/types.ts: -------------------------------------------------------------------------------- 1 | // A type for a button which can appear on an app page 2 | export type ActionButton = { 3 | label?: string 4 | icon?: string 5 | handler?: () => void 6 | href?: string 7 | primary?: boolean 8 | notice?: boolean 9 | shortcut?: string 10 | disabled?: boolean 11 | } 12 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(fullscreen)/+layout.svelte: -------------------------------------------------------------------------------- 1 |
2 |
3 | 4 |
5 |
6 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(fullscreen)/setup/(setup)/+layout.svelte: -------------------------------------------------------------------------------- 1 | 3 | 4 |
5 |
6 |
7 | 8 |
9 |
10 |
11 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(fullscreen)/setup/(setup)/connect_providers/+page.svelte: -------------------------------------------------------------------------------- 1 | 9 | 10 |
11 |
12 | logo 13 |
14 |

15 | Connect AI Providers 16 |

17 |

18 | Kiln is free, but your need to connect API keys to use AI services. 19 |

20 | 21 |
24 | 25 |
26 | 27 |
30 | 34 | 35 | 36 |
37 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(fullscreen)/setup/(setup)/create_project/+page.svelte: -------------------------------------------------------------------------------- 1 | 16 | 17 |
18 |
19 | logo 20 |
21 |

22 | Create a Project 23 |

24 |

25 | "Example" is fine if you're just trying things out. 26 |

27 |

28 | Just exploring? 29 | 32 |

33 | 34 |
37 | 38 |
39 | 40 |
41 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(fullscreen)/setup/(setup)/create_task/[slug]/+page.svelte: -------------------------------------------------------------------------------- 1 | 7 | 8 |
9 |
10 | logo 11 |
12 |

13 | Create a Task 14 |

15 |

16 | Let's define what this model should do. We call this a "task". 17 |

18 | 19 |
22 | 23 |
24 | 25 |
26 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(fullscreen)/setup/(setup)/create_task/[slug]/+page.ts: -------------------------------------------------------------------------------- 1 | export const prerender = false 2 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(fullscreen)/setup/(setup)/create_task/schema_section.svelte: -------------------------------------------------------------------------------- 1 | 44 | 45 |
46 |
47 | 57 |
58 |
59 | 69 |
70 | 71 | {#if !plaintext} 72 | 78 | {/if} 79 |
80 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(fullscreen)/setup/(setup)/intro/+page.svelte: -------------------------------------------------------------------------------- 1 | 6 | 7 |
8 |
9 | logo 10 |
11 |

12 | Introduction 13 |

14 | 15 |
18 | 19 |
20 | 21 |
22 | {#if completed} 23 | 24 | 25 | 26 | {:else} 27 | Skip Tutorial 28 | {/if} 29 |
30 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(fullscreen)/setup/(setup)/intro/tutorial_section.svelte: -------------------------------------------------------------------------------- 1 | 6 | 7 |
8 |
9 |
10 |

{title}

11 | {#each promos as promo} 12 |

{promo}

13 | {/each} 14 |
15 |
16 | {title} 17 |
18 |
19 |
20 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(fullscreen)/setup/(setup)/select_task/+page.svelte: -------------------------------------------------------------------------------- 1 | 4 | 5 |
6 |
7 | logo 8 |
9 |

10 | Select a Project and Task 11 |

12 |

13 | Select a project and task to get started. You can also create a new projects 14 | and tasks. 15 |

16 | 17 |
20 | 24 |
25 | 26 |
27 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/(fullscreen)/setup/+page.svelte: -------------------------------------------------------------------------------- 1 |
2 |
3 | logo 8 | 9 |

Welcome to Kiln

10 |

11 | The easiest way to build AI products 12 |

13 | Get Started 16 |
17 | View our License Agreement 22 |
23 |
24 |
25 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/+error.svelte: -------------------------------------------------------------------------------- 1 | 5 | 6 |
7 |
8 |
9 |

This is embarrassing...

10 |

There was an error: {$page?.error?.message}

11 |
12 | Return Home 13 |
14 |
15 |
16 |
17 | -------------------------------------------------------------------------------- /app/web_ui/src/routes/+layout.ts: -------------------------------------------------------------------------------- 1 | import posthog from "posthog-js" 2 | import { browser } from "$app/environment" 3 | import { dev } from "$app/environment" 4 | 5 | export const prerender = true 6 | export const ssr = false 7 | 8 | export const load = async () => { 9 | if (browser && !dev) { 10 | posthog.init("phc_pdNulYUFOFmRcgeQkYCOAiCQiZOC4VP8npDtRkNSirw", { 11 | api_host: "https://us.i.posthog.com", 12 | person_profiles: "identified_only", 13 | capture_pageview: false, 14 | capture_pageleave: false, 15 | }) 16 | } 17 | return 18 | } 19 | -------------------------------------------------------------------------------- /app/web_ui/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kiln-AI/Kiln/06b5dbe8560ee8cdfee1bbcc3ddfb0f5840a4508/app/web_ui/static/favicon.ico -------------------------------------------------------------------------------- /app/web_ui/static/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kiln-AI/Kiln/06b5dbe8560ee8cdfee1bbcc3ddfb0f5840a4508/app/web_ui/static/favicon.png -------------------------------------------------------------------------------- /app/web_ui/static/favicons/apple-touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kiln-AI/Kiln/06b5dbe8560ee8cdfee1bbcc3ddfb0f5840a4508/app/web_ui/static/favicons/apple-touch-icon.png -------------------------------------------------------------------------------- /app/web_ui/static/favicons/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kiln-AI/Kiln/06b5dbe8560ee8cdfee1bbcc3ddfb0f5840a4508/app/web_ui/static/favicons/favicon-16x16.png -------------------------------------------------------------------------------- /app/web_ui/static/favicons/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kiln-AI/Kiln/06b5dbe8560ee8cdfee1bbcc3ddfb0f5840a4508/app/web_ui/static/favicons/favicon-32x32.png -------------------------------------------------------------------------------- /app/web_ui/static/favicons/icon_192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kiln-AI/Kiln/06b5dbe8560ee8cdfee1bbcc3ddfb0f5840a4508/app/web_ui/static/favicons/icon_192.png -------------------------------------------------------------------------------- /app/web_ui/static/favicons/icon_512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kiln-AI/Kiln/06b5dbe8560ee8cdfee1bbcc3ddfb0f5840a4508/app/web_ui/static/favicons/icon_512.png -------------------------------------------------------------------------------- /app/web_ui/static/favicons/site.webmanifest: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Kiln", 3 | "short_name": "Kiln", 4 | "description": "The easiest way to built AI products", 5 | "start_url": "/", 6 | "display": "minimal-ui", 7 | "theme_color": "#f5f5f5", 8 | "background_color": "#f5f5f5", 9 | "icons": [ 10 | { 11 | "src": "/favicons/icon_192.png", 12 | "sizes": "192x192", 13 | "type": "image/png" 14 | }, 15 | { 16 | "src": "/favicons/icon_512.png", 17 | "sizes": "512x512", 18 | "type": "image/png" 19 | } 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /app/web_ui/static/images/add.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /app/web_ui/static/images/animated_logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /app/web_ui/static/images/anthropic.svg: -------------------------------------------------------------------------------- 1 | Anthropic -------------------------------------------------------------------------------- /app/web_ui/static/images/api.svg: -------------------------------------------------------------------------------- 1 | 2 | API 3 | -------------------------------------------------------------------------------- /app/web_ui/static/images/circle-check.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /app/web_ui/static/images/collaborate.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kiln-AI/Kiln/06b5dbe8560ee8cdfee1bbcc3ddfb0f5840a4508/app/web_ui/static/images/collaborate.png -------------------------------------------------------------------------------- /app/web_ui/static/images/data_driven.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kiln-AI/Kiln/06b5dbe8560ee8cdfee1bbcc3ddfb0f5840a4508/app/web_ui/static/images/data_driven.png -------------------------------------------------------------------------------- /app/web_ui/static/images/data_gen_tree.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kiln-AI/Kiln/06b5dbe8560ee8cdfee1bbcc3ddfb0f5840a4508/app/web_ui/static/images/data_gen_tree.png -------------------------------------------------------------------------------- /app/web_ui/static/images/delete.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /app/web_ui/static/images/developers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kiln-AI/Kiln/06b5dbe8560ee8cdfee1bbcc3ddfb0f5840a4508/app/web_ui/static/images/developers.png -------------------------------------------------------------------------------- /app/web_ui/static/images/filter.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /app/web_ui/static/images/fireworks.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /app/web_ui/static/images/gemini.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /app/web_ui/static/images/google_logo.svg: -------------------------------------------------------------------------------- 1 | 2 | ionicons-v5_logos -------------------------------------------------------------------------------- /app/web_ui/static/images/groq.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /app/web_ui/static/images/logo_grid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kiln-AI/Kiln/06b5dbe8560ee8cdfee1bbcc3ddfb0f5840a4508/app/web_ui/static/images/logo_grid.png -------------------------------------------------------------------------------- /app/web_ui/static/images/next.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /app/web_ui/static/images/openai.svg: -------------------------------------------------------------------------------- 1 | 2 | OpenAI icon -------------------------------------------------------------------------------- /app/web_ui/static/images/openrouter.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/web_ui/static/images/previous.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /app/web_ui/static/images/tag.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /app/web_ui/static/images/together_ai.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /app/web_ui/static/images/training.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kiln-AI/Kiln/06b5dbe8560ee8cdfee1bbcc3ddfb0f5840a4508/app/web_ui/static/images/training.png -------------------------------------------------------------------------------- /app/web_ui/static/images/wandb.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /app/web_ui/static/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /app/web_ui/static/styles/highlightjs.min.css: -------------------------------------------------------------------------------- 1 | /*! 2 | Theme: Grayscale Light 3 | Author: Alexandre Gavioli (https://github.com/Alexx2/) 4 | License: ~ MIT (or more permissive) [via base16-schemes-source] 5 | Maintainer: @highlightjs/core-team 6 | Version: 2021.09.0 7 | */ 8 | pre code.hljs { 9 | display: block; 10 | overflow-x: auto; 11 | padding: 1em; 12 | } 13 | code.hljs { 14 | padding: 3px 5px; 15 | } 16 | .hljs { 17 | color: #464646; 18 | background: #f7f7f7; 19 | } 20 | .hljs ::selection, 21 | .hljs::selection { 22 | background-color: #b9b9b9; 23 | color: #464646; 24 | } 25 | .hljs-comment { 26 | color: #ababab; 27 | } 28 | .hljs-tag { 29 | color: #525252; 30 | } 31 | .hljs-operator, 32 | .hljs-punctuation, 33 | .hljs-subst { 34 | color: #464646; 35 | } 36 | .hljs-operator { 37 | opacity: 0.7; 38 | } 39 | .hljs-bullet, 40 | .hljs-deletion, 41 | .hljs-name, 42 | .hljs-selector-tag, 43 | .hljs-template-variable, 44 | .hljs-variable { 45 | color: #7c7c7c; 46 | } 47 | .hljs-attr, 48 | .hljs-link, 49 | .hljs-literal, 50 | .hljs-number, 51 | .hljs-symbol, 52 | .hljs-variable.constant_ { 53 | color: rgb(1, 86, 146); 54 | } 55 | .hljs-class .hljs-title, 56 | .hljs-title, 57 | .hljs-title.class_ { 58 | color: #a0a0a0; 59 | } 60 | .hljs-strong { 61 | font-weight: 700; 62 | color: #a0a0a0; 63 | } 64 | .hljs-addition, 65 | .hljs-code, 66 | .hljs-string, 67 | .hljs-title.class_.inherited__ { 68 | color: #8e8e8e; 69 | } 70 | .hljs-built_in, 71 | .hljs-doctag, 72 | .hljs-keyword.hljs-atrule, 73 | .hljs-quote, 74 | .hljs-regexp { 75 | color: #868686; 76 | } 77 | .hljs-attribute, 78 | .hljs-function .hljs-title, 79 | .hljs-section, 80 | .hljs-title.function_, 81 | .ruby .hljs-property { 82 | color: #686868; 83 | } 84 | .diff .hljs-meta, 85 | .hljs-keyword, 86 | .hljs-template-tag, 87 | .hljs-type { 88 | color: #747474; 89 | } 90 | .hljs-emphasis { 91 | color: #747474; 92 | font-style: italic; 93 | } 94 | .hljs-meta, 95 | .hljs-meta .hljs-keyword, 96 | .hljs-meta .hljs-string { 97 | color: #5e5e5e; 98 | } 99 | .hljs-meta .hljs-keyword, 100 | .hljs-meta-keyword { 101 | font-weight: 700; 102 | } 103 | -------------------------------------------------------------------------------- /app/web_ui/svelte.config.js: -------------------------------------------------------------------------------- 1 | import adapter from "@sveltejs/adapter-static" 2 | import { vitePreprocess } from "@sveltejs/vite-plugin-svelte" 3 | 4 | /** @type {import('@sveltejs/kit').Config} */ 5 | const config = { 6 | kit: { 7 | adapter: adapter({ 8 | fallback: "404.html", 9 | }), 10 | }, 11 | prerender: { 12 | default: true, 13 | }, 14 | preprocess: vitePreprocess(), 15 | } 16 | 17 | export default config 18 | -------------------------------------------------------------------------------- /app/web_ui/tailwind.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('tailwindcss').Config} */ 2 | import daisyui from "daisyui" 3 | import typography from "@tailwindcss/typography" 4 | 5 | export default { 6 | content: ["./src/**/*.{html,js,svelte,ts}"], 7 | theme: { 8 | extend: {}, 9 | }, 10 | plugins: [typography, daisyui], 11 | daisyui: { 12 | themes: [ 13 | { 14 | kilntheme: { 15 | primary: "#415CF5", 16 | "primary-content": "#ffffff", 17 | secondary: "#131517", 18 | "secondary-content": "#ffffff", 19 | accent: "#E74D31", 20 | "accent-content": "#ffffff", 21 | neutral: "#e7e5e4", 22 | "neutral-content": "#131517", 23 | "base-100": "#ffffff", 24 | "base-200": "#F5F5F5", 25 | "base-300": "#bebebe", 26 | "base-content": "#161616", 27 | info: "#D7AAF9", 28 | "info-content": "#0a1616", 29 | success: "#33B79D", 30 | "success-content": "#ffffff", 31 | warning: "#F4B544", 32 | "warning-content": "#0a1616", 33 | error: "#E74D31", 34 | "error-content": "#ffffff", 35 | }, 36 | }, 37 | ], 38 | }, 39 | } 40 | -------------------------------------------------------------------------------- /app/web_ui/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./.svelte-kit/tsconfig.json", 3 | "compilerOptions": { 4 | "allowJs": true, 5 | "checkJs": true, 6 | "esModuleInterop": true, 7 | "forceConsistentCasingInFileNames": true, 8 | "resolveJsonModule": true, 9 | "skipLibCheck": true, 10 | "sourceMap": true, 11 | "strict": true 12 | } 13 | // Path aliases are handled by https://kit.svelte.dev/docs/configuration#alias and https://kit.svelte.dev/docs/configuration#files 14 | // 15 | // If you want to overwrite includes/excludes, make sure to copy over the relevant includes/excludes 16 | // from the referenced tsconfig.json - TypeScript does not merge them in 17 | } 18 | -------------------------------------------------------------------------------- /app/web_ui/vite.config.ts: -------------------------------------------------------------------------------- 1 | import { sveltekit } from "@sveltejs/kit/vite" 2 | import { defineConfig } from "vitest/config" 3 | 4 | export default defineConfig({ 5 | plugins: [sveltekit()], 6 | test: { 7 | include: ["src/**/*.{test,spec}.{js,ts}"], 8 | }, 9 | }) 10 | -------------------------------------------------------------------------------- /checks.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Check our project: formatting, linting, testing, building, etc. 4 | # Good to call this from .git/hooks/pre-commit 5 | 6 | # Important: run with `uv run` to setup the environment 7 | 8 | set -e 9 | 10 | # work from the root of the repo 11 | cd "$(dirname "$0")" 12 | 13 | headerStart="\n\033[4;34m=== " 14 | headerEnd=" ===\033[0m\n" 15 | 16 | echo "${headerStart}Checking Python: Ruff, format, check${headerEnd}" 17 | # I is import sorting 18 | uvx ruff check --select I 19 | uvx ruff format --check . 20 | 21 | echo "${headerStart}Checking for Misspellings${headerEnd}" 22 | find . -type f | grep -v "/node_modules/" | grep -v "/\." | grep -v "/dist/" | grep -v "/desktop/build/" | xargs misspell -error 23 | echo "No misspellings found" 24 | 25 | 26 | echo "${headerStart}Web UI: format, lint, check${headerEnd}" 27 | changed_files=$(git diff --name-only --staged) 28 | if [[ "$changed_files" == *"app/web_ui/"* ]]; then 29 | echo "${headerStart}Checking Web UI: format, lint, check${headerEnd}" 30 | cd app/web_ui 31 | npm run format_check 32 | npm run lint 33 | npm run check 34 | npm run test_run 35 | echo "Running vite build" 36 | npm run build > /dev/null 37 | cd ../.. 38 | else 39 | echo "Skipping Web UI: no files changed" 40 | fi 41 | 42 | echo "${headerStart}Checking Types${headerEnd}" 43 | pyright . 44 | 45 | echo "${headerStart}Running Python Tests${headerEnd}" 46 | python3 -m pytest --benchmark-quiet -q . 47 | -------------------------------------------------------------------------------- /conftest.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import patch 2 | 3 | import litellm 4 | import pytest 5 | from dotenv import load_dotenv 6 | from kiln_ai.utils.config import Config 7 | 8 | 9 | @pytest.fixture(autouse=True) 10 | def _clear_httpx_clients() -> None: 11 | litellm.in_memory_llm_clients_cache.flush_cache() 12 | 13 | 14 | @pytest.fixture(scope="session", autouse=True) 15 | def load_env(): 16 | load_dotenv() 17 | 18 | 19 | # mock out the settings path so we don't clobber the user's actual settings during tests 20 | @pytest.fixture(autouse=True) 21 | def use_temp_settings_dir(tmp_path): 22 | with patch.object( 23 | Config, "settings_path", return_value=str(tmp_path / "settings.yaml") 24 | ): 25 | yield 26 | 27 | 28 | def pytest_addoption(parser): 29 | parser.addoption( 30 | "--runpaid", 31 | action="store_true", 32 | default=False, 33 | help="run tests that make paid API calls", 34 | ) 35 | parser.addoption( 36 | "--runsinglewithoutchecks", 37 | action="store_true", 38 | default=False, 39 | help="if testing a single test, don't check for skips like runpaid", 40 | ) 41 | parser.addoption( 42 | "--ollama", 43 | action="store_true", 44 | default=False, 45 | help="run tests that use ollama server", 46 | ) 47 | 48 | 49 | def is_single_manual_test(config, items) -> bool: 50 | # Check if we're running manually (eg, in vscode) 51 | if not config.getoption("--runsinglewithoutchecks"): 52 | return False 53 | 54 | if len(items) == 1: 55 | return True 56 | if len(items) == 0: 57 | return False 58 | 59 | # Check if all of the items are the same prefix, expluding a.b.c[param] 60 | # This is still a 'single test' for the purposes of this flag 61 | prefix = items[0].name.split("[")[0] + "[" 62 | for item in items: 63 | if not item.name.startswith(prefix): 64 | return False 65 | return True 66 | 67 | 68 | def pytest_collection_modifyitems(config, items): 69 | # Always run test if it's a single test manually invoked 70 | if is_single_manual_test(config, items): 71 | return 72 | 73 | # Mark tests that use paid services as skipped unless --runpaid is passed 74 | if not config.getoption("--runpaid"): 75 | skip_paid = pytest.mark.skip(reason="need --runpaid option to run") 76 | for item in items: 77 | if "paid" in item.keywords: 78 | item.add_marker(skip_paid) 79 | 80 | # Mark tests that use ollama server as skipped unless --ollama is passed 81 | if not config.getoption("--ollama"): 82 | skip_ollama = pytest.mark.skip(reason="need --ollama option to run") 83 | for item in items: 84 | if "ollama" in item.keywords: 85 | item.add_marker(skip_ollama) 86 | -------------------------------------------------------------------------------- /guides/kiln_preview.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kiln-AI/Kiln/06b5dbe8560ee8cdfee1bbcc3ddfb0f5840a4508/guides/kiln_preview.avif -------------------------------------------------------------------------------- /guides/kiln_preview.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kiln-AI/Kiln/06b5dbe8560ee8cdfee1bbcc3ddfb0f5840a4508/guides/kiln_preview.gif -------------------------------------------------------------------------------- /libs/core/.python-version: -------------------------------------------------------------------------------- 1 | 3.13 2 | -------------------------------------------------------------------------------- /libs/core/LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | This license applies only to the software in the libs/core directory. 4 | 5 | ======================================================= 6 | 7 | Copyright 2024 - Chesterfield Laboratories Inc. 8 | 9 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /libs/core/kiln_ai/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. include:: ../README.md 3 | """ 4 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/adapters/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | # Adapters 3 | 4 | Adapters are used to connect Kiln to external systems, or to add new functionality to Kiln. 5 | 6 | Model adapters are used to call AI models, like Ollama, OpenAI, etc. 7 | 8 | The ml_model_list submodule contains a list of models that can be used for machine learning tasks. More can easily be added, but we keep a list here of models that are known to work well with Kiln's structured data and tool calling systems. 9 | 10 | The prompt_builders submodule contains classes that build prompts for use with the AI agents. 11 | 12 | The repair submodule contains an adapter for the repair task. 13 | 14 | The parser submodule contains parsers for the output of the AI models. 15 | 16 | The eval submodule contains the code for evaluating the performance of a model. 17 | """ 18 | 19 | from . import ( 20 | data_gen, 21 | eval, 22 | fine_tune, 23 | ml_model_list, 24 | model_adapters, 25 | prompt_builders, 26 | repair, 27 | ) 28 | 29 | __all__ = [ 30 | "model_adapters", 31 | "data_gen", 32 | "fine_tune", 33 | "ml_model_list", 34 | "prompt_builders", 35 | "repair", 36 | "eval", 37 | ] 38 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/adapters/data_gen/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | # Data Generation 3 | 4 | A task to generate synthetic data for Kiln Tasks. This generates the inputs, which then can be run through the task. 5 | 6 | Optional human guidance can be provided to guide the generation process. 7 | """ 8 | 9 | from . import data_gen_task 10 | 11 | __all__ = ["data_gen_task"] 12 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/adapters/eval/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | # Evals 3 | 4 | This module contains the code for evaluating the performance of a model. 5 | 6 | The submodules contain: 7 | 8 | - BaseEval: each eval technique implements this interface. 9 | - G-Eval: an eval implementation, that implements G-Eval and LLM as Judge. 10 | - EvalRunner: a class that runs an full evaluation (many smaller evals jobs). Includes async parallel processing, and the ability to restart where it left off. 11 | - EvalRegistry: a registry for all eval implementations. 12 | 13 | The datamodel for Evals is in the `kiln_ai.datamodel.eval` module. 14 | """ 15 | 16 | from . import ( 17 | base_eval, 18 | eval_runner, 19 | g_eval, 20 | registry, 21 | ) 22 | 23 | __all__ = [ 24 | "base_eval", 25 | "eval_runner", 26 | "g_eval", 27 | "registry", 28 | ] 29 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/adapters/eval/registry.py: -------------------------------------------------------------------------------- 1 | from kiln_ai.adapters.eval.base_eval import BaseEval 2 | from kiln_ai.adapters.eval.g_eval import GEval 3 | from kiln_ai.datamodel.eval import EvalConfigType 4 | from kiln_ai.utils.exhaustive_error import raise_exhaustive_enum_error 5 | 6 | 7 | def eval_adapter_from_type(eval_config_type: EvalConfigType) -> type[BaseEval]: 8 | match eval_config_type: 9 | case EvalConfigType.g_eval: 10 | return GEval 11 | case EvalConfigType.llm_as_judge: 12 | # Also implemented by GEval 13 | return GEval 14 | case _: 15 | # type checking will catch missing cases 16 | raise_exhaustive_enum_error(eval_config_type) 17 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/adapters/fine_tune/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | # Fine-Tuning 3 | 4 | A set of classes for fine-tuning models. 5 | """ 6 | 7 | from . import base_finetune, dataset_formatter, finetune_registry, openai_finetune 8 | 9 | __all__ = [ 10 | "base_finetune", 11 | "openai_finetune", 12 | "dataset_formatter", 13 | "finetune_registry", 14 | ] 15 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/adapters/fine_tune/finetune_registry.py: -------------------------------------------------------------------------------- 1 | from typing import Type 2 | 3 | from kiln_ai.adapters.fine_tune.base_finetune import BaseFinetuneAdapter 4 | from kiln_ai.adapters.fine_tune.fireworks_finetune import FireworksFinetune 5 | from kiln_ai.adapters.fine_tune.openai_finetune import OpenAIFinetune 6 | from kiln_ai.adapters.fine_tune.together_finetune import TogetherFinetune 7 | from kiln_ai.adapters.fine_tune.vertex_finetune import VertexFinetune 8 | from kiln_ai.adapters.ml_model_list import ModelProviderName 9 | 10 | finetune_registry: dict[ModelProviderName, Type[BaseFinetuneAdapter]] = { 11 | ModelProviderName.openai: OpenAIFinetune, 12 | ModelProviderName.fireworks_ai: FireworksFinetune, 13 | ModelProviderName.together_ai: TogetherFinetune, 14 | ModelProviderName.vertex: VertexFinetune, 15 | } 16 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/adapters/model_adapters/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | # Model Adapters 3 | 4 | Model adapters are used to call AI models, like Ollama, OpenAI, etc. 5 | 6 | """ 7 | 8 | from . import ( 9 | base_adapter, 10 | litellm_adapter, 11 | ) 12 | 13 | __all__ = [ 14 | "base_adapter", 15 | "litellm_adapter", 16 | ] 17 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/adapters/model_adapters/litellm_config.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass, field 2 | 3 | 4 | @dataclass 5 | class LiteLlmConfig: 6 | model_name: str 7 | provider_name: str 8 | # If set, over rides the provider-name based URL from litellm 9 | base_url: str | None = None 10 | # Headers to send with every request 11 | default_headers: dict[str, str] | None = None 12 | # Extra body to send with every request 13 | additional_body_options: dict[str, str] = field(default_factory=dict) 14 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/adapters/parsers/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | # Parsers 3 | 4 | Parsing utilities for JSON and models with custom output formats (R1, etc.) 5 | 6 | """ 7 | 8 | from . import base_parser, json_parser, r1_parser 9 | 10 | __all__ = ["r1_parser", "base_parser", "json_parser"] 11 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/adapters/parsers/base_parser.py: -------------------------------------------------------------------------------- 1 | from kiln_ai.adapters.run_output import RunOutput 2 | 3 | 4 | class BaseParser: 5 | def parse_output(self, original_output: RunOutput) -> RunOutput: 6 | """ 7 | Method for parsing the output of a model. Typically overridden by subclasses. 8 | """ 9 | return original_output 10 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/adapters/parsers/json_parser.py: -------------------------------------------------------------------------------- 1 | import json 2 | from typing import Any, Dict 3 | 4 | 5 | def parse_json_string(json_string: str) -> Dict[str, Any]: 6 | """ 7 | Parse a JSON string into a dictionary. Handles multiple formats: 8 | - Plain JSON 9 | - JSON wrapped in ```json code blocks 10 | - JSON wrapped in ``` code blocks 11 | 12 | Args: 13 | json_string: String containing JSON data, possibly wrapped in code blocks 14 | 15 | Returns: 16 | Dict containing parsed JSON data 17 | 18 | Raises: 19 | ValueError: If JSON parsing fails 20 | """ 21 | # Remove code block markers if present 22 | cleaned_string = json_string.strip() 23 | if cleaned_string.startswith("```"): 24 | # Split by newlines and remove first/last lines if they contain ``` 25 | lines = cleaned_string.split("\n") 26 | if lines[0].startswith("```"): 27 | lines = lines[1:] 28 | if lines and lines[-1].strip() == "```": 29 | lines = lines[:-1] 30 | cleaned_string = "\n".join(lines) 31 | 32 | try: 33 | return json.loads(cleaned_string) 34 | except json.JSONDecodeError as e: 35 | raise ValueError( 36 | f"This task requires JSON output but the model didn't return valid JSON. Search 'Troubleshooting Structured Data Issues' in our docs for more information. The model produced the following: {cleaned_string}" 37 | ) from e 38 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/adapters/parsers/parser_registry.py: -------------------------------------------------------------------------------- 1 | from typing import Type 2 | 3 | from kiln_ai.adapters.ml_model_list import ModelParserID 4 | from kiln_ai.adapters.parsers.base_parser import BaseParser 5 | from kiln_ai.adapters.parsers.r1_parser import R1ThinkingParser 6 | from kiln_ai.utils.exhaustive_error import raise_exhaustive_enum_error 7 | 8 | 9 | def model_parser_from_id(parser_id: ModelParserID | None) -> BaseParser: 10 | """ 11 | Get a model parser from its ID. 12 | """ 13 | match parser_id: 14 | case None: 15 | return BaseParser() 16 | case ModelParserID.r1_thinking: 17 | return R1ThinkingParser() 18 | case ModelParserID.optional_r1_thinking: 19 | return R1ThinkingParser(allow_missing_thinking=True) 20 | case _: 21 | raise_exhaustive_enum_error(parser_id) 22 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/adapters/parsers/request_formatters.py: -------------------------------------------------------------------------------- 1 | import json 2 | from typing import Dict, Protocol 3 | 4 | from kiln_ai.adapters.ml_model_list import ModelFormatterID 5 | from kiln_ai.utils.exhaustive_error import raise_exhaustive_enum_error 6 | 7 | 8 | class RequestFormatter(Protocol): 9 | def format_input(self, original_input: Dict | str) -> Dict | str: 10 | """ 11 | Method for formatting the input to a model. 12 | """ 13 | ... 14 | 15 | 16 | class Qwen3StyleNoThinkFormatter: 17 | def format_input(self, original_input: Dict | str) -> Dict | str: 18 | """ 19 | Format the input to a model for Qwen3 /no_think instruction 20 | """ 21 | formatted_input = ( 22 | original_input 23 | if isinstance(original_input, str) 24 | else json.dumps(original_input, indent=2) 25 | ) 26 | 27 | return formatted_input + "\n\n/no_think" 28 | 29 | 30 | def request_formatter_from_id( 31 | formatter_id: ModelFormatterID, 32 | ) -> RequestFormatter: 33 | """ 34 | Get a model parser from its ID. 35 | """ 36 | match formatter_id: 37 | case ModelFormatterID.qwen3_style_no_think: 38 | return Qwen3StyleNoThinkFormatter() 39 | case _: 40 | raise_exhaustive_enum_error(formatter_id) 41 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/adapters/parsers/test_json_parser.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from kiln_ai.adapters.parsers.json_parser import parse_json_string 4 | 5 | 6 | def test_parse_plain_json(): 7 | json_str = '{"key": "value", "number": 42}' 8 | result = parse_json_string(json_str) 9 | assert result == {"key": "value", "number": 42} 10 | 11 | 12 | def test_parse_json_with_code_block(): 13 | json_str = """``` 14 | {"key": "value", "number": 42} 15 | ```""" 16 | result = parse_json_string(json_str) 17 | assert result == {"key": "value", "number": 42} 18 | 19 | 20 | def test_parse_json_with_language_block(): 21 | json_str = """```json 22 | {"key": "value", "number": 42} 23 | ```""" 24 | result = parse_json_string(json_str) 25 | assert result == {"key": "value", "number": 42} 26 | 27 | 28 | def test_parse_json_with_whitespace(): 29 | json_str = """ 30 | { 31 | "key": "value", 32 | "number": 42 33 | } 34 | """ 35 | result = parse_json_string(json_str) 36 | assert result == {"key": "value", "number": 42} 37 | 38 | 39 | def test_parse_invalid_json(): 40 | json_str = '{"key": "value", invalid}' 41 | with pytest.raises(ValueError) as exc_info: 42 | parse_json_string(json_str) 43 | assert ( 44 | "This task requires JSON output but the model didn't return valid JSON." 45 | in str(exc_info.value) 46 | ) 47 | 48 | 49 | def test_parse_empty_code_block(): 50 | json_str = """```json 51 | ```""" 52 | with pytest.raises(ValueError) as exc_info: 53 | parse_json_string(json_str) 54 | assert ( 55 | "This task requires JSON output but the model didn't return valid JSON." 56 | in str(exc_info.value) 57 | ) 58 | 59 | 60 | def test_parse_complex_json(): 61 | json_str = """```json 62 | { 63 | "string": "hello", 64 | "number": 42, 65 | "bool": true, 66 | "null": null, 67 | "array": [1, 2, 3], 68 | "nested": { 69 | "inner": "value" 70 | } 71 | } 72 | ```""" 73 | result = parse_json_string(json_str) 74 | assert result == { 75 | "string": "hello", 76 | "number": 42, 77 | "bool": True, 78 | "null": None, 79 | "array": [1, 2, 3], 80 | "nested": {"inner": "value"}, 81 | } 82 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/adapters/parsers/test_parser_registry.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from kiln_ai.adapters.ml_model_list import ModelParserID 4 | from kiln_ai.adapters.parsers.base_parser import BaseParser 5 | from kiln_ai.adapters.parsers.parser_registry import model_parser_from_id 6 | from kiln_ai.adapters.parsers.r1_parser import R1ThinkingParser 7 | 8 | 9 | def test_model_parser_from_id_invalid(): 10 | """Test that invalid parser ID raises ValueError.""" 11 | 12 | # Create a mock enum value that isn't handled 13 | class MockModelParserID: 14 | mock_value = "mock_value" 15 | 16 | with pytest.raises(ValueError) as exc_info: 17 | model_parser_from_id(MockModelParserID.mock_value) # type: ignore 18 | 19 | assert "Unhandled enum value" in str(exc_info.value) 20 | 21 | 22 | @pytest.mark.parametrize( 23 | "parser_id,expected_class", 24 | [ 25 | (None, BaseParser), 26 | (ModelParserID.r1_thinking, R1ThinkingParser), 27 | ], 28 | ) 29 | def test_model_parser_from_id_parametrized(parser_id, expected_class): 30 | """Test all valid parser IDs using parametrize.""" 31 | parser = model_parser_from_id(parser_id) 32 | assert isinstance(parser, expected_class) 33 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/adapters/parsers/test_request_formatters.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from kiln_ai.adapters.ml_model_list import ModelFormatterID 4 | from kiln_ai.adapters.parsers.request_formatters import ( 5 | Qwen3StyleNoThinkFormatter, 6 | request_formatter_from_id, 7 | ) 8 | 9 | 10 | @pytest.fixture 11 | def qwen_formatter(): 12 | return Qwen3StyleNoThinkFormatter() 13 | 14 | 15 | def test_qwen_formatter_string_input(qwen_formatter): 16 | input_text = "Hello world" 17 | formatted = qwen_formatter.format_input(input_text) 18 | assert formatted == "Hello world\n\n/no_think" 19 | 20 | 21 | def test_qwen_formatter_dict_input(qwen_formatter): 22 | input_dict = {"key": "value", "nested": {"inner": "data"}} 23 | formatted = qwen_formatter.format_input(input_dict) 24 | expected = """{ 25 | "key": "value", 26 | "nested": { 27 | "inner": "data" 28 | } 29 | } 30 | 31 | /no_think""" 32 | assert formatted == expected 33 | 34 | 35 | def test_qwen_formatter_empty_input(qwen_formatter): 36 | # Test empty string 37 | assert qwen_formatter.format_input("") == "\n\n/no_think" 38 | 39 | # Test empty dict 40 | assert qwen_formatter.format_input({}) == "{}\n\n/no_think" 41 | 42 | 43 | def test_qwen_formatter_special_characters(qwen_formatter): 44 | input_text = "Special chars: !@#$%^&*()_+思" 45 | formatted = qwen_formatter.format_input(input_text) 46 | assert formatted == "Special chars: !@#$%^&*()_+思\n\n/no_think" 47 | 48 | 49 | def test_qwen_formatter_multiline_string(qwen_formatter): 50 | input_text = """Line 1 51 | Line 2 52 | Line 3""" 53 | formatted = qwen_formatter.format_input(input_text) 54 | assert ( 55 | formatted 56 | == """Line 1 57 | Line 2 58 | Line 3 59 | 60 | /no_think""" 61 | ) 62 | 63 | 64 | def test_request_formatter_factory(): 65 | # Test valid formatter ID 66 | formatter = request_formatter_from_id(ModelFormatterID.qwen3_style_no_think) 67 | assert isinstance(formatter, Qwen3StyleNoThinkFormatter) 68 | 69 | # Test that the formatter works 70 | assert formatter.format_input("test") == "test\n\n/no_think" 71 | 72 | 73 | def test_request_formatter_factory_invalid_id(): 74 | # Test with an invalid enum value by using a string that doesn't exist in the enum 75 | with pytest.raises(ValueError, match="Unhandled enum value"): 76 | request_formatter_from_id("invalid_formatter_id") # type: ignore 77 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/adapters/repair/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | # Repair 3 | 4 | A task to repair a task output. Used to fix up task outputs that don't meet the requirements. 5 | 6 | Not intended to be persisted into a project file. 7 | """ 8 | 9 | from . import repair_task 10 | 11 | __all__ = ["repair_task"] 12 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/adapters/run_output.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Dict 3 | 4 | from litellm.types.utils import ChoiceLogprobs 5 | 6 | 7 | @dataclass 8 | class RunOutput: 9 | output: Dict | str 10 | intermediate_outputs: Dict[str, str] | None 11 | output_logprobs: ChoiceLogprobs | None = None 12 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/adapters/test_ollama_tools.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from kiln_ai.adapters.ollama_tools import ( 4 | OllamaConnection, 5 | ollama_model_installed, 6 | parse_ollama_tags, 7 | ) 8 | 9 | 10 | def test_parse_ollama_tags_no_models(): 11 | json_response = '{"models":[{"name":"scosman_net","model":"scosman_net:latest"},{"name":"phi3.5:latest","model":"phi3.5:latest","modified_at":"2024-10-02T12:04:35.191519822-04:00","size":2176178843,"digest":"61819fb370a3c1a9be6694869331e5f85f867a079e9271d66cb223acb81d04ba","details":{"parent_model":"","format":"gguf","family":"phi3","families":["phi3"],"parameter_size":"3.8B","quantization_level":"Q4_0"}},{"name":"gemma2:2b","model":"gemma2:2b","modified_at":"2024-09-09T16:46:38.64348929-04:00","size":1629518495,"digest":"8ccf136fdd5298f3ffe2d69862750ea7fb56555fa4d5b18c04e3fa4d82ee09d7","details":{"parent_model":"","format":"gguf","family":"gemma2","families":["gemma2"],"parameter_size":"2.6B","quantization_level":"Q4_0"}},{"name":"llama3.1:latest","model":"llama3.1:latest","modified_at":"2024-09-01T17:19:43.481523695-04:00","size":4661230720,"digest":"f66fc8dc39ea206e03ff6764fcc696b1b4dfb693f0b6ef751731dd4e6269046e","details":{"parent_model":"","format":"gguf","family":"llama","families":["llama"],"parameter_size":"8.0B","quantization_level":"Q4_0"}}]}' 12 | tags = json.loads(json_response) 13 | conn = parse_ollama_tags(tags) 14 | assert "phi3.5:latest" in conn.supported_models 15 | assert "gemma2:2b" in conn.supported_models 16 | assert "llama3.1:latest" in conn.supported_models 17 | assert "scosman_net:latest" in conn.untested_models 18 | 19 | 20 | def test_parse_ollama_tags_only_untested_models(): 21 | json_response = '{"models":[{"name":"scosman_net","model":"scosman_net:latest"}]}' 22 | tags = json.loads(json_response) 23 | conn = parse_ollama_tags(tags) 24 | assert conn.supported_models == [] 25 | assert conn.untested_models == ["scosman_net:latest"] 26 | 27 | 28 | def test_ollama_model_installed(): 29 | conn = OllamaConnection( 30 | supported_models=["phi3.5:latest", "gemma2:2b", "llama3.1:latest"], 31 | message="Connected", 32 | untested_models=["scosman_net:latest"], 33 | ) 34 | assert ollama_model_installed(conn, "phi3.5:latest") 35 | assert ollama_model_installed(conn, "phi3.5") 36 | assert ollama_model_installed(conn, "gemma2:2b") 37 | assert ollama_model_installed(conn, "llama3.1:latest") 38 | assert ollama_model_installed(conn, "llama3.1") 39 | assert ollama_model_installed(conn, "scosman_net:latest") 40 | assert ollama_model_installed(conn, "scosman_net") 41 | assert not ollama_model_installed(conn, "unknown_model") 42 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/datamodel/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | See our docs for details about our datamodel classes and hierarchy: 3 | 4 | Developer docs: https://kiln-ai.github.io/Kiln/kiln_core_docs/kiln_ai.html 5 | 6 | User docs: https://docs.getkiln.ai/developers/kiln-datamodel 7 | """ 8 | 9 | # This component uses "flat" imports so we don't have too much internal structure exposed in the API. 10 | # for example you can just `from datamodel import Task, Project` instead of `from datamodel.task import Task; from datamodel.project import Project` 11 | 12 | from __future__ import annotations 13 | 14 | from kiln_ai.datamodel import dataset_split, eval, strict_mode 15 | from kiln_ai.datamodel.datamodel_enums import ( 16 | FinetuneDataStrategy, 17 | FineTuneStatusType, 18 | Priority, 19 | StructuredOutputMode, 20 | TaskOutputRatingType, 21 | ) 22 | from kiln_ai.datamodel.dataset_split import ( 23 | DatasetSplit, 24 | DatasetSplitDefinition, 25 | ) 26 | from kiln_ai.datamodel.finetune import ( 27 | Finetune, 28 | ) 29 | from kiln_ai.datamodel.project import Project 30 | from kiln_ai.datamodel.prompt import BasePrompt, Prompt 31 | from kiln_ai.datamodel.prompt_id import ( 32 | PromptGenerators, 33 | PromptId, 34 | prompt_generator_values, 35 | ) 36 | from kiln_ai.datamodel.task import Task, TaskRequirement 37 | from kiln_ai.datamodel.task_output import ( 38 | DataSource, 39 | DataSourceProperty, 40 | DataSourceType, 41 | RequirementRating, 42 | TaskOutput, 43 | TaskOutputRating, 44 | ) 45 | from kiln_ai.datamodel.task_run import ( 46 | TaskRun, 47 | Usage, 48 | ) 49 | 50 | __all__ = [ 51 | "strict_mode", 52 | "dataset_split", 53 | "eval", 54 | "Task", 55 | "Project", 56 | "TaskRun", 57 | "TaskOutput", 58 | "Priority", 59 | "DataSource", 60 | "DataSourceType", 61 | "DataSourceProperty", 62 | "Finetune", 63 | "FineTuneStatusType", 64 | "TaskOutputRatingType", 65 | "TaskRequirement", 66 | "DatasetSplitDefinition", 67 | "DatasetSplit", 68 | "RequirementRating", 69 | "TaskRequirement", 70 | "BasePrompt", 71 | "Prompt", 72 | "TaskOutputRating", 73 | "StructuredOutputMode", 74 | "FinetuneDataStrategy", 75 | "PromptId", 76 | "PromptGenerators", 77 | "prompt_generator_values", 78 | "Usage", 79 | ] 80 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/datamodel/project.py: -------------------------------------------------------------------------------- 1 | from pydantic import Field 2 | 3 | from kiln_ai.datamodel.basemodel import NAME_FIELD, KilnParentModel 4 | from kiln_ai.datamodel.task import Task 5 | 6 | 7 | class Project(KilnParentModel, parent_of={"tasks": Task}): 8 | """ 9 | A collection of related tasks. 10 | 11 | Projects organize tasks into logical groups and provide high-level descriptions 12 | of the overall goals. 13 | """ 14 | 15 | name: str = NAME_FIELD 16 | description: str | None = Field( 17 | default=None, 18 | description="A description of the project for you and your team. Will not be used in prompts/training/validation.", 19 | ) 20 | 21 | # Needed for typechecking. TODO P2: fix this in KilnParentModel 22 | def tasks(self) -> list[Task]: 23 | return super().tasks() # type: ignore 24 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/datamodel/prompt.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel, Field 2 | 3 | from kiln_ai.datamodel.basemodel import NAME_FIELD, KilnParentedModel 4 | 5 | 6 | class BasePrompt(BaseModel): 7 | """ 8 | A prompt for a task. This is the basic data storage format which can be used throughout a project. 9 | 10 | The "Prompt" model name is reserved for the custom prompts parented by a task. 11 | """ 12 | 13 | name: str = NAME_FIELD 14 | description: str | None = Field( 15 | default=None, 16 | description="A more detailed description of the prompt.", 17 | ) 18 | generator_id: str | None = Field( 19 | default=None, 20 | description="The id of the generator that created this prompt.", 21 | ) 22 | prompt: str = Field( 23 | description="The prompt for the task.", 24 | min_length=1, 25 | ) 26 | chain_of_thought_instructions: str | None = Field( 27 | default=None, 28 | description="Instructions for the model 'thinking' about the requirement prior to answering. Used for chain of thought style prompting. COT will not be used unless this is provided.", 29 | ) 30 | 31 | 32 | class Prompt(KilnParentedModel, BasePrompt): 33 | """ 34 | A prompt for a task. This is the custom prompt parented by a task. 35 | """ 36 | 37 | pass 38 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/datamodel/prompt_id.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | from typing import Annotated 3 | 4 | from pydantic import AfterValidator 5 | 6 | 7 | # Generators that can take any task and build a prompt 8 | class PromptGenerators(str, Enum): 9 | SIMPLE = "simple_prompt_builder" 10 | MULTI_SHOT = "multi_shot_prompt_builder" 11 | FEW_SHOT = "few_shot_prompt_builder" 12 | REPAIRS = "repairs_prompt_builder" 13 | SIMPLE_CHAIN_OF_THOUGHT = "simple_chain_of_thought_prompt_builder" 14 | FEW_SHOT_CHAIN_OF_THOUGHT = "few_shot_chain_of_thought_prompt_builder" 15 | MULTI_SHOT_CHAIN_OF_THOUGHT = "multi_shot_chain_of_thought_prompt_builder" 16 | SHORT = "short_prompt_builder" 17 | 18 | 19 | prompt_generator_values = [pg.value for pg in PromptGenerators] 20 | 21 | 22 | PromptId = Annotated[ 23 | str, 24 | AfterValidator(lambda v: _check_prompt_id(v)), 25 | ] 26 | """ 27 | A pydantic type that validates strings containing a valid prompt ID. 28 | 29 | Prompt IDs can be one of: 30 | - A saved prompt ID 31 | - A fine-tune prompt ID 32 | - A task run config ID 33 | - A prompt generator name 34 | """ 35 | 36 | 37 | def _check_prompt_id(id: str) -> str: 38 | """ 39 | Check that the prompt ID is valid. 40 | """ 41 | if id in prompt_generator_values: 42 | return id 43 | 44 | if id.startswith("id::"): 45 | # check it has 4 parts divided by :: -- 'id::project_id::task_id::prompt_id' 46 | parts = id.split("::") 47 | if len(parts) != 2 or len(parts[1]) == 0: 48 | raise ValueError( 49 | f"Invalid saved prompt ID: {id}. Expected format: 'id::[prompt_id]'." 50 | ) 51 | return id 52 | 53 | if id.startswith("task_run_config::"): 54 | # check it had a eval_id after the :: -- 'project_id::task_id::task_run_config_id' 55 | parts = id.split("::") 56 | if len(parts) != 4: 57 | raise ValueError( 58 | f"Invalid task run config prompt ID: {id}. Expected format: 'task_run_config::[project_id]::[task_id]::[task_run_config_id]'." 59 | ) 60 | return id 61 | 62 | if id.startswith("fine_tune_prompt::"): 63 | # check it had a fine_tune_id after the :: -- 'fine_tune_prompt::fine_tune_id' 64 | fine_tune_id = id[18:] 65 | if len(fine_tune_id) == 0: 66 | raise ValueError( 67 | f"Invalid fine-tune prompt ID: {id}. Expected format: 'fine_tune_prompt::[fine_tune_id]'." 68 | ) 69 | return id 70 | 71 | raise ValueError(f"Invalid prompt ID: {id}") 72 | 73 | 74 | def is_frozen_prompt(id: PromptId) -> bool: 75 | """ 76 | Check if the prompt ID is a frozen prompt. 77 | """ 78 | if id.startswith("id::"): 79 | return True 80 | if id.startswith("task_run_config::"): 81 | return True 82 | if id.startswith("fine_tune_prompt::"): 83 | return True 84 | return False 85 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/datamodel/registry.py: -------------------------------------------------------------------------------- 1 | from kiln_ai.datamodel import Project 2 | from kiln_ai.utils.config import Config 3 | 4 | 5 | def all_projects() -> list[Project]: 6 | project_paths = Config.shared().projects 7 | if project_paths is None: 8 | return [] 9 | projects = [] 10 | for project_path in project_paths: 11 | try: 12 | projects.append(Project.load_from_file(project_path)) 13 | except Exception: 14 | # deleted files are possible continue with the rest 15 | continue 16 | return projects 17 | 18 | 19 | def project_from_id(project_id: str) -> Project | None: 20 | project_paths = Config.shared().projects 21 | if project_paths is not None: 22 | for project_path in project_paths: 23 | try: 24 | project = Project.load_from_file(project_path) 25 | if project.id == project_id: 26 | return project 27 | except Exception: 28 | # deleted files are possible continue with the rest 29 | continue 30 | 31 | return None 32 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/datamodel/strict_mode.py: -------------------------------------------------------------------------------- 1 | """ 2 | Strict mode is a feature that enables extra validations that we want to enforce in Kiln App, ensuring everything follows the ideal schema. 3 | 4 | It's off by default when used through the library. Enable it by calling `set_strict_mode(True)`. 5 | """ 6 | 7 | # We want to be hard on ourselves for data completeness generated by the Kiln App, but don't want to make it hard for users to use the datamodel/library. 8 | # Strict mode enables extra validations that we want to enforce in Kiln App (and any other client that wants best practices), but not in the library (unless they opt in) 9 | _strict_mode: bool = False 10 | 11 | 12 | def strict_mode() -> bool: 13 | """ 14 | Get the current strict mode setting. 15 | """ 16 | return _strict_mode 17 | 18 | 19 | def set_strict_mode(value: bool) -> None: 20 | """ 21 | Set the strict mode setting. 22 | """ 23 | global _strict_mode 24 | _strict_mode = value 25 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/datamodel/test_registry.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import Mock, patch 2 | 3 | import pytest 4 | 5 | from kiln_ai.datamodel import Project 6 | from kiln_ai.datamodel.registry import all_projects, project_from_id 7 | 8 | 9 | @pytest.fixture 10 | def mock_config(): 11 | with patch("kiln_ai.datamodel.registry.Config") as mock: 12 | config_instance = Mock() 13 | mock.shared.return_value = config_instance 14 | yield config_instance 15 | 16 | 17 | @pytest.fixture 18 | def mock_project(): 19 | def create_mock_project(project_id: str = "test-id"): 20 | project = Mock(spec=Project) 21 | project.id = project_id 22 | return project 23 | 24 | return create_mock_project 25 | 26 | 27 | def test_all_projects_empty(mock_config): 28 | mock_config.projects = None 29 | assert all_projects() == [] 30 | 31 | 32 | def test_all_projects_success(mock_config, mock_project): 33 | mock_config.projects = ["path1", "path2"] 34 | 35 | project1 = mock_project("project1") 36 | project2 = mock_project("project2") 37 | 38 | with patch("kiln_ai.datamodel.Project.load_from_file") as mock_load: 39 | mock_load.side_effect = [project1, project2] 40 | 41 | result = all_projects() 42 | 43 | assert len(result) == 2 44 | assert result[0] == project1 45 | assert result[1] == project2 46 | mock_load.assert_any_call("path1") 47 | mock_load.assert_any_call("path2") 48 | 49 | 50 | def test_all_projects_with_errors(mock_config, mock_project): 51 | mock_config.projects = ["path1", "path2", "path3"] 52 | 53 | project1 = mock_project("project1") 54 | project3 = mock_project("project3") 55 | 56 | with patch("kiln_ai.datamodel.Project.load_from_file") as mock_load: 57 | mock_load.side_effect = [project1, Exception("File not found"), project3] 58 | 59 | result = all_projects() 60 | 61 | assert len(result) == 2 62 | assert result[0] == project1 63 | assert result[1] == project3 64 | 65 | 66 | def test_project_from_id_not_found(mock_config): 67 | mock_config.projects = None 68 | assert project_from_id("any-id") is None 69 | 70 | 71 | def test_project_from_id_success(mock_config, mock_project): 72 | mock_config.projects = ["path1", "path2"] 73 | 74 | project1 = mock_project("project1") 75 | project2 = mock_project("project2") 76 | 77 | with patch("kiln_ai.datamodel.Project.load_from_file") as mock_load: 78 | mock_load.side_effect = [project1, project2] 79 | 80 | result = project_from_id("project2") 81 | 82 | assert result == project2 83 | 84 | 85 | def test_project_from_id_with_errors(mock_config, mock_project): 86 | mock_config.projects = ["path1", "path2", "path3"] 87 | 88 | project1 = mock_project("project1") 89 | project3 = mock_project("target-id") 90 | 91 | with patch("kiln_ai.datamodel.Project.load_from_file") as mock_load: 92 | mock_load.side_effect = [project1, Exception("File not found"), project3] 93 | 94 | result = project_from_id("target-id") 95 | 96 | assert result == project3 97 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/utils/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | # Utils 3 | 4 | Misc utilities used in the kiln_ai library. 5 | """ 6 | 7 | from . import config, formatting 8 | 9 | __all__ = [ 10 | "config", 11 | "formatting", 12 | ] 13 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/utils/exhaustive_error.py: -------------------------------------------------------------------------------- 1 | from typing import NoReturn 2 | 3 | 4 | # Weird trick, but passing a enum to NoReturn triggers the type checker to complain unless all values are handled. 5 | def raise_exhaustive_enum_error(value: NoReturn) -> NoReturn: 6 | raise ValueError(f"Unhandled enum value: {value}") 7 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/utils/formatting.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | 4 | def snake_case(s: str) -> str: 5 | return re.sub(r"(? str: 112 | """ 113 | Generates a memorable two-word name combining a random adjective and noun. 114 | 115 | Returns: 116 | str: A memorable name in the format "Adjective Noun" 117 | 118 | Example: 119 | >>> generate_memorable_name() 120 | 'Cosmic Dragon' 121 | """ 122 | adjective = choice(ADJECTIVES) 123 | noun = choice(NOUNS) 124 | 125 | return f"{adjective} {noun}" 126 | -------------------------------------------------------------------------------- /libs/core/kiln_ai/utils/test_name_geneator.py: -------------------------------------------------------------------------------- 1 | from kiln_ai.utils.name_generator import ADJECTIVES, NOUNS, generate_memorable_name 2 | 3 | 4 | def test_generate_memorable_name_format(): 5 | """Test that generated name follows the expected format.""" 6 | name = generate_memorable_name() 7 | 8 | # Check that we get exactly two words 9 | words = name.split() 10 | assert len(words) == 2 11 | 12 | # Check that first word is an adjective and second word is a noun 13 | assert words[0] in ADJECTIVES 14 | assert words[1] in NOUNS 15 | 16 | 17 | def test_generate_memorable_name_randomness(): 18 | """Test that the function generates different names.""" 19 | names = {generate_memorable_name() for _ in range(100)} 20 | 21 | # With 50 adjectives and 50 nouns, we should get multiple unique combinations 22 | # in 100 tries. Using 50 as a reasonable lower bound. 23 | assert len(names) > 50 24 | 25 | 26 | def test_generate_memorable_name_string_type(): 27 | """Test that the generated name is a string.""" 28 | name = generate_memorable_name() 29 | assert isinstance(name, str) 30 | 31 | 32 | def test_word_lists_not_empty(): 33 | """Test that our word lists contain entries.""" 34 | assert len(ADJECTIVES) > 0 35 | assert len(NOUNS) > 0 36 | 37 | 38 | def test_word_lists_are_strings(): 39 | """Test that all entries in word lists are strings.""" 40 | assert all(isinstance(word, str) for word in ADJECTIVES) 41 | assert all(isinstance(word, str) for word in NOUNS) 42 | 43 | 44 | def test_word_lists_no_duplicates(): 45 | """Test that word lists don't contain duplicates.""" 46 | assert len(ADJECTIVES) == len(set(ADJECTIVES)) 47 | assert len(NOUNS) == len(set(NOUNS)) 48 | -------------------------------------------------------------------------------- /libs/core/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "kiln-ai" 3 | version = "0.16.0" 4 | requires-python = ">=3.10" 5 | readme = "README.md" 6 | description = 'Kiln AI' 7 | authors = [ 8 | {name = "Steve Cosman, Chesterfield Laboratories Inc", email = "scosman@users.noreply.github.com"} 9 | ] 10 | classifiers = [ 11 | "License :: OSI Approved :: MIT License", 12 | "Intended Audience :: Developers", 13 | "Programming Language :: Python :: 3.10", 14 | "Programming Language :: Python :: 3.11", 15 | "Programming Language :: Python :: 3.12", 16 | "Programming Language :: Python :: 3.13", 17 | ] 18 | 19 | 20 | dependencies = [ 21 | "boto3>=1.37.10", 22 | "coverage>=7.6.4", 23 | "google-cloud-aiplatform>=1.84.0", 24 | "jsonschema>=4.23.0", 25 | "litellm>=1.67.0", 26 | "openai>=1.53.0", 27 | "pdoc>=15.0.0", 28 | "pydantic>=2.9.2", 29 | "pytest-benchmark>=5.1.0", 30 | "pytest-cov>=6.0.0", 31 | "pyyaml>=6.0.2", 32 | "together", 33 | "typing-extensions>=4.12.2", 34 | "vertexai>=1.43.0", 35 | ] 36 | 37 | [dependency-groups] 38 | dev = [ 39 | "isort>=5.13.2", 40 | "pyright==1.1.376", 41 | "pytest-asyncio>=0.24.0", 42 | "pytest>=8.3.3", 43 | "python-dotenv>=1.0.1", 44 | "ruff>=0.9.0", 45 | ] 46 | 47 | [build-system] 48 | requires = ["hatchling"] 49 | build-backend = "hatchling.build" 50 | 51 | [tool.hatch.metadata] 52 | requires-python = ">=3.10" 53 | 54 | [tool.uv.sources] 55 | together = { git = "https://github.com/scosman/together-python" } 56 | 57 | 58 | [project.urls] 59 | Homepage = "https://getkiln.ai" 60 | Repository = "https://github.com/Kiln-AI/kiln" 61 | Documentation = "https://kiln-ai.github.io/Kiln/kiln_core_docs/kiln_ai.html" 62 | Issues = "https://github.com/Kiln-AI/kiln/issues" 63 | 64 | -------------------------------------------------------------------------------- /libs/core/setup.cfg: -------------------------------------------------------------------------------- 1 | # empty, but needed by setuptools -------------------------------------------------------------------------------- /libs/server/.python-version: -------------------------------------------------------------------------------- 1 | 3.13 2 | -------------------------------------------------------------------------------- /libs/server/LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | This license applies only to the software in the libs/server directory. 3 | 4 | ======================================================= 5 | 6 | Copyright 2024 - Chesterfield Laboratories Inc. 7 | 8 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 11 | 12 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /libs/server/README.md: -------------------------------------------------------------------------------- 1 | # Kiln AI Server 2 | 3 | [![PyPI - Version](https://img.shields.io/pypi/v/kiln-server.svg)](https://pypi.org/project/kiln-server) 4 | [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/kiln-server.svg)](https://pypi.org/project/kiln-server) 5 | 6 | --- 7 | 8 | ## About Kiln AI 9 | 10 | Learn more about Kiln AI at [getkiln.ai](https://getkiln.ai) 11 | 12 | This package is the Kiln AI server package. There is also a separate desktop application and python library package. 13 | 14 | Github: [github.com/Kiln-AI/kiln](https://github.com/Kiln-AI/kiln) 15 | 16 | ## Installation 17 | 18 | ```console 19 | pip install kiln_server 20 | ``` 21 | 22 | ## API Docs 23 | 24 | Our OpenApi docs: [https://kiln-ai.github.io/Kiln/kiln_server_openapi_docs/index.html](https://kiln-ai.github.io/Kiln/kiln_server_openapi_docs/index.html) 25 | 26 | ## Running the server 27 | 28 | ```console 29 | python -m kiln_server.server 30 | ``` 31 | 32 | With auto-reload: 33 | 34 | ```console 35 | AUTO_RELOAD=true python -m kiln_server.server 36 | ``` 37 | 38 | ## Using the server in another FastAPI app 39 | 40 | See server.py for examples, but you can connect individual API endpoints to your app like this: 41 | 42 | ```python 43 | from kiln_server.project_api import connect_project_api 44 | 45 | app = FastAPI() 46 | connect_project_api(app) 47 | ``` 48 | -------------------------------------------------------------------------------- /libs/server/kiln_server/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kiln-AI/Kiln/06b5dbe8560ee8cdfee1bbcc3ddfb0f5840a4508/libs/server/kiln_server/__init__.py -------------------------------------------------------------------------------- /libs/server/kiln_server/generate_openapi.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from fastapi.openapi.utils import get_openapi 4 | 5 | from .server import app 6 | 7 | if __name__ == "__main__": 8 | with open("openapi.json", "w") as f: 9 | json.dump( 10 | get_openapi( 11 | title=app.title, 12 | version=app.version, 13 | openapi_version=app.openapi_version, 14 | description=app.description, 15 | routes=app.routes, 16 | ), 17 | f, 18 | ) 19 | -------------------------------------------------------------------------------- /libs/server/kiln_server/server.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import uvicorn 4 | from fastapi import FastAPI 5 | from fastapi.middleware.cors import CORSMiddleware 6 | 7 | from .custom_errors import connect_custom_errors 8 | from .project_api import connect_project_api 9 | from .prompt_api import connect_prompt_api 10 | from .run_api import connect_run_api 11 | from .task_api import connect_task_api 12 | 13 | 14 | def make_app(lifespan=None): 15 | app = FastAPI( 16 | title="Kiln AI Server", 17 | summary="A REST API for the Kiln AI datamodel.", 18 | description="Learn more about Kiln AI at https://github.com/kiln-ai/kiln", 19 | lifespan=lifespan, 20 | ) 21 | 22 | @app.get("/ping") 23 | def ping(): 24 | return "pong" 25 | 26 | connect_project_api(app) 27 | connect_task_api(app) 28 | connect_prompt_api(app) 29 | connect_run_api(app) 30 | connect_custom_errors(app) 31 | 32 | allowed_origins = [ 33 | "http://localhost:5173", 34 | "http://127.0.0.1:5173", 35 | "https://localhost:5173", 36 | "https://127.0.0.1:5173", 37 | ] 38 | 39 | app.add_middleware( 40 | CORSMiddleware, 41 | allow_credentials=True, 42 | allow_origins=allowed_origins, 43 | allow_methods=["*"], 44 | allow_headers=["*"], 45 | ) 46 | 47 | return app 48 | 49 | 50 | app = make_app() 51 | if __name__ == "__main__": 52 | auto_reload = os.environ.get("AUTO_RELOAD", "").lower() in ("true", "1", "yes") 53 | uvicorn.run( 54 | "kiln_server.server:app", 55 | host="127.0.0.1", 56 | port=8757, 57 | reload=auto_reload, 58 | ) 59 | -------------------------------------------------------------------------------- /libs/server/kiln_server/test_custom_error.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from fastapi import FastAPI 3 | from fastapi.testclient import TestClient 4 | from pydantic import BaseModel, Field 5 | 6 | from kiln_server.custom_errors import ( 7 | connect_custom_errors, 8 | format_error_loc, 9 | ) 10 | 11 | 12 | @pytest.fixture 13 | def app(): 14 | app = FastAPI() 15 | connect_custom_errors(app) 16 | 17 | class Item(BaseModel): 18 | name: str = Field(..., min_length=3) 19 | price: float = Field(..., gt=0) 20 | 21 | @app.post("/items") 22 | async def create_item(item: Item): 23 | return item 24 | 25 | return app 26 | 27 | 28 | @pytest.fixture 29 | def client(app): 30 | return TestClient(app) 31 | 32 | 33 | def test_validation_error_single_field(client): 34 | response = client.post("/items", json={"name": "ab", "price": 10}) 35 | assert response.status_code == 422 36 | res = response.json() 37 | assert res["message"] == "Name: String should have at least 3 characters" 38 | assert res["error_messages"] == [ 39 | "Name: String should have at least 3 characters", 40 | ] 41 | assert len(res["source_errors"]) == 1 42 | 43 | 44 | def test_validation_error_multiple_fields(client): 45 | response = client.post("/items", json={"name": "ab", "price": -5}) 46 | assert response.status_code == 422 47 | res = response.json() 48 | assert res["error_messages"] == [ 49 | "Name: String should have at least 3 characters", 50 | "Price: Input should be greater than 0", 51 | ] 52 | assert ( 53 | res["message"] 54 | == "Name: String should have at least 3 characters.\nPrice: Input should be greater than 0" 55 | ) 56 | assert len(res["source_errors"]) == 2 57 | 58 | 59 | def test_valid_input(client): 60 | response = client.post("/items", json={"name": "abc", "price": 10}) 61 | assert response.status_code == 200 62 | assert response.json() == {"name": "abc", "price": 10} 63 | 64 | 65 | def test_format_none(): 66 | assert format_error_loc(None) == "" 67 | 68 | 69 | def test_format_error_loc_empty(): 70 | assert format_error_loc(()) == "" 71 | 72 | 73 | def test_format_error_loc_single_string(): 74 | assert format_error_loc(("body",)) == "" 75 | 76 | 77 | def test_format_error_loc_multiple_strings(): 78 | assert format_error_loc(("body", "username")) == "Username" 79 | 80 | 81 | def test_format_error_loc_with_integer(): 82 | assert format_error_loc(("items", 0, "name")) == "Items[0].Name" 83 | 84 | 85 | def test_format_error_loc_mixed_types(): 86 | assert format_error_loc(("query", "filter", 2, "value")) == "Query.Filter[2].Value" 87 | 88 | 89 | def test_format_error_loc_with_none(): 90 | assert format_error_loc(("container", None, "field")) == "Container.Field" 91 | 92 | 93 | def test_format_error_loc_with_empty_string(): 94 | assert format_error_loc(("container", "", "field")) == "Container.Field" 95 | -------------------------------------------------------------------------------- /libs/server/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "kiln-server" 3 | version = "0.16.0" 4 | requires-python = ">=3.10" 5 | description = 'Kiln AI Server' 6 | readme = "README.md" 7 | license = {file = "LICENSE.txt"} 8 | authors = [ 9 | {name = "Steve Cosman, Chesterfield Laboratories Inc", email = "scosman@users.noreply.github.com"} 10 | ] 11 | classifiers = [ 12 | "Intended Audience :: Developers", 13 | "License :: OSI Approved :: MIT License", 14 | "Programming Language :: Python :: 3.10", 15 | "Programming Language :: Python :: 3.11", 16 | "Programming Language :: Python :: 3.12", 17 | "Programming Language :: Python :: 3.13", 18 | ] 19 | 20 | dependencies = [ 21 | "fastapi>=0.115.4", 22 | "httpx>=0.27.2", 23 | "kiln-ai>=0.11.1", 24 | "pydantic>=2.9.2", 25 | "python-dotenv>=1.0.1", 26 | "python-multipart>=0.0.20", 27 | "uvicorn>=0.32.0", 28 | ] 29 | 30 | [dependency-groups] 31 | dev = [ 32 | "isort>=5.13.2", 33 | "pyright==1.1.376", 34 | "pytest-asyncio>=0.24.0", 35 | "pytest>=8.3.3", 36 | "python-dotenv>=1.0.1", 37 | "ruff>=0.9.0", 38 | ] 39 | 40 | 41 | [tool.uv.sources] 42 | kiln-ai = { workspace = true } 43 | 44 | [build-system] 45 | requires = ["hatchling"] 46 | build-backend = "hatchling.build" 47 | 48 | [tool.hatch.metadata] 49 | requires-python = ">=3.10" 50 | 51 | [project.urls] 52 | Homepage = "https://getkiln.ai" 53 | Repository = "https://github.com/Kiln-AI/kiln" 54 | Documentation = "https://github.com/Kiln-AI/kiln#readme" 55 | Issues = "https://github.com/Kiln-AI/kiln/issues" 56 | -------------------------------------------------------------------------------- /libs/server/setup.cfg: -------------------------------------------------------------------------------- 1 | # empty, but needed by setuptools -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "kiln-root" 3 | version = "0.1.0" 4 | description = "uv workspace project for Kiln AI. See kiln-ai package on pypi for our librabrary (libs/core)" 5 | readme = "README.md" 6 | requires-python = ">=3.10" 7 | 8 | dependencies = [ 9 | "kiln-ai==0.5.3", 10 | "kiln-server", 11 | "kiln-studio-desktop", 12 | ] 13 | 14 | [dependency-groups] 15 | dev = [ 16 | "isort>=5.13.2", 17 | "pyright==1.1.376", 18 | "pytest-asyncio>=0.24.0", 19 | "pytest>=8.3.3", 20 | "pytest-xdist>=3.5", 21 | "python-dotenv>=1.0.1", 22 | "ruff>=0.9.0", 23 | ] 24 | 25 | 26 | 27 | [tool.uv] 28 | # While Together depends on pyarrow, it doesn't need it, it's 80MB, and it doesn't work on MacOS 11 29 | override-dependencies = [ 30 | "pyarrow ; sys_platform == 'never'", 31 | ] 32 | 33 | [tool.uv.workspace] 34 | members = ["libs/core", "libs/server", "app/desktop"] 35 | 36 | [tool.uv.sources] 37 | kiln-server = { workspace = true } 38 | kiln-studio-desktop = { workspace = true } 39 | kiln-ai = { workspace = true } 40 | 41 | [tool.pyright] 42 | strictListInference = true 43 | reportMissingTypeArgument = true 44 | 45 | 46 | [tool.ruff] 47 | exclude = [ 48 | ] 49 | 50 | [tool.pytest.ini_options] 51 | addopts="-n auto" 52 | -------------------------------------------------------------------------------- /pyrightconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "exclude": ["**/test_*.py", "app/desktop/build/**", "app/web_ui/**", "**/.venv"] 3 | } 4 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | # Automtically detect async tests 2 | [pytest] 3 | asyncio_mode=auto 4 | # Needed to silence warning. See https://github.com/pytest-dev/pytest-asyncio/issues/924 5 | asyncio_default_fixture_loop_scope="function" 6 | 7 | markers = 8 | paid: marks tests as requring paid APIs. Not run by default, run with '--runpaid' option. 9 | ollama: marks tests as requring ollama server. Not run by default, run with '--ollama' option. 10 | 11 | # Enable parallel testing. Disabled for now as single tests are much faster without it on. It's enabled in checks.sh 12 | # addopts = -n auto -------------------------------------------------------------------------------- /start_env.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env zsh 2 | 3 | conda activate kiln 4 | 5 | --------------------------------------------------------------------------------