├── .github ├── CODEOWNERS ├── dependabot.yml └── workflows │ ├── release.yml │ ├── semantic.yml │ └── test.yml ├── .gitignore ├── .husky └── pre-commit ├── .prettierrc.json ├── .releaserc.json ├── README.md ├── __tests__ ├── README.md ├── helpers │ └── get-process-type.test.ts ├── language-model.test.ts └── preload.test.ts ├── end-to-end ├── package.json ├── src │ ├── main.ts │ └── preload.ts ├── static │ └── index.html └── tsconfig.json ├── package-lock.json ├── package.json ├── src ├── common │ └── ipc-channel-names.ts ├── constants.ts ├── helpers │ └── get-process-type.ts ├── index.ts ├── interfaces.ts ├── language-model.ts ├── main │ ├── index.ts │ └── register-ai-handlers.ts ├── preload │ └── index.ts ├── renderer │ └── index.ts └── utility │ ├── abortmanager.ts │ ├── call-ai-model-entry-point.ts │ ├── messages.ts │ └── utility-type-helpers.ts ├── tsconfig.json └── vitest.config.ts /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @electron/wg-ecosystem 2 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "monthly" 7 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | test: 10 | uses: ./.github/workflows/test.yml 11 | 12 | release: 13 | name: Release 14 | runs-on: ubuntu-latest 15 | needs: test 16 | environment: npm 17 | permissions: 18 | id-token: write # for CFA and npm provenance 19 | steps: 20 | - name: Checkout 21 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 22 | with: 23 | persist-credentials: false 24 | - name: Setup Node.js 25 | uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 26 | with: 27 | node-version: 20.x 28 | cache: 'npm' 29 | - name: Install 30 | run: npm ci 31 | - uses: continuousauth/action@4e8a2573eeb706f6d7300d6a9f3ca6322740b72d # v1.0.5 32 | timeout-minutes: 60 33 | with: 34 | project-id: ${{ secrets.CFA_PROJECT_ID }} 35 | secret: ${{ secrets.CFA_SECRET }} 36 | npm-token: ${{ secrets.NPM_TOKEN }} 37 | -------------------------------------------------------------------------------- /.github/workflows/semantic.yml: -------------------------------------------------------------------------------- 1 | name: "Check Semantic Commit" 2 | 3 | on: 4 | pull_request: 5 | types: 6 | - opened 7 | - edited 8 | - synchronize 9 | 10 | permissions: 11 | contents: read 12 | 13 | jobs: 14 | main: 15 | permissions: 16 | pull-requests: read # for amannn/action-semantic-pull-request to analyze PRs 17 | statuses: write # for amannn/action-semantic-pull-request to mark status of analyzed PR 18 | name: Validate PR Title 19 | runs-on: ubuntu-latest 20 | steps: 21 | - name: semantic-pull-request 22 | uses: amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017 # v5.5.3 23 | env: 24 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 25 | with: 26 | validateSingleCommit: false 27 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | schedule: 8 | - cron: '0 22 * * 3' 9 | workflow_call: 10 | 11 | permissions: 12 | contents: read 13 | 14 | jobs: 15 | test: 16 | name: Test 17 | strategy: 18 | fail-fast: false 19 | matrix: 20 | node-version: 21 | - '20.19' 22 | os: 23 | - macos-latest 24 | - ubuntu-latest 25 | - windows-latest 26 | runs-on: "${{ matrix.os }}" 27 | steps: 28 | - name: Checkout 29 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 30 | - name: Setup Node.js 31 | uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 32 | with: 33 | node-version: "${{ matrix.node-version }}" 34 | cache: 'npm' 35 | - name: Install dependencies 36 | run: npm ci 37 | - name: Test 38 | run: npm test 39 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .env 3 | .gclient_done 4 | **/.npmrc 5 | .tags* 6 | .vs/ 7 | .vscode/ 8 | *.log 9 | *.pyc 10 | *.sln 11 | *.swp 12 | *.VC.db 13 | *.VC.VC.opendb 14 | *.vcxproj 15 | *.vcxproj.filters 16 | *.vcxproj.user 17 | *.xcodeproj 18 | /.idea/ 19 | /dist/ 20 | node_modules/ 21 | SHASUMS256.txt 22 | compile_commands.json 23 | .envrc 24 | coverage 25 | 26 | .npmrc 27 | 28 | # Spec hash calculation 29 | spec/.hash 30 | 31 | # Eslint Cache 32 | .eslintcache* 33 | 34 | # If someone runs tsc this is where stuff will end up 35 | ts-gen 36 | 37 | out/ 38 | end-to-end/dist 39 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | . "$(dirname "$0")/_/husky.sh" 3 | npx lint-staged 4 | -------------------------------------------------------------------------------- /.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true 3 | } 4 | -------------------------------------------------------------------------------- /.releaserc.json: -------------------------------------------------------------------------------- 1 | { 2 | "plugins": [ 3 | "@semantic-release/commit-analyzer", 4 | "@semantic-release/release-notes-generator", 5 | "@continuous-auth/semantic-release-npm", 6 | "@semantic-release/github" 7 | ], 8 | "branches": ["main"] 9 | } 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # @electron/llm 2 | 3 | [![Test](https://github.com/electron/llm/actions/workflows/test.yml/badge.svg)](https://github.com/electron/llm/actions/workflows/test.yml) 4 | [![npm version](https://img.shields.io/npm/v/@electron/llm.svg)](https://npmjs.org/package/@electron/llm) 5 | 6 | This module makes it easy for developers to prototype local-first applications interacting with local large language models (LLMs), especially in chat contexts. 7 | 8 | It aims for an API surface similar to Chromium's `window.AI` API, except that you can supply any GGUF model. Under the hood, `@electron/llm` makes use of [node-llama-cpp](https://github.com/withcatai/node-llama-cpp). Our goal is to make use of native LLM capabilities in Electron _easier_ than if you consumed a Llama.cpp implementation directly - but not more feature-rich. Today, this module provides a reference implementation of `node-llama-cpp` that loads the model in a utility process and uses Chromium Mojo IPC pipes to efficiently facilitate streaming of responses between the utility process and renderers. If you're building an advanced app with LLM, you might want to use this module as a reference for your process architecture. 9 | 10 | `@electron/llm` is an experimental package. The Electron maintainers are exploring different ways to support and enable developers interested in running language models locally - and this package is just one of the potential avenues we're exploring. It's possible that we'll go in a different direction. Before using this package in a production app, be aware that you might have to migrate to something else! 11 | 12 | # Quick Start 13 | 14 | ## Installing the module, getting a model 15 | 16 | First, install the module in your Electron app: 17 | 18 | ``` 19 | npm i --save @electron/llm 20 | ``` 21 | 22 | Then, you need to load a model. The AI space seems to move at the speed of light, [so pick whichever GGUF model suits your purposes best](https://huggingface.co/models?library=gguf). If you just want to work with a small chat model that works well, we recommend `Meta-Llama-3-8B-Instruct.Q4_K_M.gguf`, which you can download [here](https://huggingface.co/MaziyarPanahi/Meta-Llama-3-8B-Instruct-GGUF/tree/main). Put this file in a path reachable by your app. 23 | 24 | ## Loading `@electron/llm` 25 | 26 | Then, in your `main` process, load the module. Make sure to do _before_ you load any windows to make sure that the `window.electronAi` API 27 | is available. 28 | 29 | ```ts:main.js 30 | import { app } from "electron" 31 | import { loadElectronLlm } from "@electron/llm" 32 | 33 | app.on("ready", () => { 34 | await loadElectronLlm() 35 | await createBrowserWindow() 36 | }) 37 | 38 | async function createBrowserWindow() { 39 | // ... 40 | } 41 | ``` 42 | 43 | ## Chatting with the model 44 | 45 | You can now use this module in any renderer. By default, `@electron/llm` auto-injects a preload script that exposes `window.electronAi`. 46 | 47 | ``` 48 | // First, load the model 49 | await window.electronAi.create({ 50 | modelPath: "/full/path/to/model.gguf" 51 | }) 52 | 53 | // Then, talk to it 54 | const response = await window.electronAi.prompt("Hi! How are you doing today?") 55 | ``` 56 | 57 | ## API 58 | 59 | ### Main Process API 60 | 61 | #### `loadElectronLlm(options?: LoadOptions): Promise` 62 | 63 | Loads the LLM module in the main process. 64 | 65 | - `options`: Optional configuration 66 | - `isAutomaticPreloadDisabled`: If true, the automatic preload script injection is disabled 67 | - `getModelPath`: A function that takes a model alias and returns the full path to the GGUF model file. By default, this function returns a path in the app's userData directory: `path.join(app.getPath('userData'), 'models', modelAlias)`. You can override this to customize where models are stored. 68 | 69 | ### Renderer Process API 70 | 71 | The renderer process API is exposed via `window.electronAi` once loaded via preload and provides the following methods: 72 | 73 | #### `create(options: LanguageModelCreateOptions): Promise` 74 | 75 | Creates and initializes a language model instance. This module will at most create one utility process with one model loaded. If you call `create` multiple times, it will return the existing instance. If you call it with new (not deep equal) options, it will stop and unload previously loaded models and load the model defined in the new options. 76 | 77 | - `options`: Configuration for the language model 78 | - `modelAlias`: Name of the model you want to load. Will be passed to `getModelPath()`. 79 | - `systemPrompt`: Optional system prompt to initialize the model 80 | - `initialPrompts`: Optional array of initial prompts to provide context 81 | - `topK`: Optional parameter to control diversity of generated text. 10 by default. 82 | - `temperature`: Optional parameter to control randomness of generated text. 0.7 by default. 83 | - `requestUUID`: Optional UUID to cancel the model loading using 84 | 85 | #### `destroy(): Promise` 86 | 87 | Destroys the current language model instance and frees resources. 88 | 89 | #### `prompt(input: string, options?: LanguageModelPromptOptions): Promise` 90 | 91 | Sends a prompt to the model and returns the complete response as a string. 92 | 93 | - `input`: The prompt text to send to the model 94 | - `options`: Optional configuration for the prompt 95 | - `responseJSONSchema`: Optional JSON schema to format the response as structured data 96 | - `signal`: Optional AbortSignal to cancel the request 97 | - `timeout`: Optional timeout in milliseconds (defaults to 20000ms) 98 | - `requestUUID`: Optional UUID to cancel the model loading using 99 | - Returns: A promise that resolves to the model's response 100 | 101 | #### `promptStreaming(input: string, options?: LanguageModelPromptOptions): Promise>` 102 | 103 | Sends a prompt to the model and returns the response as a stream of text chunks. 104 | 105 | - `input`: The prompt text to send to the model 106 | - `options`: Optional configuration for the prompt 107 | - `responseJSONSchema`: Optional JSON schema to format the response as structured data 108 | - `signal`: Optional AbortSignal to cancel the request 109 | - `timeout`: Optional timeout in milliseconds (defaults to 20000ms) 110 | - `requestUUID`: Optional UUID to cancel the model loading using 111 | - Returns: A promise that resolves to an async iterator of response chunks 112 | 113 | #### `abortRequest(requestUUID: string): Promise` 114 | 115 | Allows the abortion of a currently running model load or prompting request. To use this API, make sure to pass in `requestUUID` to your 116 | requests. 117 | 118 | # Testing 119 | 120 | Tests are implemented using [Vitest](https://vitest.dev/). To run the tests, use the following commands: 121 | 122 | ```bash 123 | # Run tests once 124 | npm test 125 | 126 | # Run tests in watch mode (useful during development) 127 | npm run test:watch 128 | 129 | # Run tests with coverage report 130 | npm run test:coverage 131 | ``` 132 | 133 | For more details, see [\_\_tests\_\_/README.md](\_\_tests\_\_/README.md). 134 | -------------------------------------------------------------------------------- /__tests__/README.md: -------------------------------------------------------------------------------- 1 | # Testing 2 | 3 | Tests are implemented using [Vitest](https://vitest.dev/). To run the tests, use the following commands: 4 | 5 | ```bash 6 | # Run tests once 7 | npm test 8 | 9 | # Run tests in watch mode (useful during development) 10 | npm run test:watch 11 | 12 | # Run tests with coverage report 13 | npm run test:coverage 14 | ``` 15 | 16 | ## Test Structure 17 | 18 | Each test file corresponds to a module. For example: 19 | 20 | - `get-process-type.test.ts` tests the functionality in `get-process-type.ts` 21 | 22 | ## Adding New Tests 23 | 24 | When adding new tests: 25 | 26 | 1. Create a new test file with the `.test.ts` extension in the __tests__ directory 27 | 2. Import the necessary testing utilities from Vitest 28 | 3. Import the functions to test from the parent directory 29 | 4. Write your tests using the `describe`, `it`, and `expect` functions 30 | 31 | ## Mocking 32 | 33 | For functions that rely on external dependencies or environment variables, use Vitest's mocking capabilities. 34 | 35 | Example: 36 | 37 | ```typescript 38 | import { describe, it, expect, vi } from 'vitest'; 39 | 40 | // Mock a module 41 | vi.mock('module-name', () => { 42 | return { 43 | functionName: vi.fn() 44 | }; 45 | }); 46 | ``` 47 | -------------------------------------------------------------------------------- /__tests__/helpers/get-process-type.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; 2 | import { getProcessType } from '../../src/helpers/get-process-type'; 3 | 4 | // Mock the electron module 5 | vi.mock('electron', () => { 6 | return { 7 | contextBridge: undefined, 8 | }; 9 | }); 10 | 11 | describe('getProcessType', () => { 12 | const originalProcessType = process.type; 13 | 14 | beforeEach(() => { 15 | vi.resetAllMocks(); 16 | }); 17 | 18 | afterEach(() => { 19 | // Restore process.type after each test 20 | Object.defineProperty(process, 'type', { 21 | value: originalProcessType, 22 | configurable: true, 23 | }); 24 | }); 25 | 26 | it('should return "main" when process.type is "browser"', async () => { 27 | Object.defineProperty(process, 'type', { 28 | value: 'browser', 29 | configurable: true, 30 | }); 31 | 32 | const result = await getProcessType(); 33 | expect(result).toBe('main'); 34 | }); 35 | 36 | it('should return "worker" when process.type is "worker"', async () => { 37 | Object.defineProperty(process, 'type', { 38 | value: 'worker', 39 | configurable: true, 40 | }); 41 | 42 | const result = await getProcessType(); 43 | expect(result).toBe('worker'); 44 | }); 45 | 46 | it('should return "utility" when process.type is "utility"', async () => { 47 | Object.defineProperty(process, 'type', { 48 | value: 'utility', 49 | configurable: true, 50 | }); 51 | 52 | const result = await getProcessType(); 53 | expect(result).toBe('utility'); 54 | }); 55 | 56 | it('should return "unknown" for any other process.type', async () => { 57 | Object.defineProperty(process, 'type', { 58 | value: 'something-else', 59 | configurable: true, 60 | }); 61 | 62 | const result = await getProcessType(); 63 | expect(result).toBe('unknown'); 64 | }); 65 | }); 66 | -------------------------------------------------------------------------------- /__tests__/language-model.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect, vi } from 'vitest'; 2 | import { LanguageModel } from '../src/language-model.js'; 3 | import { 4 | LanguageModelPromptRole, 5 | LanguageModelPromptType, 6 | } from '../src/interfaces.js'; 7 | 8 | vi.mock('node-llama-cpp', () => { 9 | return { 10 | getLlama: async () => { 11 | return { 12 | loadModel: async ({ modelPath }: { modelPath: string }) => { 13 | return { 14 | createContext: async () => { 15 | return { 16 | getSequence: () => 'dummy-sequence', 17 | }; 18 | }, 19 | }; 20 | }, 21 | }; 22 | }, 23 | LlamaChatSession: class { 24 | contextSequence: string; 25 | constructor({ contextSequence }: { contextSequence: string }) { 26 | this.contextSequence = contextSequence; 27 | } 28 | async prompt(input: string, options?: any): Promise { 29 | return `Mocked response to: ${input}`; 30 | } 31 | }, 32 | }; 33 | }); 34 | 35 | describe('LanguageModel with mocks', () => { 36 | const opts = { 37 | modelAlias: 'dummy-model', 38 | modelPath: 'dummy-model.gguf', 39 | topK: 5, 40 | temperature: 0.8, 41 | systemPrompt: 'Test system prompt', 42 | }; 43 | 44 | it('should create a LanguageModel and get a response', async () => { 45 | const model = await LanguageModel.create(opts); 46 | expect(model).toBeDefined(); 47 | 48 | const promptPayload = { 49 | role: LanguageModelPromptRole.USER, 50 | type: LanguageModelPromptType.TEXT, 51 | content: 'Hello, test!', 52 | }; 53 | 54 | const response = await model.prompt(promptPayload); 55 | expect(response).toContain('Mocked response to:'); 56 | }); 57 | 58 | it('should stream a response via promptStreaming', async () => { 59 | const model = await LanguageModel.create(opts); 60 | expect(model).toBeDefined(); 61 | 62 | const promptPayload = { 63 | role: LanguageModelPromptRole.USER, 64 | type: LanguageModelPromptType.TEXT, 65 | content: 'Tell me a test story.', 66 | }; 67 | 68 | // mock streaming behaviour 69 | model.promptStreaming = (payload, options) => { 70 | return new ReadableStream({ 71 | start(controller) { 72 | controller.enqueue('chunk1 '); 73 | controller.enqueue('chunk2 '); 74 | controller.close(); 75 | }, 76 | }); 77 | }; 78 | 79 | const stream = model.promptStreaming(promptPayload); 80 | const reader = stream.getReader(); 81 | let output = ''; 82 | 83 | while (true) { 84 | const { done, value } = await reader.read(); 85 | if (done) break; 86 | output += value; 87 | } 88 | 89 | expect(output).toEqual('chunk1 chunk2 '); 90 | }); 91 | 92 | it('should throw an error when using an unsupported prompt role', async () => { 93 | const model = await LanguageModel.create(opts); 94 | expect(model).toBeDefined(); 95 | 96 | // TODO: SYSTEM role unsupported for now 97 | const promptPayload = { 98 | role: LanguageModelPromptRole.SYSTEM, 99 | type: LanguageModelPromptType.TEXT, 100 | content: 'This should fail.', 101 | }; 102 | 103 | await expect(model.prompt(promptPayload)).rejects.toThrow( 104 | 'NotSupportedError', 105 | ); 106 | }); 107 | }); 108 | -------------------------------------------------------------------------------- /__tests__/preload.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect, vi, beforeEach } from 'vitest'; 2 | import { loadElectronLlm } from '../src/preload/index.ts'; 3 | import { IpcRendererMessage } from '../src/common/ipc-channel-names.js'; 4 | 5 | vi.mock('electron', () => { 6 | return { 7 | ipcRenderer: { 8 | invoke: vi.fn(), 9 | on: vi.fn(), 10 | once: vi.fn(), 11 | send: vi.fn(), 12 | }, 13 | contextBridge: { 14 | exposeInMainWorld: vi.fn((key, api) => { 15 | (globalThis as any)[key] = api; 16 | }), 17 | }, 18 | }; 19 | }); 20 | 21 | describe('Preload Interface', () => { 22 | let ipcRenderer: any; 23 | 24 | beforeEach(async () => { 25 | (globalThis as any).electronAi = undefined; 26 | vi.clearAllMocks(); 27 | await loadElectronLlm(); 28 | ipcRenderer = (await import('electron')).ipcRenderer; 29 | }); 30 | 31 | it('should expose electronAi on globalThis', () => { 32 | expect((globalThis as any).electronAi).toBeDefined(); 33 | }); 34 | 35 | it('create should invoke with correct ipcMessage and options', async () => { 36 | const options = { modelAlias: 'dummy-model' }; 37 | await (globalThis as any).electronAi.create(options); 38 | expect(ipcRenderer.invoke).toHaveBeenCalledWith( 39 | IpcRendererMessage.ELECTRON_LLM_CREATE, 40 | options, 41 | ); 42 | }); 43 | 44 | it('destroy should invoke with correct ipcMessage', async () => { 45 | await (globalThis as any).electronAi.destroy(); 46 | expect(ipcRenderer.invoke).toHaveBeenCalledWith( 47 | IpcRendererMessage.ELECTRON_LLM_DESTROY, 48 | ); 49 | }); 50 | 51 | it('prompt should invoke with correct params', async () => { 52 | const input = 'Test prompt'; 53 | const options = { responseJSONSchema: { type: 'string' } }; 54 | await (globalThis as any).electronAi.prompt(input, options); 55 | expect(ipcRenderer.invoke).toHaveBeenCalledWith( 56 | IpcRendererMessage.ELECTRON_LLM_PROMPT, 57 | input, 58 | options, 59 | ); 60 | }); 61 | 62 | it('promptStreaming should invoke with correct params', async () => { 63 | const input = 'Test prompt for streaming'; 64 | 65 | // Mock the MessagePort for streaming responses 66 | const mockPort = { 67 | start: vi.fn(), 68 | close: vi.fn(), 69 | onmessage: null, 70 | }; 71 | 72 | // Mock the event with ports array 73 | const mockEvent = { 74 | ports: [mockPort], 75 | }; 76 | 77 | // Set up the once handler to be called with our mock event 78 | ipcRenderer.once.mockImplementation((_channel, callback) => { 79 | callback(mockEvent); 80 | 81 | expect(mockPort.onmessage).toBeDefined(); 82 | 83 | if (mockPort.onmessage) { 84 | (mockPort as unknown as MessagePort).onmessage!({ 85 | data: { 86 | type: 'chunk', 87 | chunk: 'Hello, world!', 88 | }, 89 | } as MessageEvent); 90 | 91 | (mockPort as unknown as MessagePort).onmessage!({ 92 | data: { 93 | type: 'done', 94 | }, 95 | } as MessageEvent); 96 | } 97 | }); 98 | 99 | await (globalThis as any).electronAi.promptStreaming(input); 100 | 101 | expect(ipcRenderer.send).toHaveBeenCalledWith( 102 | IpcRendererMessage.ELECTRON_LLM_PROMPT_STREAMING_REQUEST, 103 | input, 104 | undefined, 105 | ); 106 | expect(mockPort.start).toHaveBeenCalled(); 107 | }); 108 | }); 109 | -------------------------------------------------------------------------------- /end-to-end/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "main": "dist/main/index.js", 3 | "dependencies": { 4 | "@electron/llm": "file:../dist" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /end-to-end/src/main.ts: -------------------------------------------------------------------------------- 1 | import { app, BrowserWindow } from 'electron'; 2 | import { loadElectronLlm } from '../../dist'; 3 | 4 | import path from 'path'; 5 | async function createWindow() { 6 | const mainWindow = new BrowserWindow({ 7 | width: 800, 8 | height: 800, 9 | webPreferences: { 10 | preload: path.join(__dirname, 'preload.js'), 11 | }, 12 | }); 13 | 14 | mainWindow.loadFile('../static/index.html'); 15 | } 16 | 17 | async function setupLlm() { 18 | await loadElectronLlm(); 19 | } 20 | 21 | async function onReady() { 22 | await setupLlm(); 23 | createWindow(); 24 | } 25 | 26 | app.on('ready', onReady); 27 | -------------------------------------------------------------------------------- /end-to-end/src/preload.ts: -------------------------------------------------------------------------------- 1 | import { contextBridge, webUtils } from 'electron'; 2 | 3 | contextBridge.exposeInMainWorld('electron', { 4 | getPathForFile(file: File) { 5 | return webUtils.getPathForFile(file); 6 | }, 7 | }); 8 | -------------------------------------------------------------------------------- /end-to-end/static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | AI Chat Interface 7 | 96 | 97 | 98 |

@electron/llm

99 | 100 |
101 |
102 | 103 | 104 |
105 | 106 |
107 | 108 | 109 |
110 | 111 |
112 | 113 | 114 |
115 | 116 |
117 | 118 | 119 |
120 | 121 |
122 | 123 | 124 |
125 | 126 | 127 |
128 | 129 |
130 |
131 |
132 | 136 | 137 |
138 |
139 | 140 | 208 | 209 | 210 | -------------------------------------------------------------------------------- /end-to-end/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "compilerOptions": { 4 | "jsx": "react-jsx", 5 | "module": "CommonJS", 6 | "moduleResolution": "Node", 7 | "outDir": "dist", 8 | "rootDir": "src" 9 | }, 10 | "include": ["src"] 11 | } 12 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@electron/llm", 3 | "version": "0.0.0-development", 4 | "description": "Load and use an LLM model directly in Electron. Experimental.", 5 | "main": "dist/index.js", 6 | "types": "dist/index.d.ts", 7 | "workspaces": [ 8 | "end-to-end" 9 | ], 10 | "files": [ 11 | "dist", 12 | "README.md" 13 | ], 14 | "publishConfig": { 15 | "provenance": true 16 | }, 17 | "exports": { 18 | ".": { 19 | "types": "./dist/index.d.ts", 20 | "default": "./dist/index.js" 21 | }, 22 | "./main": { 23 | "types": "./dist/main/index.d.ts", 24 | "default": "./dist/main/index.js" 25 | }, 26 | "./renderer": { 27 | "types": "./dist/renderer/index.d.ts", 28 | "default": "./dist/renderer/index.js" 29 | }, 30 | "./package.json": "./package.json" 31 | }, 32 | "scripts": { 33 | "tsc": "tsc --build", 34 | "lint:check": "prettier --check \"**/*.{ts,js}\"", 35 | "lint:fix": "prettier --write \"**/*.{ts,js}\"", 36 | "test": "vitest run", 37 | "test:watch": "vitest", 38 | "test:coverage": "vitest run --coverage", 39 | "test:e2e": "npm run test:e2e:tsc && npm run test:e2e:rebuild && npm run test:e2e:run", 40 | "test:e2e:tsc": "tsc -p end-to-end/tsconfig.json", 41 | "test:e2e:rebuild": "electron-rebuild", 42 | "test:e2e:run": "electron end-to-end/dist/main.js", 43 | "prepublishOnly": "npm run tsc", 44 | "prepare": "husky" 45 | }, 46 | "lint-staged": { 47 | "*.{js,ts}": [ 48 | "prettier --write" 49 | ] 50 | }, 51 | "author": "Electron Community", 52 | "keywords": [ 53 | "electron", 54 | "llm", 55 | "ai", 56 | "chat", 57 | "chatbot", 58 | "chatgpt", 59 | "llama", 60 | "llama.cpp" 61 | ], 62 | "license": "MIT", 63 | "repository": { 64 | "type": "git", 65 | "url": "git+https://github.com/electron/llm.git" 66 | }, 67 | "dependencies": { 68 | "node-llama-cpp": "^3.7.0" 69 | }, 70 | "devDependencies": { 71 | "@electron/rebuild": "^3.7.1", 72 | "@tsconfig/node22": "^22.0.0", 73 | "@vitest/coverage-v8": "^3.0.7", 74 | "electron": "^35.1.4", 75 | "eslint": "^8.57.1", 76 | "husky": "^9.1.7", 77 | "lint-staged": "^15.4.3", 78 | "prettier": "^3.5.2", 79 | "typescript": "^5.7.3", 80 | "vitest": "^3.0.7" 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/common/ipc-channel-names.ts: -------------------------------------------------------------------------------- 1 | export const IpcRendererMessage = { 2 | ELECTRON_LLM_ABORT_REQUEST: 'ELECTRON_LLM_ABORT_REQUEST', 3 | ELECTRON_LLM_CREATE: 'ELECTRON_LLM_CREATE', 4 | ELECTRON_LLM_DESTROY: 'ELECTRON_LLM_DESTROY', 5 | ELECTRON_LLM_PROMPT: 'ELECTRON_LLM_PROMPT', 6 | ELECTRON_LLM_PROMPT_STREAMING: 'ELECTRON_LLM_PROMPT_STREAMING', 7 | ELECTRON_LLM_PROMPT_STREAMING_REQUEST: 8 | 'ELECTRON_LLM_PROMPT_STREAMING_REQUEST', 9 | } as const; 10 | -------------------------------------------------------------------------------- /src/constants.ts: -------------------------------------------------------------------------------- 1 | export const IPC_PREFIX = 'ELECTRON_LLM_'; 2 | -------------------------------------------------------------------------------- /src/helpers/get-process-type.ts: -------------------------------------------------------------------------------- 1 | export type ProcessType = 2 | | 'main' 3 | | 'renderer' 4 | | 'preload' 5 | | 'utility' 6 | | 'worker' 7 | | 'unknown'; 8 | 9 | /** 10 | * Returns the current Electron process type 11 | * 12 | * @returns Promise 13 | */ 14 | export async function getProcessType(): Promise { 15 | // Check if we're in the main process 16 | if (process.type === 'browser') { 17 | return 'main'; 18 | } 19 | 20 | // Check if we're in the renderer process 21 | if (process.type === 'renderer') { 22 | if (await isContextIsolatedPreload()) { 23 | return 'preload'; 24 | } 25 | 26 | return 'renderer'; 27 | } 28 | 29 | // Check for worker or utility processes 30 | if (process.type === 'worker') { 31 | return 'worker'; 32 | } 33 | 34 | if (process.type === 'utility') { 35 | return 'utility'; 36 | } 37 | 38 | // If none of the above 39 | return 'unknown'; 40 | } 41 | 42 | /** 43 | * Returns true if the current process is a context isolated preload script. 44 | * 45 | * @returns boolean 46 | */ 47 | async function isContextIsolatedPreload() { 48 | try { 49 | const electron = await import('electron'); 50 | 51 | return !!electron.contextBridge; 52 | } catch (error) { 53 | return false; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import { getProcessType } from './helpers/get-process-type.js'; 2 | import { 3 | MainLoadFunction, 4 | LoadOptions, 5 | RendererLoadFunction, 6 | } from './interfaces.js'; 7 | 8 | export * from './interfaces.js'; 9 | export * from './constants.js'; 10 | 11 | export async function loadElectronLlm(options?: LoadOptions) { 12 | const processType = await getProcessType(); 13 | let loadFunction: MainLoadFunction | RendererLoadFunction; 14 | 15 | if (processType === 'main') { 16 | loadFunction = (await import('./main/index.js')).loadElectronLlm; 17 | } else if (processType === 'renderer') { 18 | loadFunction = (await import('./renderer/index.js')).loadElectronLlm; 19 | } else if (processType === 'preload') { 20 | loadFunction = (await import('./preload/index.js')).loadElectronLlm; 21 | } else { 22 | throw new Error(`Unsupported process type: ${processType}`); 23 | } 24 | 25 | await loadFunction(options); 26 | } 27 | -------------------------------------------------------------------------------- /src/interfaces.ts: -------------------------------------------------------------------------------- 1 | // Shared interfaces 2 | export interface ElectronLlmShared {} 3 | 4 | export type LanguageModelPromptContent = string | ArrayBuffer; 5 | 6 | export enum LanguageModelPromptType { 7 | TEXT = 'text', 8 | IMAGE = 'image', 9 | AUDIO = 'audio', 10 | } 11 | 12 | export enum LanguageModelPromptRole { 13 | SYSTEM = 'system', 14 | USER = 'user', 15 | ASSISTANT = 'assistant', 16 | } 17 | 18 | export interface LanguageModelPrompt { 19 | role: LanguageModelPromptRole; 20 | type: LanguageModelPromptType; 21 | content: LanguageModelPromptContent; 22 | } 23 | 24 | export interface LanguageModelCreateOptions { 25 | systemPrompt?: string; 26 | initialPrompts?: LanguageModelPrompt[]; 27 | topK?: number; 28 | temperature?: number; 29 | requestUUID?: string; 30 | modelAlias: string; 31 | } 32 | 33 | export interface InternalLanguageModelCreateOptions 34 | extends LanguageModelCreateOptions { 35 | modelPath: string; 36 | signal?: AbortSignal; 37 | } 38 | 39 | export interface LanguageModelPromptOptions { 40 | responseJSONSchema?: object; 41 | requestUUID?: string; 42 | timeout?: number; 43 | } 44 | 45 | export interface InternalLanguageModelPromptOptions 46 | extends LanguageModelPromptOptions { 47 | signal?: AbortSignal; 48 | } 49 | 50 | export type AiProcessModelCreateData = InternalLanguageModelCreateOptions; 51 | 52 | export interface AiProcessSendPromptData { 53 | options: LanguageModelPromptOptions; 54 | stream?: boolean; 55 | input: string; 56 | } 57 | 58 | // Renderer interfaces 59 | export interface ElectronLlmRenderer { 60 | create: (options: LanguageModelCreateOptions) => Promise; 61 | destroy: () => Promise; 62 | prompt: ( 63 | input: string, 64 | options?: LanguageModelPromptOptions, 65 | ) => Promise; 66 | promptStreaming: ( 67 | input: string, 68 | options?: LanguageModelPromptOptions, 69 | ) => Promise>; 70 | abortRequest: (requestUUID: string) => void; 71 | } 72 | 73 | // Main interfaces 74 | export interface ElectronLlmMain {} 75 | 76 | export type MainLoadOptions = { 77 | isAutomaticPreloadDisabled?: boolean; 78 | getModelPath?: GetModelPathFunction; 79 | }; 80 | 81 | export type LoadOptions = MainLoadOptions; 82 | export type ElectronAi = ElectronLlmRenderer; 83 | 84 | export type MainLoadFunction = (options?: LoadOptions) => Promise; 85 | export type RendererLoadFunction = () => Promise; 86 | 87 | export type GetModelPathFunction = ( 88 | modelAlias: string, 89 | ) => Promise | string | null; 90 | -------------------------------------------------------------------------------- /src/language-model.ts: -------------------------------------------------------------------------------- 1 | import type { 2 | ChatHistoryItem, 3 | LlamaChatSession, 4 | LlamaModel, 5 | } from 'node-llama-cpp' with { 'resolution-mode': 'import' }; 6 | import { 7 | LanguageModelPrompt, 8 | LanguageModelPromptType, 9 | LanguageModelPromptRole, 10 | LanguageModelPromptContent, 11 | InternalLanguageModelCreateOptions, 12 | InternalLanguageModelPromptOptions, 13 | } from './interfaces'; 14 | 15 | interface LanguageModelParams { 16 | readonly defaultTopK: number; 17 | readonly maxTopK: number; 18 | readonly defaultTemperature: number; 19 | readonly maxTemperature: number; 20 | } 21 | 22 | interface AIAvailability { 23 | status: 'unavailable' | 'downloadable' | 'downloading' | 'available'; 24 | reason?: string; 25 | } 26 | 27 | // prettier-ignore 28 | let _llamaCpp: typeof import('node-llama-cpp', { with: { 'resolution-mode': 'import' } }); 29 | async function getLlamaCpp() { 30 | if (!_llamaCpp) { 31 | _llamaCpp = await import('node-llama-cpp'); 32 | } 33 | 34 | return _llamaCpp; 35 | } 36 | 37 | export class LanguageModel { 38 | readonly topK: number; 39 | readonly temperature: number; 40 | private systemPrompt?: string; 41 | private initialPrompts?: LanguageModelPrompt[]; 42 | private context?: any; 43 | private session?: LlamaChatSession; 44 | 45 | private static readonly paramsData: LanguageModelParams = { 46 | defaultTopK: 10, 47 | maxTopK: 100, 48 | defaultTemperature: 0.7, 49 | maxTemperature: 2.0, 50 | }; 51 | 52 | private constructor( 53 | options: InternalLanguageModelCreateOptions, 54 | model: LlamaModel, 55 | context: any, 56 | session: LlamaChatSession, 57 | ) { 58 | this.topK = options.topK ?? 10; 59 | this.temperature = options.temperature ?? 0.7; 60 | this.systemPrompt = options.systemPrompt; 61 | this.initialPrompts = options.initialPrompts; 62 | this.context = context; 63 | this.session = session; 64 | } 65 | 66 | static async create( 67 | options: InternalLanguageModelCreateOptions, 68 | ): Promise { 69 | try { 70 | const llamaCpp = await getLlamaCpp(); 71 | const llama = await llamaCpp.getLlama(); 72 | const model = await llama.loadModel({ modelPath: options.modelPath }); 73 | const context = await model.createContext(); 74 | const session = new llamaCpp.LlamaChatSession({ 75 | contextSequence: context.getSequence(), 76 | systemPrompt: options.systemPrompt, 77 | }); 78 | 79 | if (options.initialPrompts && options.initialPrompts.length > 0) { 80 | session.setChatHistory( 81 | options.initialPrompts.map(this.initialPromptToChatHistoryItem), 82 | ); 83 | } 84 | 85 | process.parentPort?.postMessage({ 86 | type: 'modelLoaded', 87 | data: 'Model loaded successfully.', 88 | }); 89 | 90 | return new LanguageModel(options, model, context, session); 91 | } catch (error) { 92 | throw error; 93 | } 94 | } 95 | 96 | static async availability(): Promise { 97 | try { 98 | const llamaCpp = await getLlamaCpp(); 99 | const llama = await llamaCpp.getLlama(); 100 | 101 | if (!llama) { 102 | return { 103 | status: 'unavailable', 104 | reason: 'Llama runtime is not accessible.', 105 | }; 106 | } 107 | 108 | return { status: 'available' }; 109 | } catch (error) { 110 | return { status: 'unavailable', reason: (error as Error).message }; 111 | } 112 | } 113 | 114 | static async params(): Promise { 115 | return Promise.resolve(LanguageModel.paramsData); 116 | } 117 | 118 | private parseContent(content: LanguageModelPromptContent): string { 119 | if (typeof content === 'string') { 120 | return content; 121 | } else if (content instanceof ArrayBuffer) { 122 | return Buffer.from(content).toString('utf-8'); 123 | } 124 | throw new Error('Unsupported content type.'); 125 | } 126 | 127 | async prompt( 128 | input: LanguageModelPrompt | LanguageModelPrompt[], 129 | options?: InternalLanguageModelPromptOptions, 130 | ): Promise { 131 | if (!this.session) { 132 | process.parentPort?.postMessage({ 133 | type: 'error', 134 | data: 'Model session is not initialized.', 135 | }); 136 | 137 | throw new Error('Model session is not initialized.'); 138 | } 139 | 140 | const prompts = Array.isArray(input) ? input : [input]; 141 | prompts.forEach(this.validatePrompt); 142 | 143 | const processedInput = prompts 144 | .map((p) => this.parseContent(p.content)) 145 | .join('\n'); 146 | 147 | const response = await this.session.prompt(processedInput, { 148 | temperature: this.temperature, 149 | signal: options?.signal, 150 | stopOnAbortSignal: true, 151 | topK: this.topK, 152 | }); 153 | 154 | return response; 155 | } 156 | 157 | promptStreaming( 158 | input: LanguageModelPrompt | LanguageModelPrompt[], 159 | options?: InternalLanguageModelPromptOptions, 160 | ): ReadableStream { 161 | if (!this.session) { 162 | process.parentPort.postMessage({ 163 | type: 'error', 164 | data: 'Model session is not initialized.', 165 | }); 166 | throw new Error('Model session is not initialized.'); 167 | } 168 | 169 | const prompts = Array.isArray(input) ? input : [input]; 170 | prompts.forEach(this.validatePrompt); 171 | 172 | if (prompts[0].type !== LanguageModelPromptType.TEXT) { 173 | throw new Error( 174 | 'NotSupportedError: Only text prompts are supported for streaming', 175 | ); 176 | } 177 | const processedInput = prompts 178 | .map((p) => this.parseContent(p.content)) 179 | .join('\n'); 180 | 181 | return new ReadableStream({ 182 | start: async (controller) => { 183 | await this.session!.prompt(processedInput, { 184 | temperature: this.temperature, 185 | signal: options?.signal, 186 | stopOnAbortSignal: true, 187 | topK: this.topK, 188 | 189 | onTextChunk: (chunk: string) => { 190 | controller.enqueue(chunk); 191 | process.parentPort?.postMessage({ type: 'stream', data: chunk }); 192 | }, 193 | }); 194 | 195 | controller.close(); 196 | process.parentPort?.postMessage({ type: 'done' }); 197 | }, 198 | }); 199 | } 200 | 201 | private validatePrompt(prompt: LanguageModelPrompt) { 202 | if (prompt.role === LanguageModelPromptRole.SYSTEM) { 203 | throw new Error( 204 | "NotSupportedError: 'system' role is not allowed in prompt()", 205 | ); 206 | } 207 | } 208 | 209 | private static initialPromptToChatHistoryItem( 210 | prompt: LanguageModelPrompt, 211 | ): ChatHistoryItem { 212 | if (prompt.role === LanguageModelPromptRole.SYSTEM) { 213 | return { 214 | type: 'system', 215 | text: prompt.content.toString(), 216 | }; 217 | } 218 | 219 | if (prompt.role === LanguageModelPromptRole.USER) { 220 | return { 221 | type: 'user', 222 | text: prompt.content.toString(), 223 | }; 224 | } 225 | 226 | if (prompt.role === LanguageModelPromptRole.ASSISTANT) { 227 | return { 228 | type: 'model', 229 | response: [prompt.content.toString()], 230 | }; 231 | } 232 | 233 | throw new Error('Invalid prompt role.'); 234 | } 235 | 236 | destroy(): void { 237 | this.session = undefined; 238 | this.context = undefined; 239 | 240 | process.parentPort.postMessage({ 241 | type: 'stopped', 242 | data: 'Model session destroyed.', 243 | }); 244 | } 245 | } 246 | -------------------------------------------------------------------------------- /src/main/index.ts: -------------------------------------------------------------------------------- 1 | import { session, app, BrowserWindow } from 'electron'; 2 | import fs from 'node:fs'; 3 | import path from 'node:path'; 4 | 5 | import { GetModelPathFunction, MainLoadFunction } from '../interfaces.js'; 6 | import { registerAiHandlers } from './register-ai-handlers.js'; 7 | 8 | export const loadElectronLlm: MainLoadFunction = async (options) => { 9 | if (!options?.isAutomaticPreloadDisabled) { 10 | // Developers might load @electron/llm in their main process after BrowserWindows have already been created. 11 | // This can cause issues because the preload script won't be loaded in those windows. 12 | // This function checks for this case and warns the developer. 13 | warnIfWindowsExist(); 14 | 15 | // Register preload for default session 16 | registerPreload(session.defaultSession); 17 | 18 | // Also register preload for new sessions 19 | app.on('session-created', (session) => { 20 | registerPreload(session); 21 | }); 22 | } 23 | 24 | const getModelPath: GetModelPathFunction = 25 | options?.getModelPath || 26 | (async (modelAlias: string) => { 27 | const baseModelsPath = path.resolve(app.getPath('userData'), 'models'); 28 | if (!fs.existsSync(baseModelsPath)) { 29 | return null; 30 | } 31 | const finalPath = path.resolve(baseModelsPath, modelAlias); 32 | if (!fs.existsSync(finalPath)) { 33 | // Let's not load a model that doesn't exist 34 | return null; 35 | } 36 | const realPath = await fs.promises.realpath(finalPath); 37 | const relativePath = path.relative(baseModelsPath, realPath); 38 | if (relativePath.startsWith('..') || path.isAbsolute(relativePath)) { 39 | // Model is outside of the base models path 40 | return null; 41 | } 42 | return realPath; 43 | }); 44 | 45 | // Register handler 46 | registerAiHandlers({ getModelPath }); 47 | }; 48 | 49 | /** 50 | * Registers the preload script for the given session. 51 | * This function is used to ensure that the preload script is loaded for all sessions. 52 | */ 53 | function registerPreload(session: Electron.Session) { 54 | const filePath = path.join(__dirname, '../preload/index.js'); 55 | 56 | session.registerPreloadScript({ 57 | filePath, 58 | type: 'frame', 59 | }); 60 | } 61 | 62 | /** 63 | * Warns the developer if BrowserWindows have already been created before loading @electron/llm. 64 | * This can cause issues because the preload script won't be loaded in those windows. 65 | */ 66 | function warnIfWindowsExist() { 67 | if (app.isPackaged) { 68 | return; 69 | } 70 | 71 | if (BrowserWindow.getAllWindows().length > 0) { 72 | console.warn( 73 | "electron/llm: You're loading @electron/llm after BrowserWindows have already been created. " + 74 | "Those windows will not have access to the LLM APIs. To fix this, call @electron/llm's " + 75 | 'load() function before creating any BrowserWindows.', 76 | ); 77 | } 78 | } 79 | 80 | // Re-export any types or interfaces specific to the main process 81 | export * from '../interfaces.js'; 82 | -------------------------------------------------------------------------------- /src/main/register-ai-handlers.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ipcMain, 3 | utilityProcess, 4 | UtilityProcess, 5 | MessageChannelMain, 6 | } from 'electron'; 7 | import { once } from 'node:events'; 8 | import path from 'node:path'; 9 | import { deepEqual } from 'node:assert/strict'; 10 | 11 | import { IpcRendererMessage } from '../common/ipc-channel-names.js'; 12 | import { UTILITY_MESSAGE_TYPES } from '../utility/messages.js'; 13 | import { 14 | AiProcessSendPromptData, 15 | GetModelPathFunction, 16 | LanguageModelCreateOptions, 17 | LanguageModelPromptOptions, 18 | } from '../interfaces.js'; 19 | 20 | let aiProcess: UtilityProcess | null = null; 21 | let aiProcessCreationOptions: LanguageModelCreateOptions | null = null; 22 | 23 | export interface RegisterAiHandlersOptions { 24 | getModelPath: GetModelPathFunction; 25 | } 26 | 27 | export function registerAiHandlers({ 28 | getModelPath, 29 | }: RegisterAiHandlersOptions) { 30 | ipcMain.handle(IpcRendererMessage.ELECTRON_LLM_DESTROY, () => stopModel()); 31 | 32 | ipcMain.handle( 33 | IpcRendererMessage.ELECTRON_LLM_CREATE, 34 | async (_event, options: LanguageModelCreateOptions) => { 35 | if (!shouldStartNewAiProcess(options)) { 36 | return; 37 | } 38 | 39 | const modelPath = await getModelPath(options.modelAlias); 40 | if (!modelPath) { 41 | throw new Error( 42 | `Model path not found for alias: ${options.modelAlias}. Please provide a valid model path.`, 43 | ); 44 | } 45 | 46 | aiProcessCreationOptions = options; 47 | 48 | if (aiProcess) { 49 | try { 50 | await stopModel(); 51 | } catch (error) { 52 | throw new Error( 53 | `Failed to stop previous AI model process: ${(error as Error).message || 'Unknown error'}`, 54 | ); 55 | } 56 | } 57 | 58 | aiProcess = await startAiModel(); 59 | 60 | const messagePromise = once(aiProcess, 'message'); 61 | 62 | aiProcess.postMessage({ 63 | type: UTILITY_MESSAGE_TYPES.LOAD_MODEL, 64 | data: { 65 | ...options, 66 | modelPath, 67 | }, 68 | }); 69 | 70 | const timeoutPromise = new Promise((_, reject) => { 71 | setTimeout( 72 | () => reject(new Error('AI model process start timed out.')), 73 | 60000, 74 | ); 75 | }); 76 | 77 | // Give the AI model process 60 seconds to load the model 78 | const [data] = await Promise.race([messagePromise, timeoutPromise]); 79 | const { type, data: responseData } = data; 80 | 81 | if (type === UTILITY_MESSAGE_TYPES.MODEL_LOADED) { 82 | return; 83 | } else if (type === UTILITY_MESSAGE_TYPES.ERROR) { 84 | // Try to clean up 85 | try { 86 | await stopModel(); 87 | } catch (error) { 88 | console.error( 89 | `Failed to stop AI model process after error: ${(error as Error).message || 'Unknown error'}`, 90 | ); 91 | } 92 | 93 | throw new Error(responseData); 94 | } else { 95 | throw new Error(`Unexpected message type: ${type}`); 96 | } 97 | }, 98 | ); 99 | 100 | ipcMain.handle( 101 | IpcRendererMessage.ELECTRON_LLM_PROMPT, 102 | async (_event, input: string, options: LanguageModelPromptOptions) => { 103 | if (!aiProcess) { 104 | throw new Error( 105 | 'AI model process not started. Please do so with `electronAi.create()`', 106 | ); 107 | } 108 | 109 | const data: AiProcessSendPromptData = { 110 | input, 111 | stream: false, 112 | options, 113 | }; 114 | 115 | aiProcess.postMessage({ 116 | type: UTILITY_MESSAGE_TYPES.SEND_PROMPT, 117 | data, 118 | }); 119 | 120 | const responsePromise = once(aiProcess, 'message').then(([msg]) => { 121 | const { type, data } = msg; 122 | if (type === UTILITY_MESSAGE_TYPES.DONE) { 123 | return data; 124 | } else if (type === UTILITY_MESSAGE_TYPES.ERROR) { 125 | throw new Error(data); 126 | } else { 127 | throw new Error(`Unexpected message type: ${type}`); 128 | } 129 | }); 130 | 131 | // Set a timeout in case the child process doesn't reply. 132 | const timeoutPromise = new Promise((_, reject) => { 133 | setTimeout( 134 | () => reject(new Error('Prompt response timed out.')), 135 | options.timeout || 20000, 136 | ); 137 | }); 138 | 139 | return await Promise.race([responsePromise, timeoutPromise]); 140 | }, 141 | ); 142 | 143 | ipcMain.on( 144 | IpcRendererMessage.ELECTRON_LLM_PROMPT_STREAMING_REQUEST, 145 | (event, input: string, options: LanguageModelPromptOptions) => { 146 | if (!aiProcess) { 147 | event.sender.send( 148 | 'ELECTRON_LLM_PROMPT_STREAMING_ERROR', 149 | 'AI model process not started.', 150 | ); 151 | return; 152 | } 153 | 154 | // Create two message channels 155 | const { port1: rendererPort1, port2: rendererPort2 } = 156 | new MessageChannelMain(); 157 | const { port1: utilityPort1, port2: utilityPort2 } = 158 | new MessageChannelMain(); 159 | 160 | // Connect the two ports directly 161 | rendererPort1.on('message', (event) => { 162 | utilityPort1.postMessage(event.data); 163 | }); 164 | 165 | utilityPort1.on('message', (event) => { 166 | rendererPort1.postMessage(event.data); 167 | }); 168 | 169 | // Start both ports 170 | rendererPort1.start(); 171 | utilityPort1.start(); 172 | 173 | // Send one port to the renderer 174 | event.sender.postMessage('ELECTRON_LLM_PROMPT_STREAMING_PORT', null, [ 175 | rendererPort2, 176 | ]); 177 | 178 | // Send the other port to the utility process 179 | aiProcess.postMessage( 180 | { 181 | type: UTILITY_MESSAGE_TYPES.SEND_PROMPT, 182 | data: { input, stream: true, options }, 183 | }, 184 | [utilityPort2], 185 | ); 186 | }, 187 | ); 188 | 189 | ipcMain.handle( 190 | IpcRendererMessage.ELECTRON_LLM_ABORT_REQUEST, 191 | (_event, { requestUUID } = {}) => { 192 | if (!aiProcess) { 193 | return; 194 | } 195 | 196 | aiProcess.postMessage({ 197 | type: UTILITY_MESSAGE_TYPES.REQUEST_ABORTED, 198 | data: { requestUUID }, 199 | }); 200 | }, 201 | ); 202 | } 203 | 204 | export async function startAiModel(): Promise { 205 | const utilityScriptPath = path.join( 206 | __dirname, 207 | '../utility/call-ai-model-entry-point.js', 208 | ); 209 | 210 | const aiProcess = utilityProcess.fork(utilityScriptPath, [], { 211 | stdio: ['ignore', 'pipe', 'pipe', 'pipe'], 212 | }); 213 | 214 | if (aiProcess.stdout) { 215 | aiProcess.stdout.on('data', (data) => { 216 | console.info(`AI model child process stdout: ${data}`); 217 | }); 218 | } 219 | 220 | if (aiProcess.stderr) { 221 | aiProcess.stderr.on('data', (data) => { 222 | console.error(`AI model child process stderror: ${data}`); 223 | }); 224 | } 225 | 226 | aiProcess.on('exit', () => { 227 | console.info('AI model child process exited.'); 228 | }); 229 | 230 | return aiProcess; 231 | } 232 | 233 | /** 234 | * Stops the AI model process. If the process doesn't exit after 3 seconds, it will be killed. 235 | */ 236 | async function stopModel(): Promise { 237 | if (aiProcess) { 238 | const exitPromise = once(aiProcess, 'exit'); 239 | aiProcess.postMessage({ type: UTILITY_MESSAGE_TYPES.STOP }); 240 | 241 | // If the process doesn't exit after 3 seconds, kill it 242 | try { 243 | await Promise.race([ 244 | exitPromise, 245 | new Promise((_, reject) => 246 | setTimeout(() => reject(new Error('Process exit timeout')), 3000), 247 | ), 248 | ]); 249 | } catch (error) { 250 | aiProcess.kill(); 251 | } 252 | 253 | aiProcess = null; 254 | } 255 | } 256 | 257 | function shouldStartNewAiProcess(options: LanguageModelCreateOptions): boolean { 258 | if (!aiProcess || !aiProcess.pid || !aiProcessCreationOptions) { 259 | return true; 260 | } 261 | 262 | try { 263 | deepEqual(options, aiProcessCreationOptions); 264 | return false; 265 | } catch { 266 | return true; 267 | } 268 | } 269 | -------------------------------------------------------------------------------- /src/preload/index.ts: -------------------------------------------------------------------------------- 1 | import { contextBridge, ipcRenderer } from 'electron'; 2 | 3 | import type { 4 | ElectronLlmRenderer, 5 | LanguageModelCreateOptions, 6 | LanguageModelPromptOptions, 7 | RendererLoadFunction, 8 | } from '../interfaces.js'; 9 | 10 | /** 11 | * Validates the options for creating a language model. 12 | * 13 | * @param options - The options to validate. 14 | * @throws {TypeError} If the options are invalid. 15 | */ 16 | function validateCreateOptions(options?: LanguageModelCreateOptions): void { 17 | if (!options) return; 18 | 19 | if ( 20 | options.modelAlias === undefined || 21 | typeof options.modelAlias !== 'string' 22 | ) { 23 | throw new TypeError('modelAlias is required and must be a string'); 24 | } 25 | 26 | if ( 27 | options.systemPrompt !== undefined && 28 | typeof options.systemPrompt !== 'string' 29 | ) { 30 | throw new TypeError('systemPrompt must be a string'); 31 | } 32 | 33 | if ( 34 | options.initialPrompts !== undefined && 35 | !Array.isArray(options.initialPrompts) 36 | ) { 37 | throw new TypeError('initialPrompts must be an array'); 38 | } 39 | 40 | if ( 41 | options.topK !== undefined && 42 | (typeof options.topK !== 'number' || options.topK <= 0) 43 | ) { 44 | throw new TypeError('topK must be a positive number'); 45 | } 46 | 47 | if ( 48 | options.temperature !== undefined && 49 | (typeof options.temperature !== 'number' || options.temperature < 0) 50 | ) { 51 | throw new TypeError('temperature must be a non-negative number'); 52 | } 53 | } 54 | 55 | /** 56 | * Validates the options for prompting a language model. 57 | * 58 | * @param options - The options to validate. 59 | * @throws {TypeError} If the options are invalid. 60 | */ 61 | function validatePromptOptions(options?: LanguageModelPromptOptions): void { 62 | if (!options) return; 63 | 64 | if ( 65 | options.responseJSONSchema !== undefined && 66 | typeof options.responseJSONSchema !== 'object' 67 | ) { 68 | throw new TypeError('responseJSONSchema must be an object'); 69 | } 70 | } 71 | 72 | const electronAi: ElectronLlmRenderer = { 73 | create: async (options?: LanguageModelCreateOptions): Promise => { 74 | validateCreateOptions(options); 75 | 76 | return ipcRenderer.invoke('ELECTRON_LLM_CREATE', options); 77 | }, 78 | destroy: async (): Promise => 79 | ipcRenderer.invoke('ELECTRON_LLM_DESTROY'), 80 | prompt: async ( 81 | input: string = '', 82 | options?: LanguageModelPromptOptions, 83 | ): Promise => { 84 | validatePromptOptions(options); 85 | return ipcRenderer.invoke('ELECTRON_LLM_PROMPT', input, options); 86 | }, 87 | promptStreaming: async ( 88 | input: string = '', 89 | options?: LanguageModelPromptOptions, 90 | ): Promise> => { 91 | validatePromptOptions(options); 92 | 93 | // Create a promise that will resolve with the port from main process 94 | return new Promise((resolve) => { 95 | ipcRenderer.once('ELECTRON_LLM_PROMPT_STREAMING_PORT', (event) => { 96 | // Access the port from the event's ports array 97 | const [port] = event.ports; 98 | 99 | // Start the port to receive messages 100 | port.start(); 101 | 102 | const iterator: AsyncIterableIterator = { 103 | async next(): Promise> { 104 | const message = await new Promise>( 105 | (resolve, reject) => { 106 | port.onmessage = (event) => { 107 | if (event.data.type === 'error') { 108 | reject(new Error(event.data.error)); 109 | } else if (event.data.type === 'done') { 110 | resolve({ done: true, value: undefined }); 111 | } else { 112 | resolve({ value: event.data.chunk, done: false }); 113 | } 114 | }; 115 | }, 116 | ); 117 | return message; 118 | }, 119 | async return() { 120 | port.close(); 121 | return { done: true, value: undefined }; 122 | }, 123 | async throw(error) { 124 | port.close(); 125 | throw error; 126 | }, 127 | [Symbol.asyncIterator]() { 128 | return this; 129 | }, 130 | }; 131 | 132 | resolve(iterator); 133 | }); 134 | 135 | // Request streaming from main process 136 | ipcRenderer.send('ELECTRON_LLM_PROMPT_STREAMING_REQUEST', input, options); 137 | }); 138 | }, 139 | abortRequest(requestUUID: string): Promise { 140 | return ipcRenderer.invoke('ELECTRON_LLM_ABORT_REQUEST', { requestUUID }); 141 | }, 142 | }; 143 | 144 | export const loadElectronLlm: RendererLoadFunction = async () => { 145 | contextBridge.exposeInMainWorld('electronAi', electronAi); 146 | }; 147 | 148 | loadElectronLlm(); 149 | -------------------------------------------------------------------------------- /src/renderer/index.ts: -------------------------------------------------------------------------------- 1 | import { RendererLoadFunction } from '../interfaces.js'; 2 | 3 | export const loadElectronLlm: RendererLoadFunction = async () => { 4 | throw new Error('Please load the module via preload'); 5 | }; 6 | 7 | // Re-export any types or interfaces specific to the renderer process 8 | export * from '../interfaces.js'; 9 | -------------------------------------------------------------------------------- /src/utility/abortmanager.ts: -------------------------------------------------------------------------------- 1 | import { 2 | InternalLanguageModelCreateOptions, 3 | InternalLanguageModelPromptOptions, 4 | LanguageModelCreateOptions, 5 | LanguageModelPromptOptions, 6 | } from '../interfaces'; 7 | 8 | export class AbortSignalUtilityManager { 9 | private uuidToControllerMap = new Map(); 10 | 11 | public getSignalForUUID(uuid: string): AbortSignal { 12 | if (!this.uuidToControllerMap.has(uuid)) { 13 | this.uuidToControllerMap.set(uuid, new AbortController()); 14 | } 15 | 16 | return this.uuidToControllerMap.get(uuid)!.signal; 17 | } 18 | 19 | public abortSignalForUUID(uuid?: string): void { 20 | if (!uuid) { 21 | return; 22 | } 23 | 24 | const controller = this.uuidToControllerMap.get(uuid); 25 | 26 | if (controller) { 27 | controller.abort(); 28 | this.uuidToControllerMap.delete(uuid); 29 | } 30 | } 31 | 32 | public getWithSignalFromCreateOptions( 33 | input: InternalLanguageModelCreateOptions, 34 | ): InternalLanguageModelCreateOptions { 35 | const { requestUUID, ...rest } = input; 36 | 37 | if (requestUUID) { 38 | return { ...rest, signal: this.getSignalForUUID(requestUUID) }; 39 | } 40 | 41 | return rest as InternalLanguageModelCreateOptions; 42 | } 43 | 44 | public getWithSignalFromPromptOptions( 45 | input: LanguageModelPromptOptions, 46 | ): InternalLanguageModelPromptOptions { 47 | const { requestUUID, ...rest } = input; 48 | 49 | if (requestUUID) { 50 | return { 51 | ...rest, 52 | requestUUID, 53 | signal: this.getSignalForUUID(requestUUID), 54 | }; 55 | } 56 | 57 | return rest as LanguageModelPromptOptions; 58 | } 59 | 60 | public removeUUID(uuid?: string): void { 61 | if (uuid) { 62 | this.uuidToControllerMap.delete(uuid); 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/utility/call-ai-model-entry-point.ts: -------------------------------------------------------------------------------- 1 | import { UTILITY_MESSAGE_TYPES } from './messages.js'; 2 | import { 3 | LanguageModelPromptRole, 4 | LanguageModelPromptType, 5 | LanguageModelPrompt, 6 | } from '../interfaces.js'; 7 | import { LanguageModel } from '../language-model.js'; 8 | import { AbortSignalUtilityManager } from './abortmanager.js'; 9 | import { 10 | isAbortMessage, 11 | isLoadModelMessage, 12 | isPromptMessage, 13 | isStopMessage, 14 | LoadModelMessage, 15 | parseMessageEvent, 16 | PromptMessage, 17 | } from './utility-type-helpers.js'; 18 | 19 | let languageModel: LanguageModel; 20 | const abortSignalManager = new AbortSignalUtilityManager(); 21 | 22 | async function loadModel(message: LoadModelMessage) { 23 | try { 24 | const optionsWithSignal = abortSignalManager.getWithSignalFromCreateOptions( 25 | message.data, 26 | ); 27 | 28 | languageModel = await LanguageModel.create(optionsWithSignal); 29 | } catch (error) { 30 | console.error(error); 31 | 32 | process.parentPort?.postMessage({ 33 | type: UTILITY_MESSAGE_TYPES.ERROR, 34 | data: error, 35 | }); 36 | } finally { 37 | abortSignalManager.removeUUID(message.data.requestUUID); 38 | } 39 | } 40 | 41 | async function generateResponse(message: PromptMessage) { 42 | const { port, data } = message; 43 | 44 | if (!languageModel) { 45 | if (port) { 46 | port.postMessage({ type: 'error', error: 'Language model not loaded.' }); 47 | } 48 | return; 49 | } 50 | 51 | const options = abortSignalManager.getWithSignalFromPromptOptions( 52 | data.options, 53 | ); 54 | 55 | try { 56 | // Format the prompt payload correctly for the language model 57 | const promptPayload: LanguageModelPrompt = { 58 | role: LanguageModelPromptRole.USER, 59 | type: LanguageModelPromptType.TEXT, 60 | content: data.input, 61 | }; 62 | 63 | if (data.stream && message.port) { 64 | // Stream response through the provided port 65 | const readable = languageModel.promptStreaming(promptPayload, options); 66 | const reader = readable.getReader(); 67 | 68 | while (true) { 69 | const { done, value } = await reader.read(); 70 | if (done) break; 71 | message.port.postMessage({ 72 | type: UTILITY_MESSAGE_TYPES.CHUNK, 73 | chunk: value, 74 | }); 75 | } 76 | 77 | message.port.postMessage({ type: UTILITY_MESSAGE_TYPES.DONE }); 78 | } else { 79 | // Handle non-streaming case 80 | process.parentPort?.postMessage({ 81 | type: UTILITY_MESSAGE_TYPES.DONE, 82 | data: await languageModel.prompt(promptPayload, options), 83 | }); 84 | } 85 | } catch (error) { 86 | if (message.port) { 87 | message.port.postMessage({ 88 | type: 'error', 89 | error: error instanceof Error ? error.message : String(error), 90 | }); 91 | } else { 92 | process.parentPort?.postMessage({ 93 | type: UTILITY_MESSAGE_TYPES.ERROR, 94 | data: error, 95 | }); 96 | } 97 | } finally { 98 | // abortSignalManager.removeUUID(options.requestUUID); 99 | } 100 | } 101 | 102 | function stopModel() { 103 | if (languageModel) { 104 | languageModel.destroy(); 105 | } 106 | 107 | process.parentPort.postMessage({ 108 | type: UTILITY_MESSAGE_TYPES.STOPPED, 109 | data: 'Model session reset.', 110 | }); 111 | 112 | process.exit(0); 113 | } 114 | 115 | process.parentPort.on('message', async (messageEvent) => { 116 | const message = parseMessageEvent(messageEvent); 117 | 118 | if (isLoadModelMessage(message)) { 119 | await loadModel(message); 120 | } else if (isPromptMessage(message)) { 121 | await generateResponse(message); 122 | } else if (isStopMessage(message)) { 123 | stopModel(); 124 | } else if (isAbortMessage(message)) { 125 | abortSignalManager.abortSignalForUUID(message.data.requestUUID); 126 | } 127 | }); 128 | -------------------------------------------------------------------------------- /src/utility/messages.ts: -------------------------------------------------------------------------------- 1 | export const UTILITY_MESSAGE_TYPES = { 2 | LOAD_MODEL: 'loadModel', 3 | MODEL_LOADED: 'modelLoaded', 4 | SEND_PROMPT: 'sendPrompt', 5 | REQUEST_ABORTED: 'requestAborted', 6 | STOP: 'stop', 7 | STREAM: 'stream', 8 | DONE: 'done', 9 | ERROR: 'error', 10 | STOPPED: 'stopped', 11 | CHUNK: 'chunk', 12 | }; 13 | -------------------------------------------------------------------------------- /src/utility/utility-type-helpers.ts: -------------------------------------------------------------------------------- 1 | import { 2 | AiProcessModelCreateData, 3 | AiProcessSendPromptData, 4 | } from '../interfaces'; 5 | import { UTILITY_MESSAGE_TYPES } from './messages'; 6 | 7 | export interface UnknownMessage { 8 | type: string; 9 | data: unknown; 10 | port?: Electron.MessagePortMain; 11 | } 12 | 13 | export interface LoadModelMessage extends UnknownMessage { 14 | type: typeof UTILITY_MESSAGE_TYPES.LOAD_MODEL; 15 | data: AiProcessModelCreateData; 16 | } 17 | 18 | export interface PromptMessage extends UnknownMessage { 19 | type: typeof UTILITY_MESSAGE_TYPES.SEND_PROMPT; 20 | data: AiProcessSendPromptData; 21 | } 22 | 23 | export interface AbortMessage extends UnknownMessage { 24 | type: typeof UTILITY_MESSAGE_TYPES.REQUEST_ABORTED; 25 | data: { requestUUID: string }; 26 | } 27 | 28 | export interface StopMessage extends UnknownMessage { 29 | type: typeof UTILITY_MESSAGE_TYPES.STOP; 30 | data: undefined; 31 | } 32 | 33 | function isMessage(message: unknown): message is UnknownMessage { 34 | return typeof message === 'object' && message !== null && 'type' in message; 35 | } 36 | 37 | export function isLoadModelMessage( 38 | message: unknown, 39 | ): message is LoadModelMessage { 40 | return ( 41 | isMessage(message) && 42 | message.type === UTILITY_MESSAGE_TYPES.LOAD_MODEL && 43 | 'data' in message 44 | ); 45 | } 46 | 47 | export function isPromptMessage(message: unknown): message is PromptMessage { 48 | return ( 49 | isMessage(message) && 50 | message.type === UTILITY_MESSAGE_TYPES.SEND_PROMPT && 51 | 'data' in message 52 | ); 53 | } 54 | 55 | export function isAbortMessage(message: unknown): message is AbortMessage { 56 | return ( 57 | isMessage(message) && 58 | message.type === UTILITY_MESSAGE_TYPES.REQUEST_ABORTED && 59 | 'data' in message 60 | ); 61 | } 62 | 63 | export function isStopMessage(message: unknown): message is StopMessage { 64 | return isMessage(message) && message.type === UTILITY_MESSAGE_TYPES.STOP; 65 | } 66 | 67 | /** 68 | * Parses the message event, returning properly typed messages. 69 | * 70 | * @param messageEvent The message event to parse. 71 | * @throws {Error} If the message data is invalid. 72 | * @returns 73 | */ 74 | export function parseMessageEvent(messageEvent: Electron.MessageEvent) { 75 | const data = messageEvent.data.data; 76 | const type = messageEvent.data.type; 77 | const [port] = messageEvent.ports || []; 78 | const message = { type, data, port }; 79 | 80 | if (type === UTILITY_MESSAGE_TYPES.LOAD_MODEL) { 81 | return message as LoadModelMessage; 82 | } 83 | 84 | if (type === UTILITY_MESSAGE_TYPES.SEND_PROMPT) { 85 | return message as PromptMessage; 86 | } 87 | 88 | if (type === UTILITY_MESSAGE_TYPES.REQUEST_ABORTED) { 89 | return message as AbortMessage; 90 | } 91 | 92 | if (type === UTILITY_MESSAGE_TYPES.STOP) { 93 | return message as StopMessage; 94 | } 95 | 96 | throw new Error(`Unknown message type: ${type}`); 97 | } 98 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@tsconfig/node22/tsconfig.json", 3 | "compilerOptions": { 4 | "module": "NodeNext", 5 | "moduleResolution": "nodenext", 6 | "sourceMap": true, 7 | "outDir": "dist", 8 | "types": ["node", "electron"], 9 | "declaration": true, 10 | "declarationMap": true, 11 | "esModuleInterop": true, 12 | "strict": true, 13 | "skipLibCheck": true, 14 | "lib": ["ES2021", "DOM"], 15 | }, 16 | "include": ["src"], 17 | "exclude": ["node_modules", "dist"] 18 | } 19 | -------------------------------------------------------------------------------- /vitest.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vitest/config'; 2 | 3 | export default defineConfig({ 4 | test: { 5 | environment: 'node', 6 | include: ['__tests__/**/*.{test,spec}.{js,ts}'], 7 | globals: true, 8 | }, 9 | }); 10 | --------------------------------------------------------------------------------