13 |
--------------------------------------------------------------------------------
/scripts/postVersion.ts:
--------------------------------------------------------------------------------
1 | import path from "path";
2 | import {fileURLToPath} from "url";
3 | import fs from "fs-extra";
4 |
5 | const __dirname = path.dirname(fileURLToPath(import.meta.url));
6 | const packageJsonPath = path.join(__dirname, "..", "package.json");
7 |
8 | const packageJson = await fs.readJson(packageJsonPath);
9 | const currentVersion = packageJson.version;
10 |
11 | if (packageJson.optionalDependencies != null) {
12 | for (const packageName of Object.keys(packageJson.optionalDependencies)) {
13 | if (!packageName.startsWith("@node-llama-cpp/"))
14 | continue;
15 |
16 | console.info(`Updating optional dependency "${packageName}" to version "${currentVersion}"`);
17 | packageJson.optionalDependencies[packageName] = currentVersion;
18 | }
19 | }
20 |
21 | await fs.writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2), "utf8");
22 |
--------------------------------------------------------------------------------
/scripts/prepareCreateNodeLlamaCppModuleForPublish.ts:
--------------------------------------------------------------------------------
1 | import path from "path";
2 | import {fileURLToPath} from "url";
3 | import yargs from "yargs";
4 | import {hideBin} from "yargs/helpers";
5 | import fs from "fs-extra";
6 |
7 | const __dirname = path.dirname(fileURLToPath(import.meta.url));
8 | const createPackageModulePackageJsonPath = path.join(__dirname, "..", "packages", "create-node-llama-cpp", "package.json");
9 |
10 | const argv = await yargs(hideBin(process.argv))
11 | .option("packageVersion", {
12 | type: "string",
13 | demandOption: true
14 | })
15 | .argv;
16 |
17 | const {packageVersion} = argv;
18 | if (packageVersion === "")
19 | throw new Error("packageVersion is empty");
20 |
21 | const packageJson = await fs.readJson(createPackageModulePackageJsonPath);
22 | packageJson.version = packageVersion;
23 | packageJson.dependencies["node-llama-cpp"] = packageVersion;
24 | delete packageJson.devDependencies;
25 |
26 | await fs.writeJson(createPackageModulePackageJsonPath, packageJson, {spaces: 2});
27 | console.info(`Updated "create-node-llama-cpp/package.json" to version "${packageVersion}"`);
28 |
--------------------------------------------------------------------------------
/scripts/prepareStandalonePrebuiltBinaryModules.ts:
--------------------------------------------------------------------------------
1 | import path from "path";
2 | import {fileURLToPath} from "url";
3 | import fs from "fs-extra";
4 | import {$, cd} from "zx";
5 |
6 | const __dirname = path.dirname(fileURLToPath(import.meta.url));
7 | const packageDirectory = path.join(__dirname, "..", "packages");
8 | const packageScope = "@node-llama-cpp";
9 | const subPackagesDirectory = path.join(packageDirectory, packageScope);
10 |
11 | for (const packageName of await fs.readdir(subPackagesDirectory)) {
12 | const packagePath = path.join(subPackagesDirectory, packageName);
13 | const packagePackageJsonPath = path.join(packagePath, "package.json");
14 |
15 | if ((await fs.stat(packagePath)).isFile())
16 | continue;
17 |
18 | $.verbose = true;
19 | cd(packagePath);
20 | await $`npm ci -f`;
21 | await $`npm run build`;
22 |
23 | const packageJson = await fs.readJson(packagePackageJsonPath);
24 | delete packageJson.devDependencies;
25 | const postinstall = packageJson.scripts?.postinstall;
26 | delete packageJson.scripts;
27 |
28 | if (postinstall != null)
29 | packageJson.scripts = {postinstall};
30 |
31 | await fs.writeJson(packagePackageJsonPath, packageJson, {spaces: 2});
32 | }
33 |
--------------------------------------------------------------------------------
/scripts/resolveLatestReleaseVersion.ts:
--------------------------------------------------------------------------------
1 | import path from "path";
2 | import yargs from "yargs";
3 | import {hideBin} from "yargs/helpers";
4 | import fs from "fs-extra";
5 |
6 | const argv = await yargs(hideBin(process.argv))
7 | .option("saveVersionToFile", {
8 | type: "string"
9 | })
10 | .argv;
11 |
12 | const {saveVersionToFile} = argv;
13 |
14 | const releaseRes = await fetch("https://api.github.com/repos/withcatai/node-llama-cpp/releases/latest");
15 | const release: Release = await releaseRes.json();
16 |
17 | let latestReleaseVersion = release.tag_name;
18 | if (latestReleaseVersion.toLowerCase().startsWith("v"))
19 | latestReleaseVersion = latestReleaseVersion.slice("v".length);
20 |
21 | if (latestReleaseVersion === "")
22 | throw new Error("Could not get latest release version");
23 |
24 | console.log("Latest release version:", latestReleaseVersion);
25 |
26 | if (saveVersionToFile != null) {
27 | const resolvedPath = path.resolve(process.cwd(), saveVersionToFile);
28 |
29 | console.info("Writing latest release version to file:", resolvedPath);
30 | await fs.writeFile(resolvedPath, latestReleaseVersion, "utf8");
31 | }
32 |
33 | type Release = {
34 | tag_name: string
35 | };
36 |
--------------------------------------------------------------------------------
/src/apiDocsIndex.ts:
--------------------------------------------------------------------------------
1 | /** @internal */
2 | import {
3 | _LlamaText
4 | } from "./utils/LlamaText.js";
5 |
6 | /** @internal */
7 | export * from "./index.js";
8 |
9 | /** @internal */
10 | export {
11 | _LlamaText as LlamaText
12 | };
13 |
--------------------------------------------------------------------------------
/src/bindings/consts.ts:
--------------------------------------------------------------------------------
1 | import {BuildGpu} from "./types.js";
2 |
3 | const prettyBuildGpuNames: Record
, string> = {
4 | metal: "Metal",
5 | cuda: "CUDA",
6 | vulkan: "Vulkan"
7 | };
8 |
9 | export function getPrettyBuildGpuName(gpu: BuildGpu | undefined) {
10 | if (gpu == null)
11 | return "unknown GPU";
12 |
13 | if (gpu == false)
14 | return "no GPU";
15 |
16 | return prettyBuildGpuNames[gpu] ?? ('"' + gpu + '"');
17 | }
18 |
--------------------------------------------------------------------------------
/src/bindings/utils/NoBinaryFoundError.ts:
--------------------------------------------------------------------------------
1 | export class NoBinaryFoundError extends Error {
2 | /** @internal */
3 | public constructor(message: string = "NoBinaryFoundError") {
4 | super(message);
5 | }
6 | }
7 |
--------------------------------------------------------------------------------
/src/bindings/utils/asyncEvery.ts:
--------------------------------------------------------------------------------
1 | import {getConsoleLogPrefix} from "../../utils/getConsoleLogPrefix.js";
2 |
3 | /**
4 | * Returns a promise that resolves to true if every promise in the array resolves to true, otherwise false.
5 | * Note that this function will not throw on error and instead will log the error to the console.
6 | */
7 | export async function asyncEvery(promises: Promise[]): Promise {
8 | try {
9 | return (await Promise.all(promises)).every(Boolean);
10 | } catch (err) {
11 | console.error(getConsoleLogPrefix(false, false), err);
12 |
13 | return false;
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/src/bindings/utils/asyncSome.ts:
--------------------------------------------------------------------------------
1 | import {getConsoleLogPrefix} from "../../utils/getConsoleLogPrefix.js";
2 |
3 | /**
4 | * Returns a promise that fulfills as soon as any of the promises return `true`.
5 | * Note that this function will not throw on error and instead will log the error to the console.
6 | */
7 | export async function asyncSome(promises: Promise[]): Promise {
8 | if (promises.length === 0)
9 | return Promise.resolve(false);
10 |
11 | return new Promise((resolve) => {
12 | let fulfilled = 0;
13 |
14 | for (const promise of promises) {
15 | promise
16 | .then((result) => {
17 | if (result)
18 | return void resolve(true);
19 |
20 | fulfilled++;
21 | if (fulfilled === promises.length)
22 | resolve(false);
23 | })
24 | .catch((err) => {
25 | console.error(getConsoleLogPrefix(false, false), err);
26 |
27 | fulfilled++;
28 | if (fulfilled === promises.length)
29 | resolve(false);
30 | });
31 | }
32 | });
33 | }
34 |
--------------------------------------------------------------------------------
/src/bindings/utils/binariesGithubRelease.ts:
--------------------------------------------------------------------------------
1 | import fs from "fs-extra";
2 | import {binariesGithubReleasePath} from "../../config.js";
3 |
4 | type BinariesGithubReleaseFile = {
5 | release: "latest" | string
6 | };
7 |
8 | export async function getBinariesGithubRelease() {
9 | const binariesGithubRelease: BinariesGithubReleaseFile = await fs.readJson(binariesGithubReleasePath);
10 |
11 | return binariesGithubRelease.release;
12 | }
13 |
14 | export async function setBinariesGithubRelease(release: BinariesGithubReleaseFile["release"]) {
15 | const binariesGithubReleaseJson: BinariesGithubReleaseFile = {
16 | release: release
17 | };
18 |
19 | await fs.writeJson(binariesGithubReleasePath, binariesGithubReleaseJson, {
20 | spaces: 4
21 | });
22 | }
23 |
--------------------------------------------------------------------------------
/src/bindings/utils/getCanUsePrebuiltBinaries.ts:
--------------------------------------------------------------------------------
1 | import {builtinLlamaCppGitHubRepo, builtinLlamaCppRelease} from "../../config.js";
2 | import {getClonedLlamaCppRepoReleaseInfo} from "./cloneLlamaCppRepo.js";
3 |
4 | export async function getCanUsePrebuiltBinaries() {
5 | const clonedLlamaCppRepoReleaseInfo = await getClonedLlamaCppRepoReleaseInfo();
6 |
7 | return clonedLlamaCppRepoReleaseInfo == null || (
8 | clonedLlamaCppRepoReleaseInfo.tag === builtinLlamaCppRelease &&
9 | clonedLlamaCppRepoReleaseInfo.llamaCppGithubRepo === builtinLlamaCppGitHubRepo
10 | );
11 | }
12 |
--------------------------------------------------------------------------------
/src/bindings/utils/getLlamaWithoutBackend.ts:
--------------------------------------------------------------------------------
1 | import {withLock} from "lifecycle-utils";
2 | import {getLlamaForOptions} from "../getLlama.js";
3 | import {LlamaLogLevel} from "../types.js";
4 | import {Llama} from "../Llama.js";
5 |
6 | let sharedLlamaWithoutBackend: Llama | null = null;
7 |
8 | /**
9 | * This is used to access various methods in the addon side without actually using a backend
10 | */
11 | export async function getLlamaWithoutBackend() {
12 | if (sharedLlamaWithoutBackend != null)
13 | return sharedLlamaWithoutBackend;
14 |
15 | return await withLock(getLlamaWithoutBackend, "loadAddon", async () => {
16 | if (sharedLlamaWithoutBackend != null)
17 | return sharedLlamaWithoutBackend;
18 |
19 | sharedLlamaWithoutBackend = await getLlamaForOptions({
20 | gpu: false,
21 | progressLogs: false,
22 | logLevel: LlamaLogLevel.error,
23 | build: "never",
24 | usePrebuiltBinaries: true,
25 | vramPadding: 0
26 | }, {
27 | skipLlamaInit: true
28 | });
29 |
30 | return sharedLlamaWithoutBackend;
31 | });
32 | }
33 |
--------------------------------------------------------------------------------
/src/bindings/utils/getPlatform.ts:
--------------------------------------------------------------------------------
1 | import process from "process";
2 |
3 | export function getPlatform() {
4 | switch (process.platform) {
5 | case "win32":
6 | case "cygwin":
7 | return "win";
8 |
9 | case "linux":
10 | case "android":
11 | return "linux";
12 |
13 | case "darwin":
14 | return "mac";
15 | }
16 |
17 | return process.platform;
18 | }
19 |
20 | export type BinaryPlatform = ReturnType;
21 |
--------------------------------------------------------------------------------
/src/bindings/utils/getPlatformInfo.ts:
--------------------------------------------------------------------------------
1 | import os from "os";
2 | import {getPlatform} from "./getPlatform.js";
3 | import {getLinuxDistroInfo} from "./getLinuxDistroInfo.js";
4 |
5 | export async function getPlatformInfo(): Promise<{name: string, version: string}> {
6 | const currentPlatform = getPlatform();
7 |
8 | if (currentPlatform === "mac")
9 | return {
10 | name: "macOS",
11 | version: os.release()
12 | };
13 | else if (currentPlatform === "linux") {
14 | const linuxDistroInfo = await getLinuxDistroInfo();
15 |
16 | return {
17 | name: linuxDistroInfo.name,
18 | version: linuxDistroInfo.version
19 | };
20 | } else if (currentPlatform === "win")
21 | return {
22 | name: "Windows",
23 | version: os.release()
24 | };
25 |
26 | return {
27 | name: "Unknown",
28 | version: os.release()
29 | };
30 | }
31 |
32 | export type BinaryPlatformInfo = Awaited>;
33 |
--------------------------------------------------------------------------------
/src/bindings/utils/hasBuildingFromSourceDependenciesInstalled.ts:
--------------------------------------------------------------------------------
1 | import which from "which";
2 | import {asyncEvery} from "./asyncEvery.js";
3 |
4 | export async function hasBuildingFromSourceDependenciesInstalled() {
5 | return await asyncEvery([
6 | hasGit(),
7 | hasNpm()
8 | ]);
9 | }
10 |
11 | export async function hasGit() {
12 | try {
13 | const resolvedPath = await which("git");
14 | return resolvedPath !== "";
15 | } catch (err) {
16 | return false;
17 | }
18 | }
19 |
20 | export async function hasNpm() {
21 | try {
22 | const resolvedPath = await which("npm");
23 | return resolvedPath !== "";
24 | } catch (err) {
25 | return false;
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/src/bindings/utils/lastBuildInfo.ts:
--------------------------------------------------------------------------------
1 | import fs from "fs-extra";
2 | import {lastBuildInfoJsonPath} from "../../config.js";
3 |
4 | type LastBuildInfo = {
5 | folderName: string
6 | };
7 |
8 | export async function getLastBuildInfo() {
9 | try {
10 | const buildInfo: LastBuildInfo = await fs.readJson(lastBuildInfoJsonPath);
11 |
12 | return buildInfo;
13 | } catch (err) {
14 | return null;
15 | }
16 | }
17 |
18 | export async function setLastBuildInfo(buildInfo: LastBuildInfo) {
19 | await fs.writeJson(lastBuildInfoJsonPath, buildInfo, {
20 | spaces: 4
21 | });
22 | }
23 |
--------------------------------------------------------------------------------
/src/bindings/utils/resolveActualBindingBinaryPath.ts:
--------------------------------------------------------------------------------
1 | import path from "path";
2 | import fs from "fs-extra";
3 | import {runningInElectron} from "../../utils/runtime.js";
4 |
5 | export async function resolveActualBindingBinaryPath(binaryPath: string) {
6 | const absolutePath = path.resolve(binaryPath);
7 | if (!runningInElectron)
8 | return absolutePath;
9 |
10 | const fixedAsarPath = absolutePath.replace(".asar" + path.sep, ".asar.unpacked" + path.sep);
11 | try {
12 | if (await fs.pathExists(fixedAsarPath))
13 | return fixedAsarPath;
14 |
15 | return absolutePath;
16 | } catch (err) {
17 | return absolutePath;
18 | }
19 | }
20 |
--------------------------------------------------------------------------------
/src/bindings/utils/testCmakeBinary.ts:
--------------------------------------------------------------------------------
1 | import process from "process";
2 | import {execFile} from "node:child_process";
3 | import path from "path";
4 | import {fileURLToPath} from "url";
5 | import fs from "fs-extra";
6 |
7 | const __dirname = path.dirname(fileURLToPath(import.meta.url));
8 |
9 | export async function testCmakeBinary(cmakeBinaryPath?: string, {
10 | cwd = __dirname, env = process.env
11 | }: {
12 | cwd?: string, env?: typeof process.env
13 | } = {}) {
14 | if (cmakeBinaryPath == null || !(await fs.pathExists(cmakeBinaryPath)))
15 | return false;
16 |
17 | return new Promise((resolve, reject) => {
18 | const child = execFile(cmakeBinaryPath, ["--version"], {
19 | cwd,
20 | env,
21 | windowsHide: true
22 | });
23 |
24 | child.on("exit", (code) => {
25 | if (code == 0)
26 | resolve(true);
27 | else
28 | reject(false);
29 | });
30 | child.on("error", reject);
31 | child.on("disconnect", () => resolve(false));
32 | child.on("close", (code) => {
33 | if (code == 0)
34 | resolve(true);
35 | else
36 | resolve(false);
37 | });
38 | });
39 | }
40 |
--------------------------------------------------------------------------------
/src/chatWrappers/EmptyChatWrapper.ts:
--------------------------------------------------------------------------------
1 | import {ChatWrapper} from "../ChatWrapper.js";
2 |
3 | export class EmptyChatWrapper extends ChatWrapper {
4 | public readonly wrapperName: string = "Empty";
5 | }
6 |
--------------------------------------------------------------------------------
/src/chatWrappers/generic/utils/UniqueIdGenerator.ts:
--------------------------------------------------------------------------------
1 | export class UniqueIdGenerator {
2 | public readonly antiText: string;
3 | private readonly _ids = new Set();
4 |
5 | public constructor(antiText: string) {
6 | this.antiText = antiText;
7 | }
8 |
9 | public generateId(numbersOnly: boolean = false): string {
10 | let id: string;
11 |
12 | do {
13 | if (numbersOnly) {
14 | do {
15 | id = (
16 | Math.random()
17 | .toString(10)
18 | .slice(2)
19 | .slice(0, String(Number.MAX_SAFE_INTEGER).length - 1)
20 | );
21 | } while (id.startsWith("0"));
22 | } else
23 | id = "W" + (
24 | Math.random()
25 | .toString(36)
26 | .slice(2)
27 | ) + "W";
28 | } while (this._ids.has(id) || this.antiText.includes(id));
29 |
30 | this._ids.add(id);
31 |
32 | return id;
33 | }
34 |
35 | public removeId(id: string) {
36 | this._ids.delete(id);
37 | }
38 | }
39 |
40 |
--------------------------------------------------------------------------------
/src/chatWrappers/generic/utils/getFirstValidResult.ts:
--------------------------------------------------------------------------------
1 | /**
2 | * Call the functions in the array one by one and return the result of the first one that doesn't throw an error.
3 | *
4 | * If all functions throw an error, throw the error of the last function.
5 | */
6 | export function getFirstValidResult any)[]>(options: T): ReturnType {
7 | for (let i = 0; i < options.length; i++) {
8 | if (i === options.length - 1)
9 | return options[i]!();
10 |
11 | try {
12 | return options[i]!();
13 | } catch (err) {
14 | // do nothing
15 | }
16 | }
17 |
18 | throw new Error("All options failed");
19 | }
20 |
--------------------------------------------------------------------------------
/src/chatWrappers/utils/getModelLinageNames.ts:
--------------------------------------------------------------------------------
1 | import {GgufMetadata} from "../../gguf/types/GgufMetadataTypes.js";
2 |
3 | export function getModelLinageNames(ggufMetadata?: GgufMetadata) {
4 | const res: string[][] = [];
5 |
6 | if (ggufMetadata == null)
7 | return res;
8 |
9 | const currentModelInfo = [ggufMetadata?.general?.name, ggufMetadata?.general?.basename]
10 | .filter((v): v is string => v != null);
11 | if (currentModelInfo.length > 0)
12 | res.push(currentModelInfo);
13 |
14 | if (typeof ggufMetadata?.general?.base_model?.count === "number") {
15 | for (let i = 0; i < ggufMetadata.general.base_model.count; i++) {
16 | const baseModel = ggufMetadata.general.base_model[String(i) as `${bigint}`];
17 | if (baseModel?.name != null)
18 | res.push([baseModel.name]);
19 | }
20 | }
21 |
22 | return res;
23 | }
24 |
--------------------------------------------------------------------------------
/src/chatWrappers/utils/isLlama3_2LightweightModel.ts:
--------------------------------------------------------------------------------
1 | import {ChatWrapperCheckModelCompatibilityParams} from "../../types.js";
2 | import {includesText} from "../../utils/includesText.js";
3 | import {getModelLinageNames} from "./getModelLinageNames.js";
4 |
5 | export function isLlama3_2LightweightModel(options: ChatWrapperCheckModelCompatibilityParams) {
6 | const isLlama3_2 = getModelLinageNames(options.fileInfo?.metadata)
7 | .some((modelNames) => includesText(modelNames, ["llama 3.2", "llama-3.2", "llama3.2"]));
8 | const isSmallModel = (["1B", "3B"] as string[]).includes(options.fileInfo?.metadata?.general?.size_label ?? "");
9 |
10 | return isLlama3_2 && isSmallModel;
11 | }
12 |
--------------------------------------------------------------------------------
/src/chatWrappers/utils/jsonDumps.ts:
--------------------------------------------------------------------------------
1 | /**
2 | * Like `JSON.stringify` but results in a value formatted in the format that Python produces when using `json.dumps(value)`.
3 | *
4 | * We need to format results this way since this is what many models use in their training data,
5 | * so this is what many models expect to have in their context state.
6 | */
7 | export function jsonDumps(value: any) {
8 | return JSON.stringify(value, null, 1)
9 | .split("\n")
10 | .map((line) => {
11 | line = line.trim();
12 |
13 | if (line.endsWith(","))
14 | line += " ";
15 |
16 | return line;
17 | })
18 | .join("");
19 | }
20 |
--------------------------------------------------------------------------------
/src/cli/commands/inspect/InspectCommand.ts:
--------------------------------------------------------------------------------
1 | import {CommandModule} from "yargs";
2 | import {withCliCommandDescriptionDocsUrl} from "../../utils/withCliCommandDescriptionDocsUrl.js";
3 | import {documentationPageUrls} from "../../../config.js";
4 | import {InspectGgufCommand} from "./commands/InspectGgufCommand.js";
5 | import {InspectGpuCommand} from "./commands/InspectGpuCommand.js";
6 | import {InspectMeasureCommand} from "./commands/InspectMeasureCommand.js";
7 | import {InspectEstimateCommand} from "./commands/InspectEstimateCommand.js";
8 |
9 | type InspectCommand = {
10 | // no options for now
11 | };
12 |
13 | export const InspectCommand: CommandModule = {
14 | command: "inspect ",
15 | describe: withCliCommandDescriptionDocsUrl(
16 | "Inspect the inner workings of `node-llama-cpp`",
17 | documentationPageUrls.CLI.Inspect.index
18 | ),
19 | builder(yargs) {
20 | return yargs
21 | .command(InspectGpuCommand)
22 | .command(InspectGgufCommand)
23 | .command(InspectMeasureCommand)
24 | .command(InspectEstimateCommand);
25 | },
26 | async handler() {
27 | // this function must exist, even though we do nothing here
28 | }
29 | };
30 |
--------------------------------------------------------------------------------
/src/cli/commands/source/SourceCommand.ts:
--------------------------------------------------------------------------------
1 | import {CommandModule} from "yargs";
2 | import {withCliCommandDescriptionDocsUrl} from "../../utils/withCliCommandDescriptionDocsUrl.js";
3 | import {documentationPageUrls} from "../../../config.js";
4 | import {DownloadCommand} from "./commands/DownloadCommand.js";
5 | import {BuildCommand} from "./commands/BuildCommand.js";
6 | import {ClearCommand} from "./commands/ClearCommand.js";
7 |
8 | type SourceCommand = {
9 | // no options for now
10 | };
11 |
12 | export const SourceCommand: CommandModule = {
13 | command: "source ",
14 | describe: withCliCommandDescriptionDocsUrl(
15 | "Manage `llama.cpp` source code",
16 | documentationPageUrls.CLI.Source.index
17 | ),
18 | builder(yargs) {
19 | return yargs
20 | .command(DownloadCommand)
21 | .command(BuildCommand)
22 | .command(ClearCommand);
23 | },
24 | async handler() {
25 | // this function must exist, even though we do nothing here
26 | }
27 | };
28 |
--------------------------------------------------------------------------------
/src/cli/projectTemplates.ts:
--------------------------------------------------------------------------------
1 | export type ProjectTemplateOption = {
2 | title: string,
3 | name: string,
4 | titleFormat?(title: string): string,
5 | description?: string
6 | };
7 | export const projectTemplates: ProjectTemplateOption[] = [{
8 | title: "Node + TypeScript",
9 | name: "node-typescript",
10 | description: "A Node.js project with TypeScript using vite-node, some ESLint configuration, basic setup with a selected model file, and a working example of a simple usage of node-llama-cpp with the model"
11 | }, {
12 | title: "Electron + TypeScript + React",
13 | name: "electron-typescript-react",
14 | description: "An Electron project with TypeScript and React using Vite-Electron, some ESLint configuration, basic setup with a selected model file, and a working example of a simple usage of node-llama-cpp with the model"
15 | }];
16 |
--------------------------------------------------------------------------------
/src/cli/startCreateCli.ts:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env node
2 |
3 | import yargs from "yargs";
4 | import {hideBin} from "yargs/helpers";
5 | import {setIsRunningFromCLI} from "../state.js";
6 | import {CreateCliCommand} from "./commands/InitCommand.js";
7 |
8 | /** @internal */
9 | export function _startCreateCli({
10 | cliBinName,
11 | packageVersion,
12 | _enable
13 | }: {
14 | cliBinName: string,
15 | packageVersion: string,
16 | _enable?: any
17 | }) {
18 | if (_enable !== Symbol.for("internal"))
19 | return;
20 |
21 | setIsRunningFromCLI(true);
22 |
23 | const yarg = yargs(hideBin(process.argv));
24 |
25 | yarg
26 | .scriptName(cliBinName)
27 | .usage("Usage: $0 [options]")
28 | .command(CreateCliCommand)
29 | .demandCommand(1)
30 | .strict()
31 | .strictCommands()
32 | .alias("v", "version")
33 | .help("h")
34 | .alias("h", "help")
35 | .version(packageVersion)
36 | .wrap(Math.min(100, yarg.terminalWidth()))
37 | .parse();
38 | }
39 |
--------------------------------------------------------------------------------
/src/cli/utils/getReadablePath.ts:
--------------------------------------------------------------------------------
1 | import os from "os";
2 | import path from "path";
3 |
4 | export function getReadablePath(fsPath: string) {
5 | const resolvedPath = path.resolve(process.cwd(), fsPath);
6 |
7 | if (process.platform === "win32" || process.platform === "cygwin")
8 | return resolvedPath;
9 |
10 | let homedir = os.homedir();
11 | if (!homedir.endsWith("/"))
12 | homedir += "/";
13 |
14 | if (resolvedPath.startsWith(homedir))
15 | return "~" + resolvedPath.slice(homedir.length - "/".length);
16 |
17 | return resolvedPath;
18 | }
19 |
--------------------------------------------------------------------------------
/src/cli/utils/isRunningUnderRosetta.ts:
--------------------------------------------------------------------------------
1 | import path from "path";
2 | import {fileURLToPath} from "url";
3 | import process from "process";
4 | import {getPlatform} from "../../bindings/utils/getPlatform.js";
5 | import {spawnCommand} from "../../utils/spawnCommand.js";
6 |
7 | const __dirname = path.dirname(fileURLToPath(import.meta.url));
8 |
9 | export async function isRunningUnderRosetta() {
10 | const platform = getPlatform();
11 |
12 | // // only check for rosetta on macOS when x64 is detected
13 | if (platform !== "mac" || process.arch !== "x64")
14 | return false;
15 |
16 | try {
17 | const res = await spawnCommand("sysctl", ["-n", "sysctl.proc_translated"], __dirname, process.env, false);
18 |
19 | return res.combinedStd.trim() === "1";
20 | } catch (err) {
21 | return false;
22 | }
23 | }
24 |
--------------------------------------------------------------------------------
/src/cli/utils/logUsedGpuTypeOption.ts:
--------------------------------------------------------------------------------
1 | import chalk from "chalk";
2 | import {BuildGpu} from "../../bindings/types.js";
3 | import {getPrettyBuildGpuName} from "../../bindings/consts.js";
4 |
5 | export function logUsedGpuTypeOption(gpu: BuildGpu) {
6 | if (gpu == false)
7 | console.log(`${chalk.yellow("GPU:")} disabled`);
8 | else
9 | console.log(`${chalk.yellow("GPU:")} ${getPrettyBuildGpuName(gpu)}`);
10 | }
11 |
--------------------------------------------------------------------------------
/src/cli/utils/printModelDestination.ts:
--------------------------------------------------------------------------------
1 | import chalk from "chalk";
2 | import {ResolveModelDestination} from "../../utils/resolveModelDestination.js";
3 | import {getReadablePath} from "./getReadablePath.js";
4 |
5 | export function printModelDestination(modelDestination: ResolveModelDestination) {
6 | if (modelDestination.type === "url")
7 | console.info(`${chalk.yellow("URL:")} ${modelDestination.url}`);
8 | else if (modelDestination.type === "uri")
9 | console.info(`${chalk.yellow("URI:")} ${modelDestination.uri}`);
10 | else
11 | console.info(`${chalk.yellow("File:")} ${getReadablePath(modelDestination.path)}`);
12 | }
13 |
--------------------------------------------------------------------------------
/src/cli/utils/renderModelCompatibilityPercentageWithColors.ts:
--------------------------------------------------------------------------------
1 | import chalk from "chalk";
2 |
3 | export function renderModelCompatibilityPercentageWithColors(percentage: number, {
4 | greenBright = 100,
5 | green = 95,
6 | yellow = 85,
7 | yellowBright = 75
8 | }: {
9 | greenBright?: number,
10 | green?: number,
11 | yellow?: number,
12 | yellowBright?: number
13 | } = {}): string {
14 | const percentageText = String(Math.floor(percentage)) + "%";
15 |
16 | if (percentage >= greenBright)
17 | return chalk.greenBright(percentageText);
18 | else if (percentage >= green)
19 | return chalk.green(percentageText);
20 | else if (percentage >= yellow)
21 | return chalk.yellow(percentageText);
22 | else if (percentage >= yellowBright)
23 | return chalk.yellowBright(percentageText);
24 |
25 | return chalk.red(percentageText);
26 | }
27 |
--------------------------------------------------------------------------------
/src/cli/utils/resolveHeaderFlag.ts:
--------------------------------------------------------------------------------
1 | export function resolveHeaderFlag(header?: string[] | string) {
2 | if (typeof header === "string")
3 | header = [header];
4 |
5 | if (header == null || header.length === 0)
6 | return {};
7 |
8 | const res: Record = {};
9 |
10 | for (const headerItem of header) {
11 | const colonIndex = headerItem.indexOf(":");
12 |
13 | if (colonIndex < 0)
14 | throw new Error(`Invalid header item: ${headerItem}`);
15 |
16 | const key = headerItem.slice(0, colonIndex).trim();
17 |
18 | if (Object.hasOwn(res, key))
19 | throw new Error(`Duplicate header key: ${key}`);
20 |
21 | let value = headerItem.slice(colonIndex + 1);
22 | if (value.startsWith(" "))
23 | value = value.slice(1);
24 |
25 | res[key] = value;
26 | }
27 |
28 | return res;
29 | }
30 |
--------------------------------------------------------------------------------
/src/cli/utils/toBytes.ts:
--------------------------------------------------------------------------------
1 | import bytes from "bytes";
2 |
3 | export function toBytes(value: number): string {
4 | return bytes(value) ?? String(value);
5 | }
6 |
--------------------------------------------------------------------------------
/src/cli/utils/withCliCommandDescriptionDocsUrl.ts:
--------------------------------------------------------------------------------
1 | import {getIsInDocumentationMode} from "../../state.js";
2 | import {documentationPageUrls} from "../../config.js";
3 |
4 | export function withCliCommandDescriptionDocsUrl(description: string, docsUrl: string) {
5 | const isInDocumentationMode = getIsInDocumentationMode();
6 |
7 | if (isInDocumentationMode)
8 | return description;
9 |
10 | return [
11 | description,
12 | docsUrl
13 | ].join("\n").trim();
14 | }
15 |
16 | export function withoutCliCommandDescriptionDocsUrl(description: string | boolean) {
17 | if (typeof description !== "string")
18 | return description;
19 |
20 | const lines = description.split("\n");
21 | if (lines.length > 0 && lines[lines.length - 1]!.startsWith(documentationPageUrls.CLI.index))
22 | return lines
23 | .slice(0, -1)
24 | .join("\n")
25 | .trim();
26 |
27 | return description;
28 | }
29 |
--------------------------------------------------------------------------------
/src/commands.ts:
--------------------------------------------------------------------------------
1 | import {BuildLlamaCppCommand} from "./cli/commands/source/commands/BuildCommand.js";
2 | import {DownloadLlamaCppCommand} from "./cli/commands/source/commands/DownloadCommand.js";
3 | import {ClearLlamaCppBuildCommand} from "./cli/commands/source/commands/ClearCommand.js";
4 | import {_startCreateCli} from "./cli/startCreateCli.js";
5 | import {getBuildDefaults} from "./utils/getBuildDefaults.js";
6 |
7 | export {BuildLlamaCppCommand, DownloadLlamaCppCommand, ClearLlamaCppBuildCommand, getBuildDefaults};
8 |
9 | /** @internal */
10 | export {_startCreateCli};
11 |
--------------------------------------------------------------------------------
/src/consts.ts:
--------------------------------------------------------------------------------
1 | import isUnicodeSupported from "is-unicode-supported";
2 |
3 | const unicodeSupported = isUnicodeSupported();
4 |
5 | export const maxRecentDetokenizerTokens = 3;
6 | export const UNKNOWN_UNICODE_CHAR = "\ufffd";
7 | export const clockChar = unicodeSupported
8 | ? "\u25f7"
9 | : "+";
10 | export const arrowChar = unicodeSupported
11 | ? "\u276f"
12 | : ">";
13 |
--------------------------------------------------------------------------------
/src/evaluator/LlamaChat/utils/LlamaFunctionCallValidationError.ts:
--------------------------------------------------------------------------------
1 | import {ChatModelFunctions} from "../../../types.js";
2 | import {ChatWrapper} from "../../../ChatWrapper.js";
3 |
4 |
5 | export class LlamaFunctionCallValidationError extends Error {
6 | public readonly functions: Functions;
7 | public readonly chatWrapper: ChatWrapper;
8 | public readonly callText: string;
9 |
10 | public constructor(message: string, functions: Functions, chatWrapper: ChatWrapper, callText: string) {
11 | super(message);
12 |
13 | this.functions = functions;
14 | this.chatWrapper = chatWrapper;
15 | this.callText = callText;
16 | }
17 | }
18 |
--------------------------------------------------------------------------------
/src/evaluator/LlamaChatSession/utils/defineChatSessionFunction.ts:
--------------------------------------------------------------------------------
1 | import {GbnfJsonSchema, GbnfJsonSchemaToType} from "../../../utils/gbnfJson/types.js";
2 | import {ChatSessionModelFunction} from "../../../types.js";
3 |
4 | /**
5 | * Define a function that can be used by the model in a chat session, and return it.
6 | *
7 | * This is a helper function to facilitate defining functions with full TypeScript type information.
8 | *
9 | * The handler function can return a Promise, and the return value will be awaited before being returned to the model.
10 | * @param functionDefinition
11 | */
12 | export function defineChatSessionFunction({
13 | description,
14 | params,
15 | handler
16 | }: {
17 | description?: string,
18 | params?: Readonly,
19 | handler: (params: GbnfJsonSchemaToType) => Promise | any
20 | }): ChatSessionModelFunction {
21 | return {
22 | description,
23 | params,
24 | handler
25 | };
26 | }
27 |
--------------------------------------------------------------------------------
/src/evaluator/LlamaContext/utils/batchItemsPrioritizationStrategies/firstInFirstOutStrategy.ts:
--------------------------------------------------------------------------------
1 | import {BatchItem, PrioritizedBatchItem} from "../../types.js";
2 |
3 | export function firstInFirstOutStrategy({items, size}: {items: readonly BatchItem[], size: number}) {
4 | const res: PrioritizedBatchItem[] = [];
5 |
6 | const sortedItems = items
7 | .slice()
8 | .sort((a, b) => b.evaluationPriority - a.evaluationPriority);
9 |
10 | let leftFreeTokens = size;
11 | for (const item of sortedItems) {
12 | const processAmount = Math.min(item.tokens.length, leftFreeTokens);
13 | res.push({item, processAmount});
14 | leftFreeTokens -= processAmount;
15 |
16 | if (leftFreeTokens === 0)
17 | break;
18 | }
19 |
20 | return res;
21 | }
22 |
--------------------------------------------------------------------------------
/src/evaluator/LlamaContext/utils/resolveBatchItemsPrioritizationStrategy.ts:
--------------------------------------------------------------------------------
1 | import {BatchingOptions} from "../types.js";
2 | import {maximumParallelismStrategy} from "./batchItemsPrioritizationStrategies/maximumParallelismStrategy.js";
3 | import {firstInFirstOutStrategy} from "./batchItemsPrioritizationStrategies/firstInFirstOutStrategy.js";
4 |
5 | export function resolveBatchItemsPrioritizationStrategy(strategy: Required["itemPrioritizationStrategy"]) {
6 | if (strategy instanceof Function)
7 | return strategy;
8 | else if (strategy === "maximumParallelism")
9 | return maximumParallelismStrategy;
10 | else if (strategy === "firstInFirstOut")
11 | return firstInFirstOutStrategy;
12 |
13 | void (strategy satisfies never);
14 |
15 | throw new Error(`Unknown batch items prioritize strategy: ${strategy}`);
16 | }
17 |
--------------------------------------------------------------------------------
/src/gguf/consts.ts:
--------------------------------------------------------------------------------
1 | import retry from "async-retry";
2 |
3 | export const ggufDefaultFetchRetryOptions: retry.Options = {
4 | retries: 10,
5 | factor: 2,
6 | minTimeout: 1000,
7 | maxTimeout: 1000 * 16
8 | } as const;
9 |
10 | export const defaultExtraAllocationSize = 1024 * 1024 * 4; // 4MB
11 |
12 | export const noDirectSubNestingGGufMetadataKeys: readonly string[] = [
13 | "general.license",
14 | "tokenizer.chat_template"
15 | ];
16 |
--------------------------------------------------------------------------------
/src/gguf/errors/InvalidGgufMagicError.ts:
--------------------------------------------------------------------------------
1 | export class InvalidGgufMagicError extends Error {
2 | public constructor(expectedGgufMagic: string, actualGgufMagic: string) {
3 | super(`Invalid GGUF magic. Expected "${expectedGgufMagic}" but got "${actualGgufMagic}".`);
4 | }
5 | }
6 |
--------------------------------------------------------------------------------
/src/gguf/errors/UnsupportedGgufValueTypeError.ts:
--------------------------------------------------------------------------------
1 | export class UnsupportedGgufValueTypeError extends Error {
2 | public readonly ggufValueType: number;
3 |
4 | public constructor(ggufValueType: number) {
5 | super(`Unsupported GGUF value type "${ggufValueType}"`);
6 |
7 | Object.defineProperty(this, "ggufValueType" satisfies keyof this, {enumerable: false});
8 |
9 | this.ggufValueType = ggufValueType;
10 | }
11 | }
12 |
--------------------------------------------------------------------------------
/src/gguf/insights/utils/getRamUsageFromUnifiedVram.ts:
--------------------------------------------------------------------------------
1 | export function getRamUsageFromUnifiedVram(vramUsage: number, vramState: {total: number, free: number, unifiedSize: number}) {
2 | const onlyVramSize = vramState.total - vramState.unifiedSize;
3 | const existingUsage = Math.max(0, vramState.total - vramState.free);
4 |
5 | const unifiedRamUsage = Math.min(vramState.unifiedSize, Math.max(0, vramUsage - Math.max(0, onlyVramSize - existingUsage)));
6 |
7 | return unifiedRamUsage;
8 | }
9 |
--------------------------------------------------------------------------------
/src/gguf/insights/utils/scoreLevels.ts:
--------------------------------------------------------------------------------
1 | export function scoreLevels(num: number, levels: {start: number, end?: number, points: number}[]) {
2 | let res = 0;
3 |
4 | for (let i = 0; i < levels.length; i++) {
5 | const level = levels[i]!;
6 | const start = level.start;
7 | const end = level.end ?? levels[i + 1]?.start ?? Math.max(start, num);
8 |
9 | if (num < start)
10 | break;
11 | else if (num >= end)
12 | res += level.points;
13 | else
14 | res += level.points * ((num - start) / (end - start));
15 | }
16 |
17 | return res;
18 | }
19 |
--------------------------------------------------------------------------------
/src/gguf/parser/GgufV3Parser.ts:
--------------------------------------------------------------------------------
1 | import {GgufV2Parser} from "./GgufV2Parser.js";
2 |
3 | export class GgufV3Parser extends GgufV2Parser {
4 | // the implementation is the same as version 2 for now
5 | }
6 |
--------------------------------------------------------------------------------
/src/gguf/utils/GgufReadOffset.ts:
--------------------------------------------------------------------------------
1 | export class GgufReadOffset {
2 | public offset: number;
3 |
4 | public constructor(offset: number | GgufReadOffset) {
5 | if (offset instanceof GgufReadOffset)
6 | this.offset = offset.offset;
7 | else
8 | this.offset = offset;
9 | }
10 |
11 | public moveBy(amount: number) {
12 | this.offset += amount;
13 | }
14 |
15 | public static resolveReadOffset(offset: number | GgufReadOffset) {
16 | if (offset instanceof GgufReadOffset)
17 | return offset;
18 |
19 | return new GgufReadOffset(offset);
20 | }
21 | }
22 |
--------------------------------------------------------------------------------
/src/gguf/utils/getGgufFileTypeName.ts:
--------------------------------------------------------------------------------
1 | import {GgufFileType} from "../types/GgufMetadataTypes.js";
2 |
3 | const fileTypeNumberToNameMap = new Map();
4 | for (const [key, value] of Object.entries(GgufFileType)) {
5 | if (typeof value === "number")
6 | fileTypeNumberToNameMap.set(value, key as keyof typeof GgufFileType);
7 | }
8 |
9 | /**
10 | * Convert a GGUF file type number to its corresponding type name
11 | */
12 | export function getGgufFileTypeName(fileType?: number) {
13 | return fileTypeNumberToNameMap.get(fileType!) ?? undefined;
14 | }
15 |
--------------------------------------------------------------------------------
/src/gguf/utils/getGgufMetadataArchitectureData.ts:
--------------------------------------------------------------------------------
1 | import {GgufArchitectureType, GgufMetadata} from "../types/GgufMetadataTypes.js";
2 | import {MergeOptionalUnionTypes} from "../../utils/mergeUnionTypes.js";
3 |
4 | export function getGgufMetadataArchitectureData(ggufMetadata: GgufMetadata): (
5 | GgufArchitectureType extends T
6 | ? MergeOptionalUnionTypes>
7 | : GgufMetadata[T]
8 | ) {
9 | return ggufMetadata[ggufMetadata.general?.architecture] ?? {} as any;
10 | }
11 |
--------------------------------------------------------------------------------
/src/gguf/utils/getGgufMetadataKeyValue.ts:
--------------------------------------------------------------------------------
1 | export function getGgufMetadataKeyValue(metadata: Record, key: string) {
2 | return readMedataKey(metadata, key.split("."));
3 | }
4 |
5 | function readMedataKey(metadata: Record, keyParts: string[]): any {
6 | for (const [metadataKey, value] of Object.entries(metadata)) {
7 | const matchLength = checkMatchLength(metadataKey, keyParts);
8 | if (matchLength === 0)
9 | continue;
10 |
11 | if (matchLength === keyParts.length)
12 | return value;
13 |
14 | const res = readMedataKey(value, keyParts.slice(matchLength));
15 | if (res !== undefined)
16 | return res;
17 | }
18 |
19 | return undefined;
20 | }
21 |
22 | function checkMatchLength(metadataKey: string, keyParts: string[]) {
23 | const metadataKeyParts = metadataKey.split(".");
24 |
25 | if (metadataKeyParts.length > keyParts.length)
26 | return 0;
27 |
28 | for (let i = 0; i < metadataKeyParts.length; i++) {
29 | if (metadataKeyParts[i] !== keyParts[i])
30 | return 0;
31 | }
32 |
33 | return metadataKeyParts.length;
34 | }
35 |
--------------------------------------------------------------------------------
/src/gguf/utils/normalizeGgufDownloadUrl.ts:
--------------------------------------------------------------------------------
1 | import {ModelDownloadEndpoints, resolveHuggingFaceEndpoint} from "../../utils/modelDownloadEndpoints.js";
2 |
3 | export function normalizeGgufDownloadUrl(url: string, endpoints?: ModelDownloadEndpoints) {
4 | const parsedUrl = new URL(url);
5 |
6 | if (parsedUrl.hostname === "huggingface.co" || parsedUrl.hostname === "hf.co" ||
7 | parsedUrl.hostname === (new URL(resolveHuggingFaceEndpoint(endpoints))).hostname
8 | ) {
9 | const pathnameParts = parsedUrl.pathname.split("/");
10 |
11 | if (pathnameParts.length > 3 && pathnameParts[3] === "blob") {
12 | const newUrl = new URL(url);
13 | pathnameParts[3] = "resolve";
14 | newUrl.pathname = pathnameParts.join("/");
15 |
16 | if (newUrl.searchParams.get("download") !== "true")
17 | newUrl.searchParams.set("download", "true");
18 |
19 | return newUrl.href;
20 | }
21 | }
22 |
23 | return url;
24 | }
25 |
--------------------------------------------------------------------------------
/src/state.ts:
--------------------------------------------------------------------------------
1 | let isInDocumentationMode = false;
2 | let isInCLI = false;
3 | let forceShowConsoleLogPrefix = false;
4 |
5 | export function getIsInDocumentationMode() {
6 | return isInDocumentationMode;
7 | }
8 |
9 | export function setIsInDocumentationMode(value: boolean) {
10 | isInDocumentationMode = value;
11 | }
12 |
13 | export function getIsRunningFromCLI() {
14 | return isInCLI;
15 | }
16 |
17 | export function setIsRunningFromCLI(value: boolean) {
18 | isInCLI = value;
19 | }
20 |
21 | export function getForceShowConsoleLogPrefix() {
22 | return forceShowConsoleLogPrefix;
23 | }
24 |
25 | export function setForceShowConsoleLogPrefix(value: boolean) {
26 | forceShowConsoleLogPrefix = value;
27 | }
28 |
--------------------------------------------------------------------------------
/src/utils/InsufficientMemoryError.ts:
--------------------------------------------------------------------------------
1 | export class InsufficientMemoryError extends Error {
2 | public constructor(message: string = "Insufficient memory") {
3 | super(message);
4 | }
5 | }
6 |
--------------------------------------------------------------------------------
/src/utils/OverridesObject.ts:
--------------------------------------------------------------------------------
1 | /**
2 | * Makes all the properties of an object optional, including nested objects,
3 | * and strips all keys that their value is not of the specified allowed value types.
4 | */
5 | export type OverridesObject = T extends object
6 | ? {[P in keyof T]?: OverridesObject}
7 | : T extends Array
8 | ? AllowedValueTypes extends Array
9 | ? Array>
10 | : never
11 | : T extends ReadonlyArray
12 | ? AllowedValueTypes extends ReadonlyArray
13 | ? ReadonlyArray>
14 | : never
15 | : AllowedValueTypes extends T
16 | ? T
17 | : never;
18 |
--------------------------------------------------------------------------------
/src/utils/UnsupportedError.ts:
--------------------------------------------------------------------------------
1 | export class UnsupportedError extends Error {
2 | /** @internal */
3 | public constructor(message: string = "UnsupportedError") {
4 | super(message);
5 | }
6 | }
7 |
--------------------------------------------------------------------------------
/src/utils/appendUserMessageToChatHistory.ts:
--------------------------------------------------------------------------------
1 | import {ChatHistoryItem, ChatUserMessage} from "../types.js";
2 |
3 | /**
4 | * Appends a user message to the chat history.
5 | * If the last message in the chat history is also a user message, the new message will be appended to it.
6 | */
7 | export function appendUserMessageToChatHistory(chatHistory: readonly ChatHistoryItem[], message: string) {
8 | const newChatHistory = chatHistory.slice();
9 |
10 | if (newChatHistory.length > 0 && newChatHistory[newChatHistory.length - 1]!.type === "user") {
11 | const lastUserMessage = newChatHistory[newChatHistory.length - 1]! as ChatUserMessage;
12 |
13 | newChatHistory[newChatHistory.length - 1] = {
14 | ...lastUserMessage,
15 | text: [lastUserMessage.text, message].join("\n\n")
16 | };
17 | } else {
18 | newChatHistory.push({
19 | type: "user",
20 | text: message
21 | });
22 | }
23 |
24 | return newChatHistory;
25 | }
26 |
--------------------------------------------------------------------------------
/src/utils/clearTempFolder.ts:
--------------------------------------------------------------------------------
1 | import process from "process";
2 | import fs from "fs-extra";
3 | import {tempDownloadDirectory} from "../config.js";
4 |
5 | export async function clearTempFolder() {
6 | if (process.platform === "win32") {
7 | try {
8 | await fs.remove(tempDownloadDirectory);
9 | } catch (err) {
10 | // do nothing as it fails sometime on Windows, and since it's a temp folder, it's not a big deal
11 | }
12 |
13 | return;
14 | }
15 |
16 | await fs.remove(tempDownloadDirectory);
17 | }
18 |
--------------------------------------------------------------------------------
/src/utils/compareTokens.ts:
--------------------------------------------------------------------------------
1 | import {Token} from "../types.js";
2 |
3 | export function compareTokens(token1?: Token, token2?: Token) {
4 | return token1 === token2;
5 | }
6 |
--------------------------------------------------------------------------------
/src/utils/findBestOption.ts:
--------------------------------------------------------------------------------
1 | export function findBestOption({generator, score}: {
2 | generator: () => Generator,
3 | score: (option: O) => number | null
4 | }) {
5 | let bestOption: O | null = null;
6 | let bestScore: number | null = null;
7 |
8 | for (const option of generator()) {
9 | const currentScore = score(option);
10 |
11 | if (currentScore === Infinity)
12 | return option;
13 |
14 | if (currentScore != null && (bestScore == null || currentScore > bestScore)) {
15 | bestOption = option;
16 | bestScore = currentScore;
17 | }
18 | }
19 |
20 | return bestOption;
21 | }
22 |
--------------------------------------------------------------------------------
/src/utils/gbnfJson/getGbnfGrammarForGbnfJsonSchema.ts:
--------------------------------------------------------------------------------
1 | import {GbnfJsonSchema} from "./types.js";
2 | import {getGbnfJsonTerminalForGbnfJsonSchema} from "./utils/getGbnfJsonTerminalForGbnfJsonSchema.js";
3 | import {GbnfGrammarGenerator} from "./GbnfGrammarGenerator.js";
4 | import {GbnfJsonScopeState} from "./utils/GbnfJsonScopeState.js";
5 |
6 |
7 | export function getGbnfGrammarForGbnfJsonSchema(schema: Readonly, {
8 | allowNewLines = true,
9 | scopePadSpaces = 4
10 | }: {
11 | allowNewLines?: boolean,
12 | scopePadSpaces?: number
13 | } = {}): string {
14 | const grammarGenerator = new GbnfGrammarGenerator();
15 | const scopeState = new GbnfJsonScopeState({allowNewLines, scopePadSpaces});
16 | const rootTerminal = getGbnfJsonTerminalForGbnfJsonSchema(schema, grammarGenerator, scopeState);
17 | const rootGrammar = rootTerminal.getGrammar(grammarGenerator);
18 |
19 | return grammarGenerator.generateGbnfFile(rootGrammar + ` "${"\\n".repeat(4)}"` + " [\\n]*");
20 | }
21 |
--------------------------------------------------------------------------------
/src/utils/gbnfJson/terminals/GbnfBoolean.ts:
--------------------------------------------------------------------------------
1 | import {GbnfTerminal} from "../GbnfTerminal.js";
2 | import {reservedRuleNames} from "./gbnfConsts.js";
3 |
4 |
5 | export class GbnfBoolean extends GbnfTerminal {
6 | public getGrammar(): string {
7 | return this._getGrammar();
8 | }
9 |
10 | protected override getGrammarFromResolve(): string {
11 | return this._getGrammar(false);
12 | }
13 |
14 | private _getGrammar(wrap: boolean = true): string {
15 | const values: string[] = ['"true"', '"false"'];
16 |
17 | if (wrap)
18 | return [
19 | "(", values.join(" | "), ")"
20 | ].join(" ");
21 |
22 | return values.join(" | ");
23 | }
24 |
25 | protected override getRuleName(): string {
26 | return reservedRuleNames.boolean;
27 | }
28 | }
29 |
--------------------------------------------------------------------------------
/src/utils/gbnfJson/terminals/GbnfBooleanValue.ts:
--------------------------------------------------------------------------------
1 | import {GbnfTerminal} from "../GbnfTerminal.js";
2 |
3 |
4 | export class GbnfBooleanValue extends GbnfTerminal {
5 | public readonly value: boolean;
6 |
7 | public constructor(value: boolean) {
8 | super();
9 | this.value = value;
10 | }
11 |
12 | public getGrammar(): string {
13 | if (this.value)
14 | return '"true"';
15 |
16 | return '"false"';
17 | }
18 |
19 | public override resolve(): string {
20 | return this.getGrammar();
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/src/utils/gbnfJson/terminals/GbnfGrammar.ts:
--------------------------------------------------------------------------------
1 | import {GbnfTerminal} from "../GbnfTerminal.js";
2 | import {GbnfGrammarGenerator} from "../GbnfGrammarGenerator.js";
3 |
4 |
5 | export class GbnfGrammar extends GbnfTerminal {
6 | public readonly grammar: string | string[];
7 | public readonly resolveToRawGrammar: boolean;
8 |
9 | public constructor(grammar: string | string[], resolveToRawGrammar: boolean = false) {
10 | super();
11 | this.grammar = grammar;
12 | this.resolveToRawGrammar = resolveToRawGrammar;
13 | }
14 |
15 | public getGrammar(): string {
16 | if (this.grammar instanceof Array)
17 | return this.grammar
18 | .filter((item) => item !== "")
19 | .join(" ");
20 |
21 | return this.grammar;
22 | }
23 |
24 | public override resolve(grammarGenerator: GbnfGrammarGenerator): string {
25 | if (this.resolveToRawGrammar)
26 | return this.getGrammar();
27 |
28 | return super.resolve(grammarGenerator);
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/src/utils/gbnfJson/terminals/GbnfInsideStringChar.ts:
--------------------------------------------------------------------------------
1 | import {GbnfTerminal} from "../GbnfTerminal.js";
2 | import {reservedRuleNames} from "./gbnfConsts.js";
3 |
4 | export class GbnfInsideStringChar extends GbnfTerminal {
5 | public getGrammar(): string {
6 | return [
7 | negatedCharacterSet([
8 | '"',
9 | "\\\\",
10 | "\\x7F",
11 | "\\x00-\\x1F"
12 | ]),
13 |
14 | // escape sequences
15 | '"\\\\" ["\\\\/bfnrt]',
16 | '"\\\\u" [0-9a-fA-F]{4}'
17 | ].join(" | ");
18 | }
19 |
20 | protected override getRuleName(): string {
21 | return reservedRuleNames.stringChar;
22 | }
23 | }
24 |
25 | function negatedCharacterSet(characterDefinitions: string[]) {
26 | return "[^" + characterDefinitions.join("") + "]";
27 | }
28 |
--------------------------------------------------------------------------------
/src/utils/gbnfJson/terminals/GbnfNull.ts:
--------------------------------------------------------------------------------
1 | import {GbnfTerminal} from "../GbnfTerminal.js";
2 | import {reservedRuleNames} from "./gbnfConsts.js";
3 |
4 |
5 | export class GbnfNull extends GbnfTerminal {
6 | public getGrammar(): string {
7 | return '"null"';
8 | }
9 |
10 | protected override getRuleName(): string {
11 | return reservedRuleNames.null;
12 | }
13 | }
14 |
--------------------------------------------------------------------------------
/src/utils/gbnfJson/terminals/GbnfNumber.ts:
--------------------------------------------------------------------------------
1 | import {GbnfTerminal} from "../GbnfTerminal.js";
2 | import {reservedRuleNames} from "./gbnfConsts.js";
3 |
4 |
5 | export class GbnfNumber extends GbnfTerminal {
6 | public readonly allowFractional: boolean;
7 |
8 | public constructor({allowFractional = true}: {allowFractional: boolean}) {
9 | super();
10 | this.allowFractional = allowFractional;
11 | }
12 |
13 | public getGrammar(): string {
14 | const num = '"-"? ("0" | [1-9] [0-9]{0,15})';
15 | const exponent = ' ([eE] [-+]? ("0" | [1-9] [0-9]{0,15}))?';
16 |
17 | if (this.allowFractional)
18 | return num + ' ("." [0-9]{1,16})?' + exponent;
19 |
20 | return num + exponent;
21 | }
22 |
23 | protected override getRuleName(): string {
24 | if (this.allowFractional)
25 | return reservedRuleNames.number.fractional;
26 |
27 | return reservedRuleNames.number.integer;
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/src/utils/gbnfJson/terminals/GbnfNumberValue.ts:
--------------------------------------------------------------------------------
1 | import {GbnfTerminal} from "../GbnfTerminal.js";
2 | import {GbnfGrammarGenerator} from "../GbnfGrammarGenerator.js";
3 |
4 |
5 | export class GbnfNumberValue extends GbnfTerminal {
6 | public readonly value: number;
7 |
8 | public constructor(value: number) {
9 | super();
10 | this.value = value;
11 | }
12 |
13 | public override getGrammar(): string {
14 | return '"' + JSON.stringify(this.value) + '"';
15 | }
16 |
17 | public override resolve(grammarGenerator: GbnfGrammarGenerator): string {
18 | const grammar = this.getGrammar();
19 | if (grammar.length <= grammarGenerator.getProposedLiteralValueRuleNameLength())
20 | return grammar;
21 |
22 | return super.resolve(grammarGenerator);
23 | }
24 |
25 | protected override generateRuleName(grammarGenerator: GbnfGrammarGenerator): string {
26 | return grammarGenerator.generateRuleNameForLiteralValue(this.value);
27 | }
28 | }
29 |
--------------------------------------------------------------------------------
/src/utils/gbnfJson/terminals/GbnfStringValue.ts:
--------------------------------------------------------------------------------
1 | import {GbnfTerminal} from "../GbnfTerminal.js";
2 | import {GbnfGrammarGenerator} from "../GbnfGrammarGenerator.js";
3 |
4 |
5 | export class GbnfStringValue extends GbnfTerminal {
6 | public readonly value: string;
7 |
8 | public constructor(value: string) {
9 | super();
10 | this.value = value;
11 | }
12 |
13 | public override getGrammar(): string {
14 | return [
15 | '"',
16 | '\\"',
17 | this.value
18 | .replaceAll("\\", "\\\\\\\\")
19 | .replaceAll("\t", "\\\\t")
20 | .replaceAll("\r", "\\\\r")
21 | .replaceAll("\n", "\\\\n")
22 | .replaceAll('"', "\\\\" + '\\"'),
23 | '\\"',
24 | '"'
25 | ].join("");
26 | }
27 |
28 | protected override generateRuleName(grammarGenerator: GbnfGrammarGenerator): string {
29 | return grammarGenerator.generateRuleNameForLiteralValue(this.value);
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/src/utils/gbnfJson/terminals/GbnfVerbatimText.ts:
--------------------------------------------------------------------------------
1 | import {GbnfTerminal} from "../GbnfTerminal.js";
2 |
3 |
4 | export class GbnfVerbatimText extends GbnfTerminal {
5 | public readonly value: string;
6 |
7 | public constructor(value: string) {
8 | super();
9 | this.value = value;
10 | }
11 |
12 | public override getGrammar(): string {
13 | return [
14 | '"',
15 | this.value
16 | .replaceAll("\\", "\\\\")
17 | .replaceAll('"', '\\"')
18 | .replaceAll("\t", "\\t")
19 | .replaceAll("\r", "\\r")
20 | .replaceAll("\n", "\\n"),
21 | '"'
22 | ].join("");
23 | }
24 | }
25 |
--------------------------------------------------------------------------------
/src/utils/gbnfJson/utils/GbnfJsonScopeState.ts:
--------------------------------------------------------------------------------
1 | export type GbnfJsonScopeSettings = {
2 | readonly allowNewLines: boolean,
3 | readonly scopePadSpaces: number
4 | };
5 |
6 | export class GbnfJsonScopeState {
7 | public readonly settings: GbnfJsonScopeSettings;
8 | public readonly currentNestingScope: number;
9 |
10 | public constructor(settings: GbnfJsonScopeSettings = {
11 | allowNewLines: true,
12 | scopePadSpaces: 4
13 | }, currentNestingScope: number = 0) {
14 | this.settings = settings;
15 | this.currentNestingScope = currentNestingScope;
16 | }
17 |
18 | public getForNewScope(): GbnfJsonScopeState {
19 | return new GbnfJsonScopeState(this.settings, this.currentNestingScope + 1);
20 | }
21 | }
22 |
23 |
--------------------------------------------------------------------------------
/src/utils/gbnfJson/utils/getGbnfJsonTerminalForLiteral.ts:
--------------------------------------------------------------------------------
1 | import {GbnfNull} from "../terminals/GbnfNull.js";
2 | import {GbnfBooleanValue} from "../terminals/GbnfBooleanValue.js";
3 | import {GbnfNumberValue} from "../terminals/GbnfNumberValue.js";
4 | import {GbnfStringValue} from "../terminals/GbnfStringValue.js";
5 |
6 |
7 | export function getGbnfJsonTerminalForLiteral(literal: string | number | boolean | null) {
8 | if (literal === null)
9 | return new GbnfNull();
10 |
11 | if (typeof literal === "boolean")
12 | return new GbnfBooleanValue(literal);
13 |
14 | if (typeof literal === "number")
15 | return new GbnfNumberValue(literal);
16 |
17 | if (typeof literal === "string")
18 | return new GbnfStringValue(literal);
19 |
20 | throw new Error(`Unrecognized literal type: ${typeof literal}`);
21 | }
22 |
--------------------------------------------------------------------------------
/src/utils/getBuildDefaults.ts:
--------------------------------------------------------------------------------
1 | import {defaultLlamaCppGitHubRepo, defaultLlamaCppGpuSupport, defaultLlamaCppRelease} from "../config.js";
2 |
3 | export async function getBuildDefaults() {
4 | return {
5 | repo: defaultLlamaCppGitHubRepo,
6 | release: defaultLlamaCppRelease,
7 | gpuSupport: defaultLlamaCppGpuSupport
8 | };
9 | }
10 |
--------------------------------------------------------------------------------
/src/utils/getChatWrapperSegmentDefinition.ts:
--------------------------------------------------------------------------------
1 | import {ChatModelSegmentType, ChatWrapperSettings, ChatWrapperSettingsSegment} from "../types.js";
2 |
3 | export function getChatWrapperSegmentDefinition(
4 | chatWrapperSetting: ChatWrapperSettings,
5 | segmentType: ChatModelSegmentType
6 | ): ChatWrapperSettingsSegment | undefined {
7 | if (segmentType === "thought")
8 | return chatWrapperSetting.segments?.thought;
9 |
10 | void (segmentType satisfies never);
11 | return undefined;
12 | }
13 |
--------------------------------------------------------------------------------
/src/utils/getConsoleLogPrefix.ts:
--------------------------------------------------------------------------------
1 | import chalk from "chalk";
2 | import {getForceShowConsoleLogPrefix, getIsRunningFromCLI} from "../state.js";
3 |
4 | export function getConsoleLogPrefix(forcePrefix: boolean = false, padEnd: boolean = true) {
5 | const isInCLI = getIsRunningFromCLI();
6 | const forceShowLogPrefix = getForceShowConsoleLogPrefix();
7 |
8 | if (!isInCLI || forceShowLogPrefix || forcePrefix)
9 | return chalk.gray("[node-llama-cpp]") + (padEnd ? " " : "");
10 |
11 | return "";
12 | }
13 |
--------------------------------------------------------------------------------
/src/utils/getGrammarsFolder.ts:
--------------------------------------------------------------------------------
1 | import fs from "fs-extra";
2 | import {llamaBinsGrammarsDirectory, llamaCppGrammarsDirectory} from "../config.js";
3 | import {Llama} from "../bindings/Llama.js";
4 | import {isLlamaCppRepoCloned} from "../bindings/utils/cloneLlamaCppRepo.js";
5 |
6 | export async function getGrammarsFolder(buildType: Llama["buildType"]) {
7 | if (buildType === "localBuild") {
8 | if (await isLlamaCppRepoCloned(true) && await fs.pathExists(llamaCppGrammarsDirectory))
9 | return llamaCppGrammarsDirectory;
10 | } else if (buildType === "prebuilt") {
11 | if (await fs.pathExists(llamaBinsGrammarsDirectory))
12 | return llamaBinsGrammarsDirectory;
13 | else if (await isLlamaCppRepoCloned(false) && await fs.pathExists(llamaCppGrammarsDirectory))
14 | return llamaCppGrammarsDirectory;
15 | } else
16 | void (buildType satisfies never);
17 |
18 | throw new Error("Grammars folder not found");
19 | }
20 |
--------------------------------------------------------------------------------
/src/utils/getLlamaClasses.ts:
--------------------------------------------------------------------------------
1 | import {LlamaChatSession} from "../evaluator/LlamaChatSession/LlamaChatSession.js";
2 | import {LlamaChat} from "../evaluator/LlamaChat/LlamaChat.js";
3 | import {LlamaCompletion} from "../evaluator/LlamaCompletion.js";
4 |
5 | export type LlamaClasses = {
6 | readonly LlamaChatSession: typeof LlamaChatSession,
7 | readonly LlamaChat: typeof LlamaChat,
8 | readonly LlamaCompletion: typeof LlamaCompletion
9 | };
10 |
11 | let cachedClasses: LlamaClasses | undefined = undefined;
12 |
13 | export function getLlamaClasses(): LlamaClasses {
14 | if (cachedClasses == null)
15 | cachedClasses = Object.seal({
16 | LlamaChatSession,
17 | LlamaChat,
18 | LlamaCompletion
19 | });
20 |
21 | return cachedClasses;
22 | }
23 |
--------------------------------------------------------------------------------
/src/utils/getModuleVersion.ts:
--------------------------------------------------------------------------------
1 | import path from "path";
2 | import {fileURLToPath} from "url";
3 | import fs from "fs-extra";
4 |
5 | const __dirname = path.dirname(fileURLToPath(import.meta.url));
6 |
7 | let moduleVersion: string | null = null;
8 | export async function getModuleVersion(): Promise {
9 | if (moduleVersion != null)
10 | return moduleVersion;
11 |
12 | const packageJson = await fs.readJson(path.join(__dirname, "..", "..", "package.json"));
13 |
14 | moduleVersion = packageJson.version as string;
15 |
16 | return moduleVersion;
17 | }
18 |
--------------------------------------------------------------------------------
/src/utils/getReadableContextSize.ts:
--------------------------------------------------------------------------------
1 | export function getReadableContextSize(contextSize: number) {
2 | return contextSize.toLocaleString("en-US", {
3 | notation: "compact",
4 | compactDisplay: "short"
5 | });
6 | }
7 |
--------------------------------------------------------------------------------
/src/utils/hashString.ts:
--------------------------------------------------------------------------------
1 | import * as crypto from "node:crypto";
2 |
3 | export async function hashString(text: string): Promise {
4 | const hashBuffer = await crypto.subtle.digest("SHA-1", Buffer.from(text));
5 |
6 | return Array.from(new Uint8Array(hashBuffer))
7 | .map((b) => b.toString(36))
8 | .join("");
9 | }
10 |
--------------------------------------------------------------------------------
/src/utils/includesText.ts:
--------------------------------------------------------------------------------
1 | export function includesText(
2 | value: string | string[] | null | undefined,
3 | textToCheckFor: string | string[],
4 | strictCase: boolean = false
5 | ): boolean {
6 | if (value instanceof Array)
7 | return value.some((v) => includesText(v, textToCheckFor, strictCase));
8 | else if (typeof value !== "string")
9 | return false;
10 |
11 | if (textToCheckFor instanceof Array)
12 | return textToCheckFor.some((t) => includesText(value, t, strictCase));
13 |
14 | if (strictCase)
15 | return value.includes(textToCheckFor);
16 |
17 | return value.toLowerCase().includes(textToCheckFor.toLowerCase());
18 | }
19 |
--------------------------------------------------------------------------------
/src/utils/isLockfileActive.ts:
--------------------------------------------------------------------------------
1 | import lockfile from "proper-lockfile";
2 | import {isLockActive} from "lifecycle-utils";
3 | import {lockfileLockScope} from "./withLockfile.js";
4 |
5 | export async function isLockfileActive({
6 | resourcePath, staleDuration = 1000 * 10
7 | }: {
8 | resourcePath: string, staleDuration?: number
9 | }) {
10 | if (isLockActive(lockfileLockScope, resourcePath))
11 | return true;
12 |
13 | const lockfileActive = await lockfile.check(resourcePath, {stale: staleDuration, realpath: false});
14 | if (lockfileActive)
15 | return true;
16 |
17 | return isLockActive(lockfileLockScope, resourcePath);
18 | }
19 |
--------------------------------------------------------------------------------
/src/utils/isToken.ts:
--------------------------------------------------------------------------------
1 | import {Token} from "../types.js";
2 |
3 | export function isToken(token: any): token is Token {
4 | return typeof token === "number";
5 | }
6 |
--------------------------------------------------------------------------------
/src/utils/isUrl.ts:
--------------------------------------------------------------------------------
1 | export function isUrl(text: string, throwOnInvalidUrl: boolean = true) {
2 | if (text.toLowerCase().startsWith("http://") || text.toLowerCase().startsWith("https://")) {
3 | try {
4 | new URL(text);
5 | return true;
6 | } catch {
7 | if (throwOnInvalidUrl)
8 | throw new Error(`Invalid URL: ${text}`);
9 |
10 | return false;
11 | }
12 | }
13 |
14 | return false;
15 | }
16 |
--------------------------------------------------------------------------------
/src/utils/mergeUnionTypes.ts:
--------------------------------------------------------------------------------
1 | type UnionToIntersection = (
2 | U extends any
3 | ? ((k: U) => void)
4 | : never
5 | ) extends ((k: infer I) => void)
6 | ? I
7 | : never;
8 |
9 | type DistributeUnion = {
10 | [K in keyof U]: U[K]
11 | };
12 | type OptionalDistributeUnion = {
13 | [K in keyof U]?: U[K]
14 | };
15 |
16 | export type MergeUnionTypes = DistributeUnion>;
17 | export type MergeOptionalUnionTypes = OptionalDistributeUnion>;
18 |
--------------------------------------------------------------------------------
/src/utils/pushAll.ts:
--------------------------------------------------------------------------------
1 | /**
2 | * Pushes all items from the given array or set to the given array.
3 | * @param array - The array to push the items to
4 | * @param items - The items to push to the array
5 | */
6 | export function pushAll(array: T[], items: readonly NoInfer[] | ReadonlySet>): T[] {
7 | for (const item of items)
8 | array.push(item);
9 |
10 | return array;
11 | }
12 |
--------------------------------------------------------------------------------
/src/utils/removeNullFields.ts:
--------------------------------------------------------------------------------
1 | export function removeNullFields(obj: T): T {
2 | const newObj: T = Object.assign({}, obj);
3 |
4 | for (const key in obj) {
5 | if (newObj[key] == null)
6 | delete newObj[key];
7 | }
8 |
9 | return newObj;
10 | }
11 |
12 | export function removeUndefinedFields(obj: T): T {
13 | const newObj: T = Object.assign({}, obj);
14 |
15 | for (const key in obj) {
16 | if (newObj[key] === undefined)
17 | delete newObj[key];
18 | }
19 |
20 | return newObj;
21 | }
22 |
--------------------------------------------------------------------------------
/src/utils/resolveLastTokens.ts:
--------------------------------------------------------------------------------
1 | import {Token} from "../types.js";
2 | import {maxRecentDetokenizerTokens} from "../consts.js";
3 |
4 | export function resolveLastTokens(tokenArrays: Token[][], maxTokens: number = maxRecentDetokenizerTokens) {
5 | const lastTokens: Token[] = [];
6 | for (let i = tokenArrays.length - 1; i >= 0 && lastTokens.length < maxTokens; i--) {
7 | const tokens = tokenArrays[i]!;
8 |
9 | for (let j = tokens.length - 1; j >= 0 && lastTokens.length < maxTokens; j--) {
10 | lastTokens.unshift(tokens[j]!);
11 | }
12 | }
13 |
14 | return lastTokens;
15 | }
16 |
--------------------------------------------------------------------------------
/src/utils/runtime.ts:
--------------------------------------------------------------------------------
1 | import path from "path";
2 | import {fileURLToPath} from "url";
3 |
4 | const __filename = fileURLToPath(import.meta.url);
5 |
6 | export const runningInElectron = process.versions.electron != null;
7 | export const runningInsideAsar = runningInElectron && __filename.toLowerCase().includes(".asar" + path.sep);
8 | export const runningInBun = process.versions.bun != null;
9 | export const runningInNode = !runningInElectron && !runningInBun;
10 |
11 |
--------------------------------------------------------------------------------
/src/utils/tokenizeInput.ts:
--------------------------------------------------------------------------------
1 | import {Token, Tokenizer} from "../types.js";
2 | import {isLlamaText, LlamaText} from "./LlamaText.js";
3 | import {isToken} from "./isToken.js";
4 |
5 | export function tokenizeInput(
6 | input: Token | Token[] | string | LlamaText,
7 | tokenizer: Tokenizer,
8 | options?: "trimLeadingSpace",
9 | clone: boolean = false
10 | ) {
11 | if (typeof input === "string")
12 | return tokenizer(input, false, options);
13 | else if (isLlamaText(input))
14 | return input.tokenize(tokenizer, options);
15 | else if (isToken(input))
16 | return [input];
17 |
18 | if (clone)
19 | return input.slice();
20 |
21 | return input;
22 | }
23 |
--------------------------------------------------------------------------------
/src/utils/utilTypes.ts:
--------------------------------------------------------------------------------
1 | export type Writable = {
2 | -readonly [P in keyof T]: T[P];
3 | };
4 |
5 | export type PickOptions<
6 | Value extends Readonly>,
7 | Options extends {readonly [key: string]: boolean | undefined}
8 | > = Pick;
15 |
--------------------------------------------------------------------------------
/src/utils/waitForLockfileRelease.ts:
--------------------------------------------------------------------------------
1 | import lockfile from "proper-lockfile";
2 | import {isLockActive, waitForLockRelease} from "lifecycle-utils";
3 | import {lockfileLockScope} from "./withLockfile.js";
4 |
5 | export async function waitForLockfileRelease({
6 | resourcePath, checkInterval = 1000 * 5.5, staleDuration = 1000 * 10
7 | }: {
8 | resourcePath: string, checkInterval?: number, staleDuration?: number
9 | }) {
10 | while (true) {
11 | if (isLockActive(lockfileLockScope, resourcePath)) {
12 | await waitForLockRelease(lockfileLockScope, resourcePath);
13 | continue;
14 | }
15 |
16 | const lockfileActive = await lockfile.check(resourcePath, {stale: staleDuration, realpath: false});
17 | const lockIsActive = isLockActive(lockfileLockScope, resourcePath);
18 |
19 | if (lockIsActive)
20 | continue;
21 |
22 | if (!lockfileActive)
23 | return;
24 |
25 | await new Promise((resolve) => setTimeout(resolve, checkInterval));
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/src/utils/wrapAbortSignal.ts:
--------------------------------------------------------------------------------
1 | export function wrapAbortSignal(abortSignal?: AbortSignal): [controller: AbortController, dispose: (() => void)] {
2 | const controller = new AbortController();
3 |
4 | function onAbort() {
5 | controller.abort(abortSignal!.reason);
6 | }
7 |
8 | function dispose() {
9 | if (abortSignal != null)
10 | abortSignal.removeEventListener("abort", onAbort);
11 | }
12 |
13 | if (abortSignal != null)
14 | abortSignal.addEventListener("abort", onAbort);
15 |
16 | return [controller, dispose];
17 | }
18 |
--------------------------------------------------------------------------------
/templates/.gitignore:
--------------------------------------------------------------------------------
1 | /*/**/package-lock.json
2 |
--------------------------------------------------------------------------------
/templates/README.md:
--------------------------------------------------------------------------------
1 | Use the following command to scaffold a project from a template:
2 | ```bash
3 | npm create node-llama-cpp@latest
4 | ```
5 |
6 | > Don't use the templates in this repository directly. They are built to be used with the `npm create` command ([learn more](https://node-llama-cpp.withcat.ai/guide/))
7 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/.editorconfig:
--------------------------------------------------------------------------------
1 | root = true
2 |
3 | [*]
4 | indent_style = space
5 | indent_size = 4
6 |
7 | [{*.ts,*.tsx,*.js,*.jsx,*.css,*.scss}]
8 | insert_final_newline = true
9 |
10 | [{package.json,package-lock.json,manifest.json,electron-builder.json5}]
11 | indent_size = 2
12 |
13 | [*.yml]
14 | indent_size = 2
15 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/.gitignore:
--------------------------------------------------------------------------------
1 | /.idea
2 | /.vscode
3 | node_modules
4 | .DS_Store
5 |
6 | /dist
7 | /dist-electron
8 | /release
9 | /models
10 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/README.md:
--------------------------------------------------------------------------------
1 | # Electron + TypeScript + React + Vite + `node-llama-cpp`
2 | This template provides a minimal setup to get an Electron app working with TypeScript and `node-llama-cpp`, React with TypeScript for the renderer, and some ESLint rules.
3 |
4 | ## Get started
5 | Install node modules and download the model files used by `node-llama-cpp`:
6 | ```bash
7 | npm install
8 | ```
9 |
10 | Start the project:
11 | ```bash
12 | npm start
13 | ```
14 |
15 | > Generated using `npm create node-llama-cpp@latest` ([learn more](https://node-llama-cpp.withcat.ai/guide/))
16 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/electron/electron-env.d.ts:
--------------------------------------------------------------------------------
1 | ///
2 |
3 | declare namespace NodeJS {
4 | interface ProcessEnv {
5 | /**
6 | * The built directory structure
7 | *
8 | * ```tree
9 | * ├─┬─┬ dist
10 | * │ │ └── index.html
11 | * │ │
12 | * │ ├─┬ dist-electron
13 | * │ │ ├── index.js
14 | * │ │ └── preload.mjs
15 | * │
16 | * ```
17 | */
18 | APP_ROOT: string,
19 | /** /dist/ or /public/ */
20 | VITE_PUBLIC: string
21 | }
22 | }
23 |
24 | // Used in Renderer process, expose in `preload.ts`
25 | interface Window {
26 | ipcRenderer: import("electron").IpcRenderer
27 | }
28 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/electron/preload.ts:
--------------------------------------------------------------------------------
1 | import {ipcRenderer, contextBridge} from "electron";
2 |
3 | // --------- Expose some API to the Renderer process ---------
4 | contextBridge.exposeInMainWorld("ipcRenderer", {
5 | on(...args: Parameters) {
6 | const [channel, listener] = args;
7 | return ipcRenderer.on(channel, (event, ...args) => listener(event, ...args));
8 | },
9 | off(...args: Parameters) {
10 | const [channel, ...omit] = args;
11 | return ipcRenderer.off(channel, ...omit);
12 | },
13 | send(...args: Parameters) {
14 | const [channel, ...omit] = args;
15 | return ipcRenderer.send(channel, ...omit);
16 | },
17 | invoke(...args: Parameters) {
18 | const [channel, ...omit] = args;
19 | return ipcRenderer.invoke(channel, ...omit);
20 | }
21 |
22 | // You can expose other APIs you need here
23 | // ...
24 | });
25 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/electron/utils/createElectronSideBirpc.ts:
--------------------------------------------------------------------------------
1 | import {BrowserWindow, ipcMain} from "electron";
2 | import {createBirpc} from "birpc";
3 |
4 | export function createElectronSideBirpc<
5 | const RendererFunction = Record,
6 | const ElectronFunctions extends object = Record
7 | >(
8 | toRendererEventName: string,
9 | fromRendererEventName: string,
10 | window: BrowserWindow,
11 | electronFunctions: ElectronFunctions
12 | ) {
13 | return createBirpc(electronFunctions, {
14 | post: (data) => window.webContents.send(toRendererEventName, data),
15 | on: (onData) => ipcMain.on(fromRendererEventName, (event, data) => {
16 | if (BrowserWindow.fromWebContents(event.sender) === window)
17 | onData(data);
18 | }),
19 | serialize: (value) => JSON.stringify(value),
20 | deserialize: (value) => JSON.parse(value)
21 | });
22 | }
23 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/src/App/components/ChatHistory/ChatHistory.css:
--------------------------------------------------------------------------------
1 | .appChatHistory {
2 | flex: 1;
3 | display: flex;
4 | flex-direction: column;
5 | text-align: start;
6 | overflow: auto;
7 | padding: 24px 0px;
8 | }
9 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/src/App/components/ChatHistory/components/ModelMessage/components/ModelMessageCopyButton/ModelMessageCopyButton.css:
--------------------------------------------------------------------------------
1 | .appChatHistory > .message.model > .buttons {
2 | > .copyButton {
3 | display: grid;
4 | grid-template-areas: "icon";
5 | padding: 6px;
6 | border: none;
7 |
8 | transition: background-color 0.1s ease-in-out;
9 |
10 | &:not(:hover, :focus-visible) {
11 | background-color: transparent;
12 | }
13 |
14 | &.copied {
15 | > .icon.copy {
16 | opacity: 0;
17 | transition-delay: 0s;
18 | }
19 |
20 | > .icon.check {
21 | opacity: 1;
22 | transition-delay: 0.1s;
23 | }
24 | }
25 |
26 | > .icon {
27 | grid-area: icon;
28 | width: 18px;
29 | height: 18px;
30 |
31 | transition: opacity 0.3s ease-in-out;
32 |
33 | &.copy {
34 | opacity: 1;
35 | transition-delay: 0.1s;
36 | }
37 | &.check {
38 | opacity: 0;
39 | }
40 | }
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/src/App/components/ChatHistory/components/UserMessage/UserMessage.css:
--------------------------------------------------------------------------------
1 | .appChatHistory > .message.user {
2 | align-self: flex-end;
3 | background-color: var(--user-message-background-color);
4 | padding: 8px 12px;
5 | border-radius: 12px;
6 | margin-top: 0px;
7 | margin-bottom: 16px;
8 | margin-inline-start: 48px;
9 | margin-inline-end: 12px;
10 | color: var(--user-message-text-color);
11 | word-break: break-word;
12 | max-width: calc(100% - 48px - 12px);
13 | box-sizing: border-box;
14 |
15 | &:not(:first-child) {
16 | margin-top: 36px;
17 | }
18 | }
19 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/src/App/components/ChatHistory/components/UserMessage/UserMessage.tsx:
--------------------------------------------------------------------------------
1 | import {MessageMarkdown} from "../../../MessageMarkdown/MessageMarkdown.js";
2 | import {SimplifiedUserChatItem} from "../../../../../../electron/state/llmState.js";
3 |
4 | import "./UserMessage.css";
5 |
6 | export function UserMessage({message}: UserMessageProps) {
7 | return
8 | {message.message}
9 | ;
10 | }
11 |
12 | type UserMessageProps = {
13 | message: SimplifiedUserChatItem
14 | };
15 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/src/App/components/Header/components/UpdateBadge.css:
--------------------------------------------------------------------------------
1 | .appHeader > .updateBadge {
2 | pointer-events: all;
3 | display: flex;
4 | flex-direction: row;
5 | align-items: center;
6 | flex-shrink: 0;
7 | margin-inline-start: 16px;
8 |
9 | > .currentVersion {
10 | opacity: 0.4;
11 | margin: 14px;
12 |
13 | > code {
14 | background-color: transparent;
15 | }
16 | }
17 |
18 | > .newVersion {
19 | background-color: var(--update-badge-background-color);
20 | border: solid 1px var(--update-badge-border-color);
21 | box-shadow: var(--update-badge-box-shadow);
22 | border-radius: 12px;
23 | backdrop-filter: blur(8px);
24 | padding: 8px 12px;
25 | margin: 0px 8px;
26 |
27 | > .version {
28 | margin: 0px 4px;
29 | font-family: monospace;
30 | }
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/src/hooks/useExternalState.ts:
--------------------------------------------------------------------------------
1 | import {useEffect, useState} from "react";
2 | import {State} from "lifecycle-utils";
3 |
4 | export function useExternalState(state: State, selector: ((state: StateType) => R)): R;
5 | export function useExternalState(state: State): StateType;
6 | export function useExternalState(state: State, selector?: ((state: StateType) => any) | null): StateType {
7 | const [currentState, setCurrentState] = useState(() => (
8 | selector == null
9 | ? state.state
10 | : selector(state.state)
11 | ));
12 |
13 | useEffect(() => {
14 | return state.createChangeListener((newState) => {
15 | setCurrentState(
16 | selector == null
17 | ? newState
18 | : selector(newState)
19 | );
20 | }, true).dispose;
21 | }, [state]);
22 |
23 | return currentState;
24 | }
25 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/src/icons/AbortIconSVG.tsx:
--------------------------------------------------------------------------------
1 | import {SVGProps} from "react";
2 |
3 | export function AbortIconSVG(props: SVGProps) {
4 | return ;
7 | }
8 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/src/icons/AddMessageIconSVG.tsx:
--------------------------------------------------------------------------------
1 | import {SVGProps} from "react";
2 |
3 | export function AddMessageIconSVG(props: SVGProps) {
4 | return ;
7 | }
8 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/src/icons/CheckIconSVG.tsx:
--------------------------------------------------------------------------------
1 | import {SVGProps} from "react";
2 |
3 | export function CheckIconSVG(props: SVGProps) {
4 | return ;
7 | }
8 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/src/icons/CopyIconSVG.tsx:
--------------------------------------------------------------------------------
1 | import {SVGProps} from "react";
2 |
3 | export function CopyIconSVG(props: SVGProps) {
4 | return ;
7 | }
8 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/src/icons/DeleteIconSVG.tsx:
--------------------------------------------------------------------------------
1 | import {SVGProps} from "react";
2 |
3 | export function DeleteIconSVG(props: SVGProps) {
4 | return ;
7 | }
8 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/src/icons/DownloadIconSVG.tsx:
--------------------------------------------------------------------------------
1 | import {SVGProps} from "react";
2 |
3 | export function DownloadIconSVG(props: SVGProps) {
4 | return ;
7 | }
8 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/src/icons/LoadFileIconSVG.tsx:
--------------------------------------------------------------------------------
1 | import {SVGProps} from "react";
2 |
3 | export function LoadFileIconSVG(props: SVGProps) {
4 | return ;
7 | }
8 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/src/icons/RightChevronIconSVG.tsx:
--------------------------------------------------------------------------------
1 | import {SVGProps} from "react";
2 |
3 | export function RightChevronIconSVG(props: SVGProps) {
4 | return ;
7 | }
8 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/src/icons/SearchIconSVG.tsx:
--------------------------------------------------------------------------------
1 | import {SVGProps} from "react";
2 |
3 | export function SearchIconSVG(props: SVGProps) {
4 | return ;
7 | }
8 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/src/icons/StarIconSVG.tsx:
--------------------------------------------------------------------------------
1 | import {SVGProps} from "react";
2 |
3 | export function StarIconSVG(props: SVGProps) {
4 | return ;
7 | }
8 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/src/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 | Electron + TypeScript + React + Vite + node-llama-cpp
8 |
9 |
10 |
11 |
12 |
13 |
14 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/src/index.tsx:
--------------------------------------------------------------------------------
1 | import React from "react";
2 | import ReactDOM from "react-dom/client";
3 | import "@fontsource-variable/inter/opsz-italic.css";
4 | import {App} from "./App/App.tsx";
5 | import "./index.css";
6 |
7 | ReactDOM.createRoot(document.getElementById("root")!).render(
8 |
9 |
10 |
11 | );
12 |
13 | // Use contextBridge
14 | window.ipcRenderer.on("main-process-message", (_event, message) => {
15 | console.log(message);
16 | });
17 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/src/rpc/llmRpc.ts:
--------------------------------------------------------------------------------
1 | import {ElectronFunctions} from "../../electron/rpc/llmRpc.ts";
2 | import {createRendererSideBirpc} from "../utils/createRendererSideBirpc.ts";
3 | import {llmState} from "../state/llmState.ts";
4 | import {LlmState} from "../../electron/state/llmState.ts";
5 |
6 |
7 | const renderedFunctions = {
8 | updateState(state: LlmState) {
9 | llmState.state = state;
10 | }
11 | } as const;
12 | export type RenderedFunctions = typeof renderedFunctions;
13 |
14 | export const electronLlmRpc = createRendererSideBirpc("llmRpc", "llmRpc", renderedFunctions);
15 |
16 | electronLlmRpc.getState()
17 | .then((state) => {
18 | llmState.state = state;
19 | })
20 | .catch((error) => {
21 | console.error("Failed to get the initial state from the main process", error);
22 | });
23 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/src/state/llmState.ts:
--------------------------------------------------------------------------------
1 | import {State} from "lifecycle-utils";
2 |
3 | import {LlmState} from "../../electron/state/llmState.ts";
4 |
5 | export const llmState = new State({
6 | appVersion: undefined,
7 | llama: {
8 | loaded: false
9 | },
10 | model: {
11 | loaded: false
12 | },
13 | context: {
14 | loaded: false
15 | },
16 | contextSequence: {
17 | loaded: false
18 | },
19 | chatSession: {
20 | loaded: false,
21 | generatingResult: false,
22 | simplifiedChat: [],
23 | draftPrompt: {
24 | prompt: "",
25 | completion: ""
26 | }
27 | }
28 | });
29 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/src/utils/createRendererSideBirpc.ts:
--------------------------------------------------------------------------------
1 | import {createBirpc} from "birpc";
2 |
3 | export function createRendererSideBirpc<
4 | const RendererFunction = Record,
5 | const ElectronFunctions extends object = Record
6 | >(
7 | toRendererEventName: string,
8 | fromRendererEventName: string,
9 | electronFunctions: ElectronFunctions
10 | ) {
11 | return createBirpc(electronFunctions, {
12 | post: (data) => window.ipcRenderer.send(fromRendererEventName, data),
13 | on: (onData) => window.ipcRenderer.on(toRendererEventName, (event, data) => {
14 | onData(data);
15 | }),
16 | serialize: (value) => JSON.stringify(value),
17 | deserialize: (value) => JSON.parse(value)
18 | });
19 | }
20 |
21 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/src/vite-env.d.ts:
--------------------------------------------------------------------------------
1 | ///
2 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "target": "es2022",
4 | "lib": ["es2022", "DOM", "DOM.Iterable"],
5 | "module": "es2022",
6 | "skipLibCheck": true,
7 |
8 | "esModuleInterop": true,
9 | "removeComments": false,
10 | "allowSyntheticDefaultImports": true,
11 | "forceConsistentCasingInFileNames": true,
12 | "noFallthroughCasesInSwitch": true,
13 | "noUncheckedIndexedAccess": true,
14 | "moduleDetection": "force",
15 |
16 | "moduleResolution": "bundler",
17 | "allowImportingTsExtensions": true,
18 | "resolveJsonModule": true,
19 | "isolatedModules": true,
20 | "noEmit": true,
21 | "jsx": "react-jsx",
22 |
23 | "strict": true,
24 | "strictNullChecks": true,
25 | "noImplicitAny": true,
26 | "noImplicitReturns": true,
27 | "noImplicitThis": true,
28 | "noImplicitOverride": true
29 | },
30 | "include": [
31 | "./src",
32 | "./electron"
33 | ],
34 | "references": [{ "path": "./tsconfig.node.json" }]
35 | }
36 |
--------------------------------------------------------------------------------
/templates/electron-typescript-react/tsconfig.node.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "target": "es2022",
4 | "lib": ["es2022"],
5 | "module": "es2022",
6 | "skipLibCheck": true,
7 |
8 | "esModuleInterop": true,
9 | "removeComments": false,
10 | "allowSyntheticDefaultImports": true,
11 | "forceConsistentCasingInFileNames": true,
12 | "noFallthroughCasesInSwitch": true,
13 | "noUncheckedIndexedAccess": true,
14 | "moduleDetection": "force",
15 |
16 | "moduleResolution": "bundler",
17 | "resolveJsonModule": false,
18 | "isolatedModules": true,
19 | "composite": true,
20 |
21 | "strict": true,
22 | "strictNullChecks": true,
23 | "noImplicitAny": true,
24 | "noImplicitReturns": true,
25 | "noImplicitThis": true,
26 | "noImplicitOverride": true
27 | },
28 | "include": [
29 | "vite.config.ts"
30 | ]
31 | }
32 |
--------------------------------------------------------------------------------
/templates/node-typescript/.editorconfig:
--------------------------------------------------------------------------------
1 | root = true
2 |
3 | [*]
4 | indent_style = space
5 | indent_size = 4
6 |
7 | [{*.ts,*.tsx,*.js,*.jsx,*.css,*.scss}]
8 | insert_final_newline = true
9 |
10 | [{package.json,package-lock.json,manifest.json}]
11 | indent_size = 2
12 |
13 | [*.yml]
14 | indent_size = 2
15 |
--------------------------------------------------------------------------------
/templates/node-typescript/.gitignore:
--------------------------------------------------------------------------------
1 | /.idea
2 | /.vscode
3 | node_modules
4 | .DS_Store
5 |
6 | /dist
7 | /models
8 |
--------------------------------------------------------------------------------
/templates/node-typescript/README.md:
--------------------------------------------------------------------------------
1 | # Node + TypeScript + `node-llama-cpp`
2 | This template provides a minimal setup to get Node working with TypeScript and `node-llama-cpp`, and some ESLint rules.
3 |
4 | ## Get started
5 | Install node modules and download the model files used by `node-llama-cpp`:
6 | ```bash
7 | npm install
8 | ```
9 |
10 | Start the project:
11 | ```bash
12 | npm start
13 | ```
14 |
15 | > Generated using `npm create node-llama-cpp@latest` ([learn more](https://node-llama-cpp.withcat.ai/guide/))
16 |
--------------------------------------------------------------------------------
/templates/node-typescript/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "lib": ["es2022"],
4 | "module": "es2022",
5 | "target": "es2022",
6 | "esModuleInterop": true,
7 | "noImplicitAny": true,
8 | "noImplicitReturns": true,
9 | "noImplicitThis": true,
10 | "noImplicitOverride": true,
11 | "removeComments": false,
12 | "allowSyntheticDefaultImports": true,
13 | "forceConsistentCasingInFileNames": true,
14 | "noFallthroughCasesInSwitch": true,
15 | "noUncheckedIndexedAccess": true,
16 | "moduleDetection": "force",
17 | "skipLibCheck": true,
18 | "moduleResolution": "node",
19 | "resolveJsonModule": false,
20 | "strictNullChecks": true,
21 | "isolatedModules": true,
22 | "noEmit": false,
23 | "outDir": "./dist",
24 | "strict": true,
25 | "sourceMap": true,
26 | "composite": false,
27 | "declaration": true
28 | },
29 | "files": [
30 | "./src/index.ts"
31 | ],
32 | "include": [
33 | "./src"
34 | ]
35 | }
36 |
--------------------------------------------------------------------------------
/templates/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "eslint-dependencies-for-ci",
3 | "private": true,
4 | "version": "0.0.0",
5 | "type": "module",
6 | "devDependencies": {
7 | "eslint-plugin-react-hooks": "^4.6.0",
8 | "eslint-plugin-react-refresh": "^0.4.5"
9 | }
10 | }
11 |
--------------------------------------------------------------------------------
/test/modelDependent/llama3.2/completion.test.ts:
--------------------------------------------------------------------------------
1 | import {describe, expect, test} from "vitest";
2 | import {LlamaChatSession, Llama3_2LightweightChatWrapper} from "../../../src/index.js";
3 | import {getModelFile} from "../../utils/modelFiles.js";
4 | import {getTestLlama} from "../../utils/getTestLlama.js";
5 |
6 | describe("llama 3.2", () => {
7 | describe("chatSession", () => {
8 | test("resolved to the correct chat wrapper", {timeout: 1000 * 60 * 60 * 2}, async () => {
9 | const modelPath = await getModelFile("Llama-3.2-3B-Instruct.Q4_K_M.gguf");
10 | const llama = await getTestLlama();
11 |
12 | const model = await llama.loadModel({
13 | modelPath
14 | });
15 | const context = await model.createContext({
16 | contextSize: 4096
17 | });
18 | const chatSession = new LlamaChatSession({
19 | contextSequence: context.getSequence()
20 | });
21 |
22 | expect(chatSession.chatWrapper).to.be.instanceof(Llama3_2LightweightChatWrapper);
23 | });
24 | });
25 | });
26 |
--------------------------------------------------------------------------------
/test/standalone/utils/createSplitPartFilename.test.ts:
--------------------------------------------------------------------------------
1 | import {describe, expect, test} from "vitest";
2 | import {createSplitPartFilename} from "../../../src/gguf/utils/resolveSplitGgufParts.js";
3 |
4 |
5 | describe("utils", () => {
6 | describe("createSplitPartFilename", () => {
7 | test("simple initial part filename", async () => {
8 | const partFilename = createSplitPartFilename("Meta-Llama-3.1-70B-Instruct-Q5_K_L.gguf", 3, 10);
9 |
10 | expect(partFilename).to.eql("Meta-Llama-3.1-70B-Instruct-Q5_K_L-00003-of-00010.gguf");
11 | });
12 |
13 | test("initial part filename with existing parts information", async () => {
14 | const partFilename = createSplitPartFilename("Meta-Llama-3.1-70B-Instruct-Q5_K_L-00001-of-00002.gguf", 3, 10);
15 |
16 | expect(partFilename).to.eql("Meta-Llama-3.1-70B-Instruct-Q5_K_L-00003-of-00010.gguf");
17 | });
18 |
19 | test("initial part filename with existing parts information 2", async () => {
20 | const partFilename = createSplitPartFilename("Meta-Llama-3.1-70B-Instruct-Q5_K_L-00002-of-00002.gguf", 3, 10);
21 |
22 | expect(partFilename).to.eql("Meta-Llama-3.1-70B-Instruct-Q5_K_L-00003-of-00010.gguf");
23 | });
24 | });
25 | });
26 |
--------------------------------------------------------------------------------
/test/standalone/utils/modelDownloader.test.ts:
--------------------------------------------------------------------------------
1 | import {describe, expect, test} from "vitest";
2 | import {getFilenameForBinarySplitGgufPartUrls} from "../../../src/gguf/utils/resolveBinarySplitGgufPartUrls.js";
3 |
4 |
5 | describe("utils", () => {
6 | describe("modelDownloader", () => {
7 | test("getFilenameForBinarySplitGgufPartUrls", async () => {
8 | const res = getFilenameForBinarySplitGgufPartUrls([
9 | "https://example.com/model.Q6_K.gguf.part1of2",
10 | "https://example.com/model.Q6_K.gguf.part2of2"
11 | ]);
12 | expect(res).to.eql("model.Q6_K.gguf");
13 | });
14 |
15 | test("getFilenameForBinarySplitGgufPartUrls with search params", async () => {
16 | const res = getFilenameForBinarySplitGgufPartUrls([
17 | "https://example.com/model.Q6_K.gguf.part1of2?hi=true",
18 | "https://example.com/model.Q6_K.gguf.part2of2?hello=hi"
19 | ]);
20 | expect(res).to.eql("model.Q6_K.gguf");
21 | });
22 | });
23 | });
24 |
--------------------------------------------------------------------------------
/test/utils/getTestLlama.ts:
--------------------------------------------------------------------------------
1 | import {getLlama, Llama} from "../../src/index.js";
2 |
3 | let llamaPromise: Promise | null = null;
4 |
5 | export async function getTestLlama() {
6 | if (llamaPromise != null)
7 | return await llamaPromise;
8 |
9 | llamaPromise = createTestLlama();
10 |
11 | return await llamaPromise;
12 | }
13 |
14 | export function createTestLlama() {
15 | return getLlama();
16 | }
17 |
--------------------------------------------------------------------------------
/test/utils/helpers/SpecialTokenSerializer.ts:
--------------------------------------------------------------------------------
1 | import {SnapshotSerializer} from "vitest";
2 |
3 | export default {
4 | serialize(value, config, indentation, depth, refs, printer) {
5 | return "new SpecialTokensText(" + printer(value.value, config, indentation, depth, refs) + ")";
6 | },
7 | test(value) {
8 | return value != null && Object.getPrototypeOf(value).constructor.name === "SpecialTokensText";
9 | }
10 | } satisfies SnapshotSerializer;
11 |
--------------------------------------------------------------------------------
/test/utils/helpers/SpecialTokensTextSerializer.ts:
--------------------------------------------------------------------------------
1 | import {SnapshotSerializer} from "vitest";
2 |
3 | export default {
4 | serialize(value, config, indentation, depth, refs, printer) {
5 | return "new SpecialToken(" + printer(value.value, config, indentation, depth, refs) + ")";
6 | },
7 | test(value) {
8 | return value != null && Object.getPrototypeOf(value).constructor.name === "SpecialToken";
9 | }
10 | } satisfies SnapshotSerializer;
11 |
--------------------------------------------------------------------------------
/test/utils/helpers/getTempTestDir.ts:
--------------------------------------------------------------------------------
1 | import path from "path";
2 | import {fileURLToPath} from "url";
3 | import fs from "fs-extra";
4 |
5 | const __dirname = path.dirname(fileURLToPath(import.meta.url));
6 | const tempDir = path.join(__dirname, "..", "..", ".temp");
7 |
8 | export async function getTempTestDir() {
9 | await fs.ensureDir(tempDir);
10 | return tempDir;
11 | }
12 |
13 | export async function getTempTestFilePath(fileName: string) {
14 | const tempDir = await getTempTestDir();
15 | const currentDate = new Date();
16 | const fileDateString = [
17 | String(currentDate.getFullYear()),
18 | String(currentDate.getMonth() + 1).padStart(2, "0"),
19 | String(currentDate.getDate()).padStart(2, "0"),
20 | String(currentDate.getHours()).padStart(2, "0"),
21 | String(currentDate.getMinutes()).padStart(2, "0"),
22 | String(currentDate.getSeconds()).padStart(2, "0"),
23 | String(currentDate.getMilliseconds()).padStart(3, "0")
24 | ].join("");
25 | const randomText = Math.random()
26 | .toString(36)
27 | .slice(2, 10 + 2);
28 |
29 | const resolvedFilename = fileDateString + "_" + randomText + "_" + fileName;
30 |
31 | return path.join(tempDir, resolvedFilename);
32 | }
33 |
--------------------------------------------------------------------------------
/test/utils/helpers/llamaTextSerializer.ts:
--------------------------------------------------------------------------------
1 | import {SnapshotSerializer} from "vitest";
2 | import {isLlamaText} from "../../../src/index.js";
3 |
4 | export default {
5 | serialize(value, config, indentation, depth, refs, printer) {
6 | return "LlamaText(" + printer(value.values, config, indentation, depth, refs) + ")";
7 | },
8 | test(value) {
9 | return isLlamaText(value);
10 | }
11 | } satisfies SnapshotSerializer;
12 |
--------------------------------------------------------------------------------
/test/utils/helpers/simplifyGgufInfoForTestSnapshot.ts:
--------------------------------------------------------------------------------
1 | import {GgufFileInfo} from "../../../src/gguf/types/GgufFileInfoTypes.js";
2 |
3 | export function simplifyGgufInfoForTestSnapshot(ggufFileInfo: GgufFileInfo) {
4 | const ggufFileInfoCopy = structuredClone(ggufFileInfo);
5 |
6 | // these keys are ignored in tests because they contain very long values, so we don't want to include them in full,
7 | // to make sure we won't make the test snapshots huge, keeping them readable and maintainable
8 | shortenArray(ggufFileInfoCopy.metadata.tokenizer.ggml.tokens, 10);
9 | shortenArray(ggufFileInfoCopy.metadata.tokenizer.ggml.scores, 10);
10 | shortenArray(ggufFileInfoCopy.metadata.tokenizer.ggml.token_type, 10);
11 | shortenArray(ggufFileInfoCopy.metadata.tokenizer.ggml.merges, 10);
12 |
13 | shortenArray(ggufFileInfoCopy.tensorInfo, 4);
14 | shortenArray(ggufFileInfoCopy.fullTensorInfo, 4);
15 |
16 | return ggufFileInfoCopy;
17 | }
18 |
19 | function shortenArray(array?: readonly any[], maxSize: number = 10) {
20 | if (array == null)
21 | return;
22 |
23 | (array as any[]).splice(maxSize);
24 | }
25 |
--------------------------------------------------------------------------------
/test/utils/helpers/testSetup.ts:
--------------------------------------------------------------------------------
1 | import util from "node:util";
2 |
3 | // Increase the default depth of the inspect function to 6 (from the default of 2)
4 | util.inspect.defaultOptions.depth = 6;
5 |
--------------------------------------------------------------------------------
/test/utils/jsonSchemas/message.json:
--------------------------------------------------------------------------------
1 | {
2 | "type": "object",
3 | "properties": {
4 | "responseMessage": {
5 | "type": "string"
6 | },
7 | "requestPositivityScoreFromOneToTen": {
8 | "type": "number"
9 | }
10 | }
11 | }
12 |
--------------------------------------------------------------------------------
/test/utils/scripts/downloadAllTestModels.ts:
--------------------------------------------------------------------------------
1 | import {downloadAllModels} from "../modelFiles.js";
2 |
3 | await downloadAllModels();
4 | process.exit(0);
5 |
--------------------------------------------------------------------------------
/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "lib": ["es2022"],
4 | "module": "es2022",
5 | "target": "es2022",
6 | "esModuleInterop": true,
7 | "noImplicitAny": true,
8 | "noImplicitReturns": true,
9 | "noImplicitThis": true,
10 | "noImplicitOverride": true,
11 | "removeComments": false,
12 | "allowSyntheticDefaultImports": true,
13 | "forceConsistentCasingInFileNames": true,
14 | "noFallthroughCasesInSwitch": true,
15 | "noUncheckedIndexedAccess": true,
16 | "moduleDetection": "force",
17 | "skipLibCheck": true,
18 | "moduleResolution": "node",
19 | "resolveJsonModule": false,
20 | "strictNullChecks": true,
21 | "isolatedModules": true,
22 | "noEmit": false,
23 | "outDir": "./dist",
24 | "strict": true,
25 | "sourceMap": true,
26 | "composite": false,
27 | "declaration": true,
28 | "stripInternal": true
29 | },
30 | "files": [
31 | "./src/index.ts"
32 | ],
33 | "include": [
34 | "./src"
35 | ]
36 | }
37 |
--------------------------------------------------------------------------------
/vitest.config.ts:
--------------------------------------------------------------------------------
1 | import {defineConfig} from "vitest/config";
2 |
3 | export default defineConfig({
4 | test: {
5 | pool: "forks",
6 | maxWorkers: 1,
7 | minWorkers: 1,
8 | maxConcurrency: 1,
9 | poolOptions: {
10 | forks: {
11 | minForks: 1,
12 | maxForks: 1,
13 | singleFork: true
14 |
15 | // uncomment for profiling
16 | // execArgv: [
17 | // "--cpu-prof",
18 | // "--cpu-prof-dir=test-runner-profile",
19 | // "--heap-prof",
20 | // "--heap-prof-dir=test-runner-profile"
21 | // ]
22 | }
23 | },
24 | snapshotSerializers: [
25 | "./test/utils/helpers/llamaTextSerializer.ts",
26 | "./test/utils/helpers/SpecialTokensTextSerializer.ts",
27 | "./test/utils/helpers/SpecialTokenSerializer.ts"
28 | ],
29 | setupFiles: ["./test/utils/helpers/testSetup.ts"]
30 | }
31 | });
32 |
--------------------------------------------------------------------------------