├── tests ├── integration │ ├── assets │ │ ├── test-file-for-cache.txt │ │ ├── invoice.jpg │ │ ├── timer_video.mp4 │ │ ├── google_invoice.pdf │ │ └── two_minute_rules.mp3 │ └── client │ │ ├── domains.test.ts │ │ ├── model.test.ts │ │ ├── hub.test.ts │ │ ├── audio-predictions.test.ts │ │ ├── video-predictions.test.ts │ │ ├── image-predictions.test.ts │ │ └── agent.test.ts ├── unit │ └── client │ │ ├── domains.test.ts │ │ ├── model.test.ts │ │ ├── vlmrun.test.ts │ │ ├── hub.test.ts │ │ ├── executions.test.ts │ │ ├── exceptions.test.ts │ │ ├── requestor.test.ts │ │ ├── datasets.test.ts │ │ ├── fine_tuning.test.ts │ │ ├── audio-predictions.test.ts │ │ └── video-predictions.test.ts └── e2e │ └── client │ └── chat-completions.test.ts ├── src ├── utils │ ├── index.ts │ ├── utils.ts │ ├── image.ts │ ├── file.ts │ └── webhook.ts ├── client │ ├── models.ts │ ├── domains.ts │ ├── hub.ts │ ├── feedback.ts │ ├── executions.ts │ ├── datasets.ts │ ├── base_requestor.ts │ ├── fine_tuning.ts │ ├── files.ts │ ├── exceptions │ │ └── index.ts │ └── agent.ts └── index.ts ├── .gitignore ├── scripts ├── format ├── lint ├── bootstrap ├── utils │ ├── git-swap.sh │ ├── fix-index-exports.cjs │ ├── check-is-in-git-install.sh │ ├── make-dist-package-json.cjs │ ├── check-version.cjs │ └── postprocess-files.cjs ├── mock ├── test └── build ├── tsup.config.ts ├── .eslintrc.js ├── jest.config.js ├── jest.e2e.config.js ├── jest.integration.all.config.js ├── jest.integration.config.js ├── tsconfig.json ├── examples ├── models.ts ├── feedback.ts ├── files.ts └── predictions.ts ├── bin └── publish-npm ├── .github └── workflows │ ├── ci.yml │ └── release-on-merge.yml ├── package.json ├── CLAUDE.md ├── CHANGELOG.md ├── docs └── CONTRIBUTING.md └── README.md /tests/integration/assets/test-file-for-cache.txt: -------------------------------------------------------------------------------- 1 | Test non-existent cached file -------------------------------------------------------------------------------- /tests/integration/assets/invoice.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vlm-run/vlmrun-node-sdk/main/tests/integration/assets/invoice.jpg -------------------------------------------------------------------------------- /src/utils/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./image"; 2 | export * from "./file"; 3 | export * from "./utils"; 4 | export * from "./webhook"; 5 | -------------------------------------------------------------------------------- /tests/integration/assets/timer_video.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vlm-run/vlmrun-node-sdk/main/tests/integration/assets/timer_video.mp4 -------------------------------------------------------------------------------- /tests/integration/assets/google_invoice.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vlm-run/vlmrun-node-sdk/main/tests/integration/assets/google_invoice.pdf -------------------------------------------------------------------------------- /tests/integration/assets/two_minute_rules.mp3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vlm-run/vlmrun-node-sdk/main/tests/integration/assets/two_minute_rules.mp3 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .prism.log 2 | node_modules 3 | yarn-error.log 4 | codegen.log 5 | Brewfile.lock.json 6 | dist 7 | /*.tgz 8 | .idea/ 9 | .env 10 | .env.test 11 | .vscode/ 12 | .DS_Store -------------------------------------------------------------------------------- /scripts/format: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")/.." 6 | 7 | echo "==> Running eslint --fix" 8 | ESLINT_USE_FLAT_CONFIG="false" ./node_modules/.bin/eslint --fix --ext ts,js . 9 | -------------------------------------------------------------------------------- /tsup.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'tsup'; 2 | 3 | export default defineConfig({ 4 | entry: ['./src/index.ts'], 5 | format: ['cjs', 'esm'], 6 | dts: true, 7 | sourcemap: true, 8 | skipNodeModulesBundle: true, 9 | clean: true, 10 | }); -------------------------------------------------------------------------------- /scripts/lint: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")/.." 6 | 7 | echo "==> Running eslint" 8 | ESLINT_USE_FLAT_CONFIG="false" ./node_modules/.bin/eslint --ext ts,js . 9 | 10 | echo "==> Running tsc" 11 | ./node_modules/.bin/tsc --noEmit 12 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | parser: '@typescript-eslint/parser', 3 | plugins: ['@typescript-eslint', 'unused-imports', 'prettier'], 4 | rules: { 5 | 'no-unused-vars': 'off', 6 | 'prettier/prettier': 'error', 7 | 'unused-imports/no-unused-imports': 'error', 8 | }, 9 | root: true, 10 | }; 11 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: 'ts-jest', 3 | testEnvironment: 'node', 4 | testMatch: ['**/tests/unit/**/*.test.ts'], 5 | moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'], 6 | transform: { 7 | '^.+\\.tsx?$': ['ts-jest', { 8 | useESM: true, 9 | }] 10 | }, 11 | extensionsToTreatAsEsm: [".ts"], 12 | }; 13 | -------------------------------------------------------------------------------- /jest.e2e.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: 'ts-jest', 3 | testEnvironment: 'node', 4 | testMatch: ['**/tests/e2e/**/*.test.ts'], 5 | moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'], 6 | transform: { 7 | '^.+\\.tsx?$': ['ts-jest', { 8 | useESM: true, 9 | }] 10 | }, 11 | extensionsToTreatAsEsm: [".ts"], 12 | }; 13 | -------------------------------------------------------------------------------- /jest.integration.all.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: 'ts-jest', 3 | testEnvironment: 'node', 4 | testMatch: ['**/tests/integration/**/*.test.ts'], 5 | moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'], 6 | transform: { 7 | '^.+\\.tsx?$': ['ts-jest', { 8 | useESM: true, 9 | }] 10 | }, 11 | extensionsToTreatAsEsm: [".ts"], 12 | }; -------------------------------------------------------------------------------- /scripts/bootstrap: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")/.." 6 | 7 | if [ -f "Brewfile" ] && [ "$(uname -s)" = "Darwin" ]; then 8 | brew bundle check >/dev/null 2>&1 || { 9 | echo "==> Installing Homebrew dependencies…" 10 | brew bundle 11 | } 12 | fi 13 | 14 | echo "==> Installing Node dependencies…" 15 | 16 | PACKAGE_MANAGER=$(command -v yarn >/dev/null 2>&1 && echo "yarn" || echo "npm") 17 | 18 | $PACKAGE_MANAGER install 19 | -------------------------------------------------------------------------------- /scripts/utils/git-swap.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -exuo pipefail 3 | # the package is published to NPM from ./dist 4 | # we want the final file structure for git installs to match the npm installs, so we 5 | 6 | # delete everything except ./dist and ./node_modules 7 | find . -maxdepth 1 -mindepth 1 ! -name 'dist' ! -name 'node_modules' -exec rm -rf '{}' + 8 | 9 | # move everything from ./dist to . 10 | mv dist/* . 11 | 12 | # delete the now-empty ./dist 13 | rmdir dist 14 | -------------------------------------------------------------------------------- /jest.integration.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: 'ts-jest', 3 | testEnvironment: 'node', 4 | testMatch: ['**/tests/integration/**/*.test.ts'], 5 | testPathIgnorePatterns: [ 6 | 'audio-predictions.test.ts', 7 | 'video-predictions.test.ts' 8 | ], 9 | moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'], 10 | transform: { 11 | '^.+\\.tsx?$': ['ts-jest', { 12 | useESM: true, 13 | }] 14 | }, 15 | extensionsToTreatAsEsm: [".ts"], 16 | }; 17 | -------------------------------------------------------------------------------- /scripts/utils/fix-index-exports.cjs: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const path = require('path'); 3 | 4 | const indexJs = 5 | process.env['DIST_PATH'] ? 6 | path.resolve(process.env['DIST_PATH'], 'index.js') 7 | : path.resolve(__dirname, '..', '..', 'dist', 'index.js'); 8 | 9 | let before = fs.readFileSync(indexJs, 'utf8'); 10 | let after = before.replace( 11 | /^\s*exports\.default\s*=\s*(\w+)/m, 12 | 'exports = module.exports = $1;\nexports.default = $1', 13 | ); 14 | fs.writeFileSync(indexJs, after, 'utf8'); 15 | -------------------------------------------------------------------------------- /scripts/utils/check-is-in-git-install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Check if you happen to call prepare for a repository that's already in node_modules. 3 | [ "$(basename "$(dirname "$PWD")")" = 'node_modules' ] || 4 | # The name of the containing directory that 'npm` uses, which looks like 5 | # $HOME/.npm/_cacache/git-cloneXXXXXX 6 | [ "$(basename "$(dirname "$PWD")")" = 'tmp' ] || 7 | # The name of the containing directory that 'yarn` uses, which looks like 8 | # $(yarn cache dir)/.tmp/XXXXX 9 | [ "$(basename "$(dirname "$PWD")")" = '.tmp' ] 10 | -------------------------------------------------------------------------------- /src/client/models.ts: -------------------------------------------------------------------------------- 1 | import { Client, APIRequestor } from './base_requestor'; 2 | import { ModelInfoResponse } from './types'; 3 | 4 | export class Models { 5 | private client: Client; 6 | private requestor: APIRequestor; 7 | 8 | constructor(client: Client) { 9 | this.client = client; 10 | this.requestor = new APIRequestor(client); 11 | } 12 | 13 | async list(): Promise { 14 | const [response] = await this.requestor.request( 15 | 'GET', 16 | 'models' 17 | ); 18 | return response; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2019", 4 | "strict": true, 5 | "declaration": true, 6 | "declarationDir": "dist/types", 7 | "sourceMap": true, 8 | "esModuleInterop": true, 9 | "moduleResolution": "node", 10 | "outDir": "./dist", 11 | "rootDir": ".", 12 | "skipLibCheck": true, 13 | "forceConsistentCasingInFileNames": true, 14 | "resolveJsonModule": true, 15 | "lib": ["ES2019", "DOM"], 16 | "types": ["jest", "node"] 17 | }, 18 | "include": ["src/**/*", "tests/**/*"] 19 | } 20 | -------------------------------------------------------------------------------- /examples/models.ts: -------------------------------------------------------------------------------- 1 | import { VlmRun } from "../src"; 2 | 3 | const client = new VlmRun({ 4 | apiKey: "your-api-key", // Replace with your actual API key 5 | }); 6 | 7 | async function listModels() { 8 | try { 9 | const models = await client.models.list(); 10 | 11 | console.log("Available models:"); 12 | models.forEach((model) => { 13 | console.log(`- Model: ${model.model}, Domain: ${model.domain}`); 14 | }); 15 | 16 | return models; 17 | } catch (error) { 18 | console.error("Error listing models:", error); 19 | throw error; 20 | } 21 | } 22 | 23 | listModels(); 24 | -------------------------------------------------------------------------------- /bin/publish-npm: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eux 4 | 5 | npm config set '//registry.npmjs.org/:_authToken' "$NPM_TOKEN" 6 | 7 | # Build the project 8 | yarn build 9 | 10 | # Navigate to the dist directory 11 | cd dist 12 | 13 | # Get the version from package.json 14 | VERSION="$(node -p "require('../package.json').version")" 15 | 16 | # Extract the pre-release tag if it exists 17 | if [[ "$VERSION" =~ -([a-zA-Z]+) ]]; then 18 | # Extract the part before any dot in the pre-release identifier 19 | TAG="${BASH_REMATCH[1]}" 20 | else 21 | TAG="latest" 22 | fi 23 | 24 | # Publish with the appropriate tag 25 | yarn publish --access public --tag "$TAG" 26 | -------------------------------------------------------------------------------- /scripts/utils/make-dist-package-json.cjs: -------------------------------------------------------------------------------- 1 | const pkgJson = require(process.env['PKG_JSON_PATH'] || '../../package.json'); 2 | 3 | function processExportMap(m) { 4 | for (const key in m) { 5 | const value = m[key]; 6 | if (typeof value === 'string') m[key] = value.replace(/^\.\/dist\//, './'); 7 | else processExportMap(value); 8 | } 9 | } 10 | processExportMap(pkgJson.exports); 11 | 12 | for (const key of ['types', 'main', 'module']) { 13 | if (typeof pkgJson[key] === 'string') pkgJson[key] = pkgJson[key].replace(/^(\.\/)?dist\//, './'); 14 | } 15 | 16 | delete pkgJson.devDependencies; 17 | delete pkgJson.scripts.prepack; 18 | delete pkgJson.scripts.prepublishOnly; 19 | delete pkgJson.scripts.prepare; 20 | 21 | console.log(JSON.stringify(pkgJson, null, 2)); 22 | -------------------------------------------------------------------------------- /scripts/utils/check-version.cjs: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const path = require('path'); 3 | 4 | const main = () => { 5 | const pkg = require('../../package.json'); 6 | const version = pkg['version']; 7 | if (!version) throw 'The version property is not set in the package.json file'; 8 | if (typeof version !== 'string') { 9 | throw `Unexpected type for the package.json version field; got ${typeof version}, expected string`; 10 | } 11 | 12 | const versionFile = path.resolve(__dirname, '..', '..', 'src', 'version.ts'); 13 | const contents = fs.readFileSync(versionFile, 'utf8'); 14 | const output = contents.replace(/(export const VERSION = ')(.*)(')/g, `$1${version}$3`); 15 | fs.writeFileSync(versionFile, output); 16 | }; 17 | 18 | if (require.main === module) { 19 | main(); 20 | } 21 | -------------------------------------------------------------------------------- /src/utils/utils.ts: -------------------------------------------------------------------------------- 1 | import { ZodType } from "zod"; 2 | import zodToJsonSchema from "zod-to-json-schema"; 3 | 4 | /** 5 | * Converts a value to JSON schema if it's a Zod schema, otherwise returns it as-is 6 | * @param schema - The schema to convert (can be Zod schema or plain JSON schema) 7 | * @returns Converted JSON schema or the original value 8 | */ 9 | export function convertToJsonSchema( 10 | schema: ZodType | Record | null | undefined, 11 | zodToJsonParams?: any 12 | ): Record | null | undefined { 13 | const isZodSchema = 14 | schema instanceof ZodType || 15 | typeof (schema as any)?.safeParse === "function"; 16 | 17 | if (isZodSchema && schema) { 18 | return zodToJsonSchema(schema as ZodType, zodToJsonParams); 19 | } 20 | 21 | return schema; 22 | } 23 | -------------------------------------------------------------------------------- /scripts/mock: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")/.." 6 | 7 | if [[ -n "$1" && "$1" != '--'* ]]; then 8 | URL="$1" 9 | shift 10 | else 11 | URL="$(grep 'openapi_spec_url' .stats.yml | cut -d' ' -f2)" 12 | fi 13 | 14 | # Check if the URL is empty 15 | if [ -z "$URL" ]; then 16 | echo "Error: No OpenAPI spec path/url provided or found in .stats.yml" 17 | exit 1 18 | fi 19 | 20 | echo "==> Starting mock server with URL ${URL}" 21 | 22 | # Run prism mock on the given spec 23 | if [ "$1" == "--daemon" ]; then 24 | npm exec --package=@stainless-api/prism-cli@5.8.5 -- prism mock "$URL" &> .prism.log & 25 | 26 | # Wait for server to come online 27 | echo -n "Waiting for server" 28 | while ! grep -q "✖ fatal\|Prism is listening" ".prism.log" ; do 29 | echo -n "." 30 | sleep 0.1 31 | done 32 | 33 | if grep -q "✖ fatal" ".prism.log"; then 34 | cat .prism.log 35 | exit 1 36 | fi 37 | 38 | echo 39 | else 40 | npm exec --package=@stainless-api/prism-cli@5.8.5 -- prism mock "$URL" 41 | fi 42 | -------------------------------------------------------------------------------- /tests/integration/client/domains.test.ts: -------------------------------------------------------------------------------- 1 | import { DomainInfo } from "../../../src/client/types"; 2 | import { VlmRun } from "../../../src/index"; 3 | import { config } from "dotenv"; 4 | 5 | jest.setTimeout(60000); 6 | 7 | describe("Integration: Domains", () => { 8 | let client: VlmRun; 9 | 10 | beforeAll(() => { 11 | config({ path: ".env.test" }); 12 | 13 | client = new VlmRun({ 14 | apiKey: process.env.TEST_API_KEY ?? "", 15 | baseURL: process.env.TEST_BASE_URL, 16 | }); 17 | }); 18 | 19 | describe("listDomains()", () => { 20 | it("should successfully fetch domains list", async () => { 21 | const result = await client.domains.list(); 22 | 23 | expect(Array.isArray(result)).toBe(true); 24 | if (result.length > 0) { 25 | const domain: DomainInfo = result[0]; 26 | 27 | expect(domain).toHaveProperty("domain"); 28 | } 29 | }); 30 | 31 | it("should handle API errors with invalid credentials", async () => { 32 | const clientWithInvalidKey = new VlmRun({ 33 | apiKey: "invalid-api-key", 34 | baseURL: process.env.TEST_BASE_URL, 35 | }); 36 | 37 | await expect(clientWithInvalidKey.domains.list()).rejects.toThrow(); 38 | }); 39 | }); 40 | }); 41 | -------------------------------------------------------------------------------- /tests/integration/client/model.test.ts: -------------------------------------------------------------------------------- 1 | import { ModelInfoResponse } from "../../../src/client/types"; 2 | import { VlmRun } from "../../../src/index"; 3 | import { config } from 'dotenv'; 4 | 5 | jest.setTimeout(60000); 6 | 7 | describe("Integration: Models", () => { 8 | let client: VlmRun; 9 | 10 | beforeAll(() => { 11 | config({ path: '.env.test' }); 12 | 13 | client = new VlmRun({ 14 | apiKey: process.env.TEST_API_KEY ?? '', 15 | baseURL: process.env.TEST_BASE_URL, 16 | }); 17 | }); 18 | 19 | describe("list()", () => { 20 | it("should successfully fetch models list", async () => { 21 | const result = await client.models.list(); 22 | 23 | expect(Array.isArray(result)).toBe(true); 24 | if (result.length > 0) { 25 | const model: ModelInfoResponse = result[0]; 26 | expect(model).toHaveProperty("model"); 27 | expect(model).toHaveProperty("domain"); 28 | } 29 | }); 30 | 31 | it("should handle API errors", async () => { 32 | const clientWithInvalidKey = new VlmRun({ 33 | apiKey: "invalid-api-key", 34 | baseURL: process.env.TEST_BASE_URL, 35 | }); 36 | 37 | await expect(clientWithInvalidKey.models.list()).rejects.toThrow(); 38 | }); 39 | }); 40 | }); 41 | -------------------------------------------------------------------------------- /tests/unit/client/domains.test.ts: -------------------------------------------------------------------------------- 1 | import { VlmRun } from "../../../src"; 2 | import { DomainInfo } from "../../../src/client/types"; 3 | 4 | describe("Domains", () => { 5 | let client: VlmRun; 6 | 7 | beforeEach(() => { 8 | client = new VlmRun({ 9 | apiKey: "test-api-key", 10 | baseURL: "https://api.example.com", 11 | }); 12 | }); 13 | 14 | describe("listDomains", () => { 15 | it("should return list of domains", async () => { 16 | const mockResponse: DomainInfo[] = [ 17 | { 18 | domain: "document.invoice", 19 | name: "Invoice", 20 | description: "Invoice document type", 21 | }, 22 | ]; 23 | 24 | jest 25 | .spyOn(client.domains["requestor"], "request") 26 | .mockResolvedValueOnce([mockResponse, 200, {}]); 27 | 28 | const result = await client.domains.list(); 29 | expect(result).toEqual(mockResponse); 30 | expect(client.domains["requestor"].request).toHaveBeenCalledWith( 31 | "GET", 32 | "/domains" 33 | ); 34 | }); 35 | 36 | it("should handle errors", async () => { 37 | jest 38 | .spyOn(client.domains["requestor"], "request") 39 | .mockRejectedValueOnce(new Error("Network error")); 40 | 41 | await expect(client.domains.list()).rejects.toThrow( 42 | "Failed to list domains" 43 | ); 44 | }); 45 | }); 46 | }); 47 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: 3 | push: 4 | branches: 5 | - main 6 | pull_request: 7 | branches: 8 | - main 9 | - next 10 | 11 | jobs: 12 | security: 13 | name: security 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | - uses: actions/checkout@v4 18 | 19 | - name: Set up Node 20 | uses: actions/setup-node@v4 21 | with: 22 | node-version: "20" 23 | 24 | - name: Install dependencies 25 | run: npm install 26 | 27 | - name: Security audit 28 | run: npm audit --audit-level moderate 29 | 30 | build: 31 | name: build 32 | runs-on: ubuntu-latest 33 | needs: security 34 | 35 | steps: 36 | - uses: actions/checkout@v4 37 | 38 | - name: Set up Node 39 | uses: actions/setup-node@v4 40 | with: 41 | node-version: "20" 42 | 43 | - name: Install dependencies 44 | run: npm install 45 | 46 | - name: Build 47 | run: npm run build 48 | 49 | test: 50 | name: test 51 | runs-on: ubuntu-latest 52 | needs: security 53 | 54 | steps: 55 | - uses: actions/checkout@v4 56 | 57 | - name: Set up Node 58 | uses: actions/setup-node@v4 59 | with: 60 | node-version: "20" 61 | 62 | - name: Install dependencies 63 | run: npm install 64 | 65 | - name: Run tests 66 | run: npm run test 67 | -------------------------------------------------------------------------------- /examples/feedback.ts: -------------------------------------------------------------------------------- 1 | import { VlmRun } from "../src"; 2 | 3 | const client = new VlmRun({ 4 | apiKey: "your-api-key", // Replace with your actual API key 5 | }); 6 | 7 | async function submitFeedback() { 8 | try { 9 | console.log("Making a prediction..."); 10 | const imageUrl = "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/document.invoice/invoice_1.jpg"; 11 | 12 | const prediction = await client.image.generate({ 13 | images: [imageUrl], 14 | model: "vlm-1", 15 | domain: "document.invoice", 16 | }); 17 | 18 | console.log(`Prediction completed with ID: ${prediction.id}`); 19 | 20 | console.log("Submitting positive feedback..."); 21 | const positiveFeedback = await client.feedback.submit( 22 | prediction.id, 23 | { corrected_total: 1250.00 }, 24 | "The extraction was accurate and complete!" 25 | ); 26 | 27 | console.log("Positive feedback submitted:", positiveFeedback); 28 | 29 | console.log("Submitting negative feedback with corrections..."); 30 | const negativeFeedback = await client.feedback.submit( 31 | prediction.id, 32 | { total_amount: 1250.00 }, 33 | "The total amount was incorrect" 34 | ); 35 | 36 | console.log("Negative feedback submitted:", negativeFeedback); 37 | 38 | return { positiveFeedback, negativeFeedback }; 39 | } catch (error) { 40 | console.error("Error submitting feedback:", error); 41 | throw error; 42 | } 43 | } 44 | 45 | submitFeedback(); 46 | -------------------------------------------------------------------------------- /tests/integration/client/hub.test.ts: -------------------------------------------------------------------------------- 1 | import { DomainInfo } from "../../../src/client/types"; 2 | import { VlmRun } from "../../../src/index"; 3 | import { config } from "dotenv"; 4 | 5 | jest.setTimeout(60000); 6 | 7 | describe("Integration: Hubs", () => { 8 | let client: VlmRun; 9 | 10 | beforeAll(() => { 11 | config({ path: ".env.test" }); 12 | 13 | client = new VlmRun({ 14 | apiKey: process.env.TEST_API_KEY ?? "", 15 | baseURL: process.env.TEST_BASE_URL, 16 | }); 17 | }); 18 | 19 | describe("listDomains()", () => { 20 | it("should successfully fetch domains list", async () => { 21 | const result = await client.hub.listDomains(); 22 | 23 | expect(Array.isArray(result)).toBe(true); 24 | if (result.length > 0) { 25 | const domain: DomainInfo = result[0]; 26 | 27 | expect(domain).toHaveProperty("domain"); 28 | } 29 | }); 30 | }); 31 | 32 | describe("info()", () => { 33 | it("should successfully fetch info", async () => { 34 | const result = await client.hub.info(); 35 | 36 | expect(result).toHaveProperty("version"); 37 | }); 38 | }); 39 | 40 | describe("getSchema()", () => { 41 | it("should successfully fetch schema without gql_stmt", async () => { 42 | const result = await client.hub.getSchema({ domain: "document.invoice" }); 43 | expect(result).toHaveProperty("json_schema"); 44 | expect(result).toHaveProperty("schema_version"); 45 | expect(result).toHaveProperty("schema_hash"); 46 | }); 47 | }); 48 | }); 49 | -------------------------------------------------------------------------------- /examples/files.ts: -------------------------------------------------------------------------------- 1 | import { VlmRun } from "../src"; 2 | 3 | const client = new VlmRun({ 4 | apiKey: "your-api-key", // Replace with your actual API key 5 | }); 6 | 7 | async function fileOperations() { 8 | try { 9 | console.log("Listing files..."); 10 | const files = await client.files.list({ limit: 10 }); 11 | console.log(`Found ${files.length} files`); 12 | 13 | console.log("Uploading a file..."); 14 | const uploadResult = await client.files.upload({ 15 | filePath: "path/to/your/document.pdf", // Replace with actual file path 16 | purpose: "assistants", 17 | checkDuplicate: true, 18 | }); 19 | 20 | console.log(`File uploaded successfully:`); 21 | console.log(`- ID: ${uploadResult.id}`); 22 | console.log(`- Filename: ${uploadResult.filename}`); 23 | console.log(`- Size: ${uploadResult.bytes} bytes`); 24 | 25 | console.log("Getting file details..."); 26 | const fileDetails = await client.files.get(uploadResult.id); 27 | console.log(`File details: ${JSON.stringify(fileDetails, null, 2)}`); 28 | 29 | console.log("Checking for cached file..."); 30 | const cachedFile = await client.files.getCachedFile("path/to/your/document.pdf"); 31 | if (cachedFile) { 32 | console.log(`Cached file found: ${cachedFile.id}`); 33 | } else { 34 | console.log("No cached file found"); 35 | } 36 | 37 | return uploadResult; 38 | } catch (error) { 39 | console.error("Error with file operations:", error); 40 | throw error; 41 | } 42 | } 43 | 44 | fileOperations(); 45 | -------------------------------------------------------------------------------- /src/utils/image.ts: -------------------------------------------------------------------------------- 1 | import { extname } from 'path'; 2 | import { DependencyError, InputError } from '../client/exceptions'; 3 | 4 | /** 5 | * Encodes an image file to base64 6 | * @param imagePath Path to the image file 7 | * @returns Base64 encoded image with data URI prefix 8 | */ 9 | export function encodeImage(imagePath: string): string { 10 | if (typeof window !== 'undefined') { 11 | throw new DependencyError('Image encoding is not supported in the browser', 'browser_limitation', 'Use server-side image encoding instead'); 12 | } 13 | const { readFileSync } = require('fs'); 14 | const imageBuffer = readFileSync(imagePath); 15 | const ext = extname(imagePath).toLowerCase().slice(1); 16 | const mimeType = `image/${ext === 'jpg' ? 'jpeg' : ext}`; 17 | const base64Image = imageBuffer.toString('base64'); 18 | return `data:${mimeType};base64,${base64Image}`; 19 | } 20 | 21 | /** 22 | * Checks if a file is an image based on its extension 23 | * @param image Path to the file or base64 encoded image 24 | * @returns string with base64 encoded image 25 | */ 26 | export function processImage(image: string): string { 27 | if (image.startsWith('data:image/')) { 28 | return image; 29 | } else if (image.startsWith('https://') || image.startsWith('http://')) { 30 | return image; 31 | } else if (['.jpg', '.jpeg', '.png', '.gif', '.webp'].includes(extname(image).toLowerCase())) { 32 | return encodeImage(image); 33 | } 34 | 35 | throw new InputError(`Invalid image file: ${image}`, "invalid_format", "Provide a valid image file path, URL, or base64 encoded image"); 36 | } 37 | -------------------------------------------------------------------------------- /scripts/test: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")/.." 6 | 7 | RED='\033[0;31m' 8 | GREEN='\033[0;32m' 9 | YELLOW='\033[0;33m' 10 | NC='\033[0m' # No Color 11 | 12 | function prism_is_running() { 13 | curl --silent "http://localhost:4010" >/dev/null 2>&1 14 | } 15 | 16 | kill_server_on_port() { 17 | pids=$(lsof -t -i tcp:"$1" || echo "") 18 | if [ "$pids" != "" ]; then 19 | kill "$pids" 20 | echo "Stopped $pids." 21 | fi 22 | } 23 | 24 | function is_overriding_api_base_url() { 25 | [ -n "$TEST_API_BASE_URL" ] 26 | } 27 | 28 | if ! is_overriding_api_base_url && ! prism_is_running ; then 29 | # When we exit this script, make sure to kill the background mock server process 30 | trap 'kill_server_on_port 4010' EXIT 31 | 32 | # Start the dev server 33 | ./scripts/mock --daemon 34 | fi 35 | 36 | if is_overriding_api_base_url ; then 37 | echo -e "${GREEN}✔ Running tests against ${TEST_API_BASE_URL}${NC}" 38 | echo 39 | elif ! prism_is_running ; then 40 | echo -e "${RED}ERROR:${NC} The test suite will not run without a mock Prism server" 41 | echo -e "running against your OpenAPI spec." 42 | echo 43 | echo -e "To run the server, pass in the path or url of your OpenAPI" 44 | echo -e "spec to the prism command:" 45 | echo 46 | echo -e " \$ ${YELLOW}npm exec --package=@stoplight/prism-cli@~5.3.2 -- prism mock path/to/your.openapi.yml${NC}" 47 | echo 48 | 49 | exit 1 50 | else 51 | echo -e "${GREEN}✔ Mock prism server is running with your OpenAPI spec${NC}" 52 | echo 53 | fi 54 | 55 | echo "==> Running tests" 56 | ./node_modules/.bin/jest "$@" 57 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "vlmrun", 3 | "version": "1.0.7", 4 | "description": "The official TypeScript library for the VlmRun API", 5 | "author": "VlmRun ", 6 | "main": "dist/index.js", 7 | "module": "dist/index.mjs", 8 | "types": "dist/index.d.ts", 9 | "type": "commonjs", 10 | "repository": "github:vlm-run/vlmrun-node-sdk", 11 | "license": "Apache-2.0", 12 | "keywords": [], 13 | "packageManager": "yarn@1.22.22", 14 | "private": false, 15 | "files": [ 16 | "**/*" 17 | ], 18 | "scripts": { 19 | "build": "tsup", 20 | "clean": "rm -rf dist", 21 | "test": "jest", 22 | "test:watch": "jest --watch", 23 | "test:coverage": "jest --coverage", 24 | "test:integration": "jest --config=jest.integration.config.js", 25 | "test:integration:all": "jest --config=jest.integration.all.config.js", 26 | "test:e2e": "jest --config=jest.e2e.config.js" 27 | }, 28 | "dependencies": { 29 | "axios": "^1.12.2", 30 | "axios-retry": "^4.5.0", 31 | "dotenv": "^16.4.7", 32 | "mime-types": "^2.1.35", 33 | "path": "^0.12.7", 34 | "tar": "^7.4.3", 35 | "zod": "~3.24.2", 36 | "zod-to-json-schema": "~3.24.1" 37 | }, 38 | "devDependencies": { 39 | "@types/jest": "^29.5.14", 40 | "@types/node": "^22.13.0", 41 | "jest": "^30.1.3", 42 | "openai": "^4.0.0", 43 | "ts-jest": "^29.4.1", 44 | "tsup": "^8.3.6", 45 | "typescript": "^5.7.3" 46 | }, 47 | "peerDependencies": { 48 | "openai": "^4.0.0" 49 | }, 50 | "peerDependenciesMeta": { 51 | "openai": { 52 | "optional": true 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/client/domains.ts: -------------------------------------------------------------------------------- 1 | import { Client, APIRequestor } from "./base_requestor"; 2 | import { APIError } from "./exceptions"; 3 | import { 4 | DomainInfo, 5 | SchemaResponse, 6 | GenerationConfig, 7 | GenerationConfigInput 8 | } from "./types"; 9 | 10 | export class Domains { 11 | private client: Client; 12 | private requestor: APIRequestor; 13 | 14 | constructor(client: Client) { 15 | this.client = client; 16 | this.requestor = new APIRequestor(client); 17 | } 18 | 19 | /** 20 | * Get the list of supported domains. 21 | * @returns List of domain information 22 | * @throws APIError if the request fails 23 | */ 24 | async list(): Promise { 25 | try { 26 | const [response] = await this.requestor.request( 27 | "GET", 28 | "/domains" 29 | ); 30 | return response; 31 | } catch (e) { 32 | throw new APIError(`Failed to list domains: ${e}`); 33 | } 34 | } 35 | 36 | /** 37 | * Get the schema for a domain. 38 | * @param domain Domain name (e.g. "document.invoice") 39 | * @param config Optional generation config 40 | * @returns Schema response containing JSON schema and metadata 41 | * @throws APIError if the request fails 42 | */ 43 | async getSchema( 44 | domain: string, 45 | config?: GenerationConfigInput 46 | ): Promise { 47 | try { 48 | const configObj = config instanceof GenerationConfig ? config : new GenerationConfig(config); 49 | const [response] = await this.requestor.request( 50 | "POST", 51 | "/schema", 52 | undefined, 53 | { domain, config: configObj.toJSON() } 54 | ); 55 | return response; 56 | } catch (e) { 57 | throw new APIError(`Failed to get schema for domain ${domain}: ${e}`); 58 | } 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /tests/unit/client/model.test.ts: -------------------------------------------------------------------------------- 1 | import { ModelInfoResponse } from "../../../src/client/types"; 2 | import { VlmRun } from "../../../src/index"; 3 | 4 | describe("Models", () => { 5 | let client: VlmRun; 6 | 7 | beforeEach(() => { 8 | client = new VlmRun({ 9 | apiKey: "test-api-key", 10 | baseURL: "https://api.example.com", 11 | }); 12 | }); 13 | 14 | describe("list()", () => { 15 | it("should successfully fetch models list", async () => { 16 | // Mock response data 17 | const mockModels: ModelInfoResponse[] = [ 18 | { 19 | model: "vlm-1", 20 | domain: "healthcare.patient-medical-history", 21 | }, 22 | { 23 | model: "vlm-1", 24 | domain: "document.invoice", 25 | }, 26 | ]; 27 | 28 | // Mock the requestor.request method 29 | jest 30 | .spyOn(client.models["requestor"], "request") 31 | .mockResolvedValueOnce([mockModels, 200, {}]); 32 | 33 | // Make the API call 34 | const result = await client.models.list(); 35 | 36 | // Verify the results 37 | expect(result).toEqual(mockModels); 38 | expect(client.models["requestor"].request).toHaveBeenCalledWith( 39 | "GET", 40 | "models" 41 | ); 42 | expect(client.models["requestor"].request).toHaveBeenCalledTimes(1); 43 | }); 44 | 45 | it("should handle API errors", async () => { 46 | // Mock an API error 47 | const errorMessage = "API Error"; 48 | jest 49 | .spyOn(client.models["requestor"], "request") 50 | .mockRejectedValueOnce(new Error(errorMessage)); 51 | 52 | // Attempt the API call and expect it to throw 53 | await expect(client.models.list()).rejects.toThrow(errorMessage); 54 | expect(client.models["requestor"].request).toHaveBeenCalledWith( 55 | "GET", 56 | "models" 57 | ); 58 | expect(client.models["requestor"].request).toHaveBeenCalledTimes(1); 59 | }); 60 | }); 61 | }); 62 | -------------------------------------------------------------------------------- /scripts/build: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -exuo pipefail 4 | 5 | cd "$(dirname "$0")/.." 6 | 7 | node scripts/utils/check-version.cjs 8 | 9 | # Build into dist and will publish the package from there, 10 | # so that src/resources/foo.ts becomes /resources/foo.js 11 | # This way importing from `"vlmrun/resources/foo"` works 12 | # even with `"moduleResolution": "node"` 13 | 14 | rm -rf dist; mkdir dist 15 | # Copy src to dist/src and build from dist/src into dist, so that 16 | # the source map for index.js.map will refer to ./src/index.ts etc 17 | cp -rp src README.md dist 18 | rm dist/src/_shims/*-deno.ts dist/src/_shims/auto/*-deno.ts 19 | for file in LICENSE CHANGELOG.md; do 20 | if [ -e "${file}" ]; then cp "${file}" dist; fi 21 | done 22 | if [ -e "bin/cli" ]; then 23 | mkdir dist/bin 24 | cp -p "bin/cli" dist/bin/; 25 | fi 26 | # this converts the export map paths for the dist directory 27 | # and does a few other minor things 28 | node scripts/utils/make-dist-package-json.cjs > dist/package.json 29 | 30 | # build to .js/.mjs/.d.ts files 31 | npm exec tsc-multi 32 | # copy over handwritten .js/.mjs/.d.ts files 33 | cp src/_shims/*.{d.ts,js,mjs,md} dist/_shims 34 | cp src/_shims/auto/*.{d.ts,js,mjs} dist/_shims/auto 35 | # we need to add exports = module.exports = Vlm to index.js; 36 | # No way to get that from index.ts because it would cause compile errors 37 | # when building .mjs 38 | node scripts/utils/fix-index-exports.cjs 39 | # with "moduleResolution": "nodenext", if ESM resolves to index.d.ts, 40 | # it'll have TS errors on the default import. But if it resolves to 41 | # index.d.mts the default import will work (even though both files have 42 | # the same export default statement) 43 | cp dist/index.d.ts dist/index.d.mts 44 | cp tsconfig.dist-src.json dist/src/tsconfig.json 45 | 46 | node scripts/utils/postprocess-files.cjs 47 | 48 | # make sure that nothing crashes when we require the output CJS or 49 | # import the output ESM 50 | (cd dist && node -e 'require("vlmrun")') 51 | (cd dist && node -e 'import("vlmrun")' --input-type=module) 52 | 53 | if [ -e ./scripts/build-deno ] 54 | then 55 | ./scripts/build-deno 56 | fi 57 | -------------------------------------------------------------------------------- /src/utils/file.ts: -------------------------------------------------------------------------------- 1 | import { DependencyError } from '../client/exceptions'; 2 | 3 | export const readFileFromPathAsFile = async (filePath: string): Promise => { 4 | try { 5 | if (typeof window === 'undefined') { 6 | const fs = require("fs/promises"); 7 | const path = require("path"); 8 | const mime = require("mime-types"); 9 | 10 | const fileBuffer = await fs.readFile(filePath); 11 | const fileName = path.basename(filePath); 12 | const mimeType = mime.lookup(fileName) || "application/octet-stream"; 13 | 14 | return new File([fileBuffer], fileName, { type: mimeType }); 15 | } else { 16 | throw new DependencyError("File reading is not supported in the browser", "browser_limitation", "Use server-side file operations instead"); 17 | } 18 | } catch (error: any) { 19 | throw new DependencyError(`Error reading file at ${filePath}: ${error.message}`, "file_operation_error", "Check file permissions and path"); 20 | } 21 | }; 22 | 23 | export const createArchive = async (directory: string, archiveName: string): Promise => { 24 | try { 25 | if (typeof window === 'undefined') { 26 | const fs = require('fs'); 27 | const path = require('path'); 28 | const os = require('os'); 29 | const tar = require('tar'); 30 | 31 | const tarPath = path.join(os.tmpdir(), `${archiveName}.tar.gz`); 32 | 33 | const files = fs.readdirSync(directory); 34 | 35 | await tar.create( 36 | { 37 | gzip: true, 38 | file: tarPath, 39 | cwd: directory, 40 | }, 41 | files 42 | ); 43 | 44 | return tarPath; 45 | } else { 46 | throw new DependencyError("createArchive is not supported in a browser environment.", "browser_limitation", "Use server-side environment to create archives"); 47 | } 48 | } catch (error: any) { 49 | throw new DependencyError(`Error creating archive for ${directory}: ${error.message}`, "file_operation_error", "Check directory permissions and path"); 50 | } 51 | }; 52 | -------------------------------------------------------------------------------- /src/utils/webhook.ts: -------------------------------------------------------------------------------- 1 | import crypto from "crypto"; 2 | 3 | /** 4 | * Verify webhook HMAC signature 5 | * 6 | * This function verifies that a webhook request came from VLM Run by validating 7 | * the HMAC signature in the X-VLMRun-Signature header. The signature is computed 8 | * using SHA256 HMAC with your webhook secret. 9 | * 10 | * @param rawBody - Raw request body as Buffer or string 11 | * @param signatureHeader - X-VLMRun-Signature header value (format: "sha256=") 12 | * @param secret - Your webhook secret from VLM Run dashboard 13 | * @returns True if the signature is valid, false otherwise 14 | * 15 | * @example 16 | * ```typescript 17 | * import express from 'express'; 18 | * import { verifyWebhook } from 'vlmrun'; 19 | * 20 | * app.post('/webhook', 21 | * express.raw({ type: 'application/json' }), 22 | * (req, res) => { 23 | * const rawBody = req.body; 24 | * const signature = req.headers['x-vlmrun-signature']; 25 | * const secret = process.env.WEBHOOK_SECRET; 26 | * 27 | * if (!verifyWebhook(rawBody, signature, secret)) { 28 | * return res.status(401).json({ error: 'Invalid signature' }); 29 | * } 30 | * 31 | * // Process webhook 32 | * const data = JSON.parse(rawBody.toString('utf8')); 33 | * res.json({ status: 'success' }); 34 | * } 35 | * ); 36 | * ``` 37 | */ 38 | export function verifyWebhook( 39 | rawBody: Buffer | string, 40 | signatureHeader: string | undefined, 41 | secret: string 42 | ): boolean { 43 | if (!signatureHeader || !signatureHeader.startsWith("sha256=")) { 44 | return false; 45 | } 46 | 47 | if (!secret) { 48 | return false; 49 | } 50 | 51 | const receivedSig = signatureHeader.replace("sha256=", ""); 52 | 53 | const bodyBuffer = Buffer.isBuffer(rawBody) 54 | ? rawBody 55 | : Buffer.from(rawBody, "utf8"); 56 | 57 | const expectedSig = crypto 58 | .createHmac("sha256", secret) 59 | .update(bodyBuffer) 60 | .digest("hex"); 61 | 62 | try { 63 | return crypto.timingSafeEqual( 64 | Buffer.from(receivedSig, "hex"), 65 | Buffer.from(expectedSig, "hex") 66 | ); 67 | } catch (error) { 68 | return false; 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/client/hub.ts: -------------------------------------------------------------------------------- 1 | import { Client, APIRequestor } from "./base_requestor"; 2 | import { APIError } from "./exceptions"; 3 | import { 4 | DomainInfo, 5 | HubInfoResponse, 6 | HubSchemaParams, 7 | HubSchemaResponse, 8 | } from "./types"; 9 | 10 | export class Hub { 11 | private client: Client; 12 | private requestor: APIRequestor; 13 | 14 | constructor(client: Client) { 15 | this.client = client; 16 | this.requestor = new APIRequestor(client); 17 | } 18 | 19 | /** 20 | * Get the hub info. 21 | * @returns HubInfoResponse containing hub version 22 | * @throws APIError if the request fails 23 | */ 24 | async info(): Promise { 25 | try { 26 | const [response] = await this.requestor.request( 27 | "GET", 28 | "/hub/info" 29 | ); 30 | return response; 31 | } catch (e) { 32 | throw new APIError(`Failed to check hub health: ${e}`); 33 | } 34 | } 35 | 36 | /** 37 | * Get the list of supported domains. 38 | * @returns List of domain information 39 | * @throws APIError if the request fails 40 | */ 41 | async listDomains(): Promise { 42 | try { 43 | const [response] = await this.requestor.request( 44 | "GET", 45 | "/hub/domains" 46 | ); 47 | return response; 48 | } catch (e) { 49 | throw new APIError(`Failed to list domains: ${e}`); 50 | } 51 | } 52 | 53 | /** 54 | * Get the JSON schema for a given domain. 55 | * @param params Object containing domain and optional gql_stmt 56 | * @param params.domain Domain identifier (e.g. "document.invoice") 57 | * @param params.gql_stmt Optional GraphQL statement for the domain 58 | * @returns HubSchemaResponse containing schema details 59 | * @throws APIError if the request fails or domain is not found 60 | */ 61 | async getSchema(params: HubSchemaParams): Promise { 62 | try { 63 | const [response] = await this.requestor.request( 64 | "POST", 65 | "/hub/schema", 66 | undefined, 67 | params 68 | ); 69 | return response; 70 | } catch (e) { 71 | throw new APIError( 72 | `Failed to get schema for domain ${params.domain}: ${e}` 73 | ); 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/client/feedback.ts: -------------------------------------------------------------------------------- 1 | import { Client, APIRequestor } from "./base_requestor"; 2 | import { 3 | FeedbackSubmitResponse, 4 | FeedbackSubmitRequest, 5 | FeedbackListResponse, 6 | } from "./types"; 7 | 8 | export class Feedback { 9 | private client: Client; 10 | private requestor: APIRequestor; 11 | 12 | constructor(client: Client) { 13 | this.client = client; 14 | this.requestor = new APIRequestor({ 15 | ...client, 16 | baseURL: `${client.baseURL}`, 17 | }); 18 | } 19 | 20 | async get( 21 | entityId: string, 22 | options?: { 23 | type?: "request" | "agent_execution" | "chat"; 24 | limit?: number; 25 | offset?: number; 26 | } 27 | ): Promise { 28 | const type = options?.type || "request"; 29 | const limit = options?.limit || 10; 30 | const offset = options?.offset || 0; 31 | 32 | const [response] = await this.requestor.request( 33 | "GET", 34 | `feedback/${entityId}`, 35 | { type, limit, offset } 36 | ); 37 | return response; 38 | } 39 | 40 | async submit(options: { 41 | requestId?: string; 42 | agentExecutionId?: string; 43 | chatId?: string; 44 | response?: Record | null; 45 | notes?: string | null; 46 | }): Promise { 47 | const { requestId, agentExecutionId, chatId, response, notes } = options; 48 | 49 | const idCount = [requestId, agentExecutionId, chatId].filter( 50 | (id) => id != null 51 | ).length; 52 | if (idCount !== 1) { 53 | throw new Error( 54 | "Must provide exactly one of: requestId, agentExecutionId, or chatId" 55 | ); 56 | } 57 | 58 | if (response === null && notes === null) { 59 | throw new Error( 60 | "`response` or `notes` parameter is required and cannot be null" 61 | ); 62 | } 63 | 64 | const feedbackData: FeedbackSubmitRequest = { 65 | request_id: requestId || null, 66 | agent_execution_id: agentExecutionId || null, 67 | chat_id: chatId || null, 68 | response, 69 | notes, 70 | }; 71 | 72 | const [responseData] = await this.requestor.request( 73 | "POST", 74 | `feedback/submit`, 75 | undefined, 76 | feedbackData 77 | ); 78 | return responseData; 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/client/executions.ts: -------------------------------------------------------------------------------- 1 | import { Client, APIRequestor } from "./base_requestor"; 2 | import { AgentExecutionResponse, ListParams } from "./types"; 3 | import { RequestTimeoutError } from "./exceptions"; 4 | 5 | export class Executions { 6 | private client: Client; 7 | private requestor: APIRequestor; 8 | 9 | constructor(client: Client) { 10 | this.client = client; 11 | this.requestor = new APIRequestor({ ...client, timeout: 120000 }); 12 | } 13 | 14 | /** 15 | * List all executions. 16 | * 17 | * @param params - List parameters 18 | * @returns List of execution objects 19 | */ 20 | async list(params: ListParams = {}): Promise { 21 | const [response] = await this.requestor.request( 22 | "GET", 23 | "agent/executions", 24 | { skip: params.skip, limit: params.limit } 25 | ); 26 | return response; 27 | } 28 | 29 | /** 30 | * Get execution by ID. 31 | * 32 | * @param id - ID of execution to retrieve 33 | * @returns Execution metadata 34 | */ 35 | async get(id: string): Promise { 36 | const [response] = await this.requestor.request( 37 | "GET", 38 | `agent/executions/${id}` 39 | ); 40 | 41 | if (typeof response !== 'object') { 42 | throw new TypeError("Expected object response"); 43 | } 44 | 45 | return response; 46 | } 47 | 48 | /** 49 | * Wait for execution to complete. 50 | * 51 | * @param id - ID of execution to wait for 52 | * @param timeout - Maximum number of seconds to wait (default: 300) 53 | * @param sleep - Time to wait between checks in seconds (default: 5) 54 | * @returns Completed execution 55 | * @throws RequestTimeoutError if execution does not complete within timeout 56 | */ 57 | async wait( 58 | id: string, 59 | timeout: number = 300, 60 | sleep: number = 5 61 | ): Promise { 62 | const startTime = Date.now(); 63 | const timeoutMs = timeout * 1000; 64 | 65 | while (Date.now() - startTime < timeoutMs) { 66 | const response = await this.get(id); 67 | if (response.status === "completed") { 68 | return response; 69 | } 70 | 71 | const elapsed = Date.now() - startTime; 72 | if (elapsed >= timeoutMs) { 73 | break; 74 | } 75 | 76 | await new Promise((resolve) => 77 | setTimeout(resolve, Math.min(sleep * 1000, timeoutMs - elapsed)) 78 | ); 79 | } 80 | 81 | throw new RequestTimeoutError( 82 | `Execution ${id} did not complete within ${timeout} seconds`, 83 | undefined, 84 | undefined, 85 | undefined, 86 | "execution_timeout", 87 | "Try increasing the timeout or check if the execution is taking longer than expected" 88 | ); 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import { Models } from "./client/models"; 2 | import { Files } from "./client/files"; 3 | import { Client } from "./client/base_requestor"; 4 | import { 5 | Predictions, 6 | ImagePredictions, 7 | DocumentPredictions, 8 | AudioPredictions, 9 | VideoPredictions, 10 | WebPredictions, 11 | } from "./client/predictions"; 12 | import { Feedback } from "./client/feedback"; 13 | import { Finetuning } from "./client/fine_tuning"; 14 | import { Datasets } from "./client/datasets"; 15 | import { Hub } from "./client/hub"; 16 | import { Agent } from "./client/agent"; 17 | import { Executions } from "./client/executions"; 18 | import { Domains } from "./client/domains"; 19 | 20 | export * from "./client/types"; 21 | export * from "./client/base_requestor"; 22 | export * from "./client/models"; 23 | export * from "./client/files"; 24 | export * from "./client/predictions"; 25 | export * from "./client/feedback"; 26 | export * from "./client/fine_tuning"; 27 | export * from "./client/exceptions"; 28 | export * from "./client/agent"; 29 | export * from "./client/executions"; 30 | 31 | export * from "./utils"; 32 | 33 | export interface VlmRunConfig { 34 | apiKey: string; 35 | baseURL?: string; 36 | timeout?: number; 37 | maxRetries?: number; 38 | } 39 | 40 | export class VlmRun { 41 | private client: Client; 42 | readonly models: Models; 43 | readonly files: Files; 44 | readonly predictions: Predictions; 45 | readonly image: ImagePredictions; 46 | readonly document: ReturnType; 47 | readonly audio: ReturnType; 48 | readonly video: ReturnType; 49 | readonly web: WebPredictions; 50 | readonly feedback: Feedback; 51 | readonly finetuning: Finetuning; 52 | readonly dataset: Datasets; 53 | readonly hub: Hub; 54 | readonly agent: Agent; 55 | readonly executions: Executions; 56 | readonly domains: Domains; 57 | 58 | constructor(config: VlmRunConfig) { 59 | this.client = { 60 | apiKey: config.apiKey, 61 | baseURL: config.baseURL ?? "https://api.vlm.run/v1", 62 | timeout: config.timeout, 63 | maxRetries: config.maxRetries, 64 | }; 65 | 66 | this.models = new Models(this.client); 67 | this.files = new Files({ ...this.client, timeout: 0 }); 68 | this.predictions = new Predictions(this.client); 69 | this.image = new ImagePredictions(this.client); 70 | this.document = DocumentPredictions(this.client); 71 | this.audio = AudioPredictions(this.client); 72 | this.video = VideoPredictions(this.client); 73 | this.web = new WebPredictions(this.client); 74 | this.feedback = new Feedback(this.client); 75 | this.finetuning = new Finetuning(this.client); 76 | this.dataset = new Datasets(this.client); 77 | this.hub = new Hub(this.client); 78 | this.agent = new Agent(this.client); 79 | this.executions = new Executions(this.client); 80 | this.domains = new Domains(this.client); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /examples/predictions.ts: -------------------------------------------------------------------------------- 1 | import { VlmRun } from "../src"; 2 | import { z } from "zod"; 3 | 4 | const client = new VlmRun({ 5 | apiKey: "your-api-key", // Replace with your actual API key 6 | }); 7 | 8 | async function imagePredictions() { 9 | try { 10 | console.log("Processing image with default schema..."); 11 | const imageUrl = "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/document.invoice/invoice_1.jpg"; 12 | 13 | const result1 = await client.image.generate({ 14 | images: [imageUrl], 15 | model: "vlm-1", 16 | domain: "document.invoice", 17 | }); 18 | 19 | console.log("Result:", JSON.stringify(result1.response, null, 2)); 20 | 21 | console.log("Processing image with custom JSON schema..."); 22 | const result2 = await client.image.generate({ 23 | images: [imageUrl], 24 | domain: "document.invoice", 25 | config: { 26 | jsonSchema: { 27 | type: "object", 28 | properties: { 29 | invoice_number: { type: "string" }, 30 | total_amount: { type: "number" }, 31 | }, 32 | }, 33 | }, 34 | }); 35 | 36 | console.log("Custom schema result:", JSON.stringify(result2.response, null, 2)); 37 | 38 | console.log("Processing image with Zod schema..."); 39 | const schema = z.object({ 40 | invoice_id: z.string(), 41 | total: z.number(), 42 | customer: z.string(), 43 | }); 44 | 45 | const result3 = await client.image.generate({ 46 | images: [imageUrl], 47 | domain: "document.invoice", 48 | config: { 49 | responseModel: schema, 50 | }, 51 | }); 52 | 53 | const typedResponse = result3.response as z.infer; 54 | console.log("Zod schema result:", typedResponse); 55 | 56 | console.log("Processing local image file..."); 57 | const result4 = await client.image.generate({ 58 | images: ["path/to/your/image.jpg"], // Replace with actual file path 59 | model: "vlm-1", 60 | domain: "document.invoice", 61 | }); 62 | 63 | console.log("Local file result:", JSON.stringify(result4.response, null, 2)); 64 | 65 | return result1; 66 | } catch (error) { 67 | console.error("Error with predictions:", error); 68 | throw error; 69 | } 70 | } 71 | 72 | async function documentPredictions() { 73 | try { 74 | console.log("Processing document from URL..."); 75 | const documentUrl = "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/document.invoice/google_invoice.pdf"; 76 | 77 | const result = await client.document.generate({ 78 | url: documentUrl, 79 | model: "vlm-1", 80 | domain: "document.invoice", 81 | }); 82 | 83 | console.log("Document result:", JSON.stringify(result.response, null, 2)); 84 | return result; 85 | } catch (error) { 86 | console.error("Error with document predictions:", error); 87 | throw error; 88 | } 89 | } 90 | 91 | async function runExamples() { 92 | await imagePredictions(); 93 | await documentPredictions(); 94 | } 95 | 96 | runExamples(); 97 | -------------------------------------------------------------------------------- /tests/unit/client/vlmrun.test.ts: -------------------------------------------------------------------------------- 1 | import { VlmRun } from "../../../src"; 2 | import { 3 | DomainInfo, 4 | SchemaResponse, 5 | GenerationConfig, 6 | } from "../../../src/client/types"; 7 | 8 | describe("Domains class methods", () => { 9 | let client: VlmRun; 10 | 11 | beforeEach(() => { 12 | client = new VlmRun({ 13 | apiKey: "test-api-key", 14 | baseURL: "https://api.example.com", 15 | }); 16 | }); 17 | 18 | describe("getSchema", () => { 19 | it("should call /schema endpoint with domain and config", async () => { 20 | const mockResponse: SchemaResponse = { 21 | json_schema: { type: "object" }, 22 | schema_version: "1.0.0", 23 | schema_hash: "abc123", 24 | domain: "document.invoice", 25 | gql_stmt: "", 26 | description: "Invoice document type", 27 | }; 28 | 29 | jest 30 | .spyOn(client.domains["requestor"], "request") 31 | .mockResolvedValueOnce([mockResponse, 200, {}]); 32 | 33 | const result = await client.domains.getSchema("document.invoice"); 34 | expect(result).toEqual(mockResponse); 35 | expect(client.domains["requestor"].request).toHaveBeenCalledWith( 36 | "POST", 37 | "/schema", 38 | undefined, 39 | { domain: "document.invoice", config: expect.any(Object) } 40 | ); 41 | }); 42 | 43 | it("should call /schema endpoint with domain and custom config", async () => { 44 | const mockResponse: SchemaResponse = { 45 | json_schema: { type: "object" }, 46 | schema_version: "1.0.0", 47 | schema_hash: "abc123", 48 | domain: "document.invoice", 49 | gql_stmt: "", 50 | description: "Invoice document type", 51 | }; 52 | 53 | const customConfig = new GenerationConfig({ detail: "hi", confidence: true }); 54 | 55 | jest 56 | .spyOn(client.domains["requestor"], "request") 57 | .mockResolvedValueOnce([mockResponse, 200, {}]); 58 | 59 | const result = await client.domains.getSchema("document.invoice", customConfig); 60 | expect(result).toEqual(mockResponse); 61 | expect(client.domains["requestor"].request).toHaveBeenCalledWith( 62 | "POST", 63 | "/schema", 64 | undefined, 65 | { 66 | domain: "document.invoice", 67 | config: { 68 | detail: "hi", 69 | json_schema: null, 70 | confidence: true, 71 | grounding: false, 72 | gql_stmt: null, 73 | } 74 | } 75 | ); 76 | }); 77 | }); 78 | 79 | 80 | describe("list", () => { 81 | it("should call /domains endpoint", async () => { 82 | const mockResponse: DomainInfo[] = [ 83 | { 84 | domain: "document.invoice", 85 | name: "Invoice", 86 | description: "Invoice document type", 87 | }, 88 | ]; 89 | 90 | jest 91 | .spyOn(client.domains["requestor"], "request") 92 | .mockResolvedValueOnce([mockResponse, 200, {}]); 93 | 94 | const result = await client.domains.list(); 95 | expect(result).toEqual(mockResponse); 96 | expect(client.domains["requestor"].request).toHaveBeenCalledWith( 97 | "GET", 98 | "/domains" 99 | ); 100 | }); 101 | }); 102 | }); 103 | -------------------------------------------------------------------------------- /CLAUDE.md: -------------------------------------------------------------------------------- 1 | # CLAUDE.md 2 | 3 | This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. 4 | 5 | ## Development Commands 6 | 7 | ### Build 8 | - `npm run build` or `./scripts/build` - Full production build with TypeScript compilation and package preparation 9 | - `npm run clean` - Remove dist directory 10 | 11 | ### Testing 12 | - `npm test` or `./scripts/test` - Run unit tests with automatic mock server setup 13 | - `npm run test:watch` - Run tests in watch mode 14 | - `npm run test:coverage` - Run tests with coverage reporting 15 | - `npm run test:integration` - Run integration tests only 16 | - `npm run test:integration:all` - Run all integration tests including those requiring real API keys 17 | 18 | ### Linting & Type Checking 19 | - `./scripts/lint` - Run ESLint and TypeScript type checking 20 | - No separate lint script in package.json, use the shell script 21 | 22 | ## Project Architecture 23 | 24 | ### Core Structure 25 | This is the official TypeScript SDK for the VLM Run API platform. The main entry point is `src/index.ts` which exports the `VlmRun` class and all client modules. 26 | 27 | ### Client Architecture 28 | The SDK follows a modular client pattern where each API resource has its own client class: 29 | 30 | - `VlmRun` class (src/index.ts) - Main SDK entry point that initializes all sub-clients 31 | - `Client` interface (src/client/base_requestor.ts) - Base HTTP client configuration 32 | - Resource clients in `src/client/`: 33 | - `Models` - Model listing and info 34 | - `Files` - File upload/management 35 | - `Predictions` - Core prediction functionality 36 | - `ImagePredictions`, `DocumentPredictions`, `AudioPredictions`, `VideoPredictions`, `WebPredictions` - Specialized prediction clients 37 | - `Feedback` - Submit prediction feedback 38 | - `Finetuning` - Model fine-tuning 39 | - `Datasets` - Dataset management 40 | - `Hub` - Schema and domain information 41 | - `Agent` - Agent execution 42 | - `Domains` - Domain listing 43 | 44 | ### Key Features 45 | - **Zod Integration**: Native support for Zod schemas via `responseModel` parameter 46 | - **File Handling**: Automatic file upload for local paths, supports URLs and file IDs 47 | - **Type Safety**: Full TypeScript support with comprehensive type definitions in `src/client/types.ts` 48 | - **Retry Logic**: Built-in retry mechanism via axios-retry 49 | - **Multiple Input Types**: Support for images, documents, audio, video, and web content 50 | 51 | ### Configuration 52 | - Main config interface: `VlmRunConfig` with apiKey, baseURL, timeout, maxRetries 53 | - Generation config: `GenerationConfig` class for prediction parameters (detail level, JSON schema, confidence, grounding) 54 | - Request metadata: `RequestMetadata` class for environment, session tracking, training permissions 55 | 56 | ### Testing Strategy 57 | - Unit tests in `tests/unit/` mirror the `src/` structure 58 | - Integration tests in `tests/integration/` test real API interactions 59 | - Mock server setup using Prism for unit tests 60 | - Test assets in `tests/integration/assets/` 61 | 62 | ## Important Notes 63 | 64 | - The SDK uses CommonJS module format (`"type": "commonjs"`) 65 | - Build process creates both CJS and ESM outputs in `dist/` 66 | - File uploads have special timeout handling (timeout: 0 for `Files` client) 67 | - Integration tests require `TEST_API_BASE_URL` environment variable or running mock server 68 | - Package uses Yarn as package manager (`packageManager: "yarn@1.22.22"`) -------------------------------------------------------------------------------- /.github/workflows/release-on-merge.yml: -------------------------------------------------------------------------------- 1 | name: Release on Merge 2 | 3 | on: 4 | pull_request: 5 | types: 6 | - closed 7 | 8 | jobs: 9 | release: 10 | if: github.event.pull_request.merged == true 11 | runs-on: ubuntu-latest 12 | 13 | permissions: 14 | contents: write 15 | 16 | steps: 17 | - name: Checkout code 18 | uses: actions/checkout@v4 19 | with: 20 | fetch-depth: 0 # Ensures full history for comparison 21 | 22 | - name: Setup Node.js 23 | uses: actions/setup-node@v4 24 | with: 25 | node-version: "20" 26 | 27 | - name: Install dependencies 28 | run: npm ci 29 | 30 | - name: Check for version change 31 | id: check_version 32 | run: | 33 | PREV_VERSION=$(git show HEAD^:package.json | jq -r .version) 34 | NEW_VERSION=$(jq -r .version package.json) 35 | echo "Previous version: $PREV_VERSION" 36 | echo "New version: $NEW_VERSION" 37 | if [ "$PREV_VERSION" != "$NEW_VERSION" ]; then 38 | echo "Version has changed to $NEW_VERSION" 39 | echo "VERSION_CHANGED=true" >> $GITHUB_ENV 40 | echo "RELEASE_TAG=v$NEW_VERSION" >> $GITHUB_ENV 41 | else 42 | echo "No version change detected." 43 | echo "VERSION_CHANGED=false" >> $GITHUB_ENV 44 | fi 45 | 46 | - name: Create GitHub Release 47 | if: env.VERSION_CHANGED == 'true' 48 | uses: softprops/action-gh-release@v1 49 | with: 50 | tag_name: ${{ env.RELEASE_TAG }} 51 | name: Release ${{ env.RELEASE_TAG }} 52 | draft: false 53 | prerelease: false 54 | generate_release_notes: true 55 | env: 56 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 57 | 58 | - name: Publish to NPM 59 | if: env.VERSION_CHANGED == 'true' 60 | run: | 61 | bash ./bin/publish-npm 62 | env: 63 | NPM_TOKEN: ${{ secrets.VLM_NPM_TOKEN || secrets.NPM_TOKEN }} 64 | 65 | - name: Notify Slack on Success 66 | if: success() && env.VERSION_CHANGED == 'true' 67 | uses: slackapi/slack-github-action@v2.0.0 68 | with: 69 | webhook: ${{ secrets.SLACK_WEBHOOK_URL }} 70 | webhook-type: incoming-webhook 71 | payload: | 72 | { 73 | "blocks": [ 74 | { 75 | "type": "section", 76 | "text": { 77 | "type": "mrkdwn", 78 | "text": ":white_check_mark: *vlmrun-node-sdk :nodejs: ${{ env.RELEASE_TAG }} Published to NPM*\n\n*Version:* `${{ env.RELEASE_TAG }}`\n*Commit:* <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>" 79 | } 80 | } 81 | ] 82 | } 83 | 84 | - name: Notify Slack on Failure 85 | if: failure() && env.VERSION_CHANGED == 'true' 86 | uses: slackapi/slack-github-action@v2.0.0 87 | with: 88 | webhook: ${{ secrets.SLACK_WEBHOOK_URL }} 89 | webhook-type: incoming-webhook 90 | payload: | 91 | { 92 | "blocks": [ 93 | { 94 | "type": "section", 95 | "text": { 96 | "type": "mrkdwn", 97 | "text": ":x: *vlmrun-node-sdk :nodejs: Publish Failed*\n\n*Tag:* `${{ env.RELEASE_TAG }}`\n*Workflow:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>" 98 | } 99 | } 100 | ] 101 | } 102 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 0.4.1 (2025-01-16) 4 | 5 | Full Changelog: [v0.4.0...v0.4.1](https://github.com/vlm-run/vlmrun-node-sdk/compare/v0.4.0...v0.4.1) 6 | 7 | ### Features 8 | 9 | * bump patch version to 0.4.1 10 | 11 | ## 0.1.3 (2025-01-10) 12 | 13 | Full Changelog: [v0.1.1...v0.1.3](https://github.com/vlm-run/vlmrun-node-sdk/compare/v0.1.1...v0.1.3) 14 | 15 | ### Features 16 | 17 | * update package version to 0.1.3 ([#20](https://github.com/vlm-run/vlmrun-node-sdk/issues/20)) 18 | 19 | ## 0.1.1 (2025-01-08) 20 | 21 | Full Changelog: [v0.1.0...v0.1.1](https://github.com/vlm-run/vlmrun-node-sdk/compare/v0.1.0...v0.1.1) 22 | 23 | ### Features 24 | 25 | * update package version to 0.1.1 ([#18](https://github.com/vlm-run/vlmrun-node-sdk/issues/18)) 26 | 27 | ## 0.1.0-alpha.2 (2025-01-07) 28 | 29 | Full Changelog: [v0.1.0-alpha.1...v0.1.0-alpha.2](https://github.com/vlm-run/vlmrun-node-sdk/compare/v0.1.0-alpha.1...v0.1.0-alpha.2) 30 | 31 | ### Chores 32 | 33 | * **client:** simplify `unknown | null` to just `unknown` ([#16](https://github.com/vlm-run/vlmrun-node-sdk/issues/16)) ([52aa167](https://github.com/vlm-run/vlmrun-node-sdk/commit/52aa16767d5e3343cf3e34994dc95fc2fe580c76)) 34 | 35 | ## 0.1.0-alpha.1 (2025-01-05) 36 | 37 | Full Changelog: [v0.0.1-alpha.2...v0.1.0-alpha.1](https://github.com/vlm-run/vlmrun-node-sdk/compare/v0.0.1-alpha.2...v0.1.0-alpha.1) 38 | 39 | ### Features 40 | 41 | * fix: updated class name ([d4f46aa](https://github.com/vlm-run/vlmrun-node-sdk/commit/d4f46aab78c38e816a24db9886a4341c388d1a13)) 42 | * fix: updated version ([a0af40a](https://github.com/vlm-run/vlmrun-node-sdk/commit/a0af40a705b34f19527e1400ddd7b75d002fe071)) 43 | 44 | ## 0.0.1-alpha.2 (2025-01-04) 45 | 46 | Full Changelog: [v0.0.1-alpha.1...v0.0.1-alpha.2](https://github.com/vlm-run/vlmrun-node-sdk/compare/v0.0.1-alpha.1...v0.0.1-alpha.2) 47 | 48 | ### Chores 49 | 50 | * update SDK settings ([#7](https://github.com/vlm-run/vlmrun-node-sdk/issues/7)) ([b1e100a](https://github.com/vlm-run/vlmrun-node-sdk/commit/b1e100a83ed4a281b9c3dcd30d0786eea10f88c4)) 51 | * update SDK settings ([#9](https://github.com/vlm-run/vlmrun-node-sdk/issues/9)) ([959d196](https://github.com/vlm-run/vlmrun-node-sdk/commit/959d196b34d968d96ead6d6ab6598818ad47b309)) 52 | 53 | ## 0.0.1-alpha.1 (2025-01-03) 54 | 55 | Full Changelog: [v0.0.1-alpha.0...v0.0.1-alpha.1](https://github.com/vlm-run/vlmrun-node-sdk/compare/v0.0.1-alpha.0...v0.0.1-alpha.1) 56 | 57 | ### Bug Fixes 58 | 59 | * **client:** normalize method ([dc50ded](https://github.com/vlm-run/vlmrun-node-sdk/commit/dc50ded48581ee1e305910ee07b42553413d2496)) 60 | 61 | 62 | ### Chores 63 | 64 | * go live ([#1](https://github.com/vlm-run/vlmrun-node-sdk/issues/1)) ([d5f0059](https://github.com/vlm-run/vlmrun-node-sdk/commit/d5f00599abd7f8d9b24ddfa051d41444c1c14b2d)) 65 | * **internal:** codegen related update ([8b9ee68](https://github.com/vlm-run/vlmrun-node-sdk/commit/8b9ee68f662919cfc9e91e74218772867216e5dd)) 66 | * **internal:** codegen related update ([bdb3282](https://github.com/vlm-run/vlmrun-node-sdk/commit/bdb32827625b6e05961021d2df148454b5111613)) 67 | * **internal:** codegen related update ([d703fcb](https://github.com/vlm-run/vlmrun-node-sdk/commit/d703fcbb0d5e4f46c74cbe9b06acce90e8f29b83)) 68 | * **internal:** codegen related update ([bad9391](https://github.com/vlm-run/vlmrun-node-sdk/commit/bad93911df586d77a07c39975ea4cc8c234e5888)) 69 | * **internal:** fix some typos ([c85fff3](https://github.com/vlm-run/vlmrun-node-sdk/commit/c85fff3f218e51eeaf3b9b619806a11372f1495b)) 70 | * **internal:** fix some typos ([0645bb3](https://github.com/vlm-run/vlmrun-node-sdk/commit/0645bb3aa580503a158af8efd99f8113ab15b242)) 71 | * update SDK settings ([#3](https://github.com/vlm-run/vlmrun-node-sdk/issues/3)) ([c372d79](https://github.com/vlm-run/vlmrun-node-sdk/commit/c372d79e19803ef3f414aaf384251e23fda841f9)) 72 | 73 | 74 | ### Documentation 75 | 76 | * minor formatting changes ([2bb0eb9](https://github.com/vlm-run/vlmrun-node-sdk/commit/2bb0eb940e4a87be9284d086445470c9497ad254)) 77 | -------------------------------------------------------------------------------- /src/client/datasets.ts: -------------------------------------------------------------------------------- 1 | import { Client, APIRequestor } from "./base_requestor"; 2 | import { DatasetResponse, DatasetCreateParams, DatasetListParams } from "./types"; 3 | import { createArchive } from "../utils"; 4 | import { Files } from "../index"; 5 | import { DependencyError, InputError, ServerError } from "./exceptions"; 6 | 7 | export class Datasets { 8 | private requestor: APIRequestor; 9 | private files: Files; 10 | 11 | constructor(client: Client) { 12 | this.requestor = new APIRequestor({ 13 | ...client, 14 | baseURL: `${client.baseURL}/datasets`, 15 | }); 16 | 17 | this.files = new Files(client); 18 | } 19 | 20 | /** 21 | * Create a dataset from a directory of files. 22 | * 23 | * @param params Dataset creation parameters. 24 | * @returns The dataset creation response. 25 | */ 26 | async create(params: DatasetCreateParams): Promise { 27 | const validTypes = ["images", "videos", "documents"]; 28 | 29 | if (typeof window !== "undefined") { 30 | throw new DependencyError("createArchive is not supported in a browser environment.", "browser_limitation", "Use server-side environment to create datasets"); 31 | } 32 | 33 | if (!validTypes.includes(params.datasetType)) { 34 | throw new InputError("dataset_type must be one of: images, videos, documents", "invalid_parameter", "Provide a valid dataset type: images, videos, or documents"); 35 | } 36 | 37 | // Create tar.gz archive of the dataset directory. 38 | const tarPath = await createArchive(params.datasetDirectory, params.datasetName); 39 | const fs = require('fs'); 40 | const tarSizeMB = (fs.statSync(tarPath).size / 1024 / 1024).toFixed(2); 41 | console.debug(`Created tar.gz file [path=${tarPath}, size=${tarSizeMB} MB]`); 42 | 43 | // Upload the tar.gz file using the client's file upload method. 44 | const fileResponse = await this.files.upload({ 45 | filePath: tarPath, 46 | purpose: "datasets", 47 | }); 48 | const fileSizeMB = (fileResponse.bytes / 1024 / 1024).toFixed(2); 49 | console.debug( 50 | `Uploaded tar.gz file [path=${tarPath}, file_id=${fileResponse.id}, size=${fileSizeMB} MB]` 51 | ); 52 | 53 | // Create the dataset by sending a POST request. 54 | const [response] = await this.requestor.request( 55 | "POST", 56 | "create", 57 | undefined, // No query parameters 58 | { 59 | file_id: fileResponse.id, 60 | domain: params.domain, 61 | dataset_name: params.datasetName, 62 | dataset_type: params.datasetType, 63 | wandb_base_url: params.wandbBaseUrl, 64 | wandb_project_name: params.wandbProjectName, 65 | wandb_api_key: params.wandbApiKey, 66 | } 67 | ); 68 | 69 | return response; 70 | } 71 | 72 | /** 73 | * Get dataset information by its ID. 74 | * 75 | * @param datasetId The ID of the dataset to retrieve. 76 | * @returns The dataset information. 77 | */ 78 | async get(datasetId: string): Promise { 79 | const [response] = await this.requestor.request( 80 | "GET", 81 | datasetId 82 | ); 83 | return response; 84 | } 85 | 86 | /** 87 | * List all datasets with pagination support. 88 | * 89 | * @param skip Number of datasets to skip. 90 | * @param limit Maximum number of datasets to return. 91 | * @returns A list of dataset responses. 92 | */ 93 | async list(params?: DatasetListParams): Promise { 94 | const [items] = await this.requestor.request( 95 | "GET", 96 | "", 97 | { 98 | skip: params?.skip ?? 0, 99 | limit: params?.limit ?? 10, 100 | } 101 | ); 102 | if (!Array.isArray(items)) { 103 | throw new ServerError("Expected array response", undefined, undefined, undefined, "unexpected_response", "Contact support if this issue persists"); 104 | } 105 | return items; 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /docs/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to VLM Run Node.js SDK 2 | 3 | We love your input! We want to make contributing to VLM Run Node.js SDK as easy and transparent as possible, whether it's: 4 | 5 | - Reporting a bug 6 | - Discussing the current state of the code 7 | - Submitting a fix 8 | - Proposing new features 9 | - Becoming a maintainer 10 | 11 | ## We Develop with Github 12 | 13 | We use GitHub to host code, to track issues and feature requests, as well as accept pull requests. 14 | 15 | ## Development Process 16 | 17 | We use Github Flow, so all code changes happen through pull requests. Pull requests are the best way to propose changes to the codebase. We actively welcome your pull requests: 18 | 19 | 1. Fork the repo and create your branch from `main`. 20 | 2. If you've added code that should be tested, add tests. 21 | 3. If you've changed APIs, update the documentation. 22 | 4. Ensure the test suite passes. 23 | 5. Make sure your code lints. 24 | 6. Issue that pull request! 25 | 26 | ## Local Development Setup 27 | 28 | 1. Clone the repository: 29 | 30 | ```bash 31 | git clone https://github.com/vlm-run/vlmrun-node-sdk.git 32 | cd vlmrun-node-sdk 33 | ``` 34 | 35 | 2. Install dependencies: 36 | 37 | ```bash 38 | npm install 39 | ``` 40 | 41 | 3. Run tests: 42 | 43 | Add .env.test file with the following variables: 44 | 45 | ```bash 46 | TEST_API_KEY= 47 | TEST_BASE_URL=https://dev.vlm.run/v1 48 | ``` 49 | 50 | ```bash 51 | npm test 52 | npm run test:integration 53 | ``` 54 | 55 | ## Code Style 56 | 57 | - We use TypeScript for type safety 58 | - Follow the existing code style 59 | - Use meaningful variable and function names 60 | - Add comments for complex logic 61 | - Keep functions focused and modular 62 | 63 | ## Running Tests 64 | 65 | - Run all tests: `npm test` 66 | - Run specific test: `npm test -- -t "test name"` 67 | - Run integration tests (excluding audio and video): `npm run test:integration` 68 | - Run all integration tests: `npm run test:integration:all` 69 | - Run specific integration test: `npm run test:integration -- -t "test name"` 70 | - Run with coverage: `npm run test:coverage` 71 | Example 72 | 73 | ```bash 74 | # Run just the files integration tests 75 | npm run test:integration -- tests/integration/client/files.test.ts 76 | 77 | # Run 78 | npm run test:e2e -- tests/e2e/client/chat-completions.test.ts 79 | 80 | # Run specific test patterns 81 | npm run test:integration -- -t "upload methods" 82 | npm run test:integration -- -t "should upload file using direct method" 83 | ``` 84 | 85 | ## Commit Messages 86 | 87 | We follow conventional commits specification. Your commit messages should be structured as follows: 88 | 89 | ``` 90 | feat: add new feature 91 | fix: correct bug 92 | docs: update documentation 93 | test: add tests 94 | chore: update dependencies 95 | ``` 96 | 97 | ## Any contributions you make will be under the Apache-2.0 Software License 98 | 99 | In short, when you submit code changes, your submissions are understood to be under the same [Apache-2.0 License](http://choosealicense.com/licenses/apache-2.0/) that covers the project. Feel free to contact the maintainers if that's a concern. 100 | 101 | ## Report bugs using Github's [issue tracker](https://github.com/vlm-run/vlmrun-node-sdk/issues) 102 | 103 | We use GitHub issues to track public bugs. Report a bug by [opening a new issue](https://github.com/vlm-run/vlmrun-node-sdk/issues/new); it's that easy! 104 | 105 | ## Write bug reports with detail, background, and sample code 106 | 107 | **Great Bug Reports** tend to have: 108 | 109 | - A quick summary and/or background 110 | - Steps to reproduce 111 | - Be specific! 112 | - Give sample code if you can 113 | - What you expected would happen 114 | - What actually happens 115 | - Notes (possibly including why you think this might be happening, or stuff you tried that didn't work) 116 | 117 | ## License 118 | 119 | By contributing, you agree that your contributions will be licensed under its Apache-2.0 License. 120 | 121 | ## References 122 | 123 | This document was adapted from the open-source contribution guidelines for [Facebook's Draft](https://github.com/facebook/draft-js/blob/master/CONTRIBUTING.md). 124 | -------------------------------------------------------------------------------- /tests/unit/client/hub.test.ts: -------------------------------------------------------------------------------- 1 | import { VlmRun } from "../../../src"; 2 | import { 3 | HubInfoResponse, 4 | DomainInfo, 5 | HubSchemaResponse, 6 | } from "../../../src/client/types"; 7 | 8 | describe("Hub", () => { 9 | let client: VlmRun; 10 | 11 | beforeEach(() => { 12 | client = new VlmRun({ 13 | apiKey: "test-api-key", 14 | baseURL: "https://api.example.com", 15 | }); 16 | }); 17 | 18 | describe("info", () => { 19 | it("should return hub info", async () => { 20 | const mockResponse: HubInfoResponse = { 21 | version: "1.0.0", 22 | }; 23 | 24 | jest 25 | .spyOn(client.hub["requestor"], "request") 26 | .mockResolvedValueOnce([mockResponse, 200, {}]); 27 | 28 | const result = await client.hub.info(); 29 | expect(result).toEqual(mockResponse); 30 | expect(client.hub["requestor"].request).toHaveBeenCalledWith( 31 | "GET", 32 | "/hub/info" 33 | ); 34 | }); 35 | 36 | it("should handle errors", async () => { 37 | jest 38 | .spyOn(client.hub["requestor"], "request") 39 | .mockRejectedValueOnce(new Error("Network error")); 40 | 41 | await expect(client.hub.info()).rejects.toThrow( 42 | "Failed to check hub health" 43 | ); 44 | }); 45 | }); 46 | 47 | describe("listDomains", () => { 48 | it("should return list of domains", async () => { 49 | const mockResponse: DomainInfo[] = [ 50 | { 51 | domain: "document.invoice", 52 | name: "Invoice", 53 | description: "Invoice document type", 54 | }, 55 | ]; 56 | 57 | jest 58 | .spyOn(client.hub["requestor"], "request") 59 | .mockResolvedValueOnce([mockResponse, 200, {}]); 60 | 61 | const result = await client.hub.listDomains(); 62 | expect(result).toEqual(mockResponse); 63 | expect(client.hub["requestor"].request).toHaveBeenCalledWith( 64 | "GET", 65 | "/hub/domains" 66 | ); 67 | }); 68 | 69 | it("should handle errors", async () => { 70 | jest 71 | .spyOn(client.hub["requestor"], "request") 72 | .mockRejectedValueOnce(new Error("Network error")); 73 | 74 | await expect(client.hub.listDomains()).rejects.toThrow( 75 | "Failed to list domains" 76 | ); 77 | }); 78 | }); 79 | 80 | describe("getSchema", () => { 81 | it("should return schema for domain without gql_stmt", async () => { 82 | const mockResponse: HubSchemaResponse = { 83 | json_schema: { type: "object" }, 84 | schema_version: "1.0.0", 85 | schema_hash: "abc123", 86 | domain: "document.invoice", 87 | gql_stmt: "", 88 | description: "Invoice document type", 89 | }; 90 | 91 | jest 92 | .spyOn(client.hub["requestor"], "request") 93 | .mockResolvedValueOnce([mockResponse, 200, {}]); 94 | 95 | const result = await client.hub.getSchema({ domain: "document.invoice" }); 96 | expect(result).toEqual(mockResponse); 97 | expect(client.hub["requestor"].request).toHaveBeenCalledWith( 98 | "POST", 99 | "/hub/schema", 100 | undefined, 101 | { domain: "document.invoice" } 102 | ); 103 | }); 104 | 105 | it("should return schema for domain with gql_stmt", async () => { 106 | const mockResponse: HubSchemaResponse = { 107 | json_schema: { type: "object" }, 108 | schema_version: "1.0.0", 109 | schema_hash: "abc123", 110 | domain: "document.invoice", 111 | gql_stmt: "query { field }", 112 | description: "Invoice document type", 113 | }; 114 | 115 | jest 116 | .spyOn(client.hub["requestor"], "request") 117 | .mockResolvedValueOnce([mockResponse, 200, {}]); 118 | 119 | const gqlStmt = "query { field }"; 120 | const result = await client.hub.getSchema({ 121 | domain: "document.invoice", 122 | gql_stmt: gqlStmt, 123 | }); 124 | expect(result).toEqual(mockResponse); 125 | expect(client.hub["requestor"].request).toHaveBeenCalledWith( 126 | "POST", 127 | "/hub/schema", 128 | undefined, 129 | { domain: "document.invoice", gql_stmt: gqlStmt } 130 | ); 131 | }); 132 | 133 | it("should handle errors", async () => { 134 | jest 135 | .spyOn(client.hub["requestor"], "request") 136 | .mockRejectedValueOnce(new Error("Network error")); 137 | 138 | await expect( 139 | client.hub.getSchema({ domain: "invalid.domain" }) 140 | ).rejects.toThrow("Failed to get schema for domain invalid.domain"); 141 | }); 142 | }); 143 | }); 144 | -------------------------------------------------------------------------------- /tests/unit/client/executions.test.ts: -------------------------------------------------------------------------------- 1 | import { Client } from '../../../src/client/base_requestor'; 2 | import { Executions } from '../../../src/client/executions'; 3 | import { AgentExecutionResponse } from '../../../src/client/types'; 4 | 5 | jest.mock('../../../src/client/base_requestor'); 6 | 7 | describe('Executions', () => { 8 | let client: jest.Mocked; 9 | let executions: Executions; 10 | 11 | beforeEach(() => { 12 | client = { 13 | apiKey: 'test-api-key', 14 | baseURL: 'https://api.example.com', 15 | } as jest.Mocked; 16 | 17 | executions = new Executions(client); 18 | }); 19 | 20 | describe('list', () => { 21 | it('should list executions successfully', async () => { 22 | const mockResponse: AgentExecutionResponse[] = [ 23 | { 24 | id: 'exec_123', 25 | name: 'test-agent', 26 | created_at: '2023-01-01T00:00:00Z', 27 | completed_at: '2023-01-01T00:00:01Z', 28 | response: { result: 'success' }, 29 | status: 'completed', 30 | usage: { credits_used: 10 }, 31 | }, 32 | ]; 33 | 34 | jest.spyOn(executions['requestor'], 'request').mockResolvedValue([mockResponse, 200, {}]); 35 | 36 | const result = await executions.list(); 37 | 38 | expect(result).toEqual(mockResponse); 39 | expect(executions['requestor'].request).toHaveBeenCalledWith( 40 | 'GET', 41 | 'agent/executions', 42 | { skip: undefined, limit: undefined } 43 | ); 44 | }); 45 | 46 | it('should list executions with pagination', async () => { 47 | const mockResponse: AgentExecutionResponse[] = []; 48 | 49 | jest.spyOn(executions['requestor'], 'request').mockResolvedValue([mockResponse, 200, {}]); 50 | 51 | const result = await executions.list({ skip: 10, limit: 5 }); 52 | 53 | expect(result).toEqual(mockResponse); 54 | expect(executions['requestor'].request).toHaveBeenCalledWith( 55 | 'GET', 56 | 'agent/executions', 57 | { skip: 10, limit: 5 } 58 | ); 59 | }); 60 | }); 61 | 62 | describe('get', () => { 63 | it('should get execution by ID successfully', async () => { 64 | const mockResponse: AgentExecutionResponse = { 65 | id: 'exec_123', 66 | name: 'test-agent', 67 | created_at: '2023-01-01T00:00:00Z', 68 | completed_at: '2023-01-01T00:00:01Z', 69 | response: { result: 'success' }, 70 | status: 'completed', 71 | usage: { credits_used: 10 }, 72 | }; 73 | 74 | jest.spyOn(executions['requestor'], 'request').mockResolvedValue([mockResponse, 200, {}]); 75 | 76 | const result = await executions.get('exec_123'); 77 | 78 | expect(result).toEqual(mockResponse); 79 | expect(executions['requestor'].request).toHaveBeenCalledWith( 80 | 'GET', 81 | 'agent/executions/exec_123' 82 | ); 83 | }); 84 | 85 | it('should throw error for non-object response', async () => { 86 | jest.spyOn(executions['requestor'], 'request').mockResolvedValue(['not-an-object', 200, {}]); 87 | 88 | await expect(executions.get('exec_123')).rejects.toThrow('Expected object response'); 89 | }); 90 | }); 91 | 92 | describe('wait', () => { 93 | it('should wait for execution to complete', async () => { 94 | const mockResponse: AgentExecutionResponse = { 95 | id: 'exec_123', 96 | name: 'test-agent', 97 | created_at: '2023-01-01T00:00:00Z', 98 | completed_at: '2023-01-01T00:00:01Z', 99 | response: { result: 'success' }, 100 | status: 'completed', 101 | usage: { credits_used: 10 }, 102 | }; 103 | 104 | jest.spyOn(executions['requestor'], 'request').mockResolvedValue([mockResponse, 200, {}]); 105 | 106 | const result = await executions.wait('exec_123', 10, 1); 107 | 108 | expect(result).toEqual(mockResponse); 109 | expect(executions['requestor'].request).toHaveBeenCalledWith( 110 | 'GET', 111 | 'agent/executions/exec_123' 112 | ); 113 | }); 114 | 115 | it('should throw timeout error when execution does not complete', async () => { 116 | const mockResponse: AgentExecutionResponse = { 117 | id: 'exec_123', 118 | name: 'test-agent', 119 | created_at: '2023-01-01T00:00:00Z', 120 | status: 'running', 121 | usage: { credits_used: 0 }, 122 | }; 123 | 124 | jest.spyOn(executions['requestor'], 'request').mockResolvedValue([mockResponse, 200, {}]); 125 | 126 | await expect(executions.wait('exec_123', 1, 0.1)).rejects.toThrow( 127 | 'Execution exec_123 did not complete within 1 seconds' 128 | ); 129 | }); 130 | }); 131 | }); 132 | -------------------------------------------------------------------------------- /tests/e2e/client/chat-completions.test.ts: -------------------------------------------------------------------------------- 1 | import { config } from "dotenv"; 2 | config({ path: ".env.test" }); 3 | 4 | // Import from the built dist folder 5 | import { VlmRun } from "../../../dist"; 6 | 7 | jest.setTimeout(120000); 8 | 9 | describe("E2E: Chat Completions", () => { 10 | let client: VlmRun; 11 | 12 | beforeAll(() => { 13 | const apiKey = process.env.TEST_API_KEY; 14 | const baseURL = process.env.AGENT_BASE_URL; 15 | 16 | if (!apiKey) { 17 | throw new Error( 18 | "TEST_API_KEY environment variable is required. Set it in .env.test" 19 | ); 20 | } 21 | 22 | if (!baseURL) { 23 | throw new Error( 24 | "AGENT_BASE_URL environment variable is required. Set it in .env.test" 25 | ); 26 | } 27 | 28 | client = new VlmRun({ 29 | apiKey, 30 | baseURL, 31 | timeout: 60000, 32 | maxRetries: 2, 33 | }); 34 | }); 35 | 36 | describe("agent.completions", () => { 37 | describe("create()", () => { 38 | it("should create a basic chat completion", async () => { 39 | const response = await client.agent.completions.create({ 40 | model: "vlmrun-orion-1", 41 | messages: [ 42 | { 43 | role: "user", 44 | content: "Say 'Hello, World!' and nothing else.", 45 | }, 46 | ], 47 | }); 48 | 49 | expect(response).toBeTruthy(); 50 | expect(response).toHaveProperty("id"); 51 | expect(response).toHaveProperty("choices"); 52 | expect(response).toHaveProperty("model"); 53 | expect(response).toHaveProperty("usage"); 54 | 55 | expect(Array.isArray(response.choices)).toBe(true); 56 | expect(response.choices.length).toBeGreaterThan(0); 57 | 58 | const choice = response.choices[0]; 59 | expect(choice).toHaveProperty("message"); 60 | expect(choice.message).toHaveProperty("role", "assistant"); 61 | expect(choice.message).toHaveProperty("content"); 62 | expect(typeof choice.message.content).toBe("string"); 63 | }); 64 | 65 | it("should handle multi-turn conversation", async () => { 66 | const response = await client.agent.completions.create({ 67 | model: "vlmrun-orion-1", 68 | messages: [ 69 | { 70 | role: "system", 71 | content: "You are a helpful assistant that responds concisely.", 72 | }, 73 | { 74 | role: "user", 75 | content: "What is 2 + 2?", 76 | }, 77 | { 78 | role: "assistant", 79 | content: "4", 80 | }, 81 | { 82 | role: "user", 83 | content: "What is 3 + 3?", 84 | }, 85 | ], 86 | }); 87 | console.log(response); 88 | 89 | expect(response).toBeTruthy(); 90 | expect(response.choices).toBeTruthy(); 91 | expect(response.choices.length).toBeGreaterThan(0); 92 | 93 | const choice = response.choices[0]; 94 | expect(choice.message.role).toBe("assistant"); 95 | expect(choice.message.content).toBeTruthy(); 96 | }); 97 | 98 | it("should include usage statistics in response when available", async () => { 99 | const response = await client.agent.completions.create({ 100 | model: "vlmrun-orion-1", 101 | messages: [ 102 | { 103 | role: "user", 104 | content: "Hi", 105 | }, 106 | ], 107 | }); 108 | 109 | expect(response).toBeTruthy(); 110 | 111 | // Usage statistics may or may not be included depending on the model/endpoint 112 | if (response.usage) { 113 | const { usage } = response; 114 | expect(typeof usage.prompt_tokens).toBe("number"); 115 | expect(typeof usage.completion_tokens).toBe("number"); 116 | expect(typeof usage.total_tokens).toBe("number"); 117 | expect(usage.total_tokens).toBe( 118 | usage.prompt_tokens + usage.completion_tokens 119 | ); 120 | } else { 121 | // If usage is not provided, that's acceptable 122 | expect(response.usage).toBeNull(); 123 | } 124 | }); 125 | }); 126 | 127 | describe("streaming", () => { 128 | it("should handle streaming responses", async () => { 129 | const stream = await client.agent.completions.create({ 130 | model: "vlmrun-orion-1", 131 | messages: [ 132 | { 133 | role: "user", 134 | content: "Count from 1 to 5.", 135 | }, 136 | ], 137 | stream: true, 138 | }); 139 | 140 | expect(stream).toBeTruthy(); 141 | 142 | const chunks: string[] = []; 143 | for await (const chunk of stream) { 144 | expect(chunk).toHaveProperty("id"); 145 | expect(chunk).toHaveProperty("choices"); 146 | 147 | if (chunk.choices[0]?.delta?.content) { 148 | chunks.push(chunk.choices[0].delta.content); 149 | } 150 | } 151 | 152 | expect(chunks.length).toBeGreaterThan(0); 153 | const fullContent = chunks.join(""); 154 | expect(fullContent.length).toBeGreaterThan(0); 155 | }); 156 | }); 157 | }); 158 | }); 159 | -------------------------------------------------------------------------------- /tests/integration/client/audio-predictions.test.ts: -------------------------------------------------------------------------------- 1 | import { config } from "dotenv"; 2 | 3 | import { VlmRun } from "../../../src/index"; 4 | import { z } from "zod"; 5 | 6 | jest.setTimeout(60000); 7 | 8 | describe("Integration: Audio Predictions", () => { 9 | let client: VlmRun; 10 | 11 | beforeAll(() => { 12 | config({ path: ".env.test" }); 13 | 14 | client = new VlmRun({ 15 | apiKey: process.env.TEST_API_KEY as string, 16 | baseURL: process.env.TEST_BASE_URL as string, 17 | }); 18 | }); 19 | 20 | describe("AudioPredictions", () => { 21 | const testFilePath = "tests/integration/assets/two_minute_rules.mp3"; 22 | 23 | it("should generate audio predictions using file id with batch processing", async () => { 24 | const uploadedAudio = await client.files.upload({ 25 | filePath: testFilePath, 26 | checkDuplicate: false, 27 | }); 28 | 29 | const result = await client.audio.generate({ 30 | fileId: uploadedAudio.id, 31 | batch: true, 32 | domain: "audio.transcription", 33 | }); 34 | 35 | // Initial response should be pending for batch processing 36 | expect(result).toHaveProperty("id"); 37 | expect(result).toHaveProperty("created_at"); 38 | expect(result).toHaveProperty("completed_at"); 39 | expect(result).toHaveProperty("usage"); 40 | expect(result.status).toBe("pending"); 41 | expect(result.response).toBeNull(); 42 | expect(result.completed_at).toBeNull(); 43 | 44 | // Wait for completion 45 | const waitResponse = await client.predictions.wait(result.id); 46 | 47 | expect(waitResponse.status).toBe("completed"); 48 | expect(waitResponse.response).not.toBeNull(); 49 | expect(waitResponse.completed_at).not.toBeNull(); 50 | 51 | // Check usage information 52 | expect(waitResponse.usage).toBeDefined(); 53 | expect(waitResponse.usage!).toHaveProperty("elements_processed"); 54 | expect(waitResponse.usage!).toHaveProperty("element_type"); 55 | expect(waitResponse.usage!).toHaveProperty("credits_used"); 56 | expect(waitResponse.usage!.element_type).toBe("audio"); 57 | 58 | // Check response structure 59 | expect(waitResponse.response).toHaveProperty("metadata"); 60 | expect(waitResponse.response).toHaveProperty("segments"); 61 | 62 | // Check metadata 63 | expect(waitResponse.response.metadata).toHaveProperty("duration"); 64 | expect(typeof waitResponse.response.metadata.duration).toBe("number"); 65 | expect(waitResponse.response.metadata.duration).toBeGreaterThan(0); 66 | 67 | // Check segments 68 | expect(Array.isArray(waitResponse.response.segments)).toBe(true); 69 | expect(waitResponse.response.segments.length).toBeGreaterThan(0); 70 | 71 | // Check first segment structure 72 | const firstSegment = waitResponse.response.segments[0]; 73 | expect(firstSegment).toHaveProperty("start_time"); 74 | expect(firstSegment).toHaveProperty("end_time"); 75 | expect(firstSegment).toHaveProperty("content"); 76 | expect(typeof firstSegment.start_time).toBe("number"); 77 | expect(typeof firstSegment.end_time).toBe("number"); 78 | expect(typeof firstSegment.content).toBe("string"); 79 | expect(firstSegment.content.length).toBeGreaterThan(0); 80 | }); 81 | 82 | it("should generate audio predictions using url", async () => { 83 | const audioUrl = 84 | "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/audio.transcription-summary/two_minute_rules.mp3"; 85 | 86 | const result = await client.audio.generate({ 87 | url: audioUrl, 88 | domain: "audio.transcription", 89 | batch: true, 90 | }); 91 | 92 | // Initial response should be pending for batch processing 93 | expect(result).toHaveProperty("id"); 94 | expect(result).toHaveProperty("created_at"); 95 | expect(result).toHaveProperty("completed_at"); 96 | expect(result).toHaveProperty("usage"); 97 | expect(result.status).toBe("pending"); 98 | expect(result.response).toBeNull(); 99 | expect(result.completed_at).toBeNull(); 100 | 101 | // Wait for completion 102 | const waitResponse = await client.predictions.wait(result.id); 103 | 104 | expect(waitResponse.status).toBe("completed"); 105 | expect(waitResponse.response).not.toBeNull(); 106 | expect(waitResponse.completed_at).not.toBeNull(); 107 | 108 | // Check usage information 109 | expect(waitResponse.usage).toBeDefined(); 110 | expect(waitResponse.usage!).toHaveProperty("elements_processed"); 111 | expect(waitResponse.usage!).toHaveProperty("element_type"); 112 | expect(waitResponse.usage!).toHaveProperty("credits_used"); 113 | expect(waitResponse.usage!.element_type).toBe("audio"); 114 | 115 | // Check response structure 116 | expect(waitResponse.response).toHaveProperty("metadata"); 117 | expect(waitResponse.response).toHaveProperty("segments"); 118 | 119 | // Check metadata 120 | expect(waitResponse.response.metadata).toHaveProperty("duration"); 121 | expect(typeof waitResponse.response.metadata.duration).toBe("number"); 122 | expect(waitResponse.response.metadata.duration).toBeGreaterThan(0); 123 | 124 | // Check segments 125 | expect(Array.isArray(waitResponse.response.segments)).toBe(true); 126 | expect(waitResponse.response.segments.length).toBeGreaterThan(0); 127 | 128 | // Test get endpoint 129 | const getResponse = await client.predictions.get(result.id); 130 | expect(getResponse.status).toBe("completed"); 131 | expect(getResponse.response).not.toBeNull(); 132 | expect(getResponse.response).toHaveProperty("segments"); 133 | }); 134 | }); 135 | }); 136 | -------------------------------------------------------------------------------- /src/client/base_requestor.ts: -------------------------------------------------------------------------------- 1 | import axios, { AxiosHeaders, AxiosInstance, AxiosError } from "axios"; 2 | import axiosRetry from "axios-retry"; 3 | import { 4 | APIError, 5 | AuthenticationError, 6 | ValidationError, 7 | RateLimitError, 8 | ServerError, 9 | ResourceNotFoundError, 10 | RequestTimeoutError, 11 | NetworkError, 12 | } from "./exceptions"; 13 | 14 | const DEFAULT_TIMEOUT = 120000; // 120 seconds in ms 15 | const DEFAULT_MAX_RETRIES = 5; 16 | const INITIAL_RETRY_DELAY = 1000; // 1 second in ms 17 | const MAX_RETRY_DELAY = 10000; // 10 seconds in ms 18 | 19 | export interface Client { 20 | apiKey: string; 21 | baseURL: string; 22 | timeout?: number; 23 | maxRetries?: number; 24 | } 25 | 26 | export class APIRequestor { 27 | private client: Client; 28 | private axios: AxiosInstance; 29 | private timeout: number; 30 | private maxRetries: number; 31 | 32 | constructor(client: Client) { 33 | this.client = client; 34 | this.timeout = client.timeout ?? DEFAULT_TIMEOUT; 35 | this.maxRetries = client.maxRetries ?? DEFAULT_MAX_RETRIES; 36 | 37 | this.axios = axios.create({ 38 | baseURL: client.baseURL, 39 | headers: { 40 | Authorization: `Bearer ${client.apiKey}`, 41 | "Content-Type": "application/json", 42 | }, 43 | timeout: this.timeout, 44 | }); 45 | 46 | axiosRetry(this.axios, { 47 | retries: this.maxRetries, 48 | retryDelay: (retryCount, error) => { 49 | const delay = Math.min( 50 | INITIAL_RETRY_DELAY * 51 | Math.pow(2, retryCount - 1) * 52 | (0.5 + Math.random() * 0.5), 53 | MAX_RETRY_DELAY 54 | ); 55 | return delay; 56 | }, 57 | retryCondition: (error) => { 58 | return ( 59 | axiosRetry.isNetworkError(error) || 60 | error.response?.status === 429 || 61 | (error.response?.status && 62 | error.response?.status >= 500 && 63 | error.response?.status < 600) || 64 | error.code === "ECONNABORTED" 65 | ); 66 | }, 67 | }); 68 | } 69 | 70 | async request( 71 | method: string, 72 | url: string, 73 | params?: Record, 74 | data?: any, 75 | files?: { [key: string]: any } 76 | ): Promise<[T, number, Record]> { 77 | try { 78 | let headers = new AxiosHeaders(this.axios.defaults.headers); 79 | 80 | if (files) { 81 | const formData = new FormData(); 82 | Object.entries(files).forEach(([key, value]) => { 83 | formData.append(key, value); 84 | }); 85 | data = formData; 86 | headers.set("Content-Type", "multipart/form-data"); 87 | } 88 | 89 | const response = await this.axios.request({ 90 | method, 91 | url, 92 | params, 93 | data, 94 | headers, 95 | }); 96 | 97 | return [ 98 | response.data as T, 99 | response.status, 100 | response.headers as Record, 101 | ]; 102 | } catch (error) { 103 | if (axios.isAxiosError(error) && error.response) { 104 | let errorMessage = "API request failed"; 105 | let errorType: string | undefined; 106 | let requestId: string | undefined; 107 | 108 | try { 109 | const errorData = error.response?.data; 110 | 111 | if (Array.isArray(errorData.detail)) { 112 | errorMessage = 113 | errorData.detail[0].msg || errorData.detail[0] || errorMessage; 114 | } else { 115 | errorMessage = errorData.detail || error.message || errorMessage; 116 | } 117 | 118 | errorType = (error.cause as any)?.name; 119 | requestId = error.response?.request?.id; 120 | } catch (e) { 121 | errorMessage = error.message || errorMessage; 122 | } 123 | 124 | const status = error.response.status; 125 | const headers = error.response.headers; 126 | 127 | if (status === 401) { 128 | throw new AuthenticationError( 129 | errorMessage, 130 | status, 131 | headers, 132 | requestId, 133 | errorType 134 | ); 135 | } else if (status === 400) { 136 | throw new ValidationError( 137 | errorMessage, 138 | status, 139 | headers, 140 | requestId, 141 | errorType 142 | ); 143 | } else if (status === 404) { 144 | throw new ResourceNotFoundError( 145 | errorMessage, 146 | status, 147 | headers, 148 | requestId, 149 | errorType 150 | ); 151 | } else if (status === 429) { 152 | throw new RateLimitError( 153 | errorMessage, 154 | status, 155 | headers, 156 | requestId, 157 | errorType 158 | ); 159 | } else if (status >= 500 && status < 600) { 160 | throw new ServerError( 161 | errorMessage, 162 | status, 163 | headers, 164 | requestId, 165 | errorType 166 | ); 167 | } else { 168 | throw new APIError( 169 | errorMessage, 170 | status, 171 | headers, 172 | requestId, 173 | errorType 174 | ); 175 | } 176 | } else if (axios.isAxiosError(error)) { 177 | if (error.code === "ECONNABORTED") { 178 | throw new RequestTimeoutError(`Request timed out: ${error.message}`); 179 | } else { 180 | throw new NetworkError(`Network error: ${error.message}`); 181 | } 182 | } 183 | throw new APIError( 184 | error instanceof Error ? error.message : "Unknown error occurred" 185 | ); 186 | } 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /scripts/utils/postprocess-files.cjs: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const path = require('path'); 3 | const { parse } = require('@typescript-eslint/parser'); 4 | 5 | const pkgImportPath = process.env['PKG_IMPORT_PATH'] ?? 'vlmrun/'; 6 | 7 | const distDir = 8 | process.env['DIST_PATH'] ? 9 | path.resolve(process.env['DIST_PATH']) 10 | : path.resolve(__dirname, '..', '..', 'dist'); 11 | const distSrcDir = path.join(distDir, 'src'); 12 | 13 | /** 14 | * Quick and dirty AST traversal 15 | */ 16 | function traverse(node, visitor) { 17 | if (!node || typeof node.type !== 'string') return; 18 | visitor.node?.(node); 19 | visitor[node.type]?.(node); 20 | for (const key in node) { 21 | const value = node[key]; 22 | if (Array.isArray(value)) { 23 | for (const elem of value) traverse(elem, visitor); 24 | } else if (value instanceof Object) { 25 | traverse(value, visitor); 26 | } 27 | } 28 | } 29 | 30 | /** 31 | * Helper method for replacing arbitrary ranges of text in input code. 32 | * 33 | * The `replacer` is a function that will be called with a mini-api. For example: 34 | * 35 | * replaceRanges('foobar', ({ replace }) => replace([0, 3], 'baz')) // 'bazbar' 36 | * 37 | * The replaced ranges must not be overlapping. 38 | */ 39 | function replaceRanges(code, replacer) { 40 | const replacements = []; 41 | replacer({ replace: (range, replacement) => replacements.push({ range, replacement }) }); 42 | 43 | if (!replacements.length) return code; 44 | replacements.sort((a, b) => a.range[0] - b.range[0]); 45 | const overlapIndex = replacements.findIndex( 46 | (r, index) => index > 0 && replacements[index - 1].range[1] > r.range[0], 47 | ); 48 | if (overlapIndex >= 0) { 49 | throw new Error( 50 | `replacements overlap: ${JSON.stringify(replacements[overlapIndex - 1])} and ${JSON.stringify( 51 | replacements[overlapIndex], 52 | )}`, 53 | ); 54 | } 55 | 56 | const parts = []; 57 | let end = 0; 58 | for (const { 59 | range: [from, to], 60 | replacement, 61 | } of replacements) { 62 | if (from > end) parts.push(code.substring(end, from)); 63 | parts.push(replacement); 64 | end = to; 65 | } 66 | if (end < code.length) parts.push(code.substring(end)); 67 | return parts.join(''); 68 | } 69 | 70 | /** 71 | * Like calling .map(), where the iteratee is called on the path in every import or export from statement. 72 | * @returns the transformed code 73 | */ 74 | function mapModulePaths(code, iteratee) { 75 | const ast = parse(code, { range: true }); 76 | return replaceRanges(code, ({ replace }) => 77 | traverse(ast, { 78 | node(node) { 79 | switch (node.type) { 80 | case 'ImportDeclaration': 81 | case 'ExportNamedDeclaration': 82 | case 'ExportAllDeclaration': 83 | case 'ImportExpression': 84 | if (node.source) { 85 | const { range, value } = node.source; 86 | const transformed = iteratee(value); 87 | if (transformed !== value) { 88 | replace(range, JSON.stringify(transformed)); 89 | } 90 | } 91 | } 92 | }, 93 | }), 94 | ); 95 | } 96 | 97 | async function* walk(dir) { 98 | for await (const d of await fs.promises.opendir(dir)) { 99 | const entry = path.join(dir, d.name); 100 | if (d.isDirectory()) yield* walk(entry); 101 | else if (d.isFile()) yield entry; 102 | } 103 | } 104 | 105 | async function postprocess() { 106 | for await (const file of walk(path.resolve(__dirname, '..', '..', 'dist'))) { 107 | if (!/\.([cm]?js|(\.d)?[cm]?ts)$/.test(file)) continue; 108 | 109 | const code = await fs.promises.readFile(file, 'utf8'); 110 | 111 | let transformed = mapModulePaths(code, (importPath) => { 112 | if (file.startsWith(distSrcDir)) { 113 | if (importPath.startsWith(pkgImportPath)) { 114 | // convert self-references in dist/src to relative paths 115 | let relativePath = path.relative( 116 | path.dirname(file), 117 | path.join(distSrcDir, importPath.substring(pkgImportPath.length)), 118 | ); 119 | if (!relativePath.startsWith('.')) relativePath = `./${relativePath}`; 120 | return relativePath; 121 | } 122 | return importPath; 123 | } 124 | if (importPath.startsWith('.')) { 125 | // add explicit file extensions to relative imports 126 | const { dir, name } = path.parse(importPath); 127 | const ext = /\.mjs$/.test(file) ? '.mjs' : '.js'; 128 | return `${dir}/${name}${ext}`; 129 | } 130 | return importPath; 131 | }); 132 | 133 | if (file.startsWith(distSrcDir) && !file.endsWith('_shims/index.d.ts')) { 134 | // strip out `unknown extends Foo ? never :` shim guards in dist/src 135 | // to prevent errors from appearing in Go To Source 136 | transformed = transformed.replace( 137 | new RegExp('unknown extends (typeof )?\\S+ \\? \\S+ :\\s*'.replace(/\s+/, '\\s+'), 'gm'), 138 | // replace with same number of characters to avoid breaking source maps 139 | (match) => ' '.repeat(match.length), 140 | ); 141 | } 142 | 143 | if (file.endsWith('.d.ts')) { 144 | // work around bad tsc behavior 145 | // if we have `import { type Readable } from 'vlmrun/_shims/index'`, 146 | // tsc sometimes replaces `Readable` with `import("stream").Readable` inline 147 | // in the output .d.ts 148 | transformed = transformed.replace(/import\("stream"\).Readable/g, 'Readable'); 149 | } 150 | 151 | // strip out lib="dom" and types="node" references; these are needed at build time, 152 | // but would pollute the user's TS environment 153 | transformed = transformed.replace( 154 | /^ *\/\/\/ * ' '.repeat(match.length - 1) + '\n', 157 | ); 158 | 159 | if (transformed !== code) { 160 | await fs.promises.writeFile(file, transformed, 'utf8'); 161 | console.error(`wrote ${path.relative(process.cwd(), file)}`); 162 | } 163 | } 164 | } 165 | postprocess(); 166 | -------------------------------------------------------------------------------- /tests/unit/client/exceptions.test.ts: -------------------------------------------------------------------------------- 1 | import { 2 | VLMRunError, 3 | APIError, 4 | AuthenticationError, 5 | ValidationError, 6 | RateLimitError, 7 | ServerError, 8 | ResourceNotFoundError, 9 | ClientError, 10 | ConfigurationError, 11 | DependencyError, 12 | InputError, 13 | RequestTimeoutError, 14 | NetworkError 15 | } from '../../../src/client/exceptions'; 16 | 17 | describe('Error classes', () => { 18 | describe('VLMRunError', () => { 19 | it('should create a VLMRunError with the correct name and message', () => { 20 | const error = new VLMRunError('Test error'); 21 | expect(error.name).toBe('VLMRunError'); 22 | expect(error.message).toBe('Test error'); 23 | expect(error instanceof Error).toBe(true); 24 | }); 25 | }); 26 | 27 | describe('APIError', () => { 28 | it('should create an APIError with the correct properties', () => { 29 | const headers = { 'x-request-id': '123' }; 30 | const error = new APIError( 31 | 'API error', 32 | 404, 33 | headers, 34 | '123', 35 | 'not_found', 36 | 'Check the ID' 37 | ); 38 | 39 | expect(error.name).toBe('APIError'); 40 | expect(error.message).toBe('API error'); 41 | expect(error.http_status).toBe(404); 42 | expect(error.headers).toEqual(headers); 43 | expect(error.request_id).toBe('123'); 44 | expect(error.error_type).toBe('not_found'); 45 | expect(error.suggestion).toBe('Check the ID'); 46 | expect(error instanceof VLMRunError).toBe(true); 47 | }); 48 | 49 | it('should format toString properly', () => { 50 | const error = new APIError( 51 | 'API error', 52 | 404, 53 | { 'x-request-id': '123' }, 54 | '123', 55 | 'not_found', 56 | 'Check the ID' 57 | ); 58 | 59 | expect(error.toString()).toBe('[status=404, type=not_found, id=123] API error (Suggestion: Check the ID)'); 60 | }); 61 | }); 62 | 63 | describe('Specialized API errors', () => { 64 | it('should create AuthenticationError with correct defaults', () => { 65 | const error = new AuthenticationError(); 66 | expect(error.message).toBe('Authentication failed'); 67 | expect(error.http_status).toBe(401); 68 | expect(error.error_type).toBe('authentication_error'); 69 | expect(error.suggestion).toBe('Check your API key and ensure it is valid'); 70 | expect(error instanceof APIError).toBe(true); 71 | }); 72 | 73 | it('should create ValidationError with correct defaults', () => { 74 | const error = new ValidationError(); 75 | expect(error.message).toBe('Validation failed'); 76 | expect(error.http_status).toBe(400); 77 | expect(error.error_type).toBe('validation_error'); 78 | expect(error instanceof APIError).toBe(true); 79 | }); 80 | 81 | it('should create RateLimitError with correct defaults', () => { 82 | const error = new RateLimitError(); 83 | expect(error.message).toBe('Rate limit exceeded'); 84 | expect(error.http_status).toBe(429); 85 | expect(error.error_type).toBe('rate_limit_error'); 86 | expect(error instanceof APIError).toBe(true); 87 | }); 88 | 89 | it('should create ServerError with correct defaults', () => { 90 | const error = new ServerError(); 91 | expect(error.message).toBe('Server error'); 92 | expect(error.http_status).toBe(500); 93 | expect(error.error_type).toBe('server_error'); 94 | expect(error instanceof APIError).toBe(true); 95 | }); 96 | 97 | it('should create ResourceNotFoundError with correct defaults', () => { 98 | const error = new ResourceNotFoundError(); 99 | expect(error.message).toBe('Resource not found'); 100 | expect(error.http_status).toBe(404); 101 | expect(error.error_type).toBe('not_found_error'); 102 | expect(error instanceof APIError).toBe(true); 103 | }); 104 | 105 | it('should create RequestTimeoutError with correct defaults', () => { 106 | const error = new RequestTimeoutError(); 107 | expect(error.message).toBe('Request timed out'); 108 | expect(error.http_status).toBe(408); 109 | expect(error.error_type).toBe('timeout_error'); 110 | expect(error instanceof APIError).toBe(true); 111 | }); 112 | 113 | it('should create NetworkError with correct defaults', () => { 114 | const error = new NetworkError(); 115 | expect(error.message).toBe('Network error'); 116 | expect(error.error_type).toBe('network_error'); 117 | expect(error instanceof APIError).toBe(true); 118 | }); 119 | }); 120 | 121 | describe('ClientError', () => { 122 | it('should create a ClientError with the correct properties', () => { 123 | const error = new ClientError( 124 | 'Client error', 125 | 'test_error', 126 | 'Fix it' 127 | ); 128 | 129 | expect(error.name).toBe('ClientError'); 130 | expect(error.message).toBe('Client error'); 131 | expect(error.error_type).toBe('test_error'); 132 | expect(error.suggestion).toBe('Fix it'); 133 | expect(error instanceof VLMRunError).toBe(true); 134 | }); 135 | 136 | it('should format toString properly', () => { 137 | const error = new ClientError( 138 | 'Client error', 139 | 'test_error', 140 | 'Fix it' 141 | ); 142 | 143 | expect(error.toString()).toBe('[type=test_error] Client error (Suggestion: Fix it)'); 144 | }); 145 | }); 146 | 147 | describe('Specialized Client errors', () => { 148 | it('should create ConfigurationError with correct defaults', () => { 149 | const error = new ConfigurationError(); 150 | expect(error.message).toBe('Invalid configuration'); 151 | expect(error.error_type).toBe('configuration_error'); 152 | expect(error instanceof ClientError).toBe(true); 153 | }); 154 | 155 | it('should create DependencyError with correct defaults', () => { 156 | const error = new DependencyError(); 157 | expect(error.message).toBe('Missing dependency'); 158 | expect(error.error_type).toBe('dependency_error'); 159 | expect(error instanceof ClientError).toBe(true); 160 | }); 161 | 162 | it('should create InputError with correct defaults', () => { 163 | const error = new InputError(); 164 | expect(error.message).toBe('Invalid input'); 165 | expect(error.error_type).toBe('input_error'); 166 | expect(error instanceof ClientError).toBe(true); 167 | }); 168 | }); 169 | }); 170 | -------------------------------------------------------------------------------- /tests/unit/client/requestor.test.ts: -------------------------------------------------------------------------------- 1 | import axios from 'axios'; 2 | import { APIRequestor } from '../../../src/client/base_requestor'; 3 | import { 4 | AuthenticationError, 5 | ValidationError, 6 | ResourceNotFoundError, 7 | RateLimitError, 8 | ServerError, 9 | RequestTimeoutError, 10 | NetworkError 11 | } from '../../../src/client/exceptions'; 12 | 13 | jest.mock('axios'); 14 | jest.mock('axios-retry', () => jest.fn()); 15 | const mockedAxios = axios as jest.Mocked; 16 | 17 | describe('APIRequestor', () => { 18 | const client = { 19 | apiKey: 'test-api-key', 20 | baseURL: 'https://api.example.com', 21 | timeout: 5000, 22 | maxRetries: 3 23 | }; 24 | 25 | let requestor: APIRequestor; 26 | 27 | beforeEach(() => { 28 | jest.clearAllMocks(); 29 | 30 | const mockAxiosInstance = { 31 | request: jest.fn(), 32 | defaults: { 33 | headers: { 34 | common: {}, 35 | get: {}, 36 | post: {} 37 | } 38 | }, 39 | interceptors: { 40 | request: { 41 | use: jest.fn(), 42 | eject: jest.fn(), 43 | clear: jest.fn() 44 | }, 45 | response: { 46 | use: jest.fn(), 47 | eject: jest.fn(), 48 | clear: jest.fn() 49 | } 50 | } 51 | }; 52 | 53 | mockedAxios.create.mockReturnValue(mockAxiosInstance as any); 54 | mockedAxios.isAxiosError.mockImplementation((error) => error && error.isAxiosError === true); 55 | 56 | requestor = new APIRequestor(client); 57 | 58 | (requestor as any).axios = mockAxiosInstance; 59 | }); 60 | 61 | it('should initialize with the correct configuration', () => { 62 | expect(mockedAxios.create).toHaveBeenCalledWith({ 63 | baseURL: client.baseURL, 64 | headers: { 65 | Authorization: `Bearer ${client.apiKey}`, 66 | 'Content-Type': 'application/json' 67 | }, 68 | timeout: client.timeout 69 | }); 70 | }); 71 | 72 | it('should return successful response', async () => { 73 | const mockResponse = { 74 | data: { result: 'success' }, 75 | status: 200, 76 | headers: { 'content-type': 'application/json' } 77 | }; 78 | 79 | (requestor as any).axios.request.mockResolvedValue(mockResponse); 80 | 81 | const result = await requestor.request('GET', '/test'); 82 | expect(result).toEqual([ 83 | mockResponse.data, 84 | mockResponse.status, 85 | mockResponse.headers 86 | ]); 87 | }); 88 | 89 | it('should throw AuthenticationError for 401 responses', async () => { 90 | const mockError = { 91 | response: { 92 | data: { error: { message: 'Invalid API key', type: 'auth_error', id: '123' } }, 93 | status: 401, 94 | headers: { 'x-request-id': '123' } 95 | }, 96 | isAxiosError: true, 97 | message: 'Request failed with status code 401' 98 | }; 99 | 100 | mockedAxios.isAxiosError.mockReturnValue(true); 101 | 102 | (requestor as any).axios.request.mockRejectedValue(mockError); 103 | 104 | await expect(requestor.request('GET', '/test')).rejects.toThrow(AuthenticationError); 105 | await expect(requestor.request('GET', '/test')).rejects.toMatchObject({ 106 | http_status: 401, 107 | error_type: 'authentication_error', 108 | request_id: undefined 109 | }); 110 | }); 111 | 112 | it('should throw ValidationError for 400 responses', async () => { 113 | const mockError = { 114 | response: { 115 | data: { error: { message: 'Invalid parameters', type: 'validation_error', id: '123' } }, 116 | status: 400, 117 | headers: { 'x-request-id': '123' } 118 | }, 119 | isAxiosError: true, 120 | message: 'Request failed with status code 400' 121 | }; 122 | 123 | mockedAxios.isAxiosError.mockReturnValue(true); 124 | (requestor as any).axios.request.mockRejectedValue(mockError); 125 | 126 | await expect(requestor.request('GET', '/test')).rejects.toThrow(ValidationError); 127 | }); 128 | 129 | it('should throw ResourceNotFoundError for 404 responses', async () => { 130 | const mockError = { 131 | response: { 132 | data: { error: { message: 'Resource not found', type: 'not_found', id: '123' } }, 133 | status: 404, 134 | headers: { 'x-request-id': '123' } 135 | }, 136 | isAxiosError: true, 137 | message: 'Request failed with status code 404' 138 | }; 139 | 140 | mockedAxios.isAxiosError.mockReturnValue(true); 141 | (requestor as any).axios.request.mockRejectedValue(mockError); 142 | 143 | await expect(requestor.request('GET', '/test')).rejects.toThrow(ResourceNotFoundError); 144 | }); 145 | 146 | it('should throw RateLimitError for 429 responses', async () => { 147 | const mockError = { 148 | response: { 149 | data: { error: { message: 'Too many requests', type: 'rate_limit', id: '123' } }, 150 | status: 429, 151 | headers: { 'x-request-id': '123' } 152 | }, 153 | isAxiosError: true, 154 | message: 'Request failed with status code 429' 155 | }; 156 | 157 | mockedAxios.isAxiosError.mockReturnValue(true); 158 | (requestor as any).axios.request.mockRejectedValue(mockError); 159 | 160 | await expect(requestor.request('GET', '/test')).rejects.toThrow(RateLimitError); 161 | }); 162 | 163 | it('should throw ServerError for 5xx responses', async () => { 164 | const mockError = { 165 | response: { 166 | data: { error: { message: 'Internal server error', type: 'server_error', id: '123' } }, 167 | status: 500, 168 | headers: { 'x-request-id': '123' } 169 | }, 170 | isAxiosError: true, 171 | message: 'Request failed with status code 500' 172 | }; 173 | 174 | mockedAxios.isAxiosError.mockReturnValue(true); 175 | (requestor as any).axios.request.mockRejectedValue(mockError); 176 | 177 | await expect(requestor.request('GET', '/test')).rejects.toThrow(ServerError); 178 | }); 179 | 180 | it('should throw RequestTimeoutError for timeout errors', async () => { 181 | const mockError = { 182 | code: 'ECONNABORTED', 183 | isAxiosError: true, 184 | message: 'timeout of 5000ms exceeded' 185 | }; 186 | 187 | mockedAxios.isAxiosError.mockReturnValue(true); 188 | (requestor as any).axios.request.mockRejectedValue(mockError); 189 | 190 | await expect(requestor.request('GET', '/test')).rejects.toThrow(RequestTimeoutError); 191 | }); 192 | 193 | it('should throw NetworkError for network errors', async () => { 194 | const mockError = { 195 | isAxiosError: true, 196 | message: 'Network Error', 197 | code: 'ERR_NETWORK' 198 | }; 199 | 200 | mockedAxios.isAxiosError.mockReturnValue(true); 201 | (requestor as any).axios.request.mockRejectedValue(mockError); 202 | 203 | await expect(requestor.request('GET', '/test')).rejects.toThrow(NetworkError); 204 | }); 205 | }); 206 | -------------------------------------------------------------------------------- /tests/integration/client/video-predictions.test.ts: -------------------------------------------------------------------------------- 1 | import { config } from "dotenv"; 2 | 3 | import { VlmRun } from "../../../src/index"; 4 | import { z } from "zod"; 5 | 6 | jest.setTimeout(300000); 7 | 8 | describe("Integration: Video Predictions", () => { 9 | let client: VlmRun; 10 | 11 | beforeAll(() => { 12 | config({ path: ".env.test" }); 13 | 14 | client = new VlmRun({ 15 | apiKey: process.env.TEST_API_KEY as string, 16 | baseURL: process.env.TEST_BASE_URL as string, 17 | }); 18 | }); 19 | 20 | describe("VideoPredictions", () => { 21 | const testFilePath = "tests/integration/assets/timer_video.mp4"; 22 | 23 | it("should generate video predictions using file id with batch processing", async () => { 24 | const uploadedVideo = await client.files.upload({ 25 | filePath: testFilePath, 26 | checkDuplicate: false, 27 | }); 28 | 29 | const result = await client.video.generate({ 30 | fileId: uploadedVideo.id, 31 | batch: true, 32 | domain: "video.transcription", 33 | }); 34 | 35 | // Initial response should be pending for batch processing 36 | expect(result).toHaveProperty("id"); 37 | expect(result).toHaveProperty("created_at"); 38 | expect(result).toHaveProperty("completed_at"); 39 | expect(result).toHaveProperty("usage"); 40 | expect(result.status).toBe("pending"); 41 | expect(result.response).toBeNull(); 42 | expect(result.completed_at).toBeNull(); 43 | 44 | // Wait for completion 45 | const waitResponse = await client.predictions.wait(result.id, 300); 46 | 47 | expect(waitResponse.status).toBe("completed"); 48 | expect(waitResponse.response).not.toBeNull(); 49 | expect(waitResponse.completed_at).not.toBeNull(); 50 | 51 | // Check usage information 52 | expect(waitResponse.usage).toBeDefined(); 53 | expect(waitResponse.usage!).toHaveProperty("elements_processed"); 54 | expect(waitResponse.usage!).toHaveProperty("element_type"); 55 | expect(waitResponse.usage!).toHaveProperty("credits_used"); 56 | expect(waitResponse.usage!.element_type).toBe("video"); 57 | 58 | // Check response structure 59 | expect(waitResponse.response).toHaveProperty("metadata"); 60 | expect(waitResponse.response).toHaveProperty("segments"); 61 | 62 | // Check metadata 63 | expect(waitResponse.response.metadata).toHaveProperty("duration"); 64 | expect(typeof waitResponse.response.metadata.duration).toBe("number"); 65 | expect(waitResponse.response.metadata.duration).toBeGreaterThan(0); 66 | 67 | // Check segments 68 | expect(Array.isArray(waitResponse.response.segments)).toBe(true); 69 | expect(waitResponse.response.segments.length).toBeGreaterThan(0); 70 | 71 | // Check first segment structure 72 | const firstSegment = waitResponse.response.segments[0]; 73 | expect(firstSegment).toHaveProperty("start_time"); 74 | expect(firstSegment).toHaveProperty("end_time"); 75 | expect(firstSegment).toHaveProperty("audio"); 76 | expect(firstSegment).toHaveProperty("video"); 77 | expect(typeof firstSegment.start_time).toBe("number"); 78 | expect(typeof firstSegment.end_time).toBe("number"); 79 | 80 | // Check audio content in segment 81 | expect(firstSegment.audio).toHaveProperty("content"); 82 | expect(typeof firstSegment.audio.content).toBe("string"); 83 | expect(firstSegment.audio.content.length).toBeGreaterThan(0); 84 | 85 | // Check video content in segment 86 | expect(firstSegment.video).toHaveProperty("content"); 87 | expect(typeof firstSegment.video.content).toBe("string"); 88 | expect(firstSegment.video.content.length).toBeGreaterThan(0); 89 | }); 90 | 91 | it("should generate video predictions using url", async () => { 92 | const videoUrl = 93 | "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/video.transcription/timer_video.mp4"; 94 | 95 | const result = await client.video.generate({ 96 | url: videoUrl, 97 | domain: "video.transcription", 98 | batch: true, 99 | }); 100 | 101 | // Initial response should be pending for batch processing 102 | expect(result).toHaveProperty("id"); 103 | expect(result).toHaveProperty("created_at"); 104 | expect(result).toHaveProperty("completed_at"); 105 | expect(result).toHaveProperty("usage"); 106 | expect(result.status).toBe("pending"); 107 | expect(result.response).toBeNull(); 108 | expect(result.completed_at).toBeNull(); 109 | 110 | // Wait for completion 111 | const waitResponse = await client.predictions.wait(result.id, 300); 112 | 113 | expect(waitResponse.status).toBe("completed"); 114 | expect(waitResponse.response).not.toBeNull(); 115 | expect(waitResponse.completed_at).not.toBeNull(); 116 | 117 | // Check usage information 118 | expect(waitResponse.usage).toBeDefined(); 119 | expect(waitResponse.usage!).toHaveProperty("elements_processed"); 120 | expect(waitResponse.usage!).toHaveProperty("element_type"); 121 | expect(waitResponse.usage!).toHaveProperty("credits_used"); 122 | expect(waitResponse.usage!.element_type).toBe("video"); 123 | 124 | // Check response structure 125 | expect(waitResponse.response).toHaveProperty("metadata"); 126 | expect(waitResponse.response).toHaveProperty("segments"); 127 | 128 | // Check metadata 129 | expect(waitResponse.response.metadata).toHaveProperty("duration"); 130 | expect(typeof waitResponse.response.metadata.duration).toBe("number"); 131 | expect(waitResponse.response.metadata.duration).toBeGreaterThan(0); 132 | 133 | // Check segments 134 | expect(Array.isArray(waitResponse.response.segments)).toBe(true); 135 | expect(waitResponse.response.segments.length).toBeGreaterThan(0); 136 | 137 | // Check first segment structure 138 | const firstSegment = waitResponse.response.segments[0]; 139 | expect(firstSegment).toHaveProperty("start_time"); 140 | expect(firstSegment).toHaveProperty("end_time"); 141 | expect(firstSegment).toHaveProperty("audio"); 142 | expect(firstSegment).toHaveProperty("video"); 143 | expect(typeof firstSegment.start_time).toBe("number"); 144 | expect(typeof firstSegment.end_time).toBe("number"); 145 | 146 | // Check audio content in segment 147 | expect(firstSegment.audio).toHaveProperty("content"); 148 | expect(typeof firstSegment.audio.content).toBe("string"); 149 | expect(firstSegment.audio.content.length).toBeGreaterThan(0); 150 | 151 | // Check video content in segment 152 | expect(firstSegment.video).toHaveProperty("content"); 153 | expect(typeof firstSegment.video.content).toBe("string"); 154 | expect(firstSegment.video.content.length).toBeGreaterThan(0); 155 | 156 | // Test get endpoint 157 | const getResponse = await client.predictions.get(result.id); 158 | expect(getResponse.status).toBe("completed"); 159 | expect(getResponse.response).not.toBeNull(); 160 | expect(getResponse.response).toHaveProperty("segments"); 161 | }); 162 | }); 163 | }); 164 | -------------------------------------------------------------------------------- /tests/integration/client/image-predictions.test.ts: -------------------------------------------------------------------------------- 1 | import { config } from "dotenv"; 2 | 3 | import { VlmRun } from "../../../src/index"; 4 | import { z } from "zod"; 5 | 6 | jest.setTimeout(60000); 7 | 8 | describe("Integration: Image Predictions", () => { 9 | let client: VlmRun; 10 | 11 | beforeAll(() => { 12 | config({ path: ".env.test" }); 13 | 14 | client = new VlmRun({ 15 | apiKey: process.env.TEST_API_KEY as string, 16 | baseURL: process.env.TEST_BASE_URL as string, 17 | }); 18 | }); 19 | 20 | describe("ImagePredictions", () => { 21 | it("should generate image predictions with default options", async () => { 22 | const testImagePath = "tests/integration/assets/invoice.jpg"; 23 | 24 | const result = await client.image.generate({ 25 | images: [testImagePath], 26 | model: "vlm-1", 27 | domain: "document.invoice", 28 | }); 29 | 30 | expect(result).toHaveProperty("id"); 31 | expect(result.status).toBe("completed"); 32 | expect(result.response).toHaveProperty("invoice_id"); 33 | expect(result.response).toHaveProperty("invoice_issue_date"); 34 | 35 | expect(typeof result.response.customer).toBe("string"); 36 | expect(result.response).toHaveProperty("customer_email"); 37 | expect(result.response).toHaveProperty("customer_phone"); 38 | expect(result.response).toHaveProperty("customer_billing_address"); 39 | expect(result.response.customer_billing_address).toHaveProperty("street"); 40 | expect(result.response.customer_billing_address).toHaveProperty("city"); 41 | expect(result.response.customer_billing_address).toHaveProperty("state"); 42 | expect(result.response.customer_billing_address).toHaveProperty( 43 | "postal_code" 44 | ); 45 | expect(result.response).toHaveProperty("customer_shipping_address"); 46 | expect(result.response.customer_shipping_address).toHaveProperty( 47 | "street" 48 | ); 49 | expect(result.response.customer_shipping_address).toHaveProperty("city"); 50 | expect(result.response.customer_shipping_address).toHaveProperty("state"); 51 | expect(result.response.customer_shipping_address).toHaveProperty( 52 | "postal_code" 53 | ); 54 | expect(result.response).toHaveProperty("items"); 55 | expect(result.response).toHaveProperty("subtotal"); 56 | expect(result.response).toHaveProperty("total"); 57 | 58 | expect(result.response).toHaveProperty("subtotal"); 59 | expect(result.response).toHaveProperty("total"); 60 | }); 61 | 62 | it("should generate image predictions from path with zod schema", async () => { 63 | const testImagePath = "tests/integration/assets/invoice.jpg"; 64 | 65 | const schema = z.object({ 66 | invoice_id: z.string(), 67 | invoice_issue_date: z.string(), 68 | customer: z.string(), 69 | customer_email: z.string(), 70 | customer_phone: z.string(), 71 | total: z.number(), 72 | }); 73 | 74 | const result = await client.image.generate({ 75 | images: [testImagePath], 76 | model: "vlm-1", 77 | domain: "document.invoice", 78 | config: { 79 | responseModel: schema, 80 | }, 81 | }); 82 | 83 | const response = result.response as z.infer; 84 | expect(response).toHaveProperty("invoice_id"); 85 | expect(response).toHaveProperty("total"); 86 | }); 87 | 88 | it("should generate image predictions from url with zod schema", async () => { 89 | const imageUrl = 90 | "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/document.invoice/invoice_1.jpg"; 91 | 92 | const schema = z.object({ 93 | invoice_id: z.string(), 94 | total: z.number(), 95 | }); 96 | 97 | const result = await client.image.generate({ 98 | images: [imageUrl], 99 | model: "vlm-1", 100 | domain: "document.invoice", 101 | config: { 102 | responseModel: schema, 103 | }, 104 | }); 105 | 106 | const response = result.response as z.infer; 107 | expect(response).toHaveProperty("invoice_id"); 108 | expect(response).toHaveProperty("total"); 109 | }); 110 | 111 | it("should generate image predictions with custom options", async () => { 112 | const imageUrl = 113 | "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/document.invoice/invoice_1.jpg"; 114 | 115 | const result = await client.image.generate({ 116 | urls: [imageUrl], 117 | model: "vlm-1", 118 | domain: "document.invoice", 119 | config: { 120 | jsonSchema: { 121 | type: "object", 122 | properties: { 123 | invoice_number: { type: "string" }, 124 | total_amount: { type: "number" }, 125 | }, 126 | }, 127 | }, 128 | }); 129 | 130 | expect(result).toHaveProperty("id"); 131 | expect(result.status).toBe("completed"); 132 | expect(result.response).toHaveProperty("invoice_number"); 133 | expect(result.response).toHaveProperty("total_amount"); 134 | 135 | expect(result.response).not.toHaveProperty("invoice_issue_date"); 136 | expect(result.response).not.toHaveProperty("customer"); 137 | expect(result.response).not.toHaveProperty("customer_email"); 138 | expect(result.response).not.toHaveProperty("customer_phone"); 139 | expect(result.response).not.toHaveProperty("customer_billing_address"); 140 | expect(result.response).not.toHaveProperty("customer_shipping_address"); 141 | expect(result.response).not.toHaveProperty("items"); 142 | }); 143 | 144 | describe("schema", () => { 145 | it("should generate schema from image path", async () => { 146 | const testImagePath = "tests/integration/assets/invoice.jpg"; 147 | 148 | const result = await client.image.schema({ 149 | images: [testImagePath], 150 | }); 151 | 152 | expect(result).toHaveProperty("id"); 153 | expect(result.status).toBe("completed"); 154 | expect(result.response).toHaveProperty("json_schema"); 155 | expect(result.response).toHaveProperty("schema_version"); 156 | expect(result.response).toHaveProperty("schema_hash"); 157 | expect(result.response).toHaveProperty("domain"); 158 | expect(result.response).toHaveProperty("description"); 159 | 160 | // The schema should be for an invoice document 161 | expect(result.response.json_schema).toHaveProperty("properties"); 162 | }); 163 | 164 | it("should throw an error when neither images nor urls are provided", async () => { 165 | await expect(client.image.schema({})).rejects.toThrow( 166 | "Either `images` or `urls` must be provided" 167 | ); 168 | }); 169 | 170 | it("should throw an error when both images and urls are provided", async () => { 171 | const testImagePath = "tests/integration/assets/invoice.jpg"; 172 | const imageUrl = 173 | "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/document.invoice/invoice_1.jpg"; 174 | 175 | await expect( 176 | client.image.schema({ 177 | images: [testImagePath], 178 | urls: [imageUrl], 179 | }) 180 | ).rejects.toThrow("Only one of `images` or `urls` can be provided"); 181 | }); 182 | }); 183 | }); 184 | }); 185 | -------------------------------------------------------------------------------- /src/client/fine_tuning.ts: -------------------------------------------------------------------------------- 1 | import { Client, APIRequestor } from "./base_requestor"; 2 | import { FinetuningResponse, FinetuningProvisionResponse, FinetuningGenerateParams, FinetuningListParams, PredictionResponse, FinetuningCreateParams, FinetuningProvisionParams } from "./types"; 3 | import { processImage } from "../utils"; 4 | import { InputError, ConfigurationError } from "./exceptions"; 5 | 6 | export class Finetuning { 7 | private requestor: APIRequestor; 8 | 9 | constructor(client: Client) { 10 | this.requestor = new APIRequestor({ 11 | ...client, 12 | baseURL: `${client.baseURL}/fine_tuning`, 13 | }); 14 | } 15 | 16 | /** 17 | * Create a fine-tuning job 18 | * @param {Object} params - Fine-tuning parameters 19 | * @param {string} params.model - Base model to fine-tune 20 | * @param {string} params.training_file_id - File ID for training data 21 | * @param {string} [params.validation_file_id] - File ID for validation data 22 | * @param {number} [params.num_epochs=1] - Number of epochs 23 | * @param {number|string} [params.batch_size="auto"] - Batch size for training 24 | * @param {number} [params.learning_rate=2e-4] - Learning rate for training 25 | * @param {string} [params.suffix] - Suffix for the fine-tuned model 26 | * @param {string} [params.wandb_api_key] - Weights & Biases API key 27 | * @param {string} [params.wandb_base_url] - Weights & Biases base URL 28 | * @param {string} [params.wandb_project_name] - Weights & Biases project name 29 | */ 30 | async create(params: FinetuningCreateParams): Promise { 31 | if (params.suffix) { 32 | // Ensure suffix contains only alphanumeric, hyphens or underscores 33 | if (!/^[a-zA-Z0-9_-]+$/.test(params.suffix)) { 34 | throw new InputError( 35 | "Suffix must be alphanumeric, hyphens or underscores without spaces", 36 | "invalid_format", 37 | "Use only letters, numbers, hyphens, and underscores in the suffix" 38 | ); 39 | } 40 | } 41 | 42 | const [response] = await this.requestor.request( 43 | "POST", 44 | "create", 45 | undefined, 46 | { 47 | callback_url: params.callbackUrl, 48 | model: params.model, 49 | training_file: params.trainingFile, 50 | validation_file: params.validationFile, 51 | num_epochs: params.numEpochs ?? 1, 52 | batch_size: params.batchSize ?? 1, 53 | learning_rate: params.learningRate ?? 2e-4, 54 | suffix: params.suffix, 55 | wandb_api_key: params.wandbApiKey, 56 | wandb_base_url: params.wandbBaseUrl, 57 | wandb_project_name: params.wandbProjectName, 58 | } 59 | ); 60 | 61 | return response; 62 | } 63 | 64 | /** 65 | * Provision a fine-tuning model 66 | * @param {Object} params - Provisioning parameters 67 | * @param {string} params.model - Model to provision 68 | * @param {number} [params.duration=600] - Duration for the provisioned model (in seconds) 69 | * @param {number} [params.concurrency=1] - Concurrency for the provisioned model 70 | */ 71 | async provision(params: FinetuningProvisionParams): Promise { 72 | const [response] = await this.requestor.request( 73 | "POST", 74 | "provision", 75 | undefined, 76 | { 77 | model: params.model, 78 | duration: params.duration ?? 600, // 10 minutes default 79 | concurrency: params.concurrency ?? 1, 80 | } 81 | ); 82 | 83 | return response; 84 | } 85 | 86 | /** 87 | * Generate a prediction using a fine-tuned model 88 | * @param {FinetuningGenerateParams} params - Generation parameters 89 | */ 90 | async generate(params: FinetuningGenerateParams): Promise { 91 | if (!params.jsonSchema) { 92 | throw new InputError("JSON schema is required for fine-tuned model predictions", "missing_parameter", "Provide a JSON schema for the prediction"); 93 | } 94 | 95 | if (!params.prompt) { 96 | throw new InputError("Prompt is required for fine-tuned model predictions", "missing_parameter", "Provide a prompt for the prediction"); 97 | } 98 | 99 | if (params.domain) { 100 | throw new ConfigurationError("Domain is not supported for fine-tuned model predictions", "unsupported_parameter", "Remove the domain parameter for fine-tuned models"); 101 | } 102 | 103 | if (params.detail && params.detail !== "auto") { 104 | throw new ConfigurationError("Detail level is not supported for fine-tuned model predictions", "unsupported_parameter", "Use 'auto' or remove the detail parameter for fine-tuned models"); 105 | } 106 | 107 | if (params.batch) { 108 | throw new ConfigurationError("Batch mode is not supported for fine-tuned models", "unsupported_parameter", "Remove the batch parameter for fine-tuned models"); 109 | } 110 | 111 | if (params.callbackUrl) { 112 | throw new ConfigurationError("Callback URL is not supported for fine-tuned model predictions", "unsupported_parameter", "Remove the callbackUrl parameter for fine-tuned models"); 113 | } 114 | 115 | const [response] = await this.requestor.request( 116 | "POST", 117 | "generate", 118 | undefined, 119 | { 120 | images: params.images.map((image) => processImage(image)), 121 | model: params.model, 122 | config: { 123 | prompt: params.prompt, 124 | json_schema: params.jsonSchema, 125 | detail: params.detail ?? "auto", 126 | response_model: params.responseModel, 127 | confidence: params.confidence ?? false, 128 | grounding: params.grounding ?? false, 129 | max_retries: params.maxRetries ?? 3, 130 | max_tokens: params.maxTokens ?? 65535, 131 | }, 132 | max_new_tokens: params.maxNewTokens ?? 65535, 133 | temperature: params.temperature ?? 0.0, 134 | metadata: { 135 | environment: params?.environment ?? "dev", 136 | session_id: params?.sessionId, 137 | allow_training: params?.allowTraining ?? true, 138 | }, 139 | batch: params.batch ?? false, 140 | callback_url: params.callbackUrl, 141 | } 142 | ); 143 | 144 | return response; 145 | } 146 | 147 | /** 148 | * List all fine-tuning jobs 149 | * @param {FinetuningListParams} params - List parameters 150 | */ 151 | async list(params?: FinetuningListParams): Promise { 152 | const [response] = await this.requestor.request( 153 | "GET", 154 | "jobs", 155 | { 156 | skip: params?.skip ?? 0, 157 | limit: params?.limit ?? 10, 158 | } 159 | ); 160 | 161 | return response; 162 | } 163 | 164 | /** 165 | * Get fine-tuning job details 166 | * @param {string} jobId - ID of job to retrieve 167 | */ 168 | async get(jobId: string): Promise { 169 | const [response] = await this.requestor.request( 170 | "GET", 171 | `jobs/${jobId}` 172 | ); 173 | 174 | return response; 175 | } 176 | 177 | /** 178 | * Cancel a fine-tuning job 179 | * @param {string} jobId - ID of job to cancel 180 | */ 181 | async cancel(jobId: string): Promise> { 182 | throw new ConfigurationError("Cancel method is not implemented yet", "not_implemented", "This feature will be available in a future release"); 183 | } 184 | } 185 | -------------------------------------------------------------------------------- /src/client/files.ts: -------------------------------------------------------------------------------- 1 | import { createHash } from "crypto"; 2 | import axios from "axios"; 3 | import { Client, APIRequestor } from "./base_requestor"; 4 | import { 5 | FileResponse, 6 | ListParams, 7 | FileUploadParams, 8 | PresignedUrlResponse, 9 | PresignedUrlRequest, 10 | } from "./types"; 11 | import { readFileFromPathAsFile } from "../utils/file"; 12 | import { DependencyError, InputError } from "./exceptions"; 13 | 14 | export class Files { 15 | private client: Client; 16 | private requestor: APIRequestor; 17 | 18 | constructor(client: Client) { 19 | this.client = client; 20 | this.requestor = new APIRequestor(client); 21 | } 22 | 23 | async list(params: ListParams = {}): Promise { 24 | const [response] = await this.requestor.request( 25 | "GET", 26 | "files", 27 | { skip: params.skip, limit: params.limit } 28 | ); 29 | return response; 30 | } 31 | 32 | /** 33 | * Calculate the MD5 hash of a file by reading it in chunks 34 | * @param filePath Path to the file to hash 35 | * @returns MD5 hash of the file as a hex string 36 | * @private 37 | */ 38 | private async calculateMD5(filePath: string): Promise { 39 | if (typeof window !== "undefined") { 40 | throw new DependencyError( 41 | "File hashing is not supported in the browser", 42 | "browser_limitation", 43 | "Use server-side file hashing instead" 44 | ); 45 | } 46 | 47 | const fs = require("fs"); 48 | const hash = createHash("md5"); 49 | const chunkSize = 4 * 1024 * 1024; // 4MB chunks, same as Python implementation 50 | 51 | return new Promise((resolve, reject) => { 52 | const stream = fs.createReadStream(filePath, { 53 | highWaterMark: chunkSize, 54 | }); 55 | 56 | stream.on("data", (chunk: Buffer) => { 57 | hash.update(chunk); 58 | }); 59 | 60 | stream.on("end", () => { 61 | const fileHash = hash.digest("hex"); 62 | resolve(fileHash); 63 | }); 64 | 65 | stream.on("error", (error: Error) => { 66 | reject(error); 67 | }); 68 | }); 69 | } 70 | 71 | /** 72 | * Get a cached file from the API by calculating its MD5 hash 73 | * @param filePath Path to the file to check 74 | * @returns FileResponse if the file exists, null otherwise 75 | */ 76 | async getCachedFile(filePath: string): Promise { 77 | const fileHash = await this.calculateMD5(filePath); 78 | 79 | try { 80 | const [response] = await this.requestor.request( 81 | "GET", 82 | `files/hash/${fileHash}` 83 | ); 84 | return response; 85 | } catch (error) { 86 | // If the file doesn't exist or there's an error, return null 87 | return null; 88 | } 89 | } 90 | 91 | // Keep the old method for backward compatibility 92 | async checkFileExists(filePath: string): Promise { 93 | return this.getCachedFile(filePath); 94 | } 95 | 96 | async upload(params: FileUploadParams): Promise { 97 | let fileToUpload: File; 98 | let filePath: string | undefined; 99 | 100 | if (params.file) { 101 | fileToUpload = params.file; 102 | } else if (params.filePath) { 103 | filePath = params.filePath; 104 | 105 | if (typeof window === "undefined") { 106 | const fs = require("fs"); 107 | if (!fs.existsSync(filePath)) { 108 | throw new InputError( 109 | `File does not exist: ${filePath}`, 110 | "file_not_found", 111 | "Provide a valid file path" 112 | ); 113 | } 114 | } 115 | 116 | if (params.checkDuplicate !== false && !params.force) { 117 | const existingFile = await this.getCachedFile(params.filePath); 118 | if (existingFile) { 119 | return existingFile; 120 | } 121 | } 122 | 123 | fileToUpload = await readFileFromPathAsFile(params.filePath); 124 | } else { 125 | throw new InputError( 126 | "Either file or filePath must be provided.", 127 | "missing_parameter", 128 | "Provide either a file object or a filePath string" 129 | ); 130 | } 131 | 132 | let method = params.method || "auto"; 133 | if (method === "auto") { 134 | if (fileToUpload.size > 32 * 1024 * 1024) { 135 | method = "presigned-url"; 136 | } else { 137 | method = "direct"; 138 | } 139 | } 140 | 141 | if (method === "presigned-url") { 142 | const [presignedResponse] = 143 | await this.requestor.request( 144 | "POST", 145 | "files/presigned-url", 146 | undefined, 147 | { 148 | filename: fileToUpload.name, 149 | purpose: params.purpose ?? "assistants", 150 | } 151 | ); 152 | 153 | if (!presignedResponse.url || !presignedResponse.id) { 154 | throw new Error("Invalid presigned URL response"); 155 | } 156 | 157 | const startTime = Date.now(); 158 | try { 159 | const putResponse = await axios.put( 160 | presignedResponse.url, 161 | fileToUpload, 162 | { 163 | headers: { 164 | "Content-Type": 165 | presignedResponse.content_type || "application/octet-stream", 166 | }, 167 | } 168 | ); 169 | 170 | const endTime = Date.now(); 171 | 172 | if (putResponse.status === 200) { 173 | const [verifyResponse] = await this.requestor.request( 174 | "GET", 175 | `files/verify-upload/${presignedResponse.id}` 176 | ); 177 | return verifyResponse; 178 | } else { 179 | throw new Error( 180 | `Failed to upload file to presigned URL: ${putResponse.statusText}` 181 | ); 182 | } 183 | } catch (error) { 184 | if (axios.isAxiosError(error)) { 185 | throw new Error( 186 | `Failed to upload file to presigned URL: ${error.message}` 187 | ); 188 | } 189 | throw error; 190 | } 191 | } else if (method === "direct") { 192 | const [response] = await this.requestor.request( 193 | "POST", 194 | "files", 195 | { 196 | purpose: params.purpose ?? "assistants", 197 | generate_public_url: params.generatePublicUrl ?? true, 198 | }, 199 | undefined, 200 | { file: fileToUpload } 201 | ); 202 | return response; 203 | } else { 204 | throw new InputError( 205 | `Invalid upload method: ${method}`, 206 | "invalid_parameter", 207 | "Use 'auto', 'direct', or 'presigned-url'" 208 | ); 209 | } 210 | } 211 | 212 | async get( 213 | fileId: string, 214 | generatePublicUrl?: boolean 215 | ): Promise { 216 | const [response] = await this.requestor.request( 217 | "GET", 218 | `files/${fileId}`, 219 | generatePublicUrl !== undefined 220 | ? { generate_public_url: generatePublicUrl } 221 | : undefined 222 | ); 223 | return response; 224 | } 225 | 226 | async delete(fileId: string): Promise { 227 | await this.requestor.request("DELETE", `files/${fileId}`); 228 | } 229 | 230 | async generatePresignedUrl( 231 | params: PresignedUrlRequest 232 | ): Promise { 233 | const [response] = await this.requestor.request( 234 | "POST", 235 | "files/presigned-url", 236 | undefined, 237 | { 238 | filename: params.filename, 239 | purpose: params.purpose, 240 | } 241 | ); 242 | return response; 243 | } 244 | } 245 | -------------------------------------------------------------------------------- /tests/unit/client/datasets.test.ts: -------------------------------------------------------------------------------- 1 | import { Client } from '../../../src/client/base_requestor'; 2 | import { Datasets } from '../../../src/client/datasets'; 3 | import { DatasetResponse, FileResponse } from '../../../src/client/types'; 4 | import { Files } from '../../../src/client/files'; 5 | 6 | jest.mock('../../../src/client/base_requestor'); 7 | jest.mock('../../../src/client/files'); 8 | jest.mock('../../../src/utils/file', () => ({ 9 | createArchive: jest.fn().mockResolvedValue('/tmp/test-dataset.tar.gz'), 10 | })); 11 | 12 | jest.mock('fs', () => ({ 13 | ...jest.requireActual('fs'), 14 | statSync: jest.fn().mockReturnValue({ size: 1024 * 1024 }), // 1 MB mock file 15 | })); 16 | 17 | jest.mock('../../../src/client/files', () => ({ 18 | Files: jest.fn().mockImplementation(() => ({ 19 | upload: jest.fn().mockResolvedValue({ 20 | id: 'file_123', 21 | filename: 'test-dataset.tar.gz', 22 | bytes: 1024 * 1024, // Mocking a 1 MB file 23 | purpose: 'datasets', 24 | created_at: new Date().toISOString(), 25 | object: 'file', 26 | }), 27 | })), 28 | })); 29 | 30 | 31 | describe('Datasets', () => { 32 | let client: jest.Mocked; 33 | let datasets: Datasets; 34 | let mockFiles: jest.Mocked; 35 | 36 | beforeEach(() => { 37 | client = { 38 | apiKey: 'test-api-key', 39 | baseURL: 'https://api.example.com', 40 | } as jest.Mocked; 41 | 42 | mockFiles = { 43 | upload: jest.fn(), 44 | } as unknown as jest.Mocked; 45 | 46 | datasets = new Datasets(client); 47 | // @ts-ignore - Accessing private property for testing 48 | datasets.vlmClient = { files: mockFiles }; 49 | }); 50 | 51 | describe('create', () => { 52 | it('should create a dataset with minimal parameters', async () => { 53 | const mockFileResponse: FileResponse = { 54 | id: 'file_123', 55 | filename: 'test-dataset.tar.gz', 56 | bytes: 1024, 57 | purpose: 'datasets', 58 | created_at: new Date().toISOString(), 59 | object: 'file', 60 | }; 61 | 62 | const mockDatasetResponse: DatasetResponse = { 63 | id: 'ds_123', 64 | status: 'running', 65 | domain: 'test-domain', 66 | dataset_name: 'test-dataset', 67 | dataset_type: 'images', 68 | file_id: 'file_123', 69 | created_at: new Date().toISOString(), 70 | }; 71 | 72 | mockFiles.upload.mockResolvedValue(mockFileResponse); 73 | jest.spyOn(datasets['requestor'], 'request').mockResolvedValue([mockDatasetResponse, 200, {}]); 74 | 75 | const result = await datasets.create({ 76 | datasetDirectory: '/path/to/dataset', 77 | domain: 'test-domain', 78 | datasetName: 'test-dataset', 79 | datasetType: 'images', 80 | }); 81 | 82 | expect(result).toEqual(mockDatasetResponse); 83 | expect(datasets['requestor'].request).toHaveBeenCalledWith( 84 | 'POST', 85 | 'create', 86 | undefined, 87 | { 88 | file_id: 'file_123', 89 | domain: 'test-domain', 90 | dataset_name: 'test-dataset', 91 | dataset_type: 'images', 92 | wandb_base_url: undefined, 93 | wandb_project_name: undefined, 94 | wandb_api_key: undefined, 95 | } 96 | ); 97 | }); 98 | 99 | it('should create a dataset with minimal parameters', async () => { 100 | mockFiles.upload.mockResolvedValueOnce({ 101 | id: 'file_123', 102 | filename: 'test-dataset.tar.gz', 103 | bytes: 1024 * 1024, // Ensure 'bytes' field is present 104 | purpose: 'datasets', 105 | created_at: new Date().toISOString(), 106 | object: 'file', 107 | }); 108 | 109 | const mockDatasetResponse: DatasetResponse = { 110 | id: 'ds_123', 111 | status: 'running', 112 | domain: 'test-domain', 113 | dataset_name: 'test-dataset', 114 | dataset_type: 'images', 115 | file_id: 'file_123', 116 | created_at: new Date().toISOString(), 117 | }; 118 | 119 | jest.spyOn(datasets['requestor'], 'request').mockResolvedValue([mockDatasetResponse, 200, {}]); 120 | 121 | const result = await datasets.create({ 122 | datasetDirectory: '/path/to/dataset', 123 | domain: 'test-domain', 124 | datasetName: 'test-dataset', 125 | datasetType: 'images', 126 | }); 127 | 128 | expect(result).toEqual(mockDatasetResponse); 129 | }); 130 | 131 | it('should throw error for invalid dataset type', async () => { 132 | await expect(datasets.create({ 133 | datasetDirectory: '/path/to/dataset', 134 | domain: 'test-domain', 135 | datasetName: 'test-dataset', 136 | datasetType: 'invalid' as any, 137 | })).rejects.toThrow('dataset_type must be one of: images, videos, documents'); 138 | }); 139 | }); 140 | 141 | describe('get', () => { 142 | it('should get dataset by ID', async () => { 143 | const mockResponse: DatasetResponse = { 144 | id: 'ds_123', 145 | status: 'completed', 146 | domain: 'test-domain', 147 | dataset_name: 'test-dataset', 148 | dataset_type: 'images', 149 | file_id: 'file_123', 150 | created_at: new Date().toISOString(), 151 | completed_at: new Date().toISOString(), 152 | }; 153 | 154 | jest.spyOn(datasets['requestor'], 'request').mockResolvedValue([mockResponse, 200, {}]); 155 | 156 | const result = await datasets.get('ds_123'); 157 | 158 | expect(result).toEqual(mockResponse); 159 | expect(datasets['requestor'].request).toHaveBeenCalledWith( 160 | 'GET', 161 | 'ds_123' 162 | ); 163 | }); 164 | }); 165 | 166 | describe('list', () => { 167 | it('should list datasets with default pagination', async () => { 168 | const mockResponse: DatasetResponse[] = [ 169 | { 170 | id: 'ds_123', 171 | status: 'completed', 172 | domain: 'test-domain', 173 | dataset_name: 'test-dataset-1', 174 | dataset_type: 'images', 175 | file_id: 'file_123', 176 | created_at: new Date().toISOString(), 177 | }, 178 | { 179 | id: 'ds_456', 180 | status: 'running', 181 | domain: 'test-domain', 182 | dataset_name: 'test-dataset-2', 183 | dataset_type: 'videos', 184 | file_id: 'file_456', 185 | created_at: new Date().toISOString(), 186 | }, 187 | ]; 188 | 189 | jest.spyOn(datasets['requestor'], 'request').mockResolvedValue([mockResponse, 200, {}]); 190 | 191 | const result = await datasets.list(); 192 | 193 | expect(result).toEqual(mockResponse); 194 | expect(datasets['requestor'].request).toHaveBeenCalledWith( 195 | 'GET', 196 | '', 197 | { 198 | skip: 0, 199 | limit: 10, 200 | } 201 | ); 202 | }); 203 | 204 | it('should list datasets with custom pagination', async () => { 205 | const mockResponse: DatasetResponse[] = [ 206 | { 207 | id: 'ds_789', 208 | status: 'completed', 209 | domain: 'test-domain', 210 | dataset_name: 'test-dataset-3', 211 | dataset_type: 'documents', 212 | file_id: 'file_789', 213 | created_at: new Date().toISOString(), 214 | }, 215 | ]; 216 | 217 | jest.spyOn(datasets['requestor'], 'request').mockResolvedValue([mockResponse, 200, {}]); 218 | 219 | const result = await datasets.list({ skip: 2, limit: 1 }); 220 | 221 | expect(result).toEqual(mockResponse); 222 | expect(datasets['requestor'].request).toHaveBeenCalledWith( 223 | 'GET', 224 | '', 225 | { 226 | skip: 2, 227 | limit: 1, 228 | } 229 | ); 230 | }); 231 | 232 | it('should throw error for non-array response', async () => { 233 | jest.spyOn(datasets['requestor'], 'request').mockResolvedValue([{}, 200, {}]); 234 | 235 | await expect(datasets.list()).rejects.toThrow('Expected array response'); 236 | }); 237 | }); 238 | }); 239 | -------------------------------------------------------------------------------- /tests/unit/client/fine_tuning.test.ts: -------------------------------------------------------------------------------- 1 | import { Client } from '../../../src/client/base_requestor'; 2 | import { Finetuning } from '../../../src/client/fine_tuning'; 3 | import { FinetuningResponse, FinetuningProvisionResponse, PredictionResponse } from '../../../src/client/types'; 4 | 5 | jest.mock('../../../src/client/base_requestor'); 6 | 7 | describe('Finetuning', () => { 8 | let client: jest.Mocked; 9 | let finetuning: Finetuning; 10 | 11 | beforeEach(() => { 12 | client = { 13 | apiKey: 'test-api-key', 14 | baseURL: 'https://api.example.com', 15 | } as jest.Mocked; 16 | finetuning = new Finetuning(client); 17 | }); 18 | 19 | describe('create', () => { 20 | it('should create a fine-tuning job with minimal parameters', async () => { 21 | const mockResponse: FinetuningResponse = { 22 | id: 'ft_123', 23 | status: 'running', 24 | model: 'base-model', 25 | created_at: new Date().toISOString(), 26 | training_file_id: 'file_123', 27 | num_epochs: 1, 28 | batch_size: 1, 29 | learning_rate: 2e-4, 30 | }; 31 | jest.spyOn(finetuning['requestor'], 'request').mockResolvedValue([mockResponse, 200, {}]); 32 | 33 | const result = await finetuning.create({ 34 | model: 'base-model', 35 | trainingFile: 'file_123', 36 | }); 37 | 38 | expect(result).toEqual(mockResponse); 39 | expect(finetuning['requestor'].request).toHaveBeenCalledWith( 40 | 'POST', 41 | 'create', 42 | undefined, 43 | { 44 | callback_url: undefined, 45 | model: 'base-model', 46 | training_file: 'file_123', 47 | validation_file: undefined, 48 | num_epochs: 1, 49 | batch_size: 1, 50 | learning_rate: 2e-4, 51 | suffix: undefined, 52 | wandb_api_key: undefined, 53 | wandb_base_url: undefined, 54 | wandb_project_name: undefined, 55 | } 56 | ); 57 | }); 58 | 59 | it('should create a fine-tuning job with all parameters', async () => { 60 | const mockResponse: FinetuningResponse = { 61 | id: 'ft_123', 62 | status: 'running', 63 | model: 'base-model', 64 | created_at: new Date().toISOString(), 65 | training_file_id: 'file_123', 66 | num_epochs: 5, 67 | batch_size: 8, 68 | learning_rate: 1e-4, 69 | }; 70 | jest.spyOn(finetuning['requestor'], 'request').mockResolvedValue([mockResponse, 200, {}]); 71 | 72 | const result = await finetuning.create({ 73 | model: 'base-model', 74 | trainingFile: 'file_123', 75 | validationFile: 'file_456', 76 | numEpochs: 5, 77 | batchSize: 8, 78 | learningRate: 1e-4, 79 | suffix: 'test-model', 80 | wandbApiKey: 'wandb-key', 81 | wandbBaseUrl: 'https://wandb.example.com', 82 | wandbProjectName: 'test-project', 83 | callbackUrl: 'https://callback.example.com', 84 | }); 85 | 86 | expect(result).toEqual(mockResponse); 87 | expect(finetuning['requestor'].request).toHaveBeenCalledWith( 88 | 'POST', 89 | 'create', 90 | undefined, 91 | { 92 | callback_url: 'https://callback.example.com', 93 | model: 'base-model', 94 | training_file: 'file_123', 95 | validation_file: 'file_456', 96 | num_epochs: 5, 97 | batch_size: 8, 98 | learning_rate: 1e-4, 99 | suffix: 'test-model', 100 | wandb_api_key: 'wandb-key', 101 | wandb_base_url: 'https://wandb.example.com', 102 | wandb_project_name: 'test-project', 103 | } 104 | ); 105 | }); 106 | 107 | it('should throw error for invalid suffix', async () => { 108 | await expect(finetuning.create({ 109 | model: 'base-model', 110 | trainingFile: 'file_123', 111 | suffix: 'invalid suffix', 112 | })).rejects.toThrow('Suffix must be alphanumeric, hyphens or underscores without spaces'); 113 | }); 114 | }); 115 | 116 | describe('provision', () => { 117 | it('should provision a model with default parameters', async () => { 118 | const mockResponse: FinetuningProvisionResponse = { 119 | id: 'prov_123', 120 | status: 'running', 121 | model: 'ft_model', 122 | created_at: new Date().toISOString(), 123 | duration: 600, 124 | concurrency: 1, 125 | }; 126 | jest.spyOn(finetuning['requestor'], 'request').mockResolvedValue([mockResponse, 200, {}]); 127 | 128 | const result = await finetuning.provision({ 129 | model: 'ft_model', 130 | }); 131 | 132 | expect(result).toEqual(mockResponse); 133 | expect(finetuning['requestor'].request).toHaveBeenCalledWith( 134 | 'POST', 135 | 'provision', 136 | undefined, 137 | { 138 | model: 'ft_model', 139 | duration: 600, 140 | concurrency: 1, 141 | } 142 | ); 143 | }); 144 | 145 | it('should provision a model with custom parameters', async () => { 146 | const mockResponse: FinetuningProvisionResponse = { 147 | id: 'prov_123', 148 | status: 'running', 149 | model: 'ft_model', 150 | created_at: new Date().toISOString(), 151 | duration: 1200, 152 | concurrency: 2, 153 | }; 154 | jest.spyOn(finetuning['requestor'], 'request').mockResolvedValue([mockResponse, 200, {}]); 155 | 156 | const result = await finetuning.provision({ 157 | model: 'ft_model', 158 | duration: 1200, 159 | concurrency: 2, 160 | }); 161 | 162 | expect(result).toEqual(mockResponse); 163 | expect(finetuning['requestor'].request).toHaveBeenCalledWith( 164 | 'POST', 165 | 'provision', 166 | undefined, 167 | { 168 | model: 'ft_model', 169 | duration: 1200, 170 | concurrency: 2, 171 | } 172 | ); 173 | }); 174 | }); 175 | 176 | describe('generate', () => { 177 | it('should generate a prediction with valid parameters', async () => { 178 | const mockResponse: PredictionResponse = { 179 | id: 'pred_123', 180 | created_at: new Date().toISOString(), 181 | status: 'completed', 182 | response: { 183 | text: 'test prediction' 184 | }, 185 | }; 186 | jest.spyOn(finetuning['requestor'], 'request').mockResolvedValue([mockResponse, 200, {}]); 187 | 188 | const result = await finetuning.generate({ 189 | model: 'ft_model', 190 | prompt: 'test prompt', 191 | jsonSchema: { type: 'string' }, 192 | images: [], 193 | }); 194 | 195 | expect(result).toEqual(mockResponse); 196 | }); 197 | 198 | it('should throw error when JSON schema is missing', async () => { 199 | await expect(finetuning.generate({ 200 | model: 'ft_model', 201 | prompt: 'test prompt', 202 | images: [], 203 | })).rejects.toThrow('JSON schema is required for fine-tuned model predictions'); 204 | }); 205 | 206 | it('should throw error when prompt is missing', async () => { 207 | await expect(finetuning.generate({ 208 | model: 'ft_model', 209 | jsonSchema: { type: 'string' }, 210 | images: [], 211 | })).rejects.toThrow('Prompt is required for fine-tuned model predictions'); 212 | }); 213 | 214 | it('should throw error when domain is provided', async () => { 215 | await expect(finetuning.generate({ 216 | model: 'ft_model', 217 | prompt: 'test prompt', 218 | jsonSchema: { type: 'string' }, 219 | domain: 'test-domain', 220 | images: [], 221 | })).rejects.toThrow('Domain is not supported for fine-tuned model predictions'); 222 | }); 223 | 224 | it('should throw error when detail level is provided', async () => { 225 | await expect(finetuning.generate({ 226 | model: 'ft_model', 227 | prompt: 'test prompt', 228 | jsonSchema: { type: 'string' }, 229 | detail: 'hi', 230 | images: [], 231 | })).rejects.toThrow('Detail level is not supported for fine-tuned model predictions'); 232 | }); 233 | }); 234 | }); 235 | -------------------------------------------------------------------------------- /tests/unit/client/audio-predictions.test.ts: -------------------------------------------------------------------------------- 1 | import { Client } from "../../../src/client/base_requestor"; 2 | import { AudioPredictions } from "../../../src/client/predictions"; 3 | 4 | jest.mock("../../../src/client/base_requestor"); 5 | 6 | describe("AudioPredictions", () => { 7 | let client: jest.Mocked; 8 | let audioPredictions: ReturnType; 9 | let requestMock: jest.SpyInstance; 10 | 11 | beforeEach(() => { 12 | client = { 13 | apiKey: "test-api-key", 14 | baseURL: "https://api.example.com", 15 | } as jest.Mocked; 16 | 17 | audioPredictions = AudioPredictions(client); 18 | requestMock = jest.spyOn(audioPredictions["requestor"], "request"); 19 | }); 20 | 21 | afterEach(() => { 22 | requestMock.mockReset(); 23 | }); 24 | 25 | describe("generate", () => { 26 | it("should generate audio predictions with fileId and default options", async () => { 27 | const mockResponse = { id: "pred_123", status: "completed" }; 28 | requestMock.mockResolvedValue([mockResponse, 200, {}]); 29 | 30 | const result = await audioPredictions.generate({ 31 | fileId: "audio1.mp3", 32 | model: "model1", 33 | domain: "audio.transcription", 34 | }); 35 | 36 | expect(result).toEqual(mockResponse); 37 | expect(requestMock).toHaveBeenCalledWith( 38 | "POST", 39 | "/audio/generate", 40 | undefined, 41 | { 42 | file_id: "audio1.mp3", 43 | model: "model1", 44 | domain: "audio.transcription", 45 | batch: false, 46 | config: { 47 | confidence: false, 48 | detail: "auto", 49 | grounding: false, 50 | gql_stmt: null, 51 | json_schema: undefined, 52 | }, 53 | metadata: { 54 | environment: "dev", 55 | session_id: undefined, 56 | allow_training: true, 57 | }, 58 | callback_url: undefined, 59 | } 60 | ); 61 | }); 62 | 63 | it("should generate audio predictions with fileId and custom options", async () => { 64 | const mockResponse = { id: "pred_456", status: "pending" }; 65 | requestMock.mockResolvedValue([mockResponse, 200, {}]); 66 | 67 | const result = await audioPredictions.generate({ 68 | fileId: "audio2.wav", 69 | model: "whisper-large", 70 | domain: "audio.transcription", 71 | batch: true, 72 | config: { 73 | confidence: true, 74 | detail: "hi", 75 | grounding: true, 76 | }, 77 | metadata: { 78 | environment: "prod", 79 | sessionId: "session_123", 80 | allowTraining: false, 81 | }, 82 | callbackUrl: "https://example.com/callback", 83 | }); 84 | 85 | expect(result).toEqual(mockResponse); 86 | expect(requestMock).toHaveBeenCalledWith( 87 | "POST", 88 | "/audio/generate", 89 | undefined, 90 | { 91 | file_id: "audio2.wav", 92 | model: "whisper-large", 93 | domain: "audio.transcription", 94 | batch: true, 95 | config: { 96 | confidence: true, 97 | detail: "hi", 98 | grounding: true, 99 | gql_stmt: null, 100 | }, 101 | metadata: { 102 | environment: "prod", 103 | session_id: "session_123", 104 | allow_training: false, 105 | }, 106 | callback_url: "https://example.com/callback", 107 | } 108 | ); 109 | }); 110 | 111 | it("should generate audio predictions with URL", async () => { 112 | const mockResponse = { id: "pred_789", status: "completed" }; 113 | requestMock.mockResolvedValue([mockResponse, 200, {}]); 114 | 115 | const result = await audioPredictions.generate({ 116 | url: "https://example.com/audio.mp3", 117 | model: "model1", 118 | domain: "audio.transcription", 119 | }); 120 | 121 | expect(result).toEqual(mockResponse); 122 | expect(requestMock).toHaveBeenCalledWith( 123 | "POST", 124 | "/audio/generate", 125 | undefined, 126 | { 127 | url: "https://example.com/audio.mp3", 128 | model: "model1", 129 | domain: "audio.transcription", 130 | batch: false, 131 | config: { 132 | confidence: false, 133 | detail: "auto", 134 | grounding: false, 135 | gql_stmt: null, 136 | json_schema: undefined, 137 | }, 138 | metadata: { 139 | environment: "dev", 140 | session_id: undefined, 141 | allow_training: true, 142 | }, 143 | callback_url: undefined, 144 | } 145 | ); 146 | }); 147 | 148 | it("should generate audio predictions with URL and custom options", async () => { 149 | const mockResponse = { id: "pred_101112", status: "pending" }; 150 | requestMock.mockResolvedValue([mockResponse, 200, {}]); 151 | 152 | const result = await audioPredictions.generate({ 153 | url: "https://example.com/podcast.mp3", 154 | model: "whisper-large", 155 | domain: "audio.summary", 156 | batch: true, 157 | config: { 158 | confidence: true, 159 | detail: "lo", 160 | grounding: false, 161 | }, 162 | metadata: { 163 | environment: "staging", 164 | sessionId: null, 165 | allowTraining: true, 166 | }, 167 | callbackUrl: "https://webhook.site/callback", 168 | }); 169 | 170 | expect(result).toEqual(mockResponse); 171 | expect(requestMock).toHaveBeenCalledWith( 172 | "POST", 173 | "/audio/generate", 174 | undefined, 175 | { 176 | url: "https://example.com/podcast.mp3", 177 | model: "whisper-large", 178 | domain: "audio.summary", 179 | batch: true, 180 | config: { 181 | confidence: true, 182 | detail: "lo", 183 | grounding: false, 184 | gql_stmt: null, 185 | json_schema: undefined, 186 | }, 187 | metadata: { 188 | environment: "staging", 189 | session_id: null, 190 | allow_training: true, 191 | }, 192 | callback_url: "https://webhook.site/callback", 193 | } 194 | ); 195 | }); 196 | 197 | it("should throw an error when neither fileId nor url are provided", async () => { 198 | await expect( 199 | audioPredictions.generate({ 200 | model: "model1", 201 | domain: "audio.transcription", 202 | }) 203 | ).rejects.toThrow("Either `fileId` or `url` must be provided"); 204 | 205 | expect(requestMock).not.toHaveBeenCalled(); 206 | }); 207 | 208 | it("should throw an error when both fileId and url are provided", async () => { 209 | await expect( 210 | audioPredictions.generate({ 211 | fileId: "audio1.mp3", 212 | url: "https://example.com/audio.mp3", 213 | model: "model1", 214 | domain: "audio.transcription", 215 | }) 216 | ).rejects.toThrow("Only one of `fileId` or `url` can be provided"); 217 | 218 | expect(requestMock).not.toHaveBeenCalled(); 219 | }); 220 | 221 | it("should handle API error responses", async () => { 222 | const errorResponse = { error: "Invalid audio format" }; 223 | requestMock.mockRejectedValue(new Error("API Error")); 224 | 225 | await expect( 226 | audioPredictions.generate({ 227 | fileId: "invalid-audio.txt", 228 | model: "model1", 229 | domain: "audio.transcription", 230 | }) 231 | ).rejects.toThrow("API Error"); 232 | 233 | expect(requestMock).toHaveBeenCalledWith( 234 | "POST", 235 | "/audio/generate", 236 | undefined, 237 | { 238 | file_id: "invalid-audio.txt", 239 | model: "model1", 240 | domain: "audio.transcription", 241 | batch: false, 242 | config: { 243 | confidence: false, 244 | detail: "auto", 245 | grounding: false, 246 | gql_stmt: null, 247 | json_schema: undefined, 248 | }, 249 | metadata: { 250 | environment: "dev", 251 | session_id: undefined, 252 | allow_training: true, 253 | }, 254 | callback_url: undefined, 255 | } 256 | ); 257 | }); 258 | }); 259 | }); 260 | -------------------------------------------------------------------------------- /tests/unit/client/video-predictions.test.ts: -------------------------------------------------------------------------------- 1 | import { Client } from "../../../src/client/base_requestor"; 2 | import { VideoPredictions } from "../../../src/client/predictions"; 3 | 4 | jest.mock("../../../src/client/base_requestor"); 5 | 6 | describe("VideoPredictions", () => { 7 | let client: jest.Mocked; 8 | let videoPredictions: ReturnType; 9 | let requestMock: jest.SpyInstance; 10 | 11 | beforeEach(() => { 12 | client = { 13 | apiKey: "test-api-key", 14 | baseURL: "https://api.example.com", 15 | } as jest.Mocked; 16 | 17 | videoPredictions = VideoPredictions(client); 18 | requestMock = jest.spyOn(videoPredictions["requestor"], "request"); 19 | }); 20 | 21 | afterEach(() => { 22 | requestMock.mockReset(); 23 | }); 24 | 25 | describe("generate", () => { 26 | it("should generate video predictions with fileId and default options", async () => { 27 | const mockResponse = { id: "pred_123", status: "completed" }; 28 | requestMock.mockResolvedValue([mockResponse, 200, {}]); 29 | 30 | const result = await videoPredictions.generate({ 31 | fileId: "video1.mp4", 32 | model: "model1", 33 | domain: "video.transcription", 34 | }); 35 | 36 | expect(result).toEqual(mockResponse); 37 | expect(requestMock).toHaveBeenCalledWith( 38 | "POST", 39 | "/video/generate", 40 | undefined, 41 | { 42 | file_id: "video1.mp4", 43 | model: "model1", 44 | domain: "video.transcription", 45 | batch: false, 46 | config: { 47 | confidence: false, 48 | detail: "auto", 49 | grounding: false, 50 | gql_stmt: null, 51 | json_schema: undefined, 52 | }, 53 | metadata: { 54 | environment: "dev", 55 | session_id: undefined, 56 | allow_training: true, 57 | }, 58 | callback_url: undefined, 59 | } 60 | ); 61 | }); 62 | 63 | it("should generate video predictions with fileId and custom options", async () => { 64 | const mockResponse = { id: "pred_456", status: "pending" }; 65 | requestMock.mockResolvedValue([mockResponse, 200, {}]); 66 | 67 | const result = await videoPredictions.generate({ 68 | fileId: "video2.avi", 69 | model: "video-large", 70 | domain: "video.analysis", 71 | batch: true, 72 | config: { 73 | confidence: true, 74 | detail: "hi", 75 | grounding: true, 76 | }, 77 | metadata: { 78 | environment: "prod", 79 | sessionId: "session_123", 80 | allowTraining: false, 81 | }, 82 | callbackUrl: "https://example.com/callback", 83 | }); 84 | 85 | expect(result).toEqual(mockResponse); 86 | expect(requestMock).toHaveBeenCalledWith( 87 | "POST", 88 | "/video/generate", 89 | undefined, 90 | { 91 | file_id: "video2.avi", 92 | model: "video-large", 93 | domain: "video.analysis", 94 | batch: true, 95 | config: { 96 | confidence: true, 97 | detail: "hi", 98 | grounding: true, 99 | gql_stmt: null, 100 | json_schema: undefined, 101 | }, 102 | metadata: { 103 | environment: "prod", 104 | session_id: "session_123", 105 | allow_training: false, 106 | }, 107 | callback_url: "https://example.com/callback", 108 | } 109 | ); 110 | }); 111 | 112 | it("should generate video predictions with URL", async () => { 113 | const mockResponse = { id: "pred_789", status: "completed" }; 114 | requestMock.mockResolvedValue([mockResponse, 200, {}]); 115 | 116 | const result = await videoPredictions.generate({ 117 | url: "https://example.com/video.mp4", 118 | model: "model1", 119 | domain: "video.transcription", 120 | }); 121 | 122 | expect(result).toEqual(mockResponse); 123 | expect(requestMock).toHaveBeenCalledWith( 124 | "POST", 125 | "/video/generate", 126 | undefined, 127 | { 128 | url: "https://example.com/video.mp4", 129 | model: "model1", 130 | domain: "video.transcription", 131 | batch: false, 132 | config: { 133 | confidence: false, 134 | detail: "auto", 135 | grounding: false, 136 | gql_stmt: null, 137 | json_schema: undefined, 138 | }, 139 | metadata: { 140 | environment: "dev", 141 | session_id: undefined, 142 | allow_training: true, 143 | }, 144 | callback_url: undefined, 145 | } 146 | ); 147 | }); 148 | 149 | it("should generate video predictions with URL and custom options", async () => { 150 | const mockResponse = { id: "pred_101112", status: "pending" }; 151 | requestMock.mockResolvedValue([mockResponse, 200, {}]); 152 | 153 | const result = await videoPredictions.generate({ 154 | url: "https://example.com/lecture.mov", 155 | model: "video-large", 156 | domain: "video.summary", 157 | batch: true, 158 | config: { 159 | confidence: true, 160 | detail: "lo", 161 | grounding: false, 162 | }, 163 | metadata: { 164 | environment: "staging", 165 | sessionId: null, 166 | allowTraining: true, 167 | }, 168 | callbackUrl: "https://webhook.site/callback", 169 | }); 170 | 171 | expect(result).toEqual(mockResponse); 172 | expect(requestMock).toHaveBeenCalledWith( 173 | "POST", 174 | "/video/generate", 175 | undefined, 176 | { 177 | url: "https://example.com/lecture.mov", 178 | model: "video-large", 179 | domain: "video.summary", 180 | batch: true, 181 | config: { 182 | confidence: true, 183 | detail: "lo", 184 | grounding: false, 185 | gql_stmt: null, 186 | json_schema: undefined, 187 | }, 188 | metadata: { 189 | environment: "staging", 190 | session_id: null, 191 | allow_training: true, 192 | }, 193 | callback_url: "https://webhook.site/callback", 194 | } 195 | ); 196 | }); 197 | 198 | it("should throw an error when neither fileId nor url are provided", async () => { 199 | await expect( 200 | videoPredictions.generate({ 201 | model: "model1", 202 | domain: "video.transcription", 203 | }) 204 | ).rejects.toThrow("Either `fileId` or `url` must be provided"); 205 | 206 | expect(requestMock).not.toHaveBeenCalled(); 207 | }); 208 | 209 | it("should throw an error when both fileId and url are provided", async () => { 210 | await expect( 211 | videoPredictions.generate({ 212 | fileId: "video1.mp4", 213 | url: "https://example.com/video.mp4", 214 | model: "model1", 215 | domain: "video.transcription", 216 | }) 217 | ).rejects.toThrow("Only one of `fileId` or `url` can be provided"); 218 | 219 | expect(requestMock).not.toHaveBeenCalled(); 220 | }); 221 | 222 | it("should handle API error responses", async () => { 223 | const errorResponse = { error: "Invalid video format" }; 224 | requestMock.mockRejectedValue(new Error("API Error")); 225 | 226 | await expect( 227 | videoPredictions.generate({ 228 | fileId: "invalid-video.txt", 229 | model: "model1", 230 | domain: "video.transcription", 231 | }) 232 | ).rejects.toThrow("API Error"); 233 | 234 | expect(requestMock).toHaveBeenCalledWith( 235 | "POST", 236 | "/video/generate", 237 | undefined, 238 | { 239 | file_id: "invalid-video.txt", 240 | model: "model1", 241 | domain: "video.transcription", 242 | batch: false, 243 | config: { 244 | confidence: false, 245 | detail: "auto", 246 | grounding: false, 247 | gql_stmt: null, 248 | json_schema: undefined, 249 | }, 250 | metadata: { 251 | environment: "dev", 252 | session_id: undefined, 253 | allow_training: true, 254 | }, 255 | callback_url: undefined, 256 | } 257 | ); 258 | }); 259 | }); 260 | }); 261 | -------------------------------------------------------------------------------- /src/client/exceptions/index.ts: -------------------------------------------------------------------------------- 1 | import { AxiosResponseHeaders, RawAxiosResponseHeaders } from 'axios'; 2 | 3 | /** 4 | * Base exception for all VLM Run errors. 5 | */ 6 | export class VLMRunError extends Error { 7 | constructor(message: string) { 8 | super(message); 9 | this.name = this.constructor.name; 10 | Object.setPrototypeOf(this, VLMRunError.prototype); 11 | } 12 | } 13 | 14 | /** 15 | * Base exception for API errors. 16 | */ 17 | export class APIError extends VLMRunError { 18 | /** 19 | * HTTP status code 20 | */ 21 | http_status?: number; 22 | 23 | /** 24 | * Response headers 25 | */ 26 | headers?: Record; 27 | 28 | /** 29 | * Request ID from the server 30 | */ 31 | request_id?: string; 32 | 33 | /** 34 | * Error type from the server 35 | */ 36 | error_type?: string; 37 | 38 | /** 39 | * Suggestion on how to fix the error 40 | */ 41 | suggestion?: string; 42 | 43 | constructor( 44 | message: string, 45 | http_status?: number, 46 | headers?: Record | RawAxiosResponseHeaders | AxiosResponseHeaders, 47 | request_id?: string, 48 | error_type?: string, 49 | suggestion?: string 50 | ) { 51 | super(message); 52 | this.http_status = http_status; 53 | this.headers = headers ? Object.fromEntries(Object.entries(headers)) : undefined; 54 | this.request_id = request_id; 55 | this.error_type = error_type; 56 | this.suggestion = suggestion; 57 | Object.setPrototypeOf(this, APIError.prototype); 58 | } 59 | 60 | toString(): string { 61 | const parts: string[] = []; 62 | if (this.http_status) { 63 | parts.push(`status=${this.http_status}`); 64 | } 65 | if (this.error_type) { 66 | parts.push(`type=${this.error_type}`); 67 | } 68 | if (this.request_id) { 69 | parts.push(`id=${this.request_id}`); 70 | } 71 | 72 | let formatted = parts.length > 0 ? `[${parts.join(', ')}] ${this.message}` : this.message; 73 | if (this.suggestion) { 74 | formatted += ` (Suggestion: ${this.suggestion})`; 75 | } 76 | return formatted; 77 | } 78 | } 79 | 80 | /** 81 | * Exception raised when authentication fails. 82 | */ 83 | export class AuthenticationError extends APIError { 84 | constructor( 85 | message = "Authentication failed", 86 | http_status = 401, 87 | headers?: Record | RawAxiosResponseHeaders | AxiosResponseHeaders, 88 | request_id?: string, 89 | error_type = "authentication_error", 90 | suggestion = "Check your API key and ensure it is valid" 91 | ) { 92 | super(message, http_status, headers, request_id, error_type, suggestion); 93 | Object.setPrototypeOf(this, AuthenticationError.prototype); 94 | } 95 | } 96 | 97 | /** 98 | * Exception raised when request validation fails. 99 | */ 100 | export class ValidationError extends APIError { 101 | constructor( 102 | message = "Validation failed", 103 | http_status = 400, 104 | headers?: Record | RawAxiosResponseHeaders | AxiosResponseHeaders, 105 | request_id?: string, 106 | error_type = "validation_error", 107 | suggestion = "Check your request parameters" 108 | ) { 109 | super(message, http_status, headers, request_id, error_type, suggestion); 110 | Object.setPrototypeOf(this, ValidationError.prototype); 111 | } 112 | } 113 | 114 | /** 115 | * Exception raised when rate limit is exceeded. 116 | */ 117 | export class RateLimitError extends APIError { 118 | constructor( 119 | message = "Rate limit exceeded", 120 | http_status = 429, 121 | headers?: Record | RawAxiosResponseHeaders | AxiosResponseHeaders, 122 | request_id?: string, 123 | error_type = "rate_limit_error", 124 | suggestion = "Reduce request frequency or contact support to increase your rate limit" 125 | ) { 126 | super(message, http_status, headers, request_id, error_type, suggestion); 127 | Object.setPrototypeOf(this, RateLimitError.prototype); 128 | } 129 | } 130 | 131 | /** 132 | * Exception raised when server returns 5xx error. 133 | */ 134 | export class ServerError extends APIError { 135 | constructor( 136 | message = "Server error", 137 | http_status = 500, 138 | headers?: Record | RawAxiosResponseHeaders | AxiosResponseHeaders, 139 | request_id?: string, 140 | error_type = "server_error", 141 | suggestion = "Please try again later or contact support if the issue persists" 142 | ) { 143 | super(message, http_status, headers, request_id, error_type, suggestion); 144 | Object.setPrototypeOf(this, ServerError.prototype); 145 | } 146 | } 147 | 148 | /** 149 | * Exception raised when resource is not found. 150 | */ 151 | export class ResourceNotFoundError extends APIError { 152 | constructor( 153 | message = "Resource not found", 154 | http_status = 404, 155 | headers?: Record | RawAxiosResponseHeaders | AxiosResponseHeaders, 156 | request_id?: string, 157 | error_type = "not_found_error", 158 | suggestion = "Check the resource ID or path" 159 | ) { 160 | super(message, http_status, headers, request_id, error_type, suggestion); 161 | Object.setPrototypeOf(this, ResourceNotFoundError.prototype); 162 | } 163 | } 164 | 165 | /** 166 | * Base exception for client-side errors. 167 | */ 168 | export class ClientError extends VLMRunError { 169 | /** 170 | * Error type 171 | */ 172 | error_type?: string; 173 | 174 | /** 175 | * Suggestion on how to fix the error 176 | */ 177 | suggestion?: string; 178 | 179 | constructor( 180 | message: string, 181 | error_type?: string, 182 | suggestion?: string 183 | ) { 184 | super(message); 185 | this.error_type = error_type; 186 | this.suggestion = suggestion; 187 | Object.setPrototypeOf(this, ClientError.prototype); 188 | } 189 | 190 | toString(): string { 191 | const parts: string[] = []; 192 | if (this.error_type) { 193 | parts.push(`type=${this.error_type}`); 194 | } 195 | 196 | let formatted = parts.length > 0 ? `[${parts.join(', ')}] ${this.message}` : this.message; 197 | if (this.suggestion) { 198 | formatted += ` (Suggestion: ${this.suggestion})`; 199 | } 200 | return formatted; 201 | } 202 | } 203 | 204 | /** 205 | * Exception raised when client configuration is invalid. 206 | */ 207 | export class ConfigurationError extends ClientError { 208 | constructor( 209 | message = "Invalid configuration", 210 | error_type = "configuration_error", 211 | suggestion = "Check your client configuration" 212 | ) { 213 | super(message, error_type, suggestion); 214 | Object.setPrototypeOf(this, ConfigurationError.prototype); 215 | } 216 | } 217 | 218 | /** 219 | * Exception raised when a required dependency is missing. 220 | */ 221 | export class DependencyError extends ClientError { 222 | constructor( 223 | message = "Missing dependency", 224 | error_type = "dependency_error", 225 | suggestion = "Install the required dependency" 226 | ) { 227 | super(message, error_type, suggestion); 228 | Object.setPrototypeOf(this, DependencyError.prototype); 229 | } 230 | } 231 | 232 | /** 233 | * Exception raised when input is invalid. 234 | */ 235 | export class InputError extends ClientError { 236 | constructor( 237 | message = "Invalid input", 238 | error_type = "input_error", 239 | suggestion = "Check your input parameters" 240 | ) { 241 | super(message, error_type, suggestion); 242 | Object.setPrototypeOf(this, InputError.prototype); 243 | } 244 | } 245 | 246 | /** 247 | * Exception raised when a request times out. 248 | */ 249 | export class RequestTimeoutError extends APIError { 250 | constructor( 251 | message = "Request timed out", 252 | http_status = 408, 253 | headers?: Record | RawAxiosResponseHeaders | AxiosResponseHeaders, 254 | request_id?: string, 255 | error_type = "timeout_error", 256 | suggestion = "Try again later or increase the timeout" 257 | ) { 258 | super(message, http_status, headers, request_id, error_type, suggestion); 259 | Object.setPrototypeOf(this, RequestTimeoutError.prototype); 260 | } 261 | } 262 | 263 | /** 264 | * Exception raised when a network error occurs. 265 | */ 266 | export class NetworkError extends APIError { 267 | constructor( 268 | message = "Network error", 269 | http_status?: number, 270 | headers?: Record | RawAxiosResponseHeaders | AxiosResponseHeaders, 271 | request_id?: string, 272 | error_type = "network_error", 273 | suggestion = "Check your internet connection and try again" 274 | ) { 275 | super(message, http_status, headers, request_id, error_type, suggestion); 276 | Object.setPrototypeOf(this, NetworkError.prototype); 277 | } 278 | } 279 | -------------------------------------------------------------------------------- /tests/integration/client/agent.test.ts: -------------------------------------------------------------------------------- 1 | import { config } from "dotenv"; 2 | config({ path: ".env.test" }); 3 | 4 | import { VlmRun } from "../../../src/index"; 5 | import { AgentCreationConfig } from "../../../src/client/types"; 6 | 7 | jest.setTimeout(60000); 8 | 9 | describe("Integration: Agent", () => { 10 | let client: VlmRun; 11 | const testAgentName = process.env.TEST_AGENT_NAME || "resume-analyzer"; 12 | const testDocumentUrl = 13 | "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/document.invoice/google_invoice.pdf"; 14 | const testImageUrl = 15 | "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/media.tv-news/finance_bb_3_speakers.jpg"; 16 | const faceRedactionPrompt = 17 | "Detect all the faces in the image, and blur all the faces. Return the URL of the blurred image."; 18 | 19 | beforeAll(() => { 20 | client = new VlmRun({ 21 | apiKey: process.env.TEST_API_KEY ?? "", 22 | baseURL: process.env.AGENT_BASE_URL ?? "", 23 | }); 24 | }); 25 | 26 | describe("get()", () => { 27 | it("should get agent by name", async () => { 28 | const result = await client.agent.get({ 29 | name: testAgentName, 30 | }); 31 | 32 | expect(result).toBeTruthy(); 33 | expect(result).toHaveProperty("id"); 34 | expect(result).toHaveProperty("name"); 35 | expect(result).toHaveProperty("status"); 36 | expect(result.name).toBe(testAgentName); 37 | }); 38 | 39 | it("should get agent by ID", async () => { 40 | // First get an agent to obtain its ID 41 | const agentByName = await client.agent.get({ 42 | name: testAgentName, 43 | }); 44 | 45 | // Then get the same agent by ID 46 | const result = await client.agent.get({ 47 | id: agentByName.id, 48 | }); 49 | 50 | expect(result).toBeTruthy(); 51 | expect(result).toHaveProperty("id"); 52 | expect(result).toHaveProperty("name"); 53 | expect(result).toHaveProperty("status"); 54 | expect(result.id).toBe(agentByName.id); 55 | expect(result.name).toBe(agentByName.name); 56 | }); 57 | 58 | it("should throw error when multiple parameters are provided", async () => { 59 | await expect( 60 | client.agent.get({ 61 | id: "some-id", 62 | name: testAgentName, 63 | }) 64 | ).rejects.toThrow("Only one of `id` or `name` or `prompt` can be provided"); 65 | }); 66 | 67 | it("should throw error when no parameters are provided", async () => { 68 | await expect(client.agent.get({})).rejects.toThrow( 69 | "Either `id` or `name` or `prompt` must be provided" 70 | ); 71 | }); 72 | }); 73 | 74 | describe("list()", () => { 75 | it("should list all agents successfully", async () => { 76 | const result = await client.agent.list(); 77 | 78 | expect(result).toBeTruthy(); 79 | expect(Array.isArray(result)).toBe(true); 80 | expect(result.length).toBeGreaterThan(0); 81 | }); 82 | 83 | it("should return agents with required properties", async () => { 84 | const result = await client.agent.list(); 85 | 86 | expect(Array.isArray(result)).toBe(true); 87 | 88 | if (result.length > 0) { 89 | const agent = result[0]; 90 | expect(agent).toHaveProperty("id"); 91 | expect(agent).toHaveProperty("name"); 92 | expect(agent).toHaveProperty("status"); 93 | expect(agent).toHaveProperty("created_at"); 94 | expect(agent).toHaveProperty("updated_at"); 95 | 96 | // Verify property types 97 | expect(typeof agent.id).toBe("string"); 98 | expect(typeof agent.name).toBe("string"); 99 | expect(typeof agent.status).toBe("string"); 100 | expect(typeof agent.created_at).toBe("string"); 101 | expect(typeof agent.updated_at).toBe("string"); 102 | } 103 | }); 104 | 105 | it("should include the test agent in the list", async () => { 106 | const result = await client.agent.list(); 107 | 108 | expect(Array.isArray(result)).toBe(true); 109 | 110 | // Find the test agent in the list 111 | const testAgent = result.find((agent) => agent.name === testAgentName); 112 | expect(testAgent).toBeTruthy(); 113 | 114 | if (testAgent) { 115 | expect(testAgent.name).toBe(testAgentName); 116 | expect(testAgent).toHaveProperty("id"); 117 | expect(testAgent).toHaveProperty("status"); 118 | } 119 | }); 120 | }); 121 | 122 | describe("create()", () => { 123 | const testAgentNameForCreation = `test-face-redaction-${Date.now()}`; 124 | 125 | it("should create agent with config object", async () => { 126 | const result = await client.agent.create({ 127 | name: testAgentNameForCreation, 128 | inputs: { 129 | image: testImageUrl, 130 | }, 131 | config: { 132 | prompt: faceRedactionPrompt, 133 | }, 134 | }); 135 | 136 | expect(result).toBeTruthy(); 137 | expect(result).toHaveProperty("id"); 138 | expect(result).toHaveProperty("name"); 139 | expect(result).toHaveProperty("status"); 140 | expect(result).toHaveProperty("created_at"); 141 | expect(result).toHaveProperty("updated_at"); 142 | expect(result.name).toBe(testAgentNameForCreation); 143 | expect(typeof result.id).toBe("string"); 144 | }); 145 | 146 | it("should create agent with AgentCreationConfig class", async () => { 147 | const config = new AgentCreationConfig({ 148 | prompt: faceRedactionPrompt, 149 | }); 150 | 151 | const result = await client.agent.create({ 152 | name: `${testAgentNameForCreation}-config-class`, 153 | inputs: { 154 | image: testImageUrl, 155 | }, 156 | config: config, 157 | }); 158 | 159 | expect(result).toBeTruthy(); 160 | expect(result).toHaveProperty("id"); 161 | expect(result).toHaveProperty("name"); 162 | expect(result).toHaveProperty("status"); 163 | expect(result.name).toBe(`${testAgentNameForCreation}-config-class`); 164 | }); 165 | 166 | it("should create agent with inputs and callback URL", async () => { 167 | const callbackUrl = "https://webhook.example.com/agent-callback"; 168 | 169 | const result = await client.agent.create({ 170 | name: `${testAgentNameForCreation}-with-callback`, 171 | inputs: { 172 | image: testImageUrl, 173 | additionalParam: "test value", 174 | }, 175 | config: { 176 | prompt: faceRedactionPrompt, 177 | }, 178 | callbackUrl: callbackUrl, 179 | }); 180 | 181 | expect(result).toBeTruthy(); 182 | expect(result).toHaveProperty("id"); 183 | expect(result).toHaveProperty("name"); 184 | expect(result).toHaveProperty("status"); 185 | expect(result.name).toBe(`${testAgentNameForCreation}-with-callback`); 186 | }); 187 | 188 | it("should create agent without specifying name (auto-generated)", async () => { 189 | const result = await client.agent.create({ 190 | inputs: { 191 | image: testImageUrl, 192 | }, 193 | config: { 194 | prompt: faceRedactionPrompt, 195 | }, 196 | }); 197 | 198 | expect(result).toBeTruthy(); 199 | expect(result).toHaveProperty("id"); 200 | expect(result).toHaveProperty("name"); 201 | expect(result).toHaveProperty("status"); 202 | expect(typeof result.name).toBe("string"); 203 | expect(result.name.length).toBeGreaterThan(0); 204 | }); 205 | 206 | it("should throw error when prompt is missing", async () => { 207 | await expect( 208 | client.agent.create({ 209 | name: "test-agent-no-prompt", 210 | inputs: { 211 | image: testImageUrl, 212 | }, 213 | config: { 214 | // No prompt provided 215 | }, 216 | }) 217 | ).rejects.toThrow( 218 | "Prompt is not provided as a request parameter, please provide a prompt" 219 | ); 220 | }); 221 | 222 | it("should create agent and verify it can be retrieved", async () => { 223 | const createResult = await client.agent.create({ 224 | name: `${testAgentNameForCreation}-verify`, 225 | inputs: { 226 | image: testImageUrl, 227 | }, 228 | config: { 229 | prompt: faceRedactionPrompt, 230 | }, 231 | }); 232 | 233 | expect(createResult).toBeTruthy(); 234 | expect(createResult.id).toBeDefined(); 235 | 236 | // Verify the created agent can be retrieved by ID 237 | const getResult = await client.agent.get({ 238 | id: createResult.id, 239 | }); 240 | 241 | expect(getResult).toBeTruthy(); 242 | expect(getResult.id).toBe(createResult.id); 243 | expect(getResult.name).toBe(createResult.name); 244 | }); 245 | }); 246 | }); 247 | -------------------------------------------------------------------------------- /src/client/agent.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * VLM Run API Agent resource. 3 | */ 4 | 5 | import { Client, APIRequestor } from "./base_requestor"; 6 | import { InputError, ServerError, DependencyError } from "./exceptions"; 7 | import { 8 | PredictionResponse, 9 | GenerationConfig, 10 | RequestMetadata, 11 | AgentGetParams, 12 | AgentExecuteParams, 13 | AgentInfo, 14 | AgentExecutionResponse, 15 | AgentCreateParams, 16 | AgentCreationResponse, 17 | AgentExecuteParamsNew, 18 | AgentExecutionConfig, 19 | AgentCreationConfig, 20 | } from "./types"; 21 | 22 | export class Agent { 23 | /** 24 | * Agent resource for VLM Run API. 25 | */ 26 | private client: Client; 27 | private requestor: APIRequestor; 28 | private _completions: any = null; 29 | 30 | constructor(client: Client) { 31 | /** 32 | * Initialize Agent resource with VLMRun instance. 33 | * 34 | * @param client - VLM Run API instance 35 | */ 36 | this.client = client; 37 | this.requestor = new APIRequestor(client); 38 | } 39 | 40 | /** 41 | * OpenAI-compatible chat completions interface. 42 | * 43 | * Returns an OpenAI Completions object configured to use the VLMRun 44 | * agent endpoint. This allows you to use the familiar OpenAI API 45 | * for chat completions. 46 | * 47 | * @example 48 | * ```typescript 49 | * import { VlmRun } from "vlmrun"; 50 | * 51 | * const client = new VlmRun({ 52 | * apiKey: "your-key", 53 | * baseURL: "https://agent.vlm.run/v1" 54 | * }); 55 | * 56 | * const response = await client.agent.completions.create({ 57 | * model: "vlmrun-orion-1", 58 | * messages: [ 59 | * { role: "user", content: "Hello!" } 60 | * ] 61 | * }); 62 | * ``` 63 | * 64 | * @throws {DependencyError} If openai package is not installed 65 | * @returns OpenAI Completions object configured for VLMRun agent endpoint 66 | */ 67 | get completions(): any { 68 | if (this._completions) { 69 | return this._completions; 70 | } 71 | 72 | let OpenAI: any; 73 | try { 74 | // Dynamic import to handle optional dependency 75 | OpenAI = require("openai").default; 76 | } catch (e) { 77 | throw new DependencyError( 78 | "OpenAI SDK is not installed", 79 | "missing_dependency", 80 | "Install it with `npm install openai` or `yarn add openai`" 81 | ); 82 | } 83 | 84 | const baseUrl = `${this.client.baseURL}/openai`; 85 | const openaiClient = new OpenAI({ 86 | apiKey: this.client.apiKey, 87 | baseURL: baseUrl, 88 | timeout: this.client.timeout, 89 | maxRetries: this.client.maxRetries ?? 1, 90 | }); 91 | 92 | this._completions = openaiClient.chat.completions; 93 | return this._completions; 94 | } 95 | 96 | /** 97 | * Get an agent by name, id, or prompt. Only one of `name`, `id`, or `prompt` can be provided. 98 | * 99 | * @param params - Agent lookup parameters 100 | * @returns Agent information 101 | */ 102 | async get(params: { 103 | name?: string; 104 | id?: string; 105 | prompt?: string; 106 | }): Promise { 107 | const { name, id, prompt } = params; 108 | 109 | if (id) { 110 | if (name || prompt) { 111 | throw new InputError("Only one of `id` or `name` or `prompt` can be provided"); 112 | } 113 | } else if (name) { 114 | if (id || prompt) { 115 | throw new InputError("Only one of `id` or `name` or `prompt` can be provided"); 116 | } 117 | } else if (prompt) { 118 | if (id || name) { 119 | throw new InputError("Only one of `id` or `name` or `prompt` can be provided"); 120 | } 121 | } else { 122 | throw new InputError("Either `id` or `name` or `prompt` must be provided"); 123 | } 124 | 125 | const data: Record = {}; 126 | if (id) { 127 | data.id = id; 128 | } else if (name) { 129 | data.name = name; 130 | } else if (prompt) { 131 | data.prompt = prompt; 132 | } 133 | 134 | const [response] = await this.requestor.request( 135 | "POST", 136 | "agent/lookup", 137 | undefined, 138 | data 139 | ); 140 | 141 | if (typeof response !== "object") { 142 | throw new TypeError("Expected object response"); 143 | } 144 | 145 | return response; 146 | } 147 | 148 | /** 149 | * List all agents. 150 | * 151 | * @returns List of agent information 152 | */ 153 | async list(): Promise { 154 | const [response] = await this.requestor.request( 155 | "GET", 156 | "agent" 157 | ); 158 | 159 | if (!Array.isArray(response)) { 160 | throw new TypeError("Expected array response"); 161 | } 162 | 163 | return response; 164 | } 165 | 166 | /** 167 | * Create an agent. 168 | * 169 | * @param params - Agent creation parameters 170 | * @returns Agent creation response 171 | */ 172 | async create(params: AgentCreateParams): Promise { 173 | const { config, name, inputs, callbackUrl } = params; 174 | 175 | const configObj = 176 | config instanceof AgentCreationConfig 177 | ? config 178 | : new AgentCreationConfig(config); 179 | if (!configObj.prompt) { 180 | throw new InputError( 181 | "Prompt is not provided as a request parameter, please provide a prompt" 182 | ); 183 | } 184 | 185 | const data: Record = { 186 | name, 187 | inputs, 188 | config: configObj.toJSON(), 189 | }; 190 | 191 | if (callbackUrl) { 192 | data.callback_url = callbackUrl; 193 | } 194 | 195 | const [response] = await this.requestor.request( 196 | "POST", 197 | "agent/create", 198 | undefined, 199 | data 200 | ); 201 | 202 | if (typeof response !== "object") { 203 | throw new TypeError("Expected object response"); 204 | } 205 | 206 | return response; 207 | } 208 | 209 | /** 210 | * Execute an agent with the given arguments (new method). 211 | * 212 | * @param params - Agent execution parameters 213 | * @returns Agent execution response 214 | */ 215 | async execute( 216 | params: AgentExecuteParamsNew 217 | ): Promise { 218 | const { 219 | name, 220 | inputs, 221 | batch = true, 222 | config, 223 | metadata, 224 | callbackUrl, 225 | } = params; 226 | 227 | if (!batch) { 228 | throw new InputError("Batch mode is required for agent execution"); 229 | } 230 | 231 | const data: Record = { 232 | name, 233 | batch, 234 | inputs, 235 | }; 236 | 237 | if (config) { 238 | const configObj = 239 | config instanceof AgentExecutionConfig 240 | ? config 241 | : new AgentExecutionConfig(config); 242 | data.config = configObj.toJSON(); 243 | } 244 | 245 | if (metadata) { 246 | data.metadata = 247 | metadata instanceof RequestMetadata ? metadata.toJSON() : metadata; 248 | } 249 | 250 | if (callbackUrl) { 251 | data.callback_url = callbackUrl; 252 | } 253 | 254 | const [response] = await this.requestor.request( 255 | "POST", 256 | "agent/execute", 257 | undefined, 258 | data 259 | ); 260 | 261 | if (typeof response !== "object") { 262 | throw new ServerError("Expected object response"); 263 | } 264 | 265 | return response; 266 | } 267 | 268 | /** 269 | * Execute an agent with the given arguments (legacy method for backward compatibility). 270 | * 271 | * @param params - Agent execution parameters 272 | * @returns Agent execution response 273 | * @deprecated Use the new execute method with AgentExecuteParamsNew 274 | */ 275 | async executeLegacy(params: AgentExecuteParams): Promise { 276 | const { 277 | name, 278 | version = "latest", 279 | fileIds, 280 | urls, 281 | batch = true, 282 | config, 283 | metadata, 284 | callbackUrl, 285 | } = params; 286 | 287 | if (!fileIds && !urls) { 288 | throw new InputError("Either `fileIds` or `urls` must be provided"); 289 | } 290 | 291 | if (fileIds && urls) { 292 | throw new InputError("Only one of `fileIds` or `urls` can be provided"); 293 | } 294 | 295 | const data: Record = { 296 | name, 297 | version, 298 | batch, 299 | }; 300 | 301 | if (fileIds) { 302 | data.file_ids = fileIds; 303 | } 304 | 305 | if (urls) { 306 | data.urls = urls; 307 | } 308 | 309 | if (config) { 310 | data.config = 311 | config instanceof GenerationConfig ? config.toJSON() : config; 312 | } 313 | 314 | if (metadata) { 315 | data.metadata = 316 | metadata instanceof RequestMetadata ? metadata.toJSON() : metadata; 317 | } 318 | 319 | if (callbackUrl) { 320 | data.callback_url = callbackUrl; 321 | } 322 | 323 | const [response] = await this.requestor.request( 324 | "POST", 325 | "agent/execute", 326 | undefined, 327 | data 328 | ); 329 | 330 | if (typeof response !== "object") { 331 | throw new ServerError("Expected object response"); 332 | } 333 | 334 | return response; 335 | } 336 | } 337 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 |

3 | VLM Run Logo
4 |

5 |

Node.js SDK

6 |

Website | Platform | Docs | Blog | Discord 7 |

8 |

9 | npm Version 10 | npm Downloads 11 | npm Types
12 | License 13 | Discord 14 | Twitter Follow 15 |

16 |
17 | 18 | The [VLM Run Node.js SDK](https://www.npmjs.com/package/vlmrun) is the official Node.js client for [VLM Run API platform](https://docs.vlm.run), providing a convenient way to interact with our REST APIs. 19 | 20 | ## 🚀 Getting Started 21 | 22 | ### Installation 23 | 24 | ```bash 25 | # Using npm 26 | npm install vlmrun 27 | 28 | # Using yarn 29 | yarn add vlmrun 30 | 31 | # Using pnpm 32 | pnpm add vlmrun 33 | ``` 34 | 35 | ### Basic Usage 36 | 37 | ### Image Predictions 38 | 39 | ```typescript 40 | import { VlmRun } from "vlmrun"; 41 | 42 | // Initialize the client 43 | const client = new VlmRun({ 44 | apiKey: "your-api-key", 45 | }); 46 | 47 | // Process an image (using image url) 48 | const imageUrl = 49 | "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/document.invoice/invoice_1.jpg"; 50 | const response = await client.image.generate({ 51 | images: [imageUrl], 52 | domain: "document.invoice", 53 | config: { 54 | jsonSchema: { 55 | type: "object", 56 | properties: { 57 | invoice_number: { type: "string" }, 58 | total_amount: { type: "number" }, 59 | }, 60 | }, 61 | }, 62 | }); 63 | console.log(response); 64 | 65 | // Process an image passing zod schema 66 | import { z } from "zod"; 67 | 68 | const imageUrl = 69 | "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/document.invoice/invoice_1.jpg"; 70 | 71 | const schema = z.object({ 72 | invoice_number: z.string(), 73 | total_amount: z.number(), 74 | }); 75 | 76 | const response = await client.image.generate({ 77 | images: [imageUrl], 78 | domain: "document.invoice", 79 | config: { 80 | responseModel: schema, 81 | }, 82 | }); 83 | const response = response.response as z.infer; 84 | console.log(response); 85 | 86 | // Process an image (using local file path) 87 | const response = await client.image.generate({ 88 | images: ["tests/integration/assets/invoice.jpg"], 89 | model: "vlm-1", 90 | domain: "document.invoice", 91 | }); 92 | console.log(response); 93 | ``` 94 | 95 | ### Document Predictions 96 | 97 | ```typescript 98 | import { VlmRun } from "vlmrun"; 99 | 100 | // Initialize the client 101 | const client = new VlmRun({ 102 | apiKey: "your-api-key", 103 | }); 104 | 105 | // Upload a document 106 | const file = await client.files.upload({ 107 | filePath: "path/to/invoice.pdf", 108 | }); 109 | 110 | // Process a document (using file id) 111 | const response = await client.document.generate({ 112 | fileId: file.id, 113 | model: "vlm-1", 114 | domain: "document.invoice", 115 | }); 116 | console.log(response); 117 | 118 | // Process a document (using url) 119 | const documentUrl = 120 | "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/document.invoice/google_invoice.pdf"; 121 | const response = await client.document.generate({ 122 | url: documentUrl, 123 | model: "vlm-1", 124 | domain: "document.invoice", 125 | }); 126 | console.log(response); 127 | 128 | // Process a document passing zod schema 129 | import { z } from "zod"; 130 | 131 | const schema = z.object({ 132 | invoice_id: z.string(), 133 | total: z.number(), 134 | sub_total: z.number(), 135 | tax: z.number(), 136 | items: z.array( 137 | z.object({ 138 | name: z.string(), 139 | quantity: z.number(), 140 | price: z.number(), 141 | total: z.number(), 142 | }) 143 | ), 144 | }); 145 | 146 | const response = await client.document.generate({ 147 | url: documentUrl, 148 | domain: "document.invoice", 149 | config: { responseModel: schema }, 150 | }); 151 | 152 | const response = response.response as z.infer; 153 | console.log(response); 154 | ``` 155 | 156 | ### Using Callback URLs for Async Processing 157 | 158 | VLM Run supports callback URLs for asynchronous processing. When you provide a callback URL, the API will send a webhook notification to your endpoint when the prediction is complete. 159 | 160 | ```typescript 161 | import { VlmRun } from "vlmrun"; 162 | 163 | // Initialize the client 164 | const client = new VlmRun({ 165 | apiKey: "your-api-key", 166 | }); 167 | 168 | // Process a document with callback URL 169 | const url = 170 | "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/document.invoice/google_invoice.pdf"; 171 | const response = await client.document.generate({ 172 | url: url, 173 | domain: "document.invoice", 174 | batch: true, // Enable batch processing for async execution 175 | callbackUrl: "https://your-webhook-endpoint.com/vlm-callback", 176 | }); 177 | 178 | console.log(response.status); // "pending" 179 | console.log(response.id); // Use this ID to track the prediction 180 | ``` 181 | 182 | #### Webhook Payload 183 | 184 | When the prediction is complete, VLM Run will send a POST request to your callback URL with the following payload: 185 | 186 | ```json 187 | { 188 | "id": "pred_abc123", 189 | "status": "completed", 190 | "response": { 191 | "invoice_id": "INV-001", 192 | "total": 1250.0, 193 | "items": [] 194 | }, 195 | "created_at": "2024-01-15T10:30:00Z", 196 | "completed_at": "2024-01-15T10:30:45Z" 197 | } 198 | ``` 199 | 200 | ### Document Predictions with Zod Definitions 201 | 202 | ```typescript 203 | import { VlmRun } from "vlmrun"; 204 | import { z } from "zod"; 205 | // Initialize the client 206 | const client = new VlmRun({ 207 | apiKey: "your-api-key", 208 | }); 209 | 210 | // Define enums and base schemas 211 | enum PaymentStatus { 212 | PAID = "Paid", 213 | UNPAID = "Unpaid", 214 | PARTIAL = "Partial", 215 | OVERDUE = "Overdue", 216 | } 217 | 218 | enum PaymentMethod { 219 | CREDIT_CARD = "Credit Card", 220 | BANK_TRANSFER = "Bank Transfer", 221 | CHECK = "Check", 222 | CASH = "Cash", 223 | PAYPAL = "PayPal", 224 | OTHER = "Other", 225 | } 226 | 227 | const currencySchema = z 228 | .number() 229 | .min(0, "Currency values must be non-negative"); 230 | 231 | const dateSchema = z 232 | .string() 233 | .regex(/^\d{4}-\d{2}-\d{2}$/, "Date must be in YYYY-MM-DD format"); 234 | 235 | // Define address schema 236 | const addressSchema = z.object({ 237 | street: z.string().nullable(), 238 | city: z.string().nullable(), 239 | state: z.string().nullable(), 240 | postal_code: z.string().nullable(), 241 | country: z.string().nullable(), 242 | }); 243 | 244 | // Define line item schema 245 | const lineItemSchema = z.object({ 246 | description: z.string(), 247 | quantity: z.number().positive(), 248 | unit_price: currencySchema, 249 | total: currencySchema, 250 | }); 251 | 252 | // Define company schema 253 | const companySchema = z.object({ 254 | name: z.string(), 255 | address: addressSchema.nullable(), 256 | tax_id: z.string().nullable(), 257 | phone: z.string().nullable(), 258 | email: z.string().nullable(), 259 | website: z.string().nullable(), 260 | }); 261 | 262 | // Define invoice schema using the definitions 263 | const invoiceSchema = z.object({ 264 | invoice_id: z.string(), 265 | invoice_date: dateSchema, 266 | due_date: dateSchema.nullable(), 267 | vendor: companySchema, 268 | customer: companySchema, 269 | items: z.array(lineItemSchema), 270 | subtotal: currencySchema, 271 | tax: currencySchema.nullable(), 272 | total: currencySchema, 273 | payment_status: z.nativeEnum(PaymentStatus).nullable(), 274 | payment_method: z.nativeEnum(PaymentMethod).nullable(), 275 | notes: z.string().nullable(), 276 | }); 277 | 278 | const documentUrl = 279 | "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/document.invoice/google_invoice.pdf"; 280 | 281 | const result = await client.document.generate({ 282 | url: documentUrl, 283 | domain: "document.invoice", 284 | config: { 285 | responseModel: invoiceSchema, 286 | zodToJsonParams: { 287 | definitions: { 288 | address: addressSchema, 289 | lineItem: lineItemSchema, 290 | company: companySchema, 291 | }, 292 | $refStrategy: "none", 293 | }, 294 | }, 295 | }); 296 | ``` 297 | 298 | ### OpenAI-Compatible Chat Completions 299 | 300 | The VLM Run SDK provides OpenAI-compatible chat completions through the agent endpoint. This allows you to use the familiar OpenAI API with VLM Run's powerful vision-language models. 301 | 302 | ```typescript 303 | import { VlmRun } from "vlmrun"; 304 | 305 | // Initialize the client with the agent endpoint 306 | const client = new VlmRun({ 307 | apiKey: "your-api-key", 308 | baseURL: "https://agent.vlm.run/v1", 309 | }); 310 | 311 | // Use OpenAI-compatible chat completions 312 | const response = await client.agent.completions.create({ 313 | model: "vlmrun-orion-1", 314 | messages: [{ role: "user", content: "Hello! How can you help me today?" }], 315 | }); 316 | 317 | console.log(response.choices[0].message.content); 318 | ``` 319 | 320 | #### Streaming Responses 321 | 322 | ```typescript 323 | const stream = await client.agent.completions.create({ 324 | model: "vlmrun-orion-1", 325 | messages: [{ role: "user", content: "Tell me a story" }], 326 | stream: true, 327 | }); 328 | 329 | for await (const chunk of stream) { 330 | process.stdout.write(chunk.choices[0]?.delta?.content || ""); 331 | } 332 | ``` 333 | 334 | **Note:** The OpenAI SDK is an optional peer dependency. Install it with: 335 | 336 | ```bash 337 | npm install openai 338 | # or 339 | yarn add openai 340 | ``` 341 | 342 | ## 🛠️ Examples 343 | 344 | Check out the [examples](./examples) directory for more detailed usage examples: 345 | 346 | - [Models](./examples/models.ts) - List available models 347 | - [Files](./examples/files.ts) - Upload and manage files 348 | - [Predictions](./examples/predictions.ts) - Make predictions with different types of inputs 349 | - [Feedback](./examples/feedback.ts) - Submit feedback for predictions 350 | 351 | ## 🔑 Authentication 352 | 353 | To use the VLM Run API, you'll need an API key. You can obtain one by: 354 | 355 | 1. Create an account at [VLM Run](https://app.vlm.run) 356 | 2. Navigate to dashboard Settings -> API Keys 357 | 358 | Then use it to initialize the client: 359 | 360 | ```typescript 361 | const client = new VlmRun({ 362 | apiKey: "your-api-key", 363 | }); 364 | ``` 365 | 366 | ## 📚 Documentation 367 | 368 | For detailed documentation and API reference, visit our [documentation site](https://docs.vlm.run). 369 | 370 | ## 🤝 Contributing 371 | 372 | We welcome contributions! Please check out our [contributing guidelines](docs/CONTRIBUTING.md) for details. 373 | 374 | ## 📝 License 375 | 376 | This project is licensed under the Apache-2.0 License - see the [LICENSE](LICENSE) file for details. 377 | --------------------------------------------------------------------------------