├── .eslintrc ├── .github └── workflows │ └── main.yml ├── .gitignore ├── license ├── package.json ├── readme.md ├── src ├── fetch-api.ts ├── index.ts ├── pinecone-client.ts ├── types.ts └── utils.ts ├── test ├── e2e-tests.ts └── run-tests.ts ├── tsconfig.dist.json ├── tsconfig.json └── yarn.lock /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "root": true, 3 | "extends": ["hckrs"] 4 | } 5 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - master 10 | 11 | jobs: 12 | lint: 13 | name: Lint 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v3 17 | - uses: actions/setup-node@v3 18 | with: 19 | node-version: 18 20 | cache: 'yarn' 21 | - run: yarn install --frozen-lockfile 22 | - run: yarn lint 23 | types: 24 | name: Types 25 | runs-on: ubuntu-latest 26 | steps: 27 | - uses: actions/checkout@v3 28 | - uses: actions/setup-node@v3 29 | with: 30 | node-version: 18 31 | cache: 'yarn' 32 | - run: yarn install --frozen-lockfile 33 | - run: yarn typecheck 34 | e2e: 35 | name: E2E Tests 36 | runs-on: ubuntu-latest 37 | steps: 38 | - uses: actions/checkout@v3 39 | - uses: actions/setup-node@v3 40 | with: 41 | node-version: 18.18.1 42 | cache: 'yarn' 43 | - run: yarn install --frozen-lockfile 44 | - run: yarn test:e2e 45 | env: 46 | PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }} 47 | PINECONE_BASE_URL: ${{ secrets.PINECONE_BASE_URL }} 48 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | .env 4 | -------------------------------------------------------------------------------- /license: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Riley Tomasek 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pinecone-client", 3 | "version": "2.0.0", 4 | "description": "Pinecone.io client powered by fetch", 5 | "type": "module", 6 | "exports": { 7 | ".": { 8 | "types": "./dist/index.d.ts", 9 | "import": "./dist/index.js" 10 | }, 11 | "./package.json": "./package.json" 12 | }, 13 | "types": "dist/index.d.ts", 14 | "main": "dist/index.js", 15 | "files": [ 16 | "dist" 17 | ], 18 | "repository": "dexaai/pinecone-client", 19 | "author": { 20 | "name": "Riley Tomasek", 21 | "email": "hi@rile.yt", 22 | "url": "https://rile.yt" 23 | }, 24 | "scripts": { 25 | "build": "rm -rf dist && tsc --project tsconfig.dist.json", 26 | "clean": "rm -rf dist node_modules", 27 | "format": "prettier --write src test", 28 | "lint": "eslint src", 29 | "prepare": "yarn build", 30 | "prepublishOnly": "yarn lint && yarn typecheck", 31 | "release": "np", 32 | "test:e2e": "tsx ./test/run-tests.ts", 33 | "typecheck": "tsc" 34 | }, 35 | "license": "MIT", 36 | "private": false, 37 | "dependencies": { 38 | "ky": "^1.1.0" 39 | }, 40 | "devDependencies": { 41 | "@types/node": "^20.8.7", 42 | "eslint": "^8.52.0", 43 | "eslint-config-hckrs": "^0.0.4", 44 | "np": "^8.0.4", 45 | "prettier": "^3.0.3", 46 | "tsx": "^3.14.0", 47 | "type-fest": "^4.5.0", 48 | "typescript": "^5.2.2" 49 | }, 50 | "publishConfig": { 51 | "registry": "https://registry.npmjs.org" 52 | }, 53 | "prettier": { 54 | "singleQuote": true 55 | }, 56 | "engines": { 57 | "node": ">=18" 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Unofficial Pinecone.io Client 2 | 3 | [![Build Status](https://github.com/rileytomasek/pinecone-client/actions/workflows/main.yml/badge.svg)](https://github.com/rileytomasek/pinecone-client/actions/workflows/main.yml) [![npm version](https://img.shields.io/npm/v/pinecone-client.svg?color=0c0)](https://www.npmjs.com/package/pinecone-client) 4 | 5 | An unofficial fetch based client for the [Pinecone.io](https://www.pinecone.io/) vector database with excellent TypeScript support. 6 | 7 | Pinecone recently released a [similar client](https://github.com/pinecone-io/pinecone-ts-client). It's a great option if you aren't picky about fully typed metadata. 8 | 9 | ## Highlights 10 | 11 | - Support for all vector operation endpoints 12 | - Fully typed metadata with TypeScript generics 13 | - Automatically remove null metadata values (Pinecone doesn't nulls) 14 | - Supports modern fetch based runtimes (Cloudlflare workers, Deno, etc) 15 | - In-editor documentation with IntelliSense/TS server 16 | - Tiny package size. [Less than 5kb gzipped](https://bundlephobia.com/package/pinecone-client) 17 | - Full [e2e test coverage](/test) 18 | 19 | ## Example Usage 20 | 21 | ```ts 22 | import { PineconeClient } from 'pinecone-client'; 23 | 24 | // Specify the type of your metadata 25 | type Metadata = { size: number, tags?: string[] | null }; 26 | 27 | // Instantiate a client 28 | const pinecone = new PineconeClient({ namespace: 'test' }); 29 | 30 | // Upsert vectors with metadata. 31 | await pinecone.upsert({ 32 | vectors: [ 33 | { id: '1', values: [1, 2, 3], metadata: { size: 3, tags: ['a', 'b', 'c'] } }, 34 | { id: '2', values: [4, 5, 6], metadata: { size: 10, tags: null } }, 35 | ], 36 | }); 37 | 38 | // Query vectors with metadata filters. 39 | const { matches } = await pinecone.query({ 40 | topK: 2, 41 | id: '2', 42 | filter: { size: { $lt: 20 } }, 43 | includeMetadata: true, 44 | }); 45 | 46 | // typeof matches = { 47 | // id: string; 48 | // score: number; 49 | // metadata: Metadata; 50 | // }[]; 51 | ``` 52 | 53 | ## Install 54 | 55 | **Warning:** This package is native [ESM](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Modules) and no longer provides a CommonJS export. If your project uses CommonJS, you will have to [convert to ESM](https://gist.github.com/sindresorhus/a39789f98801d908bbc7ff3ecc99d99c) or use the [dynamic `import()`](https://v8.dev/features/dynamic-import) function. Please don't open issues for questions regarding CommonJS / ESM. 56 | 57 | **Runtimes** 58 | - Supported: Deno, Node v18+, Cloudflare Workers, browsers 59 | - Unsupported: Anything without a native fetch implementation (Node({ 82 | apiKey: '', 83 | baseUrl: '', 84 | namespace: 'testing', 85 | }); 86 | ``` 87 | 88 | Both apiKey and baseUrl are optional and will be read from the following environment variables: 89 | - `process.env.PINECONE_API_KEY` 90 | - `process.env.PINECONE_BASE_URL` 91 | 92 | ## API 93 | 94 | The client supports all of the vector operations from the Pinecone API using the same method names and parameters. It also supports creating and deleting indexes. 95 | 96 | For detailed documentation with links to the Pinecone docs, see [the source code](/src/pinecone-client.ts). 97 | 98 | ### Supported methods: 99 | 100 | - `pinecone.delete()` 101 | - `pinecone.describeIndexStats()` 102 | - `pinecone.fetch()` 103 | - `pinecone.query()` 104 | - `pinecone.update()` 105 | - `pinecone.upsert()` 106 | - `pinecone.createIndex()` 107 | - `pinecone.deleteIndex()` 108 | 109 | You can also find more example usage in the [e2e tests](/test/e2e-tests.ts). 110 | -------------------------------------------------------------------------------- /src/fetch-api.ts: -------------------------------------------------------------------------------- 1 | import ky from 'ky'; 2 | import type { Options } from 'ky'; 3 | 4 | type KyInstance = ReturnType; 5 | export interface FetchOptions extends Options { 6 | credentials?: 'include' | 'omit' | 'same-origin'; 7 | } 8 | 9 | /** 10 | * Create an instance of Ky with options shared by all requests. 11 | */ 12 | export function createApiInstance(opts: { 13 | apiKey: string; 14 | baseUrl: string; 15 | fetchOptions?: FetchOptions; 16 | }): KyInstance { 17 | const { headers, ...restFetchOptions } = opts.fetchOptions || {}; 18 | return ky.extend({ 19 | prefixUrl: opts.baseUrl, 20 | timeout: 30 * 1000, 21 | headers: { 22 | 'Api-Key': opts.apiKey, 23 | ...headers, 24 | }, 25 | hooks: { 26 | beforeError: [ 27 | // @ts-ignore 28 | async (error) => { 29 | const { response } = error; 30 | if (response && response.body) { 31 | try { 32 | const body = await response.clone().json(); 33 | if (body.message) { 34 | return new PineconeError(body.message, { 35 | code: body.code, 36 | details: body.details, 37 | status: response.status, 38 | cause: error, 39 | }); 40 | } 41 | } catch (e) { 42 | console.error('Failed reading HTTPError response body', e); 43 | } 44 | } 45 | return error; 46 | }, 47 | ], 48 | }, 49 | ...restFetchOptions, 50 | }); 51 | } 52 | 53 | type PineconeErrorDetail = { typeUrl: string; value: string }; 54 | 55 | export class PineconeError extends Error { 56 | public code: number; 57 | public details?: PineconeErrorDetail[]; 58 | public status: number; 59 | 60 | constructor( 61 | message: string, 62 | opts: { 63 | cause?: Error; 64 | code: number; 65 | details?: PineconeErrorDetail[]; 66 | status: number; 67 | }, 68 | ) { 69 | if (opts.cause) { 70 | // @ts-ignore not sure why TS can't handle this 71 | super(message, { cause: opts.cause }); 72 | } else { 73 | super(message); 74 | } 75 | 76 | // Ensure the name of this error is the same as the class name 77 | this.name = this.constructor.name; 78 | 79 | // Set stack trace to caller 80 | if (Error.captureStackTrace) { 81 | Error.captureStackTrace(this, this.constructor); 82 | } 83 | 84 | this.code = opts.code; 85 | this.status = opts.status; 86 | 87 | if (opts.details) { 88 | this.details = opts.details; 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * This version of the client is for environments that support fetch natively. 3 | * See /fetch-polyfill for a version that works in environments that don't. 4 | */ 5 | export { PineconeClient } from './pinecone-client.js'; 6 | 7 | export type { Filter, Vector, QueryParams, QueryResults } from './types.js'; 8 | -------------------------------------------------------------------------------- /src/pinecone-client.ts: -------------------------------------------------------------------------------- 1 | import type { FetchOptions } from './fetch-api.js'; 2 | import { createApiInstance } from './fetch-api.js'; 3 | import { removeNullValues } from './utils.js'; 4 | import type { 5 | RootMetadata, 6 | QueryParams, 7 | Filter, 8 | Vector, 9 | QueryResults, 10 | SparseValues, 11 | } from './types.js'; 12 | import type { JsonObject, SetRequired } from 'type-fest'; 13 | 14 | type ConfigOpts = { 15 | /** 16 | * The API key used to authenticate with the Pinecone API. 17 | * Get yours from the Pinecone console: https://app.pinecone.io/ 18 | */ 19 | apiKey?: string; 20 | /** 21 | * The HTTP endpoint for the Pinecone index. 22 | * Use an empty string if there is no baseUrl yet because the index is being created. 23 | * @see https://www.pinecone.io/docs/manage-data/#specify-an-index-endpoint 24 | */ 25 | baseUrl?: string; 26 | /** 27 | * The index namespace to use for all requests. This can't be changed after 28 | * the client is created to ensure metadata type safety. 29 | * @see https://www.pinecone.io/docs/namespaces/ 30 | */ 31 | namespace?: string; 32 | /** 33 | * Fetch options that will be added to all requests (like credentials, etc.). 34 | */ 35 | fetchOptions?: FetchOptions; 36 | }; 37 | 38 | /** 39 | * PineconeClient class used to interact with a Pinecone index. 40 | */ 41 | export class PineconeClient { 42 | api: ReturnType; 43 | apiKey: string; 44 | baseUrl: string; 45 | namespace?: string; 46 | 47 | constructor(config: ConfigOpts) { 48 | const process = globalThis.process || { env: {} }; 49 | const apiKey = config.apiKey || process.env['PINECONE_API_KEY']; 50 | const baseUrl = config.baseUrl || process.env['PINECONE_BASE_URL']; 51 | if (!apiKey) { 52 | throw new Error( 53 | 'Missing Pinecone API key. Please provide one in the config or set the PINECONE_API_KEY environment variable.', 54 | ); 55 | } 56 | if (!baseUrl) { 57 | throw new Error( 58 | 'Missing Pinecone base URL. Please provide one in the config or set the PINECONE_BASE_URL environment variable.', 59 | ); 60 | } 61 | this.apiKey = apiKey; 62 | this.baseUrl = baseUrl; 63 | this.namespace = config.namespace; 64 | this.api = createApiInstance({ 65 | apiKey: this.apiKey, 66 | baseUrl: this.baseUrl, 67 | fetchOptions: config.fetchOptions, 68 | }); 69 | } 70 | 71 | /** 72 | * The Delete operation deletes vectors, by id, from a single namespace. You 73 | * can delete items by their id, from a single namespace. 74 | * @param params.ids The ids of the vectors to delete. 75 | * @param params.deleteAll Deletes all vectors in the index if true. 76 | * @param params.filter Metadata filter to apply to the delete. 77 | * @see https://docs.pinecone.io/reference/delete/ 78 | */ 79 | async delete(params: { 80 | ids?: string[]; 81 | deleteAll?: boolean; 82 | filter?: Filter; 83 | }): Promise { 84 | return this.api 85 | .post('vectors/delete', { 86 | json: { 87 | namespace: this.namespace, 88 | ...removeNullValues(params), 89 | }, 90 | }) 91 | .json(); 92 | } 93 | 94 | /** 95 | * The DescribeIndexStats operation returns statistics about the index's 96 | * contents, including the vector count per namespace, the number of 97 | * dimensions, and the index fullness. 98 | * @params params.filter Metadata filter to apply to the describe. 99 | * @see https://docs.pinecone.io/reference/describe_index_stats_post 100 | */ 101 | async describeIndexStats(params?: { filter?: Filter }): Promise<{ 102 | namespaces: { [namespace: string]: { vectorCount: number } }; 103 | dimension: number; 104 | indexFullness: number; 105 | totalVectorCount: number; 106 | }> { 107 | return this.api 108 | .post('describe_index_stats', { 109 | json: removeNullValues(params), 110 | }) 111 | .json(); 112 | } 113 | 114 | /** 115 | * The Fetch operation looks up and returns vectors, by ID, from a single 116 | * namespace. The returned vectors include the vector data and/or metadata. 117 | * @param params.ids The ids of the vectors to fetch. 118 | * @see https://docs.pinecone.io/reference/fetch 119 | */ 120 | async fetch(params: { ids: string[] }): Promise<{ 121 | namespace: string; 122 | vectors: { [vectorId: string]: Vector }; 123 | }> { 124 | const searchParams: string[][] = []; 125 | if (this.namespace) searchParams.push(['namespace', this.namespace]); 126 | params.ids.forEach((id) => searchParams.push(['ids', id])); 127 | return this.api.get('vectors/fetch', { searchParams }).json(); 128 | } 129 | 130 | /** 131 | * The Query operation searches a namespace, using a query vector. It 132 | * retrieves the ids of the most similar items in a namespace, along with 133 | * their similarity scores. 134 | * @param params.topK The number of results to return. 135 | * @param params.minScore Filter out results with a score below this value. 136 | * @param params.filter Metadata filter to apply to the query. 137 | * @param params.id The id of the vector in the index to be used as the query vector. 138 | * @param params.vector A dense vector to be used as the query vector. 139 | * @param params.sparseVector A sparse vector to be used as the query vector. 140 | * @param params.hybridAlpha Dense vs sparse weighting. 0.0 is all sparse, 1.0 is all dense. 141 | * @param params.includeMetadata Whether to include metadata in the results. 142 | * @param params.includeValues Whether to include vector values in the results. 143 | * @note One of `vector` or `id` is required. 144 | * @see https://docs.pinecone.io/reference/query 145 | */ 146 | async query>( 147 | params: Params, 148 | ): Promise> { 149 | const { hybridAlpha, minScore, ...restParams } = params; 150 | // Apply hybrid scoring if requested. 151 | if (hybridAlpha != undefined) { 152 | const { vector, sparseVector } = params; 153 | if (!vector || !sparseVector) { 154 | throw new Error( 155 | `Hybrid queries require vector and sparseVector parameters.`, 156 | ); 157 | } 158 | const weighted = hybridScoreNorm(vector, sparseVector, hybridAlpha); 159 | restParams.vector = weighted.values; 160 | restParams.sparseVector = weighted.sparseValues; 161 | } 162 | const results: QueryResults = await this.api 163 | .post('query', { 164 | json: { 165 | namespace: this.namespace, 166 | ...removeNullValues(restParams), 167 | }, 168 | }) 169 | .json(); 170 | // Filter out results below the minimum score. 171 | if (typeof minScore === 'number') { 172 | results.matches = results.matches.filter((r) => r.score >= minScore); 173 | } 174 | return results; 175 | } 176 | 177 | /** 178 | * The Update operation updates vector in a namespace. If a value is 179 | * included, it will overwrite the previous value. If a set_metadata 180 | * is included, the values of the fields specified in it will be added 181 | * or overwrite the previous value. 182 | * @param params.id The id of the vector to update. 183 | * @param params.values The new dense vector values. 184 | * @param params.sparseValues The new sparse vector values. 185 | * @param params.setMetadata Metadata to set for the vector. 186 | * @see https://docs.pinecone.io/reference/update 187 | */ 188 | async update(params: { 189 | id: string; 190 | values?: number[]; 191 | sparseValues?: SparseValues; 192 | setMetadata?: Metadata; 193 | }): Promise { 194 | return this.api 195 | .post('vectors/update', { 196 | json: { 197 | namespace: this.namespace, 198 | ...removeNullValues(params), 199 | }, 200 | }) 201 | .json(); 202 | } 203 | 204 | /** 205 | * The Upsert operation writes vectors into a namespace. If a new value is 206 | * upserted for an existing vector id, it will overwrite the previous value. 207 | * @param params.vectors The vectors to upsert. 208 | * @param params.batchSize The number of vectors to upsert in each batch. 209 | * @note This will automatically chunk the requests into batches of 1000 vectors. 210 | * @see https://docs.pinecone.io/reference/upsert 211 | */ 212 | async upsert(params: { 213 | vectors: SetRequired, 'metadata'>[]; 214 | batchSize?: number; 215 | }): Promise { 216 | // Don't upsert more than `params.batchSize` vectors in a single request 217 | const batchSize = params.batchSize || 50; 218 | for (let i = 0; i < params.vectors.length; i += batchSize) { 219 | const vectors = params.vectors.slice(i, i + batchSize); 220 | const vectorsWithoutMetadataNulls = vectors.map(removeNullValues); 221 | await this.api 222 | .post('vectors/upsert', { 223 | json: { 224 | namespace: this.namespace, 225 | vectors: vectorsWithoutMetadataNulls, 226 | }, 227 | }) 228 | .json(); 229 | } 230 | } 231 | 232 | /** 233 | * This operation creates a Pinecone index. You can use it to specify the measure of similarity, the dimension of vectors to be stored in the index, the numbers of shards and replicas to use, and more. 234 | * @param params.environment The environment to create the index in. Eg: us-east-1-aws or us-west1-gcp 235 | * @param params.name The name of the index to be created. The maximum length is 45 characters. 236 | * @param params.dimension The dimensions of the vectors to be inserted in the index 237 | * @param params.metric The distance metric to be used for similarity search. You can use 'euclidean', 'cosine', or 'dotproduct'. 238 | * @param params.pods The number of pods for the index to use,including replicas. 239 | * @param params.replicas The number of replicas. Replicas duplicate your index. They provide higher availability and throughput. 240 | * @param params.shards The number of shards to be used in the index. 241 | * @param params.pod_type The type of pod to use. One of s1, p1, or p2 appended with . and one of x1, x2, x4, or x8. 242 | * @param params.metadata_config Configuration for the behavior of Pinecone's internal metadata index. By default, all metadata is indexed; when metadata_config is present, only specified metadata fields are indexed. 243 | * @param params.source_collection The name of the collection to create an index from. 244 | * @see https://docs.pinecone.io/reference/create_index 245 | */ 246 | async createIndex(params: { 247 | environment: string; 248 | name: string; 249 | dimension: number; 250 | metric?: 'euclidean' | 'cosine' | 'dotproduct'; 251 | pods?: number; 252 | replicas?: number; 253 | shards?: number; 254 | pod_type?: string; 255 | metadata_config?: JsonObject; 256 | source_collection?: string; 257 | }): Promise { 258 | const { environment, ...rest } = params; 259 | const indexApi = this.api.extend({ 260 | prefixUrl: `https://controller.${environment}.pinecone.io`, 261 | }); 262 | await indexApi.post('databases', { json: rest }); 263 | } 264 | 265 | /** 266 | * This operation deletes an existing index. 267 | * @param params.environment The environment the index is in. Eg: us-east-1-aws or us-west1-gcp 268 | * @param params.name The name of the index to delete. 269 | * @see https://docs.pinecone.io/reference/delete_index 270 | */ 271 | async deleteIndex(params: { 272 | environment: string; 273 | name: string; 274 | }): Promise { 275 | const { environment, name } = params; 276 | const indexApi = this.api.extend({ 277 | prefixUrl: `https://controller.${environment}.pinecone.io`, 278 | }); 279 | await indexApi.delete(`databases/${name}`); 280 | } 281 | } 282 | 283 | /** 284 | * Hybrid score using a convex combination: alpha * dense + (1 - alpha) * sparse 285 | * @see: https://docs.pinecone.io/docs/hybrid-search#sparse-dense-queries-do-not-support-explicit-weighting 286 | */ 287 | function hybridScoreNorm( 288 | dense: number[], 289 | sparse: SparseValues, 290 | alpha: number, 291 | ): { 292 | values: number[]; 293 | sparseValues: SparseValues; 294 | } { 295 | if (alpha < 0 || alpha > 1) { 296 | throw new Error('Alpha must be between 0 and 1'); 297 | } 298 | const sparseValues: SparseValues = { 299 | indices: sparse.indices, 300 | values: sparse.values.map((v) => v * (1 - alpha)), 301 | }; 302 | const values: number[] = dense.map((v) => v * alpha); 303 | return { values, sparseValues }; 304 | } 305 | -------------------------------------------------------------------------------- /src/types.ts: -------------------------------------------------------------------------------- 1 | import type { JsonObject, RequireExactlyOne } from 'type-fest'; 2 | 3 | /** 4 | * All metadata must extend a JSON object. 5 | * @see https://www.pinecone.io/docs/metadata-filtering/#supported-metadata-types 6 | */ 7 | export type RootMetadata = JsonObject; 8 | 9 | /** 10 | * The possible leaf values for filter objects. 11 | * @note Null values aren't supported in metadata for filters, but are allowed here and automatically removed for convenience. 12 | */ 13 | type FilterValue = string | number | boolean | null | string[] | number[]; 14 | type FilterOperator = 15 | | '$eq' 16 | | '$ne' 17 | | '$gt' 18 | | '$gte' 19 | | '$lt' 20 | | '$lte' 21 | | '$in' 22 | | '$nin'; 23 | 24 | /** 25 | * An object of metadata filters. 26 | * @see https://www.pinecone.io/docs/metadata-filtering/ 27 | */ 28 | export type Filter = { 29 | [key in keyof Metadata | FilterOperator]?: 30 | | FilterValue 31 | | { 32 | [key in keyof Metadata | FilterOperator]?: FilterValue; 33 | }; 34 | }; 35 | 36 | /** 37 | * Sparse vector data. Represented as a list of indices and a list of corresponded values, which must be the same length. 38 | */ 39 | export type SparseValues = { 40 | indices: number[]; 41 | values: number[]; 42 | }; 43 | 44 | /** 45 | * The base vector object with strongly typed metadata. 46 | */ 47 | export type Vector = { 48 | id: string; 49 | values: number[]; 50 | sparseValues?: SparseValues; 51 | metadata?: Metadata; 52 | }; 53 | 54 | /** 55 | * The parameters for a vector query. 56 | */ 57 | export type QueryParams = RequireExactlyOne< 58 | { 59 | topK: number; 60 | minScore?: number; 61 | filter?: Filter; 62 | includeMetadata?: boolean; 63 | includeValues?: boolean; 64 | vector?: number[]; 65 | sparseVector?: SparseValues; 66 | hybridAlpha?: number; 67 | id?: string; 68 | }, 69 | // Queries must have either a vector or an id and cannot have both. 70 | 'vector' | 'id' 71 | >; 72 | 73 | type ScoredVector = { 74 | id: string; 75 | score: number; 76 | }; 77 | 78 | /** 79 | * Query results without metadata or vector values. 80 | */ 81 | export type QueryResultsBase = { 82 | namespace: string; 83 | matches: ScoredVector[]; 84 | }; 85 | 86 | /** 87 | * Query results with vector values and no metadata. 88 | */ 89 | export type QueryResultsValues = { 90 | namespace: string; 91 | matches: Prettify< 92 | ScoredVector & { values: number[]; sparseValues?: SparseValues } 93 | >[]; 94 | }; 95 | 96 | /** 97 | * Query results with metadata and no vector values. 98 | */ 99 | export type QueryResultsMetadata = { 100 | namespace: string; 101 | matches: Prettify[]; 102 | }; 103 | 104 | /** 105 | * Query results with metadata and vector values. 106 | */ 107 | export type QueryResultsAll = { 108 | namespace: string; 109 | matches: Prettify< 110 | ScoredVector & { 111 | metadata: Metadata; 112 | values: number[]; 113 | sparseValues?: SparseValues; 114 | } 115 | >[]; 116 | }; 117 | 118 | /** 119 | * Query results with metadata and vector values narrowed by the query parameters. 120 | */ 121 | export type QueryResults< 122 | Metadata extends RootMetadata, 123 | Params extends { includeMetadata?: boolean; includeValues?: boolean }, 124 | > = Params extends { includeValues: true; includeMetadata: true } 125 | ? QueryResultsAll 126 | : Params extends { includeValues: true } 127 | ? QueryResultsValues 128 | : Params extends { includeMetadata: true } 129 | ? QueryResultsMetadata 130 | : QueryResultsBase; 131 | 132 | /** 133 | * The parameters that need null values removed. 134 | */ 135 | export type NoNullParams = { 136 | filter?: Filter; 137 | metadata?: Metadata; 138 | setMetadata?: Metadata; 139 | }; 140 | 141 | /** Helper type to expand complex types into simple previews. */ 142 | type Prettify = { 143 | [K in keyof T]: T[K]; 144 | } & {}; 145 | -------------------------------------------------------------------------------- /src/utils.ts: -------------------------------------------------------------------------------- 1 | import type { NoNullParams, RootMetadata } from './types.js'; 2 | 3 | /** 4 | * Recursively remove keys with null values from an object. 5 | * Also handles accepting undefined to prevent repeating this logic at each call site. 6 | */ 7 | export function removeNullValuesFromObject( 8 | obj?: T, 9 | ): T | undefined { 10 | if (obj === undefined) return undefined; 11 | for (const key in obj) { 12 | const value = obj[key]; 13 | if (value === null) delete obj[key]; 14 | else if (typeof value == 'object') removeNullValuesFromObject(value); 15 | } 16 | return obj; 17 | } 18 | 19 | /** 20 | * This remove null values from the metadata and filter properties of the given 21 | * object. This makes it easier to work with Pinecones lack of support for null. 22 | */ 23 | export function removeNullValues< 24 | Metadata extends RootMetadata, 25 | T extends NoNullParams, 26 | >(obj: T | undefined): T | undefined { 27 | if (obj === undefined) return undefined; 28 | const { metadata, filter, setMetadata, ...rest } = obj; 29 | return { 30 | filter: removeNullValuesFromObject(filter), 31 | metadata: removeNullValuesFromObject(metadata), 32 | setMetadata: removeNullValuesFromObject(setMetadata), 33 | ...rest, 34 | } as T; 35 | } 36 | -------------------------------------------------------------------------------- /test/e2e-tests.ts: -------------------------------------------------------------------------------- 1 | import { strict as assert } from 'node:assert'; 2 | import type { PineconeClient } from '../src/index.js'; 3 | 4 | export const NAMESPACE = 'pinecone-fetch-e2e'; 5 | 6 | export type Metadata = { 7 | count: number; 8 | tags?: string[] | null; 9 | approved?: boolean; 10 | }; 11 | 12 | /** 13 | * Minimal E2E tests until I setup a proper test suite. 14 | * Run by the native and polyfilled test files. 15 | */ 16 | export async function e2eTests(pinecone: PineconeClient) { 17 | const vectors = { 18 | 1: { 19 | id: '1', 20 | values: [1, 1, 1, 1], 21 | sparseValues: { 22 | indices: [0, 10, 20, 30], 23 | values: [1.1, 1.1, 1.1, 1.1], 24 | }, 25 | metadata: { count: 1, tags: ['a', 'b'], approved: true }, 26 | }, 27 | 2: { 28 | id: '2', 29 | values: [2, 2, 2, 2], 30 | sparseValues: { 31 | indices: [1, 11, 21, 31], 32 | values: [1.2, 1.2, 1.2, 1.2], 33 | }, 34 | metadata: { count: 2, tags: ['b', 'c'], approved: false }, 35 | }, 36 | }; 37 | 38 | // Clear the namespace before starting 39 | await pinecone.delete({ deleteAll: true }); 40 | 41 | // Basic upsert 42 | await pinecone.upsert({ vectors: [vectors[1], vectors[2]] }); 43 | 44 | // Verify upsert with fetch 45 | const r1 = await pinecone.fetch({ ids: ['1', '2'] }); 46 | assert(r1.namespace === NAMESPACE, 'namespace should match'); 47 | assert(Object.keys(r1.vectors).length === 2, 'Expected 2 vectors'); 48 | assert.deepStrictEqual( 49 | r1.vectors['1'], 50 | { ...vectors[1] }, 51 | 'Vector 1 should match', 52 | ); 53 | assert.deepStrictEqual( 54 | r1.vectors['2'], 55 | { ...vectors[2] }, 56 | 'Vector 2 should match', 57 | ); 58 | 59 | // Query by vector 60 | const r2 = await pinecone.query({ 61 | topK: 2, 62 | vector: [3, 3, 3, 3], 63 | sparseVector: { 64 | indices: [2, 12, 22, 32], 65 | values: [1.3, 1.3, 1.3, 1.3], 66 | }, 67 | }); 68 | assert(r2.namespace === NAMESPACE, 'namespace should match'); 69 | // @ts-expect-error 70 | assert(r2.matches[0].metadata === undefined, 'metadata should be undefined'); 71 | // @ts-expect-error - they return an empty array for some reason 72 | assert(r2.matches[0].values.length === 0, 'values should be []'); 73 | assert(r2.matches.length === 2, 'Expected 2 matches'); 74 | assert( 75 | typeof r2.matches[0]?.score === 'number', 76 | 'Expected score to be a number', 77 | ); 78 | assert( 79 | typeof r2.matches[1]?.score === 'number', 80 | 'Expected score to be a number', 81 | ); 82 | 83 | // Query by vector id 84 | const r3 = await pinecone.query({ 85 | topK: 2, 86 | id: '2', 87 | }); 88 | // @ts-expect-error 89 | assert(r2.matches[0].metadata === undefined, 'metadata should be undefined'); 90 | // @ts-expect-error - they return an empty array for some reason 91 | assert(r2.matches[0].values.length === 0, 'values should be []'); 92 | assert(r3.matches.length === 2, 'Expected 2 matches'); 93 | 94 | // Query by vector id with smaller topK 95 | const r4 = await pinecone.query({ 96 | topK: 1, 97 | id: '2', 98 | }); 99 | assert(r4.matches.length === 1, 'Expected 1 matches'); 100 | 101 | // Query by vector id with metadata 102 | const r5 = await pinecone.query({ 103 | topK: 2, 104 | id: '2', 105 | includeMetadata: true, 106 | }); 107 | // @ts-expect-error - they return an empty array for some reason 108 | assert(r5.matches[0].values.length === 0, 'values should be []'); 109 | const v1 = r5.matches.find((v) => v.id === '1'); 110 | assert(v1?.id === '1', 'Expected id to be 1'); 111 | assert.deepStrictEqual( 112 | v1?.metadata, 113 | vectors[1].metadata, 114 | 'Metadata should match', 115 | ); 116 | 117 | // Query by vector id with vector values 118 | const r6 = await pinecone.query({ 119 | topK: 2, 120 | id: '2', 121 | includeValues: true, 122 | }); 123 | // @ts-expect-error 124 | assert(r2.matches[0].metadata === undefined, 'metadata should be undefined'); 125 | const v2 = r6.matches.find((v) => v.id === '1'); 126 | assert(v2?.id === '1', 'Expected id to be 1'); 127 | assert.deepStrictEqual(v2.values, vectors[1].values, 'Values should match'); 128 | assert.deepStrictEqual( 129 | v2.sparseValues, 130 | vectors[1].sparseValues, 131 | 'Sparse values should match', 132 | ); 133 | 134 | // Query by vector id with vector values and metadata 135 | const r7 = await pinecone.query({ 136 | topK: 2, 137 | id: '2', 138 | includeValues: true, 139 | includeMetadata: true, 140 | }); 141 | const v3 = r7.matches.find((v) => v.id === '1'); 142 | assert(v3?.id === '1', 'Expected id to be 1'); 143 | assert.deepStrictEqual(v3.values, vectors[1].values, 'Values should match'); 144 | assert.deepStrictEqual( 145 | v3.sparseValues, 146 | vectors[1].sparseValues, 147 | 'Sparse values should match', 148 | ); 149 | assert.deepStrictEqual( 150 | v3?.metadata, 151 | vectors[1].metadata, 152 | 'Metadata should match', 153 | ); 154 | 155 | // Query with filter: simple 156 | const r8 = await pinecone.query({ 157 | topK: 2, 158 | vector: [3, 3, 3, 3], 159 | sparseVector: { 160 | indices: [2, 12, 22, 32], 161 | values: [1.3, 1.3, 1.3, 1.3], 162 | }, 163 | filter: { count: 1 }, 164 | }); 165 | assert(r8.matches.length === 1, 'Expected 1 matches'); 166 | 167 | // Query with filter: advanced 168 | const r9 = await pinecone.query({ 169 | topK: 2, 170 | vector: [3, 3, 3, 3], 171 | sparseVector: { 172 | indices: [2, 12, 22, 32], 173 | values: [1.3, 1.3, 1.3, 1.3], 174 | }, 175 | filter: { 176 | count: { $gte: 1 }, 177 | tags: { $in: ['a'] }, 178 | }, 179 | }); 180 | assert(r9.matches.length === 1, 'Expected 1 matches'); 181 | 182 | // Query with filter: null value 183 | const r10 = await pinecone.query({ 184 | topK: 2, 185 | vector: [3, 3, 3, 3], 186 | sparseVector: { 187 | indices: [2, 12, 22, 32], 188 | values: [1.3, 1.3, 1.3, 1.3], 189 | }, 190 | filter: { 191 | count: { $lte: 1 }, 192 | tags: null, 193 | }, 194 | }); 195 | assert(r10.matches.length === 1, 'Expected 1 matches'); 196 | 197 | // Describe index stats 198 | const r11 = await pinecone.describeIndexStats(); 199 | assert(r11.dimension === 4, 'Expected dimension to be 4'); 200 | assert.deepStrictEqual( 201 | r11.namespaces[NAMESPACE], 202 | { vectorCount: 2 }, 203 | 'Expected namespace object to match', 204 | ); 205 | 206 | // Update vector values 207 | await pinecone.update({ 208 | id: '1', 209 | values: [11, 11, 11, 11], 210 | sparseValues: { 211 | indices: [2, 12, 22, 32], 212 | values: [1.3, 1.3, 1.3, 1.3], 213 | }, 214 | }); 215 | const r12 = await pinecone.fetch({ ids: ['1'] }); 216 | assert.deepStrictEqual( 217 | r12.vectors['1']?.values, 218 | [11, 11, 11, 11], 219 | 'Values should be updated', 220 | ); 221 | 222 | // Update vector metadata 223 | // null value shouldn't throw and should be ignored 224 | await pinecone.update({ 225 | id: '1', 226 | setMetadata: { count: 11, tags: null }, 227 | }); 228 | const r13 = await pinecone.fetch({ ids: ['1'] }); 229 | assert.deepStrictEqual( 230 | r13.vectors['1']?.metadata, 231 | { 232 | count: 11, 233 | tags: ['a', 'b'], 234 | approved: true, 235 | }, 236 | 'Metadata should be udpated', 237 | ); 238 | 239 | // Upsert with null metadata 240 | await pinecone.upsert({ 241 | vectors: [ 242 | { 243 | id: '3', 244 | values: [3, 3, 3, 3], 245 | sparseValues: { 246 | indices: [2, 12, 22, 32], 247 | values: [3, 3, 3, 3], 248 | }, 249 | metadata: { count: 3, tags: null, approved: false }, 250 | }, 251 | ], 252 | }); 253 | const r14 = await pinecone.fetch({ ids: ['3'] }); 254 | assert.deepStrictEqual( 255 | r14.vectors['3'], 256 | { 257 | id: '3', 258 | values: [3, 3, 3, 3], 259 | sparseValues: { 260 | indices: [2, 12, 22, 32], 261 | values: [3, 3, 3, 3], 262 | }, 263 | metadata: { count: 3, approved: false }, 264 | }, 265 | 'Upserted vector is correct', 266 | ); 267 | 268 | // Delete a vector 269 | await pinecone.delete({ ids: ['1'] }); 270 | const r15 = await pinecone.query({ 271 | topK: 3, 272 | vector: [3, 3, 3, 3], 273 | sparseValues: { 274 | indices: [2, 12, 22, 32], 275 | values: [3, 3, 3, 3], 276 | }, 277 | }); 278 | assert(r15.matches.length === 2, 'Expected 2 matches'); 279 | const deletedVector = r15.matches.find((v) => v.id === '1'); 280 | assert(deletedVector === undefined, 'Deleted vector should not be returned'); 281 | 282 | // Delete all vectors 283 | await pinecone.delete({ deleteAll: true }); 284 | const r16 = await pinecone.query({ 285 | topK: 3, 286 | vector: [3, 3, 3, 3], 287 | }); 288 | assert(r16.matches.length === 0, 'Expected all vectors to be deleted'); 289 | } 290 | -------------------------------------------------------------------------------- /test/run-tests.ts: -------------------------------------------------------------------------------- 1 | import { PineconeClient } from '../src/index.js'; 2 | import type { Metadata } from './e2e-tests.js'; 3 | import { NAMESPACE, e2eTests } from './e2e-tests.js'; 4 | 5 | /** 6 | * This runs the E2E tests against a real Pinecone instance. 7 | * The env vars need to be set to run the tests locally. 8 | */ 9 | async function main() { 10 | try { 11 | const pinecone = new PineconeClient({ namespace: NAMESPACE }); 12 | await e2eTests(pinecone); 13 | console.log('E2E tests passed'); 14 | process.exit(0); 15 | } catch (e) { 16 | console.error('E2E tests failed', e); 17 | process.exit(1); 18 | } 19 | } 20 | 21 | main(); 22 | -------------------------------------------------------------------------------- /tsconfig.dist.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig", 3 | "compilerOptions": { 4 | "sourceMap": true, 5 | "inlineSources": true, 6 | "rootDir": "./src", 7 | "outDir": "./dist" 8 | }, 9 | "include": ["src"] 10 | } 11 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "include": ["src", "test"], 3 | "exclude": ["**/node_modules", "**/.*/"], 4 | "compilerOptions": { 5 | "outDir": "dist", 6 | "module": "node16", 7 | "moduleResolution": "node16", 8 | "moduleDetection": "force", 9 | "target": "ES2022", // Node.js 18 10 | "lib": ["DOM", "DOM.Iterable", "ES2022"], 11 | "allowSyntheticDefaultImports": true, // To provide backwards compatibility, Node.js allows you to import most CommonJS packages with a default import. This flag tells TypeScript that it's okay to use import on CommonJS modules. 12 | "resolveJsonModule": false, // ESM doesn't yet support JSON modules. 13 | "jsx": "react", 14 | "declaration": true, 15 | "pretty": true, 16 | "stripInternal": true, 17 | "strict": true, 18 | "noImplicitReturns": true, 19 | "noImplicitOverride": true, 20 | "noUnusedLocals": true, 21 | "noUnusedParameters": true, 22 | "noFallthroughCasesInSwitch": true, 23 | "noUncheckedIndexedAccess": true, 24 | "noPropertyAccessFromIndexSignature": true, 25 | "noEmitOnError": true, 26 | "useDefineForClassFields": true, 27 | "forceConsistentCasingInFileNames": true, 28 | "skipLibCheck": true 29 | } 30 | } 31 | --------------------------------------------------------------------------------