├── index.ts ├── bun.lockb ├── .env.example ├── .husky ├── pre-push └── pre-commit ├── .vscode ├── extensions.json └── settings.json ├── assets └── how-semantic-cache-works.png ├── tsup.config.ts ├── prettier.config.mjs ├── tsconfig.json ├── .github └── workflows │ ├── tests.yaml │ └── release.yaml ├── LICENSE ├── package.json ├── commitlint.config.js ├── .eslintrc.json ├── .gitignore ├── src ├── semantic-cache.ts └── semantic-cache.test.ts └── readme.md /index.ts: -------------------------------------------------------------------------------- 1 | export { SemanticCache } from "./src/semantic-cache"; 2 | -------------------------------------------------------------------------------- /bun.lockb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/upstash/semantic-cache/HEAD/bun.lockb -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | UPSTASH_VECTOR_REST_URL=XXXXXX 2 | UPSTASH_VECTOR_REST_TOKEN=XXXXXX 3 | -------------------------------------------------------------------------------- /.husky/pre-push: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | . "$(dirname "$0")/_/husky.sh" 3 | 4 | bun run build 5 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": ["esbenp.prettier-vscode", "dbaeumer.vscode-eslint"] 3 | } 4 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | . "$(dirname "$0")/_/husky.sh" 3 | 4 | bun run lint && bun run fmt && bun run test run 5 | -------------------------------------------------------------------------------- /assets/how-semantic-cache-works.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/upstash/semantic-cache/HEAD/assets/how-semantic-cache-works.png -------------------------------------------------------------------------------- /tsup.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "tsup"; 2 | 3 | export default defineConfig({ 4 | entry: ["index.ts"], 5 | format: ["cjs", "esm"], 6 | clean: true, 7 | dts: true, 8 | }); 9 | -------------------------------------------------------------------------------- /prettier.config.mjs: -------------------------------------------------------------------------------- 1 | /** 2 | * @type {import('prettier').Config} 3 | */ 4 | const config = { 5 | endOfLine: "lf", 6 | singleQuote: false, 7 | tabWidth: 2, 8 | trailingComma: "es5", 9 | printWidth: 100, 10 | arrowParens: "always", 11 | }; 12 | 13 | export default config; 14 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["ESNext"], 4 | "module": "esnext", 5 | "target": "esnext", 6 | "moduleResolution": "bundler", 7 | "moduleDetection": "force", 8 | "allowImportingTsExtensions": true, 9 | "noEmit": true, 10 | "strict": true, 11 | "downlevelIteration": true, 12 | "skipLibCheck": true, 13 | "allowSyntheticDefaultImports": true, 14 | "forceConsistentCasingInFileNames": true, 15 | "allowJs": true, 16 | "types": [ 17 | "bun-types" // add Bun global 18 | ] 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.formatOnSave": true, 3 | "git.autofetch": true, 4 | "editor.codeActionsOnSave": { 5 | "source.fixAll": "explicit" 6 | }, 7 | "[html]": { 8 | "editor.defaultFormatter": "esbenp.prettier-vscode" 9 | }, 10 | "[json]": { 11 | "editor.defaultFormatter": "esbenp.prettier-vscode" 12 | }, 13 | "[javascript]": { 14 | "editor.defaultFormatter": "esbenp.prettier-vscode" 15 | }, 16 | "[jsonc]": { 17 | "editor.defaultFormatter": "esbenp.prettier-vscode" 18 | }, 19 | "[typescript]": { 20 | "editor.defaultFormatter": "esbenp.prettier-vscode" 21 | }, 22 | "editor.defaultFormatter": "esbenp.prettier-vscode" 23 | } 24 | -------------------------------------------------------------------------------- /.github/workflows/tests.yaml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | on: 3 | push: 4 | branches: 5 | - master 6 | pull_request: 7 | schedule: 8 | - cron: "0 0 * * *" # daily 9 | 10 | env: 11 | UPSTASH_VECTOR_REST_URL: ${{ secrets.UPSTASH_VECTOR_REST_URL }} 12 | UPSTASH_VECTOR_REST_TOKEN: ${{ secrets.UPSTASH_VECTOR_REST_TOKEN }} 13 | jobs: 14 | test: 15 | runs-on: ubuntu-latest 16 | concurrency: test 17 | 18 | name: Tests 19 | steps: 20 | - name: Setup repo 21 | uses: actions/checkout@v3 22 | 23 | - name: Setup Bun 24 | uses: oven-sh/setup-bun@v1 25 | with: 26 | bun-version: latest 27 | 28 | - name: Install Dependencies 29 | run: bun install 30 | 31 | - name: Run Lint 32 | run: bun run fmt 33 | 34 | - name: Run tests 35 | run: bun run test 36 | 37 | - name: Run Build 38 | run: bun run build 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Upstash 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | release: 5 | types: 6 | - published 7 | 8 | jobs: 9 | release: 10 | name: Release 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Checkout Repo 14 | uses: actions/checkout@v3 15 | 16 | - name: Set env 17 | run: echo "VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV 18 | 19 | - name: Setup Node 20 | uses: actions/setup-node@v2 21 | with: 22 | node-version: 18 23 | 24 | - name: Set package version 25 | run: echo $(jq --arg v "${{ env.VERSION }}" '(.version) = $v' package.json) > package.json 26 | 27 | - name: Setup Bun 28 | uses: oven-sh/setup-bun@v1 29 | with: 30 | bun-version: latest 31 | 32 | - name: Install dependencies 33 | run: bun install 34 | 35 | - name: Build 36 | run: bun run build 37 | 38 | - name: Add npm token 39 | run: echo "//registry.npmjs.org/:_authToken=${{secrets.NPM_TOKEN}}" > .npmrc 40 | 41 | - name: Publish release candidate 42 | if: "github.event.release.prerelease" 43 | run: npm publish --access public --tag=canary --no-git-checks 44 | 45 | - name: Publish 46 | if: "!github.event.release.prerelease" 47 | run: npm publish --access public --no-git-checks 48 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@upstash/semantic-cache", 3 | "version": "1.0.3", 4 | "main": "./dist/index.js", 5 | "module": "./dist/index.mjs", 6 | "types": "./dist/index.d.ts", 7 | "scripts": { 8 | "test": "bun test", 9 | "fmt": "prettier --write .", 10 | "lint": "tsc && eslint \"src/**/*.{js,ts,tsx}\" --quiet --fix", 11 | "build": "tsup", 12 | "prepare": "husky install" 13 | }, 14 | "bugs": { 15 | "url": "https://github.com/upstash/semantic-cache/issues" 16 | }, 17 | "description": "A Semantic cache built on top of Upstash Vector DB", 18 | "files": [ 19 | "dist" 20 | ], 21 | "homepage": "https://upstash.com/vector", 22 | "keywords": [ 23 | "vector", 24 | "upstash", 25 | "db", 26 | "semantic-cache", 27 | "cache", 28 | "classification" 29 | ], 30 | "author": "Enes Akar ", 31 | "repository": { 32 | "type": "git", 33 | "url": "https://github.com/upstash/semantic-cache" 34 | }, 35 | "license": "ISC", 36 | "devDependencies": { 37 | "@commitlint/cli": "^19.2.2", 38 | "@commitlint/config-conventional": "^19.2.2", 39 | "@types/node": "^20.12.2", 40 | "@typescript-eslint/eslint-plugin": "^7.0.1", 41 | "@typescript-eslint/parser": "^7.0.1", 42 | "bun-types": "latest", 43 | "eslint": "^8", 44 | "eslint-plugin-unicorn": "^51.0.1", 45 | "husky": "^9.0.10", 46 | "prettier": "^3.2.5", 47 | "tsup": "^8.2.3", 48 | "typescript": "^5.4.5", 49 | "vitest": "latest" 50 | }, 51 | "dependencies": { 52 | "@upstash/vector": "^1.1.3" 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /commitlint.config.js: -------------------------------------------------------------------------------- 1 | // build: Changes that affect the build system or external dependencies (example scopes: gulp, broccoli, npm) 2 | // ci: Changes to our CI configuration files and scripts (example scopes: Travis, Circle, BrowserStack, SauceLabs) 3 | // docs: Documentation only changes 4 | // feat: A new feature 5 | // fix: A bug fix 6 | // perf: A code change that improves performance 7 | // refactor: A code change that neither fixes a bug nor adds a feature 8 | // style: Changes that do not affect the meaning of the code (white-space, formatting, missing semi-colons, etc) 9 | // test: Adding missing tests or correcting existing tests 10 | 11 | module.exports = { 12 | extends: ["@commitlint/config-conventional"], 13 | rules: { 14 | "body-leading-blank": [1, "always"], 15 | "body-max-line-length": [2, "always", 100], 16 | "footer-leading-blank": [1, "always"], 17 | "footer-max-line-length": [2, "always", 100], 18 | "header-max-length": [2, "always", 100], 19 | "scope-case": [2, "always", "lower-case"], 20 | "subject-case": [2, "never", ["sentence-case", "start-case", "pascal-case", "upper-case"]], 21 | "subject-empty": [2, "never"], 22 | "subject-full-stop": [2, "never", "."], 23 | "type-case": [2, "always", "lower-case"], 24 | "type-empty": [2, "never"], 25 | "type-enum": [ 26 | 2, 27 | "always", 28 | [ 29 | "build", 30 | "chore", 31 | "ci", 32 | "docs", 33 | "feat", 34 | "fix", 35 | "perf", 36 | "refactor", 37 | "revert", 38 | "style", 39 | "test", 40 | "translation", 41 | "security", 42 | "changeset", 43 | ], 44 | ], 45 | }, 46 | }; 47 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "es2024": true 4 | }, 5 | "extends": [ 6 | "eslint:recommended", 7 | "plugin:unicorn/recommended", 8 | "plugin:@typescript-eslint/strict-type-checked", 9 | "plugin:@typescript-eslint/stylistic-type-checked" 10 | ], 11 | "plugins": ["@typescript-eslint", "unicorn"], 12 | "parserOptions": { 13 | "project": "./tsconfig.json" 14 | }, 15 | "ignorePatterns": ["*.config.*"], 16 | "rules": { 17 | "no-console": ["error", { "allow": ["warn", "error"] }], 18 | "@typescript-eslint/no-magic-numbers": [ 19 | "error", 20 | { "ignore": [-1, 0, 1, 100], "ignoreArrayIndexes": true } 21 | ], 22 | "@typescript-eslint/unbound-method": "off", 23 | "@typescript-eslint/prefer-as-const": "error", 24 | "@typescript-eslint/consistent-type-imports": "error", 25 | "@typescript-eslint/restrict-template-expressions": "off", 26 | "@typescript-eslint/consistent-type-definitions": ["error", "type"], 27 | "@typescript-eslint/no-unused-vars": [ 28 | "error", 29 | { "varsIgnorePattern": "^_", "argsIgnorePattern": "^_" } 30 | ], 31 | "@typescript-eslint/prefer-ts-expect-error": "off", 32 | "@typescript-eslint/no-misused-promises": [ 33 | "error", 34 | { 35 | "checksVoidReturn": false 36 | } 37 | ], 38 | "unicorn/prevent-abbreviations": [ 39 | 2, 40 | { 41 | "replacements": { 42 | "args": false, 43 | "props": false, 44 | "db": false 45 | } 46 | } 47 | ], 48 | "no-implicit-coercion": ["error", { "boolean": true }], 49 | "no-extra-boolean-cast": ["error", { "enforceForLogicalOperands": true }], 50 | "no-unneeded-ternary": ["error", { "defaultAssignment": true }], 51 | "unicorn/no-array-reduce": ["off"], 52 | "unicorn/no-nested-ternary": "off" 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore 2 | 3 | # Logs 4 | 5 | logs 6 | _.log 7 | npm-debug.log_ 8 | yarn-debug.log* 9 | yarn-error.log* 10 | lerna-debug.log* 11 | .pnpm-debug.log* 12 | 13 | # Diagnostic reports (https://nodejs.org/api/report.html) 14 | 15 | report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json 16 | 17 | # Runtime data 18 | 19 | pids 20 | _.pid 21 | _.seed 22 | \*.pid.lock 23 | 24 | # Directory for instrumented libs generated by jscoverage/JSCover 25 | 26 | lib-cov 27 | 28 | # Coverage directory used by tools like istanbul 29 | 30 | coverage 31 | \*.lcov 32 | 33 | # nyc test coverage 34 | 35 | .nyc_output 36 | 37 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 38 | 39 | .grunt 40 | 41 | # Bower dependency directory (https://bower.io/) 42 | 43 | bower_components 44 | 45 | # node-waf configuration 46 | 47 | .lock-wscript 48 | 49 | # Compiled binary addons (https://nodejs.org/api/addons.html) 50 | 51 | build/Release 52 | 53 | # Dependency directories 54 | 55 | node_modules/ 56 | jspm_packages/ 57 | 58 | # Snowpack dependency directory (https://snowpack.dev/) 59 | 60 | web_modules/ 61 | 62 | # TypeScript cache 63 | 64 | \*.tsbuildinfo 65 | 66 | # Optional npm cache directory 67 | 68 | .npm 69 | 70 | # Optional eslint cache 71 | 72 | .eslintcache 73 | 74 | # Optional stylelint cache 75 | 76 | .stylelintcache 77 | 78 | # Microbundle cache 79 | 80 | .rpt2_cache/ 81 | .rts2_cache_cjs/ 82 | .rts2_cache_es/ 83 | .rts2_cache_umd/ 84 | 85 | # Optional REPL history 86 | 87 | .node_repl_history 88 | 89 | # Output of 'npm pack' 90 | 91 | \*.tgz 92 | 93 | # Yarn Integrity file 94 | 95 | .yarn-integrity 96 | 97 | # dotenv environment variable files 98 | 99 | .env 100 | .env.development.local 101 | .env.test.local 102 | .env.production.local 103 | .env.local 104 | 105 | # parcel-bundler cache (https://parceljs.org/) 106 | 107 | .cache 108 | .parcel-cache 109 | 110 | # Next.js build output 111 | 112 | .next 113 | out 114 | 115 | # Nuxt.js build / generate output 116 | 117 | .nuxt 118 | dist 119 | 120 | # Gatsby files 121 | 122 | .cache/ 123 | 124 | # Comment in the public line in if your project uses Gatsby and not Next.js 125 | 126 | # https://nextjs.org/blog/next-9-1#public-directory-support 127 | 128 | # public 129 | 130 | # vuepress build output 131 | 132 | .vuepress/dist 133 | 134 | # vuepress v2.x temp and cache directory 135 | 136 | .temp 137 | .cache 138 | 139 | # Docusaurus cache and generated files 140 | 141 | .docusaurus 142 | 143 | # Serverless directories 144 | 145 | .serverless/ 146 | 147 | # FuseBox cache 148 | 149 | .fusebox/ 150 | 151 | # DynamoDB Local files 152 | 153 | .dynamodb/ 154 | 155 | # TernJS port file 156 | 157 | .tern-port 158 | 159 | # Stores VSCode versions used for testing VSCode extensions 160 | 161 | .vscode-test 162 | 163 | # yarn v2 164 | 165 | .yarn/cache 166 | .yarn/unplugged 167 | .yarn/build-state.yml 168 | .yarn/install-state.gz 169 | .pnp.\* 170 | 171 | # IntelliJ based IDEs 172 | .idea 173 | 174 | # Finder (MacOS) folder config 175 | .DS_Store 176 | -------------------------------------------------------------------------------- /src/semantic-cache.ts: -------------------------------------------------------------------------------- 1 | import type { Index } from "@upstash/vector"; 2 | 3 | type SemanticCacheConfig = { 4 | /** 5 | * A value between 0 and 1. If you set is to 1.0 then it acts like a hash map which means only exact lexical matches will be returned. 6 | * If you set it to 0.0 then it acts like a full text search query which means a value with the best proximity score (closest value) will be returned. 7 | * @default 0.9 8 | */ 9 | minProximity: number; 10 | /** 11 | * Upstash serverless vector client 12 | */ 13 | index: Index; 14 | 15 | /** 16 | * Optional namespace for the cache 17 | */ 18 | namespace?: string; 19 | }; 20 | 21 | export class SemanticCache { 22 | private minProximity: number; 23 | private index: Index; 24 | private namespace?: string; 25 | 26 | constructor(config: SemanticCacheConfig) { 27 | this.minProximity = config.minProximity; 28 | 29 | if (config.namespace) { 30 | this.index = config.index.namespace(config.namespace) as unknown as Index; 31 | this.namespace = config.namespace; 32 | } else { 33 | this.index = config.index; 34 | } 35 | } 36 | 37 | async get(key: string): Promise; 38 | async get(keys: string[]): Promise<(string | undefined)[]>; 39 | 40 | async get(keyOrKeys: string | string[]): Promise { 41 | if (typeof keyOrKeys === "string") { 42 | const result = await this.queryKey(keyOrKeys); 43 | return result; 44 | } 45 | 46 | if (Array.isArray(keyOrKeys)) { 47 | // Multiple keys fetch 48 | const results = await Promise.all(keyOrKeys.map((key) => this.queryKey(key))); 49 | return results; 50 | } 51 | } 52 | 53 | private async queryKey(key: string): Promise { 54 | const result = await this.index.query({ 55 | data: key, 56 | topK: 1, 57 | includeVectors: false, 58 | includeMetadata: true, 59 | }); 60 | if (result.length > 0 && result[0].score > this.minProximity) { 61 | return result[0]?.metadata?.value as string; 62 | } 63 | return; 64 | } 65 | 66 | async set(key: string, value: string): Promise; 67 | async set(keys: string[], values: string[]): Promise; 68 | 69 | async set(keyOrKeys: string | string[], valueOrValues?: string | string[]): Promise { 70 | if (typeof keyOrKeys === "string" && typeof valueOrValues === "string") { 71 | await this.index.upsert({ 72 | id: keyOrKeys, 73 | data: keyOrKeys, 74 | metadata: { value: valueOrValues }, 75 | }); 76 | } 77 | 78 | if (Array.isArray(keyOrKeys) && Array.isArray(valueOrValues)) { 79 | const upserts = keyOrKeys.map((key, index) => ({ 80 | id: key, 81 | data: key, 82 | metadata: { value: valueOrValues[index] }, 83 | })); 84 | for (const upsert of upserts) { 85 | await this.index.upsert(upsert); 86 | } 87 | } 88 | } 89 | 90 | async delete(key: string): Promise { 91 | const result = await this.index.delete(key); 92 | return result.deleted; 93 | } 94 | 95 | async bulkDelete(keys: string[]): Promise { 96 | const result = await this.index.delete(keys); 97 | return result.deleted; 98 | } 99 | 100 | async flush(): Promise { 101 | if (this.namespace) { 102 | await this.index.reset({ namespace: this.namespace }); 103 | return; 104 | } 105 | 106 | await this.index.reset(); 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /src/semantic-cache.test.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-non-null-assertion */ 2 | import { afterEach, describe, expect, test } from "bun:test"; 3 | import { SemanticCache } from "./semantic-cache"; 4 | import { sleep } from "bun"; 5 | import { Index } from "@upstash/vector"; 6 | 7 | const PROXIMITY_THRESHOLD = 0.9; 8 | const DELAY_MS = 1000; 9 | const cache = new SemanticCache({ 10 | index: new Index(), 11 | minProximity: PROXIMITY_THRESHOLD, 12 | }); 13 | 14 | describe("semantic-cache", () => { 15 | afterEach(() => cache.flush()); 16 | 17 | test("should return cached value successfully", async () => { 18 | const cachedValue = "paris"; 19 | await cache.set("capital of france", cachedValue); 20 | await sleep(DELAY_MS); 21 | const result = await cache.get("france's capital"); 22 | 23 | expect(result).toBe(cachedValue); 24 | }); 25 | 26 | test("should return cached values in bulk successfully", async () => { 27 | const cacheValues = [ 28 | { 29 | question: "capital of france", 30 | answer: "paris", 31 | }, 32 | { 33 | question: "best drink on a hot day", 34 | answer: "water", 35 | }, 36 | ]; 37 | 38 | for await (const pair of cacheValues) { 39 | await cache.set(pair.question, pair.answer); 40 | } 41 | await sleep(DELAY_MS); 42 | 43 | const result = await cache.get(cacheValues.map((p) => p.question)); 44 | expect(result).toEqual(cacheValues.map((p) => p.answer)); 45 | }); 46 | 47 | test("should return cached value successfully", async () => { 48 | const cachedValue1 = "water"; 49 | const cachedValue2 = "H2O"; 50 | 51 | await cache.set("best drink on a hot day", cachedValue1); 52 | await cache.set("chemical formula for water", cachedValue2); 53 | await sleep(DELAY_MS); 54 | 55 | const firstResult = await cache.get("what to drink when it's hot"); 56 | const secondResult = await cache.get("what is water's chemical formula"); 57 | 58 | expect(firstResult).toBe(cachedValue1); 59 | expect(secondResult).toBe(cachedValue2); 60 | }); 61 | 62 | test("should return undefined when doesn't exist", async () => { 63 | const firstResult = await cache.get("what to drink when it's hot"); 64 | 65 | expect(firstResult).toBeUndefined(); 66 | }); 67 | 68 | test("should delete cached value from cache", async () => { 69 | const cachedValue1 = "water"; 70 | const key = "best drink on a hot day"; 71 | 72 | await cache.set(key, cachedValue1); 73 | await sleep(DELAY_MS); 74 | 75 | let firstResult = await cache.get("what to drink when it's hot"); 76 | expect(firstResult).toBe(cachedValue1); 77 | 78 | await cache.delete(key); 79 | firstResult = await cache.get("what to drink when it's hot"); 80 | 81 | expect(firstResult).toBeUndefined(); 82 | }); 83 | 84 | test("should work with namespaces", async () => { 85 | const cache1 = new SemanticCache({ 86 | index: new Index(), 87 | minProximity: PROXIMITY_THRESHOLD, 88 | namespace: "cache1", 89 | }); 90 | 91 | const cache2 = new SemanticCache({ 92 | index: new Index(), 93 | minProximity: PROXIMITY_THRESHOLD, 94 | namespace: "cache2", 95 | }); 96 | 97 | const cachedValue1 = "water"; 98 | const cachedValue2 = "H2O"; 99 | 100 | await cache1.set("best drink on a hot day", cachedValue1); 101 | await cache2.set("chemical formula for water", cachedValue2); 102 | await sleep(DELAY_MS); 103 | 104 | const result1 = await cache1.get("what to drink when it's hot"); 105 | const result2 = await cache2.get("what is water's chemical formula"); 106 | 107 | expect(result1).toBe(cachedValue1); 108 | expect(result2).toBe(cachedValue2); 109 | 110 | // Cleanup 111 | await cache1.flush(); 112 | await cache2.flush(); 113 | }); 114 | }); 115 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Semantic Cache 2 | 3 | 4 | Semantic Cache is a tool for caching natural text based on semantic similarity. It's ideal for any task that involves querying or retrieving information based on meaning, such as natural language classification or caching AI responses. Two pieces of text can be similar but not identical (e.g., "great places to check out in Spain" vs. "best places to visit in Spain"). Traditional caching doesn't recognize this semantic similarity and misses opportunities for reuse. 5 | 6 | Semantic Cache allows you to: 7 | 8 | - Easily classify natural text into predefined categories 9 | - Avoid redundant LLM work by caching AI responses 10 | - Reduce API latency by responding to similar queries with already cached values 11 | 12 | 13 | 14 | ## Highlights 15 | 16 | - **Uses semantic similarity**: Stores cache entries by their meaning, not just the literal characters 17 | - **Handles synonyms**: Recognizes and handles synonyms 18 | - **Multi-language support**: Works across different languages (if configured with multilingual vector models) 19 | - **Complex query support**: Understands long and nested user queries 20 | - **Easy integration**: Simple API for usage in Node.js applications 21 | - **Customizable**: Set a custom proximity threshold to filter out less relevant results 22 | 23 | ## Getting Started 24 | 25 | ### Prerequisites 26 | 27 | - An Upstash Vector database (create one [here](https://console.upstash.com/vector)) 28 | 29 | ### Installation 30 | 31 | Install the package: 32 | 33 | ```bash 34 | npm install @upstash/semantic-cache @upstash/vector 35 | ``` 36 | 37 | ### Setup 38 | 39 | First, create an Upstash Vector database [here](https://console.upstash.com/vector). You'll need the `url` and `token` credentials to connect your semantic cache. Important: Choose any pre-made embedding model when creating your database. 40 | 41 | > [!NOTE] 42 | > Different embedding models are great for different use cases. For example, if low latency is a priority, choose a model with a smaller dimension size like `bge-small-en-v1.5`. If accuracy is important, choose a model with more dimensions. 43 | 44 | Create a `.env` file in the root directory of your project and add your Upstash Vector URL and token: 45 | 46 | ```plaintext 47 | UPSTASH_VECTOR_REST_URL=https://example.upstash.io 48 | UPSTASH_VECTOR_REST_TOKEN=your_secret_token_here 49 | ``` 50 | 51 | ### Using Semantic Cache 52 | 53 | Here’s how you can use Semantic Cache in your Node.js application: 54 | 55 | ```typescript 56 | import { SemanticCache } from "@upstash/semantic-cache"; 57 | import { Index } from "@upstash/vector"; 58 | 59 | // 👇 your vector database 60 | const index = new Index(); 61 | 62 | // 👇 your semantic cache 63 | const semanticCache = new SemanticCache({ index, minProximity: 0.95 }); 64 | 65 | async function runDemo() { 66 | await semanticCache.set("Capital of Turkey", "Ankara"); 67 | await delay(1000); 68 | 69 | // 👇 outputs: "Ankara" 70 | const result = await semanticCache.get("What is Turkey's capital?"); 71 | console.log(result); 72 | } 73 | 74 | function delay(ms: number) { 75 | return new Promise((resolve) => setTimeout(resolve, ms)); 76 | } 77 | 78 | runDemo(); 79 | ``` 80 | 81 | ### The `minProximity` Parameter 82 | 83 | The `minProximity` parameter ranges from `0` to `1`. It lets you define the minimum relevance score to determine a cache hit. The higher this number, the more similar your user input must be to the cached content to be a hit. In practice, a score of 0.95 indicates a very high similarity, while a score of 0.75 already indicates a low similarity. For example, a value of 1.00, the highest possible, would only accept an _exact_ match of your user query and cache content as a cache hit. 84 | 85 | ### Namespace Support 86 | 87 | You can seperate your data into partitions with namespaces. 88 | 89 | ```typescript 90 | import { SemanticCache } from "@upstash/semantic-cache"; 91 | import { Index } from "@upstash/vector"; 92 | 93 | // 👇 your vector database 94 | const index = new Index(); 95 | 96 | // 👇 your semantic cache 97 | const semanticCache = new SemanticCache({ index, minProximity: 0.95, namespace: "user1" }); 98 | 99 | await semanticCache.set("Capital of Turkey", "Ankara"); 100 | ``` 101 | 102 | ## Examples 103 | 104 | The following examples demonstrate how you can utilize Semantic Cache in various use cases: 105 | 106 | > [!NOTE] 107 | > We add a 1-second delay after setting the data to allow time for the vector index to update. This delay is necessary to ensure that the data is available for retrieval. 108 | 109 | ### Basic Semantic Retrieval 110 | 111 | ```typescript 112 | await semanticCache.set("Capital of France", "Paris"); 113 | await delay(1000); 114 | 115 | // 👇 outputs "Paris" 116 | const result = await semanticCache.get("What's the capital of France?"); 117 | ``` 118 | 119 | ### Handling Synonyms 120 | 121 | ```typescript 122 | await semanticCache.set("largest city in USA by population", "New York"); 123 | await delay(1000); 124 | 125 | // 👇 outputs "New York" 126 | const result = await semanticCache.get("which is the most populated city in the USA?"); 127 | ``` 128 | 129 | ### Multilingual Queries 130 | 131 | Note: Your embedding model needs to support the languages you intend to use. 132 | 133 | ```typescript 134 | await semanticCache.set("German Chancellor", "Olaf Scholz"); 135 | await delay(1000); 136 | 137 | // 👇 "Who is the chancellor of Germany?" -> outputs "Olaf Scholz" 138 | const result = await semanticCache.get("Wer ist der Bundeskanzler von Deutschland?"); 139 | ``` 140 | 141 | ### Complex Queries 142 | 143 | ```typescript 144 | await semanticCache.set("year in which the Berlin wall fell", "1989"); 145 | await delay(1000); 146 | 147 | // 👇 outputs "1989" 148 | const result = await semanticCache.get("what's the year the Berlin wall destroyed?"); 149 | ``` 150 | 151 | ### Different Contexts 152 | 153 | ```typescript 154 | await semanticCache.set("the chemical formula for water", "H2O"); 155 | await semanticCache.set("the healthiest drink on a hot day", "water"); 156 | 157 | await delay(1000); 158 | 159 | // 👇 outputs "water" 160 | const result = await semanticCache.get("what should i drink when it's hot outside?"); 161 | 162 | // 👇 outputs "H2O" 163 | const result = await semanticCache.get("tell me water's chemical formula"); 164 | ``` 165 | 166 | ## Contributing 167 | 168 | We appreciate your contributions! If you'd like to contribute to this project, please fork the repository, make your changes, and submit a pull request. 169 | 170 | ## License 171 | 172 | Distributed under the MIT License. See `LICENSE` for more information. 173 | --------------------------------------------------------------------------------