├── user-session-management-go ├── .gitignore ├── docker-compose.yml ├── app │ ├── handler │ │ ├── password.go │ │ ├── db.go │ │ └── handler.go │ ├── main.go │ └── session │ │ └── access_token.go ├── go.mod └── go.sum ├── cache-in-5mins-hono ├── .gitignore ├── drizzle │ ├── 0000_medical_reptil.sql │ └── meta │ │ ├── _journal.json │ │ └── 0000_snapshot.json ├── tsconfig.json ├── drizzle.config.ts ├── package.json ├── src │ ├── schema.ts │ ├── validator.ts │ └── index.ts ├── docker-compose.yml └── README.md ├── feast-recommendation-duckdb-dragonfly ├── .python-version ├── feature_store.yaml ├── .feastignore ├── pyproject.toml ├── recommendation_05_service.py ├── feature_store_docker.yaml ├── recommendation_04_online_features.py ├── recommendation_03_historical_features.py ├── Dockerfile ├── recommendation_01_data.py ├── recommendation_02_repo.py ├── .gitignore └── README.md ├── live-migration-app ├── requirements.txt └── README.md ├── ecommerce-cache ├── app │ ├── test.db │ ├── go.mod │ ├── main.go │ ├── go.sum │ ├── recommendations.go │ ├── viewed_items.go │ └── orders.go └── load │ ├── go.mod │ ├── go.sum │ └── main.go ├── ad-server-cache-bun ├── bun.lockb ├── package.json ├── src │ ├── types.ts │ ├── index.ts │ └── ads.ts ├── .gitignore └── README.md ├── monitoring-in-memory-stores ├── prometheus.yml └── docker-compose.yaml ├── sentinel-migration ├── sentinel.conf ├── test.sh ├── go.mod ├── go.sum └── main.go ├── redis-cluster-application-example ├── Dockerfile ├── app │ ├── go.mod │ ├── go.sum │ └── main.go ├── docker-compose.yml ├── README.md └── start-cluster.sh ├── cache-aside-with-expiration ├── app │ ├── model.go │ ├── cache_store.go │ └── main.go ├── docker-compose.yml ├── go.mod └── README.md ├── high-availability ├── sentinel │ ├── config │ │ ├── sentinel-0 │ │ │ └── sentinel.conf │ │ ├── sentinel-1 │ │ │ └── sentinel.conf │ │ └── sentinel-2 │ │ │ └── sentinel.conf │ ├── README.md │ └── docker-compose.yml ├── k8s-operator │ ├── dragonfly-sample.yaml │ └── README.md └── README.md ├── leaderboard ├── config │ ├── sentinel-1 │ │ └── sentinel.conf │ ├── sentinel-2 │ │ └── sentinel.conf │ └── sentinel-3 │ │ └── sentinel.conf ├── go.mod ├── Dockerfile ├── app │ ├── model │ │ └── user.go │ ├── main.go │ └── memstore │ │ └── leaderboard.go ├── go.sum ├── docker-compose.yml └── README.md ├── celery-transaction-processing ├── docker-compose.yml ├── README.md ├── models.sql ├── models.py ├── eth.py ├── requirements.txt ├── tasks.py ├── deps.py └── utils.py ├── ad-server-cache ├── go.mod ├── go.sum └── main.go ├── redlock-go ├── Dockerfile ├── go.mod ├── docker-compose.yml ├── go.sum ├── app │ └── main.go └── README.md ├── pubsub-bullmq-node ├── tsconfig.json ├── docker-compose.yml ├── src │ ├── utils │ │ ├── dragonfly.client.ts │ │ └── hashtag.ts │ ├── pubsub │ │ ├── pub.ts │ │ └── sub.ts │ ├── bullmq-keys.txt │ ├── mq │ │ ├── queue.ts │ │ └── worker.ts │ └── index.ts ├── package.json └── .gitignore ├── real-time-statistics-node ├── src │ ├── utils │ │ ├── dragonflyClient.ts │ │ └── keyGenerator.ts │ ├── middleware │ │ ├── activeUser.ts │ │ └── leaderboard.ts │ └── index.ts ├── tsconfig.json ├── package.json └── .gitignore ├── cache-refresh-ahead ├── docker-compose.yml ├── go.mod ├── app │ ├── model.go │ └── main.go ├── README.md └── go.sum ├── langchain-memory ├── database.py ├── README.md ├── models.sql ├── schemas.py ├── models.py ├── requirements.txt └── main.py ├── background-process-example ├── go.mod ├── README.md ├── producer │ └── producer.go ├── go.sum └── worker │ └── worker.go ├── search-blogs-by-vector ├── README.md └── search-blogs-by-vector.ipynb ├── README.md └── .gitignore /user-session-management-go/.gitignore: -------------------------------------------------------------------------------- 1 | app.db -------------------------------------------------------------------------------- /cache-in-5mins-hono/.gitignore: -------------------------------------------------------------------------------- 1 | # deps 2 | node_modules/ 3 | -------------------------------------------------------------------------------- /feast-recommendation-duckdb-dragonfly/.python-version: -------------------------------------------------------------------------------- 1 | 3.11.9 2 | -------------------------------------------------------------------------------- /live-migration-app/requirements.txt: -------------------------------------------------------------------------------- 1 | flask==3.0.2 2 | redis==4.5.1 -------------------------------------------------------------------------------- /ecommerce-cache/app/test.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dragonflydb/dragonfly-examples/HEAD/ecommerce-cache/app/test.db -------------------------------------------------------------------------------- /ad-server-cache-bun/bun.lockb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dragonflydb/dragonfly-examples/HEAD/ad-server-cache-bun/bun.lockb -------------------------------------------------------------------------------- /monitoring-in-memory-stores/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 15s 3 | 4 | scrape_configs: 5 | - job_name: dragonfly_metrics 6 | static_configs: 7 | - targets: ['dragonfly:6379'] -------------------------------------------------------------------------------- /sentinel-migration/sentinel.conf: -------------------------------------------------------------------------------- 1 | port 5000 2 | sentinel monitor the_master 127.0.0.1 6379 1 3 | sentinel down-after-milliseconds the_master 5000 4 | sentinel failover-timeout the_master 60000 5 | -------------------------------------------------------------------------------- /redis-cluster-application-example/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM redis 2 | WORKDIR /data 3 | 4 | COPY start-cluster.sh ./start-cluster.sh 5 | RUN chmod a+x start-cluster.sh 6 | #CMD [ "./start-cluster.sh" ] 7 | #ENTRYPOINT [ "/bin/bash","start-cluster.sh" ] -------------------------------------------------------------------------------- /cache-aside-with-expiration/app/model.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | type User struct { 4 | ID string `json:"id"` 5 | Name string `json:"name"` 6 | } 7 | 8 | type Blog struct { 9 | ID string `json:"id"` 10 | Content string `json:"content"` 11 | } 12 | -------------------------------------------------------------------------------- /feast-recommendation-duckdb-dragonfly/feature_store.yaml: -------------------------------------------------------------------------------- 1 | project: feast_recommendation_duckdb_dragonfly 2 | registry: data/registry.db 3 | provider: local 4 | offline_store: 5 | type: duckdb 6 | online_store: 7 | type: redis 8 | connection_string: "localhost:6379" 9 | -------------------------------------------------------------------------------- /user-session-management-go/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | dragonfly: 4 | container_name: "dragonfly" 5 | image: 'docker.dragonflydb.io/dragonflydb/dragonfly' 6 | ulimits: 7 | memlock: -1 8 | ports: 9 | - "6380:6379" 10 | -------------------------------------------------------------------------------- /high-availability/sentinel/config/sentinel-0/sentinel.conf: -------------------------------------------------------------------------------- 1 | port 8000 2 | sentinel monitor default-primary dragonfly-0 6379 2 3 | sentinel down-after-milliseconds default-primary 1000 4 | sentinel failover-timeout default-primary 2000 5 | sentinel parallel-syncs default-primary 1 6 | -------------------------------------------------------------------------------- /high-availability/sentinel/config/sentinel-1/sentinel.conf: -------------------------------------------------------------------------------- 1 | port 8001 2 | sentinel monitor default-primary dragonfly-0 6379 2 3 | sentinel down-after-milliseconds default-primary 1000 4 | sentinel failover-timeout default-primary 2000 5 | sentinel parallel-syncs default-primary 1 6 | -------------------------------------------------------------------------------- /high-availability/sentinel/config/sentinel-2/sentinel.conf: -------------------------------------------------------------------------------- 1 | port 8002 2 | sentinel monitor default-primary dragonfly-0 6379 2 3 | sentinel down-after-milliseconds default-primary 1000 4 | sentinel failover-timeout default-primary 2000 5 | sentinel parallel-syncs default-primary 1 6 | -------------------------------------------------------------------------------- /leaderboard/config/sentinel-1/sentinel.conf: -------------------------------------------------------------------------------- 1 | port 5000 2 | sentinel monitor leaderboard-primary redis-source 6379 2 3 | sentinel down-after-milliseconds leaderboard-primary 1000 4 | sentinel failover-timeout leaderboard-primary 2000 5 | sentinel parallel-syncs leaderboard-primary 1 -------------------------------------------------------------------------------- /leaderboard/config/sentinel-2/sentinel.conf: -------------------------------------------------------------------------------- 1 | port 5000 2 | sentinel monitor leaderboard-primary redis-source 6379 2 3 | sentinel down-after-milliseconds leaderboard-primary 1000 4 | sentinel failover-timeout leaderboard-primary 2000 5 | sentinel parallel-syncs leaderboard-primary 1 -------------------------------------------------------------------------------- /leaderboard/config/sentinel-3/sentinel.conf: -------------------------------------------------------------------------------- 1 | port 5000 2 | sentinel monitor leaderboard-primary redis-source 6379 2 3 | sentinel down-after-milliseconds leaderboard-primary 1000 4 | sentinel failover-timeout leaderboard-primary 2000 5 | sentinel parallel-syncs leaderboard-primary 1 -------------------------------------------------------------------------------- /ecommerce-cache/load/go.mod: -------------------------------------------------------------------------------- 1 | module load 2 | 3 | go 1.19 4 | 5 | require github.com/redis/go-redis/v9 v9.0.5 6 | 7 | require ( 8 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 9 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 10 | ) 11 | -------------------------------------------------------------------------------- /leaderboard/go.mod: -------------------------------------------------------------------------------- 1 | module leaderboard 2 | 3 | go 1.20 4 | 5 | require github.com/redis/go-redis/v9 v9.1.0 6 | 7 | require ( 8 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 9 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 10 | ) 11 | -------------------------------------------------------------------------------- /sentinel-migration/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | while true; do 4 | for ((id=1; id<=10; id++)); do 5 | url="http://localhost:8080/$id" 6 | 7 | curl -X POST "$url" 8 | curl "$url" 9 | 10 | sleep 2 11 | done 12 | done -------------------------------------------------------------------------------- /cache-in-5mins-hono/drizzle/0000_medical_reptil.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE "short_links" ( 2 | "id" uuid PRIMARY KEY NOT NULL, 3 | "original_url" varchar(4096) NOT NULL, 4 | "short_code" varchar(30) NOT NULL, 5 | "created_at" timestamp with time zone NOT NULL, 6 | "expires_at" timestamp with time zone NOT NULL 7 | ); 8 | -------------------------------------------------------------------------------- /cache-in-5mins-hono/drizzle/meta/_journal.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "7", 3 | "dialect": "postgresql", 4 | "entries": [ 5 | { 6 | "idx": 0, 7 | "version": "7", 8 | "when": 1751923943613, 9 | "tag": "0000_medical_reptil", 10 | "breakpoints": true 11 | } 12 | ] 13 | } -------------------------------------------------------------------------------- /cache-in-5mins-hono/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ESNext", 4 | "module": "NodeNext", 5 | "strict": true, 6 | "types": ["node"], 7 | "jsx": "react-jsx", 8 | "jsxImportSource": "hono/jsx", 9 | "outDir": "./dist" 10 | }, 11 | "exclude": ["node_modules"] 12 | } 13 | -------------------------------------------------------------------------------- /celery-transaction-processing/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | celery-dragonfly: 4 | container_name: "celery-dragonfly" 5 | image: 'docker.dragonflydb.io/dragonflydb/dragonfly' 6 | ulimits: 7 | memlock: -1 8 | ports: 9 | - "6380:6379" 10 | command: 11 | - "--dir=/data" 12 | -------------------------------------------------------------------------------- /sentinel-migration/go.mod: -------------------------------------------------------------------------------- 1 | module sentinel-app 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/gorilla/mux v1.8.0 7 | github.com/redis/go-redis/v9 v9.0.4 8 | ) 9 | 10 | require ( 11 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 12 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 13 | ) 14 | -------------------------------------------------------------------------------- /ecommerce-cache/app/go.mod: -------------------------------------------------------------------------------- 1 | module query-cache 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/mattn/go-sqlite3 v1.14.17 7 | github.com/redis/go-redis/v9 v9.0.5 8 | ) 9 | 10 | require ( 11 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 12 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 13 | ) 14 | -------------------------------------------------------------------------------- /ad-server-cache/go.mod: -------------------------------------------------------------------------------- 1 | module redis-ads 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/go-redis/redis v6.15.9+incompatible 7 | github.com/redis/go-redis/v9 v9.0.5 8 | ) 9 | 10 | require ( 11 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 12 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 13 | ) 14 | -------------------------------------------------------------------------------- /redis-cluster-application-example/app/go.mod: -------------------------------------------------------------------------------- 1 | module demo 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/gorilla/mux v1.8.0 7 | github.com/redis/go-redis/v9 v9.0.2 8 | ) 9 | 10 | require ( 11 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 12 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 13 | ) 14 | -------------------------------------------------------------------------------- /leaderboard/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.20 as builder 2 | 3 | WORKDIR /app 4 | 5 | COPY go.mod go.sum ./ 6 | RUN go mod download 7 | 8 | COPY app/ ./app/ 9 | 10 | RUN CGO_ENABLED=0 GOOS=linux go build -v -o /go/bin/app ./app/main.go 11 | 12 | FROM alpine:latest 13 | 14 | COPY --from=builder /go/bin/app /go/bin/app 15 | 16 | CMD ["/go/bin/app"] 17 | -------------------------------------------------------------------------------- /redlock-go/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.20 as builder 2 | 3 | WORKDIR /app 4 | 5 | COPY go.mod go.sum ./ 6 | RUN go mod download 7 | 8 | COPY app/ ./app/ 9 | 10 | RUN CGO_ENABLED=0 GOOS=linux go build -v -o /go/bin/app ./app/main.go 11 | 12 | FROM alpine:latest 13 | 14 | COPY --from=builder /go/bin/app /go/bin/app 15 | 16 | CMD ["/go/bin/app"] 17 | -------------------------------------------------------------------------------- /pubsub-bullmq-node/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES6", 4 | "module": "commonjs", 5 | "strict": true, 6 | "esModuleInterop": true, 7 | "skipLibCheck": true, 8 | "forceConsistentCasingInFileNames": true, 9 | "outDir": "./dist" 10 | }, 11 | "include": ["src/**/*.ts"], 12 | "exclude": ["node_modules"] 13 | } -------------------------------------------------------------------------------- /real-time-statistics-node/src/utils/dragonflyClient.ts: -------------------------------------------------------------------------------- 1 | import { Redis as Dragonfly } from 'ioredis'; 2 | 3 | // Create and export the Dragonfly client. 4 | // Make sure a Dragonfly server is running locally on port 6379 (default). 5 | const dragonfly = new Dragonfly({ 6 | host: 'localhost', 7 | port: 6379, 8 | }); 9 | 10 | export default dragonfly; 11 | -------------------------------------------------------------------------------- /cache-aside-with-expiration/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | cache-aside-dragonfly: 4 | container_name: "cache-aside-dragonfly" 5 | image: 'docker.dragonflydb.io/dragonflydb/dragonfly' 6 | ulimits: 7 | memlock: -1 8 | ports: 9 | - "6379:6379" 10 | command: 11 | - "--maxmemory=2GB" 12 | - "--cache_mode=true" 13 | -------------------------------------------------------------------------------- /cache-refresh-ahead/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | cache-refresh-ahead-dragonfly: 4 | container_name: "cache-refresh-ahead-dragonfly" 5 | image: 'docker.dragonflydb.io/dragonflydb/dragonfly' 6 | ulimits: 7 | memlock: -1 8 | ports: 9 | - "6379:6379" 10 | command: 11 | - "--maxmemory=2GB" 12 | - "--cache_mode=true" 13 | -------------------------------------------------------------------------------- /pubsub-bullmq-node/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | pubsub-bullmq-node-dragonfly: 4 | container_name: "dragonfly" 5 | image: "docker.dragonflydb.io/dragonflydb/dragonfly" 6 | ulimits: 7 | memlock: -1 8 | ports: 9 | - "6380:6379" 10 | command: 11 | - "--dir=/data" 12 | - "--cluster_mode=emulated" 13 | - "--lock_on_hashtags" 14 | -------------------------------------------------------------------------------- /real-time-statistics-node/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES6", 4 | "module": "commonjs", 5 | "strict": true, 6 | "esModuleInterop": true, 7 | "skipLibCheck": true, 8 | "forceConsistentCasingInFileNames": true, 9 | "outDir": "./dist" 10 | }, 11 | "include": ["src/**/*.ts"], 12 | "exclude": ["node_modules"] 13 | } -------------------------------------------------------------------------------- /feast-recommendation-duckdb-dragonfly/.feastignore: -------------------------------------------------------------------------------- 1 | # Ignore virtual environment 2 | venv 3 | 4 | # The following files contain the feature store initialization (i.e., store = FeatureStore(repo_path=".")). 5 | # Essentially, we only want to apply Feast objects (e.g., FeatureView, Entity, etc.) specified in 6 | # the '02_recommendation_repo.py' file. 7 | 03_recommendation_historical_features.py 8 | 04_recommendation_online_features.py 9 | -------------------------------------------------------------------------------- /feast-recommendation-duckdb-dragonfly/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "feast-recommendation" 3 | version = "0.1.0" 4 | description = "Add your description here" 5 | readme = "README.md" 6 | requires-python = "==3.11.9" 7 | dependencies = [ 8 | "docker>=7.1.0", 9 | "feast[redis,duckdb]==0.51.0", 10 | "numpy>=2.3.2", 11 | "pandas>=2.3.1", 12 | "pyarrow<=17.0.0", 13 | "python-dateutil>=2.9.0.post0", 14 | ] 15 | -------------------------------------------------------------------------------- /pubsub-bullmq-node/src/utils/dragonfly.client.ts: -------------------------------------------------------------------------------- 1 | import { Redis as Dragonfly } from 'ioredis'; 2 | 3 | export const createDragonflyClient = (): Dragonfly => { 4 | return new Dragonfly({ 5 | host: process.env.DRAGONFLY_HOST || 'localhost', 6 | port: parseInt(process.env.DRAGONFLY_PORT || '6380', 10), 7 | 8 | // This is required by BullMQ workers. 9 | maxRetriesPerRequest: null, 10 | }); 11 | }; 12 | -------------------------------------------------------------------------------- /redlock-go/go.mod: -------------------------------------------------------------------------------- 1 | module example.com/m/v2 2 | 3 | go 1.20 4 | 5 | require github.com/redis/go-redis/v9 v9.4.0 6 | 7 | require ( 8 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 9 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 10 | github.com/go-redsync/redsync/v4 v4.11.0 // indirect 11 | github.com/hashicorp/errwrap v1.1.0 // indirect 12 | github.com/hashicorp/go-multierror v1.1.1 // indirect 13 | ) 14 | -------------------------------------------------------------------------------- /cache-in-5mins-hono/drizzle.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "drizzle-kit"; 2 | 3 | // For simplicity, we are using a local PostgreSQL instance. 4 | // Please ensure it is running locally and adjust the connection details as needed. 5 | export default defineConfig({ 6 | dialect: "postgresql", 7 | schema: "./src/schema.ts", 8 | dbCredentials: { 9 | url: "postgresql://local_user_dev:local_pwd_dev@localhost:5432/appdb", 10 | }, 11 | }); 12 | -------------------------------------------------------------------------------- /ad-server-cache-bun/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ad-server-cache-bun", 3 | "version": "1.0.50", 4 | "scripts": { 5 | "test": "echo \"Error: no test specified\" && exit 1", 6 | "dev": "bun run --watch src/index.ts" 7 | }, 8 | "dependencies": { 9 | "@sinclair/typebox": "^0.31.18", 10 | "elysia": "latest", 11 | "ioredis": "^5.3.2" 12 | }, 13 | "devDependencies": { 14 | "bun-types": "latest" 15 | }, 16 | "module": "src/index.js" 17 | } -------------------------------------------------------------------------------- /langchain-memory/database.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import create_engine 2 | from sqlalchemy.ext.declarative import declarative_base 3 | from sqlalchemy.orm import sessionmaker 4 | 5 | DB_URL = "sqlite:///./data.db" 6 | # DB_URL = "postgresql://user:password@postgresserver/db" 7 | 8 | engine = create_engine( 9 | DB_URL, connect_args={"check_same_thread": False} 10 | ) 11 | SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) 12 | 13 | Base = declarative_base() 14 | -------------------------------------------------------------------------------- /redis-cluster-application-example/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | services: 3 | dragonfly: 4 | image: 'docker.dragonflydb.io/dragonflydb/dragonfly' 5 | pull_policy: 'always' 6 | command: '--cluster_mode=emulated' 7 | ulimits: 8 | memlock: -1 9 | ports: 10 | - "6380:6379" 11 | volumes: 12 | - dragonflydata:/data 13 | rediscluster: 14 | build: . 15 | ports: 16 | - "7000:7000" 17 | - "7001:7001" 18 | - "7002:7002" 19 | volumes: 20 | dragonflydata: -------------------------------------------------------------------------------- /background-process-example/go.mod: -------------------------------------------------------------------------------- 1 | module mail-processing-pipeline 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/Pallinder/go-randomdata v1.2.0 7 | github.com/gocelery/gocelery v0.0.0-20201111034804-825d89059344 8 | github.com/gomodule/redigo v2.0.0+incompatible 9 | github.com/hashicorp/go-uuid v1.0.3 10 | ) 11 | 12 | require ( 13 | github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect 14 | github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271 // indirect 15 | golang.org/x/text v0.7.0 // indirect 16 | ) 17 | -------------------------------------------------------------------------------- /feast-recommendation-duckdb-dragonfly/recommendation_05_service.py: -------------------------------------------------------------------------------- 1 | from feast import FeatureService 2 | 3 | from recommendation_02_repo import ( 4 | user_features, 5 | item_features, 6 | interaction_features, 7 | ) 8 | 9 | # FeatureService combines FeatureViews we defined earlier for recommendation use cases. 10 | recommendation_feature_service = FeatureService( 11 | name="recommendation_service", 12 | features=[ 13 | user_features, 14 | item_features, 15 | interaction_features, 16 | ], 17 | ) 18 | -------------------------------------------------------------------------------- /langchain-memory/README.md: -------------------------------------------------------------------------------- 1 | ## How to Run 2 | 3 | - It is recommended to use a [virtual environment](https://docs.python.org/3/library/venv.html) to run this example. 4 | - Python 3.9.6+ is required. 5 | - pip v24.0.0+ is required. 6 | 7 | ```bash 8 | # OpenAI API is used in this example. 9 | # Thus, you need to have an API key and set it as an environment variable. 10 | export OPENAI_API_KEY={API_KEY} 11 | 12 | # Install dependencies. 13 | pip install -r requirements.txt 14 | 15 | # Run the server. 16 | uvicorn main:app --reload 17 | ``` 18 | -------------------------------------------------------------------------------- /feast-recommendation-duckdb-dragonfly/feature_store_docker.yaml: -------------------------------------------------------------------------------- 1 | # This file is the same as feature_store.yaml but with the online store connection string 2 | # set to "host.docker.internal:6379" to allow access from within a Docker container. 3 | # Assume Dragonfly is running on port 6379 within Docker on the host machine. 4 | project: feast_recommendation_duckdb_dragonfly 5 | registry: data/registry.db 6 | provider: local 7 | offline_store: 8 | type: duckdb 9 | online_store: 10 | type: redis 11 | connection_string: "host.docker.internal:6379" 12 | -------------------------------------------------------------------------------- /high-availability/sentinel/README.md: -------------------------------------------------------------------------------- 1 | # Example: Dragonfly High Availability with Sentinel 2 | 3 | - This example demonstrates how to set up Dragonfly with high availability using Sentinel, as part of our blog post [Keeping Dragonfly Always-On: High Availability Options Explained](https://www.dragonflydb.io/blog/dragonfly-high-availability-options-explained). 4 | - Make sure all commands are executed within the current repository: 5 | 6 | ```bash 7 | $> cd /PATH/TO/dragonfly-examples/high-availability/sentinel 8 | 9 | $> docker compose up -d 10 | ``` 11 | -------------------------------------------------------------------------------- /pubsub-bullmq-node/src/pubsub/pub.ts: -------------------------------------------------------------------------------- 1 | import { Redis as Dragonfly } from 'ioredis'; 2 | import { createDragonflyClient } from '../utils/dragonfly.client'; 3 | 4 | export class DragonflyPublisher { 5 | private pub: Dragonfly; 6 | private channel: string; 7 | 8 | constructor(channel: string) { 9 | this.pub = createDragonflyClient(); 10 | this.channel = channel; 11 | } 12 | 13 | publish(message: string) { 14 | this.pub.publish(this.channel, message); 15 | } 16 | 17 | close() { this.pub.quit(); } 18 | } 19 | -------------------------------------------------------------------------------- /ad-server-cache-bun/src/types.ts: -------------------------------------------------------------------------------- 1 | import {Type, Static} from '@sinclair/typebox' 2 | 3 | export const AdMetadata = Type.Object({ 4 | id: Type.String(), 5 | title: Type.String(), 6 | category: Type.String(), 7 | clickURL: Type.String(), 8 | imageURL: Type.String(), 9 | }); 10 | 11 | export type AdMetadata = Static; 12 | 13 | export const UserAdPreferences = Type.Object({ 14 | userId: Type.String(), 15 | categories: Type.Array(Type.String()), 16 | }); 17 | 18 | export type UserAdPreferences = Static; 19 | -------------------------------------------------------------------------------- /high-availability/k8s-operator/dragonfly-sample.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: dragonflydb.io/v1alpha1 2 | kind: Dragonfly 3 | metadata: 4 | labels: 5 | app.kubernetes.io/name: dragonfly 6 | app.kubernetes.io/instance: dragonfly-sample 7 | app.kubernetes.io/part-of: dragonfly-operator 8 | app.kubernetes.io/managed-by: kustomize 9 | app.kubernetes.io/created-by: dragonfly-operator 10 | name: dragonfly-sample 11 | spec: 12 | replicas: 3 13 | resources: 14 | requests: 15 | cpu: 8 16 | memory: 200Gi 17 | limits: 18 | cpu: 8 19 | memory: 300Gi 20 | -------------------------------------------------------------------------------- /real-time-statistics-node/src/utils/keyGenerator.ts: -------------------------------------------------------------------------------- 1 | // Utility function to construct a key for the current month. 2 | export const keyForCurrMonth = (prefix: String): string => { 3 | const now = new Date(); 4 | return keyForMonth(prefix, now); 5 | }; 6 | 7 | // Utility function to construct a key for a given month. 8 | export const keyForMonth = (prefix: String, date: Date): string => { 9 | // Zero-fill the month. 10 | const year = date.getFullYear(); 11 | const month = String(date.getMonth() + 1).padStart(2, '0'); 12 | return `${prefix}:${year}:${month}`; 13 | }; 14 | -------------------------------------------------------------------------------- /redis-cluster-application-example/README.md: -------------------------------------------------------------------------------- 1 | # README 2 | 3 | This repository contains sample code and scripts that accompany our [**Migrating from a Redis Cluster to a DragonflyDB on a single node**](https://www.dragonflydb.io/blog/migrating-from-a-redis-cluster-to-a-dragonfly-on-a-single-node) tutorial. 4 | 5 | In this tutorial, you learn how to migrate data from a Redis Cluster to a single-node DragonflyDB instance. You can use the sample application `/app` to demonstrate the migration process and cover everything step by step. 6 | 7 | Please refer to the tutorial for step by step instructions. 8 | -------------------------------------------------------------------------------- /user-session-management-go/app/handler/password.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "fmt" 5 | 6 | "golang.org/x/crypto/bcrypt" 7 | ) 8 | 9 | func hashPassword(password string) (hash string, err error) { 10 | hashedBytes, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) 11 | if err != nil { 12 | return "", fmt.Errorf("failed to hash password: %w", err) 13 | } 14 | hash = string(hashedBytes) 15 | return hash, nil 16 | } 17 | 18 | func verifyPassword(password, hash string) error { 19 | return bcrypt.CompareHashAndPassword([]byte(hash), []byte(password)) 20 | } 21 | -------------------------------------------------------------------------------- /pubsub-bullmq-node/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pubsub-bullmq-node", 3 | "version": "0.0.1", 4 | "description": "Messaging & Event Streaming with Dragonfly using Pub/Sub and BullMQ", 5 | "main": "index.js", 6 | "scripts": { 7 | "start": "ts-node src/index.ts", 8 | "dev": "ts-node-dev src/index.ts" 9 | }, 10 | "dependencies": { 11 | "bullmq": "^5.50.0", 12 | "express": "^4.21.2", 13 | "ioredis": "^5.6.0" 14 | }, 15 | "devDependencies": { 16 | "@types/express": "^4.17.21", 17 | "ts-node": "^10.9.1", 18 | "ts-node-dev": "^2.0.0", 19 | "typescript": "^5.8.2" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /pubsub-bullmq-node/src/utils/hashtag.ts: -------------------------------------------------------------------------------- 1 | export const containsExactlyOneHashtag = (str?: string): boolean => { 2 | if (!str || str.trim() === '') { 3 | return false; 4 | } 5 | 6 | const openCount = (str.match(/\{/g) || []).length; 7 | const closeCount = (str.match(/\}/g) || []).length; 8 | 9 | const openIndex = str.indexOf('{'); 10 | const closeIndex = str.indexOf('}'); 11 | 12 | // 1. Exactly one opening brace. 13 | // 2. Exactly one closing brace. 14 | // 3. Opening brace must come before closing brace. 15 | return openCount === 1 && closeCount === 1 && openIndex < closeIndex; 16 | } 17 | -------------------------------------------------------------------------------- /user-session-management-go/app/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | 6 | "usm/app/handler" 7 | 8 | "github.com/gofiber/fiber/v2" 9 | "github.com/gofiber/fiber/v2/middleware/logger" 10 | _ "github.com/mattn/go-sqlite3" 11 | ) 12 | 13 | func main() { 14 | app := fiber.New() 15 | app.Use(logger.New()) 16 | 17 | app.Post("/user/register", handler.UserRegistration) 18 | app.Post("/user/login", handler.UserLogin) 19 | app.Post("/user/logout", handler.UserLogout) 20 | app.Post("/user/refresh-token", handler.UserRefreshToken) 21 | app.Get("/resource", handler.UserReadResource) 22 | 23 | log.Fatal(app.Listen(":8080")) 24 | } 25 | -------------------------------------------------------------------------------- /real-time-statistics-node/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "real-time-statistics-node", 3 | "version": "0.0.1", 4 | "description": "Real-Time Statistics using HyperLogLog and Sorted Set in Dragonfly", 5 | "main": "index.js", 6 | "scripts": { 7 | "start": "ts-node src/index.ts", 8 | "dev": "ts-node-dev src/index.ts" 9 | }, 10 | "dependencies": { 11 | "express": "^4.21.2", 12 | "ioredis": "^5.6.0" 13 | }, 14 | "devDependencies": { 15 | "@types/express": "^4.17.21", 16 | "ts-node": "^10.9.1", 17 | "ts-node-dev": "^2.0.0", 18 | "typescript": "^5.8.2" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /ad-server-cache-bun/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # next.js 12 | /.next/ 13 | /out/ 14 | 15 | # production 16 | /build 17 | 18 | # misc 19 | .DS_Store 20 | *.pem 21 | 22 | # debug 23 | npm-debug.log* 24 | yarn-debug.log* 25 | yarn-error.log* 26 | 27 | # local env files 28 | .env.local 29 | .env.development.local 30 | .env.test.local 31 | .env.production.local 32 | 33 | # vercel 34 | .vercel 35 | 36 | **/*.trace 37 | **/*.zip 38 | **/*.tar.gz 39 | **/*.tgz 40 | **/*.log 41 | package-lock.json 42 | **/*.bun -------------------------------------------------------------------------------- /high-availability/README.md: -------------------------------------------------------------------------------- 1 | # Example: Dragonfly High Availability Setups 2 | 3 | This sub-repository demonstrates how to set up Dragonfly for high availability using the following ways: 4 | 5 | - [Redis Sentinel](https://redis.io/docs/latest/operate/oss_and_stack/management/sentinel/) 6 | - [Dragonfly Kubernetes Operator](https://www.dragonflydb.io/docs/managing-dragonfly/operator/installation) 7 | - [Dragonfly Cloud](https://www.dragonflydb.io/docs/cloud/datastores#high-availability) 8 | 9 | These examples serve as a companion to our blog post [Keeping Dragonfly Always-On: High Availability Options Explained](https://www.dragonflydb.io/blog/dragonfly-high-availability-options-explained). 10 | -------------------------------------------------------------------------------- /langchain-memory/models.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE chat_sessions 2 | ( 3 | id INTEGER PRIMARY KEY, 4 | llm_name TEXT NOT NULL 5 | ); 6 | 7 | CREATE TABLE chat_histories 8 | ( 9 | id INTEGER PRIMARY KEY, 10 | chat_session_id INTEGER NOT NULL, 11 | is_human_message INTEGER NOT NULL, 12 | content TEXT NOT NULL, 13 | metadata_completion_tokens INTEGER, 14 | metadata_prompt_tokens INTEGER, 15 | metadata_total_tokens INTEGER, 16 | metadata_system_fingerprint INTEGER, 17 | external_id TEXT, 18 | FOREIGN KEY (chat_session_id) REFERENCES chat_sessions (id) 19 | ); 20 | -------------------------------------------------------------------------------- /user-session-management-go/app/handler/db.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "database/sql" 5 | "log" 6 | 7 | _ "github.com/mattn/go-sqlite3" 8 | ) 9 | 10 | var db *sql.DB 11 | 12 | func init() { 13 | // Initialize the SQLite database. 14 | var err error 15 | db, err = sql.Open("sqlite3", "./app.db") 16 | if err != nil { 17 | log.Fatalf("failed to connect to database: %v", err) 18 | } 19 | 20 | // Create the 'users' table. 21 | _, err = db.Exec(` 22 | CREATE TABLE IF NOT EXISTS users ( 23 | id TEXT PRIMARY KEY, 24 | username TEXT UNIQUE, 25 | password_hash TEXT 26 | ); 27 | `) 28 | if err != nil { 29 | log.Fatalf("failed to initialize database: %v", err) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /feast-recommendation-duckdb-dragonfly/recommendation_04_online_features.py: -------------------------------------------------------------------------------- 1 | from feast import FeatureStore 2 | 3 | 4 | def fetch_online_features(): 5 | store = FeatureStore(repo_path=".") 6 | 7 | online_features = store.get_online_features( 8 | features=[ 9 | "user_features:age", 10 | "user_features:gender", 11 | "item_features:price", 12 | "interaction_features:view_count" 13 | ], 14 | entity_rows=[ 15 | {"user_id": 1, "item_id": 101}, 16 | {"user_id": 2, "item_id": 102} 17 | ] 18 | ).to_dict() 19 | 20 | print(online_features) 21 | 22 | 23 | if __name__ == "__main__": 24 | fetch_online_features() 25 | -------------------------------------------------------------------------------- /langchain-memory/schemas.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from pydantic import BaseModel 4 | 5 | 6 | # For creating new sessions, excluding IDs and relations. 7 | class ChatSessionCreate(BaseModel): 8 | llm_name: str 9 | 10 | 11 | class ChatMessageCreate(BaseModel): 12 | content: str 13 | 14 | 15 | # For creating new history entries, excluding IDs and relations. 16 | class ChatHistoryCreate(ChatMessageCreate): 17 | is_human_message: bool 18 | metadata_completion_tokens: Optional[int] = None 19 | metadata_prompt_tokens: Optional[int] = None 20 | metadata_total_tokens: Optional[int] = None 21 | metadata_system_fingerprint: Optional[str] = None 22 | external_id: Optional[str] = None 23 | -------------------------------------------------------------------------------- /redis-cluster-application-example/start-cluster.sh: -------------------------------------------------------------------------------- 1 | cd /data 2 | mkdir 7000 7001 7002 3 | 4 | redis-server --port 7000 --cluster-enabled yes --protected-mode no --cluster-config-file 7000/nodes.conf --cluster-node-timeout 5000 >/dev/null& 5 | sleep 1 6 | redis-server --port 7001 --cluster-enabled yes --protected-mode no --cluster-config-file 7001/nodes.conf --cluster-node-timeout 5000 >/dev/null& 7 | sleep 1 8 | redis-server --port 7002 --cluster-enabled yes --protected-mode no --cluster-config-file 7002/nodes.conf --cluster-node-timeout 5000 >/dev/null& 9 | sleep 1 10 | 11 | redis-cli --cluster create 127.0.0.1:7000 127.0.0.1:7001 127.0.0.1:7002 --cluster-yes >/dev/null 12 | 13 | echo "Redis Cluster ready" 14 | 15 | exec "$@" 16 | -------------------------------------------------------------------------------- /leaderboard/app/model/user.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | type User struct { 4 | ID string `json:"id" redis:"id"` 5 | Email string `json:"email,omitempty" redis:"email"` 6 | NickName string `json:"nick_name,omitempty" redis:"nick_name"` 7 | ImageURL string `json:"image_url,omitempty" redis:"image_url"` 8 | } 9 | 10 | type UserIncreaseScoreRequest struct { 11 | User `json:",inline"` 12 | Score float64 `json:"score"` 13 | } 14 | 15 | type LeaderboardUser struct { 16 | User `json:",inline"` 17 | Score float64 `json:"score"` 18 | Rank int64 `json:"rank"` 19 | } 20 | 21 | type LeaderboardResponse struct { 22 | TopUsers []LeaderboardUser `json:"topUsers"` 23 | CurrentUser LeaderboardUser `json:"currentUser"` 24 | } 25 | -------------------------------------------------------------------------------- /background-process-example/README.md: -------------------------------------------------------------------------------- 1 | # background-process-example 2 | 3 | This repository contains sample code that accompanies our [**Building a background processing pipeline with DragonflyDB**](https://www.dragonflydb.io/blog/building-a-background-processing-pipeline-with-dragonfly) tutorial. 4 | 5 | In particular, this repository contains the sample code for: 6 | 7 | - A producer application that simulates a website user registration process. 8 | - A worker application that processes these user registration requests and sends welcome emails. 9 | 10 | In this tutorial, you learn how to use Redis Lists to build a background processing pipeline with DragonflyDB. 11 | 12 | Please refer to the tutorial for step by step instructions. 13 | 14 | 15 | -------------------------------------------------------------------------------- /cache-in-5mins-hono/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cache-in-5mins-hono", 3 | "scripts": { 4 | "dev": "bun run --hot src/index.ts", 5 | "dev-node": "tsx watch src/index.ts" 6 | }, 7 | "dependencies": { 8 | "@hono/node-server": "^1.15.0", 9 | "@hono/zod-validator": "^0.7.0", 10 | "drizzle": "^1.4.0", 11 | "drizzle-kit": "^0.31.4", 12 | "drizzle-orm": "^0.44.2", 13 | "drizzle-zod": "^0.8.2", 14 | "hono": "^4.8.3", 15 | "ioredis": "^5.6.1", 16 | "pg": "^8.16.3", 17 | "uuid": "^11.1.0", 18 | "zod": "^3.25.74" 19 | }, 20 | "devDependencies": { 21 | "@types/bun": "latest", 22 | "@types/node": "^20.11.17", 23 | "@types/pg": "^8.15.4", 24 | "tsx": "^4.7.1", 25 | "typescript": "^5.8.3" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /leaderboard/go.sum: -------------------------------------------------------------------------------- 1 | github.com/bsm/ginkgo/v2 v2.9.5 h1:rtVBYPs3+TC5iLUVOis1B9tjLTup7Cj5IfzosKtvTJ0= 2 | github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= 3 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 4 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 5 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 6 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 7 | github.com/redis/go-redis/v9 v9.1.0 h1:137FnGdk+EQdCbye1FW+qOEcY5S+SpY9T0NiuqvtfMY= 8 | github.com/redis/go-redis/v9 v9.1.0/go.mod h1:urWj3He21Dj5k4TK1y59xH8Uj6ATueP8AH1cY3lZl4c= 9 | -------------------------------------------------------------------------------- /monitoring-in-memory-stores/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | 5 | dragonfly: 6 | image: 'docker.dragonflydb.io/dragonflydb/dragonfly' 7 | #pull_policy: 'always' 8 | ulimits: 9 | memlock: -1 10 | ports: 11 | - "6379:6379" 12 | volumes: 13 | - dragonflydata:/data 14 | 15 | prometheus: 16 | image: prom/prometheus:latest 17 | restart: always 18 | ports: 19 | - "9090:9090" 20 | volumes: 21 | - .:/etc/prometheus 22 | command: 23 | - '--config.file=/etc/prometheus/prometheus.yml' 24 | depends_on: 25 | - dragonfly 26 | 27 | grafana: 28 | image: grafana/grafana:latest 29 | restart: always 30 | ports: 31 | - "3000:3000" 32 | 33 | volumes: 34 | dragonflydata: -------------------------------------------------------------------------------- /ecommerce-cache/load/go.sum: -------------------------------------------------------------------------------- 1 | github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao= 2 | github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= 3 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 4 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 5 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 6 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 7 | github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o= 8 | github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= 9 | -------------------------------------------------------------------------------- /ecommerce-cache/app/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "net/http" 8 | 9 | _ "github.com/mattn/go-sqlite3" 10 | "github.com/redis/go-redis/v9" 11 | ) 12 | 13 | var rdb *redis.Client 14 | 15 | func init() { 16 | 17 | rdb = redis.NewClient(&redis.Options{ 18 | Addr: "localhost:6379", 19 | }) 20 | 21 | err := rdb.Ping(context.Background()).Err() 22 | if err != nil { 23 | log.Fatal("failed to connect with redis", err) 24 | } 25 | 26 | fmt.Println("connected to redis") 27 | 28 | } 29 | 30 | func main() { 31 | 32 | http.HandleFunc("/orders", getOrders) 33 | http.HandleFunc("/topviewed", getMostViewedItems) 34 | http.HandleFunc("/recommendations", getRecommendations) 35 | 36 | log.Fatal(http.ListenAndServe(":8080", nil)) 37 | } 38 | -------------------------------------------------------------------------------- /ecommerce-cache/app/go.sum: -------------------------------------------------------------------------------- 1 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 2 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 3 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 4 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 5 | github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= 6 | github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= 7 | github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o= 8 | github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= 9 | -------------------------------------------------------------------------------- /cache-in-5mins-hono/src/schema.ts: -------------------------------------------------------------------------------- 1 | import { pgTable, uuid, varchar, timestamp } from "drizzle-orm/pg-core"; 2 | 3 | // Table schema for 'short_links'. 4 | export const shortLinksTable = pgTable("short_links", { 5 | id: uuid().primaryKey(), 6 | // The length varies for different browsers and systems. 4096 is a safe value that covers most use cases and is the default value for NGINX. 7 | originalUrl: varchar("original_url", { length: 4096 }).notNull(), 8 | // In our implementation, the short code is the base64-encoded ID (which is a UUID) without padding, taking 22 characters. 9 | shortCode: varchar("short_code", { length: 30 }).notNull(), 10 | createdAt: timestamp("created_at", { withTimezone: true }).notNull(), 11 | expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), 12 | }); 13 | -------------------------------------------------------------------------------- /ad-server-cache/go.sum: -------------------------------------------------------------------------------- 1 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 2 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 3 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 4 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 5 | github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= 6 | github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= 7 | github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o= 8 | github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= 9 | -------------------------------------------------------------------------------- /celery-transaction-processing/README.md: -------------------------------------------------------------------------------- 1 | ```bash 2 | pip install -r requirements.txt 3 | ``` 4 | 5 | ```bash 6 | uvicorn main:api_app --reload 7 | ``` 8 | 9 | ```bash 10 | celery -A tasks worker --loglevel=INFO 11 | ``` 12 | 13 | - A Web3 node provider URL (i.e., Infura) is required to interact with the Ethereum network. 14 | The URL should be set as the `DF_WEB3_PROVIDER_URI` environment variable. 15 | **NOTE: To test the application locally, you should use the Ethereum testnet (Sepolia) node provider URL.** 16 | - An Ethereum wallet private key is required to sign transactions. 17 | The private key should be set as the `DF_SYSTEM_ACCOUNT_PRIVATE_KEY` environment variable. 18 | **NOTE: The private key should be kept secret, ideally encrypted, and NOT shared with others.** 19 | **Do NOT use a wallet with real funds for testing purposes.** 20 | -------------------------------------------------------------------------------- /cache-refresh-ahead/go.mod: -------------------------------------------------------------------------------- 1 | module example.com/m/v2 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/gofiber/fiber/v2 v2.49.1 7 | github.com/redis/go-redis/v9 v9.1.0 8 | ) 9 | 10 | require ( 11 | github.com/andybalholm/brotli v1.0.5 // indirect 12 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 13 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 14 | github.com/google/uuid v1.3.1 // indirect 15 | github.com/klauspost/compress v1.16.7 // indirect 16 | github.com/mattn/go-colorable v0.1.13 // indirect 17 | github.com/mattn/go-isatty v0.0.19 // indirect 18 | github.com/mattn/go-runewidth v0.0.15 // indirect 19 | github.com/rivo/uniseg v0.2.0 // indirect 20 | github.com/valyala/bytebufferpool v1.0.0 // indirect 21 | github.com/valyala/fasthttp v1.49.0 // indirect 22 | github.com/valyala/tcplisten v1.0.0 // indirect 23 | golang.org/x/sys v0.11.0 // indirect 24 | ) 25 | -------------------------------------------------------------------------------- /high-availability/k8s-operator/README.md: -------------------------------------------------------------------------------- 1 | # Example: Dragonfly High Availability with Kubernetes Operator 2 | 3 | - This example demonstrates how to set up Dragonfly with high availability using its Kubernetes Operator, as part of our blog post [Keeping Dragonfly Always-On: High Availability Options Explained](https://www.dragonflydb.io/blog/dragonfly-high-availability-options-explained). 4 | - Firstly, make sure your Kubernetes cluster is up and running. 5 | - To install Dragonfly Operator, run the following command: 6 | 7 | ```bash 8 | $> kubectl apply -f https://raw.githubusercontent.com/dragonflydb/dragonfly-operator/main/manifests/dragonfly-operator.yaml 9 | ``` 10 | 11 | - Use the example YAML file to run a Dragonfly high availability setup (1 primary, 2 replicas): 12 | 13 | ```bash 14 | $> cd /PATH/TO/dragonfly-examples/high-availability/k8s-operator 15 | 16 | $> kubectl apply -f dragonfly-sample.yaml 17 | ``` 18 | -------------------------------------------------------------------------------- /real-time-statistics-node/src/middleware/activeUser.ts: -------------------------------------------------------------------------------- 1 | import { Request, Response, NextFunction } from 'express' 2 | import { Redis as Dragonfly } from 'ioredis' 3 | import { keyForCurrMonth } from '../utils/keyGenerator' 4 | 5 | export const MONTHLY_ACTIVE_USER_PREFIX = 'monthly_active_users' 6 | 7 | // Middleware to track monthly active users. 8 | export const trackMonthlyActiveUsers = (dragonfly: Dragonfly) => { 9 | return async (req: Request, res: Response, next: NextFunction) => { 10 | // Assume userId is passed in the request body. 11 | const userId = req.body.userId 12 | if (!userId) { 13 | return res.status(400).json({ error: 'userId is required' }) 14 | } 15 | 16 | // Add user to the active user key (HyperLogLog) for the current month. 17 | const key = keyForCurrMonth(MONTHLY_ACTIVE_USER_PREFIX) 18 | await dragonfly.pfadd(key, userId) 19 | 20 | console.log(`Tracked user ${userId} for ${key}`) 21 | next() 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /sentinel-migration/go.sum: -------------------------------------------------------------------------------- 1 | github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao= 2 | github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= 3 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 4 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 5 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 6 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 7 | github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= 8 | github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= 9 | github.com/redis/go-redis/v9 v9.0.4 h1:FC82T+CHJ/Q/PdyLW++GeCO+Ol59Y4T7R4jbgjvktgc= 10 | github.com/redis/go-redis/v9 v9.0.4/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= 11 | -------------------------------------------------------------------------------- /feast-recommendation-duckdb-dragonfly/recommendation_03_historical_features.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from feast import FeatureStore 3 | 4 | 5 | def fetch_historical_features(): 6 | store = FeatureStore(repo_path=".") 7 | 8 | # Build an entity dataframe as input. 9 | entity_df = pd.DataFrame({ 10 | "user_id": [1, 2], 11 | "item_id": [101, 102], 12 | "event_timestamp": [pd.Timestamp("2025-08-28 00:00:00", tz="UTC") for _ in range(2)] 13 | }) 14 | 15 | # Specify which features to retrieve. 16 | features = [ 17 | "user_features:age", 18 | "user_features:gender", 19 | "item_features:price", 20 | "interaction_features:view_count" 21 | ] 22 | 23 | historical_df = store.get_historical_features( 24 | entity_df=entity_df, 25 | features=features 26 | ).to_df() 27 | 28 | print(historical_df) 29 | 30 | 31 | if __name__ == "__main__": 32 | fetch_historical_features() 33 | -------------------------------------------------------------------------------- /redlock-go/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | dragonfly-instance-0: 4 | container_name: "dragonfly-instance-0" 5 | image: 'ghcr.io/dragonflydb/dragonfly:v1.14.3-ubuntu' 6 | ulimits: 7 | memlock: -1 8 | ports: 9 | - "6379:6379" 10 | dragonfly-instance-1: 11 | container_name: "dragonfly-instance-1" 12 | image: 'ghcr.io/dragonflydb/dragonfly:v1.14.3-ubuntu' 13 | ulimits: 14 | memlock: -1 15 | ports: 16 | - "6380:6379" 17 | dragonfly-instance-2: 18 | container_name: "dragonfly-instance-2" 19 | image: 'ghcr.io/dragonflydb/dragonfly:v1.14.3-ubuntu' 20 | ulimits: 21 | memlock: -1 22 | ports: 23 | - "6381:6379" 24 | redlock-service: 25 | container_name: "redlock-service" 26 | build: 27 | context: . 28 | dockerfile: Dockerfile 29 | depends_on: 30 | - dragonfly-instance-0 31 | - dragonfly-instance-1 32 | - dragonfly-instance-2 33 | ports: 34 | - "8080:8080" 35 | -------------------------------------------------------------------------------- /cache-aside-with-expiration/app/cache_store.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/redis/go-redis/v9" 8 | ) 9 | 10 | // CacheStorage implements the fiber.Storage interface. 11 | type CacheStorage struct { 12 | client *redis.Client 13 | } 14 | 15 | func (m *CacheStorage) Get(key string) ([]byte, error) { 16 | // Use the actual GET command of Dragonfly. 17 | return m.client.Get(context.Background(), key).Bytes() 18 | } 19 | 20 | func (m *CacheStorage) Set(key string, val []byte, exp time.Duration) error { 21 | // Use the actual SET command of Dragonfly. 22 | return m.client.Set(context.Background(), key, val, exp).Err() 23 | } 24 | 25 | func (m *CacheStorage) Delete(key string) error { 26 | // Use the actual DEL command of Dragonfly. 27 | return m.client.Del(context.Background(), key).Err() 28 | } 29 | 30 | func (m *CacheStorage) Close() error { 31 | return m.client.Close() 32 | } 33 | 34 | func (m *CacheStorage) Reset() error { 35 | return nil 36 | } 37 | -------------------------------------------------------------------------------- /cache-in-5mins-hono/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | dragonfly: 3 | image: "docker.dragonflydb.io/dragonflydb/dragonfly" 4 | container_name: "cache-with-hono-dragonfly" 5 | ulimits: 6 | memlock: -1 7 | ports: 8 | - "6380:6379" 9 | command: 10 | # The following configs are friendly for local development. 11 | # In production, Dragonfly can utilize many more CPU cores and much more memory with ease. 12 | - "--proactor_threads=2" 13 | - "--maxmemory=2GB" 14 | redis: 15 | image: "redis:latest" 16 | container_name: "cache-with-hono-redis" 17 | ports: 18 | - "6379:6379" 19 | postgres: 20 | image: "postgres:17" 21 | container_name: "cache-with-hono-postgres" 22 | environment: 23 | POSTGRES_USER: local_user_dev 24 | POSTGRES_PASSWORD: local_pwd_dev 25 | POSTGRES_DB: appdb 26 | ports: 27 | - "5432:5432" 28 | volumes: 29 | - pgdata:/var/lib/postgresql/data 30 | 31 | volumes: 32 | pgdata: 33 | -------------------------------------------------------------------------------- /user-session-management-go/go.mod: -------------------------------------------------------------------------------- 1 | module usm 2 | 3 | go 1.22.2 4 | 5 | require ( 6 | github.com/gofiber/fiber/v2 v2.52.5 7 | github.com/golang-jwt/jwt/v5 v5.2.1 8 | github.com/google/uuid v1.6.0 9 | github.com/mattn/go-sqlite3 v1.14.24 10 | github.com/redis/go-redis/v9 v9.7.0 11 | golang.org/x/crypto v0.30.0 12 | ) 13 | 14 | require ( 15 | github.com/andybalholm/brotli v1.1.1 // indirect 16 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 17 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 18 | github.com/klauspost/compress v1.17.11 // indirect 19 | github.com/mattn/go-colorable v0.1.13 // indirect 20 | github.com/mattn/go-isatty v0.0.20 // indirect 21 | github.com/mattn/go-runewidth v0.0.16 // indirect 22 | github.com/rivo/uniseg v0.4.7 // indirect 23 | github.com/valyala/bytebufferpool v1.0.0 // indirect 24 | github.com/valyala/fasthttp v1.57.0 // indirect 25 | github.com/valyala/tcplisten v1.0.0 // indirect 26 | golang.org/x/sys v0.28.0 // indirect 27 | ) 28 | -------------------------------------------------------------------------------- /cache-aside-with-expiration/go.mod: -------------------------------------------------------------------------------- 1 | module example.com/m/v2 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/gofiber/fiber/v2 v2.49.0 7 | github.com/redis/go-redis/v9 v9.1.0 8 | ) 9 | 10 | require ( 11 | github.com/andybalholm/brotli v1.0.5 // indirect 12 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 13 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 14 | github.com/google/uuid v1.3.1 // indirect 15 | github.com/klauspost/compress v1.16.7 // indirect 16 | github.com/mattn/go-colorable v0.1.13 // indirect 17 | github.com/mattn/go-isatty v0.0.19 // indirect 18 | github.com/mattn/go-runewidth v0.0.15 // indirect 19 | github.com/philhofer/fwd v1.1.2 // indirect 20 | github.com/rivo/uniseg v0.2.0 // indirect 21 | github.com/tinylib/msgp v1.1.8 // indirect 22 | github.com/valyala/bytebufferpool v1.0.0 // indirect 23 | github.com/valyala/fasthttp v1.48.0 // indirect 24 | github.com/valyala/tcplisten v1.0.0 // indirect 25 | golang.org/x/sys v0.11.0 // indirect 26 | ) 27 | -------------------------------------------------------------------------------- /pubsub-bullmq-node/src/bullmq-keys.txt: -------------------------------------------------------------------------------- 1 | # This is an example showing how BullMQ uses multiple keys of different Dragonfly data types to manage a single queue. 2 | 3 | dragonfly$> KEYS "{my-prefix}:*" 4 | 1) "{my-prefix}:my-queue:events" 5 | 2) "{my-prefix}:my-queue:repeat:dc8f96808dbf0ca8802f8088560e1ac4:1745770085000" 6 | 3) "{my-prefix}:my-queue:1" 7 | 4) "{my-prefix}:my-queue:id" 8 | 5) "{my-prefix}:my-queue:2" 9 | 6) "{my-prefix}:my-queue:meta" 10 | 7) "{my-prefix}:my-queue:3" 11 | 8) "{my-prefix}:my-queue:repeat:dc8f96808dbf0ca8802f8088560e1ac4" 12 | 9) "{my-prefix}:my-queue:stalled-check" 13 | 10) "{my-prefix}:my-queue:repeat:dc8f96808dbf0ca8802f8088560e1ac4:1745770080000" 14 | 11) "{my-prefix}:my-queue:repeat" 15 | 12) "{my-prefix}:my-queue:repeat:dc8f96808dbf0ca8802f8088560e1ac4:1745770090000" 16 | 13) "{my-prefix}:my-queue:completed" 17 | 18 | dragonfly$> TYPE "{my-prefix}:my-queue:events" 19 | stream 20 | dragonfly$> TYPE "{my-prefix}:my-queue:meta" 21 | hash 22 | dragonfly$> TYPE "{my-prefix}:my-queue:completed" 23 | zset 24 | -------------------------------------------------------------------------------- /celery-transaction-processing/models.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE user_accounts 2 | ( 3 | id INTEGER PRIMARY KEY, 4 | available_balance_in_wei INTEGER NOT NULL, 5 | current_balance_in_wei INTEGER NOT NULL 6 | ); 7 | 8 | CREATE TABLE user_account_transactions 9 | ( 10 | id INTEGER PRIMARY KEY, 11 | user_account_id INTEGER NOT NULL, 12 | transaction_hash TEXT NOT NULL, 13 | from_public_address TEXT NOT NULL, 14 | to_public_address TEXT NOT NULL, 15 | transaction_amount_in_wei INTEGER NOT NULL, 16 | transaction_fee_total_in_wei INTEGER NOT NULL, 17 | transaction_fee_blockchain_in_wei INTEGER NOT NULL, 18 | status TEXT NOT NULL DEFAULT 'PENDING' CHECK (status IN ('PENDING', 'SUCCESSFUL', 'FAILED')), 19 | 20 | FOREIGN KEY (user_account_id) REFERENCES user_accounts (id) 21 | ); 22 | 23 | CREATE INDEX idx_user_account_id ON user_account_transactions (user_account_id); 24 | -------------------------------------------------------------------------------- /ecommerce-cache/app/recommendations.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "strconv" 9 | ) 10 | 11 | const recommendations = "user:42:recommendations" 12 | 13 | func init() { 14 | 15 | //we are loading sample data. but imagine this being populated in real-time while user is interacting with the ecommerce system 16 | 17 | for i := 1; i <= 5; i++ { 18 | rdb.SAdd(context.Background(), recommendations, "product-"+strconv.Itoa(i)) 19 | } 20 | 21 | fmt.Println("loaded recommendations sample data") 22 | } 23 | 24 | func getRecommendations(rw http.ResponseWriter, req *http.Request) { 25 | 26 | items := rdb.SMembers(context.Background(), recommendations).Val() 27 | 28 | resp := RecommendationsResponse{UserID: 42, Items: items} 29 | 30 | err := json.NewEncoder(rw).Encode(resp) 31 | if err != nil { 32 | http.Error(rw, err.Error(), 500) 33 | return 34 | } 35 | } 36 | 37 | type RecommendationsResponse struct { 38 | UserID int `json:"user_id"` 39 | Items []string `json:"recommended"` 40 | } 41 | -------------------------------------------------------------------------------- /langchain-memory/models.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Boolean, Column, ForeignKey, Integer, String 2 | from sqlalchemy.orm import relationship 3 | 4 | from database import Base 5 | 6 | 7 | class ChatSession(Base): 8 | __tablename__ = "chat_sessions" 9 | 10 | id = Column(Integer, primary_key=True) 11 | llm_name = Column(String, nullable=False) 12 | 13 | chat_histories = relationship("ChatHistory", back_populates="chat_session") 14 | 15 | 16 | class ChatHistory(Base): 17 | __tablename__ = "chat_histories" 18 | 19 | id = Column(Integer, primary_key=True) 20 | chat_session_id = Column(Integer, ForeignKey("chat_sessions.id"), nullable=False) 21 | is_human_message = Column(Boolean, nullable=False) 22 | content = Column(String, nullable=False) 23 | metadata_completion_tokens = Column(Integer) 24 | metadata_prompt_tokens = Column(Integer) 25 | metadata_total_tokens = Column(Integer) 26 | metadata_system_fingerprint = Column(String) 27 | external_id = Column(String) 28 | 29 | chat_session = relationship("ChatSession", back_populates="chat_histories") 30 | -------------------------------------------------------------------------------- /pubsub-bullmq-node/src/mq/queue.ts: -------------------------------------------------------------------------------- 1 | import { Queue, QueueOptions } from 'bullmq'; 2 | import { createDragonflyClient } from '../utils/dragonfly.client'; 3 | import { containsExactlyOneHashtag } from '../utils/hashtag'; 4 | 5 | type DragonflyQueueOptions = Omit; 6 | 7 | export class DragonflyQueue extends Queue { 8 | private constructor(queueName: string, opts?: QueueOptions) { 9 | super(queueName, opts); 10 | } 11 | 12 | // Factory method that sanitizes the queue name and prefix for a Dragonfly-backed BullMQ queue. 13 | static create(queueName: string, opts?: DragonflyQueueOptions): DragonflyQueue { 14 | const fullQueueName = opts?.prefix ? `${opts.prefix}:${queueName}` : queueName; 15 | if (!containsExactlyOneHashtag(fullQueueName)) { 16 | throw new Error('The queue name (with prefix if provided) must contain exactly one hashtag'); 17 | } 18 | 19 | const connection = createDragonflyClient(); 20 | const queueOptions = { 21 | ...opts, 22 | connection 23 | }; 24 | 25 | return new DragonflyQueue(queueName, queueOptions); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /real-time-statistics-node/src/middleware/leaderboard.ts: -------------------------------------------------------------------------------- 1 | import { Request, Response, NextFunction } from 'express'; 2 | import { Redis as Dragonfly } from 'ioredis'; 3 | import { keyForCurrMonth } from '../utils/keyGenerator'; 4 | 5 | export const MONTHLY_LEADERBOARD_PREFIX = 'monthly_leaderboard'; 6 | 7 | // Middleware to add points to user's leaderboard score. 8 | export const addLeaderboardPoints = (dragonfly: Dragonfly) => { 9 | return async (req: Request, res: Response, next: NextFunction) => { 10 | // Assume userId is passed in the request body 11 | const userId = req.body.userId; 12 | if (!userId) { 13 | return res.status(400).json({ error: 'userId is required' }); 14 | } 15 | 16 | // Add 10 points for the user to the leaderboard key (Sorted Set) for the current month. 17 | // Of course, you can define different points for different user actions. 18 | const leaderboardKey = keyForCurrMonth(MONTHLY_LEADERBOARD_PREFIX) 19 | await dragonfly.zincrby(leaderboardKey, 10, userId); 20 | 21 | console.log(`Added 10 points for user ${userId} in ${leaderboardKey}`); 22 | next(); 23 | }; 24 | }; 25 | -------------------------------------------------------------------------------- /feast-recommendation-duckdb-dragonfly/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11.9-slim AS base 2 | 3 | ## Builder Stage ## 4 | FROM base AS builder 5 | 6 | # Install uv, the Python package manager that actually works. 7 | COPY --from=ghcr.io/astral-sh/uv:0.8.12 /uv /bin/uv 8 | WORKDIR /app 9 | 10 | # Copy uv files for better cache. 11 | COPY uv.lock pyproject.toml /app/ 12 | 13 | # Install dependencies using uv. 14 | RUN uv sync --frozen --no-dev 15 | 16 | # Copy the rest of the application code (necessary files only) to run the Feast server. 17 | # Note that we use 'feature_store_docker.yaml' to allow the container to talk to the Dragonfly 18 | # server running on port 6379 within Docker on the host machine. 19 | RUN mkdir -p /app/data 20 | COPY data/registry.db /app/data/registry.db 21 | COPY feature_store_docker.yaml /app/feature_store.yaml 22 | COPY recommendation_02_repo.py recommendation_05_service.py /app/ 23 | 24 | ## Runtime Stage ## 25 | FROM base 26 | COPY --from=builder /app /app 27 | WORKDIR /app 28 | 29 | # Activate the virtual environment created by uv. 30 | ENV PATH="/app/.venv/bin:$PATH" 31 | 32 | # Run the Feast server. 33 | CMD ["feast", "serve", "--host", "0.0.0.0", "--port", "6566"] 34 | -------------------------------------------------------------------------------- /ad-server-cache-bun/src/index.ts: -------------------------------------------------------------------------------- 1 | import {Elysia} from "elysia"; 2 | import {Redis as Dragonfly} from 'ioredis'; 3 | 4 | import {AdMetadataStore} from "./ads"; 5 | import {AdMetadata, UserAdPreferences} from "./types"; 6 | 7 | const client = new Dragonfly(); 8 | 9 | const app = new Elysia() 10 | .decorate("adMetadataCache", new AdMetadataStore(client)) 11 | .post( 12 | "/ads", 13 | async (context) => { 14 | await context.adMetadataCache.createAdMetadata(context.body); 15 | context.set.status = 201; 16 | return; 17 | }, 18 | {body: AdMetadata} 19 | ) 20 | .post( 21 | "/ads/user_preferences", 22 | async (context) => { 23 | await context.adMetadataCache.createUserPreference(context.body); 24 | context.set.status = 201; 25 | return; 26 | }, 27 | {body: UserAdPreferences} 28 | ) 29 | .get( 30 | "/ads/user_preferences/:userId", 31 | async (context) => { 32 | return await context.adMetadataCache.getAdMetadataListByUserPreference(context.params.userId); 33 | }, 34 | ) 35 | .listen(3888); 36 | 37 | console.log( 38 | `Ad server API is running at ${app.server?.hostname}:${app.server?.port}` 39 | ); 40 | -------------------------------------------------------------------------------- /redis-cluster-application-example/app/go.sum: -------------------------------------------------------------------------------- 1 | github.com/bsm/ginkgo/v2 v2.5.0 h1:aOAnND1T40wEdAtkGSkvSICWeQ8L3UASX7YVCqQx+eQ= 2 | github.com/bsm/gomega v1.20.0 h1:JhAwLmtRzXFTx2AkALSLa8ijZafntmhSoU63Ok18Uq8= 3 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 4 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 5 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 6 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 7 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 8 | github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= 9 | github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= 10 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 11 | github.com/redis/go-redis/v9 v9.0.2 h1:BA426Zqe/7r56kCcvxYLWe1mkaz71LKF77GwgFzSxfE= 12 | github.com/redis/go-redis/v9 v9.0.2/go.mod h1:/xDTe9EF1LM61hek62Poq2nzQSGj0xSrEtEHbBQevps= 13 | github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= 14 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 15 | -------------------------------------------------------------------------------- /pubsub-bullmq-node/src/mq/worker.ts: -------------------------------------------------------------------------------- 1 | import { Worker, WorkerOptions, Processor } from 'bullmq'; 2 | import { createDragonflyClient } from '../utils/dragonfly.client'; 3 | import { containsExactlyOneHashtag } from '../utils/hashtag'; 4 | 5 | type DragonflyWorkerOptions = Omit; 6 | type DragonflyWorkerProcessor = string | URL | null | Processor; 7 | 8 | export class DragonflyWorker extends Worker { 9 | private constructor(queueName: string, processor?: DragonflyWorkerProcessor, opts?: WorkerOptions) { 10 | super(queueName, processor, opts); 11 | } 12 | 13 | // Factory method that sanitizes the queue name and prefix for a Dragonfly-backed BullMQ worker. 14 | static create(queueName: string, processor?: DragonflyWorkerProcessor, opts?: DragonflyWorkerOptions): DragonflyWorker { 15 | const fullQueueName = opts?.prefix ? `${opts.prefix}:${queueName}` : queueName; 16 | if (!containsExactlyOneHashtag(fullQueueName)) { 17 | throw new Error('The queue name (with prefix if provided) must contain exactly one hashtag'); 18 | } 19 | 20 | const connection = createDragonflyClient(); 21 | const workerOptions = { 22 | ...opts, 23 | connection 24 | }; 25 | 26 | return new DragonflyWorker(queueName, processor, workerOptions); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /search-blogs-by-vector/README.md: -------------------------------------------------------------------------------- 1 | # Example: Search Blogs by Dragonfly Vector Search 2 | 3 | In this example, we will build a search engine for Dragonfly blogs using Dragonfly Vector Search with the OpenAI API. 4 | 5 | ## Local Setup 6 | 7 | - The following steps assume that you are working in the root directory of this example (dragonfly-examples/search-blogs-by-vector). 8 | - Start with a clean Python environment: 9 | 10 | ```shell 11 | python3 -m venv venv 12 | ``` 13 | 14 | - After the virtual environment is created, activate it: 15 | 16 | ```shell 17 | # Windows 18 | venv\Scripts\activate 19 | ``` 20 | 21 | ```shell 22 | # Linux / macOS 23 | source venv/bin/activate 24 | ``` 25 | 26 | - Install Jupyter Notebook and other dependencies: 27 | 28 | ```shell 29 | (venv)$> pip install notebook==7.0.6 30 | (venv)$> pip install openai==1.3.7 31 | (venv)$> pip install pandas==2.1.3 32 | (venv)$> pip install numpy==1.26.2 33 | (venv)$> pip install redis==5.0.1 34 | ``` 35 | 36 | ## Run Dragonfly & Jupyter Notebook 37 | 38 | - Run a Dragonfly instance using Docker (v1.13 or above) locally: 39 | 40 | ```bash 41 | docker run -p 6379:6379 --ulimit memlock=-1 ghcr.io/dragonflydb/dragonfly:v1.13.0-ubuntu 42 | ``` 43 | 44 | - Run Jupyter Notebook server locally: 45 | 46 | ```shell 47 | (venv)$> jupyter notebook 48 | ``` 49 | 50 | - Run cells in `search-blogs-by-vector.ipynb` from the Jupyter Notebook Web UI at `http://localhost:8888/` 51 | -------------------------------------------------------------------------------- /redlock-go/go.sum: -------------------------------------------------------------------------------- 1 | github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= 2 | github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= 3 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 4 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 5 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 6 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 7 | github.com/go-redsync/redsync/v4 v4.11.0 h1:OPEcAxHBb95EzfwCKWM93ksOwHd5bTce2BD4+R14N6k= 8 | github.com/go-redsync/redsync/v4 v4.11.0/go.mod h1:ZfayzutkgeBmEmBlUR3j+rF6kN44UUGtEdfzhBFZTPc= 9 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 10 | github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= 11 | github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 12 | github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= 13 | github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= 14 | github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk= 15 | github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= 16 | -------------------------------------------------------------------------------- /leaderboard/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | redis-source: 4 | container_name: "redis-source" # used in sentinel.conf 5 | image: redis:6.0-alpine 6 | command: redis-server 7 | ports: 8 | - "6379:6379" 9 | dragonfly: 10 | container_name: "dragonfly" 11 | image: "ghcr.io/dragonflydb/dragonfly:v1.10.0-ubuntu" 12 | ulimits: 13 | memlock: -1 14 | ports: 15 | - "6380:6379" 16 | command: 17 | - "--dir=/data" 18 | sentinel-1: 19 | container_name: "sentinel-1" 20 | image: redis:6.0-alpine 21 | ports: 22 | - "5001:5000" 23 | command: redis-server /etc/redis-config/sentinel.conf --sentinel 24 | volumes: 25 | - "./config/sentinel-1:/etc/redis-config" 26 | sentinel-2: 27 | container_name: "sentinel-2" 28 | image: redis:6.0-alpine 29 | ports: 30 | - "5002:5000" 31 | command: redis-server /etc/redis-config/sentinel.conf --sentinel 32 | volumes: 33 | - "./config/sentinel-2:/etc/redis-config" 34 | sentinel-3: 35 | container_name: "sentinel-3" 36 | image: redis:6.0-alpine 37 | ports: 38 | - "5003:5000" 39 | command: redis-server /etc/redis-config/sentinel.conf --sentinel 40 | volumes: 41 | - "./config/sentinel-3:/etc/redis-config" 42 | leaderboard: 43 | build: 44 | context: . 45 | dockerfile: Dockerfile 46 | container_name: leaderboard 47 | ports: 48 | - "8080:8080" 49 | -------------------------------------------------------------------------------- /ecommerce-cache/app/viewed_items.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "log" 8 | "math/rand" 9 | "net/http" 10 | "strconv" 11 | 12 | "github.com/redis/go-redis/v9" 13 | ) 14 | 15 | const key = "user:42:item_views" 16 | 17 | func init() { 18 | 19 | //we are loading sample data. but imagine this being populated in real-time while user is interacting with the ecommerce system 20 | 21 | var items []redis.Z 22 | 23 | for i := 1; i <= 10; i++ { 24 | item := "product-" + strconv.Itoa(i) 25 | items = append(items, redis.Z{Score: float64(rand.Intn(5000) + 1), Member: item}) 26 | } 27 | 28 | err := rdb.ZAdd(context.Background(), key, items...).Err() 29 | if err != nil { 30 | log.Fatal("failed to load items data", err) 31 | } 32 | 33 | fmt.Println("loaded items views sample data") 34 | } 35 | 36 | func getMostViewedItems(rw http.ResponseWriter, req *http.Request) { 37 | items := rdb.ZRevRangeWithScores(context.Background(), key, 0, 4).Val() 38 | 39 | var resp ViewedItemsResponse 40 | resp.UserID = 42 41 | var viewed []string 42 | 43 | for _, item := range items { 44 | viewed = append(viewed, fmt.Sprint(item.Member)) 45 | } 46 | 47 | resp.Items = viewed 48 | 49 | err := json.NewEncoder(rw).Encode(resp) 50 | if err != nil { 51 | http.Error(rw, err.Error(), 500) 52 | return 53 | } 54 | } 55 | 56 | type ViewedItemsResponse struct { 57 | UserID int `json:"user_id"` 58 | Items []string `json:"viewed"` 59 | } 60 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Dragonfly Example Repository 2 | 3 | This is a repository of examples for [Dragonfly](https://www.dragonflydb.io/), a high-performance, drop-in Redis replacement. Use these examples to understand Dragonfly and develop your own powerful and efficient applications. 4 | 5 | ## Usage 6 | 7 | These examples are organized into folders that correspond to tutorials that appeared on the DragonflyDB blog. 8 | 9 | ## Examples 10 | 11 | - [`/redis-cluster-application-example`](/redis-cluster-application-example) contains the code corresponding to our [Migrating from a Redis Cluster to a DragonflyDB on a single node](https://www.dragonflydb.io/blog/migrating-from-a-redis-cluster-to-a-dragonfly-on-a-single-node) tutorial. In this tutorial, you learn how to migrate data from a Redis Cluster to a single-node Dragonfly instance. You can use the sample application /app to demonstrate the migration process and cover everything step by step. 12 | - [`/background-process-example`](/background-process-example) contains the code corresponding to our [Building a background processing pipeline with Dragonfly](https://www.dragonflydb.io/blog/building-a-background-processing-pipeline-with-dragonfly) tutorial. In this tutorial, you learn how to use Redis Lists to build a background processing pipeline with Dragonfly. 13 | - [`/monitoring-in-memory-stores`](/monitoring-in-memory-stores) contains yaml corresponding to our [Monitoring in-memory datastores] tutorial. 14 | 15 | ## LICENSE 16 | 17 | The [MIT license](LICENSE). 18 | -------------------------------------------------------------------------------- /cache-in-5mins-hono/src/validator.ts: -------------------------------------------------------------------------------- 1 | import { createInsertSchema, createSelectSchema } from "drizzle-zod"; 2 | import { v7 as uuidv7, stringify as uuidStringify } from "uuid"; 3 | import { z } from "zod/v4"; 4 | 5 | import { shortLinksTable } from "./schema"; 6 | 7 | // Validator and transformer for creating a new 'short_links' entry. 8 | // Only the original URL is validated. 9 | // All other fields are transformed/generated by our predefined rules. 10 | export const shortLinkInsertSchema = createInsertSchema(shortLinksTable, { 11 | originalUrl: (val) => z.url(), 12 | }) 13 | .strict() 14 | .omit({ 15 | id: true, 16 | shortCode: true, 17 | createdAt: true, 18 | expiresAt: true, 19 | }) 20 | .transform((data) => { 21 | const idBytes = new Uint8Array(16); 22 | uuidv7(undefined, idBytes); 23 | const id = uuidStringify(idBytes); 24 | const shortCode = Buffer.from(idBytes).toString("base64url"); 25 | const createdAt = new Date(); 26 | const expiresAt = new Date(createdAt); 27 | expiresAt.setDate(expiresAt.getDate() + 30); // Expire in 30 days. 28 | return { 29 | ...data, 30 | id, 31 | shortCode, 32 | createdAt, 33 | expiresAt, 34 | }; 35 | }); 36 | 37 | export type ShortLinkInsert = z.infer; 38 | 39 | // Validator for selecting 'short_links' entries. 40 | const shortLinkSelectSchema = createSelectSchema(shortLinksTable); 41 | export type ShortLinkSelect = z.infer; 42 | -------------------------------------------------------------------------------- /celery-transaction-processing/models.py: -------------------------------------------------------------------------------- 1 | import enum 2 | 3 | from sqlalchemy import Column, ForeignKey, Integer, String, Enum 4 | from sqlalchemy.ext.declarative import declarative_base 5 | from sqlalchemy.orm import relationship 6 | 7 | Base = declarative_base() 8 | 9 | 10 | class UserAccount(Base): 11 | __tablename__ = "user_accounts" 12 | 13 | id = Column(Integer, primary_key=True) 14 | available_balance_in_wei = Column(Integer, nullable=False) 15 | current_balance_in_wei = Column(Integer, nullable=False) 16 | 17 | user_account_transactions = relationship("UserAccountTransaction", back_populates="user_account") 18 | 19 | 20 | class UserAccountTransactionStatus(enum.Enum): 21 | PENDING = 0 22 | SUCCESSFUL = 1 23 | FAILED = 2 24 | 25 | 26 | class UserAccountTransaction(Base): 27 | __tablename__ = "user_account_transactions" 28 | 29 | id = Column(Integer, primary_key=True) 30 | user_account_id = Column(Integer, ForeignKey("user_accounts.id"), nullable=False) 31 | transaction_hash = Column(String, nullable=False) 32 | from_public_address = Column(String, nullable=False) 33 | to_public_address = Column(String, nullable=False) 34 | transaction_amount_in_wei = Column(Integer, nullable=False) 35 | transaction_fee_total_in_wei = Column(Integer, nullable=False) 36 | transaction_fee_blockchain_in_wei = Column(Integer, nullable=False) 37 | status = Column(Enum(UserAccountTransactionStatus), nullable=False) 38 | 39 | user_account = relationship("UserAccount", back_populates="user_account_transactions") 40 | -------------------------------------------------------------------------------- /feast-recommendation-duckdb-dragonfly/recommendation_01_data.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | users = pd.DataFrame({ 4 | "user_id": [1, 2, 3, 4, 5], 5 | "age": [25, 31, 45, 22, 38], 6 | "gender": ["M", "F", "F", "M", "M"], 7 | "avg_rating": [4.2, 3.8, 4.5, 3.2, 4.1], 8 | "preferred_category": ["electronics", "books", "clothing", "electronics", "home"], 9 | "event_timestamp": [pd.Timestamp("2025-08-28 00:00:00", tz="UTC") for _ in range(5)] 10 | }) 11 | 12 | items = pd.DataFrame({ 13 | "item_id": [101, 102, 103, 104, 105], 14 | "category": ["electronics", "books", "clothing", "electronics", "home"], 15 | "price": [299.99, 14.99, 49.99, 199.99, 89.99], 16 | "popularity_score": [0.85, 0.72, 0.91, 0.68, 0.77], 17 | "avg_rating": [4.3, 4.1, 4.7, 3.9, 4.2], 18 | "event_timestamp": [pd.Timestamp("2025-08-28 00:00:00", tz="UTC") for _ in range(5)] 19 | }) 20 | 21 | interactions = pd.DataFrame({ 22 | "user_id": [1, 1, 2, 3, 4, 5, 2, 3], 23 | "item_id": [101, 102, 103, 104, 105, 101, 102, 103], 24 | "view_count": [5, 2, 3, 1, 4, 2, 1, 3], 25 | "last_rating": [4.5, 3.5, 4.0, 3.0, 4.5, 4.0, 3.5, 4.5], 26 | "time_since_last_interaction": [2.5, 7.2, 1.0, 5.5, 0.5, 3.0, 8.0, 2.0], 27 | "event_timestamp": [pd.Timestamp("2025-08-28 00:00:00", tz="UTC") for _ in range(8)] 28 | }) 29 | 30 | users.to_parquet("data/users.parquet", engine="pyarrow", index=False) 31 | items.to_parquet("data/items.parquet", engine="pyarrow", index=False) 32 | interactions.to_parquet("data/interactions.parquet", engine="pyarrow", index=False) 33 | -------------------------------------------------------------------------------- /pubsub-bullmq-node/src/pubsub/sub.ts: -------------------------------------------------------------------------------- 1 | import { Redis as Dragonfly } from 'ioredis'; 2 | import { createDragonflyClient } from '../utils/dragonfly.client'; 3 | 4 | type MessageHandler = (channel: string, message: string) => void; 5 | 6 | export class DragonflySubscriber { 7 | private readonly sub: Dragonfly; 8 | private readonly channel: string; 9 | private readonly handler: MessageHandler; 10 | 11 | // Note that a connection can't play both publisher and subscriber roles at the same time. 12 | // More specifically, when a client issues subscribe() or psubscribe(), it enters the "subscriber" mode. 13 | // From that point, only commands that modify the subscription set are valid. 14 | // These commands are: subscribe(), psubscribe(), unsubscribe(), punsubscribe(), ping, and quit(). 15 | // When the subscription set is empty (via unsubscribe/punsubscribe), the connection is put back into the regular mode. 16 | constructor(channel: string, handler: MessageHandler) { 17 | this.sub = createDragonflyClient(); 18 | this.channel = channel; 19 | this.handler = handler; 20 | this.initialize(); 21 | } 22 | 23 | private initialize() { 24 | this.sub.on('message', this.handler); 25 | this.sub.subscribe(this.channel, (err, _) => { 26 | if (err) { 27 | console.error(`Failed to subscribe to channel "${this.channel}": ${err.message}`); 28 | return; 29 | } 30 | console.log(`Successfully subscribed to channel "${this.channel}"`); 31 | }); 32 | } 33 | 34 | close() { this.sub.quit(); } 35 | } 36 | -------------------------------------------------------------------------------- /user-session-management-go/app/session/access_token.go: -------------------------------------------------------------------------------- 1 | package session 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/golang-jwt/jwt/v5" 9 | ) 10 | 11 | const ( 12 | jwtSecret = "your-secret-key" 13 | jwtExpiry = 15 * time.Minute 14 | 15 | jwtClaimAlgorithm = "alg" 16 | jwtClaimUserID = "sub" 17 | jwtClaimExpiry = "exp" 18 | ) 19 | 20 | // generateAccessToken creates a JWT. 21 | // Since JWTs are stateless, the access token is only valid for a short period of time. 22 | func generateAccessToken(userID string) (string, error) { 23 | claims := jwt.MapClaims{ 24 | jwtClaimUserID: userID, 25 | jwtClaimExpiry: time.Now().UTC().Add(jwtExpiry).Unix(), 26 | } 27 | token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) 28 | return token.SignedString([]byte(jwtSecret)) 29 | } 30 | 31 | // VerifyAccessToken validates the JWT and returns the user ID. 32 | func VerifyAccessToken(tokenString string) (string, error) { 33 | token, err := jwt.Parse(tokenString, func(token *jwt.Token) (interface{}, error) { 34 | if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { 35 | return nil, fmt.Errorf("unexpected signing method: %v", token.Header[jwtClaimAlgorithm]) 36 | } 37 | return []byte(jwtSecret), nil 38 | }) 39 | if err != nil || !token.Valid { 40 | return "", fmt.Errorf("invalid token: %w", err) 41 | } 42 | 43 | claims, ok := token.Claims.(jwt.MapClaims) 44 | if !ok || !token.Valid { 45 | return "", errors.New("invalid token claims") 46 | } 47 | 48 | userID, ok := claims[jwtClaimUserID].(string) 49 | if !ok { 50 | return "", errors.New("invalid user ID claim") 51 | } 52 | 53 | return userID, nil 54 | } 55 | -------------------------------------------------------------------------------- /cache-in-5mins-hono/drizzle/meta/0000_snapshot.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "f4ec2f56-49f2-4907-8902-d77ab9723dd0", 3 | "prevId": "00000000-0000-0000-0000-000000000000", 4 | "version": "7", 5 | "dialect": "postgresql", 6 | "tables": { 7 | "public.short_links": { 8 | "name": "short_links", 9 | "schema": "", 10 | "columns": { 11 | "id": { 12 | "name": "id", 13 | "type": "uuid", 14 | "primaryKey": true, 15 | "notNull": true 16 | }, 17 | "original_url": { 18 | "name": "original_url", 19 | "type": "varchar(4096)", 20 | "primaryKey": false, 21 | "notNull": true 22 | }, 23 | "short_code": { 24 | "name": "short_code", 25 | "type": "varchar(30)", 26 | "primaryKey": false, 27 | "notNull": true 28 | }, 29 | "created_at": { 30 | "name": "created_at", 31 | "type": "timestamp with time zone", 32 | "primaryKey": false, 33 | "notNull": true 34 | }, 35 | "expires_at": { 36 | "name": "expires_at", 37 | "type": "timestamp with time zone", 38 | "primaryKey": false, 39 | "notNull": true 40 | } 41 | }, 42 | "indexes": {}, 43 | "foreignKeys": {}, 44 | "compositePrimaryKeys": {}, 45 | "uniqueConstraints": {}, 46 | "policies": {}, 47 | "checkConstraints": {}, 48 | "isRLSEnabled": false 49 | } 50 | }, 51 | "enums": {}, 52 | "schemas": {}, 53 | "sequences": {}, 54 | "roles": {}, 55 | "policies": {}, 56 | "views": {}, 57 | "_meta": { 58 | "columns": {}, 59 | "schemas": {}, 60 | "tables": {} 61 | } 62 | } -------------------------------------------------------------------------------- /background-process-example/producer/producer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "os/signal" 8 | "syscall" 9 | "time" 10 | 11 | rdata "github.com/Pallinder/go-randomdata" 12 | "github.com/gocelery/gocelery" 13 | "github.com/gomodule/redigo/redis" 14 | ) 15 | 16 | const ( 17 | redisHostEnvVar = "REDIS_HOST" 18 | redisPasswordEnvVar = "REDIS_PASSWORD" 19 | taskName = "users.registration.email" 20 | ) 21 | 22 | var ( 23 | redisHost string 24 | ) 25 | 26 | func init() { 27 | redisHost = os.Getenv(redisHostEnvVar) 28 | if redisHost == "" { 29 | redisHost = "localhost:6379" 30 | } 31 | 32 | } 33 | 34 | func main() { 35 | redisPool := &redis.Pool{ 36 | Dial: func() (redis.Conn, error) { 37 | c, err := redis.Dial("tcp", redisHost) 38 | 39 | if err != nil { 40 | return nil, err 41 | } 42 | return c, err 43 | }, 44 | } 45 | 46 | celeryClient, err := gocelery.NewCeleryClient( 47 | gocelery.NewRedisBroker(redisPool), 48 | &gocelery.RedisCeleryBackend{Pool: redisPool}, 49 | 1, 50 | ) 51 | 52 | if err != nil { 53 | log.Fatal(err) 54 | } 55 | 56 | exit := make(chan os.Signal, 1) 57 | signal.Notify(exit, syscall.SIGINT, syscall.SIGTERM) 58 | closed := false 59 | 60 | go func() { 61 | fmt.Println("celery producer started...") 62 | 63 | for !closed { 64 | res, err := celeryClient.Delay(taskName, rdata.FullName(rdata.RandomGender)+","+rdata.Email()) 65 | if err != nil { 66 | panic(err) 67 | } 68 | fmt.Println("sent data and generated task", res.TaskID, "for worker") 69 | time.Sleep(1 * time.Second) 70 | } 71 | }() 72 | 73 | <-exit 74 | log.Println("exit signalled") 75 | 76 | closed = true 77 | log.Println("celery producer stopped") 78 | } 79 | -------------------------------------------------------------------------------- /live-migration-app/README.md: -------------------------------------------------------------------------------- 1 | # Live Migration to Dragonfly Cloud 2 | 3 | This repo shows a sample python application that acts like a real-time client to a 4 | Redis-compatible database. 5 | 6 | This is used to test and showcase live migration of Redis data to Dragonfly Cloud using 7 | [RedisShake](https://github.com/tair-opensource/RedisShake). With Live Migrations, There is no downtime and the application can continue to read and write to the database without any 8 | interruption. 9 | 10 | ## How to run 11 | 12 | 1. Clone the repository 13 | 14 | ```bash 15 | git clone https://github.com/dragonflydb/live-migration-demo.git 16 | ``` 17 | 18 | 2. Install the dependencies 19 | 20 | ```bash 21 | pip install -r requirements.txt 22 | ``` 23 | 24 | 3. Run the application 25 | 26 | ```bash 27 | python redis_migration_demo.py 28 | ``` 29 | 30 | ## How to test live migration 31 | 32 | 1. Create a new database in Dragonfly Cloud 33 | 34 | 2. Once the data is inserted, Start running RedisShake to migrate the data to the new database 35 | 36 | ```bash 37 | docker docker run -v "$(pwd)/redis-shake.toml:/redis-shake-config.toml" \ 38 | --entrypoint ./redis-shake \ 39 | ghcr.io/tair-opensource/redisshake:latest \ 40 | /redis-shake-config.toml 41 | ``` 42 | 43 | 3. Once the main sync is done, and keyspace notification sync is taking place, you can switch the application *live* to the new database by 44 | 45 | ```bash 46 | curl -X POST \ 47 | -H "Content-Type: application/json" \ 48 | -d '{"host": ".dragonflydb.cloud", "port": 6385, "password": ""}' \ 49 | http://127.0.0.1:8080/update_redis 50 | ``` 51 | 52 | 4. As you can see, Even after the migration, the application is still able to read and write to the database without any interruption and no 53 | key loss. -------------------------------------------------------------------------------- /leaderboard/app/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "io/ioutil" 7 | "net/http" 8 | 9 | "leaderboard/app/memstore" 10 | "leaderboard/app/model" 11 | 12 | "github.com/redis/go-redis/v9" 13 | ) 14 | 15 | func main() { 16 | client := redis.NewFailoverClient(&redis.FailoverOptions{ 17 | MasterName: "leaderboard-primary", 18 | SentinelAddrs: []string{"sentinel-1:5000", "sentinel-2:5000", "sentinel-3:5000"}, 19 | }) 20 | 21 | ctx := context.Background() 22 | _, err := client.Ping(ctx).Result() 23 | if err != nil { 24 | panic(err) 25 | } 26 | 27 | http.Handle("/leaderboard", http.HandlerFunc( 28 | func(w http.ResponseWriter, r *http.Request) { 29 | userID := r.URL.Query().Get("user_id") 30 | if userID == "" { 31 | panic("id is empty") 32 | } 33 | 34 | resp, err := memstore.UseLeaderboard(client).ReadTopRankList(ctx, userID) 35 | if err != nil { 36 | panic(err) 37 | } 38 | 39 | respBytes, err := json.Marshal(resp) 40 | if err != nil { 41 | panic(err) 42 | } 43 | 44 | _, _ = w.Write(respBytes) 45 | }), 46 | ) 47 | 48 | http.Handle("/leaderboard/user", http.HandlerFunc( 49 | func(w http.ResponseWriter, r *http.Request) { 50 | body, err := ioutil.ReadAll(r.Body) 51 | if err != nil { 52 | panic(err) 53 | } 54 | defer func() { 55 | _ = r.Body.Close() 56 | }() 57 | 58 | req := &model.UserIncreaseScoreRequest{} 59 | err = json.Unmarshal(body, req) 60 | if err != nil { 61 | panic(err) 62 | } 63 | 64 | err = memstore.UseLeaderboard(client).IncreaseUserScore(ctx, &req.User, req.Score) 65 | if err != nil { 66 | panic(err) 67 | } 68 | }), 69 | ) 70 | 71 | if err := http.ListenAndServe(":8080", nil); err != nil { 72 | panic(err) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /celery-transaction-processing/eth.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | from eth_typing import HexStr 4 | from web3.types import TxReceipt 5 | 6 | from deps import get_deps 7 | from models import UserAccountTransactionStatus 8 | 9 | 10 | @dataclass 11 | class TransactionStatusResponse: 12 | tx_hash: str 13 | tx_block_number: int 14 | current_block_number: int 15 | transaction_fee_blockchain_in_wei: int 16 | status: UserAccountTransactionStatus 17 | 18 | 19 | def get_transaction_status( 20 | tx_hash: str, 21 | number_of_blocks_to_wait: int = 10 22 | ) -> TransactionStatusResponse: 23 | receipt = get_deps().get_web3().eth.get_transaction_receipt(HexStr(tx_hash)) 24 | gas_price = get_deps().get_web3().eth.get_transaction(HexStr(tx_hash)).gasPrice 25 | current_block_number = get_deps().get_web3().eth.get_block_number() 26 | return TransactionStatusResponse( 27 | tx_hash=tx_hash, 28 | tx_block_number=receipt['blockNumber'], 29 | current_block_number=current_block_number, 30 | transaction_fee_blockchain_in_wei=receipt['gasUsed'] * gas_price, 31 | status=__calculate_transaction_status(receipt, current_block_number, number_of_blocks_to_wait) 32 | ) 33 | 34 | 35 | def __calculate_transaction_status( 36 | receipt: TxReceipt, 37 | current_block_number: int, 38 | number_of_blocks_to_wait: int 39 | ) -> UserAccountTransactionStatus: 40 | if receipt is None: 41 | return UserAccountTransactionStatus.PENDING 42 | elif receipt['status'] == 0: 43 | return UserAccountTransactionStatus.FAILED 44 | else: 45 | if receipt['blockNumber'] + number_of_blocks_to_wait <= current_block_number: 46 | return UserAccountTransactionStatus.SUCCESSFUL 47 | else: 48 | return UserAccountTransactionStatus.PENDING 49 | -------------------------------------------------------------------------------- /cache-aside-with-expiration/README.md: -------------------------------------------------------------------------------- 1 | # Example: Cache-Aside 2 | 3 | Cache-Aside (or Lazy-Loading) is the most common caching pattern available. 4 | The fundamental data retrieval logic can be summarized as: 5 | 6 | - When a service needs a specific piece of data, it queries the cache first. 7 | - Ideally, the required data is readily available in the cache, which is often referred to as a **cache hit**. 8 | - If data is absent from the cache (aptly termed a **cache miss**), the service redirects the query to the primary database instead. 9 | - The cache is also populated with the data retrieved from the database. 10 | 11 | ## Packages Used 12 | 13 | - The [Fiber](https://docs.gofiber.io/) web framework is used in this example, as the route handlers are much easier to write than the standard library. 14 | - The [Fiber cache middleware](https://docs.gofiber.io/api/middleware/cache) is used to demonstrate the Cache-Aside pattern. 15 | - The [go-redis](https://github.com/redis/go-redis) client is used. It has strongly typed methods for various commands. 16 | It is also the recommended Go client to interact with Dragonfly. 17 | 18 | ## Local Setup 19 | 20 | - Make sure that you have [Go v1.20+](https://go.dev/dl/) installed locally. 21 | - Make sure that you have [Docker](https://docs.docker.com/engine/install/) installed locally. 22 | 23 | ## Run Dragonfly & Service Application 24 | 25 | - Run a Dragonfly instance with `docker compose` using configurations in the `docker-compose.yml` file: 26 | 27 | ```bash 28 | # within the root directory of this example (dragonfly-examples/cache-aside-with-expiration) 29 | docker compose up -d --remove-orphans cache-aside-dragonfly 30 | ``` 31 | 32 | - Install dependencies and run the service application: 33 | 34 | ```bash 35 | go mod vender 36 | 37 | # within the root directory of this example (dragonfly-examples/cache-aside-with-expiration) 38 | cd app && go run . 39 | ``` 40 | -------------------------------------------------------------------------------- /celery-transaction-processing/requirements.txt: -------------------------------------------------------------------------------- 1 | aiohttp==3.9.5 2 | aiosignal==1.3.1 3 | amqp==5.2.0 4 | annotated-types==0.6.0 5 | anyio==4.3.0 6 | async-timeout==4.0.3 7 | attrs==23.2.0 8 | billiard==4.2.0 9 | bitarray==2.9.2 10 | celery==5.4.0 11 | certifi==2024.2.2 12 | charset-normalizer==3.3.2 13 | ckzg==1.0.1 14 | click==8.1.7 15 | click-didyoumean==0.3.1 16 | click-plugins==1.1.1 17 | click-repl==0.3.0 18 | cytoolz==0.12.3 19 | dnspython==2.6.1 20 | email_validator==2.1.1 21 | eth-account==0.11.2 22 | eth-hash==0.7.0 23 | eth-keyfile==0.8.1 24 | eth-keys==0.5.1 25 | eth-rlp==1.0.1 26 | eth-typing==4.2.2 27 | eth-utils==4.1.0 28 | eth_abi==5.1.0 29 | exceptiongroup==1.2.1 30 | fastapi==0.111.0 31 | fastapi-cli==0.0.3 32 | frozenlist==1.4.1 33 | h11==0.14.0 34 | hexbytes==0.3.1 35 | httpcore==1.0.5 36 | httptools==0.6.1 37 | httpx==0.27.0 38 | idna==3.7 39 | Jinja2==3.1.4 40 | jsonschema==4.22.0 41 | jsonschema-specifications==2023.12.1 42 | kombu==5.3.7 43 | lru-dict==1.2.0 44 | markdown-it-py==3.0.0 45 | MarkupSafe==2.1.5 46 | mdurl==0.1.2 47 | msgpack==1.0.8 48 | multidict==6.0.5 49 | orjson==3.10.3 50 | parsimonious==0.10.0 51 | prompt-toolkit==3.0.43 52 | protobuf==5.26.1 53 | pycryptodome==3.20.0 54 | pydantic==2.7.1 55 | pydantic_core==2.18.2 56 | Pygments==2.18.0 57 | python-dateutil==2.9.0.post0 58 | python-dotenv==1.0.1 59 | python-multipart==0.0.9 60 | pyunormalize==15.1.0 61 | PyYAML==6.0.1 62 | redis==5.0.4 63 | referencing==0.35.1 64 | regex==2024.4.28 65 | requests==2.31.0 66 | rich==13.7.1 67 | rlp==4.0.1 68 | rpds-py==0.18.0 69 | shellingham==1.5.4 70 | six==1.16.0 71 | sniffio==1.3.1 72 | SQLAlchemy==2.0.30 73 | starlette==0.37.2 74 | toolz==0.12.1 75 | typer==0.12.3 76 | typing_extensions==4.11.0 77 | tzdata==2024.1 78 | ujson==5.9.0 79 | urllib3==2.2.1 80 | uvicorn==0.29.0 81 | uvloop==0.19.0 82 | vine==5.1.0 83 | watchfiles==0.21.0 84 | wcwidth==0.2.13 85 | web3==6.18.0 86 | websockets==12.0 87 | yarl==1.9.4 88 | -------------------------------------------------------------------------------- /ecommerce-cache/load/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "strings" 9 | "sync" 10 | 11 | "github.com/redis/go-redis/v9" 12 | ) 13 | 14 | var ctx = context.Background() 15 | 16 | // Number of goroutines to use 17 | const numGoroutines = 10 18 | 19 | // Length of each value in bytes 20 | const valueLength = 1024 21 | 22 | func main() { 23 | var dataSizeGB int 24 | var keyPrefix string 25 | 26 | flag.IntVar(&dataSizeGB, "dataSizeGB", 1, "Amount of data to create in GB") 27 | flag.StringVar(&keyPrefix, "keyPrefix", "", "prefix for key") 28 | flag.Parse() 29 | 30 | if keyPrefix == "" { 31 | log.Fatal("missing keyPrefix") 32 | } 33 | 34 | // value that is approximately 10 KB 35 | 36 | value := strings.Repeat("a", 10*1024) 37 | 38 | // Calculate the number of keys to create 39 | numKeys := (dataSizeGB * 1024 * 1024 * 1024) / len(value) 40 | 41 | fmt.Println("loading", dataSizeGB, "GB data") 42 | fmt.Println("numKeys ==", numKeys) 43 | 44 | // Connect to Redis 45 | rdb := redis.NewClient(&redis.Options{ 46 | Addr: "localhost:6379", 47 | }) 48 | 49 | err := rdb.Ping(context.Background()).Err() 50 | if err != nil { 51 | log.Fatal("failed to connect with redis", err) 52 | } 53 | 54 | fmt.Println("connected to redis") 55 | 56 | ctx := context.Background() 57 | var wg sync.WaitGroup 58 | 59 | // Create 10 Go routines 60 | for i := 0; i < 10; i++ { 61 | wg.Add(1) 62 | go func(i int) { 63 | defer wg.Done() 64 | for j := i * numKeys / 10; j < (i+1)*numKeys/10; j += 1000 { 65 | pipe := rdb.Pipeline() 66 | for k := 0; k < 1000 && j+k < numKeys; k++ { 67 | pipe.Set(ctx, fmt.Sprintf(keyPrefix+"-key%d", j+k), value, 0) 68 | } 69 | _, err := pipe.Exec(ctx) 70 | if err != nil { 71 | log.Fatal("pipeline exec failed: ", err) 72 | } 73 | } 74 | }(i) 75 | } 76 | 77 | wg.Wait() 78 | 79 | rdb.Close() 80 | 81 | fmt.Println("finished loading data") 82 | } 83 | -------------------------------------------------------------------------------- /cache-refresh-ahead/app/model.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "time" 6 | 7 | "github.com/google/uuid" 8 | ) 9 | 10 | type Repo struct{} 11 | 12 | // UseRepo returns a singleton instance of Repo. 13 | // Practically, the singleton instance should be initialized with a database connection 14 | // and other necessary dependencies, concurrency-safely using sync.Once. 15 | func UseRepo() *Repo { 16 | return repoSingleton 17 | } 18 | 19 | var repoSingleton = &Repo{} 20 | 21 | // User model and repository methods. 22 | type User struct { 23 | ID string `json:"id"` 24 | Name string `json:"name"` 25 | } 26 | 27 | func (r *Repo) ReadUserByID(id uuid.UUID) (*User, error) { 28 | return r.readUserByID(id) 29 | } 30 | 31 | func (r *Repo) ReadUserBytesByID(id uuid.UUID) ([]byte, error) { 32 | user, err := r.readUserByID(id) 33 | if err != nil { 34 | return nil, err 35 | } 36 | return json.Marshal(user) 37 | } 38 | 39 | func (r *Repo) readUserByID(id uuid.UUID) (*User, error) { 40 | // This is for simplicity and demonstration. 41 | // Pretend we are reading user from database. 42 | time.Sleep(time.Millisecond * 200) 43 | return &User{ 44 | ID: id.String(), 45 | Name: "John Doe", 46 | }, nil 47 | } 48 | 49 | // Blog model and repository methods. 50 | type Blog struct { 51 | ID string `json:"id"` 52 | Content string `json:"content"` 53 | } 54 | 55 | func (r *Repo) ReadBlogByID(id uuid.UUID) (*Blog, error) { 56 | return r.readBlogByID(id) 57 | } 58 | 59 | func (r *Repo) ReadBlogBytesByID(id uuid.UUID) ([]byte, error) { 60 | blog, err := r.readBlogByID(id) 61 | if err != nil { 62 | return nil, err 63 | } 64 | return json.Marshal(blog) 65 | } 66 | 67 | func (r *Repo) readBlogByID(id uuid.UUID) (*Blog, error) { 68 | // This is for simplicity and demonstration. 69 | // Pretend we are reading blog from database. 70 | time.Sleep(time.Millisecond * 200) 71 | return &Blog{ 72 | ID: id.String(), 73 | Content: "This is a micro-blog limited to 140 characters.", 74 | }, nil 75 | } 76 | -------------------------------------------------------------------------------- /cache-refresh-ahead/README.md: -------------------------------------------------------------------------------- 1 | # Example: Cache Refresh-Ahead 2 | 3 | Refresh-Ahead is a strategy that takes a proactive approach by refreshing the cache before the cached key expires, keeping data hot until it's no longer in high demand. 4 | 5 | - Start by choosing a refresh-ahead factor, which determines the time before the cached key expires when the cache should be refreshed. 6 | - For example, if your cached data has a lifetime of 100 seconds, and you choose a refresh-ahead factor of 50%: 7 | - If the data is queried before the 50th second, the cached data will be returned as usual. 8 | - If the data is queried after the 50th second, the cached data will still be returned, but a background worker will trigger the data refresh. 9 | - It's important to ensure that only one background worker is responsible for refreshing the data to avoid redundant reads to the database. 10 | 11 | ## Packages Used 12 | 13 | - The [Fiber](https://docs.gofiber.io/) web framework is used in this example, as the route handlers are much easier to write than the standard library. 14 | - The [go-redis](https://github.com/redis/go-redis) client is used. It has strongly typed methods for various commands. 15 | It is also the recommended Go client to interact with Dragonfly. 16 | 17 | ## Local Setup 18 | 19 | - Make sure that you have [Go v1.20+](https://go.dev/dl/) installed locally. 20 | - Make sure that you have [Docker](https://docs.docker.com/engine/install/) installed locally. 21 | 22 | ## Run Dragonfly & Service Application 23 | 24 | - Run a Dragonfly instance with `docker compose` using configurations in the `docker-compose.yml` file: 25 | 26 | ```bash 27 | # within the root directory of this example (dragonfly-examples/cache-refresh-ahead) 28 | docker compose up -d --remove-orphans cache-refresh-ahead-dragonfly 29 | ``` 30 | 31 | - Install dependencies and run the service application: 32 | 33 | ```bash 34 | go mod vender 35 | 36 | # within the root directory of this example (dragonfly-examples/cache-refresh-ahead) 37 | cd app && go run . 38 | ``` 39 | -------------------------------------------------------------------------------- /feast-recommendation-duckdb-dragonfly/recommendation_02_repo.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | 3 | from feast import Entity, FeatureView, Field, ValueType, FileSource 4 | from feast.data_format import ParquetFormat 5 | from feast.types import Float32, Int64, String 6 | 7 | # Data Sources 8 | users_file_source = FileSource( 9 | file_format=ParquetFormat(), 10 | path="data/users.parquet", 11 | ) 12 | 13 | items_file_source = FileSource( 14 | file_format=ParquetFormat(), 15 | path="data/items.parquet", 16 | ) 17 | 18 | interactions_file_source = FileSource( 19 | file_format=ParquetFormat(), 20 | path="data/interactions.parquet", 21 | ) 22 | 23 | # Entities 24 | user = Entity(name="user", value_type=ValueType.INT64, join_keys=["user_id"]) 25 | item = Entity(name="item", value_type=ValueType.INT64, join_keys=["item_id"]) 26 | 27 | # User Features 28 | user_features = FeatureView( 29 | name="user_features", 30 | source=users_file_source, 31 | entities=[user], 32 | schema=[ 33 | Field(name="age", dtype=Int64), 34 | Field(name="gender", dtype=String), 35 | Field(name="avg_rating", dtype=Float32), 36 | Field(name="preferred_category", dtype=String), 37 | ], 38 | ttl=timedelta(days=365), 39 | ) 40 | 41 | # Item Features 42 | item_features = FeatureView( 43 | name="item_features", 44 | source=items_file_source, 45 | entities=[item], 46 | schema=[ 47 | Field(name="category", dtype=String), 48 | Field(name="price", dtype=Float32), 49 | Field(name="popularity_score", dtype=Float32), 50 | Field(name="avg_rating", dtype=Float32), 51 | ], 52 | ttl=timedelta(days=365), 53 | ) 54 | 55 | # Interaction Features (User-Item Pairs) 56 | interaction_features = FeatureView( 57 | name="interaction_features", 58 | source=interactions_file_source, 59 | entities=[user, item], 60 | schema=[ 61 | Field(name="view_count", dtype=Int64), 62 | Field(name="last_rating", dtype=Float32), 63 | Field(name="time_since_last_interaction", dtype=Float32), 64 | ], 65 | ttl=timedelta(days=90), 66 | ) 67 | -------------------------------------------------------------------------------- /leaderboard/README.md: -------------------------------------------------------------------------------- 1 | # Example: Leaderboard 2 | 3 | This example demonstrates how to use Dragonfly as an in-memory store to build a real-time leaderboard application. 4 | 5 | - We use the `Sorted-Set` data type to store user scores & IDs: 6 | 7 | ```bash 8 | dragonfly:6380$> ZREVRANGE leaderboard:user_scores 0 10 WITHSCORES 9 | 1) "leaderboard:users:1" 10 | 2) "500" 11 | 3) "leaderboard:users:2" 12 | 4) "400" 13 | 5) "leaderboard:users:3" 14 | 6) "300" 15 | 7) "leaderboard:users:4" 16 | 8) "200" 17 | ``` 18 | 19 | - We use the `Hash` data type to store user details based on IDs: 20 | 21 | ```bash 22 | dragonfly:6380$> HGETALL leaderboard:users:1 23 | 1) "id" 24 | 2) "1" 25 | 3) "email" 26 | 4) "joe@dragonflydb.io" 27 | 5) "nick_name" 28 | 6) "Joe" 29 | 7) "image_url" 30 | 8) "joe_avatar.png" 31 | ``` 32 | 33 | ## Packages Used 34 | 35 | - The [go-redis](https://github.com/redis/go-redis) client is used. It has strongly typed methods for various commands. 36 | It is also the recommended Go client to interact with Dragonfly. 37 | 38 | ## Local Setup 39 | 40 | - Make sure that you have [Go v1.20+](https://go.dev/dl/) installed locally. 41 | - Make sure that you have [Docker](https://docs.docker.com/engine/install/) installed locally. 42 | 43 | ## Leaderboard Application & Migration 44 | 45 | - Run in-memory store instances and the leaderboard application service with `docker compose` using configurations in the `docker-compose.yml` file: 46 | 47 | ```bash 48 | # within the root directory of this example (dragonfly-examples/leaderboard) 49 | docker compose build --no-cache && docker compose up 50 | ``` 51 | 52 | - At this point, the Redis instance (primary) is running on port `6379`. 53 | The Dragonfly instance (replica) is running on port `6380`. 54 | - The primary-replica topology is monitored and managed by a Sentinel cluster of 3 nodes. 55 | - Now, issue the following command to the Redis instance, and the Sentinel cluster will promote Dragonfly as the primary instance: 56 | 57 | ```bash 58 | redis:6379$> SHUTDOWN 59 | ``` 60 | 61 | - Alternatively, we can directly instruct the Sentinel cluster to promote Dragonfly as the primary instance: 62 | 63 | ```bash 64 | sentinel:5001$> SENTINEL FAILOVER leaderboard-primary 65 | OK 66 | ``` -------------------------------------------------------------------------------- /cache-refresh-ahead/app/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/gofiber/fiber/v2" 8 | "github.com/gofiber/fiber/v2/log" 9 | "github.com/gofiber/fiber/v2/middleware/logger" 10 | "github.com/google/uuid" 11 | "github.com/redis/go-redis/v9" 12 | ) 13 | 14 | // Use local Dragonfly instance address & credentials. 15 | func createDragonflyClient() *redis.Client { 16 | client := redis.NewClient(&redis.Options{Addr: "localhost:6379"}) 17 | 18 | err := client.Ping(context.Background()).Err() 19 | if err != nil { 20 | log.Fatal("failed to connect with dragonfly", err) 21 | } 22 | 23 | return client 24 | } 25 | 26 | func createServiceApp() *fiber.App { 27 | var ( 28 | client = createDragonflyClient() 29 | cacheExpiration = time.Second * 100 30 | refreshAheadFactor = 0.5 31 | ) 32 | 33 | // Create cache middleware using the refresh-ahead strategy connecting to the local Dragonfly instance. 34 | userCache := NewCacheRefreshAheadMiddleware( 35 | client, 36 | cacheExpiration, 37 | refreshAheadFactor, 38 | UseRepo().ReadUserBytesByID, 39 | ) 40 | blogCache := NewCacheRefreshAheadMiddleware( 41 | client, 42 | cacheExpiration, 43 | refreshAheadFactor, 44 | UseRepo().ReadBlogBytesByID, 45 | ) 46 | 47 | // Create Fiber application. 48 | app := fiber.New() 49 | app.Use(logger.New()) 50 | 51 | // Register middleware & handlers. 52 | app.Get("/users/:id", userCache.Handler, getUserHandler) 53 | app.Get("/blogs/:id", blogCache.Handler, getBlogHandler) 54 | 55 | return app 56 | } 57 | 58 | func getUserHandler(c *fiber.Ctx) error { 59 | uid, err := uuid.Parse(c.Params("id")) 60 | if err != nil { 61 | return err 62 | } 63 | user, err := UseRepo().ReadUserByID(uid) 64 | if err != nil { 65 | return err 66 | } 67 | return c.JSON(user) 68 | } 69 | 70 | func getBlogHandler(c *fiber.Ctx) error { 71 | uid, err := uuid.Parse(c.Params("id")) 72 | if err != nil { 73 | return err 74 | } 75 | blog, err := UseRepo().ReadBlogByID(uid) 76 | if err != nil { 77 | return err 78 | } 79 | return c.JSON(blog) 80 | } 81 | 82 | func main() { 83 | app := createServiceApp() 84 | if err := app.Listen(":8080"); err != nil { 85 | log.Fatal("failed to start service app", err) 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /background-process-example/go.sum: -------------------------------------------------------------------------------- 1 | github.com/Pallinder/go-randomdata v1.2.0 h1:DZ41wBchNRb/0GfsePLiSwb0PHZmT67XY00lCDlaYPg= 2 | github.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y= 3 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 4 | github.com/gocelery/gocelery v0.0.0-20201111034804-825d89059344 h1:CdLzugydeppabz3V7nQ2k+coT17zqGGwSO/4NiMbdWo= 5 | github.com/gocelery/gocelery v0.0.0-20201111034804-825d89059344/go.mod h1:EVn6ocyTN24XewNuGszlIdaovxPM9/1db4bIAhjyr/A= 6 | github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= 7 | github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= 8 | github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= 9 | github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 10 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 11 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 12 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 13 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 14 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= 15 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= 16 | github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= 17 | github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= 18 | github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271 h1:WhxRHzgeVGETMlmVfqhRn8RIeeNoPr2Czh33I4Zdccw= 19 | github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= 20 | golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= 21 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 22 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= 23 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 24 | -------------------------------------------------------------------------------- /redis-cluster-application-example/app/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "log" 8 | "net/http" 9 | "os" 10 | "strconv" 11 | "strings" 12 | 13 | "github.com/gorilla/mux" 14 | "github.com/redis/go-redis/v9" 15 | ) 16 | 17 | var client redis.UniversalClient 18 | 19 | func init() { 20 | hosts := os.Getenv("REDIS_HOSTS") 21 | if hosts == "" { 22 | log.Fatal("enter value for REDIS_HOSTS") 23 | } 24 | 25 | client = redis.NewClusterClient(&redis.ClusterOptions{Addrs: strings.Split(hosts, ",")}) 26 | err := client.Ping(context.Background()).Err() 27 | if err != nil { 28 | log.Fatal("failed to connect", err) 29 | } 30 | 31 | fmt.Println("connected to redis", hosts) 32 | 33 | load := os.Getenv("LOAD_DATA") 34 | if load != "" { 35 | _load, err := strconv.ParseBool(load) 36 | if err != nil { 37 | log.Fatal("invalid value for LOAD_DATA. use true/false", load) 38 | } 39 | 40 | if _load { 41 | loadData() 42 | } 43 | } 44 | } 45 | 46 | func loadData() { 47 | fmt.Println("loading sample data into redis.....") 48 | 49 | user := map[string]string{} 50 | 51 | for i := 0; i < 100; i++ { 52 | key := "user:" + strconv.Itoa(i) 53 | name := "user-" + strconv.Itoa(i) 54 | email := name + "@foo.com" 55 | user["name"] = name 56 | user["email"] = email 57 | 58 | err := client.HMSet(context.Background(), key, user).Err() 59 | if err != nil { 60 | log.Fatal("failed to load data", err) 61 | } 62 | } 63 | 64 | fmt.Println("data load complete") 65 | } 66 | 67 | type User struct { 68 | Name string `redis:"name" json:"name"` 69 | Email string `redis:"email" json:"email"` 70 | } 71 | 72 | func main() { 73 | r := mux.NewRouter() 74 | r.HandleFunc("/{id}", func(w http.ResponseWriter, r *http.Request) { 75 | id := mux.Vars(r)["id"] 76 | hashName := "user:" + id 77 | fmt.Println("getting data for", hashName) 78 | 79 | var user User 80 | err := client.HMGet(context.Background(), hashName, "name", "email").Scan(&user) 81 | 82 | if err != nil { 83 | http.Error(w, err.Error(), http.StatusInternalServerError) 84 | return 85 | } 86 | 87 | err = json.NewEncoder(w).Encode(user) 88 | if err != nil { 89 | http.Error(w, err.Error(), http.StatusInternalServerError) 90 | return 91 | } 92 | }) 93 | 94 | log.Fatal(http.ListenAndServe(":8080", r)) 95 | } 96 | -------------------------------------------------------------------------------- /cache-aside-with-expiration/app/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/gofiber/fiber/v2" 8 | "github.com/gofiber/fiber/v2/log" 9 | "github.com/gofiber/fiber/v2/middleware/cache" 10 | "github.com/redis/go-redis/v9" 11 | ) 12 | 13 | // Use local Dragonfly instance address & credentials. 14 | func createDragonflyClient() *redis.Client { 15 | client := redis.NewClient(&redis.Options{Addr: "localhost:6379"}) 16 | 17 | err := client.Ping(context.Background()).Err() 18 | if err != nil { 19 | log.Fatal("failed to connect with dragonfly", err) 20 | } 21 | 22 | return client 23 | } 24 | 25 | func createServiceApp() *fiber.App { 26 | client := createDragonflyClient() 27 | 28 | // Create cache middleware connecting to the local Dragonfly instance. 29 | cacheMiddleware := cache.New(cache.Config{ 30 | Storage: &CacheStorage{client: client}, 31 | Expiration: time.Second * 30, 32 | Methods: []string{fiber.MethodGet}, 33 | MaxBytes: 0, // 0 means no limit 34 | StoreResponseHeaders: false, 35 | }) 36 | 37 | // Create Fiber application. 38 | app := fiber.New() 39 | 40 | // Register the cache middleware globally. 41 | // However, the cache middleware itself will only cache GET requests. 42 | app.Use(cacheMiddleware) 43 | 44 | // Register handlers. 45 | app.Get("/users/:id", getUserHandler) 46 | app.Get("/blogs/:id", getBlogHandler) 47 | 48 | return app 49 | } 50 | 51 | func getUserHandler(c *fiber.Ctx) error { 52 | // This is for simplicity and demonstration. 53 | // Handler should read blog from database by ID, let's pretend we are doing so. 54 | time.Sleep(time.Millisecond * 200) 55 | user := User{ 56 | ID: c.Params("id"), 57 | Name: "John Doe", 58 | } 59 | return c.JSON(user) 60 | } 61 | 62 | func getBlogHandler(c *fiber.Ctx) error { 63 | // This is for simplicity and demonstration. 64 | // Handler should read blog from database by ID, let's pretend we are doing so. 65 | time.Sleep(time.Millisecond * 200) 66 | blog := Blog{ 67 | ID: c.Params("id"), 68 | Content: "This is a micro-blog limited to 140 characters.", 69 | } 70 | return c.JSON(blog) 71 | } 72 | 73 | func main() { 74 | app := createServiceApp() 75 | if err := app.Listen(":8080"); err != nil { 76 | log.Fatal("failed to start service app", err) 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /redlock-go/app/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | 7 | "github.com/go-redsync/redsync/v4" 8 | redsyncredis "github.com/go-redsync/redsync/v4/redis" 9 | redsyncpool "github.com/go-redsync/redsync/v4/redis/goredis/v9" 10 | "github.com/redis/go-redis/v9" 11 | ) 12 | 13 | // These hosts are reachable from within the Docker network. 14 | const ( 15 | dragonflyHost0 = "dragonfly-instance-0:6379" 16 | dragonflyHost1 = "dragonfly-instance-1:6379" 17 | dragonflyHost2 = "dragonfly-instance-2:6379" 18 | ) 19 | 20 | const ( 21 | // The name of the global lock. 22 | globalLockKeyName = "my-global-lock" 23 | 24 | // The expiry of the global lock. 25 | globalLockExpiry = time.Minute 26 | 27 | // Number of retries to acquire the global lock. 28 | globalLockRetries = 8 29 | 30 | // The delay between retries to acquire the global lock. 31 | globalLockRetryDelay = 10 * time.Millisecond 32 | ) 33 | 34 | func main() { 35 | // Create three clients for each instance of Dragonfly. 36 | var ( 37 | hosts = []string{dragonflyHost0, dragonflyHost1, dragonflyHost2} 38 | clients = make([]redsyncredis.Pool, len(hosts)) 39 | ) 40 | for idx, addr := range hosts { 41 | client := redis.NewClient(&redis.Options{ 42 | Addr: addr, 43 | }) 44 | clients[idx] = redsyncpool.NewPool(client) 45 | } 46 | 47 | // Create an instance of 'Redsync' to work with locks. 48 | rs := redsync.New(clients...) 49 | 50 | // Create a global lock mutex. 51 | globalMutex := rs.NewMutex( 52 | globalLockKeyName, 53 | redsync.WithExpiry(globalLockExpiry), 54 | redsync.WithTries(globalLockRetries), 55 | redsync.WithRetryDelay(globalLockRetryDelay), 56 | ) 57 | 58 | // Create an HTTP server that exposes an endpoint to acquire the global lock. 59 | // Normally, the lock should be released after the work is done. 60 | // For demonstration purposes, the lock is released after the configured expiry, 61 | // so that you can check the lock keys in Dragonfly instances. 62 | http.HandleFunc("/try-to-acquire-global-lock", func(w http.ResponseWriter, r *http.Request) { 63 | if err := globalMutex.Lock(); err != nil { 64 | http.Error(w, err.Error(), http.StatusBadRequest) 65 | return 66 | } 67 | w.WriteHeader(http.StatusOK) 68 | _, _ = w.Write([]byte("lock acquired")) 69 | }) 70 | 71 | if err := http.ListenAndServe(":8080", nil); err != nil { 72 | panic(err) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /ad-server-cache-bun/README.md: -------------------------------------------------------------------------------- 1 | # Example: Ad Server using Bun, ElysiaJS, and Dragonfly 2 | 3 | In this example, we will build a real-time ad server API using Bun, ElysiaJS, and Dragonfly. 4 | In terms of data types, we use `Hash` to store ad metadata and `Set` to store ad categories and user preferences. 5 | 6 | ## Packages Used 7 | 8 | - [ElysiaJS](https://elysiajs.com/) is a TypeScript framework supercharged by the [Bun](https://bun.sh/) runtime with end-to-end type safety. 9 | - [ioredis](https://github.com/redis/ioredis) is a Redis client for Node.js that can be used to interact with Dragonfly. 10 | - [typebox](https://github.com/sinclairzx81/typebox) is a JSON schema type builder and validator with static type resolution for TypeScript. 11 | 12 | ## Local Setup 13 | 14 | - Make sure that you have [Bun v1.0.6+](https://bun.sh/) installed locally. 15 | - Make sure that you have [Docker](https://docs.docker.com/engine/install/) installed locally. 16 | 17 | ## Run Dragonfly & Service Application 18 | 19 | - Run a Dragonfly instance using Docker: 20 | 21 | ```bash 22 | docker run -p 6379:6379 --ulimit memlock=-1 docker.dragonflydb.io/dragonflydb/dragonfly 23 | ``` 24 | 25 | - Install dependencies and run the service application: 26 | 27 | ```bash 28 | # within the root directory of this example (dragonfly-examples/ad-server-cache-bun) 29 | bun install 30 | bun run dev # (or `bun dev`) 31 | ``` 32 | 33 | - The ad server API would be running on `http://localhost:3000/` 34 | 35 | ## Interact with the Ad Server API 36 | 37 | - Create ad metadata with the following request: 38 | 39 | ```shell 40 | curl --request POST \ 41 | --url http://localhost:3888/ads \ 42 | --header 'Content-Type: application/json' \ 43 | --data '{ 44 | "id": "1", 45 | "title": "Dragonfly - a data store built for modern workloads", 46 | "category": "technology", 47 | "clickURL": "https://www.dragonflydb.io/", 48 | "imageURL": "https://www.dragonflydb.io/blog" 49 | }' 50 | ``` 51 | 52 | - Create or update user preferences with the following request: 53 | 54 | ```shell 55 | curl --request POST \ 56 | --url http://localhost:3888/ads/user_preferences \ 57 | --header 'Content-Type: application/json' \ 58 | --data '{ 59 | "userId": "1", 60 | "categories": ["technology", "sports"] 61 | }' 62 | ``` 63 | 64 | - Retrieve ads for a specific user with the following request: 65 | 66 | ```shell 67 | curl --request GET \ 68 | --url http://localhost:3888/ads/user_preferences/1 69 | ``` 70 | -------------------------------------------------------------------------------- /high-availability/sentinel/docker-compose.yml: -------------------------------------------------------------------------------- 1 | networks: 2 | ha-sentinel-net: 3 | driver: bridge 4 | 5 | services: 6 | dragonfly-0: 7 | container_name: "dragonfly-0" 8 | image: "docker.dragonflydb.io/dragonflydb/dragonfly" 9 | networks: 10 | - "ha-sentinel-net" 11 | ulimits: 12 | memlock: -1 13 | ports: 14 | - "6379:6379" 15 | command: 16 | - "--dir=/data" 17 | - "--port=6379" 18 | dragonfly-1: 19 | container_name: "dragonfly-1" 20 | image: "docker.dragonflydb.io/dragonflydb/dragonfly" 21 | networks: 22 | - "ha-sentinel-net" 23 | depends_on: 24 | - "dragonfly-0" 25 | ulimits: 26 | memlock: -1 27 | ports: 28 | - "6380:6380" 29 | command: 30 | - "--dir=/data" 31 | - "--port=6380" 32 | - "--replicaof=dragonfly-0:6379" 33 | dragonfly-2: 34 | container_name: "dragonfly-2" 35 | image: "docker.dragonflydb.io/dragonflydb/dragonfly" 36 | networks: 37 | - "ha-sentinel-net" 38 | depends_on: 39 | - "dragonfly-0" 40 | ulimits: 41 | memlock: -1 42 | ports: 43 | - "6381:6381" 44 | command: 45 | - "--dir=/data" 46 | - "--port=6381" 47 | - "--replicaof=dragonfly-0:6379" 48 | sentinel-0: 49 | container_name: "sentinel-0" 50 | image: "redis:6.0-alpine" 51 | networks: 52 | - "ha-sentinel-net" 53 | depends_on: 54 | - "dragonfly-0" 55 | - "dragonfly-1" 56 | - "dragonfly-2" 57 | ports: 58 | - "8000:8000" 59 | command: "redis-server /etc/sentinel-config/sentinel.conf --sentinel" 60 | volumes: 61 | - "./config/sentinel-0:/etc/sentinel-config" 62 | sentinel-1: 63 | container_name: "sentinel-1" 64 | image: "redis:6.0-alpine" 65 | networks: 66 | - "ha-sentinel-net" 67 | depends_on: 68 | - "dragonfly-0" 69 | - "dragonfly-1" 70 | - "dragonfly-2" 71 | ports: 72 | - "8001:8001" 73 | command: "redis-server /etc/sentinel-config/sentinel.conf --sentinel" 74 | volumes: 75 | - "./config/sentinel-1:/etc/sentinel-config" 76 | sentinel-2: 77 | container_name: "sentinel-2" 78 | image: "redis:6.0-alpine" 79 | networks: 80 | - "ha-sentinel-net" 81 | depends_on: 82 | - "dragonfly-0" 83 | - "dragonfly-1" 84 | - "dragonfly-2" 85 | ports: 86 | - "8002:8002" 87 | command: "redis-server /etc/sentinel-config/sentinel.conf --sentinel" 88 | volumes: 89 | - "./config/sentinel-2:/etc/sentinel-config" 90 | -------------------------------------------------------------------------------- /redlock-go/README.md: -------------------------------------------------------------------------------- 1 | # Example: Redlock with Go 2 | 3 | [Redlock](https://redis.io/docs/manual/patterns/distributed-locks/) is a recognized algorithm based on Redis for 4 | distributed locks, ensuring consistent operation and protection against failures such as network partitions and Redis crashes. 5 | It operates by having a client application send lock requests, using [`SET`](https://www.dragonflydb.io/docs/command-reference/strings/set.md) commands, to multiple **primary Redis instances**. 6 | The lock is successfully acquired when more than half of these instances agree on the lock acquisition. 7 | To release the lock, the client issues [`DEL`](https://www.dragonflydb.io/docs/command-reference/generic/del) commands to all the instances involved. 8 | Redlock also takes into account the lock validity time, retry on failure, lock extension, and many other aspects, which makes it a robust and reliable solution for distributed locking. 9 | 10 | Since Dragonfly is highly compatible with Redis and both `SET` and `DEL` commands are fully supported, Redlock implementations can be easily used with Dragonfly. 11 | 12 | ## Packages Used 13 | 14 | - The [go-redis](https://github.com/redis/go-redis) client is used. It has strongly typed methods for various commands. 15 | It is also the recommended Go client to interact with Dragonfly. 16 | - The [redsync](https://github.com/go-redsync/redsync) is a Go implementation of the Redlock algorithm. 17 | It is used to acquire and release locks in a distributed environment. 18 | 19 | ## Local Setup 20 | 21 | - Make sure that you have [Go v1.20+](https://go.dev/dl/) installed locally. 22 | - Make sure that you have [Docker](https://docs.docker.com/engine/install/) installed locally. 23 | - Run Dragonfly instances & the Redlock service: 24 | 25 | ```bash 26 | # within the root directory of this example (dragonfly-examples/redlock-go) 27 | docker compose build --no-cache && docker compose up 28 | ``` 29 | 30 | - Use the API endpoint to acquire the lock: 31 | 32 | ```bash 33 | curl -v --url http://localhost:8080/try-to-acquire-global-lock 34 | ``` 35 | 36 | - For the first time, the lock should be acquired successfully: 37 | 38 | ```text 39 | HTTP/1.1 200 OK 40 | Date: Thu, 15 Feb 2024 19:29:09 GMT 41 | Content-Length: 13 42 | Content-Type: text/plain; charset=utf-8 43 | 44 | lock acquired 45 | ``` 46 | 47 | - If we try to acquire the lock again in a short period, it should fail: 48 | 49 | ```text 50 | HTTP/1.1 400 Bad Request 51 | Content-Type: text/plain; charset=utf-8 52 | X-Content-Type-Options: nosniff 53 | Date: Thu, 15 Feb 2024 19:29:56 GMT 54 | Content-Length: 42 55 | 56 | lock already taken, locked nodes: [0 1 2] 57 | ``` 58 | -------------------------------------------------------------------------------- /cache-in-5mins-hono/src/index.ts: -------------------------------------------------------------------------------- 1 | import { eq, and, gt } from "drizzle-orm"; 2 | import { drizzle } from "drizzle-orm/node-postgres"; 3 | import { Hono } from "hono"; 4 | import { serve } from "@hono/node-server"; 5 | import { zValidator } from "@hono/zod-validator"; 6 | import { Redis as Cache } from "ioredis"; 7 | import { stringify as uuidStringify } from "uuid"; 8 | 9 | import { shortLinksTable } from "./schema"; 10 | import * as schema from "./schema"; 11 | import { 12 | shortLinkInsertSchema, 13 | ShortLinkInsert, 14 | ShortLinkSelect, 15 | } from "./validator"; 16 | 17 | // For simplicity, we are using local Dragonfly/Redis and PostgreSQL instances. 18 | // Please ensure they are running locally and adjust the connection details as needed. 19 | const cache = new Cache({ 20 | host: "localhost", 21 | // port: 6379, // Redis running locally. 22 | port: 6380, // Dragonfly running locally. 23 | }); 24 | 25 | const db = drizzle( 26 | "postgresql://local_user_dev:local_pwd_dev@localhost:5432/appdb", 27 | { schema: schema }, 28 | ); 29 | 30 | const app = new Hono(); 31 | 32 | app.post( 33 | "/short-links", 34 | zValidator("json", shortLinkInsertSchema), 35 | async (c) => { 36 | // Validate and transform the request. 37 | const req: ShortLinkInsert = c.req.valid("json"); 38 | 39 | // Save the new record in the database. 40 | await db.insert(shortLinksTable).values(req).execute(); 41 | 42 | // Cache the new record in Redis/Dragonfly. 43 | const expiresAt = Math.trunc(req.expiresAt.getTime() / 1000); 44 | await cache.set(req.id, req.originalUrl, "EXAT", expiresAt); 45 | return c.json(req); 46 | }, 47 | ); 48 | 49 | app.get("/:shortCode", async (c) => { 50 | // Parse the short code as a UUIDv7. 51 | const shortCode = c.req.param("shortCode"); 52 | const idBytes = new Uint8Array(Buffer.from(shortCode, "base64url")); 53 | const id = uuidStringify(idBytes); 54 | 55 | // Read from cache. 56 | const originalUrl = await cache.get(id); 57 | 58 | // Cache hit: redirect. 59 | if (originalUrl) { 60 | return c.redirect(originalUrl); 61 | } 62 | 63 | // Cache miss: read from database, cache the record again if it exists. 64 | const result: ShortLinkSelect | undefined = 65 | await db.query.shortLinksTable.findFirst({ 66 | where: and( 67 | eq(shortLinksTable.id, id), 68 | gt(shortLinksTable.expiresAt, new Date()), 69 | ), 70 | }); 71 | if (!result) { 72 | return c.notFound(); 73 | } 74 | const expiresAt = Math.trunc(result.expiresAt.getTime() / 1000); 75 | await cache.set(result.id, result.originalUrl, "EXAT", expiresAt); 76 | return c.redirect(result.originalUrl); 77 | }); 78 | 79 | // Run the server. 80 | const PORT = 3000; 81 | serve({ fetch: app.fetch, port: PORT }); 82 | console.log(`Server running on http://localhost:${PORT}`); 83 | -------------------------------------------------------------------------------- /langchain-memory/requirements.txt: -------------------------------------------------------------------------------- 1 | aiohttp==3.9.5 2 | aiosignal==1.3.1 3 | annotated-types==0.6.0 4 | anyio==4.3.0 5 | asgiref==3.8.1 6 | async-timeout==4.0.3 7 | attrs==23.2.0 8 | backoff==2.2.1 9 | bcrypt==4.1.2 10 | build==1.2.1 11 | cachetools==5.3.3 12 | certifi==2024.2.2 13 | charset-normalizer==3.3.2 14 | chroma-hnswlib==0.7.3 15 | chromadb==0.4.24 16 | click==8.1.7 17 | coloredlogs==15.0.1 18 | dataclasses-json==0.6.5 19 | Deprecated==1.2.14 20 | distro==1.9.0 21 | exceptiongroup==1.2.1 22 | fastapi==0.110.3 23 | filelock==3.14.0 24 | flatbuffers==24.3.25 25 | frozenlist==1.4.1 26 | fsspec==2024.3.1 27 | google-auth==2.29.0 28 | googleapis-common-protos==1.63.0 29 | grpcio==1.62.2 30 | h11==0.14.0 31 | httpcore==1.0.5 32 | httptools==0.6.1 33 | httpx==0.27.0 34 | huggingface-hub==0.22.2 35 | humanfriendly==10.0 36 | idna==3.7 37 | importlib-metadata==7.0.0 38 | importlib_resources==6.4.0 39 | jsonpatch==1.33 40 | jsonpointer==2.4 41 | kubernetes==29.0.0 42 | langchain==0.1.16 43 | langchain-chroma==0.1.0 44 | langchain-community==0.0.34 45 | langchain-core==0.1.46 46 | langchain-openai==0.1.4 47 | langchain-text-splitters==0.0.1 48 | langsmith==0.1.52 49 | markdown-it-py==3.0.0 50 | marshmallow==3.21.1 51 | mdurl==0.1.2 52 | mmh3==4.1.0 53 | monotonic==1.6 54 | mpmath==1.3.0 55 | multidict==6.0.5 56 | mypy-extensions==1.0.0 57 | numpy==1.26.4 58 | oauthlib==3.2.2 59 | onnxruntime==1.17.3 60 | openai==1.24.0 61 | opentelemetry-api==1.24.0 62 | opentelemetry-exporter-otlp-proto-common==1.24.0 63 | opentelemetry-exporter-otlp-proto-grpc==1.24.0 64 | opentelemetry-instrumentation==0.45b0 65 | opentelemetry-instrumentation-asgi==0.45b0 66 | opentelemetry-instrumentation-fastapi==0.45b0 67 | opentelemetry-proto==1.24.0 68 | opentelemetry-sdk==1.24.0 69 | opentelemetry-semantic-conventions==0.45b0 70 | opentelemetry-util-http==0.45b0 71 | orjson==3.10.1 72 | overrides==7.7.0 73 | packaging==23.2 74 | posthog==3.5.0 75 | protobuf==4.25.3 76 | pulsar-client==3.5.0 77 | pyasn1==0.6.0 78 | pyasn1_modules==0.4.0 79 | pydantic==2.7.1 80 | pydantic_core==2.18.2 81 | Pygments==2.17.2 82 | PyPika==0.48.9 83 | pyproject_hooks==1.1.0 84 | python-dateutil==2.9.0.post0 85 | python-dotenv==1.0.1 86 | PyYAML==6.0.1 87 | redis==5.0.4 88 | regex==2024.4.28 89 | requests==2.31.0 90 | requests-oauthlib==2.0.0 91 | rich==13.7.1 92 | rsa==4.9 93 | shellingham==1.5.4 94 | six==1.16.0 95 | sniffio==1.3.1 96 | SQLAlchemy==2.0.29 97 | starlette==0.37.2 98 | sympy==1.12 99 | tenacity==8.2.3 100 | tiktoken==0.6.0 101 | tokenizers==0.19.1 102 | tomli==2.0.1 103 | tqdm==4.66.2 104 | typer==0.12.3 105 | typing-inspect==0.9.0 106 | typing_extensions==4.11.0 107 | urllib3==2.2.1 108 | uvicorn==0.29.0 109 | uvloop==0.19.0 110 | watchfiles==0.21.0 111 | websocket-client==1.8.0 112 | websockets==12.0 113 | wrapt==1.16.0 114 | yarl==1.9.4 115 | zipp==3.18.1 116 | -------------------------------------------------------------------------------- /ad-server-cache/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "log" 8 | "math/rand" 9 | "net/http" 10 | "strconv" 11 | 12 | "github.com/redis/go-redis/v9" 13 | ) 14 | 15 | var client *redis.Client 16 | 17 | func init() { 18 | // Connect to Redis 19 | client = redis.NewClient(&redis.Options{ 20 | Addr: "localhost:6379", 21 | }) 22 | } 23 | 24 | func main() { 25 | http.HandleFunc("/load", loadAds) 26 | http.HandleFunc("/ads", fetchAds) 27 | log.Fatal(http.ListenAndServe(":8080", nil)) 28 | } 29 | 30 | func loadAds(w http.ResponseWriter, r *http.Request) { 31 | 32 | ctx := context.Background() 33 | 34 | categories := []string{"sports", "technology", "fashion"} 35 | for i := 1; i <= 10; i++ { 36 | adID := "ad:" + strconv.Itoa(i) 37 | title := "Ad " + strconv.Itoa(i) 38 | category := categories[rand.Intn(len(categories))] // random category 39 | imageURL := "http://example.com/images/" + adID // mock image URL 40 | clickURL := "http://example.com/ads/" + adID // mock click URL 41 | client.HSet(ctx, adID, "title", title, "category", category, "imageURL", imageURL, "clickURL", clickURL) 42 | client.SAdd(ctx, "ad_category:"+category, adID) 43 | } 44 | 45 | // Seeding user interest data 46 | for i := 1; i <= 5; i++ { 47 | userID := "user:" + strconv.Itoa(i) 48 | numInterests := rand.Intn(3) + 1 // random number of interests between 1 and 3 49 | interests := make([]interface{}, numInterests) 50 | for j := 0; j < numInterests; j++ { 51 | interests[j] = categories[rand.Intn(len(categories))] // random interests 52 | } 53 | client.SAdd(ctx, userID+":interests", interests...) 54 | } 55 | 56 | fmt.Fprintf(w, "Ads and user interests loaded") 57 | } 58 | 59 | func fetchAds(w http.ResponseWriter, r *http.Request) { 60 | 61 | ctx := r.Context() 62 | 63 | user := r.URL.Query().Get("user") // The user ID should be passed as a query parameter 64 | if user == "" { 65 | http.Error(w, "User not specified", http.StatusBadRequest) 66 | return 67 | } 68 | 69 | fmt.Println("getting ad for user", user) 70 | 71 | interests, err := client.SMembers(ctx, "user:"+user+":interests").Result() 72 | if err != nil { 73 | http.Error(w, "Error fetching user interests", http.StatusInternalServerError) 74 | return 75 | } 76 | 77 | fmt.Println("user", user, "interests", interests) 78 | 79 | var ads []map[string]string 80 | for _, interest := range interests { 81 | 82 | fmt.Println("getting ad for", interest) 83 | 84 | adIDs, _ := client.SMembers(ctx, "ad_category:"+interest).Result() 85 | for _, adID := range adIDs { 86 | //var ad Ad 87 | ad, err := client.HGetAll(ctx, adID).Result() 88 | if err != nil { 89 | http.Error(w, "Error fetching ad info", http.StatusInternalServerError) 90 | return 91 | } 92 | //json.Unmarshal([]byte(adJson), &ad) 93 | ads = append(ads, ad) 94 | } 95 | } 96 | 97 | json.NewEncoder(w).Encode(ads) 98 | } 99 | -------------------------------------------------------------------------------- /pubsub-bullmq-node/.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.toptal.com/developers/gitignore/api/node 2 | # Edit at https://www.toptal.com/developers/gitignore?templates=node 3 | 4 | ### Node ### 5 | # Logs 6 | logs 7 | *.log 8 | npm-debug.log* 9 | yarn-debug.log* 10 | yarn-error.log* 11 | lerna-debug.log* 12 | .pnpm-debug.log* 13 | 14 | # Diagnostic reports (https://nodejs.org/api/report.html) 15 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 16 | 17 | # Runtime data 18 | pids 19 | *.pid 20 | *.seed 21 | *.pid.lock 22 | 23 | # Directory for instrumented libs generated by jscoverage/JSCover 24 | lib-cov 25 | 26 | # Coverage directory used by tools like istanbul 27 | coverage 28 | *.lcov 29 | 30 | # nyc test coverage 31 | .nyc_output 32 | 33 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 34 | .grunt 35 | 36 | # Bower dependency directory (https://bower.io/) 37 | bower_components 38 | 39 | # node-waf configuration 40 | .lock-wscript 41 | 42 | # Compiled binary addons (https://nodejs.org/api/addons.html) 43 | build/Release 44 | 45 | # Dependency directories 46 | node_modules/ 47 | jspm_packages/ 48 | 49 | # Snowpack dependency directory (https://snowpack.dev/) 50 | web_modules/ 51 | 52 | # TypeScript cache 53 | *.tsbuildinfo 54 | 55 | # Optional npm cache directory 56 | .npm 57 | 58 | # Optional eslint cache 59 | .eslintcache 60 | 61 | # Optional stylelint cache 62 | .stylelintcache 63 | 64 | # Microbundle cache 65 | .rpt2_cache/ 66 | .rts2_cache_cjs/ 67 | .rts2_cache_es/ 68 | .rts2_cache_umd/ 69 | 70 | # Optional REPL history 71 | .node_repl_history 72 | 73 | # Output of 'npm pack' 74 | *.tgz 75 | 76 | # Yarn Integrity file 77 | .yarn-integrity 78 | 79 | # dotenv environment variable files 80 | .env 81 | .env.development.local 82 | .env.test.local 83 | .env.production.local 84 | .env.local 85 | 86 | # parcel-bundler cache (https://parceljs.org/) 87 | .cache 88 | .parcel-cache 89 | 90 | # Next.js build output 91 | .next 92 | out 93 | 94 | # Nuxt.js build / generate output 95 | .nuxt 96 | dist 97 | 98 | # Gatsby files 99 | .cache/ 100 | # Comment in the public line in if your project uses Gatsby and not Next.js 101 | # https://nextjs.org/blog/next-9-1#public-directory-support 102 | # public 103 | 104 | # vuepress build output 105 | .vuepress/dist 106 | 107 | # vuepress v2.x temp and cache directory 108 | .temp 109 | 110 | # Docusaurus cache and generated files 111 | .docusaurus 112 | 113 | # Serverless directories 114 | .serverless/ 115 | 116 | # FuseBox cache 117 | .fusebox/ 118 | 119 | # DynamoDB Local files 120 | .dynamodb/ 121 | 122 | # TernJS port file 123 | .tern-port 124 | 125 | # Stores VSCode versions used for testing VSCode extensions 126 | .vscode-test 127 | 128 | # yarn v2 129 | .yarn/cache 130 | .yarn/unplugged 131 | .yarn/build-state.yml 132 | .yarn/install-state.gz 133 | .pnp.* 134 | 135 | ### Node Patch ### 136 | # Serverless Webpack directories 137 | .webpack/ 138 | 139 | # Optional stylelint cache 140 | 141 | # SvelteKit build / generate output 142 | .svelte-kit 143 | 144 | # End of https://www.toptal.com/developers/gitignore/api/node -------------------------------------------------------------------------------- /real-time-statistics-node/.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.toptal.com/developers/gitignore/api/node 2 | # Edit at https://www.toptal.com/developers/gitignore?templates=node 3 | 4 | ### Node ### 5 | # Logs 6 | logs 7 | *.log 8 | npm-debug.log* 9 | yarn-debug.log* 10 | yarn-error.log* 11 | lerna-debug.log* 12 | .pnpm-debug.log* 13 | 14 | # Diagnostic reports (https://nodejs.org/api/report.html) 15 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 16 | 17 | # Runtime data 18 | pids 19 | *.pid 20 | *.seed 21 | *.pid.lock 22 | 23 | # Directory for instrumented libs generated by jscoverage/JSCover 24 | lib-cov 25 | 26 | # Coverage directory used by tools like istanbul 27 | coverage 28 | *.lcov 29 | 30 | # nyc test coverage 31 | .nyc_output 32 | 33 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 34 | .grunt 35 | 36 | # Bower dependency directory (https://bower.io/) 37 | bower_components 38 | 39 | # node-waf configuration 40 | .lock-wscript 41 | 42 | # Compiled binary addons (https://nodejs.org/api/addons.html) 43 | build/Release 44 | 45 | # Dependency directories 46 | node_modules/ 47 | jspm_packages/ 48 | 49 | # Snowpack dependency directory (https://snowpack.dev/) 50 | web_modules/ 51 | 52 | # TypeScript cache 53 | *.tsbuildinfo 54 | 55 | # Optional npm cache directory 56 | .npm 57 | 58 | # Optional eslint cache 59 | .eslintcache 60 | 61 | # Optional stylelint cache 62 | .stylelintcache 63 | 64 | # Microbundle cache 65 | .rpt2_cache/ 66 | .rts2_cache_cjs/ 67 | .rts2_cache_es/ 68 | .rts2_cache_umd/ 69 | 70 | # Optional REPL history 71 | .node_repl_history 72 | 73 | # Output of 'npm pack' 74 | *.tgz 75 | 76 | # Yarn Integrity file 77 | .yarn-integrity 78 | 79 | # dotenv environment variable files 80 | .env 81 | .env.development.local 82 | .env.test.local 83 | .env.production.local 84 | .env.local 85 | 86 | # parcel-bundler cache (https://parceljs.org/) 87 | .cache 88 | .parcel-cache 89 | 90 | # Next.js build output 91 | .next 92 | out 93 | 94 | # Nuxt.js build / generate output 95 | .nuxt 96 | dist 97 | 98 | # Gatsby files 99 | .cache/ 100 | # Comment in the public line in if your project uses Gatsby and not Next.js 101 | # https://nextjs.org/blog/next-9-1#public-directory-support 102 | # public 103 | 104 | # vuepress build output 105 | .vuepress/dist 106 | 107 | # vuepress v2.x temp and cache directory 108 | .temp 109 | 110 | # Docusaurus cache and generated files 111 | .docusaurus 112 | 113 | # Serverless directories 114 | .serverless/ 115 | 116 | # FuseBox cache 117 | .fusebox/ 118 | 119 | # DynamoDB Local files 120 | .dynamodb/ 121 | 122 | # TernJS port file 123 | .tern-port 124 | 125 | # Stores VSCode versions used for testing VSCode extensions 126 | .vscode-test 127 | 128 | # yarn v2 129 | .yarn/cache 130 | .yarn/unplugged 131 | .yarn/build-state.yml 132 | .yarn/install-state.gz 133 | .pnp.* 134 | 135 | ### Node Patch ### 136 | # Serverless Webpack directories 137 | .webpack/ 138 | 139 | # Optional stylelint cache 140 | 141 | # SvelteKit build / generate output 142 | .svelte-kit 143 | 144 | # End of https://www.toptal.com/developers/gitignore/api/node -------------------------------------------------------------------------------- /cache-refresh-ahead/go.sum: -------------------------------------------------------------------------------- 1 | github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= 2 | github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= 3 | github.com/bsm/ginkgo/v2 v2.9.5 h1:rtVBYPs3+TC5iLUVOis1B9tjLTup7Cj5IfzosKtvTJ0= 4 | github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= 5 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 6 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 7 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 8 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 9 | github.com/gofiber/fiber/v2 v2.49.1 h1:0W2DRWevSirc8pJl4o8r8QejDR8TV6ZUCawHxwbIdOk= 10 | github.com/gofiber/fiber/v2 v2.49.1/go.mod h1:nPUeEBUeeYGgwbDm59Gp7vS8MDyScL6ezr/Np9A13WU= 11 | github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= 12 | github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 13 | github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= 14 | github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= 15 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 16 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 17 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 18 | github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= 19 | github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 20 | github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= 21 | github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 22 | github.com/redis/go-redis/v9 v9.1.0 h1:137FnGdk+EQdCbye1FW+qOEcY5S+SpY9T0NiuqvtfMY= 23 | github.com/redis/go-redis/v9 v9.1.0/go.mod h1:urWj3He21Dj5k4TK1y59xH8Uj6ATueP8AH1cY3lZl4c= 24 | github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= 25 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 26 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 27 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 28 | github.com/valyala/fasthttp v1.49.0 h1:9FdvCpmxB74LH4dPb7IJ1cOSsluR07XG3I1txXWwJpE= 29 | github.com/valyala/fasthttp v1.49.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= 30 | github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= 31 | github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= 32 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 33 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 34 | golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= 35 | golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 36 | -------------------------------------------------------------------------------- /background-process-example/worker/worker.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "math/rand" 7 | "net/smtp" 8 | "os" 9 | "os/signal" 10 | "strings" 11 | "syscall" 12 | "time" 13 | 14 | "github.com/gocelery/gocelery" 15 | "github.com/gomodule/redigo/redis" 16 | "github.com/hashicorp/go-uuid" 17 | ) 18 | 19 | const ( 20 | redisHostEnvVar = "REDIS_HOST" 21 | taskNameEnvVar = "TASK_NAME" 22 | smtpServer = "localhost:1025" 23 | taskName = "users.registration.email" 24 | ) 25 | 26 | const fromEmail = "admin@foo.com" 27 | 28 | const emailBodyTemplate = "Hi %s!!\n\nHere is your auto-generated password %s. Visit https://foobar.com/login to login update your password.\n\nCheers,\nTeam FooBar.\n\n[processed by %s]" 29 | 30 | const autogenPassword = "foobarbaz_foobarbaz" 31 | 32 | const emailHeaderTemplate = "From: %s" + "\n" + 33 | "To: %s" + "\n" + 34 | "Subject: Welcome to FooBar! Here are your login instructions\n\n" + 35 | "%s" 36 | 37 | var ( 38 | redisHost string 39 | workerID string 40 | ) 41 | 42 | func init() { 43 | redisHost = os.Getenv(redisHostEnvVar) 44 | if redisHost == "" { 45 | redisHost = "localhost:6379" 46 | } 47 | 48 | rnd, _ := uuid.GenerateUUID() 49 | workerID = "worker-" + rnd 50 | } 51 | 52 | func main() { 53 | redisPool := &redis.Pool{ 54 | Dial: func() (redis.Conn, error) { 55 | c, err := redis.Dial("tcp", redisHost) 56 | if err != nil { 57 | return nil, err 58 | } 59 | return c, err 60 | }, 61 | } 62 | 63 | celeryClient, err := gocelery.NewCeleryClient( 64 | gocelery.NewRedisBroker(redisPool), 65 | &gocelery.RedisCeleryBackend{Pool: redisPool}, 66 | 1, 67 | ) 68 | 69 | if err != nil { 70 | log.Fatal("failed to create celery client ", err) 71 | } 72 | 73 | sendEmail := func(registrationEvent string) { 74 | name := strings.Split(registrationEvent, ",")[0] 75 | userEmail := strings.Split(registrationEvent, ",")[1] 76 | 77 | fmt.Println("user registration info:", name, userEmail) 78 | 79 | sleepFor := rand.Intn(9) + 1 80 | time.Sleep(time.Duration(sleepFor) * time.Second) 81 | 82 | body := fmt.Sprintf(emailBodyTemplate, name, autogenPassword, workerID) 83 | msg := fmt.Sprintf(emailHeaderTemplate, fromEmail, userEmail, body) 84 | 85 | //fmt.Println("email contents\n", msg) 86 | 87 | err := smtp.SendMail(smtpServer, nil, "test@localhost", []string{"foo@bar.com"}, []byte(msg)) 88 | if err != nil { 89 | log.Fatal("failed to send email - ", err) 90 | } 91 | 92 | fmt.Println("sent email to", userEmail) 93 | 94 | /*_, err = redisPool.Get().Do("INCR", workerID) 95 | if err != nil { 96 | fmt.Println("failed to increment counter for", workerID) 97 | }*/ 98 | } 99 | 100 | celeryClient.Register(taskName, sendEmail) 101 | 102 | go func() { 103 | celeryClient.StartWorker() 104 | fmt.Println("celery worker started. worker ID", workerID) 105 | }() 106 | 107 | exit := make(chan os.Signal, 1) 108 | signal.Notify(exit, syscall.SIGINT, syscall.SIGTERM) 109 | 110 | <-exit 111 | fmt.Println("exit signalled") 112 | 113 | celeryClient.StopWorker() 114 | fmt.Println("celery worker stopped") 115 | } 116 | -------------------------------------------------------------------------------- /cache-in-5mins-hono/README.md: -------------------------------------------------------------------------------- 1 | # Caching with Dragonfly in 5 Minutes (JavaScript/TypeScript) 2 | 3 | Learn how to build a URL-shortener backend (with PostgreSQL for storage and Dragonfly for caching) locally in under 5 minutes! 4 | 5 | ## Prerequisites 6 | 7 | In this example, we use the Bun toolchain to run our TypeScript code. [Bun](https://github.com/oven-sh/bun) is an all-in-one TypeScript/JavaScript runtime & toolkit designed for speed, complete with a bundler, test runner, and Node.js-compatible package manager. 8 | 9 | We also use [Docker](https://github.com/docker), which is a platform used to develop, ship, and run applications inside containers. A container is a lightweight, standalone executable that includes everything needed to run a piece of software—code, runtime, system tools, libraries, and settings. 10 | 11 | Last but not least, we also use [`npx`](https://docs.npmjs.com/cli/v8/commands/npx) to make the database migration process easier, since it is used by the DrizzleORM kit. However, if you want to skip `npx`, you can find the migration script ending with the `.sql` extension within this repository and apply it to your local database directly as well. 12 | 13 | Make sure the prerequisites above are installed before moving forward. 14 | 15 | - [Install Bun](https://bun.sh/): v1.2.17+ 16 | - [Install Docker Engine](https://docs.docker.com/engine/install/): Client v27.4.0+, Docker Desktop v4.37.2+ 17 | - [Install `npx`](https://docs.npmjs.com/cli/v8/commands/npx): v11.0.0+ 18 | - If you want to try running the project with NodeJS natively, make sure [NodeJS is installed](https://nodejs.org/en/download): v22.7.0+ 19 | 20 | --- 21 | 22 | ## How to Run 23 | 24 | Note that all commands listed below should be run within the current example project's directory: `dragonfly-examples/cache-in-5mins-hono`, instead of the root directory of `dragonfly-examples`. 25 | 26 | To install dependencies: 27 | 28 | ```sh 29 | bun install 30 | 31 | #=> bun install v1.2.17 (282dda62) 32 | #=> + @types/bun@1.2.17 33 | #=> + @types/pg@8.15.4 34 | #=> + @hono/zod-validator@0.7.0 35 | #=> + drizzle@1.4.0 36 | #=> + drizzle-kit@0.31.4 37 | #=> + drizzle-orm@0.44.2 38 | #=> + drizzle-zod@0.8.2 39 | #=> + hono@4.8.3 40 | #=> + ioredis@5.6.1 41 | #=> + pg@8.16.3 42 | #=> + uuid@11.1.0 43 | #=> + zod@3.25.74 44 | ``` 45 | 46 | To set up the environment locally (for Dragonfly/Redis and PostgreSQL): 47 | 48 | ```sh 49 | docker compose up -d 50 | 51 | #=> [+] Running 4/4 52 | #=> ✔ Network cache-in-5mins-hono_default Created 0.0s 53 | #=> ✔ Container cache-with-hono-redis Started 0.2s 54 | #=> ✔ Container cache-with-hono-dragonfly Started 0.2s 55 | #=> ✔ Container cache-with-hono-postgres Started 0.2s 56 | ``` 57 | 58 | To run the migrations targeting the local PostgreSQL database: 59 | 60 | ```sh 61 | npx drizzle-kit migrate 62 | 63 | #=> No config path provided, using default 'drizzle.config.ts' 64 | #=> Reading config file '/cache-in-5mins-hono/drizzle.config.ts' 65 | #=> Using 'pg' driver for database querying 66 | ``` 67 | 68 | To run the server: 69 | 70 | ```sh 71 | bun run dev 72 | 73 | #=> $ bun run --hot src/index.ts 74 | #=> Started development server: http://localhost:3000 75 | ``` 76 | 77 | --- 78 | 79 | ## Test the API 80 | 81 | To create a new short link: 82 | 83 | ```sh 84 | curl --request POST \ 85 | --url http://localhost:3000/short-links \ 86 | --header 'Content-Type: application/json' \ 87 | --data '{ 88 | "originalUrl": "https://www.google.com/" 89 | }' 90 | ``` 91 | 92 | To test page redirect, use a URL with the returned short code: 93 | 94 | ```sh 95 | localhost:3000/AZfjZEnZd82iKkXXXXXXXX 96 | ``` 97 | -------------------------------------------------------------------------------- /sentinel-migration/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "log" 9 | "net/http" 10 | 11 | "github.com/gorilla/mux" 12 | "github.com/redis/go-redis/v9" 13 | ) 14 | 15 | var client *redis.Client 16 | 17 | const ( 18 | httpPort = ":8080" 19 | sentinelEndpoint = "localhost:5000" 20 | masterNameInSentinelConf = "the_master" 21 | ) 22 | 23 | func main() { 24 | 25 | r := mux.NewRouter() 26 | 27 | r.HandleFunc("/{id}", get).Methods(http.MethodGet) 28 | r.HandleFunc("/{id}", set).Methods(http.MethodPost) 29 | 30 | log.Fatal(http.ListenAndServe(httpPort, r)) 31 | 32 | } 33 | 34 | func init() { 35 | client = redis.NewFailoverClient(&redis.FailoverOptions{ 36 | MasterName: masterNameInSentinelConf, 37 | SentinelAddrs: []string{sentinelEndpoint}, 38 | }) 39 | 40 | err := client.Ping(context.Background()).Err() 41 | if err != nil { 42 | log.Fatalf("failed to connect %v", err) 43 | } 44 | 45 | fmt.Println("connected to redis") 46 | 47 | } 48 | 49 | func get(rw http.ResponseWriter, req *http.Request) { 50 | id := mux.Vars(req)["id"] 51 | val, err := client.Get(context.Background(), "key-"+id).Result() 52 | 53 | if err != nil { 54 | if errors.Is(err, redis.Nil) { 55 | //http.Error(rw, "key-"+id+" not found", http.StatusNotFound) 56 | errResp := ErrorResponse{Op: "get", Message: "key-" + id + " not found", Code: http.StatusNotFound} 57 | json.NewEncoder(rw).Encode(errResp) 58 | return 59 | } 60 | 61 | //http.Error(rw, err.Error(), http.StatusInternalServerError) 62 | fmt.Println("get failed with error", err) 63 | 64 | errResp := ErrorResponse{Op: "get", Message: err.Error(), Code: http.StatusInternalServerError} 65 | json.NewEncoder(rw).Encode(errResp) 66 | return 67 | } 68 | 69 | s := redis.NewSentinelClient(&redis.Options{ 70 | Addr: sentinelEndpoint, 71 | }) 72 | 73 | info, err := s.Master(context.Background(), masterNameInSentinelConf).Result() 74 | if err != nil { 75 | //http.Error(rw, err.Error(), http.StatusInternalServerError) 76 | fmt.Println("sentinel command failed with error", err) 77 | 78 | errResp := ErrorResponse{Op: "get", Message: err.Error(), Code: http.StatusInternalServerError} 79 | json.NewEncoder(rw).Encode(errResp) 80 | 81 | return 82 | } 83 | 84 | node := info["ip"] + ":" + info["port"] 85 | 86 | response := GetResponse{Key: "key-" + id, Value: val, FromNode: node} 87 | 88 | err = json.NewEncoder(rw).Encode(response) 89 | if err != nil { 90 | http.Error(rw, err.Error(), http.StatusInternalServerError) 91 | fmt.Println("json response encoding failed with error", err) 92 | return 93 | } 94 | 95 | fmt.Println("key-" + id + "=" + val + " (from) " + node) 96 | 97 | } 98 | 99 | type GetResponse struct { 100 | Key string `json:"key"` 101 | Value string `json:"value"` 102 | FromNode string `json:"from_node"` 103 | } 104 | 105 | type ErrorResponse struct { 106 | Op string `json:"operation"` 107 | Message string `json:"msg"` 108 | Code int `json:"code"` 109 | } 110 | 111 | func set(rw http.ResponseWriter, req *http.Request) { 112 | id := mux.Vars(req)["id"] 113 | err := client.Set(context.Background(), "key-"+id, "value-"+id, 0).Err() 114 | 115 | if err != nil { 116 | //http.Error(rw, err.Error(), http.StatusInternalServerError) 117 | fmt.Println("set failed with error", err) 118 | 119 | errResp := ErrorResponse{Op: "set", Message: err.Error(), Code: http.StatusInternalServerError} 120 | json.NewEncoder(rw).Encode(errResp) 121 | return 122 | } 123 | 124 | fmt.Println("set key-" + id + "=" + "value-" + id) 125 | } 126 | -------------------------------------------------------------------------------- /real-time-statistics-node/src/index.ts: -------------------------------------------------------------------------------- 1 | import express from 'express' 2 | import dragonfly from './utils/dragonflyClient' 3 | import { keyForCurrMonth, keyForMonth } from './utils/keyGenerator' 4 | import { MONTHLY_ACTIVE_USER_PREFIX, trackMonthlyActiveUsers } from './middleware/activeUser' 5 | import { MONTHLY_LEADERBOARD_PREFIX, addLeaderboardPoints } from './middleware/leaderboard' 6 | 7 | // The Express application. 8 | const app = express() 9 | app.use(express.json()) 10 | 11 | // Dummy endpoint to simulate user actions. 12 | // For the current month: 13 | // - Track this user as an active user. 14 | // - Add points for the user in the leaderboard. 15 | app.post('/user-action', trackMonthlyActiveUsers(dragonfly), addLeaderboardPoints(dragonfly), (req, res) => { 16 | res.json({ message: 'User action finished successfully!' }) 17 | }) 18 | 19 | // Endpoint to get current monthly active users count. 20 | app.get('/active-users-current-month', async (req, res) => { 21 | const key = keyForCurrMonth(MONTHLY_ACTIVE_USER_PREFIX) 22 | const count = await dragonfly.pfcount(key) 23 | res.json({ count }) 24 | }) 25 | 26 | // Endpoint to get active users count for the current and past month as a single value. 27 | app.get('/active-users-past-two-months', async (req, res) => { 28 | const now = new Date() 29 | const lastMonth = new Date(now.getFullYear(), now.getMonth() - 1) 30 | 31 | // Construct the HyperLogLog keys for the current and last months. 32 | const currMonthKey = keyForCurrMonth(MONTHLY_ACTIVE_USER_PREFIX) 33 | const lastMonthKey = keyForMonth(MONTHLY_ACTIVE_USER_PREFIX, lastMonth) 34 | 35 | // Calculate the last second of the current month in Unix timestamp. 36 | // The merge key should expire at that time to allow a new round of "past two months." 37 | const lastDayOfMonth = new Date(now.getFullYear(), now.getMonth() + 1, 0) 38 | lastDayOfMonth.setHours(23, 59, 59, 999); 39 | const expireAt = Math.floor(lastDayOfMonth.getTime() / 1000) 40 | 41 | // PFMERGE destkey [sourcekey [sourcekey ...]] 42 | // 43 | // The PFMERGE command merges the cardinalities from two (or more) source HyperLogLog keys. 44 | // The merged cardinality value is set to the destination key. 45 | // If the destination key exists, it is treated as one of the sources, and its cardinality will be included. 46 | // This design is intended, since HyperLogLog counts a unique value only once. 47 | // 48 | // The following code pipelines 3 commands: PFMERGE, PFCOUNT, and EXPIREAT. 49 | // The result of PFCOUNT is returned as the value. 50 | const mergeKey = 'active_users_past_two_months' 51 | const result = (await dragonfly 52 | .pipeline() 53 | .pfmerge(mergeKey, currMonthKey, lastMonthKey) 54 | .pfcount(mergeKey) 55 | .expireat(mergeKey, expireAt) 56 | .exec()) as [error: Error | null, result: unknown][] 57 | 58 | const [, [, count]] = result 59 | res.json({ count }) 60 | }) 61 | 62 | // Endpoint to get top 10 users from current month's leaderboard. 63 | app.get('/leaderboard-current-month', async (req, res) => { 64 | const key = keyForCurrMonth(MONTHLY_LEADERBOARD_PREFIX) 65 | // ZREVRANGE returns members ordered from highest to lowest score. 66 | // Using the WITHSCORES option to include the scores in result. 67 | const leaderboard = await dragonfly.zrevrange(key, 0, 9, 'WITHSCORES') 68 | 69 | // Convert flat array of [member,score,member,score,...] to array of objects. 70 | const rankings = [] 71 | for (let i = 0; i < leaderboard.length; i += 2) { 72 | rankings.push({ 73 | userId: leaderboard[i], 74 | score: parseInt(leaderboard[i + 1]), 75 | }) 76 | } 77 | 78 | res.json({ rankings }) 79 | }) 80 | 81 | // Start the server. 82 | const PORT = 3000 83 | app.listen(PORT, () => { 84 | console.log(`Server is running on http://localhost:${PORT}`) 85 | }) 86 | -------------------------------------------------------------------------------- /ad-server-cache-bun/src/ads.ts: -------------------------------------------------------------------------------- 1 | import {Redis as Dragonfly} from 'ioredis'; 2 | import {Value} from '@sinclair/typebox/value' 3 | 4 | import {AdMetadata, UserAdPreferences} from './types'; 5 | 6 | export class AdMetadataStore { 7 | private client: Dragonfly; 8 | static readonly AD_PREFIX = 'ad'; 9 | static readonly AD_METADATA_PREFIX = `${AdMetadataStore.AD_PREFIX}:metadata`; 10 | static readonly AD_CATEGORY_PREFIX = `${AdMetadataStore.AD_PREFIX}:category`; 11 | static readonly AD_USER_PREFERENCE_PREFIX = `${AdMetadataStore.AD_PREFIX}:user_preference`; 12 | static readonly SET_MEMBER_COUNT = 10; 13 | 14 | constructor(client: Dragonfly) { 15 | this.client = client; 16 | } 17 | 18 | private metadataKey(metadataId: string): string { 19 | return `${AdMetadataStore.AD_METADATA_PREFIX}:${metadataId}`; 20 | } 21 | 22 | private categoryKey(metadataCategory: string): string { 23 | return `${AdMetadataStore.AD_CATEGORY_PREFIX}:${metadataCategory}`; 24 | } 25 | 26 | private userPreferenceKey(userId: string): string { 27 | return `${AdMetadataStore.AD_USER_PREFERENCE_PREFIX}:${userId}`; 28 | } 29 | 30 | async createAdMetadata(metadata: AdMetadata): Promise { 31 | await this.client.hmset(this.metadataKey(metadata.id), metadata); 32 | await this.client.sadd(this.categoryKey(metadata.category), metadata.id); 33 | } 34 | 35 | async createUserPreference(userAdPreferences: UserAdPreferences): Promise { 36 | await this.client.sadd(this.userPreferenceKey(userAdPreferences.userId), userAdPreferences.categories); 37 | } 38 | 39 | async getAdMetadataListByUserPreference(userId: string): Promise> { 40 | const [_, categoryKeys] = await this.client.sscan( 41 | this.userPreferenceKey(userId), 42 | 0, 43 | 'COUNT', 44 | AdMetadataStore.SET_MEMBER_COUNT, 45 | ); 46 | const pipeline = this.client.pipeline(); 47 | categoryKeys.forEach((category: string) => { 48 | pipeline.sscan( 49 | this.categoryKey(category), 50 | 0, 51 | 'COUNT', 52 | AdMetadataStore.SET_MEMBER_COUNT, 53 | ); 54 | }); 55 | const results = await pipeline.exec(); 56 | if (!results) { 57 | return []; 58 | } 59 | const adIds = results.map(([error, scanResult]) => { 60 | if (error) { 61 | throw error; 62 | } 63 | if (!scanResult) { 64 | return []; 65 | } 66 | // scanResult is a tuple of [string, Array] 67 | // The 1st element is the cursor. 68 | // The 2nd element is the array of ids. 69 | const [_, ids] = scanResult as [string, Array]; 70 | return ids; 71 | }).flat(); 72 | return await this.getAdMetadataList(adIds); 73 | } 74 | 75 | private async getAdMetadataList(ids: Array): Promise> { 76 | const pipeline = this.client.pipeline(); 77 | ids.forEach((id: string) => { 78 | pipeline.hgetall(this.metadataKey(id)) 79 | }); 80 | const results = await pipeline.exec(); 81 | if (!results) { 82 | return []; 83 | } 84 | return results.reduce((acc: Array, [error, hash]) => { 85 | if (error) { 86 | throw error; 87 | } 88 | if (!hash) { 89 | return acc; 90 | } 91 | if (!Value.Check(AdMetadata, hash)) { 92 | return acc; 93 | } 94 | acc.push(hash); 95 | return acc; 96 | }, []); 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /celery-transaction-processing/tasks.py: -------------------------------------------------------------------------------- 1 | from typing import Final 2 | 3 | from celery import Celery 4 | from celery.utils.log import get_task_logger 5 | from web3 import exceptions as web3_exceptions 6 | 7 | import eth 8 | import models 9 | from deps import get_deps, get_constants 10 | 11 | logger = get_task_logger(__name__) 12 | 13 | 14 | class TaskRetryException(Exception): 15 | def __init__(self, message): 16 | self.message = message 17 | super().__init__(message) 18 | 19 | 20 | class TaskNotRetryException(Exception): 21 | def __init__(self, message): 22 | self.message = message 23 | super().__init__(message) 24 | 25 | 26 | # Celery app configuration. 27 | # Use Dragonfly as the message broker and the result storage backend for Celery. 28 | # Dragonfly is wire-protocol compatible with Redis. 29 | # We can use the same Redis URL(s) as long as Dragonfly is running on the port specified. 30 | app = Celery( 31 | 'tasks', 32 | broker=get_constants().get_celery_broker_url(), 33 | backend=get_constants().get_celery_backend_url(), 34 | ) 35 | 36 | TASK_MAX_RETRIES: Final[int] = 32 37 | TASK_RETRY_BACKOFF: Final[int] = 100 38 | TASK_RETRY_BACKOFF_MAX: Final[int] = 1600 39 | 40 | 41 | # Define a Celery task to reconcile a transaction. 42 | # We check the transaction status from the blockchain and update the database accordingly. 43 | # The blockchain is just an example. Many finical systems have similar reconciliation or settlement processes. 44 | # 45 | # Maximum retry count is 32, and the delay between retries is exponential. 46 | # 47 | # The retry delays are [1, 2, 4, 8, 16, ...] x 100 seconds. 48 | # So, the first retry is after 100 seconds, the second retry is after 200 seconds, and so on. 49 | # 50 | # The maximum backoff delay is set to 1600 seconds. 51 | # Thus, the retry delays in this configuration are [100, 200, 400, 800, 1600, 1600, ...] seconds up to 32 retries. 52 | @app.task( 53 | bind=True, 54 | autoretry_for=(TaskRetryException,), 55 | max_retries=TASK_MAX_RETRIES, 56 | retry_backoff=TASK_RETRY_BACKOFF, 57 | retry_backoff_max=TASK_RETRY_BACKOFF_MAX, 58 | retry_jitter=False, 59 | ) 60 | def reconcile_transaction(_self, txn_id: str): 61 | db_gen = get_deps().get_db_session() 62 | db = next(db_gen) 63 | try: 64 | # Read the transaction and the user account from the database. 65 | # Stop if the transaction is already reconciled. 66 | txn = db.query(models.UserAccountTransaction).get(txn_id) 67 | if txn is None: 68 | raise TaskNotRetryException('TransactionNotFound') 69 | if txn.status != models.UserAccountTransactionStatus.PENDING: 70 | raise TaskNotRetryException('TransactionStatusNotPending') 71 | account = db.query(models.UserAccount).get(txn.user_account_id) 72 | 73 | # Check the transaction status from the blockchain. 74 | status_response = eth.get_transaction_status(txn.transaction_hash) 75 | logger.info(status_response) 76 | if status_response.status == models.UserAccountTransactionStatus.PENDING: 77 | raise TaskRetryException('TransactionStatus=PENDING') 78 | 79 | # Update the transaction status and the user account balance within a database transaction. 80 | if status_response.status == models.UserAccountTransactionStatus.SUCCESSFUL: 81 | total_amount_in_wei = txn.transaction_amount_in_wei + txn.transaction_fee_total_in_wei 82 | txn.status = status_response.status 83 | txn.transaction_fee_blockchain_in_wei = status_response.transaction_fee_blockchain_in_wei 84 | account.current_balance_in_wei -= total_amount_in_wei 85 | db.commit() 86 | except web3_exceptions.TransactionNotFound as e: 87 | logger.info(f'TransactionNotFoundOnChainYet: {e}') 88 | raise TaskRetryException('TransactionNotFoundOnChainYet') 89 | finally: 90 | logger.info('transaction reconciliation attempted') 91 | -------------------------------------------------------------------------------- /pubsub-bullmq-node/src/index.ts: -------------------------------------------------------------------------------- 1 | import express from 'express'; 2 | import { Job } from 'bullmq'; 3 | import { DragonflyPublisher } from './pubsub/pub'; 4 | import { DragonflySubscriber } from './pubsub/sub'; 5 | import { DragonflyQueue } from './mq/queue'; 6 | import { DragonflyWorker } from './mq/worker'; 7 | 8 | // Express application. 9 | const app = express(); 10 | app.use(express.json()); 11 | 12 | // Dragonfly native Pub/Sub messaging. 13 | const pubsubChannel = 'my-pub-sub-channel'; 14 | const pubsubMessagerHandler = async (channel: string, message: string) => { 15 | // Simulate some relatively long-running work. 16 | await new Promise(resolve => setTimeout(resolve, 2000)); 17 | console.log(`Pub/Sub message processed: ${message}`); 18 | }; 19 | const dragonflyPub = new DragonflyPublisher(pubsubChannel); 20 | const dragonflySub = new DragonflySubscriber(pubsubChannel, pubsubMessagerHandler); 21 | 22 | app.post('/pub-sub-message', (req, res) => { 23 | dragonflyPub.publish(JSON.stringify(req.body)); 24 | res.json({ message: 'Pub/Sub message received' }); 25 | }); 26 | 27 | // BullMQ's queue, worker, and jobs backed by Dragonfly. 28 | const workerProcessor = async (job: Job) => { 29 | // Simulate some relatively long-running work. 30 | await new Promise(resolve => setTimeout(resolve, 2000)); 31 | } 32 | const dragonflyQueue = DragonflyQueue.create('my-queue', { prefix: '{my-prefix}' }); 33 | const dragonflyWorker = DragonflyWorker.create('my-queue', workerProcessor, { prefix: '{my-prefix}' }) 34 | dragonflyWorker.on('completed', (job) => { 35 | console.log(`BullMQ job (${job.id}) completed:`, job.data); 36 | }); 37 | dragonflyWorker.on('failed', (job, err) => { 38 | console.error(`BullMQ job (${job?.id}) failed:`, err); 39 | }); 40 | 41 | app.post('/bullmq-single-job', (req, res) => { 42 | dragonflyQueue.add('single-job', req.body); 43 | res.json({ message: 'BullMQ single job received' }); 44 | }); 45 | 46 | app.post('/bullmq-delayed-job', (req, res) => { 47 | dragonflyQueue.add('delayed-job', req.body, { delay: 5000 }); 48 | res.json({ message: 'BullMQ delayed job received' }); 49 | }); 50 | 51 | app.post('/bullmq-repeatable-job', (req, res) => { 52 | dragonflyQueue.add('repeatable-job', req.body, { 53 | repeat: { 54 | every: 5000, 55 | limit: 3, 56 | } 57 | }); 58 | res.json({ message: 'BullMQ repeatable job received' }); 59 | }); 60 | 61 | // Server initialization. 62 | const PORT = parseInt(process.env.DRAGONFLY_PORT || '3000', 10); 63 | app.listen(PORT, () => { 64 | console.log(`Server is running on http://localhost:${PORT}`); 65 | }); 66 | 67 | // Server graceful shutdown. 68 | const gracefulShutdown = async () => { 69 | console.log('Shutting down gracefully...'); 70 | 71 | // Close the Express application server first. 72 | await new Promise((resolve) => { 73 | app.listen().close(() => { 74 | console.log('Express server closed'); 75 | resolve(); 76 | }); 77 | }); 78 | 79 | // Close Pub/Sub connections. 80 | dragonflySub.close(); 81 | dragonflyPub.close(); 82 | console.log('Dragonfly Pub/Sub closed'); 83 | 84 | // Close BullMQ connections. 85 | try { 86 | await Promise.all([ 87 | dragonflyWorker.close(), 88 | dragonflyQueue.close() 89 | ]); 90 | console.log('Dragonfly BullMQ closed'); 91 | } catch (error) { 92 | console.error('Error closing BullMQ connections:', error); 93 | } 94 | 95 | process.exit(0); 96 | }; 97 | 98 | // Register the shutdown handlers 99 | process.on('SIGTERM', () => { 100 | gracefulShutdown().catch(error => { 101 | console.error('Error during shutdown:', error); 102 | process.exit(1); 103 | }); 104 | }); 105 | 106 | process.on('SIGINT', () => { 107 | gracefulShutdown().catch(error => { 108 | console.error('Error during shutdown:', error); 109 | process.exit(1); 110 | }); 111 | }); 112 | -------------------------------------------------------------------------------- /ecommerce-cache/app/orders.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "encoding/json" 7 | "fmt" 8 | "log" 9 | "net/http" 10 | "time" 11 | 12 | "github.com/redis/go-redis/v9" 13 | ) 14 | 15 | var db *sql.DB 16 | 17 | const userID = 42 18 | 19 | func init() { 20 | 21 | var err error 22 | // Connect to SQLite 23 | db, err = sql.Open("sqlite3", "./test.db") 24 | if err != nil { 25 | log.Fatal(err) 26 | } 27 | 28 | loadData(20) 29 | 30 | rdb = redis.NewClient(&redis.Options{ 31 | Addr: "localhost:6379", 32 | }) 33 | 34 | } 35 | 36 | func getOrders(rw http.ResponseWriter, req *http.Request) { 37 | key := fmt.Sprintf("user:%d:orders", userID) 38 | var orders []Order 39 | 40 | val, err := rdb.Get(ctx, key).Result() 41 | if err == redis.Nil { 42 | 43 | fmt.Println("fetching data from 'orders' table") 44 | 45 | // Cache miss - get the data from the database 46 | rows, err := db.Query("SELECT order_id, user_id, product_id, quantity FROM orders WHERE user_id = ?", userID) 47 | if err != nil { 48 | http.Error(rw, err.Error(), 500) 49 | return 50 | } 51 | defer rows.Close() 52 | 53 | for rows.Next() { 54 | var order Order 55 | err = rows.Scan(&order.OrderID, &order.UserID, &order.ProductID, &order.Quantity) 56 | if err != nil { 57 | http.Error(rw, err.Error(), 500) 58 | return 59 | } 60 | orders = append(orders, order) 61 | } 62 | 63 | // Convert orders to JSON 64 | jsonOrders, err := json.Marshal(orders) 65 | if err != nil { 66 | http.Error(rw, err.Error(), 500) 67 | return 68 | } 69 | 70 | // Store the data in Redis with an expiration time of 5 seconds (for demonstration purposes) 71 | err = rdb.Set(ctx, key, string(jsonOrders), 5*time.Second).Err() 72 | if err != nil { 73 | http.Error(rw, err.Error(), 500) 74 | return 75 | } 76 | 77 | err = json.NewEncoder(rw).Encode(orders) 78 | if err != nil { 79 | http.Error(rw, err.Error(), 500) 80 | return 81 | } 82 | 83 | return 84 | 85 | } else if err != nil { 86 | http.Error(rw, err.Error(), 500) 87 | return 88 | } 89 | 90 | fmt.Println("fetching cached order data") 91 | 92 | // Cache hit - return the data from Redis 93 | 94 | err = json.Unmarshal([]byte(val), &orders) 95 | if err != nil { 96 | http.Error(rw, err.Error(), 500) 97 | return 98 | } 99 | 100 | err = json.NewEncoder(rw).Encode(orders) 101 | if err != nil { 102 | http.Error(rw, err.Error(), 500) 103 | return 104 | } 105 | } 106 | 107 | var ctx = context.Background() 108 | 109 | func loadData(numberOfOrders int) { 110 | 111 | // Drop the existing orders table 112 | _, err := db.Exec("DROP TABLE IF EXISTS orders") 113 | if err != nil { 114 | log.Fatal("failed drop table", err) 115 | } 116 | 117 | // Create the orders table 118 | _, err = db.Exec(`CREATE TABLE orders ( 119 | order_id INTEGER, 120 | user_id INTEGER, 121 | product_id INTEGER, 122 | quantity INTEGER 123 | )`) 124 | if err != nil { 125 | log.Fatal("failed create table", err) 126 | } 127 | 128 | fmt.Println("created table 'orders'") 129 | 130 | tx, err := db.Begin() 131 | if err != nil { 132 | log.Fatal("failed to begin db tx", err) 133 | } 134 | 135 | stmt, err := tx.Prepare("INSERT INTO orders(order_id, user_id, product_id, quantity) VALUES (?, ?, ?, ?)") 136 | if err != nil { 137 | log.Fatal("prepared statement creation failed failed", err) 138 | } 139 | defer stmt.Close() 140 | 141 | // Insert the orders 142 | for i := 1; i <= numberOfOrders; i++ { 143 | _, err = stmt.Exec(i, userID, i, i*2) // assuming product_id is same as i and quantity is twice i 144 | if err != nil { 145 | log.Fatal("load data failed", err) 146 | } 147 | } 148 | 149 | err = tx.Commit() 150 | if err != nil { 151 | log.Fatal("commit failed", err) 152 | } 153 | 154 | fmt.Println("inserted data in 'orders' table") 155 | 156 | } 157 | 158 | type Order struct { 159 | OrderID int `json:"order_id"` 160 | UserID int `json:"-"` 161 | ProductID int `json:"product_id"` 162 | Quantity int `json:"quantity"` 163 | } 164 | -------------------------------------------------------------------------------- /user-session-management-go/go.sum: -------------------------------------------------------------------------------- 1 | github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= 2 | github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= 3 | github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= 4 | github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= 5 | github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= 6 | github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= 7 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 8 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 9 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 10 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 11 | github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo= 12 | github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ= 13 | github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= 14 | github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= 15 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 16 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 17 | github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= 18 | github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= 19 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 20 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 21 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 22 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 23 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 24 | github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= 25 | github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 26 | github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= 27 | github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= 28 | github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E= 29 | github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= 30 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 31 | github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= 32 | github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= 33 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 34 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 35 | github.com/valyala/fasthttp v1.57.0 h1:Xw8SjWGEP/+wAAgyy5XTvgrWlOD1+TxbbvNADYCm1Tg= 36 | github.com/valyala/fasthttp v1.57.0/go.mod h1:h6ZBaPRlzpZ6O3H5t2gEk1Qi33+TmLvfwgLLp0t9CpE= 37 | github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= 38 | github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= 39 | github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= 40 | github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= 41 | golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= 42 | golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= 43 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 44 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 45 | golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= 46 | golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 47 | -------------------------------------------------------------------------------- /search-blogs-by-vector/search-blogs-by-vector.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "4f51b900-ac24-4cf0-ae2b-6fc6f3ce5b8f", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import pandas as pd\n", 11 | "\n", 12 | "posts = pd.read_csv('blog-with-embeddings.csv', delimiter=',', quotechar='\"', converters={'embedding': pd.eval})\n", 13 | "posts.head()" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "id": "5fa8c493-ee34-4c18-baf1-a2f0a975a25a", 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "import redis\n", 24 | "from redis.commands.search.field import (TextField, VectorField)\n", 25 | "from redis.commands.search.indexDefinition import (IndexDefinition, IndexType)\n", 26 | "\n", 27 | "# Create a Redis client communicating with the local Dragonfly instance.\n", 28 | "client = redis.Redis()\n", 29 | "\n", 30 | "# Create an index 'posts', using the TEXT type for 'title', and the VECTOR type for 'embedding'.\n", 31 | "client.ft(\"posts\").create_index(\n", 32 | " fields = [TextField(\"title\"), VectorField(\"embedding\", \"FLAT\", {\"DIM\": \"1536\"})],\n", 33 | " definition = IndexDefinition(prefix=[\"post:\"], index_type=IndexType.HASH)\n", 34 | ")" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": null, 40 | "id": "b526ef68-a4eb-4e02-92f3-d494320ed660", 41 | "metadata": {}, 42 | "outputs": [], 43 | "source": [ 44 | "import numpy as np\n", 45 | "\n", 46 | "# Store blog post data as HASH values in Dragonfly.\n", 47 | "# Since the index is created for all keys with the 'post:' prefix, these documents will be indexed.\n", 48 | "for i, post in posts.iterrows():\n", 49 | " embedding_bytes = np.array(post['embedding']).astype(np.float32).tobytes()\n", 50 | " client.hset(f\"post:{i}\", mapping={**post, 'embedding': embedding_bytes})" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": null, 56 | "id": "6ac91061-a9a5-4224-8cf6-0a7d3f4e6411", 57 | "metadata": {}, 58 | "outputs": [], 59 | "source": [ 60 | "from redis.commands.search.query import Query\n", 61 | "import openai\n", 62 | "\n", 63 | "# How to get an OpenAI API key: https://platform.openai.com/docs/api-reference/introduction\n", 64 | "# NOTE: Do not share your API key with anyone, do not commit it to git, do not hardcode it in your code.\n", 65 | "openai.api_key = \"{YOUR_OPENAI_API_KEY}\"\n", 66 | "EMBEDDING_MODEL = \"text-embedding-ada-002\"\n", 67 | "\n", 68 | "# Create a vector for a query string using the OpenAI API.\n", 69 | "query = \"How to switch from a multi node redis setup to Dragonfly\"\n", 70 | "query_vec = openai.embeddings.create(input=query, model=EMBEDDING_MODEL).data[0].embedding\n", 71 | "\n", 72 | "# Compose a search query for Dragonfly.\n", 73 | "query_expr = Query(\"*=>[KNN 3 @embedding $query_vector AS vector_score]\").return_fields(\"title\", \"vector_score\").paging(0, 30)\n", 74 | "params = {\"query_vector\": np.array(query_vec).astype(dtype=np.float32).tobytes()}\n", 75 | "\n", 76 | "# Search by query.\n", 77 | "docs = client.ft(\"posts\").search(query_expr, params).docs\n", 78 | "for i, doc in enumerate(docs):\n", 79 | " print(i+1, doc.vector_score, doc.title)" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": null, 85 | "id": "50bd113c-c48a-4554-bc32-6847dbe4395a", 86 | "metadata": {}, 87 | "outputs": [], 88 | "source": [ 89 | "# Get index information.\n", 90 | "client.ft(\"posts\").info()" 91 | ] 92 | } 93 | ], 94 | "metadata": { 95 | "kernelspec": { 96 | "display_name": "Python 3 (ipykernel)", 97 | "language": "python", 98 | "name": "python3" 99 | }, 100 | "language_info": { 101 | "codemirror_mode": { 102 | "name": "ipython", 103 | "version": 3 104 | }, 105 | "file_extension": ".py", 106 | "mimetype": "text/x-python", 107 | "name": "python", 108 | "nbconvert_exporter": "python", 109 | "pygments_lexer": "ipython3", 110 | "version": "3.11.4" 111 | } 112 | }, 113 | "nbformat": 4, 114 | "nbformat_minor": 5 115 | } 116 | -------------------------------------------------------------------------------- /feast-recommendation-duckdb-dragonfly/.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.toptal.com/developers/gitignore/api/python 2 | # Edit at https://www.toptal.com/developers/gitignore?templates=python 3 | 4 | ### Python ### 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | share/python-wheels/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | MANIFEST 32 | 33 | # PyInstaller 34 | # Usually these files are written by a python script from a template 35 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 36 | *.manifest 37 | *.spec 38 | 39 | # Installer logs 40 | pip-log.txt 41 | pip-delete-this-directory.txt 42 | 43 | # Unit test / coverage reports 44 | htmlcov/ 45 | .tox/ 46 | .nox/ 47 | .coverage 48 | .coverage.* 49 | .cache 50 | nosetests.xml 51 | coverage.xml 52 | *.cover 53 | *.py,cover 54 | .hypothesis/ 55 | .pytest_cache/ 56 | cover/ 57 | 58 | # Translations 59 | *.mo 60 | *.pot 61 | 62 | # Django stuff: 63 | *.log 64 | local_settings.py 65 | db.sqlite3 66 | db.sqlite3-journal 67 | 68 | # Flask stuff: 69 | instance/ 70 | .webassets-cache 71 | 72 | # Scrapy stuff: 73 | .scrapy 74 | 75 | # Sphinx documentation 76 | docs/_build/ 77 | 78 | # PyBuilder 79 | .pybuilder/ 80 | target/ 81 | 82 | # Jupyter Notebook 83 | .ipynb_checkpoints 84 | 85 | # IPython 86 | profile_default/ 87 | ipython_config.py 88 | 89 | # pyenv 90 | # For a library or package, you might want to ignore these files since the code is 91 | # intended to run in multiple environments; otherwise, check them in: 92 | # .python-version 93 | 94 | # pipenv 95 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 96 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 97 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 98 | # install all needed dependencies. 99 | #Pipfile.lock 100 | 101 | # poetry 102 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 103 | # This is especially recommended for binary packages to ensure reproducibility, and is more 104 | # commonly ignored for libraries. 105 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 106 | #poetry.lock 107 | 108 | # pdm 109 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 110 | #pdm.lock 111 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 112 | # in version control. 113 | # https://pdm.fming.dev/#use-with-ide 114 | .pdm.toml 115 | 116 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 117 | __pypackages__/ 118 | 119 | # Celery stuff 120 | celerybeat-schedule 121 | celerybeat.pid 122 | 123 | # SageMath parsed files 124 | *.sage.py 125 | 126 | # Environments 127 | .env 128 | .venv 129 | env/ 130 | venv/ 131 | ENV/ 132 | env.bak/ 133 | venv.bak/ 134 | 135 | # Spyder project settings 136 | .spyderproject 137 | .spyproject 138 | 139 | # Rope project settings 140 | .ropeproject 141 | 142 | # mkdocs documentation 143 | /site 144 | 145 | # mypy 146 | .mypy_cache/ 147 | .dmypy.json 148 | dmypy.json 149 | 150 | # Pyre type checker 151 | .pyre/ 152 | 153 | # pytype static type analyzer 154 | .pytype/ 155 | 156 | # Cython debug symbols 157 | cython_debug/ 158 | 159 | # PyCharm 160 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 161 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 162 | # and can be added to the global gitignore or merged into this file. For a more nuclear 163 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 164 | #.idea/ 165 | 166 | ### Python Patch ### 167 | # Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration 168 | poetry.toml 169 | 170 | # ruff 171 | .ruff_cache/ 172 | 173 | # LSP config files 174 | pyrightconfig.json 175 | 176 | # End of https://www.toptal.com/developers/gitignore/api/python 177 | 178 | data/* -------------------------------------------------------------------------------- /celery-transaction-processing/deps.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import Final 3 | 4 | from redis import Redis as Dragonfly, ConnectionPool as DragonflyConnectionPool 5 | from sqlalchemy import create_engine 6 | from sqlalchemy.orm import Session 7 | from sqlalchemy.orm import sessionmaker 8 | from web3 import Web3 9 | 10 | import utils 11 | 12 | 13 | class _Constants: 14 | # Use Dragonfly as the default broker and the result backend for Celery. 15 | DEFAULT_DRAGONFLY_URL: Final[str] = 'redis://localhost:6380/0' 16 | DRAGONFLY_URL_ENV: Final[str] = 'DF_DRAGONFLY_URL' 17 | CELERY_BROKER_URL_ENV: Final[str] = 'DF_CELERY_BROKER_URL' 18 | CELERY_BACKEND_URL_ENV: Final[str] = 'DF_CELERY_BACKEND_URL' 19 | 20 | __dragonfly_url = DEFAULT_DRAGONFLY_URL 21 | if DRAGONFLY_URL_ENV in os.environ: 22 | __dragonfly_url = os.environ[DRAGONFLY_URL_ENV] 23 | 24 | __celery_broker_url = DEFAULT_DRAGONFLY_URL 25 | if CELERY_BROKER_URL_ENV in os.environ: 26 | __celery_broker_url = os.environ[CELERY_BROKER_URL_ENV] 27 | 28 | __celery_backend_url = DEFAULT_DRAGONFLY_URL 29 | if CELERY_BACKEND_URL_ENV in os.environ: 30 | __celery_backend_url = os.environ[CELERY_BACKEND_URL_ENV] 31 | 32 | @staticmethod 33 | def get_dragonfly_url() -> str: 34 | return _Constants.__dragonfly_url 35 | 36 | @staticmethod 37 | def get_celery_broker_url() -> str: 38 | return _Constants.__celery_broker_url 39 | 40 | @staticmethod 41 | def get_celery_backend_url() -> str: 42 | return _Constants.__celery_backend_url 43 | 44 | # Use SQLite as the default database. 45 | DEFAULT_DATABASE_URL: Final[str] = 'sqlite:///./data.db' 46 | DATABASE_URL_ENV: Final[str] = 'DF_DATABASE_URL' 47 | 48 | __database_url = DEFAULT_DATABASE_URL 49 | if DATABASE_URL_ENV in os.environ: 50 | __database_url = os.environ[DATABASE_URL_ENV] 51 | 52 | @staticmethod 53 | def get_database_url() -> str: 54 | return _Constants.__database_url 55 | 56 | # Web3 related constants. 57 | SYSTEM_ACCOUNT_PRIVATE_KEY_ENV: Final[str] = 'DF_SYSTEM_ACCOUNT_PRIVATE_KEY' 58 | WEB3_PROVIDER_URL_KEY: Final[str] = 'DF_WEB3_PROVIDER_URI' 59 | 60 | __system_account_private_key = '' 61 | if SYSTEM_ACCOUNT_PRIVATE_KEY_ENV in os.environ: 62 | __system_account_private_key = os.environ[SYSTEM_ACCOUNT_PRIVATE_KEY_ENV] 63 | else: 64 | raise ValueError(f'{SYSTEM_ACCOUNT_PRIVATE_KEY_ENV} environment variable is not set') 65 | 66 | __web3_provider_url = '' 67 | if WEB3_PROVIDER_URL_KEY in os.environ: 68 | __web3_provider_url = os.environ[WEB3_PROVIDER_URL_KEY] 69 | else: 70 | raise ValueError(f'{WEB3_PROVIDER_URL_KEY} environment variable is not set') 71 | 72 | @staticmethod 73 | def get_system_account_private_key() -> str: 74 | return _Constants.__system_account_private_key 75 | 76 | @staticmethod 77 | def get_web3_provider_url() -> str: 78 | return _Constants.__web3_provider_url 79 | 80 | 81 | __constants_instance = _Constants() 82 | 83 | 84 | def get_constants(): 85 | return __constants_instance 86 | 87 | 88 | class _Deps: 89 | # Database client/session. 90 | __engine = create_engine( 91 | get_constants().get_database_url(), connect_args={"check_same_thread": False} 92 | ) 93 | __session_local = sessionmaker(autocommit=False, autoflush=False, bind=__engine) 94 | 95 | @staticmethod 96 | def get_db_session() -> Session: 97 | db = _Deps.__session_local() 98 | try: 99 | yield db 100 | finally: 101 | db.close() 102 | 103 | # Dragonfly client. 104 | __dragonfly_url = utils.parse_dragonfly_url(get_constants().get_dragonfly_url()) 105 | __dragonfly_conn_pool = DragonflyConnectionPool( 106 | host=__dragonfly_url.host, 107 | port=__dragonfly_url.port, 108 | db=__dragonfly_url.db, 109 | ) 110 | __dragonfly_client = Dragonfly(connection_pool=__dragonfly_conn_pool) 111 | 112 | @staticmethod 113 | def get_dragonfly() -> Dragonfly: 114 | return _Deps.__dragonfly_client 115 | 116 | # Web3 client. 117 | __web3_provider = Web3(Web3.HTTPProvider(get_constants().get_web3_provider_url())) 118 | 119 | @staticmethod 120 | def get_web3() -> Web3: 121 | return _Deps.__web3_provider 122 | 123 | 124 | __deps_instance = _Deps() 125 | 126 | 127 | def get_deps(): 128 | return __deps_instance 129 | -------------------------------------------------------------------------------- /celery-transaction-processing/utils.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Final 3 | from urllib.parse import urlparse 4 | 5 | from redis import Redis as Dragonfly 6 | 7 | import models 8 | 9 | CACHE_NORMAL_EXPIRATION_SECONDS: Final[int] = 60 10 | CACHE_EMPTY_EXPIRATION_SECONDS: Final[int] = 30 11 | 12 | # We use a lock to prevent concurrent transactions for the same user account. 13 | LOCK_EXPIRATION_SECONDS: Final[int] = 10 14 | LOCK_VALUE: Final[str] = "locked" 15 | 16 | # This chain ID is for the Ethereum Sepolia testnet. 17 | # Thus, the 'DF_WEB3_PROVIDER_URI' environment variable should be set to the Sepolia testnet, 18 | # which looks like 'https://sepolia.infura.io/v3/7eaXXXXXXXXXX' if you are using Infura. 19 | SEPOLIA_CHAIN_ID: Final[int] = 11155111 20 | 21 | # We charge a fixed fee for each transaction. 22 | # This fee should be enough to cover the blockchain transaction fee, so we can make a profit. 23 | # Blockchain transaction fees are not fixed and can vary depending on the network congestion. 24 | # We set the total transaction fee to 0.008 ETH, which is an estimation based on the Sepolia testnet. 25 | # Note that 1 ETH = 10^18 Wei, so 0.008 ETH = 8000000000000000 Wei 26 | TOTAL_TRANSACTION_FEE_IN_WEI: Final[int] = 8000000000000000 27 | 28 | 29 | @dataclass 30 | class TransactionRequest: 31 | user_account_id: int 32 | to_public_address: str 33 | transaction_amount_in_wei: int 34 | 35 | 36 | @dataclass 37 | class TransactionResponse: 38 | id: int 39 | transaction_hash: str 40 | from_public_address: str 41 | to_public_address: str 42 | transaction_amount_in_wei: int 43 | transaction_fee_total_in_wei: int 44 | transaction_fee_blockchain_in_wei: int 45 | status: str 46 | 47 | 48 | def user_account_lock_key(user_account_id: int) -> str: 49 | return f"user_account_lock:{user_account_id}" 50 | 51 | 52 | def txn_cache_key(txn_id: int) -> str: 53 | return f"user_account_transaction:{txn_id}" 54 | 55 | 56 | def txn_to_response(txn: models.UserAccountTransaction) -> TransactionResponse: 57 | return TransactionResponse( 58 | id=txn.id, 59 | transaction_hash=txn.transaction_hash, 60 | from_public_address=txn.from_public_address, 61 | to_public_address=txn.to_public_address, 62 | transaction_amount_in_wei=txn.transaction_amount_in_wei, 63 | transaction_fee_total_in_wei=txn.transaction_fee_total_in_wei, 64 | transaction_fee_blockchain_in_wei=txn.transaction_fee_blockchain_in_wei, 65 | status=txn.status.name, 66 | ) 67 | 68 | 69 | def txn_to_dict(txn: models.UserAccountTransaction) -> dict: 70 | return { 71 | "id": txn.id, 72 | "transaction_hash": txn.transaction_hash, 73 | "from_public_address": txn.from_public_address, 74 | "to_public_address": txn.to_public_address, 75 | "transaction_amount_in_wei": txn.transaction_amount_in_wei, 76 | "transaction_fee_total_in_wei": txn.transaction_fee_total_in_wei, 77 | "transaction_fee_blockchain_in_wei": txn.transaction_fee_blockchain_in_wei, 78 | "status": txn.status.name, 79 | } 80 | 81 | 82 | def txn_dict_to_response(txn: dict) -> TransactionResponse: 83 | return TransactionResponse( 84 | id=txn[b"id"], 85 | transaction_hash=txn[b"transaction_hash"], 86 | from_public_address=txn[b"from_public_address"], 87 | to_public_address=txn[b"to_public_address"], 88 | transaction_amount_in_wei=txn[b"transaction_amount_in_wei"], 89 | transaction_fee_total_in_wei=txn[b"transaction_fee_total_in_wei"], 90 | transaction_fee_blockchain_in_wei=txn[b"transaction_fee_blockchain_in_wei"], 91 | status=txn[b"status"], 92 | ) 93 | 94 | 95 | @dataclass 96 | class DragonflyURLParsed: 97 | host: str 98 | port: int 99 | db: int 100 | 101 | 102 | def parse_dragonfly_url(dragonfly_url) -> DragonflyURLParsed: 103 | result = urlparse(dragonfly_url) 104 | host = result.hostname 105 | port = result.port 106 | db = 0 107 | if result.path: 108 | try: 109 | db = int(result.path.strip('/')) 110 | except ValueError: 111 | raise ValueError("Invalid database index in URI") 112 | 113 | if port is None: 114 | port = 6379 115 | 116 | return DragonflyURLParsed(host=host, port=port, db=db) 117 | 118 | 119 | def hset_and_expire( 120 | df: Dragonfly, 121 | key: str, 122 | mapping: dict, 123 | expiration: int, 124 | ): 125 | pipe = df.pipeline() 126 | pipe.hset(key, mapping=mapping) 127 | pipe.expire(key, expiration) 128 | pipe.execute() 129 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.toptal.com/developers/gitignore/api/go,python 2 | # Edit at https://www.toptal.com/developers/gitignore?templates=go,python 3 | 4 | ### Go ### 5 | # If you prefer the allow list template instead of the deny list, see community template: 6 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 7 | # 8 | # Binaries for programs and plugins 9 | *.exe 10 | *.exe~ 11 | *.dll 12 | *.so 13 | *.dylib 14 | 15 | # Test binary, built with `go test -c` 16 | *.test 17 | 18 | # Output of the go coverage tool, specifically when used with LiteIDE 19 | *.out 20 | 21 | # Dependency directories (remove the comment below to include it) 22 | vendor/ 23 | 24 | # Go workspace file 25 | go.work 26 | 27 | ### Python ### 28 | # Byte-compiled / optimized / DLL files 29 | __pycache__/ 30 | *.py[cod] 31 | *$py.class 32 | 33 | # C extensions 34 | 35 | # Distribution / packaging 36 | .Python 37 | build/ 38 | develop-eggs/ 39 | dist/ 40 | downloads/ 41 | eggs/ 42 | .eggs/ 43 | lib/ 44 | lib64/ 45 | parts/ 46 | sdist/ 47 | var/ 48 | wheels/ 49 | share/python-wheels/ 50 | *.egg-info/ 51 | .installed.cfg 52 | *.egg 53 | MANIFEST 54 | 55 | # PyInstaller 56 | # Usually these files are written by a python script from a template 57 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 58 | *.manifest 59 | *.spec 60 | 61 | # Installer logs 62 | pip-log.txt 63 | pip-delete-this-directory.txt 64 | 65 | # Unit test / coverage reports 66 | htmlcov/ 67 | .tox/ 68 | .nox/ 69 | .coverage 70 | .coverage.* 71 | .cache 72 | nosetests.xml 73 | coverage.xml 74 | *.cover 75 | *.py,cover 76 | .hypothesis/ 77 | .pytest_cache/ 78 | cover/ 79 | 80 | # Translations 81 | *.mo 82 | *.pot 83 | 84 | # Django stuff: 85 | *.log 86 | local_settings.py 87 | db.sqlite3 88 | db.sqlite3-journal 89 | 90 | # Flask stuff: 91 | instance/ 92 | .webassets-cache 93 | 94 | # Scrapy stuff: 95 | .scrapy 96 | 97 | # Sphinx documentation 98 | docs/_build/ 99 | 100 | # PyBuilder 101 | .pybuilder/ 102 | target/ 103 | 104 | # Jupyter Notebook 105 | .ipynb_checkpoints 106 | 107 | # IPython 108 | profile_default/ 109 | ipython_config.py 110 | 111 | # pyenv 112 | # For a library or package, you might want to ignore these files since the code is 113 | # intended to run in multiple environments; otherwise, check them in: 114 | # .python-version 115 | 116 | # pipenv 117 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 118 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 119 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 120 | # install all needed dependencies. 121 | #Pipfile.lock 122 | 123 | # poetry 124 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 125 | # This is especially recommended for binary packages to ensure reproducibility, and is more 126 | # commonly ignored for libraries. 127 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 128 | #poetry.lock 129 | 130 | # pdm 131 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 132 | #pdm.lock 133 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 134 | # in version control. 135 | # https://pdm.fming.dev/#use-with-ide 136 | .pdm.toml 137 | 138 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 139 | __pypackages__/ 140 | 141 | # Celery stuff 142 | celerybeat-schedule 143 | celerybeat.pid 144 | 145 | # SageMath parsed files 146 | *.sage.py 147 | 148 | # Environments 149 | .env 150 | .venv 151 | env/ 152 | venv/ 153 | ENV/ 154 | env.bak/ 155 | venv.bak/ 156 | 157 | # Spyder project settings 158 | .spyderproject 159 | .spyproject 160 | 161 | # Rope project settings 162 | .ropeproject 163 | 164 | # mkdocs documentation 165 | /site 166 | 167 | # mypy 168 | .mypy_cache/ 169 | .dmypy.json 170 | dmypy.json 171 | 172 | # Pyre type checker 173 | .pyre/ 174 | 175 | # pytype static type analyzer 176 | .pytype/ 177 | 178 | # Cython debug symbols 179 | cython_debug/ 180 | 181 | # PyCharm 182 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 183 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 184 | # and can be added to the global gitignore or merged into this file. For a more nuclear 185 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 186 | #.idea/ 187 | 188 | ### Python Patch ### 189 | # Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration 190 | poetry.toml 191 | 192 | # ruff 193 | .ruff_cache/ 194 | 195 | # LSP config files 196 | pyrightconfig.json 197 | 198 | # End of https://www.toptal.com/developers/gitignore/api/go,python -------------------------------------------------------------------------------- /feast-recommendation-duckdb-dragonfly/README.md: -------------------------------------------------------------------------------- 1 | ## Feast with DuckDB & DragonflyDB for Recommendation System Example 2 | 3 | This repository showcases a complete example of building a simple feature store for a recommendation system using Feast 4 | with DuckDB as the offline store and Dragonfly as the Redis-compatible high-performance online store. 5 | 6 | This current repository is called a feature repository for Feast. 7 | We configure the feature store in `feature_store.yaml`, generating sample Parquet data representing users, items, 8 | and their interactions, and defining feature views and entities in a Python repository file. 9 | The repository demonstrates how to retrieve historical offline features with point-in-time correctness using timestamps 10 | and how to serve real-time online features for model inference. We also cover best practices for materializing features 11 | into the online store and managing timestamps with UTC awareness, and thus using Feast effectively for both offline 12 | training and online serving in recommendation systems. 13 | 14 | ## How to Run 15 | 16 | - First of all, let's make sure a Dragonfly server instance is running locally: 17 | 18 | ```bash 19 | docker run -p 6379:6379 --ulimit memlock=-1 docker.dragonflydb.io/dragonflydb/dragonfly 20 | ```` 21 | 22 | - Let's use [`uv`](https://github.com/astral-sh/uv) as our Python package and project manager. 23 | 24 | ```bash 25 | # Install uv with curl: 26 | $> curl -LsSf https://astral.sh/uv/install.sh | sh 27 | 28 | # Install uv with wget: 29 | $> wget -qO- https://astral.sh/uv/install.sh | sh 30 | ``` 31 | 32 | - Make sure commands below are executed within the current project's directory (`dragonfly-examples/feast-recommendation`). 33 | 34 | ```bash 35 | $> cd /PATH/TO/dragonfly-examples/feast-recommendation-duckdb-dragonfly 36 | ``` 37 | 38 | - Install dependencies by using `uv sync` (which also automatically creates a `venv`): 39 | 40 | ```bash 41 | $> uv sync 42 | ``` 43 | 44 | - From here on, we always use `uv` to run commands, which ensures that the correct Python version and dependencies are used. 45 | - Generate sample data for the recommendation system: 46 | 47 | ```bash 48 | # Create the 'data' directory to store the Feast registry file and the offline store (DuckDB) database files. 49 | $> mkdir data 50 | 51 | # Run the data generation script to create sample Parquet files representing users, items, and interactions. 52 | $> uv run recommendation_01_data.py 53 | ``` 54 | 55 | - Register the Feast objects (entities, feature views, data sources): 56 | 57 | ```bash 58 | # The 'feast apply' command scans Python files in the current directory for Feast object definitions, 59 | # such as feature views, entities, and data sources. It validates these definitions and then registers them 60 | # by syncing the metadata to the Feast registry. The registry is typically a protobuf binary file stored locally 61 | # or in an object store. As configured in the 'feature_store.yaml', our registry will reside in 'data/registry.db'. 62 | $> uv run feast apply 63 | ``` 64 | 65 | - Loads feature values from offline batch data source (DuckDB) into the online feature store (Dragonfly): 66 | 67 | ```bash 68 | # The 'feast materialize' command loads feature values from offline batch data sources into 69 | # the online feature store over a specified historical time range. It queries the batch sources 70 | # for all relevant feature views during the given time range and writes the latest feature values into 71 | # the configured online store to make them available for low-latency serving during inference. 72 | $> uv run feast materialize '2024-08-01T00:00:00' '2025-08-31T23:59:59' 73 | ``` 74 | 75 | - Retrieve feature values from the offline store (DuckDB) example: 76 | 77 | ```bash 78 | $> uv run recommendation_03_historical_features.py 79 | ``` 80 | 81 | - Retrieve feature values from the online store (Dragonfly) example: 82 | 83 | ```bash 84 | $> uv run recommendation_04_online_features.py 85 | ``` 86 | 87 | - Run the Feast server to serve features via HTTP: 88 | 89 | ```bash 90 | $> uv run feast serve 91 | ``` 92 | 93 | - Query the Feast server for online features example: 94 | 95 | ```bash 96 | $> curl --request POST \ 97 | --url http://localhost:6566/get-online-features \ 98 | --header 'Content-Type: application/json' \ 99 | --data '{ 100 | "features": [ 101 | "user_features:age", 102 | "user_features:gender", 103 | "user_features:avg_rating", 104 | "user_features:preferred_category", 105 | "item_features:category", 106 | "item_features:price", 107 | "item_features:popularity_score", 108 | "item_features:avg_rating", 109 | "interaction_features:view_count", 110 | "interaction_features:last_rating", 111 | "interaction_features:time_since_last_interaction" 112 | ], 113 | "entities": { 114 | "user_id": [1, 2], 115 | "item_id": [101, 102] 116 | }, 117 | "full_feature_names": true 118 | }' 119 | ``` 120 | 121 | - Build and run the Feast server as a Docker image: 122 | 123 | ```bash 124 | $> docker build -t my-feast-server . 125 | $> docker run -p 6566:6566 my-feast-server 126 | ``` 127 | -------------------------------------------------------------------------------- /leaderboard/app/memstore/leaderboard.go: -------------------------------------------------------------------------------- 1 | package memstore 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "time" 8 | 9 | "leaderboard/app/model" 10 | 11 | "github.com/redis/go-redis/v9" 12 | ) 13 | 14 | type Leaderboard struct { 15 | client *redis.Client 16 | } 17 | 18 | func UseLeaderboard(client *redis.Client) *Leaderboard { 19 | return &Leaderboard{client: client} 20 | } 21 | 22 | func (l *Leaderboard) hashKey(userID string) string { 23 | return fmt.Sprintf("leaderboard:users:%v", userID) 24 | } 25 | 26 | func (l *Leaderboard) zsetKey() string { 27 | return fmt.Sprintf("leaderboard:user_scores") 28 | } 29 | 30 | func (l *Leaderboard) IncreaseUserScore( 31 | ctx context.Context, 32 | user *model.User, 33 | score float64, 34 | ) error { 35 | if score <= 0 { 36 | return nil 37 | } 38 | 39 | var ( 40 | now = time.Now().UTC() 41 | zsetKey = l.zsetKey() 42 | hashKey = l.hashKey(user.ID) 43 | nextMonday = MondayOfTime(now.Add(7 * 24 * time.Hour)).Sub(now) 44 | ) 45 | 46 | _, err := l.client.Pipelined(ctx, func(pipe redis.Pipeliner) error { 47 | pipe.ZIncrBy(ctx, zsetKey, score, hashKey) 48 | pipe.HSet(ctx, hashKey, user) 49 | pipe.Expire(ctx, zsetKey, nextMonday) 50 | pipe.Expire(ctx, hashKey, nextMonday) 51 | return nil 52 | }) 53 | 54 | return err 55 | } 56 | 57 | func (l *Leaderboard) ReadTopRankList( 58 | ctx context.Context, 59 | userID string, 60 | ) (*model.LeaderboardResponse, error) { 61 | var ( 62 | resp = new(model.LeaderboardResponse) 63 | zsetKey = l.zsetKey() 64 | hashKey = l.hashKey(userID) 65 | ) 66 | 67 | // Read top 100 users and the current user rank in a pipeline from the Redis Sorted-Set. 68 | commandsZ, err := l.client.Pipelined(ctx, func(pipe redis.Pipeliner) error { 69 | pipe.ZRevRangeWithScores(ctx, zsetKey, 0, 100) 70 | pipe.ZRevRank(ctx, zsetKey, hashKey) 71 | pipe.ZScore(ctx, zsetKey, hashKey) 72 | return nil 73 | }) 74 | // Need to check for the 'redis.Nil' error here because: 75 | // - These commands are read operations. 76 | // - The user may not be in the Redis Sorted-Set. 77 | if err != nil && !errors.Is(err, redis.Nil) { 78 | return nil, err 79 | } 80 | 81 | var ( 82 | topRanks = commandsZ[0].(*redis.ZSliceCmd).Val() 83 | currRank, currRankErr = commandsZ[1].(*redis.IntCmd).Result() 84 | currScore, currScoreErr = commandsZ[2].(*redis.FloatCmd).Result() 85 | ) 86 | if currRankErr != nil && !errors.Is(currRankErr, redis.Nil) { 87 | return nil, currRankErr 88 | } 89 | if currScoreErr != nil && !errors.Is(currScoreErr, redis.Nil) { 90 | return nil, currScoreErr 91 | } 92 | // Add 1 since Redis Sorted-Set rank starts from 0. 93 | currRank += 1 94 | 95 | // Read the user information for the top 100 users in a pipeline from multiple Redis Hashes. 96 | commandsH, err := l.client.Pipelined(ctx, func(pipe redis.Pipeliner) error { 97 | for idx := range topRanks { 98 | pipe.HGetAll(ctx, topRanks[idx].Member.(string)) 99 | } 100 | return nil 101 | }) 102 | 103 | // Construct the response for the current user. 104 | resp.CurrentUser = model.LeaderboardUser{ 105 | User: model.User{ 106 | ID: userID, 107 | }, 108 | } 109 | // The Redis Sorted-Set can potentially store more than constant.LeaderboardMaxRankLimit elements. 110 | // However, we only want to return the current user rank if it is within the limit. 111 | // This aligns with the behaviour of previous-week and all-time leaderboards. 112 | if currRankErr == nil && currRank > 0 { 113 | resp.CurrentUser.Rank = currRank 114 | } 115 | if currScoreErr == nil { 116 | resp.CurrentUser.Score = currScore 117 | } 118 | 119 | // Construct the response for top users with rank. 120 | // This relies on the fact that query response items are sorted DESC. 121 | var ( 122 | rank = 1 123 | prevScoreIdx = 0 124 | prevScoreVal = float64(0) 125 | ) 126 | for idx, command := range commandsH { 127 | u := model.User{} 128 | if err := command.(*redis.MapStringStringCmd).Scan(&u); err != nil { 129 | return nil, err 130 | } 131 | score := topRanks[idx].Score 132 | // Initialize 'prevScoreVal' to the first element score value. 133 | if idx == 0 { 134 | prevScoreVal = score 135 | } 136 | if score != prevScoreVal { 137 | rank += idx - prevScoreIdx 138 | prevScoreIdx = idx 139 | prevScoreVal = score 140 | } 141 | resp.TopUsers = append(resp.TopUsers, model.LeaderboardUser{ 142 | User: u, 143 | Score: score, 144 | Rank: int64(rank), 145 | }) 146 | // Adjust rank for the current user if the user is in the top rank list to get the correct 147 | // dense-rank value. If the user is not in the top rank list, we keep the value returned by 148 | // ZRevRank directly (i.e., currRank), since dense-rank is not necessary in this case. 149 | if u.ID == userID { 150 | resp.CurrentUser.Rank = int64(rank) 151 | } 152 | } 153 | 154 | return resp, nil 155 | } 156 | 157 | func MondayOfTime(ts time.Time) time.Time { 158 | weekday := ts.Weekday() 159 | if weekday == time.Monday { 160 | return ts 161 | } 162 | daysToSubtract := (weekday - time.Monday + 7) % 7 163 | return ts.AddDate(0, 0, -int(daysToSubtract)) 164 | } 165 | -------------------------------------------------------------------------------- /user-session-management-go/app/handler/handler.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "errors" 7 | "strings" 8 | 9 | "usm/app/session" 10 | 11 | "github.com/gofiber/fiber/v2" 12 | "github.com/google/uuid" 13 | ) 14 | 15 | const ( 16 | tokenHeader = "Authorization" 17 | tokenPrefix = "Bearer" 18 | ) 19 | 20 | type UserRegisterOrLoginRequest struct { 21 | Username string `json:"username"` 22 | Password string `json:"password"` 23 | } 24 | 25 | // UserRegistration registers a new user in the database. 26 | func UserRegistration(c *fiber.Ctx) error { 27 | var req UserRegisterOrLoginRequest 28 | if err := c.BodyParser(&req); err != nil { 29 | return c.Status(fiber.StatusBadRequest).SendString(err.Error()) 30 | } 31 | 32 | var count int 33 | err := db.QueryRow("SELECT COUNT(*) FROM users WHERE username = ?", req.Username).Scan(&count) 34 | if err != nil { 35 | return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) 36 | } 37 | if count > 0 { 38 | return c.Status(fiber.StatusBadRequest).SendString("username already taken") 39 | } 40 | 41 | hash, err := hashPassword(req.Password) 42 | if err != nil { 43 | return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) 44 | } 45 | 46 | userID, err := uuid.NewV7() 47 | if err != nil { 48 | return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) 49 | } 50 | _, err = db.Exec( 51 | "INSERT INTO users (id, username, password_hash) VALUES (?, ?, ?)", 52 | userID.String(), req.Username, hash, 53 | ) 54 | if err != nil { 55 | return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) 56 | } 57 | 58 | return c.SendStatus(fiber.StatusCreated) 59 | } 60 | 61 | // UserLogin verifies the user's credentials and generates a pair of refresh and access tokens. 62 | func UserLogin(c *fiber.Ctx) error { 63 | var req UserRegisterOrLoginRequest 64 | if err := c.BodyParser(&req); err != nil { 65 | return c.Status(fiber.StatusBadRequest).SendString(err.Error()) 66 | } 67 | 68 | var userIDStr, hashStr string 69 | err := db.QueryRow( 70 | "SELECT id, password_hash FROM users WHERE username = ?", req.Username, 71 | ).Scan(&userIDStr, &hashStr) 72 | if err != nil && errors.Is(err, sql.ErrNoRows) { 73 | return c.Status(fiber.StatusUnauthorized).SendString("invalid username or password") 74 | } 75 | if err != nil { 76 | return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) 77 | } 78 | 79 | if err := verifyPassword(req.Password, hashStr); err != nil { 80 | return c.Status(fiber.StatusUnauthorized).SendString(err.Error()) 81 | } 82 | 83 | userID, err := uuid.Parse(userIDStr) 84 | if err != nil { 85 | return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) 86 | } 87 | 88 | tokenResponse, err := session.GenerateTokens(context.Background(), userID) 89 | if err != nil { 90 | return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) 91 | } 92 | 93 | return c.JSON(tokenResponse) 94 | } 95 | 96 | // UserLogout requires the user's refresh token to be removed from Dragonfly. 97 | func UserLogout(c *fiber.Ctx) error { 98 | refreshTokenStr := c.Get(tokenHeader) 99 | refreshTokenStr = strings.TrimPrefix(refreshTokenStr, tokenPrefix) 100 | refreshTokenStr = strings.TrimSpace(refreshTokenStr) 101 | 102 | refreshToken := new(session.RefreshToken) 103 | if err := refreshToken.UnmarshalText([]byte(refreshTokenStr)); err != nil { 104 | return c.Status(fiber.StatusBadRequest).SendString(err.Error()) 105 | } 106 | 107 | if err := session.RemoveRefreshToken(context.Background(), *refreshToken); err != nil { 108 | return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) 109 | } 110 | 111 | return c.SendStatus(fiber.StatusOK) 112 | } 113 | 114 | // UserRefreshToken requires the user's refresh token to generate a new pair of refresh and access tokens. 115 | // The newly generated refresh token replaces the old one in Dragonfly. 116 | func UserRefreshToken(c *fiber.Ctx) error { 117 | refreshTokenStr := c.Get(tokenHeader) 118 | refreshTokenStr = strings.TrimPrefix(refreshTokenStr, tokenPrefix) 119 | refreshTokenStr = strings.TrimSpace(refreshTokenStr) 120 | 121 | refreshToken := new(session.RefreshToken) 122 | if err := refreshToken.UnmarshalText([]byte(refreshTokenStr)); err != nil { 123 | return c.Status(fiber.StatusBadRequest).SendString(err.Error()) 124 | } 125 | 126 | tokenResponse, err := session.RefreshTokens(context.Background(), *refreshToken) 127 | if err != nil && errors.Is(err, session.ErrRefreshTokensNotAuthorized) { 128 | return c.Status(fiber.StatusUnauthorized).SendString(err.Error()) 129 | } 130 | if err != nil { 131 | return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) 132 | } 133 | 134 | return c.JSON(tokenResponse) 135 | } 136 | 137 | // UserReadResource requires the user's access token to access the protected resource. 138 | // Normally, the access token would be used to authenticate the user in a middleware. 139 | func UserReadResource(c *fiber.Ctx) error { 140 | accessTokenStr := c.Get(tokenHeader) 141 | accessTokenStr = strings.TrimPrefix(accessTokenStr, tokenPrefix) 142 | accessTokenStr = strings.TrimSpace(accessTokenStr) 143 | 144 | _, err := session.VerifyAccessToken(accessTokenStr) 145 | if err != nil { 146 | return c.Status(fiber.StatusUnauthorized).SendString(err.Error()) 147 | } 148 | 149 | return c.SendStatus(fiber.StatusOK) 150 | } 151 | -------------------------------------------------------------------------------- /langchain-memory/main.py: -------------------------------------------------------------------------------- 1 | from fastapi import FastAPI, Depends, HTTPException 2 | from langchain_core.messages import AIMessage, BaseMessage, HumanMessage 3 | from langchain_openai import ChatOpenAI 4 | from redis import Redis as Dragonfly, ConnectionPool 5 | from sqlalchemy.orm import Session 6 | 7 | import service 8 | from database import SessionLocal 9 | from schemas import ChatMessageCreate, ChatSessionCreate, ChatHistoryCreate 10 | 11 | # Dependencies. 12 | LARGE_LANGUAGE_MODEL_NAME = "gpt-3.5-turbo-1106" 13 | chat_api_client = ChatOpenAI(model=LARGE_LANGUAGE_MODEL_NAME, temperature=0.2) 14 | conn_pool = ConnectionPool(host='localhost', port=6379, db=0) 15 | dragonfly_client = Dragonfly(connection_pool=conn_pool) 16 | 17 | 18 | def get_chat(): 19 | return chat_api_client 20 | 21 | 22 | def get_db_session(): 23 | db = SessionLocal() 24 | try: 25 | yield db 26 | finally: 27 | db.close() 28 | 29 | 30 | def get_dragonfly(): 31 | return dragonfly_client 32 | 33 | 34 | # Create a new FastAPI server. 35 | app = FastAPI() 36 | 37 | 38 | @app.get("/") 39 | def hello(): 40 | return "Hello, LangChain! Hello, OpenAI!" 41 | 42 | 43 | @app.post("/chat-without-memory") 44 | async def chat_without_memory( 45 | chat_message_human: ChatMessageCreate, 46 | chat: ChatOpenAI = Depends(get_chat), 47 | ): 48 | message = HumanMessage(content=chat_message_human.content) 49 | chat_message = chat.invoke([message]) 50 | return {"content": chat_message.content} 51 | 52 | 53 | @app.post("/chat") 54 | async def new_chat( 55 | chat_message_human: ChatMessageCreate, 56 | db: Session = Depends(get_db_session), 57 | df: Dragonfly = Depends(get_dragonfly), 58 | chat: ChatOpenAI = Depends(get_chat), 59 | ) -> service.ChatSessionResponse: 60 | # Invoke the OpenAI API to get the AI response. 61 | message = HumanMessage(content=chat_message_human.content) 62 | chat_message_ai = chat.invoke([message]) 63 | 64 | # Create a new chat session with the first two chat history entries. 65 | chat_session = ChatSessionCreate(llm_name=LARGE_LANGUAGE_MODEL_NAME) 66 | new_chat_histories = __messages_to_histories(chat_message_human, chat_message_ai) 67 | srv = service.DataService(db, df) 68 | chat_session_response = srv.create_chat_session(chat_session, new_chat_histories) 69 | return chat_session_response 70 | 71 | 72 | @app.patch("/chat/{chat_id}") 73 | async def continue_chat( 74 | chat_id: int, 75 | chat_message_human: ChatMessageCreate, 76 | db: Session = Depends(get_db_session), 77 | df: Dragonfly = Depends(get_dragonfly), 78 | chat: ChatOpenAI = Depends(get_chat), 79 | ) -> service.ChatSessionResponse: 80 | # Check if the chat session exists and refresh the cache. 81 | srv = service.DataService(db, df) 82 | prev_chat_session_response = srv.read_chat_histories(chat_id) 83 | if prev_chat_session_response is None: 84 | raise HTTPException(status_code=404, detail="chat not found") 85 | 86 | # Construct messages from chat histories and then append the new human message. 87 | chat_histories = prev_chat_session_response.chat_histories 88 | messages = [] 89 | for i in range(len(chat_histories)): 90 | if chat_histories[i].is_human_message: 91 | messages.append(HumanMessage(content=chat_histories[i].content)) 92 | else: 93 | messages.append(AIMessage(content=chat_histories[i].content)) 94 | messages.append(HumanMessage(content=chat_message_human.content)) 95 | 96 | # Invoke the OpenAI API to get the AI response. 97 | chat_message_ai = chat.invoke(messages) 98 | 99 | # Add two chat history entries to an existing chat session. 100 | new_chat_histories = __messages_to_histories(chat_message_human, chat_message_ai) 101 | chat_session_response = srv.add_chat_histories(prev_chat_session_response, new_chat_histories) 102 | return chat_session_response 103 | 104 | 105 | @app.get("/chat/{chat_id}") 106 | async def read_chat_histories( 107 | chat_id: int, 108 | db: Session = Depends(get_db_session), 109 | df: Dragonfly = Depends(get_dragonfly), 110 | ) -> service.ChatSessionResponse: 111 | srv = service.DataService(db, df) 112 | chat_session_response = srv.read_chat_histories(chat_id) 113 | if chat_session_response is None: 114 | raise HTTPException(status_code=404, detail="chat not found") 115 | return chat_session_response 116 | 117 | 118 | def __messages_to_histories( 119 | chat_message_human: ChatMessageCreate, 120 | chat_message_ai: BaseMessage, 121 | ) -> (ChatMessageCreate, ChatMessageCreate): 122 | chat_history_human = ChatHistoryCreate( 123 | is_human_message=True, 124 | content=chat_message_human.content, 125 | ) 126 | chat_history_ai = ChatHistoryCreate( 127 | is_human_message=False, 128 | content=chat_message_ai.content, 129 | metadata_completion_tokens=chat_message_ai.response_metadata["token_usage"]["completion_tokens"], 130 | metadata_prompt_tokens=chat_message_ai.response_metadata["token_usage"]["prompt_tokens"], 131 | metadata_total_tokens=chat_message_ai.response_metadata["token_usage"]["total_tokens"], 132 | metadata_system_fingerprint=chat_message_ai.response_metadata["system_fingerprint"], 133 | external_id=chat_message_ai.id, 134 | ) 135 | return chat_history_human, chat_history_ai 136 | --------------------------------------------------------------------------------