├── .gitignore ├── docker-compose.yml ├── cache.conf ├── README.md └── nginx.conf /.gitignore: -------------------------------------------------------------------------------- 1 | server_cache/* -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | openai-cache-proxy: 4 | image: nginx:1.23.4 5 | container_name: openai-cache-proxy 6 | ports: 7 | - 127.0.0.1:81:80 8 | volumes: 9 | - ./nginx.conf:/etc/nginx/nginx.conf:ro 10 | - ./cache.conf:/etc/nginx/cache.conf:ro 11 | - ./server_cache:/server_cache:rw -------------------------------------------------------------------------------- /cache.conf: -------------------------------------------------------------------------------- 1 | proxy_cache my_cache; 2 | proxy_cache_methods POST; 3 | 4 | # This is required to make sure that GET requests are not cached 5 | set $no_cache 0; 6 | if ($request_method = GET) { 7 | set $no_cache 1; 8 | } 9 | proxy_cache_bypass $no_cache; 10 | proxy_no_cache $no_cache; 11 | 12 | proxy_cache_key "$request_uri|$request_body"; 13 | proxy_cache_valid 200 999999d; 14 | proxy_cache_valid 404 1m; 15 | proxy_read_timeout 8m; 16 | proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; 17 | proxy_cache_background_update on; 18 | proxy_cache_lock on; 19 | access_log /dev/stdout cache_log; 20 | proxy_ignore_headers Cache-Control; 21 | proxy_ignore_headers "Set-Cookie"; 22 | proxy_hide_header "Set-Cookie"; 23 | add_header X-Cache-Status $upstream_cache_status; 24 | client_body_buffer_size 4m; 25 | 26 | proxy_set_header Connection ''; -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Docker Nginx OpenAI API Cache Reverse proxy 2 | 3 | This project is a simple Docker Nginx project that serves as a cache for the OpenAI API. 4 | 5 | nginx here is preconfigured to work on OpenAI API. 6 | 7 | ### Features: 8 | 9 | - **Works with any client** that allows you to configure the server address (as it acts as a reverse proxy) 10 | - Caches the response of the support endpoints. The key of cache is built from the request uri and body 11 | - Caches the response for the POST requests only 12 | - Use multiple LLM providers 13 | - Returns an "X-Cache-Status" header indicating whether the response was served from cache or not 14 | 15 | ### Supported endpoints: 16 | 17 | - POST /v1/* (default) 18 | - POST /openai/* (openai) 19 | - POST /openrouter/* (openrouter) 20 | - POST /anthropic/* (anthropic) 21 | - POST /groq/* (groq) 22 | - POST /mistral/* (mistral) 23 | - POST /cohere/* (cohere) 24 | - POST /ai21/* (ai21) 25 | 26 | ## Getting Started 27 | 28 | ### Prerequisites 29 | 30 | - Docker 31 | - Docker compose 32 | 33 | ### Installation 34 | 35 | 1. Clone the repository: 36 | 37 | ``` 38 | git clone https://github.com/gpt4thewin/docker-nginx-openai-api-cache.git 39 | cd docker-nginx-openai-api-cache 40 | ``` 41 | 42 | 2. Start the container: 43 | 44 | ``` 45 | docker-compose up -d 46 | ``` 47 | 48 | 3. Test the server 49 | 50 | Setup your credentials: 51 | ``` 52 | OPENAI_API_KEY="...." 53 | ``` 54 | 55 | Run this 2 times or more: 56 | ``` 57 | curl -s -o /dev/null -w "%{http_code}" http://localhost:81/v1/chat/completions \ 58 | -H "Content-Type: application/json" \ 59 | -H "Authorization: Bearer $OPENAI_API_KEY" \ 60 | -d '{ 61 | "model": "gpt-4.1-nano", 62 | "messages": [ 63 | { 64 | "role": "user", 65 | "content": "Hello there !" 66 | } 67 | ], 68 | "temperature": 0, 69 | "max_tokens": 228, 70 | "top_p": 1, 71 | "frequency_penalty": 0, 72 | "presence_penalty": 0 73 | }' 74 | ``` 75 | 76 | 4. Check the logs 77 | 78 | ``` 79 | docker-compose logs 80 | ``` 81 | 82 | The last lines should show something like this 83 | ``` 84 | openai-cache-proxy | 172.28.0.1 - - [29/Feb/2024:19:59:49 +0000] "POST /v1/chat/completions HTTP/1.1" 200 494 "-" "curl/7.80.0" Cache: MISS 85 | openai-cache-proxy | 172.28.0.1 - - [29/Feb/2024:19:59:52 +0000] "POST /v1/chat/completions HTTP/1.1" 200 494 "-" "curl/7.80.0" Cache: HIT 86 | ``` 87 | 88 | 5. Stop the container: 89 | 90 | ``` 91 | docker-compose down 92 | ``` 93 | 94 | ### Usage 95 | 96 | Set your client's API server address to `http://localhost:81/v1` 97 | Once the containers are running, you can use the OpenAI API through the cache by sending requests to the supported URIs. 98 | 99 | URIs that are supported will be forwarded, unless they are cached. 100 | URIs that are not supported will be forwarded normally. 101 | 102 | ### Configuration 103 | 104 | The cache is configured using the `nginx.conf`. You can modify this file to change the cache settings or add additional URIs. 105 | 106 | ## Contributing 107 | 108 | Contributions are welcome! Please submit a pull request or open an issue if you encounter any problems or have suggestions for improvements. 109 | -------------------------------------------------------------------------------- /nginx.conf: -------------------------------------------------------------------------------- 1 | events { 2 | worker_connections 64; 3 | } 4 | 5 | http { 6 | proxy_ssl_server_name on; 7 | 8 | proxy_cache_path /server_cache levels=1:2 keys_zone=my_cache:125m max_size=4g inactive=999999d use_temp_path=off; 9 | 10 | log_format cache_log '[$time_local] ' 11 | '"$request" $status $body_bytes_sent ' 12 | '"$http_referer" "$http_user_agent" ' 13 | 'Cache: $upstream_cache_status'; 14 | 15 | resolver 127.0.0.11 valid=30s; 16 | resolver_timeout 5s; 17 | 18 | server { 19 | listen 80; 20 | proxy_http_version 1.1; 21 | proxy_set_header Host $host; 22 | 23 | proxy_busy_buffers_size 512k; 24 | proxy_buffers 4 512k; 25 | proxy_buffer_size 256k; 26 | 27 | # default endpoints (openai) 28 | 29 | location ~* ^\/v1\/((engines\/.+\/)?(?:chat\/completions|completions|edits|moderations|answers|embeddings))$ { 30 | proxy_set_header Host api.openai.com; 31 | proxy_pass https://api.openai.com; 32 | include cache.conf; 33 | } 34 | 35 | location /v1/(.*) { 36 | proxy_set_header Host api.openai.com; 37 | proxy_pass https://api.openai.com; 38 | include cache.conf; 39 | } 40 | 41 | # provider specific endpoints 42 | 43 | location ~* ^/openai/(.*)?$ { 44 | proxy_set_header Host api.openai.com; 45 | proxy_pass https://api.openai.com/$1; 46 | include cache.conf; 47 | } 48 | 49 | location ~* ^/openrouter/(.*)?$ { 50 | proxy_set_header Host openrouter.ai; 51 | proxy_pass https://openrouter.ai/api/$1; 52 | include cache.conf; 53 | } 54 | 55 | location ~* ^/anthropic/(.*)?$ { 56 | proxy_set_header Host api.anthropic.com; 57 | proxy_pass https://api.anthropic.com/$1; 58 | include cache.conf; 59 | } 60 | 61 | # !! endpoints bellow are untested !! 62 | 63 | location ~* ^/google/(.*)?$ { 64 | proxy_set_header Host generativelanguage.googleapis.com; 65 | proxy_pass https://generativelanguage.googleapis.com/v1beta/$1; 66 | include cache.conf; 67 | } 68 | 69 | location ~* ^/cohere/(.*)?$ { 70 | proxy_set_header Host api.cohere.ai; 71 | proxy_pass https://api.cohere.ai/$1; 72 | include cache.conf; 73 | } 74 | 75 | location ~* ^/ai21/(.*)?$ { 76 | proxy_set_header Host api.ai21.com; 77 | proxy_pass https://api.ai21.com/studio/$1; 78 | include cache.conf; 79 | } 80 | 81 | location ~* ^/mistral/(.*)?$ { 82 | proxy_set_header Host api.mistral.ai; 83 | proxy_pass https://api.mistral.ai/$1; 84 | include cache.conf; 85 | } 86 | 87 | # Note: Azure OpenAI and Amazon Bedrock are not included as they require specific resource names or regions 88 | 89 | location ~* ^/huggingface/(.*)?$ { 90 | proxy_set_header Host api-inference.huggingface.co; 91 | proxy_pass https://api-inference.huggingface.co/models/$1; 92 | include cache.conf; 93 | } 94 | 95 | location ~* ^/together/(.*)?$ { 96 | proxy_set_header Host api.together.xyz; 97 | proxy_pass https://api.together.xyz/$1; 98 | include cache.conf; 99 | } 100 | } 101 | } --------------------------------------------------------------------------------