├── .gitmodules ├── LICENSE ├── README.md ├── conf └── default.conf ├── content.lua ├── init.lua ├── log.lua └── stats └── content.lua /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "external/bitset"] 2 | path = external/bitset 3 | url = git@github.com:bsm/bitset.lua.git 4 | [submodule "external/lua-resty-http"] 5 | path = external/lua-resty-http 6 | url = git@github.com:liseen/lua-resty-http.git 7 | [submodule "external/nginx_log_by_lua.git"] 8 | path = external/nginx_log_by_lua.git 9 | url = git@github.com:mtourne/nginx_log_by_lua.git 10 | [submodule "external/nginx_log_by_lua"] 11 | path = external/nginx_log_by_lua 12 | url = git@github.com:mtourne/nginx_log_by_lua.git 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Cassiano Aquino 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ranger 2 | ====== 3 | 4 | Ranger is a HTTP partial content Range header enforcement script 5 | -------------------------------------------------------------------------------- /conf/default.conf: -------------------------------------------------------------------------------- 1 | log_format rt_cache '$remote_addr - $upstream_cache_status [$time_local] ' 2 | '"$request" $status $body_bytes_sent ' 3 | '"$http_referer" "$http_user_agent" "$http_range"'; 4 | 5 | log_format ranger_cache '$remote_addr - $ranger_cache_status [$time_local] ' 6 | '"$request" $status $body_bytes_sent ' 7 | '"$http_referer" "$http_user_agent" "$http_range"'; 8 | 9 | proxy_cache_path /dev/shm/nginx/ levels=1:2 keys_zone=default:100m inactive=24h max_size=100m; 10 | 11 | map $request_method $disable_cache { 12 | HEAD 1; 13 | default 0; 14 | } 15 | 16 | lua_package_path "external/lua-resty-http/lib/?.lua;external/nginx_log_by_lua/?.lua;external/bitset/lib/?.lua;;"; 17 | 18 | lua_shared_dict file_dict 5M; 19 | lua_shared_dict log_dict 1M; 20 | lua_shared_dict cache_dict 1M; 21 | lua_shared_dict chunk_dict 10M; 22 | 23 | init_by_lua_file 'ranger/init.lua'; 24 | 25 | # Server that has the lua code and will be accessed by clients 26 | server { 27 | listen 80 default; 28 | server_name _; 29 | server_name_in_redirect off; 30 | 31 | set $ranger_cache_status $upstream_cache_status; 32 | 33 | access_log /var/log/nginx/access-default.log; 34 | access_log /var/log/nginx/ranger-default.log ranger_cache; 35 | 36 | index index.html index.htm; 37 | root /var/www; 38 | 39 | lua_check_client_abort on; 40 | lua_code_cache on; 41 | 42 | resolver 4.2.2.2; 43 | server_tokens off; 44 | resolver_timeout 1s; 45 | 46 | location / { 47 | try_files $uri $uri/ index.html; 48 | } 49 | 50 | # files that will be handled by ranger 51 | location ~ [^/]\.dat(/|$) { 52 | lua_http10_buffering off; 53 | content_by_lua_file 'ranger/content.lua'; 54 | log_by_lua_file 'ranger/log.lua'; 55 | } 56 | 57 | location = /stats/data { 58 | content_by_lua_file 'ranger/stats/content.lua'; 59 | } 60 | 61 | 62 | } 63 | 64 | # Server that works as a backend to the lua code 65 | server { 66 | listen 8080; 67 | 68 | access_log /var/log/nginx/cache.log rt_cache; 69 | 70 | resolver 4.2.2.2; 71 | server_tokens off; 72 | resolver_timeout 1s; 73 | 74 | location / { 75 | proxy_no_cache $disable_cache; 76 | proxy_cache_valid 200 24h; 77 | proxy_cache_valid 206 24h; 78 | proxy_cache_key "$scheme$proxy_host$request_uri$http_range"; 79 | proxy_set_header Range $http_range; 80 | proxy_set_header If-Range $http_if_range; 81 | proxy_set_header If-None-Match ""; 82 | proxy_set_header If-Modified-Since ""; 83 | add_header X-Cache $upstream_cache_status; 84 | proxy_ignore_headers Expires; 85 | proxy_ignore_headers Cache-Control; 86 | proxy_cache_use_stale error timeout http_502; 87 | proxy_cache default; 88 | proxy_cache_min_uses 1; 89 | proxy_set_header Host backend-hostname; 90 | proxy_pass http://backend.host.com:8080/; 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /content.lua: -------------------------------------------------------------------------------- 1 | -- includes 2 | local http = require("resty.http") -- https://github.com/liseen/lua-resty-http 3 | local cjson = require("cjson") 4 | local bslib = require("bitset") -- https://github.com/bsm/bitset.lua 5 | 6 | -- basic configuration 7 | local block_size = 256*1024 -- Block size 256k 8 | local backend = "http://127.0.0.1:8080/" -- backend 9 | local fcttl = 30 -- Time to cache HEAD requests 10 | 11 | local bypass_headers = { 12 | ["expires"] = "Expires", 13 | ["content-type"] = "Content-Type", 14 | ["last-modified"] = "Last-Modified", 15 | ["expires"] = "Expires", 16 | ["cache-control"] = "Cache-Control", 17 | ["server"] = "Server", 18 | ["content-length"] = "Content-Length", 19 | ["p3p"] = "P3P", 20 | ["accept-ranges"] = "Accept-Ranges" 21 | } 22 | 23 | local httpc = http.new() 24 | 25 | local cache_dict = ngx.shared.cache_dict 26 | local file_dict = ngx.shared.file_dict 27 | local chunk_dict = ngx.shared.chunk_dict 28 | 29 | local sub = string.sub 30 | local tonumber = tonumber 31 | local ceil = math.ceil 32 | local floor = math.floor 33 | local error = error 34 | local null = ngx.null 35 | local match = ngx.re.match 36 | 37 | local start = 0 38 | local stop = -1 39 | 40 | ngx.status = 206 -- Default HTTP status 41 | 42 | -- register on_abort callback 43 | local ok, err = ngx.on_abort(function () 44 | ngx.exit(499) 45 | end) 46 | if not ok then 47 | ngx.err(ngx.LOG, "Can't register on_abort function.") 48 | ngx.exit(500) 49 | end 50 | 51 | -- try reading values from dict, if not issue a HEAD request and save the value 52 | local updating, flags = file_dict:get(ngx.var.uri .. "-update") 53 | repeat 54 | updating, flags = file_dict:get(ngx.var.uri .. "-update") 55 | ngx.sleep(0.1) 56 | until not updating 57 | 58 | local origin_headers = {} 59 | local origin_info = file_dict:get(ngx.var.uri .. "-info") 60 | if not origin_info then 61 | file_dict:set(ngx.var.uri .. "-update", true, 5) 62 | local ok, code, headers, status, body = httpc:request { 63 | url = backend .. ngx.var.uri, 64 | method = 'HEAD' 65 | } 66 | for key, value in pairs(bypass_headers) do 67 | origin_headers[value] = headers[key] 68 | end 69 | origin_info = cjson.encode(origin_headers) 70 | file_dict:set(ngx.var.uri .. "-info", origin_info, fcttl) 71 | file_dict:delete(ngx.var.uri .. "-update") 72 | end 73 | 74 | origin_headers = cjson.decode(origin_info) 75 | 76 | -- parse range header 77 | local range_header = ngx.req.get_headers()["Range"] or "bytes=0-" 78 | local matches, err = match(range_header, "^bytes=(\\d+)?-([^\\\\]\\d+)?", "joi") 79 | if matches then 80 | if matches[1] == nil and matches[2] then 81 | stop = (origin_headers["Content-Length"] - 1) 82 | start = (stop - matches[2]) + 1 83 | else 84 | start = matches[1] or 0 85 | stop = matches[2] or (origin_headers["Content-Length"] - 1) 86 | end 87 | else 88 | stop = (origin_headers["Content-Length"] - 1) 89 | end 90 | 91 | for header, value in pairs(origin_headers) do 92 | ngx.header[header] = value 93 | end 94 | 95 | local cl = origin_headers["Content-Length"] 96 | ngx.header["Content-Length"] = (stop - (start - 1)) 97 | ngx.header["Content-Range"] = "bytes " .. start .. "-" .. stop .. "/" .. cl 98 | 99 | block_stop = (ceil(stop / block_size) * block_size) 100 | block_start = (floor(start / block_size) * block_size) 101 | 102 | -- hits / miss info 103 | local chunk_info, flags = chunk_dict:get(ngx.var.uri) 104 | local chunk_map = bslib:new() 105 | if chunk_info then 106 | chunk_map.nums = cjson.decode(chunk_info) 107 | end 108 | 109 | local bytes_miss, bytes_hit = 0, 0 110 | 111 | for block_range_start = block_start, stop, block_size do 112 | local block_range_stop = (block_range_start + block_size) - 1 113 | local block_id = (floor(block_range_start / block_size)) 114 | local content_start = 0 115 | local content_stop = block_size 116 | 117 | local block_status = chunk_map:get(block_id) 118 | 119 | if block_range_start == block_start then 120 | content_start = (start - block_range_start) 121 | end 122 | 123 | if (block_range_stop + 1) == block_stop then 124 | content_stop = (stop - block_range_start) + 1 125 | end 126 | 127 | if block_status then 128 | bytes_hit = bytes_hit + (content_stop - content_start) 129 | else 130 | bytes_miss = bytes_miss + (content_stop - content_start) 131 | end 132 | end 133 | 134 | if bytes_miss > 0 then 135 | ngx.var.ranger_cache_status = "MISS" 136 | ngx.header["X-Cache"] = "MISS" 137 | else 138 | ngx.var.ranger_cache_status = "HIT" 139 | ngx.header["X-Cache"] = "HIT" 140 | end 141 | ngx.header["X-Bytes-Hit"] = bytes_hit 142 | ngx.header["X-Bytes-Miss"] = bytes_miss 143 | 144 | ngx.send_headers() 145 | 146 | -- fetch the content from the backend 147 | for block_range_start = block_start, stop, block_size do 148 | local block_range_stop = (block_range_start + block_size) - 1 149 | local block_id = (floor(block_range_start / block_size)) 150 | local content_start = 0 151 | local content_stop = -1 152 | 153 | local req_params = { 154 | url = backend .. ngx.var.uri, 155 | method = 'GET', 156 | headers = { 157 | Range = "bytes=" .. block_range_start .. "-" .. block_range_stop, 158 | } 159 | } 160 | 161 | req_params["body_callback"] = function(data, chunked_header, ...) 162 | if chunked_header then return end 163 | ngx.print(data) 164 | ngx.flush(true) 165 | end 166 | 167 | if block_range_start == block_start then 168 | req_params["body_callback"] = nil 169 | content_start = (start - block_range_start) 170 | end 171 | 172 | if (block_range_stop + 1) == block_stop then 173 | req_params["body_callback"] = nil 174 | content_stop = (stop - block_range_start) + 1 175 | end 176 | 177 | local ok, code, headers, status, body = httpc:request(req_params) 178 | if body then 179 | ngx.print(sub(body, (content_start + 1), content_stop)) -- lua count from 1 180 | end 181 | 182 | if ngx.re.match(headers["x-cache"],"HIT") then 183 | chunk_map:set(block_id) 184 | cache_dict:incr("cache_hit", 1) 185 | else 186 | chunk_map:clear(block_id) 187 | cache_dict:incr("cache_miss", 1) 188 | end 189 | end 190 | chunk_dict:set(ngx.var.uri,cjson.encode(chunk_map.nums)) 191 | ngx.eof() 192 | return ngx.exit(206) 193 | -------------------------------------------------------------------------------- /init.lua: -------------------------------------------------------------------------------- 1 | local cache_dict = ngx.shared.cache_dict 2 | 3 | cache_dict:set("cache_hit", 0) 4 | cache_dict:set("cache_miss", 0) 5 | -------------------------------------------------------------------------------- /log.lua: -------------------------------------------------------------------------------- 1 | local logging = require("logging") 2 | 3 | local log_dict = ngx.shared.log_dict 4 | 5 | local request_time = ngx.now() - ngx.req.start_time() 6 | logging.add_plot(log_dict, "request_time", request_time) 7 | -------------------------------------------------------------------------------- /stats/content.lua: -------------------------------------------------------------------------------- 1 | local logging = require("logging") 2 | local cjson = require("cjson") 3 | 4 | local cache_dict = ngx.shared.cache_dict 5 | local log_dict = ngx.shared.log_dict 6 | 7 | local count, avg, elapsed_time = logging.get_plot(log_dict, "request_time") 8 | local hits, flags = cache_dict:get("cache_hit") 9 | local misses, flags = cache_dict:get("cache_miss") 10 | 11 | local output = {} 12 | 13 | output["last_t"] = elapsed_time 14 | output["count"] = count 15 | output["avgreqt"] = avg 16 | if avg > 0 then 17 | output["qps"] = count / elapsed_time 18 | else 19 | output["qps"] = 0 20 | end 21 | output["cache"] = {} 22 | output["cache"]["hits"] = hits 23 | output["cache"]["misses"] = misses 24 | 25 | ngx.say(cjson.encode(output)) 26 | ngx.exit(ngx.OK) 27 | --------------------------------------------------------------------------------