├── .gitignore ├── CODE_OF_CONDUCT.md ├── Makefile ├── README.rst ├── lib ├── body_reader.lua ├── rate_limit.lua ├── spec_reader.lua ├── url.lua ├── util.lua └── videur.lua ├── rational.md ├── spec ├── VAS.rst └── mig_example.json └── tests ├── support.py ├── test_rate_limit.py └── test_spec_reader.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.un~ 2 | *.pyc 3 | .Python 4 | include 5 | bin 6 | *.swp 7 | lib/python* 8 | man 9 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Community Participation Guidelines 2 | 3 | This repository is governed by Mozilla's code of conduct and etiquette guidelines. 4 | For more details, please read the 5 | [Mozilla Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/). 6 | 7 | ## How to Report 8 | For more information on how to report violations of the Community Participation Guidelines, please read our '[How to Report](https://www.mozilla.org/about/governance/policies/participation/reporting/)' page. 9 | 10 | 16 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | OPENRESTY_PREFIX=/usr/local/openresty 2 | PREFIX ?= /usr/local 3 | LUA_INCLUDE_DIR ?= $(PREFIX)/include 4 | LUA_LIB_DIR ?= $(PREFIX)/lib/lua/$(LUA_VERSION) 5 | INSTALL ?= install 6 | LUA_TREE = $(PREFIX)/lib 7 | VIRTUALENV = virtualenv 8 | 9 | .PHONY: install build test 10 | 11 | all: ; 12 | 13 | install: all 14 | $(INSTALL) -d $(DESTDIR)/$(LUA_LIB_DIR)/videur 15 | $(INSTALL) lib/*.lua $(DESTDIR)/$(LUA_LIB_DIR)/videur/ 16 | 17 | build: all 18 | luarocks --tree=$(LUA_TREE) install luasec 19 | luarocks --tree=$(LUA_TREE) install lua-resty-http 20 | luarocks --tree=$(LUA_TREE) install lua-cjson 21 | luarocks --tree=$(LUA_TREE) install lrexlib-posix 22 | luarocks --tree=$(LUA_TREE) install date 23 | 24 | export PATH := ./lib:$(PATH) 25 | 26 | test: all 27 | $(VIRTUALENV) --no-site-packages . 28 | bin/pip install git+git://github.com/tarekziade/NginxTest 29 | bin/pip install nose 30 | bin/pip install webtest 31 | bin/pip install WSGIProxy2 32 | export PATH 33 | bin/nosetests -sv tests 34 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Videur 2 | ====== 3 | 4 | Pronounced: */vidœʀ/* 5 | 6 | ***experimental project*** 7 | 8 | Videur is a Lua library for OpenResty that will automatically parse 9 | an API specification file provided by a web server and proxy incoming 10 | Nginx requests to that server. 11 | 12 | Videur takes care of rejecting requests that do not comply with the 13 | specification definitions, such as: 14 | 15 | - unknown GET arguments 16 | - bad types or out of limits arguments 17 | - missing authorization headers 18 | - POST body too big 19 | - too many requests per second on a given API 20 | - etc.. 21 | 22 | To get a detailed list of rules that can be used, 23 | look at the `Videur API Specification 0.1 24 | document `_ 25 | 26 | 27 | Installation 28 | ------------ 29 | 30 | To install Videur, you need to have an OpenResty environment deployed. 31 | 32 | Then you can run:: 33 | 34 | make install 35 | 36 | If you have a specific Lua lib directory, you can use the **LUA_LIB_DIR** and 37 | **LUA_TREE** options. 38 | 39 | This command will simply copy all the lua files of the Videur lib into 40 | the OpenResty lib directory. 41 | 42 | 43 | Usage 44 | ----- 45 | 46 | Using Videur in Nginx is done in three directives. 47 | 48 | First of all, you need to define a couple of Lua shared dicts: 49 | 50 | - **cached_spec**, where Videur will store the API specification the backend provided 51 | - **stats**, where Videur keeps track of the hits for its rate limiting feature 52 | 53 | Then you need to set a **spec_url** variable with the URL of the API spec. 54 | This URL should be a JSON document as defined in the `Videur API 55 | Specification 0.1 document `_ 56 | 57 | Last, the **access_by_lua_file** directive needs to point to the 58 | **dynamic_proxy_pass.lua** script from the Videur library. 59 | 60 | 61 | Example:: 62 | 63 | http { 64 | lua_shared_dict cached_spec 512k; 65 | lua_shared_dict stats 512k; 66 | 67 | server { 68 | listen 80; 69 | set $spec_url "http://127.0.0.1:8282/api-specs"; 70 | access_by_lua_file "dynamic_proxy_pass.lua"; 71 | } 72 | } 73 | 74 | 75 | -------------------------------------------------------------------------------- /lib/body_reader.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- body_reader 3 | -- 4 | local util = require "util" 5 | local len = string.len 6 | 7 | function check_size(max_size) 8 | local content_length = tonumber(ngx.req.get_headers()['content-length']) 9 | local method = ngx.req.get_method() 10 | 11 | if not max_size then 12 | return 13 | end 14 | 15 | if content_length then 16 | if content_length > max_size then 17 | -- if the header says it's bigger we can drop now... 18 | ngx.exit(413) 19 | end 20 | end 21 | -- ...but we won't trust it if it says it's smaller 22 | local sock, err = ngx.req.socket() 23 | if not sock then 24 | if err == 'no body' then 25 | return 26 | else 27 | return util.bad_request(err) 28 | end 29 | end 30 | 31 | local chunk_size = 4096 32 | if content_length then 33 | if content_length < chunk_size then 34 | chunk_size = content_length 35 | end 36 | end 37 | 38 | sock:settimeout(0) 39 | 40 | -- reading the request body 41 | ngx.req.init_body(128 * 1024) 42 | local size = 0 43 | 44 | while true do 45 | if content_length then 46 | if size >= content_length then 47 | break 48 | end 49 | end 50 | local data, err, partial = sock:receive(chunk_size) 51 | data = data or partial 52 | if not data then 53 | return bad_request("Missing data") 54 | end 55 | 56 | 57 | ngx.req.append_body(data) 58 | size = size + len(data) 59 | 60 | if size >= max_size then 61 | ngx.exit(413) 62 | end 63 | 64 | local less = content_length - size 65 | if less < chunk_size then 66 | chunk_size = less 67 | end 68 | end 69 | ngx.req.finish_body() 70 | end 71 | 72 | -- public interface 73 | return { 74 | check_size = check_size 75 | } 76 | -------------------------------------------------------------------------------- /lib/rate_limit.lua: -------------------------------------------------------------------------------- 1 | 2 | function check_rate(target, rates) 3 | for __, rate in pairs(rates) do 4 | 5 | local max_hits = rate.hits + 1 6 | local throttle_time = rate.seconds 7 | 8 | -- how many hits we got on this match ? 9 | local stats = ngx.shared.stats 10 | local stats_key = target .. ":" .. rate.match() 11 | local hits = stats:get(stats_key) 12 | 13 | if not hits then 14 | stats:set(stats_key, 1, throttle_time) 15 | else 16 | hits = hits + 1 17 | stats:set(stats_key, hits, throttle_time) 18 | if hits >= max_hits then 19 | ngx.status = 429 20 | ngx.header.content_type = 'text/plain; charset=us-ascii' 21 | ngx.print("Rate limit exceeded.") 22 | ngx.log(ngx.ERR, "Rate limit exceeded.") 23 | ngx.exit(ngx.HTTP_OK) 24 | end 25 | end 26 | end 27 | end 28 | 29 | 30 | -- public interface 31 | return { 32 | check_rate = check_rate 33 | } 34 | -------------------------------------------------------------------------------- /lib/spec_reader.lua: -------------------------------------------------------------------------------- 1 | 2 | -- reads the proxy server specs to generate the actual routing 3 | -- rejects anything that's not 4 | local cjson = require "cjson" 5 | local rex = require "rex_posix" 6 | local date = require "date" 7 | local len = string.len 8 | local util = require "util" 9 | 10 | 11 | function get_location(spec_url, cached_spec) 12 | local last_updated = cached_spec:get("last-updated") 13 | 14 | -- update the spec if needed 15 | -- TODO: add a Last-Modified header + check every 5mn maybe 16 | -- TODO: make sure it gets reloaded on sighup 17 | 18 | 19 | -- TODO: we need a way to invalidate the cache 20 | if not last_updated then 21 | ngx.log(ngx.INFO, "Loading the api-spec file") 22 | 23 | local body, version, resources = nil 24 | -- we need to load it from the backend 25 | body = util.fetch_http_body(spec_url) 26 | cached_spec:set("raw_body", body) 27 | body = cjson.decode(body) -- todo catch parse error 28 | 29 | -- grabbing the values and setting them in mem 30 | local service = body.service 31 | cached_spec:set('location', service.location) 32 | version = service.version 33 | cached_spec:set('version', service.version) 34 | 35 | for location, desc in pairs(service.resources) do 36 | local verbs = {} 37 | 38 | for verb, def in pairs(desc) do 39 | local definition = cjson.encode(def or {}) 40 | local t, l = location:match('(%a+):(.*)') 41 | if t == 'regexp' then 42 | cached_spec:set("regexp:" .. verb .. ":" .. l, definition) 43 | else 44 | cached_spec:set(verb .. ":" .. location, definition) 45 | end 46 | verbs[verb] = true 47 | end 48 | cached_spec:set("verbs:" .. location, util.implode(",", verbs)) 49 | end 50 | last_updated = os.time() 51 | cached_spec:set("last-updated", last_updated) 52 | return service.location 53 | else 54 | return cached_spec:get('location') 55 | end 56 | end 57 | 58 | 59 | function _repl_var(match, captures) 60 | local res = 'ngx.var.' .. match 61 | res = res .. ' or ""' 62 | return res 63 | end 64 | 65 | 66 | function _repl_header(match, captures) 67 | local res = 'ngx.var.http_' .. match 68 | res = res .. ' or ""' 69 | return res 70 | end 71 | 72 | 73 | function convert_match(expr) 74 | -- TODO: take care of the ipv4 and ipv6 fields 75 | local expr = expr:lower() 76 | expr = expr:gsub("-", "_") 77 | expr = rex.gsub(expr, 'header:([a-zA-Z\\-_0-9]+)', _repl_header) 78 | expr = rex.gsub(expr, 'var:([a-zA-Z\\-_0-9]+)', _repl_var) 79 | expr = expr:gsub("and", " .. ':::' .. ") 80 | expr = "return " .. expr 81 | return loadstring(expr) 82 | end 83 | 84 | 85 | function match(spec_url, cached_spec) 86 | -- get the location from the spec url 87 | local location = get_location(spec_url, cached_spec) 88 | 89 | -- now let's see if we have a direct match 90 | local method = ngx.req.get_method() 91 | local key = method .. ":" .. ngx.var.uri 92 | local cached_value = cached_spec:get(key) 93 | 94 | if not cached_value then 95 | -- we don't - we can try a regexp match now 96 | for __, expr in ipairs(cached_spec:get_keys()) do 97 | local t, v, l = expr:match('(%a+):(.*):(.*)') 98 | if t == 'regexp' then 99 | if rex.match(ngx.var.uri, l) and v == method then 100 | cached_value = cached_spec:get(expr) 101 | break 102 | end 103 | end 104 | end 105 | 106 | if not cached_value then 107 | -- we don't! 108 | -- if we are serving / we can send back a page 109 | -- TODO: whitelist of URLS ? 110 | if ngx.var.uri == '/' then 111 | ngx.say("Welcome to Nginx/Videur") 112 | return ngx.exit(200) 113 | else 114 | local existing_verbs = cached_spec:get("verbs:/dashboard") 115 | 116 | if not existing_verbs then 117 | return ngx.exit(ngx.HTTP_NOT_FOUND) 118 | else 119 | -- XXX that does not seem to be applied 120 | ngx.header['Allow'] = existing_verbs 121 | return ngx.exit(ngx.HTTP_NOT_ALLOWED) 122 | end 123 | end 124 | end 125 | end 126 | 127 | -- 128 | -- checking the query arguments 129 | -- 130 | local definition = cjson.decode(cached_value) 131 | local params = definition.parameters or {} 132 | local limits = definition.limits or {} 133 | local args = ngx.req.get_uri_args() 134 | 135 | -- let's check if we have all required args first 136 | local provided_args = util.Keys(args) 137 | 138 | for key, value in pairs(params) do 139 | if value.required and not provided_args[key] then 140 | return util.bad_request("Missing " .. key) 141 | end 142 | end 143 | 144 | -- now let's validate the args we got 145 | -- TODO: we should build all those regexps when we read the spec file 146 | -- and have them loaded in the cache so we don't 147 | -- do it again 148 | for key, val in pairs(args) do 149 | local constraint = params[key] 150 | if constraint then 151 | if constraint['validation'] then 152 | local validation = constraint['validation'] 153 | local t, v = validation:match('(%a+):(.*)') 154 | if not t then 155 | -- not a prefix: 156 | t = validation 157 | v = '' 158 | end 159 | 160 | if t == 'regexp' then 161 | if not rex.match(val, v) then 162 | -- the value does not match the constraints 163 | return util.bad_request("Field does not match " .. key) 164 | end 165 | elseif t == 'digits' then 166 | local pattern = '[0-9]{' .. v .. '}' 167 | if not rex.match(val, pattern) then 168 | -- the value does not match the constraints 169 | return util.bad_request("Field does not match " .. key) 170 | end 171 | elseif t == 'values' then 172 | local pattern = '(' .. v .. ')' 173 | if not rex.match(val, pattern) then 174 | -- the value does not match the constraints 175 | return util.bad_request("Field does not match " .. key) 176 | end 177 | elseif t == 'datetime' then 178 | if not pcall(function() date(val) end) then 179 | return util.bad_request("Field is not RFC3339 " .. key) 180 | end 181 | else 182 | -- XXX should be detected at indexing time 183 | return util.bad_request("Bad rule " .. t) 184 | end 185 | end 186 | else 187 | -- this field was not declared 188 | return util.bad_request("Unknown field " .. key) 189 | end 190 | end 191 | 192 | -- let's prepare the limits by converting the match value 193 | -- into a lua expression 194 | -- XXX should be cached too... 195 | local parsed_limits = {max_body_size = limits.max_body_size} 196 | for key, value in pairs(limits) do 197 | if key == 'rates' then 198 | local rates = value 199 | for __, rate in pairs(rates) do 200 | rate.match = convert_match(rate.match) 201 | end 202 | parsed_limits.rates = rates 203 | end 204 | end 205 | 206 | return location, parsed_limits, params 207 | end 208 | 209 | 210 | -- public interface 211 | return { 212 | match = match 213 | } 214 | -------------------------------------------------------------------------------- /lib/url.lua: -------------------------------------------------------------------------------- 1 | -- neturl.lua - a robust url parser and builder 2 | -- 3 | -- Bertrand Mansion, 2011-2013; License MIT 4 | -- @module neturl 5 | -- @alias M 6 | 7 | local M = {} 8 | M.version = "0.9.0" 9 | 10 | --- url options 11 | -- separator is set to `&` by default but could be anything like `&amp;` or `;` 12 | -- @todo Add an option to limit the size of the argument table 13 | M.options = { 14 | separator = '&' 15 | } 16 | 17 | --- list of known and common scheme ports 18 | -- as documented in IANA URI scheme list 19 | M.services = { 20 | acap = 674, 21 | cap = 1026, 22 | dict = 2628, 23 | ftp = 21, 24 | gopher = 70, 25 | http = 80, 26 | https = 443, 27 | iax = 4569, 28 | icap = 1344, 29 | imap = 143, 30 | ipp = 631, 31 | ldap = 389, 32 | mtqp = 1038, 33 | mupdate = 3905, 34 | news = 2009, 35 | nfs = 2049, 36 | nntp = 119, 37 | rtsp = 554, 38 | sip = 5060, 39 | snmp = 161, 40 | telnet = 23, 41 | tftp = 69, 42 | vemmi = 575, 43 | afs = 1483, 44 | jms = 5673, 45 | rsync = 873, 46 | prospero = 191, 47 | videotex = 516 48 | } 49 | 50 | local legal = { 51 | ["-"] = true, ["_"] = true, ["."] = true, ["!"] = true, 52 | ["~"] = true, ["*"] = true, ["'"] = true, ["("] = true, 53 | [")"] = true, [":"] = true, ["@"] = true, ["&"] = true, 54 | ["="] = true, ["+"] = true, ["$"] = true, [","] = true, 55 | [";"] = true -- can be used for parameters in path 56 | } 57 | 58 | local function decode(str) 59 | local str = str:gsub('+', ' ') 60 | return (str:gsub("%%(%x%x)", function(c) 61 | return string.char(tonumber(c, 16)) 62 | end)) 63 | end 64 | 65 | local function encode(str) 66 | return (str:gsub("([^A-Za-z0-9%_%.%-%~])", function(v) 67 | return string.upper(string.format("%%%02x", string.byte(v))) 68 | end)) 69 | end 70 | 71 | -- for query values, prefer + instead of %20 for spaces 72 | local function encodeValue(str) 73 | local str = encode(str) 74 | return str:gsub('%%20', '+') 75 | end 76 | 77 | local function encodeSegment(s) 78 | local legalEncode = function(c) 79 | if legal[c] then 80 | return c 81 | end 82 | return encode(c) 83 | end 84 | return s:gsub('([^a-zA-Z0-9])', legalEncode) 85 | end 86 | 87 | --- builds the url 88 | -- @return a string representing the built url 89 | function M:build() 90 | local url = '' 91 | if self.path then 92 | local path = self.path 93 | path:gsub("([^/]+)", function (s) return encodeSegment(s) end) 94 | url = url .. tostring(path) 95 | end 96 | if self.query then 97 | local qstring = tostring(self.query) 98 | if qstring ~= "" then 99 | url = url .. '?' .. qstring 100 | end 101 | end 102 | if self.host then 103 | local authority = self.host 104 | if self.port and self.scheme and M.services[self.scheme] ~= self.port then 105 | authority = authority .. ':' .. self.port 106 | end 107 | local userinfo 108 | if self.user and self.user ~= "" then 109 | userinfo = self.user 110 | if self.password then 111 | userinfo = userinfo .. ':' .. self.password 112 | end 113 | end 114 | if userinfo and userinfo ~= "" then 115 | authority = userinfo .. '@' .. authority 116 | end 117 | if authority then 118 | if url ~= "" then 119 | url = '//' .. authority .. '/' .. url:gsub('^/+', '') 120 | else 121 | url = '//' .. authority 122 | end 123 | end 124 | end 125 | if self.scheme then 126 | url = self.scheme .. ':' .. url 127 | end 128 | if self.fragment then 129 | url = url .. '#' .. self.fragment 130 | end 131 | return url 132 | end 133 | 134 | --- builds the querystring 135 | -- @param tab The key/value parameters 136 | -- @param sep The separator to use (optional) 137 | -- @param key The parent key if the value is multi-dimensional (optional) 138 | -- @return a string representing the built querystring 139 | function M.buildQuery(tab, sep, key) 140 | local query = {} 141 | if not sep then 142 | sep = M.options.separator or '&' 143 | end 144 | local keys = {} 145 | for k in pairs(tab) do 146 | keys[#keys+1] = k 147 | end 148 | table.sort(keys) 149 | for _,name in ipairs(keys) do 150 | local value = tab[name] 151 | name = encode(tostring(name)) 152 | if key then 153 | name = string.format('%s[%s]', tostring(key), tostring(name)) 154 | end 155 | if type(value) == 'table' then 156 | query[#query+1] = M.buildQuery(value, sep, name) 157 | else 158 | local value = encodeValue(tostring(value)) 159 | if value ~= "" then 160 | query[#query+1] = string.format('%s=%s', name, value) 161 | else 162 | query[#query+1] = name 163 | end 164 | end 165 | end 166 | return table.concat(query, sep) 167 | end 168 | 169 | --- Parses the querystring to a table 170 | -- This function can parse multidimensional pairs and is mostly compatible 171 | -- with PHP usage of brackets in key names like ?param[key]=value 172 | -- @param str The querystring to parse 173 | -- @param sep The separator between key/value pairs, defaults to `&` 174 | -- @todo limit the max number of parameters with M.options.max_parameters 175 | -- @return a table representing the query key/value pairs 176 | function M.parseQuery(str, sep) 177 | if not sep then 178 | sep = M.options.separator or '&' 179 | end 180 | 181 | local values = {} 182 | for key,val in str:gmatch(string.format('([^%q=]+)(=*[^%q=]*)', sep, sep)) do 183 | local key = decode(key) 184 | local keys = {} 185 | key = key:gsub('%[([^%]]*)%]', function(v) 186 | -- extract keys between balanced brackets 187 | if string.find(v, "^-?%d+$") then 188 | v = tonumber(v) 189 | else 190 | v = decode(v) 191 | end 192 | table.insert(keys, v) 193 | return "=" 194 | end) 195 | key = key:gsub('=+.*$', "") 196 | key = key:gsub('%s', "_") -- remove spaces in parameter name 197 | val = val:gsub('^=+', "") 198 | 199 | if not values[key] then 200 | values[key] = {} 201 | end 202 | if #keys > 0 and type(values[key]) ~= 'table' then 203 | values[key] = {} 204 | elseif #keys == 0 and type(values[key]) == 'table' then 205 | values[key] = decode(val) 206 | end 207 | 208 | local t = values[key] 209 | for i,k in ipairs(keys) do 210 | if type(t) ~= 'table' then 211 | t = {} 212 | end 213 | if k == "" then 214 | k = #t+1 215 | end 216 | if not t[k] then 217 | t[k] = {} 218 | end 219 | if i == #keys then 220 | t[k] = decode(val) 221 | end 222 | t = t[k] 223 | end 224 | end 225 | setmetatable(values, { __tostring = M.buildQuery }) 226 | return values 227 | end 228 | 229 | --- set the url query 230 | -- @param query Can be a string to parse or a table of key/value pairs 231 | -- @return a table representing the query key/value pairs 232 | function M:setQuery(query) 233 | local query = query 234 | if type(query) == 'table' then 235 | query = M.buildQuery(query) 236 | end 237 | self.query = M.parseQuery(query) 238 | return query 239 | end 240 | 241 | --- set the authority part of the url 242 | -- The authority is parsed to find the user, password, port and host if available. 243 | -- @param authority The string representing the authority 244 | -- @return a string with what remains after the authority was parsed 245 | function M:setAuthority(authority) 246 | self.authority = authority 247 | self.port = nil 248 | self.host = nil 249 | self.userinfo = nil 250 | self.user = nil 251 | self.password = nil 252 | 253 | authority = authority:gsub('^([^@]*)@', function(v) 254 | self.userinfo = v 255 | return '' 256 | end) 257 | authority = authority:gsub("^%[[^%]]+%]", function(v) 258 | -- ipv6 259 | self.host = v 260 | return '' 261 | end) 262 | authority = authority:gsub(':([^:]*)$', function(v) 263 | self.port = tonumber(v) 264 | return '' 265 | end) 266 | if authority ~= '' and not self.host then 267 | self.host = authority:lower() 268 | end 269 | if self.userinfo then 270 | local userinfo = self.userinfo 271 | userinfo = userinfo:gsub(':([^:]*)$', function(v) 272 | self.password = v 273 | return '' 274 | end) 275 | self.user = userinfo 276 | end 277 | return authority 278 | end 279 | 280 | --- Parse the url into the designated parts. 281 | -- Depending on the url, the following parts can be available: 282 | -- scheme, userinfo, user, password, authority, host, port, path, 283 | -- query, fragment 284 | -- @param url Url string 285 | -- @return a table with the different parts and a few other functions 286 | function M.parse(url) 287 | local comp = {} 288 | M.setAuthority(comp, "") 289 | M.setQuery(comp, "") 290 | 291 | local url = tostring(url or '') 292 | url = url:gsub('#(.*)$', function(v) 293 | comp.fragment = v 294 | return '' 295 | end) 296 | url =url:gsub('^([%w][%w%+%-%.]*)%:', function(v) 297 | comp.scheme = v:lower() 298 | return '' 299 | end) 300 | url = url:gsub('%?(.*)', function(v) 301 | M.setQuery(comp, v) 302 | return '' 303 | end) 304 | url = url:gsub('^//([^/]*)', function(v) 305 | M.setAuthority(comp, v) 306 | return '' 307 | end) 308 | comp.path = decode(url) 309 | 310 | setmetatable(comp, { 311 | __index = M, 312 | __tostring = M.build} 313 | ) 314 | return comp 315 | end 316 | 317 | --- removes dots and slashes in urls when possible 318 | -- This function will also remove multiple slashes 319 | -- @param path The string representing the path to clean 320 | -- @return a string of the path without unnecessary dots and segments 321 | function M.removeDotSegments(path) 322 | local fields = {} 323 | if string.len(path) == 0 then 324 | return "" 325 | end 326 | local startslash = false 327 | local endslash = false 328 | if string.sub(path, 1, 1) == "/" then 329 | startslash = true 330 | end 331 | if (string.len(path) > 1 or startslash == false) and string.sub(path, -1) == "/" then 332 | endslash = true 333 | end 334 | 335 | path:gsub('[^/]+', function(c) table.insert(fields, c) end) 336 | 337 | local new = {} 338 | local j = 0 339 | 340 | for i,c in ipairs(fields) do 341 | if c == '..' then 342 | if j > 0 then 343 | j = j - 1 344 | end 345 | elseif c ~= "." then 346 | j = j + 1 347 | new[j] = c 348 | end 349 | end 350 | local ret = "" 351 | if #new > 0 and j > 0 then 352 | ret = table.concat(new, '/', 1, j) 353 | else 354 | ret = "" 355 | end 356 | if startslash then 357 | ret = '/'..ret 358 | end 359 | if endslash then 360 | ret = ret..'/' 361 | end 362 | return ret 363 | end 364 | 365 | local function absolutePath(base_path, relative_path) 366 | if string.sub(relative_path, 1, 1) == "/" then 367 | return '/' .. string.gsub(relative_path, '^[%./]+', '') 368 | end 369 | local path = base_path 370 | if relative_path ~= "" then 371 | path = '/'..path:gsub("[^/]*$", "") 372 | end 373 | path = path .. relative_path 374 | path = path:gsub("([^/]*%./)", function (s) 375 | if s ~= "./" then return s else return "" end 376 | end) 377 | path = string.gsub(path, "/%.$", "/") 378 | local reduced 379 | while reduced ~= path do 380 | reduced = path 381 | path = string.gsub(reduced, "([^/]*/%.%./)", function (s) 382 | if s ~= "../../" then return "" else return s end 383 | end) 384 | end 385 | path = string.gsub(path, "([^/]*/%.%.?)$", function (s) 386 | if s ~= "../.." then return "" else return s end 387 | end) 388 | local reduced 389 | while reduced ~= path do 390 | reduced = path 391 | path = string.gsub(reduced, '^/?%.%./', '') 392 | end 393 | return '/' .. path 394 | end 395 | 396 | --- builds a new url by using the one given as parameter and resolving paths 397 | -- @param other A string or a table representing a url 398 | -- @return a new url table 399 | function M:resolve(other) 400 | if type(self) == "string" then 401 | self = M.parse(self) 402 | end 403 | if type(other) == "string" then 404 | other = M.parse(other) 405 | end 406 | if other.scheme then 407 | return other 408 | else 409 | other.scheme = self.scheme 410 | if not other.authority or other.authority == "" then 411 | other:setAuthority(self.authority) 412 | if not other.path or other.path == "" then 413 | other.path = self.path 414 | local query = other.query 415 | if not query or not next(query) then 416 | other.query = self.query 417 | end 418 | else 419 | other.path = absolutePath(self.path, other.path) 420 | end 421 | end 422 | return other 423 | end 424 | end 425 | 426 | --- normalize a url path following some common normalization rules 427 | -- described on The URL normalization page of Wikipedia 428 | -- @return the normalized path 429 | function M:normalize() 430 | if type(self) == 'string' then 431 | self = M.parse(self) 432 | end 433 | if self.path then 434 | local path = self.path 435 | path = absolutePath(path, "") 436 | -- normalize multiple slashes 437 | path = string.gsub(path, "//+", "/") 438 | self.path = path 439 | end 440 | return self 441 | end 442 | 443 | return M -------------------------------------------------------------------------------- /lib/util.lua: -------------------------------------------------------------------------------- 1 | local etlua = require "etlua" 2 | local http = require "resty.http" 3 | local _url = require "url" 4 | 5 | 6 | function get_dirname() 7 | local path = debug.getinfo(1).source:match("@(.*)$") 8 | return path:match("(.*/)") 9 | end 10 | 11 | local _dir = get_dirname() 12 | 13 | function load_template(filename) 14 | filename = _dir .. '/views/' .. filename .. ".etlua" 15 | local f = assert(io.open(filename, "r")) 16 | local t = etlua.compile(f:read("*all")) 17 | f:close() 18 | return t 19 | end 20 | 21 | 22 | function render_template(filename) 23 | return load_template(filename)() 24 | end 25 | 26 | 27 | function capture_errors(func) 28 | return function(self) 29 | local status, result = pcall(func, self) 30 | if not status then 31 | ngx.say(result) 32 | ngx.say(debug.traceback()) 33 | return ngx.exit(512) 34 | else 35 | return result 36 | end 37 | end 38 | end 39 | 40 | 41 | function bad_request(message) 42 | ngx.status = 400 43 | ngx.say(message) 44 | ngx.log(ngx.ERR, "The request did not match the spec'ed rule.") 45 | ngx.log(ngx.ERR, message) 46 | return ngx.exit(ngx.HTTP_OK) 47 | end 48 | 49 | 50 | function fetch_http_body(url) 51 | url = _url.parse(url) 52 | local host = url.host 53 | local path = url.path 54 | local port = url.port 55 | local hc = http:new() 56 | 57 | hc:set_timeout(1000) 58 | ok, err = hc:connect(host, port) 59 | if not ok then 60 | ngx.say("failed to connect: ", err) 61 | return '' 62 | end 63 | 64 | local res, err = hc:request({ path = path }) 65 | if not res then 66 | ngx.say("failed to retrieve: ", err) 67 | return '' 68 | end 69 | 70 | local body = res:read_body() 71 | local ok, err = hc:close() 72 | if not ok then 73 | ngx.say("failed to close: ", err) 74 | end 75 | 76 | return body 77 | end 78 | 79 | 80 | function Keys(list) 81 | local keys = {} 82 | for k, _ in pairs(list) do 83 | keys[k] = true 84 | end 85 | return keys 86 | end 87 | 88 | 89 | function size2int(size) 90 | if not size then 91 | return nil 92 | end 93 | assert(type(size) == "string", "size2int expects a string") 94 | size = size:lower() 95 | unit = size:sub(-1) 96 | if unit == 'k' then 97 | size = tonumber(size:sub(1, -2)) * 1024 98 | elseif unit == 'm' then 99 | size = tonumber(size:sub(1, -2)) * 1024 * 1024 100 | elseif unit == 'g' then 101 | size = tonumber(size:sub(1, -2)) * 1024 * 1024 * 1025 102 | else 103 | size = tonumber(size) 104 | end 105 | return size 106 | end 107 | 108 | 109 | function implode(delimiter, list) 110 | local len = #list 111 | if len == 0 then 112 | return "" 113 | end 114 | local string = list[1] 115 | for i = 2, len do 116 | string = string .. delimiter .. list[i] 117 | end 118 | return string 119 | end 120 | 121 | 122 | function explode(delimiter, text) 123 | local list = {} 124 | local pos = 1 125 | 126 | if string.find("", delimiter, 1) then 127 | error("delimiter matches empty string!") 128 | end 129 | while 1 do 130 | local first, last = string.find(text, delimiter, pos) 131 | print (first, last) 132 | if first then 133 | table.insert(list, string.sub(text, pos, first-1)) 134 | pos = last+1 135 | else 136 | table.insert(list, string.sub(text, pos)) 137 | break 138 | end 139 | end 140 | return list 141 | end 142 | 143 | 144 | -- public interface 145 | return { 146 | render_template = render_template, 147 | load_template = load_template, 148 | capture_errors = capture_errors, 149 | bad_request = bad_request, 150 | fetch_http_body = fetch_http_body, 151 | Keys = Keys, 152 | size2int = size2int, 153 | implode = implode, 154 | explode = explode 155 | } 156 | -------------------------------------------------------------------------------- /lib/videur.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- main script 3 | -- 4 | local util = require "util" 5 | local spec_reader = require "spec_reader" 6 | local body = require "body_reader" 7 | local rate_limit = require "rate_limit" 8 | 9 | -- read the options from the Nginx config file 10 | local cached_spec = ngx.shared.cached_spec 11 | local spec_url = ngx.var.spec_url or "http://127.0.0.1:8282/api-specs" 12 | local limits, params 13 | 14 | -- 1. abort if we don't have any user agent 15 | local key = ngx.var.http_user_agent 16 | if not key then 17 | return util.bad_request("no user-agent found") 18 | end 19 | 20 | -- 2. match the incoming request w/ the spec file and set up $target 21 | ngx.var.target, limits, params = spec_reader.match(spec_url, cached_spec) 22 | local max_size = util.size2int(limits.max_body_size) or util.size2int(ngx.var.max_body_size) 23 | 24 | -- 3. check the rating limit 25 | if limits.rates then 26 | rate_limit.check_rate(ngx.var.target, limits.rates) 27 | end 28 | 29 | -- 4. control the body size 30 | body.check_size(max_size) 31 | -------------------------------------------------------------------------------- /rational.md: -------------------------------------------------------------------------------- 1 | Rational 2 | -------- 3 | 4 | Nginx is our standard web server of choice for all our services at Mozilla. It's 5 | been used for years now and is able to leverage any web service no matter what 6 | framework is used, by reverse proxying incoming requests to a Node.js, Go or 7 | Python powered service. 8 | 9 | The classical server stack we deploy on Amazon is : 10 | 11 | ELB -> Nginx -> {NodeJS, Python, Go, ...} -> Backend 12 | 13 | 14 | Where **backend** can be a database system, or any 3rd party service the application 15 | needs to interact with. 16 | 17 | To smoothly operate a cluster of such nodes, we are currently missing a few key 18 | features like: 19 | 20 | - the ability to write efficient Web Application Firewalls 21 | - the ability to interact live with the NGinx server 22 | - the ability to provide highly customized filtering features, like 23 | interactive rate-limiting. 24 | 25 | Thanks to http://wiki.nginx.org/HttpLuaModule and OpenResty, 26 | Nginx's behavior can be completely customized to implement those features. 27 | 28 | 29 | Videur has two goals: 30 | 31 | - provide a set of Lua scripts in a library that can be added in our standard 32 | deployements 33 | - offer a development environment for developers to write WAFs 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /spec/VAS.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | Videur API Specification 3 | ======================== 4 | 5 | :version: 0.1 6 | :author: Tarek Ziadé 7 | :author: Julien Vehent 8 | 9 | The **Videur API Specification** file is a JSON document a web application 10 | can provide to describe its HTTP endpoints. 11 | 12 | The standard location to publish this document is /api-specs but it 13 | can be located elsewhere if needed. 14 | 15 | The JSON document is a mapping containing a single **service** key. 16 | 17 | The service key is in turn a mapping containing the following keys: 18 | 19 | - **location** -- the root url for the service 20 | - **version** -- the service version 21 | - **resources** -- a list of resource for the service (see below) 22 | - **configuration** -- a list of configuration options for the service (see below) 23 | - **description** -- a description of the service (see below) 24 | 25 | Examples for the **location** and **version** fields:: 26 | 27 | { 28 | "service": { 29 | "location": "http://127.0.0.1:8282", 30 | "version": "1.1", 31 | ... 32 | } 33 | } 34 | 35 | 36 | resources 37 | --------- 38 | 39 | This key contains a mapping describing all HTTP endpoints. Each resource is 40 | identified the exact URI of the endpoint or a regular expression. 41 | 42 | Examples of valid URIs: 43 | 44 | - **/dashboard** 45 | - **/action/one** 46 | - **regexp:/welp/[a-zA-Z0-9]{1,64}** 47 | 48 | Regular expression based URIs are prefixed by **regexp:** 49 | 50 | The value of each resource is a mapping of all implemented methods. 51 | 52 | Example:: 53 | 54 | "/action": { 55 | "GET": {}, 56 | "DELETE": {} 57 | } 58 | 59 | 60 | This specification does not enforce any rule about the lack of the body 61 | entity on methods like GET, as this part of the HTTP specification 62 | is a bit vague. Some software like ElasticCache for instance will define 63 | GET APIs with body content. 64 | 65 | When specifying methods on a resources, there are a list of rules 66 | that can be added in the method definition: 67 | 68 | - **parameters**: rules on the query string 69 | - **body**: rules on the body 70 | - **limits**: limits on the request (rate, size, etc.) 71 | 72 | 73 | parameters 74 | ========== 75 | 76 | A validation rule can be defined for each query string parameter, in the 77 | **paramaters** key for the resource. 78 | 79 | The rule is identified by the option name and contains two fields: 80 | 81 | - **validation**: the validation rule. 82 | - **required**: a boolean to indicate if this option is mandatory when using the 83 | resource 84 | 85 | The validation rule is a pattern the value of the option must match. It can 86 | take the following values: 87 | 88 | - **digits:,** : the value is composed of numbers. Its size is 89 | between and digits 90 | - **regexp:**: the value must follow the corresponding regexp 91 | - **values:||**: the value must be one of a, b, c. 92 | - **datetime**: the value is an ISO date 93 | 94 | Examples:: 95 | 96 | "/search": { 97 | "GET": { 98 | "parameters": { 99 | "before": { 100 | "validation":"datetime", 101 | "required": false 102 | }, 103 | "after": { 104 | "validation":"datetime", 105 | "required": false 106 | }, 107 | "type": { 108 | "validation":"values:action|command|agent", 109 | "required": false 110 | }, 111 | "report": { 112 | "validation":"regexp:[a-zA-Z0-9]{1,64}", 113 | "required": false 114 | }, 115 | "agentname": { 116 | "validation":"regexp:[\\w\\n\\r\\t ]{0,256}", 117 | "required": false 118 | }, 119 | "actionname": { 120 | "validation":"regexp:[\\w\\n\\r\\t ]{0,1024}", 121 | "required": false 122 | }, 123 | "status": { 124 | "validation":"regexp:[a-zA-Z0-9]{1,64}", 125 | "required": false 126 | }, 127 | "threatfamily": { 128 | "validation":"regexp:[a-zA-Z0-9]{1,64}", 129 | "required": false 130 | }, 131 | "limit": { 132 | "validation":"digits:1,20", 133 | "required": false 134 | } 135 | } 136 | } 137 | } 138 | 139 | 140 | 141 | body 142 | ==== 143 | 144 | Not yet defined. 145 | 146 | 147 | limits 148 | ====== 149 | 150 | limits have 2 rules: 151 | 152 | - **rates**: a list of rate rules 153 | - **max_body_size**: a maximum body size expressed in kilo. example: "10k" 154 | 155 | Each rates is defined with three fields: 156 | 157 | - **seconds**: the throttling window in seconds. 158 | - **hits**: the maximum number of hits allowed in that window. 159 | - **match**: an expression to uniquely identify a user 160 | 161 | The **match** field is a logical expression articulated with **AND** and **OR** 162 | operators. 163 | 164 | Each value can be of the form: 165 | 166 | - **header:**: takes the value of the header 167 | - **var:**: takes the value of a variable. Currently defined 168 | variables are: 169 | - **remote_address**: client IP 170 | - **binary_remote_address**: client IP in binary form 171 | 172 | 173 | Examples:: 174 | 175 | "limits": { 176 | "rates": [ 177 | { 178 | "seconds": 60, 179 | "hits": 10, 180 | "match": "header:Authorization AND header:User-Agent" 181 | }, 182 | { 183 | "seconds": 10, 184 | "hits": 100, 185 | "match": "header:X-Forwarded-For OR var:remote_addr" 186 | } 187 | ], 188 | "max_body_size": "10k" 189 | } 190 | 191 | 192 | configuration 193 | ------------- 194 | 195 | Not yet defined. 196 | 197 | description 198 | ----------- 199 | 200 | description contains informative fields. Any information can be added in this 201 | section. 202 | 203 | Suggested values: 204 | 205 | - **owner**: name of the owner of the service 206 | - **developer**: name of the main developer. 207 | - **operator**: name of the main operator 208 | 209 | Example:: 210 | 211 | "description": { 212 | "owner": "Mozilla Operations Security", 213 | "developer": "Julien Vehent ", 214 | "operator": "Julien Vehent " 215 | } 216 | 217 | 218 | 219 | -------------------------------------------------------------------------------- /spec/mig_example.json: -------------------------------------------------------------------------------- 1 | { 2 | "service": { 3 | "location": "http://127.0.0.1:8282", 4 | "version": "1.1", 5 | "resources": { 6 | "/dashboard": { 7 | "GET": { 8 | "limits": { 9 | "rates": [ 10 | { 11 | "seconds": 1, 12 | "hits": 2, 13 | "match": "header:Authorization AND header:User-Agent AND var:remote_addr" 14 | }] 15 | } 16 | } 17 | }, 18 | 19 | "regexp:/welp/[a-zA-Z0-9]{1,64}": { 20 | "GET": {} 21 | }, 22 | 23 | "/action": { 24 | "GET": { 25 | "parameters": { 26 | "actionid": { 27 | "validation": "digits:1,20", 28 | "required": true 29 | } 30 | } 31 | } 32 | }, 33 | "/command": { 34 | "GET": { 35 | "parameters": { 36 | "commandid": { 37 | "validation": "regexp:[0-9]{1,20}", 38 | "required": true 39 | } 40 | } 41 | } 42 | }, 43 | "/search": { 44 | "GET": { 45 | "parameters": { 46 | "before": { 47 | "validation":"datetime", 48 | "required": false 49 | }, 50 | "after": { 51 | "validation":"datetime", 52 | "required": false 53 | }, 54 | "type": { 55 | "validation":"values:action|command|agent", 56 | "required": false 57 | }, 58 | "report": { 59 | "validation":"regexp:[a-zA-Z0-9]{1,64}", 60 | "required": false 61 | }, 62 | "agentname": { 63 | "validation":"regexp:[\\w\\n\\r\\t ]{0,256}", 64 | "required": false 65 | }, 66 | "actionname": { 67 | "validation":"regexp:[\\w\\n\\r\\t ]{0,1024}", 68 | "required": false 69 | }, 70 | "status": { 71 | "validation":"regexp:[a-zA-Z0-9]{1,64}", 72 | "required": false 73 | }, 74 | "threatfamily": { 75 | "validation":"regexp:[a-zA-Z0-9]{1,64}", 76 | "required": false 77 | }, 78 | "limit": { 79 | "validation":"digits:1,20", 80 | "required": false 81 | } 82 | } 83 | } 84 | }, 85 | "/action/create": { 86 | "POST": { 87 | "body": { 88 | "format": "json", 89 | "must_validate": false, 90 | "json_validation": { 91 | "id": "[[float64]]", 92 | "name": "[[text]]", 93 | "target": "[[text]]", 94 | "description": { 95 | "author": "[[text]]", 96 | "email": "[[email]]", 97 | "url": "[[text]]", 98 | "revision": "[float64]" 99 | }, 100 | "threat": { 101 | "level": "[[text]]", 102 | "family": "[[text]]", 103 | "type": "[[text]]" 104 | }, 105 | "validfrom": "[[datetime]]", 106 | "expireafter": "[[datetime]]", 107 | "operations": [ 108 | { 109 | "module": "[[text]]", 110 | "parameters": "[[json]]" 111 | } 112 | ], 113 | "pgpsignatures": [ 114 | "[[text]]" 115 | ], 116 | "starttime": "[[datetime]]", 117 | "finishtime": "[[datetime]]", 118 | "lastupdatetime": "[[datetime]]", 119 | "counters": "[[json]]", 120 | "syntaxversion": "[[float64]]" 121 | } 122 | }, 123 | "limits": { 124 | "rates": [ 125 | { 126 | "seconds": 60, 127 | "hits": 10, 128 | "match": "header:Authorization AND header:User-Agent" 129 | }, 130 | { 131 | "seconds": 10, 132 | "hits": 100, 133 | "match": "header:X-Forwarded-For OR var:remote_addr" 134 | } 135 | ], 136 | "max_body_size": "10k" 137 | } 138 | } 139 | } 140 | }, 141 | "description": { 142 | "owner": "Mozilla Operations Security", 143 | "developer": "Julien Vehent ", 144 | "operator": "Julien Vehent " 145 | }, 146 | "configuration": { 147 | "authentication": "fxa assertion", 148 | "tls": { 149 | "require_backward_compatibility": false, 150 | "ocsp_stapling": { 151 | "enabled": true, 152 | "must-staple": 7776000 153 | }, 154 | "hsts": { 155 | "enabled": true, 156 | "max-age": 15768000 157 | } 158 | }, 159 | "content security policy": { 160 | "enabled": true, 161 | "Content-Security-Policy": "default-src 'none'; style-src cdn.example.com; report-uri /_/csp-reports" 162 | }, 163 | "limits": { 164 | "rates": [ 165 | { 166 | "seconds": 10, 167 | "hits": 200, 168 | "match": "header:X-Forwarded-For OR var:remote_addr" 169 | } 170 | ], 171 | "max_req_size": "500B", 172 | "permitted_origins": [ 173 | "france", 174 | "usa", 175 | "uk" 176 | ], 177 | "ip whitelist": [], 178 | "ip blacklist": [] 179 | }, 180 | "logging": { 181 | "location": "amqps://mozdef.mozilla.org/mig/" 182 | } 183 | } 184 | } 185 | } 186 | -------------------------------------------------------------------------------- /tests/support.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import unittest 4 | import time 5 | import tempfile 6 | import shutil 7 | import subprocess 8 | 9 | from webtest import TestApp 10 | from nginxtest.server import NginxServer 11 | import requests 12 | 13 | 14 | PY3 = sys.version_info.major == 3 15 | 16 | LIBDIR = os.path.normpath(os.path.join(os.path.dirname(__file__), 17 | '..', 'lib')) 18 | 19 | HTTP_OPTIONS = """\ 20 | lua_package_path "%s/?.lua;;"; 21 | lua_shared_dict cached_spec 512k; 22 | lua_shared_dict stats 100k; 23 | """ % LIBDIR 24 | 25 | 26 | SERVER_OPTIONS = """\ 27 | set $spec_url "http://127.0.0.1:8282/api-specs"; 28 | set $target ""; 29 | set $max_body_size 1M; 30 | access_by_lua_file '%s/videur.lua'; 31 | """ % LIBDIR 32 | 33 | 34 | LOCATION = """ 35 | proxy_pass $target; 36 | """ 37 | 38 | SPEC_FILE = os.path.join(os.path.dirname(__file__), 39 | '..', 'spec', 'mig_example.json') 40 | 41 | 42 | class TestMyNginx(unittest.TestCase): 43 | 44 | def start_server(self, locations=None, pages=None): 45 | if locations is None: 46 | locations = [] 47 | 48 | if pages is None: 49 | pages = ('dashboard', 'action', 'search', 'welp/1234') 50 | 51 | locations.append({'path': '/', 52 | 'definition': LOCATION}) 53 | 54 | self.serv_dir = tempfile.mkdtemp() 55 | target = os.path.join(self.serv_dir, 'api-specs') 56 | shutil.copy(SPEC_FILE, target) 57 | 58 | # and lets add some pages 59 | for page in pages: 60 | path = os.path.join(self.serv_dir, page) 61 | 62 | if not os.path.isdir(os.path.dirname(path)): 63 | os.makedirs(os.path.dirname(path)) 64 | 65 | with open(path, 'w') as f: 66 | f.write('yeah') 67 | 68 | if PY3: 69 | server = 'http.server' 70 | else: 71 | server = 'SimpleHTTPServer' 72 | 73 | self._p = subprocess.Popen([sys.executable, '-m', 74 | server, '8282'], 75 | cwd=self.serv_dir, 76 | stderr=subprocess.PIPE, 77 | stdout=subprocess.PIPE) 78 | start = time.time() 79 | res = None 80 | while time.time() - start < 2: 81 | try: 82 | res = requests.get('http://127.0.0.1:8282/api-specs') 83 | break 84 | except requests.ConnectionError: 85 | time.sleep(.1) 86 | if res is None: 87 | self._kill_python_server() 88 | 89 | raise IOError("Could not start the Py server") 90 | 91 | try: 92 | self.nginx = NginxServer(locations=locations, 93 | http_options=HTTP_OPTIONS, 94 | server_options=SERVER_OPTIONS) 95 | self.nginx.start() 96 | except Exception: 97 | self._kill_python_server() 98 | raise 99 | 100 | def _kill_python_server(self): 101 | try: 102 | self._p.terminate() 103 | os.kill(self._p.pid, 9) 104 | finally: 105 | shutil.rmtree(self.serv_dir) 106 | 107 | def stop_server(self): 108 | self._kill_python_server() 109 | self.nginx.stop() 110 | -------------------------------------------------------------------------------- /tests/test_rate_limit.py: -------------------------------------------------------------------------------- 1 | import os 2 | import unittest 3 | import time 4 | 5 | from webtest import TestApp 6 | from support import TestMyNginx 7 | 8 | 9 | class TestRateLimiting(TestMyNginx): 10 | 11 | def setUp(self): 12 | super(TestRateLimiting, self).setUp() 13 | self.start_server() 14 | # see https://github.com/openresty/lua-nginx-module/issues/379 15 | self.app = TestApp(self.nginx.root_url, lint=False) 16 | self.headers = {'User-Agent': 'Me', 'Authorization': 'some'} 17 | 18 | def tearDown(self): 19 | self.stop_server() 20 | super(TestRateLimiting, self).tearDown() 21 | 22 | def test_rate(self): 23 | # the 3rd call should be returning a 429 24 | self.app.get('/dashboard', status=200, headers=self.headers) 25 | self.app.get('/dashboard', status=200, headers=self.headers) 26 | self.app.get('/dashboard', status=429, headers=self.headers) 27 | 28 | def test_rate2(self): 29 | # the 3rd call should be returning a 200 30 | # because the blacklist is ttled 31 | self.app.get('/dashboard', status=200, headers=self.headers) 32 | self.app.get('/dashboard', status=200, headers=self.headers) 33 | time.sleep(1.1) 34 | self.app.get('/dashboard', status=200, headers=self.headers) 35 | -------------------------------------------------------------------------------- /tests/test_spec_reader.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import sys 3 | import os 4 | import unittest 5 | import time 6 | import tempfile 7 | import shutil 8 | 9 | from webtest import TestApp 10 | from nginxtest.server import NginxServer 11 | import requests 12 | from support import TestMyNginx 13 | 14 | 15 | class TestSpecReader(TestMyNginx): 16 | 17 | def setUp(self): 18 | super(TestSpecReader, self).setUp() 19 | self.start_server() 20 | self.app = TestApp(self.nginx.root_url) 21 | 22 | def tearDown(self): 23 | self.stop_server() 24 | super(TestSpecReader, self).tearDown() 25 | 26 | def test_routing(self): 27 | res = self.app.get('/dashboard', headers={'User-Agent': 'Me'}, 28 | status=200) 29 | self.assertEqual(res.body, 'yeah') 30 | 31 | def test_405(self): 32 | res = self.app.delete('/dashboard', headers={'User-Agent': 'Me'}, 33 | status=405) 34 | # XXX check the allow header 35 | 36 | def test_reject_unknown_arg(self): 37 | self.app.get('/dashboard?ok=no', headers={'User-Agent': 'Me'}, 38 | status=400) 39 | 40 | def test_reject_missing_option(self): 41 | r = self.app.get('/action', 42 | headers={'User-Agent': 'Me'}, 43 | status=400) 44 | self.assertEqual(r.body, 'Missing actionid\n') 45 | 46 | def test_reject_bad_arg(self): 47 | # make sure we just accept integers 48 | self.app.get('/action', 49 | params={'actionid': 'no'}, 50 | headers={'User-Agent': 'Me'}, 51 | status=400) 52 | 53 | r = self.app.get('/action', 54 | params={'actionid': '1234'}, 55 | headers={'User-Agent': 'Me'}, 56 | status=200) 57 | self.assertEqual(r.body, 'yeah') 58 | 59 | def test_values(self): 60 | # make sure we just accept some values 61 | self.app.get('/search', 62 | params={'type': 'meh'}, 63 | headers={'User-Agent': 'Me'}, 64 | status=400) 65 | 66 | for value in ('action', 'command', 'agent'): 67 | r = self.app.get('/search', 68 | params={'type': value}, 69 | headers={'User-Agent': 'Me'}, 70 | status=200) 71 | self.assertEqual(r.body, 'yeah') 72 | 73 | def test_date(self): 74 | self.app.get('/search', 75 | params={'before': 'bad date'}, 76 | headers={'User-Agent': 'Me'}, 77 | status=400) 78 | 79 | self.app.get('/search', 80 | params={'before': '2014-06-26T04:25:24Z'}, 81 | headers={'User-Agent': 'Me'}, 82 | status=200) 83 | 84 | def test_post_limit(self): 85 | self.app.post('/action/create', params='data', 86 | headers={'User-Agent': 'Me'}, 87 | status=501) 88 | 89 | # should not work because > 10k 90 | data = 'data' * 10 * 1024 91 | self.app.post('/action/create', params=data, 92 | headers={'User-Agent': 'Me'}, 93 | status=413) 94 | 95 | def test_regexp_routes(self): 96 | # make sure we match regexps for URLs 97 | self.app.get('/welp/1234', 98 | headers={'User-Agent': 'Me'}, 99 | status=200) 100 | 101 | self.app.get('/welp/12__', 102 | headers={'User-Agent': 'Me'}, 103 | status=404) 104 | --------------------------------------------------------------------------------