├── LICENSE
├── README.md
├── conf
├── lua
│ ├── n3r
│ │ └── urlshortener_eval.lua
│ └── resty
│ │ └── redis.lua
└── nginx.conf
└── html
└── shorten-ui.html
/LICENSE:
--------------------------------------------------------------------------------
1 | The MIT License (MIT)
2 |
3 | Copyright (c) 2013 sanshi0518
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy of
6 | this software and associated documentation files (the "Software"), to deal in
7 | the Software without restriction, including without limitation the rights to
8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
9 | the Software, and to permit persons to whom the Software is furnished to do so,
10 | subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
21 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | shorturl-nginx
2 | ----------
3 |
4 | A URL Shortener with analytics based on Nginx and Redis
5 |
6 |
7 | Setup
8 | ---------
9 |
10 | - Start a redis instance
11 | - Build your nginx with **lua-nginx-module** and **set-misc-nginx-module**
12 | - Copy the lua script conf/lua/n3r/**urlshortener_eval.lua** to your own nginx **conf/lua/n3r/** directory, and update the redis configurations in the **connect** function
13 | - Copy the html/**shorten-ui.html** to your own nginx **html** directory
14 | - Download [lua-resty-redis](https://github.com/agentzh/lua-resty-redis) and copy the lib/resty/**redis.lua** to your own nginx **conf/lua/resty/** directory
15 | - Add the follow configurations to your nginx.conf, then start your nginx
16 |
17 | > **http block:** lua_package_path
18 | >
19 | > **server block:** location a)shorten-ui.html b)shorten c)^/0[0-9a-zA-Z]{1,12}$
20 | >
21 | > You could find these configurations in my **conf/nginx.conf**
22 |
23 | Basic Usage
24 | ---------
25 |
26 | #### Create a short url
27 | http://localhost:8088/shorten?url=http://www.google.com
28 | > **NOTE:** Do not forget the **http://** prefix within the **url** parameter
29 |
30 | #### Visit the short url
31 | Just visit the url that the previous step returned
32 |
33 | #### Web UI
34 | http://localhost:8088/shorten-ui.html
35 | > **NOTE:** In this page, you could input a URL, then get the shorten form
36 |
37 | #### Create a random short url
38 | http://localhost:8088/shorten?url=http://www.google.com&random=true
39 |
40 | #### Query source url by the short
41 | http://localhost:8088/shorten-qrybyshort?url=http://t.cn/0asd
42 |
--------------------------------------------------------------------------------
/conf/lua/n3r/urlshortener_eval.lua:
--------------------------------------------------------------------------------
1 | -- TwemProxy支持EVAL命令，但只是将script作为key进行哈希映射到后端redis server
2 |
3 | -- "database scheme"
4 | -- database 0: id ~> url
5 | -- database 1: md5 ~> id (lookup if we have shorten the url already)
6 | -- database 2: key "shorten.count" storing the number of shortened urls;
7 | -- the id is generated by (this number + 1) converted to base 62
8 | -- database 3: id ~> hits
9 | -- database 4: id ~> [{referer|user_agent}]
10 | -- database 5: id ~> misses
11 | -- database 6: id ~> [{referer|user_agent}] (when id is not found)
12 |
13 | local _M = {
14 | _VERSION = '0.2'
15 | }
16 |
17 | local connect = function()
18 | local redis = require "resty.redis"
19 | local red = redis:new()
20 | red:set_timeout(1000) -- 1 sec
21 | local ok, err = red:connect("127.0.0.1", 6379)
22 | if not ok then
23 | ngx.say(err)
24 | return ngx.exit(ngx.ERROR)
25 | end
26 |
27 | return red
28 | end
29 |
30 | local pack_script = "local short_pre = ARGV[1] "
31 | .. "local arg_url = ARGV[2] "
32 | .. "local md5_url = ARGV[3] "
33 | .. "local id = ARGV[4] "
34 | .. "local checkAlready = ARGV[5] "
35 | .. "\n "
36 | .. "local basen = function(n) "
37 | .. " local digits = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' "
38 | .. " local t = {} "
39 | .. " repeat "
40 | .. " local d = (n % 62) + 1 "
41 | .. " n = math.floor(n / 62) "
42 | .. " table.insert(t, 1, digits:sub(d, d)) "
43 | .. " until n == 0 "
44 | .. " return table.concat(t, '') "
45 | .. "end "
46 | .. "\n "
47 | .. "if checkAlready or true then "
48 | .. " redis.call('select', 1) "
49 | .. " local already, err = redis.call('get', md5_url) "
50 | .. " if already then "
51 | .. " return short_pre .. 0 .. already "
52 | .. " end "
53 | .. "end "
54 | .. "\n "
55 | .. "if id then "
56 | .. " redis.call('select', 0) "
57 | .. " local url = redis.call('get', id) "
58 | .. " if url then return 'ERR:RAU' end "
59 | .. "else "
60 | .. " redis.call('select', 2) "
61 | .. " local count, err = redis.call('incr', 'shorten.count') "
62 | .. " id = basen(count) "
63 | .. "end "
64 | .. "\n "
65 | .. "if checkAlready or true then "
66 | .. " redis.call('select', 1) "
67 | .. " redis.call('set', md5_url, id) "
68 | .. "end "
69 | .. "\n "
70 | .. "redis.call('select', 0) "
71 | .. "redis.call('set', id, arg_url) "
72 | .. "\n "
73 | .. "return short_pre .. 0 .. id "
74 |
75 | function exec_pack_script(red, prefix, url, hash, random_alphanum, checkAlready)
76 | return red:eval(pack_script, 5, "short_pre", "arg_url", "md5_url", "random_alphanum", "checkAlready",
77 | prefix, url, hash, random_alphanum, checkAlready)
78 | end
79 |
80 | function _M.pack(arg_url, prefix, random, checkAlready)
81 |
82 | -- url
83 | local unescape_url = ndk.set_var.set_unescape_uri(arg_url)
84 |
85 | local url = unescape_url:gsub("^%s*(.-)%s*$", "%1")
86 |
87 | if url == "" then
88 | return ngx.exit(400)
89 | end
90 |
91 | -- prefix
92 | if prefix == nil then
93 | if ngx.var.server_port == "80" then
94 | prefix = "http://" .. ngx.var.server_name .. "/"
95 | else
96 | prefix = "http://" .. ngx.var.server_name .. ":" .. ngx.var.server_port .. "/"
97 | end
98 | else
99 | prefix = "http://" .. prefix .. "/"
100 | end
101 |
102 | local hash = ngx.md5(url)
103 |
104 | local red = connect()
105 |
106 | if random then
107 |
108 | local random_alphanum = ndk.set_var.set_secure_random_alphanum(12)
109 |
110 | local res, err = exec_pack_script(red, prefix, url, hash, random_alphanum, checkAlready)
111 |
112 | -- comment "--" was illegal in redis lua script, WTF!!
113 | -- ERR: RANDOM ALREADY USED
114 | if res and res ~= 'ERR:RAU' then
115 | ngx.say(res)
116 | return ngx.exit(ngx.OK)
117 | end
118 | end
119 |
120 | local res, err = exec_pack_script(red, prefix, url, hash, nil, checkAlready);
121 |
122 | if not res then
123 | ngx.say(err)
124 | return ngx.exit(ngx.ERROR)
125 | end
126 |
127 | ngx.say(res)
128 | return ngx.exit(ngx.OK)
129 | end
130 |
131 | local unpack_script = "local id = ARGV[1] "
132 | .. "local referer_agent = ARGV[2] "
133 | .. "local recordHits = ARGV[3] "
134 | .. "local recordReferAndUserAgent = ARGV[4] "
135 | .. "\n "
136 | .. "redis.call('select', 0) "
137 | .. "local url, err = redis.call('get', id) "
138 | .. "if not url then "
139 | .. " if recordHits then "
140 | .. " redis.call('select', 5) "
141 | .. " redis.call('incr', id) "
142 | .. " end "
143 | .. " if recordReferAndUserAgent then "
144 | .. " redis.call('select', 6) "
145 | .. " redis.call('rpush', id, referer_agent) "
146 | .. " end "
147 | .. " -- distinguish the error from misses \n "
148 | .. " return nil "
149 | .. "end "
150 | .. "\n "
151 | .. "if recordHits then "
152 | .. " redis.call('select', 3) "
153 | .. " redis.call('incr', id) "
154 | .. "end "
155 | .. "if recordReferAndUserAgent then "
156 | .. " redis.call('select', 4) "
157 | .. " redis.call('rpush', id, referer_agent) "
158 | .. "end "
159 | .. "return url "
160 |
161 | function _M.unpack(notFoundRedirect, recordHits, recordReferAndUserAgent)
162 | -- remove the prefix "/" and "0"
163 | local id = string.sub(ngx.var.request_uri, 3)
164 |
165 | -- (referer .. "|" .. user_agent) in the unpack script may result in inexplicable errors, WTF!
166 | local referer = ngx.var.http_referer or ""
167 | local user_agent = ngx.var.http_user_agent or ""
168 |
169 | local red = connect()
170 | local res, err = red:eval(unpack_script, 4, "id", "referer_agent", "recordHits", "recordReferAndUserAgent",
171 | id, referer .. "|" .. user_agent, recordHits, recordReferAndUserAgent);
172 |
173 | if res == ngx.null then
174 | if notFoundRedirect then
175 | return ngx.redirect(notFoundRedirect)
176 | end
177 | ngx.exit(ngx.HTTP_NOT_FOUND)
178 | end
179 |
180 | return ngx.redirect(res)
181 | end
182 |
183 | function _M.qry_by_short(url)
184 | if url == nil then
185 | ngx.say(nil)
186 | return ngx.exit(ngx.OK)
187 | end
188 |
189 | local id_index = string.find(url, "/0[0-9a-zA-Z]+$")
190 | if id_index == nil then
191 | ngx.say(nil)
192 | return ngx.exit(ngx.OK)
193 | end
194 |
195 | local red = connect()
196 | local res, err = red:eval(unpack_script, 4, "id", "referer_agent", "recordHits", "recordReferAndUserAgent",
197 | string.sub(url, id_index + 2), nil, nil, nil);
198 |
199 | if res == ngx.null then
200 | ngx.say(nil)
201 | return ngx.exit(ngx.OK)
202 | end
203 |
204 | ngx.say(res)
205 | return ngx.exit(ngx.OK)
206 | end
207 |
208 | return _M
209 |
--------------------------------------------------------------------------------
/conf/lua/resty/redis.lua:
--------------------------------------------------------------------------------
1 | -- Copyright (C) 2012-2013 Yichun Zhang (agentzh), CloudFlare Inc.
2 |
3 |
4 | local sub = string.sub
5 | local tcp = ngx.socket.tcp
6 | local concat = table.concat
7 | local len = string.len
8 | local null = ngx.null
9 | local pairs = pairs
10 | local unpack = unpack
11 | local setmetatable = setmetatable
12 | local tonumber = tonumber
13 | local error = error
14 |
15 |
16 | local _M = {
17 | _VERSION = '0.16'
18 | }
19 |
20 | local commands = {
21 | "append", "auth", "bgrewriteaof",
22 | "bgsave", "bitcount", "bitop",
23 | "blpop", "brpop",
24 | "brpoplpush", "client", "config",
25 | "dbsize",
26 | "debug", "decr", "decrby",
27 | "del", "discard", "dump",
28 | "echo",
29 | "eval", "exec", "exists",
30 | "expire", "expireat", "flushall",
31 | "flushdb", "get", "getbit",
32 | "getrange", "getset", "hdel",
33 | "hexists", "hget", "hgetall",
34 | "hincrby", "hincrbyfloat", "hkeys",
35 | "hlen",
36 | "hmget", --[[ "hmset", ]] "hset",
37 | "hsetnx", "hvals", "incr",
38 | "incrby", "incrbyfloat", "info",
39 | "keys",
40 | "lastsave", "lindex", "linsert",
41 | "llen", "lpop", "lpush",
42 | "lpushx", "lrange", "lrem",
43 | "lset", "ltrim", "mget",
44 | "migrate",
45 | "monitor", "move", "mset",
46 | "msetnx", "multi", "object",
47 | "persist", "pexpire", "pexpireat",
48 | "ping", "psetex", "psubscribe",
49 | "pttl",
50 | "publish", "punsubscribe", "pubsub",
51 | "quit",
52 | "randomkey", "rename", "renamenx",
53 | "restore",
54 | "rpop", "rpoplpush", "rpush",
55 | "rpushx", "sadd", "save",
56 | "scard", "script",
57 | "sdiff", "sdiffstore",
58 | "select", "set", "setbit",
59 | "setex", "setnx", "setrange",
60 | "shutdown", "sinter", "sinterstore",
61 | "sismember", "slaveof", "slowlog",
62 | "smembers", "smove", "sort",
63 | "spop", "srandmember", "srem",
64 | "strlen", "subscribe", "sunion",
65 | "sunionstore", "sync", "time",
66 | "ttl",
67 | "type", "unsubscribe", "unwatch",
68 | "watch", "zadd", "zcard",
69 | "zcount", "zincrby", "zinterstore",
70 | "zrange", "zrangebyscore", "zrank",
71 | "zrem", "zremrangebyrank", "zremrangebyscore",
72 | "zrevrange", "zrevrangebyscore", "zrevrank",
73 | "zscore", "zunionstore", "evalsha"
74 | }
75 |
76 |
77 | local mt = { __index = _M }
78 |
79 |
80 | function _M.new(self)
81 | local sock, err = tcp()
82 | if not sock then
83 | return nil, err
84 | end
85 | return setmetatable({ sock = sock }, mt)
86 | end
87 |
88 |
89 | function _M.set_timeout(self, timeout)
90 | local sock = self.sock
91 | if not sock then
92 | return nil, "not initialized"
93 | end
94 |
95 | return sock:settimeout(timeout)
96 | end
97 |
98 |
99 | function _M.connect(self, ...)
100 | local sock = self.sock
101 | if not sock then
102 | return nil, "not initialized"
103 | end
104 |
105 | return sock:connect(...)
106 | end
107 |
108 |
109 | function _M.set_keepalive(self, ...)
110 | local sock = self.sock
111 | if not sock then
112 | return nil, "not initialized"
113 | end
114 |
115 | return sock:setkeepalive(...)
116 | end
117 |
118 |
119 | function _M.get_reused_times(self)
120 | local sock = self.sock
121 | if not sock then
122 | return nil, "not initialized"
123 | end
124 |
125 | return sock:getreusedtimes()
126 | end
127 |
128 |
129 | function _M.close(self)
130 | local sock = self.sock
131 | if not sock then
132 | return nil, "not initialized"
133 | end
134 |
135 | return sock:close()
136 | end
137 |
138 |
139 | local function _read_reply(sock)
140 | local line, err = sock:receive()
141 | if not line then
142 | return nil, err
143 | end
144 |
145 | local prefix = sub(line, 1, 1)
146 |
147 | if prefix == "$" then
148 | -- print("bulk reply")
149 |
150 | local size = tonumber(sub(line, 2))
151 | if size < 0 then
152 | return null
153 | end
154 |
155 | local data, err = sock:receive(size)
156 | if not data then
157 | return nil, err
158 | end
159 |
160 | local dummy, err = sock:receive(2) -- ignore CRLF
161 | if not dummy then
162 | return nil, err
163 | end
164 |
165 | return data
166 |
167 | elseif prefix == "+" then
168 | -- print("status reply")
169 |
170 | return sub(line, 2)
171 |
172 | elseif prefix == "*" then
173 | local n = tonumber(sub(line, 2))
174 |
175 | -- print("multi-bulk reply: ", n)
176 | if n < 0 then
177 | return null
178 | end
179 |
180 | local vals = {};
181 | local nvals = 0
182 | for i = 1, n do
183 | local res, err = _read_reply(sock)
184 | if res then
185 | nvals = nvals + 1
186 | vals[nvals] = res
187 |
188 | elseif res == nil then
189 | return nil, err
190 |
191 | else
192 | -- be a valid redis error value
193 | nvals = nvals + 1
194 | vals[nvals] = {false, err}
195 | end
196 | end
197 | return vals
198 |
199 | elseif prefix == ":" then
200 | -- print("integer reply")
201 | return tonumber(sub(line, 2))
202 |
203 | elseif prefix == "-" then
204 | -- print("error reply: ", n)
205 |
206 | return false, sub(line, 2)
207 |
208 | else
209 | return nil, "unkown prefix: \"" .. prefix .. "\""
210 | end
211 | end
212 |
213 |
214 | local function _gen_req(args)
215 | local req = {"*", #args, "\r\n"}
216 | local nbits = #req
217 | local nargs = #args
218 |
219 | for i = 1, nargs do
220 | local arg = args[i]
221 |
222 | if not arg then
223 | nbits = nbits + 1
224 | req[nbits] = "$-1\r\n"
225 |
226 | else
227 | req[nbits + 1] = "$"
228 | req[nbits + 2] = len(arg)
229 | req[nbits + 3] = "\r\n"
230 | req[nbits + 4] = arg
231 | req[nbits + 5] = "\r\n"
232 | nbits = nbits + 5
233 | end
234 | end
235 |
236 | -- it is faster to do string concatenation on the Lua land
237 | return concat(req)
238 | end
239 |
240 |
241 | local function _do_cmd(self, ...)
242 | local args = {...}
243 |
244 | local sock = self.sock
245 | if not sock then
246 | return nil, "not initialized"
247 | end
248 |
249 | local req = _gen_req(args)
250 |
251 | local reqs = self._reqs
252 | if reqs then
253 | reqs[#reqs + 1] = req
254 | return
255 | end
256 |
257 | -- print("request: ", table.concat(req))
258 |
259 | local bytes, err = sock:send(req)
260 | if not bytes then
261 | return nil, err
262 | end
263 |
264 | return _read_reply(sock)
265 | end
266 |
267 |
268 | function _M.read_reply(self)
269 | local sock = self.sock
270 | if not sock then
271 | return nil, "not initialized"
272 | end
273 |
274 | return _read_reply(sock)
275 | end
276 |
277 |
278 | for i = 1, #commands do
279 | local cmd = commands[i]
280 |
281 | _M[cmd] =
282 | function (self, ...)
283 | return _do_cmd(self, cmd, ...)
284 | end
285 | end
286 |
287 |
288 | function _M.hmset(self, hashname, ...)
289 | local args = {...}
290 | if #args == 1 then
291 | local t = args[1]
292 | local array = {}
293 | local n = 0
294 | for k, v in pairs(t) do
295 | array[n + 1] = k
296 | array[n + 2] = v
297 | n = n + 2
298 | end
299 | -- print("key", hashname)
300 | return _do_cmd(self, "hmset", hashname, unpack(array))
301 | end
302 |
303 | -- backwards compatibility
304 | return _do_cmd(self, "hmset", hashname, ...)
305 | end
306 |
307 |
308 | function _M.init_pipeline(self)
309 | self._reqs = {}
310 | end
311 |
312 |
313 | function _M.cancel_pipeline(self)
314 | self._reqs = nil
315 | end
316 |
317 |
318 | function _M.commit_pipeline(self)
319 | local reqs = self._reqs
320 | if not reqs then
321 | return nil, "no pipeline"
322 | end
323 |
324 | self._reqs = nil
325 |
326 | local sock = self.sock
327 | if not sock then
328 | return nil, "not initialized"
329 | end
330 |
331 | local bytes, err = sock:send(reqs)
332 | if not bytes then
333 | return nil, err
334 | end
335 |
336 | local vals = {}
337 | local nvals = 0
338 | local nreqs = #reqs
339 | for i = 1, nreqs do
340 | local res, err = _read_reply(sock)
341 | if res then
342 | nvals = nvals + 1
343 | vals[nvals] = res
344 |
345 | elseif res == nil then
346 | return nil, err
347 |
348 | else
349 | -- be a valid redis error value
350 | nvals = nvals + 1
351 | vals[nvals] = {false, err}
352 | end
353 | end
354 |
355 | return vals
356 | end
357 |
358 |
359 | function _M.array_to_hash(self, t)
360 | local h = {}
361 | for i = 1, #t, 2 do
362 | h[t[i]] = t[i + 1]
363 | end
364 | return h
365 | end
366 |
367 |
368 | function _M.add_commands(...)
369 | local cmds = {...}
370 | for i = 1, #cmds do
371 | local cmd = cmds[i]
372 | _M[cmd] =
373 | function (self, ...)
374 | return _do_cmd(self, cmd, ...)
375 | end
376 | end
377 | end
378 |
379 |
380 | return _M
381 |
--------------------------------------------------------------------------------
/conf/nginx.conf:
--------------------------------------------------------------------------------
1 | #user nobody;
2 | worker_processes 1;
3 |
4 | #error_log logs/error.log;
5 | #error_log logs/error.log notice;
6 | #error_log logs/error.log info;
7 |
8 | #pid logs/nginx.pid;
9 |
10 |
11 | events {
12 | worker_connections 1024;
13 | }
14 |
15 |
16 | http {
17 | include mime.types;
18 | #default_type application/octet-stream;
19 | default_type text/plain;
20 |
21 | sendfile on;
22 |
23 | #keepalive_timeout 0;
24 | keepalive_timeout 65;
25 |
26 | lua_package_path ';;$prefix/conf/lua/?.lua;';
27 |
28 | server {
29 | listen 8088;
30 | server_name localhost;
31 |
32 |
33 | location /shorten-ui.html {
34 | expires epoch;
35 | root html;
36 | }
37 |
38 | #error_page 404 /404.html;
39 |
40 | # redirect server error pages to the static page /50x.html
41 | error_page 500 502 503 504 /50x.html;
42 | location = /50x.html {
43 | root html;
44 | }
45 |
46 | location = /shorten {
47 | add_header Content-Type text/plain;
48 |
49 | content_by_lua '
50 | local shorten = require "n3r.urlshortener_eval"
51 | shorten.pack(ngx.var.arg_url, ngx.var.arg_prefix, ngx.var.arg_random)
52 | ';
53 | }
54 |
55 | location ~ "^/0[0-9a-zA-Z]{1,12}$" {
56 | content_by_lua '
57 | local shorten = require "n3r.urlshortener_eval"
58 | local notFoundRedirect = "http://www.10010.com"
59 | shorten.unpack(notFoundRedirect)
60 | ';
61 | }
62 |
63 | location = /shorten-qrybyshort {
64 | add_header Content-Type text/plain;
65 |
66 | content_by_lua '
67 | local shorten = require "n3r.urlshortener_eval"
68 | shorten.qry_by_short(ngx.var.arg_url)
69 | ';
70 | }
71 | }
72 |
73 | }
74 |
--------------------------------------------------------------------------------
/html/shorten-ui.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 | Welcome to url-shorten page!
7 |
14 |
15 |
16 | Welcome to url-shorten page!
17 |
18 | 长地址:
19 |
20 |
21 | 短地址域名:
22 |
23 |
24 | 是否随机生成短地址:
25 | 是
26 | 否
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
132 |
133 |
134 |
--------------------------------------------------------------------------------