├── README.md ├── lib └── resty │ └── filecache.lua └── test └── nginx.conf /README.md: -------------------------------------------------------------------------------- 1 | # Name 2 | lua-resty-filecache is an file write module with cache 3 | 4 | 5 | #Synopsis 6 | 7 | 8 | ``` 9 | lua_package_path "/path/to/lua-resty-filecache/lib/?.lua;;"; 10 | init_worker_by_lua_block { 11 | -- Import the module 12 | local filecache = require("resty.filecache") 13 | -- Start timer, regularly clean up the handle. 14 | filecache.start_cleanup_timer(600) --600 seconds no write operation, close the file handle. 15 | } 16 | 17 | server { 18 | listen 199 default; 19 | location /writelog { 20 | content_by_lua_block { 21 | local time = ngx.localtime() 22 | local date = string.sub(time,1,4) .. string.sub(time,6,7) .. string.sub(time, 9,10) 23 | local hour = string.sub(time, 12,13) 24 | local filename = string.format("/tmp/logs/myserver/%s/%s.log", date, hour) 25 | 26 | local filecache = require("resty.filecache") 27 | ngx.req.read_body() 28 | local log = ngx.req.get_body_data() 29 | log = log .. "\n" 30 | local err = filecache.write(filename, log) 31 | if err then 32 | ngx.log(ngx.ERR, "write log to [", filename, "] failed! err:", tostring(err)) 33 | end 34 | } 35 | } 36 | } 37 | ``` 38 | 39 | # Test Write Log 40 | ``` 41 | curl http://127.0.0.1:199/writelog -d "testlog" 42 | ``` 43 | # View the log 44 | ``` 45 | tail -f /tmp/logs/myserver/$yyyymmdd/$hh.log 46 | ``` 47 | 48 | #Author 49 | Xiaojie Liu. jie123108@163.com 50 | -------------------------------------------------------------------------------- /lib/resty/filecache.lua: -------------------------------------------------------------------------------- 1 | local bit = require "bit" 2 | local ffi = require "ffi" 3 | local bor = bit.bor; 4 | 5 | local O_RDONLY = 0x0000 6 | local O_WRONLY = 0x0001 7 | local O_RDWR = 0x0002 8 | local O_CREAT = 0x0040 9 | local O_APPEND = 0x0400 10 | 11 | local S_IRWXU=0x1c0 12 | local S_IRUSR=0x100 13 | local S_IWUSR=0x80 14 | local S_IXUSR=0x40 15 | 16 | local S_IRWXG=0x38 17 | local S_IRGRP=0x20 18 | local S_IWGRP=0x10 19 | local S_IXGRP=0x8 20 | 21 | local S_IRWXO=0x7 22 | local S_IROTH=0x4 23 | local S_IWOTH=0x2 24 | local S_IXOTH=0x1 25 | 26 | ffi.cdef[[ 27 | int rename(const char *oldpath, const char *newpath); 28 | int system(const char *command); 29 | int access(const char *pathname, int mode); 30 | int unlink(const char *pathname); 31 | int remove(const char *pathname); 32 | static const int F_OK = 0; 33 | char *strerror(int errnum); 34 | 35 | int open(const char *pathname, int flags, int mode); 36 | int close(int fd); 37 | 38 | uint64_t read(int fd, void *buf, uint64_t count); 39 | uint64_t write(int fd, const void *buf, uint64_t count); 40 | ]] 41 | 42 | local function exist(filename) 43 | return ffi.C.access(filename, 0) == ffi.C.F_OK 44 | end 45 | 46 | local function pathinfo(path) 47 | local pos = string.len(path) 48 | local extpos = pos + 1 49 | while pos > 0 do 50 | local b = string.byte(path, pos) 51 | if b == 46 then -- 46 = char "." 52 | extpos = pos 53 | elseif b == 47 then -- 47 = char "/" 54 | break 55 | end 56 | pos = pos - 1 57 | end 58 | 59 | local dirname = string.sub(path, 1, pos) 60 | local filename = string.sub(path, pos + 1) 61 | extpos = extpos - pos 62 | local basename = string.sub(filename, 1, extpos - 1) 63 | local extname = string.sub(filename, extpos) 64 | return { 65 | dirname = dirname, 66 | filename = filename, 67 | basename = basename, 68 | extname = extname 69 | } 70 | end 71 | 72 | local function mkdir(dir) 73 | return os.execute("mkdir -p " .. dir) 74 | end 75 | 76 | 77 | local _M = {} 78 | 79 | _M.files = {} 80 | 81 | function _M.get_file(filename) 82 | local fileinfo = _M.files[filename] 83 | if fileinfo then 84 | --ngx.log(ngx.INFO, "get a fd from cache for :", filename) 85 | return fileinfo['fd'] 86 | end 87 | local pathinfo = pathinfo(filename) 88 | local dirname = pathinfo['dirname'] 89 | if dirname and not exist(dirname) then 90 | mkdir(dirname) 91 | end 92 | 93 | local flag = bor(O_RDWR, O_CREAT, O_APPEND) 94 | local mode = bor(S_IRUSR,S_IWUSR, S_IRGRP,S_IWGRP, S_IROTH) 95 | local fd = ffi.C.open(filename, flag, mode) 96 | if fd == -1 then 97 | -- TODO: 权限问题，返回相应的错误码。 98 | local errmsg = ffi.string(ffi.C.strerror(ffi.errno())) 99 | ngx.log(ngx.ERR, "failed to open file(", filename, ",",flag, ",", mode,"), err:", errmsg) 100 | return nil, errmsg 101 | end 102 | 103 | ngx.log(ngx.INFO, "open a new file(", filename, ",",flag, ",", mode,") ") 104 | return fd 105 | end 106 | 107 | function _M.put_file(filename, fd) 108 | -- 记录最后修改时间，用于对长时间没访问的文件进行清理。 109 | _M.files[filename] = {fd=fd, last_time=ngx.time()} 110 | end 111 | 112 | function _M.clean_timeout(timeout) 113 | local count = 0 114 | for filename, fileinfo in pairs(_M.files) do 115 | local last_time = fileinfo["last_time"] 116 | if last_time and ngx.time()-last_time > timeout then 117 | local fd = fileinfo["fd"] 118 | local ret = ffi.C.close(fd) 119 | if ret == 0 then 120 | ngx.log(ngx.INFO, "success to close file [", filename, "]") 121 | else 122 | ngx.log(ngx.ERR, "failed to close file [", filename, "]") 123 | end 124 | _M.files[filename] = nil 125 | count = count + 1 126 | end 127 | end 128 | return count 129 | end 130 | 131 | function _M.write(filename, log) 132 | local fd, err = _M.get_file(filename) 133 | if not fd then 134 | return err 135 | end 136 | 137 | local ret = ffi.C.write(fd, log, string.len(log)) 138 | local err = nil 139 | if ret < 0 then 140 | err = ffi.string(ffi.C.strerror(ffi.errno())) 141 | ngx.log(ngx.ERR, "failed to write file [", filename, "], err:", err) 142 | end 143 | 144 | _M.put_file(filename, fd) 145 | return err 146 | end 147 | 148 | function _M.start_cleanup_timer(fd_expiretime) 149 | fd_expiretime = fd_expiretime or 600 150 | local delete_cache_fd_callback = function(premature) 151 | local ok, exp = pcall(_M.clean_timeout, fd_expiretime) 152 | if not ok then 153 | ngx.log(ngx.ERR, "call _M.clean_timeout() failed! err:", exp) 154 | end 155 | 156 | _M.start_cleanup_timer(fd_expiretime) 157 | end 158 | 159 | local clean_interval = math.floor(fd_expiretime/2) 160 | local next_run_time = (clean_interval - ngx.time()%clean_interval) 161 | ngx.log(ngx.INFO, " [delete-cache-fd-timer] next run time:", next_run_time) 162 | local ok, err = ngx.timer.at(next_run_time, delete_cache_fd_callback) 163 | if not ok then 164 | ngx.log(ngx.ERR, "failed to create timer: ", err) 165 | return 166 | end 167 | end 168 | 169 | return _M 170 | -------------------------------------------------------------------------------- /test/nginx.conf: -------------------------------------------------------------------------------- 1 | user nobody; 2 | worker_processes auto; 3 | 4 | error_log logs/error.log info; 5 | pid logs/nginx.pid; 6 | master_process on; 7 | working_directory /tmp; 8 | 9 | events { 10 | worker_connections 1024; 11 | } 12 | 13 | 14 | http { 15 | # include /usr/local/ngx_openresty/nginx/conf/mime.types; 16 | #default_type application/octet-stream; 17 | 18 | log_format main '$remote_addr|$remote_user|$time_local|$request|' 19 | '$status|$request_time|$body_bytes_sent|$http_referer|' 20 | '$http_user_agent|$http_x_forwarded_for|$http_x_real_ip|'; 21 | 22 | 23 | lua_code_cache on; 24 | sendfile on; 25 | 26 | client_body_buffer_size 1m; 27 | client_max_body_size 1m; 28 | client_body_timeout 3m; 29 | 30 | tcp_nodelay on; 31 | 32 | keepalive_timeout 65; 33 | lua_socket_buffer_size 256k; 34 | lua_check_client_abort off; 35 | 36 | lua_package_path "/root/work/GitHub/lua-resty-filecache/lib/?.lua;;"; 37 | init_worker_by_lua_block { 38 | -- Import the module 39 | local filecache = require("resty.filecache") 40 | -- Start timer, regularly clean up the handle. 41 | filecache.start_cleanup_timer(600) --600 seconds no write operation, close the file handle. 42 | } 43 | 44 | server { 45 | listen 199 default; 46 | location /writelog { 47 | content_by_lua_block { 48 | local time = ngx.localtime() 49 | local date = string.sub(time,1,4) .. string.sub(time,6,7) .. string.sub(time, 9,10) 50 | local hour = string.sub(time, 12,13) 51 | local filename = string.format("/tmp/logs/myserver/%s/%s.log", date, hour) 52 | 53 | local filecache = require("resty.filecache") 54 | ngx.req.read_body() 55 | local log = ngx.req.get_body_data() 56 | log = log .. "\n" 57 | local err = filecache.write(filename, log) 58 | if err then 59 | ngx.log(ngx.ERR, "write log to [", filename, "] failed! err:", tostring(err)) 60 | end 61 | } 62 | } 63 | } 64 | } 65 | --------------------------------------------------------------------------------