├── README.md ├── bin └── server.json ├── install ├── nginx-conf ├── access_file.lua └── nginx.conf ├── pkg └── linux_amd64 │ ├── config.a │ ├── dbop.a │ └── srvlog.a └── src ├── config └── server_config.go ├── dbop ├── dbop.go └── dbop_test.go ├── fmgr └── fmgr.go └── srvlog └── srv_log.go /README.md: -------------------------------------------------------------------------------- 1 | ngx_proxy_store_file_mgr 2 | ======================== 3 | 4 | 5 | Description: 6 | this project is a solution to solve nginx proxy store without file manager. Since nginx 's proxy_store directive is just responsible for saving files to disk and does not consider time to delete files. ngx_proxy_store_file_mgr is a process used to delete files on the disk using LRU . 7 | 8 | 9 | Notes: 10 | 1. file access time saved in redis database, using sorted set; 11 | 2. ngx_proxy_store_file_mgr uses json format config files, as follows : 12 | { 13 | "MaxFileLimit" : 100000, 14 | "CheckInterval" : 20, //check every 20s 15 | "ExpireDays" : 7, 16 | "ErrorLog": true, 17 | "AccessLog": false, 18 | "SortedSetName": "defset", 19 | "HashName": "defhash", 20 | "RedisAddr" : "127.0.0.1:6379", 21 | "RoutineCount" : 32 22 | } 23 | 24 | but currently the lru condition is just disk percentage(low than 20%), and later will implement the specified config parameter. 25 | 3. provides restful api to shutdown server: curl http://ip:10000/stop 26 | -------------------------------------------------------------------------------- /bin/server.json: -------------------------------------------------------------------------------- 1 | { 2 | "MaxFileLimit" : 100000, 3 | "CheckInterval" : 20, 4 | "ExpireDays" : 7, 5 | "ErrorLog": true, 6 | "AccessLog": false, 7 | "SortedSetName": "defset", 8 | "HashName": "defhash", 9 | "RedisAddr" : "127.0.0.1:6379", 10 | "RoutineCount" : 32 11 | } -------------------------------------------------------------------------------- /install: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ ! -f install ]; then 4 | echo 'install must be run within its container folder' 1>&2 5 | exit 1 6 | fi 7 | 8 | CURDIR=`pwd` 9 | OLDGOPATH="$GOPATH" 10 | export GOPATH="$OLDGOPATH:$CURDIR" 11 | 12 | gofmt -w src 13 | 14 | go install fmgr 15 | 16 | export GOPATH="$OLDGOPATH" 17 | 18 | echo 'finished' -------------------------------------------------------------------------------- /nginx-conf/access_file.lua: -------------------------------------------------------------------------------- 1 | local redis = require "resty.redis" 2 | local red = redis:new() 3 | 4 | red:set_timeout(1000) -- 1 sec 5 | 6 | -- or connect to a unix domain socket file listened 7 | -- by a redis server: 8 | -- local ok, err = red:connect("unix:/path/to/redis.sock") 9 | 10 | local ok, err = red:connect("127.0.0.1", 6379) 11 | if not ok then 12 | ngx.log(ngx.ERR, "connect to redis failed.", err) 13 | return 14 | end 15 | local res, err = red:get("cache") 16 | if res == ngx.null then 17 | ngx.log(ngx.ERR, "get cache from redis failed.", err) 18 | elseif res == "lock" then 19 | ngx.exec("@proxyNoStore", ngx.var.args) 20 | end 21 | --ngx.say("lock name", res) 22 | local md5 = ngx.md5(ngx.var.uri) 23 | local ok, err = red:zadd("defset", ngx.now(), md5) 24 | if not ok then 25 | ngx.log(ngx.ERR, "add to defset sorted set failed.", err) 26 | return 27 | end 28 | local storepath = ngx.var.document_root..ngx.var.uri 29 | local ok, err = red:hset("defhash", md5, storepath) 30 | if not ok then 31 | ngx.log(ngx.ERR, "set to hash failed.", err) 32 | return 33 | end 34 | 35 | -- put it into the connection pool of size 100, 36 | -- with 10 seconds max idle time 37 | local ok, err = red:set_keepalive(10000, 100) 38 | if not ok then 39 | ngx.log(ngx.ERR, "failed to set keepalive", err) 40 | return 41 | end 42 | 43 | -- or just close the connection right away: 44 | -- local ok, err = red:close() 45 | -- if not ok then 46 | -- ngx.say("failed to close: ", err) 47 | -- return 48 | -- end -------------------------------------------------------------------------------- /nginx-conf/nginx.conf: -------------------------------------------------------------------------------- 1 | 2 | #user nobody; 3 | worker_processes 24; 4 | 5 | #error_log logs/error.log; 6 | #error_log logs/error.log notice; 7 | #error_log logs/error.log info; 8 | 9 | #pid logs/nginx.pid; 10 | 11 | 12 | events { 13 | use epoll; 14 | worker_connections 100000; 15 | } 16 | 17 | 18 | http { 19 | include mime.types; 20 | default_type application/octet-stream; 21 | 22 | log_format main '$remote_addr - $remote_user [$time_local] "$request" ' 23 | '$status $body_bytes_sent "$http_referer" ' 24 | '"$http_user_agent" "$http_x_forwarded_for"' 25 | '"$upstream_addr" "$upstream_response_time" "$upstream_status"'; 26 | 27 | access_log logs/access.log main; 28 | resolver 114.114.114.114 8.8.8.8 8.8.4.4; 29 | sendfile on; 30 | #tcp_nopush on; 31 | 32 | #keepalive_timeout 0; 33 | keepalive_timeout 65; 34 | 35 | #gzip on; 36 | 37 | server { 38 | listen 9000; 39 | server_name 113.56.106.66; 40 | #server_name shawn.server.com; 41 | 42 | #charset koi8-r; 43 | 44 | #access_log logs/host.access.log main; 45 | location /{ 46 | client_max_body_size 10m; 47 | client_body_buffer_size 128k; 48 | 49 | proxy_connect_timeout 75s; 50 | proxy_read_timeout 60s; 51 | proxy_send_timeout 60s; 52 | proxy_buffer_size 64k; 53 | proxy_buffers 4 32k; 54 | proxy_busy_buffers_size 64k; 55 | proxy_temp_file_write_size 64k; 56 | proxy_set_header Host $host; 57 | proxy_ignore_client_abort on; 58 | proxy_pass http://$host$request_uri; 59 | } 60 | 61 | location ~ \.(pdf|chm){ 62 | access_by_lua_file /data/lua_ex/access_file.lua; 63 | error_page 404 500 = @proxyStore; 64 | root /data/store; 65 | } 66 | 67 | location @proxyStore{ 68 | client_max_body_size 10m; 69 | client_body_buffer_size 128k; 70 | 71 | proxy_connect_timeout 75s; 72 | proxy_read_timeout 60s; 73 | proxy_send_timeout 60s; 74 | proxy_buffer_size 64k; 75 | proxy_buffers 4 32k; 76 | proxy_busy_buffers_size 64k; 77 | proxy_temp_file_write_size 64k; 78 | #set $tmp_host '127.0.0.1:8000'; 79 | #proxy_pass http://$tmp_host$request_uri; 80 | proxy_set_header Host $host; 81 | proxy_pass http://$host$request_uri; 82 | proxy_store /data/store/$uri; 83 | proxy_store_access user:rw group:rw all:r; 84 | proxy_ignore_client_abort on; 85 | root /data/store; 86 | } 87 | 88 | location @proxyNoStore{ 89 | client_max_body_size 10m; 90 | client_body_buffer_size 128k; 91 | 92 | proxy_connect_timeout 75s; 93 | proxy_read_timeout 60s; 94 | proxy_send_timeout 60s; 95 | proxy_buffer_size 64k; 96 | proxy_buffers 4 32k; 97 | proxy_busy_buffers_size 64k; 98 | proxy_temp_file_write_size 64k; 99 | #set $tmp_host '127.0.0.1:8000'; 100 | proxy_set_header Host $host; 101 | #proxy_pass http://$tmp_host$request_uri; 102 | proxy_pass http://$host$request_uri; 103 | proxy_ignore_client_abort on; 104 | } 105 | 106 | #error_page 404 /404.html; 107 | 108 | # redirect server error pages to the static page /50x.html 109 | # 110 | error_page 500 502 503 504 /50x.html; 111 | location = /50x.html { 112 | root html; 113 | } 114 | 115 | # proxy the PHP scripts to Apache listening on 127.0.0.1:80 116 | # 117 | #location ~ \.php$ { 118 | # proxy_pass http://127.0.0.1; 119 | #} 120 | 121 | # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000 122 | # 123 | #location ~ \.php$ { 124 | # root html; 125 | # fastcgi_pass 127.0.0.1:9000; 126 | # fastcgi_index index.php; 127 | # fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name; 128 | # include fastcgi_params; 129 | #} 130 | 131 | # deny access to .htaccess files, if Apache's document root 132 | # concurs with nginx's one 133 | # 134 | #location ~ /\.ht { 135 | # deny all; 136 | #} 137 | } 138 | 139 | 140 | # another virtual host using mix of IP-, name-, and port-based configuration 141 | # 142 | server { 143 | listen 8000; 144 | #listen somename:8080; 145 | server_name a.b.c; 146 | 147 | location / { 148 | content_by_lua ' 149 | ngx.say("hahah", ngx.req.raw_header()) 150 | '; 151 | } 152 | } 153 | 154 | 155 | # HTTPS server 156 | # 157 | #server { 158 | # listen 443 ssl; 159 | # server_name localhost; 160 | 161 | # ssl_certificate cert.pem; 162 | # ssl_certificate_key cert.key; 163 | 164 | # ssl_session_cache shared:SSL:1m; 165 | # ssl_session_timeout 5m; 166 | 167 | # ssl_ciphers HIGH:!aNULL:!MD5; 168 | # ssl_prefer_server_ciphers on; 169 | 170 | # location / { 171 | # root html; 172 | # index index.html index.htm; 173 | # } 174 | #} 175 | 176 | } 177 | -------------------------------------------------------------------------------- /pkg/linux_amd64/config.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cxwshawn/ngx_proxy_store_file_mgr/71318177355e6921b9ad9825fa5c9523fae60578/pkg/linux_amd64/config.a -------------------------------------------------------------------------------- /pkg/linux_amd64/dbop.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cxwshawn/ngx_proxy_store_file_mgr/71318177355e6921b9ad9825fa5c9523fae60578/pkg/linux_amd64/dbop.a -------------------------------------------------------------------------------- /pkg/linux_amd64/srvlog.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cxwshawn/ngx_proxy_store_file_mgr/71318177355e6921b9ad9825fa5c9523fae60578/pkg/linux_amd64/srvlog.a -------------------------------------------------------------------------------- /src/config/server_config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "io/ioutil" 8 | "os" 9 | "os/exec" 10 | "path/filepath" 11 | ) 12 | 13 | type FileMgrConfig struct { 14 | MaxFileLimit int 15 | CheckInterval int 16 | ExpireDays int 17 | ErrorLog bool 18 | AccessLog bool 19 | SortedSetName string 20 | HashName string 21 | DelPercentOnce float32 22 | RedisAddr string 23 | RoutineCount int 24 | RedisLockName string 25 | } 26 | 27 | var Defaultfmc *FileMgrConfig 28 | 29 | func init() { 30 | Defaultfmc = &FileMgrConfig{100000, 31 | 10 * 60, 7, true, false, "defset", "defhash", 32 | 33.3333, "127.0.0.1:6379", 32, "cache"} 33 | } 34 | 35 | func ParseConfig(configFileName string) error { 36 | exePath, err1 := exec.LookPath(os.Args[0]) 37 | if err1 != nil { 38 | fmt.Println(err1.Error()) 39 | return err1 40 | } 41 | exeFullPath, err1 := filepath.Abs(exePath) 42 | if err1 != nil { 43 | fmt.Println(err1.Error()) 44 | return err1 45 | } 46 | configFullPath := fmt.Sprintf("%s/%s", filepath.Dir(exeFullPath), configFileName) 47 | 48 | file, err1 := os.Open(configFullPath) 49 | defer file.Close() 50 | 51 | if err1 != nil { 52 | fmt.Println(err1.Error()) 53 | return err1 54 | } 55 | data, err1 := ioutil.ReadAll(file) 56 | if err1 != nil { 57 | fmt.Println(err1.Error()) 58 | return err1 59 | } 60 | if Defaultfmc == nil { 61 | fmt.Println("Defaultfmc is nil!") 62 | return errors.New("Default config object is nil.") 63 | } 64 | err1 = json.Unmarshal(data, Defaultfmc) 65 | if err1 != nil { 66 | fmt.Println(err1.Error()) 67 | return err1 68 | } 69 | return nil 70 | } 71 | 72 | func GetCheckInterval() int { 73 | return Defaultfmc.CheckInterval 74 | } 75 | 76 | func GetMaxFileLimit() int { 77 | return Defaultfmc.MaxFileLimit 78 | } 79 | 80 | func IsLogError() bool { 81 | return Defaultfmc.ErrorLog 82 | } 83 | 84 | func IsLogAccess() bool { 85 | return Defaultfmc.AccessLog 86 | } 87 | 88 | func GetExpireDays() int { 89 | return Defaultfmc.ExpireDays 90 | } 91 | 92 | func GetSortedSetName() string { 93 | return Defaultfmc.SortedSetName 94 | } 95 | 96 | func GetHashName() string { 97 | return Defaultfmc.HashName 98 | } 99 | 100 | func GetDelPercent() float32 { 101 | return Defaultfmc.DelPercentOnce 102 | } 103 | 104 | func GetRedisAddr() string { 105 | return Defaultfmc.RedisAddr 106 | } 107 | 108 | func GetRoutineCount() int { 109 | return Defaultfmc.RoutineCount 110 | } 111 | 112 | func GetRedisKeyName() string { 113 | return Defaultfmc.RedisLockName 114 | } 115 | -------------------------------------------------------------------------------- /src/dbop/dbop.go: -------------------------------------------------------------------------------- 1 | package dbop 2 | 3 | import ( 4 | "config" 5 | "container/list" 6 | "github.com/hoisie/redis" 7 | "srvlog" 8 | ) 9 | 10 | var client redis.Client 11 | 12 | // func init() { 13 | 14 | // } 15 | func InitDb(redisAddr string) { 16 | client.Addr = redisAddr 17 | } 18 | 19 | func GetSetCount() (count int, err error) { 20 | sortedSetName := config.GetSortedSetName() 21 | card, err1 := client.Zcard(sortedSetName) 22 | if err1 != nil { 23 | srvlog.EPrintf("%s", err.Error()) 24 | err = err1 25 | return 26 | } 27 | return card, nil 28 | } 29 | 30 | func LockRedis() error { 31 | keyName := config.GetRedisKeyName() 32 | err := client.Set(keyName, []byte("lock")) 33 | if err != nil { 34 | srvlog.Printf("%s", err.Error()) 35 | return err 36 | } 37 | return nil 38 | } 39 | 40 | func UnlockRedis() error { 41 | keyName := config.GetRedisKeyName() 42 | err := client.Set(keyName, []byte("unlock")) 43 | if err != nil { 44 | srvlog.Printf("%s", err.Error()) 45 | return err 46 | } 47 | return nil 48 | } 49 | 50 | func GetLeastUsedKeys() (keys [][]byte, err error) { 51 | sortedSetName := config.GetSortedSetName() 52 | card, err1 := client.Zcard(sortedSetName) 53 | if err1 != nil { 54 | srvlog.EPrintf("%s", err.Error()) 55 | err = err1 56 | return 57 | } 58 | delPercent := config.GetDelPercent() 59 | delCount := int(float32(card) * (delPercent / 100.00)) 60 | data, err1 := client.Zrange(sortedSetName, 0, delCount) 61 | if err1 != nil { 62 | srvlog.EPrintf("%s", err.Error()) 63 | err = err1 64 | return 65 | } 66 | 67 | keys = make([][]byte, 0) 68 | keys = data 69 | err = nil 70 | return 71 | } 72 | 73 | func DeleteLeastUsedKeys() error { 74 | //delete sorted element in sorted set 75 | //delete element in the hash buckets. 76 | sortedSetName := config.GetSortedSetName() 77 | card, err1 := client.Zcard(sortedSetName) 78 | if err1 != nil { 79 | srvlog.EPrintf("%s", err1.Error()) 80 | return err1 81 | } 82 | delPercent := config.GetDelPercent() 83 | delCount := int(float32(card) * (delPercent / 100.0)) 84 | keys, err1 := client.Zrange(sortedSetName, 0, delCount) 85 | if err1 != nil { 86 | srvlog.EPrintf("%s", err1.Error()) 87 | return err1 88 | } 89 | _, err1 = client.Zremrangebyrank(sortedSetName, 0, delCount) 90 | if err1 != nil { 91 | srvlog.EPrintf("%s", err1.Error()) 92 | return err1 93 | } 94 | 95 | hashName := config.GetHashName() 96 | for _, val := range keys { 97 | _, err1 := client.Hdel(hashName, string(val)) 98 | if err1 != nil { 99 | srvlog.EPrintf("%s", err1.Error()) 100 | return err1 101 | } 102 | } 103 | return nil 104 | } 105 | 106 | func GetLeastUsedFiles(keys [][]byte) (filepaths *list.List, err error) { 107 | filepaths = list.New() 108 | hashName := config.GetHashName() 109 | strKeys := make([]string, 0) 110 | for _, v := range keys { 111 | strKeys = append(strKeys, string(v)) 112 | } 113 | 114 | data, err1 := client.Hmget(hashName, strKeys...) 115 | if err1 != nil { 116 | srvlog.EPrintf("%s", err.Error()) 117 | err = err1 118 | return 119 | } 120 | //filepaths = make([]string, len(data)) 121 | srvlog.Printf("filepaths length :%d, data len:%d, strkeys len:%d, keys len:%d", 122 | filepaths.Len(), len(data), len(strKeys), len(keys)) 123 | 124 | for _, v := range data { 125 | srvlog.Printf("%s\n", string(v)) 126 | //filepaths = append(filepaths, string(v)) 127 | filepaths.PushBack(v) 128 | } 129 | srvlog.Printf("filepaths length :%d", filepaths.Len()) 130 | 131 | err = nil 132 | return 133 | } 134 | -------------------------------------------------------------------------------- /src/dbop/dbop_test.go: -------------------------------------------------------------------------------- 1 | package dbop 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func Test_GetSetCount(t *testing.T) { 8 | count, err := dbop.GetSetCount() 9 | if err != nil { 10 | t.Errorf("%s", err.Error()) 11 | } 12 | } 13 | 14 | func Test_LockRedis(t *testing.T) { 15 | err := dbop.LockRedis() 16 | if err != nil { 17 | t.Errorf("%s", err.Error()) 18 | } 19 | } 20 | 21 | func Test_UnlockRedis(t *testing.T){ 22 | err := dbop.UnlockRedis() 23 | if err != nil { 24 | t.Errorf("%s", err.Error()) 25 | } 26 | } 27 | 28 | func Test_tt(){ 29 | return 30 | } 31 | 32 | func Test_tt1(){ 33 | return 34 | } -------------------------------------------------------------------------------- /src/fmgr/fmgr.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "config" 5 | "container/list" 6 | "dbop" 7 | "flag" 8 | "fmt" 9 | "log" 10 | "net/http" 11 | "os" 12 | "runtime" 13 | "runtime/debug" 14 | "srvlog" 15 | "sync" 16 | "sync/atomic" 17 | "syscall" 18 | "time" 19 | ) 20 | 21 | var configFileName *string = flag.String("config", "server.json", "server config file name") 22 | 23 | var stopCh chan bool 24 | var stopFlag uint32 25 | var mutex sync.Mutex 26 | 27 | func init() { 28 | runtime.GOMAXPROCS(runtime.NumCPU()) 29 | flag.Parse() 30 | } 31 | func testDBOp() { 32 | count, err := dbop.GetSetCount() 33 | if err != nil { 34 | fmt.Println("%s", err.Error()) 35 | } 36 | fmt.Println("count is:", count) 37 | keys, err := dbop.GetLeastUsedKeys() 38 | if err != nil { 39 | fmt.Println(err.Error()) 40 | return 41 | } 42 | for _, v := range keys { 43 | fmt.Println(string(v)) 44 | } 45 | // filepaths, err := dbop.GetLeastUsedFiles(keys) 46 | // if err != nil { 47 | // fmt.Println(err.Error()) 48 | // return 49 | // } 50 | // for _, v := range filepaths { 51 | // fmt.Println(v) 52 | // } 53 | } 54 | 55 | func stopHandler(w http.ResponseWriter, req *http.Request) { 56 | srvlog.Println("stop handler...") 57 | 58 | if req.Method != "GET" { 59 | return 60 | } 61 | w.WriteHeader(http.StatusOK) 62 | stopCh <- true 63 | } 64 | 65 | func restAPI() { 66 | http.HandleFunc("/stop", stopHandler) 67 | 68 | s := &http.Server{ 69 | Addr: fmt.Sprintf("127.0.0.1:10000"), 70 | Handler: nil, 71 | ReadTimeout: 100 * time.Millisecond, 72 | WriteTimeout: 100 * time.Millisecond, 73 | MaxHeaderBytes: 1 << 20, 74 | } 75 | 76 | err := s.ListenAndServe() 77 | if err != nil { 78 | log.Fatal("ListenAndServe: ", err) 79 | } 80 | } 81 | 82 | var filepaths *list.List 83 | 84 | func getFiles() error { 85 | defer func() { 86 | if err, ok := recover().(error); ok { 87 | srvlog.EPrintln("WARN: panic %v", err) 88 | srvlog.EPrintln(string(debug.Stack())) 89 | } 90 | }() 91 | srvlog.Printf("getFiles...") 92 | data, err := dbop.GetLeastUsedKeys() 93 | if err != nil { 94 | srvlog.EPrintf("%s", err.Error()) 95 | return err 96 | } 97 | filepaths, err = dbop.GetLeastUsedFiles(data) 98 | if err != nil { 99 | srvlog.EPrintf("%s", err.Error()) 100 | return err 101 | } 102 | srvlog.Printf("filepaths length :%d", filepaths.Len()) 103 | elem := filepaths.Front() 104 | for { 105 | if elem != nil { 106 | srvlog.Println(string(elem.Value.([]byte))) 107 | } else { 108 | break 109 | } 110 | elem = elem.Next() 111 | } 112 | return nil 113 | } 114 | 115 | func deleteFiles(ch chan bool) { 116 | defer func(ch chan bool) { 117 | if err, ok := recover().(error); ok { 118 | srvlog.EPrintln("WARN: panic %v", err) 119 | srvlog.EPrintln(string(debug.Stack())) 120 | } 121 | srvlog.Printf("deleteFiles send over to channel.") 122 | ch <- true 123 | }(ch) 124 | 125 | srvlog.Printf("deleteFiles...") 126 | 127 | for { 128 | stop := atomic.LoadUint32(&stopFlag) 129 | if stop == 1 { 130 | break 131 | } 132 | 133 | mutex.Lock() 134 | if filepaths.Len() == 0 { 135 | mutex.Unlock() 136 | break 137 | } 138 | pElem := filepaths.Front() 139 | filepath := filepaths.Remove(pElem).([]byte) 140 | mutex.Unlock() 141 | //todo:delete file. 142 | srvlog.Printf("delete file %s", string(filepath)) 143 | // srvlog.Printf("delete file %s", filepath) 144 | err := os.Remove(string(filepath)) 145 | if err != nil { 146 | srvlog.EPrintf("%s", err.Error()) 147 | } 148 | } 149 | srvlog.Printf("deleteFiles end...") 150 | } 151 | 152 | func NeedLRU() (lru bool, err error) { 153 | fs := &syscall.Statfs_t{} 154 | err = syscall.Statfs("/", fs) 155 | if err != nil { 156 | srvlog.EPrintf("%s", err.Error()) 157 | lru = false 158 | return 159 | } 160 | 161 | all := fs.Blocks * uint64(fs.Bsize) 162 | free := fs.Bfree * uint64(fs.Bsize) 163 | leftPercent := int32(float64(free) / float64(all) * 100.0) 164 | if leftPercent < 20 { 165 | return true, nil 166 | } 167 | return false, nil 168 | } 169 | 170 | func main() { 171 | err := config.ParseConfig(*configFileName) 172 | if err != nil { 173 | fmt.Println(err.Error()) 174 | return 175 | } 176 | srvlog.InitLog("Nginx_File_Manager") 177 | dbop.InitDb(config.GetRedisAddr()) 178 | 179 | stopFlag = 0 180 | processing := false 181 | stopChs := make([]chan bool, config.GetRoutineCount()) 182 | for i := 0; i < config.GetRoutineCount(); i++ { 183 | stopChs[i] = make(chan bool) 184 | } 185 | timeout := make(chan bool) 186 | stopCh = make(chan bool) 187 | loopBreak := false 188 | 189 | go restAPI() 190 | 191 | for { 192 | go func() { 193 | time.Sleep(time.Duration(config.GetCheckInterval() * int(time.Second))) 194 | timeout <- true 195 | //close(timeout) 196 | }() 197 | select { 198 | case <-stopCh: 199 | srvlog.Printf("received stop command!") 200 | atomic.SwapUint32(&stopFlag, 1) 201 | if processing == true { 202 | for _, ch := range stopChs { 203 | <-ch 204 | } 205 | } 206 | loopBreak = true 207 | case <-timeout: 208 | srvlog.Printf("timeout, start to process.") 209 | lru, _ := NeedLRU() 210 | if lru == true { 211 | err = dbop.LockRedis() 212 | if err != nil { 213 | continue 214 | } 215 | 216 | processing = true 217 | err = getFiles() 218 | if err != nil { 219 | dbop.UnlockRedis() 220 | processing = false 221 | continue 222 | } 223 | for i := 0; i < config.GetRoutineCount(); i++ { 224 | go deleteFiles(stopChs[i]) 225 | } 226 | // for _, ch := range stopChs { 227 | // <-ch 228 | // } 229 | for i := 0; i < config.GetRoutineCount(); i++ { 230 | <-stopChs[i] 231 | } 232 | dbop.DeleteLeastUsedKeys() 233 | dbop.UnlockRedis() 234 | 235 | srvlog.Printf("timeout process end.") 236 | processing = false 237 | } 238 | } 239 | if loopBreak == true { 240 | break 241 | } 242 | } 243 | srvlog.Println("nginx file manager quitting... ") 244 | 245 | close(timeout) 246 | close(stopCh) 247 | for _, val := range stopChs { 248 | close(val) 249 | } 250 | } 251 | -------------------------------------------------------------------------------- /src/srvlog/srv_log.go: -------------------------------------------------------------------------------- 1 | package srvlog 2 | 3 | import ( 4 | "config" 5 | "fmt" 6 | "log" 7 | "os" 8 | "os/exec" 9 | "path/filepath" 10 | "syscall" 11 | ) 12 | 13 | const ( 14 | LogError, LogAccess = 0, 1 15 | ) 16 | 17 | func InitLog(serverName string) error { 18 | logError := config.IsLogError() 19 | logAccess := config.IsLogAccess() 20 | if logError == true { 21 | err := initLog(os.Args[0], serverName, LogError) 22 | if err != nil { 23 | return err 24 | } 25 | } 26 | if logAccess == true { 27 | err := initLog(os.Args[0], serverName, LogAccess) 28 | if err != nil { 29 | return err 30 | } 31 | } 32 | return nil 33 | } 34 | 35 | var DefaultLogger []*log.Logger 36 | var DefaultLogFile []*os.File 37 | 38 | func initLog(exe string, logName string, logType int) error { 39 | exePath, err := exec.LookPath(exe) 40 | if err != nil { 41 | fmt.Println(err.Error()) 42 | return err 43 | } 44 | exeFullPath, err := filepath.Abs(exePath) 45 | if err != nil { 46 | fmt.Println(err.Error()) 47 | return err 48 | } 49 | var logFullPath string 50 | if logType == LogError { 51 | logFullPath = fmt.Sprintf("%s/%s_%d_error.log", filepath.Dir(exeFullPath), logName, syscall.Getpid()) 52 | } else if logType == LogAccess { 53 | logFullPath = fmt.Sprintf("%s/%s_%d_access.log", filepath.Dir(exeFullPath), logName, syscall.Getpid()) 54 | } 55 | 56 | file, err := os.Create(logFullPath) 57 | //LogErrorFile = file 58 | DefaultLogFile = append(DefaultLogFile, file) 59 | 60 | if err != nil { 61 | fmt.Println(err.Error()) 62 | return err 63 | } 64 | srvLogger := log.New(file, logName, log.Ldate| 65 | log.Ltime|log.Lmicroseconds|log.Llongfile) 66 | DefaultLogger = append(DefaultLogger, srvLogger) 67 | 68 | return nil 69 | } 70 | 71 | // Printf calls l.Output to print to the logger. 72 | // Arguments are handled in the manner of fmt.Printf. 73 | func Printf(format string, v ...interface{}) { 74 | if config.IsLogAccess() == true { 75 | DefaultLogger[LogAccess].Output(2, fmt.Sprintf(format, v...)) 76 | } 77 | } 78 | 79 | // Print calls l.Output to print to the logger. 80 | // Arguments are handled in the manner of fmt.Print. 81 | func Print(v ...interface{}) { 82 | if config.IsLogAccess() == true { 83 | DefaultLogger[LogAccess].Output(2, fmt.Sprint(v...)) 84 | } 85 | } 86 | 87 | // Println calls l.Output to print to the logger. 88 | // Arguments are handled in the manner of fmt.Println. 89 | func Println(v ...interface{}) { 90 | if config.IsLogAccess() == true { 91 | DefaultLogger[LogAccess].Output(2, fmt.Sprintln(v...)) 92 | } 93 | } 94 | 95 | // Fatal is equivalent to l.Print() followed by a call to os.Exit(1). 96 | func Fatal(v ...interface{}) { 97 | if config.IsLogAccess() == true { 98 | DefaultLogger[LogAccess].Output(2, fmt.Sprint(v...)) 99 | os.Exit(1) 100 | } 101 | } 102 | 103 | // Printf calls l.Output to print to the logger. 104 | // Arguments are handled in the manner of fmt.Printf. 105 | func EPrintf(format string, v ...interface{}) { 106 | DefaultLogger[LogError].Output(2, fmt.Sprintf(format, v...)) 107 | } 108 | 109 | // Print calls l.Output to print to the logger. 110 | // Arguments are handled in the manner of fmt.Print. 111 | func EPrint(v ...interface{}) { DefaultLogger[LogError].Output(2, fmt.Sprint(v...)) } 112 | 113 | // Println calls l.Output to print to the logger. 114 | // Arguments are handled in the manner of fmt.Println. 115 | func EPrintln(v ...interface{}) { DefaultLogger[LogError].Output(2, fmt.Sprintln(v...)) } 116 | 117 | // Fatal is equivalent to l.Print() followed by a call to os.Exit(1). 118 | func EFatal(v ...interface{}) { 119 | DefaultLogger[LogError].Output(2, fmt.Sprint(v...)) 120 | os.Exit(1) 121 | } 122 | 123 | // Fatalf is equivalent to l.Printf() followed by a call to os.Exit(1). 124 | func Fatalf(format string, v ...interface{}) { 125 | DefaultLogger[LogError].Output(2, fmt.Sprintf(format, v...)) 126 | os.Exit(1) 127 | } 128 | 129 | // Fatalln is equivalent to l.Println() followed by a call to os.Exit(1). 130 | func Fatalln(v ...interface{}) { 131 | DefaultLogger[LogError].Output(2, fmt.Sprintln(v...)) 132 | os.Exit(1) 133 | } 134 | --------------------------------------------------------------------------------