├── .gitignore ├── LICENSE ├── README.md ├── main.go └── peers.go /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.db 3 | *.log 4 | custom/ 5 | data/ 6 | .vendor/ 7 | .idea/ 8 | *.iml 9 | public/img/avatar/ 10 | files/ 11 | 12 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 13 | *.o 14 | *.a 15 | *.so 16 | 17 | # Folders 18 | _obj 19 | _test 20 | 21 | # Architecture specific extensions/prefixes 22 | *.[568vq] 23 | [568vq].out 24 | 25 | *.cgo1.go 26 | *.cgo2.c 27 | _cgo_defun.c 28 | _cgo_gotypes.go 29 | _cgo_export.* 30 | 31 | _testmain.go 32 | 33 | *.exe 34 | *.exe~ 35 | /gogs 36 | profile/ 37 | __pycache__ 38 | *.pem 39 | output* 40 | config.codekit 41 | .brackets.json 42 | minicdn 43 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## MiniCDN 2 | 一般来说会推荐采用qiniu或者upyun,又或者是amazon之类大公司的cdn服务，不过当需要一些自己实现的场景，比如企业内部软件的加速，就需要一个私有的CDN了。 3 | 4 | 极简内容分发系统是我在公司里面的一个项目，最近把他开源出来了。可能其他企业或者组织也需要一个类似的东西。 5 | 6 | 通常来说CDN分为push和pull两种方式，push比较适合大文件，pull适合小一些的文件，但是使用起来比push要简单的多。 7 | 8 | MiniCDN采用的就是pull这种方式，目前的实现方式是所有缓存的文件存储在内存中，使用LRU算法淘汰掉就的文件，镜像的文件受限于缓冲区的大小（目前的缓冲区是512M），如果超过了这个缓冲器大小，就没有加速的效果了。 9 | 10 | 没有所有的智能DNS,直接用的是最简单的http redirect. 还没写负载均衡, 所以redirect的时候，就是随机返回一个节点（简单粗暴) 11 | 12 | MiniCDN分为manager和peer。都是写在一个程序里。 13 | 14 | 我平常用的时候，就只开一个minicdn的Manager来加速我的后端服务器。如果没有节点的话，manager就会把自己当成一个节点。然后当有特别大的下载即将要冲击我的服务器的时候。我就会找很多的同事，将minicdn部署到他们平常用的电脑上(window系列, 因为是golang语言写的，什么平台的程序都能编译的出来)。这样我在短时间内就拥有了一个性能不错的cdn集群（充分利用同事的资源）。当下载冲击结束的时候，在把这些节点撤掉就可以了。相当省事 15 | 16 | ## 技术优势 17 | MiniCDN使用了谷歌开源出来的groupcache框架，目前`dl.google.com`后台就用到了groupcache，性能而言远超那些squid或者nginx-proxy-cache. 18 | 19 | groupcache的数据获取过程很有意思，我把他翻译了过来 20 | 21 | **groupcache的运行过程** 22 | 23 | [原文地址](https://github.com/golang/groupcache#loading-process) 24 | 25 | 查找`foo.txt`的过程(节点#5 是N个节点中的一个,每个节点的代码都是一样的) 26 | 27 | 1. 判断`foo.txt`是否内存中,并且很热门(super hot)，如果在就直接使用它 28 | 2. 判断`foo.txt`是否在内存中，并且当前节点拥有它(译者注:一致性hash查到该文件属于节点#5)，如果是就使用它 29 | 3. 在所有的节点中, 如果`foo.txt`的拥有者是节点#5，就加载这个文件。如果其他请求（通过直接的，或者rpc请求），节点#5会阻塞该请求，直接加载完毕，然后给所有请求返回同样的结果。否则使用rpc请求到拥有者的节点，如果请求失败，就本地加载(译者注:这种方式比较慢) 30 | 31 | groupcache是2013年写出来的，软件也不怎么更新了。里面的HTTPPool还有两个问题一直没有修复，这两个问题直接影响到节点之间不能交换数据。因为官方不用groupcache的这部分，所以连用户提的issue都不修（真是蛋疼） 32 | 33 | 是我fork的，把这两个问题修复了，虽然提了pr，不过感觉他们一时半会不会merge的。 34 | 35 | 受python-celery的启发，我实现了peer退出时候的两种状态(Warm close and Code close). Warn close可以保证党节点不在服务的时候才退出。Code close就是强制退出，下载者可能会发现下载中断的问题。 36 | 37 | ## 编译方法 38 | ```go 39 | go get -u -v github.com/codeskyblue/minicdn 40 | # run 41 | minicdn -h 42 | ``` 43 | 44 | ## 架构 45 | 46 | * M: Manager 47 | 48 | 1. 负责维护Peer的列表,每个peer会去Manager同步这个列表。 49 | 2. 所有的请求会先请求到manager, 然后由manager重定向到不同的peer 50 | 51 | * P: Peer 52 | 53 | 1. 提供文件的下载服务 54 | 2. Peer之间会根据从manager拿到的peer列表，同步文件 55 | 56 | Manager与Peer是一对多的关系 57 | 58 | ``` 59 | [M] 60 | |`------+--------+---...... 61 | | | | 62 | [P] [P] [P] .... 63 | ``` 64 | 65 | 66 | ### Run Manager 67 | 命令行启动 68 | 69 | ```shell 70 | ./minicdn -mirror http://localhost:5000 -addr :11000 -log cdn.log 71 | ``` 72 | 73 | * 对网站 `http://localhost:5000`进行镜像加速 74 | * 监听11000端口 75 | * 日志存储在cdn.log中 76 | 77 | 源站的所有下载地址，最好都改成这个 `http://localhost:5000/something` 78 | 79 | ### Run Slave 80 | 命令行启动 81 | 82 | ```shell 83 | ./minicdn -upstream http://localhost:11000 -addr :8001 84 | ``` 85 | 86 | * 指定Server地址 `http://localhost:11000` 87 | * 监听8001端口 88 | 89 | ### TODO 90 | * token 91 | * use a slave as a master 92 | * request log 93 | * cli args to specify cache size 94 | 95 | ## LICENSE 96 | Under [MIT LICENSE](LICENSE) 97 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "flag" 6 | "fmt" 7 | "io/ioutil" 8 | "log" 9 | "net/http" 10 | "net/url" 11 | "os" 12 | "os/signal" 13 | "path/filepath" 14 | "syscall" 15 | "time" 16 | 17 | "github.com/codeskyblue/groupcache" 18 | ) 19 | 20 | var thumbNails = groupcache.NewGroup("thumbnail", 512<<20, groupcache.GetterFunc( 21 | func(ctx groupcache.Context, key string, dest groupcache.Sink) error { 22 | fileName := key 23 | bytes, err := generateThumbnail(fileName) 24 | if err != nil { 25 | return err 26 | } 27 | dest.SetBytes(bytes) 28 | return nil 29 | })) 30 | 31 | func generateThumbnail(key string) ([]byte, error) { 32 | u, _ := url.Parse(*mirror) 33 | u.Path = key 34 | resp, err := http.Get(u.String()) 35 | if err != nil { 36 | return nil, err 37 | } 38 | defer resp.Body.Close() 39 | return ioutil.ReadAll(resp.Body) 40 | } 41 | 42 | func FileHandler(w http.ResponseWriter, r *http.Request) { 43 | key := r.URL.Path 44 | 45 | state.addActiveDownload(1) 46 | defer state.addActiveDownload(-1) 47 | 48 | if *upstream == "" { // Master 49 | if slaveAddr, err := slaveMap.PeekSlave(); err == nil { 50 | u, _ := url.Parse(slaveAddr) 51 | u.Path = r.URL.Path 52 | u.RawQuery = r.URL.RawQuery 53 | http.Redirect(w, r, u.String(), 302) 54 | return 55 | } 56 | } 57 | fmt.Println("KEY:", key) 58 | var data []byte 59 | var ctx groupcache.Context 60 | err := thumbNails.Get(ctx, key, groupcache.AllocatingByteSliceSink(&data)) 61 | if err != nil { 62 | http.Error(w, err.Error(), 500) 63 | return 64 | } 65 | var modTime time.Time = time.Now() 66 | 67 | rd := bytes.NewReader(data) 68 | http.ServeContent(w, r, filepath.Base(key), modTime, rd) 69 | } 70 | 71 | var ( 72 | mirror = flag.String("mirror", "", "Mirror Web Base URL") 73 | logfile = flag.String("log", "-", "Set log file, default STDOUT") 74 | upstream = flag.String("upstream", "", "Server base URL, conflict with -mirror") 75 | address = flag.String("addr", ":5000", "Listen address") 76 | token = flag.String("token", "1234567890ABCDEFG", "slave and master token should be same") 77 | ) 78 | 79 | func InitSignal() { 80 | sig := make(chan os.Signal, 2) 81 | signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) 82 | go func() { 83 | for { 84 | s := <-sig 85 | fmt.Println("Got signal:", s) 86 | if state.Closed { 87 | fmt.Println("Cold close !!!") 88 | os.Exit(1) 89 | } 90 | fmt.Println("Warm close, waiting ...") 91 | go func() { 92 | state.Close() 93 | os.Exit(0) 94 | }() 95 | } 96 | }() 97 | } 98 | 99 | func main() { 100 | flag.Parse() 101 | 102 | if *mirror != "" && *upstream != "" { 103 | log.Fatal("Can't set both -mirror and -upstream") 104 | } 105 | if *mirror == "" && *upstream == "" { 106 | log.Fatal("Must set one of -mirror and -upstream") 107 | } 108 | if *upstream != "" { 109 | if err := InitSlave(); err != nil { 110 | log.Fatal(err) 111 | } 112 | } 113 | if *mirror != "" { 114 | if _, err := url.Parse(*mirror); err != nil { 115 | log.Fatal(err) 116 | } 117 | if err := InitMaster(); err != nil { 118 | log.Fatal(err) 119 | } 120 | } 121 | 122 | InitSignal() 123 | //fmt.Println("Hello CDN") 124 | http.HandleFunc("/", FileHandler) 125 | log.Printf("Listening on %s", *address) 126 | log.Fatal(http.ListenAndServe(*address, nil)) 127 | } 128 | -------------------------------------------------------------------------------- /peers.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "log" 6 | "math/rand" 7 | "net" 8 | "net/http" 9 | "net/url" 10 | "strings" 11 | "sync" 12 | "time" 13 | 14 | "github.com/codeskyblue/groupcache" 15 | "github.com/gorilla/websocket" 16 | ) 17 | 18 | const defaultWSURL = "/_ws/" 19 | 20 | var ( 21 | upgrader = websocket.Upgrader{ 22 | ReadBufferSize: 1024, 23 | WriteBufferSize: 1024, 24 | } 25 | state = ServerState{ 26 | ActiveDownload: 0, 27 | Closed: false, 28 | } 29 | slaveMap = SlaveMap{ 30 | m: make(map[string]Slave, 10), 31 | } 32 | 33 | pool *groupcache.HTTPPool 34 | wsclient *websocket.Conn 35 | ) 36 | 37 | type Slave struct { 38 | Name string 39 | Connection *websocket.Conn 40 | ActiveDownload int 41 | } 42 | 43 | type SlaveMap struct { 44 | sync.RWMutex 45 | m map[string]Slave 46 | } 47 | 48 | func (sm *SlaveMap) AddSlave(name string, conn *websocket.Conn) { 49 | sm.Lock() 50 | defer sm.Unlock() 51 | sm.m[name] = Slave{ 52 | Name: name, 53 | Connection: conn, 54 | } 55 | } 56 | 57 | func (sm *SlaveMap) Delete(name string) { 58 | sm.Lock() 59 | delete(sm.m, name) 60 | sm.Unlock() 61 | } 62 | 63 | func (sm *SlaveMap) Keys() []string { 64 | sm.RLock() 65 | defer sm.RUnlock() 66 | keys := []string{} 67 | for key, _ := range sm.m { 68 | keys = append(keys, key) 69 | } 70 | return keys 71 | } 72 | 73 | func (sm *SlaveMap) PeekSlave() (string, error) { 74 | // FIXME(ssx): need to order by active download count 75 | sm.RLock() 76 | defer sm.RUnlock() 77 | ridx := rand.Int() 78 | keys := []string{} 79 | for key, _ := range sm.m { 80 | keys = append(keys, key) 81 | } 82 | if len(keys) == 0 { 83 | return "", errors.New("Slave count zero") 84 | } 85 | return keys[ridx%len(keys)], nil 86 | } 87 | 88 | func (sm *SlaveMap) BroadcastJSON(v interface{}) error { 89 | var err error 90 | for _, s := range sm.m { 91 | if err = s.Connection.WriteJSON(v); err != nil { 92 | return err 93 | } 94 | } 95 | return nil 96 | } 97 | 98 | type ServerState struct { 99 | sync.Mutex 100 | ActiveDownload int 101 | Closed bool 102 | } 103 | 104 | func (s *ServerState) addActiveDownload(n int) { 105 | s.Lock() 106 | defer s.Unlock() 107 | s.ActiveDownload += n 108 | } 109 | 110 | func (s *ServerState) Close() error { 111 | s.Closed = true 112 | time.Sleep(time.Millisecond * 500) // 0.5s 113 | for { 114 | if s.ActiveDownload == 0 { // Wait until all download finished 115 | break 116 | } 117 | time.Sleep(time.Millisecond * 100) 118 | } 119 | return nil 120 | } 121 | 122 | func InitSlave() (err error) { 123 | u, err := url.Parse(*upstream) 124 | if err != nil { 125 | return 126 | } 127 | u.Path = defaultWSURL 128 | conn, err := net.Dial("tcp", u.Host) 129 | if err != nil { 130 | return 131 | } 132 | wsclient, _, err = websocket.NewClient(conn, u, nil, 1024, 1024) 133 | if err != nil { 134 | return 135 | } 136 | 137 | // Get slave name from master 138 | _, port, _ := net.SplitHostPort(*address) 139 | wsclient.WriteJSON(map[string]string{ 140 | "action": "LOGIN", 141 | "token": *token, 142 | "port": port, 143 | }) 144 | var msg = make(map[string]string) 145 | if err = wsclient.ReadJSON(&msg); err != nil { 146 | return err 147 | } 148 | if me, ok := msg["self"]; ok { 149 | if pool == nil { 150 | pool = groupcache.NewHTTPPool(me) 151 | } 152 | peers := strings.Split(msg["peers"], ",") 153 | m := msg["mirror"] 154 | mirror = &m 155 | log.Println("Self name:", me) 156 | log.Println("Peer list:", peers) 157 | log.Println("Mirror site:", *mirror) 158 | pool.Set(peers...) 159 | } else { 160 | return errors.New("'peer_name' not found in master response") 161 | } 162 | 163 | // Listen peers update 164 | go func() { 165 | for { 166 | err := wsclient.ReadJSON(&msg) 167 | if err != nil { 168 | log.Println("Connection to master closed, retry in 10 seconds") 169 | time.Sleep(time.Second * 10) 170 | InitSlave() 171 | break 172 | } 173 | action := msg["action"] 174 | switch action { 175 | case "PEER_UPDATE": 176 | peers := strings.Split(msg["peers"], ",") 177 | log.Println("Update peer list:", peers) 178 | pool.Set(peers...) 179 | } 180 | } 181 | }() 182 | 183 | return nil 184 | } 185 | 186 | func InitMaster() (err error) { 187 | http.HandleFunc(defaultWSURL, WSHandler) 188 | return nil 189 | } 190 | 191 | func WSHandler(w http.ResponseWriter, r *http.Request) { 192 | conn, err := upgrader.Upgrade(w, r, nil) 193 | if err != nil { 194 | log.Println(err) 195 | return 196 | } 197 | log.Println(conn.RemoteAddr()) 198 | defer conn.Close() 199 | 200 | var name string 201 | remoteHost, _, _ := net.SplitHostPort(conn.RemoteAddr().String()) 202 | var msg = make(map[string]string) 203 | for { 204 | var err error 205 | if err = conn.ReadJSON(&msg); err != nil { 206 | break 207 | } 208 | 209 | log.Println(msg) 210 | switch msg["action"] { 211 | case "LOGIN": 212 | name = "http://" + remoteHost + ":" + msg["port"] 213 | currKeys := slaveMap.Keys() 214 | slaveMap.AddSlave(name, conn) 215 | err = conn.WriteJSON(map[string]string{ 216 | "self": name, 217 | "peers": strings.Join(slaveMap.Keys(), ","), 218 | "mirror": *mirror, 219 | }) 220 | 221 | slaveMap.RLock() 222 | for _, key := range currKeys { 223 | if s, exists := slaveMap.m[key]; exists { 224 | s.Connection.WriteJSON(map[string]string{ 225 | "action": "PEER_UPDATE", 226 | "peers": strings.Join(slaveMap.Keys(), ","), 227 | }) 228 | } 229 | } 230 | slaveMap.RUnlock() 231 | log.Printf("Slave: %s JOIN", name) 232 | } 233 | if err != nil { 234 | break 235 | } 236 | } 237 | 238 | slaveMap.Delete(name) 239 | slaveMap.RLock() 240 | for _, key := range slaveMap.Keys() { 241 | if s, exists := slaveMap.m[key]; exists { 242 | s.Connection.WriteJSON(map[string]string{ 243 | "action": "PEER_UPDATE", 244 | "peers": strings.Join(slaveMap.Keys(), ","), 245 | }) 246 | } 247 | } 248 | slaveMap.RUnlock() 249 | log.Printf("Slave: %s QUIT", name) 250 | } 251 | --------------------------------------------------------------------------------