├── .gitignore ├── Makefile ├── Makefile.i386 ├── README.md ├── anlog.go ├── anscdn.cfg.example ├── anscdn.go ├── anscdn.tmproj ├── cdnize.go ├── cdnize_test.go ├── config.go ├── configfile.go ├── downloader.go ├── filemon.go └── utils.go /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | anscdn.cfg 3 | archive/ 4 | data/ 5 | filemon 6 | anscdn 7 | *.6 8 | .gitignore 9 | *~ 10 | *# 11 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | include $(GOROOT)/src/Make.inc 2 | 3 | CC=$(GOBIN)/6g 4 | LD=$(GOBIN)/6l 5 | 6 | OBJS =\ 7 | anlog.6 \ 8 | configfile.6 \ 9 | filemon.6 \ 10 | config.6 \ 11 | utils.6 \ 12 | downloader.6 \ 13 | cdnize.6 \ 14 | anscdn.6 15 | 16 | TESTSSRC = \ 17 | cdnize_test.go 18 | 19 | all: anscdn 20 | 21 | anscdn: $(OBJS) 22 | $(LD) -o $@ anscdn.$(O) 23 | 24 | %.6 : %.go 25 | $(GC) $< 26 | 27 | % : %.6 28 | $(LD) -L . -o $@ $^ 29 | 30 | clean: 31 | rm -f *.$(O) 32 | -------------------------------------------------------------------------------- /Makefile.i386: -------------------------------------------------------------------------------- 1 | CC=$(GOBIN)/8g 2 | LD=$(GOBIN)/8l 3 | 4 | OBJS =\ 5 | anlog.8 \ 6 | configfile.8 \ 7 | filemon.8 \ 8 | config.8 \ 9 | utils.8 \ 10 | anscdn.8 11 | 12 | all: anscdn 13 | 14 | anscdn: $(OBJS) 15 | $(LD) -o $@ anscdn.8 16 | 17 | %.8 : %.go 18 | $(CC) $< 19 | 20 | % : %.8 21 | $(LD) -L . -o $@ $^ 22 | 23 | clean: 24 | rm *.8 -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | AnsCDN is simple CDN server written in Golang 3 | ================================================ 4 | 5 | AnsCDN is experimental project for implementing CDN server to be lightweight and robust with non-blocking IO operation, it's mean can handle high request concurrently. 6 | 7 | COMPILATION 8 | ------------------ 9 | 10 | Require Golang >= 6448+ 11 | 12 | Just type 13 | 14 | $ make 15 | 16 | or 17 | 18 | $ gomake 19 | 20 | CREATING CONFIGURATION 21 | ------------------------ 22 | 23 | Please see anscdn.cfg.example 24 | 25 | RUNNING 26 | ------------------------ 27 | 28 | $ ./anscdn 29 | 30 | 31 | that's all 32 | 33 | --robin -------------------------------------------------------------------------------- /anlog.go: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * AnsCDN Copyright (C) 2010 Robin Syihab (r [at] nosql.asia) 4 | * Simple CDN server written in Golang. 5 | * 6 | * License: General Public License v2 (GPLv2) 7 | * 8 | * Copyright (c) 2009 The Go Authors. All rights reserved. 9 | * 10 | **/ 11 | 12 | package anlog 13 | 14 | 15 | 16 | import ( 17 | "fmt" 18 | "os" 19 | ) 20 | 21 | var Quiet bool 22 | 23 | func Info(format string, v ...interface{}) { 24 | if Quiet{return;} 25 | fmt.Printf("[info] " + format, v...); 26 | } 27 | func Warn(format string, v ...interface{}) {fmt.Printf("[warning] " + format, v...);} 28 | func Error(format string, v ...interface{}) {fmt.Fprintf(os.Stderr,"[error] " + format, v...);} 29 | 30 | -------------------------------------------------------------------------------- /anscdn.cfg.example: -------------------------------------------------------------------------------- 1 | [default] 2 | base_server = example.com 3 | serving_port = 2009 4 | store_dir = ./data 5 | api_store_prefix = c 6 | strict = true 7 | cache_only = false 8 | file_mon = false 9 | cache_expires = 1296000 10 | clear_cache_path = /clearDataDir 11 | ignore_no_ext = true 12 | ignore_ext = html,txt 13 | provide_api = false 14 | url_map = /static 15 | api_key = 12345 16 | cdn_server_name = static4.example.com 17 | 18 | -------------------------------------------------------------------------------- /anscdn.go: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * AnsCDN Copyright (C) 2010 Robin Syihab (r [at] nosql.asia) 4 | * Simple CDN server written in Golang. 5 | * 6 | * License: General Public License v2 (GPLv2) 7 | * 8 | * Copyright (c) 2009 The Go Authors. All rights reserved. 9 | * 10 | **/ 11 | 12 | 13 | package main; 14 | 15 | 16 | import ( 17 | "strings" 18 | "strconv" 19 | "fmt" 20 | "http" 21 | "os" 22 | "path" 23 | "mime" 24 | "utf8" 25 | "flag" 26 | "./anlog" 27 | "./filemon" 28 | "./config" 29 | "./cdnize" 30 | "./downloader" 31 | ) 32 | 33 | const ( 34 | VERSION = "0.14" 35 | ) 36 | 37 | var cfg *config.AnscdnConf 38 | var quiet bool 39 | 40 | func file_exists(file_path string) bool{ 41 | file, err := os.Open(file_path) 42 | if err != nil { 43 | return false 44 | } 45 | file.Close() 46 | return true 47 | } 48 | 49 | // Heuristic: b is text if it is valid UTF-8 and doesn't 50 | // contain any unprintable ASCII or Unicode characters. 51 | func isText(b []byte) bool { 52 | for len(b) > 0 && utf8.FullRune(b) { 53 | rune, size := utf8.DecodeRune(b) 54 | if size == 1 && rune == utf8.RuneError { 55 | // decoding error 56 | return false 57 | } 58 | if 0x80 <= rune && rune <= 0x9F { 59 | return false 60 | } 61 | if rune < ' ' { 62 | switch rune { 63 | case '\n', '\r', '\t': 64 | // okay 65 | default: 66 | // binary garbage 67 | return false 68 | } 69 | } 70 | b = b[size:] 71 | } 72 | return true 73 | } 74 | 75 | func setHeaderCond(con http.ResponseWriter, abs_path string, data []byte) { 76 | extension := path.Ext(abs_path) 77 | if ctype := mime.TypeByExtension(extension); ctype != "" { 78 | con.Header().Set("Content-Type", ctype) 79 | }else{ 80 | if isText(data) { 81 | con.Header().Set("Content-Type", "text-plain; charset=utf-8") 82 | } else { 83 | con.Header().Set("Content-Type", "application/octet-stream") // generic binary 84 | } 85 | } 86 | } 87 | 88 | func validUrlPath(url_path string) bool{ 89 | return strings.Index(url_path,"../") < 1 90 | } 91 | 92 | func write(c http.ResponseWriter, f string, v ...interface{}){fmt.Fprintf(c,f,v...);} 93 | 94 | func MainHandler(con http.ResponseWriter, r *http.Request){ 95 | 96 | url_path := r.URL.Path[1:] 97 | 98 | if len(url_path) == 0{ 99 | http.Error(con,"404",http.StatusNotFound) 100 | return 101 | } 102 | 103 | // security check 104 | if !validUrlPath(url_path){ 105 | write(con,"Invalid url path") 106 | anlog.Warn("Invalid url_path: %s\n",url_path) 107 | return 108 | } 109 | 110 | if len(cfg.UrlMap) > 1 && strings.HasPrefix(url_path,cfg.UrlMap[1:]) == true{ 111 | url_path = url_path[len(cfg.UrlMap):] 112 | } 113 | 114 | // restrict no ext 115 | if cfg.IgnoreNoExt && len(path.Ext(url_path)) == 0 { 116 | anlog.Warn("Ignoring `%s`\n", url_path) 117 | http.Error(con, "404", http.StatusNotFound) 118 | return 119 | } 120 | 121 | // restrict ext 122 | if len(cfg.IgnoreExt) > 0 { 123 | cext := path.Ext(url_path) 124 | if len(cext) > 1{ 125 | cext = strings.ToLower(cext[1:]) 126 | exts := strings.Split(cfg.IgnoreExt,",") 127 | for _, ext := range exts{ 128 | if cext == strings.Trim(ext," ") { 129 | anlog.Warn("Ignoring `%s` by extension.\n", url_path) 130 | http.Error(con, "404", http.StatusNotFound) 131 | return 132 | } 133 | } 134 | } 135 | } 136 | 137 | var abs_path string 138 | 139 | if strings.HasPrefix(cfg.StoreDir,"./"){ 140 | 141 | abs_path, _ = os.Getwd() 142 | abs_path = path.Join(abs_path, cfg.StoreDir[1:], url_path) 143 | 144 | }else{ 145 | abs_path = path.Join(cfg.StoreDir,url_path) 146 | } 147 | 148 | dir_name, _ := path.Split(abs_path) 149 | 150 | if !file_exists(abs_path) { 151 | 152 | url_source := "http://" + cfg.BaseServer + "/" + url_path 153 | 154 | err := os.MkdirAll(dir_name,0755) 155 | if err != nil { 156 | fmt.Fprintf(con,"404 Not found (e)") 157 | anlog.Error("Cannot MkdirAll. error: %s\n",err.String()) 158 | return 159 | } 160 | 161 | // download it 162 | var data []byte 163 | rv, lm, total_size := downloader.Download(url_source, abs_path, cfg.Strict, &data) 164 | if rv == false{ 165 | fmt.Fprintf(con, "404 Not found") 166 | return 167 | } 168 | 169 | // send to client for the first time. 170 | setHeaderCond(con, abs_path, data) 171 | 172 | // set Last-modified header 173 | 174 | con.Header().Set("Last-Modified", lm) 175 | 176 | for { 177 | bw, err := con.Write(data) 178 | if err != nil || bw == 0 { 179 | break 180 | } 181 | if bw >= total_size { 182 | break 183 | } 184 | } 185 | 186 | }else{ 187 | 188 | if cfg.CacheOnly { 189 | // no static serving, use external server like nginx etc. 190 | return 191 | } 192 | 193 | // if file exists, just send it 194 | 195 | file, err := os.Open(abs_path) 196 | 197 | if err != nil{ 198 | fmt.Fprintf(con,"404 Not found (e)") 199 | anlog.Error("Cannot open file `%s`. error: %s\n", abs_path,err.String()) 200 | return 201 | } 202 | 203 | defer file.Close() 204 | 205 | bufsize := 1024*4 206 | buff := make([]byte,bufsize+2) 207 | 208 | sz, err := file.Read(buff) 209 | 210 | if err != nil && err != os.EOF { 211 | fmt.Fprintf(con,"404 Not found (e)") 212 | anlog.Error("Cannot read %d bytes data in file `%s`. error: %s\n", sz, abs_path,err.String()) 213 | return 214 | } 215 | 216 | setHeaderCond(con, abs_path, buff) 217 | 218 | // check for last-modified 219 | //r.Header["If-Modified-Since"] 220 | lm, _ := filemon.GetLastModif(file) 221 | con.Header().Set("Last-Modified", lm) 222 | 223 | if r.Header.Get("If-Modified-Since") == lm { 224 | con.WriteHeader(http.StatusNotModified) 225 | return 226 | } 227 | 228 | con.Write(buff[0:sz]) 229 | 230 | for { 231 | sz, err := file.Read(buff) 232 | if err != nil { 233 | if err == os.EOF { 234 | con.Write(buff[0:sz]) 235 | break 236 | } 237 | fmt.Fprintf(con,"404 Not found (e)") 238 | anlog.Error("Cannot read %d bytes data in file `%s`. error: %s\n", sz, abs_path,err.String()) 239 | return 240 | } 241 | con.Write(buff[0:sz]) 242 | } 243 | 244 | } 245 | 246 | } 247 | 248 | func ClearCacheHandler(c http.ResponseWriter, r *http.Request){ 249 | 250 | path_to_clear := r.FormValue("p") 251 | if len(path_to_clear) == 0{ 252 | write(c,"Invalid parameter") 253 | return 254 | } 255 | 256 | // prevent canonical path 257 | if strings.HasPrefix(path_to_clear,"."){ 258 | write(c,"Bad path") 259 | return 260 | } 261 | if path_to_clear[0] == '/'{ 262 | path_to_clear = path_to_clear[1:] 263 | } 264 | path_to_clear = "./data/" + path_to_clear 265 | 266 | f, err := os.Open(path_to_clear) 267 | if err != nil{ 268 | anlog.Error("File open error %s\n", err.String()) 269 | write(c,"Invalid request") 270 | return 271 | } 272 | defer f.Close() 273 | 274 | st, err := f.Stat() 275 | if err != nil{ 276 | anlog.Error("Cannot stat file. error %s\n", err.String()) 277 | write(c,"Invalid request") 278 | return 279 | } 280 | if !st.IsDirectory(){ 281 | write(c,"Invalid path") 282 | return 283 | } 284 | 285 | err = os.RemoveAll(path_to_clear) 286 | if err!=nil{ 287 | write(c,"Cannot clear path. e: %s", err.String()) 288 | return 289 | } 290 | 291 | store_dir := cfg.StoreDir 292 | 293 | if path_to_clear == store_dir + "/"{ 294 | if err := os.Mkdir(store_dir,0775); err != nil{ 295 | anlog.Error("Cannot recreate base store_dir: `%s`\n", store_dir) 296 | } 297 | } 298 | 299 | anlog.Info("Path cleared by request from `%s`: `%s`\n", r.Host, path_to_clear) 300 | write(c,"Clear successfully") 301 | } 302 | 303 | func intro(){ 304 | fmt.Println("\n AnsCDN " + VERSION + " - a Simple CDN Server") 305 | fmt.Println(" Copyright (C) 2010 Robin Syihab (r@nosql.asia)") 306 | fmt.Println(" Under GPLv2 License\n") 307 | } 308 | 309 | 310 | func main() { 311 | 312 | intro() 313 | 314 | var cfg_file string 315 | 316 | flag.StringVar(&cfg_file,"config","anscdn.cfg","Config file.") 317 | flag.BoolVar(&quiet,"quiet",false,"Quiet.") 318 | 319 | flag.Parse() 320 | 321 | anlog.Quiet = quiet 322 | 323 | var err os.Error 324 | cfg, err = config.Parse(cfg_file) 325 | cdnize.Cfg = cfg 326 | 327 | if err != nil { 328 | fmt.Println("Invalid configuration. e: ",err.String(),"\n") 329 | os.Exit(1) 330 | } 331 | 332 | if len(cfg.BaseServer) == 0{ 333 | anlog.Error("No base server") 334 | os.Exit(3) 335 | } 336 | if cfg.ServingPort == 0{ 337 | anlog.Error("No port") 338 | os.Exit(4) 339 | } 340 | if len(cfg.StoreDir) == 0{ 341 | cfg.StoreDir = "./data" 342 | } 343 | if cfg.CacheExpires == 0{ 344 | cfg.CacheExpires = 1296000 345 | } 346 | 347 | fmt.Println("Configuration:") 348 | fmt.Println("---------------------------------------") 349 | 350 | fmt.Println("Base server: " + cfg.BaseServer) 351 | if cfg.Strict == true { 352 | fmt.Println("Strict mode ON") 353 | }else{ 354 | fmt.Println("Strict mode OFF") 355 | } 356 | if cfg.CacheOnly == true { 357 | fmt.Println("Cache only") 358 | } 359 | if cfg.IgnoreNoExt==true{fmt.Println("Ignore no extension files");} 360 | if len(cfg.IgnoreExt)>0{fmt.Println("Ignore extension for", cfg.IgnoreExt);} 361 | if len(cfg.ClearCachePath) > 0{ 362 | fmt.Println("Clear cache path: ", cfg.ClearCachePath) 363 | } 364 | 365 | fmt.Printf("Store cached data in `%s`\n", cfg.StoreDir) 366 | 367 | if cfg.FileMon == true { 368 | fmt.Println("File monitor enabled") 369 | if err != nil{ 370 | anlog.Error("Invalid cache_expires value `%d`\n", cfg.CacheExpires) 371 | os.Exit(5) 372 | } 373 | go filemon.StartFileMon(cfg.StoreDir, cfg.CacheExpires) 374 | } 375 | 376 | current_dir, _ := path.Split(os.Args[0]) 377 | os.Chdir(current_dir) 378 | current_dir, err = os.Getwd() 379 | if err != nil{ 380 | anlog.Error("Cannot get current_directory\n") 381 | os.Exit(6) 382 | } 383 | anlog.Info("Current directory: %v\n", current_dir) 384 | fi, err := os.Lstat(current_dir + cfg.StoreDir[1:]) 385 | if err != nil || fi.IsDirectory() == false{ 386 | err = os.Mkdir(current_dir + cfg.StoreDir[1:], 0755) 387 | if err != nil{ 388 | anlog.Error("Cannot create dir `%s`. %s.\n", err) 389 | os.Exit(8) 390 | } 391 | } 392 | 393 | fmt.Println("---------------------------------------\n") 394 | 395 | anlog.Info("Serving on 0.0.0.0:%d... ready.\n", cfg.ServingPort ) 396 | 397 | 398 | if len(cfg.ClearCachePath) > 0 { 399 | if cfg.ClearCachePath[0] != '/'{ 400 | anlog.Error("Invalid ccp `%s`. missing `/`\n",cfg.ClearCachePath) 401 | os.Exit(2) 402 | } 403 | http.Handle(cfg.ClearCachePath, http.HandlerFunc(ClearCacheHandler)) 404 | } 405 | if cfg.ProvideApi == true { 406 | http.Handle("/api/cdnize", http.HandlerFunc(cdnize.Handler)) 407 | 408 | fi, err := os.Lstat(current_dir + cfg.StoreDir[1:] + "/" + cfg.ApiStorePrefix) 409 | if err != nil || fi.IsDirectory() == false{ 410 | err = os.Mkdir(current_dir + cfg.StoreDir[1:] + "/" + cfg.ApiStorePrefix, 0755) 411 | if err != nil{ 412 | anlog.Error("Cannot create dir `%s`. %s.\n", err) 413 | os.Exit(8) 414 | } 415 | } 416 | 417 | http.Handle(fmt.Sprintf("/%s/", cfg.StoreDir[2:]), http.HandlerFunc(cdnize.StaticHandler)) 418 | } 419 | http.Handle("/", http.HandlerFunc(MainHandler)) 420 | if err := http.ListenAndServe("0.0.0.0:" + strconv.Itoa(cfg.ServingPort), nil); err != nil { 421 | anlog.Error("%s\n",err.String()) 422 | } 423 | } 424 | 425 | -------------------------------------------------------------------------------- /anscdn.tmproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | currentDocument 6 | Makefile 7 | documents 8 | 9 | 10 | filename 11 | cdnize.go 12 | lastUsed 13 | 2011-04-25T16:22:03Z 14 | 15 | 16 | filename 17 | downloader.go 18 | lastUsed 19 | 2011-04-25T16:42:01Z 20 | 21 | 22 | filename 23 | cdnize_test.go 24 | lastUsed 25 | 2011-04-25T16:34:52Z 26 | 27 | 28 | filename 29 | anscdn.go 30 | lastUsed 31 | 2011-04-25T16:20:14Z 32 | 33 | 34 | filename 35 | anlog.go 36 | lastUsed 37 | 2011-04-25T16:41:41Z 38 | 39 | 40 | filename 41 | config.go 42 | lastUsed 43 | 2011-04-25T15:00:40Z 44 | 45 | 46 | filename 47 | configfile.go 48 | lastUsed 49 | 2011-04-25T15:31:55Z 50 | 51 | 52 | filename 53 | filemon.go 54 | lastUsed 55 | 2011-04-25T15:32:26Z 56 | 57 | 58 | filename 59 | Makefile 60 | lastUsed 61 | 2011-04-25T16:42:01Z 62 | selected 63 | 64 | 65 | 66 | filename 67 | Makefile.i386 68 | lastUsed 69 | 2011-04-25T16:41:41Z 70 | 71 | 72 | filename 73 | README.md 74 | lastUsed 75 | 2011-04-25T16:41:41Z 76 | 77 | 78 | filename 79 | anscdn.cfg.example 80 | lastUsed 81 | 2011-04-25T16:02:44Z 82 | 83 | 84 | filename 85 | utils.go 86 | lastUsed 87 | 2011-04-25T16:02:44Z 88 | 89 | 90 | fileHierarchyDrawerWidth 91 | 200 92 | metaData 93 | 94 | Makefile 95 | 96 | caret 97 | 98 | column 99 | 24 100 | line 101 | 21 102 | 103 | firstVisibleColumn 104 | 0 105 | firstVisibleLine 106 | 0 107 | 108 | README.md 109 | 110 | caret 111 | 112 | column 113 | 15 114 | line 115 | 9 116 | 117 | firstVisibleColumn 118 | 0 119 | firstVisibleLine 120 | 0 121 | 122 | anlog.go 123 | 124 | caret 125 | 126 | column 127 | 3 128 | line 129 | 9 130 | 131 | firstVisibleColumn 132 | 0 133 | firstVisibleLine 134 | 0 135 | 136 | anscdn.cfg.example 137 | 138 | caret 139 | 140 | column 141 | 22 142 | line 143 | 8 144 | 145 | firstVisibleColumn 146 | 0 147 | firstVisibleLine 148 | 0 149 | 150 | anscdn.go 151 | 152 | caret 153 | 154 | column 155 | 16 156 | line 157 | 33 158 | 159 | firstVisibleColumn 160 | 0 161 | firstVisibleLine 162 | 0 163 | 164 | cdnize.go 165 | 166 | caret 167 | 168 | column 169 | 12 170 | line 171 | 32 172 | 173 | columnSelection 174 | 175 | firstVisibleColumn 176 | 0 177 | firstVisibleLine 178 | 20 179 | selectFrom 180 | 181 | column 182 | 5 183 | line 184 | 32 185 | 186 | selectTo 187 | 188 | column 189 | 16 190 | line 191 | 32 192 | 193 | 194 | cdnize_test.go 195 | 196 | caret 197 | 198 | column 199 | 0 200 | line 201 | 15 202 | 203 | firstVisibleColumn 204 | 0 205 | firstVisibleLine 206 | 0 207 | 208 | config.go 209 | 210 | caret 211 | 212 | column 213 | 3 214 | line 215 | 9 216 | 217 | firstVisibleColumn 218 | 0 219 | firstVisibleLine 220 | 0 221 | 222 | configfile.go 223 | 224 | caret 225 | 226 | column 227 | 0 228 | line 229 | 40 230 | 231 | firstVisibleColumn 232 | 0 233 | firstVisibleLine 234 | 21 235 | 236 | downloader.go 237 | 238 | caret 239 | 240 | column 241 | 0 242 | line 243 | 20 244 | 245 | firstVisibleColumn 246 | 0 247 | firstVisibleLine 248 | 0 249 | 250 | filemon.go 251 | 252 | caret 253 | 254 | column 255 | 19 256 | line 257 | 88 258 | 259 | firstVisibleColumn 260 | 0 261 | firstVisibleLine 262 | 69 263 | 264 | utils.go 265 | 266 | caret 267 | 268 | column 269 | 3 270 | line 271 | 9 272 | 273 | firstVisibleColumn 274 | 0 275 | firstVisibleLine 276 | 0 277 | 278 | 279 | openDocuments 280 | 281 | config.go 282 | configfile.go 283 | filemon.go 284 | Makefile 285 | README.md 286 | anlog.go 287 | downloader.go 288 | anscdn.go 289 | cdnize.go 290 | cdnize_test.go 291 | anscdn.cfg.example 292 | utils.go 293 | 294 | showFileHierarchyDrawer 295 | 296 | windowFrame 297 | {{203, 44}, {1057, 734}} 298 | 299 | 300 | -------------------------------------------------------------------------------- /cdnize.go: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * AnsCDN Copyright (C) 2010 Robin Syihab (r [at] nosql.asia) 4 | * Simple CDN server written in Golang. 5 | * 6 | * License: General Public License v2 (GPLv2) 7 | * 8 | * Copyright (c) 2009 The Go Authors. All rights reserved. 9 | * 10 | **/ 11 | 12 | package cdnize 13 | 14 | import ( 15 | "fmt" 16 | "http" 17 | "rand" 18 | "time" 19 | "path" 20 | "crypto/hmac" 21 | "os" 22 | "syscall" 23 | "json" 24 | "./anlog" 25 | "./config" 26 | "./downloader" 27 | ) 28 | 29 | var Cfg *config.AnscdnConf 30 | 31 | const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ123567890abcdefghijklmnopqrstuvwxyz_"; 32 | 33 | func RandStrings(N int) string { 34 | rand.Seed(time.Nanoseconds()) 35 | 36 | buf := make([]byte, N + 1) 37 | 38 | for i := 0; i < N; i++ { 39 | buf[i] = chars[rand.Intn(len(chars))] 40 | } 41 | return string(buf[0:N]) 42 | } 43 | 44 | func write(c http.ResponseWriter, f string, v ...interface{}){fmt.Fprintf(c,f,v...);} 45 | 46 | func Jsonize(data interface{}) string{ 47 | rv, err := json.Marshal(&data) 48 | if err != nil{ 49 | anlog.Error("Cannot jsonize `%v`\n", data) 50 | return "" 51 | } 52 | return string(rv) 53 | } 54 | 55 | func jsonError(status, info string) string{ 56 | return Jsonize(map[string]string{"Status":status,"Info":info}) 57 | } 58 | 59 | func Handler(c http.ResponseWriter, r *http.Request){ 60 | 61 | c.Header().Set("Content-Type", "application/json") 62 | 63 | api_key := r.FormValue("api_key") 64 | //base_url := r.FormValue("base_url") 65 | 66 | if api_key != Cfg.ApiKey{ 67 | write(c,jsonError("failed","Invalid api key")) 68 | return 69 | } 70 | 71 | requested_url := r.FormValue("u") 72 | if requested_url == ""{ 73 | 74 | r.ParseForm() 75 | file_name := r.FormValue("file_name") 76 | 77 | if file_name == ""{ 78 | write(c,jsonError("failed","no `file_name` parameter")) 79 | return 80 | } 81 | 82 | fmt.Printf("file_name: %v\n", file_name) 83 | 84 | file, err := r.MultipartReader() 85 | if err != nil{ 86 | write(c,jsonError("failed","cannot get multipart reader")) 87 | return 88 | } 89 | 90 | part, err := file.NextPart() 91 | if err != nil{ 92 | write(c,jsonError("failed","no `u` nor `file`")) 93 | return 94 | } 95 | var data [1000]byte 96 | var i int = 0 97 | var data_size int64 = 0 98 | md5ed := hmac.NewMD5([]byte("cdnized-2194")) 99 | abs_path := "/tmp/" + RandStrings(100) 100 | dst_file, err := os.OpenFile(abs_path,os.O_WRONLY | os.O_CREATE,0755) 101 | if err != nil { 102 | anlog.Error("Cannot create file `%s`. error: %s\n", abs_path,err.String()) 103 | write(c,jsonError("failed",fmt.Sprintf("cannot create temporary data. %v\n",err))) 104 | return 105 | } 106 | 107 | for data_size < r.ContentLength{ 108 | i, err = part.Read(data[0:999]) 109 | if err !=nil{ 110 | break 111 | } 112 | 113 | _, err := md5ed.Write(data[0:i]) 114 | if err != nil{ 115 | anlog.Error("Cannot calculate MD5 hash") 116 | write(c,jsonError("failed","cannot calculate checksum")) 117 | break 118 | } 119 | 120 | _, err = dst_file.Write(data[0:i]) 121 | if err != nil{ 122 | anlog.Error("Cannot write %d bytes data in file `%s`. error: %s\n", data_size, abs_path, err.String()) 123 | } 124 | 125 | data_size += int64(i) 126 | } 127 | 128 | dst_file.Close() 129 | 130 | //fmt.Printf("content-length: %v, file: %v, file-length: %v, i: %v\n", r.ContentLength, string(data[0:]), i, i) 131 | 132 | hash := fmt.Sprintf("%x", md5ed.Sum()) 133 | file_ext := path.Ext(file_name) 134 | file_name = hash + RandStrings(9) + file_ext 135 | new_path, err := os.Getwd() 136 | 137 | new_path = path.Join(new_path, Cfg.StoreDir[2:], Cfg.ApiStorePrefix, file_name) 138 | 139 | if err != nil { 140 | anlog.Error("Cannot getwd\n") 141 | write(c,jsonError("failed","internal error")) 142 | return 143 | } 144 | 145 | //fmt.Printf("abs_path: %v, new_path: %v\n", abs_path, new_path) 146 | if err := syscall.Rename(abs_path, new_path); err != 0{ 147 | anlog.Error("Cannot move from file `%s` to `%s`. %v.\n", abs_path, new_path, err) 148 | write(c,jsonError("failed","internal error")) 149 | return 150 | } 151 | 152 | cdnized_url := fmt.Sprintf("http://%s/%s/%s/%s", Cfg.CdnServerName, Cfg.StoreDir[2:], Cfg.ApiStorePrefix, file_name) 153 | 154 | anlog.Info("cdnized_url: %s\n", cdnized_url) 155 | 156 | os.Remove(abs_path) 157 | 158 | 159 | type success struct{ 160 | Status string 161 | Size int64 162 | Cdnized_url string 163 | } 164 | 165 | write(c, Jsonize(&success{"ok", data_size, cdnized_url})) 166 | return 167 | } 168 | 169 | 170 | 171 | //write(c, fmt.Sprintf("{Status: 'ok', url_path: '%s', gen: '%s'}", requested_url, x)) 172 | 173 | file_ext := path.Ext(requested_url) 174 | abs_path, _ := os.Getwd() 175 | abs_path = path.Join(abs_path, Cfg.StoreDir[2:], Cfg.ApiStorePrefix, RandStrings(64) + file_ext) 176 | 177 | fmt.Printf("abs_path: %s\n", abs_path) 178 | 179 | var data []byte; 180 | rv, lm, tsize := downloader.Download(requested_url, abs_path, true, &data) 181 | if rv != true{ 182 | write(c,jsonError("failed","Cannot fetch from source url")) 183 | return 184 | } 185 | 186 | md5ed := hmac.NewMD5([]byte("cdnized-2194")) 187 | for { 188 | brw, err := md5ed.Write(data) 189 | if err != nil{ 190 | anlog.Error("Cannot calculate MD5 hash") 191 | write(c,jsonError("failed","Internal error")) 192 | return 193 | } 194 | if brw >= tsize{ 195 | break; 196 | } 197 | } 198 | 199 | hash := fmt.Sprintf("%x", md5ed.Sum()) 200 | dir, _ := path.Split(abs_path) 201 | file_name := hash + RandStrings(8) + file_ext 202 | new_path := path.Join(dir, file_name) 203 | 204 | if err := syscall.Rename(abs_path, new_path); err != 0{ 205 | anlog.Error("Cannot rename from file `%s` to `%s`", abs_path, new_path) 206 | write(c,jsonError("failed","Internal error")) 207 | return 208 | } 209 | 210 | cdnized_url := fmt.Sprintf("http://%s/%s/%s/%s", Cfg.CdnServerName, Cfg.StoreDir[2:], Cfg.ApiStorePrefix, file_name) 211 | 212 | anlog.Info("cdnized_url: %s", cdnized_url) 213 | 214 | type success struct{ 215 | Status string 216 | Lm string 217 | Size int 218 | Original string 219 | Cdnized_url string 220 | } 221 | 222 | write(c, Jsonize(&success{"ok", lm, tsize, requested_url, cdnized_url})) 223 | } 224 | 225 | func StaticHandler(c http.ResponseWriter, r *http.Request){ 226 | path := r.URL.Path 227 | root, _ := os.Getwd() 228 | http.ServeFile(c, r, root + "/" + path) 229 | } 230 | 231 | 232 | -------------------------------------------------------------------------------- /cdnize_test.go: -------------------------------------------------------------------------------- 1 | package "cdnize" 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestRandString(t *testing.T) { 8 | rv := RandStrings(5) 9 | testStringNeq( t, "test RandString Not equal `abc`...", rv, "abc") 10 | if len(rv) != 5 { 11 | t.Errorf("Len should be 5 got `%d`", len(rv)) 12 | } 13 | rv2 := RandStrings(6) 14 | testStringNeq( t, "test RandString not equal `" + rv2 + "`", rv, rv2) 15 | } 16 | 17 | func testStringEq(t *testing.T, msg, actual, expected string) { 18 | if actual != expected { 19 | t.Errorf("%s: `%s` != `%s`", msg, actual, expected); 20 | } 21 | } 22 | 23 | func testStringNeq(t *testing.T, msg, actual, not_expected string) { 24 | if actual == not_expected { 25 | t.Errorf("%s: `%s` == `%s`", msg, actual, not_expected) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * AnsCDN Copyright (C) 2010 Robin Syihab (r [at] nosql.asia) 4 | * Simple CDN server written in Golang. 5 | * 6 | * License: General Public License v2 (GPLv2) 7 | * 8 | * Copyright (c) 2009 The Go Authors. All rights reserved. 9 | * 10 | **/ 11 | 12 | package config 13 | 14 | 15 | import ( 16 | "os" 17 | "./configfile" 18 | ) 19 | 20 | type AnscdnConf struct { 21 | BaseServer string 22 | ServingPort int 23 | StoreDir string 24 | ApiStorePrefix string 25 | Strict bool 26 | CacheOnly bool 27 | FileMon bool 28 | CacheExpires int64 29 | ClearCachePath string 30 | IgnoreNoExt bool 31 | IgnoreExt string 32 | ProvideApi bool 33 | ApiKey string 34 | CdnServerName string 35 | UrlMap string 36 | } 37 | 38 | func Parse(file string) (ac *AnscdnConf, err os.Error) { 39 | 40 | conf, err := configfile.ReadConfigFile(file) 41 | 42 | if err != nil{ 43 | return nil, err 44 | } 45 | 46 | BaseServer, err := conf.GetString("default","base_server") 47 | ServingPort, err := conf.GetInt("default","serving_port") 48 | StoreDir, err := conf.GetString("default","store_dir") 49 | Strict, err := conf.GetBool("default","strict") 50 | CacheOnly, err := conf.GetBool("default","cache_only") 51 | FileMon, err := conf.GetBool("default","file_mon") 52 | CacheExpires, err := conf.GetInt64("default","cache_expires") 53 | ClearCachePath, err := conf.GetString("default","clear_cache_path") 54 | IgnoreNoExt, err := conf.GetBool("default","ignore_no_ext") 55 | IgnoreExt, err := conf.GetString("default","ignore_ext") 56 | ProvideApi, err := conf.GetBool("default","provide_api") 57 | ApiKey, err := conf.GetString("default","api_key") 58 | CdnServerName, err := conf.GetString("default", "cdn_server_name") 59 | UrlMap, err := conf.GetString("default", "url_map") 60 | ApiStorePrefix, err := conf.GetString("default", "api_store_prefix") 61 | 62 | return &AnscdnConf{BaseServer, 63 | ServingPort, StoreDir, ApiStorePrefix, Strict,CacheOnly,FileMon,CacheExpires, 64 | ClearCachePath,IgnoreNoExt,IgnoreExt, ProvideApi,ApiKey,CdnServerName,UrlMap}, err 65 | } 66 | -------------------------------------------------------------------------------- /configfile.go: -------------------------------------------------------------------------------- 1 | // This package implements a parser for configuration files. 2 | // This allows easy reading and writing of structured configuration files. 3 | // 4 | // Given a sample configuration file: 5 | // 6 | // [default] 7 | // host=www.example.com 8 | // protocol=http:// 9 | // base-url=%(protocol)s%(host)s 10 | // 11 | // [service-1] 12 | // url=%(base-url)s/some/path 13 | // delegation : on 14 | // maxclients=200 # do not set this higher 15 | // comments=This is a multi-line 16 | // entry ; And this is a comment 17 | // 18 | // To read this configuration file, do: 19 | // 20 | // c, err := configfile.ReadConfigFile("config.cfg"); 21 | // c.GetString("service-1", "url"); // result is string :http://www.example.com/some/path" 22 | // c.GetInt("service-1", "maxclients"); // result is int 200 23 | // c.GetBool("service-1", "delegation"); // result is bool true 24 | // c.GetString("service-1", "comments"); // result is string "This is a multi-line\nentry" 25 | // 26 | // Note the support for unfolding variables (such as %(base-url)s), which are read from the special 27 | // (reserved) section name [default]. 28 | // 29 | // A new configuration file can also be created with: 30 | // 31 | // c := configfile.NewConfigFile(); 32 | // c.AddSection("section"); 33 | // c.AddOption("section", "option", "value"); 34 | // c.WriteConfigFile("config.cfg", 0644, "A header for this file"); // use 0644 as file permission 35 | // 36 | // This results in the file: 37 | // 38 | // # A header for this file 39 | // [section] 40 | // option=value 41 | // 42 | // Note that sections and options are case-insensitive (values are case-sensitive) 43 | // and are converted to lowercase when saved to a file. 44 | // 45 | // The functionality and workflow is loosely based on the configparser.py package 46 | // of the Python Standard Library. 47 | package configfile 48 | 49 | 50 | import ( 51 | "bufio" 52 | "os" 53 | "regexp" 54 | "strconv" 55 | "strings" 56 | ) 57 | 58 | 59 | // ConfigFile is the representation of configuration settings. 60 | // The public interface is entirely through methods. 61 | type ConfigFile struct { 62 | data map[string]map[string]string; // Maps sections to options to values. 63 | } 64 | 65 | 66 | var ( 67 | DefaultSection = "default"; // Default section name (must be lower-case). 68 | DepthValues = 200; // Maximum allowed depth when recursively substituing variable names. 69 | 70 | // Strings accepted as bool. 71 | BoolStrings = map[string]bool{ 72 | "t": true, 73 | "true": true, 74 | "y": true, 75 | "yes": true, 76 | "on": true, 77 | "1": true, 78 | "f": false, 79 | "false": false, 80 | "n": false, 81 | "no": false, 82 | "off": false, 83 | "0": false, 84 | }; 85 | 86 | varRegExp = regexp.MustCompile(`%\(([a-zA-Z0-9_.\-]+)\)s`); 87 | ) 88 | 89 | 90 | // AddSection adds a new section to the configuration. 91 | // It returns true if the new section was inserted, and false if the section already existed. 92 | func (c *ConfigFile) AddSection(section string) bool { 93 | section = strings.ToLower(section); 94 | 95 | if _, ok := c.data[section]; ok { 96 | return false 97 | } 98 | c.data[section] = make(map[string]string); 99 | 100 | return true; 101 | } 102 | 103 | 104 | // RemoveSection removes a section from the configuration. 105 | // It returns true if the section was removed, and false if section did not exist. 106 | func (c *ConfigFile) RemoveSection(section string) bool { 107 | section = strings.ToLower(section); 108 | 109 | switch _, ok := c.data[section]; { 110 | case !ok: 111 | return false 112 | case section == DefaultSection: 113 | return false // default section cannot be removed 114 | default: 115 | for o, _ := range c.data[section] { 116 | c.data[section][o] = "", false 117 | } 118 | c.data[section] = nil, false; 119 | } 120 | 121 | return true; 122 | } 123 | 124 | 125 | // AddOption adds a new option and value to the configuration. 126 | // It returns true if the option and value were inserted, and false if the value was overwritten. 127 | // If the section does not exist in advance, it is created. 128 | func (c *ConfigFile) AddOption(section string, option string, value string) bool { 129 | c.AddSection(section); // make sure section exists 130 | 131 | section = strings.ToLower(section); 132 | option = strings.ToLower(option); 133 | 134 | _, ok := c.data[section][option]; 135 | c.data[section][option] = value; 136 | 137 | return !ok; 138 | } 139 | 140 | 141 | // RemoveOption removes a option and value from the configuration. 142 | // It returns true if the option and value were removed, and false otherwise, 143 | // including if the section did not exist. 144 | func (c *ConfigFile) RemoveOption(section string, option string) bool { 145 | section = strings.ToLower(section); 146 | option = strings.ToLower(option); 147 | 148 | if _, ok := c.data[section]; !ok { 149 | return false 150 | } 151 | 152 | _, ok := c.data[section][option]; 153 | c.data[section][option] = "", false; 154 | 155 | return ok; 156 | } 157 | 158 | 159 | // NewConfigFile creates an empty configuration representation. 160 | // This representation can be filled with AddSection and AddOption and then 161 | // saved to a file using WriteConfigFile. 162 | func NewConfigFile() *ConfigFile { 163 | c := new(ConfigFile); 164 | c.data = make(map[string]map[string]string); 165 | 166 | c.AddSection(DefaultSection); // default section always exists 167 | 168 | return c; 169 | } 170 | 171 | 172 | func stripComments(l string) string { 173 | // comments are preceded by space or TAB 174 | for _, c := range []string{" ;", "\t;", " #", "\t#"} { 175 | if i := strings.Index(l, c); i != -1 { 176 | l = l[0:i] 177 | } 178 | } 179 | return l; 180 | } 181 | 182 | 183 | func firstIndex(s string, delim []byte) int { 184 | for i := 0; i < len(s); i++ { 185 | for j := 0; j < len(delim); j++ { 186 | if s[i] == delim[j] { 187 | return i 188 | } 189 | } 190 | } 191 | return -1; 192 | } 193 | 194 | 195 | func (c *ConfigFile) read(buf *bufio.Reader) (err os.Error) { 196 | var section, option string; 197 | for { 198 | l, err := buf.ReadString('\n'); // parse line-by-line 199 | if err == os.EOF { 200 | break 201 | } else if err != nil { 202 | return err 203 | } 204 | 205 | l = strings.TrimSpace(l); 206 | // switch written for readability (not performance) 207 | switch { 208 | case len(l) == 0: // empty line 209 | continue 210 | 211 | case l[0] == '#': // comment 212 | continue 213 | 214 | case l[0] == ';': // comment 215 | continue 216 | 217 | case len(l) >= 3 && strings.ToLower(l[0:3]) == "rem": // comment (for windows users) 218 | continue 219 | 220 | case l[0] == '[' && l[len(l)-1] == ']': // new section 221 | option = ""; // reset multi-line value 222 | section = strings.TrimSpace(l[1 : len(l)-1]); 223 | c.AddSection(section); 224 | 225 | case section == "": // not new section and no section defined so far 226 | return os.NewError("section not found: must start with section") 227 | 228 | default: // other alternatives 229 | i := firstIndex(l, []byte{'=', ':'}); 230 | switch { 231 | case i > 0: // option and value 232 | i := firstIndex(l, []byte{'=', ':'}); 233 | option = strings.TrimSpace(l[0:i]); 234 | value := strings.TrimSpace(stripComments(l[i+1:])); 235 | c.AddOption(section, option, value); 236 | 237 | case section != "" && option != "": // continuation of multi-line value 238 | prev, _ := c.GetRawString(section, option); 239 | value := strings.TrimSpace(stripComments(l)); 240 | c.AddOption(section, option, prev+"\n"+value); 241 | 242 | default: 243 | return os.NewError("could not parse line: " + l) 244 | } 245 | } 246 | } 247 | return nil; 248 | } 249 | 250 | 251 | // ReadConfigFile reads a file and returns a new configuration representation. 252 | // This representation can be queried with GetString, etc. 253 | func ReadConfigFile(fname string) (c *ConfigFile, err os.Error) { 254 | var file *os.File; 255 | 256 | if file, err = os.Open(fname); err != nil { 257 | return nil, err 258 | } 259 | 260 | c = NewConfigFile(); 261 | if err = c.read(bufio.NewReader(file)); err != nil { 262 | return nil, err 263 | } 264 | 265 | if err = file.Close(); err != nil { 266 | return nil, err 267 | } 268 | 269 | return c, nil; 270 | } 271 | 272 | 273 | func (c *ConfigFile) write(buf *bufio.Writer, header string) (err os.Error) { 274 | if header != "" { 275 | if _, err = buf.WriteString("# " + header + "\n"); err != nil { 276 | return err 277 | } 278 | } 279 | 280 | for section, sectionmap := range c.data { 281 | if section == DefaultSection && len(sectionmap) == 0 { 282 | continue // skip default section if empty 283 | } 284 | if _, err = buf.WriteString("[" + section + "]\n"); err != nil { 285 | return err 286 | } 287 | for option, value := range sectionmap { 288 | if _, err = buf.WriteString(option + "=" + value + "\n"); err != nil { 289 | return err 290 | } 291 | } 292 | if _, err = buf.WriteString("\n"); err != nil { 293 | return err 294 | } 295 | } 296 | 297 | return nil; 298 | } 299 | 300 | 301 | // WriteConfigFile saves the configuration representation to a file. 302 | // The desired file permissions must be passed as in os.Open. 303 | // The header is a string that is saved as a comment in the first line of the file. 304 | func (c *ConfigFile) WriteConfigFile(fname string, perm uint32, header string) (err os.Error) { 305 | var file *os.File; 306 | 307 | if file, err = os.OpenFile(fname, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm); err != nil { 308 | return err 309 | } 310 | 311 | buf := bufio.NewWriter(file); 312 | if err = c.write(buf, header); err != nil { 313 | return err 314 | } 315 | buf.Flush(); 316 | 317 | return file.Close(); 318 | } 319 | 320 | 321 | // GetSections returns the list of sections in the configuration. 322 | // (The default section always exists.) 323 | func (c *ConfigFile) GetSections() (sections []string) { 324 | sections = make([]string, len(c.data)); 325 | 326 | i := 0; 327 | for s, _ := range c.data { 328 | sections[i] = s; 329 | i++; 330 | } 331 | 332 | return sections; 333 | } 334 | 335 | 336 | // HasSection checks if the configuration has the given section. 337 | // (The default section always exists.) 338 | func (c *ConfigFile) HasSection(section string) bool { 339 | _, ok := c.data[strings.ToLower(section)]; 340 | 341 | return ok; 342 | } 343 | 344 | 345 | // GetOptions returns the list of options available in the given section. 346 | // It returns an error if the section does not exist and an empty list if the section is empty. 347 | // Options within the default section are also included. 348 | func (c *ConfigFile) GetOptions(section string) (options []string, err os.Error) { 349 | section = strings.ToLower(section); 350 | 351 | if _, ok := c.data[section]; !ok { 352 | return nil, os.NewError("section not found") 353 | } 354 | 355 | options = make([]string, len(c.data[DefaultSection])+len(c.data[section])); 356 | i := 0; 357 | for s, _ := range c.data[DefaultSection] { 358 | options[i] = s; 359 | i++; 360 | } 361 | for s, _ := range c.data[section] { 362 | options[i] = s; 363 | i++; 364 | } 365 | 366 | return options, nil; 367 | } 368 | 369 | 370 | // HasOption checks if the configuration has the given option in the section. 371 | // It returns false if either the option or section do not exist. 372 | func (c *ConfigFile) HasOption(section string, option string) bool { 373 | section = strings.ToLower(section); 374 | option = strings.ToLower(option); 375 | 376 | if _, ok := c.data[section]; !ok { 377 | return false 378 | } 379 | 380 | _, okd := c.data[DefaultSection][option]; 381 | _, oknd := c.data[section][option]; 382 | 383 | return okd || oknd; 384 | } 385 | 386 | 387 | // GetRawString gets the (raw) string value for the given option in the section. 388 | // The raw string value is not subjected to unfolding, which was illustrated in the beginning of this documentation. 389 | // It returns an error if either the section or the option do not exist. 390 | func (c *ConfigFile) GetRawString(section string, option string) (value string, err os.Error) { 391 | section = strings.ToLower(section); 392 | option = strings.ToLower(option); 393 | 394 | if _, ok := c.data[section]; ok { 395 | if value, ok = c.data[section][option]; ok { 396 | return value, nil 397 | } 398 | return "", os.NewError("option not found"); 399 | } 400 | return "", os.NewError("section not found"); 401 | } 402 | 403 | 404 | // GetString gets the string value for the given option in the section. 405 | // If the value needs to be unfolded (see e.g. %(host)s example in the beginning of this documentation), 406 | // then GetString does this unfolding automatically, up to DepthValues number of iterations. 407 | // It returns an error if either the section or the option do not exist, or the unfolding cycled. 408 | func (c *ConfigFile) GetString(section string, option string) (value string, err os.Error) { 409 | value, err = c.GetRawString(section, option); 410 | if err != nil { 411 | return "", err 412 | } 413 | 414 | section = strings.ToLower(section); 415 | 416 | var i int; 417 | 418 | for i = 0; i < DepthValues; i++ { // keep a sane depth 419 | vr := varRegExp.FindString(value); 420 | if len(vr) == 0 { 421 | break 422 | } 423 | 424 | noption := value[vr[2]:vr[3]]; 425 | noption = strings.ToLower(noption); 426 | 427 | nvalue, _ := c.data[DefaultSection][noption]; // search variable in default section 428 | if _, ok := c.data[section][noption]; ok { 429 | nvalue = c.data[section][noption] 430 | } 431 | if nvalue == "" { 432 | return "", os.NewError("option not found: " + noption) 433 | } 434 | 435 | // substitute by new value and take off leading '%(' and trailing ')s' 436 | value = value[0:vr[2]-2] + nvalue + value[vr[3]+2:]; 437 | } 438 | 439 | if i == DepthValues { 440 | return "", os.NewError("possible cycle while unfolding variables: max depth of " + strconv.Itoa(DepthValues) + " reached") 441 | } 442 | 443 | return value, nil; 444 | } 445 | 446 | // GetInt has the same behaviour as GetString but converts the response to int. 447 | func (c *ConfigFile) GetInt64(section string, option string) (value int64, err os.Error) { 448 | sv, err := c.GetString(section, option); 449 | if err == nil { 450 | value, err = strconv.Atoi64(sv) 451 | } 452 | 453 | return value, err; 454 | } 455 | 456 | // GetInt has the same behaviour as GetString but converts the response to int. 457 | func (c *ConfigFile) GetInt(section string, option string) (value int, err os.Error) { 458 | sv, err := c.GetString(section, option); 459 | if err == nil { 460 | value, err = strconv.Atoi(sv) 461 | } 462 | 463 | return value, err; 464 | } 465 | 466 | 467 | // GetFloat has the same behaviour as GetString but converts the response to float. 468 | func (c *ConfigFile) GetFloat(section string, option string) (value float64, err os.Error) { 469 | sv, err := c.GetString(section, option); 470 | if err == nil { 471 | value, err = strconv.Atof64(sv) 472 | } 473 | 474 | return value, err; 475 | } 476 | 477 | 478 | // GetBool has the same behaviour as GetString but converts the response to bool. 479 | // See constant BoolStrings for string values converted to bool. 480 | func (c *ConfigFile) GetBool(section string, option string) (value bool, err os.Error) { 481 | sv, err := c.GetString(section, option); 482 | if err != nil { 483 | return false, err 484 | } 485 | 486 | value, ok := BoolStrings[strings.ToLower(sv)]; 487 | if !ok { 488 | return false, os.NewError("could not parse bool value: " + sv) 489 | } 490 | 491 | return value, nil; 492 | } 493 | -------------------------------------------------------------------------------- /downloader.go: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * AnsCDN Copyright (C) 2010 Robin Syihab (r [at] nosql.asia) 4 | * Simple CDN server written in Golang. 5 | * 6 | * License: General Public License v2 (GPLv2) 7 | * 8 | * Copyright (c) 2009 The Go Authors. All rights reserved. 9 | * 10 | **/ 11 | 12 | package downloader 13 | 14 | import ( 15 | "io/ioutil" 16 | "strings" 17 | "http" 18 | "os" 19 | "path" 20 | "mime" 21 | "./filemon" 22 | "./anlog" 23 | "./utils" 24 | ) 25 | 26 | func Download(url_source string, abs_path string, strict bool, data *[]byte) (rv bool, lm string, total_size int) { 27 | 28 | resp, err := http.Get(url_source) 29 | if err != nil { 30 | anlog.Error("Cannot download data from `%s`. e: %s\n", url_source, err.String()) 31 | return false, "", 0 32 | } 33 | 34 | *data, err = ioutil.ReadAll(resp.Body) 35 | 36 | if err != nil { 37 | anlog.Error("Cannot read url source body `%s`. error: %s\n", abs_path,err.String()) 38 | return false, "", 0 39 | } 40 | 41 | // check for the mime 42 | content_type := resp.Header.Get("Content-Type") 43 | if endi := strings.IndexAny(content_type,";"); endi > 1 { 44 | content_type = content_type[0:endi] 45 | }else{ 46 | content_type = content_type[0:] 47 | } 48 | 49 | // fmt.Printf("Content-type: %s\n",ctype) 50 | if ext_type := mime.TypeByExtension(path.Ext(abs_path)); ext_type != "" { 51 | if endi := strings.IndexAny(ext_type,";"); endi > 1 { 52 | ext_type = ext_type[0:endi] 53 | }else{ 54 | ext_type = ext_type[0:] 55 | } 56 | content_type := utils.FixedMime(content_type) 57 | exttype := utils.FixedMime(ext_type) 58 | if exttype != content_type { 59 | anlog.Warn("Mime type different by extension. `%s` <> `%s` path `%s`\n", content_type, exttype, url_source ) 60 | if strict { 61 | return false, "", 0 62 | } 63 | } 64 | } 65 | 66 | anlog.Info("File `%s` first cached from `%s`.\n", abs_path, url_source) 67 | 68 | file, err := os.OpenFile(abs_path,os.O_WRONLY | os.O_CREATE,0755) 69 | if err != nil { 70 | anlog.Error("Cannot create file `%s`. error: %s\n", abs_path,err.String()) 71 | return false, "", 0 72 | } 73 | defer file.Close() 74 | 75 | total_size = len(*data) 76 | for { 77 | bw, err := file.Write(*data) 78 | if err != nil { 79 | anlog.Error("Cannot write %d bytes data in file `%s`. error: %s\n", total_size, abs_path,err.String()) 80 | return false, "", 0 81 | } 82 | if bw >= total_size { 83 | break 84 | } 85 | } 86 | 87 | lm, _ = filemon.GetLastModif(file) 88 | 89 | return true, lm, total_size 90 | } 91 | -------------------------------------------------------------------------------- /filemon.go: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * AnsCDN Copyright (C) 2010 Robin Syihab (r [at] nosql.asia) 4 | * Simple CDN server written in Golang. 5 | * 6 | * License: General Public License v2 (GPLv2) 7 | * 8 | * Copyright (c) 2009 The Go Authors. All rights reserved. 9 | * 10 | **/ 11 | 12 | package filemon 13 | 14 | 15 | import ( 16 | "os" 17 | "time" 18 | //"fmt" 19 | "path" 20 | "strings" 21 | "./anlog" 22 | ) 23 | 24 | 25 | var cache_expires int64 26 | 27 | 28 | func isObsolete(atime_ns int64) (rv bool, old int64) { 29 | //tf := time.SecondsToLocalTime(atime_ns / 1e9) 30 | //tn := time.SecondsToLocalTime(time.Seconds() - 1296000) 31 | //anlog.Info("ft = %s\n", tf.String()) 32 | //anlog.Info("nt = %s\n", tn.String()) 33 | tndelta := (time.Seconds() - cache_expires) 34 | old = (tndelta - (atime_ns / 1e9)) 35 | return (atime_ns < (tndelta * 1e9)), old 36 | } 37 | 38 | 39 | func rmObsolete(fpath string){ 40 | 41 | f, err := os.Open(fpath) 42 | if err != nil{return;} 43 | 44 | defer f.Close() 45 | 46 | st, err := f.Stat() 47 | 48 | if err != nil {return;} 49 | 50 | if r, old := isObsolete(st.Atime_ns); r == true { 51 | 52 | anlog.Info("File `%s` is obsolete, %d seconds old.\n", fpath, old) 53 | anlog.Info("Delete file `%s`\n", fpath) 54 | 55 | if err := os.Remove(fpath); err != nil { 56 | anlog.Error("Cannot delete file `%s`. e: %s\n", err.String()) 57 | } 58 | 59 | } 60 | 61 | } 62 | 63 | 64 | func processDir(p string){ 65 | 66 | 67 | dir, err := os.Open(p) 68 | 69 | if err != nil { 70 | anlog.Error("Cannot read dir `%s`. e: %s\n", p, err.String()) 71 | return 72 | } 73 | 74 | defer dir.Close() 75 | 76 | files, err := dir.Readdirnames(10) 77 | 78 | if err != nil { 79 | anlog.Error("Cannot read dir `%s`. e: %s\n", p, err.String()) 80 | return 81 | } 82 | 83 | for _, f := range files{ 84 | if strings.HasPrefix(f,".DS_"){ 85 | continue 86 | } 87 | pp := path.Join(p,f) 88 | //fmt.Println(pp) 89 | o, err := os.Open(pp) 90 | if err != nil { 91 | continue 92 | } 93 | defer o.Close() 94 | if st,_:=o.Stat(); st != nil{ 95 | if st.IsDirectory(){ 96 | processDir(pp) 97 | }else{ 98 | go rmObsolete(pp) 99 | } 100 | } 101 | } 102 | 103 | } 104 | 105 | func StartFileMon(store_dir string, cx int64){ 106 | 107 | cache_expires = cx 108 | 109 | anlog.Info("File monitor started. (`%s`)\n", store_dir) 110 | 111 | for { 112 | 113 | //anlog.Info("Starting file auto cleaner...\n") 114 | 115 | processDir(store_dir) 116 | 117 | time.Sleep(86400 * 1e9) // 1x per day 118 | } 119 | 120 | } 121 | 122 | func GetLastModif(file *os.File) (rv string, err os.Error) { 123 | rv = "" 124 | st, err := file.Stat() 125 | if err != nil {return rv, err;} 126 | 127 | t := time.SecondsToLocalTime( st.Mtime_ns / 1e9 ) 128 | rv = t.Format(time.RFC1123) 129 | return rv, err 130 | } 131 | 132 | /* 133 | func main(){ 134 | 135 | processDir("./data") 136 | 137 | } 138 | */ 139 | 140 | 141 | 142 | 143 | 144 | -------------------------------------------------------------------------------- /utils.go: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * AnsCDN Copyright (C) 2010 Robin Syihab (r [at] nosql.asia) 4 | * Simple CDN server written in Golang. 5 | * 6 | * License: General Public License v2 (GPLv2) 7 | * 8 | * Copyright (c) 2009 The Go Authors. All rights reserved. 9 | * 10 | **/ 11 | 12 | package utils 13 | 14 | 15 | var FixedMimeList = map[string]string{ 16 | "js" : "application/x-javascript", 17 | } 18 | 19 | var VariantMimeList = map[string]string{ 20 | "application/javascript" : FixedMimeList["js"], 21 | } 22 | 23 | 24 | func FixedMime(mimeType string) string { 25 | if v, ok := VariantMimeList[mimeType]; ok{ 26 | return v 27 | } 28 | return mimeType 29 | } 30 | --------------------------------------------------------------------------------