├── .travis.yml ├── README.md └── curl.go /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.3 4 | - 1.2 5 | - tip 6 | install: 7 | - go get github.com/nareix/curl 8 | notifications: 9 | email: 10 | recipients: hi.nareix@gmail.com 11 | on_success: change 12 | on_failure: always 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | CURL-like library for golang (NOT libcurl binding) 3 | 4 | * Custom HTTP method and header 5 | * Monitoring download progress and speed 6 | * Pause/resume control 7 | 8 | ### Status 9 | 10 | [![Travis Build Status](https://travis-ci.org/nareix/curl.svg?branch=master)](https://travis-ci.org/nareix/curl) 11 | 12 | ### Usage 13 | 14 | ```go 15 | import "github.com/nareix/curl" 16 | 17 | req := curl.New("https://kernel.org/pub/linux/kernel/v4.x/linux-4.0.4.tar.xz") 18 | 19 | req.Method("POST") // can be "PUT"/"POST"/"DELETE" ... 20 | 21 | req.Header("MyHeader", "Value") // Custom header 22 | req.Headers = http.Header { // Custom all headers 23 | "User-Agent": {"mycurl/1.0"}, 24 | } 25 | 26 | ctrl := req.ControlDownload() // Download control 27 | go func () { 28 | // control functions are thread safe 29 | ctrl.Stop() // Stop download 30 | ctrl.Pause() // Pause download 31 | ctrl.Resume() // Resume download 32 | }() 33 | 34 | req.DialTimeout(time.Second * 10) // TCP Connection Timeout 35 | req.Timeout(time.Second * 30) // Download Timeout 36 | 37 | // Print progress status per one second 38 | req.Progress(func (p curl.ProgressStatus) { 39 | log.Println( 40 | "Stat", p.Stat, // one of curl.Connecting / curl.Downloading / curl.Closed 41 | "speed", curl.PrettySpeedString(p.Speed), 42 | "len", curl.PrettySizeString(p.ContentLength), 43 | "got", curl.PrettySizeString(p.Size), 44 | "percent", p.Percent, 45 | "paused", p.Paused, 46 | ) 47 | }, time.Second) 48 | /* 49 | 2015/05/20 15:34:15 Stat 2 speed 0.0B/s len 78.5M got 0.0B percent 0 paused true 50 | 2015/05/20 15:34:16 Stat 2 speed 0.0B/s len 78.5M got 0.0B percent 0 paused true 51 | 2015/05/20 15:34:16 Stat 2 speed 394.1K/s len 78.5M got 197.5K percent 0.0024564497 paused false 52 | 2015/05/20 15:34:17 Stat 2 speed 87.8K/s len 78.5M got 241.5K percent 0.0030038392 paused false 53 | 2015/05/20 15:34:17 Stat 2 speed 79.8K/s len 78.5M got 281.5K percent 0.003501466 paused false 54 | 2015/05/20 15:34:18 Stat 2 speed 63.9K/s len 78.5M got 313.5K percent 0.0038995675 paused false 55 | */ 56 | 57 | res, err := req.Do() 58 | 59 | res.HttpResponse // related *http.Response struct 60 | log.Println(res.Body) // Body in string 61 | log.Println(res.StatusCode) // HTTP Status Code: 200,404,302 etc 62 | log.Println(res.Hearders) // Reponse headers 63 | log.Println(res.DownloadStatus.AverageSpeed) // Average speed 64 | ``` 65 | -------------------------------------------------------------------------------- /curl.go: -------------------------------------------------------------------------------- 1 | package curl 2 | 3 | import ( 4 | "bytes" 5 | _ "errors" 6 | "fmt" 7 | "io" 8 | "mime/multipart" 9 | "net" 10 | "net/http" 11 | "os" 12 | "strings" 13 | "sync" 14 | "time" 15 | ) 16 | 17 | /* 18 | var ( 19 | ErrConnectTimeout = errors.New("Connecting timeout") 20 | ErrDNSResolve = errors.New("DNS resolve failed") 21 | ErrDownloadTimeout = errors.New("Download timeout") 22 | ErrCreateDownloadFile = errors.New("Create download file error") 23 | ) 24 | */ 25 | 26 | type Request struct { 27 | url string 28 | method string 29 | Headers http.Header 30 | body string 31 | 32 | bodyUploadEntry *uploadEntry 33 | mpartUploadEntries []uploadEntry 34 | 35 | uploadMonitor *Monitor 36 | downloadMonitor *Monitor 37 | 38 | downloadToFile string 39 | 40 | stat int 41 | progressCb MonitorProgressCb 42 | 43 | Upload ProgressStatus 44 | Download ProgressStatus 45 | 46 | reqbodyTracer io.Writer 47 | reqTracer io.Writer 48 | 49 | progressCloseEvent chan int 50 | progressInterval time.Duration 51 | 52 | dialTimeout time.Duration 53 | transferTimeout time.Duration 54 | } 55 | 56 | func New(url string) *Request { 57 | req := &Request{ 58 | url: url, 59 | Headers: http.Header{ 60 | "User-Agent": {"curl/7.29.0"}, 61 | }, 62 | } 63 | 64 | req.uploadMonitor = &Monitor{ioTracker: &ioTracker{}} 65 | req.downloadMonitor = &Monitor{ioTracker: &ioTracker{}} 66 | 67 | return req 68 | } 69 | 70 | func Get(url string) *Request { 71 | return New(url).Method("GET") 72 | } 73 | 74 | func Post(url string) *Request { 75 | return New(url).Method("POST") 76 | } 77 | 78 | func (req *Request) Method(method string) *Request { 79 | req.method = method 80 | return req 81 | } 82 | 83 | func (req *Request) Header(k, v string) *Request { 84 | req.Headers[k] = []string{v} 85 | return req 86 | } 87 | 88 | func (req *Request) UserAgent(v string) *Request { 89 | return req.Header("User-Agent", v) 90 | } 91 | 92 | func (req *Request) BodyString(v string) *Request { 93 | req.body = v 94 | return req 95 | } 96 | 97 | func (req *Request) TraceRequestBody(w io.Writer) *Request { 98 | req.reqbodyTracer = w 99 | return req 100 | } 101 | 102 | func (req *Request) TraceRequest(w io.Writer) *Request { 103 | req.reqTracer = w 104 | return req 105 | } 106 | 107 | type uploadEntry struct { 108 | filename string 109 | filepath string 110 | } 111 | 112 | func (e uploadEntry) getFileReader() (reader io.Reader, length int64, err error) { 113 | var file *os.File 114 | var fileInfo os.FileInfo 115 | 116 | if file, err = os.Open(e.filepath); err != nil { 117 | return 118 | } 119 | if fileInfo, err = file.Stat(); err != nil { 120 | return 121 | } 122 | 123 | length = fileInfo.Size() 124 | reader = file 125 | 126 | return 127 | } 128 | 129 | func (req *Request) BodyUploadFile(filename, filepath string) *Request { 130 | req.bodyUploadEntry = &uploadEntry{ 131 | filename: filename, 132 | filepath: filepath, 133 | } 134 | return req 135 | } 136 | 137 | func (req *Request) SaveToFile(filepath string) *Request { 138 | req.downloadToFile = filepath 139 | return req 140 | } 141 | 142 | type ioTracker struct { 143 | io.Reader 144 | io.Writer 145 | Bytes int64 146 | whenDataComes func() 147 | pausedCond *sync.Cond 148 | pausedLock *sync.Mutex 149 | paused bool 150 | stop bool 151 | } 152 | 153 | func (tracker *ioTracker) newPauseCond() { 154 | tracker.pausedLock = &sync.Mutex{} 155 | tracker.pausedCond = sync.NewCond(tracker.pausedLock) 156 | } 157 | 158 | func (tracker *ioTracker) WhenDataComes(cb func()) { 159 | tracker.whenDataComes = cb 160 | } 161 | 162 | func (tracker *ioTracker) preIO() (err error) { 163 | if tracker.stop { 164 | return io.EOF 165 | } 166 | if tracker.whenDataComes != nil { 167 | tracker.whenDataComes() 168 | tracker.whenDataComes = nil 169 | } 170 | if tracker.pausedCond != nil { 171 | tracker.pausedCond.L.Lock() 172 | if tracker.paused { 173 | tracker.pausedCond.Wait() 174 | } 175 | tracker.pausedCond.L.Unlock() 176 | } 177 | return 178 | } 179 | 180 | func (tracker *ioTracker) Write(p []byte) (n int, err error) { 181 | if err = tracker.preIO(); err != nil { 182 | return 183 | } 184 | n, err = tracker.Writer.Write(p) 185 | tracker.Bytes += int64(n) 186 | return 187 | } 188 | 189 | func (tracker *ioTracker) Read(p []byte) (n int, err error) { 190 | if err = tracker.preIO(); err != nil { 191 | return 192 | } 193 | n, err = tracker.Reader.Read(p) 194 | tracker.Bytes += int64(n) 195 | return 196 | } 197 | 198 | func (req *Request) getBodyUploadReader() (body io.Reader, length int64, err error) { 199 | return req.bodyUploadEntry.getFileReader() 200 | } 201 | 202 | func (req *Request) getMpartUploadReaderAndContentType() (body io.Reader, contentType string) { 203 | pReader, pWriter := io.Pipe() 204 | mpart := multipart.NewWriter(pWriter) 205 | 206 | go func() { 207 | defer pWriter.Close() 208 | defer mpart.Close() 209 | 210 | entry := req.mpartUploadEntries[0] 211 | 212 | part, err := mpart.CreateFormFile(entry.filename, entry.filepath) 213 | if err != nil { 214 | return 215 | } 216 | 217 | var f io.ReadCloser 218 | if f, err = os.Open(entry.filepath); err != nil { 219 | return 220 | } 221 | defer f.Close() 222 | 223 | _, err = io.Copy(part, f) 224 | }() 225 | 226 | return pReader, mpart.FormDataContentType() 227 | } 228 | 229 | const ( 230 | Connecting = iota 231 | Uploading 232 | Downloading 233 | Closed 234 | ) 235 | 236 | type ProgressStatus struct { 237 | Stat int 238 | ContentLength int64 239 | Size int64 240 | Percent float32 241 | AverageSpeed int64 242 | Speed int64 243 | MaxSpeed int64 244 | TimeElapsed time.Duration 245 | Paused bool 246 | } 247 | 248 | type Monitor struct { 249 | ioTracker *ioTracker 250 | contentLength int64 251 | timeStarted time.Time 252 | finished bool 253 | } 254 | 255 | type MonitorProgressCb func(p ProgressStatus) 256 | 257 | type snapShot struct { 258 | bytes int64 259 | time time.Time 260 | } 261 | 262 | func (mon *Monitor) currentProgressSnapshot() (shot snapShot) { 263 | shot.bytes = mon.ioTracker.Bytes 264 | shot.time = time.Now() 265 | return 266 | } 267 | 268 | func (mon *Monitor) getProgressStatus(lastSnapshot *snapShot) (stat ProgressStatus) { 269 | now := time.Now() 270 | 271 | stat.TimeElapsed = now.Sub(mon.timeStarted) 272 | stat.ContentLength = mon.contentLength 273 | stat.Size = mon.ioTracker.Bytes 274 | 275 | if lastSnapshot != nil { 276 | stat.Speed = (stat.Size - lastSnapshot.bytes) * 1000 / (int64(now.Sub(lastSnapshot.time)) / int64(time.Millisecond)) 277 | } 278 | 279 | stat.AverageSpeed = stat.Size * 1000 / (int64(now.Sub(mon.timeStarted)) / int64(time.Millisecond)) 280 | 281 | if stat.ContentLength > 0 { 282 | stat.Percent = float32(stat.Size) / float32(stat.ContentLength) 283 | } 284 | 285 | stat.Paused = mon.ioTracker.paused 286 | 287 | return 288 | } 289 | 290 | type traceConn struct { 291 | net.Conn 292 | io.Writer 293 | } 294 | 295 | func (conn traceConn) Write(b []byte) (n int, err error) { 296 | if conn.Writer != nil { 297 | conn.Writer.Write(b) 298 | } 299 | return conn.Conn.Write(b) 300 | } 301 | 302 | func (req *Request) MonitorDownload() (mon *Monitor) { 303 | mon = &Monitor{} 304 | req.downloadMonitor = mon 305 | return 306 | } 307 | 308 | func (req *Request) MonitorUpload() (mon *Monitor) { 309 | mon = &Monitor{} 310 | req.uploadMonitor = mon 311 | return 312 | } 313 | 314 | func (req *Request) enterStat(stat int) { 315 | if req.progressCb != nil { 316 | req.progressCb(ProgressStatus{Stat: stat}) 317 | } 318 | 319 | progressCall := func(stat int, mon *Monitor) { 320 | var shot snapShot 321 | if mon != nil { 322 | shot = mon.currentProgressSnapshot() 323 | } 324 | 325 | go func() { 326 | for { 327 | select { 328 | case <-time.After(req.progressInterval): 329 | ps := ProgressStatus{} 330 | if mon != nil { 331 | ps = mon.getProgressStatus(&shot) 332 | shot = mon.currentProgressSnapshot() 333 | } 334 | ps.Stat = stat 335 | if req.progressCb != nil { 336 | req.progressCb(ps) 337 | } 338 | 339 | case <-req.progressCloseEvent: 340 | return 341 | } 342 | } 343 | }() 344 | } 345 | 346 | switch stat { 347 | case Connecting: 348 | req.progressCloseEvent = make(chan int) 349 | progressCall(stat, nil) 350 | 351 | case Uploading: 352 | req.progressCloseEvent <- 0 353 | progressCall(stat, req.uploadMonitor) 354 | 355 | case Downloading: 356 | req.progressCloseEvent <- 0 357 | progressCall(stat, req.downloadMonitor) 358 | 359 | case Closed: 360 | req.progressCloseEvent <- 0 361 | } 362 | } 363 | 364 | func (req *Request) Progress(cb MonitorProgressCb, interval time.Duration) *Request { 365 | req.progressCb = cb 366 | req.progressInterval = interval 367 | return req 368 | } 369 | 370 | func (req *Request) DialTimeout(timeout time.Duration) *Request { 371 | req.dialTimeout = timeout 372 | return req 373 | } 374 | 375 | func (req *Request) Timeout(timeout time.Duration) *Request { 376 | req.transferTimeout = timeout 377 | return req 378 | } 379 | 380 | type Control struct { 381 | ioTracker *ioTracker 382 | } 383 | 384 | func (ctrl *Control) Stop() { 385 | ctrl.ioTracker.stop = true 386 | } 387 | 388 | func (ctrl *Control) Resume() { 389 | ctrl.ioTracker.paused = false 390 | ctrl.ioTracker.pausedCond.Broadcast() 391 | } 392 | 393 | func (ctrl *Control) Pause() { 394 | ctrl.ioTracker.paused = true 395 | ctrl.ioTracker.pausedCond.Broadcast() 396 | } 397 | 398 | func (req *Request) ControlDownload() (ctrl *Control) { 399 | ctrl = &Control{ 400 | ioTracker: req.downloadMonitor.ioTracker, 401 | } 402 | ctrl.ioTracker.newPauseCond() 403 | return 404 | } 405 | 406 | func (req *Request) Do() (res Response, err error) { 407 | var httpreq *http.Request 408 | var httpres *http.Response 409 | var reqbody io.Reader 410 | var reqbodyLength int64 411 | 412 | if len(req.mpartUploadEntries) > 0 { 413 | var contentType string 414 | reqbody, contentType = req.getMpartUploadReaderAndContentType() 415 | req.Headers["Content-Type"] = []string{contentType} 416 | req.method = "POST" 417 | } else if req.bodyUploadEntry != nil { 418 | if reqbody, reqbodyLength, err = req.getBodyUploadReader(); err != nil { 419 | return 420 | } 421 | } else { 422 | reqbody = strings.NewReader(req.body) 423 | reqbodyLength = int64(len(req.body)) 424 | } 425 | 426 | if req.reqbodyTracer != nil && reqbody != nil { 427 | reqbody = io.TeeReader(reqbody, req.reqbodyTracer) 428 | } 429 | 430 | req.uploadMonitor.contentLength = reqbodyLength 431 | req.uploadMonitor.timeStarted = time.Now() 432 | req.downloadMonitor.timeStarted = time.Now() 433 | 434 | req.uploadMonitor.ioTracker.Reader = reqbody 435 | reqbody = req.uploadMonitor.ioTracker 436 | 437 | req.enterStat(Connecting) 438 | 439 | req.uploadMonitor.ioTracker.WhenDataComes(func() { 440 | req.enterStat(Uploading) 441 | }) 442 | 443 | req.downloadMonitor.ioTracker.WhenDataComes(func() { 444 | req.enterStat(Downloading) 445 | }) 446 | 447 | defer req.enterStat(Closed) 448 | 449 | if httpreq, err = http.NewRequest(req.method, req.url, reqbody); err != nil { 450 | return 451 | } 452 | httpreq.Header = req.Headers 453 | httpreq.ContentLength = reqbodyLength 454 | 455 | httptrans := &http.Transport{ 456 | Dial: func(network, addr string) (conn net.Conn, err error) { 457 | if req.dialTimeout != time.Duration(0) { 458 | conn, err = net.DialTimeout(network, addr, req.dialTimeout) 459 | } else { 460 | conn, err = net.Dial(network, addr) 461 | } 462 | if conn != nil { 463 | conn = traceConn{Conn: conn, Writer: req.reqTracer} 464 | if req.transferTimeout != time.Duration(0) { 465 | conn.SetDeadline(time.Now().Add(req.transferTimeout)) 466 | } 467 | } 468 | return 469 | }, 470 | DisableCompression: true, 471 | } 472 | 473 | httpclient := http.Client{ 474 | Transport: httptrans, 475 | } 476 | 477 | if httpres, err = httpclient.Do(httpreq); err != nil { 478 | return 479 | } 480 | defer httpres.Body.Close() 481 | 482 | var resbodyBuffer *bytes.Buffer 483 | var resbody io.Writer 484 | 485 | if req.downloadToFile != "" { 486 | var f *os.File 487 | if f, err = os.Create(req.downloadToFile); err != nil { 488 | return 489 | } 490 | resbody = f 491 | } else { 492 | resbodyBuffer = &bytes.Buffer{} 493 | resbody = resbodyBuffer 494 | } 495 | 496 | req.downloadMonitor.contentLength = httpres.ContentLength 497 | req.downloadMonitor.ioTracker.Writer = resbody 498 | resbody = req.downloadMonitor.ioTracker 499 | 500 | res.StatusCode = httpres.StatusCode 501 | res.Headers = httpres.Header 502 | res.HttpResponse = httpres 503 | 504 | if _, err = io.Copy(resbody, httpres.Body); err != nil { 505 | return 506 | } 507 | 508 | if resbodyBuffer != nil { 509 | res.Body = resbodyBuffer.String() 510 | } 511 | 512 | req.stat = Closed 513 | 514 | req.uploadMonitor.finished = true 515 | req.downloadMonitor.finished = true 516 | res.UploadStatus = req.uploadMonitor.getProgressStatus(nil) 517 | res.DownloadStatus = req.downloadMonitor.getProgressStatus(nil) 518 | 519 | return 520 | } 521 | 522 | type Response struct { 523 | HttpResponse *http.Response 524 | StatusCode int 525 | Headers http.Header 526 | Body string 527 | UploadStatus ProgressStatus 528 | DownloadStatus ProgressStatus 529 | } 530 | 531 | func PrettySizeString(size int64) string { 532 | unit := "B" 533 | fsize := float64(size) 534 | 535 | if fsize > 1024 { 536 | unit = "K" 537 | fsize /= 1024 538 | } 539 | if fsize > 1024 { 540 | unit = "M" 541 | fsize /= 1024 542 | } 543 | if fsize > 1024 { 544 | unit = "G" 545 | fsize /= 1024 546 | } 547 | 548 | return fmt.Sprintf("%.1f%s", fsize, unit) 549 | } 550 | 551 | func PrettySpeedString(speed int64) string { 552 | return fmt.Sprintf("%s/s", PrettySizeString(speed)) 553 | } 554 | --------------------------------------------------------------------------------