├── cli ├── version │ └── version.go ├── application │ ├── func.go │ ├── product.go │ ├── login.go │ └── course.go └── cmds │ ├── buy.go │ ├── course.go │ ├── cmds.go │ ├── login.go │ └── download.go ├── config ├── export.go ├── errors.go ├── geek.go └── config.go ├── requester ├── requester.go ├── http_client.go └── fetch.go ├── utils ├── chromedp_test.go ├── json.go ├── pool_test.go ├── pool.go ├── ffmpeg.go ├── utils_test.go ├── utils.go └── chromedp.go ├── go.mod ├── service ├── func.go ├── user.go ├── errors.go ├── service.go ├── buy_product.go ├── response_handler.go ├── requester.go ├── course.go └── course_types.go ├── main.go ├── LICENSE ├── .goreleaser.yml ├── downloader ├── pdf.go ├── types.go └── downloader.go ├── aeswrap ├── aeswrap.go └── aeswrap_test.go ├── login └── login_client.go ├── README.md ├── go.sum └── .gitignore /cli/version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | var ( 4 | //Version version 5 | Version = "dev" 6 | ) 7 | -------------------------------------------------------------------------------- /config/export.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | type configJSONExport struct { 4 | AcitveUID int 5 | Geektimes Geektimes 6 | } 7 | -------------------------------------------------------------------------------- /cli/application/func.go: -------------------------------------------------------------------------------- 1 | package application 2 | 3 | import ( 4 | "github.com/udoless/geektime-downloader/config" 5 | "github.com/udoless/geektime-downloader/service" 6 | ) 7 | 8 | func getService() *service.Service { 9 | return config.Instance.ActiveUserService() 10 | } 11 | -------------------------------------------------------------------------------- /requester/requester.go: -------------------------------------------------------------------------------- 1 | package requester 2 | 3 | var ( 4 | // UserAgent 浏览器标识 5 | UserAgent = "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1" 6 | 7 | //DefaultClient 默认 http 客户端 8 | DefaultClient = NewHTTPClient() 9 | ) 10 | -------------------------------------------------------------------------------- /utils/chromedp_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | func TestPrintToPdf(t *testing.T) { 9 | filename := "file.pdf" 10 | err := ColumnPrintToPDF(158248, filename, nil) 11 | 12 | if err != nil { 13 | t.Fatal("PrintToPDF test is failure", err) 14 | } else { 15 | os.Remove(filename) 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /utils/json.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "io" 5 | 6 | jsoniter "github.com/json-iterator/go" 7 | ) 8 | 9 | //UnmarshalReader 将 r 中的 json 格式的数据, 解析到 v 10 | func UnmarshalReader(r io.Reader, v interface{}) error { 11 | d := jsoniter.NewDecoder(r) 12 | return d.Decode(v) 13 | } 14 | 15 | //UnmarshalJSON 将 r 中的 json 格式的数据, 解析到 v 16 | func UnmarshalJSON(data []byte, v interface{}) error { 17 | return jsoniter.Unmarshal(data, v) 18 | } 19 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/udoless/geektime-downloader 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/cheggaaa/pb v1.0.25 7 | github.com/chromedp/cdproto v0.0.0-20210713064928-7d28b402946a 8 | github.com/chromedp/chromedp v0.7.4 9 | github.com/json-iterator/go v1.1.9 10 | github.com/knq/sysutil v0.0.0-20191005231841-15668db23d08 // indirect 11 | github.com/olekukonko/tablewriter v0.0.4 12 | github.com/sirupsen/logrus v1.5.0 13 | github.com/urfave/cli v1.22.4 14 | ) 15 | -------------------------------------------------------------------------------- /utils/pool_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "sync/atomic" 5 | "testing" 6 | ) 7 | 8 | func TestPool(t *testing.T) { 9 | wgp := NewWaitGroupPool(10) 10 | 11 | number := 1000 12 | var loop int32 13 | for i := 0; i < number; i++ { 14 | wgp.Add() 15 | go func(loop *int32) { 16 | defer wgp.Done() 17 | atomic.AddInt32(loop, 1) 18 | }(&loop) 19 | } 20 | wgp.Wait() 21 | 22 | if int(loop) != number { 23 | t.Fatal("Pool test is failure") 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /service/func.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "io" 5 | "net/http" 6 | ) 7 | 8 | func deferResponseClose(s *http.Response) { 9 | if s != nil { 10 | defer s.Body.Close() 11 | } 12 | } 13 | 14 | //handleHTTPResponse handle 15 | func handleHTTPResponse(res *http.Response, err error) (io.ReadCloser, Error) { 16 | if err != nil { 17 | deferResponseClose(res) 18 | return nil, &ErrorInfo{Err: err} 19 | } 20 | 21 | if res.StatusCode == 452 { 22 | return nil, &ErrorInfo{Err: ErrLoginOffline} 23 | } 24 | 25 | return res.Body, nil 26 | } 27 | -------------------------------------------------------------------------------- /cli/application/product.go: -------------------------------------------------------------------------------- 1 | package application 2 | 3 | import ( 4 | "github.com/udoless/geektime-downloader/service" 5 | ) 6 | 7 | //BuyProductAll product 8 | func BuyProductAll() (*service.ProductAll, error) { 9 | return getService().BuyProductAll() 10 | } 11 | 12 | //BuyColumns all columns 13 | func BuyColumns() (*service.Product, error) { 14 | all, err := BuyProductAll() 15 | return all.Columns, err 16 | } 17 | 18 | //BuyVideos all columns 19 | func BuyVideos() (*service.Product, error) { 20 | all, err := BuyProductAll() 21 | return all.Videos, err 22 | } 23 | -------------------------------------------------------------------------------- /service/user.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | //User user info 4 | type User struct { 5 | UID int `json:"uid"` 6 | Nickname string `json:"nickname"` 7 | Avatar string `json:"avatar"` 8 | Cellphone string `json:"cellphone"` 9 | } 10 | 11 | //User user info 12 | func (s *Service) User() (*User, Error) { 13 | body, err := s.requestUser() 14 | 15 | if err != nil { 16 | return nil, err 17 | } 18 | 19 | defer body.Close() 20 | 21 | user := new(User) 22 | if err := handleJSONParse(body, &user); err != nil { 23 | return nil, err 24 | } 25 | 26 | return user, nil 27 | } 28 | -------------------------------------------------------------------------------- /cli/application/login.go: -------------------------------------------------------------------------------- 1 | package application 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/udoless/geektime-downloader/login" 7 | ) 8 | 9 | //Login login 10 | func Login(phone, password string) (gcess string, gcid string, serverID string, err error) { 11 | c := login.NewLoginClient() 12 | result := c.Login(phone, password) 13 | if !result.IsLoginSuccess() { 14 | return "", "", "", errors.New(result.Error.Msg) 15 | } 16 | 17 | return result.Data.GCID, result.Data.GCESS, result.Data.ServerID, nil 18 | } 19 | 20 | //LoginedCookies get logined cookies 21 | func LoginedCookies() map[string]string { 22 | return getService().Cookies() 23 | } 24 | -------------------------------------------------------------------------------- /config/errors.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import "errors" 4 | 5 | var ( 6 | //ErrNotLogin 未登录帐号错误 7 | ErrNotLogin = errors.New("请先登录极客时间账户") 8 | //ErrHasLoginedNotLogin 有登录用户，但是当前并未有有效用户 9 | ErrHasLoginedNotLogin = errors.New("存在登录的用户，可以进行切换登录用户") 10 | //ErrConfigFilePathNotSet 未设置配置文件 11 | ErrConfigFilePathNotSet = errors.New("config file not set") 12 | //ErrConfigFileNotExist 未设置Config, 未初始化 13 | ErrConfigFileNotExist = errors.New("config file not exist") 14 | //ErrConfigFileNoPermission Config文件无权限访问 15 | ErrConfigFileNoPermission = errors.New("config file permission denied") 16 | //ErrConfigContentsParseError 解析Config数据错误 17 | ErrConfigContentsParseError = errors.New("config contents parse error") 18 | ) 19 | -------------------------------------------------------------------------------- /service/errors.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import "errors" 4 | 5 | var ( 6 | //ErrNotLogin not login 7 | ErrNotLogin = errors.New("当前未登录获取登录已失效，请先登录") 8 | //ErrLoginOffline not login 9 | ErrLoginOffline = errors.New("该账号已在其他同类设备登录，如非本人操作，则密码可能已经泄露，建议立即更换密码") 10 | ) 11 | 12 | // Error 错误信息接口 13 | type Error interface { 14 | error 15 | IsUnlogin() bool 16 | } 17 | 18 | //ErrorInfo error info 19 | type ErrorInfo struct { 20 | Err error 21 | } 22 | 23 | //IsUnlogin 是否未登录 24 | func (e *ErrorInfo) IsUnlogin() bool { 25 | return e.Err == ErrNotLogin 26 | } 27 | 28 | func (e *ErrorInfo) Error() string { 29 | if e.Err != nil { 30 | return e.Err.Error() 31 | } 32 | return "" 33 | } 34 | 35 | func (e *ErrorInfo) String() string { 36 | return e.Error() 37 | } 38 | -------------------------------------------------------------------------------- /requester/http_client.go: -------------------------------------------------------------------------------- 1 | package requester 2 | 3 | import ( 4 | "net/http" 5 | "net/http/cookiejar" 6 | "time" 7 | ) 8 | 9 | //HTTPClient client 10 | type HTTPClient struct { 11 | http.Client 12 | UserAgent string 13 | } 14 | 15 | //NewHTTPClient new client 16 | func NewHTTPClient() *HTTPClient { 17 | c := &HTTPClient{ 18 | Client: http.Client{ 19 | Timeout: 10 * time.Second, 20 | }, 21 | } 22 | 23 | c.ResetCookieJar() 24 | 25 | return c 26 | } 27 | 28 | // SetUserAgent 设置 UserAgent 浏览器标识 29 | func (h *HTTPClient) SetUserAgent(ua string) { 30 | h.UserAgent = ua 31 | } 32 | 33 | //SetCookiejar 设置 Cookie 34 | func (h *HTTPClient) SetCookiejar(jar http.CookieJar) { 35 | h.Client.Jar = jar 36 | } 37 | 38 | //ResetCookieJar 重置cookie jar 39 | func (h *HTTPClient) ResetCookieJar() { 40 | h.Jar, _ = cookiejar.New(nil) 41 | } 42 | 43 | //SetTimeout 设置超时时间 44 | func (h *HTTPClient) SetTimeout(t time.Duration) { 45 | h.Client.Timeout = t 46 | } 47 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | 8 | "github.com/sirupsen/logrus" 9 | "github.com/udoless/geektime-downloader/cli/cmds" 10 | "github.com/udoless/geektime-downloader/config" 11 | "github.com/urfave/cli" 12 | ) 13 | 14 | func init() { 15 | 16 | err := config.Instance.Init() 17 | if err != nil { 18 | fmt.Println(err) 19 | } 20 | } 21 | 22 | func main() { 23 | f, err := os.OpenFile("log.log", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) 24 | if err != nil { 25 | log.Fatalf("error opening file: %v", err) 26 | } 27 | defer f.Close() 28 | 29 | log.SetOutput(f) 30 | 31 | app := cmds.NewApp() 32 | app.Commands = []cli.Command{} 33 | app.Commands = append(app.Commands, cmds.NewLoginCommand()...) 34 | app.Commands = append(app.Commands, cmds.NewBuyCommand()...) 35 | app.Commands = append(app.Commands, cmds.NewCourseCommand()...) 36 | 37 | app.Action = cmds.DefaultAction 38 | 39 | if err := app.Run(os.Args); err != nil { 40 | logrus.Fatal(err) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright 2020-present, udoless 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /utils/pool.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "math" 5 | "sync" 6 | ) 7 | 8 | // WaitGroupPool pool of WaitGroup 9 | type WaitGroupPool struct { 10 | pool chan int 11 | wg *sync.WaitGroup 12 | } 13 | 14 | // NewWaitGroupPool creates a sized pool for WaitGroup 15 | func NewWaitGroupPool(size int) *WaitGroupPool { 16 | if size <= 0 { 17 | size = math.MaxInt32 18 | } 19 | 20 | return &WaitGroupPool{ 21 | pool: make(chan int, size), 22 | wg: &sync.WaitGroup{}, 23 | } 24 | } 25 | 26 | // Add adds delta, which may be negative, to the WaitGroup counter. 27 | // See sync.WaitGroup documentation for more information. 28 | func (p *WaitGroupPool) Add() { 29 | p.pool <- 1 30 | p.wg.Add(1) 31 | } 32 | 33 | // Done decrements the WaitGroup counter by one. 34 | // See sync.WaitGroup documentation for more information. 35 | func (p *WaitGroupPool) Done() { 36 | <-p.pool 37 | p.wg.Done() 38 | } 39 | 40 | // Wait blocks until the WaitGroup counter is zero. 41 | // See sync.WaitGroup documentation for more information. 42 | func (p *WaitGroupPool) Wait() { 43 | p.wg.Wait() 44 | } 45 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | project_name: geektime-dl 2 | env: 3 | - GO111MODULE=on 4 | builds: 5 | - env: 6 | - CGO_ENABLED=1 7 | binary: geektime-dl 8 | goos: 9 | - windows 10 | - darwin 11 | - linux 12 | goarch: 13 | - 386 14 | - amd64 15 | - arm 16 | - arm64 17 | ignore: 18 | - goos: linux 19 | goarch: arm 20 | goarm: 6 21 | - goos: darwin 22 | goarch: 386 23 | changelog: 24 | sort: asc 25 | filters: 26 | exclude: 27 | - '^docs' 28 | - '^tests' 29 | - Merge pull request 30 | - Merge branch 31 | archives: 32 | - 33 | name_template: '{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}_v{{ .Arm }}{{ end }}' 34 | format: tar.gz 35 | format_overrides: 36 | - goos: windows 37 | format: zip 38 | files: 39 | - none* 40 | wrap_in_directory: false 41 | replacements: 42 | amd64: 64-bit 43 | 386: 32-bit 44 | arm: ARM 45 | arm64: ARM64 46 | darwin: macOS 47 | linux: Linux 48 | windows: Windows 49 | openbsd: OpenBSD 50 | netbsd: NetBSD 51 | freebsd: FreeBSD 52 | release: 53 | draft: true 54 | -------------------------------------------------------------------------------- /downloader/pdf.go: -------------------------------------------------------------------------------- 1 | package downloader 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "strings" 7 | 8 | "github.com/udoless/geektime-downloader/utils" 9 | ) 10 | 11 | var replacer = strings.NewReplacer( 12 | "/", "_", 13 | "\\", "_", 14 | "*", "_", 15 | ":", "_", 16 | "\"", "“", 17 | "<", "《", 18 | ">", "》", 19 | "|", "_", 20 | "?", "", 21 | ) 22 | 23 | //PrintToPDF print to pdf 24 | func PrintToPDF(v Datum, cookies map[string]string, path string) error { 25 | 26 | name := utils.FileName(v.Title, "pdf") 27 | name = replacer.Replace(name) 28 | 29 | filename := filepath.Join(path, name) 30 | fmt.Printf("正在生成文件：【\033[37;1m%s\033[0m】 \n", name) 31 | 32 | _, exist, err := utils.FileSize(filename) 33 | 34 | if err != nil { 35 | fmt.Printf("\033[31;1m%s, err=%v\033[0m\n", "失败1", err) 36 | return err 37 | } 38 | 39 | if exist { 40 | fmt.Printf("\033[33;1m%s\033[0m\n", "已存在，跳过") 41 | return nil 42 | } 43 | 44 | err = utils.ColumnPrintToPDF(v.ID, filename, cookies) 45 | 46 | if err != nil { 47 | fmt.Printf("\033[31;1m%s, err=%v\033[0m\n", "失败2", err) 48 | return err 49 | } 50 | 51 | fmt.Printf("\033[32;1m%s\033[0m\n", "完成") 52 | 53 | return nil 54 | } 55 | -------------------------------------------------------------------------------- /cli/cmds/buy.go: -------------------------------------------------------------------------------- 1 | package cmds 2 | 3 | import ( 4 | "os" 5 | "strconv" 6 | 7 | "github.com/olekukonko/tablewriter" 8 | "github.com/udoless/geektime-downloader/cli/application" 9 | "github.com/urfave/cli" 10 | ) 11 | 12 | //NewBuyCommand login command 13 | func NewBuyCommand() []cli.Command { 14 | return []cli.Command{ 15 | { 16 | Name: "buy", 17 | Usage: "获取已购买过的专栏和视频课程", 18 | UsageText: appName + " buy", 19 | Action: buyAction, 20 | Before: authorizationFunc, 21 | }, 22 | } 23 | } 24 | 25 | func buyAction(c *cli.Context) error { 26 | products, err := application.BuyProductAll() 27 | 28 | if err != nil { 29 | return err 30 | } 31 | 32 | table := tablewriter.NewWriter(os.Stdout) 33 | table.SetHeader([]string{"#", "ID", "类型", "名称", "作者"}) 34 | 35 | i := 0 36 | for _, p := range products.Columns.List { 37 | table.Append([]string{strconv.Itoa(i), strconv.Itoa(p.Extra.ColumnID), products.Columns.Title, p.Title, p.Extra.AuthorName}) 38 | i++ 39 | } 40 | 41 | for _, p := range products.Videos.List { 42 | table.Append([]string{strconv.Itoa(i), strconv.Itoa(p.Extra.ColumnID), products.Videos.Title, p.Title, p.Extra.AuthorName}) 43 | i++ 44 | } 45 | 46 | table.Render() 47 | 48 | return nil 49 | } 50 | -------------------------------------------------------------------------------- /service/service.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "net/http" 5 | "net/url" 6 | 7 | "github.com/udoless/geektime-downloader/requester" 8 | ) 9 | 10 | var ( 11 | geekBangCommURL = &url.URL{ 12 | Scheme: "https", 13 | Host: "geekbang.org", 14 | } 15 | ) 16 | 17 | //Service geek time service 18 | type Service struct { 19 | client *requester.HTTPClient 20 | } 21 | 22 | //NewService new service 23 | func NewService(gcid, gcess, serviceID string) *Service { 24 | client := requester.NewHTTPClient() 25 | client.ResetCookieJar() 26 | cookies := []*http.Cookie{} 27 | cookies = append(cookies, &http.Cookie{ 28 | Name: "GCID", 29 | Value: gcid, 30 | Domain: "." + geekBangCommURL.Host, 31 | }) 32 | cookies = append(cookies, &http.Cookie{ 33 | Name: "GCESS", 34 | Value: gcess, 35 | Domain: "." + geekBangCommURL.Host, 36 | }) 37 | cookies = append(cookies, &http.Cookie{ 38 | Name: "SERVERID", 39 | Value: serviceID, 40 | Domain: "." + geekBangCommURL.Host, 41 | }) 42 | client.Jar.SetCookies(geekBangCommURL, cookies) 43 | 44 | return &Service{client: client} 45 | } 46 | 47 | //Cookies get cookies string 48 | func (s *Service) Cookies() map[string]string { 49 | cookies := s.client.Jar.Cookies(geekBangCommURL) 50 | 51 | cstr := map[string]string{} 52 | 53 | for _, cookie := range cookies { 54 | cstr[cookie.Name] = cookie.Value 55 | } 56 | 57 | return cstr 58 | } 59 | -------------------------------------------------------------------------------- /service/buy_product.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | // ProductAll all protuct 4 | type ProductAll struct { 5 | Columns *Product 6 | Videos *Product 7 | } 8 | 9 | //Product all product 10 | type Product struct { 11 | ID int `json:"id"` 12 | Title string `json:"title"` 13 | Page struct { 14 | More bool `json:"more"` 15 | Count int `json:"count"` 16 | } `json:"page"` 17 | List []struct { 18 | Title string `json:"title"` 19 | Conver string `json:"cover"` 20 | Type string `json:"type"` 21 | Extra struct { 22 | LastAid int `json:"last_aid"` 23 | ColumnID int `json:"column_id"` 24 | ColumnTitle string `json:"column_title"` 25 | ColumnSubtitle string `json:"column_subtitle"` 26 | AuthorName string `json:"author_name"` 27 | AuthorIntro string `json:"author_intro"` 28 | ColumnCover string `json:"column_cover"` 29 | ColumnType int `json:"column_type"` 30 | ArticleCount int `json:"article_count"` 31 | IsIncludeAudio bool `json:"is_include_audio"` 32 | } `json:"extra"` 33 | } `json:"list"` 34 | } 35 | 36 | //BuyProductAll 获取所有购买的课程信息 37 | func (s *Service) BuyProductAll() (*ProductAll, error) { 38 | body, err := s.requestBuyAll() 39 | 40 | if err != nil { 41 | return nil, err 42 | } 43 | 44 | defer body.Close() 45 | 46 | var products []*Product 47 | 48 | if err := handleJSONParse(body, &products); err != nil { 49 | return nil, err 50 | } 51 | productAll := &ProductAll{ 52 | Columns: products[0], 53 | Videos: products[1], 54 | } 55 | 56 | return productAll, nil 57 | } 58 | -------------------------------------------------------------------------------- /aeswrap/aeswrap.go: -------------------------------------------------------------------------------- 1 | package aeswrap 2 | 3 | import ( 4 | "bytes" 5 | "crypto/aes" 6 | "crypto/cipher" 7 | ) 8 | 9 | //@brief:填充明文 10 | func PKCS5Padding(plaintext []byte, blockSize int) []byte { 11 | padding := blockSize - len(plaintext)%blockSize 12 | padtext := bytes.Repeat([]byte{byte(padding)}, padding) 13 | return append(plaintext, padtext...) 14 | } 15 | 16 | //@brief:去除填充数据 17 | func PKCS5UnPadding(origData []byte) []byte { 18 | length := len(origData) 19 | unpadding := int(origData[length-1]) 20 | return origData[:(length - unpadding)] 21 | } 22 | 23 | //@brief:AES加密 24 | func AesEncrypt(origData, key []byte) ([]byte, error) { 25 | block, err := aes.NewCipher(key) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | //AES分组长度为128位，所以blockSize=16，单位字节 31 | blockSize := block.BlockSize() 32 | origData = PKCS5Padding(origData, blockSize) 33 | blockMode := cipher.NewCBCEncrypter(block, key[:blockSize]) //初始向量的长度必须等于块block的长度16字节 34 | crypted := make([]byte, len(origData)) 35 | blockMode.CryptBlocks(crypted, origData) 36 | return crypted, nil 37 | } 38 | 39 | //@brief:AES解密 40 | func AesDecrypt(crypted, key []byte) ([]byte, error) { 41 | block, err := aes.NewCipher(key) 42 | if err != nil { 43 | return nil, err 44 | } 45 | 46 | //AES分组长度为128位，所以blockSize=16，单位字节 47 | blockSize := block.BlockSize() 48 | blockMode := cipher.NewCBCDecrypter(block, key[:blockSize]) //初始向量的长度必须等于块block的长度16字节 49 | origData := make([]byte, len(crypted)) 50 | blockMode.CryptBlocks(origData, crypted) 51 | origData = PKCS5UnPadding(origData) 52 | return origData, nil 53 | } 54 | -------------------------------------------------------------------------------- /cli/cmds/course.go: -------------------------------------------------------------------------------- 1 | package cmds 2 | 3 | import ( 4 | "os" 5 | "strconv" 6 | "time" 7 | 8 | "github.com/olekukonko/tablewriter" 9 | "github.com/udoless/geektime-downloader/cli/application" 10 | "github.com/udoless/geektime-downloader/service" 11 | "github.com/urfave/cli" 12 | ) 13 | 14 | //NewCourseCommand login command 15 | func NewCourseCommand() []cli.Command { 16 | return []cli.Command{ 17 | { 18 | Name: "column", 19 | Usage: "获取专栏列表", 20 | UsageText: appName + " column", 21 | Action: columnAction, 22 | }, 23 | { 24 | Name: "video", 25 | Usage: "获取视频课程列表", 26 | UsageText: appName + " video", 27 | Action: videoAction, 28 | }, 29 | } 30 | } 31 | 32 | func columnAction(c *cli.Context) error { 33 | columns, err := application.Columns() 34 | 35 | if err != nil { 36 | return err 37 | } 38 | 39 | renderCourses(columns) 40 | 41 | return nil 42 | } 43 | 44 | func videoAction(c *cli.Context) error { 45 | videos, err := application.Videos() 46 | 47 | if err != nil { 48 | return err 49 | } 50 | 51 | renderCourses(videos) 52 | 53 | return nil 54 | } 55 | 56 | func renderCourses(courses []*service.Course) { 57 | table := tablewriter.NewWriter(os.Stdout) 58 | table.SetHeader([]string{"#", "ID", "名称", "时间", "作者", "购买"}) 59 | table.SetAutoWrapText(false) 60 | 61 | for i, p := range courses { 62 | isBuy := "" 63 | if p.HadSub { 64 | isBuy = "是" 65 | } 66 | table.Append([]string{strconv.Itoa(i), strconv.Itoa(p.ID), p.ColumnTitle, time.Unix(int64(p.ColumnCtime), 0).Format("2006-01-02"), p.AuthorName, isBuy}) 67 | } 68 | 69 | table.Render() 70 | } 71 | -------------------------------------------------------------------------------- /aeswrap/aeswrap_test.go: -------------------------------------------------------------------------------- 1 | package aeswrap 2 | 3 | import ( 4 | "encoding/base64" 5 | "fmt" 6 | "regexp" 7 | "testing" 8 | ) 9 | 10 | func TestAes128(t *testing.T) { 11 | //key的长度必须是16、24或者32字节，分别用于选择AES-128, AES-192, or AES-256 12 | var aeskey = []byte("12345678abcdefgh") 13 | pass := []byte("vdncloud123456") 14 | xpass, err := AesEncrypt(pass, aeskey) 15 | if err != nil { 16 | fmt.Println(err) 17 | return 18 | } 19 | 20 | pass64 := base64.StdEncoding.EncodeToString(xpass) 21 | fmt.Printf("加密后:%v\n", pass64) 22 | 23 | bytesPass, err := base64.StdEncoding.DecodeString(pass64) 24 | if err != nil { 25 | fmt.Println(err) 26 | return 27 | } 28 | 29 | tpass, err := AesDecrypt(bytesPass, aeskey) 30 | if err != nil { 31 | fmt.Println(err) 32 | return 33 | } 34 | fmt.Printf("解密后:%s\n", tpass) 35 | } 36 | 37 | func TestRegex(t *testing.T) { 38 | a := "#EXT-X-KEY:METHOD=AES-128,URI=\"https://misc.geekbang.org/serv/v1/decrypt/decryptkms/?Ciphertext=MjUyNWM3ZTQtZGM4Mi00NzI4LTg3YjAtMjk1YWE5N2ViMDg5Sjh2TzJ1NERYU0IyazlEVFdLbGZZZitYMjBhL202U01BQUFBQUFBQUFBQXY2Y2cxNWQxM2pMZ1crWE8vWDR5WFdHUngxNldLbVlPVEp3enlTdC84RnFPYlJhK0g0QWVL&MediaId=c4dddb5fe1a64490939f0ec2e46186ef&MtsHlsUriToken=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJjb2RlIjoiNDFlN2UzNzhlYjhjOGExYmZiODczODczNzQyYTIwZmZkNzIwN2ZhNCIsImV4cCI6MTY0OTIxNTc1MDQxMSwiZXh0cmEiOnsidmlkIjoiYzRkZGRiNWZlMWE2NDQ5MDkzOWYwZWMyZTQ2MTg2ZWYiLCJhaWQiOjIxODQsInVpZCI6MTI1Mjk0MCwicyI6IiJ9LCJzIjoyLCJ0IjoxLCJ2IjoxfQ.sacehfECZw3wyGj4kRFutptGOAT55hfq8aJJIOIkm3Y\"" 39 | re := regexp.MustCompile("#EXT-X-KEY:METHOD=AES-128,URI=\"(.*)\"") 40 | 41 | fmt.Printf("%s\n", re.FindStringSubmatch(a)[1]) 42 | } 43 | -------------------------------------------------------------------------------- /utils/ffmpeg.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "os" 7 | "os/exec" 8 | ) 9 | 10 | func runMergeCmd(cmd *exec.Cmd, paths []string, mergeFilePath string) error { 11 | var stderr bytes.Buffer 12 | cmd.Stderr = &stderr 13 | err := cmd.Run() 14 | if err != nil { 15 | return fmt.Errorf("%s\n%s", err, stderr.String()) 16 | } 17 | 18 | if mergeFilePath != "" { 19 | os.Remove(mergeFilePath) 20 | } 21 | // remove parts 22 | for _, path := range paths { 23 | os.Remove(path) 24 | } 25 | return nil 26 | } 27 | 28 | // MergeAudioAndVideo merge audio and video 29 | func MergeAudioAndVideo(paths []string, mergedFilePath string) error { 30 | cmds := []string{ 31 | "-y", 32 | } 33 | for _, path := range paths { 34 | cmds = append(cmds, "-i", path) 35 | } 36 | cmds = append( 37 | cmds, "-c:v", "copy", "-c:a", "copy", 38 | mergedFilePath, 39 | ) 40 | return runMergeCmd(exec.Command("ffmpeg", cmds...), paths, "") 41 | } 42 | 43 | // MergeToMP4 merge video parts to MP4 44 | func MergeToMP4(paths []string, mergedFilePath string, filename string) error { 45 | mergeFilePath := filename + ".txt" // merge list file should be in the current directory 46 | 47 | // write ffmpeg input file list 48 | mergeFile, _ := os.Create(mergeFilePath) 49 | for _, path := range paths { 50 | mergeFile.Write([]byte(fmt.Sprintf("file '%s'\n", path))) 51 | } 52 | mergeFile.Close() 53 | 54 | cmd := exec.Command( 55 | "ffmpeg", "-y", "-f", "concat", "-safe", "false", 56 | "-i", mergeFilePath, "-c", "copy", "-bsf:a", "aac_adtstoasc", mergedFilePath, 57 | ) 58 | return runMergeCmd(cmd, paths, mergeFilePath) 59 | } 60 | -------------------------------------------------------------------------------- /utils/utils_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "os" 5 | "runtime" 6 | "testing" 7 | ) 8 | 9 | func TestFilePath(t *testing.T) { 10 | type args struct { 11 | name string 12 | ext string 13 | escape bool 14 | } 15 | tests := []struct { 16 | name string 17 | args args 18 | want string 19 | }{ 20 | { 21 | name: "normal test", 22 | args: args{ 23 | name: "hello", 24 | ext: "txt", 25 | escape: false, 26 | }, 27 | want: "hello.txt", 28 | }, 29 | { 30 | name: "normal test", 31 | args: args{ 32 | name: "hello:world", 33 | ext: "txt", 34 | escape: true, 35 | }, 36 | want: "hello：world.txt", 37 | }, 38 | } 39 | for _, tt := range tests { 40 | t.Run(tt.name, func(t *testing.T) { 41 | if got, _ := FilePath(tt.args.name, tt.args.ext, tt.args.escape); got != tt.want { 42 | t.Errorf("FilePath() = %v, want %v", got, tt.want) 43 | } 44 | }) 45 | } 46 | } 47 | 48 | func TestMkdir(t *testing.T) { 49 | tests := []struct { 50 | name string 51 | path []string 52 | wantWin string 53 | wantUnix string 54 | }{ 55 | { 56 | name: "a", 57 | path: []string{"a"}, 58 | wantWin: "a", 59 | wantUnix: "a", 60 | }, 61 | } 62 | for _, tt := range tests { 63 | t.Run(tt.name, func(t *testing.T) { 64 | got, _ := Mkdir(tt.path...) 65 | 66 | if runtime.GOOS == "windows" { 67 | if got != tt.wantWin { 68 | t.Errorf("Mkdir() = %v, want %v", got, tt.wantWin) 69 | } 70 | } else { 71 | if got != tt.wantUnix { 72 | t.Errorf("Mkdir() = %v, want %v", got, tt.wantUnix) 73 | } 74 | } 75 | os.RemoveAll(got) 76 | }) 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /service/response_handler.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | 7 | "github.com/udoless/geektime-downloader/utils" 8 | ) 9 | 10 | type resultData []byte 11 | 12 | func (rd *resultData) UnmarshalJSON(data []byte) error { 13 | *rd = data 14 | 15 | return nil 16 | } 17 | 18 | func (rd resultData) String() string { 19 | return string(rd) 20 | } 21 | 22 | type resultError struct { 23 | Code int `json:"code"` 24 | Msg string `json:"msg"` 25 | } 26 | 27 | func (re *resultError) UnmarshalJSON(data []byte) error { 28 | str := string(data) 29 | if str == "[]" { 30 | str = "{}" 31 | } 32 | 33 | type rError resultError 34 | 35 | e := new(rError) 36 | err := utils.UnmarshalJSON([]byte(str), &e) 37 | if err != nil { 38 | return err 39 | } 40 | 41 | *re = resultError(*e) 42 | 43 | return nil 44 | } 45 | 46 | // Result 从百度服务器解析的数据结构 47 | type Result struct { 48 | Code int `json:"code"` 49 | Data resultData `json:"data"` 50 | Error resultError `json:"error"` 51 | // Extra struct { 52 | // Cost float64 `json:"cost"` 53 | // RequestID string `json:"request-id"` 54 | // } `json:"extra"` 55 | } 56 | 57 | func (r *Result) isSuccess() bool { 58 | return r.Code == 0 59 | } 60 | 61 | func handleJSONParse(reader io.Reader, v interface{}) Error { 62 | result := new(Result) 63 | 64 | // b, _ := ioutil.ReadAll(reader) 65 | // a := string(b) 66 | // fmt.Println(a) 67 | err := utils.UnmarshalReader(reader, &result) 68 | if err != nil { 69 | return &ErrorInfo{Err: err} 70 | } 71 | 72 | if !result.isSuccess() { 73 | //未登录或者登录凭证无效 74 | if result.Error.Code == -3050 || result.Error.Code == -2000 { 75 | return &ErrorInfo{Err: ErrNotLogin} 76 | } 77 | return &ErrorInfo{Err: errors.New(result.Error.Msg)} 78 | } 79 | 80 | err = utils.UnmarshalJSON(result.Data, v) 81 | if err != nil { 82 | return &ErrorInfo{Err: err} 83 | } 84 | 85 | return nil 86 | } 87 | -------------------------------------------------------------------------------- /cli/application/course.go: -------------------------------------------------------------------------------- 1 | package application 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/udoless/geektime-downloader/service" 7 | ) 8 | 9 | //Columns 专栏列表 10 | func Columns() ([]*service.Course, error) { 11 | return getService().Columns() 12 | } 13 | 14 | //Videos 视频课程列表 15 | func Videos() ([]*service.Course, error) { 16 | return getService().Videos() 17 | } 18 | 19 | //CourseWithArticles course and articles info 20 | func CourseWithArticles(id int) (*service.Course, []*service.Article, error) { 21 | course, err := getService().ShowCourse(id) 22 | if err != nil { 23 | return nil, nil, err 24 | } 25 | 26 | course.ColumnTitle = strings.TrimSpace(course.ColumnTitle) 27 | 28 | articles, err := getService().Articles(id) 29 | if err != nil { 30 | return course, nil, err 31 | } 32 | 33 | return course, articles, nil 34 | } 35 | 36 | //GetVideoPlayInfo 获取视频播放信息 37 | func GetVideoPlayInfo(aid int, videoID string) (*service.VideoPlayInfo, error) { 38 | videoPlayAuth, err := VideoPlayAuth(aid, videoID) 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | videoPlayInfo, err := VideoPlayInfo(videoPlayAuth.PlayAuth) 44 | if err != nil { 45 | return nil, err 46 | } 47 | return videoPlayInfo, nil 48 | } 49 | 50 | //VideoPlayAuth 获取视频的播放授权信息 51 | func VideoPlayAuth(aid int, videoID string) (*service.VideoPlayAuth, error) { 52 | videoPlayAuth, err := getService().VideoPlayAuth(aid, videoID) 53 | 54 | if err != nil { 55 | return nil, err 56 | } 57 | 58 | return videoPlayAuth, nil 59 | } 60 | 61 | //VideoPlayInfo 获取视频播放信息 62 | func VideoPlayInfo(playAuth string) (*service.VideoPlayInfo, error) { 63 | videoPlayInfo, err := getService().VideoPlayInfo(playAuth) 64 | 65 | if err != nil { 66 | return nil, err 67 | } 68 | 69 | return videoPlayInfo, nil 70 | } 71 | 72 | //VideoPlayInfo 获取视频播放信息 73 | func V3ArticleInfo(aid int) (*service.V3ArticleInfo, error) { 74 | v3ArticleInfo, err := getService().V3ArticleInfo(aid) 75 | 76 | if err != nil { 77 | return nil, err 78 | } 79 | 80 | return v3ArticleInfo, nil 81 | } 82 | -------------------------------------------------------------------------------- /config/geek.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/udoless/geektime-downloader/service" 7 | ) 8 | 9 | //User geek time user info 10 | type User struct { 11 | ID int `json:"id"` 12 | Name string `json:"name"` 13 | Avatar string `json:"avatar"` 14 | } 15 | 16 | //Geektime geek time info 17 | type Geektime struct { 18 | User 19 | PHONE string `json:"phone"` 20 | PASSWORD string `json:"password"` 21 | GCID string `json:"gcid"` 22 | GCESS string `json:"gcess"` 23 | ServerID string `json:"serverId"` 24 | Ticket string `json:"ticket"` 25 | CookieString string `json:"cookieString"` 26 | } 27 | 28 | //Service geek time service 29 | func (g *Geektime) Service() *service.Service { 30 | ser := service.NewService(g.GCID, g.GCESS, g.ServerID) 31 | 32 | return ser 33 | } 34 | 35 | //SetUserByGcidAndGcess set user 36 | func (c *ConfigsData) SetUserByGcidAndGcess(gcid, gcess, serverID, phone, password string) (*Geektime, error) { 37 | ser := service.NewService(gcid, gcess, serverID) 38 | user, err := ser.User() 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | c.DeleteUser(&User{ID: user.UID}) 44 | 45 | geektime := &Geektime{ 46 | User: User{ 47 | ID: user.UID, 48 | Name: user.Nickname, 49 | Avatar: user.Avatar, 50 | }, 51 | PHONE: phone, 52 | PASSWORD: password, 53 | GCID: gcid, 54 | GCESS: gcess, 55 | ServerID: serverID, 56 | } 57 | 58 | c.Geektimes = append(c.Geektimes, geektime) 59 | // c.service = ser 60 | 61 | c.setActiveUser(geektime) 62 | 63 | return geektime, nil 64 | } 65 | 66 | //DeleteUser delete 67 | func (c *ConfigsData) DeleteUser(u *User) { 68 | for k, gk := range c.Geektimes { 69 | if gk.ID == u.ID { 70 | c.Geektimes = append(c.Geektimes[:k], c.Geektimes[k+1:]...) 71 | break 72 | } 73 | } 74 | } 75 | 76 | func (c *ConfigsData) setActiveUser(g *Geektime) { 77 | c.AcitveUID = g.ID 78 | c.activeUser = g 79 | } 80 | 81 | //LoginUserCount 登录用户数量 82 | func (c *ConfigsData) LoginUserCount() int { 83 | return len(c.Geektimes) 84 | } 85 | 86 | //SwitchUser switch user 87 | func (c *ConfigsData) SwitchUser(u *User) error { 88 | for _, g := range c.Geektimes { 89 | if g.ID == u.ID { 90 | c.setActiveUser(g) 91 | return nil 92 | } 93 | } 94 | 95 | return errors.New("用户不存在") 96 | } 97 | -------------------------------------------------------------------------------- /downloader/types.go: -------------------------------------------------------------------------------- 1 | package downloader 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "regexp" 7 | "sort" 8 | "strconv" 9 | 10 | "github.com/olekukonko/tablewriter" 11 | ) 12 | 13 | //URL for url information 14 | type URL struct { 15 | URL string `json:"url"` 16 | Size int `json:"size"` 17 | Ext string `json:"ext"` 18 | } 19 | 20 | //Data 课程信息 21 | type Data struct { 22 | Title string `json:"title"` 23 | Type string `json:"type"` 24 | Data []Datum `json:"articles"` 25 | } 26 | 27 | //Datum download information 28 | type Datum struct { 29 | ID int `json:"id"` 30 | Title string `json:"title"` 31 | Type string `json:"type"` 32 | IsCanDL bool `json:"is_can_dl"` 33 | 34 | Streams map[string]Stream `json:"streams"` 35 | sortedStreams []Stream 36 | } 37 | 38 | //Stream data 39 | type Stream struct { 40 | URLs []URL `json:"urls"` 41 | AesKeyBytes []byte 42 | Size int `json:"size"` 43 | Quality string `json:"quality"` 44 | name string 45 | } 46 | 47 | //VideoMediaMap 视频大小信息 48 | type VideoMediaMap struct { 49 | Size int `json:"size"` 50 | } 51 | 52 | //EmptyData empty data list 53 | var EmptyData = make([]Datum, 0) 54 | 55 | //PrintInfo print info 56 | func (data *Data) PrintInfo() { 57 | if len(data.Data) == 0 { 58 | fmt.Println(data.Type + "目录为空") 59 | return 60 | } 61 | 62 | table := tablewriter.NewWriter(os.Stdout) 63 | 64 | header := []string{"#", "ID", "类型", "名称"} 65 | for key := range data.Data[0].Streams { 66 | header = append(header, key) 67 | } 68 | header = append(header, "下载") 69 | 70 | table.SetHeader(header) 71 | table.SetAutoWrapText(false) 72 | i := 0 73 | for _, p := range data.Data { 74 | reg, _ := regexp.Compile(" \\| ") 75 | title := reg.ReplaceAllString(p.Title, " ") 76 | 77 | isCanDL := "" 78 | if p.IsCanDL { 79 | isCanDL = " ✔" 80 | } 81 | 82 | value := []string{strconv.Itoa(i), strconv.Itoa(p.ID), p.Type, title} 83 | 84 | if len(p.Streams) > 0 { 85 | for _, stream := range p.Streams { 86 | value = append(value, fmt.Sprintf("%.2fM", float64(stream.Size)/1024/1024)) 87 | } 88 | } else { 89 | for range data.Data[0].Streams { 90 | value = append(value, " -") 91 | } 92 | } 93 | 94 | value = append(value, isCanDL) 95 | 96 | table.Append(value) 97 | i++ 98 | } 99 | table.Render() 100 | } 101 | 102 | func (v *Datum) genSortedStreams() { 103 | for k, data := range v.Streams { 104 | if data.Size == 0 { 105 | data.calculateTotalSize() 106 | } 107 | data.name = k 108 | v.Streams[k] = data 109 | v.sortedStreams = append(v.sortedStreams, data) 110 | } 111 | if len(v.Streams) > 1 { 112 | sort.Slice( 113 | v.sortedStreams, func(i, j int) bool { return v.sortedStreams[i].Size > v.sortedStreams[j].Size }, 114 | ) 115 | } 116 | } 117 | 118 | func (stream *Stream) calculateTotalSize() { 119 | 120 | if stream.Size > 0 { 121 | return 122 | } 123 | 124 | size := 0 125 | for _, url := range stream.URLs { 126 | size += url.Size 127 | } 128 | stream.Size = size 129 | } 130 | -------------------------------------------------------------------------------- /service/requester.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "io" 5 | "net/url" 6 | ) 7 | 8 | //获取用户信息 9 | func (s *Service) requestUser() (io.ReadCloser, Error) { 10 | res, err := s.client.Req("POST", "https://account.geekbang.org/account/user", nil, map[string]string{"Origin": "https://account.geekbang.org"}) 11 | return handleHTTPResponse(res, err) 12 | } 13 | 14 | //所有购买的课程 15 | func (s *Service) requestBuyAll() (io.ReadCloser, Error) { 16 | res, err := s.client.Req("POST", "https://time.geekbang.org/serv/v1/my/products/all", nil, map[string]string{"Origin": "https://account.geekbang.org"}) 17 | return handleHTTPResponse(res, err) 18 | } 19 | 20 | //所有课程 21 | func (s *Service) requestCourses(couseType int) (io.ReadCloser, Error) { 22 | res, err := s.client.Req("POST", "https://time.geekbang.org/serv/v1/column/newAll", map[string]int{"type": couseType}, map[string]string{"Origin": "https://time.geekbang.org"}) 23 | return handleHTTPResponse(res, err) 24 | } 25 | 26 | //获取课程信息 27 | func (s *Service) requestCourseDetail(ids []int) (io.ReadCloser, Error) { 28 | ii := map[string]interface{}{"ids": ids} 29 | res, err := s.client.Req("POST", "https://time.geekbang.org/serv/v1/column/details", ii, map[string]string{"Origin": "https://time.geekbang.org"}) 30 | return handleHTTPResponse(res, err) 31 | } 32 | 33 | //课程详细信息 34 | func (s *Service) requestCourseIntro(id int) (io.ReadCloser, Error) { 35 | res, err := s.client.Req("POST", "https://time.geekbang.org/serv/v1/column/intro", map[string]interface{}{"cid": id, "with_groupbuy": true}, map[string]string{"Origin": "https://time.geekbang.org"}) 36 | return handleHTTPResponse(res, err) 37 | } 38 | 39 | //课程的文章列表信息 40 | func (s *Service) requestCourseArticles(id int) (io.ReadCloser, Error) { 41 | data := map[string]interface{}{ 42 | "cid": id, 43 | "order": "earliest", 44 | "prev": 0, 45 | "sample": false, 46 | "size": 500, 47 | } 48 | res, err := s.client.Req("POST", "https://time.geekbang.org/serv/v1/column/articles", data, map[string]string{"Origin": "https://time.geekbang.org"}) 49 | return handleHTTPResponse(res, err) 50 | } 51 | 52 | //获取文章信息 53 | func (s *Service) requestArticleInfo(aid int) (io.ReadCloser, Error) { 54 | data := map[string]interface{}{ 55 | "id": aid, 56 | } 57 | res, err := s.client.Req("POST", "https://time.geekbang.org/serv/v3/article/info", data, map[string]string{"Origin": "https://time.geekbang.org"}) 58 | return handleHTTPResponse(res, err) 59 | } 60 | 61 | //获取视频的播放授权信息 62 | func (s *Service) requestVideoPlayAuth(aid int, videoID string) (io.ReadCloser, Error) { 63 | data := map[string]interface{}{ 64 | "source_type": 1, 65 | "aid": aid, 66 | "video_id": videoID, 67 | } 68 | res, err := s.client.Req("POST", "https://time.geekbang.org/serv/v3/source_auth/video_play_auth", data, map[string]string{"Origin": "https://time.geekbang.org"}) 69 | return handleHTTPResponse(res, err) 70 | } 71 | 72 | //获取视频的播放信息 73 | func (s *Service) requestVideoPlayInfo(playAuth string) (io.ReadCloser, Error) { 74 | res, err := s.client.Req("GET", "http://ali.mantv.top/play/info?playAuth="+url.QueryEscape(playAuth), nil, nil) 75 | return handleHTTPResponse(res, err) 76 | } 77 | -------------------------------------------------------------------------------- /utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net/url" 7 | "os" 8 | "path/filepath" 9 | "regexp" 10 | "runtime" 11 | "strings" 12 | 13 | "github.com/udoless/geektime-downloader/requester" 14 | ) 15 | 16 | // MAXLENGTH Maximum length of file name 17 | const MAXLENGTH = 80 18 | 19 | //FileName filter invalid string 20 | func FileName(name string, ext string) string { 21 | rep := strings.NewReplacer("\n", " ", "/", " ", "|", "-", ": ", "：", ":", "：", "'", "’", "\t", " ") 22 | name = rep.Replace(name) 23 | 24 | if runtime.GOOS == "windows" { 25 | rep := strings.NewReplacer("\"", " ", "?", " ", "*", " ", "\\", " ", "<", " ", ">", " ", ":", " ", "：", " ") 26 | name = rep.Replace(name) 27 | } 28 | 29 | name = strings.TrimSpace(name) 30 | 31 | limitedName := LimitLength(name, MAXLENGTH) 32 | if ext != "" { 33 | return fmt.Sprintf("%s.%s", limitedName, ext) 34 | } 35 | return limitedName 36 | } 37 | 38 | //LimitLength cut string 39 | func LimitLength(s string, length int) string { 40 | ellipses := "..." 41 | 42 | str := []rune(s) 43 | if len(str) > length { 44 | s = string(str[:length-len(ellipses)]) + ellipses 45 | } 46 | 47 | return s 48 | } 49 | 50 | // FilePath gen valid file path 51 | func FilePath(name, ext string, escape bool) (string, error) { 52 | var outputPath string 53 | 54 | var fileName string 55 | if escape { 56 | fileName = FileName(name, ext) 57 | } else { 58 | fileName = fmt.Sprintf("%s.%s", name, ext) 59 | } 60 | outputPath = filepath.Join(fileName) 61 | return outputPath, nil 62 | } 63 | 64 | //Mkdir mkdir path 65 | func Mkdir(elem ...string) (string, error) { 66 | path := filepath.Join(elem...) 67 | 68 | err := os.MkdirAll(path, os.ModePerm) 69 | 70 | return path, err 71 | } 72 | 73 | // FileSize return the file size of the specified path file 74 | func FileSize(filePath string) (int, bool, error) { 75 | file, err := os.Stat(filePath) 76 | if err != nil { 77 | if os.IsNotExist(err) { 78 | return 0, false, nil 79 | } 80 | return 0, false, err 81 | } 82 | return int(file.Size()), true, nil 83 | } 84 | 85 | var aesRegex = regexp.MustCompile("#EXT-X-KEY:METHOD=AES-128,URI=\"(.*)\"") 86 | 87 | // M3u8URLs get all ts urls from m3u8 url 88 | func M3u8URLsAndAesKey(uri string) ([]string, []byte, error) { 89 | if len(uri) == 0 { 90 | return nil, nil, errors.New("M3u8地址为空") 91 | } 92 | 93 | html, err := requester.HTTPGet(uri) 94 | if err != nil { 95 | return nil, nil, err 96 | } 97 | lines := strings.Split(string(html), "\n") 98 | var urls []string 99 | var aesBytes []byte 100 | for _, line := range lines { 101 | line = strings.TrimSpace(line) 102 | if strings.HasPrefix(line, "#EXT-X-KEY") { 103 | aesApi := aesRegex.FindStringSubmatch(line)[1] 104 | aesBytes, err = requester.HTTPGet(aesApi) 105 | if err != nil { 106 | return nil, nil, err 107 | } 108 | continue 109 | } 110 | if line != "" && !strings.HasPrefix(line, "#") { 111 | if strings.HasPrefix(line, "http") { 112 | urls = append(urls, line) 113 | } else { 114 | base, err := url.Parse(uri) 115 | if err != nil { 116 | continue 117 | } 118 | u, err := url.Parse(line) 119 | if err != nil { 120 | continue 121 | } 122 | urls = append(urls, fmt.Sprintf("%s", base.ResolveReference(u))) 123 | } 124 | } 125 | } 126 | return urls, aesBytes, nil 127 | } 128 | -------------------------------------------------------------------------------- /cli/cmds/cmds.go: -------------------------------------------------------------------------------- 1 | package cmds 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "time" 9 | 10 | "github.com/sirupsen/logrus" 11 | 12 | "github.com/udoless/geektime-downloader/cli/version" 13 | "github.com/udoless/geektime-downloader/config" 14 | "github.com/udoless/geektime-downloader/utils" 15 | "github.com/urfave/cli" 16 | ) 17 | 18 | var ( 19 | _debug bool 20 | _info bool 21 | _stream string 22 | _pdf bool 23 | _mp3 bool 24 | appName = filepath.Base(os.Args[0]) 25 | configSaveFunc = func(c *cli.Context) error { 26 | err := config.Instance.Save() 27 | if err != nil { 28 | return errors.New("保存配置错误：" + err.Error()) 29 | } 30 | return nil 31 | } 32 | authorizationFunc = func(c *cli.Context) error { 33 | if config.Instance.AcitveUID <= 0 { 34 | if len(config.Instance.Geektimes) > 0 { 35 | return config.ErrHasLoginedNotLogin 36 | } 37 | return config.ErrNotLogin 38 | } 39 | 40 | return nil 41 | } 42 | ) 43 | 44 | //NewApp cli app 45 | func NewApp() *cli.App { 46 | app := cli.NewApp() 47 | app.Name = appName 48 | app.Usage = "极客时间下载客户端" 49 | app.Version = fmt.Sprintf("%s", version.Version) 50 | cli.VersionPrinter = func(c *cli.Context) { 51 | fmt.Printf("%s version %s\n", app.Name, app.Version) 52 | } 53 | app.Flags = []cli.Flag{ 54 | cli.BoolFlag{ 55 | Name: "debug,d", 56 | Usage: "Turn on debug logs", 57 | Destination: &_debug, 58 | }, 59 | cli.BoolFlag{ 60 | Name: "info, i", 61 | Usage: "只输出视频信息", 62 | Destination: &_info, 63 | }, 64 | cli.StringFlag{ 65 | Name: "stream, s", 66 | Usage: "选择要下载的指定类型", 67 | Destination: &_stream, 68 | }, 69 | cli.BoolFlag{ 70 | Name: "pdf, p", 71 | Usage: "下载专栏PDF文档", 72 | Destination: &_pdf, 73 | }, 74 | cli.BoolFlag{ 75 | Name: "mp3, m", 76 | Usage: "下载专栏MP3音频", 77 | Destination: &_mp3, 78 | }, 79 | } 80 | 81 | app.Before = func(c *cli.Context) error { 82 | if _debug { 83 | logrus.SetLevel(logrus.DebugLevel) 84 | } 85 | return nil 86 | } 87 | 88 | return app 89 | } 90 | 91 | //DefaultAction default action 92 | func DefaultAction(c *cli.Context) error { 93 | _, err := config.Instance.ActiveUserService().User() 94 | if err != nil { 95 | return err 96 | } 97 | loginCh := make(chan struct{}) 98 | loginRespCh := make(chan struct{}) 99 | go func() { 100 | for { 101 | select { 102 | case <-loginCh: 103 | fmt.Println("Relogin....") 104 | err := loginByPhoneAndPassword(config.Instance.ActiveUser().PHONE, config.Instance.ActiveUser().PASSWORD) 105 | if err != nil { 106 | fmt.Printf("login failed: %s\n", err) 107 | continue 108 | } 109 | err = configSaveFunc(c) 110 | if err != nil { 111 | fmt.Printf("config save failed: %s\n", err) 112 | continue 113 | } 114 | err = config.Instance.Init() 115 | if err != nil { 116 | fmt.Printf("config reinit failed: %s\n", err) 117 | continue 118 | } 119 | loginRespCh <- struct{}{} 120 | } 121 | time.Sleep(3 * time.Second) 122 | } 123 | }() 124 | utils.LoginCh = loginCh 125 | utils.LoginRespCh = loginRespCh 126 | 127 | if len(c.Args()) == 0 { 128 | cli.ShowAppHelp(c) 129 | return nil 130 | } 131 | 132 | dlc := &NewDownloadCommand()[0] 133 | if dlc != nil { 134 | return dlc.Run(c) 135 | } 136 | 137 | return nil 138 | } 139 | -------------------------------------------------------------------------------- /login/login_client.go: -------------------------------------------------------------------------------- 1 | package login 2 | 3 | import ( 4 | "net/http/cookiejar" 5 | "net/url" 6 | "regexp" 7 | "strings" 8 | 9 | jsoniter "github.com/json-iterator/go" 10 | "github.com/udoless/geektime-downloader/requester" 11 | ) 12 | 13 | //Client login client 14 | type Client struct { 15 | *requester.HTTPClient 16 | } 17 | 18 | // Result 从百度服务器解析的数据结构 19 | type Result struct { 20 | Code int `json:"code"` 21 | Data struct { 22 | UID int `json:"uid"` 23 | Name string `json:"nickname"` 24 | Avatar string `json:"avatar"` 25 | GCID string `json:"gcid"` 26 | GCESS string `json:"gcess"` 27 | ServerID string `json:"serverId"` 28 | Ticket string `json:"ticket"` 29 | CookieString string `json:"cookieString"` 30 | } `json:"data"` 31 | Error struct { 32 | Code int `json:"code"` 33 | Msg string `json:"msg"` 34 | } `json:"error"` 35 | Extra struct { 36 | Cost float64 `json:"cost"` 37 | RequestID string `json:"request-id"` 38 | } `json:"extra"` 39 | } 40 | 41 | //NewLoginClient new login client 42 | func NewLoginClient() *Client { 43 | c := &Client{ 44 | HTTPClient: requester.NewHTTPClient(), 45 | } 46 | 47 | c.InitLoginPage() 48 | 49 | return c 50 | } 51 | 52 | //InitLoginPage init 53 | func (c *Client) InitLoginPage() { 54 | res, _ := c.Get("https://account.geekbang.org/signin?redirect=https%3A%2F%2Ftime.geekbang.org%2F") 55 | defer (func() { 56 | if res != nil { 57 | res.Body.Close() 58 | } 59 | })() 60 | } 61 | 62 | //Login by phone and dpassword 63 | func (c *Client) Login(phone, password string) *Result { 64 | result := &Result{} 65 | post := map[string]string{ 66 | "country": "86", 67 | "cellphone": phone, 68 | "password": password, 69 | // "captcha": "", 70 | "remeber": "1", 71 | "platform": "3", 72 | "appid": "1", 73 | } 74 | 75 | header := map[string]string{ 76 | "Referer": "https://account.geekbang.org/signin?redirect=https%3A%2F%2Ftime.geekbang.org%2F", 77 | "Accept": "application/json", 78 | "Connection": "keep-alive", 79 | } 80 | body, err := c.Fetch("POST", "https://account.geekbang.org/account/ticket/login", post, header) 81 | if err != nil { 82 | result.Code = -1 83 | result.Error.Code = -1 84 | result.Error.Msg = "网络请求失败, " + err.Error() 85 | 86 | return result 87 | } 88 | 89 | rex, _ := regexp.Compile("\\[\\]") 90 | body = rex.ReplaceAll(body, []byte("{}")) 91 | 92 | if err = jsoniter.Unmarshal(body, &result); err != nil { 93 | result.Code = -1 94 | result.Error.Code = -1 95 | result.Error.Msg = "发送登录请求错误: " + err.Error() 96 | 97 | return result 98 | } 99 | 100 | if result.IsLoginSuccess() { 101 | result.parseCookies("https://account.geekbang.org", c.Jar.(*cookiejar.Jar)) 102 | } 103 | 104 | return result 105 | } 106 | 107 | //parseCookies 解析cookie 108 | func (r *Result) parseCookies(targetURL string, jar *cookiejar.Jar) { 109 | url, _ := url.Parse(targetURL) 110 | cookies := jar.Cookies(url) 111 | 112 | cookieArr := []string{} 113 | for _, cookie := range cookies { 114 | switch cookie.Name { 115 | case "GCID": 116 | r.Data.GCID = cookie.Value 117 | case "GCESS": 118 | r.Data.GCESS = cookie.Value 119 | case "SERVERID": 120 | r.Data.ServerID = cookie.Value 121 | } 122 | cookieArr = append(cookieArr, cookie.String()) 123 | } 124 | r.Data.CookieString = strings.Join(cookieArr, ";") 125 | } 126 | 127 | //IsLoginSuccess 是否登陆成功 128 | func (r *Result) IsLoginSuccess() bool { 129 | return r.Code == 0 130 | } 131 | -------------------------------------------------------------------------------- /service/course.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "github.com/udoless/geektime-downloader/utils" 5 | ) 6 | 7 | //Columns 获取专栏 8 | func (s *Service) Columns() ([]*Course, error) { 9 | return s.getCourses(1) 10 | } 11 | 12 | //Videos 获取专栏 13 | func (s *Service) Videos() ([]*Course, error) { 14 | return s.getCourses(3) 15 | } 16 | 17 | //获取课程信息 18 | func (s *Service) getCourses(courseType int) ([]*Course, error) { 19 | ids, err := s.courses(courseType) 20 | 21 | if err != nil { 22 | return nil, err 23 | } 24 | 25 | return s.getCourseDetail(ids) 26 | } 27 | 28 | func (s *Service) courses(courseType int) ([]int, error) { 29 | body, err := s.requestCourses(courseType) 30 | 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | defer body.Close() 36 | 37 | courses := new(CourseList) 38 | if err := handleJSONParse(body, &courses); err != nil { 39 | return nil, err 40 | } 41 | 42 | var ids []int 43 | for _, item := range courses.List { 44 | ids = append(ids, item.ID) 45 | } 46 | 47 | return ids, nil 48 | } 49 | 50 | func (s *Service) getCourseDetail(ids []int) ([]*Course, error) { 51 | body, err := s.requestCourseDetail(ids) 52 | if err != nil { 53 | return nil, err 54 | } 55 | 56 | defer body.Close() 57 | 58 | var courses []*Course 59 | if err := handleJSONParse(body, &courses); err != nil { 60 | return nil, err 61 | } 62 | 63 | return courses, nil 64 | } 65 | 66 | //ShowCourse 获取课程信息 67 | func (s *Service) ShowCourse(id int) (*Course, error) { 68 | body, err := s.requestCourseIntro(id) 69 | if err != nil { 70 | return nil, err 71 | } 72 | defer body.Close() 73 | 74 | course := new(Course) 75 | if err := handleJSONParse(body, &course); err != nil { 76 | return nil, err 77 | } 78 | 79 | return course, nil 80 | } 81 | 82 | //Articles get course articles 83 | func (s *Service) Articles(id int) ([]*Article, error) { 84 | body, err := s.requestCourseArticles(id) 85 | if err != nil { 86 | return nil, err 87 | } 88 | defer body.Close() 89 | 90 | articleResult := &articleResult{} 91 | if err := handleJSONParse(body, articleResult); err != nil { 92 | return nil, err 93 | } 94 | 95 | return articleResult.Articles, nil 96 | } 97 | 98 | //VideoPlayAuth 获取视频的播放授权信息 99 | func (s *Service) VideoPlayAuth(aid int, videoID string) (*VideoPlayAuth, error) { 100 | body, err := s.requestVideoPlayAuth(aid, videoID) 101 | if err != nil { 102 | return nil, err 103 | } 104 | defer body.Close() 105 | 106 | videoPlayAuth := &VideoPlayAuth{} 107 | if err := handleJSONParse(body, videoPlayAuth); err != nil { 108 | return nil, err 109 | } 110 | 111 | return videoPlayAuth, nil 112 | } 113 | 114 | //VideoPlayInfo 获取视频播放信息 115 | func (s *Service) VideoPlayInfo(playAuth string) (*VideoPlayInfo, error) { 116 | body, err := s.requestVideoPlayInfo(playAuth) 117 | if err != nil { 118 | return nil, err 119 | } 120 | defer body.Close() 121 | 122 | videoPlayInfo := &VideoPlayInfo{} 123 | if err := utils.UnmarshalReader(body, &videoPlayInfo); err != nil { 124 | return nil, err 125 | } 126 | 127 | return videoPlayInfo, nil 128 | } 129 | 130 | //ArticleInfo 获取文章信息 131 | func (s *Service) V3ArticleInfo(aid int) (*V3ArticleInfo, error) { 132 | body, err := s.requestArticleInfo(aid) 133 | if err != nil { 134 | return nil, err 135 | } 136 | defer body.Close() 137 | 138 | v3ArticleInfo := &V3ArticleInfo{} 139 | if err := utils.UnmarshalReader(body, &v3ArticleInfo); err != nil { 140 | return nil, err 141 | } 142 | 143 | return v3ArticleInfo, nil 144 | } 145 | -------------------------------------------------------------------------------- /requester/fetch.go: -------------------------------------------------------------------------------- 1 | package requester 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "net/http" 11 | "strconv" 12 | "strings" 13 | "time" 14 | ) 15 | 16 | //HTTPGet 简单实现 http 访问 GET 请求 17 | func HTTPGet(urlStr string) ([]byte, error) { 18 | res, err := DefaultClient.Get(urlStr) 19 | 20 | if res != nil { 21 | defer res.Body.Close() 22 | } 23 | 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | return ioutil.ReadAll(res.Body) 29 | } 30 | 31 | // Req 参见 *HTTPClient.Req, 使用默认 http 客户端 32 | func Req(method string, urlStr string, post interface{}, header map[string]string) (*http.Response, error) { 33 | return DefaultClient.Req(method, urlStr, post, header) 34 | } 35 | 36 | // Fetch 参见 *HTTPClient.Fetch, 使用默认 http 客户端 37 | func Fetch(method string, urlStr string, post interface{}, header map[string]string) ([]byte, error) { 38 | return DefaultClient.Fetch(method, urlStr, post, header) 39 | } 40 | 41 | // Headers return the HTTP Headers of the url 42 | func Headers(url string) (http.Header, error) { 43 | res, err := Req(http.MethodGet, url, nil, nil) 44 | if err != nil { 45 | return nil, err 46 | } 47 | return res.Header, nil 48 | } 49 | 50 | // Size get size of the url 51 | func Size(url string) (int, error) { 52 | h, err := Headers(url) 53 | if err != nil { 54 | return 0, err 55 | } 56 | s := h.Get("Content-Length") 57 | if s == "" { 58 | return 0, errors.New("Content-Length is not present") 59 | } 60 | size, err := strconv.Atoi(s) 61 | if err != nil { 62 | return 0, err 63 | } 64 | return size, nil 65 | } 66 | 67 | // Req 实现 http／https 访问， 68 | // 根据给定的 method (GET, POST, HEAD, PUT 等等), 69 | // urlStr (网址), 70 | // post (post 数据), 71 | // header (header 请求头数据), 进行网站访问。 72 | // 返回值分别为 *http.Response, 错误信息 73 | func (h *HTTPClient) Req(method string, urlStr string, post interface{}, header map[string]string) (*http.Response, error) { 74 | var ( 75 | req *http.Request 76 | obody io.Reader 77 | ) 78 | 79 | if post != nil { 80 | switch value := post.(type) { 81 | case io.Reader: 82 | obody = value 83 | case map[string]string, map[string]int, map[string]interface{}, []int, []string: 84 | postData, err := json.Marshal(value) 85 | if err != nil { 86 | return nil, err 87 | } 88 | header["Content-Type"] = "application/json" 89 | obody = bytes.NewReader(postData) 90 | case string: 91 | obody = strings.NewReader(value) 92 | case []byte: 93 | obody = bytes.NewReader(value) 94 | default: 95 | return nil, fmt.Errorf("request.Req: unknow post type: %s", post) 96 | } 97 | } 98 | 99 | req, err := http.NewRequest(method, urlStr, obody) 100 | if err != nil { 101 | return nil, err 102 | } 103 | 104 | // 设置浏览器标识 105 | req.Header.Set("User-Agent", UserAgent) 106 | 107 | if header != nil { 108 | for k, v := range header { 109 | req.Header.Set(k, v) 110 | } 111 | } 112 | 113 | h.SetTimeout(20 * time.Minute) 114 | 115 | return h.Client.Do(req) 116 | } 117 | 118 | // Fetch 实现 http／https 访问， 119 | // 根据给定的 method (GET, POST, HEAD, PUT 等等), 120 | // urlStr (网址), 121 | // post (post 数据), 122 | // header (header 请求头数据), 进行网站访问。 123 | // 返回值分别为 []byte, 错误信息 124 | func (h *HTTPClient) Fetch(method string, urlStr string, post interface{}, header map[string]string) ([]byte, error) { 125 | res, err := h.Req(method, urlStr, post, header) 126 | 127 | if res != nil { 128 | defer res.Body.Close() 129 | } 130 | 131 | if err != nil { 132 | return nil, err 133 | } 134 | 135 | return ioutil.ReadAll(res.Body) 136 | } 137 | -------------------------------------------------------------------------------- /service/course_types.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import "encoding/json" 4 | 5 | //CourseList 课程列表基础信息 6 | type CourseList struct { 7 | List []struct { 8 | ID int `json:"id"` 9 | ColumnCtime int `json:"column_ctime"` 10 | ColumnGroupbuy int `json:"column_groupbuy"` 11 | ColumnPrice int `json:"column_price"` 12 | ColumnPriceMarket int `json:"column_price_market"` 13 | ColumnSku int `json:"column_sku"` 14 | ColumnType int `json:"column_type"` 15 | HadSub bool `json:"had_sub"` 16 | IsChannel int `json:"is_channel"` 17 | IsExperience bool `json:"is_experience"` 18 | LastAid int `json:"last_aid"` 19 | LastChapterID int `json:"last_chapter_id"` 20 | PriceType int `json:"price_type"` 21 | SubCount int `json:"sub_count"` 22 | } `json:"list"` 23 | } 24 | 25 | //Course 课程信息 26 | type Course struct { 27 | ID int `json:"id"` 28 | Authorintro string `json:"author_intro"` 29 | AuthorName string `json:"author_name"` 30 | ChannelBackAmount int `json:"channel_back_amount"` 31 | ColumnBgcolor string `json:"column_bgcolor"` 32 | ColumnCover string `json:"column_cover"` 33 | ColumnCtime int `json:"column_ctime"` 34 | ColumnPrice int `json:"column_price"` 35 | ColumnPriceMarket int `json:"column_price_market"` 36 | ColumnPriceSale int `json:"column_price_sale"` 37 | ColumnSku int `json:"column_sku"` 38 | ColumnSubtitle string `json:"column_subtitle"` 39 | ColumnTitle string `json:"column_title"` 40 | ColumnType int `json:"column_type"` 41 | ColumnTnit string `json:"column_unit"` 42 | HadSub bool `json:"had_sub"` 43 | IsChannel bool `json:"is_channel"` 44 | IsExperience bool `json:"is_experience"` 45 | IsOnboard bool `json:"is_onboard"` 46 | PriceType int `json:"price_type"` 47 | SubCount int `json:"sub_count"` 48 | ShowChapter bool `json:"show_chapter"` 49 | UpdateFrequency string `json:"update_frequency"` 50 | } 51 | 52 | //Article 课程文章信息 53 | type Article struct { 54 | ID int `json:"id"` 55 | ArticleTitle string `json:"article_title"` 56 | ArticleSummary string `json:"article_summary"` 57 | ArticleCover string `json:"article_cover"` 58 | ArticleTime int `json:"article_ctime"` 59 | ChapterID int `json:"chapter_id string"` 60 | IncludeAudio bool `json:"include_audio"` 61 | //Is can preview 62 | ColumnHadSub bool `json:"column_had_sub"` 63 | ArticleCouldPreview bool `json:"article_could_preview"` 64 | //Audio info 65 | AudioDownloadURL string `json:"audio_download_url"` 66 | AudioSize int `json:"audio_size"` 67 | //Viode info 68 | VideoMediaMap json.RawMessage `json:"video_media_map"` 69 | VideoID string `json:"video_id"` 70 | VideoCover string `json:"video_cover"` 71 | } 72 | 73 | //VideoPlayAuth 视频的播放授权信息 74 | type VideoPlayAuth struct { 75 | PlayAuth string `json:"play_auth"` 76 | } 77 | 78 | //VideoPlayInfo 视频播放信息 79 | type VideoPlayInfo struct { 80 | VideoBase struct { 81 | VideoID string `json:"VideoId"` 82 | Title string `json:"Title"` 83 | CoverURL string `josn:"CoverURL"` 84 | } `json:"VideoBase"` 85 | PlayInfoList struct { 86 | PlayInfo []struct { 87 | URL string `json:"PlayURL"` 88 | Size int64 `json:"Size"` 89 | Definition string `json:"Definition"` 90 | } `json:"PlayInfo"` 91 | } `json:"PlayInfoList"` 92 | } 93 | 94 | type articleResult struct { 95 | Articles []*Article `json:"list"` 96 | Page struct { 97 | Count int `json:"count"` 98 | More bool `json:"more"` 99 | } `json:"page"` 100 | } 101 | 102 | //V3ArticleInfo V3文章信息 103 | type V3ArticleInfo struct { 104 | Data struct { 105 | Info struct { 106 | Video struct { 107 | HlsMedias []struct { 108 | Quality string `json:"quality"` 109 | Url string `json:"url"` 110 | Size int `json:"size"` 111 | } `json:"hls_medias"` 112 | } `json:"video"` 113 | } `json:"info"` 114 | } `json:"data"` 115 | } 116 | 117 | //IsColumn 是否专栏 118 | func (course *Course) IsColumn() bool { 119 | return course.ColumnType == 1 120 | } 121 | 122 | //IsVideo 是否视频 123 | func (course *Course) IsVideo() bool { 124 | return course.ColumnType == 3 125 | } 126 | 127 | //IsCanPreview 是否能看 128 | func (article *Article) IsCanPreview() bool { 129 | return article.ColumnHadSub || article.ArticleCouldPreview 130 | } 131 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "sync" 7 | "unsafe" 8 | 9 | jsoniter "github.com/json-iterator/go" 10 | "github.com/udoless/geektime-downloader/service" 11 | ) 12 | 13 | const ( 14 | // EnvConfigDir 配置路径环境变量 15 | EnvConfigDir = "GEEKTIME_GO_CONFIG_DIR" 16 | // ConfigName 配置文件名 17 | ConfigName = "config.json" 18 | ) 19 | 20 | var ( 21 | configFilePath = filepath.Join(GetConfigDir(), ConfigName) 22 | 23 | //Instance 配置信息 全局调用 24 | Instance = NewConfig(configFilePath) 25 | ) 26 | 27 | //ConfigsData 配置数据 28 | type ConfigsData struct { 29 | AcitveUID int 30 | Geektimes Geektimes 31 | DownloadPath string 32 | 33 | activeUser *Geektime 34 | configFilePath string 35 | configFile *os.File 36 | fileMu sync.Mutex 37 | service *service.Service 38 | } 39 | 40 | //Init 初始化配置 41 | func (c *ConfigsData) Init() error { 42 | if c.configFilePath == "" { 43 | return ErrConfigFilePathNotSet 44 | } 45 | 46 | //初始化默认配置 47 | c.initDefaultConfig() 48 | //从配置文件中加载配置 49 | err := c.loadConfigFromFile() 50 | if err != nil { 51 | return err 52 | } 53 | 54 | //初始化登陆用户信息 55 | err = c.initActiveUser() 56 | if err != nil { 57 | return nil 58 | } 59 | 60 | if c.activeUser != nil { 61 | c.service = c.activeUser.Service() 62 | } 63 | 64 | return nil 65 | } 66 | 67 | func (c *ConfigsData) initActiveUser() error { 68 | //如果已经初始化过，则跳过 69 | if c.AcitveUID > 0 && c.activeUser != nil && c.activeUser.ID == c.AcitveUID { 70 | return nil 71 | } 72 | 73 | if c.AcitveUID == 0 && c.activeUser != nil { 74 | c.AcitveUID = c.activeUser.ID 75 | return nil 76 | } 77 | 78 | if c.AcitveUID > 0 { 79 | for _, geek := range c.Geektimes { 80 | if geek.ID == c.AcitveUID { 81 | c.activeUser = geek 82 | return nil 83 | } 84 | } 85 | } 86 | 87 | if len(c.Geektimes) > 0 { 88 | return ErrHasLoginedNotLogin 89 | } 90 | 91 | return ErrNotLogin 92 | } 93 | 94 | //Save 保存配置 95 | func (c *ConfigsData) Save() error { 96 | err := c.lazyOpenConfigFile() 97 | if err != nil { 98 | return err 99 | } 100 | 101 | c.fileMu.Lock() 102 | defer c.fileMu.Unlock() 103 | 104 | //todo 保存配置的数据 105 | data, err := jsoniter.MarshalIndent((*configJSONExport)(unsafe.Pointer(c)), "", " ") 106 | 107 | if err != nil { 108 | panic(err) 109 | } 110 | 111 | // 减掉多余的部分 112 | err = c.configFile.Truncate(int64(len(data))) 113 | if err != nil { 114 | return err 115 | } 116 | 117 | _, err = c.configFile.Seek(0, os.SEEK_SET) 118 | if err != nil { 119 | return err 120 | } 121 | 122 | _, err = c.configFile.Write(data) 123 | if err != nil { 124 | return err 125 | } 126 | 127 | return nil 128 | } 129 | 130 | func (c *ConfigsData) initDefaultConfig() { 131 | //todo 默认配置 132 | } 133 | 134 | func (c *ConfigsData) loadConfigFromFile() error { 135 | err := c.lazyOpenConfigFile() 136 | if err != nil { 137 | return err 138 | } 139 | 140 | info, err := c.configFile.Stat() 141 | if err != nil { 142 | return err 143 | } 144 | 145 | if info.Size() == 0 { 146 | return c.Save() 147 | } 148 | 149 | c.fileMu.Lock() 150 | defer c.fileMu.Unlock() 151 | 152 | _, err = c.configFile.Seek(0, os.SEEK_SET) 153 | if err != nil { 154 | return nil 155 | } 156 | 157 | //从配置文件中加载配置 158 | decoder := jsoniter.NewDecoder(c.configFile) 159 | decoder.Decode((*configJSONExport)(unsafe.Pointer(c))) 160 | 161 | return nil 162 | } 163 | 164 | func (c *ConfigsData) lazyOpenConfigFile() (err error) { 165 | if c.configFile != nil { 166 | return nil 167 | } 168 | c.fileMu.Lock() 169 | os.MkdirAll(filepath.Dir(c.configFilePath), 0700) 170 | c.configFile, err = os.OpenFile(c.configFilePath, os.O_CREATE|os.O_RDWR, 0600) 171 | c.fileMu.Unlock() 172 | 173 | if err != nil { 174 | if os.IsPermission(err) { 175 | return ErrConfigFileNoPermission 176 | } 177 | if os.IsExist(err) { 178 | return ErrConfigFileNotExist 179 | } 180 | return err 181 | } 182 | 183 | return nil 184 | } 185 | 186 | //NewConfig new config 187 | func NewConfig(configFilePath string) *ConfigsData { 188 | c := &ConfigsData{ 189 | configFilePath: configFilePath, 190 | } 191 | 192 | return c 193 | } 194 | 195 | //Geektimes 极客时间用户 196 | type Geektimes []*Geektime 197 | 198 | //GetConfigDir 配置文件夹 199 | func GetConfigDir() string { 200 | configDir, ok := os.LookupEnv(EnvConfigDir) 201 | if ok { 202 | if filepath.IsAbs(configDir) { 203 | return configDir 204 | } 205 | } 206 | 207 | home, ok := os.LookupEnv("HOME") 208 | if ok { 209 | return filepath.Join(home, ".config", "geekbang") 210 | } 211 | 212 | return filepath.Join("/tmp", "geekbang") 213 | } 214 | 215 | //ActiveUser active user 216 | func (c *ConfigsData) ActiveUser() *Geektime { 217 | return c.activeUser 218 | } 219 | 220 | //ActiveUserService user service 221 | func (c *ConfigsData) ActiveUserService() *service.Service { 222 | if c.service == nil { 223 | c.service = c.activeUser.Service() 224 | } 225 | return c.service 226 | } 227 | -------------------------------------------------------------------------------- /cli/cmds/login.go: -------------------------------------------------------------------------------- 1 | package cmds 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "strconv" 8 | 9 | "github.com/olekukonko/tablewriter" 10 | "github.com/udoless/geektime-downloader/cli/application" 11 | "github.com/udoless/geektime-downloader/config" 12 | "github.com/urfave/cli" 13 | ) 14 | 15 | //Login login data 16 | type Login struct { 17 | phone string 18 | password string 19 | gcid string 20 | gcess string 21 | serverID string 22 | } 23 | 24 | //IsByPhoneAndPassword 通过手机号和密码登录 25 | func (l *Login) IsByPhoneAndPassword() bool { 26 | return l.phone != "" && l.password != "" 27 | } 28 | 29 | //IsByCookie cookie login 30 | func (l *Login) IsByCookie() bool { 31 | return l.gcid != "" && l.gcess != "" 32 | } 33 | 34 | //LoginConfig config 35 | var LoginConfig Login 36 | 37 | //NewLoginCommand login command 38 | func NewLoginCommand() []cli.Command { 39 | return []cli.Command{ 40 | { 41 | Name: "login", 42 | Usage: "Login geektime", 43 | UsageText: appName + " login [OPTIONS]", 44 | Action: loginAction, 45 | Flags: []cli.Flag{ 46 | cli.StringFlag{ 47 | Name: "phone", 48 | Usage: "登录手机号", 49 | Destination: &LoginConfig.phone, 50 | }, 51 | cli.StringFlag{ 52 | Name: "password", 53 | Usage: "登录密码", 54 | Destination: &LoginConfig.password, 55 | }, 56 | cli.StringFlag{ 57 | Name: "gcid", 58 | Usage: "GCID Cookie", 59 | Destination: &LoginConfig.gcid, 60 | }, 61 | cli.StringFlag{ 62 | Name: "gcess", 63 | Usage: "GCESS Cookie", 64 | Destination: &LoginConfig.gcess, 65 | }, 66 | cli.StringFlag{ 67 | Name: "serverId", 68 | Usage: "SERVERID Cookie", 69 | Destination: &LoginConfig.serverID, 70 | }, 71 | }, 72 | After: configSaveFunc, 73 | }, 74 | { 75 | Name: "who", 76 | Usage: "获取当前帐号", 77 | UsageText: appName + " who", 78 | Description: "获取当前帐号的信息", 79 | Action: whoAction, 80 | Before: authorizationFunc, 81 | }, 82 | { 83 | Name: "users", 84 | Usage: "获取帐号列表", 85 | UsageText: appName + " users", 86 | Description: "获取当前已登录的帐号列表", 87 | Action: usersAction, 88 | }, 89 | { 90 | Name: "su", 91 | Usage: "切换极客时间帐号", 92 | UsageText: appName + " su [UID]", 93 | Description: "切换已登录的极客时间账号", 94 | Action: suAction, 95 | After: configSaveFunc, 96 | }, 97 | } 98 | } 99 | 100 | func loginAction(c *cli.Context) error { 101 | //通过手机号和密码登录 102 | var ( 103 | gcid = LoginConfig.gcid 104 | gcess = LoginConfig.gcess 105 | serverID = LoginConfig.serverID 106 | err error 107 | ) 108 | if LoginConfig.IsByPhoneAndPassword() { 109 | err = loginByPhoneAndPassword(LoginConfig.phone, LoginConfig.password) 110 | if err != nil { 111 | return err 112 | } 113 | return nil 114 | } else if gcid != "" && gcess != "" { 115 | geektime, err := config.Instance.SetUserByGcidAndGcess(gcid, gcess, serverID, LoginConfig.phone, LoginConfig.password) 116 | if err != nil { 117 | return err 118 | } 119 | 120 | fmt.Println("极客时间账号登录成功：", geektime.Name) 121 | 122 | return nil 123 | } 124 | 125 | return errors.New("请输入登录凭证信息") 126 | } 127 | 128 | func loginByPhoneAndPassword(phone string, password string) error { 129 | gcid, gcess, serverID, err := application.Login(phone, password) 130 | if err != nil { 131 | return err 132 | } 133 | 134 | geektime, err := config.Instance.SetUserByGcidAndGcess(gcid, gcess, serverID, phone, password) 135 | if err != nil { 136 | return err 137 | } 138 | 139 | fmt.Println("极客时间账号登录成功：", geektime.Name) 140 | 141 | return nil 142 | } 143 | 144 | func whoAction(c *cli.Context) error { 145 | activeUser := config.Instance.ActiveUser() 146 | fmt.Printf("当前帐号 uid: %d, 用户名: %s, 头像地址: %s \n", activeUser.ID, activeUser.Name, activeUser.Avatar) 147 | 148 | return nil 149 | } 150 | 151 | func usersAction(c *cli.Context) error { 152 | table := tablewriter.NewWriter(os.Stdout) 153 | table.SetHeader([]string{"#", "UID", "昵称"}) 154 | i := 0 155 | for _, g := range config.Instance.Geektimes { 156 | table.Append([]string{strconv.Itoa(i), strconv.Itoa(g.ID), g.Name}) 157 | i++ 158 | } 159 | table.Render() 160 | 161 | return nil 162 | } 163 | 164 | func suAction(c *cli.Context) error { 165 | if config.Instance.LoginUserCount() == 0 { 166 | return errors.New("未登录任何极客时间账号，不能切换") 167 | } 168 | 169 | if c.NArg() == 0 { 170 | cli.HandleAction(usersAction, c) 171 | return errors.New("请选择登录的用户UID") 172 | } 173 | 174 | i, err := strconv.Atoi(c.Args().Get(0)) 175 | if err != nil { 176 | cli.HandleAction(usersAction, c) 177 | return errors.New("请输入用户UID") 178 | } 179 | 180 | err = config.Instance.SwitchUser(&config.User{ID: i}) 181 | 182 | if err != nil { 183 | return err 184 | } 185 | 186 | fmt.Printf("成功切换登录用户：%s\n", config.Instance.ActiveUser().Name) 187 | 188 | return nil 189 | } 190 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## geektime-dl 2 | 3 | 4 | 👾 Geektime-dl 是使用Go构建的快速、简单的 [极客时间](https://time.geekbang.org/) 下载器。支持下载文章为PDF，也支持音频，视频下载。 5 | 6 | 7 | - [安装](#安装) 8 | - [必要条件](#必要条件) 9 | - [使用`go get`安装](#%e4%bd%bf%e7%94%a8go-get%e5%ae%89%e8%a3%85) 10 | - [入门](#%e5%85%a5%e9%97%a8) 11 | - [视频和专栏的下载](#视频和专栏的下载) 12 | - [查看视频或专栏列表](#查看视频或专栏列表) 13 | - [可恢复继续下载](#可恢复继续下载) 14 | - [登录](#登录) 15 | - [参考仓库](#参考仓库) 16 | - [License](#license) 17 | 18 | ## 安装 19 | 20 | ### 必要条件 21 | 22 | 以下为必须安装依赖： 23 | 24 | * **[FFmpeg](https://www.ffmpeg.org)** 25 | 26 | > **Note**: FFmpeg的使用是为了最后视频文件合并成需要的格式。 27 | 28 | * **[Google-Chrome](https://www.google.cn/intl/zh-CN/chrome/)** 29 | 30 | > **Note**: 借助[`chromedp/chromedp`](https://github.com/chromedp/chromedp)工具导出页面为PDF文档，该功能需要谷歌浏览器的支持。 31 | 32 | ### 使用`go get`安装 33 | 34 | 安装Geektime-dl，可以使用如下`go get`命令，或者从[Releases](https://github.com/udoless/geektime-downloader/releases) 页面下载二进制文件. 35 | 36 | ```bash 37 | $ go get github.com/udoless/geektime-downloader 38 | ``` 39 | 40 | ## 入门 41 | 42 | 使用方法: 43 | 44 | ```bash 45 | #下载 46 | geektime-dl [OPTIONS] 课程ID [目录ID] 47 | #查看专栏、视频，登录及其他命令操作 48 | geektime-dl [OPTIONS] command 49 | ``` 50 | 51 | 包含命令 52 | 53 | ```text 54 | login 登录极客时间 55 | who 获取当前帐号 56 | users 获取帐号列表 57 | su 切换极客时间帐号 58 | buy 获取已购买过的专栏和视频课程 59 | column 获取专栏列表 60 | video 获取视频课程列表 61 | help, h 帮助 62 | ``` 63 | 64 | ### 登录 65 | 66 | 通过账号密码登录： 67 | 68 | ```console 69 | $ geektime-dl login --phone xxxxxx --password xxxxxx 70 | 极客时间账号登录成功： XXX 71 | ``` 72 | 73 | 通过Cookie登录： 74 | 75 | ```console 76 | $ geektime-dl login --gcid xxxxxx --gcess xxxxxx --serverId 'xxxxxxx' 77 | 极客时间账号登录成功： XXX 78 | ``` 79 | 80 | ### 视频和专栏的下载 81 | 82 | 只能下载已购买或者免费部分的视频、专栏。 83 | 84 | ```console 85 | $ geektime-dl 66 86 | 01 - 什么是微服务架构？ 107.55 MiB / 107.54 MiB [==================================] 100.01% 1.42 MiB/s 1m15s 87 | 02 - 架构师如何权衡微服务的利弊？ 92.10 MiB / 92.09 MiB [============================] 100.01% 1.69 MiB/s 54s 88 | 03 - 康威法则和微服务给架构师怎样的启示？ 69.38 MiB / 69.38 MiB [=====================] 100.01% 1.68 MiB/s 41s 89 | 04 - 企业应该在什么时候开始考虑引入微服务？ 114.20 MiB / 114.20 MiB [==================] 100.00% 1.41 MiB/s 1m21s 90 | 05 - 什么样的组织架构更适合微服务？ 121.10 MiB / 121.09 MiB [========================] 100.00% 1.66 MiB/s 1m13s 91 | 06 - 如何理解阿里巴巴提出的微服务中台战略？65.23 MiB / 126.82 MiB [==========>---------] 51.43% 1.68 MiB/s 1m15s 92 | ``` 93 | 94 | 只需下载课程中的某个目录 95 | 96 | ```console 97 | $ geektime-dl 66 2276 98 | 16 - 微服务监控系统分层和监控架构 11.22 MiB / 97.55 MiB [======>--------------------] 28.51% 1.30 MiB/s 01m06s 99 | ``` 100 | 101 | 下载专栏时，可以同时下载专栏文章内容为PDF文档（`需要谷歌浏览器支持`） 102 | 103 | ```console 104 | 04 - 静态容器：办公用品如何表达你的内容？ 13.94 MiB / 13.94 MiB [===================] 100.00% 2.23 MiB/s 6s 105 | 正在生成文件：【04 - 静态容器：办公用品如何表达你的内容？.pdf】 完成 106 | ``` 107 | 108 | > **注**: `如果生成文件提示失败，可以重复执行命令针对失败的文件再次生成`，已生成的文件不会重复生成。如果尝试多次都是失败，可以Issues提问。 109 | 110 | 查看课程中可下载的目录 111 | 112 | ```console 113 | $ geektime-dl -i 66 114 | +----+------+------+----------------------------------------------+---------+---------+---------+------+ 115 | | # | ID | 类型 | 名称 | SD | LD | HD | 下载 | 116 | +----+------+------+----------------------------------------------+---------+---------+---------+------+ 117 | | 0 | 2184 | 视频 | 01 什么是微服务架构？ | 86.52M | 53.45M | 107.54M | ✔ | 118 | | 1 | 2185 | 视频 | 02 架构师如何权衡微服务的利弊？ | 71.43M | 44.12M | 92.09M | ✔ | 119 | | 2 | 2154 | 视频 | 03 康威法则和微服务给架构师怎样的启示？ | 54.32M | 33.57M | 69.38M | ✔ | 120 | | 3 | 2186 | 视频 | 04 企业应该在什么时候开始考虑引入微服务？ | 90.07M | 55.67M | 114.20M | ✔ | 121 | | 4 | 2187 | 视频 | 05 什么样的组织架构更适合微服务？ | 90.22M | 55.79M | 121.09M | ✔ | 122 | | 5 | 2188 | 视频 | 06 如何理解阿里巴巴提出的微服务中台战略？ | 126.82M | 100.05M | 61.79M | ✔ | 123 | | 6 | 2189 | 视频 | 07 如何给出一个清晰简洁的服务分层方式？ | 45.89M | 62.07M | 61.95M | ✔ | 124 | | 7 | 2222 | 视频 | 08 微服务总体技术架构体系是怎样设计的？ | 85.67M | 52.91M | 109.83M | ✔ | 125 | | 8 | 2269 | 视频 | 09 微服务最经典的三种服务发现机制 | 94.00M | 73.18M | 45.21M | ✔ | 126 | ``` 127 | 128 | ### 查看视频或专栏列表 129 | 130 | ```bash 131 | #查看专栏列表 132 | $ geektime-dl column 133 | +----+-----+---------------------------+------------+------------------+------+ 134 | | # | ID | 名称 | 时间 | 作者 | 购买 | 135 | +----+-----+---------------------------+------------+------------------+------+ 136 | | 0 | 42 | 技术与商业案例解读 | 2017-09-07 | 徐飞 | | 137 | | 1 | 43 | AI技术内参 | 2017-09-11 | 洪亮劼 | | 138 | | 2 | 48 | 左耳听风 | 2017-09-20 | 陈皓 | 是 | 139 | | 3 | 49 | 朱赟的技术管理课 | 2017-11-09 | 朱赟 | 是 | 140 | | 4 | 50 | 邱岳的产品手记 | 2017-11-16 | 邱岳 | | 141 | | 5 | 62 | 人工智能基础课 | 2017-12-01 | 王天一 | 是 | 142 | | 6 | 63 | 赵成的运维体系管理课 | 2017-12-13 | 赵成 | | 143 | | 7 | 74 | 推荐系统三十六式 | 2018-02-23 | 刑无刀 | | 144 | | 8 | 76 | 深入浅出区块链 | 2018-03-19 | 陈浩 | 是 | 145 | 146 | 147 | #查看视频列表 148 | $ geektime-dl video 149 | +----+-----+------------------------------------------+------------+--------------+------+ 150 | | # | ID | 名称 | 时间 | 作者 | 购买 | 151 | +----+-----+------------------------------------------+------------+--------------+------+ 152 | | 0 | 66 | 微服务架构核心20讲 | 2018-01-08 | 杨波 | 是 | 153 | | 1 | 77 | 9小时搞定微信小程序开发 | 2018-03-22 | 高磊 | | 154 | | 2 | 84 | 微服务架构实战160讲 | 2018-05-03 | 杨波 | 是 | 155 | | 3 | 98 | 零基础学Python | 2018-05-25 | 尹会生 | | 156 | ``` 157 | 158 | ### 可恢复继续下载 159 | 160 | Ctrl+C 中断下载。 161 | 162 | 存在 `.download` 临时文件，使用相同的参数执行 `geektime-dl` 命令，则下载进度将从上一个会话恢复。 163 | 164 | 165 | ## 参考仓库 166 | 167 | * [geektime-dl](https://github.com/mmzou/geektime-dl) -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 2 | github.com/cheggaaa/pb v1.0.25 h1:tFpebHTkI7QZx1q1rWGOKhbunhZ3fMaxTvHDWn1bH/4= 3 | github.com/cheggaaa/pb v1.0.25/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= 4 | github.com/chromedp/cdproto v0.0.0-20210104223854-2cc87dae3ee3 h1:XeGYLuu3Yu3/2/FLDXyObe6lBYtUFDTJgjjNPcfcU40= 5 | github.com/chromedp/cdproto v0.0.0-20210104223854-2cc87dae3ee3/go.mod h1:55pim6Ht4LJKdVLlyFJV/g++HsEA1hQxPbB5JyNdZC0= 6 | github.com/chromedp/cdproto v0.0.0-20210713064928-7d28b402946a h1:B6EyBXuMsFyrUoBrNXdt+Vf3vQNpN4DU/Xv96R4BdFg= 7 | github.com/chromedp/cdproto v0.0.0-20210713064928-7d28b402946a/go.mod h1:At5TxYYdxkbQL0TSefRjhLE3Q0lgvqKKMSFUglJ7i1U= 8 | github.com/chromedp/chromedp v0.6.0 h1:jjzHzXW5pNdKt1D9cEDAKZM/yZ2EwL/hLyGbCUFldBI= 9 | github.com/chromedp/chromedp v0.6.0/go.mod h1:Yay7TUDCNOQBK8EJDUon6AUaQI12VEBOuULcGtY4uDY= 10 | github.com/chromedp/chromedp v0.7.4 h1:U+0d3WbB/Oj4mDuBOI0P7S3PJEued5UZIl5AJ3QulwU= 11 | github.com/chromedp/chromedp v0.7.4/go.mod h1:dBj+SXuQHznp6ZPwZeDDEBZKwclUwDLbZ0hjMialMYs= 12 | github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic= 13 | github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= 14 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= 15 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 16 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 17 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 18 | github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= 19 | github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= 20 | github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= 21 | github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= 22 | github.com/gobwas/ws v1.0.4 h1:5eXU1CZhpQdq5kXbKb+sECH5Ia5KiO6CYzIzdlVx6Bs= 23 | github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= 24 | github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA= 25 | github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0= 26 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 27 | github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= 28 | github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= 29 | github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= 30 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 31 | github.com/knq/sysutil v0.0.0-20191005231841-15668db23d08/go.mod h1:dFWs1zEqDjFtnBXsd1vPOZaLsESovai349994nHx3e0= 32 | github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= 33 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 34 | github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= 35 | github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= 36 | github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= 37 | github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= 38 | github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= 39 | github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= 40 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= 41 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 42 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= 43 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 44 | github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= 45 | github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= 46 | github.com/orisano/pixelmatch v0.0.0-20210112091706-4fa4c7ba91d5/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= 47 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 48 | github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= 49 | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 50 | github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= 51 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 52 | github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q= 53 | github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= 54 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 55 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 56 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 57 | github.com/urfave/cli v1.22.4 h1:u7tSpNPPswAFymm8IehJhy4uJMlUuU/GmqSkvJ1InXA= 58 | github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= 59 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 60 | golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 61 | golang.org/x/sys v0.0.0-20210104204734-6f8348627aad h1:MCsdmFSdEd4UEa5TKS5JztCRHK/WtvNei1edOj5RSRo= 62 | golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 63 | golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea h1:+WiDlPBBaO+h9vPNZi8uJ3k4BkKQB7Iow3aqwHVA5hI= 64 | golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 65 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 66 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 67 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.toptal.com/developers/gitignore/api/vim,jetbrains,vscode,git,go,tags,backup,test,emacs 2 | # Edit at https://www.toptal.com/developers/gitignore?templates=vim,jetbrains,vscode,git,go,tags,backup,test,emacs 3 | 4 | ### Backup ### 5 | *.bak 6 | *.gho 7 | *.ori 8 | *.orig 9 | *.tmp 10 | 11 | ### Git ### 12 | # Created by git for backups. To disable backups in Git: 13 | # $ git config --global mergetool.keepBackup false 14 | 15 | # Created by git when using merge tools for conflicts 16 | *.BACKUP.* 17 | *.BASE.* 18 | *.LOCAL.* 19 | *.REMOTE.* 20 | *_BACKUP_*.txt 21 | *_BASE_*.txt 22 | *_LOCAL_*.txt 23 | *_REMOTE_*.txt 24 | 25 | ### Go ### 26 | # Binaries for programs and plugins 27 | *.exe 28 | *.exe~ 29 | *.dll 30 | *.so 31 | *.dylib 32 | 33 | # Test binary, built with `go test -c` 34 | *.test 35 | 36 | # Output of the go coverage tool, specifically when used with LiteIDE 37 | *.out 38 | 39 | # Dependency directories (remove the comment below to include it) 40 | # vendor/ 41 | 42 | ### Go Patch ### 43 | /vendor/ 44 | /Godeps/ 45 | 46 | ### JetBrains ### 47 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider 48 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 49 | .idea 50 | 51 | # User-specific stuff 52 | .idea/**/workspace.xml 53 | .idea/**/tasks.xml 54 | .idea/**/usage.statistics.xml 55 | .idea/**/dictionaries 56 | .idea/**/shelf 57 | 58 | # Generated files 59 | .idea/**/contentModel.xml 60 | 61 | # Sensitive or high-churn files 62 | .idea/**/dataSources/ 63 | .idea/**/dataSources.ids 64 | .idea/**/dataSources.local.xml 65 | .idea/**/sqlDataSources.xml 66 | .idea/**/dynamic.xml 67 | .idea/**/uiDesigner.xml 68 | .idea/**/dbnavigator.xml 69 | 70 | # Gradle 71 | .idea/**/gradle.xml 72 | .idea/**/libraries 73 | 74 | # Gradle and Maven with auto-import 75 | # When using Gradle or Maven with auto-import, you should exclude module files, 76 | # since they will be recreated, and may cause churn. Uncomment if using 77 | # auto-import. 78 | # .idea/artifacts 79 | # .idea/compiler.xml 80 | # .idea/jarRepositories.xml 81 | # .idea/modules.xml 82 | # .idea/*.iml 83 | # .idea/modules 84 | # *.iml 85 | # *.ipr 86 | 87 | # CMake 88 | cmake-build-*/ 89 | 90 | # Mongo Explorer plugin 91 | .idea/**/mongoSettings.xml 92 | 93 | # File-based project format 94 | *.iws 95 | 96 | # IntelliJ 97 | out/ 98 | 99 | # mpeltonen/sbt-idea plugin 100 | .idea_modules/ 101 | 102 | # JIRA plugin 103 | atlassian-ide-plugin.xml 104 | 105 | # Cursive Clojure plugin 106 | .idea/replstate.xml 107 | 108 | # Crashlytics plugin (for Android Studio and IntelliJ) 109 | com_crashlytics_export_strings.xml 110 | crashlytics.properties 111 | crashlytics-build.properties 112 | fabric.properties 113 | 114 | # Editor-based Rest Client 115 | .idea/httpRequests 116 | 117 | # Android studio 3.1+ serialized cache file 118 | .idea/caches/build_file_checksums.ser 119 | 120 | ### JetBrains Patch ### 121 | # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 122 | 123 | # *.iml 124 | # modules.xml 125 | # .idea/misc.xml 126 | # *.ipr 127 | 128 | # Sonarlint plugin 129 | # https://plugins.jetbrains.com/plugin/7973-sonarlint 130 | .idea/**/sonarlint/ 131 | 132 | # SonarQube Plugin 133 | # https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin 134 | .idea/**/sonarIssues.xml 135 | 136 | # Markdown Navigator plugin 137 | # https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced 138 | .idea/**/markdown-navigator.xml 139 | .idea/**/markdown-navigator-enh.xml 140 | .idea/**/markdown-navigator/ 141 | 142 | # Cache file creation bug 143 | # See https://youtrack.jetbrains.com/issue/JBR-2257 144 | .idea/$CACHE_FILE$ 145 | 146 | # CodeStream plugin 147 | # https://plugins.jetbrains.com/plugin/12206-codestream 148 | .idea/codestream.xml 149 | 150 | ### Tags ### 151 | # Ignore tags created by etags, ctags, gtags (GNU global) and cscope 152 | TAGS 153 | .TAGS 154 | !TAGS/ 155 | tags 156 | .tags 157 | !tags/ 158 | gtags.files 159 | GTAGS 160 | GRTAGS 161 | GPATH 162 | GSYMS 163 | cscope.files 164 | cscope.out 165 | cscope.in.out 166 | cscope.po.out 167 | 168 | 169 | ### Test ### 170 | ### Ignore all files that could be used to test your code and 171 | ### you wouldn't want to push 172 | 173 | # Reference https://en.wikipedia.org/wiki/Metasyntactic_variable 174 | 175 | # Most common 176 | *foo 177 | *bar 178 | *fubar 179 | *foobar 180 | *baz 181 | 182 | # Less common 183 | *qux 184 | *quux 185 | *bongo 186 | *bazola 187 | *ztesch 188 | 189 | # UK, Australia 190 | *wibble 191 | *wobble 192 | *wubble 193 | *flob 194 | *blep 195 | *blah 196 | *boop 197 | *beep 198 | 199 | # Japanese 200 | *hoge 201 | *piyo 202 | *fuga 203 | *hogera 204 | *hogehoge 205 | 206 | # Portugal, Spain 207 | *fulano 208 | *sicrano 209 | *beltrano 210 | *mengano 211 | *perengano 212 | *zutano 213 | 214 | # France, Italy, the Netherlands 215 | *toto 216 | *titi 217 | *tata 218 | *tutu 219 | *pipppo 220 | *pluto 221 | *paperino 222 | *aap 223 | *noot 224 | *mies 225 | 226 | # Other names that would make sense 227 | *tests 228 | *testsdir 229 | *testsfile 230 | *testsfiles 231 | *test 232 | *testdir 233 | *testfile 234 | *testfiles 235 | *testing 236 | *testingdir 237 | *testingfile 238 | *testingfiles 239 | *temp 240 | *tempdir 241 | *tempfile 242 | *tempfiles 243 | *tmp 244 | *tmpdir 245 | *tmpfile 246 | *tmpfiles 247 | *lol 248 | 249 | ### Vim ### 250 | # Swap 251 | [._]*.s[a-v][a-z] 252 | !*.svg # comment out if you don't need vector files 253 | [._]*.sw[a-p] 254 | [._]s[a-rt-v][a-z] 255 | [._]ss[a-gi-z] 256 | [._]sw[a-p] 257 | 258 | # Session 259 | Session.vim 260 | Sessionx.vim 261 | 262 | # Temporary 263 | .netrwhist 264 | *~ 265 | # Auto-generated tag files 266 | # Persistent undo 267 | [._]*.un~ 268 | 269 | ### Emacs ### 270 | # -*- mode: gitignore; -*- 271 | *~ 272 | \#*\# 273 | /.emacs.desktop 274 | /.emacs.desktop.lock 275 | *.elc 276 | auto-save-list 277 | tramp 278 | .\#* 279 | 280 | # Org-mode 281 | .org-id-locations 282 | *_archive 283 | ltximg/** 284 | 285 | # flymake-mode 286 | *_flymake.* 287 | 288 | # eshell files 289 | /eshell/history 290 | /eshell/lastdir 291 | 292 | # elpa packages 293 | /elpa/ 294 | 295 | # reftex files 296 | *.rel 297 | 298 | # AUCTeX auto folder 299 | /auto/ 300 | 301 | # cask packages 302 | .cask/ 303 | dist/ 304 | 305 | # Flycheck 306 | flycheck_*.el 307 | 308 | # server auth directory 309 | /server/ 310 | 311 | # projectiles files 312 | .projectile 313 | 314 | # directory configuration 315 | .dir-locals.el 316 | 317 | # network security 318 | /network-security.data 319 | 320 | ### vscode ### 321 | .vscode/* 322 | !.vscode/settings.json 323 | !.vscode/tasks.json 324 | !.vscode/launch.json 325 | !.vscode/extensions.json 326 | *.code-workspace 327 | 328 | # End of https://www.toptal.com/developers/gitignore/api/vim,jetbrains,vscode,git,go,tags,backup,test 329 | 330 | # Start by iam 331 | 332 | # log 333 | *.log 334 | 335 | # Output of backend and frontend 336 | /_output 337 | /_debug 338 | 339 | # Misc 340 | .DS_Store 341 | *.env 342 | .env.* 343 | dist 344 | 345 | # files used by the developer 346 | .idea.md 347 | .todo.md 348 | .note.md 349 | 350 | # config files, may contain sensitive information 351 | 352 | configs/*.yaml 353 | 354 | .vscode 355 | test/ 356 | downloads/ 357 | .DS_Store 358 | log.log 359 | .idea 360 | .goreleaser.yml -------------------------------------------------------------------------------- /utils/chromedp.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "io/ioutil" 8 | "log" 9 | "strconv" 10 | "strings" 11 | "sync" 12 | "time" 13 | 14 | "github.com/chromedp/cdproto/cdp" 15 | "github.com/chromedp/cdproto/network" 16 | "github.com/chromedp/cdproto/page" 17 | "github.com/chromedp/cdproto/runtime" 18 | "github.com/chromedp/chromedp" 19 | "github.com/chromedp/chromedp/device" 20 | ) 21 | 22 | var LoginCh chan struct{} 23 | var LoginRespCh chan struct{} 24 | var ErrRetry = errors.New("retry error") 25 | 26 | //ColumnPrintToPDF print pdf 27 | func ColumnPrintToPDF(aid int, filename string, cookies map[string]string) error { 28 | var buf []byte 29 | // create chrome instance 30 | ctx, cancel := chromedp.NewContext( 31 | context.Background(), 32 | chromedp.WithLogf(log.Printf), 33 | ) 34 | defer cancel() 35 | 36 | // create a timeout 37 | ctx, cancel = context.WithTimeout(ctx, 120*time.Second) 38 | defer cancel() 39 | 40 | var wg sync.WaitGroup 41 | urlMap := make(map[string]struct{}) 42 | chromedp.ListenTarget(ctx, func(ev interface{}) { 43 | switch e := ev.(type) { 44 | case *network.EventRequestWillBeSent: 45 | if e.Type == "Image" && strings.Contains(e.Request.URL, "geekbang") && !strings.Contains(e.Request.URL, "aliyun") { 46 | //fmt.Printf("【network.EventRequestWillBeSent】%s, %s\n", e.Type, e.Request.URL) 47 | urlMap[e.Request.URL] = struct{}{} 48 | wg.Add(1) 49 | } 50 | case *network.EventResponseReceived: 51 | if e.Type == "Image" && strings.Contains(e.Response.URL, "geekbang") && !strings.Contains(e.Response.URL, "aliyun") { 52 | //fmt.Printf("【network.EventResponseReceived】%s, %s\n", e.Type, e.Response.URL) 53 | delete(urlMap, e.Response.URL) 54 | wg.Done() 55 | } 56 | } 57 | }) 58 | imgLoadedCh := make(chan struct{}) 59 | 60 | // url := `https://time.geekbang.org/column/article/169881` 61 | url := `https://time.geekbang.org/column/article/` + strconv.Itoa(aid) 62 | err := chromedp.Run(ctx, 63 | chromedp.Tasks{ 64 | network.SetExtraHTTPHeaders(network.Headers(map[string]interface{}{ 65 | "User-Agent": "Nimei_" + time.Now().String(), 66 | })), 67 | chromedp.Emulate(device.IPhone7), 68 | enableLifeCycleEvents(), 69 | setCookies(cookies), 70 | navigateAndWaitFor(url, "firstImagePaint"), 71 | // chromedp.WaitVisible("img", chromedp.ByQueryAll), 72 | chromedp.WaitReady("img", chromedp.ByQueryAll), 73 | chromedp.ActionFunc(func(ctx context.Context) error { 74 | s := ` 75 | document.querySelector('.iconfont').parentElement.parentElement.style.display='none'; 76 | document.querySelector('.Index_white_1gqaD>div.iconfont').style.display='none'; 77 | var audioBar = document.querySelector('.audio-float-bar'); 78 | if(audioBar){ 79 | audioBar.style.display='none' 80 | } 81 | var bottom = document.querySelector('.sub-bottom-wrapper'); 82 | if(bottom){ 83 | bottom.style.display='none' 84 | } 85 | [...document.querySelectorAll('ul>li>div>div>div:nth-child(2)>span')].map(e=>e.click()); 86 | ` 87 | _, exp, err := runtime.Evaluate(s).Do(ctx) 88 | if err != nil { 89 | return err 90 | } 91 | 92 | if exp != nil { 93 | return exp 94 | } 95 | 96 | return nil 97 | }), 98 | chromedp.ActionFunc(func(ctx context.Context) error { 99 | s := ` 100 | var divs = document.getElementsByTagName('div'); 101 | for (var i = 0; i < divs.length; ++i){ 102 | if(divs[i].innerText === "打开APP"){ 103 | divs[i].parentNode.parentNode.style.display="none"; 104 | break; 105 | } 106 | } 107 | ` 108 | _, exp, err := runtime.Evaluate(s).Do(ctx) 109 | if err != nil { 110 | return err 111 | } 112 | 113 | if exp != nil { 114 | return exp 115 | } 116 | 117 | return nil 118 | }), 119 | //尽量确保图片都已加载 120 | /* 121 | chromedp.ActionFunc(func(ctx context.Context) error { 122 | s := ` 123 | var maxTry = 3 124 | var loaded=false; 125 | var t = setInterval(function(){ 126 | var img=document.getElementsByTagName('img'); 127 | for(var i=0;i ") 255 | } 256 | } 257 | }) 258 | 259 | select { 260 | case <-ch: 261 | return nil 262 | case <-ctx.Done(): 263 | return ctx.Err() 264 | } 265 | } 266 | -------------------------------------------------------------------------------- /cli/cmds/download.go: -------------------------------------------------------------------------------- 1 | package cmds 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "strconv" 8 | "strings" 9 | 10 | "github.com/udoless/geektime-downloader/cli/application" 11 | "github.com/udoless/geektime-downloader/downloader" 12 | "github.com/udoless/geektime-downloader/service" 13 | "github.com/udoless/geektime-downloader/utils" 14 | "github.com/urfave/cli" 15 | ) 16 | 17 | //NewDownloadCommand login command 18 | func NewDownloadCommand() []cli.Command { 19 | return []cli.Command{ 20 | { 21 | Name: "", 22 | Usage: "", 23 | UsageText: "", 24 | Action: downloadAction, 25 | Before: authorizationFunc, 26 | }, 27 | } 28 | } 29 | 30 | func downloadAction(c *cli.Context) error { 31 | args := c.Parent().Args() 32 | cid, err := strconv.Atoi(args.First()) 33 | if err != nil { 34 | cli.ShowCommandHelp(c, "download") 35 | return errors.New("请输入课程ID") 36 | } 37 | 38 | //课程目录ID 39 | aid := 0 40 | if len(args) > 1 { 41 | aid, err = strconv.Atoi(args.Get(1)) 42 | if err != nil { 43 | return errors.New("课程目录ID错误") 44 | } 45 | } 46 | 47 | course, articles, err := application.CourseWithArticles(cid) 48 | if err != nil { 49 | return err 50 | } 51 | 52 | downloadData := extractDownloadData(course, articles, aid) 53 | // printExtractDownloadData(downloadData) 54 | 55 | if _info { 56 | downloadData.PrintInfo() 57 | return nil 58 | } 59 | 60 | // 专栏下载时，如果没有指定pdf和mp3，则默认同时下载 61 | if course.IsColumn() && !_pdf && !_mp3 { 62 | _pdf = true 63 | _mp3 = true 64 | } 65 | 66 | myErrors := make([]error, 0) 67 | 68 | // 视频或者音频下载 69 | if course.IsVideo() || (course.IsColumn() && _mp3) { 70 | sub := "MP4" 71 | if course.IsColumn() { 72 | sub = "MP3" 73 | } 74 | 75 | // 创建文件夹 76 | path, err := utils.Mkdir(utils.FileName(course.ColumnTitle, ""), sub) 77 | if err != nil { 78 | return err 79 | } 80 | 81 | for _, datum := range downloadData.Data { 82 | if !datum.IsCanDL { 83 | continue 84 | } 85 | if err := downloader.Download(datum, _stream, path); err != nil { 86 | myErrors = append(myErrors, err) 87 | } 88 | } 89 | 90 | if len(myErrors) > 0 { 91 | return myErrors[0] 92 | } 93 | } 94 | 95 | //如果是专栏，则需要打印内容 96 | if course.IsColumn() && _pdf { 97 | path, err := utils.Mkdir(utils.FileName(course.ColumnTitle, ""), "PDF") 98 | if err != nil { 99 | return err 100 | } 101 | cookies := application.LoginedCookies() 102 | for _, datum := range downloadData.Data { 103 | if !datum.IsCanDL { 104 | continue 105 | } 106 | for { 107 | if err := downloader.PrintToPDF(datum, cookies, path); err != nil { 108 | if errors.Is(err, utils.ErrRetry) { 109 | cookies = application.LoginedCookies() 110 | continue 111 | } 112 | myErrors = append(myErrors, err) 113 | } 114 | break 115 | } 116 | } 117 | } 118 | 119 | if len(myErrors) > 0 { 120 | return myErrors[0] 121 | } 122 | 123 | return nil 124 | } 125 | 126 | //生成下载数据 127 | func extractDownloadData(course *service.Course, articles []*service.Article, aid int) downloader.Data { 128 | 129 | downloadData := downloader.Data{ 130 | Title: course.ColumnTitle, 131 | } 132 | 133 | if course.IsColumn() { 134 | downloadData.Type = "专栏" 135 | downloadData.Data = extractColumnDownloadData(articles, aid) 136 | } else if course.IsVideo() { 137 | downloadData.Type = "视频" 138 | downloadData.Data = extractVideoDownloadData(articles, aid) 139 | } 140 | 141 | return downloadData 142 | } 143 | 144 | //生成专栏下载数据 145 | func extractColumnDownloadData(articles []*service.Article, aid int) []downloader.Datum { 146 | data := downloader.EmptyData 147 | 148 | key := "df" 149 | for _, article := range articles { 150 | if aid > 0 && article.ID != aid { 151 | continue 152 | } 153 | urls := []downloader.URL{} 154 | if article.AudioDownloadURL != "" { 155 | urls = []downloader.URL{ 156 | { 157 | URL: article.AudioDownloadURL, 158 | Size: article.AudioSize, 159 | Ext: "mp3", 160 | }, 161 | } 162 | } 163 | 164 | streams := map[string]downloader.Stream{ 165 | key: { 166 | URLs: urls, 167 | Size: article.AudioSize, 168 | Quality: key, 169 | }, 170 | } 171 | 172 | data = append(data, downloader.Datum{ 173 | ID: article.ID, 174 | Title: article.ArticleTitle, 175 | IsCanDL: article.IsCanPreview(), 176 | Streams: streams, 177 | Type: "专栏", 178 | }) 179 | } 180 | 181 | return data 182 | } 183 | 184 | //生成视频下载数据 185 | func extractVideoDownloadData(articles []*service.Article, aid int) []downloader.Datum { 186 | data := downloader.EmptyData 187 | 188 | videoIds := map[int]string{} 189 | 190 | videoData := make([]*downloader.Datum, 0) 191 | 192 | for _, article := range articles { 193 | if aid > 0 && article.ID != aid { 194 | continue 195 | } 196 | 197 | videoIds[article.ID] = article.VideoID 198 | 199 | videoMediaMaps := &map[string]downloader.VideoMediaMap{} 200 | utils.UnmarshalJSON(article.VideoMediaMap, videoMediaMaps) 201 | 202 | urls := []downloader.URL{} 203 | 204 | streams := map[string]downloader.Stream{} 205 | for key, videoMediaMap := range *videoMediaMaps { 206 | streams[key] = downloader.Stream{ 207 | URLs: urls, 208 | Size: videoMediaMap.Size, 209 | Quality: key, 210 | } 211 | } 212 | 213 | datum := &downloader.Datum{ 214 | ID: article.ID, 215 | Title: article.ArticleTitle, 216 | IsCanDL: article.IsCanPreview(), 217 | Streams: streams, 218 | Type: "视频", 219 | } 220 | 221 | videoData = append(videoData, datum) 222 | } 223 | if !_info { 224 | wgp := utils.NewWaitGroupPool(10) 225 | for _, datum := range videoData { 226 | wgp.Add() 227 | go func(datum *downloader.Datum, streams map[int]string) { 228 | defer func() { 229 | wgp.Done() 230 | }() 231 | if datum.IsCanDL { 232 | v3ArticleInfo, err := application.V3ArticleInfo(datum.ID) 233 | if err != nil { 234 | panic(err) 235 | } 236 | for _, info := range v3ArticleInfo.Data.Info.Video.HlsMedias { 237 | if info.Quality != "hd" { 238 | continue 239 | } 240 | if urls, aesBytes, err := utils.M3u8URLsAndAesKey(info.Url); err == nil { 241 | key := strings.ToLower(info.Quality) 242 | stream := datum.Streams[key] 243 | stream.AesKeyBytes = aesBytes 244 | stream.Size = info.Size 245 | for _, url := range urls { 246 | stream.URLs = append(stream.URLs, downloader.URL{ 247 | URL: url, 248 | Ext: "ts", 249 | }) 250 | } 251 | datum.Streams[key] = stream 252 | } else { 253 | fmt.Println("M3u8URLsAndAesKey error") 254 | panic(err) 255 | } 256 | } 257 | 258 | for k, v := range datum.Streams { 259 | if len(v.URLs) == 0 { 260 | delete(datum.Streams, k) 261 | } 262 | } 263 | } 264 | }(datum, videoIds) 265 | } 266 | wgp.Wait() 267 | } 268 | /* 269 | if !_info { 270 | wgp := utils.NewWaitGroupPool(10) 271 | for _, datum := range videoData { 272 | wgp.Add() 273 | go func(datum *downloader.Datum, streams map[int]string) { 274 | defer func() { 275 | wgp.Done() 276 | }() 277 | if datum.IsCanDL { 278 | playInfo, _ := application.GetVideoPlayInfo(datum.ID, streams[datum.ID]) 279 | for _, info := range playInfo.PlayInfoList.PlayInfo { 280 | if urls, err := utils.M3u8URLs(info.URL); err == nil { 281 | key := strings.ToLower(info.Definition) 282 | stream := datum.Streams[key] 283 | for _, url := range urls { 284 | stream.URLs = append(stream.URLs, downloader.URL{ 285 | URL: url, 286 | Ext: "ts", 287 | }) 288 | } 289 | datum.Streams[key] = stream 290 | } 291 | } 292 | 293 | for k, v := range datum.Streams { 294 | if len(v.URLs) == 0 { 295 | delete(datum.Streams, k) 296 | } 297 | } 298 | } 299 | }(datum, videoIds) 300 | } 301 | wgp.Wait() 302 | } 303 | 304 | */ 305 | 306 | for _, d := range videoData { 307 | data = append(data, *d) 308 | } 309 | 310 | return data 311 | } 312 | 313 | func printExtractDownloadData(v interface{}) { 314 | jsonData, err := json.MarshalIndent(v, "", " ") 315 | if err != nil { 316 | fmt.Println(err) 317 | } else { 318 | fmt.Printf("%s\n", jsonData) 319 | } 320 | } 321 | -------------------------------------------------------------------------------- /downloader/downloader.go: -------------------------------------------------------------------------------- 1 | package downloader 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "net/http" 10 | "os" 11 | "path/filepath" 12 | "strings" 13 | "sync" 14 | "time" 15 | 16 | "github.com/cheggaaa/pb" 17 | "github.com/udoless/geektime-downloader/aeswrap" 18 | "github.com/udoless/geektime-downloader/requester" 19 | "github.com/udoless/geektime-downloader/utils" 20 | ) 21 | 22 | func progressBar(size int, prefix string) *pb.ProgressBar { 23 | bar := pb.New(size).SetUnits(pb.U_BYTES).SetRefreshRate(time.Second * 1) 24 | bar.ShowSpeed = true 25 | bar.ShowFinalTime = true 26 | bar.SetMaxWidth(1000) 27 | 28 | if prefix != "" { 29 | bar.Prefix(prefix) 30 | } 31 | 32 | return bar 33 | } 34 | 35 | //Download download data 36 | func Download(v Datum, stream string, path string) error { 37 | if !v.IsCanDL { 38 | return errors.New("该课程目录未付费，或者不支持下载") 39 | } 40 | 41 | //按大到小排序 42 | v.genSortedStreams() 43 | 44 | title := utils.FileName(v.Title, "") 45 | title = replacer.Replace(title) 46 | 47 | if stream == "" { 48 | if len(v.sortedStreams) == 0 { 49 | vStr, _ := json.Marshal(v) 50 | panic("文章解析异常：" + string(vStr)) 51 | } 52 | stream = v.sortedStreams[0].name 53 | } 54 | data, ok := v.Streams[strings.ToLower(stream)] 55 | if !ok { 56 | return fmt.Errorf("指定要下载的类型不存在：%s", stream) 57 | } 58 | 59 | //判断下载连接是否存在 60 | if len(data.URLs) == 0 { 61 | return nil 62 | } 63 | 64 | filePreName := filepath.Join(path, title) 65 | fileName, err := utils.FilePath(filePreName, "mp4", false) 66 | 67 | if err != nil { 68 | return err 69 | } 70 | 71 | _, mergedFileExists, err := utils.FileSize(fileName) 72 | if err != nil { 73 | return err 74 | } 75 | 76 | // After the merge, the file size has changed, so we do not check whether the size matches 77 | if mergedFileExists { 78 | // fmt.Printf("%s: file already exists, skipping\n", mergedFilePath) 79 | return nil 80 | } 81 | 82 | bar := progressBar(data.Size, title) 83 | bar.Start() 84 | 85 | // chunkSizeMB := 1 86 | chunkSizeMB := 0 87 | 88 | if len(data.URLs) == 1 { 89 | err := Save(data.URLs[0], filePreName, bar, chunkSizeMB, data.AesKeyBytes) 90 | if err != nil { 91 | return err 92 | } 93 | bar.Finish() 94 | return nil 95 | } 96 | 97 | wgp := utils.NewWaitGroupPool(8) 98 | 99 | errs := make([]error, 0) 100 | lock := sync.Mutex{} 101 | parts := make([]string, len(data.URLs)) 102 | 103 | for index, url := range data.URLs { 104 | if len(errs) > 0 { 105 | break 106 | } 107 | 108 | partFileName := fmt.Sprintf("%s[%d]", filePreName, index) 109 | partFilePath, err := utils.FilePath(partFileName, url.Ext, false) 110 | if err != nil { 111 | return err 112 | } 113 | parts[index] = partFilePath 114 | 115 | wgp.Add() 116 | go func(url URL, fileName string, bar *pb.ProgressBar) { 117 | defer wgp.Done() 118 | err := Save(url, fileName, bar, chunkSizeMB, data.AesKeyBytes) 119 | if err != nil { 120 | lock.Lock() 121 | errs = append(errs, err) 122 | lock.Unlock() 123 | } 124 | }(url, partFileName, bar) 125 | } 126 | 127 | wgp.Wait() 128 | 129 | if len(errs) > 0 { 130 | return errs[0] 131 | } 132 | 133 | bar.Finish() 134 | 135 | if v.Type != "视频" { 136 | return nil 137 | } 138 | 139 | // merge 140 | // fmt.Printf("Merging video parts into %s\n", mergedFilePath) 141 | err = utils.MergeToMP4(parts, fileName, title) 142 | 143 | return err 144 | } 145 | 146 | // Save save url file 147 | func Save( 148 | urlData URL, fileName string, bar *pb.ProgressBar, chunkSizeMB int, aesKeyBytes []byte, 149 | ) error { 150 | if urlData.Size == 0 { 151 | size, err := requester.Size(urlData.URL) 152 | if err != nil { 153 | return err 154 | } 155 | urlData.Size = size 156 | } 157 | 158 | var err error 159 | filePath, err := utils.FilePath(fileName, urlData.Ext, false) 160 | if err != nil { 161 | return err 162 | } 163 | fileSize, exists, err := utils.FileSize(filePath) 164 | if err != nil { 165 | return err 166 | } 167 | if bar == nil { 168 | bar = progressBar(urlData.Size, fileName) 169 | bar.Start() 170 | } 171 | // Skip segment file 172 | // TODO: Live video URLs will not return the size 173 | if exists && fileSize == urlData.Size { 174 | bar.Add(fileSize) 175 | return nil 176 | } 177 | tempFilePath := filePath + ".download" 178 | tempFileSize, _, err := utils.FileSize(tempFilePath) 179 | 180 | if err != nil { 181 | return err 182 | } 183 | headers := map[string]string{} 184 | var ( 185 | file *os.File 186 | fileError error 187 | ) 188 | if tempFileSize > 0 { 189 | // range start from 0, 0-1023 means the first 1024 bytes of the file 190 | headers["Range"] = fmt.Sprintf("bytes=%d-", tempFileSize) 191 | file, fileError = os.OpenFile(tempFilePath, os.O_APPEND|os.O_WRONLY, 0644) 192 | bar.Add(tempFileSize) 193 | } else { 194 | file, fileError = os.Create(tempFilePath) 195 | } 196 | if fileError != nil { 197 | return fileError 198 | } 199 | 200 | // close and rename temp file at the end of this function 201 | defer func() { 202 | // must close the file before rename or it will cause 203 | // `The process cannot access the file because it is being used by another process.` error. 204 | file.Close() 205 | if err == nil { 206 | os.Rename(tempFilePath, filePath) 207 | } 208 | }() 209 | 210 | if chunkSizeMB > 0 { 211 | var start, end, chunkSize int 212 | chunkSize = chunkSizeMB * 1024 * 1024 213 | remainingSize := urlData.Size 214 | if tempFileSize > 0 { 215 | start = tempFileSize 216 | remainingSize -= tempFileSize 217 | } 218 | chunk := remainingSize / chunkSize 219 | if remainingSize%chunkSize != 0 { 220 | chunk++ 221 | } 222 | var i = 1 223 | for ; i <= chunk; i++ { 224 | end = start + chunkSize - 1 225 | headers["Range"] = fmt.Sprintf("bytes=%d-%d", start, end) 226 | temp := start 227 | for i := 0; ; i++ { 228 | var written int 229 | var err error 230 | if urlData.Ext == "ts" { 231 | written, err = writeFileWithDecAes(urlData.URL, file, headers, bar, aesKeyBytes) 232 | } else { 233 | written, err = writeFile(urlData.URL, file, headers, bar) 234 | } 235 | if err == nil { 236 | break 237 | } else if i+1 >= 3 { 238 | return err 239 | } 240 | temp += written 241 | headers["Range"] = fmt.Sprintf("bytes=%d-%d", temp, end) 242 | time.Sleep(1 * time.Second) 243 | } 244 | start = end + 1 245 | } 246 | } else { 247 | temp := tempFileSize 248 | for i := 0; ; i++ { 249 | var written int 250 | var err error 251 | if urlData.Ext == "ts" { 252 | written, err = writeFileWithDecAes(urlData.URL, file, headers, bar, aesKeyBytes) 253 | } else { 254 | written, err = writeFile(urlData.URL, file, headers, bar) 255 | } 256 | if err == nil { 257 | break 258 | } else if i+1 >= 3 { 259 | return err 260 | } 261 | temp += written 262 | headers["Range"] = fmt.Sprintf("bytes=%d-", temp) 263 | time.Sleep(1 * time.Second) 264 | } 265 | } 266 | 267 | return nil 268 | } 269 | 270 | func writeFile( 271 | url string, file *os.File, headers map[string]string, bar *pb.ProgressBar, 272 | ) (int, error) { 273 | res, err := requester.Req(http.MethodGet, url, nil, headers) 274 | if err != nil { 275 | return 0, err 276 | } 277 | defer res.Body.Close() 278 | 279 | writer := io.MultiWriter(file, bar) 280 | // Note that io.Copy reads 32kb(maximum) from input and writes them to output, then repeats. 281 | // So don't worry about memory. 282 | written, copyErr := io.Copy(writer, res.Body) 283 | if copyErr != nil && copyErr != io.EOF { 284 | return int(written), fmt.Errorf("file copy error: %s", copyErr) 285 | } 286 | return int(written), nil 287 | } 288 | 289 | func writeFileWithDecAes( 290 | url string, file *os.File, headers map[string]string, bar *pb.ProgressBar, aesKeyBytes []byte, 291 | ) (int, error) { 292 | res, err := requester.Req(http.MethodGet, url, nil, headers) 293 | if err != nil { 294 | return 0, err 295 | } 296 | defer res.Body.Close() 297 | 298 | bodyBytes, err := ioutil.ReadAll(res.Body) 299 | if err != nil { 300 | return 0, err 301 | } 302 | bodyBytes, err = aeswrap.AesDecrypt(bodyBytes, aesKeyBytes) 303 | if err != nil { 304 | return 0, err 305 | } 306 | 307 | writer := io.MultiWriter(file, bar) 308 | written, err := writer.Write(bodyBytes) 309 | if err != nil { 310 | return 0, err 311 | } 312 | return written, nil 313 | } 314 | --------------------------------------------------------------------------------