├── go.mod ├── .gitignore ├── mock.go ├── LICENSE ├── go.sum ├── README.md ├── main_test.go └── main.go /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/timolinn/joncalhoun-dl 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/PuerkitoBio/goquery v1.5.1 7 | github.com/stretchr/testify v1.5.1 8 | golang.org/x/net v0.0.0-20200301022130-244492dfa37a 9 | ) 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | joncalhoun-dl 8 | 9 | # Test binary, built with `go test -c` 10 | *.test 11 | 12 | # Output of the go coverage tool, specifically when used with LiteIDE 13 | *.out 14 | 15 | # Dependency directories (remove the comment below to include it) 16 | vendor/ 17 | videos 18 | cache/ 19 | 20 | .DS_STORE 21 | 22 | *.html 23 | *.ytdl 24 | *.part 25 | *.mp4 26 | -------------------------------------------------------------------------------- /mock.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "net/http" 4 | 5 | // RoundTripperFunc implements the http.RoundTripper interface 6 | type RoundTripperFunc func(r *http.Request) (*http.Response, error) 7 | 8 | // RoundTrip function 9 | func (fn RoundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) { 10 | return fn(r) 11 | } 12 | 13 | // TestingHTTPClient returns http client with stubbed transport 14 | // func TestingHTTPClient(fn RoundTripperFunc) *http.Client { 15 | // return &http.Client{ 16 | // Transport: RoundTripperFunc(fn), 17 | // } 18 | // } 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Timothy Onyiuke 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= 2 | github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= 3 | github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= 4 | github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= 5 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 6 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 7 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 8 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 9 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 10 | github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= 11 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 12 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 13 | golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 14 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 15 | golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= 16 | golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 17 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 18 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 19 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 20 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 21 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # joncalhoun-dl 🔥⬇ 2 | 3 | Downloads Go tutorial videos from 4 | 5 | > **Before you proceed, note that you must be a paid user for the paid content to download** 6 | 7 | Kindly create your account [here](https://courses.calhoun.io/signup?). Jon is a great teacher, consider buying his premium courses if you want to. 8 | 9 | ## Installation 10 | 11 | + Ensure [youtube-dl](https://github.com/ytdl-org/youtube-dl#installation) is installed and in your PATH. 12 | 13 | To install this package run 14 | 15 | ```bash 16 | $ go get -u github.com/timolinn/joncalhoun-dl 17 | ``` 18 | 19 | ### To build from source run 20 | 21 | ```bash 22 | $ git clone git@github.com:timolinn/joncalhoun-dl.git 23 | $ cd joncalhoun-dl 24 | $ go build . 25 | ``` 26 | 27 | ## How to use 28 | 29 | If you installed via `go get`, you can simply run 30 | 31 | ```bash 32 | $ joncalhoun-dl -email=jon@doe.com -password=12345 -course=gophercises -output=your-chosen-directory 33 | [joncalhoun-dl]: fetching video urls for gophercises 34 | [joncalhoun-dl]: fetching data from https://courses.calhoun.io/courses/cor_gophercises... 35 | ``` 36 | 37 | If you built from source, the compiled binary should be in the current folder. 38 | 39 | ```bash 40 | $ ./joncalhoun-dl -email=jon@doe.com -password=12345 -course=gophercises -output=your-chosen-directory 41 | [joncalhoun-dl]: fetching video urls for gophercises 42 | [joncalhoun-dl]: fetching data from https://courses.calhoun.io/courses/cor_gophercises... 43 | ``` 44 | 45 | Also note, video downloads **resumes** from where it stopped, so should you experience network interruption nothing to worry about just make sure the output directory remains the same. 46 | 47 | ### Command [OPTIONS] 48 | 49 | + `--email` [required] : Your account email address. Sign up [here](https://courses.calhoun.io/signup?) 50 | + `--password` [required]: Your account password. _Unlike the unix password prompt, this will not hide your password by default, you'll have to keep an eye over your shoulder 😉_ 51 | + `--course` [gophercises | algorithmswithgo | testwithgo | webdevwithgo]: This is the name of the course you want to download. **defaults** to `"gophercises"` 52 | + `--output` [optional]: This is the output directory, which means where you want the videos to be saved. It must be an absolute path. If this is not specified, we will try to create a `"/[course] folder"` (ie. the specified course name eg. `gophercises`) within your current working directory. 53 | + `--cache` [optional]: Specify your desired location where the cache will be saved. joncalhoun-dl uses the cached data to resume downloads incase of an interruption. It also prevents unecessary repeated calls to the remote server. This will **default** to a `joncalhoun-dl-cache` folder within the output directoy. 54 | + `--help` [optional]: Prints usage options 55 | 56 | ### Supported courses 57 | 58 | + [x] Gophercises - [gophercises](https://gophercises.com) 59 | + [x] Algorithms with Go - [algorithmswithgo](https://algorithmswithgo.com) 60 | + [x] Testing with Go - [testwithgo](https://testwithgo.com/) 61 | + [x] Web development with Go - [webdevwithgo](https://www.usegolang.com/) 62 | 63 | ### Contributing 64 | 65 | There is still a couple features to implement, check the TODO list below. 66 | 67 | + Start by forking this repo 68 | + Create your branch 69 | + Implement your fixes, changes etc. 70 | + Open a Pull Request 71 | 72 | ### Issues 73 | 74 | If you find a bug please create an [issue](https://github.com/timolinn/joncalhoun-dl/issues/new) 75 | 76 | ### Tests 77 | 78 | To run existing tests 79 | 80 | ```bash 81 | $ go test 82 | ``` 83 | 84 | ## TODO 85 | 86 | + [x] Add caching for requests 87 | + [x] Add default output directory 88 | + [x] Add output directoy flag 89 | + [x] provide packaged release 90 | + [ ] Add more unit tests 91 | + [ ] check for authentication error 92 | + [ ] prevent signin when using cache 93 | + [ ] choose video quality 94 | + [ ] reduce cache size by storing fewer data 95 | 96 | ### Authors 97 | 98 | + Timothy Onyiuke - _([github](https://github.com/timolinn), [twitter](https://twitter.com/timolinn_))_ 99 | + Damilare Lana - _([github](https://github.com/damilarelana))_ 100 | 101 | If you find this repository to be of any help, please consider giving it Star! 🔥 102 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io/ioutil" 5 | "net/http" 6 | "os" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestSaveHTMLContent(t *testing.T) { 14 | t.Run("saveHTMLContent: should saves html file to filesystem", func(t *testing.T) { 15 | filename := "temp.html" 16 | *cachelocation = "." 17 | r := strings.NewReader(` 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 30 | 31 | 32 | Gophercises | courses.calhoun.io 33 | 34 | 35 |
36 | 37 | 38 | `) 39 | saveHTMLContent(filename, r) 40 | 41 | _, err := os.Stat(filename) 42 | if err != nil { 43 | t.Error() 44 | return 45 | } 46 | assert.False(t, os.IsNotExist(err)) 47 | if err := os.Remove(filename); err != nil { 48 | t.Error() 49 | return 50 | } 51 | }) 52 | } 53 | 54 | func TestGetCourseHTML(t *testing.T) { 55 | handler := func(r *http.Request) (*http.Response, error) { 56 | body := ` 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 69 | 70 | 71 | Gophercises | courses.calhoun.io 72 | 73 | 74 | 75 | 76 |
77 |
78 |

79 |
Quiz Game
80 |

81 | 82 |
83 |

Create a program to run timed quizes via the command line.

84 | 85 |
86 | 87 |
88 | 89 | 90 |
91 |
92 | Thumbnail for Overview 93 |
94 |
95 | 96 | Overview 97 | 98 |
99 |
100 | 101 | 102 | 103 |
104 |
105 | Thumbnail for Solution - Part 1 106 |
107 |
108 | 109 | Solution - Part 1 110 | 111 |
112 |
113 | 114 | 115 | 116 |
117 |
118 | Thumbnail for Solution - Part 2 119 |
120 |
121 | 122 | Solution - Part 2 123 | 124 |
125 |
126 | 127 | 128 |
129 |
130 |
` 131 | 132 | return &http.Response{ 133 | Body: ioutil.NopCloser(strings.NewReader(body)), 134 | }, nil 135 | } 136 | t.Run("getCourseHTML: fetches the course HTML main page, this page contains links to individual lessons that make up the course", func(t *testing.T) { 137 | client, _ := NewClient(WithTransport(handler)) 138 | // set course name 139 | *course = *course + "_test" 140 | getCourseHTML(courses["gophercises"], client) 141 | assert.True(t, fileExists(*course+".html")) 142 | os.Remove(*course + ".html") 143 | }) 144 | } 145 | 146 | func TestFileExists(t *testing.T) { 147 | t.Run("fileExists: should check if a file exists when given a path", func(t *testing.T) { 148 | assert.True(t, fileExists("README.md")) 149 | assert.False(t, fileExists("nonExistingFile")) 150 | }) 151 | } 152 | 153 | func TestDirExists(t *testing.T) { 154 | t.Run("dirExists: should check if a directory exists when given a path", func(t *testing.T) { 155 | dir := "testdir" 156 | err := os.Mkdir(dir, 0755) 157 | if err != nil { 158 | t.Error() 159 | return 160 | } 161 | assert.True(t, dirExists("testdir")) 162 | assert.False(t, dirExists("nonExistingDir")) 163 | if err := os.Remove(dir); err != nil { 164 | t.Error() 165 | return 166 | } 167 | }) 168 | } 169 | 170 | func init() { 171 | os.Setenv("APP_ENV", "test") 172 | } 173 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "errors" 7 | "flag" 8 | "fmt" 9 | "io" 10 | "log" 11 | "net/http" 12 | "net/http/cookiejar" 13 | "net/url" 14 | "os" 15 | "os/exec" 16 | "strings" 17 | "time" 18 | 19 | "github.com/PuerkitoBio/goquery" 20 | "golang.org/x/net/publicsuffix" 21 | ) 22 | 23 | var email = flag.String("email", "", "your account email") 24 | var password = flag.String("password", "", "your account password") 25 | var course = flag.String("course", "gophercises", "course name") 26 | var outputdir = flag.String("output", "", "output directory\n ie. where the course videos are saved") 27 | var cachelocation = flag.String("cache", "", "specifies where the cache will be saved\nDefaults to a joncalhoun-dl-cache folder within the output directoy") 28 | var help = flag.Bool("help", false, "prints this output") 29 | 30 | // this will be used by youtube-dl binary to download video 31 | var referer = "https://courses.calhoun.io" 32 | 33 | var courses = map[string]string{ 34 | "testwithgo": "https://courses.calhoun.io/courses/cor_test", 35 | "gophercises": "https://courses.calhoun.io/courses/cor_gophercises", 36 | "algorithmswithgo": "https://courses.calhoun.io/courses/cor_algo", 37 | "webdevwithgo": "https://courses.calhoun.io/courses/cor_webdev", 38 | } 39 | var delayDuration = 5 40 | 41 | // ClientOption is the type of constructor options for NewClient(...). 42 | type ClientOption func(*http.Client) error 43 | 44 | func checkError(err error) { 45 | if err != nil { 46 | log.Fatalf("[joncalhoun-dl]:[Error]: %v.\nif you think this is unusual please create an issue %s\n", 47 | err, 48 | "https://github.com/timolinn/joncalhoun-dl/issues/new") 49 | } 50 | } 51 | 52 | // NewClient constructs anew client which can make requests 53 | // to course website 54 | func NewClient(options ...ClientOption) (*http.Client, error) { 55 | // Cookiejar provides automatic cookie management 56 | // that would normally be accessed only via the browser 57 | opts := cookiejar.Options{ 58 | PublicSuffixList: publicsuffix.List, 59 | } 60 | jar, err := cookiejar.New(&opts) 61 | checkError(err) 62 | c := &http.Client{Jar: jar} 63 | for _, option := range options { 64 | err := option(c) 65 | if err != nil { 66 | return nil, err 67 | } 68 | } 69 | return c, nil 70 | } 71 | 72 | // WithTransport configures the client to use a different transport 73 | func WithTransport(fn RoundTripperFunc) ClientOption { 74 | return func(client *http.Client) error { 75 | client.Transport = RoundTripperFunc(fn) 76 | return nil 77 | } 78 | } 79 | 80 | func main() { 81 | // Parse commandline options 82 | flag.Parse() 83 | 84 | // Print usage options 85 | if *help { 86 | flag.PrintDefaults() 87 | return 88 | } 89 | 90 | // validate input 91 | validateInput() 92 | 93 | client, err := NewClient() 94 | checkError(err) 95 | 96 | // Login 97 | signin(client) 98 | 99 | location := *outputdir + "/%(title)s.%(ext)s" 100 | if *outputdir == "" { 101 | cwd, err := os.Getwd() 102 | checkError(err) 103 | *outputdir = cwd + "/" + *course 104 | location = *outputdir + "/%(title)s.%(ext)s" 105 | } 106 | fmt.Printf("[joncalhoun-dl]: output directory is %s\n", *outputdir) 107 | 108 | // do some chores 109 | setup() 110 | 111 | // fetch video urls 112 | videoURLs := getURLs(client) 113 | 114 | ctx, cancel := context.WithCancel(context.Background()) 115 | defer cancel() 116 | for i, videoURL := range videoURLs { 117 | if videoURL != "" { 118 | fmt.Printf("[joncalhoun-dl]: downloading lesson 0%d of %s\n", i+1, *course) 119 | fmt.Printf("[joncalhoun-dl]:[exec]: youtube-dl %s --referer %s -o %s\n", videoURL, referer, location) 120 | cmd := exec.CommandContext(ctx, "youtube-dl", videoURL, "--referer", referer, "-o", location) 121 | cmd.Stdout = os.Stdout 122 | cmd.Stderr = os.Stderr 123 | if err := cmd.Start(); err != nil { 124 | log.Fatal(err) 125 | } 126 | if err := cmd.Wait(); err != nil { 127 | log.Fatal(err) 128 | } 129 | fmt.Printf("[joncalhoun-dl]: downloaded lesson 0%d\n", i+1) 130 | } else { 131 | fmt.Printf("[joncalhoun-dl]: Page for lesson 0%d does not have an embedded video \n", i+1) 132 | } 133 | } 134 | fmt.Println("Done! 🚀") 135 | } 136 | 137 | func validateInput() { 138 | if !isSupported(*course) { 139 | err := errors.New("course not supported yet") 140 | checkError(err) 141 | } 142 | 143 | if *email == "" || *password == "" { 144 | checkError(errors.New("try adding: --email=jon@examp.com --password=12345' to the command")) 145 | } 146 | } 147 | 148 | func setup() { 149 | // create output directory if it does not exist yet 150 | if !dirExists(*outputdir) { 151 | err := os.Mkdir(*outputdir, 0755) 152 | checkError(err) 153 | } 154 | 155 | if *cachelocation == "" { 156 | *cachelocation = *outputdir + "/" + "joncalhoun-dl-cache" 157 | } 158 | 159 | // create cache location if it does not exist 160 | if !dirExists(*cachelocation) { 161 | err := os.Mkdir(*cachelocation, 0755) 162 | checkError(err) 163 | } 164 | } 165 | 166 | func signin(client *http.Client) { 167 | // Login and create session 168 | 169 | fmt.Println("[joncalhoun-dl]: signing in...") 170 | _, err := client.PostForm("https://courses.calhoun.io/signin", url.Values{ 171 | "email": {*email}, 172 | "password": {*password}, 173 | }) 174 | checkError(err) 175 | fmt.Println("[joncalhoun-dl]: sign in successful") 176 | } 177 | 178 | func getCourseHTML(url string, client *http.Client) { 179 | // Make a Get Request to the course URL and fetch the HTML 180 | // user must be logged in 181 | fmt.Printf("[joncalhoun-dl]: fetching data for %s...\n", url) 182 | res, err := client.Get(url) 183 | checkError(err) 184 | defer res.Body.Close() 185 | 186 | // Write data to file 187 | saveHTMLContent(*course+".html", res.Body) 188 | } 189 | 190 | func getURLs(client *http.Client) []string { 191 | fmt.Printf("[joncalhoun-dl]: fetching video urls for %s\n", *course) 192 | var urls []string 193 | var file *os.File 194 | var err error 195 | 196 | // check if course page is cached 197 | if isCached(*course + ".html") { 198 | fmt.Printf("[joncalhoun-dl]: loading %s data from cache \n", *course) 199 | file, err = loadFromCache(*course + ".html") 200 | checkError(err) 201 | } else { 202 | // fecth from remote if not cached 203 | fmt.Printf("[joncalhoun-dl]: fetching %s data from remote\n", *course) 204 | res, err := client.Get(courses[*course]) 205 | checkError(err) 206 | defer res.Body.Close() 207 | 208 | // cache raw HTML data 209 | getCourseHTML(courses[*course], client) 210 | file, err = loadFromCache(*course + ".html") 211 | checkError(err) 212 | } 213 | 214 | doc, err := goquery.NewDocumentFromReader(file) 215 | checkError(err) 216 | 217 | // parses the HTML tree to extract url 218 | // where the lesson video is located 219 | doc.Find("a").Each(func(i int, s *goquery.Selection) { 220 | href, _ := s.Attr("href") 221 | switch *course { 222 | case "testwithgo": 223 | // each lesson link should contain this substring 224 | // else ignore 225 | if strings.Contains(href, "/lessons/les_twg") { 226 | urls = append(urls, "https://courses.calhoun.io"+href) 227 | } 228 | case "gophercises": 229 | // each lesson link should contain this substring 230 | // else ignore 231 | if strings.Contains(href, "/lessons/les_goph") { 232 | urls = append(urls, "https://courses.calhoun.io"+href) 233 | } 234 | case "webdevwithgo": 235 | if strings.Contains(href, "/lessons/les_wd") { 236 | urls = append(urls, "https://courses.calhoun.io"+href) 237 | } 238 | case "advancedwebdevwithgo": 239 | log.Fatal("'Advanced Web Development with Go' not supported yet") 240 | case "algorithmswithgo": 241 | if strings.Contains(href, "/lessons/les_algo") { 242 | urls = append(urls, "https://courses.calhoun.io"+href) 243 | } 244 | default: 245 | log.Fatal("course not supported yet. feel free to send a pull request") 246 | } 247 | }) 248 | 249 | videoURLs := []string{} 250 | for _, url := range urls { 251 | videoURLs = append(videoURLs, getVideoURL(url, client)) 252 | // we don't want to send too many requests in a short time 253 | // this naively simulates human behaviour 254 | fmt.Printf("[joncalhoun-dl]: waiting 5 seconds\n") 255 | time.Sleep(time.Duration(delayDuration) * time.Second) 256 | } 257 | return videoURLs 258 | } 259 | 260 | func getVideoURL(url string, client *http.Client) string { 261 | fmt.Printf("[joncalhoun-dl]: fetching video url for lesson %s\n", url) 262 | var videoID string 263 | var file *os.File 264 | var err error 265 | 266 | // check cache for existing webpage 267 | name := strings.Split(url, "/")[4] 268 | filename := name + ".html" 269 | if isCached(filename) { 270 | fmt.Printf("[joncalhoun-dl]: loading %s from cache\n", name) 271 | file, err = loadFromCache(filename) 272 | checkError(err) 273 | 274 | // no need to delay when loading from cash 275 | delayDuration = 0 276 | } else { 277 | // fetch web page where video lives 278 | fmt.Printf("[joncalhoun-dl]: fetching %s from remote\n", filename) 279 | res, err := client.Get(url) 280 | checkError(err) 281 | defer res.Body.Close() 282 | 283 | // To provide caching support we save the resulting 284 | // html in the cache folder 285 | saveHTMLContent(filename, res.Body) 286 | file, err = loadFromCache(filename) 287 | delayDuration = 5 288 | } 289 | 290 | // convert return data to parsable HTML Document 291 | doc, err := goquery.NewDocumentFromReader(file) 292 | checkError(err) 293 | iframe := doc.Find("iframe") 294 | videoID, _ = iframe.Attr("src") 295 | fmt.Printf("[joncalhoun-dl]:[video ID] %s\n", videoID) 296 | return videoID 297 | } 298 | 299 | func saveHTMLContent(filename string, r io.Reader) { 300 | f, err := os.Create(*cachelocation + "/" + filename) 301 | checkError(err) 302 | defer f.Close() 303 | filewriter := bufio.NewWriter(f) 304 | _, err = filewriter.ReadFrom(r) 305 | checkError(err) 306 | 307 | filewriter.Flush() 308 | } 309 | 310 | func fileExists(filename string) bool { 311 | info, err := os.Stat(filename) 312 | if os.IsNotExist(err) { 313 | return false 314 | } 315 | return !info.IsDir() 316 | } 317 | 318 | func dirExists(path string) bool { 319 | info, err := os.Stat(path) 320 | if os.IsNotExist(err) { 321 | return false 322 | } 323 | return info.IsDir() 324 | } 325 | 326 | func isCached(name string) bool { 327 | if fileExists(*cachelocation + "/" + name) { 328 | return true 329 | } 330 | return false 331 | } 332 | 333 | func loadFromCache(name string) (*os.File, error) { 334 | return os.OpenFile(*cachelocation+"/"+name, os.O_RDWR, 0666) 335 | } 336 | 337 | func isSupported(coursename string) bool { 338 | if courses[coursename] != "" { 339 | return true 340 | } 341 | return false 342 | } 343 | --------------------------------------------------------------------------------