├── .coveragerc
├── .gitignore
├── .travis.yml
├── CONTRIBUTING.md
├── README.md
├── coursera-dl
├── coursera-dl.bat
├── coursera
    ├── __init__.py
    ├── cookies.py
    ├── coursera_dl.py
    ├── credentials.py
    ├── define.py
    ├── downloaders.py
    ├── test
    │   ├── __init__.py
    │   ├── fixtures
    │   │   ├── auth
    │   │   │   ├── netrc
    │   │   │   └── not_netrc
    │   │   ├── cookies
    │   │   │   ├── chrome_cookies.txt
    │   │   │   ├── firefox_cookies.txt
    │   │   │   ├── firefox_cookies_expired.txt
    │   │   │   └── firefox_cookies_without_coursera.txt
    │   │   └── html
    │   │   │   ├── hidden-videos.html
    │   │   │   ├── links-to-wikipedia.html
    │   │   │   ├── multiple-resources-with-the-same-format.html
    │   │   │   ├── parsing-datasci-001-with-bs4.html
    │   │   │   ├── parsing-malsoftware-001-with-bs4.html
    │   │   │   ├── parsing-startup-001-with-bs4.html
    │   │   │   ├── parsing-wealthofnations-001-with-bs4.html
    │   │   │   ├── preview.html
    │   │   │   ├── regular-syllabus.html
    │   │   │   ├── sections-not-to-be-missed-2.html
    │   │   │   └── sections-not-to-be-missed.html
    │   ├── test_cookies.py
    │   ├── test_credentials.py
    │   ├── test_downloaders.py
    │   ├── test_parsing.py
    │   └── test_utils.py
    └── utils.py
└── requirements.txt


/.coveragerc:
--------------------------------------------------------------------------------
 1 | # -*- conf -*-
 2 | [report]
 3 | omit =
 4 |     */python?.?/*
 5 |     */site-packages/nose/*
 6 | 
 7 | exclude_lines =
 8 |     # Have to re-enable the standard pragma
 9 |     pragma: no cover
10 | 
11 |     # Python 2/3 compatibility
12 |     except ImportError
13 |     sys.version_info
14 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | *.py[co]
2 | *~
3 | .DS_Store
4 | _cache/
5 | .coverage
6 | 


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
 1 | language: python
 2 | python:
 3 |   - "2.6"
 4 |   - "2.7"
 5 |   - "3.2"
 6 |   - "3.3"
 7 |   - "pypy"
 8 | matrix:
 9 |   allow_failures:
10 |     - python: "pypy"
11 | # command to install dependencies
12 | install:
13 |   - pip install -r requirements.txt --use-mirrors
14 |   - pip install --use-mirrors coveralls
15 | # command to run tests
16 | script: nosetests --verbose --with-coverage --cover-package=coursera
17 | 
18 | after_success:
19 |   coveralls
20 | 


--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
 1 | Please, before sending patches, it is helpful if you:
 2 | 
 3 | * Make sure that you have all dependencies installed, like via
 4 | 
 5 |         pip install -r requirements
 6 | 
 7 | * Run the test suite with
 8 | 
 9 |         nosetests --verbose --with-coverage --cover-package=coursera
10 | 
11 | * Check for any potential bugs with the help of [`pep8`][pep8],
12 |   [`pyflakes`][pyflakes], and [`pylint`][pylint].
13 | 
14 | [pep8]: https://pypi.python.org/pypi/pep8
15 | [pyflakes]: https://pypi.python.org/pypi/pyflakes/
16 | [pylint]: https://pypi.python.org/pypi/pylint
17 | 
18 | If you happen to find any issue reported by these programs, we would welcome
19 | you to fix them, as they are usually very easy to fix and we would all
20 | benefit from a better code base.
21 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Coursera Downloader
  2 | 
  3 | [![Build Status](https://travis-ci.org/coursera-dl/coursera.png?branch=master)](https://travis-ci.org/coursera-dl/coursera)
  4 | [![Coverage Status](https://coveralls.io/repos/coursera-dl/coursera/badge.png)](https://coveralls.io/r/coursera-dl/coursera)
  5 | 
  6 | [Coursera][1] is arguably the leader in *massive open online courses* (MOOC)
  7 | with a selection of more than 300 classes from 62 different institutions [as of
  8 | February 2013][13]. Generous contributions by educators and institutions are
  9 | making excellent education available to many who could not afford it otherwise.
 10 | There are even non-profits with "feet on the ground" in remote areas of the
 11 | world who are helping spread the wealth (see the feedback below from [Tunapanda][14]).
 12 | 
 13 | This script makes it easier to batch download lecture resources (e.g., videos, ppt,
 14 | etc) for Coursera classes.  Given one or more class names and account credentials,
 15 | it obtains week and class names from the *lectures* page, and then downloads
 16 | the related materials into appropriately named files and directories.
 17 | 
 18 | Why is this helpful?  A utility like [`wget`][2] can work, but has the
 19 | following limitations:
 20 | 
 21 | 1. Video names have a number in them, but this does not correspond to the
 22 |    actual order.  Manually renaming them is a pain.
 23 | 2. Using names from the syllabus page provides more informative names.
 24 | 3. Using a wget in a for loop picks up extra videos which are not
 25 |    posted/linked, and these are sometimes duplicates.
 26 | 
 27 | *DownloadThemAll* is another possibility, but this script provides more
 28 | features such as appropriately named files.
 29 | 
 30 | This work was originally inspired in part by [youtube-dl][3] by which
 31 | I've downloaded many other good videos such as those from Khan Academy.
 32 | 
 33 | 
 34 | ## Features
 35 | 
 36 |   * Intentionally detailed names, so that it will display and sort properly
 37 |     on most interfaces (e.g., MX Video on Android phone).
 38 |   * Regex-based section (week) and lecture name filters to download only
 39 |     certain resources.
 40 |   * File format extension filter to grab resource types you want.
 41 |   * Login credentials accepted on command-line or from `.netrc` file
 42 |   * Core functionality tested on Linux, Mac and Windows.
 43 | 
 44 | 
 45 | ## Instructions
 46 | 
 47 | `coursera-dl` requires Python 2 (2.6 or newer) or Python 3 (3.2 or newer)
 48 | and a free Coursera account enrolled in the class of interest. On Windows
 49 | ensure that Python executable location is added to the PATH environment variable.
 50 | *Note:* You must already have (manually) agreed to the Honor of Code of the
 51 | particular courses that you want to use with `coursera-dl`.
 52 | 
 53 | ### Install any missing dependencies.
 54 | 
 55 | We strongly recommend that you consider installing Python packages with
 56 | [`pip`][17], as in it is the current [preferred method][18].  If you are
 57 | using `pip`, you can directly install all the dependencies from the
 58 | requirements file using `pip install -r requirements.txt`.
 59 | 
 60 | ### Installing dependencies on your own
 61 | 
 62 | **Warning:** This method is not recommended unless you know what you are
 63 | doing. Before filing bug reports, please check that the versions of your
 64 | modules are those recommended in the `requirements.txt` file.
 65 | 
 66 | You may choose to install the dependencies yourself, but our users had
 67 | issues that not all resources (videos etc.) were downloaded with versions
 68 | of the dependencies different than those listed in the `requirements.txt`
 69 | file.
 70 | 
 71 | In any case, you may want to install:
 72 | 
 73 | * [Beautiful Soup 4][5]: Required. See also html5lib below.
 74 |   - Ubuntu/Debian: `sudo apt-get install python-bs4`
 75 |   - Mac OSX + MacPorts: `sudo port install py-beautifulsoup4`
 76 |   - Other: `pip beautifulsoup4`
 77 | * [Argparse][6]: Required (but you only need to install with Python 2.6)
 78 |   - Ubuntu/Debian: `sudo apt-get install python-argparse`
 79 |   - Other: `pip argparse`
 80 | * [requests][16]: Required.
 81 |   - Ubuntu/Debian: `sudo apt-get install python-requests`
 82 |   - Mac OSX + MacPorts: `sudo port install py-requests`
 83 |   - Other: `pip requests`
 84 | * [six][19]: Required.
 85 |   - Ubuntu/Debian: `sudo apt-get install python-six`
 86 |   - Mac OSX + MacPorts: `sudo port install py27-six`
 87 |   - Other: `pip six`
 88 | * [html5lib][15]: Not required, but recommended for parsing pages.
 89 |   - Ubuntu/Debian: `sudo apt-get install python-html5lib`
 90 |   - Mac OSX + MacPorts: `sudo port install py-html5lib`
 91 |   - Other: `pip html5lib`
 92 | * [easy_install][7]: Only necessary if not using prepackaged
 93 |   dependencies. Also, `pip` supersedes it.
 94 |   - Ubuntu/Debian: `sudo apt-get install python-setuptools`
 95 | 
 96 | Again, make sure that you have the versions mentioned in the file
 97 | `requirements.txt` (later versions may be OK).
 98 | 
 99 | On Mac OSX using MacPorts, the following may be used:
100 | 
101 |     port
102 |     > select --set python python27
103 |     > install py-beautifulsoup
104 |     > install py-argparse
105 |     > install py-setuptools
106 |     > install py-requests
107 |     > install py27-six
108 | 
109 | ### Create an account with Coursera
110 | 
111 | If you don't already have one, create a [Coursera][1] account and enroll in
112 | a class. See https://www.coursera.org/courses for the list of classes.
113 | 
114 | ### Running the script
115 | 
116 | Run the script to download the materials by providing your Coursera account
117 | credentials (e.g. email address and password or a `~/.netrc` file), the class names,
118 | as well as any additional parameters:
119 | 
120 |     General:                     coursera-dl -u <user> -p <pass> modelthinking-004
121 |     Multiple classes:            coursera-dl -u <user> -p <pass> saas historyofrock1-001 algo-2012-002
122 |     Filter by section name:      coursera-dl -u <user> -p <pass> -sf "Chapter_Four" crypto-004
123 |     Filter by lecture name:      coursera-dl -u <user> -p <pass> -lf "3.1_" ml-2012-002
124 |     Download only ppt files:     coursera-dl -u <user> -p <pass> -f "ppt" qcomp-2012-001
125 |     Use a ~/.netrc file:         coursera-dl -n -- matrix-001
126 |     Get the preview classes:     coursera-dl -n -b ni-001
127 |     Specify download path:       coursera-dl -n --path=C:\Coursera\Classes\ comnetworks-002
128 |     Display help:                coursera-dl --help
129 |     
130 |     Maintain a list of classes in a dir:
131 |       Initialize:              mkdir -p CURRENT/{class1,class2,..classN}
132 |       Update:                  coursera-dl -n --path CURRENT `\ls CURRENT`
133 |     
134 |     Note: If your ls command is aliased to display a colorized output, you may experience problems. 
135 |     Be sure to escape the ls command (use \ls) to assure that no special characters get sent to the script.
136 | 
137 | On \*nix platforms, the use of a `~/.netrc` file is a good alternative to
138 | specifying both your username (i.e., your email address) and password every
139 | time on the command line. To use it, simply add a line like the one below to
140 | a file named `.netrc` in your home directory (or the [equivalent][8], if you
141 | are using Windows) with contents like:
142 | 
143 |     machine coursera-dl login <user> password <pass>
144 | 
145 | Create the file if it doesn't exist yet.  From then on, you can switch from
146 | using `-u` and `-p` to simply call `coursera-dl` with the option `-n`
147 | instead.  This is especially convenient, as typing usernames (email
148 | addresses) and passwords directly on the command line can get tiresome (even
149 | more if you happened to choose a "strong" password).
150 | 
151 | **NOTE**: If your password contains punctuation, quotes or other "funny
152 | characters" (e.g., `<`, `>`, `#`, `&`, `|` and so on), then you may have to
153 | escape them from your shell. With bash or other Bourne-shell clones (and
154 | probably with many other shells) one of the better ways to do so is to
155 | enclose your password in single quotes, so that you don't run into
156 | problems.  See issue #213 for more information.
157 | 
158 | ## Troubleshooting
159 | 
160 | If you have problems when downloading class materials, please try to see if
161 | one of the following actions solve your problem:
162 | 
163 | * Make sure the class name you are using corresponds to the resource name
164 |   used in the URL for that class:
165 |     `https://class.coursera.org/<CLASS_NAME>/class/index`
166 | 
167 | * Have you tried to clean the cached cookies/credentials with the
168 |   `--clear-cache` option?
169 | 
170 | * Note that many courses (most, perhaps?) may remove the materials after a
171 |   little while after the course is completed, while other courses may retain
172 |   the materials up to a next session/offering of the same course (to avoid
173 |   problems with academic dishonesty, apparently).
174 | 
175 |   In short, it is not guaranteed that you will be able to download after the
176 |   course is finished and this is, unfortunately, nothing that we can help
177 |   you with.
178 | 
179 | * Make sure you have installed and/or updated all of your dependencies
180 |   according to the `requirements.txt` file as described above.
181 |  
182 | * One can export a Netscape-style cookies file with a browser extension
183 |   ([1][9], [2][10]) and use it with the `-c` option. This comes in handy
184 |   when the authentication via password is not working (the authentication
185 |   process changes now and then).
186 | 
187 | * If results show 0 sections, you most likely have provided invalid
188 |   credentials (username and/or password in the command line or in your
189 |   `.netrc` file).
190 | 
191 | * For courses that have not started yet, but have had a previous iteration
192 |   sometimes a preview is available, containing all the classes from the
193 |   last course. These files can be downloaded by passing the -b parameter.
194 | 
195 | * If you are using Beautiful Soup 4, make sure you have installed
196 |   html5lib:
197 | 
198 |         $ python
199 |         >>> import html5lib
200 |         >>> print(html5lib.__version__)
201 |         1.0b2
202 | 
203 | * If you get an error like `Could not find class: <CLASS_NAME>`:
204 |     * Verify that the name of the course is correct. Current class
205 |       names in coursera are composed by a short course name e.g. `class`
206 |       and the current version of the course (a number). For example, for a
207 |       class named `class`, you would have to use `class-001`, `class-002`
208 |       etc.
209 |     * Second, verify that you are enrolled in the course. You won't be
210 |       able to access the course materials if you are not officially
211 |       enrolled and agreed to the honor course *via the website*.
212 | 
213 | * If:
214 |   - You get an error when using `-n` to specify that you want to use a
215 |     `.netrc` file and,
216 |   - You want the script to use your default netrc file and,
217 |   - You get a message saying `coursera-dl: error: too few arguments`
218 | 
219 |   Then you should specify `--` as an argument after `-n`, that is, `-n --`
220 |   or change the order in which you pass the arguments to the script, so that
221 |   the argument after `-n` begins with an hyphen (`-`).  Otherwise, Python's
222 |   `argparse` module will think that what you are passing is the name of the
223 |   netrc file that you want to use. See issue #162.
224 | 
225 | ## Filing an issue/Reporting a bug
226 | 
227 | When reporting bugs against `coursera-dl`, please don't forget to include
228 | enough information so that you can help us help you:
229 | 
230 | * Is the problem happening with the latest version of the script?
231 | * What operating system are you using?
232 | * Do you have all the recommended versions of the modules? See them in
233 |   the file `requirements.txt`.
234 | * What is the course that you are trying to access?
235 | * What is the precise command line that you are using (feel free to hide
236 |   your username and password with asterisks, but leave all other
237 |   information untouched).
238 | * What are the precise messages that you get? Please, copy and paste them.
239 |   Don't reword the messages.
240 | 
241 | ## Feedback
242 | 
243 | I enjoy getting feedback. Here are a few of the comments I've received:
244 | 
245 | * "Thanks for the good job! Knowledge will flood the World a little more thanks
246 |   to your script!"
247 |   <br>Guillaume V. 11/8/2012
248 |   
249 | * "Just wanted to send you props for your Python script to download Coursera
250 |   courses. I've been using it in Kenya for my non-profit to get online courses
251 |   to places where internet is really expensive and unreliable. Mostly kids here
252 |   can't afford high school, and downloading one of these classes by the usual
253 |   means would cost more than the average family earns in one week. Thanks!"
254 |   <br>Jay L., [Tunapanda][14] 3/20/2013
255 | 
256 | * "I am a big fan of Coursera and attend lots of different courses. Time
257 |   constraints don't allow me to attend all the courses I want at the same time.
258 |   I came across your script, and I am very happily using it!  Great stuff and
259 |   thanks for making this available on Github - well done!"
260 |   <br>William G.  2/18/2013
261 |   
262 | * "This script is awesome! I was painstakingly downloading each and every video
263 |   and ppt by hand -- looked into wget but ran into wildcard issues with HTML,
264 |   and then.. I came across your script.  Can't tell you how many hours you've
265 |   just saved me :) If you're ever in Paris / Stockholm, it is absolutely
266 |   mandatory that I buy you a beer :)"
267 |   <br>Razvan T. 11/26/2012
268 | 
269 | * "Thanks a lot! :)"
270 |   <br>Viktor V. 24/04/2013
271 | 
272 | ## Contact
273 | 
274 | Post bugs and issues on [github][11]. Send other comments to John Lehmann:
275 | first last at geemail dotcom or [@jplehmann][12]
276 | 
277 | [1]: https://www.coursera.org
278 | [2]: http://sourceforge.net/projects/gnuwin32/files/wget/1.11.4-1/wget-1.11.4-1-setup.exe
279 | [3]: https://rg3.github.com/youtube-dl
280 | [5]: http://www.crummy.com/software/BeautifulSoup
281 | [6]: http://pypi.python.org/pypi/argparse
282 | [7]: http://pypi.python.org/pypi/setuptools
283 | [8]: http://stackoverflow.com/a/6031266/962311
284 | [9]: https://chrome.google.com/webstore/detail/lopabhfecdfhgogdbojmaicoicjekelh
285 | [10]: https://addons.mozilla.org/en-US/firefox/addon/export-cookies
286 | [11]: https://github.com/coursera-dl/coursera/issues
287 | [12]: https://twitter.com/jplehmann
288 | [13]: http://techcrunch.com/2013/02/20/coursera-adds-29-schools-90-courses-and-4-new-languages-to-its-online-learning-platform
289 | [14]: http://www.tunapanda.org
290 | [15]: https://github.com/html5lib/html5lib-python
291 | [16]: http://docs.python-requests.org/en/latest/
292 | [17]: http://www.pip-installer.org/en/latest/
293 | [18]: http://python-distribute.org/pip_distribute.png
294 | [19]: https://pypi.python.org/pypi/six/
295 | 
296 | 
297 | [![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/coursera-dl/coursera/trend.png)](https://bitdeli.com/free "Bitdeli Badge")
298 | 
299 | 


--------------------------------------------------------------------------------
/coursera-dl:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf-8 -*-
3 | 
4 | from coursera import coursera_dl
5 | 
6 | coursera_dl.main()
7 | 


--------------------------------------------------------------------------------
/coursera-dl.bat:
--------------------------------------------------------------------------------
 1 | @echo off
 2 | rem Start-up script for Windows systems, ensuring out-of-the-box command execution as for *nix examples.
 3 | rem Make sure the containing folder is referenced in %PATH%
 4 | 
 5 | rem Usage: see "Running the script" section of the README for sample commands.
 6 | 
 7 | rem Run "coursera-dl" in the current folder as working directory (where "%~dp0" is the
 8 | rem location of this .bat file) via Python, passing all user-specified arguments ("%*")
 9 | python %~dp0\coursera-dl %*
10 | 


--------------------------------------------------------------------------------
/coursera/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jplehmann/coursera/c9ff3dee3cc3a97df3942e633144872e8ab7f3c2/coursera/__init__.py


--------------------------------------------------------------------------------
/coursera/cookies.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | 
  3 | """
  4 | Cookie handling module.
  5 | """
  6 | 
  7 | import logging
  8 | import os
  9 | 
 10 | import requests
 11 | import six
 12 | 
 13 | from six.moves import StringIO
 14 | from six.moves import http_cookiejar as cookielib
 15 | from .define import AUTH_URL, CLASS_URL, AUTH_REDIRECT_URL, PATH_COOKIES
 16 | from .utils import mkdir_p
 17 | 
 18 | 
 19 | # Monkey patch cookielib.Cookie.__init__.
 20 | # Reason: The expires value may be a decimal string,
 21 | # but the Cookie class uses int() ...
 22 | __orginal_init__ = cookielib.Cookie.__init__
 23 | 
 24 | 
 25 | def __fixed_init__(self, version, name, value,
 26 |                    port, port_specified,
 27 |                    domain, domain_specified, domain_initial_dot,
 28 |                    path, path_specified,
 29 |                    secure,
 30 |                    expires,
 31 |                    discard,
 32 |                    comment,
 33 |                    comment_url,
 34 |                    rest,
 35 |                    rfc2109=False,
 36 |                    ):
 37 |     if expires is not None:
 38 |         expires = float(expires)
 39 |     __orginal_init__(self, version, name, value,
 40 |                      port, port_specified,
 41 |                      domain, domain_specified, domain_initial_dot,
 42 |                      path, path_specified,
 43 |                      secure,
 44 |                      expires,
 45 |                      discard,
 46 |                      comment,
 47 |                      comment_url,
 48 |                      rest,
 49 |                      rfc2109=False,)
 50 | 
 51 | cookielib.Cookie.__init__ = __fixed_init__
 52 | 
 53 | 
 54 | class ClassNotFound(BaseException):
 55 |     """
 56 |     Raised if a course is not found in Coursera's site.
 57 |     """
 58 | 
 59 | 
 60 | class AuthenticationFailed(BaseException):
 61 |     """
 62 |     Raised if we cannot authenticate on Coursera's site.
 63 |     """
 64 | 
 65 | 
 66 | def login(session, class_name, username, password):
 67 |     """
 68 |     Login on accounts.coursera.org with the given credentials.
 69 |     This adds the following cookies to the session:
 70 |         sessionid, maestro_login, maestro_login_flag
 71 |     """
 72 | 
 73 |     try:
 74 |         session.cookies.clear('.coursera.org')
 75 |     except KeyError:
 76 |         pass
 77 | 
 78 |     # Hit class url to obtain csrf_token
 79 |     class_url = CLASS_URL.format(class_name=class_name)
 80 |     r = requests.get(class_url, allow_redirects=False)
 81 | 
 82 |     try:
 83 |         r.raise_for_status()
 84 |     except requests.exceptions.HTTPError as e:
 85 |         logging.error(e)
 86 |         raise ClassNotFound(class_name)
 87 | 
 88 |     csrftoken = r.cookies.get('csrf_token')
 89 | 
 90 |     if not csrftoken:
 91 |         raise AuthenticationFailed('Did not recieve csrf_token cookie.')
 92 | 
 93 |     # Now make a call to the authenticator url.
 94 |     headers = {
 95 |         'Cookie': 'csrftoken=' + csrftoken,
 96 |         'Referer': 'https://accounts.coursera.org/signin',
 97 |         'X-CSRFToken': csrftoken,
 98 |     }
 99 | 
100 |     data = {
101 |         'email': username,
102 |         'password': password
103 |     }
104 | 
105 |     r = session.post(AUTH_URL, data=data,
106 |                      headers=headers, allow_redirects=False)
107 |     try:
108 |         r.raise_for_status()
109 |     except requests.exceptions.HTTPError:
110 |         raise AuthenticationFailed('Cannot login on accounts.coursera.org.')
111 | 
112 |     logging.info('Logged in on accounts.coursera.org.')
113 | 
114 | 
115 | def down_the_wabbit_hole(session, class_name):
116 |     """
117 |     Authenticate on class.coursera.org
118 |     """
119 | 
120 |     auth_redirector_url = AUTH_REDIRECT_URL.format(class_name=class_name)
121 |     r = session.get(auth_redirector_url)
122 |     try:
123 |         r.raise_for_status()
124 |     except requests.exceptions.HTTPError:
125 |         raise AuthenticationFailed('Cannot login on class.coursera.org.')
126 | 
127 | 
128 | def _get_authentication_cookies(session, class_name,
129 |                                 username, password):
130 |     try:
131 |         session.cookies.clear('class.coursera.org', '/' + class_name)
132 |     except KeyError:
133 |         pass
134 | 
135 |     down_the_wabbit_hole(session, class_name)
136 | 
137 |     enough = do_we_have_enough_cookies(session.cookies, class_name)
138 | 
139 |     if not enough:
140 |         raise AuthenticationFailed('Did not find necessary cookies.')
141 | 
142 | 
143 | def get_authentication_cookies(session, class_name, username, password):
144 |     """
145 |     Get the necessary cookies to authenticate on class.coursera.org.
146 | 
147 |     To access the class pages we need two cookies on class.coursera.org:
148 |         csrf_token, session
149 |     """
150 | 
151 |     # First, check if we already have the .coursera.org cookies.
152 |     if session.cookies.get('CAUTH', domain=".coursera.org"):
153 |         logging.debug('Already logged in on accounts.coursera.org.')
154 |     else:
155 |         login(session, class_name, username, password)
156 | 
157 |     _get_authentication_cookies(
158 |         session, class_name, username, password)
159 | 
160 |     logging.info('Found authentication cookies.')
161 | 
162 | 
163 | def do_we_have_enough_cookies(cj, class_name):
164 |     """
165 |     Checks whether we have all the required cookies
166 |     to authenticate on class.coursera.org.
167 |     """
168 |     domain = 'class.coursera.org'
169 |     path = "/" + class_name
170 | 
171 |     return cj.get('csrf_token', domain=domain, path=path) is not None
172 | 
173 | 
174 | def validate_cookies(session, class_name):
175 |     """
176 |     Checks whether we have all the required cookies
177 |     to authenticate on class.coursera.org. Also check for and remove
178 |     stale session.
179 |     """
180 |     if not do_we_have_enough_cookies(session.cookies, class_name):
181 |         return False
182 | 
183 |     url = CLASS_URL.format(class_name=class_name) + '/class'
184 |     r = session.head(url, allow_redirects=False)
185 | 
186 |     if r.status_code == 200:
187 |         return True
188 |     else:
189 |         logging.debug('Stale session.')
190 |         try:
191 |             session.cookies.clear('.coursera.org')
192 |         except KeyError:
193 |             pass
194 |         return False
195 | 
196 | 
197 | def make_cookie_values(cj, class_name):
198 |     """
199 |     Makes a string of cookie keys and values.
200 |     Can be used to set a Cookie header.
201 |     """
202 |     path = "/" + class_name
203 | 
204 |     cookies = [c.name + '=' + c.value
205 |                for c in cj
206 |                if c.domain == "class.coursera.org"
207 |                and c.path == path]
208 | 
209 |     return '; '.join(cookies)
210 | 
211 | 
212 | def find_cookies_for_class(cookies_file, class_name):
213 |     """
214 |     Return a RequestsCookieJar containing the cookies for
215 |     .coursera.org and class.coursera.org found in the given cookies_file.
216 |     """
217 | 
218 |     path = "/" + class_name
219 | 
220 |     def cookies_filter(c):
221 |         return c.domain == ".coursera.org" \
222 |             or (c.domain == "class.coursera.org" and c.path == path)
223 | 
224 |     cj = get_cookie_jar(cookies_file)
225 | 
226 |     new_cj = requests.cookies.RequestsCookieJar()
227 |     for c in filter(cookies_filter, cj):
228 |         new_cj.set_cookie(c)
229 | 
230 |     return new_cj
231 | 
232 | 
233 | def load_cookies_file(cookies_file):
234 |     """
235 |     Loads the cookies file.
236 | 
237 |     We pre-pend the file with the special Netscape header because the cookie
238 |     loader is very particular about this string.
239 |     """
240 | 
241 |     cookies = StringIO()
242 |     cookies.write('# Netscape HTTP Cookie File')
243 |     cookies.write(open(cookies_file, 'rU').read())
244 |     cookies.flush()
245 |     cookies.seek(0)
246 |     return cookies
247 | 
248 | 
249 | def get_cookie_jar(cookies_file):
250 |     cj = cookielib.MozillaCookieJar()
251 |     cookies = load_cookies_file(cookies_file)
252 | 
253 |     # nasty hack: cj.load() requires a filename not a file, but if I use
254 |     # stringio, that file doesn't exist. I used NamedTemporaryFile before,
255 |     # but encountered problems on Windows.
256 |     cj._really_load(cookies, 'StringIO.cookies', False, False)
257 | 
258 |     return cj
259 | 
260 | 
261 | def get_cookies_cache_path(username):
262 |     return os.path.join(PATH_COOKIES, username + '.txt')
263 | 
264 | 
265 | def get_cookies_from_cache(username):
266 |     """
267 |     Returns a RequestsCookieJar containing the cached cookies for the given
268 |     user.
269 |     """
270 |     path = get_cookies_cache_path(username)
271 |     cj = requests.cookies.RequestsCookieJar()
272 |     try:
273 |         cached_cj = get_cookie_jar(path)
274 |         for cookie in cached_cj:
275 |             cj.set_cookie(cookie)
276 |         logging.debug(
277 |             'Loaded cookies from %s', get_cookies_cache_path(username))
278 |     except IOError:
279 |         pass
280 | 
281 |     return cj
282 | 
283 | 
284 | def write_cookies_to_cache(cj, username):
285 |     """
286 |     Saves the RequestsCookieJar to disk in the Mozilla cookies.txt file
287 |     format.  This prevents us from repeated authentications on the
288 |     accounts.coursera.org and class.coursera.org/class_name sites.
289 |     """
290 |     mkdir_p(PATH_COOKIES, 0o700)
291 |     path = get_cookies_cache_path(username)
292 |     cached_cj = cookielib.MozillaCookieJar()
293 |     for cookie in cj:
294 |         cached_cj.set_cookie(cookie)
295 |     cached_cj.save(path)
296 | 
297 | 
298 | def get_cookies_for_class(session, class_name,
299 |                           cookies_file=None,
300 |                           username=None,
301 |                           password=None):
302 |     """
303 |     Get the cookies for the given class.
304 |     We do not validate the cookies if they are loaded from a cookies file
305 |     because this is intented for debugging purposes or if the coursera
306 |     authentication process has changed.
307 |     """
308 |     if cookies_file:
309 |         cookies = find_cookies_for_class(cookies_file, class_name)
310 |         session.cookies.update(cookies)
311 |         logging.info('Loaded cookies from %s', cookies_file)
312 |     else:
313 |         cookies = get_cookies_from_cache(username)
314 |         session.cookies.update(cookies)
315 |         if validate_cookies(session, class_name):
316 |             logging.info('Already authenticated.')
317 |         else:
318 |             get_authentication_cookies(session, class_name, username, password)
319 |             write_cookies_to_cache(session.cookies, username)
320 | 


--------------------------------------------------------------------------------
/coursera/coursera_dl.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # -*- coding: utf-8 -*-
  3 | 
  4 | """
  5 | For downloading lecture resources such as videos for Coursera classes. Given
  6 | a class name, username and password, it scrapes the course listing page to
  7 | get the section (week) and lecture names, and then downloads the related
  8 | materials into appropriately named files and directories.
  9 | 
 10 | Examples:
 11 |   coursera-dl -u <user> -p <passwd> saas
 12 |   coursera-dl -u <user> -p <passwd> -l listing.html -o saas --skip-download
 13 | 
 14 | For further documentation and examples, visit the project's home at:
 15 |   https://github.com/coursera-dl/coursera
 16 | 
 17 | Authors and copyright:
 18 |     © 2012-2013, John Lehmann (first last at geemail dotcom or @jplehmann)
 19 |     © 2012-2013, Rogério Brito (r lastname at ime usp br)
 20 |     © 2013, Jonas De Taeye (first dt at fastmail fm)
 21 | 
 22 | Contributions are welcome, but please add new unit tests to test your changes
 23 | and/or features.  Also, please try to make changes platform independent and
 24 | backward compatible.
 25 | 
 26 | Legalese:
 27 | 
 28 |  This program is free software: you can redistribute it and/or modify it
 29 |  under the terms of the GNU Lesser General Public License as published by
 30 |  the Free Software Foundation, either version 3 of the License, or (at your
 31 |  option) any later version.
 32 | 
 33 |  This program is distributed in the hope that it will be useful,
 34 |  but WITHOUT ANY WARRANTY; without even the implied warranty of
 35 |  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 36 |  GNU General Public License for more details.
 37 | 
 38 |  You should have received a copy of the GNU General Public License
 39 |  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 40 | """
 41 | 
 42 | import argparse
 43 | import datetime
 44 | import json
 45 | import logging
 46 | import os
 47 | import re
 48 | import shutil
 49 | import subprocess
 50 | import sys
 51 | import time
 52 | import glob
 53 | 
 54 | from distutils.version import LooseVersion as V
 55 | 
 56 | import requests
 57 | from six import iteritems
 58 | 
 59 | try:
 60 |     from BeautifulSoup import BeautifulSoup
 61 | except ImportError:
 62 |     from bs4 import BeautifulSoup as BeautifulSoup_
 63 |     try:
 64 |         # Use html5lib for parsing if available
 65 |         import html5lib
 66 |         BeautifulSoup = lambda page: BeautifulSoup_(page, 'html5lib')
 67 |     except ImportError:
 68 |         BeautifulSoup = lambda page: BeautifulSoup_(page, 'html.parser')
 69 | 
 70 | 
 71 | from .cookies import (
 72 |     AuthenticationFailed, ClassNotFound,
 73 |     get_cookies_for_class, make_cookie_values)
 74 | from .credentials import get_credentials, CredentialsError
 75 | from .define import CLASS_URL, ABOUT_URL, PATH_CACHE
 76 | from .downloaders import get_downloader
 77 | from .utils import clean_filename, get_anchor_format, mkdir_p, fix_url
 78 | 
 79 | # URL containing information about outdated modules
 80 | _see_url = " See https://github.com/coursera-dl/coursera/issues/139"
 81 | 
 82 | # Test versions of some critical modules.
 83 | # We may, perhaps, want to move these elsewhere.
 84 | import bs4
 85 | import six
 86 | 
 87 | assert V(requests.__version__) >= V('1.2'), "Upgrade requests!" + _see_url
 88 | assert V(six.__version__) >= V('1.3'), "Upgrade six!" + _see_url
 89 | assert V(bs4.__version__) >= V('4.1'), "Upgrade bs4!" + _see_url
 90 | 
 91 | 
 92 | def get_syllabus_url(class_name, preview):
 93 |     """
 94 |     Return the Coursera index/syllabus URL, depending on if we want to only
 95 |     preview or if we are enrolled in the course.
 96 |     """
 97 |     class_type = 'preview' if preview else 'index'
 98 |     page = CLASS_URL.format(class_name=class_name) + '/lecture/' + class_type
 99 |     logging.debug('Using %s mode with page: %s', class_type, page)
100 | 
101 |     return page
102 | 
103 | 
104 | def get_page(session, url):
105 |     """
106 |     Download an HTML page using the requests session.
107 |     """
108 | 
109 |     r = session.get(url)
110 | 
111 |     try:
112 |         r.raise_for_status()
113 |     except requests.exceptions.HTTPError as e:
114 |         logging.error("Error %s getting page %s", e, url)
115 |         raise
116 | 
117 |     return r.text
118 | 
119 | 
120 | def grab_hidden_video_url(session, href):
121 |     """
122 |     Follow some extra redirects to grab hidden video URLs (like those from
123 |     University of Washington).
124 |     """
125 |     try:
126 |         page = get_page(session, href)
127 |     except requests.exceptions.HTTPError:
128 |         return None
129 | 
130 |     soup = BeautifulSoup(page)
131 |     l = soup.find('source', attrs={'type': 'video/mp4'})
132 |     if l is not None:
133 |         return l['src']
134 |     else:
135 |         return None
136 | 
137 | 
138 | def get_syllabus(session, class_name, local_page=False, preview=False):
139 |     """
140 |     Get the course listing webpage.
141 | 
142 |     If we are instructed to use a local page and it already exists, then
143 |     that page is used instead of performing a download.  If we are
144 |     instructed to use a local page and it does not exist, then we download
145 |     the page and save a copy of it for future use.
146 |     """
147 | 
148 |     if not (local_page and os.path.exists(local_page)):
149 |         url = get_syllabus_url(class_name, preview)
150 |         page = get_page(session, url)
151 |         logging.info('Downloaded %s (%d bytes)', url, len(page))
152 | 
153 |         # cache the page if we're in 'local' mode
154 |         if local_page:
155 |             with open(local_page, 'w') as f:
156 |                 f.write(page.encode("utf-8"))
157 |     else:
158 |         with open(local_page) as f:
159 |             page = f.read().decode("utf-8")
160 |         logging.info('Read (%d bytes) from local file', len(page))
161 | 
162 |     return page
163 | 
164 | 
165 | def transform_preview_url(a):
166 |     """
167 |     Given a preview lecture URL, transform it into a regular video URL.
168 | 
169 |     If the given URL is not a preview URL, we simply return None.
170 |     """
171 | 
172 |     # Example URLs
173 |     # "https://class.coursera.org/modelthinking/lecture/preview_view/8"
174 |     # "https://class.coursera.org/nlp/lecture/preview_view?lecture_id=124"
175 |     mobj = re.search(r'preview_view/(\d+)$', a)
176 |     if mobj:
177 |         return re.sub(r'preview_view/(\d+)$', r'preview_view?lecture_id=\1', a)
178 |     else:
179 |         return None
180 | 
181 | 
182 | def get_video(session, url):
183 |     """
184 |     Parses a Coursera video page
185 |     """
186 | 
187 |     page = get_page(session, url)
188 |     soup = BeautifulSoup(page)
189 |     return soup.find(attrs={'type': re.compile('^video/mp4')})['src']
190 | 
191 | 
192 | def parse_syllabus(session, page, reverse=False, intact_fnames=False):
193 |     """
194 |     Parses a Coursera course listing/syllabus page.  Each section is a week
195 |     of classes.
196 |     """
197 | 
198 |     sections = []
199 |     soup = BeautifulSoup(page)
200 | 
201 |     # traverse sections
202 |     for stag in soup.findAll(attrs={'class':
203 |                                     re.compile('^course-item-list-header')}):
204 |         assert stag.contents[0] is not None, "couldn't find section"
205 |         untouched_fname = stag.contents[0].contents[1]
206 |         section_name = clean_filename(untouched_fname, intact_fnames)
207 |         logging.info(section_name)
208 |         lectures = []  # resources for 1 lecture
209 | 
210 |         # traverse resources (e.g., video, ppt, ..)
211 |         for vtag in stag.nextSibling.findAll('li'):
212 |             assert vtag.a.contents[0], "couldn't get lecture name"
213 |             untouched_fname = vtag.a.contents[0]
214 |             vname = clean_filename(untouched_fname, intact_fnames)
215 |             logging.info('  %s', vname)
216 |             lecture = {}
217 |             lecture_page = None
218 | 
219 |             for a in vtag.findAll('a'):
220 |                 href = fix_url(a['href'])
221 |                 untouched_fname = a.get('title', '')
222 |                 title = clean_filename(untouched_fname, intact_fnames)
223 |                 fmt = get_anchor_format(href)
224 |                 logging.debug('    %s %s', fmt, href)
225 |                 if fmt:
226 |                     lecture[fmt] = lecture.get(fmt, [])
227 |                     lecture[fmt].append((href, title))
228 |                     continue
229 | 
230 |                 # Special case: find preview URLs
231 |                 lecture_page = transform_preview_url(href)
232 |                 if lecture_page:
233 |                     try:
234 |                         href = get_video(session, lecture_page)
235 |                         lecture['mp4'] = lecture.get('mp4', [])
236 |                         lecture['mp4'].append((fix_url(href), ''))
237 |                     except TypeError:
238 |                         logging.warn(
239 |                             'Could not get resource: %s', lecture_page)
240 | 
241 |             # Special case: we possibly have hidden video links---thanks to
242 |             # the University of Washington for that.
243 |             if 'mp4' not in lecture:
244 |                 for a in vtag.findAll('a'):
245 |                     if a.get('data-modal-iframe'):
246 |                         href = grab_hidden_video_url(
247 |                             session, a['data-modal-iframe'])
248 |                         href = fix_url(href)
249 |                         fmt = 'mp4'
250 |                         logging.debug('    %s %s', fmt, href)
251 |                         if href is not None:
252 |                             lecture[fmt] = lecture.get(fmt, [])
253 |                             lecture[fmt].append((href, ''))
254 | 
255 |             for fmt in lecture:
256 |                 count = len(lecture[fmt])
257 |                 for i, r in enumerate(lecture[fmt]):
258 |                     if (count == i + 1):
259 |                         # for backward compatibility, we do not add the title
260 |                         # to the filename (format_combine_number_resource and
261 |                         # format_resource)
262 |                         lecture[fmt][i] = (r[0], '')
263 |                     else:
264 |                         # make sure the title is unique
265 |                         lecture[fmt][i] = (r[0], '{0:d}_{1}'.format(i, r[1]))
266 | 
267 |             lectures.append((vname, lecture))
268 | 
269 |         sections.append((section_name, lectures))
270 | 
271 |     logging.info('Found %d sections and %d lectures on this page',
272 |                  len(sections), sum(len(s[1]) for s in sections))
273 | 
274 |     if sections and reverse:
275 |         sections.reverse()
276 | 
277 |     if not len(sections):
278 |         logging.error('The cookies file may be invalid, '
279 |                       'please re-run with the `--clear-cache` option.')
280 | 
281 |     return sections
282 | 
283 | 
284 | def download_about(session, class_name, path='', overwrite=False):
285 |     """
286 |     Download the 'about' metadata which is in JSON format and pretty-print it.
287 |     """
288 |     about_fn = os.path.join(path, class_name + '-about.json')
289 |     logging.debug('About file to be written to: %s', about_fn)
290 |     if os.path.exists(about_fn) and not overwrite:
291 |         return
292 | 
293 |     # strip off course number on end e.g. ml-001 -> ml
294 |     base_class_name = class_name.split('-')[0]
295 | 
296 |     about_url = ABOUT_URL.format(class_name=base_class_name)
297 |     logging.debug('About url: %s', about_url)
298 | 
299 |     # NOTE: should we create a directory with metadata?
300 |     logging.info('Downloading about page from: %s', about_url)
301 |     about_json = get_page(session, about_url)
302 |     data = json.loads(about_json)
303 | 
304 |     with open(about_fn, 'w') as about_file:
305 |         json_data = json.dumps(data, indent=4, separators=(',', ':'))
306 |         about_file.write(json_data)
307 | 
308 | 
309 | def download_lectures(downloader,
310 |                       class_name,
311 |                       sections,
312 |                       file_formats,
313 |                       overwrite=False,
314 |                       skip_download=False,
315 |                       section_filter=None,
316 |                       lecture_filter=None,
317 |                       resource_filter=None,
318 |                       path='',
319 |                       verbose_dirs=False,
320 |                       preview=False,
321 |                       combined_section_lectures_nums=False,
322 |                       hooks=None,
323 |                       playlist=False,
324 |                       intact_fnames=False
325 |                       ):
326 |     """
327 |     Downloads lecture resources described by sections.
328 |     Returns True if the class appears completed.
329 |     """
330 |     last_update = -1
331 | 
332 |     def format_section(num, section):
333 |         sec = '%02d_%s' % (num, section)
334 |         if verbose_dirs:
335 |             sec = class_name.upper() + '_' + sec
336 |         return sec
337 | 
338 |     def format_resource(num, name, title, fmt):
339 |         if title:
340 |             title = '_' + title
341 |         return '%02d_%s%s.%s' % (num, name, title, fmt)
342 | 
343 |     def format_combine_number_resource(secnum, lecnum, lecname, title, fmt):
344 |         if title:
345 |             title = '_' + title
346 |         return '%02d_%02d_%s%s.%s' % (secnum, lecnum, lecname, title, fmt)
347 | 
348 |     for (secnum, (section, lectures)) in enumerate(sections):
349 |         if section_filter and not re.search(section_filter, section):
350 |             logging.debug('Skipping b/c of sf: %s %s', section_filter,
351 |                           section)
352 |             continue
353 |         sec = os.path.join(path, class_name, format_section(secnum + 1,
354 |                                                             section))
355 |         for (lecnum, (lecname, lecture)) in enumerate(lectures):
356 |             if lecture_filter and not re.search(lecture_filter,
357 |                                                 lecname):
358 |                 logging.debug('Skipping b/c of lf: %s %s', lecture_filter,
359 |                               lecname)
360 |                 continue
361 | 
362 |             if not os.path.exists(sec):
363 |                 mkdir_p(sec)
364 | 
365 |             # Select formats to download
366 |             resources_to_get = []
367 |             for fmt, resources in iteritems(lecture):
368 |                 if fmt in file_formats or 'all' in file_formats:
369 |                     for r in resources:
370 |                         if resource_filter and r[1] and not re.search(resource_filter, r[1]):
371 |                             logging.debug('Skipping b/c of rf: %s %s',
372 |                                           resource_filter, r[1])
373 |                             continue
374 |                         resources_to_get.append((fmt, r[0], r[1]))
375 |                 else:
376 |                     logging.debug(
377 |                         'Skipping b/c format %s not in %s', fmt, file_formats)
378 | 
379 |             # write lecture resources
380 |             for fmt, url, title in resources_to_get:
381 |                 if combined_section_lectures_nums:
382 |                     lecfn = os.path.join(
383 |                         sec,
384 |                         format_combine_number_resource(
385 |                             secnum + 1, lecnum + 1, lecname, title, fmt))
386 |                 else:
387 |                     lecfn = os.path.join(
388 |                         sec, format_resource(lecnum + 1, lecname, title, fmt))
389 | 
390 |                 if overwrite or not os.path.exists(lecfn):
391 |                     if not skip_download:
392 |                         logging.info('Downloading: %s', lecfn)
393 |                         downloader.download(url, lecfn)
394 |                     else:
395 |                         open(lecfn, 'w').close()  # touch
396 |                     last_update = time.time()
397 |                 else:
398 |                     logging.info('%s already downloaded', lecfn)
399 |                     # if this file hasn't been modified in a long time,
400 |                     # record that time
401 |                     last_update = max(last_update, os.path.getmtime(lecfn))
402 | 
403 |         # After fetching resources, create a playlist in M3U format with the
404 |         # videos downloaded.
405 |         if playlist:
406 |             path_to_return = os.getcwd()
407 | 
408 |             for (_path, subdirs, files) in os.walk(sec):
409 |                 os.chdir(_path)
410 |                 globbed_videos = glob.glob("*.mp4")
411 |                 m3u_name = os.path.split(_path)[1] + ".m3u"
412 | 
413 |                 if len(globbed_videos):
414 |                     with open(m3u_name, "w") as m3u:
415 |                         for video in globbed_videos:
416 |                             m3u.write(video + "\n")
417 |                     os.chdir(path_to_return)
418 | 
419 |         if hooks:
420 |             for hook in hooks:
421 |                 logging.info('Running hook %s for section %s.', hook, sec)
422 |                 os.chdir(sec)
423 |                 subprocess.call(hook)
424 | 
425 |     # if we haven't updated any files in 1 month, we're probably
426 |     # done with this course
427 |     if last_update >= 0:
428 |         delta = time.time() - last_update
429 |         max_delta = total_seconds(datetime.timedelta(days=30))
430 |         if delta > max_delta:
431 |             logging.info('COURSE PROBABLY COMPLETE: ' + class_name)
432 |             return True
433 |     return False
434 | 
435 | 
436 | def total_seconds(td):
437 |     """
438 |     Compute total seconds for a timedelta.
439 | 
440 |     Added for backward compatibility, pre 2.7.
441 |     """
442 |     return (td.microseconds +
443 |            (td.seconds + td.days * 24 * 3600) * 10**6) // 10**6
444 | 
445 | 
446 | def parseArgs():
447 |     """
448 |     Parse the arguments/options passed to the program on the command line.
449 |     """
450 | 
451 |     parser = argparse.ArgumentParser(
452 |         description='Download Coursera.org lecture material and resources.')
453 | 
454 |     # positional
455 |     parser.add_argument('class_names',
456 |                         action='store',
457 |                         nargs='+',
458 |                         help='name(s) of the class(es) (e.g. "nlp")')
459 | 
460 |     parser.add_argument('-c',
461 |                         '--cookies_file',
462 |                         dest='cookies_file',
463 |                         action='store',
464 |                         default=None,
465 |                         help='full path to the cookies.txt file')
466 |     parser.add_argument('-u',
467 |                         '--username',
468 |                         dest='username',
469 |                         action='store',
470 |                         default=None,
471 |                         help='coursera username')
472 |     parser.add_argument('-n',
473 |                         '--netrc',
474 |                         dest='netrc',
475 |                         nargs='?',
476 |                         action='store',
477 |                         const=True,
478 |                         default=False,
479 |                         help='use netrc for reading passwords, uses default'
480 |                              ' location if no path specified')
481 | 
482 |     parser.add_argument('-p',
483 |                         '--password',
484 |                         dest='password',
485 |                         action='store',
486 |                         default=None,
487 |                         help='coursera password')
488 | 
489 |     # optional
490 |     parser.add_argument('--about',
491 |                         dest='about',
492 |                         action='store_true',
493 |                         default=False,
494 |                         help='download "about" metadata. (Default: False)')
495 |     parser.add_argument('-b',
496 |                         '--preview',
497 |                         dest='preview',
498 |                         action='store_true',
499 |                         default=False,
500 |                         help='get preview videos. (Default: False)')
501 |     parser.add_argument('-f',
502 |                         '--formats',
503 |                         dest='file_formats',
504 |                         action='store',
505 |                         default='all',
506 |                         help='file format extensions to be downloaded in'
507 |                              ' quotes space separated, e.g. "mp4 pdf" '
508 |                              '(default: special value "all")')
509 |     parser.add_argument('-sf',
510 |                         '--section_filter',
511 |                         dest='section_filter',
512 |                         action='store',
513 |                         default=None,
514 |                         help='only download sections which contain this'
515 |                              ' regex (default: disabled)')
516 |     parser.add_argument('-lf',
517 |                         '--lecture_filter',
518 |                         dest='lecture_filter',
519 |                         action='store',
520 |                         default=None,
521 |                         help='only download lectures which contain this regex'
522 |                              ' (default: disabled)')
523 |     parser.add_argument('-rf',
524 |                         '--resource_filter',
525 |                         dest='resource_filter',
526 |                         action='store',
527 |                         default=None,
528 |                         help='only download resources which match this regex'
529 |                              ' (default: disabled)')
530 |     parser.add_argument('--wget',
531 |                         dest='wget',
532 |                         action='store',
533 |                         nargs='?',
534 |                         const='wget',
535 |                         default=None,
536 |                         help='use wget for downloading,'
537 |                              'optionally specify wget bin')
538 |     parser.add_argument('--curl',
539 |                         dest='curl',
540 |                         action='store',
541 |                         nargs='?',
542 |                         const='curl',
543 |                         default=None,
544 |                         help='use curl for downloading,'
545 |                              ' optionally specify curl bin')
546 |     parser.add_argument('--aria2',
547 |                         dest='aria2',
548 |                         action='store',
549 |                         nargs='?',
550 |                         const='aria2c',
551 |                         default=None,
552 |                         help='use aria2 for downloading,'
553 |                              ' optionally specify aria2 bin')
554 |     parser.add_argument('--axel',
555 |                         dest='axel',
556 |                         action='store',
557 |                         nargs='?',
558 |                         const='axel',
559 |                         default=None,
560 |                         help='use axel for downloading,'
561 |                              ' optionally specify axel bin')
562 |     # We keep the wget_bin, ... options for backwards compatibility.
563 |     parser.add_argument('-w',
564 |                         '--wget_bin',
565 |                         dest='wget_bin',
566 |                         action='store',
567 |                         default=None,
568 |                         help='DEPRECATED, use --wget')
569 |     parser.add_argument('--curl_bin',
570 |                         dest='curl_bin',
571 |                         action='store',
572 |                         default=None,
573 |                         help='DEPRECATED, use --curl')
574 |     parser.add_argument('--aria2_bin',
575 |                         dest='aria2_bin',
576 |                         action='store',
577 |                         default=None,
578 |                         help='DEPRECATED, use --aria2')
579 |     parser.add_argument('--axel_bin',
580 |                         dest='axel_bin',
581 |                         action='store',
582 |                         default=None,
583 |                         help='DEPRECATED, use --axel')
584 |     parser.add_argument('-o',
585 |                         '--overwrite',
586 |                         dest='overwrite',
587 |                         action='store_true',
588 |                         default=False,
589 |                         help='whether existing files should be overwritten'
590 |                              ' (default: False)')
591 |     parser.add_argument('-l',
592 |                         '--process_local_page',
593 |                         dest='local_page',
594 |                         help='uses or creates local cached version of syllabus'
595 |                              ' page')
596 |     parser.add_argument('--skip-download',
597 |                         dest='skip_download',
598 |                         action='store_true',
599 |                         default=False,
600 |                         help='for debugging: skip actual downloading of files')
601 |     parser.add_argument('--path',
602 |                         dest='path',
603 |                         action='store',
604 |                         default='',
605 |                         help='path to save the file')
606 |     parser.add_argument('--verbose-dirs',
607 |                         dest='verbose_dirs',
608 |                         action='store_true',
609 |                         default=False,
610 |                         help='include class name in section directory name')
611 |     parser.add_argument('--debug',
612 |                         dest='debug',
613 |                         action='store_true',
614 |                         default=False,
615 |                         help='print lots of debug information')
616 |     parser.add_argument('--quiet',
617 |                         dest='quiet',
618 |                         action='store_true',
619 |                         default=False,
620 |                         help='omit as many messages as possible'
621 |                              ' (only printing errors)')
622 |     parser.add_argument('--add-class',
623 |                         dest='add_class',
624 |                         action='append',
625 |                         default=[],
626 |                         help='additional classes to get')
627 |     parser.add_argument('-r',
628 |                         '--reverse',
629 |                         dest='reverse',
630 |                         action='store_true',
631 |                         default=False,
632 |                         help='download sections in reverse order')
633 |     parser.add_argument('--combined-section-lectures-nums',
634 |                         dest='combined_section_lectures_nums',
635 |                         action='store_true',
636 |                         default=False,
637 |                         help='include lecture and section name in final files')
638 |     parser.add_argument('--hook',
639 |                         dest='hooks',
640 |                         action='append',
641 |                         default=[],
642 |                         help='hooks to run when finished')
643 |     parser.add_argument('-pl',
644 |                         '--playlist',
645 |                         dest='playlist',
646 |                         action='store_true',
647 |                         default=False,
648 |                         help='generate M3U playlists for course weeks')
649 |     parser.add_argument('--clear-cache',
650 |                         dest='clear_cache',
651 |                         action='store_true',
652 |                         default=False,
653 |                         help='clear cached cookies')
654 |     parser.add_argument('--unrestricted-filenames',
655 |                         dest='intact_fnames',
656 |                         action='store_true',
657 |                         default=False,
658 |                         help='Do not limit filenames to be ASCII-only')
659 | 
660 |     args = parser.parse_args()
661 | 
662 |     # Initialize the logging system first so that other functions
663 |     # can use it right away
664 |     if args.debug:
665 |         logging.basicConfig(level=logging.DEBUG,
666 |                             format='%(name)s[%(funcName)s] %(message)s')
667 |     elif args.quiet:
668 |         logging.basicConfig(level=logging.ERROR,
669 |                             format='%(name)s: %(message)s')
670 |     else:
671 |         logging.basicConfig(level=logging.INFO,
672 |                             format='%(message)s')
673 | 
674 |     # turn list of strings into list
675 |     args.file_formats = args.file_formats.split()
676 | 
677 |     for bin in ['wget_bin', 'curl_bin', 'aria2_bin', 'axel_bin']:
678 |         if getattr(args, bin):
679 |             logging.error('The --%s option is deprecated, please use --%s',
680 |                           bin, bin[:-4])
681 |             sys.exit(1)
682 | 
683 |     # check arguments
684 |     if args.cookies_file and not os.path.exists(args.cookies_file):
685 |         logging.error('Cookies file not found: %s', args.cookies_file)
686 |         sys.exit(1)
687 | 
688 |     if not args.cookies_file:
689 |         try:
690 |             args.username, args.password = get_credentials(
691 |                 username=args.username, password=args.password,
692 |                 netrc=args.netrc)
693 |         except CredentialsError as e:
694 |             logging.error(e)
695 |             sys.exit(1)
696 | 
697 |     return args
698 | 
699 | 
700 | def download_class(args, class_name):
701 |     """
702 |     Download all requested resources from the class given in class_name.
703 |     Returns True if the class appears completed.
704 |     """
705 | 
706 |     session = requests.Session()
707 | 
708 |     if args.preview:
709 |         # Todo, remove this.
710 |         session.cookie_values = 'dummy=dummy'
711 |     else:
712 |         get_cookies_for_class(
713 |             session,
714 |             class_name,
715 |             cookies_file=args.cookies_file,
716 |             username=args.username, password=args.password
717 |         )
718 |         session.cookie_values = make_cookie_values(session.cookies, class_name)
719 | 
720 |     # get the syllabus listing
721 |     page = get_syllabus(session, class_name, args.local_page, args.preview)
722 | 
723 |     # parse it
724 |     sections = parse_syllabus(session, page, args.reverse,
725 |                               args.intact_fnames)
726 | 
727 |     if args.about:
728 |         download_about(session, class_name, args.path, args.overwrite)
729 | 
730 |     downloader = get_downloader(session, class_name, args)
731 | 
732 |     # obtain the resources
733 |     completed = download_lectures(
734 |         downloader,
735 |         class_name,
736 |         sections,
737 |         args.file_formats,
738 |         args.overwrite,
739 |         args.skip_download,
740 |         args.section_filter,
741 |         args.lecture_filter,
742 |         args.resource_filter,
743 |         args.path,
744 |         args.verbose_dirs,
745 |         args.preview,
746 |         args.combined_section_lectures_nums,
747 |         args.hooks,
748 |         args.playlist,
749 |         args.intact_fnames)
750 | 
751 |     return completed
752 | 
753 | 
754 | def main():
755 |     """
756 |     Main entry point for execution as a program (instead of as a module).
757 |     """
758 | 
759 |     args = parseArgs()
760 |     completed_classes = []
761 | 
762 |     mkdir_p(PATH_CACHE, 0o700)
763 |     if args.clear_cache:
764 |         shutil.rmtree(PATH_CACHE)
765 | 
766 |     for class_name in args.class_names:
767 |         try:
768 |             logging.info('Downloading class: %s', class_name)
769 |             if download_class(args, class_name):
770 |                 completed_classes.append(class_name)
771 |         except requests.exceptions.HTTPError as e:
772 |             logging.error('HTTPError %s', e)
773 |         except ClassNotFound as cnf:
774 |             logging.error('Could not find class: %s', cnf)
775 |         except AuthenticationFailed as af:
776 |             logging.error('Could not authenticate: %s', af)
777 | 
778 |     if completed_classes:
779 |         logging.info(
780 |             "Classes which appear completed: " + " ".join(completed_classes))
781 | 
782 | 
783 | if __name__ == '__main__':
784 |     main()
785 | 


--------------------------------------------------------------------------------
/coursera/credentials.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | 
  3 | """
  4 | Manages the credential information (netrc, passwords, etc).
  5 | """
  6 | 
  7 | import getpass
  8 | import logging
  9 | import netrc
 10 | import os
 11 | import platform
 12 | 
 13 | 
 14 | class CredentialsError(BaseException):
 15 |     """
 16 |     Class to be thrown if the credentials are not found.
 17 |     """
 18 | 
 19 |     pass
 20 | 
 21 | 
 22 | def _getenv_or_empty(s):
 23 |     """
 24 |     Helper function that converts None gotten from the environment to the
 25 |     empty string.
 26 |     """
 27 |     return os.getenv(s) or ""
 28 | 
 29 | 
 30 | def get_config_paths(config_name): # pragma: no test
 31 |     """
 32 |     Returns a list of config files paths to try in order, given config file
 33 |     name and possibly a user-specified path.
 34 | 
 35 |     For Windows platforms, there are several paths that can be tried to
 36 |     retrieve the netrc file. There is, however, no "standard way" of doing
 37 |     things.
 38 | 
 39 |     A brief recap of the situation (all file paths are written in Unix
 40 |     convention):
 41 | 
 42 |     1. By default, Windows does not define a $HOME path. However, some
 43 |     people might define one manually, and many command-line tools imported
 44 |     from Unix will search the $HOME environment variable first. This
 45 |     includes MSYSGit tools (bash, ssh, ...) and Emacs.
 46 | 
 47 |     2. Windows defines two 'user paths': $USERPROFILE, and the
 48 |     concatenation of the two variables $HOMEDRIVE and $HOMEPATH. Both of
 49 |     these paths point by default to the same location, e.g.
 50 |     C:\\Users\\Username
 51 | 
 52 |     3. $USERPROFILE cannot be changed, however $HOMEDRIVE and $HOMEPATH
 53 |     can be changed. They are originally intended to be the equivalent of
 54 |     the $HOME path, but there are many known issues with them
 55 | 
 56 |     4. As for the name of the file itself, most of the tools ported from
 57 |     Unix will use the standard '.dotfile' scheme, but some of these will
 58 |     instead use "_dotfile". Of the latter, the two notable exceptions are
 59 |     vim, which will first try '_vimrc' before '.vimrc' (but it will try
 60 |     both) and git, which will require the user to name its netrc file
 61 |     '_netrc'.
 62 | 
 63 |     Relevant links :
 64 |     http://markmail.org/message/i33ldu4xl5aterrr
 65 |     http://markmail.org/message/wbzs4gmtvkbewgxi
 66 |     http://stackoverflow.com/questions/6031214/
 67 | 
 68 |     Because the whole thing is a mess, I suggest we tried various sensible
 69 |     defaults until we succeed or have depleted all possibilities.
 70 |     """
 71 | 
 72 |     if platform.system() != 'Windows':
 73 |         return [None]
 74 | 
 75 |     # Now, we only treat the case of Windows
 76 |     env_vars = [["HOME"],
 77 |                 ["HOMEDRIVE", "HOMEPATH"],
 78 |                 ["USERPROFILE"],
 79 |                 ["SYSTEMDRIVE"]]
 80 | 
 81 |     env_dirs = []
 82 |     for var_list in env_vars:
 83 | 
 84 |         var_values = [_getenv_or_empty(var) for var in var_list]
 85 | 
 86 |         directory = ''.join(var_values)
 87 |         if not directory:
 88 |             logging.debug('Environment var(s) %s not defined, skipping',
 89 |                           var_list)
 90 |         else:
 91 |             env_dirs.append(directory)
 92 | 
 93 |     additional_dirs = ["C:", ""]
 94 | 
 95 |     all_dirs = env_dirs + additional_dirs
 96 | 
 97 |     leading_chars = [".", "_"]
 98 | 
 99 |     res = [''.join([directory, os.sep, lc, config_name])
100 |            for directory in all_dirs
101 |            for lc in leading_chars]
102 | 
103 |     return res
104 | 
105 | 
106 | def authenticate_through_netrc(path=None):
107 |     """
108 |     Returns the tuple user / password given a path for the .netrc file.
109 |     Raises CredentialsError if no valid netrc file is found.
110 |     """
111 |     errors = []
112 |     netrc_machine = 'coursera-dl'
113 |     paths = [path] if path else get_config_paths("netrc")
114 |     for path in paths:
115 |         try:
116 |             logging.debug('Trying netrc file %s', path)
117 |             auths = netrc.netrc(path).authenticators(netrc_machine)
118 |         except (IOError, netrc.NetrcParseError) as e:
119 |             errors.append(e)
120 |         else:
121 |             if auths is None:
122 |                 errors.append('Didn\'t find any credentials for ' +
123 |                               netrc_machine)
124 |             else:
125 |                 return auths[0], auths[2]
126 | 
127 |     error_messages = '\n'.join(str(e) for e in errors)
128 |     raise CredentialsError(
129 |         'Did not find valid netrc file:\n' + error_messages)
130 | 
131 | 
132 | def get_credentials(username=None, password=None, netrc=None):
133 |     """
134 |     Returns valid username, password tuple.
135 |     Raises CredentialsError if username or password is missing.
136 |     """
137 |     if netrc:
138 |         path = None if netrc is True else netrc
139 |         return authenticate_through_netrc(path)
140 | 
141 |     if not username:
142 |         raise CredentialsError(
143 |             'Please provide a username with the -u option, '
144 |             'or a .netrc file with the -n option.')
145 | 
146 |     if not password:
147 |         password = getpass.getpass(
148 |             'Coursera password for {0}: '.format(username))
149 | 
150 |     return username, password
151 | 


--------------------------------------------------------------------------------
/coursera/define.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | 
 3 | """
 4 | This module defines the global constants.
 5 | """
 6 | 
 7 | import os
 8 | import getpass
 9 | import tempfile
10 | 
11 | AUTH_URL = 'https://accounts.coursera.org/api/v1/login'
12 | CLASS_URL = 'https://class.coursera.org/{class_name}'
13 | ABOUT_URL = 'https://www.coursera.org/maestro/api/topic/information?' \
14 |             'topic-id={class_name}'
15 | AUTH_REDIRECT_URL = 'https://class.coursera.org/{class_name}' \
16 |                     '/auth/auth_redirector?type=login&subtype=normal'
17 | 
18 | ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
19 | # define a per-user cache folder
20 | if os.name == "posix": # pragma: no cover
21 |     import pwd
22 |     user = pwd.getpwuid(os.getuid())[0]
23 | else:
24 |     user = getpass.getuser()
25 | 
26 | PATH_CACHE = os.path.join(tempfile.gettempdir(), user+"_coursera_dl_cache")
27 | PATH_COOKIES = os.path.join(PATH_CACHE, 'cookies')
28 | 


--------------------------------------------------------------------------------
/coursera/downloaders.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | 
  3 | from __future__ import print_function
  4 | 
  5 | import logging
  6 | import math
  7 | import os
  8 | import requests
  9 | import subprocess
 10 | import sys
 11 | import time
 12 | 
 13 | from six import iteritems
 14 | 
 15 | 
 16 | class Downloader(object):
 17 |     """
 18 |     Base downloader class.
 19 | 
 20 |     Every subclass should implement the _start_download method.
 21 | 
 22 |     Usage::
 23 | 
 24 |       >>> import downloaders
 25 |       >>> d = downloaders.SubclassFromDownloader()
 26 |       >>> d.download('http://example.com', 'save/to/this/file')
 27 |     """
 28 | 
 29 |     def _start_download(self, url, filename):
 30 |         """
 31 |         Actual method to download the given url to the given file.
 32 |         This method should be implemented by the subclass.
 33 |         """
 34 |         raise NotImplementedError("Subclasses should implement this")
 35 | 
 36 |     def download(self, url, filename):
 37 |         """
 38 |         Download the given url to the given file. When the download
 39 |         is aborted by the user, the partially downloaded file is also removed.
 40 |         """
 41 | 
 42 |         try:
 43 |             self._start_download(url, filename)
 44 |         except KeyboardInterrupt as e:
 45 |             logging.info(
 46 |                 'Keyboard Interrupt -- Removing partial file: %s', filename)
 47 |             try:
 48 |                 os.remove(filename)
 49 |             except OSError:
 50 |                 pass
 51 |             raise e
 52 | 
 53 | 
 54 | class ExternalDownloader(Downloader):
 55 |     """
 56 |     Downloads files with an extrnal downloader.
 57 | 
 58 |     We could possibly use python to stream files to disk,
 59 |     but this is slow compared to these external downloaders.
 60 | 
 61 |     :param session: Requests session.
 62 |     :param bin: External downloader binary.
 63 |     """
 64 | 
 65 |     # External downloader binary
 66 |     bin = None
 67 | 
 68 |     def __init__(self, session, bin=None):
 69 |         self.session = session
 70 |         self.bin = bin or self.__class__.bin
 71 | 
 72 |         if not self.bin:
 73 |             raise RuntimeError("No bin specified")
 74 | 
 75 |     def _prepare_cookies(self, command, url):
 76 |         """
 77 |         Extract cookies from the requests session and add them to the command
 78 |         """
 79 | 
 80 |         req = requests.models.Request()
 81 |         req.method = 'GET'
 82 |         req.url = url
 83 | 
 84 |         cookie_values = requests.cookies.get_cookie_header(
 85 |             self.session.cookies, req)
 86 | 
 87 |         if cookie_values:
 88 |             self._add_cookies(command, cookie_values)
 89 | 
 90 |     def _add_cookies(self, command, cookie_values):
 91 |         """
 92 |         Add the given cookie values to the command
 93 |         """
 94 | 
 95 |         raise RuntimeError("Subclasses should implement this")
 96 | 
 97 |     def _create_command(self, url, filename):
 98 |         """
 99 |         Create command to execute in a subprocess.
100 |         """
101 |         raise NotImplementedError("Subclasses should implement this")
102 | 
103 |     def _start_download(self, url, filename):
104 |         command = self._create_command(url, filename)
105 |         self._prepare_cookies(command, url)
106 |         logging.debug('Executing %s: %s', self.bin, command)
107 |         try:
108 |             subprocess.call(command)
109 |         except OSError as e:
110 |             msg = "{0}. Are you sure that '{1}' is the right bin?".format(
111 |                 e, self.bin)
112 |             raise OSError(msg)
113 | 
114 | 
115 | class WgetDownloader(ExternalDownloader):
116 |     """
117 |     Uses wget, which is robust and gives nice visual feedback.
118 |     """
119 | 
120 |     bin = 'wget'
121 | 
122 |     def _add_cookies(self, command, cookie_values):
123 |         command.extend(['--header', "Cookie: " + cookie_values])
124 | 
125 |     def _create_command(self, url, filename):
126 |         return [self.bin, url, '-O', filename, '--no-cookies',
127 |                 '--no-check-certificate']
128 | 
129 | 
130 | class CurlDownloader(ExternalDownloader):
131 |     """
132 |     Uses curl, which is robust and gives nice visual feedback.
133 |     """
134 | 
135 |     bin = 'curl'
136 | 
137 |     def _add_cookies(self, command, cookie_values):
138 |         command.extend(['--cookie', cookie_values])
139 | 
140 |     def _create_command(self, url, filename):
141 |         return [self.bin, url, '-k', '-#', '-L', '-o', filename]
142 | 
143 | 
144 | class Aria2Downloader(ExternalDownloader):
145 |     """
146 |     Uses aria2. Unfortunately, it does not give a nice visual feedback, but
147 |     gets the job done much faster than the alternatives.
148 |     """
149 | 
150 |     bin = 'aria2c'
151 | 
152 |     def _add_cookies(self, command, cookie_values):
153 |         command.extend(['--header', "Cookie: " + cookie_values])
154 | 
155 |     def _create_command(self, url, filename):
156 |         return [self.bin, url, '-o', filename,
157 |                 '--check-certificate=false', '--log-level=notice',
158 |                 '--max-connection-per-server=4', '--min-split-size=1M']
159 | 
160 | 
161 | class AxelDownloader(ExternalDownloader):
162 |     """
163 |     Uses axel, which is robust and it both gives nice
164 |     visual feedback and get the job done fast.
165 |     """
166 | 
167 |     bin = 'axel'
168 | 
169 |     def _add_cookies(self, command, cookie_values):
170 |         command.extend(['-H', "Cookie: " + cookie_values])
171 | 
172 |     def _create_command(self, url, filename):
173 |         return [self.bin, '-o', filename, '-n', '4', '-a', url]
174 | 
175 | 
176 | def format_bytes(bytes):
177 |     """
178 |     Get human readable version of given bytes.
179 |     Ripped from https://github.com/rg3/youtube-dl
180 |     """
181 |     if bytes is None:
182 |         return 'N/A'
183 |     if type(bytes) is str:
184 |         bytes = float(bytes)
185 |     if bytes == 0.0:
186 |         exponent = 0
187 |     else:
188 |         exponent = int(math.log(bytes, 1024.0))
189 |     suffix = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'][exponent]
190 |     converted = float(bytes) / float(1024 ** exponent)
191 |     return '{0:.2f}{1}'.format(converted, suffix)
192 | 
193 | 
194 | class DownloadProgress(object):
195 |     """
196 |     Report download progress.
197 |     Inspired by https://github.com/rg3/youtube-dl
198 |     """
199 | 
200 |     def __init__(self, total):
201 |         if total in [0, '0', None]:
202 |             self._total = None
203 |         else:
204 |             self._total = int(total)
205 | 
206 |         self._current = 0
207 |         self._start = 0
208 |         self._now = 0
209 | 
210 |         self._finished = False
211 | 
212 |     def start(self):
213 |         self._now = time.time()
214 |         self._start = self._now
215 | 
216 |     def stop(self):
217 |         self._now = time.time()
218 |         self._finished = True
219 |         self._total = self._current
220 |         self.report_progress()
221 | 
222 |     def read(self, bytes):
223 |         self._now = time.time()
224 |         self._current += bytes
225 |         self.report_progress()
226 | 
227 |     def calc_percent(self):
228 |         if self._total is None:
229 |             return '--%'
230 |         percentage = int(float(self._current) / float(self._total) * 100.0)
231 |         done = int(percentage/2)
232 |         return '[{0: <50}] {1}%'.format(done * '#', percentage)
233 | 
234 |     def calc_speed(self):
235 |         dif = self._now - self._start
236 |         if self._current == 0 or dif < 0.001:  # One millisecond
237 |             return '---b/s'
238 |         return '{0}/s'.format(format_bytes(float(self._current) / dif))
239 | 
240 |     def report_progress(self):
241 |         """Report download progress."""
242 |         percent = self.calc_percent()
243 |         total = format_bytes(self._total)
244 | 
245 |         speed = self.calc_speed()
246 |         total_speed_report = '{0} at {1}'.format(total, speed)
247 | 
248 |         report = '\r{0: <56} {1: >30}'.format(percent, total_speed_report)
249 | 
250 |         if self._finished:
251 |             print(report)
252 |         else:
253 |             print(report, end="")
254 |         sys.stdout.flush()
255 | 
256 | 
257 | class NativeDownloader(Downloader):
258 |     """
259 |     'Native' python downloader -- slower than the external downloaders.
260 | 
261 |     :param session: Requests session.
262 |     """
263 | 
264 |     def __init__(self, session):
265 |         self.session = session
266 | 
267 |     def _start_download(self, url, filename):
268 |         logging.info('Downloading %s -> %s', url, filename)
269 | 
270 |         attempts_count = 0
271 |         error_msg = ''
272 |         while attempts_count < 5:
273 |             r = self.session.get(url, stream=True)
274 | 
275 |             if r.status_code is not 200:
276 |                 logging.warn(
277 |                     'Probably the file is missing from the AWS repository...'
278 |                     ' waiting.')
279 | 
280 |                 if r.reason:
281 |                     error_msg = r.reason + ' ' + str(r.status_code)
282 |                 else:
283 |                     error_msg = 'HTTP Error ' + str(r.status_code)
284 | 
285 |                 wait_interval = 2 ** (attempts_count + 1)
286 |                 msg = 'Error downloading, will retry in {0} seconds ...'
287 |                 print(msg.format(wait_interval))
288 |                 time.sleep(wait_interval)
289 |                 attempts_count += 1
290 |                 continue
291 | 
292 |             content_length = r.headers.get('content-length')
293 |             progress = DownloadProgress(content_length)
294 |             chunk_sz = 1048576
295 |             with open(filename, 'wb') as f:
296 |                 progress.start()
297 |                 while True:
298 |                     data = r.raw.read(chunk_sz)
299 |                     if not data:
300 |                         progress.stop()
301 |                         break
302 |                     progress.read(len(data))
303 |                     f.write(data)
304 |             r.close()
305 |             return True
306 | 
307 |         if attempts_count == 5:
308 |             logging.warn('Skipping, can\'t download file ...')
309 |             logging.error(error_msg)
310 |             return False
311 | 
312 | 
313 | def get_downloader(session, class_name, args):
314 |     """
315 |     Decides which downloader to use.
316 |     """
317 | 
318 |     external = {
319 |         'wget': WgetDownloader,
320 |         'curl': CurlDownloader,
321 |         'aria2': Aria2Downloader,
322 |         'axel': AxelDownloader,
323 |     }
324 | 
325 |     for bin, class_ in iteritems(external):
326 |         if getattr(args, bin):
327 |             return class_(session, bin=getattr(args, bin))
328 | 
329 |     return NativeDownloader(session)
330 | 


--------------------------------------------------------------------------------
/coursera/test/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jplehmann/coursera/c9ff3dee3cc3a97df3942e633144872e8ab7f3c2/coursera/test/__init__.py


--------------------------------------------------------------------------------
/coursera/test/fixtures/auth/netrc:
--------------------------------------------------------------------------------
1 | machine coursera-dl login user@mail.com password secret


--------------------------------------------------------------------------------
/coursera/test/fixtures/auth/not_netrc:
--------------------------------------------------------------------------------
1 | fake


--------------------------------------------------------------------------------
/coursera/test/fixtures/cookies/chrome_cookies.txt:
--------------------------------------------------------------------------------
 1 | # Cookies for domains related to coursera.org.
 2 | # This content may be pasted into a cookies.txt file and used by wget
 3 | # Example:  wget -x --load-cookies cookies.txt https://class.coursera.org/class-001/class/index
 4 | #
 5 | class.coursera.org	FALSE	/class-001	FALSE	2373759602.166928	csrf_token	csrfclass001
 6 | class.coursera.org	FALSE	/class-001	FALSE	2373759602.166998	session	sessionclass001
 7 | www.coursera.org	FALSE	/	FALSE	2373765776.224814	maestro_login_flag	1
 8 | www.coursera.org	FALSE	/	FALSE	2372383376.224882	sessionid	c5b654a47a953e6c44ab519649da42df
 9 | www.coursera.org	FALSE	/	FALSE	2373765776.224931	maestro_login	fake
10 | www.coursera.org	FALSE	/	TRUE	42843173776	maestro_user	fake


--------------------------------------------------------------------------------
/coursera/test/fixtures/cookies/firefox_cookies.txt:
--------------------------------------------------------------------------------
 1 | # Netscape HTTP Cookie File
 2 | # http://www.netscape.com/newsref/std/cookie_spec.html
 3 | # This is a generated file!  Do not edit.
 4 | 
 5 | github.com	FALSE	/	FALSE	1371001812	_gauges_unique_hour	1
 6 | github.com	FALSE	/	FALSE	1371084612	_gauges_unique_day	1
 7 | github.com	FALSE	/	FALSE	1373676612	_gauges_unique_month	1
 8 | .coursera.org	TRUE	/	TRUE	2381580073	CAUTH	fake
 9 | .coursera.org	TRUE	/	FALSE	2381580073	maestro_login_flag	1
10 | .coursera.org	TRUE	/	FALSE	2377260083	__utmd	1
11 | class.coursera.org	FALSE	/	FALSE	2401754774	__204u	fake
12 | class.coursera.org	FALSE	/class-002	FALSE	2373594799	csrf_token	csrfclass002
13 | class.coursera.org	FALSE	/class-00Z	FALSE	2373594799	session	sessionclass002
14 | www.coursera.org	FALSE	/	TRUE	42843133365	maestro_user	fake
15 | .coursera.org	TRUE	/	FALSE	2434228385	__utma	fake
16 | class.coursera.org	FALSE	/class-001	FALSE	2373748385	csrf_token	csrfclass001
17 | class.coursera.org	FALSE	/class-001	FALSE	2373748385	session	sessionclass1
18 | disqus.com	FALSE	/	FALSE	1366621292	__jid	fake
19 | 


--------------------------------------------------------------------------------
/coursera/test/fixtures/cookies/firefox_cookies_expired.txt:
--------------------------------------------------------------------------------
 1 | # Netscape HTTP Cookie File
 2 | # http://www.netscape.com/newsref/std/cookie_spec.html
 3 | # This is a generated file!  Do not edit.
 4 | 
 5 | github.com	FALSE	/	FALSE	1371001812	_gauges_unique_hour	1
 6 | github.com	FALSE	/	FALSE	1371084612	_gauges_unique_day	1
 7 | github.com	FALSE	/	FALSE	1373676612	_gauges_unique_month	1
 8 | .coursera.org	TRUE	/	TRUE	1281580262	CAUTH	IL
 9 | .coursera.org	TRUE	/	FALSE	2281580262	maestro_login_flag	1
10 | class.coursera.org	FALSE	/	FALSE	2401754774	__204u	fake
11 | class.coursera.org	FALSE	/class-002	FALSE	2373594799	csrf_token	csrfclass002
12 | class.coursera.org	FALSE	/class-00Z	FALSE	2373594799	session	sessionclass002
13 | www.coursera.org	FALSE	/	TRUE	42843133365	maestro_user	fake
14 | .coursera.org	TRUE	/	FALSE	2434228385	__utma	fake
15 | class.coursera.org	FALSE	/class-001	FALSE	1273748385	csrf_token	csrfclass001
16 | class.coursera.org	FALSE	/class-001	FALSE	1273748385	session	sessionclass1
17 | disqus.com	FALSE	/	FALSE	1366621292	__jid	fake
18 | 


--------------------------------------------------------------------------------
/coursera/test/fixtures/cookies/firefox_cookies_without_coursera.txt:
--------------------------------------------------------------------------------
1 | # Netscape HTTP Cookie File
2 | # http://www.netscape.com/newsref/std/cookie_spec.html
3 | # This is a generated file!  Do not edit.
4 | 
5 | github.com	FALSE	/	FALSE	1371001812	_gauges_unique_hour	1
6 | github.com	FALSE	/	FALSE	1371084612	_gauges_unique_day	1
7 | github.com	FALSE	/	FALSE	1373676612	_gauges_unique_month	1


--------------------------------------------------------------------------------
/coursera/test/fixtures/html/hidden-videos.html:
--------------------------------------------------------------------------------
  1 | <!DOCTYPE html>
  2 | <html lang="en-US">
  3 | <head>
  4 | <meta charset="utf-8" />
  5 | <title>Announcements | Scientific Computing</title>
  6 | <link rel="stylesheet" href="https://dt5zaw6a98blc.cloudfront.net/site-static/90f5c29fdd72ac361e43e11980466765a9db8889/css/spark.main.css" />
  7 | <link rel="stylesheet" href="https://www.coursera.org/maestro/api/course/340/course.css">
  8 | 
  9 | <link rel="icon" href="https://spark-public.s3.amazonaws.com/scientificcomp/static/images/favicon.ico" />
 10 | 
 11 | </head>
 12 | 
 13 | <body>
 14 |     <a href="#course-page-content" class="hidden">Skip Navigation</a>
 15 |     <div class="hidden">
 16 |         This page features MathJax technology to render mathematical formulae.
 17 |         If you are using a screen reader, please visit <a href="http://www.dessci.com/en/products/mathplayer/">MathPlayer</a> to download the plugin for your browser. Please note that this is an Internet Explorer-only plugin at this time.
 18 |     </div>
 19 | 
 20 |     <h1 class="hidden">Scientific Computing</h1>
 21 | 
 22 |     
 23 |     
 24 | 
 25 | <div class="course-topbar container-fluid">
 26 | 
 27 |     <div class="row-fluid">
 28 |         
 29 |         <div class="span3">
 30 |           <a href="https://www.coursera.org/" class="course-topbar-logo" data-if-linkable="add-referrer"></a>
 31 |         </div>
 32 | 
 33 |         <div class="course-topbar-nav-container span9" style="float:right;">
 34 |             <h2 class="hidden">Top Navigation Bar</h2>
 35 | 
 36 |             <ul class="course-topbar-nav-list">
 37 |                 <li class="course-topbar-nav-list-item">
 38 |                   <a href="https://www.coursera.org/courses" data-if-linkable="add-referrer">Courses</a>
 39 |                 </li>
 40 | 
 41 |                 
 42 |                                 
 43 |                 <li class="course-topbar-nav-list-item"
 44 |                     tabindex="0" role="button" aria-haspopup="true" aria-expanded="false" aria-owns="course-topbar-my"
 45 |                     data-popup="#course-topbar-my" data-popup-bind-open="mouseenter" data-popup-close style="cursor:pointer;" data-popup-direction="se">
 46 |                                         <a data-user-id="xxxxxx">  
 47 |                         Joe Doe
 48 |                         <i class="icon-caret-down"></i>
 49 |                     </a>
 50 |                 </li>
 51 |                
 52 |                             </ul>
 53 |         </div>
 54 |     </div>
 55 | 
 56 |     <div id="course-topbar-aboutus" class="course-topbar-sublist">
 57 |         <a class="course-topbar-sublist-item" href="https://www.coursera.org/about/jobs" target="_new" data-if-linkable="add-referrer">Jobs</a>
 58 |         <a class="course-topbar-sublist-item" href="https://www.coursera.org/about/team" target="_new" data-if-linkable="add-referrer">Team</a>
 59 |         <a class="course-topbar-sublist-item" href="https://www.coursera.org/about/contact" target="_new" data-if-linkable="add-referrer">Contact Us</a>
 60 |         <a class="course-topbar-sublist-item" href="https://www.coursera.org/about/" target="_new" data-if-linkable="add-referrer">About Us</a>
 61 |     </div>
 62 | 
 63 |     <div id="course-topbar-my" class="course-topbar-sublist">
 64 |         <a class="course-topbar-sublist-item" href="https://www.coursera.org/" target="_new">Courses</a>
 65 |         <a class="course-topbar-sublist-item" href="https://www.coursera.org/account/profile" target="_new">Profile</a>
 66 |         <a class="course-topbar-sublist-item" href="https://www.coursera.org/account/records" target="_new">Course Records</a>
 67 |         <a class="course-topbar-sublist-item" href="https://www.coursera.org/account/settings" target="_new">Settings</a>
 68 |         <a class="course-topbar-sublist-item" href="https://class.coursera.org/scientificcomp-2012-001/class/preferences">Course Preferences</a>
 69 |         <a class="course-topbar-sublist-item" href="https://class.coursera.org/scientificcomp-2012-001/auth/logout">Logout</a>
 70 |     </div>
 71 | 
 72 | </div>
 73 | 
 74 | 
 75 | 
 76 | 
 77 | 
 78 | 
 79 | 
 80 | 
 81 | 
 82 | <style type="text/css">
 83 |  
 84 | .course-topbar {
 85 |   background-color: #4d4d4d;
 86 |   box-shadow: inset 0px -6px 18px 0px rgba(0, 0, 0, 0.25);
 87 | }
 88 | 
 89 | .course-topbar-logo {
 90 |    margin-top: 12px;
 91 |    margin-left: 21px;
 92 | }
 93 | 
 94 | div.course-topbar-sublist {
 95 |   border-top: solid 3e3e3e 1px;
 96 |   background: #3e3e3e;
 97 | }
 98 | 
 99 | li.course-topbar-nav-list-item {
100 |   text-shadow 0px 1px rgba(0, 0, 0, 0.5);
101 | }
102 | 
103 | .course-topbar-nav-container a {
104 |    color: #3e3e3e;
105 | }
106 | 
107 | div.course-topbar-sublist a.course-topbar-sublist-item {
108 |   color: #3e3e3e;
109 | }
110 | 
111 | 
112 | .course-topbanner {
113 |   border-bottom: 1px solid #BCBEC0;
114 |   height: 71px;
115 |   box-shadow 0px 0px 1px 1px #ccc;
116 |   position: relative;
117 |   overflow: hidden;
118 |   background-color: white;
119 | }
120 | 
121 | .course-topbanner-university-logo {
122 |   width: 200px;
123 |   overflow: hidden;
124 |   float: left;
125 |   margin-top: 12px;
126 |   margin-left: 21px;
127 |   margin-right: 20px;
128 |   height: 59px;
129 | }
130 | 
131 | .course-topbanner-university-logo img {
132 |   max-height: 48px;
133 |   max-width: 200px;
134 | }
135 | 
136 | .course-topbanner-header {
137 |   font-size: 16px;
138 |   line-height: 21px;
139 |   vertical-align: top;
140 |   margin-top: 13px;
141 |   margin-left: 26px;
142 |   margin-bottom: 0px;
143 |   margin-right: 10px;
144 |   height: 50px;
145 |   overflow: hidden;
146 |   text-overflow: ellipsis;
147 |   white-space: nowrap;
148 | }
149 |  
150 | .course-topbanner-name {
151 |   font-size: 18px;
152 | }
153 |  
154 | .course-topbanner-instructor {
155 |   font-size: 15px;
156 |   font-weight: lighter;
157 | }
158 | 
159 | .course-topbanner-side {
160 |   float: right;
161 |   margin-top: 12px;
162 |   margin-left: 20px;
163 |   margin-right: 20px;
164 |   min-width: 120px;
165 |   text-align: right;
166 | }
167 | 
168 | .course-topbanner-side img {
169 |   height: 48px;
170 |   border: 1px solid #eee;
171 | }
172 | 
173 | </style>
174 | <div class="course-topbanner">
175 |   <div class="course-topbanner-university-logo">
176 |     <a href="https://www.coursera.org/uw" class="coursera-university-color" target="_blank" data-if-linkable="add-referrer">
177 |       <img src="https://www.coursera.org/maestro/api/course/340/university_logo" alt="University logo">
178 |     </a>
179 |   </div>
180 | 
181 |   
182 |   <h1 class="course-topbanner-header">
183 |       <a class="course-topbanner-name coursera-university-color" href="https://class.coursera.org/scientificcomp-2012-001/class/index" data-if-linkable="modal-lock">
184 |         Scientific Computing      </a>
185 |       <br>
186 |       <span class="course-topbanner-instructor" style="">
187 |         by J. Nathan Kutz      </span>
188 |   </h1>
189 | </div>
190 |     
191 |         <div class="container-fluid" style="position:relative;">
192 |         <div class="row-fluid">
193 |             <div id="course-page-sidebar">
194 |                 <div class="course-navbar-container">
195 | 		<a href="https://class.coursera.org/scientificcomp-2012-001/class/index"
196 |       >
197 | 	<img src="https://www.coursera.org//maestro/api/course/340/logo" style="width: 210px;height: 112px;margin: -15px 0px 8px 1px; box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.35);">
198 | 	</a>
199 | 	<h2 class="hidden">Side Navigation Bar</h2>
200 | <ul class="course-navbar-list">
201 | 		
202 | 	<li class="course-navbar-item">
203 | 								<a href="https://class.coursera.org/scientificcomp-2012-001/class/index"  data-if-linkable="modal-lock,gray,lock-icon" class="coursera-university-color">
204 | 				
205 | 				Home - Announcements							</a>
206 | 						</li>
207 | 	
208 | 		
209 | 	<li class="course-navbar-item">
210 | 								<a href="https://class.coursera.org/scientificcomp-2012-001/wiki/view?page=WelcometotheCourse"  data-if-linkable="modal-lock,gray,lock-icon" class="coursera-university-color">
211 | 				
212 | 				Welcome to the Course							</a>
213 | 						</li>
214 | 	
215 | 		
216 | 	<li class="course-navbar-item">
217 | 								<a href="https://class.coursera.org/scientificcomp-2012-001/wiki/view?page=CourseResources"  data-if-linkable="modal-lock,gray,lock-icon" class="coursera-university-color">
218 | 				
219 | 				Course Resources							</a>
220 | 						</li>
221 | 	
222 | 		
223 | 	<li class="course-navbar-item">
224 | 								<a href="https://class.coursera.org/scientificcomp-2012-001/lecture/index"  data-if-linkable="modal-lock,gray,lock-icon" class="coursera-university-color">
225 | 				
226 | 				Video Lectures							</a>
227 | 						</li>
228 | 	
229 | 		
230 | 	<li class="course-navbar-item">
231 | 								<a href="https://class.coursera.org/scientificcomp-2012-001/quiz/index"  data-if-linkable="modal-lock,gray,lock-icon" class="coursera-university-color">
232 | 				
233 | 				Quizzes							</a>
234 | 						</li>
235 | 	
236 | 		
237 | 	<li class="course-navbar-item">
238 | 								<a href="https://class.coursera.org/scientificcomp-2012-001/quiz/index?quiz_type=homework"  data-if-linkable="modal-lock,gray,lock-icon" class="coursera-university-color">
239 | 				
240 | 				Homework							</a>
241 | 						</li>
242 | 	
243 | 		
244 | 	<li class="course-navbar-item">
245 | 								<a href="https://class.coursera.org/scientificcomp-2012-001/forum/index"  data-if-linkable="modal-lock,gray,lock-icon" class="coursera-university-color">
246 | 				
247 | 				Discussion Forums							</a>
248 | 						</li>
249 | 	
250 | 		
251 | 	<li class="course-navbar-item">
252 | 								<a href="https://class.coursera.org/scientificcomp-2012-001/wiki/view?page=TipsforCourseraUse"  data-if-linkable="modal-lock,gray,lock-icon" class="coursera-university-color">
253 | 				
254 | 				Tips for Coursera Use							</a>
255 | 						</li>
256 | 	
257 | 		
258 | 	<li class="course-navbar-item">
259 | 								<a href="https://class.coursera.org/scientificcomp-2012-001/wiki/view?page=CourseFAQScientificComputing"  data-if-linkable="modal-lock,gray,lock-icon" class="coursera-university-color">
260 | 				
261 | 				FAQ about the Course							</a>
262 | 						</li>
263 | 	
264 | 		
265 | 	<li class="course-navbar-item">
266 | 								<a href="https://class.coursera.org/scientificcomp-2012-001/wiki/view?page=Administration"  data-if-linkable="modal-lock,gray,lock-icon" class="coursera-university-color">
267 | 				
268 | 				Administration							</a>
269 | 						</li>
270 | 	
271 | 	
272 | 		<li class="course-navbar-item">
273 | 		<a href="https://share.coursera.org/wiki/index.php/scientificcomp:Main" target="_blank" class="coursera-university-color">
274 | 			Course Wiki
275 | 			<span class="icon-share"></span>
276 | 		</a>
277 | 	</li>
278 | 		
279 | 		<li class="course-navbar-item">
280 | 		<a href="http://www.meetup.com/Coursera/" target="_blank" class="coursera-university-color">
281 | 			Join a Meetup
282 | 			<span class="icon-share"></span>
283 | 		</a>
284 | 	</li>
285 | 	
286 |     </ul>
287 | 
288 | </div>
289 |             </div>
290 |             <div id="course-page-content">
291 |                                 
292 | 
293 | <div style="width:240px;float:right;" class="course-page-sidebar">
294 | 
295 |   <div style="width:100%;background-color:#f8f8f8;padding:20px;border:1px solid #d0d0d0;">
296 | 
297 |     <h2 class="hidden">Reminders</h2>
298 | 
299 |         
300 |     
301 |     <h4>Upcoming Deadlines</h4>
302 | 
303 |     
304 |        
305 | 
306 |     
307 |        
308 | 
309 |     
310 |        
311 | 
312 |     
313 |        
314 | 
315 |     
316 |        
317 | 
318 |     
319 |     <!-- DEADLINES FOOTER -->
320 |     
321 |     <hr>
322 |     
323 |        
324 | 
325 | 
326 |     
327 |   </div>
328 | 
329 | </div>
330 | 
331 | 
332 | <div style="padding-right:280px">
333 |   
334 |     <h2 class="course-page-header">Announcements</h2>
335 |     <!-- ANNOUNCEMENTS PREAMBLE -->    
336 |                         <div style="border-bottom:1px solid #f0f0f0;padding-top:16px;padding-bottom:16px;">
337 |                 <div style="overflow:hidden;margin-top:-5px;padding-right:25px;">
338 |                                         <h3>Scientific Computing course ending information</h3>
339 |                                         <div>On December 16th, Scientific Computing will be closing for this session. <br><br />
340 | All quizzes and homework assignments need to be completed and finalized by December 15th at 11:59pm PST.  Currently enrolled students can continue to access the course via a “Course Archive” button on the Coursera landing page for the course. <br><br />
341 | If you or others are interested in taking this Scientific Computing class again we plan to offer the course again on Coursera beginning January 7th, 2013. <br><br />
342 | Dr. Nathan Kutz will be offering a new course beginning January 7th entitled Computational Methods for Data Analysis which can be found on Coursera. <br><br />
343 | Thanks you for your participation in this course. <br><br />
344 | The University of Washington<br />
345 | </div>
346 |                                         <div style="text-align:right;padding-top:10px;font-style:italic;">Mon  3 Dec 2012  2:00 PM PST -0800</div>
347 |                                     </div>
348 |             </div>
349 |                                 <div style="border-bottom:1px solid #f0f0f0;padding-top:16px;padding-bottom:16px;">
350 |                 <div style="overflow:hidden;margin-top:-5px;padding-right:25px;">
351 |                                         <h3>Week 8 cheb.m matlab file to be accessed.</h3>
352 |                                         <div>Hello and welcome to week 8.<br />
353 | <br />
354 | Please find the needed file that you will need for Week 8 at the following link: <a href="http://spark-public.s3.amazonaws.com/scientificcomp/Lecture%20Note%20Packet/cheb.m" target="_blank">cheb.m MATLAB Document for week 8</a></div>
355 |                                         <div style="text-align:right;padding-top:10px;font-style:italic;">Tue 13 Nov 2012  1:03 PM PST -0800</div>
356 |                                     </div>
357 |             </div>
358 |                                 <div style="border-bottom:1px solid #f0f0f0;padding-top:16px;padding-bottom:16px;">
359 |                 <div style="overflow:hidden;margin-top:-5px;padding-right:25px;">
360 |                                         <h3>Quiz Attempts</h3>
361 |                                         <div><strong>Weekly Lecture Quiz attempts</strong><br />
362 | <ul><br />
363 |   <li>You now have TWO attempts ONLY to complete the quizzes assigned. (This is retroactive.)</li><br />
364 |   <li>Quizes are timed.</li><br />
365 |   <li>Once opened, You have 3 hours to complete them before they  close; Give yourself enough time to complete the quiz.</li><br />
366 |   <li>Quiz names have been updated to be more reflective of their association with each lecture. </li><br />
367 | </ul><br />
368 | Before attempting, be  sure that you have viewed the lectures and read the notes so that you are fully  prepared to answer the questions.</div>
369 |                                         <div style="text-align:right;padding-top:10px;font-style:italic;">Tue  2 Oct 2012  8:54 AM PDT -0700</div>
370 |                                     </div>
371 |             </div>
372 |                                 <div style="border-bottom:1px solid #f0f0f0;padding-top:16px;padding-bottom:16px;">
373 |                 <div style="overflow:hidden;margin-top:-5px;padding-right:25px;">
374 |                                         <h3>Video Lecture Downloads</h3>
375 |                                         <div>Thank you for enrolling in this free course. The University  of Washington is committed to working with some of the world's leading  instructors to provide high quality, free education, globally.<br><br />
376 | <p>We have removed the ability to download course video from  Coursera for a number of reasons:<br />
377 | <ol><br />
378 |     <li>Our video contains functionality which only  works within the Coursera environment.&nbsp; Downloading it to another  platform, such as YouTube, prevents this from working  as designed.</li><br />
379 |     <li>Our ability to bring you a free course means that  we must protect its content so that it is viewed as intended, as a component of an instructionally coherent educational program in Coursera.</li></ol><br />
380 | We apologize if this is an  inconvenience for some students who would prefer more flexibility for viewing  this free content. Along with the free course offering, we also provide the  added opportunity for Coursera students to enroll in one of our leading  Certificate programs.<br />
381 | <p><br />
382 | We hope that you understand our need to maintain a quality  standard and continue to enjoy the course.<br />
383 | <br><p>The University of Washington</div>
384 |                                         <div style="text-align:right;padding-top:10px;font-style:italic;">Fri 28 Sep 2012  9:37 AM PDT -0700</div>
385 |                                     </div>
386 |             </div>
387 |                                 <div style="border-bottom:1px solid #f0f0f0;padding-top:16px;padding-bottom:16px;">
388 |                 <div style="overflow:hidden;margin-top:-5px;padding-right:25px;">
389 |                                         <h3>Keep Learning about Scientific Computing</h3>
390 |                                         <div>Thank you for enrolling in <em>Scientific Computing.</em> Given your interest  in this area of study, you may want to learn more about the <a href="http://www.pce.uw.edu/certificates/scientific-computing-coursera.html" target="_blank">Certificate in Scientific Computing</a> offered online through  the <a href="http://www.pce.uw.edu/" target="_blank">University of Washington</a>.<br><br />
391 | By enrolling in the UW certificate  program, you gain access to the enhanced version of each of the courses, which  includes interaction with an instructor, additional assignments, readings, and  multimedia material. Plus, you earn a valuable UW credential.<br><br />
392 |  The deadline to apply for this program  is 5 p.m. Pacific Daylight Time on October 8, 2012. Details on <a href="http://www.pce.uw.edu/certificates/scientific-computing-coursera/online-autumn-2012/apply/" target="_blank">how to apply</a> are available online. Feel free to contact a  UW enrollment adviser by e-mail at <a href="mailto:info@pce.uw.edu">info@pce.uw.edu</a> or by phone at 888-469-6499, if you have additional questions. <br><br />
393 |  If you’re unable to commit to the  full program at this time but would like to consider enrolling in the future, sign  up for e-mail notifications on the <a href="http://www.pce.uw.edu/certificates/scientific-computing-coursera.html" target="_blank">UW Certificate in Scientific Computing website</a>. <br><br />
394 | Once again, thank you for joining  us in this course!<br />
395 | <br />
396 | </div>
397 |                                         <div style="text-align:right;padding-top:10px;font-style:italic;">Mon 24 Sep 2012 12:00 PM PDT -0700</div>
398 |                                     </div>
399 |             </div>
400 |                                 <div style="border-bottom:1px solid #f0f0f0;padding-top:16px;padding-bottom:16px;">
401 |                 <div style="overflow:hidden;margin-top:-5px;padding-right:25px;">
402 |                                         <h3>Welcome to the Scientific Computing course!</h3>
403 |                                         <div>Thank you for joining Scientific  Computing! Please take a few moments to read through the course welcome page  and watch the welcome video. There is a lot of useful information there about  the course. <br />
404 | <br />
405 | For now, you should plan to allocate between 10 and 15 hours per week on the  Scientific Computing course. There will be roughly 2 hours of lectures per  week, as well as weekly homework assignments. The homework assignments will  consist of computation exercises and assignments using information contained  in both the lecture as well as the course lecture notes packet. You will need  to download the course lecture notes packet to be able to complete some of the  quiz questions as well as a reference for the course. This may be  downloaded in the course resources or at the following link. <a href="http://spark-public.s3.amazonaws.com/scientificcomp/Lecture%20Note%20Packet/LectureNotePacketSciComputing.pdf" target="_blank">Course Lecture Notes Packet</a></p><br />
406 | A text book is  not necessary as the course is self-contained and the course notes are  provided. To be successful in the course, a strong background in linear algebra  is required. Familiarity with methods of ordinary differential equations and  basic programming structure is also required. With this background, you  should be able to develop the formulas necessary for the homework in the course.<br><br />
407 | <br />
408 | Given the computational nature of the course,  access to MATLAB (<a href="http://www.mathworks.com/">www.mathworks.com</a>) or Octave (<a href="http://www.gnu.org/software/octave/">www.gnu.org/software/octave</a>) is essential. MATLAB  provides student editions for $99 that can be downloaded via the web. Octave is  a free (or by donation) alternative to MATLAB that can also be downloaded and  installed via the web. Either software should suffice for all the needs of the  course, but MATLAB is the strongly recommended alternative.<br />
409 | <br />
410 | My goal is that by the end of the course you will be able to investigate the flexibility and power of project-oriented  computational analysis.. It has been a lot of work and a lot of fun  getting this course ready for you. I sincerely hope you enjoy it!<br><br />
411 | <br />
412 | Nathan Kutz</div>
413 |                                         <div style="text-align:right;padding-top:10px;font-style:italic;">Mon 24 Sep 2012 12:00 AM PDT -0700</div>
414 |                                     </div>
415 |             </div>
416 |             </div>
417 | 
418 | 
419 |             </div>
420 |         </div>
421 |     </div>
422 |         
423 |         <script src="https://dt5zaw6a98blc.cloudfront.net/site-static/90f5c29fdd72ac361e43e11980466765a9db8889/spark/spark.js"></script>
424 |     
425 |     <script>
426 |         $("[data-readme]").each(function() {
427 |             window.ReadMe(this);
428 |         });
429 |     </script>
430 |     <script src="https://dt5zaw6a98blc.cloudfront.net/site-static/90f5c29fdd72ac361e43e11980466765a9db8889/spark/core/js/modal_focus.js" type="text/javascript"></script>
431 |     <script src="https://dt5zaw6a98blc.cloudfront.net/site-static/90f5c29fdd72ac361e43e11980466765a9db8889/spark/core/js/get_query_parameters.js" type="text/javascript"></script>
432 |     <script src="https://dt5zaw6a98blc.cloudfront.net/site-static/90f5c29fdd72ac361e43e11980466765a9db8889/spark/core/js/signature_track.js" type="text/javascript"></script>
433 |     <script src="https://dt5zaw6a98blc.cloudfront.net/site-static/90f5c29fdd72ac361e43e11980466765a9db8889/spark/app/generic/js/item_list.js" type="text/javascript"></script>
434 |     <script src="https://dt5zaw6a98blc.cloudfront.net/site-static/90f5c29fdd72ac361e43e11980466765a9db8889/spark/app/generic/js/navbar.js" type="text/javascript"></script>
435 | 
436 |     
437 |     <script type="text/javascript">
438 |         // SET UP THE MULTITRACKER
439 |                 window.logger = window.Log({level:"error"});
440 |                 window.multitracker = window.MultiTracker({logger:window.logger});
441 |         window.multitracker.register('204', window._204 = window._204 || []);
442 |         window.multitracker.register('ga', window._gaq = window._gaq || [], 'google');
443 | 
444 |         // SET UP GA and 204
445 |         window._gaq.push(['_setAccount', 'UA-28377374-1']);
446 |         window._gaq.push(['_setDomainName', 'coursera.org']);
447 |         window._gaq.push(['_setAllowLinker', true]);  
448 |         window._gaq.push(['_trackPageview']);
449 |         
450 |         window._204.push({key:"pageview", val:{}});
451 | 
452 |         // override our multitracker queues with the ga queue when it is ready
453 |         (function() {
454 |           var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
455 |           ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
456 |           var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
457 |         
458 |           ga.onload = ga.onreadystatechange = function() {
459 |               ga.onreadystatechange = ga.onload = null;
460 |               window.multitracker.get('ga').queue = window._gaq;
461 |           };
462 |         })();
463 | 
464 |         // override our multitracker queues with the 204 queue when it is ready
465 |         (function() {
466 |           var a = document.createElement('script'); a.type = 'text/javascript'; a.async = true;
467 |           a.src = 'https://eventing.coursera.org/204.min.js?client=spark&user=xxxxxx';
468 |           var s = document.getElementsByTagName('script')[0];
469 |           s.parentNode.insertBefore(a, s);
470 |           a.onload = a.onreadystatechange = function() {
471 |               a.onreadystatechange = a.onload = null;
472 |               window.multitracker.get('204').queue = window._204;
473 |           };
474 |         })();
475 |     </script>
476 | 
477 |     <script>
478 |       // Set up shared AB testing object
479 |       (function() {
480 |           var userId = $('[data-user-id]:first').attr('data-user-id');
481 | 
482 |           // Adapted from app/www/static/js/models/user.js
483 |           if (userId == 0) {
484 |               userId = window.cookie.get('ab-session-id');
485 |               if (!userId) {
486 |                   userId =
487 |                       (Math.random() * 10000000000).toFixed() + '-' +
488 |                       (new Date()).getTime();
489 |                   cookie.set("ab-session-id", userId, { expires: 36500 });
490 |               }
491 |           }
492 | 
493 |                         window.ab = window.AB(userId, {debug:false, tracker:window.multitracker, logger:window.logger});
494 |           
495 |       })();
496 |     </script>
497 | 
498 |     <script type="text/x-mathjax-config">
499 | MathJax.Hub.Config({
500 |   config: ["MMLorHTML.js"],
501 |   
502 |   styleSheets: [],
503 |   styles: {},
504 | 
505 |   jax: ["input/TeX"],
506 |   
507 |   extensions: ["tex2jax.js"],
508 | 
509 |   preJax: null,
510 |   postJax: null,
511 | 
512 |   preRemoveClass: "MathJax_Preview",
513 | 
514 |   showProcessingMessages: true,
515 | 
516 |   messageStyle: "none",
517 |   
518 |   displayAlign: "center",
519 |   displayIndent: "0em",
520 |   
521 |   delayStartupUntil: "none",
522 | 
523 |   skipStartupTypeset: false,
524 |   
525 |   elements: [],
526 |   
527 |   tex2jax: {
528 | 	    inlineMath: [
529 | 	                 ['$$','$$'],      // uncomment this for standard TeX math delimiters
530 | 	                 ['\\(','\\)']
531 | 	                 ],
532 | 
533 | 	                 displayMath: [
534 | 	                 ['\\[','\\]']
535 | 	                 ],
536 | 
537 |     skipTags: ["script","noscript","style","textarea","pre","code"],
538 |     ignoreClass: "tex2jax_ignore",
539 |     processClass: "tex2jax_process",
540 |     processEscapes: false,
541 |     processEnvironments: true,
542 |     preview: "TeX"
543 |     
544 |   },
545 |   
546 |   mml2jax: {
547 |     preview: "alttext"
548 |     
549 |   },
550 |   
551 |   jsMath2jax: {
552 |     preview: "TeX"
553 |     
554 |   },
555 | 
556 |   TeX: {
557 |     TagSide: "right",
558 |     TagIndent: ".8em",
559 |     MultLineWidth: "85%",
560 |     Macros: {},
561 | 
562 |     extensions: ["AMSmath.js", "AMSsymbols.js"]
563 |     
564 |   },
565 | 
566 |   //============================================================================
567 |   //
568 |   //  These parameters control the MathML inupt jax.
569 |   //
570 |   MathML: {
571 |     //
572 |     //  This specifies whether to use TeX spacing or MathML spacing when the
573 |     //  HTML-CSS output jax is used.
574 |     //
575 |     useMathMLspacing: false
576 |   },
577 |   
578 |   //============================================================================
579 |   //
580 |   //  These parameters control the HTML-CSS output jax.
581 |   //
582 |   "HTML-CSS": {
583 |     
584 |     scale: 100,
585 |     
586 |     availableFonts: ["STIX","TeX"],
587 |     
588 |     preferredFont: "TeX",
589 |     
590 |     webFont: "TeX",
591 |     
592 |     imageFont: "TeX",
593 |     
594 |     undefinedFamily: "STIXGeneral,'Arial Unicode MS',serif",
595 |       
596 |     showMathMenu: true,
597 | 
598 |     styles: {},
599 |     
600 |     tooltip: {
601 |       delayPost: 600,          // milliseconds delay before tooltip is posted after mouseover
602 |       delayClear: 600,         // milliseconds delay before tooltip is cleared after mouseout
603 |       offsetX: 10, offsetY: 5  // pixels to offset tooltip from mouse position
604 |     }
605 |   },
606 |   
607 |   //============================================================================
608 |   //
609 |   //  These parameters control the NativeMML output jax.
610 |   //
611 |   NativeMML: {
612 | 
613 |     scale: 100,
614 | 
615 |     showMathMenu: true,
616 |     showMathMenuMSIE: true,
617 | 
618 |     styles: {}
619 |   },
620 |   
621 |   MathMenu: {
622 |     delay: 400,
623 |     
624 |     helpURL: "http://www.mathjax.org/help/user/",
625 | 
626 |     showRenderer: true,
627 |     showFontMenu: false,
628 |     showContext:  false,
629 | 
630 |     windowSettings: {
631 |       status: "no", toolbar: "no", locationbar: "no", menubar: "no",
632 |       directories: "no", personalbar: "no", resizable: "yes", scrollbars: "yes",
633 |       width: 100, height: 50
634 |     },
635 |     
636 |     styles: {}
637 |     
638 |   },
639 | 
640 |   MMLorHTML: {
641 |     prefer: {
642 |       MSIE:    "MML",
643 |       Firefox: "MML",
644 |       Opera:   "HTML",
645 |       other:   "HTML"
646 |     }
647 |   }
648 | });
649 | </script>
650 | <script type="text/javascript">
651 | (function () {
652 | 
653 |   function loadMathJax() {
654 |     var script = document.createElement("script");
655 |     script.type = "text/javascript";
656 |     script.src  = "https://duqnjvq4jwr55.cloudfront.net/2.1/MathJax.js";
657 |     document.getElementsByTagName("head")[0].appendChild(script);
658 |   }
659 | 
660 |   window.loadOrRefreshMathJax = function(domId) {
661 |     if (window.MathJax) {
662 |       if (domId) {
663 |         MathJax.Hub.Queue(["Typeset", MathJax.Hub, domId]);
664 |       } else {
665 |         MathJax.Hub.Queue(["Typeset", MathJax.Hub]);
666 |       }
667 |     } else {
668 |       loadMathJax();
669 |     }
670 |   }
671 | })();
672 | </script>
673 |     
674 |     <script type="text/javascript">
675 | 
676 |         function hideMaintenanceMessage() {
677 |             $('#maintenance_message').hide();
678 |             document.cookie = "spark_maintmsghide_2=1;expires=0;path=/;";
679 |         }
680 | 
681 |         var site_base_url = "https:\/\/www.coursera.org\/";
682 |         var site_static_asset_url = "https:\/\/dt5zaw6a98blc.cloudfront.net\/site-static\/";
683 |         var spark_class_short_name = "scientificcomp";
684 |         var spark_class_id = 340;
685 |         var spark_class_url = "\/scientificcomp-2012-001";
686 |         var spark_signature_url = "https:\/\/class.coursera.org\/scientificcomp-2012-001\/signature\/";
687 |         
688 |         var course_strings_name = "Scientific Computing";
689 |         var student_full_name = "Joe Doe";
690 |     
691 |     </script>
692 | 
693 |     
694 | <script type="text/javascript">$(function() { loadOrRefreshMathJax();})</script>
695 | 
696 | </body>
697 | 
698 | </html>
699 | 


--------------------------------------------------------------------------------
/coursera/test/fixtures/html/parsing-malsoftware-001-with-bs4.html:
--------------------------------------------------------------------------------
   1 | 
   2 | 
   3 | <!DOCTYPE html>
   4 | <html lang="en-US">
   5 | <head>
   6 | <meta charset="utf-8" />
   7 | <title> | Malicious Software and its Underground Economy: Two Sides to Every Story</title>
   8 | <link rel="stylesheet" href="https://dt5zaw6a98blc.cloudfront.net/site-static/d1190b5daf5c2f7b9e72f105a4951386409e0be1/css/spark.main.css" />
   9 | <link rel="stylesheet" href="https://www.coursera.org/maestro/api/course/970255/course.css">
  10 | <link rel="stylesheet" href="https://dt5zaw6a98blc.cloudfront.net/site-static/d1190b5daf5c2f7b9e72f105a4951386409e0be1/bundles/jquery-ui/css/smoothness/jquery-ui.css" />
  11 | <link rel="icon" href="https://spark-public.s3.amazonaws.com/malsoftware/static/images/favicon.ico" />
  12 | </head>
  13 | 
  14 | <body>
  15 |     
  16 |     <a href="#course-page-content" class="hidden">Skip Navigation</a>
  17 |     <div class="hidden">
  18 |         This page features MathJax technology to render mathematical formulae.
  19 |         If you are using a screen reader, please visit <a href="http://www.dessci.com/en/products/mathplayer/">MathPlayer</a> to download the plugin for your browser. Please note that this is an Internet Explorer-only plugin at this time.
  20 |     </div>
  21 | 
  22 |     <h1 class="hidden">Malicious Software and its Underground Economy: Two Sides to Every Story</h1>
  23 | 
  24 |     
  25 | 
  26 | <div class="course-topbar container-fluid" role="banner">
  27 | 
  28 |     <div class="row-fluid">
  29 |         
  30 |         <div class="span3">
  31 |           <a href="https://www.coursera.org/" class="course-topbar-logo" data-if-linkable="add-referrer"></a>
  32 |         </div>
  33 | 
  34 |         <div class="course-topbar-nav-container span9" style="float:right;">
  35 |             <h2 class="hidden">Top Navigation Bar</h2>
  36 | 
  37 |             <ul class="course-topbar-nav-list">
  38 |                 <li class="course-topbar-nav-list-item">
  39 |                   <a href="https://www.coursera.org/courses" data-if-linkable="add-referrer">Courses</a>
  40 |                 </li>
  41 | 
  42 |                                 
  43 |                 <li class="course-topbar-nav-list-item"
  44 |                     tabindex="0" role="button" aria-haspopup="true" aria-expanded="false" aria-owns="course-topbar-my"
  45 |                     data-popup="#course-topbar-my" data-popup-bind-open="mouseenter" data-popup-close style="cursor:pointer;" data-popup-direction="se">
  46 |                                         <a data-user-id="1576064">  
  47 |                         Daniël W. Crompton 
  48 |                         <i class="icon-caret-down"></i>
  49 |                     </a>
  50 |                 </li>
  51 |                
  52 |                             </ul>
  53 |         </div>
  54 |     </div>
  55 | 
  56 |     <div id="course-topbar-aboutus" class="course-topbar-sublist">
  57 |         <a class="course-topbar-sublist-item" href="https://www.coursera.org/about/jobs" target="_new" data-if-linkable="add-referrer">Jobs</a>
  58 |         <a class="course-topbar-sublist-item" href="https://www.coursera.org/about/team" target="_new" data-if-linkable="add-referrer">Team</a>
  59 |         <a class="course-topbar-sublist-item" href="https://www.coursera.org/about/contact" target="_new" data-if-linkable="add-referrer">Contact Us</a>
  60 |         <a class="course-topbar-sublist-item" href="https://www.coursera.org/about/" target="_new" data-if-linkable="add-referrer">About Us</a>
  61 |     </div>
  62 | 
  63 |     <div id="course-topbar-my" class="course-topbar-sublist">
  64 |         <a class="course-topbar-sublist-item" href="https://www.coursera.org/" target="_new">Courses</a>
  65 |         <a class="course-topbar-sublist-item" href="https://www.coursera.org/account/profile" target="_new">Profile</a>
  66 |         <a class="course-topbar-sublist-item" href="https://www.coursera.org/account/records" target="_new">Course Records</a>
  67 |         <a class="course-topbar-sublist-item" href="https://www.coursera.org/account/settings" target="_new">Settings</a>
  68 |         <a class="course-topbar-sublist-item" href="https://class.coursera.org/malsoftware-001/class/preferences">Course Preferences</a>
  69 |         <a class="course-topbar-sublist-item" href="https://class.coursera.org/malsoftware-001/auth/logout">Sign Out</a>
  70 |     </div>
  71 | 
  72 | </div>
  73 | 
  74 | 
  75 | 
  76 | 
  77 | 
  78 | 
  79 | 
  80 | 
  81 | <style type="text/css">
  82 |  
  83 | 
  84 | 
  85 | </style>
  86 | <div class="course-topbanner">
  87 |   <div class="course-topbanner-university-logo">
  88 |     <a href="https://www.coursera.org/london" class="coursera-university-color" target="_blank" data-if-linkable="add-referrer" title="https://www.coursera.org/london">
  89 |       <img src="https://www.coursera.org/maestro/api/course/970255/university_logo" alt="london">
  90 |     </a>
  91 |   </div>
  92 | 
  93 |               <div class="course-signaturetrack-topbanner-side-join">
  94 |           <div class='course-signaturetrack-bannerbox'>
  95 |               <div class='course-signaturetrack-bannerbox-join'>
  96 |                   <a target='_new' href='https://www.coursera.org/signature/course/malsoftware/970255'>Join Signature Track</a>
  97 |               </div>
  98 |               <div class='course-signaturetrack-bannerbox-daysleft'>8 hours and 6 minutes left!</div>
  99 |           </div>
 100 |       </div>
 101 |   
 102 |   <h1 class="course-topbanner-header">
 103 |       <a class="course-topbanner-name coursera-university-color" href="https://class.coursera.org/malsoftware-001/class/index" data-if-linkable="modal-lock">
 104 |         Malicious Software and its Underground Economy: Two Sides to Every Story      </a>
 105 |       <br>
 106 |       <span class="course-topbanner-instructor" style="">
 107 |         by Lorenzo  Cavallaro      </span>
 108 |   </h1>
 109 | </div>
 110 |     
 111 |         <div class="container-fluid" style="position:relative;">
 112 |         <div class="row-fluid">
 113 |             <div id="course-page-sidebar" role="navigation">
 114 |                 <div class="course-navbar-container">
 115 | 	<a href="https://class.coursera.org/malsoftware-001/class/index"
 116 |       >
 117 | 	<img src="https://www.coursera.org//maestro/api/course/970255/logo" style="width: 210px;height: 112px;margin: -15px 0px 8px 1px; box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.35);" alt="Course Home Page">
 118 | 	</a>
 119 | <h2 class="hidden">Side Navigation Bar</h2>
 120 | <ul class="course-navbar-list">
 121 | 		
 122 | 	<li class="course-navbar-item">
 123 | 								<a href="https://class.coursera.org/malsoftware-001/class/index"  data-if-linkable="modal-lock,gray,lock-icon" class="coursera-university-color">
 124 | 				
 125 | 				Home							</a>
 126 | 						</li>
 127 | 	
 128 | 		
 129 | 	<li class="course-navbar-item">
 130 | 					<h3></h3>
 131 | 			</li>
 132 | 	
 133 | 		
 134 | 	<li class="course-navbar-item">
 135 | 								<a href="https://class.coursera.org/malsoftware-001/lecture/index"  data-if-linkable="modal-lock,gray,lock-icon" class="coursera-university-color">
 136 | 				
 137 | 				Video Lectures							</a>
 138 | 						</li>
 139 | 	
 140 | 		
 141 | 	<li class="course-navbar-item">
 142 | 								<a href="https://class.coursera.org/malsoftware-001/wiki/view?page=recommendedreadings"  data-if-linkable="modal-lock,gray,lock-icon" class="coursera-university-color">
 143 | 				
 144 | 				Recommended Readings							</a>
 145 | 						</li>
 146 | 	
 147 | 		
 148 | 	<li class="course-navbar-item">
 149 | 								<a href="https://class.coursera.org/malsoftware-001/quiz/index"  data-if-linkable="modal-lock,gray,lock-icon" class="coursera-university-color">
 150 | 				
 151 | 				Quizzes							</a>
 152 | 						</li>
 153 | 	
 154 | 		
 155 | 	<li class="course-navbar-item">
 156 | 								<a href="https://class.coursera.org/malsoftware-001/forum/index"  data-if-linkable="modal-lock,gray,lock-icon" class="coursera-university-color">
 157 | 				
 158 | 				Discussion Forums							</a>
 159 | 						</li>
 160 | 	
 161 | 		
 162 | 	<li class="course-navbar-item">
 163 | 					<h3></h3>
 164 | 			</li>
 165 | 	
 166 | 		
 167 | 	<li class="course-navbar-item">
 168 | 								<a href="https://class.coursera.org/malsoftware-001/wiki/view?page=courselogistics"  data-if-linkable="modal-lock,gray,lock-icon" class="coursera-university-color">
 169 | 				
 170 | 				Course Logistics							</a>
 171 | 						</li>
 172 | 	
 173 | 		
 174 | 	<li class="course-navbar-item">
 175 | 								<a href="https://class.coursera.org/malsoftware-001/wiki/view?page=syllabus"  data-if-linkable="modal-lock,gray,lock-icon" class="coursera-university-color">
 176 | 				
 177 | 				Learning Outcomes and Syllabus							</a>
 178 | 						</li>
 179 | 	
 180 | 		
 181 | 	<li class="course-navbar-item">
 182 | 					<h3></h3>
 183 | 			</li>
 184 | 	
 185 | 		
 186 | 	<li class="course-navbar-item">
 187 | 								<a href="https://class.coursera.org/malsoftware-001/quiz/index?quiz_type=survey"  data-if-linkable="modal-lock,gray,lock-icon" class="coursera-university-color">
 188 | 				
 189 | 				Surveys 							</a>
 190 | 						</li>
 191 | 	
 192 | 		
 193 | 	<li class="course-navbar-item">
 194 | 								<a href="https://class.coursera.org/malsoftware-001/wiki/view?page=aboutus"  data-if-linkable="modal-lock,gray,lock-icon" class="coursera-university-color">
 195 | 				
 196 | 				Course Staff							</a>
 197 | 						</li>
 198 | 	
 199 | 	
 200 | 		
 201 | 	
 202 |     </ul>
 203 | 
 204 | <ul style="list-style-type:none;margin-left: 10px;">
 205 |   <li><a href="http://help.coursera.org" class="coursera-university-color" target="_blank" title="Open help articles in new window"><i class="icon-question-sign"></i> Help Articles</a></li>
 206 | </ul>
 207 | 
 208 | <ul style="list-style-type:none;margin-left: 10px;">
 209 | 	
 210 | </ul>
 211 | 
 212 | </div>
 213 |             </div>
 214 |             <div id="course-page-content" role="main">
 215 |                                 
 216 | <div class="course-lectures-list">
 217 |   <h2 class="course-page-header">Video Lectures
 218 |     <a class="coursera-reporter-link" target="_blank" href="https://class.coursera.org/malsoftware-001/class/reporter?url=http%3A%2F%2Fclass.coursera.org%2Fmalsoftware-001%2Flecture%2Findex&area=video&title=Video+Lectures">
 219 |      Report a problem
 220 |     </a>
 221 | 
 222 |   </h2>
 223 | 
 224 |   <a data-coursera-admin-helpwidget-link rel="help" href="http://support.coursera.org/customer/portal/articles/712646-adding-resources-and-links" title="Adding resources and links"></a>
 225 |   <a data-coursera-admin-helpwidget-link rel="help" href="http://support.coursera.org/customer/portal/articles/749155-creating-a-lecture-on-the-course-site" title="Creating a lecture"></a>
 226 |   <a data-coursera-admin-helpwidget-link rel="discuss" href='https://class.coursera.org/mooc/forum/tag?name=LectureVideos&forum_id=7'></a>
 227 |   
 228 |   <div><!-- Lecture List Preamble Here --></div>
 229 | 
 230 |   
 231 |   
 232 |   <p style="margin-bottom:20px">
 233 |     Having trouble viewing lectures?
 234 |     Try changing players. Your current player format is html5. <a href="https://class.coursera.org/malsoftware-001/lecture/index?lecture_player=flash">Change to flash</a>.
 235 |   <p>
 236 | 
 237 |   <div class="course-item-list">
 238 | <div class="course-item-list-header contracted"><h3><span class="icon-chevron-right" style="width:18px;display:inline-block;"></span> &nbsp;Week 1 — Introduction</h3> <span class="hidden">(collapsed, click to expand)</span></div><ul class="course-item-list-section-list"><li class="viewed"><span class="icon-ok"></span><span class="hidden">Completed</span><a data-lecture-id="53"
 239 |    data-modal-iframe="https://class.coursera.org/malsoftware-001/lecture/view?lecture_id=53"
 240 |    href="https://class.coursera.org/malsoftware-001/lecture/53"
 241 |    data-modal=".course-modal-frame"
 242 |    rel="lecture-link"
 243 |    class="lecture-link">
 244 | Course Teaser — Introduction (01:11)</a>
 245 | <div class="course-lecture-item-resource">
 246 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/subtitles?q=53_en&format=txt"
 247 |            data-if-linkable="modal-lock"
 248 |            data-placement="top" rel="tooltip" 
 249 |            title="Subtitles (text)" >
 250 |            <i class="icon-align-justify resource" ></i>
 251 |            <div class="hidden">Subtitles (text) for Course Teaser — Introduction (01:11)</div>
 252 |         </a>
 253 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/subtitles?q=53_en&format=srt"
 254 |            data-if-linkable="modal-lock"
 255 |            data-placement="top" rel="tooltip" 
 256 |            title="Subtitles (srt)" >
 257 |            <i class="icon-list resource" ></i>
 258 |            <div class="hidden">Subtitles (srt) for Course Teaser — Introduction (01:11)</div>
 259 |         </a>
 260 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/download.mp4?lecture_id=53"
 261 |            data-if-linkable="modal-lock"
 262 |            data-placement="top" rel="tooltip" 
 263 |            title="Video (MP4)" >
 264 |            <i class="icon-download-alt resource" ></i>
 265 |            <div class="hidden">Video (MP4) for Course Teaser — Introduction (01:11)</div>
 266 |         </a>
 267 | </div>
 268 | </li><li class="viewed"><span class="icon-ok"></span><span class="hidden">Completed</span><a data-lecture-id="3"
 269 |    data-modal-iframe="https://class.coursera.org/malsoftware-001/lecture/view?lecture_id=3"
 270 |    href="https://class.coursera.org/malsoftware-001/lecture/3"
 271 |    data-modal=".course-modal-frame"
 272 |    rel="lecture-link"
 273 |    class="lecture-link">
 274 | (Should we care? A botnet takeover storytelling) (11:39)</a>
 275 | <div class="course-lecture-item-resource">
 276 |         <a target="_new" href="https://d396qusza40orc.cloudfront.net/malsoftware/w1%2Fw1-1_intro.pdf"
 277 |            data-if-linkable="modal-lock"
 278 |            data-placement="top" rel="tooltip" 
 279 |            title="Introduction" >
 280 |            <i class="icon-file resource" ></i>
 281 |            <div class="hidden">Introduction for (Should we care? A botnet takeover storytelling) (11:39)</div>
 282 |         </a>
 283 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/subtitles?q=3_en&format=txt"
 284 |            data-if-linkable="modal-lock"
 285 |            data-placement="top" rel="tooltip" 
 286 |            title="Subtitles (text)" >
 287 |            <i class="icon-align-justify resource" ></i>
 288 |            <div class="hidden">Subtitles (text) for (Should we care? A botnet takeover storytelling) (11:39)</div>
 289 |         </a>
 290 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/subtitles?q=3_en&format=srt"
 291 |            data-if-linkable="modal-lock"
 292 |            data-placement="top" rel="tooltip" 
 293 |            title="Subtitles (srt)" >
 294 |            <i class="icon-list resource" ></i>
 295 |            <div class="hidden">Subtitles (srt) for (Should we care? A botnet takeover storytelling) (11:39)</div>
 296 |         </a>
 297 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/download.mp4?lecture_id=3"
 298 |            data-if-linkable="modal-lock"
 299 |            data-placement="top" rel="tooltip" 
 300 |            title="Video (MP4)" >
 301 |            <i class="icon-download-alt resource" ></i>
 302 |            <div class="hidden">Video (MP4) for (Should we care? A botnet takeover storytelling) (11:39)</div>
 303 |         </a>
 304 | </div>
 305 | </li><li class="viewed"><span class="icon-ok"></span><span class="hidden">Completed</span><a data-lecture-id="25"
 306 |    data-modal-iframe="https://class.coursera.org/malsoftware-001/lecture/view?lecture_id=25"
 307 |    href="https://class.coursera.org/malsoftware-001/lecture/25"
 308 |    data-modal=".course-modal-frame"
 309 |    rel="lecture-link"
 310 |    class="lecture-link">
 311 | Admin blabbing (11:38)</a>
 312 | <div class="course-lecture-item-resource">
 313 |         <a target="_new" href="https://d396qusza40orc.cloudfront.net/malsoftware/w1%2Fw1-2_admin.pdf"
 314 |            data-if-linkable="modal-lock"
 315 |            data-placement="top" rel="tooltip" 
 316 |            title="Admin blabbing" >
 317 |            <i class="icon-file resource" ></i>
 318 |            <div class="hidden">Admin blabbing for Admin blabbing (11:38)</div>
 319 |         </a>
 320 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/subtitles?q=25_en&format=txt"
 321 |            data-if-linkable="modal-lock"
 322 |            data-placement="top" rel="tooltip" 
 323 |            title="Subtitles (text)" >
 324 |            <i class="icon-align-justify resource" ></i>
 325 |            <div class="hidden">Subtitles (text) for Admin blabbing (11:38)</div>
 326 |         </a>
 327 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/subtitles?q=25_en&format=srt"
 328 |            data-if-linkable="modal-lock"
 329 |            data-placement="top" rel="tooltip" 
 330 |            title="Subtitles (srt)" >
 331 |            <i class="icon-list resource" ></i>
 332 |            <div class="hidden">Subtitles (srt) for Admin blabbing (11:38)</div>
 333 |         </a>
 334 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/download.mp4?lecture_id=25"
 335 |            data-if-linkable="modal-lock"
 336 |            data-placement="top" rel="tooltip" 
 337 |            title="Video (MP4)" >
 338 |            <i class="icon-download-alt resource" ></i>
 339 |            <div class="hidden">Video (MP4) for Admin blabbing (11:38)</div>
 340 |         </a>
 341 | </div>
 342 | </li><li class="viewed"><span class="icon-ok"></span><span class="hidden">Completed</span><a data-lecture-id="27"
 343 |    data-modal-iframe="https://class.coursera.org/malsoftware-001/lecture/view?lecture_id=27"
 344 |    href="https://class.coursera.org/malsoftware-001/lecture/27"
 345 |    data-modal=".course-modal-frame"
 346 |    rel="lecture-link"
 347 |    class="lecture-link">
 348 | Malicious Software (20:09)</a>
 349 | <div class="course-lecture-item-resource">
 350 |         <a target="_new" href="https://d396qusza40orc.cloudfront.net/malsoftware/w1%2Fw1-3_malware.pdf"
 351 |            data-if-linkable="modal-lock"
 352 |            data-placement="top" rel="tooltip" 
 353 |            title="Malicious Software" >
 354 |            <i class="icon-file resource" ></i>
 355 |            <div class="hidden">Malicious Software for Malicious Software (20:09)</div>
 356 |         </a>
 357 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/subtitles?q=27_en&format=txt"
 358 |            data-if-linkable="modal-lock"
 359 |            data-placement="top" rel="tooltip" 
 360 |            title="Subtitles (text)" >
 361 |            <i class="icon-align-justify resource" ></i>
 362 |            <div class="hidden">Subtitles (text) for Malicious Software (20:09)</div>
 363 |         </a>
 364 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/subtitles?q=27_en&format=srt"
 365 |            data-if-linkable="modal-lock"
 366 |            data-placement="top" rel="tooltip" 
 367 |            title="Subtitles (srt)" >
 368 |            <i class="icon-list resource" ></i>
 369 |            <div class="hidden">Subtitles (srt) for Malicious Software (20:09)</div>
 370 |         </a>
 371 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/download.mp4?lecture_id=27"
 372 |            data-if-linkable="modal-lock"
 373 |            data-placement="top" rel="tooltip" 
 374 |            title="Video (MP4)" >
 375 |            <i class="icon-download-alt resource" ></i>
 376 |            <div class="hidden">Video (MP4) for Malicious Software (20:09)</div>
 377 |         </a>
 378 | </div>
 379 | </li><li class="viewed"><span class="icon-ok"></span><span class="hidden">Completed</span><a data-lecture-id="29"
 380 |    data-modal-iframe="https://class.coursera.org/malsoftware-001/lecture/view?lecture_id=29"
 381 |    href="https://class.coursera.org/malsoftware-001/lecture/29"
 382 |    data-modal=".course-modal-frame"
 383 |    rel="lecture-link"
 384 |    class="lecture-link">
 385 | (a glimpse at) Botnets (26:09)</a>
 386 | <div class="course-lecture-item-resource">
 387 |         <a target="_new" href="https://d396qusza40orc.cloudfront.net/malsoftware/w1%2Fw1-4_botnet.pdf"
 388 |            data-if-linkable="modal-lock"
 389 |            data-placement="top" rel="tooltip" 
 390 |            title="(a glimpse at) Botnets" >
 391 |            <i class="icon-file resource" ></i>
 392 |            <div class="hidden">(a glimpse at) Botnets for (a glimpse at) Botnets (26:09)</div>
 393 |         </a>
 394 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/subtitles?q=29_en&format=txt"
 395 |            data-if-linkable="modal-lock"
 396 |            data-placement="top" rel="tooltip" 
 397 |            title="Subtitles (text)" >
 398 |            <i class="icon-align-justify resource" ></i>
 399 |            <div class="hidden">Subtitles (text) for (a glimpse at) Botnets (26:09)</div>
 400 |         </a>
 401 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/subtitles?q=29_en&format=srt"
 402 |            data-if-linkable="modal-lock"
 403 |            data-placement="top" rel="tooltip" 
 404 |            title="Subtitles (srt)" >
 405 |            <i class="icon-list resource" ></i>
 406 |            <div class="hidden">Subtitles (srt) for (a glimpse at) Botnets (26:09)</div>
 407 |         </a>
 408 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/download.mp4?lecture_id=29"
 409 |            data-if-linkable="modal-lock"
 410 |            data-placement="top" rel="tooltip" 
 411 |            title="Video (MP4)" >
 412 |            <i class="icon-download-alt resource" ></i>
 413 |            <div class="hidden">Video (MP4) for (a glimpse at) Botnets (26:09)</div>
 414 |         </a>
 415 | </div>
 416 | </li><li class="viewed"><span class="icon-ok"></span><span class="hidden">Completed</span><a data-lecture-id="31"
 417 |    data-modal-iframe="https://class.coursera.org/malsoftware-001/lecture/view?lecture_id=31"
 418 |    href="https://class.coursera.org/malsoftware-001/lecture/31"
 419 |    data-modal=".course-modal-frame"
 420 |    rel="lecture-link"
 421 |    class="lecture-link">
 422 | (a glimpse at) Botnets detection and Rootkits (42:42)</a>
 423 | <div class="course-lecture-item-resource">
 424 |         <a target="_new" href="https://d396qusza40orc.cloudfront.net/malsoftware/w1%2Fw1-5_rootkit.pdf"
 425 |            data-if-linkable="modal-lock"
 426 |            data-placement="top" rel="tooltip" 
 427 |            title="(a glimpse at) Botnets detection and Rookits" >
 428 |            <i class="icon-file resource" ></i>
 429 |            <div class="hidden">(a glimpse at) Botnets detection and Rookits for (a glimpse at) Botnets detection and Rootkits (42:42)</div>
 430 |         </a>
 431 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/subtitles?q=31_en&format=txt"
 432 |            data-if-linkable="modal-lock"
 433 |            data-placement="top" rel="tooltip" 
 434 |            title="Subtitles (text)" >
 435 |            <i class="icon-align-justify resource" ></i>
 436 |            <div class="hidden">Subtitles (text) for (a glimpse at) Botnets detection and Rootkits (42:42)</div>
 437 |         </a>
 438 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/subtitles?q=31_en&format=srt"
 439 |            data-if-linkable="modal-lock"
 440 |            data-placement="top" rel="tooltip" 
 441 |            title="Subtitles (srt)" >
 442 |            <i class="icon-list resource" ></i>
 443 |            <div class="hidden">Subtitles (srt) for (a glimpse at) Botnets detection and Rootkits (42:42)</div>
 444 |         </a>
 445 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/download.mp4?lecture_id=31"
 446 |            data-if-linkable="modal-lock"
 447 |            data-placement="top" rel="tooltip" 
 448 |            title="Video (MP4)" >
 449 |            <i class="icon-download-alt resource" ></i>
 450 |            <div class="hidden">Video (MP4) for (a glimpse at) Botnets detection and Rootkits (42:42)</div>
 451 |         </a>
 452 | </div>
 453 | </li></ul><div class="course-item-list-header contracted"><h3><span class="icon-chevron-right" style="width:18px;display:inline-block;"></span> &nbsp;Week 2 — Static analysis and its limitations</h3> <span class="hidden">(collapsed, click to expand)</span></div><ul class="course-item-list-section-list"><li class="viewed"><span class="icon-ok"></span><span class="hidden">Completed</span><a data-lecture-id="5"
 454 |    data-modal-iframe="https://class.coursera.org/malsoftware-001/lecture/view?lecture_id=5"
 455 |    href="https://class.coursera.org/malsoftware-001/lecture/5"
 456 |    data-modal=".course-modal-frame"
 457 |    rel="lecture-link"
 458 |    class="lecture-link">
 459 | Early days, AV industry (15:36)</a>
 460 | <div class="course-lecture-item-resource">
 461 |         <a target="_new" href="https://d396qusza40orc.cloudfront.net/malsoftware/w2%2Fw2-1_earlydays.pdf"
 462 |            data-if-linkable="modal-lock"
 463 |            data-placement="top" rel="tooltip" 
 464 |            title="Early days, AV industry" >
 465 |            <i class="icon-file resource" ></i>
 466 |            <div class="hidden">Early days, AV industry for Early days, AV industry (15:36)</div>
 467 |         </a>
 468 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/subtitles?q=5_en&format=txt"
 469 |            data-if-linkable="modal-lock"
 470 |            data-placement="top" rel="tooltip" 
 471 |            title="Subtitles (text)" >
 472 |            <i class="icon-align-justify resource" ></i>
 473 |            <div class="hidden">Subtitles (text) for Early days, AV industry (15:36)</div>
 474 |         </a>
 475 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/subtitles?q=5_en&format=srt"
 476 |            data-if-linkable="modal-lock"
 477 |            data-placement="top" rel="tooltip" 
 478 |            title="Subtitles (srt)" >
 479 |            <i class="icon-list resource" ></i>
 480 |            <div class="hidden">Subtitles (srt) for Early days, AV industry (15:36)</div>
 481 |         </a>
 482 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/download.mp4?lecture_id=5"
 483 |            data-if-linkable="modal-lock"
 484 |            data-placement="top" rel="tooltip" 
 485 |            title="Video (MP4)" >
 486 |            <i class="icon-download-alt resource" ></i>
 487 |            <div class="hidden">Video (MP4) for Early days, AV industry (15:36)</div>
 488 |         </a>
 489 | </div>
 490 | </li><li class="viewed"><span class="icon-ok"></span><span class="hidden">Completed</span><a data-lecture-id="33"
 491 |    data-modal-iframe="https://class.coursera.org/malsoftware-001/lecture/view?lecture_id=33"
 492 |    href="https://class.coursera.org/malsoftware-001/lecture/33"
 493 |    data-modal=".course-modal-frame"
 494 |    rel="lecture-link"
 495 |    class="lecture-link">
 496 | (a glimpse at) Reverse engineering (part 1) (30:41)</a>
 497 | <div class="course-lecture-item-resource">
 498 |         <a target="_new" href="https://d396qusza40orc.cloudfront.net/malsoftware/w2%2Fw2-2_rev1.pdf"
 499 |            data-if-linkable="modal-lock"
 500 |            data-placement="top" rel="tooltip" 
 501 |            title="(a glimpse at) Reverse engineering (part 1)" >
 502 |            <i class="icon-file resource" ></i>
 503 |            <div class="hidden">(a glimpse at) Reverse engineering (part 1) for (a glimpse at) Reverse engineering (part 1) (30:41)</div>
 504 |         </a>
 505 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/subtitles?q=33_en&format=txt"
 506 |            data-if-linkable="modal-lock"
 507 |            data-placement="top" rel="tooltip" 
 508 |            title="Subtitles (text)" >
 509 |            <i class="icon-align-justify resource" ></i>
 510 |            <div class="hidden">Subtitles (text) for (a glimpse at) Reverse engineering (part 1) (30:41)</div>
 511 |         </a>
 512 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/subtitles?q=33_en&format=srt"
 513 |            data-if-linkable="modal-lock"
 514 |            data-placement="top" rel="tooltip" 
 515 |            title="Subtitles (srt)" >
 516 |            <i class="icon-list resource" ></i>
 517 |            <div class="hidden">Subtitles (srt) for (a glimpse at) Reverse engineering (part 1) (30:41)</div>
 518 |         </a>
 519 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/download.mp4?lecture_id=33"
 520 |            data-if-linkable="modal-lock"
 521 |            data-placement="top" rel="tooltip" 
 522 |            title="Video (MP4)" >
 523 |            <i class="icon-download-alt resource" ></i>
 524 |            <div class="hidden">Video (MP4) for (a glimpse at) Reverse engineering (part 1) (30:41)</div>
 525 |         </a>
 526 | </div>
 527 | </li><li class="viewed"><span class="icon-ok"></span><span class="hidden">Completed</span><a data-lecture-id="35"
 528 |    data-modal-iframe="https://class.coursera.org/malsoftware-001/lecture/view?lecture_id=35"
 529 |    href="https://class.coursera.org/malsoftware-001/lecture/35"
 530 |    data-modal=".course-modal-frame"
 531 |    rel="lecture-link"
 532 |    class="lecture-link">
 533 | (a glimpse at) Reverse engineering (part 2) (09:06)</a>
 534 | <div class="course-lecture-item-resource">
 535 |         <a target="_new" href="https://d396qusza40orc.cloudfront.net/malsoftware/w2%2Fw2-3_rev2.pdf"
 536 |            data-if-linkable="modal-lock"
 537 |            data-placement="top" rel="tooltip" 
 538 |            title="(a glimpse at) Reverse engineering (part 2)" >
 539 |            <i class="icon-file resource" ></i>
 540 |            <div class="hidden">(a glimpse at) Reverse engineering (part 2) for (a glimpse at) Reverse engineering (part 2) (09:06)</div>
 541 |         </a>
 542 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/subtitles?q=35_en&format=txt"
 543 |            data-if-linkable="modal-lock"
 544 |            data-placement="top" rel="tooltip" 
 545 |            title="Subtitles (text)" >
 546 |            <i class="icon-align-justify resource" ></i>
 547 |            <div class="hidden">Subtitles (text) for (a glimpse at) Reverse engineering (part 2) (09:06)</div>
 548 |         </a>
 549 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/subtitles?q=35_en&format=srt"
 550 |            data-if-linkable="modal-lock"
 551 |            data-placement="top" rel="tooltip" 
 552 |            title="Subtitles (srt)" >
 553 |            <i class="icon-list resource" ></i>
 554 |            <div class="hidden">Subtitles (srt) for (a glimpse at) Reverse engineering (part 2) (09:06)</div>
 555 |         </a>
 556 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/download.mp4?lecture_id=35"
 557 |            data-if-linkable="modal-lock"
 558 |            data-placement="top" rel="tooltip" 
 559 |            title="Video (MP4)" >
 560 |            <i class="icon-download-alt resource" ></i>
 561 |            <div class="hidden">Video (MP4) for (a glimpse at) Reverse engineering (part 2) (09:06)</div>
 562 |         </a>
 563 | </div>
 564 | </li><li class="viewed"><span class="icon-ok"></span><span class="hidden">Completed</span><a data-lecture-id="37"
 565 |    data-modal-iframe="https://class.coursera.org/malsoftware-001/lecture/view?lecture_id=37"
 566 |    href="https://class.coursera.org/malsoftware-001/lecture/37"
 567 |    data-modal=".course-modal-frame"
 568 |    rel="lecture-link"
 569 |    class="lecture-link">
 570 | (a glimpse at) Polymorphism, code obfuscation (18:23)</a>
 571 | <div class="course-lecture-item-resource">
 572 |         <a target="_new" href="https://d396qusza40orc.cloudfront.net/malsoftware/w2%2Fw2-4_polymorphism.pdf"
 573 |            data-if-linkable="modal-lock"
 574 |            data-placement="top" rel="tooltip" 
 575 |            title="(a glimpse at) Polymorphism, code obfuscation" >
 576 |            <i class="icon-file resource" ></i>
 577 |            <div class="hidden">(a glimpse at) Polymorphism, code obfuscation for (a glimpse at) Polymorphism, code obfuscation (18:23)</div>
 578 |         </a>
 579 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/subtitles?q=37_en&format=txt"
 580 |            data-if-linkable="modal-lock"
 581 |            data-placement="top" rel="tooltip" 
 582 |            title="Subtitles (text)" >
 583 |            <i class="icon-align-justify resource" ></i>
 584 |            <div class="hidden">Subtitles (text) for (a glimpse at) Polymorphism, code obfuscation (18:23)</div>
 585 |         </a>
 586 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/subtitles?q=37_en&format=srt"
 587 |            data-if-linkable="modal-lock"
 588 |            data-placement="top" rel="tooltip" 
 589 |            title="Subtitles (srt)" >
 590 |            <i class="icon-list resource" ></i>
 591 |            <div class="hidden">Subtitles (srt) for (a glimpse at) Polymorphism, code obfuscation (18:23)</div>
 592 |         </a>
 593 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/download.mp4?lecture_id=37"
 594 |            data-if-linkable="modal-lock"
 595 |            data-placement="top" rel="tooltip" 
 596 |            title="Video (MP4)" >
 597 |            <i class="icon-download-alt resource" ></i>
 598 |            <div class="hidden">Video (MP4) for (a glimpse at) Polymorphism, code obfuscation (18:23)</div>
 599 |         </a>
 600 | </div>
 601 | </li><li class="viewed"><span class="icon-ok"></span><span class="hidden">Completed</span><a data-lecture-id="39"
 602 |    data-modal-iframe="https://class.coursera.org/malsoftware-001/lecture/view?lecture_id=39"
 603 |    href="https://class.coursera.org/malsoftware-001/lecture/39"
 604 |    data-modal=".course-modal-frame"
 605 |    rel="lecture-link"
 606 |    class="lecture-link">
 607 | IDA Pro—a very simple example. . . (24:17)</a>
 608 | <div class="course-lecture-item-resource">
 609 |         <a target="_new" href="https://d396qusza40orc.cloudfront.net/malsoftware/w2%2Fw2-5_ida.pdf"
 610 |            data-if-linkable="modal-lock"
 611 |            data-placement="top" rel="tooltip" 
 612 |            title="IDA Pro—a very simple example. . ." >
 613 |            <i class="icon-file resource" ></i>
 614 |            <div class="hidden">IDA Pro—a very simple example. . . for IDA Pro—a very simple example. . . (24:17)</div>
 615 |         </a>
 616 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/subtitles?q=39_en&format=txt"
 617 |            data-if-linkable="modal-lock"
 618 |            data-placement="top" rel="tooltip" 
 619 |            title="Subtitles (text)" >
 620 |            <i class="icon-align-justify resource" ></i>
 621 |            <div class="hidden">Subtitles (text) for IDA Pro—a very simple example. . . (24:17)</div>
 622 |         </a>
 623 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/subtitles?q=39_en&format=srt"
 624 |            data-if-linkable="modal-lock"
 625 |            data-placement="top" rel="tooltip" 
 626 |            title="Subtitles (srt)" >
 627 |            <i class="icon-list resource" ></i>
 628 |            <div class="hidden">Subtitles (srt) for IDA Pro—a very simple example. . . (24:17)</div>
 629 |         </a>
 630 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/download.mp4?lecture_id=39"
 631 |            data-if-linkable="modal-lock"
 632 |            data-placement="top" rel="tooltip" 
 633 |            title="Video (MP4)" >
 634 |            <i class="icon-download-alt resource" ></i>
 635 |            <div class="hidden">Video (MP4) for IDA Pro—a very simple example. . . (24:17)</div>
 636 |         </a>
 637 | </div>
 638 | </li><li class="viewed"><span class="icon-ok"></span><span class="hidden">Completed</span><a data-lecture-id="67"
 639 |    data-modal-iframe="https://class.coursera.org/malsoftware-001/lecture/view?lecture_id=67"
 640 |    href="https://class.coursera.org/malsoftware-001/lecture/67"
 641 |    data-modal=".course-modal-frame"
 642 |    rel="lecture-link"
 643 |    class="lecture-link">
 644 | reverse-ex — the binary (to download)</a>
 645 | <div class="course-lecture-item-resource">
 646 |         <a target="_new" href="https://d396qusza40orc.cloudfront.net/malsoftware/w2%2Freverse-ex"
 647 |            data-if-linkable="modal-lock"
 648 |            data-placement="top" rel="tooltip" 
 649 |            title="reverse-ex" >
 650 |            <i class="icon-file resource" ></i>
 651 |            <div class="hidden">reverse-ex for reverse-ex — the binary (to download)</div>
 652 |         </a>
 653 | </div>
 654 | </li><li class="viewed"><span class="icon-ok"></span><span class="hidden">Completed</span><a data-lecture-id="69"
 655 |    data-modal-iframe="https://class.coursera.org/malsoftware-001/lecture/view?lecture_id=69"
 656 |    href="https://class.coursera.org/malsoftware-001/lecture/69"
 657 |    data-modal=".course-modal-frame"
 658 |    rel="lecture-link"
 659 |    class="lecture-link">
 660 | reverse-ex.c — the C source (to download)</a>
 661 | <div class="course-lecture-item-resource">
 662 |         <a target="_new" href="https://d396qusza40orc.cloudfront.net/malsoftware/w2%2Freverse-ex.c"
 663 |            data-if-linkable="modal-lock"
 664 |            data-placement="top" rel="tooltip" 
 665 |            title="reverse-ex.c" >
 666 |            <i class="icon-file resource" ></i>
 667 |            <div class="hidden">reverse-ex.c for reverse-ex.c — the C source (to download)</div>
 668 |         </a>
 669 | </div>
 670 | </li></ul><div class="course-item-list-header expanded"><h3><span class="icon-chevron-down" style="width:18px;display:inline-block;"></span> &nbsp;Week 3 — Dynamic analysis and its limitations</h3> <span class="hidden">(expanded, click to collapse)</span></div><ul class="course-item-list-section-list"><li class="unviewed"><a data-lecture-id="75"
 671 |    data-modal-iframe="https://class.coursera.org/malsoftware-001/lecture/view?lecture_id=75"
 672 |    href="https://class.coursera.org/malsoftware-001/lecture/75"
 673 |    data-modal=".course-modal-frame"
 674 |    rel="lecture-link"
 675 |    class="lecture-link">
 676 | Toward dynamic analysis (15:05)</a>
 677 | <div class="course-lecture-item-resource">
 678 |         <a target="_new" href="https://d396qusza40orc.cloudfront.net/malsoftware/w3%2Fw3-1_packing.pdf"
 679 |            data-if-linkable="modal-lock"
 680 |            data-placement="top" rel="tooltip" 
 681 |            title="Toward dynamic analysis" >
 682 |            <i class="icon-file resource" ></i>
 683 |            <div class="hidden">Toward dynamic analysis for Toward dynamic analysis (15:05)</div>
 684 |         </a>
 685 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/download.mp4?lecture_id=75"
 686 |            data-if-linkable="modal-lock"
 687 |            data-placement="top" rel="tooltip" 
 688 |            title="Video (MP4)" >
 689 |            <i class="icon-download-alt resource" ></i>
 690 |            <div class="hidden">Video (MP4) for Toward dynamic analysis (15:05)</div>
 691 |         </a>
 692 | </div>
 693 | </li><li class="unviewed"><a data-lecture-id="9"
 694 |    data-modal-iframe="https://class.coursera.org/malsoftware-001/lecture/view?lecture_id=9"
 695 |    href="https://class.coursera.org/malsoftware-001/lecture/9"
 696 |    data-modal=".course-modal-frame"
 697 |    rel="lecture-link"
 698 |    class="lecture-link">
 699 | (a glimpse at) Dynamic analysis (part 1) (25:59)</a>
 700 | <div class="course-lecture-item-resource">
 701 |         <a target="_new" href="https://d396qusza40orc.cloudfront.net/malsoftware/w3%2Fw3-2_behavior1.pdf"
 702 |            data-if-linkable="modal-lock"
 703 |            data-placement="top" rel="tooltip" 
 704 |            title="(a glimpse at) Dynamic analysis (part 1)" >
 705 |            <i class="icon-file resource" ></i>
 706 |            <div class="hidden">(a glimpse at) Dynamic analysis (part 1) for (a glimpse at) Dynamic analysis (part 1) (25:59)</div>
 707 |         </a>
 708 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/download.mp4?lecture_id=9"
 709 |            data-if-linkable="modal-lock"
 710 |            data-placement="top" rel="tooltip" 
 711 |            title="Video (MP4)" >
 712 |            <i class="icon-download-alt resource" ></i>
 713 |            <div class="hidden">Video (MP4) for (a glimpse at) Dynamic analysis (part 1) (25:59)</div>
 714 |         </a>
 715 | </div>
 716 | </li><li class="unviewed"><a data-lecture-id="55"
 717 |    data-modal-iframe="https://class.coursera.org/malsoftware-001/lecture/view?lecture_id=55"
 718 |    href="https://class.coursera.org/malsoftware-001/lecture/55"
 719 |    data-modal=".course-modal-frame"
 720 |    rel="lecture-link"
 721 |    class="lecture-link">
 722 | (a glimpse at) Dynamic analysis (part 2) (19:14)</a>
 723 | <div class="course-lecture-item-resource">
 724 |         <a target="_new" href="https://d396qusza40orc.cloudfront.net/malsoftware/w3%2Fw3-2_behavior2.pdf"
 725 |            data-if-linkable="modal-lock"
 726 |            data-placement="top" rel="tooltip" 
 727 |            title="(a glimpse at) Dynamic analysis (part 2)" >
 728 |            <i class="icon-file resource" ></i>
 729 |            <div class="hidden">(a glimpse at) Dynamic analysis (part 2) for (a glimpse at) Dynamic analysis (part 2) (19:14)</div>
 730 |         </a>
 731 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/download.mp4?lecture_id=55"
 732 |            data-if-linkable="modal-lock"
 733 |            data-placement="top" rel="tooltip" 
 734 |            title="Video (MP4)" >
 735 |            <i class="icon-download-alt resource" ></i>
 736 |            <div class="hidden">Video (MP4) for (a glimpse at) Dynamic analysis (part 2) (19:14)</div>
 737 |         </a>
 738 | </div>
 739 | </li><li class="unviewed"><a data-lecture-id="77"
 740 |    data-modal-iframe="https://class.coursera.org/malsoftware-001/lecture/view?lecture_id=77"
 741 |    href="https://class.coursera.org/malsoftware-001/lecture/77"
 742 |    data-modal=".course-modal-frame"
 743 |    rel="lecture-link"
 744 |    class="lecture-link">
 745 | (a glimpse at) Limits of dynamic analysis (11:31)</a>
 746 | <div class="course-lecture-item-resource">
 747 |         <a target="_new" href="https://d396qusza40orc.cloudfront.net/malsoftware/w3%2Fw3-4_limits.pdf"
 748 |            data-if-linkable="modal-lock"
 749 |            data-placement="top" rel="tooltip" 
 750 |            title="(a glimpse at) Limits of dynamic analysis" >
 751 |            <i class="icon-file resource" ></i>
 752 |            <div class="hidden">(a glimpse at) Limits of dynamic analysis for (a glimpse at) Limits of dynamic analysis (11:31)</div>
 753 |         </a>
 754 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/subtitles?q=77_en&format=txt"
 755 |            data-if-linkable="modal-lock"
 756 |            data-placement="top" rel="tooltip" 
 757 |            title="Subtitles (text)" >
 758 |            <i class="icon-align-justify resource" ></i>
 759 |            <div class="hidden">Subtitles (text) for (a glimpse at) Limits of dynamic analysis (11:31)</div>
 760 |         </a>
 761 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/subtitles?q=77_en&format=srt"
 762 |            data-if-linkable="modal-lock"
 763 |            data-placement="top" rel="tooltip" 
 764 |            title="Subtitles (srt)" >
 765 |            <i class="icon-list resource" ></i>
 766 |            <div class="hidden">Subtitles (srt) for (a glimpse at) Limits of dynamic analysis (11:31)</div>
 767 |         </a>
 768 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/download.mp4?lecture_id=77"
 769 |            data-if-linkable="modal-lock"
 770 |            data-placement="top" rel="tooltip" 
 771 |            title="Video (MP4)" >
 772 |            <i class="icon-download-alt resource" ></i>
 773 |            <div class="hidden">Video (MP4) for (a glimpse at) Limits of dynamic analysis (11:31)</div>
 774 |         </a>
 775 | </div>
 776 | </li><li class="unviewed"><a data-lecture-id="59"
 777 |    data-modal-iframe="https://class.coursera.org/malsoftware-001/lecture/view?lecture_id=59"
 778 |    href="https://class.coursera.org/malsoftware-001/lecture/59"
 779 |    data-modal=".course-modal-frame"
 780 |    rel="lecture-link"
 781 |    class="lecture-link">
 782 | AccessMiner — system-centric models (10:38)</a>
 783 | <div class="course-lecture-item-resource">
 784 |         <a target="_new" href="https://d396qusza40orc.cloudfront.net/malsoftware/w3%2Fw3-5_accessminer.pdf"
 785 |            data-if-linkable="modal-lock"
 786 |            data-placement="top" rel="tooltip" 
 787 |            title="AccessMiner — system-centric models" >
 788 |            <i class="icon-file resource" ></i>
 789 |            <div class="hidden">AccessMiner — system-centric models for AccessMiner — system-centric models (10:38)</div>
 790 |         </a>
 791 |         <a target="_new" href="https://class.coursera.org/malsoftware-001/lecture/download.mp4?lecture_id=59"
 792 |            data-if-linkable="modal-lock"
 793 |            data-placement="top" rel="tooltip" 
 794 |            title="Video (MP4)" >
 795 |            <i class="icon-download-alt resource" ></i>
 796 |            <div class="hidden">Video (MP4) for AccessMiner — system-centric models (10:38)</div>
 797 |         </a>
 798 | </div>
 799 | </li></ul></div>
 800 | </div>
 801 | 
 802 | 
 803 | 
 804 | 
 805 | <div class="course-modal-frame " data-modal-overlay-class="course-modal-overlay-dark" data-modal-position="absolute" data-modal-position-top="50px" data-modal-overlay-close="false">
 806 | <div style="position:absolute;top:-50px;height:50px;width:100%;background:#000;font-size:18px;padding-top:20px;padding-bottom:10px;border-bottom:1px solid #999999;">
 807 |     <div style="left:5px;color:white;float:left;padding-left:10px;height:24px;width:800px;overflow:hidden" class="course-modal-frame-title"></div>
 808 | 
 809 |     <div style="float:right;">
 810 |       <a class="coursera-reporter-link course-modal-frame-title-reporter-link" style="color:white;float:none; margin-right:10px;" target="_blank" href="https://class.coursera.org/malsoftware-001/class/reporter?area=video&url=http%3A%2F%2Fclass.coursera.org%2Fmalsoftware-001%2Flecture%2Findex">
 811 |        Report a problem
 812 |       </a>
 813 |       <span class="icon-remove" style="color:white;cursor:pointer;padding-right:10px" data-modal-close></span>
 814 |     </div>
 815 |     <div style="clear:both"></div>
 816 | </div>
 817 | </div>
 818 |             </div>
 819 |         </div>
 820 |     </div>
 821 |         
 822 |     
 823 |     <script src="https://dt5zaw6a98blc.cloudfront.net/site-static/d1190b5daf5c2f7b9e72f105a4951386409e0be1/spark/spark.js"></script>
 824 | 
 825 | <script type="text/javascript">
 826 |     // SET UP THE MULTITRACKER
 827 |         window.logger = window.Log({level:"error"});
 828 |         var parts = location.host.split(".");
 829 |     while(parts.length > 2) { parts.shift(); }
 830 | 
 831 |     window.multitracker = window.MultiTracker({logger:window.logger});
 832 |     window.multitracker.register('204', window._204 = window._204 || []);
 833 |     window.multitracker.register('ga', window._gaq = window._gaq || [], 'google');
 834 | 
 835 |     // SET UP GA and 204
 836 |     window._gaq.push(['_setAccount', 'UA-28377374-1']);
 837 |     window._gaq.push(['_setDomainName', 'coursera.org']);
 838 |     window._gaq.push(['_setAllowLinker', true]);  
 839 |     window._gaq.push(['_trackPageview']);
 840 |     
 841 |     window._204.push({key:"pageview", val:{}});
 842 |     window._204.push(["client", "spark"]);
 843 |     window._204.push(["user", 1576064]);
 844 |     window._204.push(["domain", "." + parts.join(".")]);
 845 | 
 846 |     // override our multitracker queues with the ga queue when it is ready
 847 |     (function() {
 848 |       var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
 849 |       ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
 850 |       var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
 851 |     
 852 |       ga.onload = ga.onreadystatechange = function() {
 853 |           ga.onreadystatechange = ga.onload = null;
 854 |           window.multitracker.get('ga').queue = window._gaq;
 855 |       };
 856 |     })();
 857 | 
 858 |     // override our multitracker queues with the 204 queue when it is ready
 859 |     (function() {
 860 |       var a = document.createElement('script'); a.type = 'text/javascript'; a.async = true;
 861 |       a.src = 'https://eventing.coursera.org/204.min.js';
 862 |       var s = document.getElementsByTagName('script')[0];
 863 |       s.parentNode.insertBefore(a, s);
 864 |       a.onload = a.onreadystatechange = function() {
 865 |           a.onreadystatechange = a.onload = null;
 866 |           window.multitracker.get('204').queue = window._204;
 867 |       };
 868 |     })();
 869 | </script>
 870 | 
 871 | <script>
 872 |   // Set up shared AB testing object
 873 |   (function() {
 874 |     window.ab = {};
 875 | 
 876 |     // we want to share the 204 session cookie with the AB session cookie...
 877 |     var sessionId = AB.makeSession("__204u", {expires:356, domain:"." + location.host});
 878 |         var debug = true;
 879 |     
 880 |     window.ab.user = window.AB(1576064, "user", {debug:debug, tracker:window.multitracker, logger:window.logger});
 881 |     window.ab.session = window.AB(sessionId, "session", {debug:debug, tracker:window.multitracker, logger:window.logger});
 882 | 
 883 |     if(window.CourseraABExperiments.user) {
 884 |         window.ab.user.addExperiments(window.CourseraABExperiments.user);
 885 |         window.ab.user.runExperimentsOn("body");
 886 |     }
 887 | 
 888 |     if(window.CourseraABExperiments.session) {
 889 |         window.ab.session.addExperiments(window.CourseraABExperiments.session);
 890 |         window.ab.session.runExperimentsOn("body");
 891 |     }
 892 |   })();
 893 | </script>
 894 | 
 895 | 
 896 |     <script>
 897 |       // Set up help widget
 898 |       (function() {
 899 |         $("[data-helpwidget]").each(function() {
 900 |             window.HelpWidget(this, {tracker: window.multitracker});
 901 |         });
 902 |        })();
 903 |  
 904 |         $("[data-readme]").each(function() {
 905 |             window.ReadMe(this);
 906 |         });
 907 |     </script>
 908 | 
 909 |     <script src="https://dt5zaw6a98blc.cloudfront.net/site-static/d1190b5daf5c2f7b9e72f105a4951386409e0be1/spark/core/js/modal_focus.js" type="text/javascript"></script>
 910 |     <script src="https://dt5zaw6a98blc.cloudfront.net/site-static/d1190b5daf5c2f7b9e72f105a4951386409e0be1/spark/core/js/get_query_parameters.js" type="text/javascript"></script>
 911 |     <script src="https://dt5zaw6a98blc.cloudfront.net/site-static/d1190b5daf5c2f7b9e72f105a4951386409e0be1/spark/core/js/signature_track.js" type="text/javascript"></script>
 912 |     <script src="https://dt5zaw6a98blc.cloudfront.net/site-static/d1190b5daf5c2f7b9e72f105a4951386409e0be1/spark/app/generic/js/item_list.js" type="text/javascript"></script>
 913 |     <script src="https://dt5zaw6a98blc.cloudfront.net/site-static/d1190b5daf5c2f7b9e72f105a4951386409e0be1/spark/app/generic/js/navbar.js" type="text/javascript"></script>
 914 | 
 915 |     
 916 |     <script>
 917 |      // Set up help widget
 918 |       (function() {
 919 |         $("[data-helpwidget]").each(function() {
 920 |             window.HelpWidget(this, {tracker: window.multitracker});
 921 |         });
 922 |        })();
 923 |     </script>
 924 | 
 925 |     
 926 |     <script type="text/x-mathjax-config">
 927 | MathJax.Hub.Config({
 928 |   config: ["MMLorHTML.js"],
 929 |   
 930 |   styleSheets: [],
 931 |   styles: {},
 932 | 
 933 |   jax: ["input/TeX"],
 934 |   
 935 |   extensions: ["tex2jax.js"],
 936 | 
 937 |   preJax: null,
 938 |   postJax: null,
 939 | 
 940 |   preRemoveClass: "MathJax_Preview",
 941 | 
 942 |   showProcessingMessages: true,
 943 | 
 944 |   messageStyle: "none",
 945 |   
 946 |   displayAlign: "center",
 947 |   displayIndent: "0em",
 948 |   
 949 |   delayStartupUntil: "none",
 950 | 
 951 |   skipStartupTypeset: false,
 952 |   
 953 |   elements: [],
 954 |   
 955 |   tex2jax: {
 956 | 	    inlineMath: [
 957 | 	                 ['$$','$$'],      // uncomment this for standard TeX math delimiters
 958 | 	                 ['\\(','\\)']
 959 | 	                 ],
 960 | 
 961 | 	                 displayMath: [
 962 | 	                 ['\\[','\\]']
 963 | 	                 ],
 964 | 
 965 |     skipTags: ["script","noscript","style","textarea","pre","code"],
 966 |     ignoreClass: "tex2jax_ignore",
 967 |     processClass: "tex2jax_process",
 968 |     processEscapes: false,
 969 |     processEnvironments: true,
 970 |     preview: "TeX"
 971 |     
 972 |   },
 973 |   
 974 |   mml2jax: {
 975 |     preview: "alttext"
 976 |     
 977 |   },
 978 |   
 979 |   jsMath2jax: {
 980 |     preview: "TeX"
 981 |     
 982 |   },
 983 | 
 984 |   TeX: {
 985 |     TagSide: "right",
 986 |     TagIndent: ".8em",
 987 |     MultLineWidth: "85%",
 988 |     Macros: {},
 989 | 
 990 |     extensions: ["AMSmath.js", "AMSsymbols.js"]
 991 |     
 992 |   },
 993 | 
 994 |   //============================================================================
 995 |   //
 996 |   //  These parameters control the MathML inupt jax.
 997 |   //
 998 |   MathML: {
 999 |     //
1000 |     //  This specifies whether to use TeX spacing or MathML spacing when the
1001 |     //  HTML-CSS output jax is used.
1002 |     //
1003 |     useMathMLspacing: false
1004 |   },
1005 |   
1006 |   //============================================================================
1007 |   //
1008 |   //  These parameters control the HTML-CSS output jax.
1009 |   //
1010 |   "HTML-CSS": {
1011 |     
1012 |     scale: 100,
1013 |     
1014 |     availableFonts: ["STIX","TeX"],
1015 |     
1016 |     preferredFont: "TeX",
1017 |     
1018 |     webFont: "TeX",
1019 |     
1020 |     imageFont: "TeX",
1021 |     
1022 |     undefinedFamily: "STIXGeneral,'Arial Unicode MS',serif",
1023 |       
1024 |     showMathMenu: true,
1025 | 
1026 |     styles: {},
1027 |     
1028 |     tooltip: {
1029 |       delayPost: 600,          // milliseconds delay before tooltip is posted after mouseover
1030 |       delayClear: 600,         // milliseconds delay before tooltip is cleared after mouseout
1031 |       offsetX: 10, offsetY: 5  // pixels to offset tooltip from mouse position
1032 |     }
1033 |   },
1034 |   
1035 |   //============================================================================
1036 |   //
1037 |   //  These parameters control the NativeMML output jax.
1038 |   //
1039 |   NativeMML: {
1040 | 
1041 |     scale: 100,
1042 | 
1043 |     showMathMenu: true,
1044 |     showMathMenuMSIE: true,
1045 | 
1046 |     styles: {}
1047 |   },
1048 |   
1049 |   MathMenu: {
1050 |     delay: 400,
1051 |     
1052 |     helpURL: "http://www.mathjax.org/help/user/",
1053 | 
1054 |     showRenderer: true,
1055 |     showFontMenu: false,
1056 |     showContext:  false,
1057 | 
1058 |     windowSettings: {
1059 |       status: "no", toolbar: "no", locationbar: "no", menubar: "no",
1060 |       directories: "no", personalbar: "no", resizable: "yes", scrollbars: "yes",
1061 |       width: 100, height: 50
1062 |     },
1063 |     
1064 |     styles: {}
1065 |     
1066 |   },
1067 | 
1068 |   MMLorHTML: {
1069 |     prefer: {
1070 |       MSIE:    "MML",
1071 |       Firefox: "MML",
1072 |       Opera:   "HTML",
1073 |       other:   "HTML"
1074 |     }
1075 |   }
1076 | });
1077 | </script>
1078 | <script type="text/javascript">
1079 | (function () {
1080 | 
1081 |   function loadMathJax() {
1082 |     var script = document.createElement("script");
1083 |     script.type = "text/javascript";
1084 |     script.src  = "https://duqnjvq4jwr55.cloudfront.net/2.1/MathJax.js";
1085 |     document.getElementsByTagName("head")[0].appendChild(script);
1086 |   }
1087 | 
1088 |   window.loadOrRefreshMathJax = function(domId) {
1089 |     if (window.MathJax) {
1090 |       if (domId) {
1091 |         MathJax.Hub.Queue(["Typeset", MathJax.Hub, domId]);
1092 |       } else {
1093 |         MathJax.Hub.Queue(["Typeset", MathJax.Hub]);
1094 |       }
1095 |     } else {
1096 |       loadMathJax();
1097 |     }
1098 |   }
1099 | })();
1100 | </script>
1101 |     
1102 |     <script type="text/javascript">
1103 |         var site_base_url = "https:\/\/www.coursera.org\/";
1104 |         var site_static_asset_url = "https:\/\/dt5zaw6a98blc.cloudfront.net\/site-static\/";
1105 |         var spark_class_short_name = "malsoftware";
1106 |         var spark_class_id = 970255;
1107 |         var spark_class_url = "\/malsoftware-001";
1108 |         var spark_signature_url = "https:\/\/class.coursera.org\/malsoftware-001\/signature\/";
1109 |         
1110 |         var course_strings_name = "Malicious Software and its Underground Economy: Two Sides to Every Story";
1111 |         var student_full_name = "Dani\u00ebl W. Crompton";
1112 |     </script>
1113 | 
1114 |     
1115 | <script src="https://dt5zaw6a98blc.cloudfront.net/site-static/d1190b5daf5c2f7b9e72f105a4951386409e0be1/spark/core/js/jquery_ui.js"></script>
1116 | <script src="https://dt5zaw6a98blc.cloudfront.net/site-static/d1190b5daf5c2f7b9e72f105a4951386409e0be1/spark/core/js/jquery.history.js"></script>
1117 | <script src="https://dt5zaw6a98blc.cloudfront.net/site-static/d1190b5daf5c2f7b9e72f105a4951386409e0be1/js/lib/bootstrap.tooltip.js"></script>
1118 | <script src="https://dt5zaw6a98blc.cloudfront.net/site-static/d1190b5daf5c2f7b9e72f105a4951386409e0be1/spark/core/js/get_query_parameters.js"></script>
1119 | <script src="https://dt5zaw6a98blc.cloudfront.net/site-static/d1190b5daf5c2f7b9e72f105a4951386409e0be1/spark/app/lecture/js/lecture_list_functions.js?view_url=https://class.coursera.org/malsoftware-001/lecture/view"></script>
1120 | <script src="https://dt5zaw6a98blc.cloudfront.net/site-static/d1190b5daf5c2f7b9e72f105a4951386409e0be1/spark/app/generic/js/video_modal.js"></script>
1121 | 
1122 | </body>
1123 | 
1124 | </html>


--------------------------------------------------------------------------------
/coursera/test/test_cookies.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # -*- coding: utf-8 -*-
  3 | 
  4 | """
  5 | Test syllabus parsing.
  6 | """
  7 | 
  8 | import os.path
  9 | import unittest
 10 | 
 11 | import six
 12 | 
 13 | from coursera import cookies
 14 | 
 15 | FIREFOX_COOKIES = \
 16 |     os.path.join(os.path.dirname(__file__),
 17 |                  "fixtures", "cookies", "firefox_cookies.txt")
 18 | 
 19 | CHROME_COOKIES = \
 20 |     os.path.join(os.path.dirname(__file__),
 21 |                  "fixtures", "cookies", "chrome_cookies.txt")
 22 | 
 23 | FIREFOX_COOKIES_WITHOUT_COURSERA = \
 24 |     os.path.join(os.path.dirname(__file__),
 25 |                  "fixtures", "cookies", "firefox_cookies_without_coursera.txt")
 26 | 
 27 | FIREFOX_COOKIES_EXPIRED = \
 28 |     os.path.join(os.path.dirname(__file__),
 29 |                  "fixtures", "cookies", "firefox_cookies_expired.txt")
 30 | 
 31 | 
 32 | class MockResponse:
 33 |     def raise_for_status(self):
 34 |         pass
 35 | 
 36 | 
 37 | class MockSession:
 38 |     def __init__(self):
 39 |         self.called = False
 40 | 
 41 |     def get(self, url):
 42 |         self.called = True
 43 |         return MockResponse()
 44 | 
 45 | 
 46 | class CookiesFileTestCase(unittest.TestCase):
 47 | 
 48 |     def test_get_cookiejar_from_firefox_cookies(self):
 49 |         from six.moves import http_cookiejar as cookielib
 50 |         cj = cookies.get_cookie_jar(FIREFOX_COOKIES)
 51 |         self.assertTrue(isinstance(cj, cookielib.MozillaCookieJar))
 52 | 
 53 |     def test_get_cookiejar_from_chrome_cookies(self):
 54 |         from six.moves import http_cookiejar as cookielib
 55 |         cj = cookies.get_cookie_jar(CHROME_COOKIES)
 56 |         self.assertTrue(isinstance(cj, cookielib.MozillaCookieJar))
 57 | 
 58 |     def test_find_cookies_for_class(self):
 59 |         import requests
 60 |         cj = cookies.find_cookies_for_class(FIREFOX_COOKIES, 'class-001')
 61 |         self.assertTrue(isinstance(cj, requests.cookies.RequestsCookieJar))
 62 | 
 63 |         self.assertEquals(len(cj), 6)
 64 | 
 65 |         domains = cj.list_domains()
 66 |         self.assertEquals(len(domains), 2)
 67 |         self.assertTrue('.coursera.org' in domains)
 68 |         self.assertTrue('class.coursera.org' in domains)
 69 | 
 70 |         paths = cj.list_paths()
 71 |         self.assertEquals(len(paths), 2)
 72 |         self.assertTrue('/' in paths)
 73 |         self.assertTrue('/class-001' in paths)
 74 | 
 75 |     def test_did_not_find_cookies_for_class(self):
 76 |         import requests
 77 |         cj = cookies.find_cookies_for_class(
 78 |             FIREFOX_COOKIES_WITHOUT_COURSERA, 'class-001')
 79 |         self.assertTrue(isinstance(cj, requests.cookies.RequestsCookieJar))
 80 | 
 81 |         self.assertEquals(len(cj), 0)
 82 | 
 83 |     def test_did_not_find_expired_cookies_for_class(self):
 84 |         import requests
 85 |         cj = cookies.find_cookies_for_class(
 86 |             FIREFOX_COOKIES_EXPIRED, 'class-001')
 87 |         self.assertTrue(isinstance(cj, requests.cookies.RequestsCookieJar))
 88 | 
 89 |         self.assertEquals(len(cj), 2)
 90 | 
 91 |     def test_we_have_enough_cookies(self):
 92 |         cj = cookies.find_cookies_for_class(FIREFOX_COOKIES, 'class-001')
 93 | 
 94 |         enough = cookies.do_we_have_enough_cookies(cj, 'class-001')
 95 |         self.assertTrue(enough)
 96 | 
 97 |     def test_we_dont_have_enough_cookies(self):
 98 |         cj = cookies.find_cookies_for_class(
 99 |             FIREFOX_COOKIES_WITHOUT_COURSERA, 'class-001')
100 | 
101 |         enough = cookies.do_we_have_enough_cookies(cj, 'class-001')
102 |         self.assertFalse(enough)
103 | 
104 |     def test_make_cookie_values(self):
105 |         cj = cookies.find_cookies_for_class(FIREFOX_COOKIES, 'class-001')
106 | 
107 |         values = 'csrf_token=csrfclass001; session=sessionclass1'
108 |         cookie_values = cookies.make_cookie_values(cj, 'class-001')
109 |         self.assertEquals(cookie_values, values)
110 | 


--------------------------------------------------------------------------------
/coursera/test/test_credentials.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | 
 3 | """
 4 | Test retrieving the credentials.
 5 | """
 6 | 
 7 | import os.path
 8 | import unittest
 9 | 
10 | from coursera import credentials
11 | 
12 | NETRC = \
13 |     os.path.join(os.path.dirname(__file__),
14 |                  "fixtures", "auth", "netrc")
15 | 
16 | NOT_NETRC = \
17 |     os.path.join(os.path.dirname(__file__),
18 |                  "fixtures", "auth", "not_netrc")
19 | 
20 | 
21 | class CredentialsTestCase(unittest.TestCase):
22 | 
23 |     def test_authenticate_through_netrc_with_given_path(self):
24 |         username, password = credentials.authenticate_through_netrc(NETRC)
25 |         self.assertEquals(username, 'user@mail.com')
26 |         self.assertEquals(password, 'secret')
27 | 
28 |     def test_authenticate_through_netrc_raises_exception(self):
29 |         self.assertRaises(
30 |             credentials.CredentialsError,
31 |             credentials.authenticate_through_netrc,
32 |             NOT_NETRC)
33 | 
34 |     def test_get_credentials_with_netrc(self):
35 |         username, password = credentials.get_credentials(netrc=NETRC)
36 |         self.assertEquals(username, 'user@mail.com')
37 |         self.assertEquals(password, 'secret')
38 | 
39 |     def test_get_credentials_with_invalid_netrc_raises_exception(self):
40 |         self.assertRaises(
41 |             credentials.CredentialsError,
42 |             credentials.get_credentials,
43 |             netrc=NOT_NETRC)
44 | 
45 |     def test_get_credentials_with_username_and_password_given(self):
46 |         username, password = credentials.get_credentials(
47 |             username='user', password='pass')
48 |         self.assertEquals(username, 'user')
49 |         self.assertEquals(password, 'pass')
50 | 
51 |     def test_get_credentials_with_username_given(self):
52 |         import getpass
53 |         _getpass = getpass.getpass
54 |         getpass.getpass = lambda x: 'pass'
55 | 
56 |         username, password = credentials.get_credentials(
57 |             username='user')
58 |         self.assertEquals(username, 'user')
59 |         self.assertEquals(password, 'pass')
60 | 
61 |         getpass.getpass = _getpass
62 | 
63 |     def test_get_credentials_without_username_given_raises_exception(self):
64 |         self.assertRaises(
65 |             credentials.CredentialsError,
66 |             credentials.get_credentials)
67 | 


--------------------------------------------------------------------------------
/coursera/test/test_downloaders.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | 
  3 | """
  4 | Test the downloaders.
  5 | """
  6 | 
  7 | import unittest
  8 | 
  9 | from coursera import downloaders
 10 | 
 11 | 
 12 | class ExternalDownloaderTestCase(unittest.TestCase):
 13 | 
 14 |     def _get_session(self):
 15 |         import time
 16 |         import requests
 17 | 
 18 |         expires = int(time.time() + 60*60*24*365*50)
 19 | 
 20 |         s = requests.Session()
 21 |         s.cookies.set('csrf_token', 'csrfclass001',
 22 |                       domain="www.coursera.org", expires=expires)
 23 |         s.cookies.set('session', 'sessionclass1',
 24 |                       domain="www.coursera.org", expires=expires)
 25 |         s.cookies.set('k', 'v',
 26 |                       domain="www.example.org", expires=expires)
 27 | 
 28 |         return s
 29 | 
 30 |     def test_bin_not_specified(self):
 31 |         self.assertRaises(RuntimeError, downloaders.ExternalDownloader, None)
 32 | 
 33 |     def test_bin_not_found_raises_exception(self):
 34 |         d = downloaders.ExternalDownloader(None, bin='no_way_this_exists')
 35 |         d._prepare_cookies = lambda cmd, cv: None
 36 |         d._create_command = lambda x, y: ['no_way_this_exists']
 37 |         self.assertRaises(OSError, d._start_download, 'url', 'filename')
 38 | 
 39 |     def test_bin_is_set(self):
 40 |         d = downloaders.ExternalDownloader(None, bin='test')
 41 |         self.assertEquals(d.bin, 'test')
 42 | 
 43 |     def test_prepare_cookies(self):
 44 |         s = self._get_session()
 45 | 
 46 |         d = downloaders.ExternalDownloader(s, bin="test")
 47 | 
 48 |         def mock_add_cookies(cmd, cv):
 49 |             cmd.append(cv)
 50 | 
 51 |         d._add_cookies = mock_add_cookies
 52 |         command = []
 53 |         d._prepare_cookies(command, 'http://www.coursera.org')
 54 |         self.assertTrue('csrf_token=csrfclass001' in command[0])
 55 |         self.assertTrue('session=sessionclass1' in command[0])
 56 | 
 57 |     def test_prepare_cookies_does_nothing(self):
 58 |         s = self._get_session()
 59 |         s.cookies.clear(domain="www.coursera.org")
 60 | 
 61 |         d = downloaders.ExternalDownloader(s, bin="test")
 62 |         command = []
 63 | 
 64 |         def mock_add_cookies(cmd, cookie_values):
 65 |             pass
 66 | 
 67 |         d._add_cookies = mock_add_cookies
 68 | 
 69 |         d._prepare_cookies(command, 'http://www.coursera.org')
 70 |         self.assertEquals(command, [])
 71 | 
 72 |     def test_start_command_raises_exception(self):
 73 |         d = downloaders.ExternalDownloader(None, bin='test')
 74 |         d._add_cookies = lambda cmd, cookie_values: None
 75 |         self.assertRaises(
 76 |             NotImplementedError,
 77 |             d._create_command, 'url', 'filename')
 78 | 
 79 |     def test_wget(self):
 80 |         s = self._get_session()
 81 | 
 82 |         d = downloaders.WgetDownloader(s)
 83 |         command = d._create_command('download_url', 'save_to')
 84 |         self.assertEquals(command[0], 'wget')
 85 |         self.assertTrue('download_url' in command)
 86 |         self.assertTrue('save_to' in command)
 87 | 
 88 |         d._prepare_cookies(command, 'http://www.coursera.org')
 89 |         self.assertTrue(any("Cookie: " in e for e in command))
 90 |         self.assertTrue(any("csrf_token=csrfclass001" in e for e in command))
 91 |         self.assertTrue(any("session=sessionclass1" in e for e in command))
 92 | 
 93 |     def test_curl(self):
 94 |         s = self._get_session()
 95 | 
 96 |         d = downloaders.CurlDownloader(s)
 97 |         command = d._create_command('download_url', 'save_to')
 98 |         self.assertEquals(command[0], 'curl')
 99 |         self.assertTrue('download_url' in command)
100 |         self.assertTrue('save_to' in command)
101 | 
102 |         d._prepare_cookies(command, 'http://www.coursera.org')
103 |         self.assertTrue(any("csrf_token=csrfclass001" in e for e in command))
104 |         self.assertTrue(any("session=sessionclass1" in e for e in command))
105 | 
106 |     def test_aria2(self):
107 |         s = self._get_session()
108 | 
109 |         d = downloaders.Aria2Downloader(s)
110 |         command = d._create_command('download_url', 'save_to')
111 |         self.assertEquals(command[0], 'aria2c')
112 |         self.assertTrue('download_url' in command)
113 |         self.assertTrue('save_to' in command)
114 | 
115 |         d._prepare_cookies(command, 'http://www.coursera.org')
116 |         self.assertTrue(any("Cookie: " in e for e in command))
117 |         self.assertTrue(any("csrf_token=csrfclass001" in e for e in command))
118 |         self.assertTrue(any("session=sessionclass1" in e for e in command))
119 | 
120 |     def test_axel(self):
121 |         s = self._get_session()
122 | 
123 |         d = downloaders.AxelDownloader(s)
124 |         command = d._create_command('download_url', 'save_to')
125 |         self.assertEquals(command[0], 'axel')
126 |         self.assertTrue('download_url' in command)
127 |         self.assertTrue('save_to' in command)
128 | 
129 |         d._prepare_cookies(command, 'http://www.coursera.org')
130 |         self.assertTrue(any("Cookie: " in e for e in command))
131 |         self.assertTrue(any("csrf_token=csrfclass001" in e for e in command))
132 |         self.assertTrue(any("session=sessionclass1" in e for e in command))
133 | 
134 | 
135 | class NativeDownloaderTestCase(unittest.TestCase):
136 | 
137 |     def test_all_attempts_have_failed(self):
138 |         import time
139 | 
140 |         class IObject(object):
141 |             pass
142 | 
143 |         class MockSession:
144 | 
145 |             def get(self, url, stream=True):
146 |                 object_ = IObject()
147 |                 object_.status_code = 400
148 |                 object_.reason = None
149 |                 return object_
150 | 
151 |         _sleep = time.sleep
152 |         time.sleep = lambda interval: 0
153 | 
154 |         session = MockSession()
155 |         d = downloaders.NativeDownloader(session)
156 |         self.assertFalse(d._start_download('download_url', 'save_to'))
157 | 
158 |         time.sleep = _sleep
159 | 
160 | 
161 | class DownloadProgressTestCase(unittest.TestCase):
162 | 
163 |     def _get_progress(self, total):
164 |         p = downloaders.DownloadProgress(total)
165 |         p.report_progress = lambda: None
166 | 
167 |         return p
168 | 
169 |     def test_calc_percent_if_total_is_zero(self):
170 |         p = self._get_progress(0)
171 |         self.assertEquals(p.calc_percent(), '--%')
172 | 
173 |         p.read(10)
174 |         self.assertEquals(p.calc_percent(), '--%')
175 | 
176 |     def test_calc_percent_if_not_yet_read(self):
177 |         p = self._get_progress(100)
178 |         self.assertEquals(
179 |             p.calc_percent(),
180 |             '[                                                  ] 0%')
181 | 
182 |     def test_calc_percent_if_read(self):
183 |         p = self._get_progress(100)
184 |         p.read(2)
185 |         self.assertEquals(
186 |             p.calc_percent(),
187 |             '[#                                                 ] 2%')
188 | 
189 |         p.read(18)
190 |         self.assertEquals(
191 |             p.calc_percent(),
192 |             '[##########                                        ] 20%')
193 | 
194 |         p = self._get_progress(2300)
195 |         p.read(177)
196 |         self.assertEquals(
197 |             p.calc_percent(),
198 |             '[###                                               ] 7%')
199 | 
200 |     def test_calc_speed_if_total_is_zero(self):
201 |         p = self._get_progress(0)
202 |         self.assertEquals(p.calc_speed(), '---b/s')
203 | 
204 |     def test_calc_speed_if_not_yet_read(self):
205 |         p = self._get_progress(100)
206 |         self.assertEquals(p.calc_speed(), '---b/s')
207 | 
208 |     def test_calc_speed_ifread(self):
209 |         p = self._get_progress(10000)
210 |         p.read(2000)
211 |         p._now = p._start + 1000
212 |         self.assertEquals(p.calc_speed(), '2.00B/s')
213 | 


--------------------------------------------------------------------------------
/coursera/test/test_parsing.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # -*- coding: utf-8 -*-
  3 | 
  4 | """
  5 | Test functionality of coursera module.
  6 | """
  7 | 
  8 | import os.path
  9 | import unittest
 10 | 
 11 | from six import iteritems
 12 | 
 13 | from coursera import coursera_dl
 14 | 
 15 | 
 16 | class TestSyllabusParsing(unittest.TestCase):
 17 | 
 18 |     def setUp(self):
 19 |         """
 20 |         As setup, we mock some methods that would, otherwise, create
 21 |         repeateadly many web requests.
 22 | 
 23 |         More specifically, we mock:
 24 | 
 25 |         * the search for hidden videos
 26 |         * the actual download of videos
 27 |         """
 28 | 
 29 |         # Mock coursera_dl.grab_hidden_video_url
 30 |         self.__grab_hidden_video_url = coursera_dl.grab_hidden_video_url
 31 | 
 32 |         def new_grab_hidden_video_url(session, href):
 33 |             """
 34 |             Mock function to prevent network requests.
 35 |             """
 36 |             return None
 37 |         coursera_dl.grab_hidden_video_url = new_grab_hidden_video_url
 38 | 
 39 |         # Mock coursera_dl.get_video
 40 |         self.__get_video = coursera_dl.get_video
 41 | 
 42 |         def new_get_video(session, href):
 43 |             """
 44 |             Mock function to prevent network requests.
 45 |             """
 46 |             return None
 47 |         coursera_dl.get_video = new_get_video
 48 | 
 49 |     def tearDown(self):
 50 |         """
 51 |         We unmock the methods mocked in set up.
 52 |         """
 53 |         coursera_dl.grab_hidden_video_url = self.__grab_hidden_video_url
 54 |         coursera_dl.get_video = self.__get_video
 55 | 
 56 |     def _assert_parse(self, filename, num_sections, num_lectures,
 57 |                       num_resources, num_videos):
 58 |         filename = os.path.join(
 59 |             os.path.dirname(__file__), "fixtures", "html",
 60 |             filename)
 61 | 
 62 |         with open(filename) as syllabus:
 63 |             syllabus_page = syllabus.read()
 64 | 
 65 |             sections = coursera_dl.parse_syllabus(None, syllabus_page, None)
 66 | 
 67 |             # section count
 68 |             self.assertEqual(len(sections), num_sections)
 69 | 
 70 |             # lecture count
 71 |             lectures = [lec for sec in sections for lec in sec[1]]
 72 |             self.assertEqual(len(lectures), num_lectures)
 73 | 
 74 |             # resource count
 75 |             resources = [(res[0], len(res[1]))
 76 |                          for lec in lectures for res in iteritems(lec[1])]
 77 |             self.assertEqual(sum(r for f, r in resources), num_resources)
 78 | 
 79 |             # mp4 count
 80 |             self.assertEqual(
 81 |                 sum(r for f, r in resources if f == "mp4"),
 82 |                 num_videos)
 83 | 
 84 |     def test_parse(self):
 85 |         self._assert_parse(
 86 |             "regular-syllabus.html",
 87 |             num_sections=23,
 88 |             num_lectures=102,
 89 |             num_resources=502,
 90 |             num_videos=102)
 91 | 
 92 |     def test_links_to_wikipedia(self):
 93 |         self._assert_parse(
 94 |             "links-to-wikipedia.html",
 95 |             num_sections=5,
 96 |             num_lectures=37,
 97 |             num_resources=158,
 98 |             num_videos=36)
 99 | 
100 |     def test_parse_preview(self):
101 |         self._assert_parse(
102 |             "preview.html",
103 |             num_sections=20,
104 |             num_lectures=106,
105 |             num_resources=106,
106 |             num_videos=106)
107 | 
108 |     def test_sections_missed(self):
109 |         self._assert_parse(
110 |             "sections-not-to-be-missed.html",
111 |             num_sections=9,
112 |             num_lectures=61,
113 |             num_resources=224,
114 |             num_videos=61)
115 | 
116 |     def test_sections_missed2(self):
117 |         self._assert_parse(
118 |             "sections-not-to-be-missed-2.html",
119 |             num_sections=20,
120 |             num_lectures=121,
121 |             num_resources=397,
122 |             num_videos=121)
123 | 
124 |     def test_parse_classes_with_bs4(self):
125 |         classes = {
126 |             'datasci-001': (10, 97, 358, 97),  # issue 134
127 |             'startup-001': (4, 44, 136, 44),   # issue 137
128 |             'wealthofnations-001': (8, 74, 296, 74),  # issue 131
129 |             'malsoftware-001': (3, 18, 56, 16)  # issue 148
130 |         }
131 | 
132 |         for class_, counts in iteritems(classes):
133 |             filename = "parsing-{0}-with-bs4.html".format(class_)
134 |             self._assert_parse(
135 |                 filename,
136 |                 num_sections=counts[0],
137 |                 num_lectures=counts[1],
138 |                 num_resources=counts[2],
139 |                 num_videos=counts[3])
140 | 
141 |     def test_multiple_resources_with_the_same_format(self):
142 |         self._assert_parse(
143 |             "multiple-resources-with-the-same-format.html",
144 |             num_sections=18,
145 |             num_lectures=97,
146 |             num_resources=478,
147 |             num_videos=97)
148 | 
149 | 
150 | if __name__ == "__main__":
151 |     unittest.main()
152 | 


--------------------------------------------------------------------------------
/coursera/test/test_utils.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | 
 3 | """
 4 | Test the utility functions.
 5 | """
 6 | 
 7 | import unittest
 8 | 
 9 | from six import iteritems
10 | 
11 | from coursera import utils
12 | 
13 | 
14 | class UtilsTestCase(unittest.TestCase):
15 | 
16 |     def test_clean_filename(self):
17 |         strings = {
18 |             '(23:90)': '',
19 |             '(:': '',
20 |             'a téest &and a@noòtheèr': 'a_test_and_another',
21 |             'Lecture 2.7 - Evaluation and Operators (16:25)':
22 |             'Lecture_2.7_-_Evaluation_and_Operators',
23 |             'Week 3: Data and Abstraction':
24 |             'Week_3-_Data_and_Abstraction'
25 |         }
26 |         for k, v in iteritems(strings):
27 |             self.assertEquals(utils.clean_filename(k), v)
28 | 
29 |     def test_clean_filename_minimal_change(self):
30 |         strings = {
31 |             '(23:90)': '(23-90)',
32 |             '(:': '(-',
33 |             'a téest &and a@noòtheèr': 'a téest &and a@noòtheèr',
34 |             'Lecture 2.7 - Evaluation and Operators (16:25)':
35 |             'Lecture 2.7 - Evaluation and Operators (16-25)',
36 |             'Week 3: Data and Abstraction':
37 |             'Week 3- Data and Abstraction',
38 |             '  (Week 1) BRANDING:  Marketing Strategy and Brand Positioning':
39 |             '  (Week 1) BRANDING-  Marketing Strategy and Brand Positioning'
40 |         }
41 |         for k, v in iteritems(strings):
42 |             self.assertEquals(utils.clean_filename(k, minimal_change=True), v)
43 | 
44 |     def test_get_anchor_format(self):
45 |         strings = {
46 |             'https://class.coursera.org/sub?q=123_en&format=txt': 'txt',
47 |             'https://class.coursera.org/sub?q=123_en&format=srt': 'srt',
48 |             'https://d396qusza40orc.cloudfront.net/week7-4.pdf': 'pdf',
49 |             'https://class.coursera.org/download.mp4?lecture_id=123': 'mp4'
50 |         }
51 |         for k, v in iteritems(strings):
52 |             self.assertEquals(utils.get_anchor_format(k), v)
53 | 
54 |     def test_fix_url_ads_sheme(self):
55 |         url = "www.coursera.org"
56 |         self.assertEquals(utils.fix_url(url), 'http://www.coursera.org')
57 | 
58 |     def test_fix_url_removes_sheme(self):
59 |         url = " www.coursera.org "
60 |         self.assertEquals(utils.fix_url(url), 'http://www.coursera.org')
61 | 
62 |     def test_fix_url_doesnt_alters_empty_url(self):
63 |         url = None
64 |         self.assertEquals(utils.fix_url(url), None)
65 | 
66 |         url = ""
67 |         self.assertEquals(utils.fix_url(url), "")
68 | 


--------------------------------------------------------------------------------
/coursera/utils.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | 
 3 | """
 4 | This module provides utility functions that are used within the script.
 5 | """
 6 | 
 7 | import errno
 8 | import os
 9 | import re
10 | import string
11 | 
12 | import six
13 | 
14 | #  six.moves doesn’t support urlparse
15 | if six.PY3: # pragma: no cover
16 |     from urllib.parse import urlparse
17 | else:
18 |     from urlparse import urlparse
19 | 
20 | 
21 | def clean_filename(s, minimal_change=False):
22 |     """
23 |     Sanitize a string to be used as a filename.
24 | 
25 |     If minimal_change is set to true, then we only strip the bare minimum of
26 |     characters that are problematic for filesystems (namely, ':', '/' and
27 |     '\x00', '\n').
28 |     """
29 | 
30 |     # strip paren portions which contain trailing time length (...)
31 |     s = s.replace(':', '-').replace('/', '-').replace('\x00', '-').replace('\n', '')
32 | 
33 |     if minimal_change:
34 |         return s
35 | 
36 |     s = re.sub(r"\([^\(]*$", '', s)
37 |     s = s.replace('nbsp', '')
38 |     s = s.strip().replace(' ', '_')
39 |     valid_chars = '-_.()%s%s' % (string.ascii_letters, string.digits)
40 |     return ''.join(c for c in s if c in valid_chars)
41 | 
42 | 
43 | def get_anchor_format(a):
44 |     """
45 |     Extract the resource file-type format from the anchor.
46 |     """
47 | 
48 |     # (. or format=) then (file_extension) then (? or $)
49 |     # e.g. "...format=txt" or "...download.mp4?..."
50 |     fmt = re.search(r"(?:\.|format=)(\w+)(?:\?.*)?$", a)
51 |     return (fmt.group(1) if fmt else None)
52 | 
53 | 
54 | def mkdir_p(path, mode=0o777):
55 |     """
56 |     Create subdirectory hierarchy given in the paths argument.
57 |     """
58 | 
59 |     try:
60 |         os.makedirs(path, mode)
61 |     except OSError as exc:
62 |         if exc.errno == errno.EEXIST and os.path.isdir(path):
63 |             pass
64 |         else:
65 |             raise
66 | 
67 | 
68 | def fix_url(url):
69 |     """
70 |     Strip whitespace characters from the beginning and the end of the url
71 |     and add a default scheme.
72 |     """
73 |     if url is None:
74 |         return None
75 | 
76 |     url = url.strip()
77 | 
78 |     if url and not urlparse(url).scheme:
79 |         url = "http://" + url
80 | 
81 |     return url
82 | 


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | argparse==1.2.1
2 | beautifulsoup4==4.1.3
3 | coverage==3.7
4 | html5lib==1.0b2
5 | nose==1.3.0
6 | requests==1.2.3
7 | six==1.3.0
8 | 


--------------------------------------------------------------------------------