├── .gitignore ├── lesson_downloader.js ├── lesson_info.js ├── lesson_pager.js ├── login.js ├── readme.md ├── series_downloader.js └── series_fetcher.js /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # cookie file # 3 | ###################### 4 | *.cookie 5 | *.txt 6 | *.json 7 | 8 | # OS generated files # 9 | ###################### 10 | .DS_Store 11 | .DS_Store? 12 | ehthumbs.db 13 | Icon? 14 | Thumbs.db 15 | 16 | 17 | storage/ 18 | laracasts/ 19 | -------------------------------------------------------------------------------- /lesson_downloader.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var child_process = require('child_process'); 4 | var download_string = ''; 5 | 6 | var fs = require('fs'); 7 | var download_link_file = 'storage/download_link.txt'; 8 | 9 | // This is the video file download dictory 10 | var lesson_dir = '../../Learning/Laravel/Laracasts/Lesson/'; 11 | 12 | var util = require('util'); 13 | 14 | var lessen_infos; 15 | var lessens_links_file = "storage/lessen_infos.json"; 16 | var cookie_header = '--header=Cookie: ' + '_trak_0303ff918bab012c714b118edfe4ef84a8d22ebb_fo....'; 17 | var user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:26.0) Gecko/20100101 Firefox/26.0'; 18 | 19 | function download() 20 | { 21 | var download_option = [ 22 | cookie_header, 23 | '--user-agent='+user_agent, 24 | '--check-certificate=false', 25 | '--continue=true', 26 | '--max-concurrent-downloads=5', 27 | '--input-file='+download_link_file 28 | ] 29 | child = child_process.spawn('aria2c', download_option); 30 | child.stdout.setEncoding('utf8'); 31 | child.stdout.on('data', function(data) 32 | { 33 | console.log(data); 34 | }); 35 | } 36 | 37 | lessen_infos = JSON.parse(fs.readFileSync(lessens_links_file, 'utf8')); 38 | 39 | for(var key in lessen_infos) 40 | { 41 | var lesson = lessen_infos[key]; 42 | var lesson_file_name = lesson.Title + '.mp4'; 43 | download_string += lesson.DownloadLink + "\r\n"; 44 | download_string += "\t" + 'out=' + lesson_dir + lesson_file_name + "\r\n"; 45 | } 46 | 47 | fs.writeFileSync(download_link_file, download_string); 48 | 49 | download(); 50 | -------------------------------------------------------------------------------- /lesson_info.js: -------------------------------------------------------------------------------- 1 | 2 | var fs = require('fs'); 3 | 4 | var lessen_infos; 5 | var new_lessen_infos = []; 6 | 7 | var lessens_links_file = "storage/lessen_infos.json"; 8 | 9 | var casper = require('casper').create 10 | ({ 11 | // verbose: true, 12 | // logLevel: 'info', 13 | waitTimeout: 30000, 14 | // clientScripts: ["jquery.js"], 15 | pageSettings: { 16 | userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36', 17 | loadImages: false, // The WebPage instance used by Casper will 18 | loadPlugins: false // use these settings 19 | } 20 | }); 21 | 22 | // check lessen_infos file 23 | if (fs.exists(lessens_links_file)) { 24 | lessen_infos = JSON.parse(fs.read(lessens_links_file)); 25 | } else { 26 | casper.log('Info file cannot be found, make sure you run fetch_course_info.js first.', 'error'); 27 | casper.exit(); 28 | } 29 | 30 | function get_lessen_link() 31 | { 32 | return $("li#playback-rate").prev().find("a").attr("href"); 33 | } 34 | 35 | // enabling the for testing 36 | phantom.casperTest = true; 37 | 38 | casper.start().each(lessen_infos, function(self, lesson) 39 | { 40 | self.thenOpen(lesson.CourseLink, function() { 41 | 42 | console.log('Current Lesson: '+lesson.Title); 43 | console.log(' - Request Url:: '+lesson.CourseLink); 44 | 45 | var link = this.evaluate(get_lessen_link); 46 | lesson.DownloadLink = 'https://laracasts.com' + link; 47 | 48 | console.log(' - Download Link is: ' + lesson.DownloadLink); 49 | new_lessen_infos.push(lesson); 50 | console.log(' '); 51 | }); 52 | }) 53 | 54 | casper.then(function() 55 | { 56 | fs.write(lessens_links_file, JSON.stringify(new_lessen_infos), 'w'); 57 | console.log("Lessons Data: "); 58 | require('utils').dump(new_lessen_infos); 59 | }); 60 | 61 | casper.run(); 62 | 63 | -------------------------------------------------------------------------------- /lesson_pager.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env casperjs 2 | 3 | var fs = require('fs'); 4 | var lesson_url = 'https://laracasts.com/lessons'; 5 | 6 | var lessen_infos = []; 7 | var lessens_links_file = "storage/lessen_infos.json"; 8 | var cookie_file = 'storage/cookies.txt'; 9 | 10 | var currentPage = 1; 11 | 12 | var casper = require('casper').create 13 | ({ 14 | verbose: true, 15 | logLevel: 'debug', 16 | waitTimeout: 30000, 17 | // clientScripts: ["jquery.js"], 18 | pageSettings: { 19 | userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36', 20 | loadImages: false, // The WebPage instance used by Casper will 21 | loadPlugins: false // use these settings 22 | } 23 | }); 24 | 25 | 26 | // check cookie file 27 | if (fs.exists(cookie_file)) { 28 | cookies = JSON.parse(fs.read(cookie_file)); 29 | } else { 30 | casper.log('cookie file cannot be found, are you sure you are login.', 'error'); 31 | casper.exit(); 32 | } 33 | 34 | function get_lessen_infos() 35 | { 36 | var articles = []; 37 | $(".piece").first().find("article.lesson-block-lesson").each(function(index){ 38 | var article = {}; 39 | article['Title'] = $(this).find('h3 a').text(); 40 | article['CourseLink'] = $(this).find('h3 a').attr('href'); 41 | article['DownloadLink'] = ''; 42 | article['ReleaseDate'] = $(this).find('div.meta div.lesson-date small').text(); 43 | article['Duration'] = $(this).find('div.meta div.lesson-length span').text(); 44 | articles.push(article); 45 | }); 46 | return articles; 47 | } 48 | 49 | function correct_number (num) { 50 | if ( num<10 ) 51 | num= '00'+num; 52 | else if( num<100) 53 | num= '0'+num; 54 | return num; 55 | } 56 | 57 | var terminate = function() { 58 | lessen_infos.reverse(); 59 | for(var i=0; i Cookie Setted."); 110 | }); 111 | 112 | // Fire up 113 | casper.thenOpen(lesson_url, function() 114 | { 115 | this.echo("Start Page Title " + this.getTitle()); 116 | processPage.call(casper); 117 | }); 118 | 119 | 120 | casper.run(); 121 | 122 | -------------------------------------------------------------------------------- /login.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env casperjs 2 | 3 | var fs = require('fs'); 4 | var cookie_file = 'storage/cookies.txt'; 5 | 6 | var login_url = 'https://laracasts.com/login'; 7 | var login_test_url = 'https://laracasts.com/admin/history'; 8 | var login_username = 'here is the usrename'; 9 | var login_password = 'pasword here'; 10 | 11 | var casper = require('casper').create 12 | ({ 13 | verbose: true, 14 | logLevel: 'debug', 15 | userAgent: 'Mozilla/5.0 (Windows NT 6.1; rv:17.0) Gecko/20100101 Firefox/17.0', 16 | pageSettings: { 17 | loadImages: false, // The WebPage instance used by Casper will 18 | loadPlugins: false // use these settings 19 | } 20 | }); 21 | 22 | phantom.casperTest = true; 23 | 24 | casper.start(login_url, function() { 25 | 26 | console.log("------------------------> page loaded"); 27 | this.echo('=====================================>'+this.getTitle()); 28 | 29 | this.test.assertExists('div#form form', 'Form is found'); 30 | 31 | this.fill('div#form form', { 32 | email: login_username, 33 | password: login_password, 34 | remember: 'on' 35 | }, true); 36 | 37 | this.then(function() { 38 | console.log('Login From Submitted!'); 39 | }); 40 | }); 41 | 42 | casper.thenOpen(login_test_url, function() { 43 | this.echo('=====================================>'+this.getTitle()); 44 | this.test.assertExists('div.admin-nav', '√√√√√√√√√√√√√√√√√√√√√ -----> Login Success!'); 45 | }); 46 | 47 | casper.run(function() 48 | { 49 | var cookies = JSON.stringify((this.page.cookies)); 50 | fs.write(cookie_file, cookies, 'w'); 51 | this.exit(); 52 | }); 53 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | 2 | ## Introduction 3 | 4 | This is little script for download video from http://www.laracasts.com/. 5 | 6 | ## Requirements 7 | 8 | Reqirements: 9 | 10 | * nodejs 11 | * caspterjs - Headless browser 12 | * aria2 - Powerfull command line downloader 13 | 14 | ### Install nodejs 15 | 16 | ➜ brew update 17 | ➜ brew install node 18 | 19 | ### Install caspterjs 20 | 21 | ➜ brew install casperjs 22 | 23 | ### Install aria2 24 | 25 | ➜ brew install aria2 26 | 27 | ## Usage: Downloading Lessons 28 | 29 | > Please follow the steps. 30 | 31 | ### 1. Fetching all the lessons info. 32 | 33 | casperjs lesson_pager.js 34 | 35 | ### 2. Collecting download links. 36 | 37 | casperjs lesson_info.js 38 | 39 | ### 3. Configurate the lesson_downloader.js file 40 | 41 | The website will check for user login credential, we must setting up the cookie for downloader `aria2`; 42 | 43 | #### 1). Get the cookie string 44 | 45 | 1. Open Chrome Broswer link -> http://www.laracasts.com/ and login; 46 | 2. Using chrome debuger -> network, refresh the page, click for request detail, copy the cookie string; 47 | 3. Paste it onto the following link in `lesson_downloader.js` file: 48 | 49 | ``` 50 | var cookie_header = '--header=Cookie: ' + 'PASTE THE COOKE HERE'; 51 | ``` 52 | 53 | #### 2). Modifiy the download destination folder 54 | 55 | 56 | ``` 57 | // This is the video file download dictory 58 | var lesson_dir = '../../Learning/Laravel/Laracasts/Lesson/'; 59 | ``` 60 | 61 | ### 4. Start the download 62 | 63 | node lesson_downloader.js 64 | 65 | 66 | ## Usage: Downloading Series 67 | 68 | ### 1. Gathering series info 69 | 70 | ``` 71 | casperjs series_fetcher.js --web-security=no --series=https://laracasts.com/series/whats-new-in-laravel-5 72 | ``` 73 | ### 2. Configurate the series_downloader.js file 74 | 75 | Just like [before](#3-configurate-the-lesson_downloaderjs-file). 76 | 77 | ### 3. Start the download 78 | 79 | node series_downloader.js 80 | 81 | 82 | -------------------------------------------------------------------------------- /series_downloader.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var child_process = require('child_process'); 4 | var fs = require('fs'); 5 | var util = require('util'); 6 | var download_link_file = 'storage/series_download_link.txt'; 7 | 8 | var cookie_header = '--header=Cookie: '+'_trak_0303ff918bab012c714b118edfe4ef84a8d22ebb_foo=; ...'; 9 | var user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:26.0) Gecko/20100101 Firefox/26.0'; 10 | 11 | function download() 12 | { 13 | var download_option = [ 14 | cookie_header, 15 | '--user-agent='+user_agent, 16 | '--check-certificate=false', 17 | '--continue=true', 18 | '--max-concurrent-downloads=2', 19 | '--input-file='+download_link_file 20 | ] 21 | child = child_process.spawn('aria2c', download_option); 22 | child.stdout.setEncoding('utf8'); 23 | child.stdout.on('data', function(data) 24 | { 25 | console.log(data); 26 | }); 27 | } 28 | 29 | download(); 30 | -------------------------------------------------------------------------------- /series_fetcher.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env casperjs 2 | 3 | var fs = require('fs'); 4 | var lessen_infos = new_lessen_infos = []; 5 | 6 | var lesson_dir = 'storage/Laracasts/Series/'; 7 | var download_link_file = 'storage/series_download_link.txt'; 8 | var series_name; 9 | 10 | var casper = require('casper').create 11 | ({ 12 | verbose: true, 13 | // logLevel: 'debug', 14 | logLevel: 'error', 15 | userAgent: 'Mozilla/5.0 (Windows NT 6.1; rv:17.0) Gecko/20100101 Firefox/17.0', 16 | pageSettings: { 17 | loadImages: false, // The WebPage instance used by Casper will 18 | loadPlugins: false // use these settings 19 | } 20 | }); 21 | 22 | phantom.casperTest = true; 23 | 24 | // get course from command line 25 | series_url = casper.cli.options.series; 26 | if (!series_url) { 27 | casper.log('Usage: "casperjs series_fetcher.js --web-security=no --series=https://laracasts.com/series/whats-new-in-laravel-5" ', 'error'); 28 | casper.exit(); 29 | } else { 30 | console.log('Download Series: '); 31 | require("utils").dump(series_url); 32 | console.log('Fetching information... '); 33 | } 34 | 35 | function get_lessen_infos() 36 | { 37 | var articles = []; 38 | $("table.episode-outline tr.episode-wrap").each(function(index){ 39 | var article = {}; 40 | var title = $(this).find('.episode-title a').text().replace(/(\r\n|\n|\r)/gm,"").replace(/\s+/g, ' ').trim(); 41 | article['Title'] = $(this).find('.episode-index').text().trim() + '.' + title; 42 | article['CourseLink'] = "https://laracasts.com" + $(this).find('.episode-title a').attr('href'); 43 | article['DownloadLink'] = ''; 44 | articles.push(article); 45 | }); 46 | return articles; 47 | } 48 | function get_lessen_link() 49 | { 50 | return $("li#playback-rate").prev().find("a").attr("href"); 51 | } 52 | function get_lessen_title() 53 | { 54 | return $("h1.lesson-title").text().trim(); 55 | } 56 | 57 | casper.start(series_url, function() { 58 | 59 | console.log("------------------------> page loaded"); 60 | series_name = this.getTitle(); 61 | this.echo('=====================================>'+this.getTitle()); 62 | 63 | lessen_infos = this.evaluate(get_lessen_infos); 64 | // console.log(lessen_infos); 65 | require("utils").dump(lessen_infos); 66 | 67 | console.log('Beginning to get lesson download links ... '); 68 | 69 | this.each(lessen_infos, function(self, lesson) 70 | { 71 | self.thenOpen(lesson.CourseLink, function() { 72 | 73 | console.log('Current Lesson: '+lesson.Title); 74 | console.log(' - Request Url:: '+lesson.CourseLink); 75 | 76 | lesson.DownloadLink = 'https://laracasts.com' + this.evaluate(get_lessen_link); 77 | console.log(' - Download Link is: ' + lesson.DownloadLink); 78 | new_lessen_infos.push(lesson); 79 | console.log(' '); 80 | }); 81 | }) 82 | 83 | }); 84 | 85 | // ------------------------------------------------------------- 86 | // -------------------- Sorting Out the Download Link --------- 87 | // ------------------------------------------------------------- 88 | 89 | casper.then(function() 90 | { 91 | var download_string = ''; 92 | for(var key in new_lessen_infos) 93 | { 94 | var lesson = new_lessen_infos[key]; 95 | var lesson_file_name = lesson.Title + '.mp4'; 96 | download_string += lesson.DownloadLink + "\r\n"; 97 | download_string += "\t" + 'out=' + lesson_dir + series_name + '/' + lesson_file_name + "\r\n"; 98 | } 99 | 100 | fs.write(download_link_file, download_string, 'w'); 101 | 102 | console.log("Lessons Data: "); 103 | require('utils').dump(new_lessen_infos); 104 | }); 105 | 106 | 107 | casper.run(); 108 | --------------------------------------------------------------------------------