├── api ├── models │ ├── .gitkeep │ ├── Notice.js │ ├── Job.js │ ├── Report.js │ ├── DataModel.js │ ├── Request.js │ ├── Analytic.js │ └── User.js ├── services │ ├── .gitkeep │ ├── FlashService.js │ ├── LoggerService.js │ ├── SessionService.js │ ├── QueueService.js │ ├── AuthService.js │ └── UtilService.js ├── controllers │ ├── .gitkeep │ ├── NoticeController.js │ ├── JobController.js │ ├── HomeController.js │ ├── DataScriptController.js │ ├── AdminController.js │ └── ReportController.js ├── policies │ ├── getOnly.js │ ├── postOnly.js │ ├── flash.js │ ├── isAuthenticated.js │ ├── sessionAuth.js │ ├── hasID.js │ └── isAdmin.js └── responses │ ├── ok.js │ ├── badRequest.js │ ├── forbidden.js │ ├── serverError.js │ └── notFound.js ├── assets ├── images │ ├── .gitkeep │ ├── banner.jpg │ ├── favicon.png │ ├── sort_asc.png │ ├── sort_both.png │ ├── sort_desc.png │ ├── back_enabled.png │ ├── Sorting icons.psd │ ├── back_disabled.png │ ├── forward_disabled.png │ ├── forward_enabled.png │ ├── sort_asc_disabled.png │ ├── back_enabled_hover.png │ ├── sort_desc_disabled.png │ ├── forward_enabled_hover.png │ ├── glyphicons-halflings.png │ └── glyphicons-halflings-white.png ├── templates │ ├── .gitkeep │ └── errorHandler.ejs ├── favicon.ico ├── fonts │ ├── glyphicons-halflings-regular.eot │ ├── glyphicons-halflings-regular.ttf │ └── glyphicons-halflings-regular.woff ├── robots.txt ├── styles │ └── importer.less ├── scaffolds │ └── d3_scaffold.ejs └── js │ └── app.js ├── edX-datascrub ├── src │ ├── logs │ │ ├── __init__.py │ │ ├── makeUserList.py │ │ ├── buildUnknownLog.py │ │ ├── moveRawLogs.py │ │ ├── makeUserIpList.py │ │ ├── moveWeeklyLogs.py │ │ ├── buildFullLog.py │ │ ├── distillUnknownLabels.py │ │ ├── selectLogRange.py │ │ ├── countInteract.py │ │ ├── listClassEntries.py │ │ ├── cleanLogDups.py │ │ ├── findLogDups.py │ │ ├── processLogData.py │ │ ├── buildWeekLog.py │ │ └── checkDates.py │ ├── checkData │ │ ├── __init__.py │ │ ├── getCertsFromId.py │ │ ├── certsAndusers.py │ │ ├── compUser.py │ │ ├── corrUsers.py │ │ └── checkUsersTimes.py │ ├── convertfiles │ │ ├── __init__.py │ │ ├── __init__.pyc │ │ ├── sqltocsv.pyc │ │ ├── sqltocsv.py │ │ ├── makeIdCountryFile.py │ │ └── killListedFiles.py │ ├── demographics │ │ ├── __init__.py │ │ ├── scrubstudentprofile.py │ │ ├── buildAnonProfile.py │ │ └── getenrollmentdata.py │ ├── processAll.py │ ├── getIP.py │ ├── getProblemIds.py │ ├── utils.py │ ├── deleteUnchangedFiles.py │ ├── scrapeEmail.py │ ├── course_struct.py │ ├── findEventTypes.py │ ├── findBrowserEventTypes.py │ ├── countcerts.py │ ├── buildEmailList.py │ ├── usersAndClasses.py │ ├── diffUsers.py │ ├── buildCompRoster.py │ ├── moveWeeklyToClass.py │ ├── buildClassList.py │ ├── coursestudentstate.py │ ├── buildCertList.py │ └── buildAllStudents.py └── shellscripts │ ├── transformOneLog.sh │ └── processSmallLogData.sh ├── bin ├── stop_moocRP.sh ├── setup │ ├── db_setup.sql │ └── setup.sh ├── start_moocRP.sh └── data_scripts │ └── distribution │ └── zip.sh ├── test ├── mocha.opts ├── services │ ├── EncryptionService.spec.js │ └── SessionService.spec.js ├── bootstrap.js ├── SampleController.spec.js ├── controllers │ └── UserController.spec.js └── stubs.js ├── .sailsrc ├── .travis.yml ├── config ├── constants.js ├── locales │ ├── de.json │ ├── en.json │ ├── fr.json │ ├── es.json │ └── _README.md ├── models.js ├── env │ ├── test.js │ └── development.js ├── bootstrap.js ├── ssl │ ├── server.csr │ ├── server.crt │ ├── server.key │ └── server.key.secure ├── paths.js ├── log.js ├── i18n.js └── csrf.js ├── tasks ├── register │ ├── test.js │ ├── default.js │ ├── syncAssets.js │ ├── build.js │ ├── compileAssets.js │ ├── buildProd.js │ ├── linkAssets.js │ ├── linkAssetsBuild.js │ ├── linkAssetsBuildProd.js │ └── prod.js ├── config │ ├── clean.js │ ├── uglify.js │ ├── cssmin.js │ ├── mochaTest.js │ ├── sync.js │ ├── less.js │ ├── concat.js │ ├── coffee.js │ ├── copy.js │ ├── watch.js │ └── jst.js ├── pipeline.js └── README.md ├── .editorconfig ├── ssl └── cert.pem ├── views ├── analytic │ └── edit.ejs ├── home │ ├── about.ejs │ ├── license.ejs │ ├── privacy.ejs │ └── contact.ejs ├── report │ └── index.ejs ├── user │ └── edit.ejs └── datamodel │ └── info.ejs ├── documentation ├── data_models.md ├── data_distribution.md └── installation.md ├── app.js ├── package.json └── Gruntfile.js /api/models/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /api/services/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /assets/images/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /api/controllers/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /assets/templates/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /edX-datascrub/src/logs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /edX-datascrub/src/checkData/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /edX-datascrub/src/convertfiles/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /edX-datascrub/src/demographics/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /bin/stop_moocRP.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | pm2 kill 3 | -------------------------------------------------------------------------------- /test/mocha.opts: -------------------------------------------------------------------------------- 1 | --reporter spec 2 | --colors 3 | --timeout 5000 -------------------------------------------------------------------------------- /.sailsrc: -------------------------------------------------------------------------------- 1 | { 2 | "generators": { 3 | "modules": {} 4 | } 5 | } -------------------------------------------------------------------------------- /bin/setup/db_setup.sql: -------------------------------------------------------------------------------- 1 | CREATE DATABASE IF NOT EXISTS moocRP; 2 | -------------------------------------------------------------------------------- /assets/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CAHLR/moocRP/HEAD/assets/favicon.ico -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "0.10" 4 | services: 5 | - redis-server 6 | -------------------------------------------------------------------------------- /assets/images/banner.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CAHLR/moocRP/HEAD/assets/images/banner.jpg -------------------------------------------------------------------------------- /assets/images/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CAHLR/moocRP/HEAD/assets/images/favicon.png -------------------------------------------------------------------------------- /bin/start_moocRP.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | redis-server& 3 | NODE_ENV=development pm2 start app.js 4 | -------------------------------------------------------------------------------- /config/constants.js: -------------------------------------------------------------------------------- 1 | module.exports.constants = { 2 | SUCCESS: true, 3 | FAILURE: false 4 | } -------------------------------------------------------------------------------- /assets/images/sort_asc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CAHLR/moocRP/HEAD/assets/images/sort_asc.png -------------------------------------------------------------------------------- /assets/images/sort_both.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CAHLR/moocRP/HEAD/assets/images/sort_both.png -------------------------------------------------------------------------------- /assets/images/sort_desc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CAHLR/moocRP/HEAD/assets/images/sort_desc.png -------------------------------------------------------------------------------- /assets/images/back_enabled.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CAHLR/moocRP/HEAD/assets/images/back_enabled.png -------------------------------------------------------------------------------- /config/locales/de.json: -------------------------------------------------------------------------------- 1 | { 2 | "Welcome": "Willkommen", 3 | "A brand new app.": "Eine neue App." 4 | } 5 | -------------------------------------------------------------------------------- /config/locales/en.json: -------------------------------------------------------------------------------- 1 | { 2 | "Welcome": "Welcome", 3 | "A brand new app.": "A brand new app." 4 | } 5 | -------------------------------------------------------------------------------- /assets/images/Sorting icons.psd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CAHLR/moocRP/HEAD/assets/images/Sorting icons.psd -------------------------------------------------------------------------------- /assets/images/back_disabled.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CAHLR/moocRP/HEAD/assets/images/back_disabled.png -------------------------------------------------------------------------------- /assets/images/forward_disabled.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CAHLR/moocRP/HEAD/assets/images/forward_disabled.png -------------------------------------------------------------------------------- /assets/images/forward_enabled.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CAHLR/moocRP/HEAD/assets/images/forward_enabled.png -------------------------------------------------------------------------------- /assets/images/sort_asc_disabled.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CAHLR/moocRP/HEAD/assets/images/sort_asc_disabled.png -------------------------------------------------------------------------------- /assets/images/back_enabled_hover.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CAHLR/moocRP/HEAD/assets/images/back_enabled_hover.png -------------------------------------------------------------------------------- /assets/images/sort_desc_disabled.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CAHLR/moocRP/HEAD/assets/images/sort_desc_disabled.png -------------------------------------------------------------------------------- /config/locales/fr.json: -------------------------------------------------------------------------------- 1 | { 2 | "Welcome": "Bienvenue", 3 | "A brand new app.": "Une toute nouvelle application." 4 | } 5 | -------------------------------------------------------------------------------- /assets/images/forward_enabled_hover.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CAHLR/moocRP/HEAD/assets/images/forward_enabled_hover.png -------------------------------------------------------------------------------- /assets/images/glyphicons-halflings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CAHLR/moocRP/HEAD/assets/images/glyphicons-halflings.png -------------------------------------------------------------------------------- /config/locales/es.json: -------------------------------------------------------------------------------- 1 | { 2 | "Welcome": "Bienvenido", 3 | "A brand new app.": "Una aplicación de la nueva marca." 4 | } 5 | -------------------------------------------------------------------------------- /assets/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CAHLR/moocRP/HEAD/assets/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /assets/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CAHLR/moocRP/HEAD/assets/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /assets/images/glyphicons-halflings-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CAHLR/moocRP/HEAD/assets/images/glyphicons-halflings-white.png -------------------------------------------------------------------------------- /edX-datascrub/src/convertfiles/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CAHLR/moocRP/HEAD/edX-datascrub/src/convertfiles/__init__.pyc -------------------------------------------------------------------------------- /edX-datascrub/src/convertfiles/sqltocsv.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CAHLR/moocRP/HEAD/edX-datascrub/src/convertfiles/sqltocsv.pyc -------------------------------------------------------------------------------- /tasks/register/test.js: -------------------------------------------------------------------------------- 1 | module.exports = function (grunt) { 2 | grunt.registerTask('test', [ 3 | 'mochaTest' 4 | ]); 5 | }; 6 | -------------------------------------------------------------------------------- /assets/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CAHLR/moocRP/HEAD/assets/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /tasks/register/default.js: -------------------------------------------------------------------------------- 1 | module.exports = function (grunt) { 2 | grunt.registerTask('default', ['compileAssets', 'linkAssets', 'watch']); 3 | }; 4 | -------------------------------------------------------------------------------- /bin/data_scripts/distribution/zip.sh: -------------------------------------------------------------------------------- 1 | #################### 2 | # Author Kevin Kao # 3 | #################### 4 | 5 | #!/bin/bash 6 | for i in *; do zip -r "${i%/}.zip" "$i"; done -------------------------------------------------------------------------------- /tasks/register/syncAssets.js: -------------------------------------------------------------------------------- 1 | module.exports = function (grunt) { 2 | grunt.registerTask('syncAssets', [ 3 | 'jst:dev', 4 | 'less:dev', 5 | 'sync:dev', 6 | 'coffee:dev' 7 | ]); 8 | }; 9 | -------------------------------------------------------------------------------- /tasks/register/build.js: -------------------------------------------------------------------------------- 1 | module.exports = function (grunt) { 2 | grunt.registerTask('build', [ 3 | 'compileAssets', 4 | 'linkAssetsBuild', 5 | 'clean:build', 6 | 'copy:build' 7 | ]); 8 | }; 9 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | -------------------------------------------------------------------------------- /tasks/register/compileAssets.js: -------------------------------------------------------------------------------- 1 | module.exports = function (grunt) { 2 | grunt.registerTask('compileAssets', [ 3 | 'clean:dev', 4 | 'jst:dev', 5 | 'less:dev', 6 | 'copy:dev', 7 | 'coffee:dev' 8 | ]); 9 | }; 10 | -------------------------------------------------------------------------------- /tasks/register/buildProd.js: -------------------------------------------------------------------------------- 1 | module.exports = function (grunt) { 2 | grunt.registerTask('buildProd', [ 3 | 'compileAssets', 4 | 'concat', 5 | 'uglify', 6 | 'cssmin', 7 | 'linkAssetsBuildProd', 8 | 'clean:build', 9 | 'copy:build' 10 | ]); 11 | }; 12 | -------------------------------------------------------------------------------- /edX-datascrub/src/logs/makeUserList.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import json 4 | import sys 5 | 6 | f1 = open(sys.argv[1], 'r') 7 | f2 = open(sys.argv[2], 'w') 8 | dc = json.JSONDecoder() 9 | 10 | for line in f1: 11 | dcl = dc.decode(line) 12 | f2.write(dcl['username'] + '\n') 13 | -------------------------------------------------------------------------------- /api/models/Notice.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Notice.js 3 | * 4 | * @description :: TODO: You might write a short summary of how this model works and what it represents here. 5 | * @docs :: http://sailsjs.org/#!documentation/models 6 | */ 7 | 8 | module.exports = { 9 | 10 | attributes: { 11 | 12 | } 13 | }; 14 | 15 | -------------------------------------------------------------------------------- /assets/robots.txt: -------------------------------------------------------------------------------- 1 | # The robots.txt file is used to control how search engines index your live URLs. 2 | # See http://www.robotstxt.org/wc/norobots.html for more information. 3 | 4 | 5 | 6 | # To prevent search engines from seeing the site altogether, uncomment the next two lines: 7 | # User-Agent: * 8 | # Disallow: / 9 | -------------------------------------------------------------------------------- /tasks/register/linkAssets.js: -------------------------------------------------------------------------------- 1 | module.exports = function (grunt) { 2 | grunt.registerTask('linkAssets', [ 3 | 'sails-linker:devJs', 4 | 'sails-linker:devStyles', 5 | 'sails-linker:devTpl', 6 | 'sails-linker:devJsJade', 7 | 'sails-linker:devStylesJade', 8 | 'sails-linker:devTplJade' 9 | ]); 10 | }; 11 | -------------------------------------------------------------------------------- /tasks/register/linkAssetsBuild.js: -------------------------------------------------------------------------------- 1 | module.exports = function (grunt) { 2 | grunt.registerTask('linkAssetsBuild', [ 3 | 'sails-linker:devJsRelative', 4 | 'sails-linker:devStylesRelative', 5 | 'sails-linker:devTpl', 6 | 'sails-linker:devJsRelativeJade', 7 | 'sails-linker:devStylesRelativeJade', 8 | 'sails-linker:devTplJade' 9 | ]); 10 | }; 11 | -------------------------------------------------------------------------------- /api/policies/getOnly.js: -------------------------------------------------------------------------------- 1 | /** 2 | * getOnly 3 | * 4 | * @module :: Policy 5 | * @description :: Only allows GET requests 6 | * @docs :: http://sailsjs.org/#!documentation/policies 7 | * 8 | */ 9 | module.exports = function(req, res, next) { 10 | if (req.method === 'GET') { 11 | return next(); 12 | } 13 | return res.badRequest(); 14 | }; -------------------------------------------------------------------------------- /api/services/FlashService.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | success: function(req, message) { 3 | req.session.messages['success'].push(message); 4 | }, 5 | warning: function(req, message) { 6 | req.session.messages['warning'].push(message); 7 | }, 8 | error: function(req, message) { 9 | req.session.messages['error'].push(message); 10 | } 11 | } -------------------------------------------------------------------------------- /edX-datascrub/shellscripts/transformOneLog.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Transform one json log into nicely formatted csv log. 3 | 4 | # The script takes 1 argument: 5 | # argument 1: name of log file to be processed. 6 | 7 | echo transform begins... 8 | log=$1 9 | csv=${log:0:`expr length $log - 4`}.csv 10 | makePersonClick.py $log $2 $csv 11 | echo transform ends... -------------------------------------------------------------------------------- /api/policies/postOnly.js: -------------------------------------------------------------------------------- 1 | /** 2 | * postOnly 3 | * 4 | * @module :: Policy 5 | * @description :: Only allows POST requests 6 | * @docs :: http://sailsjs.org/#!documentation/policies 7 | * 8 | */ 9 | module.exports = function(req, res, next) { 10 | if (req.method === 'POST') { 11 | return next(); 12 | } 13 | return res.badRequest(); 14 | }; -------------------------------------------------------------------------------- /tasks/register/linkAssetsBuildProd.js: -------------------------------------------------------------------------------- 1 | module.exports = function (grunt) { 2 | grunt.registerTask('linkAssetsBuildProd', [ 3 | 'sails-linker:prodJsRelative', 4 | 'sails-linker:prodStylesRelative', 5 | 'sails-linker:devTpl', 6 | 'sails-linker:prodJsRelativeJade', 7 | 'sails-linker:prodStylesRelativeJade', 8 | 'sails-linker:devTplJade' 9 | ]); 10 | }; 11 | -------------------------------------------------------------------------------- /tasks/register/prod.js: -------------------------------------------------------------------------------- 1 | module.exports = function (grunt) { 2 | grunt.registerTask('prod', [ 3 | 'compileAssets', 4 | 'concat', 5 | 'uglify', 6 | 'cssmin', 7 | 'sails-linker:prodJs', 8 | 'sails-linker:prodStyles', 9 | 'sails-linker:devTpl', 10 | 'sails-linker:prodJsJade', 11 | 'sails-linker:prodStylesJade', 12 | 'sails-linker:devTplJade' 13 | ]); 14 | }; 15 | -------------------------------------------------------------------------------- /edX-datascrub/src/logs/buildUnknownLog.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | Created on Nov 3, 2013 4 | 5 | @author: waldo 6 | ''' 7 | 8 | import glob 9 | import json 10 | import buildWeekLog as blog 11 | 12 | if __name__ == '__main__': 13 | logFiles = glob.glob('prod*/unknown*') 14 | fLog = blog.combineLogs('unknown', logFiles) 15 | blog.writeCombLog('unknownProd.log', fLog) -------------------------------------------------------------------------------- /edX-datascrub/src/demographics/scrubstudentprofile.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | Scrub a csv file containing student profile records 4 | 5 | Removes any line in a csv file of student profile records that does 6 | not contain the correct number of entries. Useful for cleaning up 7 | incremental parsing of the XML files. 8 | 9 | Created on Feb 24, 2013 10 | 11 | @author: waldo 12 | ''' 13 | import userprofile 14 | import sys 15 | 16 | userprofile.scrubprofile(sys.argv[1], sys.argv[2]) -------------------------------------------------------------------------------- /api/services/LoggerService.js: -------------------------------------------------------------------------------- 1 | // Logging service to remove logging during a 'test' env 2 | module.exports = { 3 | info: function(msg) { 4 | if (process.env.NODE_ENV !== 'test') { 5 | sails.log.info(msg); 6 | } 7 | }, 8 | debug: function(msg) { 9 | if (process.env.NODE_ENV !== 'test') { 10 | sails.log.debug(msg); 11 | } 12 | }, 13 | error: function(msg) { 14 | if (process.env.NODE_ENV !== 'test') { 15 | sails.log.error(msg); 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /edX-datascrub/src/processAll.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys, os, csv 4 | 5 | if __name__ == '__main__': 6 | info_file = sys.argv[1] 7 | clfile = open(info_file, 'rU') 8 | clreader = csv.reader(clfile) 9 | for cname, start, end in clreader: 10 | os.system("rm dates.txt") 11 | os.system("processLogData.py " + cname + " " + start + " " + end) 12 | os.system("transformOneLog.sh " + cname + ".log " + info_file[:info_file.rfind('/')] + "/" + cname + "_axis.csv") 13 | -------------------------------------------------------------------------------- /edX-datascrub/src/getIP.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import json 4 | import csv 5 | import sys 6 | 7 | inFile = open(sys.argv[1], 'r') 8 | outfile = csv.writer(open(sys.argv[2], 'w')) 9 | ipDict = {} 10 | 11 | for line in inFile: 12 | elems = json.loads(line) 13 | ipAddr = elems['ip'] 14 | if ipAddr in ipDict: 15 | ipDict[ipAddr] += 1 16 | else: 17 | ipDict[ipAddr] = 1 18 | 19 | for ipA in sorted(iter(ipDict)): 20 | outfile.writerow([ipA, ipDict[ipA]]) 21 | -------------------------------------------------------------------------------- /tasks/config/clean.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Clean files and folders. 3 | * 4 | * --------------------------------------------------------------- 5 | * 6 | * This grunt task is configured to clean out the contents in the .tmp/public of your 7 | * sails project. 8 | * 9 | * For usage docs see: 10 | * https://github.com/gruntjs/grunt-contrib-clean 11 | */ 12 | module.exports = function(grunt) { 13 | 14 | grunt.config.set('clean', { 15 | dev: ['.tmp/public/**'], 16 | build: ['www'] 17 | }); 18 | 19 | grunt.loadNpmTasks('grunt-contrib-clean'); 20 | }; 21 | -------------------------------------------------------------------------------- /tasks/config/uglify.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Minify files with UglifyJS. 3 | * 4 | * --------------------------------------------------------------- 5 | * 6 | * Minifies client-side javascript `assets`. 7 | * 8 | * For usage docs see: 9 | * https://github.com/gruntjs/grunt-contrib-uglify 10 | * 11 | */ 12 | module.exports = function(grunt) { 13 | 14 | grunt.config.set('uglify', { 15 | dist: { 16 | src: ['.tmp/public/concat/production.js'], 17 | dest: '.tmp/public/min/production.min.js' 18 | } 19 | }); 20 | 21 | grunt.loadNpmTasks('grunt-contrib-uglify'); 22 | }; 23 | -------------------------------------------------------------------------------- /tasks/config/cssmin.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Compress CSS files. 3 | * 4 | * --------------------------------------------------------------- 5 | * 6 | * Minifies css files and places them into .tmp/public/min directory. 7 | * 8 | * For usage docs see: 9 | * https://github.com/gruntjs/grunt-contrib-cssmin 10 | */ 11 | module.exports = function(grunt) { 12 | 13 | grunt.config.set('cssmin', { 14 | dist: { 15 | src: ['.tmp/public/concat/production.css'], 16 | dest: '.tmp/public/min/production.min.css' 17 | } 18 | }); 19 | 20 | grunt.loadNpmTasks('grunt-contrib-cssmin'); 21 | }; 22 | -------------------------------------------------------------------------------- /edX-datascrub/src/logs/moveRawLogs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | Created on Feb 23, 2014 4 | 5 | Move the raw log files from their position under the various weekly directories to 6 | a full directory of all of the raw log files. 7 | ''' 8 | import shutil 9 | import glob 10 | import sys 11 | 12 | if __name__ == '__main__': 13 | if len(sys.argv) < 2: 14 | print 'Usage: moveRawLogs destdir' 15 | exit() 16 | destDir = sys.argv[1] 17 | flist = glob.glob('*/*/2014*.log') 18 | for f in flist: 19 | destf = destDir + '/' + f 20 | shutil.move(f, destf) -------------------------------------------------------------------------------- /edX-datascrub/src/getProblemIds.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import csv 4 | import sys 5 | 6 | inf = csv.reader(open(sys.argv[1], 'r')) 7 | idDict = {} 8 | 9 | for row in inf: 10 | if (row[1] == 'problem'): 11 | modId = row[2] 12 | modId = modId[modId.rfind('/')+1:] 13 | if modId not in idDict: 14 | idDict[modId] = 1 15 | else: 16 | idDict[modId] += 1 17 | 18 | outf = csv.writer(open('pidCounts.csv', 'w')) 19 | outf.writerow(['UUID', 'count']) 20 | 21 | for pId in iter(idDict): 22 | outf.writerow([pId, idDict[pId]]) 23 | 24 | 25 | -------------------------------------------------------------------------------- /edX-datascrub/src/utils.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Module for utility functions, used commonly in the rest of the HarvardX scripts and programs 3 | 4 | Utility functions that are commonly used by HarvardX scripts and programs. Rather 5 | than cut-and-paste these functions, they can be placed in this module and used 6 | throughout. 7 | ''' 8 | 9 | import os 10 | 11 | def getFileName(prompt): 12 | while (True): 13 | fname = raw_input('Please enter file name for ' + prompt + ': ') 14 | if os.path.exists(fname): 15 | return (fname) 16 | else: 17 | print('Entered name does not exist, please retry') 18 | -------------------------------------------------------------------------------- /ssl/cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIBkDCCAToCCQD5BELJ1fVSRDANBgkqhkiG9w0BAQUFADBPMQswCQYDVQQGEwJV 3 | UzELMAkGA1UECAwCQ0ExETAPBgNVBAcMCEJlcmtlbGV5MQ8wDQYDVQQKDAZtb29j 4 | UlAxDzANBgNVBAMMBm1vb2NSUDAeFw0xNDAzMTEwODU5MDdaFw00MTA3MjYwODU5 5 | MDdaME8xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTERMA8GA1UEBwwIQmVya2Vs 6 | ZXkxDzANBgNVBAoMBm1vb2NSUDEPMA0GA1UEAwwGbW9vY1JQMFwwDQYJKoZIhvcN 7 | AQEBBQADSwAwSAJBANuLjFbSr6ykK1wi2HyjPPQ8zKVfcoJ13QyGEsNVlm6KjGmj 8 | f1dCpXTGHNuEVxgNQmALKraxLxGVY77d8jGA35UCAwEAATANBgkqhkiG9w0BAQUF 9 | AANBAHsbnMH1xn5A8Or35/QOm8Oh2AMgrn8blxqKl/FuMUQH9RK1U1elVR4N1uam 10 | GYGXgmbowqA0MaArj7xIXQYd08o= 11 | -----END CERTIFICATE----- 12 | -------------------------------------------------------------------------------- /test/services/EncryptionService.spec.js: -------------------------------------------------------------------------------- 1 | var request = require('supertest'); 2 | var assert = require('assert'); 3 | var async = require('async'); 4 | var stubs = require('../stubs.js'); 5 | 6 | describe('EncryptionService', function() { 7 | var testUser; 8 | 9 | before(function () { 10 | var userParams = stubs.userStub(); 11 | User.create(userParams, function (err, user) { testUser = user}); 12 | }); 13 | 14 | describe('#encrypt()', function() { 15 | describe('TODO', function() { 16 | it('should encrypt a dataset.', function (done) { 17 | request(sails.hooks.http.app); 18 | done(); 19 | }); 20 | }); 21 | }); 22 | }); -------------------------------------------------------------------------------- /api/policies/flash.js: -------------------------------------------------------------------------------- 1 | // module.exports = function(req, res, next) { 2 | // res.locals.flash = {}; 3 | // if (!req.session.flash) return next(); 4 | // res.locals.flash = _.clone(req.session.flash); 5 | // req.session.flash = {}; 6 | // next(); 7 | // }; 8 | 9 | module.exports = function(req, res, next) { 10 | res.locals.messages = { success: [], error: [], warning: [] }; 11 | 12 | if(!req.session.messages) { 13 | req.session.messages = { success: [], error: [], warning: [] }; 14 | return next(); 15 | } 16 | res.locals.messages = _.clone(req.session.messages); 17 | 18 | // Clear flash 19 | req.session.messages = { success: [], error: [], warning: [] }; 20 | return next(); 21 | }; -------------------------------------------------------------------------------- /tasks/config/mochaTest.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Compiles LESS files into CSS. 3 | * 4 | * --------------------------------------------------------------- 5 | * 6 | * Only the `assets/styles/importer.less` is compiled. 7 | * This allows you to control the ordering yourself, i.e. import your 8 | * dependencies, mixins, variables, resets, etc. before other stylesheets) 9 | * 10 | * For usage docs see: 11 | * https://github.com/gruntjs/grunt-contrib-less 12 | */ 13 | module.exports = function(grunt) { 14 | 15 | grunt.config.set('mochaTest', { 16 | dev: { 17 | options: { 18 | reporter: 'spec' 19 | }, 20 | src: ['tests/grunt/**/*.spec.js'] 21 | } 22 | }); 23 | 24 | grunt.loadNpmTasks('grunt-mocha-test'); 25 | }; 26 | -------------------------------------------------------------------------------- /tasks/config/sync.js: -------------------------------------------------------------------------------- 1 | /** 2 | * A grunt task to keep directories in sync. It is very similar to grunt-contrib-copy 3 | * but tries to copy only those files that has actually changed. 4 | * 5 | * --------------------------------------------------------------- 6 | * 7 | * Synchronize files from the `assets` folder to `.tmp/public`, 8 | * smashing anything that's already there. 9 | * 10 | * For usage docs see: 11 | * https://github.com/tomusdrw/grunt-sync 12 | * 13 | */ 14 | module.exports = function(grunt) { 15 | 16 | grunt.config.set('sync', { 17 | dev: { 18 | files: [{ 19 | cwd: './assets', 20 | src: ['**/*.!(coffee)'], 21 | dest: '.tmp/public' 22 | }] 23 | } 24 | }); 25 | 26 | grunt.loadNpmTasks('grunt-sync'); 27 | }; 28 | -------------------------------------------------------------------------------- /assets/templates/errorHandler.ejs: -------------------------------------------------------------------------------- 1 | <% if (messages && messages['error'].length > 0) { %> 2 |
3 | <% messages['error'].forEach(function(message) { %> 4 | <%= message %> 5 |
6 | <% }); %> 7 |
8 |
9 | <% } %> 10 | <% if (messages && messages['warning'].length > 0) { %> 11 |
12 | <% messages['warning'].forEach(function(message) { %> 13 | <%= message %> 14 |
15 | <% }); %> 16 |
17 |
18 | <% } %> 19 | <% if (messages && messages['success'].length > 0) { %> 20 |
21 | <% messages['success'].forEach(function(message) { %> 22 | <%= message %> 23 |
24 | <% }); %> 25 |
26 |
27 | <% } %> -------------------------------------------------------------------------------- /api/policies/isAuthenticated.js: -------------------------------------------------------------------------------- 1 | /** 2 | * isAuthenticated 3 | * 4 | * @module :: Policy 5 | * @description :: Simple policy to allow any authenticated user 6 | * Assumes that your login action in one of your controllers sets `req.session.authenticated = true;` 7 | * @docs :: http://sailsjs.org/#!documentation/policies 8 | * 9 | */ 10 | module.exports = function(req, res, next) { 11 | 12 | // User is allowed, proceed to the next policy, 13 | // or if this is the last policy, the controller 14 | if (req.session.authenticated && req.session.user) { 15 | return next(); 16 | } 17 | 18 | // User is not allowed 19 | // (default res.forbidden() behavior can be overridden in `config/403.js`) 20 | return res.redirect('/login'); 21 | }; 22 | -------------------------------------------------------------------------------- /tasks/config/less.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Compiles LESS files into CSS. 3 | * 4 | * --------------------------------------------------------------- 5 | * 6 | * Only the `assets/styles/importer.less` is compiled. 7 | * This allows you to control the ordering yourself, i.e. import your 8 | * dependencies, mixins, variables, resets, etc. before other stylesheets) 9 | * 10 | * For usage docs see: 11 | * https://github.com/gruntjs/grunt-contrib-less 12 | */ 13 | module.exports = function(grunt) { 14 | 15 | grunt.config.set('less', { 16 | dev: { 17 | files: [{ 18 | expand: true, 19 | cwd: 'assets/styles/', 20 | src: ['importer.less'], 21 | dest: '.tmp/public/styles/', 22 | ext: '.css' 23 | }] 24 | } 25 | }); 26 | 27 | grunt.loadNpmTasks('grunt-contrib-less'); 28 | }; 29 | -------------------------------------------------------------------------------- /api/services/SessionService.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | createSession: function(req, user) { 3 | sails.log.debug('User ' + user.id + ' logged in'); 4 | FlashService.success(req, 'Successfully logged in.'); 5 | req.session.user = user; 6 | var oldDateObj = new Date(); 7 | var newDateObj = new Date(oldDateObj.getTime() + sails.config.cookieExpiration); // one hour before expiring 8 | req.session.cookie.expires = newDateObj; 9 | req.session.authenticated = true; 10 | }, 11 | 12 | destroySession: function(req, res) { 13 | if (req.session.authenticated) { 14 | req.session.user = null; 15 | req.session.authenticated = false; 16 | return res.redirect(AuthService.logoutRoute()); 17 | } else { 18 | return res.redirect('/home'); 19 | } 20 | } 21 | } -------------------------------------------------------------------------------- /api/policies/sessionAuth.js: -------------------------------------------------------------------------------- 1 | /** 2 | * sessionAuth 3 | * 4 | * @module :: Policy 5 | * @description :: Simple policy to allow any authenticated user 6 | * Assumes that your login action in one of your controllers sets `req.session.authenticated = true;` 7 | * @docs :: http://sailsjs.org/#!documentation/policies 8 | * 9 | */ 10 | module.exports = function(req, res, next) { 11 | 12 | // User is allowed, proceed to the next policy, 13 | // or if this is the last policy, the controller 14 | // if (req.session.authenticated) { 15 | // return next(); 16 | // } 17 | 18 | // User is not allowed 19 | // (default res.forbidden() behavior can be overridden in `config/403.js`) 20 | // return res.forbidden('You are not permitted to perform this action.'); 21 | }; 22 | -------------------------------------------------------------------------------- /api/models/Job.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Job.js 3 | * 4 | * @description :: TODO: You might write a short summary of how this model works and what it represents here. 5 | * @docs :: http://sailsjs.org/#!documentation/models 6 | */ 7 | 8 | module.exports = { 9 | 10 | attributes: { 11 | jobName: { 12 | type: 'STRING', 13 | required: true 14 | }, 15 | queueJobId: { 16 | type: 'STRING', 17 | required: true, 18 | unique: true 19 | }, 20 | jobType: { 21 | type: 'STRING' 22 | }, 23 | status: { 24 | type: 'STRING', 25 | required: true 26 | }, 27 | startTime: { 28 | type: 'DATETIME', 29 | defaultsTo: null 30 | }, 31 | endTime: { 32 | type: 'DATETIME', 33 | defaultsTo: null 34 | } 35 | } 36 | }; 37 | 38 | -------------------------------------------------------------------------------- /edX-datascrub/src/logs/makeUserIpList.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | Created on Nov 3, 2013 4 | 5 | @author: waldo 6 | ''' 7 | 8 | import sys 9 | import json 10 | import csv 11 | 12 | if __name__ == '__main__': 13 | logFile = sys.argv[1] 14 | nameIpDict = {} 15 | fin = open(logFile, 'r') 16 | for line in fin: 17 | dline = json.loads(line) 18 | uname = dline['username'] 19 | ipAddr = dline['ip'] 20 | if uname in nameIpDict: 21 | if ipAddr not in nameIpDict[uname]: 22 | nameIpDict[uname].append(ipAddr) 23 | else: 24 | nameIpDict[uname] = [ipAddr] 25 | outfile = csv.writer(open(sys.argv[2], 'w')) 26 | for n in sorted(iter(nameIpDict)): 27 | outfile.writerow([n, nameIpDict[n]]) 28 | 29 | -------------------------------------------------------------------------------- /tasks/config/concat.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Concatenate files. 3 | * 4 | * --------------------------------------------------------------- 5 | * 6 | * Concatenates files javascript and css from a defined array. Creates concatenated files in 7 | * .tmp/public/contact directory 8 | * [concat](https://github.com/gruntjs/grunt-contrib-concat) 9 | * 10 | * For usage docs see: 11 | * https://github.com/gruntjs/grunt-contrib-concat 12 | */ 13 | module.exports = function(grunt) { 14 | 15 | grunt.config.set('concat', { 16 | js: { 17 | src: require('../pipeline').jsFilesToInject, 18 | dest: '.tmp/public/concat/production.js' 19 | }, 20 | css: { 21 | src: require('../pipeline').cssFilesToInject, 22 | dest: '.tmp/public/concat/production.css' 23 | } 24 | }); 25 | 26 | grunt.loadNpmTasks('grunt-contrib-concat'); 27 | }; 28 | -------------------------------------------------------------------------------- /edX-datascrub/src/deleteUnchangedFiles.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | Created on Dec 24, 2013 5 | 6 | @author: waldo 7 | ''' 8 | import filecmp 9 | import glob 10 | import os 11 | import sys 12 | 13 | 14 | if __name__ == '__main__': 15 | print len(sys.argv) 16 | if len(sys.argv) == 2: 17 | cmpDir = sys.argv[1] 18 | else: 19 | print 'Usage: deleteUnchangedFile destDir' 20 | sys.exit(1) 21 | 22 | allFiles = glob.glob('*/*/*') 23 | unchangedListf = open('UnchangedFileList', 'w') 24 | unchangedListf.write(cmpDir+'\n') 25 | for f in allFiles: 26 | f2 = cmpDir + '/' + f 27 | if os.path.exists(f2): 28 | if filecmp.cmp(f, f2): 29 | unchangedListf.write(f + '\n') 30 | #os.remove(f2) 31 | unchangedListf.close() 32 | -------------------------------------------------------------------------------- /config/models.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Default model configuration 3 | * (sails.config.models) 4 | * 5 | * Unless you override them, the following properties will be included 6 | * in each of your models. 7 | * 8 | * For more info on Sails models, see: 9 | * http://sailsjs.org/#/documentation/concepts/ORM 10 | */ 11 | 12 | module.exports.models = { 13 | 14 | /*************************************************************************** 15 | * * 16 | * Your app's default connection. i.e. the name of one of your app's * 17 | * connections (see `config/connections.js`) * 18 | * * 19 | ***************************************************************************/ 20 | 21 | connection: 'mysql' 22 | }; 23 | -------------------------------------------------------------------------------- /edX-datascrub/src/logs/moveWeeklyLogs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | Move the weekly logs created by separateClassLogs to the directory for the course, 4 | and rename the log WeekLog. 5 | Created on Feb 23, 2014 6 | 7 | @author: waldo 8 | ''' 9 | import sys 10 | import shutil 11 | import glob 12 | 13 | if __name__ == '__main__': 14 | if len(sys.argv) < 3: 15 | print 'Usage: moveWeeklyLogs srcDir destDir' 16 | exit() 17 | 18 | srcDir = sys.argv[1] 19 | destDir = sys.argv[2] 20 | 21 | fileList = glob.glob(srcDir + '/*.log') 22 | for fname in fileList: 23 | if 'unknown' not in fname: 24 | cname = fname[:fname.find('.log')] 25 | destFile = destDir + '/' + cname + '/WeekLog' 26 | try: 27 | shutil.move(fname, destFile) 28 | except: 29 | print 'unable to move file', fname, 'to', destFile -------------------------------------------------------------------------------- /config/env/test.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Test environment settings 3 | * 4 | * This file can include shared settings for a test team, 5 | * such as API keys or remote database passwords. If you're using 6 | * a version control solution for your Sails app, this file will 7 | * be committed to your repository unless you add it to your .gitignore 8 | * file. If your repository will be publicly viewable, don't add 9 | * any private information to this file! 10 | * 11 | */ 12 | 13 | module.exports = { 14 | 15 | /*************************************************************************** 16 | * Set the default database connection for models in the test * 17 | * environment (see config/connections.js and config/models.js ) * 18 | ***************************************************************************/ 19 | 20 | models: { 21 | connection: 'mysql', 22 | migrate: 'alter' 23 | } 24 | 25 | }; 26 | -------------------------------------------------------------------------------- /edX-datascrub/src/logs/buildFullLog.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | Combine all of the log files in a particular directory into a single log file 4 | 5 | 6 | ''' 7 | 8 | import json 9 | import os 10 | 11 | lineDict = {} 12 | dc = json.JSONDecoder() 13 | 14 | dirName = os.getcwd() 15 | className = dirName[dirName.rindex('/')+1:] 16 | logList = os.listdir(dirName) 17 | outfile = open(className + 'FullLog', 'w') 18 | 19 | for fname in logList: 20 | inf = open(fname, 'r') 21 | for line in inf: 22 | dcl = dc.decode(line) 23 | ts = dcl['time'] 24 | if ts not in lineDict: 25 | lineDict[ts] = [line] 26 | else: 27 | lineDict[ts].append(line) 28 | inf.close() 29 | 30 | i = 0 31 | for d in sorted(iter(lineDict)): 32 | for l in lineDict[d]: 33 | i += 1 34 | outfile.write(l) 35 | 36 | print 'wrote ' + str(i) + ' lines to output file' 37 | outfile.close() 38 | -------------------------------------------------------------------------------- /api/models/Report.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Report.js 3 | * 4 | * @description :: TODO: You might write a short summary of how this model works and what it represents here. 5 | * @docs :: http://sailsjs.org/#!documentation/models 6 | */ 7 | 8 | module.exports = { 9 | 10 | attributes: { 11 | firstName: { 12 | type: 'STRING', 13 | required: true 14 | }, 15 | lastName: { 16 | type: 'STRING', 17 | required: true 18 | }, 19 | emailAddress: { 20 | type: 'EMAIL', 21 | required: true 22 | }, 23 | type: { 24 | type: 'STRING', 25 | required: true 26 | }, 27 | emailMessage: { 28 | type: 'TEXT', 29 | required: true 30 | }, 31 | status: { 32 | // Statuses include: 'OPEN', 'IN-PROGRESS', 'CLOSED', 'RESOLVED' 33 | type: 'STRING', 34 | defaultsTo: 'OPEN' 35 | }, 36 | user: { 37 | model: 'user' 38 | } 39 | } 40 | }; 41 | 42 | -------------------------------------------------------------------------------- /edX-datascrub/src/convertfiles/sqltocsv.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | Converts tab-separated lines from the data dumps into a csv file. 4 | The program takes an sql file and the name of the csv file to 5 | be produced. 6 | 7 | Created on Apr 7, 2013 8 | 9 | @author: waldo 10 | ''' 11 | 12 | 13 | import sys 14 | import csv 15 | 16 | def convertFile(fileNameIn, fileNameOut): 17 | f1 = open(fileNameIn, 'r') 18 | f2 = open(fileNameOut, 'w') 19 | f3 = csv.writer(f2) 20 | 21 | for line in f1: 22 | f3.writerow(line[:-1].split('\t')) 23 | 24 | f1.close() 25 | f2.close() 26 | 27 | if __name__ == '__main__': 28 | if (len(sys.argv) < 3): 29 | print ('Usage: sqltocsv.py file1 file2 where') 30 | print ('file1 is an existing .sql file from edx and') 31 | print ('file2 is the name of the .csv file to produce') 32 | 33 | convertFile(sys.argv[1], sys.argv[2]) 34 | -------------------------------------------------------------------------------- /edX-datascrub/src/scrapeEmail.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import user 4 | import certificates 5 | import os 6 | import csv 7 | 8 | def getFileName(prompt): 9 | while (True): 10 | fname = raw_input("Please enter file name for " + prompt + ' : ') 11 | if os.path.exists(fname): 12 | return fname 13 | else: 14 | print ("file entered does not exist, please retry") 15 | 16 | f1name = getFileName('Enter name of the user file') 17 | f1 = csv.reader(open(f1name, 'r')) 18 | 19 | f2name = getFileName('Enter the name of the certificates file') 20 | f2 = csv.reader(open(f2name, 'r')) 21 | 22 | udict = user.builddict(f1) 23 | cdict = certificates.builddict(f2) 24 | 25 | out1 = open('allmail', 'w') 26 | out2 = open('certMail', 'w') 27 | 28 | for u in iter(udict): 29 | out1.write(udict[u].email + '\n') 30 | if u in cdict: 31 | out2.write(udict[u].email + '\n') 32 | 33 | 34 | -------------------------------------------------------------------------------- /api/policies/hasID.js: -------------------------------------------------------------------------------- 1 | /** 2 | * hasID 3 | * 4 | * @module :: Policy 5 | * @description :: Simple policy to allow any authenticated user 6 | * Assumes that your login action in one of your controllers sets `req.session.authenticated = true;` 7 | * @docs :: http://sailsjs.org/#!documentation/policies 8 | * 9 | */ 10 | module.exports = function(req, res, next) { 11 | 12 | // User is allowed, proceed to the next policy, 13 | // or if this is the last policy, the controller 14 | if (req.session.authenticated && req.session.user && req.param('id')) { 15 | return next(); 16 | } 17 | 18 | // User is not allowed 19 | // (default res.forbidden() behavior can be overridden in `config/403.js`) 20 | //return res.forbidden('You are not permitted to perform this action.'); 21 | sails.log.error("Error occurred while attempting to access page without passing in 'id' param"); 22 | return res.redirect('/'); 23 | }; -------------------------------------------------------------------------------- /test/services/SessionService.spec.js: -------------------------------------------------------------------------------- 1 | var request = require('supertest'); 2 | var assert = require('assert'); 3 | var async = require('async'); 4 | var stubs = require('../stubs.js'); 5 | 6 | describe('SessionService', function() { 7 | var testUser; 8 | 9 | before(function () { 10 | var userParams = stubs.userStub(); 11 | User.create(userParams, function (err, user) { testUser = user}); 12 | }); 13 | 14 | describe('#createSession()', function() { 15 | describe('TODO after modularizing authentication.', function() { 16 | it('should create a new session.', function (done) { 17 | request(sails.hooks.http.app); 18 | done(); 19 | }); 20 | }); 21 | }); 22 | 23 | describe('#destroySession()', function() { 24 | describe('TODO after modularizing authentication.', function() { 25 | it('should destroy the existing session.', function (done) { 26 | done(); 27 | }); 28 | }); 29 | }); 30 | }); -------------------------------------------------------------------------------- /tasks/config/coffee.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Compile CoffeeScript files to JavaScript. 3 | * 4 | * --------------------------------------------------------------- 5 | * 6 | * Compiles CoffeeScript files from `assets/js` into Javascript and places them into 7 | * `.tmp/public/js` directory. 8 | * 9 | * For usage docs see: 10 | * https://github.com/gruntjs/grunt-contrib-coffee 11 | */ 12 | module.exports = function(grunt) { 13 | 14 | grunt.config.set('coffee', { 15 | dev: { 16 | options: { 17 | bare: true, 18 | sourceMap: true, 19 | sourceRoot: './' 20 | }, 21 | files: [{ 22 | expand: true, 23 | cwd: 'assets/js/', 24 | src: ['**/*.coffee'], 25 | dest: '.tmp/public/js/', 26 | ext: '.js' 27 | }, { 28 | expand: true, 29 | cwd: 'assets/js/', 30 | src: ['**/*.coffee'], 31 | dest: '.tmp/public/js/', 32 | ext: '.js' 33 | }] 34 | } 35 | }); 36 | 37 | grunt.loadNpmTasks('grunt-contrib-coffee'); 38 | }; 39 | -------------------------------------------------------------------------------- /api/policies/isAdmin.js: -------------------------------------------------------------------------------- 1 | /** 2 | * isAdmin 3 | * 4 | * @module :: Policy 5 | * @description :: Simple policy to allow any authenticated user 6 | * Assumes that your login action in one of your controllers sets `req.session.authenticated = true;` 7 | * @docs :: http://sailsjs.org/#!documentation/policies 8 | * 9 | */ 10 | module.exports = function(req, res, next) { 11 | 12 | // User is allowed, proceed to the next policy, 13 | // or if this is the last policy, the controller 14 | if (req.session.authenticated && req.session.user && req.session.user.admin) { 15 | return next(); 16 | } 17 | var requireAdminError = 'You must be an admin.'; 18 | req.session.messages = { error: [requireAdminError] }; 19 | 20 | // User is not allowed 21 | // (default res.forbidden() behavior can be overridden in `config/403.js`) 22 | //return res.forbidden('You are not permitted to perform this action.'); 23 | return res.redirect('/login'); 24 | }; -------------------------------------------------------------------------------- /config/env/development.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Development environment settings 3 | * 4 | * This file can include shared settings for a development team, 5 | * such as API keys or remote database passwords. If you're using 6 | * a version control solution for your Sails app, this file will 7 | * be committed to your repository unless you add it to your .gitignore 8 | * file. If your repository will be publicly viewable, don't add 9 | * any private information to this file! 10 | * 11 | */ 12 | 13 | module.exports = { 14 | 15 | /*************************************************************************** 16 | * Set the default database connection for models in the development * 17 | * environment (see config/connections.js and config/models.js ) * 18 | ***************************************************************************/ 19 | 20 | models: { 21 | connection: 'mysql', 22 | migrate: 'alter' /** SET TO 'safe' DURING PRODUCTION **/ 23 | } 24 | 25 | }; 26 | -------------------------------------------------------------------------------- /tasks/config/copy.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copy files and folders. 3 | * 4 | * --------------------------------------------------------------- 5 | * 6 | * # dev task config 7 | * Copies all directories and files, exept coffescript and less fiels, from the sails 8 | * assets folder into the .tmp/public directory. 9 | * 10 | * # build task config 11 | * Copies all directories nd files from the .tmp/public directory into a www directory. 12 | * 13 | * For usage docs see: 14 | * https://github.com/gruntjs/grunt-contrib-copy 15 | */ 16 | module.exports = function(grunt) { 17 | 18 | grunt.config.set('copy', { 19 | dev: { 20 | files: [{ 21 | expand: true, 22 | cwd: './assets', 23 | src: ['**/*.!(coffee|less)'], 24 | dest: '.tmp/public' 25 | }] 26 | }, 27 | build: { 28 | files: [{ 29 | expand: true, 30 | cwd: '.tmp/public', 31 | src: ['**/*'], 32 | dest: 'www' 33 | }] 34 | } 35 | }); 36 | 37 | grunt.loadNpmTasks('grunt-contrib-copy'); 38 | }; 39 | -------------------------------------------------------------------------------- /edX-datascrub/src/course_struct.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import csv 4 | import json 5 | import sys 6 | 7 | class course_struct: 8 | 9 | def __init__(self, category, children, metad): 10 | self.category = category 11 | self.children = children 12 | self.metad = metad 13 | 14 | def getChildNames(dict, entr): 15 | ent = dict[entr] 16 | children = ent['children'] 17 | retlist = [] 18 | for c in children: 19 | retlist.append(dict[c]['metadata']['display_name']) 20 | return retlist 21 | 22 | def save_csv(ctree, f): 23 | 24 | f.writerow(['id', 'category', 'metadata', 'children']) 25 | for c in iter(ctree): 26 | chNames = getChildNames(ctree, c) 27 | f.writerow([ c, ctree[c]['category'], ctree[c]['metadata'], chNames]) 28 | 29 | inf = open(sys.argv[1], 'r') 30 | inline = inf.readline() 31 | ctdict = json.loads(inline) 32 | 33 | outf = csv.writer(open(sys.argv[2], 'w')) 34 | save_csv(ctdict, outf) 35 | 36 | 37 | -------------------------------------------------------------------------------- /edX-datascrub/src/findEventTypes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | '''Find all of the event types in a JSON-style log dump 4 | 5 | This script will take as input an edX activity log file and find all 6 | of the event types in that file. The resulting event types will be 7 | written to a file, one type per line, and the number of event types 8 | (which may be very large) will be displayed at the end of the 9 | run. The script will deal with non-ascii characters, writing the 10 | output file as a latin-1 encoding 11 | 12 | ''' 13 | import json 14 | import sys 15 | import codecs 16 | 17 | infile = open(sys.argv[1], 'r') 18 | outfile = codecs.open(sys.argv[2],'w', 'latin-1', 'replace') 19 | typelist = set() 20 | i = 0 21 | 22 | for line in infile: 23 | elems = json.loads(line) 24 | etype = elems['event_type'] 25 | if etype not in typelist: 26 | typelist.add(etype) 27 | outfile.write(etype + '\n') 28 | i = i + 1 29 | 30 | print i 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /views/analytic/edit.ejs: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Analytics

4 | <% if (messages && messages['error'].length > 0) { %> 5 |
6 | <% messages['error'].forEach(function(message) { %> 7 | <%= message %> 8 |
9 | <% }); %> 10 |
11 |
12 | <% } %> 13 | <% if (messages && messages['warning'].length > 0) { %> 14 |
15 | <% messages['warning'].forEach(function(message) { %> 16 | <%= message %> 17 |
18 | <% }); %> 19 |
20 |
21 | <% } %> 22 | <% if (messages && messages['success'].length > 0) { %> 23 |
24 | <% messages['success'].forEach(function(message) { %> 25 | <%= message %> 26 |
27 | <% }); %> 28 |
29 |
30 | <% } %> 31 |
32 |
33 | 34 | -------------------------------------------------------------------------------- /assets/styles/importer.less: -------------------------------------------------------------------------------- 1 | /** 2 | * importer.less 3 | * 4 | * By default, new Sails projects are configured to compile this file 5 | * from LESS to CSS. Unlike CSS files, LESS files are not compiled and 6 | * included automatically unless they are imported below. 7 | * 8 | * The LESS files imported below are compiled and included in the order 9 | * they are listed. Mixins, variables, etc. should be imported first 10 | * so that they can be accessed by subsequent LESS stylesheets. 11 | * 12 | * (Just like the rest of the asset pipeline bundled in Sails, you can 13 | * always omit, customize, or replace this behavior with SASS, SCSS, 14 | * or any other Grunt tasks you like.) 15 | */ 16 | 17 | 18 | 19 | // For example: 20 | // 21 | // @import 'variables/colors.less'; 22 | // @import 'mixins/foo.less'; 23 | // @import 'mixins/bar.less'; 24 | // @import 'mixins/baz.less'; 25 | // 26 | // @import 'styleguide.less'; 27 | // @import 'pages/login.less'; 28 | // @import 'pages/signup.less'; 29 | // 30 | // etc. 31 | -------------------------------------------------------------------------------- /edX-datascrub/src/checkData/getCertsFromId.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | A simple interactive program to compare the ids in a file with those in a 4 | certificates file 5 | 6 | This program will prompt the user for the name of a csv file containing 7 | only user ids, and a csv of a certificates file, and see if there are any 8 | ids in the first file that correspond to entries in the certificates file. 9 | 10 | ''' 11 | 12 | import csv 13 | import sys 14 | import certificates 15 | import utils 16 | 17 | if len(sys.argv) > 2: 18 | f1name = sys.argv[1] 19 | f2name = sys.argv[2] 20 | else: 21 | f1name = utils.getFileName('Enter csv file with ids : ') 22 | f2name = utils.getFileName('Enter certificates csv file name : ') 23 | 24 | f1 = csv.reader(open(f1name, 'r')) 25 | f2 = csv.reader(open(f2name, 'r')) 26 | certdict = certificates.builddict(f2) 27 | 28 | f1.readrow() 29 | 30 | for [ident] in f1: 31 | if ident in certdict: 32 | print 'found new identifier ' + ident + ' in certificates file' 33 | 34 | -------------------------------------------------------------------------------- /edX-datascrub/src/checkData/certsAndusers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | Looks for differences between the user listed in the users file and those in the certificates file 5 | 6 | In particular, looks for any users not found in the users file who have received a certificate. 7 | ''' 8 | import csv 9 | import certificates 10 | import user 11 | 12 | ufile = csv.reader(open('users.csv', 'r')) 13 | udict = user.builddict(ufile) 14 | cfile = csv.reader(open('certificates.csv', 'r')) 15 | cDict = certificates.builddict(cfile) 16 | 17 | certsMissing = [] 18 | 19 | for c in iter(cDict): 20 | if (cDict[c].status == 'downloadable') and (c not in udict): 21 | certsMissing.append(c) 22 | 23 | if len(certsMissing) > 0: 24 | print 'found ' + str(len(certsMissing)) + ' certificates with no associated user' 25 | outfile = csv.writer(open('certsAndusers.csv', 'w')) 26 | outfile.writerow(['Missing user ids that have certificates']) 27 | for u in certsMissing: 28 | outfile.writerow([u]) 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /api/controllers/NoticeController.js: -------------------------------------------------------------------------------- 1 | /** 2 | * NoticeController 3 | * 4 | * @description :: Server-side logic for managing notices 5 | * @help :: See http://links.sailsjs.org/docs/controllers 6 | */ 7 | 8 | module.exports = { 9 | create: function(req, res) { 10 | Notice.create(req.params.all(), function(err, notice) { 11 | if (err || !notice) { 12 | sails.log.error(err); 13 | FlashService.error(req, "Unable to create notice."); 14 | return res.redirect('/admin/manage_notices'); 15 | } 16 | 17 | FlashService.success(req, 'Successfully created a notice.'); 18 | return res.redirect('/admin/manage_notices'); 19 | }); 20 | }, 21 | destroy: function(req, res) { 22 | Notice.destroy(req.param('id'), function(err) { 23 | if (err) { 24 | FlashService.error(req, "Unable to destroy notice."); 25 | } else { 26 | FlashService.success(req, "Successfully destroyed notice."); 27 | } 28 | return res.redirect('/admin/manage_notices'); 29 | }); 30 | } 31 | }; 32 | 33 | -------------------------------------------------------------------------------- /edX-datascrub/src/findBrowserEventTypes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | '''Find all of the event types in a JSON-style log dump 4 | 5 | This script will take as input an edX activity log file and find all 6 | of the event types in that file. The resulting event types will be 7 | written to a file, one type per line, and the number of event types 8 | (which may be very large) will be displayed at the end of the 9 | run. The script will deal with non-ascii characters, writing the 10 | output file as a latin-1 encoding 11 | 12 | ''' 13 | import json 14 | import sys 15 | import codecs 16 | 17 | infile = open(sys.argv[1], 'r') 18 | outfile = codecs.open(sys.argv[2],'w', 'latin-1', 'replace') 19 | typelist = set() 20 | i = 0 21 | 22 | for line in infile: 23 | elems = json.loads(line) 24 | etype = elems['event_type'] 25 | if (elems['event_source'] == 'browser'): 26 | if etype not in typelist: 27 | typelist.add(etype) 28 | outfile.write(etype + '\n') 29 | i = i + 1 30 | 31 | print i 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /edX-datascrub/src/logs/distillUnknownLabels.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | Created on Oct 26, 2013 4 | Goes through all of the files containing log entries that were not able 5 | to be classified by course, and creates a list of the event types and 6 | the counts for that event type. Will print out the event types in increasing 7 | order so that the most common will be at the end. 8 | @author: waldo 9 | ''' 10 | import json 11 | import glob 12 | 13 | def buildList(fname, rdict): 14 | with open (fname, 'r') as fin: 15 | for line in fin: 16 | lstr = json.loads(line) 17 | st = lstr['event_type'] 18 | if st not in rdict: 19 | rdict[st] = 1 20 | else: 21 | rdict[st] += 1 22 | return rdict 23 | 24 | if __name__ == '__main__': 25 | ukdict = {} 26 | fname = glob.glob('*/unknown*.log') 27 | for n in fname: 28 | ukdict = buildList(n, ukdict) 29 | 30 | s = sorted(ukdict.items(), key = lambda(k,v):(v,k)) 31 | for i in s: 32 | print i 33 | 34 | 35 | -------------------------------------------------------------------------------- /config/bootstrap.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Bootstrap 3 | * (sails.config.bootstrap) 4 | * 5 | * An asynchronous bootstrap function that runs before your Sails app gets lifted. 6 | * This gives you an opportunity to set up your data model, run jobs, or perform some special logic. 7 | * 8 | * For more information on bootstrapping your app, check out: 9 | * http://sailsjs.org/#/documentation/reference/sails.config/sails.config.bootstrap.html 10 | */ 11 | 12 | module.exports.bootstrap = function(cb) { 13 | 14 | // Prevents logging in the 'test' env 15 | if (process.env.NODE_ENV == 'test') { 16 | sails.log.debug = LoggerService.debug; 17 | sails.log.info = LoggerService.info; 18 | sails.log.error = LoggerService.error; 19 | } 20 | 21 | // Launch message 22 | sails.log.info('moocRP: Learning Analytics Platform'); 23 | 24 | 25 | // Launch jobs queue 26 | QueueService.launchQueue(); 27 | 28 | // It's very important to trigger this callback method when you are finished 29 | // with the bootstrap! (otherwise your server will never lift, since it's waiting on the bootstrap) 30 | cb(); 31 | }; 32 | -------------------------------------------------------------------------------- /config/ssl/server.csr: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE REQUEST----- 2 | MIICyDCCAbACAQAwgYIxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTERMA8GA1UE 3 | BwwIQmVya2VsZXkxDzANBgNVBAoMBm1vb2NSUDEPMA0GA1UECwwGbW9vY1JQMQ8w 4 | DQYDVQQDDAZtb29jUlAxIDAeBgkqhkiG9w0BCQEWEWtrYW9AYmVya2VsZXkuZWR1 5 | MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuIxAdZjcW4Tx5Z91PPaC 6 | ZqxKHdsMWQ9NV12+3V+j/lMxXGzvRaMuo7zaL7eHsO9fgG4n0lkxS2YrSdMaO2Rt 7 | eTglmHoTmEGTmIF+WjvOnGxvsgeAcA3qwVfImw/8mL337tqmoS1TW1cMo+Wqa2y/ 8 | ERuu7hrgAmRpcUIk879iJKEZZPe5+AM8agKBrmzWOkbtqYw8Aj6/Ftbl/iJmjxD7 9 | tKwAeDLbDUoglSI6lCmcGDpHIVQNLmiWFM9WsxdST3iS7IYZP49X055mm7mfEFgK 10 | NaxLyA+89nKSXoPKRmSM2bbqyB8yERh93RZbCX6iupJtvgNzZ/DLCQ6AonLnHKwR 11 | BwIDAQABoAAwDQYJKoZIhvcNAQELBQADggEBAFcaWSODL1drRmQyAuIrOfYDqVK9 12 | gAnSf8noLY2XSLQ1rlAD+0jiBXooLqTCSGrRiLHmvSWY4dnGG/UBXBnjdABpneBc 13 | IxSnncyHiF2iuTHSkUhhC490qkqA8v7BYBlisGVc6JYyq0smDiaBu1dLXnarwK90 14 | 3TizOUIX+SGYXKroLn5HoMAubbj8QSuV1el54ir/k3oCqg5yCC6WAAAe9paim4Aa 15 | 9UQGi4UTN7EEHRnsczfcDWj2fVQ6S2Ex9+34i1MBpGdVV46dVe7TrXMAX+M44+My 16 | H1FEI/L8dO9wrzLpyJr5y0fQs6G5zUNIsEc3/TcgdeN+2XOOeTDbt5TeBlg= 17 | -----END CERTIFICATE REQUEST----- 18 | -------------------------------------------------------------------------------- /edX-datascrub/src/logs/selectLogRange.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | Created on Dec 22, 2013 5 | 6 | @author: waldo 7 | ''' 8 | 9 | import shutil 10 | import sys 11 | import glob 12 | 13 | def getFiles(srcDir, fromDate, toDate): 14 | retList = [] 15 | candidates = glob.glob(srcDir + '/*/*.log') 16 | for f in candidates: 17 | fname = f[f.rfind('/')+1:] 18 | fdate = fname[:fname.find('_')] 19 | if (fromDate <= fdate) and (fdate <= toDate): 20 | retList.append(f) 21 | return retList 22 | 23 | if __name__ == '__main__': 24 | if len(sys.argv) < 4: 25 | print "usage: selectLogRange srcDir destDir fromDate [toDate]" 26 | sys.exit(1) 27 | 28 | srcDir = sys.argv[1] 29 | print srcDir 30 | destDir = sys.argv[2] 31 | print destDir 32 | fromDate = sys.argv[3] 33 | print fromDate 34 | if len(sys.argv) > 4: 35 | toDate = sys.argv[4] 36 | else: 37 | toDate = '2020' 38 | print toDate 39 | copyList = getFiles(srcDir, fromDate, toDate) 40 | print copyList 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /views/home/about.ejs: -------------------------------------------------------------------------------- 1 |
2 |
3 |

About

4 |
5 |
6 |
7 | 8 |
9 |

About moocRP

10 |
11 |

moocRP is currently being developed as part of a research project at University of California, Berkeley. moocRP aims to be a platform where research conducted on large amounts of MOOC data can be shared, through a revolutionizing way of sharing data analytics and visualizations.

12 |

moocRP serves as a platform to bring the educational research community together and further the potential of research discoveries through the sharing of research results, tackling the problem of "reproducible research."

13 |
14 |
15 |
16 | 17 |
18 | 19 |
20 |

Team Members

21 |
22 |

23 | Lead Developer: 24 |
25 | Kevin Kao 26 |

27 |

28 | Professor & Advisor: 29 |
30 | Zach Pardos 31 |

32 |

33 | Advisor: 34 |
35 | Derrick Coetzee 36 |

37 |
38 |
-------------------------------------------------------------------------------- /edX-datascrub/src/logs/countInteract.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import json 4 | import sys 5 | import csv 6 | 7 | class InteractRec: 8 | def __init__(self): 9 | self.totalInt = 0 10 | self.diffDays = 0 11 | self.idic = {} 12 | 13 | def addRec(self, when): 14 | self.totalInt += 1 15 | if (when in self.idic): 16 | self.idic[when] += 1 17 | else: 18 | self.diffDays += 1 19 | self.idic[when] = 1 20 | 21 | f1 = open(sys.argv[1], 'r') 22 | f2 = csv.writer(open(sys.argv[2] + '.csv', 'w')) 23 | f3 = csv.writer(open(sys.argv[2] + 'detail.csv', 'w')) 24 | dc = json.JSONDecoder() 25 | interAct = {} 26 | 27 | for line in f1: 28 | dcl = dc.decode(line) 29 | user = dcl['username'] 30 | if (user not in interAct): 31 | interAct[user] = InteractRec() 32 | dt = dcl['time'] 33 | d = dt[ :dt.find('T')] 34 | interAct[user].addRec(d) 35 | 36 | 37 | for user in interAct: 38 | f2.writerow((user, interAct[user].totalInt, interAct[user].diffDays)) 39 | i = interAct[user].idic 40 | for d in i: 41 | f3.writerow((user, d, i[d])) 42 | -------------------------------------------------------------------------------- /edX-datascrub/shellscripts/processSmallLogData.sh: -------------------------------------------------------------------------------- 1 | # Process all of the log data from an edX dump. This script will result in a small (weekly) log for each course in the given list. 2 | 3 | # The script takes 3 parameters: 4 | # 1) a list of class names to be processed, seperated by ',' WITHOUT whitespace. 5 | # Class name is in school-class-term format. E.g. BerkeleyX-CS191x-Spring_2013. 6 | # 2) the day from which the log entries are to be processed in YYYY-MM-DD format. 7 | # 3) the last day from which the log entries are to be processed in YYYY-MM-DD format. 8 | 9 | 10 | checkDates.py $1 $2 $3 11 | 12 | if [ "$?" = "0" ]; then 13 | # Separate out the log entries in each directory by the class. 14 | for line in `ls | grep prod`; 15 | do 16 | echo $line 17 | cd $line 18 | separateClassLogs.py $1 19 | cd .. 20 | done 21 | 22 | # Build a log for the week for each of the classes, writing the log to the current directory 23 | # TODO: this part can be run in parallel 24 | courses=$(echo $1 | tr "," "\n") 25 | 26 | for x in $courses 27 | do 28 | buildWeekLog.py $x 29 | done 30 | 31 | mv NewClassList.csv ClassList.csv 32 | fi 33 | 34 | -------------------------------------------------------------------------------- /tasks/config/watch.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Run predefined tasks whenever watched file patterns are added, changed or deleted. 3 | * 4 | * --------------------------------------------------------------- 5 | * 6 | * Watch for changes on 7 | * - files in the `assets` folder 8 | * - the `tasks/pipeline.js` file 9 | * and re-run the appropriate tasks. 10 | * 11 | * For usage docs see: 12 | * https://github.com/gruntjs/grunt-contrib-watch 13 | * 14 | */ 15 | module.exports = function(grunt) { 16 | 17 | grunt.config.set('watch', { 18 | api: { 19 | 20 | // API files to watch: 21 | files: ['api/**/*'] 22 | }, 23 | assets: { 24 | 25 | // Assets to watch: 26 | files: ['assets/**/*', 'tasks/pipeline.js'], 27 | 28 | // When assets are changed: 29 | tasks: ['syncAssets' , 'linkAssets'] 30 | }, 31 | 32 | // datasets: { 33 | // // Datasets to watch 34 | // files: ['../datasets/non_pii/**/**', '../datasets/pii/**/**'], 35 | 36 | // // To-do when new datasets are added: 37 | // tasks: ['extractDatasets'], 38 | 39 | // // Only when datasets are *added* 40 | // options: { 41 | // events: ['added', 'changed'] 42 | // } 43 | // } 44 | }); 45 | 46 | grunt.loadNpmTasks('grunt-contrib-watch'); 47 | }; 48 | -------------------------------------------------------------------------------- /config/paths.js: -------------------------------------------------------------------------------- 1 | var path = require('path'); 2 | 3 | module.exports.paths = { 4 | 5 | // Analytic Paths 6 | UPLOAD_PATH: path.resolve('..', 'analytics', 'archives'), 7 | EXTRACT_PATH: path.resolve('..', 'analytics', 'tmp'), 8 | PUBLIC_SHARE_PATH: path.resolve('views', 'analyticdisplay', 'share'), 9 | 10 | // Data scripts 11 | ANALYTICS_SCRIPTS_PATH: path.resolve('..', 'data_scripts', 'analytics'), 12 | 13 | // Stored Scaffolds Folder 14 | STORED_SCAFFOLDS_PATH: path.resolve('assets', 'scaffolds'), 15 | 16 | // Assets Folder 17 | ANALYTICS_ASSETS_PATH: path.join('assets', 'analytics'), 18 | ANALYTICS_REWRITE_PATH: '../../../../../analytics', 19 | 20 | // Dataset Paths 21 | DATASET_ROOT: path.resolve('..', 'datasets'), 22 | DATASET_DOWNLOAD_ROOT: path.resolve('..', 'datasets', 'available'), 23 | DATASET_NON_PII: path.resolve('..', 'datasets', 'available', 'non_pii'), 24 | DATASET_PII: path.resolve('..', 'datasets', 'available', 'pii'), 25 | DATASET_DROP: path.resolve('..', 'datasets', 'data_drop'), 26 | DATASET_EXTRACT_PATH: path.resolve('..', 'datasets', 'extracted'), 27 | DATASET_ENCRYPT_PATH: path.resolve('..', 'datasets', 'encrypted') 28 | 29 | // Public Key Paths 30 | } -------------------------------------------------------------------------------- /api/services/QueueService.js: -------------------------------------------------------------------------------- 1 | var kue = require('kue'); 2 | var jobs; 3 | 4 | module.exports = { 5 | launchQueue: function() { 6 | sails.log.info('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'); 7 | sails.log.info('====> Launching moocRP jobs queue...'); 8 | jobs = kue.createQueue(); 9 | kue.app.listen(3000); 10 | sails.log.info('====> Jobs queue launched!'); 11 | sails.log.info('====> Listening on port 3000'); 12 | sails.log.info('====> BETA IMPLEMENTATION'); 13 | sails.log.info('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'); 14 | }, 15 | 16 | getQueue: function() { 17 | return jobs; 18 | }, 19 | 20 | createJob: function(jobName, jobParams) { 21 | var job = jobs.create('test job', { title: 'a test job', user: '991426' }); 22 | job 23 | .on('complete', function () { 24 | sails.log.info('[Job ID: ' + job.id + '] Job "' + jobName + '" 100% complete.'); 25 | }).on('failed', function () { 26 | sails.log.error('[Job ID: ' + job.id + '] Job "' + jobName + '" failed.'); 27 | }).on('progress', function (progress) { 28 | sails.log.info('[Job ID: ' + job.id + '] Job "' + jobName + '" ' + progress + '% complete.'); 29 | }); 30 | job.save(); 31 | return job; 32 | } 33 | 34 | } -------------------------------------------------------------------------------- /api/models/DataModel.js: -------------------------------------------------------------------------------- 1 | /** 2 | * DataModel.js 3 | * 4 | * @description :: TODO: You might write a short summary of how this model works and what it represents here. 5 | * @docs :: http://sailsjs.org/#!documentation/models 6 | */ 7 | 8 | module.exports = { 9 | 10 | attributes: { 11 | displayName: { 12 | type: 'STRING', 13 | unique: true, 14 | required: true 15 | }, 16 | fileSafeName: { 17 | type: 'STRING', 18 | unique: true, 19 | required: true 20 | }, 21 | files: { 22 | type: 'ARRAY', 23 | defaultsTo: [] 24 | }, 25 | request: { 26 | collection: 'request', 27 | via: 'dataModel' 28 | }, 29 | description: { 30 | type: 'TEXT' 31 | } 32 | }, 33 | 34 | beforeCreate: function (values, next) { 35 | // We use __ as a separator, so disallow it in names. 36 | var prohibit = /__/; 37 | if (prohibit.test(values.displayName) || prohibit.test(values.fileSafeName)) { 38 | // Handle somehow 39 | return next(new Error('Please use a filename that does not have __ in it.')); 40 | } 41 | if (!values.description || values.description == '') { 42 | values.description = 'No description available.' 43 | } 44 | return next(); 45 | } 46 | }; 47 | 48 | -------------------------------------------------------------------------------- /config/locales/_README.md: -------------------------------------------------------------------------------- 1 | # Internationalization / Localization Settings 2 | 3 | > Also see the official docs on internationalization/localization: 4 | > http://links.sailsjs.org/docs/config/locales 5 | 6 | ## Locales 7 | All locale files live under `config/locales`. Here is where you can add translations 8 | as JSON key-value pairs. The name of the file should match the language that you are supporting, which allows for automatic language detection based on request headers. 9 | 10 | Here is an example locale stringfile for the Spanish language (`config/locales/es.json`): 11 | ```json 12 | { 13 | "Hello!": "Hola!", 14 | "Hello %s, how are you today?": "¿Hola %s, como estas?", 15 | } 16 | ``` 17 | ## Usage 18 | Locales can be accessed in controllers/policies through `res.i18n()`, or in views through the `__(key)` or `i18n(key)` functions. 19 | Remember that the keys are case sensitive and require exact key matches, e.g. 20 | 21 | ```ejs 22 |

<%= __('Welcome to PencilPals!') %>

23 |

<%= i18n('Hello %s, how are you today?', 'Pencil Maven') %>

24 |

<%= i18n('That\'s right-- you can use either i18n() or __()') %>

25 | ``` 26 | 27 | ## Configuration 28 | Localization/internationalization config can be found in `config/i18n.js`, from where you can set your supported locales. 29 | -------------------------------------------------------------------------------- /config/ssl/server.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDgjCCAmoCCQDymUeVnFgzXjANBgkqhkiG9w0BAQsFADCBgjELMAkGA1UEBhMC 3 | VVMxCzAJBgNVBAgMAkNBMREwDwYDVQQHDAhCZXJrZWxleTEPMA0GA1UECgwGbW9v 4 | Y1JQMQ8wDQYDVQQLDAZtb29jUlAxDzANBgNVBAMMBm1vb2NSUDEgMB4GCSqGSIb3 5 | DQEJARYRa2thb0BiZXJrZWxleS5lZHUwHhcNMTUwMTEwMDYyMzI0WhcNMTYwMTEw 6 | MDYyMzI0WjCBgjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMREwDwYDVQQHDAhC 7 | ZXJrZWxleTEPMA0GA1UECgwGbW9vY1JQMQ8wDQYDVQQLDAZtb29jUlAxDzANBgNV 8 | BAMMBm1vb2NSUDEgMB4GCSqGSIb3DQEJARYRa2thb0BiZXJrZWxleS5lZHUwggEi 9 | MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC4jEB1mNxbhPHln3U89oJmrEod 10 | 2wxZD01XXb7dX6P+UzFcbO9Foy6jvNovt4ew71+AbifSWTFLZitJ0xo7ZG15OCWY 11 | ehOYQZOYgX5aO86cbG+yB4BwDerBV8ibD/yYvffu2qahLVNbVwyj5aprbL8RG67u 12 | GuACZGlxQiTzv2IkoRlk97n4AzxqAoGubNY6Ru2pjDwCPr8W1uX+ImaPEPu0rAB4 13 | MtsNSiCVIjqUKZwYOkchVA0uaJYUz1azF1JPeJLshhk/j1fTnmabuZ8QWAo1rEvI 14 | D7z2cpJeg8pGZIzZturIHzIRGH3dFlsJfqK6km2+A3Nn8MsJDoCicuccrBEHAgMB 15 | AAEwDQYJKoZIhvcNAQELBQADggEBACI/Yk7fEHo0O3a1a5JuUam/qLrZD6y9wDun 16 | mYhMjYh0fjUjrfvYBVIqNa9/I4RLeufLLzHfczUgl1hkz4nCTPQM/HQd7PCok2Do 17 | xnvgXOw0wTnhapSVftkN84zJrxMDiAcB3jLVe7Quh4RClbiLi+f2J/97xNOOr9vS 18 | 0ryfBl6lLTJMAmBzrWjAE98rdR44e+yknQbZkdYKPBlZFGOQIFOcrYvXMW/acLtP 19 | 7ZtsIOtGqxV2WwmCFpg4D9OmWvY+t0uGHwJ606X0qqsBqUFTU2U0JLc5j4Tvm27f 20 | vmHI4H4SqjVcKCDNKjoFaDc8crV8xLeeQ5r7h6fPF6DUy56qkW0= 21 | -----END CERTIFICATE----- 22 | -------------------------------------------------------------------------------- /edX-datascrub/src/logs/listClassEntries.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | Created on Dec 13, 2013 4 | 5 | @author: waldo 6 | ''' 7 | import json 8 | import glob 9 | import csv 10 | import sys 11 | 12 | def readFromLog(toDict,fromFile, inEdge): 13 | #dc = json.JSONDecoder() 14 | for line in fromFile: 15 | try: 16 | dcl = json.loads(line) 17 | cl = dcl['context']['course_id'] 18 | cl = cl[cl.find('/')+1:] 19 | cl = cl.replace('/', '-') 20 | if inEdge: 21 | cl = cl + '-edge' 22 | if cl in toDict: 23 | toDict[cl] += 1 24 | else: 25 | toDict[cl] = 1 26 | except ValueError: 27 | pass 28 | return toDict 29 | 30 | if __name__ == '__main__': 31 | if (len(sys.argv) > 1): 32 | d = sys.argv[1] 33 | else: 34 | d = '' 35 | flist = glob.glob(d + '/' + '*.log') 36 | cDict = {} 37 | for f in flist: 38 | if 'edge' in f: 39 | inEdge = True 40 | else: 41 | inEdge = False 42 | fin = open(f, 'r') 43 | cDict = readFromLog(cDict, fin, inEdge) 44 | fin.close() 45 | 46 | fout = csv.writer(open('ClassList.csv', 'w')) 47 | fout.writerow(['Classname', 'Count']) 48 | for v in iter(cDict): 49 | fout.writerow([v, cDict[v]]) 50 | print v -------------------------------------------------------------------------------- /edX-datascrub/src/countcerts.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | Totals the number of students who passed, did not pass, or are restricted from a certificate 4 | 5 | Goes over a certificates file for a course that has completed, finding 6 | the total number of students, the number who were awarded a certificate, 7 | those that did not receive a certificate, and those that are restricted 8 | from receiving a certificate (because of being in an embargoed country) 9 | Created on Feb 20, 2013 10 | 11 | @author: waldo 12 | ''' 13 | 14 | import certificates 15 | import csv 16 | import sys 17 | 18 | if __name__ == '__main__': 19 | pass 20 | 21 | infile = csv.reader(open(sys.argv[1], 'r')) 22 | infile.next() 23 | certDict = certificates.builddict(infile) 24 | passed = unfinished = restrict = total = unknown = 0 25 | citer = iter(certDict) 26 | for c in citer: 27 | if certDict[c].status == "downloadable": 28 | passed += 1 29 | elif certDict[c].status == 'notpassing': 30 | unfinished += 1 31 | elif certDict[c].status == 'restricted': 32 | restrict += 1 33 | else: 34 | print certDict[c].status 35 | unknown += 1 36 | total += 1 37 | 38 | print "Total records = " + str(total) 39 | print "Total passed = " + str(passed) 40 | print 'Total not passing = ' + str(unfinished) 41 | print 'Total restricted = ' + str(restrict) 42 | print 'Total unknown = ' + str(unknown) -------------------------------------------------------------------------------- /edX-datascrub/src/buildEmailList.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | Builds a csv file of First Name, Last Name, email address for a course 4 | 5 | This file needs to be run from a directory in which the users.csv and profiles.csv files 6 | exist. When run, it produces a mailAddresses.csv file with the above format. It will only 7 | include entries for students that are in the user.csv file. If there is a student in the 8 | users.csv file but not in the profiles.csv file, the student's name will read "Missing 9 | Profile" 10 | 11 | Created on Nov 13, 2013 12 | 13 | @author: waldo 14 | ''' 15 | import demographics.userprofile as userp 16 | import user 17 | import sys 18 | import csv 19 | 20 | def split(name): 21 | spIn = name.rfind(' ') 22 | first = name[ :spIn] 23 | last = name[spIn + 1: ] 24 | return [first, last] 25 | 26 | if __name__ == '__main__': 27 | csv.field_size_limit(sys.maxsize) 28 | ufile = csv.reader(open('users.csv','r')) 29 | udict = user.builddict(ufile) 30 | pfile = csv.reader(open('profiles.csv', 'r')) 31 | pdict = userp.builddict(pfile) 32 | 33 | outfile = csv.writer(open('mailAddresses.csv','w')) 34 | 35 | for uid in iter(udict): 36 | if uid in pdict: 37 | name = pdict[uid].name 38 | else : 39 | name = 'Missing Profile' 40 | [first, last] = split(name) 41 | outfile.writerow([first, last, udict[uid].email]) 42 | -------------------------------------------------------------------------------- /assets/scaffolds/d3_scaffold.ejs: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Analytics

4 |
5 |
6 |
7 |
8 |

Step 2: Visualize.

9 |
10 |
11 | 100% 12 |
13 |
14 |
15 |
16 | 17 |
18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 |
28 |
29 |
30 | Back to Analytics 31 |
32 | 33 | 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /documentation/data_models.md: -------------------------------------------------------------------------------- 1 | [Back to README](../README.md) 2 | 3 | Data Models 4 | ================ 5 | This page is a work in-progress. You can contribute to it by forking this repository and making pull requests. 6 | 7 | ## Raw edX Event Tracking 8 | 9 | ## Raw edX-Database 10 | 11 | ## X-API (HarvardX) 12 | The schema of the HarvardX data model is: 13 | time,secs_to_next,actor,verb,object_name,object_type,result,meta,ip,event,event_type,page,agent 14 | 15 | The data is stored in a CSV format, so one can use the ```csv``` function in d3.js to read in the data in a visualization. 16 | 17 | * _time_: the time of the action 18 | * _secs_to_next_: the number of seconds to the next actor 19 | * _actor_: who is performing the action 20 | * _verb_: the action being performed 21 | * _object_name_: the object that the action is being performed on 22 | * _result_: 23 | * _meta_: 24 | * _ip_: 25 | * _event_: 26 | * _event_type_: 27 | * _page_: 28 | * _agent_: 29 | 30 | _Sample Entry_: 31 | ``` 32 | 2014-01-23T07:48:43.123460+00:00,5.462487,applevo,page_view,Course Info,tab_name,,,1.2.3.4,"{""POST"": {}, ""GET"": {}}",/courses/University/Course/info,,"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" 33 | ``` 34 | 35 | The filename to retrieve from the data dictionary for this data model is _course_name_.csv. 36 | 37 | ## MOOCdb 38 | 39 | ## Stanford 40 | 41 | ## Placeholder 42 | -------------------------------------------------------------------------------- /edX-datascrub/src/logs/cleanLogDups.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | Reads an edX supplied log file, finding and eliminating any duplicate lines 4 | 5 | Reads an edX log file, finding any duplicate lines. Writes a file with the same 6 | name as the log file with the additional postfix "scrub" that contains only the 7 | non-duplicated lines. 8 | 9 | At the end of the run, prints out the number of lines read, the number of duplicate 10 | lines, and the number of non-duplicate lines. 11 | ''' 12 | import json 13 | import sys 14 | 15 | class Line(object): 16 | def __init__(self, lineNo, lineCont): 17 | self.line = lineNo 18 | self.content = lineCont 19 | 20 | 21 | linedict = {} 22 | 23 | f1 = open(sys.argv[1], 'r') 24 | f2 = open(sys.argv[1] + 'scrub', 'w') 25 | ln = 0; 26 | duplines = 0; 27 | 28 | dc = json.JSONDecoder() 29 | 30 | for line in f1: 31 | ln += 1; 32 | dl = dc.decode(line) 33 | key = dl['time'] + dl['username'] 34 | if key not in linedict: 35 | f2.write(line) 36 | lo = Line(ln, line) 37 | linedict[key] = lo 38 | else: 39 | if linedict[key].content == line: 40 | duplines += 1 41 | print line 42 | else: 43 | f2.write(line) 44 | 45 | print 'total number of lines = ' + str(ln) 46 | print 'total number of duplicate lines = ' + str(duplines) 47 | print 'total lines of real data = ' + str(ln - duplines) 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /api/responses/ok.js: -------------------------------------------------------------------------------- 1 | /** 2 | * 200 (OK) Response 3 | * 4 | * Usage: 5 | * return res.ok(); 6 | * return res.ok(data); 7 | * return res.ok(data, 'auth/login'); 8 | * 9 | * @param {Object} data 10 | * @param {String|Object} options 11 | * - pass string to render specified view 12 | */ 13 | 14 | module.exports = function sendOK (data, options) { 15 | 16 | // Get access to `req`, `res`, & `sails` 17 | var req = this.req; 18 | var res = this.res; 19 | var sails = req._sails; 20 | 21 | sails.log.silly('res.ok() :: Sending 200 ("OK") response'); 22 | 23 | // Set status code 24 | res.status(200); 25 | 26 | // If appropriate, serve data as JSON(P) 27 | if (req.wantsJSON) { 28 | return res.jsonx(data); 29 | } 30 | 31 | // If second argument is a string, we take that to mean it refers to a view. 32 | // If it was omitted, use an empty object (`{}`) 33 | options = (typeof options === 'string') ? { view: options } : options || {}; 34 | 35 | // If a view was provided in options, serve it. 36 | // Otherwise try to guess an appropriate view, or if that doesn't 37 | // work, just send JSON. 38 | if (options.view) { 39 | return res.view(options.view, { data: data }); 40 | } 41 | 42 | // If no second argument provided, try to serve the implied view, 43 | // but fall back to sending JSON(P) if no view can be inferred. 44 | else return res.guessView({ data: data }, function couldNotGuessView () { 45 | return res.jsonx(data); 46 | }); 47 | 48 | }; 49 | -------------------------------------------------------------------------------- /tasks/config/jst.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Precompiles Underscore templates to a `.jst` file. 3 | * 4 | * --------------------------------------------------------------- 5 | * 6 | * (i.e. basically it takes HTML files and turns them into tiny little 7 | * javascript functions that you pass data to and return HTML. This can 8 | * speed up template rendering on the client, and reduce bandwidth usage.) 9 | * 10 | * For usage docs see: 11 | * https://github.com/gruntjs/grunt-contrib-jst 12 | * 13 | */ 14 | 15 | module.exports = function(grunt) { 16 | 17 | var templateFilesToInject = [ 18 | 'templates/**/*.html' 19 | ]; 20 | 21 | grunt.config.set('jst', { 22 | dev: { 23 | 24 | // To use other sorts of templates, specify a regexp like the example below: 25 | // options: { 26 | // templateSettings: { 27 | // interpolate: /\{\{(.+?)\}\}/g 28 | // } 29 | // }, 30 | 31 | // Note that the interpolate setting above is simply an example of overwriting lodash's 32 | // default interpolation. If you want to parse templates with the default _.template behavior 33 | // (i.e. using
), there's no need to overwrite `templateSettings.interpolate`. 34 | 35 | 36 | files: { 37 | // e.g. 38 | // 'relative/path/from/gruntfile/to/compiled/template/destination' : ['relative/path/to/sourcefiles/**/*.html'] 39 | '.tmp/public/jst.js': require('../pipeline').templateFilesToInject 40 | } 41 | } 42 | }); 43 | 44 | grunt.loadNpmTasks('grunt-contrib-jst'); 45 | }; 46 | -------------------------------------------------------------------------------- /edX-datascrub/src/checkData/compUser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | A simple script to compare two user files and a certificates file 4 | 5 | Generates a report of users differences between multiple user files, 6 | with the additional check to see if those users that are in one of the 7 | files but not the other are in the certificates files. This script assumes 8 | that the certificates file being used is in the directory in which the 9 | script is run, and takes as arguments the file names of the two user 10 | files to be compared. 11 | ''' 12 | 13 | import user 14 | import csv 15 | import sys 16 | import certificates 17 | 18 | f1 = csv.reader(open(sys.argv[1], 'r')) 19 | f2 = csv.reader(open(sys.argv[2], 'r')) 20 | f3 = csv.writer(open('additions.csv', 'w')) 21 | f4 = csv.reader(open('certificates.csv', 'r')) 22 | f3.writerow(['id', 'in certificate file']) 23 | f3.writerow(['User ids in first file, not in second']) 24 | u1 = user.builddict(f1) 25 | u2 = user.builddict(f2) 26 | cdict = certificates.builddict(f4) 27 | 28 | for key in u1.iterkeys(): 29 | if u1[key].id not in u2: 30 | if key in cdict: 31 | f3.writerow([key, 'Yes']) 32 | else: 33 | f3.writerow([key, 'No']) 34 | 35 | f3.writerow(['User ids in second file, not in first']) 36 | 37 | for key in u2.iterkeys(): 38 | if u2[key].id not in u1: 39 | if key in cdict: 40 | f3.writerow([key, 'Yes']) 41 | else: 42 | f3.writerow([key, 'No']) 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /api/models/Request.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Request 3 | * 4 | * @module :: Model 5 | * @description :: A short summary of how this model works and what it represents. 6 | * @docs :: http://sailsjs.org/#!documentation/models 7 | */ 8 | 9 | module.exports = { 10 | 11 | attributes: { 12 | 13 | /* e.g. 14 | nickname: 'string' 15 | */ 16 | firstName: { 17 | type: 'STRING', 18 | required: true 19 | }, 20 | lastName: { 21 | type: 'STRING', 22 | required: true 23 | }, 24 | email: { 25 | type: 'STRING', 26 | email: true, 27 | required: true 28 | }, 29 | reason: { 30 | type: 'TEXT', 31 | }, 32 | requestingUser: { 33 | model: 'user', 34 | required: true 35 | }, 36 | dataModel: { 37 | model: 'datamodel', 38 | required: true 39 | }, 40 | dataset: { 41 | type: 'STRING', 42 | required: true 43 | }, 44 | requestType: { 45 | type: 'STRING', 46 | required: true 47 | }, 48 | approved: { 49 | type: 'BOOLEAN', 50 | defaultsTo: false 51 | }, 52 | rejected: { 53 | type: 'BOOLEAN', 54 | defaultsTo: false 55 | }, 56 | downloaded: { 57 | type: 'BOOLEAN', 58 | defaultsTo: false 59 | } 60 | }, 61 | beforeCreate: function (values, next) { 62 | // Default reason value 63 | if (values.reason == '') { 64 | values.reason = 'No reason specified'; 65 | } 66 | return next(); 67 | } 68 | 69 | }; 70 | -------------------------------------------------------------------------------- /api/models/Analytic.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Analytic 3 | * 4 | * @module :: Model 5 | * @description :: A short summary of how this model works and what it represents. 6 | * @docs :: http://sailsjs.org/#!documentation/models 7 | */ 8 | 9 | module.exports = { 10 | 11 | attributes: { 12 | 13 | /* e.g. 14 | nickname: 'string' 15 | */ 16 | owner: { 17 | model: 'user', 18 | required: true 19 | }, 20 | name: { 21 | type: 'STRING', 22 | required: true 23 | }, 24 | fileName: { 25 | type: 'STRING', 26 | required: true 27 | }, 28 | seededFileName: { 29 | type: 'STRING', 30 | required: true 31 | }, 32 | description: { 33 | type: 'TEXT', 34 | required: true 35 | }, 36 | url: { 37 | type: 'STRING' 38 | }, 39 | approved: { 40 | type: 'BOOLEAN', 41 | defaultsTo: false 42 | }, 43 | rejected: { 44 | type: 'BOOLEAN', 45 | defaultsTo: false 46 | }, 47 | // i.e. D3, R, Plateau 48 | type: { 49 | type: 'STRING', 50 | required: true 51 | }, 52 | // i.e. HarvardX, StanfordX, moocDB 53 | dataModels: { 54 | type: 'ARRAY', 55 | required: true 56 | }, 57 | usersWhoStarred: { 58 | collection: 'user', 59 | via: 'starredAnalytics' 60 | }, 61 | 62 | // Instance methods 63 | toJSON: function() { 64 | var obj = this.toObject(); 65 | delete obj._csrf; 66 | return obj; 67 | }, 68 | } 69 | }; 70 | -------------------------------------------------------------------------------- /edX-datascrub/src/usersAndClasses.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import glob 5 | import user 6 | import csv 7 | import ipGeoloc as loc 8 | import sys 9 | 10 | class userClasses: 11 | def __init__(self, uname, className): 12 | self.numClasses = 1 13 | self.uname = uname 14 | self.country = '' 15 | self.classList = [className] 16 | 17 | 18 | geoFile = csv.reader(open(sys.argv[1], 'r')) 19 | geoDict = loc.builddict(geoFile) 20 | 21 | dirList = glob.glob('[A-Z]*') 22 | classDict = {} 23 | 24 | for d in dirList: 25 | filein = open(d+'/users.csv', 'r') 26 | fin = csv.reader(filein) 27 | cName = d 28 | fin.next() 29 | udict = user.builddict(fin) 30 | for u in iter(udict): 31 | if u in classDict: 32 | classDict[u].numClasses += 1 33 | classDict[u].classList.append(cName) 34 | if udict[u].username != classDict[u].uname: 35 | classDict[u].uname = 'Duplicate user name' 36 | else: 37 | classDict[u] = userClasses(udict[u].username, cName) 38 | if udict[u].username in geoDict: 39 | classDict[u].country = geoDict[udict[u].username] 40 | 41 | filein.close() 42 | 43 | outf = csv.writer(open('studentClassList.csv', 'w')) 44 | outf.writerow(['user Id', 'User name','country', 'number of classes', 'classes']) 45 | for u in iter(classDict): 46 | outf.writerow([u, classDict[u].uname, classDict[u].country, classDict[u].numClasses, classDict[u].classList]) 47 | -------------------------------------------------------------------------------- /edX-datascrub/src/checkData/corrUsers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | Checks to insure that users appear in the enrollment and profiles file 4 | 5 | Looks at the user, enrollment, and profile file in the directory in which 6 | the script is run to insure that all of the entries in the user file 7 | have entries in the enrollment and profiles file, and that fall of the 8 | entries in the profiles and enrollment file have entries in the user file. 9 | ''' 10 | 11 | 12 | import csv 13 | import user 14 | import demographics.userprofile as uprofile 15 | import course_enrollment as ce 16 | 17 | csv.field_size_limit(1000000) 18 | uIn = csv.reader(open('users.csv', 'r')) 19 | uDict = user.builddict(uIn) 20 | 21 | upIn = csv.reader(open('profiles.csv', 'r')) 22 | upDict = uprofile.builddict(upIn) 23 | 24 | ceIn = csv.reader(open('enrollment.csv', 'r')) 25 | ceDict = ce.builddict(ceIn) 26 | 27 | of = csv.writer(open('userDiffs.csv', 'w')) 28 | 29 | of.writerow(['ids in user file, not in profiles file']) 30 | for u in iter(uDict): 31 | if u not in upDict: 32 | of.writerow([u]) 33 | 34 | of.writerow(['ids in profiles file, not in user file']) 35 | for p in iter(upDict): 36 | if p not in uDict: 37 | of.writerow([p]) 38 | 39 | of.writerow(['ids in user file, not in enrollment file']) 40 | for u in iter(uDict): 41 | if u not in ceDict: 42 | of.writerow([u]) 43 | 44 | of.writerow(['ids in enrollment file, not in user file']) 45 | for e in iter(ceDict): 46 | if e not in uDict: 47 | of.writerow([u]) 48 | 49 | 50 | -------------------------------------------------------------------------------- /views/home/license.ejs: -------------------------------------------------------------------------------- 1 |
2 |
3 |

License

4 |
5 |
6 |
7 |
8 |

MIT License

9 |
10 |
11 |
12 |

13 | The MIT License (MIT) 14 |

15 |

16 | Copyright (c) 2014-2015 moocRP. 17 |

18 |

19 | Permission is hereby granted, free of charge, to any person obtaining a copy 20 | of this software and associated documentation files (the "Software"), to deal 21 | in the Software without restriction, including without limitation the rights 22 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 23 | copies of the Software, and to permit persons to whom the Software is 24 | furnished to do so, subject to the following conditions: 25 |

26 |

27 | The above copyright notice and this permission notice shall be included in 28 | all copies or substantial portions of the Software. 29 |

30 |

31 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 32 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 33 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 34 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 35 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 36 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 37 | THE SOFTWARE. 38 |

39 |
-------------------------------------------------------------------------------- /api/services/AuthService.js: -------------------------------------------------------------------------------- 1 | // Various CAS settings 2 | var settings = sails.config; 3 | var casOptions = settings.casOptions; 4 | 5 | module.exports = { 6 | loginRoute: function(params) { 7 | // Can override with own login system 8 | var baseURL = casOptions.casURL + casOptions.login + '?service=' + AuthService.serviceURL + '/user/validate'; 9 | if (params) { 10 | for (var key in params) { 11 | if (params.hasOwnProperty(key)) baseURL = baseURL + '&' + key + '=' + params[key]; 12 | } 13 | } 14 | return baseURL 15 | }, 16 | 17 | logoutRoute: function() { 18 | return casOptions.casURL + casOptions.logout + '?url=' + AuthService.serviceURL; 19 | }, 20 | 21 | validateRoute: function(params) { 22 | var baseURL = casOptions.casURL + casOptions.validate + '?service=' + AuthService.serviceURL + '/user/validate'; 23 | if (params) { 24 | for (var key in params) { 25 | if (params.hasOwnProperty(key)) baseURL = baseURL + '&' + key + '=' + params[key]; 26 | } 27 | } 28 | return baseURL 29 | }, 30 | 31 | validate: function(url, cb) { 32 | var request = require('request'); 33 | 34 | request({uri: url, secureProtocol: 'TLSv1_method' }, function(err, response, body) { 35 | var uid = undefined; 36 | 37 | if (!err && body) { 38 | var lines = body.split('\n'); 39 | if (lines && lines[0] == 'yes') { 40 | uid = lines[1]; 41 | } 42 | } 43 | return cb(err, uid); 44 | }); 45 | }, 46 | 47 | serviceURL: settings.protocol + settings.appEnvMap[settings.environment] + ":" + settings.port 48 | } -------------------------------------------------------------------------------- /test/bootstrap.js: -------------------------------------------------------------------------------- 1 | // http://stackoverflow.com/questions/26837522/sails-js-how-to-actually-run-tests 2 | // https://github.com/albertosouza/sails-test-example 3 | /** 4 | * Test starter - with this version of sails.js we can only start one sails server, 5 | * to solve this problem we use only one before All and after All to start and 6 | * stop the server 7 | */ 8 | var Sails = require('sails'); 9 | var _ = require('lodash'); 10 | 11 | global.DOMAIN = 'http://localhost'; 12 | global.PORT = 1337; 13 | global.HOST = DOMAIN + ':' + PORT; 14 | 15 | before(function(callback) { 16 | this.timeout(7000); 17 | 18 | var configs = { 19 | log: { 20 | level: 'info' 21 | }, 22 | connections: { 23 | memory: { 24 | // lets use memory tests ... 25 | adapter : 'sails-memory' 26 | } 27 | }, 28 | models: { 29 | connection: 'memory' 30 | }, 31 | port: PORT, 32 | environment: 'test', 33 | 34 | // @TODO needs support to csrf token 35 | csrf: false, 36 | 37 | // we dont need this configs in API test 38 | hooks: { 39 | grunt: false, 40 | socket: false, 41 | pubsub: false 42 | } 43 | }; 44 | 45 | // Sails.load() might be better, but lift() loads bootstrap.js in config, for 46 | // removing logging within the app 47 | Sails.lift(configs, function(err, sails) { 48 | if (err) { 49 | console.error(err); 50 | return callback(err); 51 | } 52 | 53 | // here you can load fixtures, etc. 54 | callback(err, sails); 55 | }); 56 | }); 57 | 58 | after(function(done) { 59 | // here you can clear fixtures, etc. 60 | sails.lower(done); 61 | }); -------------------------------------------------------------------------------- /config/log.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Built-in Log Configuration 3 | * (sails.config.log) 4 | * 5 | * Configure the log level for your app, as well as the transport 6 | * (Underneath the covers, Sails uses Winston for logging, which 7 | * allows for some pretty neat custom transports/adapters for log messages) 8 | * 9 | * For more information on the Sails logger, check out: 10 | * http://sailsjs.org/#/documentation/concepts/Logging 11 | */ 12 | 13 | var winston = require('winston'); 14 | 15 | var customLogger = new winston.Logger({ 16 | transports: [ 17 | new(winston.transports.File)({ 18 | level: 'debug', 19 | filename: './logs/development.log' 20 | }), 21 | ], 22 | }); 23 | 24 | module.exports.log = { 25 | 26 | /*************************************************************************** 27 | * * 28 | * Valid `level` configs: i.e. the minimum log level to capture with * 29 | * sails.log.*() * 30 | * * 31 | * The order of precedence for log levels from lowest to highest is: * 32 | * silly, verbose, info, debug, warn, error * 33 | * * 34 | * You may also set the level to "silent" to suppress all logs. * 35 | * * 36 | ***************************************************************************/ 37 | 38 | //colors: false, 39 | //custom: customLogger 40 | 41 | }; 42 | -------------------------------------------------------------------------------- /api/controllers/JobController.js: -------------------------------------------------------------------------------- 1 | /** 2 | * JobController 3 | * 4 | * @description :: Server-side logic for managing jobs 5 | * @help :: See http://links.sailsjs.org/docs/controllers 6 | */ 7 | 8 | var updateStartTime = function (jobID, startTime, cb) { 9 | Job.findOne(jobID, function (err, job) { 10 | if (err || !job) return cb(err); 11 | params = { status: status }; 12 | job.startTime = startTime; 13 | job.save(function (err2) { 14 | if (err2) return cb(err2); 15 | return cb(); 16 | }); 17 | }); 18 | } 19 | 20 | var updateEndTime = function (jobID, endTime, cb) { 21 | Job.findOne(jobID, function (err, job) { 22 | if (err || !job) return cb(err); 23 | params = { status: status }; 24 | job.endTime = endTime; 25 | job.save(function (err2) { 26 | if (err2) return cb(err2);W 27 | return cb(); 28 | }); 29 | }); 30 | } 31 | 32 | var updateStatus = function(jobID, status, cb) { 33 | Job.findOne(jobID, function (err, job) { 34 | if (err || !job) return cb(err); 35 | params = { status: status }; 36 | job.status = status; 37 | job.save(function (err2) { 38 | if (err2) return cb(err2); 39 | return cb(); 40 | }); 41 | }); 42 | } 43 | 44 | module.exports = { 45 | create: function(req, res) { 46 | var params = req.params.all(); 47 | Job.create(params, function (err, job) { 48 | if (err || !job) return res.json( {'success': false }); 49 | return res.json({ 'success': true }); 50 | }); 51 | }, 52 | destroy: function(req, res) { 53 | Job.destroy(req.param['id'], function (err, job) { 54 | return res.json({ 'success': true }); 55 | }); 56 | }, 57 | update: function(req, res) { 58 | // TODO 59 | } 60 | }; 61 | 62 | -------------------------------------------------------------------------------- /config/ssl/server.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEowIBAAKCAQEAuIxAdZjcW4Tx5Z91PPaCZqxKHdsMWQ9NV12+3V+j/lMxXGzv 3 | RaMuo7zaL7eHsO9fgG4n0lkxS2YrSdMaO2RteTglmHoTmEGTmIF+WjvOnGxvsgeA 4 | cA3qwVfImw/8mL337tqmoS1TW1cMo+Wqa2y/ERuu7hrgAmRpcUIk879iJKEZZPe5 5 | +AM8agKBrmzWOkbtqYw8Aj6/Ftbl/iJmjxD7tKwAeDLbDUoglSI6lCmcGDpHIVQN 6 | LmiWFM9WsxdST3iS7IYZP49X055mm7mfEFgKNaxLyA+89nKSXoPKRmSM2bbqyB8y 7 | ERh93RZbCX6iupJtvgNzZ/DLCQ6AonLnHKwRBwIDAQABAoIBACyEYJQ9gIJvKm7q 8 | rTw8dq5Pxz02dt8Q6uY1TfJWvNd/t/uEp59Tws9qofM6wXez9oSjjeWW8GYwyiZv 9 | zvceva2tFpyWbh2fS/xQ2Grp3GgtXDBE0P22zbc/9rs4+wTZZnJuuh1NMrto3zq2 10 | DwsE235EAEmdoAXTtP/GJeXKd+E/tlAK8OtAmnMPeaCDy22+6R7h8oVSlOVu+N1u 11 | 32sC1gla+jmKj2RUVzKlWVz9gmng+UeULB+3R0fLpbKQis7yEXMZCqEORzca3m8V 12 | r9dACNko3AGHbixLZnC/ZIU7p/xvyU6PdEw1n7SsmTXmEjtBOH8Fxeja0tVDe8BV 13 | DTnehwkCgYEA4e9FDk9I4NmUw6KAYl7ZbSwb2/1m656oiDcqdFP3nEI3GA621FHE 14 | hm8HVw9AGLGQXxc66IP9g24qREoLS3YhVSlUMXu8WXq/vz7lHI2b/g24U5pIF7Iv 15 | pPjvsFDhmH9qrjU8kh8camBJtcvT4ct2nptrY2+AsNcdl9AvA3MxhjUCgYEA0RsX 16 | nshDmcztl/XH0wARVO7vz1PMy3xkvRWfqaeCWnC1IycpARlZqUHmbBqvpecWJ/Yw 17 | d2YsUuehNb2e6QNVAAwOj0ygcyjSpVKw7y9bc+ttWg+jFf4il/VgtzfikossQkzz 18 | QfLT93ffKAKOjxoYM7lZbYJP4fhzHp5WIU+sscsCgYAZOnSFkojawrD/32dilKDG 19 | tgQuXm9dpAvBmhddgfrGMgag4xO7RZ4iPMefCw7nMvyiAaAMUqC+SlDh6zqzpG84 20 | aTMDi4OOokxC+KzwsUdX6QRKIZInQzhavYlWMNHgC2pIJZ2r21l672GLsUTpk6Sd 21 | NVGaetrt5DdjulVllzlSeQKBgBcNbpAxscot3m+nR+1KD58WbFel/GjegNibnqt8 22 | bRF9ZWrHsWyOl+Th//4g/wZCMJ4dNQCkwfQt1wburaswk9laeuxvXSz07iwNrrXf 23 | uaxQ4xBPswIEr1mjUpNAVPkk4K86foLhu16H15E4nvDFxq9FGsfI2velhUN13zby 24 | 6q37AoGBANSZ+oEukbYyo0s4EGhK0a85eD9lNjxfRPGfMScyqpqZRERG485AYBYY 25 | zVBLxEiTNKirY4RJgHcD/8KpEUEWQYLIfxCNlQtsRvobKqyj5zcqPsoTLDNm/bI7 26 | dmWzfP3t/86gGoeJax2RhuX+xLQUXLjFMVpXJ3kSlxhn3875w1XG 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /edX-datascrub/src/logs/findLogDups.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | Find and report on duplicate lines in an edX log file 5 | 6 | Reads the edX log file supplied as the first parameter. Building a dictionary 7 | keyed by a concatenation of the username and time of the log entry, compares any 8 | two lines with the same key for equivalence. If they are the same, increments the 9 | count of duplicate lines and prints out the contents of the line, along with the 10 | sequence number of the duplicated line and the sequence number of the line it 11 | duplicates. 12 | 13 | At the end of the run, prints out the number of log lines read, the number of 14 | duplicate lines found, and the number of non-duplicate lines. 15 | 16 | If duplicate lines are found, they can be eliminated by running the script 17 | cleanLogDups.py 18 | ''' 19 | import json 20 | import sys 21 | 22 | class Line(object): 23 | def __init__(self, lineNo, lineCont): 24 | self.line = lineNo 25 | self.content = lineCont 26 | 27 | 28 | linedict = {} 29 | 30 | f1 = open(sys.argv[1], 'r') 31 | ln = 0; 32 | duplines = 0; 33 | 34 | dc = json.JSONDecoder() 35 | 36 | for line in f1: 37 | ln += 1; 38 | dl = dc.decode(line) 39 | key = dl['time'] + dl['username'] 40 | if key not in linedict: 41 | lo = Line(ln, line) 42 | linedict[key] = lo 43 | else: 44 | if linedict[key].content == line: 45 | duplines += 1 46 | print line 47 | print 'line no ' + str(lo) + 'duplicates line no ' + str(linedict[key].line) 48 | 49 | print 'total number of lines = ' + str(ln) 50 | print 'total number of duplicate lines = ' + str(duplines) 51 | print 'total lines of real data = ' + str(ln - duplines) 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /edX-datascrub/src/convertfiles/makeIdCountryFile.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | Takes a file mapping usernames to country and converts to userId to country 4 | 5 | Used for an earlier version of the edX geolocation files, this takes a csv for a particular 6 | course that mapped the username to the country indicated by the most-often used IP address 7 | for the student and creates a file that maps from the userId to the country. 8 | 9 | This is largely obsoleted by the new format for a country mapping. 10 | 11 | Created on May 4, 2013 12 | 13 | @author: waldo 14 | ''' 15 | 16 | import user 17 | import csv 18 | import os 19 | import utils 20 | import sys 21 | 22 | 23 | def buildNameCountry(cfile): 24 | retDict = {} 25 | for [country, username] in cfile: 26 | retDict[username] = country 27 | return retDict 28 | 29 | if len(sys.argv) > 3: 30 | cFileName = sys.argv[1] 31 | userFileName = sys.argv[2] 32 | clName = sys.argv[3] 33 | else: 34 | cFileName = utils.getFileName('user name to country file') 35 | userFileName = utils.getFileName('user file') 36 | clName = raw_input("Please enter the name of the class : ") 37 | 38 | cfile = csv.reader(open(cFileName, 'r')) 39 | nameDict = buildNameCountry(cfile) 40 | ufile = csv.reader(open(userFileName, 'r')) 41 | userDict = user.builddict(ufile) 42 | 43 | 44 | clfName = clName + '_id_country.csv' 45 | outfile = csv.writer(open(clfName, 'w')) 46 | 47 | users = userDict.keys() 48 | 49 | outfile.writerow(['User id', 'Country']) 50 | for u in users: 51 | userName = userDict[u].username 52 | if (userName in nameDict): 53 | country = nameDict[userDict[u].username] 54 | outfile.writerow([u, country]) 55 | else: 56 | print ('unknown userName ' + userName) 57 | 58 | 59 | -------------------------------------------------------------------------------- /config/ssl/server.key.secure: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | Proc-Type: 4,ENCRYPTED 3 | DEK-Info: DES-EDE3-CBC,73D65A98A60F9174 4 | 5 | Z0GAgB2erUHdf9RU/FgAIXwz5C7hab950KParYritXMbKUBA2XA4iHJoxElSrfDq 6 | 8BmeNtB6Ha8VTbtfLN1x23xEm5sag59jyYy3pqd9vQFGF5wZIbD1JI9i6n2hl4V5 7 | s86++wRUPbzQkw3JpjSCjt738FeEc05k0X07vvMiQ1i0b6fstDUWhpS05i8nGAEH 8 | nmbqa22uXvYp5s2Jq2E9hfOY3qjIUIFYuDETmUy+WTk4JjkPY/l4lhUJXd4x4ksd 9 | dDKkiRA1zOJ612T3vl4wtD75wXjpJg5TOW35fROGgkgvVnB1jYyD9yNKOcHWS2Xb 10 | 8Eh1YLb9v8SQt405rbVxSW+WP/SaPRZZPUH4GYvgtDXsl6JJj8Rzas61ymQ+c4v0 11 | MvheoOyVSF2+28jqi9vxwsAO/tHA3gYbSn6JCf9fM3j9qPotk+okDrEL2HvGYOKW 12 | nICtUxaeLDtLwmpmCQsZ2Da221J1Q0EoHAZEb0fv/BVLZObe5EWyrrcNQpHNsuOU 13 | F5ilem++4eEYia18W8MttzIRL5/QeWVDZljjo6EyvxaKqG2llX3lANbg1TdX7ABU 14 | D0+8FU5bg10C0sF4FYE6mo2ocqiK0Plt85nrqSoLsCFZorUMc/3ecxLlcyIWxn0L 15 | u7ZRUm9XLZWiEq9HWKSMoCvBJjJG9rvSQeupcV2DxrwZ40f2jl9YBYeEe6uz8K8w 16 | 2HttyX2KTdzqicX1/1wk65AC8NPC6KZyKw6uoLaqf9ldWZh5amTcVFD/QzIIG8Q6 17 | ByBHxpWvPEIOceV4i957qjXA7WrUqpqueJKrLHKMQMPj81ActUTAX862T68I0gqX 18 | DAqmTpTHo3G1VTRZuA/GflAXkicOMc3la05Czic7xxOODjzP8PYo651k/od4GVkf 19 | ZXZpCxVqdwvd4f1ccqwRzf/4Mm4SXdfahOgMu4zXScoCaS+wYlluf4ITVJGIDqYC 20 | yF7/9ktGRZJXatLTe+TcNxNvNqtlhTE3OjfUStG48EZ1fcQqCW0vOBDyGaoBobII 21 | xW2F5Iz+wRG2M9FlRrd4Co228MDDucnRNf0hw2OM4S9QAYAqM5h7huzmkrHyEi4O 22 | 6TNhHmRFzLzD9qoeBv/ZHE0c4DtpRPXvZesBot795atEqjq7gAlMERS5Qh7teMxh 23 | rljnVk80A9MFdCsboORVIPkRaqAvkdUBwfqeigPBndIbuFyaJ5DOSeH+af8lQ1BZ 24 | STM9bVamS2dE6UomA5v9owUVHocVHAKi/IY7pjxVxXFUhv7Hf667gRWd0ZCqr2YW 25 | Rw71Hko8qXO/g++d1lGR8FLMKrdkO/WTlIRGILb3YN8GGa8LgV0GXEVRVWkL4Wyh 26 | 0VwC5ylAXX8xDXxjHrZjlEOJDP1xGFZCQLaFqUowhOL/orDACCj0ogCNAJDrk3OH 27 | OZQx1uVBZqyZ1+M7iIOp5mCPo6EpMx9oJ8YX56acl1Lsyug4itWgDpO+HRJdva2c 28 | m4GqF8qEnog8NyuBpji5NXEDvxfNDiU3/fsighf1FEn3LNlsnvkZ97dpM9pLJUlM 29 | 6fL+IllL1mDGH9mtSiT+u8LgExX9EKGl8puzyP8eKnOxR1oW6yhpHw== 30 | -----END RSA PRIVATE KEY----- 31 | -------------------------------------------------------------------------------- /app.js: -------------------------------------------------------------------------------- 1 | /** 2 | * app.js 3 | * 4 | * Use `app.js` to run your app without `sails lift`. 5 | * To start the server, run: `node app.js`. 6 | * 7 | * This is handy in situations where the sails CLI is not relevant or useful. 8 | * 9 | * For example: 10 | * => `node app.js` 11 | * => `forever start app.js` 12 | * => `node debug app.js` 13 | * => `modulus deploy` 14 | * => `heroku scale` 15 | * 16 | * 17 | * The same command-line arguments are supported, e.g.: 18 | * `node app.js --silent --port=80 --prod` 19 | */ 20 | 21 | // Ensure a "sails" can be located: 22 | (function() { 23 | var sails; 24 | try { 25 | sails = require('sails'); 26 | } catch (e) { 27 | console.error('To run an app using `node app.js`, you usually need to have a version of `sails` installed in the same directory as your app.'); 28 | console.error('To do that, run `npm install sails`'); 29 | console.error(''); 30 | console.error('Alternatively, if you have sails installed globally (i.e. you did `npm install -g sails`), you can use `sails lift`.'); 31 | console.error('When you run `sails lift`, your app will still use a local `./node_modules/sails` dependency if it exists,'); 32 | console.error('but if it doesn\'t, the app will run with the global sails instead!'); 33 | return; 34 | } 35 | 36 | // Try to get `rc` dependency 37 | var rc; 38 | try { 39 | rc = require('rc'); 40 | } catch (e0) { 41 | try { 42 | rc = require('sails/node_modules/rc'); 43 | } catch (e1) { 44 | console.error('Could not find dependency: `rc`.'); 45 | console.error('Your `.sailsrc` file(s) will be ignored.'); 46 | console.error('To resolve this, run:'); 47 | console.error('npm install rc --save'); 48 | rc = function () { return {}; }; 49 | } 50 | } 51 | 52 | 53 | // Start server 54 | sails.lift(rc('sails')); 55 | })(); 56 | -------------------------------------------------------------------------------- /api/controllers/HomeController.js: -------------------------------------------------------------------------------- 1 | /******************************************* 2 | * Copyright 2014, moocRP * 3 | * Author: Kevin Kao * 4 | *******************************************/ 5 | 6 | /** 7 | * HomeController 8 | * 9 | * @module :: Controller 10 | * @description :: A set of functions called `actions`. 11 | * 12 | * Actions contain code telling Sails how to respond to a certain type of request. 13 | * (i.e. do stuff, then send some JSON, show an HTML page, or redirect to another URL) 14 | * 15 | * You can configure the blueprint URLs which trigger these actions (`config/controllers.js`) 16 | * and/or override them with custom routes (`config/routes.js`) 17 | * 18 | * NOTE: The code you write here supports both HTTP and Socket.io automatically. 19 | * 20 | * @docs :: http://sailsjs.org/#!documentation/controllers 21 | */ 22 | 23 | module.exports = { 24 | /** 25 | * Overrides for the settings in `config/controllers.js` 26 | * (specific to HomeController) 27 | */ 28 | _config: {}, 29 | 30 | about: function(req, res) { 31 | res.view({ 32 | title: 'About' 33 | }); 34 | }, 35 | 36 | index: function(req, res) { 37 | if (req.session.authenticated) { 38 | res.redirect('/dashboard'); 39 | } else { 40 | res.view({ 41 | title: 'Home' 42 | }); 43 | } 44 | }, 45 | 46 | tos: function(req, res) { 47 | res.view({ 48 | title: 'Terms of Service' 49 | }) 50 | }, 51 | 52 | privacy: function(req, res) { 53 | res.view({ 54 | title: 'Privacy Policy' 55 | }) 56 | }, 57 | 58 | license: function(req, res) { 59 | res.view({ 60 | title: 'MIT License' 61 | }) 62 | }, 63 | 64 | contact: function(req, res) { 65 | res.view({ 66 | title: 'Contact' 67 | }); 68 | } 69 | 70 | }; 71 | -------------------------------------------------------------------------------- /edX-datascrub/src/diffUsers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | Examine a set of users.csv files for a class over some weeks, finding any differences 4 | 5 | This program is meant to be run from a directory of weekly dumps from edX. It 6 | finds all sub-directories and then looks for the class number supplied as a 7 | command-line argument. It goes into those class directories, and compares the 8 | users.csv files in those directories, creating a .csv file with the user id and 9 | user name for any user who is in one week and not in the next, or not in the first 10 | week and then in the next. 11 | 12 | Command line arguments 13 | ====================== 14 | The name of a class (assumed to be the name of a sub-directory of the current 15 | directory) that is to have the user lists compared 16 | 17 | ''' 18 | 19 | import glob 20 | import sys 21 | import csv 22 | import user 23 | 24 | course = sys.argv[1] 25 | flist = glob.glob('harvardx*/' + course + '/users.csv') 26 | if len(flist) < 2: 27 | exit() 28 | 29 | f = iter(flist).next() 30 | flist.remove(f) 31 | ufile = open(f, 'r') 32 | oldDict = user.builddict(ufile) 33 | ufile.close() 34 | out = csv.writer(open(course+'diffs.csv', 'w')) 35 | for f in flist: 36 | ufile = open(f, 'r') 37 | newDict = user.builddict(csv.reader(ufile)) 38 | ufile.close() 39 | out.writerow(['In older course list, not new']) 40 | i = 0 41 | for u in iter(oldDict): 42 | if u not in newDict: 43 | out.writerow([u, oldDict[u].username]) 44 | i += 1 45 | out.writerow(['Total deleted between files: ', str(i)]) 46 | i = 0 47 | out.writerow(['In new course list, not old']) 48 | for u in iter(newDict): 49 | if u not in oldDict: 50 | out.writerow([u, newDict[u].username]) 51 | i += 1 52 | out.writerow(['Total added between files : ', str(i)]) 53 | oldDict = newDict 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /edX-datascrub/src/demographics/buildAnonProfile.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | Created on Apr 22, 2013 5 | 6 | Create a file of FERPA de-identified demographic data 7 | 8 | The program takes five arguments: 9 | A string that names the course; 10 | A file containing the userprofile data, in CSV format; 11 | A file containing the user data, in CSV format; 12 | A file containing the mapping from usernames to countries; 13 | A file containing the users who received certificates (optional) 14 | 15 | 16 | @author: waldo 17 | ''' 18 | 19 | import sys 20 | import csv 21 | import userprofile as prof 22 | import user 23 | import certificates as cs 24 | import ipGeoloc as geo 25 | 26 | 27 | if (len(sys.argv) < 2): 28 | print('Usage: buildAnonProfile.py courseName profileFile userFile countryFile certFile') 29 | sys.exit() 30 | 31 | csv.field_size_limit(1000000) 32 | 33 | out_name = sys.argv[1] + 'anonProfile.csv' 34 | o1 = csv.writer(open(out_name, 'w')) 35 | 36 | ufile = csv.reader(open(sys.argv[2], 'r')) 37 | uprof = prof.builddict(ufile) 38 | 39 | udfile = csv.reader(open(sys.argv[3], 'r')) 40 | udict = user.builddict(udfile) 41 | 42 | countryFile = csv.reader(open(sys.argv[4], 'r')) 43 | locDict = geo.builddict(countryFile) 44 | 45 | certs = False 46 | if (len(sys.argv) > 5): 47 | certfile = csv.reader(open(sys.argv[5], 'r')) 48 | certDict = cs.builddict(certfile) 49 | certs = True 50 | 51 | 52 | students = uprof.keys() 53 | for s in students: 54 | p = uprof[s] 55 | if (s in udict): 56 | usrName = udict[s].username 57 | if (usrName in locDict): 58 | loc = locDict[usrName] 59 | else: 60 | loc = '' 61 | else: 62 | loc = '' 63 | 64 | if (certs): 65 | o1.writerow([p.gender, p.yob, p.ledu, p.goal, loc, (p.user_id in certDict)]) 66 | else: 67 | o1.writerow([p.gender, p.yob, p.ledu, p.goal, loc]) 68 | 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /edX-datascrub/src/logs/processLogData.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | Take the log files from all servers between startDate and endDate, 5 | and generate a log file for each course in the given input list. 6 | 7 | Command line arguments: 8 | 1) a list of class names to be processed, seperated by ',' WITHOUT whitespace. 9 | Class name is in school-class-term format. E.g. BerkeleyX-CS191x-Spring_2013. 10 | 2) start date in YYYY-MM-DD format 11 | 3) end date in YYYY-MM-DD format 12 | 13 | @auther: mangpo 14 | ''' 15 | 16 | import sys, os, glob 17 | 18 | if __name__ == '__main__': 19 | names = sys.argv[1] 20 | start = sys.argv[2] 21 | end = sys.argv[3] 22 | 23 | (yy1,mm1,dd1) = [int(x) for x in start.split('-')] 24 | (yy2,mm2,dd2) = [int(x) for x in end.split('-')] 25 | 26 | # Clear unknownLogs 27 | logFiles = glob.glob('prod*') 28 | for d in logFiles: 29 | os.system("rm %s/unknownLogs" % d) 30 | 31 | first = True 32 | 33 | # Divide a long period into multiple week periods to reduce sorting time 34 | while yy1 < yy2 or mm1 < mm2 or dd1 < dd2: 35 | (y,m,d) = (yy1,mm1,dd1) 36 | if d >= 31: 37 | if m >= 12: 38 | y = y+1 39 | m = 1 40 | d = 1 41 | else: 42 | m = m+1 43 | d = 1 44 | else: 45 | d = d+7 46 | if y >= yy2 and m >= mm2 and d > mm2: 47 | (y,m,d) = (yy2,mm2,dd2) 48 | 49 | if first: 50 | command = "processSmallLogData.sh %s %04d-%02d-%02d %04d-%02d-%02d" % (names,yy1,mm1,dd1,y,m,d) 51 | first = False 52 | else: 53 | command = "processSmallLogData.sh %s - %04d-%02d-%02d" % (names,y,m,d) 54 | 55 | 56 | print command 57 | status = os.system(command) 58 | 59 | if status == 0: 60 | (yy1,mm1,dd1) = (y,m,d+1) 61 | else: 62 | exit(status) 63 | 64 | 65 | -------------------------------------------------------------------------------- /edX-datascrub/src/buildCompRoster.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | Build a cumulative class roster for a class 5 | 6 | Run from a directory that contains all of the weekly dumps of data for a class from 7 | edX, this script will build a roster for all students who have ever shown up in the 8 | weekly profiles.csv files. The roster will include student id, name, country, age, 9 | education level, and gender. This should allow us to see all students in a course, 10 | even those that have withdrawn, as long as they remained in the course for a single 11 | data dump. 12 | 13 | Created on Jul 11, 2013 14 | Modified on Jul 28, 2013 15 | 16 | @author: waldo 17 | ''' 18 | 19 | import glob 20 | import csv 21 | import sys 22 | import utils 23 | import buildClassRoster as bcr 24 | import demographics.userprofile as prof 25 | import ipGeoloc as geo 26 | import user 27 | 28 | def main(locname, relLoc="./"): 29 | csv.field_size_limit(sys.maxsize) 30 | locD = geo.readIdToLoc(locname) 31 | flist = glob.glob(relLoc + '20*') 32 | fname = flist.pop() 33 | fin = csv.reader(open(fname + '/profiles.csv', 'r')) 34 | pDict = prof.builddict(fin) 35 | uin = csv.reader(open(fname +'/users.csv', 'r')) 36 | uDict = user.builddict(uin) 37 | fullR = bcr.buildRosterDict(pDict, uDict, locD) 38 | 39 | for f in flist: 40 | fin = csv.reader(open(f + '/profiles.csv', 'r')) 41 | addDict = prof.builddict(fin) 42 | uin = csv.reader(open(f + '/users.csv', 'r')) 43 | uDict = user.builddict(uin) 44 | addR = bcr.buildRosterDict(addDict, uDict, locD) 45 | for i in iter(addR): 46 | if i not in fullR: 47 | fullR[i] = addR[i] 48 | 49 | outname = relLoc + 'FullRoster.csv' 50 | bcr.writeRoster(fullR, outname) 51 | 52 | if __name__ == '__main__': 53 | if len(sys.argv) > 1: 54 | locname = sys.argv[1] 55 | else: 56 | locname = utils.getFileName('Enter name of the id=>location file :') 57 | 58 | main(locname) 59 | -------------------------------------------------------------------------------- /views/home/privacy.ejs: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Privacy Policy

4 |
5 |
6 |
7 |

8 | Your privacy is very important to us. Accordingly, we have developed this Policy in order for you to understand how we collect, use, communicate and disclose and make use of personal information. The following outlines our privacy policy. 9 |

10 | 11 | 34 | 35 |

36 | We are committed to conducting our business in accordance with these principles in order to ensure that the confidentiality of personal information is protected and maintained. 37 |

38 |
-------------------------------------------------------------------------------- /api/responses/badRequest.js: -------------------------------------------------------------------------------- 1 | /** 2 | * 400 (Bad Request) Handler 3 | * 4 | * Usage: 5 | * return res.badRequest(); 6 | * return res.badRequest(data); 7 | * return res.badRequest(data, 'some/specific/badRequest/view'); 8 | * 9 | * e.g.: 10 | * ``` 11 | * return res.badRequest( 12 | * 'Please choose a valid `password` (6-12 characters)', 13 | * 'trial/signup' 14 | * ); 15 | * ``` 16 | */ 17 | 18 | module.exports = function badRequest(data, options) { 19 | 20 | // Get access to `req`, `res`, & `sails` 21 | var req = this.req; 22 | var res = this.res; 23 | var sails = req._sails; 24 | 25 | // Set status code 26 | res.status(400); 27 | 28 | // Log error to console 29 | if (data !== undefined) { 30 | sails.log.verbose('Sending 400 ("Bad Request") response: \n',data); 31 | } 32 | else sails.log.verbose('Sending 400 ("Bad Request") response'); 33 | 34 | // Only include errors in response if application environment 35 | // is not set to 'production'. In production, we shouldn't 36 | // send back any identifying information about errors. 37 | if (sails.config.environment === 'production') { 38 | data = undefined; 39 | } 40 | 41 | // If the user-agent wants JSON, always respond with JSON 42 | if (req.wantsJSON) { 43 | return res.jsonx(data); 44 | } 45 | 46 | // If second argument is a string, we take that to mean it refers to a view. 47 | // If it was omitted, use an empty object (`{}`) 48 | options = (typeof options === 'string') ? { view: options } : options || {}; 49 | 50 | // If a view was provided in options, serve it. 51 | // Otherwise try to guess an appropriate view, or if that doesn't 52 | // work, just send JSON. 53 | if (options.view) { 54 | return res.view(options.view, { data: data }); 55 | } 56 | 57 | // If no second argument provided, try to serve the implied view, 58 | // but fall back to sending JSON(P) if no view can be inferred. 59 | else return res.guessView({ data: data }, function couldNotGuessView () { 60 | return res.jsonx(data); 61 | }); 62 | 63 | }; 64 | 65 | -------------------------------------------------------------------------------- /api/controllers/DataScriptController.js: -------------------------------------------------------------------------------- 1 | /** 2 | * DataScriptController 3 | * 4 | * @description :: Server-side logic for managing Datascripts 5 | * @help :: See http://links.sailsjs.org/docs/controllers 6 | */ 7 | 8 | var PATH_CONFIG = sails.config.paths; 9 | var DATA_DROP_PATH = PATH_CONFIG.DATASET_DROP_PATH; 10 | var kue = require('kue'); 11 | 12 | module.exports = { 13 | create: function(req, res) { 14 | return res.redirect('/admin/manage_data_scripts'); 15 | }, 16 | destroy: function(req, res) { 17 | return res.redirect('/admin/manage_data_scripts'); 18 | }, 19 | 20 | script_test: function(req, res) { 21 | var jobs = QueueService.getQueue(); 22 | var job = QueueService.createJob('test job', { user: 1, test: 'testparam' }); 23 | 24 | jobs.process('archive job', 1, function (job, done) { 25 | var frames = job.data.frames; 26 | 27 | function next(i) { 28 | if (i > 10) { 29 | done(); 30 | } else { 31 | sails.log(i); 32 | next(i+1); 33 | } 34 | } 35 | next(0); 36 | }); 37 | return res.json({ 'progress': 'Running' }); 38 | }, 39 | 40 | // BUILT-IN DATA MANAGEMENT TOOLS 41 | // http://blog.thesparktree.com/post/92465942639/ducktyping-sailsjs-core-for-background-tasks-via 42 | script_archive: function(req, res) { 43 | var jobs = QueueService.getQueue(); 44 | var job = QueueService.createJob('test job', { user: 1, test: 'testparam' }); 45 | 46 | jobs.process('archive job', 1, function (job, done) { 47 | var frames = job.data.frames; 48 | 49 | function next(i) { 50 | if (i > 10) { 51 | done(); 52 | } else { 53 | sails.log(i); 54 | next(i+1); 55 | } 56 | } 57 | next(0); 58 | }); 59 | return res.json({ 'progress': 'Running' }); 60 | }, 61 | 62 | script_move: function(req, res) { 63 | return res.redirect('/admin/manage_data_scripts'); 64 | }, 65 | 66 | upload: function(req, res) { 67 | return res.redirect('/admin/manage_data_scripts'); 68 | } 69 | }; 70 | 71 | -------------------------------------------------------------------------------- /test/SampleController.spec.js: -------------------------------------------------------------------------------- 1 | var request = require('supertest'); 2 | var assert = require('assert'); 3 | var async = require('async'); 4 | var stubs = require('../stubs.js'); 5 | 6 | 7 | describe('Post', function() { 8 | 9 | // // use after all to create custom stub data 10 | // var user; 11 | // before(function(done) { 12 | // User.create(uStub) 13 | // .exec( function(err, u){ 14 | // if( err ) { 15 | // console.log(err); 16 | // return done(err); 17 | // } 18 | 19 | // user = u; 20 | // user.password = password; 21 | // done(); 22 | // }); 23 | // }); 24 | 25 | describe('Authenticated', function() { 26 | // use supertest.agent for store cookies ... 27 | // logged in agent 28 | var agent ; 29 | // after authenticated requests login the user 30 | // before(function(done) { 31 | // agent = request.agent(sails.hooks.http.app); 32 | // agent.post('/auth/login') 33 | // .send({ 34 | // email: user.email, 35 | // password: user.password 36 | // }) 37 | // .end(function(err) { 38 | // done(err); 39 | // }); 40 | // }) 41 | 42 | describe('JSON Requests', function() { 43 | it('/relato should create one post with authenticated user as creator'); 44 | }); 45 | }) 46 | 47 | describe('UnAuthenticated', function() { 48 | describe('JSON Requests', function() { 49 | it('/relato should create one post and return 201',function(done) { 50 | 51 | var postStub = stubs.postStub(); 52 | request(sails.hooks.http.app) 53 | .post('/post') 54 | .send(postStub) 55 | .set('Accept', 'application/json') 56 | .expect('Content-Type', /json/) 57 | .end(function (err, res) { 58 | if(err) return done(err); 59 | 60 | assert.ok(res.body.title); 61 | assert.ok(res.body.body); 62 | assert.equal(res.body.title, postStub.title); 63 | assert.equal(res.body.body, postStub.body); 64 | 65 | done(); 66 | }); 67 | }); 68 | }); 69 | }) 70 | 71 | }) -------------------------------------------------------------------------------- /bin/setup/setup.sh: -------------------------------------------------------------------------------- 1 | ########################## 2 | # Author: Kevin Kao # 3 | ########################## 4 | #!/bin/bash 5 | 6 | echo "------------------------------------------------------------------" 7 | echo "| Welcome to moocRP setup. Creating file directory structure... |" 8 | echo "------------------------------------------------------------------" 9 | echo "" 10 | 11 | # Grab this script's absolute filepath to its directory 12 | SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 13 | 14 | DATASET_FOLDERS=( "encrypted" "extracted" "available" "available/non_pii" "available/pii" ) 15 | ANALYTIC_FOLDERS=("tmp" "archives") 16 | LOG_FILES=(production.log development.log) 17 | DATA_SCRIPTS=( "analytics" ) 18 | 19 | DATABASE_SETUP=db_setup.sql 20 | 21 | ERRORS=0 22 | 23 | for folder in "${DATASET_FOLDERS[@]}"; do CREATE="$SCRIPT_PATH/../../../datasets/${folder}"; echo "[x] Creating $CREATE"; mkdir -p "$CREATE"; done 24 | for folder in "${ANALYTIC_FOLDERS[@]}"; do CREATE="$SCRIPT_PATH/../../../analytics/${folder}"; echo "[x] Creating $CREATE"; mkdir -p "$CREATE"; done 25 | for folder in "${DATA_SCRIPTS[@]}"; do CREATE="$SCRIPT_PATH/../../../data_scripts/${folder}"; echo "[x] Creating $CREATE"; mkdir -p "$CREATE"; done 26 | 27 | 28 | mkdir -p "$SCRIPT_PATH/../../logs" 29 | 30 | for files in "${LOG_FILES[@]}" 31 | do 32 | CREATE="$SCRIPT_PATH/../../logs/${files}" 33 | echo "[x] Creating $CREATE" 34 | 35 | if [ ! -e "$CREATE" ] ; then 36 | touch "$CREATE" 37 | fi 38 | 39 | if [ ! -w "$CREATE" ] ; then 40 | echo "Unable to write to $CREATE - run this script as sudo" 41 | COUNTER=$((COUNTER + 1)) 42 | continue 43 | fi 44 | done 45 | 46 | echo "" 47 | echo "Setting up MySQL database..." 48 | printf "Please enter your MySQL username: " 49 | read -e mysqlUser 50 | 51 | printf "Please enter your MySQL password: " 52 | read -e -s mysqlPass 53 | echo "" 54 | 55 | printf "Ignore WARNING message from MySQL: " 56 | $(mysql -u $mysqlUser -p$mysqlPass < "$SCRIPT_PATH/$DATABASE_SETUP") 57 | 58 | echo "" 59 | 60 | if [[ $COUNTER -gt "0" ]] ; then 61 | echo "Setup encountered $COUNTER error(s)" 62 | exit 1 63 | fi 64 | echo "==== Setup was successful ====" 65 | exit 0 66 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "moocRP", 3 | "private": true, 4 | "version": "0.1.5", 5 | "description": "A learning analytics platform for MOOC research.", 6 | "keywords": [ 7 | "moocRP", 8 | "analytics", 9 | "edx", 10 | "research", 11 | "visualizations", 12 | "d3" 13 | ], 14 | "dependencies": { 15 | "adm-zip": "latest", 16 | "bcrypt": "latest", 17 | "cheerio": "~0.17.0", 18 | "deasync": "latest", 19 | "ejs": "~0.8.4", 20 | "fs-extra": "latest", 21 | "grunt": "0.4.2", 22 | "grunt-contrib-clean": "~0.5.0", 23 | "grunt-contrib-coffee": "~0.10.1", 24 | "grunt-contrib-concat": "~0.3.0", 25 | "grunt-contrib-copy": "~0.5.0", 26 | "grunt-contrib-cssmin": "~0.9.0", 27 | "grunt-contrib-jst": "~0.6.0", 28 | "grunt-contrib-less": "0.11.1", 29 | "grunt-contrib-uglify": "~0.4.0", 30 | "grunt-contrib-watch": "~0.5.3", 31 | "grunt-sails-linker": "~0.9.5", 32 | "grunt-sync": "~0.0.4", 33 | "include-all": "~0.1.3", 34 | "kue": "latest", 35 | "lodash": "~2.4.1", 36 | "nodemailer": "latest", 37 | "pm2": "latest", 38 | "process": "latest", 39 | "rc": "~0.5.0", 40 | "request": "latest", 41 | "rimraf": "latest", 42 | "sails": "~0.11", 43 | "sails-disk": "~0.10.0", 44 | "sails-mysql": "latest", 45 | "shelljs": "latest", 46 | "shortid": "latest", 47 | "winston": "^0.8.3" 48 | }, 49 | "bin": { 50 | "setup": "./bin/setup/setup.sh" 51 | }, 52 | "scripts": { 53 | "start": "node app.js", 54 | "debug": "node debug app.js", 55 | "test": "./node_modules/.bin/mocha test/bootstrap.js test/**/*.spec.js" 56 | }, 57 | "main": "app.js", 58 | "repository": { 59 | "type": "git", 60 | "url": "git://github.com/kk415kk/moocRP.git" 61 | }, 62 | "author": "Kevin Kao ", 63 | "license": "MIT", 64 | "devDependencies": { 65 | "grunt-cli": "latest", 66 | "grunt-contrib-watch": "^0.5.3", 67 | "grunt-mocha-test": "latest", 68 | "mocha": "^1.21.4", 69 | "sinon": "latest", 70 | "async": "latest", 71 | "barrels": "latest", 72 | "assert": "latest", 73 | "supertest": "latest", 74 | "sails-memory": "latest" 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /views/report/index.ejs: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Reports

4 |
5 |
6 | 7 |
8 |
9 |

Manage Notices & Reports

10 |
11 |
12 | 13 |
14 |
15 |
16 |

Reports

17 |

All reports by users (and non-users) will appear here, as well as in the inbox of the email address registered with this moocRP instance. Currently, messages can only be responded to through external email inboxes. Integration of replies, etc. is currently being developed. - 1/16/15

18 | <% _.each(reports, function(report) { %> 19 |
20 | <% if (report.type == 'Bug') { %> 21 | [<%= report.type %>] 22 | <% } else if (report.type == 'Inquiry'){ %> 23 | [<%= report.type %>] 24 | <% } else { %> 25 | [<%= report.type %>] 26 | <% } %> 27 | <%= report.createdAt %>: [Report ID: <%= report.id %>] 28 |
29 | <% }); %> 30 | 31 |
32 | 33 |
34 |

Message

35 |
36 |
37 |
38 |
39 |
40 | 41 | 42 | -------------------------------------------------------------------------------- /Gruntfile.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Gruntfile 3 | * 4 | * This Node script is executed when you run `grunt` or `sails lift`. 5 | * It's purpose is to load the Grunt tasks in your project's `tasks` 6 | * folder, and allow you to add and remove tasks as you see fit. 7 | * For more information on how this works, check out the `README.md` 8 | * file that was generated in your `tasks` folder. 9 | * 10 | * WARNING: 11 | * Unless you know what you're doing, you shouldn't change this file. 12 | * Check out the `tasks` directory instead. 13 | */ 14 | 15 | module.exports = function(grunt) { 16 | 17 | 18 | // Load the include-all library in order to require all of our grunt 19 | // configurations and task registrations dynamically. 20 | var includeAll; 21 | try { 22 | includeAll = require('include-all'); 23 | } catch (e0) { 24 | try { 25 | includeAll = require('sails/node_modules/include-all'); 26 | } 27 | catch(e1) { 28 | console.error('Could not find `include-all` module.'); 29 | console.error('Skipping grunt tasks...'); 30 | console.error('To fix this, please run:'); 31 | console.error('npm install include-all --save`'); 32 | console.error(); 33 | 34 | grunt.registerTask('default', []); 35 | return; 36 | } 37 | } 38 | 39 | 40 | /** 41 | * Loads Grunt configuration modules from the specified 42 | * relative path. These modules should export a function 43 | * that, when run, should either load/configure or register 44 | * a Grunt task. 45 | */ 46 | function loadTasks(relPath) { 47 | return includeAll({ 48 | dirname: require('path').resolve(__dirname, relPath), 49 | filter: /(.+)\.js$/ 50 | }) || {}; 51 | } 52 | 53 | /** 54 | * Invokes the function from a Grunt configuration module with 55 | * a single argument - the `grunt` object. 56 | */ 57 | function invokeConfigFn(tasks) { 58 | for (var taskName in tasks) { 59 | if (tasks.hasOwnProperty(taskName)) { 60 | tasks[taskName](grunt); 61 | } 62 | } 63 | } 64 | 65 | 66 | 67 | 68 | // Load task functions 69 | var taskConfigurations = loadTasks('./tasks/config'), 70 | registerDefinitions = loadTasks('./tasks/register'); 71 | 72 | // (ensure that a default task exists) 73 | if (!registerDefinitions.default) { 74 | registerDefinitions.default = function (grunt) { grunt.registerTask('default', []); }; 75 | } 76 | 77 | // Run task functions to configure Grunt. 78 | invokeConfigFn(taskConfigurations); 79 | invokeConfigFn(registerDefinitions); 80 | 81 | }; 82 | -------------------------------------------------------------------------------- /assets/js/app.js: -------------------------------------------------------------------------------- 1 | /** 2 | * app.js 3 | * 4 | * This file contains some conventional defaults for working with Socket.io + Sails. 5 | * It is designed to get you up and running fast, but is by no means anything special. 6 | * 7 | * Feel free to change none, some, or ALL of this file to fit your needs! 8 | */ 9 | 10 | 11 | (function (io) { 12 | 13 | // as soon as this file is loaded, connect automatically, 14 | var socket = io.connect(); 15 | if (typeof console !== 'undefined') { 16 | log('Connecting to Sails.js...'); 17 | } 18 | 19 | socket.on('connect', function socketConnected() { 20 | 21 | // Listen for Comet messages from Sails 22 | socket.on('message', function messageReceived(message) { 23 | 24 | /////////////////////////////////////////////////////////// 25 | // Replace the following with your own custom logic 26 | // to run when a new message arrives from the Sails.js 27 | // server. 28 | /////////////////////////////////////////////////////////// 29 | log('New comet message received :: ', message); 30 | ////////////////////////////////////////////////////// 31 | 32 | }); 33 | 34 | 35 | /////////////////////////////////////////////////////////// 36 | // Here's where you'll want to add any custom logic for 37 | // when the browser establishes its socket connection to 38 | // the Sails.js server. 39 | /////////////////////////////////////////////////////////// 40 | log( 41 | 'Socket is now connected and globally accessible as `socket`.\n' + 42 | 'e.g. to send a GET request to Sails, try \n' + 43 | '`socket.get("/", function (response) ' + 44 | '{ console.log(response); })`' 45 | ); 46 | /////////////////////////////////////////////////////////// 47 | 48 | 49 | }); 50 | 51 | 52 | // Expose connected `socket` instance globally so that it's easy 53 | // to experiment with from the browser console while prototyping. 54 | window.socket = socket; 55 | 56 | 57 | // Simple log function to keep the example simple 58 | function log () { 59 | if (typeof console !== 'undefined') { 60 | try { 61 | console.log.apply(console, arguments); 62 | } catch (err) { 63 | console.log.apply(console, err); 64 | } 65 | } 66 | } 67 | 68 | 69 | })( 70 | 71 | // In case you're wrapping socket.io to prevent pollution of the global namespace, 72 | // you can replace `window.io` with your own `io` here: 73 | window.io 74 | 75 | ); 76 | -------------------------------------------------------------------------------- /api/services/UtilService.js: -------------------------------------------------------------------------------- 1 | /************************************************* 2 | * Copyright 2014, moocRP * 3 | * Author: Kevin Kao * 4 | *************************************************/ 5 | 6 | var path = require('path'); 7 | var sid = require('shortid'); 8 | var process = require('process'); 9 | var fs = require('fs-extra'); 10 | 11 | var SUCCESS = sails.config.constants.SUCCESS, 12 | FAILURE = sails.config.constants.FAILURE; 13 | 14 | // Randomized Seeding - currently unused 15 | sid.characters('0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ+@'); 16 | sid.seed(42); 17 | 18 | module.exports = { 19 | 20 | addFileExt: function(fileName, extension) { 21 | if (fileName == null || extension == null) return ''; 22 | return fileName + extension; 23 | }, 24 | 25 | /** 26 | * @return a random alphanumeric string 27 | */ 28 | generateSID: function() { 29 | return sid.generate(); 30 | }, 31 | 32 | /** 33 | * @return a cleaned up name for a valid file 34 | */ 35 | safeFilename: function(name) { 36 | name = name.replace(/ /g, '-'); 37 | name = name.replace(/[^A-Za-z0-9-_\.]/g, ''); 38 | name = name.replace(/\.+/g, '.'); 39 | name = name.replace(/-+/g, '-'); 40 | name = name.replace(/_+/g, '_'); 41 | return name; 42 | }, 43 | 44 | /** 45 | * @return the filename without its extension 46 | */ 47 | fileMinusExt: function(fileName) { 48 | return fileName.split('.').slice(0, -1).join('.'); 49 | }, 50 | 51 | /** 52 | * @return the file extension of the file 53 | */ 54 | fileExtension: function(fileName) { 55 | return fileName.split('.').slice(-1)[0]; 56 | }, 57 | 58 | /** 59 | * @param filePath: path to file/folder to be moved 60 | * @param destination: path to folder to move to 61 | * @param allContents: if origin filePath is a folder, you can pass in a boolean true to move all contents of the folder but not the folder itself 62 | * @return the callback fn. with true if successful, false otherwise 63 | */ 64 | moveCommand: function(filePath, destination, allContents, cb) { 65 | var cmd = allContents ? 'mv ' + filePath + '/* ' + destination : 'mv ' + filePath + ' ' + destination 66 | exec = require('child_process').exec; 67 | sails.log('Moving files: ' + cmd); 68 | 69 | exec(cmd, function(error, stdout, stderr) { 70 | if (error) { 71 | return cb(error, FAILURE); 72 | } else { 73 | return cb(null, SUCCESS); 74 | } 75 | }); 76 | } 77 | }; -------------------------------------------------------------------------------- /test/controllers/UserController.spec.js: -------------------------------------------------------------------------------- 1 | var request = require('supertest'); 2 | var assert = require('assert'); 3 | var async = require('async'); 4 | var stubs = require('../stubs.js'); 5 | 6 | describe('User', function() { 7 | var testUser; 8 | var userParams = stubs.userStub(); 9 | 10 | describe('#create()', function() { 11 | describe('user registration', function() { 12 | it('should create a new user with the correct attributes.', function (done) { 13 | User.create(userParams, function(err, user) { 14 | if (err || !user) { 15 | throw new Error(err); 16 | } 17 | testUser = user; 18 | done(); 19 | }); 20 | }); 21 | }); 22 | 23 | describe('user attributes', function() { 24 | it('should have the correct user ID.', function (done) { 25 | assert.equal(testUser.id, userParams.id); 26 | done(); 27 | }); 28 | 29 | it('should have the correct public key ID.', function (done) { 30 | assert.equal(testUser.publicKeyID, userParams.publicKeyID); 31 | done(); 32 | }); 33 | 34 | it('should default to an administrator role for the first user.', function (done) { 35 | assert.equal(testUser.admin, true); 36 | done(); 37 | }); 38 | 39 | it('should not default to an administrator role for a second user.', function (done) { 40 | var userParams2 = stubs.userStub2(); 41 | User.create(userParams2, function (err, user) { 42 | if (err || !user) { 43 | throw new Error(err); 44 | } 45 | assert.equal(user.admin, false); 46 | done(); 47 | }); 48 | }); 49 | 50 | it('should have the correct first and last name.', function (done) { 51 | assert.equal(testUser.firstName, userParams.firstName); 52 | assert.equal(testUser.lastName, userParams.lastName); 53 | done(); 54 | }); 55 | 56 | it('should have the correct email address.', function (done) { 57 | assert.equal(testUser.email, userParams.email); 58 | done(); 59 | }); 60 | }); 61 | }); 62 | 63 | describe('#edit()', function() { 64 | describe('email', function() { 65 | it('TODO should edit the email address of the user.', function (done) { 66 | done(); 67 | }); 68 | }); 69 | }); 70 | 71 | after(function (done) { 72 | // Clean up the users in the system 73 | User.destroy(userParams.id, function (err) {}); 74 | User.destroy(stubs.userStub2().id, function (err) {}); 75 | done(); 76 | }) 77 | 78 | }); -------------------------------------------------------------------------------- /api/models/User.js: -------------------------------------------------------------------------------- 1 | /** 2 | * User 3 | * 4 | * @module :: Model 5 | * @description :: A short summary of how this model works and what it represents. 6 | * @docs :: http://sailsjs.org/#!documentation/models 7 | */ 8 | 9 | module.exports = { 10 | //migrate: 'drop', // use to drop all tables 11 | schema: true, 12 | 13 | attributes: { 14 | /* e.g. 15 | nickname: 'string' 16 | */ 17 | 18 | // Instance variables 19 | id: { 20 | type: 'STRING', 21 | required: true, 22 | unique: true, 23 | primaryKey: true 24 | }, 25 | firstName: { 26 | type: 'STRING', 27 | required: true 28 | }, 29 | lastName: { 30 | type: 'STRING', 31 | required: true 32 | }, 33 | email: { 34 | type: 'EMAIL', 35 | unique: true, 36 | }, 37 | registered: { 38 | type: 'BOOLEAN', 39 | defaultsTo: false 40 | }, 41 | admin: { 42 | type: 'BOOLEAN', 43 | defaultsTo: false 44 | }, 45 | publicKey: { 46 | type: 'TEXT', 47 | required: true 48 | }, 49 | publicKeyID: { 50 | type: 'STRING', 51 | required: true 52 | }, 53 | requests: { 54 | collection: 'request', 55 | via: 'requestingUser' 56 | }, 57 | analytics: { 58 | collection: 'analytic', 59 | via: 'owner' 60 | }, 61 | starredAnalytics: { 62 | dominant: true, 63 | collection: 'analytic', 64 | via: 'usersWhoStarred' 65 | }, 66 | 67 | // Instance methods 68 | toJSON: function() { 69 | var obj = this.toObject(); 70 | delete obj._csrf; 71 | return obj; 72 | }, 73 | }, 74 | 75 | // 'beforeCreate' is a lifecycle callback; essentially, these are hooks that we 76 | // can tape into to change data 77 | // See Sails documentation on Models for more info/examples 78 | beforeCreate: function (values, next) { 79 | // Comment out if other emails are allowed 80 | User.find().exec(function (err, users) { 81 | 82 | if (err) { 83 | sails.log('Error occurred while creating user: ' + err); 84 | } 85 | // Seed initial user as an admin 86 | if (!users || users.length == 0) { 87 | values.admin = true; 88 | } 89 | 90 | var matchingBerkeleyEmail = /berkeley.edu$/; 91 | if (!matchingBerkeleyEmail.test(values.email)) { 92 | // TODO: Add some new page 93 | return next("Please use a Berkeley email address."); 94 | } 95 | return next(); 96 | }); 97 | } 98 | 99 | }; 100 | -------------------------------------------------------------------------------- /api/controllers/AdminController.js: -------------------------------------------------------------------------------- 1 | /******************************************* 2 | * Copyright 2014, moocRP * 3 | * Author: Kevin Kao * 4 | *******************************************/ 5 | 6 | /** 7 | * AdminController 8 | * 9 | * @module :: Controller 10 | * @description :: A set of functions called `actions`. 11 | * 12 | * Actions contain code telling Sails how to respond to a certain type of request. 13 | * (i.e. do stuff, then send some JSON, show an HTML page, or redirect to another URL) 14 | * 15 | * You can configure the blueprint URLs which trigger these actions (`config/controllers.js`) 16 | * and/or override them with custom routes (`config/routes.js`) 17 | * 18 | * NOTE: The code you write here supports both HTTP and Socket.io automatically. 19 | * 20 | * @docs :: http://sailsjs.org/#!documentation/controllers 21 | */ 22 | 23 | module.exports = { 24 | 25 | /** 26 | * Overrides for the settings in `config/controllers.js` 27 | * (specific to AdminController) 28 | */ 29 | _config: {}, 30 | 31 | // View to show all users on page 32 | manage_users: function(req, res) { 33 | User.find(function foundUsers(err, users) { 34 | return res.view({ title: 'User Management', users: users }); 35 | }); 36 | }, 37 | 38 | // View to manage requests by researchers 39 | manage_requests: function (req, res) { 40 | Request.find().populate('requestingUser').populate('dataModel').exec(function foundRequests(err, requests) { 41 | return res.view({ title: 'Data Request Management', requests: requests }); 42 | }); 43 | }, 44 | 45 | // View to manage uploads by researchers 46 | manage_analytics: function(req, res) { 47 | Analytic.find().populate('owner').exec(function foundAnalytics(err, analytics) { 48 | return res.view({ title: 'Module Management', analytics: analytics }); 49 | }); 50 | }, 51 | 52 | manage_data_models: function(req, res) { 53 | DataModel.find(function foundDataModels(err, dataModels) { 54 | return res.view({ title: 'Data Model Management', dataModels: dataModels }); 55 | }); 56 | }, 57 | 58 | manage_notices: function(req, res) { 59 | Notice.find(function foundNotices(err, notices) { 60 | return res.view({ title: 'Notice Management', notices: notices}); 61 | }); 62 | }, 63 | 64 | manage_data_scripts: function(req, res) { 65 | DataModel.find(function foundDataModels(err, dataModels) { 66 | return res.view({ title: 'Data Script Management', dataModels: dataModels }); 67 | }); 68 | }, 69 | 70 | }; 71 | -------------------------------------------------------------------------------- /api/responses/forbidden.js: -------------------------------------------------------------------------------- 1 | /** 2 | * 403 (Forbidden) Handler 3 | * 4 | * Usage: 5 | * return res.forbidden(); 6 | * return res.forbidden(err); 7 | * return res.forbidden(err, 'some/specific/forbidden/view'); 8 | * 9 | * e.g.: 10 | * ``` 11 | * return res.forbidden('Access denied.'); 12 | * ``` 13 | */ 14 | 15 | module.exports = function forbidden (data, options) { 16 | 17 | // Get access to `req`, `res`, & `sails` 18 | var req = this.req; 19 | var res = this.res; 20 | var sails = req._sails; 21 | 22 | // Set status code 23 | res.status(403); 24 | 25 | // Log error to console 26 | if (data !== undefined) { 27 | sails.log.verbose('Sending 403 ("Forbidden") response: \n',data); 28 | } 29 | else sails.log.verbose('Sending 403 ("Forbidden") response'); 30 | 31 | // Only include errors in response if application environment 32 | // is not set to 'production'. In production, we shouldn't 33 | // send back any identifying information about errors. 34 | if (sails.config.environment === 'production') { 35 | data = undefined; 36 | } 37 | 38 | // If the user-agent wants JSON, always respond with JSON 39 | if (req.wantsJSON) { 40 | return res.jsonx(data); 41 | } 42 | 43 | // If second argument is a string, we take that to mean it refers to a view. 44 | // If it was omitted, use an empty object (`{}`) 45 | options = (typeof options === 'string') ? { view: options } : options || {}; 46 | 47 | // If a view was provided in options, serve it. 48 | // Otherwise try to guess an appropriate view, or if that doesn't 49 | // work, just send JSON. 50 | if (options.view) { 51 | return res.view(options.view, { data: data }); 52 | } 53 | 54 | // If no second argument provided, try to serve the default view, 55 | // but fall back to sending JSON(P) if any errors occur. 56 | else return res.view('403', { data: data }, function (err, html) { 57 | 58 | // If a view error occured, fall back to JSON(P). 59 | if (err) { 60 | // 61 | // Additionally: 62 | // • If the view was missing, ignore the error but provide a verbose log. 63 | if (err.code === 'E_VIEW_FAILED') { 64 | sails.log.verbose('res.forbidden() :: Could not locate view for error page (sending JSON instead). Details: ',err); 65 | } 66 | // Otherwise, if this was a more serious error, log to the console with the details. 67 | else { 68 | sails.log.warn('res.forbidden() :: When attempting to render error page view, an error occured (sending JSON instead). Details: ', err); 69 | } 70 | return res.jsonx(data); 71 | } 72 | 73 | return res.send(html); 74 | }); 75 | 76 | }; 77 | 78 | -------------------------------------------------------------------------------- /tasks/pipeline.js: -------------------------------------------------------------------------------- 1 | /** 2 | * grunt/pipeline.js 3 | * 4 | * The order in which your css, javascript, and template files should be 5 | * compiled and linked from your views and static HTML files. 6 | * 7 | * (Note that you can take advantage of Grunt-style wildcard/glob/splat expressions 8 | * for matching multiple files.) 9 | */ 10 | 11 | 12 | 13 | // CSS files to inject in order 14 | // 15 | // (if you're using LESS with the built-in default config, you'll want 16 | // to change `assets/styles/importer.less` instead.) 17 | var cssFilesToInject = [ 18 | 'styles/**/bootstrap.css', 19 | 'styles/**/bootstrap-glyphicons.css', 20 | 'styles/**/dataTables.bootstrap.css', 21 | 'styles/**/jquery.dataTables.css', 22 | 'styles/**/jquery.dataTables_themeroller.css', 23 | 'styles/**/*.css' 24 | ]; 25 | 26 | 27 | // Client-side javascript files to inject in order 28 | // (uses Grunt-style wildcard/glob/splat expressions) 29 | var jsFilesToInject = [ 30 | 31 | // Load sails.io before everything else 32 | 'js/dependencies/sails.io.js', 33 | 34 | // Dependencies like jQuery, or Angular are brought in here 35 | 'js/dependencies/**/*.js', 36 | 37 | // Other required dependencies come first 38 | 'js/jquery-1.11.0.min.js', 39 | 'js/bootstrap.js', 40 | 'js/jquery.dataTables.js', 41 | 42 | // All of the rest of your client-side js files 43 | // will be injected here in no particular order. 44 | 'js/**/*.js' 45 | ]; 46 | 47 | 48 | // Client-side HTML templates are injected using the sources below 49 | // The ordering of these templates shouldn't matter. 50 | // (uses Grunt-style wildcard/glob/splat expressions) 51 | // 52 | // By default, Sails uses JST templates and precompiles them into 53 | // functions for you. If you want to use jade, handlebars, dust, etc., 54 | // with the linker, no problem-- you'll just want to make sure the precompiled 55 | // templates get spit out to the same file. Be sure and check out `tasks/README.md` 56 | // for information on customizing and installing new tasks. 57 | var templateFilesToInject = [ 58 | 'templates/**/*.html' 59 | ]; 60 | 61 | 62 | 63 | // Prefix relative paths to source files so they point to the proper locations 64 | // (i.e. where the other Grunt tasks spit them out, or in some cases, where 65 | // they reside in the first place) 66 | module.exports.cssFilesToInject = cssFilesToInject.map(function(path) { 67 | return '.tmp/public/' + path; 68 | }); 69 | module.exports.jsFilesToInject = jsFilesToInject.map(function(path) { 70 | return '.tmp/public/' + path; 71 | }); 72 | module.exports.templateFilesToInject = templateFilesToInject.map(function(path) { 73 | return 'assets/' + path; 74 | }); 75 | -------------------------------------------------------------------------------- /api/controllers/ReportController.js: -------------------------------------------------------------------------------- 1 | /******************************************* 2 | * Copyright 2014, moocRP * 3 | * Author: Kevin Kao * 4 | *******************************************/ 5 | 6 | /** 7 | * ReportController 8 | * 9 | * @description :: Server-side logic for managing reports 10 | * @help :: See http://links.sailsjs.org/docs/controllers 11 | */ 12 | 13 | module.exports = { 14 | 15 | index: function(req, res) { 16 | Report.find(function (err, reports) { 17 | return res.view({ title: 'Reports', reports: reports}); 18 | }); 19 | }, 20 | 21 | send: function(req, res) { 22 | var nodemailer = require('nodemailer'); 23 | var transporter = nodemailer.createTransport(sails.config.transporter); 24 | var mailOptions = JSON.parse(JSON.stringify(sails.config.mailOptions)); 25 | 26 | var params = req.params.all(); 27 | var firstName = params['firstName'], 28 | lastName = params['lastName'], 29 | type = params['type'], 30 | emailAddress = params['emailAddress'], 31 | emailMessage = params['emailMessage'] 32 | 33 | var HTMLtext = '' 34 | HTMLtext += 'From: ' + firstName + ' ' + lastName; 35 | HTMLtext += '
Type: ' + type; 36 | HTMLtext += '
Email: ' + emailAddress; 37 | HTMLtext += '
Message:
'; 38 | HTMLtext += emailMessage; 39 | 40 | mailOptions['subject'] = mailOptions['subject'] + type; 41 | mailOptions['text'] = HTMLtext; 42 | mailOptions['html'] = HTMLtext; 43 | 44 | if (req.session.user && req.session.user.id) params['user'] = req.session.user.id; 45 | Report.create(params, function (err, report) { 46 | 47 | transporter.sendMail(mailOptions, function(error, info) { 48 | if (error) { 49 | sails.log(error) 50 | FlashService.error(req, "Unable to send message at this time. Please try again later."); 51 | return res.redirect('/contact'); 52 | } else { 53 | sails.log('Message sent: ' + info.response); 54 | FlashService.success(req, "Successfully sent message. Please allow 1-2 business days for a response."); 55 | return res.redirect('/contact'); 56 | } 57 | }); 58 | }); 59 | }, 60 | 61 | destroy: function(req, res) { 62 | Report.destroy(req.param('id'), function (err, report) { 63 | FlashService.success(req, 'Successfully deleted report.'); 64 | return res.redirect('/report'); 65 | }); 66 | }, 67 | 68 | reply: function (req, res) { 69 | //TODO 70 | }, 71 | 72 | update_status: function (req, res) { 73 | //TODO 74 | } 75 | 76 | }; 77 | 78 | -------------------------------------------------------------------------------- /edX-datascrub/src/moveWeeklyToClass.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | Moves files from an existing weekly directory (the default IQSS format) 4 | into directories organized by course: 5 | 6 | i.e. HarvardX/{COURSE}/ --> sys.argv[1]/{COURSE}/sys.argv[2]/ 7 | 8 | Runs from an existing weekly directory directly containing 9 | a directory for each course. Pass the following args at command line: 10 | 11 | 1) destination directory (containing directories for each course) 12 | 2) name of new directory for the week of files being moved 13 | 3) [optional] 'v' as a third argument to signal "verbose" printing 14 | of progress; errors are printed either way 15 | 16 | Created on July 16, 2013 17 | 18 | @author: tmullaney 19 | ''' 20 | 21 | import sys 22 | import os 23 | import shutil 24 | 25 | def main(): 26 | # check args 27 | if(len(sys.argv) < 3): 28 | print "[Error] Too few arguments" 29 | return -1 30 | 31 | dst_dir = sys.argv[1] 32 | new_dir_name = sys.argv[2] 33 | verbose = len(sys.argv) >= 4 and sys.argv[3] == 'v' 34 | cwd = os.getcwd() 35 | num_files_moved = 0 36 | 37 | if(verbose): print "Destination: " + sys.argv[1] + "/{COURSE}/" + sys.argv[2] 38 | 39 | # loop through each course's files 40 | src_dirs = os.listdir(cwd) 41 | for course_name in src_dirs: 42 | if(not os.path.isdir(course_name)): 43 | # some stray file, so skip it 44 | continue 45 | 46 | if(verbose): print "Moving files for " + course_name 47 | 48 | src_files = os.listdir(os.path.join(cwd, course_name)) 49 | for course_file_name in src_files: 50 | src = os.path.join(cwd, course_name, course_file_name) 51 | dst = os.path.join(dst_dir, course_name, new_dir_name) 52 | 53 | #if there is no directory for this course, make one 54 | if (not os.path.isdir(os.path.join(dst_dir, course_name))): 55 | try: 56 | os.mkdir(os.path.join(dst_dir, course_name)) 57 | except OSError as err: pass 58 | 59 | # make new week dir if necessary 60 | if(not os.path.isdir(dst)): 61 | try: os.mkdir(dst) 62 | except OSError as err: pass # display error on move 63 | 64 | # move file 65 | try: 66 | shutil.move(src, dst) 67 | num_files_moved += 1 68 | except: 69 | print 'error copying file', src 70 | 71 | if(verbose): print "Done. " + str(num_files_moved) + " files moved." 72 | 73 | if __name__ == "__main__": 74 | main() -------------------------------------------------------------------------------- /views/home/contact.ejs: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Contact

4 | <% if (messages && messages['error'].length > 0) { %> 5 |
6 | <% messages['error'].forEach(function(message) { %> 7 | <%= message %> 8 |
9 | <% }); %> 10 |
11 |
12 | <% } %> 13 | <% if (messages && messages['warning'].length > 0) { %> 14 |
15 | <% messages['warning'].forEach(function(message) { %> 16 | <%= message %> 17 |
18 | <% }); %> 19 |
20 |
21 | <% } %> 22 | <% if (messages && messages['success'].length > 0) { %> 23 |
24 | <% messages['success'].forEach(function(message) { %> 25 | <%= message %> 26 |
27 | <% }); %> 28 |
29 |
30 | <% } %> 31 |
32 |
33 |
34 |
35 |

If you have any questions/concerns, please contact us through this form below. We will try to respond within 2 business days.

36 |
37 | 38 | 39 |
40 |
41 | 42 | 43 |
44 |
45 | 46 | 47 |
48 |
49 | 50 |
51 | 52 | 53 | 54 | 55 |
56 |
57 | 58 |
59 | 60 | 61 |
62 | 63 | 64 |
65 |
-------------------------------------------------------------------------------- /edX-datascrub/src/demographics/getenrollmentdata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | This program will construct a table of when the students of a course registered, 4 | and will track the self-reported gender of those students. The result is a csv 5 | file that contains the date of enrollment, the number of self-reported males, 6 | the number of self-reported females, and the number who did not report, along 7 | with the total number that registered on that day. A final entry will give 8 | totals for the three possible gender choices (M, F, no report) and the total for 9 | the course. 10 | 11 | Usage: 12 | getenrollmentdata.py file1 file2 file3 13 | 14 | where: 15 | file1 is a csv representation of the student demographic information (user_profile), 16 | file2 is a csv representation of the course enrollment date (courseenrollent) 17 | file3 is a csv file that will be written with the enrollment information by day 18 | 19 | Created on Feb 18, 2013 20 | 21 | @author: waldo 22 | ''' 23 | 24 | import userprofile 25 | import csv 26 | import sys 27 | import logging 28 | 29 | if __name__ == '__main__': 30 | pass 31 | 32 | class enrollday: 33 | def __init__(self): 34 | self.m = 0 35 | self.f = 0 36 | self.n = 0 37 | self.t = 0 38 | 39 | 40 | 41 | def buildenrolldict(e, profiles): 42 | retdict = {} 43 | lineno = 0; 44 | for line in e: 45 | lineno += 1 46 | if len(line) != 4: 47 | logging.warning('bad enrollment line at ' + str(lineno)) 48 | continue 49 | [rid, uid, cid, date] = line 50 | if uid not in profiles: 51 | continue 52 | day = date[:date.find(' ')] 53 | gender = profiles[uid].gender 54 | if day not in retdict: 55 | retdict[day]= enrollday() 56 | if gender == 'm': 57 | retdict[day].m += 1 58 | elif gender == 'f': 59 | retdict[day].f += 1 60 | else : 61 | retdict[day].n += 1 62 | retdict[day].t += 1 63 | return retdict 64 | 65 | 66 | csv.field_size_limit(1000000000) 67 | f = csv.reader(open(sys.argv[1], 'r')) 68 | profdict = userprofile.builddict(f) 69 | e = csv.reader(open(sys.argv[2], 'r')) 70 | enrdict = buildenrolldict(e, profdict) 71 | it = sorted(enrdict.keys()) 72 | outfile = csv.writer(open(sys.argv[3], 'w')) 73 | mt = ft = nt = 0 74 | outfile.writerow(['Enroll Date', 'Male', 'Female', 'Unspecified', 'Total']) 75 | for date in it: 76 | rec = enrdict[date] 77 | outfile.writerow([date, rec.m, rec.f, rec.n, rec.t]) 78 | mt += rec.m 79 | ft += rec.f 80 | nt += rec.n 81 | 82 | outfile.writerow(['', mt, ft, nt, mt+ft+nt]) 83 | 84 | -------------------------------------------------------------------------------- /api/responses/serverError.js: -------------------------------------------------------------------------------- 1 | /** 2 | * 500 (Server Error) Response 3 | * 4 | * Usage: 5 | * return res.serverError(); 6 | * return res.serverError(err); 7 | * return res.serverError(err, 'some/specific/error/view'); 8 | * 9 | * NOTE: 10 | * If something throws in a policy or controller, or an internal 11 | * error is encountered, Sails will call `res.serverError()` 12 | * automatically. 13 | */ 14 | 15 | module.exports = function serverError (data, options) { 16 | 17 | // Get access to `req`, `res`, & `sails` 18 | var req = this.req; 19 | var res = this.res; 20 | var sails = req._sails; 21 | 22 | // Set status code 23 | res.status(500); 24 | 25 | // Log error to console 26 | if (data !== undefined) { 27 | sails.log.error('Sending 500 ("Server Error") response: \n',data); 28 | } 29 | else sails.log.error('Sending empty 500 ("Server Error") response'); 30 | 31 | // Only include errors in response if application environment 32 | // is not set to 'production'. In production, we shouldn't 33 | // send back any identifying information about errors. 34 | if (sails.config.environment === 'production') { 35 | data = undefined; 36 | } 37 | 38 | // If the user-agent wants JSON, always respond with JSON 39 | if (req.wantsJSON) { 40 | return res.jsonx(data); 41 | } 42 | 43 | // If second argument is a string, we take that to mean it refers to a view. 44 | // If it was omitted, use an empty object (`{}`) 45 | options = (typeof options === 'string') ? { view: options } : options || {}; 46 | 47 | // If a view was provided in options, serve it. 48 | // Otherwise try to guess an appropriate view, or if that doesn't 49 | // work, just send JSON. 50 | if (options.view) { 51 | return res.view(options.view, { data: data }); 52 | } 53 | 54 | // If no second argument provided, try to serve the default view, 55 | // but fall back to sending JSON(P) if any errors occur. 56 | else return res.view('500', { data: data }, function (err, html) { 57 | 58 | // If a view error occured, fall back to JSON(P). 59 | if (err) { 60 | // 61 | // Additionally: 62 | // • If the view was missing, ignore the error but provide a verbose log. 63 | if (err.code === 'E_VIEW_FAILED') { 64 | sails.log.verbose('res.serverError() :: Could not locate view for error page (sending JSON instead). Details: ',err); 65 | } 66 | // Otherwise, if this was a more serious error, log to the console with the details. 67 | else { 68 | sails.log.warn('res.serverError() :: When attempting to render error page view, an error occured (sending JSON instead). Details: ', err); 69 | } 70 | return res.jsonx(data); 71 | } 72 | 73 | return res.send(html); 74 | }); 75 | 76 | }; 77 | 78 | -------------------------------------------------------------------------------- /views/user/edit.ejs: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Edit Profile

4 | <% if (messages && messages['error'].length > 0) { %> 5 |
6 | <% messages['error'].forEach(function(message) { %> 7 | <%= message %> 8 |
9 | <% }); %> 10 |
11 |
12 | <% } %> 13 | <% if (messages && messages['warning'].length > 0) { %> 14 |
15 | <% messages['warning'].forEach(function(message) { %> 16 | <%= message %> 17 |
18 | <% }); %> 19 |
20 |
21 | <% } %> 22 | <% if (messages && messages['success'].length > 0) { %> 23 |
24 | <% messages['success'].forEach(function(message) { %> 25 | <%= message %> 26 |
27 | <% }); %> 28 |
29 |
30 | <% } %> 31 |
32 |
33 |
34 | 73 |
1: 81 | os.chdir(sys.argv[1]) 82 | flist = glob.glob('*.mongo') 83 | clist = reduceName(flist) 84 | ofile = open('weeklyClassList', 'w') 85 | writeList(ofile, clist) 86 | 87 | -------------------------------------------------------------------------------- /edX-datascrub/src/logs/buildWeekLog.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | Combine all of the log files in a directory into a single log file 4 | 5 | This script assumes that all of the log files to be combined are in a 6 | single directory, and that the name of the directory is the name of the 7 | class. The program will combine all of the files into a single file, with 8 | the lines in timestamp order. While unlikely, it is possible that multiple 9 | lines have the same timestamp; if that happens they will be ordered in the 10 | order they are seen when read. 11 | 12 | The main use for this script is to combine all of the files for a given 13 | week (as obtained from edX) into a file for that week. However, if multiple 14 | weeks log files are in the directory, they will all be combined. 15 | ''' 16 | 17 | import json 18 | import glob 19 | import csv 20 | import sys 21 | import os 22 | 23 | def buildClassList(): 24 | classes = [] 25 | cin = open('ClassList.csv', 'rU') 26 | cread = csv.reader(cin) 27 | for course, count in cread: 28 | if course not in classes: 29 | classes.append(course) 30 | return iter(classes) 31 | 32 | def combineLogs(className, logFiles): 33 | lineDict = {} 34 | dc = json.JSONDecoder() 35 | for fname in logFiles: 36 | inf = open(fname, 'r') 37 | lineNo = 1 38 | for line in inf: 39 | try: 40 | dcl = dc.decode(line) 41 | ts = dcl['time'] 42 | if ts not in lineDict: 43 | lineDict[ts] = [line] 44 | else: 45 | lineDict[ts].append(line) 46 | lineNo += 1 47 | except ValueError: 48 | print 'JSON error at line', str(lineNo) 49 | inf.close() 50 | return lineDict 51 | 52 | def writeCombLog(fname, log): 53 | i = 0 54 | if len(log) < 1: 55 | print 'Nothing to write for log', fname 56 | return 57 | outfile = open(fname, 'a') 58 | for d in sorted(iter(log)): 59 | last_l = None 60 | for l in sorted(log[d]): 61 | if not (l == last_l): 62 | i += 1 63 | outfile.write(l) 64 | last_l = l 65 | print 'wrote', str(i), 'lines to output file', fname 66 | print '-----------------' 67 | outfile.close() 68 | 69 | if __name__ == '__main__': 70 | cl = sys.argv[1] 71 | 72 | print 'about to process logs for', cl 73 | prodLogs = [] 74 | logFiles = glob.glob('prod*/' + cl + '-*') 75 | print logFiles 76 | for f in logFiles: 77 | print 'processing log', f 78 | prodLogs.append(f) 79 | 80 | prodDict = combineLogs(cl, prodLogs) 81 | writeCombLog(cl + '.log', prodDict) 82 | 83 | for f in logFiles: 84 | os.system("rm %s" % f) 85 | 86 | 87 | -------------------------------------------------------------------------------- /api/responses/notFound.js: -------------------------------------------------------------------------------- 1 | /** 2 | * 404 (Not Found) Handler 3 | * 4 | * Usage: 5 | * return res.notFound(); 6 | * return res.notFound(err); 7 | * return res.notFound(err, 'some/specific/notfound/view'); 8 | * 9 | * e.g.: 10 | * ``` 11 | * return res.notFound(); 12 | * ``` 13 | * 14 | * NOTE: 15 | * If a request doesn't match any explicit routes (i.e. `config/routes.js`) 16 | * or route blueprints (i.e. "shadow routes", Sails will call `res.notFound()` 17 | * automatically. 18 | */ 19 | 20 | module.exports = function notFound (data, options) { 21 | 22 | // Get access to `req`, `res`, & `sails` 23 | var req = this.req; 24 | var res = this.res; 25 | var sails = req._sails; 26 | 27 | // Set status code 28 | res.status(404); 29 | 30 | // Log error to console 31 | if (data !== undefined) { 32 | sails.log.verbose('Sending 404 ("Not Found") response: \n',data); 33 | } 34 | else sails.log.verbose('Sending 404 ("Not Found") response'); 35 | 36 | // Only include errors in response if application environment 37 | // is not set to 'production'. In production, we shouldn't 38 | // send back any identifying information about errors. 39 | if (sails.config.environment === 'production') { 40 | data = undefined; 41 | } 42 | 43 | // If the user-agent wants JSON, always respond with JSON 44 | if (req.wantsJSON) { 45 | return res.jsonx(data); 46 | } 47 | 48 | // If second argument is a string, we take that to mean it refers to a view. 49 | // If it was omitted, use an empty object (`{}`) 50 | options = (typeof options === 'string') ? { view: options } : options || {}; 51 | 52 | // If a view was provided in options, serve it. 53 | // Otherwise try to guess an appropriate view, or if that doesn't 54 | // work, just send JSON. 55 | if (options.view) { 56 | return res.view(options.view, { data: data }); 57 | } 58 | 59 | // If no second argument provided, try to serve the default view, 60 | // but fall back to sending JSON(P) if any errors occur. 61 | else return res.view('404', { data: data }, function (err, html) { 62 | 63 | // If a view error occured, fall back to JSON(P). 64 | if (err) { 65 | // 66 | // Additionally: 67 | // • If the view was missing, ignore the error but provide a verbose log. 68 | if (err.code === 'E_VIEW_FAILED') { 69 | sails.log.verbose('res.notFound() :: Could not locate view for error page (sending JSON instead). Details: ',err); 70 | } 71 | // Otherwise, if this was a more serious error, log to the console with the details. 72 | else { 73 | sails.log.warn('res.notFound() :: When attempting to render error page view, an error occured (sending JSON instead). Details: ', err); 74 | } 75 | return res.jsonx(data); 76 | } 77 | 78 | return res.send(html); 79 | }); 80 | 81 | }; 82 | 83 | -------------------------------------------------------------------------------- /config/i18n.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Internationalization / Localization Settings 3 | * (sails.config.i18n) 4 | * 5 | * If your app will touch people from all over the world, i18n (or internationalization) 6 | * may be an important part of your international strategy. 7 | * 8 | * 9 | * For more informationom i18n in Sails, check out: 10 | * http://sailsjs.org/#/documentation/concepts/Internationalization 11 | * 12 | * For a complete list of i18n options, see: 13 | * https://github.com/mashpie/i18n-node#list-of-configuration-options 14 | * 15 | * 16 | */ 17 | 18 | module.exports.i18n = { 19 | 20 | /*************************************************************************** 21 | * * 22 | * Which locales are supported? * 23 | * * 24 | ***************************************************************************/ 25 | 26 | // locales: ['en', 'es', 'fr', 'de'] 27 | 28 | /**************************************************************************** 29 | * * 30 | * What is the default locale for the site? Note that this setting will be * 31 | * overridden for any request that sends an "Accept-Language" header (i.e. * 32 | * most browsers), but it's still useful if you need to localize the * 33 | * response for requests made by non-browser clients (e.g. cURL). * 34 | * * 35 | ****************************************************************************/ 36 | 37 | // defaultLocale: 'en', 38 | 39 | /**************************************************************************** 40 | * * 41 | * Automatically add new keys to locale (translation) files when they are * 42 | * encountered during a request? * 43 | * * 44 | ****************************************************************************/ 45 | 46 | // updateFiles: false, 47 | 48 | /**************************************************************************** 49 | * * 50 | * Path (relative to app root) of directory to store locale (translation) * 51 | * files in. * 52 | * * 53 | ****************************************************************************/ 54 | 55 | // localesDirectory: '/config/locales' 56 | 57 | }; 58 | -------------------------------------------------------------------------------- /edX-datascrub/src/coursestudentstate.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Object definition and utility funcitons for the course student state data dumps 3 | 4 | Contains a definition of a coursestudentstate object, which contains all of the 5 | information exported in the coursestudentstate files from edx. Also contains a 6 | function allowing the construction of a dictionary, indexed by the student id, 7 | that maps to this state. As usual, there is also a function that will scrub the 8 | file of entries that are the wrong size. 9 | 10 | Created on Feb 24, 2013 11 | 12 | @author: waldo 13 | ''' 14 | 15 | import convertfiles.xmltocsv 16 | 17 | class coursestudentstate(object): 18 | ''' 19 | A representation of the state of student work in a course 20 | 21 | This object encapsulates the work a student has done in any 22 | edX course, and the course state itself. 23 | 24 | classdocs 25 | ''' 26 | 27 | 28 | def __init__(self, sid, mod_type, mod_id, student_id, state, grade, created, mod, 29 | max_grade, done, course_id): 30 | ''' 31 | Constructor 32 | ''' 33 | self.sid = sid 34 | self.mod_type = mod_type 35 | self.mod_id = mod_id 36 | self.student_id = student_id 37 | self.state = state 38 | self.grade = grade 39 | self.created = created 40 | self.modified = mod 41 | self.max_grade = max_grade 42 | self.done = done 43 | self.course_id = course_id 44 | 45 | def builddict(f, ptype = ''): 46 | ''' 47 | Build a dictionary, indexed by the state id, of the course state 48 | 49 | This function builds a dictionary of the student state of a course. 50 | Since the state is large, this also allows building a dictionary of 51 | only one part of the state, determined by the ptype that is handed 52 | in. 53 | ''' 54 | retdict = {} 55 | lineno = 0 56 | for line in f: 57 | lineno += 1 58 | if len(line) != 11: 59 | print 'bad row size at line ' + str(lineno) 60 | continue 61 | [sid, modt, modi, st_id, state, gr, cr, modif, mgr, done, c_id] = line 62 | if (ptype != '') and (modt != ptype): 63 | continue 64 | rec = coursestudentstate(sid, modt, modi, st_id, state, gr, cr, modif, 65 | mgr, done, c_id) 66 | retdict[sid] = rec 67 | 68 | return retdict 69 | 70 | def scrubcsstate(f1, f2): 71 | ''' 72 | Clean up the state of a cvs file representation of the student state 73 | 74 | This function will traverse a cvs file representation of the student 75 | course state, removing any entries that do not have the right number 76 | of fields (which can happen because of bad or dirty input) 77 | ''' 78 | convertfiles.xmltocsv.scrubcsv(f1, f2, 11) 79 | -------------------------------------------------------------------------------- /edX-datascrub/src/buildCertList.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | Builds a dictionary of all of the certificates a user has been 4 | considered for, indexed by the user id. 5 | 6 | It is given a directory which has all of the courses. Each course 7 | directory should have at least one date dump 8 | 9 | Writes a FullCertList.json file to the given directory 10 | 11 | Created on October 11, 2013 12 | 13 | @author: lazovich 14 | ''' 15 | 16 | import certificates 17 | from certificates import cert, CertEncoder 18 | import csv 19 | import sys 20 | import glob 21 | import json 22 | 23 | 24 | def processCerts(dir): 25 | ''' 26 | Construct a dictionary of certificate recipients for a course, 27 | given the directory of the certificates.csv file 28 | 29 | Parameters 30 | ----------- 31 | dir: A string corresponding to the directory of the certificates.csv 32 | ''' 33 | 34 | try: 35 | f = open(dir+"/certificates.csv", 'r') 36 | except IOError: 37 | return None 38 | 39 | infile = csv.reader(f) 40 | certDict = certificates.builddict(infile) 41 | 42 | return certDict 43 | 44 | 45 | def mergeCertDicts(dict1, dict2): 46 | ''' 47 | Take two dictionaries and merge them. Combines values with the 48 | same key as a list 49 | 50 | Parameters 51 | ----------- 52 | dict1, dict2: dictionaries to be merged 53 | ''' 54 | 55 | for key in dict2: 56 | if key in dict1: 57 | obj1 = dict1[key] 58 | obj2 = dict2[key] 59 | merged = None 60 | 61 | if isinstance(obj1, cert): 62 | merged = [obj1, obj2] 63 | else: 64 | obj1.append(obj2) 65 | merged = obj1 66 | 67 | dict1[key] = merged 68 | else: 69 | dict1[key] = dict2[key] 70 | 71 | return dict1 72 | 73 | 74 | def main(): 75 | if len(sys.argv) != 2: 76 | print "Usage: buildCertList.py dir_name" 77 | return 1 78 | 79 | indir = sys.argv[1] 80 | dirList = glob.glob(indir+"/"+"*x*20*") 81 | 82 | allCerts = {} 83 | 84 | # Iterate over all courses 85 | for dir in dirList: 86 | fList = glob.glob(dir+"/"+"*20*") 87 | 88 | # Iterate over all dumps 89 | allCourseCerts = {} 90 | 91 | for f in fList: 92 | certDict = processCerts(f) 93 | 94 | # Overwrites cert from earlier dumps 95 | # if user is already there 96 | allCourseCerts.update(certDict) 97 | 98 | mergeCertDicts(allCerts, allCourseCerts) 99 | 100 | outfile = open(indir + "/" + "FullCertList.json", 'w') 101 | outfile.write(json.dumps(allCerts, cls=CertEncoder)) 102 | 103 | 104 | if __name__ == '__main__': 105 | main() 106 | -------------------------------------------------------------------------------- /documentation/data_distribution.md: -------------------------------------------------------------------------------- 1 | [Back to README](../README.md) 2 | 3 | Data Distribution 4 | ================ 5 | This page is a work in-progress. You can contribute to it by forking this repository and making pull requests. 6 | 7 | Data distribution is the process of distributing the data dumps of various MOOCs. Essentially, for each course's data that you want to make available, the following steps should be followed (suppose the course was BerkeleyX-CS169.1.x): 8 | 9 | * Provided you have the dataset saved in a folder somewhere already, transfer it to the server that moocRP is on. 10 | * Archive the contents of the folder (the data for the course) into a .ZIP file, where the name of the archive is the name of the course, i.e. BerkeleyX-CS169.1.x.zip. 11 | * Move the archive to `moocRP_base/datasets/available/non_pii` or `moocRP_base/datasets/available/pii`, whichever is relevant. 12 | 13 | Then, the dataset will appear on the dashboard page of moocRP, available for a user to request. 14 | 15 | Analytics Data 16 | ============== 17 | To make a dataset available to be selected from the Analytics tab of moocRP, the following steps should be followed: 18 | 19 | * Using the same name as the archive for data distribution (minus the .ZIP at the end), create a folder in `moocRP_base/datasets/extracted/DATA_MODEL_NAME/` named after the archive name. 20 | * Extract the files of the dataset archive into the newly created folder. Make sure all the files are in the new folder and NOT nested inside some other folder, i.e. `moocRP_base/datasets/extracted/DATA_MODEL_NAME/ARCHIVE_NAME` should be as deep as the hierarchy of folders goes. 21 | 22 | The datasets should then appear in the Analytics tab, available for users to choose from if they've been granted access to the dataset. 23 | 24 | IMPORTANT NOTE: the name of the archive file for data distribution, the folder name for analytics data MUST be consistent and must be the name of the course offering. moocRP relies on the names of the folders and archive files to detect which datasets are referring to the same course offering. 25 | 26 | Example: 27 | ``` 28 | moocRP_base 29 | - datasets 30 | - keys 31 | - moocRP 32 | - available // for data distribution 33 | - non_pii 34 | - data_model_1 35 | - BerkeleyX-CS169.1x.zip // a dataset that's archived into a ZIP file 36 | - data_model_2 37 | - BerkeleyX-CS169.1x.zip // same dataset as above, but scrubbed into a diff format 38 | - pii 39 | ... 40 | - extracted // for analytics 41 | - data_model_1 42 | - BerkeleyX-CS169.1x // folder name is the same as the archive 43 | - data_model_1_data_file1 44 | - data_model_1_data_file2 45 | - data_model_2 46 | - BerkeleyX-CS169.1x 47 | - data_model_2_data_file1 48 | - data_model_2_data_file2 49 | - data_model_2_data_file3 50 | - encrypted 51 | ... 52 | ``` -------------------------------------------------------------------------------- /views/datamodel/info.ejs: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Develop

4 | <% if (messages && messages['error'].length > 0) { %> 5 |
6 | <% messages['error'].forEach(function(message) { %> 7 | <%= message %> 8 |
9 | <% }); %> 10 |
11 |
12 | <% } %> 13 | <% if (messages && messages['warning'].length > 0) { %> 14 |
15 | <% messages['warning'].forEach(function(message) { %> 16 | <%= message %> 17 |
18 | <% }); %> 19 |
20 |
21 | <% } %> 22 | <% if (messages && messages['success'].length > 0) { %> 23 |
24 | <% messages['success'].forEach(function(message) { %> 25 | <%= message %> 26 |
27 | <% }); %> 28 |
29 |
30 | <% } %> 31 |
32 |
33 |
34 |
35 |

Guide for contributing analytics modules

36 |
37 |
38 | 39 |
40 |
41 |
42 |

Welcome to the Developer's Guide for moocRP.

43 |

This page contains information about each data model: its files, the file schemas, and other information. Basic module development resources can also be found here. If anything is out of date, please contact us below.

44 |
45 |
46 | 47 |
48 |
49 |

Data Models

50 | <% _.each(dataModels, function(dataModel) { %> 51 |
<%= dataModel.displayName %>
52 | <% }) %> 53 |
54 | 55 |
56 |

Data Model Description

57 |
58 |

Click on a data model to view its description.

59 |
60 |
61 |
62 |
63 | 64 | -------------------------------------------------------------------------------- /edX-datascrub/src/checkData/checkUsersTimes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | Compare the user files for a class over multiple weeks, and determine if any of those not 4 | active in both weeks were awarded a certificate, and the date of enrollment of those who 5 | are in one list but not the other 6 | 7 | Looks at the user, enrollment, and certificate files in multiple directories for a class, 8 | and produces a list of those users that are in one user list and not the other. For those 9 | users, the script looks into the enrollment file to determine when the student enrolled, 10 | and at the certificates file to see if the student was awarded a certificate. 11 | 12 | Currently, the class for which this is done is specified as a command-line argument. The 13 | weeks compared is hard-coded. 14 | 15 | ''' 16 | 17 | import os 18 | import sys 19 | import csv 20 | import user 21 | import certificates 22 | import course_enrollment 23 | 24 | def compareUsers(d1, d2): 25 | retDict = {} 26 | for u in iter(d1): 27 | if u not in d2: 28 | retDict[u] = 'n' 29 | return retDict 30 | 31 | ck_course = sys.argv[1] 32 | 33 | dump2 = 'harvardx-2013-06-16' 34 | dump1 = 'harvardx-2013-06-02' 35 | userFile = '/' + ck_course + '/users.csv' 36 | certFile = '/' + ck_course + '/certificates.csv' 37 | enroll = '/' + ck_course + '/enrollment.csv' 38 | uf1 = csv.reader(open(dump1 + userFile, 'r')) 39 | uf2 = csv.reader(open(dump2 + userFile, 'r')) 40 | cf1 = csv.reader(open(dump1 + certFile, 'r')) 41 | cf2 = csv.reader(open(dump2 + certFile, 'r')) 42 | ef1 = csv.reader(open(dump1 + enroll, 'r')) 43 | ef2 = csv.reader(open(dump2 + enroll, 'r')) 44 | 45 | u1dict = user.builddict(uf1) 46 | u2dict = user.builddict(uf2) 47 | c1dict = certificates.builddict(cf1) 48 | c2dict = certificates.builddict(cf2) 49 | e1dict = course_enrollment.builddict(ef1) 50 | e2dict = course_enrollment.builddict(ef2) 51 | 52 | OneNotTwo = compareUsers(u1dict, u2dict) 53 | TwoNotOne = compareUsers(u2dict, u1dict) 54 | 55 | for u in iter(OneNotTwo): 56 | if u in c1dict and c1dict[u].status =='downloadable': 57 | OneNotTwo[u] = 'y' 58 | 59 | for u in iter(TwoNotOne): 60 | if u in c2dict and c2dict[u].status == 'downloadable': 61 | TwoNotOne[u] = 'y' 62 | 63 | outfile = csv.writer(open('userDiff06020616' + ck_course +'.csv', 'w')) 64 | outfile.writerow(['Users in 06/02 list but not in 06/16 list']) 65 | outfile.writerow(['User id', 'Certificate granted', 'Date enrolled']) 66 | for u in iter(OneNotTwo): 67 | if u in e1dict: 68 | signdate = e1dict[u].enroll_d 69 | else: 70 | signdate = '' 71 | outfile.writerow([u, OneNotTwo[u], signdate]) 72 | 73 | outfile.writerow(['Users in 06/16 list but not in 06/02 list']) 74 | outfile.writerow(['User id', 'Certificate granted', 'Date enrolled']) 75 | for u in iter(TwoNotOne): 76 | if u in e2dict: 77 | signdate = e2dict[u].enroll_d 78 | else: 79 | signdate = '' 80 | outfile.writerow([u, TwoNotOne[u], signdate]) 81 | 82 | 83 | 84 | -------------------------------------------------------------------------------- /test/stubs.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Stubs for use in tests 3 | * A GPG key must be generated for 4 | */ 5 | var stubs = {}; 6 | 7 | /************************* 8 | ** DATA MODEL STUBS ** 9 | *************************/ 10 | stubs.dataModelStub = function postStub() { 11 | return { 12 | displayName: 'Test Data Model', 13 | fileSafeName: 'test_data_model' 14 | } 15 | } 16 | 17 | stubs.dataModelStub2 = function postStub() { 18 | return { 19 | displayName: 'Test Data Model 2', 20 | fileSafeName: 'test_data_model_2' 21 | } 22 | } 23 | 24 | /************************** 25 | ** USER STUBS ** 26 | **************************/ 27 | stubs.userStub = function createUser() { 28 | return { 29 | id: '111111', 30 | firstName: 'John', 31 | lastName: 'Johnson', 32 | email: 'john.johnson@berkeley.edu', 33 | publicKey: testGPGKey, 34 | publicKeyID: '13779E47' 35 | } 36 | } 37 | 38 | stubs.userStub2 = function createUser() { 39 | return { 40 | id: '222222', 41 | firstName: 'Kevin', 42 | lastName: 'Chan', 43 | email: 'kevin.chan@berkeley.edu', 44 | publicKey: testGPGKey, 45 | publicKeyID: '13779E47' 46 | } 47 | } 48 | 49 | var testGPGKey = "-----BEGIN PGP PUBLIC KEY BLOCK-----" + 50 | "Version: GnuPG v1" + 51 | "" + 52 | "mQENBFSdIf4BCADATizHrS35lAsfqgI2PkvQoTUZSq+T0eSjmDjw0hrSChC+UpzW" + 53 | "l8Vqr/hVz3nhT9yVP+pyQhaWXDlGERmWej7rYW0JickunYvhM8D0RLRe7eXAYHQG" + 54 | "lWZimdOs9jarcnfh1cB94wgT8bgcEvgwmnHDpKwF/iVL7XTpUbwjGuQ5uNYqwM1w" + 55 | "nFpgr9tF9KRqqUJOpdzrr9LeIn+FtbP6l/WrYBn4+mmC203JXaV+7B0HQjHNIKUT" + 56 | "k4jJSrrMEmb2IkRRur5wA3f7QTDtb5XBkEpCrL8CA74LE8WXopK1fQpb82OmnqUV" + 57 | "OSrUdpzqRZk6UgBga5JCjkTVeFirtpJdxkm1ABEBAAG0OkpvaG4gSm9obnNvbiAo" + 58 | "bW9vY1JQIHRlc3Qga2V5KSA8am9obi5qb2huc29uQGJlcmtlbGV5LmVkdT6JATgE" + 59 | "EwECACIFAlSdIf4CGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJECGa274T" + 60 | "d55HqtUIAJQkxNYs7OKBPRYxVBURD1A9XSKAJBoJPSfEo95Gh8kCZRTK0yvoFVT4" + 61 | "vZBclFLIsg4mJ69sYboXRha4onpdOQkriCP4nG+kIgpcKO6fBYthvCsAUpR0Rz22" + 62 | "+BOBZFATYcytbSaKqhDg80F++MbDUd8ImiCbWtIfjfWRydn+YXatJ36zkX3wiJsh" + 63 | "wKOsvlcxwlc5V4MfPM2P7mRgYr+ger+FutBldPSdfsH+yJuAlQC3D2rw4q2SekS/" + 64 | "ud4baPnTD2+q6xNNQcP69/zUkz2q8Y0pmYqjDis9Cr3pWOJUgZ3vcUF3adRynMLC" + 65 | "8kGHwfHZDhfTnbfNuDXbM2/JTMTIYse5AQ0EVJ0h/gEIAJ5DfkkkG7kGry77A0Y0" + 66 | "lKsm0FMeshhdH7FHfpTdtRhPLamZVtUxABETbtTiaU7n4H4PxWmZ7CZriZpoZcf8" + 67 | "2+58JVhx2aNmJYjizdoY28XyOT+E+JhQgU7uxwUOaeGK0l3JU9fr0ynBCXY/Zv/M" + 68 | "/jsx6qFueYPVHZzHsdD1hZ9Sp3OdCKIlptbWq6fwnN0ZvehvHuzpNM6ybH9d8+LT" + 69 | "usZYQ49a2YUqHvsAKtHiEPOIXjj5D13r1sMXDnzowBsgRL3ARqYLphGUe6xwYpDa" + 70 | "qy5Z4Rj7LSjTd/koYL0gFLSYPFIBWxneWMsK45OYCo2/t6aXrjKYtbghgzV7hA85" + 71 | "MlUAEQEAAYkBHwQYAQIACQUCVJ0h/gIbDAAKCRAhmtu+E3eeR1JpB/wOKl/rwVLn" + 72 | "PFFp8ikrNZQaHG1Ycch9mX0UXjoaKlpta7W67dP2MBZmhMDK6fu9qHHpXeS7MwWp" + 73 | "jshZ3c30dOSNbeP/FkWOfg7vjHeix3KoeoaQoBmVO0WreWjm8JTsCtN7dOQQG1Ui" + 74 | "nbd44hBdQrnwudMl7kGV/kDYdQql5Luqtqlnzenthc+eR67DLTD7Q/K0HtZduvz3" + 75 | "h1BN1GSrkentod23Jp8+H0HXtEO3ddzl9qMWAlOjVJneSW5fCZMPlpsr8LHZf3mV" + 76 | "zp9YQwspPuwiPooKLKm775J6s+BrFlsszXo5DF8XDmzTMSl9SSqxQB1O93YUJzHr" + 77 | "Ai/xmb5/2vO0" + 78 | "=r0uC" + 79 | "-----END PGP PUBLIC KEY BLOCK-----"; 80 | 81 | module.exports = stubs; -------------------------------------------------------------------------------- /config/csrf.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Cross-Site Request Forgery Protection Settings 3 | * (sails.config.csrf) 4 | * 5 | * CSRF tokens are like a tracking chip. While a session tells the server that a user 6 | * "is who they say they are", a csrf token tells the server "you are where you say you are". 7 | * 8 | * When enabled, all non-GET requests to the Sails server must be accompanied by 9 | * a special token, identified as the '_csrf' parameter. 10 | * 11 | * This option protects your Sails app against cross-site request forgery (or CSRF) attacks. 12 | * A would-be attacker needs not only a user's session cookie, but also this timestamped, 13 | * secret CSRF token, which is refreshed/granted when the user visits a URL on your app's domain. 14 | * 15 | * This allows us to have certainty that our users' requests haven't been hijacked, 16 | * and that the requests they're making are intentional and legitimate. 17 | * 18 | * This token has a short-lived expiration timeline, and must be acquired by either: 19 | * 20 | * (a) For traditional view-driven web apps: 21 | * Fetching it from one of your views, where it may be accessed as 22 | * a local variable, e.g.: 23 | *
24 | * 25 | *
26 | * 27 | * or (b) For AJAX/Socket-heavy and/or single-page apps: 28 | * Sending a GET request to the `/csrfToken` route, where it will be returned 29 | * as JSON, e.g.: 30 | * { _csrf: 'ajg4JD(JGdajhLJALHDa' } 31 | * 32 | * 33 | * Enabling this option requires managing the token in your front-end app. 34 | * For traditional web apps, it's as easy as passing the data from a view into a form action. 35 | * In AJAX/Socket-heavy apps, just send a GET request to the /csrfToken route to get a valid token. 36 | * 37 | * For more information on CSRF, check out: 38 | * http://en.wikipedia.org/wiki/Cross-site_request_forgery 39 | * 40 | * For more information on this configuration file, including info on CSRF + CORS, see: 41 | * http://beta.sailsjs.org/#/documentation/reference/sails.config/sails.config.csrf.html 42 | * 43 | */ 44 | 45 | /**************************************************************************** 46 | * * 47 | * Enabled CSRF protection for your site? * 48 | * * 49 | ****************************************************************************/ 50 | 51 | module.exports.csrf = true; 52 | 53 | /**************************************************************************** 54 | * * 55 | * You may also specify more fine-grained settings for CSRF, including the * 56 | * domains which are allowed to request the CSRF token via AJAX. These * 57 | * settings override the general CORS settings in your config/cors.js file. * 58 | * * 59 | ****************************************************************************/ 60 | 61 | // module.exports.csrf = { 62 | // grantTokenViaAjax: true, 63 | // origin: '' 64 | // } 65 | -------------------------------------------------------------------------------- /edX-datascrub/src/buildAllStudents.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | ''' 4 | This is a file for building a complete list of users who have taken a course 5 | and what they've taken. As such, it's going to be saved as a JSON file. 6 | 7 | This is meant to be run from within the repository. Clone the repo into the 8 | /Harvard directory on the shared space, then cd to this local directory and 9 | execute the script. As you can see below, some assumptions are made about the 10 | relative location of things, so it's important to either do as I have outlined 11 | or update the logic below. 12 | 13 | @author EJ Bensing 14 | ''' 15 | 16 | import glob 17 | import buildCompRoster as bcr 18 | import csv 19 | import json 20 | 21 | # yah, this isn't the most hacky thing ever... like this is really bad 22 | dirsToExclude = ['Logs', 'LOGS-FROM-2012', 'Unknown', 'idtolocation.csv', 23 | 'rosters.tar', 'Course_Axes', 'ebensing-scripts', 'HarvardX-Tools', 24 | 'globalis2name.csv', 'globalname2id.csv'] 25 | 26 | # relative path to the course data top level folder 27 | relPath='../../../../' 28 | 29 | # location of the id=>location file, from the relPath directory 30 | idLocFile='idtolocation.csv' 31 | 32 | def main(saveName='StudentCourses.json'): 33 | # since we're changing the value of the global variable, we need to declare 34 | # it. I guess python != javascript 35 | global idLocFile 36 | global dirsToExclude 37 | dirsToExclude.append(saveName) 38 | rdirsToExclude = [relPath + x for x in dirsToExclude] 39 | idLocFile = relPath + idLocFile 40 | # get the dirs we're going to search 41 | courseDirs = [x + '/' for x in glob.glob(relPath + '*') if x not in rdirsToExclude] 42 | 43 | # build all of the class roster CSV files 44 | for courseDir in courseDirs: 45 | print "Processing " + courseDir 46 | bcr.main(idLocFile, courseDir) 47 | 48 | rosterFiles = [x + 'FullRoster.csv' for x in courseDirs] 49 | fullUsers = {} 50 | # iterate over all of the full class rosters and create the new dict that 51 | # we'll write out 52 | for rfile in rosterFiles: 53 | with open(rfile, 'r') as csvfile: 54 | reader = csv.DictReader(csvfile) 55 | for row in reader: 56 | 57 | id = row['Student ID'] 58 | if id not in fullUsers: 59 | fullUsers[id] = {} 60 | fullUsers[id]["courses"] = [] 61 | for key,val in row.iteritems(): 62 | # grab all the demographic information except the 63 | # student ID since that is the key to the dictionary 64 | # anyway 65 | if key != 'Student ID': 66 | fullUsers[id][key] = val 67 | 68 | # hey, what's the worst way we could get the course name? I 69 | # think this is coming close... 70 | fullUsers[id]["courses"].append(rfile.replace(relPath, 71 | "").replace("FullRoster.csv","").replace("/","")) 72 | 73 | with open(relPath + saveName, 'w') as wfile: 74 | wfile.write(json.dumps(fullUsers)) 75 | 76 | 77 | 78 | 79 | 80 | if __name__ == '__main__': 81 | main() 82 | -------------------------------------------------------------------------------- /tasks/README.md: -------------------------------------------------------------------------------- 1 | # About the `tasks` folder 2 | 3 | The `tasks` directory is a suite of Grunt tasks and their configurations, bundled for your convenience. The Grunt integration is mainly useful for bundling front-end assets, (like stylesheets, scripts, & markup templates) but it can also be used to run all kinds of development tasks, from browserify compilation to database migrations. 4 | 5 | If you haven't used [Grunt](http://gruntjs.com/) before, be sure to check out the [Getting Started](http://gruntjs.com/getting-started) guide, as it explains how to create a [Gruntfile](http://gruntjs.com/sample-gruntfile) as well as install and use Grunt plugins. Once you're familiar with that process, read on! 6 | 7 | 8 | ### How does this work? 9 | 10 | The asset pipeline bundled in Sails is a set of Grunt tasks configured with conventional defaults designed to make your project more consistent and productive. 11 | 12 | The entire front-end asset workflow in Sails is completely customizable-- while it provides some suggestions out of the box, Sails makes no pretense that it can anticipate all of the needs you'll encounter building the browser-based/front-end portion of your application. Who's to say you're even building an app for a browser? 13 | 14 | 15 | 16 | ### What tasks does Sails run automatically? 17 | 18 | Sails runs some of these tasks (the ones in the `tasks/register` folder) automatically when you run certain commands. 19 | 20 | ###### `sails lift` 21 | 22 | Runs the `default` task (`tasks/register/default.js`). 23 | 24 | ###### `sails lift --prod` 25 | 26 | Runs the `prod` task (`tasks/register/prod.js`). 27 | 28 | ###### `sails www` 29 | 30 | Runs the `build` task (`tasks/register/build.js`). 31 | 32 | ###### `sails www --prod` (production) 33 | 34 | Runs the `buildProd` task (`tasks/register/buildProd.js`). 35 | 36 | 37 | ### Can I customize this for SASS, Angular, client-side Jade templates, etc? 38 | 39 | You can modify, omit, or replace any of these Grunt tasks to fit your requirements. You can also add your own Grunt tasks- just add a `someTask.js` file in the `grunt/config` directory to configure the new task, then register it with the appropriate parent task(s) (see files in `grunt/register/*.js`). 40 | 41 | 42 | ### Do I have to use Grunt? 43 | 44 | Nope! To disable Grunt integration in Sails, just delete your Gruntfile or disable the Grunt hook. 45 | 46 | 47 | ### What if I'm not building a web frontend? 48 | 49 | That's ok! A core tenant of Sails is client-agnosticism-- it's especially designed for building APIs used by all sorts of clients; native Android/iOS/Cordova, serverside SDKs, etc. 50 | 51 | You can completely disable Grunt by following the instructions above. 52 | 53 | If you still want to use Grunt for other purposes, but don't want any of the default web front-end stuff, just delete your project's `assets` folder and remove the front-end oriented tasks from the `grunt/register` and `grunt/config` folders. You can also run `sails new myCoolApi --no-frontend` to omit the `assets` folder and front-end-oriented Grunt tasks for future projects. You can also replace your `sails-generate-frontend` module with alternative community generators, or create your own. This allows `sails new` to create the boilerplate for native iOS apps, Android apps, Cordova apps, SteroidsJS apps, etc. 54 | 55 | -------------------------------------------------------------------------------- /edX-datascrub/src/convertfiles/killListedFiles.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | A script made necessary by the inability to distinguish between experimental 4 | or test courses and the real thing on the edX platform. 5 | 6 | This script will get rid of a lot of files that are generated by the data 7 | extraction programs used by edX. The strings that are used to match files 8 | that can be removed are obtained by experience, and have to be updated 9 | every now and then as new experiments are tried. 10 | 11 | It might be better to use the course list generated by the log processing 12 | scripts as the set of classes to keep, and then delete any file associated with 13 | a class that hasn't had any log data generated for it during the week. But for 14 | the moment this is the technique being used. 15 | 16 | @author: waldo 17 | ''' 18 | 19 | import os 20 | import glob 21 | 22 | killList = ['00001', 23 | '001-Intro_to_Learning_Management_Eco_System', 24 | '007Horsey', 25 | '100', 26 | '101-My', 27 | '101-Sept', 28 | '12345', 29 | '1795', 30 | '50000', 31 | '50-Take_2_on_edX', 32 | 'AIU12x', 33 | 'AV101', 34 | 'Biblical_Literacy', 35 | 'CB22x*edge', 36 | 'colin', 37 | 'Colin', 38 | 'Demo', 39 | 'Dogs', 40 | 'DOGS', 41 | 'Dummy', 42 | 'Edelman102', 43 | 'FAS2.1x', 44 | 'Fly_Fishing_', 45 | 'Gov2001', 46 | 'GSE102x', 47 | 'HarvardX-101', 48 | 'HeroesX-HeroesX', 49 | 'HKS-211', 50 | 'HKS211.1x-Central', 51 | 'HLS1x*edge', 52 | 'HS121x-Fall*edge', 53 | 'HSD1544.1x-3T', 54 | 'HX101', 55 | 'ITCx-ITCx', 56 | 'JandR', 57 | 'JS101', 58 | 'JS50', 59 | 'KMH1-Kuriyama_Prototype', 60 | 'Law-LRW_2', 61 | 'SLW1-Legal_Research_for_Non-Lawyers', 62 | 'MCB63', 63 | 'Math101', 64 | 'Mockup', 65 | 'NA_001', 66 | 'NA001', 67 | 'PC10', 68 | 'PH207x-*edge', 69 | 'PH207x-Health', 70 | 'QUANTUM', 71 | 'SAI-HGHI', 72 | 'SHAKE1x', 73 | 'sheep', 74 | 'Sheep', 75 | 'Slow_Cooking_Basics', 76 | 'SP001', 77 | 'SPU17x-3T2013', 78 | 'SPU27X', 79 | 'SW-12X', 80 | 'SW12-ChinaX', 81 | 'SW12x-2013_Oct', 82 | 'SW12.4X', 83 | 'T5532', 84 | 'TBD', 85 | 'test', 86 | 'Test', 87 | 'TR101', 88 | 'Tropicana', 89 | 'TT01x', 90 | 'UH001', 91 | 'WW-TFU1', 92 | 'wiki', 93 | 'WLX', 94 | 'WP1', 95 | 'xxx' 96 | ] 97 | 98 | def killFiles(fileList): 99 | for f in fileList: 100 | os.remove(f) 101 | 102 | if __name__ == '__main__': 103 | for k in killList: 104 | l = glob.glob('*'+k+'*') 105 | killFiles(l) 106 | -------------------------------------------------------------------------------- /edX-datascrub/src/logs/checkDates.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' Check that 4 | 1) start date is '-' if that course has already been processed. 5 | 2) end date is after the end date of the previous run. 6 | 7 | This script is useful if processSmallLogData.sh is executed manually. 8 | 9 | @auther: mangpo 10 | ''' 11 | 12 | 13 | import os, sys 14 | 15 | def get_class_list(): 16 | if os.path.isfile('ClassList.csv'): 17 | f = open('ClassList.csv','r') 18 | courses = {} 19 | for line in f: 20 | tokens = line.split(',') 21 | if len(tokens) == 3: 22 | (course,start,end) = tokens 23 | courses[course] = (start,end) 24 | f.close() 25 | return courses 26 | else: 27 | return {} 28 | 29 | def write_class_list(courses): 30 | f = open('NewClassList.csv','w') 31 | for course in courses: 32 | (start,end) = courses[course] 33 | f.write(course + "," + start + "," + end + "\n") 34 | f.close() 35 | 36 | def same_or_before(start,date): 37 | if start[0] > date[0]: 38 | return False 39 | elif start[0] < date[0]: 40 | return True 41 | elif start[1] > date[1]: 42 | return False 43 | elif start[1] < date[1]: 44 | return True 45 | else: 46 | return start[2] <= date[2] 47 | 48 | if __name__ == '__main__': 49 | names = sys.argv[1].split(',') 50 | startDate = sys.argv[2] 51 | endDate = sys.argv[3] 52 | 53 | if endDate == "-": 54 | print "Abort:", cl, "illegal end date." 55 | exit(1) 56 | 57 | class_list = get_class_list() 58 | dates = {} 59 | newStartDate = startDate 60 | newEndDate = endDate 61 | for cl in names: 62 | if cl in class_list: 63 | (oldStartDate,oldEndDate) = class_list[cl] 64 | if oldEndDate >= endDate: 65 | print "Abort:", cl, "logs have already been processed until", oldEndDate 66 | exit(1) 67 | elif not(startDate == "-"): 68 | print "Abort:", cl,"logs have already been processed." 69 | print "Use \'-\' for \'start date\' to continue processing from the latest date." 70 | exit(1) 71 | 72 | class_list[cl] = (oldStartDate, endDate) 73 | (yy,mm,dd) = oldEndDate.split('-') 74 | newStartDate = "%s-%s-%02d" % (yy,mm,int(dd)+1) 75 | dates[cl] = (newStartDate, newEndDate) 76 | else: 77 | if startDate == "-": 78 | print "Abort:", cl, "logs have NOT been processed before. Start date cannot be \'-\'" 79 | exit(1) 80 | 81 | class_list[cl] = (startDate, endDate) 82 | dates[cl] = (startDate,endDate) 83 | 84 | for cl in dates: 85 | (start,end) = dates[cl] 86 | if not(start == newStartDate): 87 | print "Cannot process this group of courses together because of the dates." 88 | print dates 89 | exit(1) 90 | 91 | print class_list 92 | write_class_list(class_list) 93 | f = open("dates.txt","w") 94 | f.write(newStartDate + "\n") 95 | f.write(newEndDate + "\n") 96 | f.close() 97 | -------------------------------------------------------------------------------- /documentation/installation.md: -------------------------------------------------------------------------------- 1 | [Back to README](../README.md) 2 | 3 | Installation 4 | ================ 5 | 6 | ## Dependencies 7 | 8 | **Ubuntu Instructions** 9 | 10 | * Install ````git````: ````sudo apt-get install git```` 11 | * Install Node.js ~0.10.25, minimum 0.10.x: 12 | 13 | ```` 14 | sudo apt-get install python-software-properties 15 | sudo apt-add-repository ppa:chris-lea/node.js 16 | sudo apt-get update 17 | sudo apt-get install nodejs-legacy 18 | ```` 19 | 20 | * Install npm (Node.js package manager) ~1.3.10: 21 | 22 | Use ````aptitude```` to install npm and downgrade Node.js through the prompt if conflicts occur. 23 | ```` 24 | sudo apt-get install aptitude 25 | sudo aptitude install npm 26 | ```` 27 | 28 | * Install MySQL server: ````sudo apt-get install mysql-server-5.6```` 29 | * Install Redis (latest installation instructions [here](http://redis.io/topics/quickstart)): 30 | 31 | ``` 32 | wget http://download.redis.io/redis-stable.tar.gz 33 | tar xvzf redis-stable.tar.gz 34 | cd redis-stable 35 | make 36 | make test 37 | sudo make install 38 | ``` 39 | 40 | * Install Sails.js ~0.10.5, minimum 0.10.x: ````sudo npm install -g sails```` 41 | 42 | ## Setup Instructions 43 | Make sure MySQL and Redis are running before launching moocRP. 44 | 45 | First, create a new folder called moocRP_base to clone this repository to: 46 | ```` 47 | mkdir moocRP_base 48 | cd moocRP_base 49 | git clone http://github.com:kk415kk/moocRP.git 50 | ```` 51 | 52 | After cloning this repository, run the setup script to create the correct directory structure. Enter in the correct MySQL user and password when prompted. This will create the database as well. 53 | ```` 54 | ./bin/setup/setup.sh 55 | ```` 56 | 57 | Once the setup script is run, the file structure setup should be in this format: 58 | ```` 59 | /moocRP_base 60 | ---- /moocRP (web application cloned from Github) 61 | -------- /api 62 | -------- /assets 63 | ------------ /scaffolds 64 | ------------ /analytics 65 | -------- ... 66 | -------- /logs 67 | -------- /bin 68 | ------------ setup.sh [setup script to create directory structure] 69 | ---- /datasets 70 | -------- /available 71 | ---------- /non_pii 72 | ---------- /pii 73 | ---------- /data_drop 74 | -------- /extracted 75 | -------- /encrypted 76 | ---- /analytics 77 | -------- /tmp 78 | -------- /archives 79 | ```` 80 | 81 | Then, we need to install all npm package dependencies before launch: 82 | ```` 83 | cd moocRP_base/moocRP 84 | sudo npm install 85 | ```` 86 | 87 | There is also a bug where Grunt is not installed properly. To fix this: 88 | ```` 89 | cd moocRP_base/moocRP/node_modules/sails 90 | sudo npm install grunt-cli 91 | ```` 92 | 93 | Configuration 94 | ================ 95 | See the [configuration documentation](configuration.md) to configure moocRP before launch. Most importantly, note that `local.js` must be created before launching. 96 | 97 | 98 | Launching moocRP 99 | ================ 100 | To launch the application, first launch the Redis server: 101 | ```` 102 | redis-server& 103 | ```` 104 | 105 | Then, launch moocRP in a new command window: 106 | ```` 107 | cd moocRP_base/moocRP 108 | sails lift 109 | ```` 110 | 111 | Note that if you configure moocRP to use SSL, you will need to run moocRP as admin: 112 | ```` 113 | sudo sails lift 114 | ```` 115 | --------------------------------------------------------------------------------