├── AutoConfig.bat
├── AutoConfig.ps1
├── CanvasDataAuth_Schema.cdconfig
├── CanvasDataAuth_Tables.cdconfig
├── CanvasDataDL_Schema_Latest.cdconfig
├── CanvasDataDL_Tables_Latest.cdconfig
├── CanvasDataLevel1_042417_1330.sql
├── CanvasDataStore_042417_1332.sql
├── DownloadLatestSchemaAndTables.bat
├── README.md
├── SampleConfigurationInputs.txt
└── mods.bat


/AutoConfig.bat:
--------------------------------------------------------------------------------
 1 | @echo off
 2 | set /p configdir= Node.js Path: 
 3 | set /p APIS= Canvas Data API Secret: 
 4 | set /p APIK= Canvas Data API Key: 
 5 | set /p dldir= Download Path:
 6 | set /p cdsvr= Server Name:
 7 | set /p cdusr= Server User:
 8 | set /p cdpw= Server USR PW:
 9 | powershell -ExecutionPolicy ByPass -File "%configdir%"/AutoConfig.ps1 "%configdir%" "%APIS%" "%APIK%" "%dldir%" "%cdsvr%" "%cdusr%" "%cdpw%"
10 | pause
11 | cd "%configdir%" 
12 | ren *.cdconfig *.js


--------------------------------------------------------------------------------
/AutoConfig.ps1:
--------------------------------------------------------------------------------
 1 | $configdir=$args[0]
 2 | $APIS=$args[1]
 3 | $APIK=$args[2]
 4 | $dldir=$args[3]
 5 | $cdsvr=$args[4]
 6 | $cdusr=$args[5]
 7 | $cdpw=$args[6]
 8 | 
 9 | $configfiles = Get-ChildItem $configdir\*.cdconfig -rec
10 | foreach ($file in $configfiles)
11 | {
12 |   (get-content $file) |
13 |   foreach-object {$_ -replace "IRSECRETREPLACEME", $APIS} |
14 |   foreach-object {$_ -replace "IRKEYREPLACEME", $APIK} | 
15 |   foreach-object {$_ -replace "IRCDDLDIRREPLACEME", $dldir} | 
16 |   foreach-object {$_ -replace "IRCDSVRREPLACEME", $cdsvr} | 
17 |   foreach-object {$_ -replace "IRCDSVRUSRREPLACEME", $cdusr} | 
18 |   foreach-object {$_ -replace "IRCDSVRUSRPWREPLACEME", $cdpw} | 
19 |   set-content $file
20 | }


--------------------------------------------------------------------------------
/CanvasDataAuth_Schema.cdconfig:
--------------------------------------------------------------------------------
 1 | //Adding necessary node modules
 2 | var querystring = require('querystring');
 3 | var https = require('https');
 4 | var dateFormat = require('dateformat');
 5 | var crypto = require('crypto');  
 6 | var url = require('url');  
 7 | 
 8 | //Declaring and assembling pieces for HMAC auth
 9 | var date = Date();
10 | var timestamp = dateFormat(date, "ddd, d mmm yyyy H:MM:ss Z");
11 | 
12 | //Insert your CanvasAPI API Secret
13 | var secret = 'IRSECRETREPLACEME';  
14 | 
15 | //Insert your CanvasAPI API Key 
16 | var key = 'IRKEYREPLACEME';  
17 | 
18 | var opts = {  
19 |   method: 'GET',  
20 |   host: 'portal.inshosteddata.com',  
21 |   path: 'https://portal.inshosteddata.com/api/schema/latest'  
22 | };  
23 | var HMAC_ALG = 'sha256';  
24 | 
25 | //Assembling Canvas Data Auth credentials
26 | var apiAuth = module.exports = {  
27 |   buildMessage: function(secret, timestamp, reqOpts) {  
28 |     var urlInfo = url.parse(reqOpts.path, true)  
29 |     var sortedParams = Object.keys(urlInfo.query).sort(function(a, b) {  
30 |       return a.localeCompare(b)  
31 |     })  
32 |     var sortedParts = []  
33 |     for (var i = 0; i < sortedParams.length; i++) {  
34 |       var paramName = sortedParams[i]  
35 |       sortedParts.push(paramName + '=' + urlInfo.query[paramName])  
36 |     }  
37 |     var parts = [  
38 |       reqOpts.method.toUpperCase(),  
39 |       reqOpts.host || '',  
40 |       reqOpts.contentType || '',  
41 |       reqOpts.contentMD5 || '',  
42 |       urlInfo.pathname,  
43 |       sortedParts.join('&') || '',  
44 |       timestamp,  
45 |       secret  
46 |     ]  
47 |     return parts.join('\n')  
48 |   },  
49 |   buildHmacSig: function(secret, timestamp, reqOpts) {  
50 |     var message = apiAuth.buildMessage(secret, timestamp, reqOpts)  
51 |     var hmac = crypto.createHmac(HMAC_ALG, new Buffer(secret))  
52 |     hmac.update(message)  
53 |     return hmac.digest('base64')  
54 |   }  
55 | }  
56 | 
57 | var HMACSig = apiAuth.buildHmacSig(secret, timestamp, opts);
58 | var Auth = 'HMACAuth ' + key;
59 | 
60 | 
61 | //Preparing HTTP headers for Canvas Data get request
62 | var GETheaders = {
63 |     'Authorization' : Auth+':'+HMACSig,
64 |     'Date' : timestamp
65 | }; 
66 | 
67 | var optionsget = {
68 |     headers: GETheaders,
69 |     host : 'portal.inshosteddata.com', // here only the domain name
70 |     // (no http/https !)
71 |     port : 443,
72 |     // this is the endpoint for the latest data dump
73 |     path : 'https://portal.inshosteddata.com/api/schema/latest',  //the rest of the url with parameters if needed
74 |     method : 'GET' // do GET
75 | };
76 | 
77 | 
78 | //This function (with the necessary callback) is where we return the object containing our URL(s) for table(s)
79 | var CanvasDataAPICall = function(input, cb) {
80 | 
81 |   getCall = function(response) {
82 |     var data = [];
83 |     response.setEncoding('utf8');
84 |     response.on('data', function (chunk) {
85 |         data.push(chunk)
86 |       });
87 |     response.on('end', function () {
88 |         var result = JSON.parse(data.join(''))
89 |         cb(result);
90 |       });
91 |   }
92 |   var req = https.request(optionsget, getCall);
93 |   req.end();
94 | 
95 | }  
96 | 
97 | module.exports = CanvasDataAPICall;
98 | 


--------------------------------------------------------------------------------
/CanvasDataAuth_Tables.cdconfig:
--------------------------------------------------------------------------------
 1 | //Adding necessary node modules
 2 | var querystring = require('querystring');
 3 | var https = require('https');
 4 | var dateFormat = require('dateformat');
 5 | var crypto = require('crypto');  
 6 | var url = require('url');  
 7 | 
 8 | //Declaring and assembling pieces for HMAC auth
 9 | var date = Date();
10 | var timestamp = dateFormat(date, "ddd, d mmm yyyy H:MM:ss Z");
11 | 
12 | //Insert your CanvasAPI API Secret
13 | var secret = 'IRSECRETREPLACEME';  
14 | 
15 | //Insert your CanvasAPI API Key 
16 | var key = 'IRKEYREPLACEME';  
17 | 
18 | var opts = {  
19 |   method: 'GET',  
20 |   host: 'portal.inshosteddata.com',  
21 |   path: 'https://portal.inshosteddata.com/api/account/self/file/latest'  
22 | };  
23 | var HMAC_ALG = 'sha256';  
24 | 
25 | //Assembling Canvas Data Auth credentials
26 | var apiAuth = module.exports = {  
27 |   buildMessage: function(secret, timestamp, reqOpts) {  
28 |     var urlInfo = url.parse(reqOpts.path, true)  
29 |     var sortedParams = Object.keys(urlInfo.query).sort(function(a, b) {  
30 |       return a.localeCompare(b)  
31 |     })  
32 |     var sortedParts = []  
33 |     for (var i = 0; i < sortedParams.length; i++) {  
34 |       var paramName = sortedParams[i]  
35 |       sortedParts.push(paramName + '=' + urlInfo.query[paramName])  
36 |     }  
37 |     var parts = [  
38 |       reqOpts.method.toUpperCase(),  
39 |       reqOpts.host || '',  
40 |       reqOpts.contentType || '',  
41 |       reqOpts.contentMD5 || '',  
42 |       urlInfo.pathname,  
43 |       sortedParts.join('&') || '',  
44 |       timestamp,  
45 |       secret  
46 |     ]  
47 |     return parts.join('\n')  
48 |   },  
49 |   buildHmacSig: function(secret, timestamp, reqOpts) {  
50 |     var message = apiAuth.buildMessage(secret, timestamp, reqOpts)  
51 |     var hmac = crypto.createHmac(HMAC_ALG, new Buffer(secret))  
52 |     hmac.update(message)  
53 |     return hmac.digest('base64')  
54 |   }  
55 | }  
56 | 
57 | var HMACSig = apiAuth.buildHmacSig(secret, timestamp, opts);
58 | var Auth = 'HMACAuth ' + key;
59 | 
60 | 
61 | //Preparing HTTP headers for Canvas Data get request
62 | var GETheaders = {
63 |     'Authorization' : Auth+':'+HMACSig,
64 |     'Date' : timestamp
65 | }; 
66 | 
67 | var optionsget = {
68 |     headers: GETheaders,
69 |     host : 'portal.inshosteddata.com', // here only the domain name
70 |     // (no http/https !)
71 |     port : 443,
72 |     // this is the endpoint for the latest data dump
73 |     path : 'https://portal.inshosteddata.com/api/account/self/file/latest',  //the rest of the url with parameters if needed
74 |     method : 'GET' // do GET
75 | };
76 | 
77 | 
78 | //This function (with the necessary callback) is where we return the object containing our URL(s) for table(s)
79 | var CanvasDataAPICall = function(input, cb) {
80 | 
81 |   getCall = function(response) {
82 |     var data = [];
83 |     response.setEncoding('utf8');
84 |     response.on('data', function (chunk) {
85 |         data.push(chunk)
86 |       });
87 |     response.on('end', function () {
88 |         var result = JSON.parse(data.join(''))
89 |         cb(result);
90 |       });
91 |   }
92 |   var req = https.request(optionsget, getCall);
93 |   req.end();
94 | 
95 | }  
96 | 
97 | module.exports = CanvasDataAPICall;
98 | 


--------------------------------------------------------------------------------
/CanvasDataDL_Schema_Latest.cdconfig:
--------------------------------------------------------------------------------
  1 | var fs = require('fs');
  2 | var https = require('https');
  3 | //var json2csv = require('json2csv');
  4 | var CanvasDataAPICall = require('./CanvasDataAuth_Schema.js');
  5 | var sql = require("seriate");
  6 | var datetime = new Date();
  7 | var datetimeSimple = (datetime.getMonth() + 1) + '-' + datetime.getDate() + '-' +  datetime.getFullYear();
  8 | 
  9 | var request = require('request');
 10 | 
 11 |     var schKey=[];
 12 |     var schVal=[];
 13 | 
 14 | var download = function(uri, filename, callback){
 15 |   request.head(uri, function(err, res, body){
 16 | 
 17 |     var r = request(uri).pipe(fs.createWriteStream(filename));
 18 |     r.on('close', callback);
 19 | 
 20 |   });
 21 | };
 22 | 
 23 | JSON.flatten = function(data) {
 24 |           var flattenResult = {};
 25 |           function recurse (cur, prop) {
 26 |               if (Object(cur) !== cur) {
 27 |                   flattenResult[prop] = cur;
 28 |               } else if (Array.isArray(cur)) {
 29 |                    for(var i=0, l=cur.length; i<l; i++)
 30 |                        recurse(cur[i], prop + "[" + i + "]");
 31 |                   if (l == 0)
 32 |                       flattenResult[prop] = [];
 33 |               } else {
 34 |                   var isEmpty = true;
 35 |                   for (var p in cur) {
 36 |                       isEmpty = false;
 37 |                       recurse(cur[p], prop ? prop+"."+p : p);
 38 |                   }
 39 |                   if (isEmpty && prop)
 40 |                       flattenResult[prop] = {};
 41 |               }
 42 |           }
 43 |           recurse(data, "");
 44 |           //console.log(flattenResult)
 45 |           return flattenResult;
 46 | }
 47 | 
 48 | var config = {  
 49 |     "server": "IRCDSVRREPLACEME",
 50 |     "user": "IRCDSVRUSRREPLACEME",
 51 |     "password": "IRCDSVRUSRPWREPLACEME",
 52 |     "database": "CanvasDataStore"
 53 | };
 54 | 
 55 | sql.setDefaultConfig( config );
 56 | 
 57 | CanvasDataAPICall('', function(result) {
 58 |        respAPIObj=result;
 59 |        respAPIObjSchema=respAPIObj.schema
 60 | 
 61 |       flattenResult = JSON.flatten(respAPIObjSchema);
 62 |       //console.log(result);
 63 |       //console.log(flattenResult);
 64 |       
 65 |       for (var k in flattenResult){
 66 |             if (flattenResult.hasOwnProperty(k)) {
 67 |                  //console.log(k);
 68 |                  //console.log(flattenResult[k]);
 69 | 
 70 |                 value0=JSON.stringify(k);
 71 |                 value1=JSON.stringify(flattenResult[k]);
 72 | 
 73 |                 schKey.push(k)
 74 |                 schVal.push(flattenResult[k])
 75 |             }
 76 |       }
 77 | 
 78 |       console.log(schKey)
 79 |       console.log(schVal)
 80 | 
 81 |     sql.execute( {  
 82 |         query: "TRUNCATE TABLE RAWSchemaKey " +
 83 |                 "TRUNCATE TABLE RAWSchemaVal"
 84 |     } ).then( function( results ) {
 85 |         console.log( results );
 86 |     }, function( err ) {
 87 |         console.log( "Something bad happened:", err );
 88 |     } );
 89 | //////////////
 90 |     sql.execute( {  
 91 |         query: "INSERT INTO RawSchemaKey ([schKey]) " +
 92 |         "SELECT value FROM @schKey",
 93 |         params: {
 94 |           schKey: {
 95 |             val: schKey,
 96 |             type: sql.VARCHAR(4096),
 97 |             asTable: true
 98 |           }
 99 |         }
100 |     } ).then( function( results ) {
101 |         console.log( results );
102 |     }, function( err ) {
103 |         console.log( "Something bad happened:", err );
104 |     } );
105 | //////////////
106 |     sql.execute( {  
107 |         query: "INSERT INTO RawSchemaVal ([schVal]) " +
108 |         "SELECT value FROM @schVal",
109 |         params: {
110 |           schVal: {
111 |             val: schVal,
112 |             type: sql.VARCHAR(4096),
113 |             asTable: true
114 |           }
115 |         }
116 |     } ).then( function( results ) {
117 |         console.log( results );
118 |     }, function( err ) {
119 |         console.log( "Something bad happened:", err );
120 |     } );
121 | });
122 | 


--------------------------------------------------------------------------------
/CanvasDataDL_Tables_Latest.cdconfig:
--------------------------------------------------------------------------------
 1 | var fs = require('fs');
 2 | var async = require('async');
 3 | var https = require('https');
 4 | var request = require('request');
 5 | var CanvasDataAPICall = require('./CanvasDataAuth_Tables.js');
 6 | 
 7 | var dir = "IRCDDLDIRREPLACEME";
 8 | 
 9 | var q = async.queue(function(task, cb) {
10 |   var dl = fs.createWriteStream(dir + '/' + task.target)
11 | 
12 |   request
13 |     .get(task.url)
14 |     .on('response', function(response) {
15 |       var conlenmb = response.headers['content-length'] / 1024 / 1024;
16 |       var size = conlenmb | 0;
17 |       console.log(task.target + ', Response: ' + response.statusCode + ', Size in MB: ' + size + ', Type: ' +  response.headers['content-type']);
18 |     })
19 |     .on('error', function(err) {
20 |       console.log(err);
21 |       cb(err);
22 |     })
23 |     .pipe(dl);
24 |       dl.on('finish', function(){
25 |         cb();
26 |       });
27 | }, 20);
28 | 
29 | q.drain = function() {
30 |   console.log('Done.')
31 | };
32 | 
33 | CanvasDataAPICall('', function(result) {
34 |       
35 |       if (!fs.existsSync(dir)){
36 |           fs.mkdirSync(dir);
37 |       }
38 | 
39 |       res=result
40 |         //console.log(res);
41 |       abt=res.artifactsByTable
42 |         //console.log(tables);
43 | 
44 |       for (var tables in abt) {
45 |         if (abt.hasOwnProperty(tables)) {
46 |           var tableObj = abt[tables];
47 |           console.log(tableObj);
48 | 
49 |             tableObj.files.forEach(function(file) {
50 |             var url = file.url;
51 |             var filename = file.filename;
52 |             file.target = filename;
53 | 
54 |             q.push(file, function(err) {
55 |               if (err) {
56 |                 console.log(err);
57 |               }
58 |             });
59 |           });
60 |         }
61 |       }
62 | });      
63 | 


--------------------------------------------------------------------------------
/CanvasDataLevel1_042417_1330.sql:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/EMU-CFE/CanvasDataViewer/687c1ba620958eac42ec8efb3e92e92824b1697a/CanvasDataLevel1_042417_1330.sql


--------------------------------------------------------------------------------
/CanvasDataStore_042417_1332.sql:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/EMU-CFE/CanvasDataViewer/687c1ba620958eac42ec8efb3e92e92824b1697a/CanvasDataStore_042417_1332.sql


--------------------------------------------------------------------------------
/DownloadLatestSchemaAndTables.bat:
--------------------------------------------------------------------------------
1 | cd "C:\Program Files\nodejs" 
2 | node CanvasDataDL_Schema_Latest.js
3 | node CanvasDataDL_Tables_Latest.js


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Welcome to CanvasDataViewer!
  2 | 
  3 | ### A Project at Eastern Michigan University
  4 | 
  5 | ## Project Description
  6 | 
  7 | Our project is used to automatically download Canvas Data files (specifically, the latest/daily set that can be downloaded) into a SQL Server database where you can query and analyze the information. Canvas is a proprietary learning management system widely used in higher education and K12, and Canvas Data is the big data service that allows you to download all user activity data for your own use.
  8 | 
  9 | Canvas Data files are very large--currently a full download includes at least one file for each of 80+ tables with many tables requiring more than one file. For a large campus each daily download can include more than 10GB of files that contain millions of database records. Manually downloading Canvas Data files from the Canvas web interface and loading them into a database can take hours. Our CanvasDataViewer automates this process. When set up with a SQL job agent to run the process at night, you can come to work every morning with fresh Canvas Data waiting for you to analyze.
 10 | 
 11 | ## Who will find this useful?
 12 | 
 13 | We think the most likely use case will be campuses that want to evaluate the Canvas Data product as a source of information for teachers and program managers. When schools implement a long-term Canvas Data program they will most likely want to purchase a license for a hosted data repository like Amazon Redshift that can automatically accept Canvas Data.
 14 | CanvasDataViewer may be suitable for long-term use--we have built the system to download the latest data schema from Canvas and built the database table to suit. But we have seen the Canvas Data service change over time, so we can't be sure that our application won't break at some point and need updates.
 15 | 
 16 | ## What do I need?
 17 | 
 18 | CanvasDataViewer need to be installed on Microsoft SQL Server, preferably SQL Server 2012 or later. You cannot use the free SQL Server Express product because it has a 10GB limit. All other components, like the node.js libraries, can be downloaded for free.
 19 | 
 20 | ## Who needs to set it up and how long will it take?
 21 | 
 22 | The initial setup needs to be performed by a SQL Server administrator--someone who knows their way around SQL Server Management Studio and Windows Server. The basic setup can be done in a few hours, although some users may run into problems that can make it a multi-day project.
 23 | 
 24 | ## Installation
 25 | 
 26 | ### Requirements
 27 | 
 28 | * Windows OS
 29 |   * Windows 7 or newer
 30 |   * – or –
 31 |   * Windows 2012 Server or newer
 32 |   * 4GB+ RAM (2GB will run too slow)
 33 | * Admin privileges on the user account
 34 |   * Many instructions tell you to “Run as Admin”
 35 |   * Usually you right-click on the file and “Run as Admin” is a menu choice
 36 | * 7-zip (http://www.7-zip.org)
 37 |   * Required in the scripts that automatically decompress the Canvas Data files
 38 | 
 39 | ### Step 1 – Install SQL Server
 40 | 
 41 | Require SQL Server 2012 or higher
 42 | 
 43 | ### Step 2 – Enable xpcmdshell
 44 | 
 45 | Execute the following TSQL as a SQL Server admin (copy/paste into a New Query screen and run) 
 46 | ```
 47 | EXEC sp_configure 'show advanced options', 1
 48 | GO
 49 | RECONFIGURE
 50 | GO
 51 | EXEC sp_configure 'xp_cmdshell', 1
 52 | GO
 53 | RECONFIGURE
 54 | GO
 55 | ```
 56 | 
 57 | ### Step 3 – Install Node.js
 58 | 
 59 | Download and install Node.js 
 60 | 1. Link: https://nodejs.org/en/ 
 61 | 2. Select the “LTS” (long time support) version recommended for most users 
 62 | 3. Install with the standard “out of box” options
 63 | 
 64 | ### Step 4 – Download and Unzip EMU CDV
 65 | 
 66 | Download and install EMU’s Canvas Data Viewer (CDV) 
 67 | 1. Link: https://github.com/EMU-CFE/CanvasDataViewer 
 68 | 2. Download CDV.zip 
 69 | 3. Extract all files to your preferred location
 70 | 
 71 | ### Step 5– Gather configuration information
 72 | 
 73 | Use the SampleConfigurationInputs.txt file to collect your configuration information.
 74 | Items needed: 
 75 | 1. Your Node.js install path (example: C:\Program Files\nodejs) 
 76 | 2. Your Canvas Data credentials (from the Canvas Data Portal menu in your Canvas site) 
 77 |    1. Canvas Data API Secret 
 78 |    2. Canvas Data API Key 
 79 | 3. Your SQL Server Information 
 80 |    1. Server Instance Name 
 81 |    2. Server account user name (example: sa) 
 82 |    3. Server account user password
 83 | 
 84 | ### Step 6 – Install the required Node.js modules
 85 | 
 86 | From the CanvasDataViewer folder that you unzipped
 87 | 
 88 | 1. Copy mods.bat file to the Node.js directory (C:\Program Files\nodejs)
 89 | 2. Right-click mods.bat and Run as Administrator (This will download/install the Node.js modules (extensions) that CanvasDataViewer needs)
 90 | 
 91 | ### Step 7 – Install the CanvasDataViewer JavaScript files
 92 | 
 93 | Copy/paste to the Node.js directory (C:\Program Files\nodejs) 
 94 | * AutoConfig.bat 
 95 | * AutoConfig.ps1 
 96 | * CanvasData_Schema_Latest.cdconfig 
 97 | * CanvasData_Tables_Latest.cdconfig 
 98 | * CanvasDataAuth_Schema.cdconfig 
 99 | * CanvasDataAuth_Tables.cdconfig
100 | * DownloadLatestSchemaAndTables.bat
101 | 
102 | Right-click Run AutoConfig.bat and Run as Administrator. Enter your configuration information as gathered from Step 5 (this step will modify the scripts and insert the information you provided)
103 | 
104 | ### Step 8 – Create CanvasDataStore database in SQL Server
105 | 
106 | CanvasDataStore is a SQL Server database that houses Canvas Data and the stored procedures that download and install the data.
107 | 
108 | Open SQL Server Management Studio
109 | * In the ObjectExplorer click on Connect and select your database engine that you installed
110 | * In the database engine, right-click on Databases and select New Database
111 | * In Database Name enter “CanvasDataStore” and click OK (Leave all other settings as default. You must use this name because it is hard-coded into all the scripts.)
112 | * Right-click on Databases and click Refresh to make sure CanvasDataStore has appeared
113 | * In the top menu click on File > Open > File. In the dialog box, find/highlight the CanvasDataStore_mmddyy_hhmm.sql ... file in the CDV scripts. The script should open in a New Query window. Click Execute (F5).
114 | * (N.B. The “Warning! The maximum key length is 900 bytes. …” error message is expected. It doesn’t affect the installation and we’re working on eliminating it.)
115 | * Open the CanvasDataStore database to confirm that tables have installed.
116 | 
117 | ### Step 9 – Run the datafile download and database loading
118 | 
119 | * Open SQL Server Management Studio
120 | * In the ObjectExplorer open the CanvasDataStore database and go to CanvasDataStore/Programmability/Stored Procedures
121 | * Right-click on dbo.CanvasData_General_DownloadSchemaAndTables and click “Execute Stored Procedure…”. This process accesses Canvas API endpoints to download the current Canvas Data table schema. It also downloads all the actual Canvas Data data, which is packaged in comma-delimited text files. This can take up to an hour or more.
122 | * Right-click on dbo.CanvasData_General_TableBuild and click “Execute Stored Procedure…”. This process creates tables to hold the data according to the latest schema. Then it loads all the datafiles into import tables. Finally it transfers all the data to production tables and builds indexes on them to aid searching. This process can take several hours depending on the amount of data in your instance.
123 | 
124 | ### Step 10 - Set up SQL Server Agent Jobs
125 | 
126 | SQL Server Agent is a scheduler system that can run processes automatically.
127 | 
128 | * Open SQL Server Agent
129 | * Right-click on Jobs and select New Job
130 | * Enter the following info:
131 |    * General/Name
132 | * Steps - click on New:
133 |    * Step Name
134 |    * Database - change to CanvasDataStore
135 |    * In Command window - "EXEC dbo.CanvasData_General_DownloadSchemaAndTables"
136 | * Click OK
137 | * Schedules - click on New:
138 |    * Name
139 |    * Frequency - "Daily"
140 |    * Daily frequency - 1:00 AM
141 |    * Click OK
142 |    
143 | Repeat this process to create a job for dbo.CanvasData_General_TableBuild at 2:00 AM
144 | 
145 | 
146 | ### Step 11 – Create CanvasDataLevel1 database in SQL Server
147 | 
148 | CanvasDataLevel1 is a SQL Server database that contains views to query Canvas Data. These views all draw on data in the CanvasDataStore database. (CanvasDataLevel1 doesn't store any data of its own.)
149 | 
150 | You need to install CanvasDataLevel1 after you've successfully downloaded your Canvas Data into CanvasDataStore. Otherwise the install will fail.
151 | 
152 | Open SQL Server Management Studio
153 | * In the ObjectExplorer click on Connect and select your database engine that you installed
154 | * In the database engine, right-click on Databases and select New Database
155 | * In Database Name enter “CanvasDataLevel1” and click OK (Leave all other settings as default. You must use this name because it is hard-coded into all the scripts.)
156 | * Right-click on Databases and click Refresh to make sure CanvasDataLevel has appeared
157 | * In the top menu click on File > Open > File. In the dialog box, find/highlight the CanvasDataLevel1_mmddyy_hhmm.sql ... file in the CDV scripts. The script should open in a New Query window. Click Execute (F5).
158 | * Open the CanvasDataLevel1 database to confirm that the views have installed.
159 | 
160 | ### Contributing
161 | 
162 | 
163 | ### Credits
164 | 
165 | [Bill Jones](https://www.linkedin.com/in/wirjones525/)
166 | 
167 | [Andrew Anders](https://www.linkedin.com/in/andrew-a-54108)
168 | 
169 | ### Licensing
170 | 


--------------------------------------------------------------------------------
/SampleConfigurationInputs.txt:
--------------------------------------------------------------------------------
 1 | Node.js Path: 
 2 | (The Node.js installer will ask you where you want to install the files.  Accept the default, which is C:\Program Files\nodejs.  You will need to copy a number of files into this folder in a later step.)
 3 | 
 4 | Canvas Data API Secret: 
 5 | (These can be found in your Canvas Data Portal page.  You find this in your main Canvas account left-hand menu.  Copy the credentials string right off the webpage.)
 6 | 
 7 | Canvas Data API Key: 
 8 | (These can be found in your Canvas Data Portal page.  You find this in your main Canvas account left-hand menu.  Copy the credentials string right off the webpage.)
 9 | 
10 | Downloaded Files path: 
11 | (For example: C:/CanvasDataDownload. Be mindful of the forward slash here)
12 | 
13 | Server Name: MyCampus
14 | (This is the name of your SQL Server instance.  In the SQL Server Management Studio ObjectExplorer, right-click on the main server and select Properties.  The server name is the first variable shown.)
15 | 
16 | Server User: Foo
17 | 
18 | Server USR PW: Bar
19 | (During SQL Server installation you will create a password for a default server account, usually called "sa".  You can use this account/password, or you can create a different one to use with CanvasDataViewer in the Security/Logins folder.)
20 | 


--------------------------------------------------------------------------------
/mods.bat:
--------------------------------------------------------------------------------
1 | cd "C:\Program Files\nodejs" 
2 | npm install filesystem && npm install https && npm install async && npm install seriate && npm install request && npm install querystring && npm install dateformat && npm install crypto && npm install url
3 | pause


--------------------------------------------------------------------------------