├── .dockerignore ├── helpers ├── error.js ├── logging.js ├── cookie.js ├── date.js ├── config.js ├── progressbar.js ├── utils.js ├── moodle.js ├── download.js └── webex.js ├── Dockerfile ├── docker.sh ├── config.example.json ├── .eslintrc.json ├── .github ├── ISSUE_TEMPLATE │ └── bug_report.md └── workflows │ └── compile.yml ├── package.json ├── LICENSE ├── .gitignore ├── app.js └── README.md /.dockerignore: -------------------------------------------------------------------------------- 1 | .gitignore -------------------------------------------------------------------------------- /helpers/error.js: -------------------------------------------------------------------------------- 1 | class AuthenticationError extends Error { 2 | constructor(message) { 3 | super(message); 4 | this.name = 'AuthenticationError'; 5 | Error.captureStackTrace(this, AuthenticationError); 6 | } 7 | } 8 | 9 | module.exports = { 10 | AuthenticationError 11 | }; 12 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:16-alpine 2 | 3 | WORKDIR /app 4 | 5 | RUN chown 1000:1000 /app 6 | 7 | USER 1000:1000 8 | 9 | COPY package*.json ./ 10 | 11 | RUN npm ci --production 12 | 13 | COPY . . 14 | 15 | VOLUME /app/config 16 | VOLUME /app/downloads 17 | 18 | ENV CONFIG_PATH=/app/config/config.json 19 | CMD ["node", "app.js"] 20 | -------------------------------------------------------------------------------- /docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | if ! docker image inspect unifi-webex-dl:latest &>/dev/null; then 6 | docker build -t unifi-webex-dl . 7 | fi 8 | 9 | if [ ! -f "./config.json" ]; then 10 | echo "Missing './config.json' file"; 11 | exit 1 12 | fi 13 | 14 | if [ ! -d "./downloads" ]; then 15 | echo "Missing './downloads' folder, creating it"; 16 | mkdir ./downloads 17 | fi 18 | 19 | docker run --rm --init -it \ 20 | -v "$PWD/config.json":/app/config/config.json \ 21 | -v "$PWD/downloads":/app/downloads \ 22 | unifi-webex-dl 23 | -------------------------------------------------------------------------------- /config.example.json: -------------------------------------------------------------------------------- 1 | { 2 | "credentials": { 3 | "username": "00000", 4 | "password": "*****" 5 | }, 6 | "download": { 7 | "base_path": "./downloads", 8 | "progress_bar": true, 9 | "show_existing": true, 10 | "max_concurrent_downloads": 10 11 | }, 12 | "courses": [ 13 | { 14 | "name": "Cookie2020", 15 | "id": 42, 16 | "skip_names": "test\\d" 17 | }, 18 | { 19 | "name": "HackThePlanet", 20 | "id": 1337, 21 | "skip_before_date": "2020-01-01", 22 | "skip_after_date": "2020-12-31" 23 | } 24 | ] 25 | } -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "commonjs": true, 4 | "es2021": true, 5 | "node": true 6 | }, 7 | "extends": "eslint:recommended", 8 | "parserOptions": { 9 | "ecmaVersion": 12 10 | }, 11 | "rules": { 12 | "indent": [ 13 | "error", 14 | 4 15 | ], 16 | "linebreak-style": [ 17 | "error", 18 | "unix" 19 | ], 20 | "quotes": [ 21 | "error", 22 | "single" 23 | ], 24 | "semi": [ 25 | "error", 26 | "always" 27 | ] 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report of an error encountered 4 | title: '' 5 | labels: bug 6 | assignees: beryxz 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear description of what the bug is. 12 | 13 | **To Reproduce** 14 | Provide the following information to help reproduce the behavior: 15 | - Which recordings are affected 16 | - ID of the course on Moodle 17 | - Which OS are you using? Also, are you using docker? 18 | - Does it persists through time? 19 | 20 | **Screenshots** 21 | If possible, add screenshots to help explain your problem. 22 | 23 | **Additional context** 24 | Add any other context about the problem here. 25 | -------------------------------------------------------------------------------- /helpers/logging.js: -------------------------------------------------------------------------------- 1 | const { createLogger, format, transports } = require('winston'); 2 | const { combine, cli, errors, label, timestamp, printf } = format; 3 | 4 | function create(logLabel) { 5 | return createLogger({ 6 | level: process.env['LOG_LEVEL'] || 'info', 7 | format: combine( 8 | cli(), 9 | errors(), 10 | label({ label: logLabel, message: false }), 11 | timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), 12 | printf(info => `[${info.timestamp}] ${info.label}.${info.level}: ${info.message}`) 13 | ), 14 | transports: [ 15 | new transports.Console() 16 | ] 17 | }); 18 | } 19 | 20 | module.exports = create; -------------------------------------------------------------------------------- /.github/workflows/compile.yml: -------------------------------------------------------------------------------- 1 | name: Compile 2 | on: 3 | push: 4 | tags: 5 | - "v*.*.*" 6 | 7 | jobs: 8 | compile: 9 | runs-on: ubuntu-latest 10 | 11 | steps: 12 | - uses: actions/checkout@v2 13 | - uses: actions/setup-node@v2 14 | with: 15 | node-version: '16' 16 | cache: 'npm' 17 | - name: Install dependencies 18 | run: | 19 | npm ci 20 | npm install -g pkg 21 | 22 | - name: Compile executables 23 | run: npm run compile 24 | 25 | - name: Release 26 | uses: softprops/action-gh-release@v1 27 | with: 28 | draft: true 29 | files: dist/unifi-webex-dl-* 30 | 31 | -------------------------------------------------------------------------------- /helpers/cookie.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Formats an array of cookies as a single string to use in http headers 3 | * @param {Array} cookieJar List of cookies to format 4 | */ 5 | function getCookies(cookieJar) { 6 | return cookieJar 7 | .map(c => c.match(/^(.+?)(?:;[\s]?|$)/)[1]) // Get only the cookie 8 | .join('; '); 9 | } 10 | 11 | /** 12 | * Retrieves the MoodleSession cookie from the array if exists. Throw an error otherwise 13 | * @param {Array} cookies Cookies jar 14 | * @returns {string} The MoodleSession cookie if found 15 | * @throws {Error} If the MoodleSession cookie couldn't be found 16 | */ 17 | function checkMoodleCookie(cookies) { 18 | for (const c of cookies) { 19 | if (c.startsWith('MoodleSession=')) 20 | return c; 21 | } 22 | 23 | throw new Error('Invalid cookies'); 24 | } 25 | 26 | module.exports = { 27 | checkMoodleCookie, getCookies 28 | }; 29 | -------------------------------------------------------------------------------- /helpers/date.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Converts the given date to UTC format: YYYYMMDD 3 | * @param {string} date The date string 4 | * @param {string} separator The separator to place in between year,month,day 5 | * @returns {string} 6 | */ 7 | function getUTCDateTimestamp(date, separator = '') { 8 | const d = new Date(date); 9 | return [ 10 | pad0(''+d.getUTCFullYear(), 4), 11 | pad0(''+(d.getUTCMonth()+1), 2), 12 | pad0(''+d.getUTCDate(), 2) 13 | ].join(separator); 14 | } 15 | 16 | /** 17 | * Prepend the given text with 0s up to length 18 | * @param {string} text the text to pad with 0 19 | * @param {number} length the length of the resulting string 20 | * @returns {string} 21 | */ 22 | function pad0(text, length = 0) { 23 | const pad = text.length < length ? '0'.repeat(length - text.length) : ''; 24 | return pad + text; 25 | } 26 | 27 | module.exports = { 28 | getUTCDateTimestamp 29 | }; 30 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "unifi-webex-dl", 3 | "version": "9.0.0", 4 | "description": "Download recorded lessons from unifi webex platform passing by the Moodle platform.", 5 | "main": "app.js", 6 | "bin": "app.js", 7 | "scripts": { 8 | "start": "node app.js", 9 | "compile": "pkg --no-build ." 10 | }, 11 | "repository": { 12 | "type": "git", 13 | "url": "git+https://github.com/beryxz/unifi-webex-dl.git" 14 | }, 15 | "author": "beryxz", 16 | "license": "MIT", 17 | "bugs": { 18 | "url": "https://github.com/beryxz/unifi-webex-dl/issues" 19 | }, 20 | "homepage": "https://github.com/beryxz/unifi-webex-dl#readme", 21 | "dependencies": { 22 | "axios": "^0.21.2", 23 | "bytes": "^3.1.0", 24 | "cheerio": "^1.0.0-rc.5", 25 | "progress": "^2.0.3", 26 | "qs": "^6.10.3", 27 | "winston": "^3.3.3", 28 | "yaml": "2.0.0-3" 29 | }, 30 | "devDependencies": { 31 | "eslint": "^7.19.0" 32 | }, 33 | "pkg": { 34 | "targets": [ 35 | "node16-linux-x64", 36 | "node16-win-x64", 37 | "node16-macos-x64" 38 | ], 39 | "outputPath": "dist" 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Lorenzo 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | config.json 3 | config.yaml 4 | downloads 5 | .tmp 6 | test.js 7 | tmp 8 | 9 | # Created by https://www.toptal.com/developers/gitignore/api/vscode,node 10 | # Edit at https://www.toptal.com/developers/gitignore?templates=vscode,node 11 | 12 | ### Node ### 13 | # Logs 14 | logs 15 | *.log 16 | npm-debug.log* 17 | yarn-debug.log* 18 | yarn-error.log* 19 | lerna-debug.log* 20 | 21 | # Diagnostic reports (https://nodejs.org/api/report.html) 22 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 23 | 24 | # Runtime data 25 | pids 26 | *.pid 27 | *.seed 28 | *.pid.lock 29 | 30 | # Directory for instrumented libs generated by jscoverage/JSCover 31 | lib-cov 32 | 33 | # Coverage directory used by tools like istanbul 34 | coverage 35 | *.lcov 36 | 37 | # nyc test coverage 38 | .nyc_output 39 | 40 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 41 | .grunt 42 | 43 | # Bower dependency directory (https://bower.io/) 44 | bower_components 45 | 46 | # node-waf configuration 47 | .lock-wscript 48 | 49 | # Compiled binary addons (https://nodejs.org/api/addons.html) 50 | build/Release 51 | 52 | # Dependency directories 53 | node_modules/ 54 | jspm_packages/ 55 | 56 | # TypeScript v1 declaration files 57 | typings/ 58 | 59 | # TypeScript cache 60 | *.tsbuildinfo 61 | 62 | # Optional npm cache directory 63 | .npm 64 | 65 | # Optional eslint cache 66 | .eslintcache 67 | 68 | # Microbundle cache 69 | .rpt2_cache/ 70 | .rts2_cache_cjs/ 71 | .rts2_cache_es/ 72 | .rts2_cache_umd/ 73 | 74 | # Optional REPL history 75 | .node_repl_history 76 | 77 | # Output of 'npm pack' 78 | *.tgz 79 | 80 | # Yarn Integrity file 81 | .yarn-integrity 82 | 83 | # dotenv environment variables file 84 | .env 85 | .env.test 86 | .env*.local 87 | 88 | # parcel-bundler cache (https://parceljs.org/) 89 | .cache 90 | .parcel-cache 91 | 92 | # Next.js build output 93 | .next 94 | 95 | # Nuxt.js build / generate output 96 | .nuxt 97 | dist 98 | 99 | # Gatsby files 100 | .cache/ 101 | # Comment in the public line in if your project uses Gatsby and not Next.js 102 | # https://nextjs.org/blog/next-9-1#public-directory-support 103 | # public 104 | 105 | # vuepress build output 106 | .vuepress/dist 107 | 108 | # Serverless directories 109 | .serverless/ 110 | 111 | # FuseBox cache 112 | .fusebox/ 113 | 114 | # DynamoDB Local files 115 | .dynamodb/ 116 | 117 | # TernJS port file 118 | .tern-port 119 | 120 | # Stores VSCode versions used for testing VSCode extensions 121 | .vscode-test 122 | 123 | ### vscode ### 124 | .vscode/* 125 | !.vscode/settings.json 126 | !.vscode/tasks.json 127 | !.vscode/launch.json 128 | !.vscode/extensions.json 129 | *.code-workspace 130 | 131 | # End of https://www.toptal.com/developers/gitignore/api/vscode,node 132 | -------------------------------------------------------------------------------- /helpers/config.js: -------------------------------------------------------------------------------- 1 | const { existsSync, readFileSync } = require('fs'); 2 | const { isNone, isFilenameValidOnWindows } = require('./utils'); 3 | const logger = require('./logging')('config'); 4 | const yaml = require('yaml'); 5 | const { execSync } = require('child_process'); 6 | 7 | /** 8 | * @typedef Course 9 | * @type {object} 10 | * @property {string} id 11 | * @property {string} name 12 | * @property {string} custom_webex_id 13 | * @property {string} skip_names 14 | * @property {string} skip_before_date 15 | * @property {string} skip_after_date 16 | * @property {boolean} prepend_date 17 | */ 18 | 19 | /** 20 | * @typedef ConfigDownload 21 | * @type {object} 22 | * @property {string} base_path 23 | * @property {boolean} progress_bar 24 | * @property {boolean} show_existing 25 | * @property {number} max_concurrent_downloads 26 | * @property {boolean} fix_streams_with_ffmpeg 27 | */ 28 | 29 | /** 30 | * @typedef ConfigCredentials 31 | * @type {object} 32 | * @property {string} username 33 | * @property {string} password 34 | */ 35 | 36 | /** 37 | * @typedef Config 38 | * @type {object} 39 | * @property {ConfigCredentials} credentials 40 | * @property {ConfigDownload} download 41 | * @property {Course[]} courses 42 | */ 43 | 44 | /** 45 | * Assert that every config isn't undefined or null 46 | * @param {object} configs Object of configs 47 | * @throws {Error} If some config is invalid. Check the error object for the precise reason. 48 | */ 49 | function checkConfigs(configs) { 50 | for (const config in configs) { 51 | if (isNone(configs[config])) 52 | throw new Error(`Missing config: ${config}`); 53 | } 54 | } 55 | 56 | /** 57 | * Check that each Course object in the array is valid 58 | * @param {Course[]} courses Array of course objects 59 | * @throws {Error} If the config file is invalid. Check the error object for the precise reason. 60 | */ 61 | function checkCourses(courses) { 62 | for (const c of courses) { 63 | if (isNone(c.id)) 64 | throw new Error('Invalid config file. A course is missing the \'id\''); 65 | if (typeof(c.id) !== 'number') 66 | throw new Error('Invalid config file. A course\'s \'id\' is not a number'); 67 | if (isNone(c.name)) 68 | throw new Error(`Invalid config file. The [${c.id}] course is missing the 'name'`); 69 | if (typeof(c.name) !== 'string') 70 | throw new Error(`Invalid config file. The [${c.id}] course's 'name' is not a string`); 71 | 72 | if (!isFilenameValidOnWindows(c.name)) 73 | logger.warn(`The [${c.id}] course has a 'name' which contains Windows reserved chars. On Windows this won't work!`); 74 | } 75 | } 76 | 77 | /** 78 | * Check for the ffmpeg binary in the system path 79 | */ 80 | function checkFFmpeg() { 81 | try { 82 | execSync('ffmpeg -version -hide_banner ', {windowsHide: true, stdio: 'ignore'}); 83 | } catch { 84 | throw new Error('ffmpeg binary not present in PATH'); 85 | } 86 | } 87 | 88 | /** 89 | * @param {string} configPath 90 | */ 91 | function parseJson(configPath) { 92 | return JSON.parse(readFileSync(configPath, 'utf8')); 93 | } 94 | /** 95 | * @param {string} configPath 96 | */ 97 | function parseYaml(configPath) { 98 | return yaml.parse(readFileSync(configPath, 'utf8')); 99 | } 100 | /** 101 | * @param {string} configPath 102 | */ 103 | function parseConfigFile(configPath) { 104 | if (!existsSync(configPath)) { 105 | logger.warn(`Missing file ${configPath}`); 106 | return {}; 107 | } 108 | 109 | switch (configPath.match(/\.([a-z]+)$/)?.[1]) { 110 | case 'json': 111 | return parseJson(configPath); 112 | case 'yaml': 113 | return parseYaml(configPath); 114 | default: 115 | return {}; 116 | } 117 | } 118 | 119 | /** 120 | * Read configs from file and/or env variables if set. 121 | * @param {string} configPath 122 | * @return {Config} Configs object 123 | */ 124 | async function load(configPath) { 125 | logger.debug(`Loading ${configPath}`); 126 | 127 | // Try to load file 128 | let config = parseConfigFile(configPath); 129 | 130 | // Read env variables and if not exists, assign config file values 131 | logger.debug('Reading env variables'); 132 | let username = process.env['CREDENTIALS__USERNAME'] || config.credentials?.username, 133 | password = process.env['CREDENTIALS__PASSWORD'] || config.credentials?.password, 134 | base_path = (process.env['DOWNLOAD__BASE_PATH']) || config.download?.base_path, 135 | progress_bar = ((process.env['DOWNLOAD__PROGRESS_BAR']) || config.download?.progress_bar) ?? true, 136 | show_existing = ((process.env['DOWNLOAD__SHOW_EXISTING']) || config.download?.show_existing) ?? true, 137 | max_concurrent_downloads = ((process.env['DOWNLOAD__MAX_CONCURRENT_DOWNLOADS']) || config.download?.max_concurrent_downloads) ?? 3, 138 | fix_streams_with_ffmpeg = ((process.env['DOWNLOAD__FIX_STREAMS_WITH_FFMPEG']) || config.download?.fix_streams_with_ffmpeg) ?? false, 139 | courses; 140 | 141 | // Work on course objects 142 | if (process.env['COURSES']) { 143 | // 123000=Course1,234000=Course2 ... 144 | courses = process.env['COURSES'] 145 | .split(',') 146 | .map(c => c.split('=')) 147 | .map(c => { return { id: c[0], name: c[1] }; }); 148 | } else { 149 | courses = config.courses; 150 | } 151 | 152 | // check for all required configs 153 | checkConfigs({username, password, base_path}); 154 | checkCourses(courses); 155 | if (fix_streams_with_ffmpeg) 156 | checkFFmpeg(); 157 | 158 | return { 159 | credentials: { 160 | username, 161 | password 162 | }, 163 | download: { 164 | base_path, 165 | progress_bar: !!progress_bar, 166 | show_existing: !!show_existing, 167 | max_concurrent_downloads, 168 | fix_streams_with_ffmpeg: !!fix_streams_with_ffmpeg 169 | }, 170 | courses 171 | }; 172 | } 173 | 174 | module.exports = { 175 | load 176 | }; -------------------------------------------------------------------------------- /helpers/progressbar.js: -------------------------------------------------------------------------------- 1 | /* 2 | based on: 3 | - https://gist.github.com/nuxlli/b425344b92ac1ff99c74 4 | - https://github.com/pitaj/multi-progress 5 | */ 6 | 7 | const ProgressBar = require('progress'); 8 | 9 | class MultiProgressBar { 10 | /** 11 | * @param {boolean} clearOnTerminate Clear all the bars when the last one terminates 12 | */ 13 | constructor(clearOnTerminate = false) { 14 | this.stream = process.stderr; 15 | this.cursor = 0; 16 | this.bars = []; 17 | this.clearOnTerminate = clearOnTerminate; 18 | return this; 19 | } 20 | 21 | newBar(schema, options) { 22 | options.stream = this.stream; 23 | var bar = new ProgressBar(schema, options); 24 | this.bars.push(bar); 25 | var index = this.bars.length - 1; 26 | 27 | // allocate line 28 | this.move(index); 29 | this.stream.write('\n'); 30 | this.cursor += 1; 31 | 32 | // replace original 33 | bar.otick = bar.tick; 34 | bar.oterminate = bar.terminate; 35 | bar.oupdate = bar.update; 36 | bar.index = index; 37 | bar.tick = (value, options) => { 38 | this.tick(bar.index, value, options); 39 | }; 40 | bar.terminate = () => { 41 | if (this.clearOnTerminate) { 42 | if (this.bars.every(v => v.complete)) { 43 | this.terminate(); 44 | } 45 | } 46 | }; 47 | bar.update = (value, options) => { 48 | this.update(bar.index, value, options); 49 | }; 50 | 51 | return bar; 52 | } 53 | 54 | terminateSingleBar(index) { 55 | delete this.bars[index]; 56 | this.bars.splice(index, 1); 57 | for (let i = 0; i < this.bars.length; i++) { 58 | if (i < index) continue; 59 | this.bars[i].index -= 1; 60 | } 61 | } 62 | 63 | terminate() { 64 | for (let i = 0; i < this.bars.length; i++) { 65 | this.move(i); 66 | this.stream.clearLine(0); 67 | this.stream.cursorTo(0); 68 | } 69 | this.move(0); 70 | } 71 | 72 | move(index) { 73 | this.stream.moveCursor(0, index - this.cursor); 74 | this.cursor = index; 75 | } 76 | 77 | tick(index, value, options) { 78 | const bar = this.bars[index]; 79 | if (bar) { 80 | this.move(index); 81 | bar.otick(value, options); 82 | this.moveCursorToStart(); 83 | } 84 | } 85 | 86 | update(index, value, options) { 87 | const bar = this.bars[index]; 88 | if (bar) { 89 | this.move(index); 90 | bar.oupdate(value, options); 91 | this.moveCursorToStart(); 92 | } 93 | } 94 | 95 | moveCursorToStart() { 96 | this.stream.cursorTo(0); 97 | this.move(0); 98 | } 99 | } 100 | 101 | /** 102 | * Returns a value to be used, it could be based on the additional data retrieved 103 | * @callback GetterCallback 104 | * @param {object} data additional data 105 | * @returns {any} 106 | */ 107 | 108 | /** 109 | * Progress bar for tracking status of an EventEmitter, inside a MultiProgressBar group 110 | */ 111 | class StatusProgressBar { 112 | /** 113 | * @param {MultiProgressBar} multiProgressBar MultiProgressBar instance where to create the new bar 114 | * @param {EventEmitter} emitter Event emitter that emits 'init' for bar creation, and 'data' for updating bar with bar tick. 115 | * @param {GetterCallback} titleGetter Function that returns the value to be used as the title for the bar creation. Uses `data` from the 'init' event. 116 | * @param {GetterCallback} totalGetter Function that returns the value to be used as the total for the bar creation. Uses `data` from the 'init' event. 117 | * @param {GetterCallback} tickAmountGetter Function that returns the value to be used as the tick amount for the bar update. Uses `data` from the 'data' event. 118 | */ 119 | constructor(multiProgressBar, emitter, titleGetter, totalGetter, tickAmountGetter) { 120 | this.bar = null; 121 | this._multiProgressBar = multiProgressBar; 122 | this._emitter = emitter; 123 | this._titleGetter = titleGetter; 124 | this._totalGetter = totalGetter; 125 | this._tickAmountGetter = tickAmountGetter; 126 | 127 | emitter.on('init', (data) => { 128 | this.bar = multiProgressBar.newBar(`${this._titleGetter(data)} > [:bar] :percent :etas`, { 129 | width: 20, 130 | complete: '=', 131 | incomplete: ' ', 132 | renderThrottle: 100, 133 | clear: true, // bar.terminate has been hooked, this should always stay true 134 | total: this._totalGetter(data) 135 | }); 136 | this.bar.tick(0); // show the progress bar instantly 137 | }); 138 | 139 | emitter.on('data', (data) => { 140 | this.bar.tick(this._tickAmountGetter(data)); 141 | }); 142 | 143 | emitter.on('error', () => { 144 | this._multiProgressBar.terminateSingleBar(this.bar.index); 145 | }); 146 | } 147 | } 148 | 149 | class OneShotProgressBar { 150 | constructor(multiProgressBar, title) { 151 | /** @type {ProgressBar} */ 152 | this.bar = null; 153 | this._multiProgressBar = multiProgressBar; 154 | this._title = title; 155 | } 156 | 157 | init() { 158 | if (this.bar) return; 159 | 160 | this.bar = this._multiProgressBar.newBar(`${this._title} > [:bar] :percent :etas`, { 161 | width: 20, 162 | complete: '=', 163 | incomplete: ' ', 164 | renderThrottle: 100, 165 | clear: true, // bar.terminate has been hooked, this should always stay true 166 | total: 100 167 | }); 168 | this.bar.tick(0); // show the progress bar instantly 169 | } 170 | 171 | complete() { 172 | if (!this.bar) return; 173 | 174 | this.bar.update(1); 175 | } 176 | } 177 | 178 | module.exports = { 179 | MultiProgressBar, 180 | StatusProgressBar, 181 | OneShotProgressBar 182 | }; 183 | -------------------------------------------------------------------------------- /helpers/utils.js: -------------------------------------------------------------------------------- 1 | const { access, mkdir, writeFileSync, unlinkSync, readFileSync, renameSync } = require('fs'); 2 | const util = require('util'); 3 | const exec = util.promisify(require('child_process').exec); 4 | 5 | /** 6 | * Split the source array in chunks of fixed length. 7 | * The last chunk is filled with the remaining objects that might not be less than fixed chunk length 8 | * 9 | * source: https://codereview.stackexchange.com/questions/245346/javascript-split-array-into-n-subarrays-size-of-chunks-dont-matter 10 | * @param {any[]} sourceArray 11 | * @param {number} chunkLength max number of items in each chunk, If less than 1, a single chunk with all the items is going to be returned. 12 | * @returns {any[][]} array of chunks 13 | */ 14 | function splitArrayInChunksOfFixedLength(sourceArray, chunkLength) { 15 | if (!sourceArray) throw new Error('Undefined source array to be split'); 16 | if (!chunkLength) throw new Error('Undefined chunk length'); 17 | if (chunkLength < 1) return [sourceArray]; 18 | 19 | const srcLen = sourceArray.length; 20 | const numOfChunks = Math.ceil(srcLen / chunkLength); 21 | 22 | const chunks = Array.from(Array(numOfChunks), () => []); 23 | for (let i = 0; i < srcLen ; i++) { 24 | chunks[Math.floor(i / chunkLength)].push(sourceArray[i]); 25 | } 26 | return chunks; 27 | } 28 | 29 | /** 30 | * Check whether the given object is undefined, null, empty string or empty object 31 | * @param {any} object to entity to check 32 | */ 33 | function isNone(object) { 34 | return typeof object === 'undefined' || object === null || object === '' || object === {}; 35 | } 36 | 37 | /** 38 | * Check if the filename contains any characters what Windows consider reserved, 39 | * and can't therefore be used in files and folders names. 40 | * @param {string} filename 41 | * @returns {boolean} true if it the filename doesn't contain any reserved char. false otherwise 42 | */ 43 | function isFilenameValidOnWindows(filename) { 44 | // eslint-disable-next-line no-control-regex 45 | return !( /[<>:"/\\|?*\x00-\x1F\r\n]/.test(filename) || /^(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])$/.test(filename) ); 46 | } 47 | 48 | /** 49 | * Return the input text string with all special windows char replaced by another value 50 | * @param {string} text The text from which to replace the windows special chars 51 | * @param {string} replaceValue A string containing the text to replace the matches with 52 | */ 53 | function replaceWindowsSpecialChars(text, replaceValue) { 54 | // eslint-disable-next-line no-control-regex 55 | return text.replace(/[<>:"/\\|?*\x00-\x1F\r\n]/g, replaceValue); 56 | } 57 | 58 | /** 59 | * Return the input text string with all whitespace characters replaced by another value 60 | * @param {string} text The text from which to replace the whitespace characters 61 | * @param {string} replaceValue A string containing the text to replace the matches with 62 | */ 63 | function replaceWhitespaceChars(text, replaceValue) { 64 | return text.replace(/\s/g, replaceValue); 65 | } 66 | 67 | /** 68 | * Retries a promise until it's resolved or it fails too many times 69 | * @param {number} maxRetries the max number of retries before throwing an error if the functions keep failing 70 | * @param {number} timeoutOnError Time to wait before trying again to call fn 71 | * @param {*} fn the function that is called each try 72 | * @returns {Promise} 73 | */ 74 | function retryPromise(maxRetries, timeoutOnError, fn) { 75 | return fn().catch(async function (err) { 76 | if (maxRetries <= 0) { 77 | throw err; 78 | } 79 | await sleep(timeoutOnError); 80 | return retryPromise(maxRetries - 1, timeoutOnError, fn); 81 | }); 82 | } 83 | 84 | /** 85 | * Return a promise that resolves after 'timeout' ms 86 | * @param {number} timeout Sleep timeout in ms 87 | * @returns Promise that is resolved after 'timeout' ms 88 | */ 89 | function sleep(timeout) { 90 | return new Promise(resolve => { 91 | setTimeout(() => { 92 | resolve(); 93 | }, timeout); 94 | }); 95 | } 96 | 97 | /** 98 | * Asynchronously make the dir path if it doesn't exists 99 | * @param {string} dirPath The path to the dir 100 | * @returns {Promise} 101 | */ 102 | function mkdirIfNotExists(dirPath) { 103 | return new Promise((resolve, reject) => { 104 | // try to access 105 | access(dirPath, (err) => { 106 | if (err && err.code === 'ENOENT') { 107 | // dir doesn't exist, creating it 108 | mkdir(dirPath, { recursive: true }, (err) => { 109 | if (err) 110 | reject(`Error creating directory. ${err.code}`); 111 | resolve(); 112 | }); 113 | } else { 114 | // dir exists 115 | resolve(); 116 | } 117 | }); 118 | }); 119 | } 120 | 121 | /** 122 | * Move a file. 123 | * First, it tries to rename the file. If it doesn't work, it then tries to copy it to the new destination, deleting the old one. 124 | * @param {string} srcPath source path 125 | * @param {string} dstPath destination path 126 | * @throws Throws an error if it fails all move strategies 127 | */ 128 | function moveFile(srcPath, dstPath) { 129 | try { 130 | renameSync(srcPath, dstPath); 131 | } catch (err) { 132 | if (err.code === 'EXDEV') { 133 | // Cannot move files that are not in the top OverlayFS layer (e.g.: inside volumes) 134 | // Probably inside a Docker container, falling back to copy-and-unlink 135 | const fileContents = readFileSync(srcPath); 136 | writeFileSync(dstPath, fileContents); 137 | unlinkSync(srcPath); 138 | } else { 139 | throw err; 140 | } 141 | } 142 | } 143 | 144 | /** 145 | * Demux and Remux a video file, using ffmpeg, to fix container format and metadata issues. 146 | * Useful for downloaded HLS stream where resulting file is a mess of concatenated segments. 147 | * Note: Also useful for Webex recordings as they have many issues. 148 | * @param {string} inputFilePath path to the file to remux 149 | * @param {string} outputFilePath path where to save the remuxed file 150 | * @returns {Promise} resolved on success, rejected on failure 151 | */ 152 | async function remuxVideoWithFFmpeg(inputFilePath, outputFilePath) { 153 | let sanitizedInput = inputFilePath.replace('"', '_'); 154 | let sanitizedOutput = outputFilePath.replace('"', '_'); 155 | 156 | return exec(`ffmpeg -hide_banner -v warning -y -i "${sanitizedInput}" -c copy "${sanitizedOutput}"`, {windowsHide: true}) 157 | .then(({ stdout, stderr }) => { 158 | // if stdout is not empty, an error or warning occurred. 159 | if (stdout || stderr) { 160 | throw new Error(`FFmpeg failed the remux process: \n\n${stdout}\n\n${stderr}\n`); 161 | } 162 | }) 163 | .catch(err => { throw err; }); 164 | } 165 | 166 | module.exports = { 167 | splitArrayInChunksOfFixedLength, 168 | isNone, 169 | isFilenameValidOnWindows, 170 | sleep, 171 | retryPromise, 172 | replaceWindowsSpecialChars, 173 | replaceWhitespaceChars, 174 | mkdirIfNotExists, 175 | moveFile, 176 | remuxVideoWithFFmpeg 177 | }; 178 | -------------------------------------------------------------------------------- /helpers/moodle.js: -------------------------------------------------------------------------------- 1 | const axios = require('axios').default; 2 | const qs = require('qs'); 3 | const cheerio = require('cheerio'); 4 | const logger = require('./logging')('moodle'); 5 | const { checkMoodleCookie } = require('./cookie'); 6 | const { AuthenticationError } = require('./error'); 7 | 8 | /** 9 | * @typedef WebexLaunchOptions 10 | * @type {object} 11 | * @property {string|null} launchParameters Parameters to set in the post request to launch webex. Should be 'null' if webex course id is also 'null' 12 | * @property {string} webexCourseId Id of the course on Webex 13 | * @property {string} moodleCourseId Id of the course on Moodle 14 | */ 15 | 16 | class Moodle { 17 | /** 18 | * Session token for moodle requests 19 | * @type {string} 20 | */ 21 | get sessionToken() { 22 | return this._sessionToken; 23 | } 24 | set sessionToken(value) { 25 | this._sessionToken = value; 26 | } 27 | 28 | constructor() { 29 | this._sessionToken = null; 30 | } 31 | 32 | /** 33 | * Throws an error if the instance isn't authenticated to Moodle 34 | * @throws instance is not authenticated to moodle 35 | */ 36 | checkAuth() { 37 | if (!this.sessionToken) throw new AuthenticationError('Not authenticated to Moodle'); 38 | } 39 | 40 | /** 41 | * Login into Moodle platform through the "Autenticazione Unica UniFi" portal. 42 | * @param {string} username Moodle username 43 | * @param {string} password Moodle password 44 | * @returns {Promise} 45 | */ 46 | async loginMoodleUnifiedAuth(username, password) { 47 | let res, executionToken, cookie; 48 | 49 | let loginPortalUrl = 'https://identity.unifi.it/cas/login?service=https://e-l.unifi.it/login/index.php?authCASattras=CASattras'; 50 | 51 | // get loginToken 52 | logger.debug('Loading login portal'); 53 | res = await axios.get(loginPortalUrl); 54 | executionToken = res.data.match(/name="execution" value="(.+?)"/)[1]; 55 | 56 | logger.debug('Posting form to login portal'); 57 | res = await axios.post(loginPortalUrl, qs.stringify({ 58 | username: username, 59 | password: password, 60 | execution: executionToken, 61 | _eventId: 'submit', 62 | geolocation: '' 63 | }), { 64 | headers: { 65 | 'Content-Type': 'application/x-www-form-urlencoded' 66 | }, 67 | maxRedirects: 0, 68 | validateStatus: status => status === 302 || status === 303 69 | }); 70 | 71 | logger.debug('Login on Moodle with ticket'); 72 | res = await axios.get(res.headers.location, { 73 | maxRedirects: 0, 74 | validateStatus: status => status === 302 || status === 303 75 | }); 76 | cookie = checkMoodleCookie(res.headers['set-cookie']); 77 | 78 | logger.debug('Getting authorized session token'); 79 | res = await axios.get(res.headers.location, { 80 | headers: { 81 | 'Cookie': cookie 82 | }, 83 | maxRedirects: 0, 84 | validateStatus: status => status === 302 || status === 303 85 | }); 86 | this.sessionToken = checkMoodleCookie(res.headers['set-cookie']); 87 | this.checkAuth(); 88 | } 89 | 90 | /** 91 | * Extract the course name from the moodle course page 92 | * 93 | * Requires the login method to be called first. 94 | * @param {number} courseId Moodle course id 95 | * @returns {Promise} The course name if it was found, null otherwise 96 | * @throws {AuthenticationError} If not alredy authenticated 97 | * @throws {Error} when axios request wasn't successful 98 | */ 99 | async getCourseName(courseId) { 100 | this.checkAuth(); 101 | 102 | const res = await axios.get('https://e-l.unifi.it/course/view.php', { 103 | params: { 104 | id: courseId 105 | }, 106 | headers: { 107 | 'Cookie': this.sessionToken 108 | } 109 | }); 110 | 111 | // Match the course name 112 | return cheerio.load(res.data)('h1').text(); 113 | } 114 | 115 | /** 116 | * Extract the webex id from the moodle course page 117 | * 118 | * Requires the login method to be called first. 119 | * @param {number} courseId Moodle course id 120 | * @returns {Promise} The id if it was found, null otherwise 121 | * @throws {AuthenticationError} If not alredy authenticated 122 | * @throws {Error} when axios request wasn't successful 123 | */ 124 | async getWebexId(courseId) { 125 | this.checkAuth(); 126 | 127 | const res = await axios.get('https://e-l.unifi.it/course/view.php', { 128 | params: { 129 | id: courseId 130 | }, 131 | headers: { 132 | 'Cookie': this.sessionToken 133 | } 134 | }); 135 | 136 | // Match the webex id 137 | // Match the canonical `launch.php` path 138 | let match = res.data.match(/https:\/\/e-l\.unifi\.it\/mod\/lti\/(?:launch)\.php\?id=(\d+)/); 139 | if (!match) { 140 | // Check for unreliable `view.php` paths 141 | let matches = [ ...res.data.matchAll(/https:\/\/e-l\.unifi\.it\/mod\/lti\/(?:view)\.php\?id=(\d+)/g) ]; 142 | if (matches.length === 1) { 143 | // If there's only one match, use that 144 | match = matches[0]; 145 | } else { 146 | // If there are multiple `view.php` entries, try to use the one with the webex logo 147 | match = res.data.match(/https:\/\/e-l\.unifi\.it\/mod\/lti\/(?:view)\.php\?id=(\d+)"> !row.startsWith('#')); 91 | } 92 | 93 | /** 94 | * Download an HLS playlist stream from an m3u8 url to a file 95 | * 96 | * Emit download events through the statusEmitter instance: 97 | * - 'init' event on download start, or merge start. 98 | * - 'data' event on segment downloaded, or segment merged 99 | * - 'error' event on error 100 | * - 'finish' event emitted when recording finished downloading and has been merged successfully. 101 | * 102 | * `init` is called multiple times with the `stage` of the download, either "DOWNLOAD" or "MERGE". 103 | * 104 | * Use the 'finish' event to check when the download has finished. 105 | * 106 | * @param {string} playlistUrl The HLS m3u8 playlist file url. 107 | * @param {string} savePath Path in which to save the downloaded file. 108 | * @param {string} tmpFolderPath Path to a temp folder to be used internally to save intermediary segments. 109 | * @param {EventEmitter} statusEmitter EventEmitter instance where to emit status updates on the download 110 | */ 111 | function downloadHLS(playlistUrl, savePath, tmpFolderPath, statusEmitter) { 112 | _downloadHLSPlaylistSegments(playlistUrl, tmpFolderPath, statusEmitter) 113 | .then(downloadedSegmentsCount => 114 | _mergeHLSPlaylistSegments(tmpFolderPath, savePath, downloadedSegmentsCount, statusEmitter)) 115 | .then(() => 116 | statusEmitter.emit('finish')) 117 | .catch(err => 118 | statusEmitter.emit('error', err)); 119 | } 120 | 121 | /** 122 | * Download each segment of an HLS playlist stream from an m3u8 url to single files named `segNum.ts` 123 | * @param {string} playlistUrl The HLS m3u8 playlist file url 124 | * @param {string} savePath Existing path to folder where to save the stream segments 125 | * @param {EventEmitter} statusEmitter EventEmitter instance where to emit status updates on the download 126 | * @returns {Promise} Number of downloaded segments. 127 | */ 128 | async function _downloadHLSPlaylistSegments(playlistUrl, savePath, statusEmitter) { 129 | //NOTE: When there's only one large segment, the progress status is useless as it only updates on completion. 130 | 131 | // Download the hls stream 132 | const segments = await parseHLSPlaylistSegments(playlistUrl); 133 | if (!Array.isArray(segments) || segments.length === 0) 134 | throw new Error('Playlist is empty'); 135 | const totSegments = segments.length; 136 | 137 | statusEmitter.emit('init', { 138 | stage: 'DOWNLOAD', 139 | segmentsCount: totSegments, 140 | }); 141 | 142 | // download each segment 143 | let segmentNum = 1; 144 | let chunks = splitArrayInChunksOfFixedLength(segments, HLS_CONFIG.MAX_PARALLEL_SEGMENTS); 145 | 146 | for (const chunk of chunks) { 147 | let segments = chunk.map(segmentUrl => { 148 | const TMP_NUM = segmentNum++; 149 | 150 | // download segment 151 | return new Promise((resolve, reject) => { 152 | let dwnlFn = async () => { 153 | const res = await axios.get(url.resolve(playlistUrl, segmentUrl), { 154 | responseType: 'stream' 155 | }); 156 | 157 | let fileStream = createWriteStream(join(savePath, `${TMP_NUM}.ts`)); 158 | 159 | // wait for segment to download 160 | res.data.pipe(fileStream); 161 | res.data.on('end', () => { 162 | statusEmitter.emit('data', { segmentDownloaded: TMP_NUM }); 163 | resolve(); 164 | }); 165 | }; 166 | 167 | retryPromise(HLS_CONFIG.SEGMENT_RETRY_COUNT, HLS_CONFIG.SEGMENT_RETRY_DELAY, dwnlFn) 168 | .catch(err => { 169 | reject(new Error(`Segment ${segmentNum}: ${err.message}`)); 170 | }); 171 | }); 172 | }); 173 | 174 | await Promise.all(segments).catch(err => {throw err;}); 175 | } 176 | 177 | return totSegments; 178 | } 179 | 180 | /** 181 | * Merge the segments downloaded with downloadHLSPlaylist() 182 | * @param {string} segmentsPath Path to the folder containing the downloaded hls segments 183 | * @param {string} resultFilePath Path where to save the merged file 184 | * @param {number} downloadedSegments Number of segments to merge 185 | * @param {EventEmitter} statusEmitter EventEmitter instance where to emit status updates on the download 186 | * @returns {Promise} 187 | */ 188 | async function _mergeHLSPlaylistSegments(segmentsPath, resultFilePath, downloadedSegments, statusEmitter) { 189 | const outputFile = createWriteStream(resultFilePath); 190 | 191 | statusEmitter.emit('init', { 192 | stage: 'MERGE', 193 | segmentsCount: downloadedSegments, 194 | }); 195 | 196 | for (let segmentNum = 1; segmentNum <= downloadedSegments; segmentNum++) { 197 | let segmentPath = join(segmentsPath, `${segmentNum}.ts`); 198 | if (!existsSync(segmentPath)) throw new Error(`Missing segment number ${segmentNum}`); 199 | 200 | const segment = createReadStream(segmentPath); 201 | 202 | segment.pipe(outputFile, { end: false }); 203 | await new Promise((resolve, reject) => { 204 | segment.on('end', () => { 205 | statusEmitter.emit('data', { segmentMerged: segmentNum }); 206 | resolve(); 207 | }); 208 | segment.on('error', (err) => { 209 | reject(err); 210 | }); 211 | }); 212 | 213 | try { 214 | unlinkSync(segmentPath); 215 | } catch (err) { 216 | logger.debug(`Error deleting tmp segment: ${err.message}`); 217 | } 218 | } 219 | } 220 | 221 | module.exports = { 222 | downloadStream, 223 | downloadHLS, 224 | parseHLSPlaylistSegments 225 | }; -------------------------------------------------------------------------------- /helpers/webex.js: -------------------------------------------------------------------------------- 1 | const axios = require('axios').default; 2 | const logger = require('./logging')('webex'); 3 | const cheerio = require('cheerio'); 4 | const qs = require('qs'); 5 | const { getCookies } = require('./cookie'); 6 | 7 | /** 8 | * @typedef Recording 9 | * @type {object} 10 | * @property {number} id 11 | * @property {string} name 12 | * @property {string} created_at 13 | * @property {string} updated_at 14 | * @property {string} recording_url 15 | * @property {string} timezone 16 | * @property {number} duration_hour 17 | * @property {number} duration_min 18 | * @property {number} duration_sec 19 | * @property {string} file_url 20 | * @property {string} format 21 | * @property {string} password 22 | */ 23 | 24 | /** 25 | * @typedef WebexLaunchObject 26 | * @type {object} 27 | * @property {string} webexCourseId Id of the course on webex 28 | * @property {string} launchParameters urlencoded string to send as post body to access webex 29 | * @property {string} cookies Session cookies to call webex endpoints 30 | */ 31 | 32 | /** 33 | * Time in milliseconds before timing out webex requests 34 | * @type {number} 35 | */ 36 | const WEBEX_REQUEST_TIMEOUT = 5000; 37 | 38 | /** 39 | * Launch the webex platform and retrieve the JWT and Cookies 40 | * @param {import('./moodle').WebexLaunchOptions} webexLaunchOptions 41 | * @returns {Promise} 42 | */ 43 | async function launchWebex(webexLaunchOptions) { 44 | logger.debug(`[${webexLaunchOptions.webexCourseId}] Launching Webex`); 45 | 46 | let reqConfig = { 47 | headers: { 48 | 'Content-Type': 'application/x-www-form-urlencoded' 49 | }, 50 | timeout: WEBEX_REQUEST_TIMEOUT 51 | }; 52 | return axios.post('https://lti.educonnector.io/launches', webexLaunchOptions.launchParameters, reqConfig) 53 | .then(res => { 54 | return { 55 | webexCourseId: webexLaunchOptions.webexCourseId, 56 | launchParameters: webexLaunchOptions.launchParameters, 57 | cookies: getCookies(res.headers['set-cookie']) 58 | }; 59 | }) 60 | .catch(err => { throw new Error(`Couldn't launch webex. ${err.message}`); }); 61 | 62 | //NOTE: API changed, shouldn't be necessary anymore. Anyway, "json_web_token" changed to "session_ticket" 63 | // match JWT 64 | // let jwt = res.data.match(/(?:"|")json_web_token(?:"|"):(?:"|")([a-zA-Z0-9.\-_=+/]+?)(?:"|")/); 65 | // if (jwt === null) 66 | // throw new Error('JWT not found'); 67 | // jwt = jwt[1]; 68 | // logger.debug(`├─ jwt: ${jwt}`); 69 | // logger.debug(`└─ cookies: ${cookies}`); 70 | } 71 | 72 | /** 73 | * Get all available recordings for the given webex course 74 | * @param {WebexLaunchObject} webexObject Required to interact with webex endpoints 75 | * @returns {Promise} List of all the available recordings 76 | */ 77 | async function getWebexRecordings(webexObject) { 78 | logger.debug(`[${webexObject.webexCourseId}] Get recordings`); 79 | 80 | let requestConfig = { 81 | headers: { 82 | 'Cookie': webexObject.cookies 83 | }, 84 | timeout: WEBEX_REQUEST_TIMEOUT 85 | }; 86 | return axios.get('https://lti.educonnector.io/api/webex/recordings', requestConfig) 87 | .then(res => res.data) 88 | .catch(err => { 89 | throw new Error(`Error retrieving recordings: ${err.message}`); 90 | }); 91 | } 92 | 93 | /** 94 | * Get the nbrshared response for the events 95 | * @param {AxiosResponse} res The response from fileUrl 96 | * @param {Promise} password The recording password 97 | */ 98 | async function eventRecordingPassword(res, password) { 99 | let url, params; 100 | logger.debug('Event recording'); 101 | 102 | // Serialize params for recordAction.do 103 | params = cheerio.load(res.data)('form') 104 | .serialize() 105 | .replace(/playbackPasswd=/, `playbackPasswd=${password}`) 106 | .replace(/theAction=[a-zA-Z]*/, 'theAction=check_pass') 107 | .replace(/accessType=[a-zA-Z]*/, 'accessType=downloadRecording'); 108 | url = 'https://unifirenze.webex.com/ec3300/eventcenter/recording/recordAction.do'; 109 | 110 | // Check credentials to recordAction.do 111 | logger.debug('Posting to recordAction.do'); 112 | res = await axios.post(url, params, { 113 | headers: { 114 | 'Content-Type': 'application/x-www-form-urlencoded' 115 | } 116 | }); 117 | const cookies = getCookies(res.headers['set-cookie']); 118 | 119 | // parse params for viewrecord.do 120 | let formId = res.data.match(/[?&]formId=(\d+)/)?.[1], 121 | siteurl = res.data.match(/[?&]siteurl=(\w+)/)?.[1], 122 | accessType = res.data.match(/[?&]accessType=(\w+)/)?.[1], 123 | internalPBRecordTicket = res.data.match(/[?&]internalPBRecordTicket=(\w+)/)?.[1], 124 | internalDWRecordTicket = res.data.match(/[?&]internalDWRecordTicket=(\w+)/)?.[1]; 125 | // Check if params were parsed successfully 126 | if (formId === null || siteurl === null || accessType === null || internalPBRecordTicket === null || internalDWRecordTicket === null) 127 | throw new Error('Some required parameters couldn\'t be parsed'); 128 | logger.debug(`├─ formId: ${formId}`); 129 | logger.debug(`├─ siteurl: ${siteurl}`); 130 | logger.debug(`├─ accessType: ${accessType}`); 131 | logger.debug(`├─ iPBRT: ${internalPBRecordTicket}`); 132 | logger.debug(`└─ iDWRT: ${internalDWRecordTicket}`); 133 | 134 | // post to viewrecord.do 135 | logger.debug('Posting to viewrecord.do'); 136 | url = 'https://unifirenze.webex.com/ec3300/eventcenter/enroll/viewrecord.do'; 137 | params = { 138 | firstName: 'Anonymous', 139 | lastName: 'Anonymous', 140 | email: null, 141 | siteurl, 142 | directview: 1, 143 | AT: 'ViewAction', 144 | recordId: formId, 145 | accessType, 146 | internalPBRecordTicket, 147 | internalDWRecordTicket 148 | }; 149 | res = await axios.post(url, qs.stringify(params), { 150 | headers: { 151 | 'Content-Type': 'application/x-www-form-urlencoded', 152 | 'Cookie': cookies 153 | } 154 | }); 155 | 156 | // parse params for nbrshared.do 157 | let recordKey = res.data.match(/(?:&|\\x3f|\\x26)recordKey(?:=|\\x3d)(\w+)/)?.[1], 158 | recordID = res.data.match(/(?:&|\\x3f|\\x26)recordID(?:=|\\x3d)(\d+)/)?.[1], 159 | serviceRecordID = res.data.match(/(?:&|\\x3f|\\x26)serviceRecordID(?:=|\\x3d)(\d+)/)?.[1]; 160 | // Check if params were parsed successfully 161 | if (recordKey === null || recordID === null || serviceRecordID === null) 162 | throw new Error('Some required parameters couldn\'t be parsed'); 163 | logger.debug(`├─ recordKey: ${recordKey}`); 164 | logger.debug(`├─ recordID: ${recordID}`); 165 | logger.debug(`└─ serviceRecordID: ${serviceRecordID}`); 166 | 167 | // post to nbrshared.do 168 | logger.debug('Posting to nbrshared.do'); 169 | url = 'https://unifirenze.webex.com/mw3300/mywebex/nbrshared.do'; 170 | params = { 171 | action: 'publishfile', 172 | siteurl: siteurl, 173 | recordKey, 174 | recordID, 175 | serviceRecordID 176 | }; 177 | res = await axios.post(url, qs.stringify(params), { 178 | headers: { 179 | 'Content-Type': 'application/x-www-form-urlencoded' 180 | } 181 | }); 182 | return res; 183 | } 184 | 185 | /** 186 | * Get the nbrshared response for the meetings 187 | * @param {AxiosResponse} res The response from fileUrl 188 | * @param {string} password The recording password 189 | * @param {string} fileUrl The webex recording file url from which the download procedure started 190 | */ 191 | async function meetingRecordingPassword(res, password, fileUrl) { 192 | let resultResponse; 193 | logger.debug('Meeting recording'); 194 | 195 | // Check if password is required 196 | if (/recordingpasswordcheck\.do/.test(res.data)) { 197 | // Get params for recordingpasswordcheck.do 198 | logger.debug('Getting params for recordingpasswordcheck.do'); 199 | const form = cheerio.load(res.data)('form'); 200 | // format params as urlEncoded 201 | let params = form.serialize().replace(/password=/, `password=${password}`); 202 | if (res.data.includes('document.forms[0].firstEntry.value=false;')) { // Don't know the reason. 203 | params = params.replace('firstEntry=true', 'firstEntry=false'); 204 | } 205 | // add origin to relative actionUrl 206 | let origin = new URL(fileUrl).origin; 207 | let actionUrl = new URL(form.attr('action'), origin); 208 | 209 | // Check recordingpasswordcheck.do 210 | logger.debug('Checking params with recordingpasswordcheck.do'); 211 | resultResponse = await axios.post(actionUrl.toString(), params, { 212 | headers: { 213 | 'Content-Type': 'application/x-www-form-urlencoded' 214 | }, 215 | }); 216 | } else { 217 | logger.debug('No password required'); 218 | 219 | // Refer to README. Sometimes, when no password is required, the response is already the output of `nbrshared`. 220 | if (!res.data.includes('commonGet2PostForm')) 221 | return res; 222 | 223 | resultResponse = res; 224 | } 225 | 226 | // parse params for nbrshared.do in response 227 | let url = new URL(resultResponse.data.match(/href=['"](http.+?nbrshared\.do.+?)['"]/)[1]); 228 | 229 | // post to nbrshared.do 230 | logger.debug('Posting to nbrshared.do'); 231 | resultResponse = await axios.post(url.origin + url.pathname, url.search.substring(1), { 232 | headers: { 233 | 'Content-Type': 'application/x-www-form-urlencoded' 234 | } 235 | }); 236 | return resultResponse; 237 | } 238 | 239 | /** 240 | * Given a file_url of a recording, gets the URL for downloading the recording 241 | * of Meetings or Events that have webex download feature enabled. 242 | * @param {string} fileUrl The webex recording file url from which to start the download procedure 243 | * @param {string} password The webex recording password 244 | * @throws {Error} If an error occurred. 245 | * @return {Promise} The url from which to download the recording 246 | */ 247 | async function getWebexRecordingDownloadUrl(fileUrl, password) { 248 | let res, params; 249 | 250 | res = await axios.get(fileUrl); 251 | if (/(ico-warning|TblContentFont2)/.test(res.data)) 252 | throw new Error('Recording deleted, not available, or not downloadable.'); 253 | 254 | // res is the response from nbrshared 255 | res = (/internalRecordTicket/.test(res.data)) 256 | ? await eventRecordingPassword(res, password) // Event recording 257 | : await meetingRecordingPassword(res, password, fileUrl); // Meeting recording 258 | 259 | // parse nbrPrepare.do params 260 | logger.debug('Parsing nbrPrepare params'); 261 | let recordId = res.data.match(/var recordId\s*?=\s*?(\d+);/)?.[1], 262 | serviceRecordId = res.data.match(/var serviceRecordId\s*?=\s*?(\d+);/)?.[1], 263 | prepareTicket = res.data.match(/var prepareTicket\s*?=\s*?['"]([a-f0-9]+)['"];/)?.[1], 264 | downloadUrl = res.data.match(/var downloadUrl\s*?=\s*?['"](http.+?)['"][\s;]/)?.[1]; 265 | // Check if params were parsed successfully 266 | if (recordId === null || prepareTicket === null || downloadUrl === null) 267 | throw new Error('Some required parameters couldn\'t be parsed'); 268 | logger.debug(`├─ recordId: ${recordId}`); 269 | logger.debug(`├─ serviceRecordId: ${serviceRecordId}`); 270 | logger.debug(`├─ downloadUrl: ${downloadUrl}`); 271 | logger.debug(`└─ ticket: ${prepareTicket}`); 272 | // Prepare params object 273 | params = { recordid: recordId, prepareTicket }; 274 | if (serviceRecordId !== null && serviceRecordId > 0) 275 | params.serviceRecordId = serviceRecordId; 276 | 277 | // Wait for recording to be ready and then download it 278 | let status; 279 | while (status !== 'OKOK') { 280 | // get nbrPrepare.do 281 | logger.debug('Checking recording status nbrPrepare.do'); 282 | res = await axios.get('https://unifirenze.webex.com/mw3300/mywebex/nbrPrepare.do', { 283 | params: { 284 | siteurl: 'unifirenze', 285 | ...params 286 | } 287 | }); 288 | // parse `window.parent.func_prepare(status, url, ticket)' 289 | let groups = res.data.match(/func_prepare\(['"](.*?)['"],['"](.*?)['"],['"](.*?)['"]\);/); 290 | if (groups === null || !['OKOK', 'Preparing'].includes(groups[1])) 291 | throw new Error('Unknown error while waiting for recording to be ready'); 292 | params = { status: groups[1], url: groups[2], ticket: groups[3]}; 293 | logger.debug(`├─ status: ${params.status}`); 294 | logger.debug(`├─ url: ${params.url}`); 295 | logger.debug(`└─ ticket: ${params.ticket}`); 296 | 297 | // 'status' case switch 298 | if (params.status === 'OKOK') { 299 | // Write to file 300 | logger.debug('Recording ready'); 301 | return downloadUrl + params.ticket; 302 | } 303 | 304 | logger.debug('Recording not ready, waiting 1s...'); 305 | await new Promise(r => setTimeout(r, 1000)); 306 | } 307 | } 308 | 309 | /** 310 | * Given a recording_url retrieves the stream options. 311 | * Stream options contains parameters required for download hls playlist and more. 312 | * @param {string} recording_url The recording_url of the recording object 313 | * @param {string} password The password of the recording 314 | * @throws {Error} if some requests fails 315 | * @returns {Promise} the stream options 316 | */ 317 | async function getWebexRecordingStreamOptions(recording_url, password) { 318 | // get recordingId 319 | let res = await axios.get(recording_url); 320 | if (/(You can\\'t access this recording|Impossibile accedere a questa registrazione)/.test(res.data)) 321 | throw new Error('Recording has been deleted or isn\'t available at the moment'); 322 | 323 | const recordingId = res.data.match(/location.href.+?https:\/\/unifirenze\.webex.+?playback\/([a-zA-Z0-9]+)/)?.[1]; 324 | if (recordingId === null) 325 | throw new Error('Couldn\'t match recordingId'); 326 | 327 | // get stream options 328 | res = await axios.get(`https://unifirenze.webex.com/webappng/api/v1/recordings/${recordingId}/stream?siteurl=unifirenze`, { 329 | headers: { 330 | accessPwd: password 331 | } 332 | }); 333 | if (!res.data?.mp4StreamOption) 334 | throw new Error('Invalid response. No stream options'); 335 | 336 | return res.data; 337 | } 338 | 339 | /** 340 | * Given a recording_url of a recording, retrieves url of the hls playlist used for streaming the recording. 341 | * This function uses the `mp4StreamOption` property. 342 | * @param {string} recording_url The recording_url of the recording 343 | * @param {string} password The password of the recording 344 | * @throws {Error} if some requests fails 345 | * @returns {Promise} { playlistUrl, filesize } 346 | */ 347 | async function getWebexRecordingHLSPlaylist(recording_url, password) { 348 | // get mp4StreamOption 349 | logger.debug('Getting stream options'); 350 | const streamOptions = await getWebexRecordingStreamOptions(recording_url, password); 351 | if (!streamOptions?.mp4StreamOption) 352 | throw new Error('Invalid recording stream options'); 353 | const mp4StreamOption = streamOptions.mp4StreamOption; 354 | 355 | // get playlist filename 356 | logger.debug('Getting playlist filename'); 357 | const res = await axios({ 358 | method: 'post', 359 | url: 'https://nfg1vss.webex.com/apis/html5-pipeline.do', 360 | params: { 361 | recordingDir: mp4StreamOption.recordingDir, 362 | timestamp: mp4StreamOption.timestamp, 363 | token: mp4StreamOption.token, 364 | xmlName: mp4StreamOption.xmlName 365 | } 366 | }); 367 | const playlistFilename = res.data.match(/(.+?)<\/Sequence>/)?.[1]; 368 | if (playlistFilename === null) 369 | throw new Error('Recording file not found'); 370 | const playlistUrl = `https://nfg1vss.webex.com/hls-vod/recordingDir/${mp4StreamOption.recordingDir}/timestamp/${mp4StreamOption.timestamp}/token/${mp4StreamOption.token}/fileName/${playlistFilename}.m3u8`; 371 | 372 | const filesize = (streamOptions.fileSize ?? 0) + (streamOptions.mediaDetectInfo?.audioSize ?? 0); 373 | logger.debug(`└─ playlistUrl: ${playlistUrl}`); 374 | logger.debug(`└─ filesize: ${filesize}`); 375 | return { playlistUrl, filesize }; 376 | } 377 | 378 | module.exports = { 379 | launchWebex, 380 | getWebexRecordings, 381 | getWebexRecordingDownloadUrl, 382 | getWebexRecordingHLSPlaylist, 383 | getWebexRecordingStreamOptions 384 | }; 385 | -------------------------------------------------------------------------------- /app.js: -------------------------------------------------------------------------------- 1 | const config = require('./helpers/config'); 2 | const { createHash } = require('crypto'); 3 | const Moodle = require('./helpers/moodle'); 4 | const bytes = require('bytes'); 5 | const { launchWebex, getWebexRecordings, getWebexRecordingDownloadUrl, getWebexRecordingHLSPlaylist, getWebexRecordingStreamOptions } = require('./helpers/webex'); 6 | const logger = require('./helpers/logging')('app'); 7 | const { join } = require('path'); 8 | const { existsSync, readdirSync, unlinkSync, rmSync } = require('fs'); 9 | const { downloadHLS, downloadStream, parseHLSPlaylistSegments } = require('./helpers/download'); 10 | const { getUTCDateTimestamp } = require('./helpers/date'); 11 | const { MultiProgressBar, StatusProgressBar, OneShotProgressBar } = require('./helpers/progressbar'); 12 | const { splitArrayInChunksOfFixedLength, retryPromise, sleep, replaceWindowsSpecialChars, replaceWhitespaceChars, mkdirIfNotExists, moveFile, remuxVideoWithFFmpeg } = require('./helpers/utils'); 13 | const { default: axios } = require('axios'); 14 | const { EventEmitter } = require('stream'); 15 | 16 | /** 17 | * @typedef FetchedCourse 18 | * @type {object} 19 | * @property {boolean} success if the course was fetched succesfully. If false, check the `err` property for additional details. 20 | * @property {FetchedRecordings} [recordings] List of recordings of the course 21 | * @property {config.Course} [course] 22 | * @property {any} [err] 23 | */ 24 | 25 | /** 26 | * @typedef FetchedRecordings 27 | * @type {object} 28 | * @property {import('./helpers/webex').Recording[]} recordings 29 | * @property {number} totalCount 30 | * @property {number} filteredCount 31 | */ 32 | 33 | /** 34 | * @typedef LogsConfig 35 | * @type {object} 36 | * @property {MultiProgressBar} [multiProgressBar=null] MultiProgressBar instance to render download status 37 | * @property {string} logStatusName Name of the download to show in the download status 38 | */ 39 | 40 | /** 41 | * @type {bytes.BytesOptions} 42 | */ 43 | const BYTES_OPTIONS = { 44 | decimalPlaces: 2, 45 | fixedDecimals: true, 46 | thousandsSeparator: '', 47 | unit: 'MB', 48 | unitSeparator: '', 49 | }; 50 | 51 | /** 52 | * Helper to load the proper config file. 53 | * First it tries to load config.json, if it doesn't exist, config.yaml is tried next 54 | * @return {Promise} configs 55 | */ 56 | async function loadConfig() { 57 | let configPath = process.env['CONFIG_PATH']; 58 | if (!configPath) { 59 | if (existsSync('./config.json')) { 60 | logger.info('Loading config.json'); 61 | configPath = './config.json'; 62 | } else if (existsSync('./config.yaml')) { 63 | logger.info('Loading config.yaml'); 64 | configPath = './config.yaml'; 65 | } else { 66 | throw new Error('Config file not found. Are you in the same directory as the script?'); 67 | } 68 | } else { 69 | logger.info(`Loading ${configPath}`); 70 | } 71 | 72 | return config.load(configPath); 73 | } 74 | 75 | /** 76 | * Setup common configs for axios static instance 77 | */ 78 | function setupAxios() { 79 | Object.assign(axios.defaults, { 80 | headers: { 81 | 'User-Agent': 'Mozilla/5.0' 82 | } 83 | }); 84 | } 85 | 86 | /** 87 | * Helper to create the temp folder removing all temp files of previous executions that were abruptly interrupted 88 | * @returns {Promise} 89 | * @throws {Error} If temp directory couldn't be created 90 | */ 91 | async function createTempFolder() { 92 | try { 93 | await mkdirIfNotExists('./tmp'); 94 | readdirSync('./tmp').forEach(tmpfile => { 95 | rmSync(join('./tmp/', tmpfile), { recursive: true, force: true }); 96 | }); 97 | } catch (err) { 98 | throw new Error(`Error while creating tmp folder: ${err.message}`); 99 | } 100 | } 101 | 102 | /** 103 | * Helper to login to Moodle 104 | * @param {Moodle} moodle 105 | * @param {config.Config} configs 106 | * @returns {Promise} 107 | */ 108 | async function loginToMoodle(moodle, configs) { 109 | logger.info('Logging into Moodle'); 110 | return moodle.loginMoodleUnifiedAuth(configs.credentials.username, configs.credentials.password); 111 | } 112 | 113 | /** 114 | * Get all recordings, applying filters specified in the course's config 115 | * @param {config.Course} course 116 | * @param {Moodle} moodle 117 | * @return {Promise} 118 | */ 119 | async function getRecordings(course, moodle) { 120 | const recordingsAll = await moodle.getWebexLaunchOptions(course.id, course?.custom_webex_id) 121 | .then(webexLaunch => launchWebex(webexLaunch)) 122 | .then(webexObject => getWebexRecordings(webexObject)) 123 | .catch(err => { throw err; }); 124 | 125 | const recordingsFiltered = recordingsAll.filter(rec => { 126 | try { 127 | let createdAt = new Date(rec.created_at).getTime(); 128 | return !( 129 | (course.skip_before_date && new Date(course.skip_before_date) > createdAt) || 130 | (course.skip_after_date && new Date(course.skip_after_date) < createdAt) || 131 | (course.skip_names && RegExp(course.skip_names).test(rec.name)) 132 | ); 133 | } catch (err) { 134 | return true; 135 | } 136 | }); 137 | 138 | return { 139 | recordings: recordingsFiltered, 140 | totalCount: recordingsAll.length, 141 | filteredCount: recordingsAll.length - recordingsFiltered.length 142 | }; 143 | } 144 | 145 | /** 146 | * Process a moodle course's recordings, and download all missing ones from webex 147 | * @param {config.Course} course The moodle course to process 148 | * @param {import('./helpers/webex').Recording[]} recordings Recordings to process 149 | * @param {Promise} downloadConfigs Download section configs 150 | */ 151 | async function processCourseRecordings(course, recordings, downloadConfigs) { 152 | const courseDownloadPath = join( 153 | downloadConfigs.base_path, 154 | course.name ? `${course.name}_${course.id}` : `${course.id}` 155 | ); 156 | await mkdirIfNotExists(courseDownloadPath); 157 | 158 | const chunks = splitArrayInChunksOfFixedLength(recordings, downloadConfigs.max_concurrent_downloads); 159 | 160 | for (const chunk of chunks) { 161 | const multiProgressBar = (downloadConfigs.progress_bar ? new MultiProgressBar(false) : null); 162 | 163 | const downloads = chunk.map(async (recording) => { 164 | try { 165 | let filename = replaceWhitespaceChars(replaceWindowsSpecialChars(`${recording.name}.${recording.format}`, '_'), '_'); 166 | if (course.prepend_date) 167 | filename = `${getUTCDateTimestamp(recording.created_at, '')}-${filename}`; 168 | 169 | await downloadRecording(recording, filename, courseDownloadPath, downloadConfigs, multiProgressBar); 170 | } catch (err) { 171 | logger.error(` └─ Skipping "${recording.name}": ${err.message}`); 172 | } 173 | }); 174 | 175 | // For simplicity, individual promises must not throw an error, as is the case here. Otherwise Promise.all fails and the entire course is skipped. 176 | await Promise.all(downloads); 177 | if (multiProgressBar) multiProgressBar.terminate(); 178 | } 179 | } 180 | 181 | /** 182 | * Wrapper to download a stream file from an url to a file. 183 | * 184 | * The wrapper manages the various initialization and the progress bar. 185 | * Returns a promise that resolves if the stream was downloaded successfully, rejects it otherwise. 186 | * 187 | * @param {string} url URL of the resource to download as a stream. 188 | * @param {string} savePath Path in which to save the downloaded file. 189 | * @param {LogsConfig} logsConfig Configs for logs and progressBar. 190 | * @returns {Promise} A promise that resolves if the stream was downloaded successfully, rejects it otherwise. 191 | */ 192 | async function downloadStreamWrapper(url, savePath, logsConfig) { 193 | const statusEmitter = new EventEmitter(); 194 | if (logsConfig.multiProgressBar !== null) 195 | new StatusProgressBar( 196 | logsConfig.multiProgressBar, 197 | statusEmitter, 198 | (data) => `[${logsConfig.logStatusName}] ${bytes(parseInt(data.filesize), BYTES_OPTIONS).padStart(9)}`, 199 | (data) => data.filesize, 200 | (data) => data.chunkLength); 201 | 202 | downloadStream(url, savePath, statusEmitter); 203 | 204 | return new Promise((resolve, reject) => { 205 | statusEmitter.on('finish', resolve); 206 | statusEmitter.on('error', (err) => { 207 | reject(err); 208 | }); 209 | }); 210 | } 211 | 212 | /** 213 | * Wrapper to download an HLS stream file from an url to a file. 214 | * 215 | * The wrapper manages the various initialization and the progress bar. 216 | * Returns a promise that resolves if the stream was downloaded successfully, rejects it otherwise. 217 | * 218 | * @param {string} playlistUrl URL of the m3u8 playlist to download as an HLS stream. 219 | * @param {string} filesize Expected filesize used to track progress in logging. 220 | * @param {string} savePath Path in which to save the downloaded file. 221 | * @param {string} tmpFolderPath Path to a temp folder to be used internally to save intermediary segments. 222 | * @param {LogsConfig} logsConfig Configs for logs and progressBar. 223 | * @returns {Promise} A promise that resolves if the HLS stream was downloaded successfully, rejects it otherwise. 224 | */ 225 | async function downloadHLSWrapper(playlistUrl, filesize, savePath, tmpFolderPath, logsConfig) { 226 | const statusEmitter = new EventEmitter(); 227 | if (logsConfig.multiProgressBar !== null) { 228 | new StatusProgressBar( 229 | logsConfig.multiProgressBar, 230 | statusEmitter, 231 | (data) => `[${logsConfig.logStatusName}] ${(data.stage === 'DOWNLOAD') ? (bytes(parseInt(filesize), BYTES_OPTIONS).padStart(9)) : 'MERGE'}`, 232 | (data) => data.segmentsCount, 233 | () => null); 234 | } 235 | 236 | downloadHLS(playlistUrl, savePath, tmpFolderPath, statusEmitter); 237 | 238 | return new Promise((resolve, reject) => { 239 | statusEmitter.on('finish', resolve); 240 | statusEmitter.on('error', (err) => { 241 | reject(err); 242 | }); 243 | }); 244 | } 245 | 246 | /** 247 | * Download the recording using the `file_url` url. 248 | * @param {import('./helpers/webex').Recording} recording The recording to download, using the `file_url` property. 249 | * @param {string} savePath Path in which to save the downloaded file. 250 | * @param {LogsConfig} logsConfig Configs for logs and progressBar. 251 | * @returns A promise that resolves if the stream was downloaded successfully, rejects it otherwise. 252 | */ 253 | async function downloadStreamStrategy(recording, savePath, logsConfig) { 254 | logger.debug(` └─ [${logsConfig.logStatusName}] Trying to download stream`); 255 | 256 | const downloadUrl = await getWebexRecordingDownloadUrl(recording.file_url, recording.password); 257 | 258 | return downloadStreamWrapper(downloadUrl, savePath, logsConfig); 259 | } 260 | 261 | /** 262 | * Download the recording using the `recording_url` url, and the `fallbackPlaySrc` property. 263 | * @param {import('./helpers/webex').Recording} recording The reecording to download using the `recording_url` property. 264 | * @param {string} savePath Path in which to save the downloaded file. 265 | * @param {LogsConfig} logsConfig Configs for logs and progressBar. 266 | * @returns A promise that resolves if the stream was downloaded successfully, rejects it otherwise. 267 | */ 268 | async function downloadFallbackPlaySrcStrategy(recording, savePath, logsConfig) { 269 | logger.debug(` └─ [${logsConfig.logStatusName}] Trying to download fallbackPlaySrc`); 270 | 271 | const streamOptions = await getWebexRecordingStreamOptions(recording.recording_url, recording.password); 272 | if (!streamOptions.fallbackPlaySrc) throw new Error('fallbackPlaySrc property not found'); 273 | 274 | return downloadStreamWrapper(streamOptions.fallbackPlaySrc, savePath, logsConfig); 275 | } 276 | 277 | /** 278 | * Download the recording using the `recording_url` url, and the `downloadRecordingInfo.downloadInfo.hlsURL` property. 279 | * @param {import('./helpers/webex').Recording} recording The reecording to download using the `recording_url` property. 280 | * @param {string} savePath Path in which to save the downloaded file. 281 | * @param {LogsConfig} logsConfig Configs for logs and progressBar. 282 | * @returns A promise that resolves if the stream was downloaded successfully, rejects it otherwise. 283 | */ 284 | async function downloadHlsURLStrategy(recording, savePath, logsConfig) { 285 | logger.debug(` └─ [${logsConfig.logStatusName}] Trying to download hlsURL`); 286 | 287 | const streamOptions = await getWebexRecordingStreamOptions(recording.recording_url, recording.password); 288 | if (!streamOptions?.downloadRecordingInfo?.downloadInfo?.hlsURL) throw new Error('hlsURL property not found'); 289 | const hlsURL = streamOptions.downloadRecordingInfo.downloadInfo.hlsURL; 290 | const playlistSegments = await parseHLSPlaylistSegments(hlsURL); 291 | const streamUrl = hlsURL.replace('hls.m3u8', playlistSegments[0]); 292 | if (playlistSegments.length !== 1) throw new Error('HLS playlist has more than 1 segment'); 293 | 294 | return downloadStreamWrapper(streamUrl, savePath, logsConfig); 295 | } 296 | 297 | /** 298 | * Download the recording using the `recording_url` url, and the `mp4StreamOption` property. 299 | * @param {import('./helpers/webex').Recording} recording The reecording to download using the `recording_url` property. 300 | * @param {string} savePath Path in which to save the downloaded file. 301 | * @param {LogsConfig} logsConfig Configs for logs and progressBar. 302 | * @returns A promise that resolves if the HLS stream was downloaded successfully, rejects it otherwise. 303 | */ 304 | async function downloadHLSStrategy(recording, savePath, tmpFolderPath, logsConfig) { 305 | logger.debug(` └─ [${logsConfig.logStatusName}] Trying to download HLS`); 306 | 307 | const { playlistUrl, filesize } = await retryPromise(10, 2000, 308 | () => getWebexRecordingHLSPlaylist(recording.recording_url, recording.password)); 309 | 310 | return downloadHLSWrapper(playlistUrl, filesize, savePath, tmpFolderPath, logsConfig); 311 | } 312 | 313 | /** 314 | * If the recording doesn't alredy exists, download the recording and save it. 315 | * @param {import('./helpers/webex').Recording} recording Webex Recording object to download. 316 | * @param {string} filename The filename of the recording. 317 | * @param {string} courseDownloadPath The course's download folder where to save recordings. 318 | * @param {config.ConfigDownload} downloadConfigs Download section configs. 319 | * @param {MultiProgressBar} [multiProgressBar=null] MultiProgressBar instance to render download status. 320 | * @returns {Promise} A promise that resolves if the download completed successfully, rejects it otherwise 321 | */ 322 | async function downloadRecording(recording, filename, courseDownloadPath, downloadConfigs, multiProgressBar = null) { 323 | /** Final file save-path after download its complete */ 324 | const downloadFilePath = join(courseDownloadPath, filename); 325 | if (existsSync(downloadFilePath)) { 326 | if (downloadConfigs.show_existing) 327 | logger.info(` └─ Already exists: ${recording.name}`); 328 | return; 329 | } 330 | logger.info(` └─ Downloading: ${recording.name}`); 331 | 332 | /** hash of the recording's resulting filename */ 333 | const filenameHash = createHash('sha1').update(filename).digest('hex'); 334 | 335 | /** Path to a temporary folder where to save all files related to a recording. */ 336 | const tmpDownloadFolderPath = join('./tmp/', filenameHash); 337 | await mkdirIfNotExists(tmpDownloadFolderPath); 338 | 339 | /** Path to the temporary file in which to download the recording */ 340 | const tmpDownloadFilePath = join(tmpDownloadFolderPath, 'recording.mp4'); 341 | 342 | const logStatusName = getUTCDateTimestamp(recording.created_at, ''); 343 | const logsConfig = { 344 | multiProgressBar: downloadConfigs.progress_bar ? multiProgressBar : null, 345 | logStatusName: logStatusName 346 | }; 347 | 348 | // Try to download the recording in different ways. 349 | //TODO: log error messages to debug output 350 | await downloadStreamStrategy(recording, tmpDownloadFilePath, logsConfig) 351 | .catch(async () => 352 | await downloadFallbackPlaySrcStrategy(recording, tmpDownloadFilePath, logsConfig)) 353 | .catch(async () => 354 | await downloadHlsURLStrategy(recording, tmpDownloadFilePath, logsConfig)) 355 | .catch(async () => 356 | await downloadHLSStrategy(recording, tmpDownloadFilePath, tmpDownloadFolderPath, logsConfig)); 357 | 358 | // Download was successful, move rec to destination. 359 | if (downloadConfigs.fix_streams_with_ffmpeg) { 360 | let progressBar = new OneShotProgressBar(multiProgressBar, `[${logStatusName}] REMUX`); 361 | progressBar.init(); 362 | 363 | await remuxVideoWithFFmpeg(tmpDownloadFilePath, downloadFilePath); 364 | unlinkSync(tmpDownloadFilePath); 365 | 366 | progressBar.complete(); 367 | } else { 368 | moveFile(tmpDownloadFilePath, downloadFilePath); 369 | } 370 | } 371 | 372 | /** 373 | * Fetch the recordings list for each course 374 | * @param {Moodle} moodle Moodle instance 375 | * @param {config.Course[]} courses List of courses to fetchs 376 | * @return {Array.>} 377 | */ 378 | function getCourses(moodle, courses) { 379 | logger.info('Fetching recordings lists'); 380 | 381 | return courses.map((course) => 382 | retryPromise(3, 500, () => getRecordings(course, moodle)) 383 | .then(/** @returns {FetchedCourse} */ 384 | recordings => ({ 385 | success: true, 386 | recordings: recordings, 387 | course: course 388 | })) 389 | .catch(/** @returns {FetchedCourse} */ 390 | err => ({ 391 | success: false, 392 | err: err, 393 | course: course 394 | })) 395 | ); 396 | } 397 | 398 | /** 399 | * Process all moodle courses specified in the configs. 400 | * 401 | * Initially, the recordings list are fetched simultaneously. 402 | * Then, each recordings list is processed individually. 403 | * @param {Moodle} moodle 404 | * @param {config.Config} configs 405 | * @returns {Promise} 406 | */ 407 | async function processCourses(moodle, configs) { 408 | const coursesToProcess = getCourses(moodle, configs.courses); 409 | 410 | for (const curCourse of coursesToProcess) { 411 | let { success, err, recordings, course } = await curCourse; 412 | logger.info(`Working on course: ${course.id} - ${course.name ?? ''}`); 413 | 414 | if (!success) { 415 | logger.error(`└─ Error retrieving recordings: ${err.message}`); 416 | continue; 417 | } 418 | logger.info(`└─ Found ${recordings.totalCount} recordings (${recordings.filteredCount} filtered)`); 419 | 420 | try { 421 | await processCourseRecordings(course, recordings.recordings, configs.download); 422 | } catch (err) { 423 | logger.error(`└─ Error processing recordings: ${err.message}`); 424 | continue; 425 | } 426 | } 427 | } 428 | 429 | (async () => { 430 | try { 431 | setupAxios(); 432 | await createTempFolder(); 433 | 434 | let configs = await loadConfig(); 435 | 436 | const moodle = new Moodle(); 437 | await loginToMoodle(moodle, configs); 438 | 439 | await processCourses(moodle, configs); 440 | 441 | logger.info('Done'); 442 | } catch (err) { 443 | logger.error(err); 444 | logger.warn('Exiting in 5s...'); 445 | await sleep(5000); 446 | } 447 | })(); 448 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # unifi-webex-dl 2 | 3 | > Download recorded lessons from UniFi's Webex platform passing by the Moodle platform. 4 | 5 | This utility can automatically download all UniFi courses' recordings saved on Webex. 6 | 7 | ## ⚠ Update 2023-09 ⚠ 8 | 9 | Our University decided to not renew the contract with webex and therefore the recordings are not available anymore. 10 | 11 | For anyone interested in reusing this for webex recordings found somewhere else, it should be trivial to extract the webex logic as the downloader is modular. 12 | I'd suggest to use the `webex.js` helper together with the `downloadRecording` function defined in `app.js`. (All assuming webex didn't change the requests flow...) 13 | 14 | I'll archive this repository for the moment. 15 | 16 | ## Index 17 | 18 | - [Quick Start](#quick-start) 19 | - [Quick Start with Node](#optional-quick-start-with-node) 20 | - [Quick Start with Docker](#optional-quick-start-with-docker) 21 | - [PLEASE NOTE - Known issues](#please-note---known-issues) 22 | - [Config](#config) 23 | - [Credentials](#credentials) 24 | - [Download](#download) 25 | - [Courses](#courses) 26 | - [Environment variables](#environment-variables) 27 | - [Logging](#logging) 28 | - [How it works](#how-it-works) 29 | 30 | ## Quick Start 31 | 32 | The easiest way to start is by downloading the pre-compiled application from the latest release available. 33 | 34 | Then, copy the `config.example.json` file to a new file named `config.json` and change the credentials and courses ids accordingly. 35 | 36 | Done, that's it! 37 | 38 | While being the easiest method, it does come with a drawback. To update it, you'll have to manually check the repository once in a while and download the latest version. For this reason, if possible, it is recommended to use the following method that uses Node directly. 39 | 40 | ### [Optional] Quick Start with Node 41 | 42 |
43 | [Optional] Quick Start with Node 44 | 45 | Node.js v14 or newer is required. 46 | 47 | - Install project dependencies: `npm ci` 48 | 49 | - Copy `config.example.json` to a new file named `config.json` and change credentials and courses ids accordingly. 50 | 51 | - Run the app with: `npm start` 52 | 53 | When you pull new updates, remember to update project dependencies using `npm ci`. 54 | 55 |
56 | 57 | ### [Optional] Quick Start with Docker 58 | 59 |
60 | [Optional] Quick Start with Docker 61 | 62 | Suppose you are on Linux and have docker. In that case, you can execute the `docker.sh` to automatically execute the downloader inside of a container. 63 | 64 | Note a few things: 65 | 66 | - Make sure to use the same UID and GID of your user in the `Dockerfile`. By default, they are both set to 1000; 67 | - If you use `.yaml` configs instead of `.json`, change the extension accordingly in `docker.sh` 68 | 69 |
70 | 71 | ## PLEASE NOTE - Known issues 72 | 73 | Errors related to stream downloads: 74 | 75 | - If a recording doesn't seem to have the audio while reproducing it with the Windows Media Player, try with a different player such as VLC. 76 | - If there are stutters while scrubbing the timeline or segments missing audio, this is caused by the way HLS recordings are downloaded and by how poorly Webex encodes the recordings. To solve this, install `ffmpeg`, enable the `fix_streams_with_ffmpeg` option and then delete-and-redownload the faulty recordings. 77 | 78 | If a recording gives you an error, verify on Webex that it can actually be opened before opening an issue. Recordings could be disabled by the course organizer. 79 | 80 | If you get a `429 Error`, it means that Webex received too many requests. In this case, you should wait some time before trying again. 81 | 82 | If the tool repeatedly fails to download a specific recording, feel free to open an issue to let me know what happens. 83 | 84 | ## Config 85 | 86 |
87 | Config section 88 | 89 | > The config file has 3 sections. 90 | 91 | Currently, both **.json** and **.yaml** file are supported, JSON being the default one. 92 | 93 | The default config file path is `config.json` inside the root directory; you can change it with the environment variable `CONFIG_PATH`. 94 | 95 | ### Credentials 96 | 97 | | Key name | Value type | Optional | Default value | Description | 98 | |------------|------------|----------|---------------|----------------------------------------------------------| 99 | | `username` | string | No | | Username used for authenticating to the Moodle Platform. | 100 | | `password` | string | No | | Password used for authenticating to the Moodle Platform. | 101 | 102 | ### Download 103 | 104 | | Key name | Value type | Optional | Default value | Description | 105 | |----------------------------|------------|----------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 106 | | `base_path` | string | No | | Path in which to download recordings. | 107 | | `progress_bar` | boolean | Yes | true | Show a progress bar while downloading the recordings. | 108 | | `show_existing` | boolean | Yes | true | Show already downloaded recordings. | 109 | | `max_concurrent_downloads` | number | Yes | 3 | maximum number of parallel downloads. | 110 | | `fix_streams_with_ffmpeg` | boolean | Yes | false | Remux all recordings using ffmpeg. This requires ffmpeg to be installed and available on system path. Additionally, check that the h264/mp4/m4v formats are supported for remuxing operations. On Linux you can check this using `ffmpeg -formats | egrep '(h264|mp4|m4v)'` | 111 | 112 | ### Courses 113 | 114 | > Array of objects, one for each course. The object contains the following fields. 115 | 116 | Please note that on Windows the `name` field shouldn't contain any of the not allowed characters, such as `: " *`. It is therefore recommended to keep the name simple using only letters, numbers, hyphens, and underscores. 117 | 118 | | Key name | Value type | Optional | Description | 119 | |--------------------|------------|----------|---------------------------------------------------------------------------------------------------------------------| 120 | | `id` | string | No | Id of the course shown in the URL bar of the Moodle's course page. | 121 | | `name` | string | No | Name prepended to the folder name and also shown in the logs. | 122 | | `custom_webex_id` | string | Yes | Manually set the id of the Webex page instead of trying to find it in the course page. | 123 | | `skip_names` | string | Yes | Regex to match recordings names to skip. Exclude slashes and flags from strings. E.g. `'test'` and NOT `'/test/i'`. | 124 | | `skip_before_date` | string | Yes | Skip recordings before the date `YYYY-MM-DD`. | 125 | | `skip_after_date` | string | Yes | Skip recordings after the date `YYYY-MM-DD`. | 126 | | `prepend_date` | boolean | Yes | Prepend the date of the recording (`YYYYMMDD-`) to the filenames. | 127 | 128 |
129 | 130 | ## Environment variables 131 | 132 | The app tries to be as docker-friendly as possible. 133 | 134 | In fact, as an alternative, the configs may all be specified using environment variables. Just convert the config names to uppercase. In the case of nested properties, separate them with two underscores. 135 | 136 | E.g. `credentials.username` => `CREDENTIALS__USERNAME`; `download.base_path` => `DOWNLOAD__BASE_PATH` 137 | 138 | Courses can also be specified through the `COURSES` env variable using the following format, although limited to only `id` and `name`: 139 | 140 | `COURSE_ID=COURSE_NAME,12003=WhiteRabbit` 141 | 142 | ## Logging 143 | 144 | To modify the default log level of 'info', set the env variable `LOG_LEVEL` with one of [winston available log_level](https://github.com/winstonjs/winston#logging-levels). 145 | 146 | ## How it works 147 | 148 |
149 | Inner workings of the downloader (Optional) 150 | 151 | Unfortunately, UniFi Moodle doesn't make use of REST APIs. So we have to do a bit of guessing and matching on the response body. 152 | 153 | This approach works for now but is prone to errors and will stop working if things change. Feel free to open an issue or a PR to report these changes. 154 | 155 | ### Login to Moodle 156 | 157 | _As of March 2021, they use this new unified authentication system for accessing their services._ 158 | 159 | > GET 160 | 161 | In the response body match 162 | 163 | ``. 164 | 165 | Then post the form with the `execution` field. 166 | 167 | > POST 168 | > 169 | > Content-Type: application/x-www-form-urlencoded 170 | 171 | The request body should match the following format: 172 | 173 | ```json 174 | { 175 | "username": 00000, 176 | "password": "*****", 177 | "execution": "...", 178 | "_eventId": "submit", 179 | "geolocation": "" 180 | } 181 | ``` 182 | 183 | If the credentials are wrong, a status code `401` should be returned from the POST request. 184 | 185 | Otherwise, follow the `Location` header that should have a ticket in the URL parameters. 186 | 187 | Set `MoodleSession` Cookie from the Set-Cookie response header and follow the `Location` header again. 188 | 189 | Finally, get the authenticated `MoodleSession` Cookie from the Set-Cookie response header. 190 | 191 | ### Get Webex Id 192 | 193 | To launch Webex, we have to get the Webex course id relative to the Moodle course id. 194 | 195 | > GET 196 | 197 | In the body, match the launch URL: 198 | 199 | - `https://e-l.unifi.it/mod/lti/launch.php?id=***` 200 | 201 | Retrieve the id parameter 202 | 203 | ### Get Webex launch parameters 204 | 205 | > GET > 206 | > 207 | > Cookie: MoodleSession 208 | 209 | Serialize from the HTML body all the name attributes in input tags 210 | 211 | ### Launch Webex 212 | 213 | > POST 214 | > 215 | > Content-Type: application/x-www-form-urlencoded 216 | 217 | In the body send the parameters retrieved [from Moodle](#get-webex-launch-parameters) 218 | 219 | From the response: 220 | 221 | Get cookies [`ahoy_visitor`, `ahoy_visit`, `_ea_involvio_lti_session`] 222 | 223 | ### Get Webex course recordings 224 | 225 | > GET 226 | 227 | The request headers should match the following 228 | 229 | ```html 230 | Cookie: ahoy_visitor=***,ahoy_visit=***,_ea_involvio_lti_session=*** 231 | ``` 232 | 233 | The response is an array of objects like the following 234 | 235 | ```json 236 | [ 237 | { 238 | "created_at": "2020-01-13T00:00:00.000-07:00", 239 | "duration_hour": 0, 240 | "duration_min": 0, 241 | "duration_sec": 0, 242 | "file_url": "https://unifirenze.webex.com/unifirenze/lsr.php?RCID=******", 243 | "format": "MP4", 244 | "id": 0, 245 | "name": "", 246 | "password": "", 247 | "recording_url": "https://unifirenze.webex.com/unifirenze/ldr.php?RCID=******", 248 | "timezone": "Europe/Rome", 249 | "updated_at": "2020-01-13T00:00:00.000-07:00" 250 | } 251 | ] 252 | ``` 253 | 254 | ### Download a recording - STEP 1 255 | 256 | Before starting, to better understand the following operations, from what I understood, there are two types of recordings. 257 | 258 | There are recordings of `Meetings` and recordings of `Events`. Some of the following strategies are structured upon this idea. 259 | 260 | As some teachers disable the download functionality, I've implemented multiple strategies to try and download the recordings. The following strategies are tried in order, if one fails, the next one is tried. If they all fail, the recording is skipped and an error is logged. 261 | 262 | 1. [Webex download functionality](#download-with-webex-functionality---step-1): uses the `file_url` property. 263 | 2. [fallbackPlaySrc property](#download-using-fallbackplaysrc): Using the `recording_url`, download the `fallbackPlaySrc` property. 264 | 3. [hlsURL property](#download-using-hlsurl): Using the `recording_url`, download using the `downloadRecordingInfo.downloadInfo.hlsURL` property. 265 | 4. [HLS stream](#download-hls-stream---step-1): Using the `recording_url`, download using the `mp4StreamOptions` property. 266 | 267 | ### Download with Webex functionality - STEP 1 268 | 269 | > GET `file_url` 270 | 271 | 1. If the response matches `Error` then, there's been an error. This means that the recording has the download functionality disabled, or has been deleted, or isn't available at the moment. Try with another strategy. 272 | 273 | 2. If the response contains `'internalRecordTicket'` then you're downloading an event. Go to [STEP 2b](#download-with-webex-functionality---step-2b) 274 | 275 | 3. If none of the above, then you're downloading a meeting. Go to [STEP 2a](#download-with-webex-functionality---step-2a) 276 | 277 | ### Download with Webex functionality - STEP 2a 278 | 279 | If the response of the previous step doesn't contain `recordingpasswordcheck`, the recording doesn't need a password, and you can skip to [STEP 3](#download-with-webex-functionality---step-3). Also, note that if the response doesn't contain "commonGet2PostForm", you should instead skip to STEP 3 after the first request to `nbrshared.do`. 280 | 281 | Otherwise, follow along... 282 | 283 | Get all `name` and `values` attributes from the input tags. 284 | 285 | Note that you may need to change `firstEntry` to false since the JS does it there: 286 | 287 | ```js 288 | document.forms[0].firstEntry.value=false; 289 | ``` 290 | 291 | > POST 292 | 293 | The body should contain the input attributes from the previous request and the password of the recording. 294 | 295 | Go to [STEP 3](#download-with-webex-functionality---step-3) 296 | 297 | ### Download with Webex functionality - STEP 2b 298 | 299 | > Follow the `redirect` of the previous request. 300 | 301 | Serialize the form inputs and: 302 | 303 | - add password to `playbackPasswd=` 304 | - change `theAction=...` to `theAction=check_pass` 305 | - change `accessType=...` to `accessType=downloadRecording` 306 | 307 | > POST `https://unifirenze.webex.com/ec3300/eventcenter/recording/recordAction.do` 308 | > 309 | > Content-Type: application/x-www-form-urlencoded 310 | 311 | Save cookies from the response header. 312 | 313 | Parse from the response the following fields: 314 | 315 | - formId 316 | - accessType 317 | - internalPBRecordTicket 318 | - internalDWRecordTicket 319 | 320 | > POST `https://unifirenze.webex.com/ec3300/eventcenter/enroll/viewrecord.do` 321 | > 322 | > Content-Type: application/x-www-form-urlencoded 323 | > 324 | > Cookie: From the previous step 325 | 326 | Request body: 327 | 328 | ```jsonc 329 | { 330 | "firstName": "Anonymous", 331 | "lastName": "Anonymous", 332 | "siteurl": "unifirenze", 333 | "directview": 1, 334 | "AT": "ViewAction", 335 | "recordId": 0000, // formId of the previous step 336 | "accessType": "downloadRecording", 337 | "internalPBRecordTicket": "4832534b000000040...", 338 | "internalDWRecordTicket": "4832534b00000004f..." 339 | } 340 | ``` 341 | 342 | Parse from the response the following fields: 343 | 344 | - siteurl 345 | - recordKey 346 | - recordID 347 | - serviceRecordID 348 | 349 | Go to [STEP 3](#download-with-webex-functionality---step-3) 350 | 351 | ### Download with Webex functionality - STEP 3 352 | 353 | From the previous request match `var href='https://unifirenze.webex.com/mw3300/mywebex/nbrshared.do?siteurl=unifirenze-en&action=publishfile&recordID=***&serviceRecordID=***&recordKey=***';` 354 | 355 | Parse the URL arguments and make the following request. 356 | 357 | > POST `https://unifirenze.webex.com/mw3300/mywebex/nbrshared.do` 358 | > 359 | > Content-Type: application/x-www-form-urlencoded 360 | 361 | Request body: 362 | 363 | ```jsonc 364 | { 365 | "action": "publishfile", // always required 366 | "siteurl": "unifirenze", // could also be 'unifirenze-en' 367 | "recordKey": "***", 368 | "recordID": "***", 369 | "serviceRecordID": "***", 370 | } 371 | ``` 372 | 373 | Match the following part 374 | 375 | ```js 376 | function download(){ 377 | document.title="Download file"; 378 | var recordId = 000; 379 | var serviceRecordId = 000; 380 | var prepareTicket = '******'; 381 | var comeFrom = ''; 382 | var url = "https://unifirenze.webex.com/mw3300/mywebex/nbrPrepare.do?siteurl=unifirenze-en" + "&recordid=" + recordId+"&prepareTicket=" + prepareTicket; 383 | if (serviceRecordId > 0) { 384 | url = url + "&serviceRecordId=" + serviceRecordId; 385 | } 386 | 387 | _refreshIFrame(url,1); 388 | } 389 | ``` 390 | 391 | > GET `matched nbrPrepare.do url` 392 | 393 | Match `window.parent.func_prepare('***','***','***');` 394 | 395 | This is the function declaration `func_prepare(status, url, ticket)` that I'll refer to. 396 | 397 | Check the `status` that could be one of the following [`OKOK`, `Preparing`, `Error`, "null if bug?"] 398 | 399 | - Error: 400 | - Throw error and skip this file 401 | - Preparing: 402 | - Fetch again GET "https://unifirenze.webex.com/mw3300/mywebex/nbrPrepare.do?siteurl=unifirenze-en" + `url` 403 | - OKOK: 404 | - Match `var downloadUrl = 'https://***.webex.com/nbr/MultiThreadDownloadServlet?siteid=***&recordid=***&confid=***&language=1&userid=***&serviceRecordID=***&ticket=' + ticket;` 405 | 406 | > GET `MultiThreadDownloadServlet` 407 | 408 | The response is the recording that can be saved as `name`.`format` (from the recording object). 409 | 410 | ### Download using hlsURL 411 | 412 | > GET `recording_url` 413 | 414 | If in the response JSON object DOES NOT contain the `downloadRecordingInfo.downloadInfo.hlsURL`, this strategy won't work. Try another one. 415 | 416 | In this case the HLS playlist doesn't contain a list of segments but only the name of the recording with a bunch of ranges. We only need the recording name to download it and ignore the ranges part. 417 | 418 | > GET 419 | 420 | In the response, containing the m3u8 playlist, match the `#EXT-X-MAP:URI="filename.mp4", ...` line and extract the filename. 421 | 422 | Download the recording replacing `hls.m3u8` with `filename.mp4` in the last URL -> 423 | 424 | ### Download using fallbackPlaySrc 425 | 426 | > GET `recording_url` 427 | 428 | If in the response JSON object DOES NOT contain the `fallbackPlaySrc`, this strategy won't work. Try another one. 429 | 430 | > GET `fallbackPlaySrc` 431 | 432 | ### Download HLS Stream - STEP 1 433 | 434 | > GET `recording_url` 435 | 436 | 1. If the response matches `Error` then, there's been an error. Try another strategy. 437 | 438 | 2. If the response contains `'internalRecordTicket'`, then you're downloading an event. This is a `WIP` since I never found an event recording with download disabled. Feel free to open an Issue to solve this. 439 | 440 | 3. If none of the above, then you're downloading a meeting. Go to [Download HLS Stream](#download-hls-stream---step-2) 441 | 442 | ### Download HLS Stream - STEP 2 443 | 444 | From the response of the `recording_url`, match the recording ID. 445 | 446 | ```js 447 | location.href='https://unifirenze.webex.com/recordingservice/sites/unifirenze/recording/playback/RECORDING_ID'; 448 | ``` 449 | 450 | > GET 451 | 452 | In the request, also add the following custom header 453 | 454 | `accessPwd: RECORDING_PASSWORD` 455 | 456 | **Optionally**: if you wanna get the approximate filesize, sum `fileSize` with `mediaDetectInfo.audioSize` 457 | 458 | Save the `mp4StreamOption` parameter and proceed to [Download HLS Stream - STEP 3](#download-hls-stream---step-3) 459 | 460 | ### Download HLS Stream - STEP 3 461 | 462 | > POST 463 | 464 | In the request, add the following query parameters from the `mp4StreamOption` object of the previous step: 465 | 466 | ```json 467 | { 468 | "recordingDir": "", 469 | "timestamp": 0, 470 | "token": "", 471 | "xmlName": "" 472 | } 473 | ``` 474 | 475 | From the response match `HLS_FILE` 476 | 477 | ```xml 478 | 479 | HLS_FILE 480 | 481 | ``` 482 | 483 | Use all parameters in this URL, and you get the HLS Playlist file to download 484 | 485 | ```js 486 | let playlistFile = `https://nfg1vss.webex.com/hls-vod/recordingDir/${mp4StreamOption.recordingDir}/timestamp/${mp4StreamOption.timestamp}/token/${mp4StreamOption.token}/fileName/${HLS_FILE}.m3u8` 487 | ``` 488 | 489 |
490 | --------------------------------------------------------------------------------