├── .editorconfig ├── .env.defaults ├── .gitignore ├── .npmignore ├── .travis.yml ├── LICENSE ├── README.md ├── chuck.png ├── package.json ├── postman ├── chuck.postman_collection.json └── devel.postman_environment.json ├── src ├── admin │ ├── admin.pug │ └── index.ts ├── api │ ├── api_keys_cache.ts │ ├── conversions_api.ts │ ├── conversions_api_events.ts │ ├── index.ts │ └── middlewares.ts ├── cli │ ├── api │ │ ├── generate_key.ts │ │ ├── index.ts │ │ └── revoke_key.ts │ └── index.ts ├── config.ts ├── converter │ ├── job.ts │ ├── job_events.ts │ ├── plugin_steps.ts │ ├── queue.ts │ ├── queue_event_handlers.ts │ ├── queue_processor.ts │ └── steps │ │ ├── 01_download_assets.ts │ │ ├── 02_exec_assetbundlecompiler.ts │ │ ├── 03_upload_bundle.ts │ │ ├── index.ts │ │ └── step.ts ├── entry │ ├── app_standalone.ts │ ├── bootstrap.ts │ ├── cli.ts │ ├── libchuck.ts │ └── standalone.ts ├── express_utils.ts ├── logger.ts ├── models │ ├── IApiKey.ts │ ├── IConversion.ts │ ├── api_key.ts │ ├── conversion.ts │ └── index.ts ├── mongoose.ts ├── safe_error_serialize.ts └── web_uis.ts ├── stubs └── toureiro.d.ts ├── test ├── test.js └── test.sh ├── tsconfig.json ├── tslint.json └── yarn.lock /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | indent_style = space 6 | indent_size = 4 7 | end_of_line = lf 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | 11 | [*.md] 12 | indent_size = 2 13 | 14 | [*.json] 15 | indent_size = 2 16 | -------------------------------------------------------------------------------- /.env.defaults: -------------------------------------------------------------------------------- 1 | CHUCK_ENV=development 2 | CHUCK_LOGLEVEL=verbose 3 | CHUCK_SERVERPORT=3001 4 | CHUCK_MONGOURL=mongodb://localhost/chuck 5 | CHUCK_RAVENDSN= 6 | CHUCK_REDIS_HOST=localhost 7 | CHUCK_REDIS_PORT=6379 8 | CHUCK_REDIS_DB=0 9 | CHUCK_ADMINWEBUIS_ENABLE=false 10 | CHUCK_ADMINWEBUIS_USER=admin 11 | CHUCK_ADMINWEBUIS_PASSWORD=admin 12 | CHUCK_AZURE_ENABLEEMU=false 13 | CHUCK_STEPMODULEPLUGINS= 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | node_modules/ 3 | .env 4 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | postman/* 2 | src/* 3 | stubs/* 4 | .editorconfig 5 | .env 6 | .gitignore 7 | .npmignore 8 | .travis.yml 9 | chuck.png 10 | tsconfig.json 11 | tslint.json 12 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | 3 | cache: 4 | directories: 5 | - node_modules 6 | 7 | notifications: 8 | email: false 9 | 10 | node_js: 11 | - '7' 12 | - '6' 13 | 14 | before_script: 15 | - yarn build 16 | 17 | after_success: 18 | - yarn semantic-release 19 | 20 | branches: 21 | except: 22 | - /^v\d+\.\d+\.\d+$/ 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Method in the Madness 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | chuck webservice logo 2 | 3 | # chuck [![npm version](https://img.shields.io/npm/v/@mitm/chuck.svg?style=flat-square)](https://www.npmjs.com/package/@mitm/chuck) ![license](https://img.shields.io/github/license/mitmadness/chuck.svg?style=flat-square) [![Travis Build](https://img.shields.io/travis/mitmadness/chuck.svg?style=flat-square)](https://travis-ci.org/mitmadness/chuck) 4 | 5 | chuck is a webservice that converts files that Unity understands to Unity3D AssetBundles. 6 | 7 | From the Unity3D [documentation](https://docs.unity3d.com/Manual/AssetBundlesIntro.html): 8 | 9 | > AssetBundles are files which you can export from Unity to contain Assets of your choice, [that] can be loaded on demand by your application. This allows you to stream content, such as models, Textures, audio clips, or even entire Scenes [...]. 10 | 11 | Chuck notably features REST & Server-Sent APIs, a command-line interface, an admin page for managing API keys, and integrates [Toureiro](https://github.com/Epharmix/Toureiro) for viewing queued jobs. 12 | 13 | *:point_right: See also: [@mitm/assetbundlecompiler](https://github.com/mitmadness/AssetBundleCompiler), fluent JavaScript API to create AssetBundles from any files.* 14 | 15 | *:point_right: See also: [@mitm/unityinvoker](https://github.com/mitmadness/UnityInvoker), invoke Unity3D CLI with a fluent JavaScript API.* 16 | 17 | ---------------- 18 | 19 | - [Requirements](#requirements) 20 | - [Installation](#installation): [Standalone](#standalone-installation) / [Embedded](#embedded-installation) 21 | - [Configuration](#configuration) 22 | - [Public REST API](#public-rest--sse-api) 23 | - Extra tools: [Command Line Interface](#command-line-interface) / [Admin Web Interface](#admin-web-interface) / [Toureiro](#toureiro) 24 | - [Development & Contributing](#development--contributing) 25 | 26 | ---------------- 27 | 28 | ## Requirements 29 | 30 | - **[Node.js](https://nodejs.org/en/)** v.7 or higher ; 31 | - **[yarn](https://yarnpkg.com/en/)** dependency manager, for development or standalone installation ; 32 | - An Azure account where chuck will send its generated asset bundles (limitation) ; 33 | - A running **[MongoDB](https://www.mongodb.com/)** server ; 34 | - A running **[Redis](https://redis.io/)** server ; 35 | - :warning: **An _activated_ installation of Unity on the machine** :warning: 36 | - If Unity is not installed in the standard path, configure the path via the [Configuration](#configuration) 37 | - You must activate Unity if not already done, even with a free plan, read [Unity activation](https://github.com/mitmadness/AssetBundleCompiler#unity-activation) from AssetBundleCompiler 38 | 39 | Also, except logging them, chuck can optionally report fatal errors on [Sentry](https://sentry.io/welcome/). To enable this, [configure](#configuration) your Sentry DSN. 40 | 41 | ## Installation 42 | 43 | Please note that chuck is not a pure library, nor a reusable Express middleware. It's a standalone application with its own logging, database, etc. But it can also be installed as an npm package for writing plugins or configuring and boot it. 44 | 45 | ### Standalone installation 46 | 47 |
48 | Classical approach. Use this to deploy chuck directly without integrating it to another application (except API communication of course) 49 | 50 | #### 1. Install the application and dependencies 51 | 52 | ``` 53 | $ git clone git@github.com:mitmadness/chuck.git && cd chuck 54 | $ yarn install 55 | $ yarn build 56 | ``` 57 | 58 | #### 2. Configure Chuck 59 | 60 | Create a blank `.env` file at the root. 61 | 62 | Look at the `.env.defaults` file (don't delete it, it's part of the application), it contains key/value pairs of environment variables. 63 | 64 | You can now override values from `.env.defaults` to match your own environment when the default values are incorrect. 65 | 66 | You can also, of course, set environment variables by hand with your preferred method (exports, inline variables when launching the command...). 67 | 68 | *:point_right: Read more about how configuration is mapped with environment variables in [Using environment variables](#using-environment-variables)* 69 | 70 | #### 3. Run it 71 | 72 | Run `yarn start`. That's it. 73 | 74 | To use chuck plugins, install them (ex. `yarn add @mitm/chuck-ifc`) then ask chuck to load them via the `CHUCK_STEPMODULEPLUGINS` environment variable: 75 | 76 | ``` 77 | CHUCK_STEPMODULEPLUGINS=@mitm/chuck-ifc,myplugin yarn start 78 | ``` 79 |
80 | 81 | ### Embedded installation 82 | 83 |
84 | Chuck app integration into another package: chuck as a npm package. This is the preferred method and it gives you more flexibility, doesn't need compilation, permits better versioning, etc. 85 | 86 | #### 1. Install chuck via its package 87 | 88 | ``` 89 | yarn add @mitm/chuck 90 | ``` 91 | 92 | #### 2. Configure & boot it 93 | 94 | You can either set up your own entry point and configure it in JavaScript (see [Configuration](#configuration)), or you can still use environment variables. 95 | 96 | **With environment variables**, do it like this: 97 | 98 | ``` 99 | CHUCK_SERVERPORT=80 CHUCK_MONGOURL=mongodb://localhost/chuck yarn chuck 100 | ``` 101 | 102 | You can also, of course, set environment variables by hand with your preferred method (exports, inline variables when launching the command...). 103 | 104 | Run the CLI similarly with `yarn chuck-cli`. 105 | 106 | *:point_right: Read more about how configuration is mapped with environment variables in [Using environment variables](#using-environment-variables)* 107 | 108 | **With your own entry point**: 109 | 110 | Create a file named, for example, `chuck.ts` in your project: 111 | 112 | ```typescript 113 | import { boot } from '@mitm/chuck'; 114 | 115 | boot({ 116 | serverPort: 80, 117 | mongoUrl: 'mongodb://localhost/chuck' 118 | }); 119 | ``` 120 | 121 | You can still use environment variables with `boot()`. 122 |
123 | 124 | ## Configuration 125 | 126 | ### Available configuration keys 127 | 128 |
129 | This is the interface for the available configuration 130 | 131 | ```ts 132 | interface IChuckConfig { 133 | // aka NODE_ENV. Configures the mode (`development` or `production`) in which the server is running. 134 | // development: permissive CORS rules are set on the API 135 | // production: timestamps in log messages and more verbose HTTP logs 136 | // Defaults to process.env.NODE_ENV or, if undefined, "development" 137 | env: EnvType; 138 | 139 | // Minimum log level (npm log levels, see https://github.com/winstonjs/winston#logging-levels). 140 | // Defaults to "verbose" 141 | logLevel: string; 142 | 143 | // Server HTTP port. 144 | // Defaults to 3001 145 | serverPort: number; 146 | 147 | // Connection string to a MongoDB database. 148 | // Defaults to mongodb://localhost/chuck 149 | mongoUrl: string; 150 | 151 | // DSN for Sentry error reporting. 152 | // Reporting is disabled if this is not set. 153 | ravenDsn: string; 154 | 155 | // Redis connection informations. 156 | // Defaults to { host: 'localhost', port: 6379, db: 0 } 157 | redis: { host: string; port: number; db: number }; 158 | 159 | // Admin Web UIs configuration. Used by the admin interface and Toureiro. 160 | // Default to { enable: false, user: 'admin', password: 'admin' } 161 | adminWebUis: { enable: boolean, user: string; password: string; }; 162 | 163 | // Unity Editor path (if not installed in the standard path), see https://github.com/mitmadness/AssetBundleCompiler#changing-unitys-executable-path 164 | // Default to undefined (auto) 165 | unityPath: string; 166 | 167 | // Azure configuration. 168 | // Default to { enableEmu: false } 169 | azure: { enableEmu: boolean; }; 170 | 171 | /** 172 | * An array of module names. 173 | * Those modules will be loaded dynamically as step plugins. 174 | */ 175 | stepModulePlugins: string[]; 176 | } 177 | ``` 178 |
179 | 180 | ### Using environment variables 181 | 182 |
183 | Chuck is primarily configurable via environment variables. Read here how to map the configuration interface options on environment variables. 184 | 185 | You can set environement variables in the way you prefer. Tou can set them inline, in the CLI command that launches chuck, via a shell `export`, or for example, if you use the standalone installation, via an `.env` file at root that overrides Chuck's `.env.defaults` values (only for redefined keys). 186 | 187 | Then, environment variables are simply mapped to the real configuration. Take those example: 188 | 189 | - To set `config.logLevel`, set `CHUCK_LOGLEVEL` 190 | - To set `config.adminWebUis.enable`, set `CHUCK_ADMINWEBUIS_ENABLE`. 191 | 192 | Etc. Prefix with `CHUCK_` and replace dots with underscores, all uppercase. 193 |
194 | 195 | ## Public REST / SSE API 196 | 197 | Chuck exposes a simple REST and [SSE (Server-Sent Events)](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events) API for converting files to asset bundles. 198 | 199 | ### Create a new conversion request 200 | 201 | `POST /api/conversions` 202 | 203 |
204 | This endpoint will create a conversion request. It will immediately push on the conversions queue, so the job will start as soon as possible (conversions are ran sequentially). 205 | 206 | #### Request 207 | 208 | :closed_lock_with_key: This endpoint requires authentication using an API key. You can generate one either via the CLI, or via the web interface. Pass the API key like this: `Authorization: Bearer YOUR_API_KEY`. 209 | 210 | Note: `compilerOptions` is an object of options to pass to AssetBundleCompiler ([abcompiler's reference](https://github.com/mitmadness/AssetBundleCompiler#link-simple-fluent-api)). 211 | 212 | Note: `azure.sharedAccessSignatureToken` is an [Azure SAS token](https://docs.microsoft.com/en-us/azure/storage/storage-dotnet-shared-access-signature-part-1) that lets chuck create the asset bundle binary blob on Azure, without giving your Azure credentials to chuck. You can automate token creation with Azure CLI or Azure SDKs. 213 | 214 | ``` 215 | POST /api/conversions 216 | Content-Type: application/json 217 | Authorization: Bearer {{apiKey}} 218 | ``` 219 | ```json 220 | { 221 | "assetBundleName": "myassetbundle.ab", 222 | "compilerOptions": { 223 | "targeting": "webgl" 224 | }, 225 | "assetUrls": [ 226 | "https://url/to/a/file.fbx", 227 | "https://url/to/another/file.ifc" 228 | ], 229 | "azure": { 230 | "host": "{{azureHost}}", 231 | "container": "{{azureContainer}}", 232 | "sharedAccessSignatureToken": "{{azureSharedAccessSignatureToken}}" 233 | } 234 | } 235 | ``` 236 | 237 | #### Successful response 238 | 239 | ``` 240 | HTTP/1.1 202 Accepted 241 | Content-Type: application/json; charset=utf-8 242 | ``` 243 | ```json 244 | { 245 | "assetBundleName": "myassetbundle.ab", 246 | "conversion": { 247 | "logs": [], 248 | "assetBundleUrl": null, 249 | "error": null, 250 | "step": null, 251 | "isCompleted": false, 252 | "jobId": "138" 253 | }, 254 | "compilerOptions": { 255 | "targeting": "webgl", 256 | "editorScripts": [], 257 | "buildOptions": {} 258 | }, 259 | "azure": { 260 | "host": "{{azureHost}}", 261 | "container": "{{azureContainer}}", 262 | "sharedAccessSignatureToken": "{{azureSharedAccessSignatureToken}}" 263 | }, 264 | "assetUrls": [ 265 | "https://url/to/a/file.fbx", 266 | "https://url/to/another/file.ifc" 267 | ], 268 | "code": "00cad557-5acc-4c6b-a987-79f650d67197" 269 | } 270 | ``` 271 |
272 | 273 | ### Retrieve a conversion request 274 | 275 | `GET /api/conversions/{{code}}` 276 | 277 |
278 | Retrieves a previously-posted conversion request. This endpoint has no authentication as the conversion code adds a first layer of security (moreover, conversion requests are not editable). 279 | 280 | #### Request 281 | 282 | ``` 283 | GET /api/conversions/{{conversionCode}} 284 | ``` 285 | 286 | #### Successful response (completed conversion) 287 | 288 | ``` 289 | HTTP/1.1 200 OK 290 | Content-Type: application/json; charset=utf-8 291 | ``` 292 | ```json 293 | { 294 | "assetBundleName": "myassetbundle.ab", 295 | "conversion": { 296 | "assetBundleUrl": "https://{{azureHost}}/{{azureContainer}}/myassetbundle.ab", 297 | "error": null, 298 | "step": null, 299 | "isCompleted": true, 300 | "jobId": "139" 301 | }, 302 | "compilerOptions": { 303 | "targeting": "webgl", 304 | "editorScripts": [], 305 | "buildOptions": {} 306 | }, 307 | "azure": { 308 | "host": "{{azureHost}}", 309 | "container": "{{azureContainer}}", 310 | "sharedAccessSignatureToken": "{{azureSharedAccessSignatureToken}}" 311 | }, 312 | "assetUrls": [ 313 | "https://url/to/a/file.fbx", 314 | "https://url/to/another/file.ifc" 315 | ], 316 | "code": "00cad557-5acc-4c6b-a987-79f650d67197" 317 | } 318 | ``` 319 | 320 | #### Successful response (failed conversion) 321 | 322 | ``` 323 | HTTP/1.1 200 OK 324 | Content-Type: application/json; charset=utf-8 325 | ``` 326 | ```json 327 | { 328 | ... 329 | "conversion": { 330 | "assetBundleUrl": null, 331 | "error": { 332 | "name": "Error", 333 | "message": "Error(s) while downloading assets", 334 | "errors": [ 335 | { 336 | "name": "FetchError", 337 | "message": "request to https://url/to/a/file.fbx failed, reason: getaddrinfo ENOTFOUND url url:443" 338 | }, 339 | { 340 | "name": "FetchError", 341 | "message": "request to https://url/to/another/file.ifc failed, reason: getaddrinfo ENOTFOUND url url:443" 342 | } 343 | ] 344 | }, 345 | "step": "cleanup", 346 | "isCompleted": true, 347 | "jobId": "140" 348 | }, 349 | ... 350 | } 351 | ``` 352 |
353 | 354 | ### Real-time conversion job's events 355 | 356 | `GET /api/conversions/{{code}}/events` 357 | 358 |
359 | This is the only way to know precisely when a conversion is completed (or failed). It also sends various events concerning the job state, which step is running, and various logging informations. 360 | 361 | **This is an Server-Sent Events (SSE) endpoint**, use the browser's native interface `EventSource` to access it, or a browser/node.js polyfill like the [eventsource](https://www.npmjs.com/package/eventsource) package on npm. 362 | 363 | #### Request 364 | 365 | ``` 366 | GET /api/conversions/{{conversionCode}}/events 367 | ``` 368 | 369 | Available query parameters: 370 | 371 | - `?sseType={events|data}`: whether to use data-only SSE messages or event+data. If using `data`, the event name will be in the `type` property of the data-only message. Defaults to `events`. 372 | - `?replay={true|false}`: whether to dump missed events between the job start and the connection to the SSE endpoint. Defaults to `true`. 373 | 374 | #### Unsuccessful response 375 | 376 | Two classic HTTP errors may occur before the SSE session starts (errors are formatted the same way there are on the rest of the API). 377 | 378 | - `404 Not Found` when the conversion code given in the URL is unknown ; 379 | - `410 Gone` when `?replay=false` and the conversion is already terminated (success or error). 380 | 381 | The client can handle this using `EventSource.onerror`. 382 | 383 | #### Successful response (completed conversion) 384 | 385 | :bulb: Note: The server will close the connection as soon as the conversion is terminated, error or success. As the SSE spec does not allow a server to close the connection gracefully, the client MUST listen for errors, and call EventSource#close() itself to avoid automatic reconnection. 386 | 387 | ```typescript 388 | ev.onerror = (event) => { 389 | if (event.eventPhase == EventSource.CLOSED) { 390 | ev.close(); 391 | } 392 | }; 393 | ``` 394 | 395 | :bulb: Note: The client will probably most interested in the `queue/conversion-ended` event that contains either an error or an URL to the resulting asset bundle. 396 | 397 | ``` 398 | HTTP/1.1 200 OK 399 | Content-Type:text/event-stream; charset=utf-8 400 | Cache-Control: no-cache 401 | 402 | : sse-start 403 | event: processor/step-change 404 | data: {"step":{"priority":10,"name":"Download remote assets","code":"download-assets"},"message":"Starting \"Download remote assets\""} 405 | 406 | event: queue/conversion-start 407 | data: {"message":"Conversion started"} 408 | 409 | event: processor/download-assets/start-download 410 | data: {"message":"Downloading \"https://i.ytimg.com/vi/qIIN64bUbsI/maxresdefault.jpg\""} 411 | 412 | event: processor/step-change 413 | data: {"step":{"priority":30,"name":"Execute AssetBundleCompiler to assemble the asset bundle","code":"exec-assetbundlecompiler"},"message":"Starting \"Execute AssetBundleCompiler to assemble the asset bundle\""} 414 | 415 | event: processor/exec-assetbundlecompiler/abcompiler-log 416 | data: {"message":"Preparing Unity project in /tmp/AssetBundleCompiler"} 417 | 418 | event: processor/exec-assetbundlecompiler/abcompiler-log 419 | data: {"message":"Copying assets to /tmp/AssetBundleCompiler/Assets/CopiedAssets"} 420 | 421 | event: processor/exec-assetbundlecompiler/abcompiler-log 422 | data: {"message":"Copying custom editor scripts to /tmp/AssetBundleCompiler/Assets/Editor/CopiedScripts"} 423 | 424 | event: processor/exec-assetbundlecompiler/abcompiler-log 425 | data: {"message":"Generating asset bundle in /tmp/AssetBundleCompiler/GeneratedAssetBundles"} 426 | 427 | event: processor/exec-assetbundlecompiler/abcompiler-log 428 | data: {"message":"Updating resource: maxresdefault.jpg"} 429 | 430 | event: processor/exec-assetbundlecompiler/abcompiler-log 431 | data: {"message":"Moving asset bundle to target destination"} 432 | 433 | event: processor/exec-assetbundlecompiler/abcompiler-log 434 | data: {"message":"Cleaning up the Unity project"} 435 | 436 | event: processor/exec-assetbundlecompiler/abcompiler-log 437 | data: {"message":"Done."} 438 | 439 | event: processor/step-change 440 | data: {"message":"Starting \"Upload the AssetBundle on Azure\"","step":{"code":"upload-bundle","name":"Upload the AssetBundle on Azure","priority":40}} 441 | 442 | event: processor/upload-bundle/upload-start 443 | data: {"message":"Uploading \"/tmp/chuck-exec-assetbundlecompiler-1495536189347/myassetbundle.ab2\" to Azure"} 444 | 445 | event: processor/upload-bundle/upload-end 446 | data: {"message":"Upload terminated with success","blobUrl":"https://mitmtest.blob.core.windows.net/assetbundles/myassetbundle.ab2","blobResult":{"container":"assetbundles","name":"myassetbundle.ab2","lastModified":"Tue, 23 May 2017 10:43:19 GMT","etag":"\"0x8D4A1C886E0CAC4\"","requestId":"54ef0e35-0001-0088-4db1-d3adde000000","contentSettings":{"contentMD5":"xRF+eIadlPTWCVp8Y8IkjA=="}}} 447 | 448 | event: processor/step-change 449 | data: {"message":"Performing cleanup for steps: download-assets, exec-assetbundlecompiler, upload-bundle (All steps have terminated successfuly)","step":{"code":"cleanup","name":"Conversion artifacts cleanup","priority":null}} 450 | 451 | event: queue/conversion-ended 452 | data: {"message":"Conversion terminated with success!","assetBundleUrl":"https://mitmtest.blob.core.windows.net/assetbundles/myassetbundle.ab2","error":null} 453 | 454 | : sse-keep-alive 455 | : sse-keep-alive 456 | ``` 457 |
458 | 459 | ## Extras tools 460 | 461 | ### Command Line Interface 462 | 463 | Chuck provides a CLI tool that is exported as the `bin` file in the package. In standalone mode, use it with `yarn cli -- --arguments`. 464 | 465 | Examples (using chuck as a package): 466 | 467 | - **`yarn chuck-cli help`** get available commands 468 | - **`yarn chuck-cli help `** displays informations about a command and available arguments 469 | - **`yarn chuck-cli api:generate-key`** generates an API key. Pass `--save` to save the generated key to the database. 470 | - **`yarn chuck-cli api:revoke-key `** revokes an API key stored in the database. 471 | 472 | ### Admin Web Interface 473 | 474 | A very, very simple administration interface is available under https://chuck/admin and uses HTTP Basic authentication, with the same credentials as Toureiro (see [Configuration](#configuration) section). 475 | 476 | It lets you create and revoke API keys as an alternative to the CLI. 477 | 478 | ### Toureiro 479 | 480 | Chuck embarks [Toureiro](https://github.com/Epharmix/Toureiro), which is... 481 | 482 | > A graphical monitoring interface for the distributed job queue bull built using express and react. Toureiro provides simple monitoring features as well as the ability to promote delayed jobs directly. 483 | 484 | Toureiro's interface can be found on https://chuck/toureiro, and is protected by the same HTTP Basic Auth and credentials used to login to the administration interface (see [Configuration](#configuration) section). 485 | 486 | ![Toureiro Web Interface](https://raw.githubusercontent.com/Epharmix/Toureiro/screenshots/public/screenshots/Job%20List.png) 487 | 488 | ## Development & Contributing 489 | 490 | The workflow is based on npm scripts: 491 | 492 | - `yarn watch`: starts the TypeScript compiler in watch mode ; 493 | - `yarn build`: compile TypeScript sources ; 494 | - `yarn start`: (run `watch` or `build` before!) starts the server and restarts it when the compiled files change (production or development, but for production you could use [pm2](http://pm2.keymetrics.io/) with `yarn standalone`) ; 495 | - `yarn cli`: shortcut to chuck's CLI (usage: `yarn cli -- command --arg1 --arg2`) ; 496 | - `yarn standalone`: starts the Express server without nodemon ; 497 | - `yarn lint`: checks code style on the TypeScript sources (recommended: install typescript and tslint extensions for your editor). 498 | 499 | So, basically, to start a development session, run in a terminal: 500 | 501 | ``` 502 | yarn install 503 | yarn watch 504 | ``` 505 | 506 | In a terminal aside of the first one, run: 507 | 508 | ``` 509 | yarn start 510 | ``` 511 | 512 | You can also create an `.env` file at the project root to override the default environment variables in `.env.defaults`. 513 | -------------------------------------------------------------------------------- /chuck.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mitmadness/chuck/b9a201f5b2382c94d2507dc0b15143feecc71f2c/chuck.png -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@mitm/chuck", 3 | "version": "0.0.0-development", 4 | "description": "chuck is a webservice that converts various 3D file-formats to Unity3D AssetBundles", 5 | "repository": "git@github.com:mitmadness/chuck.git", 6 | "author": "Morgan Touverey Quilling ", 7 | "license": "MIT", 8 | "main": "dist/entry/libchuck.js", 9 | "types": "dist/entry/libchuck.d.ts", 10 | "bin": { 11 | "chuck": "dist/entry/standalone.js", 12 | "chuck-cli": "dist/entry/cli.js" 13 | }, 14 | "scripts": { 15 | "start": "nodemon dist/entry/standalone.js", 16 | "watch": "yarn copy-pug && tsc -w", 17 | "build": "yarn lint && tsc && yarn copy-pug", 18 | "cli": "node dist/entry/cli.js", 19 | "standalone": "node dist/entry/standalone.js", 20 | "lint": "tslint \"src/**/*.ts\" \"stubs/**/*.ts\"", 21 | "semantic-release": "semantic-release pre && npm publish --access public && semantic-release post", 22 | "test": "node test/test.js", 23 | "copy-pug": "cpy \"admin/*.pug\" \"../dist/\" --cwd=src --parents" 24 | }, 25 | "devDependencies": { 26 | "@types/body-parser": "^1.16.3", 27 | "@types/boom": "^4.3.2", 28 | "@types/bull": "^2.1.0", 29 | "@types/cors": "^2.8.1", 30 | "@types/dotenv": "^4.0.0", 31 | "@types/express": "^4.0.35", 32 | "@types/mongoose": "^4.7.11", 33 | "@types/morgan": "^1.7.32", 34 | "@types/node-fetch": "^1.6.7", 35 | "@types/pify": "^0.0.28", 36 | "@types/raven": "^1.2.2", 37 | "@types/source-map-support": "^0.2.28", 38 | "@types/uuid": "^2.0.29", 39 | "@types/winston": "^2.3.1", 40 | "@types/yargs": "^6.6.0", 41 | "nodemon": "^1.11.0", 42 | "semantic-release": "^6.3.6", 43 | "tslint": "^5.1.0", 44 | "typescript": "^2.2.2" 45 | }, 46 | "dependencies": { 47 | "@mitm/assetbundlecompiler": "^1.4.0", 48 | "@toverux/expresse": "^2.1.0", 49 | "@types/sanitize-filename": "^1.1.28", 50 | "azure-storage": "^2.1.0", 51 | "body-parser": "^1.17.1", 52 | "boom": "^5.1.0", 53 | "bull": "^2.2.6", 54 | "compose-middleware": "^2.2.0", 55 | "cors": "^2.8.3", 56 | "cpy-cli": "^1.0.1", 57 | "dotenv": "^4.0.0", 58 | "dotenv-extended": "^2.0.0", 59 | "express": "^4.15.2", 60 | "express-basic-auth": "^1.0.2", 61 | "mongoose": "^5.13.20", 62 | "morgan": "^1.8.1", 63 | "node-fetch": "^1.6.3", 64 | "pify": "^3.0.0", 65 | "pug": "^2.0.0-rc.1", 66 | "raven": "^2.1.0", 67 | "sanitize-filename": "^1.6.1", 68 | "source-map-support": "^0.4.15", 69 | "toureiro": "^0.2.13", 70 | "uuid": "^3.0.1", 71 | "winston": "^2.3.1", 72 | "yargs": "^8.0.1" 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /postman/chuck.postman_collection.json: -------------------------------------------------------------------------------- 1 | { 2 | "variables": [], 3 | "info": { 4 | "name": "Chuck", 5 | "_postman_id": "ae2ec073-82b3-a253-8ce8-84bbe667348c", 6 | "description": "", 7 | "schema": "https://schema.getpostman.com/json/collection/v2.0.0/collection.json" 8 | }, 9 | "item": [ 10 | { 11 | "name": "Push conversion request", 12 | "event": [ 13 | { 14 | "listen": "test", 15 | "script": { 16 | "type": "text/javascript", 17 | "exec": [ 18 | "if (responseCode.code === 202) {", 19 | " var conversion = JSON.parse(responseBody);", 20 | " postman.setGlobalVariable(\"lastConversionId\", conversion.code);", 21 | "}" 22 | ] 23 | } 24 | } 25 | ], 26 | "request": { 27 | "url": "{{baseUrl}}/api/conversions", 28 | "method": "POST", 29 | "header": [ 30 | { 31 | "key": "Content-Type", 32 | "value": "application/json", 33 | "description": "" 34 | }, 35 | { 36 | "key": "Authorization", 37 | "value": "Bearer {{apiKey}}", 38 | "description": "" 39 | } 40 | ], 41 | "body": { 42 | "mode": "raw", 43 | "raw": "{\n \"assetBundleName\": \"myassetbundle.ab2\",\n \"compilerOptions\": {\n \"targeting\": \"webgl\"\n },\n \"assetUrls\": [\n \"https://file/to/asset.fbx\",\n \"https://file/to/texture.png\"\n ],\n \"azure\": {\n \"host\": \"{{azureHost}}\",\n \"container\": \"{{azureContainer}}\",\n \"sharedAccessSignatureToken\": \"{{azureSharedAccessSignatureToken}}\"\n }\n}" 44 | }, 45 | "description": "Crée une nouvelle demande de conversion, placée dans la queue de conversion." 46 | }, 47 | "response": [] 48 | }, 49 | { 50 | "name": "Get conversion info", 51 | "request": { 52 | "url": "{{baseUrl}}/api/conversions/{{lastConversionId}}", 53 | "method": "GET", 54 | "header": [], 55 | "body": {}, 56 | "description": "Récupère les informations sur une conversion, avec le statut du job dans la queue." 57 | }, 58 | "response": [] 59 | } 60 | ] 61 | } -------------------------------------------------------------------------------- /postman/devel.postman_environment.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "631dbadb-2860-21f6-9563-f4f63a883827", 3 | "name": "Devel", 4 | "values": [ 5 | { 6 | "enabled": true, 7 | "key": "baseUrl", 8 | "value": "http://localhost:3001", 9 | "type": "text" 10 | }, 11 | { 12 | "enabled": true, 13 | "key": "apiKey", 14 | "value": "a2b48c5f-b7b8-4bf6-a34a-6003d0ea908b", 15 | "type": "text" 16 | }, 17 | { 18 | "enabled": true, 19 | "key": "azureHost", 20 | "value": "secret", 21 | "type": "text" 22 | }, 23 | { 24 | "enabled": true, 25 | "key": "azureContainer", 26 | "value": "secret", 27 | "type": "text" 28 | }, 29 | { 30 | "enabled": true, 31 | "key": "azureSharedAccessSignatureToken", 32 | "value": "secret", 33 | "type": "text" 34 | } 35 | ], 36 | "timestamp": 1495197669447, 37 | "_postman_variable_scope": "environment", 38 | "_postman_exported_at": "2017-05-19T12:47:33.254Z", 39 | "_postman_exported_using": "Postman/4.10.7" 40 | } -------------------------------------------------------------------------------- /src/admin/admin.pug: -------------------------------------------------------------------------------- 1 | html 2 | head 3 | title= 'Chuck Webservice Admin' 4 | 5 | body 6 | form(action='' method='post') 7 | input(type='submit' name='generateKey' value='Generate a new API key') 8 | table 9 | caption List of API Keys 10 | 11 | each val in keys 12 | tr 13 | th= val.key 14 | th 15 | form(action="admin/delete/"+val.key method='post') 16 | input(type='submit' name='deleteKey' value='Delete') 17 | 18 | a(href='/toureiro') Job list 19 | -------------------------------------------------------------------------------- /src/admin/index.ts: -------------------------------------------------------------------------------- 1 | import * as express from 'express'; 2 | import { ApiKey } from '../models/api_key'; 3 | import { wrapAsync} from '../express_utils'; 4 | 5 | const router: express.Router = express.Router(); 6 | 7 | router.get('/', wrapAsync(async (req, res, next) => { 8 | const keys = await ApiKey.find({}); 9 | 10 | res.render('admin/admin', { keys }); 11 | 12 | next(); 13 | })); 14 | 15 | router.post('/', wrapAsync(async (req, res, next) => { 16 | await ApiKey.create({}); 17 | 18 | res.redirect('/admin'); 19 | 20 | next(); 21 | })); 22 | 23 | router.post('/delete/:key', wrapAsync(async (req, res, next) => { 24 | await ApiKey.findOneAndRemove({ key: req.params.key }); 25 | 26 | res.redirect('/admin'); 27 | 28 | next(); 29 | })); 30 | 31 | export default router; 32 | -------------------------------------------------------------------------------- /src/api/api_keys_cache.ts: -------------------------------------------------------------------------------- 1 | import { ApiKey } from '../models'; 2 | 3 | const apiKeysCache: string[] = []; 4 | 5 | export async function isKeyValid(key: string): Promise { 6 | if (apiKeysCache.includes(key)) { 7 | return true; 8 | } 9 | 10 | const apiKeyDocument = await ApiKey.findOne({ key }); 11 | 12 | if (apiKeyDocument) { 13 | apiKeysCache.push(key); 14 | return true; 15 | } 16 | 17 | return false; 18 | } 19 | -------------------------------------------------------------------------------- /src/api/conversions_api.ts: -------------------------------------------------------------------------------- 1 | import { notFound } from 'boom'; 2 | import * as express from 'express'; 3 | import { Conversion, safeData as safeConversionData } from '../models/conversion'; 4 | import converterQueue from '../converter/queue'; 5 | import { wrapAsync, safeOutData } from '../express_utils'; 6 | import conversionsEventsMiddleware from './conversions_api_events'; 7 | import { hasValidApiKey } from './middlewares'; 8 | 9 | const router: express.Router = express.Router(); 10 | 11 | /** 12 | * Create a new conversion request. 13 | * Creates a Conversion document, and pushes it onto the queue - the job will start asap. 14 | */ 15 | router.post('/', hasValidApiKey(), wrapAsync(async (req, res, next) => { 16 | const conversionData = safeConversionData(req.body); 17 | const conversion = await Conversion.create(conversionData); 18 | 19 | const job = await converterQueue.add(conversion); 20 | 21 | conversion.conversion.jobId = job.jobId; 22 | await conversion.save(); 23 | 24 | res.status(202).json(safeOutData(conversion)); 25 | 26 | next(); 27 | })); 28 | 29 | /** 30 | * Retrieves a conversion request by it's code identifier. 31 | */ 32 | router.get('/:code', wrapAsync(async (req, res, next) => { 33 | const conversion = await Conversion.findOne({ code: req.params.code }, { 'conversion.logs': false }); 34 | if (!conversion) { 35 | throw notFound(); 36 | } 37 | 38 | res.status(200).json(safeOutData(conversion)); 39 | 40 | next(); 41 | })); 42 | 43 | /** 44 | * A Server-Sent Events endpoint to get realtime events about the conversion. 45 | */ 46 | router.get('/:code/events', conversionsEventsMiddleware); 47 | 48 | export default router; 49 | -------------------------------------------------------------------------------- /src/api/conversions_api_events.ts: -------------------------------------------------------------------------------- 1 | import { sse, ISseResponse } from '@toverux/expresse'; 2 | import { notFound, resourceGone } from 'boom'; 3 | import { compose } from 'compose-middleware'; 4 | import { Request, Response, NextFunction } from 'express'; 5 | import converterQueue from '../converter/queue'; 6 | import { IProgressReportJob } from '../converter/job'; 7 | import { IEvent, isQueueConversionEndedEvent } from '../converter/job_events'; 8 | import { Conversion, IConversionModel } from '../models/conversion'; 9 | import { wrapAsync } from '../express_utils'; 10 | 11 | interface InitSseRequest extends Request { 12 | isReplay: boolean; 13 | conversion: IConversionModel; 14 | } 15 | 16 | export default compose( 17 | //=> 1. Loads the conversion, checks request validity 18 | wrapAsync(loadConversion), 19 | //=> 2. Init SSE session 20 | sse(), 21 | //=> 3. Send conversion events (replay or realtime or both) 22 | wrapAsync(conversionSseEvents), 23 | //=> 4. Close connection 24 | terminateSseSession 25 | ); 26 | 27 | async function loadConversion(req: InitSseRequest, res: Response, next: NextFunction): Promise { 28 | req.isReplay = req.query.replay !== 'false'; 29 | req.conversion = await Conversion.findOne({ code: req.params.code }); 30 | 31 | //=> Check that we can continue 32 | if (!req.conversion) { 33 | throw notFound(); 34 | } else if (!req.isReplay && req.conversion.conversion.isCompleted) { 35 | throw resourceGone('This conversion is terminated'); 36 | } 37 | 38 | next(); 39 | } 40 | 41 | async function conversionSseEvents(req: InitSseRequest, res: ISseResponse, next: NextFunction): Promise { 42 | // @todo Unsafe typecast -- a Bull queue is an EventEmitter, but typings are not complete 43 | const emitterQueue = (converterQueue as any) as NodeJS.EventEmitter; 44 | const sse = sseSend.bind(null, req.query.sseType); 45 | 46 | //=> Replay mode: dump all progress records 47 | if (req.isReplay && req.conversion.conversion.logs) { 48 | req.conversion.conversion.logs.forEach(event => sse(res, event)); 49 | 50 | // optim: conversion is terminated, don't watch for subsequent events 51 | if (req.conversion.conversion.isCompleted) return next(); 52 | } 53 | 54 | //=> Listen queue for progress events, filter, and send if the jobId matches 55 | emitterQueue.on('progress', handleProgress); 56 | 57 | function handleProgress(job: IProgressReportJob, event: IEvent): void { 58 | if (job.id != req.conversion.conversion.jobId) return; 59 | sse(res, event); 60 | 61 | if (isQueueConversionEndedEvent(event)) { 62 | emitterQueue.removeListener('progress', handleProgress); 63 | next(); 64 | } 65 | } 66 | } 67 | 68 | function terminateSseSession(req: Request, res: Response, next: NextFunction): void { 69 | res.end(); // avoid hanging for nothing, would be too easy to DDoS it 70 | next(); 71 | } 72 | 73 | function sseSend(type: 'events'|'data', res: ISseResponse, event: IEvent): void { 74 | if (type == 'data') { 75 | res.sse.data(event); 76 | } else { 77 | const eventPayload = { ...event }; 78 | delete eventPayload.type; 79 | 80 | res.sse.event(event.type, eventPayload); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/api/index.ts: -------------------------------------------------------------------------------- 1 | import * as express from 'express'; 2 | import * as raven from 'raven'; 3 | import conversionsApi from './conversions_api'; 4 | import { fatalErrorsHandler, recoverableErrorsHandler } from './middlewares'; 5 | 6 | const router: express.Router = express.Router(); 7 | 8 | //=> API endpoint roots 9 | router.use('/conversions', conversionsApi); 10 | 11 | //=> Install error handlers 12 | router.use(recoverableErrorsHandler()); 13 | router.use(raven.errorHandler()); 14 | router.use(fatalErrorsHandler()); 15 | 16 | export default router; 17 | -------------------------------------------------------------------------------- /src/api/middlewares.ts: -------------------------------------------------------------------------------- 1 | import { BoomError, unauthorized } from 'boom'; 2 | import { ErrorRequestHandler, Handler, NextFunction, Request, Response } from 'express'; 3 | import logger from '../logger'; 4 | import { wrapAsync } from '../express_utils'; 5 | import { safeErrorSerialize } from '../safe_error_serialize'; 6 | import { isKeyValid } from './api_keys_cache'; 7 | 8 | interface ISentryResponse extends Response { 9 | sentry?: string; 10 | } 11 | 12 | export function hasValidApiKey(): Handler { 13 | return wrapAsync(async (req: Request, res: Response, next: NextFunction): Promise => { 14 | const authHeader = req.header('Authorization'); 15 | if (!authHeader || !authHeader.startsWith('Bearer ')) { 16 | throw unauthorized('The API key must be provided in the Authorization header, with scheme Bearer'); 17 | } 18 | 19 | const apiKey = authHeader.split(' ')[1]; 20 | 21 | if (!await isKeyValid(apiKey)) { 22 | throw unauthorized(`The API key ${apiKey} does not seem to exist`); 23 | } 24 | 25 | next(); 26 | }); 27 | } 28 | 29 | export function recoverableErrorsHandler(): ErrorRequestHandler { 30 | return (err: any, req: Request, res: Response, next: NextFunction): void => { 31 | //=> Headers already sent, let Express handle the thing 32 | if (res.headersSent) { 33 | return void next(err); 34 | } 35 | 36 | //=> Mongoose validation errors 37 | if (err.name === 'ValidationError') { 38 | return void res.status(400).json(err); 39 | } 40 | 41 | //=> Boom's HTTP errors 42 | const boomError = err as BoomError; 43 | if (boomError.isBoom) { 44 | return void res.status(boomError.output.statusCode).json({ 45 | name: boomError.output.payload.error, 46 | message: boomError.message 47 | }); 48 | } 49 | 50 | //=> We can't recognize this error, pass to the next error handler. 51 | next(err); 52 | }; 53 | } 54 | 55 | export function fatalErrorsHandler(): ErrorRequestHandler { 56 | return (err: any, req: Request, res: ISentryResponse, next: NextFunction): void => { 57 | //=> We don't know what this error is, return a 500 error 58 | res.status(500).json(safeErrorSerialize(err, res.sentry)); 59 | 60 | //=> Log it because that's, well, unexpected 61 | logger.error(err); 62 | 63 | next(); 64 | }; 65 | } 66 | -------------------------------------------------------------------------------- /src/cli/api/generate_key.ts: -------------------------------------------------------------------------------- 1 | import { CommandModule } from 'yargs'; 2 | import config from '../../config'; 3 | import logger from '../../logger'; 4 | import { connectDatabase, disconnectFromDatabase } from '../../mongoose'; 5 | import { ApiKey } from '../../models'; 6 | 7 | const command: CommandModule = { 8 | command: 'api:generate-key', 9 | describe: 'Generate an API key', 10 | builder: { 11 | save: { 12 | alias: 'S', 13 | type: 'boolean', 14 | desc: 'Save the API key to the database for direct use', 15 | default: false 16 | } 17 | }, 18 | handler 19 | }; 20 | 21 | interface IArgs { 22 | save: boolean; 23 | } 24 | 25 | async function handler(args: IArgs): Promise { 26 | const key = new ApiKey(); 27 | 28 | logger.info(`Generated key: ${key.key}`); 29 | 30 | if (args.save) { 31 | await connectDatabase(config.mongoUrl); 32 | await key.save(); 33 | await disconnectFromDatabase(); 34 | 35 | logger.info('Key saved to database!'); 36 | } 37 | } 38 | 39 | export default command; 40 | -------------------------------------------------------------------------------- /src/cli/api/index.ts: -------------------------------------------------------------------------------- 1 | import { CommandModule } from 'yargs'; 2 | import generateKeyCommand from './generate_key'; 3 | import revokeKeyCommand from './revoke_key'; 4 | 5 | export default [generateKeyCommand, revokeKeyCommand] as CommandModule[]; 6 | -------------------------------------------------------------------------------- /src/cli/api/revoke_key.ts: -------------------------------------------------------------------------------- 1 | import { CommandModule } from 'yargs'; 2 | import config from '../../config'; 3 | import logger from '../../logger'; 4 | import { connectDatabase, disconnectFromDatabase } from '../../mongoose'; 5 | import { ApiKey } from '../../models'; 6 | 7 | const command: CommandModule = { 8 | command: 'api:revoke-key ', 9 | describe: 'Revoke an API key', 10 | handler 11 | }; 12 | 13 | interface IArgs { 14 | key: string; 15 | } 16 | 17 | async function handler(args: IArgs): Promise { 18 | await connectDatabase(config.mongoUrl); 19 | 20 | await ApiKey.remove({ key: args.key }); 21 | 22 | logger.info(`The key ${args.key} has been deleted or was nonexistent.`); 23 | 24 | await disconnectFromDatabase(); 25 | } 26 | 27 | export default command; 28 | -------------------------------------------------------------------------------- /src/cli/index.ts: -------------------------------------------------------------------------------- 1 | import * as apiCommands from './api'; 2 | 3 | export default [apiCommands]; 4 | -------------------------------------------------------------------------------- /src/config.ts: -------------------------------------------------------------------------------- 1 | import * as dotenvx from 'dotenv-extended'; 2 | import * as path from 'path'; 3 | 4 | export type EnvType = 'development'|'production'; 5 | 6 | export interface IChuckConfig { 7 | /** 8 | * The release number of chuck, as specified in the package.json. 9 | * This will only have a real value in the npm distribution of chuck -- value is otherwise "0.0.0-development". 10 | */ 11 | release: string; 12 | 13 | /** 14 | * aka NODE_ENV. Configures the mode (`development` or `production`) in which the server is running. 15 | * - development: permissive CORS rules are set on the API 16 | * - production: timestamps in log messages and more verbose HTTP logs 17 | * 18 | * @default process.env.NODE_ENV || 'development' 19 | */ 20 | env: EnvType; 21 | 22 | /** 23 | * Minimum log level (npm standard log levels are used). 24 | * 25 | * @see https://github.com/winstonjs/winston#logging-levels 26 | * @default 'verbose' 27 | */ 28 | logLevel: string; 29 | 30 | /** 31 | * Chuck HTTP server port. 32 | * 33 | * @default 3001 34 | */ 35 | serverPort: number; 36 | 37 | /** 38 | * Connection string to a MongoDB database. 39 | * 40 | * @default 'mongodb://localhost/chuck' 41 | */ 42 | mongoUrl: string; 43 | 44 | /** 45 | * DSN for Sentry error reporting. 46 | */ 47 | ravenDsn: string; 48 | 49 | /** 50 | * Redis connection informations. 51 | * 52 | * @default { host: 'localhost', port: 6379, db: 0 } 53 | */ 54 | redis: { host: string; port: number; db: number }; 55 | 56 | /** 57 | * Admin Web UIs configuration. Used by the admin interface and Toureiro. 58 | * 59 | * @default { enable: false, user: 'admin', password: 'admin' } 60 | */ 61 | adminWebUis: { enable: boolean, user: string; password: string; }; 62 | 63 | /** 64 | * Unity Editor path (if not installed in the standard path). 65 | * 66 | * @see https://github.com/mitmadness/AssetBundleCompiler#changing-unitys-executable-path 67 | * @default undefined (auto) 68 | */ 69 | unityPath: string|undefined; 70 | 71 | /** 72 | * Azure configuration. 73 | * 74 | * @default { enableEmu: false } 75 | */ 76 | azure: { enableEmu: boolean; }; 77 | 78 | /** 79 | * An array of module names. 80 | * Those modules will be loaded dynamically as step plugins. 81 | */ 82 | stepModulePlugins: string[]; 83 | } 84 | 85 | //=> Load default environment variables with dotenv-extended 86 | dotenvx.load({ 87 | defaults: path.resolve(`${__dirname}/../.env.defaults`) 88 | }); 89 | 90 | //=> Determine release number 91 | // tslint:disable-next-line:no-var-requires 92 | const release = require('../package.json').version; 93 | 94 | //=> Hydrate config with the environment variables 95 | const config: IChuckConfig = { 96 | release, 97 | env: process.env.NODE_ENV || process.env.CHUCK_ENV, 98 | logLevel: process.env.CHUCK_LOGLEVEL, 99 | serverPort: parseInt(process.env.CHUCK_SERVERPORT, 10), 100 | mongoUrl: process.env.CHUCK_MONGOURL, 101 | ravenDsn: process.env.CHUCK_RAVENDSN, 102 | redis: { 103 | host: process.env.CHUCK_REDIS_HOST, 104 | port: parseInt(process.env.CHUCK_REDIS_PORT, 10), 105 | db: parseInt(process.env.CHUCK_REDIS_DB, 10) 106 | }, 107 | adminWebUis: { 108 | enable: process.env.CHUCK_ADMINWEBUIS_ENABLE === 'true', 109 | user: process.env.CHUCK_ADMINWEBUIS_USER, 110 | password: process.env.CHUCK_ADMINWEBUIS_PASSWORD 111 | }, 112 | unityPath: process.env.CHUCK_UNITYPATH, 113 | azure: { 114 | enableEmu: process.env.CHUCK_AZURE_ENABLEEMU === 'true' 115 | }, 116 | stepModulePlugins: process.env.CHUCK_STEPMODULEPLUGINS ? process.env.CHUCK_STEPMODULEPLUGINS.split(',') : [] 117 | }; 118 | 119 | export default config; 120 | -------------------------------------------------------------------------------- /src/converter/job.ts: -------------------------------------------------------------------------------- 1 | import { DocumentQuery } from 'mongoose'; 2 | import { Job } from 'bull'; 3 | import { Conversion } from '../models'; 4 | import { IConversion } from '../models/IConversion'; 5 | import { IConversionModel } from '../models/conversion'; 6 | import { IEvent } from './job_events'; 7 | 8 | /** 9 | * This interface is the common subset of a normal Job object (w/ methods) and a job in a progress report. 10 | */ 11 | export interface IConversionDataJob { 12 | data: IConversion; 13 | } 14 | 15 | /** 16 | * This is a more-typed specialization of Bull's Job interface. 17 | * This is a normal Job object with methods like progress(). 18 | */ 19 | export interface IConversionJob extends Job { 20 | data: IConversion; 21 | progress(event: T): Promise; 22 | } 23 | 24 | /** 25 | * This is an imperfect subset of a normal Job object. 26 | * It has no methods (plain object) and has "id" instead of "jobId". 27 | * It is received is a progress event handler, for example. 28 | */ 29 | export interface IProgressReportJob { 30 | id: string; 31 | data: IConversion; 32 | } 33 | 34 | export function updateConversion(job: IConversionDataJob, update: object): DocumentQuery { 35 | return Conversion.findOneAndUpdate({ code: job.data.code }, update); 36 | } 37 | -------------------------------------------------------------------------------- /src/converter/job_events.ts: -------------------------------------------------------------------------------- 1 | import { safeErrorSerialize } from '../safe_error_serialize'; 2 | import { IStepDescription } from './steps/step'; 3 | 4 | export interface IEvent { 5 | type: string; 6 | message: string; 7 | } 8 | 9 | // QueueConversionStartEvent 10 | // ------------------------- 11 | 12 | export function queueConversionStartEvent(message: string): IEvent { 13 | return { type: 'queue/conversion-start', message }; 14 | } 15 | 16 | export function isQueueConversionStartEvent(event: IEvent): event is IEvent { 17 | return event.type == 'queue/conversion-start'; 18 | } 19 | 20 | // QueueConversionEndedEvent 21 | // ------------------------- 22 | 23 | export interface IQueueConversionEndedEvent extends IEvent { 24 | assetBundleUrl: string|null; 25 | error: any|null; 26 | } 27 | 28 | export function queueConversionEndedEvent( 29 | message: string, 30 | assetBundleUrl: string|null, 31 | error: any = null 32 | ): IQueueConversionEndedEvent { 33 | if (error) { 34 | error = safeErrorSerialize(error); 35 | } 36 | 37 | return { type: 'queue/conversion-ended', message, assetBundleUrl, error }; 38 | } 39 | 40 | export function isQueueConversionEndedEvent(event: IEvent): event is IQueueConversionEndedEvent { 41 | return event.type == 'queue/conversion-ended'; 42 | } 43 | 44 | // ProcessorStepChangeEvent 45 | // ------------------------ 46 | 47 | export interface IProcessorStepChangeEvent extends IEvent { 48 | step: IStepDescription; 49 | } 50 | 51 | export function processorStepChangeEvent(message: string, step: IStepDescription): IProcessorStepChangeEvent { 52 | return { type: 'processor/step-change', message, step }; 53 | } 54 | 55 | export function isProcessorStepChangeEvent(event: IEvent): event is IProcessorStepChangeEvent { 56 | return event.type == 'processor/step-change'; 57 | } 58 | 59 | // ProcessorCleanupErrorEvent 60 | // -------------------------- 61 | 62 | export interface IProcessorCleanupErrorEvent extends IEvent { 63 | step: IStepDescription; 64 | error: any; 65 | } 66 | 67 | export function processorCleanupErrorEvent( 68 | message: string, 69 | step: IStepDescription, 70 | error: any 71 | ): IProcessorCleanupErrorEvent { 72 | return { type: 'processor/cleanup-error', message, step, error }; 73 | } 74 | 75 | export function isProcessorCleanupErrorEvent(event: IEvent): event is IProcessorCleanupErrorEvent { 76 | return event.type == 'processor/cleanup-error'; 77 | } 78 | 79 | // ProcessorStepProgressEvent 80 | // -------------------------- 81 | 82 | export interface IProcessorStepProgressEvent extends IEvent { 83 | [customKey: string]: any; 84 | } 85 | 86 | export function processorStepProgressEvent( 87 | stepCode: string, 88 | type: string, 89 | message: string, 90 | data?: any 91 | ): IProcessorStepProgressEvent { 92 | return { type: `processor/${stepCode}/${type}`, message, ...data }; 93 | } 94 | -------------------------------------------------------------------------------- /src/converter/plugin_steps.ts: -------------------------------------------------------------------------------- 1 | import { IStepModule } from './steps/step'; 2 | 3 | const pluginSteps: IStepModule[] = []; 4 | 5 | export function register(step: IStepModule): void { 6 | pluginSteps.push(step); 7 | } 8 | 9 | export function all(): IStepModule[] { 10 | return pluginSteps; 11 | } 12 | -------------------------------------------------------------------------------- /src/converter/queue.ts: -------------------------------------------------------------------------------- 1 | import * as queue from 'bull'; 2 | import config from '../config'; 3 | import coreSteps from './steps'; 4 | import * as pluginSteps from './plugin_steps'; 5 | import { processor } from './queue_processor'; 6 | import * as handlers from './queue_event_handlers'; 7 | 8 | //=> Get plug-in steps and merge with the core steps 9 | const steps = coreSteps.concat(pluginSteps.all()); 10 | 11 | //=> Sort all steps with their declared priority 12 | const sortedSteps = steps.sort((a, b) => a.describe().priority - b.describe().priority); 13 | 14 | //=> Initialize the Bull queue 15 | const conversionsQueue = queue('chuck-conversion-queue', config.redis.port, config.redis.host); 16 | 17 | //=> Initialize the job processor for the conversions queue 18 | conversionsQueue.process(processor.bind(null, sortedSteps)); 19 | 20 | //=> Queue-related events 21 | conversionsQueue.on('ready', handlers.onQueueReady); 22 | conversionsQueue.on('paused', handlers.onQueuePaused); 23 | conversionsQueue.on('resumed', handlers.onQueueResumed); 24 | conversionsQueue.on('error', handlers.onQueueError); 25 | conversionsQueue.on('cleaned', handlers.onQueueCleaned); 26 | 27 | //=> Job-related events 28 | conversionsQueue.on('progress', handlers.onJobProgress); 29 | conversionsQueue.on('active', handlers.onJobActive); 30 | conversionsQueue.on('completed', handlers.onJobCompleted); 31 | conversionsQueue.on('failed', handlers.onJobFailed); 32 | 33 | export default conversionsQueue; 34 | -------------------------------------------------------------------------------- /src/converter/queue_event_handlers.ts: -------------------------------------------------------------------------------- 1 | import * as raven from 'raven'; 2 | import config from '../config'; 3 | import logger from '../logger'; 4 | import { safeErrorSerialize } from '../safe_error_serialize'; 5 | import { IConversionJob, IProgressReportJob, updateConversion } from './job'; 6 | import { IEvent, isProcessorStepChangeEvent, queueConversionEndedEvent, queueConversionStartEvent } from './job_events'; 7 | 8 | // GLOBAL QUEUE EVENTS 9 | //-------------------- 10 | 11 | export function onQueueReady(): void { 12 | logger.info('convqueue: Now ready'); 13 | } 14 | 15 | export function onQueuePaused(): void { 16 | logger.info('convqueue: Now paused'); 17 | } 18 | 19 | export function onQueueResumed(): void { 20 | logger.info('convqueue: Now resumed'); 21 | } 22 | 23 | export function onQueueError(err: any): void { 24 | logger.error('convqueue: The queue encountered an error!', err); 25 | } 26 | 27 | export function onQueueCleaned(jobs: IConversionJob[]): void { 28 | const jobIdsStr = jobs.map(job => job.jobId).join(', '); 29 | logger.info(`convqueue: ${jobs.length} terminated jobs have been deleted: ${jobIdsStr}`); 30 | } 31 | 32 | // JOB-RELATED QUEUE EVENTS 33 | //------------------------- 34 | 35 | export async function onJobProgress(job: IProgressReportJob, progress: IEvent): Promise { 36 | logger.verbose(`convqueue: job #${job.id} [${progress.type}] ${progress.message}`); 37 | 38 | let updateQuery = { 39 | $push: { 'conversion.logs': progress } 40 | }; 41 | 42 | isProcessorStepChangeEvent(progress) && (updateQuery = { 43 | ...updateQuery, 44 | $set: { 'conversion.step': progress.step.code } 45 | }); 46 | 47 | await updateConversion(job, updateQuery); 48 | } 49 | 50 | export async function onJobActive(job: IConversionJob): Promise { 51 | logger.verbose(`convqueue: job #${job.jobId} has started`, job.data); 52 | 53 | await job.progress(queueConversionStartEvent('Conversion started')); 54 | } 55 | 56 | export async function onJobCompleted(job: IConversionJob, assetBundleUrl: string): Promise { 57 | logger.verbose(`convqueue: job #${job.jobId} is completed`, job.data); 58 | 59 | await job.progress(queueConversionEndedEvent('Conversion terminated with success!', assetBundleUrl)); 60 | 61 | await updateConversion(job, { 62 | $set: { 63 | 'conversion.isCompleted': true, 64 | 'conversion.step': null, 65 | 'conversion.assetBundleUrl': assetBundleUrl 66 | } 67 | }); 68 | } 69 | 70 | export async function onJobFailed(job: IConversionJob, error: any): Promise { 71 | //=> Log & report on Sentry 72 | logger.error(`convqueue: job #${job.jobId} has failed!`, error); 73 | 74 | if (config.ravenDsn) { 75 | raven.captureException(error); 76 | } 77 | 78 | //=> Report in job's progress log 79 | const progressTask = job.progress(queueConversionEndedEvent( 80 | 'Conversion failed, an error occured!', null, error 81 | )); 82 | 83 | //=> Update the conversion document infos about progress. 84 | // We don't update conversion.step to let the client know where the fail occured. 85 | const updateTask = updateConversion(job, { 86 | $set: { 87 | 'conversion.isCompleted': true, 88 | 'conversion.error': safeErrorSerialize(error) 89 | } 90 | }); 91 | 92 | await Promise.all([progressTask, updateTask]); 93 | } 94 | -------------------------------------------------------------------------------- /src/converter/queue_processor.ts: -------------------------------------------------------------------------------- 1 | import { IConversionJob } from './job'; 2 | import { IStepModule, IStepsContext } from './steps/step'; 3 | import { processorCleanupErrorEvent, processorStepChangeEvent, processorStepProgressEvent } from './job_events'; 4 | 5 | /** 6 | * The processor is the function that is passed to queue.process(). 7 | * It iterates over conversion steps, handling context and errors (cleanup, etc). 8 | */ 9 | export async function processor(steps: IStepModule[], job: IConversionJob): Promise { 10 | //=> Initialize context 11 | const stepsStack: IStepModule[] = []; 12 | const context: IStepsContext = { assetsPaths: [] }; 13 | const cleanup = stepsCleanupProcessor.bind(null, stepsStack, job, context); 14 | 15 | //=> Execute all steps in order, sequentially 16 | for (const step of steps) { 17 | //=> Pass or record passage 18 | if (!step.shouldProcess(job.data, context)) continue; 19 | stepsStack.push(step); 20 | 21 | const stepInfo = step.describe(); 22 | 23 | //=> Signal progress 24 | await job.progress(processorStepChangeEvent(`Starting "${stepInfo.name}"`, stepInfo)); 25 | 26 | //=> Function for a step to signal its progress 27 | function stepSignalProgress(type: string, message: string, data: any = {}) { 28 | return job.progress(processorStepProgressEvent(stepInfo.code, type, message, data)); 29 | } 30 | 31 | //=> Execute the step 32 | try { 33 | await step.process(job.data, context, stepSignalProgress); 34 | } catch (err) { 35 | await cleanup(`An error occured while running step ${stepInfo.code}`); 36 | throw err; 37 | } 38 | } 39 | 40 | //=> Perform cleanup 41 | await cleanup('All steps have terminated successfuly'); 42 | 43 | return context.assetBundleUrl as string; 44 | } 45 | 46 | /** 47 | * The cleanup processor handles conversion artifacts cleanup. 48 | * It is called by the processor after a successful conversion, or just after an error, before re-throw. 49 | * It calls each step's cleanup() method (if any). 50 | */ 51 | export async function stepsCleanupProcessor( 52 | stepsStack: IStepModule[], 53 | job: IConversionJob, 54 | context: IStepsContext, 55 | reason: string 56 | ): Promise { 57 | //=> Signal cleanup 58 | const stepNames = stepsStack.map(step => step.describe().code).join(', '); 59 | 60 | await job.progress(processorStepChangeEvent( 61 | `Performing cleanup for steps: ${stepNames} (${reason})`, 62 | { code: 'cleanup', name: 'Conversion artifacts cleanup', priority: Infinity } 63 | )); 64 | 65 | //=> Call each step's cleanup() function 66 | while (stepsStack.length) { 67 | const step = stepsStack.pop() as IStepModule; 68 | if (!step.cleanup) continue; // implementing cleanup() isn't mandatory 69 | 70 | try { 71 | await step.cleanup(context); 72 | } catch (error) { 73 | const stepInfo = step.describe(); 74 | 75 | await job.progress(processorCleanupErrorEvent( 76 | `Error during calling cleanup() of step ${stepInfo.code}, ignoring`, 77 | stepInfo, error 78 | )); 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/converter/steps/01_download_assets.ts: -------------------------------------------------------------------------------- 1 | import fetch from 'node-fetch'; 2 | import * as fs from 'fs'; 3 | import * as os from 'os'; 4 | import * as path from 'path'; 5 | import * as pify from 'pify'; 6 | import { IConversion } from '../../models/IConversion'; 7 | import { IStepDescription, IStepsContext, ProgressFn } from './step'; 8 | 9 | export interface IDownloadAssetsStepsContext extends IStepsContext { 10 | assetsPaths: string[]; 11 | downloadedAssetsDir: string; 12 | downloadedAssetsPaths: string[]; 13 | } 14 | 15 | export function describe(): IStepDescription { 16 | return { 17 | code: 'download-assets', 18 | name: 'Download remote assets', 19 | priority: 10 20 | }; 21 | } 22 | 23 | export function shouldProcess(conv: IConversion): boolean { 24 | return true; 25 | } 26 | 27 | export async function process( 28 | conv: IConversion, 29 | context: IDownloadAssetsStepsContext, 30 | progress: ProgressFn 31 | ): Promise { 32 | //=> Create a temporary folder for the assets 33 | const tmpDir = path.resolve(`${os.tmpdir()}/chuck-dl-assets-${Date.now()}`); 34 | await pify(fs.mkdir)(tmpDir); 35 | 36 | context.downloadedAssetsDir = tmpDir; 37 | context.downloadedAssetsPaths = []; 38 | 39 | //=> Start downloading all assets 40 | const downloads = conv.assetUrls.map(url => Promise.all([ 41 | progress('start-download', `Downloading "${url}"`), 42 | downloadAndStoreAsset(context, url, tmpDir) 43 | ])); 44 | 45 | //=> Await downloads and check if there are errors. 46 | let remainingDownloads = downloads.length; 47 | const errors: any[] = []; 48 | 49 | return new Promise((resolve, reject) => { 50 | downloads.forEach((dl) => { 51 | dl.catch(err => errors.push(err)).then(() => { 52 | if (--remainingDownloads > 0) return; 53 | 54 | errors.length 55 | ? reject(new MasterDownloadError('Error(s) while downloading assets', errors)) 56 | : resolve(); 57 | }); 58 | }); 59 | }); 60 | } 61 | 62 | export async function cleanup(context: Readonly): Promise { 63 | const rms = context.downloadedAssetsPaths.map(path => pify(fs.unlink)(path)); 64 | 65 | await Promise.all(rms); 66 | await pify(fs.rmdir)(context.downloadedAssetsDir); 67 | } 68 | 69 | async function downloadAndStoreAsset( 70 | context: IDownloadAssetsStepsContext, 71 | assetUrl: string, 72 | directory: string 73 | ): Promise { 74 | //=> Find the dest path for the asset based on `directory` 75 | const fileName = assetUrl.split(/[?#]/)[0].split('/').reverse()[0]; 76 | const filePath = path.resolve(`${directory}/${fileName}`); 77 | 78 | //=> Make HTTP request and create a write stream to the temp asset file 79 | const onlineAsset = await downloadAsset(assetUrl); 80 | const localAsset = fs.createWriteStream(filePath); 81 | 82 | context.downloadedAssetsPaths.push(filePath); 83 | 84 | //=> Pipe the HTTP response content to the write stream, then resolve 85 | return new Promise((resolve) => { 86 | onlineAsset.pipe(localAsset).on('finish', () => { 87 | localAsset.close(); 88 | 89 | context.assetsPaths.push(filePath); 90 | resolve(filePath); 91 | }); 92 | }); 93 | } 94 | 95 | async function downloadAsset(assetUrl: string): Promise { 96 | const response = await fetch(assetUrl); 97 | if (!response.ok) { 98 | throw new Error(`Server responded with HTTP error code ${response.status} while downloading ${assetUrl}`); 99 | } 100 | 101 | return response.body; 102 | } 103 | 104 | class MasterDownloadError extends Error { 105 | constructor(message: string, public errors: any[]) { 106 | super(message); 107 | 108 | Object.setPrototypeOf(this, MasterDownloadError.prototype); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/converter/steps/02_exec_assetbundlecompiler.ts: -------------------------------------------------------------------------------- 1 | import * as fs from 'fs'; 2 | import * as os from 'os'; 3 | import * as path from 'path'; 4 | import * as pify from 'pify'; 5 | import * as sanitize from 'sanitize-filename'; 6 | import { bundle, setUnityPath } from '@mitm/assetbundlecompiler'; 7 | import config from '../../config'; 8 | import { IConversion } from '../../models/IConversion'; 9 | import { IStepDescription, ProgressFn } from './step'; 10 | import { IDownloadAssetsStepsContext } from './01_download_assets'; 11 | 12 | export interface IExecAssetBundleCompilerStepContext extends IDownloadAssetsStepsContext { 13 | assetBundleDir: string; 14 | assetBundlePath: string; 15 | } 16 | 17 | export function describe(): IStepDescription { 18 | return { 19 | code: 'exec-assetbundlecompiler', 20 | name: 'Execute AssetBundleCompiler to assemble the asset bundle', 21 | priority: 20 22 | }; 23 | } 24 | 25 | export function shouldProcess(conv: IConversion, context: IExecAssetBundleCompilerStepContext): boolean { 26 | return !!(context.assetsPaths && context.assetsPaths.length); 27 | } 28 | 29 | export async function process( 30 | conv: IConversion, 31 | context: IExecAssetBundleCompilerStepContext, 32 | progress: ProgressFn 33 | ): Promise { 34 | const tmpDir = path.resolve(`${os.tmpdir()}/chuck-exec-assetbundlecompiler-${Date.now()}`); 35 | const sanitizedBundleName = sanitize(conv.assetBundleName).trim().toLowerCase(); 36 | const assetBundlePath = path.join(tmpDir, sanitizedBundleName); 37 | 38 | await pify(fs.mkdir)(tmpDir); 39 | 40 | context.assetBundleDir = tmpDir; 41 | context.assetBundlePath = assetBundlePath; 42 | 43 | if (config.unityPath) { 44 | setUnityPath(config.unityPath); 45 | } 46 | 47 | const options = conv.compilerOptions; 48 | 49 | await bundle(...context.assetsPaths) 50 | .targeting(options.targeting) 51 | .includingEditorScripts(...options.editorScripts) 52 | .withBuildOptions(options.buildOptions) 53 | .withLogger(async log => await progress('abcompiler-log', log)) 54 | .to(assetBundlePath); 55 | } 56 | 57 | export async function cleanup(context: Readonly): Promise { 58 | await pify(fs.unlink)(context.assetBundlePath); 59 | await pify(fs.rmdir)(context.assetBundleDir); 60 | } 61 | -------------------------------------------------------------------------------- /src/converter/steps/03_upload_bundle.ts: -------------------------------------------------------------------------------- 1 | import * as azure from 'azure-storage'; 2 | import * as pify from 'pify'; 3 | import * as sanitize from 'sanitize-filename'; 4 | import config from '../../config'; 5 | import { IConversion } from '../../models/IConversion'; 6 | import { IStepDescription, ProgressFn } from './step'; 7 | import { IExecAssetBundleCompilerStepContext as ICompilerStepContext } from './02_exec_assetbundlecompiler'; 8 | 9 | export function describe(): IStepDescription { 10 | return { 11 | code: 'upload-bundle', 12 | name: 'Upload the AssetBundle on Azure', 13 | priority: 30 14 | }; 15 | } 16 | 17 | export function shouldProcess(): boolean { 18 | return true; 19 | } 20 | 21 | export async function process(conv: IConversion, context: ICompilerStepContext, progress: ProgressFn): Promise { 22 | await progress('upload-start', `Uploading "${context.assetBundlePath}" to Azure`); 23 | 24 | const blobService = getBlobService(conv); 25 | 26 | const blobName = sanitize(conv.assetBundleName).trim().toLowerCase(); 27 | const container = conv.azure.container; 28 | 29 | const createBlob = pify(blobService.createBlockBlobFromLocalFile.bind(blobService)); 30 | const blobResult = await createBlob(container, blobName, context.assetBundlePath); 31 | 32 | const blobUrl = blobService.getUrl(container, blobName); 33 | 34 | await progress('upload-end', 'Upload terminated with success', { blobUrl, blobResult }); 35 | 36 | context.assetBundleUrl = blobUrl; 37 | } 38 | 39 | function getBlobService(conv: IConversion): azure.BlobService { 40 | if (config.azure.enableEmu) { 41 | const devStoreCreds = azure.generateDevelopmentStorageCredentials(); 42 | return azure.createBlobService(devStoreCreds); 43 | } else { 44 | return azure.createBlobServiceWithSas(conv.azure.host, conv.azure.sharedAccessSignatureToken); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/converter/steps/index.ts: -------------------------------------------------------------------------------- 1 | import { IStepModule } from './step'; 2 | import * as downloadAssets from './01_download_assets'; 3 | import * as execAssetBundleCompiler from './02_exec_assetbundlecompiler'; 4 | import * as uploadBundle from './03_upload_bundle'; 5 | 6 | export default [downloadAssets, execAssetBundleCompiler, uploadBundle] as IStepModule[]; 7 | -------------------------------------------------------------------------------- /src/converter/steps/step.ts: -------------------------------------------------------------------------------- 1 | import { IConversion } from '../../models/IConversion'; 2 | 3 | export type ProgressFn = (type: string, message: string, data?: any) => Promise; 4 | 5 | export interface IStepsContext { 6 | [customKey: string]: any; 7 | assetBundleUrl?: string; // mandatory ouput 8 | } 9 | 10 | export interface IStepDescription { 11 | code: string; 12 | name: string; 13 | priority: number; 14 | } 15 | 16 | export interface IStepModule { 17 | describe(): IStepDescription; 18 | shouldProcess(conversion: IConversion, context: IStepsContext): boolean; 19 | process(conversion: IConversion, context: IStepsContext, progress: ProgressFn): Promise; 20 | cleanup?(context: Readonly): Promise; 21 | } 22 | -------------------------------------------------------------------------------- /src/entry/app_standalone.ts: -------------------------------------------------------------------------------- 1 | import * as os from 'os'; 2 | import * as express from 'express'; 3 | import * as cors from 'cors'; 4 | import * as bodyParser from 'body-parser'; 5 | import * as path from 'path'; 6 | import * as morgan from 'morgan'; 7 | import './bootstrap'; 8 | import config from '../config'; 9 | import logger, { morganStreamWriter } from '../logger'; 10 | import { connectDatabase } from '../mongoose'; 11 | import api from '../api'; 12 | import converterQueue from '../converter/queue'; 13 | 14 | //=> Resume the conversions queue 15 | converterQueue.resume().catch((error) => { 16 | logger.error(error.message); 17 | process.exit(1); 18 | }); 19 | 20 | //=> Create an Express app 21 | const app = express(); 22 | const port = config.serverPort; 23 | 24 | //=> Load the view engine Pug 25 | app.set('view engine', 'pug'); 26 | app.set('views', path.resolve(`${__dirname}/..`)); 27 | 28 | //=> Connect to the MongoDB database 29 | connectDatabase(config.mongoUrl).catch((error) => { 30 | logger.error(error.message); 31 | process.exit(1); 32 | }); 33 | 34 | //=> Enable CORS 35 | app.use(cors()); 36 | 37 | //=> Logging of HTTP requests with morgan 38 | const morganFormat = config.env == 'production' 39 | ? ':remote-addr - :method :url [:status], resp. :response-time ms, :res[content-length] bytes, referrer ":referrer"' 40 | : 'dev'; 41 | 42 | app.use(morgan(morganFormat, { stream: morganStreamWriter })); 43 | 44 | //=> Decode JSON request bodies 45 | app.use( 46 | bodyParser.json(), 47 | bodyParser.urlencoded({ extended: true }) 48 | ); 49 | 50 | //=> Mount the API 51 | app.use('/api', api); 52 | 53 | if (config.adminWebUis.enable) { 54 | // Load web uis on-demand with require() 55 | // tslint:disable-next-line:no-var-requires 56 | app.use(require('../web_uis').default); 57 | } 58 | 59 | //=> Start the HTTP server 60 | app.listen(port, () => { 61 | logger.info(`🌍 Up and running @ http://${os.hostname()}:${port}`); 62 | logger.info(`Running for env: ${config.env}`); 63 | }); 64 | -------------------------------------------------------------------------------- /src/entry/bootstrap.ts: -------------------------------------------------------------------------------- 1 | import * as raven from 'raven'; 2 | import * as sourceMapSupport from 'source-map-support'; 3 | import config from '../config'; 4 | import logger from '../logger'; 5 | 6 | sourceMapSupport.install(); 7 | 8 | config.ravenDsn && raven.config(config.ravenDsn, { 9 | release: config.release, 10 | environment: config.env 11 | }).install(); 12 | 13 | process.on('unhandledRejection', (reason: any): void => { 14 | logger.error('UNHANDLED REJECTION', reason); 15 | }); 16 | 17 | process.on('uncaughtException', (err: any): void => { 18 | logger.error('UNCAUGHT EXCEPTION', err); 19 | }); 20 | -------------------------------------------------------------------------------- /src/entry/cli.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import * as yargs from 'yargs'; 3 | import allCommands from '../cli/api'; 4 | 5 | import './bootstrap'; 6 | 7 | yargs.usage('Usage: $0 [arguments]'); 8 | 9 | (allCommands as yargs.CommandModule[]) 10 | .forEach(command => yargs.command(command)); 11 | 12 | yargs.help().argv; 13 | -------------------------------------------------------------------------------- /src/entry/libchuck.ts: -------------------------------------------------------------------------------- 1 | import defaultConfig, { IChuckConfig } from '../config'; 2 | 3 | //=> Export IChuckConfig 4 | export { IChuckConfig, EnvType } from '../config'; 5 | 6 | //=> Export core symbols for writing plugin steps 7 | export { IConversion } from '../models/IConversion'; 8 | export { ProgressFn, IStepDescription, IStepsContext, IStepModule } from '../converter/steps/step'; 9 | 10 | //=> Export core steps' context for reuse in plugin steps 11 | export { IDownloadAssetsStepsContext } from '../converter/steps/01_download_assets'; 12 | export { IExecAssetBundleCompilerStepContext } from '../converter/steps/02_exec_assetbundlecompiler'; 13 | 14 | export function boot(config: Partial = {}): void { 15 | Object.assign(defaultConfig, config); 16 | 17 | //=> Launch Chuck by its main entry point 18 | require('./standalone'); 19 | } 20 | -------------------------------------------------------------------------------- /src/entry/standalone.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import config from '../config'; 3 | import logger from '../logger'; 4 | import * as pluginSteps from '../converter/plugin_steps'; 5 | 6 | if (config.stepModulePlugins.length) { 7 | logger.info(`Loading plugins: ${config.stepModulePlugins.join(', ')}`); 8 | } 9 | 10 | //=> Require each plugin module buits name 11 | config.stepModulePlugins.map(require).forEach(pluginSteps.register); 12 | 13 | //=> Launch Chuck by its main entry point 14 | // tslint:disable-next-line:no-var-requires 15 | require('./app_standalone'); 16 | -------------------------------------------------------------------------------- /src/express_utils.ts: -------------------------------------------------------------------------------- 1 | import { Request, RequestHandler, Response, NextFunction } from 'express'; 2 | import { Document } from 'mongoose'; 3 | 4 | export function wrapAsync(handler: RequestHandler): RequestHandler { 5 | return (req: Request, res: Response, next: NextFunction) => handler(req, res, next).catch(next); 6 | } 7 | 8 | export function safeOutData(document: T): Partial { 9 | const safeDocument = document.toObject() as any; 10 | 11 | delete safeDocument.__v; 12 | delete safeDocument._id; 13 | 14 | return safeDocument; 15 | } 16 | -------------------------------------------------------------------------------- /src/logger.ts: -------------------------------------------------------------------------------- 1 | import * as winston from 'winston'; 2 | import { StreamOptions } from 'morgan'; 3 | import config from './config'; 4 | 5 | const logger = new winston.Logger({ 6 | transports: [ 7 | new winston.transports.Console({ 8 | level: config.logLevel, 9 | json: false, 10 | colorize: true, 11 | timestamp: config.env == 'production' 12 | }) 13 | ] 14 | }); 15 | 16 | export default logger; 17 | 18 | export const morganStreamWriter: StreamOptions = { 19 | write(message: string): void { 20 | logger.info(message.replace(/\n$/, '')); 21 | } 22 | }; 23 | -------------------------------------------------------------------------------- /src/models/IApiKey.ts: -------------------------------------------------------------------------------- 1 | export interface IApiKey { 2 | key: string; 3 | } 4 | -------------------------------------------------------------------------------- /src/models/IConversion.ts: -------------------------------------------------------------------------------- 1 | import { IBuildOptionsMap } from '@mitm/assetbundlecompiler'; 2 | import { IEvent } from '../converter/job_events'; 3 | 4 | export interface IConversion { 5 | code: string; 6 | assetBundleName: string; 7 | assetUrls: string[]; 8 | conversionOptions: string[]; 9 | 10 | azure: { 11 | host: string; 12 | sharedAccessSignatureToken: string; 13 | container: string; 14 | }; 15 | 16 | compilerOptions: { 17 | targeting: string; 18 | buildOptions: IBuildOptionsMap; 19 | editorScripts: string[]; 20 | }; 21 | 22 | conversion: { 23 | jobId: string|null; 24 | isCompleted: boolean; 25 | step: string|null; 26 | error: any|null; 27 | assetBundleUrl: string|null; 28 | logs: IEvent[] 29 | }; 30 | } 31 | -------------------------------------------------------------------------------- /src/models/api_key.ts: -------------------------------------------------------------------------------- 1 | import * as mongoose from 'mongoose'; 2 | import { Document, Schema } from 'mongoose'; 3 | import * as uuid from 'uuid'; 4 | import { IApiKey } from './IApiKey'; 5 | 6 | export type IApiKeyModel = IApiKey & Document; 7 | 8 | const ApiKeySchema = new Schema({ 9 | key: { type: String, unique: true, index: true, default: () => uuid.v4() } 10 | }); 11 | 12 | export const ApiKey = mongoose.model('ApiKey', ApiKeySchema); 13 | -------------------------------------------------------------------------------- /src/models/conversion.ts: -------------------------------------------------------------------------------- 1 | import * as mongoose from 'mongoose'; 2 | import { Document, Schema } from 'mongoose'; 3 | import * as uuid from 'uuid'; 4 | import { IConversion } from './IConversion'; 5 | 6 | export type IConversionModel = IConversion & Document; 7 | 8 | const ConversionSchema = new Schema({ 9 | code: { type: String, unique: true, index: true, default: () => uuid.v4() }, 10 | 11 | assetBundleName: { type: String, required: true }, 12 | 13 | assetUrls: { 14 | type: Array, 15 | required: true, 16 | validate(urls: any[]) { 17 | return urls.every(value => typeof value === 'string'); 18 | } 19 | }, 20 | 21 | conversionOptions: { 22 | type: Array, 23 | default: [], 24 | validate(options: any[]) { 25 | return options.every(value => typeof value === 'string'); 26 | } 27 | }, 28 | 29 | azure: { 30 | host: { type: String, required: true }, 31 | sharedAccessSignatureToken: { type: String, required: true }, 32 | container : { type: String, required: true } 33 | }, 34 | 35 | compilerOptions: { 36 | targeting: { type: String, required: true }, 37 | buildOptions: { 38 | type: Schema.Types.Mixed, 39 | default: {} 40 | }, 41 | editorScripts: { 42 | type: Array, 43 | default: [], 44 | validate(script: any[]) { 45 | return script.every(value => typeof value === 'string'); 46 | } 47 | } 48 | }, 49 | 50 | conversion: { 51 | jobId: { type: String, default: null }, 52 | isCompleted: { type: Boolean, default: false }, 53 | step: { type: String, default: null }, 54 | error: { type: Schema.Types.Mixed, default: null }, 55 | assetBundleUrl: { type: String, default: null }, 56 | logs: [ 57 | { type: Schema.Types.Mixed, default: [] } 58 | ] 59 | } 60 | }, { minimize: false }); 61 | 62 | export function safeData({ 63 | assetBundleName, 64 | assetUrls, 65 | conversionOptions, 66 | azure, 67 | compilerOptions 68 | }: IConversion): Partial { 69 | return { assetBundleName, assetUrls, conversionOptions, azure, compilerOptions }; 70 | } 71 | 72 | export const Conversion = mongoose.model('Conversion', ConversionSchema); 73 | -------------------------------------------------------------------------------- /src/models/index.ts: -------------------------------------------------------------------------------- 1 | export { ApiKey } from './api_key'; 2 | export { Conversion } from './conversion'; 3 | -------------------------------------------------------------------------------- /src/mongoose.ts: -------------------------------------------------------------------------------- 1 | import * as mongoose from 'mongoose'; 2 | import logger from './logger'; 3 | 4 | // tslint:disable:only-arrow-functions 5 | 6 | //=> Tell Mongoose to use native V8 promises 7 | (mongoose as any).Promise = Promise; 8 | 9 | //=> Setup logging of queries 10 | mongoose.set('debug', function(collection: string, method: string, ...methodArgs: object[]) { 11 | // Basically extracted from https://goo.gl/OYCxAV (otherwise, Mongoose writes directly to stderr). 12 | const args: string[] = []; 13 | for (let j = methodArgs.length - 1; j >= 0; --j) { 14 | if (this.$format(methodArgs[j]) || args.length) { 15 | args.unshift(this.$format(methodArgs[j])); 16 | } 17 | } 18 | 19 | logger.debug(`mongo: ${collection}.${method}(${args.join(', ')})`); 20 | }); 21 | 22 | export function connectDatabase(mongoUrl: string): Promise { 23 | return new Promise((resolve, reject) => { 24 | mongoose.connect(mongoUrl, (error) => { 25 | error 26 | ? reject(new Error(`Could not connect to the MongoDB instance! ${error.message}`)) 27 | : resolve(); 28 | }); 29 | }); 30 | } 31 | 32 | export function disconnectFromDatabase(): Promise { 33 | return new Promise((resolve, reject) => { 34 | mongoose.disconnect(error => error ? reject() : resolve()); 35 | }); 36 | } 37 | -------------------------------------------------------------------------------- /src/safe_error_serialize.ts: -------------------------------------------------------------------------------- 1 | export interface IPlainObjectError { 2 | name: string; 3 | message: string; 4 | errorId?: string; 5 | errors?: any[]; 6 | } 7 | 8 | export function safeErrorSerialize(error: any, sentryErrorId?: string): IPlainObjectError { 9 | const plainError: any = {}; 10 | 11 | //=> Error name 12 | plainError.name = error.name || error.constructor.name; 13 | 14 | //=> Error message 15 | plainError.message = error.message || 'Unknown error'; 16 | 17 | //=> Set error's ID if given 18 | if (sentryErrorId) { 19 | plainError.errorId = sentryErrorId; 20 | } 21 | 22 | //=> Serialize recursively sub-errors if any 23 | if (error.errors) { 24 | plainError.errors = error.errors.map(safeErrorSerialize); 25 | } 26 | 27 | return plainError; 28 | } 29 | -------------------------------------------------------------------------------- /src/web_uis.ts: -------------------------------------------------------------------------------- 1 | import * as basicAuth from 'express-basic-auth'; 2 | import * as express from 'express'; 3 | import * as toureiro from 'toureiro'; 4 | import admin from './admin'; 5 | import config from './config'; 6 | 7 | const router: express.Router = express.Router(); 8 | 9 | //=> Set a middleware that authenticate administrators 10 | const adminAuth = basicAuth({ 11 | challenge: true, 12 | realm: 'Chuck Administration', 13 | users: { [config.adminWebUis.user]: config.adminWebUis.password } 14 | }); 15 | 16 | //=> Mount Toureiro 17 | router.use('/toureiro', adminAuth, toureiro({ 18 | redis: { host: config.redis.host, port: config.redis.port, db: 0 } 19 | })); 20 | 21 | //=> Route of the admin interface 22 | router.use('/admin', adminAuth, admin); 23 | 24 | export default router; 25 | -------------------------------------------------------------------------------- /stubs/toureiro.d.ts: -------------------------------------------------------------------------------- 1 | declare module 'toureiro' { 2 | import { RequestHandler } from 'express'; 3 | 4 | function toureiro(options: toureiro.IRedisOptions): RequestHandler; 5 | 6 | namespace toureiro { 7 | interface IRedisOptions { 8 | redis: { host: string, port: number, db: number, auth_pass?: string }; 9 | } 10 | } 11 | 12 | export = toureiro; 13 | } 14 | -------------------------------------------------------------------------------- /test/test.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mitmadness/chuck/b9a201f5b2382c94d2507dc0b15143feecc71f2c/test/test.js -------------------------------------------------------------------------------- /test/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Hello, world!" 4 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compileOnSave": false, 3 | "buildOnSave": false, 4 | "compilerOptions": { 5 | "outDir": "dist", 6 | "target": "es2015", 7 | "module": "commonjs", 8 | "moduleResolution": "node", 9 | "declaration": true, 10 | "removeComments": true, 11 | "emitDecoratorMetadata": true, 12 | "experimentalDecorators": true, 13 | "sourceMap": true, 14 | "noImplicitAny": true, 15 | "noUnusedLocals": true, 16 | "noUnusedParameters": false, 17 | "strictNullChecks": true, 18 | "lib": ["es7"] 19 | }, 20 | "include": ["src/**/*", "stubs/*.d.ts"], 21 | "exclude": ["node_modules"] 22 | } 23 | -------------------------------------------------------------------------------- /tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "tslint:latest", 3 | "rules": { 4 | "curly": false, 5 | "quotemark": ["double"], 6 | "ordered-imports": false, 7 | "object-literal-sort-keys": false, 8 | "comment-format": [], 9 | "triple-equals": false, 10 | "trailing-comma": [true, { "multiline": "never", "singleline": "never" }], 11 | "arrow-parens": false, 12 | "new-parens": false, 13 | "no-unused-expression": false 14 | } 15 | } 16 | --------------------------------------------------------------------------------