├── .editorconfig
├── .env.defaults
├── .gitignore
├── .npmignore
├── .travis.yml
├── LICENSE
├── README.md
├── chuck.png
├── package.json
├── postman
├── chuck.postman_collection.json
└── devel.postman_environment.json
├── src
├── admin
│ ├── admin.pug
│ └── index.ts
├── api
│ ├── api_keys_cache.ts
│ ├── conversions_api.ts
│ ├── conversions_api_events.ts
│ ├── index.ts
│ └── middlewares.ts
├── cli
│ ├── api
│ │ ├── generate_key.ts
│ │ ├── index.ts
│ │ └── revoke_key.ts
│ └── index.ts
├── config.ts
├── converter
│ ├── job.ts
│ ├── job_events.ts
│ ├── plugin_steps.ts
│ ├── queue.ts
│ ├── queue_event_handlers.ts
│ ├── queue_processor.ts
│ └── steps
│ │ ├── 01_download_assets.ts
│ │ ├── 02_exec_assetbundlecompiler.ts
│ │ ├── 03_upload_bundle.ts
│ │ ├── index.ts
│ │ └── step.ts
├── entry
│ ├── app_standalone.ts
│ ├── bootstrap.ts
│ ├── cli.ts
│ ├── libchuck.ts
│ └── standalone.ts
├── express_utils.ts
├── logger.ts
├── models
│ ├── IApiKey.ts
│ ├── IConversion.ts
│ ├── api_key.ts
│ ├── conversion.ts
│ └── index.ts
├── mongoose.ts
├── safe_error_serialize.ts
└── web_uis.ts
├── stubs
└── toureiro.d.ts
├── test
├── test.js
└── test.sh
├── tsconfig.json
├── tslint.json
└── yarn.lock
/.editorconfig:
--------------------------------------------------------------------------------
1 | root = true
2 |
3 | [*]
4 | charset = utf-8
5 | indent_style = space
6 | indent_size = 4
7 | end_of_line = lf
8 | insert_final_newline = true
9 | trim_trailing_whitespace = true
10 |
11 | [*.md]
12 | indent_size = 2
13 |
14 | [*.json]
15 | indent_size = 2
16 |
--------------------------------------------------------------------------------
/.env.defaults:
--------------------------------------------------------------------------------
1 | CHUCK_ENV=development
2 | CHUCK_LOGLEVEL=verbose
3 | CHUCK_SERVERPORT=3001
4 | CHUCK_MONGOURL=mongodb://localhost/chuck
5 | CHUCK_RAVENDSN=
6 | CHUCK_REDIS_HOST=localhost
7 | CHUCK_REDIS_PORT=6379
8 | CHUCK_REDIS_DB=0
9 | CHUCK_ADMINWEBUIS_ENABLE=false
10 | CHUCK_ADMINWEBUIS_USER=admin
11 | CHUCK_ADMINWEBUIS_PASSWORD=admin
12 | CHUCK_AZURE_ENABLEEMU=false
13 | CHUCK_STEPMODULEPLUGINS=
14 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | dist/
2 | node_modules/
3 | .env
4 |
--------------------------------------------------------------------------------
/.npmignore:
--------------------------------------------------------------------------------
1 | postman/*
2 | src/*
3 | stubs/*
4 | .editorconfig
5 | .env
6 | .gitignore
7 | .npmignore
8 | .travis.yml
9 | chuck.png
10 | tsconfig.json
11 | tslint.json
12 |
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: node_js
2 |
3 | cache:
4 | directories:
5 | - node_modules
6 |
7 | notifications:
8 | email: false
9 |
10 | node_js:
11 | - '7'
12 | - '6'
13 |
14 | before_script:
15 | - yarn build
16 |
17 | after_success:
18 | - yarn semantic-release
19 |
20 | branches:
21 | except:
22 | - /^v\d+\.\d+\.\d+$/
23 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2017 Method in the Madness
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 |
3 | # chuck [](https://www.npmjs.com/package/@mitm/chuck)  [](https://travis-ci.org/mitmadness/chuck)
4 |
5 | chuck is a webservice that converts files that Unity understands to Unity3D AssetBundles.
6 |
7 | From the Unity3D [documentation](https://docs.unity3d.com/Manual/AssetBundlesIntro.html):
8 |
9 | > AssetBundles are files which you can export from Unity to contain Assets of your choice, [that] can be loaded on demand by your application. This allows you to stream content, such as models, Textures, audio clips, or even entire Scenes [...].
10 |
11 | Chuck notably features REST & Server-Sent APIs, a command-line interface, an admin page for managing API keys, and integrates [Toureiro](https://github.com/Epharmix/Toureiro) for viewing queued jobs.
12 |
13 | *:point_right: See also: [@mitm/assetbundlecompiler](https://github.com/mitmadness/AssetBundleCompiler), fluent JavaScript API to create AssetBundles from any files.*
14 |
15 | *:point_right: See also: [@mitm/unityinvoker](https://github.com/mitmadness/UnityInvoker), invoke Unity3D CLI with a fluent JavaScript API.*
16 |
17 | ----------------
18 |
19 | - [Requirements](#requirements)
20 | - [Installation](#installation): [Standalone](#standalone-installation) / [Embedded](#embedded-installation)
21 | - [Configuration](#configuration)
22 | - [Public REST API](#public-rest--sse-api)
23 | - Extra tools: [Command Line Interface](#command-line-interface) / [Admin Web Interface](#admin-web-interface) / [Toureiro](#toureiro)
24 | - [Development & Contributing](#development--contributing)
25 |
26 | ----------------
27 |
28 | ## Requirements
29 |
30 | - **[Node.js](https://nodejs.org/en/)** v.7 or higher ;
31 | - **[yarn](https://yarnpkg.com/en/)** dependency manager, for development or standalone installation ;
32 | - An Azure account where chuck will send its generated asset bundles (limitation) ;
33 | - A running **[MongoDB](https://www.mongodb.com/)** server ;
34 | - A running **[Redis](https://redis.io/)** server ;
35 | - :warning: **An _activated_ installation of Unity on the machine** :warning:
36 | - If Unity is not installed in the standard path, configure the path via the [Configuration](#configuration)
37 | - You must activate Unity if not already done, even with a free plan, read [Unity activation](https://github.com/mitmadness/AssetBundleCompiler#unity-activation) from AssetBundleCompiler
38 |
39 | Also, except logging them, chuck can optionally report fatal errors on [Sentry](https://sentry.io/welcome/). To enable this, [configure](#configuration) your Sentry DSN.
40 |
41 | ## Installation
42 |
43 | Please note that chuck is not a pure library, nor a reusable Express middleware. It's a standalone application with its own logging, database, etc. But it can also be installed as an npm package for writing plugins or configuring and boot it.
44 |
45 | ### Standalone installation
46 |
47 |
48 | Classical approach. Use this to deploy chuck directly without integrating it to another application (except API communication of course)
49 |
50 | #### 1. Install the application and dependencies
51 |
52 | ```
53 | $ git clone git@github.com:mitmadness/chuck.git && cd chuck
54 | $ yarn install
55 | $ yarn build
56 | ```
57 |
58 | #### 2. Configure Chuck
59 |
60 | Create a blank `.env` file at the root.
61 |
62 | Look at the `.env.defaults` file (don't delete it, it's part of the application), it contains key/value pairs of environment variables.
63 |
64 | You can now override values from `.env.defaults` to match your own environment when the default values are incorrect.
65 |
66 | You can also, of course, set environment variables by hand with your preferred method (exports, inline variables when launching the command...).
67 |
68 | *:point_right: Read more about how configuration is mapped with environment variables in [Using environment variables](#using-environment-variables)*
69 |
70 | #### 3. Run it
71 |
72 | Run `yarn start`. That's it.
73 |
74 | To use chuck plugins, install them (ex. `yarn add @mitm/chuck-ifc`) then ask chuck to load them via the `CHUCK_STEPMODULEPLUGINS` environment variable:
75 |
76 | ```
77 | CHUCK_STEPMODULEPLUGINS=@mitm/chuck-ifc,myplugin yarn start
78 | ```
79 |
80 |
81 | ### Embedded installation
82 |
83 |
84 | Chuck app integration into another package: chuck as a npm package. This is the preferred method and it gives you more flexibility, doesn't need compilation, permits better versioning, etc.
85 |
86 | #### 1. Install chuck via its package
87 |
88 | ```
89 | yarn add @mitm/chuck
90 | ```
91 |
92 | #### 2. Configure & boot it
93 |
94 | You can either set up your own entry point and configure it in JavaScript (see [Configuration](#configuration)), or you can still use environment variables.
95 |
96 | **With environment variables**, do it like this:
97 |
98 | ```
99 | CHUCK_SERVERPORT=80 CHUCK_MONGOURL=mongodb://localhost/chuck yarn chuck
100 | ```
101 |
102 | You can also, of course, set environment variables by hand with your preferred method (exports, inline variables when launching the command...).
103 |
104 | Run the CLI similarly with `yarn chuck-cli`.
105 |
106 | *:point_right: Read more about how configuration is mapped with environment variables in [Using environment variables](#using-environment-variables)*
107 |
108 | **With your own entry point**:
109 |
110 | Create a file named, for example, `chuck.ts` in your project:
111 |
112 | ```typescript
113 | import { boot } from '@mitm/chuck';
114 |
115 | boot({
116 | serverPort: 80,
117 | mongoUrl: 'mongodb://localhost/chuck'
118 | });
119 | ```
120 |
121 | You can still use environment variables with `boot()`.
122 |
123 |
124 | ## Configuration
125 |
126 | ### Available configuration keys
127 |
128 |
129 | This is the interface for the available configuration
130 |
131 | ```ts
132 | interface IChuckConfig {
133 | // aka NODE_ENV. Configures the mode (`development` or `production`) in which the server is running.
134 | // development: permissive CORS rules are set on the API
135 | // production: timestamps in log messages and more verbose HTTP logs
136 | // Defaults to process.env.NODE_ENV or, if undefined, "development"
137 | env: EnvType;
138 |
139 | // Minimum log level (npm log levels, see https://github.com/winstonjs/winston#logging-levels).
140 | // Defaults to "verbose"
141 | logLevel: string;
142 |
143 | // Server HTTP port.
144 | // Defaults to 3001
145 | serverPort: number;
146 |
147 | // Connection string to a MongoDB database.
148 | // Defaults to mongodb://localhost/chuck
149 | mongoUrl: string;
150 |
151 | // DSN for Sentry error reporting.
152 | // Reporting is disabled if this is not set.
153 | ravenDsn: string;
154 |
155 | // Redis connection informations.
156 | // Defaults to { host: 'localhost', port: 6379, db: 0 }
157 | redis: { host: string; port: number; db: number };
158 |
159 | // Admin Web UIs configuration. Used by the admin interface and Toureiro.
160 | // Default to { enable: false, user: 'admin', password: 'admin' }
161 | adminWebUis: { enable: boolean, user: string; password: string; };
162 |
163 | // Unity Editor path (if not installed in the standard path), see https://github.com/mitmadness/AssetBundleCompiler#changing-unitys-executable-path
164 | // Default to undefined (auto)
165 | unityPath: string;
166 |
167 | // Azure configuration.
168 | // Default to { enableEmu: false }
169 | azure: { enableEmu: boolean; };
170 |
171 | /**
172 | * An array of module names.
173 | * Those modules will be loaded dynamically as step plugins.
174 | */
175 | stepModulePlugins: string[];
176 | }
177 | ```
178 |
179 |
180 | ### Using environment variables
181 |
182 |
183 | Chuck is primarily configurable via environment variables. Read here how to map the configuration interface options on environment variables.
184 |
185 | You can set environement variables in the way you prefer. Tou can set them inline, in the CLI command that launches chuck, via a shell `export`, or for example, if you use the standalone installation, via an `.env` file at root that overrides Chuck's `.env.defaults` values (only for redefined keys).
186 |
187 | Then, environment variables are simply mapped to the real configuration. Take those example:
188 |
189 | - To set `config.logLevel`, set `CHUCK_LOGLEVEL`
190 | - To set `config.adminWebUis.enable`, set `CHUCK_ADMINWEBUIS_ENABLE`.
191 |
192 | Etc. Prefix with `CHUCK_` and replace dots with underscores, all uppercase.
193 |
194 |
195 | ## Public REST / SSE API
196 |
197 | Chuck exposes a simple REST and [SSE (Server-Sent Events)](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events) API for converting files to asset bundles.
198 |
199 | ### Create a new conversion request
200 |
201 | `POST /api/conversions`
202 |
203 |
204 | This endpoint will create a conversion request. It will immediately push on the conversions queue, so the job will start as soon as possible (conversions are ran sequentially).
205 |
206 | #### Request
207 |
208 | :closed_lock_with_key: This endpoint requires authentication using an API key. You can generate one either via the CLI, or via the web interface. Pass the API key like this: `Authorization: Bearer YOUR_API_KEY`.
209 |
210 | Note: `compilerOptions` is an object of options to pass to AssetBundleCompiler ([abcompiler's reference](https://github.com/mitmadness/AssetBundleCompiler#link-simple-fluent-api)).
211 |
212 | Note: `azure.sharedAccessSignatureToken` is an [Azure SAS token](https://docs.microsoft.com/en-us/azure/storage/storage-dotnet-shared-access-signature-part-1) that lets chuck create the asset bundle binary blob on Azure, without giving your Azure credentials to chuck. You can automate token creation with Azure CLI or Azure SDKs.
213 |
214 | ```
215 | POST /api/conversions
216 | Content-Type: application/json
217 | Authorization: Bearer {{apiKey}}
218 | ```
219 | ```json
220 | {
221 | "assetBundleName": "myassetbundle.ab",
222 | "compilerOptions": {
223 | "targeting": "webgl"
224 | },
225 | "assetUrls": [
226 | "https://url/to/a/file.fbx",
227 | "https://url/to/another/file.ifc"
228 | ],
229 | "azure": {
230 | "host": "{{azureHost}}",
231 | "container": "{{azureContainer}}",
232 | "sharedAccessSignatureToken": "{{azureSharedAccessSignatureToken}}"
233 | }
234 | }
235 | ```
236 |
237 | #### Successful response
238 |
239 | ```
240 | HTTP/1.1 202 Accepted
241 | Content-Type: application/json; charset=utf-8
242 | ```
243 | ```json
244 | {
245 | "assetBundleName": "myassetbundle.ab",
246 | "conversion": {
247 | "logs": [],
248 | "assetBundleUrl": null,
249 | "error": null,
250 | "step": null,
251 | "isCompleted": false,
252 | "jobId": "138"
253 | },
254 | "compilerOptions": {
255 | "targeting": "webgl",
256 | "editorScripts": [],
257 | "buildOptions": {}
258 | },
259 | "azure": {
260 | "host": "{{azureHost}}",
261 | "container": "{{azureContainer}}",
262 | "sharedAccessSignatureToken": "{{azureSharedAccessSignatureToken}}"
263 | },
264 | "assetUrls": [
265 | "https://url/to/a/file.fbx",
266 | "https://url/to/another/file.ifc"
267 | ],
268 | "code": "00cad557-5acc-4c6b-a987-79f650d67197"
269 | }
270 | ```
271 |
272 |
273 | ### Retrieve a conversion request
274 |
275 | `GET /api/conversions/{{code}}`
276 |
277 |
278 | Retrieves a previously-posted conversion request. This endpoint has no authentication as the conversion code adds a first layer of security (moreover, conversion requests are not editable).
279 |
280 | #### Request
281 |
282 | ```
283 | GET /api/conversions/{{conversionCode}}
284 | ```
285 |
286 | #### Successful response (completed conversion)
287 |
288 | ```
289 | HTTP/1.1 200 OK
290 | Content-Type: application/json; charset=utf-8
291 | ```
292 | ```json
293 | {
294 | "assetBundleName": "myassetbundle.ab",
295 | "conversion": {
296 | "assetBundleUrl": "https://{{azureHost}}/{{azureContainer}}/myassetbundle.ab",
297 | "error": null,
298 | "step": null,
299 | "isCompleted": true,
300 | "jobId": "139"
301 | },
302 | "compilerOptions": {
303 | "targeting": "webgl",
304 | "editorScripts": [],
305 | "buildOptions": {}
306 | },
307 | "azure": {
308 | "host": "{{azureHost}}",
309 | "container": "{{azureContainer}}",
310 | "sharedAccessSignatureToken": "{{azureSharedAccessSignatureToken}}"
311 | },
312 | "assetUrls": [
313 | "https://url/to/a/file.fbx",
314 | "https://url/to/another/file.ifc"
315 | ],
316 | "code": "00cad557-5acc-4c6b-a987-79f650d67197"
317 | }
318 | ```
319 |
320 | #### Successful response (failed conversion)
321 |
322 | ```
323 | HTTP/1.1 200 OK
324 | Content-Type: application/json; charset=utf-8
325 | ```
326 | ```json
327 | {
328 | ...
329 | "conversion": {
330 | "assetBundleUrl": null,
331 | "error": {
332 | "name": "Error",
333 | "message": "Error(s) while downloading assets",
334 | "errors": [
335 | {
336 | "name": "FetchError",
337 | "message": "request to https://url/to/a/file.fbx failed, reason: getaddrinfo ENOTFOUND url url:443"
338 | },
339 | {
340 | "name": "FetchError",
341 | "message": "request to https://url/to/another/file.ifc failed, reason: getaddrinfo ENOTFOUND url url:443"
342 | }
343 | ]
344 | },
345 | "step": "cleanup",
346 | "isCompleted": true,
347 | "jobId": "140"
348 | },
349 | ...
350 | }
351 | ```
352 |
353 |
354 | ### Real-time conversion job's events
355 |
356 | `GET /api/conversions/{{code}}/events`
357 |
358 |
359 | This is the only way to know precisely when a conversion is completed (or failed). It also sends various events concerning the job state, which step is running, and various logging informations.
360 |
361 | **This is an Server-Sent Events (SSE) endpoint**, use the browser's native interface `EventSource` to access it, or a browser/node.js polyfill like the [eventsource](https://www.npmjs.com/package/eventsource) package on npm.
362 |
363 | #### Request
364 |
365 | ```
366 | GET /api/conversions/{{conversionCode}}/events
367 | ```
368 |
369 | Available query parameters:
370 |
371 | - `?sseType={events|data}`: whether to use data-only SSE messages or event+data. If using `data`, the event name will be in the `type` property of the data-only message. Defaults to `events`.
372 | - `?replay={true|false}`: whether to dump missed events between the job start and the connection to the SSE endpoint. Defaults to `true`.
373 |
374 | #### Unsuccessful response
375 |
376 | Two classic HTTP errors may occur before the SSE session starts (errors are formatted the same way there are on the rest of the API).
377 |
378 | - `404 Not Found` when the conversion code given in the URL is unknown ;
379 | - `410 Gone` when `?replay=false` and the conversion is already terminated (success or error).
380 |
381 | The client can handle this using `EventSource.onerror`.
382 |
383 | #### Successful response (completed conversion)
384 |
385 | :bulb: Note: The server will close the connection as soon as the conversion is terminated, error or success. As the SSE spec does not allow a server to close the connection gracefully, the client MUST listen for errors, and call EventSource#close() itself to avoid automatic reconnection.
386 |
387 | ```typescript
388 | ev.onerror = (event) => {
389 | if (event.eventPhase == EventSource.CLOSED) {
390 | ev.close();
391 | }
392 | };
393 | ```
394 |
395 | :bulb: Note: The client will probably most interested in the `queue/conversion-ended` event that contains either an error or an URL to the resulting asset bundle.
396 |
397 | ```
398 | HTTP/1.1 200 OK
399 | Content-Type:text/event-stream; charset=utf-8
400 | Cache-Control: no-cache
401 |
402 | : sse-start
403 | event: processor/step-change
404 | data: {"step":{"priority":10,"name":"Download remote assets","code":"download-assets"},"message":"Starting \"Download remote assets\""}
405 |
406 | event: queue/conversion-start
407 | data: {"message":"Conversion started"}
408 |
409 | event: processor/download-assets/start-download
410 | data: {"message":"Downloading \"https://i.ytimg.com/vi/qIIN64bUbsI/maxresdefault.jpg\""}
411 |
412 | event: processor/step-change
413 | data: {"step":{"priority":30,"name":"Execute AssetBundleCompiler to assemble the asset bundle","code":"exec-assetbundlecompiler"},"message":"Starting \"Execute AssetBundleCompiler to assemble the asset bundle\""}
414 |
415 | event: processor/exec-assetbundlecompiler/abcompiler-log
416 | data: {"message":"Preparing Unity project in /tmp/AssetBundleCompiler"}
417 |
418 | event: processor/exec-assetbundlecompiler/abcompiler-log
419 | data: {"message":"Copying assets to /tmp/AssetBundleCompiler/Assets/CopiedAssets"}
420 |
421 | event: processor/exec-assetbundlecompiler/abcompiler-log
422 | data: {"message":"Copying custom editor scripts to /tmp/AssetBundleCompiler/Assets/Editor/CopiedScripts"}
423 |
424 | event: processor/exec-assetbundlecompiler/abcompiler-log
425 | data: {"message":"Generating asset bundle in /tmp/AssetBundleCompiler/GeneratedAssetBundles"}
426 |
427 | event: processor/exec-assetbundlecompiler/abcompiler-log
428 | data: {"message":"Updating resource: maxresdefault.jpg"}
429 |
430 | event: processor/exec-assetbundlecompiler/abcompiler-log
431 | data: {"message":"Moving asset bundle to target destination"}
432 |
433 | event: processor/exec-assetbundlecompiler/abcompiler-log
434 | data: {"message":"Cleaning up the Unity project"}
435 |
436 | event: processor/exec-assetbundlecompiler/abcompiler-log
437 | data: {"message":"Done."}
438 |
439 | event: processor/step-change
440 | data: {"message":"Starting \"Upload the AssetBundle on Azure\"","step":{"code":"upload-bundle","name":"Upload the AssetBundle on Azure","priority":40}}
441 |
442 | event: processor/upload-bundle/upload-start
443 | data: {"message":"Uploading \"/tmp/chuck-exec-assetbundlecompiler-1495536189347/myassetbundle.ab2\" to Azure"}
444 |
445 | event: processor/upload-bundle/upload-end
446 | data: {"message":"Upload terminated with success","blobUrl":"https://mitmtest.blob.core.windows.net/assetbundles/myassetbundle.ab2","blobResult":{"container":"assetbundles","name":"myassetbundle.ab2","lastModified":"Tue, 23 May 2017 10:43:19 GMT","etag":"\"0x8D4A1C886E0CAC4\"","requestId":"54ef0e35-0001-0088-4db1-d3adde000000","contentSettings":{"contentMD5":"xRF+eIadlPTWCVp8Y8IkjA=="}}}
447 |
448 | event: processor/step-change
449 | data: {"message":"Performing cleanup for steps: download-assets, exec-assetbundlecompiler, upload-bundle (All steps have terminated successfuly)","step":{"code":"cleanup","name":"Conversion artifacts cleanup","priority":null}}
450 |
451 | event: queue/conversion-ended
452 | data: {"message":"Conversion terminated with success!","assetBundleUrl":"https://mitmtest.blob.core.windows.net/assetbundles/myassetbundle.ab2","error":null}
453 |
454 | : sse-keep-alive
455 | : sse-keep-alive
456 | ```
457 |
458 |
459 | ## Extras tools
460 |
461 | ### Command Line Interface
462 |
463 | Chuck provides a CLI tool that is exported as the `bin` file in the package. In standalone mode, use it with `yarn cli -- --arguments`.
464 |
465 | Examples (using chuck as a package):
466 |
467 | - **`yarn chuck-cli help`** get available commands
468 | - **`yarn chuck-cli help `** displays informations about a command and available arguments
469 | - **`yarn chuck-cli api:generate-key`** generates an API key. Pass `--save` to save the generated key to the database.
470 | - **`yarn chuck-cli api:revoke-key `** revokes an API key stored in the database.
471 |
472 | ### Admin Web Interface
473 |
474 | A very, very simple administration interface is available under https://chuck/admin and uses HTTP Basic authentication, with the same credentials as Toureiro (see [Configuration](#configuration) section).
475 |
476 | It lets you create and revoke API keys as an alternative to the CLI.
477 |
478 | ### Toureiro
479 |
480 | Chuck embarks [Toureiro](https://github.com/Epharmix/Toureiro), which is...
481 |
482 | > A graphical monitoring interface for the distributed job queue bull built using express and react. Toureiro provides simple monitoring features as well as the ability to promote delayed jobs directly.
483 |
484 | Toureiro's interface can be found on https://chuck/toureiro, and is protected by the same HTTP Basic Auth and credentials used to login to the administration interface (see [Configuration](#configuration) section).
485 |
486 | 
487 |
488 | ## Development & Contributing
489 |
490 | The workflow is based on npm scripts:
491 |
492 | - `yarn watch`: starts the TypeScript compiler in watch mode ;
493 | - `yarn build`: compile TypeScript sources ;
494 | - `yarn start`: (run `watch` or `build` before!) starts the server and restarts it when the compiled files change (production or development, but for production you could use [pm2](http://pm2.keymetrics.io/) with `yarn standalone`) ;
495 | - `yarn cli`: shortcut to chuck's CLI (usage: `yarn cli -- command --arg1 --arg2`) ;
496 | - `yarn standalone`: starts the Express server without nodemon ;
497 | - `yarn lint`: checks code style on the TypeScript sources (recommended: install typescript and tslint extensions for your editor).
498 |
499 | So, basically, to start a development session, run in a terminal:
500 |
501 | ```
502 | yarn install
503 | yarn watch
504 | ```
505 |
506 | In a terminal aside of the first one, run:
507 |
508 | ```
509 | yarn start
510 | ```
511 |
512 | You can also create an `.env` file at the project root to override the default environment variables in `.env.defaults`.
513 |
--------------------------------------------------------------------------------
/chuck.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mitmadness/chuck/b9a201f5b2382c94d2507dc0b15143feecc71f2c/chuck.png
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "@mitm/chuck",
3 | "version": "0.0.0-development",
4 | "description": "chuck is a webservice that converts various 3D file-formats to Unity3D AssetBundles",
5 | "repository": "git@github.com:mitmadness/chuck.git",
6 | "author": "Morgan Touverey Quilling ",
7 | "license": "MIT",
8 | "main": "dist/entry/libchuck.js",
9 | "types": "dist/entry/libchuck.d.ts",
10 | "bin": {
11 | "chuck": "dist/entry/standalone.js",
12 | "chuck-cli": "dist/entry/cli.js"
13 | },
14 | "scripts": {
15 | "start": "nodemon dist/entry/standalone.js",
16 | "watch": "yarn copy-pug && tsc -w",
17 | "build": "yarn lint && tsc && yarn copy-pug",
18 | "cli": "node dist/entry/cli.js",
19 | "standalone": "node dist/entry/standalone.js",
20 | "lint": "tslint \"src/**/*.ts\" \"stubs/**/*.ts\"",
21 | "semantic-release": "semantic-release pre && npm publish --access public && semantic-release post",
22 | "test": "node test/test.js",
23 | "copy-pug": "cpy \"admin/*.pug\" \"../dist/\" --cwd=src --parents"
24 | },
25 | "devDependencies": {
26 | "@types/body-parser": "^1.16.3",
27 | "@types/boom": "^4.3.2",
28 | "@types/bull": "^2.1.0",
29 | "@types/cors": "^2.8.1",
30 | "@types/dotenv": "^4.0.0",
31 | "@types/express": "^4.0.35",
32 | "@types/mongoose": "^4.7.11",
33 | "@types/morgan": "^1.7.32",
34 | "@types/node-fetch": "^1.6.7",
35 | "@types/pify": "^0.0.28",
36 | "@types/raven": "^1.2.2",
37 | "@types/source-map-support": "^0.2.28",
38 | "@types/uuid": "^2.0.29",
39 | "@types/winston": "^2.3.1",
40 | "@types/yargs": "^6.6.0",
41 | "nodemon": "^1.11.0",
42 | "semantic-release": "^6.3.6",
43 | "tslint": "^5.1.0",
44 | "typescript": "^2.2.2"
45 | },
46 | "dependencies": {
47 | "@mitm/assetbundlecompiler": "^1.4.0",
48 | "@toverux/expresse": "^2.1.0",
49 | "@types/sanitize-filename": "^1.1.28",
50 | "azure-storage": "^2.1.0",
51 | "body-parser": "^1.17.1",
52 | "boom": "^5.1.0",
53 | "bull": "^2.2.6",
54 | "compose-middleware": "^2.2.0",
55 | "cors": "^2.8.3",
56 | "cpy-cli": "^1.0.1",
57 | "dotenv": "^4.0.0",
58 | "dotenv-extended": "^2.0.0",
59 | "express": "^4.15.2",
60 | "express-basic-auth": "^1.0.2",
61 | "mongoose": "^5.13.20",
62 | "morgan": "^1.8.1",
63 | "node-fetch": "^1.6.3",
64 | "pify": "^3.0.0",
65 | "pug": "^2.0.0-rc.1",
66 | "raven": "^2.1.0",
67 | "sanitize-filename": "^1.6.1",
68 | "source-map-support": "^0.4.15",
69 | "toureiro": "^0.2.13",
70 | "uuid": "^3.0.1",
71 | "winston": "^2.3.1",
72 | "yargs": "^8.0.1"
73 | }
74 | }
75 |
--------------------------------------------------------------------------------
/postman/chuck.postman_collection.json:
--------------------------------------------------------------------------------
1 | {
2 | "variables": [],
3 | "info": {
4 | "name": "Chuck",
5 | "_postman_id": "ae2ec073-82b3-a253-8ce8-84bbe667348c",
6 | "description": "",
7 | "schema": "https://schema.getpostman.com/json/collection/v2.0.0/collection.json"
8 | },
9 | "item": [
10 | {
11 | "name": "Push conversion request",
12 | "event": [
13 | {
14 | "listen": "test",
15 | "script": {
16 | "type": "text/javascript",
17 | "exec": [
18 | "if (responseCode.code === 202) {",
19 | " var conversion = JSON.parse(responseBody);",
20 | " postman.setGlobalVariable(\"lastConversionId\", conversion.code);",
21 | "}"
22 | ]
23 | }
24 | }
25 | ],
26 | "request": {
27 | "url": "{{baseUrl}}/api/conversions",
28 | "method": "POST",
29 | "header": [
30 | {
31 | "key": "Content-Type",
32 | "value": "application/json",
33 | "description": ""
34 | },
35 | {
36 | "key": "Authorization",
37 | "value": "Bearer {{apiKey}}",
38 | "description": ""
39 | }
40 | ],
41 | "body": {
42 | "mode": "raw",
43 | "raw": "{\n \"assetBundleName\": \"myassetbundle.ab2\",\n \"compilerOptions\": {\n \"targeting\": \"webgl\"\n },\n \"assetUrls\": [\n \"https://file/to/asset.fbx\",\n \"https://file/to/texture.png\"\n ],\n \"azure\": {\n \"host\": \"{{azureHost}}\",\n \"container\": \"{{azureContainer}}\",\n \"sharedAccessSignatureToken\": \"{{azureSharedAccessSignatureToken}}\"\n }\n}"
44 | },
45 | "description": "Crée une nouvelle demande de conversion, placée dans la queue de conversion."
46 | },
47 | "response": []
48 | },
49 | {
50 | "name": "Get conversion info",
51 | "request": {
52 | "url": "{{baseUrl}}/api/conversions/{{lastConversionId}}",
53 | "method": "GET",
54 | "header": [],
55 | "body": {},
56 | "description": "Récupère les informations sur une conversion, avec le statut du job dans la queue."
57 | },
58 | "response": []
59 | }
60 | ]
61 | }
--------------------------------------------------------------------------------
/postman/devel.postman_environment.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "631dbadb-2860-21f6-9563-f4f63a883827",
3 | "name": "Devel",
4 | "values": [
5 | {
6 | "enabled": true,
7 | "key": "baseUrl",
8 | "value": "http://localhost:3001",
9 | "type": "text"
10 | },
11 | {
12 | "enabled": true,
13 | "key": "apiKey",
14 | "value": "a2b48c5f-b7b8-4bf6-a34a-6003d0ea908b",
15 | "type": "text"
16 | },
17 | {
18 | "enabled": true,
19 | "key": "azureHost",
20 | "value": "secret",
21 | "type": "text"
22 | },
23 | {
24 | "enabled": true,
25 | "key": "azureContainer",
26 | "value": "secret",
27 | "type": "text"
28 | },
29 | {
30 | "enabled": true,
31 | "key": "azureSharedAccessSignatureToken",
32 | "value": "secret",
33 | "type": "text"
34 | }
35 | ],
36 | "timestamp": 1495197669447,
37 | "_postman_variable_scope": "environment",
38 | "_postman_exported_at": "2017-05-19T12:47:33.254Z",
39 | "_postman_exported_using": "Postman/4.10.7"
40 | }
--------------------------------------------------------------------------------
/src/admin/admin.pug:
--------------------------------------------------------------------------------
1 | html
2 | head
3 | title= 'Chuck Webservice Admin'
4 |
5 | body
6 | form(action='' method='post')
7 | input(type='submit' name='generateKey' value='Generate a new API key')
8 | table
9 | caption List of API Keys
10 |
11 | each val in keys
12 | tr
13 | th= val.key
14 | th
15 | form(action="admin/delete/"+val.key method='post')
16 | input(type='submit' name='deleteKey' value='Delete')
17 |
18 | a(href='/toureiro') Job list
19 |
--------------------------------------------------------------------------------
/src/admin/index.ts:
--------------------------------------------------------------------------------
1 | import * as express from 'express';
2 | import { ApiKey } from '../models/api_key';
3 | import { wrapAsync} from '../express_utils';
4 |
5 | const router: express.Router = express.Router();
6 |
7 | router.get('/', wrapAsync(async (req, res, next) => {
8 | const keys = await ApiKey.find({});
9 |
10 | res.render('admin/admin', { keys });
11 |
12 | next();
13 | }));
14 |
15 | router.post('/', wrapAsync(async (req, res, next) => {
16 | await ApiKey.create({});
17 |
18 | res.redirect('/admin');
19 |
20 | next();
21 | }));
22 |
23 | router.post('/delete/:key', wrapAsync(async (req, res, next) => {
24 | await ApiKey.findOneAndRemove({ key: req.params.key });
25 |
26 | res.redirect('/admin');
27 |
28 | next();
29 | }));
30 |
31 | export default router;
32 |
--------------------------------------------------------------------------------
/src/api/api_keys_cache.ts:
--------------------------------------------------------------------------------
1 | import { ApiKey } from '../models';
2 |
3 | const apiKeysCache: string[] = [];
4 |
5 | export async function isKeyValid(key: string): Promise {
6 | if (apiKeysCache.includes(key)) {
7 | return true;
8 | }
9 |
10 | const apiKeyDocument = await ApiKey.findOne({ key });
11 |
12 | if (apiKeyDocument) {
13 | apiKeysCache.push(key);
14 | return true;
15 | }
16 |
17 | return false;
18 | }
19 |
--------------------------------------------------------------------------------
/src/api/conversions_api.ts:
--------------------------------------------------------------------------------
1 | import { notFound } from 'boom';
2 | import * as express from 'express';
3 | import { Conversion, safeData as safeConversionData } from '../models/conversion';
4 | import converterQueue from '../converter/queue';
5 | import { wrapAsync, safeOutData } from '../express_utils';
6 | import conversionsEventsMiddleware from './conversions_api_events';
7 | import { hasValidApiKey } from './middlewares';
8 |
9 | const router: express.Router = express.Router();
10 |
11 | /**
12 | * Create a new conversion request.
13 | * Creates a Conversion document, and pushes it onto the queue - the job will start asap.
14 | */
15 | router.post('/', hasValidApiKey(), wrapAsync(async (req, res, next) => {
16 | const conversionData = safeConversionData(req.body);
17 | const conversion = await Conversion.create(conversionData);
18 |
19 | const job = await converterQueue.add(conversion);
20 |
21 | conversion.conversion.jobId = job.jobId;
22 | await conversion.save();
23 |
24 | res.status(202).json(safeOutData(conversion));
25 |
26 | next();
27 | }));
28 |
29 | /**
30 | * Retrieves a conversion request by it's code identifier.
31 | */
32 | router.get('/:code', wrapAsync(async (req, res, next) => {
33 | const conversion = await Conversion.findOne({ code: req.params.code }, { 'conversion.logs': false });
34 | if (!conversion) {
35 | throw notFound();
36 | }
37 |
38 | res.status(200).json(safeOutData(conversion));
39 |
40 | next();
41 | }));
42 |
43 | /**
44 | * A Server-Sent Events endpoint to get realtime events about the conversion.
45 | */
46 | router.get('/:code/events', conversionsEventsMiddleware);
47 |
48 | export default router;
49 |
--------------------------------------------------------------------------------
/src/api/conversions_api_events.ts:
--------------------------------------------------------------------------------
1 | import { sse, ISseResponse } from '@toverux/expresse';
2 | import { notFound, resourceGone } from 'boom';
3 | import { compose } from 'compose-middleware';
4 | import { Request, Response, NextFunction } from 'express';
5 | import converterQueue from '../converter/queue';
6 | import { IProgressReportJob } from '../converter/job';
7 | import { IEvent, isQueueConversionEndedEvent } from '../converter/job_events';
8 | import { Conversion, IConversionModel } from '../models/conversion';
9 | import { wrapAsync } from '../express_utils';
10 |
11 | interface InitSseRequest extends Request {
12 | isReplay: boolean;
13 | conversion: IConversionModel;
14 | }
15 |
16 | export default compose(
17 | //=> 1. Loads the conversion, checks request validity
18 | wrapAsync(loadConversion),
19 | //=> 2. Init SSE session
20 | sse(),
21 | //=> 3. Send conversion events (replay or realtime or both)
22 | wrapAsync(conversionSseEvents),
23 | //=> 4. Close connection
24 | terminateSseSession
25 | );
26 |
27 | async function loadConversion(req: InitSseRequest, res: Response, next: NextFunction): Promise {
28 | req.isReplay = req.query.replay !== 'false';
29 | req.conversion = await Conversion.findOne({ code: req.params.code });
30 |
31 | //=> Check that we can continue
32 | if (!req.conversion) {
33 | throw notFound();
34 | } else if (!req.isReplay && req.conversion.conversion.isCompleted) {
35 | throw resourceGone('This conversion is terminated');
36 | }
37 |
38 | next();
39 | }
40 |
41 | async function conversionSseEvents(req: InitSseRequest, res: ISseResponse, next: NextFunction): Promise {
42 | // @todo Unsafe typecast -- a Bull queue is an EventEmitter, but typings are not complete
43 | const emitterQueue = (converterQueue as any) as NodeJS.EventEmitter;
44 | const sse = sseSend.bind(null, req.query.sseType);
45 |
46 | //=> Replay mode: dump all progress records
47 | if (req.isReplay && req.conversion.conversion.logs) {
48 | req.conversion.conversion.logs.forEach(event => sse(res, event));
49 |
50 | // optim: conversion is terminated, don't watch for subsequent events
51 | if (req.conversion.conversion.isCompleted) return next();
52 | }
53 |
54 | //=> Listen queue for progress events, filter, and send if the jobId matches
55 | emitterQueue.on('progress', handleProgress);
56 |
57 | function handleProgress(job: IProgressReportJob, event: IEvent): void {
58 | if (job.id != req.conversion.conversion.jobId) return;
59 | sse(res, event);
60 |
61 | if (isQueueConversionEndedEvent(event)) {
62 | emitterQueue.removeListener('progress', handleProgress);
63 | next();
64 | }
65 | }
66 | }
67 |
68 | function terminateSseSession(req: Request, res: Response, next: NextFunction): void {
69 | res.end(); // avoid hanging for nothing, would be too easy to DDoS it
70 | next();
71 | }
72 |
73 | function sseSend(type: 'events'|'data', res: ISseResponse, event: IEvent): void {
74 | if (type == 'data') {
75 | res.sse.data(event);
76 | } else {
77 | const eventPayload = { ...event };
78 | delete eventPayload.type;
79 |
80 | res.sse.event(event.type, eventPayload);
81 | }
82 | }
83 |
--------------------------------------------------------------------------------
/src/api/index.ts:
--------------------------------------------------------------------------------
1 | import * as express from 'express';
2 | import * as raven from 'raven';
3 | import conversionsApi from './conversions_api';
4 | import { fatalErrorsHandler, recoverableErrorsHandler } from './middlewares';
5 |
6 | const router: express.Router = express.Router();
7 |
8 | //=> API endpoint roots
9 | router.use('/conversions', conversionsApi);
10 |
11 | //=> Install error handlers
12 | router.use(recoverableErrorsHandler());
13 | router.use(raven.errorHandler());
14 | router.use(fatalErrorsHandler());
15 |
16 | export default router;
17 |
--------------------------------------------------------------------------------
/src/api/middlewares.ts:
--------------------------------------------------------------------------------
1 | import { BoomError, unauthorized } from 'boom';
2 | import { ErrorRequestHandler, Handler, NextFunction, Request, Response } from 'express';
3 | import logger from '../logger';
4 | import { wrapAsync } from '../express_utils';
5 | import { safeErrorSerialize } from '../safe_error_serialize';
6 | import { isKeyValid } from './api_keys_cache';
7 |
8 | interface ISentryResponse extends Response {
9 | sentry?: string;
10 | }
11 |
12 | export function hasValidApiKey(): Handler {
13 | return wrapAsync(async (req: Request, res: Response, next: NextFunction): Promise => {
14 | const authHeader = req.header('Authorization');
15 | if (!authHeader || !authHeader.startsWith('Bearer ')) {
16 | throw unauthorized('The API key must be provided in the Authorization header, with scheme Bearer');
17 | }
18 |
19 | const apiKey = authHeader.split(' ')[1];
20 |
21 | if (!await isKeyValid(apiKey)) {
22 | throw unauthorized(`The API key ${apiKey} does not seem to exist`);
23 | }
24 |
25 | next();
26 | });
27 | }
28 |
29 | export function recoverableErrorsHandler(): ErrorRequestHandler {
30 | return (err: any, req: Request, res: Response, next: NextFunction): void => {
31 | //=> Headers already sent, let Express handle the thing
32 | if (res.headersSent) {
33 | return void next(err);
34 | }
35 |
36 | //=> Mongoose validation errors
37 | if (err.name === 'ValidationError') {
38 | return void res.status(400).json(err);
39 | }
40 |
41 | //=> Boom's HTTP errors
42 | const boomError = err as BoomError;
43 | if (boomError.isBoom) {
44 | return void res.status(boomError.output.statusCode).json({
45 | name: boomError.output.payload.error,
46 | message: boomError.message
47 | });
48 | }
49 |
50 | //=> We can't recognize this error, pass to the next error handler.
51 | next(err);
52 | };
53 | }
54 |
55 | export function fatalErrorsHandler(): ErrorRequestHandler {
56 | return (err: any, req: Request, res: ISentryResponse, next: NextFunction): void => {
57 | //=> We don't know what this error is, return a 500 error
58 | res.status(500).json(safeErrorSerialize(err, res.sentry));
59 |
60 | //=> Log it because that's, well, unexpected
61 | logger.error(err);
62 |
63 | next();
64 | };
65 | }
66 |
--------------------------------------------------------------------------------
/src/cli/api/generate_key.ts:
--------------------------------------------------------------------------------
1 | import { CommandModule } from 'yargs';
2 | import config from '../../config';
3 | import logger from '../../logger';
4 | import { connectDatabase, disconnectFromDatabase } from '../../mongoose';
5 | import { ApiKey } from '../../models';
6 |
7 | const command: CommandModule = {
8 | command: 'api:generate-key',
9 | describe: 'Generate an API key',
10 | builder: {
11 | save: {
12 | alias: 'S',
13 | type: 'boolean',
14 | desc: 'Save the API key to the database for direct use',
15 | default: false
16 | }
17 | },
18 | handler
19 | };
20 |
21 | interface IArgs {
22 | save: boolean;
23 | }
24 |
25 | async function handler(args: IArgs): Promise {
26 | const key = new ApiKey();
27 |
28 | logger.info(`Generated key: ${key.key}`);
29 |
30 | if (args.save) {
31 | await connectDatabase(config.mongoUrl);
32 | await key.save();
33 | await disconnectFromDatabase();
34 |
35 | logger.info('Key saved to database!');
36 | }
37 | }
38 |
39 | export default command;
40 |
--------------------------------------------------------------------------------
/src/cli/api/index.ts:
--------------------------------------------------------------------------------
1 | import { CommandModule } from 'yargs';
2 | import generateKeyCommand from './generate_key';
3 | import revokeKeyCommand from './revoke_key';
4 |
5 | export default [generateKeyCommand, revokeKeyCommand] as CommandModule[];
6 |
--------------------------------------------------------------------------------
/src/cli/api/revoke_key.ts:
--------------------------------------------------------------------------------
1 | import { CommandModule } from 'yargs';
2 | import config from '../../config';
3 | import logger from '../../logger';
4 | import { connectDatabase, disconnectFromDatabase } from '../../mongoose';
5 | import { ApiKey } from '../../models';
6 |
7 | const command: CommandModule = {
8 | command: 'api:revoke-key ',
9 | describe: 'Revoke an API key',
10 | handler
11 | };
12 |
13 | interface IArgs {
14 | key: string;
15 | }
16 |
17 | async function handler(args: IArgs): Promise {
18 | await connectDatabase(config.mongoUrl);
19 |
20 | await ApiKey.remove({ key: args.key });
21 |
22 | logger.info(`The key ${args.key} has been deleted or was nonexistent.`);
23 |
24 | await disconnectFromDatabase();
25 | }
26 |
27 | export default command;
28 |
--------------------------------------------------------------------------------
/src/cli/index.ts:
--------------------------------------------------------------------------------
1 | import * as apiCommands from './api';
2 |
3 | export default [apiCommands];
4 |
--------------------------------------------------------------------------------
/src/config.ts:
--------------------------------------------------------------------------------
1 | import * as dotenvx from 'dotenv-extended';
2 | import * as path from 'path';
3 |
4 | export type EnvType = 'development'|'production';
5 |
6 | export interface IChuckConfig {
7 | /**
8 | * The release number of chuck, as specified in the package.json.
9 | * This will only have a real value in the npm distribution of chuck -- value is otherwise "0.0.0-development".
10 | */
11 | release: string;
12 |
13 | /**
14 | * aka NODE_ENV. Configures the mode (`development` or `production`) in which the server is running.
15 | * - development: permissive CORS rules are set on the API
16 | * - production: timestamps in log messages and more verbose HTTP logs
17 | *
18 | * @default process.env.NODE_ENV || 'development'
19 | */
20 | env: EnvType;
21 |
22 | /**
23 | * Minimum log level (npm standard log levels are used).
24 | *
25 | * @see https://github.com/winstonjs/winston#logging-levels
26 | * @default 'verbose'
27 | */
28 | logLevel: string;
29 |
30 | /**
31 | * Chuck HTTP server port.
32 | *
33 | * @default 3001
34 | */
35 | serverPort: number;
36 |
37 | /**
38 | * Connection string to a MongoDB database.
39 | *
40 | * @default 'mongodb://localhost/chuck'
41 | */
42 | mongoUrl: string;
43 |
44 | /**
45 | * DSN for Sentry error reporting.
46 | */
47 | ravenDsn: string;
48 |
49 | /**
50 | * Redis connection informations.
51 | *
52 | * @default { host: 'localhost', port: 6379, db: 0 }
53 | */
54 | redis: { host: string; port: number; db: number };
55 |
56 | /**
57 | * Admin Web UIs configuration. Used by the admin interface and Toureiro.
58 | *
59 | * @default { enable: false, user: 'admin', password: 'admin' }
60 | */
61 | adminWebUis: { enable: boolean, user: string; password: string; };
62 |
63 | /**
64 | * Unity Editor path (if not installed in the standard path).
65 | *
66 | * @see https://github.com/mitmadness/AssetBundleCompiler#changing-unitys-executable-path
67 | * @default undefined (auto)
68 | */
69 | unityPath: string|undefined;
70 |
71 | /**
72 | * Azure configuration.
73 | *
74 | * @default { enableEmu: false }
75 | */
76 | azure: { enableEmu: boolean; };
77 |
78 | /**
79 | * An array of module names.
80 | * Those modules will be loaded dynamically as step plugins.
81 | */
82 | stepModulePlugins: string[];
83 | }
84 |
85 | //=> Load default environment variables with dotenv-extended
86 | dotenvx.load({
87 | defaults: path.resolve(`${__dirname}/../.env.defaults`)
88 | });
89 |
90 | //=> Determine release number
91 | // tslint:disable-next-line:no-var-requires
92 | const release = require('../package.json').version;
93 |
94 | //=> Hydrate config with the environment variables
95 | const config: IChuckConfig = {
96 | release,
97 | env: process.env.NODE_ENV || process.env.CHUCK_ENV,
98 | logLevel: process.env.CHUCK_LOGLEVEL,
99 | serverPort: parseInt(process.env.CHUCK_SERVERPORT, 10),
100 | mongoUrl: process.env.CHUCK_MONGOURL,
101 | ravenDsn: process.env.CHUCK_RAVENDSN,
102 | redis: {
103 | host: process.env.CHUCK_REDIS_HOST,
104 | port: parseInt(process.env.CHUCK_REDIS_PORT, 10),
105 | db: parseInt(process.env.CHUCK_REDIS_DB, 10)
106 | },
107 | adminWebUis: {
108 | enable: process.env.CHUCK_ADMINWEBUIS_ENABLE === 'true',
109 | user: process.env.CHUCK_ADMINWEBUIS_USER,
110 | password: process.env.CHUCK_ADMINWEBUIS_PASSWORD
111 | },
112 | unityPath: process.env.CHUCK_UNITYPATH,
113 | azure: {
114 | enableEmu: process.env.CHUCK_AZURE_ENABLEEMU === 'true'
115 | },
116 | stepModulePlugins: process.env.CHUCK_STEPMODULEPLUGINS ? process.env.CHUCK_STEPMODULEPLUGINS.split(',') : []
117 | };
118 |
119 | export default config;
120 |
--------------------------------------------------------------------------------
/src/converter/job.ts:
--------------------------------------------------------------------------------
1 | import { DocumentQuery } from 'mongoose';
2 | import { Job } from 'bull';
3 | import { Conversion } from '../models';
4 | import { IConversion } from '../models/IConversion';
5 | import { IConversionModel } from '../models/conversion';
6 | import { IEvent } from './job_events';
7 |
8 | /**
9 | * This interface is the common subset of a normal Job object (w/ methods) and a job in a progress report.
10 | */
11 | export interface IConversionDataJob {
12 | data: IConversion;
13 | }
14 |
15 | /**
16 | * This is a more-typed specialization of Bull's Job interface.
17 | * This is a normal Job object with methods like progress().
18 | */
19 | export interface IConversionJob extends Job {
20 | data: IConversion;
21 | progress(event: T): Promise;
22 | }
23 |
24 | /**
25 | * This is an imperfect subset of a normal Job object.
26 | * It has no methods (plain object) and has "id" instead of "jobId".
27 | * It is received is a progress event handler, for example.
28 | */
29 | export interface IProgressReportJob {
30 | id: string;
31 | data: IConversion;
32 | }
33 |
34 | export function updateConversion(job: IConversionDataJob, update: object): DocumentQuery {
35 | return Conversion.findOneAndUpdate({ code: job.data.code }, update);
36 | }
37 |
--------------------------------------------------------------------------------
/src/converter/job_events.ts:
--------------------------------------------------------------------------------
1 | import { safeErrorSerialize } from '../safe_error_serialize';
2 | import { IStepDescription } from './steps/step';
3 |
4 | export interface IEvent {
5 | type: string;
6 | message: string;
7 | }
8 |
9 | // QueueConversionStartEvent
10 | // -------------------------
11 |
12 | export function queueConversionStartEvent(message: string): IEvent {
13 | return { type: 'queue/conversion-start', message };
14 | }
15 |
16 | export function isQueueConversionStartEvent(event: IEvent): event is IEvent {
17 | return event.type == 'queue/conversion-start';
18 | }
19 |
20 | // QueueConversionEndedEvent
21 | // -------------------------
22 |
23 | export interface IQueueConversionEndedEvent extends IEvent {
24 | assetBundleUrl: string|null;
25 | error: any|null;
26 | }
27 |
28 | export function queueConversionEndedEvent(
29 | message: string,
30 | assetBundleUrl: string|null,
31 | error: any = null
32 | ): IQueueConversionEndedEvent {
33 | if (error) {
34 | error = safeErrorSerialize(error);
35 | }
36 |
37 | return { type: 'queue/conversion-ended', message, assetBundleUrl, error };
38 | }
39 |
40 | export function isQueueConversionEndedEvent(event: IEvent): event is IQueueConversionEndedEvent {
41 | return event.type == 'queue/conversion-ended';
42 | }
43 |
44 | // ProcessorStepChangeEvent
45 | // ------------------------
46 |
47 | export interface IProcessorStepChangeEvent extends IEvent {
48 | step: IStepDescription;
49 | }
50 |
51 | export function processorStepChangeEvent(message: string, step: IStepDescription): IProcessorStepChangeEvent {
52 | return { type: 'processor/step-change', message, step };
53 | }
54 |
55 | export function isProcessorStepChangeEvent(event: IEvent): event is IProcessorStepChangeEvent {
56 | return event.type == 'processor/step-change';
57 | }
58 |
59 | // ProcessorCleanupErrorEvent
60 | // --------------------------
61 |
62 | export interface IProcessorCleanupErrorEvent extends IEvent {
63 | step: IStepDescription;
64 | error: any;
65 | }
66 |
67 | export function processorCleanupErrorEvent(
68 | message: string,
69 | step: IStepDescription,
70 | error: any
71 | ): IProcessorCleanupErrorEvent {
72 | return { type: 'processor/cleanup-error', message, step, error };
73 | }
74 |
75 | export function isProcessorCleanupErrorEvent(event: IEvent): event is IProcessorCleanupErrorEvent {
76 | return event.type == 'processor/cleanup-error';
77 | }
78 |
79 | // ProcessorStepProgressEvent
80 | // --------------------------
81 |
82 | export interface IProcessorStepProgressEvent extends IEvent {
83 | [customKey: string]: any;
84 | }
85 |
86 | export function processorStepProgressEvent(
87 | stepCode: string,
88 | type: string,
89 | message: string,
90 | data?: any
91 | ): IProcessorStepProgressEvent {
92 | return { type: `processor/${stepCode}/${type}`, message, ...data };
93 | }
94 |
--------------------------------------------------------------------------------
/src/converter/plugin_steps.ts:
--------------------------------------------------------------------------------
1 | import { IStepModule } from './steps/step';
2 |
3 | const pluginSteps: IStepModule[] = [];
4 |
5 | export function register(step: IStepModule): void {
6 | pluginSteps.push(step);
7 | }
8 |
9 | export function all(): IStepModule[] {
10 | return pluginSteps;
11 | }
12 |
--------------------------------------------------------------------------------
/src/converter/queue.ts:
--------------------------------------------------------------------------------
1 | import * as queue from 'bull';
2 | import config from '../config';
3 | import coreSteps from './steps';
4 | import * as pluginSteps from './plugin_steps';
5 | import { processor } from './queue_processor';
6 | import * as handlers from './queue_event_handlers';
7 |
8 | //=> Get plug-in steps and merge with the core steps
9 | const steps = coreSteps.concat(pluginSteps.all());
10 |
11 | //=> Sort all steps with their declared priority
12 | const sortedSteps = steps.sort((a, b) => a.describe().priority - b.describe().priority);
13 |
14 | //=> Initialize the Bull queue
15 | const conversionsQueue = queue('chuck-conversion-queue', config.redis.port, config.redis.host);
16 |
17 | //=> Initialize the job processor for the conversions queue
18 | conversionsQueue.process(processor.bind(null, sortedSteps));
19 |
20 | //=> Queue-related events
21 | conversionsQueue.on('ready', handlers.onQueueReady);
22 | conversionsQueue.on('paused', handlers.onQueuePaused);
23 | conversionsQueue.on('resumed', handlers.onQueueResumed);
24 | conversionsQueue.on('error', handlers.onQueueError);
25 | conversionsQueue.on('cleaned', handlers.onQueueCleaned);
26 |
27 | //=> Job-related events
28 | conversionsQueue.on('progress', handlers.onJobProgress);
29 | conversionsQueue.on('active', handlers.onJobActive);
30 | conversionsQueue.on('completed', handlers.onJobCompleted);
31 | conversionsQueue.on('failed', handlers.onJobFailed);
32 |
33 | export default conversionsQueue;
34 |
--------------------------------------------------------------------------------
/src/converter/queue_event_handlers.ts:
--------------------------------------------------------------------------------
1 | import * as raven from 'raven';
2 | import config from '../config';
3 | import logger from '../logger';
4 | import { safeErrorSerialize } from '../safe_error_serialize';
5 | import { IConversionJob, IProgressReportJob, updateConversion } from './job';
6 | import { IEvent, isProcessorStepChangeEvent, queueConversionEndedEvent, queueConversionStartEvent } from './job_events';
7 |
8 | // GLOBAL QUEUE EVENTS
9 | //--------------------
10 |
11 | export function onQueueReady(): void {
12 | logger.info('convqueue: Now ready');
13 | }
14 |
15 | export function onQueuePaused(): void {
16 | logger.info('convqueue: Now paused');
17 | }
18 |
19 | export function onQueueResumed(): void {
20 | logger.info('convqueue: Now resumed');
21 | }
22 |
23 | export function onQueueError(err: any): void {
24 | logger.error('convqueue: The queue encountered an error!', err);
25 | }
26 |
27 | export function onQueueCleaned(jobs: IConversionJob[]): void {
28 | const jobIdsStr = jobs.map(job => job.jobId).join(', ');
29 | logger.info(`convqueue: ${jobs.length} terminated jobs have been deleted: ${jobIdsStr}`);
30 | }
31 |
32 | // JOB-RELATED QUEUE EVENTS
33 | //-------------------------
34 |
35 | export async function onJobProgress(job: IProgressReportJob, progress: IEvent): Promise {
36 | logger.verbose(`convqueue: job #${job.id} [${progress.type}] ${progress.message}`);
37 |
38 | let updateQuery = {
39 | $push: { 'conversion.logs': progress }
40 | };
41 |
42 | isProcessorStepChangeEvent(progress) && (updateQuery = {
43 | ...updateQuery,
44 | $set: { 'conversion.step': progress.step.code }
45 | });
46 |
47 | await updateConversion(job, updateQuery);
48 | }
49 |
50 | export async function onJobActive(job: IConversionJob): Promise {
51 | logger.verbose(`convqueue: job #${job.jobId} has started`, job.data);
52 |
53 | await job.progress(queueConversionStartEvent('Conversion started'));
54 | }
55 |
56 | export async function onJobCompleted(job: IConversionJob, assetBundleUrl: string): Promise {
57 | logger.verbose(`convqueue: job #${job.jobId} is completed`, job.data);
58 |
59 | await job.progress(queueConversionEndedEvent('Conversion terminated with success!', assetBundleUrl));
60 |
61 | await updateConversion(job, {
62 | $set: {
63 | 'conversion.isCompleted': true,
64 | 'conversion.step': null,
65 | 'conversion.assetBundleUrl': assetBundleUrl
66 | }
67 | });
68 | }
69 |
70 | export async function onJobFailed(job: IConversionJob, error: any): Promise {
71 | //=> Log & report on Sentry
72 | logger.error(`convqueue: job #${job.jobId} has failed!`, error);
73 |
74 | if (config.ravenDsn) {
75 | raven.captureException(error);
76 | }
77 |
78 | //=> Report in job's progress log
79 | const progressTask = job.progress(queueConversionEndedEvent(
80 | 'Conversion failed, an error occured!', null, error
81 | ));
82 |
83 | //=> Update the conversion document infos about progress.
84 | // We don't update conversion.step to let the client know where the fail occured.
85 | const updateTask = updateConversion(job, {
86 | $set: {
87 | 'conversion.isCompleted': true,
88 | 'conversion.error': safeErrorSerialize(error)
89 | }
90 | });
91 |
92 | await Promise.all([progressTask, updateTask]);
93 | }
94 |
--------------------------------------------------------------------------------
/src/converter/queue_processor.ts:
--------------------------------------------------------------------------------
1 | import { IConversionJob } from './job';
2 | import { IStepModule, IStepsContext } from './steps/step';
3 | import { processorCleanupErrorEvent, processorStepChangeEvent, processorStepProgressEvent } from './job_events';
4 |
5 | /**
6 | * The processor is the function that is passed to queue.process().
7 | * It iterates over conversion steps, handling context and errors (cleanup, etc).
8 | */
9 | export async function processor(steps: IStepModule[], job: IConversionJob): Promise {
10 | //=> Initialize context
11 | const stepsStack: IStepModule[] = [];
12 | const context: IStepsContext = { assetsPaths: [] };
13 | const cleanup = stepsCleanupProcessor.bind(null, stepsStack, job, context);
14 |
15 | //=> Execute all steps in order, sequentially
16 | for (const step of steps) {
17 | //=> Pass or record passage
18 | if (!step.shouldProcess(job.data, context)) continue;
19 | stepsStack.push(step);
20 |
21 | const stepInfo = step.describe();
22 |
23 | //=> Signal progress
24 | await job.progress(processorStepChangeEvent(`Starting "${stepInfo.name}"`, stepInfo));
25 |
26 | //=> Function for a step to signal its progress
27 | function stepSignalProgress(type: string, message: string, data: any = {}) {
28 | return job.progress(processorStepProgressEvent(stepInfo.code, type, message, data));
29 | }
30 |
31 | //=> Execute the step
32 | try {
33 | await step.process(job.data, context, stepSignalProgress);
34 | } catch (err) {
35 | await cleanup(`An error occured while running step ${stepInfo.code}`);
36 | throw err;
37 | }
38 | }
39 |
40 | //=> Perform cleanup
41 | await cleanup('All steps have terminated successfuly');
42 |
43 | return context.assetBundleUrl as string;
44 | }
45 |
46 | /**
47 | * The cleanup processor handles conversion artifacts cleanup.
48 | * It is called by the processor after a successful conversion, or just after an error, before re-throw.
49 | * It calls each step's cleanup() method (if any).
50 | */
51 | export async function stepsCleanupProcessor(
52 | stepsStack: IStepModule[],
53 | job: IConversionJob,
54 | context: IStepsContext,
55 | reason: string
56 | ): Promise {
57 | //=> Signal cleanup
58 | const stepNames = stepsStack.map(step => step.describe().code).join(', ');
59 |
60 | await job.progress(processorStepChangeEvent(
61 | `Performing cleanup for steps: ${stepNames} (${reason})`,
62 | { code: 'cleanup', name: 'Conversion artifacts cleanup', priority: Infinity }
63 | ));
64 |
65 | //=> Call each step's cleanup() function
66 | while (stepsStack.length) {
67 | const step = stepsStack.pop() as IStepModule;
68 | if (!step.cleanup) continue; // implementing cleanup() isn't mandatory
69 |
70 | try {
71 | await step.cleanup(context);
72 | } catch (error) {
73 | const stepInfo = step.describe();
74 |
75 | await job.progress(processorCleanupErrorEvent(
76 | `Error during calling cleanup() of step ${stepInfo.code}, ignoring`,
77 | stepInfo, error
78 | ));
79 | }
80 | }
81 | }
82 |
--------------------------------------------------------------------------------
/src/converter/steps/01_download_assets.ts:
--------------------------------------------------------------------------------
1 | import fetch from 'node-fetch';
2 | import * as fs from 'fs';
3 | import * as os from 'os';
4 | import * as path from 'path';
5 | import * as pify from 'pify';
6 | import { IConversion } from '../../models/IConversion';
7 | import { IStepDescription, IStepsContext, ProgressFn } from './step';
8 |
9 | export interface IDownloadAssetsStepsContext extends IStepsContext {
10 | assetsPaths: string[];
11 | downloadedAssetsDir: string;
12 | downloadedAssetsPaths: string[];
13 | }
14 |
15 | export function describe(): IStepDescription {
16 | return {
17 | code: 'download-assets',
18 | name: 'Download remote assets',
19 | priority: 10
20 | };
21 | }
22 |
23 | export function shouldProcess(conv: IConversion): boolean {
24 | return true;
25 | }
26 |
27 | export async function process(
28 | conv: IConversion,
29 | context: IDownloadAssetsStepsContext,
30 | progress: ProgressFn
31 | ): Promise {
32 | //=> Create a temporary folder for the assets
33 | const tmpDir = path.resolve(`${os.tmpdir()}/chuck-dl-assets-${Date.now()}`);
34 | await pify(fs.mkdir)(tmpDir);
35 |
36 | context.downloadedAssetsDir = tmpDir;
37 | context.downloadedAssetsPaths = [];
38 |
39 | //=> Start downloading all assets
40 | const downloads = conv.assetUrls.map(url => Promise.all([
41 | progress('start-download', `Downloading "${url}"`),
42 | downloadAndStoreAsset(context, url, tmpDir)
43 | ]));
44 |
45 | //=> Await downloads and check if there are errors.
46 | let remainingDownloads = downloads.length;
47 | const errors: any[] = [];
48 |
49 | return new Promise((resolve, reject) => {
50 | downloads.forEach((dl) => {
51 | dl.catch(err => errors.push(err)).then(() => {
52 | if (--remainingDownloads > 0) return;
53 |
54 | errors.length
55 | ? reject(new MasterDownloadError('Error(s) while downloading assets', errors))
56 | : resolve();
57 | });
58 | });
59 | });
60 | }
61 |
62 | export async function cleanup(context: Readonly): Promise {
63 | const rms = context.downloadedAssetsPaths.map(path => pify(fs.unlink)(path));
64 |
65 | await Promise.all(rms);
66 | await pify(fs.rmdir)(context.downloadedAssetsDir);
67 | }
68 |
69 | async function downloadAndStoreAsset(
70 | context: IDownloadAssetsStepsContext,
71 | assetUrl: string,
72 | directory: string
73 | ): Promise {
74 | //=> Find the dest path for the asset based on `directory`
75 | const fileName = assetUrl.split(/[?#]/)[0].split('/').reverse()[0];
76 | const filePath = path.resolve(`${directory}/${fileName}`);
77 |
78 | //=> Make HTTP request and create a write stream to the temp asset file
79 | const onlineAsset = await downloadAsset(assetUrl);
80 | const localAsset = fs.createWriteStream(filePath);
81 |
82 | context.downloadedAssetsPaths.push(filePath);
83 |
84 | //=> Pipe the HTTP response content to the write stream, then resolve
85 | return new Promise((resolve) => {
86 | onlineAsset.pipe(localAsset).on('finish', () => {
87 | localAsset.close();
88 |
89 | context.assetsPaths.push(filePath);
90 | resolve(filePath);
91 | });
92 | });
93 | }
94 |
95 | async function downloadAsset(assetUrl: string): Promise {
96 | const response = await fetch(assetUrl);
97 | if (!response.ok) {
98 | throw new Error(`Server responded with HTTP error code ${response.status} while downloading ${assetUrl}`);
99 | }
100 |
101 | return response.body;
102 | }
103 |
104 | class MasterDownloadError extends Error {
105 | constructor(message: string, public errors: any[]) {
106 | super(message);
107 |
108 | Object.setPrototypeOf(this, MasterDownloadError.prototype);
109 | }
110 | }
111 |
--------------------------------------------------------------------------------
/src/converter/steps/02_exec_assetbundlecompiler.ts:
--------------------------------------------------------------------------------
1 | import * as fs from 'fs';
2 | import * as os from 'os';
3 | import * as path from 'path';
4 | import * as pify from 'pify';
5 | import * as sanitize from 'sanitize-filename';
6 | import { bundle, setUnityPath } from '@mitm/assetbundlecompiler';
7 | import config from '../../config';
8 | import { IConversion } from '../../models/IConversion';
9 | import { IStepDescription, ProgressFn } from './step';
10 | import { IDownloadAssetsStepsContext } from './01_download_assets';
11 |
12 | export interface IExecAssetBundleCompilerStepContext extends IDownloadAssetsStepsContext {
13 | assetBundleDir: string;
14 | assetBundlePath: string;
15 | }
16 |
17 | export function describe(): IStepDescription {
18 | return {
19 | code: 'exec-assetbundlecompiler',
20 | name: 'Execute AssetBundleCompiler to assemble the asset bundle',
21 | priority: 20
22 | };
23 | }
24 |
25 | export function shouldProcess(conv: IConversion, context: IExecAssetBundleCompilerStepContext): boolean {
26 | return !!(context.assetsPaths && context.assetsPaths.length);
27 | }
28 |
29 | export async function process(
30 | conv: IConversion,
31 | context: IExecAssetBundleCompilerStepContext,
32 | progress: ProgressFn
33 | ): Promise {
34 | const tmpDir = path.resolve(`${os.tmpdir()}/chuck-exec-assetbundlecompiler-${Date.now()}`);
35 | const sanitizedBundleName = sanitize(conv.assetBundleName).trim().toLowerCase();
36 | const assetBundlePath = path.join(tmpDir, sanitizedBundleName);
37 |
38 | await pify(fs.mkdir)(tmpDir);
39 |
40 | context.assetBundleDir = tmpDir;
41 | context.assetBundlePath = assetBundlePath;
42 |
43 | if (config.unityPath) {
44 | setUnityPath(config.unityPath);
45 | }
46 |
47 | const options = conv.compilerOptions;
48 |
49 | await bundle(...context.assetsPaths)
50 | .targeting(options.targeting)
51 | .includingEditorScripts(...options.editorScripts)
52 | .withBuildOptions(options.buildOptions)
53 | .withLogger(async log => await progress('abcompiler-log', log))
54 | .to(assetBundlePath);
55 | }
56 |
57 | export async function cleanup(context: Readonly): Promise {
58 | await pify(fs.unlink)(context.assetBundlePath);
59 | await pify(fs.rmdir)(context.assetBundleDir);
60 | }
61 |
--------------------------------------------------------------------------------
/src/converter/steps/03_upload_bundle.ts:
--------------------------------------------------------------------------------
1 | import * as azure from 'azure-storage';
2 | import * as pify from 'pify';
3 | import * as sanitize from 'sanitize-filename';
4 | import config from '../../config';
5 | import { IConversion } from '../../models/IConversion';
6 | import { IStepDescription, ProgressFn } from './step';
7 | import { IExecAssetBundleCompilerStepContext as ICompilerStepContext } from './02_exec_assetbundlecompiler';
8 |
9 | export function describe(): IStepDescription {
10 | return {
11 | code: 'upload-bundle',
12 | name: 'Upload the AssetBundle on Azure',
13 | priority: 30
14 | };
15 | }
16 |
17 | export function shouldProcess(): boolean {
18 | return true;
19 | }
20 |
21 | export async function process(conv: IConversion, context: ICompilerStepContext, progress: ProgressFn): Promise {
22 | await progress('upload-start', `Uploading "${context.assetBundlePath}" to Azure`);
23 |
24 | const blobService = getBlobService(conv);
25 |
26 | const blobName = sanitize(conv.assetBundleName).trim().toLowerCase();
27 | const container = conv.azure.container;
28 |
29 | const createBlob = pify(blobService.createBlockBlobFromLocalFile.bind(blobService));
30 | const blobResult = await createBlob(container, blobName, context.assetBundlePath);
31 |
32 | const blobUrl = blobService.getUrl(container, blobName);
33 |
34 | await progress('upload-end', 'Upload terminated with success', { blobUrl, blobResult });
35 |
36 | context.assetBundleUrl = blobUrl;
37 | }
38 |
39 | function getBlobService(conv: IConversion): azure.BlobService {
40 | if (config.azure.enableEmu) {
41 | const devStoreCreds = azure.generateDevelopmentStorageCredentials();
42 | return azure.createBlobService(devStoreCreds);
43 | } else {
44 | return azure.createBlobServiceWithSas(conv.azure.host, conv.azure.sharedAccessSignatureToken);
45 | }
46 | }
47 |
--------------------------------------------------------------------------------
/src/converter/steps/index.ts:
--------------------------------------------------------------------------------
1 | import { IStepModule } from './step';
2 | import * as downloadAssets from './01_download_assets';
3 | import * as execAssetBundleCompiler from './02_exec_assetbundlecompiler';
4 | import * as uploadBundle from './03_upload_bundle';
5 |
6 | export default [downloadAssets, execAssetBundleCompiler, uploadBundle] as IStepModule[];
7 |
--------------------------------------------------------------------------------
/src/converter/steps/step.ts:
--------------------------------------------------------------------------------
1 | import { IConversion } from '../../models/IConversion';
2 |
3 | export type ProgressFn = (type: string, message: string, data?: any) => Promise;
4 |
5 | export interface IStepsContext {
6 | [customKey: string]: any;
7 | assetBundleUrl?: string; // mandatory ouput
8 | }
9 |
10 | export interface IStepDescription {
11 | code: string;
12 | name: string;
13 | priority: number;
14 | }
15 |
16 | export interface IStepModule {
17 | describe(): IStepDescription;
18 | shouldProcess(conversion: IConversion, context: IStepsContext): boolean;
19 | process(conversion: IConversion, context: IStepsContext, progress: ProgressFn): Promise;
20 | cleanup?(context: Readonly): Promise;
21 | }
22 |
--------------------------------------------------------------------------------
/src/entry/app_standalone.ts:
--------------------------------------------------------------------------------
1 | import * as os from 'os';
2 | import * as express from 'express';
3 | import * as cors from 'cors';
4 | import * as bodyParser from 'body-parser';
5 | import * as path from 'path';
6 | import * as morgan from 'morgan';
7 | import './bootstrap';
8 | import config from '../config';
9 | import logger, { morganStreamWriter } from '../logger';
10 | import { connectDatabase } from '../mongoose';
11 | import api from '../api';
12 | import converterQueue from '../converter/queue';
13 |
14 | //=> Resume the conversions queue
15 | converterQueue.resume().catch((error) => {
16 | logger.error(error.message);
17 | process.exit(1);
18 | });
19 |
20 | //=> Create an Express app
21 | const app = express();
22 | const port = config.serverPort;
23 |
24 | //=> Load the view engine Pug
25 | app.set('view engine', 'pug');
26 | app.set('views', path.resolve(`${__dirname}/..`));
27 |
28 | //=> Connect to the MongoDB database
29 | connectDatabase(config.mongoUrl).catch((error) => {
30 | logger.error(error.message);
31 | process.exit(1);
32 | });
33 |
34 | //=> Enable CORS
35 | app.use(cors());
36 |
37 | //=> Logging of HTTP requests with morgan
38 | const morganFormat = config.env == 'production'
39 | ? ':remote-addr - :method :url [:status], resp. :response-time ms, :res[content-length] bytes, referrer ":referrer"'
40 | : 'dev';
41 |
42 | app.use(morgan(morganFormat, { stream: morganStreamWriter }));
43 |
44 | //=> Decode JSON request bodies
45 | app.use(
46 | bodyParser.json(),
47 | bodyParser.urlencoded({ extended: true })
48 | );
49 |
50 | //=> Mount the API
51 | app.use('/api', api);
52 |
53 | if (config.adminWebUis.enable) {
54 | // Load web uis on-demand with require()
55 | // tslint:disable-next-line:no-var-requires
56 | app.use(require('../web_uis').default);
57 | }
58 |
59 | //=> Start the HTTP server
60 | app.listen(port, () => {
61 | logger.info(`🌍 Up and running @ http://${os.hostname()}:${port}`);
62 | logger.info(`Running for env: ${config.env}`);
63 | });
64 |
--------------------------------------------------------------------------------
/src/entry/bootstrap.ts:
--------------------------------------------------------------------------------
1 | import * as raven from 'raven';
2 | import * as sourceMapSupport from 'source-map-support';
3 | import config from '../config';
4 | import logger from '../logger';
5 |
6 | sourceMapSupport.install();
7 |
8 | config.ravenDsn && raven.config(config.ravenDsn, {
9 | release: config.release,
10 | environment: config.env
11 | }).install();
12 |
13 | process.on('unhandledRejection', (reason: any): void => {
14 | logger.error('UNHANDLED REJECTION', reason);
15 | });
16 |
17 | process.on('uncaughtException', (err: any): void => {
18 | logger.error('UNCAUGHT EXCEPTION', err);
19 | });
20 |
--------------------------------------------------------------------------------
/src/entry/cli.ts:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env node
2 | import * as yargs from 'yargs';
3 | import allCommands from '../cli/api';
4 |
5 | import './bootstrap';
6 |
7 | yargs.usage('Usage: $0 [arguments]');
8 |
9 | (allCommands as yargs.CommandModule[])
10 | .forEach(command => yargs.command(command));
11 |
12 | yargs.help().argv;
13 |
--------------------------------------------------------------------------------
/src/entry/libchuck.ts:
--------------------------------------------------------------------------------
1 | import defaultConfig, { IChuckConfig } from '../config';
2 |
3 | //=> Export IChuckConfig
4 | export { IChuckConfig, EnvType } from '../config';
5 |
6 | //=> Export core symbols for writing plugin steps
7 | export { IConversion } from '../models/IConversion';
8 | export { ProgressFn, IStepDescription, IStepsContext, IStepModule } from '../converter/steps/step';
9 |
10 | //=> Export core steps' context for reuse in plugin steps
11 | export { IDownloadAssetsStepsContext } from '../converter/steps/01_download_assets';
12 | export { IExecAssetBundleCompilerStepContext } from '../converter/steps/02_exec_assetbundlecompiler';
13 |
14 | export function boot(config: Partial = {}): void {
15 | Object.assign(defaultConfig, config);
16 |
17 | //=> Launch Chuck by its main entry point
18 | require('./standalone');
19 | }
20 |
--------------------------------------------------------------------------------
/src/entry/standalone.ts:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env node
2 | import config from '../config';
3 | import logger from '../logger';
4 | import * as pluginSteps from '../converter/plugin_steps';
5 |
6 | if (config.stepModulePlugins.length) {
7 | logger.info(`Loading plugins: ${config.stepModulePlugins.join(', ')}`);
8 | }
9 |
10 | //=> Require each plugin module buits name
11 | config.stepModulePlugins.map(require).forEach(pluginSteps.register);
12 |
13 | //=> Launch Chuck by its main entry point
14 | // tslint:disable-next-line:no-var-requires
15 | require('./app_standalone');
16 |
--------------------------------------------------------------------------------
/src/express_utils.ts:
--------------------------------------------------------------------------------
1 | import { Request, RequestHandler, Response, NextFunction } from 'express';
2 | import { Document } from 'mongoose';
3 |
4 | export function wrapAsync(handler: RequestHandler): RequestHandler {
5 | return (req: Request, res: Response, next: NextFunction) => handler(req, res, next).catch(next);
6 | }
7 |
8 | export function safeOutData(document: T): Partial {
9 | const safeDocument = document.toObject() as any;
10 |
11 | delete safeDocument.__v;
12 | delete safeDocument._id;
13 |
14 | return safeDocument;
15 | }
16 |
--------------------------------------------------------------------------------
/src/logger.ts:
--------------------------------------------------------------------------------
1 | import * as winston from 'winston';
2 | import { StreamOptions } from 'morgan';
3 | import config from './config';
4 |
5 | const logger = new winston.Logger({
6 | transports: [
7 | new winston.transports.Console({
8 | level: config.logLevel,
9 | json: false,
10 | colorize: true,
11 | timestamp: config.env == 'production'
12 | })
13 | ]
14 | });
15 |
16 | export default logger;
17 |
18 | export const morganStreamWriter: StreamOptions = {
19 | write(message: string): void {
20 | logger.info(message.replace(/\n$/, ''));
21 | }
22 | };
23 |
--------------------------------------------------------------------------------
/src/models/IApiKey.ts:
--------------------------------------------------------------------------------
1 | export interface IApiKey {
2 | key: string;
3 | }
4 |
--------------------------------------------------------------------------------
/src/models/IConversion.ts:
--------------------------------------------------------------------------------
1 | import { IBuildOptionsMap } from '@mitm/assetbundlecompiler';
2 | import { IEvent } from '../converter/job_events';
3 |
4 | export interface IConversion {
5 | code: string;
6 | assetBundleName: string;
7 | assetUrls: string[];
8 | conversionOptions: string[];
9 |
10 | azure: {
11 | host: string;
12 | sharedAccessSignatureToken: string;
13 | container: string;
14 | };
15 |
16 | compilerOptions: {
17 | targeting: string;
18 | buildOptions: IBuildOptionsMap;
19 | editorScripts: string[];
20 | };
21 |
22 | conversion: {
23 | jobId: string|null;
24 | isCompleted: boolean;
25 | step: string|null;
26 | error: any|null;
27 | assetBundleUrl: string|null;
28 | logs: IEvent[]
29 | };
30 | }
31 |
--------------------------------------------------------------------------------
/src/models/api_key.ts:
--------------------------------------------------------------------------------
1 | import * as mongoose from 'mongoose';
2 | import { Document, Schema } from 'mongoose';
3 | import * as uuid from 'uuid';
4 | import { IApiKey } from './IApiKey';
5 |
6 | export type IApiKeyModel = IApiKey & Document;
7 |
8 | const ApiKeySchema = new Schema({
9 | key: { type: String, unique: true, index: true, default: () => uuid.v4() }
10 | });
11 |
12 | export const ApiKey = mongoose.model('ApiKey', ApiKeySchema);
13 |
--------------------------------------------------------------------------------
/src/models/conversion.ts:
--------------------------------------------------------------------------------
1 | import * as mongoose from 'mongoose';
2 | import { Document, Schema } from 'mongoose';
3 | import * as uuid from 'uuid';
4 | import { IConversion } from './IConversion';
5 |
6 | export type IConversionModel = IConversion & Document;
7 |
8 | const ConversionSchema = new Schema({
9 | code: { type: String, unique: true, index: true, default: () => uuid.v4() },
10 |
11 | assetBundleName: { type: String, required: true },
12 |
13 | assetUrls: {
14 | type: Array,
15 | required: true,
16 | validate(urls: any[]) {
17 | return urls.every(value => typeof value === 'string');
18 | }
19 | },
20 |
21 | conversionOptions: {
22 | type: Array,
23 | default: [],
24 | validate(options: any[]) {
25 | return options.every(value => typeof value === 'string');
26 | }
27 | },
28 |
29 | azure: {
30 | host: { type: String, required: true },
31 | sharedAccessSignatureToken: { type: String, required: true },
32 | container : { type: String, required: true }
33 | },
34 |
35 | compilerOptions: {
36 | targeting: { type: String, required: true },
37 | buildOptions: {
38 | type: Schema.Types.Mixed,
39 | default: {}
40 | },
41 | editorScripts: {
42 | type: Array,
43 | default: [],
44 | validate(script: any[]) {
45 | return script.every(value => typeof value === 'string');
46 | }
47 | }
48 | },
49 |
50 | conversion: {
51 | jobId: { type: String, default: null },
52 | isCompleted: { type: Boolean, default: false },
53 | step: { type: String, default: null },
54 | error: { type: Schema.Types.Mixed, default: null },
55 | assetBundleUrl: { type: String, default: null },
56 | logs: [
57 | { type: Schema.Types.Mixed, default: [] }
58 | ]
59 | }
60 | }, { minimize: false });
61 |
62 | export function safeData({
63 | assetBundleName,
64 | assetUrls,
65 | conversionOptions,
66 | azure,
67 | compilerOptions
68 | }: IConversion): Partial {
69 | return { assetBundleName, assetUrls, conversionOptions, azure, compilerOptions };
70 | }
71 |
72 | export const Conversion = mongoose.model('Conversion', ConversionSchema);
73 |
--------------------------------------------------------------------------------
/src/models/index.ts:
--------------------------------------------------------------------------------
1 | export { ApiKey } from './api_key';
2 | export { Conversion } from './conversion';
3 |
--------------------------------------------------------------------------------
/src/mongoose.ts:
--------------------------------------------------------------------------------
1 | import * as mongoose from 'mongoose';
2 | import logger from './logger';
3 |
4 | // tslint:disable:only-arrow-functions
5 |
6 | //=> Tell Mongoose to use native V8 promises
7 | (mongoose as any).Promise = Promise;
8 |
9 | //=> Setup logging of queries
10 | mongoose.set('debug', function(collection: string, method: string, ...methodArgs: object[]) {
11 | // Basically extracted from https://goo.gl/OYCxAV (otherwise, Mongoose writes directly to stderr).
12 | const args: string[] = [];
13 | for (let j = methodArgs.length - 1; j >= 0; --j) {
14 | if (this.$format(methodArgs[j]) || args.length) {
15 | args.unshift(this.$format(methodArgs[j]));
16 | }
17 | }
18 |
19 | logger.debug(`mongo: ${collection}.${method}(${args.join(', ')})`);
20 | });
21 |
22 | export function connectDatabase(mongoUrl: string): Promise {
23 | return new Promise((resolve, reject) => {
24 | mongoose.connect(mongoUrl, (error) => {
25 | error
26 | ? reject(new Error(`Could not connect to the MongoDB instance! ${error.message}`))
27 | : resolve();
28 | });
29 | });
30 | }
31 |
32 | export function disconnectFromDatabase(): Promise {
33 | return new Promise((resolve, reject) => {
34 | mongoose.disconnect(error => error ? reject() : resolve());
35 | });
36 | }
37 |
--------------------------------------------------------------------------------
/src/safe_error_serialize.ts:
--------------------------------------------------------------------------------
1 | export interface IPlainObjectError {
2 | name: string;
3 | message: string;
4 | errorId?: string;
5 | errors?: any[];
6 | }
7 |
8 | export function safeErrorSerialize(error: any, sentryErrorId?: string): IPlainObjectError {
9 | const plainError: any = {};
10 |
11 | //=> Error name
12 | plainError.name = error.name || error.constructor.name;
13 |
14 | //=> Error message
15 | plainError.message = error.message || 'Unknown error';
16 |
17 | //=> Set error's ID if given
18 | if (sentryErrorId) {
19 | plainError.errorId = sentryErrorId;
20 | }
21 |
22 | //=> Serialize recursively sub-errors if any
23 | if (error.errors) {
24 | plainError.errors = error.errors.map(safeErrorSerialize);
25 | }
26 |
27 | return plainError;
28 | }
29 |
--------------------------------------------------------------------------------
/src/web_uis.ts:
--------------------------------------------------------------------------------
1 | import * as basicAuth from 'express-basic-auth';
2 | import * as express from 'express';
3 | import * as toureiro from 'toureiro';
4 | import admin from './admin';
5 | import config from './config';
6 |
7 | const router: express.Router = express.Router();
8 |
9 | //=> Set a middleware that authenticate administrators
10 | const adminAuth = basicAuth({
11 | challenge: true,
12 | realm: 'Chuck Administration',
13 | users: { [config.adminWebUis.user]: config.adminWebUis.password }
14 | });
15 |
16 | //=> Mount Toureiro
17 | router.use('/toureiro', adminAuth, toureiro({
18 | redis: { host: config.redis.host, port: config.redis.port, db: 0 }
19 | }));
20 |
21 | //=> Route of the admin interface
22 | router.use('/admin', adminAuth, admin);
23 |
24 | export default router;
25 |
--------------------------------------------------------------------------------
/stubs/toureiro.d.ts:
--------------------------------------------------------------------------------
1 | declare module 'toureiro' {
2 | import { RequestHandler } from 'express';
3 |
4 | function toureiro(options: toureiro.IRedisOptions): RequestHandler;
5 |
6 | namespace toureiro {
7 | interface IRedisOptions {
8 | redis: { host: string, port: number, db: number, auth_pass?: string };
9 | }
10 | }
11 |
12 | export = toureiro;
13 | }
14 |
--------------------------------------------------------------------------------
/test/test.js:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mitmadness/chuck/b9a201f5b2382c94d2507dc0b15143feecc71f2c/test/test.js
--------------------------------------------------------------------------------
/test/test.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | echo "Hello, world!"
4 |
--------------------------------------------------------------------------------
/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compileOnSave": false,
3 | "buildOnSave": false,
4 | "compilerOptions": {
5 | "outDir": "dist",
6 | "target": "es2015",
7 | "module": "commonjs",
8 | "moduleResolution": "node",
9 | "declaration": true,
10 | "removeComments": true,
11 | "emitDecoratorMetadata": true,
12 | "experimentalDecorators": true,
13 | "sourceMap": true,
14 | "noImplicitAny": true,
15 | "noUnusedLocals": true,
16 | "noUnusedParameters": false,
17 | "strictNullChecks": true,
18 | "lib": ["es7"]
19 | },
20 | "include": ["src/**/*", "stubs/*.d.ts"],
21 | "exclude": ["node_modules"]
22 | }
23 |
--------------------------------------------------------------------------------
/tslint.json:
--------------------------------------------------------------------------------
1 | {
2 | "extends": "tslint:latest",
3 | "rules": {
4 | "curly": false,
5 | "quotemark": ["double"],
6 | "ordered-imports": false,
7 | "object-literal-sort-keys": false,
8 | "comment-format": [],
9 | "triple-equals": false,
10 | "trailing-comma": [true, { "multiline": "never", "singleline": "never" }],
11 | "arrow-parens": false,
12 | "new-parens": false,
13 | "no-unused-expression": false
14 | }
15 | }
16 |
--------------------------------------------------------------------------------