├── logs
    └── .gitkeep
├── repos
    └── .gitkeep
├── .nvmrc
├── .prettierignore
├── .dockerignore
├── renovate.json
├── logo.png
├── .gitignore
├── .vscode
    ├── extensions.json
    ├── tasks.json
    ├── settings.json
    └── launch.json
├── Makefile
├── .prettierrc
├── src
    ├── types.d.ts
    ├── constants.ts
    ├── stats.ts
    ├── daemon.ts
    ├── error.ts
    ├── app
    │   ├── log-details.tsx
    │   ├── index.tsx
    │   ├── log.tsx
    │   ├── util.ts
    │   ├── app-shell.tsx
    │   ├── local-images.tsx
    │   └── debug.tsx
    ├── logger.ts
    ├── config.ts
    ├── middlewares.ts
    ├── image-runner.ts
    ├── builder.ts
    ├── index.ts
    └── api.ts
├── test
    ├── __mocks__
    │   └── nodegit.js
    ├── api.test.ts
    └── logger.test.ts
├── tsconfig.json
├── Dockerfile
├── .circleci
    └── config.yml
├── package.json
└── README.md


/logs/.gitkeep:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/repos/.gitkeep:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/.nvmrc:
--------------------------------------------------------------------------------
1 | v12.16.3
2 | 


--------------------------------------------------------------------------------
/.prettierignore:
--------------------------------------------------------------------------------
1 | .vscode
2 | 


--------------------------------------------------------------------------------
/.dockerignore:
--------------------------------------------------------------------------------
1 | node_modules
2 | .git
3 | logs
4 | repos
5 | 


--------------------------------------------------------------------------------
/renovate.json:
--------------------------------------------------------------------------------
1 | {
2 | 	"extends": [ "config:base" ]
3 | }
4 | 


--------------------------------------------------------------------------------
/logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Automattic/dserve/HEAD/logo.png


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules
2 | yarn-error.log
3 | build
4 | logs/*
5 | repos/*
6 | 


--------------------------------------------------------------------------------
/.vscode/extensions.json:
--------------------------------------------------------------------------------
1 | {
2 |     "recommendations": [
3 |         "eg2.tslint",
4 |         "streetsidesoftware.code-spell-checker"
5 |     ]
6 | }


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | build:
2 | 	docker build -t dserve .
3 | 
4 | run:
5 | 	docker run -it --rm -p 80:3000 -v /var/run/docker.sock:/var/run/docker.sock dserve
6 | 
7 | .PHONY: build


--------------------------------------------------------------------------------
/.prettierrc:
--------------------------------------------------------------------------------
 1 | useTabs: true
 2 | tabWidth: 2
 3 | printWidth: 100
 4 | singleQuote: true
 5 | trailingComma: es5
 6 | bracketSpacing: true
 7 | parenSpacing: true
 8 | jsxBracketSameLine: false
 9 | semi: true
10 | arrowParens: avoid
11 | 


--------------------------------------------------------------------------------
/src/types.d.ts:
--------------------------------------------------------------------------------
 1 | declare module 'docker-parse-image' {
 2 | 	export default function parse(
 3 | 		imgName: string
 4 | 	): {
 5 | 		registry: string;
 6 | 		namespace: string;
 7 | 		repository: string;
 8 | 		tag: string;
 9 | 	};
10 | }
11 | 


--------------------------------------------------------------------------------
/test/__mocks__/nodegit.js:
--------------------------------------------------------------------------------
1 | /*
2 |  * Since nodegit is a native module, it gives jest all kinds of hell.
3 |  * This makes it so that it never actually gets used in any test...whatsoever
4 |  */
5 | module.exports = jest.fn();
6 | module.exports.enableThreadSafety = jest.fn();


--------------------------------------------------------------------------------
/src/constants.ts:
--------------------------------------------------------------------------------
1 | export const ONE_SECOND = 1000;
2 | export const ONE_MINUTE = 60 * ONE_SECOND;
3 | export const FIVE_MINUTES = 5 * ONE_MINUTE;
4 | export const TEN_MINUTES = 10 * ONE_MINUTE;
5 | export const CONTAINER_EXPIRY_TIME = 60 * ONE_MINUTE;
6 | 
7 | export const START_TIME = Date.now();
8 | 


--------------------------------------------------------------------------------
/tsconfig.json:
--------------------------------------------------------------------------------
 1 | {
 2 | 	"compilerOptions": {
 3 | 		"module": "commonjs",
 4 | 		"target": "es6",
 5 | 		"noImplicitAny": true,
 6 | 		"removeComments": true,
 7 | 		"sourceMap": true,
 8 | 		"outDir": "build",
 9 | 		"jsx": "react",
10 | 		"lib": [ "es7", "dom" ],
11 | 		"esModuleInterop": true
12 | 	},
13 | 	"include": [ "src/*" ]
14 | }
15 | 


--------------------------------------------------------------------------------
/.vscode/tasks.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     // See https://go.microsoft.com/fwlink/?LinkId=733558
 3 |     // for the documentation about the tasks.json format
 4 |     "version": "2.0.0",
 5 |     "tasks": [
 6 |         {
 7 |             "type": "npm",
 8 |             "script": "build",
 9 |             "group": {
10 |                 "kind": "build",
11 |                 "isDefault": true
12 |             }
13 |         }
14 |     ]
15 | }


--------------------------------------------------------------------------------
/.vscode/settings.json:
--------------------------------------------------------------------------------
 1 | // Place your settings in this file to overwrite default and user settings.
 2 | {
 3 |     "search.exclude": {
 4 |         "**/node_modules": true,
 5 |         "**/bower_components": true,
 6 |         "**/dist": true
 7 |     },
 8 |     "typescript.referencesCodeLens.enabled": true,
 9 |     "tslint.ignoreDefinitionFiles": false,
10 |     "tslint.autoFixOnSave": true,
11 |     "tslint.exclude": "**/node_modules/**/*" 
12 | }


--------------------------------------------------------------------------------
/src/stats.ts:
--------------------------------------------------------------------------------
 1 | import { StatsD } from 'hot-shots';
 2 | 
 3 | const statsd = new StatsD( {
 4 | 	host: process.env.STATSD_HOST || 'localhost',
 5 | 	port: +process.env.STATSD_PORT || 8125,
 6 | } );
 7 | 
 8 | export function increment( stat: string ) {
 9 | 	statsd.increment( `dserve.${ stat }` );
10 | }
11 | 
12 | export function decrement( stat: string ) {
13 | 	statsd.decrement( `dserve.${ stat }` );
14 | }
15 | 
16 | export function gauge( stat: string, value: number ) {
17 | 	statsd.gauge( `dserve.${ stat }`, value );
18 | }
19 | 
20 | export function timing( stat: string, value: number ) {
21 | 	statsd.timing( `dserve.${ stat }`, value );
22 | }
23 | 


--------------------------------------------------------------------------------
/src/daemon.ts:
--------------------------------------------------------------------------------
 1 | import forever from 'forever-monitor';
 2 | 
 3 | import { l } from './logger';
 4 | 
 5 | const child = new forever.Monitor( 'build/index.js', {
 6 | 	watch: false,
 7 | 	silent: false,
 8 | 	max: Infinity,
 9 | 	minUptime: 2000,
10 | } );
11 | 
12 | child.on( 'error', err => {
13 | 	l.error( { err }, 'forever: Error during run' );
14 | } );
15 | 
16 | child.on( 'restart', () => {
17 | 	l.info( 'forever: Restarting' );
18 | } );
19 | 
20 | child.on( 'exit:code', ( code, signal ) => {
21 | 	l.info( { code, signal }, 'forever: exited child', code, signal );
22 | } );
23 | 
24 | child.on( 'exit', ( child, spinning ) => {
25 | 	l.info( { child, spinning }, 'forever: really exited', child, spinning );
26 | } );
27 | 
28 | child.on( 'stop', childData => {
29 | 	l.info( { data: childData }, 'forever: child stopping' );
30 | } );
31 | 
32 | child.start();
33 | 


--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
 1 | # THIS DOES NOT CURRENTLY WORK
 2 | # Docker for Mac and Docker for Linux have different networking configuration requirements for making dserve work
 3 | # On Mac all of docker runs in a vm, in Linux it can kind-of run on the host.
 4 | #
 5 | # We need the dserve container to share a localhost with the host in order to proxy to various other containers
 6 | 
 7 | from node:alpine
 8 | LABEL maintainer="Automattic"
 9 | 
10 | # All for installing dependencies of nodegit
11 | RUN apk update && \
12 |     apk upgrade && \
13 |     apk add git libgit2-dev && \
14 |     apk add python tzdata pkgconfig build-base && \
15 |     yarn install --production nodegit
16 | 
17 | # install rest of dependencies
18 | COPY package.json yarn.lock tsconfig.json ./
19 | RUN yarn --production
20 | 
21 | COPY src ./src
22 | RUN mkdir logs
23 | RUN mkdir repos
24 | 
25 | RUN yarn build-ts
26 | 
27 | CMD yarn serve:forever
28 | 


--------------------------------------------------------------------------------
/.circleci/config.yml:
--------------------------------------------------------------------------------
 1 | # Javascript Node CircleCI 2.0 configuration file
 2 | #
 3 | # Check https://circleci.com/docs/2.0/language-javascript/ for more details
 4 | #
 5 | version: 2
 6 | jobs:
 7 |   build:
 8 |     docker:
 9 |       - image: circleci/node:10
10 | 
11 |     working_directory: ~/repo
12 | 
13 |     steps:
14 |       - checkout
15 | 
16 |       # Download and cache dependencies
17 |       - restore_cache:
18 |           key: v2-dependencies-{{ checksum "yarn.lock" }}
19 | 
20 |       - run:
21 |           name: Install node dependencies
22 |           command: |
23 |             if [ ! -d node_modules ]; then
24 |               yarn
25 |             fi
26 | 
27 |       - save_cache:
28 |           key: v2-dependencies-{{ checksum "yarn.lock" }}
29 |           paths:
30 |             - node_modules
31 | 
32 |       # Ensure TypeScript builds
33 |       - run: yarn run build-ts
34 | 
35 |       # run tests!
36 |       - run: yarn test
37 | 
38 | 


--------------------------------------------------------------------------------
/src/error.ts:
--------------------------------------------------------------------------------
 1 | 
 2 | export class ContainerError extends Error {
 3 |     containerName: string;
 4 |     constructor(containerName:string, message:string) {
 5 |         super(message);
 6 |         this.containerName = containerName;
 7 |         Error.captureStackTrace(this, ContainerError);
 8 |     }
 9 | }
10 | 
11 | export class ImageError extends Error {
12 |     imageName: string;
13 |     constructor(imageName:string, message:string) {
14 |         super(message);
15 |         this.imageName = imageName;
16 |         Error.captureStackTrace(this, ImageError);
17 |     }
18 | }
19 | 
20 | export class ImageNotFound extends ImageError {
21 |     constructor(name:string) {
22 |         super(name, "Docker image not found");
23 |     }
24 | }
25 | 
26 | export class InvalidImage extends ImageError {
27 |     constructor(name:string) {
28 |         super(name, "Image is invalid");
29 |     }
30 | }
31 | 
32 | export class InvalidRegistry extends Error {
33 |     registry: string;
34 |     constructor(registry:string) {
35 |         super("Docker registry is invalid");
36 |         this.registry = registry;
37 |         Error.captureStackTrace(this, InvalidRegistry);
38 |     }
39 | }
40 | 
41 | 
42 | 


--------------------------------------------------------------------------------
/.vscode/launch.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     // Use IntelliSense to learn about possible Node.js debug attributes.
 3 |     // Hover to view descriptions of existing attributes.
 4 |     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
 5 |     "version": "0.2.0",
 6 |     "configurations": [
 7 |         {
 8 |             "name": "Launch",
 9 |             "request": "launch",
10 |             "runtimeArgs": [
11 |                 "run",
12 |                 "serve:debug"
13 |             ],
14 |             "runtimeExecutable": "yarn",
15 |             "skipFiles": [
16 |                 "<node_internals>/**"
17 |             ],
18 |             "outFiles": [
19 |                 "${workspaceFolder}/build/**/*.js",
20 |                 "!**/node_modules/**"
21 |             ],
22 |             "preLaunchTask": "npm: build-ts",
23 |             "type": "pwa-node",
24 |             "outputCapture": "std"
25 |         },
26 |         {
27 |             "request": "launch",
28 |             "name": "Run tests",
29 |             "program": "${workspaceFolder}/node_modules/.bin/jest",
30 |             "skipFiles": [
31 |                 "<node_internals>/**"
32 |             ],
33 |               "args": [
34 |                 "--runInBand",
35 |                 "${file}"
36 |             ],
37 |             "type": "pwa-node",
38 |         }
39 |     ]
40 | }
41 | 
42 | 


--------------------------------------------------------------------------------
/src/app/log-details.tsx:
--------------------------------------------------------------------------------
 1 | import React from 'react';
 2 | import { humanTimeSpan } from './util';
 3 | 
 4 | const interestingDetails = new Set( [
 5 | 	'buildConcurrency',
 6 | 	'buildImageTime',
 7 | 	'checkoutTime',
 8 | 	'cloneTime',
 9 | 	'code',
10 | 	'commitHash',
11 | 	'containerId',
12 | 	'data',
13 | 	'err',
14 | 	'error',
15 | 	'freePort',
16 | 	'imageName',
17 | 	'reason',
18 | 	'signal',
19 | 	'success',
20 | ] );
21 | 
22 | const serialize = ( value: any, key: string ) => {
23 | 	switch ( key ) {
24 | 		case 'buildImageTime':
25 | 		case 'checkoutTime':
26 | 		case 'cloneTime':
27 | 			return humanTimeSpan( +value );
28 | 		case 'commitHash':
29 | 			return (
30 | 				<React.Fragment>
31 | 					<a href={ `/?hash=${ value }` } title={ value }>
32 | 						{ value.substr( 0, 8 ) }
33 | 					</a>{' '}
34 | 					<a href={ `https://github.com/Automattic/wp-calypso/commit/${ value }` }>(github)</a>
35 | 				</React.Fragment>
36 | 			);
37 | 		default:
38 | 			return typeof value === 'object' ? JSON.stringify( value, null, 2 ) : value.toString();
39 | 	}
40 | };
41 | 
42 | const LogDetails = ( { data, details }: any ) => {
43 | 	details = details || interestingDetails;
44 | 	const detailsToShow = new Map();
45 | 	for ( let detail of details ) {
46 | 		if ( data[ detail ] ) {
47 | 			detailsToShow.set( detail, data[ detail ] );
48 | 		}
49 | 	}
50 | 	if ( detailsToShow.size === 0 ) {
51 | 		return null;
52 | 	}
53 | 	return (
54 | 		<div className="details">
55 | 			{ Array.from( detailsToShow.entries() ).map( ( [ key, value ] ) => (
56 | 				<pre key={ key }>
57 | 					{ key }: { serialize( value, key ) }
58 | 				</pre>
59 | 			) ) }
60 | 		</div>
61 | 	);
62 | };
63 | 
64 | export default LogDetails;
65 | 


--------------------------------------------------------------------------------
/src/app/index.tsx:
--------------------------------------------------------------------------------
 1 | import React from 'react';
 2 | import ReactDOMServer from 'react-dom/server';
 3 | import { ONE_SECOND } from '../constants';
 4 | 
 5 | import { Shell } from './app-shell';
 6 | import stripAnsi from 'strip-ansi';
 7 | import LogDetails from './log-details';
 8 | 
 9 | const BUILD_LOG_DETAILS = new Set( [ 'error', 'err' ] );
10 | 
11 | class BuildLog extends React.Component< { log: string } > {
12 | 	render() {
13 | 		const { log } = this.props;
14 | 		const formattedLog = log
15 | 			.trim()
16 | 			.split( '\n' )
17 | 			.map( str => {
18 | 				try {
19 | 					const line = JSON.parse( str );
20 | 					return [ line, `Time=${ line.time } | ${ line.msg }` ];
21 | 				} catch ( err ) {
22 | 					return [ {}, '' ];
23 | 				}
24 | 			} )
25 | 			.map( ( [ data, str ], i ) => (
26 | 				<li key={ i }>
27 | 					{ stripAnsi( str ) }
28 | 					<LogDetails data={ data } details={ BUILD_LOG_DETAILS } />
29 | 				</li>
30 | 			) );
31 | 		return <ol>{ formattedLog }</ol>;
32 | 	}
33 | }
34 | 
35 | const App = ( { buildLog, message, startedServerAt }: RenderContext ) => (
36 | 	<Shell refreshInterval={ 3 * ONE_SECOND } startedServerAt={ startedServerAt } showReset={ true }>
37 | 		<div
38 | 			dangerouslySetInnerHTML={ {
39 | 				__html: `
40 | 			<style>
41 |                 .dserve-toolbar a {
42 | 					transition: background 200ms ease-in;
43 | 				}
44 | 
45 | 				@keyframes progress-bar-animation {
46 | 						0%   { background-position: 400px 50px; }
47 | 						100% {  }
48 | 				}
49 | 			</style>
50 | 		`,
51 | 			} }
52 | 		/>
53 | 		<pre>
54 | 			{ buildLog && <BuildLog log={ buildLog } /> }
55 | 			{ message && <p> { message }</p> }
56 | 		</pre>
57 | 	</Shell>
58 | );
59 | 
60 | type RenderContext = { buildLog?: string; message?: string; startedServerAt: Date };
61 | export default function renderApp( renderContext: RenderContext ) {
62 | 	return ReactDOMServer.renderToStaticNodeStream( <App { ...renderContext } /> );
63 | }
64 | 


--------------------------------------------------------------------------------
/src/logger.ts:
--------------------------------------------------------------------------------
 1 | import bunyan from 'bunyan';
 2 | import _ from 'lodash';
 3 | import { Writable } from 'stream';
 4 | 
 5 | import { CommitHash, getImageName } from './api';
 6 | import { getLogPath } from './builder';
 7 | import { config } from './config';
 8 | 
 9 | export const ringbuffer = new bunyan.RingBuffer( { limit: 1000 } );
10 | 
11 | const dserveLogger = bunyan.createLogger( {
12 | 	name: 'dserve',
13 | 	streams: [
14 | 		{
15 | 			type: 'rotating-file',
16 | 			path: './logs/log.txt',
17 | 			period: '1d',
18 | 			count: 7,
19 | 		},
20 | 		{
21 | 			level: bunyan.DEBUG,
22 | 			type: 'raw',
23 | 			stream: ringbuffer,
24 | 		},
25 | 		{
26 | 			stream: process.stdout,
27 | 			level: bunyan.DEBUG,
28 | 		},
29 | 	],
30 | 	serializers: bunyan.stdSerializers, // allows one to use err, req, and res as special keys
31 | 	src: true,
32 | } );
33 | 
34 | /* super convenient name */
35 | export const l = dserveLogger;
36 | 
37 | /**
38 |  * Creates a child logger that outputs to the build directory and
39 |  * outputs errors to the console.
40 |  *
41 |  * @param commitHash - hash to make logger for
42 |  */
43 | export function getLoggerForBuild( commitHash: CommitHash ) {
44 | 	const path = getLogPath( commitHash );
45 | 	const logger = dserveLogger.child( {
46 | 		streams: [
47 | 			{
48 | 				type: 'file',
49 | 				path,
50 | 			},
51 | 			{
52 | 				stream: process.stdout,
53 | 				level: bunyan.INFO,
54 | 			},
55 | 		],
56 | 		commitHash,
57 | 		imageName: getImageName( commitHash ),
58 | 	} );
59 | 
60 | 	// we want it to be a child so that
61 | 	// it inherits al the same properties as the parent
62 | 	// except we don't want any of the parents streams
63 | 	// so this line removes them all
64 | 	// @ts-ignore this needs to be fixed with proper typing
65 | 	( ( logger as any ) as Logger ).streams = _.filter( ( logger as any ).streams, { path } );
66 | 
67 | 	return logger;
68 | }
69 | 
70 | type Logger = { streams: Array< { stream: Writable; type: string } > };
71 | export function closeLogger( logger: Logger ) {
72 | 	logger.streams.forEach( stream => {
73 | 		stream.stream.end();
74 | 	} );
75 | }
76 | 


--------------------------------------------------------------------------------
/test/api.test.ts:
--------------------------------------------------------------------------------
 1 | import { getExpiredContainers, getImageName, state } from '../src/api';
 2 | 
 3 | import { CONTAINER_EXPIRY_TIME } from '../src/constants';
 4 | 
 5 | describe( 'api', () => {
 6 | 	describe( 'getExpiredContainers', () => {
 7 | 		const RealNow = Date.now;
 8 | 		const fakeNow = RealNow() + 24 * 60 * 1000;
 9 | 		Date.now = () => fakeNow;
10 | 
11 | 		afterAll( () => {
12 | 			Date.now = RealNow;
13 | 		} );
14 | 
15 | 		const EXPIRED_TIME = Date.now() - CONTAINER_EXPIRY_TIME - 1;
16 | 		const GOOD_TIME = Date.now() - CONTAINER_EXPIRY_TIME + 1;
17 | 		const images = [
18 | 			{ Image: getImageName( '1' ), Id: 1, Created: EXPIRED_TIME / 1000, Names: [ '/foo' ] },
19 | 			{ Image: getImageName( '2' ), Id: 1, Created: EXPIRED_TIME / 1000, Names: [ '/bar' ] },
20 | 		];
21 | 
22 | 		afterEach( () => {
23 | 			state.accesses = new Map();
24 | 			state.containers = new Map();
25 | 		} );
26 | 
27 | 		beforeEach( () => {
28 | 			state.containers = new Map( images.map( image => [ image.Image, { ...image } ] ) as any );
29 | 		} );
30 | 
31 | 		test( 'returns nothing for empty list of containers', () => {
32 | 			state.containers = new Map();
33 | 			expect( getExpiredContainers() ).toEqual( [] );
34 | 		} );
35 | 
36 | 		test( 'returns the whole list if everything is expired', () => {
37 | 			expect( getExpiredContainers() ).toEqual( images );
38 | 		} );
39 | 
40 | 		test.only( 'returns empty list if everything was accessed before expiry', () => {
41 | 			state.accesses.set( 'foo', GOOD_TIME );
42 | 			state.accesses.set( 'bar', GOOD_TIME );
43 | 
44 | 			expect( getExpiredContainers() ).toEqual( [] );
45 | 		} );
46 | 
47 | 		test( 'returns list of only images that have not expired', () => {
48 | 			state.accesses.set( 'foo', Date.now() );
49 | 
50 | 			expect( getExpiredContainers() ).toEqual( [ state.containers.get( getImageName( '2' ) ) ] );
51 | 		} );
52 | 
53 | 		test( 'young images are not returned, regardless of access time', () => {
54 | 			state.containers.get( getImageName( '1' ) ).Created = Date.now() / 1000;
55 | 
56 | 			expect( getExpiredContainers() ).toEqual( [ state.containers.get( getImageName( '2' ) ) ] );
57 | 		} );
58 | 	} );
59 | } );
60 | 


--------------------------------------------------------------------------------
/src/config.ts:
--------------------------------------------------------------------------------
 1 | import Dockerode from 'dockerode';
 2 | import { DockerRepository, RunEnv } from './api';
 3 | 
 4 | type Readonly< T > = { readonly [ P in keyof T ]: T[ P ] };
 5 | type AppConfig = Readonly< {
 6 | 	build: BuildConfig;
 7 | 	repo: RepoConfig;
 8 | 	envs: EnvsConfig;
 9 | 	allowedDockerRepositories: AllowedDockerRepositories;
10 | 	allowedLabels: AllowedLabels;
11 | 	proxyRetry: number;
12 | } >;
13 | 
14 | type BuildConfig = Readonly< {
15 | 	containerCreateOptions?: Dockerode.ContainerCreateOptions;
16 | 	exposedPort: number;
17 | 	logFilename: string;
18 | 	tagPrefix: string;
19 | } >;
20 | 
21 | type RepoConfig = Readonly< {
22 | 	project: string;
23 | } >;
24 | 
25 | type EnvsConfig = Readonly< RunEnv[] >;
26 | 
27 | type AllowedDockerRepositories = Readonly< DockerRepository[] >;
28 | 
29 | type AllowedLabels = Readonly< Record< string, string > >;
30 | 
31 | export const config: AppConfig = {
32 | 	build: {
33 | 		containerCreateOptions: {},
34 | 		exposedPort: 3000,
35 | 		logFilename: 'dserve-build-log.txt',
36 | 		tagPrefix: 'dserve-wpcalypso',
37 | 	},
38 | 
39 | 	repo: {
40 | 		project: 'Automattic/wp-calypso',
41 | 	},
42 | 
43 | 	envs: [ 'calypso', 'jetpack', 'a8c-for-agencies', 'dashboard' ],
44 | 
45 | 	allowedDockerRepositories: [ 'registry.a8c.com' ],
46 | 
47 | 	allowedLabels: {
48 | 		'com.a8c.target': 'calypso-live',
49 | 	},
50 | 
51 | 	// When the proxy to the container fails with a ECONNRESET error, retry this number
52 | 	// of times.
53 | 	proxyRetry: 3,
54 | };
55 | 
56 | export function envContainerConfig( environment: RunEnv ): Dockerode.ContainerCreateOptions {
57 | 	switch ( environment ) {
58 | 		case 'calypso':
59 | 		default:
60 | 			return {
61 | 				Env: [ 'NODE_ENV=wpcalypso', 'CALYPSO_ENV=wpcalypso' ],
62 | 			};
63 | 		case 'jetpack':
64 | 			return {
65 | 				Env: [ 'NODE_ENV=jetpack-cloud-horizon', 'CALYPSO_ENV=jetpack-cloud-horizon' ],
66 | 			};
67 | 		case 'a8c-for-agencies':
68 | 			return {
69 | 				Env: [ 'NODE_ENV=a8c-for-agencies-horizon', 'CALYPSO_ENV=a8c-for-agencies-horizon' ],
70 | 			};
71 | 		case 'dashboard':
72 | 			return {
73 | 				Env: [ 'NODE_ENV=dashboard-horizon', 'CALYPSO_ENV=dashboard-horizon' ],
74 | 			};
75 | 			
76 | 	}
77 | }
78 | 


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 | 	"name": "calypso-docker-branches",
 3 | 	"version": "0.1.0",
 4 | 	"description": "",
 5 | 	"main": "src/index.js",
 6 | 	"devDependencies": {
 7 | 		"prettier": "npm:wp-prettier@1.18.2"
 8 | 	},
 9 | 	"dependencies": {
10 | 		"@types/bunyan": "^1.8.4",
11 | 		"@types/dockerode": "^2.5.5",
12 | 		"@types/express": "^4.16.0",
13 | 		"@types/express-session": "^1.15.10",
14 | 		"@types/forever-monitor": "^1.7.3",
15 | 		"@types/fs-extra": "^8.0.0",
16 | 		"@types/http-proxy": "^1.16.2",
17 | 		"@types/jest": "^24.0.0",
18 | 		"@types/lodash": "^4.14.116",
19 | 		"@types/node": "^10.12.2",
20 | 		"@types/node-fetch": "^2.1.2",
21 | 		"@types/nodegit": "^0.26.2",
22 | 		"@types/react": "^16.4.8",
23 | 		"@types/react-dom": "^16.0.7",
24 | 		"@types/strip-ansi": "^3.0.0",
25 | 		"@types/striptags": "^3.1.1",
26 | 		"@types/tar-fs": "^1.16.1",
27 | 		"@types/useragent": "^2.1.1",
28 | 		"bunyan": "^1.8.12",
29 | 		"docker-parse-image": "^3.0.1",
30 | 		"dockerode": "^3.0.0",
31 | 		"express": "^4.16.3",
32 | 		"express-session": "^1.15.6",
33 | 		"forever-monitor": "^1.7.1",
34 | 		"fs-extra": "^8.0.0",
35 | 		"get-port": "^5.1.1",
36 | 		"hot-shots": "^6.3.0",
37 | 		"http-proxy": "^1.17.0",
38 | 		"jest": "^24.0.0",
39 | 		"lodash": "^4.17.10",
40 | 		"node-fetch": "^2.2.1",
41 | 		"nodegit": "^0.26.5",
42 | 		"react": "^16.4.2",
43 | 		"react-dom": "^16.4.2",
44 | 		"strip-ansi": "^5.0.0",
45 | 		"striptags": "^3.1.1",
46 | 		"tar-fs": "^2.0.0",
47 | 		"ts-jest": "^24.0.0",
48 | 		"typescript": "^3.1.6",
49 | 		"useragent": "^2.3.0"
50 | 	},
51 | 	"scripts": {
52 | 		"test": "jest --testURL=http://localhost/",
53 | 		"start": "yarn run build-ts && yarn run serve",
54 | 		"serve": "node build/index.js",
55 | 		"serve:debug": "node build/index.js | bunyan",
56 | 		"serve:forever": "node build/daemon.js",
57 | 		"build-ts": "tsc",
58 | 		"watch-ts": "tsc -w",
59 | 		"reformat-files": "./node_modules/.bin/prettier --write \"**/*.{ts,tsx,json}\""
60 | 	},
61 | 	"repository": {
62 | 		"type": "git",
63 | 		"url": "https://github.com/Automattic/dserve"
64 | 	},
65 | 	"author": "samouri",
66 | 	"license": "MIT",
67 | 	"jest": {
68 | 		"transform": {
69 | 			"^.+\\.tsx?$": "ts-jest"
70 | 		},
71 | 		"testRegex": "(/__tests__/.*|(\\.|/)(test|spec))\\.tsx?$",
72 | 		"moduleFileExtensions": [
73 | 			"ts",
74 | 			"tsx",
75 | 			"js",
76 | 			"jsx",
77 | 			"json",
78 | 			"node"
79 | 		]
80 | 	}
81 | }
82 | 


--------------------------------------------------------------------------------
/test/logger.test.ts:
--------------------------------------------------------------------------------
 1 | jest.mock( 'bunyan', () => ( {
 2 | 	createLogger: jest.fn( () => ( {
 3 | 		info: () => {},
 4 | 		warn: () => {},
 5 | 		error: () => {},
 6 | 	} ) ),
 7 | 	RingBuffer: jest.fn(),
 8 | } ) );
 9 | 
10 | describe( 'logger', () => {
11 | 	let logger: any;
12 | 	let createLogger: any;
13 | 	beforeEach( () => {
14 | 		jest.clearAllMocks();
15 | 		jest.resetModules();
16 | 		logger = {
17 | 			info: jest.fn(),
18 | 			warn: jest.fn(),
19 | 			error: jest.fn(),
20 | 			child: jest.fn( options => ( { streams: options.streams } ) ),
21 | 		};
22 | 		createLogger = () => logger;
23 | 	} );
24 | 
25 | 	describe( '#closeLogger', () => {
26 | 		test( 'should close all of the streams in a logger logger', () => {} );
27 | 	} );
28 | 
29 | 	describe( 'l', () => {
30 | 		test( 'should only make one base logger', () => {
31 | 			const bunyan = require( 'bunyan' );
32 | 			const { l, closeLogger, getLoggerForBuild } = require( '../src/logger' );
33 | 			expect( bunyan.createLogger.mock.calls.length ).toBe( 1 );
34 | 			expect( bunyan.createLogger.mock.instances.length ).toBe( 1 );
35 | 		} );
36 | 
37 | 		test( 'l should call the underlying loggers info and error functions', () => {
38 | 			jest.setMock( 'bunyan', { createLogger, RingBuffer: jest.fn() } );
39 | 			const { l, closeLogger, getLoggerForBuild } = require( '../src/logger' );
40 | 
41 | 			l.info( 'testLog' );
42 | 			l.warn( 'testLog' );
43 | 			l.error( 'testLog' );
44 | 
45 | 			expect( logger.info.mock.calls.length ).toBe( 1 );
46 | 			expect( logger.warn.mock.calls.length ).toBe( 1 );
47 | 			expect( logger.error.mock.calls.length ).toBe( 1 );
48 | 		} );
49 | 	} );
50 | 
51 | 	describe( '#getLoggerForBuild', () => {
52 | 		test( 'should make a child logger', () => {
53 | 			const bunyan = require( 'bunyan' );
54 | 			const { l, closeLogger, getLoggerForBuild } = require( '../src/logger' );
55 | 			getLoggerForBuild( 'build-hash' );
56 | 			expect( logger.child.mock.calls.length ).toBe( 1 );
57 | 		} );
58 | 
59 | 		test( 'should write to a file at the correct path', () => {
60 | 			const bunyan = require( 'bunyan' );
61 | 			const { l, closeLogger, getLoggerForBuild } = require( '../src/logger' );
62 | 			const childLogger = getLoggerForBuild( 'build-hash' );
63 | 			expect( childLogger.streams ).toEqual( [
64 | 				{
65 | 					path: expect.stringContaining(
66 | 						'dserve-build-Automattic-wp-calypso-build-hash/dserve-build-log.txt'
67 | 					),
68 | 					type: 'file',
69 | 				},
70 | 			] );
71 | 		} );
72 | 	} );
73 | } );
74 | 


--------------------------------------------------------------------------------
/src/app/log.tsx:
--------------------------------------------------------------------------------
 1 | import React from 'react';
 2 | import ReactDOMServer from 'react-dom/server';
 3 | 
 4 | import { Shell } from './app-shell';
 5 | import { errorClass, humanRelativeTime } from './util';
 6 | import { ONE_MINUTE } from '../constants';
 7 | import LogDetails from './log-details';
 8 | 
 9 | const Log = ( { log, startedServerAt }: RenderContext ) => {
10 | 	return (
11 | 		<Shell refreshInterval={ ONE_MINUTE } startedServerAt={ startedServerAt }>
12 | 			<style
13 | 				dangerouslySetInnerHTML={ {
14 | 					__html: `
15 |                 .dserve-log-lines {
16 |                     list-style: none;
17 |                 }
18 | 
19 |                 .dserve-log-line {
20 |                     color: #00d8ff;
21 |                     margin-bottom: 4px;
22 |                 }
23 | 
24 |                 .dserve-log-line time {
25 |                     color: #eee;
26 |                     display: inline-block;
27 |                     min-width: 6em;
28 |                     text-align: right;
29 |                 }
30 | 
31 |                 .dserve-log-line.error span.info::before {
32 |                     content: ' - 🚨 - ';
33 |                 }
34 | 
35 |                 .dserve-log-line.info span.info::before {
36 |                     content: ' - ℹ - ';
37 |                 }
38 | 
39 |                 .dserve-log-line .details {
40 |                     font-family: monospace;
41 |                     color: #999;
42 |                     margin-left: 15em;
43 |                 }
44 | 
45 |                 .dserve-log-line .details pre {
46 |                     margin: 0;
47 |                 }
48 | 
49 |                 .dserve-log-line a[href] {
50 |                   color: lightblue;
51 |                 }
52 |         `,
53 | 				} }
54 | 			/>
55 | 			<ol className="dserve-log-lines">
56 | 				{ log
57 | 					.map( ( data, i ) => {
58 | 						const at = Date.parse( data.time );
59 | 
60 | 						return (
61 | 							<li className={ `dserve-log-line ${ errorClass( data.level ) }` } key={ `${ i }` }>
62 | 								<time
63 | 									dateTime={ new Date( at ).toISOString() }
64 | 									title={ new Date( at ).toLocaleTimeString( undefined, {
65 | 										timeZoneName: 'long',
66 | 										hour12: true,
67 | 									} ) }
68 | 								>
69 | 									{ humanRelativeTime( at / 1000 ) }
70 | 								</time>{' '}
71 | 								<span className="info">{ data.msg }</span>
72 | 								<LogDetails data={ data } />
73 | 							</li>
74 | 						);
75 | 					} )
76 | 					.reverse() }
77 | 			</ol>
78 | 		</Shell>
79 | 	);
80 | };
81 | 
82 | type RenderContext = { log: any[]; startedServerAt: Date };
83 | export default function renderLog( renderContext: RenderContext ) {
84 | 	return ReactDOMServer.renderToStaticMarkup( <Log { ...renderContext } /> );
85 | }
86 | 


--------------------------------------------------------------------------------
/src/app/util.ts:
--------------------------------------------------------------------------------
  1 | export function round( a: number, digits: number ): number {
  2 | 	const scale = Math.pow( 10, digits );
  3 | 
  4 | 	return Math.round( a * scale ) / scale;
  5 | }
  6 | 
  7 | export function percent( numerator: number, denominator: number ): number {
  8 | 	return round( ( numerator / ( Number.EPSILON + denominator ) ) * 100, 2 );
  9 | }
 10 | 
 11 | export function splitTime( t: number, u: number, n: number ): [ number, number ] {
 12 | 	const whole = Math.floor( t / ( u * n ) );
 13 | 	const part = Math.round( ( t - whole * u * n ) / u );
 14 | 
 15 | 	return [ whole, part ];
 16 | }
 17 | 
 18 | export function humanSize( size: number ): string {
 19 | 	if ( size > 1024 ** 3 ) {
 20 | 		return `${ round( size / 1024 ** 3, 2 ) } GB`;
 21 | 	}
 22 | 
 23 | 	if ( size > 1024 ** 2 ) {
 24 | 		return `${ round( size / 1024 ** 2, 1 ) } MB`;
 25 | 	}
 26 | 
 27 | 	if ( size > 1024 ) {
 28 | 		return `${ Math.round( size / 1024 ) } KB`;
 29 | 	}
 30 | 
 31 | 	return `${ size } B`;
 32 | }
 33 | 
 34 | const formatMs = ( tic: number ) => tic + 'ms';
 35 | 
 36 | const formatSec = ( tic: number ) => {
 37 | 	const secs = ( tic / 1000 ).toFixed( 2 );
 38 | 	return `${ secs }s`;
 39 | };
 40 | 
 41 | const formatMin = ( tic: number ) => {
 42 | 	const mins = Math.floor( tic / ( 1000 * 60 ) );
 43 | 	const secs = tic - mins * 60 * 1000;
 44 | 	let str = mins + 'm';
 45 | 	if ( secs > 0 ) {
 46 | 		str += ' ' + formatSec( secs );
 47 | 	}
 48 | 	return str;
 49 | };
 50 | 
 51 | const formatHours = ( tic: number ) => {
 52 | 	const hours = Math.floor( tic / ( 1000 * 60 * 24 ) );
 53 | 	const mins = tic - hours * 1000 * 60 * 24;
 54 | 	let str = hours + 'h';
 55 | 	if ( mins > 0 ) {
 56 | 		str += ' ' + formatMin( mins );
 57 | 	}
 58 | 	return str;
 59 | };
 60 | 
 61 | export function humanTimeSpan( tic: number ): string {
 62 | 	let negate = false;
 63 | 	if ( tic < 0 ) {
 64 | 		tic = -1 * tic;
 65 | 		negate = true;
 66 | 	}
 67 | 	let span;
 68 | 	if ( tic < 1000 ) {
 69 | 		span = formatMs( tic );
 70 | 	} else if ( tic < 1000 * 60 ) {
 71 | 		span = formatSec( tic );
 72 | 	} else if ( tic < 1000 * 60 * 24 ) {
 73 | 		span = formatMin( tic );
 74 | 	} else {
 75 | 		span = formatHours( tic );
 76 | 	}
 77 | 
 78 | 	return ( negate ? '-' : '' ) + span;
 79 | }
 80 | 
 81 | export function humanRelativeTime( tic: number ): string {
 82 | 	const toc = Date.now() / 1000;
 83 | 	const span = toc - tic;
 84 | 
 85 | 	if ( span < 60 ) {
 86 | 		return `${ Math.round( span ) }s ago`;
 87 | 	}
 88 | 
 89 | 	if ( span < 60 * 60 ) {
 90 | 		const [ m, s ] = splitTime( span, 60, 1 );
 91 | 
 92 | 		return s ? `${ m }m ${ s }s ago` : `${ m }m ago`;
 93 | 	}
 94 | 
 95 | 	if ( span < 24 * 3600 ) {
 96 | 		const [ h, m ] = splitTime( span, 60, 60 );
 97 | 
 98 | 		return m ? `${ h }h ${ m }m ago` : `${ h }h ago`;
 99 | 	}
100 | 
101 | 	if ( span < 7 * 24 * 3600 ) {
102 | 		const [ d, h ] = splitTime( span, 3600, 24 );
103 | 
104 | 		return h ? `${ d }d ${ h }h ago` : `${ d }d ago`;
105 | 	}
106 | 
107 | 	if ( span < 30 * 7 * 24 * 3600 ) {
108 | 		const [ w, d ] = splitTime( span, 24 * 3600, 7 );
109 | 
110 | 		return d ? `${ w }w ${ d }d ago` : `${ w }w ago`;
111 | 	}
112 | 
113 | 	return `${ Math.round( span / ( 7 * 24 * 3600 ) ) }w ago`;
114 | }
115 | 
116 | export function errorClass( errorLevel: number ): string {
117 | 	if ( errorLevel <= 10 ) {
118 | 		return 'trace';
119 | 	}
120 | 
121 | 	if ( errorLevel <= 20 ) {
122 | 		return 'debug';
123 | 	}
124 | 
125 | 	if ( errorLevel <= 30 ) {
126 | 		return 'info';
127 | 	}
128 | 
129 | 	if ( errorLevel <= 40 ) {
130 | 		return 'warn';
131 | 	}
132 | 
133 | 	if ( errorLevel <= 50 ) {
134 | 		return 'error';
135 | 	}
136 | 
137 | 	return 'fatal';
138 | }
139 | 


--------------------------------------------------------------------------------
/src/app/app-shell.tsx:
--------------------------------------------------------------------------------
  1 | import React from 'react';
  2 | import { start } from 'repl';
  3 | import { humanRelativeTime } from './util';
  4 | import { pendingHashes, buildQueue } from '../builder';
  5 | 
  6 | class Reloader extends React.Component< { milliseconds: number } > {
  7 | 	render() {
  8 | 		return (
  9 | 			<div
 10 | 				dangerouslySetInnerHTML={ {
 11 | 					__html: `
 12 | 					<script>
 13 | 						setTimeout(() => window.location.reload(), ${ this.props.milliseconds });
 14 | 					</script>
 15 | 				`,
 16 | 				} }
 17 | 			/>
 18 | 		);
 19 | 	}
 20 | }
 21 | 
 22 | export const Shell = ( { refreshInterval, startedServerAt, showReset = false, children }: any ) => (
 23 | 	<html>
 24 | 		<head>
 25 | 			{ 'number' === typeof refreshInterval && <Reloader milliseconds={ refreshInterval } /> }
 26 | 			<style
 27 | 				dangerouslySetInnerHTML={ {
 28 | 					__html: `
 29 | 					body {
 30 | 						font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "Roboto", "Oxygen-Sans", "Ubuntu", "Cantarell", "Helvetica Neue", sans-serif;
 31 | 						background: #091e25;
 32 | 						color: #ffffff;
 33 | 						margin: 0;
 34 | 					}
 35 | 
 36 | 					.dserve-toolbar {
 37 | 						position: absolute;
 38 | 						top: 12px;
 39 | 						right: 4px;
 40 | 						box-sizing: border-box;
 41 | 						display: flex;
 42 | 					}
 43 | 
 44 | 					.dserve-toolbar a {
 45 | 						display: inline-block;
 46 | 						border: 1px solid #ffffff;
 47 | 						border-radius: 3px;
 48 | 						padding: 4px 6px;
 49 | 						margin: 0 10px 0 0;
 50 | 						fontSize: 12;
 51 | 						text-decoration: none;
 52 | 						color: #ffffff;
 53 | 						background: #2e4453;
 54 | 					}
 55 | 
 56 | 					.dserve-toolbar a:hover {
 57 | 						background: #ffffff;
 58 | 						color: #2e4453;
 59 | 					}
 60 | 
 61 | 					.dserve-message {
 62 | 						position: fixed;
 63 | 						width: 100%;
 64 | 						box-sizing: border-box;
 65 | 						min-height: 55px;
 66 | 						max-height: 80px;
 67 | 						overflow: hidden;
 68 | 						padding: 16px 20px;
 69 | 						margin: 0;
 70 | 						background: #2e4453;
 71 | 						color: #ffffff;
 72 | 						animation: progress-bar-animation 3300ms infinite linear;
 73 | 						background-image: linear-gradient( -45deg, #3d596d 28%, #334b5c 28%, #334b5c 72%, #3d596d 72%);
 74 | 						background-size: 200px 100%;
 75 | 						background-repeat: repeat-x;
 76 | 						background-position: 0 50px;
 77 | 					}
 78 | 
 79 | 					.dserve-main-contents {
 80 | 						box-sizing: border-box;
 81 | 						width: 100%;
 82 | 						border: 0;
 83 | 						padding: 86px 20px 10px 20px;
 84 | 					}
 85 | 			`,
 86 | 				} }
 87 | 			/>
 88 | 		</head>
 89 | 		<body>
 90 | 			<div className="dserve-message">
 91 | 				DServe Calypso - Started{' '}
 92 | 				{
 93 | 					<time
 94 | 						dateTime={ startedServerAt.toISOString() }
 95 | 						title={ startedServerAt.toLocaleTimeString( undefined, {
 96 | 							timeZoneName: 'long',
 97 | 							hour12: true,
 98 | 						} ) }
 99 | 					>
100 | 						{ humanRelativeTime( startedServerAt / 1000 ) }
101 | 					</time>
102 | 				}
103 | 				<span className="build-count">
104 | 					, { pendingHashes.size ? pendingHashes.size + ' Building' : 'Idle' }
105 | 				</span>
106 | 				<span className="build-count">
107 | 					, { buildQueue.length ? buildQueue.length + ' Queued' : 'Empty Queue' }
108 | 				</span>
109 | 				<div className="dserve-toolbar">
110 | 					{ showReset && <a href="/?reset=true">Reset Branch</a> }
111 | 					<a href="/log">Logs</a>
112 | 					<a href="/localimages">Local Images</a>
113 | 					<a href="https://github.com/Automattic/dserve/issues">Report issues</a>
114 | 				</div>
115 | 			</div>
116 | 			<div className="dserve-main-contents">{ children }</div>
117 | 		</body>
118 | 	</html>
119 | );
120 | 


--------------------------------------------------------------------------------
/src/middlewares.ts:
--------------------------------------------------------------------------------
  1 | // external
  2 | import express from 'express';
  3 | import expressSession from 'express-session';
  4 | 
  5 | // internal
  6 | import { getCommitHashForBranch, refreshRemoteBranches, CommitHash, RunEnv } from './api';
  7 | import { config } from './config';
  8 | 
  9 | const hashPattern = /(?:^|.*?\.)(\w*)-?hash-([a-f0-9]+)\./;
 10 | 
 11 | function assembleSubdomainUrlForHash(
 12 | 	req: express.Request,
 13 | 	commitHash: CommitHash,
 14 | 	environment: RunEnv
 15 | ) {
 16 | 	const protocol = req.secure || req.headers.host.indexOf( 'calypso.live' ) > -1 ? 'https' : 'http';
 17 | 
 18 | 	const subdomainEnv = environment && environment !== config.envs[ 0 ] ? environment + '-' : '';
 19 | 
 20 | 	const newUrl = new URL(
 21 | 		`${ protocol }://${ subdomainEnv }hash-${ commitHash }.${ stripCommitHashSubdomainFromHost(
 22 | 			req.headers.host
 23 | 		) }`
 24 | 	);
 25 | 	newUrl.pathname = req.path;
 26 | 	for ( let [ key, value ] of Object.entries( req.query ) ) {
 27 | 		if ( key === 'hash' || key === 'branch' || key === 'env' ) {
 28 | 			continue;
 29 | 		}
 30 | 		newUrl.searchParams.set( key, String( value ) );
 31 | 	}
 32 | 
 33 | 	return newUrl.toString();
 34 | }
 35 | 
 36 | function stripCommitHashSubdomainFromHost( host: string ) {
 37 | 	return host.replace( hashPattern, '' );
 38 | }
 39 | 
 40 | function getCommitHashFromSubdomain( host: string ) {
 41 | 	const match = host.match( hashPattern );
 42 | 
 43 | 	if ( ! match ) {
 44 | 		return null;
 45 | 	}
 46 | 
 47 | 	// https://github.com/prettier/prettier/issues/1013
 48 | 	// prettier-ignore
 49 | 	const [ /* full match */, /* environment */, hash ] = match;
 50 | 	return hash;
 51 | }
 52 | 
 53 | function getEnvironmentFromSubdomain( host: string ) {
 54 | 	const match = host.match( hashPattern );
 55 | 
 56 | 	if ( ! match ) {
 57 | 		return null;
 58 | 	}
 59 | 
 60 | 	// https://github.com/prettier/prettier/issues/1013
 61 | 	// prettier-ignore
 62 | 	const [ /* full match */, environment ] = match;
 63 | 	return environment;
 64 | }
 65 | 
 66 | export function redirectHashFromQueryStringToSubdomain(
 67 | 	req: express.Request,
 68 | 	res: express.Response,
 69 | 	next: express.NextFunction,
 70 | 	retry: number = 2
 71 | ) {
 72 | 	const isHashSpecified = req.query && ( req.query.hash || req.query.branch );
 73 | 
 74 | 	if ( ! isHashSpecified ) {
 75 | 		next();
 76 | 		return;
 77 | 	}
 78 | 
 79 | 	const commitHash = req.query.hash || getCommitHashForBranch( req.query.branch );
 80 | 	const environment = req.query.env;
 81 | 
 82 | 	const sendError = () => {
 83 | 		res.send( 'could not find a hash for that branch' );
 84 | 		res.end();
 85 | 	};
 86 | 
 87 | 	if ( ! commitHash ) {
 88 | 		// could not find a hash for the branch... refresh the remotes and try again
 89 | 		if ( retry > 0 ) {
 90 | 			refreshRemoteBranches()
 91 | 				.then( () => {
 92 | 					redirectHashFromQueryStringToSubdomain( req, res, next, retry - 1 );
 93 | 				} )
 94 | 				.catch( sendError );
 95 | 			return;
 96 | 		}
 97 | 		sendError();
 98 | 	}
 99 | 
100 | 	res.redirect( assembleSubdomainUrlForHash( req, commitHash, environment ) );
101 | 
102 | 	res.end();
103 | }
104 | 
105 | export function determineCommitHash(
106 | 	req: express.Request,
107 | 	res: express.Response,
108 | 	next: express.NextFunction
109 | ) {
110 | 	const isHashInSession = !! req.session.commitHash;
111 | 	const subdomainCommitHash = getCommitHashFromSubdomain( req.headers.host );
112 | 
113 | 	if ( isHashInSession && ! subdomainCommitHash ) {
114 | 		next();
115 | 		return;
116 | 	}
117 | 
118 | 	if ( ! subdomainCommitHash ) {
119 | 		// @todo Render a nicer page here.
120 | 		res.send( 'Please specify a branch to load' );
121 | 		return;
122 | 	}
123 | 
124 | 	req.session.commitHash = subdomainCommitHash;
125 | 
126 | 	next();
127 | }
128 | 
129 | export function determineEnvironment(
130 | 	req: express.Request,
131 | 	res: express.Response,
132 | 	next: express.NextFunction
133 | ) {
134 | 	const subdomainEnvironment = getEnvironmentFromSubdomain( req.headers.host );
135 | 	if ( config.envs.includes( subdomainEnvironment ) ) {
136 | 		req.session.runEnv = subdomainEnvironment;
137 | 	} else {
138 | 		req.session.runEnv = config.envs[ 0 ];
139 | 	}
140 | 	next();
141 | }
142 | export const session = expressSession( {
143 | 	secret: 'keyboard cat',
144 | 	cookie: {},
145 | 	resave: false,
146 | 	saveUninitialized: true,
147 | } );
148 | 


--------------------------------------------------------------------------------
/src/app/local-images.tsx:
--------------------------------------------------------------------------------
  1 | import React from 'react';
  2 | import ReactDOMServer from 'react-dom/server';
  3 | import Docker from 'dockerode';
  4 | import { config } from '../config';
  5 | 
  6 | import { Shell } from './app-shell';
  7 | import { humanSize, humanRelativeTime } from './util';
  8 | import { BranchName, CommitHash } from '../api';
  9 | import { ONE_MINUTE, ONE_SECOND } from '../constants';
 10 | 
 11 | const LocalImages = ( {
 12 | 	branchHashes,
 13 | 	knownBranches,
 14 | 	localImages,
 15 | 	startedServerAt,
 16 | }: RenderContext ) => (
 17 | 	<Shell
 18 | 		refreshInterval={ knownBranches.size > 0 ? ONE_MINUTE : 5 * ONE_SECOND }
 19 | 		startedServerAt={ startedServerAt }
 20 | 	>
 21 | 		<style
 22 | 			dangerouslySetInnerHTML={ {
 23 | 				__html: `
 24 |                 .dserve-branch-selector select,
 25 |                 .dserve-branch-selector input[type="button"] {
 26 |                     background: none;
 27 |                     color: #fff;
 28 |                     font-size: 14px;
 29 |                 }
 30 | 
 31 |                 .dserve-branch-selector input[type="button"] {
 32 |                     margin-left: 8px;
 33 |                     border-radius: 4px;
 34 |                 }
 35 | 
 36 |                 .dserve-branch-selector input[type="button"]:hover {
 37 |                     background: #fff;
 38 |                     color: #222;
 39 |                 }
 40 | 
 41 |                 .dserve-image-header {
 42 |                     color: #00d8ff;
 43 |                 }
 44 | 
 45 |                 .dserve-image-header a {
 46 |                     color: #fff;
 47 |                     text-decoration: none;
 48 |                 }
 49 | 
 50 |                 .dserve-image-header a:hover,
 51 |                 .dserve-image-header a:visited:hover {
 52 |                     color: #00ff0c;
 53 |                     text-decoration: underline;
 54 |                 }
 55 | 
 56 |                 .dserve-image-header a:visited {
 57 |                     color: #fff;
 58 |                 }
 59 | 
 60 |                 time {
 61 |                     color: #00ff0c;
 62 |                 }
 63 |         `,
 64 | 			} }
 65 | 		/>
 66 | 		<div className="dserve-branch-selector">
 67 | 			{ knownBranches.size > 0 ? (
 68 | 				<React.Fragment>
 69 | 					<select id="selected-branch">
 70 | 						{ Array.from( knownBranches )
 71 | 							.sort( ( a, b ) => a[ 0 ].localeCompare( b[ 0 ] ) )
 72 | 							.map( ( [ branchName, hash ] ) => (
 73 | 								<option key={ branchName }>{ branchName }</option>
 74 | 							) ) }
 75 | 					</select>
 76 | 					<input id="branch-selector-button" type="button" value="Open" />
 77 | 					<div
 78 | 						dangerouslySetInnerHTML={ {
 79 | 							__html: `
 80 |                         <script>
 81 |                             (function() {
 82 |                                 var button = document.getElementById( 'branch-selector-button' );
 83 | 
 84 |                                 if ( button.getAttribute( 'data-has-listener' ) ) {
 85 |                                     return;
 86 |                                 }
 87 | 
 88 |                                 button.addEventListener( 'click', function() {
 89 |                                     var branch = document.getElementById( 'selected-branch' );
 90 | 
 91 |                                     window.open(
 92 |                                         '/?branch=' + branch.value,
 93 |                                         '_blank'
 94 |                                     );
 95 |                                 } );
 96 |                             })();
 97 |                         </script>
 98 |                     `,
 99 | 						} }
100 | 					/>
101 | 				</React.Fragment>
102 | 			) : (
103 | 				'Loading remote branches…'
104 | 			) }
105 | 		</div>
106 | 		<dl>
107 | 			{ Object.keys( localImages ).map( repoTags => {
108 | 				const info = localImages[ repoTags ];
109 | 				const createdAt = new Date( info.Created * 1000 );
110 | 				const match = repoTags.match( new RegExp( `${ config.build.tagPrefix }:([a-f0-9]+)` ) );
111 | 				const title =
112 | 					match && branchHashes.has( match[ 1 ] ) ? branchHashes.get( match[ 1 ] ) : repoTags;
113 | 
114 | 				return (
115 | 					<React.Fragment key={ info.Id }>
116 | 						<dt className="dserve-image-header">
117 | 							{ title }
118 | 							{ match && (
119 | 								<React.Fragment>
120 | 									{ ' - ' }
121 | 									<a href={ `https://github.com/${ config.repo.project }/commit/${ match[ 1 ] }` }>
122 | 										Github
123 | 									</a>
124 | 									{ ' - ' }
125 | 									<a href={ `/?hash=${ match[ 1 ] }` } target="_blank">
126 | 										Open
127 | 									</a>
128 | 								</React.Fragment>
129 | 							) }
130 | 						</dt>
131 | 						<dd>
132 | 							<p>
133 | 								Created{' '}
134 | 								{
135 | 									<time
136 | 										dateTime={ createdAt.toISOString() }
137 | 										title={ createdAt.toLocaleTimeString( undefined, {
138 | 											timeZoneName: 'long',
139 | 											hour12: true,
140 | 										} ) }
141 | 									>
142 | 										{ humanRelativeTime( info.Created ) }
143 | 									</time>
144 | 								}
145 | 							</p>
146 | 							<p>ID { info.Id }</p>
147 | 							<p>Size { humanSize( info.Size ) }</p>
148 | 						</dd>
149 | 					</React.Fragment>
150 | 				);
151 | 			} ) }
152 | 		</dl>
153 | 	</Shell>
154 | );
155 | 
156 | type RenderContext = {
157 | 	branchHashes: Map< CommitHash, BranchName >;
158 | 	knownBranches: Map< BranchName, CommitHash >;
159 | 	localImages: { [ s: string ]: Docker.ImageInfo };
160 | 	startedServerAt: Date;
161 | };
162 | export default function renderLocalImages( renderContext: RenderContext ) {
163 | 	return ReactDOMServer.renderToStaticMarkup( <LocalImages { ...renderContext } /> );
164 | }
165 | 


--------------------------------------------------------------------------------
/src/image-runner.ts:
--------------------------------------------------------------------------------
  1 | // external
  2 | import express from 'express';
  3 | import { ContainerInfo } from 'dockerode';
  4 | 
  5 | // internal
  6 | import {
  7 | 	getAllImages,
  8 | 	findContainer,
  9 | 	pullImage,
 10 | 	deleteContainer,
 11 | 	ContainerName,
 12 | 	proxyRequestToContainer,
 13 | 	reviveContainer,
 14 | 	createContainer,
 15 | 	getContainerName,
 16 | 	isValidImage,
 17 | 	refreshLocalImages,
 18 | } from './api';
 19 | import { config } from './config';
 20 | import { l } from './logger';
 21 | import dockerParseImage from 'docker-parse-image';
 22 | import {ImageNotFound, InvalidImage, InvalidRegistry} from './error';
 23 | 
 24 | const containerPattern = /^container-(?<container>[A-Za-z0-9_-]+)\./;
 25 | 
 26 | function stripImageHashSubdomainFromHost( host: string ) {
 27 | 	return host.replace( containerPattern, '' );
 28 | }
 29 | 
 30 | function assembleSubdomainUrlForContainer( req: express.Request, container: ContainerInfo ) {
 31 | 	const protocol = req.secure || req.headers.host.indexOf( 'calypso.live' ) > -1 ? 'https' : 'http';
 32 | 
 33 | 	// Docker names are in the form <string>_<string>, see https://github.com/docker/engine/blob/master/pkg/namesgenerator/names-generator.go#L843
 34 | 	// But we don't want to generate URLs with `_` because in a few other systems (eg: Calypso, wpcom) we assume the URL will match [a-zA-Z0-9-].calypso.live
 35 | 	const name = getContainerName( container ).replace( /_/g, '-' );
 36 | 
 37 | 	const newUrl = new URL(
 38 | 		`${ protocol }://container-${ name }.${ stripImageHashSubdomainFromHost(
 39 | 			req.headers.host
 40 | 		) }`
 41 | 	);
 42 | 	newUrl.pathname = req.path;
 43 | 	for ( let [ key, value ] of Object.entries( req.query ) ) {
 44 | 		if ( key === 'hash' || key === 'branch' || key === 'env' || key === 'image' ) {
 45 | 			continue;
 46 | 		}
 47 | 		newUrl.searchParams.set( key, String( value ) );
 48 | 	}
 49 | 
 50 | 	return newUrl.toString();
 51 | }
 52 | 
 53 | function getContainerNameFromSubdomain( host: string ) {
 54 | 	const match = host.match( containerPattern );
 55 | 	if ( ! match ) {
 56 | 		return null;
 57 | 	}
 58 | 
 59 | 	return match.groups.container;
 60 | }
 61 | 
 62 | /**
 63 |  * Gets an image name from the query string, finds (or creates) a container for that image
 64 |  * and redirects to http://container-<containername>.calypso.live
 65 |  */
 66 | async function loadImage( req: express.Request, res: express.Response ) {
 67 | 	res.header( 'Cache-control', 'no-cache' );
 68 | 
 69 | 	const imageName = req.query.image;
 70 | 	const environment = req.query.env || config.envs[ 0 ];
 71 | 
 72 | 	const { registry } = dockerParseImage( imageName );
 73 | 	if ( ! config.allowedDockerRepositories.includes( registry ) ) {
 74 | 		throw new InvalidRegistry(registry)
 75 | 	}
 76 | 
 77 | 	// Check if there is a container for this image+environment
 78 | 	const existingContainer = findContainer( {
 79 | 		image: imageName,
 80 | 		env: environment,
 81 | 	} );
 82 | 	if ( existingContainer ) {
 83 | 		// Redirect to http://container-<name>.calypso.live
 84 | 		res.redirect( assembleSubdomainUrlForContainer( req, existingContainer ) );
 85 | 		return;
 86 | 	}
 87 | 
 88 | 	// There is no container. Check if we have the image already downloaded
 89 | 	const image = getAllImages().get( imageName );
 90 | 	if (!image) {
 91 | 		// No image in local repo, try to fetch it and reload the page when it's done.
 92 | 		try{
 93 | 			let responseStarted = false;
 94 | 			await pullImage( imageName, data => {
 95 | 				// This callback is called when the image starts to download. At this point we know the image exists.
 96 | 				// Start streaming the download logs.
 97 | 				if (!responseStarted) {
 98 | 					res.status( 202 ).write( '<!DOCTYPE html><body><pre>' );
 99 | 					responseStarted=true;
100 | 				}
101 | 				res.write( `${ Date.now() } - ${ JSON.stringify( data ) }\n` );
102 | 			} );
103 | 
104 | 			// After the imge is pulled, reload the page. This will invoke this same express handler, but now that the
105 | 			// image has been downloaded it will follow a different code path
106 | 			res.write( `</pre><script>document.location.reload()</script></body>` );
107 | 			res.end();
108 | 			return;
109 | 		}catch(err) {
110 | 			if (err.message.match(/HTTP code 404/)) {
111 | 				throw new ImageNotFound(imageName)
112 | 			} else {
113 | 				throw err;
114 | 			}
115 | 		}
116 | 	}
117 | 
118 | 	// We have the image! Validate it (i.e. validate it was built with TeamCity)
119 | 	if ( ! isValidImage( image ) ) {
120 | 		throw new InvalidImage(imageName)
121 | 	}
122 | 
123 | 	// The image is valid, create the container and redirect to http://container-<name>.calypso.live
124 | 	const container = await createContainer( imageName, environment );
125 | 	res.redirect( assembleSubdomainUrlForContainer( req, container ) );
126 | 	return;
127 | }
128 | 
129 | /**
130 |  * Gets a container name from the subdmain, starts it if necessary and proxies all requests to it.
131 |  */
132 | async function startAndProxyRequestsToContainer( req: express.Request, res: express.Response ) {
133 | 	const containerName: ContainerName = getContainerNameFromSubdomain( req.headers.host );
134 | 	let container: ContainerInfo;
135 | 	const shouldDelete = 'delete' in req.query;
136 | 
137 | 	container = findContainer( {
138 | 		sanitizedName: containerName,
139 | 	} );
140 | 	if ( ! container ) {
141 | 		throw new Error( `Container ${ containerName } not found` );
142 | 	}
143 | 
144 | 	if ( shouldDelete ) {
145 | 		const containerRealName = getContainerName( container );
146 | 		l.info( { containerRealName }, `Hard reset for ${ containerRealName }` );
147 | 		await deleteContainer( container );
148 | 		res.send( `Container ${ containerRealName } deleted` );
149 | 		return;
150 | 	}
151 | 
152 | 	if ( container.State !== 'running' ) {
153 | 		container = await reviveContainer( container );
154 | 	}
155 | 
156 | 	proxyRequestToContainer( req, res, container );
157 | }
158 | 
159 | /**
160 |  * Main middleware for the image runner. This single middleware will take care of both cases:
161 |  *
162 |  *  - Image in query string (http://calypso.live?<registry>/image=calypso/app:build-4)
163 |  *  - Container name in subdomain (http://container-agitated_hypatia.calypso.live/)
164 |  */
165 | export function middleware(
166 | 	req: express.Request,
167 | 	res: express.Response,
168 | 	next: express.NextFunction
169 | ) {
170 | 	const imageName = req.query && req.query.image;
171 | 	if ( imageName ) {
172 | 		loadImage( req, res ).catch( next );
173 | 		return;
174 | 	}
175 | 
176 | 	const containerName = getContainerNameFromSubdomain( req.headers.host );
177 | 	if ( containerName ) {
178 | 		startAndProxyRequestsToContainer( req, res ).catch( next );
179 | 		return;
180 | 	}
181 | 
182 | 	next();
183 | }
184 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # dserve
  2 | 
  3 | [![CircleCI](https://circleci.com/gh/Automattic/dserve/tree/master.svg?style=svg&circle-token=061a56710d3d75a9251ff74141b1c758a0790461)](https://circleci.com/gh/Automattic/dserve/tree/master)
  4 | 
  5 | <img src="https://raw.githubusercontent.com/Automattic/dserve/f699948673de4600a181484f5ab96a4a153eb552/logo.png" width=256 />
  6 | 
  7 | A development server for serving branches of your docker-based web application an on-demand basis.
  8 | 
  9 | It can build images for any hash, run containers, stop containers that haven't been accessed in a
 10 | while, proxy requests to the right container based on query params, etc.
 11 | 
 12 | ## Install
 13 | 
 14 | ```bash
 15 | git clone git@github.com:Automattic/dserve.git
 16 | cd dserve
 17 | nvm use
 18 | yarn
 19 | yarn start
 20 | ```
 21 | 
 22 | ## Use
 23 | 
 24 | You will need to modify your hosts file to include the following line:
 25 | 
 26 | ```
 27 | 127.0.0.1 calypso.localhost
 28 | ```
 29 | 
 30 | Then you may either specify a branch or a hash to load.
 31 | 
 32 | 1. branch: calypso.localhost:3000?branch={branchName}
 33 | 2. hash: calypso.localhost:3000?hash={commitHash}
 34 | 
 35 | ## Source Code Overview
 36 | 
 37 | At the end of the day, dserve is node express server written in typescript. It can trigger docker image
 38 | builds and deletions, start and stop containers, and a few other tricks.
 39 | 
 40 | Here is an example flow of what happens when requesting a never-requested-before commit sha:
 41 | 
 42 | 1. User tries to access `https://calypso.live?hash=hash`.
 43 | 2. dserve will query the local fs and docker daemon to determine the status of the corresponding image. It will discover that `hash` has never been requested before and needs to build an image for it. Therefore it will add the hash to the build queue and send the user a screen saying "starting a build for requested hash".
 44 | 3. If the branch or hash exist, dserve will redirect the user to https://hash-$hash.calypso.live, isolating the build to a subdomain
 45 | 4. Internally dserve checks the build queue very frequently and will initate a build within seconds. The build takes places within its own temporary directory in a place like: `/tmp/dserve-calyspo-hash/repo` and logs will be stored in `/tmp/dserve-calypso-hash/dserve-build-log.txt`.
 46 | 5. When a user requests the branch while the build is happening, dserve will recognize that the build is in progress and show the user the build's status.
 47 | 6. Finally when the build completes, the next time a user requests the branch they will see: "starting container, this page will refresh in a couple of seconds".
 48 | 
 49 | **index.ts**: this acts as the entry point for dserve. it sets up the server, initializes the routes, and contains the request handling for each incoming route.
 50 | 
 51 | **middlewares.ts** this file contains all of the middlewares that dserve uses.
 52 | 
 53 | 1. _redirectHashFromQueryStringToSubdomain_: This middleware will look for a branch or hash in the query string and redirect to a corresponding subdomain matching the commit hash.
 54 | 2. _determineCommitHash_: Every request to dserve needs to be associated with a commit hash or else it cannot be fulfilled. this middleware will attach a `commitHash` to the express request based on the subdomain.
 55 | 3. _session_: Standard session middleware so that each request doesn't need to specify a hash with a query param.
 56 | 
 57 | **api.ts**: Contains all of the code that interfaces with external things like the fs, docker, or github. there are two kinds of entities that exist in this file, those that periodically update data and the other is helper functions for things that need to be done on-demand.
 58 | 
 59 | _periodically repeating_: updating the git branches to commit hash mapping, updating which docker images are available from the local docker server, stopping unused containers.
 60 | 
 61 | _on-demand helper functions_: this includes functionality for recording how recently a commitHash was accessed, a helper for proxying a request to the right container, helpers for checking the progress/state of a a commit, etc.
 62 | 
 63 | **builder.ts**: Contains all of the code for building the docker images for a specific commit hash. This includes making build queue and rate limiting dserve to N builds at a time.
 64 | 
 65 | **logger.ts**: Exports a couple key items around loggers including the application logger and a getter for configuring a specific logger per-docker build of commits.
 66 | 
 67 | ## Operations
 68 | 
 69 | Are you in a situation where you are suddently tasked with maintaining dserve even though you didn't write it?
 70 | Once the flood of mixed feelings towards the original authors settles, it'd be a good idea to read this section.
 71 | You probably want to know how to do things like, deploy new code, debug issues, and e2e test dserve locally.
 72 | Here goes nothing:
 73 | 
 74 | **deploying**
 75 | This GitHub repo is polled every 15 minutes. If there have been updates to the repo, then the latest sha is deployed.
 76 | Thats it. Merge, and it'll be deployed. In a high severity situation where dserve is broken, you'll want to make sure you time your attempts to fix it _before_ the next 15 minute mark.
 77 | 
 78 | **debugging**
 79 | dserve has a couple helpful urls for debugging issues for times when you don't have ssh access.
 80 | Note that any time you see `branch=${branchName}` you can subsitute `hash=${sha}`.
 81 | 
 82 | - Application Log: https://calypso.live/log
 83 | - List of local Docker images: https://calypso.live/localimages
 84 | - Delete a build directory: add reset=1 as a query param like so https://calypso.live?branch=${branchName}&reset=1
 85 | - Build Status: https://calypso.live/status?branch=${branchName}
 86 | 
 87 | **e2e test locally**
 88 | 
 89 | 1. start up dserve with `yarn start`
 90 | 2. try to access a branch that you've never built before by going to localhost:3000?branch=${branchName}. After a successful build you should be proxied to calypso
 91 | 3. try to access an already built branch (by looking at the result of `docker images` you can find repo-tags with the right sha to specify). After a succesfful build you should be proxied to that branch's version of calypso.
 92 | 4. you might need access to the private Docker registry: PCYsg-stw-p2
 93 | 
 94 | **fixing errors**
 95 | 
 96 | - Docker connection error ("connect ENOENT /var/run/docker.sock"): You can easily fix this by running [sudo ln -s ~/.docker/run/docker.sock /var/run/docker.sock](https://github.com/lando/lando/issues/3533#issuecomment-1464252377)
 97 | - Error when building image: Make sure the image of the branch you are trying to build is available.
 98 | 
 99 | **things that have broken in the past**
100 | 
101 | 1. We were running an older version of docker that had `buildCache` issues. disabling the build cache (as a setting in the `buildImage` function) until we could upgrade docker versions solved the issue
102 | 2. The Docker Daemon ran into problems: there was one instance where builds seemed to just hang randomly and there was no obvious cause. all builds had failed. Systems restarting the docker daemon solved the issue.
103 | 3. Double slash branches: there is an interesting property of git with respect to how branches get stored on the local filesystem. Each slash in a branchname actually means that it occupies a nested folder. That means if locally you have a branch named `thing/thing2` then you _cannot_ pull down a remote branch with the name `thing`. The reason the remote repo was capable of having branch `thing/thing2` is because `thing` had already been deleted in its repo. The fix here is to always run a `git prune` when pulling down new branches which automatically deletes the appropriates local branches that no longer exist in the remote repo.
104 | 


--------------------------------------------------------------------------------
/src/builder.ts:
--------------------------------------------------------------------------------
  1 | import fs from 'fs-extra';
  2 | import git from 'nodegit';
  3 | import os from 'os';
  4 | import path from 'path';
  5 | import tar from 'tar-fs';
  6 | import { Readable } from 'stream';
  7 | import { sample } from 'lodash';
  8 | 
  9 | import {
 10 | 	CommitHash,
 11 | 	getImageName,
 12 | 	docker,
 13 | 	BranchName,
 14 | 	refreshLocalImages,
 15 | 	refreshRemoteBranches,
 16 | 	getCommitHashForBranch,
 17 | } from './api';
 18 | import { config } from './config';
 19 | import { closeLogger, l, getLoggerForBuild } from './logger';
 20 | import { ONE_SECOND, ONE_MINUTE, FIVE_MINUTES } from './constants';
 21 | import { increment, timing, gauge } from './stats';
 22 | 
 23 | // hidden method in nodegit that turns on thread safety
 24 | // see https://github.com/nodegit/nodegit/pull/836
 25 | ( git as any ).enableThreadSafety();
 26 | 
 27 | export const MAX_CONCURRENT_BUILDS = 4;
 28 | 
 29 | const MAX_CORES = [ 12 ];
 30 | export const getBuildConcurrency = () =>
 31 | 	Math.max(
 32 | 		1,
 33 | 		Math.min( Math.floor( os.cpus().length / MAX_CONCURRENT_BUILDS ), sample( MAX_CORES ) )
 34 | 	);
 35 | 
 36 | export const buildQueue: Array< CommitHash > = [];
 37 | export const pendingHashes: Set< CommitHash > = new Set();
 38 | const failedHashes: Set< CommitHash > = new Set();
 39 | 
 40 | export const getLogPath = ( hash: CommitHash ) =>
 41 | 	path.join( getBuildDir( hash ), config.build.logFilename );
 42 | export async function isBuildInProgress( hash: CommitHash ): Promise< boolean > {
 43 | 	if ( pendingHashes.has( hash ) ) {
 44 | 		return true;
 45 | 	}
 46 | 
 47 | 	const pathExists = await fs.pathExists( getBuildDir( hash ) );
 48 | 
 49 | 	return pendingHashes.has( hash ) || pathExists;
 50 | }
 51 | 
 52 | export function didBuildFail( hash: CommitHash ): boolean {
 53 | 	return failedHashes.has( hash );
 54 | }
 55 | 
 56 | export async function readBuildLog( hash: CommitHash ): Promise< string | null > {
 57 | 	return ( await fs.pathExists( getLogPath( hash ) ) )
 58 | 		? fs.readFile( getLogPath( hash ), 'utf-8' )
 59 | 		: null;
 60 | }
 61 | 
 62 | export function getBuildDir( hash: CommitHash ) {
 63 | 	const tmpDir = os.tmpdir();
 64 | 	return path.join( tmpDir, `dserve-build-${ config.repo.project.replace( '/', '-' ) }-${ hash }` );
 65 | }
 66 | 
 67 | export async function cleanupBuildDir( hash: CommitHash ) {
 68 | 	const buildDir = getBuildDir( hash );
 69 | 	l.info( `removing directory: ${ buildDir }` );
 70 | 	pendingHashes.delete( hash );
 71 | 	return fs.remove( buildDir );
 72 | }
 73 | 
 74 | const waitingOnCommits: Set< CommitHash > = new Set();
 75 | async function waitForCommit( commitHash: CommitHash ): Promise< boolean > {
 76 | 	let repo = await git.Repository.open( path.join( __dirname, '..', 'repos', 'wp-calypso' ) );
 77 | 	try {
 78 | 		await repo.getCommit( commitHash );
 79 | 		return true;
 80 | 	} catch {
 81 | 		// commit not found, refresh
 82 | 		await refreshRemoteBranches();
 83 | 		try {
 84 | 			await repo.getCommit( commitHash );
 85 | 			return true;
 86 | 		} catch {
 87 | 			return false;
 88 | 		}
 89 | 	}
 90 | }
 91 | 
 92 | export async function addToBuildQueue( commitHash: CommitHash ) {
 93 | 	if ( waitingOnCommits.has( commitHash ) ) {
 94 | 		return;
 95 | 	}
 96 | 	waitingOnCommits.add( commitHash );
 97 | 	try {
 98 | 		const foundCommit = await waitForCommit( commitHash );
 99 | 		if ( ! foundCommit ) {
100 | 			l.error( { commitHash }, 'Cannot find commit' );
101 | 			return;
102 | 		}
103 | 	} finally {
104 | 		waitingOnCommits.delete( commitHash );
105 | 	}
106 | 
107 | 	if ( buildQueue.includes( commitHash ) || pendingHashes.has( commitHash ) ) {
108 | 		return;
109 | 	}
110 | 	l.info(
111 | 		{ buildQueueSize: buildQueue.length, commitHash },
112 | 		'Adding a commitHash to the buildQueue'
113 | 	);
114 | 	increment( 'build.queued' );
115 | 	failedHashes.delete( commitHash );
116 | 	buildQueue.push( commitHash );
117 | 	gauge( 'build_queue', buildQueue.length );
118 | }
119 | 
120 | export async function buildImageForHash( commitHash: CommitHash ): Promise< void > {
121 | 	let buildStream: NodeJS.ReadableStream;
122 | 
123 | 	const buildDir = getBuildDir( commitHash );
124 | 	const repoDir = path.join( buildDir, 'repo' );
125 | 	const imageName = getImageName( commitHash );
126 | 	let imageStart: number;
127 | 
128 | 	if ( await isBuildInProgress( commitHash ) ) {
129 | 		l.info( { commitHash, buildDir }, 'Skipping build because a build is already in progress' );
130 | 		return;
131 | 	}
132 | 
133 | 	pendingHashes.add( commitHash );
134 | 	failedHashes.delete( commitHash );
135 | 
136 | 	increment( 'build.start' );
137 | 
138 | 	try {
139 | 		await fs.mkdir( buildDir );
140 | 	} catch ( err ) {
141 | 		l.error( { err, buildDir, commitHash }, 'Could not create directory for the build' );
142 | 		increment( 'build.error' );
143 | 		pendingHashes.delete( commitHash );
144 | 		failedHashes.add( commitHash );
145 | 		return;
146 | 	}
147 | 	const buildLogger = getLoggerForBuild( commitHash );
148 | 	const buildConcurrency = getBuildConcurrency();
149 | 
150 | 	try {
151 | 		l.info(
152 | 			{ commitHash, buildDir, repoDir, imageName, buildConcurrency },
153 | 			'Attempting to build image.'
154 | 		);
155 | 
156 | 		const cloneStart = Date.now();
157 | 		buildLogger.info( 'Cloning git repo' );
158 | 		const calypsoDir = path.join( __dirname, '../repos', 'wp-calypso' );
159 | 		const repo = await git.Clone.clone( calypsoDir, repoDir );
160 | 		buildLogger.info( 'Finished cloning repo' );
161 | 		const cloneTime = Date.now() - cloneStart;
162 | 		timing( 'git.build.clone', cloneTime );
163 | 		l.info( { commitHash, cloneTime }, 'Finished cloning repo' );
164 | 
165 | 		const checkoutStart = Date.now();
166 | 		const commit = await repo.getCommit( commitHash );
167 | 		const branch = await repo.createBranch( 'dserve', commit, true );
168 | 		await repo.checkoutBranch( branch );
169 | 		const checkoutTime = Date.now() - checkoutStart;
170 | 		timing( 'git.build.checkout', checkoutTime );
171 | 		l.info( { commitHash, checkoutTime }, 'Checked out branch' );
172 | 		buildLogger.info( 'Checked out the correct branch' );
173 | 
174 | 		buildLogger.info( 'Placing all the contents into a tarball stream for docker\n' );
175 | 		l.info(
176 | 			{ commitHash, repoDir, imageName, buildConcurrency },
177 | 			'Placing contents of repoDir into a tarball and sending to docker for a build'
178 | 		);
179 | 		const tarStream = tar.pack( repoDir );
180 | 		buildLogger.info( 'Handing off tarball to Docker for the rest of the legwork\n' );
181 | 
182 | 		imageStart = Date.now();
183 | 		buildStream = await docker.buildImage( tarStream, {
184 | 			t: imageName,
185 | 			nocache: false,
186 | 			forcerm: true,
187 | 			buildargs: {
188 | 				commit_sha: commitHash,
189 | 				workers: String( buildConcurrency ),
190 | 			},
191 | 		} );
192 | 	} catch ( err ) {
193 | 		buildLogger.error(
194 | 			{ err },
195 | 			`Encountered error while git checking out, tarballing, or handing to docker`
196 | 		);
197 | 		l.error(
198 | 			{ err },
199 | 			`Encountered error while git checking out, tarballing, or handing to docker`
200 | 		);
201 | 		increment( 'build.error' );
202 | 		pendingHashes.delete( commitHash );
203 | 		failedHashes.add( commitHash );
204 | 	}
205 | 
206 | 	if ( ! buildStream ) {
207 | 		l.error( { buildStream, commitHash }, "Failed to build image but didn't throw an error" );
208 | 		increment( 'build.error' );
209 | 		pendingHashes.delete( commitHash );
210 | 		failedHashes.add( commitHash );
211 | 		closeLogger( buildLogger as any );
212 | 		return;
213 | 	}
214 | 
215 | 	async function onFinished( err: Error ) {
216 | 		if ( ! err ) {
217 | 			const buildImageTime = Date.now() - imageStart;
218 | 			timing( 'build_image', buildImageTime );
219 | 			timing( `build_image_by_core.${ buildConcurrency }_cores`, buildImageTime );
220 | 			increment( 'build.success' );
221 | 			try {
222 | 				await refreshLocalImages();
223 | 			} catch ( err ) {
224 | 				l.info( { commitHash, err }, 'Error refreshing local images' );
225 | 			}
226 | 			l.info(
227 | 				{ commitHash, buildImageTime, repoDir, imageName, buildConcurrency },
228 | 				`Successfully built image. Now cleaning up build directory`
229 | 			);
230 | 			cleanupBuildDir( commitHash );
231 | 		} else {
232 | 			increment( 'build.error' );
233 | 			buildLogger.error( { err }, 'Encountered error when building image' );
234 | 			l.error( { err, commitHash }, `Failed to build image for. Leaving build files in place` );
235 | 			failedHashes.add( commitHash );
236 | 		}
237 | 		pendingHashes.delete( commitHash );
238 | 		closeLogger( buildLogger as any );
239 | 	}
240 | 
241 | 	function onProgress( event: any ) {
242 | 		if ( event.stream ) {
243 | 			buildLogger.info( { fromDocker: true }, event.stream );
244 | 		} else {
245 | 			buildLogger.info( { fromDocker: true }, event );
246 | 		}
247 | 	}
248 | 
249 | 	docker.modem.followProgress( buildStream, onFinished, onProgress );
250 | }
251 | 
252 | // Background tasks
253 | 
254 | const loop = ( f: Function, delay: number ) => {
255 | 	const run = () => {
256 | 		f();
257 | 		//console.log( 'running loop with %o and %d', f, delay );
258 | 		setTimeout( run, delay );
259 | 	};
260 | 
261 | 	run();
262 | };
263 | 
264 | function warnOnQueueBuildup() {
265 | 	if ( buildQueue.length > MAX_CONCURRENT_BUILDS ) {
266 | 		l.info(
267 | 			{ buildQueue },
268 | 			'There are images waiting to be built that are stuck because of too many concurrent builds'
269 | 		);
270 | 	}
271 | }
272 | export function buildFromQueue() {
273 | 	const currentLength = buildQueue.length;
274 | 	buildQueue
275 | 		.splice( 0, MAX_CONCURRENT_BUILDS - pendingHashes.size ) // grab the next batch of builds
276 | 		.forEach( commitHash => {
277 | 			l.info( { commitHash }, 'Popping a commitHash off of the buildQueue' );
278 | 			buildImageForHash( commitHash );
279 | 		} );
280 | 	if ( buildQueue.length !== currentLength ) {
281 | 		gauge( 'build_queue', buildQueue.length );
282 | 	}
283 | }
284 | 
285 | function reportQueueDepth() {
286 | 	gauge( 'build_queue', buildQueue.length );
287 | }
288 | 
289 | loop( warnOnQueueBuildup, ONE_MINUTE );
290 | loop( buildFromQueue, ONE_SECOND );
291 | 
292 | // report the queue depth every five seconds to keep statsd aggregations happy
293 | loop( reportQueueDepth, ONE_SECOND * 5 );
294 | 


--------------------------------------------------------------------------------
/src/index.ts:
--------------------------------------------------------------------------------
  1 | // external
  2 | import express from 'express';
  3 | import fs from 'fs-extra';
  4 | import striptags from 'striptags';
  5 | import useragent from 'useragent';
  6 | import { exec } from 'child_process';
  7 | 
  8 | // internal
  9 | import {
 10 | 	getCommitHashForBranch,
 11 | 	getKnownBranches,
 12 | 	hasHashLocally,
 13 | 	CommitHash,
 14 | 	NotFound,
 15 | 	getPortForContainer,
 16 | 	startContainer,
 17 | 	isContainerRunning,
 18 | 	proxyRequestToHash as proxy,
 19 | 	deleteImage,
 20 | 	getLocalImages,
 21 | 	getBranchHashes,
 22 | } from './api';
 23 | 
 24 | import { ONE_MINUTE, ONE_SECOND, TEN_MINUTES } from './constants';
 25 | 
 26 | import {
 27 | 	isBuildInProgress,
 28 | 	buildImageForHash,
 29 | 	readBuildLog,
 30 | 	addToBuildQueue,
 31 | 	cleanupBuildDir,
 32 | 	buildQueue,
 33 | 	didBuildFail,
 34 | } from './builder';
 35 | 
 36 | import {
 37 | 	redirectHashFromQueryStringToSubdomain,
 38 | 	determineCommitHash,
 39 | 	session,
 40 | 	determineEnvironment,
 41 | } from './middlewares';
 42 | import { middleware as imageRunnerMiddleware } from './image-runner';
 43 | import renderApp from './app/index';
 44 | import renderLocalImages from './app/local-images';
 45 | import renderLog from './app/log';
 46 | import renderDebug from './app/debug';
 47 | import { l, ringbuffer } from './logger';
 48 | import { increment } from './stats';
 49 | 
 50 | import {
 51 | 	refreshLocalImages,
 52 | 	refreshRemoteBranches,
 53 | 	refreshContainers,
 54 | 	cleanupExpiredContainers,
 55 | } from './api';
 56 | import { ContainerError, ImageError, ImageNotFound, InvalidImage, InvalidRegistry } from './error';
 57 | 
 58 | const startedServerAt = new Date();
 59 | increment( 'server_start' );
 60 | 
 61 | // calypso proxy server.
 62 | // checks branch names, decides to start a build or a container,
 63 | // and also proxies request to currently active container
 64 | const calypsoServer = express();
 65 | calypsoServer.use( session );
 66 | 
 67 | // global node process junk - catched unhandled errors
 68 | process.on( 'uncaughtException', error => {
 69 | 	l.info( { error }, 'Crashing on uncaught error' );
 70 | 	increment( 'uncaught_error' );
 71 | } );
 72 | 
 73 | export const promiseRejections: Map<
 74 | 	Promise< any >,
 75 | 	[ Date, any, 'reported' | 'unreported' ]
 76 | > = new Map();
 77 | const logRejections = () => {
 78 | 	const now = new Date();
 79 | 	Array.from( promiseRejections.entries() )
 80 | 		.filter(
 81 | 			( [ , [ ts, , status ] ] ) =>
 82 | 				'unreported' === status && now.getTime() - ts.getTime() > ONE_MINUTE
 83 | 		)
 84 | 		.forEach( ( [ promise, [ ts, reason ] ] ) => {
 85 | 			l.info( { reason }, 'Unhandled rejection sitting in queue for at least one minute' );
 86 | 			promiseRejections.set( promise, [ ts, reason, 'reported' ] );
 87 | 		} );
 88 | 
 89 | 	setTimeout( logRejections, ONE_MINUTE );
 90 | };
 91 | process.on( 'unhandledRejection', ( reason, promise ) => {
 92 | 	promiseRejections.set( promise, [ new Date(), reason, 'unreported' ] );
 93 | 	increment( 'unhandled_promise_rejection' );
 94 | } );
 95 | process.on( 'rejectionHandled', promise => promiseRejections.delete( promise ) );
 96 | logRejections();
 97 | 
 98 | // get application log for debugging
 99 | calypsoServer.get( '/log', ( req: express.Request, res: express.Response ) => {
100 | 	isBrowser( req )
101 | 		? res.send( renderLog( { log: ringbuffer.records, startedServerAt } ) )
102 | 		: res.send( ringbuffer.records );
103 | } );
104 | 
105 | calypsoServer.get( '/api/queue-size', ( req: express.Request, res: express.Response ) => {
106 | 	res.contentType( 'text/plain' );
107 | 	res.send( '' + buildQueue.length );
108 | 	res.end();
109 | } );
110 | 
111 | calypsoServer.get( '/localimages', ( req: express.Request, res: express.Response ) => {
112 | 	const branchHashes = getBranchHashes();
113 | 	const knownBranches = getKnownBranches();
114 | 	const localImages = Array.from( getLocalImages() ).reduce(
115 | 		( images, [ repoTags, image ] ) => ( { ...images, [ repoTags ]: image } ),
116 | 		{}
117 | 	);
118 | 
119 | 	isBrowser( req )
120 | 		? res.send(
121 | 				renderLocalImages( {
122 | 					branchHashes,
123 | 					knownBranches,
124 | 					localImages,
125 | 					startedServerAt,
126 | 				} )
127 | 		  )
128 | 		: res.send( JSON.stringify( localImages ) );
129 | } );
130 | 
131 | calypsoServer.get( '/debug', async ( req: express.Request, res: express.Response ) => {
132 | 	try {
133 | 		res.send(
134 | 			await renderDebug( {
135 | 				startedServerAt,
136 | 			} )
137 | 		);
138 | 	} catch ( err ) {
139 | 		l.error( { err }, 'Error rendering debug' );
140 | 		res.send( 'error rendering debug' );
141 | 		res.end();
142 | 	}
143 | } );
144 | 
145 | calypsoServer.use( imageRunnerMiddleware );
146 | calypsoServer.use( redirectHashFromQueryStringToSubdomain );
147 | calypsoServer.use( determineCommitHash );
148 | calypsoServer.use( determineEnvironment );
149 | 
150 | calypsoServer.get( '/status', async ( req: express.Request, res: express.Response ) => {
151 | 	const commitHash = req.session.commitHash;
152 | 	let status;
153 | 	if ( isContainerRunning( commitHash ) ) {
154 | 		status = 'Ready';
155 | 	} else if ( didBuildFail( commitHash ) ) {
156 | 		status = 'FAIL';
157 | 	} else if ( await hasHashLocally( commitHash ) ) {
158 | 		status = 'NeedsPriming';
159 | 	} else if ( await isBuildInProgress( commitHash ) ) {
160 | 		status = 'Building';
161 | 	} else {
162 | 		status = 'NotBuilt';
163 | 	}
164 | 	res.send( status );
165 | 	res.end();
166 | } );
167 | 
168 | calypsoServer.get( '*', async ( req: express.Request, res: express.Response ) => {
169 | 	const { commitHash, runEnv } = req.session;
170 | 	const hasLocally = await hasHashLocally( commitHash );
171 | 	const isCurrentlyBuilding = ! hasLocally && ( await isBuildInProgress( commitHash ) );
172 | 	const needsToBuild = ! isCurrentlyBuilding && ! hasLocally;
173 | 	const shouldStartContainer = hasLocally && ! isContainerRunning( commitHash, runEnv );
174 | 	const shouldReset = req.query.reset;
175 | 
176 | 	if ( shouldReset ) {
177 | 		l.info( { commitHash }, `Hard reset for ${ commitHash }` );
178 | 		increment( 'hash_reset' );
179 | 		await deleteImage( commitHash );
180 | 		await cleanupBuildDir( commitHash );
181 | 		const response = `hard reset hash: ${ commitHash } and loading it now...`;
182 | 		res.set( 'Refresh', `5;url=${ req.path }` );
183 | 		res.send( striptags( response ) );
184 | 		return;
185 | 	}
186 | 
187 | 	if ( isContainerRunning( commitHash, runEnv ) ) {
188 | 		proxy( req, res );
189 | 		return;
190 | 	}
191 | 
192 | 	let buildLog;
193 | 	let message;
194 | 	if ( isCurrentlyBuilding ) {
195 | 		buildLog = await readBuildLog( commitHash );
196 | 	} else if ( needsToBuild ) {
197 | 		message = 'Starting build now';
198 | 		addToBuildQueue( commitHash );
199 | 	} else if ( shouldStartContainer ) {
200 | 		//message = 'Just started your hash, this page will restart automatically';
201 | 		// TODO: fix race condition where multiple containers may be spun up
202 | 		// within the same subsecond time period.
203 | 		try {
204 | 			await startContainer( commitHash, runEnv );
205 | 			res.set( 'Refresh', `1;url=${ req.path }` );
206 | 			res.send( striptags( 'build complete, loading now...' ) );
207 | 			return;
208 | 		} catch ( err ) {
209 | 			message = 'Error starting that commit...';
210 | 			l.error( { err }, 'Error starting commit' );
211 | 		}
212 | 	}
213 | 
214 | 	renderApp( { message, buildLog, startedServerAt } ).pipe( res );
215 | } );
216 | 
217 | // log errors
218 | calypsoServer.use(
219 | 	( err: Error, req: express.Request, res: express.Response, next: express.NextFunction ) => {
220 | 
221 | 		if (err instanceof ImageNotFound) {
222 | 			l.warn( {err, url:req.originalUrl, image:err.name});
223 | 			res.status(404).send(`Image ${err.name} not found`);
224 | 			return;
225 | 		}
226 | 
227 | 		if (err instanceof InvalidImage) {
228 | 			l.warn( {err, url:req.originalUrl, image:err.name});
229 | 			res.status(403).send(err.message);
230 | 			return;
231 | 		}
232 | 
233 | 		if (err instanceof InvalidRegistry) {
234 | 			l.warn( {err, url:req.originalUrl, registry:err.registry});
235 | 			res.status(403).send(`Registry ${err.registry} is not valid`);
236 | 			return;
237 | 		}
238 | 
239 | 		if (err instanceof ContainerError) {
240 | 			l.error( {err, url:req.originalUrl, containerName: err.containerName});
241 | 			res.status(500).send(err.message);
242 | 			return;
243 | 		}
244 | 
245 | 		if (err instanceof ImageError) {
246 | 			l.error( {err, url:req.originalUrl, imageName: err.imageName});
247 | 			res.status(500).send(err.message);
248 | 			return;
249 | 		}
250 | 
251 | 		// Catch all for unknown errors
252 | 		l.error(err);
253 | 		res.status(500).send(err.message);
254 | 	}
255 | );
256 | 
257 | const server = calypsoServer.listen( 3000, () =>
258 | 	l.info(
259 | 		`✅ dserve is listening on 3000 - started at ${ startedServerAt.toLocaleTimeString( undefined, {
260 | 			timeZoneName: 'long',
261 | 			hour12: true,
262 | 		} ) }`
263 | 	)
264 | );
265 | 
266 | server.on( 'error', err => {
267 | 	console.log( 'err' );
268 | 	l.error( err, 'Error serving request' );
269 | } );
270 | server.on( 'close', () => {
271 | 	console.log( 'close' );
272 | 	l.info( {}, 'Server shutting down' );
273 | } );
274 | 
275 | function isBrowser( req: express.Request ): Boolean {
276 | 	const ua = useragent.lookup( req.header( 'user-agent' ) );
277 | 	const family = ua.family.toLocaleLowerCase();
278 | 
279 | 	return (
280 | 		family === 'chrome' ||
281 | 		family === 'safari' ||
282 | 		family === 'firefox' ||
283 | 		family === 'chrome mobile' ||
284 | 		family === 'mobile safari'
285 | 	);
286 | }
287 | 
288 | if ( process.env.NODE_ENV !== 'test' ) {
289 | 	const loop = ( f: Function, delay: number ) => {
290 | 		const run = async () => {
291 | 			try {
292 | 				await f();
293 | 			} catch ( e ) {
294 | 				l.error( e );
295 | 			} finally {
296 | 				setTimeout( run, delay );
297 | 			}
298 | 		};
299 | 
300 | 		run();
301 | 	};
302 | 
303 | 	loop( refreshLocalImages, 5 * ONE_SECOND );
304 | 	loop( refreshContainers, 5 * ONE_SECOND );
305 | 	loop( refreshRemoteBranches, ONE_MINUTE );
306 | 	// Wait a bit before starting the expired container cleanup.
307 | 	// This gives us some time to accumulate accesses to existing containers across app restarts
308 | 	setTimeout( () => loop( cleanupExpiredContainers, ONE_MINUTE ), TEN_MINUTES );
309 | }
310 | 


--------------------------------------------------------------------------------
/src/app/debug.tsx:
--------------------------------------------------------------------------------
  1 | import React from 'react';
  2 | import ReactDOMServer from 'react-dom/server';
  3 | import Dockerode from 'dockerode';
  4 | import os from 'os';
  5 | 
  6 | import { ONE_MINUTE } from '../constants';
  7 | import { Shell } from './app-shell';
  8 | import { promiseRejections } from '../index';
  9 | import { humanSize, humanRelativeTime, percent, round } from './util';
 10 | 
 11 | import {
 12 | 	getAllContainers,
 13 | 	getLocalImages,
 14 | 	extractCommitFromImage,
 15 | 	extractEnvironmentFromImage,
 16 | 	getContainerAccessTime,
 17 | 	getContainerName,
 18 | } from '../api';
 19 | import { buildQueue, pendingHashes } from '../builder';
 20 | 
 21 | const Docker = new Dockerode();
 22 | 
 23 | async function maxTime( p: Promise< any >, timeoutMs: number ) {
 24 | 	const timeout = new Promise( ( resolve, reject ) => {
 25 | 		setTimeout( () => reject( 'timeout' ), timeoutMs );
 26 | 	} );
 27 | 	return Promise.race( [ p, timeout ] );
 28 | }
 29 | 
 30 | const Debug = ( c: RenderContext ) => {
 31 | 	const memUsage = process.memoryUsage();
 32 | 	const heapU = memUsage.heapUsed;
 33 | 	const heapT = memUsage.heapTotal;
 34 | 	const memTotal = os.totalmem();
 35 | 	const memUsed = memTotal - os.freemem();
 36 | 	const images = Array.from( getLocalImages().entries() ) as Array<
 37 | 		[ string, Dockerode.ImageInfo ]
 38 | 	>;
 39 | 	const apiContainers = Array.from( getAllContainers().entries() );
 40 | 
 41 | 	const shortHash = ( hash: string = '', length = 30 ) => (
 42 | 		<span title={ hash }>{ hash.slice( 0, length ) }…</span>
 43 | 	);
 44 | 
 45 | 	return (
 46 | 		<Shell refreshInterval={ ONE_MINUTE } startedServerAt={ c.startedServerAt }>
 47 | 			<style
 48 | 				dangerouslySetInnerHTML={ {
 49 | 					__html: `
 50 |                     .dserve-debug-cards {
 51 |                         display: grid;
 52 |                         grid-template-columns: repeat(6, 1fr);
 53 |                         grid-gap: 10px;
 54 |                         grid-auto-rows: minmax(100px, auto);
 55 |                     }
 56 | 
 57 |                     .dserve-debug-cards figure {
 58 |                         border: 1px solid gray;
 59 |                         border-radius: 4px;
 60 |                         padding: 8px;
 61 |                         position: relative;
 62 |                     }
 63 | 
 64 |                     .system {
 65 |                         grid-row: 1;
 66 |                         grid-column: 1 / 3;
 67 |                     }
 68 | 
 69 |                     .queue {
 70 |                         grid-row: 1;
 71 |                         grid-column: 3 / 5;
 72 |                     }
 73 | 
 74 |                     .promises {
 75 |                         grid-row: 1;
 76 |                         grid-column: 5 / 7;
 77 |                     }
 78 | 
 79 |                     .api {
 80 |                         grid-row: 2;
 81 |                         grid-column: 1 / 4;
 82 |                     }
 83 | 
 84 |                     .docker {
 85 |                         grid-row: 2;
 86 |                         grid-column: 4 / 7;
 87 |                     }
 88 | 
 89 |                     .dserve-debug-cards figure figcaption {
 90 |                         border: 1px solid gray;
 91 |                         border-radius: 2px;
 92 |                         padding: 4px;
 93 |                         position: absolute;
 94 |                         top: 2px;
 95 |                         right: 2px;
 96 |                     }
 97 | 
 98 |                     .dserve-debug-cards a[href] {
 99 |                         color: #00d8ff;
100 |                     }
101 | 
102 |                     .dserve-container-list li ,
103 |                     .dserve-image-list li {
104 |                         margin-bottom: 8px;
105 |                     }
106 | 
107 |                     .dserve-container-list li.created strong:before {
108 |                         content: '🈺 ';
109 |                     }
110 | 
111 |                     .dserve-container-list li.exited strong:before {
112 |                         content: '🈵 ';
113 |                     }
114 | 
115 |                     .dserve-container-list li.running strong:before {
116 |                         content: '✅ ';
117 |                     }`,
118 | 				} }
119 | 			/>
120 | 			<div className="dserve-debug-cards">
121 | 				<figure className="system">
122 | 					<p>
123 | 						CPU (x{ os.cpus().length }):{' '}
124 | 						{ os
125 | 							.loadavg()
126 | 							.map( a => round( a, 2 ) )
127 | 							.join( ', ' ) }{' '}
128 | 						(1m, 5m, 15m)
129 | 					</p>
130 | 					<p>Memory</p>
131 | 					<ul>
132 | 						<li>
133 | 							heap: { humanSize( heapU ) } / { humanSize( heapT ) } ({ percent( heapU, heapT ) }%)
134 | 						</li>
135 | 						<li>
136 | 							system: { humanSize( memUsed ) } / { humanSize( memTotal ) } (
137 | 							{ percent( memUsed, memTotal ) }%)
138 | 						</li>
139 | 					</ul>
140 | 					<figcaption>System Load</figcaption>
141 | 				</figure>
142 | 
143 | 				<figure className="queue">
144 | 					<p>Building</p>
145 | 					{ pendingHashes.size ? (
146 | 						<ul>
147 | 							{ Array.from( pendingHashes ).map( hash => (
148 | 								<li key={ hash }>
149 | 									<a href={ `/?hash=${ hash }` }>{ hash.substr( 0, 8 ) }</a>{' '}
150 | 									<a href={ `https://github.com/Automattic/wp-calypso/commit/${ hash }` }>github</a>
151 | 								</li>
152 | 							) ) }
153 | 						</ul>
154 | 					) : (
155 | 						<p>
156 | 							<em>Nothing building</em>
157 | 						</p>
158 | 					) }
159 | 					<p>Queue</p>
160 | 					{ buildQueue.length ? (
161 | 						<ul>
162 | 							{ buildQueue.map( hash => (
163 | 								<li key={ hash }>
164 | 									<a href={ `/?hash=${ hash }` }>{ hash.substr( 0, 8 ) }</a>{' '}
165 | 									<a href={ `https://github.com/Automattic/wp-calypso/commit/${ hash }` }>github</a>
166 | 								</li>
167 | 							) ) }
168 | 						</ul>
169 | 					) : (
170 | 						<p>
171 | 							<em>Nothing queued</em>
172 | 						</p>
173 | 					) }
174 | 					<figcaption>Builder</figcaption>
175 | 				</figure>
176 | 
177 | 				<figure className="promises">
178 | 					{ promiseRejections.size ? (
179 | 						<ul>
180 | 							{ Array.from( promiseRejections.values() ).map( ( [ ts, reason ], i ) => (
181 | 								<li key={ i }>
182 | 									<time
183 | 										dateTime={ ts.toISOString() }
184 | 										title={ ts.toLocaleTimeString( undefined, {
185 | 											timeZoneName: 'long',
186 | 											hour12: true,
187 | 										} ) }
188 | 									>
189 | 										{ humanRelativeTime( ts.getTime() / 1000 ) }
190 | 									</time>{' '}
191 | 									{ reason.toString() }
192 | 								</li>
193 | 							) ) }
194 | 						</ul>
195 | 					) : (
196 | 						<p>
197 | 							<em>No unhandled rejected promises</em>
198 | 						</p>
199 | 					) }
200 | 					<figcaption>Rejected Promises</figcaption>
201 | 				</figure>
202 | 
203 | 				<figure className="api">
204 | 					<p>Running Containers</p>
205 | 					<ul>
206 | 						{ apiContainers.length === 0 ? (
207 | 							<li>
208 | 								<em>No running containers</em>
209 | 							</li>
210 | 						) : (
211 | 							apiContainers.map( ( [ key, info ] ) => {
212 | 								const commit = extractCommitFromImage( info.Image );
213 | 								const env = extractEnvironmentFromImage( info );
214 | 								return (
215 | 									<li key={ info.Id } className={ info.State }>
216 | 										<strong>{ info.Names }</strong> - { shortHash( info.Id ) }
217 | 										<br />
218 | 										Commit:{' '}
219 | 										{ commit ? (
220 | 											<a href={ `/?hash=${ commit }&env=${ env || '' }` }>{ commit }</a>
221 | 										) : (
222 | 											'none'
223 | 										) }
224 | 										<br />
225 | 										Image ID: { shortHash( info.ImageID ) }
226 | 										<br />
227 | 										Environment: { env || 'unknown' }
228 | 										<br />
229 | 										Status: { info.Status }
230 | 										<br />
231 | 										Last Access:{' '}
232 | 										{ getContainerAccessTime( getContainerName( info ) )
233 | 											? humanRelativeTime(
234 | 													getContainerAccessTime( getContainerName( info ) ) / 1000
235 | 											  )
236 | 											: 'never' }
237 | 									</li>
238 | 								);
239 | 							} )
240 | 						) }
241 | 					</ul>
242 | 
243 | 					<p>DServe Images</p>
244 | 					<p>
245 | 						Total storage size:{' '}
246 | 						{ humanSize( images.reduce( ( size, [ , info ] ) => size + info.Size, 0 ) ) }
247 | 					</p>
248 | 					<ul>
249 | 						{ images.length === 0 ? (
250 | 							<li>
251 | 								<em>No dserve images</em>
252 | 							</li>
253 | 						) : (
254 | 							images.map( ( [ key, info ] ) => (
255 | 								<li key={ info.Id }>
256 | 									Commit:{' '}
257 | 									<strong>
258 | 										{ info.RepoTags ? (
259 | 											<a href={ `/?hash=${ extractCommitFromImage( info.RepoTags.join( ' ' ) ) }` }>
260 | 												{ extractCommitFromImage( info.RepoTags.join( ' ' ) ) }{' '}
261 | 											</a>
262 | 										) : (
263 | 											'None'
264 | 										) }
265 | 									</strong>
266 | 									<br />
267 | 									Id: { shortHash( info.Id ) }
268 | 									<br />
269 | 									Size: { humanSize( info.Size ) }
270 | 									<br />
271 | 									Created: { humanRelativeTime( info.Created ) }
272 | 								</li>
273 | 							) )
274 | 						) }
275 | 					</ul>
276 | 
277 | 					<figcaption>API</figcaption>
278 | 				</figure>
279 | 
280 | 				<figure className="docker">
281 | 					<p>All Containers</p>
282 | 					<ul className="dserve-container-list">
283 | 						{ c.docker.containers.length === 0 ? (
284 | 							<li>
285 | 								<em>No containers</em>
286 | 							</li>
287 | 						) : (
288 | 							c.docker.containers
289 | 								.sort( ( a, b ) => b.Created - a.Created )
290 | 								.map( info => (
291 | 									<li key={ info.Id } className={ info.State }>
292 | 										<strong>{ info.Names }</strong> - { shortHash( info.Id ) }
293 | 										<br />
294 | 										Image ID: { shortHash( info.ImageID ) }
295 | 										<br />
296 | 										Status: { info.State } - { info.Status }
297 | 										<br />
298 | 										Environment: { extractEnvironmentFromImage( info ) || 'unknown' }
299 | 									</li>
300 | 								) )
301 | 						) }
302 | 					</ul>
303 | 
304 | 					<p>All Images</p>
305 | 					<ul className="dserve-image-list">
306 | 						{ c.docker.images.length === 0 ? (
307 | 							<li>
308 | 								<em>No docker images</em>
309 | 							</li>
310 | 						) : (
311 | 							c.docker.images
312 | 								.sort( ( a, b ) => b.Created - a.Created )
313 | 								.map( info => (
314 | 									<li key={ info.Id }>
315 | 										RepoTags:{' '}
316 | 										<strong>
317 | 											{ info.RepoTags ? shortHash( info.RepoTags.join( ', ' ), 38 ) : 'None' }
318 | 										</strong>
319 | 										<br />
320 | 										Id: { shortHash( info.Id ) }
321 | 										<br />
322 | 										Size: { humanSize( info.Size ) }
323 | 										<br />
324 | 										Created: { humanRelativeTime( info.Created ) }
325 | 									</li>
326 | 								) )
327 | 						) }
328 | 					</ul>
329 | 					<figcaption>Docker</figcaption>
330 | 				</figure>
331 | 			</div>
332 | 		</Shell>
333 | 	);
334 | };
335 | 
336 | type RenderContext = {
337 | 	startedServerAt: Date;
338 | 	docker: {
339 | 		containers: Array< Dockerode.ContainerInfo >;
340 | 		images: Array< Dockerode.ImageInfo >;
341 | 	};
342 | };
343 | 
344 | export default async function renderDebug( { startedServerAt }: { startedServerAt: Date } ) {
345 | 	let containers, images;
346 | 	try {
347 | 		containers = await maxTime( Docker.listContainers( { all: true } ), 5000 );
348 | 	} catch ( err ) {
349 | 		containers = [];
350 | 	}
351 | 	try {
352 | 		images = await maxTime( Docker.listImages(), 5000 );
353 | 	} catch ( err ) {
354 | 		images = [];
355 | 	}
356 | 	return ReactDOMServer.renderToStaticMarkup(
357 | 		<Debug
358 | 			startedServerAt={ startedServerAt }
359 | 			docker={ {
360 | 				containers,
361 | 				images,
362 | 			} }
363 | 		/>
364 | 	);
365 | }
366 | 


--------------------------------------------------------------------------------
/src/api.ts:
--------------------------------------------------------------------------------
  1 | import httpProxy from 'http-proxy';
  2 | import Docker, { ImageInfo } from 'dockerode';
  3 | import _ from 'lodash';
  4 | import getPort from 'get-port';
  5 | import git from 'nodegit';
  6 | import fs from 'fs-extra';
  7 | import path from 'path';
  8 | import { promisify } from 'util';
  9 | import { ContainerInfo } from 'dockerode';
 10 | 
 11 | import { config, envContainerConfig } from './config';
 12 | import { l } from './logger';
 13 | import { pendingHashes } from './builder';
 14 | import { exec } from 'child_process';
 15 | 
 16 | import { CONTAINER_EXPIRY_TIME } from './constants';
 17 | import { timing } from './stats';
 18 | import { ContainerError, ImageError } from './error';
 19 | 
 20 | type APIState = {
 21 | 	accesses: Map< ContainerName, number >;
 22 | 	branchHashes: Map< CommitHash, BranchName >;
 23 | 	containers: Map< string, Docker.ContainerInfo >;
 24 | 	localImages: Map< ImageName, Docker.ImageInfo >;
 25 | 	pullingImages: Map< ImageName, Promise< DockerodeStream > >;
 26 | 	remoteBranches: Map< BranchName, CommitHash >;
 27 | 	startingContainers: Map< CommitHash, Promise< ContainerInfo > >;
 28 | };
 29 | 
 30 | export const state: APIState = {
 31 | 	accesses: new Map(),
 32 | 	branchHashes: new Map(),
 33 | 	containers: new Map(),
 34 | 	localImages: new Map(),
 35 | 	pullingImages: new Map(),
 36 | 	remoteBranches: new Map(),
 37 | 	startingContainers: new Map(),
 38 | };
 39 | 
 40 | export const docker = new Docker();
 41 | 
 42 | // types
 43 | export type NotFound = Error;
 44 | export type CommitHash = string;
 45 | export type BranchName = string;
 46 | export type PortNumber = number;
 47 | export type ImageStatus = 'NoImage' | 'Inactive' | PortNumber;
 48 | export type RunEnv = string;
 49 | export type DockerRepository = string;
 50 | export type ImageName = string;
 51 | export type ContainerName = string;
 52 | export type DockerodeStream = any;
 53 | export type ContainerSearchOptions = {
 54 | 	image?: ImageName;
 55 | 	env?: RunEnv;
 56 | 	status?: string;
 57 | 	id?: string;
 58 | 	name?: string;
 59 | 	sanitizedName?: string;
 60 | };
 61 | 
 62 | export const getImageName = ( hash: CommitHash ) => `${ config.build.tagPrefix }:${ hash }`;
 63 | export const extractCommitFromImage = ( imageName: string ): CommitHash => {
 64 | 	const [ prefix, sha ] = imageName.split( ':' );
 65 | 	if ( prefix !== config.build.tagPrefix ) {
 66 | 		return null;
 67 | 	}
 68 | 	return sha;
 69 | };
 70 | 
 71 | export const extractEnvironmentFromImage = ( image: ContainerInfo ): RunEnv => {
 72 | 	return image.Labels.calypsoEnvironment || undefined;
 73 | };
 74 | 
 75 | /**
 76 |  * Polls the local Docker daemon to fetch an updated list of images
 77 |  *
 78 |  * It saves them in the Map `stcate.localImages`, indexed by tag name. If an image has more than
 79 |  * one tag it will appear multiple times in the map.
 80 |  */
 81 | export async function refreshLocalImages() {
 82 | 	const images = await docker.listImages();
 83 | 	state.localImages = new Map(
 84 | 		images.reduce(
 85 | 			( acc, image ) => [
 86 | 				...acc,
 87 | 				...( image.RepoTags || [] ).map( tag => [ tag, image ] as [ ImageName, ImageInfo ] ),
 88 | 			],
 89 | 			[]
 90 | 		)
 91 | 	);
 92 | }
 93 | 
 94 | /**
 95 |  * Returns the list of images built by dserve
 96 |  */
 97 | export function getLocalImages() {
 98 | 	return new Map(
 99 | 		Array.from( state.localImages.entries() ).filter( ( [ imageName ] ) =>
100 | 			imageName.startsWith( config.build.tagPrefix )
101 | 		)
102 | 	);
103 | }
104 | 
105 | /**
106 |  * Returns the list of all images
107 |  */
108 | export function getAllImages() {
109 | 	return state.localImages;
110 | }
111 | 
112 | export function getAllContainers() {
113 | 	return state.containers;
114 | }
115 | 
116 | export function isValidImage( imageInfo: ImageInfo ) {
117 | 	if ( ! imageInfo.Labels ) return false;
118 | 
119 | 	for ( const [ label, value ] of Object.entries( config.allowedLabels ) ) {
120 | 		if ( imageInfo.Labels[ label ] !== value ) return false;
121 | 	}
122 | 	return true;
123 | }
124 | 
125 | export async function hasHashLocally( hash: CommitHash ): Promise< boolean > {
126 | 	return getLocalImages().has( getImageName( hash ) );
127 | }
128 | 
129 | export async function deleteImage( hash: CommitHash ) {
130 | 	l.info( { commitHash: hash }, 'attempting to remove image for hash' );
131 | 
132 | 	const runningContainer = state.containers.get( getImageName( hash ) );
133 | 	if ( runningContainer ) {
134 | 		await docker.getContainer( runningContainer.Id ).stop();
135 | 	}
136 | 
137 | 	let img;
138 | 	try {
139 | 		img = docker.getImage( getImageName( hash ) );
140 | 		if ( ! img ) {
141 | 			l.info(
142 | 				{ commitHash: hash },
143 | 				'did not have an image locally with name' + getImageName( hash )
144 | 			);
145 | 			return;
146 | 		}
147 | 	} catch ( err ) {
148 | 		l.info(
149 | 			{ commitHash: hash, err },
150 | 			'error trying to find image locally with name' + getImageName( hash )
151 | 		);
152 | 		return;
153 | 	}
154 | 
155 | 	try {
156 | 		await img.remove( { force: true } );
157 | 		l.info( { commitHash: hash }, 'succesfully removed image' );
158 | 	} catch ( err ) {
159 | 		l.error( { err, commitHash: hash }, 'failed to remove image' );
160 | 	}
161 | }
162 | export async function startContainer( commitHash: CommitHash, env: RunEnv ) {
163 | 	//l.info( { commitHash }, `Request to start a container for ${ commitHash }` );
164 | 	const image = getImageName( commitHash );
165 | 	const containerId = `${ env }:${ image }`;
166 | 
167 | 	// do we have an existing container?
168 | 	const existingContainer = getRunningContainerForHash( commitHash, env );
169 | 	if ( existingContainer ) {
170 | 		l.info(
171 | 			{ commitHash, containerId: existingContainer.Id },
172 | 			`Found a running container for ${ commitHash }`
173 | 		);
174 | 		return Promise.resolve( existingContainer );
175 | 	}
176 | 
177 | 	// are we starting one already?
178 | 	if ( state.startingContainers.has( containerId ) ) {
179 | 		//l.info( { commitHash }, `Already starting a container for ${ commitHash }` );
180 | 		return state.startingContainers.get( containerId );
181 | 	}
182 | 
183 | 	async function start(
184 | 		image: string,
185 | 		commitHash: CommitHash,
186 | 		env: RunEnv
187 | 	): Promise< ContainerInfo > {
188 | 		// ok, try to start one
189 | 		let freePort: number;
190 | 		try {
191 | 			freePort = await getPort();
192 | 		} catch ( err ) {
193 | 			l.error(
194 | 				{ err, image, commitHash },
195 | 				`Error while attempting to find a free port for ${ image }`
196 | 			);
197 | 			throw err;
198 | 		}
199 | 
200 | 		const exposedPort = `${ config.build.exposedPort }/tcp`;
201 | 		const dockerPromise = new Promise( ( resolve, reject ) => {
202 | 			let runError: any;
203 | 
204 | 			l.info( { image, commitHash }, `Starting a container for ${ commitHash }` );
205 | 
206 | 			docker.run(
207 | 				image,
208 | 				[],
209 | 				process.stdout,
210 | 				{
211 | 					...config.build.containerCreateOptions,
212 | 					...envContainerConfig( env ),
213 | 					ExposedPorts: { [ exposedPort ]: {} },
214 | 					PortBindings: { [ exposedPort ]: [ { HostPort: freePort.toString() } ] },
215 | 					Tty: false,
216 | 					Labels: {
217 | 						calypsoEnvironment: env,
218 | 					},
219 | 				},
220 | 				err => {
221 | 					runError = err;
222 | 				}
223 | 			);
224 | 
225 | 			// run will never callback for calypso when things work as intended.
226 | 			// wait 5 seconds. If we don't see an error by then, assume run worked and resolve
227 | 			setTimeout( () => {
228 | 				if ( runError ) {
229 | 					reject( { error: runError, freePort } );
230 | 				} else {
231 | 					resolve( { freePort } );
232 | 				}
233 | 			}, 5000 );
234 | 		} );
235 | 		return dockerPromise.then(
236 | 			( { success, freePort } ) => {
237 | 				l.info(
238 | 					{ image, freePort, commitHash },
239 | 					`Successfully started container for ${ image } on ${ freePort }`
240 | 				);
241 | 				return refreshContainers().then( () => getRunningContainerForHash( commitHash ) );
242 | 			},
243 | 			( { error, freePort } ) => {
244 | 				l.error(
245 | 					{ image, freePort, error, commitHash },
246 | 					`Failed starting container for ${ image } on ${ freePort }`
247 | 				);
248 | 				throw error;
249 | 			}
250 | 		);
251 | 	}
252 | 
253 | 	const startPromise = start( image, commitHash, env );
254 | 
255 | 	state.startingContainers.set( containerId, startPromise );
256 | 	startPromise.then(
257 | 		s => {
258 | 			state.startingContainers.delete( containerId );
259 | 			return s;
260 | 		},
261 | 		err => {
262 | 			state.startingContainers.delete( containerId );
263 | 			throw err;
264 | 		}
265 | 	);
266 | 	return startPromise;
267 | }
268 | 
269 | export async function refreshContainers() {
270 | 	const containers = await docker.listContainers( { all: true } );
271 | 	state.containers = new Map(
272 | 		containers.map( container => [ container.Id, container ] as [ string, ContainerInfo ] )
273 | 	);
274 | }
275 | 
276 | export function getRunningContainerForHash( hash: CommitHash, env?: RunEnv ): ContainerInfo | null {
277 | 	const image = getImageName( hash );
278 | 	return Array.from( state.containers.values() ).find(
279 | 		ci =>
280 | 			ci.Image === image &&
281 | 			ci.State === 'running' &&
282 | 			( ! env || env === extractEnvironmentFromImage( ci ) )
283 | 	);
284 | }
285 | 
286 | export function isContainerRunning( hash: CommitHash, env?: RunEnv ): boolean {
287 | 	return !! getRunningContainerForHash( hash, env );
288 | }
289 | 
290 | export function getPortForContainer( hash: CommitHash, env: RunEnv ): number | boolean {
291 | 	const container = getRunningContainerForHash( hash, env );
292 | 
293 | 	if ( ! container ) {
294 | 		return false;
295 | 	}
296 | 
297 | 	const ports = container.Ports;
298 | 
299 | 	return ports.length > 0 ? ports[ 0 ].PublicPort : false;
300 | }
301 | 
302 | async function getRemoteBranches(): Promise< Map< string, string > > {
303 | 	const repoDir = path.join( __dirname, '../repos' );
304 | 	const calypsoDir = path.join( repoDir, 'wp-calypso' );
305 | 	let repo: git.Repository;
306 | 
307 | 	const start = Date.now();
308 | 
309 | 	try {
310 | 		if ( ! ( await fs.pathExists( repoDir ) ) ) {
311 | 			await fs.mkdir( repoDir );
312 | 		}
313 | 		if ( ! ( await fs.pathExists( calypsoDir ) ) ) {
314 | 			repo = await git.Clone.clone( `https://github.com/${ config.repo.project }`, calypsoDir );
315 | 		} else {
316 | 			repo = await git.Repository.open( calypsoDir );
317 | 		}
318 | 
319 | 		// this code here is all for retrieving origin
320 | 		// and then pruning out old branches
321 | 		const origin: git.Remote = await repo.getRemote( 'origin' );
322 | 		await origin.connect( git.Enums.DIRECTION.FETCH, {} );
323 | 		await origin.download( null );
324 | 		const pruneError = origin.prune( new git.RemoteCallbacks() );
325 | 		if ( pruneError ) {
326 | 			throw new Error( `invoking remote prune returned error code: ${ pruneError }` );
327 | 		}
328 | 		//
329 | 		await repo.fetchAll();
330 | 	} catch ( err ) {
331 | 		l.error( { err }, 'Could not fetch repo to update branches list' );
332 | 	}
333 | 
334 | 	if ( ! repo ) {
335 | 		l.error( 'Something went very wrong while trying to refresh branches' );
336 | 	}
337 | 
338 | 	timing( 'git.refresh', Date.now() - start );
339 | 
340 | 	try {
341 | 		const branchesReferences = ( await repo.getReferences() ).filter(
342 | 			( x: git.Reference ) => x.isBranch
343 | 		);
344 | 
345 | 		const branchToCommitHashMap: Map< string, string > = new Map(
346 | 			branchesReferences.map( reference => {
347 | 				const name = reference.shorthand().replace( 'origin/', '' );
348 | 				const commitHash = reference.target().tostrS();
349 | 
350 | 				return [ name, commitHash ] as [ string, CommitHash ];
351 | 			} )
352 | 		);
353 | 
354 | 		// gc the repo if no builds are running
355 | 		if ( pendingHashes.size === 0 ) {
356 | 			try {
357 | 				await promisify( exec )( 'git gc', {
358 | 					cwd: calypsoDir,
359 | 				} );
360 | 			} catch ( err ) {
361 | 				l.error( { err }, 'git gc failed' );
362 | 			}
363 | 		}
364 | 
365 | 		return branchToCommitHashMap;
366 | 	} catch ( err ) {
367 | 		l.error(
368 | 			{ err, repository: config.repo.project },
369 | 			'Error creating branchName --> commitSha map'
370 | 		);
371 | 		return;
372 | 	}
373 | }
374 | 
375 | let refreshingPromise: Promise< any > = null;
376 | export async function refreshRemoteBranches() {
377 | 	if ( refreshingPromise ) {
378 | 		return refreshingPromise;
379 | 	}
380 | 
381 | 	refreshingPromise = ( async () => {
382 | 		const branches = await getRemoteBranches();
383 | 
384 | 		if ( branches ) {
385 | 			state.branchHashes = new Map(
386 | 				Array.from( branches ).map( ( [ a, b ] ) => [ b, a ] as [ CommitHash, BranchName ] )
387 | 			);
388 | 
389 | 			state.remoteBranches = branches;
390 | 		}
391 | 	} )();
392 | 
393 | 	function letItGo() {
394 | 		refreshingPromise = null;
395 | 	}
396 | 
397 | 	refreshingPromise.then( letItGo, letItGo ); // errors never bothered me anyway
398 | 
399 | 	return refreshingPromise;
400 | }
401 | 
402 | export function getBranchHashes() {
403 | 	return state.branchHashes;
404 | }
405 | 
406 | export function getKnownBranches() {
407 | 	return state.remoteBranches;
408 | }
409 | 
410 | export function getCommitHashForBranch( branch: BranchName ): CommitHash | undefined {
411 | 	return state.remoteBranches.get( branch );
412 | }
413 | 
414 | export function touchCommit( hash: CommitHash, env?: RunEnv ) {
415 | 	const container = getRunningContainerForHash( hash, env );
416 | 	if ( ! container ) throw `Running container for commit ${ hash } not found}`;
417 | 
418 | 	const name = getContainerName( container );
419 | 	touchContainer( name );
420 | }
421 | 
422 | export function touchContainer( name: ContainerName ) {
423 | 	state.accesses.set( name, Date.now() );
424 | }
425 | 
426 | export function getContainerAccessTime( name: ContainerName ): number | undefined {
427 | 	return state.accesses.get( name );
428 | }
429 | 
430 | /*
431 |  * Get all currently running containers that have expired.
432 |  * Expired means have not been accessed in EXPIRED_DURATION
433 |  */
434 | export function getExpiredContainers() {
435 | 	// Filter off containers that are still valid
436 | 	return Array.from( state.containers.values() ).filter( ( container: ContainerInfo ) => {
437 | 		if ( container.State === 'dead' || container.State === 'created' ) {
438 | 			// ignore dead and just created containers
439 | 			return false;
440 | 		}
441 | 
442 | 		if ( container.State === 'exited' ) {
443 | 			// these are done, remove 'em
444 | 			return true;
445 | 		}
446 | 
447 | 		const createdAgo = Date.now() - container.Created * 1000;
448 | 		const lastAccessed = getContainerAccessTime( getContainerName( container ) );
449 | 
450 | 		return (
451 | 			createdAgo > CONTAINER_EXPIRY_TIME &&
452 | 			( _.isUndefined( lastAccessed ) || Date.now() - lastAccessed > CONTAINER_EXPIRY_TIME )
453 | 		);
454 | 	} );
455 | }
456 | 
457 | // stop any container that hasn't been accessed within ten minutes
458 | export async function cleanupExpiredContainers() {
459 | 	await refreshContainers();
460 | 	const expiredContainers = getExpiredContainers();
461 | 	for ( let container of expiredContainers ) {
462 | 		const containerName = getContainerName( container )
463 | 
464 | 		const log = l.child({
465 | 			containerId: container.Id,
466 | 			containerName
467 | 		})
468 | 
469 | 		log.info( 'Cleaning up stale container' );
470 | 
471 | 		if ( container.State === 'running' ) {
472 | 			try {
473 | 				await docker.getContainer( container.Id ).stop();
474 | 				log.info( `Successfully stopped container` );
475 | 			} catch ( err ) {
476 | 				log.error( err, 'Failed to stop container' );
477 | 			}
478 | 		}
479 | 		try {
480 | 			await docker.getContainer( container.Id ).remove();
481 | 			log.info(`Successfully removed container` );
482 | 		} catch ( err ) {
483 | 			log.error( err, 'Failed to remove container' );
484 | 		}
485 | 	}
486 | 	refreshContainers();
487 | }
488 | 
489 | const proxy = httpProxy.createProxyServer( {} ); // See (†)
490 | export async function proxyRequestToHash( req: any, res: any ) {
491 | 	const { commitHash, runEnv } = req.session;
492 | 	let port = await getPortForContainer( commitHash, runEnv );
493 | 
494 | 	if ( ! port ) {
495 | 		l.info( { port, commitHash, runEnv }, `Could not find port for commitHash` );
496 | 		res.send( 'Error setting up port!' );
497 | 		res.end();
498 | 		return;
499 | 	}
500 | 
501 | 	touchCommit( commitHash, runEnv );
502 | 	proxy.web( req, res, { target: `http://localhost:${ port }` }, err => {
503 | 		if ( err && ( err as any ).code === 'ECONNRESET' ) {
504 | 			return;
505 | 		}
506 | 		l.info( { err, req, res, commitHash }, 'unexpected error occured while proxying' );
507 | 	} );
508 | }
509 | 
510 | export function getContainerName( container: ContainerInfo ) {
511 | 	// The first character is a `/`, skip it
512 | 	return container.Names[ 0 ].substring( 1 );
513 | }
514 | 
515 | export function findContainer( {
516 | 	id,
517 | 	image,
518 | 	env,
519 | 	status,
520 | 	name,
521 | 	sanitizedName,
522 | }: ContainerSearchOptions ) {
523 | 	return Array.from( state.containers.values() ).find( container => {
524 | 		if ( image && ( container.Image !== image && container.ImageID !== image ) ) return false;
525 | 		if ( env && container.Labels[ 'calypsoEnvironment' ] !== env ) return false;
526 | 		if ( status && container.Status !== status ) return false;
527 | 		if ( id && container.Id !== id ) return false;
528 | 		// In the Docker internal list, names start with `/`
529 | 		if ( name && ! container.Names.includes( '/' + name ) ) return false;
530 | 
531 | 		// Sanitized name is the URL friendly version of the container's name (`_` got replaced with `-`)
532 | 		if (
533 | 			sanitizedName &&
534 | 			! container.Names.map( name => name.replace( /_/g, '-' ).substr( 1 ) ).includes(
535 | 				sanitizedName
536 | 			)
537 | 		) {
538 | 			return false;
539 | 		}
540 | 
541 | 		return true;
542 | 	} );
543 | }
544 | 
545 | export async function proxyRequestToContainer( req: any, res: any, container: ContainerInfo ) {
546 | 	// In the Docker internal list, names start with `/`
547 | 	const containerName = getContainerName( container );
548 | 
549 | 	if ( ! container.Ports[ 0 ] ) {
550 | 		l.info( { containerName }, `Could not find port for container` );
551 | 		throw new Error( `Could not find port for container ${ containerName }` );
552 | 	}
553 | 	const port = container.Ports[ 0 ].PublicPort;
554 | 
555 | 	let retryCounter = config.proxyRetry;
556 | 	const proxyToContainer = () =>
557 | 		proxy.web( req, res, { target: `http://localhost:${ port }` }, errorHandler );
558 | 
559 | 	const errorHandler = ( err: any ) => {
560 | 		if ( err && err.code === 'ECONNRESET' ) {
561 | 			retryCounter--;
562 | 			if ( retryCounter > 0 ) {
563 | 				l.warn( { err, containerName }, `ECONNRESET error occured while proxying, retrying (${retryCounter} retries left)` );
564 | 				setTimeout( proxyToContainer, 1000 );
565 | 				return
566 | 			} else {
567 | 				l.error( { err, containerName }, 'ECONNRESET error occured while proxying, no more retries left' );
568 | 			}
569 | 		}
570 | 
571 | 		throw new Error( 'unexpected error occured while proxying' );
572 | 	};
573 | 	touchContainer( containerName );
574 | 	proxyToContainer();
575 | }
576 | 
577 | /**
578 |  * Pulls an image. Calls onProgress() when there is an update, resolves the returned promise
579 |  * when the image is pulled
580 |  */
581 | export async function pullImage( imageName: ImageName, onProgress: ( data: any ) => void ):Promise<void> {
582 | 	// Store the stream in memory, so other requets can "join" and listen for the progress
583 | 	if ( ! state.pullingImages.has( imageName ) ) {
584 | 		const stream = docker.pull( imageName, {} ) as Promise< DockerodeStream >;
585 | 		state.pullingImages.set( imageName, stream );
586 | 	}
587 | 
588 | 	const stream = state.pullingImages.get( imageName );
589 | 	return new Promise( async ( resolve, reject ) => {
590 | 		try {
591 | 			docker.modem.followProgress(
592 | 				await stream,
593 | 				( err: any ) => {
594 | 					state.pullingImages.delete( imageName );
595 | 					if ( err ) reject( err );
596 | 					else resolve();
597 | 				},
598 | 				onProgress
599 | 			);
600 | 		} catch( err ) {
601 | 			reject(err);
602 | 		}
603 | 	} );
604 | }
605 | 
606 | /**
607 |  * Asks a container nicely to stop, waits for 10 seconds and then obliterates it
608 |  */
609 | export async function deleteContainer( containerInfo: ContainerInfo ) {
610 | 	const container = docker.getContainer( containerInfo.Id );
611 | 	if ( containerInfo.State === 'running' ) {
612 | 		await container.stop( { t: 10 } );
613 | 	}
614 | 	await container.remove( { force: true } );
615 | 	await refreshContainers();
616 | }
617 | 
618 | /**
619 |  * Creates a container
620 |  *
621 |  * createContainer is async, but we don't keep a list of container being creates to ensure atomicity for a few reasons:
622 |  *
623 |  * - Creating container is quite fast (a few ms), so the chances of collisions are quite low
624 |  * - Even if we get two requests with the same image+env at the same time, creating two separate containers for the same
625 |  *   image is ok. Each one will get a different URL, and if one of them is not used it will get eventually cleaned up.
626 |  */
627 | export async function createContainer( imageName: ImageName, env: RunEnv ) {
628 | 	const exposedPort = `${ config.build.exposedPort }/tcp`;
629 | 
630 | 	let freePort: number;
631 | 	try {
632 | 		freePort = await getPort();
633 | 	} catch ( err ) {
634 | 		throw new ImageError(imageName, `Error while attempting to find a free port: ${err.message}`);
635 | 	}
636 | 
637 | 	try {
638 | 		const container = await docker.createContainer( {
639 | 			...config.build.containerCreateOptions,
640 | 			...envContainerConfig( env ),
641 | 			Image: imageName,
642 | 			ExposedPorts: { [ exposedPort ]: {} },
643 | 			HostConfig: {
644 | 				PortBindings: { [ exposedPort ]: [ { HostPort: freePort.toString() } ] },
645 | 			},
646 | 			Labels: {
647 | 				calypsoEnvironment: env,
648 | 			},
649 | 		} );
650 | 		l.info( { imageName }, `Successfully created container for ${ imageName }` );
651 | 		await refreshContainers();
652 | 
653 | 		// Returns a ContainerInfo for the created container, in order to avoid exposing a real Container object.
654 | 		return findContainer( {
655 | 			id: container.id,
656 | 		} );
657 | 	} catch ( error ) {
658 | 		throw new ImageError(imageName, `Failed creating container: ${error.message}`);
659 | 	}
660 | }
661 | 
662 | /**
663 |  * Starts a container that was dormant (either never started, or stopped)
664 |  */
665 | export async function reviveContainer( containerInfo: ContainerInfo ) {
666 | 	const containerName = getContainerName( containerInfo );
667 | 	const container = docker.getContainer( containerInfo.Id );
668 | 
669 | 	// Try to start the container. Due concurrent requests the container
670 | 	// maybe alredy be starting at this point anyway.
671 | 	try {
672 | 		await container.start();
673 | 		l.info( { containerName }, `Successfully started container` );
674 | 	} catch(error) {
675 | 		if (error.statusCode === 304) {
676 | 			l.info( { containerName }, `Container already started` );
677 | 		} else {
678 | 			throw new ContainerError(containerName, `Failed reviving ${containerInfo.State} container: ${error.message}`);
679 | 		}
680 | 	}
681 | 
682 | 	// Refresh intenerl info about container and return the updated info for this container
683 | 	await refreshContainers();
684 | 	return findContainer( { id: container.id } );
685 | }
686 | 


--------------------------------------------------------------------------------