├── .gitignore ├── README.md ├── ninelet ├── run.sh └── Dockerfile ├── LICENSE ├── Dockerfile ├── Makefile └── run.sh /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *~ 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ninemaster 2 | [Private] Repository for ninecluster 3 | -------------------------------------------------------------------------------- /ninelet/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | start_docker() { 4 | local port="${1}"; shift 5 | 6 | while :; do 7 | docker run \ 8 | --rm \ 9 | --privileged \ 10 | --publish="${port}:22" \ 11 | --volume=/alloc:/alloc \ 12 | --volume=/ninemaster/tmp:/var/tmp \ 13 | --volume=/ninemaster/config:/usr/imos/config \ 14 | local/ninelet "$@" 15 | sleep 1 16 | done 17 | } 18 | 19 | cd "$(dirname $0)" 20 | 21 | date > dummy 22 | docker build --tag=local/ninelet . 23 | 24 | mkdir -p /ninemaster/tmp 25 | chmod ugo=rwxrwxrwt /ninemaster/tmp 26 | mkdir -p /ninemaster/config 27 | 28 | for port in $(seq 2201 2209); do 29 | start_docker "${port}" & 30 | done 31 | 32 | for port in $(seq 2210 2219); do 33 | start_docker "${port}" -d & 34 | done 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Kentaro IMAJO 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:14.04 2 | MAINTAINER imos@docker.com 3 | 4 | # Let's start with some basic stuff. 5 | RUN apt-get update -qq && apt-get install -qqy \ 6 | apt-transport-https \ 7 | ca-certificates \ 8 | curl \ 9 | lxc \ 10 | iptables 11 | 12 | # Install Docker from Docker Inc. repositories. 13 | RUN curl -sSL https://get.docker.com/ubuntu/ | sh 14 | 15 | RUN apt-get update -qq & \ 16 | apt-get install -qqy openssh-server 17 | RUN mkdir /var/run/sshd 18 | 19 | RUN useradd \ 20 | --home-dir=/home/ninemaster \ 21 | --create-home \ 22 | --uid=20601 \ 23 | --user-group \ 24 | --shell=/bin/bash \ 25 | ninemaster 26 | RUN mkdir -p /home/ninemaster/.ssh 27 | RUN echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCrHaL3kdZ2RekCdpkie3fsiv2yVyyWRBOO6Q68Kr+tFStRqtF8q1/UoeteUIOxzwKaAmHoaM9PkItdMBki0BLQDimCZwjjbkosritGDMTJXGd21O72mWaTv+nfq+/ishCdt6gdBYXTejvpPJhq8ZMYhTYJZkWqlGO2CKrWcnHHu1HhnValeqNWS5nh8BULOTMKaixjdzXIkWgm8HyiewvqjZXC3tZlfFDErRpiS7SYfJHd4PujjFCNyiVxZ5yOvEGMXQa1UFxQlfX8H+lAr6qObK50osAdUbvjjbhIhMvZT2higSNNtv/yiaLRnLbbOHomObvqxob5TUVdCkazXX3N imos@docker' > /home/ninemaster/.ssh/authorized_keys 28 | RUN echo 'ninemaster ALL=(ALL:ALL) NOPASSWD: ALL' > /etc/sudoers.d/ninemaster 29 | 30 | # Install Ninelet. 31 | ADD ./ninelet /usr/local/ninelet 32 | 33 | # Install the magic wrapper. 34 | ADD ./run.sh /usr/local/bin/run.sh 35 | RUN chmod +x /usr/local/bin/run.sh 36 | 37 | # Define additional metadata for our image. 38 | VOLUME /var/lib/docker 39 | 40 | CMD ["run.sh"] 41 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SERVICE := ninemaster 2 | DISK ?= 50G 3 | ALLOC_DISK ?= 10G 4 | 5 | start: check 6 | make mount 7 | if ! docker top $(SERVICE) >/dev/null 2>/dev/null; then \ 8 | set -e -x; \ 9 | mkdir -p /ninecluster/alloc; \ 10 | mkdir -p /ninecluster/ninemaster; \ 11 | docker build --tag=local/$(SERVICE) .; \ 12 | docker rm --force $(SERVICE) 2>/dev/null || true; \ 13 | docker run \ 14 | --privileged \ 15 | --name=$(SERVICE) \ 16 | --hostname=$(SERVICE) \ 17 | --volume=/ninecluster/alloc:/alloc \ 18 | --volume=/ninecluster/ninemaster:/ninemaster \ 19 | --publish=2200:22 \ 20 | --publish=2201-2219:2201-2219 \ 21 | local/$(SERVICE); \ 22 | fi 23 | .PHONY: start 24 | 25 | stop: check 26 | -docker kill $(SERVICE) 2>/dev/null 27 | -docker rm --force $(SERVICE) 2>/dev/null 28 | -make unmount 29 | .PHONY: stop 30 | 31 | restart: check 32 | make stop 33 | make start 34 | .PHONY: restart 35 | 36 | mount: check 37 | if ! mountpoint -q /ninecluster/ninemaster; then \ 38 | set -e -x; \ 39 | mkdir -p /ninecluster/ninemaster; \ 40 | if [ ! -f /ninecluster/ninemaster.dmg ]; then \ 41 | dd if=/dev/zero of=/ninecluster/ninemaster.dmg count=1 bs=1M seek=10000; \ 42 | mkfs.ext4 /ninecluster/ninemaster.dmg; \ 43 | fi; \ 44 | e2fsck -y -f /ninecluster/ninemaster.dmg; \ 45 | resize2fs /ninecluster/ninemaster.dmg $(DISK); \ 46 | mount -t auto -o loop /ninecluster/ninemaster.dmg /ninecluster/ninemaster; \ 47 | fi 48 | .PHONY: mount 49 | 50 | unmount: check 51 | -fuser --kill /ninecluster/ninemaster 2>/dev/null 52 | -umount -f /ninecluster/ninemaster 2>/dev/null 53 | .PHONY: unmount 54 | 55 | defrag: check 56 | -e4defrag -c /ninecluster/ninemaster.dmg 57 | -e2fsck -y -f /ninecluster/ninemaster.dmg 58 | -resize2fs -M /ninecluster/ninemaster.dmg 59 | .PHONY: defrag 60 | 61 | backup: check 62 | xz --stdout --compress /ninecluster/ninemaster.dmg > /ninecluster/ninemaster.dmg.xz 63 | .PHONY: backup 64 | 65 | install: 66 | test "$$(whoami)" = 'root' 67 | apt-get update -qq 68 | apt-get install -qqy apt-transport-https ca-certificates curl lxc iptables 69 | curl -sSL https://get.docker.com/ubuntu/ | sh 70 | sed -i 's/^GRUB_CMDLINE_LINUX=".*"/GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1"/' /etc/default/grub 71 | update-grub 72 | .PHONY: install 73 | 74 | check: 75 | test "$$(whoami)" = 'root' 76 | .PHONY: check 77 | -------------------------------------------------------------------------------- /ninelet/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:14.04 2 | MAINTAINER imos@docker.com 3 | 4 | # Install SSH server. 5 | RUN apt-get update -qq && \ 6 | apt-get install -qqy openssh-server 7 | RUN mkdir /var/run/sshd 8 | 9 | # Install useful tools. 10 | RUN apt-get update -qq && \ 11 | apt-get install -qqy \ 12 | build-essential devscripts ubuntu-standard software-properties-common \ 13 | screen lxc traceroute gdb \ 14 | vim git subversion mercurial cmake make \ 15 | dos2unix nkf curl xz-utils graphviz imagemagick 16 | 17 | # Install C++. 18 | RUN apt-get update -qq && apt-get install -qqy clang-3.6 clang-format-3.6 19 | 20 | # Install scripts (python, php, ruby). 21 | RUN apt-get update -qq && \ 22 | apt-get install -qqy \ 23 | python3.4 ruby1.9.3 rubygems-integration php5-cli python-pip 24 | 25 | # Install C#. 26 | RUN apt-get update -qq && apt-get install -qqy mono-devel 27 | 28 | # Install Haskell. 29 | RUN apt-get update -qq && apt-get install -qqy haskell-platform && cabal update 30 | 31 | # Install Go. 32 | RUN apt-get update -qq && apt-get install -qqy golang 33 | 34 | # Install Java. 35 | RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 \ 36 | select true | debconf-set-selections && \ 37 | add-apt-repository -y ppa:webupd8team/java && \ 38 | apt-get update -qq && apt-get install -qqy oracle-java8-installer 39 | 40 | # Install JavaScript. 41 | # RUN wget -O - https://deb.nodesource.com/setup | bash && \ 42 | # apt-get install -qqy nodejs npm 43 | 44 | # Install AWS command. 45 | RUN pip install awscli 46 | 47 | # Install utilities for Bazel. 48 | RUN apt-get update -qq && \ 49 | apt-get install -qqy libarchive-dev pkg-config zip g++ zlib1g-dev 50 | 51 | # Add ninelet user. 52 | RUN useradd \ 53 | --home-dir=/home/ninelet \ 54 | --create-home \ 55 | --uid=20602 \ 56 | --user-group \ 57 | --shell=/bin/bash \ 58 | ninelet 59 | RUN mkdir -p /home/ninelet/.ssh 60 | RUN echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCrHaL3kdZ2RekCdpkie3fsiv2yVyyWRBOO6Q68Kr+tFStRqtF8q1/UoeteUIOxzwKaAmHoaM9PkItdMBki0BLQDimCZwjjbkosritGDMTJXGd21O72mWaTv+nfq+/ishCdt6gdBYXTejvpPJhq8ZMYhTYJZkWqlGO2CKrWcnHHu1HhnValeqNWS5nh8BULOTMKaixjdzXIkWgm8HyiewvqjZXC3tZlfFDErRpiS7SYfJHd4PujjFCNyiVxZ5yOvEGMXQa1UFxQlfX8H+lAr6qObK50osAdUbvjjbhIhMvZT2higSNNtv/yiaLRnLbbOHomObvqxob5TUVdCkazXX3N imos@docker' > /home/ninelet/.ssh/authorized_keys 61 | RUN echo 'ninelet ALL=(ALL:ALL) NOPASSWD: ALL' > /etc/sudoers.d/ninelet 62 | RUN echo 'ninelet:$1$yLREWjDA$pR8cjWbX984KgIzMu71Z4.' | chpasswd --encrypted 63 | RUN echo 'source imos-bashrc' >> /home/ninelet/.bashrc 64 | 65 | # Set language settings. 66 | RUN echo 'LANGUAGE=en_US.UTF8' >> /etc/environment 67 | RUN echo 'LANG=en_US.UTF8' >> /etc/environment 68 | RUN echo 'LC_ALL=C.UTF-8' >> /etc/environment 69 | 70 | # Set a ninecluster role. 71 | RUN echo 'NINECLUSTER=ninelet' >> /etc/environment 72 | 73 | # Set up imos/bin. 74 | RUN grep '/usr/imos/bin:' /etc/environment || \ 75 | sed -i 's%PATH="%PATH="/usr/imos/bin:%' /etc/environment 76 | ENV PATH /usr/imos/bin:$PATH 77 | RUN echo "Defaults secure_path = \"$PATH\"" >/etc/sudoers.d/00-secure_path 78 | RUN mkdir -p /usr/imos/bin 79 | RUN git clone --depth 1 'https://github.com/imos/bin' /usr/imos/bin 80 | 81 | # Clear cache if necessary. 82 | ADD ./dummy /var/tmp/dummy 83 | RUN cd /usr/imos/bin && git pull 84 | 85 | EXPOSE 22 86 | ENTRYPOINT ["/bin/bash", "-c", "/usr/sbin/sshd -D -o 'ClientAliveInterval 5' -o 'ClientAliveCountMax 3'"] 87 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Ensure that all nodes in /dev/mapper correspond to mapped devices currently loaded by the device-mapper kernel driver 4 | dmsetup mknodes 5 | 6 | # First, make sure that cgroups are mounted correctly. 7 | CGROUP=/sys/fs/cgroup 8 | : {LOG:=stdio} 9 | 10 | [ -d $CGROUP ] || 11 | mkdir $CGROUP 12 | 13 | mountpoint -q $CGROUP || 14 | mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP || { 15 | echo "Could not make a tmpfs mount. Did you use --privileged?" 16 | exit 1 17 | } 18 | 19 | if [ -d /sys/kernel/security ] && ! mountpoint -q /sys/kernel/security 20 | then 21 | mount -t securityfs none /sys/kernel/security || { 22 | echo "Could not mount /sys/kernel/security." 23 | echo "AppArmor detection and --privileged mode might break." 24 | } 25 | fi 26 | 27 | # Mount the cgroup hierarchies exactly as they are in the parent system. 28 | for SUBSYS in $(cut -d: -f2 /proc/1/cgroup) 29 | do 30 | [ -d $CGROUP/$SUBSYS ] || mkdir $CGROUP/$SUBSYS 31 | mountpoint -q $CGROUP/$SUBSYS || 32 | mount -n -t cgroup -o $SUBSYS cgroup $CGROUP/$SUBSYS 33 | 34 | # The two following sections address a bug which manifests itself 35 | # by a cryptic "lxc-start: no ns_cgroup option specified" when 36 | # trying to start containers withina container. 37 | # The bug seems to appear when the cgroup hierarchies are not 38 | # mounted on the exact same directories in the host, and in the 39 | # container. 40 | 41 | # Named, control-less cgroups are mounted with "-o name=foo" 42 | # (and appear as such under /proc//cgroup) but are usually 43 | # mounted on a directory named "foo" (without the "name=" prefix). 44 | # Systemd and OpenRC (and possibly others) both create such a 45 | # cgroup. To avoid the aforementioned bug, we symlink "foo" to 46 | # "name=foo". This shouldn't have any adverse effect. 47 | echo $SUBSYS | grep -q ^name= && { 48 | NAME=$(echo $SUBSYS | sed s/^name=//) 49 | ln -s $SUBSYS $CGROUP/$NAME 50 | } 51 | 52 | # Likewise, on at least one system, it has been reported that 53 | # systemd would mount the CPU and CPU accounting controllers 54 | # (respectively "cpu" and "cpuacct") with "-o cpuacct,cpu" 55 | # but on a directory called "cpu,cpuacct" (note the inversion 56 | # in the order of the groups). This tries to work around it. 57 | [ $SUBSYS = cpuacct,cpu ] && ln -s $SUBSYS $CGROUP/cpu,cpuacct 58 | done 59 | 60 | # Note: as I write those lines, the LXC userland tools cannot setup 61 | # a "sub-container" properly if the "devices" cgroup is not in its 62 | # own hierarchy. Let's detect this and issue a warning. 63 | grep -q :devices: /proc/1/cgroup || 64 | echo "WARNING: the 'devices' cgroup should be in its own hierarchy." 65 | grep -qw devices /proc/1/cgroup || 66 | echo "WARNING: it looks like the 'devices' cgroup is not mounted." 67 | 68 | # Now, close extraneous file descriptors. 69 | pushd /proc/self/fd >/dev/null 70 | for FD in * 71 | do 72 | case "$FD" in 73 | # Keep stdin/stdout/stderr 74 | [012]) 75 | ;; 76 | # Nuke everything else 77 | *) 78 | eval exec "$FD>&-" 79 | ;; 80 | esac 81 | done 82 | popd >/dev/null 83 | 84 | 85 | # If a pidfile is still around (for example after a container restart), 86 | # delete it so that docker can start. 87 | rm -rf /var/run/docker.pid 88 | 89 | # If we were given a PORT environment variable, start as a simple daemon; 90 | # otherwise, spawn a shell as well 91 | pid="$$" 92 | docker --daemon --graph=/ninemaster/docker \ 93 | -H unix:///var/run/docker.sock $DOCKER_DAEMON_ARGS & 94 | (( timeout = 10 + SECONDS )) 95 | until docker info >/dev/null 2>&1 96 | do 97 | if (( SECONDS >= timeout )); then 98 | echo 'Timed out trying to connect to internal docker host.' >&2 99 | exit 1 100 | fi 101 | sleep 1 102 | done 103 | bash /usr/local/ninelet/run.sh & 104 | exec /usr/sbin/sshd -D 105 | --------------------------------------------------------------------------------