├── cephbox ├── .gitignore ├── osd.sh ├── ceph.conf.sh ├── bootstrap.sh ├── mon.sh └── Dockerfile ├── README.md └── run.sh /cephbox/.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | docker-ceph 2 | =========== 3 | 4 | ## Example Setup 5 | 6 | ### Create fake volumes on the host 7 | ``` 8 | sudo sh ./run.sh 9 | ``` 10 | 11 | This will give you a Ceph container 12 | 13 | ### Validation 14 | The Container uses host network and persists Ceph configuration and keyrings on host /etc/ceph. 15 | 16 | -------------------------------------------------------------------------------- /cephbox/osd.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Configures and launches a new OSD. 4 | # 5 | 6 | ceph osd create 7 | ceph-osd -i $1 --mkfs --mkkey 8 | ceph auth add osd.$1 osd 'allow *' mon 'allow rwx' -i /var/lib/ceph/osd/ceph-$1/keyring 9 | ceph osd crush add $1 1 root=default host=cephbox 10 | ceph-osd -i $1 -k /var/lib/ceph/osd/ceph-$1/keyring 11 | -------------------------------------------------------------------------------- /cephbox/ceph.conf.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Configures /etc/ceph.conf from a template. 4 | # 5 | 6 | echo " 7 | [global] 8 | auth cluster required = cephx 9 | auth service required = cephx 10 | auth client required = cephx 11 | 12 | [mon.a] 13 | host = cephbox 14 | mon addr = $1 15 | 16 | [osd] 17 | osd journal size = 128 18 | journal dio = false 19 | 20 | [osd.0] 21 | osd host = cephbox 22 | " > /etc/ceph/ceph.conf 23 | 24 | -------------------------------------------------------------------------------- /cephbox/bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Bootstraps a CEPH monitor, and launches the services. 4 | # 5 | 6 | sh ./ceph.conf.sh `hostname -i` 7 | 8 | sh ./mon.sh `hostname -i` 9 | 10 | sh ./osd.sh 0 11 | sh ./osd.sh 1 12 | 13 | ceph osd tree 14 | 15 | 16 | ps -ef |grep ceph 17 | 18 | # create a pool for kube test 19 | ceph osd pool create kube 24 20 | # create an image inside kube pool 21 | rbd create foo --size 10 --pool kube 22 | 23 | ceph -w 24 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docker build -t ceph cephbox 4 | 5 | rm -f /etc/ceph/* 6 | 7 | mkdir -p /home/disks 8 | 9 | for i in 0 1 10 | do 11 | umount /tmp/ceph_disk${i} 12 | dd if=/dev/zero of=/home/disks/d${i} bs=1024M count=6 conv=notrunc 13 | mkfs -t xfs -f /home/disks/d${i} 14 | mkdir -p /tmp/ceph_disk${i} 15 | mount -t xfs -o loop /home/disks/d${i} /tmp/ceph_disk${i} 16 | done 17 | 18 | docker run --privileged --net=host -i -t -v /tmp/ceph_disk0:/var/lib/ceph/osd/ceph-0 -v /tmp/ceph_disk1:/var/lib/ceph/osd/ceph-1 -v /etc/ceph:/etc/ceph -t ceph /bin/bash ./bootstrap.sh 19 | -------------------------------------------------------------------------------- /cephbox/mon.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Configures and launches a new MON. 4 | # 5 | 6 | # monitor setup 7 | monmaptool --create --clobber --fsid `uuidgen` --add a $1:6789 /etc/ceph/monmap 8 | ceph-authtool --create-keyring /var/lib/ceph/mon/keyring --gen-key -n mon. 9 | ceph-authtool /var/lib/ceph/mon/keyring --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' 10 | mkdir /var/lib/ceph/mon/ceph-a 11 | cp /var/lib/ceph/mon/keyring /var/lib/ceph/mon/ceph-a 12 | ceph-mon -i a --mkfs --monmap /etc/ceph/monmap -k /var/lib/ceph/mon/ceph-a/keyring 13 | ceph-mon -i a --monmap /etc/ceph/monmap -k /var/lib/ceph/mon/ceph-a/keyring 14 | 15 | # client setup (handy) 16 | cp /var/lib/ceph/mon/keyring /etc/ceph 17 | 18 | # for this test we want to 19 | ceph osd getcrushmap -o /tmp/crushc 20 | crushtool -d /tmp/crushc -o /tmp/crushd 21 | sed -i 's/step chooseleaf firstn 0 type host/step chooseleaf firstn 0 type osd/' /tmp/crushd 22 | crushtool -c /tmp/crushd -o /tmp/crushc 23 | ceph osd setcrushmap -i /tmp/crushc 24 | -------------------------------------------------------------------------------- /cephbox/Dockerfile: -------------------------------------------------------------------------------- 1 | # CEPH all in one 2 | # 3 | # VERSION 0.0.1 4 | 5 | FROM ubuntu:precise 6 | MAINTAINER Ricardo Rocha, ricardo@catalyst.net.nz 7 | 8 | # Base repositories 9 | RUN echo "deb http://nz.archive.ubuntu.com/ubuntu precise main universe" >> /etc/apt/sources.list 10 | RUN apt-get update 11 | RUN apt-get upgrade -y 12 | 13 | # Workaround for /sbin/init overwrite by docker 14 | # https://github.com/dotcloud/docker/issues/1024 15 | RUN dpkg-divert --local --rename --add /sbin/initctl 16 | RUN ln -fs /bin/true /sbin/initctl 17 | 18 | # Base Packages 19 | RUN apt-get install -y wget sudo net-tools vim openssh-server less iputils-ping 20 | 21 | # Fake fuse (otherwise package install fails) 22 | # https://gist.github.com/henrik-muehe/6155333 23 | RUN apt-get install libfuse2 24 | RUN cd /tmp ; apt-get download fuse 25 | RUN cd /tmp ; dpkg-deb -x fuse_* . 26 | RUN cd /tmp ; dpkg-deb -e fuse_* 27 | RUN cd /tmp ; rm fuse_*.deb 28 | RUN cd /tmp ; echo -en '#!/bin/bash\nexit 0\n' > DEBIAN/postinst 29 | RUN cd /tmp ; dpkg-deb -b . /fuse.deb 30 | RUN cd /tmp ; dpkg -i /fuse.deb 31 | 32 | # Ceph repositories 33 | RUN /usr/bin/wget -q -O- 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc' | sudo apt-key add - 34 | RUN echo "deb http://ceph.com/debian-emperor/ precise main" > /etc/apt/sources.list.d/ceph.list 35 | RUN apt-get update 36 | 37 | # Ceph Packages 38 | RUN apt-get install -y ceph 39 | 40 | # Get ports exposed 41 | EXPOSE 6789 42 | 43 | ADD ./bootstrap.sh /bootstrap.sh 44 | ADD ./mon.sh /mon.sh 45 | ADD ./osd.sh /osd.sh 46 | ADD ./ceph.conf.sh /ceph.conf.sh 47 | 48 | CMD /bootstrap.sh 49 | 50 | # Volumes 51 | # dd if=/dev/zero of=/tmp/ceph0 bs=1024000 count=3000 52 | # mkfs.ext4 /tmp/ceph0 53 | # mkdir /tmp/ceph0d 54 | # sudo mount -o loop /tmp/ceph0 /tmp/ceph0d 55 | # 56 | # Build 57 | # sudo docker build -t ceph . 58 | # 59 | # Run 60 | # sudo docker run -h cephbox -i -v /tmp/ceph0d:/var/lib/ceph/osd/ceph-0 -v /tmp/ceph1d:/var/lib/ceph/osd/ceph-1 -p 6789:6789 -t ceph /bin/bash 61 | # ./bootstrap.sh 62 | # 63 | # NOTES: 64 | # Need to update docker config 65 | # vim /etc/init/docker.conf 66 | # ... 67 | # start... 68 | # stop.... 69 | # limit nofile 65536 65536 70 | --------------------------------------------------------------------------------