├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── Vagrantfile ├── containers ├── base │ ├── Dockerfile │ └── haconiwa │ │ └── rootfs │ │ └── etc │ │ ├── motd │ │ ├── resolv.conf │ │ ├── ssh │ │ └── sshd_config │ │ └── vim │ │ └── vimrc.local ├── nginx │ └── Dockerfile ├── postfix │ ├── Dockerfile │ ├── entry.sh │ └── main.cf └── ssh │ └── Dockerfile ├── experiment ├── convert-to-csv.sh └── save-logs.sh ├── misc ├── multitenancy-fig.png └── overview-fig.png ├── nginx ├── .build_revision ├── Dockerfile ├── build_config.rb ├── builds │ └── .keep └── entry.sh ├── out └── .keep ├── provision ├── containers │ ├── batch.sh │ ├── cleanip │ ├── dstat.service │ ├── dstat_postfix_proc_count.py │ ├── iptables.rules │ ├── nginx │ │ ├── conf.d │ │ │ ├── dispatcher.rb │ │ │ ├── http.conf │ │ │ ├── mail.conf │ │ │ ├── spec.yml │ │ │ └── stream.conf │ │ ├── nginx.conf │ │ └── nginx.service │ ├── script.sh │ └── warp.service ├── dist │ ├── .keep │ └── postfix-workdir.tar.gz ├── hacos │ ├── nginx.haco │ ├── postfix-standalone.haco │ ├── postfix.haco │ ├── ssh.haco │ ├── test.haco │ └── wordpress.haco ├── hosts ├── monolith │ ├── dstat.service │ ├── nginx.conf │ ├── nginx.service │ └── script.sh ├── mxtarpit │ ├── mxtarpit.service │ └── script.sh ├── postfix.sh ├── sandbox-container.sh └── sender │ └── script.sh └── test ├── integration ├── smtp └── ssh /.gitignore: -------------------------------------------------------------------------------- 1 | dist 2 | /*console.log 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Tomohisa Oda 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | default: build_all 2 | 3 | dispatcher: 4 | docker build -t nginx-haconiwa ./nginx 5 | docker run --rm -v $(PWD)/nginx/builds:/builds -t nginx-haconiwa 6 | mv ./nginx/builds/nginx* ./provision/dist/ 7 | 8 | base: 9 | docker build -t haconiwa-container:base ./containers/base 10 | 11 | nginx: 12 | docker build -t haconiwa-container:nginx ./containers/nginx 13 | docker run -d -h nginx --rm --name haconiwa-nginx -t haconiwa-container:nginx 14 | docker export haconiwa-nginx > ./provision/dist/nginx.image.tar 15 | docker stop haconiwa-nginx 16 | docker rmi haconiwa-container:nginx 17 | 18 | ssh: 19 | docker build -t haconiwa-container:ssh ./containers/ssh 20 | docker run -d -h ssh --rm --name haconiwa-ssh -t haconiwa-container:ssh 21 | docker export haconiwa-ssh > ./provision/dist/ssh.image.tar 22 | docker stop haconiwa-ssh 23 | docker rmi haconiwa-container:ssh 24 | 25 | postfix: 26 | docker build -t haconiwa-container:postfix ./containers/postfix 27 | docker run -d -h postfix --rm --name haconiwa-postfix -t haconiwa-container:postfix 28 | docker export haconiwa-postfix > ./provision/dist/postfix.image.tar 29 | docker stop haconiwa-postfix 30 | docker rmi haconiwa-container:postfix 31 | 32 | build: base nginx ssh postfix 33 | build_all: dispatcher build 34 | 35 | .PHONY: dispatcher nginx 36 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | FastContainer: Nginx with Haconiwa 2 | == 3 | 4 | This is an example of [FastContainer][fastcontainer] implementation by [nginx][nginx], [ngx_mruby][ngx_mruby] and [haconiwa][haconiwa]. 5 | 6 | [fastcontainer]: https://speakerdeck.com/matsumoto_r/fastcontainer-at-iot38 7 | [nginx]: https://github.com/nginx/nginx 8 | [ngx_mruby]: https://github.com/matsumotory/ngx_mruby 9 | [haconiwa]: https://github.com/haconiwa/haconiwa 10 | 11 | ![overview](misc/overview-fig.png) 12 | 13 | Requirement 14 | -- 15 | 16 | - docker >= 18.06.0-ce 17 | - vagrant >= 2.1.1 18 | 19 | Usage 20 | -- 21 | 22 | To create an image to use in this implementation, run GNU make and vagrant to run the VM on the Virtualbox. 23 | 24 | ```sh 25 | # Create nginx and container images to `./provision/dist` 26 | $ make 27 | 28 | # Create instance, and provision by `./provision/provisioner.sh` 29 | $ vagrant up 30 | ``` 31 | 32 | When the VM starts up, let's start the container with the request trigger in the following way. 33 | 34 | ### HTTP 35 | 36 | HTTP container starts 37 | 38 | ```sh 39 | $ curl http://127.0.0.1:8080/ 40 | 41 | # HTTP foo container responds 42 | $ curl -H 'Host: foo.test' http://127.0.0.1:8080/ 43 | 44 | # HTTP bar container responds 45 | $ curl -H 'Host: bar.test' http://127.0.0.1:8080/ 46 | ``` 47 | 48 | ### SSH 49 | 50 | SSH container starts 51 | 52 | ```sh 53 | $ ssh root@127.0.0.1 -p 8022 54 | # => password: screencast 55 | ``` 56 | 57 | ### SMTP 58 | 59 | SMTP container starts 60 | 61 | ```sh 62 | $ printf "%s\0%s\0%s" foo foo password | openssl base64 -e 63 | Zm9vAGZvbwBwYXNzd29yZA== 64 | 65 | $ telnet 127.0.0.1 8025 66 | Trying 127.0.0.1... 67 | Connected to localhost. 68 | Escape character is '^]'. 69 | 220 ubuntu-xenial ESMTP ready 70 | HELO local 71 | 250 ubuntu-xenial 72 | AUTH PLAIN Zm9vAGZvbwBwYXNzd29yZA== 73 | 235 2.0.0 OK 74 | # SMTP foo container responds 75 | 76 | $ printf "%s\0%s\0%s" bar bar password | openssl base64 -e 77 | YmFyAGJhcgBwYXNzd29yZA== 78 | 79 | $ telnet 127.0.0.1 8025 80 | Trying 127.0.0.1... 81 | Connected to localhost. 82 | Escape character is '^]'. 83 | 220 ubuntu-xenial ESMTP ready 84 | HELO local 85 | 250 ubuntu-xenial 86 | AUTH PLAIN YmFyAGJhcgBwYXNzd29yZA== 87 | 235 2.0.0 OK 88 | # SMTP bar container responds 89 | ``` 90 | 91 | Multitenancy 92 | -- 93 | 94 | HTTP determines the domain from the request host and SMTP determines the domain from SMTP AUTH. Then, upstream containers are dynamically selected from the discrimination. 95 | 96 | ![multitenancy](misc/multitenancy-fig.png) 97 | 98 | SMTP AUTH uses [ngx_mail_core_module](http://nginx.org/en/docs/mail/ngx_mail_core_module.html). 99 | 100 | Todo 101 | -- 102 | 103 | - [x] Multitenancy 104 | - [ ] Auto Scaling 105 | - [ ] Multi Host 106 | 107 | Author 108 | -- 109 | 110 | - [linyows][linyows] 111 | 112 | [linyows]: https://github.com/linyows 113 | 114 | License 115 | -- 116 | 117 | This project is under the MIT License. 118 | 119 | - nginx: https://github.com/nginx/nginx/blob/master/docs/text/LICENSE 120 | - ngx_mruby: https://github.com/matsumotory/ngx_mruby/blob/master/MITL 121 | - haconiwa: https://github.com/haconiwa/haconiwa/blob/master/LICENSE 122 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | Vagrant.configure('2') do |config| 5 | config.vm.box = 'ubuntu/bionic64' 6 | config.vm.provider 'virtualbox' do |vb| 7 | vb.memory = 512 * 4 8 | vb.cpus = 1 * 2 9 | end 10 | 11 | autostart_bench = !!ENV['BENCH'] 12 | autostart_smtp = !!ENV['SMTP'] 13 | 14 | config.vm.define 'containers', primary: true do |c| 15 | #c.disksize.size = '50GB' 16 | c.vm.provider 'virtualbox' do |vb| 17 | vb.memory = 512 * 6 18 | vb.cpus = 8 19 | end 20 | %w(80 443).each do |port| 21 | c.vm.network 'forwarded_port', guest: port, host: "8#{port.rjust(3, '0')}" 22 | end 23 | %w(22 25).each do |port| 24 | p = "8#{port.rjust(3, '0')}" 25 | c.vm.network 'forwarded_port', guest: p, host: p 26 | end 27 | c.vm.synced_folder './provision', '/data' 28 | c.vm.provision 'shell', path: 'provision/containers/script.sh' 29 | c.vm.hostname = 'containers' 30 | c.vm.network :private_network, ip:'192.168.30.10' 31 | c.vm.network :private_network, ip:'192.168.30.99' 32 | end 33 | 34 | config.vm.define :bench, autostart: autostart_bench do |c| 35 | c.disksize.size = '50GB' 36 | c.vm.provider 'virtualbox' do |vb| 37 | vb.memory = 512 * 2 38 | vb.cpus = 4 39 | end 40 | c.vm.synced_folder './out', '/opt/out' 41 | c.vm.synced_folder './bench', '/opt/bench' 42 | c.vm.provision 'shell', inline: (<<-SHELL) 43 | apt update 44 | apt -y install apache2-utils 45 | SHELL 46 | c.vm.network 'private_network', ip: '192.168.199.20' 47 | end 48 | 49 | config.vm.define 'monolith', autostart: autostart_smtp do |c| 50 | c.vm.synced_folder './provision', '/data' 51 | c.vm.provision 'file', source: './provision/hosts', destination: '/tmp/hosts' 52 | c.vm.provision 'shell', path: 'provision/postfix.sh' 53 | c.vm.provision 'shell', path: 'provision/monolith/script.sh' 54 | c.vm.hostname = 'monolith' 55 | c.vm.network :private_network, ip:'192.168.30.11' 56 | end 57 | 58 | config.vm.define 'sender', autostart: autostart_smtp do |c| 59 | c.vm.provision 'file', source: './provision/hosts', destination: '/tmp/hosts' 60 | c.vm.provision 'shell', path: 'provision/postfix.sh' 61 | c.vm.provision 'shell', path: 'provision/sender/script.sh' 62 | c.vm.hostname = 'sender' 63 | c.vm.network :private_network, ip:'192.168.30.12' 64 | end 65 | 66 | config.vm.define 'recipient', autostart: autostart_smtp do |c| 67 | c.vm.provision 'file', source: './provision/hosts', destination: '/tmp/hosts' 68 | c.vm.provision 'shell', path: 'provision/postfix.sh' 69 | c.vm.hostname = 'recipient' 70 | c.vm.network :private_network, ip:'192.168.30.13' 71 | end 72 | 73 | config.vm.define 'mxtarpit', autostart: autostart_smtp do |c| 74 | c.vm.provision 'file', source: './provision/hosts', destination: '/tmp/hosts' 75 | c.vm.provision 'file', source: './provision/mxtarpit/mxtarpit.service', destination: '/tmp/mxtarpit.service' 76 | c.vm.provision 'shell', path: 'provision/mxtarpit/script.sh' 77 | c.vm.hostname = 'mxtarpit' 78 | c.vm.network :private_network, ip:'192.168.30.14' 79 | end 80 | end 81 | -------------------------------------------------------------------------------- /containers/base/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM buildpack-deps:xenial 2 | 3 | ENV HACONIWA_ROOT /var/lib/haconiwa 4 | 5 | ADD haconiwa $HACONIWA_ROOT 6 | RUN mkdir /usr/local/libexec && \ 7 | mkdir -p $HACONIWA_ROOT/rootfs/var/log/container && \ 8 | chown www-data:www-data $HACONIWA_ROOT/rootfs/var/log/container && \ 9 | chmod 750 $HACONIWA_ROOT/rootfs/var/log/container 10 | 11 | RUN apt -yy update && \ 12 | apt upgrade -yy bash && \ 13 | apt install -yy net-tools bridge-utils iproute2 iputils-ping vim 14 | -------------------------------------------------------------------------------- /containers/base/haconiwa/rootfs/etc/motd: -------------------------------------------------------------------------------- 1 | _ _ 2 | | | _ ____ ____ ___ ____ (_)_ _ _ ____ 3 | | || \ / _ |/ ___) _ \| _ \| | | | |/ _ | 4 | | | | ( ( | ( (__| |_| | | | | | | | ( ( | | 5 | |_| |_|\_||_|\____)___/|_| |_|_|\____|\_||_| 6 | 7 | https://github.com/FastContainer/nginx-haconiwa 8 | -------------------------------------------------------------------------------- /containers/base/haconiwa/rootfs/etc/resolv.conf: -------------------------------------------------------------------------------- 1 | nameserver 10.1.0.1 2 | -------------------------------------------------------------------------------- /containers/base/haconiwa/rootfs/etc/ssh/sshd_config: -------------------------------------------------------------------------------- 1 | AuthorizedKeysCommand /usr/sbin/mc-key-wrapper 2 | AuthorizedKeysCommandUser root 3 | ChallengeResponseAuthentication no 4 | PasswordAuthentication no 5 | PermitRootLogin no 6 | PubkeyAuthentication yes 7 | UsePAM no 8 | Subsystem sftp /usr/lib/openssh/sftp-server 9 | -------------------------------------------------------------------------------- /containers/base/haconiwa/rootfs/etc/vim/vimrc.local: -------------------------------------------------------------------------------- 1 | set encoding=utf-8 2 | set fileencodings=iso-2022-jp,euc-jp,sjis,utf-8 3 | set fileformats=unix,dos,mac 4 | -------------------------------------------------------------------------------- /containers/nginx/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM haconiwa-container:base 2 | 3 | ENV IMAGE_NAME nginx 4 | RUN apt update -yy && \ 5 | apt install -yy nginx 6 | 7 | EXPOSE 80 8 | CMD ["nginx", "-g", "daemon off;"] 9 | -------------------------------------------------------------------------------- /containers/postfix/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM haconiwa-container:base 2 | 3 | ENV IMAGE_NAME postfix 4 | ENV DEBIAN_FRONTEND noninteractive 5 | 6 | RUN mkdir /etc/postfix 7 | ADD main.cf /etc/postfix/main.cf 8 | 9 | RUN apt update -yy && \ 10 | apt-get install -yy postfix rsyslog uuid-dev 11 | 12 | ADD entry.sh /entry.sh 13 | RUN chmod +x /entry.sh 14 | 15 | EXPOSE 25 16 | CMD ["/entry.sh"] 17 | -------------------------------------------------------------------------------- /containers/postfix/entry.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "$maildomain" == "" ]; then 4 | maildomain="example.test" 5 | fi 6 | 7 | postconf -e myhostname="$maildomain" 8 | postconf -F '*/*/chroot = n' 9 | 10 | # allow private network 11 | postconf -e mynetworks='127.0.0.0/8 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8' 12 | # lookup hosts 13 | postconf -e smtp_host_lookup='native' 14 | postconf -e smtp_dns_support_level='disabled' 15 | 16 | if [ "$bench" == "true" ]; then 17 | # for smtp-sink 18 | postconf -e relayhost='127.0.0.1:8025' 19 | smtp-sink -R /root -u root -d sink/%Y%m%d%H/%M. 127.0.0.1:8025 5 & 20 | fi 21 | 22 | if [ "$relayhost" != "" ]; then 23 | # relay 24 | postconf -e relayhost="$relayhost" 25 | fi 26 | 27 | if [ ! -f /var/log/mail.log ]; then 28 | touch /var/log/mail.log 29 | fi 30 | 31 | service rsyslog start 32 | service postfix start 33 | 34 | trap "echo got sig-term to exit; exit 0" TERM 35 | while : 36 | do 37 | sleep 1 38 | done 39 | -------------------------------------------------------------------------------- /containers/postfix/main.cf: -------------------------------------------------------------------------------- 1 | # See /usr/share/postfix/main.cf.dist for a commented, more complete version 2 | 3 | 4 | # Debian specific: Specifying a file name will cause the first 5 | # line of that file to be used as the name. The Debian default 6 | # is /etc/mailname. 7 | #myorigin = /etc/mailname 8 | 9 | smtpd_banner = $myhostname ESMTP $mail_name (Ubuntu) 10 | biff = no 11 | 12 | # appending .domain is the MUA's job. 13 | append_dot_mydomain = no 14 | 15 | # Uncomment the next line to generate "delayed mail" warnings 16 | #delay_warning_time = 4h 17 | 18 | readme_directory = no 19 | 20 | # TLS parameters 21 | smtpd_tls_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 22 | smtpd_tls_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 23 | smtpd_use_tls=yes 24 | smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache 25 | smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache 26 | 27 | # See /usr/share/doc/postfix/TLS_README.gz in the postfix-doc package for 28 | # information on enabling SSL in the smtp client. 29 | 30 | smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination 31 | myhostname = example.test 32 | alias_maps = hash:/etc/aliases 33 | alias_database = hash:/etc/aliases 34 | mydestination = $myhostname, localhost, localhost.localdomain, , localhost 35 | relayhost = 36 | mynetworks = 127.0.0.0/8 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8 37 | mailbox_size_limit = 0 38 | recipient_delimiter = + 39 | inet_interfaces = all 40 | inet_protocols = all 41 | smtp_host_lookup = native 42 | smtp_dns_support_level = disabled 43 | -------------------------------------------------------------------------------- /containers/ssh/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM haconiwa-container:base 2 | 3 | ENV IMAGE_NAME ssh 4 | ENV NOTVISIBLE "in users profile" 5 | 6 | RUN apt update -yy && \ 7 | apt install -yy openssh-server 8 | 9 | RUN mkdir /var/run/sshd 10 | RUN echo 'root:screencast' | chpasswd 11 | RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config 12 | RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd 13 | RUN echo "export VISIBLE=now" >> /etc/profile 14 | 15 | EXPOSE 22 16 | CMD ["/usr/sbin/sshd", "-D"] 17 | -------------------------------------------------------------------------------- /experiment/convert-to-csv.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PREFIX="6th" 4 | PROXY="$PREFIX/proxy" 5 | RELAY="$PREFIX/relay" 6 | 7 | # Proxy 8 | grep container-1 $PROXY/playback.log | awk -v 'OFS= ' '{print $1,$2,$3}' | \ 9 | xargs -IXXX gdate --date="XXX" +"%s" | \ 10 | awk '{print "10, "$1}' > $PREFIX/proxy-sent-to-recipient.csv 11 | 12 | grep container-2 $PROXY/playback.log | awk -v 'OFS= ' '{print $1,$2,$3}' | \ 13 | xargs -IXXX gdate --date="XXX" +"%s" | \ 14 | awk '{print "10, "$1}' > $PREFIX/proxy-sent-to-mxtarpit.csv 15 | 16 | grep delivered $PROXY/recipient.mail.log | awk -v 'OFS= ' '{print $1,$2,$3}' | \ 17 | xargs -IXXX gdate --date="XXX" +"%s" | sort | uniq -c | \ 18 | awk -v 'OFS=,' '{print $1, $2}' > $PREFIX/proxy-received.csv 19 | 20 | # Relay 21 | grep container-1 $RELAY/playback.log | awk -v 'OFS= ' '{print $1,$2,$3}' | \ 22 | xargs -IXXX gdate --date="XXX" +"%s" | \ 23 | awk '{print "10, "$1}' > $PREFIX/relay-sent-to-recipient.csv 24 | 25 | grep container-2 $RELAY/playback.log | awk -v 'OFS= ' '{print $1,$2,$3}' | \ 26 | xargs -IXXX gdate --date="XXX" +"%s" | \ 27 | awk '{print "10, "$1}' > $PREFIX/relay-sent-to-mxtarpit.csv 28 | 29 | grep delivered $RELAY/recipient.mail.log | awk -v 'OFS= ' '{print $1,$2,$3}' | \ 30 | xargs -IXXX gdate --date="XXX" +"%s" | sort | uniq -c | \ 31 | awk -v 'OFS=,' '{print $1, $2}' > $PREFIX/relay-received.csv 32 | -------------------------------------------------------------------------------- /experiment/save-logs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Diff 4 | # - /provision/hacos/postfix.haco 5 | # ENV['RELAY'] 6 | # - /provision/containers/iptables.rules 7 | # WARP DNAT 8 | # - /provision/containers/script.sh 9 | # WARP installation 10 | 11 | # Do 12 | # vagrant up mxtarpit && vagrant up recipient && vagrant up sender && vagrant up monolith && vagrant up containers 13 | # or 14 | # vagrant up mxtarpit && vagrant up recipient && vagrant up sender && vagrant up containers 15 | # vagrant ssh sender -- "playback -c bulk > /vagrant/experiment/playback.log" 16 | 17 | vagrant ssh mxtarpit -- "sudo cp /var/log/syslog /vagrant/experiment/mxtarpit.syslog" 18 | vagrant ssh recipient -- "sudo cp /var/log/syslog /vagrant/experiment/recipient.syslog;\ 19 | sudo cp /var/log/mail.log /vagrant/experiment/recipient.mail.log;\ 20 | sudo cp /var/spool/mail/root /vagrant/experiment/recipient.mail" 21 | vagrant ssh containers -- "sudo cp /var/log/syslog /vagrant/experiment/containers.syslog;\ 22 | sudo cp /var/log/nginx/error.log /vagrant/experiment/containers.nginx.error.log;\ 23 | sudo cp /var/log/nginx/haconiwa.log /vagrant/experiment/containers.nginx.haconiwa.log;\ 24 | sudo cp /var/lib/haconiwa/rootfs/postfix-10-1-100-32/var/log/syslog /vagrant/experiment/container-1.syslog;\ 25 | sudo cp /var/lib/haconiwa/rootfs/postfix-10-1-100-32/var/log/haconiwa.out /vagrant/experiment/container-1.haconiwa.out;\ 26 | sudo cp /var/lib/haconiwa/rootfs/postfix-10-1-100-33/var/log/syslog /vagrant/experiment/container-2.syslog;\ 27 | sudo cp /var/lib/haconiwa/rootfs/postfix-10-1-100-33/var/log/haconiwa.out /vagrant/experiment/container-2.haconiwa.out;\ 28 | sudo cp /var/lib/haconiwa/rootfs/postfix-10-1-100-33/var/log/haconiwa.err /vagrant/experiment/container-2.haconiwa.err;\ 29 | sudo apt install tree; 30 | sudo tree /var/lib/haconiwa/rootfs/postfix-10-1-100-32/var/spool/postfix > /vagrant/experiment/container-1.tree;\ 31 | sudo tree /var/lib/haconiwa/rootfs/postfix-10-1-100-33/var/spool/postfix > /vagrant/experiment/container-2.tree" 32 | 33 | vagrant ssh monolith -- "sudo cp /var/log/syslog /vagrant/experiment/relay.syslog;\ 34 | sudo cp /var/log/mail.log /vagrant/experiment/relay.mail.log;\ 35 | sudo postqueue -p > /vagrant/experiment/relay.postqueue" 36 | -------------------------------------------------------------------------------- /misc/multitenancy-fig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FastContainer/nginx-haconiwa/14c0928f1fc7bcc918c7362abca8cd05443f5cf2/misc/multitenancy-fig.png -------------------------------------------------------------------------------- /misc/overview-fig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FastContainer/nginx-haconiwa/14c0928f1fc7bcc918c7362abca8cd05443f5cf2/misc/overview-fig.png -------------------------------------------------------------------------------- /nginx/.build_revision: -------------------------------------------------------------------------------- 1 | a3242586e23e011c44eb06bf20f79353 2 | -------------------------------------------------------------------------------- /nginx/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM buildpack-deps:xenial 2 | 3 | ENV NGINX_VERSION 1.19.9 4 | ENV OPENSSL_VERSION 1.1.1l 5 | ENV YAML_VERSION 0.2.1 6 | ENV NGX_VTS_REV 46d85558e344dfe2b078ce757fd36c69a1ec2dd3 7 | ENV NGX_MRUBY_REV 76a55dad0528862ec5ad34c76fb4d19c17217358 8 | 9 | RUN apt -y update 10 | RUN apt -y install build-essential bison libpcre3-dev libpcre++-dev debhelper \ 11 | flex gcc make automake autoconf libtool git libreadline6-dev \ 12 | zlib1g-dev libncurses5-dev libssl-dev rake libpam0g-dev \ 13 | autotools-dev cgroup-lite git 14 | RUN git clone --depth=100 https://github.com/vozlt/nginx-module-vts && \ 15 | cd nginx-module-vts && git checkout $NGX_VTS_REV 16 | RUN git clone --depth=100 https://github.com/matsumotory/ngx_mruby && \ 17 | cd ngx_mruby && git checkout $NGX_MRUBY_REV 18 | RUN wget http://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz && \ 19 | tar zxf openssl-$OPENSSL_VERSION.tar.gz 20 | RUN wget http://nginx.org/download/nginx-$NGINX_VERSION.tar.gz &&\ 21 | tar zxf nginx-$NGINX_VERSION.tar.gz 22 | RUN wget https://pyyaml.org/download/libyaml/yaml-$YAML_VERSION.tar.gz && \ 23 | tar zxf yaml-$YAML_VERSION.tar.gz && \ 24 | cd yaml-$YAML_VERSION && ./configure && make && make install 25 | 26 | ADD build_config.rb /ngx_mruby/build_config.rb 27 | ADD entry.sh /entry.sh 28 | RUN chmod +x /entry.sh 29 | 30 | CMD ["/entry.sh"] 31 | -------------------------------------------------------------------------------- /nginx/build_config.rb: -------------------------------------------------------------------------------- 1 | MRuby::Build.new('host') do |conf| 2 | toolchain :gcc 3 | 4 | conf.gembox 'full-core' 5 | 6 | conf.cc do |cc| 7 | cc.flags << '-fPIC' if ENV['BUILD_DYNAMIC_MODULE'] 8 | cc.flags << ENV['NGX_MRUBY_CFLAGS'] if ENV['NGX_MRUBY_CFLAGS'] 9 | end 10 | 11 | # Recommended for ngx_mruby 12 | conf.gem mgem: 'mruby-env' 13 | conf.gem mgem: 'mruby-dir' 14 | conf.gem mgem: 'mruby-digest' 15 | conf.gem mgem: 'mruby-process' 16 | conf.gem mgem: 'mruby-json' 17 | conf.gem mgem: 'mruby-onig-regexp' 18 | conf.gem mgem: 'mruby-redis' 19 | conf.gem mgem: 'mruby-vedis' 20 | conf.gem mgem: 'mruby-userdata' 21 | conf.gem mgem: 'mruby-uname' 22 | conf.gem mgem: 'mruby-httprequest' 23 | conf.gem mgem: 'mruby-mutex' 24 | conf.gem mgem: 'mruby-localmemcache' 25 | conf.gem mgem: 'mruby-base64' 26 | conf.gem mgem: 'mruby-fast-remote-check' 27 | conf.gem github: 'udzura/mruby-clean-spawn' 28 | conf.gem github: 'AndrewBelt/mruby-yaml' 29 | # ngx_mruby extended class 30 | conf.gem './mrbgems/ngx_mruby_mrblib' 31 | conf.gem './mrbgems/rack-based-api' 32 | end 33 | 34 | MRuby::Build.new('test') do |conf| 35 | # load specific toolchain settings 36 | 37 | # Gets set by the VS command prompts. 38 | if ENV['VisualStudioVersion'] || ENV['VSINSTALLDIR'] 39 | toolchain :visualcpp 40 | else 41 | toolchain :gcc 42 | end 43 | 44 | enable_debug 45 | 46 | conf.gem mgem: 'mruby-simplehttp' 47 | conf.gem mgem: 'mruby-httprequest' 48 | conf.gem mgem: 'mruby-uname' 49 | conf.gem mgem: 'mruby-ngx-mruby-ext' 50 | conf.gem mgem: 'mruby-simpletest' 51 | conf.gem mgem: 'mruby-http' 52 | conf.gem mgem: 'mruby-json' 53 | conf.gem mgem: 'mruby-env' 54 | 55 | # include the default GEMs 56 | conf.gembox 'full-core' 57 | end 58 | -------------------------------------------------------------------------------- /nginx/builds/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FastContainer/nginx-haconiwa/14c0928f1fc7bcc918c7362abca8cd05443f5cf2/nginx/builds/.keep -------------------------------------------------------------------------------- /nginx/entry.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xe 2 | 3 | prefix=/usr/local/nginx-$NGINX_VERSION 4 | cd /ngx_mruby 5 | 6 | ./configure \ 7 | --with-ngx-src-root=/nginx-$NGINX_VERSION \ 8 | --with-ngx-config-opt="--with-http_v2_module \ 9 | --with-http_ssl_module \ 10 | --with-stream --with-debug \ 11 | --with-mail --with-mail_ssl_module \ 12 | --add-module=/nginx-module-vts \ 13 | --with-http_stub_status_module \ 14 | --without-stream_access_module \ 15 | --prefix=$prefix" \ 16 | --with-openssl-src=/openssl-$OPENSSL_VERSION 17 | 18 | make && make install 19 | 20 | cd /usr/local 21 | tar -zcvf nginx-$NGINX_VERSION.tar.gz nginx-$NGINX_VERSION 22 | mv nginx-$NGINX_VERSION.tar.gz /builds 23 | -------------------------------------------------------------------------------- /out/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FastContainer/nginx-haconiwa/14c0928f1fc7bcc918c7362abca8cd05443f5cf2/out/.keep -------------------------------------------------------------------------------- /provision/containers/batch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | 3 | sysctl -w net.core.somaxconn=4096 4 | ulimit -n 100000 5 | 6 | # expand postfix image 7 | postfix_rootfs_path=/var/lib/haconiwa/rootfs/shared/postfix 8 | postfix_image_path=/var/lib/haconiwa/images/postfix.image.tar 9 | postfix_workdir_path=/var/lib/haconiwa/images/postfix-workdir.tar.gz 10 | test -d ${postfix_rootfs_path} || mkdir -m 755 -p ${postfix_rootfs_path} 11 | test -n "`ls ${postfix_rootfs_path}`" || tar xfp ${postfix_image_path} -C ${postfix_rootfs_path} 12 | for i in `seq 2 201`; do 13 | path=/var/lib/haconiwa/rootfs/postfix-10-1-1-${i} 14 | if [ ! -d $path ]; then 15 | mkdir -m 755 -p ${path} && tar xzfp ${postfix_workdir_path} -C ${path} 16 | fi 17 | sleep 1 18 | done 19 | for i in `seq 2 201`; do 20 | path=/var/lib/haconiwa/rootfs/postfix-10-1-2-${i} 21 | if [ ! -d $path ]; then 22 | mkdir -m 755 -p ${path} && tar xzfp ${postfix_workdir_path} -C ${path} 23 | fi 24 | sleep 1 25 | done 26 | for i in `seq 2 201`; do 27 | path=/var/lib/haconiwa/rootfs/postfix-10-1-3-${i} 28 | if [ ! -d $path ]; then 29 | mkdir -m 755 -p ${path} && tar xzfp ${postfix_workdir_path} -C ${path} 30 | fi 31 | sleep 1 32 | done 33 | for i in `seq 2 201`; do 34 | path=/var/lib/haconiwa/rootfs/postfix-10-1-4-${i} 35 | if [ ! -d $path ]; then 36 | mkdir -m 755 -p ${path} && tar xzfp ${postfix_workdir_path} -C ${path} 37 | fi 38 | sleep 1 39 | done 40 | for i in `seq 2 201`; do 41 | path=/var/lib/haconiwa/rootfs/postfix-10-1-5-${i} 42 | if [ ! -d $path ]; then 43 | mkdir -m 755 -p ${path} && tar xzfp ${postfix_workdir_path} -C ${path} 44 | fi 45 | sleep 1 46 | done 47 | 48 | # start postfix 49 | for i in `seq 2 201`; do 50 | env IP=10.1.1.${i} \ 51 | ID=postfix-10-1-1-${i} \ 52 | DOMAIN=container1-${i}.test \ 53 | PORT=25 \ 54 | PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ 55 | /usr/bin/haconiwa start /var/lib/haconiwa/hacos/postfix-standalone.haco 56 | done 57 | 58 | for i in `seq 2 201`; do 59 | env IP=10.1.2.${i} \ 60 | ID=postfix-10-1-2-${i} \ 61 | DOMAIN=container2-${i}.test \ 62 | PORT=25 \ 63 | PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ 64 | /usr/bin/haconiwa start /var/lib/haconiwa/hacos/postfix-standalone.haco 65 | done 66 | 67 | for i in `seq 2 201`; do 68 | env IP=10.1.3.${i} \ 69 | ID=postfix-10-1-3-${i} \ 70 | DOMAIN=container3-${i}.test \ 71 | PORT=25 \ 72 | PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ 73 | /usr/bin/haconiwa start /var/lib/haconiwa/hacos/postfix-standalone.haco 74 | done 75 | 76 | for i in `seq 2 201`; do 77 | env IP=10.1.4.${i} \ 78 | ID=postfix-10-1-4-${i} \ 79 | DOMAIN=container4-${i}.test \ 80 | PORT=25 \ 81 | PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ 82 | /usr/bin/haconiwa start /var/lib/haconiwa/hacos/postfix-standalone.haco 83 | done 84 | -------------------------------------------------------------------------------- /provision/containers/cleanip: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | CLEANIF=enp0s9 4 | CLEANIP=$(ifconfig $CLEANIF | grep inet | grep -v inet6 | awk '{print $2}') 5 | 6 | CMD=$1 7 | PRIVATEIP=$2 8 | 9 | usage() { 10 | echo $"Usage: $0 {add|del} [PRIVATE IP]" 11 | exit 2 12 | } 13 | 14 | if [ -z "$CMD" ] || [ -z "$PRIVATEIP" ]; then 15 | usage 16 | fi 17 | 18 | case "$CMD" in 19 | "add") 20 | /sbin/iptables -t nat -I POSTROUTING -o $CLEANIF -s $PRIVATEIP ! -d $PRIVATEIP -j MASQUERADE && exit 0 21 | ;; 22 | "del") 23 | number=/sbin/iptables -t nat -L --line-numbers | grep $PRIVATEIP | awk '{print $1}' 24 | test -z $number && exit 1 25 | /sbin/iptables -t nat -D POSTROUTING $number && exit 0 26 | ;; 27 | *) 28 | usage 29 | esac 30 | -------------------------------------------------------------------------------- /provision/containers/dstat.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Dstat 3 | Requires=network-online.target 4 | After=network-online.target 5 | 6 | [Service] 7 | ExecStart=/usr/bin/dstat -tcdnymp --postfix-proc-count --output /data/dstat.container.csv 8 | KillSignal=SIGINT 9 | Restart=on-failure 10 | 11 | [Install] 12 | WantedBy=multi-user.target 13 | -------------------------------------------------------------------------------- /provision/containers/dstat_postfix_proc_count.py: -------------------------------------------------------------------------------- 1 | 2 | class dstat_plugin(dstat): 3 | """ 4 | Total Number of postfix processes on this system. 5 | """ 6 | def __init__(self): 7 | self.name = 'pprocs' 8 | self.vars = ('total',) 9 | self.type = 'd' 10 | self.width = 4 11 | self.scale = 10 12 | 13 | def extract(self): 14 | import commands 15 | cmd = '/bin/ps ax | /bin/grep "[/]usr/lib/postfix/sbin/master" | /usr/bin/wc -l' 16 | self.val['total'] = int(commands.getoutput(cmd)) 17 | -------------------------------------------------------------------------------- /provision/containers/iptables.rules: -------------------------------------------------------------------------------- 1 | *nat 2 | :PREROUTING ACCEPT [1:100] 3 | :INPUT ACCEPT [1:100] 4 | :OUTPUT ACCEPT [0:0] 5 | :POSTROUTING ACCEPT [0:0] 6 | :HACONIWA - [0:0] 7 | 8 | # WARP 9 | -A PREROUTING -s 10.1.0.0/16 ! -d 10.0.2.0/24 -p tcp --dport 25 -j DNAT --to-destination 192.168.30.10:10025 10 | 11 | -A PREROUTING -m addrtype --dst-type LOCAL -j HACONIWA 12 | -A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j HACONIWA 13 | -A POSTROUTING -s 10.1.0.0/16 ! -d 10.1.0.0/16 -j MASQUERADE 14 | COMMIT 15 | 16 | *filter 17 | :INPUT ACCEPT [133:10308] 18 | :FORWARD ACCEPT [0:0] 19 | :OUTPUT ACCEPT [72:9600] 20 | -A FORWARD -o haconiwa0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 21 | -A FORWARD -i haconiwa0 ! -o haconiwa0 -j ACCEPT 22 | -A FORWARD -i haconiwa0 -o haconiwa0 -j ACCEPT 23 | COMMIT 24 | -------------------------------------------------------------------------------- /provision/containers/nginx/conf.d/dispatcher.rb: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # frozen_string_literal: true 3 | 4 | HACONIWA_BIN_PATH = '/usr/bin/haconiwa' 5 | IMAGES_ROOT = '/tmp/criu/images' 6 | CleanSpawn.cgroup_root_path = '/sys/fs/cgroup/systemd' 7 | 8 | module Container 9 | class << self 10 | def dispatch_http 11 | containers = conf['containers']['http'] 12 | req = Nginx::Request.new 13 | haco, cip = if containers.include?(req.hostname) 14 | [containers[req.hostname]['haco'], containers[req.hostname]['ip']] 15 | else 16 | [containers['default']['haco'], containers['default']['ip']] 17 | end 18 | dispatch(haco, cip, 80, [], req.hostname) 19 | end 20 | 21 | def dispatch_ssh 22 | containers = conf['containers']['ssh'] 23 | haco = containers['haco'] 24 | cip = containers['ip'] 25 | cport = 22 26 | c = Nginx::Stream::Connection.new 'dynamic_server' 27 | c.upstream_server = "#{cip}:#{cport}" 28 | dispatch(haco, cip, cport) 29 | end 30 | 31 | def dispatch_smtp_no_auth(name) 32 | containers = conf['containers']['smtp'] 33 | haco = containers[name]['haco'] 34 | cip = containers[name]['ip'] 35 | cport = 25 36 | c = Nginx::Stream::Connection.new 'dynamic_server' 37 | c.upstream_server = "#{cip}:#{cport}" 38 | dispatch(haco, cip, cport, ["DOMAIN=#{name}"]) 39 | end 40 | 41 | def dispatch_smtp_no_auth_no_conf(number) 42 | haco = 'postfix' 43 | cport = 25 44 | cip = if number < 200 45 | "10.1.1.#{number}" 46 | elsif number < 400 47 | "10.1.2.#{number - 200}" 48 | elsif number < 600 49 | "10.1.3.#{number - 400}" 50 | elsif number < 800 51 | "10.1.4.#{number - 600}" 52 | else 53 | "10.1.5.#{number - 800}" 54 | end 55 | c = Nginx::Stream::Connection.new 'dynamic_server' 56 | c.upstream_server = "#{cip}:#{cport}" 57 | 58 | raise "Not enough container info -- haco: #{haco}, ip: #{cip} port: #{cport}" \ 59 | if haco.nil? || cip.nil? || cport.nil? 60 | 61 | env = ["DOMAIN=container-#{number}.test", "SHARED=true"] 62 | d = Dispatcher.new(cip, cport, haco, env, '') 63 | d.rootfs_path = '/var/lib/haconiwa/rootfs/shared/postfix' 64 | return d.run 65 | end 66 | 67 | def dispatch_smtp_after_smtp_auth 68 | containers = conf['containers']['smtp'] 69 | req = Nginx::Request.new 70 | 71 | user = req.headers_in['Auth-User'] 72 | prot = req.headers_in['Auth-Protocol'] 73 | cip = containers[user]['ip'] 74 | haco = containers[user]['haco'] 75 | cport = 25 76 | result = "#{cip}:#{cport}" 77 | 78 | req.headers_out['Auth-Status'] = -> do 79 | unless containers.keys.include? user 80 | debug("SMTP AUTH failed: unknown #{user}") 81 | return 'invalid user' 82 | end 83 | 84 | req.headers_out['Auth-Server'] = cip 85 | req.headers_out['Auth-Port'] = "#{cport}" 86 | dispatch(haco, cip, cport) 87 | 88 | debug("SMTP AUTH success: #{user} to #{result}") 89 | return 'OK' 90 | end.call 91 | 92 | return result 93 | end 94 | 95 | def dispatch(haco = nil, ip = nil, port = nil, env = [], hostname = '') 96 | raise "Not enough container info -- haco: #{haco}, ip: #{ip} port: #{port}" \ 97 | if haco.nil? || ip.nil? || port.nil? 98 | 99 | if ! File.exist?(IMAGES_ROOT + "/apache/core-1.img") 100 | return Dispatcher.new(ip, port, haco, env, hostname).run 101 | else 102 | return Dispatcher.new(ip, port, haco, env, hostname).restore 103 | end 104 | rescue => e 105 | err(e.message) 106 | return '' 107 | end 108 | 109 | def debug(m) 110 | Nginx.errlogger Nginx::LOG_DEBUG, "#{self.name} -- #{m}" 111 | rescue 112 | Nginx::Stream.log Nginx::Stream::LOG_DEBUG, "#{self.name} -- #{m}" 113 | end 114 | 115 | def err(m) 116 | Nginx.errlogger Nginx::LOG_ERR, "#{self.name} -- #{m}" 117 | rescue 118 | Nginx::Stream.log Nginx::Stream::LOG_ERR, "#{self.name} -- #{m}" 119 | end 120 | 121 | def conf 122 | @@_conf ||= load_conf 123 | end 124 | 125 | def load_conf 126 | path = '/etc/nginx/conf.d/spec.yml' 127 | io = File.open(path, 'r') 128 | Container.debug("Loaded conf: #{path}") 129 | YAML.load(io.read) 130 | end 131 | end 132 | 133 | class Dispatcher 134 | def initialize(ip, port, haco, env = [], hostname = '') 135 | @ip = ip 136 | @port = port 137 | @haco = haco 138 | 139 | @root = '/var/lib/haconiwa' 140 | @id = "#{@haco}-#{@ip.gsub('.', '-')}" 141 | @environment = ["IP=#{@ip}", "PORT=#{@port}", "ID=#{@id}", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"] 142 | @environment.concat(env) if env.length > 0 143 | @hostname = hostname 144 | end 145 | 146 | def run 147 | result = "#{@ip}:#{@port}" 148 | if listen? 149 | Container.debug('Already container launched!') 150 | return result 151 | end 152 | 153 | if booting? 154 | wait_for_listen("/var/lock/.#{@id}.hacolock") 155 | return result 156 | end 157 | 158 | booting! 159 | 160 | Container.debug('Launching a container...') 161 | setup_rootfs 162 | start_haconiwa 163 | wait_for_listen("/var/lock/.#{@id}.hacolock") 164 | 165 | not_booting! 166 | 167 | Container.debug("Return ip: #{@ip} port: #{@port}") 168 | return result 169 | end 170 | 171 | def restore 172 | result = "#{@ip}:#{@port}" 173 | if listen? 174 | Container.debug('Already container launched!') 175 | return result 176 | end 177 | 178 | Container.debug('Restoreing a container...') 179 | 180 | restore_haconiwa 181 | wait_for_listen("/var/lock/.#{@id}.hacolock") 182 | 183 | Container.debug("Return ip: #{@ip} port: #{@port}") 184 | result 185 | end 186 | 187 | def rootfs_path 188 | @rootfs_path ||= "#{@root}/rootfs/#{@id}" 189 | end 190 | 191 | def rootfs_path=(path) 192 | @rootfs_path = path 193 | end 194 | 195 | def image_path 196 | "#{@root}/images/#{@haco}.image.tar" 197 | end 198 | 199 | def setup_rootfs 200 | return if File.exist?(rootfs_path) 201 | system "/bin/mkdir -m 755 -p #{rootfs_path}" 202 | system "/bin/tar xfp #{image_path} -C #{rootfs_path}" 203 | setup_hosts(rootfs_path) 204 | setup_welcome_html(rootfs_path) if @haco == 'nginx' 205 | end 206 | 207 | def setup_hosts(root) 208 | cmd = [ 209 | "/bin/echo \"127.0.0.1 localhost #{@id}\" >> #{root}/etc/hosts", 210 | "/bin/cat /data/hosts >> #{root}/etc/hosts" 211 | ].join(' && ') 212 | Container.debug(cmd) 213 | system cmd 214 | end 215 | 216 | def setup_welcome_html(root) 217 | html = "#{root}/var/www/html/index.nginx-debian.html" 218 | cmd = ['/bin/sed', '-i', "'s/Welcome to nginx!/Welcome to #{@hostname}/g'", html].join(' ') 219 | Container.debug(cmd) 220 | system cmd 221 | end 222 | 223 | def env 224 | @environment.unshift('/usr/bin/env').join(' ') 225 | end 226 | 227 | def command 228 | [env, HACONIWA_BIN_PATH, 'start', "#{@root}/hacos/#{@haco}.haco"].join(' ') 229 | end 230 | 231 | def restore_command 232 | [env, HACONIWA_BIN_PATH, 'restore', "#{@root}/hacos/#{@haco}.haco"].join(' ') 233 | end 234 | 235 | def start_haconiwa 236 | shell = ['/bin/bash', '-c', "#{command} >> /var/log/nginx/haconiwa.log 2>&1"] 237 | Container.debug(shell.join(' ')) 238 | clean_spawn(*shell) 239 | end 240 | 241 | def restore_haconiwa 242 | shell = ['/bin/bash', '-c', "#{restore_command} >> /var/log/nginx/haconiwa.log 2>&1"] 243 | Container.debug(shell.join(' ')) 244 | clean_spawn(*shell) 245 | end 246 | 247 | def wait_for_listen(lockfile, max = 1000000) 248 | while true 249 | listen = listen? 250 | file = File.exist?(lockfile) 251 | 252 | return if listen && file 253 | Container.debug("Stil no listen: #{@ip}:#{@port}") unless listen 254 | Container.debug("Stil no lockfile: #{lockfile}'") unless file 255 | 256 | usleep 10 * 1000 257 | max -= 1 258 | raise 'It take too long time to begin listening, timeout' if max <= 0 259 | end 260 | end 261 | 262 | def listen? 263 | if ret = ::FastRemoteCheck.new('127.0.0.1', 0, @ip, @port, 3).connectable? 264 | Container.debug("listen!") 265 | else 266 | Container.debug("not listen") 267 | end 268 | ret 269 | rescue => e 270 | Container.debug("FastRemoteCheck error: #{e.message} retry") 271 | false 272 | end 273 | 274 | def booting_flag_path 275 | "#{@root}/rootfs/#{@id}/.booting" 276 | end 277 | 278 | def booting! 279 | File.open(booting_flag_path, 'w') {|f| 280 | f.flock(File::LOCK_EX) 281 | } 282 | end 283 | 284 | def not_booting! 285 | ::File.delete(booting_flag_path) 286 | end 287 | 288 | def booting? 289 | ::File.exists?(booting_flag_path) 290 | end 291 | end 292 | end 293 | 294 | def nginx_local_port 295 | Nginx::Stream::Connection.local_port 296 | rescue 297 | req = Nginx::Request.new 298 | req.var.server_port.to_i 299 | end 300 | 301 | lambda do 302 | return Container.dispatch_smtp_no_auth_no_conf(nginx_local_port - 60000) \ 303 | if nginx_local_port > 60000 304 | 305 | return case nginx_local_port 306 | when 58080 then Container.dispatch_smtp_after_smtp_auth 307 | when 58025 then Container.dispatch_smtp_no_auth('container-1.test') 308 | when 58026 then Container.dispatch_smtp_no_auth('container-2.test') 309 | when 58027 then Container.dispatch_smtp_no_auth('container-3.test') 310 | when 58028 then Container.dispatch_smtp_no_auth('container-4.test') 311 | when 80 then Container.dispatch_http 312 | when 8022 then Container.dispatch_ssh 313 | end 314 | end.call 315 | -------------------------------------------------------------------------------- /provision/containers/nginx/conf.d/http.conf: -------------------------------------------------------------------------------- 1 | limit_conn_zone $backend zone=limit_proxy_host:100m; 2 | limit_conn limit_proxy_host 40; 3 | limit_conn_status 503; 4 | 5 | server { 6 | listen 80; 7 | listen 19998; 8 | server_name localhost; 9 | proxy_set_header Host $http_host; 10 | proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; 11 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 12 | proxy_buffers 30 32k; 13 | proxy_http_version 1.1; 14 | proxy_set_header Connection ""; 15 | proxy_pass_header Accept; 16 | proxy_pass_header Server; 17 | 18 | proxy_read_timeout 60; 19 | proxy_connect_timeout 60; 20 | proxy_send_timeout 60; 21 | proxy_redirect off; 22 | 23 | location /status { 24 | vhost_traffic_status_display; 25 | vhost_traffic_status_display_format json; 26 | access_log off; 27 | allow 127.0.0.1; 28 | deny all; 29 | } 30 | 31 | location /nginx_status { 32 | stub_status on; 33 | access_log off; 34 | allow 127.0.0.1; 35 | deny all; 36 | } 37 | 38 | location / { 39 | mruby_set $backend /etc/nginx/conf.d/dispatcher.rb cache; 40 | if ($backend = "") { 41 | return 500; 42 | } 43 | proxy_pass http://$backend; 44 | } 45 | } 46 | 47 | server { 48 | listen 443 ssl http2; 49 | server_name fastcontainer.local 50 | add_header Front-End-Https on; 51 | server_tokens off; 52 | server_name_in_redirect off; 53 | 54 | client_max_body_size 100M; 55 | keepalive_timeout 5; 56 | 57 | ssl_certificate /etc/nginx/tls.crt; 58 | ssl_certificate_key /etc/nginx/tls.key; 59 | ssl_protocols TLSv1 TLSv1.1 TLSv1.2; 60 | ssl_ciphers AESGCM:HIGH:!aNULL:!MD5:!LOW:!SSLv2:!EXP:!eNULL; 61 | ssl_prefer_server_ciphers on; 62 | 63 | proxy_set_header X-Real-IP $remote_addr; 64 | proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; 65 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 66 | proxy_set_header Host $http_host; 67 | proxy_set_header X-Forwarded-Ssl on; 68 | proxy_pass_header Accept; 69 | proxy_pass_header Server; 70 | proxy_set_header Authorization $http_authorization; 71 | 72 | proxy_read_timeout 60; 73 | proxy_connect_timeout 60; 74 | proxy_send_timeout 60; 75 | proxy_redirect off; 76 | 77 | location / { 78 | mruby_set $backend /etc/nginx/conf.d/dispatcher.rb cache; 79 | if ($backend = "") { 80 | return 500; 81 | } 82 | proxy_pass https://$backend; 83 | } 84 | } 85 | 86 | server { 87 | listen 127.0.0.1:58080; 88 | server_name fastcontainer.local; 89 | 90 | location /smtp_auth { 91 | mruby_set $backend /etc/nginx/conf.d/dispatcher.rb cache; 92 | if ($backend = "") { 93 | return 500; 94 | } 95 | proxy_pass https://$backend; 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /provision/containers/nginx/conf.d/mail.conf: -------------------------------------------------------------------------------- 1 | proxy on; 2 | proxy_pass_error_message on; 3 | 4 | auth_http 127.0.0.1:58080/smtp_auth; 5 | smtp_capabilities PIPELINING 8BITMIME "SIZE 20480000"; 6 | imap_capabilities IMAP4rev1 LITERAL+ SASL-IR LOGIN-REFERRALS ID ENABLE IDLE AUTH=LOGIN; 7 | smtp_auth login plain; 8 | 9 | server { 10 | listen 8025; 11 | protocol smtp; 12 | xclient off; 13 | } 14 | -------------------------------------------------------------------------------- /provision/containers/nginx/conf.d/spec.yml: -------------------------------------------------------------------------------- 1 | containers: 2 | http: 3 | default: 4 | haco: 'nginx' 5 | ip: '10.1.100.10' 6 | foo.test: 7 | haco: 'nginx' 8 | ip: '10.1.100.11' 9 | bar.test: 10 | haco: 'nginx' 11 | ip: '10.1.100.12' 12 | wordpress.test: 13 | haco: 'wordpress' 14 | ip: '10.1.100.13' 15 | ssh: 16 | haco: 'ssh' 17 | ip: '10.1.100.20' 18 | smtp: 19 | foo: 20 | haco: 'postfix' 21 | ip: '10.1.100.30' 22 | bar: 23 | haco: 'postfix' 24 | ip: '10.1.100.31' 25 | container-1.test: 26 | haco: 'postfix' 27 | ip: '10.1.100.32' 28 | container-2.test: 29 | haco: 'postfix' 30 | ip: '10.1.100.33' 31 | container-3.test: 32 | haco: 'postfix' 33 | ip: '10.1.100.34' 34 | container-4.test: 35 | haco: 'postfix' 36 | ip: '10.1.100.35' 37 | -------------------------------------------------------------------------------- /provision/containers/nginx/conf.d/stream.conf: -------------------------------------------------------------------------------- 1 | upstream dynamic_server { 2 | server 127.0.0.1:8080; 3 | } 4 | 5 | server { 6 | mruby_stream_server_context_code ' 7 | %w(8022 58025 58026 58027 58028).each { |p| Nginx::Stream.add_listener({ address: "#{p}" }) } 8 | 1000.times { |i| Nginx::Stream.add_listener({ address: "#{60001 + i}" }) } 9 | '; 10 | mruby_stream /etc/nginx/conf.d/dispatcher.rb; 11 | proxy_pass dynamic_server; 12 | } 13 | -------------------------------------------------------------------------------- /provision/containers/nginx/nginx.conf: -------------------------------------------------------------------------------- 1 | worker_processes auto; 2 | user root; 3 | error_log logs/error.log debug; 4 | pid /run/nginx.pid; 5 | 6 | worker_rlimit_nofile 100000; 7 | 8 | events { 9 | worker_connections 4096; 10 | multi_accept on; 11 | } 12 | 13 | http { 14 | include mime.types; 15 | default_type application/octet-stream; 16 | sendfile on; 17 | keepalive_timeout 65; 18 | client_max_body_size 20m; 19 | lingering_close on; 20 | lingering_time 30s; 21 | lingering_timeout 15s; 22 | vhost_traffic_status_zone shared:vhost_traffic_status:50m; 23 | include /etc/nginx/conf.d/http.conf; 24 | } 25 | 26 | stream { 27 | include /etc/nginx/conf.d/stream.conf; 28 | } 29 | 30 | mail { 31 | include /etc/nginx/conf.d/mail.conf; 32 | } 33 | -------------------------------------------------------------------------------- /provision/containers/nginx/nginx.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=The NGINX HTTP and reverse proxy server 3 | After=syslog.target network.target remote-fs.target nss-lookup.target 4 | 5 | [Service] 6 | Type=forking 7 | PIDFile=/run/nginx.pid 8 | ExecStartPre=/usr/sbin/nginx -t 9 | ExecStart=/usr/sbin/nginx 10 | ExecReload=/bin/kill -s HUP $MAINPID 11 | ExecStop=/bin/kill -s QUIT $MAINPID 12 | PrivateTmp=false 13 | LimitNOFILE=30000 14 | 15 | [Install] 16 | WantedBy=multi-user.target 17 | -------------------------------------------------------------------------------- /provision/containers/script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | export DEBIAN_FRONTEND noninteractive 4 | 5 | nginx_ver=1.19.9 6 | common_name=fastcontainer.example 7 | images=("nginx" "ssh" "postfix") 8 | 9 | grep 192.168.30 /etc/hosts >/dev/null || cat /data/hosts >> /etc/hosts 10 | 11 | apt update -y 12 | apt upgrade -y 13 | apt install -y bridge-utils openssl curl 14 | locale-gen ja_JP.UTF-8 15 | 16 | # install haconiwa 17 | type haconiwa >/dev/null 2>&1 || \ 18 | curl -s https://packagecloud.io/install/repositories/udzura/haconiwa/script.deb.sh | bash 19 | 20 | apt-get install -y criu 21 | haconiwa_ver=0.10.4 22 | wget https://github.com/haconiwa/haconiwa/releases/download/v${haconiwa_ver}/haconiwa-v${haconiwa_ver}.x86_64-pc-linux-gnu.tgz 23 | tar xzf haconiwa-v${haconiwa_ver}.x86_64-pc-linux-gnu.tgz 24 | rm -rf /usr/bin/haco* 25 | install hacorb hacoirb haconiwa /usr/bin 26 | 27 | # deploy hacofile 28 | test -d /var/log/haconiwa || mkdir -p /var/log/haconiwa 29 | test -d /var/lib/haconiwa/rootfs || mkdir -p /var/lib/haconiwa/rootfs 30 | rm -rf /var/lib/haconiwa/hacos && ln -s /data/hacos /var/lib/haconiwa/hacos 31 | rm -rf /var/lib/haconiwa/images && ln -s /data/dist /var/lib/haconiwa/images 32 | 33 | # setup network 34 | brctl show haconiwa0 2>&1 | grep -i "no such device" && \ 35 | haconiwa init --bridge --bridge-ip=10.1.0.1/16 36 | test $(/sbin/sysctl net.ipv4.ip_forward | awk '{print $3}') -eq 0 && \ 37 | /sbin/sysctl -w net.ipv4.ip_forward=1 38 | /sbin/iptables-restore < /data/containers/iptables.rules 39 | 40 | # install nginx 41 | cp /data/dist/nginx-${nginx_ver}.tar.gz /usr/local/src/nginx.tar.gz 42 | tar xf /usr/local/src/nginx.tar.gz -C /usr/local/ 43 | test -d /etc/nginx || ln -s /usr/local/nginx-${nginx_ver}/conf /etc/nginx 44 | test -x /usr/sbin/nginx || ln -s /usr/local/nginx-${nginx_ver}/sbin/nginx /usr/sbin/nginx 45 | test -d /var/log/nginx || ln -s /usr/local/nginx-${nginx_ver}/logs /var/log/nginx 46 | id nginx >/dev/null 2>&1 || useradd -d /var/www -s /bin/false nginx 47 | 48 | test -f /etc/systemd/system/nginx.service || \ 49 | cp /data/containers/nginx/nginx.service /etc/systemd/system/nginx.service && systemctl daemon-reload 50 | rm -rf /etc/nginx/nginx.conf && \ 51 | ln -s /data/containers/nginx/nginx.conf /etc/nginx/nginx.conf 52 | rm -rf /etc/nginx/conf.d && \ 53 | ln -s /data/containers/nginx/conf.d /etc/nginx/conf.d 54 | 55 | test -f /etc/nginx/tls.crt || \ 56 | openssl req -x509 -days 365 -newkey rsa:2048 -nodes \ 57 | -out /etc/nginx/tls.crt -keyout /etc/nginx/tls.key \ 58 | -subj "/C=JP/ST=Fukuoka/L=Fukuoka/O=FastContainer/OU=Haconiwa/CN=${common_name}" >/dev/null 2>&1 59 | 60 | # add script 61 | rm -rf /usr/local/bin/cleanip && ln -s /data/containers/cleanip /usr/local/bin/cleanip 62 | 63 | # dstat daemon 64 | #apt-get install -y dstat 65 | #test -f /etc/systemd/system/dstat.service || \ 66 | # cp /data/containers/dstat.service /etc/systemd/system/dstat.service && systemctl daemon-reload 67 | #systemctl enable dstat && systemctl start dstat 68 | #systemctl enable nginx && systemctl start nginx 69 | 70 | # install warp 71 | warp_ver=0.4.0 72 | wget https://github.com/linyows/warp/releases/download/v${warp_ver}/warp_linux_x86_64.tar.gz 73 | tar xzf warp_linux_x86_64.tar.gz 74 | rm -rf /usr/bin/warp 75 | install warp /usr/bin 76 | test -f /etc/systemd/system/warp.service || \ 77 | cp /data/containers/warp.service /etc/systemd/system/warp.service && systemctl daemon-reload 78 | systemctl enable warp && systemctl start warp 79 | 80 | # stack the containers 81 | # /data/containers/batch.sh 82 | -------------------------------------------------------------------------------- /provision/containers/warp.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Warp 3 | Requires=network-online.target 4 | After=network-online.target 5 | 6 | [Service] 7 | ExecStart=/usr/bin/warp -ip 192.168.30.10 -port 10025 8 | KillSignal=SIGINT 9 | Restart=on-failure 10 | 11 | [Install] 12 | WantedBy=multi-user.target 13 | -------------------------------------------------------------------------------- /provision/dist/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FastContainer/nginx-haconiwa/14c0928f1fc7bcc918c7362abca8cd05443f5cf2/provision/dist/.keep -------------------------------------------------------------------------------- /provision/dist/postfix-workdir.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FastContainer/nginx-haconiwa/14c0928f1fc7bcc918c7362abca8cd05443f5cf2/provision/dist/postfix-workdir.tar.gz -------------------------------------------------------------------------------- /provision/hacos/nginx.haco: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | def Kernel.container_working?(ip, port, veth) 4 | system "netstat -nt | grep -v TIME_WAIT | awk '{print $5}' | grep #{ip}:#{port}" 5 | end 6 | 7 | Haconiwa.define do |c| 8 | id = ENV['ID'] 9 | ip = ENV['IP'] 10 | port = ENV['PORT'] 11 | 12 | c.name = id 13 | c.cgroup_name = "#{id}-#{Time.now.to_i}" 14 | c.workdir = '/root' 15 | 16 | root = Pathname.new("/var/lib/haconiwa/rootfs/#{id}") 17 | c.init_command = ['/usr/sbin/nginx', '-g', 'daemon off;'] 18 | c.command.set_stdout(file: "/var/log/haconiwa.out") 19 | c.command.set_stderr(file: "/var/log/haconiwa.err") 20 | c.chroot_to root 21 | c.daemonize! 22 | 23 | c.lxcfs_root = '/var/lib/lxcfs' 24 | 25 | c.mount_independent 'procfs' 26 | c.mount_independent 'sysfs' 27 | c.mount_independent 'devtmpfs' 28 | c.mount_independent 'devpts' 29 | c.mount_independent 'shm' 30 | c.add_mount_point('/etc/resolv.conf', to: "#{root}/etc/resolv.conf") 31 | 32 | c.namespace.unshare 'mount' 33 | c.namespace.unshare 'ipc' 34 | c.namespace.unshare 'uts' 35 | c.namespace.unshare 'pid' 36 | 37 | c.network.namespace = id 38 | c.network.container_ip = ip 39 | c.network.bridge_name = 'haconiwa0' 40 | c.network.veth_host = veth = "veth#{::SHA1.sha1_hex(id)[0, 4]}" 41 | c.network.veth_guest = 'eth0' 42 | 43 | c.cgroup['pids.max'] = 1024 44 | c.capabilities.allow 'cap_sys_chroot' 45 | c.capabilities.allow 'cap_net_bind_service' 46 | 47 | c.add_async_hook(msec: 60 * 1000) do |base| 48 | if container_working?(ip, port, veth) 49 | Haconiwa::Logger.info "shutdown canceled because container(#{ip}:#{port} on #{veth}) still seems working. wait next shutdown" 50 | next 51 | end 52 | Haconiwa::Logger.info "Process killed: #{base.pid}, container not working on #{ip}:#{port}" 53 | ::Process.kill :TERM, base.pid 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /provision/hacos/postfix-standalone.haco: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | Haconiwa.define do |c| 5 | id = ENV['ID'] 6 | domain = ENV['DOMAIN'] || id 7 | ip = ENV['IP'] 8 | port = ENV['PORT'] 9 | 10 | c.name = id 11 | c.cgroup_name = "#{id}-#{Time.now.to_i}" 12 | c.environ = { 'maildomain' => domain } 13 | 14 | root = Pathname.new("/var/lib/haconiwa/rootfs/shared/postfix") 15 | c.init_command = ['/entry.sh'] 16 | c.command.set_stdout(file: "/var/log/haconiwa.out") 17 | c.command.set_stderr(file: "/var/log/haconiwa.err") 18 | c.chroot_to root 19 | c.daemonize! 20 | 21 | c.lxcfs_root = '/var/lib/lxcfs' 22 | 23 | c.mount_independent 'procfs' 24 | c.mount_independent 'sysfs' 25 | c.mount_independent 'devtmpfs' 26 | c.mount_independent 'devpts' 27 | c.mount_independent 'shm' 28 | c.add_mount_point('/etc/resolv.conf', to: "#{root}/etc/resolv.conf") 29 | c.add_mount_point("/var/lib/haconiwa/rootfs/#{id}/etc", to: "#{root}/etc/postfix") 30 | c.add_mount_point("/var/lib/haconiwa/rootfs/#{id}/spool", to: "#{root}/var/spool/postfix") 31 | c.add_mount_point("/var/lib/haconiwa/rootfs/#{id}/lib", to: "#{root}/var/lib/postfix") 32 | 33 | c.namespace.unshare 'mount' 34 | c.namespace.unshare 'ipc' 35 | c.namespace.unshare 'uts' 36 | c.namespace.unshare 'pid' 37 | 38 | c.network.namespace = id 39 | c.network.container_ip = ip 40 | c.network.bridge_name = 'haconiwa0' 41 | c.network.veth_host = veth = "veth#{::SHA1.sha1_hex(id)[0, 4]}" 42 | c.network.veth_guest = 'eth0' 43 | 44 | c.cgroup['pids.max'] = 1024 45 | c.capabilities.allow 'cap_sys_chroot' 46 | c.capabilities.allow 'cap_net_bind_service' 47 | c.capabilities.allow 'cap_kill' 48 | c.capabilities.allow 'cap_audit_write' 49 | 50 | c.add_general_hook :before_start_wait do |base| 51 | Haconiwa::Logger.info "haconiwa hook: setup -- #{base.name}" 52 | end 53 | 54 | c.add_general_hook :teardown do |base| 55 | Haconiwa::Logger.info "haconiwa hook: teardown -- #{base.name}" 56 | system "/usr/local/bin/delroute del #{ip}" 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /provision/hacos/postfix.haco: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | Haconiwa.define do |c| 5 | def Kernel.container_working?(ip, port, veth) 6 | system "netstat -nt | grep -v TIME_WAIT | awk '{print $5}' | grep #{ip}:#{port}" 7 | end 8 | 9 | def Kernel.mailq_exists? 10 | system "test `mailq | grep -c \"^[A-F0-9]\"` != 0" 11 | end 12 | 13 | id = ENV['ID'] 14 | domain = ENV['DOMAIN'] || id 15 | ip = ENV['IP'] 16 | port = ENV['PORT'] 17 | bench = ENV['BENCH'] || 'false' 18 | #relay = '' 19 | relay = ENV['RELAY'] || 'monolith' 20 | shared = ENV['SHARED'] || 'false' 21 | 22 | c.name = id 23 | c.cgroup_name = "#{id}-#{Time.now.to_i}" 24 | c.environ = { 'maildomain' => domain, 'bench' => bench, 'relayhost' => relay } 25 | 26 | root = Pathname.new("/var/lib/haconiwa/rootfs/#{shared == 'true' ? 'shared/postfix' : id}") 27 | c.init_command = ['/entry.sh'] 28 | c.command.set_stdout(file: "/var/log/haconiwa.out") 29 | c.command.set_stderr(file: "/var/log/haconiwa.err") 30 | c.chroot_to root 31 | c.daemonize! 32 | 33 | c.lxcfs_root = '/var/lib/lxcfs' 34 | 35 | c.mount_independent 'procfs' 36 | c.mount_independent 'sysfs' 37 | c.mount_independent 'devtmpfs' 38 | c.mount_independent 'devpts' 39 | c.mount_independent 'shm' 40 | c.add_mount_point('/etc/resolv.conf', to: "#{root}/etc/resolv.conf") 41 | if shared == 'true' 42 | c.add_mount_point("/var/lib/haconiwa/rootfs/#{id}/etc", to: "#{root}/etc/postfix") 43 | c.add_mount_point("/var/lib/haconiwa/rootfs/#{id}/spool", to: "#{root}/var/spool/postfix") 44 | c.add_mount_point("/var/lib/haconiwa/rootfs/#{id}/lib", to: "#{root}/var/lib/postfix") 45 | end 46 | 47 | c.namespace.unshare 'mount' 48 | c.namespace.unshare 'ipc' 49 | c.namespace.unshare 'uts' 50 | c.namespace.unshare 'pid' 51 | 52 | c.network.namespace = id 53 | c.network.container_ip = ip 54 | c.network.bridge_name = 'haconiwa0' 55 | c.network.veth_host = veth = "veth#{::SHA1.sha1_hex(id)[0, 4]}" 56 | c.network.veth_guest = 'eth0' 57 | 58 | c.cgroup['pids.max'] = 1024 59 | c.capabilities.allow 'cap_sys_chroot' 60 | c.capabilities.allow 'cap_net_bind_service' 61 | c.capabilities.allow 'cap_kill' 62 | c.capabilities.allow 'cap_audit_write' 63 | 64 | c.add_async_hook(min: 3, interval_msec: 3 * 60 * 1000) do |base| 65 | if container_working?(ip, 25, veth) 66 | Haconiwa::Logger.info "shutdown canceled because container(#{ip}:#{port} on #{veth}) still seems working. wait next shutdown" 67 | next 68 | end 69 | if mailq_exists? 70 | Haconiwa::Logger.info "shutdown canceled because container(#{ip}:#{port} on #{veth}) still have mail queue. wait next shutdown" 71 | next 72 | end 73 | Haconiwa::Logger.info "process killed: #{base.pid}, container not working on #{ip}:#{port}" 74 | ::Process.kill :TERM, base.pid 75 | end 76 | 77 | c.add_general_hook :before_start_wait do |base| 78 | Haconiwa::Logger.info "haconiwa hook: setup -- #{base.name}" 79 | system "/usr/local/bin/cleanip add #{ip}" 80 | end 81 | 82 | c.add_general_hook :teardown do |base| 83 | Haconiwa::Logger.info "haconiwa hook: teardown -- #{base.name}" 84 | system "/usr/local/bin/delroute del #{ip}" 85 | end 86 | end 87 | -------------------------------------------------------------------------------- /provision/hacos/ssh.haco: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | Haconiwa.define do |c| 5 | def Kernel.container_working?(ip, port, veth) 6 | system "netstat -nt | grep -v TIME_WAIT | awk '{print $5}' | grep #{ip}:#{port}" 7 | end 8 | 9 | id = ENV['ID'] 10 | ip = ENV['IP'] 11 | port = ENV['PORT'] 12 | 13 | c.name = id 14 | c.cgroup_name = "#{id}-#{Time.now.to_i}" 15 | c.workdir = '/root' 16 | 17 | root = Pathname.new("/var/lib/haconiwa/rootfs/#{id}") 18 | c.init_command = ['/usr/sbin/sshd', '-D'] 19 | c.command.set_stdout(file: "/var/log/haconiwa.out") 20 | c.command.set_stderr(file: "/var/log/haconiwa.err") 21 | c.chroot_to root 22 | c.daemonize! 23 | 24 | c.lxcfs_root = '/var/lib/lxcfs' 25 | 26 | c.mount_independent 'procfs' 27 | c.mount_independent 'sysfs' 28 | c.mount_independent 'devtmpfs' 29 | c.mount_independent 'devpts' 30 | c.mount_independent 'shm' 31 | c.add_mount_point('/etc/resolv.conf', to: "#{root}/etc/resolv.conf") 32 | 33 | c.namespace.unshare 'mount' 34 | c.namespace.unshare 'ipc' 35 | c.namespace.unshare 'uts' 36 | c.namespace.unshare 'pid' 37 | 38 | c.network.namespace = id 39 | c.network.container_ip = ip 40 | c.network.bridge_name = 'haconiwa0' 41 | c.network.veth_host = veth = "veth#{::SHA1.sha1_hex(id)[0, 4]}" 42 | c.network.veth_guest = 'eth0' 43 | 44 | c.cgroup['pids.max'] = 1024 45 | c.capabilities.allow 'cap_sys_chroot' 46 | c.capabilities.allow 'cap_net_bind_service' 47 | c.capabilities.allow 'cap_kill' 48 | c.capabilities.allow 'cap_audit_write' 49 | 50 | c.add_async_hook(msec: 3 * 60 * 1000) do |base| 51 | if container_working?(ip, port, veth) 52 | Haconiwa::Logger.info "shutdown canceled because container(#{ip}:#{port} on #{veth}) still seems working. wait next shutdown" 53 | next 54 | end 55 | Haconiwa::Logger.info "Process killed: #{base.pid}, container not working on #{ip}:#{port}" 56 | ::Process.kill :TERM, base.pid 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /provision/hacos/test.haco: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | Haconiwa.define do |c| 4 | id = ENV['ID'] 5 | ip = ENV['IP'] 6 | port = ENV['PORT'] 7 | 8 | c.name = id 9 | c.cgroup_name = "#{id}-#{Time.now.to_i}" 10 | c.workdir = '/root' 11 | 12 | root = Pathname.new("/var/lib/haconiwa/rootfs/#{id}") 13 | c.init_command = ['/bin/sh'] 14 | c.command.set_stdout(file: "/var/log/haconiwa.out") 15 | c.command.set_stderr(file: "/var/log/haconiwa.err") 16 | c.chroot_to root 17 | #c.daemonize! 18 | 19 | c.lxcfs_root = '/var/lib/lxcfs' 20 | 21 | c.mount_independent 'procfs' 22 | c.mount_independent 'sysfs' 23 | c.mount_independent 'devtmpfs' 24 | c.mount_independent 'devpts' 25 | c.mount_independent 'shm' 26 | c.add_mount_point('/etc/resolv.conf', to: "#{root}/etc/resolv.conf") 27 | 28 | c.namespace.unshare 'mount' 29 | c.namespace.unshare 'ipc' 30 | c.namespace.unshare 'uts' 31 | c.namespace.unshare 'pid' 32 | 33 | c.network.namespace = id 34 | c.network.container_ip = ip 35 | c.network.bridge_name = 'haconiwa0' 36 | c.network.veth_host = veth = "veth#{::SHA1.sha1_hex(id)[0, 4]}" 37 | c.network.veth_guest = 'eth0' 38 | 39 | c.cgroup['pids.max'] = 1024 40 | c.capabilities.allow 'cap_sys_chroot' 41 | c.capabilities.allow 'cap_net_bind_service' 42 | end 43 | -------------------------------------------------------------------------------- /provision/hacos/wordpress.haco: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | def Kernel.container_working?(ip, port, veth) 4 | system "netstat -nt | grep -v TIME_WAIT | awk '{print $5}' | grep #{ip}:#{port}" 5 | end 6 | 7 | Haconiwa.define do |c| 8 | id = ENV['ID'] || "wordpress-10-1-100-20" 9 | ip = ENV['IP'] || "10.1.100.20" 10 | port = ENV['PORT'] || "80" 11 | 12 | c.name = id 13 | c.cgroup_name = "#{id}-#{Time.now.to_i}" 14 | c.workdir = '/root' 15 | 16 | root = Pathname.new("/var/lib/haconiwa/rootfs/#{id}") 17 | c.init_command = ['/usr/sbin/apache2', '-k', 'start', '-DFOREGROUND'] 18 | # c.init_command = ['/bin/sh', '-c', "sleep 1; exec /usr/sbin/apache2 -k start -DFOREGROUND"] 19 | c.command.set_stdout(file: "/var/log/haconiwa.out") 20 | c.command.set_stderr(file: "/var/log/haconiwa.err") 21 | c.chroot_to root 22 | c.environ = ENV.to_hash.merge({ 23 | 'APACHE_PORT' => '80', 24 | 'APACHE_RUN_DIR' => '/var/run/apache2', 25 | 'APACHE_PID_FILE' => "/var/run/apache2/apache2.#{$$}.pid", 26 | 'APACHE_LOCK_DIR' => '/var/lock/apache2', 27 | 'APACHE_RUN_USER' => 'www-data', 28 | 'APACHE_RUN_GROUP' => 'www-data', 29 | 'APACHE_LOG_DIR' => '/var/log/apache2', 30 | }) 31 | c.daemonize! 32 | 33 | # c.lxcfs_root = '/var/lib/lxcfs' 34 | 35 | c.mount_independent 'procfs' 36 | c.mount_independent 'sysfs' 37 | c.mount_independent 'devtmpfs' 38 | c.mount_independent 'devpts' 39 | c.mount_independent 'shm' 40 | #c.mount_network_etc(root, host_root: '/etc') 41 | 42 | c.namespace.unshare 'mount' 43 | c.namespace.unshare 'ipc' 44 | c.namespace.unshare 'uts' 45 | c.namespace.unshare 'pid' 46 | 47 | c.network.namespace = id 48 | c.network.container_ip = ip 49 | c.network.bridge_name = 'haconiwa0' 50 | c.network.veth_host = veth = "veth#{::SHA1.sha1_hex(id)[0, 4]}" 51 | c.network.veth_guest = 'veth0' 52 | 53 | c.cgroup['pids.max'] = 1024 54 | c.capabilities.allow 'cap_sys_chroot' 55 | c.capabilities.allow 'cap_net_bind_service' 56 | 57 | c.checkpoint do |checkpoint| 58 | checkpoint.target_syscall :listen, 0 59 | checkpoint.images_dir = "/tmp/criu/images/apache" 60 | checkpoint.criu_log_file = "-" # "/var/log/criu.all" 61 | checkpoint.log_level = 4 62 | checkpoint.criu_service_address = "/var/run/criu_service.socket" 63 | end 64 | 65 | c.add_async_hook(msec: 60 * 60 * 1000) do |base| 66 | Haconiwa::Logger.info "Process going to stop gracefully: #{base.pid}, container not working on #{ip}:#{port}" 67 | #::Process.kill :WINCH, base.pid 68 | system "/bin/kill -WINCH #{base.pid}" 69 | 70 | loop do 71 | if container_working?(ip, port, veth) 72 | Haconiwa::Logger.info "shutdown canceled because container(#{ip}:#{port} on #{veth}) still seems working. wait next shutdown" 73 | usleep 10 * 1000 74 | else 75 | break 76 | end 77 | end 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /provision/hosts: -------------------------------------------------------------------------------- 1 | 192.168.30.10 containers container-1.test container-2.test container-3.test container-4.test 2 | 192.168.30.12 sender 3 | 192.168.30.13 recipient 4 | 192.168.30.14 mxtarpit 5 | 6 | 192.168.30.11 monolith mono-0.test mono-1.test mono-2.test mono-3.test mono-4.test mono-5.test mono-6.test mono-7.test mono-8.test mono-9.test mono-10.test mono-11.test mono-12.test mono-13.test mono-14.test mono-15.test mono-16.test mono-17.test mono-18.test mono-19.test mono-20.test mono-21.test mono-22.test mono-23.test mono-24.test mono-25.test mono-26.test mono-27.test mono-28.test mono-29.test mono-30.test mono-31.test mono-32.test mono-33.test mono-34.test mono-35.test mono-36.test mono-37.test mono-38.test mono-39.test mono-40.test mono-41.test mono-42.test mono-43.test mono-44.test mono-45.test mono-46.test mono-47.test mono-48.test mono-49.test mono-50.test mono-51.test mono-52.test mono-53.test mono-54.test mono-55.test mono-56.test mono-57.test mono-58.test mono-59.test mono-60.test mono-61.test mono-62.test mono-63.test mono-64.test mono-65.test mono-66.test mono-67.test mono-68.test mono-69.test mono-70.test mono-71.test mono-72.test mono-73.test mono-74.test mono-75.test mono-76.test mono-77.test mono-78.test mono-79.test mono-80.test mono-81.test mono-82.test mono-83.test mono-84.test mono-85.test mono-86.test mono-87.test mono-88.test mono-89.test mono-90.test mono-91.test mono-92.test mono-93.test mono-94.test mono-95.test mono-96.test mono-97.test mono-98.test mono-99.test mono-100.test mono-101.test mono-102.test mono-103.test mono-104.test mono-105.test mono-106.test mono-107.test mono-108.test mono-109.test mono-110.test mono-111.test mono-112.test mono-113.test mono-114.test mono-115.test mono-116.test mono-117.test mono-118.test mono-119.test mono-120.test mono-121.test mono-122.test mono-123.test mono-124.test mono-125.test mono-126.test mono-127.test mono-128.test mono-129.test mono-130.test mono-131.test mono-132.test mono-133.test mono-134.test mono-135.test mono-136.test mono-137.test mono-138.test mono-139.test mono-140.test mono-141.test mono-142.test mono-143.test mono-144.test mono-145.test mono-146.test mono-147.test mono-148.test mono-149.test mono-150.test mono-151.test mono-152.test mono-153.test mono-154.test mono-155.test mono-156.test mono-157.test mono-158.test mono-159.test mono-160.test mono-161.test mono-162.test mono-163.test mono-164.test mono-165.test mono-166.test mono-167.test mono-168.test mono-169.test mono-170.test mono-171.test mono-172.test mono-173.test mono-174.test mono-175.test mono-176.test mono-177.test mono-178.test mono-179.test mono-180.test mono-181.test mono-182.test mono-183.test mono-184.test mono-185.test mono-186.test mono-187.test mono-188.test mono-189.test mono-190.test mono-191.test mono-192.test mono-193.test mono-194.test mono-195.test mono-196.test mono-197.test mono-198.test mono-199.test mono-200.test mono-201.test mono-202.test mono-203.test mono-204.test mono-205.test mono-206.test mono-207.test mono-208.test mono-209.test mono-210.test mono-211.test mono-212.test mono-213.test mono-214.test mono-215.test mono-216.test mono-217.test mono-218.test mono-219.test mono-220.test mono-221.test mono-222.test mono-223.test mono-224.test mono-225.test mono-226.test mono-227.test mono-228.test mono-229.test mono-230.test mono-231.test mono-232.test mono-233.test mono-234.test mono-235.test mono-236.test mono-237.test mono-238.test mono-239.test mono-240.test mono-241.test mono-242.test mono-243.test mono-244.test mono-245.test mono-246.test mono-247.test mono-248.test mono-249.test mono-250.test mono-251.test mono-252.test mono-253.test mono-254.test mono-255.test mono-256.test mono-257.test mono-258.test mono-259.test mono-260.test mono-261.test mono-262.test mono-263.test mono-264.test mono-265.test mono-266.test mono-267.test mono-268.test mono-269.test mono-270.test mono-271.test mono-272.test mono-273.test mono-274.test mono-275.test mono-276.test mono-277.test mono-278.test mono-279.test mono-280.test mono-281.test mono-282.test mono-283.test mono-284.test mono-285.test mono-286.test mono-287.test mono-288.test mono-289.test mono-290.test mono-291.test mono-292.test mono-293.test mono-294.test mono-295.test mono-296.test mono-297.test mono-298.test mono-299.test mono-300.test mono-301.test mono-302.test mono-303.test mono-304.test mono-305.test mono-306.test mono-307.test mono-308.test mono-309.test mono-310.test mono-311.test mono-312.test mono-313.test mono-314.test mono-315.test mono-316.test mono-317.test mono-318.test mono-319.test mono-320.test mono-321.test mono-322.test mono-323.test mono-324.test mono-325.test mono-326.test mono-327.test mono-328.test mono-329.test mono-330.test mono-331.test mono-332.test mono-333.test mono-334.test mono-335.test mono-336.test mono-337.test mono-338.test mono-339.test mono-340.test mono-341.test mono-342.test mono-343.test mono-344.test mono-345.test mono-346.test mono-347.test mono-348.test mono-349.test mono-350.test mono-351.test mono-352.test mono-353.test mono-354.test mono-355.test mono-356.test mono-357.test mono-358.test mono-359.test mono-360.test mono-361.test mono-362.test mono-363.test mono-364.test mono-365.test mono-366.test mono-367.test mono-368.test mono-369.test mono-370.test mono-371.test mono-372.test mono-373.test mono-374.test mono-375.test mono-376.test mono-377.test mono-378.test mono-379.test mono-380.test mono-381.test mono-382.test mono-383.test mono-384.test mono-385.test mono-386.test mono-387.test mono-388.test mono-389.test mono-390.test mono-391.test mono-392.test mono-393.test mono-394.test mono-395.test mono-396.test mono-397.test mono-398.test mono-399.test mono-400.test mono-401.test mono-402.test mono-403.test mono-404.test mono-405.test mono-406.test mono-407.test mono-408.test mono-409.test mono-410.test mono-411.test mono-412.test mono-413.test mono-414.test mono-415.test mono-416.test mono-417.test mono-418.test mono-419.test mono-420.test mono-421.test mono-422.test mono-423.test mono-424.test mono-425.test mono-426.test mono-427.test mono-428.test mono-429.test mono-430.test mono-431.test mono-432.test mono-433.test mono-434.test mono-435.test mono-436.test mono-437.test mono-438.test mono-439.test mono-440.test mono-441.test mono-442.test mono-443.test mono-444.test mono-445.test mono-446.test mono-447.test mono-448.test mono-449.test mono-450.test mono-451.test mono-452.test mono-453.test mono-454.test mono-455.test mono-456.test mono-457.test mono-458.test mono-459.test mono-460.test mono-461.test mono-462.test mono-463.test mono-464.test mono-465.test mono-466.test mono-467.test mono-468.test mono-469.test mono-470.test mono-471.test mono-472.test mono-473.test mono-474.test mono-475.test mono-476.test mono-477.test mono-478.test mono-479.test mono-480.test mono-481.test mono-482.test mono-483.test mono-484.test mono-485.test mono-486.test mono-487.test mono-488.test mono-489.test mono-490.test mono-491.test mono-492.test mono-493.test mono-494.test mono-495.test mono-496.test mono-497.test mono-498.test mono-499.test mono-500.test mono-501.test mono-502.test mono-503.test mono-504.test mono-505.test mono-506.test mono-507.test mono-508.test mono-509.test mono-510.test mono-511.test mono-512.test mono-513.test mono-514.test mono-515.test mono-516.test mono-517.test mono-518.test mono-519.test mono-520.test mono-521.test mono-522.test mono-523.test mono-524.test mono-525.test mono-526.test mono-527.test mono-528.test mono-529.test mono-530.test mono-531.test mono-532.test mono-533.test mono-534.test mono-535.test mono-536.test mono-537.test mono-538.test mono-539.test mono-540.test mono-541.test mono-542.test mono-543.test mono-544.test mono-545.test mono-546.test mono-547.test mono-548.test mono-549.test mono-550.test mono-551.test mono-552.test mono-553.test mono-554.test mono-555.test mono-556.test mono-557.test mono-558.test mono-559.test mono-560.test mono-561.test mono-562.test mono-563.test mono-564.test mono-565.test mono-566.test mono-567.test mono-568.test mono-569.test mono-570.test mono-571.test mono-572.test mono-573.test mono-574.test mono-575.test mono-576.test mono-577.test mono-578.test mono-579.test mono-580.test mono-581.test mono-582.test mono-583.test mono-584.test mono-585.test mono-586.test mono-587.test mono-588.test mono-589.test mono-590.test mono-591.test mono-592.test mono-593.test mono-594.test mono-595.test mono-596.test mono-597.test mono-598.test mono-599.test mono-600.test mono-601.test mono-602.test mono-603.test mono-604.test mono-605.test mono-606.test mono-607.test mono-608.test mono-609.test mono-610.test mono-611.test mono-612.test mono-613.test mono-614.test mono-615.test mono-616.test mono-617.test mono-618.test mono-619.test mono-620.test mono-621.test mono-622.test mono-623.test mono-624.test mono-625.test mono-626.test mono-627.test mono-628.test mono-629.test mono-630.test mono-631.test mono-632.test mono-633.test mono-634.test mono-635.test mono-636.test mono-637.test mono-638.test mono-639.test mono-640.test mono-641.test mono-642.test mono-643.test mono-644.test mono-645.test mono-646.test mono-647.test mono-648.test mono-649.test mono-650.test mono-651.test mono-652.test mono-653.test mono-654.test mono-655.test mono-656.test mono-657.test mono-658.test mono-659.test mono-660.test mono-661.test mono-662.test mono-663.test mono-664.test mono-665.test mono-666.test mono-667.test mono-668.test mono-669.test mono-670.test mono-671.test mono-672.test mono-673.test mono-674.test mono-675.test mono-676.test mono-677.test mono-678.test mono-679.test mono-680.test mono-681.test mono-682.test mono-683.test mono-684.test mono-685.test mono-686.test mono-687.test mono-688.test mono-689.test mono-690.test mono-691.test mono-692.test mono-693.test mono-694.test mono-695.test mono-696.test mono-697.test mono-698.test mono-699.test mono-700.test mono-701.test mono-702.test mono-703.test mono-704.test mono-705.test mono-706.test mono-707.test mono-708.test mono-709.test mono-710.test mono-711.test mono-712.test mono-713.test mono-714.test mono-715.test mono-716.test mono-717.test mono-718.test mono-719.test mono-720.test mono-721.test mono-722.test mono-723.test mono-724.test mono-725.test mono-726.test mono-727.test mono-728.test mono-729.test mono-730.test mono-731.test mono-732.test mono-733.test mono-734.test mono-735.test mono-736.test mono-737.test mono-738.test mono-739.test mono-740.test mono-741.test mono-742.test mono-743.test mono-744.test mono-745.test mono-746.test mono-747.test mono-748.test mono-749.test mono-750.test mono-751.test mono-752.test mono-753.test mono-754.test mono-755.test mono-756.test mono-757.test mono-758.test mono-759.test mono-760.test mono-761.test mono-762.test mono-763.test mono-764.test mono-765.test mono-766.test mono-767.test mono-768.test mono-769.test mono-770.test mono-771.test mono-772.test mono-773.test mono-774.test mono-775.test mono-776.test mono-777.test mono-778.test mono-779.test mono-780.test mono-781.test mono-782.test mono-783.test mono-784.test mono-785.test mono-786.test mono-787.test mono-788.test mono-789.test mono-790.test mono-791.test mono-792.test mono-793.test mono-794.test mono-795.test mono-796.test mono-797.test mono-798.test mono-799.test mono-800.test mono-801.test mono-802.test mono-803.test mono-804.test mono-805.test mono-806.test mono-807.test mono-808.test mono-809.test mono-810.test mono-811.test mono-812.test mono-813.test mono-814.test mono-815.test mono-816.test mono-817.test mono-818.test mono-819.test mono-820.test mono-821.test mono-822.test mono-823.test mono-824.test mono-825.test mono-826.test mono-827.test mono-828.test mono-829.test mono-830.test mono-831.test mono-832.test mono-833.test mono-834.test mono-835.test mono-836.test mono-837.test mono-838.test mono-839.test mono-840.test mono-841.test mono-842.test mono-843.test mono-844.test mono-845.test mono-846.test mono-847.test mono-848.test mono-849.test mono-850.test mono-851.test mono-852.test mono-853.test mono-854.test mono-855.test mono-856.test mono-857.test mono-858.test mono-859.test mono-860.test mono-861.test mono-862.test mono-863.test mono-864.test mono-865.test mono-866.test mono-867.test mono-868.test mono-869.test mono-870.test mono-871.test mono-872.test mono-873.test mono-874.test mono-875.test mono-876.test mono-877.test mono-878.test mono-879.test mono-880.test mono-881.test mono-882.test mono-883.test mono-884.test mono-885.test mono-886.test mono-887.test mono-888.test mono-889.test mono-890.test mono-891.test mono-892.test mono-893.test mono-894.test mono-895.test mono-896.test mono-897.test mono-898.test mono-899.test mono-900.test mono-901.test mono-902.test mono-903.test mono-904.test mono-905.test mono-906.test mono-907.test mono-908.test mono-909.test mono-910.test mono-911.test mono-912.test mono-913.test mono-914.test mono-915.test mono-916.test mono-917.test mono-918.test mono-919.test mono-920.test mono-921.test mono-922.test mono-923.test mono-924.test mono-925.test mono-926.test mono-927.test mono-928.test mono-929.test mono-930.test mono-931.test mono-932.test mono-933.test mono-934.test mono-935.test mono-936.test mono-937.test mono-938.test mono-939.test mono-940.test mono-941.test mono-942.test mono-943.test mono-944.test mono-945.test mono-946.test mono-947.test mono-948.test mono-949.test mono-950.test mono-951.test mono-952.test mono-953.test mono-954.test mono-955.test mono-956.test mono-957.test mono-958.test mono-959.test mono-960.test mono-961.test mono-962.test mono-963.test mono-964.test mono-965.test mono-966.test mono-967.test mono-968.test mono-969.test mono-970.test mono-971.test mono-972.test mono-973.test mono-974.test mono-975.test mono-976.test mono-977.test mono-978.test mono-979.test mono-980.test mono-981.test mono-982.test mono-983.test mono-984.test mono-985.test mono-986.test mono-987.test mono-988.test mono-989.test mono-990.test mono-991.test mono-992.test mono-993.test mono-994.test mono-995.test mono-996.test mono-997.test mono-998.test mono-999.test 7 | -------------------------------------------------------------------------------- /provision/monolith/dstat.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Dstat 3 | Requires=network-online.target 4 | After=network-online.target 5 | 6 | [Service] 7 | ExecStart=/usr/bin/dstat -tcdnymp --output /data/dstat.monolith.csv 8 | KillSignal=SIGINT 9 | Restart=on-failure 10 | 11 | [Install] 12 | WantedBy=multi-user.target 13 | -------------------------------------------------------------------------------- /provision/monolith/nginx.conf: -------------------------------------------------------------------------------- 1 | worker_processes 2; 2 | user root; 3 | error_log logs/error.log debug; 4 | pid /run/nginx.pid; 5 | 6 | events { 7 | worker_connections 1024; 8 | } 9 | 10 | http { 11 | include mime.types; 12 | default_type application/octet-stream; 13 | sendfile on; 14 | keepalive_timeout 65; 15 | client_max_body_size 20m; 16 | lingering_close on; 17 | lingering_time 30s; 18 | lingering_timeout 15s; 19 | vhost_traffic_status_zone shared:vhost_traffic_status:50m; 20 | server { 21 | listen 127.0.0.1:58080; 22 | server_name localhost; 23 | 24 | location /smtp_auth { 25 | mruby_content_handler_code ' 26 | r = Nginx::Request.new 27 | user = r.headers_in["Auth-User"] 28 | pass = r.headers_in["Auth-Pass"] 29 | prot = r.headers_in["Auth-Protocol"] 30 | Nginx.rputs "SMTP Auth" 31 | 32 | r.headers_out["Auth-Status"] = -> do 33 | users = [] 34 | 10000.times { |i| users.push("user#{i}") } 35 | 36 | if users.include?(user) && users.include?(pass) 37 | r.headers_out["Auth-Server"] = "127.0.0.1" 38 | r.headers_out["Auth-Port"] = "25" 39 | return "OK" 40 | end 41 | 42 | return "Invalid login or password" 43 | end.call 44 | 45 | Nginx.log Nginx::LOG_NOTICE, "auth info: #{user} #{prot}" 46 | '; 47 | } 48 | } 49 | } 50 | 51 | mail { 52 | proxy on; 53 | proxy_pass_error_message on; 54 | 55 | auth_http 127.0.0.1:58080/smtp_auth; 56 | smtp_capabilities PIPELINING 8BITMIME "SIZE 20480000"; 57 | imap_capabilities IMAP4rev1 LITERAL+ SASL-IR LOGIN-REFERRALS ID ENABLE IDLE AUTH=LOGIN; 58 | smtp_auth login plain; 59 | 60 | server { 61 | listen 127.0.0.1:8025; 62 | protocol smtp; 63 | xclient off; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /provision/monolith/nginx.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=The NGINX HTTP and reverse proxy server 3 | After=syslog.target network.target remote-fs.target nss-lookup.target 4 | 5 | [Service] 6 | Type=forking 7 | PIDFile=/run/nginx.pid 8 | ExecStartPre=/usr/sbin/nginx -t 9 | ExecStart=/usr/sbin/nginx 10 | ExecReload=/bin/kill -s HUP $MAINPID 11 | ExecStop=/bin/kill -s QUIT $MAINPID 12 | PrivateTmp=false 13 | LimitNOFILE=30000 14 | 15 | [Install] 16 | WantedBy=multi-user.target 17 | -------------------------------------------------------------------------------- /provision/monolith/script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | apt upgrade -y 4 | apt install -y bridge-utils openssl curl dstat 5 | locale-gen ja_JP.UTF-8 6 | 7 | # install nginx 8 | nginx_ver=1.13.12 9 | cp /data/dist/nginx-${nginx_ver}.tar.gz /usr/local/src/nginx.tar.gz 10 | tar xf /usr/local/src/nginx.tar.gz -C /usr/local/ 11 | test -d /etc/nginx || ln -s /usr/local/nginx-${nginx_ver}/conf /etc/nginx 12 | test -x /usr/sbin/nginx || ln -s /usr/local/nginx-${nginx_ver}/sbin/nginx /usr/sbin/nginx 13 | test -d /var/log/nginx || ln -s /usr/local/nginx-${nginx_ver}/logs /var/log/nginx 14 | id nginx >/dev/null 2>&1 || useradd -d /var/www -s /bin/false nginx 15 | 16 | test -f /etc/systemd/system/nginx.service || \ 17 | cp /data/monolith/nginx.service /etc/systemd/system/nginx.service && systemctl daemon-reload 18 | rm -rf /etc/nginx/nginx.conf && \ 19 | ln -s /data/monolith/nginx.conf /etc/nginx/nginx.conf 20 | 21 | systemctl enable nginx && systemctl start nginx 22 | 23 | postconf -e default_process_limit=10000 24 | postconf -e smtpd_client_connection_count_limit=10000 25 | 26 | systemctl restart postfix 27 | 28 | # install dstat 29 | #test -f /etc/systemd/system/dstat.service || \ 30 | # cp /data/monolith/dstat.service /etc/systemd/system/dstat.service && systemctl daemon-reload 31 | #systemctl enable dstat && systemctl start dstat 32 | -------------------------------------------------------------------------------- /provision/mxtarpit/mxtarpit.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Fake SMTP Backup MX Tarpit 3 | After=syslog.target network.target remote-fs.target nss-lookup.target 4 | 5 | [Service] 6 | Type=simple 7 | PIDFile=/run/mxtarpit.pid 8 | ExecStart=/usr/sbin/mxtarpit -F 9 | ExecReload=/bin/kill -s HUP $MAINPID 10 | ExecStop=/bin/kill -s QUIT $MAINPID 11 | PrivateTmp=false 12 | 13 | [Install] 14 | WantedBy=multi-user.target 15 | -------------------------------------------------------------------------------- /provision/mxtarpit/script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | apt-get update -yy 4 | apt-get install -yy make build-essential libev-dev 5 | locale-gen ja_JP.UTF-8 6 | 7 | test -d /var/empty || mkdir /var/empty 8 | test -d /usr/src/mxtarpit || \ 9 | git clone https://github.com/martinh/mxtarpit.git /usr/src/mxtarpit 10 | cd /usr/src/mxtarpit 11 | 12 | # change interval 13 | sed -e 's/app.spam_stutter_interval = 3/app.spam_stutter_interval = 0.25/' mxtarpit.c 14 | 15 | make linux 16 | mv ./mxtarpit /usr/sbin/mxtarpit 17 | test -f /etc/systemd/system/mxtarpit.service || \ 18 | cp /tmp/mxtarpit.service /etc/systemd/system/mxtarpit.service && systemctl daemon-reload 19 | systemctl start mxtarpit 20 | -------------------------------------------------------------------------------- /provision/postfix.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | export DEBIAN_FRONTEND=noninteractive 4 | 5 | grep 192.168.30 /etc/hosts >/dev/null || cat /tmp/hosts >> /etc/hosts 6 | 7 | apt update -y 8 | apt install -y postfix curl 9 | locale-gen ja_JP.UTF-8 10 | 11 | postconf -e myhostname="$(hostname)" 12 | postconf -e mynetworks='127.0.0.0/8 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8' 13 | postconf -e smtp_host_lookup='native' 14 | postconf -e smtp_dns_support_level='disabled' 15 | postconf -e default_process_limit='5' 16 | 17 | systemctl restart postfix 18 | -------------------------------------------------------------------------------- /provision/sandbox-container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | NAME=postfix 4 | IP=10.1.100.88 5 | PORT=19999 6 | 7 | ID=$NAME-sandbox 8 | ROOTFS=/var/lib/haconiwa/rootfs/$ID 9 | IMAGE=/var/lib/haconiwa/images/$NAME.image.tar 10 | HACOFILE=/var/lib/haconiwa/hacos/$NAME.haco 11 | PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin 12 | 13 | if [ ! -d $ROOTFS ]; then 14 | /bin/mkdir -m 755 -p $ROOTFS 15 | /bin/tar xfp $IMAGE -C $ROOTFS 16 | fi 17 | 18 | /bin/echo "127.0.0.1 localhost $ID" >> $ROOTFS/etc/hosts && 19 | /bin/cat /data/hosts >> $ROOTFS/etc/hosts 20 | 21 | /bin/env IP=$IP PORT=$PORT ID=$ID PATH=$PATH /usr/bin/haconiwa start $HACOFILE 22 | -------------------------------------------------------------------------------- /provision/sender/script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | export GOLANG_VERSION=1.12.6 4 | export GOPATH=/go 5 | 6 | apt-get update 7 | apt-get install -y --no-install-recommends g++ gcc libc6-dev make pkg-config 8 | wget -O go.tgz "https://golang.org/dl/go${GOLANG_VERSION}.linux-amd64.tar.gz" 9 | tar -C /usr/local -xzf go.tgz 10 | rm go.tgz 11 | 12 | mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" 13 | mkdir -p "$GOPATH/src/github.com/FastContainer" 14 | git clone https://github.com/FastContainer/playback.git "$GOPATH/src/github.com/FastContainer/playback" 15 | 16 | ln -s /usr/local/go/bin/go /usr/local/bin/go 17 | cd /go/src/github.com/FastContainer/playback 18 | export GO111MODULE=on 19 | make 20 | mv ./playback /usr/local/bin/playback 21 | -------------------------------------------------------------------------------- /test/integration: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | CLR_PASS="\\033[0;32m" 4 | CLR_FAIL="\\033[0;31m" 5 | CLR_WARN="\\033[0;33m" 6 | CLR_INFO="\\033[0;34m" 7 | CLR_RESET="\\033[0;39m" 8 | ALL_PASSED=0 9 | 10 | email=$1 11 | namespace='fast_container' 12 | 13 | function pass() { 14 | echo -e "[${CLR_PASS}PASS${CLR_RESET}] $namespace $(echo $1 | sed -e "s/test_//")" 15 | } 16 | 17 | function fail() { 18 | ALL_PASSED=1 19 | echo -e "[${CLR_FAIL}FAIL${CLR_RESET}] $namespace $(echo $1 | sed -e "s/test_//")" 20 | echo -e "${CLR_INFO}Expected${CLR_RESET}:" 21 | echo -e "$2" 22 | echo -e "${CLR_WARN}Actual${CLR_RESET}:" 23 | echo -e "$3" 24 | } 25 | 26 | function test_http_ok_as_localhost() { 27 | expected='200' 28 | actual="$(curl -s http://127.0.0.1:8080/ -o /dev/null -w '%{http_code}')" 29 | 30 | if [ "$actual" == "$expected" ]; then 31 | pass "${FUNCNAME[0]}" 32 | else 33 | fail "${FUNCNAME[0]}" "$expected" "$actual" 34 | fi 35 | } 36 | 37 | function test_http_ok_as_foo() { 38 | domain='foo.test' 39 | expected='200' 40 | actual="$(curl -s -H "Host: $domain" http://127.0.0.1:8080/ -o /dev/null -w '%{http_code}')" 41 | 42 | if [ "$actual" == "$expected" ]; then 43 | pass "${FUNCNAME[0]}" 44 | else 45 | fail "${FUNCNAME[0]}" "$expected" "$actual" 46 | fi 47 | } 48 | 49 | function test_http_ok_as_bar() { 50 | domain='bar.test' 51 | expected='200' 52 | actual="$(curl -s -H "Host: $domain" http://127.0.0.1:8080/ -o /dev/null -w '%{http_code}')" 53 | 54 | if [ "$actual" == "$expected" ]; then 55 | pass "${FUNCNAME[0]}" 56 | else 57 | fail "${FUNCNAME[0]}" "$expected" "$actual" 58 | fi 59 | } 60 | 61 | function test_ssh_login() { 62 | expected='0' 63 | actual="$(test/ssh 127.0.0.1 8022 root screencast; echo $?)" 64 | 65 | if [ "$actual" == "$expected" ]; then 66 | pass "${FUNCNAME[0]}" 67 | else 68 | fail "${FUNCNAME[0]}" "$expected" "$actual" 69 | fi 70 | } 71 | 72 | function test_smtp_send_as_foo() { 73 | auth='Zm9vAGZvbwBwYXNzd29yZA==' 74 | expected='0' 75 | actual="$(test/smtp localhost $email $email $auth; echo $?)" 76 | 77 | if [ "$actual" == "$expected" ]; then 78 | pass "${FUNCNAME[0]}" 79 | else 80 | fail "${FUNCNAME[0]}" "$expected" "$actual" 81 | fi 82 | } 83 | 84 | function test_smtp_send_as_bar() { 85 | auth='YmFyAGJhcgBwYXNzd29yZA==' 86 | expected='0' 87 | actual="$(test/smtp localhost $email $email $auth; echo $?)" 88 | 89 | if [ "$actual" == "$expected" ]; then 90 | pass "${FUNCNAME[0]}" 91 | else 92 | fail "${FUNCNAME[0]}" "$expected" "$actual" 93 | fi 94 | } 95 | 96 | function run_test() { 97 | self=$(cd $(dirname $0) && pwd)/$(basename $0) 98 | tests="$(grep "^function test_" $self | sed -E "s/function (.*)\(\) \{/\1/g")" 99 | for t in $(echo $tests); do 100 | $t 101 | done 102 | } 103 | 104 | if [ "$email" == "" ];then 105 | echo "Usage: $0 test@example.com" 106 | exit 1 107 | fi 108 | run_test 109 | exit $ALL_PASSED 110 | -------------------------------------------------------------------------------- /test/smtp: -------------------------------------------------------------------------------- 1 | #!/usr/bin/expect 2 | 3 | set timeout 10 4 | 5 | proc abort {} { 6 | puts $expect_out(buffer) 7 | exit 2 8 | } 9 | 10 | if {$argc != 4} { 11 | puts "Usage: $argv0 smtp_host from_addr to_addr auth" 12 | exit 1 13 | } 14 | 15 | set host [lindex $argv 0] 16 | set fadr [lindex $argv 1] 17 | set tadr [lindex $argv 2] 18 | set auth [lindex $argv 3] 19 | 20 | log_user 0 21 | spawn telnet $host 8025 22 | expect default abort -re "220.*\n" 23 | send "HELO $host\r" 24 | expect default abort -re "250.*\n" 25 | send "AUTH PLAIN $auth\r" 26 | expect default abort -re "235.*\n" 27 | send "MAIL FROM:<$fadr>\r" 28 | expect default abort -re "250.*\n" 29 | send "RCPT TO:<$tadr>\r" 30 | expect default abort -re "250.*\n" 31 | send "DATA\r" 32 | expect default abort -re "354.*\n" 33 | send "From: $fadr\rTo: $tadr\rSubject: Hello\rThis is from postfix on fast container.\r.\r" 34 | expect default abort -re "250.*\n" 35 | send "QUIT\r" 36 | expect default abort -re "Connection closed.*\n" 37 | -------------------------------------------------------------------------------- /test/ssh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/expect 2 | 3 | set timeout 10 4 | 5 | proc abort {} { 6 | puts $expect_out(buffer) 7 | exit 2 8 | } 9 | 10 | if {$argc != 4} { 11 | puts "Usage: $argv0 host port user password" 12 | exit 1 13 | } 14 | 15 | set host [lindex $argv 0] 16 | set port [lindex $argv 1] 17 | set user [lindex $argv 2] 18 | set pass [lindex $argv 3] 19 | 20 | log_user 0 21 | spawn ssh $user@$host -p $port 22 | expect default abort -re ".*password: " 23 | send "$pass\n" 24 | expect default abort -re ".*:~#.*" 25 | send "exit\n" 26 | expect default abort -re ".*Connection to $host closed.*\n" 27 | --------------------------------------------------------------------------------