├── .gitignore
├── inventory
├── roles
    ├── common
    │   └── tasks
    │   │   ├── main.yml
    │   │   └── create_sshkey.yml
    ├── defunctzombie.coreos-bootstrap
    │   ├── tests
    │   │   └── test.yml
    │   ├── meta
    │   │   ├── .galaxy_install_info
    │   │   └── main.yml
    │   ├── files
    │   │   ├── runner
    │   │   └── bootstrap.sh
    │   ├── .editorconfig
    │   ├── .travis.yml
    │   ├── tasks
    │   │   └── main.yml
    │   ├── LICENSE
    │   └── README.md
    ├── etcd
    │   ├── tasks
    │   │   ├── main.yml
    │   │   ├── create_secgroup.yml
    │   │   ├── create_vm.yml
    │   │   └── create_secgroup_rules.yml
    │   └── templates
    │   │   └── etcd.j2
    └── k8s
    │   ├── tasks
    │       ├── create_inv.yml
    │       ├── main.yml
    │       ├── create_secgroup.yml
    │       ├── create_vm.yml
    │       └── create_secgroup_rules.yml
    │   └── templates
    │       ├── inventory.j2
    │       ├── k8s-node.j2
    │       └── k8s-master.j2
├── ansible.cfg
├── k8s.yml
├── etcd.yml
├── README.md
└── LICENSE


/.gitignore:
--------------------------------------------------------------------------------
1 | cloudstack.ini
2 | *.retry


--------------------------------------------------------------------------------
/inventory:
--------------------------------------------------------------------------------
1 | [default]
2 | <cloudstackserver>
3 | 


--------------------------------------------------------------------------------
/roles/common/tasks/main.yml:
--------------------------------------------------------------------------------
1 | ---
2 |   - include: create_sshkey.yml
3 | 


--------------------------------------------------------------------------------
/ansible.cfg:
--------------------------------------------------------------------------------
1 | [defaults]
2 | hostfile = ./inventory
3 | library = library
4 | 


--------------------------------------------------------------------------------
/roles/defunctzombie.coreos-bootstrap/tests/test.yml:
--------------------------------------------------------------------------------
1 | ---
2 | - hosts: all
3 |   roles:
4 |     - coreos-bootstrap
5 | 


--------------------------------------------------------------------------------
/roles/defunctzombie.coreos-bootstrap/meta/.galaxy_install_info:
--------------------------------------------------------------------------------
1 | {install_date: 'Wed Feb 25 23:17:32 2015', version: master}
2 | 


--------------------------------------------------------------------------------
/roles/etcd/tasks/main.yml:
--------------------------------------------------------------------------------
1 | ---
2 |   - include: create_secgroup.yml
3 |   - include: create_secgroup_rules.yml
4 |   - include: create_vm.yml
5 | 


--------------------------------------------------------------------------------
/roles/defunctzombie.coreos-bootstrap/files/runner:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | LD_LIBRARY_PATH=$HOME/pypy/lib:$LD_LIBRARY_PATH $HOME/pypy/bin/$(basename $0) $@
3 | 


--------------------------------------------------------------------------------
/roles/k8s/tasks/create_inv.yml:
--------------------------------------------------------------------------------
1 | # Create inventory file
2 | 
3 |   - name: Create inventory file
4 |     template: src=inventory.j2 dest=./inventory
5 |     tags: inventory
6 | 


--------------------------------------------------------------------------------
/roles/k8s/tasks/main.yml:
--------------------------------------------------------------------------------
1 | ---
2 |   - include: create_secgroup.yml
3 |   - include: create_secgroup_rules.yml
4 |   - include: create_vm.yml
5 |   - include: create_inv.yml
6 | 


--------------------------------------------------------------------------------
/roles/k8s/tasks/create_secgroup.yml:
--------------------------------------------------------------------------------
1 | # Create k8s security group
2 | 
3 |   - name: Create k8s Security Group
4 |     local_action:
5 |      module: cs_securitygroup
6 |      name: "{{ k8s_security_group_name }}" 
7 |      description: k8s 
8 | 


--------------------------------------------------------------------------------
/roles/etcd/tasks/create_secgroup.yml:
--------------------------------------------------------------------------------
1 | # Create Backend security group
2 | 
3 |   - name: Create etcd Security Group
4 |     local_action:
5 |      module: cs_securitygroup
6 |      name: "{{ etcd_security_group_name }}"
7 |      description: etcd 
8 | 


--------------------------------------------------------------------------------
/roles/defunctzombie.coreos-bootstrap/meta/main.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | dependencies: []
 3 | 
 4 | galaxy_info:
 5 |   author: defunctzombie
 6 |   description: "bootstrap coreos hosts to run ansible"
 7 |   license: "MIT"
 8 |   min_ansible_version: 1.4
 9 |   categories:
10 |     - system
11 | 


--------------------------------------------------------------------------------
/roles/defunctzombie.coreos-bootstrap/.editorconfig:
--------------------------------------------------------------------------------
 1 | # EditorConfig is awesome: http://EditorConfig.org
 2 | 
 3 | # top-most EditorConfig file
 4 | root = true
 5 | 
 6 | # Unix-style newlines with a newline ending every file
 7 | [*]
 8 | end_of_line = lf
 9 | insert_final_newline = true
10 | indent_style = space
11 | indent_size = 2
12 | 


--------------------------------------------------------------------------------
/k8s.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | - hosts: <cloudstackserver>
 3 |   connection: ssh
 4 |   vars:
 5 |     ssh_key: k8s
 6 |     k8s_version: v0.11.0
 7 |     k8s_num_nodes: 2
 8 |     k8s_security_group_name: k8s
 9 |     k8s_node_prefix: k8s
10 |     k8s_template: <templatename>
11 |     k8s_instance_type: <serviceofferingname>
12 | 
13 |   roles:
14 |     - common
15 |     - k8s
16 | 


--------------------------------------------------------------------------------
/roles/etcd/tasks/create_vm.yml:
--------------------------------------------------------------------------------
 1 |   - name: Start etcd nodes
 2 |     local_action:
 3 |       module: cs_instance
 4 |       name: "{{ etcd_node_prefix }}-node-{{ item }}"
 5 |       template: "{{ etcd_template }}"
 6 |       service_offering: "{{ etcd_instance_type }}"
 7 |       ssh_key: "{{ ssh_key }}"
 8 |       user_data: "{{ lookup('template', '../templates/etcd.j2') }}"
 9 |     with_sequence: count={{ etcd_num_nodes }}
10 | 


--------------------------------------------------------------------------------
/roles/etcd/templates/etcd.j2:
--------------------------------------------------------------------------------
 1 | #cloud-config
 2 | 
 3 | coreos:
 4 |   fleet:
 5 |     metadata: role=etcd
 6 |   etcd:
 7 |     discovery: "{{ token.stdout }}" 
 8 |     addr: $public_ipv4:4001
 9 |     peer-addr: $public_ipv4:7001
10 |     snapshot: true
11 |   units:
12 |     - name: etcd.service
13 |       command: start
14 |     - name: fleet.service
15 |       command: start
16 |   update:
17 |     group: alpha
18 |     reboot-strategy: off
19 | 


--------------------------------------------------------------------------------
/roles/k8s/templates/inventory.j2:
--------------------------------------------------------------------------------
 1 | [default]
 2 | localhost ansible_python_interpreter=/opt/local/Library/Frameworks/Python.framework/Versions/2.7/bin/python
 3 | 
 4 | [nodes]
 5 | {{ k8s_master.default_ip }}
 6 | {% for item in k8s_nodes.results %}
 7 | {{ item.default_ip }}
 8 | {% endfor %}
 9 | 
10 | [nodes:vars]
11 | ansible_ssh_user=core
12 | ansible_ssh_private_key_file=~/.ssh/id_rsa_{{ ssh_key }}
13 | ansible_python_interpreter="PATH=/home/core/bin:$PATH python"
14 | 


--------------------------------------------------------------------------------
/roles/defunctzombie.coreos-bootstrap/.travis.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | language: python
 3 | python: "2.7"
 4 | 
 5 | env:
 6 |   - SITE=test.yml
 7 | 
 8 | before_install:
 9 |   - sudo apt-get update -qq
10 |   - sudo apt-get install -y curl
11 | 
12 | install:
13 |   - pip install ansible
14 | 
15 |   # Add ansible.cfg to pick up roles path.
16 |   - "printf '[defaults]\nroles_path = ../' > ansible.cfg"
17 | 
18 | script:
19 |   - "ansible-playbook -i tests/inventory tests/$SITE --syntax-check"
20 | 


--------------------------------------------------------------------------------
/etcd.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | - hosts: localhost
 3 |   connection: local
 4 | 
 5 |   vars:
 6 |     ssh_key: etcd
 7 |     etcd_num_nodes: 3
 8 |     etcd_security_group_name: etcd
 9 |     etcd_node_prefix: etcd
10 |     etcd_template: Linux CoreOS alpha 435 64-bit 10GB Disk 
11 |     etcd_instance_type: Tiny
12 | 
13 |   pre_tasks:
14 |     - name: Get bootstrap token for etcd
15 |       command: curl -w "\n" https://discovery.etcd.io/new
16 |       register: token
17 |     - debug: msg='Etcd token is {{ token.stdout }}'
18 |     
19 |   roles:
20 |     - common
21 |     - etcd
22 | 


--------------------------------------------------------------------------------
/roles/common/tasks/create_sshkey.yml:
--------------------------------------------------------------------------------
 1 |   - name: Create SSH Key
 2 |     local_action:
 3 |       module: cs_sshkeypair
 4 |       name: "{{ ssh_key }}"
 5 |     register: key
 6 |     tags: sshkey
 7 | 
 8 |   - debug: msg='private key is {{ key.private_key }}'
 9 |     when: key.changed
10 |     tags: sshkey
11 |  
12 |   - local_action: copy content="{{ key.private_key }}" dest="~/.ssh/id_rsa_{{ ssh_key }}"
13 |     when: key.changed
14 |     tags: sshkey
15 | 
16 |   - file: path="~/.ssh/id_rsa_{{ ssh_key }}" mode=0600
17 |     when: key.changed
18 |     tags: sshkey
19 | 


--------------------------------------------------------------------------------
/roles/defunctzombie.coreos-bootstrap/files/bootstrap.sh:
--------------------------------------------------------------------------------
 1 | #/bin/bash
 2 | 
 3 | set -e
 4 | 
 5 | cd
 6 | 
 7 | if [-e $HOME/.bootstrapped]; then
 8 |   exit 0
 9 | fi
10 | 
11 | PYPY_VERSION=2.4.0
12 | 
13 | wget https://bitbucket.org/pypy/pypy/downloads/pypy-$PYPY_VERSION-linux64.tar.bz2
14 | tar -xf pypy-$PYPY_VERSION-linux64.tar.bz2
15 | mv pypy-$PYPY_VERSION-linux64 pypy
16 | 
17 | ## library fixup
18 | mkdir pypy/lib
19 | ln -s /lib64/libncurses.so.5.9 $HOME/pypy/lib/libtinfo.so.5
20 | 
21 | mkdir -p $HOME/bin
22 | 
23 | cat > $HOME/bin/python <<EOF
24 | #!/bin/bash
25 | LD_LIBRARY_PATH=$HOME/pypy/lib:$LD_LIBRARY_PATH $HOME/pypy/bin/pypy "\$@"
26 | EOF
27 | 
28 | chmod +x $HOME/bin/python
29 | $HOME/bin/python --version
30 | 
31 | touch $HOME/.bootstrapped
32 | 
33 | rm -f pypy-$PYPY_VERSION-linux64.tar.bz2
34 | 


--------------------------------------------------------------------------------
/roles/defunctzombie.coreos-bootstrap/tasks/main.yml:
--------------------------------------------------------------------------------
 1 | - name: Check if bootstrap is needed
 2 |   raw: stat $HOME/.bootstrapped
 3 |   register: need_bootstrap
 4 |   ignore_errors: True
 5 | 
 6 | - name: Run bootstrap.sh
 7 |   script: bootstrap.sh
 8 |   when: need_bootstrap | failed
 9 | 
10 | - name: Check if we need to install pip
11 |   shell: "{{ansible_python_interpreter}} -m pip --version"
12 |   register: need_pip
13 |   ignore_errors: True
14 |   changed_when: false
15 | 
16 | - name: Copy get-pip.py
17 |   copy: src=get-pip.py dest=~/get-pip.py
18 |   when: need_pip | failed
19 | 
20 | - name: Install pip
21 |   shell: "{{ansible_python_interpreter}} ~/get-pip.py"
22 |   when: need_pip | failed
23 | 
24 | - name: Remove get-pip.py
25 |   command: rm -f ~/get-pip.py
26 |   ignore_errors: True
27 | 
28 | - name: Install pip launcher
29 |   copy: src=runner dest=~/bin/pip mode=0755
30 | 


--------------------------------------------------------------------------------
/roles/k8s/tasks/create_vm.yml:
--------------------------------------------------------------------------------
 1 |   - name: Start k8s head node
 2 |     local_action:
 3 |       module: cs_instance
 4 |       name: "{{ k8s_node_prefix }}-head"
 5 |       template: "{{ k8s_template }}"
 6 |       service_offering: "{{ k8s_instance_type }}"
 7 |       ssh_key: "{{ ssh_key }}"
 8 |       security_groups: [ '{{ k8s_security_group_name }}' ]
 9 |       user_data: "{{ lookup('template', '../templates/k8s-master.j2') }}"
10 |     register: k8s_master
11 |     
12 |   - debug: msg='k8s master IP is {{ k8s_master.default_ip }}'
13 |               
14 |   - name: Start k8s nodes
15 |     local_action:
16 |       module: cs_instance
17 |       name: "{{ k8s_node_prefix }}-node-{{ item }}"
18 |       template: "{{ k8s_template }}"
19 |       service_offering: "{{ k8s_instance_type }}"
20 |       ssh_key: "{{ ssh_key }}"
21 |       security_groups: [ '{{ k8s_security_group_name }}' ]
22 |       user_data: "{{ lookup('template', '../templates/k8s-node.j2') }}"
23 |     with_sequence: count={{ k8s_num_nodes }}
24 |     register: k8s_nodes
25 | 


--------------------------------------------------------------------------------
/roles/etcd/tasks/create_secgroup_rules.yml:
--------------------------------------------------------------------------------
 1 | # Rules for the etcd security group
 2 | 
 3 |   - name: SSH for etcd
 4 |     local_action:
 5 |      module: cs_securitygroup_rule
 6 |      name: "{{ etcd_security_group_name }}"
 7 |      start_port: 22
 8 |      end_port: 22
 9 | 
10 |   - name: etcd 4001
11 |     local_action:
12 |      module: cs_securitygroup_rule
13 |      name: "{{ etcd_security_group_name }}"
14 |      start_port: 4001
15 |      end_port: 4001
16 |      user_security_group: "{{ etcd_security_group_name }}"
17 | 
18 |   - name: etcd 7001
19 |     local_action:
20 |      module: cs_securitygroup_rule
21 |      name: "{{ etcd_security_group_name }}"
22 |      start_port: 7001
23 |      end_port: 7001
24 |      user_security_group: "{{ etcd_security_group_name }}"
25 | 
26 | #  - name: k8s 4001
27 | #    local_action:
28 | #     module: cs_securitygroup_rule
29 | #     name: "{{ etcd_security_group_name }}"
30 | #     start_port: 4001
31 | #     end_port: 4001
32 | #     user_security_group: "{{ k8s_security_group_name }}"
33 | 
34 | 


--------------------------------------------------------------------------------
/roles/defunctzombie.coreos-bootstrap/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (c) 2014 Roman Shtylman
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/roles/k8s/tasks/create_secgroup_rules.yml:
--------------------------------------------------------------------------------
 1 | # Add Rule to etcd security group
 2 | 
 3 | #  - name: etcd access for k8s
 4 | #    local_action:
 5 | #     module: cs_securitygroup_rule
 6 | #     name: "{{ etcd_security_group_name }}"
 7 | #     start_port: 4001
 8 | #     end_port: 4001
 9 | #     user_security_group: "{{ k8s_security_group_name }}"
10 | 
11 | # Rules for the k8s security group
12 | 
13 |   - name: SSH for k8s
14 |     local_action:
15 |      module: cs_securitygroup_rule
16 |      name: "{{ k8s_security_group_name }}"
17 |      protocol: tcp
18 |      start_port: 22
19 |      end_port: 22
20 | 
21 |   - name: k8s 4001
22 |     local_action:
23 |      module: cs_securitygroup_rule
24 |      name: "{{ k8s_security_group_name }}" 
25 |      start_port: 4001
26 |      end_port: 4001
27 |      user_security_group: "{{ k8s_security_group_name }}" 
28 | 
29 |   - name: k8s 7001
30 |     local_action:
31 |      module: cs_securitygroup_rule
32 |      name: "{{ k8s_security_group_name }}" 
33 |      start_port: 7001
34 |      end_port: 7001
35 |      user_security_group: "{{ k8s_security_group_name }}" 
36 | 
37 |   - name: k8s 10250
38 |     local_action:
39 |      module: cs_securitygroup_rule
40 |      name: "{{ k8s_security_group_name }}" 
41 |      start_port: 10250
42 |      end_port: 10250
43 |      user_security_group: "{{ k8s_security_group_name }}" 
44 | 
45 |   - name: k8s UDP 8285
46 |     local_action:
47 |      module: cs_securitygroup_rule
48 |      name: "{{ k8s_security_group_name }}" 
49 |      start_port: 8285
50 |      end_port: 8285
51 |      protocol: udp
52 |      user_security_group: "{{ k8s_security_group_name }}" 
53 | 
54 |   - name: k8s 8080
55 |     local_action:
56 |      module: cs_securitygroup_rule
57 |      name: "{{ k8s_security_group_name }}"
58 |      start_port: 8080
59 |      end_port: 8080
60 |      user_security_group: "{{ k8s_security_group_name }}"
61 | 
62 | 


--------------------------------------------------------------------------------
/roles/defunctzombie.coreos-bootstrap/README.md:
--------------------------------------------------------------------------------
 1 | # coreos-bootstrap
 2 | 
 3 | In order to effectively run ansible, the target machine needs to have a python interpreter. Coreos machines are minimal and do not ship with any version of python. To get around this limitation we can install [pypy](http://pypy.org/), a lightweight python interpreter. The coreos-bootstrap role will install pypy for us and we will update our inventory file to use the installed python interpreter.
 4 | 
 5 | # install
 6 | 
 7 | ```
 8 | ansible-galaxy install defunctzombie.coreos-bootstrap
 9 | ```
10 | 
11 | # Configure your project
12 | 
13 | Unlike a typical role, you need to configure Ansible to use an alternative python interpreter for coreos hosts. This can be done by adding a `coreos` group to your inventory file and setting the group's vars to use the new python interpreter. This way, you can use ansible to manage CoreOS and non-CoreOS hosts. Simply put every host that has CoreOS into the `coreos` inventory group and it will automatically use the specified python interpreter.
14 | ```
15 | [coreos]
16 | host-01
17 | host-02
18 | 
19 | [coreos:vars]
20 | ansible_ssh_user=core
21 | ansible_python_interpreter="PATH=/home/core/bin:$PATH python"
22 | ```
23 | 
24 | This will configure ansible to use the python interpreter at `/home/core/bin/python` which will be created by the coreos-bootstrap role.
25 | 
26 | ## Bootstrap Playbook
27 | 
28 | Now you can simply add the following to your playbook file and include it in your `site.yml` so that it runs on all hosts in the coreos group.
29 | 
30 | ```yaml
31 | - hosts: coreos
32 |   gather_facts: False
33 |   roles:
34 |     - defunctzombie.coreos-bootstrap
35 | ```
36 | 
37 | Make sure that `gather_facts` is set to false, otherwise ansible will try to first gather system facts using python which is not yet installed!
38 | 
39 | ## Example Playbook
40 | 
41 | After bootstrap, you can use ansible as usual to manage system services, install python modules (via pip), and run containers. Below is a basic example that starts the `etcd` service, installs the `docker-py` module and then uses the ansible `docker` module to pull and start a basic nginx container.
42 | 
43 | ```yaml
44 | - name: Nginx Example
45 |   hosts: web
46 |   sudo: true
47 |   tasks:
48 |     - name: Start etcd
49 |       service: name=etcd.service state=started
50 | 
51 |     - name: Install docker-py
52 |       pip: name=docker-py
53 | 
54 |     - name: pull container
55 |       raw: docker pull nginx:1.7.1
56 | 
57 |     - name: launch nginx container
58 |       docker:
59 |         image="nginx:1.7.1"
60 |         name="example-nginx"
61 |         ports="8080:80"
62 |         state=running
63 | ```
64 | 
65 | # License
66 | MIT
67 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | Ansible Recipes to Install Kubernetes on CloudStack
 2 | =====================================
 3 | 
 4 | Basic recipes using the ansible cloudstack module to create ssh keys, sec group etc and deploy [Kubernetes](http://kubernetes.io) on [CoreOS](http://coreos.com).
 5 | 
 6 | Prerequisites
 7 | -------------
 8 | 
 9 | You will need Ansible and [cs](https://github.com/exoscale/cs) :)
10 | 
11 |     $ sudo apt-get install -y python-pip libssl-dev
12 |     $ sudo pip install cs
13 |     $ sudo pip install sshpubkeys
14 |     $ sudo apt-get install software-properties-common
15 |     $ sudo apt-add-repository ppa:ansible/ansible
16 |     $ sudo apt-get update
17 |     $ sudo apt-get install ansible
18 | 
19 | Setup cs
20 | --------
21 | 
22 | Create a `~/.cloudstack.ini` file with your creds and cloudstack endpoint:
23 | 
24 |     [cloudstack]
25 |     endpoint = <cloudstackapiendpoint>
26 |     key = <apiaccesskey> 
27 |     secret = <apisecretkey> 
28 |     method = post
29 | 
30 | We need to use the http POST method to pass the userdata to the coreOS instances.
31 | 
32 | We can also use variables:
33 | 
34 |     CLOUDSTACK_ENDPOINT=<cloudstackapiendpoint>
35 |     CLOUDSTACK_KEY=<apiaccesskey>
36 |     CLOUDSTACK_SECRET=<apisecretkey>
37 |     CLOUDSTACK_METHOD=post
38 | 
39 | On CloudStack server you have to install libselinux-python
40 | 
41 |     yum install libselinux-python
42 | 
43 | Clone the repository
44 | ---------------
45 | 
46 |     $ git clone https://github.com/apachecloudstack/k8s
47 |     $ cd ansible-kubernetes
48 | 
49 | Create a Kubernetes cluster
50 | ---------------------------
51 | 
52 |     $ ansible-playbook k8s.yml
53 | 
54 | Some variables can be edited in the `k8s.yml` file.
55 | This will start a Kubernetes master node and a number of compute nodes.
56 | This is all setup via coreOS instances and passing userdata.
57 | 
58 | Check the tasks and templates in `roles/k8s`
59 | 
60 | If you retrieve an error during the ssh key copy:
61 | 
62 |     "msg": "file (/root/.ssh/id_rsa_k8s) is absent, cannot continue",
63 | 
64 | Please run the Playbook a second time [(related issue)](https://github.com/apachecloudstack/k8s/issues/5)
65 | 
66 | Create etcd cluster
67 | -------------------
68 | 
69 | That's a bonus to this work, there is a playbook to create an independent etcd cluster.
70 | 
71 |     $ ansible-playbook etcd.yml
72 | 
73 | Edit some of the variables in the `etcd.yml` file directly.
74 | 
75 | Important Notice
76 | -------------
77 | 
78 | If you want to run it on a CloudStack environment with VMware ESX hosts cluster. You have to comment the lines refer to secgroups and secgroup_rules (specific to KVM) :
79 |     k8s.yml
80 |     k8s/tasks/main.yml
81 |     k8s/tasks/create_vm.yml
82 |     etcd.yml
83 |     etcd/tasks/main.yml
84 | 


--------------------------------------------------------------------------------
/roles/k8s/templates/k8s-node.j2:
--------------------------------------------------------------------------------
  1 | #cloud-config
  2 | 
  3 | coreos:
  4 |   units:
  5 |     - name: etcd.service
  6 |       mask: true
  7 |     - name: fleet.service
  8 |       command: start
  9 |       content: |
 10 |         [Unit]
 11 |         Description=fleet daemon
 12 |         Wants=fleet.socket
 13 |         After=fleet.socket
 14 | 
 15 |         [Service]
 16 |         Environment="FLEET_ETCD_SERVERS=http://{{ k8s_master.default_ip}}:4001"
 17 |         Environment="FLEET_METADATA=role=node"
 18 |         ExecStart=/usr/bin/fleetd
 19 |         Restart=always
 20 |         RestartSec=10s
 21 |     - name: flannel.service
 22 |       command: start
 23 |       content: |
 24 |         [Unit]
 25 |         After=network-online.target 
 26 |         Wants=network-online.target
 27 |         Description=flannel is an etcd backed overlay network for containers
 28 | 
 29 |         [Service]
 30 |         Type=notify
 31 |         ExecStartPre=-/usr/bin/mkdir -p /opt/bin
 32 |         ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/k8s/flanneld
 33 |         ExecStartPre=/usr/bin/chmod +x /opt/bin/flanneld
 34 |         ExecStart=/opt/bin/flanneld -etcd-endpoints http://{{ k8s_master.default_ip}}:4001
 35 |     - name: docker.service
 36 |       command: start
 37 |       content: |
 38 |         [Unit]
 39 |         After=flannel.service
 40 |         Wants=flannel.service
 41 |         Description=Docker Application Container Engine
 42 |         Documentation=http://docs.docker.io
 43 | 
 44 |         [Service]
 45 |         EnvironmentFile=/run/flannel/subnet.env
 46 |         ExecStartPre=/bin/mount --make-rprivate /
 47 |         ExecStart=/usr/bin/docker -d --bip=${FLANNEL_SUBNET} --mtu=${FLANNEL_MTU} -s=overlay -H fd://
 48 | 
 49 |         [Install]
 50 |         WantedBy=multi-user.target
 51 |     - name: setup-network-environment.service
 52 |       command: start
 53 |       content: |
 54 |         [Unit]
 55 |         Description=Setup Network Environment
 56 |         Documentation=https://github.com/kelseyhightower/setup-network-environment
 57 |         Requires=network-online.target
 58 |         After=network-online.target
 59 | 
 60 |         [Service]
 61 |         ExecStartPre=-/usr/bin/mkdir -p /opt/bin
 62 |         ExecStartPre=/usr/bin/wget -N -P /opt/bin https://github.com/kelseyhightower/setup-network-environment/releases/download/v1.0.0/setup-network-environment 
 63 |         ExecStartPre=/usr/bin/chmod +x /opt/bin/setup-network-environment
 64 |         ExecStart=/opt/bin/setup-network-environment
 65 |         RemainAfterExit=yes
 66 |         Type=oneshot
 67 |     - name: kube-proxy.service
 68 |       command: start
 69 |       content: |
 70 |         [Unit]
 71 |         Description=Kubernetes Proxy
 72 |         Documentation=https://github.com/GoogleCloudPlatform/kubernetes
 73 |         Requires=setup-network-environment.service
 74 |         After=setup-network-environment.service
 75 | 
 76 |         [Service]
 77 |         ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/{{ k8s_version }}/bin/linux/amd64/kube-proxy
 78 |         ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-proxy
 79 |         ExecStart=/opt/bin/kube-proxy \
 80 |         --etcd_servers=http://{{ k8s_master.default_ip}}:4001 \
 81 |         --logtostderr=true
 82 |         Restart=always
 83 |         RestartSec=10
 84 |     - name: kube-kubelet.service
 85 |       command: start
 86 |       content: |
 87 |         [Unit]
 88 |         Description=Kubernetes Kubelet
 89 |         Documentation=https://github.com/GoogleCloudPlatform/kubernetes
 90 |         Requires=setup-network-environment.service
 91 |         After=setup-network-environment.service
 92 | 
 93 |         [Service]
 94 |         EnvironmentFile=/etc/network-environment
 95 |         ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/{{ k8s_version }}/bin/linux/amd64/kubelet
 96 |         ExecStartPre=/usr/bin/chmod +x /opt/bin/kubelet
 97 |         ExecStart=/opt/bin/kubelet \
 98 |         --address=0.0.0.0 \
 99 |         --port=10250 \
100 |         --hostname_override=$private_ipv4 \
101 |         --etcd_servers=http://{{ k8s_master.default_ip}}:4001 \
102 |         --logtostderr=true
103 |         Restart=always
104 |         RestartSec=10
105 |   update:
106 |     group: alpha
107 |     reboot-strategy: off
108 | 


--------------------------------------------------------------------------------
/roles/k8s/templates/k8s-master.j2:
--------------------------------------------------------------------------------
  1 | #cloud-config
  2 | 
  3 | ---
  4 | write_files:
  5 | - path: /opt/bin/waiter.sh
  6 |   owner: root
  7 |   content: |
  8 |     #! /usr/bin/bash
  9 |     until curl http://127.0.0.1:4001/v2/machines; do sleep 2; done
 10 | coreos:
 11 |   units:
 12 |     - name: setup-network-environment.service
 13 |       command: start
 14 |       content: |
 15 |         [Unit]
 16 |         Description=Setup Network Environment
 17 |         Documentation=https://github.com/kelseyhightower/setup-network-environment
 18 |         Requires=network-online.target
 19 |         After=network-online.target
 20 | 
 21 |         [Service]
 22 |         ExecStartPre=-/usr/bin/mkdir -p /opt/bin
 23 |         ExecStartPre=/usr/bin/wget -N -P /opt/bin https://github.com/kelseyhightower/setup-network-environment/releases/download/v1.0.0/setup-network-environment
 24 |         ExecStartPre=/usr/bin/chmod +x /opt/bin/setup-network-environment
 25 |         ExecStart=/opt/bin/setup-network-environment
 26 |         RemainAfterExit=yes
 27 |         Type=oneshot
 28 |     - name: etcd.service
 29 |       command: start
 30 |       content: |
 31 |         [Unit]
 32 |         Description=etcd
 33 |         Requires=setup-network-environment.service
 34 |         After=setup-network-environment.service
 35 | 
 36 |         [Service]
 37 |         EnvironmentFile=/etc/network-environment
 38 |         User=etcd
 39 |         PermissionsStartOnly=true
 40 |         ExecStart=/usr/bin/etcd \
 41 |         --name ${DEFAULT_IPV4} \
 42 |         --addr ${DEFAULT_IPV4}:4001 \
 43 |         --bind-addr 0.0.0.0 \
 44 |         --cluster-active-size 1 \
 45 |         --data-dir /var/lib/etcd \
 46 |         --http-read-timeout 86400 \
 47 |         --peer-addr ${DEFAULT_IPV4}:7001 \
 48 |         --snapshot true
 49 |         Restart=always
 50 |         RestartSec=10s
 51 |     - name: fleet.socket
 52 |       command: start
 53 |       content: |
 54 |         [Socket]
 55 |         ListenStream=/var/run/fleet.sock
 56 |     - name: fleet.service
 57 |       command: start
 58 |       content: |
 59 |         [Unit]
 60 |         Description=fleet daemon
 61 |         Wants=etcd.service
 62 |         After=etcd.service
 63 |         Wants=fleet.socket
 64 |         After=fleet.socket
 65 | 
 66 |         [Service]
 67 |         Environment="FLEET_ETCD_SERVERS=http://127.0.0.1:4001"
 68 |         Environment="FLEET_METADATA=role=master"
 69 |         ExecStart=/usr/bin/fleetd
 70 |         Restart=always
 71 |         RestartSec=10s
 72 |     - name: etcd-waiter.service
 73 |       command: start
 74 |       content: |
 75 |         [Unit]
 76 |         Description=etcd waiter
 77 |         Wants=network-online.target
 78 |         Wants=etcd.service
 79 |         After=etcd.service
 80 |         After=network-online.target
 81 |         Before=flannel.service
 82 |         Before=setup-network-environment.service
 83 | 
 84 |         [Service]
 85 |         ExecStartPre=/usr/bin/chmod +x /opt/bin/waiter.sh
 86 |         ExecStart=/usr/bin/bash /opt/bin/waiter.sh
 87 |         RemainAfterExit=true
 88 |         Type=oneshot
 89 |     - name: flannel.service
 90 |       command: start
 91 |       content: |
 92 |         [Unit]
 93 |         Wants=etcd-waiter.service
 94 |         After=etcd-waiter.service
 95 |         Requires=etcd.service
 96 |         After=etcd.service
 97 |         After=network-online.target
 98 |         Wants=network-online.target
 99 |         Description=flannel is an etcd backed overlay network for containers
100 | 
101 |         [Service]
102 |         Type=notify
103 |         ExecStartPre=-/usr/bin/mkdir -p /opt/bin
104 |         ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/k8s/flanneld
105 |         ExecStartPre=/usr/bin/chmod +x /opt/bin/flanneld
106 |         ExecStartPre=/usr/bin/etcdctl mk /coreos.com/network/config '{"Network":"10.244.0.0/16", "Backend": {"Type": "vxlan"}}'
107 |         ExecStart=/opt/bin/flanneld
108 |     - name: kube-apiserver.service
109 |       command: start
110 |       content: |
111 |         [Unit]
112 |         Description=Kubernetes API Server
113 |         Documentation=https://github.com/GoogleCloudPlatform/kubernetes
114 |         Requires=etcd.service
115 |         After=etcd.service
116 | 
117 |         [Service]
118 |         ExecStartPre=-/usr/bin/mkdir -p /opt/bin
119 |         ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/{{ k8s_version }}/bin/linux/amd64/kube-apiserver
120 |         ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-apiserver
121 |         ExecStart=/opt/bin/kube-apiserver \
122 |         --address=0.0.0.0 \
123 |         --port=8080 \
124 |         --portal_net=10.244.0.0/16 \
125 |         --etcd_servers=http://127.0.0.1:4001 \
126 |         --public_address_override=$private_ipv4 \
127 |         --logtostderr=true
128 |         Restart=always
129 |         RestartSec=10
130 |     - name: kube-controller-manager.service 
131 |       command: start
132 |       content: |
133 |         [Unit]
134 |         Description=Kubernetes Controller Manager
135 |         Documentation=https://github.com/GoogleCloudPlatform/kubernetes
136 |         Requires=kube-apiserver.service
137 |         After=kube-apiserver.service
138 | 
139 |         [Service]
140 |         ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/{{ k8s_version }}/bin/linux/amd64/kube-controller-manager
141 |         ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-controller-manager
142 |         ExecStart=/opt/bin/kube-controller-manager \
143 |         --master=127.0.0.1:8080 \
144 |         --logtostderr=true
145 |         Restart=always
146 |         RestartSec=10
147 |     - name: kube-scheduler.service
148 |       command: start
149 |       content: |
150 |         [Unit]
151 |         Description=Kubernetes Scheduler
152 |         Documentation=https://github.com/GoogleCloudPlatform/kubernetes
153 |         Requires=kube-apiserver.service
154 |         After=kube-apiserver.service
155 | 
156 |         [Service]
157 |         ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/{{ k8s_version }}/bin/linux/amd64/kube-scheduler
158 |         ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-scheduler
159 |         ExecStart=/opt/bin/kube-scheduler --master=127.0.0.1:8080
160 |         Restart=always
161 |         RestartSec=10
162 |     - name: kube-register.service
163 |       command: start
164 |       content: |
165 |         [Unit]
166 |         Description=Kubernetes Registration Service
167 |         Documentation=https://github.com/kelseyhightower/kube-register
168 |         Requires=kube-apiserver.service
169 |         After=kube-apiserver.service
170 |         Requires=fleet.service
171 |         After=fleet.service
172 | 
173 |         [Service]
174 |         ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/k8s/kube-register
175 |         ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-register
176 |         ExecStart=/opt/bin/kube-register \
177 |         --metadata=role=node \
178 |         --fleet-endpoint=unix:///var/run/fleet.sock \
179 |         --api-endpoint=http://127.0.0.1:8080
180 |         Restart=always
181 |         RestartSec=10
182 |   update:
183 |     group: alpha
184 |     reboot-strategy: off
185 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                                  Apache License
  2 |                            Version 2.0, January 2004
  3 |                         http://www.apache.org/licenses/
  4 | 
  5 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 |    1. Definitions.
  8 | 
  9 |       "License" shall mean the terms and conditions for use, reproduction,
 10 |       and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |       "Licensor" shall mean the copyright owner or entity authorized by
 13 |       the copyright owner that is granting the License.
 14 | 
 15 |       "Legal Entity" shall mean the union of the acting entity and all
 16 |       other entities that control, are controlled by, or are under common
 17 |       control with that entity. For the purposes of this definition,
 18 |       "control" means (i) the power, direct or indirect, to cause the
 19 |       direction or management of such entity, whether by contract or
 20 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |       outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |       "You" (or "Your") shall mean an individual or Legal Entity
 24 |       exercising permissions granted by this License.
 25 | 
 26 |       "Source" form shall mean the preferred form for making modifications,
 27 |       including but not limited to software source code, documentation
 28 |       source, and configuration files.
 29 | 
 30 |       "Object" form shall mean any form resulting from mechanical
 31 |       transformation or translation of a Source form, including but
 32 |       not limited to compiled object code, generated documentation,
 33 |       and conversions to other media types.
 34 | 
 35 |       "Work" shall mean the work of authorship, whether in Source or
 36 |       Object form, made available under the License, as indicated by a
 37 |       copyright notice that is included in or attached to the work
 38 |       (an example is provided in the Appendix below).
 39 | 
 40 |       "Derivative Works" shall mean any work, whether in Source or Object
 41 |       form, that is based on (or derived from) the Work and for which the
 42 |       editorial revisions, annotations, elaborations, or other modifications
 43 |       represent, as a whole, an original work of authorship. For the purposes
 44 |       of this License, Derivative Works shall not include works that remain
 45 |       separable from, or merely link (or bind by name) to the interfaces of,
 46 |       the Work and Derivative Works thereof.
 47 | 
 48 |       "Contribution" shall mean any work of authorship, including
 49 |       the original version of the Work and any modifications or additions
 50 |       to that Work or Derivative Works thereof, that is intentionally
 51 |       submitted to Licensor for inclusion in the Work by the copyright owner
 52 |       or by an individual or Legal Entity authorized to submit on behalf of
 53 |       the copyright owner. For the purposes of this definition, "submitted"
 54 |       means any form of electronic, verbal, or written communication sent
 55 |       to the Licensor or its representatives, including but not limited to
 56 |       communication on electronic mailing lists, source code control systems,
 57 |       and issue tracking systems that are managed by, or on behalf of, the
 58 |       Licensor for the purpose of discussing and improving the Work, but
 59 |       excluding communication that is conspicuously marked or otherwise
 60 |       designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |       on behalf of whom a Contribution has been received by Licensor and
 64 |       subsequently incorporated within the Work.
 65 | 
 66 |    2. Grant of Copyright License. Subject to the terms and conditions of
 67 |       this License, each Contributor hereby grants to You a perpetual,
 68 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |       copyright license to reproduce, prepare Derivative Works of,
 70 |       publicly display, publicly perform, sublicense, and distribute the
 71 |       Work and such Derivative Works in Source or Object form.
 72 | 
 73 |    3. Grant of Patent License. Subject to the terms and conditions of
 74 |       this License, each Contributor hereby grants to You a perpetual,
 75 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |       (except as stated in this section) patent license to make, have made,
 77 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |       where such license applies only to those patent claims licensable
 79 |       by such Contributor that are necessarily infringed by their
 80 |       Contribution(s) alone or by combination of their Contribution(s)
 81 |       with the Work to which such Contribution(s) was submitted. If You
 82 |       institute patent litigation against any entity (including a
 83 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |       or a Contribution incorporated within the Work constitutes direct
 85 |       or contributory patent infringement, then any patent licenses
 86 |       granted to You under this License for that Work shall terminate
 87 |       as of the date such litigation is filed.
 88 | 
 89 |    4. Redistribution. You may reproduce and distribute copies of the
 90 |       Work or Derivative Works thereof in any medium, with or without
 91 |       modifications, and in Source or Object form, provided that You
 92 |       meet the following conditions:
 93 | 
 94 |       (a) You must give any other recipients of the Work or
 95 |           Derivative Works a copy of this License; and
 96 | 
 97 |       (b) You must cause any modified files to carry prominent notices
 98 |           stating that You changed the files; and
 99 | 
100 |       (c) You must retain, in the Source form of any Derivative Works
101 |           that You distribute, all copyright, patent, trademark, and
102 |           attribution notices from the Source form of the Work,
103 |           excluding those notices that do not pertain to any part of
104 |           the Derivative Works; and
105 | 
106 |       (d) If the Work includes a "NOTICE" text file as part of its
107 |           distribution, then any Derivative Works that You distribute must
108 |           include a readable copy of the attribution notices contained
109 |           within such NOTICE file, excluding those notices that do not
110 |           pertain to any part of the Derivative Works, in at least one
111 |           of the following places: within a NOTICE text file distributed
112 |           as part of the Derivative Works; within the Source form or
113 |           documentation, if provided along with the Derivative Works; or,
114 |           within a display generated by the Derivative Works, if and
115 |           wherever such third-party notices normally appear. The contents
116 |           of the NOTICE file are for informational purposes only and
117 |           do not modify the License. You may add Your own attribution
118 |           notices within Derivative Works that You distribute, alongside
119 |           or as an addendum to the NOTICE text from the Work, provided
120 |           that such additional attribution notices cannot be construed
121 |           as modifying the License.
122 | 
123 |       You may add Your own copyright statement to Your modifications and
124 |       may provide additional or different license terms and conditions
125 |       for use, reproduction, or distribution of Your modifications, or
126 |       for any such Derivative Works as a whole, provided Your use,
127 |       reproduction, and distribution of the Work otherwise complies with
128 |       the conditions stated in this License.
129 | 
130 |    5. Submission of Contributions. Unless You explicitly state otherwise,
131 |       any Contribution intentionally submitted for inclusion in the Work
132 |       by You to the Licensor shall be under the terms and conditions of
133 |       this License, without any additional terms or conditions.
134 |       Notwithstanding the above, nothing herein shall supersede or modify
135 |       the terms of any separate license agreement you may have executed
136 |       with Licensor regarding such Contributions.
137 | 
138 |    6. Trademarks. This License does not grant permission to use the trade
139 |       names, trademarks, service marks, or product names of the Licensor,
140 |       except as required for reasonable and customary use in describing the
141 |       origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 |    7. Disclaimer of Warranty. Unless required by applicable law or
144 |       agreed to in writing, Licensor provides the Work (and each
145 |       Contributor provides its Contributions) on an "AS IS" BASIS,
146 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |       implied, including, without limitation, any warranties or conditions
148 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |       PARTICULAR PURPOSE. You are solely responsible for determining the
150 |       appropriateness of using or redistributing the Work and assume any
151 |       risks associated with Your exercise of permissions under this License.
152 | 
153 |    8. Limitation of Liability. In no event and under no legal theory,
154 |       whether in tort (including negligence), contract, or otherwise,
155 |       unless required by applicable law (such as deliberate and grossly
156 |       negligent acts) or agreed to in writing, shall any Contributor be
157 |       liable to You for damages, including any direct, indirect, special,
158 |       incidental, or consequential damages of any character arising as a
159 |       result of this License or out of the use or inability to use the
160 |       Work (including but not limited to damages for loss of goodwill,
161 |       work stoppage, computer failure or malfunction, or any and all
162 |       other commercial damages or losses), even if such Contributor
163 |       has been advised of the possibility of such damages.
164 | 
165 |    9. Accepting Warranty or Additional Liability. While redistributing
166 |       the Work or Derivative Works thereof, You may choose to offer,
167 |       and charge a fee for, acceptance of support, warranty, indemnity,
168 |       or other liability obligations and/or rights consistent with this
169 |       License. However, in accepting such obligations, You may act only
170 |       on Your own behalf and on Your sole responsibility, not on behalf
171 |       of any other Contributor, and only if You agree to indemnify,
172 |       defend, and hold each Contributor harmless for any liability
173 |       incurred by, or claims asserted against, such Contributor by reason
174 |       of your accepting any such warranty or additional liability.
175 | 
176 |    END OF TERMS AND CONDITIONS
177 | 
178 |    APPENDIX: How to apply the Apache License to your work.
179 | 
180 |       To apply the Apache License to your work, attach the following
181 |       boilerplate notice, with the fields enclosed by brackets "{}"
182 |       replaced with your own identifying information. (Don't include
183 |       the brackets!)  The text should be enclosed in the appropriate
184 |       comment syntax for the file format. We also recommend that a
185 |       file or class name and description of purpose be included on the
186 |       same "printed page" as the copyright notice for easier
187 |       identification within third-party archives.
188 | 
189 |    Copyright {yyyy} {name of copyright owner}
190 | 
191 |    Licensed under the Apache License, Version 2.0 (the "License");
192 |    you may not use this file except in compliance with the License.
193 |    You may obtain a copy of the License at
194 | 
195 |        http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 |    Unless required by applicable law or agreed to in writing, software
198 |    distributed under the License is distributed on an "AS IS" BASIS,
199 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 |    See the License for the specific language governing permissions and
201 |    limitations under the License.
202 | 


--------------------------------------------------------------------------------