├── .gitignore ├── roles ├── opencontrail │ ├── meta │ │ └── main.yml │ ├── files │ │ ├── vrouter.conf │ │ ├── contrail-vrouter-agent.upstart │ │ ├── ifmap-server.manifest │ │ ├── rabbitmq.manifest │ │ ├── ifdown-vhost │ │ ├── contrail-schema.manifest │ │ ├── contrail-api.manifest │ │ ├── cassandra.manifest │ │ ├── zookeeper.manifest │ │ ├── contrail-control.manifest │ │ ├── contrail-vrouter-agent.manifest │ │ └── ifup-vhost │ ├── templates │ │ ├── ifcfg-gateway.j2 │ │ ├── route-gateway0.j2 │ │ ├── ifcfg-eth.j2 │ │ ├── route-gateway1.j2 │ │ ├── contrail-api.conf.j2 │ │ ├── contrail-schema.conf.j2 │ │ ├── node-route-redhat.j2 │ │ ├── interface.cfg.j2 │ │ ├── ifcfg-vhost0.j2 │ │ ├── contrail-control.conf.j2 │ │ ├── contrail-vrouter-agent.conf.node.j2 │ │ ├── rabbitmq.service.j2 │ │ ├── zookeeper.service.j2 │ │ ├── ifmap-server.service.j2 │ │ ├── kube-network.conf.j2 │ │ ├── contrail-vrouter-agent.service.j2 │ │ ├── contrail-control.service.j2 │ │ ├── kube-network-manager.manifest.j2 │ │ ├── cassandra.service.j2 │ │ ├── kube-network-manager.service.j2 │ │ ├── contrail-schema.service.j2 │ │ ├── contrail-api.service.j2 │ │ ├── gateway.cfg.j2 │ │ ├── systemd.service.j2 │ │ ├── Dockerfile.debian.j2 │ │ ├── contrail-vrouter-agent.conf.gateway.j2 │ │ ├── vhost0.cfg.j2 │ │ └── Dockerfile.redhat.j2 │ ├── tasks │ │ ├── nodes-redhat.yml │ │ ├── docker_config.yml │ │ ├── vrouter-ubuntu.yml │ │ ├── kmod.yml │ │ ├── masters.yml │ │ ├── vrouter-redhat.yml │ │ ├── gateway-redhat.yml │ │ ├── gateway-ubuntu.yml │ │ ├── iptables.yml │ │ ├── kmod-build-ubuntu.yml │ │ ├── masters-control.yml │ │ ├── vrouter.yml │ │ ├── main.yml │ │ ├── masters-services.yml │ │ ├── gateways.yml │ │ ├── kmod-build-redhat.yml │ │ ├── kmod-artifacts.yml │ │ ├── masters-config.yml │ │ └── nodes.yml │ ├── vars │ │ ├── kubernetes.yml │ │ ├── openshift.yml │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ └── README.md ├── opencontrail_provision │ ├── files │ │ └── namespace.yml │ ├── vars │ │ └── main.yml │ └── tasks │ │ ├── nodes.yml │ │ ├── main.yml │ │ └── master.yml └── opencontrail_facts │ ├── vars │ ├── kubernetes.yml │ ├── main.yml │ └── openshift.yml │ └── tasks │ ├── vhost_ansible_facts.yml │ ├── interface_inventory_facts.yml │ ├── interface_ansible_facts.yml │ └── main.yml ├── test ├── ec2-k8s │ ├── localhost │ ├── roles │ │ ├── basic │ │ │ ├── library │ │ │ ├── templates │ │ │ │ ├── status.j2 │ │ │ │ └── inventory.j2 │ │ │ └── tasks │ │ │ │ └── main.yml │ │ ├── clean │ │ │ ├── library │ │ │ └── tasks │ │ │ │ └── main.yml │ │ ├── workspace │ │ │ ├── files │ │ │ │ ├── ansible.cfg │ │ │ │ └── cluster.patch │ │ │ └── tasks │ │ │ │ └── main.yml │ │ └── deployer_install │ │ │ └── tasks │ │ │ └── main.yml │ ├── .gitignore │ ├── ansible.cfg │ ├── clean.yml │ ├── group_vars │ │ └── all.yml │ └── playbook.yml ├── ec2-origin │ ├── localhost │ ├── roles │ │ ├── clean │ │ │ ├── library │ │ │ └── tasks │ │ │ │ └── main.yml │ │ ├── cluster │ │ │ ├── library │ │ │ ├── templates │ │ │ │ ├── status.j2 │ │ │ │ └── inventory.j2 │ │ │ └── tasks │ │ │ │ └── main.yml │ │ ├── workspace │ │ │ ├── vars │ │ │ │ └── main.yml │ │ │ ├── files │ │ │ │ ├── ansible.cfg │ │ │ │ ├── systemd_workaround.yml │ │ │ │ ├── applications.yml │ │ │ │ ├── rails-postgresql.patch.j2 │ │ │ │ ├── system-install.yml │ │ │ │ ├── deployment_config_set.py │ │ │ │ ├── openshift_provision.yml │ │ │ │ ├── opencontrail.yml │ │ │ │ ├── opencontrail_provision.yml │ │ │ │ └── opencontrail_validate.py │ │ │ └── tasks │ │ │ │ └── main.yml │ │ ├── dns_forwarder │ │ │ ├── handlers │ │ │ │ └── main.yml │ │ │ ├── templates │ │ │ │ └── unbound.conf.j2 │ │ │ └── tasks │ │ │ │ └── main.yml │ │ ├── key_data │ │ │ └── tasks │ │ │ │ └── main.yml │ │ └── deployer_install │ │ │ └── tasks │ │ │ └── main.yml │ ├── .gitignore │ ├── group_vars │ │ └── all.yml │ ├── key-data.yml │ ├── ansible.cfg │ ├── clean.yml │ └── playbook.yml ├── common │ ├── resolution.yml │ ├── examples.yml │ ├── library │ │ ├── ec2_vpc_facts.py │ │ ├── ec2_vpc_rtb_update.py │ │ └── ec2_remote_facts.py │ └── validate.yml ├── README.md └── jenkins.groovy ├── Jenkinsfile ├── filter_plugins └── ip_filters.py ├── README.md └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | **~ 2 | filter_plugins/*.pyc 3 | -------------------------------------------------------------------------------- /roles/opencontrail/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | -------------------------------------------------------------------------------- /test/ec2-k8s/localhost: -------------------------------------------------------------------------------- 1 | [hosts] 2 | localhost 3 | -------------------------------------------------------------------------------- /test/ec2-origin/localhost: -------------------------------------------------------------------------------- 1 | [hosts] 2 | localhost 3 | -------------------------------------------------------------------------------- /test/ec2-k8s/roles/basic/library: -------------------------------------------------------------------------------- 1 | ../../../common/library -------------------------------------------------------------------------------- /test/ec2-k8s/roles/clean/library: -------------------------------------------------------------------------------- 1 | ../../../common/library -------------------------------------------------------------------------------- /test/ec2-origin/roles/clean/library: -------------------------------------------------------------------------------- 1 | ../../../common/library -------------------------------------------------------------------------------- /test/ec2-origin/roles/cluster/library: -------------------------------------------------------------------------------- 1 | ../../../common/library -------------------------------------------------------------------------------- /test/ec2-origin/.gitignore: -------------------------------------------------------------------------------- 1 | cluster.status 2 | inventory.cluster 3 | -------------------------------------------------------------------------------- /test/ec2-k8s/.gitignore: -------------------------------------------------------------------------------- 1 | cluster.status 2 | inventory.cluster 3 | vpc.status 4 | -------------------------------------------------------------------------------- /test/ec2-origin/roles/workspace/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | path_src: /home/centos/src 3 | -------------------------------------------------------------------------------- /roles/opencontrail/files/vrouter.conf: -------------------------------------------------------------------------------- 1 | options vrouter vr_flow_entries=65536 vr_bridge_entries=1024 -------------------------------------------------------------------------------- /roles/opencontrail/templates/ifcfg-gateway.j2: -------------------------------------------------------------------------------- 1 | DEVICE={{ item }} 2 | TYPE=vhost 3 | MACADDR=00:00:5e:00:01:00 4 | -------------------------------------------------------------------------------- /test/ec2-origin/roles/workspace/files/ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | host_key_checking = False 3 | pipelining = True 4 | -------------------------------------------------------------------------------- /test/ec2-origin/group_vars/all.yml: -------------------------------------------------------------------------------- 1 | --- 2 | aws_region: us-west-1 3 | ec2_image: ami-f77fbeb3 4 | ssh_user: centos 5 | 6 | -------------------------------------------------------------------------------- /roles/opencontrail/templates/route-gateway0.j2: -------------------------------------------------------------------------------- 1 | {{ opencontrail_public_subnet }} dev gateway0 src {{ opencontrail_host_address }} -------------------------------------------------------------------------------- /roles/opencontrail_provision/files/namespace.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: opencontrail 5 | -------------------------------------------------------------------------------- /test/ec2-origin/roles/dns_forwarder/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart unbound 3 | service: name=unbound state=restarted 4 | -------------------------------------------------------------------------------- /roles/opencontrail/templates/ifcfg-eth.j2: -------------------------------------------------------------------------------- 1 | DEVICE={{ opencontrail_host_interface }} 2 | BOOTPROTO=none 3 | ONBOOT=yes 4 | IPADDR=0.0.0.0 5 | -------------------------------------------------------------------------------- /roles/opencontrail/templates/route-gateway1.j2: -------------------------------------------------------------------------------- 1 | {{ opencontrail_all_service_addresses }} dev gateway1 src {{ opencontrail_host_address }} 2 | -------------------------------------------------------------------------------- /test/ec2-origin/key-data.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | connection: local 4 | gather_facts: False 5 | roles: 6 | - key_data 7 | 8 | -------------------------------------------------------------------------------- /test/ec2-k8s/ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | host_key_checking = False 3 | pipelining = True 4 | 5 | [ssh_connection] 6 | control_path = %(directory)s/%%h-%%r 7 | -------------------------------------------------------------------------------- /roles/opencontrail_facts/vars/kubernetes.yml: -------------------------------------------------------------------------------- 1 | --- 2 | opencontrail_all_service_addresses: "{{ kube_service_addresses }}" 3 | opencontrail_kube_master_port: 443 4 | -------------------------------------------------------------------------------- /test/ec2-origin/ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | host_key_checking = False 3 | pipelining = True 4 | 5 | [ssh_connection] 6 | control_path = %(directory)s/%%h-%%r 7 | -------------------------------------------------------------------------------- /roles/opencontrail_provision/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | opencontrail_config_dns_forwarder: "{{ opencontrail_dns_forwarder | default(opencontrail_master_host_address) }}" 3 | -------------------------------------------------------------------------------- /roles/opencontrail/tasks/nodes-redhat.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Static routes 3 | template: src=node-route-redhat.j2 dest=/etc/sysconfig/network-scripts/route-vhost0 4 | 5 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | def tests 2 | 3 | node { 4 | checkout scm 5 | def script = load 'test/jenkins.groovy' 6 | tests = script.getTestMatrix() 7 | } 8 | 9 | parallel tests 10 | -------------------------------------------------------------------------------- /roles/opencontrail/templates/contrail-api.conf.j2: -------------------------------------------------------------------------------- 1 | [DEFAULTS] 2 | {% if opencontrail_master_ifmap_port is defined %} 3 | ifmap_server_port = {{ opencontrail_master_ifmap_port }} 4 | {% endif %} -------------------------------------------------------------------------------- /test/ec2-origin/roles/key_data/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: fetch registry certificate 3 | fetch: src=/var/lib/docker-registry/cert.crt dest=registry.crt fail_on_missing=yes flat=yes 4 | -------------------------------------------------------------------------------- /roles/opencontrail/templates/contrail-schema.conf.j2: -------------------------------------------------------------------------------- 1 | [DEFAULTS] 2 | {% if opencontrail_master_ifmap_port is defined %} 3 | ifmap_server_port = {{ opencontrail_master_ifmap_port }} 4 | {% endif %} -------------------------------------------------------------------------------- /test/ec2-k8s/roles/workspace/files/ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | host_key_checking = False 3 | pipelining=True 4 | 5 | [ssh_connection] 6 | ssh_args = -o ControlMaster=no -o ControlPersist=60s 7 | -------------------------------------------------------------------------------- /roles/opencontrail/templates/node-route-redhat.j2: -------------------------------------------------------------------------------- 1 | {% if 'gateways' in groups %} 2 | {{ opencontrail_all_service_addresses }} via {{ hostvars[groups['gateways'][0]]['opencontrail_host_address'] }} 3 | {% endif %} -------------------------------------------------------------------------------- /roles/opencontrail/templates/interface.cfg.j2: -------------------------------------------------------------------------------- 1 | auto {{ opencontrail_host_interface }} 2 | iface {{ opencontrail_host_interface }} inet static 3 | address 0.0.0.0 4 | up ip link set $IFACE up 5 | down ip link set $IFACE down 6 | -------------------------------------------------------------------------------- /test/ec2-origin/clean.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Executed with the inventory created by the playbook. 3 | # ansible-playbook -i cluster.status clean.yml 4 | # 5 | - hosts: localhost 6 | connection: local 7 | gather_facts: False 8 | roles: 9 | - clean 10 | -------------------------------------------------------------------------------- /test/ec2-k8s/clean.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Executed with the inventory created by the playbook. 3 | # ansible-playbook -i cluster.status clean.yml 4 | # 5 | - hosts: localhost 6 | connection: local 7 | gather_facts: False 8 | roles: 9 | - clean 10 | 11 | -------------------------------------------------------------------------------- /roles/opencontrail_facts/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # host variables 3 | opencontrail_host_interface: "{{ opencontrail_interface | default('eth0') }}" 4 | opencontrail_host_use_vrouter: inventory_hostname in groups['nodes'] or ('gateways' in groups and inventory_hostname in groups['gateways']) 5 | -------------------------------------------------------------------------------- /roles/opencontrail/templates/ifcfg-vhost0.j2: -------------------------------------------------------------------------------- 1 | DEVICE=vhost0 2 | BOOTPROTO=none 3 | DEVICETYPE=vhost 4 | IPADDR="{{ opencontrail_host_ipaddr }}" 5 | PHYSDEV="{{ opencontrail_host_interface }}" 6 | {% if opencontrail_host_gateway -%} 7 | GATEWAY={{ opencontrail_host_gateway }} 8 | {% endif -%} 9 | -------------------------------------------------------------------------------- /roles/opencontrail/vars/kubernetes.yml: -------------------------------------------------------------------------------- 1 | --- 2 | opencontrail_master_kube_service: kube-apiserver 3 | opencontrail_use_systemd: False 4 | opencontrail_all_kube_config_dir: /etc/kubernetes 5 | opencontrail_all_kube_manifest_dir: /etc/kubernetes/manifests 6 | opencontrail_cluster_services_namespace: kube-system 7 | -------------------------------------------------------------------------------- /roles/opencontrail/tasks/docker_config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Disable usage of iptables by docker 3 | lineinfile: 4 | dest: /etc/sysconfig/docker-network 5 | regexp: "^DOCKER_NETWORK_OPTIONS=" 6 | line: "DOCKER_NETWORK_OPTIONS=--iptables=false --ip-masq=false" 7 | when: ansible_os_family == "RedHat" 8 | -------------------------------------------------------------------------------- /test/ec2-k8s/group_vars/all.yml: -------------------------------------------------------------------------------- 1 | --- 2 | aws_region: us-west-1 3 | 4 | # username for ssh into remote VMs 5 | ssh_user: ubuntu 6 | # path used to store k8s binaries in management host 7 | path_tmp: /home/ubuntu/tmp 8 | # path used to store ansible playbook in management host 9 | path_src: /home/ubuntu/src 10 | 11 | 12 | -------------------------------------------------------------------------------- /roles/opencontrail/templates/contrail-control.conf.j2: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | hostip={{ opencontrail_host_address }} 3 | hostname={{ ansible_hostname }} 4 | 5 | [IFMAP] 6 | server_url=https://localhost:{{ opencontrail_master_ifmap_port if opencontrail_master_ifmap_port is defined else 8443 }} 7 | user=control-node 8 | password=control-node 9 | -------------------------------------------------------------------------------- /filter_plugins/ip_filters.py: -------------------------------------------------------------------------------- 1 | import netaddr 2 | 3 | class FilterModule(object): 4 | ''' Custom ansible filter ''' 5 | 6 | @staticmethod 7 | def netmask2prefixlen(data): 8 | net = netaddr.IPNetwork('0.0.0.0/%s' % data) 9 | return net.prefixlen 10 | 11 | def filters(self): 12 | return { 13 | "netmask2prefixlen": self.netmask2prefixlen 14 | } -------------------------------------------------------------------------------- /test/ec2-origin/roles/dns_forwarder/templates/unbound.conf.j2: -------------------------------------------------------------------------------- 1 | server: 2 | interface: 0.0.0.0 3 | access-control: 10.0.0.0/8 allow 4 | verbosity: 1 5 | 6 | forward-zone: 7 | name: "cluster.local." 8 | forward-addr: {{ hostvars[groups['masters'][0]]['instance_ip'] }} 9 | 10 | forward-zone: 11 | name: "." 12 | forward-addr: {{ aws_dns_resolver }} -------------------------------------------------------------------------------- /roles/opencontrail/tasks/vrouter-ubuntu.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Interface configuration file (physical) 3 | template: src=interface.cfg.j2 dest=/etc/network/interfaces.d/{{ opencontrail_host_interface }}.cfg 4 | 5 | - name: Interface configuration file (vhost0) 6 | template: src=vhost0.cfg.j2 dest=/etc/network/interfaces.d/vhost0.cfg 7 | 8 | - name: Enable vhost0 interface 9 | command: ifup vhost0 10 | 11 | -------------------------------------------------------------------------------- /roles/opencontrail/templates/contrail-vrouter-agent.conf.node.j2: -------------------------------------------------------------------------------- 1 | [CONTROL-NODE] 2 | server={{ opencontrail_master_host_addresses }} 3 | 4 | [VIRTUAL-HOST-INTERFACE] 5 | name=vhost0 6 | ip={{ opencontrail_host_ipaddr }} 7 | physical_interface={{ opencontrail_host_interface }} 8 | {% if opencontrail_host_gateway -%} 9 | gateway={{ opencontrail_host_gateway }} 10 | {% endif -%} 11 | 12 | [HYPERVISOR] 13 | type=kvm 14 | -------------------------------------------------------------------------------- /roles/opencontrail_facts/vars/openshift.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # The default value for portal_net is embeded in openshift_facts.py and is not accessible 3 | # unless overriden by the user configurable variable. 4 | opencontrail_all_service_addresses: "{{ hostvars[groups['masters'][0]]['openshift_master_portal_net'] if 'openshift_master_portal_net' in hostvars[groups['masters'][0]] else '172.30.0.0/16'}}" 5 | opencontrail_kube_master_port: 8443 6 | -------------------------------------------------------------------------------- /roles/opencontrail/tasks/kmod.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create directory 3 | file: path=/tmp/.ansible/build/{{ opencontrail_host_kernel_tag }} state=directory 4 | delegate_to: "{{ groups['masters'][0] }}" 5 | run_once: true 6 | 7 | - include: kmod-build-redhat.yml 8 | when: ansible_os_family == "RedHat" 9 | 10 | - include: kmod-build-ubuntu.yml 11 | when: ansible_distribution == "Ubuntu" 12 | 13 | - include: kmod-artifacts.yml 14 | -------------------------------------------------------------------------------- /test/common/resolution.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: address resolution 3 | hosts: all 4 | sudo: yes 5 | tasks: 6 | - name: Set hostname 7 | hostname: name="{{ inventory_hostname }}" 8 | 9 | - name: Add entries to /etc/hosts 10 | lineinfile: 11 | dest=/etc/hosts 12 | regexp="^{{ hostvars[item].ansible_ssh_host }}" 13 | line="{{ hostvars[item].ansible_ssh_host }} {{ item }}" 14 | with_items: groups['all'] 15 | -------------------------------------------------------------------------------- /test/ec2-k8s/roles/deployer_install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Update apt cache 3 | apt: update_cache=yes cache_valid_time=3600 4 | 5 | - name: Python requirements 6 | apt: name="{{ item }}" state=present 7 | with_items: 8 | - python-dev 9 | - python-pip 10 | - python-netaddr 11 | - python-markupsafe 12 | 13 | - name: Requires ansible 14 | pip: name=ansible state=present 15 | 16 | - name: Requires git 17 | apt: name=git state=present 18 | -------------------------------------------------------------------------------- /roles/opencontrail/templates/rabbitmq.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=RabbitMQ 3 | 4 | After=docker.service 5 | Requires=docker.service 6 | 7 | {% set service = { 8 | 'name': 'rabbitmq', 9 | 'image': 'rabbitmq:3.5.4', 10 | 'network_mode': 'host', 11 | 'env': ['RABBITMQ_ERLANG_COOKIE=opencontrail'] 12 | } 13 | -%} 14 | 15 | {% import 'systemd.service.j2' as systemd with context -%} 16 | {{ systemd.systemd_docker_service(service) }} 17 | 18 | [Install] 19 | WantedBy=multi-user.target 20 | -------------------------------------------------------------------------------- /roles/opencontrail_provision/tasks/nodes.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create virtual-router object 3 | command: > 4 | docker run --net=host opencontrail/config:2.20 /usr/share/contrail-utils/provision_vrouter.py 5 | --api_server_ip localhost 6 | --host_name "{{ ansible_hostname }}{% if hostvars[inventory_hostname]['ansible_domain'] != "" %}.{{ hostvars[inventory_hostname]['ansible_domain'] }}{% endif %}" 7 | --host_ip "{{ opencontrail_host_address }}" 8 | delegate_to: "{{ groups['masters'][0] }}" 9 | -------------------------------------------------------------------------------- /roles/opencontrail/vars/openshift.yml: -------------------------------------------------------------------------------- 1 | --- 2 | opencontrail_master_kube_service: "{{ openshift.common.service_type }}-master" 3 | opencontrail_use_systemd: true 4 | opencontrail_all_kube_config_dir: "{{ openshift.common.config_base }}" 5 | opencontrail_cluster_services_namespace: default 6 | opencontrail_gateway_public_subnet: "{{ hostvars[groups['masters'][0]]['opencontrail_public_subnet'] if 'opencontrail_public_subnet' in hostvars[groups['masters'][0]] else None }}" 7 | opencontrail_master_ifmap_port: 8444 8 | 9 | -------------------------------------------------------------------------------- /roles/opencontrail/tasks/masters.yml: -------------------------------------------------------------------------------- 1 | --- 2 | #- name: Set selinux permissive to enable directory mounts 3 | # selinux: state=permissive policy={{ ansible_selinux.type }} 4 | # when: ansible_selinux is defined and ansible_selinux.status == "enabled" 5 | 6 | - name: Assures manifests dir exists 7 | file: path="{{ opencontrail_all_kube_manifest_dir }}" state=directory 8 | when: not opencontrail_use_systemd 9 | 10 | - include: masters-services.yml 11 | 12 | - include: masters-config.yml 13 | 14 | - include: masters-control.yml 15 | -------------------------------------------------------------------------------- /roles/opencontrail/tasks/vrouter-redhat.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Interface up/down scripts 3 | copy: src={{ item }} dest=/etc/sysconfig/network-scripts mode=755 4 | with_items: 5 | - ifup-vhost 6 | - ifdown-vhost 7 | 8 | - name: Interface configuration file (physical) 9 | template: src=ifcfg-eth.j2 dest="/etc/sysconfig/network-scripts/ifcfg-{{ opencontrail_host_interface }}" 10 | 11 | - name: Interface configuration file (vhost0) 12 | template: src=ifcfg-vhost0.j2 dest=/etc/sysconfig/network-scripts/ifcfg-vhost0 13 | 14 | -------------------------------------------------------------------------------- /roles/opencontrail_provision/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Assert that input variables are defined. 4 | assert: 5 | that: 6 | - opencontrail_host_address is defined 7 | - opencontrail_all_service_addresses is defined 8 | 9 | - name: controller provisioning 10 | include: master.yml 11 | when: inventory_hostname == groups['masters'][0] 12 | 13 | - name: node provisioning 14 | include: nodes.yml 15 | when: inventory_hostname in groups['nodes'] or ('gateways' in groups and inventory_hostname in groups['gateways']) -------------------------------------------------------------------------------- /roles/opencontrail/templates/zookeeper.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Apache Zookeeper 3 | After=docker.service 4 | Requires=docker.service 5 | 6 | {% set service = { 7 | 'name': 'zookeeper', 8 | 'image': 'mesoscloud/zookeeper:3.4.6', 9 | 'network_mode': 'host', 10 | 'mounts': ['/var/lib/zookeeper:/var/lib/zookeeper:Z'], 11 | 'env': ['MYID=1', 'SERVERS=localhost'] 12 | } 13 | -%} 14 | 15 | {% import 'systemd.service.j2' as systemd with context -%} 16 | {{ systemd.systemd_docker_service(service) }} 17 | 18 | [Install] 19 | WantedBy=multi-user.target 20 | -------------------------------------------------------------------------------- /test/ec2-origin/roles/dns_forwarder/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install unbound package 3 | yum: name=unbound state=present 4 | 5 | - name: read nameserver configuration 6 | command: awk '/nameserver/ { print $2; }' /etc/resolv.conf 7 | register: _resolv_ns 8 | 9 | - set_fact: 10 | aws_dns_resolver: "{{ _resolv_ns.stdout }}" 11 | 12 | - name: unbound configuration file 13 | template: src=unbound.conf.j2 dest=/etc/unbound/unbound.conf 14 | notify: 15 | - restart unbound 16 | 17 | - name: start unbound 18 | service: name=unbound state=started -------------------------------------------------------------------------------- /test/ec2-origin/playbook.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | connection: local 4 | gather_facts: False 5 | roles: 6 | - cluster 7 | tags: 8 | - cluster 9 | 10 | - hosts: jenkins-master 11 | gather_facts: False 12 | roles: 13 | - key_data 14 | tags: 15 | - key-data 16 | 17 | - hosts: deployer 18 | sudo: yes 19 | gather_facts: False 20 | roles: 21 | - deployer_install 22 | - dns_forwarder 23 | tags: 24 | - deployer-install 25 | 26 | - hosts: deployer 27 | gather_facts: False 28 | roles: 29 | - workspace 30 | tags: 31 | - workspace 32 | -------------------------------------------------------------------------------- /roles/opencontrail/templates/ifmap-server.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=OpenContrail ifmap-server 3 | 4 | After=docker.service 5 | Requires=docker.service 6 | 7 | {% set service = { 8 | 'name': 'ifmap-server', 9 | 'image': 'opencontrail/ifmap-server:2.20', 10 | 'network_mode': 'host', 11 | 'env': ['IFMAP_BASIC_PORT=8444', 'IFMAP_CERT_PORT=8445'], 12 | 'mounts': ['/var/log/contrail:/var/log/contrail:z'] 13 | } 14 | -%} 15 | 16 | {% import 'systemd.service.j2' as systemd with context -%} 17 | {{ systemd.systemd_docker_service(service) }} 18 | 19 | [Install] 20 | WantedBy=multi-user.target 21 | -------------------------------------------------------------------------------- /test/ec2-origin/roles/workspace/files/systemd_workaround.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: 3 | - masters 4 | gather_facts: false 5 | sudo: yes 6 | tasks: 7 | - name: Ensure services are running 8 | service: name="{{ item }}" state=running 9 | with_items: 10 | - kube-network-manager 11 | - contrail-api 12 | - contrail-schema 13 | - contrail-control 14 | - ifmap-server 15 | 16 | - hosts: 17 | - nodes 18 | gather_facts: false 19 | sudo: yes 20 | tasks: 21 | - name: Ensure agent is running 22 | service: name=contrail-vrouter-agent state=running 23 | -------------------------------------------------------------------------------- /test/ec2-k8s/playbook.yml: -------------------------------------------------------------------------------- 1 | # 2 | # This playbook expects AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to be 3 | # defined. 4 | # The AIM user needs priviledges to manage VPCs and EC2 instances. 5 | # 6 | - hosts: localhost 7 | connection: local 8 | gather_facts: False 9 | roles: 10 | - basic 11 | tags: 12 | - create 13 | vars: 14 | # trusty hvm:ebs us-west-1 15 | ec2_image: ami-f898f698 16 | 17 | - hosts: deployer 18 | sudo: yes 19 | roles: 20 | - deployer_install 21 | tags: 22 | - deployer-install 23 | 24 | - hosts: deployer 25 | roles: 26 | - workspace 27 | tags: 28 | - workspace 29 | -------------------------------------------------------------------------------- /roles/opencontrail/tasks/gateway-redhat.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - set_fact: 3 | interface_list: ['gateway1'] 4 | 5 | - set_fact: 6 | interface_list: ['gateway0', 'gateway1'] 7 | when: opencontrail_public_subnet is defined 8 | 9 | - name: Interface configuration file 10 | template: src=ifcfg-gateway.j2 dest=/etc/sysconfig/network-scripts/ifcfg-{{ item }} 11 | with_items: interface_list 12 | 13 | - name: Static routes 14 | template: src=route-{{ item }}.j2 dest=/etc/sysconfig/network-scripts/route-{{ item }} 15 | with_items: interface_list 16 | 17 | - name: Activate gateway interfaces 18 | command: ifup {{ item }} 19 | with_items: interface_list 20 | 21 | -------------------------------------------------------------------------------- /roles/opencontrail/templates/kube-network.conf.j2: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | {% if openshift is defined -%} 3 | kubeconfig = /etc/kubernetes/master/openshift-master.kubeconfig 4 | {% endif -%} 5 | service-cluster-ip-range = {{ opencontrail_all_service_addresses }} 6 | 7 | [opencontrail] 8 | public-ip-range = {{ opencontrail_public_subnet }} 9 | {% if opencontrail_private_subnet is defined %} 10 | private-ip-range = {{ opencontrail_private_subnet }} 11 | {% endif %} 12 | cluster-service = {{ opencontrail_cluster_services_namespace }}/default 13 | {% if openshift is defined -%} 14 | network-label = opencontrail.org/name 15 | service-label = opencontrail.org/services 16 | {% endif -%} -------------------------------------------------------------------------------- /roles/opencontrail/files/contrail-vrouter-agent.upstart: -------------------------------------------------------------------------------- 1 | description "OpenContrail VRouter agent" 2 | 3 | start on started docker.io 4 | stop on runlevel [!2345] 5 | 6 | respawn 7 | 8 | pre-start script 9 | /usr/bin/docker pull opencontrail/vrouter-agent:2.20 10 | /usr/bin/docker rm vrouter-agent || echo "ignore error" 11 | end script 12 | 13 | post-stop script 14 | /usr/bin/docker kill vrouter-agent 15 | end script 16 | 17 | script 18 | /usr/bin/docker run --name vrouter-agent --privileged=true --net=host -v /etc/contrail:/etc/contrail -v /var/log/contrail:/var/log/contrail opencontrail/vrouter-agent:2.20 /usr/bin/contrail-vrouter-agent 19 | end script 20 | -------------------------------------------------------------------------------- /roles/opencontrail/templates/contrail-vrouter-agent.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=OpenContrail VRouter agent 3 | After=docker.service 4 | Requires=docker.service 5 | 6 | {% set service = { 7 | 'name': 'vrouter-agent', 8 | 'image': 'opencontrail/vrouter-agent:2.20', 9 | 'network_mode': 'host', 10 | 'mounts': ['/etc/contrail:/etc/contrail:z', '/var/log/contrail:/var/log/contrail:z'], 11 | 'extra_opts': '--privileged=true', 12 | 'command': '/usr/bin/contrail-vrouter-agent' 13 | } 14 | -%} 15 | 16 | {% import 'systemd.service.j2' as systemd with context -%} 17 | {{ systemd.systemd_docker_service(service) }} 18 | 19 | [Install] 20 | WantedBy=multi-user.target 21 | -------------------------------------------------------------------------------- /roles/opencontrail/templates/contrail-control.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=OpenContrail control-node 3 | 4 | After=ifmap-server.service 5 | Requires=ifmap-server.service 6 | 7 | {% set service = { 8 | 'name': 'contrail-control', 9 | 'image': 'opencontrail/control:2.20', 10 | 'network_mode': 'host', 11 | 'mounts': ['/etc/contrail:/etc/contrail:z', '/var/log/contrail:/var/log/contrail:z'], 12 | 'extra_opts': '--cap-add=NET_BIND_SERVICE', 13 | 'command': '/usr/bin/contrail-control' 14 | } 15 | -%} 16 | 17 | {% import 'systemd.service.j2' as systemd with context -%} 18 | {{ systemd.systemd_docker_service(service) }} 19 | 20 | [Install] 21 | WantedBy=multi-user.target 22 | -------------------------------------------------------------------------------- /test/ec2-origin/roles/deployer_install/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: epel 7 3 | yum: name="http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm" state=present 4 | 5 | - name: Python requirements 6 | yum: name="{{ item }}" state=present 7 | with_items: 8 | - gcc 9 | - openssl-devel 10 | - python-devel 11 | - python-dns 12 | - python-pip 13 | - python-netaddr 14 | - python-markupsafe 15 | - pyOpenSSL 16 | - util-linux 17 | - libffi-devel 18 | - python-cryptography 19 | 20 | - name: Requires ansible 21 | pip: name=ansible state=present version=1.9.4 22 | 23 | - name: Requires git 24 | yum: name=git state=present 25 | -------------------------------------------------------------------------------- /roles/opencontrail/tasks/gateway-ubuntu.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Interface configuration file 3 | template: src=gateway.cfg.j2 dest=/etc/network/interfaces.d/{{ item }}.cfg 4 | with_items: 5 | - gateway0 6 | - gateway1 7 | 8 | - name: Activate gateway0 9 | command: ifup gateway0 10 | 11 | - name: Activate gateway1 12 | command: ifup gateway1 13 | 14 | - name: Install upstart configuration file 15 | copy: 16 | src: contrail-vrouter-agent.upstart 17 | dest: /etc/init/contrail-vrouter-agent.conf 18 | mode: 0644 19 | 20 | - name: Create bridge sysvinit via upstart-job 21 | file: 22 | src: /lib/init/upstart-job 23 | dest: /etc/init.d/contrail-vrouter-agent 24 | state: link 25 | -------------------------------------------------------------------------------- /roles/opencontrail/tasks/iptables.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Check whether iptables are configured 3 | command: iptables -L OS_FIREWALL_ALLOW 4 | register: iptables_chain 5 | ignore_errors: True 6 | 7 | - name: Remove openshift iptables rules 8 | command: iptables -F OS_FIREWALL_ALLOW 9 | when: iptables_chain is defined and iptables_chain.rc == 0 10 | ignore_errors: True 11 | 12 | 13 | - name: Remove jump rule 14 | command: iptables -D INPUT -p all -j OS_FIREWALL_ALLOW 15 | when: iptables_chain is defined and iptables_chain.rc == 0 16 | ignore_errors: True 17 | 18 | - name: Delete openshift chain 19 | command: iptables -X OS_FIREWALL_ALLOW 20 | when: iptables_chain is defined and iptables_chain.rc == 0 21 | -------------------------------------------------------------------------------- /roles/opencontrail/templates/kube-network-manager.manifest.j2: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "v1", 3 | "kind": "Pod", 4 | "metadata": { 5 | "namespace": "opencontrail", 6 | "name": "kube-network-manager" 7 | }, 8 | "spec":{ 9 | "hostNetwork": true, 10 | "containers":[{ 11 | "name": "kube-network-manager", 12 | "image": "opencontrail/kube-network-manager{% if opencontrail_kube_release is defined -%}:{{ opencontrail_kube_release }}{% endif -%}", 13 | "volumeMounts": [{ 14 | "name": "config", 15 | "mountPath": "/etc/kubernetes" 16 | }] 17 | }], 18 | "volumes": [{ 19 | "name": "config", 20 | "hostPath": {"path": "/etc/kubernetes"} 21 | }] 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /roles/opencontrail/templates/cassandra.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=OpenContrail Cassandra database 3 | After=docker.service 4 | Requires=docker.service 5 | 6 | {% set service = { 7 | 'name': 'opencontrail-config-db', 8 | 'image': 'cassandra:2.2.0', 9 | 'network_mode': 'host', 10 | 'mounts': ['/var/lib/cassandra:/var/lib/cassandra:Z'], 11 | 'env': ['CASSANDRA_CLUSTER_NAME=OpenContrail-config'], 12 | 'command': "/bin/sh -c \"sed -ri 's/^(start_rpc:) .*/\\1 true/' /etc/cassandra/cassandra.yaml && /docker-entrypoint.sh cassandra -f\"" 13 | } 14 | -%} 15 | 16 | {% import 'systemd.service.j2' as systemd with context -%} 17 | {{ systemd.systemd_docker_service(service) }} 18 | 19 | [Install] 20 | WantedBy=multi-user.target 21 | -------------------------------------------------------------------------------- /roles/opencontrail/templates/kube-network-manager.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes OpenContrail plugin 3 | 4 | After=contrail-api.service {{ opencontrail_master_kube_service }}.service 5 | Requires=contrail-api.service {{ opencontrail_master_kube_service }}.service 6 | 7 | {% set service = { 8 | 'name': 'kube-network-manager', 9 | 'image': 'opencontrail/kube-network-manager:' + opencontrail_kube_release, 10 | 'network_mode': 'host', 11 | 'mounts': [ opencontrail_all_kube_config_dir + ':/etc/kubernetes:z'] 12 | } 13 | -%} 14 | 15 | {% import 'systemd.service.j2' as systemd with context -%} 16 | {{ systemd.systemd_docker_service(service) }} 17 | 18 | [Install] 19 | WantedBy={{ opencontrail_master_kube_service }}.service 20 | -------------------------------------------------------------------------------- /roles/opencontrail_facts/tasks/vhost_ansible_facts.yml: -------------------------------------------------------------------------------- 1 | # Determine the IP address configuration of the host from the vhost0 interface. 2 | --- 3 | - name: IP address information (vhost0) 4 | set_fact: 5 | opencontrail_host_address: "{{ ansible_vhost0.ipv4.address }}" 6 | opencontrail_host_netmask: "{{ ansible_vhost0.ipv4.netmask }}" 7 | opencontrail_host_gateway: "{{ ansible_default_ipv4.gateway if 'interface' in ansible_default_ipv4 and ansible_default_ipv4.interface == 'vhost0' else None}}" 8 | 9 | - set_fact: 10 | opencontrail_host_prefixlen: "{{ opencontrail_host_netmask | netmask2prefixlen }}" 11 | - set_fact: 12 | opencontrail_host_ipaddr: "{{ [opencontrail_host_address, opencontrail_host_prefixlen] | join('/') }}" 13 | -------------------------------------------------------------------------------- /roles/opencontrail/templates/contrail-schema.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=OpenContrail schema transformer 3 | 4 | After=contrail-api.service zookeeper.service cassandra.service 5 | Requires=contrail-api.service zookeeper.service cassandra.service 6 | 7 | {% set service = { 8 | 'name': 'contrail-schema', 9 | 'image': 'opencontrail/config:2.20', 10 | 'network_mode': 'host', 11 | 'mounts': ['/etc/contrail:/etc/contrail:z', '/var/log/contrail:/var/log/contrail:z'], 12 | 'command': '/usr/bin/contrail-schema --conf_file /etc/contrail/contrail-schema.conf' 13 | } 14 | -%} 15 | 16 | {% import 'systemd.service.j2' as systemd with context -%} 17 | {{ systemd.systemd_docker_service(service) }} 18 | 19 | [Install] 20 | WantedBy=multi-user.target 21 | -------------------------------------------------------------------------------- /roles/opencontrail/templates/contrail-api.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=OpenContrail api server 3 | 4 | After=cassandra.service zookeeper.service rabbitmq.service ifmap-server.service 5 | Requires=cassandra.service zookeeper.service rabbitmq.service ifmap-server.service 6 | 7 | {% set service = { 8 | 'name': 'contrail-api', 9 | 'image': 'opencontrail/config:2.20', 10 | 'network_mode': 'host', 11 | 'mounts': ['/etc/contrail:/etc/contrail:z', '/var/log/contrail:/var/log/contrail:z'], 12 | 'command': '/usr/bin/contrail-api --conf_file /etc/contrail/contrail-api.conf' 13 | } 14 | -%} 15 | 16 | {% import 'systemd.service.j2' as systemd with context -%} 17 | {{ systemd.systemd_docker_service(service) }} 18 | 19 | [Install] 20 | WantedBy=multi-user.target 21 | -------------------------------------------------------------------------------- /roles/opencontrail/files/ifmap-server.manifest: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "v1", 3 | "kind": "Pod", 4 | "metadata": { 5 | "namespace": "opencontrail", 6 | "name": "ifmap-server" 7 | }, 8 | "spec":{ 9 | "hostNetwork": true, 10 | "containers":[{ 11 | "name": "ifmap-server", 12 | "image": "opencontrail/ifmap-server:2.20", 13 | "ports": [{ 14 | "name": "ifmap", 15 | "containerPort": 8443, 16 | "hostPort": 8443 17 | }], 18 | "volumeMounts": [ 19 | { 20 | "name": "logs", 21 | "mountPath": "/var/log/contrail", 22 | "readOnly": false 23 | }] 24 | }], 25 | "volumes": [ 26 | { 27 | "name": "logs", 28 | "hostPath": {"path": "/var/log/ifmap-server"} 29 | } 30 | ] 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /roles/opencontrail/templates/gateway.cfg.j2: -------------------------------------------------------------------------------- 1 | auto {{ item }} 2 | iface {{ item }} inet static 3 | pre-up modprobe vrouter 4 | pre-up ip link add name {{ item }} type vhost 5 | pre-up ip link set {{ item }} address 00:00:5e:00:01:00 6 | {% if item == "gateway0" and opencontrail_public_subnet is defined %} 7 | up ip route add {{ opencontrail_public_subnet }} dev gateway0 src {{ opencontrail_host_address }} 8 | {% endif %} 9 | {% if item == "gateway1" %} 10 | up ip route add {{ opencontrail_all_service_addresses }} dev gateway1 src {{ opencontrail_host_address }} 11 | {% endif %} 12 | address 0.0.0.0 13 | post-down vif --list | awk '/^vif.*OS: {{ item }}/ {split($1, arr, "\/"); print arr[2];}' | xargs vif --delete 14 | post-down ip link delete {{ item }} 15 | 16 | -------------------------------------------------------------------------------- /roles/opencontrail/files/rabbitmq.manifest: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "v1", 3 | "kind": "Pod", 4 | "metadata": { 5 | "namespace": "opencontrail", 6 | "name": "rabbitmq" 7 | }, 8 | "spec":{ 9 | "hostNetwork": true, 10 | "containers":[{ 11 | "name": "rabbitmq", 12 | "image": "rabbitmq:3.5.4", 13 | "ports": [{ 14 | "name": "rabbitmq", 15 | "containerPort": 5672, 16 | "hostPort": 5672 17 | }], 18 | "env": [{ 19 | "name": "RABBITMQ_NODENAME", 20 | "value": "localhost" 21 | }], 22 | "volumeMounts": [{ 23 | "name": "logs", 24 | "mountPath": "/var/log/rabbitmq" 25 | }] 26 | }], 27 | "volumes": [ 28 | { 29 | "name": "logs", 30 | "hostPath": {"path": "/var/log/rabbitmq"} 31 | } 32 | ] 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /roles/opencontrail/files/ifdown-vhost: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | . /etc/init.d/functions 4 | 5 | cd /etc/sysconfig/network-scripts 6 | . ./network-functions 7 | 8 | [ -f ../network ] && . ../network 9 | 10 | CONFIG=${1} 11 | 12 | need_config "${CONFIG}" 13 | 14 | source_config 15 | 16 | ifindex=$(vif --list | awk '/^vif.*OS: vhost0/ {split($1, arr, "/"); print arr[2];}') 17 | if [ -n "${ifindex}" ]; then 18 | vif --delete ${ifindex} 19 | fi 20 | 21 | ifindex=$(vif --list | awk -v device=${PHYSDEV} '/^vif.*OS: / { if ($3 == device) {split($1, arr, "/"); print arr[2];}}') 22 | if [ -n "${ifindex}" ]; then 23 | vif --delete ${ifindex} 24 | fi 25 | 26 | if [ -d /sys/class/net/${DEVICE} ]; then 27 | ip link delete vhost0 28 | fi 29 | 30 | exec /etc/sysconfig/network-scripts/ifdown-post ${CONFIG} ${2} 31 | -------------------------------------------------------------------------------- /test/ec2-k8s/roles/workspace/files/cluster.patch: -------------------------------------------------------------------------------- 1 | diff --git a/ansible/cluster.yml b/ansible/cluster.yml 2 | index 838737a..95df925 100644 3 | --- a/ansible/cluster.yml 4 | +++ b/ansible/cluster.yml 5 | @@ -41,7 +41,10 @@ 6 | - hosts: all 7 | sudo: yes 8 | roles: 9 | + - { role: opencontrail_facts, when: networking == 'opencontrail'} 10 | - { role: opencontrail, when: networking == 'opencontrail'} 11 | + vars: 12 | + opencontrail_cluster_type: kubernetes 13 | tags: 14 | - network-service-install 15 | 16 | @@ -76,6 +79,6 @@ 17 | - nodes 18 | sudo: yes 19 | roles: 20 | - - { role: opencontrail-provision, when: networking == 'opencontrail' } 21 | + - { role: opencontrail_provision, when: networking == 'opencontrail' } 22 | tags: 23 | - network-service-config 24 | -------------------------------------------------------------------------------- /roles/opencontrail_facts/tasks/interface_inventory_facts.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: IP address information (from inventory / hostvars) 3 | set_fact: 4 | opencontrail_host_address: "{{ opencontrail_ipaddr | ipaddr('address') }}" 5 | opencontrail_host_netmask: "{{ opencontrail_ipaddr | ipaddr('netmask') }}" 6 | opencontrail_host_prefixlen: "{{ opencontrail_ipaddr | ipaddr('subnet') }}" 7 | opencontrail_host_ipaddr: "{{ opencontrail_ipaddr }}" 8 | 9 | - set_fact: 10 | opencontrail_host_gateway: "{{ opencontrail_gateway }}" 11 | when: opencontrail_gateway is defined 12 | 13 | - set_fact: 14 | opencontrail_host_gateway: "{{ ansible_default_ipv4.gateway if 'interface' in ansible_default_ipv4 and ansible_default_ipv4.interface == opencontrail_host_interface else None }}" 15 | when: opencontrail_gateway is not defined 16 | -------------------------------------------------------------------------------- /test/ec2-k8s/roles/clean/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Cleanup routes from the VPC routing table 2 | ec2_vpc_rtb_update: 3 | region: "{{ aws_region }}" 4 | vpc_id: "{{ ec2_vpc_id }}" 5 | state: absent 6 | subnets: 7 | - "{{ ec2_public_subnet_id }}" 8 | routes: 9 | - dest: "{{ opencontrail_public_subnet }}" 10 | - dest: "{{ opencontrail_service_addresses }}" 11 | 12 | - name: Instance id list 13 | set_fact: 14 | hosts: "{{ groups['all'] | difference(['localhost']) }}" 15 | - set_fact: 16 | instance_ids: "{% set x = [] %}{% for host in hosts %}{% set _ = x.append(hostvars[host]['id']) %}{% endfor %}{{ x }}" 17 | 18 | - name: Delete virtual machines 19 | ec2: 20 | state: absent 21 | region: "{{ aws_region }}" 22 | instance_ids: "{{ hostvars[item].id }}" 23 | with_items: "{{ hosts }}" 24 | -------------------------------------------------------------------------------- /test/ec2-k8s/roles/basic/templates/status.j2: -------------------------------------------------------------------------------- 1 | [all:vars] 2 | ec2_vpc_id = {{ ec2_vpc.id }} 3 | ec2_public_subnet_id = {{ ec2_public_subnet.id }} 4 | opencontrail_public_subnet = {{ k8s_public_subnet }} 5 | opencontrail_service_addresses = {{ k8s_service_addresses }} 6 | 7 | [deployer] 8 | {{ k8s_management.tagged_instances[0].public_dns_name }} id={{ k8s_management.tagged_instances[0].id }} ansible_ssh_user=ubuntu 9 | 10 | [masters] 11 | {% for instance in k8s_master.tagged_instances %} 12 | k8s-master-{{ loop.index }} id={{ instance.id }} 13 | {% endfor %} 14 | 15 | [gateways] 16 | {% for instance in k8s_gateway.tagged_instances %} 17 | k8s-gateway-{{ loop.index }} id={{ instance.id }} 18 | {% endfor %} 19 | 20 | [nodes] 21 | {% for instance in k8s_nodes.tagged_instances %} 22 | k8s-node-{{ loop.index }} id={{ instance.id }} 23 | {% endfor %} 24 | 25 | -------------------------------------------------------------------------------- /roles/opencontrail_facts/tasks/interface_ansible_facts.yml: -------------------------------------------------------------------------------- 1 | # Determine the IP address information from the physical interface. 2 | --- 3 | - name: IP address information (from physical interface) 4 | set_fact: 5 | opencontrail_host_address: "{{ hostvars[inventory_hostname]['ansible_' + opencontrail_host_interface]['ipv4']['address'] }}" 6 | opencontrail_host_netmask: "{{ hostvars[inventory_hostname]['ansible_' + opencontrail_host_interface]['ipv4']['netmask'] }}" 7 | opencontrail_host_gateway: "{{ ansible_default_ipv4.gateway if 'interface' in ansible_default_ipv4 and ansible_default_ipv4.interface == opencontrail_host_interface else None }}" 8 | 9 | - set_fact: 10 | opencontrail_host_prefixlen: "{{ opencontrail_host_netmask | netmask2prefixlen }}" 11 | - set_fact: 12 | opencontrail_host_ipaddr: "{{ [opencontrail_host_address, opencontrail_host_prefixlen] | join('/') }}" 13 | -------------------------------------------------------------------------------- /roles/opencontrail/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # global variables 3 | opencontrail_all_release: "{{ opencontrail_release | default('2.20') }}" 4 | 5 | # master variables 6 | opencontrail_master_use_systemd: "{{ opencontrail_use_systemd | default(true) }}" 7 | 8 | opencontrail_build_http_proxy: "{{ opencontrail_http_proxy | default(None) }}" 9 | 10 | # kernel variables 11 | opencontrail_host_kernel_tag: "{{ ansible_distribution | lower}}{{ansible_distribution_version}}-{{ansible_kernel}}" 12 | opencontrail_host_kernel_devel: "{{ ansible_kernel | regex_replace('([0-9.]+)-([0-9]+)[0-9.]*([a-z0-9]+).([a-z0-9_]+)', '\\\\1-\\\\2.\\\\3.\\\\4') }}" 13 | 14 | opencontrail_host_kernel_build_dir: "/tmp/.ansible/build/{{ opencontrail_host_kernel_tag }}" 15 | opencontrail_host_kernel_install_dir: "/tmp/.ansible/install/{{ opencontrail_host_kernel_tag }}" 16 | opencontrail_host_kernel_artifact_tar: "/tmp/.ansible/opencontrail-vrouter-{{ opencontrail_host_kernel_tag }}.tgz" -------------------------------------------------------------------------------- /roles/opencontrail/files/contrail-schema.manifest: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "v1", 3 | "kind": "Pod", 4 | "metadata": { 5 | "namespace": "opencontrail", 6 | "name": "contrail-schema" 7 | }, 8 | "spec":{ 9 | "hostNetwork": true, 10 | "containers":[{ 11 | "name": "contrail-schema", 12 | "image": "opencontrail/config:2.20", 13 | "command": ["/usr/bin/contrail-schema"], 14 | "ports": [{ 15 | "name": "schema-ui", 16 | "containerPort": 8087, 17 | "hostPort": 8087 18 | }], 19 | "volumeMounts": [{ 20 | "name": "config", 21 | "mountPath": "/etc/contrail" 22 | }, { 23 | "name": "logs", 24 | "mountPath": "/var/log/contrail", 25 | "readOnly": false 26 | }] 27 | }], 28 | "volumes": [ 29 | { 30 | "name": "config", 31 | "hostPath": {"path": "/etc/contrail"} 32 | }, 33 | { 34 | "name": "logs", 35 | "hostPath": {"path": "/var/log/contrail"} 36 | } 37 | ] 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /roles/opencontrail/files/contrail-api.manifest: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "v1", 3 | "kind": "Pod", 4 | "metadata": { 5 | "namespace": "opencontrail", 6 | "name": "contrail-api" 7 | }, 8 | "spec":{ 9 | "hostNetwork": true, 10 | "containers":[{ 11 | "name": "contrail-api", 12 | "image": "opencontrail/config:2.20", 13 | "command": ["/usr/bin/contrail-api"], 14 | "ports": [{ 15 | "name": "contrail-api", 16 | "containerPort": 8082, 17 | "hostPort": 8082 18 | }], 19 | "volumeMounts": [ 20 | { 21 | "name": "config", 22 | "mountPath": "/etc/contrail" 23 | }, 24 | { 25 | "name": "logs", 26 | "mountPath": "/var/log/contrail", 27 | "readOnly": false 28 | }] 29 | }], 30 | "volumes": [ 31 | { 32 | "name": "config", 33 | "hostPath": {"path": "/etc/contrail"} 34 | }, 35 | { 36 | "name": "logs", 37 | "hostPath": {"path": "/var/log/contrail"} 38 | } 39 | ] 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /test/ec2-origin/roles/cluster/templates/status.j2: -------------------------------------------------------------------------------- 1 | [all:vars] 2 | ec2_vpc_id = "{{ ec2_vpc.id }}" 3 | ec2_public_subnet_id = "{{ ec2_public_subnet.id }}" 4 | ec2_private_subnet_id = "{{ ec2_private_subnet.id }}" 5 | opencontrail_public_subnet = {{ cluster_public_subnet }} 6 | opencontrail_service_addresses = {{ cluster_service_addresses }} 7 | 8 | [deployer] 9 | {{ origin_deployer.tagged_instances[0].public_dns_name }} id={{ origin_deployer.tagged_instances[0].id }} ansible_ssh_user={{ ssh_user }} 10 | 11 | [masters] 12 | {% for instance in origin_master.tagged_instances %} 13 | origin-master-{{ loop.index }} id={{ instance.id }} instance_ip={{ instance.private_ip }} 14 | {% endfor %} 15 | 16 | [gateways] 17 | {% for instance in origin_gateway.tagged_instances %} 18 | origin-gateway-{{ loop.index }} id={{ instance.id }} 19 | {% endfor %} 20 | 21 | [nodes] 22 | {% for instance in origin_nodes.tagged_instances %} 23 | origin-node-{{ loop.index }} id={{ instance.id }} 24 | {% endfor %} 25 | 26 | -------------------------------------------------------------------------------- /roles/opencontrail/files/cassandra.manifest: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "v1", 3 | "kind": "Pod", 4 | "metadata": { 5 | "namespace": "opencontrail", 6 | "name": "cassandra" 7 | }, 8 | "spec":{ 9 | "containers":[{ 10 | "name": "opencontrail-config-db", 11 | "image": "cassandra:2.2.0", 12 | "command": [ 13 | "/bin/sh", 14 | "-c", 15 | "sed -ri 's/^(start_rpc:) .*/\\1 true/' /etc/cassandra/cassandra.yaml && /docker-entrypoint.sh cassandra -f" 16 | ], 17 | "ports": [{ 18 | "name": "cassandra", 19 | "containerPort": 9160, 20 | "hostPort": 9160 21 | }], 22 | "env": [{ 23 | "name": "CASSANDRA_CLUSTER_NAME", 24 | "value": "OpenContrail-config" 25 | }], 26 | "volumeMounts": [{ 27 | "name": "data", 28 | "mountPath": "/var/lib/cassandra", 29 | "readOnly": false 30 | }] 31 | }], 32 | "volumes": [ 33 | { 34 | "name": "data", 35 | "hostPath": {"path": "/var/lib/cassandra"} 36 | } 37 | ] 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /test/common/examples.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: fix kubelet schema.json permissins 3 | hosts: masters 4 | sudo: yes 5 | tasks: 6 | - file: path="/tmp/kubectl.schema/v1.1.1/api/v1/schema.json" mode=0644 state=file 7 | 8 | - name: run examples 9 | hosts: masters 10 | tasks: 11 | - file: path="~/guestbook" state=directory 12 | 13 | # https://raw.githubusercontent.com/kubernetes/kubernetes/master/examples/guestbook-go 14 | - name: download files 15 | get_url: url="https://raw.githubusercontent.com/pedro-r-marques/examples/master/k8s-guestbook/{{ item }}" dest="~/guestbook" 16 | with_items: 17 | - guestbook-namespace.json 18 | - guestbook-controller.json 19 | - guestbook-service.json 20 | - redis-master-controller.json 21 | - redis-master-service.json 22 | - redis-slave-controller.json 23 | - redis-slave-service.json 24 | run_once: true 25 | 26 | - name: Create example 27 | command: kubectl create -f "~/guestbook" 28 | run_once: true -------------------------------------------------------------------------------- /roles/opencontrail/files/zookeeper.manifest: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "v1", 3 | "kind": "Pod", 4 | "metadata": { 5 | "namespace": "opencontrail", 6 | "name": "zookeeper" 7 | }, 8 | "spec":{ 9 | "hostNetwork": true, 10 | "containers":[{ 11 | "name": "zookeeper", 12 | "image": "mesoscloud/zookeeper:3.4.6", 13 | "ports": [{ 14 | "name": "zookeeper", 15 | "containerPort": 2181, 16 | "hostPort": 2181 17 | }], 18 | "env": [{ 19 | "name": "MYID", 20 | "value": "1" 21 | }, { 22 | "name": "SERVERS", 23 | "value": "localhost" 24 | }], 25 | "volumeMounts": [{ 26 | "name": "logs", 27 | "mountPath": "/var/log/zookeeper", 28 | "readOnly": false 29 | }, { 30 | "name": "data", 31 | "mountPath": "/var/lib/zookeeper", 32 | "readOnly": false 33 | }] 34 | }], 35 | "volumes": [{ 36 | "name": "logs", 37 | "hostPath": {"path": "/var/log/zookeeper"} 38 | }, { 39 | "name": "data", 40 | "hostPath": {"path": "/var/lib/zookeeper"} 41 | }] 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /roles/opencontrail/templates/systemd.service.j2: -------------------------------------------------------------------------------- 1 | {% macro systemd_docker_service(svc) %} 2 | [Service] 3 | ExecStartPre=/usr/bin/docker pull {{ svc.image }} 4 | ExecStartPre=-/usr/bin/docker rm -f {{ svc.name }} 5 | ExecStart=/usr/bin/docker run --name {{ svc.name }} {% if 'network_mode' in svc %}--net={{ svc.network_mode }}{% endif %} {%- for var in svc.env|default([]) %} -e {{ var }}{% endfor %} {%- for mount in svc.mounts|default([]) %} -v {{mount}}{% endfor %} {% if 'extra_opts' in svc %}{{ svc.extra_opts }}{% endif %} {{ svc.image }} {% if 'command' in svc %}{{ svc.command }}{% endif %} 6 | 7 | ExecStop=/usr/bin/docker stop -t 10 {{ svc.name }} 8 | Restart=on-failure 9 | # Processes that do not register an handler for SIGTERM will be killed by 10 | # the "docker stop" command after the timeout. The docker client that performs 11 | # the "run" operation will then exit with code 137. Defining status 137 as 12 | # successful prevents the systemd unit from entering "failed state". 13 | SuccessExitStatus=0 137 14 | TimeoutStartSec=0 15 | TimeoutStopSec=1m 16 | {% endmacro %} -------------------------------------------------------------------------------- /roles/opencontrail/tasks/kmod-build-ubuntu.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: kmod Dockerfile 3 | template: 4 | src: Dockerfile.debian.j2 5 | dest: /tmp/.ansible/build/{{ opencontrail_host_kernel_tag }}/Dockerfile 6 | delegate_to: "{{ groups['masters'][0] }}" 7 | run_once: true 8 | 9 | - name: Build docker container 10 | command: docker build -t opencontrail/kmod_{{ opencontrail_host_kernel_tag }} {{ opencontrail_host_kernel_tag }} 11 | args: 12 | chdir: /tmp/.ansible/build 13 | delegate_to: "{{ groups['masters'][0] }}" 14 | run_once: true 15 | 16 | - name: Build kernel module 17 | command: docker run -v /tmp/.ansible/build/{{ opencontrail_host_kernel_tag }}:/src/vrouter/build opencontrail/kmod_{{ opencontrail_host_kernel_tag }} /bin/bash -c "USER=nobody scons --optimization=production --kernel-dir=/lib/modules/{{ ansible_kernel }}/build vrouter && cp vrouter/vrouter.ko build/kbuild" 18 | args: 19 | creates: /tmp/.ansible/build/{{ opencontrail_host_kernel_tag }}/kbuild/vrouter.ko 20 | delegate_to: "{{ groups['masters'][0] }}" 21 | run_once: true 22 | 23 | -------------------------------------------------------------------------------- /roles/opencontrail/templates/Dockerfile.debian.j2: -------------------------------------------------------------------------------- 1 | FROM {{ ansible_distribution | lower }}:{{ ansible_distribution_version }} 2 | {%if opencontrail_build_http_proxy %} 3 | RUN sh -c "echo 'Acquire::http::Proxy \"{{ opencontrail_build_http_proxy }}\";' >> /etc/apt/apt.conf.d/02proxy" 4 | {% endif %} 5 | RUN apt-get update 6 | RUN {{ ansible_pkg_mgr }} install -y git make 7 | {%if opencontrail_build_http_proxy %} 8 | RUN git config --global http.proxy {{ opencontrail_build_http_proxy }} 9 | RUN git config --global https.proxy {{ opencontrail_build_http_proxy }} 10 | {% endif %} 11 | RUN {{ ansible_pkg_mgr }} install -y automake flex bison gcc g++ scons linux-headers-{{ ansible_kernel }} libboost-dev libxml2-dev 12 | RUN mkdir -p src/vrouter 13 | WORKDIR src/vrouter 14 | RUN git clone -b R2.20 https://github.com/Juniper/contrail-vrouter vrouter 15 | RUN mkdir tools 16 | RUN (cd tools && git clone https://github.com/Juniper/contrail-build build) 17 | RUN (cd tools && git clone -b R2.20 https://github.com/Juniper/contrail-sandesh sandesh) 18 | RUN cp tools/build/SConstruct . 19 | 20 | -------------------------------------------------------------------------------- /roles/opencontrail/tasks/masters-control.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Configure control-node 3 | template: src=contrail-control.conf.j2 dest=/etc/contrail/contrail-control.conf 4 | notify: 5 | - restart contrail-control 6 | 7 | - name: Install control-node manifest 8 | copy: src=contrail-control.manifest dest="{{ opencontrail_all_kube_manifest_dir }}" 9 | when: not opencontrail_use_systemd 10 | 11 | - name: Install control-node service 12 | template: src=contrail-control.service.j2 dest=/etc/systemd/system/contrail-control.service 13 | notify: 14 | - reload systemd 15 | when: opencontrail_use_systemd 16 | 17 | - name: Docker image ids (control) 18 | command: docker images -q "{{ item }}" 19 | with_items: 20 | - "opencontrail/control:{{ opencontrail_all_release }}" 21 | register: image_ids 22 | always_run: true 23 | 24 | - name: Pull docker images (control) 25 | command: docker pull "{{ item.item }}" 26 | with_items: image_ids.results 27 | when: not item.stdout 28 | 29 | - name: Start control-node service 30 | service: name="contrail-control" enabled=yes state=started 31 | when: opencontrail_use_systemd 32 | 33 | -------------------------------------------------------------------------------- /roles/opencontrail/tasks/vrouter.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - file: path="~/.ansible/files" state=directory 3 | 4 | - name: Copy vrouter tarball 5 | copy: 6 | src: "/tmp/.ansible/artifacts/opencontrail-vrouter-{{ opencontrail_host_kernel_tag }}.tgz" 7 | dest: "~/.ansible/files/opencontrail-vrouter-{{ opencontrail_host_kernel_tag }}.tgz" 8 | 9 | - name: Extract vrouter module 10 | shell: "tar zxf ~/.ansible/files/opencontrail-vrouter-{{ opencontrail_host_kernel_tag }}.tgz --no-overwrite-dir --skip-old-files" 11 | args: 12 | chdir: / 13 | 14 | - name: Depmod 15 | command: depmod 16 | 17 | # Reduce the vrouter table sizes in order to be able to execute on a 2G VM. 18 | - name: Reduce memory utilization of vrouter 19 | copy: src=vrouter.conf dest=/etc/modprobe.d 20 | when: ansible_memtotal_mb|int < 4096 21 | 22 | - name: Redhat-style interface configuration 23 | include: vrouter-redhat.yml 24 | when: ansible_os_family == "RedHat" 25 | 26 | - name: Ubuntu interface configuration 27 | include: vrouter-ubuntu.yml 28 | when: ansible_distribution == "Ubuntu" 29 | 30 | - name: Ensure configuration directory exists 31 | file: path=/etc/contrail state=directory 32 | -------------------------------------------------------------------------------- /test/ec2-k8s/roles/basic/templates/inventory.j2: -------------------------------------------------------------------------------- 1 | [opencontrail:children] 2 | masters 3 | nodes 4 | gateways 5 | 6 | [opencontrail:vars] 7 | localBuildOutput={{ path_tmp }}/kubernetes/server/bin 8 | opencontrail_public_subnet={{ k8s_public_subnet }} 9 | opencontrail_private_subnet=10.32.0.0/16 10 | opencontrail_kube_release=1.1 11 | kube_service_addresses={{ k8s_service_addresses }} 12 | 13 | [masters] 14 | {% for instance in k8s_master.tagged_instances %} 15 | k8s-master-{{ loop.index }} ansible_ssh_host={{ instance.private_ip }} ansible_ssh_user={{ ssh_user }} 16 | {% endfor %} 17 | 18 | [etcd] 19 | {% for instance in k8s_master.tagged_instances %} 20 | k8s-master-{{ loop.index }} ansible_ssh_host={{ instance.private_ip }} ansible_ssh_user={{ ssh_user }} 21 | {% endfor %} 22 | 23 | [gateways] 24 | {% for instance in k8s_gateway.tagged_instances %} 25 | k8s-gateway-{{ loop.index }} ansible_ssh_host={{ instance.private_ip }} ansible_ssh_user={{ ssh_user }} 26 | {% endfor %} 27 | 28 | [nodes] 29 | {% for instance in k8s_nodes.tagged_instances %} 30 | k8s-node-{{ loop.index }} ansible_ssh_host={{ instance.private_ip }} ansible_ssh_user={{ ssh_user }} 31 | {% endfor %} 32 | 33 | -------------------------------------------------------------------------------- /roles/opencontrail/templates/contrail-vrouter-agent.conf.gateway.j2: -------------------------------------------------------------------------------- 1 | [CONTROL-NODE] 2 | server={{ opencontrail_master_host_addresses }} 3 | 4 | 5 | [VIRTUAL-HOST-INTERFACE] 6 | name=vhost0 7 | ip={{ opencontrail_host_ipaddr }} 8 | physical_interface={{ opencontrail_host_interface }} 9 | {% if opencontrail_host_gateway -%} 10 | gateway={{ opencontrail_host_gateway }} 11 | {% endif -%} 12 | 13 | {% if opencontrail_public_subnet is defined %} 14 | [GATEWAY-0] 15 | routing_instance=default-domain:default-project:Public:Public 16 | interface=gateway0 17 | ip_blocks={{ opencontrail_public_subnet }} 18 | {% endif %} 19 | 20 | [GATEWAY-1] 21 | routing_instance=default-domain:{{ opencontrail_cluster_services_namespace }}:service-default:service-default 22 | interface=gateway1 23 | ip_blocks={{ opencontrail_all_service_addresses }} 24 | routes={{ opencontrail_host_ipaddr }} {% if 'address' in ansible_default_ipv4 and ansible_default_ipv4.address != opencontrail_host_address -%}{{ [ansible_default_ipv4.address, ansible_default_ipv4.netmask | netmask2prefixlen ] | join('/') }}{% endif -%} {% if opencontail_gateway_extra_infra_prefixes is defined -%}{{ opencontail_gateway_extra_infra_prefixes }}{% endif -%} 25 | -------------------------------------------------------------------------------- /roles/opencontrail/files/contrail-control.manifest: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "v1", 3 | "kind": "Pod", 4 | "metadata": { 5 | "namespace": "opencontrail", 6 | "name": "contrail-control" 7 | }, 8 | "spec":{ 9 | "hostNetwork": true, 10 | "containers":[{ 11 | "name": "contrail-control", 12 | "image": "opencontrail/control:2.20", 13 | "command": ["/usr/bin/contrail-control"], 14 | "securityContext": { 15 | "capabilities": { 16 | "add": ["NET_BIND_SERVICE"] 17 | } 18 | }, 19 | "ports": [{ 20 | "name": "bgp", 21 | "containerPort": 179, 22 | "hostPort": 179 23 | }, { 24 | "name": "xmpp", 25 | "containerPort": 5269, 26 | "hostPort": 5269 27 | }, { 28 | "name": "control-ui", 29 | "containerPort": 8083, 30 | "hostPort": 8083 31 | }], 32 | "volumeMounts": [ 33 | { 34 | "name": "config", 35 | "mountPath": "/etc/contrail" 36 | }, 37 | { 38 | "name": "logs", 39 | "mountPath": "/var/log/contrail", 40 | "readOnly": false 41 | }] 42 | }], 43 | "volumes": [{ 44 | "name": "config", 45 | "hostPath": {"path": "/etc/contrail"} 46 | },{ 47 | "name": "logs", 48 | "hostPath": {"path": "/var/log/contrail"} 49 | }] 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /roles/opencontrail/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # The kubernetes role invokes restart daemons. It must be defined here 3 | # since the opencontrail playbook depends on that role to setup basic 4 | # variables such as the kubernetes configuration file directory. 5 | - name: restart daemons 6 | command: /bin/true 7 | 8 | - name: reload systemd 9 | command: systemctl --system daemon-reload 10 | 11 | - name: restart contrail-vrouter-agent 12 | service: name=contrail-vrouter-agent state=restarted 13 | 14 | - name: restart contrail-control 15 | shell: docker ps | awk '/opencontrail\/control/ {print $1;}' 16 | register: control_node_id 17 | notify: 18 | - docker restart contrail-control 19 | 20 | - name: docker restart contrail-control 21 | command: docker restart {{ control_node_id.stdout }} 22 | when: control_node_id.rc == 0 and control_node_id.stdout != "" 23 | 24 | - name: restart kube-network-manager 25 | shell: docker ps | awk '/opencontrail\/kube-network-manager/ {print $1;}' 26 | register: network_manager_id 27 | notify: 28 | - docker restart kube-network-manager 29 | 30 | - name: docker restart kube-network-manager 31 | command: docker restart {{ network_manager_id.stdout }} 32 | when: network_manager_id.rc == 0 and network_manager_id.stdout != "" 33 | -------------------------------------------------------------------------------- /roles/opencontrail/files/contrail-vrouter-agent.manifest: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "v1", 3 | "kind": "Pod", 4 | "metadata": { 5 | "namespace": "opencontrail", 6 | "name":"contrail-vrouter-agent" 7 | }, 8 | "spec":{ 9 | "hostNetwork": true, 10 | "containers":[{ 11 | "name": "contrail-vrouter-agent", 12 | "image": "opencontrail/vrouter-agent:2.20", 13 | "command": ["/usr/bin/contrail-vrouter-agent"], 14 | "securityContext": { 15 | "privileged": true 16 | }, 17 | "ports": [{ 18 | "name": "vrouter-api", 19 | "containerPort": 9090, 20 | "hostPort": 9090 21 | }, { 22 | "name": "ipc", 23 | "containerPort": 9091, 24 | "hostPort":9091 25 | }, { 26 | "name": "control-ui", 27 | "containerPort": 8085, 28 | "hostPort": 8085 29 | }], 30 | "volumeMounts": [ 31 | { 32 | "name": "config", 33 | "mountPath": "/etc/contrail" 34 | }, 35 | { 36 | "name": "logs", 37 | "mountPath": "/var/log/contrail", 38 | "readOnly": false 39 | } 40 | ] 41 | }], 42 | "volumes": [{ 43 | "name": "config", 44 | "hostPath": {"path": "/etc/contrail"} 45 | },{ 46 | "name": "logs", 47 | "hostPath": {"path": "/var/log/contrail"} 48 | } 49 | ] 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /test/ec2-origin/roles/workspace/files/applications.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: 3 | - masters 4 | sudo: yes 5 | tasks: 6 | - name: Ensure patch is installed 7 | yum: name=patch state=present 8 | 9 | - hosts: 10 | - masters 11 | tasks: 12 | - name: List projects 13 | command: oc get projects 14 | register: _project_list 15 | 16 | - name: Create test project 17 | command: oc new-project test --description="Test project" 18 | when: "'test' not in _project_list.stdout" 19 | 20 | - file: path="{{ tmp_dir }}" state=directory 21 | 22 | - name: Copy example file 23 | command: cp /usr/share/openshift/examples/quickstart-templates/rails-postgresql.json "{{ tmp_dir }}" 24 | 25 | - name: Copy rails-postgresql.patch 26 | template: src=rails-postgresql.patch.j2 dest="{{ tmp_dir }}/rails-postgresql.patch" 27 | 28 | - name: Patch example 29 | patch: src="{{ tmp_dir }}/rails-postgresql.patch" remote_src=true basedir="{{ tmp_dir }}" 30 | 31 | - name: Switch to test project 32 | command: oc project test 33 | 34 | - name: Launch rails-postgresql application 35 | command: oc new-app "{{ tmp_dir }}/rails-postgresql.json" 36 | 37 | - name: Switch back to default project 38 | command: oc project default 39 | 40 | vars: 41 | tmp_dir: /home/centos/test -------------------------------------------------------------------------------- /roles/opencontrail/templates/vhost0.cfg.j2: -------------------------------------------------------------------------------- 1 | auto vhost0 2 | iface vhost0 inet static 3 | pre-up modprobe vrouter 4 | pre-up ip link add name vhost0 type vhost 5 | pre-up ip link set vhost0 address $(cat /sys/class/net/{{ opencontrail_host_interface }}/address) 6 | pre-up vif --add {{ opencontrail_host_interface }} --mac $(cat /sys/class/net/{{ opencontrail_host_interface }}/address) --vrf 0 --vhost-phys --type physical 7 | pre-up vif --add vhost0 --mac $(cat /sys/class/net/{{ opencontrail_host_interface }}/address) --vrf 0 --type vhost --xconnect {{ opencontrail_host_interface }} 8 | address {{ opencontrail_host_address }} 9 | netmask {{ opencontrail_host_netmask }} 10 | up ip addr flush {{ opencontrail_host_interface }} 11 | {% if opencontrail_host_gateway -%} 12 | up ip route add default via {{ opencontrail_host_gateway }} 13 | {% endif -%} 14 | {% if inventory_hostname in groups['nodes'] %} 15 | up ip route add {{ opencontrail_all_service_addresses }} via {{ hostvars[groups['gateways'][0]]['opencontrail_host_address'] }} 16 | {% endif %} 17 | post-down vif --list | awk '/^vif.*OS: vhost0/ {split($1, arr, "\/"); print arr[2];}' | xargs vif --delete 18 | post-down vif --list | awk '/^vif.*OS: {{ opencontrail_host_interface }}/ {split($1, arr, "\/"); print arr[2];}' | xargs vif --delete 19 | post-down ip link delete vhost0 20 | -------------------------------------------------------------------------------- /test/ec2-origin/roles/clean/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Cleanup routes from the VPC routing table 3 | ec2_vpc_rtb_update: 4 | region: "{{ aws_region }}" 5 | vpc_id: "{{ ec2_vpc_id }}" 6 | state: absent 7 | subnets: 8 | - "{{ ec2_private_subnet_id }}" 9 | routes: 10 | - dest: "{{ opencontrail_service_addresses }}" 11 | 12 | - name: Instance id list 13 | set_fact: 14 | hosts: "{{ groups['all'] | difference(['localhost']) }}" 15 | - set_fact: 16 | instance_ids: "{% set x = [] %}{% for host in hosts %}{% set _ = x.append(hostvars[host]['id']) %}{% endfor %}{{ x }}" 17 | 18 | - ec2_vol: 19 | instance: "{{ item }}" 20 | state: list 21 | region: "{{ aws_region }}" 22 | with_items: instance_ids 23 | register: volume_list 24 | 25 | - set_fact: 26 | volume_ids: "{% set x = [] %}{% for item in volume_list.results %}{% for vol in item.volumes %}{% if vol['attachment_set']['device'] == '/dev/sda1' %}{% set _ = x.append(vol.id) %}{% endif %}{% endfor %}{% endfor %}{{ x }}" 27 | 28 | - name: Delete virtual machines 29 | ec2: 30 | state: absent 31 | region: "{{ aws_region }}" 32 | instance_ids: "{{ hostvars[item].id }}" 33 | wait: yes 34 | with_items: "{{ hosts }}" 35 | 36 | - name: Delete volumes 37 | ec2_vol: 38 | region: "{{ aws_region }}" 39 | id: "{{ item }}" 40 | state: absent 41 | with_items: volume_ids 42 | ignore_errors: true -------------------------------------------------------------------------------- /roles/opencontrail/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - assert: 4 | that: 5 | - "'masters' in groups" 6 | - "'nodes' in groups" 7 | 8 | - name: Assert that interface facts are defined 9 | assert: 10 | that: 11 | - opencontrail_host_interface is defined 12 | - opencontrail_host_ipaddr is defined 13 | - opencontrail_host_address is defined 14 | - opencontrail_host_gateway is defined 15 | 16 | - include_vars: openshift.yml 17 | when: openshift is defined or 'openshift' in hostvars[groups['masters'][0]] 18 | 19 | - include_vars: kubernetes.yml 20 | when: opencontrail_cluster_type == "kubernetes" 21 | 22 | - name: Assert that cluster dependent vars are defined 23 | assert: 24 | that: 25 | - opencontrail_all_service_addresses is defined 26 | - opencontrail_all_kube_config_dir is defined 27 | 28 | - include: docker_config.yml 29 | 30 | - name: Build kernel module 31 | include: kmod.yml 32 | 33 | - name: Install vrouter 34 | include: vrouter.yml 35 | when: opencontrail_host_use_vrouter 36 | 37 | - name: Install compute nodes 38 | include: nodes.yml 39 | when: inventory_hostname in groups['nodes'] 40 | 41 | - name: Install gateways 42 | include: gateways.yml 43 | when: "'gateways' in groups and inventory_hostname in groups['gateways']" 44 | 45 | - name: Install master 46 | include: masters.yml 47 | when: inventory_hostname in groups['masters'] 48 | -------------------------------------------------------------------------------- /roles/opencontrail/tasks/masters-services.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: cassandra directory 3 | file: name=/var/lib/cassandra state=directory 4 | 5 | - name: zookeeper directory 6 | file: name=/var/lib/zookeeper state=directory 7 | 8 | - name: Install templates 9 | copy: src="{{ item }}" dest="{{ opencontrail_all_kube_manifest_dir }}" 10 | with_items: 11 | - cassandra.manifest 12 | - rabbitmq.manifest 13 | - zookeeper.manifest 14 | when: not opencontrail_use_systemd 15 | 16 | - name: master services systemd files 17 | template: src="{{ item }}.j2" dest="/etc/systemd/system/{{ item }}" 18 | with_items: 19 | - cassandra.service 20 | - zookeeper.service 21 | - rabbitmq.service 22 | notify: 23 | - reload systemd 24 | when: opencontrail_use_systemd 25 | 26 | - name: Docker image ids (services) 27 | command: docker images -q "{{ item }}" 28 | with_items: 29 | - 'mesoscloud/zookeeper:3.4.6' 30 | - 'rabbitmq:3.5.4' 31 | - 'cassandra:2.2.0' 32 | register: image_ids 33 | always_run: true 34 | 35 | - name: Pull docker images (services) 36 | command: docker pull "{{ item.item }}" 37 | with_items: image_ids.results 38 | when: not item.stdout 39 | 40 | - name: master services 41 | service: name="{{ item }}" enabled=yes state=started 42 | with_items: 43 | - cassandra.service 44 | - zookeeper.service 45 | - rabbitmq.service 46 | when: opencontrail_use_systemd -------------------------------------------------------------------------------- /roles/opencontrail/tasks/gateways.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: VRouter log directory 3 | file: path="/var/log/contrail" state=directory 4 | 5 | - name: VRouter agent configuration 6 | template: src=contrail-vrouter-agent.conf.gateway.j2 dest=/etc/contrail/contrail-vrouter-agent.conf 7 | notify: 8 | - restart contrail-vrouter-agent 9 | 10 | - name: Redhat-sytle interface configuration 11 | include: gateway-redhat.yml 12 | when: ansible_os_family == "RedHat" 13 | 14 | - name: Ubuntu interface configuration 15 | include: gateway-ubuntu.yml 16 | when: ansible_distribution == "Ubuntu" 17 | 18 | - name: Create vrouter agent service 19 | template: src=contrail-vrouter-agent.service.j2 dest=/etc/systemd/system/contrail-vrouter-agent.service 20 | notify: 21 | - reload systemd 22 | when: not (ansible_distribution == "Ubuntu" and ansible_distribution_major_version|int < 15) 23 | 24 | - name: Enable vhost0 interface 25 | command: ifup vhost0 26 | 27 | - name: Docker image ids (gateways) 28 | command: docker images -q "{{ item }}" 29 | with_items: 30 | - "opencontrail/vrouter-agent:{{ opencontrail_all_release }}" 31 | register: _docker_image_ids 32 | always_run: true 33 | 34 | - name: Pull docker images (gateways) 35 | command: docker pull "{{ item.item }}" 36 | with_items: _docker_image_ids.results 37 | when: not item.stdout 38 | 39 | - name: Start vrouter agent 40 | service: name=contrail-vrouter-agent enabled=yes state=started 41 | -------------------------------------------------------------------------------- /test/ec2-origin/roles/workspace/files/rails-postgresql.patch.j2: -------------------------------------------------------------------------------- 1 | --- /usr/share/openshift/examples/quickstart-templates/rails-postgresql.json 2016-01-04 23:35:22.893340775 +0000 2 | +++ rails-postgresql.json 2016-01-08 19:04:41.531340775 +0000 3 | @@ -73,7 +73,9 @@ 4 | "type": "Git", 5 | "git": { 6 | "uri": "${SOURCE_REPOSITORY_URL}", 7 | - "ref": "${SOURCE_REPOSITORY_REF}" 8 | + "ref": "${SOURCE_REPOSITORY_REF}", 9 | + "httpProxy": "http://{{ openshift_master_portal_net | ipaddr('net') | ipaddr(2) | ipaddr('address')}}:3128", 10 | + "httpsProxy": "http://{{ openshift_master_portal_net | ipaddr('net') | ipaddr(2) | ipaddr('address')}}:3128" 11 | }, 12 | "contextDir": "${CONTEXT_DIR}" 13 | }, 14 | @@ -84,7 +86,17 @@ 15 | "kind": "ImageStreamTag", 16 | "namespace": "openshift", 17 | "name": "ruby:2.2" 18 | - } 19 | + }, 20 | + "env": [ 21 | + { 22 | + "name": "http_proxy", 23 | + "value": "http://{{ openshift_master_portal_net | ipaddr('net') | ipaddr(2) | ipaddr('address')}}:3128" 24 | + }, 25 | + { 26 | + "name": "https_proxy", 27 | + "value": "http://{{ openshift_master_portal_net | ipaddr('net') | ipaddr(2) | ipaddr('address')}}:3128" 28 | + } 29 | + ] 30 | } 31 | }, 32 | "output": { 33 | -------------------------------------------------------------------------------- /roles/opencontrail/tasks/kmod-build-redhat.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Patch required for kernel 3.10.0-123.8.1.el7.x86_64 3 | # This kernel version is toxic due to gro bugs. 4 | # - name: Set patches 5 | # set_fact: 6 | # kmod_patches: ['kmod-centos7-3.10.0-123.8.1.patch'] 7 | 8 | # - name: copy patches 9 | # copy: src="{{ item }}" dest="/tmp/.ansible/build/{{ opencontrail_host_kernel_tag }}/{{ item }}" 10 | # with_items: kmod_patches 11 | 12 | - name: kmod Dockerfile 13 | template: 14 | src: Dockerfile.redhat.j2 15 | dest: /tmp/.ansible/build/{{ opencontrail_host_kernel_tag }}/Dockerfile 16 | delegate_to: "{{ groups['masters'][0] }}" 17 | run_once: true 18 | 19 | - name: Build docker container 20 | command: docker build -t opencontrail/kmod_{{ opencontrail_host_kernel_tag }} {{ opencontrail_host_kernel_tag }} 21 | args: 22 | chdir: /tmp/.ansible/build 23 | delegate_to: "{{ groups['masters'][0] }}" 24 | run_once: true 25 | 26 | - name: Build kernel module 27 | command: docker run -v /tmp/.ansible/build/{{ opencontrail_host_kernel_tag }}:/src/vrouter/build{{ ':Z' if ansible_selinux is defined and ansible_selinux.status == 'enabled' else '' }} opencontrail/kmod_{{ opencontrail_host_kernel_tag }} /bin/bash -c "USER=nobody scons --optimization=production --kernel-dir=/usr/src/kernels/{{ opencontrail_host_kernel_devel }} vrouter && cp vrouter/vrouter.ko build/kbuild" 28 | args: 29 | creates: /tmp/.ansible/build/{{ opencontrail_host_kernel_tag }}/kbuild/vrouter.ko 30 | delegate_to: "{{ groups['masters'][0] }}" 31 | run_once: true 32 | -------------------------------------------------------------------------------- /test/ec2-origin/roles/workspace/files/system-install.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # playbook use to configure the cluster with settings such as web-proxy 3 | # and address resolution. It is executed by the deployer against the 4 | # cluster inventory. 5 | 6 | - name: System configuration 7 | hosts: all 8 | sudo: yes 9 | tasks: 10 | - name: yum proxy configuration 11 | lineinfile: 12 | dest: /etc/yum.conf 13 | regexp: "^proxy=" 14 | line: "proxy={{ web_proxy }}" 15 | 16 | - name: environment (http) 17 | lineinfile: 18 | dest: /etc/environment 19 | regexp: "^http_proxy=" 20 | line: "http_proxy={{ web_proxy }}" 21 | 22 | - name: environment (https) 23 | lineinfile: 24 | dest: /etc/environment 25 | regexp: "^https_proxy=" 26 | line: "https_proxy={{ web_proxy }}" 27 | 28 | - name: environment (no-proxy) 29 | lineinfile: 30 | dest: /etc/environment 31 | regexp: "^no_proxy=" 32 | line: "no_proxy=localhost,127.0.0.1,{{ ansible_default_ipv4['address'] }},169.254.169.254,.dev.opencontrail.org,.svc.cluster.local,.compute.internal,::1" 33 | 34 | - name: Set hostname 35 | hostname: name="{{ inventory_hostname }}" 36 | 37 | - name: Add entries to /etc/hosts 38 | lineinfile: 39 | dest=/etc/hosts 40 | regexp="^{{ hostvars[item].ansible_ssh_host }}" 41 | line="{{ hostvars[item].ansible_ssh_host }} {{ item }}" 42 | with_items: groups['all'] 43 | 44 | vars: 45 | web_proxy: "http://web-proxy.VAR_AWS_REGION.dev.opencontrail.org:3128" 46 | -------------------------------------------------------------------------------- /test/README.md: -------------------------------------------------------------------------------- 1 | # EC2 tests 2 | 3 | By default, EC2 limits clients to 5 VPCs. As such it is not practical to create a VPC per test job. All the jenkins jobs share one VPC which is created by the aws-ci-provisioning. 4 | 5 | This VPC uses the CIDR block 10.0.0.0/16 and the following 2 subnets: 6 | 7 | | IP Prefix | Name | Description | 8 | |-----------|------|-------------| 9 | | 10.0.0.0/20 | public | For instances with public IP addresses | 10 | | 10.0.32.0/20 | private | For instances without public IP addresses | 11 | 12 | The networks used by the test clusters should exclude the VPC CIDR block. 13 | 14 | `kube-network-manager` defaults to using the same CIDR block as its private IP block. The tests override the private IP block to 10.32.0.0/16. 15 | 16 | ## kubernetes 17 | 18 | The tests require network access between the underlay and the public network. In addition, future tests may require connectivity between the underlay network and monitoring services. 19 | 20 | | IP Prefix | Name | Description | 21 | |-----------|------|-------------| 22 | | 172.16.(8 * (job_id % 32)).0/21 | public network | | 23 | | 172.18.0.0/20 | public network | default (manual tests) | 24 | | 10.(192 + job_id % 32).0.0/16 | service address range | | 25 | | 10.64.0.0/16 | service address range | default (manual tests) | 26 | 27 | 28 | 29 | ## openshift 30 | 31 | | IP Prefix | Name | Description | 32 | |-----------|------|-------------| 33 | | 172.20.( 8 * (job_id % 32)).0/21 | public network | | 34 | | 172.18.64.0/20 | public network | default (manual tests) | 35 | | 10.(160 + job_id % 32).0.0/16 | service address range | | 36 | | 10.65.0.0/16 | service address range | default (manual tests) | 37 | 38 | -------------------------------------------------------------------------------- /test/ec2-origin/roles/workspace/files/deployment_config_set.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | """ 4 | deployment_config_set [-i] filename 5 | 6 | Edit an openshift deployment config template. 7 | """ 8 | 9 | import argparse 10 | import json 11 | import sys 12 | 13 | class DeploymentConfig(object): 14 | def __init__(self, fp): 15 | self._template = json.load(fp) 16 | 17 | def _get_item(self, kind): 18 | for item in self._template['items']: 19 | if item['kind'] == kind: 20 | return item 21 | return None 22 | 23 | def set_service(self, clusterIP): 24 | svc = self._get_item("Service") 25 | svc['spec']['clusterIP'] = clusterIP 26 | 27 | def store(self, fp): 28 | return json.dump(self._template, fp, indent=4) 29 | 30 | def main(): 31 | parser = argparse.ArgumentParser() 32 | parser.add_argument('-i', help='Modify input file') 33 | parser.add_argument('--output', type=str, help='Output filename') 34 | parser.add_argument('--service', type=str, help='Set service cluster IP address') 35 | parser.add_argument('filename', help='deployment config JSON file') 36 | 37 | args = parser.parse_args() 38 | 39 | if args.i and args.output: 40 | print 'Invalid arguments: -i and --output are mutually exclusive' 41 | sys.exit(1) 42 | 43 | with open(args.filename, 'r') as fp: 44 | deployment = DeploymentConfig(fp) 45 | 46 | if args.service: 47 | deployment.set_service(args.service) 48 | 49 | if args.i or args.output: 50 | outputFile = args.output 51 | if not outputFile: 52 | outputFile = args.filename 53 | with open(outputFile, 'w') as fp: 54 | deployment.store(fp) 55 | else: 56 | deployment.store(sys.stdout) 57 | 58 | if __name__ == '__main__': 59 | main() -------------------------------------------------------------------------------- /roles/opencontrail_facts/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - include_vars: openshift.yml 3 | when: opencontrail_cluster_type == "openshift" 4 | 5 | - include_vars: kubernetes.yml 6 | when: opencontrail_cluster_type == "kubernetes" 7 | 8 | - name: Host IP address configuration (from inventory) 9 | include: interface_inventory_facts.yml 10 | when: opencontrail_ipaddr is defined 11 | 12 | - name: Host IP address configuration (from physical interface) 13 | include: interface_ansible_facts.yml 14 | when: "opencontrail_ipaddr is not defined and 'ipv4' in hostvars[inventory_hostname]['ansible_' + opencontrail_host_interface]" 15 | 16 | - name: Host IP address configuration (from vhost0) 17 | include: vhost_ansible_facts.yml 18 | when: "opencontrail_ipaddr is not defined and 'ansible_vhost0' in hostvars[inventory_hostname] and 'ipv4' in ansible_vhost0" 19 | 20 | 21 | - name: Master IP list override 22 | set_fact: 23 | opencontrail_master_host_addresses: "{{ opencontrail_masters_ipaddr }}" 24 | when: opencontrail_masters_ipaddr is defined 25 | 26 | - name: Master IP list default 27 | set_fact: 28 | opencontrail_master_host_addresses: "{%for host in groups['masters']%}{{ hostvars[host]['opencontrail_host_address'] }} {% endfor %}" 29 | when: opencontrail_master_host_addresses is not defined 30 | 31 | - name: First Master ipaddr override 32 | set_fact: 33 | opencontrail_master_host_address: "{{ opencontrail_first_masters_ipaddr }}" 34 | when: opencontrail_first_masters_ipaddr is defined 35 | 36 | - name: First Master ipaddr default 37 | set_fact: 38 | opencontrail_master_host_address: "{{ hostvars[groups['masters'][0]]['opencontrail_host_address'] }}" 39 | when: opencontrail_master_host_address is not defined 40 | 41 | -------------------------------------------------------------------------------- /test/ec2-origin/roles/workspace/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create container-networking-ansible tar 3 | local_action: command tar zcvf container-networking-ansible.tgz -C "{{ inventory_dir }}/../.." --exclude test --exclude playbooks --exclude .git . 4 | 5 | - name: workspace directory 6 | file: path="{{ path_src }}" state=directory 7 | 8 | - name: Openshift provisioning 9 | git: repo=https://github.com/pedro-r-marques/openshift-ansible dest="{{ path_src }}/openshift-ansible" version=opencontrail_3.0.55 update=no 10 | 11 | - copy: src=inventory.cluster dest="{{ path_src }}/openshift-ansible/inventory/byo/hosts" 12 | 13 | - name: Copy deployment playbooks 14 | copy: src="{{ item }}" dest="{{ path_src }}/openshift-ansible/playbooks/byo" 15 | with_items: 16 | - system-install.yml 17 | - opencontrail.yml 18 | - opencontrail_provision.yml 19 | - openshift_provision.yml 20 | - applications.yml 21 | - systemd_workaround.yml 22 | 23 | - name: variable substitution 24 | command: sed -i "s/VAR_AWS_REGION/{{ aws_region }}/" "{{ path_src }}/openshift-ansible/playbooks/byo/{{ item }}" 25 | with_items: 26 | - system-install.yml 27 | - opencontrail.yml 28 | - opencontrail_provision.yml 29 | 30 | - name: Copy registry certificate 31 | copy: src=registry.crt dest="{{ path_src }}/openshift-ansible/playbooks/byo" 32 | 33 | - name: Copy files 34 | copy: src="{{ item }}" dest="{{ path_src }}/openshift-ansible/playbooks/byo" 35 | with_items: 36 | - opencontrail_validate.py 37 | - deployment_config_set.py 38 | - rails-postgresql.patch.j2 39 | 40 | - copy: src=ansible.cfg dest="{{ path_src }}/openshift-ansible" 41 | 42 | - name: Extract opencontrail roles 43 | unarchive: src=container-networking-ansible.tgz dest="{{ path_src }}/openshift-ansible" 44 | 45 | -------------------------------------------------------------------------------- /test/common/library/ec2_vpc_facts.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | # Retrieve information on an existing VPC. 4 | # 5 | 6 | # import module snippets 7 | from ansible.module_utils.basic import * 8 | from ansible.module_utils.ec2 import * 9 | 10 | import boto.vpc 11 | 12 | 13 | def main(): 14 | argument_spec = ec2_argument_spec() 15 | argument_spec.update(dict( 16 | resource_tags=dict(type='dict', required=True) 17 | )) 18 | module = AnsibleModule(argument_spec=argument_spec) 19 | 20 | ec2_url, aws_access_key, aws_secret_key, region = get_ec2_creds(module) 21 | 22 | if not region: 23 | module.fail_json(msg="region must be specified") 24 | 25 | try: 26 | connection = boto.vpc.connect_to_region( 27 | region, 28 | aws_access_key_id=aws_access_key, 29 | aws_secret_access_key=aws_secret_key) 30 | except boto.exception.NoAuthHandlerFound, e: 31 | module.fail_json(msg=str(e)) 32 | 33 | vpcs = connection.get_all_vpcs() 34 | vpcs_w_resources = filter( 35 | lambda x: x.tags == module.params.get('resource_tags'), vpcs) 36 | if len(vpcs_w_resources) != 1: 37 | if len(vpcs_w_resources) == 0: 38 | module.fail_json(msg="No vpc found") 39 | else: 40 | module.fail_json(msg="Multiple VPCs with specified resource_tags") 41 | 42 | vpc = vpcs_w_resources[0] 43 | 44 | subnets = connection.get_all_subnets(filters={'vpc_id': vpc.id}) 45 | 46 | def subnet_data(s): 47 | d = s.__dict__ 48 | del d["connection"] 49 | del d["region"] 50 | return d 51 | 52 | data = map(subnet_data, subnets) 53 | facts = { 54 | 'ec2_vpc': { 55 | 'id': vpc.id, 56 | 'subnets': data 57 | } 58 | } 59 | module.exit_json(changed=False, ansible_facts=facts) 60 | 61 | main() 62 | -------------------------------------------------------------------------------- /roles/opencontrail/templates/Dockerfile.redhat.j2: -------------------------------------------------------------------------------- 1 | FROM {{ ansible_distribution | lower }}:{{ ansible_distribution_major_version }} 2 | {%if opencontrail_build_http_proxy %} 3 | RUN sh -c "echo proxy={{ opencontrail_build_http_proxy }}" >> /etc/yum.conf 4 | {% endif %} 5 | RUN {{ ansible_pkg_mgr }} install -y git make 6 | {%if opencontrail_build_http_proxy %} 7 | RUN git config --global http.proxy {{ opencontrail_build_http_proxy }} 8 | RUN git config --global https.proxy {{ opencontrail_build_http_proxy }} 9 | {% endif %} 10 | RUN {{ ansible_pkg_mgr }} install -y automake flex bison gcc gcc-c++ boost boost-devel libxml2-devel python-lxml 11 | RUN yum install -y yum-utils 12 | # When the running system is an older version compared to the container install the vault repository 13 | RUN release=$(awk '{print $4;}' /etc/redhat-release); if [ $release != "{{ ansible_distribution_version }} " ]; then yum-config-manager --enable C{{ ansible_distribution_version }}-base; fi 14 | # On centos the kernel package name does not match the kernel name 15 | RUN yum install -y kernel-devel-{{ opencontrail_host_kernel_devel }} 16 | {% if ansible_distribution != "Fedora" %} 17 | RUN yum -y install http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm 18 | {% endif %} 19 | RUN {{ ansible_pkg_mgr }} install -y scons 20 | RUN mkdir -p src/vrouter 21 | WORKDIR src/vrouter 22 | RUN git clone -b R2.20 https://github.com/Juniper/contrail-vrouter vrouter 23 | RUN mkdir tools 24 | RUN (cd tools && git clone https://github.com/Juniper/contrail-build build) 25 | RUN (cd tools && git clone -b R2.20 https://github.com/Juniper/contrail-sandesh sandesh) 26 | RUN cp tools/build/SConstruct . 27 | {% if opencontrail_host_kmod_patches is defined %} 28 | RUN yum install -y patch 29 | {% for patch in opencontrail_host_kmod_patches %} 30 | ADD "{{ patch }}" /src/vrouter/vrouter/ 31 | RUN (cd vrouter && patch -p1 -i "{{ patch }}") 32 | {% endfor %} 33 | {% endif %} -------------------------------------------------------------------------------- /roles/opencontrail/tasks/kmod-artifacts.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create install tree 3 | file: 4 | path: "{{ opencontrail_host_kernel_install_dir }}/usr/bin" 5 | state: directory 6 | delegate_to: "{{ groups['masters'][0] }}" 7 | run_once: true 8 | 9 | - name: Create install tree (kmod) 10 | file: 11 | path: "{{ opencontrail_host_kernel_install_dir }}/lib/modules/{{ ansible_kernel }}/extra/net" 12 | state: directory 13 | delegate_to: "{{ groups['masters'][0] }}" 14 | run_once: true 15 | 16 | - name: Install utilities 17 | command: install -m 755 "production/vrouter/utils/{{ item }}" "{{ opencontrail_host_kernel_install_dir }}/usr/bin" 18 | args: 19 | chdir: "{{ opencontrail_host_kernel_build_dir }}" 20 | creates: "{{ opencontrail_host_kernel_install_dir }}/usr/bin/{{ item }}" 21 | with_items: 22 | - dropstats 23 | - flow 24 | - mirror 25 | - mpls 26 | - nh 27 | - rt 28 | - vif 29 | - vrfstats 30 | - vrouter 31 | - vxlan 32 | delegate_to: "{{ groups['masters'][0] }}" 33 | run_once: true 34 | 35 | - name: Install kmod 36 | command: install kbuild/vrouter.ko "{{ opencontrail_host_kernel_install_dir }}/lib/modules/{{ ansible_kernel }}/extra/net" 37 | args: 38 | chdir: "{{ opencontrail_host_kernel_build_dir }}" 39 | creates: "{{ opencontrail_host_kernel_install_dir }}/lib/modules/{{ ansible_kernel }}/extra/net/vrouter.ko" 40 | delegate_to: "{{ groups['masters'][0] }}" 41 | run_once: true 42 | 43 | - name: Create tarball 44 | command: tar zcf "{{ opencontrail_host_kernel_artifact_tar }}" . 45 | args: 46 | chdir: "{{ opencontrail_host_kernel_install_dir }}" 47 | creates: "{{ opencontrail_host_kernel_artifact_tar }}" 48 | delegate_to: "{{ groups['masters'][0] }}" 49 | run_once: true 50 | 51 | - name: Fetch tarball 52 | fetch: src="{{ opencontrail_host_kernel_artifact_tar }}" dest="/tmp/.ansible/artifacts/" flat=yes 53 | delegate_to: "{{ groups['masters'][0] }}" 54 | run_once: true 55 | -------------------------------------------------------------------------------- /roles/opencontrail/files/ifup-vhost: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | . /etc/init.d/functions 4 | 5 | cd /etc/sysconfig/network-scripts 6 | . ./network-functions 7 | 8 | [ -f ../network ] && . ../network 9 | 10 | CONFIG=${1} 11 | 12 | need_config "${CONFIG}" 13 | 14 | source_config 15 | 16 | if ! /sbin/modprobe vrouter >/dev/null 2>&1; then 17 | net_log $"OpenContrail vrouter kernel module not available" 18 | exit 1 19 | fi 20 | 21 | if [ -n "${MACADDR}" ]; then 22 | hwaddr=${MACADDR} 23 | else 24 | if [ -n "${PHYSDEV}" ]; then 25 | hwaddr=$(cat /sys/class/net/${PHYSDEV}/address) 26 | fi 27 | fi 28 | 29 | if [ ! -d /sys/class/net/${DEVICE} ]; then 30 | ip link add ${DEVICE} type vhost || net_log $"Error creating interface ${DEVICE}" 31 | 32 | if [ -n "${hwaddr}" ]; then 33 | ip link set ${DEVICE} address ${hwaddr} || net_log $"Error setting mac-address on ${DEVICE}" 34 | fi 35 | 36 | if [ -n "${PHYSDEV}" ]; then 37 | vif --add ${PHYSDEV} --mac ${hwaddr} --vrf 0 --vhost-phys --type physical >/dev/null 2>&1 || net_log $"Error adding host interface to vrouter module" 38 | vif --add ${DEVICE} --mac ${hwaddr} --vrf 0 --type vhost --xconnect ${PHYSDEV} >/dev/null 2>&1 || net_log $"Error setting cross-connect on host interface" 39 | fi 40 | fi 41 | 42 | if [ -n "${IPADDR}" ]; then 43 | ip addr add dev ${DEVICE} ${IPADDR} || net_log $"Error configuring IP address on interface" 44 | fi 45 | 46 | if [ -n "${PHYSDEV}" ]; then 47 | if [ -f "/var/run/dhclient-${PHYSDEV}.pid" ]; then 48 | pid=$(cat /var/run/dhclient-${PHYSDEV}.pid) 49 | kill $pid && rm /var/run/dhclient-${PHYSDEV}.pid 50 | fi 51 | ip addr flush ${PHYSDEV} || net_log $"Error flushing ip addresses on ${PHYSDEV}" 52 | fi 53 | 54 | ip link set ${DEVICE} up || net_log $"Error setting link state up" 55 | 56 | if [ -n "${GATEWAY}" ]; then 57 | ip route replace default via ${GATEWAY} dev ${DEVICE} || net_log $"Error adding default gateway" 58 | fi 59 | 60 | exec /etc/sysconfig/network-scripts/ifup-post ${CONFIG} ${2} 61 | 62 | -------------------------------------------------------------------------------- /test/common/validate.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: sanity check the cluster 3 | hosts: masters 4 | sudo: yes 5 | tasks: 6 | - name: Check the number of XMPP sessions 7 | shell: "netstat -nt | grep -E ':5269\\s+.*ESTABLISHED'" 8 | register: _xmpp_sessions 9 | ignore_errors: true 10 | 11 | - debug: var=_xmpp_sessions.stdout 12 | - set_fact: 13 | xmpp_session_count: "{{ _xmpp_sessions.stdout_lines | length }}" 14 | - debug: var=xmpp_session_count 15 | 16 | # workaround control-node ifmap bug. 17 | - name: Determine control-node docker id 18 | shell: "docker ps -f name='k8s.*contrail-control' | awk '/^([0-9a-f]+)\\s+opencontrail/ { print $1; }'" 19 | register: _control_status 20 | 21 | - name: Examine control-node logs for NoFqnSet bug 22 | shell: docker logs --since=1m "{{ _control_status.stdout }}" | grep NoFqnSet 23 | when: _control_status.stdout != "" 24 | register: _control_logs_nofqnset 25 | ignore_errors: true 26 | 27 | - debug: var=_control_logs_nofqnset.stdout 28 | 29 | - name: Restart control-node 30 | command: docker restart "{{ _control_status.stdout }}" 31 | when: xmpp_session_count|int != 3 and _control_status.stdout != "" and _control_logs_nofqnset.rc == 0 32 | 33 | - name: dns rc is not backwards compatible (successThreshold) 34 | lineinfile: dest=/etc/kubernetes/addons/dns/skydns-rc.yaml regexp="successThreshold:" state=absent 35 | 36 | - name: dns rc is not backwards compatible (failureThreshold) 37 | lineinfile: dest=/etc/kubernetes/addons/dns/skydns-rc.yaml regexp="failureThreshold:" state=absent 38 | 39 | - assert: 40 | that: 41 | - xmpp_session_count|int == 3 42 | 43 | # Ensure that the DNS resolver is operational 44 | - name: Check kube-system pods 45 | shell: "kubectl --namespace=kube-system get pods | awk '/kube-dns-v/{ print $2 ;}'" 46 | register: dns_status 47 | 48 | - assert: 49 | that: 50 | - dns_status.stdout == "4/4" 51 | 52 | 53 | -------------------------------------------------------------------------------- /test/ec2-origin/roles/workspace/files/openshift_provision.yml: -------------------------------------------------------------------------------- 1 | - hosts: 2 | - masters 3 | sudo: yes 4 | tasks: 5 | - name: Workaround image-stream download problem 6 | command: oc -n openshift delete is --all 7 | 8 | - name: Recreate image-streams 9 | command: oc -n openshift create -f /usr/share/openshift/examples/image-streams/image-streams-centos7.json 10 | 11 | - file: path="{{ tmp_dir }}" state=directory 12 | 13 | - name: Create OpenShift Registry template 14 | shell: >- 15 | oadm registry 16 | -o json --service-account=registry 17 | --credentials={{ openshift_master_config_dir }}/openshift-registry.kubeconfig 18 | > "{{ tmp_dir }}/registry.json.in" 19 | args: 20 | creates: "{{ tmp_dir }}/registry.json.in" 21 | 22 | - name: Set registry clusterIP address 23 | script: >- 24 | deployment_config_set.py --service={{ registry_svc_address }} 25 | --output={{ tmp_dir }}/registry.json 26 | {{ tmp_dir }}/registry.json.in 27 | args: 28 | creates: "{{ tmp_dir }}/registry.json" 29 | 30 | - name: Lookup Registry service 31 | command: oc get svc docker-registry 32 | register: _registry_svc_get 33 | ignore_errors: yes 34 | 35 | - name: Create OpenShift Registry 36 | command: oc create -f "{{ tmp_dir }}/registry.json" 37 | when: _registry_svc_get.rc != 0 38 | 39 | - name: Deploy OpenShift Router 40 | command: >- 41 | oadm router 42 | --create --service-account=router --host-network=false 43 | --credentials={{ openshift_master_config_dir }}/openshift-router.kubeconfig 44 | register: _ortr_results 45 | changed_when: "'service exists' not in _ortr_results.stdout" 46 | 47 | - name: Create test user 48 | command: htpasswd -b /etc/openshift/htpasswd test c0ntrail123 49 | 50 | vars: 51 | openshift_master_config_dir: /etc/origin/master 52 | tmp_dir: /home/centos/.ansible/tmp 53 | opencontrail_all_service_addresses: "{{ openshift_master_portal_net | default('172.30.0.0/16') }}" 54 | registry_svc_address: "{{ opencontrail_all_service_addresses | ipaddr('net') | ipaddr(32) | ipaddr('address')}}" 55 | -------------------------------------------------------------------------------- /test/ec2-origin/roles/cluster/templates/inventory.j2: -------------------------------------------------------------------------------- 1 | [OSEv3:children] 2 | masters 3 | nodes 4 | etcd 5 | 6 | # Set variables common for all OSEv3 hosts 7 | [OSEv3:vars] 8 | # SSH user, this user should allow ssh based auth without requiring a 9 | # password. If using ssh key based auth, then the key should be managed by an 10 | # ssh agent. 11 | ansible_ssh_user=centos 12 | 13 | # If ansible_ssh_user is not root, ansible_sudo must be set to true and the 14 | # user must be configured for passwordless sudo 15 | ansible_sudo=true 16 | 17 | # deployment type valid values are origin, online and enterprise 18 | #deployment_type=atomic-enterprise 19 | deployment_type=origin 20 | 21 | openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', 'challenge': 'true', 'kind': 'HTPasswdPasswordIdentityProvider', 'filename': '/etc/openshift/htpasswd'}] 22 | 23 | cli_docker_additional_registries=registry.{{ aws_region }}.dev.opencontrail.org:5000 24 | 25 | use_openshift_sdn = false 26 | sdn_network_plugin_name = opencontrail 27 | 28 | openshift_master_portal_net = {{ cluster_service_addresses }} 29 | 30 | openshift_pkg_version=-1.1.3 31 | 32 | [opencontrail:children] 33 | masters 34 | nodes 35 | gateways 36 | 37 | [opencontrail:vars] 38 | opencontrail_public_subnet={{ cluster_public_subnet }} 39 | opencontrail_private_subnet=10.32.0.0/16 40 | opencontrail_http_proxy=http://web-proxy.{{ aws_region }}.dev.opencontrail.org:3128 41 | opencontrail_kube_release=origin-1.1.3 42 | opencontrail_dns_forwarder={{ origin_deployer.tagged_instances[0].private_ip }} 43 | opencontail_gateway_extra_infra_prefixes={{ ec2_private_subnet.cidr_block }} 44 | 45 | [masters] 46 | {% for instance in origin_master.tagged_instances %} 47 | origin-master-{{ loop.index }} ansible_ssh_host={{ instance.private_ip }} 48 | {% endfor %} 49 | 50 | [etcd] 51 | {% for instance in origin_master.tagged_instances %} 52 | origin-master-{{ loop.index }} ansible_ssh_host={{ instance.private_ip }} 53 | {% endfor %} 54 | 55 | [gateways] 56 | {% for instance in origin_gateway.tagged_instances %} 57 | origin-gateway-{{ loop.index }} ansible_ssh_host={{ instance.private_ip }} 58 | {% endfor %} 59 | 60 | [nodes] 61 | {% for instance in origin_nodes.tagged_instances %} 62 | origin-node-{{ loop.index }} ansible_ssh_host={{ instance.private_ip }} 63 | {% endfor %} 64 | 65 | -------------------------------------------------------------------------------- /test/ec2-origin/roles/workspace/files/opencontrail.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # This playbook runs the opencontrail role 3 | # 4 | - hosts: 5 | - all 6 | sudo: yes 7 | tasks: 8 | - lineinfile: 9 | dest: /etc/sysconfig/docker-storage-setup 10 | regexp: "^(DEVS=)" 11 | line: "DEVS=/dev/xvdf" 12 | create: yes 13 | 14 | - lineinfile: 15 | dest: /etc/sysconfig/docker-storage-setup 16 | regexp: "^(VG=)" 17 | line: "VG=ebs" 18 | 19 | - hosts: 20 | - all 21 | sudo: yes 22 | roles: 23 | - openshift_facts 24 | - docker 25 | tags: 26 | - docker 27 | vars: 28 | docker_udev_workaround: true 29 | 30 | - hosts: 31 | - all 32 | sudo: yes 33 | tasks: 34 | - lineinfile: 35 | dest: /etc/sysconfig/docker 36 | regexp: "^http_proxy=" 37 | line: "http_proxy={{ opencontrail_http_proxy }}" 38 | notify: 39 | - restart docker 40 | - lineinfile: 41 | dest: /etc/sysconfig/docker 42 | regexp: "^https_proxy=" 43 | line: "https_proxy={{ opencontrail_http_proxy }}" 44 | notify: 45 | - restart docker 46 | - lineinfile: 47 | dest: /etc/sysconfig/docker 48 | regexp: "^no_proxy=" 49 | line: "no_proxy=.dev.opencontrail.org,.compute.internal,{{ registry_svc_address }}" 50 | notify: 51 | - restart docker 52 | - lineinfile: 53 | dest: /etc/sysconfig/docker 54 | regexp: "^(#|)ADD_REGISTRY=" 55 | line: "ADD_REGISTRY=--add-registry registry.VAR_AWS_REGION.dev.opencontrail.org:5000" 56 | notify: 57 | - restart docker 58 | 59 | - file: path=/etc/docker/certs.d/registry.VAR_AWS_REGION.dev.opencontrail.org:5000 state=directory 60 | - copy: src=registry.crt dest=/etc/docker/certs.d/registry.VAR_AWS_REGION.dev.opencontrail.org:5000/ca.crt 61 | notify: 62 | - restart docker 63 | handlers: 64 | - name: restart docker 65 | service: name=docker state=restarted 66 | vars: 67 | opencontrail_all_service_addresses: "{{ openshift_master_portal_net | default('172.30.0.0/16') }}" 68 | registry_svc_address: "{{ opencontrail_all_service_addresses | ipaddr('net') | ipaddr(32) | ipaddr('address')}}" 69 | 70 | - hosts: 71 | - masters 72 | - nodes 73 | - gateways 74 | sudo: yes 75 | roles: 76 | - openshift_facts 77 | - opencontrail_facts 78 | - opencontrail 79 | vars: 80 | opencontrail_cluster_type: openshift 81 | tags: 82 | - opencontrail 83 | -------------------------------------------------------------------------------- /roles/opencontrail/tasks/masters-config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Ensure configuration directory exists 3 | file: path="/etc/contrail" state=directory 4 | 5 | - file: path="{{ opencontrail_all_kube_config_dir }}" state=directory 6 | 7 | - name: Ensure log directory exists 8 | file: path="/var/log/contrail" state=directory 9 | 10 | - name: Configuration files 11 | template: src="{{ item }}.j2" dest="/etc/contrail/{{ item }}" 12 | with_items: 13 | - contrail-api.conf 14 | - contrail-schema.conf 15 | 16 | - name: Network manager configuration 17 | template: src=kube-network.conf.j2 dest={{ opencontrail_all_kube_config_dir }}/network.conf 18 | notify: 19 | - restart kube-network-manager 20 | 21 | - name: Install manifests 22 | copy: src="{{ item }}" dest="{{ opencontrail_all_kube_manifest_dir }}" 23 | with_items: 24 | - contrail-api.manifest 25 | - contrail-schema.manifest 26 | - ifmap-server.manifest 27 | when: not opencontrail_use_systemd 28 | 29 | - name: Install manifests (from templates) 30 | template: src="{{ item }}.j2" dest="{{ opencontrail_all_kube_manifest_dir }}/{{ item }}" 31 | with_items: 32 | - kube-network-manager.manifest 33 | when: not opencontrail_use_systemd 34 | 35 | - name: Install configuration services 36 | template: src="{{item}}.j2" dest="/etc/systemd/system/{{ item }}" 37 | with_items: 38 | - contrail-api.service 39 | - contrail-schema.service 40 | - ifmap-server.service 41 | - kube-network-manager.service 42 | notify: 43 | - reload systemd 44 | when: opencontrail_use_systemd 45 | 46 | - name: Docker image ids (config) 47 | command: docker images -q "{{ item }}" 48 | with_items: 49 | - "opencontrail/config:{{ opencontrail_all_release }}" 50 | - "opencontrail/ifmap-server:{{ opencontrail_all_release }}" 51 | - "opencontrail/kube-network-manager:{{ opencontrail_kube_release }}" 52 | register: image_ids 53 | always_run: true 54 | 55 | - name: Pull docker images (config) 56 | command: docker pull "{{ item.item }}" 57 | with_items: image_ids.results 58 | when: not item.stdout 59 | 60 | - name: Start configuration services 61 | service: name="{{ item }}" enabled=yes state=started 62 | with_items: 63 | - contrail-api 64 | - contrail-schema 65 | - ifmap-server 66 | when: opencontrail_use_systemd 67 | 68 | # kube-network-manager can only be started after origin-master is defined. 69 | - name: Enable kube-network-manager 70 | service: name=kube-network-manager enabled=yes 71 | when: opencontrail_use_systemd -------------------------------------------------------------------------------- /test/ec2-origin/roles/workspace/files/opencontrail_provision.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # Playbook that runs after openshift is installed in order to provision 4 | # the contrail-api components. 5 | # 6 | - hosts: 7 | - masters 8 | sudo: yes 9 | roles: 10 | - openshift_facts 11 | - os_firewall 12 | vars: 13 | os_firewall_use_firewalld: false 14 | os_firewall_allow: 15 | - service: contrail-api 16 | port: 8082/tcp 17 | - service: xmpp 18 | port: 5269/tcp 19 | 20 | - hosts: 21 | - masters 22 | sudo: yes 23 | tasks: 24 | - name: Add kubernetes service address to /etc/hosts 25 | lineinfile: 26 | dest: /etc/hosts 27 | line: '{{ ansible_default_ipv4.address }} kubernetes.default.svc' 28 | 29 | - name: openshift-master proxy configuration (noproxy) 30 | lineinfile: 31 | dest: /etc/sysconfig/origin-master 32 | regexp: '^(#|)NO_PROXY=' 33 | line: "NO_PROXY=localhost,127.0.0.1,::1,{{ ansible_default_ipv4.address }},{{ ansible_hostname }},.dev.opencontrail.org,.compute.internal,.cluster.local,kubernetes.default.svc" 34 | notify: 35 | - restart openshift-master 36 | 37 | - name: openshift-master proxy configuration (http) 38 | lineinfile: 39 | dest: /etc/sysconfig/origin-master 40 | regexp: '^(#|)HTTP_PROXY=' 41 | line: "HTTP_PROXY=http://web-proxy.VAR_AWS_REGION.dev.opencontrail.org:3128" 42 | notify: 43 | - restart openshift-master 44 | 45 | - name: openshift-master proxy configuration (https) 46 | lineinfile: 47 | dest: /etc/sysconfig/origin-master 48 | regexp: '^(#|)HTTPS_PROXY=' 49 | line: "HTTPS_PROXY=http://web-proxy.VAR_AWS_REGION.dev.opencontrail.org:3128" 50 | notify: 51 | - restart openshift-master 52 | 53 | 54 | handlers: 55 | - name: restart openshift-master 56 | service: name=origin-master state=restarted 57 | 58 | - hosts: 59 | - gateways 60 | sudo: yes 61 | tasks: 62 | - name: modify resolv.conf 63 | lineinfile: 64 | dest: /etc/resolv.conf 65 | regexp: '^nameserver' 66 | line: "nameserver {{ opencontrail_dns_forwarder }}" 67 | 68 | # OpenShift is automatically starting the KubeProxy on the nodes which adds 69 | # nat rules to the service addresses. This creates a problem when the docker deamon 70 | # needs to talk to the docker-registry service. 71 | - hosts: 72 | - nodes 73 | sudo: yes 74 | tasks: 75 | - name: Flush nat rules 76 | command: iptables -t nat -F 77 | - name: Delete nat tables 78 | command: iptables -t nat -X 79 | 80 | - hosts: 81 | - all 82 | sudo: yes 83 | # workaround for issue with 'delegate_to' used to provision the virtual-router object. 84 | serial: 1 85 | roles: 86 | - openshift_facts 87 | - opencontrail_facts 88 | - opencontrail_provision 89 | vars: 90 | opencontrail_cluster_type: openshift 91 | -------------------------------------------------------------------------------- /roles/opencontrail/tasks/nodes.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # note: ansible localhost requires python-netaddr to be installed 4 | 5 | - name: Redhat-style interface configuration 6 | include: nodes-redhat.yml 7 | when: ansible_os_family == "RedHat" 8 | 9 | - name: Agent configuration 10 | template: src=contrail-vrouter-agent.conf.node.j2 dest=/etc/contrail/contrail-vrouter-agent.conf 11 | 12 | - name: Ensure log directory exists 13 | file: path=/var/log/contrail state=directory 14 | 15 | - name: Create vrouter agent service 16 | template: src=contrail-vrouter-agent.service.j2 dest=/etc/systemd/system/contrail-vrouter-agent.service 17 | notify: 18 | - reload systemd 19 | when: not (ansible_distribution == "Ubuntu" and ansible_distribution_major_version|int < 15) 20 | 21 | - name: VRouter agent upstart 22 | copy: 23 | src: contrail-vrouter-agent.upstart 24 | dest: /etc/init/contrail-vrouter-agent.conf 25 | mode: 0644 26 | when: ansible_distribution == "Ubuntu" and ansible_distribution_major_version|int < 15 27 | 28 | - name: Enable vhost0 interface 29 | command: ifup vhost0 30 | 31 | - name: Docker image ids (nodes) 32 | command: docker images -q "{{ item }}" 33 | with_items: 34 | - "opencontrail/vrouter-agent:{{ opencontrail_all_release }}" 35 | register: _docker_image_ids 36 | always_run: true 37 | 38 | - name: Pull docker images (nodes) 39 | command: docker pull "{{ item.item }}" 40 | with_items: _docker_image_ids.results 41 | when: not item.stdout 42 | 43 | - name: Start vrouter agent 44 | service: name=contrail-vrouter-agent enabled=yes state=started 45 | 46 | # python-pip is provided by the epel repository on CentOS 7. 47 | - name: epel 7 48 | yum: name="http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm" state=present 49 | when: ansible_distribution == "CentOS" and ansible_distribution_major_version == "7" 50 | 51 | - name: Ensure python pip is installed 52 | action: "{{ ansible_pkg_mgr }}" 53 | args: 54 | name: python-pip 55 | state: latest 56 | 57 | - name: bridge-utils 58 | action: "{{ ansible_pkg_mgr }}" 59 | args: 60 | name: bridge-utils 61 | state: latest 62 | 63 | - name: ethtool 64 | action: "{{ ansible_pkg_mgr }}" 65 | args: 66 | name: ethtool 67 | state: latest 68 | 69 | - name: Install plugin 70 | pip: name=opencontrail-kubelet extra_args="{% if opencontrail_build_http_proxy %}--proxy={{ opencontrail_build_http_proxy }}{% endif %}" 71 | 72 | - name: Create plugin directory 73 | file: 74 | path: /usr/libexec/kubernetes/kubelet-plugins/net/exec/opencontrail 75 | state: directory 76 | 77 | - name: Default plugin path 78 | set_fact: 79 | opencontrail_plugin_path: "/usr/bin/opencontrail-kubelet-plugin" 80 | 81 | - name: Check for plugin in /usr/local/bin 82 | stat: path="/usr/local/bin/opencontrail-kubelet-plugin" 83 | register: usr_local_stat 84 | 85 | - name: Override plugin path 86 | set_fact: 87 | opencontrail_plugin_path: "/usr/local/bin/opencontrail-kubelet-plugin" 88 | when: usr_local_stat.stat.exists 89 | 90 | - name: Create symlink for plugin 91 | file: 92 | src: "{{ opencontrail_plugin_path }}" 93 | dest: /usr/libexec/kubernetes/kubelet-plugins/net/exec/opencontrail/opencontrail 94 | state: link 95 | -------------------------------------------------------------------------------- /test/ec2-k8s/roles/workspace/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create container-networking-ansible tar 3 | local_action: command tar zcvf container-networking-ansible.tgz -C "{{ inventory_dir }}/../.." --exclude test --exclude .git . 4 | 5 | - name: tmp directory 6 | file: path="{{ path_tmp }}" state=directory 7 | 8 | - name: Download kubernetes release 9 | get_url: url=https://github.com/kubernetes/kubernetes/releases/download/v1.1.1/kubernetes.tar.gz dest="{{ path_tmp }}" 10 | # creates: "{{ path_tmp }}/kubernetes.tar.gz" 11 | 12 | - name: Unpack kubernetes tarball 13 | unarchive: 14 | src: "{{ path_tmp }}/kubernetes.tar.gz" 15 | dest: "{{ path_tmp }}" 16 | copy: no 17 | creates: "{{ path_tmp }}/kubernetes/server/kubernetes-server-linux-amd64.tar.gz" 18 | 19 | - name: Unpack linux binaries 20 | unarchive: 21 | src: "{{ path_tmp }}/kubernetes/server/kubernetes-server-linux-amd64.tar.gz" 22 | dest: "{{ path_tmp }}" 23 | copy: no 24 | creates: "{{ path_tmp }}/kubernetes/server/bin/kube-apiserver" 25 | 26 | - name: Ansible configuration file 27 | copy: src=ansible.cfg dest="~/.ansible.cfg" 28 | 29 | - name: Code directory 30 | file: path="{{ path_src }}" state=directory 31 | 32 | - name: Fetch k8s provisioning scripts 33 | git: repo=https://github.com/kubernetes/contrib.git dest="{{ path_src }}/contrib" update=no version=710435a 34 | 35 | - name: Cleanup opencontrail roles 36 | file: path="{{ path_src }}/contrib/ansible/role/{{ item }}" state=absent 37 | with_items: 38 | - opencontrail_facts 39 | - opencontrail 40 | - opencontrail_provision 41 | 42 | - name: Extract opencontrail roles 43 | unarchive: src=container-networking-ansible.tgz dest="{{ path_src }}/contrib/ansible" 44 | 45 | - name: Copy inventory file 46 | copy: src="{{ inventory_dir }}/inventory.cluster" dest="{{ path_src }}/contrib/ansible/inventory" 47 | 48 | - name: Copy playbooks 49 | copy: src="../common/{{ item }}" dest="{{ path_src }}/contrib/ansible" 50 | with_items: 51 | - resolution.yml 52 | - validate.yml 53 | - examples.yml 54 | 55 | - name: Enable opencontrail in group_vars 56 | lineinfile: 57 | dest: "{{ path_src }}/contrib/ansible/group_vars/all.yml" 58 | regexp: "^networking: " 59 | line: "networking: opencontrail" 60 | 61 | # ansible 2.x changed variable precendence such that group_vars has higher precendence than 62 | # variables declared in the inventory. So copy the value from inventory to group_vars. 63 | - name: Get the value of kube_service_addresses from the inventory file 64 | command: awk '{ if (match($0, /^kube_service_addresses\s*=\s*(.*)/, groups) != 0) { print groups[1] } }' {{ path_src }}/contrib/ansible/inventory 65 | register: _inventory_svc_address 66 | when: ansible_version.major >= 2 67 | 68 | - name: Set kube_service_addresses 69 | lineinfile: 70 | dest: "{{ path_src }}/contrib/ansible/group_vars/all.yml" 71 | regexp: "^kube_service_addresses: " 72 | line: "kube_service_addresses: {{ _inventory_svc_address.stdout }}" 73 | when: ansible_version.major >= 2 74 | 75 | - name: Patch cluster.yml 76 | patch: 77 | src: cluster.patch 78 | dest: "{{ path_src }}/contrib/ansible/cluster.yml" 79 | 80 | # - name: Configure address resolution 81 | # command: ansible-playbook -i inventory resolution.yml 82 | # args: 83 | # chdir: "{{ path_src }}/contrib/ansible" 84 | -------------------------------------------------------------------------------- /roles/opencontrail_provision/tasks/master.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Ensure API server is running 3 | wait_for: port=8082 4 | 5 | - name: Create temporary directory 6 | command: mktemp -d 7 | register: mktemp_result 8 | always_run: yes 9 | 10 | - name: Temporary directory var 11 | set_fact: 12 | host_tmp_dir: "{{ mktemp_result.stdout }}" 13 | always_run: yes 14 | 15 | - name: Namespace file 16 | copy: src=namespace.yml dest={{ host_tmp_dir }} 17 | 18 | - name: Namespace status 19 | command: kubectl get namespace opencontrail 20 | ignore_errors: yes 21 | register: namespace 22 | 23 | - name: Create namespace 24 | command: kubectl create -f "{{ host_tmp_dir }}/namespace.yml" 25 | when: namespace is defined and namespace.rc != 0 26 | 27 | - name: Cleanup 28 | file: path="{{ host_tmp_dir }}" state=absent 29 | always_run: yes 30 | 31 | - set_fact: 32 | opencontrail_config_http_proxy_host: "{{ opencontrail_http_proxy | regex_replace('[a-z]+://([a-zA-Z0-9][a-zA-Z0-9\\.\\-_]{0,62})(:[0-9]+)?', '\\\\1') }}" 33 | when: opencontrail_http_proxy is defined 34 | 35 | - set_fact: 36 | opencontrail_config_http_proxy_address: "{{ opencontrail_config_http_proxy_host | ipaddr('address') }}" 37 | when: opencontrail_config_http_proxy_host is defined 38 | 39 | # lookup module dig requires python-dns package 40 | - set_fact: 41 | opencontrail_config_http_proxy_address: "{{ lookup('dig', opencontrail_config_http_proxy_host) }}" 42 | when: opencontrail_config_http_proxy_host is defined and not opencontrail_config_http_proxy_address|bool 43 | 44 | - name: kubernetes api service address 45 | command: > 46 | docker run --net=host opencontrail/config:2.20 /usr/share/contrail-utils/provision_linklocal.py 47 | --api_server_ip localhost --api_server_port 8082 48 | --linklocal_service_name kubernetes-ssl 49 | --linklocal_service_ip {{ opencontrail_all_service_addresses | ipaddr('net') | ipaddr(1) | ipaddr('address')}} 50 | --linklocal_service_port 443 51 | --ipfabric_service_ip "{{ opencontrail_host_address }}" 52 | --ipfabric_service_port {{ opencontrail_kube_master_port }} --oper add 53 | 54 | - name: DNS 55 | command: > 56 | docker run --net=host opencontrail/config:2.20 /usr/share/contrail-utils/provision_linklocal.py 57 | --api_server_ip localhost --api_server_port 8082 58 | --linklocal_service_name dns 59 | --linklocal_service_ip {{ opencontrail_all_service_addresses | ipaddr('net') | ipaddr(1) | ipaddr('address')}} 60 | --linklocal_service_port 53 61 | --ipfabric_service_ip "{{ opencontrail_config_dns_forwarder }}" 62 | --ipfabric_service_port 53 --oper add 63 | when: openshift is defined 64 | 65 | - debug: var=opencontrail_config_http_proxy_address 66 | 67 | - name: WebProxy 68 | command: > 69 | docker run --net=host opencontrail/config:2.20 /usr/share/contrail-utils/provision_linklocal.py 70 | --api_server_ip localhost --api_server_port 8082 71 | --linklocal_service_name web-proxy 72 | --linklocal_service_ip {{ opencontrail_all_service_addresses | ipaddr('net') | ipaddr(2) | ipaddr('address')}} 73 | --linklocal_service_port 3128 --ipfabric_service_ip "{{ opencontrail_config_http_proxy_address }}" 74 | --ipfabric_service_port 3128 --oper add 75 | when: opencontrail_config_http_proxy_address is defined and opencontrail_config_http_proxy_address != "" 76 | 77 | - name: Provision controller 78 | command: > 79 | docker run --net=host opencontrail/config:2.20 /usr/share/contrail-utils/provision_control.py 80 | --api_server_ip localhost --api_server_port 8082 81 | --host_name "{{ ansible_hostname }}" --host_ip "{{ opencontrail_host_address }}" 82 | --router_asn 64512 --oper add 83 | -------------------------------------------------------------------------------- /roles/opencontrail/README.md: -------------------------------------------------------------------------------- 1 | OpenContrail playbook role 2 | ========================== 3 | 4 | This playbook installs opencontrail on k8s/openshift masters and nodes. It also (optionally) 5 | installs a software gateway that provides connectivity between the underlay and the Public network 6 | as well as connectivity between the internal underlay network where masters and minions are present 7 | and a system network such as kube-system/default. 8 | 9 | ## User settings 10 | 11 | ### Per-host settings 12 | 13 | Variables that can be set on a per host basis on the inventory file. Example: 14 | ``` 15 | [nodes] 16 | kube-contrail-node-01 opencontrail_interface=eth1 17 | ``` 18 | 19 | Variable | Description | Default 20 | ----------|-------------|--------- 21 | opencontrail_interface | Physical interface used by the vrouter kernel module | eth0 22 | opencontrail_ipaddr | IP address prefix | address of vhost0 when present, or the address of opencontrail_interface 23 | opencontrail_gateway | IP address | default router on vhost0 interface 24 | 25 | ### Global settings 26 | 27 | Variables set in inventory file. 28 | 29 | The inventory file should define a variable group such as: 30 | ``` 31 | [opencontrail:children] 32 | masters 33 | nodes 34 | gateways 35 | 36 | [opencontrail:vars] 37 | opencontrail_public_subnet = 192.168.254.0/24 38 | [...] 39 | ``` 40 | 41 | | Variable | Description | Default value | 42 | |----------|-------------|---------------| 43 | | opencontrail_public_subnet | IP subnet of the Public network | (mandatory) | 44 | | opencontrail_private_subnet | IP subnet for private IP addreses | optional | 45 | | opencontrail_http_proxy | Proxy used by kmod builder | optional | 46 | | opencontrail_dns_forwarder| DNS forwarder | optional | 47 | | opencontrail_use_systemd | TODO: Use systemd to start docker containers | true | 48 | | opencontrail_release | TODO: Software release to install | 2.20 | 49 | 50 | ## Playbook 51 | 52 | The objective of this playbook is that the opencontrail (and opencontrail_provision) roles should be usable when called from the kubernetes playbook, openshift or a standalone playbook such as: 53 | 54 | ``` 55 | - hosts: all 56 | sudo: yes 57 | roles: opencontrail 58 | vars: 59 | opencontrail_cluster_type: x 60 | # ... other vars ... 61 | ``` 62 | 63 | The *opencontrail* role has the assumption that docker is installed and running in the hosts. 64 | The *opencontrail_provision* role has the assumption that both the kubernetes apiserver and the contrail-api are running and accepting requests. 65 | 66 | ## Variables used by the opencontrail playbook 67 | 68 | The interface configuration facts can be established by: 69 | - explicit configuration in the inventory; 70 | - examining the physical interface (before the vhost0 interface is configured); 71 | - examining the vhost0 interface (after the vhost0 interface is configured) 72 | 73 | |Variable | Description | 74 | |---------| ---- | 75 | | opencontrail_cluster_type | {kubernetes, openshift} | 76 | | opencontrail_host_interface | physical interface for vrouter | 77 | | opencontrail_host_ipaddr | IP address prefix for the vhost0 interface | 78 | | opencontrail_host_address | IP address of the vhost0 interface | 79 | | opencontrail_host_netmask | IP netmask of the vhost0 interface | 80 | | opencontrail_host_gateway | Default router, when the default route is through vhost0 | 81 | 82 | The following are determined from the variables passed into the role by either ansible_facts or the playbook predecessor tasks. 83 | 84 | |Variable | Description | 85 | |---------| ---- | 86 | | opencontrail_host_kernel_tag | Kernel version | 87 | | opencontrail_all_service_addresses | ClusterIP range | 88 | | opencontrail_all_release | | 89 | | opencontrail_master_ifmap_port | 8444 (openshift) | 90 | 91 | ## Pre-requisites 92 | 93 | ### kubernetes 94 | - ansible_facts 95 | 96 | ### openshift 97 | - openshift_facts 98 | -------------------------------------------------------------------------------- /test/common/library/ec2_vpc_rtb_update.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | # Module that updates the routing table of a VPC. 4 | # 5 | # It updates (create or replace routes pointing to instance). 6 | # It ignores gateway routes and doesn't flush instance routes that are no 7 | # longer specified. 8 | # 9 | 10 | # import module snippets 11 | from ansible.module_utils.basic import * 12 | from ansible.module_utils.ec2 import * 13 | 14 | import boto.vpc 15 | 16 | 17 | def rtb_update(connection, rtb, routes): 18 | """ Update the table in order to ensure that the route is present """ 19 | changed = False 20 | for route in routes: 21 | if route.get('gw') == 'igw': 22 | continue 23 | 24 | existing_rt = filter( 25 | lambda x: x.destination_cidr_block == route['dest'], rtb.routes) 26 | 27 | if len(existing_rt) > 0: 28 | if existing_rt[0].instance_id == route.get('gw'): 29 | continue 30 | success = connection.replace_route( 31 | rtb.id, route['dest'], instance_id=route.get('gw')) 32 | else: 33 | success = connection.create_route( 34 | rtb.id, route['dest'], instance_id=route.get('gw')) 35 | if success: 36 | changed = True 37 | return changed 38 | 39 | 40 | def rtb_delete(connection, rtb, routes): 41 | """ Delete a set of routes from the table """ 42 | changed = False 43 | for route in routes: 44 | if route.get('gw') == 'igw': 45 | continue 46 | 47 | existing_rt = filter( 48 | lambda x: x.destination_cidr_block == route['dest'], rtb.routes) 49 | if len(existing_rt) == 0: 50 | continue 51 | 52 | connection.delete_route(rtb.id, route['dest']) 53 | changed = True 54 | 55 | return changed 56 | 57 | 58 | def main(): 59 | argument_spec = ec2_argument_spec() 60 | argument_spec.update(dict( 61 | vpc_id=dict(required=True), 62 | subnets=dict(type='list', required=True), 63 | routes=dict(type='list'), 64 | state=dict(choices=['present', 'absent'], default='present') 65 | )) 66 | 67 | module = AnsibleModule(argument_spec=argument_spec) 68 | 69 | ec2_url, aws_access_key, aws_secret_key, region = get_ec2_creds(module) 70 | 71 | if not region: 72 | module.fail_json(msg="region must be specified") 73 | 74 | try: 75 | connection = boto.vpc.connect_to_region( 76 | region, 77 | aws_access_key_id=aws_access_key, 78 | aws_secret_access_key=aws_secret_key) 79 | except boto.exception.NoAuthHandlerFound, e: 80 | module.fail_json(msg=str(e)) 81 | 82 | tables = connection.get_all_route_tables( 83 | filters={'vpc_id': module.params.get('vpc_id')} 84 | ) 85 | 86 | def match_by_subnets(t): 87 | subnet_ids = map(lambda x: x.subnet_id, t.associations) 88 | return set(subnet_ids) == set(module.params.get('subnets')) 89 | 90 | selected_tables = filter(match_by_subnets, tables) 91 | 92 | if len(selected_tables) != 1: 93 | if len(selected_tables) > 1: 94 | module.fail_json(msg="Multiple route tables selected") 95 | 96 | rtb = connection.create_route_table(module.params.get('vpc_id')) 97 | for subnet_id in module.params.get('subnets'): 98 | connection.associate_route_table(rtb.id, subnet_id) 99 | else: 100 | rtb = selected_tables[0] 101 | 102 | changed = False 103 | if module.params.get('state') == 'present': 104 | changed = rtb_update(connection, rtb, module.params.get('routes')) 105 | elif module.params.get('state') == 'absent': 106 | changed = rtb_delete(connection, rtb, module.params.get('routes')) 107 | 108 | module.exit_json(changed=changed, rtb_id=rtb.id) 109 | 110 | main() 111 | -------------------------------------------------------------------------------- /test/ec2-k8s/roles/basic/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Locate the VPC 3 | ec2_vpc_facts: 4 | region: "{{ aws_region }}" 5 | resource_tags: 6 | "Name": opencontrail-ci-vpc 7 | 8 | - set_fact: 9 | ec2_public_subnet: "{{ ec2_vpc.subnets | selectattr('tags', 'equalto', dict(Name='opencontrail-ci-public')) | first}}" 10 | k8s_public_subnet: "{% if job_id is defined %}172.16.{{ 8 * (job_id|int % 32) }}.0/24{% else %}172.18.0.0/20{% endif %}" 11 | k8s_service_addresses: "{% if job_id is defined %}10.{{ 192 + (job_id|int % 32) }}.0.0/16{% else %}10.64.0.0/16{% endif %}" 12 | 13 | - name: Deployer security-group 14 | ec2_group: 15 | name: k8s-basic-deployer 16 | description: "k8s deployer security-group" 17 | region: "{{ aws_region }}" 18 | vpc_id: "{{ ec2_vpc.id }}" 19 | rules: 20 | - proto: tcp 21 | from_port: 22 22 | to_port: 22 23 | cidr_ip: 0.0.0.0/0 24 | rules_egress: 25 | - proto: all 26 | cidr_ip: 0.0.0.0/0 27 | register: sg_management 28 | 29 | - name: Cluster security-group 30 | ec2_group: 31 | name: k8s-basic-sg 32 | description: "k8s cluster security-group" 33 | vpc_id: "{{ ec2_vpc.id }}" 34 | region: "{{ aws_region }}" 35 | rules: 36 | - proto: all 37 | group_name: k8s-basic-sg 38 | - proto: all 39 | group_id: "{{ sg_management.group_id }}" 40 | rules_egress: 41 | - proto: all 42 | cidr_ip: 0.0.0.0/0 43 | register: sg_cluster 44 | 45 | - name: Management host 46 | ec2: 47 | image: "{{ ec2_image }}" 48 | key_name: k8s 49 | instance_tags: 50 | Name: "k8s-mgmt-{{ job_id | default('00') }}" 51 | Cluster: "k8s-ansible-{{ job_id | default('00') }}" 52 | instance_type: m3.medium 53 | region: "{{ aws_region }}" 54 | vpc_subnet_id: "{{ ec2_public_subnet.id }}" 55 | group_id: "{{ sg_management.group_id }}" 56 | assign_public_ip: yes 57 | count_tag: 58 | Name: "k8s-mgmt-{{ job_id | default('00') }}" 59 | exact_count: 1 60 | wait: true 61 | register: k8s_management 62 | 63 | - add_host: name='{{ k8s_management.tagged_instances[0].public_dns_name }}' groups=deployer ansible_ssh_user="{{ ssh_user }}" 64 | 65 | - name: Create gateway instance 66 | ec2: 67 | image: "{{ ec2_image }}" 68 | key_name: k8s 69 | instance_tags: 70 | Name: "k8s-gateway-{{ job_id | default('00') }}" 71 | Cluster: "k8s-ansible-{{ job_id | default('00') }}" 72 | instance_type: m3.medium 73 | region: "{{ aws_region }}" 74 | vpc_subnet_id: "{{ ec2_public_subnet.id }}" 75 | group_id: "{{ sg_cluster.group_id }}" 76 | assign_public_ip: yes 77 | source_dest_check: no 78 | wait: yes 79 | count_tag: 80 | Name: "k8s-gateway-{{ job_id | default('00') }}" 81 | exact_count: 1 82 | register: k8s_gateway 83 | 84 | - name: Create master instance 85 | ec2: 86 | image: "{{ ec2_image }}" 87 | key_name: k8s 88 | instance_tags: 89 | Name: "k8s-master-{{ job_id | default('00') }}" 90 | Cluster: "k8s-ansible-{{ job_id | default('00') }}" 91 | instance_type: m3.large 92 | region: "{{ aws_region }}" 93 | vpc_subnet_id: "{{ ec2_public_subnet.id }}" 94 | group_id: "{{ sg_cluster.group_id }}" 95 | assign_public_ip: yes 96 | count_tag: 97 | Name: "k8s-master-{{ job_id | default('00') }}" 98 | exact_count: 1 99 | register: k8s_master 100 | 101 | - name: Create nodes 102 | ec2: 103 | image: "{{ ec2_image }}" 104 | key_name: k8s 105 | instance_tags: 106 | Name: "k8s-node-{{ job_id | default('00') }}" 107 | Cluster: "k8s-ansible-{{ job_id | default('00') }}" 108 | instance_type: m3.medium 109 | region: "{{ aws_region }}" 110 | vpc_subnet_id: "{{ ec2_public_subnet.id }}" 111 | group_id: "{{ sg_cluster.group_id }}" 112 | assign_public_ip: yes 113 | count_tag: 114 | Name: "k8s-node-{{ job_id | default('00') }}" 115 | exact_count: 2 116 | register: k8s_nodes 117 | 118 | - name: Update the vpc routing table 119 | ec2_vpc_rtb_update: 120 | region: "{{ aws_region }}" 121 | vpc_id: "{{ ec2_vpc.id }}" 122 | subnets: 123 | - "{{ ec2_public_subnet.id }}" 124 | routes: 125 | - dest: 0.0.0.0/0 126 | gw: igw 127 | - dest: "{{ k8s_public_subnet }}" 128 | gw: "{{ k8s_gateway.tagged_instances[0].id }}" 129 | - dest: "{{ k8s_service_addresses }}" 130 | gw: "{{ k8s_gateway.tagged_instances[0].id }}" 131 | 132 | - name: Store the status of the cluster 133 | template: src=status.j2 dest="{{ inventory_dir }}/cluster.status" 134 | 135 | - name: Create inventory file 136 | template: src=inventory.j2 dest="{{ inventory_dir }}/inventory.cluster" 137 | 138 | - wait_for: host="{{ k8s_management.tagged_instances[0].public_dns_name }}" port=22 139 | 140 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # container-networking-ansible 2 | Ansible provisioning for container networking solutions using OpenContrail 3 | 4 | This repository contains provisioning instructions to install OpenContrail 5 | as a network overlay for container based cluster management solutions. 6 | 7 | The test directory defines a jenkins workflow that creates and 8 | installs a test cluster and executes an application within the 9 | cluster. 10 | 11 | For support/questions: 12 | - Join the slack team at `slack.opencontrail.org` 13 | - Developers mailing list: dev@lists.opencontrail.org 14 | 15 | The opencontrail playbook consists of the following: 16 | - filter_plugins/ip_filters.py 17 | - roles/opencontrail{,_facts,_provision} 18 | 19 | The playbooks are designed to be addons to the existing ansible provisioning for kubernetes and openshift. 20 | 21 | ### Kubernetes 22 | 23 | #### Network segmentation and access control 24 | When opencontrail is used as the kubernetes network plugin, it defaults to isolate all pods according to `namespace` and a user defined tag. External traffic is restricted to services that are annotated with a ExternalIP address or have "type" set to "LoadBalancer". This causes the opencontrail public to allocate an address on the public network and assign it to all the pods in this service. 25 | 26 | Services in the `kube-system` namespace are also available to all Pods, irrespective of the namespace of the pod. This is configured via the `cluster-service` option in /etc/kubernetes/network.conf. The cluster-service network is also connected to the underlay network where masters and nodes are present. 27 | 28 | Pods are expected to communicate with the master via its ClusterIP address. 29 | 30 | #### Deployment 31 | The kubernetes ansible playbook at https://github.com/kubernetes/contrib. 32 | 33 | - edit ansible/group_vars/all.yml 34 | ``` 35 | networking: opencontrail 36 | ``` 37 | 38 | - inventory file: 39 | ``` 40 | [opencontrail:children] 41 | masters 42 | nodes 43 | gateways 44 | 45 | [opencontrail:vars] 46 | opencontrail_public_subnet=192.0.2.0/24 47 | opencontrail_kube_release=1.1 48 | 49 | ``` 50 | 51 | - patch ansible/cluster.yml according to: 52 | https://github.com/kubernetes/contrib/pull/261 53 | 54 | - run the ansible/cluster.yml playbook (e.g. via ansible/setup.sh) 55 | 56 | ### OpenShift 57 | 58 | #### Network segmentation and access control 59 | 60 | There are several differences in design from a plain-vanilla kubernetes cluster deployment and an openshift deployment: 61 | - OpenShift expects all external traffic to be delivered through the router service. The openshift router pod is a TCP load-balancer (ha-proxy by default) that performs SSL termination and delivers traffic to the pods that implement the service. 62 | - OpenShift pods (builder/deployer) have the nasty habbit of trying to reach the master through its infrastructure IP address (rather than using the ClusterIP). 63 | - OpenShift STI builder pods expect to be able to access external git repositories as well as package repositories for popular languages (python, ruby, etc...). 64 | - OpenShift builder pods use the docker daemon in the node and expect it to be able to talk to the docker-repository service running as a pod (in the overlay). 65 | - Deployer pods expect to be able to pull images from the docker-repository into the node docker daemon. 66 | 67 | * In current test scripts, we expect the builder pods to use an http proxy in order to fetch software packages. The builder pods are spawned in the namespace of the user `project`. To provide direct external access, one would need to do so for all pods currently. Future versions of the contrail-kubernetes plugin should support source-nat for outbound access to the public network. It is also possible to add a set of prefixes that contain the software and artifact repositories used by the builder to the global `cluster-service` network. 68 | * All the traffic between underlay and overlay is expected to occur based on the `cluster-service` gateway configured for ```default:default``` 69 | 70 | #### Deployment 71 | - inventory file: 72 | ``` 73 | [OSEv3:children] 74 | masters 75 | nodes 76 | etcd 77 | 78 | # Set variables common for all OSEv3 hosts 79 | [OSEv3:vars] 80 | 81 | use_openshift_sdn = false 82 | sdn_network_plugin_name = opencontrail 83 | 84 | [opencontrail:children] 85 | masters 86 | nodes 87 | gateways 88 | 89 | [opencontrail:vars] 90 | opencontrail_public_subnet=192.0.2.0/24 91 | opencontrail_kube_release=origin-1.1 92 | ``` 93 | 94 | - provision opencontrail with the following playbook: 95 | ``` 96 | - hosts: 97 | - masters 98 | - nodes 99 | - gateways 100 | sudo: yes 101 | roles: 102 | - openshift_facts 103 | - opencontrail_facts 104 | - opencontrail 105 | vars: 106 | opencontrail_cluster_type: openshift 107 | tags: 108 | - opencontrail 109 | ``` 110 | 111 | - patch openshift-ansible with the following delta: 112 | https://github.com/openshift/openshift-ansible/compare/master...pedro-r-marques:opencontrail 113 | 114 | - install openshift via the ansible playbook 115 | - run the opencontrail_provision role 116 | -------------------------------------------------------------------------------- /test/ec2-origin/roles/cluster/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Locate the VPC 3 | ec2_vpc_facts: 4 | region: "{{ aws_region }}" 5 | resource_tags: 6 | "Name": opencontrail-ci-vpc 7 | 8 | - set_fact: 9 | ec2_public_subnet: "{{ ec2_vpc.subnets | selectattr('tags', 'equalto', dict(Name='opencontrail-ci-public')) | first}}" 10 | ec2_private_subnet: "{{ ec2_vpc.subnets | selectattr('tags', 'equalto', dict(Name='opencontrail-ci-private')) | first}}" 11 | 12 | - name: IP addressing 13 | set_fact: 14 | cluster_public_subnet: "{% if job_id is defined %}172.20.{{ 8 * (job_id|int % 32) }}.0/24{% else %}172.18.64.0/20{% endif %}" 15 | cluster_service_addresses: "{% if job_id is defined %}10.{{ 160 + (job_id|int % 32) }}.0.0/16{% else %}10.65.0.0/16{% endif %}" 16 | 17 | - name: Deployer security-group 18 | ec2_group: 19 | name: origin-deployer-group 20 | description: "origin deployer security-group" 21 | region: "{{ aws_region }}" 22 | vpc_id: "{{ ec2_vpc.id }}" 23 | rules: 24 | - proto: tcp 25 | from_port: 22 26 | to_port: 22 27 | cidr_ip: 0.0.0.0/0 28 | - proto: udp 29 | from_port: 53 30 | to_port: 53 31 | cidr_ip: 10.0.0.0/8 32 | - proto: tcp 33 | from_port: 53 34 | to_port: 53 35 | cidr_ip: 10.0.0.0/8 36 | rules_egress: 37 | - proto: all 38 | cidr_ip: 0.0.0.0/0 39 | register: sg_deployer 40 | 41 | - name: Cluster security-group 42 | ec2_group: 43 | name: origin-cluster-group 44 | description: "k8s cluster security-group" 45 | vpc_id: "{{ ec2_vpc.id }}" 46 | region: "{{ aws_region }}" 47 | rules: 48 | - proto: all 49 | group_name: origin-cluster-group 50 | - proto: all 51 | group_id: "{{ sg_deployer.group_id }}" 52 | rules_egress: 53 | - proto: all 54 | cidr_ip: 0.0.0.0/0 55 | register: sg_cluster 56 | 57 | - name: Deployer 58 | ec2: 59 | image: "{{ ec2_image }}" 60 | key_name: k8s 61 | instance_tags: 62 | Name: "origin-deployer-{{ job_id | default('00') }}" 63 | Cluster: "origin-ansible-{{ job_id | default('00') }}" 64 | instance_type: t2.micro 65 | region: "{{ aws_region }}" 66 | vpc_subnet_id: "{{ ec2_public_subnet.id }}" 67 | group_id: "{{ sg_deployer.group_id }}" 68 | assign_public_ip: yes 69 | count_tag: 70 | Name: "origin-deployer-{{ job_id | default('00') }}" 71 | exact_count: 1 72 | wait: true 73 | register: origin_deployer 74 | 75 | - name: Add deployer to inventory 76 | add_host: name='{{ origin_deployer.tagged_instances[0].public_dns_name }}' groups=deployer ansible_ssh_user="{{ ssh_user }}" 77 | 78 | - name: Create gateway instance 79 | ec2: 80 | image: "{{ ec2_image }}" 81 | key_name: k8s 82 | instance_tags: 83 | Name: "origin-gateway-{{ job_id | default('00') }}" 84 | Cluster: "origin-ansible-{{ job_id | default('00') }}" 85 | instance_type: m3.medium 86 | region: "{{ aws_region }}" 87 | vpc_subnet_id: "{{ ec2_public_subnet.id }}" 88 | group_id: "{{ sg_cluster.group_id }}" 89 | assign_public_ip: yes 90 | source_dest_check: no 91 | wait: yes 92 | count_tag: 93 | Name: "origin-gateway-{{ job_id | default('00') }}" 94 | exact_count: 1 95 | volumes: 96 | - device_name: /dev/xvdf 97 | delete_on_termination: true 98 | volume_type: generic 99 | volume_size: 40 100 | register: origin_gateway 101 | 102 | - name: Create master instance 103 | ec2: 104 | image: "{{ ec2_image }}" 105 | key_name: k8s 106 | instance_tags: 107 | Name: "origin-master-{{ job_id | default('00') }}" 108 | Cluster: "origin-ansible-{{ job_id | default('00') }}" 109 | instance_type: m3.large 110 | region: "{{ aws_region }}" 111 | vpc_subnet_id: "{{ ec2_private_subnet.id }}" 112 | group_id: "{{ sg_cluster.group_id }}" 113 | count_tag: 114 | Name: "origin-master-{{ job_id | default('00') }}" 115 | exact_count: 1 116 | volumes: 117 | - device_name: /dev/xvdf 118 | delete_on_termination: true 119 | volume_type: generic 120 | volume_size: 80 121 | register: origin_master 122 | 123 | - name: Add instance to masters group 124 | add_host: name=origin-master-1 groups=masters instance_ip="{{ origin_master.tagged_instances[0].private_ip }}" 125 | 126 | - name: Create nodes 127 | ec2: 128 | image: "{{ ec2_image }}" 129 | key_name: k8s 130 | instance_tags: 131 | Name: "origin-node-{{ job_id | default('00') }}" 132 | Cluster: "origin-ansible-{{ job_id | default('00') }}" 133 | instance_type: m3.medium 134 | region: "{{ aws_region }}" 135 | vpc_subnet_id: "{{ ec2_private_subnet.id }}" 136 | group_id: "{{ sg_cluster.group_id }}" 137 | count_tag: 138 | Name: "origin-node-{{ job_id | default('00') }}" 139 | exact_count: 2 140 | volumes: 141 | - device_name: /dev/xvdf 142 | delete_on_termination: true 143 | volume_type: generic 144 | volume_size: 80 145 | register: origin_nodes 146 | 147 | - name: Update the vpc routing table (private subnet) 148 | ec2_vpc_rtb_update: 149 | region: "{{ aws_region }}" 150 | vpc_id: "{{ ec2_vpc.id }}" 151 | subnets: 152 | - "{{ ec2_private_subnet.id }}" 153 | routes: 154 | - dest: "{{ cluster_service_addresses }}" 155 | gw: "{{ origin_gateway.tagged_instances[0].id }}" 156 | 157 | - name: Store the status of the cluster 158 | template: src=status.j2 dest="{{ inventory_dir }}/cluster.status" 159 | 160 | - name: Create inventory file 161 | template: src=inventory.j2 dest="{{ inventory_dir }}/inventory.cluster" 162 | 163 | - name: Determine jenkins-master information 164 | ec2_remote_facts: 165 | region: "{{ aws_region }}" 166 | filters: 167 | instance-state-name: running 168 | "tag:Name": jenkins-master 169 | register: jenkins_instance 170 | 171 | - name: Add jenkins-instance 172 | add_host: name="{{ jenkins_instance.instances[0].public_dns_name }}" groups=jenkins-master ansible_ssh_user=ubuntu 173 | 174 | - wait_for: host="{{ origin_deployer.tagged_instances[0].public_dns_name }}" port=22 175 | -------------------------------------------------------------------------------- /test/common/library/ec2_remote_facts.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | # This is a free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | # 8 | # This Ansible library is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | # 13 | # You should have received a copy of the GNU General Public License 14 | # along with this library. If not, see . 15 | 16 | DOCUMENTATION = ''' 17 | --- 18 | module: ec2_remote_facts 19 | short_description: Gather facts about ec2 instances in AWS 20 | description: 21 | - Gather facts about ec2 instances in AWS 22 | version_added: "2.0" 23 | options: 24 | filters: 25 | description: 26 | - A dict of filters to apply. Each dict item consists of a filter key and a filter value. See U(http://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstances.html) for possible filters. 27 | required: false 28 | default: null 29 | author: 30 | - "Michael Schuett (@michaeljs1990)" 31 | extends_documentation_fragment: 32 | - aws 33 | - ec2 34 | ''' 35 | 36 | EXAMPLES = ''' 37 | # Note: These examples do not set authentication details, see the AWS Guide for details. 38 | 39 | # Gather facts about all ec2 instances 40 | - ec2_remote_facts: 41 | 42 | # Gather facts about all running ec2 instances with a tag of Name:Example 43 | - ec2_remote_facts: 44 | filters: 45 | instance-state-name: running 46 | "tag:Name": Example 47 | 48 | # Gather facts about instance i-123456 49 | - ec2_remote_facts: 50 | filters: 51 | instance-id: i-123456 52 | 53 | # Gather facts about all instances in vpc-123456 that are t2.small type 54 | - ec2_remote_facts: 55 | filters: 56 | vpc-id: vpc-123456 57 | instance-type: t2.small 58 | 59 | ''' 60 | 61 | try: 62 | import boto.ec2 63 | from boto.exception import BotoServerError 64 | HAS_BOTO = True 65 | except ImportError: 66 | HAS_BOTO = False 67 | 68 | def get_instance_info(instance): 69 | 70 | # Get groups 71 | groups = [] 72 | for group in instance.groups: 73 | groups.append({ 'id': group.id, 'name': group.name }.copy()) 74 | 75 | # Get interfaces 76 | interfaces = [] 77 | for interface in instance.interfaces: 78 | interfaces.append({ 'id': interface.id, 'mac_address': interface.mac_address }.copy()) 79 | 80 | # If an instance is terminated, sourceDestCheck is no longer returned 81 | try: 82 | source_dest_check = instance.sourceDestCheck 83 | except AttributeError: 84 | source_dest_check = None 85 | 86 | instance_info = { 'id': instance.id, 87 | 'kernel': instance.kernel, 88 | 'instance_profile': instance.instance_profile, 89 | 'root_device_type': instance.root_device_type, 90 | 'private_dns_name': instance.private_dns_name, 91 | 'public_dns_name': instance.public_dns_name, 92 | 'ebs_optimized': instance.ebs_optimized, 93 | 'client_token': instance.client_token, 94 | 'virtualization_type': instance.virtualization_type, 95 | 'architecture': instance.architecture, 96 | 'ramdisk': instance.ramdisk, 97 | 'tags': instance.tags, 98 | 'key_name': instance.key_name, 99 | 'source_destination_check': source_dest_check, 100 | 'image_id': instance.image_id, 101 | 'groups': groups, 102 | 'interfaces': interfaces, 103 | 'spot_instance_request_id': instance.spot_instance_request_id, 104 | 'requester_id': instance.requester_id, 105 | 'monitoring_state': instance.monitoring_state, 106 | 'placement': { 107 | 'tenancy': instance._placement.tenancy, 108 | 'zone': instance._placement.zone 109 | }, 110 | 'ami_launch_index': instance.ami_launch_index, 111 | 'launch_time': instance.launch_time, 112 | 'hypervisor': instance.hypervisor, 113 | 'region': instance.region.name, 114 | 'persistent': instance.persistent, 115 | 'private_ip_address': instance.private_ip_address, 116 | 'state': instance._state.name, 117 | 'vpc_id': instance.vpc_id, 118 | } 119 | 120 | return instance_info 121 | 122 | 123 | def list_ec2_instances(connection, module): 124 | 125 | filters = module.params.get("filters") 126 | instance_dict_array = [] 127 | 128 | try: 129 | all_instances = connection.get_only_instances(filters=filters) 130 | except BotoServerError as e: 131 | module.fail_json(msg=e.message) 132 | 133 | for instance in all_instances: 134 | instance_dict_array.append(get_instance_info(instance)) 135 | 136 | module.exit_json(instances=instance_dict_array) 137 | 138 | 139 | def main(): 140 | argument_spec = ec2_argument_spec() 141 | argument_spec.update( 142 | dict( 143 | filters = dict(default=None, type='dict') 144 | ) 145 | ) 146 | 147 | module = AnsibleModule(argument_spec=argument_spec) 148 | 149 | if not HAS_BOTO: 150 | module.fail_json(msg='boto required for this module') 151 | 152 | region, ec2_url, aws_connect_params = get_aws_connection_info(module) 153 | 154 | if region: 155 | try: 156 | connection = connect_to_aws(boto.ec2, region, **aws_connect_params) 157 | except (boto.exception.NoAuthHandlerFound, AnsibleAWSError), e: 158 | module.fail_json(msg=str(e)) 159 | else: 160 | module.fail_json(msg="region must be specified") 161 | 162 | list_ec2_instances(connection, module) 163 | 164 | # import module snippets 165 | from ansible.module_utils.basic import * 166 | from ansible.module_utils.ec2 import * 167 | 168 | if __name__ == '__main__': 169 | main() 170 | -------------------------------------------------------------------------------- /test/jenkins.groovy: -------------------------------------------------------------------------------- 1 | import hudson.AbortException 2 | 3 | ssh_options = '-o StrictHostKeyChecking=no -o ForwardAgent=yes' 4 | 5 | def getDeployerHostname() { 6 | def inventory = readFile('cluster.status') 7 | 8 | def section = false 9 | def hostname 10 | 11 | for (line in inventory.split('\n')) { 12 | if (line == '[deployer]') { 13 | section = true 14 | continue 15 | } 16 | if (section) { 17 | hostname = inventory_match_item(line) 18 | break 19 | } 20 | } 21 | return hostname 22 | } 23 | 24 | @NonCPS 25 | def inventory_match_item(text) { 26 | def matcher = (text =~ /^[\w-_\.]+/) 27 | matcher ? matcher[0] : null 28 | } 29 | 30 | 31 | def getMasterIP() { 32 | def inventory = readFile('inventory.cluster') 33 | 34 | def section = false 35 | def address 36 | 37 | for (line in inventory.split('\n')) { 38 | if (line == '[masters]') { 39 | section = true 40 | continue 41 | } 42 | if (section) { 43 | address = inventory_match_ssh_host(line) 44 | break 45 | } 46 | } 47 | return address 48 | } 49 | 50 | @NonCPS 51 | def inventory_match_ssh_host(text) { 52 | def matcher = (text =~ /^([\w-_\.]+) ansible_ssh_host=([0-9\.]+)/) 53 | matcher ? matcher[0][2] : null 54 | } 55 | 56 | 57 | def k8s_deploy(deployer) { 58 | def playbooks = [ 59 | 'resolution.yml', 60 | 'cluster.yml' 61 | ] 62 | 63 | echo "Start k8s deploy stage on ${deployer}" 64 | 65 | // Use an integer as iterator so that it is serializable. 66 | // The "sh" step requires local variables to serialize. 67 | for (int i = 0; i < playbooks.size(); i++) { 68 | def playbook = playbooks[i] 69 | echo "playbook ${playbook}" 70 | sh "ssh ${ssh_options} ubuntu@${deployer} ansible-playbook -i src/contrib/ansible/inventory src/contrib/ansible/${playbook}" 71 | } 72 | } 73 | 74 | def origin_deploy(deployer) { 75 | def playbooks = [ 76 | 'system-install.yml', 77 | 'opencontrail.yml', 78 | 'config.yml', 79 | 'opencontrail_provision.yml', 80 | 'openshift_provision.yml', 81 | 'applications.yml' 82 | ] 83 | 84 | echo "Start openshift deploy stage on ${deployer}" 85 | 86 | // Use an integer as iterator so that it is serializable. 87 | // The "sh" step requires local variables to serialize. 88 | for (int i = 0; i < playbooks.size(); i++) { 89 | def playbook = playbooks[i] 90 | echo "playbook ${playbook}" 91 | sh "ssh ${ssh_options} centos@${deployer} '(cd src/openshift-ansible; ansible-playbook -i inventory/byo/hosts playbooks/byo/${playbook})'" 92 | // version 1.15 of the script-security plugin allows less-than but not greater-than comparissons 93 | if (0 < i) { 94 | try { 95 | sh "ssh ${ssh_options} centos@${deployer} '(cd src/openshift-ansible; python playbooks/byo/opencontrail_validate.py --stage ${i} inventory/byo/hosts)'" 96 | } catch (AbortException ex) { 97 | // openshift config playbook restarts docker and systemd will fail to restart some of the dependent 98 | // opencontrail services. 99 | if (i == 2) { 100 | sh "ssh ${ssh_options} centos@${deployer} '(cd src/openshift-ansible; ansible-playbook -i inventory/byo/hosts playbooks/byo/systemd_workaround.yml)'" 101 | steps.sleep(60) 102 | sh "ssh ${ssh_options} centos@${deployer} '(cd src/openshift-ansible; python playbooks/byo/opencontrail_validate.py --stage ${i} inventory/byo/hosts)'" 103 | } else { 104 | throw ex 105 | } 106 | } 107 | } 108 | } 109 | } 110 | 111 | def k8s_validate(deployer) { 112 | retry(15) { 113 | try { 114 | sh "ssh ${ssh_options} ubuntu@${deployer} ansible-playbook -i src/contrib/ansible/inventory src/contrib/ansible/validate.yml" 115 | } catch (ex) { 116 | echo "k8s_validate: ${ex}" 117 | sleep 180L 118 | throw ex 119 | } 120 | } 121 | } 122 | 123 | def k8s_run_examples(deployer) { 124 | sh "ssh ${ssh_options} ubuntu@${deployer} ansible-playbook -i src/contrib/ansible/inventory src/contrib/ansible/examples.yml" 125 | } 126 | 127 | def guestbook_status(deployer) { 128 | 129 | def master = getMasterIP() 130 | echo "master: ${master}" 131 | 132 | def svcAddress 133 | 134 | retry(6) { 135 | sh "ssh ${ssh_options} ubuntu@${deployer} -- ssh ${master} kubectl get svc guestbook > guestbook_svc.status" 136 | def svcStatus = readFile('guestbook_svc.status') 137 | svcAddress = svc_external_ip_address(svcStatus) 138 | if (!svcAddress) { 139 | error "ExternalIP is null" 140 | sleep 10L 141 | } 142 | } 143 | 144 | echo "guestbook ExternalIP: ${svcAddress}" 145 | 146 | retry(15) { 147 | def status 148 | try { 149 | sh "ssh ${ssh_options} ubuntu@${deployer} curl http://${svcAddress}:3000/info > guestbook.status" 150 | status = readFile('guestbook.status') 151 | } catch (AbortException ex) { 152 | echo "${ex}" 153 | sleep 60L 154 | throw ex 155 | } 156 | def slaves = match_connected_slaves(status) 157 | if (slaves != '2') { 158 | sleep 60L 159 | error("redis slaves: ${slaves}") 160 | } 161 | } 162 | } 163 | 164 | @NonCPS 165 | def svc_external_ip_address(text) { 166 | def matcher = (text =~ /guestbook\s+([0-9\.]+)\s+(([0-9]{1,3}\.){3}[0-9]{1,3})/) 167 | matcher ? matcher[0][2] : null 168 | } 169 | 170 | @NonCPS 171 | def match_connected_slaves(status) { 172 | def matcher = (status =~ /(?m)^connected_slaves:(\d+)/) 173 | matcher ? matcher[0][1] : null 174 | } 175 | 176 | test_ec2_k8s_basic = { 177 | node { 178 | // git url: 'https://github.com/Juniper/container-networking-ansible.git' 179 | checkout scm 180 | 181 | dir('test/ec2-k8s') { 182 | withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'k8s-provisioner', usernameVariable: 'AWS_ACCESS_KEY_ID', passwordVariable: 'AWS_SECRET_ACCESS_KEY']]) { 183 | // create cluster 184 | sh "ansible-playbook -i localhost playbook.yml --tags=create -e job_id=${env.BUILD_NUMBER}" 185 | } 186 | 187 | def deployer = getDeployerHostname() 188 | 189 | try { 190 | sshagent(credentials: ["k8s"]) { 191 | sh 'ansible-playbook -i cluster.status playbook.yml --tags=deployer-install' 192 | sh 'ansible-playbook -i cluster.status playbook.yml --tags=workspace' 193 | // ssh client steps 194 | k8s_deploy(deployer) 195 | 196 | k8s_validate(deployer) 197 | 198 | k8s_run_examples(deployer) 199 | 200 | // verify 201 | guestbook_status(deployer) 202 | } 203 | } catch(ex) { 204 | echo "${ex}" 205 | input 'Debug k8s' 206 | throw ex 207 | } finally { 208 | withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'k8s-provisioner', usernameVariable: 'AWS_ACCESS_KEY_ID', passwordVariable: 'AWS_SECRET_ACCESS_KEY']]) { 209 | // delete cluster 210 | sh 'ansible-playbook -i cluster.status clean.yml' 211 | } 212 | } 213 | } 214 | } 215 | } 216 | 217 | test_ec2_openshift_basic = { 218 | node { 219 | // Checkout repository in workspace. 220 | checkout scm 221 | 222 | dir('test/ec2-origin') { 223 | withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'k8s-provisioner', usernameVariable: 'AWS_ACCESS_KEY_ID', passwordVariable: 'AWS_SECRET_ACCESS_KEY']]) { 224 | // tags: cluster, key-data, deployer-install, workspace 225 | sh "ansible-playbook -i localhost playbook.yml --tags=cluster -e job_id=${env.BUILD_NUMBER}" 226 | } 227 | 228 | sh "ansible-playbook -i localhost key-data.yml" 229 | 230 | def deployer = getDeployerHostname() 231 | 232 | try { 233 | sshagent(credentials: ["k8s"]) { 234 | sh "ansible-playbook -i cluster.status playbook.yml --tags=deployer-install,workspace -e job_id=${env.BUILD_NUMBER}" 235 | origin_deploy(deployer) 236 | } 237 | } catch(ex) { 238 | echo "${ex}" 239 | input 'Debug openshift' 240 | throw ex 241 | } finally { 242 | withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'k8s-provisioner', usernameVariable: 'AWS_ACCESS_KEY_ID', passwordVariable: 'AWS_SECRET_ACCESS_KEY']]) { 243 | // delete cluster 244 | sh 'ansible-playbook -i cluster.status clean.yml' 245 | } 246 | } 247 | } 248 | } 249 | } 250 | 251 | def getTestMatrix() { 252 | tests = [ 253 | ec2_k8s_basic: test_ec2_k8s_basic, 254 | ec2_openshift_basic: test_ec2_openshift_basic 255 | ] 256 | return tests 257 | } 258 | 259 | return this; 260 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /test/ec2-origin/roles/workspace/files/opencontrail_validate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | """ 4 | Sanity check the status of an openshift + opencontrail cluster 5 | """ 6 | 7 | import ConfigParser 8 | import argparse 9 | import json 10 | import paramiko 11 | import re 12 | import sys 13 | import time 14 | import xml.etree.ElementTree 15 | from datetime import datetime 16 | 17 | 18 | class Executor(object): 19 | DEFAULT_USERNAME = 'centos' 20 | 21 | def __init__(self, server): 22 | """ Constructor """ 23 | self._ssh_client = paramiko.SSHClient() 24 | self._ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 25 | self._ssh_client.connect(server, username=Executor.DEFAULT_USERNAME) 26 | 27 | def run(self, cmd, sudo=False): 28 | if sudo: 29 | cmd = 'sudo ' + cmd 30 | _, stdout, stderr = self._ssh_client.exec_command(cmd, get_pty=sudo) 31 | stdout.channel.recv_exit_status() 32 | return stdout.readlines(), stderr.readlines() 33 | 34 | def __enter__(self): 35 | return self 36 | 37 | def __exit__(self, type, value, traceback): 38 | self._ssh_client.close() 39 | 40 | def __del__(self): 41 | """ Destructor """ 42 | self._ssh_client.close() 43 | 44 | 45 | def expect_listen_ports(channel, expected): 46 | stdout, stderr = channel.run('netstat -ntl') 47 | 48 | absent = expected 49 | 50 | regexp = re.compile(r'(tcp\s+[0-9]+\s+[0-9]+\s+([0-9\.]+):([0-9]+)|' 51 | 'tcp6\s+[0-9]+\s+[0-9]+\s+:::([0-9]+))\s+(.*)' 52 | '\s+LISTEN') 53 | for line in stdout: 54 | match = regexp.match(line) 55 | if match: 56 | if match.group(1).startswith('tcp6'): 57 | port = match.group(4) 58 | else: 59 | port = match.group(3) 60 | port = int(port) 61 | if port in absent: 62 | absent.pop(port) 63 | return absent 64 | 65 | 66 | def expect_docker_running(channel, containerNames): 67 | stdout, stderr = channel.run("docker ps --format='{{.ID}} {{.Names}}'", 68 | sudo=True) 69 | regexp = re.compile(r'([a-f0-9]+)\s([\w-]+)') 70 | absent = containerNames 71 | 72 | for line in stdout: 73 | m = regexp.match(line) 74 | if not m: 75 | print "Unexpected output from ps command: %s" % line 76 | continue 77 | name = m.group(2) 78 | if name in absent: 79 | absent.remove(name) 80 | 81 | return absent 82 | 83 | 84 | def contrail_services_status(channel): 85 | tcp_ports = { 86 | 8082: "contrail-api", 87 | 8444: "ifmap", 88 | 5269: "xmpp", 89 | 9160: "cassandra", 90 | 2181: "zookeeper", 91 | 5672: "rabbitmq" 92 | } 93 | absent = expect_listen_ports(channel, tcp_ports) 94 | if len(absent): 95 | print "Service ports not running:" 96 | print absent 97 | return False 98 | return True 99 | 100 | 101 | def contrail_docker_status(channel, netManager=False): 102 | containerNames = [ 103 | 'contrail-control', 'contrail-api', 'contrail-schema', 'ifmap-server' 104 | ] 105 | if netManager: 106 | containerNames.append('kube-network-manager') 107 | absent = expect_docker_running(channel, containerNames) 108 | if len(absent) > 0: 109 | print "Containers not running: ", absent 110 | return False 111 | return True 112 | 113 | 114 | def contrail_docker_agent(channel, nodeIP): 115 | absent = expect_docker_running(channel, ['vrouter-agent']) 116 | if len(absent) > 0: 117 | print 'vrouter agent node not running on %s' % nodeIP 118 | return False 119 | return True 120 | 121 | 122 | def contrail_api_status(channel): 123 | stdout, stderr = channel.run('curl http://localhost:8082') 124 | try: 125 | json.loads('\n'.join(stdout)) 126 | except Exception: 127 | print 'Unable to connect to the contrail-api server' 128 | print '\n'.join(stderr) 129 | return False 130 | return True 131 | 132 | 133 | def contrail_control_instance_status(channel): 134 | """ 135 | Verify that the control-node is not stuck with a deleted routing-instance. 136 | """ 137 | stdout, stderr = channel.run( 138 | 'curl http://localhost:8083/Snh_ShowRoutingInstanceSummaryReq') 139 | if len(stdout) == 0: 140 | print 'Unable to get routing instance summary' 141 | print '\n'.join(stderr) 142 | return False 143 | 144 | root = xml.etree.ElementTree.fromstringlist(stdout) 145 | count = 0 146 | for instance in root.findall('.//ShowRoutingInstance'): 147 | delete_tag = instance.find('deleted') 148 | if delete_tag.text == 'true': 149 | name = instance.find('name') 150 | print 'instance %s deleted' % name.text 151 | count += 1 152 | 153 | return count == 0 154 | 155 | 156 | def contrail_xmpp_sessions(channel): 157 | """ 158 | Wait for 180 secs for the sessions to come up. 159 | """ 160 | for _ in range(18): 161 | stdout, stderr = channel.run( 162 | "netstat -nt | grep -E ':5269\s+.*ESTABLISHED'") 163 | if len(stdout) == 3: 164 | return True 165 | time.sleep(10) 166 | 167 | print 'XMPP sessions:' 168 | print '\n'.join(stdout) 169 | return False 170 | 171 | 172 | def openshift_system_services(channel): 173 | """ 174 | Ensure that openshift is able to start the docker-registry and router pods. 175 | This requires the deployer pods to be able to communicate with the master. 176 | """ 177 | 178 | def patternInList(pattern, pods): 179 | regexp = re.compile(pattern) 180 | for pod in pods: 181 | if regexp.match(pod): 182 | return True 183 | return False 184 | 185 | expect = [r'docker-registry-([0-9]+)-', r'router-([0-9]+)-'] 186 | 187 | for _ in range(36): 188 | stdout, stderr = channel.run("oc get pods -o json") 189 | data = json.loads('\n'.join(stdout)) 190 | 191 | pods = [] 192 | for item in data['items']: 193 | if item['status']['phase'] != 'Running': 194 | continue 195 | if 'generateName' in item['metadata']: 196 | pods.append(item['metadata']['generateName']) 197 | 198 | absent = [] 199 | for pattern in expect: 200 | if not patternInList(pattern, pods): 201 | absent.append(pattern) 202 | if len(absent) == 0: 203 | return True 204 | time.sleep(10) 205 | 206 | print 'system pods not running' 207 | print absent 208 | stdout, stderr = channel.run("oc get pods") 209 | print '\n'.join(stdout) 210 | return False 211 | 212 | 213 | def contrail_gateway_expect_svc_routes(channel, master, gatewayIP): 214 | """ 215 | The unicast routing table for the service VRF should have routes for the 216 | system services. 217 | """ 218 | 219 | stdout, stderr = master.run( 220 | "oc get svc -o jsonpath='{.items[*].spec.clusterIP}'") 221 | if len(stdout) == 0: 222 | print 'No service IPs' 223 | print '\n'.join(stderr) 224 | return False 225 | 226 | svc = stdout[0].split() 227 | if len(svc) < 3: 228 | print 'Expected at least 3 clusterIPs' 229 | return False 230 | 231 | stdout, stderr = channel.run("vif --list") 232 | 233 | re_section = re.compile(r'vif0\/([0-9]+)\s+OS:\s(\w+)') 234 | re_vrf = re.compile(r'Vrf:([0-9]+)') 235 | 236 | vrf_index = None 237 | inSection = False 238 | for line in stdout: 239 | m = re_section.match(line) 240 | if m: 241 | if m.group(2) == 'gateway1': 242 | inSection = True 243 | continue 244 | if inSection: 245 | break 246 | if not inSection: 247 | continue 248 | tag = re_vrf.search(line) 249 | if tag: 250 | vrf_index = int(tag.group(1)) 251 | 252 | if not vrf_index: 253 | print 'Unable to determine vrf id' 254 | return False 255 | 256 | absent = svc 257 | stdout, stderr = channel.run( 258 | "curl http://localhost:8085/Snh_Inet4UcRouteReq?uc_index=%d" % 259 | vrf_index) 260 | root = xml.etree.ElementTree.fromstringlist(stdout) 261 | for route in root.findall('.//RouteUcSandeshData'): 262 | ip = route.find('src_ip') 263 | prefixlen = route.find('src_plen') 264 | if prefixlen.text == "32" and ip.text in svc: 265 | absent.remove(ip.text) 266 | 267 | if len(absent) > 1: 268 | print 'services not in gateway VRF' 269 | print absent 270 | return False 271 | return True 272 | 273 | 274 | def contrail_svc_address_ping(prober, master): 275 | """ 276 | Ensure that the specified system can reach the service IP addresses. 277 | """ 278 | 279 | stdout, stderr = master.run( 280 | "oc get svc -o jsonpath='{.items[*].spec.clusterIP}'") 281 | if len(stdout) == 0: 282 | print 'No service IPs' 283 | print '\n'.join(stderr) 284 | return False 285 | serviceIPs = stdout[0].split() 286 | for svc in serviceIPs: 287 | if svc.endswith('.0.1'): 288 | serviceIPs.remove(svc) 289 | break 290 | 291 | success = True 292 | regexp = re.compile(r'(\d+) packets transmitted, (\d+) received') 293 | for svc in serviceIPs: 294 | result = 0 295 | stdout, stderr = prober.run("ping -c 5 %s" % svc) 296 | for line in stdout: 297 | m = regexp.match(line) 298 | if m: 299 | result = int(m.group(2)) 300 | break 301 | if result != 5: 302 | print "ping %s" % svc 303 | print line 304 | success = False 305 | return success 306 | 307 | 308 | def test_application_status(master, gateway): 309 | """ Returns True if the application is running 310 | 311 | Deployment fails is any of the pods is in Error state. 312 | 313 | The test succeeds if the web-front end is reachable. 314 | Deployment takes 5/10 mins to complete. 315 | """ 316 | start = datetime.now() 317 | while (datetime.now() - start).seconds < (60 * 60): 318 | stdout, stderr = master.run( 319 | "oc --namespace=test get pods -o json") 320 | try: 321 | podInfo = json.loads('\n'.join(stdout)) 322 | except Exception as ex: 323 | print 'Unable to decode pod information %s' % ex 324 | print stderr 325 | return False 326 | 327 | run_count = 0 328 | pending = 0 329 | builder = 0 330 | for item in podInfo['items']: 331 | if item['status']['phase'] == 'Failed': 332 | print 'pod %s Failed' % item['metadata']['name'] 333 | return False 334 | elif item['status']['phase'] == 'Running': 335 | if (item['metadata']['name'].endswith('-build') or 336 | item['metadata']['name'].endswith('-deploy')): 337 | builder += 1 338 | continue 339 | run_count += 1 340 | elif item['status']['phase'] == 'Pending': 341 | pending += 1 342 | 343 | if not pending and not builder and run_count >= 2: 344 | break 345 | time.sleep(180) 346 | 347 | for _ in range(6): 348 | stdout, stderr = gateway.run( 349 | "no_proxy=* curl http://%s:%d/articles" % 350 | ('rails-postgresql-example-test.router.default.svc.cluster.local', 351 | 80)) 352 | pattern = re.compile(r'Listing articles') 353 | for line in stdout: 354 | if pattern.search(line): 355 | print "Application OK" 356 | return True 357 | time.sleep(10) 358 | 359 | print 'Application stdout:' 360 | print '\n'.join(stdout) 361 | print 'Application stderr:' 362 | print '\n'.join(stderr) 363 | return False 364 | 365 | 366 | def inventory_parse(filename): 367 | """ Parse the inventory file. 368 | 369 | Expects inventory to have the following format: 370 | [section] 371 | hostname ansible_ssh_host= 372 | """ 373 | 374 | group_names = ['masters', 'gateways', 'nodes'] 375 | groups = {} 376 | 377 | config = ConfigParser.ConfigParser(allow_no_value=True) 378 | with open(filename, 'r') as fp: 379 | config.readfp(fp) 380 | 381 | for section in group_names: 382 | try: 383 | group = [] 384 | for item in config.items(section): 385 | group.append(item[1]) 386 | groups[section] = group 387 | except ConfigParser.NoSectionError: 388 | pass 389 | 390 | return groups 391 | 392 | 393 | def main(): 394 | """ 395 | stages: 396 | 1. OpenContrail is installed (but not provisioned) 397 | 2. OpenShift is installed 398 | 3. OpenContrail is provisioned 399 | 4. OpenShift services are started 400 | 5. Test application is deployed 401 | 402 | common install problems: 403 | - control-node rejecting XMPP connections 404 | - service-default network marked as deleted on control-node 405 | """ 406 | 407 | parser = argparse.ArgumentParser() 408 | 409 | parser.add_argument('--stage', type=int, help='Install stage') 410 | parser.add_argument('inventory') 411 | 412 | args = parser.parse_args() 413 | groups = inventory_parse(args.inventory) 414 | 415 | if 'masters' not in groups: 416 | print '%s does not define a master' % args.inventory 417 | sys.exit(1) 418 | 419 | master = Executor(groups['masters'][0]) 420 | 421 | success = ( 422 | contrail_api_status(master) and 423 | contrail_docker_status(master, netManager=args.stage >= 2) and 424 | contrail_services_status(master) and 425 | (args.stage < 3 or contrail_xmpp_sessions(master)) and 426 | (args.stage < 4 or openshift_system_services(master)) and 427 | (args.stage < 4 or contrail_control_instance_status(master)) 428 | ) 429 | 430 | for node in groups['nodes']: 431 | with Executor(node) as channel: 432 | ok = contrail_docker_agent(channel, node) 433 | if not ok: 434 | success = False 435 | 436 | for gateway in groups['gateways']: 437 | with Executor(gateway) as channel: 438 | ok = contrail_docker_agent(channel, gateway) 439 | if (args.stage >= 4): 440 | if not contrail_gateway_expect_svc_routes(channel, master, 441 | gateway): 442 | ok = False 443 | if not ok: 444 | success = False 445 | 446 | if args.stage >= 4 and not contrail_svc_address_ping(master, master): 447 | success = False 448 | 449 | if args.stage >= 5: 450 | with Executor(groups['gateways'][0]) as channel: 451 | if not test_application_status(master, channel): 452 | success = False 453 | 454 | del master 455 | 456 | if not success: 457 | print 'FAIL' 458 | sys.exit(1) 459 | 460 | if __name__ == '__main__': 461 | main() 462 | --------------------------------------------------------------------------------