├── .gitignore
├── misfortune5
├── ansible
│ ├── build
│ ├── templates
│ │ └── etc
│ │ │ ├── resolv.conf
│ │ │ └── init
│ │ │ └── kubeflaskprometheus.conf
│ └── playbook.yml
├── terraform
│ ├── aws.tf
│ ├── variables.tf
│ ├── vpc.tf
│ └── public.tf
└── README.md
├── misfortune4
├── ansible
│ ├── templates
│ │ ├── spark
│ │ │ ├── sbin
│ │ │ │ ├── stop-ipython.sh
│ │ │ │ └── start-ipython.sh.j2
│ │ │ └── conf
│ │ │ │ ├── spark-defaults.conf.j2
│ │ │ │ └── spark-env.sh.j2
│ │ └── home
│ │ │ └── ubuntu
│ │ │ └── .profile.j2
│ ├── stop-workers.yml
│ ├── stop-master.yml
│ ├── start-workers.yml
│ ├── start-master.yml
│ ├── env.yml
│ └── install.yml
├── terraform
│ ├── aws.tf
│ ├── README.md
│ ├── variables.tf
│ ├── vpc.tf
│ ├── workers.tf
│ └── master.tf
└── README.md
├── misfortune1
├── ansible
│ ├── src
│ │ ├── myproject
│ │ │ ├── wsgi.py
│ │ │ ├── myproject.ini
│ │ │ └── myproject.py
│ │ └── myotherproject
│ │ │ ├── wsgi.py
│ │ │ ├── myotherproject.ini
│ │ │ └── myotherproject.py
│ ├── templates
│ │ └── etc
│ │ │ ├── init
│ │ │ ├── myproject.conf
│ │ │ └── myotherproject.conf
│ │ │ ├── nginx
│ │ │ └── sites-available
│ │ │ │ ├── myproject.j2
│ │ │ │ └── myotherproject.j2
│ │ │ └── postgresql
│ │ │ └── 9.3
│ │ │ └── main
│ │ │ └── postgresql.conf
│ └── playbook.yml
├── terraform
│ ├── aws.tf
│ ├── variables.tf
│ ├── vpc.tf
│ └── public.tf
└── README.md
├── misfortune2
├── terraform
│ ├── aws.tf
│ ├── variables.tf
│ ├── private.tf
│ ├── public.tf
│ └── vpc.tf
├── packer
│ ├── install.sh
│ └── api.json
└── README.md
├── misfortune3
├── terraform
│ ├── aws.tf
│ ├── README.md
│ ├── variables.tf
│ ├── client2.tf
│ ├── client1.tf
│ ├── rds1.tf
│ ├── rds2.tf
│ └── vpc.tf
└── README.md
└── README.md
/.gitignore:
--------------------------------------------------------------------------------
1 | terraform.tfstate
2 | terraform.tfstate.backup
3 | *.retry
4 |
--------------------------------------------------------------------------------
/misfortune5/ansible/build:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | ansible-playbook playbook.yml
4 |
--------------------------------------------------------------------------------
/misfortune4/ansible/templates/spark/sbin/stop-ipython.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | tmux kill-session -t ipython_notebook
4 |
--------------------------------------------------------------------------------
/misfortune1/ansible/src/myproject/wsgi.py:
--------------------------------------------------------------------------------
1 | from myproject import application
2 |
3 | if __name__ == "__main__":
4 | application.run()
5 |
--------------------------------------------------------------------------------
/misfortune1/ansible/src/myotherproject/wsgi.py:
--------------------------------------------------------------------------------
1 | from myotherproject import application
2 |
3 | if __name__ == "__main__":
4 | application.run()
5 |
--------------------------------------------------------------------------------
/misfortune1/terraform/aws.tf:
--------------------------------------------------------------------------------
1 | provider "aws" {
2 | access_key = "${var.aws_access_key}"
3 | secret_key = "${var.aws_secret_key}"
4 | region = "${var.aws_region}"
5 | }
6 |
--------------------------------------------------------------------------------
/misfortune2/terraform/aws.tf:
--------------------------------------------------------------------------------
1 | provider "aws" {
2 | access_key = "${var.aws_access_key}"
3 | secret_key = "${var.aws_secret_key}"
4 | region = "${var.aws_region}"
5 | }
6 |
--------------------------------------------------------------------------------
/misfortune3/terraform/aws.tf:
--------------------------------------------------------------------------------
1 | provider "aws" {
2 | access_key = "${var.aws_access_key}"
3 | secret_key = "${var.aws_secret_key}"
4 | region = "${var.aws_region}"
5 | }
6 |
--------------------------------------------------------------------------------
/misfortune4/terraform/aws.tf:
--------------------------------------------------------------------------------
1 | provider "aws" {
2 | access_key = "${var.aws_access_key}"
3 | secret_key = "${var.aws_secret_key}"
4 | region = "${var.aws_region}"
5 | }
6 |
--------------------------------------------------------------------------------
/misfortune5/terraform/aws.tf:
--------------------------------------------------------------------------------
1 | provider "aws" {
2 | access_key = "${var.aws_access_key}"
3 | secret_key = "${var.aws_secret_key}"
4 | region = "${var.aws_region}"
5 | }
6 |
--------------------------------------------------------------------------------
/misfortune4/ansible/stop-workers.yml:
--------------------------------------------------------------------------------
1 | ---
2 | - hosts: tag_Name_wom4_worker:&key_wom
3 | tasks:
4 | - name: Stop Worker
5 | command: '/usr/local/spark/sbin/stop-slave.sh'
6 |
--------------------------------------------------------------------------------
/misfortune1/ansible/src/myproject/myproject.ini:
--------------------------------------------------------------------------------
1 | [uwsgi]
2 | module = wsgi
3 |
4 | master = true
5 | processes = 5
6 |
7 | socket = myproject.sock
8 | chmod-socket = 660
9 | vacuum = true
10 |
11 | die-on-term = true
12 |
--------------------------------------------------------------------------------
/misfortune1/ansible/src/myotherproject/myotherproject.ini:
--------------------------------------------------------------------------------
1 | [uwsgi]
2 | module = wsgi
3 |
4 | master = true
5 | processes = 5
6 |
7 | socket = myotherproject.sock
8 | chmod-socket = 660
9 | vacuum = true
10 |
11 | die-on-term = true
12 |
--------------------------------------------------------------------------------
/misfortune5/ansible/templates/etc/resolv.conf:
--------------------------------------------------------------------------------
1 | # Dynamic resolv.conf(5) file for glibc resolver(3) generated by resolvconf(8)
2 | # DO NOT EDIT THIS FILE BY HAND -- YOUR CHANGES WILL BE OVERWRITTEN
3 | nameserver 127.0.0.1
4 | search us-west-2.compute.internal
5 |
--------------------------------------------------------------------------------
/misfortune4/ansible/stop-master.yml:
--------------------------------------------------------------------------------
1 | ---
2 | - hosts: tag_Name_wom4_master:&key_wom
3 | tasks:
4 | - name: Stop Master
5 | command: '/usr/local/spark/sbin/stop-master.sh'
6 | - name: Stop IPython Notebook
7 | command: '/usr/local/spark/sbin/stop-ipython.sh'
8 |
--------------------------------------------------------------------------------
/misfortune1/ansible/templates/etc/init/myproject.conf:
--------------------------------------------------------------------------------
1 | description "uWSGI server instance configured to serve myproject"
2 |
3 | start on runlevel [2345]
4 | stop on runlevel [!2345]
5 |
6 | setuid ubuntu
7 | setgid www-data
8 |
9 | chdir /home/ubuntu/myproject
10 | exec uwsgi --ini myproject.ini
11 |
--------------------------------------------------------------------------------
/misfortune1/ansible/templates/etc/nginx/sites-available/myproject.j2:
--------------------------------------------------------------------------------
1 | server {
2 | listen 80;
3 | server_name {{ansible_eth0.ipv4.address}};
4 |
5 | location / {
6 | include uwsgi_params;
7 | uwsgi_pass unix:/home/ubuntu/myproject/myproject.sock;
8 | }
9 | }
10 |
11 |
12 |
--------------------------------------------------------------------------------
/misfortune1/ansible/templates/etc/init/myotherproject.conf:
--------------------------------------------------------------------------------
1 | description "uWSGI server instance configured to serve myotherproject"
2 |
3 | start on runlevel [2345]
4 | stop on runlevel [!2345]
5 |
6 | setuid ubuntu
7 | setgid www-data
8 |
9 | chdir /home/ubuntu/myotherproject
10 | exec uwsgi --ini myotherproject.ini
11 |
--------------------------------------------------------------------------------
/misfortune1/ansible/templates/etc/nginx/sites-available/myotherproject.j2:
--------------------------------------------------------------------------------
1 | server {
2 | listen 5000;
3 | server_name {{ansible_eth0.ipv4.address}};
4 |
5 | location / {
6 | include uwsgi_params;
7 | uwsgi_pass unix:/home/ubuntu/myotherproject/myotherproject.sock;
8 | }
9 | }
10 |
11 |
12 |
--------------------------------------------------------------------------------
/misfortune1/ansible/src/myproject/myproject.py:
--------------------------------------------------------------------------------
1 | from flask import Flask
2 | application = Flask(__name__)
3 |
4 | @application.route("/")
5 | def hello():
6 | return "Hello There...I see you've found me on HTTP...
"
7 |
8 | if __name__ == "__main__":
9 | application.run(host='0.0.0.0')
10 |
--------------------------------------------------------------------------------
/misfortune2/packer/install.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | sudo apt-get update &&
4 | sudo apt-get install -y git python-pip python-dev
5 |
6 | sudo pip install flask requests prometheus_client tornado
7 |
8 | git clone https://github.com/InsightDataScience/flask-kube-prometheus.git /home/ubuntu/flask-kube-prometheus
9 |
10 |
--------------------------------------------------------------------------------
/misfortune5/ansible/templates/etc/init/kubeflaskprometheus.conf:
--------------------------------------------------------------------------------
1 | description "uWSGI server instance configured to serve kube flask prometheus API"
2 |
3 | start on runlevel [2345]
4 | stop on runlevel [!2345]
5 |
6 | setuid ubuntu
7 | setgid ubuntu
8 |
9 | chdir /home/ubuntu/flask-kube-prometheus/webapp
10 | exec ./run.sh
11 |
--------------------------------------------------------------------------------
/misfortune1/ansible/src/myotherproject/myotherproject.py:
--------------------------------------------------------------------------------
1 | from flask import Flask
2 | application = Flask(__name__)
3 |
4 | @application.route("/")
5 | def hello():
6 | return "Hello There...I see you've found me on some other TCP port...
"
7 |
8 | if __name__ == "__main__":
9 | application.run(host='0.0.0.0')
10 |
--------------------------------------------------------------------------------
/misfortune5/ansible/playbook.yml:
--------------------------------------------------------------------------------
1 | ---
2 | - hosts: tag_Name_wom4_api:&key_wom
3 | become: true
4 | tasks:
5 | - name: Copy wsgi kubeflaskprometheus.conf to remote
6 | copy:
7 | src: templates/etc/init/kubeflaskprometheus.conf
8 | dest: /etc/init/kubeflaskprometheus.conf
9 | - name: Copy faulty resolv.conf to api machine
10 | copy:
11 | src: templates/etc/resolv.conf
12 | dest: /etc/resolv.conf
13 | - name: Restart kube flask prometheus api
14 | service:
15 | name: kubeflaskprometheus
16 | state: restarted
17 |
--------------------------------------------------------------------------------
/misfortune4/README.md:
--------------------------------------------------------------------------------
1 | # Spark workers cannot connect with the Spark master
2 |
3 | ## Contents
4 |
5 | * 1 VPC
6 | * 1 Public subnet
7 | * 1 Spark Master
8 | * 3 Spark Workers
9 |
10 | ## Goals to pass
11 |
12 | * Verify there are 3 Spark workers
13 | * Ensure that work can be sent from the Master to Workers
14 | * Display the total number of records in the `s3a://venmo-json/`
15 |
16 | ## Helpful resources
17 | * [SSH](https://semaphoreci.com/community/tutorials/getting-started-with-ssh)
18 | * [Spark](http://spark.apache.org/docs/latest/cluster-overview.html)
19 |
20 |
--------------------------------------------------------------------------------
/misfortune4/ansible/templates/spark/sbin/start-ipython.sh.j2:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | tmux new-session -s ipython_notebook -n bash -d
4 |
5 | tmux send-keys -t ipython_notebook 'PYSPARK_DRIVER_PYTHON=ipython PYSPARK_DRIVER_PYTHON_OPTS="notebook --no-browser --ip="*" --port=8888" /usr/local/spark/bin/pyspark --packages com.databricks:spark-csv_2.10:1.1.0 --master spark://{{ ec2_facts.instances[0].private_ip_address }}:7077 --executor-memory {{ ((ansible_memtotal_mb-1000) * 0.9)|int }}M --driver-memory {{ ((ansible_memtotal_mb-1000) * 0.9)|int }}M --conf spark.hadoop.fs.s3a.impl\=org.apache.hadoop.fs.s3a.S3AFileSystem' C-m
6 |
--------------------------------------------------------------------------------
/misfortune3/terraform/README.md:
--------------------------------------------------------------------------------
1 | # Allow two VPCs to share their RDS postgres databases
2 |
3 | ## Contents
4 |
5 | * 2 VPCs
6 | * 1 Public subnet in each VPC
7 | * EC2 instance in each public subnet acting as client
8 | * RDS Postgres instance in each public subnet acting as a backend DB
9 |
10 | ## Goals to pass
11 |
12 | * EC2 instance in each VPC should be able to connect with their own RDS Postgres instance
13 | * EC2 instance in each VPC should be able to connect to each others RDS Postgres instance
14 |
15 | ## Helpful resources
16 | * [VPC Peering](http://docs.aws.amazon.com/AmazonVPC/latest/PeeringGuide/Welcome.html)
17 |
18 |
--------------------------------------------------------------------------------
/misfortune4/ansible/start-workers.yml:
--------------------------------------------------------------------------------
1 | ---
2 | - hosts: tag_Name_wom4_worker:&key_wom
3 | tasks:
4 | - name: Get master node private IP
5 | ec2_remote_facts:
6 | aws_secret_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}"
7 | aws_access_key: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}"
8 | region: "{{ lookup('env', 'AWS_DEFAULT_REGION') }}"
9 | filters:
10 | instance-state-name: running
11 | "tag:Name": wom4-master
12 | register: ec2_facts
13 | - name: Start Worker
14 | command: '/usr/local/spark/sbin/start-slave.sh spark://{{ ec2_facts.instances[0].private_ip_address }}:7077'
15 |
--------------------------------------------------------------------------------
/misfortune4/terraform/README.md:
--------------------------------------------------------------------------------
1 | # Allow two VPCs to share their RDS postgres databases
2 |
3 | ## Contents
4 |
5 | * 2 VPCs
6 | * 1 Public subnet in each VPC
7 | * EC2 instance in each public subnet acting as client
8 | * RDS Postgres instance in each public subnet acting as a backend DB
9 |
10 | ## Goals to pass
11 |
12 | * EC2 instance in each VPC should be able to connect with their own RDS Postgres instance
13 | * EC2 instance in each VPC should be able to connect to each others RDS Postgres instance
14 |
15 | ## Helpful resources
16 | * [VPC Peering](http://docs.aws.amazon.com/AmazonVPC/latest/PeeringGuide/Welcome.html)
17 |
18 |
--------------------------------------------------------------------------------
/misfortune4/ansible/start-master.yml:
--------------------------------------------------------------------------------
1 | ---
2 | - hosts: tag_Name_wom4_master:&key_wom
3 | tasks:
4 | - name: Get master node private IP
5 | ec2_remote_facts:
6 | aws_secret_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}"
7 | aws_access_key: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}"
8 | region: "{{ lookup('env', 'AWS_DEFAULT_REGION') }}"
9 | filters:
10 | instance-state-name: running
11 | "tag:Name": wom4-master
12 | register: ec2_facts
13 | - name: Start Master
14 | command: '/usr/local/spark/sbin/start-master.sh'
15 | - name: Start IPython Notebook
16 | command: '/usr/local/spark/sbin/start-ipython.sh'
17 |
--------------------------------------------------------------------------------
/misfortune2/README.md:
--------------------------------------------------------------------------------
1 | # Resource in the private subnet access the external internet
2 |
3 | ## Contents
4 |
5 | * VPC
6 | * Public subnet
7 | * Private subnet
8 | * EC2 instance in public subnet acting as client
9 | * EC2 instance in private subnet acting as API server
10 |
11 | ## Goals to pass
12 |
13 | Client should be able to hit the API server with a given user and receive all public repositories
14 |
15 | * GET request against WOM2-API-NODE:9999 to view Prometheus metrics
16 | * GET request against WOM2-API-NODE:5000 to view HTML of home page
17 | * GET request against WOM2-API-NODE:5000/get-repos?username={myusername}
18 |
19 | ## Helpful resources
20 | * [NAT](http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/VPC_NAT_Instance.html)
21 |
22 |
--------------------------------------------------------------------------------
/misfortune5/README.md:
--------------------------------------------------------------------------------
1 | # Two instances in public subnets can talk with each other but the API server does not get a response from Github.com.
2 |
3 | ## Contents
4 |
5 | * VPC
6 | * 1 Public subnet
7 | * 1 EC2 instance acting as client
8 | * 1 EC2 instance acting as an API server
9 | * Tornado web server on port 5000
10 | * Prometheus web server on port 9999
11 |
12 | ## Goals to pass
13 |
14 | * Make a GET request to the server on port 5000 from the client
15 | * Make a GET request to the server on port 9999 from the client
16 | * Make a GET request to one of the endpoing on the server
17 | * SERVERIP:5000/get-repos?username=aouyang1
18 |
19 | ## Helpful resources
20 |
21 | * [DNS](https://www.youtube.com/watch?v=72snZctFFtA)
22 | * [resolv.conf](https://en.wikipedia.org/wiki/Resolv.conf)
23 |
24 |
--------------------------------------------------------------------------------
/misfortune1/terraform/variables.tf:
--------------------------------------------------------------------------------
1 | variable "aws_access_key" {}
2 | variable "aws_secret_key" {}
3 | variable "aws_key_path" {}
4 | variable "aws_key_name" {}
5 |
6 | variable "client_cnt" {
7 | default = "4"
8 | }
9 |
10 | variable "aws_region" {
11 | description = "EC2 Region for the VPC"
12 | default = "us-west-2"
13 | }
14 |
15 | variable "amis" {
16 | description = "AMIs by region"
17 | default = {
18 | us-west-2 = "ami-5e63d13e"
19 | us-west-1 = "ami-3e21725e"
20 | us-east-1 = "ami-49c9295f"
21 | }
22 | }
23 |
24 | variable "vpc_cidr" {
25 | description = "CIDR for the whole VPC"
26 | default = "10.0.0.0/16"
27 | }
28 |
29 | variable "public_subnet_cidr_vpc" {
30 | description = "CIDR for the Public Subnet"
31 | default = "10.0.0.0/24"
32 | }
33 |
34 |
--------------------------------------------------------------------------------
/misfortune3/README.md:
--------------------------------------------------------------------------------
1 | # Allow instances in separate VPCs connect to each others RDS instances
2 |
3 | ## Contents
4 |
5 | * 2 VPCs
6 | * 2 Public subnet
7 | * 2 EC2 instances with one in each public subnet acting as client
8 | * 2 RDS Postgres servers with one in each public subnet
9 |
10 | ## Goals to pass
11 |
12 | Client from either subnet should be able to access their own RDS postgres server along with each others Postgres servers
13 | * Ensure each client instance can make a connection with their own database
14 | * Ensure each client instance can access each others database
15 |
16 | ## Helpful resources
17 | * [RDS](https://aws.amazon.com/rds/postgresql/)
18 | * [VPC Peering](http://docs.aws.amazon.com/AmazonVPC/latest/PeeringGuide/Welcome.html)
19 | * [Route Tables](http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/VPC_Route_Tables.html)
20 |
--------------------------------------------------------------------------------
/misfortune2/packer/api.json:
--------------------------------------------------------------------------------
1 | {
2 | "variables": {
3 | "aws_access_key": "{{env `AWS_ACCESS_KEY_ID`}}",
4 | "aws_secret_key": "{{env `AWS_SECRET_ACCESS_KEY`}}"
5 | },
6 | "builders": [{
7 | "type": "amazon-ebs",
8 | "access_key": "{{user `aws_access_key`}}",
9 | "secret_key": "{{user `aws_secret_key`}}",
10 | "region": "us-west-2",
11 | "source_ami": "ami-5189a661",
12 | "instance_type": "m4.large",
13 | "ssh_username": "ubuntu",
14 | "ami_name": "flask-kube-prometheus-{{timestamp}}",
15 | "ami_groups": "all",
16 | "tags": {
17 | "Name": "flask-kube-prometheus"
18 | }
19 | }],
20 | "provisioners": [{
21 | "type": "shell",
22 | "scripts": [
23 | "install.sh"
24 | ],
25 | "pause_before": "30s"
26 | }]
27 | }
28 |
--------------------------------------------------------------------------------
/misfortune5/terraform/variables.tf:
--------------------------------------------------------------------------------
1 | variable "aws_access_key" {}
2 | variable "aws_secret_key" {}
3 | variable "aws_key_path" {}
4 | variable "aws_key_name" {}
5 |
6 | variable "aws_region" {
7 | description = "EC2 Region for the VPC"
8 | default = "us-west-2"
9 | }
10 |
11 | variable "amis" {
12 | description = "AMIs by region"
13 | default = {
14 | us-west-2 = "ami-5e63d13e"
15 | us-west-1 = "ami-3e21725e"
16 | us-east-1 = "ami-49c9295f"
17 | }
18 | }
19 |
20 | variable "amis-api" {
21 | description = "AMIs by region"
22 | default = {
23 | us-west-2 = "ami-6a7ef20a"
24 | }
25 | }
26 |
27 | variable "vpc_cidr" {
28 | description = "CIDR for the whole VPC"
29 | default = "10.0.0.0/16"
30 | }
31 |
32 | variable "public_subnet_cidr" {
33 | description = "CIDR for the Public Subnet"
34 | default = "10.0.0.0/24"
35 | }
36 |
37 |
--------------------------------------------------------------------------------
/misfortune1/README.md:
--------------------------------------------------------------------------------
1 | # Reach a resource with different protocols from multiple clients. Modify the AWS infrastructure to pass all the goals.
2 |
3 | ## Contents
4 |
5 | * VPC
6 | * Public subnet
7 | * 4 EC2 instances acting as clients
8 | * 1 EC2 isntance acting as a server
9 | * Nginx web server on HTTP port
10 | * Nginx web server on port 5000
11 | * Postgres on port 5432
12 |
13 | ## Goals to pass
14 |
15 | * Ping the server node from any of the clients
16 | * Make a GET request to the server on HTTP from any of the clients
17 | * Make a GET request to the server on port 5000 from any of the clients
18 | * Establish a connection to Postgresql in the server on port 5432 from any of the clients
19 | * All connections should be accessible only from the clients
20 |
21 | ## Helpful resources
22 |
23 | * [AWS Security Groups](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-network-security.html)
24 | * [ICMP/TCP/UDP](http://superuser.com/a/1044369)
25 | * [NGINX](http://nginx.org/en/)
26 |
--------------------------------------------------------------------------------
/misfortune4/terraform/variables.tf:
--------------------------------------------------------------------------------
1 | variable "aws_access_key" {}
2 | variable "aws_secret_key" {}
3 | variable "aws_key_path" {}
4 | variable "aws_key_name" {}
5 |
6 | variable "num_workers" {
7 | description = "Number of Spark Workers"
8 | default = "10"
9 | }
10 |
11 | variable "instance_type" {
12 | description = "Instance type for master and workers"
13 | default = "m4.large"
14 | }
15 |
16 | variable "aws_region" {
17 | description = "EC2 Region for the VPC"
18 | default = "us-west-2"
19 | }
20 |
21 | variable "amis" {
22 | description = "AMIs by region"
23 | default = {
24 | us-west-2 = "ami-5e63d13e"
25 | us-west-1 = "ami-3e21725e"
26 | us-east-1 = "ami-49c9295f"
27 | }
28 | }
29 |
30 | variable "vpc_cidr" {
31 | description = "CIDR for the whole VPC"
32 | default = "10.0.0.0/16"
33 | }
34 |
35 | variable "public_subnet_cidr" {
36 | description = "CIDR for the Public Subnet"
37 | default = "10.0.0.0/24"
38 | }
39 |
40 |
--------------------------------------------------------------------------------
/misfortune2/terraform/variables.tf:
--------------------------------------------------------------------------------
1 | variable "aws_access_key" {}
2 | variable "aws_secret_key" {}
3 | variable "aws_key_path" {}
4 | variable "aws_key_name" {}
5 |
6 | variable "aws_region" {
7 | description = "EC2 Region for the VPC"
8 | default = "us-west-2"
9 | }
10 |
11 | variable "amis" {
12 | description = "AMIs by region"
13 | default = {
14 | us-west-2 = "ami-5e63d13e"
15 | us-west-1 = "ami-3e21725e"
16 | us-east-1 = "ami-49c9295f"
17 | }
18 | }
19 |
20 | variable "amis-api" {
21 | description = "AMIs by region for API"
22 | default = {
23 | us-west-2 = "ami-6a7ef20a"
24 | }
25 | }
26 |
27 | variable "vpc_cidr" {
28 | description = "CIDR for the whole VPC"
29 | default = "10.0.0.0/16"
30 | }
31 |
32 | variable "public_subnet_cidr" {
33 | description = "CIDR for the Public Subnet"
34 | default = "10.0.0.0/24"
35 | }
36 |
37 | variable "private_subnet_cidr" {
38 | description = "CIDR for the Private Subnet"
39 | default = "10.0.1.0/24"
40 | }
41 |
42 |
43 |
--------------------------------------------------------------------------------
/misfortune4/terraform/vpc.tf:
--------------------------------------------------------------------------------
1 | /*
2 | VPC
3 | */
4 | resource "aws_vpc" "wom4" {
5 | cidr_block = "${var.vpc_cidr}"
6 | enable_dns_hostnames = true
7 | tags {
8 | Name = "wom4"
9 | }
10 | }
11 |
12 | /*
13 | IGW
14 | */
15 | resource "aws_internet_gateway" "wom4" {
16 | vpc_id = "${aws_vpc.wom4.id}"
17 |
18 | tags {
19 | Name = "wom4"
20 | }
21 | }
22 |
23 | /*
24 | Public Subnet in VPC
25 | */
26 | resource "aws_subnet" "wom4" {
27 | vpc_id = "${aws_vpc.wom4.id}"
28 |
29 | cidr_block = "${var.public_subnet_cidr}"
30 | availability_zone = "us-west-2a"
31 |
32 | tags {
33 | Name = "wom4"
34 | }
35 | }
36 |
37 | resource "aws_route_table" "wom4" {
38 | vpc_id = "${aws_vpc.wom4.id}"
39 |
40 | route {
41 | cidr_block = "0.0.0.0/0"
42 | gateway_id = "${aws_internet_gateway.wom4.id}"
43 | }
44 |
45 | tags {
46 | Name = "wom4"
47 | }
48 | }
49 |
50 | resource "aws_route_table_association" "wom4" {
51 | subnet_id = "${aws_subnet.wom4.id}"
52 | route_table_id = "${aws_route_table.wom4.id}"
53 | }
54 |
55 |
--------------------------------------------------------------------------------
/misfortune5/terraform/vpc.tf:
--------------------------------------------------------------------------------
1 | /*
2 | VPCs
3 | */
4 | resource "aws_vpc" "wom4" {
5 | cidr_block = "${var.vpc_cidr}"
6 | enable_dns_hostnames = true
7 | tags {
8 | Name = "wom4"
9 | }
10 | }
11 |
12 | /*
13 | IGWs
14 | */
15 | resource "aws_internet_gateway" "wom4" {
16 | vpc_id = "${aws_vpc.wom4.id}"
17 |
18 | tags {
19 | Name = "wom4"
20 | }
21 | }
22 |
23 | /*
24 | Public Subnet in VPC
25 | */
26 | resource "aws_subnet" "wom4" {
27 | vpc_id = "${aws_vpc.wom4.id}"
28 |
29 | cidr_block = "${var.public_subnet_cidr}"
30 | availability_zone = "us-west-2a"
31 |
32 | tags {
33 | Name = "Public Subnet"
34 | }
35 | }
36 |
37 | resource "aws_route_table" "wom4" {
38 | vpc_id = "${aws_vpc.wom4.id}"
39 |
40 | route {
41 | cidr_block = "0.0.0.0/0"
42 | gateway_id = "${aws_internet_gateway.wom4.id}"
43 | }
44 |
45 | tags {
46 | Name = "Public Route Table for VPC"
47 | }
48 | }
49 |
50 | resource "aws_route_table_association" "wom4" {
51 | subnet_id = "${aws_subnet.wom4.id}"
52 | route_table_id = "${aws_route_table.wom4.id}"
53 | }
54 |
--------------------------------------------------------------------------------
/misfortune1/terraform/vpc.tf:
--------------------------------------------------------------------------------
1 | /*
2 | VPCs
3 | */
4 | resource "aws_vpc" "wom1" {
5 | cidr_block = "${var.vpc_cidr}"
6 | enable_dns_hostnames = true
7 | tags {
8 | Name = "wom1"
9 | }
10 | }
11 |
12 | /*
13 | IGWs
14 | */
15 | resource "aws_internet_gateway" "wom1" {
16 | vpc_id = "${aws_vpc.wom1.id}"
17 |
18 | tags {
19 | Name = "wom1"
20 | }
21 | }
22 |
23 | /*
24 | Public Subnet in VPC
25 | */
26 | resource "aws_subnet" "wom1-vpc" {
27 | vpc_id = "${aws_vpc.wom1.id}"
28 |
29 | cidr_block = "${var.public_subnet_cidr_vpc}"
30 | availability_zone = "us-west-2a"
31 |
32 | tags {
33 | Name = "Public Subnet"
34 | }
35 | }
36 |
37 | resource "aws_route_table" "wom1-vpc" {
38 | vpc_id = "${aws_vpc.wom1.id}"
39 |
40 | route {
41 | cidr_block = "0.0.0.0/0"
42 | gateway_id = "${aws_internet_gateway.wom1.id}"
43 | }
44 |
45 | tags {
46 | Name = "Public Route Table for VPC"
47 | }
48 | }
49 |
50 | resource "aws_route_table_association" "wom1-vpc" {
51 | subnet_id = "${aws_subnet.wom1-vpc.id}"
52 | route_table_id = "${aws_route_table.wom1-vpc.id}"
53 | }
54 |
--------------------------------------------------------------------------------
/misfortune4/ansible/templates/home/ubuntu/.profile.j2:
--------------------------------------------------------------------------------
1 | # ~/.profile: executed by the command interpreter for login shells.
2 | # This file is not read by bash(1), if ~/.bash_profile or ~/.bash_login
3 | # exists.
4 | # see /usr/share/doc/bash/examples/startup-files for examples.
5 | # the files are located in the bash-doc package.
6 |
7 | # the default umask is set in /etc/profile; for setting the umask
8 | # for ssh logins, install and configure the libpam-umask package.
9 | #umask 022
10 |
11 | # if running bash
12 | if [ -n "$BASH_VERSION" ]; then
13 | # include .bashrc if it exists
14 | if [ -f "$HOME/.bashrc" ]; then
15 | . "$HOME/.bashrc"
16 | fi
17 | fi
18 |
19 | # set PATH so it includes user's private bin if it exists
20 | if [ -d "$HOME/bin" ] ; then
21 | PATH="$HOME/bin:$PATH"
22 | fi
23 |
24 | export JAVA_HOME=/usr
25 | export PATH=$PATH:$JAVA_HOME/bin
26 |
27 | export SPARK_HOME=/usr/local/spark
28 | export HADOOP_HOME=/usr/local/hadoop
29 | export PATH=$PATH:$SPARK_HOME/bin:$HADOOP_HOME/bin
30 |
31 | export AWS_ACCESS_KEY_ID={{ aws_access_key_id }}
32 | export AWS_SECRET_ACCESS_KEY={{ aws_secret_access_key }}
33 | export AWS_DEFAULT_REGION={{ aws_default_region }}
34 |
--------------------------------------------------------------------------------
/misfortune3/terraform/variables.tf:
--------------------------------------------------------------------------------
1 | variable "aws_access_key" {}
2 | variable "aws_secret_key" {}
3 | variable "aws_key_path" {}
4 | variable "aws_key_name" {}
5 |
6 | variable "aws_region" {
7 | description = "EC2 Region for the VPC"
8 | default = "us-west-2"
9 | }
10 |
11 | variable "amis" {
12 | description = "AMIs by region"
13 | default = {
14 | us-west-2 = "ami-5e63d13e"
15 | us-west-1 = "ami-3e21725e"
16 | us-east-1 = "ami-49c9295f"
17 | }
18 | }
19 |
20 | variable "vpc_cidr_1" {
21 | description = "CIDR for the whole VPC"
22 | default = "10.0.0.0/16"
23 | }
24 |
25 | variable "vpc_cidr_2" {
26 | description = "CIDR for the whole VPC"
27 | default = "10.1.0.0/16"
28 | }
29 |
30 | variable "public_subnet_cidr_1_1" {
31 | description = "CIDR for the Public Subnet"
32 | default = "10.0.0.0/24"
33 | }
34 |
35 | variable "public_subnet_cidr_1_2" {
36 | description = "CIDR for the Public Subnet"
37 | default = "10.0.1.0/24"
38 | }
39 |
40 | variable "public_subnet_cidr_2_1" {
41 | description = "CIDR for the Public Subnet"
42 | default = "10.1.0.0/24"
43 | }
44 |
45 | variable "public_subnet_cidr_2_2" {
46 | description = "CIDR for the Public Subnet"
47 | default = "10.1.1.0/24"
48 | }
49 |
--------------------------------------------------------------------------------
/misfortune2/terraform/private.tf:
--------------------------------------------------------------------------------
1 | /*
2 | API Server
3 | */
4 | resource "aws_security_group" "wom2-api" {
5 | name = "wom2-api"
6 | description = "Allow incoming API connections."
7 |
8 | ingress {
9 | from_port = 22
10 | to_port = 22
11 | protocol = "tcp"
12 | security_groups = ["${aws_security_group.wom2-client.id}"]
13 | }
14 |
15 | ingress {
16 | from_port = 9999
17 | to_port = 9999
18 | protocol = "udp"
19 | security_groups = ["${aws_security_group.wom2-client.id}"]
20 | }
21 |
22 | egress {
23 | from_port = 0
24 | to_port = 0
25 | protocol = -1
26 | cidr_blocks = ["0.0.0.0/0"]
27 | }
28 |
29 | vpc_id = "${aws_vpc.wom2.id}"
30 |
31 | tags {
32 | Name = "wom2-api"
33 | }
34 | }
35 |
36 | resource "aws_instance" "wom2-api" {
37 | ami = "${lookup(var.amis-api, var.aws_region)}"
38 | availability_zone = "us-west-2a"
39 | instance_type = "t2.micro"
40 | key_name = "${var.aws_key_name}"
41 | vpc_security_group_ids = ["${aws_security_group.wom2-api.id}"]
42 | subnet_id = "${aws_subnet.wom2-private.id}"
43 | associate_public_ip_address = false
44 | source_dest_check = false
45 |
46 | tags {
47 | Name = "wom2-api"
48 | }
49 | }
50 |
--------------------------------------------------------------------------------
/misfortune3/terraform/client2.tf:
--------------------------------------------------------------------------------
1 | resource "aws_security_group" "wom3-client-2" {
2 | name = "wom3-client-2"
3 | description = "Website level security groups."
4 |
5 | ingress {
6 | from_port = 22
7 | to_port = 22
8 | protocol = "tcp"
9 | cidr_blocks = ["0.0.0.0/0"]
10 | }
11 |
12 | ingress {
13 | from_port = -1
14 | to_port = -1
15 | protocol = "icmp"
16 | cidr_blocks = ["0.0.0.0/0"]
17 | }
18 |
19 | egress {
20 | from_port = 0
21 | to_port = 0
22 | protocol = -1
23 | cidr_blocks = ["0.0.0.0/0"]
24 | }
25 |
26 | vpc_id = "${aws_vpc.wom3-2.id}"
27 |
28 | tags {
29 | Name = "wom3-client-2"
30 | }
31 | }
32 |
33 | resource "aws_instance" "wom3-client-2" {
34 | ami = "${lookup(var.amis, var.aws_region)}"
35 | availability_zone = "us-west-2a"
36 | instance_type = "t2.micro"
37 | key_name = "${var.aws_key_name}"
38 | vpc_security_group_ids = ["${aws_security_group.wom3-client-2.id}"]
39 | subnet_id = "${aws_subnet.wom3-public-2-1.id}"
40 | associate_public_ip_address = true
41 | source_dest_check = false
42 |
43 | tags {
44 | Name = "wom3-client-2"
45 | }
46 | }
47 |
48 | resource "aws_eip" "wom3-client-2" {
49 | instance = "${aws_instance.wom3-client-2.id}"
50 | vpc = true
51 | }
52 |
--------------------------------------------------------------------------------
/misfortune2/terraform/public.tf:
--------------------------------------------------------------------------------
1 | /*
2 | Client
3 | */
4 | resource "aws_security_group" "wom2-client" {
5 | name = "wom2-client"
6 | description = "Website level security groups."
7 |
8 | ingress {
9 | from_port = 22
10 | to_port = 22
11 | protocol = "tcp"
12 | cidr_blocks = ["0.0.0.0/0"]
13 | }
14 |
15 | ingress {
16 | from_port = -1
17 | to_port = -1
18 | protocol = "icmp"
19 | cidr_blocks = ["0.0.0.0/0"]
20 | }
21 |
22 | egress {
23 | from_port = 0
24 | to_port = 0
25 | protocol = -1
26 | cidr_blocks = ["0.0.0.0/0"]
27 | }
28 |
29 | vpc_id = "${aws_vpc.wom2.id}"
30 |
31 | tags {
32 | Name = "wom2-client"
33 | }
34 | }
35 |
36 | resource "aws_instance" "wom2-client" {
37 | ami = "${lookup(var.amis, var.aws_region)}"
38 | availability_zone = "us-west-2a"
39 | instance_type = "t2.micro"
40 | key_name = "${var.aws_key_name}"
41 | vpc_security_group_ids = ["${aws_security_group.wom2-client.id}"]
42 | subnet_id = "${aws_subnet.wom2-public.id}"
43 | associate_public_ip_address = true
44 | source_dest_check = false
45 |
46 | tags {
47 | Name = "wom2-client"
48 | }
49 | }
50 |
51 | resource "aws_eip" "wom2-client" {
52 | instance = "${aws_instance.wom2-client.id}"
53 | vpc = true
54 | }
55 |
--------------------------------------------------------------------------------
/misfortune3/terraform/client1.tf:
--------------------------------------------------------------------------------
1 | /*
2 | Client
3 | */
4 | resource "aws_security_group" "wom3-client-1" {
5 | name = "wom3-client-1"
6 | description = "Website level security groups."
7 |
8 | ingress {
9 | from_port = 22
10 | to_port = 22
11 | protocol = "tcp"
12 | cidr_blocks = ["0.0.0.0/0"]
13 | }
14 |
15 | ingress {
16 | from_port = -1
17 | to_port = -1
18 | protocol = "icmp"
19 | cidr_blocks = ["0.0.0.0/0"]
20 | }
21 |
22 | egress {
23 | from_port = 0
24 | to_port = 0
25 | protocol = -1
26 | cidr_blocks = ["0.0.0.0/0"]
27 | }
28 |
29 | vpc_id = "${aws_vpc.wom3-1.id}"
30 |
31 | tags {
32 | Name = "wom3-client-1"
33 | }
34 | }
35 |
36 | resource "aws_instance" "wom3-client-1" {
37 | ami = "${lookup(var.amis, var.aws_region)}"
38 | availability_zone = "us-west-2a"
39 | instance_type = "t2.micro"
40 | key_name = "${var.aws_key_name}"
41 | vpc_security_group_ids = ["${aws_security_group.wom3-client-1.id}"]
42 | subnet_id = "${aws_subnet.wom3-public-1-1.id}"
43 | associate_public_ip_address = true
44 | source_dest_check = false
45 |
46 | tags {
47 | Name = "wom3-client-1"
48 | }
49 | }
50 |
51 | resource "aws_eip" "wom3-client-1" {
52 | instance = "${aws_instance.wom3-client-1.id}"
53 | vpc = true
54 | }
55 |
56 |
--------------------------------------------------------------------------------
/misfortune3/terraform/rds1.tf:
--------------------------------------------------------------------------------
1 | /*
2 | RDS
3 | */
4 | resource "aws_security_group" "wom3-rds-1" {
5 | name = "wom3-rds-1"
6 | description = "Website level security groups."
7 |
8 | ingress {
9 | from_port = 5432
10 | to_port = 5432
11 | protocol = "tcp"
12 | security_groups = ["${aws_security_group.wom3-client-1.id}"]
13 | }
14 |
15 | egress {
16 | from_port = 0
17 | to_port = 0
18 | protocol = -1
19 | cidr_blocks = ["0.0.0.0/0"]
20 | }
21 |
22 | vpc_id = "${aws_vpc.wom3-1.id}"
23 |
24 | tags {
25 | Name = "wom3-rds-1"
26 | }
27 | }
28 |
29 | resource "aws_db_subnet_group" "wom3-rds-1" {
30 | name = "wom3-rds-1"
31 | subnet_ids = [
32 | "${aws_subnet.wom3-public-1-1.id}",
33 | "${aws_subnet.wom3-public-1-2.id}"
34 | ]
35 | tags {
36 | Name = "My DB subnet group"
37 | }
38 | }
39 |
40 | resource "aws_db_instance" "wom3-rds-1" {
41 | instance_class = "db.t2.micro"
42 | name = "db1"
43 | engine = "postgres"
44 | engine_version = "9.5"
45 | allocated_storage = "100"
46 | username = "postgres"
47 | password = "postgres"
48 | db_subnet_group_name = "${aws_db_subnet_group.wom3-rds-1.id}"
49 | vpc_security_group_ids = ["${aws_security_group.wom3-rds-1.id}"]
50 |
51 | tags {
52 | Name = "wom3-rds-1"
53 | }
54 | }
55 |
56 |
57 |
--------------------------------------------------------------------------------
/misfortune3/terraform/rds2.tf:
--------------------------------------------------------------------------------
1 | /*
2 | RDS
3 | */
4 | resource "aws_security_group" "wom3-rds-2" {
5 | name = "wom3-rds-2"
6 | description = "Website level security groups."
7 |
8 | ingress {
9 | from_port = 5432
10 | to_port = 5432
11 | protocol = "tcp"
12 | security_groups = ["${aws_security_group.wom3-client-2.id}"]
13 | }
14 |
15 | egress {
16 | from_port = 0
17 | to_port = 0
18 | protocol = -1
19 | cidr_blocks = ["0.0.0.0/0"]
20 | }
21 |
22 | vpc_id = "${aws_vpc.wom3-2.id}"
23 |
24 | tags {
25 | Name = "wom3-rds-2"
26 | }
27 | }
28 |
29 | resource "aws_db_subnet_group" "wom3-rds-2" {
30 | name = "wom3-rds-2"
31 | subnet_ids = [
32 | "${aws_subnet.wom3-public-2-1.id}",
33 | "${aws_subnet.wom3-public-2-2.id}"
34 | ]
35 | tags {
36 | Name = "My DB subnet group"
37 | }
38 | }
39 |
40 | resource "aws_db_instance" "wom3-rds-2" {
41 | instance_class = "db.t2.micro"
42 | name = "db2"
43 | engine = "postgres"
44 | engine_version = "9.5"
45 | allocated_storage = "100"
46 | username = "postgres"
47 | password = "postgres"
48 | db_subnet_group_name = "${aws_db_subnet_group.wom3-rds-2.id}"
49 | vpc_security_group_ids = ["${aws_security_group.wom3-rds-2.id}"]
50 |
51 | tags {
52 | Name = "wom3-rds-2"
53 | }
54 | }
55 |
56 |
57 |
--------------------------------------------------------------------------------
/misfortune2/terraform/vpc.tf:
--------------------------------------------------------------------------------
1 | /*
2 | VPC
3 | */
4 | resource "aws_vpc" "wom2" {
5 | cidr_block = "${var.vpc_cidr}"
6 | enable_dns_hostnames = true
7 | tags {
8 | Name = "wom2"
9 | }
10 | }
11 |
12 | /*
13 | IGW
14 | */
15 | resource "aws_internet_gateway" "wom2" {
16 | vpc_id = "${aws_vpc.wom2.id}"
17 |
18 | tags {
19 | Name = "wom2"
20 | }
21 | }
22 |
23 | /*
24 | Public Subnet in VPC
25 | */
26 | resource "aws_subnet" "wom2-public" {
27 | vpc_id = "${aws_vpc.wom2.id}"
28 |
29 | cidr_block = "${var.public_subnet_cidr}"
30 | availability_zone = "us-west-2a"
31 |
32 | tags {
33 | Name = "wom2-public"
34 | }
35 | }
36 |
37 | resource "aws_route_table" "wom2" {
38 | vpc_id = "${aws_vpc.wom2.id}"
39 |
40 | route {
41 | cidr_block = "0.0.0.0/0"
42 | gateway_id = "${aws_internet_gateway.wom2.id}"
43 | }
44 |
45 | tags {
46 | Name = "wom2"
47 | }
48 | }
49 |
50 | resource "aws_route_table_association" "wom2" {
51 | subnet_id = "${aws_subnet.wom2-public.id}"
52 | route_table_id = "${aws_route_table.wom2.id}"
53 | }
54 |
55 | /*
56 | Private Subnet in VPC
57 | */
58 | resource "aws_subnet" "wom2-private" {
59 | vpc_id = "${aws_vpc.wom2.id}"
60 |
61 | cidr_block = "${var.private_subnet_cidr}"
62 | availability_zone = "us-west-2a"
63 |
64 | tags {
65 | Name = "wom2-private"
66 | }
67 | }
68 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # wheel-of-misfortune
2 |
3 | 1. Install Terraform
4 |
5 | * download zip from [here](https://www.terraform.io/downloads.html)
6 | * `$ unzip terraform_*.zip -d /usr/local/bin`
7 |
8 | 2. Update your `bash_profile`
9 |
10 | ```
11 | export TF_VAR_aws_access_key=${AWS_ACCESS_KEY_ID}
12 | export TF_VAR_aws_secret_key=${AWS_SECRET_ACCESS_KEY}
13 | export TF_VAR_aws_region=${AWS_DEFAULT_REGION}
14 | export TF_VAR_aws_key_path="~/.ssh/wom-us-west-2.pem"
15 | export TF_VAR_aws_key_name="wom"
16 | ```
17 |
18 | 3. You should be able to go into any misfortune folder and run the following:
19 |
20 | * `terraform plan` - shows what will be created (dry-run)
21 | * `terraform apply` - brings up infra
22 | * `terraform plan -destroy` - shows what will be taken down (dry-run)
23 | * `terraform destroy` - takes down infra
24 |
25 | 4. Install Ansible
26 |
27 | * `$ sudo pip install ansible`
28 |
29 | 5. Set your Ansible default configuration in `/etc/ansible/ansible.cfg`
30 |
31 | ```
32 | [defaults]
33 | host_key_checking = False
34 | remote_user = ubuntu
35 | ```
36 |
37 | 6. Download the ec2.ini into `/etc/ansible`
38 |
39 | * `$ sudo wget https://raw.githubusercontent.com/ansible/ansible/devel/contrib/inventory/ec2.ini -P /etc/ansible`
40 |
41 | 7. Download the ec2.py into `/etc/ansible` and make it executable
42 |
43 | * `$ sudo wget https://raw.githubusercontent.com/ansible/ansible/devel/contrib/inventory/ec2.py -P /etc/ansible`
44 | * `$ sudo chmod +x /etc/ansible/ec2.py`
45 |
46 | 8. You can run the Ansible playbooks with the following if there exists a playbook in the current directory
47 |
48 | * `$ ansible-playbook playbook.yaml`
49 |
--------------------------------------------------------------------------------
/misfortune4/terraform/workers.tf:
--------------------------------------------------------------------------------
1 | /*
2 | Spark Workers
3 | */
4 | resource "aws_security_group" "wom4-worker" {
5 | name = "wom4-worker"
6 | description = "Spark worker security group."
7 |
8 | ingress {
9 | from_port = 22
10 | to_port = 22
11 | protocol = "tcp"
12 | cidr_blocks = ["0.0.0.0/0"]
13 | }
14 |
15 | ingress {
16 | from_port = -1
17 | to_port = -1
18 | protocol = "icmp"
19 | cidr_blocks = ["0.0.0.0/0"]
20 | }
21 |
22 | ingress {
23 | from_port = 0
24 | to_port = 65535
25 | protocol = "tcp"
26 | cidr_blocks = ["${var.vpc_cidr}"]
27 | }
28 |
29 | egress {
30 | from_port = 0
31 | to_port = 0
32 | protocol = -1
33 | cidr_blocks = ["0.0.0.0/0"]
34 | }
35 |
36 | vpc_id = "${aws_vpc.wom4.id}"
37 |
38 | tags {
39 | Name = "wom4-worker"
40 | }
41 | }
42 |
43 | resource "aws_instance" "wom4-worker" {
44 | ami = "${lookup(var.amis, var.aws_region)}"
45 | availability_zone = "us-west-2a"
46 | instance_type = "${var.instance_type}"
47 | key_name = "${var.aws_key_name}"
48 | vpc_security_group_ids = ["${aws_security_group.wom4-worker.id}"]
49 | subnet_id = "${aws_subnet.wom4.id}"
50 | associate_public_ip_address = true
51 | source_dest_check = false
52 | count = "${var.num_workers}"
53 |
54 | tags {
55 | Name = "wom4-worker"
56 | Misfortune = "wom4"
57 | }
58 | }
59 |
60 | resource "aws_eip" "wom4-worker" {
61 | count = "${var.num_workers}"
62 | instance = "${element(aws_instance.wom4-worker.*.id, count.index)}"
63 | vpc = true
64 | }
65 |
66 |
--------------------------------------------------------------------------------
/misfortune5/terraform/public.tf:
--------------------------------------------------------------------------------
1 | /*
2 | Server
3 | */
4 | resource "aws_security_group" "wom4" {
5 | name = "wom4"
6 | description = "wom1 server security group."
7 |
8 | ingress {
9 | from_port = 22
10 | to_port = 22
11 | protocol = "tcp"
12 | cidr_blocks = ["0.0.0.0/0"]
13 | }
14 |
15 | egress {
16 | from_port = 0
17 | to_port = 0
18 | protocol = -1
19 | cidr_blocks = ["0.0.0.0/0"]
20 | }
21 |
22 | vpc_id = "${aws_vpc.wom4.id}"
23 |
24 | tags {
25 | Name = "wom4"
26 | }
27 | }
28 |
29 | resource "aws_instance" "wom4-client" {
30 | ami = "${lookup(var.amis, var.aws_region)}"
31 | availability_zone = "us-west-2a"
32 | instance_type = "t2.micro"
33 | key_name = "${var.aws_key_name}"
34 | vpc_security_group_ids = ["${aws_security_group.wom4.id}"]
35 | subnet_id = "${aws_subnet.wom4.id}"
36 | associate_public_ip_address = true
37 | source_dest_check = false
38 |
39 | tags {
40 | Name = "wom4-client"
41 | }
42 | }
43 |
44 | resource "aws_eip" "wom4-client" {
45 | instance = "${aws_instance.wom4-client.id}"
46 | vpc = true
47 | }
48 |
49 | resource "aws_instance" "wom4-api" {
50 | ami = "${lookup(var.amis-api, var.aws_region)}"
51 | availability_zone = "us-west-2a"
52 | instance_type = "t2.micro"
53 | key_name = "${var.aws_key_name}"
54 | vpc_security_group_ids = ["${aws_security_group.wom4.id}"]
55 | subnet_id = "${aws_subnet.wom4.id}"
56 | associate_public_ip_address = true
57 | source_dest_check = false
58 |
59 | tags {
60 | Name = "wom4-api"
61 | }
62 | }
63 |
64 | resource "aws_eip" "wom4-api" {
65 | instance = "${aws_instance.wom4-api.id}"
66 | vpc = true
67 | }
68 |
--------------------------------------------------------------------------------
/misfortune4/ansible/env.yml:
--------------------------------------------------------------------------------
1 | ---
2 | - hosts: tag_Misfortune_wom4:&key_wom
3 | become: true
4 | vars:
5 | home_dir: '/home/ubuntu'
6 | sbt_ver: '0.13.7'
7 | sbt_url: 'https://dl.bintray.com/sbt/debian'
8 | tasks:
9 | - name: Add openjdk and maven3 to apt repository
10 | apt_repository: repo={{item}}
11 | with_items:
12 | - 'ppa:openjdk-r/ppa'
13 | - 'ppa:andrei-pozolotin/maven3'
14 | - name: Update package list
15 | apt: update_cache=yes cache_valid_time=36000
16 | - name: Remove any existing maven install
17 | apt: name={{item}} state=absent purge=yes
18 | with_items:
19 | - maven
20 | - maven2
21 | - maven3
22 | - name: Install base packages
23 | apt: name={{item}} state=present
24 | with_items:
25 | - ssh
26 | - rsync
27 | - openjdk-8-jdk
28 | - maven3
29 | - scala
30 | - python-setuptools
31 | - python-dev
32 | - gcc
33 | - name: Remove Downloads folder
34 | file: path='{{ home_dir }}/Downloads' state=absent
35 | - name: Create Downloads folder
36 | file: path='{{ home_dir }}/Downloads' state=directory mode=0755
37 | - name: Get sbt package into Downloads folder
38 | get_url:
39 | url: '{{ sbt_url }}/sbt-{{ sbt_ver }}.deb'
40 | dest: '{{ home_dir }}/Downloads/sbt-{{ sbt_ver }}.deb'
41 | - name: Install sbt
42 | apt: deb='{{ home_dir }}/Downloads/sbt-{{ sbt_ver }}.deb'
43 | - name: Switch to Java 8
44 | alternatives:
45 | name: java
46 | path: /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
47 | - name: Install pip
48 | easy_install: name=pip state=latest
49 | - name: Install Python packages
50 | pip: name={{item}}
51 | with_items:
52 | - nose
53 | - seaborn
54 | - boto
55 | - scikit-learn
56 | - ipython[notebook]
57 |
58 |
--------------------------------------------------------------------------------
/misfortune4/ansible/templates/spark/conf/spark-defaults.conf.j2:
--------------------------------------------------------------------------------
1 | #
2 | # Licensed to the Apache Software Foundation (ASF) under one or more
3 | # contributor license agreements. See the NOTICE file distributed with
4 | # this work for additional information regarding copyright ownership.
5 | # The ASF licenses this file to You under the Apache License, Version 2.0
6 | # (the "License"); you may not use this file except in compliance with
7 | # the License. You may obtain a copy of the License at
8 | #
9 | # http://www.apache.org/licenses/LICENSE-2.0
10 | #
11 | # Unless required by applicable law or agreed to in writing, software
12 | # distributed under the License is distributed on an "AS IS" BASIS,
13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | # See the License for the specific language governing permissions and
15 | # limitations under the License.
16 | #
17 |
18 | # Default system properties included when running spark-submit.
19 | # This is useful for setting default environmental settings.
20 |
21 | spark.master spark://{{ ec2_facts.instances[0].private_ip_address }}:7077
22 | spark.hadoop.fs.s3a.impl org.apache.hadoop.fs.s3a.S3AFileSystem
23 | spark.executor.extraClassPath /usr/local/spark/lib/aws-java-sdk-{{ aws_java_sdk_ver }}.jar:/usr/local/spark/lib/hadoop-aws-{{ hadoop_ver }}.jar
24 | spark.driver.extraClassPath /usr/local/spark/lib/aws-java-sdk-{{ aws_java_sdk_ver }}.jar:/usr/local/spark/lib/hadoop-aws-{{ hadoop_ver }}.jar
25 |
26 |
27 | # Example:
28 | # spark.master spark://master:7077
29 | # spark.eventLog.enabled true
30 | # spark.eventLog.dir hdfs://namenode:8021/directory
31 | # spark.serializer org.apache.spark.serializer.KryoSerializer
32 | # spark.driver.memory 5g
33 | # spark.executor.extraJavaOptions -XX:+PrintGCDetails -Dkey=value -Dnumbers="one two three"
34 |
--------------------------------------------------------------------------------
/misfortune4/terraform/master.tf:
--------------------------------------------------------------------------------
1 | /*
2 | Spark Master
3 | */
4 | resource "aws_security_group" "wom4-master" {
5 | name = "wom4-master"
6 | description = "Spark master security group."
7 |
8 | ingress {
9 | from_port = 22
10 | to_port = 22
11 | protocol = "tcp"
12 | cidr_blocks = ["0.0.0.0/0"]
13 | }
14 |
15 | ingress {
16 | from_port = -1
17 | to_port = -1
18 | protocol = "icmp"
19 | cidr_blocks = ["0.0.0.0/0"]
20 | }
21 |
22 | ingress {
23 | from_port = 8080
24 | to_port = 8080
25 | protocol = "tcp"
26 | cidr_blocks = ["0.0.0.0/0"]
27 | }
28 |
29 | ingress {
30 | from_port = 4040
31 | to_port = 4040
32 | protocol = "tcp"
33 | cidr_blocks = ["0.0.0.0/0"]
34 | }
35 |
36 | ingress {
37 | from_port = 8888
38 | to_port = 8888
39 | protocol = "tcp"
40 | cidr_blocks = ["0.0.0.0/0"]
41 | }
42 |
43 | ingress {
44 | from_port = 0
45 | to_port = 65535
46 | protocol = "tcp"
47 | cidr_blocks = ["${var.vpc_cidr}"]
48 | }
49 |
50 | egress {
51 | from_port = 0
52 | to_port = 0
53 | protocol = -1
54 | cidr_blocks = ["0.0.0.0/0"]
55 | }
56 |
57 | vpc_id = "${aws_vpc.wom4.id}"
58 |
59 | tags {
60 | Name = "wom4-master"
61 | }
62 | }
63 |
64 | resource "aws_instance" "wom4-master" {
65 | ami = "${lookup(var.amis, var.aws_region)}"
66 | availability_zone = "us-west-2a"
67 | instance_type = "${var.instance_type}"
68 | key_name = "${var.aws_key_name}"
69 | vpc_security_group_ids = ["${aws_security_group.wom4-master.id}"]
70 | subnet_id = "${aws_subnet.wom4.id}"
71 | associate_public_ip_address = true
72 | source_dest_check = false
73 |
74 | tags {
75 | Name = "wom4-master"
76 | Misfortune = "wom4"
77 | }
78 | }
79 |
80 | resource "aws_eip" "wom4-master" {
81 | instance = "${aws_instance.wom4-master.id}"
82 | vpc = true
83 | }
84 |
85 |
--------------------------------------------------------------------------------
/misfortune1/terraform/public.tf:
--------------------------------------------------------------------------------
1 | /*
2 | Client
3 | */
4 | resource "aws_security_group" "wom1-client" {
5 | name = "wom1-client"
6 | description = "wom1 client security group."
7 |
8 | ingress {
9 | from_port = 22
10 | to_port = 22
11 | protocol = "tcp"
12 | cidr_blocks = ["0.0.0.0/0"]
13 | }
14 |
15 | ingress {
16 | from_port = -1
17 | to_port = -1
18 | protocol = "icmp"
19 | cidr_blocks = ["0.0.0.0/0"]
20 | }
21 |
22 | egress {
23 | from_port = 0
24 | to_port = 0
25 | protocol = -1
26 | cidr_blocks = ["0.0.0.0/0"]
27 | }
28 |
29 | vpc_id = "${aws_vpc.wom1.id}"
30 |
31 | tags {
32 | Name = "wom1-client"
33 | }
34 | }
35 |
36 | resource "aws_instance" "wom1-client" {
37 | ami = "${lookup(var.amis, var.aws_region)}"
38 | availability_zone = "us-west-2a"
39 | instance_type = "t2.micro"
40 | key_name = "${var.aws_key_name}"
41 | vpc_security_group_ids = ["${aws_security_group.wom1-client.id}"]
42 | subnet_id = "${aws_subnet.wom1-vpc.id}"
43 | associate_public_ip_address = true
44 | source_dest_check = false
45 | count = "${var.client_cnt}"
46 |
47 | tags {
48 | Name = "wom1-client"
49 | }
50 | }
51 |
52 | resource "aws_eip" "wom1-client" {
53 | count = "${var.client_cnt}"
54 | instance = "${element(aws_instance.wom1-client.*.id, count.index)}"
55 | vpc = true
56 | }
57 |
58 | /*
59 | Server
60 | */
61 | resource "aws_security_group" "wom1-server" {
62 | name = "wom1-server"
63 | description = "wom1 server security group."
64 |
65 | ingress {
66 | from_port = 22
67 | to_port = 22
68 | protocol = "tcp"
69 | cidr_blocks = ["0.0.0.0/0"]
70 | }
71 |
72 | egress {
73 | from_port = 0
74 | to_port = 0
75 | protocol = -1
76 | cidr_blocks = ["0.0.0.0/0"]
77 | }
78 |
79 | vpc_id = "${aws_vpc.wom1.id}"
80 |
81 | tags {
82 | Name = "wom1-server"
83 | }
84 | }
85 |
86 | resource "aws_instance" "wom1-server" {
87 | ami = "${lookup(var.amis, var.aws_region)}"
88 | availability_zone = "us-west-2a"
89 | instance_type = "t2.micro"
90 | key_name = "${var.aws_key_name}"
91 | vpc_security_group_ids = ["${aws_security_group.wom1-server.id}"]
92 | subnet_id = "${aws_subnet.wom1-vpc.id}"
93 | associate_public_ip_address = true
94 | source_dest_check = false
95 |
96 | tags {
97 | Name = "wom1-server"
98 | }
99 | }
100 |
101 | resource "aws_eip" "wom1-server" {
102 | instance = "${aws_instance.wom1-server.id}"
103 | vpc = true
104 | }
105 |
--------------------------------------------------------------------------------
/misfortune1/ansible/playbook.yml:
--------------------------------------------------------------------------------
1 | ---
2 | - hosts: tag_Name_wom1_server:&key_wom
3 | become: true
4 | tasks:
5 | - name: Update package list
6 | apt: update_cache=yes cache_valid_time=36000
7 | - name: Install base packages
8 | apt: name={{item}} state=present
9 | with_items:
10 | - python-pip
11 | - python-dev
12 | - nginx
13 | - postgresql
14 | - name: Copy postgres config to remote
15 | copy:
16 | src: templates/etc/postgresql/9.3/main/postgresql.conf
17 | dest: /etc/postgresql/9.3/main/postgresql.conf
18 | - name: Restart postgres
19 | service:
20 | name: postgresql
21 | state: restarted
22 | - name: Install uwsgi and flask for python
23 | pip: name={{item}} state=present
24 | with_items:
25 | - uwsgi
26 | - flask
27 | - name: Copy myproject over to web server nodes
28 | synchronize:
29 | src: src/myproject
30 | dest: /home/ubuntu
31 | - name: Change ownership of myproject to ubuntu
32 | file:
33 | dest: /home/ubuntu/myproject
34 | owner: ubuntu
35 | group: ubuntu
36 | recurse: yes
37 | - name: Copy wsgi myproject.conf to remote
38 | copy:
39 | src: templates/etc/init/myproject.conf
40 | dest: /etc/init/myproject.conf
41 | - name: Copy nginx myproject config template
42 | template:
43 | src: templates/etc/nginx/sites-available/myproject.j2
44 | dest: /etc/nginx/sites-available/myproject
45 | - name: Create symbolic link for nginx myproject web application
46 | file:
47 | src: /etc/nginx/sites-available/myproject
48 | dest: /etc/nginx/sites-enabled/myproject
49 | state: link
50 | - name: Copy myotherproject over to web server nodes
51 | synchronize:
52 | src: src/myotherproject
53 | dest: /home/ubuntu
54 | - name: Change ownership of myotherproject to ubuntu
55 | file:
56 | dest: /home/ubuntu/myotherproject
57 | owner: ubuntu
58 | group: ubuntu
59 | recurse: yes
60 | - name: Copy wsgi myotherproject.conf to remote
61 | copy:
62 | src: templates/etc/init/myotherproject.conf
63 | dest: /etc/init/myotherproject.conf
64 | - name: Copy nginx myotherproject config template
65 | template:
66 | src: templates/etc/nginx/sites-available/myotherproject.j2
67 | dest: /etc/nginx/sites-available/myotherproject
68 | - name: Create symbolic link for nginx myotherproject web application
69 | file:
70 | src: /etc/nginx/sites-available/myotherproject
71 | dest: /etc/nginx/sites-enabled/myotherproject
72 | state: link
73 | - name: Delete default config for nginx web application
74 | file:
75 | path: /etc/nginx/sites-available/default
76 | state: absent
77 | - name: Start wsgi process for myproject
78 | service:
79 | name: myproject
80 | state: restarted
81 | - name: Start wsgi process for myotherproject
82 | service:
83 | name: myotherproject
84 | state: restarted
85 | - name: Start nginx process
86 | service:
87 | name: nginx
88 | state: restarted
89 |
90 |
--------------------------------------------------------------------------------
/misfortune3/terraform/vpc.tf:
--------------------------------------------------------------------------------
1 | /*
2 | VPC
3 | */
4 | resource "aws_vpc" "wom3-1" {
5 | cidr_block = "${var.vpc_cidr_1}"
6 | enable_dns_hostnames = true
7 | tags {
8 | Name = "wom3-1"
9 | }
10 | }
11 |
12 | resource "aws_vpc" "wom3-2" {
13 | cidr_block = "${var.vpc_cidr_2}"
14 | enable_dns_hostnames = true
15 | tags {
16 | Name = "wom3-2"
17 | }
18 | }
19 |
20 | /*
21 | IGW
22 | */
23 | resource "aws_internet_gateway" "wom3-1" {
24 | vpc_id = "${aws_vpc.wom3-1.id}"
25 |
26 | tags {
27 | Name = "wom3-1"
28 | }
29 | }
30 |
31 | resource "aws_internet_gateway" "wom3-2" {
32 | vpc_id = "${aws_vpc.wom3-2.id}"
33 |
34 | tags {
35 | Name = "wom3-2"
36 | }
37 | }
38 |
39 | /*
40 | Public Subnet in VPC
41 | */
42 | resource "aws_subnet" "wom3-public-1-1" {
43 | vpc_id = "${aws_vpc.wom3-1.id}"
44 |
45 | cidr_block = "${var.public_subnet_cidr_1_1}"
46 | availability_zone = "us-west-2a"
47 |
48 | tags {
49 | Name = "wom3-public-1-1"
50 | }
51 | }
52 |
53 | resource "aws_subnet" "wom3-public-1-2" {
54 | vpc_id = "${aws_vpc.wom3-1.id}"
55 |
56 | cidr_block = "${var.public_subnet_cidr_1_2}"
57 | availability_zone = "us-west-2b"
58 |
59 | tags {
60 | Name = "wom3-public-1-2"
61 | }
62 | }
63 |
64 | resource "aws_subnet" "wom3-public-2-1" {
65 | vpc_id = "${aws_vpc.wom3-2.id}"
66 |
67 | cidr_block = "${var.public_subnet_cidr_2_1}"
68 | availability_zone = "us-west-2a"
69 |
70 | tags {
71 | Name = "wom3-public-2-1"
72 | }
73 | }
74 |
75 | resource "aws_subnet" "wom3-public-2-2" {
76 | vpc_id = "${aws_vpc.wom3-2.id}"
77 |
78 | cidr_block = "${var.public_subnet_cidr_2_2}"
79 | availability_zone = "us-west-2b"
80 |
81 | tags {
82 | Name = "wom3-public-2-2"
83 | }
84 | }
85 |
86 | resource "aws_route_table" "wom3-1" {
87 | vpc_id = "${aws_vpc.wom3-1.id}"
88 |
89 | route {
90 | cidr_block = "0.0.0.0/0"
91 | gateway_id = "${aws_internet_gateway.wom3-1.id}"
92 | }
93 |
94 | tags {
95 | Name = "wom3-1"
96 | }
97 | }
98 |
99 | resource "aws_route_table" "wom3-2" {
100 | vpc_id = "${aws_vpc.wom3-2.id}"
101 |
102 | route {
103 | cidr_block = "0.0.0.0/0"
104 | gateway_id = "${aws_internet_gateway.wom3-2.id}"
105 | }
106 |
107 | tags {
108 | Name = "wom3-2"
109 | }
110 | }
111 |
112 | resource "aws_route_table_association" "wom3-1-1" {
113 | subnet_id = "${aws_subnet.wom3-public-1-1.id}"
114 | route_table_id = "${aws_route_table.wom3-1.id}"
115 | }
116 |
117 | resource "aws_route_table_association" "wom3-1-2" {
118 | subnet_id = "${aws_subnet.wom3-public-1-2.id}"
119 | route_table_id = "${aws_route_table.wom3-1.id}"
120 | }
121 |
122 | resource "aws_route_table_association" "wom3-2-1" {
123 | subnet_id = "${aws_subnet.wom3-public-2-1.id}"
124 | route_table_id = "${aws_route_table.wom3-2.id}"
125 | }
126 |
127 | resource "aws_route_table_association" "wom3-2-2" {
128 | subnet_id = "${aws_subnet.wom3-public-2-2.id}"
129 | route_table_id = "${aws_route_table.wom3-2.id}"
130 | }
131 |
132 |
133 |
--------------------------------------------------------------------------------
/misfortune4/ansible/install.yml:
--------------------------------------------------------------------------------
1 | ---
2 | - hosts: tag_Misfortune_wom4:&key_wom
3 | become: true
4 | vars:
5 | home_dir: '/home/ubuntu'
6 | hadoop_ver: '2.7.2'
7 | spark_ver: '1.6.2'
8 | spark_hadoop_ver: '2.6'
9 | aws_java_sdk_ver: '1.7.4'
10 | hadoop_url: 'http://www.us.apache.org/dist/hadoop/core'
11 | spark_url: 'http://www.us.apache.org/dist/spark'
12 | aws_access_key_id: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}"
13 | aws_secret_access_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}"
14 | aws_default_region: "{{ lookup('env', 'AWS_DEFAULT_REGION') }}"
15 | tasks:
16 | - name: Download Hadoop
17 | get_url:
18 | url: '{{ hadoop_url }}/hadoop-{{ hadoop_ver }}/hadoop-{{ hadoop_ver }}.tar.gz'
19 | dest: '{{ home_dir }}/Downloads/hadoop-{{ hadoop_ver }}.tar.gz'
20 | - name: Unpack Hadoop
21 | unarchive:
22 | remote_src: yes
23 | src: '{{ home_dir }}/Downloads/hadoop-{{ hadoop_ver }}.tar.gz'
24 | dest: /usr/local
25 | - name: Rename Hadoop directory
26 | command: 'mv /usr/local/hadoop-{{ hadoop_ver }} /usr/local/hadoop'
27 | args:
28 | creates: /usr/local/hadoop
29 | removes: '/usr/local/hadoop-{{ hadoop_ver }}'
30 | - name: Change ownership for Hadoop directory
31 | file:
32 | path: /usr/local/hadoop
33 | owner: ubuntu
34 | group: ubuntu
35 | recurse: yes
36 | - name: Download Spark
37 | get_url:
38 | url: '{{ spark_url }}/spark-{{ spark_ver }}/spark-{{ spark_ver }}-bin-hadoop{{ spark_hadoop_ver }}.tgz'
39 | dest: '{{ home_dir }}/Downloads/spark-{{ spark_ver }}-bin-hadoop{{ spark_hadoop_ver }}.tgz'
40 | - name: Unpack Spark
41 | unarchive:
42 | remote_src: yes
43 | src: '{{ home_dir }}/Downloads/spark-{{ spark_ver }}-bin-hadoop{{ spark_hadoop_ver }}.tgz'
44 | dest: /usr/local
45 | - name: Rename Spark directory
46 | command: 'mv /usr/local/spark-{{ spark_ver }}-bin-hadoop{{ spark_hadoop_ver }} /usr/local/spark'
47 | args:
48 | creates: /usr/local/spark
49 | removes: '/usr/local/spark-{{ spark_ver }}-bin-hadoop{{ spark_hadoop_ver }}'
50 | - name: Change ownership for Spark directory
51 | file:
52 | path: /usr/local/spark
53 | owner: ubuntu
54 | group: ubuntu
55 | recurse: yes
56 | - name: Copy profile to remote
57 | template:
58 | src: 'templates/{{ home_dir }}/.profile.j2'
59 | dest: '{{ home_dir }}/.profile'
60 | - name: Copy aws jars from Hadoop to Spark lib
61 | copy: remote_src=true src={{item}} dest='/usr/local/spark/lib'
62 | with_items:
63 | - '/usr/local/hadoop/share/hadoop/tools/lib/aws-java-sdk-{{ aws_java_sdk_ver }}.jar'
64 | - '/usr/local/hadoop/share/hadoop/tools/lib/hadoop-aws-{{ hadoop_ver }}.jar'
65 | - name: Get master node private IP
66 | ec2_remote_facts:
67 | aws_secret_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}"
68 | aws_access_key: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}"
69 | region: "{{ lookup('env', 'AWS_DEFAULT_REGION') }}"
70 | filters:
71 | instance-state-name: running
72 | "tag:Name": wom4-master
73 | register: ec2_facts
74 | - name: Copy spark env template to remote
75 | template:
76 | src: templates/spark/conf/spark-env.sh.j2
77 | dest: /usr/local/spark/conf/spark-env.sh
78 | - name: Copy spark conf template to remote
79 | template:
80 | src: templates/spark/conf/spark-defaults.conf.j2
81 | dest: /usr/local/spark/conf/spark-defaults.conf
82 | - name: Copy ipython start script to remote
83 | template:
84 | src: templates/spark/sbin/start-ipython.sh.j2
85 | dest: /usr/local/spark/sbin/start-ipython.sh
86 | mode: 0755
87 | - name: Copy ipython stop script to remote
88 | copy:
89 | src: templates/spark/sbin/stop-ipython.sh
90 | dest: /usr/local/spark/sbin/stop-ipython.sh
91 | mode: 0755
92 |
93 |
94 |
--------------------------------------------------------------------------------
/misfortune4/ansible/templates/spark/conf/spark-env.sh.j2:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | #
4 | # Licensed to the Apache Software Foundation (ASF) under one or more
5 | # contributor license agreements. See the NOTICE file distributed with
6 | # this work for additional information regarding copyright ownership.
7 | # The ASF licenses this file to You under the Apache License, Version 2.0
8 | # (the "License"); you may not use this file except in compliance with
9 | # the License. You may obtain a copy of the License at
10 | #
11 | # http://www.apache.org/licenses/LICENSE-2.0
12 | #
13 | # Unless required by applicable law or agreed to in writing, software
14 | # distributed under the License is distributed on an "AS IS" BASIS,
15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | # See the License for the specific language governing permissions and
17 | # limitations under the License.
18 | #
19 |
20 | # This file is sourced when running various Spark programs.
21 | # Copy it as spark-env.sh and edit that to configure Spark for your site.
22 |
23 | export JAVA_HOME=/usr
24 | export SPARK_WORKER_CORES={{ ansible_processor_cores * 3 }}
25 | export SPARK_MASTER_IP={{ ec2_facts.instances[0].private_ip_address }}
26 | export DEFAULT_HADOOP_HOME=/usr/local/hadoop
27 |
28 | # Options read when launching programs locally with
29 | # ./bin/run-example or ./bin/spark-submit
30 | # - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files
31 | # - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node
32 | # - SPARK_PUBLIC_DNS, to set the public dns name of the driver program
33 | # - SPARK_CLASSPATH, default classpath entries to append
34 |
35 | # Options read by executors and drivers running inside the cluster
36 | # - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node
37 | # - SPARK_PUBLIC_DNS, to set the public DNS name of the driver program
38 | # - SPARK_CLASSPATH, default classpath entries to append
39 | # - SPARK_LOCAL_DIRS, storage directories to use on this node for shuffle and RDD data
40 | # - MESOS_NATIVE_JAVA_LIBRARY, to point to your libmesos.so if you use Mesos
41 |
42 | # Options read in YARN client mode
43 | # - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files
44 | # - SPARK_EXECUTOR_INSTANCES, Number of executors to start (Default: 2)
45 | # - SPARK_EXECUTOR_CORES, Number of cores for the executors (Default: 1).
46 | # - SPARK_EXECUTOR_MEMORY, Memory per Executor (e.g. 1000M, 2G) (Default: 1G)
47 | # - SPARK_DRIVER_MEMORY, Memory for Driver (e.g. 1000M, 2G) (Default: 1G)
48 | # - SPARK_YARN_APP_NAME, The name of your application (Default: Spark)
49 | # - SPARK_YARN_QUEUE, The hadoop queue to use for allocation requests (Default: ‘default’)
50 | # - SPARK_YARN_DIST_FILES, Comma separated list of files to be distributed with the job.
51 | # - SPARK_YARN_DIST_ARCHIVES, Comma separated list of archives to be distributed with the job.
52 |
53 | # Options for the daemons used in the standalone deploy mode
54 | # - SPARK_MASTER_IP, to bind the master to a different IP address or hostname
55 | # - SPARK_MASTER_PORT / SPARK_MASTER_WEBUI_PORT, to use non-default ports for the master
56 | # - SPARK_MASTER_OPTS, to set config properties only for the master (e.g. "-Dx=y")
57 | # - SPARK_WORKER_CORES, to set the number of cores to use on this machine
58 | # - SPARK_WORKER_MEMORY, to set how much total memory workers have to give executors (e.g. 1000m, 2g)
59 | # - SPARK_WORKER_PORT / SPARK_WORKER_WEBUI_PORT, to use non-default ports for the worker
60 | # - SPARK_WORKER_INSTANCES, to set the number of worker processes per node
61 | # - SPARK_WORKER_DIR, to set the working directory of worker processes
62 | # - SPARK_WORKER_OPTS, to set config properties only for the worker (e.g. "-Dx=y")
63 | # - SPARK_DAEMON_MEMORY, to allocate to the master, worker and history server themselves (default: 1g).
64 | # - SPARK_HISTORY_OPTS, to set config properties only for the history server (e.g. "-Dx=y")
65 | # - SPARK_SHUFFLE_OPTS, to set config properties only for the external shuffle service (e.g. "-Dx=y")
66 | # - SPARK_DAEMON_JAVA_OPTS, to set config properties for all daemons (e.g. "-Dx=y")
67 | # - SPARK_PUBLIC_DNS, to set the public dns name of the master or workers
68 |
69 | # Generic options for the daemons used in the standalone deploy mode
70 | # - SPARK_CONF_DIR Alternate conf dir. (Default: ${SPARK_HOME}/conf)
71 | # - SPARK_LOG_DIR Where log files are stored. (Default: ${SPARK_HOME}/logs)
72 | # - SPARK_PID_DIR Where the pid file is stored. (Default: /tmp)
73 | # - SPARK_IDENT_STRING A string representing this instance of spark. (Default: $USER)
74 | # - SPARK_NICENESS The scheduling priority for daemons. (Default: 0)
75 |
--------------------------------------------------------------------------------
/misfortune1/ansible/templates/etc/postgresql/9.3/main/postgresql.conf:
--------------------------------------------------------------------------------
1 | # -----------------------------
2 | # PostgreSQL configuration file
3 | # -----------------------------
4 | #
5 | # This file consists of lines of the form:
6 | #
7 | # name = value
8 | #
9 | # (The "=" is optional.) Whitespace may be used. Comments are introduced with
10 | # "#" anywhere on a line. The complete list of parameter names and allowed
11 | # values can be found in the PostgreSQL documentation.
12 | #
13 | # The commented-out settings shown in this file represent the default values.
14 | # Re-commenting a setting is NOT sufficient to revert it to the default value;
15 | # you need to reload the server.
16 | #
17 | # This file is read on server startup and when the server receives a SIGHUP
18 | # signal. If you edit the file on a running system, you have to SIGHUP the
19 | # server for the changes to take effect, or use "pg_ctl reload". Some
20 | # parameters, which are marked below, require a server shutdown and restart to
21 | # take effect.
22 | #
23 | # Any parameter can also be given as a command-line option to the server, e.g.,
24 | # "postgres -c log_connections=on". Some parameters can be changed at run time
25 | # with the "SET" SQL command.
26 | #
27 | # Memory units: kB = kilobytes Time units: ms = milliseconds
28 | # MB = megabytes s = seconds
29 | # GB = gigabytes min = minutes
30 | # h = hours
31 | # d = days
32 |
33 |
34 | #------------------------------------------------------------------------------
35 | # FILE LOCATIONS
36 | #------------------------------------------------------------------------------
37 |
38 | # The default values of these variables are driven from the -D command-line
39 | # option or PGDATA environment variable, represented here as ConfigDir.
40 |
41 | data_directory = '/var/lib/postgresql/9.3/main' # use data in another directory
42 | # (change requires restart)
43 | hba_file = '/etc/postgresql/9.3/main/pg_hba.conf' # host-based authentication file
44 | # (change requires restart)
45 | ident_file = '/etc/postgresql/9.3/main/pg_ident.conf' # ident configuration file
46 | # (change requires restart)
47 |
48 | # If external_pid_file is not explicitly set, no extra PID file is written.
49 | external_pid_file = '/var/run/postgresql/9.3-main.pid' # write an extra PID file
50 | # (change requires restart)
51 |
52 |
53 | #------------------------------------------------------------------------------
54 | # CONNECTIONS AND AUTHENTICATION
55 | #------------------------------------------------------------------------------
56 |
57 | # - Connection Settings -
58 |
59 | listen_addresses = '*' # what IP address(es) to listen on;
60 | # comma-separated list of addresses;
61 | # defaults to 'localhost'; use '*' for all
62 | # (change requires restart)
63 | port = 5432 # (change requires restart)
64 | max_connections = 100 # (change requires restart)
65 | #superuser_reserved_connections = 3 # (change requires restart)
66 | unix_socket_directories = '/var/run/postgresql' # comma-separated list of directories
67 | # (change requires restart)
68 | #unix_socket_group = '' # (change requires restart)
69 | #unix_socket_permissions = 0777 # begin with 0 to use octal notation
70 | # (change requires restart)
71 | #bonjour = off # advertise server via Bonjour
72 | # (change requires restart)
73 | #bonjour_name = '' # defaults to the computer name
74 | # (change requires restart)
75 |
76 | # - Security and Authentication -
77 |
78 | #authentication_timeout = 1min # 1s-600s
79 | ssl = true # (change requires restart)
80 | #ssl_ciphers = 'DEFAULT:!LOW:!EXP:!MD5:@STRENGTH' # allowed SSL ciphers
81 | # (change requires restart)
82 | #ssl_renegotiation_limit = 0 # amount of data between renegotiations
83 | ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem' # (change requires restart)
84 | ssl_key_file = '/etc/ssl/private/ssl-cert-snakeoil.key' # (change requires restart)
85 | #ssl_ca_file = '' # (change requires restart)
86 | #ssl_crl_file = '' # (change requires restart)
87 | #password_encryption = on
88 | #db_user_namespace = off
89 |
90 | # Kerberos and GSSAPI
91 | #krb_server_keyfile = ''
92 | #krb_srvname = 'postgres' # (Kerberos only)
93 | #krb_caseins_users = off
94 |
95 | # - TCP Keepalives -
96 | # see "man 7 tcp" for details
97 |
98 | #tcp_keepalives_idle = 0 # TCP_KEEPIDLE, in seconds;
99 | # 0 selects the system default
100 | #tcp_keepalives_interval = 0 # TCP_KEEPINTVL, in seconds;
101 | # 0 selects the system default
102 | #tcp_keepalives_count = 0 # TCP_KEEPCNT;
103 | # 0 selects the system default
104 |
105 |
106 | #------------------------------------------------------------------------------
107 | # RESOURCE USAGE (except WAL)
108 | #------------------------------------------------------------------------------
109 |
110 | # - Memory -
111 |
112 | shared_buffers = 128MB # min 128kB
113 | # (change requires restart)
114 | #temp_buffers = 8MB # min 800kB
115 | #max_prepared_transactions = 0 # zero disables the feature
116 | # (change requires restart)
117 | # Caution: it is not advisable to set max_prepared_transactions nonzero unless
118 | # you actively intend to use prepared transactions.
119 | #work_mem = 1MB # min 64kB
120 | #maintenance_work_mem = 16MB # min 1MB
121 | #max_stack_depth = 2MB # min 100kB
122 |
123 | # - Disk -
124 |
125 | #temp_file_limit = -1 # limits per-session temp file space
126 | # in kB, or -1 for no limit
127 |
128 | # - Kernel Resource Usage -
129 |
130 | #max_files_per_process = 1000 # min 25
131 | # (change requires restart)
132 | #shared_preload_libraries = '' # (change requires restart)
133 |
134 | # - Cost-Based Vacuum Delay -
135 |
136 | #vacuum_cost_delay = 0 # 0-100 milliseconds
137 | #vacuum_cost_page_hit = 1 # 0-10000 credits
138 | #vacuum_cost_page_miss = 10 # 0-10000 credits
139 | #vacuum_cost_page_dirty = 20 # 0-10000 credits
140 | #vacuum_cost_limit = 200 # 1-10000 credits
141 |
142 | # - Background Writer -
143 |
144 | #bgwriter_delay = 200ms # 10-10000ms between rounds
145 | #bgwriter_lru_maxpages = 100 # 0-1000 max buffers written/round
146 | #bgwriter_lru_multiplier = 2.0 # 0-10.0 multipler on buffers scanned/round
147 |
148 | # - Asynchronous Behavior -
149 |
150 | #effective_io_concurrency = 1 # 1-1000; 0 disables prefetching
151 |
152 |
153 | #------------------------------------------------------------------------------
154 | # WRITE AHEAD LOG
155 | #------------------------------------------------------------------------------
156 |
157 | # - Settings -
158 |
159 | #wal_level = minimal # minimal, archive, or hot_standby
160 | # (change requires restart)
161 | #fsync = on # turns forced synchronization on or off
162 | #synchronous_commit = on # synchronization level;
163 | # off, local, remote_write, or on
164 | #wal_sync_method = fsync # the default is the first option
165 | # supported by the operating system:
166 | # open_datasync
167 | # fdatasync (default on Linux)
168 | # fsync
169 | # fsync_writethrough
170 | # open_sync
171 | #full_page_writes = on # recover from partial page writes
172 | #wal_buffers = -1 # min 32kB, -1 sets based on shared_buffers
173 | # (change requires restart)
174 | #wal_writer_delay = 200ms # 1-10000 milliseconds
175 |
176 | #commit_delay = 0 # range 0-100000, in microseconds
177 | #commit_siblings = 5 # range 1-1000
178 |
179 | # - Checkpoints -
180 |
181 | #checkpoint_segments = 3 # in logfile segments, min 1, 16MB each
182 | #checkpoint_timeout = 5min # range 30s-1h
183 | #checkpoint_completion_target = 0.5 # checkpoint target duration, 0.0 - 1.0
184 | #checkpoint_warning = 30s # 0 disables
185 |
186 | # - Archiving -
187 |
188 | #archive_mode = off # allows archiving to be done
189 | # (change requires restart)
190 | #archive_command = '' # command to use to archive a logfile segment
191 | # placeholders: %p = path of file to archive
192 | # %f = file name only
193 | # e.g. 'test ! -f /mnt/server/archivedir/%f && cp %p /mnt/server/archivedir/%f'
194 | #archive_timeout = 0 # force a logfile segment switch after this
195 | # number of seconds; 0 disables
196 |
197 |
198 | #------------------------------------------------------------------------------
199 | # REPLICATION
200 | #------------------------------------------------------------------------------
201 |
202 | # - Sending Server(s) -
203 |
204 | # Set these on the master and on any standby that will send replication data.
205 |
206 | #max_wal_senders = 0 # max number of walsender processes
207 | # (change requires restart)
208 | #wal_keep_segments = 0 # in logfile segments, 16MB each; 0 disables
209 | #wal_sender_timeout = 60s # in milliseconds; 0 disables
210 |
211 | # - Master Server -
212 |
213 | # These settings are ignored on a standby server.
214 |
215 | #synchronous_standby_names = '' # standby servers that provide sync rep
216 | # comma-separated list of application_name
217 | # from standby(s); '*' = all
218 | #vacuum_defer_cleanup_age = 0 # number of xacts by which cleanup is delayed
219 |
220 | # - Standby Servers -
221 |
222 | # These settings are ignored on a master server.
223 |
224 | #hot_standby = off # "on" allows queries during recovery
225 | # (change requires restart)
226 | #max_standby_archive_delay = 30s # max delay before canceling queries
227 | # when reading WAL from archive;
228 | # -1 allows indefinite delay
229 | #max_standby_streaming_delay = 30s # max delay before canceling queries
230 | # when reading streaming WAL;
231 | # -1 allows indefinite delay
232 | #wal_receiver_status_interval = 10s # send replies at least this often
233 | # 0 disables
234 | #hot_standby_feedback = off # send info from standby to prevent
235 | # query conflicts
236 | #wal_receiver_timeout = 60s # time that receiver waits for
237 | # communication from master
238 | # in milliseconds; 0 disables
239 |
240 |
241 | #------------------------------------------------------------------------------
242 | # QUERY TUNING
243 | #------------------------------------------------------------------------------
244 |
245 | # - Planner Method Configuration -
246 |
247 | #enable_bitmapscan = on
248 | #enable_hashagg = on
249 | #enable_hashjoin = on
250 | #enable_indexscan = on
251 | #enable_indexonlyscan = on
252 | #enable_material = on
253 | #enable_mergejoin = on
254 | #enable_nestloop = on
255 | #enable_seqscan = on
256 | #enable_sort = on
257 | #enable_tidscan = on
258 |
259 | # - Planner Cost Constants -
260 |
261 | #seq_page_cost = 1.0 # measured on an arbitrary scale
262 | #random_page_cost = 4.0 # same scale as above
263 | #cpu_tuple_cost = 0.01 # same scale as above
264 | #cpu_index_tuple_cost = 0.005 # same scale as above
265 | #cpu_operator_cost = 0.0025 # same scale as above
266 | #effective_cache_size = 128MB
267 |
268 | # - Genetic Query Optimizer -
269 |
270 | #geqo = on
271 | #geqo_threshold = 12
272 | #geqo_effort = 5 # range 1-10
273 | #geqo_pool_size = 0 # selects default based on effort
274 | #geqo_generations = 0 # selects default based on effort
275 | #geqo_selection_bias = 2.0 # range 1.5-2.0
276 | #geqo_seed = 0.0 # range 0.0-1.0
277 |
278 | # - Other Planner Options -
279 |
280 | #default_statistics_target = 100 # range 1-10000
281 | #constraint_exclusion = partition # on, off, or partition
282 | #cursor_tuple_fraction = 0.1 # range 0.0-1.0
283 | #from_collapse_limit = 8
284 | #join_collapse_limit = 8 # 1 disables collapsing of explicit
285 | # JOIN clauses
286 |
287 |
288 | #------------------------------------------------------------------------------
289 | # ERROR REPORTING AND LOGGING
290 | #------------------------------------------------------------------------------
291 |
292 | # - Where to Log -
293 |
294 | #log_destination = 'stderr' # Valid values are combinations of
295 | # stderr, csvlog, syslog, and eventlog,
296 | # depending on platform. csvlog
297 | # requires logging_collector to be on.
298 |
299 | # This is used when logging to stderr:
300 | #logging_collector = off # Enable capturing of stderr and csvlog
301 | # into log files. Required to be on for
302 | # csvlogs.
303 | # (change requires restart)
304 |
305 | # These are only used if logging_collector is on:
306 | #log_directory = 'pg_log' # directory where log files are written,
307 | # can be absolute or relative to PGDATA
308 | #log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' # log file name pattern,
309 | # can include strftime() escapes
310 | #log_file_mode = 0600 # creation mode for log files,
311 | # begin with 0 to use octal notation
312 | #log_truncate_on_rotation = off # If on, an existing log file with the
313 | # same name as the new log file will be
314 | # truncated rather than appended to.
315 | # But such truncation only occurs on
316 | # time-driven rotation, not on restarts
317 | # or size-driven rotation. Default is
318 | # off, meaning append to existing files
319 | # in all cases.
320 | #log_rotation_age = 1d # Automatic rotation of logfiles will
321 | # happen after that time. 0 disables.
322 | #log_rotation_size = 10MB # Automatic rotation of logfiles will
323 | # happen after that much log output.
324 | # 0 disables.
325 |
326 | # These are relevant when logging to syslog:
327 | #syslog_facility = 'LOCAL0'
328 | #syslog_ident = 'postgres'
329 |
330 | # This is only relevant when logging to eventlog (win32):
331 | #event_source = 'PostgreSQL'
332 |
333 | # - When to Log -
334 |
335 | #client_min_messages = notice # values in order of decreasing detail:
336 | # debug5
337 | # debug4
338 | # debug3
339 | # debug2
340 | # debug1
341 | # log
342 | # notice
343 | # warning
344 | # error
345 |
346 | #log_min_messages = warning # values in order of decreasing detail:
347 | # debug5
348 | # debug4
349 | # debug3
350 | # debug2
351 | # debug1
352 | # info
353 | # notice
354 | # warning
355 | # error
356 | # log
357 | # fatal
358 | # panic
359 |
360 | #log_min_error_statement = error # values in order of decreasing detail:
361 | # debug5
362 | # debug4
363 | # debug3
364 | # debug2
365 | # debug1
366 | # info
367 | # notice
368 | # warning
369 | # error
370 | # log
371 | # fatal
372 | # panic (effectively off)
373 |
374 | #log_min_duration_statement = -1 # -1 is disabled, 0 logs all statements
375 | # and their durations, > 0 logs only
376 | # statements running at least this number
377 | # of milliseconds
378 |
379 |
380 | # - What to Log -
381 |
382 | #debug_print_parse = off
383 | #debug_print_rewritten = off
384 | #debug_print_plan = off
385 | #debug_pretty_print = on
386 | #log_checkpoints = off
387 | #log_connections = off
388 | #log_disconnections = off
389 | #log_duration = off
390 | #log_error_verbosity = default # terse, default, or verbose messages
391 | #log_hostname = off
392 | log_line_prefix = '%t ' # special values:
393 | # %a = application name
394 | # %u = user name
395 | # %d = database name
396 | # %r = remote host and port
397 | # %h = remote host
398 | # %p = process ID
399 | # %t = timestamp without milliseconds
400 | # %m = timestamp with milliseconds
401 | # %i = command tag
402 | # %e = SQL state
403 | # %c = session ID
404 | # %l = session line number
405 | # %s = session start timestamp
406 | # %v = virtual transaction ID
407 | # %x = transaction ID (0 if none)
408 | # %q = stop here in non-session
409 | # processes
410 | # %% = '%'
411 | # e.g. '<%u%%%d> '
412 | #log_lock_waits = off # log lock waits >= deadlock_timeout
413 | #log_statement = 'none' # none, ddl, mod, all
414 | #log_temp_files = -1 # log temporary files equal or larger
415 | # than the specified size in kilobytes;
416 | # -1 disables, 0 logs all temp files
417 | log_timezone = 'UTC'
418 |
419 |
420 | #------------------------------------------------------------------------------
421 | # RUNTIME STATISTICS
422 | #------------------------------------------------------------------------------
423 |
424 | # - Query/Index Statistics Collector -
425 |
426 | #track_activities = on
427 | #track_counts = on
428 | #track_io_timing = off
429 | #track_functions = none # none, pl, all
430 | #track_activity_query_size = 1024 # (change requires restart)
431 | #update_process_title = on
432 | #stats_temp_directory = 'pg_stat_tmp'
433 |
434 |
435 | # - Statistics Monitoring -
436 |
437 | #log_parser_stats = off
438 | #log_planner_stats = off
439 | #log_executor_stats = off
440 | #log_statement_stats = off
441 |
442 |
443 | #------------------------------------------------------------------------------
444 | # AUTOVACUUM PARAMETERS
445 | #------------------------------------------------------------------------------
446 |
447 | #autovacuum = on # Enable autovacuum subprocess? 'on'
448 | # requires track_counts to also be on.
449 | #log_autovacuum_min_duration = -1 # -1 disables, 0 logs all actions and
450 | # their durations, > 0 logs only
451 | # actions running at least this number
452 | # of milliseconds.
453 | #autovacuum_max_workers = 3 # max number of autovacuum subprocesses
454 | # (change requires restart)
455 | #autovacuum_naptime = 1min # time between autovacuum runs
456 | #autovacuum_vacuum_threshold = 50 # min number of row updates before
457 | # vacuum
458 | #autovacuum_analyze_threshold = 50 # min number of row updates before
459 | # analyze
460 | #autovacuum_vacuum_scale_factor = 0.2 # fraction of table size before vacuum
461 | #autovacuum_analyze_scale_factor = 0.1 # fraction of table size before analyze
462 | #autovacuum_freeze_max_age = 200000000 # maximum XID age before forced vacuum
463 | # (change requires restart)
464 | #autovacuum_multixact_freeze_max_age = 400000000 # maximum Multixact age
465 | # before forced vacuum
466 | # (change requires restart)
467 | #autovacuum_vacuum_cost_delay = 20ms # default vacuum cost delay for
468 | # autovacuum, in milliseconds;
469 | # -1 means use vacuum_cost_delay
470 | #autovacuum_vacuum_cost_limit = -1 # default vacuum cost limit for
471 | # autovacuum, -1 means use
472 | # vacuum_cost_limit
473 |
474 |
475 | #------------------------------------------------------------------------------
476 | # CLIENT CONNECTION DEFAULTS
477 | #------------------------------------------------------------------------------
478 |
479 | # - Statement Behavior -
480 |
481 | #search_path = '"$user",public' # schema names
482 | #default_tablespace = '' # a tablespace name, '' uses the default
483 | #temp_tablespaces = '' # a list of tablespace names, '' uses
484 | # only default tablespace
485 | #check_function_bodies = on
486 | #default_transaction_isolation = 'read committed'
487 | #default_transaction_read_only = off
488 | #default_transaction_deferrable = off
489 | #session_replication_role = 'origin'
490 | #statement_timeout = 0 # in milliseconds, 0 is disabled
491 | #lock_timeout = 0 # in milliseconds, 0 is disabled
492 | #vacuum_freeze_min_age = 50000000
493 | #vacuum_freeze_table_age = 150000000
494 | #vacuum_multixact_freeze_min_age = 5000000
495 | #vacuum_multixact_freeze_table_age = 150000000
496 | #bytea_output = 'hex' # hex, escape
497 | #xmlbinary = 'base64'
498 | #xmloption = 'content'
499 | #gin_fuzzy_search_limit = 0
500 |
501 | # - Locale and Formatting -
502 |
503 | datestyle = 'iso, mdy'
504 | #intervalstyle = 'postgres'
505 | timezone = 'UTC'
506 | #timezone_abbreviations = 'Default' # Select the set of available time zone
507 | # abbreviations. Currently, there are
508 | # Default
509 | # Australia (historical usage)
510 | # India
511 | # You can create your own file in
512 | # share/timezonesets/.
513 | #extra_float_digits = 0 # min -15, max 3
514 | #client_encoding = sql_ascii # actually, defaults to database
515 | # encoding
516 |
517 | # These settings are initialized by initdb, but they can be changed.
518 | lc_messages = 'en_US.UTF-8' # locale for system error message
519 | # strings
520 | lc_monetary = 'en_US.UTF-8' # locale for monetary formatting
521 | lc_numeric = 'en_US.UTF-8' # locale for number formatting
522 | lc_time = 'en_US.UTF-8' # locale for time formatting
523 |
524 | # default configuration for text search
525 | default_text_search_config = 'pg_catalog.english'
526 |
527 | # - Other Defaults -
528 |
529 | #dynamic_library_path = '$libdir'
530 | #local_preload_libraries = ''
531 |
532 |
533 | #------------------------------------------------------------------------------
534 | # LOCK MANAGEMENT
535 | #------------------------------------------------------------------------------
536 |
537 | #deadlock_timeout = 1s
538 | #max_locks_per_transaction = 64 # min 10
539 | # (change requires restart)
540 | #max_pred_locks_per_transaction = 64 # min 10
541 | # (change requires restart)
542 |
543 |
544 | #------------------------------------------------------------------------------
545 | # VERSION/PLATFORM COMPATIBILITY
546 | #------------------------------------------------------------------------------
547 |
548 | # - Previous PostgreSQL Versions -
549 |
550 | #array_nulls = on
551 | #backslash_quote = safe_encoding # on, off, or safe_encoding
552 | #default_with_oids = off
553 | #escape_string_warning = on
554 | #lo_compat_privileges = off
555 | #quote_all_identifiers = off
556 | #sql_inheritance = on
557 | #standard_conforming_strings = on
558 | #synchronize_seqscans = on
559 |
560 | # - Other Platforms and Clients -
561 |
562 | #transform_null_equals = off
563 |
564 |
565 | #------------------------------------------------------------------------------
566 | # ERROR HANDLING
567 | #------------------------------------------------------------------------------
568 |
569 | #exit_on_error = off # terminate session on any error?
570 | #restart_after_crash = on # reinitialize after backend crash?
571 |
572 |
573 | #------------------------------------------------------------------------------
574 | # CONFIG FILE INCLUDES
575 | #------------------------------------------------------------------------------
576 |
577 | # These options allow settings to be loaded from files other than the
578 | # default postgresql.conf.
579 |
580 | #include_dir = 'conf.d' # include files ending in '.conf' from
581 | # directory 'conf.d'
582 | #include_if_exists = 'exists.conf' # include file only if it exists
583 | #include = 'special.conf' # include file
584 |
585 |
586 | #------------------------------------------------------------------------------
587 | # CUSTOMIZED OPTIONS
588 | #------------------------------------------------------------------------------
589 |
590 | # Add settings for extensions here
591 |
--------------------------------------------------------------------------------