├── .gitmodules ├── README.md └── rabbitmq_cluster ├── metadata.rb └── recipes └── default.rb /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "elasticsearch"] 2 | path = elasticsearch 3 | url = https://github.com/foxycoder/cookbook-elasticsearch.git 4 | [submodule "kibana"] 5 | path = kibana 6 | url = https://github.com/foxycoder/chef-kibana.git 7 | [submodule "logstash"] 8 | path = logstash 9 | url = https://github.com/foxycoder/chef-logstash.git 10 | [submodule "redisio"] 11 | path = redisio 12 | url = https://github.com/brianbianco/redisio.git 13 | [submodule "rabbitmq"] 14 | path = rabbitmq 15 | url = https://github.com/opscode-cookbooks/rabbitmq.git 16 | [submodule "java"] 17 | path = java 18 | url = https://github.com/opscode-cookbooks/java.git 19 | [submodule "apt"] 20 | path = apt 21 | url = https://github.com/opscode-cookbooks/apt.git 22 | [submodule "erlang"] 23 | path = erlang 24 | url = https://github.com/opscode-cookbooks/erlang.git 25 | [submodule "mysql"] 26 | path = mysql 27 | url = https://github.com/opscode-cookbooks/mysql.git 28 | [submodule "nagios"] 29 | path = nagios 30 | url = https://github.com/opscode-cookbooks/nagios.git 31 | [submodule "nginx"] 32 | path = nginx 33 | url = https://github.com/opscode-cookbooks/nginx.git 34 | [submodule "ntp"] 35 | path = ntp 36 | url = https://github.com/opscode-cookbooks/ntp.git 37 | [submodule "postgresql"] 38 | path = postgresql 39 | url = https://github.com/opscode-cookbooks/postgresql.git 40 | [submodule "runit"] 41 | path = runit 42 | url = https://github.com/opscode-cookbooks/runit.git 43 | [submodule "ant"] 44 | path = ant 45 | url = https://github.com/opscode-cookbooks/ant.git 46 | [submodule "apache2"] 47 | path = apache2 48 | url = https://github.com/opscode-cookbooks/apache2.git 49 | [submodule "ark"] 50 | path = ark 51 | url = https://github.com/opscode-cookbooks/ark.git 52 | [submodule "aws"] 53 | path = aws 54 | url = https://github.com/opscode-cookbooks/aws.git 55 | [submodule "bluepill"] 56 | path = bluepill 57 | url = https://github.com/opscode-cookbooks/bluepill.git 58 | [submodule "build-essential"] 59 | path = build-essential 60 | url = https://github.com/opscode-cookbooks/build-essential.git 61 | [submodule "chef_handler"] 62 | path = chef_handler 63 | url = https://github.com/opscode-cookbooks/chef_handler.git 64 | [submodule "database"] 65 | path = database 66 | url = https://github.com/opscode-cookbooks/database.git 67 | [submodule "dmg"] 68 | path = dmg 69 | url = https://github.com/opscode-cookbooks/dmg.git 70 | [submodule "git"] 71 | path = git 72 | url = https://github.com/opscode-cookbooks/git.git 73 | [submodule "logrotate"] 74 | path = logrotate 75 | url = https://github.com/opscode-cookbooks/logrotate.git 76 | [submodule "memcached"] 77 | path = memcached 78 | url = https://github.com/opscode-cookbooks/memcached.git 79 | [submodule "ohai"] 80 | path = ohai 81 | url = https://github.com/opscode-cookbooks/ohai.git 82 | [submodule "openssl"] 83 | path = openssl 84 | url = https://github.com/opscode-cookbooks/openssl.git 85 | [submodule "perl"] 86 | path = perl 87 | url = https://github.com/opscode-cookbooks/perl.git 88 | [submodule "php"] 89 | path = php 90 | url = https://github.com/opscode-cookbooks/php.git 91 | [submodule "postfix"] 92 | path = postfix 93 | url = https://github.com/opscode-cookbooks/postfix.git 94 | [submodule "python"] 95 | path = python 96 | url = https://github.com/opscode-cookbooks/python.git 97 | [submodule "vim"] 98 | path = vim 99 | url = https://github.com/opscode-cookbooks/vim.git 100 | [submodule "windows"] 101 | path = windows 102 | url = https://github.com/opscode-cookbooks/windows.git 103 | [submodule "xfs"] 104 | path = xfs 105 | url = https://github.com/opscode-cookbooks/xfs.git 106 | [submodule "xml"] 107 | path = xml 108 | url = https://github.com/opscode-cookbooks/xml.git 109 | [submodule "yum"] 110 | path = yum 111 | url = https://github.com/opscode-cookbooks/yum.git 112 | [submodule "rsyslog"] 113 | path = rsyslog 114 | url = https://github.com/opscode-cookbooks/rsyslog.git 115 | [submodule "hostsfile"] 116 | path = hostsfile 117 | url = https://github.com/foxycoder/hostsfile.git 118 | [submodule "rbenv"] 119 | path = rbenv 120 | url = https://github.com/ddux/rbenv-cookbook.git 121 | [submodule "yumrepo"] 122 | path = yumrepo 123 | url = https://github.com/bryanwb/cookbook-yumrepo.git 124 | [submodule "htpasswd"] 125 | path = htpasswd 126 | url = https://github.com/Youscribe/htpasswd-cookbook.git 127 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This project is **no longer maintained**. 2 | 3 | If you're interested in taking it over, we'd love to hear from you. 4 | Email [developers@springest.com](mailto:developers@springest.com), or check out 5 | one of its maintained forks. 6 | 7 | ***** 8 | 9 | # A complete Logstash stack on [AWS OpsWorks](http://aws.amazon.com/opsworks/) 10 | 11 | __UPDATE__: I wrote a [blog post](http://devblog.springest.com/complete-logstash-stack-on-aws-opsworks-in-15-minutes) that will help you set this up. 12 | 13 | This is a bunch of cookbooks that will allow you to run a complete Logstash setup on a scalable 14 | [AWS OpsWorks](http://aws.amazon.com/opsworks/) stack. At [Springest](http://www.springest.com) 15 | we use it to ship from 250 to about 1k log entries per second, depending on the RPM on our 18 servers. 16 | 17 | ![kibana](http://springest-monosnap.s3-website-eu-west-1.amazonaws.com/2013_11_05_17_30_10_abrqo.jpg) 18 | __A screenshot of [Kibana3](http://demo.kibana.org/), included in the cookbooks.__ 19 | 20 | 21 | ## The stack's layers 22 | 23 | ![layers](http://springest-monosnap.s3-website-eu-west-1.amazonaws.com/2013_11_05_11_52_21_z5pt8.jpg) 24 | __A screenshot of the layers in [AWS OpsWorks](http://aws.amazon.com/opsworks/)__ 25 | 26 | - A RabbitMQ cluster layer (Ubuntu) – Agents wil ship their logs to this cluster and the Logstash server cluster uses it as an input source. 27 | - An ElasticSearch cluster layer (Amazon Linux) – All log messages are stored and indexed here. 28 | - A LogStash cluster layer (Ubuntu) – Takes the messages from the RabbitMQ fanout and puts them into ElasticSearch. 29 | - A Kibana server (Ubuntu) - An Angular.js interface on top of ElasticSearch to search, graph etc. 30 | 31 | ## Setting up your stack 32 | 33 | - Set `git://github.com/Springest/opsworks-logstash.git` as a repository URL for your custom cookbooks. 34 | - Use the following Chef custom JSON: 35 | 36 | ```json 37 | { 38 | "chef_environment": "production", 39 | "elasticsearch": { 40 | "cluster": { 41 | "name": "logstash" 42 | }, 43 | "basic_auth": { 44 | "user": "", 45 | "password": "" 46 | } 47 | }, 48 | "rabbitmq": { 49 | "cluster": true, 50 | "erlang_cookie": "" 51 | }, 52 | "rabbitmq_cluster": { 53 | "user": "", 54 | "password": "" 55 | }, 56 | "kibana": { 57 | "webserver": "nginx", 58 | "webserver_hostname": "", 59 | "web_user": "", 60 | "web_password": "", 61 | "es_port": "9200", 62 | "es_role": "elasticsearch", 63 | "es_server": "", 64 | "es_user": "", 65 | "es_password": "" 66 | }, 67 | "logstash": { 68 | "elasticsearch_cluster": "logstash", 69 | "server": { 70 | "install_rabbitmq": false, 71 | "enable_embedded_es": false, 72 | "elasticsearch_role": "elasticsearch", 73 | "inputs": [ 74 | { 75 | "rabbitmq": { 76 | "exchange": "rawlogs", 77 | "user": "", 78 | "password": "", 79 | "host": "", 80 | "type": "all" 81 | } 82 | } 83 | ] 84 | } 85 | } 86 | } 87 | ``` 88 | 89 | ### Configure Custom Chef recipes 90 | 91 | #### ElasticSearch 92 | 93 | ![elasticsearch recipes](http://springest-monosnap.s3-website-eu-west-1.amazonaws.com/2013_11_05_17_21_21_u1qo0.jpg) 94 | 95 | #### Kibana 96 | 97 | ![kibana recipes](http://springest-monosnap.s3-website-eu-west-1.amazonaws.com/2013_11_05_17_22_50_e2bpa.jpg) 98 | 99 | #### Logstash 100 | 101 | ![logstash recipes](http://springest-monosnap.s3-website-eu-west-1.amazonaws.com/2013_11_05_17_24_04_poohu.jpg) 102 | 103 | #### RabbitMQ 104 | 105 | ![rabbitmq recipes](http://springest-monosnap.s3-website-eu-west-1.amazonaws.com/2013_11_05_17_24_45_iojw0.jpg) 106 | 107 | ## Contribute 108 | 109 | - Fork the project 110 | - Create a feature branch 111 | - Fix the code 112 | - Create a pull request 113 | 114 | ## Thanks to 115 | 116 | All of these people have made our lives a little better by writing great 117 | software. 118 | 119 | The contributors at these projects: 120 | 121 | - [ElasticSearch](http://elasticsearch.com/) for their awesome search 122 | engine. 123 | - [Kibana](http://www.elasticsearch.org/overview/kibana/) for a 124 | beautiful graphing tool. 125 | - [LogStash](http://logstash.net/) for bringing our logs together. 126 | - [RabbitMQ](http://www.rabbitmq.com/) for a robust message broker. 127 | 128 | And the people who wrote these cookbooks: 129 | 130 | - [@lusis](https://github.com/lusis) for his work on the [Kibana cookbook](https://github.com/foxycoder/chef-kibana) and the [LogStash cookbook](https://github.com/foxycoder/chef-logstash) 131 | - [@karmi](https://github.com/foxycoder/cookbook-elasticsearch) for the original, and [@ehlertij](https://github.com/ehlertij), [@tkawa](https://github.com/tkawa), and [@interu](https://github.com/interu) for their efforts on the [ElasticSearch cookbook for OpsWorks](https://github.com/sportngin/cookbook-elasticsearch). 132 | - [@brianbianco](https://github.com/brianbianco) for his [Redis cookbook](https://github.com/brianbianco/redisio) 133 | - [@sethvargo](https://github.com/sethvargo) for the [hostsfile cookbook](https://github.com/customink-webops/hostsfile) 134 | - [@ddux](https://github.com/ddux) for the [rbenv cookbook](https://github.com/ddux/rbenv-cookbook) 135 | - [@bryanwb](https://github.com/bryanwb) for the [Yumrepo cookbook](https://github.com/bryanwb/cookbook-yumrepo) 136 | - [@guilhem](https://github.com/guilhem) for his efforts on the [Htpaswd cookbook](https://github.com/Youscribe/htpasswd-cookbook) 137 | 138 | And the guys at [Opscode](http://www.opscode.com/chef/) and their 139 | contributors for Chef and cookbooks like: 140 | 141 | - [RabbitMQ](https://github.com/opscode-cookbooks/rabbitmq) 142 | - [Java](https://github.com/opscode-cookbooks/java) 143 | - [Apt](https://github.com/opscode-cookbooks/apt) 144 | - [Erlang](https://github.com/opscode-cookbooks/erlang) 145 | - [Nginx](https://github.com/opscode-cookbooks/nginx) 146 | - [NTP](https://github.com/opscode-cookbooks/ntp) 147 | - [Runit](https://github.com/opscode-cookbooks/runit) 148 | - [Ant](https://github.com/opscode-cookbooks/ant) 149 | - [Apache2](https://github.com/opscode-cookbooks/apache2) 150 | - [Ark](https://github.com/opscode-cookbooks/ark) 151 | - [AWS](https://github.com/opscode-cookbooks/aws) 152 | - [Build-essential](https://github.com/opscode-cookbooks/build-essential) 153 | - [Chef handler](https://github.com/opscode-cookbooks/chef_handler) 154 | - [Git](https://github.com/opscode-cookbooks/git) 155 | - [Ohai](https://github.com/opscode-cookbooks/ohai) 156 | - [OpenSSL](https://github.com/opscode-cookbooks/openssl) 157 | - [Perl](https://github.com/opscode-cookbooks/perl) 158 | - [Python](https://github.com/opscode-cookbooks/python) 159 | - [Windows](https://github.com/opscode-cookbooks/windows) 160 | - [xfs](https://github.com/opscode-cookbooks/xfs) 161 | - [xml](https://github.com/opscode-cookbooks/xml) 162 | - [yum](https://github.com/opscode-cookbooks/yum) 163 | - [rsyslog](https://github.com/opscode-cookbooks/rsyslog) 164 | 165 | And last but not least: 166 | 167 | - [Amazon AWS](http://aws.amazon.com) 168 | 169 | -------------------------------------------------------------------------------- /rabbitmq_cluster/metadata.rb: -------------------------------------------------------------------------------- 1 | name "rabbitmq_cluster" 2 | 3 | %w{ubuntu debian linuxmint redhat centos scientific amazon fedora oracle smartos suse}.each do |os| 4 | supports os 5 | end 6 | 7 | %w{ rabbitmq hostsfile }.each do |ckbk| 8 | depends ckbk 9 | end 10 | -------------------------------------------------------------------------------- /rabbitmq_cluster/recipes/default.rb: -------------------------------------------------------------------------------- 1 | # Add all rabbitmq nodes to the hosts file with their short name. 2 | instances = node[:opsworks][:layers][:rabbitmq][:instances] 3 | 4 | instances.each do |name, attrs| 5 | hostsfile_entry attrs['private_ip'] do 6 | hostname name 7 | unique true 8 | end 9 | end 10 | 11 | rabbit_nodes = instances.map{ |name, attrs| "rabbit@#{name}" } 12 | node.set['rabbitmq']['cluster_disk_nodes'] = rabbit_nodes 13 | 14 | include_recipe 'rabbitmq' 15 | 16 | execute "chown -R rabbitmq:rabbitmq /var/lib/rabbitmq" 17 | 18 | rabbitmq_user "guest" do 19 | action :delete 20 | end 21 | 22 | rabbitmq_user node['rabbitmq_cluster']['user'] do 23 | password node['rabbitmq_cluster']['password'] 24 | action :add 25 | end 26 | 27 | rabbitmq_user node['rabbitmq_cluster']['user'] do 28 | vhost "/" 29 | permissions ".* .* .*" 30 | action :set_permissions 31 | end 32 | --------------------------------------------------------------------------------