├── .fixtures.yml ├── .gitattributes ├── .gitignore ├── .rspec ├── .ruby-version ├── .travis.yml ├── ELK.png ├── Gemfile ├── LICENSE ├── README.md ├── Rakefile ├── hieradata ├── manifests └── site.pp ├── site └── modules │ ├── profile │ ├── files │ │ └── logstash │ │ │ ├── conf.d │ │ │ ├── indexer.conf │ │ │ └── shipper.conf │ │ │ └── logstash.json │ └── manifests │ │ ├── base.pp │ │ ├── base │ │ ├── filebeat.pp │ │ ├── firewall.pp │ │ ├── firewall │ │ │ ├── post.pp │ │ │ └── pre.pp │ │ └── yum.pp │ │ ├── elasticsearch.pp │ │ ├── elasticsearch │ │ ├── coordinating_node.pp │ │ └── data_node.pp │ │ ├── jdk.pp │ │ ├── kibana.pp │ │ ├── logstash.pp │ │ ├── nginx.pp │ │ └── redis.pp │ └── role │ └── manifests │ └── elk_stack.pp ├── spec ├── acceptance │ ├── nodesets │ │ └── default.yml │ └── role_elk_stack_spec.rb ├── classes │ └── role_elk_stack_spec.rb ├── fixtures │ ├── Puppetfile.legacy │ ├── Puppetfile.v6 │ ├── facts.d │ │ └── facts.txt │ ├── hiera.yaml │ ├── hiera.yaml.beaker │ └── hieradata │ │ └── common.yaml ├── spec_helper.rb └── spec_helper_acceptance.rb └── yamllint.yml /.fixtures.yml: -------------------------------------------------------------------------------- 1 | fixtures: 2 | # repositories: 3 | # repositories is empty because we use librarian-puppet 4 | # and this is called from the Rakefile. 5 | symlinks: 6 | 'role': "#{source_dir}/site/modules/role" 7 | 'profile': "#{source_dir}/site/modules/profile" 8 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.rb linguist-language=Puppet 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .bin/ 2 | .gems/ 3 | Gemfile.lock 4 | .vagrant/ 5 | pkg/ 6 | log/ 7 | spec/fixtures/manifests 8 | spec/fixtures/modules 9 | spec/fixtures/.librarian 10 | spec/fixtures/.tmp 11 | spec/fixtures/Puppetfile.lock 12 | spec/fixtures/Puppetfile 13 | .bundle 14 | catalogs 15 | docs/ 16 | .yardoc/ 17 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | --format d 3 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | ruby-2.4.1 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | bundler_args: --without development system_tests 3 | before_install: rm Gemfile.lock || true 4 | matrix: 5 | include: 6 | - language: ruby 7 | rvm: 2.4.1 8 | env: PUPPET_GEM_VERSION="~> 4.0" STRICT_VARIABLES=yes LIBRARIAN_VERBOSE=true 9 | script: "bundle exec rake lint && bundle exec rake librarian_spec SPEC_OPTS='--format documentation'" 10 | - language: ruby 11 | rvm: 2.4.1 12 | env: PUPPET_GEM_VERSION="~> 5.0" STRICT_VARIABLES=yes LIBRARIAN_VERBOSE=true 13 | script: "bundle exec rake lint && bundle exec rake librarian_spec SPEC_OPTS='--format documentation'" 14 | - language: ruby 15 | rvm: 2.4.1 16 | env: PUPPET_GEM_VERSION="~> 6.0" STRICT_VARIABLES=yes LIBRARIAN_VERBOSE=true 17 | script: "bundle exec rake lint && bundle exec rake librarian_spec SPEC_OPTS='--format documentation'" 18 | - language: python 19 | python: 2.7 20 | script: "pip install yamllint && yamllint -c yamllint.yml spec/fixtures/hiera* spec/acceptance/nodesets/default.yml *.yml" 21 | notifications: 22 | email: 23 | recipients: 24 | - 'alexharv074@gmail.com' 25 | on_success: never 26 | on_failure: always 27 | -------------------------------------------------------------------------------- /ELK.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex-harvey-z3q/elk/5c9c26bf1bc503014af989863ee2c01c10c00f40/ELK.png -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | group :development do 4 | gem 'pry' 5 | gem 'pry-rescue' 6 | gem 'pry-stack_explorer' 7 | gem 'debug_inspector', '<= 0.0.2' 8 | gem 'hashdiff' 9 | gem 'awesome_print' 10 | end 11 | 12 | group :tests do 13 | gem 'puppetlabs_spec_helper' 14 | gem 'librarian-puppet' 15 | gem 'versionomy' 16 | end 17 | 18 | group :system_tests do 19 | gem 'beaker' 20 | gem 'beaker-rspec' 21 | gem 'beaker-puppet_install_helper' 22 | gem 'beaker-vagrant' 23 | gem 'beaker-pe' 24 | end 25 | 26 | gem 'facter' 27 | 28 | if puppetversion = ENV['PUPPET_GEM_VERSION'] 29 | gem 'puppet', puppetversion 30 | else 31 | gem 'puppet' 32 | end 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Alex Harvey 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # elk 2 | 3 | [![Build Status](https://img.shields.io/travis/alexharv074/elk.svg)](https://travis-ci.org/alexharv074/elk) 4 | 5 | #### Table of contents 6 | 7 | 1. [ELK 7 status](#elk-7-status) 8 | 2. [Motivation](#motivation) 9 | 3. [Latest version](#latest-version) 10 | 4. [Support](#support) 11 | 5. [Architecture](#architecture) 12 | 6. [About Librarian-puppet & g10k](#about-librarian-puppet--g10k) 13 | 7. [Setup](#setup) 14 | * [Quickly build an ELK cluster in Vagrant for development and testing](#quickly-build-an-elk-cluster-in-vagrant-for-development-and-testing) 15 | - [Dependencies](#dependencies) 16 | - [Squid Man set up](#squid-man-set-up) 17 | - [Clone this project](#clone-this-project) 18 | - [Install dependencies with Bundler](#install-dependencies-with-bundler) 19 | - [Build the Vagrant box](#build-the-vagrant-box) 20 | 8. [Reference](#reference) 21 | * [Note about examples](#note-about-examples) 22 | * [Roles](#roles) 23 | - [Public Classes](#public-classes) 24 | * [Profiles](#profiles) 25 | - [Public Classes](#public-classes-2) 26 | - [Private Classes](#private-classes) 27 | 9. [Testing](#testing) 28 | * [Dependencies](#dependencies-2) 29 | * [Squid Man set up](#squid-man-set-up-2) 30 | * [Run the tests](#run-the-tests) 31 | 10. [Donate](#donate) 32 | 33 | ## ELK 7 status 34 | 35 | Update 6 November 2019. At this time the upstream Elastic Puppet modules still do not support ELK 7. If you would like ELK 7 support here, consider upvoting [this](https://github.com/elastic/puppet-elasticsearch/issues/1037) issue. I will add ELK 7 support here as soon as I am able to. 36 | 37 | ## Motivation 38 | 39 | This project is intended to solve a variety of problems: 40 | 41 | * A convenient fully-automated moderately-opinionated out-of-the-box latest-version highly-available ELK solution in Puppet. 42 | * An ELK cluster that spins up conveniently on a single node for development and testing in ELK. 43 | * A demonstration of a complex Puppet Control Repo, Roles & Profiles design with automated testing in Rspec-puppet and Beaker. 44 | 45 | ## Latest version 46 | 47 | Last tested against: 48 | 49 | - ELK 6.8.0 50 | - Elasticsearch Curator 5.7.6 51 | - java-1.8.0-openjdk-1.8.0.212.b04-0.el7_6.x86_64 52 | - Redis 3.2.12 53 | 54 | Note that ELK 7 has been released and will be supported here soon! 55 | 56 | ## Support 57 | 58 | At the time of writing my intention is to support this solution and keep it up-to-date with the latest upstream ELK components and the ELK Puppet modules that it uses. For this reason, the components pulled are always latest-everything (see `Puppetfile`). 59 | 60 | If you have any problems with it or wish to request features feel free to raise an issue and I may (or may not) fix or implement. Pull requests also welcome. 61 | 62 | Also, consider [donating](#donate). 63 | 64 | ## Architecture 65 | 66 | The basic architecture is shown in the following figure: 67 | 68 | ![Fig 1](./ELK.png) 69 | 70 | ## About Librarian-puppet & g10k 71 | 72 | The project is configured to use [librarian-puppet](https://github.com/voxpupuli/librarian-puppet) or [g10k](https://github.com/xorpaul/g10k) to check out the shared modules that this project depends upon. 73 | 74 | I prefer Librarian-puppet over r10k for its ability to install dependencies, and I prefer g10k over Librarian-puppet because it has been rewritten in Go and it's just very fast. Librarian-puppet must remain in order to run from within Travis CI. See also the Rakefile. 75 | 76 | ## Setup 77 | 78 | ### Quickly build an ELK cluster in Vagrant for development and testing 79 | 80 | If your reason for being here is that you just want to play with an ELK cluster, do this: 81 | 82 | #### Dependencies 83 | 84 | Make sure you have: 85 | 86 | * VirtualBox (download from [here](https://www.virtualbox.org/wiki/Downloads)) 87 | * Vagrant (download from [here](https://www.vagrantup.com/downloads.html)) 88 | * Ruby Gems (should be already installed on a Mac) 89 | * RVM (Ruby version 2.4.1: `rvm install ruby-2.4.1`) 90 | * bundler (already installed on newer Macs, and `gem install bundler` otherwise.) 91 | * (optional) Squid Man (`brew cask install squidman`) 92 | 93 | #### Squid Man set up 94 | 95 | Squid Man can be optionally used to speed up the Vagrant build by caching a lot of the repetitive downloads of RPMs etc. I assume you are using Squid Man. 96 | 97 | Configure Squid Man to listen on port 3128 and cache all. 98 | 99 | #### Clone this project 100 | 101 | Clone the project from Github: 102 | 103 | ~~~ text 104 | git clone https://github.com/alexharv074/elk.git 105 | ~~~ 106 | 107 | #### Install dependencies with Bundler 108 | 109 | Cd into the elk directory then: 110 | 111 | ~~~ text 112 | bundle install 113 | ~~~ 114 | 115 | #### Build the Vagrant box 116 | 117 | Finally, you can build the ELK cluster using these commands: 118 | 119 | ~~~ text 120 | export BEAKER_destroy=no 121 | export BEAKER_PACKAGE_PROXY=http://:3128/ 122 | bundle exec rake best_spec 123 | bundle exec rspec spec/acceptance 124 | ~~~ 125 | 126 | It should then take about 10 minutes or so to build. 127 | 128 | Finally, you can visit the ELK at [http://localhost:5601](http://localhost:5601). 129 | 130 | ## Reference 131 | 132 | - [**Roles**](#roles) 133 | - [**Public Classes**](#public-classes) 134 | - [Class: role::elk_stack](#class-roleelk_stack) 135 | - [**Profiles**](#profiles) 136 | - [**Public Classes**](#public-classes) 137 | - [Class: profile::base](#class-profilebase) 138 | - [Class: profile::elasticsearch::coordinating_node](#class-profileelasticsearchcoordinating_node) 139 | - [Class: profile::elasticsearch::data_node](#class-profileelasticsearchdata_node) 140 | - [Class: profile::kibana](#class-profilekibana) 141 | - [Class: profile::logstash](#class-profilelogstash) 142 | - [Class: profile::nginx](#class-profilenginx) 143 | - [Class: profile::redis](#class-profileredis) 144 | - [**Private Classes**](#private-classes) 145 | - [Class: profile::jdk](#class-profilejdk) 146 | - [Class: profile::base::filebeat](#class-profilebasefilebeat) 147 | - [Class: profile::base::firewall](#class-profilebasefirewall) 148 | - [Class: profile::base::yum](#class-profilebaseyum) 149 | - [Class: profile::elasticsearch](#class-profileelasticsearch) 150 | 151 | ### Note about examples 152 | 153 | In all of the examples below, I assume that the user is using Hiera to pass data to the profile classes and only therefore show YAML versions of the config. 154 | 155 | ### Roles 156 | 157 | #### Public Classes 158 | 159 | ##### Class: `role::elk_stack` 160 | 161 | Installs an ELK cluster with a base profile that includes the Filebeat agent and a Redis cache frontended by an Nginx reverse proxy. 162 | 163 | The purpose of this role is: 164 | 165 | 1. To do integration testing in Beaker of all the ELK components. 166 | 2. To quickly spin up an ELK cluster for development and testing. 167 | 168 | ### Profiles 169 | 170 | #### Public Classes 171 | 172 | ##### Class: `profile::base` 173 | 174 | The base profile that configures adds the Elastic Filebeat agent, and also an iptables firewall, yum repos both for the ELK and Linux, NTP and other services. 175 | 176 | The following parameters are expected: 177 | 178 | - `firewall_multis`. A Hash of firewall blocks passed to the firewall_multis module, which creates the iptables rules. For example, to allow ports 23 and 80 for SSH & Kibana: 179 | 180 | ~~~ yaml 181 | profile::base::firewall_multis: 182 | '00099 accept tcp ports for SSH and Kibana': 183 | dport: [22, 80] 184 | action: accept 185 | proto: tcp 186 | source: 187 | - 0.0.0.0/0 188 | ~~~ 189 | 190 | - `tools`. A Hash of RPMs like `vim` etc for debugging. 191 | 192 | ##### Class: `profile::elasticsearch::coordinating_node` 193 | 194 | Creates a coordinating-only node. Elasticsearch Coordinating-only nodes are essentially smart load balancers and Kibana typically uses one of these to distribute requests to cluster. The class sets `node.master => false`, `node.data => false` and `node.ingest => false` and expects other config to be provided by the user. Also included is the private `profile::elasticsearch` class (see below), which configures the elasticsearch user and group and includes the `profile::jdk`. 195 | 196 | The following parameters are expected: 197 | 198 | - `firewall_multis`. See above. 199 | - `config`. A Hash of config options that are used to populate the `elasticsearch.yml` file. For example, to set the `cluster.name` and `node.name`: 200 | 201 | ~~~ yaml 202 | profile::elasticsearch::coordinating_node::config: 203 | 'cluster.name': es01 204 | 'node.name': "es01_coordinating_%{::hostname}" 205 | ~~~ 206 | 207 | - `init_defaults`. A Hash of config options are used to populate the `/etc/sysconfig/defaults` file. For example: 208 | 209 | ~~~ yaml 210 | profile::elasticsearch::coordinating_node::init_defaults: 211 | JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.65-0.b17.el6_7.x86_64 212 | ~~~ 213 | 214 | For more information about configuring a coordinating-only node for use with Kibana, see [here](https://www.elastic.co/guide/en/kibana/current/production.html). 215 | 216 | ##### Class: `profile::elasticsearch::data_node` 217 | 218 | Creates a data node (or master-eligible data node). The class sets `node.data => true` and expects other config to be provided by the user. As part of this, there is expected to be a separate physical volume provided via the `espv` fact for the Elasticsearch datadir, and the class creates the LVM and filesystem via the `puppetlabs/lvm` module. Also included is the private `profile::elasticsearch` class (see below), which configures the elasticsearch user and group and includes the `profile::jdk`. 219 | 220 | Optional features include: 221 | - Installation of the [curator](https://github.com/elastic/curator) tool, for managing indices (e.g. maintaining disk space). 222 | - Install a list of Elasticsearch templates 223 | - Install a list of Elasticsearch plugins 224 | - Set recommended performance tuning options for the cluster 225 | 226 | The following parameters are expected: 227 | 228 | - `firewall_multis`. See above. 229 | - `config`. A Hash of config options that are used to populate the `elasticsearch.yml` file. For example, to set the `cluster.name` and `node.name`, and make this data node master-eligible: 230 | 231 | ~~~ yaml 232 | profile::elasticsearch::data_node::config: 233 | 'cluster.name': es01 234 | 'node.name': "es01_%{::hostname}" 235 | 'node.master': true 236 | ~~~ 237 | 238 | - `init_defaults`. A Hash of config options are used to populate the `/etc/sysconfig/defaults` file. For example: 239 | 240 | ~~~ yaml 241 | profile::elasticsearch::coordinating_node::init_defaults: 242 | JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.65-0.b17.el6_7.x86_64 243 | ~~~ 244 | 245 | - `es_templates`. An optional Hash of [Elasticsearch Index Templates](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html) which are passed to the `elasticsearch::template` type from the Elasticsearch Puppet module. The templates are expected to be JSON files in your Puppet code base. Example: 246 | 247 | ~~~ yaml 248 | profile::elasticsearch::data_node::es_templates: 249 | logstash: 250 | source: puppet:///modules/profile/logstash/logstash.json 251 | ~~~ 252 | 253 | - `es_plugins`. An optional Hash of Elasticsearch plugins which are passed to the `elasticsearch::plugin` type from the Elasticsearch Puppet module. For example: 254 | 255 | ~~~ yaml 256 | profile::elasticsearch::data_node::es_plugins: 257 | 'x-pack': 258 | instances: es01 259 | ~~~ 260 | 261 | - `curator_jobs`. A Hash of jobs passed to the built-in Puppet `cron` type. Example: 262 | 263 | ~~~ yaml 264 | profile::elasticsearch::data_node::curator_jobs: 265 | curator_delete: 266 | command: '/usr/bin/curator --master-only --logfile /var/log/elasticsearch/curator.log delete --older-than 20' 267 | hour: 2 268 | minute: 12 269 | curator_close: 270 | command: '/usr/bin/curator --master-only --logfile /var/log/elasticsearch/curator.log close --older-than 10' 271 | hour: 2 272 | minute: 24 273 | curator_bloom: 274 | command: '/usr/bin/curator --master-only --logfile /var/log/elasticsearch/curator.log bloom --older-than 2' 275 | hour: 2 276 | minute: 32 277 | ~~~ 278 | 279 | - `datadir`. The full path to the mount point of the data dir. 280 | 281 | - `jvm_options`. An Array of JVM options. Example: 282 | 283 | ~~~ yaml 284 | profile::elasticsearch::data_node::jvm_options: 285 | - -Xms1g 286 | - -Xmx1g 287 | ~~~ 288 | 289 | - `config`. A Hash of config options that are used to populate the `elasticsearch.yml` file. For example, to set the `cluster.name` and `node.name`: 290 | 291 | ~~~ yaml 292 | profile::elasticsearch::coordinating_node::config: 293 | 'cluster.name': es01 294 | 'node.name': "es01_coordinating_%{::hostname}" 295 | ~~~ 296 | 297 | - `init_defaults`. A Hash of config options are used to populate the `/etc/sysconfig/defaults` file. For example: 298 | 299 | ~~~ yaml 300 | profile::elasticsearch::coordinating_node::init_defaults: 301 | JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.65-0.b17.el6_7.x86_64 302 | ~~~ 303 | 304 | For more information about configuring a coordinating-only node for use with Kibana, see [here](https://www.elastic.co/guide/en/kibana/current/production.html). 305 | 306 | ##### Class: `profile::elasticsearch::data_node` 307 | 308 | Creates a data node (or master-eligible data node). The class sets `node.data => true` and expects other config to be provided by the user. As part of this, there is expected to be a separate physical volume provided via the `espv` fact for the Elasticsearch datadir, and the class creates the LVM and filesystem via the `puppetlabs/lvm` module. 309 | 310 | Optional features include: 311 | - Installation of the [curator](https://github.com/elastic/curator) tool, for managing indices (e.g. maintaining disk space). 312 | - Install a list of Elasticsearch templates 313 | - Install a list of Elasticsearch plugins 314 | - Set recommended performance tuning options for the cluster 315 | 316 | The parameters that are expected: 317 | 318 | - `firewall_multis`. See above. 319 | - `config`. A Hash of config options that are used to populate the `elasticsearch.yml` file. For example, to set the `cluster.name` and `node.name`, and make this data node master-eligible: 320 | 321 | ~~~ yaml 322 | profile::elasticsearch::data_node::config: 323 | 'cluster.name': es01 324 | 'node.name': "es01_%{::hostname}" 325 | 'node.master': true 326 | ~~~ 327 | 328 | - `init_defaults`. A Hash of config options are used to populate the `/etc/sysconfig/defaults` file. For example: 329 | 330 | ~~~ yaml 331 | profile::elasticsearch::coordinating_node::init_defaults: 332 | JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.65-0.b17.el6_7.x86_64 333 | ~~~ 334 | 335 | - `es_templates`. An optional Hash of [Elasticsearch Index Templates](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html). The templates are expected to be JSON files in your Puppet code base. Example: 336 | 337 | ~~~ yaml 338 | profile::elasticsearch::data_node::es_templates: 339 | logstash: 340 | source: puppet:///modules/profile/logstash/logstash.json 341 | ~~~ 342 | 343 | ##### Class: `profile::kibana` 344 | 345 | The Kibana profile is just a thin wrapper around the Elastic Kibana module. The module accepts just three parameters: 346 | 347 | - `firewall_multis`. See above. 348 | - `uid` and `gid`. The UID and GID of the Kibana user and group. We manage it explicitly to avoid the indeterminate assignment that occurs when created by the RPMs. Integers >= 30000 are expected here. 349 | 350 | Config can be passed to the Kibana module directly, for example: 351 | 352 | ~~~ yaml 353 | kibana::config: 354 | 'server.host': 0.0.0.0 355 | 'server.port': 5601 356 | 'elasticsearch.url': http://0.0.0.0:9201 357 | ~~~ 358 | 359 | ##### Class: `profile::nginx` 360 | 361 | The Nginx profile can be used to create a simple Reverse Proxy to place in front of Kibana so that users can access Kibana on port 80 in their browsers, rather than Kibana port 5601. 362 | 363 | Expected parameters: 364 | 365 | - `firewall_multis`. See above. 366 | - `uid` and `gid`. The UID and GID of the Logstash user and group. We manage it explicitly to avoid the indeterminate assignment that occurs when created by the RPMs. Integers >= 30000 are expected here. 367 | - `backend_host` and `backend_port`. The host and port of the Kibana service. 368 | 369 | ##### Class: `profile::logstash` 370 | 371 | The Logstash profile is used to configure Logstash instances such as Logstash Shipper or Indexer nodes or both or other kinds of Logstash nodes. It configures Logstash itself via the Elastic Puppet Logstash module, declares `logstash::configfile` and `logstash::patternfiles` from that module, creates the Logstash user and group, and installs the JDK package. 372 | 373 | Logstash Pipelines can be declared by passing data directly to the Logstash module. Note that since ELK 6.x, the [Multiple Pipelines](https://www.elastic.co/guide/en/logstash/current/multiple-pipelines.html) feature has obviated the need in earlier versions of ELK for multiple Logstash instances, e.g. separate Shipper and Indexer nodes. To run Shipper and Indexer Pipelines in the same instance, for example: 374 | 375 | ~~~ yaml 376 | logstash::pipelines: 377 | - 'pipeline.id': shipper 378 | 'path.config': /etc/logstash/conf.d/shipper.conf 379 | - 'pipeline.id': indexer 380 | 'path.config': /etc/logstash/conf.d/indexer.conf 381 | ~~~ 382 | 383 | The following parameters are expected by the Logstash profile: 384 | 385 | - `firewall_multis`. See above. 386 | - `uid` and `gid`. The UID and GID of the Logstash user and group. We manage it explicitly to avoid the indeterminate assignment that occurs when created by the RPMs. Integers >= 30000 are expected here. 387 | - `configfiles`. A Hash of `logstash::configfile`. For example: 388 | 389 | ~~~ yaml 390 | profile::logstash::configfiles: 391 | 'shipper.conf': 392 | source: puppet:///modules/profile/logstash/conf.d/shipper.conf 393 | path: /etc/logstash/conf.d/shipper.conf 394 | 'indexer.conf': 395 | source: puppet:///modules/profile/logstash/conf.d/indexer.conf 396 | path: /etc/logstash/conf.d/indexer.conf 397 | ~~~ 398 | 399 | - `patternfiles`. A Hash of `logstash::patternfile`. 400 | 401 | #### Private Classes 402 | 403 | ##### Class: `profile::jdk` 404 | 405 | This profile just installs the Open JDK package. It expects one parameter, the package name. For example: 406 | 407 | ~~~ yaml 408 | profile::jdk::package: java-1.8.0-openjdk 409 | ~~~ 410 | 411 | ##### Class: `profile::base::filebeat` 412 | 413 | Includes the (official) Pcfens/Filebeat class and also declares a `filebeat::prospector`. There is one expected parameter: 414 | 415 | - `paths`. An Array of file paths to send logs for. Example: 416 | 417 | ~~~ yaml 418 | profile::base::filebeat::paths: 419 | - /var/log/syslog 420 | - /var/log/secure 421 | ~~~ 422 | 423 | ##### Class: `profile::base::firewall`, `profile::base::firewall::pre`, `profile::base::firewall::post` 424 | 425 | These classes configure the Puppetlabs Firewall module in the standard way according to documentation found at that project. 426 | 427 | ##### Class: `profile::base::yum` 428 | 429 | All Yum repos - including for the Elasticsearch components - are managed from here. I designed it this way because I wanted to use the Puppet "purge" feature to ensure that I have control over Yum repos left in here by the Linux distro. 430 | 431 | - `repos`. A Hash of Yum Repos. Example: 432 | 433 | ~~~ yaml 434 | profile::base::yum::repos: 435 | 'ELK-6.x': 436 | descr: 'elasticsearch repo' 437 | baseurl: https://artifacts.elastic.co/packages/6.x/yum 438 | gpgcheck: 1 439 | gpgkey: https://artifacts.elastic.co/GPG-KEY-elasticsearch 440 | enabled: 1 441 | ~~~ 442 | 443 | ##### Class: `profile::elasticsearch` 444 | 445 | This class configures the Elasticsearch user and group and also includes the Elasticsearch Puppet module itself. The only expected parameters are the `uid` and `gid` (see above for these). 446 | 447 | ## Testing 448 | 449 | ### Dependencies 450 | 451 | Make sure you have: 452 | 453 | * VirtualBox (or equivalent) 454 | * Vagrant 455 | * Ruby Gems 456 | * bundler 457 | * RVM 458 | * Squid Man 459 | 460 | ### Squid Man set up 461 | 462 | Configure Squid Man to listen on port 3128 and cache all. 463 | 464 | ### Run the tests 465 | 466 | Install the necessary gems: 467 | 468 | ~~~ text 469 | bundle install 470 | ~~~ 471 | 472 | To run the unit tests from the root of the source code: 473 | 474 | ~~~ text 475 | bundle exec rake best_spec 476 | ~~~ 477 | 478 | To run the acceptance tests: 479 | 480 | ~~~ text 481 | ipaddr=$(ifconfig en0 | awk '$1 == "inet" {print $2}') 482 | export BEAKER_PACKAGE_PROXY=http://${ipaddr}:3128/ 483 | ~~~ 484 | 485 | Puppet 6.4.2: 486 | 487 | ~~~ text 488 | export BEAKER_PUPPET_COLLECTION=puppet6 489 | export BEAKER_PUPPET_INSTALL_VERSION=6.4.2 490 | ~~~ 491 | 492 | Puppet 5.5.14: 493 | 494 | ~~~ text 495 | export BEAKER_PUPPET_COLLECTION=puppet5 496 | export BEAKER_PUPPET_INSTALL_VERSION=5.5.14 497 | ~~~ 498 | 499 | Puppet 4.10.11: 500 | 501 | ~~~ text 502 | export BEAKER_PUPPET_COLLECTION=pc1 503 | export BEAKER_PUPPET_INSTALL_VERSION=1.10.12 504 | ~~~ 505 | 506 | Then: 507 | 508 | ~~~ text 509 | bundle exec rspec spec/acceptance 510 | ~~~ 511 | 512 | Tested using Puppet 4.10.11, 5.5.14, 6.4.2 and Ruby 2.4.1. 513 | 514 | ## Donate 515 | 516 | If you find that this code saved your project some significant time, consider donating: 517 | 518 | [![paypal](https://www.paypalobjects.com/en_AU/i/btn/btn_donateCC_LG.gif)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=6849RBYT6VYBQ) 519 | 520 | Also, please add a star if you find it useful! 521 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # Environment variables: 2 | # 3 | # NO_G10K, NO_LIBRARIAN, NO_CHECKOUT - all have the same effect of preventing 4 | # librarian or g10k from installing the modules from the Forge. 5 | # 6 | # LIBRARIAN_VERBOSE - pass --verbose to librarian-puppet. 7 | # 8 | # FORCE_LIBRARIAN - use librarian-puppet even if g10k is available. 9 | # 10 | require 'puppetlabs_spec_helper/rake_tasks' 11 | require 'puppet-lint/tasks/puppet-lint' 12 | require 'versionomy' 13 | require 'puppet/version' 14 | 15 | Rake::Task[:lint].clear 16 | PuppetLint.configuration.relative = true 17 | PuppetLint::RakeTask.new(:lint) do |config| 18 | config.fail_on_warnings = true 19 | config.disable_checks = [ 20 | 'class_inherits_from_params_class', 21 | 'class_parameter_defaults', 22 | 'documentation', 23 | 'single_quote_string_with_variables', 24 | 'variables_not_enclosed', 25 | 'arrow_alignment', 26 | 'arrow_on_right_operand_line', 27 | ] 28 | config.ignore_paths = ["tests/**/*.pp", "vendor/**/*.pp","examples/**/*.pp", "spec/**/*.pp", "pkg/**/*.pp"] 29 | end 30 | 31 | def run(command) 32 | puts "Running #{command}" 33 | begin 34 | system(command) 35 | rescue => e 36 | raise "#{command} failed: #{e}" 37 | end 38 | end 39 | 40 | def locate_g10k 41 | ENV['PATH'].split(':').each do |p| 42 | g10k = File.join(p, 'g10k') 43 | if File.exists?(g10k) 44 | return g10k 45 | end 46 | end 47 | nil 48 | end 49 | 50 | def no_checkout 51 | ENV['NO_G10K'] or ENV['NO_LIBRARIAN'] or ENV['NO_CHECKOUT'] 52 | end 53 | 54 | desc 'Generate Puppetfile' 55 | task :generate_puppetfile do 56 | puppetfile = 'spec/fixtures/Puppetfile.v6' 57 | if Versionomy.parse(Puppet.version) < Versionomy.parse('6.0.0') 58 | puppetfile = 'spec/fixtures/Puppetfile.legacy' 59 | end 60 | FileUtils::cp puppetfile, 'spec/fixtures/Puppetfile' 61 | end 62 | 63 | desc 'Install modules with g10k' 64 | task :g10k_spec_prep do 65 | raise "g10k_spec_prep called but set NO_CHECKOUT, NO_LIBRARIAN or NO_G10K set" if no_checkout 66 | 67 | if g10k = locate_g10k 68 | if RUBY_PLATFORM =~ /darwin/ 69 | command = "#{g10k} -puppetfile" 70 | else 71 | command = "#{g10k} -puppetfile -info" 72 | end 73 | end 74 | 75 | run("cd spec/fixtures && #{command}") 76 | end 77 | 78 | desc "Run spec tests using g10k to checkout modules" 79 | task :g10k_spec do 80 | Rake::Task[:generate_puppetfile].invoke 81 | Rake::Task[:g10k_spec_prep].invoke 82 | Rake::Task[:spec].invoke 83 | end 84 | 85 | verbose = ENV['LIBRARIAN_VERBOSE'] ? '--verbose' : '' 86 | 87 | desc 'Install puppet modules with librarian-puppet' 88 | task :librarian_spec_prep do 89 | raise "librarian_spec_prep called but set NO_CHECKOUT, NO_LIBRARIAN or NO_G10K set" if no_checkout 90 | run("cd spec/fixtures && bundle exec librarian-puppet install #{verbose}") 91 | end 92 | 93 | desc 'Update puppet modules with librarian-puppet' 94 | task :librarian_update do 95 | system('rm -f spec/fixtures/Puppetfile.lock') 96 | run("cd spec/fixtures && bundle exec librarian-puppet update #{verbose}") 97 | end 98 | 99 | desc "Run spec tests using librarian-puppet to checkout modules" 100 | task :librarian_spec do 101 | Rake::Task[:generate_puppetfile].invoke 102 | Rake::Task[:librarian_spec_prep].invoke 103 | Rake::Task[:spec].invoke 104 | end 105 | 106 | desc "Update puppet modules with g10k preferred or librarian-puppet" 107 | if locate_g10k and !ENV['FORCE_LIBRARIAN'] 108 | task :best_spec_prep => :g10k_spec_prep 109 | else 110 | task :best_spec_prep => :librarian_spec_prep 111 | end 112 | 113 | desc "Run spec tests using fastest tool to checkout modules" 114 | task :best_spec do 115 | Rake::Task[:generate_puppetfile].invoke 116 | Rake::Task[:best_spec_prep].invoke 117 | Rake::Task[:spec].invoke 118 | end 119 | 120 | desc "Clean Puppet 6-only modules" 121 | task :clean_puppet_6 do 122 | modules = ['yum','mount','cron','augeas'] 123 | modules.map { |x| FileUtils::rm_rf "spec/fixtures/modules/#{x}*" } 124 | end 125 | 126 | desc "Clean all modules and lock file" 127 | task :clean_all do 128 | Dir.glob("spec/fixtures/modules/*").each { |x| FileUtils::rm_rf x } 129 | FileUtils::rm 'spec/fixtures/Puppetfile.lock' 130 | end 131 | -------------------------------------------------------------------------------- /hieradata: -------------------------------------------------------------------------------- 1 | spec/fixtures/hieradata -------------------------------------------------------------------------------- /manifests/site.pp: -------------------------------------------------------------------------------- 1 | stage { 'pre': before => Stage['main'] } 2 | 3 | Firewall { 4 | require => Class['profile::base::firewall::pre'], 5 | before => Class['profile::base::firewall::post'], 6 | } 7 | -------------------------------------------------------------------------------- /site/modules/profile/files/logstash/conf.d/indexer.conf: -------------------------------------------------------------------------------- 1 | ### MANAGED BY PUPPET ### 2 | input { 3 | redis { 4 | data_type => "list" 5 | key => "logstash" 6 | codec => "json" 7 | # codec => multiline { 8 | # pattern => "(^\s+at .+)" 9 | # what => "previous" 10 | # } 11 | } 12 | } 13 | 14 | output { 15 | elasticsearch { 16 | manage_template => false 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /site/modules/profile/files/logstash/conf.d/shipper.conf: -------------------------------------------------------------------------------- 1 | ### MANAGED BY PUPPET ### 2 | input { 3 | beats { 4 | port => "5044" 5 | } 6 | } 7 | 8 | output { 9 | redis { 10 | data_type => "list" 11 | key => "logstash" 12 | } 13 | } 14 | 15 | -------------------------------------------------------------------------------- /site/modules/profile/files/logstash/logstash.json: -------------------------------------------------------------------------------- 1 | { 2 | "index_patterns": ["logstash-*"], 3 | "settings": { 4 | "index.refresh_interval": "5s", 5 | "index.number_of_replicas": "0" 6 | }, 7 | "mappings": { 8 | "_default_": { 9 | "dynamic_templates": [ 10 | { 11 | "message_field": { 12 | "path_match": "message", 13 | "match_mapping_type": "string", 14 | "mapping": { 15 | "type": "text", 16 | "norms": "false" 17 | } 18 | } 19 | }, 20 | { 21 | "string_fields": { 22 | "match": "*", 23 | "match_mapping_type": "string", 24 | "mapping": { 25 | "type": "text", "norms": "false", 26 | "fields": { 27 | "keyword": { "type": "keyword", "ignore_above": 256 } 28 | } 29 | } 30 | } 31 | } 32 | ], 33 | "properties": { 34 | "@timestamp": { "type": "date"}, 35 | "@version": { "type": "keyword"}, 36 | "geoip": { 37 | "dynamic": "true", 38 | "properties": { 39 | "ip": { "type": "ip" }, 40 | "location": { "type": "geo_point" }, 41 | "latitude": { "type": "half_float" }, 42 | "longitude": { "type": "half_float" } 43 | } 44 | } 45 | } 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /site/modules/profile/manifests/base.pp: -------------------------------------------------------------------------------- 1 | class profile::base ( 2 | Hash[String, Hash] $firewall_multis, 3 | Hash[String, Hash] $tools, 4 | ) { 5 | create_resources(firewall_multi, $firewall_multis) 6 | create_resources(package, $tools) 7 | 8 | include profile::base::firewall 9 | include profile::base::yum 10 | include ntp 11 | include profile::base::filebeat 12 | # include profile::base::logrotate 13 | } 14 | -------------------------------------------------------------------------------- /site/modules/profile/manifests/base/filebeat.pp: -------------------------------------------------------------------------------- 1 | class profile::base::filebeat ( 2 | Array[Stdlib::Absolutepath] $paths, 3 | ) { 4 | include filebeat 5 | 6 | filebeat::input { 'syslogs': 7 | paths => $paths, 8 | doc_type => 'syslog-beat', 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /site/modules/profile/manifests/base/firewall.pp: -------------------------------------------------------------------------------- 1 | class profile::base::firewall { 2 | resources { 'firewall': 3 | purge => true, 4 | } 5 | 6 | include profile::base::firewall::pre 7 | include profile::base::firewall::post 8 | 9 | include firewall 10 | } 11 | -------------------------------------------------------------------------------- /site/modules/profile/manifests/base/firewall/post.pp: -------------------------------------------------------------------------------- 1 | # Rules which are applied to all nodes AFTER any others. 2 | class profile::base::firewall::post () { 3 | firewall { '999 drop all': 4 | proto => 'all', 5 | action => 'drop', 6 | before => undef, 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /site/modules/profile/manifests/base/firewall/pre.pp: -------------------------------------------------------------------------------- 1 | # Rules which are applied to all nodes before any others. 2 | class profile::base::firewall::pre { 3 | Firewall { 4 | require => undef, 5 | } 6 | 7 | firewall { '000 accept all icmp': 8 | proto => 'icmp', 9 | action => 'accept', 10 | } 11 | -> 12 | firewall { '001 accept all to lo interface': 13 | proto => 'all', 14 | iniface => 'lo', 15 | action => 'accept', 16 | } 17 | -> 18 | firewall { '002 reject local traffic not on loopback interface': 19 | iniface => '! lo', 20 | proto => 'all', 21 | destination => '127.0.0.1/8', 22 | action => 'reject', 23 | } 24 | -> 25 | firewall { '003 accept related established rules': 26 | proto => 'all', 27 | state => ['RELATED', 'ESTABLISHED'], 28 | action => 'accept', 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /site/modules/profile/manifests/base/yum.pp: -------------------------------------------------------------------------------- 1 | class profile::base::yum ( 2 | Hash[String, Hash[String, Variant[Integer, String]]] $repos, 3 | ) { 4 | create_resources(yumrepo, $repos) 5 | 6 | # Since we must purge the file resources in 7 | # /etc/yum.repos.d/, we must also declare the 8 | # associated files to prevent them also 9 | # being purged. 10 | 11 | keys($repos).each |String $yumrepo| { 12 | file { "/etc/yum.repos.d/${yumrepo}.repo": } 13 | -> 14 | Yumrepo[$yumrepo] 15 | } 16 | file { '/etc/yum.repos.d/': 17 | ensure => directory, 18 | recurse => true, 19 | purge => true, 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /site/modules/profile/manifests/elasticsearch.pp: -------------------------------------------------------------------------------- 1 | class profile::elasticsearch ( 2 | Integer[30000] $uid, 3 | Integer[30000] $gid, 4 | ) { 5 | include profile::jdk 6 | 7 | # Manage the user and group to prevent random UID and 8 | # GID assignment by the RPM. 9 | 10 | group { 'elasticsearch': 11 | ensure => present, 12 | gid => $gid, 13 | } 14 | user { 'elasticsearch': 15 | ensure => present, 16 | uid => $uid, 17 | gid => $gid, 18 | home => '/usr/share/elasticsearch', 19 | shell => '/sbin/nologin', 20 | managehome => false, 21 | } 22 | include elasticsearch 23 | User['elasticsearch'] -> Package['elasticsearch'] 24 | } 25 | -------------------------------------------------------------------------------- /site/modules/profile/manifests/elasticsearch/coordinating_node.pp: -------------------------------------------------------------------------------- 1 | class profile::elasticsearch::coordinating_node ( 2 | Hash[String, Hash] $firewall_multis, 3 | Hash $config, 4 | Hash $init_defaults, 5 | ) { 6 | 7 | # Users of this profile should always allow this class to configure ES as a 8 | # client node. 9 | # 10 | ['node.master', 'node.data', 'node.ingest'].each |String $key| { 11 | if has_key($config, $key) { 12 | fail("Do not specify ${key} in ${module_name}") 13 | } 14 | } 15 | 16 | create_resources(firewall_multi, $firewall_multis) 17 | 18 | include elasticsearch 19 | include profile::elasticsearch 20 | 21 | $cluster_name = $config['cluster.name'] 22 | 23 | elasticsearch::instance { "${cluster_name}-coordinating-instance": 24 | init_defaults => $init_defaults, 25 | config => merge($config, { 26 | 'node.master' => false, 27 | 'node.data' => false, 28 | 'node.ingest' => false, 29 | }), 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /site/modules/profile/manifests/elasticsearch/data_node.pp: -------------------------------------------------------------------------------- 1 | class profile::elasticsearch::data_node ( 2 | 3 | # Mandatory params. 4 | Stdlib::Absolutepath $datadir, 5 | Hash[String, Hash] $firewall_multis, 6 | 7 | # Volume groups. 8 | Hash[String, Struct[{ 9 | physical_volumes => Array[String], 10 | logical_volumes => Hash[String, Struct[{ 11 | mountpath => Stdlib::Absolutepath 12 | }]] 13 | }]] $volume_groups, 14 | 15 | # Optional params. 16 | Hash $config = {}, 17 | Hash $init_defaults = {}, 18 | Hash $es_plugins = {}, 19 | Hash $curator_jobs = {}, 20 | Array[Pattern[/^-/]] $jvm_options = [], 21 | Integer[0,1] $vm_swappiness = undef, 22 | Integer $vm_max_map_count = undef, 23 | 24 | # ES templates. 25 | Hash[String, Struct[{ 26 | source => Pattern[/puppet:\/\/\//] 27 | }]] $es_templates = {}, 28 | 29 | ) { 30 | 31 | # Users of this profile should always allow this class to configure ES as a 32 | # data node. 33 | # 34 | if has_key($config, 'node.data') { 35 | fail("Do not specify node.data in ${module_name}") 36 | } 37 | 38 | unless length($volume_groups) == 1 { 39 | fail('You must specify one volume group for the data dir') 40 | } 41 | 42 | # Note that it is expected that the user has created a custom fact espv 43 | # points to the physical volume for the data node filesystem. 44 | # 45 | assert_type(Stdlib::Absolutepath, $facts['espv']) 46 | 47 | create_resources(firewall_multi, $firewall_multis) 48 | create_resources(lvm::volume_group, $volume_groups) 49 | 50 | include elasticsearch 51 | include profile::elasticsearch 52 | 53 | $cluster_name = $config['cluster.name'] 54 | 55 | elasticsearch::instance { $cluster_name: 56 | config => merge($config, {'node.data' => true}), 57 | datadir => $datadir, 58 | init_defaults => $init_defaults, 59 | jvm_options => $jvm_options, 60 | } 61 | Mount[$datadir] -> File[$datadir] 62 | 63 | create_resources(elasticsearch::template, $es_templates) 64 | create_resources(elasticsearch::plugin, $es_plugins) 65 | 66 | unless empty($curator_jobs) { 67 | package { 'elasticsearch-curator': 68 | ensure => installed, 69 | } 70 | create_resources(cron, $curator_jobs) 71 | } 72 | 73 | Sysctl { 74 | before => Service["elasticsearch-instance-${cluster_name}"] 75 | } 76 | 77 | if $vm_swappiness { 78 | sysctl { 'vm.swappiness': value => $vm_swappiness } 79 | } 80 | 81 | if $vm_max_map_count { 82 | sysctl { 'vm.max_map_count': value => $vm_max_map_count } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /site/modules/profile/manifests/jdk.pp: -------------------------------------------------------------------------------- 1 | class profile::jdk ( 2 | Pattern[/java-\d+\.\d+\.\d+-openjdk/] $package, #/ # comment tricks vim highlighting. 3 | ) { 4 | package { $package: 5 | ensure => installed, 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /site/modules/profile/manifests/kibana.pp: -------------------------------------------------------------------------------- 1 | class profile::kibana ( 2 | Hash[String, Hash] $firewall_multis, 3 | Integer[30000] $uid, 4 | Integer[30000] $gid, 5 | ) { 6 | # Pass in an empty hash here if used in conjunction with 7 | # profile::nginx. 8 | create_resources(firewall_multi, $firewall_multis) 9 | 10 | # Manage the user and group to prevent random UID and 11 | # GID assignment by the RPM. 12 | 13 | group { 'kibana': 14 | ensure => present, 15 | gid => $gid, 16 | } 17 | user { 'kibana': 18 | ensure => present, 19 | uid => $uid, 20 | gid => $gid, 21 | home => '/home/kibana', 22 | shell => '/bin/bash', 23 | managehome => false, 24 | } 25 | 26 | include profile::jdk 27 | include kibana 28 | } 29 | -------------------------------------------------------------------------------- /site/modules/profile/manifests/logstash.pp: -------------------------------------------------------------------------------- 1 | class profile::logstash ( 2 | Integer[30000] $uid, 3 | Integer[30000] $gid, 4 | 5 | Hash[String, Hash] $firewall_multis, 6 | 7 | Hash[String, Struct[{ 8 | source => Pattern[/puppet:\/\/\//], 9 | path => Stdlib::Absolutepath }]] $configfiles, 10 | 11 | Hash[String, Struct[{ 12 | source => Pattern[/puppet:\/\/\//], 13 | path => Stdlib::Absolutepath }]] $patternfiles, 14 | 15 | ) { 16 | create_resources(firewall_multi, $firewall_multis) 17 | 18 | # Manage the user and group to prevent random UID and 19 | # GID assignment by the RPM. 20 | # 21 | group { 'logstash': 22 | ensure => present, 23 | gid => $gid, 24 | } 25 | user { 'logstash': 26 | ensure => present, 27 | uid => $uid, 28 | gid => $gid, 29 | comment => 'logstash', 30 | home => '/opt/logstash', 31 | shell => '/sbin/nologin', 32 | managehome => false, 33 | } 34 | User['logstash'] -> Package['logstash'] 35 | 36 | include logstash 37 | 38 | create_resources(logstash::configfile, $configfiles) 39 | create_resources(logstash::patternfile, $patternfiles) 40 | 41 | include profile::jdk 42 | Package[$profile::jdk::package] -> Package['logstash'] 43 | } 44 | -------------------------------------------------------------------------------- /site/modules/profile/manifests/nginx.pp: -------------------------------------------------------------------------------- 1 | class profile::nginx ( 2 | Hash[String, Hash] $firewall_multis, 3 | Pattern[/(\d+(\.|$)){4}/] $backend_host, #/ # comment tricks vim highlighting. 4 | Integer $backend_port, 5 | Integer[30000] $uid, 6 | Integer[30000] $gid, 7 | ) { 8 | create_resources(firewall_multi, $firewall_multis) 9 | 10 | # Manage the user and group to prevent random UID and 11 | # GID assignment by the RPM. 12 | 13 | group { 'nginx': 14 | ensure => present, 15 | gid => $gid, 16 | } 17 | user { 'nginx': 18 | ensure => present, 19 | uid => $uid, 20 | gid => $gid, 21 | comment => 'nginx user', 22 | home => '/var/cache/nginx', 23 | shell => '/sbin/nologin', 24 | managehome => false, 25 | } 26 | 27 | include nginx 28 | 29 | nginx::resource::server { $facts['fqdn']: 30 | proxy => "http://$backend_host:$backend_port", 31 | } 32 | 33 | User['nginx'] -> Package['nginx'] 34 | } 35 | -------------------------------------------------------------------------------- /site/modules/profile/manifests/redis.pp: -------------------------------------------------------------------------------- 1 | class profile::redis ( 2 | Hash[String, Hash] $firewall_multis, 3 | 4 | Hash[ 5 | String, Struct[{ 6 | physical_volumes => Array[String], 7 | logical_volumes => Hash[String, Struct[{ 8 | mountpath => Stdlib::Absolutepath }]] }]] $volume_groups, 9 | 10 | Integer[30000] $uid, 11 | Integer[30000] $gid, 12 | 13 | Integer[0,1] $vm_overcommit_memory, 14 | Integer $net_core_somaxconn, 15 | ) { 16 | assert_type(Stdlib::Absolutepath, $facts['redispv']) 17 | 18 | create_resources(firewall_multi, $firewall_multis) 19 | 20 | # Our redis has a separate filesystem for its work dir. 21 | 22 | include redis 23 | $workdir = $::redis::workdir 24 | 25 | # Manage the user and group to prevent random UID and 26 | # GID assignment by the RPM. 27 | 28 | group { 'redis': 29 | ensure => present, 30 | gid => $gid, 31 | } 32 | user { 'redis': 33 | ensure => present, 34 | uid => $uid, 35 | gid => $gid, 36 | home => $workdir, 37 | comment => 'Redis Server', 38 | shell => '/sbin/nologin', 39 | managehome => false, 40 | } 41 | User['redis'] -> Package['redis'] 42 | 43 | # We manage the work dir's file explicitly because the 44 | # Redis module does not do this. 45 | 46 | create_resources(lvm::volume_group, $volume_groups) 47 | 48 | # TODO: Redis now apparently does manage the workdir. Must find 49 | # out what the other implications are of the change. 50 | # 51 | # file { $workdir: 52 | # ensure => directory, 53 | # owner => 'redis', 54 | # group => 'redis', 55 | # mode => '0755', 56 | # } 57 | 58 | Mount[$workdir] -> File[$workdir] 59 | 60 | include disable_transparent_hugepage 61 | Service['disable-transparent-hugepage'] 62 | -> 63 | sysctl { 'vm.overcommit_memory': value => $vm_overcommit_memory } 64 | -> 65 | sysctl { 'net.core.somaxconn': value => $net_core_somaxconn } 66 | -> 67 | Service['redis'] 68 | } 69 | -------------------------------------------------------------------------------- /site/modules/role/manifests/elk_stack.pp: -------------------------------------------------------------------------------- 1 | class role::elk_stack { 2 | include profile::base 3 | include profile::elasticsearch::data_node 4 | include profile::elasticsearch::coordinating_node 5 | include profile::kibana 6 | include profile::nginx 7 | include profile::redis 8 | include profile::logstash 9 | 10 | # The disable_transparent_hugepage module, included by the Redis profile, 11 | # runs a script that resets the value of vm.swappiness, breaking idempotence. 12 | # So the enable-tuned-profile has run before the swappiness setting is 13 | # enabled. 14 | # 15 | Exec['enable-tuned-profile'] 16 | -> 17 | Sysctl['vm.swappiness'] 18 | 19 | # See Issue #7. Something around the time of the Logstash package is changing 20 | # vm.swappiness to 30. This ordering ensures idempotence. 21 | # 22 | Package['logstash'] 23 | -> 24 | Sysctl['vm.swappiness'] 25 | 26 | Wait_for { 27 | polling_frequency => 5, # Wait up to 2 minutes. 28 | max_retries => 24, 29 | refreshonly => true, 30 | } 31 | 32 | $cluster_name = $::profile::elasticsearch::data_node::config['cluster.name'] 33 | 34 | # The order in which things need to start to ensure that everything 35 | # starts cleanly. 36 | # 37 | Service["elasticsearch-instance-${cluster_name}"] 38 | ~> 39 | wait_for { 'es-master': 40 | query => 'cat /var/log/elasticsearch/es01/es01.log 2> /dev/null', 41 | regex => 'o.e.n.Node.*started', 42 | } 43 | -> 44 | Service["elasticsearch-instance-${cluster_name}-coordinating-instance"] 45 | ~> 46 | wait_for { 'es-client': 47 | query => 'cat /var/log/elasticsearch/es01-coordinating-instance/es01.log 2> /dev/null', 48 | regex => 'o.e.n.Node.*started', 49 | } 50 | -> 51 | Service['kibana'] 52 | ~> 53 | wait_for { 'kibana': 54 | query => 'journalctl -u kibana.service', 55 | regex => 'Server running at.*5601', 56 | } 57 | 58 | Wait_for['es-master'] -> Service['logstash'] 59 | 60 | Service['redis'] 61 | ~> 62 | wait_for { 'redis': 63 | query => 'cat /var/log/redis/redis.log 2> /dev/null', 64 | regex => 'The server is now ready to accept connections on port 6379', 65 | } 66 | -> 67 | Service['logstash'] 68 | ~> 69 | wait_for { 'logstash': 70 | query => 'cat /var/log/logstash/logstash-plain.log 2> /dev/null', 71 | regex => 'Successfully started Logstash API endpoint', 72 | } 73 | -> 74 | Service['filebeat'] 75 | 76 | # This seems to be the best way to set index.number_of_replicas to 0 for 77 | # all indices, as required in an all-in-one configuration. 78 | # 79 | elasticsearch::template { 'zero_replicas': 80 | content => { 81 | 'index_patterns' => ['*'], 82 | 'settings' => { 83 | 'index' => { 84 | 'number_of_replicas' => '0', 85 | } 86 | } 87 | }, 88 | } 89 | 90 | Elasticsearch::Template['zero_replicas'] -> Service['logstash'] 91 | } 92 | -------------------------------------------------------------------------------- /spec/acceptance/nodesets/default.yml: -------------------------------------------------------------------------------- 1 | --- 2 | HOSTS: 3 | centos-72-x64: 4 | platform: el-7-x86_64 5 | box: puppetlabs/centos-7.2-64-nocm 6 | hypervisor: vagrant 7 | ip: 10.10.2.15 8 | vagrant_memsize: 8192 9 | volumes: 10 | sdb: 11 | size: 5000 12 | sdc: 13 | size: 20000 14 | sdd: 15 | size: 20000 16 | forwarded_ports: 17 | nginx: 18 | from: 10080 19 | to: 80 20 | kibana: 21 | from: 5601 22 | to: 5601 23 | CONFIG: 24 | log_level: verbose 25 | type: foss 26 | masterless: true 27 | -------------------------------------------------------------------------------- /spec/acceptance/role_elk_stack_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper_acceptance' 2 | 3 | elk_version = '6.8.4' 4 | curator_version = '5.8.1' 5 | openjdk = 'java-1.8.0-openjdk-1.8.0.232.b09-0.el7_7.x86_64' 6 | 7 | pp = <<-EOS 8 | stage { 'pre': before => Stage['main'] } 9 | 10 | Firewall { 11 | require => Class['profile::base::firewall::pre'], 12 | before => Class['profile::base::firewall::post'], 13 | } 14 | 15 | include role::elk_stack 16 | EOS 17 | 18 | describe 'role::elk_stack' do 19 | context 'puppet apply' do 20 | it 'is expected to be idempotent and apply without errors' do 21 | 22 | apply_manifest pp, :catch_failures => true 23 | 24 | # test for idempotence 25 | expect(apply_manifest(pp, :catch_failures => true).exit_code).to be_zero 26 | end 27 | end 28 | 29 | context 'filebeat' do 30 | context 'packages' do 31 | describe package('filebeat') do 32 | it { should be_installed.with_version(elk_version) } 33 | end 34 | end 35 | 36 | context 'config files' do 37 | describe file('/etc/filebeat/filebeat.yml') do 38 | its(:content) { should match /managed by Puppet/ } 39 | end 40 | 41 | describe file('/etc/filebeat/conf.d/syslogs.yml') do 42 | it { should be_file } 43 | end 44 | end 45 | 46 | context 'log files' do 47 | describe file('/var/log/filebeat/filebeat') do 48 | its(:content) { should match /filebeat start running/ } 49 | end 50 | end 51 | 52 | context 'process' do 53 | describe process('filebeat') do 54 | its(:args) { should match %r{-c /etc/filebeat/filebeat.yml} } 55 | its(:args) { should match %r{-path.home /usr/share/filebeat} } 56 | its(:args) { should match %r{-path.config /etc/filebeat} } 57 | its(:args) { should match %r{-path.data /var/lib/filebeat} } 58 | its(:args) { should match %r{-path.logs /var/log/filebeat} } 59 | end 60 | end 61 | end 62 | 63 | context 'logstash' do 64 | context 'packages' do 65 | describe package('logstash') do 66 | it { should be_installed.with_version(elk_version) } 67 | end 68 | end 69 | 70 | context 'user' do 71 | describe user('logstash') do 72 | it { should exist } 73 | it { should have_uid 30001 } 74 | end 75 | end 76 | 77 | context 'executable' do 78 | describe file('/usr/share/logstash/bin/logstash') do 79 | it { should be_executable } 80 | end 81 | 82 | describe file('/usr/share/logstash/bin/logstash-plugin') do 83 | it { should be_executable } 84 | end 85 | end 86 | 87 | context 'config files' do 88 | describe file('/etc/logstash/conf.d/shipper.conf') do 89 | its(:content) { should match /MANAGED BY PUPPET/ } 90 | end 91 | 92 | describe file('/etc/logstash/conf.d/indexer.conf') do 93 | its(:content) { should match /MANAGED BY PUPPET/ } 94 | end 95 | 96 | describe file('/etc/logstash/jvm.options') do 97 | its(:content) { should match /managed by Puppet/ } 98 | end 99 | 100 | describe file('/etc/logstash/log4j2.properties') do 101 | it { should be_file } 102 | end 103 | 104 | describe file('/etc/logstash/startup.options') do 105 | it { should be_file } 106 | end 107 | 108 | describe file('/etc/logstash/logstash.yml') do 109 | its(:content) { should match %r{path.data.*/var/lib/logstash} } 110 | its(:content) { should match %r{path.logs.*/var/log/logstash} } 111 | end 112 | 113 | describe file('/etc/logstash/pipelines.yml') do 114 | its(:content) { should match /pipeline.id.*shipper/ } 115 | its(:content) { should match /pipeline.id.*indexer/ } 116 | end 117 | end 118 | 119 | context 'log files' do 120 | describe file('/var/log/logstash/logstash-plain.log') do 121 | its(:content) { should match /Starting Logstash/ } 122 | its(:content) { should match /No persistent UUID file found. Generating new UUID/ } 123 | its(:content) { should match /Starting pipeline.*indexer/ } 124 | its(:content) { should match /Starting pipeline.*shipper/ } 125 | its(:content) { should match /Beats inputs: Starting input listener.*5044/ } 126 | its(:content) { should match /Pipeline started successfully.*shipper/ } 127 | its(:content) { should match /Elasticsearch pool URLs updated/ } 128 | its(:content) { should match /Restored connection to ES instance/ } 129 | its(:content) { should match /ES Output version determined/ } 130 | its(:content) { should match /Detected a 6.x and above cluster: the.*type.*event field won.*t be used to determine the document _type/ } 131 | its(:content) { should match /New Elasticsearch output/ } 132 | its(:content) { should match /Registering Redis/ } 133 | its(:content) { should match /Pipeline started successfully.*indexer/ } 134 | its(:content) { should match /Pipelines running.*count=>2/ } 135 | its(:content) { should match /Starting server on port: 5044/ } 136 | its(:content) { should match /Successfully started Logstash API endpoint.*9600/ } 137 | end 138 | 139 | describe file('/var/log/logstash/logstash-slowlog-plain.log') do 140 | its(:size) { should be_zero } 141 | end 142 | end 143 | 144 | # FIXME. This is failing and I don't know why. 145 | # 146 | # context 'commands' do 147 | # describe command('echo hello world | /usr/share/logstash/bin/logstash -e "input { stdin { type => stdin } } output { stdout { } }"') do 148 | # its(:stdout) { should match /"message" => "hello world"/ } 149 | # end 150 | # end 151 | end 152 | 153 | context 'redis' do 154 | context 'packages' do 155 | describe package('redis') do 156 | it { should be_installed.with_version('3.2.12') } 157 | end 158 | end 159 | 160 | context 'mount points' do 161 | describe file('/var/lib/redis') do 162 | it { should be_directory } 163 | it { should be_owned_by 'redis' } 164 | it { should be_grouped_into 'redis' } 165 | it { should be_mounted.with(:type => 'ext4') } 166 | end 167 | end 168 | 169 | context 'log files' do 170 | describe file('/var/log/redis/redis.log') do 171 | its(:content) { should match /Server started, Redis version \d+\.\d+\.\d+/ } 172 | end 173 | end 174 | 175 | context 'ports' do 176 | describe port(6379) do 177 | it { should be_listening } 178 | end 179 | end 180 | 181 | describe 'end to end test' do 182 | before(:all) do 183 | shell('redis-cli lpush mylist foo') 184 | end 185 | it 'can push and pop to a list' do 186 | shell('redis-cli lpop mylist') do |r| 187 | expect(r.stdout).to match /foo/ 188 | end 189 | end 190 | end 191 | end 192 | 193 | context 'elasticsearch' do 194 | 195 | context 'packages' do 196 | [ 197 | ['java-1.8.0-openjdk', '1.8.0'], 198 | ['java-1.8.0-openjdk-headless', '1.8.0'], 199 | ['elasticsearch', elk_version], 200 | ['elasticsearch-curator', curator_version], 201 | 202 | ].each do |package, version| 203 | 204 | describe package(package) do 205 | it { should be_installed.with_version(version) } 206 | end 207 | end 208 | end 209 | 210 | context 'user' do 211 | describe user('elasticsearch') do 212 | it { should exist } 213 | it { should have_uid 30000 } 214 | end 215 | end 216 | 217 | context 'ports' do 218 | [9200, 9300, 9600].each do |port| 219 | describe port(port) do 220 | it { should be_listening } 221 | end 222 | end 223 | end 224 | 225 | context 'directories' do 226 | describe file("/usr/lib/jvm/#{openjdk}") do 227 | it { should be_directory } 228 | end 229 | end 230 | 231 | context 'config files' do 232 | describe file('/usr/lib/tmpfiles.d/elasticsearch.conf') do 233 | its(:content) { should match /elasticsearch/ } 234 | end 235 | 236 | describe file('/etc/elasticsearch/es01/logging.yml') do 237 | its(:content) { should match /managed by Puppet/ } 238 | end 239 | 240 | describe file('/etc/elasticsearch/es01/elasticsearch.yml') do 241 | its(:content) { should match /MANAGED BY PUPPET/ } 242 | end 243 | 244 | describe file('/lib/systemd/system/elasticsearch-es01.service') do 245 | it { should be_file } 246 | end 247 | end 248 | 249 | context 'log files' do 250 | describe file('/var/log/elasticsearch/es01/es01.log') do 251 | its(:content) { should match /initialized/ } 252 | its(:content) { should match /using.*data paths, mounts/ } 253 | its(:content) { should match /heap size/ } 254 | its(:content) { should match /node name.*node ID/ } 255 | its(:content) { should match /JVM arguments/ } 256 | its(:content) { should match /loaded module/ } 257 | its(:content) { should match /no plugins loaded/ } 258 | its(:content) { should match /using discovery type.*zen/ } 259 | its(:content) { should match /initialized/ } 260 | its(:content) { should match /starting .../ } 261 | its(:content) { should match /publish_address.*127.0.0.1:9300/ } 262 | its(:content) { should match /zen-disco-elected-as-master.*reason: new_master/ } 263 | its(:content) { should match /publish_address.*127.0.0.1:9200/ } 264 | its(:content) { should match /started/ } 265 | its(:content) { should match /WARN.*Failed to clear cache for realms/ } # What is this? 266 | its(:content) { should match /adding template/ } 267 | its(:content) { should match /license.*mode.*basic.*valid/ } 268 | end 269 | 270 | describe file('/var/log/elasticsearch/es01/es01_index_search_slowlog.log') do 271 | its(:size) { should be_zero } 272 | end 273 | 274 | describe file('/var/log/elasticsearch/es01/es01_index_indexing_slowlog.log') do 275 | its(:size) { should be_zero } 276 | end 277 | 278 | describe file('/var/log/elasticsearch/es01/gc.log.0.current') do 279 | its(:content) { should match /OpenJDK 64-Bit Server VM/ } 280 | end 281 | end 282 | 283 | context 'mount points' do 284 | describe file('/srv/es') do 285 | it { should be_directory } 286 | it { should be_owned_by 'elasticsearch' } 287 | it { should be_grouped_into 'root' } 288 | it { should be_mounted.with(:type => 'ext4') } 289 | end 290 | end 291 | 292 | context 'kernel parameters' do 293 | describe linux_kernel_parameter('vm.max_map_count') do 294 | its(:value) { should eq 262144 } 295 | end 296 | 297 | describe linux_kernel_parameter('vm.swappiness') do 298 | its(:value) { should eq 1 } 299 | end 300 | end 301 | 302 | context 'commands' do 303 | describe command('curl 0.0.0.0:9200') do 304 | its(:stdout) { should match /cluster_name.*es01/ } 305 | end 306 | 307 | describe command('curl 0.0.0.0:9200/_cluster/health?pretty') do 308 | its(:stdout) { should match /green/ } 309 | end 310 | 311 | # TODO. Extend based on this page: 312 | # https://www.safaribooksonline.com/library/view/mastering-elastic-stack/9781786460011/ch03s15.html 313 | # 314 | describe command('curl 0.0.0.0:9600/?pretty') do 315 | its(:stdout) { should match /host.*centos/ } 316 | end 317 | 318 | it 'add some data' do 319 | shell('curl -XPUT 0.0.0.0:9200/blog/user/dilbert -H'"'Content-Type: application/json' -d '{"'"name":"dilbert"}'"' ; sleep 5") 320 | expect(command("curl '0.0.0.0:9200/blog/user/_search?q=name:Dilbert&pretty'").stdout).to match /_id.*dilbert/ 321 | end 322 | end 323 | end 324 | 325 | context 'ES coordinating-only instance' do 326 | context 'ports' do 327 | [9201, 9301].each do |port| 328 | describe port(port) do 329 | it { should be_listening } 330 | end 331 | end 332 | end 333 | 334 | context 'config files' do 335 | describe file('/usr/lib/tmpfiles.d/elasticsearch.conf') do 336 | its(:content) { should match /elasticsearch/ } 337 | end 338 | 339 | describe file('/etc/elasticsearch/es01-coordinating-instance/logging.yml') do 340 | its(:content) { should match /managed by Puppet/ } 341 | end 342 | 343 | describe file('/etc/elasticsearch/es01-coordinating-instance/elasticsearch.yml') do 344 | its(:content) { should match /MANAGED BY PUPPET/ } 345 | end 346 | 347 | describe file('/lib/systemd/system/elasticsearch-es01-coordinating-instance.service') do 348 | it { should be_file } 349 | end 350 | end 351 | 352 | context 'log files' do 353 | describe file('/var/log/elasticsearch/es01-coordinating-instance/es01.log') do 354 | its(:content) { should match /publish_address.*127.0.0.1:9301/ } 355 | its(:content) { should match /detected_master.*reason: apply cluster state/ } 356 | its(:content) { should match /publish_address.*127.0.0.1:9201/ } 357 | its(:content) { should match /started/ } 358 | its(:content) { should match /WARN.*Failed to clear cache for realms/ } # What is this? 359 | end 360 | 361 | describe file('/var/log/elasticsearch/es01-coordinating-instance/es01_index_search_slowlog.log') do 362 | its(:size) { should be_zero } 363 | end 364 | 365 | describe file('/var/log/elasticsearch/es01-coordinating-instance/es01_index_indexing_slowlog.log') do 366 | its(:size) { should be_zero } 367 | end 368 | 369 | describe file('/var/log/elasticsearch/es01-coordinating-instance/gc.log.0.current') do 370 | its(:content) { should match /OpenJDK 64-Bit Server VM/ } 371 | end 372 | end 373 | 374 | context 'commands' do 375 | describe command('curl 0.0.0.0:9201') do 376 | its(:stdout) { should match /name.*es01_coordinating/ } 377 | end 378 | 379 | describe command('curl 0.0.0.0:9201/_cluster/health?pretty') do 380 | its(:stdout) { should match /green/ } 381 | end 382 | end 383 | end 384 | 385 | context 'kibana' do 386 | context 'packages' do 387 | describe package('kibana') do 388 | it { should be_installed.with_version(elk_version) } 389 | end 390 | end 391 | 392 | context 'user' do 393 | describe user('kibana') do 394 | it { should exist } 395 | it { should have_uid 30002 } 396 | end 397 | end 398 | 399 | context 'commands' do 400 | describe command('journalctl -u kibana.service') do 401 | its(:stdout) { should match /Started Kibana/ } 402 | end 403 | 404 | describe command('curl 0.0.0.0:5601/status -I') do 405 | its(:stdout) { should match %r{200 OK} } 406 | end 407 | end 408 | end 409 | end 410 | -------------------------------------------------------------------------------- /spec/classes/role_elk_stack_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'puppet/file_serving' 3 | 4 | describe 'role::elk_stack' do 5 | 6 | # This setup is to work-around the Elasticsearch Template custom type as seen 7 | # here: https://github.com/elastic/puppet-elasticsearch/blob/ 8 | # eef10e8ac99d9295c4297234b684471bdad42014/lib/puppet/type/ 9 | # elasticsearch_template.rb#L90-L109 10 | # 11 | # where this custom type retrieves content from Puppet's FileServing 12 | # indirection. 13 | # 14 | class Fake 15 | def content 16 | File.read('./site/modules/profile/files/logstash/logstash.json') 17 | end 18 | end 19 | 20 | before(:each) do 21 | allow(Puppet::FileServing::Metadata.indirection).to receive(:find). 22 | and_call_original 23 | 24 | allow(Puppet::FileServing::Metadata.indirection).to receive(:find). 25 | with('puppet:///modules/profile/logstash/logstash.json').and_return(true) 26 | 27 | allow(Puppet::FileServing::Content.indirection).to receive(:find). 28 | and_call_original 29 | 30 | allow(Puppet::FileServing::Content.indirection).to receive(:find). 31 | with( 32 | 'puppet:///modules/profile/logstash/logstash.json', 33 | hash_including(environment: instance_of(Puppet::Node::Environment)) 34 | ).and_return(Fake.new) 35 | end 36 | 37 | it 'should write a compiled catalog' do 38 | is_expected.to compile.with_all_deps 39 | File.write( 40 | 'catalogs/role__elk_stack.json', 41 | PSON.pretty_generate(catalogue) 42 | ) 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /spec/fixtures/Puppetfile.legacy: -------------------------------------------------------------------------------- 1 | forge 'https://forgeapi.puppetlabs.com' 2 | 3 | # To analyse these dependencies: 4 | # cat spec/**/metadata.json | jq -r '.name as $n | .dependencies[] | 5 | # select(.name=="puppetlabs/stdlib") | [$n, .version_requirement] | @tsv' 6 | # 7 | # To try reinstalling: 8 | # rm -f spec/fixtures/Puppetfile.lock ; LIBRARIAN_VERBOSE=true \ 9 | # bundle exec rake generate_puppetfile librarian_spec_prep 10 | 11 | mod 'puppetlabs/stdlib', '< 6.0.0' 12 | 13 | mod 'puppetlabs/ntp' 14 | mod 'puppetlabs/lvm' 15 | mod 'puppetlabs/firewall' 16 | mod 'alexharvey/firewall_multi' 17 | mod 'alexharvey/disable_transparent_hugepage' 18 | mod 'pcfens/filebeat' 19 | mod 'elastic/elasticsearch' 20 | mod 'elastic/logstash' 21 | 22 | # At this point arioch/redis is the only module still requiring stdlib 23 | # < 5.0.0. My branch updates arioch/redis to support latest stdlib. 24 | # 25 | mod 'arioch/redis', 26 | :git => 'https://github.com/alexharv074/puppet-redis.git', 27 | :branch => 'alexharvey/fix_up_tests' 28 | 29 | mod 'elastic/kibana' 30 | mod 'puppet/nginx' 31 | mod 'camptocamp/archive' 32 | mod 'thias/sysctl' 33 | mod 'heini/wait_for' 34 | 35 | # Dependencies. 36 | # 37 | # Although librarian-puppet can resolve the dependencies itself, g10k can't. 38 | # The commented-out entries are specified in metadata.json by the parent 39 | # module, but we don't actually need them. 40 | 41 | # Dependencies of arioch/redis. 42 | mod 'herculesteam/augeasproviders_core' 43 | mod 'herculesteam/augeasproviders_sysctl' 44 | #mod 'puppetlabs/apt' 45 | mod 'stahnma/epel' 46 | 47 | # Dependencies of all the elastic modules 48 | #mod 'elastic/elastic_stack' 49 | 50 | # Dependencies of elastic/elastic_stack 51 | #mod 'puppet/yum' 52 | 53 | # Dependencies of elastic/elasticsearch 54 | mod 'richardc/datacat' 55 | 56 | # Dependencies of pcfens/filebeat 57 | #mod 'puppetlabs/powershell' 58 | 59 | # Dependencies of puppet/nginx 60 | mod 'puppetlabs/concat', '< 6.0.0' 61 | -------------------------------------------------------------------------------- /spec/fixtures/Puppetfile.v6: -------------------------------------------------------------------------------- 1 | forge 'https://forgeapi.puppetlabs.com' 2 | 3 | # To analyse these dependencies: 4 | # cat spec/**/metadata.json | jq -r '.name as $n | .dependencies[] | 5 | # select(.name=="puppetlabs/stdlib") | [$n, .version_requirement] | @tsv' 6 | # 7 | # To try reinstalling: 8 | # rm -f spec/fixtures/Puppetfile.lock ; LIBRARIAN_VERBOSE=true \ 9 | # bundle exec rake generate_puppetfile librarian_spec_prep 10 | 11 | mod 'puppetlabs/stdlib', '< 6.0.0' 12 | 13 | mod 'puppetlabs/ntp' 14 | mod 'puppetlabs/lvm' 15 | mod 'puppetlabs/firewall' 16 | mod 'alexharvey/firewall_multi' 17 | mod 'alexharvey/disable_transparent_hugepage' 18 | mod 'pcfens/filebeat' 19 | mod 'elastic/elasticsearch' 20 | mod 'elastic/logstash' 21 | 22 | # At this point arioch/redis is the only module still requiring stdlib 23 | # < 5.0.0. My branch updates arioch/redis to support latest stdlib. 24 | # 25 | mod 'arioch/redis', 26 | :git => 'https://github.com/alexharv074/puppet-redis.git', 27 | :branch => 'alexharvey/fix_up_tests' 28 | 29 | mod 'elastic/kibana' 30 | mod 'puppet/nginx' 31 | mod 'camptocamp/archive' 32 | mod 'thias/sysctl' 33 | mod 'heini/wait_for' 34 | 35 | # Needed for Puppet 6. 36 | mod 'puppetlabs/yumrepo_core' 37 | mod 'puppetlabs/augeas_core' 38 | mod 'puppetlabs/mount_core' 39 | mod 'puppetlabs/cron_core' 40 | 41 | # Dependencies. 42 | # 43 | # Although librarian-puppet can resolve the dependencies itself, g10k can't. 44 | # The commented-out entries are specified in metadata.json by the parent 45 | # module, but we don't actually need them. 46 | 47 | # Dependencies of arioch/redis. 48 | mod 'herculesteam/augeasproviders_core' 49 | mod 'herculesteam/augeasproviders_sysctl' 50 | #mod 'puppetlabs/apt' 51 | mod 'stahnma/epel' 52 | 53 | # Dependencies of all the elastic modules 54 | #mod 'elastic/elastic_stack' 55 | 56 | # Dependencies of elastic/elastic_stack 57 | #mod 'puppet/yum' 58 | 59 | # Dependencies of elastic/elasticsearch 60 | mod 'richardc/datacat' 61 | 62 | # Dependencies of pcfens/filebeat 63 | #mod 'puppetlabs/powershell' 64 | 65 | # Dependencies of puppet/nginx 66 | mod 'puppetlabs/concat', '< 6.0.0' 67 | -------------------------------------------------------------------------------- /spec/fixtures/facts.d/facts.txt: -------------------------------------------------------------------------------- 1 | redispv=/dev/sdb 2 | espv=/dev/sdc 3 | -------------------------------------------------------------------------------- /spec/fixtures/hiera.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: 5 3 | hierarchy: 4 | - name: Yaml backend 5 | data_hash: yaml_data 6 | paths: 7 | - 'common.yaml' 8 | datadir: ./hieradata 9 | -------------------------------------------------------------------------------- /spec/fixtures/hiera.yaml.beaker: -------------------------------------------------------------------------------- 1 | --- 2 | version: 5 3 | hierarchy: 4 | - name: Yaml backend 5 | data_hash: yaml_data 6 | paths: 7 | - 'common.yaml' 8 | datadir: /etc/puppetlabs/code/hieradata 9 | -------------------------------------------------------------------------------- /spec/fixtures/hieradata/common.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # profile::base 4 | # 5 | ntp::servers: 6 | - 0.au.pool.ntp.org 7 | - 1.au.pool.ntp.org 8 | - 2.au.pool.ntp.org 9 | - 3.au.pool.ntp.org 10 | ntp::restrict: 11 | - default kod nomodify notrap nopeer noquery 12 | - -6 default kod nomodify notrap nopeer noquery 13 | 14 | profile::base::yum::repos: 15 | 'C7.0.1406-base': 16 | ensure: present 17 | baseurl: http://vault.centos.org/7.0.1406/os/$basearch/ 18 | descr: CentOS-7.0.1406 - Base 19 | enabled: 0 20 | gpgcheck: 1 21 | gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 22 | 'C7.0.1406-centosplus': 23 | ensure: present 24 | baseurl: http://vault.centos.org/7.0.1406/centosplus/$basearch/ 25 | descr: CentOS-7.0.1406 - CentOSPlus 26 | enabled: 0 27 | gpgcheck: 1 28 | gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 29 | 'C7.0.1406-extras': 30 | ensure: present 31 | baseurl: http://vault.centos.org/7.0.1406/extras/$basearch/ 32 | descr: CentOS-7.0.1406 - Extras 33 | enabled: 0 34 | gpgcheck: 1 35 | gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 36 | 'C7.0.1406-fasttrack': 37 | ensure: present 38 | baseurl: http://vault.centos.org/7.0.1406/fasttrack/$basearch/ 39 | descr: CentOS-7.0.1406 - CentOSPlus 40 | enabled: 0 41 | gpgcheck: 1 42 | gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 43 | 'C7.0.1406-updates': 44 | ensure: present 45 | baseurl: http://vault.centos.org/7.0.1406/updates/$basearch/ 46 | descr: CentOS-7.0.1406 - Updates 47 | enabled: 0 48 | gpgcheck: 1 49 | gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 50 | 'C7.1.1503-base': 51 | ensure: present 52 | baseurl: http://vault.centos.org/7.1.1503/os/$basearch/ 53 | descr: CentOS-7.1.1503 - Base 54 | enabled: 0 55 | gpgcheck: 1 56 | gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 57 | 'C7.1.1503-centosplus': 58 | ensure: present 59 | baseurl: http://vault.centos.org/7.1.1503/centosplus/$basearch/ 60 | descr: CentOS-7.1.1503 - CentOSPlus 61 | enabled: 0 62 | gpgcheck: 1 63 | gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 64 | 'C7.1.1503-extras': 65 | ensure: present 66 | baseurl: http://vault.centos.org/7.1.1503/extras/$basearch/ 67 | descr: CentOS-7.1.1503 - Extras 68 | enabled: 0 69 | gpgcheck: 1 70 | gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 71 | 'C7.1.1503-fasttrack': 72 | ensure: present 73 | baseurl: http://vault.centos.org/7.1.1503/fasttrack/$basearch/ 74 | descr: CentOS-7.1.1503 - CentOSPlus 75 | enabled: 0 76 | gpgcheck: 1 77 | gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 78 | 'C7.1.1503-updates': 79 | ensure: present 80 | baseurl: http://vault.centos.org/7.1.1503/updates/$basearch/ 81 | descr: CentOS-7.1.1503 - Updates 82 | enabled: 0 83 | gpgcheck: 1 84 | gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 85 | 'base': 86 | ensure: present 87 | descr: CentOS-$releasever - Base 88 | gpgcheck: 1 89 | gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 90 | #mirrorlist: http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=os&infra=$infra 91 | baseurl: http://mirror.centos.org/centos/$releasever/os/$basearch/ 92 | 'base-debuginfo': 93 | ensure: present 94 | baseurl: http://debuginfo.centos.org/7/$basearch/ 95 | descr: CentOS-7 - Debuginfo 96 | enabled: 0 97 | gpgcheck: 1 98 | gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-Debug-7 99 | 'base-source': 100 | ensure: present 101 | baseurl: http://vault.centos.org/centos/$releasever/os/Source/ 102 | descr: CentOS-$releasever - Base Sources 103 | enabled: 0 104 | gpgcheck: 1 105 | gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 106 | 'c7-media': 107 | ensure: present 108 | baseurl: 'file:///media/CentOS/ 109 | file:///media/cdrom/ 110 | file:///media/cdrecorder/' 111 | descr: CentOS-$releasever - Media 112 | enabled: 0 113 | gpgcheck: 1 114 | gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 115 | 'centosplus': 116 | ensure: present 117 | descr: CentOS-$releasever - Plus 118 | enabled: 0 119 | gpgcheck: 1 120 | gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 121 | #mirrorlist: http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=centosplus&infra=$infra 122 | baseurl: http://mirror.centos.org/centos/$releasever/centosplus/$basearch/ 123 | 'centosplus-source': 124 | ensure: present 125 | baseurl: http://vault.centos.org/centos/$releasever/centosplus/Source/ 126 | descr: CentOS-$releasever - Plus Sources 127 | enabled: 0 128 | gpgcheck: 1 129 | gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 130 | 'cr': 131 | ensure: present 132 | baseurl: http://mirror.centos.org/centos/$releasever/cr/$basearch/ 133 | descr: CentOS-$releasever - cr 134 | enabled: 0 135 | gpgcheck: 1 136 | gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 137 | 'epel': 138 | ensure: present 139 | descr: Extra Packages for Enterprise Linux 7 - $basearch 140 | enabled: 1 141 | failovermethod: priority 142 | gpgcheck: 1 143 | gpgkey: http://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7 144 | #mirrorlist: https://mirrors.fedoraproject.org/metalink?repo=epel-7&arch=$basearch 145 | baseurl: http://download.fedoraproject.org/pub/epel/7/$basearch 146 | 'epel-debuginfo': 147 | ensure: present 148 | descr: Extra Packages for Enterprise Linux 7 - $basearch - Debug 149 | enabled: 0 150 | failovermethod: priority 151 | gpgcheck: 1 152 | gpgkey: http://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7 153 | #mirrorlist: https://mirrors.fedoraproject.org/metalink?repo=epel-debug-7&arch=$basearch 154 | baseurl: http://download.fedoraproject.org/pub/epel/7/$basearch/debug 155 | 'epel-source': 156 | ensure: present 157 | descr: Extra Packages for Enterprise Linux 7 - $basearch - Source 158 | enabled: 0 159 | failovermethod: priority 160 | gpgcheck: 1 161 | gpgkey: http://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7 162 | #mirrorlist: https://mirrors.fedoraproject.org/metalink?repo=epel-source-7&arch=$basearch 163 | baseurl: http://download.fedoraproject.org/pub/epel/7/SRPMS 164 | 'epel-testing': 165 | ensure: present 166 | descr: Extra Packages for Enterprise Linux 7 - Testing - $basearch 167 | enabled: 0 168 | failovermethod: priority 169 | gpgcheck: 1 170 | gpgkey: http://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7 171 | #mirrorlist: https://mirrors.fedoraproject.org/metalink?repo=testing-epel7&arch=$basearch 172 | baseurl: http://download.fedoraproject.org/pub/epel/testing/7/$basearch 173 | 'epel-testing-debuginfo': 174 | ensure: present 175 | descr: Extra Packages for Enterprise Linux 7 - Testing - $basearch - Debug 176 | enabled: 0 177 | failovermethod: priority 178 | gpgcheck: 1 179 | gpgkey: http://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7 180 | #mirrorlist: https://mirrors.fedoraproject.org/metalink?repo=testing-debug-epel7&arch=$basearch 181 | baseurl: http://download.fedoraproject.org/pub/epel/testing/7/$basearch/debug 182 | 'epel-testing-source': 183 | ensure: present 184 | descr: Extra Packages for Enterprise Linux 7 - Testing - $basearch - Source 185 | enabled: 0 186 | failovermethod: priority 187 | gpgcheck: 1 188 | gpgkey: http://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7 189 | #mirrorlist: https://mirrors.fedoraproject.org/metalink?repo=testing-source-epel7&arch=$basearch 190 | baseurl: http://download.fedoraproject.org/pub/epel/testing/7/$basearch/SRPMS 191 | 'extras': 192 | ensure: present 193 | descr: CentOS-$releasever - Extras 194 | gpgcheck: 1 195 | gpgkey: http://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7 196 | #mirrorlist: http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=extras&infra=$infra 197 | baseurl: http://mirror.centos.org/centos/$releasever/extras/$basearch/ 198 | 'extras-source': 199 | ensure: present 200 | baseurl: http://vault.centos.org/centos/$releasever/extras/Source/ 201 | descr: CentOS-$releasever - Extras Sources 202 | enabled: 0 203 | gpgcheck: 1 204 | gpgkey: http://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7 205 | 'fasttrack': 206 | ensure: present 207 | descr: CentOS-7 - fasttrack 208 | enabled: 0 209 | gpgcheck: 1 210 | gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 211 | #mirrorlist: http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=fasttrack&infra=$infra 212 | baseurl: http://mirror.centos.org/centos/$releasever/fasttrack/$basearch/ 213 | 'updates': 214 | ensure: present 215 | descr: CentOS-$releasever - Updates 216 | gpgcheck: 1 217 | gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 218 | #mirrorlist: http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=updates&infra=$infra 219 | baseurl: http://mirror.centos.org/centos/$releasever/updates/$basearch/ 220 | 'updates-source': 221 | ensure: present 222 | baseurl: http://vault.centos.org/centos/$releasever/updates/Source/ 223 | descr: CentOS-$releasever - Updates Sources 224 | enabled: 0 225 | gpgcheck: 1 226 | gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 227 | 'nginx-release': 228 | descr: nginx repo 229 | baseurl: http://nginx.org/packages/centos/7/$basearch/ 230 | gpgcheck: 1 231 | gpgkey: http://nginx.org/keys/nginx_signing.key 232 | enabled: 1 233 | 'ELK-6.x': 234 | descr: elasticsearch repo 235 | baseurl: https://artifacts.elastic.co/packages/6.x/yum 236 | gpgcheck: 1 237 | gpgkey: https://artifacts.elastic.co/GPG-KEY-elasticsearch 238 | enabled: 1 239 | 'curator-5': 240 | descr: CentOS/RHEL 6 repository for Elasticsearch Curator 5.x packages 241 | baseurl: https://packages.elastic.co/curator/5/centos/6 242 | gpgcheck: 1 243 | gpgkey: https://packages.elastic.co/GPG-KEY-elasticsearch 244 | enabled: 1 245 | 246 | profile::base::firewall_multis: 247 | '00099 accept tcp ports for SSH and Kibana': 248 | dport: 249 | - 22 250 | - 80 251 | - 5601 252 | action: accept 253 | proto: tcp 254 | source: 255 | - 0.0.0.0/0 256 | 257 | # profile::base::filebeat 258 | # 259 | filebeat::manage_repo: false 260 | filebeat::package_ensure: latest 261 | filebeat::major_version: '6' # Must be a String. 262 | filebeat::outputs: 263 | logstash: 264 | hosts: 265 | - 0.0.0.0:5044 266 | loadbalance: false 267 | 268 | profile::base::filebeat::paths: 269 | - /var/log/messages 270 | - /var/log/testlog 271 | 272 | profile::base::tools: 273 | vim: 274 | ensure: installed 275 | 276 | # profile::jdk 277 | # 278 | profile::jdk::package: java-1.8.0-openjdk 279 | 280 | # profile::logstash 281 | # 282 | profile::logstash::uid: 30001 283 | profile::logstash::gid: 30001 284 | 285 | # NOTE: concerning the heap sizes: 286 | # test VM has 8G so: 287 | # 1g for LS shipper heap 288 | # 1g for LS indexer heap 289 | # 500m for Redis 290 | # 1g for ES heap 291 | # leaving ~ 4G for OS. 292 | 293 | profile::logstash::firewall_multis: {} 294 | 295 | logstash::manage_repo: false 296 | logstash::jvm_options: 297 | - -Xms1g 298 | - -Xmx1g 299 | logstash::pipelines: 300 | - 'pipeline.id': shipper 301 | 'path.config': /etc/logstash/conf.d/shipper.conf 302 | - 'pipeline.id': indexer 303 | 'path.config': /etc/logstash/conf.d/indexer.conf 304 | 305 | profile::logstash::configfiles: 306 | 'shipper.conf': 307 | source: puppet:///modules/profile/logstash/conf.d/shipper.conf 308 | path: /etc/logstash/conf.d/shipper.conf 309 | 'indexer.conf': 310 | source: puppet:///modules/profile/logstash/conf.d/indexer.conf 311 | path: /etc/logstash/conf.d/indexer.conf 312 | 313 | profile::logstash::patternfiles: {} 314 | 315 | # profile::redis 316 | # 317 | redis::workdir: /var/lib/redis 318 | redis::maxmemory: 16gb 319 | 320 | profile::redis::firewall_multis: {} 321 | profile::redis::volume_groups: 322 | redisvg00: 323 | physical_volumes: 324 | - "%{facts.redispv}" 325 | logical_volumes: 326 | redislv00: 327 | mountpath: "%{lookup('redis::workdir')}" 328 | profile::redis::uid: 30003 329 | profile::redis::gid: 30003 330 | profile::redis::vm_overcommit_memory: 1 331 | profile::redis::net_core_somaxconn: 1024 332 | 333 | # profile::elasticsearch 334 | # 335 | elasticsearch::api_host: 0.0.0.0 336 | elasticsearch::api_timeout: 120 337 | elasticsearch::manage_repo: false 338 | 339 | profile::elasticsearch::uid: 30000 340 | profile::elasticsearch::gid: 30000 341 | 342 | # profile::elasticsearch::data_node 343 | # 344 | profile::elasticsearch::data_node::firewall_multis: {} 345 | profile::elasticsearch::data_node::datadir: /srv/es 346 | profile::elasticsearch::data_node::volume_groups: 347 | esvg00: 348 | physical_volumes: 349 | - "%{facts.espv}" 350 | logical_volumes: 351 | eslv00: 352 | mountpath: "%{lookup('profile::elasticsearch::data_node::datadir')}" 353 | profile::elasticsearch::data_node::config: 354 | 'cluster.name': es01 355 | 'node.name': "es01_%{facts.hostname}" 356 | 'node.master': true 357 | profile::elasticsearch::data_node::init_defaults: 358 | JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.181-3.b13.el7_5.x86_64 359 | profile::elasticsearch::data_node::es_plugins: {} 360 | profile::elasticsearch::data_node::curator_jobs: 361 | curator_delete: 362 | command: /usr/bin/curator --master-only --logfile /var/log/elasticsearch/curator.log delete --older-than 20 363 | hour: 2 364 | minute: 12 365 | curator_close: 366 | command: /usr/bin/curator --master-only --logfile /var/log/elasticsearch/curator.log close --older-than 10 367 | hour: 2 368 | minute: 24 369 | curator_bloom: 370 | command: /usr/bin/curator --master-only --logfile /var/log/elasticsearch/curator.log bloom --older-than 2 371 | hour: 2 372 | minute: 32 373 | profile::elasticsearch::data_node::jvm_options: 374 | - -Xms1g 375 | - -Xmx1g 376 | - "-XX:HeapDumpPath=%{lookup('profile::elasticsearch::data_node::datadir')}" 377 | profile::elasticsearch::data_node::vm_swappiness: 1 378 | profile::elasticsearch::data_node::vm_max_map_count: 262144 379 | 380 | # see comments on https://www.coredump.id.au/inability-to-create-new-logstash-indexes-in-elasticsearch-2-0/#comments 381 | profile::elasticsearch::data_node::es_templates: 382 | logstash: 383 | source: puppet:///modules/profile/logstash/logstash.json 384 | 385 | # profile::elasticsearch::coordinating_node 386 | # 387 | profile::elasticsearch::coordinating_node::firewall_multis: {} 388 | profile::elasticsearch::coordinating_node::config: 389 | 'cluster.name': es01 390 | 'node.name': "es01_coordinating_%{facts.hostname}" 391 | profile::elasticsearch::coordinating_node::init_defaults: 392 | JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.65-0.b17.el6_7.x86_64 393 | ES_JAVA_OPTS: '"-XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:/var/log/elasticsearch/es01-coordinating-instance/gc.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M"' 394 | 395 | # profile::kibana 396 | # 397 | kibana::ensure: latest 398 | kibana::manage_repo: false 399 | kibana::config: 400 | 'server.host': 0.0.0.0 401 | 'server.port': 5601 402 | 'elasticsearch.url': http://0.0.0.0:9201 # 9201 by default on a single VM because 9200 is taken by the ES data node instance. 403 | 404 | profile::kibana::firewall_multis: {} 405 | profile::kibana::uid: 30002 406 | profile::kibana::gid: 30002 407 | 408 | # profile::nginx 409 | # 410 | nginx::manage_repo: false 411 | 412 | profile::nginx::firewall_multis: {} 413 | profile::nginx::backend_host: 0.0.0.0 414 | profile::nginx::backend_port: 5601 415 | profile::nginx::uid: 30004 416 | profile::nginx::gid: 30004 417 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # Without this, a warning is seen: 2 | # 3 | # puppetlabs_spec_helper: defaults `mock_with` to `:mocha`. 4 | # 5 | # but the RSpec.configure needs to come before require 6 | # puppetlabs_spec_helper. 7 | # 8 | RSpec.configure do |c| 9 | c.mock_with :mocha 10 | end 11 | 12 | require 'puppetlabs_spec_helper/module_spec_helper' 13 | 14 | FileUtils::mkdir_p 'catalogs' 15 | 16 | RSpec.configure do |c| 17 | c.color = true 18 | c.hiera_config = 'spec/fixtures/hiera.yaml' 19 | 20 | c.formatter = :documentation 21 | c.mock_framework = :rspec 22 | 23 | c.default_facts = { 24 | # required by our roles & profiles. 25 | :espv => '/dev/sdc', 26 | :redispv => '/dev/sdb', 27 | # firewall module. 28 | :puppetversion => '5.5.1', 29 | :selinux => false, 30 | :kernel => 'Linux', 31 | :osfamily => 'RedHat', 32 | :operatingsystem => 'RedHat', 33 | :operatingsystemrelease => '7.2.1511', 34 | # redis module. 35 | :operatingsystemmajrelease => '7', 36 | # filebeat, disable_transparent_hugepage etc. 37 | :os => {'family' => 'RedHat', 'release' => {'major' => '7', 'minor' => '1', 'full' => '7.1.1503'}}, 38 | # filebeat module. 39 | :filebeat_version => '6', 40 | } 41 | end 42 | -------------------------------------------------------------------------------- /spec/spec_helper_acceptance.rb: -------------------------------------------------------------------------------- 1 | # Environment variables: 2 | # 3 | # For Puppet 5.x and 6.x: 4 | # 5 | # ENV['BEAKER_PUPPET_COLLECTION'] 6 | # For Puppet 5 & this must be set to 'puppet5' or 'puppet6' 7 | # 8 | # ENV['BEAKER_PUPPET_AGENT_VERSION'] 9 | # Agent Version should == Puppet Version. 10 | # 11 | # Usage example: 12 | # 13 | # BEAKER_PUPPET_COLLECTION=puppet6 BEAKER_PUPPET_INSTALL_VERSION=6.4.2 \ 14 | # bundle exec rspec spec/acceptance 15 | # 16 | # For Puppet 4.x: 17 | # 18 | # ENV['BEAKER_PUPPET_COLLECTION'] 19 | # For Puppet 4 this must be set to 'pc1' 20 | # 21 | # ENV['BEAKER_PUPPET_AGENT_VERSION'] 22 | # For Puppet 4, Agent Version != Puppet Version. The conversion is 23 | # documented here: 24 | # https://puppet.com/docs/puppet/4.10/about_agent.html 25 | # 26 | # Usage example (for Puppet 4.10.12): 27 | # 28 | # BEAKER_PUPPET_COLLECTION=pc1 BEAKER_PUPPET_INSTALL_VERSION=1.10.12 \ 29 | # bundle exec rspec spec/acceptance 30 | # 31 | # Other variables: 32 | # 33 | # ENV['BEAKER_destroy'] 34 | # If set to 'no' Beaker will not tear down the Vagrant VM after the 35 | # tests run. Use this if you want the VM to keep running for 36 | # manual checking. 37 | # 38 | # ENV['YUM_UPDATE'] 39 | # If set, a yum update will be run before testing. 40 | 41 | require 'beaker-pe' 42 | require 'beaker-puppet' 43 | require 'beaker-rspec' 44 | require 'beaker/puppet_install_helper' 45 | require 'puppet' 46 | 47 | def copy_modules_to(host, opts = {}) 48 | Dir["#{opts[:source]}/*"].each do |dir| 49 | if File.symlink?(dir) 50 | scp_to host, dir, opts[:module_dir], 51 | {:ignore => 'spec/fixtures/modules'} 52 | else 53 | scp_to host, dir, opts[:dist_dir] 54 | end 55 | end 56 | end 57 | 58 | def copy_hiera_files_to(host, opts = {}) 59 | scp_to host, opts[:hiera_yaml], opts[:target] + '/hiera.yaml' 60 | scp_to host, opts[:hieradata], opts[:target] 61 | end 62 | 63 | def copy_external_facts_to(host, opts = {}) 64 | on host, "mkdir -p #{opts[:target]}" 65 | scp_to host, opts[:source], opts[:target] 66 | end 67 | 68 | run_puppet_install_helper 69 | 70 | RSpec.configure do |c| 71 | proj_root = File.expand_path(File.join(File.dirname(__FILE__), '..')) 72 | 73 | c.formatter = :documentation 74 | 75 | # Configure all nodes in nodeset 76 | # See spec/acceptance/nodesets/default.yml 77 | # 78 | c.before :suite do 79 | host = hosts[0] 80 | 81 | if ENV['YUM_UPDATE'] == 'yes' 82 | on host, 'yum -y update' 83 | end 84 | 85 | system 'bundle exec rake best_spec_prep' 86 | system 'bundle exec rake spec_prep' 87 | 88 | copy_modules_to(host, { 89 | :source => "#{proj_root}/spec/fixtures/modules", 90 | :dist_dir => '/etc/puppetlabs/code/modules', 91 | :module_dir => '/etc/puppetlabs/code/environments/production/modules' 92 | }) 93 | 94 | copy_hiera_files_to(host, { 95 | :hieradata => "#{proj_root}/spec/fixtures/hieradata", 96 | :hiera_yaml => "#{proj_root}/spec/fixtures/hiera.yaml.beaker", 97 | :target => '/etc/puppetlabs/code', 98 | }) 99 | 100 | copy_external_facts_to(host, { 101 | :source => "#{proj_root}/spec/fixtures/facts.d", 102 | :target => '/etc/facter', 103 | }) 104 | 105 | # https://tickets.puppetlabs.com/browse/MODULES-3153 106 | # https://tickets.puppetlabs.com/browse/MODULES-3184 107 | on host, 'yum -y install iptables-services' 108 | on host, 'systemctl start iptables.service' 109 | on host, 'systemctl start ip6tables.service' 110 | end 111 | end 112 | -------------------------------------------------------------------------------- /yamllint.yml: -------------------------------------------------------------------------------- 1 | --- 2 | rules: 3 | braces: 4 | min-spaces-inside: 0 5 | max-spaces-inside: 0 6 | min-spaces-inside-empty: -1 7 | max-spaces-inside-empty: -1 8 | brackets: 9 | min-spaces-inside: 0 10 | max-spaces-inside: 0 11 | min-spaces-inside-empty: -1 12 | max-spaces-inside-empty: -1 13 | colons: 14 | max-spaces-before: 0 15 | max-spaces-after: 1 16 | commas: 17 | max-spaces-before: 0 18 | min-spaces-after: 1 19 | max-spaces-after: 1 20 | document-end: disable 21 | document-start: 22 | level: error 23 | present: true 24 | empty-lines: 25 | max: 1 26 | max-start: 0 27 | max-end: 0 28 | empty-values: 29 | forbid-in-block-mappings: false 30 | forbid-in-flow-mappings: false 31 | hyphens: 32 | max-spaces-after: 1 33 | indentation: 34 | spaces: consistent 35 | indent-sequences: true 36 | check-multi-line-strings: false 37 | key-duplicates: enable 38 | key-ordering: disable 39 | new-line-at-end-of-file: enable 40 | new-lines: 41 | type: unix 42 | octal-values: 43 | forbid-implicit-octal: false 44 | forbid-explicit-octal: false 45 | trailing-spaces: enable 46 | truthy: 47 | level: warning 48 | --------------------------------------------------------------------------------