├── .fixtures.yml ├── .gitattributes ├── .gitignore ├── .gitlab-ci.yml ├── .pdkignore ├── .project ├── .rspec ├── .rubocop.yml ├── .travis.yml ├── .yardopts ├── CHANGELOG.md ├── DESIGN.md ├── Gemfile ├── LICENSE ├── README.md ├── Rakefile ├── appveyor.yml ├── autotune.jpeg ├── examples ├── mono.yaml ├── mono_04_08.yaml ├── mono_04_08_cm.yaml ├── mono_08_16.yaml ├── mono_08_16_cm.yaml ├── mono_16_32.yaml ├── mono_16_32_cm.yaml ├── mono_cms_xl.yaml ├── mono_external.yaml ├── mono_ha.yaml ├── mono_ha_cms.yaml ├── mono_ha_cms_xl.yaml ├── split.yaml ├── split_external.yaml └── split_external_cms.yaml ├── fixtures ├── mono.yaml ├── no_nodes.yaml └── syntax_error.yaml ├── lib ├── puppet │ ├── application │ │ └── pe.rb │ └── face │ │ └── pe.rb └── puppet_x │ └── puppetlabs │ ├── tune.rb │ └── tune │ ├── calculate.rb │ ├── cli.rb │ ├── inventory.rb │ ├── peconf.rb │ └── query.rb ├── metadata.json ├── spec ├── README.md ├── acceptance │ ├── nodesets │ │ └── default.yml │ └── run_spec.rb ├── default_facts.yml ├── spec_helper.rb ├── spec_helper_acceptance.rb └── unit │ └── puppet_x │ └── puppetlabs │ ├── tune │ ├── calculate_spec.rb │ ├── inventory_spec.rb │ └── query_spec.rb │ └── tune_spec.rb └── tasks ├── init.json └── init.rb /.fixtures.yml: -------------------------------------------------------------------------------- 1 | # This file can be used to install module dependencies for unit testing 2 | # See https://github.com/puppetlabs/puppetlabs_spec_helper#using-fixtures for details 3 | --- 4 | fixtures: 5 | forge_modules: 6 | # stdlib: "puppetlabs/stdlib" 7 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.rb eol=lf 2 | *.erb eol=lf 3 | *.pp eol=lf 4 | *.sh eol=lf 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .git/ 2 | .*.sw[op] 3 | .metadata 4 | .yardoc 5 | .yardwarns 6 | *.iml 7 | /.bundle/ 8 | /.idea/ 9 | /.vagrant/ 10 | /coverage/ 11 | /bin/ 12 | /doc/ 13 | /Gemfile.local 14 | /Gemfile.lock 15 | /junit/ 16 | /log/ 17 | /pkg/ 18 | /spec/fixtures/manifests/ 19 | /spec/fixtures/modules/ 20 | /tmp/ 21 | /vendor/ 22 | /convert_report.txt 23 | /update_report.txt 24 | .DS_Store 25 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | --- 2 | stages: 3 | - syntax 4 | - unit 5 | 6 | cache: 7 | paths: 8 | - vendor/bundle 9 | 10 | before_script: 11 | - bundle -v 12 | - rm Gemfile.lock || true 13 | - gem update --system 14 | - gem --version 15 | - bundle -v 16 | - bundle install --without system_tests --path vendor/bundle --jobs $(nproc) 17 | 18 | parallel_spec-Ruby 2.1.9-Puppet ~> 4.0: 19 | stage: unit 20 | image: ruby:2.1.9 21 | script: 22 | - bundle exec rake parallel_spec 23 | variables: 24 | PUPPET_GEM_VERSION: '~> 4.0' 25 | 26 | syntax lint metadata_lint check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop-Ruby 2.4.4-Puppet ~> 5.5: 27 | stage: syntax 28 | image: ruby:2.4.4 29 | script: 30 | - bundle exec rake syntax lint metadata_lint check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop 31 | variables: 32 | PUPPET_GEM_VERSION: '~> 5.5' 33 | 34 | parallel_spec-Ruby 2.4.4-Puppet ~> 5.5: 35 | stage: unit 36 | image: ruby:2.4.4 37 | script: 38 | - bundle exec rake parallel_spec 39 | variables: 40 | PUPPET_GEM_VERSION: '~> 5.5' 41 | 42 | -------------------------------------------------------------------------------- /.pdkignore: -------------------------------------------------------------------------------- 1 | .git/ 2 | .*.sw[op] 3 | .metadata 4 | .yardoc 5 | .yardwarns 6 | *.iml 7 | /.bundle/ 8 | /.idea/ 9 | /.vagrant/ 10 | /coverage/ 11 | /bin/ 12 | /doc/ 13 | /Gemfile.local 14 | /Gemfile.lock 15 | /junit/ 16 | /log/ 17 | /pkg/ 18 | /spec/fixtures/manifests/ 19 | /spec/fixtures/modules/ 20 | /tmp/ 21 | /vendor/ 22 | /convert_report.txt 23 | /update_report.txt 24 | .DS_Store 25 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | tkishel-pe_tune 4 | 5 | 6 | 7 | 8 | 9 | com.puppetlabs.geppetto.pp.dsl.ui.modulefileBuilder 10 | 11 | 12 | 13 | 14 | org.eclipse.xtext.ui.shared.xtextBuilder 15 | 16 | 17 | 18 | 19 | 20 | com.puppetlabs.geppetto.pp.dsl.ui.puppetNature 21 | org.eclipse.xtext.ui.shared.xtextNature 22 | 23 | 24 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | --format documentation 3 | 4 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | --- 2 | require: rubocop-rspec 3 | AllCops: 4 | DisplayCopNames: true 5 | TargetRubyVersion: '2.1' 6 | Include: 7 | - "./**/*.rb" 8 | Exclude: 9 | - bin/* 10 | - ".vendor/**/*" 11 | - "**/Gemfile" 12 | - "**/Rakefile" 13 | - pkg/**/* 14 | - spec/fixtures/**/* 15 | - vendor/**/* 16 | - "**/Puppetfile" 17 | - "**/Vagrantfile" 18 | - "**/Guardfile" 19 | Metrics/LineLength: 20 | Description: People have wide screens, use them. 21 | Max: 200 22 | #GetText/DecorateString: 23 | # Description: We don't want to decorate test output. 24 | # Exclude: 25 | # - spec/* 26 | RSpec/BeforeAfterAll: 27 | Description: Beware of using after(:all) as it may cause state to leak between tests. 28 | A necessary evil in acceptance testing. 29 | Exclude: 30 | - spec/acceptance/**/*.rb 31 | RSpec/HookArgument: 32 | Description: Prefer explicit :each argument, matching existing module's style 33 | EnforcedStyle: each 34 | Style/BlockDelimiters: 35 | Description: Prefer braces for chaining. Mostly an aesthetical choice. Better to 36 | be consistent then. 37 | EnforcedStyle: braces_for_chaining 38 | Style/ClassAndModuleChildren: 39 | Description: Compact style reduces the required amount of indentation. 40 | EnforcedStyle: compact 41 | Style/EmptyElse: 42 | Description: Enforce against empty else clauses, but allow `nil` for clarity. 43 | EnforcedStyle: empty 44 | Style/FormatString: 45 | Description: Following the main puppet project's style, prefer the % format format. 46 | EnforcedStyle: percent 47 | Style/FormatStringToken: 48 | Description: Following the main puppet project's style, prefer the simpler template 49 | tokens over annotated ones. 50 | EnforcedStyle: template 51 | Style/Lambda: 52 | Description: Prefer the keyword for easier discoverability. 53 | EnforcedStyle: literal 54 | Style/RegexpLiteral: 55 | Description: Community preference. See https://github.com/voxpupuli/modulesync_config/issues/168 56 | EnforcedStyle: percent_r 57 | Style/TernaryParentheses: 58 | Description: Checks for use of parentheses around ternary conditions. Enforce parentheses 59 | on complex expressions for better readability, but seriously consider breaking 60 | it up. 61 | EnforcedStyle: require_parentheses_when_complex 62 | Style/TrailingCommaInArguments: 63 | Description: Prefer always trailing comma on multiline argument lists. This makes 64 | diffs, and re-ordering nicer. 65 | EnforcedStyleForMultiline: comma 66 | Style/TrailingCommaInLiteral: 67 | Description: Prefer always trailing comma on multiline literals. This makes diffs, 68 | and re-ordering nicer. 69 | EnforcedStyleForMultiline: comma 70 | Style/SymbolArray: 71 | Description: Using percent style obscures symbolic intent of array's contents. 72 | EnforcedStyle: brackets 73 | RSpec/MessageSpies: 74 | EnforcedStyle: receive 75 | Style/Documentation: 76 | Exclude: 77 | - lib/puppet/parser/functions/**/* 78 | - spec/**/* 79 | Style/WordArray: 80 | EnforcedStyle: brackets 81 | Style/CollectionMethods: 82 | Enabled: true 83 | Style/MethodCalledOnDoEndBlock: 84 | Enabled: true 85 | Style/StringMethods: 86 | Enabled: true 87 | Layout/EndOfLine: 88 | Enabled: false 89 | Layout/IndentHeredoc: 90 | Enabled: false 91 | Metrics/AbcSize: 92 | Enabled: false 93 | Metrics/BlockLength: 94 | Enabled: false 95 | Metrics/ClassLength: 96 | Enabled: false 97 | Metrics/CyclomaticComplexity: 98 | Enabled: false 99 | Metrics/MethodLength: 100 | Enabled: false 101 | Metrics/ModuleLength: 102 | Enabled: false 103 | Metrics/ParameterLists: 104 | Enabled: false 105 | Metrics/PerceivedComplexity: 106 | Enabled: false 107 | RSpec/DescribeClass: 108 | Enabled: false 109 | RSpec/ExampleLength: 110 | Enabled: false 111 | RSpec/MessageExpectation: 112 | Enabled: false 113 | RSpec/MultipleExpectations: 114 | Enabled: false 115 | RSpec/NestedGroups: 116 | Enabled: false 117 | Style/AsciiComments: 118 | Enabled: false 119 | Style/IfUnlessModifier: 120 | Enabled: false 121 | Style/SymbolProc: 122 | Enabled: false 123 | # TJK Overridden 124 | Metrics/LineLength: 125 | Max: 255 126 | Style/ClassAndModuleChildren: 127 | Enabled: false 128 | Style/TrailingCommaInLiteral: 129 | Enabled: false 130 | Style/WordArray: 131 | Enabled: false 132 | # TJK New 133 | Layout/AlignArray: 134 | Enabled: false 135 | Layout/ExtraSpacing: 136 | Enabled: false 137 | Layout/MultilineArrayBraceLayout: 138 | Enabled: false 139 | RSpec/FilePath: 140 | Enabled: false 141 | Style/ClassAndModuleCamelCase: 142 | Enabled: false 143 | Style/ColonMethodCall: 144 | Enabled: false 145 | Style/GuardClause: 146 | Enabled: false 147 | Style/HashSyntax: 148 | Enabled: false 149 | Style/NumericLiterals: 150 | Enabled: false 151 | Style/StringLiterals: 152 | Enabled: false 153 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dist: trusty 3 | language: ruby 4 | cache: bundler 5 | before_install: 6 | - bundle -v 7 | - rm -f Gemfile.lock 8 | - gem update --system $RUBYGEMS_VERSION 9 | - gem --version 10 | - bundle -v 11 | script: 12 | - 'bundle exec rake $CHECK' 13 | bundler_args: --without system_tests 14 | rvm: 15 | - 2.5.3 16 | stages: 17 | - static 18 | - spec 19 | - acceptance 20 | - 21 | if: tag =~ ^v\d 22 | name: deploy 23 | matrix: 24 | fast_finish: true 25 | include: 26 | - 27 | env: CHECK="check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop syntax lint metadata_lint" 28 | stage: static 29 | - 30 | env: PUPPET_GEM_VERSION="~> 5.0" CHECK=parallel_spec 31 | rvm: 2.4.5 32 | stage: spec 33 | - 34 | env: PUPPET_GEM_VERSION="~> 6.0" CHECK=parallel_spec 35 | rvm: 2.5.3 36 | stage: spec 37 | - 38 | env: PUPPET_GEM_VERSION="~> 6.9" CHECK=parallel_spec 39 | rvm: 2.5.6 40 | stage: spec 41 | - 42 | env: DEPLOY_TO_FORGE=yes 43 | stage: deploy 44 | branches: 45 | only: 46 | - master 47 | - /^v\d/ 48 | notifications: 49 | email: false 50 | deploy: 51 | provider: puppetforge 52 | user: puppet 53 | password: 54 | secure: "" 55 | on: 56 | tags: true 57 | all_branches: true 58 | condition: "$DEPLOY_TO_FORGE = yes" 59 | -------------------------------------------------------------------------------- /.yardopts: -------------------------------------------------------------------------------- 1 | --markup markdown 2 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | # 3.1.2 6 | 7 | - Improve debug data 8 | 9 | # 3.1.1 10 | 11 | - Manually add the Orchestrator class to the Replica class list 12 | 13 | # 3.1.0 14 | 15 | - Accommodates `pe_compilers` 16 | - Increases the amount of unallocated memory from 25% to 20% 17 | - Decreases the amount of memory allocated to PostgreSQL `shared_buffers` from 25% to 20% 18 | 19 | # 3.0.2 20 | 21 | - Define an intuitive output order 22 | - Improve documentation 23 | 24 | # 3.0.1 25 | 26 | - Convert puppetserver from percent to remainder of unallocated processors 27 | - Remove database settings not based upon resources 28 | - Document formulas used 29 | - Output common data as data 30 | - Simplify spec test math 31 | - Drop percent_clamp 32 | 33 | # 3.0.0 34 | 35 | - Convert calculations to percentages 36 | 37 | # 2.4.2 38 | 39 | - Improve debugging output 40 | - Replace the concept of PE XL with PE Compilers 41 | - Set log_temp_files equal to work_mem to minimize PostgreSQL logging 42 | - Remove UseG1G. 43 | 44 | # 2.4.1 45 | 46 | - Do not output settings to stdout when outputting to files. 47 | - Disable puppetdb garbage collection on compile masters. 48 | - Add UseG1G. 49 | - Support 2 CPU / 6 GB systems. 50 | 51 | # 2.4.0 52 | 53 | - Add flag to write to pe.conf. 54 | - Discard STDERR when setting variables from command line commands. 55 | - Tune more PostgreSQL settings. 56 | - Avoid PuppetDB queries on older versions of PE. 57 | 58 | # 2.3.0 59 | 60 | - Implement compare option. 61 | 62 | # 2.2.1 63 | 64 | - Output details for duplicates. 65 | 66 | # 2.2.0 67 | 68 | - Add use_current_memory_per_jruby option. 69 | - Buffer collection of current settings. 70 | 71 | # 2.1.0 72 | 73 | - Rename face. 74 | 75 | # 2.0.2 76 | 77 | - Update face documentation. 78 | 79 | # 2.0.1 80 | 81 | - Rename task. 82 | - Note that the name of this module (and its face and task) are in flux. 83 | 84 | # 2.0.0 85 | 86 | - Implement as a module with a face and a task. 87 | 88 | # 1.7.0 89 | 90 | - Refactor use of PuppetDB 91 | 92 | # 1.6.1 93 | 94 | - Move cli code to separate file. 95 | - Rename configuration class to query. 96 | - Handle exceptions when PuppetDB is unavailable. 97 | - Add spec tests. 98 | 99 | # 1.6.0 100 | 101 | - Improve estimate output and translation test. 102 | 103 | # 1.5.0 104 | 105 | - Implement extra large reference architecture. 106 | 107 | # 1.3.0 108 | 109 | - Add translation method 110 | - Limit command_processing_threads 111 | - Move inventory into its own class 112 | 113 | # 1.2.3 114 | 115 | - Refactor calculate methods. 116 | 117 | ## 1.2.1 118 | 119 | - Remove dependency on pe.conf. 120 | - Refactor roles, profiles, and components mappings. 121 | 122 | ## 1.2.0 123 | 124 | - Allow use of the local system or an inventory file instead of PuppetDB. 125 | 126 | ## 1.1.1 127 | 128 | - Update to PDK 1.7.0 129 | -------------------------------------------------------------------------------- /DESIGN.md: -------------------------------------------------------------------------------- 1 | # pe_tune 2 | 3 | ## Background 4 | 5 | The default settings for Puppet Enterprise services are tuned, but not necessarily optimized for PE Infrastructure type and the combination of PE services competing for system resources on each PE Infrastructure host. 6 | 7 | The `tune` command outputs optimized settings for Puppet Enterprise services based upon available system resources. 8 | 9 | The command expects that you have provisioned the PE Infrastructure hosts with the system resources required to handle the workload, given agent count and code and environment complexity. 10 | 11 | ## Methodology 12 | 13 | 1. Query PuppetDB for PE Infrastructure hosts (query for declared PE classes) 14 | 1. Identify PE Infrastructure type: Standard, Large, Extra Large (legacy: Split) 15 | 1. Query PuppetDB for CPU and RAM facts for each PE Infrastructure host (query for processors, memory) 16 | 1. Output settings for PE services for each PE Infrastructure host (as parameters for the declared PE classes) 17 | 18 | ## Resource Allocation 19 | 20 | ### Ratios, Minimums, and Maximums 21 | 22 | The `tune` command calculates settings for each service based upon a ratio of system resources (processors and/or memory) limited by a minimum and maximum. 23 | 24 | The ratio, minimum, and maximum vary based upon the PE Infrastructure type and the combination of PE services competing for system resources on each PE Infrastructure host. 25 | 26 | The minimum system resources for the `tune` command to function are 4 CPU / 8 GB RAM. 27 | 28 | Notes: 29 | 30 | > The following CPU values are percentages, and RAM values are megabytes. 31 | 32 | > Any Replica should/would/will receive the same settings as the Primary Master, as a Replica is required to have the same system resources as the Primary Master. 33 | 34 | 35 | #### Standard Reference Architecture 36 | 37 | A Standard Reference Architecture is a Master-only install. 38 | 39 | ##### Master 40 | 41 | Allocations are calculated in the following order. 42 | 43 | ###### Database Service (pe-postgresql) 44 | 45 | ``` 46 | CPU No Allocation 47 | ``` 48 | 49 | ``` 50 | RAM Percent = 0.25 51 | RAM Minimum = 2048 52 | RAM Maximum = 16384 53 | ``` 54 | 55 | If the total number of potential database connections from all PuppetDB services exceeds the default, we increase `max_connections` by the number of potential database connections times `1.10`. 56 | 57 | ###### PuppetDB Service (pe-puppetdb) 58 | 59 | ``` 60 | CPU Percent = 0.25 61 | CPU Minimum = 1 62 | ``` 63 | 64 | ``` 65 | RAM Percent = 0.10 66 | RAM Minimum = 512 67 | RAM Maximum = 8192 68 | ``` 69 | 70 | ###### Console Service (pe-console-services) 71 | 72 | ``` 73 | CPU No Allocation 74 | ``` 75 | 76 | ``` 77 | RAM Percent = 0.08 78 | RAM Minimum = 512 79 | RAM Maximum = 1024 80 | ``` 81 | 82 | ###### Orchestrator Service (pe-orchestration-services) 83 | 84 | ``` 85 | CPU No Allocation 86 | ``` 87 | 88 | ``` 89 | RAM Percent = 0.08 90 | RAM Minimum = 512 91 | RAM Maximum = 1024 92 | ``` 93 | 94 | In PE 2019.2.x, Orchestrator has JRubies and is allocated additional memory as follows. 95 | 96 | 97 | ``` 98 | RAM Percent = 0.10 99 | RAM Maximum = N/A 100 | ``` 101 | 102 | Orchestrator JRubies do not require a CPU allocation as the are bound by I/O. 103 | But we limit the number of Orchestrator JRubies based upon how many fit into the memory allocated to Orchestrator. 104 | 105 | ``` 106 | minimum jrubies orchestrator = 1 107 | maximum jrubies orchestrator = 4 108 | maximum jrubies orchestrator limited by memory = (allocated memory / memory per jruby) 109 | orchestrator_jruby_max_active_instances = (maximum jrubies orchestrator limited by memory).clamp(minimum jrubies orchestrator, maximum jrubies puppetserver orchestrator) 110 | ``` 111 | 112 | 113 | ###### ActiveMQ Service (pe-activemq) * 114 | 115 | ``` 116 | CPU No Allocation 117 | ``` 118 | 119 | ``` 120 | RAM Percent = 0.08 121 | RAM Minimum = 512 122 | RAM Maximum = 1024 123 | ``` 124 | 125 | \* ActiveMQ (used by MCollective) is deprecated in PE 2018.x and removed in PE 2019.x. 126 | 127 | ###### Puppet Server Service (pe-puppetserver) 128 | 129 | Since PuppetServer is allocated the remainder of system resources, it does not have explicit ratios of CPU or RAM, or a maximum of RAM. 130 | 131 | ``` 132 | CPU Percent = N/A 133 | CPU Minimum = 2 134 | CPU Maximum = 24 135 | ``` 136 | 137 | Since ReservedCodeCache is limited to a maximum of 2 GB, and each JRuby requires an estimated 96 MB of ReservedCodeCache, the maximum number of JRubies is effectively limited to 24. 138 | 139 | ``` 140 | RAM Percent Heap = N/A 141 | RAM Minimum Heap = 512 142 | ``` 143 | 144 | ``` 145 | RAM Percent Reserved Code Cache = N/A 146 | RAM Minimum Reserved Code Cache = 128 147 | RAM Maximum Reserved Code Cache = 2048 148 | ``` 149 | 150 | ``` 151 | RAM Heap Per JRuby = (512, 768, 1024) when RAM equals (4-7 GB, 8-16 GB, 16 GB+) 152 | RAM Reserved Code Cache Per JRuby = 96 153 | ``` 154 | 155 | Puppet Server JRubies are constrained based on both how many JRubies fit into unallocated memory and unallocated processors (one JRuby per processor). 156 | Puppet Server memory is then set to the amount of memory required for the total calculated number of JRubies. 157 | 158 | ``` 159 | minimum jrubies puppetserver = 2 160 | maximum jrubies puppetserver = 24 161 | maximum jrubies puppetserver limited by processors = (available processors).clamp(minimum jrubies puppetserver, maximum jrubies puppetserver) 162 | maximum jrubies puppetserver limited by memory = (available memory / (memory per jruby + memory per jruby reserved code cache)) 163 | puppetserver_jruby_max_active_instances = (maximum jrubies puppetserver limited by memory).clamp(minimum jrubies puppetserver, maximum jrubies puppetserver limited by processors) 164 | ``` 165 | 166 | ###### Operating System and Other Services 167 | 168 | ``` 169 | CPU Reserved = 1 170 | ``` 171 | 172 | ``` 173 | RAM Reserved Percentage = 0.20 174 | ``` 175 | 176 | 177 | #### Large Reference Architecture 178 | 179 | A Large Reference Architecture is a Master plus Compilers. 180 | 181 | ##### Master 182 | 183 | Calculations for the Master in a Large Reference Architecture use the same algorithms used for the [Standard Reference Architecture Master](#Standard-Master) with the following exceptions: 184 | 185 | ``` 186 | PuppetDB CPU Percent = 0.50 # up from 0.25 187 | ``` 188 | 189 | ``` 190 | PuppetDB RAM Percent = 0.15 # up from 0.10 191 | ``` 192 | 193 | Rationale: 194 | 195 | Puppet Server on the Master will process catalog requests only for PE Infrastructure hosts. 196 | PuppetDB on the Master is expected to handle requests from the Puppet Server services on multiple Compilers that by definition serve more agents than the Standard Reference Architecture. 197 | 198 | ##### Compilers 199 | 200 | Calculations for Compilers in a Large Reference Architecture use the same algorithms used for the [Standard Reference Architecture Master](#Standard-Master) with the following exceptions. 201 | 202 | ``` 203 | PuppetDB CPU Maximum = 3 204 | ``` 205 | 206 | Compilers in a Large Reference Architecture include a local PuppetDB service. 207 | The local PuppetDB service connects to the same PostgreSQL service as the PuppetDB service on the Master. 208 | We lower the local PuppetDB allocation of CPU to enforce a limited number of connections to PostgreSQL, preventing an overallocation of connections to PostgreSQL. 209 | In addition, we disable the local PuppetDB service garbage collection, as garbage collection is already performed by the PuppetDB service on the Master. 210 | 211 | 212 | #### Extra Large Reference Architecture 213 | 214 | An Extra Large Reference Architecture is a Master plus Compilers with PuppetDB and PostgreSQL services on a PuppetDB host. 215 | 216 | ##### Master 217 | 218 | Calculations for the Master in an Extra Large Reference Architecture use the same algorithms used for the [Large Reference Architecture Master](#Large-Master) 219 | 220 | ##### Compilers 221 | 222 | Calculations for Compilers in an Extra Large Reference Architecture use the same algorithms used for the [Large Reference Architecture Compilers](#Large-Compilers) 223 | 224 | ##### PuppetDB Host 225 | 226 | Calculations for the PuppetDB Host use the same algorithms used for the [Standard Reference Architecture Master](#Standard-Master). 227 | 228 | The below are the same settings for these two services as would be seen on a Standard Reference Architecture Master. 229 | 230 | ###### Database Service (pe-postgresql) 231 | 232 | Same as [Standard Reference Architecture Database Service (pe-postgresql)](#Standard-Database) 233 | 234 | ###### PuppetDB Service (pe-puppetdb) 235 | 236 | Same as [Standard Reference Architecture PuppetDB Service (pe-puppetdb)](#Standard-PuppetDB) 237 | 238 | 239 | #### Legacy Split Architecture 240 | 241 | ##### Master 242 | 243 | Calculations for a Split Master use the same algorithms used for the [Standard Reference Architecture Master](#Standard-Master) minus allocations for the services moved to the other hosts. 244 | 245 | ##### Console Host 246 | 247 | ###### Console Service (pe-console-services) 248 | 249 | ``` 250 | CPU No Allocation 251 | ``` 252 | 253 | ``` 254 | RAM Percent = 0.75 255 | RAM Minimum = 512 256 | RAM Maximum = 4096 257 | ``` 258 | 259 | ##### Database Host 260 | 261 | ###### Database Service (pe-postgresql) 262 | 263 | ``` 264 | CPU No Allocation 265 | ``` 266 | 267 | ``` 268 | RAM Percent = 0.25 269 | RAM Minimum = 2048 270 | RAM Maximum = 16384 271 | ``` 272 | 273 | ###### PuppetDB Service (pe-puppetdb) 274 | 275 | ``` 276 | CPU Percent = 0.50 277 | CPU Minimum = 1 278 | ``` 279 | 280 | ``` 281 | RAM Percent = 0.25 282 | RAM Minimum = 512 283 | RAM Maximum = 8192 284 | ``` 285 | 286 | If PostgreSQL is moved to an External PostgreSQL Host the following change: 287 | 288 | ``` 289 | PuppetDB RAM Percent = 0.50 290 | ``` 291 | 292 | ##### External PostgreSQL Host 293 | 294 | ###### Database Service (pe-postgresql) 295 | 296 | ``` 297 | RAM Percent = 0.25 298 | RAM Minimum = 2048 299 | RAM Maximum = 16384 300 | ``` 301 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source ENV['GEM_SOURCE'] || 'https://rubygems.org' 2 | 3 | def location_for(place_or_version, fake_version = nil) 4 | git_url_regex = %r{\A(?(https?|git)[:@][^#]*)(#(?.*))?} 5 | file_url_regex = %r{\Afile:\/\/(?.*)} 6 | 7 | if place_or_version && (git_url = place_or_version.match(git_url_regex)) 8 | [fake_version, { git: git_url[:url], branch: git_url[:branch], require: false }].compact 9 | elsif place_or_version && (file_url = place_or_version.match(file_url_regex)) 10 | ['>= 0', { path: File.expand_path(file_url[:path]), require: false }] 11 | else 12 | [place_or_version, { require: false }] 13 | end 14 | end 15 | 16 | ruby_version_segments = Gem::Version.new(RUBY_VERSION.dup).segments 17 | minor_version = ruby_version_segments[0..1].join('.') 18 | 19 | group :development do 20 | gem "fast_gettext", '1.1.0', require: false if Gem::Version.new(RUBY_VERSION.dup) < Gem::Version.new('2.1.0') 21 | gem "fast_gettext", require: false if Gem::Version.new(RUBY_VERSION.dup) >= Gem::Version.new('2.1.0') 22 | gem "json_pure", '<= 2.0.1', require: false if Gem::Version.new(RUBY_VERSION.dup) < Gem::Version.new('2.0.0') 23 | gem "json", '= 1.8.1', require: false if Gem::Version.new(RUBY_VERSION.dup) == Gem::Version.new('2.1.9') 24 | gem "json", '<= 2.0.4', require: false if Gem::Version.new(RUBY_VERSION.dup) == Gem::Version.new('2.4.4') 25 | gem "puppet-module-posix-default-r#{minor_version}", require: false, platforms: [:ruby] 26 | gem "puppet-module-posix-dev-r#{minor_version}", require: false, platforms: [:ruby] 27 | gem "puppet-module-win-default-r#{minor_version}", require: false, platforms: [:mswin, :mingw, :x64_mingw] 28 | gem "puppet-module-win-dev-r#{minor_version}", require: false, platforms: [:mswin, :mingw, :x64_mingw] 29 | end 30 | 31 | group :system_tests do 32 | gem "puppet-module-posix-system-r#{minor_version}", require: false, platforms: [:ruby] 33 | gem "beaker", *location_for(ENV['BEAKER_VERSION'] || '~> 4.4') 34 | gem "beaker-abs", *location_for(ENV['BEAKER_ABS_VERSION'] || '~> 0.1') 35 | gem "beaker-pe", require: false 36 | gem "beaker-hostgenerator" 37 | gem "beaker-rspec" 38 | end 39 | 40 | puppet_version = ENV['PUPPET_GEM_VERSION'] 41 | facter_version = ENV['FACTER_GEM_VERSION'] 42 | hiera_version = ENV['HIERA_GEM_VERSION'] 43 | 44 | gems = {} 45 | 46 | gems['puppet'] = location_for(puppet_version) 47 | 48 | # If facter or hiera versions have been specified via the environment 49 | # variables 50 | 51 | gems['facter'] = location_for(facter_version) if facter_version 52 | gems['hiera'] = location_for(hiera_version) if hiera_version 53 | 54 | if Gem.win_platform? && puppet_version =~ %r{^(file:///|git://)} 55 | # If we're using a Puppet gem on Windows which handles its own win32-xxx gem 56 | # dependencies (>= 3.5.0), set the maximum versions (see PUP-6445). 57 | gems['win32-dir'] = ['<= 0.4.9', require: false] 58 | gems['win32-eventlog'] = ['<= 0.6.5', require: false] 59 | gems['win32-process'] = ['<= 0.7.5', require: false] 60 | gems['win32-security'] = ['<= 0.2.5', require: false] 61 | gems['win32-service'] = ['0.8.8', require: false] 62 | end 63 | 64 | gems.each do |gem_name, gem_params| 65 | gem gem_name, *gem_params 66 | end 67 | 68 | # Evaluate Gemfile.local and ~/.gemfile if they exist 69 | extra_gemfiles = [ 70 | "#{__FILE__}.local", 71 | File.join(Dir.home, '.gemfile'), 72 | ] 73 | 74 | extra_gemfiles.each do |gemfile| 75 | if File.file?(gemfile) && File.readable?(gemfile) 76 | eval(File.read(gemfile), binding) 77 | end 78 | end 79 | # vim: syntax=ruby 80 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2018 Thomas Kishel 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pe_tune 2 | 3 | #### Table of Contents 4 | 5 | 1. [Description - What the module does and why it is useful](#description) 6 | 1. [Setup - Getting started with this module](#setup) 7 | 1. [Usage - Command parameters and how to use them](#usage) 8 | 1. [Reference - How the module works and how to use its output](#reference) 9 | 1. [Limitations - Supported infrastructures and versions](#limitations) 10 | 11 | ## Description 12 | 13 | > The fault, dear Brutus, is not in our stars, but in our defaults, that we are under-allocating system resources. 14 | 15 | The default settings for Puppet Enterprise services are tuned, but not necessarily optimized for PE Infrastructure type and the combination of PE services competing for system resources on each PE Infrastructure host. 16 | 17 | This module provides a Puppet command `puppet pe tune` that outputs optimized settings for Puppet Enterprise services based upon available system resources. 18 | This command expects that you have provisioned the PE Infrastructure hosts with the system resources required to handle the workload, given agent count and code and environment complexity. 19 | 20 | Puppet Enterprise 2018.1.3 and newer includes this functionality via the `puppet infrastructure tune` command. 21 | To use this command instead of `puppet infrastructure tune` with Puppet Enterprise 2018.1.3 and newer, refer to [Limitations](#limitations). 22 | 23 | ## Setup 24 | 25 | Install this module on the Primary Master. 26 | 27 | For example, to install the latest version: 28 | 29 | ```shell 30 | git clone --depth=1 --branch=master https://github.com/tkishel/pe_tune.git /etc/puppetlabs/code/modules/pe_tune 31 | ``` 32 | 33 | Or to install the latest release: 34 | 35 | ```shell 36 | wget -q -O - https://api.github.com/repos/tkishel/pe_tune/releases/latest | grep -oP '"tarball_url": "\K(.*)(?=")' | wget -q -i - -O - | tar -xzf - && mv tkishel-pe_tune* /etc/puppetlabs/code/modules/pe_tune 37 | ``` 38 | 39 | ## Usage 40 | 41 | 1. Run the `puppet pe tune` command as root on the Primary Master. 42 | 1. Verify the optimized settings. 43 | 1. Add the optimized settings to Hiera. 44 | 1. Remove any duplicate settings from the Console. 45 | 1. Run `puppet agent -t` on each PE Infrastructure host to apply the optimized settings. 46 | 47 | #### Parameters 48 | 49 | ##### `--common` 50 | 51 | Extract common settings from node-specific settings when outputting optimized settings. 52 | 53 | A common setting is one with a value that is identical on multiple hosts. 54 | This option extracts and outputs common settings separately from node-specific settings, potentially reducing the number of node-specific settings. 55 | 56 | ##### `--compare` 57 | 58 | Output a comparison of currently-defined and optimized settings. 59 | 60 | ##### `--current` 61 | 62 | Output currently-defined settings, in JSON format. 63 | 64 | Settings may be defined either in the Classifier (the Console) or in Hiera, with Classifier settings taking precedence over Hiera settings. 65 | This option also identifies duplicate settings defined in both the Classifier and Hiera. 66 | Best practice is to define settings in Hiera (preferred) or the Classifier, but not both. 67 | 68 | ##### `--debug` 69 | 70 | Enable logging of debug information. 71 | 72 | ##### `--hiera DIRECTORY` 73 | 74 | Output optimized settings to the specified directory, as YAML files for use in Hiera. 75 | 76 | > Do not specify a directory in your Hiera hierarchy if that directory is managed by Code Manager. Instead: specify a temporary directory, verify the settings in resulting files, and merge them into the control repository that contains your Hiera data. 77 | 78 | ##### `--force` 79 | 80 | Do not enforce minimum system requirements (4 CPU / 8 GB RAM) for PE Infrastructure hosts. 81 | 82 | ##### `--inventory FILE` 83 | 84 | Use the specified YAML file to define PE Infrastructure hosts. 85 | 86 | This eliminates a dependency upon PuppetDB to query facts and classes for PE Infrastructure hosts. 87 | Refer to the [examples](examples) directory of this module for details. 88 | 89 | ##### `--local` 90 | 91 | Use the local system to define a monolithic master host. 92 | 93 | This eliminates a dependency upon PuppetDB to query facts and classes for PE Infrastructure hosts, and is only useful after a clean install of a monolithic master host. 94 | 95 | ##### `--memory_per_jruby MB` 96 | 97 | Amount of RAM to allocate for each Puppet Server JRuby. 98 | 99 | ##### `--memory_reserved_for_os MB` 100 | 101 | Amount of RAM to reserve for the operating system and other services. 102 | 103 | ##### `--node CERTNAME` 104 | 105 | Limit output to a single PE Infrastructure host. 106 | 107 | ##### `--use_current_memory_per_jruby` 108 | 109 | Use currently-defined settings to determine `memory_per_jruby`. 110 | 111 | ## Reference 112 | 113 | This command outputs optimized settings for PE services as follows. 114 | 115 | 1. Query PuppetDB for PE Infrastructure hosts (query for declared PE classes) 116 | 1. Identify PE Infrastructure type: Standard, Large, Extra Large (legacy: Split) 117 | 1. Query PuppetDB for CPU and RAM facts for each PE Infrastructure host (query for processors, memory) 118 | 1. Output settings for PE services for each PE Infrastructure host (as parameters for the declared PE classes) 119 | 120 | ### Output 121 | 122 | By default, settings are output to STDOUT. 123 | 124 | For example: 125 | 126 | ```shell 127 | [root@pe-master ~] puppet pe tune 128 | # Puppet Infrastructure Summary: Found a Monolithic Infrastructure 129 | 130 | # Found 8 CPU(s) / 16384 MB RAM for Primary Master pe-master.puppetdebug.vlan 131 | # Specify the following optimized settings in Hiera in nodes/pe-master.puppetdebug.vlan.yaml 132 | 133 | --- 134 | puppet_enterprise::profile::database::shared_buffers: 4096MB 135 | puppet_enterprise::puppetdb::command_processing_threads: 2 136 | puppet_enterprise::master::puppetserver::jruby_max_active_instances: 5 137 | puppet_enterprise::master::puppetserver::reserved_code_cache: 1024m 138 | puppet_enterprise::profile::master::java_args: 139 | Xms: 3840m 140 | Xmx: 3840m 141 | puppet_enterprise::profile::puppetdb::java_args: 142 | Xms: 1638m 143 | Xmx: 1638m 144 | puppet_enterprise::profile::console::java_args: 145 | Xms: 768m 146 | Xmx: 768m 147 | puppet_enterprise::profile::orchestrator::java_args: 148 | Xms: 768m 149 | Xmx: 768m 150 | 151 | # CPU Summary: Total/Used/Free: 8/7/1 for pe-master.puppetdebug.vlan 152 | # RAM Summary: Total/Used/Free: 16384/12134/4250 for pe-master.puppetdebug.vlan 153 | # JVM Summary: Using 768 MB per Puppet Server JRuby for pe-master.puppetdebug.vlan 154 | ``` 155 | 156 | By default, this command outputs node-specific settings for use in node-specific YAML files in a node-specific Hiera hierarchy. 157 | 158 | For example: 159 | 160 | #### Hiera 3.x 161 | 162 | ```yaml 163 | --- 164 | :hierarchy: 165 | - "nodes/%{trusted.certname}" 166 | - "common" 167 | ``` 168 | 169 | #### Hiera 5.x 170 | 171 | ```yaml 172 | --- 173 | version: 5 174 | hierarchy: 175 | - name: "Per-Node Data" 176 | path: "nodes/%{trusted.certname}.yaml" 177 | - name: "Common values" 178 | path: "common.yaml" 179 | ``` 180 | 181 | ### Reference Links: 182 | 183 | For more information, review: 184 | 185 | * [PE Hardware Requirements](https://puppet.com/docs/pe/latest/installing/hardware_requirements.html) 186 | * [PE Configuration](https://puppet.com/docs/pe/latest/configuring/config_intro.html) 187 | * [PE Java Arguments](https://puppet.com/docs/pe/latest/configuring/config_java_args.html) 188 | * [PE Puppetserver Configuration](https://puppet.com/docs/pe/latest/configuring/config_puppetserver.html) 189 | * [PE Console Configuration](https://puppet.com/docs/pe/latest/configuring/config_console.html) 190 | * [PE PuppetDB Configuration](https://puppet.com/docs/pe/latest/configuring/config_puppetdb.html) 191 | * [PE Tuning Monolithic](https://puppet.com/docs/pe/latest/configuring/tuning_monolithic.html) 192 | * [PE Puppetserver Tuning Guide](https://puppet.com/docs/puppetserver/latest/tuning_guide.html) 193 | * [Hiera](https://puppet.com/docs/puppet/latest/hiera_intro.html) 194 | 195 | ## Limitations 196 | 197 | ### Infrastructure Support 198 | 199 | Support is limited to the following infrastructures: 200 | 201 | * Monolithic Master 202 | * Monolithic Master with HA 203 | * Monolithic Master with Compile Masters 204 | * Monolithic Master with Compile Masters with HA 205 | * Monolithic Master with Compile Masters with External PostgreSQL 206 | * Monolithic Master with External PostgreSQL 207 | * Split Infrastructure 208 | * Split Infrastructure with Compile Masters 209 | * Split Infrastructure with Compile Masters with External PostgreSQL 210 | * Split Infrastructure with External PostgreSQL 211 | 212 | ### Version Support 213 | 214 | Support is limited to the following versions: 215 | 216 | * PE 2016.4.x (\*) 217 | * PE 2017.3.x (\*) 218 | * PE 2018.x.x 219 | * PE 2019.x.x 220 | 221 | \* In these versions, this command is unable to identify PE Database hosts or tune PE PostgreSQL services. 222 | 223 | #### Puppet Enterprise 2018.1.3 and Newer 224 | 225 | This command is the upstream version of the `puppet infrastructure tune` command built into Puppet Enterprise 2018.1.3 and newer. 226 | Installing this module in Puppet Enterprise 2018.1.3 and newer will result in a conflict with the `puppet infrastructure tune` subcommand. 227 | To avoid that conflict, install this module and run this command outside the `modulepath`. 228 | 229 | For example: 230 | 231 | ```shell 232 | mkdir -p /tmp/puppet_modules 233 | (which git > /dev/null 2>&1 && git clone --depth=1 --branch=master --quiet https://github.com/tkishel/pe_tune.git /tmp/puppet_modules/pe_tune) || \ 234 | (wget -q -O - https://api.github.com/repos/tkishel/pe_tune/releases/latest | grep -oP '"tarball_url": "\K(.*)(?=")' | wget -q -i - -O - | tar -xzf - && mv tkishel-pe_tune* /tmp/puppet_modules/pe_tune) 235 | ls -l /tmp/puppet_modules 236 | ``` 237 | 238 | ``` 239 | puppet pe tune --modulepath /tmp/puppet_modules 240 | ``` 241 | 242 | #### Puppet Enterprise 2018.1.2 and Older 243 | 244 | This module may not be able to query PuppetDB in older versions of Puppet Enterprise. 245 | To avoid that error, install this module and run the command outside the `modulepath`. 246 | 247 | ```shell 248 | mkdir -p /tmp/puppet_modules 249 | (which git > /dev/null 2>&1 && git clone --depth=1 --branch=master --quiet https://github.com/tkishel/pe_tune.git /tmp/puppet_modules/pe_tune) || \ 250 | (wget -q -O - https://api.github.com/repos/tkishel/pe_tune/releases/latest | grep -oP '"tarball_url": "\K(.*)(?=")' | wget -q -i - -O - | tar -xzf - && mv tkishel-pe_tune* /tmp/puppet_modules/pe_tune) 251 | ls -l /tmp/puppet_modules 252 | ``` 253 | 254 | ```shell 255 | /tmp/puppet_modules/pe_tune/lib/puppet_x/puppetlabs/tune.rb 256 | ``` 257 | 258 | ![AUTOTUNE](autotune.jpeg?raw=true "AUTOTUNE") 259 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'puppetlabs_spec_helper/rake_tasks' 2 | require 'puppet-syntax/tasks/puppet-syntax' 3 | require 'puppet_blacksmith/rake_tasks' if Bundler.rubygems.find_name('puppet-blacksmith').any? 4 | require 'github_changelog_generator/task' if Bundler.rubygems.find_name('github_changelog_generator').any? 5 | 6 | def changelog_user 7 | return unless Rake.application.top_level_tasks.include? "changelog" 8 | returnVal = nil || JSON.load(File.read('metadata.json'))['author'] 9 | raise "unable to find the changelog_user in .sync.yml, or the author in metadata.json" if returnVal.nil? 10 | puts "GitHubChangelogGenerator user:#{returnVal}" 11 | returnVal 12 | end 13 | 14 | def changelog_project 15 | return unless Rake.application.top_level_tasks.include? "changelog" 16 | returnVal = nil || JSON.load(File.read('metadata.json'))['name'] 17 | raise "unable to find the changelog_project in .sync.yml or the name in metadata.json" if returnVal.nil? 18 | puts "GitHubChangelogGenerator project:#{returnVal}" 19 | returnVal 20 | end 21 | 22 | def changelog_future_release 23 | return unless Rake.application.top_level_tasks.include? "changelog" 24 | returnVal = JSON.load(File.read('metadata.json'))['version'] 25 | raise "unable to find the future_release (version) in metadata.json" if returnVal.nil? 26 | puts "GitHubChangelogGenerator future_release:#{returnVal}" 27 | returnVal 28 | end 29 | 30 | PuppetLint.configuration.send('disable_relative') 31 | 32 | if Bundler.rubygems.find_name('github_changelog_generator').any? 33 | GitHubChangelogGenerator::RakeTask.new :changelog do |config| 34 | raise "Set CHANGELOG_GITHUB_TOKEN environment variable eg 'export CHANGELOG_GITHUB_TOKEN=valid_token_here'" if Rake.application.top_level_tasks.include? "changelog" and ENV['CHANGELOG_GITHUB_TOKEN'].nil? 35 | config.user = "#{changelog_user}" 36 | config.project = "#{changelog_project}" 37 | config.future_release = "#{changelog_future_release}" 38 | config.exclude_labels = ['maintenance'] 39 | config.header = "# Change log\n\nAll notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org)." 40 | config.add_pr_wo_labels = true 41 | config.issues = false 42 | config.merge_prefix = "### UNCATEGORIZED PRS; GO LABEL THEM" 43 | config.configure_sections = { 44 | "Changed" => { 45 | "prefix" => "### Changed", 46 | "labels" => ["backwards-incompatible"], 47 | }, 48 | "Added" => { 49 | "prefix" => "### Added", 50 | "labels" => ["feature", "enhancement"], 51 | }, 52 | "Fixed" => { 53 | "prefix" => "### Fixed", 54 | "labels" => ["bugfix"], 55 | }, 56 | } 57 | end 58 | else 59 | desc 'Generate a Changelog from GitHub' 60 | task :changelog do 61 | raise <= Gem::Version.new('2.2.2')" 72 | EOM 73 | end 74 | end 75 | 76 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: 1.1.x.{build} 3 | skip_commits: 4 | message: /^\(?doc\)?.*/ 5 | clone_depth: 10 6 | init: 7 | - SET 8 | - 'mkdir C:\ProgramData\PuppetLabs\code && exit 0' 9 | - 'mkdir C:\ProgramData\PuppetLabs\facter && exit 0' 10 | - 'mkdir C:\ProgramData\PuppetLabs\hiera && exit 0' 11 | - 'mkdir C:\ProgramData\PuppetLabs\puppet\var && exit 0' 12 | environment: 13 | matrix: 14 | - 15 | RUBY_VERSION: 24-x64 16 | CHECK: syntax lint metadata_lint check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop 17 | - 18 | PUPPET_GEM_VERSION: ~> 4.0 19 | RUBY_VERSION: 21 20 | CHECK: parallel_spec 21 | - 22 | PUPPET_GEM_VERSION: ~> 4.0 23 | RUBY_VERSION: 21-x64 24 | CHECK: parallel_spec 25 | - 26 | PUPPET_GEM_VERSION: ~> 5.0 27 | RUBY_VERSION: 24 28 | CHECK: parallel_spec 29 | - 30 | PUPPET_GEM_VERSION: ~> 5.0 31 | RUBY_VERSION: 24-x64 32 | CHECK: parallel_spec 33 | - 34 | PUPPET_GEM_VERSION: ~> 6.0 35 | RUBY_VERSION: 25 36 | CHECK: parallel_spec 37 | - 38 | PUPPET_GEM_VERSION: ~> 6.0 39 | RUBY_VERSION: 25-x64 40 | CHECK: parallel_spec 41 | matrix: 42 | fast_finish: true 43 | install: 44 | - set PATH=C:\Ruby%RUBY_VERSION%\bin;%PATH% 45 | - bundle install --jobs 4 --retry 2 --without system_tests 46 | - type Gemfile.lock 47 | build: off 48 | test_script: 49 | - bundle exec puppet -V 50 | - ruby -v 51 | - gem -v 52 | - bundle -v 53 | - bundle exec rake %CHECK% 54 | notifications: 55 | - provider: Email 56 | to: 57 | - nobody@nowhere.com 58 | on_build_success: false 59 | on_build_failure: false 60 | on_build_status_changed: false 61 | -------------------------------------------------------------------------------- /autotune.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tkishel/pe_tune/fabd31dd367b6c114773983ce43a580b9025c188/autotune.jpeg -------------------------------------------------------------------------------- /examples/mono.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Use lowercase for all nodes, roles, and components keys and values. 3 | # Use 'g' for GB (the default) and 'm' for MB for ram values. 4 | 5 | nodes: 6 | master.example.com: 7 | resources: 8 | cpu: 4 9 | ram: 8 10 | 11 | # Infrastructure roles include: 12 | # 13 | # puppet_master_host (string) 14 | # primary_master_replica (string) 15 | # console_host (string) 16 | # puppetdb_host (string or an array) 17 | # database_host (string or an array) 18 | # compile_master (string or an array) 19 | 20 | roles: 21 | puppet_master_host: master.example.com 22 | -------------------------------------------------------------------------------- /examples/mono_04_08.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Use lowercase for all nodes, roles, and components keys and values. 3 | # Use 'g' for GB (the default) and 'm' for MB for ram values. 4 | 5 | nodes: 6 | master.example.com: 7 | resources: 8 | cpu: 4 9 | ram: 8 10 | 11 | # Infrastructure roles include: 12 | # 13 | # puppet_master_host (string) 14 | # primary_master_replica (string) 15 | # console_host (string) 16 | # puppetdb_host (string or an array) 17 | # database_host (string or an array) 18 | # compile_master (string or an array) 19 | 20 | roles: 21 | puppet_master_host: master.example.com 22 | -------------------------------------------------------------------------------- /examples/mono_04_08_cm.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Use lowercase for all nodes, roles, and components keys and values. 3 | # Use 'g' for GB (the default) and 'm' for MB for ram values. 4 | 5 | nodes: 6 | master.example.com: 7 | resources: 8 | cpu: 4 9 | ram: 8 10 | compile-master1.example.com: 11 | resources: 12 | cpu: 4 13 | ram: 8 14 | 15 | # Infrastructure roles include: 16 | # 17 | # puppet_master_host (string) 18 | # primary_master_replica (string) 19 | # console_host (string) 20 | # puppetdb_host (string or an array) 21 | # database_host (string or an array) 22 | # compile_master (string or an array) 23 | 24 | roles: 25 | puppet_master_host: master.example.com 26 | compile_master: 27 | - compile-master1.example.com 28 | -------------------------------------------------------------------------------- /examples/mono_08_16.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Use lowercase for all nodes, roles, and components keys and values. 3 | # Use 'g' for GB (the default) and 'm' for MB for ram values. 4 | 5 | nodes: 6 | master.example.com: 7 | resources: 8 | cpu: 8 9 | ram: 16 10 | 11 | # Infrastructure roles include: 12 | # 13 | # puppet_master_host (string) 14 | # primary_master_replica (string) 15 | # console_host (string) 16 | # puppetdb_host (string or an array) 17 | # database_host (string or an array) 18 | # compile_master (string or an array) 19 | 20 | roles: 21 | puppet_master_host: master.example.com 22 | -------------------------------------------------------------------------------- /examples/mono_08_16_cm.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Use lowercase for all nodes, roles, and components keys and values. 3 | # Use 'g' for GB (the default) and 'm' for MB for ram values. 4 | 5 | nodes: 6 | master.example.com: 7 | resources: 8 | cpu: 8 9 | ram: 16 10 | compile-master1.example.com: 11 | resources: 12 | cpu: 4 13 | ram: 8 14 | 15 | # Infrastructure roles include: 16 | # 17 | # puppet_master_host (string) 18 | # primary_master_replica (string) 19 | # console_host (string) 20 | # puppetdb_host (string or an array) 21 | # database_host (string or an array) 22 | # compile_master (string or an array) 23 | 24 | roles: 25 | puppet_master_host: master.example.com 26 | compile_master: 27 | - compile-master1.example.com 28 | -------------------------------------------------------------------------------- /examples/mono_16_32.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Use lowercase for all nodes, roles, and components keys and values. 3 | # Use 'g' for GB (the default) and 'm' for MB for ram values. 4 | 5 | nodes: 6 | master.example.com: 7 | resources: 8 | cpu: 16 9 | ram: 32 10 | 11 | # Infrastructure roles include: 12 | # 13 | # puppet_master_host (string) 14 | # primary_master_replica (string) 15 | # console_host (string) 16 | # puppetdb_host (string or an array) 17 | # database_host (string or an array) 18 | # compile_master (string or an array) 19 | 20 | roles: 21 | puppet_master_host: master.example.com 22 | -------------------------------------------------------------------------------- /examples/mono_16_32_cm.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Use lowercase for all nodes, roles, and components keys and values. 3 | # Use 'g' for GB (the default) and 'm' for MB for ram values. 4 | 5 | nodes: 6 | master.example.com: 7 | resources: 8 | cpu: 16 9 | ram: 32 10 | compile-master1.example.com: 11 | resources: 12 | cpu: 4 13 | ram: 8 14 | # Infrastructure roles include: 15 | # 16 | # puppet_master_host (string) 17 | # primary_master_replica (string) 18 | # console_host (string) 19 | # puppetdb_host (string or an array) 20 | # database_host (string or an array) 21 | # compile_master (string or an array) 22 | 23 | roles: 24 | puppet_master_host: master.example.com 25 | compile_master: 26 | - compile-master1.example.com 27 | -------------------------------------------------------------------------------- /examples/mono_cms_xl.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Use lowercase for all nodes, roles, and components keys and values. 3 | # Use 'g' for GB (the default) and 'm' for MB for ram values. 4 | 5 | nodes: 6 | mono-master.example.com: 7 | resources: 8 | cpu: 16 9 | ram: 32 10 | mono-master-database.example.com: 11 | resources: 12 | cpu: 16 13 | ram: 32 14 | compile-master1.example.com: 15 | resources: 16 | cpu: 16 17 | ram: 32 18 | compile-master2.example.com: 19 | resources: 20 | cpu: 16 21 | ram: 32 22 | 23 | # Infrastructure roles include: 24 | # 25 | # puppet_master_host (string) 26 | # primary_master_replica (string) 27 | # console_host (string) 28 | # puppetdb_host (string or an array) 29 | # database_host (string or an array) 30 | # compile_master (string or an array) 31 | 32 | roles: 33 | puppet_master_host: mono-master.example.com 34 | database_host: mono-master-database.example.com 35 | compile_master: 36 | - compile-master1.example.com 37 | - compile-master2.example.com 38 | -------------------------------------------------------------------------------- /examples/mono_external.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Use lowercase for all nodes, roles, and components keys and values. 3 | # Use 'g' for GB (the default) and 'm' for MB for ram values. 4 | 5 | nodes: 6 | master.example.com: 7 | resources: 8 | cpu: 16 9 | ram: 32 10 | external-database.example.com: 11 | resources: 12 | cpu: 8 13 | ram: 16 14 | 15 | # Infrastructure roles include: 16 | # 17 | # puppet_master_host (string) 18 | # primary_master_replica (string) 19 | # console_host (string) 20 | # puppetdb_host (string or an array) 21 | # database_host (string or an array) 22 | # compile_master (string or an array) 23 | 24 | roles: 25 | puppet_master_host: master.example.com 26 | database_host: external-database.example.com 27 | -------------------------------------------------------------------------------- /examples/mono_ha.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Use lowercase for all nodes, roles, and components keys and values. 3 | # Use 'g' for GB (the default) and 'm' for MB for ram values. 4 | 5 | nodes: 6 | mono-master.example.com: 7 | resources: 8 | cpu: 16 9 | ram: 32 10 | replica-master.example.com: 11 | resources: 12 | cpu: 16 13 | ram: 32 14 | 15 | # Infrastructure roles include: 16 | # 17 | # puppet_master_host (string) 18 | # primary_master_replica (string) 19 | # console_host (string) 20 | # puppetdb_host (string or an array) 21 | # database_host (string or an array) 22 | # compile_master (string or an array) 23 | 24 | roles: 25 | puppet_master_host: mono-master.example.com 26 | primary_master_replica: replica-master.example.com 27 | -------------------------------------------------------------------------------- /examples/mono_ha_cms.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Use lowercase for all nodes, roles, and components keys and values. 3 | # Use 'g' for GB (the default) and 'm' for MB for ram values. 4 | 5 | nodes: 6 | mono-master.example.com: 7 | resources: 8 | cpu: 16 9 | ram: 32 10 | replica-master.example.com: 11 | resources: 12 | cpu: 16 13 | ram: 32 14 | compile-master1.example.com: 15 | resources: 16 | cpu: 4 17 | ram: 8 18 | compile-master2.example.com: 19 | resources: 20 | cpu: 4 21 | ram: 8 22 | 23 | # Infrastructure roles include: 24 | # 25 | # puppet_master_host (string) 26 | # primary_master_replica (string) 27 | # console_host (string) 28 | # puppetdb_host (string or an array) 29 | # database_host (string or an array) 30 | # compile_master (string or an array) 31 | 32 | roles: 33 | puppet_master_host: mono-master.example.com 34 | primary_master_replica: replica-master.example.com 35 | compile_master: 36 | - compile-master1.example.com 37 | - compile-master2.example.com 38 | -------------------------------------------------------------------------------- /examples/mono_ha_cms_xl.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Use lowercase for all nodes, roles, and components keys and values. 3 | # Use 'g' for GB (the default) and 'm' for MB for ram values. 4 | 5 | nodes: 6 | mono-master.example.com: 7 | resources: 8 | cpu: 16 9 | ram: 32 10 | mono-master-database.example.com: 11 | resources: 12 | cpu: 16 13 | ram: 32 14 | replica-master.example.com: 15 | resources: 16 | cpu: 16 17 | ram: 32 18 | replica-master-database.example.com: 19 | resources: 20 | cpu: 16 21 | ram: 32 22 | compile-master1.example.com: 23 | resources: 24 | cpu: 16 25 | ram: 32 26 | compile-master2.example.com: 27 | resources: 28 | cpu: 16 29 | ram: 32 30 | 31 | # Infrastructure roles include: 32 | # 33 | # puppet_master_host (string) 34 | # primary_master_replica (string) 35 | # console_host (string) 36 | # puppetdb_host (string or an array) 37 | # database_host (string or an array) 38 | # compile_master (string or an array) 39 | 40 | roles: 41 | puppet_master_host: mono-master.example.com 42 | primary_master_replica: replica-master.example.com 43 | database_host: 44 | - mono-master-database.example.com 45 | - replica-master-database.example.com 46 | compile_master: 47 | - compile-master1.example.com 48 | - compile-master2.example.com 49 | -------------------------------------------------------------------------------- /examples/split.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Use lowercase for all nodes, roles, and components keys and values. 3 | # Use 'g' for GB (the default) and 'm' for MB for ram values. 4 | 5 | nodes: 6 | split-master.example.com: 7 | resources: 8 | cpu: 16 9 | ram: 32 10 | split-console.example.com: 11 | resources: 12 | cpu: 8 13 | ram: 16 14 | split-db.example.com: 15 | resources: 16 | cpu: 8 17 | ram: 16 18 | 19 | # Infrastructure roles include: 20 | # 21 | # puppet_master_host (string) 22 | # primary_master_replica (string) 23 | # console_host (string) 24 | # puppetdb_host (string or an array) 25 | # database_host (string or an array) 26 | # compile_master (string or an array) 27 | 28 | roles: 29 | puppet_master_host: split-master.example.com 30 | console_host: split-console.example.com 31 | puppetdb_host: split-db.example.com 32 | -------------------------------------------------------------------------------- /examples/split_external.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Use lowercase for all nodes, roles, and components keys and values. 3 | # Use 'g' for GB (the default) and 'm' for MB for ram values. 4 | 5 | nodes: 6 | split-master.example.com: 7 | resources: 8 | cpu: 16 9 | ram: 32 10 | split-console.example.com: 11 | resources: 12 | cpu: 8 13 | ram: 16 14 | split-db.example.com: 15 | resources: 16 | cpu: 8 17 | ram: 16 18 | external-database.example.com: 19 | resources: 20 | cpu: 8 21 | ram: 16 22 | 23 | # Infrastructure roles include: 24 | # 25 | # puppet_master_host (string) 26 | # primary_master_replica (string) 27 | # console_host (string) 28 | # puppetdb_host (string or an array) 29 | # database_host (string or an array) 30 | # compile_master (string or an array) 31 | 32 | roles: 33 | puppet_master_host: split-master.example.com 34 | console_host: split-console.example.com 35 | puppetdb_host: split-db.example.com 36 | database_host: external-database.example.com 37 | -------------------------------------------------------------------------------- /examples/split_external_cms.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Use lowercase for all nodes, roles, and components keys and values. 3 | # Use 'g' for GB (the default) and 'm' for MB for ram values. 4 | 5 | nodes: 6 | split-master.example.com: 7 | resources: 8 | cpu: 16 9 | ram: 32 10 | split-console.example.com: 11 | resources: 12 | cpu: 8 13 | ram: 16 14 | split-db.example.com: 15 | resources: 16 | cpu: 8 17 | ram: 16 18 | external-database.example.com: 19 | resources: 20 | cpu: 8 21 | ram: 16 22 | compile-master1.example.com: 23 | resources: 24 | cpu: 4 25 | ram: 8 26 | compile-master2.example.com: 27 | resources: 28 | cpu: 4 29 | ram: 8 30 | 31 | # Infrastructure roles include: 32 | # 33 | # puppet_master_host (string) 34 | # primary_master_replica (string) 35 | # console_host (string) 36 | # puppetdb_host (string or an array) 37 | # database_host (string or an array) 38 | # compile_master (string or an array) 39 | 40 | roles: 41 | puppet_master_host: split-master.example.com 42 | console_host: split-console.example.com 43 | puppetdb_host: split-db.example.com 44 | database_host: external-database.example.com 45 | compile_master: 46 | - compile-master1.example.com 47 | - compile-master2.example.com 48 | -------------------------------------------------------------------------------- /fixtures/mono.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | nodes: 3 | master.example.com: 4 | resources: 5 | cpu: 4 6 | ram: 8 7 | roles: 8 | puppet_master_host: master.example.com 9 | -------------------------------------------------------------------------------- /fixtures/no_nodes.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | roles: 3 | puppet_master_host: master.example.com 4 | -------------------------------------------------------------------------------- /fixtures/syntax_error.yaml: -------------------------------------------------------------------------------- 1 | --- ` 2 | -------------------------------------------------------------------------------- /lib/puppet/application/pe.rb: -------------------------------------------------------------------------------- 1 | require 'puppet/application/face_base' 2 | 3 | # This subcommand is implemented as a face. 4 | # The definition of the application can be found in lib/puppet/face/pe.rb. 5 | 6 | class Puppet::Application::Pe < Puppet::Application::FaceBase 7 | end 8 | -------------------------------------------------------------------------------- /lib/puppet/face/pe.rb: -------------------------------------------------------------------------------- 1 | require 'puppet/indirector/face' 2 | 3 | # Load puppet enterprise modules. 4 | # Note that the location of enterprise modules varies from version to version. 5 | 6 | enterprise_modules = ['pe_infrastructure', 'pe_install', 'pe_manager'] 7 | env_mod = '/opt/puppetlabs/server/data/environments/enterprise/modules' 8 | ent_mod = '/opt/puppetlabs/server/data/enterprise/modules' 9 | enterprise_module_paths = [env_mod, ent_mod] 10 | enterprise_module_paths.each do |enterprise_module_path| 11 | next unless File.directory?(enterprise_module_path) 12 | enterprise_modules.each do |enterprise_module| 13 | enterprise_module_lib = "#{enterprise_module_path}/#{enterprise_module}/lib" 14 | next if $LOAD_PATH.include?(enterprise_module_lib) 15 | Puppet.debug("Adding #{enterprise_module} to LOAD_PATH: #{enterprise_module_lib}") 16 | $LOAD_PATH.unshift(enterprise_module_lib) 17 | end 18 | end 19 | 20 | # Load this module's classes (instead of those in pe_manager). 21 | 22 | require_relative '../../puppet_x/puppetlabs/tune' 23 | require_relative '../../puppet_x/puppetlabs/tune/calculate' 24 | require_relative '../../puppet_x/puppetlabs/tune/inventory' 25 | require_relative '../../puppet_x/puppetlabs/tune/query' 26 | require_relative '../../puppet_x/puppetlabs/tune/peconf' 27 | 28 | Puppet::Face.define(:pe, '1.0.0') do 29 | summary _('Inspect infrastructure and output settings') 30 | description <<-'DESC' 31 | Collects information about your Puppet Enterprise installation. 32 | Outputs current or optimized settings that tune Puppet Enterprise services. 33 | DESC 34 | 35 | action(:tune) do 36 | summary _('Inspect infrastructure and output settings') 37 | description <<-'DESC' 38 | Collects information about your Puppet Enterprise installation. 39 | Outputs current or optimized settings that tune Puppet Enterprise services. 40 | DESC 41 | 42 | option '--common' do 43 | summary _('Extract common settings from node-specific settings') 44 | default_to { false } 45 | end 46 | 47 | option '--compare' do 48 | summary _('Output comparison of currently-defined and optimized settings, and exit') 49 | default_to { false } 50 | end 51 | 52 | option '--current' do 53 | summary _('Output currently-defined settings, and exit') 54 | default_to { false } 55 | end 56 | 57 | option '--estimate' do 58 | summary _('Output an estimated capacity summary') 59 | default_to { false } 60 | end 61 | 62 | option '--force' do 63 | summary _('Do not enforce minimum system requirements') 64 | default_to { false } 65 | end 66 | 67 | option '--hiera DIRECTORY' do 68 | summary _('Output Hiera YAML files to a directory') 69 | default_to { nil } 70 | end 71 | 72 | option '--inventory FILE' do 73 | summary _('Use a YAML file to define nodes') 74 | default_to { nil } 75 | end 76 | 77 | option '--local' do 78 | summary _('Use the local system to define a node') 79 | default_to { false } 80 | end 81 | 82 | option '--memory_per_jruby MB' do 83 | summary _('Amount of RAM to allocate for each JRuby') 84 | default_to { nil } 85 | end 86 | 87 | option '--memory_reserved_for_os MB' do 88 | summary _('Amount of RAM to reserve for the OS') 89 | default_to { nil } 90 | end 91 | 92 | option '--node CERTNAME' do 93 | summary _('The node to tune') 94 | default_to { nil } 95 | end 96 | 97 | option '--pe_conf' do 98 | summary _('Output HOCON to pe.conf') 99 | default_to { false } 100 | end 101 | 102 | option '--use_current_memory_per_jruby' do 103 | summary _('Use currently-defined settings to determine memory_per_jruby') 104 | default_to { false } 105 | end 106 | 107 | when_invoked do |*args| 108 | options = args.pop 109 | Puppet.debug("Command Options: #{options}") 110 | Tune = PuppetX::Puppetlabs::Tune.new(options) 111 | 112 | Tune.collect_infrastructure_nodes 113 | Tune.output_infrastructure 114 | 115 | if options[:compare] 116 | Tune.output_compare_current_and_optimized_settings 117 | elsif options[:current] 118 | Tune.output_current_settings 119 | else 120 | Tune.output_optimized_settings 121 | end 122 | 123 | return 124 | end 125 | end 126 | end 127 | -------------------------------------------------------------------------------- /lib/puppet_x/puppetlabs/tune/calculate.rb: -------------------------------------------------------------------------------- 1 | # Note: Calculate the number of jrubies by the number of jrubies that will fit into RAM rather than CPU. 2 | 3 | module PuppetX 4 | module Puppetlabs 5 | # Query infrastructure and show current, or calculate optimized settings. 6 | class Tune 7 | # Calculate optimized settings. 8 | class Calculate 9 | # Calculon Compute! 10 | 11 | def initialize(options) 12 | @defaults = {} 13 | 14 | # For use when estimating capacity. 15 | @defaults[:compile_time_factor] = 2 16 | 17 | # Round up when memory is close to the next level of our leveled settings. See fit_to_memory(). 18 | @defaults[:fit_to_memory_percentage] = 5 19 | 20 | # Memory reserved for the operating system (and other applications). 21 | @defaults[:memory_reserved_for_os_percentage] = 0.25 22 | 23 | @options = {} 24 | 25 | # Users may override these defaults via command line options. 26 | @options[:memory_per_jruby] = options[:memory_per_jruby] || 0 27 | @options[:memory_reserved_for_os] = options[:memory_reserved_for_os] || 0 28 | 29 | Puppet.debug("Using optional #{@options[:memory_per_jruby]} MB RAM per JRuby") if @options[:memory_per_jruby] != 0 30 | Puppet.debug("Using optional #{@options[:memory_reserved_for_os]} MB RAM reserved for the operating system") if @options[:memory_reserved_for_os] != 0 31 | end 32 | 33 | # 34 | # PE Infrastructure Roles 35 | # 36 | 37 | # Masters, Replicas, and Compilers, in Monolithic or Split Infrastructures 38 | # Services: pe-puppetserver and (optionally) all other services. 39 | # Ratios model https://puppet.com/docs/pe/latest/configuring/tuning_monolithic.html 40 | 41 | def calculate_master_settings(node) 42 | percent_cpu_puppetdb = 0.25 43 | minimum_cpu_puppetdb = 1 44 | maximum_cpu_puppetdb = (node['resources']['cpu'] * 0.50).to_i 45 | 46 | minimum_cpu_orchestrator = 1 47 | maximum_cpu_orchestrator = 4 48 | 49 | minimum_cpu_puppetserver = 2 50 | maximum_cpu_puppetserver = 24 51 | 52 | percent_ram_database = 0.20 53 | percent_ram_puppetdb = 0.10 54 | percent_ram_orchestrator = 0.08 55 | percent_ram_console = 0.08 56 | percent_ram_activemq = 0.08 57 | 58 | minimum_ram_puppetserver = 512 59 | 60 | minimum_ram_code_cache = 96 61 | maximum_ram_code_cache = 2048 62 | 63 | maximum_ram_database = 16384 64 | 65 | minimum_ram_puppetdb = 512 66 | maximum_ram_puppetdb = 8192 67 | 68 | minimum_ram_orchestrator = 512 69 | maximum_ram_orchestrator = 1024 70 | 71 | minimum_ram_console = 512 72 | maximum_ram_console = 1024 73 | 74 | minimum_ram_activemq = 256 75 | maximum_ram_activemq = 512 76 | 77 | ram_per_jruby = fit_to_memory(node['resources']['ram'], 512, 768, 1024) 78 | ram_per_jruby_code_cache = 96 79 | 80 | cpu_reserved = 1 81 | ram_reserved = select_reserved_memory(node['resources']['ram']) 82 | 83 | # https://github.com/puppetlabs/puppet-enterprise-modules/blob/irving/modules/puppet_enterprise/manifests/profile/database.pp 84 | default_database_max_connections = 400 85 | 86 | percent_database_connections = 1.10 87 | 88 | settings = initialize_settings(node) 89 | 90 | # Optionally use memory_per_jruby, if defined. 91 | 92 | ram_per_jruby = @options[:memory_per_jruby] if @options[:memory_per_jruby] != 0 93 | 94 | # Optionally use current_memory_per_jruby, if defined. 95 | 96 | ram_per_jruby = node['current_memory_per_jruby'] if node['current_memory_per_jruby'] && node['current_memory_per_jruby'] != 0 97 | 98 | # Reallocate resources between puppetserver and puppetdb, if this host is a monolithic master or replica master, with compile masters or compilers. 99 | 100 | if node['type']['is_monolithic_master'] || node['type']['is_replica_master'] 101 | if node['infrastructure']['with_compile_masters'] 102 | percent_cpu_puppetdb = 0.50 103 | percent_ram_puppetdb = 0.15 104 | end 105 | end 106 | 107 | # The Vegas Renormalization: allow for testing with vmpooler (2 CPU / 6 GB RAM) VMs. 108 | 109 | if node['resources']['cpu'] < 3 110 | minimum_cpu_puppetserver = 1 111 | maximum_cpu_puppetserver = 1 112 | cpu_reserved = 1 113 | ram_reserved = 256 114 | end 115 | 116 | # Do not allocate memory for reserved_code_cache, depending upon jruby version. 117 | 118 | unless node['type']['with_jruby9k_enabled'] 119 | minimum_ram_code_cache = 0 120 | ram_per_jruby_code_cache = 0 121 | end 122 | 123 | # Allocate processors and memory for PE Infrastructure services ... 124 | 125 | if node['classes']['database'] 126 | ram_database = [(node['resources']['ram'] * percent_ram_database).to_i, maximum_ram_database].min 127 | settings['params']['puppet_enterprise::profile::database::shared_buffers'] = "#{ram_database}MB" 128 | settings['totals']['RAM']['used'] += ram_database 129 | end 130 | 131 | if node['classes']['puppetdb'] 132 | # Reallocate resources between puppetserver and puppetdb, if this host is a compiler (puppetserver plus puppetdb). 133 | if node['type']['is_compile_master'] || node['type']['is_compiler'] 134 | percent_cpu_puppetdb = 0.25 135 | minimum_cpu_puppetdb = 1 136 | maximum_cpu_puppetdb = 3 137 | end 138 | 139 | command_processing_threads = (node['resources']['cpu'] * percent_cpu_puppetdb).to_i.clamp(minimum_cpu_puppetdb, maximum_cpu_puppetdb) 140 | settings['params']['puppet_enterprise::puppetdb::command_processing_threads'] = command_processing_threads 141 | settings['totals']['CPU']['used'] += command_processing_threads 142 | 143 | ram_puppetdb = (node['resources']['ram'] * percent_ram_puppetdb).to_i.clamp(minimum_ram_puppetdb, maximum_ram_puppetdb) 144 | settings['params']['puppet_enterprise::profile::puppetdb::java_args'] = { 'Xms' => "#{ram_puppetdb}m", 'Xmx' => "#{ram_puppetdb}m" } 145 | settings['totals']['RAM']['used'] += ram_puppetdb 146 | end 147 | 148 | if node['classes']['orchestrator'] 149 | if node['type']['with_orchestrator_jruby'] 150 | # Reallocate resources between orchestrator and puppetserver, if orchestrator has jrubies. 151 | percent_ram_orchestrator = 0.10 152 | 153 | # Note: orchestrator_jruby_max_active_instances is constrained based on how many jrubies fit into orchestrator memory. 154 | ram_orchestrator = [(node['resources']['ram'] * percent_ram_orchestrator).to_i, minimum_ram_orchestrator].max 155 | max_jrubies_in_ram_orchestrator = (ram_orchestrator / ram_per_jruby).to_i 156 | orchestrator_jruby_max_active_instances = max_jrubies_in_ram_orchestrator.clamp(minimum_cpu_orchestrator, maximum_cpu_orchestrator) 157 | settings['params']['puppet_enterprise::profile::orchestrator::jruby_max_active_instances'] = orchestrator_jruby_max_active_instances 158 | # Note: orchestrator_jruby_max_active_instances is not a dedicated allocation, do not add it to settings['totals']['CPU']['used']. 159 | 160 | # ram_orchestrator_code_cache = orchestrator_jruby_max_active_instances * ram_per_jruby_code_cache 161 | # settings['params']['puppet_enterprise::profile::orchestrator::reserved_code_cache'] = ram_orchestrator_code_cache 162 | # settings['totals']['RAM']['used'] += ram_orchestrator_code_cache 163 | else 164 | ram_orchestrator = (node['resources']['ram'] * percent_ram_orchestrator).to_i.clamp(minimum_ram_orchestrator, maximum_ram_orchestrator) 165 | end 166 | settings['params']['puppet_enterprise::profile::orchestrator::java_args'] = { 'Xms' => "#{ram_orchestrator}m", 'Xmx' => "#{ram_orchestrator}m" } 167 | settings['totals']['RAM']['used'] += ram_orchestrator 168 | end 169 | 170 | if node['classes']['console'] 171 | ram_console = (node['resources']['ram'] * percent_ram_console).to_i.clamp(minimum_ram_console, maximum_ram_console) 172 | settings['params']['puppet_enterprise::profile::console::java_args'] = { 'Xms' => "#{ram_console}m", 'Xmx' => "#{ram_console}m" } 173 | settings['totals']['RAM']['used'] += ram_console 174 | end 175 | 176 | if node['classes']['amq::broker'] 177 | ram_activemq = (node['resources']['ram'] * percent_ram_activemq).to_i.clamp(minimum_ram_activemq, maximum_ram_activemq) 178 | settings['params']['puppet_enterprise::profile::amq::broker::heap_mb'] = ram_activemq 179 | settings['totals']['RAM']['used'] += ram_activemq 180 | end 181 | 182 | # Note: puppetserver is not allocated a percentage of memory: it is initially allocated all unused memory. 183 | 184 | ram_puppetserver = (node['resources']['ram'] - ram_reserved - settings['totals']['RAM']['used']) 185 | if ram_puppetserver < (minimum_ram_puppetserver + minimum_ram_code_cache) 186 | Puppet.debug("Error: available memory for puppetserver: #{ram_puppetserver} MB is less than minimum required: #{minimum_ram_puppetserver} + #{minimum_ram_code_cache} MB") 187 | return 188 | end 189 | 190 | # Note: puppetserver_jruby_max_active_instances is constrained based on both how many jrubies fit into unallocated memory and unallocated processors. 191 | 192 | maximum_cpu_puppetserver = (node['resources']['cpu'] - cpu_reserved - settings['totals']['CPU']['used']).clamp(minimum_cpu_puppetserver, maximum_cpu_puppetserver) 193 | max_jrubies_in_ram_puppetserver = (ram_puppetserver / (ram_per_jruby + ram_per_jruby_code_cache)).to_i 194 | puppetserver_jruby_max_active_instances = max_jrubies_in_ram_puppetserver.clamp(minimum_cpu_puppetserver, maximum_cpu_puppetserver) 195 | settings['params']['puppet_enterprise::master::puppetserver::jruby_max_active_instances'] = puppetserver_jruby_max_active_instances 196 | settings['totals']['CPU']['used'] += puppetserver_jruby_max_active_instances 197 | 198 | ram_puppetserver = [minimum_ram_puppetserver, (puppetserver_jruby_max_active_instances * ram_per_jruby)].max 199 | settings['params']['puppet_enterprise::profile::master::java_args'] = { 'Xms' => "#{ram_puppetserver}m", 'Xmx' => "#{ram_puppetserver}m" } 200 | settings['totals']['RAM']['used'] += ram_puppetserver 201 | 202 | if node['type']['with_jruby9k_enabled'] 203 | code_cache_based_upon_jrubies = puppetserver_jruby_max_active_instances * ram_per_jruby_code_cache 204 | ram_puppetserver_code_cache = code_cache_based_upon_jrubies.clamp(minimum_ram_code_cache, maximum_ram_code_cache) 205 | settings['params']['puppet_enterprise::master::puppetserver::reserved_code_cache'] = "#{ram_puppetserver_code_cache}m" 206 | settings['totals']['RAM']['used'] += ram_puppetserver_code_cache 207 | end 208 | 209 | settings['totals']['MB_PER_JRUBY'] = ram_per_jruby 210 | 211 | # Detune puppetdb to avoid making too many connections to the database and disable garbage collection, if this host is a compiler (puppetserver with puppetdb). 212 | 213 | if node['classes']['puppetdb'] && (node['type']['is_compile_master'] || node['type']['is_compiler']) 214 | read_maximum_pool_size = puppetserver_jruby_max_active_instances + [1, (puppetserver_jruby_max_active_instances / 2).to_i].max 215 | write_maximum_pool_size = (command_processing_threads * 2) 216 | settings['params']['puppet_enterprise::puppetdb::read_maximum_pool_size'] = read_maximum_pool_size 217 | settings['params']['puppet_enterprise::puppetdb::write_maximum_pool_size'] = write_maximum_pool_size 218 | settings['params']['puppet_enterprise::profile::puppetdb::gc_interval'] = 0 219 | end 220 | 221 | # Increase max_connections when the potential number of connections from compilers exceeds the default. 222 | 223 | if node['classes']['database'] && node['infrastructure']['compiler_connections'] 224 | if node['infrastructure']['compiler_connections'] > default_database_max_connections 225 | settings['params']['puppet_enterprise::profile::database::max_connections'] = (node['infrastructure']['compiler_connections'] * percent_database_connections).to_i 226 | end 227 | end 228 | 229 | # Do not return any settings when overallocating. 230 | 231 | if settings['totals']['CPU']['used'] > settings['totals']['CPU']['total'] 232 | Puppet.debug("Error: calculations overallocated processors: #{settings}") 233 | return 234 | end 235 | if (settings['totals']['RAM']['used'] + ram_reserved) > settings['totals']['RAM']['total'] 236 | Puppet.debug("Error: calculations overallocated memory: reserved: #{ram_reserved} settings: #{settings}") 237 | return 238 | end 239 | 240 | settings 241 | end 242 | 243 | # Console Hosts in Split Infrastructures 244 | # Services: pe-console-services 245 | 246 | def calculate_console_settings(node) 247 | percent_ram_console = 0.75 248 | minimum_ram_console = 512 249 | maximum_ram_console = 4096 250 | 251 | # Unused here, as this role is deprecated. 252 | # ram_reserved = select_reserved_memory(node['resources']['ram']) 253 | 254 | settings = initialize_settings(node) 255 | 256 | ram_console = (node['resources']['ram'] * percent_ram_console).to_i.clamp(minimum_ram_console, maximum_ram_console) 257 | settings['params']['puppet_enterprise::profile::console::java_args'] = { 'Xms' => "#{ram_console}m", 'Xmx' => "#{ram_console}m" } 258 | settings['totals']['RAM']['used'] += ram_console 259 | 260 | settings 261 | end 262 | 263 | # PuppetDB Hosts in Split Infrastructures 264 | # Services: pe-puppetdb and (by default, but optionally) pe-postgresql 265 | 266 | def calculate_puppetdb_settings(node) 267 | percent_cpu_puppetdb = 0.50 268 | minimum_cpu_puppetdb = 1 269 | maximum_cpu_puppetdb = (node['resources']['cpu'] * percent_cpu_puppetdb).to_i 270 | 271 | percent_ram_puppetdb = 0.50 272 | minimum_ram_puppetdb = 512 273 | maximum_ram_puppetdb = 8192 274 | 275 | # Unused here, as this role is deprecated. 276 | # ram_reserved = select_reserved_memory(node['resources']['ram']) 277 | 278 | settings = initialize_settings(node) 279 | 280 | if node['classes']['database'] 281 | percent_ram_puppetdb = 0.25 282 | database_settings = calculate_database_settings(node) 283 | settings['params'].merge!(database_settings['params']) 284 | settings['totals']['CPU']['used'] += database_settings['totals']['CPU']['used'] 285 | settings['totals']['RAM']['used'] += database_settings['totals']['RAM']['used'] 286 | end 287 | 288 | command_processing_threads = [minimum_cpu_puppetdb, maximum_cpu_puppetdb].max 289 | settings['params']['puppet_enterprise::puppetdb::command_processing_threads'] = command_processing_threads 290 | settings['totals']['CPU']['used'] += command_processing_threads 291 | 292 | ram_puppetdb = (node['resources']['ram'] * percent_ram_puppetdb).to_i.clamp(minimum_ram_puppetdb, maximum_ram_puppetdb) 293 | settings['params']['puppet_enterprise::profile::puppetdb::java_args'] = { 'Xms' => "#{ram_puppetdb}m", 'Xmx' => "#{ram_puppetdb}m" } 294 | settings['totals']['RAM']['used'] += ram_puppetdb 295 | 296 | settings 297 | end 298 | 299 | # External PostgreSQL Hosts in Monolithic and/or PuppetDB Hosts in Split Infrastructures 300 | # Services: pe-postgresql 301 | 302 | def calculate_database_settings(node) 303 | percent_ram_database = 0.20 304 | minimum_ram_database = 2048 305 | maximum_ram_database = 16384 306 | 307 | ram_reserved = select_reserved_memory(node['resources']['ram']) 308 | 309 | # https://github.com/puppetlabs/puppet-enterprise-modules/blob/irving/modules/puppet_enterprise/manifests/profile/database.pp 310 | default_database_max_connections = 400 311 | 312 | percent_database_connections = 1.10 313 | 314 | settings = initialize_settings(node) 315 | 316 | ram_database = ((node['resources']['ram'] - ram_reserved) * percent_ram_database).to_i.clamp(minimum_ram_database, maximum_ram_database) 317 | settings['params']['puppet_enterprise::profile::database::shared_buffers'] = "#{ram_database}MB" 318 | settings['totals']['RAM']['used'] += ram_database 319 | 320 | # Increase max_connections when the potential number of connections from compilers exceeds the default. 321 | 322 | if node['infrastructure']['compiler_connections'] 323 | if node['infrastructure']['compiler_connections'] > default_database_max_connections 324 | settings['params']['puppet_enterprise::profile::database::max_connections'] = (node['infrastructure']['compiler_connections'] * percent_database_connections).to_i 325 | end 326 | end 327 | 328 | settings 329 | end 330 | 331 | # Return a new settings structure. 332 | 333 | def initialize_settings(node) 334 | { 335 | 'params' => {}, 336 | 'totals' => { 337 | 'CPU' => { 'total' => node['resources']['cpu'], 'used' => 0 }, 338 | 'RAM' => { 'total' => node['resources']['ram'], 'used' => 0 }, 339 | }, 340 | } 341 | end 342 | 343 | # 344 | # Utilities 345 | # 346 | 347 | # Estimate a reasonable sample of agent runs based upon node count and run interval. 348 | 349 | def calculate_run_sample(active_nodes, run_interval) 350 | maximum = 10000 351 | seconds_per_day = 86400 352 | # If running continuously ... 353 | return [active_nodes, maximum].min if run_interval.zero? 354 | runs_per_day = (seconds_per_day / run_interval).to_i 355 | # If running less than once per day ... 356 | return [active_nodes * 7, maximum].min if runs_per_day < 1 357 | [active_nodes * runs_per_day, maximum].min 358 | end 359 | 360 | # Little's Law 361 | # 362 | # L = λ * W 363 | # 364 | # Where: 365 | # 366 | # L = Number of requests in the queue. 367 | # λ = Average effective arrival rate of requests. 368 | # W = Average time spent processing a request. 369 | 370 | # Estimate the theoretical maximum number of nodes that can managed by an infrastructure. 371 | 372 | def calculate_maximum_nodes(average_compile_time, available_jrubies, run_interval) 373 | jruby_lock_time = average_compile_time.to_f * @defaults[:compile_time_factor] 374 | ((run_interval.to_f * available_jrubies.to_f) / jruby_lock_time.to_f).ceil 375 | end 376 | 377 | # Estimate the theoretical minimum number of jrubies required to manage an infrastructure. 378 | 379 | def calculate_minimum_jrubies(active_nodes, average_compile_time, run_interval) 380 | jruby_lock_time = average_compile_time.to_f * @defaults[:compile_time_factor] 381 | ((active_nodes.to_f * jruby_lock_time.to_f) / run_interval.to_f).ceil 382 | end 383 | 384 | # Return the option or the default. 385 | 386 | def select_reserved_memory(memory) 387 | return @options[:memory_reserved_for_os] if @options[:memory_reserved_for_os] != 0 388 | (memory * @defaults[:memory_reserved_for_os_percentage]).to_i 389 | end 390 | 391 | # Model https://puppet.com/docs/pe/latest/configuring/tuning_monolithic.html 392 | 393 | def fit_to_memory(memory, small, medium, large) 394 | # Round up to the nearest power of two (31500 -> 32768) if within a percentage. 395 | target_memory = nearest_power_of_two(memory) 396 | if (memory < target_memory) && within_percent?(memory, target_memory, @defaults[:fit_to_memory_percentage]) 397 | Puppet.debug("Rounding #{memory} up to #{target_memory} for fit_to_memory") 398 | memory = target_memory 399 | end 400 | return small if memory <= 8192 401 | return medium if memory <= 16384 402 | return medium if memory < 32768 403 | Puppet.debug('Using a maximum value for fit_to_memory') 404 | return large if memory >= 32768 405 | end 406 | 407 | # Test if a number is within a percentage of another number. 408 | 409 | def within_percent?(actual, target, percentage) 410 | return false if actual == target 411 | (Float(target - actual) / target * 100).ceil <= percentage 412 | end 413 | 414 | # Return a number as a computer-science number. 415 | 416 | def nearest_power_of_two(number) 417 | return 0 if number <= 0 418 | exponent = Math.log2 number 419 | higher_power = 2**exponent.ceil 420 | lower_power = 2**exponent.floor 421 | ((higher_power - number) <= (number - lower_power)) ? higher_power : lower_power 422 | end 423 | end 424 | end 425 | end 426 | end 427 | -------------------------------------------------------------------------------- /lib/puppet_x/puppetlabs/tune/cli.rb: -------------------------------------------------------------------------------- 1 | require 'optparse' 2 | require 'puppet' 3 | require 'json' 4 | require 'yaml' 5 | 6 | # Load puppet enterprise modules. 7 | # Note that the location of enterprise modules varies from version to version. 8 | 9 | enterprise_modules = ['pe_infrastructure', 'pe_install', 'pe_manager'] 10 | env_mod = '/opt/puppetlabs/server/data/environments/enterprise/modules' 11 | ent_mod = '/opt/puppetlabs/server/data/enterprise/modules' 12 | enterprise_module_paths = [env_mod, ent_mod] 13 | enterprise_module_paths.each do |enterprise_module_path| 14 | next unless File.directory?(enterprise_module_path) 15 | enterprise_modules.each do |enterprise_module| 16 | enterprise_module_lib = "#{enterprise_module_path}/#{enterprise_module}/lib" 17 | next if $LOAD_PATH.include?(enterprise_module_lib) 18 | Puppet.debug("Adding #{enterprise_module} to LOAD_PATH: #{enterprise_module_lib}") 19 | $LOAD_PATH.unshift(enterprise_module_lib) 20 | end 21 | end 22 | 23 | # Load this module's classes (instead of those in pe_manager). 24 | 25 | require_relative 'calculate' 26 | require_relative 'inventory' 27 | require_relative 'peconf' 28 | require_relative 'query' 29 | 30 | options = {} 31 | parser = OptionParser.new do |opts| 32 | opts.banner = 'Usage: tune.rb [options]' 33 | opts.separator '' 34 | opts.separator 'Summary: Inspect infrastructure and output settings' 35 | opts.separator '' 36 | opts.separator 'Options:' 37 | opts.separator '' 38 | options[:common] = false 39 | opts.on('--common', 'Extract common settings from node-specific settings') do 40 | options[:common] = true 41 | end 42 | options[:compare] = false 43 | opts.on('--compare', 'Output comparison of currently-defined and optimized settings, and exit') do 44 | options[:compare] = true 45 | end 46 | options[:current] = false 47 | opts.on('--current', 'Output currently-defined settings, and exit') do 48 | options[:current] = true 49 | end 50 | options[:debug] = false 51 | opts.on('--debug', 'Enable logging of debug information') do 52 | options[:debug] = true 53 | end 54 | options[:estimate] = false 55 | opts.on('--estimate', 'Output an estimated capacity summary') do 56 | options[:estimate] = true 57 | end 58 | options[:force] = false 59 | opts.on('--force', 'Do not enforce minimum system requirements') do 60 | options[:force] = true 61 | end 62 | opts.on('--hiera DIRECTORY', 'Output Hiera YAML files to a directory') do |hi| 63 | options[:hiera] = hi 64 | end 65 | opts.on('--inventory FILE', 'Use a YAML file to define nodes') do |no| 66 | options[:inventory] = no 67 | end 68 | options[:local] = false 69 | opts.on('--local', 'Use the local system to define a node') do 70 | options[:local] = true 71 | end 72 | opts.on('--memory_per_jruby MB', 'Amount of RAM to allocate for each JRuby') do |me| 73 | options[:memory_per_jruby] = me 74 | end 75 | opts.on('--memory_reserved_for_os MB', 'Amount of RAM to reserve for the OS') do |mo| 76 | options[:memory_reserved_for_os] = mo 77 | end 78 | opts.on('--node CERTNAME', 'The node to tune') do |no| 79 | options[:node] = no 80 | end 81 | options[:pe_conf] = false 82 | opts.on('--pe_conf', 'Output HOCON to pe.conf') do 83 | options[:pe_conf] = true 84 | end 85 | options[:use_current_memory_per_jruby] = false 86 | opts.on('--use_current_memory_per_jruby', 'Use currently-defined settings to determine memory_per_jruby') do 87 | options[:use_current_memory_per_jruby] = true 88 | end 89 | opts.on('-h', '--help', 'Display help') do 90 | puts opts 91 | puts 92 | exit 0 93 | end 94 | end 95 | parser.parse! 96 | 97 | Puppet.initialize_settings 98 | Puppet::Util::Log.newdestination :console 99 | Puppet.debug = options[:debug] 100 | Puppet.debug("Command Options: #{options}") 101 | Tune = PuppetX::Puppetlabs::Tune.new(options) 102 | Puppet.warning "Unable to identify Database Hosts or tune PostgreSQL services in PE 2017.x and older\n" unless Tune.pe_2018_or_newer? 103 | 104 | Tune.collect_infrastructure_nodes 105 | Tune.output_infrastructure 106 | 107 | if options[:compare] 108 | Tune.output_compare_current_and_optimized_settings 109 | elsif options[:current] 110 | Tune.output_current_settings 111 | else 112 | Tune.output_optimized_settings 113 | end 114 | -------------------------------------------------------------------------------- /lib/puppet_x/puppetlabs/tune/inventory.rb: -------------------------------------------------------------------------------- 1 | require 'yaml' 2 | 3 | # Notes: 4 | # 5 | # Mappings vary between PE Infrastructure roles, profiles, and classes. 6 | # See: https://github.com/puppetlabs/puppetlabs-pe_infrastructure/blob/irving/lib/puppet_x/puppetlabs/meep/defaults.rb 7 | 8 | module PuppetX 9 | module Puppetlabs 10 | # Query infrastructure and show current, or calculate optimized settings. 11 | class Tune 12 | # Use the local system or a file as inventory, instead of querying PuppeDB. 13 | class Inventory 14 | attr_reader :nodes 15 | attr_reader :roles 16 | attr_reader :classes 17 | 18 | def initialize 19 | @nodes = {} 20 | @roles = {} 21 | @classes = {} 22 | end 23 | 24 | # Inventory PE Infrastructure roles. 25 | 26 | def default_inventory_roles 27 | { 28 | 'puppet_master_host' => nil, 29 | 'console_host' => nil, 30 | 'puppetdb_host' => [], 31 | 'database_host' => [], 32 | 'primary_master_replica' => nil, 33 | 'compile_master' => [] 34 | } 35 | end 36 | 37 | # Inventory PE Infrastructure classes. 38 | 39 | def default_inventory_classes 40 | { 41 | 'master' => [].to_set, 42 | 'console' => [].to_set, 43 | 'puppetdb' => [].to_set, 44 | 'database' => [].to_set, 45 | 'amq::broker' => [].to_set, 46 | 'orchestrator' => [].to_set, 47 | 'primary_master' => [].to_set, 48 | 'primary_master_replica' => [].to_set, 49 | 'compile_master' => [].to_set 50 | } 51 | end 52 | 53 | # Use the local system to define a Monolithic PE Infrastructure node. 54 | # This eliminates the dependency upon PuppetDB during a clean install when there is no PuppetDB to query. 55 | 56 | def read_inventory_from_local_system 57 | Puppet.debug('Querying the local system to define a monolithic infrastructure master node') 58 | hostname = Puppet::Util::Execution.execute('hostname -f', 'combine' => false).chomp 59 | cpu = Puppet::Util::Execution.execute('nproc --all', 'combine' => false).chomp 60 | ram = Puppet::Util::Execution.execute('free -b | grep Mem', 'combine' => false).chomp.split(' ')[1] 61 | ram << 'b' 62 | nodes = { 63 | hostname => { 64 | 'resources' => { 65 | 'cpu' => cpu, 66 | 'ram' => ram, 67 | } 68 | } 69 | } 70 | Puppet.debug("Found resources on the local system: #{nodes}") 71 | roles = { 72 | 'puppet_master_host' => hostname 73 | } 74 | 75 | @nodes = nodes 76 | @roles = default_inventory_roles.merge(roles) 77 | @classes = default_inventory_classes 78 | end 79 | 80 | # Use an inventory file to define PE Infrastructure nodes. 81 | # This eliminates the dependency upon PuppetDB to query node resources and classes. 82 | # The compile_master, database_host, and puppetdb_host roles can be an Array or a String. 83 | 84 | def read_inventory_from_inventory_file(file) 85 | unless File.exist?(file) 86 | Puppet.err _("The inventory file %{file} does not exist") % { file: file } 87 | return 88 | end 89 | Puppet.debug("Using the inventory file #{file} to define infrastructure nodes") 90 | begin 91 | file_inventory = YAML.safe_load(File.read(file)) 92 | rescue Psych::SyntaxError 93 | Puppet.err _("The inventory file %{file} contains a syntax error") % { file: file } 94 | return 95 | end 96 | unless file_inventory['nodes'] 97 | Puppet.err _('The inventory file does not contain a nodes hash') 98 | return 99 | end 100 | file_inventory['roles'] = {} unless file_inventory['roles'] 101 | file_inventory['roles']['compile_master'] = Array(file_inventory['roles']['compile_master']) 102 | file_inventory['roles']['database_host'] = Array(file_inventory['roles']['database_host']) 103 | file_inventory['roles']['puppetdb_host'] = Array(file_inventory['roles']['puppetdb_host']) 104 | 105 | @nodes = file_inventory['nodes'] 106 | @roles = default_inventory_roles.merge(file_inventory['roles']) 107 | @classes = default_inventory_classes 108 | end 109 | 110 | # Convert inventory roles to PE Infrastructure classes, using Set instead of Array to prevent duplicates. 111 | 112 | def convert_inventory_roles_to_classes 113 | if @roles['database_host'] 114 | @roles['database_host'].each do |database_host| 115 | Puppet.debug("Converting database_host role to classes for: #{database_host}") 116 | @classes['database'] << database_host 117 | end 118 | end 119 | 120 | if @roles['console_host'] 121 | console_host = @roles['console_host'] 122 | Puppet.debug("Converting console_host role to classes for: #{console_host}") 123 | @classes['console'] << console_host 124 | end 125 | 126 | is_split = not_set?(@roles['console_host']) 127 | is_split_local_database = is_split && @roles['database_host'].count.zero? 128 | 129 | if @roles['puppetdb_host'] 130 | @roles['puppetdb_host'].each do |puppetdb_host| 131 | Puppet.debug("Converting puppetdb_host role to classes for: #{puppetdb_host}") 132 | @classes['puppetdb'] << puppetdb_host 133 | @classes['database'] << @roles['puppetdb_host'].first if is_split_local_database 134 | end 135 | end 136 | 137 | is_mono = is_split == false 138 | is_ha = not_set?(@roles['primary_master_replica']) 139 | is_mono_local_database = is_mono && @roles['database_host'].count.zero? 140 | is_mono_extra_large = is_mono && !is_ha && @roles['database_host'].count == 1 141 | is_mono_ha_extra_large = is_mono && is_ha && @roles['database_host'].count == 2 142 | is_extra_large = is_mono_extra_large || is_mono_ha_extra_large 143 | 144 | if @roles['puppet_master_host'] 145 | puppet_master_host = @roles['puppet_master_host'] 146 | Puppet.debug("Converting puppet_master_host role to classes for: #{puppet_master_host}") 147 | @classes['primary_master'] << puppet_master_host 148 | @classes['master'] << puppet_master_host 149 | @classes['console'] << puppet_master_host if is_mono 150 | @classes['puppetdb'] << puppet_master_host if is_mono 151 | @classes['database'] << puppet_master_host if is_mono_local_database || is_extra_large 152 | @classes['amq::broker'] << puppet_master_host 153 | @classes['orchestrator'] << puppet_master_host 154 | end 155 | 156 | if @roles['primary_master_replica'] 157 | primary_master_replica = @roles['primary_master_replica'] 158 | Puppet.debug("Converting primary_master_replica role to classes for: #{primary_master_replica}") 159 | @classes['primary_master_replica'] << primary_master_replica 160 | @classes['master'] << primary_master_replica 161 | @classes['console'] << primary_master_replica 162 | @classes['puppetdb'] << primary_master_replica 163 | @classes['database'] << primary_master_replica 164 | @classes['amq::broker'] << primary_master_replica 165 | @classes['orchestrator'] << primary_master_replica 166 | end 167 | 168 | if @roles['compile_master'] 169 | @roles['compile_master'].each do |compile_master| 170 | Puppet.debug("Converting compile_master role to classes for: #{compile_master}") 171 | @classes['compile_master'] << compile_master 172 | @classes['master'] << compile_master 173 | @classes['puppetdb'] << compile_master if is_extra_large 174 | end 175 | end 176 | end 177 | 178 | # 179 | # Utilities 180 | # 181 | 182 | # Array or String 183 | 184 | def not_set?(variable) 185 | return false if variable.nil? || variable.empty? 186 | true 187 | end 188 | end 189 | end 190 | end 191 | end 192 | -------------------------------------------------------------------------------- /lib/puppet_x/puppetlabs/tune/peconf.rb: -------------------------------------------------------------------------------- 1 | module PuppetX 2 | module Puppetlabs 3 | # Query infrastructure and show current, or calculate optimized settings. 4 | class Tune 5 | # Interface to PuppetX::Puppetlabs::Meep 6 | class PEConf 7 | attr_reader :file 8 | 9 | def initialize(meep_config_path) 10 | @file = "#{meep_config_path}/conf.d/pe.conf" 11 | require 'puppet_x/puppetlabs/meep/config' 12 | # An error is raised if the file does not exist. Create? 13 | @meep_meep = PuppetX::Puppetlabs::Meep::Modify.new(meep_config_path) 14 | end 15 | 16 | def write(settings) 17 | settings.each do |key, optimized| 18 | previous = @meep_meep::get_in_pe_conf(key) 19 | Puppet.debug("Previous value in pe.conf for #{key}: #{previous}") 20 | if previous.is_a?(Hash) 21 | merged = previous.merge(optimized) 22 | @meep_meep::set_in_pe_conf(key, merged) 23 | else 24 | @meep_meep::set_in_pe_conf(key, optimized) 25 | end 26 | updated = @meep_meep::get_in_pe_conf(key) 27 | Puppet.debug("Updated value in pe.conf for #{key}: #{updated}") 28 | end 29 | true 30 | end 31 | end 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/puppet_x/puppetlabs/tune/query.rb: -------------------------------------------------------------------------------- 1 | module PuppetX 2 | module Puppetlabs 3 | # Query infrastructure and show current, or calculate optimized settings. 4 | class Tune 5 | # Interface to Puppet::Util::Puppetdb, Puppet::Util::Pe_conf, and Puppet::Util::Pe_conf::Recover 6 | class Query 7 | attr_reader :environment 8 | attr_reader :environmentpath 9 | 10 | def initialize 11 | @environment = 'production' 12 | @environmentpath = '/etc/puppetlabs/code/environments' 13 | end 14 | 15 | # PE-15116 overrides 'environment' and 'environmentpath' in the 'puppet infrastructure' face. 16 | # The original values are required by methods in this class. 17 | 18 | def pe_environment(certname) 19 | environment = catalog_environment(certname) 20 | if environment 21 | @environment = environment 22 | else 23 | Puppet.debug("No Environment found in PuppetDB using: #{certname}") 24 | Puppet.debug("Querying 'puppet config print environment' for Environment") 25 | @environment = Puppet::Util::Execution.execute('/opt/puppetlabs/puppet/bin/puppet config print environment --section master', 'combine' => false).chomp 26 | end 27 | @environmentpath = Puppet::Util::Execution.execute('/opt/puppetlabs/puppet/bin/puppet config print environmentpath --section master', 'combine' => false).chomp 28 | end 29 | 30 | # Query PuppetDB for the environment of a PE Infrastructure node. 31 | 32 | def catalog_environment(certname) 33 | Puppet.debug("Querying PuppetDB for Environment using: #{certname}") 34 | pql = ['from', 'nodes', 35 | ['extract', ['certname', 'catalog_environment'], 36 | ['and', 37 | ['=', 'certname', certname], 38 | ] 39 | ] 40 | ] 41 | results = query_puppetdb(pql) 42 | if results.nil? 43 | Puppet.debug('No result for PuppetDB Query') 44 | return nil 45 | end 46 | if results.empty? 47 | Puppet.debug('No results for PuppetDB Query') 48 | return nil 49 | end 50 | Puppet.debug('Begin Query Results') 51 | Puppet.debug(results) 52 | Puppet.debug('End Query Results') 53 | results[0]['catalog_environment'] 54 | end 55 | 56 | # Query PuppetDB for the count of active nodes. 57 | 58 | def active_node_count 59 | Puppet.debug('Querying PuppetDB for Active Nodes') 60 | pql = ['from', 'nodes', 61 | ['extract', ['certname'], 62 | ['and', 63 | ['=', ['node', 'active'], true], 64 | ] 65 | ] 66 | ] 67 | results = query_puppetdb(pql) 68 | if results.nil? 69 | Puppet.debug('No result for PuppetDB Query') 70 | return nil 71 | end 72 | if results.empty? 73 | Puppet.debug('No results for PuppetDB Query') 74 | end 75 | Puppet.debug('Begin Query Results') 76 | Puppet.debug(results) 77 | Puppet.debug('End Query Results') 78 | results.count 79 | end 80 | 81 | # Query PuppetDB for config_retrieval or total time metrics. 82 | 83 | # "name" => "catalog_application", 84 | # "name" => "config_retrieval", 85 | # "name" => "convert_catalog", 86 | # "name" => "fact_generation", 87 | # "name" => "node_retrieval", 88 | # "name" => "plugin_sync", 89 | # "name" => "transaction_evaluation", 90 | # "name" => "total", 91 | 92 | def average_compile_time(query_limit = 1000) 93 | Puppet.debug('Querying PuppetDB for Average Compile Time') 94 | pql = ['from', 'reports', 95 | ['extract', 96 | ['hash', 'start_time', 'end_time', 'metrics'], 97 | ], 98 | ['limit', query_limit] 99 | ] 100 | results = query_puppetdb(pql) 101 | if results.nil? 102 | Puppet.debug('No result for PuppetDB Query') 103 | return nil 104 | end 105 | if results.empty? 106 | Puppet.debug('No results for PuppetDB Query') 107 | return nil 108 | end 109 | random_report_hash = results.sample['hash'] 110 | Puppet.debug("Random report: #{random_report_hash}") 111 | average_metric_time(results, 'config_retrieval', 'total') 112 | end 113 | 114 | # Query PuppetDB for average config_retrieval or time metrics for a given time range. 115 | # 116 | # @param [String] start_time Value of start_time to query data for 117 | # @param [String] end_time Value of end_time to query data for 118 | # 119 | # @return [Integer] Average time for config_retrieval or total time in data set 120 | 121 | def average_compile_time_for_range(start_time, end_time) 122 | start_time = Time.parse(start_time) 123 | end_time = Time.parse(end_time) 124 | # Ensure that start is less than end 125 | start_time, end_time = [start_time, end_time].sort 126 | 127 | # Ensure that time zone is not emitted in string sent to pdb endpoint 128 | start_time = start_time.strftime "%Y-%m-%d %H:%M:%S" 129 | end_time = end_time.strftime "%Y-%m-%d %H:%M:%S" 130 | 131 | # Extract metrics from reports that only fall within the time fence 132 | Puppet.debug('Querying PuppetDB for Average Compile Time for Range') 133 | pql = ['from', 'reports', 134 | ['extract', 'metrics', 135 | ['and', 136 | ['>', 'start_time', start_time], 137 | ['<', 'end_time', end_time] 138 | ] 139 | ] 140 | ] 141 | results = query_puppetdb(pql) 142 | if results.nil? 143 | Puppet.debug('No result for PuppetDB Query') 144 | return nil 145 | end 146 | if results.empty? 147 | Puppet.debug('No results for PuppetDB Query') 148 | return nil 149 | end 150 | average_metric_time(results, 'config_retrieval', 'total') 151 | end 152 | 153 | # Calculate average time for PuppetDB metric on a given data set. 154 | # 155 | # @param [Array] data Results from call to query_puppetdb(pql) 156 | # @param [String] metric Primary metric to find and average time values for 157 | # @param [String] failover_metric Secondary metric to use if primary is not found 158 | # 159 | # @return [Integer] Average time for metric in data set 160 | 161 | def average_metric_time(data, metric, failover_metric) 162 | result = nil 163 | begin 164 | data.delete_if { |report| report['metrics']['data'].empty? } 165 | config_retrieval_times = data.map do |report| 166 | report['metrics']['data'].select { |md| 167 | md['category'] == 'time' && (md['name'] == metric || md['name'] == failover_metric) 168 | }.first.fetch('value') 169 | end 170 | avg_config_retrieval_time = config_retrieval_times.reduce(0.0, :+) / config_retrieval_times.size 171 | result = avg_config_retrieval_time.ceil 172 | rescue # rubocop:disable Lint/HandleExceptions 173 | end 174 | result 175 | end 176 | 177 | # Query PuppetDB for nodes with a PE Infrastructure class. 178 | 179 | def infra_nodes_with_class(classname) 180 | Puppet.debug("Querying PuppetDB for Class: Puppet_enterprise::Profile::#{classname}") 181 | pql = ['from', 'resources', 182 | ['extract', ['certname', 'parameters'], 183 | ['and', 184 | ['=', 'type', 'Class'], 185 | ['=', 'environment', @environment], 186 | ['=', 'title', "Puppet_enterprise::Profile::#{classname}"], 187 | ['=', ['node', 'active'], true], 188 | ] 189 | ] 190 | ] 191 | results = query_puppetdb(pql) 192 | if results.nil? 193 | Puppet.debug('No result for PuppetDB Query') 194 | return nil 195 | end 196 | if results.empty? 197 | Puppet.debug('No results for PuppetDB Query') 198 | end 199 | Puppet.debug('Begin Query Results') 200 | Puppet.debug(results) 201 | Puppet.debug('End Query Results') 202 | results.map { |resource| resource.fetch('certname') } 203 | end 204 | 205 | # Query PuppetDB for facts for a node. 206 | 207 | def node_facts(certname) 208 | Puppet.debug("Querying PuppetDB for Facts for: #{certname}") 209 | pql = ['from', 'facts', 210 | ['extract', ['name', 'value'], 211 | ['and', 212 | ['=', 'certname', certname], 213 | ['=', 'environment', @environment], 214 | ] 215 | ] 216 | ] 217 | results = query_puppetdb(pql) 218 | if results.nil? 219 | Puppet.debug('No result for PuppetDB Query') 220 | return nil 221 | end 222 | if results.empty? 223 | Puppet.debug('No results for PuppetDB Query') 224 | end 225 | Puppet.debug('Begin Query Results') 226 | Puppet.debug(results) 227 | Puppet.debug('End Query Results') 228 | facts = {} 229 | results.each do |fact, _nil| 230 | facts[fact['name']] = fact['value'] 231 | end 232 | facts 233 | end 234 | 235 | # Return settings configured in Hiera and the Classifier. 236 | 237 | def hiera_classifier_settings(certname, setting_names) 238 | overrides_hiera, overrides_classifier = hiera_classifier_overrides(certname, setting_names) 239 | Puppet.debug("Settings: #{setting_names}") 240 | Puppet.debug("Settings from Hiera for: #{certname}: #{overrides_hiera}") 241 | Puppet.debug("Settings from Classifier for: #{certname}: #{overrides_classifier}") 242 | return { 'hiera' => overrides_hiera, 'classifier' => overrides_classifier } 243 | rescue Puppet::Error 244 | return nil 245 | end 246 | 247 | # Internal helper methods. 248 | 249 | private 250 | 251 | # If 'puppet/util/puppetdb' would be required at the top of the file, 252 | # then it would be autoloaded/required as part of the install process, 253 | # resulting in an error. 254 | 255 | def query_puppetdb(pql) 256 | require 'puppet/util/puppetdb' 257 | return Puppet::Util::Puppetdb.query_puppetdb(pql) 258 | rescue Puppet::Error 259 | return nil 260 | end 261 | 262 | # Extract the beating heart of a puppet compiler for lookup purposes. 263 | 264 | def hiera_classifier_overrides(certname, settings) 265 | require 'puppet/util/pe_conf' 266 | require 'puppet/util/pe_conf/recover' 267 | 268 | if recover_with_instance_method? 269 | recover = Puppet::Util::Pe_conf::Recover.new 270 | recover_node_facts = recover.facts_for_node(certname, @environment) 271 | node_terminus = recover.get_node_terminus 272 | overrides_hiera = recover.find_hiera_overrides(certname, settings, recover_node_facts, @environment, node_terminus) 273 | overrides_classifier = recover.classifier_overrides_for_node(certname, recover_node_facts, recover_node_facts['::trusted']) 274 | else 275 | recover_node_facts = Puppet::Util::Pe_conf::Recover.facts_for_node(certname, @environment) 276 | if recover_with_node_terminus_method? 277 | node_terminus = Puppet::Util::Pe_conf::Recover.get_node_terminus 278 | overrides_hiera = Puppet::Util::Pe_conf::Recover.find_hiera_overrides(certname, settings, recover_node_facts, @environment, node_terminus) 279 | else 280 | overrides_hiera = Puppet::Util::Pe_conf::Recover.find_hiera_overrides(settings, recover_node_facts, @environment) 281 | end 282 | overrides_classifier = Puppet::Util::Pe_conf::Recover.classifier_overrides_for_node(certname, recover_node_facts, recover_node_facts['::trusted']) 283 | end 284 | [overrides_hiera, overrides_classifier] 285 | end 286 | 287 | # PE-24106 changes Recover to a class with instance methods. 288 | 289 | def recover_with_instance_method? 290 | defined?(Puppet::Util::Pe_conf::Recover.facts_for_node) != 'method' 291 | end 292 | 293 | # In some versions, Puppet::Util::Pe_conf::Recover does not implement get_node_terminus() and implements find_hiera_overrides(params, facts, environment) 294 | 295 | def recover_with_node_terminus_method? 296 | defined?(Puppet::Util::Pe_conf::Recover.get_node_terminus) == 'method' 297 | end 298 | end 299 | end 300 | end 301 | end 302 | -------------------------------------------------------------------------------- /metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tkishel-pe_tune", 3 | "version": "3.1.2", 4 | "author": "Thomas Kishel", 5 | "summary": "Inspect infrastructure and suggest tuned settings for services.", 6 | "license": "Apache-2.0", 7 | "source": "https://github.com/tkishel/pe_tune.git", 8 | "project_page": "https://github.com/tkishel/pe_tune", 9 | "issues_url": "https://github.com/tkishel/pe_tune/issues", 10 | "dependencies": [ 11 | 12 | ], 13 | "operatingsystem_support": [ 14 | { 15 | "operatingsystem": "RedHat", 16 | "operatingsystemrelease": [ 17 | "5", 18 | "6", 19 | "7" 20 | ] 21 | }, 22 | { 23 | "operatingsystem": "CentOS", 24 | "operatingsystemrelease": [ 25 | "5", 26 | "6", 27 | "7" 28 | ] 29 | }, 30 | { 31 | "operatingsystem": "Ubuntu", 32 | "operatingsystemrelease": [ 33 | "12.04", 34 | "14.04", 35 | "16.04", 36 | "18.04" 37 | ] 38 | } 39 | ], 40 | "requirements": [ 41 | { 42 | "name": "puppet", 43 | "version_requirement": "> 4.7.0 < 7.0.0" 44 | } 45 | ], 46 | "description": "Collects information about your Puppet Enterprise installation. Outputs optimized settings for PE services based upon CPU and RAM.", 47 | "pdk-version": "1.7.1", 48 | "template-url": "file:///opt/puppetlabs/pdk/share/cache/pdk-templates.git", 49 | "template-ref": "1.7.1-0-g810b982" 50 | } 51 | -------------------------------------------------------------------------------- /spec/README.md: -------------------------------------------------------------------------------- 1 | # pe_tune 2 | 3 | #### Outline 4 | 5 | 1. Stub out reading puppet.conf 6 | 1. Stub out reading pe.conf 7 | 1. Stub out PDB queries for groups and facts 8 | 1. Write test for a monolithic infrastructure 9 | 1. Write tests for various combinations of infrastructures 10 | 1. Write tests for reading various combinations of processors and memory 11 | 1. Combine combinations into a matrix -------------------------------------------------------------------------------- /spec/acceptance/nodesets/default.yml: -------------------------------------------------------------------------------- 1 | --- 2 | HOSTS: 3 | masterblaster: 4 | pe_dir: http://neptune.puppetlabs.lan/2018.1/ci-ready 5 | pe_ver: 6 | pe_upgrade_dir: 7 | pe_upgrade_ver: 8 | hypervisor: vmpooler 9 | # hypervisor: none 10 | # ip: xyz.delivery.puppetlabs.net 11 | platform: el-7-x86_64 12 | template: centos-7-x86_64 13 | roles: 14 | - agent 15 | - master 16 | - database 17 | - dashboard 18 | CONFIG: 19 | nfs_server: none 20 | consoleport: 443 21 | pooling_api: http://vmpooler.delivery.puppetlabs.net/ 22 | ssh: 23 | keys: "~/.ssh/id_rsa-acceptance" 24 | -------------------------------------------------------------------------------- /spec/acceptance/run_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper_acceptance' 2 | 3 | # BEAKER_provision=yes PUPPET_INSTALL_TYPE=pe bundle exec rake beaker:default 4 | 5 | describe 'run' do 6 | context 'puppet pe tune' do 7 | it 'and output optimized settings' do 8 | on(master, puppet('pe', 'tune'), acceptable_exit_codes: 0) do |result| 9 | expect(result.stdout).to match(%r{CPU Summary}) 10 | end 11 | end 12 | 13 | it 'and output current settings' do 14 | on(master, puppet('pe', 'tune', '--current'), acceptable_exit_codes: 0) do |result| 15 | expect(result.stdout).to match(%r{Found (defined|default) settings}) 16 | end 17 | end 18 | 19 | it 'and compare current and optimized settings' do 20 | on(master, puppet('pe', 'tune', '--compare'), acceptable_exit_codes: 0) do |result| 21 | expect(result.stdout).to match(%r{No defined settings to compare}) # or %r{Defined and optimized settings (match|vary)} 22 | end 23 | end 24 | 25 | it 'and output current settings to pe_conf' do 26 | on(master, puppet('pe', 'tune', '--local', '--pe_conf'), acceptable_exit_codes: 0) do |result| 27 | expect(result.stdout).to match(%r{Merged optimized settings}) 28 | end 29 | end 30 | 31 | it 'and output current settings to hiera' do 32 | on(master, puppet('pe', 'tune', '--hiera', '/tmp/xyz'), acceptable_exit_codes: 0) do |result| 33 | expect(result.stdout).to match(%r{Wrote Hiera YAML file}) 34 | end 35 | end 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /spec/default_facts.yml: -------------------------------------------------------------------------------- 1 | # Use default_module_facts.yml for module specific facts. 2 | # 3 | # Facts specified here will override the values provided by rspec-puppet-facts. 4 | --- 5 | concat_basedir: "" 6 | ipaddress: "172.16.254.254" 7 | is_pe: false 8 | macaddress: "AA:AA:AA:AA:AA:AA" 9 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | RSpec.configure do |c| 2 | c.mock_with :rspec 3 | end 4 | 5 | require 'puppetlabs_spec_helper/module_spec_helper' 6 | require 'rspec-puppet-facts' 7 | 8 | begin 9 | require 'spec_helper_local' if File.file?(File.join(File.dirname(__FILE__), 'spec_helper_local.rb')) 10 | rescue LoadError => loaderror 11 | warn "Could not require spec_helper_local: #{loaderror.message}" 12 | end 13 | 14 | require 'simplecov' 15 | require 'simplecov-console' 16 | 17 | SimpleCov.start do 18 | add_filter '/.bundle' 19 | add_filter '/spec' 20 | formatter SimpleCov::Formatter::MultiFormatter.new([SimpleCov::Formatter::HTMLFormatter, SimpleCov::Formatter::Console]) 21 | end 22 | 23 | include RspecPuppetFacts 24 | 25 | default_facts = { 26 | puppetversion: Puppet.version, 27 | facterversion: Facter.version, 28 | } 29 | 30 | default_facts_path = File.expand_path(File.join(File.dirname(__FILE__), 'default_facts.yml')) 31 | default_module_facts_path = File.expand_path(File.join(File.dirname(__FILE__), 'default_module_facts.yml')) 32 | 33 | if File.exist?(default_facts_path) && File.readable?(default_facts_path) 34 | default_facts.merge!(YAML.safe_load(File.read(default_facts_path))) 35 | end 36 | 37 | if File.exist?(default_module_facts_path) && File.readable?(default_module_facts_path) 38 | default_facts.merge!(YAML.safe_load(File.read(default_module_facts_path))) 39 | end 40 | 41 | RSpec.configure do |c| 42 | c.default_facts = default_facts 43 | c.before :each do 44 | # set to strictest setting for testing 45 | # by default Puppet runs at warning level 46 | Puppet.settings[:strict] = :warning 47 | end 48 | c.mock_with :rspec 49 | end 50 | 51 | def ensure_module_defined(module_name) 52 | module_name.split('::').reduce(Object) do |last_module, next_module| 53 | last_module.const_set(next_module, Module.new) unless last_module.const_defined?(next_module) 54 | last_module.const_get(next_module) 55 | end 56 | end 57 | 58 | # 'spec_overrides' from sync.yml will appear below this line 59 | -------------------------------------------------------------------------------- /spec/spec_helper_acceptance.rb: -------------------------------------------------------------------------------- 1 | require 'beaker-pe' 2 | require 'beaker-puppet' 3 | require 'beaker-rspec/helpers/serverspec' 4 | require 'beaker-rspec/spec_helper' 5 | require 'beaker-task_helper' 6 | require 'beaker/module_install_helper' 7 | require 'beaker/puppet_install_helper' 8 | 9 | if ENV['BEAKER_provision'] != 'no' 10 | run_puppet_install_helper 11 | install_module_on(hosts) 12 | end 13 | 14 | RSpec.configure do |c| 15 | c.before :suite do 16 | unless ENV['BEAKER_TESTMODE'] == 'local' 17 | unless ENV['BEAKER_provision'] == 'no' 18 | # intentionally blank 19 | end 20 | hosts.each do |host| 21 | # intentionally blank 22 | end 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /spec/unit/puppet_x/puppetlabs/tune/calculate_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | require 'puppet_x/puppetlabs/tune/calculate.rb' 4 | 5 | describe PuppetX::Puppetlabs::Tune::Calculate do 6 | options = {} 7 | subject(:calculator) { described_class.new(options) } 8 | 9 | # Allows mergeups in the PE implementation of this class. 10 | pe_2019_or_newer = Gem::Version.new(Puppet.version) >= Gem::Version.new('6.0.0') 11 | 12 | percent_cpu_puppetdb = 0.25 13 | percent_cpu_puppetdb_with_compilers = 0.50 14 | 15 | percent_ram_database = 0.20 16 | percent_ram_puppetdb = 0.10 17 | percent_ram_puppetdb_with_compilers = 0.15 18 | percent_ram_puppetdb_split = 0.25 19 | percent_ram_puppetdb_split_external = 0.50 20 | percent_ram_orchestrator = 0.08 21 | percent_ram_orchestrator_with_jrubies = 0.10 22 | percent_ram_console = 0.08 23 | percent_ram_activemq = 0.08 24 | 25 | minimum_ram_database_split = 2048 26 | maximum_ram_database = 16384 27 | 28 | minimum_ram_puppetserver = 512 29 | 30 | minimum_ram_orchestrator = 512 31 | maximum_ram_orchestrator = 1024 32 | 33 | minimum_ram_console = 512 34 | maximum_ram_console = 1024 35 | 36 | minimum_ram_activemq = 256 37 | maximum_ram_activemq = 512 38 | 39 | ram_per_jruby_code_cache = 96 40 | 41 | context 'with a monolithic infrastructure' do 42 | it 'can calculate master host settings, in vmpooler' do 43 | resources = { 44 | 'cpu' => 2, 45 | 'ram' => 6144, 46 | } 47 | infrastructure = { 48 | 'is_monolithic' => true, 49 | 'with_compile_masters' => false, 50 | } 51 | type = { 52 | 'is_monolithic_master' => true, 53 | 'is_replica_master' => false, 54 | 'is_compile_master' => false, 55 | 'with_jruby9k_enabled' => false, 56 | } 57 | classes = { 58 | 'amq::broker' => true, 59 | 'console' => true, 60 | 'database' => true, 61 | 'orchestrator' => true, 62 | 'puppetdb' => true, 63 | } 64 | node = { 'resources' => resources, 'infrastructure' => infrastructure, 'type' => type, 'classes' => classes } 65 | 66 | cpu_puppetdb = 1 67 | cpu_puppetserver = 1 68 | ram_per_jruby = 512 69 | ram_database = (resources['ram'] * percent_ram_database).to_i 70 | ram_puppetdb = (resources['ram'] * percent_ram_puppetdb).to_i 71 | ram_puppetserver = minimum_ram_puppetserver 72 | ram_orchestrator = minimum_ram_orchestrator 73 | ram_console = minimum_ram_console 74 | ram_activemq = (resources['ram'] * percent_ram_activemq).to_i.clamp(minimum_ram_activemq, maximum_ram_activemq) 75 | 76 | params = { 77 | 'puppet_enterprise::profile::database::shared_buffers' => "#{ram_database}MB", 78 | 'puppet_enterprise::puppetdb::command_processing_threads' => cpu_puppetdb, 79 | 'puppet_enterprise::master::puppetserver::jruby_max_active_instances' => cpu_puppetserver, 80 | 'puppet_enterprise::profile::puppetdb::java_args' => { 'Xms' => "#{ram_puppetdb}m", 'Xmx' => "#{ram_puppetdb}m" }, 81 | 'puppet_enterprise::profile::master::java_args' => { 'Xms' => "#{ram_puppetserver}m", 'Xmx' => "#{ram_puppetserver}m" }, 82 | 'puppet_enterprise::profile::orchestrator::java_args' => { 'Xms' => "#{ram_orchestrator}m", 'Xmx' => "#{ram_orchestrator}m" }, 83 | 'puppet_enterprise::profile::console::java_args' => { 'Xms' => "#{ram_console}m", 'Xmx' => "#{ram_console}m" }, 84 | 'puppet_enterprise::profile::amq::broker::heap_mb' => ram_activemq, 85 | } 86 | 87 | total_cpu = params['puppet_enterprise::puppetdb::command_processing_threads'] + 88 | params['puppet_enterprise::master::puppetserver::jruby_max_active_instances'] 89 | total_ram = ram_database + ram_puppetdb + ram_puppetserver + ram_orchestrator + ram_console + ram_activemq 90 | totals = { 91 | 'CPU' => { 'total' => resources['cpu'], 'used' => total_cpu }, 92 | 'RAM' => { 'total' => resources['ram'], 'used' => total_ram }, 93 | 'MB_PER_JRUBY' => ram_per_jruby, 94 | } 95 | 96 | settings = { 'params' => params, 'totals' => totals } 97 | 98 | if pe_2019_or_newer 99 | node['type']['with_jruby9k_enabled'] = true 100 | node['classes'].delete('amq::broker') 101 | reserved_code_cache = settings['params']['puppet_enterprise::master::puppetserver::jruby_max_active_instances'] * ram_per_jruby_code_cache 102 | settings['params']['puppet_enterprise::master::puppetserver::reserved_code_cache'] = "#{reserved_code_cache}m" 103 | settings['totals']['RAM']['used'] += reserved_code_cache 104 | settings['totals']['RAM']['used'] -= settings['params']['puppet_enterprise::profile::amq::broker::heap_mb'] 105 | settings['params'].delete('puppet_enterprise::profile::amq::broker::heap_mb') 106 | end 107 | 108 | expect(calculator::calculate_master_settings(node)).to eq(settings) 109 | end 110 | 111 | it 'can calculate master host settings, server size small' do 112 | resources = { 113 | 'cpu' => 4, 114 | 'ram' => 8192, 115 | } 116 | infrastructure = { 117 | 'is_monolithic' => true, 118 | 'with_compile_masters' => false, 119 | } 120 | type = { 121 | 'is_monolithic_master' => true, 122 | 'is_replica_master' => false, 123 | 'is_compile_master' => false, 124 | 'with_jruby9k_enabled' => false, 125 | } 126 | classes = { 127 | 'amq::broker' => true, 128 | 'console' => true, 129 | 'database' => true, 130 | 'orchestrator' => true, 131 | 'puppetdb' => true, 132 | } 133 | node = { 'resources' => resources, 'infrastructure' => infrastructure, 'type' => type, 'classes' => classes } 134 | 135 | cpu_puppetdb = (resources['cpu'] * percent_cpu_puppetdb).to_i 136 | cpu_puppetserver = 2 137 | ram_per_jruby = 512 138 | ram_database = (resources['ram'] * percent_ram_database).to_i 139 | ram_puppetdb = (resources['ram'] * percent_ram_puppetdb).to_i 140 | ram_puppetserver = cpu_puppetserver * ram_per_jruby 141 | ram_orchestrator = (resources['ram'] * percent_ram_orchestrator).to_i 142 | ram_console = (resources['ram'] * percent_ram_console).to_i 143 | ram_activemq = (resources['ram'] * percent_ram_activemq).to_i.clamp(minimum_ram_activemq, maximum_ram_activemq) 144 | 145 | params = { 146 | 'puppet_enterprise::profile::database::shared_buffers' => "#{ram_database}MB", 147 | 'puppet_enterprise::puppetdb::command_processing_threads' => cpu_puppetdb, 148 | 'puppet_enterprise::master::puppetserver::jruby_max_active_instances' => cpu_puppetserver, 149 | 'puppet_enterprise::profile::puppetdb::java_args' => { 'Xms' => "#{ram_puppetdb}m", 'Xmx' => "#{ram_puppetdb}m" }, 150 | 'puppet_enterprise::profile::master::java_args' => { 'Xms' => "#{ram_puppetserver}m", 'Xmx' => "#{ram_puppetserver}m" }, 151 | 'puppet_enterprise::profile::orchestrator::java_args' => { 'Xms' => "#{ram_orchestrator}m", 'Xmx' => "#{ram_orchestrator}m" }, 152 | 'puppet_enterprise::profile::console::java_args' => { 'Xms' => "#{ram_console}m", 'Xmx' => "#{ram_console}m" }, 153 | 'puppet_enterprise::profile::amq::broker::heap_mb' => ram_activemq, 154 | } 155 | 156 | total_cpu = params['puppet_enterprise::puppetdb::command_processing_threads'] + 157 | params['puppet_enterprise::master::puppetserver::jruby_max_active_instances'] 158 | total_ram = ram_database + ram_puppetdb + ram_puppetserver + ram_orchestrator + ram_console + ram_activemq 159 | totals = { 160 | 'CPU' => { 'total' => resources['cpu'], 'used' => total_cpu }, 161 | 'RAM' => { 'total' => resources['ram'], 'used' => total_ram }, 162 | 'MB_PER_JRUBY' => ram_per_jruby, 163 | } 164 | 165 | settings = { 'params' => params, 'totals' => totals } 166 | 167 | if pe_2019_or_newer 168 | node['type']['with_jruby9k_enabled'] = true 169 | node['classes'].delete('amq::broker') 170 | reserved_code_cache = settings['params']['puppet_enterprise::master::puppetserver::jruby_max_active_instances'] * ram_per_jruby_code_cache 171 | settings['params']['puppet_enterprise::master::puppetserver::reserved_code_cache'] = "#{reserved_code_cache}m" 172 | settings['totals']['RAM']['used'] += reserved_code_cache 173 | settings['totals']['RAM']['used'] -= settings['params']['puppet_enterprise::profile::amq::broker::heap_mb'] 174 | settings['params'].delete('puppet_enterprise::profile::amq::broker::heap_mb') 175 | end 176 | 177 | expect(calculator::calculate_master_settings(node)).to eq(settings) 178 | end 179 | 180 | it 'can calculate master host settings, server size medium' do 181 | resources = { 182 | 'cpu' => 8, 183 | 'ram' => 16384, 184 | } 185 | infrastructure = { 186 | 'is_monolithic' => true, 187 | 'with_compile_masters' => false, 188 | } 189 | type = { 190 | 'is_monolithic_master' => true, 191 | 'is_replica_master' => false, 192 | 'is_compile_master' => false, 193 | 'with_jruby9k_enabled' => false, 194 | } 195 | classes = { 196 | 'amq::broker' => true, 197 | 'console' => true, 198 | 'database' => true, 199 | 'orchestrator' => true, 200 | 'puppetdb' => true, 201 | } 202 | node = { 'resources' => resources, 'infrastructure' => infrastructure, 'type' => type, 'classes' => classes } 203 | 204 | cpu_puppetdb = (resources['cpu'] * percent_cpu_puppetdb).to_i 205 | cpu_puppetserver = 5 206 | ram_per_jruby = 768 207 | ram_database = (resources['ram'] * percent_ram_database).to_i 208 | ram_puppetdb = (resources['ram'] * percent_ram_puppetdb).to_i 209 | ram_puppetserver = cpu_puppetserver * ram_per_jruby 210 | ram_orchestrator = maximum_ram_orchestrator 211 | ram_console = maximum_ram_console 212 | ram_activemq = maximum_ram_activemq 213 | 214 | params = { 215 | 'puppet_enterprise::profile::database::shared_buffers' => "#{ram_database}MB", 216 | 'puppet_enterprise::puppetdb::command_processing_threads' => cpu_puppetdb, 217 | 'puppet_enterprise::master::puppetserver::jruby_max_active_instances' => cpu_puppetserver, 218 | 'puppet_enterprise::profile::puppetdb::java_args' => { 'Xms' => "#{ram_puppetdb}m", 'Xmx' => "#{ram_puppetdb}m" }, 219 | 'puppet_enterprise::profile::master::java_args' => { 'Xms' => "#{ram_puppetserver}m", 'Xmx' => "#{ram_puppetserver}m" }, 220 | 'puppet_enterprise::profile::orchestrator::java_args' => { 'Xms' => "#{ram_orchestrator}m", 'Xmx' => "#{ram_orchestrator}m" }, 221 | 'puppet_enterprise::profile::console::java_args' => { 'Xms' => "#{ram_console}m", 'Xmx' => "#{ram_console}m" }, 222 | 'puppet_enterprise::profile::amq::broker::heap_mb' => ram_activemq, 223 | } 224 | 225 | total_cpu = params['puppet_enterprise::puppetdb::command_processing_threads'] + 226 | params['puppet_enterprise::master::puppetserver::jruby_max_active_instances'] 227 | total_ram = ram_database + ram_puppetdb + ram_puppetserver + ram_orchestrator + ram_console + ram_activemq 228 | totals = { 229 | 'CPU' => { 'total' => resources['cpu'], 'used' => total_cpu }, 230 | 'RAM' => { 'total' => resources['ram'], 'used' => total_ram }, 231 | 'MB_PER_JRUBY' => ram_per_jruby, 232 | } 233 | 234 | settings = { 'params' => params, 'totals' => totals } 235 | 236 | if pe_2019_or_newer 237 | node['type']['with_jruby9k_enabled'] = true 238 | node['classes'].delete('amq::broker') 239 | reserved_code_cache = settings['params']['puppet_enterprise::master::puppetserver::jruby_max_active_instances'] * ram_per_jruby_code_cache 240 | settings['params']['puppet_enterprise::master::puppetserver::reserved_code_cache'] = "#{reserved_code_cache}m" 241 | settings['totals']['RAM']['used'] += reserved_code_cache 242 | settings['totals']['RAM']['used'] -= settings['params']['puppet_enterprise::profile::amq::broker::heap_mb'] 243 | settings['params'].delete('puppet_enterprise::profile::amq::broker::heap_mb') 244 | end 245 | 246 | expect(calculator::calculate_master_settings(node)).to eq(settings) 247 | end 248 | 249 | it 'can calculate master host settings, server size large' do 250 | resources = { 251 | 'cpu' => 16, 252 | 'ram' => 32768, 253 | } 254 | infrastructure = { 255 | 'is_monolithic' => true, 256 | 'with_compile_masters' => false, 257 | } 258 | type = { 259 | 'is_monolithic_master' => true, 260 | 'is_replica_master' => false, 261 | 'is_compile_master' => false, 262 | 'with_jruby9k_enabled' => false, 263 | } 264 | classes = { 265 | 'amq::broker' => true, 266 | 'console' => true, 267 | 'database' => true, 268 | 'orchestrator' => true, 269 | 'puppetdb' => true, 270 | } 271 | node = { 'resources' => resources, 'infrastructure' => infrastructure, 'type' => type, 'classes' => classes } 272 | 273 | cpu_puppetdb = (resources['cpu'] * percent_cpu_puppetdb).to_i 274 | cpu_puppetserver = 11 275 | ram_per_jruby = 1024 276 | ram_database = (resources['ram'] * percent_ram_database).to_i 277 | ram_puppetdb = (resources['ram'] * percent_ram_puppetdb).to_i 278 | ram_puppetserver = cpu_puppetserver * ram_per_jruby 279 | ram_orchestrator = maximum_ram_orchestrator 280 | ram_console = maximum_ram_console 281 | ram_activemq = maximum_ram_activemq 282 | 283 | params = { 284 | 'puppet_enterprise::profile::database::shared_buffers' => "#{ram_database}MB", 285 | 'puppet_enterprise::puppetdb::command_processing_threads' => cpu_puppetdb, 286 | 'puppet_enterprise::master::puppetserver::jruby_max_active_instances' => cpu_puppetserver, 287 | 'puppet_enterprise::profile::puppetdb::java_args' => { 'Xms' => "#{ram_puppetdb}m", 'Xmx' => "#{ram_puppetdb}m" }, 288 | 'puppet_enterprise::profile::master::java_args' => { 'Xms' => "#{ram_puppetserver}m", 'Xmx' => "#{ram_puppetserver}m" }, 289 | 'puppet_enterprise::profile::orchestrator::java_args' => { 'Xms' => "#{ram_orchestrator}m", 'Xmx' => "#{ram_orchestrator}m" }, 290 | 'puppet_enterprise::profile::console::java_args' => { 'Xms' => "#{ram_console}m", 'Xmx' => "#{ram_console}m" }, 291 | 'puppet_enterprise::profile::amq::broker::heap_mb' => ram_activemq, 292 | } 293 | 294 | total_cpu = params['puppet_enterprise::puppetdb::command_processing_threads'] + 295 | params['puppet_enterprise::master::puppetserver::jruby_max_active_instances'] 296 | total_ram = ram_database + ram_puppetdb + ram_puppetserver + ram_orchestrator + ram_console + ram_activemq 297 | totals = { 298 | 'CPU' => { 'total' => resources['cpu'], 'used' => total_cpu }, 299 | 'RAM' => { 'total' => resources['ram'], 'used' => total_ram }, 300 | 'MB_PER_JRUBY' => ram_per_jruby, 301 | } 302 | 303 | settings = { 'params' => params, 'totals' => totals } 304 | 305 | if pe_2019_or_newer 306 | node['type']['with_jruby9k_enabled'] = true 307 | node['classes'].delete('amq::broker') 308 | reserved_code_cache = settings['params']['puppet_enterprise::master::puppetserver::jruby_max_active_instances'] * ram_per_jruby_code_cache 309 | settings['params']['puppet_enterprise::master::puppetserver::reserved_code_cache'] = "#{reserved_code_cache}m" 310 | settings['totals']['RAM']['used'] += reserved_code_cache 311 | settings['totals']['RAM']['used'] -= settings['params']['puppet_enterprise::profile::amq::broker::heap_mb'] 312 | settings['params'].delete('puppet_enterprise::profile::amq::broker::heap_mb') 313 | end 314 | 315 | expect(calculator::calculate_master_settings(node)).to eq(settings) 316 | end 317 | 318 | it 'can calculate master host settings, server size large, with jruby9k and orchestrator jruby' do 319 | resources = { 320 | 'cpu' => 16, 321 | 'ram' => 32768, 322 | } 323 | infrastructure = { 324 | 'is_monolithic' => true, 325 | 'with_compile_masters' => false, 326 | } 327 | type = { 328 | 'is_monolithic_master' => true, 329 | 'is_replica_master' => false, 330 | 'is_compile_master' => false, 331 | 'with_jruby9k_enabled' => true, 332 | 'with_orchestrator_jruby' => true, 333 | } 334 | classes = { 335 | 'console' => true, 336 | 'database' => true, 337 | 'orchestrator' => true, 338 | 'puppetdb' => true, 339 | } 340 | node = { 'resources' => resources, 'infrastructure' => infrastructure, 'type' => type, 'classes' => classes } 341 | 342 | cpu_puppetdb = (resources['cpu'] * percent_cpu_puppetdb).to_i 343 | cpu_puppetserver = 9 344 | ram_per_jruby = 1024 345 | ram_database = (resources['ram'] * percent_ram_database).to_i 346 | ram_puppetdb = (resources['ram'] * percent_ram_puppetdb).to_i 347 | ram_orchestrator = (resources['ram'] * percent_ram_orchestrator_with_jrubies).to_i 348 | ram_puppetserver = cpu_puppetserver * ram_per_jruby 349 | ram_puppetserver_cc = cpu_puppetserver * ram_per_jruby_code_cache 350 | ram_console = maximum_ram_console 351 | 352 | cpu_orchestrator = 3 353 | # ram_orchestrator_cc = cpu_orchestrator * ram_per_jruby_code_cache 354 | 355 | params = { 356 | 'puppet_enterprise::profile::database::shared_buffers' => "#{ram_database}MB", 357 | 'puppet_enterprise::puppetdb::command_processing_threads' => cpu_puppetdb, 358 | 'puppet_enterprise::master::puppetserver::jruby_max_active_instances' => cpu_puppetserver, 359 | 'puppet_enterprise::master::puppetserver::reserved_code_cache' => "#{ram_puppetserver_cc}m", 360 | 'puppet_enterprise::profile::puppetdb::java_args' => { 'Xms' => "#{ram_puppetdb}m", 'Xmx' => "#{ram_puppetdb}m" }, 361 | 'puppet_enterprise::profile::master::java_args' => { 'Xms' => "#{ram_puppetserver}m", 'Xmx' => "#{ram_puppetserver}m" }, 362 | 'puppet_enterprise::profile::orchestrator::jruby_max_active_instances' => cpu_orchestrator, 363 | # 'puppet_enterprise::profile::orchestrator::reserved_code_cache' => "#{ram_orchestrator_cc}m", 364 | 'puppet_enterprise::profile::orchestrator::java_args' => { 'Xms' => "#{ram_orchestrator}m", 'Xmx' => "#{ram_orchestrator}m" }, 365 | 'puppet_enterprise::profile::console::java_args' => { 'Xms' => "#{ram_console}m", 'Xmx' => "#{ram_console}m" }, 366 | } 367 | 368 | total_cpu = params['puppet_enterprise::puppetdb::command_processing_threads'] + 369 | params['puppet_enterprise::master::puppetserver::jruby_max_active_instances'] 370 | total_ram = ram_database + ram_puppetdb + ram_puppetserver + ram_puppetserver_cc + ram_orchestrator + ram_console 371 | totals = { 372 | 'CPU' => { 'total' => resources['cpu'], 'used' => total_cpu }, 373 | 'RAM' => { 'total' => resources['ram'], 'used' => total_ram }, 374 | 'MB_PER_JRUBY' => ram_per_jruby, 375 | } 376 | 377 | settings = { 'params' => params, 'totals' => totals } 378 | 379 | expect(calculator::calculate_master_settings(node)).to eq(settings) 380 | end 381 | 382 | it 'can calculate master host settings with compilers' do 383 | resources = { 384 | 'cpu' => 4, 385 | 'ram' => 8192, 386 | } 387 | infrastructure = { 388 | 'is_monolithic' => true, 389 | 'with_compile_masters' => true, 390 | } 391 | type = { 392 | 'is_monolithic_master' => true, 393 | 'is_replica_master' => false, 394 | 'is_compile_master' => false, 395 | 'with_jruby9k_enabled' => false, 396 | } 397 | classes = { 398 | 'amq::broker' => true, 399 | 'console' => true, 400 | 'database' => true, 401 | 'orchestrator' => true, 402 | 'puppetdb' => true, 403 | } 404 | node = { 'resources' => resources, 'infrastructure' => infrastructure, 'type' => type, 'classes' => classes } 405 | 406 | cpu_puppetdb = (resources['cpu'] * percent_cpu_puppetdb_with_compilers).to_i 407 | cpu_puppetserver = 2 408 | ram_per_jruby = 512 409 | ram_database = (resources['ram'] * percent_ram_database).to_i 410 | ram_puppetdb = (resources['ram'] * percent_ram_puppetdb_with_compilers).to_i 411 | ram_puppetserver = cpu_puppetserver * ram_per_jruby 412 | ram_orchestrator = (resources['ram'] * percent_ram_orchestrator).to_i 413 | ram_console = (resources['ram'] * percent_ram_console).to_i 414 | ram_activemq = (resources['ram'] * percent_ram_activemq).to_i.clamp(minimum_ram_activemq, maximum_ram_activemq) 415 | 416 | params = { 417 | 'puppet_enterprise::profile::database::shared_buffers' => "#{ram_database}MB", 418 | 'puppet_enterprise::puppetdb::command_processing_threads' => cpu_puppetdb, 419 | 'puppet_enterprise::master::puppetserver::jruby_max_active_instances' => cpu_puppetserver, 420 | 'puppet_enterprise::profile::puppetdb::java_args' => { 'Xms' => "#{ram_puppetdb}m", 'Xmx' => "#{ram_puppetdb}m" }, 421 | 'puppet_enterprise::profile::master::java_args' => { 'Xms' => "#{ram_puppetserver}m", 'Xmx' => "#{ram_puppetserver}m" }, 422 | 'puppet_enterprise::profile::orchestrator::java_args' => { 'Xms' => "#{ram_orchestrator}m", 'Xmx' => "#{ram_orchestrator}m" }, 423 | 'puppet_enterprise::profile::console::java_args' => { 'Xms' => "#{ram_console}m", 'Xmx' => "#{ram_console}m" }, 424 | 'puppet_enterprise::profile::amq::broker::heap_mb' => ram_activemq, 425 | } 426 | 427 | total_cpu = params['puppet_enterprise::puppetdb::command_processing_threads'] + 428 | params['puppet_enterprise::master::puppetserver::jruby_max_active_instances'] 429 | total_ram = ram_database + ram_puppetdb + ram_puppetserver + ram_orchestrator + ram_console + ram_activemq 430 | totals = { 431 | 'CPU' => { 'total' => resources['cpu'], 'used' => total_cpu }, 432 | 'RAM' => { 'total' => resources['ram'], 'used' => total_ram }, 433 | 'MB_PER_JRUBY' => ram_per_jruby, 434 | } 435 | 436 | settings = { 'params' => params, 'totals' => totals } 437 | 438 | if pe_2019_or_newer 439 | node['type']['with_jruby9k_enabled'] = true 440 | node['classes'].delete('amq::broker') 441 | reserved_code_cache = settings['params']['puppet_enterprise::master::puppetserver::jruby_max_active_instances'] * ram_per_jruby_code_cache 442 | settings['params']['puppet_enterprise::master::puppetserver::reserved_code_cache'] = "#{reserved_code_cache}m" 443 | settings['totals']['RAM']['used'] += reserved_code_cache 444 | settings['totals']['RAM']['used'] -= settings['params']['puppet_enterprise::profile::amq::broker::heap_mb'] 445 | settings['params'].delete('puppet_enterprise::profile::amq::broker::heap_mb') 446 | end 447 | 448 | expect(calculator::calculate_master_settings(node)).to eq(settings) 449 | end 450 | 451 | it 'can calculate master host settings with compilers and an external database' do 452 | resources = { 453 | 'cpu' => 4, 454 | 'ram' => 8192, 455 | } 456 | infrastructure = { 457 | 'is_monolithic' => true, 458 | 'with_compile_masters' => true, 459 | } 460 | type = { 461 | 'is_monolithic_master' => true, 462 | 'is_replica_master' => false, 463 | 'is_compile_master' => false, 464 | 'with_jruby9k_enabled' => false, 465 | } 466 | classes = { 467 | 'amq::broker' => true, 468 | 'console' => true, 469 | 'database' => false, 470 | 'orchestrator' => true, 471 | 'puppetdb' => true, 472 | } 473 | node = { 'resources' => resources, 'infrastructure' => infrastructure, 'type' => type, 'classes' => classes } 474 | 475 | cpu_puppetdb = (resources['cpu'] * percent_cpu_puppetdb_with_compilers).to_i 476 | cpu_puppetserver = 2 477 | ram_per_jruby = 512 478 | ram_database = 0 479 | ram_puppetdb = (resources['ram'] * percent_ram_puppetdb_with_compilers).to_i 480 | ram_puppetserver = cpu_puppetserver * ram_per_jruby 481 | ram_orchestrator = (resources['ram'] * percent_ram_orchestrator).to_i 482 | ram_console = (resources['ram'] * percent_ram_console).to_i 483 | ram_activemq = (resources['ram'] * percent_ram_activemq).to_i.clamp(minimum_ram_activemq, maximum_ram_activemq) 484 | 485 | params = { 486 | 'puppet_enterprise::puppetdb::command_processing_threads' => cpu_puppetdb, 487 | 'puppet_enterprise::master::puppetserver::jruby_max_active_instances' => cpu_puppetserver, 488 | 'puppet_enterprise::profile::puppetdb::java_args' => { 'Xms' => "#{ram_puppetdb}m", 'Xmx' => "#{ram_puppetdb}m" }, 489 | 'puppet_enterprise::profile::master::java_args' => { 'Xms' => "#{ram_puppetserver}m", 'Xmx' => "#{ram_puppetserver}m" }, 490 | 'puppet_enterprise::profile::orchestrator::java_args' => { 'Xms' => "#{ram_orchestrator}m", 'Xmx' => "#{ram_orchestrator}m" }, 491 | 'puppet_enterprise::profile::console::java_args' => { 'Xms' => "#{ram_console}m", 'Xmx' => "#{ram_console}m" }, 492 | 'puppet_enterprise::profile::amq::broker::heap_mb' => ram_activemq, 493 | } 494 | 495 | total_cpu = params['puppet_enterprise::puppetdb::command_processing_threads'] + 496 | params['puppet_enterprise::master::puppetserver::jruby_max_active_instances'] 497 | total_ram = ram_database + ram_puppetdb + ram_puppetserver + ram_orchestrator + ram_console + ram_activemq 498 | totals = { 499 | 'CPU' => { 'total' => resources['cpu'], 'used' => total_cpu }, 500 | 'RAM' => { 'total' => resources['ram'], 'used' => total_ram }, 501 | 'MB_PER_JRUBY' => ram_per_jruby, 502 | } 503 | 504 | settings = { 'params' => params, 'totals' => totals } 505 | 506 | if pe_2019_or_newer 507 | node['type']['with_jruby9k_enabled'] = true 508 | node['classes'].delete('amq::broker') 509 | reserved_code_cache = settings['params']['puppet_enterprise::master::puppetserver::jruby_max_active_instances'] * ram_per_jruby_code_cache 510 | settings['params']['puppet_enterprise::master::puppetserver::reserved_code_cache'] = "#{reserved_code_cache}m" 511 | settings['totals']['RAM']['used'] += reserved_code_cache 512 | settings['totals']['RAM']['used'] -= settings['params']['puppet_enterprise::profile::amq::broker::heap_mb'] 513 | settings['params'].delete('puppet_enterprise::profile::amq::broker::heap_mb') 514 | end 515 | 516 | expect(calculator::calculate_master_settings(node)).to eq(settings) 517 | end 518 | 519 | it 'can calculate master host settings with a large number of compiler connections' do 520 | resources = { 521 | 'cpu' => 4, 522 | 'ram' => 8192, 523 | } 524 | infrastructure = { 525 | 'is_monolithic' => true, 526 | 'with_compile_masters' => true, 527 | 'compiler_connections' => 500, 528 | } 529 | type = { 530 | 'is_monolithic_master' => true, 531 | 'is_replica_master' => false, 532 | 'is_compile_master' => false, 533 | 'with_jruby9k_enabled' => false, 534 | } 535 | classes = { 536 | 'amq::broker' => true, 537 | 'console' => true, 538 | 'database' => true, 539 | 'orchestrator' => true, 540 | 'puppetdb' => true, 541 | } 542 | node = { 'resources' => resources, 'infrastructure' => infrastructure, 'type' => type, 'classes' => classes } 543 | 544 | cpu_puppetdb = (resources['cpu'] * percent_cpu_puppetdb_with_compilers).to_i 545 | cpu_puppetserver = 2 546 | ram_per_jruby = 512 547 | ram_database = (resources['ram'] * percent_ram_database).to_i 548 | ram_puppetdb = (resources['ram'] * percent_ram_puppetdb_with_compilers).to_i 549 | ram_puppetserver = cpu_puppetserver * ram_per_jruby 550 | ram_orchestrator = (resources['ram'] * percent_ram_orchestrator).to_i 551 | ram_console = (resources['ram'] * percent_ram_console).to_i 552 | ram_activemq = (resources['ram'] * percent_ram_activemq).to_i.clamp(minimum_ram_activemq, maximum_ram_activemq) 553 | 554 | max_connections = (infrastructure['compiler_connections'] * 1.10).to_i 555 | 556 | params = { 557 | 'puppet_enterprise::profile::database::shared_buffers' => "#{ram_database}MB", 558 | 'puppet_enterprise::puppetdb::command_processing_threads' => cpu_puppetdb, 559 | 'puppet_enterprise::master::puppetserver::jruby_max_active_instances' => cpu_puppetserver, 560 | 'puppet_enterprise::profile::puppetdb::java_args' => { 'Xms' => "#{ram_puppetdb}m", 'Xmx' => "#{ram_puppetdb}m" }, 561 | 'puppet_enterprise::profile::master::java_args' => { 'Xms' => "#{ram_puppetserver}m", 'Xmx' => "#{ram_puppetserver}m" }, 562 | 'puppet_enterprise::profile::orchestrator::java_args' => { 'Xms' => "#{ram_orchestrator}m", 'Xmx' => "#{ram_orchestrator}m" }, 563 | 'puppet_enterprise::profile::console::java_args' => { 'Xms' => "#{ram_console}m", 'Xmx' => "#{ram_console}m" }, 564 | 'puppet_enterprise::profile::amq::broker::heap_mb' => ram_activemq, 565 | 'puppet_enterprise::profile::database::max_connections' => max_connections, 566 | } 567 | 568 | total_cpu = params['puppet_enterprise::puppetdb::command_processing_threads'] + 569 | params['puppet_enterprise::master::puppetserver::jruby_max_active_instances'] 570 | total_ram = ram_database + ram_puppetdb + ram_puppetserver + ram_orchestrator + ram_console + ram_activemq 571 | totals = { 572 | 'CPU' => { 'total' => resources['cpu'], 'used' => total_cpu }, 573 | 'RAM' => { 'total' => resources['ram'], 'used' => total_ram }, 574 | 'MB_PER_JRUBY' => ram_per_jruby, 575 | } 576 | 577 | settings = { 'params' => params, 'totals' => totals } 578 | 579 | if pe_2019_or_newer 580 | node['type']['with_jruby9k_enabled'] = true 581 | node['classes'].delete('amq::broker') 582 | reserved_code_cache = settings['params']['puppet_enterprise::master::puppetserver::jruby_max_active_instances'] * ram_per_jruby_code_cache 583 | settings['params']['puppet_enterprise::master::puppetserver::reserved_code_cache'] = "#{reserved_code_cache}m" 584 | settings['totals']['RAM']['used'] += reserved_code_cache 585 | settings['totals']['RAM']['used'] -= settings['params']['puppet_enterprise::profile::amq::broker::heap_mb'] 586 | settings['params'].delete('puppet_enterprise::profile::amq::broker::heap_mb') 587 | end 588 | 589 | expect(calculator::calculate_master_settings(node)).to eq(settings) 590 | end 591 | end 592 | 593 | context 'with a split infrastructure' do 594 | it 'can calculate master host settings' do 595 | resources = { 596 | 'cpu' => 4, 597 | 'ram' => 8192, 598 | } 599 | infrastructure = { 600 | 'is_monolithic' => false, 601 | 'with_compile_masters' => false, 602 | } 603 | type = { 604 | 'is_monolithic_master' => false, 605 | 'is_replica_master' => false, 606 | 'is_compile_master' => false, 607 | 'with_jruby9k_enabled' => false, 608 | } 609 | classes = { 610 | 'amq::broker' => false, 611 | 'console' => false, 612 | 'database' => false, 613 | 'orchestrator' => true, 614 | 'puppetdb' => false, 615 | } 616 | node = { 'resources' => resources, 'infrastructure' => infrastructure, 'type' => type, 'classes' => classes } 617 | 618 | cpu_puppetserver = 3 619 | ram_per_jruby = 512 620 | ram_puppetserver = cpu_puppetserver * ram_per_jruby 621 | ram_orchestrator = (resources['ram'] * percent_ram_orchestrator).to_i 622 | 623 | params = { 624 | 'puppet_enterprise::master::puppetserver::jruby_max_active_instances' => cpu_puppetserver, 625 | 'puppet_enterprise::profile::master::java_args' => { 'Xms' => "#{ram_puppetserver}m", 'Xmx' => "#{ram_puppetserver}m" }, 626 | 'puppet_enterprise::profile::orchestrator::java_args' => { 'Xms' => "#{ram_orchestrator}m", 'Xmx' => "#{ram_orchestrator}m" }, 627 | } 628 | 629 | total_cpu = params['puppet_enterprise::master::puppetserver::jruby_max_active_instances'] 630 | total_ram = ram_puppetserver + ram_orchestrator 631 | totals = { 632 | 'CPU' => { 'total' => resources['cpu'], 'used' => total_cpu }, 633 | 'RAM' => { 'total' => resources['ram'], 'used' => total_ram }, 634 | 'MB_PER_JRUBY' => ram_per_jruby, 635 | } 636 | 637 | settings = { 'params' => params, 'totals' => totals } 638 | 639 | expect(calculator::calculate_master_settings(node)).to eq(settings) 640 | end 641 | 642 | it 'can calculate console host settings' do 643 | resources = { 644 | 'cpu' => 4, 645 | 'ram' => 8192, 646 | } 647 | node = { 'resources' => resources, 'infrastructure' => {}, 'type' => {}, 'classes' => {} } 648 | 649 | params = { 650 | 'puppet_enterprise::profile::console::java_args' => { 'Xms' => '4096m', 'Xmx' => '4096m' }, 651 | } 652 | 653 | totals = { 654 | 'CPU' => { 'total' => 4, 'used' => 0 }, 655 | 'RAM' => { 'total' => 8192, 'used' => 4096 } 656 | } 657 | 658 | settings = { 'params' => params, 'totals' => totals } 659 | 660 | expect(calculator::calculate_console_settings(node)).to eq(settings) 661 | end 662 | 663 | it 'can calculate puppetdb host settings' do 664 | resources = { 665 | 'cpu' => 4, 666 | 'ram' => 8192, 667 | } 668 | infrastructure = { 669 | 'is_monolithic' => false, 670 | 'with_compile_masters' => false, 671 | } 672 | classes = { 673 | 'database' => true, 674 | } 675 | node = { 'resources' => resources, 'infrastructure' => infrastructure, 'type' => {}, 'classes' => classes } 676 | 677 | cpu_puppetdb = 2 678 | ram_database = (resources['ram'] * percent_ram_database).to_i.clamp(minimum_ram_database_split, maximum_ram_database) 679 | ram_puppetdb = (resources['ram'] * percent_ram_puppetdb_split).to_i 680 | 681 | params = { 682 | 'puppet_enterprise::profile::database::shared_buffers' => "#{ram_database}MB", 683 | 'puppet_enterprise::puppetdb::command_processing_threads' => cpu_puppetdb, 684 | 'puppet_enterprise::profile::puppetdb::java_args' => { 'Xms' => "#{ram_puppetdb}m", 'Xmx' => "#{ram_puppetdb}m" }, 685 | } 686 | 687 | total_cpu = params['puppet_enterprise::puppetdb::command_processing_threads'] 688 | total_ram = ram_database + ram_puppetdb 689 | totals = { 690 | 'CPU' => { 'total' => resources['cpu'], 'used' => total_cpu }, 691 | 'RAM' => { 'total' => resources['ram'], 'used' => total_ram }, 692 | } 693 | 694 | settings = { 'params' => params, 'totals' => totals } 695 | 696 | expect(calculator::calculate_puppetdb_settings(node)).to eq(settings) 697 | end 698 | 699 | it 'can calculate puppetdb host settings with an external database' do 700 | resources = { 701 | 'cpu' => 4, 702 | 'ram' => 8192, 703 | } 704 | infrastructure = { 705 | 'is_monolithic' => false, 706 | 'with_compile_masters' => false, 707 | } 708 | classes = { 709 | 'database' => false, 710 | } 711 | node = { 'resources' => resources, 'infrastructure' => infrastructure, 'type' => {}, 'classes' => classes } 712 | 713 | cpu_puppetdb = 2 714 | ram_puppetdb = (resources['ram'] * percent_ram_puppetdb_split_external).to_i 715 | 716 | params = { 717 | 'puppet_enterprise::puppetdb::command_processing_threads' => cpu_puppetdb, 718 | 'puppet_enterprise::profile::puppetdb::java_args' => { 'Xms' => "#{ram_puppetdb}m", 'Xmx' => "#{ram_puppetdb}m" }, 719 | } 720 | 721 | total_cpu = params['puppet_enterprise::puppetdb::command_processing_threads'] 722 | total_ram = ram_puppetdb 723 | totals = { 724 | 'CPU' => { 'total' => resources['cpu'], 'used' => total_cpu }, 725 | 'RAM' => { 'total' => resources['ram'], 'used' => total_ram }, 726 | } 727 | 728 | settings = { 'params' => params, 'totals' => totals } 729 | 730 | expect(calculator::calculate_puppetdb_settings(node)).to eq(settings) 731 | end 732 | end 733 | 734 | context 'with a monolithic or split infrastructure' do 735 | it 'can calculate compile master host settings' do 736 | resources = { 737 | 'cpu' => 8, 738 | 'ram' => 16384, 739 | } 740 | infrastructure = { 741 | 'is_monolithic' => false, 742 | 'with_compile_masters' => true, 743 | } 744 | type = { 745 | 'is_monolithic_master' => false, 746 | 'is_replica_master' => false, 747 | 'is_compile_master' => true, 748 | 'with_jruby9k_enabled' => false, 749 | } 750 | classes = { 751 | 'amq::broker' => false, 752 | 'console' => false, 753 | 'database' => false, 754 | 'orchestrator' => false, 755 | 'puppetdb' => false, 756 | } 757 | node = { 'resources' => resources, 'infrastructure' => infrastructure, 'type' => type, 'classes' => classes } 758 | 759 | cpu_puppetserver = 7 760 | ram_per_jruby = 768 761 | ram_puppetserver = cpu_puppetserver * ram_per_jruby 762 | 763 | params = { 764 | 'puppet_enterprise::master::puppetserver::jruby_max_active_instances' => cpu_puppetserver, 765 | 'puppet_enterprise::profile::master::java_args' => { 'Xms' => "#{ram_puppetserver}m", 'Xmx' => "#{ram_puppetserver}m" }, 766 | } 767 | 768 | total_cpu = params['puppet_enterprise::master::puppetserver::jruby_max_active_instances'] 769 | total_ram = ram_puppetserver 770 | totals = { 771 | 'CPU' => { 'total' => resources['cpu'], 'used' => total_cpu }, 772 | 'RAM' => { 'total' => resources['ram'], 'used' => total_ram }, 773 | 'MB_PER_JRUBY' => ram_per_jruby, 774 | } 775 | 776 | settings = { 'params' => params, 'totals' => totals } 777 | 778 | expect(calculator::calculate_master_settings(node)).to eq(settings) 779 | end 780 | 781 | it 'can calculate database host settings' do 782 | resources = { 783 | 'cpu' => 4, 784 | 'ram' => 8192, 785 | } 786 | infrastructure = { 787 | 'is_monolithic' => false, 788 | 'with_compile_masters' => false, 789 | } 790 | node = { 'resources' => resources, 'infrastructure' => infrastructure, 'type' => {}, 'classes' => {} } 791 | 792 | ram_database = 2048 793 | 794 | params = { 795 | 'puppet_enterprise::profile::database::shared_buffers' => "#{ram_database}MB", 796 | } 797 | 798 | total_cpu = 0 799 | total_ram = ram_database 800 | totals = { 801 | 'CPU' => { 'total' => resources['cpu'], 'used' => total_cpu }, 802 | 'RAM' => { 'total' => resources['ram'], 'used' => total_ram }, 803 | } 804 | 805 | settings = { 'params' => params, 'totals' => totals } 806 | 807 | expect(calculator::calculate_database_settings(node)).to eq(settings) 808 | end 809 | end 810 | 811 | context 'with a monolithic infrastructure with compilers' do 812 | it 'can calculate compiler host settings, server size small' do 813 | resources = { 814 | 'cpu' => 4, 815 | 'ram' => 8192, 816 | } 817 | infrastructure = { 818 | 'is_monolithic' => true, 819 | 'with_compile_masters' => true, 820 | } 821 | type = { 822 | 'is_monolithic_master' => false, 823 | 'is_replica_master' => false, 824 | 'is_compile_master' => true, 825 | 'with_jruby9k_enabled' => false, 826 | } 827 | classes = { 828 | 'amq::broker' => false, 829 | 'console' => false, 830 | 'database' => false, 831 | 'orchestrator' => false, 832 | 'puppetdb' => true, 833 | } 834 | node = { 'resources' => resources, 'infrastructure' => infrastructure, 'type' => type, 'classes' => classes } 835 | 836 | cpu_puppetdb = 1 837 | cpu_puppetserver = 2 838 | ram_per_jruby = 512 839 | ram_puppetdb = (resources['ram'] * percent_ram_puppetdb).to_i 840 | ram_puppetserver = cpu_puppetserver * ram_per_jruby 841 | 842 | params = { 843 | 'puppet_enterprise::puppetdb::command_processing_threads' => cpu_puppetdb, 844 | 'puppet_enterprise::master::puppetserver::jruby_max_active_instances' => cpu_puppetserver, 845 | 'puppet_enterprise::profile::puppetdb::java_args' => { 'Xms' => "#{ram_puppetdb}m", 'Xmx' => "#{ram_puppetdb}m" }, 846 | 'puppet_enterprise::profile::master::java_args' => { 'Xms' => "#{ram_puppetserver}m", 'Xmx' => "#{ram_puppetserver}m" }, 847 | 'puppet_enterprise::puppetdb::write_maximum_pool_size' => 2, 848 | 'puppet_enterprise::puppetdb::read_maximum_pool_size' => 3, 849 | 'puppet_enterprise::profile::puppetdb::gc_interval' => 0, 850 | } 851 | 852 | total_cpu = params['puppet_enterprise::puppetdb::command_processing_threads'] + 853 | params['puppet_enterprise::master::puppetserver::jruby_max_active_instances'] 854 | total_ram = ram_puppetdb + ram_puppetserver 855 | totals = { 856 | 'CPU' => { 'total' => resources['cpu'], 'used' => total_cpu }, 857 | 'RAM' => { 'total' => resources['ram'], 'used' => total_ram }, 858 | 'MB_PER_JRUBY' => ram_per_jruby, 859 | } 860 | 861 | settings = { 'params' => params, 'totals' => totals } 862 | 863 | expect(calculator::calculate_master_settings(node)).to eq(settings) 864 | end 865 | 866 | it 'can calculate compiler host settings, server size medium' do 867 | resources = { 868 | 'cpu' => 8, 869 | 'ram' => 16384, 870 | } 871 | infrastructure = { 872 | 'is_monolithic' => true, 873 | 'with_compile_masters' => true, 874 | } 875 | type = { 876 | 'is_monolithic_master' => false, 877 | 'is_replica_master' => false, 878 | 'is_compile_master' => true, 879 | 'with_jruby9k_enabled' => false, 880 | } 881 | classes = { 882 | 'amq::broker' => false, 883 | 'console' => false, 884 | 'database' => false, 885 | 'orchestrator' => false, 886 | 'puppetdb' => true, 887 | } 888 | node = { 'resources' => resources, 'infrastructure' => infrastructure, 'type' => type, 'classes' => classes } 889 | 890 | cpu_puppetdb = 2 891 | cpu_puppetserver = 5 892 | ram_per_jruby = 768 893 | ram_puppetdb = (resources['ram'] * percent_ram_puppetdb).to_i 894 | ram_puppetserver = cpu_puppetserver * ram_per_jruby 895 | 896 | params = { 897 | 'puppet_enterprise::puppetdb::command_processing_threads' => cpu_puppetdb, 898 | 'puppet_enterprise::master::puppetserver::jruby_max_active_instances' => cpu_puppetserver, 899 | 'puppet_enterprise::profile::puppetdb::java_args' => { 'Xms' => "#{ram_puppetdb}m", 'Xmx' => "#{ram_puppetdb}m" }, 900 | 'puppet_enterprise::profile::master::java_args' => { 'Xms' => "#{ram_puppetserver}m", 'Xmx' => "#{ram_puppetserver}m" }, 901 | 'puppet_enterprise::puppetdb::write_maximum_pool_size' => 4, 902 | 'puppet_enterprise::puppetdb::read_maximum_pool_size' => 7, 903 | 'puppet_enterprise::profile::puppetdb::gc_interval' => 0, 904 | } 905 | 906 | total_cpu = params['puppet_enterprise::puppetdb::command_processing_threads'] + 907 | params['puppet_enterprise::master::puppetserver::jruby_max_active_instances'] 908 | total_ram = ram_puppetdb + ram_puppetserver 909 | totals = { 910 | 'CPU' => { 'total' => resources['cpu'], 'used' => total_cpu }, 911 | 'RAM' => { 'total' => resources['ram'], 'used' => total_ram }, 912 | 'MB_PER_JRUBY' => ram_per_jruby, 913 | } 914 | 915 | settings = { 'params' => params, 'totals' => totals } 916 | 917 | expect(calculator::calculate_master_settings(node)).to eq(settings) 918 | end 919 | 920 | it 'can calculate compiler host settings, server size large' do 921 | resources = { 922 | 'cpu' => 16, 923 | 'ram' => 32768, 924 | } 925 | infrastructure = { 926 | 'is_monolithic' => true, 927 | 'with_compile_masters' => true, 928 | } 929 | type = { 930 | 'is_monolithic_master' => false, 931 | 'is_replica_master' => false, 932 | 'is_compile_master' => true, 933 | 'with_jruby9k_enabled' => false, 934 | } 935 | classes = { 936 | 'amq::broker' => false, 937 | 'console' => false, 938 | 'database' => false, 939 | 'orchestrator' => false, 940 | 'puppetdb' => true, 941 | } 942 | node = { 'resources' => resources, 'infrastructure' => infrastructure, 'type' => type, 'classes' => classes } 943 | 944 | cpu_puppetdb = 3 945 | cpu_puppetserver = 12 946 | ram_per_jruby = 1024 947 | ram_puppetdb = (resources['ram'] * percent_ram_puppetdb).to_i 948 | ram_puppetserver = cpu_puppetserver * ram_per_jruby 949 | 950 | params = { 951 | 'puppet_enterprise::puppetdb::command_processing_threads' => cpu_puppetdb, 952 | 'puppet_enterprise::master::puppetserver::jruby_max_active_instances' => cpu_puppetserver, 953 | 'puppet_enterprise::profile::puppetdb::java_args' => { 'Xms' => "#{ram_puppetdb}m", 'Xmx' => "#{ram_puppetdb}m" }, 954 | 'puppet_enterprise::profile::master::java_args' => { 'Xms' => "#{ram_puppetserver}m", 'Xmx' => "#{ram_puppetserver}m" }, 955 | 'puppet_enterprise::puppetdb::write_maximum_pool_size' => 6, 956 | 'puppet_enterprise::puppetdb::read_maximum_pool_size' => 18, 957 | 'puppet_enterprise::profile::puppetdb::gc_interval' => 0, 958 | } 959 | 960 | total_cpu = params['puppet_enterprise::puppetdb::command_processing_threads'] + 961 | params['puppet_enterprise::master::puppetserver::jruby_max_active_instances'] 962 | total_ram = ram_puppetdb + ram_puppetserver 963 | totals = { 964 | 'CPU' => { 'total' => resources['cpu'], 'used' => total_cpu }, 965 | 'RAM' => { 'total' => resources['ram'], 'used' => total_ram }, 966 | 'MB_PER_JRUBY' => ram_per_jruby, 967 | } 968 | 969 | settings = { 'params' => params, 'totals' => totals } 970 | 971 | expect(calculator::calculate_master_settings(node)).to eq(settings) 972 | end 973 | end 974 | 975 | context 'with its supporting methods' do 976 | it 'can calculate a reasonable sample of agent runs based upon node count and run interval' do 977 | expect((calculator.send :calculate_run_sample, 1, 30)).to eq(2880) 978 | expect((calculator.send :calculate_run_sample, 5000, 0)).to eq(5000) 979 | expect((calculator.send :calculate_run_sample, 5000, 30)).to eq(10000) 980 | end 981 | 982 | it 'can calculate the theoretical maximum number of nodes that can managed by an infrastructure' do 983 | expect((calculator.send :calculate_maximum_nodes, 20, 1, 30)).to eq(1) 984 | end 985 | 986 | it 'can calculate the theoretical minimum number of jrubies required for an infrastructure' do 987 | expect((calculator.send :calculate_minimum_jrubies, 100, 20, 30)).to eq(134) 988 | end 989 | 990 | it 'can calculate the default memory reserved for the operating system' do 991 | expect((calculator.send :select_reserved_memory, 4096)).to eq(1024) 992 | expect((calculator.send :select_reserved_memory, 8192)).to eq(2048) 993 | expect((calculator.send :select_reserved_memory, 16384)).to eq(4096) 994 | end 995 | 996 | it 'can calculate the optional memory reserved for the operating system' do 997 | calculator.instance_variable_set(:@options, :memory_reserved_for_os => 2048) 998 | expect((calculator.send :select_reserved_memory, 32768)).to eq(2048) 999 | end 1000 | 1001 | it 'can calculate a setting based upon amount of memory' do 1002 | expect((calculator.send :fit_to_memory, 4096, 'S', 'M', 'L')).to eq('S') 1003 | expect((calculator.send :fit_to_memory, 8192, 'S', 'M', 'L')).to eq('S') 1004 | expect((calculator.send :fit_to_memory, 16384, 'S', 'M', 'L')).to eq('M') 1005 | expect((calculator.send :fit_to_memory, 32768, 'S', 'M', 'L')).to eq('L') 1006 | expect((calculator.send :fit_to_memory, 32769, 'S', 'M', 'L')).to eq('L') 1007 | end 1008 | 1009 | it 'can test if a number is within a percentage of another number' do 1010 | expect((calculator.send :within_percent?, 80, 100, 5)).to eq(false) 1011 | expect((calculator.send :within_percent?, 95, 100, 5)).to eq(true) 1012 | expect((calculator.send :within_percent?, 100, 100, 5)).to eq(false) 1013 | end 1014 | 1015 | it 'can calculate the nearest power of two' do 1016 | expect((calculator.send :nearest_power_of_two, 511)).to eq(512) 1017 | expect((calculator.send :nearest_power_of_two, 512)).to eq(512) 1018 | expect((calculator.send :nearest_power_of_two, 513)).to eq(512) 1019 | expect((calculator.send :nearest_power_of_two, 767)).to eq(512) 1020 | expect((calculator.send :nearest_power_of_two, 768)).to eq(1024) 1021 | expect((calculator.send :nearest_power_of_two, 1023)).to eq(1024) 1022 | expect((calculator.send :nearest_power_of_two, 1024)).to eq(1024) 1023 | expect((calculator.send :nearest_power_of_two, 1025)).to eq(1024) 1024 | end 1025 | end 1026 | end 1027 | -------------------------------------------------------------------------------- /spec/unit/puppet_x/puppetlabs/tune/inventory_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | require 'puppet_x/puppetlabs/tune/inventory.rb' 4 | 5 | def suppress_standard_output 6 | allow(STDOUT).to receive(:puts) 7 | end 8 | 9 | describe PuppetX::Puppetlabs::Tune::Inventory do 10 | subject(:inventory) { described_class.new } 11 | 12 | context 'with its supporting methods' do 13 | let(:empty_roles) do 14 | { 15 | 'puppet_master_host' => nil, 16 | 'console_host' => nil, 17 | 'puppetdb_host' => [], 18 | 'database_host' => [], 19 | 'primary_master_replica' => nil, 20 | 'compile_master' => [] 21 | } 22 | end 23 | 24 | let(:empty_classes) do 25 | { 26 | 'master' => [].to_set, 27 | 'console' => [].to_set, 28 | 'puppetdb' => [].to_set, 29 | 'database' => [].to_set, 30 | 'amq::broker' => [].to_set, 31 | 'orchestrator' => [].to_set, 32 | 'primary_master' => [].to_set, 33 | 'primary_master_replica' => [].to_set, 34 | 'compile_master' => [].to_set 35 | } 36 | end 37 | 38 | it 'can use the local system as inventory' do 39 | allow(Puppet::Util::Execution).to receive(:execute).with('hostname -f', 'combine' => false).and_return('master.example.com') 40 | allow(Puppet::Util::Execution).to receive(:execute).with('nproc --all', 'combine' => false).and_return('4') 41 | allow(Puppet::Util::Execution).to receive(:execute).with('free -b | grep Mem', 'combine' => false).and_return('Mem: 8589934592') 42 | output = { 43 | 'nodes' => { 44 | 'master.example.com' => { 45 | 'resources' => { 46 | 'cpu' => '4', 47 | 'ram' => '8589934592b', 48 | } 49 | } 50 | }, 51 | 'roles' => { 52 | 'puppet_master_host' => 'master.example.com', 53 | 'console_host' => nil, 54 | 'puppetdb_host' => [], 55 | 'database_host' => [], 56 | 'primary_master_replica' => nil, 57 | 'compile_master' => [], 58 | }, 59 | } 60 | 61 | inventory::read_inventory_from_local_system 62 | 63 | expect(inventory.instance_variable_get(:@nodes)).to eq(output['nodes']) 64 | expect(inventory.instance_variable_get(:@roles)).to eq(output['roles']) 65 | end 66 | 67 | it 'can use a file as inventory' do 68 | output = { 69 | 'nodes' => { 70 | 'master.example.com' => { 71 | 'resources' => { 72 | 'cpu' => 4, 73 | 'ram' => 8, 74 | } 75 | } 76 | }, 77 | 'roles' => { 78 | 'puppet_master_host' => 'master.example.com', 79 | 'console_host' => nil, 80 | 'puppetdb_host' => [], 81 | 'database_host' => [], 82 | 'primary_master_replica' => nil, 83 | 'compile_master' => [], 84 | }, 85 | } 86 | 87 | inventory::read_inventory_from_inventory_file('fixtures/mono.yaml') 88 | 89 | expect(inventory.instance_variable_get(:@nodes)).to eq(output['nodes']) 90 | expect(inventory.instance_variable_get(:@roles)).to eq(output['roles']) 91 | end 92 | 93 | it 'can handles errors with a file as inventory' do 94 | inventory::read_inventory_from_inventory_file('fixtures/does_not_exist.yaml') 95 | 96 | expect(inventory.instance_variable_get(:@nodes)).to eq({}) 97 | expect(inventory.instance_variable_get(:@roles)).to eq({}) 98 | end 99 | 100 | it 'can handles syntax errors with a file as inventory' do 101 | inventory::read_inventory_from_inventory_file('fixtures/syntax_error.yaml') 102 | 103 | expect(inventory.instance_variable_get(:@nodes)).to eq({}) 104 | expect(inventory.instance_variable_get(:@roles)).to eq({}) 105 | end 106 | 107 | it 'can handles node errors with a file as inventory' do 108 | inventory::read_inventory_from_inventory_file('fixtures/no_nodes.yaml') 109 | 110 | expect(inventory.instance_variable_get(:@nodes)).to eq({}) 111 | expect(inventory.instance_variable_get(:@roles)).to eq({}) 112 | end 113 | 114 | it 'can convert mono inventory roles to classes' do 115 | inputs = { 116 | 'roles' => { 117 | 'puppet_master_host' => 'master', 118 | 'console_host' => nil, 119 | 'puppetdb_host' => nil, 120 | 'database_host' => [], 121 | 'primary_master_replica' => nil, 122 | 'compile_master' => nil 123 | }, 124 | 'classes' => empty_classes 125 | } 126 | output = { 127 | 'roles' => inputs['roles'], 128 | 'classes' => { 129 | 'master' => ['master'].to_set, 130 | 'console' => ['master'].to_set, 131 | 'puppetdb' => ['master'].to_set, 132 | 'database' => ['master'].to_set, 133 | 'amq::broker' => ['master'].to_set, 134 | 'orchestrator' => ['master'].to_set, 135 | 'primary_master' => ['master'].to_set, 136 | 'primary_master_replica' => [].to_set, 137 | 'compile_master' => [].to_set 138 | } 139 | } 140 | 141 | inventory.instance_variable_set(:@roles, inputs['roles']) 142 | inventory.instance_variable_set(:@classes, inputs['classes']) 143 | 144 | inventory::convert_inventory_roles_to_classes 145 | 146 | expect(inventory.instance_variable_get(:@roles)).to eq(output['roles']) 147 | expect(inventory.instance_variable_get(:@classes)).to eq(output['classes']) 148 | end 149 | 150 | it 'can convert mono inventory roles to classes with a compile master' do 151 | inputs = { 152 | 'roles' => { 153 | 'puppet_master_host' => 'master', 154 | 'console_host' => nil, 155 | 'puppetdb_host' => [], 156 | 'database_host' => [], 157 | 'primary_master_replica' => nil, 158 | 'compile_master' => ['compile'] 159 | }, 160 | 'classes' => empty_classes 161 | } 162 | output = { 163 | 'roles' => inputs['roles'], 164 | 'classes' => { 165 | 'master' => ['master', 'compile'].to_set, 166 | 'console' => ['master'].to_set, 167 | 'puppetdb' => ['master'].to_set, 168 | 'database' => ['master'].to_set, 169 | 'amq::broker' => ['master'].to_set, 170 | 'orchestrator' => ['master'].to_set, 171 | 'primary_master' => ['master'].to_set, 172 | 'primary_master_replica' => [].to_set, 173 | 'compile_master' => ['compile'].to_set 174 | } 175 | } 176 | 177 | inventory.instance_variable_set(:@roles, inputs['roles']) 178 | inventory.instance_variable_set(:@classes, inputs['classes']) 179 | 180 | inventory::convert_inventory_roles_to_classes 181 | 182 | expect(inventory.instance_variable_get(:@roles)).to eq(output['roles']) 183 | expect(inventory.instance_variable_get(:@classes)).to eq(output['classes']) 184 | end 185 | 186 | it 'can convert split inventory roles to classes' do 187 | inputs = { 188 | 'nodes' => {}, 189 | 'roles' => { 190 | 'puppet_master_host' => 'master', 191 | 'console_host' => 'console', 192 | 'puppetdb_host' => ['puppetdb'], 193 | 'database_host' => [], 194 | 'primary_master_replica' => nil, 195 | 'compile_master' => nil 196 | }, 197 | 'classes' => empty_classes 198 | } 199 | output = { 200 | 'roles' => inputs['roles'], 201 | 'classes' => { 202 | 'master' => ['master'].to_set, 203 | 'console' => ['console'].to_set, 204 | 'puppetdb' => ['puppetdb'].to_set, 205 | 'database' => ['puppetdb'].to_set, 206 | 'amq::broker' => ['master'].to_set, 207 | 'orchestrator' => ['master'].to_set, 208 | 'primary_master' => ['master'].to_set, 209 | 'primary_master_replica' => [].to_set, 210 | 'compile_master' => [].to_set 211 | } 212 | } 213 | 214 | inventory.instance_variable_set(:@roles, inputs['roles']) 215 | inventory.instance_variable_set(:@classes, inputs['classes']) 216 | 217 | inventory::convert_inventory_roles_to_classes 218 | 219 | expect(inventory.instance_variable_get(:@roles)).to eq(output['roles']) 220 | expect(inventory.instance_variable_get(:@classes)).to eq(output['classes']) 221 | end 222 | 223 | it 'can convert split inventory roles to classes with a database host' do 224 | inputs = { 225 | 'roles' => { 226 | 'puppet_master_host' => 'master', 227 | 'console_host' => 'console', 228 | 'puppetdb_host' => ['puppetdb'], 229 | 'database_host' => ['database'], 230 | 'primary_master_replica' => nil, 231 | 'compile_master' => nil 232 | }, 233 | 'classes' => empty_classes 234 | } 235 | output = { 236 | 'roles' => inputs['roles'], 237 | 'classes' => { 238 | 'master' => ['master'].to_set, 239 | 'console' => ['console'].to_set, 240 | 'puppetdb' => ['puppetdb'].to_set, 241 | 'database' => ['database'].to_set, 242 | 'amq::broker' => ['master'].to_set, 243 | 'orchestrator' => ['master'].to_set, 244 | 'primary_master' => ['master'].to_set, 245 | 'primary_master_replica' => [].to_set, 246 | 'compile_master' => [].to_set 247 | } 248 | } 249 | 250 | inventory.instance_variable_set(:@roles, inputs['roles']) 251 | inventory.instance_variable_set(:@classes, inputs['classes']) 252 | 253 | inventory::convert_inventory_roles_to_classes 254 | 255 | expect(inventory.instance_variable_get(:@roles)).to eq(output['roles']) 256 | expect(inventory.instance_variable_get(:@classes)).to eq(output['classes']) 257 | end 258 | 259 | it 'can convert split inventory roles to classes with an array of puppetdb hosts' do 260 | inputs = { 261 | 'roles' => { 262 | 'puppet_master_host' => 'master', 263 | 'console_host' => 'console', 264 | 'puppetdb_host' => ['puppetdb1', 'puppetdb2'], 265 | 'database_host' => [], 266 | 'primary_master_replica' => nil, 267 | 'compile_master' => nil 268 | }, 269 | 'classes' => empty_classes 270 | } 271 | output = { 272 | 'roles' => inputs['roles'], 273 | 'classes' => { 274 | 'master' => ['master'].to_set, 275 | 'console' => ['console'].to_set, 276 | 'puppetdb' => ['puppetdb1', 'puppetdb2'].to_set, 277 | 'database' => ['puppetdb1'].to_set, 278 | 'amq::broker' => ['master'].to_set, 279 | 'orchestrator' => ['master'].to_set, 280 | 'primary_master' => ['master'].to_set, 281 | 'primary_master_replica' => [].to_set, 282 | 'compile_master' => [].to_set 283 | } 284 | } 285 | 286 | inventory.instance_variable_set(:@roles, inputs['roles']) 287 | inventory.instance_variable_set(:@classes, inputs['classes']) 288 | 289 | inventory::convert_inventory_roles_to_classes 290 | 291 | expect(inventory.instance_variable_get(:@roles)).to eq(output['roles']) 292 | expect(inventory.instance_variable_get(:@classes)).to eq(output['classes']) 293 | end 294 | 295 | it 'can convert split inventory roles to classes with a database host and an array of puppetdb hosts' do 296 | inputs = { 297 | 'roles' => { 298 | 'puppet_master_host' => 'master', 299 | 'console_host' => 'console', 300 | 'puppetdb_host' => ['puppetdb1', 'puppetdb2'], 301 | 'database_host' => ['database'], 302 | 'primary_master_replica' => nil, 303 | 'compile_master' => nil 304 | }, 305 | 'classes' => empty_classes 306 | } 307 | output = { 308 | 'roles' => inputs['roles'], 309 | 'classes' => { 310 | 'master' => ['master'].to_set, 311 | 'console' => ['console'].to_set, 312 | 'puppetdb' => ['puppetdb1', 'puppetdb2'].to_set, 313 | 'database' => ['database'].to_set, 314 | 'amq::broker' => ['master'].to_set, 315 | 'orchestrator' => ['master'].to_set, 316 | 'primary_master' => ['master'].to_set, 317 | 'primary_master_replica' => [].to_set, 318 | 'compile_master' => [].to_set 319 | } 320 | } 321 | 322 | inventory.instance_variable_set(:@roles, inputs['roles']) 323 | inventory.instance_variable_set(:@classes, inputs['classes']) 324 | 325 | inventory::convert_inventory_roles_to_classes 326 | 327 | expect(inventory.instance_variable_get(:@roles)).to eq(output['roles']) 328 | expect(inventory.instance_variable_get(:@classes)).to eq(output['classes']) 329 | end 330 | 331 | it 'can convert mono inventory with ha roles to classes' do 332 | inputs = { 333 | 'roles' => { 334 | 'puppet_master_host' => 'master', 335 | 'console_host' => nil, 336 | 'puppetdb_host' => nil, 337 | 'database_host' => [], 338 | 'primary_master_replica' => 'replica', 339 | 'compile_master' => nil 340 | }, 341 | 'classes' => empty_classes 342 | } 343 | output = { 344 | 'roles' => inputs['roles'], 345 | 'classes' => { 346 | 'master' => ['master', 'replica'].to_set, 347 | 'console' => ['master', 'replica'].to_set, 348 | 'puppetdb' => ['master', 'replica'].to_set, 349 | 'database' => ['master', 'replica'].to_set, 350 | 'amq::broker' => ['master', 'replica'].to_set, 351 | 'orchestrator' => ['master', 'replica'].to_set, 352 | 'primary_master' => ['master'].to_set, 353 | 'primary_master_replica' => ['replica'].to_set, 354 | 'compile_master' => [].to_set 355 | } 356 | } 357 | 358 | inventory.instance_variable_set(:@roles, inputs['roles']) 359 | inventory.instance_variable_set(:@classes, inputs['classes']) 360 | 361 | inventory::convert_inventory_roles_to_classes 362 | 363 | expect(inventory.instance_variable_get(:@roles)).to eq(output['roles']) 364 | expect(inventory.instance_variable_get(:@classes)).to eq(output['classes']) 365 | end 366 | 367 | it 'can convert extra large inventory roles to classes' do 368 | inputs = { 369 | 'roles' => { 370 | 'puppet_master_host' => 'master', 371 | 'console_host' => nil, 372 | 'puppetdb_host' => [], 373 | 'database_host' => ['database'], 374 | 'primary_master_replica' => nil, 375 | 'compile_master' => ['compile1', 'compile2'] 376 | }, 377 | 'classes' => empty_classes 378 | } 379 | output = { 380 | 'roles' => inputs['roles'], 381 | 'classes' => { 382 | 'master' => ['master', 'compile1', 'compile2'].to_set, 383 | 'console' => ['master'].to_set, 384 | 'puppetdb' => ['master', 'compile1', 'compile2'].to_set, 385 | 'database' => ['database', 'master'].to_set, 386 | 'amq::broker' => ['master'].to_set, 387 | 'orchestrator' => ['master'].to_set, 388 | 'primary_master' => ['master'].to_set, 389 | 'primary_master_replica' => [].to_set, 390 | 'compile_master' => ['compile1', 'compile2'].to_set 391 | } 392 | } 393 | 394 | inventory.instance_variable_set(:@roles, inputs['roles']) 395 | inventory.instance_variable_set(:@classes, inputs['classes']) 396 | 397 | inventory::convert_inventory_roles_to_classes 398 | 399 | expect(inventory.instance_variable_get(:@roles)).to eq(output['roles']) 400 | expect(inventory.instance_variable_get(:@classes)).to eq(output['classes']) 401 | end 402 | 403 | it 'can convert extra large inventory with ha roles to classes' do 404 | inputs = { 405 | 'roles' => { 406 | 'puppet_master_host' => 'master', 407 | 'console_host' => nil, 408 | 'puppetdb_host' => [], 409 | 'database_host' => ['database1', 'database2'], 410 | 'primary_master_replica' => 'replica', 411 | 'compile_master' => ['compile1', 'compile2'] 412 | }, 413 | 'classes' => empty_classes 414 | } 415 | output = { 416 | 'roles' => inputs['roles'], 417 | 'classes' => { 418 | 'master' => ['master', 'replica', 'compile1', 'compile2'].to_set, 419 | 'console' => ['master', 'replica'].to_set, 420 | 'puppetdb' => ['master', 'replica', 'compile1', 'compile2'].to_set, 421 | 'database' => ['database1', 'database2', 'master', 'replica'].to_set, 422 | 'amq::broker' => ['master', 'replica'].to_set, 423 | 'orchestrator' => ['master', 'replica'].to_set, 424 | 'primary_master' => ['master'].to_set, 425 | 'primary_master_replica' => ['replica'].to_set, 426 | 'compile_master' => ['compile1', 'compile2'].to_set 427 | } 428 | } 429 | 430 | inventory.instance_variable_set(:@roles, inputs['roles']) 431 | inventory.instance_variable_set(:@classes, inputs['classes']) 432 | 433 | inventory::convert_inventory_roles_to_classes 434 | 435 | expect(inventory.instance_variable_get(:@roles)).to eq(output['roles']) 436 | expect(inventory.instance_variable_get(:@classes)).to eq(output['classes']) 437 | end 438 | end 439 | end 440 | -------------------------------------------------------------------------------- /spec/unit/puppet_x/puppetlabs/tune/query_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | require 'puppet_x/puppetlabs/tune/query.rb' 4 | 5 | def suppress_standard_output 6 | allow(STDOUT).to receive(:puts) 7 | end 8 | 9 | describe PuppetX::Puppetlabs::Tune::Query do 10 | subject(:query) { described_class.new } 11 | 12 | let(:pql_data) do 13 | [ 14 | { "metrics" => 15 | { "data" => 16 | [ 17 | { "name" => "foo", "value" => 75.181073364999975, "category" => "time" }, 18 | { "name" => "bar", "value" => 28.589005453, "category" => "time" }, 19 | { "name" => "config_retrieval", "value" => 50.181073364999975, "category" => "time" }, 20 | ] } }, 21 | { "metrics" => 22 | { "data" => 23 | [ 24 | { "name" => "foo", "value" => 8.181073364999975, "category" => "time" }, 25 | { "name" => "bar", "value" => 28.589005453, "category" => "time" }, 26 | { "name" => "config_retrieval", "value" => 50.181073364999975, "category" => "time" }, 27 | ] } } 28 | ] 29 | end 30 | 31 | describe '#average_metric_time' do 32 | context 'when data contains time values for the given metric' do 33 | it 'returns the average of the metric\'s data' do 34 | expect(query.average_metric_time(pql_data, 'foo', 'bar')).to eq(42) 35 | end 36 | end 37 | 38 | context 'when data does not contain time values for the given metric' do 39 | it 'returns nil' do 40 | expect(query.average_metric_time([{}], 'foo', 'bar')).to eq(nil) 41 | end 42 | end 43 | end 44 | 45 | # describe '#average_compile_time_for_range' do 46 | # context 'when data contains values in the given time range' do 47 | # it 'returns the average of the config_retrieval data' do 48 | # pending("Cannot test without puppetdb installed") 49 | # expect(query.average_compile_time_for_range('2019-01-01', '2019-12-31')).to eq(50) 50 | # end 51 | # end 52 | # end 53 | end 54 | -------------------------------------------------------------------------------- /spec/unit/puppet_x/puppetlabs/tune_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | require 'puppet_x/puppetlabs/tune.rb' 4 | 5 | def suppress_standard_output 6 | allow(STDOUT).to receive(:puts) 7 | end 8 | 9 | describe PuppetX::Puppetlabs::Tune do 10 | # Do not query PuppetDB when unit testing this class. 11 | subject(:tune) { described_class.new(:local => true) } 12 | 13 | # Allows mergeups in the PE implementation of this class. 14 | pe_2019_or_newer = Gem::Version.new(Puppet.version) >= Gem::Version.new('6.0.0') 15 | pe_2019_2_or_newer = Gem::Version.new(Puppet.version) >= Gem::Version.new('6.9.0') 16 | 17 | before(:each) do 18 | suppress_standard_output 19 | end 20 | 21 | context 'with its tunable methods' do 22 | it 'tunes a known set of classes' do 23 | class_names = [ 24 | 'certificate_authority', 25 | 'master', 26 | 'console', 27 | 'puppetdb', 28 | 'database', 29 | 'amq::broker', 30 | 'orchestrator', 31 | 'primary_master', 32 | 'primary_master_replica', 33 | 'enabled_primary_master_replica', 34 | 'compile_master', 35 | ] 36 | class_names.delete('amq::broker') if pe_2019_or_newer 37 | expect(tune::tunable_class_names).to eq(class_names) 38 | end 39 | 40 | it 'tunes a known set of settings' do 41 | param_names = [ 42 | 'puppet_enterprise::master::puppetserver::jruby_max_active_instances', 43 | 'puppet_enterprise::master::puppetserver::reserved_code_cache', 44 | 'puppet_enterprise::profile::amq::broker::heap_mb', 45 | 'puppet_enterprise::profile::console::java_args', 46 | 'puppet_enterprise::profile::database::shared_buffers', 47 | 'puppet_enterprise::profile::master::java_args', 48 | 'puppet_enterprise::profile::orchestrator::java_args', 49 | 'puppet_enterprise::profile::orchestrator::jruby_max_active_instances', 50 | # 'puppet_enterprise::profile::orchestrator::reserved_code_cache', 51 | 'puppet_enterprise::profile::puppetdb::java_args', 52 | 'puppet_enterprise::puppetdb::command_processing_threads', 53 | ] 54 | param_names.delete('puppet_enterprise::profile::amq::broker::heap_mb') if pe_2019_or_newer 55 | unless pe_2019_2_or_newer 56 | param_names.delete('puppet_enterprise::profile::orchestrator::jruby_max_active_instances') 57 | # param_names.delete('puppet_enterprise::profile::orchestrator::reserved_code_cache') 58 | end 59 | expect(tune::tunable_param_names).to eq(param_names) 60 | end 61 | end 62 | 63 | context 'with its supporting methods' do 64 | it 'can detect an unknown infrastructure' do 65 | nodes = { 'primary_masters' => [] } 66 | tune.instance_variable_set(:@nodes_with_role, nodes) 67 | 68 | expect(tune::unknown_infrastructure?).to eq(true) 69 | end 70 | 71 | it 'can detect a monolithic infrastructure' do 72 | nodes = { 73 | 'console_hosts' => [], 74 | 'puppetdb_hosts' => [], 75 | } 76 | tune.instance_variable_set(:@nodes_with_role, nodes) 77 | 78 | expect(tune::monolithic?).to eq(true) 79 | end 80 | 81 | it 'can detect a split infrastructure' do 82 | nodes = { 83 | 'console_hosts' => ['console'], 84 | 'puppetdb_hosts' => ['puppetdb'], 85 | } 86 | tune.instance_variable_set(:@nodes_with_role, nodes) 87 | 88 | expect(tune::monolithic?).to eq(false) 89 | end 90 | 91 | it 'can detect ha infrastructure' do 92 | nodes = { 'replica_masters' => ['replica'] } 93 | tune.instance_variable_set(:@nodes_with_role, nodes) 94 | 95 | expect(tune::with_ha?).to eq(true) 96 | end 97 | 98 | it 'can detect compile masters' do 99 | nodes = { 'compile_masters' => ['compile'] } 100 | tune.instance_variable_set(:@nodes_with_role, nodes) 101 | 102 | expect(tune::with_compile_masters?).to eq(true) 103 | end 104 | 105 | it 'can detect an external database host' do 106 | nodes = { 107 | 'primary_masters' => ['master'], 108 | 'database_hosts' => ['postgresql'], 109 | } 110 | tune.instance_variable_set(:@nodes_with_role, nodes) 111 | 112 | expect(tune::with_external_database?).to eq(true) 113 | end 114 | 115 | it 'can detect local and external databases' do 116 | nodes_with_class = { 117 | 'database' => ['master', 'postgresql'] 118 | } 119 | nodes = { 120 | 'primary_masters' => ['master'], 121 | 'replica_masters' => [], 122 | 'database_hosts' => ['master', 'postgresql'], 123 | } 124 | tune.instance_variable_set(:@nodes_with_class, nodes_with_class) 125 | tune.instance_variable_set(:@nodes_with_role, nodes) 126 | 127 | expect(tune::with_local_and_external_databases?).to eq(true) 128 | end 129 | 130 | it 'can detect puppetdb on all masters' do 131 | nodes_with_class = { 132 | 'puppetdb' => ['master', 'replica', 'compile'] 133 | } 134 | nodes = { 135 | 'primary_masters' => ['master'], 136 | 'replica_masters' => ['replica'], 137 | 'compile_masters' => ['compile'], 138 | } 139 | tune.instance_variable_set(:@nodes_with_class, nodes_with_class) 140 | tune.instance_variable_set(:@nodes_with_role, nodes) 141 | 142 | expect(tune::with_puppetdb_on_all_masters?).to eq(true) 143 | end 144 | 145 | it 'can detect a monolithic master' do 146 | nodes = { 147 | 'primary_masters' => ['master'], 148 | 'console_hosts' => [], 149 | 'puppetdb_hosts' => [], 150 | } 151 | tune.instance_variable_set(:@nodes_with_role, nodes) 152 | 153 | expect(tune::monolithic_master?('master')).to eq(true) 154 | end 155 | 156 | it 'can detect a replica master' do 157 | nodes = { 158 | 'primary_masters' => ['master'], 159 | 'replica_masters' => ['replica'], 160 | 'console_hosts' => [], 161 | 'puppetdb_hosts' => [], 162 | } 163 | tune.instance_variable_set(:@nodes_with_role, nodes) 164 | 165 | expect(tune::replica_master?('replica')).to eq(true) 166 | end 167 | 168 | it 'can detect a compile master' do 169 | nodes = { 170 | 'compile_masters' => ['compile'], 171 | } 172 | tune.instance_variable_set(:@nodes_with_role, nodes) 173 | 174 | expect(tune::compile_master?('compile')).to eq(true) 175 | end 176 | 177 | it 'can detect a compiler' do 178 | nodes_with_class = { 179 | 'puppetdb' => ['master', 'compile1'] 180 | } 181 | nodes = { 182 | 'primary_masters' => ['master'], 183 | 'replica_masters' => [], 184 | 'console_hosts' => [], 185 | 'compile_masters' => ['compile1'], 186 | 'puppetdb_hosts' => ['compile1'], 187 | 'database_hosts' => [], 188 | } 189 | tune.instance_variable_set(:@nodes_with_class, nodes_with_class) 190 | tune.instance_variable_set(:@nodes_with_role, nodes) 191 | 192 | expect(tune::with_compilers?).to eq(true) 193 | end 194 | 195 | it 'can detect a class on a host' do 196 | nodes_with_class = { 'console' => ['console'] } 197 | tune.instance_variable_set(:@nodes_with_class, nodes_with_class) 198 | 199 | expect(tune::node_with_class?('console', 'console')).to eq(true) 200 | end 201 | 202 | it 'can detect classes on a host' do 203 | nodes_with_class = { 204 | 'certificate_authority' => [], 205 | 'master' => [], 206 | 'console' => ['console'], 207 | 'puppetdb' => [], 208 | 'database' => [], 209 | 'amq::broker' => [], 210 | 'orchestrator' => [], 211 | 'primary_master' => [], 212 | 'primary_master_replica' => [], 213 | 'enabled_primary_master_replica' => [], 214 | 'compile_master' => [], 215 | } 216 | classes_for_node = { 217 | 'certificate_authority' => false, 218 | 'master' => false, 219 | 'console' => true, 220 | 'puppetdb' => false, 221 | 'database' => false, 222 | 'amq::broker' => false, 223 | 'orchestrator' => false, 224 | 'primary_master' => false, 225 | 'primary_master_replica' => false, 226 | 'enabled_primary_master_replica' => false, 227 | 'compile_master' => false, 228 | } 229 | nodes_with_class.delete('amq::broker') if pe_2019_or_newer 230 | classes_for_node.delete('amq::broker') if pe_2019_or_newer 231 | tune.instance_variable_set(:@nodes_with_class, nodes_with_class) 232 | 233 | expect(tune::tunable_classes_for_node('console')).to eq(classes_for_node) 234 | end 235 | 236 | # it 'can detect that JRuby9K is enabled for the puppetsever service' do 237 | # end 238 | 239 | it 'can extract common settings' do 240 | tune.instance_variable_set(:@options, :common => true) 241 | tune.instance_variable_set(:@collected_settings_common, {}) 242 | collected_nodes = { 243 | 'node_1' => { 244 | 'settings' => { 245 | 'params' => { 246 | 'a' => 1, 247 | 'b' => 'b' 248 | } 249 | } 250 | }, 251 | 'node_2' => { 252 | 'settings' => { 253 | 'params' => { 254 | 'a' => 2, 255 | 'b' => 'b' 256 | } 257 | } 258 | } 259 | } 260 | collected_nodes_without_common_settings = { 261 | 'node_1' => { 'settings' => { 'params' => { 'a' => 1 } } }, 262 | 'node_2' => { 'settings' => { 'params' => { 'a' => 2 } } } 263 | } 264 | collected_settings_common = { 'b' => 'b' } 265 | 266 | tune.instance_variable_set(:@collected_nodes, collected_nodes) 267 | tune::collect_optimized_settings_common_to_all_nodes 268 | 269 | expect(tune.instance_variable_get(:@collected_settings_common)).to eq(collected_settings_common) 270 | expect(tune.instance_variable_get(:@collected_nodes)).to eq(collected_nodes_without_common_settings) 271 | end 272 | 273 | it 'can enforce minimum system requirements' do 274 | tune.instance_variable_set(:@options, :force => false) 275 | 276 | resources = { 'cpu' => 1, 'ram' => 4096 } 277 | expect(tune::meets_minimum_system_requirements?(resources)).to eq(false) 278 | 279 | resources = { 'cpu' => 2, 'ram' => 5120 } 280 | expect(tune::meets_minimum_system_requirements?(resources)).to eq(false) 281 | 282 | resources = { 'cpu' => 2, 'ram' => 5806 } 283 | expect(tune::meets_minimum_system_requirements?(resources)).to eq(true) 284 | 285 | resources = { 'cpu' => 2, 'ram' => 6144 } 286 | expect(tune::meets_minimum_system_requirements?(resources)).to eq(true) 287 | end 288 | 289 | it 'can disable minimum system requirements' do 290 | tune.instance_variable_set(:@options, :force => true) 291 | resources = { 'cpu' => 1, 'ram' => 4096 } 292 | 293 | expect(tune::meets_minimum_system_requirements?(resources)).to eq(true) 294 | end 295 | 296 | it 'can convert a string to bytes with a g unit' do 297 | bytes_string = '16g' 298 | bytes = 17179869184 299 | expect(tune::string_to_bytes(bytes_string)).to eq(bytes) 300 | end 301 | 302 | it 'can convert a string to bytes with a m unit' do 303 | bytes_string = '16384m' 304 | bytes = 17179869184 305 | expect(tune::string_to_bytes(bytes_string)).to eq(bytes) 306 | end 307 | 308 | it 'can convert a string to bytes with a k unit' do 309 | bytes_string = '16777216k' 310 | bytes = 17179869184 311 | expect(tune::string_to_bytes(bytes_string)).to eq(bytes) 312 | end 313 | 314 | it 'can convert a string to bytes with a b unit' do 315 | bytes_string = '17179869184b' 316 | bytes = 17179869184 317 | expect(tune::string_to_bytes(bytes_string)).to eq(bytes) 318 | end 319 | 320 | it 'can convert a string to bytes without a unit' do 321 | bytes_string = '16' 322 | bytes = 17179869184 323 | expect(tune::string_to_bytes(bytes_string)).to eq(bytes) 324 | end 325 | 326 | it 'can convert a string to megabytes with a g unit' do 327 | bytes_string = '1g' 328 | megabytes = 1024 329 | expect(tune::string_to_megabytes(bytes_string)).to eq(megabytes) 330 | end 331 | 332 | it 'can convert a string to megabytes with a m unit' do 333 | bytes_string = '1024m' 334 | megabytes = 1024 335 | expect(tune::string_to_megabytes(bytes_string)).to eq(megabytes) 336 | end 337 | 338 | it 'can convert a string to megabytes without a unit' do 339 | bytes_string = '1024' 340 | megabytes = 1024 341 | expect(tune::string_to_megabytes(bytes_string)).to eq(megabytes) 342 | end 343 | end 344 | end 345 | -------------------------------------------------------------------------------- /tasks/init.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Run 'puppet pe tune' on the primary master", 3 | "input_method": "stdin", 4 | "supports_noop": false, 5 | "parameters": { 6 | "common": { 7 | "description": "Extract common settings from node-specific settings", 8 | "type": "Optional[Variant[Boolean,Enum['true','false']]]" 9 | }, 10 | "hiera": { 11 | "description": "Output Hiera YAML files to the specified directorypriomary ", 12 | "type": "Optional[String]" 13 | }, 14 | "memory_per_jruby": { 15 | "description": "Amount of RAM to allocate for each Puppet Server JRuby", 16 | "type": "Optional[Integer]" 17 | }, 18 | "memory_reserved_for_os": { 19 | "description": "Amount of RAM to reserve for the operating system", 20 | "type": "Optional[Integer]" 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /tasks/init.rb: -------------------------------------------------------------------------------- 1 | #!/opt/puppetlabs/puppet/bin/ruby 2 | 3 | require 'facter' 4 | require 'json' 5 | require 'open3' 6 | require 'puppet' 7 | require 'timeout' 8 | 9 | Puppet.initialize_settings 10 | 11 | # Read parameters, set defaults, and validate values. 12 | 13 | def read_parameters 14 | input = read_stdin 15 | output = {} 16 | 17 | # Validate parameter values or return errors. 18 | 19 | unless safe_hiera_path?(input['hiera']) 20 | return_error _("The 'hiera' parameter must be a subdirectory in %{paths}") % { paths: safe_hiera_paths.join(' or ') } 21 | end 22 | 23 | return_error("The 'memory_per_jruby' parameter must be an integer") unless safe_memory?(input['memory_per_jruby']) 24 | return_error("The 'memory_reserved_for_os' parameter must be an integer") unless safe_memory?(input['memory_reserved_for_os']) 25 | 26 | output['common'] = optional_true(input['common']) ? '--common' : '' 27 | output['hiera'] = (input['hiera']) ? "--hiera=#{input['hiera']}" : '' 28 | output['memory_per_jruby'] = (input['memory_per_jruby']) ? "--memory_per_jruby=#{input['memory_per_jruby'].to_i}" : '' 29 | output['memory_reserved_for_os'] = (input['memory_reserved_for_os']) ? "--memory_reserved_for_os=#{input['memory_reserved_for_os'].to_i}" : '' 30 | 31 | output 32 | end 33 | 34 | # Read parameters as JSON from STDIN. 35 | 36 | def read_stdin 37 | input = {} 38 | begin 39 | Timeout.timeout(3) do 40 | input = JSON.parse(STDIN.read) 41 | end 42 | rescue Timeout::Error 43 | return_error('Cannot read parameters as JSON from STDIN') 44 | end 45 | input 46 | end 47 | 48 | # Handle "Optional[Variant[Boolean,Enum['true','false']]]" 49 | 50 | def optional_true(param) 51 | return false unless param 52 | return true if param == true 53 | param == 'true' 54 | end 55 | 56 | # Validation 57 | 58 | def safe_hiera_paths 59 | ['/tmp/', "#{Puppet[:environmentpath]}/#{Puppet[:environment]}/"] 60 | end 61 | 62 | def safe_hiera_path?(param) 63 | return true unless param 64 | path = File.absolute_path(param) 65 | safe_hiera_paths.each do |safe_hiera_path| 66 | return true if path.start_with?(safe_hiera_path) 67 | end 68 | false 69 | end 70 | 71 | def safe_memory?(param) 72 | return true unless param 73 | (param =~ %r{^\d+$}) != nil 74 | end 75 | 76 | # Execute a command with an array of arguments and return the result as a hash. 77 | 78 | def execute_command(command, args = []) 79 | # '/opt/puppetlabs/bin' is not in PATH, but is require to execute 'puppet infra' commands. 80 | command_env = { 'PATH' => "#{ENV['PATH']}:/opt/puppetlabs/bin" } 81 | # Convert each element of the args array to a string. 82 | args = args.reject { |a| a.empty? }.map(&:to_s) 83 | # Execute the command with the arguments passed as a variable length argument list using the asterisk operator. 84 | stdout, stderr, status = Open3.capture3(command_env, command, *args) 85 | # Merge the command and args into a string. 86 | command_line = args.unshift(command).join(' ') 87 | { command: command_line, status: status.exitstatus, stdout: stdout.strip, stderr: stderr.strip } 88 | end 89 | 90 | # Return an error and exit. 91 | 92 | def return_error(message) 93 | result = {} 94 | result[:_error] = { 95 | msg: message, 96 | kind: 'pe_tune/failure', 97 | details: {} 98 | } 99 | puts result.to_json 100 | exit 1 101 | end 102 | 103 | # Return the error results of a command and exit. 104 | 105 | def return_command_error(params, command_results) 106 | result = {} 107 | result[:status] = 'failure' 108 | result[:command] = command_results[:command] 109 | result[:error] = command_results[:stderr] 110 | result[:results] = command_results[:stdout] 111 | result[:params] = params 112 | puts result.to_json 113 | exit 1 114 | end 115 | 116 | # Return the results of a command and exit. 117 | 118 | def return_command_results(_params, command_results) 119 | result = {} 120 | result[:status] = 'success' 121 | result[:command] = command_results[:command] 122 | result[:results] = command_results[:stdout] 123 | puts result.to_json 124 | exit 0 125 | end 126 | 127 | ###### 128 | # Main 129 | ###### 130 | 131 | # Master validation. 132 | 133 | unless File.exist?('/opt/puppetlabs/bin/puppetserver') 134 | return_error('This node does not appear to be the primary master') 135 | end 136 | 137 | params = read_parameters 138 | 139 | command = 'puppet' 140 | options = ['pe', 'tune', 141 | params['common'], 142 | params['hiera'], 143 | params['memory_per_jruby'], 144 | params['memory_reserved_for_os'], 145 | ] 146 | 147 | results = execute_command(command, options) 148 | 149 | if results[:status] != 0 150 | return_command_error(params, results) 151 | else 152 | return_command_results(params, results) 153 | end 154 | --------------------------------------------------------------------------------