├── .github └── workflows │ ├── gem-push.yml │ └── ruby.yml ├── .gitignore ├── .rspec ├── Gemfile ├── LICENSE ├── README.md ├── Rakefile ├── catalog-info.yaml ├── config.ru ├── lib └── talos.rb ├── spec ├── fixtures │ ├── hiera.yaml │ ├── master │ ├── master.3fa3fd97848a72ae539b75bccd6028cd1d4e92e3 │ │ ├── common.yaml │ │ ├── fqdn │ │ │ └── foo.bar.yaml │ │ ├── role │ │ │ ├── foobar │ │ │ │ ├── testing.yaml │ │ │ │ └── z.yaml │ │ │ └── puppet.yaml │ │ └── site │ │ │ └── sjc.yaml │ └── talos.yaml ├── spec_helper.rb └── talos_spec.rb └── talos.gemspec /.github/workflows/gem-push.yml: -------------------------------------------------------------------------------- 1 | name: Ruby Gem 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | 7 | 8 | jobs: 9 | build: 10 | name: Build + Publish 11 | runs-on: ubuntu-latest 12 | permissions: 13 | contents: read 14 | packages: write 15 | 16 | steps: 17 | - uses: actions/checkout@v2 18 | - name: Set up Ruby 2.7 19 | uses: ruby/setup-ruby@v1 20 | with: 21 | ruby-version: 2.7 22 | 23 | - name: Publish to GPR 24 | run: | 25 | mkdir -p $HOME/.gem 26 | touch $HOME/.gem/credentials 27 | chmod 0600 $HOME/.gem/credentials 28 | printf -- "---\n:github: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials 29 | gem build *.gemspec 30 | gem push --KEY github --host https://rubygems.pkg.github.com/${OWNER} *.gem 31 | env: 32 | GEM_HOST_API_KEY: "Bearer ${{secrets.GITHUB_TOKEN}}" 33 | OWNER: ${{ github.repository_owner }} 34 | 35 | - name: Publish to RubyGems 36 | run: | 37 | mkdir -p $HOME/.gem 38 | touch $HOME/.gem/credentials 39 | chmod 0600 $HOME/.gem/credentials 40 | printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials 41 | gem build *.gemspec 42 | gem push *.gem 43 | env: 44 | GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_KEY}}" 45 | -------------------------------------------------------------------------------- /.github/workflows/ruby.yml: -------------------------------------------------------------------------------- 1 | # This workflow uses actions that are not certified by GitHub. 2 | # They are provided by a third-party and are governed by 3 | # separate terms of service, privacy policy, and support 4 | # documentation. 5 | # This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake 6 | # For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby 7 | 8 | name: Ruby 9 | 10 | on: 11 | push: 12 | branches: [ master ] 13 | pull_request: 14 | branches: [ master ] 15 | 16 | jobs: 17 | test: 18 | 19 | runs-on: ubuntu-latest 20 | strategy: 21 | matrix: 22 | ruby-version: ['2.6', '2.7'] 23 | 24 | steps: 25 | - uses: actions/checkout@v2 26 | - name: Set up Ruby 27 | # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby, 28 | # change this to (see https://github.com/ruby/setup-ruby#versioning): 29 | uses: ruby/setup-ruby@v1 30 | with: 31 | ruby-version: ${{ matrix.ruby-version }} 32 | bundler-cache: true # runs 'bundle install' and caches installed gems automatically 33 | - name: Run tests 34 | run: bundle exec rake 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | .bundle 4 | .config 5 | .yardoc 6 | Gemfile.lock 7 | InstalledFiles 8 | _yardoc 9 | coverage 10 | doc/ 11 | lib/bundler/man 12 | pkg 13 | rdoc 14 | spec/reports 15 | test/tmp 16 | test/version_tmp 17 | tmp 18 | vendor 19 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | gemspec 3 | 4 | gem "codeclimate-test-reporter", group: :test, require: nil 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2013-2016 Spotify AB 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Talos 2 | ===== 3 | 4 | *After faithfully serving Spotify with secrets for 10 years Talos has now been sunset and will not see any new releases.* 5 | 6 | 7 | [![Gem Version](https://badge.fury.io/rb/talos.svg)](http://badge.fury.io/rb/talos) 8 | 9 | Talos is a rack application which serves Hiera yaml files over HTTP. 10 | It authorizes clients based on the SSL certificates issued by the Puppet CA and returns only the files in the 11 | [Hiera scope](https://github.com/puppetlabs/docs-archive/blob/master/hiera/3.3/command_line.markdown#json-and-yaml-scopes). 12 | 13 | Talos is used to store and distribute secrets via Hiera to the masterless puppet clients. 14 | 15 | How it works 16 | ------------ 17 | Talos listens for incoming HTTP requests and returns compressed hiera 18 | tree based on the client's SSL certificate. 19 | 20 | To determine the list of files to send, Talos matches the certificate 21 | common name against a list of regular expressions. 22 | 23 | Fetching the tree 24 | ----------------- 25 | 26 | It's possible to run a cron task or create a wrapper around the puppet 27 | agent. Here's an example of the client-side code which uses local puppet SSL key 28 | to authenticate: 29 | 30 | ```ruby 31 | require 'puppet' 32 | Puppet[:confdir] = '/etc/puppetlabs/puppet/' 33 | `/usr/bin/curl -s --fail -X GET -k https://talos.internal}/ \ 34 | --cert #{Puppet[:hostcert]} --key #{Puppet[:hostprivkey]} \ 35 | --data-urlencode pool=#{Facter.value(:pool)} > /etc/talos/tree.tar.gz` 36 | `/bin/tar xzf /etc/talos/tree.tar.gz -C /etc/talos/hiera_secrets` 37 | ``` 38 | 39 | In this example the client also passes `pool` variable which will 40 | be included in the Hiera scope if `unsafe_scopes` option is enabled. 41 | 42 | The received copy of the tree could be included in the local hiera config 43 | and used in the normal puppet runs. 44 | 45 | Configuration 46 | ------------- 47 | Talos configuration is stored in `/etc/talos/talos.yaml`: 48 | 49 | ```yaml 50 | scopes: 51 | # lon-puppet-a1: site = lon, role = puppet, pool = a 52 | - match: '(?[[:alpha:]]+)-(?[a-z0-9]+)-(?[[:alpha:]]+)' 53 | facts: 54 | environment: production 55 | - match: 'cloud\.example\.com' 56 | facts: 57 | environment: testing 58 | 59 | unsafe_scopes: true 60 | ssl: true 61 | ``` 62 | 63 | When receiving a request, Talos iterates over `scopes` list and matches 64 | the client certificate against the `match` blocks. If the match is 65 | successful, Talos does 2 things: 66 | 67 | 1. Adds all the named captures from the regexp to the Hiera scope 68 | 2. Adds all the `facts` to the Hiera scope 69 | 70 | Talos will iterate over all the regexps updating the 71 | Hiera scope, meaning that the later matches will override the existing 72 | scope on collision. 73 | 74 | If `unsafe_scopes` option is enabled, Talos will also add all the parameters 75 | passed by the client to the Hiera scope. 76 | 77 | The `ssl` option defaults to enabled. When disabled, the `fqdn` query parameter 78 | is used to determine scopes rather than the client certificate. 79 | 80 | Hiera 81 | ----- 82 | You need to provide `/etc/talos/hiera.yaml` file to configure Hiera 83 | backend on the Talos server: 84 | 85 | ```yaml 86 | --- 87 | :backends: 88 | - yaml 89 | :hierarchy: 90 | - 'hiera-secrets/fqdn/%{fqdn}' 91 | - 'hiera-secrets/role/%{role}/%{pod}/%{pool}' 92 | - 'hiera-secrets/role/%{role}/%{pod}' 93 | - 'hiera-secrets/role/%{role}' 94 | - 'hiera-secrets/pod/%{pod}' 95 | - 'hiera-secrets/common' 96 | :yaml: 97 | :datadir: '/etc/puppet' 98 | :merge_behavior: :deeper 99 | ``` 100 | 101 | Talos will use the `datadir` option to search for YAML files and it 102 | will return only the files that match the Hiera scope of the clients. 103 | 104 | 105 | Installing 106 | ---------- 107 | 108 | You can use [spotify/talos](https://github.com/spotify/puppet-talos) 109 | puppet module to install Talos. 110 | 111 | ### Manual installation 112 | 113 | First, install talos using rubygems: 114 | 115 | $ gem install talos 116 | 117 | Create a separate user and Document Root for the Rack application: 118 | 119 | $ useradd talos --system --create-home --home-dir /var/lib/talos 120 | $ mkdir -p /var/lib/talos/public /var/lib/talos/tmp /etc/talos 121 | $ chown -R talos:talos /var/lib/talos/ /etc/talos 122 | 123 | Then copy [config.ru](config.ru) to `/var/lib/talos/` directory. 124 | 125 | You also need to copy and adjust [hiera.yaml](spec/fixtures/hiera.yaml) and 126 | [talos.yaml](spec/fixtures/talos.yaml) configs in `/etc/talos` directory. 127 | 128 | ### Hiera repository 129 | 130 | You need to have a copy of the hiera-secrets repository available on the 131 | talos server. Make sure it's located at the `datadir` specified in 132 | `/etc/talos/hiera.yaml` 133 | 134 | ### Apache 135 | 136 | You can run Talos using Passenger or any other application server. Make 137 | sure you use Puppet SSL keys to validate the client certificates and to 138 | forward `SSL_CLIENT_S_DN_CN` header: 139 | 140 | ```apacheconf 141 | 142 | DocumentRoot "/var/lib/talos/public" 143 | 144 | 145 | Require all granted 146 | 147 | 148 | SSLEngine on 149 | SSLCertificateFile "/etc/puppetlabs/puppet/ssl/certs/talos.internal.pem" 150 | SSLCertificateKeyFile "/etc/puppetlabs/puppet/ssl/private_keys/talos.internal.pem" 151 | SSLCertificateChainFile "/etc/puppetlabs/puppet/ssl/certs/ca.pem" 152 | SSLCACertificatePath "/etc/ssl/certs" 153 | SSLCACertificateFile "/etc/puppetlabs/puppet/ssl/certs/ca.pem" 154 | SSLCARevocationFile "/etc/puppetlabs/puppet/ssl/crl.pem" 155 | SSLVerifyClient require 156 | SSLOptions +StdEnvVars +FakeBasicAuth 157 | RequestHeader set SSL_CLIENT_S_DN_CN "%{SSL_CLIENT_S_DN_CN}s" 158 | 159 | ``` 160 | 161 | Contributing 162 | ------------ 163 | 1. Fork the project on github 164 | 2. Create your feature branch 165 | 3. Open a Pull Request 166 | 167 | This project adheres to the [Open Code of Conduct][code-of-conduct]. By 168 | participating, you are expected to honor this code. 169 | 170 | [code-of-conduct]: 171 | https://github.com/spotify/code-of-conduct/blob/master/code-of-conduct.md 172 | 173 | License 174 | ------- 175 | ```text 176 | Copyright 2013-2016 Spotify AB 177 | 178 | Licensed under the Apache License, Version 2.0 (the "License"); 179 | you may not use this file except in compliance with the License. 180 | You may obtain a copy of the License at 181 | 182 | http://www.apache.org/licenses/LICENSE-2.0 183 | 184 | Unless required by applicable law or agreed to in writing, software 185 | distributed under the License is distributed on an "AS IS" BASIS, 186 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 187 | See the License for the specific language governing permissions and 188 | limitations under the License. 189 | ``` 190 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env rake 2 | require "bundler/gem_tasks" 3 | 4 | require 'rake' 5 | require 'rspec/core/rake_task' 6 | 7 | task :default => :test 8 | task :test => [:spec] 9 | 10 | RSpec::Core::RakeTask.new(:spec) do |t| 11 | t.pattern = 'spec/*_spec.rb' 12 | end 13 | -------------------------------------------------------------------------------- /catalog-info.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: backstage.io/v1alpha1 2 | kind: Component 3 | metadata: 4 | name: talos 5 | spec: 6 | type: library 7 | owner: wasabi 8 | -------------------------------------------------------------------------------- /config.ru: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | # uncomment if you wish to run from source code 3 | # libdir = File.join(File.dirname(__FILE__), 'lib') 4 | # $LOAD_PATH.unshift(libdir) unless $LOAD_PATH.include?(libdir) 5 | require 'talos' 6 | run Talos 7 | -------------------------------------------------------------------------------- /lib/talos.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | #-- 3 | # Copyright 2015 Spotify AB 4 | # 5 | # The contents of this file are licensed under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with the 7 | # License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | # License for the specific language governing permissions and limitations under 15 | # the License. 16 | #++ 17 | 18 | require 'sinatra/base' 19 | require 'json' 20 | require 'hiera' 21 | require 'stringio' 22 | require 'zlib' 23 | require 'archive/tar/minitar' 24 | require 'pathname' 25 | include Archive::Tar 26 | 27 | 28 | class Talos < Sinatra::Base 29 | def self.prepare_config(path) 30 | set :talos, YAML.load_file(path) 31 | settings.talos['ssl'] = true if settings.talos['ssl'].nil? 32 | settings.talos['scopes'].each do |scope_config| 33 | begin 34 | scope_config['regexp'] = Regexp.new(scope_config['match']) 35 | rescue 36 | fail "Invalid regexp: #{scope_config['match']}" 37 | end 38 | end 39 | end 40 | 41 | configure :development, :test do 42 | require 'sinatra/reloader' 43 | register Sinatra::Reloader 44 | set :hiera, Hiera::Config::load(File.expand_path('spec/fixtures/hiera.yaml')) 45 | prepare_config('spec/fixtures/talos.yaml') 46 | set :show_exceptions, false 47 | end 48 | 49 | configure :production do 50 | set :hiera, Hiera::Config::load(File.expand_path('/etc/talos/hiera.yaml')) 51 | prepare_config('/etc/talos/talos.yaml') 52 | warn("SECURITY WARNING: use of ssl is disabled, client requests cannot be authenticated") if !settings.talos['ssl'] 53 | warn("SECURITY WARNING: unsafe_scopes are enabled, SSL authentication bypass is possible") if settings.talos['unsafe_scopes'] 54 | end 55 | 56 | def absolute_datadir 57 | datadir = settings.hiera[:yaml][:datadir] 58 | Pathname.new(datadir).relative? ? File.join(File.dirname(__FILE__), '..', datadir) : datadir 59 | end 60 | 61 | # Extracts scopes from FQDN using regexp with named captures 62 | # Falls back to insecure arguments passed by the puppet agent 63 | # (needed for the hosts not following naming convention) 64 | def get_scope(fqdn) 65 | scope = {'fqdn' => fqdn} 66 | settings.talos['scopes'].each do | scope_config| 67 | if m = fqdn.match(scope_config['regexp']) 68 | scope.update(Hash[ m.names.zip( m.captures ) ]) 69 | scope.update(scope_config['facts']) 70 | end 71 | end 72 | 73 | unsafe_scope = settings.talos['unsafe_scopes'] ? request.env['rack.request.query_hash'] : {} 74 | scope.update(unsafe_scope) 75 | # scope = {"pod"=>"lon3", "site"=>"lon", "role"=>"puppet", "pool"=>"a"} 76 | scope 77 | end 78 | 79 | def files_in_scope(scope) 80 | files = [] 81 | Hiera::Backend.datasources(scope, nil) do |source, yamlfile| 82 | yamlfile = Hiera::Backend.datafile(:yaml, scope, source, 'yaml') || next 83 | next unless File.exist?(yamlfile) 84 | # Strip path from filename 85 | files << yamlfile.gsub(settings.hiera[:yaml][:datadir] + '/', '') 86 | end 87 | files 88 | end 89 | 90 | def compress_files(files) 91 | output = StringIO.new 92 | begin 93 | sgz = Zlib::GzipWriter.new(output) 94 | tar = Minitar::Output.new(sgz) 95 | Dir.chdir(absolute_datadir) { files.each { |f| Minitar.pack_file(f, tar) } } 96 | ensure 97 | tar.close 98 | end 99 | output 100 | end 101 | 102 | get '/' do 103 | fqdn_env = request.env['HTTP_SSL_CLIENT_S_DN_CN'] ? request.env['HTTP_SSL_CLIENT_S_DN_CN'] : request.env['SSL_CLIENT_S_DN_CN'] 104 | fqdn = (settings.development? || !settings.talos['ssl']) ? params[:fqdn] : fqdn_env 105 | scope = get_scope(fqdn) 106 | files_to_pack = files_in_scope(scope) 107 | archive = compress_files(files_to_pack) 108 | content_type 'application/x-gzip' 109 | headers['content-encoding'] = 'gzip' 110 | archive.string 111 | end 112 | 113 | # Get the checksum the data folder symlink to 114 | # Internal API 115 | get '/status' do 116 | begin 117 | File.readlink(absolute_datadir).split('.').last 118 | rescue 119 | halt 'Failed to fetch commit id' 120 | end 121 | end 122 | 123 | run! if app_file == $0 124 | end 125 | -------------------------------------------------------------------------------- /spec/fixtures/hiera.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | :backends: 3 | - yaml 4 | 5 | :hierarchy: 6 | - 'fqdn/%{fqdn}' 7 | - 'role/%{role}/%{environment}/%{pool}' 8 | - 'role/%{role}/%{pool}' 9 | - 'role/%{role}/%{environment}' 10 | - 'role/%{role}' 11 | - 'lsbdistcodename/%{lsbdistcodename}' 12 | - 'domain/%{domain}' 13 | - 'pod/%{pod}' 14 | - 'site/%{site}' 15 | - 'environment/%{environment}' 16 | - common 17 | 18 | :yaml: 19 | :datadir: spec/fixtures/master 20 | 21 | :logger: puppet 22 | -------------------------------------------------------------------------------- /spec/fixtures/master: -------------------------------------------------------------------------------- 1 | master.3fa3fd97848a72ae539b75bccd6028cd1d4e92e3 -------------------------------------------------------------------------------- /spec/fixtures/master.3fa3fd97848a72ae539b75bccd6028cd1d4e92e3/common.yaml: -------------------------------------------------------------------------------- 1 | sysadmin: 'admin@example' 2 | classes: 3 | - 'hostbase' 4 | -------------------------------------------------------------------------------- /spec/fixtures/master.3fa3fd97848a72ae539b75bccd6028cd1d4e92e3/fqdn/foo.bar.yaml: -------------------------------------------------------------------------------- 1 | bar: bo 2 | -------------------------------------------------------------------------------- /spec/fixtures/master.3fa3fd97848a72ae539b75bccd6028cd1d4e92e3/role/foobar/testing.yaml: -------------------------------------------------------------------------------- 1 | foo::bar: baz 2 | -------------------------------------------------------------------------------- /spec/fixtures/master.3fa3fd97848a72ae539b75bccd6028cd1d4e92e3/role/foobar/z.yaml: -------------------------------------------------------------------------------- 1 | foo::bar: 'buzz' -------------------------------------------------------------------------------- /spec/fixtures/master.3fa3fd97848a72ae539b75bccd6028cd1d4e92e3/role/puppet.yaml: -------------------------------------------------------------------------------- 1 | sysadmin: 'sysadmin@example.com' 2 | classes: 3 | - 'puppet' 4 | - 'puppet::master' 5 | -------------------------------------------------------------------------------- /spec/fixtures/master.3fa3fd97848a72ae539b75bccd6028cd1d4e92e3/site/sjc.yaml: -------------------------------------------------------------------------------- 1 | foo::bar: baz 2 | -------------------------------------------------------------------------------- /spec/fixtures/talos.yaml: -------------------------------------------------------------------------------- 1 | unsafe_scopes: true 2 | 3 | scopes: 4 | # lon3-puppet-a1: pod = lon3, site = lon3, role = puppet, pool = a 5 | - match: '(?(?[[:alpha:]]+)\d*)-(?[a-z0-9]+)-(?[[:alpha:]]+)\d+' 6 | facts: 7 | environment: production 8 | - match: 'cloud\.example\.com' 9 | facts: 10 | environment: testing 11 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require File.join(File.dirname(__FILE__), '../lib/talos.rb') 2 | 3 | require 'sinatra' 4 | require 'rack/test' 5 | Talos.environment = :development 6 | set :run, false 7 | set :raise_errors, true 8 | set :logging, true 9 | 10 | def app 11 | Talos 12 | end 13 | 14 | RSpec.configure do |config| 15 | config.include Rack::Test::Methods 16 | end 17 | -------------------------------------------------------------------------------- /spec/talos_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'tempfile' 3 | 4 | describe 'talos' do 5 | 6 | def match_query_to_files(query, files) 7 | get query 8 | expect(last_response).to be_ok 9 | Tempfile.open('spec') do |file| 10 | file.write(last_response.body) 11 | file.flush 12 | files_in_archive = `tar -tf #{file.path}`.split 13 | files.each { |f| expect(files.sort).to eq(files_in_archive.sort) } 14 | end 15 | end 16 | 17 | it 'should detect scope and return YAML files' do 18 | { '/?fqdn=testing.int.sto.example.com' => 19 | %w(common.yaml), 20 | '/?role=puppet&pod=sto3&fqdn=sto3-puppet-a1.sto3.example.com' => 21 | %w(common.yaml role/puppet.yaml), 22 | '/?fqdn=sto3-puppet-a1.sto3.example.com' => 23 | %w(common.yaml role/puppet.yaml), 24 | '/?role=puppet&pod=sto3&fqdn=foo.bar' => 25 | %w(common.yaml role/puppet.yaml fqdn/foo.bar.yaml), 26 | '/?fqdn=something.random&role=foobar&pool=z' => 27 | %w(common.yaml role/foobar/z.yaml), 28 | '/?fqdn=sjc1-puppet-a1' => 29 | %w(common.yaml role/puppet.yaml site/sjc.yaml), 30 | '/?fqdn=sjc1-foobar-a1.cloud.example.com' => 31 | %w(common.yaml site/sjc.yaml role/foobar/testing.yaml), 32 | }.each do |query, files| 33 | match_query_to_files(query, files) 34 | end 35 | end 36 | 37 | it 'should resturn the checksum master symlink to' do 38 | get '/status' 39 | expect(last_response).to be_ok 40 | expect(last_response.body).to match('3fa3fd97848a72ae539b75bccd6028cd1d4e92e3') 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /talos.gemspec: -------------------------------------------------------------------------------- 1 | Gem::Specification.new do |s| 2 | s.version = '1.0.5' 3 | s.name = 'talos' 4 | s.authors = ['Alexey Lapitsky', 'Johan Haals', 'Wasabi'] 5 | s.email = 'wasabi@spotify.com' 6 | s.summary = %q{Hiera secrets distribution over HTTP} 7 | s.description = %q{Distribute compressed hiera yaml files to authenticated puppet clients over HTTP} 8 | s.homepage = 'https://github.com/spotify/talos' 9 | s.license = 'Apache-2.0' 10 | 11 | s.files = `git ls-files`.split($\) 12 | s.executables = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) } 13 | s.test_files = s.files.grep(%r{^(test|spec|features)/}) 14 | s.require_paths = ['lib'] 15 | 16 | s.add_dependency 'rack', '2.2.8.1' 17 | s.add_dependency 'sinatra', '~> 2.2.0' 18 | s.add_dependency 'hiera', '~> 3.6.0' 19 | s.add_dependency 'minitar', '~> 0.9' 20 | s.add_development_dependency 'rake' 21 | s.add_development_dependency 'rack-test', '~> 1.1.0' 22 | s.add_development_dependency 'rspec', '>= 2.9' 23 | s.add_development_dependency 'sinatra-contrib', '~> 2.2.0' 24 | end 25 | --------------------------------------------------------------------------------