├── .gitignore ├── .rspec ├── .travis.yml ├── Gemfile ├── LICENSE.txt ├── README.md ├── Rakefile ├── bin └── etcenv ├── etcenv.gemspec ├── lib ├── etcenv.rb └── etcenv │ ├── cli.rb │ ├── dockerenv_file.rb │ ├── dotenv_file.rb │ ├── environment.rb │ ├── utils.rb │ ├── variable_expander.rb │ ├── version.rb │ └── watcher.rb ├── scripts ├── console └── setup └── spec ├── dockerenv_file_spec.rb ├── dotenv_file_spec.rb ├── environment_spec.rb ├── etcenv_spec.rb ├── spec_helper.rb └── variable_expander_spec.rb /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /Gemfile.lock 4 | /_yardoc/ 5 | /coverage/ 6 | /doc/ 7 | /pkg/ 8 | /spec/reports/ 9 | /tmp/ 10 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --format documentation 2 | --color 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | sudo: false 3 | cache: bundler 4 | rvm: 5 | - "2.2" 6 | - "2.1" 7 | - "2.0.0" 8 | - "ruby-head" 9 | 10 | matrix: 11 | allow_failures: 12 | - rvm: 13 | - "ruby-head" 14 | fast_finish: true 15 | 16 | script: bundle exec rspec -fd ./spec 17 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in etcenv.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Shota Fukumori (sora_h) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Etcenv: Dump etcd keys into dotenv file or docker env file 2 | 3 | [![Build Status](https://travis-ci.org/sorah/etcenv.svg)](https://travis-ci.org/sorah/etcenv) 4 | 5 | ## Installation 6 | 7 | $ gem install etcenv 8 | 9 | ## Simple Usage 10 | 11 | ### Create directory and keys on etcd 12 | 13 | ``` 14 | etcdctl mkdir /my-app 15 | etcdctl set /my-app/AWESOME_SERVICE_CREDENTIAL xxxxxx 16 | ``` 17 | 18 | ### Run `etcenv` 19 | 20 | #### One-shot 21 | 22 | This will output generated .env file to STDOUT. 23 | 24 | ``` 25 | $ etcenv /my-app 26 | AWESOME_SERVICE_CREDENTIAL=xxxxxx 27 | ``` 28 | 29 | or 30 | 31 | ``` 32 | $ etcenv -o .env /my-app 33 | $ cat .env 34 | AWESOME_SERVICE_CREDENTIAL=xxxxxx 35 | ``` 36 | 37 | to save as file. 38 | 39 | #### Continuously update 40 | 41 | Etcenv also supports watching etcd server. In `--watch` mode, Etcenv updates dotenv file when value gets updated: 42 | 43 | ``` 44 | $ etcenv --watch -o .env /my-env 45 | ``` 46 | 47 | Also you can start it as daemon: 48 | 49 | ``` 50 | $ etcenv --watch --daemon /path/to/pidfile.pid -o .env /my-env 51 | ``` 52 | 53 | #### For docker 54 | 55 | Use `--docker` flag to generate file for docker's `--env-file` option. 56 | 57 | In docker mode, etcenv evaluates `${...}` expansion like dotenv do. 58 | 59 | ## Options 60 | 61 | ### etcd options 62 | 63 | - `--etcd`: URL of etcd to connect to. Path in URL will be ignored. 64 | - `--etcd-ca-file`: Path to CA certificate file (PEM) of etcd server. 65 | - `--etcd-cert-file`: Path to client certificate file for etcd. 66 | - `--etcd-key-file`: Path to private key file of client certificate file for etcd. 67 | 68 | ## Advanced usage 69 | 70 | ### Include other directory's variables 71 | 72 | Set directory path to `.include`. Directories can be specified multiple, separated by comma. 73 | 74 | ``` 75 | etcdctl mkdir /common 76 | etcdctl set /common/COMMON_SECRET xxx 77 | etcdctl set /my-app/.include /common 78 | ``` 79 | 80 | Also, you can omit path of parent directory: 81 | 82 | ``` 83 | etcdctl mkdir /envs/common 84 | etcdctl set /envs/common/COMMON_SECRET xxx 85 | 86 | etcdctl mkdir /envs/my-app 87 | etcdctl set /envs/my-app/.include common 88 | ``` 89 | 90 | - `.include` will be applied recursively (up to 10 times by default). If `.include` is looping, it'll be an error. 91 | - For multiple `.include`, value for same key may be overwritten. 92 | - If `a` includes `b`,`c` and `b` includes `d`, result for `a` will be: `d`, `b`, `c`, then `a`. 93 | 94 | ## Development 95 | 96 | After checking out the repo, run `scripts/setup` to install dependencies. Then, run `scripts/console` for an interactive prompt that will allow you to experiment. 97 | 98 | To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org). 99 | 100 | ## Contributing 101 | 102 | 1. Fork it ( https://github.com/sorah/etcenv/fork ) 103 | 2. Create your feature branch (`git checkout -b my-new-feature`) 104 | 3. Commit your changes (`git commit -am 'Add some feature'`) 105 | 4. Push to the branch (`git push origin my-new-feature`) 106 | 5. Create a new Pull Request 107 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | -------------------------------------------------------------------------------- /bin/etcenv: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'etcenv' 3 | require 'etcenv/cli' 4 | 5 | exit Etcenv::Cli.new(*ARGV).run 6 | -------------------------------------------------------------------------------- /etcenv.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'etcenv/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = "etcenv" 8 | spec.version = Etcenv::VERSION 9 | spec.authors = ["Shota Fukumori (sora_h)"] 10 | spec.email = ["sorah@cookpad.com"] 11 | 12 | spec.summary = %q{Dump etcd keys into dotenv file or docker env file} 13 | spec.homepage = "https://github.com/sorah/etcenv" 14 | spec.license = "MIT" 15 | 16 | spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } 17 | spec.bindir = "bin" 18 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 19 | spec.require_paths = ["lib"] 20 | 21 | spec.add_development_dependency "bundler" 22 | spec.add_development_dependency "rake", "~> 10.0" 23 | spec.add_development_dependency "rspec" 24 | 25 | spec.add_dependency "etcd", ">= 0.3.0" 26 | spec.add_dependency "etcd-etcvault" 27 | end 28 | -------------------------------------------------------------------------------- /lib/etcenv.rb: -------------------------------------------------------------------------------- 1 | require 'etcenv/version' 2 | require 'etcenv/environment' 3 | require 'etcenv/variable_expander' 4 | require 'etcenv/dockerenv_file' 5 | require 'etcenv/dotenv_file' 6 | -------------------------------------------------------------------------------- /lib/etcenv/cli.rb: -------------------------------------------------------------------------------- 1 | require 'optparse' 2 | require 'thread' 3 | require 'openssl' 4 | require 'etcd' 5 | require 'uri' 6 | require 'etcenv/dockerenv_file' 7 | require 'etcenv/dotenv_file' 8 | require 'etcenv/watcher' 9 | 10 | module Etcenv 11 | class Cli 12 | def initialize(*argv) 13 | @argv = argv 14 | @options = { 15 | etcd: URI.parse("http://localhost:2379"), 16 | format: DotenvFile, 17 | perm: 0600, 18 | mode: :oneshot, 19 | } 20 | parse! 21 | end 22 | 23 | attr_reader :argv, :options 24 | 25 | def run 26 | parse! 27 | 28 | case options[:mode] 29 | when :oneshot 30 | oneshot 31 | when :watch 32 | watch 33 | else 34 | raise "[BUG] unknown mode" 35 | end 36 | end 37 | 38 | def parse! 39 | parser = OptionParser.new do |opts| 40 | opts.banner = "Usage: etcenv [options] key ..." 41 | 42 | opts.on("-h", "--help", "show this help") do 43 | puts opts 44 | exit 0 45 | end 46 | 47 | opts.on("--watch", "-w", "continuous update mode") do 48 | options[:mode] = :watch 49 | end 50 | 51 | opts.on("--daemon", "-d", "daemonize") do 52 | options[:daemonize] = true 53 | end 54 | 55 | opts.on("--pidfile PIDFILE", "-p PIDFILE", "pidfile to use when deamonizing") do |pidpath| 56 | options[:pidfile] = pidpath 57 | end 58 | 59 | opts.on("-o PATH", "--output PATH", "save to speciifed file") do |path| 60 | options[:output] = path 61 | end 62 | 63 | opts.on("-m MODE", "--mode MODE", "mode (permission) to use when creating --output file, in octal. Default: 0600") do |perm| 64 | options[:perm] = perm.to_i(8) 65 | end 66 | 67 | opts.on("--docker", "use docker env-file format instead of dotenv.gem format") do 68 | options[:format] = DockerenvFile 69 | end 70 | 71 | opts.on("--etcd URL", "URL of etcd to connect; Any paths are ignored.") do |url| 72 | options[:etcd] = URI.parse(url) 73 | end 74 | 75 | opts.on("--etcd-ca-file PATH", "Path to CA certificate file (PEM) of etcd server") do |path| 76 | options[:etcd_ca_file] = path 77 | end 78 | 79 | opts.on("--etcd-cert-file PATH", "Path to client certificate file (PEM) for etcd server") do |path| 80 | options[:etcd_tls_cert] = OpenSSL::X509::Certificate.new(File.read(path)) 81 | end 82 | 83 | opts.on("--etcd-key-file PATH", "Path to private key file for client certificate for etcd server") do |path| 84 | options[:etcd_tls_key] = OpenSSL::PKey::RSA.new(File.read(path)) 85 | end 86 | end 87 | parser.parse!(argv) 88 | end 89 | 90 | def etcd 91 | @etcd ||= Etcd.client( 92 | host: options[:etcd].host, 93 | port: options[:etcd].port, 94 | use_ssl: options[:etcd].scheme == 'https', 95 | ca_file: options[:etcd_ca_file], 96 | ssl_cert: options[:etcd_tls_cert], 97 | ssl_key: options[:etcd_tls_key], 98 | ) 99 | end 100 | 101 | def oneshot 102 | if argv.empty? 103 | $stderr.puts "error: no KEY specified. See --help for detail" 104 | return 1 105 | end 106 | env = argv.inject(nil) do |env, key| 107 | new_env = Environment.new(etcd, key).expanded_env 108 | env ? env.merge(new_env) : new_env 109 | end 110 | dump_env(env) 111 | 112 | 0 113 | end 114 | 115 | def watch 116 | if argv.empty? 117 | $stderr.puts "error: no KEY specified. See --help for detail" 118 | return 1 119 | end 120 | 121 | if options[:daemonize] 122 | $stderr.puts "Daemonizing" 123 | Process.daemon(nil, true) 124 | if options[:pidfile] 125 | File.write options[:pidfile], "#{$$}\n" 126 | end 127 | end 128 | 129 | envs = argv.map { |key| Environment.new(etcd, key) } 130 | 131 | watchers = envs.map { |env| Watcher.new(env, verbose: true) } 132 | 133 | dumper_ch = Queue.new 134 | dumper = Thread.new do 135 | loop do 136 | $stderr.puts "[dumper] dumping env" 137 | env = envs.inject(nil) do |result, env| 138 | new_env = env.expanded_env 139 | result ? result.merge(new_env) : new_env 140 | end 141 | dump_env(env) 142 | dumper_ch.pop 143 | end 144 | end 145 | dumper.abort_on_exception = true 146 | 147 | watchers.map do |watcher| 148 | Thread.new do 149 | watcher.auto_reload_loop do 150 | dumper_ch << true 151 | end 152 | end 153 | end 154 | 155 | loop { sleep 1 } 156 | end 157 | 158 | private 159 | 160 | def dump_env(env) 161 | env_file = options[:format].new(env) 162 | if options[:output] 163 | open(options[:output], "w", options[:perm]) do |io| 164 | io.puts env_file.to_s 165 | end 166 | else 167 | $stdout.puts env_file.to_s 168 | end 169 | end 170 | end 171 | end 172 | -------------------------------------------------------------------------------- /lib/etcenv/dockerenv_file.rb: -------------------------------------------------------------------------------- 1 | require 'etcenv/variable_expander' 2 | 3 | module Etcenv 4 | class DockerenvFile 5 | def initialize(env) 6 | @env = env 7 | end 8 | 9 | attr_reader :env 10 | 11 | def lines 12 | env.map { |k,v| "#{k}=#{v}" } 13 | end 14 | 15 | def to_s 16 | lines.join(?\n) + ?\n 17 | end 18 | 19 | private 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /lib/etcenv/dotenv_file.rb: -------------------------------------------------------------------------------- 1 | require 'etcenv/variable_expander' 2 | 3 | module Etcenv 4 | class DotenvFile 5 | def initialize(env) 6 | @env = env 7 | end 8 | 9 | attr_reader :env 10 | 11 | def lines 12 | env.map { |k, v| make_dotenv_line(k, v) } 13 | end 14 | 15 | def to_s 16 | lines.join(?\n) + ?\n 17 | end 18 | 19 | private 20 | 21 | SHOULD_QUOTE = /\r|\n|"|#|\$/ 22 | def make_dotenv_line(k,v) 23 | if v.match(SHOULD_QUOTE) 24 | v.gsub!('"', '\"') 25 | v.gsub!(/\r?\n/, '\n') 26 | v.gsub!(/\$([^(])/, '\$\1') 27 | "#{k}=\"#{v}\"" 28 | else 29 | "#{k}=#{v}" 30 | end 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/etcenv/environment.rb: -------------------------------------------------------------------------------- 1 | require 'etcenv/variable_expander' 2 | require 'etcd' 3 | require 'etcd/etcvault' 4 | 5 | module Etcenv 6 | class Environment 7 | class NotDirectory < StandardError; end 8 | class DepthLimitError < StandardError; end 9 | class LoopError < StandardError; end 10 | class KeyNotFound < StandardError; end 11 | class EtcvaultFailure < StandardError; end 12 | 13 | INCLUDE_KEY = '.include' 14 | MAX_DEPTH_DEFAULT = 10 15 | 16 | def initialize(etcd, root_key, max_depth: MAX_DEPTH_DEFAULT) 17 | @etcd = etcd 18 | @root_key = root_key 19 | @max_depth = max_depth 20 | @lock = Mutex.new 21 | load 22 | end 23 | 24 | attr_reader :root_key, :env, :etcd, :cluster_index 25 | attr_accessor :max_depth 26 | 27 | def expanded_env 28 | VariableExpander.expand(env) 29 | end 30 | 31 | def modified_indices 32 | @modified_indices ||= {} 33 | end 34 | 35 | def keys 36 | modified_indices.keys 37 | end 38 | 39 | def load 40 | @lock.synchronize do 41 | flush 42 | env = {} 43 | includes.each do |name| 44 | env.merge! fetch(name) 45 | end 46 | env.delete '.include' 47 | @env = env 48 | end 49 | self 50 | end 51 | 52 | private 53 | 54 | def flush 55 | @env = {} 56 | @includes = nil 57 | @cache = {} 58 | @modified_indices = {} 59 | @cluster_index = nil 60 | self 61 | end 62 | 63 | def includes 64 | @includes ||= solve_include_order(root_key) 65 | end 66 | 67 | def default_prefix 68 | root_key.sub(/^(.*)\/.+?$/, '\1') 69 | end 70 | 71 | def resolve_key(key) 72 | if key.start_with?('/') 73 | key 74 | else 75 | default_prefix + '/' + key 76 | end 77 | end 78 | 79 | def cache 80 | @cache ||= {} 81 | end 82 | 83 | def fetch(name) 84 | key = resolve_key(name) 85 | return cache[key] if cache[key] 86 | 87 | resp = @etcd.get(key) 88 | node = resp.node 89 | @cluster_index = [@cluster_index, resp.etcd_index].compact.min 90 | 91 | if node.etcvault_error 92 | raise EtcvaultFailure, node.etcvault_error 93 | end 94 | 95 | if node.dir 96 | dir = {} 97 | index = 0 98 | 99 | node.children.each do |child| 100 | name = child.key.sub(/^.*\//, '') 101 | 102 | index = [index, child.modified_index].max 103 | if child.dir 104 | next 105 | else 106 | dir[name] = child.value 107 | if child.etcvault_error 108 | raise EtcvaultFailure, child.etcvault_error 109 | end 110 | end 111 | end 112 | else 113 | dir = {key.sub(/^.*\//, '') => node.value} 114 | index = node.modified_index 115 | end 116 | 117 | modified_indices[key] = index 118 | cache[key] = dir 119 | rescue Etcd::KeyNotFound 120 | raise KeyNotFound, "Couldn't find key #{key}" 121 | end 122 | 123 | def solve_include_order(name, path = []) 124 | if path.include?(name) 125 | raise LoopError, "Found an include loop at path: #{path.inspect}" 126 | end 127 | 128 | path = path + [name] 129 | if max_depth < path.size 130 | raise DepthLimitError, "Reached maximum depth (path: #{path.inspect})" 131 | end 132 | 133 | node = fetch(name) 134 | 135 | if node[INCLUDE_KEY] 136 | node[INCLUDE_KEY].split(/,\s*/).flat_map do |x| 137 | solve_include_order(x, path) 138 | end + [name] 139 | else 140 | [name] 141 | end 142 | end 143 | end 144 | end 145 | -------------------------------------------------------------------------------- /lib/etcenv/utils.rb: -------------------------------------------------------------------------------- 1 | module Etcenv 2 | module Utils 3 | class << self 4 | def uniq_with_keeping_first_appearance(array) 5 | set = {} 6 | result = [] 7 | array.each do |x| 8 | next if set[x] 9 | result.push x 10 | set[x] = true 11 | end 12 | result 13 | end 14 | end 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /lib/etcenv/variable_expander.rb: -------------------------------------------------------------------------------- 1 | require 'etcenv/utils' 2 | 3 | module Etcenv 4 | class VariableExpander 5 | class LoopError < StandardError; end 6 | class DepthLimitError < StandardError; end 7 | 8 | MAX_DEPTH_DEFAULT = 50 9 | 10 | def self.expand(variables, max = MAX_DEPTH_DEFAULT) 11 | new(variables).expand(max) 12 | end 13 | 14 | def initialize(variables) 15 | @variables = variables.dup.freeze 16 | end 17 | 18 | attr_reader :variables 19 | 20 | def expand(max = MAX_DEPTH_DEFAULT) 21 | detect_loop! 22 | 23 | result = {} 24 | solve_order(max).map do |x| 25 | result[x] = single_expand(@variables[x], result) 26 | end 27 | result 28 | end 29 | 30 | def variable_with_deps 31 | @variables.map { |k, v| [k, [v, detect_variables(v)]] } 32 | end 33 | 34 | def dependees_by_variable 35 | @dependees_by_variable ||= variable_with_deps.inject({}) do |r, x| 36 | k, v, deps = x[0], x[1][0], x[1][1] 37 | 38 | deps.each do |dep| 39 | r[dep] ||= [] 40 | r[dep] << k 41 | end 42 | r 43 | end.freeze 44 | end 45 | 46 | def root_variables 47 | @root_variables ||= variable_with_deps.inject([]) do |r, x| 48 | k, v, deps = x[0], x[1][0], x[1][1] 49 | if deps.empty? 50 | r << k 51 | end 52 | r 53 | end + (variables.keys - dependees_by_variable.keys) 54 | end 55 | 56 | def detect_loop! 57 | if !variables.empty? && root_variables.empty? 58 | raise LoopError, "there's no possible root variables (variables which don't have dependee)" 59 | else 60 | dependees_by_variable.each do |k, deps| 61 | deps.each do |dep| 62 | if (dependees_by_variable[dep] || []).include?(k) 63 | raise LoopError, "There's a loop between $#{dep} and $#{k}" 64 | end 65 | end 66 | end 67 | end 68 | end 69 | 70 | def solve_order(max_depth = 10) 71 | order = [] 72 | solve = nil 73 | solve = ->(vs, depth = 0) do 74 | raise DepthLimitError if depth.succ > max_depth 75 | vs.each do |x| 76 | order.concat solve.call(dependees_by_variable[x] || [], depth.succ) + [x] 77 | end 78 | end 79 | solve.call(root_variables) 80 | 81 | Utils.uniq_with_keeping_first_appearance(order).reverse 82 | end 83 | 84 | private 85 | 86 | VARIABLE = / 87 | (?\\?) 88 | \$ 89 | ( 90 | {(?[a-zA-Z0-9_]+)} 91 | | 92 | \g 93 | ) 94 | /x 95 | 96 | def detect_variables(str) 97 | result = [] 98 | 99 | pos = 0 100 | while match = str.match(VARIABLE, pos) 101 | pos = match.end(0) 102 | next if match['escape'] == '\\' 103 | 104 | result << match['name'] 105 | end 106 | 107 | result 108 | end 109 | 110 | def single_expand(str, variables) 111 | str.gsub(VARIABLE) do |variable| 112 | match = $~ 113 | 114 | if match['escape'] == '\\' 115 | variable[1..-1] 116 | else 117 | variables[match['name']].to_s 118 | end 119 | end 120 | end 121 | end 122 | end 123 | -------------------------------------------------------------------------------- /lib/etcenv/version.rb: -------------------------------------------------------------------------------- 1 | module Etcenv 2 | VERSION = "0.6.0" 3 | end 4 | -------------------------------------------------------------------------------- /lib/etcenv/watcher.rb: -------------------------------------------------------------------------------- 1 | require 'thread' 2 | 3 | module Etcenv 4 | class Watcher 5 | WATCH_TIMEOUT = 120 6 | 7 | def initialize(env, verbose: false) 8 | @env = env 9 | @verbose = verbose 10 | @indices = {} 11 | @lock = Mutex.new 12 | end 13 | 14 | attr_reader :env, :verbose 15 | 16 | def etcd 17 | env.etcd 18 | end 19 | 20 | def watch 21 | ch = Queue.new 22 | threads = env.keys.map do |key| 23 | Thread.new(ch, key, env.cluster_index, &method(:watch_thread)).tap do |th| 24 | th.abort_on_exception = true 25 | end 26 | end 27 | report = ch.pop 28 | threads.each(&:kill) 29 | report 30 | end 31 | 32 | def auto_reload_loop 33 | retries = 0 34 | loop do 35 | begin 36 | watch 37 | $stderr.puts "[watcher] reloading env #{env.root_key}" if verbose 38 | env.load 39 | yield env if block_given? 40 | rescue => e 41 | retries += 1 42 | interval = (2**retries) * 0.1 43 | 44 | $stderr.puts "[watcher][error] Failed to reload env #{env.root_key}: #{e.inspect}" 45 | $stderr.puts "\t#{e.backtrace.join("\n\t")}" 46 | $stderr.puts "[watcher][error] RETRYING reload #{env.root_key} in #{'%.2f' % interval} sec" 47 | sleep interval 48 | else 49 | retries = 0 50 | end 51 | end 52 | end 53 | 54 | private 55 | 56 | def watch_thread(ch, key, index) 57 | tries = 0 58 | loop do 59 | if try_watch(key, index) 60 | ch << key 61 | break 62 | end 63 | interval = (2 ** tries) * 0.1 64 | $stderr.puts "[watcher] RETRYING; #{key.inspect} watch will resume after #{'%.2f' % interval} sec" 65 | sleep interval 66 | tries += 1 67 | end 68 | end 69 | 70 | def try_watch(key, index) 71 | $stderr.puts "[watcher] waiting for change on #{key} (index: #{index.succ})" if verbose 72 | index = [@indices[key], index].compact.max 73 | index += 1 if index 74 | response = etcd.watch(key, recursive: true, index: index, timeout: WATCH_TIMEOUT) 75 | @lock.synchronize do 76 | # Record modified_index in watcher itself; Because the latest index may be hidden in normal response 77 | # e.g. unlisted keys, removed keys 78 | @indices[key] = response.node.modified_index 79 | end 80 | $stderr.puts "[watcher] dir #{key} has updated" if verbose 81 | ch << key 82 | return true 83 | rescue Etcd::EventIndexCleared => e 84 | $stderr.puts "[watcher][warn] #{e.inspect} on key #{key.inspect}, trying to get X-Etcd-Index" if verbose 85 | @lock.synchronize do 86 | @indices[key] = etcd.get(key).etcd_index 87 | end 88 | $stderr.puts "[watcher][warn] Updated #{key.inspect} index to #{@indices[key]}" if verbose 89 | return nil 90 | rescue Net::ReadTimeout 91 | $stderr.puts "[watcher] #{e.inspect} on key #{key.inspect}" if verbose 92 | return nil 93 | end 94 | end 95 | end 96 | -------------------------------------------------------------------------------- /scripts/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "bundler/setup" 4 | require "etcenv" 5 | 6 | # You can add fixtures and/or initialization code here to make experimenting 7 | # with your gem easier. You can also use a different console, if you like. 8 | 9 | # (If you use this, don't forget to add pry to your Gemfile!) 10 | # require "pry" 11 | # Pry.start 12 | 13 | require "irb" 14 | IRB.start 15 | -------------------------------------------------------------------------------- /scripts/setup: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | 5 | bundle install 6 | 7 | # Do any other automated setup that you need to do here 8 | -------------------------------------------------------------------------------- /spec/dockerenv_file_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'etcenv/dockerenv_file' 3 | require 'etcenv/variable_expander' 4 | 5 | describe Etcenv::DockerenvFile do 6 | let(:env) { {'KEY' => 'VALUE'} } 7 | subject(:dotenv_file) { described_class.new(env) } 8 | 9 | describe "#to_s" do 10 | subject { dotenv_file.to_s.lines.sort_by { |_| _.split(?=,2)[0] }.join } 11 | 12 | context "for normal env" do 13 | let(:env) { {'KEY' => 'VALUE'} } 14 | 15 | it { is_expected.to eq "KEY=VALUE\n" } 16 | end 17 | 18 | context "for multiple env" do 19 | let(:env) { {'KEY' => 'VALUE', 'KEY2' => 'VALUE2'} } 20 | 21 | it { is_expected.to eq "KEY=VALUE\nKEY2=VALUE2\n" } 22 | end 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /spec/dotenv_file_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'etcenv/dotenv_file' 3 | 4 | describe Etcenv::DotenvFile do 5 | let(:env) { {'KEY' => 'VALUE'} } 6 | subject(:dotenv_file) { described_class.new(env) } 7 | 8 | describe "#to_s" do 9 | subject { dotenv_file.to_s.lines.sort_by { |_| _.split(?=,2)[0] }.join } 10 | 11 | context "for normal env" do 12 | let(:env) { {'KEY' => 'VALUE'} } 13 | 14 | it { is_expected.to eq "KEY=VALUE\n" } 15 | end 16 | 17 | context "for multiple env" do 18 | let(:env) { {'KEY' => 'VALUE', 'KEY2' => 'VALUE2'} } 19 | 20 | it { is_expected.to eq "KEY=VALUE\nKEY2=VALUE2\n" } 21 | end 22 | 23 | context "for env contains multiple lines" do 24 | let(:env) { {'KEY' => "a\nb"} } 25 | 26 | it { is_expected.to eq 'KEY="a\nb"' + "\n" } 27 | end 28 | 29 | context "for env contains multiple lines (CR+LF)" do 30 | let(:env) { {'KEY' => "a\r\nb"} } 31 | 32 | it { is_expected.to eq 'KEY="a\nb"' + "\n" } 33 | end 34 | 35 | context "for env contains #" do 36 | let(:env) { {'KEY' => "a#b"} } 37 | 38 | it { is_expected.to eq 'KEY="a#b"' + "\n" } 39 | end 40 | 41 | context 'for env contains "' do 42 | let(:env) { {'KEY' => 'a"b'} } 43 | 44 | it { is_expected.to eq 'KEY="a\"b"' + "\n" } 45 | end 46 | 47 | context 'for env contains $(..)' do 48 | let(:env) { {'KEY' => 'a$(..)'} } 49 | 50 | it { is_expected.to eq 'KEY="a$(..)"' + "\n" } 51 | end 52 | 53 | context 'for env contains \${..}' do 54 | let(:env) { {'KEY' => 'a\${XX}'} } 55 | 56 | it { is_expected.to eq 'KEY="a\\\\${XX}"' + "\n" } 57 | end 58 | 59 | context 'for env contains \$..' do 60 | let(:env) { {'KEY' => 'a\$FOO'} } 61 | 62 | it { is_expected.to eq 'KEY="a\\\\$FOO"' + "\n" } 63 | end 64 | 65 | context 'for env contains \$(..)' do 66 | let(:env) { {'KEY' => 'a\$(..)'} } 67 | 68 | it { is_expected.to eq 'KEY="a\$(..)"' + "\n" } 69 | end 70 | end 71 | end 72 | -------------------------------------------------------------------------------- /spec/environment_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'etcd' 3 | require 'etcenv/environment' 4 | 5 | describe Etcenv::Environment do 6 | def generate_etcd_tree(tree, path = []) 7 | Hash[tree.map do |k,v| 8 | new_path = path + [k] 9 | case v 10 | when Hash 11 | [k.to_s, generate_etcd_tree(v, new_path)] 12 | else 13 | [k.to_s, double("etcd-node #{new_path.join(?/)}", 14 | key: "/#{new_path.join(?/)}", 15 | value: v, 16 | dir: false, 17 | modified_index: 0, 18 | etcvault_error: nil, 19 | )] 20 | end 21 | end] 22 | end 23 | 24 | def generate_etcd_dir(tree, path) 25 | children = tree.map do |k,v| 26 | case v 27 | when Hash 28 | generate_etcd_dir(v, path + [k]) 29 | else 30 | v 31 | end 32 | end 33 | 34 | double("etcd dir #{path.join(?/)}", 35 | key: "/#{path.join(?/)}", 36 | children: children, 37 | value: nil, 38 | dir: true, 39 | etcvault_error: nil, 40 | ) 41 | end 42 | 43 | def generate_etcd_double(tree) 44 | double('etcd').tap do |etcd| 45 | allow(etcd).to receive(:get) do |key| 46 | path = key.split(?/).tap(&:shift) 47 | raw_value = path.inject(tree) do |head, path_part| 48 | head && head[path_part] 49 | end 50 | 51 | if raw_value 52 | node = case raw_value 53 | when Hash 54 | generate_etcd_dir(raw_value,path) 55 | else 56 | raw_value 57 | end 58 | 59 | double('etcd response', 60 | node: node, 61 | value: node.value, 62 | etcd_index: 1, 63 | ) 64 | else 65 | raise Etcd::KeyNotFound 66 | end 67 | end 68 | end 69 | end 70 | 71 | def mock_etcd(tree) 72 | generate_etcd_double(generate_etcd_tree(tree)) 73 | end 74 | 75 | 76 | let(:tree) do 77 | { 78 | } 79 | end 80 | 81 | let(:etcd) { mock_etcd(tree) } 82 | let(:root_key) { '/a' } 83 | subject(:environment) { described_class.new(etcd, root_key) } 84 | 85 | subject { environment.env } 86 | 87 | context "with simple namespace" do 88 | let(:tree) do 89 | { 90 | a: { 91 | FOO: "foo", 92 | BAR: "bar", 93 | } 94 | } 95 | end 96 | 97 | it { is_expected.to eq("FOO" => "foo", "BAR" => "bar",) } 98 | end 99 | 100 | context "with one including" do 101 | let(:tree) do 102 | { 103 | a: { 104 | ".include" => "b", 105 | FOO: "foo", 106 | BAR: "bar", 107 | }, 108 | b: { 109 | BAZ: "baz", 110 | } 111 | } 112 | end 113 | 114 | it { is_expected.to eq("FOO" => "foo", "BAR" => "bar", "BAZ" => "baz",) } 115 | end 116 | 117 | context "with including (conflict)" do 118 | let(:tree) do 119 | { 120 | a: { 121 | ".include" => "b", 122 | FOO: "foo", 123 | BAR: "bar", 124 | }, 125 | b: { 126 | BAR: "baz", 127 | } 128 | } 129 | end 130 | 131 | it { is_expected.to eq("FOO" => "foo", "BAR" => "bar",) } 132 | end 133 | 134 | context "with complex including" do 135 | let(:tree) do 136 | { 137 | a: { 138 | ".include" => "b0", 139 | A0: "a0", 140 | }, 141 | b0: { 142 | ".include" => "c0,c1,c2,d1", 143 | B0: "b0", 144 | }, 145 | c0: { 146 | ".include" => "d0", 147 | C0: "c0", 148 | }, 149 | c1: { 150 | ".include" => "d0", 151 | C1: "c1", 152 | }, 153 | c2: { 154 | ".include" => "d0,d1", 155 | C2: "c2", 156 | }, 157 | d0: { 158 | D0: "d0", 159 | }, 160 | d1: { 161 | D1: "d1", 162 | }, 163 | } 164 | end 165 | 166 | it { is_expected.to eq("A0" => "a0", "B0" => "b0", "C0" => "c0", "C1" => "c1", "C2" => "c2", "D0" => "d0", "D1" => "d1",) } 167 | end 168 | 169 | context "with nested including (conflict)" do 170 | let(:tree) do 171 | { 172 | a: { 173 | ".include" => "b", 174 | A: "a", 175 | }, 176 | b: { 177 | ".include" => "c", 178 | A: "b", 179 | B: "b", 180 | }, 181 | c: { 182 | A: "c", 183 | B: "c", 184 | C: "c", 185 | }, 186 | } 187 | end 188 | 189 | it { 190 | is_expected.to eq( 191 | "A" => "a", 192 | "B" => "b", 193 | "C" => "c", 194 | ) 195 | } 196 | end 197 | 198 | context "with complex nested including (conflict)" do 199 | let(:tree) do 200 | { 201 | a: { 202 | ".include" => "b", 203 | A: "a", 204 | }, 205 | b: { 206 | ".include" => "c,d", 207 | A: "b", 208 | B: "b", 209 | D: "b", 210 | }, 211 | c: { 212 | ".include" => "d", 213 | A: "c", 214 | B: "c", 215 | C: "c", 216 | D: "c", 217 | }, 218 | d: { 219 | D: "d", 220 | } 221 | } 222 | end 223 | 224 | it { 225 | is_expected.to eq( 226 | "A" => "a", 227 | "B" => "b", 228 | "C" => "c", 229 | "D" => "b", 230 | ) 231 | } 232 | end 233 | 234 | context "with too deep including" do 235 | let(:tree) do 236 | { 237 | a: { ".include" => "b", A: "0", }, 238 | b: { ".include" => "c", A: "1", }, 239 | c: { ".include" => "d", A: "2", }, 240 | d: { ".include" => "e", A: "3", }, 241 | e: { ".include" => "f", A: "4", }, 242 | f: { ".include" => "g", A: "5", }, 243 | g: { ".include" => "h", A: "6", }, 244 | h: { ".include" => "i", A: "7", }, 245 | i: { ".include" => "j", A: "8", }, 246 | j: { ".include" => "k", A: "9", }, 247 | k: { A: "10", }, 248 | } 249 | end 250 | 251 | specify { 252 | expect { subject }.to raise_error(Etcenv::Environment::DepthLimitError) 253 | } 254 | end 255 | 256 | context "with looped including" do 257 | let(:tree) do 258 | { 259 | a: { 260 | ".include" => "b", 261 | A: "a", 262 | }, 263 | b: { 264 | ".include" => "a", 265 | B: "b", 266 | }, 267 | } 268 | end 269 | 270 | specify { 271 | expect { subject }.to raise_error(Etcenv::Environment::LoopError) 272 | } 273 | end 274 | 275 | context "with nested looped including" do 276 | let(:tree) do 277 | { 278 | a: { 279 | ".include" => "b", 280 | A: "a", 281 | }, 282 | b: { 283 | ".include" => "c", 284 | }, 285 | c: { 286 | ".include" => "a", 287 | C: "c", 288 | }, 289 | } 290 | end 291 | 292 | specify { 293 | expect { subject }.to raise_error(Etcenv::Environment::LoopError) 294 | } 295 | end 296 | 297 | context "with relative include path" do 298 | let(:root_key) { '/prefix/a' } 299 | let(:tree) do 300 | { 301 | prefix: { 302 | a: { 303 | ".include" => "b", 304 | A: "a", 305 | }, 306 | b: { 307 | B: "b", 308 | }, 309 | }, 310 | } 311 | end 312 | 313 | it { is_expected.to eq("A" => "a", "B" => "b",) } 314 | end 315 | 316 | context "with absolute include path" do 317 | let(:root_key) { '/prefix/a' } 318 | let(:tree) do 319 | { 320 | prefix: { 321 | a: { 322 | ".include" => "/another_dir/b", 323 | A: "a", 324 | }, 325 | }, 326 | another_dir: { 327 | b: { 328 | B: "b", 329 | }, 330 | }, 331 | } 332 | end 333 | 334 | it { is_expected.to eq("A" => "a", "B" => "b",) } 335 | end 336 | 337 | context "with including key" do 338 | let(:tree) do 339 | { 340 | a: { 341 | ".include" => "secrets/B", 342 | A: "a", 343 | }, 344 | secrets: { 345 | B: "b", 346 | C: "c", 347 | }, 348 | } 349 | end 350 | 351 | it { is_expected.to eq("A" => "a", "B" => "b",) } 352 | end 353 | end 354 | -------------------------------------------------------------------------------- /spec/etcenv_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Etcenv do 4 | end 5 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) 2 | require 'etcenv' 3 | -------------------------------------------------------------------------------- /spec/variable_expander_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'etcenv/variable_expander' 3 | 4 | describe Etcenv::VariableExpander do 5 | let(:string) { '' } 6 | let(:variables) { { 'VAR' => 'var', 'STR' => string } } 7 | subject(:result) { described_class.expand(variables) } 8 | 9 | describe "single expansion" do 10 | subject { result['STR'] } 11 | 12 | context "no variables" do 13 | it { is_expected.to eq '' } 14 | end 15 | 16 | context "with variable" do 17 | let(:string) { "$VAR" } 18 | 19 | it { is_expected.to eq 'var' } 20 | end 21 | 22 | context "with {variable}" do 23 | let(:string) { "${VAR}" } 24 | 25 | it { is_expected.to eq 'var' } 26 | end 27 | 28 | context "with unexist variable" do 29 | let(:string) { "$NOVAR" } 30 | 31 | it { is_expected.to eq '' } 32 | end 33 | 34 | context "with string including variable" do 35 | let(:string) { "foo $VAR baz" } 36 | 37 | it { is_expected.to eq "foo var baz" } 38 | end 39 | 40 | context "with string including variable" do 41 | let(:string) { "foo ${VAR} baz" } 42 | 43 | it { is_expected.to eq "foo var baz" } 44 | end 45 | 46 | context "with multiple variable expansions" do 47 | let(:string) { "${VAR}${VAR}" } 48 | 49 | it { is_expected.to eq "varvar" } 50 | end 51 | 52 | context "with multiple variables expansions" do 53 | let(:variables) { { 'VAR' => 'var', 'VAR2' => 'var2', 'STR' => string } } 54 | let(:string) { "${VAR}${VAR2}" } 55 | 56 | it { is_expected.to eq "varvar2" } 57 | end 58 | 59 | context "with multiple variables expansions" do 60 | let(:variables) { { 'VAR' => 'var', 'VAR2' => 'var2', 'STR' => string } } 61 | let(:string) { "${VAR}${VAR2}" } 62 | 63 | it { is_expected.to eq "varvar2" } 64 | end 65 | 66 | context "with escaped variable" do 67 | let(:string) { "\\$VAR" } 68 | 69 | it { is_expected.to eq "$VAR" } 70 | end 71 | 72 | context "with escaped {variable}" do 73 | let(:string) { "\\${VAR}" } 74 | 75 | it { is_expected.to eq "${VAR}" } 76 | end 77 | 78 | context "with escaped variable and variable" do 79 | let(:string) { "$VAR \\$VAR $VAR" } 80 | 81 | it { is_expected.to eq "var $VAR var" } 82 | end 83 | end 84 | 85 | describe "multiple expansions" do 86 | let(:variables) do 87 | { 88 | 'VAR2' => '${VAR}', 89 | 'VAR' => 'var' 90 | } 91 | end 92 | 93 | it "resolves correctly" do 94 | expect(subject['VAR']).to eq 'var' 95 | expect(subject['VAR2']).to eq 'var' 96 | end 97 | end 98 | 99 | describe "nested multiple expansions" do 100 | # VALUE 101 | # VAR 102 | # FOO, BAZ 103 | # BAR 104 | # VAR2 105 | let(:variables) do 106 | { 107 | 'VAR2' => '${VAR}', 108 | 'BAR' => '${FOO} bar ${BAZ}', 109 | 'VAR' => '${VALUE}', 110 | 'FOO' => 'foo ${VAR}', 111 | 'BAZ' => 'baz ${VAR}', 112 | 'VALUE' => 'var' 113 | } 114 | end 115 | 116 | it "resolves correctly" do 117 | expect(result).to eq( 118 | 'VALUE' => 'var', 119 | 'VAR' => 'var', 120 | 'FOO' => 'foo var', 121 | 'BAZ' => 'baz var', 122 | 'BAR' => 'foo var bar baz var', 123 | 'VAR2' => 'var', 124 | ) 125 | end 126 | end 127 | 128 | context "looped multiple expansions with root" do 129 | # VAR 130 | # VAR2 131 | let(:variables) do 132 | { 133 | 'ROOT' => '${VAR}', 134 | 'VAR2' => '${VAR}', 135 | 'VAR' => '${VAR2}', 136 | } 137 | end 138 | 139 | it "raises error" do 140 | expect { subject }.to raise_error(Etcenv::VariableExpander::LoopError) 141 | end 142 | end 143 | 144 | 145 | context "looped multiple expansions" do 146 | # VAR 147 | # VAR2 148 | let(:variables) do 149 | { 150 | 'VAR2' => '${VAR}', 151 | 'VAR' => '${VAR2}', 152 | } 153 | end 154 | 155 | it "raises error" do 156 | expect { subject }.to raise_error(Etcenv::VariableExpander::LoopError) 157 | end 158 | end 159 | 160 | context "nested looped multiple expansions" do 161 | # VAR 162 | # VAR2 163 | # VAR3 164 | let(:variables) do 165 | { 166 | 'VAR' => '${VAR2}', 167 | 'VAR2' => '${VAR3}', 168 | 'VAR3' => '${VAR}', 169 | } 170 | end 171 | 172 | it "raises error" do 173 | expect { subject }.to raise_error(Etcenv::VariableExpander::LoopError) 174 | end 175 | end 176 | 177 | context "exceed max_depth" do 178 | # VAR 179 | # VAR2 180 | # VAR3 181 | let(:variables) do 182 | Hash[(1..50).map do |i| 183 | ["VAR#{i}", (1...i).map { |_| "${VAR#{_}}" }.join] 184 | end] 185 | end 186 | 187 | it "raises error" do 188 | expect { subject }.to raise_error(Etcenv::VariableExpander::DepthLimitError) 189 | end 190 | end 191 | 192 | context "empty" do 193 | let(:variables) do 194 | {} 195 | end 196 | 197 | it { is_expected.to eq({}) } 198 | end 199 | end 200 | --------------------------------------------------------------------------------