├── spec ├── sample.txt ├── included2.yml ├── included.yml ├── spec_helper.rb ├── sample.yml └── yaml_master_spec.rb ├── .rspec ├── lib ├── yaml_master │ ├── version.rb │ └── yaml_tree_builder.rb └── yaml_master.rb ├── Gemfile ├── .travis.yml ├── Rakefile ├── bin ├── setup └── console ├── .gitignore ├── Dockerfile ├── yaml_master.gemspec ├── exe └── yaml_master └── README.md /spec/sample.txt: -------------------------------------------------------------------------------- 1 | dummy 2 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --format documentation 2 | --color 3 | -------------------------------------------------------------------------------- /lib/yaml_master/version.rb: -------------------------------------------------------------------------------- 1 | class YamlMaster 2 | VERSION = "1.0.1" 3 | end 4 | -------------------------------------------------------------------------------- /spec/included2.yml: -------------------------------------------------------------------------------- 1 | - foo: bar 2 | - hoge: fuga 3 | - :abc: 4 | [1, 2, 3] 5 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in yaml_master.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /spec/included.yml: -------------------------------------------------------------------------------- 1 | xyz: hoge 2 | db: 3 | <<: *database_development 4 | abc: 5 | - 1 6 | - 2.3 7 | - {a: 1, b: 2} 8 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | sudo: false 3 | rvm: 4 | - 2.5 5 | - 2.6 6 | - 2.7 7 | before_install: gem update bundler 8 | cache: bundler 9 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | require "rspec/core/rake_task" 3 | 4 | RSpec::Core::RakeTask.new(:spec) 5 | 6 | task :default => :spec 7 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | 5 | bundle install 6 | 7 | # Do any other automated setup that you need to do here 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /Gemfile.lock 4 | /_yardoc/ 5 | /coverage/ 6 | /doc/ 7 | /pkg/ 8 | /spec/reports/ 9 | /spec/examples.txt 10 | /tmp/ 11 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ruby:2.4-alpine 2 | 3 | ARG version 4 | 5 | RUN if [[ "$version" = "" ]]; then gem install yaml_master --no-document; else gem install yaml_master --no-document --version ${version}; fi 6 | 7 | ENTRYPOINT ["yaml_master"] 8 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "bundler/setup" 4 | require "yaml_master" 5 | 6 | # You can add fixtures and/or initialization code here to make experimenting 7 | # with your gem easier. You can also use a different console, if you like. 8 | 9 | # (If you use this, don't forget to add pry to your Gemfile!) 10 | # require "pry" 11 | # Pry.start 12 | 13 | require "irb" 14 | IRB.start 15 | -------------------------------------------------------------------------------- /yaml_master.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'yaml_master/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = "yaml_master" 8 | spec.version = YamlMaster::VERSION 9 | spec.authors = ["joker1007"] 10 | spec.email = ["kakyoin.hierophant@gmail.com"] 11 | 12 | spec.summary = %q{Helper of YAML file generation from single master YAML file} 13 | spec.description = %q{Helper of YAML file generation from single master YAML file} 14 | spec.homepage = "https://github.com/joker1007/yaml_master" 15 | 16 | spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } 17 | spec.bindir = "exe" 18 | spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } 19 | spec.require_paths = ["lib"] 20 | 21 | spec.add_development_dependency "bundler", ">= 1.15", "< 3" 22 | spec.add_development_dependency "rake", ">= 10.0" 23 | spec.add_development_dependency "rspec" 24 | spec.add_development_dependency "fakefs" 25 | end 26 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) 2 | require 'yaml_master' 3 | require 'pp' 4 | require 'fakefs/safe' 5 | 6 | ENV["CONFIG_DIR"] = "." 7 | ENV["RAILS_ENV"] = "production" 8 | 9 | RSpec.configure do |config| 10 | config.expect_with :rspec do |expectations| 11 | # This option will default to `true` in RSpec 4. It makes the `description` 12 | # and `failure_message` of custom matchers include text for helper methods 13 | # defined using `chain`, e.g.: 14 | # be_bigger_than(2).and_smaller_than(4).description 15 | # # => "be bigger than 2 and smaller than 4" 16 | # ...rather than: 17 | # # => "be bigger than 2" 18 | expectations.include_chain_clauses_in_custom_matcher_descriptions = true 19 | end 20 | 21 | config.mock_with :rspec do |mocks| 22 | # Prevents you from mocking or stubbing a method that does not exist on 23 | # a real object. This is generally recommended, and will default to 24 | # `true` in RSpec 4. 25 | mocks.verify_partial_doubles = true 26 | end 27 | 28 | config.example_status_persistence_file_path = "spec/examples.txt" 29 | 30 | config.disable_monkey_patching! 31 | 32 | config.warnings = true 33 | 34 | if config.files_to_run.one? 35 | config.default_formatter = 'doc' 36 | end 37 | 38 | config.profile_examples = 10 39 | 40 | config.order = :random 41 | end 42 | -------------------------------------------------------------------------------- /exe/yaml_master: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "optparse" 4 | require "yaml_master" 5 | 6 | options = {properties: []} 7 | 8 | opt_parser = OptionParser.new do |opts| 9 | opts.on("-m", "--master=MASTER_FILE", "master yaml file") do |master| 10 | options[:master] = master 11 | end 12 | 13 | opts.on("-k", "--key=KEY", "single generate target key (in data: block)") do |key| 14 | options[:key] = key 15 | end 16 | 17 | opts.on("-o", "--output=OUTPUT", "output filename for single generate target") do |output| 18 | options[:output] = output 19 | end 20 | 21 | opts.on("-a", "--all", "target all key (defined in yaml_master: block)") do |all| 22 | options[:all] = true 23 | end 24 | 25 | opts.on("-d", "--dump", "dump evaluated master yaml") do || 26 | options[:dump] = true 27 | end 28 | 29 | opts.on("-p", "--properties=PROPERTIES", %q{set property (--properties="NAME=VALUE,NAME=VALUE" or -p "NAME=VALUE" -p "NAME=VALUE")}) do |properties| 30 | options[:properties].concat(properties.split(/\s*,\s*/)) 31 | end 32 | 33 | opts.on("-v", "--verbose", "verbose mode") do |verbose| 34 | options[:verbose] = true 35 | end 36 | 37 | opts.on("--version", "Print version") do 38 | puts YamlMaster::VERSION 39 | exit 40 | end 41 | 42 | opts.on("-h", "--help", "Prints this help") do 43 | puts opts 44 | exit 45 | end 46 | end 47 | 48 | opt_parser.parse!(ARGV) 49 | 50 | unless options[:master] 51 | puts "--master options is necessary" 52 | puts opt_parser 53 | exit 1 54 | end 55 | 56 | if options[:all].nil? && options[:key].nil? && options[:dump].nil? 57 | puts "--all or --key or --dump is necessary" 58 | puts opt_parser 59 | exit 1 60 | end 61 | 62 | yaml_master = YamlMaster.new(options[:master], options[:properties]) 63 | 64 | if options[:all] 65 | yaml_master.generate_all(verbose: options[:verbose]) 66 | elsif options[:dump] 67 | result = yaml_master.dump(options[:output], {verbose: options[:verbose]}) 68 | puts result unless options[:output] 69 | else 70 | result = yaml_master.generate(options[:key], options[:output], {verbose: options[:verbose]}) 71 | puts result unless options[:output] 72 | end 73 | -------------------------------------------------------------------------------- /spec/sample.yml: -------------------------------------------------------------------------------- 1 | yaml_master: 2 | database_yml: <%= ENV["CONFIG_DIR"] %>/database.yml 3 | embedded_methods: <%= ENV["CONFIG_DIR"] %>/embedded_methods.yml 4 | 5 | database_config: &database_config 6 | development: &database_development 7 | adapter: mysql2 8 | encoding: utf8 9 | database: development 10 | pool: 5 11 | host: &database_development_host 12 | username: &database_development_username root 13 | password: &database_development_password 14 | socket: /tmp/mysql.sock 15 | 16 | test: &database_test 17 | adapter: mysql2 18 | encoding: utf8 19 | database: test 20 | host: &database_test_host 21 | username: &database_test_username root 22 | password: &database_test_password 23 | 24 | production: &database_production 25 | adapter: mysql2 26 | encoding: utf8 27 | database: production 28 | pool: 5 29 | host: &database_production_host "192.168.1.100" 30 | username: &database_production_username root 31 | password: &database_production_password 32 | socket: /tmp/mysql.sock 33 | 34 | data: 35 | database_yml: 36 | <<: *database_config 37 | 38 | embulk_yml: 39 | in: 40 | type: file 41 | path_prefix: example.csv 42 | parser: 43 | type: csv 44 | skip_header_lines: 1 45 | columns: 46 | - {name: key_name, type: string} 47 | - {name: day, type: timestamp, format: '%Y-%m-%d'} 48 | - {name: new_clients, type: long} 49 | 50 | out: 51 | type: mysql 52 | host: *database_<%= ENV["RAILS_ENV"] %>_host 53 | user: *database_<%= ENV["RAILS_ENV"] %>_username 54 | password: *database_<%= ENV["RAILS_ENV"] %>_password 55 | database: my_database 56 | table: my_table 57 | mode: insert 58 | 59 | embedded_methods: 60 | included: !include included.yml 61 | included2: !include included2.yml 62 | master_path: !master_path 63 | master_path2: <%= master_path %> 64 | user_home: !user_home 65 | user_home2: <%= user_home %> 66 | env: !env HOME 67 | fullpath: !fullpath sample.txt 68 | properties: !properties foo 69 | properties2: !properties foo2 70 | fullpath: !fullpath sample.txt 71 | read_file_if_exist: !read_file_if_exist sample.txt 72 | read_file_if_exist_nothing: !read_file_if_exist nothing.txt 73 | read_file_if_exist2: "<%= read_file_if_exist(master_path.dirname + "sample.txt") %>" 74 | 75 | -------------------------------------------------------------------------------- /lib/yaml_master/yaml_tree_builder.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: false 2 | require 'yaml' 3 | require 'pathname' 4 | 5 | class YamlMaster::YAMLTreeBuilder < YAML::TreeBuilder 6 | def initialize(master_path, properties, parser) 7 | super() 8 | @master_path = master_path 9 | @properties = properties 10 | @parser = parser 11 | end 12 | 13 | def scalar(value, anchor, tag, plain, quoted, style) 14 | case tag 15 | when "!include" 16 | ensure_tag_argument(tag, value) 17 | 18 | path = Pathname(value) 19 | path = path.absolute? ? path : @master_path.dirname.join(path) 20 | tree = YAML.parse(File.read(path)) 21 | @last.children << tree.children[0] 22 | when "!master_path" 23 | s = build_scalar_node(@master_path.to_s) 24 | @last.children << s 25 | s 26 | when "!fullpath" 27 | path = Pathname(value) 28 | path = path.absolute? ? path : @master_path.dirname.join(path) 29 | s = build_scalar_node(path.to_s) 30 | @last.children << s 31 | s 32 | when "!user_home" 33 | s = build_scalar_node(ENV["HOME"]) 34 | @last.children << s 35 | s 36 | when "!properties" 37 | ensure_tag_argument(tag, value) 38 | 39 | s = build_scalar_node(@properties[value]) 40 | @last.children << s 41 | s 42 | when "!env" 43 | ensure_tag_argument(tag, value) 44 | 45 | s = build_scalar_node(ENV[value]) 46 | @last.children << s 47 | s 48 | when "!read_file_if_exist" 49 | ensure_tag_argument(tag, value) 50 | 51 | path = Pathname(value) 52 | path = path.absolute? ? path : @master_path.dirname.join(path) 53 | content = path.file? && path.readable? ? File.read(path) : nil 54 | s = build_scalar_node(content, false, true, 4) 55 | @last.children << s 56 | s 57 | else 58 | super 59 | end 60 | end 61 | 62 | private 63 | 64 | def ensure_tag_argument(tag, value) 65 | if value.empty? 66 | mark = @parser.mark 67 | $stderr.puts "tag format error" 68 | $stderr.puts "#{tag} requires 1 argument at #{mark.line}:#{mark.column}" 69 | exit 1 70 | end 71 | end 72 | 73 | def build_scalar_node(value, plain = true, quoted = true, style = 1) 74 | if value 75 | YAML::Nodes::Scalar.new(value, nil, nil, plain, quoted, style) 76 | else 77 | YAML::Nodes::Scalar.new("null", nil, nil, true, false, 1) 78 | end 79 | end 80 | end 81 | -------------------------------------------------------------------------------- /spec/yaml_master_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | RSpec.describe YamlMaster do 4 | let!(:yaml_master) { YamlMaster.new(File.expand_path("../sample.yml", __FILE__), ["foo=bar"]) } 5 | 6 | after do 7 | FakeFS.deactivate! 8 | end 9 | 10 | it "#generate_all" do 11 | FakeFS.activate! 12 | 13 | aggregate_failures do 14 | expect(File.exist?("./database.yml")).to be_falsey 15 | yaml_master.generate_all 16 | expect(File.exist?("./database.yml")).to be_truthy 17 | 18 | yaml1 = YAML.load_file("./database.yml") 19 | expect(yaml1["development"]["adapter"]).to eq "mysql2" 20 | expect(yaml1["test"]["adapter"]).to eq "mysql2" 21 | expect(yaml1["production"]["adapter"]).to eq "mysql2" 22 | 23 | yaml2 = YAML.load_file("./embedded_methods.yml") 24 | expect(yaml2["master_path"]).to eq File.expand_path("../sample.yml", __FILE__) 25 | expect(yaml2["master_path2"]).to eq File.expand_path("../sample.yml", __FILE__) 26 | expect(yaml2["user_home"]).to eq ENV["HOME"] 27 | expect(yaml2["user_home2"]).to eq ENV["HOME"] 28 | expect(yaml2["env"]).to eq ENV["HOME"] 29 | expect(yaml2["properties"]).to eq "bar" 30 | expect(yaml2["properties2"]).to be_nil 31 | expect(yaml2["fullpath"]).to eq File.expand_path("../sample.txt", __FILE__) 32 | expect(yaml2["read_file_if_exist"]).to match(/dummy/) 33 | expect(yaml2["read_file_if_exist_nothing"]).to be_nil 34 | expect(yaml2["read_file_if_exist2"]).to match(/dummy/) 35 | expect(yaml2["included"]["xyz"]).to eq "hoge" 36 | expect(yaml2["included"]["abc"][0]).to eq 1 37 | expect(yaml2["included"]["db"]["database"]).to eq "development" 38 | end 39 | 40 | FakeFS.deactivate! 41 | end 42 | 43 | it "#generate" do 44 | FakeFS.activate! 45 | 46 | aggregate_failures do 47 | expect(File.exist?("./embulk.yml")).to be_falsey 48 | yaml_master.generate("embulk_yml", "./embulk.yml") 49 | expect(File.exist?("./embulk.yml")).to be_truthy 50 | 51 | config = YAML.load_file("./embulk.yml") 52 | expect(config["out"]["host"]).to eq "192.168.1.100" 53 | expect(config["out"]["user"]).to eq "root" 54 | expect(config["out"]["password"]).to be_nil 55 | end 56 | 57 | FakeFS.deactivate! 58 | end 59 | 60 | it "#generate (fetch nested data)" do 61 | config = YAML.load(yaml_master.generate("embulk_yml.in.parser.columns.[1]")) 62 | aggregate_failures do 63 | expect(config["name"]).to eq "day" 64 | expect(config["type"]).to eq "timestamp" 65 | expect(config["format"]).to eq "%Y-%m-%d" 66 | end 67 | end 68 | 69 | it "#generate (nothing key)" do 70 | aggregate_failures do 71 | expect { yaml_master.generate("no_data") }.to \ 72 | raise_error(YamlMaster::KeyFetchError) 73 | expect { yaml_master.generate("embulk_yml.in.parser.columns.[3]") }.to \ 74 | raise_error(YamlMaster::KeyFetchError) 75 | end 76 | end 77 | 78 | context "Hash style properties" do 79 | let!(:yaml_master) { YamlMaster.new(File.expand_path("../sample.yml", __FILE__), {"foo" => "bar", "foo2" => [{"a" => 1}]}) } 80 | 81 | it "#generate_all" do 82 | FakeFS.activate! 83 | 84 | yaml_master.generate_all 85 | 86 | yaml2 = YAML.load_file("./embedded_methods.yml") 87 | expect(yaml2["properties"]).to eq "bar" 88 | expect(yaml2["properties2"]).to eq [{"a" => 1}] 89 | 90 | FakeFS.deactivate! 91 | end 92 | end 93 | end 94 | -------------------------------------------------------------------------------- /lib/yaml_master.rb: -------------------------------------------------------------------------------- 1 | require "yaml_master/version" 2 | 3 | require "yaml" 4 | require "erb" 5 | require "pathname" 6 | 7 | require "yaml_master/yaml_tree_builder" 8 | 9 | class YamlMaster 10 | class KeyFetchError < StandardError 11 | def initialize(data, key) 12 | super("cannot fetch key \"#{key}\" from\n#{data.pretty_inspect}") 13 | end 14 | end 15 | attr_reader :master, :master_path, :properties 16 | 17 | def initialize(io_or_filename, property_strings = []) 18 | case io_or_filename 19 | when String 20 | @master_path = Pathname(io_or_filename).expand_path 21 | when File 22 | @master_path = Pathname(io_or_filename.absolute_path) 23 | end 24 | 25 | @properties = PropertyParser.parse_properties(property_strings) 26 | yaml = Context.new(master_path, @properties).render_master 27 | 28 | parser = YAML::Parser.new 29 | parser.handler = YamlMaster::YAMLTreeBuilder.new(@master_path, @properties, parser) 30 | @tree = parser.parse(yaml).handler.root 31 | @master = @tree.to_ruby[0] 32 | end 33 | 34 | def dump(output = nil, options = {}) 35 | yaml = @tree.to_yaml 36 | write_to_output(yaml, output, options[:verbose]) 37 | end 38 | 39 | def generate(key, output = nil, options = {}) 40 | raise "data key is necessary on toplevel" unless @master["data"] 41 | yaml = YAML.dump(fetch_data_from_master(key)) 42 | write_to_output(yaml, output, options[:verbose]) 43 | end 44 | 45 | def generate_all(options = {}) 46 | raise "yaml_master key is necessary on toplevel" unless @master["yaml_master"] 47 | raise "data key is necessary on toplevel" unless @master["data"] 48 | @master["yaml_master"].each do |key, output| 49 | generate(key, output, options) 50 | end 51 | end 52 | 53 | private 54 | 55 | def fetch_data_from_master(key) 56 | keys = split_key(key) 57 | keys.inject(@master["data"]) do |data, k| 58 | data.fetch(k) 59 | end 60 | rescue 61 | raise KeyFetchError.new(@master["data"], key) 62 | end 63 | 64 | def split_key(key) 65 | keys = key.split(".") 66 | array_pattern = /\[(\d+)\]/ 67 | keys.map do |k| 68 | if k.match(array_pattern) 69 | Regexp.last_match[1].to_i 70 | else 71 | k 72 | end 73 | end 74 | end 75 | 76 | def write_to_output(yaml, output, verbose) 77 | if output && verbose 78 | puts <<~VERBOSE 79 | gen: #{output} 80 | #{yaml} 81 | VERBOSE 82 | end 83 | 84 | return yaml unless output 85 | 86 | File.open(output, 'w') do |f| 87 | f.write(yaml) 88 | end 89 | end 90 | 91 | module PropertyParser 92 | class ParseError < StandardError; end 93 | 94 | def self.parse_properties(property_strings_or_hash) 95 | if property_strings_or_hash.is_a?(Hash) 96 | property_strings_or_hash 97 | else 98 | property_strings_or_hash.each_with_object({}) do |str, hash| 99 | key, value = str.split("=") 100 | raise ParseError.new("#{str} is invalid format") unless key && value 101 | hash[key] = value 102 | end 103 | end 104 | end 105 | end 106 | 107 | class Context 108 | attr_reader :master_path, :properties 109 | 110 | def initialize(master_path, properties) 111 | @master_path = master_path 112 | @properties = properties 113 | end 114 | 115 | def user_home 116 | Pathname(ENV["HOME"]) 117 | end 118 | 119 | def read_file_if_exist(path) 120 | return nil unless File.exist?(path) 121 | File.read(path) 122 | end 123 | 124 | def render_master 125 | ERB.new(File.read(@master_path)).result(binding) 126 | end 127 | end 128 | end 129 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # YamlMaster 2 | [![Gem Version](https://badge.fury.io/rb/yaml_master.svg)](https://badge.fury.io/rb/yaml_master) 3 | [![Build Status](https://travis-ci.org/joker1007/yaml_master.svg)](https://travis-ci.org/joker1007/yaml_master) 4 | 5 | This gem is helper of yaml file generation from single master yaml file. 6 | And this gem has some useful helpers. For example `!include` 7 | 8 | ## Installation 9 | 10 | Add this line to your application's Gemfile: 11 | 12 | ```ruby 13 | gem 'yaml_master' 14 | ``` 15 | 16 | And then execute: 17 | 18 | $ bundle 19 | 20 | Or install it yourself as: 21 | 22 | $ gem install yaml_master 23 | 24 | ## Usage 25 | 26 | ### command help 27 | ``` 28 | Usage: yaml_master [options] 29 | -m, --master=MASTER_FILE master yaml file 30 | -k, --key=KEY single generate target key (in data: block) 31 | -o, --output=OUTPUT output filename for single generate target 32 | -a, --all target all key (defined in yaml_master: block) 33 | -d, --dump dump evaluated master yaml 34 | -p, --properties=PROPERTIES set property (--properties="NAME=VALUE,NAME=VALUE" or -p "NAME=VALUE" -p "NAME=VALUE") 35 | -v, --verbose verbose mode 36 | --version Print version 37 | -h, --help Prints this help 38 | ``` 39 | 40 | ### Support YAML tags 41 | 42 | | tag | description | 43 | | ------------------------ | ------------------------------------------------------------------------------------- | 44 | | !include {yaml_filename} | Replace value by other yaml file content. included file can use alias in master file. | 45 | | !master_path | Replace value by master file path | 46 | | !user_home | Replace value by `ENV["HOME"]`. | 47 | | !env {key} | Replace value by `ENV["{key}"]`. | 48 | | !properties {key} | Replace value by fetched value from given properties. | 49 | | !read_file_if_exist {filename} | Replace value by content of {filename}. if {filename} does not exist, Replace value by `null` | 50 | 51 | at first, Write master.yml 52 | 53 | ```yaml 54 | yaml_master: 55 | database_yml: <%= ENV["CONFIG_DIR"] %>/database.yml 56 | embulk_yml: <%= ENV["CONFIG_DIR"] %>/embulk.yml 57 | 58 | database_config: &database_config 59 | development: &database_development 60 | adapter: mysql2 61 | encoding: utf8 62 | database: development 63 | pool: 5 64 | host: &database_development_host 65 | username: &database_development_username root 66 | password: &database_development_password 67 | socket: /tmp/mysql.sock 68 | 69 | test: &database_test 70 | adapter: mysql2 71 | encoding: utf8 72 | database: test 73 | host: &database_test_host 74 | username: &database_test_username root 75 | password: &database_test_password 76 | 77 | production: &database_production 78 | adapter: mysql2 79 | encoding: utf8 80 | database: production 81 | pool: 5 82 | host: &database_production_host "192.168.1.100" 83 | username: &database_production_username root 84 | password: &database_production_password 85 | socket: /tmp/mysql.sock 86 | 87 | 88 | data: 89 | database_yml: 90 | <<: *database_config 91 | 92 | embulk_yml: 93 | in: 94 | type: file 95 | path_prefix: example.csv 96 | parser: 97 | type: csv 98 | skip_header_lines: 1 99 | columns: 100 | - {name: key_name, type: string} 101 | - {name: day, type: timestamp, format: '%Y-%m-%d'} 102 | - {name: new_clients, type: long} 103 | 104 | out: 105 | type: mysql 106 | host: *database_<%= ENV["RAILS_ENV"] %>_host 107 | user: *database_<%= ENV["RAILS_ENV"] %>_username 108 | password: *database_<%= ENV["RAILS_ENV"] %>_password 109 | database: my_database 110 | table: my_table 111 | mode: insert 112 | 113 | tag_sample: 114 | included: !include included.yml 115 | master_path: !master_path 116 | user_home: !user_home 117 | env: !env HOME 118 | properties: !properties foo 119 | read_file_if_exist: !read_file_if_exist sample.txt 120 | read_file_if_exist_nothing: !read_file_if_exist nothing.txt 121 | ``` 122 | 123 | ### single output 124 | ```sh 125 | $ RAILS_ENV=production CONFIG_DIR="." yaml_master -m master.yml -k embulk_yml -o embulk_config.yml 126 | ``` 127 | 128 | ```yaml 129 | # ./embulk_config.yml 130 | 131 | in: 132 | type: file 133 | path_prefix: example.csv 134 | parser: 135 | type: csv 136 | skip_header_lines: 1 137 | columns: 138 | - {name: key_name, type: string} 139 | - {name: day, type: timestamp, format: '%Y-%m-%d'} 140 | - {name: new_clients, type: long} 141 | 142 | out: 143 | type: mysql 144 | host: *database_<%= ENV["RAILS_ENV"] %>_host 145 | user: *database_<%= ENV["RAILS_ENV"] %>_username 146 | password: *database_<%= ENV["RAILS_ENV"] %>_password 147 | database: my_database 148 | table: my_table 149 | mode: insert 150 | ``` 151 | 152 | ### all output 153 | ```sh 154 | $ RAILS_ENV=production CONFIG_DIR="." yaml_master -m master.yml --all 155 | ``` 156 | 157 | outputs is following. 158 | 159 | ```yaml 160 | # ./database.yml 161 | 162 | --- 163 | development: 164 | adapter: mysql2 165 | encoding: utf8 166 | database: development 167 | pool: 5 168 | host: 169 | username: root 170 | password: 171 | socket: "/tmp/mysql.sock" 172 | test: 173 | adapter: mysql2 174 | encoding: utf8 175 | database: test 176 | host: 177 | username: root 178 | password: 179 | production: 180 | adapter: mysql2 181 | encoding: utf8 182 | database: production 183 | pool: 5 184 | host: 192.168.1.100 185 | username: root 186 | password: 187 | socket: "/tmp/mysql.sock" 188 | ``` 189 | 190 | ```yaml 191 | # ./embulk.yml 192 | 193 | --- 194 | in: 195 | type: file 196 | path_prefix: example.csv 197 | parser: 198 | type: csv 199 | skip_header_lines: 1 200 | columns: 201 | - name: key_name 202 | type: string 203 | - name: day 204 | type: timestamp 205 | format: "%Y-%m-%d" 206 | - name: new_clients 207 | type: long 208 | out: 209 | type: mysql 210 | host: 192.168.1.100 211 | user: root 212 | password: 213 | database: my_database 214 | table: my_table 215 | mode: insert 216 | ``` 217 | 218 | ```yaml 219 | # ./tag_sample.yml 220 | 221 | --- 222 | included: 223 | xyz: hoge 224 | db: 225 | adapter: mysql2 226 | encoding: utf8 227 | database: development 228 | pool: 5 229 | host: 230 | username: root 231 | password: 232 | socket: "/tmp/mysql.sock" 233 | abc: 234 | - 1 235 | - 2.3 236 | - a: 1 237 | b: 2 238 | included2: 239 | - foo: bar 240 | - hoge: fuga 241 | - :abc: 242 | - 1 243 | - 2 244 | - 3 245 | master_path: "/home/joker/.ghq/github.com/joker1007/yaml_master/spec/sample.yml" 246 | master_path2: "/home/joker/.ghq/github.com/joker1007/yaml_master/spec/sample.yml" 247 | user_home: "/home/joker" 248 | user_home2: "/home/joker" 249 | env: "/home/joker" 250 | properties: '24' 251 | read_file_if_exist: 'dummy 252 | 253 | ' 254 | read_file_if_exist_nothing: 255 | read_file_if_exist2: 'dummy ' 256 | ``` 257 | 258 | ## Raw Dump 259 | 260 | Any yaml file can use yaml_master feature. 261 | 262 | ```yaml 263 | # tag_sample.yml 264 | 265 | included: !include included.yml 266 | master_path: !master_path 267 | user_home: !user_home 268 | env: !env HOME 269 | properties: !properties foo 270 | read_file_if_exist: !read_file_if_exist sample.txt 271 | read_file_if_exist_nothing: !read_file_if_exist nothing.txt 272 | ``` 273 | 274 | ```sh 275 | % yaml_master -m tag_sample.yml --dump 276 | ``` 277 | 278 | output 279 | 280 | ```yaml 281 | included: 282 | xyz: hoge 283 | db: 284 | adapter: mysql2 285 | encoding: utf8 286 | database: development 287 | pool: 5 288 | host: 289 | username: root 290 | password: 291 | socket: "/tmp/mysql.sock" 292 | abc: 293 | - 1 294 | - 2.3 295 | - a: 1 296 | b: 2 297 | included2: 298 | - foo: bar 299 | - hoge: fuga 300 | - :abc: 301 | - 1 302 | - 2 303 | - 3 304 | master_path: "/home/joker/.ghq/github.com/joker1007/yaml_master/spec/sample.yml" 305 | master_path2: "/home/joker/.ghq/github.com/joker1007/yaml_master/spec/sample.yml" 306 | user_home: "/home/joker" 307 | user_home2: "/home/joker" 308 | env: "/home/joker" 309 | properties: '24' 310 | read_file_if_exist: 'dummy 311 | 312 | ' 313 | read_file_if_exist_nothing: 314 | read_file_if_exist2: 'dummy ' 315 | ``` 316 | 317 | ## How to use with Docker 318 | 319 | ```sh 320 | docker run --rm \ 321 | -v `pwd`/:/vol \ 322 | joker1007/yaml_master -m /vol/spec/sample.yml -k database_yml -o /vol/test.yml 323 | ``` 324 | 325 | ## Development 326 | 327 | After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake rspec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. 328 | 329 | To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org). 330 | 331 | ## Contributing 332 | 333 | Bug reports and pull requests are welcome on GitHub at https://github.com/joker1007/yaml_master. 334 | 335 | --------------------------------------------------------------------------------