├── tasks ├── core │ ├── setup_build_root.rb │ ├── setup_ruby.rb │ ├── core_setup.rb │ └── setup_process_helper.rb ├── lib │ └── cbm │ │ ├── logger.rb │ │ ├── git_branches_parser.rb │ │ ├── pipeline_updater.rb │ │ ├── branch_manager.rb │ │ └── pipeline_generator.rb ├── manage_branches.rb └── manage-branches.yml ├── examples ├── config │ └── my-repo-branch-manager-config.yml ├── credentials │ └── my-repo-branch-manager-credentials.yml ├── templates │ ├── my-repo-common-resources-template.yml.erb │ ├── my-repo-branch-resource-template.yml.erb │ └── my-repo-branch-job-template.yml.erb ├── tasks │ ├── my-repo-branch-task.yml │ └── my-repo-branch-task-script └── pipelines │ └── branch-manager-example.yml ├── Gemfile ├── ruby-lint.yml ├── spec ├── static_analysis_spec.rb ├── fixtures │ └── my-repo-branch-job-template-without-common-resource.yml.erb ├── spec_helper.rb ├── pipeline_updater_spec.rb ├── git_branches_parser_spec.rb ├── branch_manager_spec.rb └── pipeline_generator_spec.rb ├── .rubocop.yml ├── Gemfile.lock ├── .gitignore ├── LICENSE ├── README.md └── branches.svg /tasks/core/setup_build_root.rb: -------------------------------------------------------------------------------- 1 | ENV['BUILD_ROOT'] = File.expand_path('../../../../', __FILE__) 2 | -------------------------------------------------------------------------------- /tasks/core/setup_ruby.rb: -------------------------------------------------------------------------------- 1 | # requires commonly used in ruby ci scripting 2 | require 'tmpdir' 3 | require 'fileutils' 4 | -------------------------------------------------------------------------------- /examples/config/my-repo-branch-manager-config.yml: -------------------------------------------------------------------------------- 1 | EXAMPLE_LOAD_VARS_FROM_CONFIG_KEY: example_load_vars_from_config_value 2 | -------------------------------------------------------------------------------- /examples/credentials/my-repo-branch-manager-credentials.yml: -------------------------------------------------------------------------------- 1 | EXAMPLE_LOAD_VARS_FROM_CREDENTIALS_KEY: example_load_vars_from_credentials_value 2 | -------------------------------------------------------------------------------- /tasks/core/core_setup.rb: -------------------------------------------------------------------------------- 1 | require_relative 'setup_ruby.rb' 2 | require_relative 'setup_build_root.rb' 3 | require_relative 'setup_process_helper.rb' 4 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem 'bundler' 4 | gem 'process_helper' 5 | gem 'rspec', '~> 3.0' 6 | gem 'ruby-lint', '~> 2.0' 7 | gem 'rubocop', '>= 0.27.0' 8 | -------------------------------------------------------------------------------- /examples/templates/my-repo-common-resources-template.yml.erb: -------------------------------------------------------------------------------- 1 | - name: my-repo-common-resource-master 2 | type: git 3 | source: 4 | uri: <%= uri %> 5 | branch: master 6 | -------------------------------------------------------------------------------- /examples/templates/my-repo-branch-resource-template.yml.erb: -------------------------------------------------------------------------------- 1 | - name: my-repo-branch-<%= branch_name %> 2 | type: git 3 | source: 4 | uri: <%= uri %> 5 | branch: <%= branch_name %> 6 | -------------------------------------------------------------------------------- /tasks/lib/cbm/logger.rb: -------------------------------------------------------------------------------- 1 | module Cbm 2 | # logger proxy for puts so we can avoid spamming log messages in specs 3 | module Logger 4 | def log(msg) 5 | puts msg 6 | end 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /tasks/manage_branches.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require_relative 'core/core_setup' 4 | require_relative 'lib/cbm/branch_manager' 5 | 6 | puts 'Running BranchManager...' 7 | Cbm::BranchManager.new.run 8 | -------------------------------------------------------------------------------- /tasks/core/setup_process_helper.rb: -------------------------------------------------------------------------------- 1 | system('gem install process_helper --no-ri --no-rdoc --version 0.0.3') || 2 | fail('failed to install process_helper gem') 3 | require 'rubygems' 4 | require 'process_helper' 5 | include ProcessHelper 6 | -------------------------------------------------------------------------------- /examples/tasks/my-repo-branch-task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///radial/busyboxplus#git 5 | 6 | inputs: 7 | - name: my-repo-branch 8 | - name: my-repo-common-resource 9 | 10 | run: 11 | path: my-repo-branch/examples/tasks/my-repo-branch-task-script 12 | -------------------------------------------------------------------------------- /ruby-lint.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # http://code.yorickpeterse.com/ruby-lint/latest/ 3 | analysis_classes: 4 | - argument_amount 5 | - pedantics 6 | - shadowing_variables 7 | # - undefined_methods 8 | # - undefined_variables 9 | # - unused_variables 10 | - useless_equality_checks 11 | directories: 12 | - spec 13 | - tasks 14 | - examples 15 | -------------------------------------------------------------------------------- /spec/static_analysis_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative 'spec_helper' 2 | 3 | describe 'static analysis checks' do 4 | it 'ruby-lint' do 5 | ruby_lint_cmd = "bundle exec ruby-lint #{File.expand_path('../..', __FILE__)}" 6 | process(ruby_lint_cmd, out: :error, out_ex: true) 7 | end 8 | 9 | it 'rubocop' do 10 | process('bundle exec rubocop', out: :error, out_ex: true) 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /tasks/manage-branches.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image_resource: 5 | type: docker-image 6 | source: 7 | repository: tracker/busyboxplus 8 | tag: ruby-git 9 | 10 | inputs: 11 | - name: concourse-branch-manager 12 | - name: git-branches 13 | - name: template-repo 14 | - name: config-repo 15 | - name: credentials-repo 16 | 17 | params: 18 | CONCOURSE_URL: null 19 | CONCOURSE_USERNAME: null 20 | CONCOURSE_PASSWORD: null 21 | CONCOURSE_TEAM: null 22 | 23 | run: 24 | path: concourse-branch-manager/tasks/manage_branches.rb 25 | -------------------------------------------------------------------------------- /spec/fixtures/my-repo-branch-job-template-without-common-resource.yml.erb: -------------------------------------------------------------------------------- 1 | - name: my-repo-branch-job-<%= branch_name %> 2 | plan: 3 | - get: my-repo-branch 4 | resource: my-repo-branch-<%= branch_name %> 5 | params: {depth: 20} 6 | trigger: true 7 | - task: my-repo-branch-task 8 | file: my-repo-branch/examples/tasks/my-repo-branch-task.yml 9 | config: 10 | params: 11 | BRANCH_NAME: <%= branch_name %> 12 | EXAMPLE_LOAD_VARS_FROM_CONFIG_KEY: {{EXAMPLE_LOAD_VARS_FROM_CONFIG_KEY}} 13 | EXAMPLE_LOAD_VARS_FROM_CREDENTIALS_KEY: {{EXAMPLE_LOAD_VARS_FROM_CREDENTIALS_KEY}} 14 | -------------------------------------------------------------------------------- /examples/tasks/my-repo-branch-task-script: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | echo "In my-repo-branch-task-script, running `git status` for branch '${BRANCH_NAME}'..." 4 | 5 | cd my-repo-branch 6 | git status 7 | git branch -a 8 | cd .. 9 | 10 | echo "Pipeline load-vars-from values:" 11 | echo " EXAMPLE_LOAD_VARS_FROM_CONFIG_KEY: $EXAMPLE_LOAD_VARS_FROM_CONFIG_KEY" 12 | echo " EXAMPLE_LOAD_VARS_FROM_CREDENTIALS_KEY: $EXAMPLE_LOAD_VARS_FROM_CREDENTIALS_KEY" 13 | echo 14 | echo "First line of readme from PIPELINE_COMMON_RESOURCES_TEMPLATE entry:" 15 | echo "'$(head -n 1 my-repo-common-resource/README.md)'" 16 | echo 17 | echo "Successfully ran my-repo-branch-task-script for branch '${BRANCH_NAME}'!" 18 | -------------------------------------------------------------------------------- /examples/templates/my-repo-branch-job-template.yml.erb: -------------------------------------------------------------------------------- 1 | - name: my-repo-branch-job-<%= branch_name %> 2 | plan: 3 | - get: my-repo-branch 4 | resource: my-repo-branch-<%= branch_name %> 5 | params: {depth: 20} 6 | trigger: true 7 | - get: my-repo-common-resource 8 | resource: my-repo-common-resource-master 9 | params: {depth: 20} 10 | trigger: true 11 | - task: my-repo-branch-task 12 | file: my-repo-branch/examples/tasks/my-repo-branch-task.yml 13 | config: 14 | params: 15 | BRANCH_NAME: <%= branch_name %> 16 | EXAMPLE_LOAD_VARS_FROM_CONFIG_KEY: {{EXAMPLE_LOAD_VARS_FROM_CONFIG_KEY}} 17 | EXAMPLE_LOAD_VARS_FROM_CREDENTIALS_KEY: {{EXAMPLE_LOAD_VARS_FROM_CREDENTIALS_KEY}} 18 | -------------------------------------------------------------------------------- /tasks/lib/cbm/git_branches_parser.rb: -------------------------------------------------------------------------------- 1 | require_relative 'logger' 2 | require 'json' 3 | 4 | module Cbm 5 | # Given a local git branches resource root containing git-branches.json, 6 | # parses it and returns the repo uri and array of branches it contains 7 | class GitBranchesParser 8 | include Logger 9 | attr_reader :git_branches_root 10 | 11 | def initialize(git_branches_root) 12 | @git_branches_root = git_branches_root 13 | end 14 | 15 | def parse 16 | log 'Reading uri and git branches...' 17 | git_branches_json = File.read("#{git_branches_root}/git-branches.json") 18 | git_branches_hash = JSON.parse(git_branches_json) 19 | branches = git_branches_hash.fetch('branches').sort 20 | uri = git_branches_hash.fetch('uri') 21 | 22 | [uri, branches] 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # https://github.com/bbatsov/rubocop/blob/master/config/default.yml 3 | Metrics/LineLength: 4 | Max: 99 5 | 6 | Metrics/MethodLength: 7 | CountComments: false # count full line comments? 8 | Max: 20 9 | 10 | Style/MultilineOperationIndentation: 11 | EnforcedStyle: indented 12 | 13 | Style/SpaceAroundEqualsInParameterDefault: 14 | # compatibility with RubyMine defaults (apparently can't override?) 15 | EnforcedStyle: space 16 | 17 | Style/TrailingComma: 18 | # can't make this only apply to arrays and not params, so it's disabled 19 | Enabled: false 20 | #EnforcedStyleForMultiline: comma 21 | 22 | Style/MultilineOperationIndentation: 23 | # can't convince rubymine 7 to not indent 4 spaces with mult-line expect(x).to receive ... 24 | Enabled: false 25 | 26 | Style/RegexpLiteral: 27 | # This is required or else rubocop argues with itself when trying to escape a slash on a single-line regex 28 | AllowInnerSlashes: true 29 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'rspec' 2 | require 'tmpdir' 3 | require 'json' 4 | require 'process_helper' 5 | require_relative '../tasks/lib/cbm/logger' 6 | 7 | # RSpec config 8 | RSpec.configure do |c| 9 | c.before(:suite) do 10 | ENV['GIT_AUTHOR_NAME'] = 'cbm' 11 | ENV['GIT_AUTHOR_EMAIL'] = 'cbm@example.com' 12 | ENV['GIT_COMMITTER_NAME'] = 'cbm' 13 | ENV['GIT_COMMITTER_EMAIL'] = 'cbm@example.com' 14 | end 15 | 16 | c.before(:each) do 17 | # squelch log messages during specs 18 | allow_any_instance_of(Cbm::Logger).to receive(:log) 19 | end 20 | end 21 | 22 | # RSpec helper methods 23 | module SpecHelper 24 | include ProcessHelper 25 | 26 | def make_git_branches_root 27 | git_branches_root = Dir.mktmpdir 28 | 29 | git_branches_hash = { 30 | 'uri' => 'https://github.com/user/repo.git', 31 | 'branches' => [ 32 | 'master', 33 | 'feature-1' 34 | ] 35 | } 36 | git_branches_json = JSON.dump(git_branches_hash) 37 | 38 | FileUtils.cd(git_branches_root) do 39 | File.open('git-branches.json', 'w') do |file| 40 | file.write(git_branches_json) 41 | end 42 | end 43 | 44 | git_branches_root 45 | end 46 | end 47 | 48 | include SpecHelper 49 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | ast (2.1.0) 5 | astrolabe (1.3.1) 6 | parser (~> 2.2) 7 | diff-lcs (1.2.5) 8 | parser (2.2.3.0) 9 | ast (>= 1.1, < 3.0) 10 | powerpack (0.1.1) 11 | process_helper (0.0.3) 12 | rainbow (2.0.0) 13 | rspec (3.4.0) 14 | rspec-core (~> 3.4.0) 15 | rspec-expectations (~> 3.4.0) 16 | rspec-mocks (~> 3.4.0) 17 | rspec-core (3.4.0) 18 | rspec-support (~> 3.4.0) 19 | rspec-expectations (3.4.0) 20 | diff-lcs (>= 1.2.0, < 2.0) 21 | rspec-support (~> 3.4.0) 22 | rspec-mocks (3.4.0) 23 | diff-lcs (>= 1.2.0, < 2.0) 24 | rspec-support (~> 3.4.0) 25 | rspec-support (3.4.0) 26 | rubocop (0.35.1) 27 | astrolabe (~> 1.3) 28 | parser (>= 2.2.3.0, < 3.0) 29 | powerpack (~> 0.1) 30 | rainbow (>= 1.99.1, < 3.0) 31 | ruby-progressbar (~> 1.7) 32 | tins (<= 1.6.0) 33 | ruby-lint (2.0.5) 34 | parser (~> 2.2) 35 | slop (~> 3.4, >= 3.4.7) 36 | ruby-progressbar (1.7.5) 37 | slop (3.6.0) 38 | tins (1.6.0) 39 | 40 | PLATFORMS 41 | ruby 42 | 43 | DEPENDENCIES 44 | bundler 45 | process_helper 46 | rspec (~> 3.0) 47 | rubocop (>= 0.27.0) 48 | ruby-lint (~> 2.0) 49 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | /.config 4 | /coverage/ 5 | /InstalledFiles 6 | /pkg/ 7 | /spec/reports/ 8 | /test/tmp/ 9 | /test/version_tmp/ 10 | /tmp/ 11 | 12 | ## Specific to RubyMotion: 13 | .dat* 14 | .repl_history 15 | build/ 16 | 17 | ## Documentation cache and generated files: 18 | /.yardoc/ 19 | /_yardoc/ 20 | /doc/ 21 | /rdoc/ 22 | 23 | ## Environment normalisation: 24 | /.bundle/ 25 | /lib/bundler/man/ 26 | 27 | # for a library or gem, you might want to ignore these files since the code is 28 | # intended to run in multiple environments; otherwise, check them in: 29 | # Gemfile.lock 30 | # .ruby-version 31 | # .ruby-gemset 32 | 33 | # unless supporting rvm < 1.11.0 or doing something fancy, ignore this: 34 | .rvmrc 35 | 36 | # OSX 37 | .DS_Store 38 | .AppleDouble 39 | .LSOverride 40 | 41 | # Icon must end with two \r 42 | Icon 43 | 44 | # Thumbnails 45 | ._* 46 | 47 | # Files that might appear on external disk 48 | .Spotlight-V100 49 | .Trashes 50 | 51 | # Directories potentially created on remote AFP share 52 | .AppleDB 53 | .AppleDesktop 54 | Network Trash Folder 55 | Temporary Items 56 | .apdisk 57 | 58 | # RubyMine 59 | .idea 60 | 61 | # Bash build dir 62 | spec/bashs 63 | 64 | # Git build dir 65 | spec/gits 66 | 67 | # ignore any local secrets 68 | secrets.yml 69 | -------------------------------------------------------------------------------- /spec/pipeline_updater_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative 'spec_helper' 2 | require_relative '../tasks/lib/cbm/logger' 3 | require_relative '../tasks/lib/cbm/pipeline_updater' 4 | 5 | describe Cbm::PipelineUpdater do 6 | it 'updates pipeline' do 7 | url = 'http://myconcourse.example.com' 8 | username = 'admin' 9 | password = 'password' 10 | pipeline_file = double 11 | load_vars_from_1 = 'path/to/config' 12 | load_vars_from_2 = 'path/to/credentials' 13 | load_vars_from_entries = [load_vars_from_1, load_vars_from_2] 14 | 15 | subject = Cbm::PipelineUpdater.new( 16 | url, 17 | username, 18 | password, 19 | pipeline_file, 20 | load_vars_from_entries, 21 | 'my-pipeline') 22 | 23 | allow(subject).to receive(:fly_path).and_return('/path/to/fly') 24 | 25 | fly_download_url = 'http://myconcourse.example.com/api/v1/cli?arch=amd64&platform=linux' 26 | creds = %w(admin password) 27 | stream = double 28 | expect(subject).to receive(:open) 29 | .with(fly_download_url, 'rb', http_basic_authentication: creds) 30 | .and_return(stream) 31 | expect(IO).to receive(:copy_stream).with(stream, '/path/to/fly') 32 | 33 | expect(subject).to receive(:process).with('chmod +x /path/to/fly') 34 | 35 | login_cmd = '/path/to/fly --target=concourse login ' \ 36 | '--concourse-url=http://myconcourse.example.com' 37 | expect(subject).to receive(:process) 38 | .with(login_cmd, timeout: 5, input_lines: %w(admin password)) 39 | 40 | expect(pipeline_file).to receive(:to_s).and_return('/tmp/pipeline.yml') 41 | set_pipeline_cmd = '/path/to/fly --target=concourse set-pipeline ' \ 42 | '--config=/tmp/pipeline.yml --pipeline=my-pipeline ' \ 43 | '--load-vars-from=path/to/config --load-vars-from=path/to/credentials' 44 | expect(subject).to receive(:process).with(set_pipeline_cmd, timeout: 5, input_lines: %w(y)) 45 | 46 | unpause_cmd = '/path/to/fly --target=concourse unpause-pipeline --pipeline=my-pipeline' 47 | expect(subject).to receive(:process).with(unpause_cmd, timeout: 5) 48 | 49 | subject.set_pipeline 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /tasks/lib/cbm/pipeline_updater.rb: -------------------------------------------------------------------------------- 1 | require 'process_helper' 2 | require 'tmpdir' 3 | require 'open-uri' 4 | 5 | module Cbm 6 | # Creates/updates pipeline via fly 7 | class PipelineUpdater 8 | include Logger 9 | include ProcessHelper 10 | 11 | attr_reader :url, :username, :password, :team, :pipeline_file, :fly_path 12 | attr_reader :load_vars_from_entries, :pipeline_name 13 | 14 | # TODO: do http://www.refactoring.com/catalog/introduceParameterObject.html 15 | # rubocop:disable Metrics/ParameterLists 16 | def initialize(url, username, password, team, pipeline_file, load_vars_from_entries, pipeline_name) 17 | @url = url 18 | @username = username 19 | @password = password 20 | @team = team 21 | @pipeline_file = pipeline_file 22 | @fly_path = "#{Dir.mktmpdir}/fly" 23 | @load_vars_from_entries = load_vars_from_entries 24 | @pipeline_name = pipeline_name 25 | end 26 | 27 | def set_pipeline 28 | download_fly 29 | 30 | log 'Logging into concourse...' 31 | team_argument = team != nil && team != "" ? "--team-name=#{team}" : '' 32 | process( 33 | "#{fly_path} --target=concourse login --concourse-url=#{url} #{team_argument}", 34 | timeout: 5, 35 | input_lines: [username, password]) 36 | 37 | log 'Updating pipeline...' 38 | process(generate_set_pipeline_cmd, timeout: 5, input_lines: %w(y)) 39 | 40 | log 'Unpausing pipeline...' 41 | unpause_pipeline_cmd = "#{fly_path} --target=concourse unpause-pipeline " \ 42 | "--pipeline=#{pipeline_name}" 43 | process(unpause_pipeline_cmd, timeout: 5) 44 | end 45 | 46 | private 47 | 48 | def generate_set_pipeline_cmd 49 | load_vars_from_options = load_vars_from_entries.reduce('') do |options, entry| 50 | "#{options}--load-vars-from=#{entry} " 51 | end.strip 52 | "#{fly_path} --target=concourse set-pipeline --config=#{pipeline_file} " \ 53 | "--pipeline=#{pipeline_name} #{load_vars_from_options}" 54 | end 55 | 56 | def download_fly 57 | log 'Downloading fly executable...' 58 | 59 | fly_download_url = "#{url}/api/v1/cli?arch=amd64&platform=linux" 60 | read_binary_open_mode = 'rb' 61 | stream = open( 62 | fly_download_url, 63 | read_binary_open_mode, 64 | http_basic_authentication: [username, password]) 65 | IO.copy_stream(stream, fly_path) 66 | process("chmod +x #{fly_path}") 67 | end 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /tasks/lib/cbm/branch_manager.rb: -------------------------------------------------------------------------------- 1 | require_relative('git_branches_parser') 2 | require_relative('pipeline_generator') 3 | require_relative('pipeline_updater') 4 | require_relative('logger') 5 | require 'json' 6 | 7 | module Cbm 8 | # Main class and entry point 9 | class BranchManager 10 | attr_reader :build_root, :url, :username, :password, :username, :team, :resource_template_file 11 | attr_reader :job_template_file, :load_vars_from_entries, :pipeline_name 12 | attr_reader :common_resources_template, :group_per_branch, :resource_type_template_file 13 | 14 | def initialize 15 | @build_root = ENV.fetch('BUILD_ROOT') 16 | @url = ENV.fetch('CONCOURSE_URL') 17 | @username = ENV.fetch('CONCOURSE_USERNAME') 18 | @password = ENV.fetch('CONCOURSE_PASSWORD') 19 | @team = ENV.fetch('CONCOURSE_TEAM', nil) 20 | @resource_template_file = ENV.fetch('BRANCH_RESOURCE_TEMPLATE') 21 | @job_template_file = ENV.fetch('BRANCH_JOB_TEMPLATE') 22 | @pipeline_name = ENV.fetch('PIPELINE_NAME', nil) 23 | @load_vars_from_entries = parse_load_vars_from_entries 24 | @common_resources_template = ENV.fetch('PIPELINE_COMMON_RESOURCES_TEMPLATE', nil) 25 | @resource_type_template_file = ENV.fetch('PIPELINE_RESOURCE_TYPE_TEMPLATE', nil) 26 | @group_per_branch = ENV.fetch('GROUP_PER_BRANCH', 'true') == 'true' 27 | end 28 | 29 | # TODO: do http://www.refactoring.com/catalog/introduceParameterObject.html 30 | # rubocop:disable Metrics/AbcSize 31 | def run 32 | git_uri, branches = Cbm::GitBranchesParser.new(git_branches_root).parse 33 | 34 | pipeline_file = Cbm::PipelineGenerator.new( 35 | git_uri, 36 | branches, 37 | resource_template_file, 38 | job_template_file, 39 | common_resources_template, 40 | resource_type_template_file, 41 | group_per_branch).generate 42 | Cbm::PipelineUpdater.new( 43 | url, 44 | username, 45 | password, 46 | team, 47 | pipeline_file, 48 | load_vars_from_entries, 49 | pipeline_name_or_default(git_uri)).set_pipeline 50 | end 51 | 52 | private 53 | 54 | def git_branches_root 55 | "#{build_root}/git-branches" 56 | end 57 | 58 | def pipeline_name_or_default(git_uri) 59 | repo = git_uri.split('/').last.gsub('.git', '') 60 | pipeline_name || "cbm-#{repo}" 61 | end 62 | 63 | def parse_load_vars_from_entries 64 | entries = ENV.keys.map do |key| 65 | regexp = /^(PIPELINE_LOAD_VARS_FROM_\d+)$/ 66 | matches = regexp.match(key) 67 | ENV.fetch(matches[1]) if matches 68 | end 69 | entries.compact 70 | end 71 | end 72 | end 73 | -------------------------------------------------------------------------------- /spec/git_branches_parser_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative 'spec_helper' 2 | require_relative '../tasks/lib/cbm/git_branches_parser' 3 | 4 | describe Cbm::GitBranchesParser do 5 | it 'lists uri and remote branches alphabetically' do 6 | git_branches_root = make_git_branches_root 7 | 8 | subject = Cbm::GitBranchesParser.new(git_branches_root) 9 | uri, branches = subject.parse 10 | expect(uri).to eq('https://github.com/user/repo.git') 11 | expect(branches).to eq(%w(feature-1 master)) 12 | end 13 | # 14 | # it 'lists only branches matching the specified regex' do 15 | # local_repo = make_cloned_repo[:local] 16 | # 17 | # FileUtils.cd(local_repo) do 18 | # process('git checkout -b feature-1', out: :error) 19 | # process('git push', out: :error) 20 | # process('git checkout -b feature-2', out: :error) 21 | # process('git push', out: :error) 22 | # process('git checkout -b zzz', out: :error) 23 | # process('git push', out: :error) 24 | # end 25 | # 26 | # subject = Cbm::BranchLister.new(local_repo, 'feature-', 20) 27 | # branches = subject.list 28 | # expect(branches).to eq(%w(feature-1 feature-2)) 29 | # end 30 | # 31 | # it 'handles branches containing slashes and regex special characters' do 32 | # local_repo = make_cloned_repo[:local] 33 | # 34 | # FileUtils.cd(local_repo) do 35 | # process('git checkout -b feat/feature-1', out: :error) 36 | # process('git push', out: :error) 37 | # process('git checkout -b feat{feature-2', out: :error) 38 | # process('git push', out: :error) 39 | # process('git checkout -b zzz', out: :error) 40 | # process('git push', out: :error) 41 | # end 42 | # 43 | # subject = Cbm::BranchLister.new(local_repo, 'feat(\/|\{)feature-', 20) 44 | # branches = subject.list 45 | # expect(branches).to eq(%w(feat/feature-1 feat{feature-2)) 46 | # end 47 | # 48 | # it 'fails if more than MAX_BRANCHES matches' do 49 | # local_repo = make_cloned_repo[:local] 50 | # 51 | # FileUtils.cd(local_repo) do 52 | # process('git checkout -b feature-1', out: :error) 53 | # process('git push', out: :error) 54 | # process('git checkout -b feature-2', out: :error) 55 | # process('git push', out: :error) 56 | # process('git checkout -b feature-3', out: :error) 57 | # process('git push', out: :error) 58 | # end 59 | # 60 | # max_branches = 2 61 | # subject = Cbm::BranchLister.new(local_repo, 'feature-', max_branches) 62 | # expected_msg = '3 branches found. Increase MAX_BRANCHES, ' \ 63 | # 'or provide a more specific regular expression.' 64 | # expect { subject.list }.to raise_error(RuntimeError, expected_msg) 65 | # end 66 | end 67 | -------------------------------------------------------------------------------- /examples/pipelines/branch-manager-example.yml: -------------------------------------------------------------------------------- 1 | --- 2 | groups: 3 | - name: branch-manager 4 | jobs: 5 | - branch-manager 6 | 7 | resources: 8 | - name: concourse-branch-manager 9 | type: git 10 | source: 11 | uri: https://github.com/pivotaltracker/concourse-branch-manager.git 12 | branch: master 13 | ignore_paths: [Gemfile, Gemfile.lock] 14 | 15 | # This `git-branches` input resource determines which branches will be processed 16 | - name: branch-manager-git-branches 17 | type: git-branches 18 | source: 19 | # Set this to the uri of your repo for which you want to dynamically build arbitrary branches 20 | uri: https://github.com/pivotaltracker/concourse-branch-manager.git 21 | branch_regexp: ".*" 22 | max_branches: 20 23 | 24 | # This repo containing your resource/job templates can be the same repo as 25 | # the one in the git-branches resource above, but it doesn't have to be 26 | - name: branch-manager-templates 27 | type: git 28 | source: 29 | # Set this to the uri of your repo containing your resource/job templates for building branches 30 | uri: https://github.com/pivotaltracker/concourse-branch-manager.git 31 | branch: master 32 | # paths: [examples/templates/*] 33 | 34 | # This repo contains any non-secret non-credential config that needs to be 35 | # passed into your generated pipeline via fly --load-vars-from options 36 | - name: branch-manager-config 37 | type: git 38 | source: 39 | # Set this to the uri of your repo containing your non-secret non-credential config 40 | uri: https://github.com/pivotaltracker/concourse-branch-manager.git 41 | branch: master 42 | # paths: [examples/config/*] 43 | 44 | # This repo contains any secret credential config that needs to be 45 | # passed into your generated pipeline via fly --load-vars-from options 46 | - name: branch-manager-credentials 47 | type: git 48 | source: 49 | # Set this to the uri of your repo containing your non-secret non-credential config 50 | uri: https://github.com/pivotaltracker/concourse-branch-manager.git 51 | branch: master 52 | # paths: [examples/credentials/*] 53 | 54 | jobs: 55 | - name: branch-manager 56 | serial: true 57 | plan: 58 | - get: concourse-branch-manager 59 | params: {depth: 20} 60 | trigger: true 61 | - get: git-branches 62 | resource: branch-manager-git-branches 63 | trigger: true 64 | - get: template-repo 65 | resource: branch-manager-templates 66 | params: {depth: 20} 67 | trigger: true 68 | - get: config-repo 69 | resource: branch-manager-config 70 | params: {depth: 20} 71 | trigger: true 72 | - get: credentials-repo 73 | resource: branch-manager-credentials 74 | params: {depth: 20} 75 | trigger: true 76 | - task: manage-branches 77 | file: concourse-branch-manager/tasks/manage-branches.yml 78 | config: 79 | params: 80 | BRANCH_RESOURCE_TEMPLATE: template-repo/examples/templates/my-repo-branch-resource-template.yml.erb 81 | BRANCH_JOB_TEMPLATE: template-repo/examples/templates/my-repo-branch-job-template.yml.erb 82 | PIPELINE_COMMON_RESOURCES_TEMPLATE: template-repo/examples/templates/my-repo-common-resources-template.yml.erb 83 | CONCOURSE_URL: {{CONCOURSE_URL}} 84 | CONCOURSE_USERNAME: {{CONCOURSE_USERNAME}} 85 | CONCOURSE_PASSWORD: {{CONCOURSE_PASSWORD}} 86 | PIPELINE_LOAD_VARS_FROM_1: config-repo/examples/config/my-repo-branch-manager-config.yml 87 | PIPELINE_LOAD_VARS_FROM_2: credentials-repo/examples/credentials/my-repo-branch-manager-credentials.yml 88 | PIPELINE_NAME: cbm-example-branches 89 | GROUP_PER_BRANCH: true 90 | -------------------------------------------------------------------------------- /spec/branch_manager_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative 'spec_helper' 2 | require_relative '../tasks/lib/cbm/git_branches_parser' 3 | require_relative '../tasks/lib/cbm/branch_manager' 4 | require_relative '../tasks/lib/cbm/pipeline_generator' 5 | 6 | describe Cbm::BranchManager do 7 | attr_reader :git_uri, :concourse_url, :pipeline_file, :pipeline_updater, :branches 8 | attr_reader :pipeline_generator 9 | 10 | before do 11 | allow(ENV).to receive(:fetch).and_call_original 12 | 13 | expect(ENV).to receive(:fetch).with('BUILD_ROOT').and_return('/build-root') 14 | @concourse_url = 'http://my-concourse.example.com' 15 | expect(ENV).to receive(:fetch).with('CONCOURSE_URL').and_return(@concourse_url) 16 | expect(ENV).to receive(:fetch).with('CONCOURSE_USERNAME').and_return('username') 17 | expect(ENV).to receive(:fetch).with('CONCOURSE_PASSWORD').and_return('password') 18 | expect(ENV).to receive(:fetch).with('BRANCH_RESOURCE_TEMPLATE') 19 | .and_return('template-repo/resource.yml.erb') 20 | expect(ENV).to receive(:fetch).with('BRANCH_JOB_TEMPLATE') 21 | .and_return('template-repo/job.yml.erb') 22 | allow(ENV).to receive(:keys).and_return(%w(UNRELATED IRRELEVANT)) 23 | allow(ENV) 24 | .to receive(:fetch) 25 | .with('PIPELINE_COMMON_RESOURCES_TEMPLATE', nil).and_return(nil) 26 | allow(ENV).to receive(:fetch).with('GROUP_PER_BRANCH', 'true').and_return('false') 27 | 28 | git_branches_parser = double 29 | expect(Cbm::GitBranchesParser).to receive(:new) 30 | .with('/build-root/git-branches') 31 | .and_return(git_branches_parser) 32 | @git_uri = 'https://github.com/user/repo.git' 33 | @branches = %w(branch1 master) 34 | expect(git_branches_parser).to receive(:parse).and_return([git_uri, branches]) 35 | 36 | @pipeline_generator = double 37 | allow(Cbm::PipelineGenerator).to receive(:new) 38 | .with( 39 | git_uri, 40 | branches, 41 | 'template-repo/resource.yml.erb', 42 | 'template-repo/job.yml.erb', 43 | nil, 44 | false) 45 | .and_return(pipeline_generator) 46 | @pipeline_file = double 47 | expect(pipeline_generator).to receive(:generate).and_return(pipeline_file) 48 | 49 | @pipeline_updater = double 50 | expect(pipeline_updater).to receive(:set_pipeline) 51 | end 52 | 53 | it 'works with none of the optional params specified' do 54 | subject = Cbm::BranchManager.new 55 | expect(Cbm::PipelineGenerator).to receive(:new) 56 | .with( 57 | git_uri, 58 | branches, 59 | 'template-repo/resource.yml.erb', 60 | 'template-repo/job.yml.erb', 61 | nil, 62 | false) 63 | .and_return(pipeline_generator) 64 | expect(Cbm::PipelineUpdater).to receive(:new) 65 | .with(concourse_url, 'username', 'password', pipeline_file, [], 'cbm-repo') 66 | .and_return(pipeline_updater) 67 | subject.run 68 | end 69 | 70 | it 'has no syntax errors in #run' do 71 | subject = Cbm::BranchManager.new 72 | expect(Cbm::PipelineUpdater).to receive(:new) 73 | .with(concourse_url, 'username', 'password', pipeline_file, [], 'cbm-repo') 74 | .and_return(pipeline_updater) 75 | subject.run 76 | end 77 | 78 | it 'handles PIPELINE_LOAD_VARS_FROM_n' do 79 | expect(ENV).to receive(:keys) 80 | .and_return(%w(PIPELINE_LOAD_VARS_FROM_1 PIPELINE_LOAD_VARS_FROM_2 ZED)) 81 | expect(ENV).to receive(:fetch).with('PIPELINE_LOAD_VARS_FROM_1') 82 | .and_return('path/to/config') 83 | expect(ENV).to receive(:fetch).with('PIPELINE_LOAD_VARS_FROM_2') 84 | .and_return('path/to/credentials') 85 | subject = Cbm::BranchManager.new 86 | expected_load_vars_from_entries = [ 87 | 'path/to/config', 88 | 'path/to/credentials', 89 | ] 90 | expect(Cbm::PipelineUpdater).to receive(:new) 91 | .with( 92 | concourse_url, 93 | 'username', 94 | 'password', 95 | pipeline_file, 96 | expected_load_vars_from_entries, 97 | 'cbm-repo') 98 | .and_return(pipeline_updater) 99 | subject.run 100 | end 101 | 102 | it 'allows optional override of PIPELINE_NAME' do 103 | expect(ENV).to receive(:fetch).with('PIPELINE_NAME', nil).and_return('name') 104 | subject = Cbm::BranchManager.new 105 | expect(Cbm::PipelineUpdater).to receive(:new) 106 | .with(concourse_url, 'username', 'password', pipeline_file, [], 'name') 107 | .and_return(pipeline_updater) 108 | subject.run 109 | end 110 | 111 | it 'handles the PIPELINE_COMMON_RESOURCES_TEMPLATE param' do 112 | expect(ENV) 113 | .to receive(:fetch) 114 | .with('PIPELINE_COMMON_RESOURCES_TEMPLATE', nil).and_return('path/to/template') 115 | subject = Cbm::BranchManager.new 116 | expect(Cbm::PipelineGenerator).to receive(:new) 117 | .with( 118 | git_uri, 119 | branches, 120 | 'template-repo/resource.yml.erb', 121 | 'template-repo/job.yml.erb', 122 | 'path/to/template', 123 | false) 124 | .and_return(pipeline_generator) 125 | expect(Cbm::PipelineUpdater).to receive(:new) 126 | .with( 127 | concourse_url, 128 | 'username', 129 | 'password', 130 | pipeline_file, 131 | [], 132 | 'cbm-repo') 133 | .and_return(pipeline_updater) 134 | subject.run 135 | end 136 | end 137 | -------------------------------------------------------------------------------- /tasks/lib/cbm/pipeline_generator.rb: -------------------------------------------------------------------------------- 1 | require 'yaml' 2 | require 'tmpdir' 3 | require 'erb' 4 | 5 | module Cbm 6 | # Generates pipeline yml based on branches 7 | class PipelineGenerator 8 | include Logger 9 | attr_reader :git_uri, :branches, :resource_template_file, :job_template_file 10 | attr_reader :common_resource_template_file, :group_per_branch, :resource_type_template_file 11 | 12 | # TODO: do http://www.refactoring.com/catalog/introduceParameterObject.html 13 | # rubocop:disable Metrics/LineLength, Metrics/ParameterLists 14 | def initialize(git_uri, branches, resource_template_file, job_template_file, common_resource_template_file, resource_type_template_file, group_per_branch) 15 | @git_uri = git_uri 16 | @branches = branches 17 | @resource_template_file = resource_template_file 18 | @job_template_file = job_template_file 19 | @common_resource_template_file = common_resource_template_file 20 | @resource_type_template_file = resource_type_template_file 21 | @group_per_branch = group_per_branch 22 | end 23 | 24 | def generate 25 | log 'Generating pipeline file...' 26 | 27 | pipeline_yml = build_yml 28 | 29 | log 'Generated pipeline yml:' 30 | log '-' * 80 31 | log pipeline_yml 32 | log '-' * 80 33 | 34 | write_pipeline_file(pipeline_yml) 35 | end 36 | 37 | private 38 | 39 | def build_yml 40 | binding_class = create_binding_class 41 | 42 | resource_entries = create_entries_from_template(binding_class, resource_template_file) 43 | 44 | common_resource_entries = create_common_entries_from_template( 45 | binding_class, 46 | common_resource_template_file 47 | ) 48 | 49 | resource_type_entries = create_resource_types_from_template( 50 | binding_class, 51 | resource_type_template_file 52 | ) 53 | 54 | groups, job_entries = create_groups_and_jobs_entries(binding_class) 55 | 56 | create_complete_yaml(groups, resource_entries, common_resource_entries, job_entries, resource_type_entries) 57 | end 58 | 59 | def create_groups_and_jobs_entries(binding_class) 60 | groups = '' 61 | all_jobs_entry = [{ 'name' => '000-all', 'jobs' => [], }] 62 | job_entries = create_entries_from_template( 63 | binding_class, job_template_file) do |branch, job_entry_yml| 64 | groups += create_group_entry(branch, job_entry_yml, all_jobs_entry) if group_per_branch 65 | end 66 | 67 | if group_per_branch 68 | all_jobs_entry_yaml = YAML.dump(all_jobs_entry).gsub(/^---\n/, '') 69 | groups = "groups:\n" + all_jobs_entry_yaml + groups 70 | end 71 | [groups, job_entries] 72 | end 73 | 74 | def create_complete_yaml(groups, resource_entries, common_resource_entries, job_entries, resource_type_entries) 75 | "---\n" \ 76 | "#{groups}" \ 77 | "resources:\n" \ 78 | "#{resource_entries}\n" \ 79 | "#{common_resource_entries}\n" \ 80 | "jobs:\n" \ 81 | "#{job_entries}\n" \ 82 | "resource_types:\n" \ 83 | "#{resource_type_entries}\n" 84 | end 85 | 86 | def write_pipeline_file(pipeline_yml) 87 | tmpdir = Dir.mktmpdir 88 | pipeline_file = "#{tmpdir}/pipeline.yml" 89 | File.open(pipeline_file, 'w') do |file| 90 | file.write(pipeline_yml) 91 | end 92 | pipeline_file 93 | end 94 | 95 | def create_binding_class 96 | binding_class = Class.new 97 | binding_class.class_eval( 98 | <<-BINDING_CLASS 99 | attr_accessor :uri, :branch_name 100 | def get_binding 101 | binding() 102 | end 103 | BINDING_CLASS 104 | ) 105 | binding_class 106 | end 107 | 108 | def create_group_entry(branch, job_entry_yml, all_jobs_entry) 109 | job_entry_hashes = YAML.load(job_entry_yml) 110 | job_names = job_entry_hashes.reduce([]) do |names, job| 111 | names << job.fetch('name') 112 | end 113 | all_jobs_entry.first.fetch('jobs').concat(job_names) 114 | group_entry_yaml = YAML.dump([{ 'name' => branch, 'jobs' => job_names, }]) 115 | group_entry_yaml.gsub(/^---\n/, '') 116 | end 117 | 118 | def create_common_entries_from_template(binding_class, template_file) 119 | return '' unless template_file 120 | template = open(template_file).read 121 | erb_binding = binding_class.new 122 | erb_binding.uri = git_uri 123 | ERB.new(template).result(erb_binding.get_binding) 124 | end 125 | 126 | def create_entries_from_template(binding_class, template_file, &block) 127 | template = open(template_file).read 128 | 129 | branches.reduce('') do |entries_memo, branch| 130 | erb_binding = binding_class.new 131 | erb_binding.uri = git_uri 132 | erb_binding.branch_name = branch 133 | entry_yml = ERB.new(template).result(erb_binding.get_binding) 134 | yield(branch, entry_yml) if block 135 | entries_memo.concat(entry_yml) 136 | end 137 | end 138 | 139 | def create_resource_types_from_template(binding_class, template_file) 140 | return '' unless template_file 141 | template = open(template_file).read 142 | erb_binding = binding_class.new 143 | erb_binding.uri = git_uri 144 | ERB.new(template).result(erb_binding.get_binding) 145 | end 146 | end 147 | end 148 | -------------------------------------------------------------------------------- /spec/pipeline_generator_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative 'spec_helper' 2 | require_relative '../tasks/lib/cbm/logger' 3 | require_relative '../tasks/lib/cbm/pipeline_generator' 4 | require 'yaml' 5 | 6 | describe Cbm::PipelineGenerator do 7 | attr_reader :uri, :branches, :resource_template_fixture, :expected_pipeline_yml_hash 8 | 9 | before do 10 | @uri = 'https://github.com/user/repo.git' 11 | @branches = %w(branch1 master) 12 | @resource_template_fixture = File.expand_path( 13 | '../../examples/templates/my-repo-branch-resource-template.yml.erb', __FILE__ 14 | ) 15 | 16 | @expected_pipeline_yml_hash = { 17 | 'groups' => [ 18 | { 19 | 'name' => '000-all', 20 | 'jobs' => [ 21 | 'my-repo-branch-job-branch1', 22 | 'my-repo-branch-job-master', 23 | ], 24 | }, 25 | { 26 | 'name' => 'branch1', 27 | 'jobs' => [ 28 | 'my-repo-branch-job-branch1', 29 | ], 30 | }, 31 | { 32 | 'name' => 'master', 33 | 'jobs' => [ 34 | 'my-repo-branch-job-master', 35 | ], 36 | }, 37 | ], 38 | 'resources' => [ 39 | { 40 | 'name' => 'my-repo-branch-branch1', 41 | 'type' => 'git', 42 | 'source' => { 43 | 'uri' => 'https://github.com/user/repo.git', 44 | 'branch' => 'branch1', 45 | }, 46 | }, 47 | { 48 | 'name' => 'my-repo-branch-master', 49 | 'type' => 'git', 50 | 'source' => { 51 | 'uri' => 'https://github.com/user/repo.git', 52 | 'branch' => 'master', 53 | }, 54 | }, 55 | { 56 | 'name' => 'my-repo-common-resource-master', 57 | 'type' => 'git', 58 | 'source' => { 59 | 'uri' => 'https://github.com/user/repo.git', 60 | 'branch' => 'master', 61 | }, 62 | }, 63 | ], 64 | 'jobs' => [ 65 | { 66 | 'name' => 'my-repo-branch-job-branch1', 67 | 'plan' => [ 68 | { 69 | 'get' => 'my-repo-branch', 70 | 'resource' => 'my-repo-branch-branch1', 71 | 'params' => { 'depth' => 20 }, 72 | 'trigger' => true, 73 | }, 74 | { 75 | 'get' => 'my-repo-common-resource', 76 | 'resource' => 'my-repo-common-resource-master', 77 | 'params' => { 'depth' => 20 }, 78 | 'trigger' => true, 79 | }, 80 | { 81 | 'task' => 'my-repo-branch-task', 82 | 'file' => 'my-repo-branch/examples/tasks/my-repo-branch-task.yml', 83 | 'config' => { 84 | 'params' => { 85 | 'BRANCH_NAME' => 'branch1', 86 | 'EXAMPLE_LOAD_VARS_FROM_CONFIG_KEY' => 87 | '{{EXAMPLE_LOAD_VARS_FROM_CONFIG_KEY}}', 88 | 'EXAMPLE_LOAD_VARS_FROM_CREDENTIALS_KEY' => 89 | '{{EXAMPLE_LOAD_VARS_FROM_CREDENTIALS_KEY}}', 90 | }, 91 | }, 92 | }, 93 | ], 94 | }, 95 | { 96 | 'name' => 'my-repo-branch-job-master', 97 | 'plan' => [ 98 | { 99 | 'get' => 'my-repo-branch', 100 | 'resource' => 'my-repo-branch-master', 101 | 'params' => { 'depth' => 20 }, 102 | 'trigger' => true, 103 | }, 104 | { 105 | 'get' => 'my-repo-common-resource', 106 | 'resource' => 'my-repo-common-resource-master', 107 | 'params' => { 'depth' => 20 }, 108 | 'trigger' => true, 109 | }, 110 | { 111 | 'task' => 'my-repo-branch-task', 112 | 'file' => 'my-repo-branch/examples/tasks/my-repo-branch-task.yml', 113 | 'config' => { 114 | 'params' => { 115 | 'BRANCH_NAME' => 'master', 116 | 'EXAMPLE_LOAD_VARS_FROM_CONFIG_KEY' => 117 | '{{EXAMPLE_LOAD_VARS_FROM_CONFIG_KEY}}', 118 | 'EXAMPLE_LOAD_VARS_FROM_CREDENTIALS_KEY' => 119 | '{{EXAMPLE_LOAD_VARS_FROM_CREDENTIALS_KEY}}', 120 | }, 121 | }, 122 | } 123 | ], 124 | }, 125 | ] 126 | } 127 | end 128 | 129 | it 'generates pipeline yml' do 130 | common_resource_fixture = File.expand_path( 131 | '../../examples/templates/my-repo-common-resources-template.yml.erb', __FILE__ 132 | ) 133 | 134 | job_template_fixture = File.expand_path( 135 | '../../examples/templates/my-repo-branch-job-template.yml.erb', __FILE__ 136 | ) 137 | subject = Cbm::PipelineGenerator.new( 138 | uri, branches, resource_template_fixture, job_template_fixture, common_resource_fixture, true 139 | ) 140 | 141 | pipeline_file = subject.generate 142 | perform_assertion(expected_pipeline_yml_hash, pipeline_file) 143 | end 144 | 145 | it 'generates pipeline yml without optional common resources template specified' do 146 | expected_pipeline_yml_hash['resources'].delete_at(2) 147 | expected_pipeline_yml_hash['jobs'][0]['plan'].delete_at(1) 148 | expected_pipeline_yml_hash['jobs'][1]['plan'].delete_at(1) 149 | 150 | job_template_fixture = File.expand_path( 151 | '../fixtures/my-repo-branch-job-template-without-common-resource.yml.erb', __FILE__ 152 | ) 153 | subject = Cbm::PipelineGenerator.new( 154 | uri, branches, resource_template_fixture, job_template_fixture, nil, true 155 | ) 156 | 157 | pipeline_file = subject.generate 158 | perform_assertion(expected_pipeline_yml_hash, pipeline_file) 159 | end 160 | 161 | it 'generates pipeline yml with group_per_branch set to false' do 162 | expected_pipeline_yml_hash.delete('groups') 163 | 164 | common_resource_fixture = File.expand_path( 165 | '../../examples/templates/my-repo-common-resources-template.yml.erb', __FILE__ 166 | ) 167 | 168 | job_template_fixture = File.expand_path( 169 | '../../examples/templates/my-repo-branch-job-template.yml.erb', __FILE__ 170 | ) 171 | subject = Cbm::PipelineGenerator.new( 172 | uri, 173 | branches, 174 | resource_template_fixture, 175 | job_template_fixture, 176 | common_resource_fixture, 177 | false 178 | ) 179 | 180 | pipeline_file = subject.generate 181 | perform_assertion(expected_pipeline_yml_hash, pipeline_file) 182 | end 183 | 184 | def perform_assertion(expected_pipeline_yml_hash, pipeline_file) 185 | pipeline_yml = File.read(pipeline_file) 186 | 187 | # convert concourse pipeline param delimiters to strings so we can compare 188 | # as a hash for this test 189 | pipeline_yml.gsub!( 190 | '{{EXAMPLE_LOAD_VARS_FROM_CONFIG_KEY}}', 191 | '"{{EXAMPLE_LOAD_VARS_FROM_CONFIG_KEY}}"') 192 | pipeline_yml.gsub!( 193 | '{{EXAMPLE_LOAD_VARS_FROM_CREDENTIALS_KEY}}', 194 | '"{{EXAMPLE_LOAD_VARS_FROM_CREDENTIALS_KEY}}"') 195 | 196 | pipeline_yml_hash = YAML.load(pipeline_yml) 197 | 198 | expect(pipeline_yml_hash).to eq(expected_pipeline_yml_hash) 199 | end 200 | end 201 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # UPDATE 2 | 3 | This tool and repo is no longer used nor maintained by pivotaltracker. If anyone wishes to take over ownership 4 | and maintenance, please open an issue. 5 | 6 | # UPDATE 2 7 | 8 | The new [work on spatial resources in Concourse](https://github.com/concourse/concourse/issues/2292) should hopefully 9 | eliminate the need for the "pipelines creating pipelines" approach in this repo. 10 | 11 | # Concourse Branch Manager 12 | 13 | Automatically build arbitrary branches on [Concourse CI](http://concourse.ci/) without relying on Github pull requests. 14 | 15 | See and vote for [this issue](https://github.com/concourse/concourse/issues/239) to get official support for building arbitrary branches without pull requests added to Concourse. 16 | 17 | ![Branches](https://cdn.rawgit.com/pivotaltracker/concourse-branch-manager/master/branches.svg) 18 | 19 | ## Overview 20 | 21 | This is a Concourse build task to find all existing branch names which match selected (1) criteria. 22 | 23 | Then, a `branch-manager` pipeline will be dynamically created/updated, 24 | using the concourse Fly CLI to create it "on the fly" (no pun intended), 25 | which will contain for a job + build plan for each branch, based on 26 | YAML ERB templates (2). 27 | 28 | It is intended to be used with the 29 | [Concourse git-branches-resource](https://github.com/pivotaltracker/git-branches-resource) (3) 30 | which is published on Docker Hub at 31 | [tracker/git-branches-resource](https://hub.docker.com/r/tracker/git-branches-resource/) 32 | to determine when and which branches should be built. However, any resource that 33 | fulfills the same input contract at the git-branches-resource can be used. 34 | **NOTE: That there is now an 35 | [identically named git-branches-resource](https://github.com/vito/git-branches-resource) 36 | which is published on Docker Hub at 37 | [cfcommunity/git-branches-resource](https://hub.docker.com/r/cfcommunity/git-branches-resource/), 38 | however, this one is does NOT currently have some of the features which 39 | [tracker/git-branches-resource](https://hub.docker.com/r/tracker/git-branches-resource/) 40 | has, such as regex matching and a max branches limit. The rest of this 41 | documentation assumes you are using the latter. Caveat Emptor.** 42 | 43 | * (1) The "selected" branches can be based a regex config option passed 44 | to the git-branches-resource (3) or other resource which fulfills the 45 | resource 'input' contract to write out a `git-branches.json` containing 46 | a hash with a `uri` to the repo, and an array of `branches`. 47 | This list of branches can be controlled with a regex 48 | (e.g. all branches starting with a string like feature-, release-candidate-, etc), 49 | and also limit the maximum number of branches listed. See the 50 | [Concourse git-branches-resource](https://github.com/pivotaltracker/git-branches-resource) 51 | documentation for more details. ***NOTE:*** *In the future, support will be added 52 | for automatically building branches for GitHub pull requests, via this same 53 | `git-branches.json` input resource interface.* 54 | 55 | * (2) the specified resource and job YAML ERB templates can contain whatever job/plan 56 | processing is needed, and the name of the dynamically-selected git branch will be passed as a param, 57 | to be interpolated into the YAML via ERB. 58 | 59 | ## How it works 60 | 61 | * In your Concourse pipeline, you will add a git resource and a job which will run 62 | Concourse Branch Manager, and specify the necessary parameters, including credentials 63 | to manage your Concourse instance. 64 | * When the job runs its tasks, it will dynamically create/update a new pipeline which will 65 | contain resources and jobs for all your dynamically processed branches. 66 | * The resources and jobs/plans/tasks which are automatically created in the pipeline 67 | are configurable to do whatever is needed for your particular situation and 68 | build/deployment environment. 69 | 70 | ## Setup and Usage 71 | 72 | ### 0. Enable Concourse authentication 73 | 74 | * The branch manager will not work with non-authenticating (development mode) Concourse instances. Any password based authentication mecanism will do (Basic, OAuth...) 75 | 76 | ### 1. Edit and update your Concourse pipeline to add the three required concourse-branch-manager resources 77 | 78 | * Add the following resources to your Concourse pipeline YAML file: 79 | 80 | ```yaml 81 | - name: concourse-branch-manager 82 | type: git 83 | source: 84 | uri: https://github.com/pivotaltracker/concourse-branch-manager.git 85 | branch: master 86 | ignore_paths: [Gemfile, Gemfile.lock] 87 | 88 | # This `git-branches` input resource determines which branches will be processed 89 | - name: branch-manager-git-branches 90 | type: git-branches 91 | source: 92 | # Set this to the uri of your repo for which you want to dynamically build arbitrary branches 93 | uri: https://github.com/mygithubuser/my-repo 94 | branch_regexp: ".*" 95 | max_branches: 20 96 | 97 | # This repo containing your resource/job templates can be the same repo as 98 | # the one in the git-branches resource above, but it doesn't have to be 99 | - name: branch-manager-templates 100 | type: git 101 | source: 102 | uri: https://github.com/mygithubuser/my-template-repo 103 | branch: master 104 | paths: [ci/templates/*] 105 | 106 | # This repo contains any non-secret non-credential config that needs to be 107 | # passed into your generated pipeline via fly --load-vars-from options 108 | - name: branch-manager-config 109 | type: git 110 | source: 111 | # Set this to the uri of your repo containing your non-secret non-credential config 112 | uri: https://github.com/pivotaltracker/concourse-branch-manager.git 113 | branch: master 114 | # paths: [ci/config/*] 115 | 116 | # This repo contains any secret credential config that needs to be 117 | # passed into your generated pipeline via fly --load-vars-from options 118 | - name: branch-manager-credentials 119 | type: git 120 | source: 121 | # Set this to the uri of your repo containing your non-secret non-credential config 122 | uri: https://github.com/pivotaltracker/concourse-branch-manager.git 123 | branch: master 124 | # paths: [ci/credentials/*] 125 | ``` 126 | 127 | * The `concourse-branch-manager` resource will always point to the `concourse-branch-manager` 128 | public repo on github, and should look exactly as the above example. This is where the logic 129 | for the branch-building task lives. 130 | 131 | * Set the `uri` of the `branch-manager-git-branches` resource with the uri of your 132 | repo for which you want to dynamically build arbitrary branches. 133 | 134 | * Set the `uri` of the `branch-manager-templates` resource with the uri of your 135 | your `git` resource containing your resource/job templates for building branches. NOTE: This 136 | may both point to the same git repo as your `git-branches` resource, but it doesn't have to. 137 | 138 | ### 2. Edit and update your Concourse pipeline to add the branch-manager job: 139 | 140 | * Add the following job to your Concourse pipeline YAML file: 141 | 142 | ```yaml 143 | - name: branch-manager 144 | serial: true 145 | plan: 146 | - get: concourse-branch-manager 147 | params: {depth: 20} 148 | trigger: true 149 | - get: git-branches 150 | resource: branch-manager-git-branches 151 | trigger: true 152 | - get: template-repo 153 | resource: branch-manager-templates 154 | params: {depth: 20} 155 | trigger: true 156 | - get: config-repo 157 | resource: branch-manager-config 158 | params: {depth: 20} 159 | trigger: true 160 | - get: credentials-repo 161 | resource: branch-manager-credentials 162 | params: {depth: 20} 163 | trigger: true 164 | - task: manage-branches 165 | file: concourse-branch-manager/tasks/manage-branches.yml 166 | config: 167 | params: 168 | BRANCH_RESOURCE_TEMPLATE: template-repo/ci/templates/my-repo-branch-resource-template.yml.erb 169 | BRANCH_JOB_TEMPLATE: template-repo/ci/templates/my-repo-branch-job-template.yml.erb 170 | CONCOURSE_URL: {{CONCOURSE_URL}} 171 | CONCOURSE_USERNAME: {{CONCOURSE_USERNAME}} 172 | CONCOURSE_PASSWORD: {{CONCOURSE_PASSWORD}} 173 | CONCOURSE_TEAM: {{CONCOURSE_TEAM}} # Add this if you have a team 174 | ``` 175 | 176 | You may specify the `CONCOURSE_*` params directly in your pipeline YAML file, but 177 | since they are sensitive credentials, you should handle them via Concourse's 178 | support for [template variables](http://concourse.ci/fly-cli.html#parameters). 179 | 180 | If you're using the [teams feature](http://concourse.ci/teams.html) and want to add the pipeline to a team other than `main`, specify it in the `CONCOURSE_TEAM` params. If this params is omitted, no team will be specified. 181 | 182 | There are several params available to configure the templates. Each is a path to a file containing an ERB template. 183 | 184 | * `BRANCH_RESOURCE_TEMPLATE` - resources to be added to each branch 185 | * `BRANCH_JOB_TEMPLATE` - jobs to be added to each branch 186 | * `PIPELINE_COMMON_RESOURCES_TEMPLATE` - resources which will be added only once to the pipeline 187 | * `PIPELINE_RESOURCE_TYPE_TEMPLATE` - [resource _types_](https://concourse.ci/configuring-resource-types.html) to be added to the pipeline 188 | 189 | These templates can 190 | live in your managed repo, but they don't have to - you could add an additional 191 | resource to the `branch-manager` job to contain them. More details on this below... 192 | 193 | ***TODO: Document PIPELINE_LOAD_VARS_FROM_N params*** 194 | ***TODO: Document PIPELINE_NAME param*** 195 | ***TODO: Document GROUP_PER_BRANCH param*** 196 | 197 | ### 3. Edit and update your Concourse pipeline to add the branch-manager group (optional): 198 | 199 | ```yaml 200 | - name: branch-manager 201 | jobs: 202 | - branch-manager 203 | ``` 204 | 205 | ### 4. Add the git-branch resource type to your pipeline (optional) 206 | 207 | ```yaml 208 | resource_types: 209 | - name: git-branches 210 | type: docker-image 211 | source: 212 | repository: tracker/git-branches-resource 213 | ``` 214 | 215 | ### 5. Create a YAML ERB templates for your resource and job which will be run for each branch 216 | 217 | Each arbitrary branch which is dynamically detected will have a Concourse 218 | [resource]() automatically created for it, and a Concourse 219 | [job](http://concourse.ci/configuring-jobs.html), [build plan](http://concourse.ci/build-plans.html), 220 | and [task](http://concourse.ci/task-step.html) will also be automatically created to process it. 221 | 222 | You have control over what this resource and job do by providing 223 | [ERB](http://apidock.com/ruby/ERB) templates which will be used to build 224 | separate resource and job entries in the generated pipeline for each branch 225 | which is processed. 226 | 227 | The path to your ERB templates is specified in the `BRANCH_RESOURCE_TEMPLATE` and 228 | `BRANCH_JOB_TEMPLATE` parameters, as documented above. They are paths 229 | to ERB templates which will be used to dynamically generate a resource and 230 | job for each of your branches. These templates can 231 | live in your managed repo, but they don't have to - you could add an additional 232 | resource to the `branch-manager` job to contain them. 233 | 234 | The only requirement for these ERB templates is that they use ERB to interpolate 235 | the `uri` and `branch_name` variables, which are automatically set to contain the 236 | repo uri and name of the branch which was automatically detected and processed, 237 | and for which builds should be triggered. 238 | 239 | And, of course, you must create and properly reference a 240 | [task configuration and associated script](http://concourse.ci/running-tasks.html#configuring-tasks), 241 | as well as any additional resources you need, 242 | to do the actual work of your job. This can be whatever you want - using a 243 | Concourse git or s3 resource output to push a branch or s3 artifact, which 244 | can then be used by other hardcoded jobs in your pipeline. Here is an example 245 | [task configuration](https://github.com/pivotaltracker/concourse-branch-manager/blob/master/examples/tasks/my-repo-branch-task.yml) and 246 | [script](https://github.com/pivotaltracker/concourse-branch-manager/blob/master/examples/tasks/my-repo-branch-task-script) 247 | from the concourse-branch-manager project itself. 248 | 249 | Here is an example branch resource template (this is an 250 | [actual example from the concourse-branch-manager project itself](https://github.com/pivotaltracker/concourse-branch-manager/blob/master/examples/templates/my-repo-branch-resource-template.yml.erb)): 251 | 252 | ```yaml 253 | name: my-repo-branch-<%= branch_name %> 254 | type: git 255 | source: 256 | uri: <%= uri %> 257 | branch: <%= branch_name %> 258 | ``` 259 | 260 | And here is an example branch job template (this is an 261 | [actual example from the concourse-branch-manager project itself](https://github.com/pivotaltracker/concourse-branch-manager/blob/master/examples/templates/my-repo-branch-job-template.yml.erb)): 262 | 263 | ```yaml 264 | name: my-repo-branch-job-<%= branch_name %> 265 | plan: 266 | - get: my-repo-branch 267 | resource: my-repo-branch-<%= branch_name %> 268 | params: {depth: 20} 269 | trigger: true 270 | - task: my-repo-branch-task 271 | file: my-repo-branch/ci/tasks/my-repo-branch-task.yml 272 | config: 273 | params: 274 | BRANCH_NAME: <%= branch_name %> 275 | ``` 276 | 277 | ### 6. Create/Update the Concourse pipeline 278 | 279 | * Update your Concourse pipeline with the new resources and job 280 | using the [`fly set-pipeline`](http://concourse.ci/fly-cli.html#fly-set-pipeline) 281 | command. 282 | 283 | ## Alternate configuration options 284 | 285 | The implementation above is flexible. Based on your situation, you can any combination of one or more 286 | pipelines, `git-branches` resources, template `git` resources, `branch-manager` jobs, resource templates, 287 | and job templates. 288 | 289 | For example, you can have one pipeline per branch, one pipeline with multiple branches, multiple 290 | pipelines with multiple branches, multiple jobs with different templates using different resources - 291 | whatever works for you. 292 | 293 | The resource naming is also flexible to allow you to have multiple `git-branches` or template `git` 294 | resources in the same pipeline YAML file. The only requirement is that the `get` entries in your 295 | task must be named `concourse-branch-manager`, `git-branches`, and `template-repo`, since these 296 | are referred to by the task branch-building logic. 297 | 298 | ## Live Example 299 | 300 | There is a working example pipeline which points to 301 | [example templates and a dummy task in the concourse-branch-manager project repo](https://github.com/pivotaltracker/concourse-branch-manager/blob/master/examples)) 302 | 303 | To try it out yourself: 304 | 305 | 1. (at the beginning of the day) Login to fly and save credentials in a `ci` target: 306 | 307 | ``` 308 | fly login --target=ci --concourse-url=https://my-concourse-server 309 | ``` 310 | 311 | 2. Clone a local copy of the 312 | [`branch-manager-example` repo](https://github.com/pivotaltracker/concourse-branch-manager). 313 | Note that this has all the Concourse config in an `examples` directory, but in your actual projects 314 | you may want to want to put this in a `ci` directory by convention. 315 | 316 | 3. Create a `secrets.yml` file containing the vars specifying the uri and credentials of your Concourse 317 | server. Put this some where secret and don't check it in! For convenience, `secrets.yml` in the root 318 | of this repo is gitignored, so you can put it there if you want: 319 | 320 | ``` 321 | # secrets.yml 322 | CONCOURSE_URL: https://my-concourse-server.example.com 323 | CONCOURSE_USERNAME: my-basic-auth-concourse-username 324 | CONCOURSE_PASSWORD: my-basic-auth-concourse-password 325 | ``` 326 | 327 | 4. Use the `fly set-pipeline` command to create/update the `branch-manager-example` pipeline: 328 | 329 | ``` 330 | fly --target=ci set-pipeline --config=examples/pipelines/branch-manager-example.yml --load-vars-from=secrets.yml --pipeline=branch-manager-example 331 | ``` 332 | 333 | 5. (first time only) Use the `fly unpause-pipeline` commmand to unpause the pipeline the first 334 | time it is created (or just click unpause in the Concourse UI): 335 | 336 | ``` 337 | fly --target=ci unpause-pipeline --pipeline=branch-manager-example 338 | ``` 339 | 340 | 6. Go to your Concourse UI (e.g. `https://my-concourse-server.example.com`), and select 341 | the `branch-manager-example` pipeline from the upper-left hamburger menu. 342 | 343 | 7. It should successfully build automatically in a few seconds and go green 344 | 345 | 8. Refresh the page, and select the autocreated `branch-manager` pipeline from the upper-left 346 | hamburger menu. It should contain some of the dummy branches on the `concourse-branch-manager` 347 | repo. They should also all successfully build after a few seconds. 348 | 349 | That's it! Use this as a template and example for using concourse-branch-manager in your 350 | own project! 351 | 352 | ## Dealing with Concourse UI issues due to many branches/groups 353 | 354 | * It is recommended that you keep the number of branches limited. The git-branches-resource 355 | resource by default limits you to 20, but the Concourse UI starts having layout issues 356 | with fewer than that. The biggest problem is z-index issues that prevent usage of the 357 | pipelines menu. 358 | * [This issue against Concourse ATC](https://github.com/concourse/atc/issues/39) 359 | (the Concourse UI) reports these issues, and contains a javascript bookmarklet 360 | that makes some hacks to fix the issues. The recommended fix by the concourse team 361 | would be to make the groups scrollable like the build numbers at the top of a build 362 | page. A pull request would be welcome ;) 363 | -------------------------------------------------------------------------------- /branches.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | 270 | 271 | 272 | 273 | 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | 282 | 283 | 284 | 285 | 286 | 287 | 288 | 289 | 290 | 291 | 292 | 293 | 294 | 295 | 296 | 297 | 298 | 299 | 300 | 301 | 302 | 303 | 304 | 305 | 306 | 307 | 308 | 309 | 310 | 311 | 312 | 313 | 314 | 315 | 316 | 317 | 318 | 319 | 320 | 321 | 322 | 323 | 324 | 325 | 326 | 327 | 328 | 329 | 330 | 331 | 332 | 333 | 334 | 335 | 336 | 337 | 338 | 339 | --------------------------------------------------------------------------------