├── README.md
├── lib
    └── puppet
    │   ├── application
    │       └── partial.rb
    │   └── face
    │       └── partial.rb
└── manifests
    ├── blank.pp
    ├── init.pp
    ├── mirror.pp
    ├── rpm.pp
    └── rpmbuild.pp


/README.md:
--------------------------------------------------------------------------------
 1 | puppet-partial
 2 | ==============
 3 | 
 4 | Puppet Face for building useful partial catalogs and applying them
 5 | 
 6 | Version 0.0.1
 7 | 
 8 | ## Overview
 9 | 
10 | The puppet-partial module is used to compile an existing puppet catalog and pull
11 | out the parts that belong in golden images.
12 | 
13 | It can also create a local repo of all packages + package deps that are needed to
14 | install a given catalog.
15 | 
16 | ## Usage
17 | 
18 | the face is called partial, and the action for golden images is image_build:
19 | 
20 |     puppet partial image_build <nodename>
21 | 
22 | the action for building a repo is
23 | 
24 |     puppet partial repo_build <role_name>
25 | 
26 | the action of listing packages managed by a catalog:
27 |     puppet partial resource_list --resource package <nodename>
28 | 
29 | the action of listing services managed by a catalog:
30 |     puppet partial resource_list --resource service <nodename>
31 | 
32 | You can use resource_list to list any type of Puppet resource.
33 | 
34 | where role name will set a fact 'role' => <role_name> that can be used to select
35 | which role to build for. To get the most out of this now, create a role that simply
36 | includes every profile, so that all packages are downloaded.
37 | 
38 | TODO: add support for a list of roles, so that we don't have to make an all role.
39 | 
40 | ## Limitations
41 | 
42 | All the image_build action currently does is pull out resources of the type 'package' and 'yumrepo'
43 | and separate them with an anchor.
44 | 
45 | ## License
46 | Copyright (C) 2014 Aptira and Authors
47 | 
48 | Original Author: Michael Chapman
49 | 
50 | Aptira can be contacted at info@aptira.com
51 | 
52 | 
53 | Licensed under the Apache License, Version 2.0 (the "License");
54 | you may not use this file except in compliance with the License.
55 | You may obtain a copy of the License at
56 | 
57 |     http://www.apache.org/licenses/LICENSE-2.0
58 | 
59 |     Unless required by applicable law or agreed to in writing, software
60 |     distributed under the License is distributed on an "AS IS" BASIS,
61 |     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
62 |     See the License for the specific language governing permissions and
63 |     limitations under the License.
64 | 


--------------------------------------------------------------------------------
/lib/puppet/application/partial.rb:
--------------------------------------------------------------------------------
1 | require 'puppet/face'
2 | require 'puppet/application/indirection_base'
3 | 
4 | class Puppet::Application::Partial < Puppet::Application::IndirectionBase
5 |   #option("--outfile", "-o") do |arg|
6 |   #  options[:outfile] = arg
7 |   #end
8 | end
9 | 


--------------------------------------------------------------------------------
/lib/puppet/face/partial.rb:
--------------------------------------------------------------------------------
  1 | # Select and show a list of resources of a given type.
  2 | Puppet::Face.define(:partial, '0.0.1') do
  3 |   action :repo_build do
  4 |     summary "Retrieve a catalog for a given role, filter it for resources like packages and repos, and create a local yum repository to serve"
  5 |     arguments "<role>"
  6 | 
  7 |     returns <<-'EOT'
  8 |       "Applies a catalog and doesn't return anything of note"
  9 |     EOT
 10 |     option "--repo_path REPO_PATH" do
 11 |       summary "The path to place package files in"
 12 |     end
 13 | 
 14 |     option "--installed INSTALLED" do
 15 |       summary "Whether to also load installed packages from this node into the mirror"
 16 |     end
 17 |     when_invoked do |role, options|
 18 |       facts = Puppet::Face[:facts, :current].find('node')
 19 |       facts.values['role'] = role
 20 | 
 21 |       node = Puppet::Node.new('role', options={:parameters => facts.values})
 22 | 
 23 |       catalog = Puppet::Resource::Catalog.indirection.find('role', options = {:use_node => node})
 24 | 
 25 |       if options.has_key? :repo_path
 26 |         path = options[:repo_path]
 27 |       else
 28 |         path = '/usr/share/yumrepo'
 29 |       end
 30 | 
 31 |       if options.has_key? :installed
 32 |         installed = options[:installed]
 33 |       else
 34 |         installed = true
 35 |       end
 36 | 
 37 |       tcat = Puppet::Resource::Catalog.new('test', Puppet::Node::Environment.new('production'))
 38 |       tcat.make_default_resources
 39 |       anchor = tcat.create_resource('anchor', {'title' => 'start'})
 40 |       anchor = tcat.create_resource('anchor', {'title' => 'repos'})
 41 | 
 42 |       tcat.create_resource('file', {'title' => path, 'ensure' => 'directory', 'before' => 'Package[yum-utils]' })
 43 |       tcat.create_resource('package', {'title' => 'yum-utils', 'ensure' => 'installed', 'require' => 'Anchor[repos]' })
 44 | 
 45 |       catalog.resources.each do |res|
 46 |         if res.type.downcase == 'package' then
 47 |           tcat.create_resource('exec', { 'title' => "exec_#{res.title}", 'path' => '/usr/bin:/bin:/usr/sbin:/sbin', 'timeout' => 0, 'command' => "repotrack -a x86_64 -p #{path} #{res['name']}", 'require' => 'Package[yum-utils]'})
 48 |         elsif res.type.downcase == 'yumrepo'
 49 |           newres = res.to_hash
 50 |           newres[:before] = 'Anchor[repos]'
 51 |           newres.delete(:notify)
 52 |           newres.delete(:require)
 53 |           newres[:title] = res.title
 54 |           puts newres
 55 |           tcat.create_resource('yumrepo', newres)
 56 |         end
 57 |       end
 58 |       tcat.finalize
 59 |       transaction = tcat.apply()
 60 |       return
 61 |     end
 62 |   end
 63 | 
 64 |   action :image_build do
 65 |     summary "Retrieve a catalog, filter it for image building resources like packages and repos, and apply it"
 66 | 
 67 |     arguments "<host>"
 68 | 
 69 |     returns <<-'EOT'
 70 |       A puppet manifest containing the package and repository resources separated
 71 |       by an anchor.
 72 |     EOT
 73 | 
 74 |     description <<-'EOT'
 75 |     EOT
 76 | 
 77 |     notes <<-'NOTES'
 78 |       Work in progress as a packer provider
 79 |     NOTES
 80 | 
 81 |     examples <<-'EOT'
 82 |       Compile a catalog and select the resources for image building for the node compute1
 83 |       and output a simplified manifest to /root/image.pp
 84 | 
 85 |       $ puppet partial image_build somenode.magpie.lan --outfile=/root/image.pp
 86 |     EOT
 87 | 
 88 |     when_invoked do |host, options|
 89 |       catalog = Puppet::Resource::Catalog.indirection.find(host)
 90 | 
 91 |       tcat = Puppet::Resource::Catalog.new('test', Puppet::Node::Environment.new('production'))
 92 |       tcat.make_default_resources
 93 |       anchor = tcat.create_resource('anchor', {'title' => 'break'})
 94 | 
 95 |       catalog.resources.each do |res|
 96 |         if res.type.downcase == 'package' then
 97 |           tcat.create_resource('package', {'title' => res['name'], 'require' => 'Anchor[break]'})
 98 |         elsif res.type.downcase == 'yumrepo'
 99 |           tcat.create_resource('yumrepo', {'title' => res.title, 'name' => res['name'], 'baseurl' => res['baseurl'], 'before' => 'Anchor[break]' })
100 |         end
101 |       end
102 |       tcat.finalize
103 |       transaction = tcat.apply()
104 |       return
105 |     end
106 |   end
107 | 
108 |   action :resource_list do
109 |     summary "Retrieve a catalog, filter resource list and create a list."
110 | 
111 |     arguments "<hosts>"
112 | 
113 |     option "--resource <package,service,file,(...)>" do
114 |       summary "List the resources on a specific type"
115 |     end
116 | 
117 |     option "--tag <tag>" do
118 |       summary "List the resources on a specific tag"
119 |     end
120 | 
121 |     returns <<-'EOT'
122 |       A list containing the resources.
123 |     EOT
124 | 
125 |     description <<-'EOT'
126 |     EOT
127 | 
128 |     notes <<-'NOTES'
129 |     NOTES
130 | 
131 |     examples <<-'EOT'
132 |       Compile a catalog and select the resources managed by Puppet on one node.
133 | 
134 |       $ puppet partial resource_list somenode.magpie.lan
135 |       $ puppet partial resource_list --tag neutron::server --resource package somenode.magpie.lan
136 |     EOT
137 | 
138 |     when_invoked do |host, options|
139 |       catalog = Puppet::Resource::Catalog.indirection.find(host)
140 | 
141 |       tcat = Puppet::Resource::Catalog.new('test', Puppet::Node::Environment.new('production'))
142 |       tcat.make_default_resources
143 | 
144 |       catalog.resources.each do |res|
145 |         resource = options[:resource]
146 |         if options.has_key? :tag
147 |           tag = options[:tag]
148 |           if res.type.downcase == resource and res.tags.include?(tag) then
149 |             puts "#{res['name']}"
150 |           end
151 |         else
152 |           if res.type.downcase == resource then
153 |             puts "#{res['name']}"
154 |           end
155 |         end
156 |       end
157 |       return
158 |     end
159 |   end
160 | 
161 | end
162 | 


--------------------------------------------------------------------------------
/manifests/blank.pp:
--------------------------------------------------------------------------------
1 | class partial::blank(
2 |  $param = 'test'
3 | ) {
4 |  notice($param)
5 | }
6 | 


--------------------------------------------------------------------------------
/manifests/init.pp:
--------------------------------------------------------------------------------
 1 | class partial(
 2 |   $server_name = $::fqdn,
 3 |   $repo_path   = '/usr/share/yumrepo'
 4 | )
 5 | {
 6 |   include ::apache
 7 |   include ::apache::mod::autoindex
 8 |   include ::apache::mod::dir
 9 | 
10 |   apache::vhost { "${server_name}":
11 |     port           => '80',
12 |     docroot        => $repo_path,
13 |     directoryindex => 'index.html'
14 |   }
15 | 
16 |   firewall { '100 accept all tcp 80 for apache':
17 |     proto => 'tcp',
18 |     action => 'accept',
19 |     port => [80, 443]
20 |   }
21 | 
22 |   package { 'createrepo':
23 |     ensure => 'installed',
24 |   }
25 |   
26 |   file  { $repo_path:
27 |     ensure => directory 
28 |   } ->
29 | 
30 |   exec { 'vagrant_rpm_cache':
31 |     command => "bash /vagrant/provision/rpmcache.sh /vagrant/rpmcache ${repo_path}",
32 |     onlyif  => "file -f /vagrant/provision/rpmcache.sh && file -f /vagrant/rpmcache",
33 |     path    => ['/usr/bin', '/usr/local/bin','/usr/sbin','/sbin' ],
34 |     timeout => 0,
35 |     before  => Exec['create_yum_repo'],
36 |   }
37 | 
38 |   exec { 'create_yum_repo':
39 |     command => "rm -rf ${repo_path}/repodata; createrepo ${repo_path}",
40 |     path    => ['/usr/bin', '/usr/local/bin','/usr/sbin','/sbin' ],
41 |     timeout => 0,
42 |     require => [Package['createrepo'], File[$repo_path]]
43 |   }
44 | }
45 | 


--------------------------------------------------------------------------------
/manifests/mirror.pp:
--------------------------------------------------------------------------------
 1 | class partial::mirror(
 2 |   $all_role          = 'all',
 3 |   $repo_path         = '/usr/share/yumrepo',
 4 |   $upstream_cache    = false,
 5 |   $mirror_installed  = true
 6 | )
 7 | {
 8 |   package { 'wget':
 9 |     ensure => 'installed'
10 |   }
11 | 
12 |   # pull from a local copy to speed things up if possible
13 |   if $upstream_cache {
14 |     exec { 'pull_from_upstream_cache':
15 |       command => "cd ${repo_path}; wget -r -nH --cut-dirs=2 --no-parent --reject=\"index.html*\" http://${upstream_cache} || true",
16 |       path    => ['/usr/bin', '/usr/local/bin','/usr/sbin','/sbin' ],
17 |       before  => Exec['build_repo'],
18 |       timeout => 0,
19 |       creates => '/usr/share/yumrepo/repodata',
20 |       require => Package['wget']
21 |     }
22 |   }
23 | 
24 |   # this can take a very long time
25 | 
26 |   Exec<| title == 'vagrant_rpm_cache' |> -> Exec <| title == 'build_repo' |>
27 |   Exec<| title == 'vagrant_rpm_cache' |> -> Exec <| title == 'build_installed_repo' |>
28 | 
29 |   Yumrepo<||> ->
30 |   exec { 'build_repo':
31 |     command => "puppet partial repo_build --repo_path=${repo_path} all",
32 |     path    => ['/usr/bin', '/usr/local/bin','/usr/sbin','/sbin' ],
33 |     timeout => 0
34 |   }
35 | 
36 |   if $mirror_installed {
37 |     exec { 'build_installed_repo':
38 |       command => "ret=1; for i in `puppet resource package | grep package | cut -d \"'\" -f 2`; do repotrack -a x86_64 -p ${repo_path} \$i; done;",
39 |       path    => ['/usr/bin', '/usr/local/bin','/usr/sbin','/sbin' ],
40 |       timeout => 0,
41 |       provider => shell
42 |     }
43 |     Exec['build_installed_repo'] ~> Exec<| title == 'create_yum_repo' |>
44 |   }
45 | 
46 |   Exec['build_repo'] ~> Exec<| title == 'create_yum_repo' |>
47 | }
48 | 


--------------------------------------------------------------------------------
/manifests/rpm.pp:
--------------------------------------------------------------------------------
 1 | class partial::rpm(
 2 |   $rpms = {}
 3 | )
 4 | {
 5 |   package { ['git', 'gcc', 'rpmdevtools', 'mock']:
 6 |     ensure => installed
 7 |   }
 8 |   notice($rpms)
 9 |   create_resources('partial::rpmbuild', $rpms )
10 | }
11 | 


--------------------------------------------------------------------------------
/manifests/rpmbuild.pp:
--------------------------------------------------------------------------------
 1 | define partial::rpmbuild(
 2 |   $repo_source,
 3 |   $build_command,
 4 |   $repo_provider = 'git',
 5 |   $repo_revision = 'master',
 6 |   $build_environment = 'HOME=/tmp',
 7 |   $mirror_path = '/usr/share/yumrepo',
 8 |   $install_rpm = false,
 9 | )
10 | {
11 | 
12 |   notice($title)
13 |   notice($name)
14 | 
15 |   if $install_rpm {
16 |     $install_string = 'yum install -y /tmp/*.rpm; '
17 |   } else {
18 |     $install_string = ''
19 |   }
20 | 
21 |   vcsrepo { "/tmp/${title}":
22 |     ensure   => present,
23 |     provider => $repo_provider,
24 |     revision => $repo_revision,
25 |     source   => $repo_source,
26 |   } ~>
27 |   exec { "build_rpm_${title}":
28 |     path          => '/usr/bin:/bin:/usr/sbin:/sbin',
29 |     provider      => shell,
30 |     command       => "cd /tmp; ${build_command}; cp /tmp/*.rpm ${mirror_path}; ${install_string}",
31 |     refreshonly   => true,
32 |     environment   => $build_environment
33 |   }
34 | 
35 |   Exec["build_rpm_${title}"] ~> Exec<| title == 'create_yum_repo' |>
36 | }
37 | 


--------------------------------------------------------------------------------