├── .gitignore ├── .rspec ├── Gemfile ├── LICENSE ├── NOTICE ├── README.md ├── Rakefile ├── bin └── buildpack-packager ├── buildpack-packager.gemspec ├── doc └── disconnected_environments.md ├── lib ├── buildpack │ ├── manifest_dependency.rb │ ├── manifest_validator.rb │ ├── packager.rb │ └── packager │ │ ├── default_versions_presenter.rb │ │ ├── dependencies_presenter.rb │ │ ├── manifest_schema.yml │ │ ├── package.rb │ │ ├── table_presentation.rb │ │ ├── version.rb │ │ └── zip_file_excluder.rb └── kwalify │ └── parser │ └── yaml-patcher.rb └── spec ├── buildpack └── packager_spec.rb ├── fixtures ├── buildpack-with-uri-credentials │ ├── VERSION │ └── manifest.yml ├── buildpack-without-uri-credentials │ ├── VERSION │ └── manifest.yml └── manifests │ ├── manifest_invalid-sha224.yml │ ├── manifest_invalid-sha224_and_defaults.yml │ ├── manifest_opensus.yml │ ├── manifest_valid.yml │ ├── manifest_valid_new_deprecation_dates_no_url_map.yml │ ├── manifest_valid_plus_deprecation_dates.yml │ └── manifest_windows.yml ├── helpers ├── cache_directory_helpers.rb ├── fake_binary_hosting_helpers.rb └── file_system_helpers.rb ├── integration ├── bin │ ├── buildpack_packager │ │ └── download_caching_spec.rb │ └── buildpack_packager_spec.rb ├── buildpack │ ├── directory_name_spec.rb │ └── packager_spec.rb ├── default_versions_spec.rb └── output_spec.rb ├── spec_helper.rb └── unit ├── buildpack └── packager │ └── zip_file_excluder_spec.rb ├── manifest_dependency_spec.rb ├── manifest_validator_spec.rb └── packager └── package_spec.rb /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /Gemfile.lock 4 | /_yardoc/ 5 | /coverage/ 6 | /pkg/ 7 | /spec/reports/ 8 | /spec/**/*.zip 9 | /tmp/ 10 | *.bundle 11 | *.so 12 | *.o 13 | *.a 14 | mkmf.log 15 | .idea/ 16 | vendor/ 17 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --format documentation 2 | --color 3 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in buildpack-packager.gemspec 4 | gemspec 5 | gem 'pry' 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | buildpack-packager 2 | 3 | Copyright (c) 2013-Present CloudFoundry.org Foundation, Inc. All Rights Reserved. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Deprecated 2 | 3 | This repo has been deprecated in favor of https://github.com/cloudfoundry/libbuildpack/tree/master/packager/buildpack-packager 4 | 5 | # Buildpack Packager 6 | 7 | A [RubyGem](https://rubygems.org/) that provides tooling to package a buildpack for upload to Cloud Foundry. 8 | 9 | For officially supported Cloud Foundry buildpacks, it is used in conjunction with [compile-extensions](https://github.com/cloudfoundry-incubator/compile-extensions). 10 | 11 | 12 | # Usage 13 | 14 | ## Packaging a Buildpack 15 | 16 | Within your buildpack directory: 17 | 18 | 1. Create a `Gemfile` and the line `gem 'buildpack-packager', git: 'https://github.com/cloudfoundry-incubator/buildpack-packager'`. 19 | 1. Run `bundle install`. 20 | 1. Create a `manifest.yml` 21 | 1. Run the packager for uncached or cached mode: 22 | 23 | ``` 24 | buildpack-packager [cached|uncached] 25 | ``` 26 | 27 | In either mode, the packager will add (almost) everything in your 28 | buildpack directory into a zip file. It will exclude anything marked 29 | for exclusion in your manifest. 30 | 31 | In `cached` mode, the packager will download and add dependencies as 32 | described in the manifest. 33 | 34 | 35 | ## Examining a manifest 36 | 37 | If you'd like to get a pretty-printed summary of what's in a manifest, 38 | run in `list` mode: 39 | 40 | ``` 41 | buildpack-packager list 42 | ``` 43 | 44 | Example output: 45 | 46 | ``` 47 | +----------------------------+------------------------------+------------+ 48 | | name | version | cf_stacks | 49 | +----------------------------+------------------------------+------------+ 50 | | ruby | 2.0.0 | cflinuxfs2 | 51 | | ruby | 2.1.5 | cflinuxfs2 | 52 | | ruby | 2.1.6 | cflinuxfs2 | 53 | | ruby | 2.2.1 | cflinuxfs2 | 54 | | ruby | 2.2.2 | cflinuxfs2 | 55 | | jruby | ruby-1.9.3-jruby-1.7.19 | cflinuxfs2 | 56 | | jruby | ruby-2.0.0-jruby-1.7.19 | cflinuxfs2 | 57 | | jruby | ruby-2.2.2-jruby-9.0.0.0.rc1 | cflinuxfs2 | 58 | | node | 0.12.2 | cflinuxfs2 | 59 | | bundler | 1.9.7 | cflinuxfs2 | 60 | | libyaml | 0.1.6 | cflinuxfs2 | 61 | | openjdk1.8-latest | - | cflinuxfs2 | 62 | | rails3_serve_static_assets | - | cflinuxfs2 | 63 | | rails_log_stdout | - | cflinuxfs2 | 64 | +----------------------------+------------------------------+------------+ 65 | ``` 66 | 67 | ### Option Flags 68 | 69 | #### --force-download 70 | 71 | By default, `buildpack-packager` stores the dependencies that it 72 | downloads while building a cached buildpack in a local cache at 73 | `~/.buildpack-packager`. This is in order to avoid redownloading them 74 | when repackaging similar buildpacks. Running `buildpack-packager 75 | cached` with the `--force-download` option will force the packager 76 | to download dependencies from the s3 host and ignore the local cache. 77 | 78 | #### --use-custom-manifest 79 | 80 | If you would like to include a different manifest file in your 81 | packaged buildpack, you may call `buildpack-packager` with the 82 | `--use-custom-manifest [path/to/manifest.yml]` 83 | option. `buildpack-packager` will generate a buildpack with the 84 | specified manifest. If you are building a cached buildpack, 85 | `buildpack-packager` will vendor dependencies from the specified 86 | manifest as well. 87 | 88 | 89 | # Manifest 90 | 91 | The packager looks for a `manifest.yml` file in the current working 92 | directory, which should be the root of your buildpack. 93 | 94 | A sample manifest (all keys (excepting dependency_deprecation_dates) are required): 95 | 96 | ```yaml 97 | --- 98 | language: ruby 99 | 100 | url_to_dependency_map: 101 | - match: bundler-(\d+\.\d+\.\d+) 102 | name: bundler 103 | version: $1 104 | - match: ruby-(\d+\.\d+\.\d+) 105 | name: ruby 106 | version: $1 107 | 108 | dependencies: 109 | - name: bundler 110 | version: 1.7.12 111 | uri: https://pivotal-buildpacks.s3.amazonaws.com/ruby/binaries/lucid64/bundler-1.7.12.tgz 112 | sha256: 09b15ac14f7b46ac6d0a85102cef4671d95f6d6581e01dbcdbab0e64df83c4d5 113 | cf_stacks: 114 | - lucid64 115 | - cflinuxfs2 116 | - name: ruby 117 | version: 2.1.4 118 | uri: https://pivotal-buildpacks.s3.amazonaws.com/ruby/binaries/lucid64/ruby-2.1.4.tgz 119 | sha256: e3e6023764357324260e9efab1f1690b7bcc0c69f82e8589797b262eb5df2831 120 | cf_stacks: 121 | - lucid64 122 | 123 | dependency_deprecation_dates: 124 | - match: 2.1.\\d 125 | version_line: 2.1 126 | name: ruby 127 | date: 2016-03-30 128 | 129 | exclude_files: 130 | - .gitignore 131 | - private.key 132 | ``` 133 | 134 | ## language (required) 135 | 136 | The language key is used to give your zip file a meaningful name. 137 | 138 | 139 | ## url_to_dependency_map (required) 140 | 141 | A list of regular expressions that extract and map the values of `name` and `version` to a corresponding dependency. 142 | 143 | 144 | ## dependencies (required) 145 | 146 | 147 | The dependencies key specifies the name, version, uri, sha256, and the 148 | cf_stacks (the root file system(s) for which it is compiled for) of a 149 | resource which the buildpack attempts to download during staging. By 150 | specifying them here, the packager can download them and install them 151 | into the `dependencies/` folder in the zip file. 152 | 153 | All keys are required: 154 | 155 | - `name`, `version`, and `uri`: 156 | Required for `url_to_dependency_map` to work. Make sure to create a new entry in the `url_to_dependency_map` if a matching regex does not exist for the dependency to be curled. 157 | 158 | - `sha256`: 159 | Required to ensure that dependencies being packaged for 'cached' mode have not been compromised 160 | 161 | - `cf_stacks`: 162 | Required to ensure the right binary is selected for the root file system in which an app will be running on. Currently supported root file systems are lucid64(default) and cflinuxfs2. *Note that if the same dependency is used for both root file systems, both can be listed under the `cf_stacks` key.* 163 | 164 | Read more on the [compile-extensions repo](https://github.com/cloudfoundry-incubator/compile-extensions). 165 | 166 | 167 | ## dependency_deprecation_dates (optional) 168 | 169 | 170 | The dependency_deprecation_dates specifies the date at which dependencies 171 | will be end of life (and thus removed from the buildpack). By specifying 172 | EOL here, buildpack maintainers can set a date which will trigger warnings 173 | for users 30 days before the EOL date is reached. 174 | 175 | All keys are required: 176 | 177 | - `name`, `match`: 178 | Required for `dependency_deprecation_dates` to work. Dependencies are matched to the 179 | name and a regexp of `match` and the dependency version 180 | 181 | - `date`, `version_line`: 182 | Required to generate the warning message for users, eg: 183 | `WARNING: Ruby 2.1 will no longer be available in new buildpacks released after 2016-03-30` 184 | 185 | ## exclude_files (required) 186 | 187 | The exclude key lists files you do not want in your buildpack. This is 188 | useful to remove sensitive information or extraneous files before uploading. 189 | 190 | # Development 191 | 192 | ## Propagating Packager Changes to CF Buildpacks 193 | 194 | Latest changes to master will not be automatically reflected in the various Cloud Foundry buildpacks. 195 | To propagate buildpack-packager changes: 196 | 197 | 1. Update the version in `lib/buildpack/packager/version.rb`. 198 | 2. Commit this change and push to master. 199 | 3. Tag the new version with a release tag (e.g., `git tag v2.2.6`). 200 | 4. Push the tag to origin with `git push --tags`. 201 | 5. Update the `cf.Gemfile`s in the various buildpacks with the new release tag like so: 202 | ``` 203 | gem 'buildpack-packager', git: 'https://github.com/cloudfoundry/buildpack-packager', tag: 'v2.2.6' 204 | ``` 205 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'bundler/gem_tasks' 2 | require 'rspec/core/rake_task' 3 | 4 | RSpec::Core::RakeTask.new(:spec) 5 | 6 | task default: :spec 7 | -------------------------------------------------------------------------------- /bin/buildpack-packager: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'buildpack/packager' 3 | require 'optparse' 4 | require 'buildpack/manifest_validator' 5 | 6 | SUPPORTED_MODES = %w(cached uncached list defaults).freeze 7 | USAGE_MSG = "USAGE: #{File.basename(__FILE__)} < #{SUPPORTED_MODES.map { |mode| '--' + mode }.join(' | ')} >".freeze 8 | 9 | options = { 10 | root_dir: Dir.pwd, 11 | manifest_path: 'manifest.yml', 12 | force_download: false 13 | } 14 | 15 | optparser = OptionParser.new do |opts| 16 | opts.banner = USAGE_MSG 17 | 18 | opts.on('--cached', 'Package dependencies with the built buildpack.') do 19 | options[:mode] = :cached 20 | end 21 | 22 | opts.on('--uncached', 'Will NOT package dependencies with the built buildpack.') do 23 | options[:mode] = :uncached 24 | end 25 | 26 | opts.on('--any-stack', '') do 27 | options[:stack] = :any_stack 28 | end 29 | 30 | opts.on('--stack=STACK', '') do |stack| 31 | options[:stack] = stack 32 | end 33 | 34 | opts.on('--list', 'List dependencies supported by the buildpack.') do 35 | options[:mode] = :list 36 | end 37 | 38 | opts.on('--defaults', 'List default dependency versions specified by the buildpack.') do 39 | options[:mode] = :defaults 40 | end 41 | 42 | opts.on('--force-download', 'Force dependencies to be downloaded from the internet. (default is false)') do 43 | options[:force_download] = true 44 | end 45 | 46 | opts.on('--use-custom-manifest=PATH_TO_MANIFEST', 'Use a custom manifest, passing the path as an argument.') do |manifest_path| 47 | options[:manifest_path] = manifest_path 48 | end 49 | 50 | opts.on_tail('-h', '--help', 'Show this message') do 51 | puts opts 52 | exit 53 | end 54 | end 55 | 56 | optparser.parse! 57 | 58 | unless options[:mode] 59 | puts optparser 60 | exit 1 61 | end 62 | 63 | if options[:mode] == :cached || options[:mode] == :uncached 64 | unless options[:stack] 65 | puts optparser 66 | exit 1 67 | end 68 | end 69 | 70 | manifest_path = options[:manifest_path] 71 | unless File.exist?(manifest_path) 72 | STDERR.puts "Could not find #{manifest_path}" 73 | exit 1 74 | end 75 | 76 | validator = Buildpack::ManifestValidator.new(manifest_path) 77 | unless validator.valid? 78 | errors = validator.errors 79 | if errors[:manifest_parser_errors] 80 | STDERR.print "The manifest does not conform to the schema:\n\t- " 81 | STDERR.puts errors[:manifest_parser_errors].join("\n\t- ") 82 | end 83 | 84 | STDERR.puts errors[:default_version].join("\n\t") 85 | exit 1 86 | end 87 | 88 | if options[:mode] == :list 89 | puts Buildpack::Packager.list(options) 90 | elsif options[:mode] == :defaults 91 | puts Buildpack::Packager.defaults(options) 92 | else 93 | Buildpack::Packager.package(options) 94 | end 95 | -------------------------------------------------------------------------------- /buildpack-packager.gemspec: -------------------------------------------------------------------------------- 1 | 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'buildpack/packager/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = 'buildpack-packager' 8 | spec.version = Buildpack::Packager::VERSION 9 | spec.authors = ['Cloud Foundry Buildpacks Team'] 10 | spec.email = ['cf-buildpacks-eng@pivotal.io'] 11 | spec.summary = 'Tool that packages your buildpacks based on a manifest' 12 | spec.description = 'Tool that packages your Cloud Foundry buildpacks based on a manifest' 13 | spec.homepage = 'https://github.com/cloudfoundry/buildpack-packager' 14 | spec.license = 'Apache-2.0' 15 | 16 | spec.files = `git ls-files -z`.split("\x0") 17 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 18 | spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) 19 | spec.require_paths = ['lib'] 20 | 21 | spec.required_ruby_version = '~> 3.0' 22 | 23 | spec.add_dependency 'activesupport', '~> 4.1' 24 | spec.add_dependency 'kwalify', '~> 0' 25 | spec.add_dependency 'semantic' 26 | spec.add_dependency 'terminal-table', '~> 1.4' 27 | 28 | spec.add_development_dependency 'bundler', '~> 2.4.3' 29 | spec.add_development_dependency 'rake', '~> 10.0' 30 | spec.add_development_dependency 'rspec', '~> 3.5' 31 | spec.add_development_dependency 'rubocop', '~> 0.52' 32 | spec.add_development_dependency 'rubocop-rspec', '~> 1.23' 33 | spec.add_development_dependency 'rubyzip', '~> 1.2' 34 | end 35 | -------------------------------------------------------------------------------- /doc/disconnected_environments.md: -------------------------------------------------------------------------------- 1 | # Disconnected environments 2 | 3 | ### Deploying Apps on disconnected environments 4 | Cached buildpacks only ensure that the buildpack's dependencies are cached, not your applications. 5 | 6 | When you work with a disconnected environment, it's important to use your package manager 7 | to 'vendor' your application's dependencies. 8 | 9 | The specific mechanism varies between platforms. See your buildpack's documentation for 'vendoring' advice. 10 | 11 | ## Building a cached buildpack 12 | 1. Make sure you have fetched submodules 13 | 14 | ```shell 15 | git submodule update --init 16 | ``` 17 | 18 | 1. Get the latest buildpack dependencies 19 | 20 | ```shell 21 | BUNDLE_GEMFILE=cf.Gemfile bundle 22 | ``` 23 | 24 | 1. Build the buildpack 25 | 26 | ```shell 27 | BUNDLE_GEMFILE=cf.Gemfile bundle exec buildpack-packager [ uncached | cached ] 28 | ``` 29 | 30 | This produces a buildpack for use on Cloud Foundry. 31 | 32 | 'cached' generates a zip with all the dependencies cached. 33 | 34 | 'uncached' does not include the dependencies, however it excludes some files as specified 35 | in manifest.yml. 36 | 37 | 1. Use in Cloud Foundry 38 | 39 | Currently, you can only specify cached buildpacks by creating Cloud Foundry Admin Buildpacks. 40 | 41 | This means you need admin rights. See the 42 | [open source admin documentation](http://docs.cloudfoundry.org/adminguide/buildpacks.html) 43 | for more information. 44 | 45 | Upload the buildpack to your Cloud Foundry and specify it by name: 46 | 47 | ```shell 48 | cf create-buildpack custom_ruby_buildpack ruby_buildpack-cached-custom.zip 1 49 | ``` 50 | 51 | -------------------------------------------------------------------------------- /lib/buildpack/manifest_dependency.rb: -------------------------------------------------------------------------------- 1 | module Buildpack 2 | class ManifestDependency 3 | attr_reader :name, :version 4 | 5 | def initialize(name, version) 6 | @name = name 7 | @version = version 8 | end 9 | 10 | def ==(other_object) 11 | name == other_object.name && version == other_object.version 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /lib/buildpack/manifest_validator.rb: -------------------------------------------------------------------------------- 1 | require 'kwalify' 2 | require 'kwalify/parser/yaml-patcher' 3 | require 'uri' 4 | require 'semantic' 5 | 6 | require 'buildpack/manifest_dependency' 7 | 8 | module Buildpack 9 | class ManifestValidator 10 | class ManifestValidationError < StandardError; end 11 | 12 | SCHEMA_FILE = File.join(File.dirname(__FILE__), 'packager', 'manifest_schema.yml') 13 | 14 | attr_reader :errors 15 | 16 | def initialize(manifest_path) 17 | @manifest_path = manifest_path 18 | end 19 | 20 | def valid? 21 | validate 22 | errors.empty? 23 | end 24 | 25 | private 26 | 27 | def validate 28 | schema = Kwalify::Yaml.load_file(SCHEMA_FILE) 29 | validator = Kwalify::Validator.new(schema) 30 | parser = Kwalify::Yaml::Parser.new(validator) 31 | manifest_data = parser.parse_file(@manifest_path) 32 | 33 | @errors = {} 34 | @errors[:manifest_parser_errors] = parser.errors unless parser.errors.empty? 35 | 36 | if manifest_data['default_versions'] && !@errors[:manifest_parser_errors] 37 | default_version_errors = validate_default_versions(manifest_data) 38 | @errors[:default_version] = default_version_errors unless default_version_errors.empty? 39 | end 40 | end 41 | 42 | def validate_default_versions(manifest_data) 43 | error_messages = [] 44 | 45 | default_versions = create_manifest_dependencies(manifest_data['default_versions']) 46 | dependency_versions = create_manifest_dependencies(manifest_data['dependencies']) 47 | 48 | error_messages += validate_no_duplicate_names(default_versions) 49 | error_messages += validate_defaults_in_dependencies(default_versions, dependency_versions) 50 | wrap_errors_with_common_text(error_messages) 51 | 52 | error_messages 53 | end 54 | 55 | def create_manifest_dependencies(dependency_entries) 56 | dependency_entries.map do |dependency| 57 | ManifestDependency.new(dependency['name'], dependency['version']) 58 | end 59 | end 60 | 61 | def validate_no_duplicate_names(default_versions) 62 | default_versions_names = default_versions.map { |default_version| default_version.name } 63 | duplicate_names = default_versions_names.find_all { |dep| default_versions_names.count(dep) > 1 }.uniq 64 | 65 | duplicate_names.map do |name| 66 | "- #{name} had more than one 'default_versions' entry in the buildpack manifest." 67 | end 68 | end 69 | 70 | def validate_defaults_in_dependencies(default_versions, dependency_versions) 71 | unmatched_dependencies = default_versions.reject { |d| version_exists?(d, dependency_versions) } 72 | unmatched_dependencies.map do |dependency| 73 | name_version = "#{dependency.name} #{dependency.version}" 74 | 75 | "- a 'default_versions' entry for #{name_version} was specified by the buildpack manifest, " + 76 | "but no 'dependencies' entry for #{name_version} was found in the buildpack manifest." 77 | end 78 | end 79 | 80 | def wrap_errors_with_common_text(error_messages) 81 | if error_messages.any? 82 | error_messages.unshift('The buildpack manifest is malformed:') 83 | error_messages<< 'For more information, see https://docs.cloudfoundry.org/buildpacks/custom.html#specifying-default-versions' 84 | end 85 | end 86 | 87 | private 88 | 89 | def version_exists?(default_dependency, dependency_versions) 90 | major, minor, patch = default_dependency.version.to_s.split('.') 91 | major = major.gsub('v','') 92 | 93 | if patch == 'x' 94 | dependency_versions.each do |d| 95 | d_version = Semantic::Version.new(d.version) 96 | return true if d.name == default_dependency.name && d_version.major.to_s == major && d_version.minor.to_s == minor 97 | end 98 | elsif patch.nil? && minor == 'x' 99 | dependency_versions.each do |d| 100 | d_version = Semantic::Version.new(d.version) 101 | return true if d.name == default_dependency.name && d_version.major.to_s == major 102 | end 103 | else 104 | return dependency_versions.include?(default_dependency) 105 | end 106 | 107 | return false 108 | end 109 | end 110 | end 111 | 112 | -------------------------------------------------------------------------------- /lib/buildpack/packager.rb: -------------------------------------------------------------------------------- 1 | require 'buildpack/packager/version' 2 | require 'buildpack/packager/table_presentation' 3 | require 'buildpack/packager/dependencies_presenter' 4 | require 'buildpack/packager/default_versions_presenter' 5 | require 'buildpack/packager/package' 6 | require 'active_support/core_ext/hash/indifferent_access' 7 | require 'open3' 8 | require 'fileutils' 9 | require 'tmpdir' 10 | require 'yaml' 11 | require 'shellwords' 12 | 13 | module Buildpack 14 | module Packager 15 | class CheckSumError < StandardError; end 16 | 17 | def self.package(options) 18 | check_for_zip 19 | 20 | package = Package.new(options) 21 | 22 | Dir.mktmpdir do |temp_dir| 23 | package.copy_buildpack_to_temp_dir(temp_dir) 24 | 25 | package.build_dependencies(temp_dir) if options[:mode] == :cached 26 | 27 | Dir.chdir(temp_dir) do 28 | package.run_pre_package 29 | end 30 | 31 | package.build_zip_file(temp_dir) 32 | end 33 | 34 | buildpack_stack = options[:stack] == :any_stack ? "any stack" : options[:stack] 35 | buildpack_type = options[:mode] == :cached ? "Cached" : "Uncached" 36 | human_readable_size = `du -h #{package.zip_file_path} | cut -f1` 37 | puts "#{buildpack_type} buildpack for #{buildpack_stack} created and saved as #{package.zip_file_path} with a size of #{human_readable_size.strip}" 38 | 39 | package 40 | end 41 | 42 | def self.list(options) 43 | package = Package.new(options) 44 | package.list 45 | end 46 | 47 | def self.defaults(options) 48 | package = Package.new(options) 49 | package.defaults 50 | end 51 | 52 | def self.check_for_zip 53 | _, _, status = Open3.capture3('which zip') 54 | 55 | if status.to_s.include?('exit 1') 56 | raise "Zip is not installed\nTry: apt-get install zip\nAnd then rerun" 57 | end 58 | end 59 | end 60 | end 61 | -------------------------------------------------------------------------------- /lib/buildpack/packager/default_versions_presenter.rb: -------------------------------------------------------------------------------- 1 | require 'terminal-table' 2 | 3 | module Buildpack 4 | module Packager 5 | class DefaultVersionsPresenter < Struct.new(:default_versions) 6 | include TablePresentation 7 | 8 | attr_reader :default_versions 9 | 10 | def initialize(default_versions) 11 | default_versions = [] if default_versions.nil? 12 | @default_versions = default_versions 13 | end 14 | 15 | def inspect 16 | table = Terminal::Table.new do |table| 17 | default_versions.sort_by do |dependency| 18 | sort_string_for dependency 19 | end.each do |dependency| 20 | columns = [ 21 | dependency['name'], 22 | sanitize_version_string(dependency['version']) 23 | ] 24 | table.add_row columns 25 | end 26 | end 27 | 28 | table.headings = %w(name version) 29 | 30 | table.to_s 31 | end 32 | 33 | def present 34 | to_markdown(inspect) 35 | end 36 | end 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /lib/buildpack/packager/dependencies_presenter.rb: -------------------------------------------------------------------------------- 1 | require 'terminal-table' 2 | 3 | module Buildpack 4 | module Packager 5 | class DependenciesPresenter < Struct.new(:dependencies) 6 | include TablePresentation 7 | 8 | def inspect 9 | has_modules = dependencies.any? { |dependency| dependency['modules'] } 10 | 11 | table = Terminal::Table.new do |table| 12 | dependencies.sort_by do |dependency| 13 | sort_string_for dependency 14 | end.each do |dependency| 15 | columns = [ 16 | dependency['name'], 17 | sanitize_version_string(dependency['version']), 18 | dependency['cf_stacks'].sort.join(',') 19 | ] 20 | if has_modules 21 | columns += [dependency.fetch('modules', []).sort.join(', ')] 22 | end 23 | table.add_row columns 24 | end 25 | end 26 | 27 | table.headings = if has_modules 28 | %w(name version cf_stacks modules) 29 | else 30 | %w(name version cf_stacks) 31 | end 32 | 33 | table.to_s 34 | end 35 | 36 | def present 37 | to_markdown(inspect) 38 | end 39 | end 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /lib/buildpack/packager/manifest_schema.yml: -------------------------------------------------------------------------------- 1 | type: map 2 | mapping: 3 | "language": 4 | type: str 5 | required: yes 6 | "dependency_deprecation_dates": 7 | type: seq 8 | required: no 9 | sequence: 10 | - type: map 11 | mapping: 12 | "match": 13 | type: str 14 | required: no 15 | "version_line": 16 | type: text 17 | required: yes 18 | "name": 19 | type: str 20 | required: yes 21 | "date": 22 | type: date 23 | required: yes 24 | "link": 25 | type: str 26 | required: no 27 | "url_to_dependency_map": 28 | type: seq 29 | required: no 30 | sequence: 31 | - type: map 32 | mapping: 33 | "match": 34 | type: str 35 | required: yes 36 | "name": 37 | type: str 38 | required: yes 39 | "version": 40 | type: text 41 | required: yes 42 | "dependencies": 43 | type: seq 44 | required: yes 45 | sequence: 46 | - type: map 47 | mapping: 48 | "name": 49 | type: str 50 | required: yes 51 | "version": 52 | type: text 53 | required: yes 54 | "source": 55 | type: str 56 | required: no 57 | "osl": 58 | type: str 59 | required: no 60 | "uri": 61 | type: str 62 | required: yes 63 | "modules": 64 | type: seq 65 | required: no 66 | sequence: 67 | - type: str 68 | "source_sha256": 69 | type: str 70 | required: no 71 | "sha256": 72 | type: str 73 | required: yes 74 | "cf_stacks": 75 | type: seq 76 | required: yes 77 | sequence: 78 | - type: str 79 | enum: [ lucid64, cflinuxfs2, cflinuxfs3, cflinuxfs4, windows2012R2, windows2016, opensuse42, sle12, sle15 ] 80 | "dependencies": 81 | type: seq 82 | required: no 83 | sequence: 84 | - type: map 85 | mapping: 86 | "name": 87 | type: str 88 | required: yes 89 | "version": 90 | type: text 91 | required: no 92 | "exclude_files": 93 | type: seq 94 | required: yes 95 | sequence: 96 | - type: str 97 | "default_versions": 98 | type: seq 99 | required: no 100 | sequence: 101 | - type: map 102 | required: yes 103 | mapping: 104 | "name": 105 | type: str 106 | required: yes 107 | "version": 108 | type: text 109 | required: yes 110 | "pre_package": 111 | type: str 112 | required: no 113 | -------------------------------------------------------------------------------- /lib/buildpack/packager/package.rb: -------------------------------------------------------------------------------- 1 | require 'active_support/core_ext/hash/indifferent_access' 2 | require 'open3' 3 | require 'fileutils' 4 | require 'tmpdir' 5 | require 'yaml' 6 | require 'shellwords' 7 | require 'buildpack/packager/zip_file_excluder' 8 | require 'digest' 9 | 10 | module Buildpack 11 | module Packager 12 | class Package < Struct.new(:options) 13 | def copy_buildpack_to_temp_dir(temp_dir) 14 | FileUtils.cp_r(File.join(options[:root_dir], '.'), temp_dir) 15 | 16 | a_manifest = YAML.load_file(options[:manifest_path]).with_indifferent_access 17 | unless options[:stack] == :any_stack 18 | a_manifest = edit_manifest_for_stack(a_manifest) 19 | end 20 | File.open(File.join(temp_dir, 'manifest.yml'), 'w') { |f| f.write(a_manifest.to_hash.to_yaml) } 21 | end 22 | 23 | def edit_manifest_for_stack(a_manifest) 24 | a_manifest[:stack] = options[:stack] 25 | 26 | a_manifest[:dependencies] = a_manifest[:dependencies] 27 | .select { |dep| dep.fetch('cf_stacks', []).include? options[:stack] } 28 | .each { |dep| dep.delete('cf_stacks') } 29 | 30 | a_manifest 31 | end 32 | 33 | def build_dependencies(temp_dir) 34 | local_cache_directory = options[:cache_dir] || "#{ENV['HOME']}/.buildpack-packager/cache" 35 | FileUtils.mkdir_p(local_cache_directory) 36 | 37 | dependency_dir = File.join(temp_dir, "dependencies") 38 | FileUtils.mkdir_p(dependency_dir) 39 | 40 | download_dependencies(manifest[:dependencies], local_cache_directory, dependency_dir) 41 | end 42 | 43 | def download_dependencies(dependencies, local_cache_directory, dependency_dir) 44 | dependencies.each do |dependency| 45 | if options[:stack] == :any_stack || dependency.fetch(:cf_stacks, []).include?(options[:stack]) 46 | safe_uri = uri_without_credentials(dependency['uri']) 47 | translated_filename = uri_cache_path(safe_uri) 48 | local_cached_file = File.expand_path(File.join(local_cache_directory, translated_filename)) 49 | 50 | if options[:force_download] || !File.exist?(local_cached_file) 51 | puts "Downloading #{dependency['name']} version #{dependency['version']} from: #{safe_uri}" 52 | download_file(dependency['uri'], local_cached_file) 53 | human_readable_size = `du -h #{local_cached_file} | cut -f1`.strip 54 | puts " Using #{dependency['name']} version #{dependency['version']} with size #{human_readable_size}" 55 | 56 | from_local_cache = false 57 | else 58 | human_readable_size = `du -h #{local_cached_file} | cut -f1`.strip 59 | puts "Using #{dependency['name']} version #{dependency['version']} from local cache at: #{local_cached_file} with size #{human_readable_size}" 60 | from_local_cache = true 61 | end 62 | 63 | ensure_correct_dependency_checksum(**{ 64 | local_cached_file: local_cached_file, 65 | dependency: dependency, 66 | from_local_cache: from_local_cache 67 | }) 68 | 69 | FileUtils.cp(local_cached_file, dependency_dir) 70 | end 71 | end 72 | end 73 | 74 | def build_zip_file(temp_dir) 75 | FileUtils.rm_rf(zip_file_path) 76 | zip_files(temp_dir, zip_file_path, manifest[:exclude_files]) 77 | end 78 | 79 | def list 80 | DependenciesPresenter.new(manifest['dependencies']).present 81 | end 82 | 83 | def defaults 84 | DefaultVersionsPresenter.new(manifest['default_versions']).present 85 | end 86 | 87 | def zip_file_path 88 | Shellwords.escape(File.join(options[:root_dir], zip_file_name)) 89 | end 90 | 91 | def run_pre_package 92 | if manifest['pre_package'] && !Kernel.system(manifest['pre_package']) 93 | raise "Failed to run pre_package script: #{manifest['pre_package']}" 94 | end 95 | end 96 | 97 | private 98 | 99 | def uri_without_credentials(uri_string) 100 | uri = URI(uri_string) 101 | if uri.userinfo 102 | uri.user = "-redacted-" if uri.user 103 | uri.password = "-redacted-" if uri.password 104 | end 105 | uri.to_s 106 | end 107 | 108 | def uri_cache_path uri 109 | uri.gsub(/[:\/\?&]/, '_') 110 | end 111 | 112 | def manifest 113 | @manifest ||= YAML.load_file(options[:manifest_path]).with_indifferent_access 114 | end 115 | 116 | def zip_file_name 117 | stack = options[:stack] == :any_stack ? '' : "-#{options[:stack]}" 118 | "#{manifest[:language]}_buildpack#{cached_identifier}#{stack}-v#{buildpack_version}.zip" 119 | end 120 | 121 | def buildpack_version 122 | File.read("#{options[:root_dir]}/VERSION").chomp 123 | end 124 | 125 | def cached_identifier 126 | return '' unless options[:mode] == :cached 127 | '-cached' 128 | end 129 | 130 | def ensure_correct_dependency_checksum(local_cached_file:, dependency:, from_local_cache:) 131 | if dependency['sha256'] != Digest::SHA256.file(local_cached_file).hexdigest 132 | if from_local_cache 133 | FileUtils.rm_rf(local_cached_file) 134 | 135 | download_file(dependency['uri'], local_cached_file) 136 | ensure_correct_dependency_checksum(**{ 137 | local_cached_file: local_cached_file, 138 | dependency: dependency, 139 | from_local_cache: false 140 | }) 141 | else 142 | raise CheckSumError, 143 | "File: #{dependency['name']}, version: #{dependency['version']} downloaded at location #{dependency['uri']}\n\tis reporting a different checksum than the one specified in the manifest." 144 | end 145 | else 146 | puts " #{dependency['name']} version #{dependency['version']} matches the manifest provided sha256 checksum of #{dependency['sha256']}\n\n" 147 | end 148 | end 149 | 150 | def download_file(url, file) 151 | raise "Failed to download file from #{url}" unless system("curl -s --retry 15 --retry-delay 2 #{url} -o #{file} -L --fail -f") 152 | end 153 | 154 | def zip_files(source_dir, zip_file_path, excluded_files) 155 | excluder = ZipFileExcluder.new 156 | manifest_exclusions = excluder.generate_manifest_exclusions excluded_files 157 | gitfile_exclusions = excluder.generate_exclusions_from_git_files source_dir 158 | all_exclusions = manifest_exclusions + ' ' + gitfile_exclusions 159 | `cd #{source_dir} && zip -r #{zip_file_path} ./ #{all_exclusions}` 160 | end 161 | end 162 | end 163 | end 164 | 165 | -------------------------------------------------------------------------------- /lib/buildpack/packager/table_presentation.rb: -------------------------------------------------------------------------------- 1 | require 'terminal-table' 2 | 3 | module Buildpack 4 | module Packager 5 | module TablePresentation 6 | def to_markdown(table_contents) 7 | table_contents.split("\n")[1...-1].tap { |lines| lines[1].tr!('+', '|') }.join("\n") 8 | end 9 | 10 | def sanitize_version_string(version) 11 | version == 0 ? '-' : version 12 | end 13 | 14 | def sort_string_for(dependency) 15 | interpreter_names = %w(ruby jruby php hhvm python go node) 16 | sort_index = interpreter_names.index(dependency['name']) || 9999 17 | sprintf '%s-%s-%s', sort_index, dependency['name'], dependency['version'] 18 | end 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /lib/buildpack/packager/version.rb: -------------------------------------------------------------------------------- 1 | module Buildpack 2 | module Packager 3 | VERSION = '2.3.23'.freeze 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /lib/buildpack/packager/zip_file_excluder.rb: -------------------------------------------------------------------------------- 1 | module Buildpack 2 | module Packager 3 | class ZipFileExcluder 4 | def generate_manifest_exclusions(excluded_files) 5 | generate_exclusion_string excluded_files 6 | end 7 | 8 | def generate_exclusions_from_git_files(dir) 9 | Dir.chdir dir do 10 | git_files = Dir.glob('**/.git*').map do |elt| 11 | File.directory?(elt) ? "#{elt}/" : elt 12 | end 13 | 14 | generate_exclusion_string git_files 15 | end 16 | end 17 | 18 | private 19 | 20 | def generate_exclusion_string(file_list) 21 | file_list.map do |file| 22 | if file.chars.last == '/' 23 | "-x #{file}\\* -x \\*/#{file}\\*" 24 | else 25 | "-x #{file} -x \\*/#{file}" 26 | end 27 | end.join(' ') 28 | end 29 | end 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /lib/kwalify/parser/yaml-patcher.rb: -------------------------------------------------------------------------------- 1 | class Kwalify::Yaml::Parser < Kwalify::BaseParser 2 | def parse_block_value(level, rule, path, uniq_table, container) 3 | skip_spaces_and_comments 4 | ## nil 5 | return nil if @column < level || (@column == level && !match?(/-\s+/)) || eos? 6 | ## anchor and alias 7 | name = nil 8 | if scan(/\&([-\w]+)/) 9 | name = parse_anchor(rule, path, uniq_table, container) 10 | elsif scan(/\*([-\w]+)/) 11 | return parse_alias(rule, path, uniq_table, container) 12 | end 13 | ## type 14 | skip_spaces_and_comments if scan(/!!?\w+/) 15 | ## sequence 16 | if match?(/-\s+/) 17 | if rule && !rule.sequence 18 | # _validate_error("sequence is not expected.", path) 19 | rule = nil 20 | end 21 | seq = create_sequence(rule, @linenum, @column) 22 | @anchors[name] = seq if name 23 | parse_block_seq(seq, rule, path, uniq_table) 24 | return seq 25 | end 26 | ## mapping 27 | if match?(MAPKEY_PATTERN) 28 | if rule && !rule.mapping 29 | # _validate_error("mapping is not expected.", path) 30 | rule = nil 31 | end 32 | map = create_mapping(rule, @linenum, @column) 33 | @anchors[name] = map if name 34 | parse_block_map(map, rule, path, uniq_table) 35 | return map 36 | end 37 | ## sequence (flow-style) 38 | if match?(/\[/) 39 | if rule && !rule.sequence 40 | # _validate_error("sequence is not expected.", path) 41 | rule = nil 42 | end 43 | seq = create_sequence(rule, @linenum, @column) 44 | @anchors[name] = seq if name 45 | parse_flow_seq(seq, rule, path, uniq_table) 46 | return seq 47 | end 48 | ## mapping (flow-style) 49 | if match?(/\{/) 50 | if rule && !rule.mapping 51 | # _validate_error("mapping is not expected.", path) 52 | rule = nil 53 | end 54 | map = create_mapping(rule, @linenum, @column) 55 | @anchors[name] = map if name 56 | parse_flow_map(map, rule, path, uniq_table) 57 | return map 58 | end 59 | ## block text 60 | if match?(/[|>]/) 61 | text = parse_block_text(level, rule, path, uniq_table) 62 | @anchors[name] = text if name 63 | return text 64 | end 65 | ## scalar 66 | scalar = parse_block_scalar(rule, path, uniq_table) 67 | @anchors[name] = scalar if name 68 | scalar 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /spec/buildpack/packager_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Buildpack::Packager do 4 | it 'has a version number' do 5 | expect(Buildpack::Packager::VERSION).not_to be nil 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /spec/fixtures/buildpack-with-uri-credentials/VERSION: -------------------------------------------------------------------------------- 1 | 1.7.8 -------------------------------------------------------------------------------- /spec/fixtures/buildpack-with-uri-credentials/manifest.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: go 3 | url_to_dependency_map: 4 | - match: go(\d+\.\d+(.*)) 5 | name: go 6 | version: "$1" 7 | - match: godep 8 | name: godep 9 | version: v74 10 | dependencies: 11 | - name: go 12 | version: 1.6.3 13 | uri: https://login:password@buildpacks.cloudfoundry.org/concourse-binaries/go/go1.6.3.linux-amd64.tar.gz 14 | sha256: 5ac238cd321a66a35c646d61e4cafd922929af0800c78b00375312c76e702e11 15 | cf_stacks: 16 | - cflinuxfs2 17 | - name: godep 18 | version: v74 19 | uri: https://api-token:x-oauth-basic@buildpacks.cloudfoundry.org/concourse-binaries/godep/godep-v74-linux-x64.tgz 20 | sha256: 6e6761b71e1518bf7716b3f383f10598afe5ce4592e6eea0184f17b85fd93813 21 | cf_stacks: 22 | - cflinuxfs2 23 | exclude_files: 24 | - ".git/" 25 | - ".gitignore" 26 | - ".gitmodules" 27 | - cf_spec/ 28 | - log/ 29 | - bin/package 30 | - buildpack-packager/ 31 | - test-godir/ 32 | - test/ 33 | - Makefile 34 | - PULL_REQUEST_TEMPLATE 35 | - ISSUE_TEMPLATE 36 | - go_buildpack-*v*.zip 37 | -------------------------------------------------------------------------------- /spec/fixtures/buildpack-without-uri-credentials/VERSION: -------------------------------------------------------------------------------- 1 | 1.7.8 -------------------------------------------------------------------------------- /spec/fixtures/buildpack-without-uri-credentials/manifest.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: go 3 | url_to_dependency_map: 4 | - match: go(\d+\.\d+(.*)) 5 | name: go 6 | version: "$1" 7 | - match: godep 8 | name: godep 9 | version: v74 10 | dependencies: 11 | - name: go 12 | version: 1.6.3 13 | uri: https://buildpacks.cloudfoundry.org/concourse-binaries/go/go1.6.3.linux-amd64.tar.gz 14 | sha256: 5ac238cd321a66a35c646d61e4cafd922929af0800c78b00375312c76e702e11 15 | cf_stacks: 16 | - cflinuxfs2 17 | - name: godep 18 | version: v74 19 | uri: https://pivotal-buildpacks.s3.amazonaws.com/concourse-binaries/godep/godep-v74-linux-x64.tgz 20 | sha256: 6e6761b71e1518bf7716b3f383f10598afe5ce4592e6eea0184f17b85fd93813 21 | cf_stacks: 22 | - cflinuxfs2 23 | exclude_files: 24 | - ".git/" 25 | - ".gitignore" 26 | - ".gitmodules" 27 | - cf_spec/ 28 | - log/ 29 | - bin/package 30 | - buildpack-packager/ 31 | - test-godir/ 32 | - test/ 33 | - Makefile 34 | - PULL_REQUEST_TEMPLATE 35 | - ISSUE_TEMPLATE 36 | - go_buildpack-*v*.zip 37 | -------------------------------------------------------------------------------- /spec/fixtures/manifests/manifest_invalid-sha224.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: go 3 | 4 | url_to_dependency_map: 5 | - match: go(\d+\.\d+(\.\d+)?) 6 | name: go 7 | version: $1 8 | dependencies: 9 | - name: go 10 | version: 1.2 11 | uri: http://go.googlecode.com/files/go1.2.linux-amd64.tar.gz 12 | sha224: 68901bbf8a04e71e0b30aa19c3946b21 13 | cf_stacks: 14 | - lucid64 15 | - cflinuxfs2 16 | 17 | exclude_files: 18 | - .git/ 19 | -------------------------------------------------------------------------------- /spec/fixtures/manifests/manifest_invalid-sha224_and_defaults.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: go 3 | 4 | url_to_dependency_map: 5 | - match: go(\d+\.\d+(\.\d+)?) 6 | name: go 7 | version: $1 8 | default_versions: 9 | - name: go 10 | version: 1.3 11 | dependencies: 12 | - name: go 13 | version: 1.2 14 | uri: http://go.googlecode.com/files/go1.2.linux-amd64.tar.gz 15 | sha244: 68901bbf8a04e71e0b30aa19c3946b21 16 | cf_stacks: 17 | - lucid64 18 | - cflinuxfs2 19 | 20 | exclude_files: 21 | - .git/ 22 | -------------------------------------------------------------------------------- /spec/fixtures/manifests/manifest_opensus.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: open_sus_language 3 | dependencies: 4 | - name: mr_open_sus_the_third 5 | version: 3.0.0 6 | uri: https://buildpacks.cloudfoundry.org/dependencies/garbage/not_real_url.zip 7 | sha256: banana 8 | cf_stacks: 9 | - opensuse42 10 | - sle12 11 | - sle15 12 | pre_package: scripts/build.sh 13 | 14 | exclude_files: [] 15 | -------------------------------------------------------------------------------- /spec/fixtures/manifests/manifest_valid.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: go 3 | 4 | url_to_dependency_map: 5 | - match: go(\d+\.\d+(\.\d+)?) 6 | name: go 7 | version: $1 8 | 9 | dependencies: 10 | - name: go 11 | version: 1.2 12 | uri: http://go.googlecode.com/files/go1.2.linux-amd64.tar.gz 13 | source: some-source 14 | source_sha256: "asdASDadsadasdasdasdasdasdasdasdasdasdasdasdasdasdboo" 15 | osl: some-osl 16 | sha256: c0ffee 17 | cf_stacks: 18 | - lucid64 19 | - cflinuxfs2 20 | 21 | exclude_files: 22 | - .git/ 23 | -------------------------------------------------------------------------------- /spec/fixtures/manifests/manifest_valid_new_deprecation_dates_no_url_map.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: go 3 | 4 | dependencies: 5 | - name: go 6 | version: 1.2 7 | uri: http://go.googlecode.com/files/go1.2.linux-amd64.tar.gz 8 | sha256: c0ffee 9 | cf_stacks: 10 | - lucid64 11 | - cflinuxfs2 12 | 13 | dependency_deprecation_dates: 14 | - version_line: 1.2.x 15 | name: go 16 | date: 2016-01-18 17 | 18 | exclude_files: 19 | - .git/ 20 | -------------------------------------------------------------------------------- /spec/fixtures/manifests/manifest_valid_plus_deprecation_dates.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: go 3 | 4 | url_to_dependency_map: 5 | - match: go(\d+\.\d+(\.\d+)?) 6 | name: go 7 | version: $1 8 | 9 | dependencies: 10 | - name: go 11 | version: 1.2 12 | uri: http://go.googlecode.com/files/go1.2.linux-amd64.tar.gz 13 | sha256: c0ffee 14 | cf_stacks: 15 | - lucid64 16 | - cflinuxfs2 17 | 18 | dependency_deprecation_dates: 19 | - match: 1.2.\\d 20 | version_line: 1.2 21 | name: go 22 | date: 2016-01-18 23 | link: https://link.io 24 | 25 | exclude_files: 26 | - .git/ 27 | -------------------------------------------------------------------------------- /spec/fixtures/manifests/manifest_windows.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: hwc 3 | dependencies: 4 | - name: hwc 5 | version: 3.0.0 6 | uri: https://buildpacks.cloudfoundry.org/dependencies/hwc/hwc-3.0.0-windows-amd64-acb65777.zip 7 | sha256: c0ffee 8 | cf_stacks: 9 | - windows2012R2 10 | - windows2016 11 | pre_package: scripts/build.sh 12 | 13 | exclude_files: [] 14 | -------------------------------------------------------------------------------- /spec/helpers/cache_directory_helpers.rb: -------------------------------------------------------------------------------- 1 | module CacheDirectoryHelpers 2 | BUILDPACK_PACKAGER_CACHE_DIR = File.join(ENV['HOME'], '.buildpack-packager', 'cache') 3 | 4 | def uri_to_cache_filename(uri) 5 | uri.gsub(/[\/:]/, '_') 6 | end 7 | 8 | def uri_to_cache_path(uri) 9 | File.join(BUILDPACK_PACKAGER_CACHE_DIR, uri_to_cache_filename(uri)) 10 | end 11 | 12 | def remove_from_cache_dir(uri) 13 | FileUtils.rm_f(uri_to_cache_path(uri)) 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /spec/helpers/fake_binary_hosting_helpers.rb: -------------------------------------------------------------------------------- 1 | module FakeBinaryHostingHelpers 2 | def upstream_host_dir 3 | @upstream_host_dir ||= Dir.mktmpdir('upstream_host_') 4 | end 5 | 6 | def create_upstream_file(file_name, file_content) 7 | file_path = generate_upstream_file_path(file_name) 8 | File.write(file_path, file_content) 9 | file_path 10 | end 11 | 12 | def remove_upstream_file(file_name) 13 | FileUtils.rm_f(generate_upstream_file_path(file_name)) 14 | end 15 | 16 | private 17 | 18 | def generate_upstream_file_path(file_name) 19 | File.join(upstream_host_dir, file_name) 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /spec/helpers/file_system_helpers.rb: -------------------------------------------------------------------------------- 1 | require 'zip' 2 | require 'fileutils' 3 | 4 | module FileSystemHelpers 5 | def run_packager_binary(buildpack_dir, flags) 6 | packager_binary_file = File.join(`pwd`.chomp, 'bin', 'buildpack-packager') 7 | Open3.capture2e("cd #{buildpack_dir} && #{packager_binary_file} #{flags}") 8 | end 9 | 10 | def make_fake_files(root, file_list) 11 | file_list.each do |file| 12 | full_path = File.join(root, file) 13 | FileUtils.mkdir_p(File.dirname(full_path)) 14 | File.write(full_path, 'a') 15 | end 16 | end 17 | 18 | def all_files(root) 19 | Dir["#{root}/*"].map do |filename| 20 | filename.gsub(root, '').gsub(/^\//, '') 21 | end 22 | end 23 | 24 | def get_zip_contents(zip_path) 25 | Zip::File.open(zip_path) do |zip_file| 26 | zip_file 27 | .map(&:name) 28 | .select { |name| name[/\/$/].nil? } 29 | end 30 | end 31 | 32 | def get_manifest_from_zip(zip_path) 33 | Zip::File.open(zip_path) do |zip_file| 34 | entry = zip_file.glob('manifest.yml').first 35 | return YAML.load(entry.get_input_stream.read).with_indifferent_access 36 | end 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /spec/integration/bin/buildpack_packager/download_caching_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | # Special note 4 | # ============= 5 | # 6 | # There are two uses of the term 'caching' in buildpack packager. 7 | # 1. A 'cached' buildpack is a buildpack zipball that contains the buildpacks dependencies 8 | # 2. 'Download caching' is where the packager keeps a copy of downloaded dependencies for 9 | # the next time it is run. 10 | # 11 | # This integration test is concerned with 'Download caching' 12 | 13 | describe 'reusing previously downloaded files' do 14 | let(:upstream_file_path) do 15 | create_upstream_file('sample_download.ignore_me', 16 | 'sample_download original text') 17 | end 18 | 19 | let(:upstream_file_uri) { "file://#{upstream_file_path}" } 20 | 21 | let(:buildpack_dir) { Dir.mktmpdir('buildpack_') } 22 | 23 | let(:sha256) { Digest::SHA256.file(upstream_file_path).hexdigest } 24 | let(:manifest) do 25 | { 26 | 'exclude_files' => [], 27 | 'language' => 'sample', 28 | 'url_to_dependency_map' => [], 29 | 'dependencies' => [{ 30 | 'version' => '1.0', 31 | 'name' => 'sample_download.ignore_me', 32 | 'cf_stacks' => [], 33 | 'sha256' => sha256, 34 | 'uri' => upstream_file_uri 35 | }] 36 | } 37 | end 38 | 39 | before do 40 | File.write(File.join(buildpack_dir, 'manifest.yml'), manifest.to_yaml) 41 | 42 | `echo "1.2.3" > #{File.join(buildpack_dir, 'VERSION')}` 43 | end 44 | 45 | context 'the file is not in the cache' do 46 | specify 'the file should be kept in a cache when it is downloaded' do 47 | _, status = run_packager_binary(buildpack_dir, '--cached --any-stack') 48 | 49 | expect(status).to be_success 50 | expect(File).to exist(uri_to_cache_path(upstream_file_uri)) 51 | 52 | expect(File.read(uri_to_cache_path(upstream_file_uri))).to include('sample_download original text') 53 | end 54 | end 55 | 56 | context 'the file has been downloaded before' do 57 | specify 'the file in the cache should be used instead of downloading' do 58 | run_packager_binary(buildpack_dir, '--cached --any-stack') 59 | 60 | remove_upstream_file('sample_download.ignore_me') # taking this away means packager must use the cache 61 | 62 | _, status = run_packager_binary(buildpack_dir, '--cached --any-stack') 63 | 64 | expect(status).to be_success 65 | end 66 | 67 | context 'however the file has changed, and the manifest is updated to reflect the new sha256' do 68 | before do 69 | run_packager_binary(buildpack_dir, '--cached --any-stack') 70 | 71 | create_upstream_file('sample_download.ignore_me', 'sample_download updated text') 72 | new_sha256 = Digest::SHA256.file(upstream_file_path).hexdigest 73 | manifest['dependencies'].first['sha256'] = new_sha256 74 | File.write(File.join(buildpack_dir, 'manifest.yml'), manifest.to_yaml) 75 | end 76 | 77 | specify 'the cache should now contain the new upstream file' do 78 | output, status = run_packager_binary(buildpack_dir, '--cached --any-stack') 79 | 80 | expect(status).to be_success 81 | expect(File.read(uri_to_cache_path(upstream_file_uri))).to include('sample_download updated text') 82 | end 83 | end 84 | end 85 | end 86 | -------------------------------------------------------------------------------- /spec/integration/bin/buildpack_packager_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'tmpdir' 3 | require 'yaml' 4 | require 'open3' 5 | 6 | describe 'buildpack_packager binary' do 7 | def create_manifests 8 | create_manifest 9 | create_full_manifest 10 | end 11 | 12 | def create_manifest 13 | File.open(File.join(buildpack_dir, 'manifest.yml'), 'w') do |manifest_file| 14 | manifest_file.write <<-MANIFEST 15 | --- 16 | language: sample 17 | 18 | url_to_dependency_map: 19 | - match: fake_name(\d+\.\d+(\.\d+)?) 20 | name: fake_name 21 | version: $1 22 | 23 | dependencies: 24 | - name: fake_name 25 | version: 1.2 26 | uri: file://#{file_location} 27 | sha256: #{sha256} 28 | modules: ["one", "two", "three"] 29 | cf_stacks: 30 | - lucid64 31 | - cflinuxfs2 32 | 33 | exclude_files: 34 | - .gitignore 35 | - lib/ephemeral_junkpile 36 | MANIFEST 37 | end 38 | 39 | files_to_include << 'manifest.yml' 40 | end 41 | 42 | def create_full_manifest 43 | File.open(File.join(buildpack_dir, 'manifest-including-default-versions.yml'), 'w') do |manifest_file| 44 | manifest_file.write <<-MANIFEST 45 | --- 46 | language: sample 47 | 48 | url_to_dependency_map: 49 | - match: fake_name(\d+\.\d+(\.\d+)?) 50 | name: fake_name 51 | version: $1 52 | 53 | default_versions: 54 | - name: fake_name 55 | version: 1.2 56 | 57 | dependencies: 58 | - name: fake_name 59 | version: 1.2 60 | uri: file://#{file_location} 61 | sha256: #{sha256} 62 | cf_stacks: 63 | - lucid64 64 | - cflinuxfs2 65 | - name: fake_name 66 | version: 1.1 67 | uri: file://#{deprecated_file_location} 68 | sha256: #{deprecated_sha256} 69 | cf_stacks: 70 | - lucid64 71 | - cflinuxfs2 72 | 73 | exclude_files: 74 | - .gitignore 75 | - lib/ephemeral_junkpile 76 | MANIFEST 77 | end 78 | 79 | files_to_include << 'manifest-including-default-versions.yml' 80 | end 81 | 82 | def create_invalid_manifest 83 | File.open(File.join(buildpack_dir, 'manifest.yml'), 'w') do |manifest_file| 84 | manifest_file.write <<-MANIFEST 85 | --- 86 | language: sample 87 | dependencies: 88 | - name: fake_name 89 | version: 1.2 90 | uri: file://#{file_location} 91 | sha256: sha256 92 | cf_stacks: [cflinuxfs2] 93 | MANIFEST 94 | end 95 | 96 | files_to_include << 'manifest.yml' 97 | end 98 | 99 | let(:tmp_dir) { Dir.mktmpdir } 100 | let(:buildpack_dir) { File.join(tmp_dir, 'sample-buildpack-root-dir') } 101 | let(:remote_dependencies_dir) { File.join(tmp_dir, 'remote_dependencies') } 102 | let(:file_location) { "#{remote_dependencies_dir}/dep1.txt" } 103 | let(:sha256) { Digest::SHA256.file(file_location).hexdigest } 104 | let(:deprecated_file_location) { "#{remote_dependencies_dir}/dep2.txt" } 105 | let(:deprecated_sha256) { Digest::SHA256.file(deprecated_file_location).hexdigest } 106 | 107 | let(:files_to_include) do 108 | [ 109 | 'VERSION', 110 | 'README.md', 111 | 'lib/sai.to', 112 | 'lib/rash' 113 | ] 114 | end 115 | 116 | let(:files_to_exclude) do 117 | [ 118 | '.gitignore', 119 | 'lib/ephemeral_junkpile' 120 | ] 121 | end 122 | 123 | let(:files) { files_to_include + files_to_exclude } 124 | 125 | let(:dependencies) do 126 | ['dep1.txt', 'dep2.txt'] 127 | end 128 | 129 | before do 130 | make_fake_files( 131 | remote_dependencies_dir, 132 | dependencies 133 | ) 134 | 135 | make_fake_files( 136 | buildpack_dir, 137 | files 138 | ) 139 | 140 | `echo "1.2.3" > #{File.join(buildpack_dir, 'VERSION')}` 141 | end 142 | 143 | after do 144 | FileUtils.remove_entry tmp_dir 145 | end 146 | 147 | describe 'flags' do 148 | describe '--list flag' do 149 | let(:flags) { '--list' } 150 | 151 | before do 152 | create_manifests 153 | end 154 | 155 | context 'default manifest' do 156 | it 'emits a table of contents' do 157 | output = run_packager_binary(buildpack_dir, flags) 158 | stdout = output.first 159 | 160 | expect(stdout).to match(/fake_name.*1\.2/) 161 | expect(stdout).to match(/------/) # it's a table! 162 | end 163 | 164 | context 'and there are modules' do 165 | it 'emit a table for modules' do 166 | output = run_packager_binary(buildpack_dir, flags) 167 | stdout = output.first 168 | 169 | expect(stdout).to match /modules/ 170 | expect(stdout).to match /one, three, two/ 171 | end 172 | end 173 | end 174 | 175 | context 'custom manifest' do 176 | let(:flags) { '--list --use-custom-manifest=manifest-including-default-versions.yml' } 177 | 178 | it 'emits a table of contents' do 179 | output = run_packager_binary(buildpack_dir, flags) 180 | stdout = output.first 181 | 182 | expect(stdout).to match(/fake_name.*1\.1/) 183 | expect(stdout).to match(/fake_name.*1\.2/) 184 | expect(stdout).to match(/------/) # it's a table! 185 | end 186 | 187 | context 'and there are no modules' do 188 | it 'ensures there is no modules column' do 189 | output = run_packager_binary(buildpack_dir, flags) 190 | stdout = output.first 191 | 192 | expect(stdout).to_not match /modules/ 193 | end 194 | end 195 | end 196 | end 197 | 198 | describe '--defaults flag' do 199 | let(:flags) { '--defaults' } 200 | 201 | before do 202 | create_manifests 203 | end 204 | 205 | context 'default manifest with no default_versions' do 206 | it 'emits an empty table' do 207 | output = run_packager_binary(buildpack_dir, flags) 208 | stdout = output.first 209 | 210 | expect(stdout).to match(/ name | version/) 211 | expect(stdout).to match(/------/) # it's a table! 212 | expect(stdout.split("\m").length).to eq(2) 213 | end 214 | end 215 | 216 | context 'custom manifest with default_versions' do 217 | let(:flags) { '--defaults --use-custom-manifest=manifest-including-default-versions.yml' } 218 | 219 | it 'emits a table of the manifest specified default dependency versions' do 220 | output = run_packager_binary(buildpack_dir, flags) 221 | stdout = output.first 222 | 223 | expect(stdout).to match(/ name | version/) 224 | expect(stdout).to match(/fake_name.*1\.2/) 225 | expect(stdout).to match(/------/) # it's a table! 226 | end 227 | end 228 | end 229 | 230 | describe '--use-custom-manifest' do 231 | let(:flags) { '--uncached --any-stack' } 232 | 233 | before do 234 | create_manifests 235 | end 236 | 237 | context 'with the flag' do 238 | let(:flags) { '--uncached --any-stack --use-custom-manifest=manifest-including-default-versions.yml' } 239 | 240 | it 'uses the specified manifest' do 241 | run_packager_binary(buildpack_dir, flags) 242 | 243 | manifest_location = File.join(Dir.mktmpdir, 'manifest.yml') 244 | zip_file_path = File.join(buildpack_dir, 'sample_buildpack-v1.2.3.zip') 245 | 246 | Zip::File.open(zip_file_path) do |zip_file| 247 | generated_manifest = zip_file.find { |file| file.name == 'manifest.yml' } 248 | generated_manifest.extract(manifest_location) 249 | end 250 | 251 | manifest_contents = YAML.load_file(manifest_location) 252 | file_manifest_contents = YAML.load_file(File.join(buildpack_dir, 'manifest-including-default-versions.yml')) 253 | expect(manifest_contents).to eq(file_manifest_contents) 254 | end 255 | end 256 | 257 | context 'without the flag' do 258 | it 'uses the skinny manifest' do 259 | run_packager_binary(buildpack_dir, flags) 260 | 261 | manifest_location = File.join(Dir.mktmpdir, 'manifest.yml') 262 | zip_file_path = File.join(buildpack_dir, 'sample_buildpack-v1.2.3.zip') 263 | 264 | Zip::File.open(zip_file_path) do |zip_file| 265 | generated_manifest = zip_file.find { |file| file.name == 'manifest.yml' } 266 | generated_manifest.extract(manifest_location) 267 | end 268 | 269 | manifest_contents = YAML.load_file(manifest_location) 270 | file_manifest_contents = YAML.load_file(File.join(buildpack_dir, 'manifest.yml')) 271 | expect(manifest_contents).to eq(file_manifest_contents) 272 | end 273 | end 274 | end 275 | end 276 | 277 | context 'without a manifest' do 278 | let(:flags) { '--uncached --any-stack' } 279 | 280 | specify do 281 | output, status = run_packager_binary(buildpack_dir, flags) 282 | 283 | expect(output).to include('Could not find manifest.yml') 284 | expect(status).not_to be_success 285 | end 286 | end 287 | 288 | context 'with an invalid manifest' do 289 | let(:flags) { '--uncached --any-stack' } 290 | 291 | before do 292 | create_invalid_manifest 293 | end 294 | 295 | specify do 296 | output, status = run_packager_binary(buildpack_dir, flags) 297 | 298 | expect(output).to include('conform to the schema') 299 | expect(status).not_to be_success 300 | end 301 | end 302 | 303 | describe 'usage' do 304 | context 'without a flag' do 305 | let(:flags) { '' } 306 | specify do 307 | output, status = run_packager_binary(buildpack_dir, flags) 308 | 309 | expect(output).to include('USAGE: buildpack-packager < --cached | --uncached | --list | --defaults >') 310 | expect(status).not_to be_success 311 | end 312 | end 313 | 314 | context 'with an invalid flag' do 315 | let(:flags) { 'beast' } 316 | 317 | it 'reports proper usage' do 318 | output, status = run_packager_binary(buildpack_dir, flags) 319 | 320 | expect(output).to include('USAGE: buildpack-packager < --cached | --uncached | --list | --defaults >') 321 | expect(status).not_to be_success 322 | end 323 | end 324 | end 325 | 326 | context 'with a manifest' do 327 | before do 328 | create_manifest 329 | end 330 | 331 | describe 'the zip file contents' do 332 | context 'an uncached buildpack' do 333 | let(:flags) { '--uncached --any-stack' } 334 | 335 | specify do 336 | output, status = run_packager_binary(buildpack_dir, flags) 337 | 338 | zip_file_path = File.join(buildpack_dir, 'sample_buildpack-v1.2.3.zip') 339 | zip_contents = get_zip_contents(zip_file_path) 340 | 341 | expect(zip_contents).to match_array(files_to_include) 342 | expect(status).to be_success 343 | end 344 | end 345 | 346 | context 'a cached buildpack' do 347 | let(:flags) { '--cached --any-stack' } 348 | 349 | specify do 350 | _, status = run_packager_binary(buildpack_dir, flags) 351 | 352 | zip_file_path = File.join(buildpack_dir, 'sample_buildpack-cached-v1.2.3.zip') 353 | zip_contents = get_zip_contents(zip_file_path) 354 | 355 | dependencies_in_manifest = YAML.load_file(File.join(buildpack_dir, 'manifest.yml'))['dependencies'] 356 | 357 | dependencies_with_translation = dependencies_in_manifest 358 | .map { |dep| "file://#{remote_dependencies_dir}/#{dep['uri'].split('/').last}" } 359 | .map { |path| path.gsub(/[:\/]/, '_') } 360 | 361 | deps_with_path = dependencies_with_translation.map { |dep| "dependencies/#{dep}" } 362 | 363 | expect(zip_contents).to match_array(files_to_include + deps_with_path) 364 | expect(status).to be_success 365 | end 366 | 367 | context 'vendored dependencies with invalid checksums' do 368 | let(:sha256) { 'InvalidSHA256_123' } 369 | 370 | specify do 371 | stdout, status = run_packager_binary(buildpack_dir, flags) 372 | expect(status).not_to be_success 373 | end 374 | end 375 | end 376 | end 377 | end 378 | end 379 | -------------------------------------------------------------------------------- /spec/integration/buildpack/directory_name_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'buildpack/packager' 3 | 4 | module Buildpack 5 | describe Packager do 6 | let(:fake_file_uri) do 7 | location = File.join(Dir.mktmpdir, 'fake.file') 8 | File.write(location, 'fake text') 9 | "file://#{location}" 10 | end 11 | 12 | let(:manifest_path) do 13 | location = File.join(Dir.mktmpdir, 'fake.manifest') 14 | File.write(location, {language: 'text', 15 | dependencies: []}.to_yaml) 16 | location 17 | end 18 | 19 | let(:sha256) { Digest::SHA256.file(fake_file_uri.gsub(/file:\/\//, '')).hexdigest } 20 | 21 | let(:manifest) do 22 | { 23 | language: 'fake', 24 | exclude_files: [], 25 | dependencies: [ 26 | { 27 | 'name' => 'fake', 28 | 'uri' => fake_file_uri, 29 | 'sha256' => sha256 30 | } 31 | ] 32 | } 33 | end 34 | 35 | let(:mode) { :uncached } 36 | let(:cache_dir) { '' } 37 | 38 | let(:root_dir) do 39 | dir_name = Dir.mktmpdir('fake-buildpack') 40 | File.write(File.join(dir_name, 'mock.txt'), 'fake!') 41 | dir_name 42 | end 43 | 44 | let(:options) do 45 | { 46 | root_dir: root_dir, 47 | manifest_path: manifest_path, 48 | mode: mode, 49 | stack: :any_stack, 50 | force_download: false, 51 | cache_dir: cache_dir 52 | } 53 | end 54 | 55 | before do 56 | @pwd ||= Dir.pwd 57 | Dir.chdir(root_dir) 58 | end 59 | 60 | after do 61 | Dir.chdir(@pwd) 62 | end 63 | 64 | describe 'directory naming structure' do 65 | before do 66 | allow_any_instance_of(Packager::Package).to receive(:buildpack_version).and_return('1.0.0') 67 | allow_any_instance_of(Packager::Package).to receive(:manifest).and_return(manifest) 68 | allow(FileUtils).to receive(:cp) 69 | Packager.package(options) 70 | end 71 | 72 | context 'directory has no space' do 73 | let(:root_dir) do 74 | dir_name = Dir.mktmpdir('nospace') 75 | File.write(File.join(dir_name, 'mock.txt'), "don't read this") 76 | dir_name 77 | end 78 | 79 | it 'puts the zip file in the right place' do 80 | expect(File.exist?(File.join(root_dir, 'fake_buildpack-v1.0.0.zip'))).to be(true) 81 | end 82 | end 83 | 84 | context 'directory has a space' do 85 | let(:root_dir) do 86 | dir_name = Dir.mktmpdir('a space') 87 | File.write(File.join(dir_name, 'mock.txt'), "don't read this") 88 | dir_name 89 | end 90 | 91 | it 'it puts the zip file in the right place' do 92 | expect(File.exist?(File.join(root_dir, 'fake_buildpack-v1.0.0.zip'))).to be(true) 93 | end 94 | end 95 | end 96 | end 97 | end 98 | -------------------------------------------------------------------------------- /spec/integration/buildpack/packager_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'tmpdir' 3 | 4 | module Buildpack 5 | describe Packager do 6 | let(:tmp_dir) do 7 | dir = FileUtils.mkdir_p(File.join(Dir.mktmpdir, rand.to_s[2..-1]))[0] 8 | puts dir 9 | dir 10 | end 11 | let(:buildpack_dir) { File.join(tmp_dir, 'sample-buildpack-root-dir') } 12 | let(:stack) { :any_stack } 13 | let(:cache_dir) { File.join(tmp_dir, 'cache-dir') } 14 | 15 | def file_location(id = '') 16 | location = File.join(tmp_dir, "sample_host#{id}") 17 | File.write(location, 'contents!') 18 | location 19 | end 20 | 21 | let(:translated_file_location) { 'file___' + file_location.gsub(/[:\/]/, '_') } 22 | 23 | let(:sha256) { Digest::SHA256.file(file_location).hexdigest } 24 | 25 | let(:options) do 26 | { 27 | root_dir: buildpack_dir, 28 | mode: buildpack_mode, 29 | stack: stack, 30 | cache_dir: cache_dir, 31 | manifest_path: manifest_path 32 | } 33 | end 34 | 35 | let(:dependencies) { 36 | [{ 37 | 'version' => '1.0', 38 | 'name' => 'etc_host', 39 | 'sha256' => sha256, 40 | 'uri' => "file://#{file_location}", 41 | 'cf_stacks' => ['cflinuxfs2'] 42 | }] 43 | } 44 | let(:manifest_path) { 'manifest.yml' } 45 | let(:manifest) do 46 | { 47 | exclude_files: [], 48 | language: 'sample', 49 | url_to_dependency_map: [ 50 | { 51 | match: 'ruby-(d+.d+.d+)', 52 | name: 'ruby', 53 | version: '$1' 54 | } 55 | ], 56 | dependencies: dependencies 57 | } 58 | end 59 | 60 | let(:buildpack_files) do 61 | [ 62 | 'VERSION', 63 | 'README.md', 64 | 'lib/sai.to', 65 | 'lib/rash', 66 | 'log/log.txt', 67 | 'first-level/log/log.txt', 68 | 'log.txt', 69 | 'blog.txt', 70 | 'blog/blog.txt', 71 | '.gitignore', 72 | '.gitmodules', 73 | 'lib/.git' 74 | ] 75 | end 76 | 77 | let(:git_files) { ['.gitignore', '.gitmodules', 'lib/.git'] } 78 | let(:cached_file) { File.join(cache_dir, translated_file_location) } 79 | 80 | def create_manifest(options = {}) 81 | manifest.merge!(options) 82 | File.write(File.join(buildpack_dir, manifest_path), manifest.to_yaml) 83 | end 84 | 85 | before do 86 | make_fake_files(buildpack_dir, buildpack_files) 87 | buildpack_files << 'manifest.yml' 88 | create_manifest 89 | `echo "1.2.3" > #{File.join(buildpack_dir, 'VERSION')}` 90 | 91 | @pwd ||= Dir.pwd 92 | Dir.chdir(buildpack_dir) 93 | end 94 | 95 | after do 96 | Dir.chdir(@pwd) 97 | FileUtils.remove_entry tmp_dir 98 | end 99 | 100 | describe '#list' do 101 | let(:buildpack_mode) { :list } 102 | 103 | context 'default manifest.yml' do 104 | specify do 105 | create_manifest 106 | table = Packager.list(options) 107 | expect(table.to_s).to match(/etc_host.*1\.0.*cflinuxfs2/) 108 | end 109 | end 110 | 111 | context 'alternate manifest path' do 112 | let(:manifest_path) { 'my-manifest.yml' } 113 | 114 | specify do 115 | create_manifest 116 | table = Packager.list(options) 117 | expect(table.to_s).to match(/etc_host.*1\.0.*cflinuxfs2/) 118 | end 119 | end 120 | 121 | context 'sorted output' do 122 | def create_manifest_dependency_skeleton(dependencies) 123 | manifest = {} 124 | manifest['dependencies'] = [] 125 | dependencies.each do |dependency| 126 | manifest['dependencies'].push('name' => dependency.first, 127 | 'version' => dependency.last, 128 | 'cf_stacks' => ['cflinuxfs2']) 129 | end 130 | File.write(File.join(buildpack_dir, manifest_path), manifest.to_yaml) 131 | end 132 | 133 | %w(go hhvm jruby node php python ruby).each do |interpreter| 134 | it "sorts #{interpreter} interpreter first" do 135 | create_manifest_dependency_skeleton([ 136 | ['aaaaa', '1.0'], 137 | [interpreter, '1.0'], 138 | ['zzzzz', '1.0'] 139 | ]) 140 | table = Packager.list(options) 141 | stdout = table.to_s.split("\n") 142 | 143 | position_of_a = stdout.index(stdout.grep(/aaaaa/).first) 144 | position_of_interpreter = stdout.index(stdout.grep(/ #{interpreter} /).first) 145 | position_of_z = stdout.index(stdout.grep(/zzzzz/).first) 146 | 147 | expect(position_of_interpreter).to be < position_of_a 148 | expect(position_of_interpreter).to be < position_of_z 149 | end 150 | end 151 | 152 | it 'sorts using `name` as secondary key' do 153 | create_manifest_dependency_skeleton([ 154 | ['b_foobar', '1.0'], 155 | ['a_foobar', '1.0'], 156 | ['c_foobar', '1.0'] 157 | ]) 158 | table = Packager.list(options) 159 | stdout = table.to_s.split("\n") 160 | 161 | position_of_a = stdout.index(stdout.grep(/a_foobar/).first) 162 | position_of_b = stdout.index(stdout.grep(/b_foobar/).first) 163 | position_of_c = stdout.index(stdout.grep(/c_foobar/).first) 164 | 165 | expect(position_of_a).to be < position_of_b 166 | expect(position_of_b).to be < position_of_c 167 | end 168 | 169 | it 'sorts using `version` as secondary key' do 170 | create_manifest_dependency_skeleton([ 171 | ['foobar', '1.1'], 172 | ['foobar', '1.2'], 173 | ['foobar', '1.0'] 174 | ]) 175 | table = Packager.list(options) 176 | stdout = table.to_s.split("\n") 177 | 178 | position_of_10 = stdout.index(stdout.grep(/1\.0/).first) 179 | position_of_11 = stdout.index(stdout.grep(/1\.1/).first) 180 | position_of_12 = stdout.index(stdout.grep(/1\.2/).first) 181 | 182 | expect(position_of_10).to be < position_of_11 183 | expect(position_of_11).to be < position_of_12 184 | end 185 | end 186 | end 187 | 188 | describe 'a well formed zip file name' do 189 | context 'an uncached buildpack' do 190 | let(:buildpack_mode) { :uncached } 191 | 192 | specify do 193 | Packager.package(options) 194 | 195 | expect(all_files(buildpack_dir)).to include('sample_buildpack-v1.2.3.zip') 196 | end 197 | end 198 | 199 | context 'a cached buildpack' do 200 | let(:buildpack_mode) { :cached } 201 | 202 | specify do 203 | puts `ls #{buildpack_dir}` 204 | 205 | puts Packager.package(options) 206 | 207 | expect(all_files(buildpack_dir)).to include('sample_buildpack-cached-v1.2.3.zip') 208 | end 209 | end 210 | end 211 | 212 | describe 'the zip file contents' do 213 | context 'an uncached buildpack' do 214 | let(:buildpack_mode) { :uncached } 215 | 216 | specify do 217 | Packager.package(options) 218 | 219 | zip_file_path = File.join(buildpack_dir, 'sample_buildpack-v1.2.3.zip') 220 | zip_contents = get_zip_contents(zip_file_path) 221 | 222 | expect(zip_contents).to match_array(buildpack_files - git_files) 223 | end 224 | 225 | context 'with stack set' do 226 | let(:stack) { 'cflinuxfs2' } 227 | let(:cflinuxfs2_dependency) { { 228 | 'version' => '1.0', 229 | 'name' => 'fs2_dep', 230 | 'sha256' => sha256 + '2', 231 | 'uri' => "file://#{file_location('_cflinuxfs2')}", 232 | 'cf_stacks' => ['cflinuxfs2'] 233 | } } 234 | let(:dependencies) { 235 | [ 236 | cflinuxfs2_dependency, 237 | { 238 | 'version' => '1.0', 239 | 'name' => 'fs3_dep', 240 | 'sha256' => sha256 + '3', 241 | 'uri' => "file://#{file_location('_cflinuxfs3')}", 242 | 'cf_stacks' => ['cflinuxfs3'] 243 | } 244 | ] 245 | } 246 | 247 | it "sets stack on manifest" do 248 | Packager.package(options) 249 | 250 | zip_file_path = File.join(buildpack_dir, 'sample_buildpack-cflinuxfs2-v1.2.3.zip') 251 | new_manifest = get_manifest_from_zip(zip_file_path) 252 | 253 | expect(new_manifest[:stack]).to eq("cflinuxfs2") 254 | end 255 | 256 | it "doesn't set cf_stacks on deps" do 257 | Packager.package(options) 258 | 259 | zip_file_path = File.join(buildpack_dir, 'sample_buildpack-cflinuxfs2-v1.2.3.zip') 260 | new_manifest = get_manifest_from_zip(zip_file_path) 261 | 262 | new_manifest[:dependencies].each do |dep| 263 | expect(dep['cf_stacks']).to be_nil 264 | end 265 | end 266 | 267 | it "excludes deps that don't match stack from manifest" do 268 | Packager.package(options) 269 | 270 | zip_file_path = File.join(buildpack_dir, 'sample_buildpack-cflinuxfs2-v1.2.3.zip') 271 | new_manifest = get_manifest_from_zip(zip_file_path) 272 | 273 | expect(new_manifest[:dependencies].map { |dep| dep['name'] }).to match_array([cflinuxfs2_dependency['name']]) 274 | end 275 | end 276 | end 277 | 278 | context 'a cached buildpack' do 279 | let(:buildpack_mode) { :cached } 280 | 281 | specify do 282 | Packager.package(options) 283 | 284 | zip_file_path = File.join(buildpack_dir, 'sample_buildpack-cached-v1.2.3.zip') 285 | zip_contents = get_zip_contents(zip_file_path) 286 | dependencies = ["dependencies/#{translated_file_location}"] 287 | 288 | expect(zip_contents).to match_array(buildpack_files + dependencies - git_files) 289 | end 290 | 291 | context 'set the stack' do 292 | let(:stack) { 'cflinuxfs3' } 293 | let(:dependencies) { 294 | [{ 295 | 'version' => '1.0', 296 | 'name' => 'etc_host', 297 | 'sha256' => sha256, 298 | 'uri' => "file://#{file_location('_cflinuxfs2')}", 299 | 'cf_stacks' => ['cflinuxfs2'] 300 | }, { 301 | 'version' => '1.0', 302 | 'name' => 'etc_host', 303 | 'sha256' => sha256, 304 | 'uri' => "file://#{file_location('_cflinuxfs3')}", 305 | 'cf_stacks' => ['cflinuxfs3'] 306 | }] 307 | } 308 | specify do 309 | Packager.package(options) 310 | 311 | zip_file_path = File.join(buildpack_dir, 'sample_buildpack-cached-cflinuxfs3-v1.2.3.zip') 312 | zip_contents = get_zip_contents(zip_file_path) 313 | dependencies = ["dependencies/#{translated_file_location}_cflinuxfs3"] 314 | 315 | expect(zip_contents).to match_array(buildpack_files + dependencies - git_files) 316 | end 317 | end 318 | end 319 | end 320 | 321 | describe 'excluded files' do 322 | let(:buildpack_mode) { :uncached } 323 | 324 | context 'when specifying files for exclusion' do 325 | it 'excludes .git files from zip files' do 326 | create_manifest(exclude_files: ['.gitignore']) 327 | Packager.package(options) 328 | 329 | zip_file_path = File.join(buildpack_dir, 'sample_buildpack-v1.2.3.zip') 330 | zip_contents = get_zip_contents(zip_file_path) 331 | 332 | expect(zip_contents).to_not include('.gitignore') 333 | expect(zip_contents).to_not include('.gitmodules') 334 | expect(zip_contents).to_not include('lib/.git') 335 | end 336 | end 337 | 338 | context 'when using a directory pattern in exclude_files' do 339 | it 'excludes directories with that name' do 340 | create_manifest(exclude_files: ['log/']) 341 | Packager.package(options) 342 | 343 | zip_file_path = File.join(buildpack_dir, 'sample_buildpack-v1.2.3.zip') 344 | zip_contents = get_zip_contents(zip_file_path) 345 | 346 | expect(zip_contents).to_not include('first-level/log/log.txt') 347 | expect(zip_contents).to_not include('log/log.txt') 348 | expect(zip_contents).to include('blog/blog.txt') 349 | expect(zip_contents).to include('log.txt') 350 | end 351 | end 352 | 353 | context 'when using glob patterns in exclude_files' do 354 | it 'can accept glob patterns' do 355 | create_manifest(exclude_files: ['*log.txt']) 356 | Packager.package(options) 357 | 358 | zip_file_path = File.join(buildpack_dir, 'sample_buildpack-v1.2.3.zip') 359 | zip_contents = get_zip_contents(zip_file_path) 360 | 361 | expect(zip_contents).to_not include('log.txt') 362 | expect(zip_contents).to_not include('log/log.txt') 363 | expect(zip_contents).to_not include('first-level/log/log.txt') 364 | expect(zip_contents).to_not include('blog/blog.txt') 365 | expect(zip_contents).to_not include('blog.txt') 366 | end 367 | 368 | it 'does not do fuzzy matching by default' do 369 | create_manifest(exclude_files: ['log.txt']) 370 | Packager.package(options) 371 | 372 | zip_file_path = File.join(buildpack_dir, 'sample_buildpack-v1.2.3.zip') 373 | zip_contents = get_zip_contents(zip_file_path) 374 | 375 | expect(zip_contents).to_not include('log.txt') 376 | expect(zip_contents).to include('blog.txt') 377 | end 378 | end 379 | end 380 | 381 | describe 'caching of dependencies' do 382 | context 'an uncached buildpack' do 383 | let(:buildpack_mode) { :uncached } 384 | 385 | specify do 386 | Packager.package(options) 387 | 388 | expect(File).to_not exist(cached_file) 389 | end 390 | end 391 | 392 | context 'a cached buildpack' do 393 | let(:buildpack_mode) { :cached } 394 | 395 | context 'by default' do 396 | specify do 397 | Packager.package(options) 398 | expect(File).to exist(cached_file) 399 | end 400 | end 401 | 402 | context 'with the force download enabled' do 403 | context 'and the cached file does not exist' do 404 | it 'will overwrite the cache file' do 405 | expect(File).to_not exist(cached_file) 406 | 407 | Packager.package(options.merge(force_download: true)) 408 | 409 | expect(File).to exist(cached_file) 410 | expect(Digest::SHA256.file(cached_file).hexdigest).to eq sha256 411 | end 412 | end 413 | 414 | context 'and the request fails' do 415 | let(:file_location) { 'fake-file-that-no-one-should-have.txt' } 416 | let(:sha256) { nil } 417 | 418 | it 'does not cache the file' do 419 | expect(File).to_not exist(cached_file) 420 | 421 | expect do 422 | Packager.package(options.merge(force_download: true)) 423 | end.to raise_error(RuntimeError) 424 | 425 | expect(File).to_not exist(cached_file) 426 | end 427 | 428 | it 'raises an error about a failed download' do 429 | expect do 430 | Packager.package(options.merge(force_download: true)) 431 | end.to raise_error(RuntimeError, 'Failed to download file from file://fake-file-that-no-one-should-have.txt') 432 | end 433 | end 434 | 435 | context 'on subsequent calls' do 436 | context 'and they are successful' do 437 | it 'does not use the cached file and overwrites it' do 438 | Packager.package(options.merge(force_download: true)) 439 | File.write(cached_file, 'asdf') 440 | 441 | Packager.package(options.merge(force_download: true)) 442 | expect(Digest::SHA256.file(cached_file).hexdigest).to eq sha256 443 | end 444 | end 445 | 446 | context 'and they fail' do 447 | it 'does not override the cached file' do 448 | Packager.package(options.merge(force_download: true)) 449 | File.write(cached_file, 'asdf') 450 | 451 | File.delete(file_location) 452 | 453 | expect do 454 | Packager.package(options.merge(force_download: true)) 455 | end.to raise_error(RuntimeError) 456 | 457 | expect(File.read(cached_file)).to eq 'asdf' 458 | end 459 | end 460 | end 461 | end 462 | 463 | context 'with the force download disabled' do 464 | context 'and the cached file does not exist' do 465 | it 'will write the cache file' do 466 | Packager.package(options.merge(force_download: false)) 467 | expect(File).to exist(cached_file) 468 | end 469 | end 470 | 471 | context 'on subsequent calls' do 472 | it 'does use the cached file' do 473 | Packager.package(options.merge(force_download: false)) 474 | 475 | expect_any_instance_of(Packager::Package).not_to receive(:download_file) 476 | Packager.package(options.merge(force_download: false)) 477 | end 478 | end 479 | end 480 | end 481 | end 482 | 483 | describe 'when checking checksums' do 484 | context 'with an invalid SHA256' do 485 | let(:sha256) { 'wompwomp' } 486 | 487 | context 'in cached mode' do 488 | let(:buildpack_mode) { :cached } 489 | 490 | it 'raises an error' do 491 | expect do 492 | Packager.package(options) 493 | end.to raise_error(Packager::CheckSumError) 494 | end 495 | end 496 | 497 | context 'in uncached mode' do 498 | let(:buildpack_mode) { :uncached } 499 | 500 | it 'does not raise an error' do 501 | expect do 502 | Packager.package(options) 503 | end.to_not raise_error 504 | end 505 | end 506 | end 507 | end 508 | 509 | describe 'existence of zip' do 510 | let(:buildpack_mode) { :uncached } 511 | 512 | context 'zip is installed' do 513 | specify do 514 | expect { Packager.package(options) }.not_to raise_error 515 | end 516 | end 517 | 518 | context 'zip is not installed' do 519 | before do 520 | allow(Open3).to receive(:capture3) 521 | .with('which zip') 522 | .and_return(['', '', 'exit 1']) 523 | end 524 | 525 | specify do 526 | expect { Packager.package(options) }.to raise_error(RuntimeError) 527 | end 528 | end 529 | end 530 | 531 | describe 'avoid changing state of buildpack folder, other than creating the artifact (.zip)' do 532 | context 'create an cached buildpack' do 533 | let(:buildpack_mode) { :cached } 534 | 535 | specify 'user does not see dependencies directory in their buildpack folder' do 536 | Packager.package(options) 537 | 538 | expect(all_files(buildpack_dir)).not_to include('dependencies') 539 | end 540 | end 541 | end 542 | 543 | describe 'pre package script' do 544 | let(:buildpack_mode) { :uncached } 545 | 546 | before do 547 | FileUtils.mkdir_p(File.join(buildpack_dir, 'scripts')) 548 | script_path = File.join(buildpack_dir, 'scripts/run.sh') 549 | File.write(script_path, 'mkdir .cloudfoundry && touch .cloudfoundry/hwc.exe') 550 | File.chmod(0755, script_path) 551 | end 552 | 553 | it 'runs the pre package script if specified' do 554 | create_manifest(pre_package: 'scripts/run.sh') 555 | Packager.package(options) 556 | 557 | zip_file_path = File.join(buildpack_dir, 'sample_buildpack-v1.2.3.zip') 558 | zip_contents = get_zip_contents(zip_file_path) 559 | 560 | expect(zip_contents).to include('.cloudfoundry/hwc.exe') 561 | end 562 | 563 | it 'does not run the pre package script if not specified' do 564 | create_manifest(pre_package: nil) 565 | Packager.package(options) 566 | 567 | zip_file_path = File.join(buildpack_dir, 'sample_buildpack-v1.2.3.zip') 568 | zip_contents = get_zip_contents(zip_file_path) 569 | 570 | expect(zip_contents).not_to include('.cloudfoundry/hwc.exe') 571 | end 572 | end 573 | end 574 | end 575 | -------------------------------------------------------------------------------- /spec/integration/default_versions_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'fileutils' 3 | 4 | describe 'Buildpack packager default_versions validation' do 5 | let(:flags) { '--uncached --any-stack' } 6 | let(:buildpack_dir) { Dir.mktmpdir } 7 | let(:cache_dir) { Dir.mktmpdir } 8 | let(:base_manifest_contents) { <<-BASE 9 | language: python 10 | url_to_dependency_map: [] 11 | exclude_files: [] 12 | BASE 13 | } 14 | 15 | before do 16 | Dir.chdir(buildpack_dir) do 17 | File.write('manifest.yml', manifest) 18 | File.write('VERSION', '1.7.8') 19 | end 20 | end 21 | 22 | after do 23 | FileUtils.rm_rf(buildpack_dir) 24 | FileUtils.rm_rf(cache_dir) 25 | end 26 | 27 | shared_examples_for "general output that helps with the error is produced" do 28 | it "outputs a link to the Cloud Foundry custom buildpacks page" do 29 | output, status = run_packager_binary(buildpack_dir, flags) 30 | expect(output).to include("For more information, see https://docs.cloudfoundry.org/buildpacks/custom.html#specifying-default-versions") 31 | end 32 | 33 | it "states the buildpack manifest is malformed" do 34 | output, status = run_packager_binary(buildpack_dir, flags) 35 | expect(output).to include("The buildpack manifest is malformed:") 36 | end 37 | end 38 | 39 | context 'defaults and dependencies are in agreement' do 40 | let(:manifest) {<<-MANIFEST 41 | #{base_manifest_contents} 42 | default_versions: 43 | - name: python 44 | version: 3.3.5 45 | - name: pip 46 | version: 3.x 47 | - name: ruby 48 | version: 5.5.x 49 | dependencies: 50 | - name: python 51 | version: 3.3.5 52 | uri: http://example.com/ 53 | sha256: aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f 54 | cf_stacks: 55 | - cflinuxfs2 56 | - name: pip 57 | version: 3.3.4 58 | uri: http://example.com/ 59 | sha256: aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f 60 | cf_stacks: 61 | - cflinuxfs2 62 | - name: pip 63 | version: 3.3.2 64 | uri: http://example.com/ 65 | sha256: aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f 66 | cf_stacks: 67 | - cflinuxfs2 68 | - name: ruby 69 | version: 5.4.3 70 | uri: http://example.com/ 71 | sha256: aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f 72 | cf_stacks: 73 | - cflinuxfs2 74 | - name: ruby 75 | version: 5.5.3 76 | uri: http://example.com/ 77 | sha256: aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f 78 | cf_stacks: 79 | - cflinuxfs2 80 | MANIFEST 81 | } 82 | 83 | it 'emits no errors' do 84 | stdout, status = run_packager_binary(buildpack_dir, flags) 85 | 86 | puts stdout 87 | expect(status).to be_success 88 | end 89 | end 90 | 91 | context 'multiple default versions for a dependency' do 92 | let(:manifest) {<<-MANIFEST 93 | #{base_manifest_contents} 94 | default_versions: 95 | - name: python 96 | version: 3.3.5 97 | - name: python 98 | version: 3.3.2 99 | - name: pip 100 | version: 7.7.x 101 | - name: pip 102 | version: 7.7.2 103 | - name: ruby 104 | version: 4.x 105 | - name: ruby 106 | version: 4.3.2 107 | dependencies: 108 | - name: python 109 | uri: https://a.org 110 | version: 3.3.5 111 | sha256: 3a2 112 | cf_stacks: [cflinuxfs2] 113 | - name: python 114 | uri: https://a.org 115 | version: 3.3.2 116 | sha256: 3a2 117 | cf_stacks: [cflinuxfs2] 118 | - name: ruby 119 | uri: https://a.org 120 | version: 4.3.5 121 | sha256: 3a2 122 | cf_stacks: [cflinuxfs2] 123 | - name: ruby 124 | uri: https://a.org 125 | version: 4.3.2 126 | sha256: 3a2 127 | cf_stacks: [cflinuxfs2] 128 | - name: pip 129 | uri: https://a.org 130 | version: 7.7.7 131 | sha256: 3a2 132 | cf_stacks: [cflinuxfs2] 133 | - name: pip 134 | uri: https://a.org 135 | version: 7.7.2 136 | sha256: 3a2 137 | cf_stacks: [cflinuxfs2] 138 | MANIFEST 139 | } 140 | 141 | it_behaves_like "general output that helps with the error is produced" 142 | 143 | it 'fails and errors stating the context' do 144 | output, status = run_packager_binary(buildpack_dir, flags) 145 | 146 | expect(output).to include("python had more " + 147 | "than one 'default_versions' entry in the buildpack manifest.") 148 | expect(output).to include("ruby had more " + 149 | "than one 'default_versions' entry in the buildpack manifest.") 150 | expect(output).to include("pip had more " + 151 | "than one 'default_versions' entry in the buildpack manifest.") 152 | expect(status).to_not be_success 153 | end 154 | end 155 | 156 | context 'no dependency with name found for default in manifest' do 157 | let(:manifest) {<<-MANIFEST 158 | #{base_manifest_contents} 159 | default_versions: 160 | - name: python 161 | version: 3.3.5 162 | - name: ruby 163 | version: 4.x 164 | - name: pip 165 | version: 7.7.x 166 | dependencies: [] 167 | MANIFEST 168 | } 169 | 170 | it_behaves_like "general output that helps with the error is produced" 171 | 172 | it 'fails and errors stating the context' do 173 | output, status = run_packager_binary(buildpack_dir, flags) 174 | 175 | expect(output).to include("a 'default_versions' entry for python 3.3.5 was specified by the buildpack manifest, " + 176 | "but no 'dependencies' entry for python 3.3.5 was found in the buildpack manifest.") 177 | expect(output).to include("a 'default_versions' entry for ruby 4.x was specified by the buildpack manifest, " + 178 | "but no 'dependencies' entry for ruby 4.x was found in the buildpack manifest.") 179 | expect(output).to include("a 'default_versions' entry for pip 7.7.x was specified by the buildpack manifest, " + 180 | "but no 'dependencies' entry for pip 7.7.x was found in the buildpack manifest.") 181 | expect(status).to_not be_success 182 | end 183 | end 184 | 185 | context 'no dependency with version found for default in manifest' do 186 | let(:manifest) {<<-MANIFEST 187 | #{base_manifest_contents} 188 | default_versions: 189 | - name: ruby 190 | version: 1.1.1 191 | - name: python 192 | version: 3.3.5 193 | - name: pip 194 | version: 7.7.x 195 | dependencies: 196 | - name: ruby 197 | uri: https://a.org 198 | version: 9.9.9 199 | sha256: 3a2 200 | cf_stacks: [cflinuxfs2] 201 | - name: python 202 | uri: https://a.org 203 | version: 9.9.9 204 | sha256: 3a2 205 | cf_stacks: [cflinuxfs2] 206 | - name: pip 207 | uri: https://a.org 208 | version: 9.9.9 209 | sha256: 3a2 210 | cf_stacks: [cflinuxfs2] 211 | MANIFEST 212 | } 213 | 214 | it_behaves_like "general output that helps with the error is produced" 215 | 216 | it 'fails and errors stating the context' do 217 | output, status = run_packager_binary(buildpack_dir, flags) 218 | 219 | expect(output).to include("a 'default_versions' entry for python 3.3.5 was specified by the buildpack manifest, " + 220 | "but no 'dependencies' entry for python 3.3.5 was found in the buildpack manifest.") 221 | expect(output).to include("a 'default_versions' entry for ruby 1.1.1 was specified by the buildpack manifest, " + 222 | "but no 'dependencies' entry for ruby 1.1.1 was found in the buildpack manifest.") 223 | expect(output).to include("a 'default_versions' entry for pip 7.7.x was specified by the buildpack manifest, " + 224 | "but no 'dependencies' entry for pip 7.7.x was found in the buildpack manifest.") 225 | expect(status).to_not be_success 226 | end 227 | end 228 | end 229 | -------------------------------------------------------------------------------- /spec/integration/output_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'fileutils' 3 | 4 | describe 'Buildpack packager output' do 5 | let(:buildpack_type) { '--uncached' } 6 | let(:stack) { '--any-stack' } 7 | let(:fixture_name) { 'buildpack-without-uri-credentials' } 8 | let(:buildpack_fixture) { File.join(File.dirname(__FILE__), '..', 'fixtures', fixture_name) } 9 | let(:tmpdir) { Dir.mktmpdir } 10 | 11 | before do 12 | FileUtils.rm_rf(tmpdir) 13 | end 14 | 15 | def buildpack_packager_execute(buildpack_dir, home_dir) 16 | Dir.chdir(buildpack_dir) do 17 | `HOME=#{home_dir} buildpack-packager #{buildpack_type} #{stack}` 18 | end 19 | end 20 | 21 | subject { buildpack_packager_execute(buildpack_fixture, tmpdir) } 22 | 23 | context 'building the uncached buildpack' do 24 | it 'outputs the type of buildpack created, where and its human readable size' do 25 | expect(subject).to include("Uncached buildpack for any stack created and saved as") 26 | expect(subject).to include("spec/fixtures/#{fixture_name}/go_buildpack-v1.7.8.zip") 27 | expect(subject).to match(/with a size of 4\.0K$/) 28 | end 29 | end 30 | 31 | context 'building the cached buildpack' do 32 | let(:buildpack_type) { '--cached' } 33 | 34 | it 'outputs the dependencies downloaded, their versions, and download source url' do 35 | expect(subject).to include("Downloading go version 1.6.3 from: https://buildpacks.cloudfoundry.org/concourse-binaries/go/go1.6.3.linux-amd64.tar.gz") 36 | expect(subject).to include("Using go version 1.6.3 with size") 37 | expect(subject).to include("go version 1.6.3 matches the manifest provided sha256 checksum of 5ac238cd321a66a35c646d61e4cafd922929af0800c78b00375312c76e702e11") 38 | 39 | expect(subject).to include("Downloading godep version v74 from: https://pivotal-buildpacks.s3.amazonaws.com/concourse-binaries/godep/godep-v74-linux-x64.tgz") 40 | expect(subject).to include("Using godep version v74 with size 2.8M") 41 | expect(subject).to include("godep version v74 matches the manifest provided sha256 checksum of 6e6761b71e1518bf7716b3f383f10598afe5ce4592e6eea0184f17b85fd93813") 42 | end 43 | 44 | it 'outputs the type of buildpack created, where and its human readable size' do 45 | expect(subject).to include("Cached buildpack for any stack created and saved as") 46 | expect(subject).to include("spec/fixtures/#{fixture_name}/go_buildpack-cached-v1.7.8.zip") 47 | expect(subject).to match(/with a size of [\d]+M$/) 48 | end 49 | 50 | context 'with a buildpack packager dependency cache intact' do 51 | before { buildpack_packager_execute(buildpack_fixture, tmpdir) } 52 | 53 | it 'outputs the dependencies downloaded, their versions, and cache location' do 54 | expect(subject).to include("Using go version 1.6.3 from local cache at: #{tmpdir}/.buildpack-packager/cache/https___buildpacks.cloudfoundry.org_concourse-binaries_go_go1.6.3.linux-amd64.tar.gz with size") 55 | expect(subject).to include("go version 1.6.3 matches the manifest provided sha256 checksum of 5ac238cd321a66a35c646d61e4cafd922929af0800c78b00375312c76e702e11") 56 | 57 | expect(subject).to include("Using godep version v74 from local cache at: #{tmpdir}/.buildpack-packager/cache/https___pivotal-buildpacks.s3.amazonaws.com_concourse-binaries_godep_godep-v74-linux-x64.tgz with size") 58 | expect(subject).to include("godep version v74 matches the manifest provided sha256 checksum of 6e6761b71e1518bf7716b3f383f10598afe5ce4592e6eea0184f17b85fd93813") 59 | end 60 | end 61 | 62 | context 'with auth credentials in the dependency uri' do 63 | let(:fixture_name) { 'buildpack-with-uri-credentials' } 64 | 65 | it 'outputs the dependencies download source url without the credentials' do 66 | expect(subject).to include('Downloading go version 1.6.3 from: https://-redacted-:-redacted-@buildpacks.cloudfoundry.org/concourse-binaries/go/go1.6.3.linux-amd64.tar.gz') 67 | expect(subject).to include('Downloading godep version v74 from: https://-redacted-:-redacted-@buildpacks.cloudfoundry.org/concourse-binaries/godep/godep-v74-linux-x64.tgz') 68 | end 69 | end 70 | end 71 | end 72 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | $LOAD_PATH << File.expand_path('../../lib', __FILE__) 2 | $LOAD_PATH << File.expand_path('../../spec/helpers', __FILE__) 3 | require 'buildpack/packager' 4 | require 'file_system_helpers' 5 | require 'cache_directory_helpers' 6 | require 'fake_binary_hosting_helpers' 7 | 8 | RSpec.configure do |config| 9 | config.include FileSystemHelpers 10 | config.include CacheDirectoryHelpers 11 | config.include FakeBinaryHostingHelpers 12 | config.filter_run focus: true 13 | config.run_all_when_everything_filtered = true 14 | end 15 | -------------------------------------------------------------------------------- /spec/unit/buildpack/packager/zip_file_excluder_spec.rb: -------------------------------------------------------------------------------- 1 | require 'buildpack/packager/zip_file_excluder' 2 | require 'spec_helper' 3 | require 'tmpdir' 4 | require 'securerandom' 5 | 6 | module Buildpack 7 | module Packager 8 | describe ZipFileExcluder do 9 | describe '#generate_exclusions_from_manifest' do 10 | let(:excluder) { ZipFileExcluder.new } 11 | let(:file1) { SecureRandom.uuid } 12 | let(:file2) { SecureRandom.uuid } 13 | let(:dir) { "#{SecureRandom.uuid}/" } 14 | let(:excluded_files) { [file1, file2] } 15 | subject do 16 | excluder.generate_manifest_exclusions excluded_files 17 | end 18 | 19 | context 'does not include directories' do 20 | it do 21 | is_expected.to eq "-x #{file1} -x \\*/#{file1} -x #{file2} -x \\*/#{file2}" 22 | end 23 | end 24 | 25 | context 'includes directories' do 26 | let(:excluded_files) { [file1, dir] } 27 | it do 28 | is_expected.to eq "-x #{file1} -x \\*/#{file1} -x #{dir}\\* -x \\*/#{dir}\\*" 29 | end 30 | end 31 | 32 | it 'uses short flags to support zip 2.3.*, see [#107948062]' do 33 | is_expected.to include('-x') 34 | is_expected.to_not include('--exclude') 35 | end 36 | end 37 | end 38 | 39 | describe '#generate_exclusions_from_git_files' do 40 | let(:excluder) { ZipFileExcluder.new } 41 | 42 | context 'git files exist' do 43 | let (:files) { ['.gitignore', '.git/', '.gitmodules', 'lib/.git'] } 44 | 45 | it 'returns an exclusion string with all the git files' do 46 | Dir.mktmpdir do |dir| 47 | Dir.mkdir "#{dir}/lib" 48 | 49 | files.each do |gitfilename| 50 | if gitfilename =~ /.*\/$/ 51 | Dir.mkdir "#{dir}/#{gitfilename}" 52 | else 53 | File.new "#{dir}/#{gitfilename}", 'w' 54 | end 55 | end 56 | 57 | git_exclusions = excluder.generate_exclusions_from_git_files dir 58 | 59 | expect(git_exclusions).to include '-x .gitignore -x \\*/.gitignore' 60 | expect(git_exclusions).to include '-x lib/.git -x \\*/lib/.git' 61 | expect(git_exclusions).to include '-x .gitmodules -x \\*/.gitmodules' 62 | expect(git_exclusions).to include '-x .git/\\* -x \\*/.git/\\*' 63 | end 64 | end 65 | end 66 | end 67 | end 68 | end 69 | -------------------------------------------------------------------------------- /spec/unit/manifest_dependency_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'buildpack/manifest_dependency' 3 | 4 | describe Buildpack::ManifestDependency do 5 | describe '#==' do 6 | context 'two objects with the same name and same version' do 7 | let(:first_dependency) { described_class.new('name', 'version') } 8 | let(:second_dependency) { described_class.new('name', 'version') } 9 | 10 | it 'returns true' do 11 | expect(first_dependency == second_dependency).to be true 12 | end 13 | end 14 | context 'two objects with the same name only' do 15 | let(:first_dependency) { described_class.new('name', 'version 11111111') } 16 | let(:second_dependency) { described_class.new('name', 'version 22222222') } 17 | 18 | it 'returns false' do 19 | expect(first_dependency == second_dependency).to be false 20 | end 21 | end 22 | 23 | context 'two objects with the same version only' do 24 | let(:first_dependency) { described_class.new('name 111', 'version') } 25 | let(:second_dependency) { described_class.new('name 222', 'version') } 26 | 27 | it 'returns false' do 28 | expect(first_dependency == second_dependency).to be false 29 | end 30 | end 31 | 32 | context 'two objects with different names and different versions' do 33 | let(:first_dependency) { described_class.new('name 111111', 'version 11111111') } 34 | let(:second_dependency) { described_class.new('name 222222', 'version 22222222') } 35 | 36 | it 'returns false' do 37 | expect(first_dependency == second_dependency).to be false 38 | end 39 | end 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /spec/unit/manifest_validator_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'buildpack/manifest_validator' 3 | 4 | describe Buildpack::ManifestValidator do 5 | let(:manifest_path) { "#{File.dirname(__FILE__)}/../fixtures/manifests/#{manifest_file_name}" } 6 | let(:validator) { Buildpack::ManifestValidator.new(manifest_path) } 7 | 8 | context 'with a valid manifest' do 9 | let(:manifest_file_name) { 'manifest_valid.yml' } 10 | 11 | it 'reports valid manifests correctly' do 12 | expect(validator.valid?).to be(true) 13 | expect(validator.errors).to be_empty 14 | end 15 | 16 | context 'and deprecation dates' do 17 | let(:manifest_file_name) { 'manifest_valid_plus_deprecation_dates.yml' } 18 | 19 | it 'reports valid manifests correctly' do 20 | expect(validator.valid?).to be(true) 21 | expect(validator.errors).to be_empty 22 | end 23 | end 24 | 25 | context 'and new deprecation dates with no url mapping' do 26 | let(:manifest_file_name) { 'manifest_valid_new_deprecation_dates_no_url_map.yml' } 27 | 28 | it 'reports valid manifests correctly' do 29 | expect(validator.valid?).to be(true) 30 | expect(validator.errors).to be_empty 31 | end 32 | end 33 | 34 | context 'with a manifest with windows stacks' do 35 | let (:manifest_file_name) { 'manifest_windows.yml' } 36 | 37 | it 'reports valid manifests correctly' do 38 | expect(validator.valid?).to be(true) 39 | expect(validator.errors).to be_empty 40 | end 41 | end 42 | 43 | context 'with a manifest with opensus stacks' do 44 | let (:manifest_file_name) { 'manifest_opensus.yml' } 45 | 46 | it 'reports valid manifests correctly' do 47 | expect(validator.valid?).to be(true) 48 | expect(validator.errors).to be_empty 49 | end 50 | end 51 | end 52 | 53 | context 'with a manifest with an invalid sha256 key' do 54 | let(:manifest_file_name) { 'manifest_invalid-sha224.yml' } 55 | 56 | it 'reports invalid manifests correctly' do 57 | expect(validator.valid?).to be(false) 58 | expect(validator.errors[:manifest_parser_errors]).not_to be_empty 59 | end 60 | 61 | context 'and incorrect defaults' do 62 | let(:manifest_file_name) { 'manifest_invalid-sha224_and_defaults.yml' } 63 | 64 | it 'reports manifest parser errors only' do 65 | expect(validator).to_not receive(:validate_default_versions) 66 | expect(validator.valid?).to be(false) 67 | expect(validator.errors[:manifest_parser_errors]).not_to be_empty 68 | end 69 | end 70 | end 71 | end 72 | -------------------------------------------------------------------------------- /spec/unit/packager/package_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'tmpdir' 3 | 4 | module Buildpack 5 | module Packager 6 | describe Package do 7 | let(:packager) { Buildpack::Packager::Package.new(options) } 8 | let(:manifest_path) { 'manifest.yml' } 9 | let(:dependency) { double(:dependency) } 10 | let(:dependencies) { [dependency] } 11 | let(:mode) { :uncached } 12 | let(:stack) { :any_stack } 13 | let(:local_cache_dir) { nil } 14 | let(:force_download) { false } 15 | let(:options) do 16 | { 17 | root_dir: 'root_dir', 18 | mode: mode, 19 | cache_dir: local_cache_dir, 20 | manifest_path: manifest_path, 21 | force_download: force_download, 22 | stack: stack 23 | } 24 | end 25 | 26 | let(:manifest) do 27 | { 28 | language: 'fake_language', 29 | dependencies: dependencies, 30 | exclude_files: ['.DS_Store', '.gitignore'] 31 | } 32 | end 33 | 34 | before do 35 | allow(YAML).to receive(:load_file).and_return(manifest) 36 | end 37 | 38 | describe 'with a specific stack' do 39 | let(:stack) { "cflinuxfs2" } 40 | let(:cflinuxfs2_dependency) { double(:cflinuxfs2_dependency) } 41 | let(:dependencies) { [dependency, cflinuxfs2_dependency] } 42 | 43 | describe '#download_dependencies' do 44 | let(:local_cache_dir) { 'local_cache_dir' } 45 | let(:dependency_dir) { File.join('hello_dir', 'dependencies') } 46 | let(:url_with_parameters) { 'http://some.cdn/with?parameters=true&secondParameter=present' } 47 | 48 | before do 49 | allow(cflinuxfs2_dependency).to receive(:[]) 50 | allow(cflinuxfs2_dependency).to receive(:fetch).with(:cf_stacks, []).and_return(['cflinuxfs2']) 51 | allow(cflinuxfs2_dependency).to receive(:[]).with('uri').and_return('file:///fake_uri_cflinuxfs2.tgz') 52 | 53 | allow(dependency).to receive(:[]) 54 | allow(dependency).to receive(:fetch).with(:cf_stacks, []).and_return([]) 55 | allow(dependency).to receive(:[]).with('uri').and_return('file:///fake_uri.tgz') 56 | 57 | allow(packager).to receive(:ensure_correct_dependency_checksum) 58 | allow(FileUtils).to receive(:cp) 59 | allow(packager).to receive(:download_file) 60 | end 61 | 62 | context 'before dependency has been cached locally' do 63 | before do 64 | allow(File).to receive(:exist?).and_return(false) 65 | end 66 | 67 | it 'downloads the dependency to the local cache' do 68 | expanded_local_file_location = File.expand_path(File.join('local_cache_dir', 'file____fake_uri_cflinuxfs2.tgz')) 69 | expect(packager).to receive(:download_file).with('file:///fake_uri_cflinuxfs2.tgz', expanded_local_file_location) 70 | packager.download_dependencies(dependencies, local_cache_dir, dependency_dir) 71 | end 72 | 73 | it 'copies the dependency from the local cache to the dependency_dir' do 74 | expanded_local_file_location = File.expand_path(File.join('local_cache_dir', 'file____fake_uri_cflinuxfs2.tgz')) 75 | expect(FileUtils).to receive(:cp).with(expanded_local_file_location, dependency_dir) 76 | packager.download_dependencies(dependencies, local_cache_dir, dependency_dir) 77 | end 78 | end 79 | 80 | context 'after dependency has been cached locally' do 81 | before do 82 | allow(File).to receive(:exist?).and_return(true) 83 | end 84 | 85 | it 'does not re-download the dependency' do 86 | expect(packager).not_to receive(:download_file) 87 | packager.download_dependencies(dependencies, local_cache_dir, dependency_dir) 88 | end 89 | 90 | it 'copies the dependency from the local cache to the dependency_dir' do 91 | expanded_local_file_location = File.expand_path(File.join('local_cache_dir', 'file____fake_uri_cflinuxfs2.tgz')) 92 | expect(FileUtils).to receive(:cp).with(expanded_local_file_location, dependency_dir) 93 | packager.download_dependencies(dependencies, local_cache_dir, dependency_dir) 94 | end 95 | end 96 | 97 | context 'with :force_download option active and a locally cached dependency' do 98 | let(:force_download) { true } 99 | 100 | before do 101 | allow(File).to receive(:exist?).and_return(true) 102 | end 103 | 104 | it 're-downloads the dependency anyway' do 105 | expect(packager).to receive(:download_file) 106 | packager.download_dependencies(dependencies, local_cache_dir, dependency_dir) 107 | end 108 | end 109 | 110 | it 'translates ? and & characters in the url to underscores' do 111 | package = Package.new 112 | expect(package.send(:uri_cache_path, url_with_parameters)).to eq("http___some.cdn_with_parameters=true_secondParameter=present") 113 | end 114 | 115 | context 'url has login and password authentication credentials' do 116 | let(:url_with_credentials) { 'http://log!i213:pas!9sword@some.cdn/with' } 117 | 118 | it 'redacts the credentials in the resulting file path' do 119 | package = Package.new 120 | expect(package.send(:uri_without_credentials, url_with_credentials)).to eq("http://-redacted-:-redacted-@some.cdn/with") 121 | end 122 | end 123 | 124 | context 'url has a login authentication credential' do 125 | let(:url_with_credentials) { 'http://log!i213@some.cdn/with' } 126 | 127 | it 'redacts the credential in the resulting file path' do 128 | package = Package.new 129 | expect(package.send(:uri_without_credentials, url_with_credentials)).to eq("http://-redacted-@some.cdn/with") 130 | end 131 | end 132 | end 133 | end 134 | 135 | describe 'with any-stack' do 136 | describe '#download_dependencies' do 137 | let(:local_cache_dir) { 'local_cache_dir' } 138 | let(:dependency_dir) { File.join('hello_dir', 'dependencies') } 139 | let(:url_with_parameters) { 'http://some.cdn/with?parameters=true&secondParameter=present' } 140 | 141 | before do 142 | allow(dependency).to receive(:[]) 143 | allow(dependency).to receive(:[]).with('uri').and_return('file:///fake_uri.tgz') 144 | allow(packager).to receive(:ensure_correct_dependency_checksum) 145 | allow(FileUtils).to receive(:cp) 146 | allow(packager).to receive(:download_file) 147 | end 148 | 149 | context 'before dependency has been cached locally' do 150 | before do 151 | allow(File).to receive(:exist?).and_return(false) 152 | end 153 | 154 | it 'downloads the dependency to the local cache' do 155 | expanded_local_file_location = File.expand_path(File.join('local_cache_dir', 'file____fake_uri.tgz')) 156 | expect(packager).to receive(:download_file).with('file:///fake_uri.tgz', expanded_local_file_location) 157 | packager.download_dependencies([dependency], local_cache_dir, dependency_dir) 158 | end 159 | 160 | it 'copies the dependency from the local cache to the dependency_dir' do 161 | expanded_local_file_location = File.expand_path(File.join('local_cache_dir', 'file____fake_uri.tgz')) 162 | expect(FileUtils).to receive(:cp).with(expanded_local_file_location, dependency_dir) 163 | packager.download_dependencies([dependency], local_cache_dir, dependency_dir) 164 | end 165 | end 166 | 167 | context 'after dependency has been cached locally' do 168 | before do 169 | allow(File).to receive(:exist?).and_return(true) 170 | end 171 | 172 | it 'does not re-download the dependency' do 173 | expect(packager).not_to receive(:download_file) 174 | packager.download_dependencies([dependency], local_cache_dir, dependency_dir) 175 | end 176 | 177 | it 'copies the dependency from the local cache to the dependency_dir' do 178 | expanded_local_file_location = File.expand_path(File.join('local_cache_dir', 'file____fake_uri.tgz')) 179 | expect(FileUtils).to receive(:cp).with(expanded_local_file_location, dependency_dir) 180 | packager.download_dependencies([dependency], local_cache_dir, dependency_dir) 181 | end 182 | end 183 | 184 | context 'with :force_download option active and a locally cached dependency' do 185 | let(:force_download) { true } 186 | 187 | before do 188 | allow(File).to receive(:exist?).and_return(true) 189 | end 190 | 191 | it 're-downloads the dependency anyway' do 192 | expect(packager).to receive(:download_file) 193 | packager.download_dependencies([dependency], local_cache_dir, dependency_dir) 194 | end 195 | end 196 | 197 | it 'translates ? and & characters in the url to underscores' do 198 | package = Package.new 199 | expect(package.send(:uri_cache_path, url_with_parameters)).to eq("http___some.cdn_with_parameters=true_secondParameter=present") 200 | end 201 | 202 | context 'url has login and password authentication credentials' do 203 | let(:url_with_credentials) { 'http://log!i213:pas!9sword@some.cdn/with' } 204 | 205 | it 'redacts the credentials in the resulting file path' do 206 | package = Package.new 207 | expect(package.send(:uri_without_credentials, url_with_credentials)).to eq("http://-redacted-:-redacted-@some.cdn/with") 208 | end 209 | end 210 | 211 | context 'url has a login authentication credential' do 212 | let(:url_with_credentials) { 'http://log!i213@some.cdn/with' } 213 | 214 | it 'redacts the credential in the resulting file path' do 215 | package = Package.new 216 | expect(package.send(:uri_without_credentials, url_with_credentials)).to eq("http://-redacted-@some.cdn/with") 217 | end 218 | end 219 | 220 | context 'file url has authentication credential' do 221 | let(:url_with_credentials) { 'file://log!i213:pas!9sword@/fake_uri.tgz' } 222 | 223 | it 'ignores the credential in the resulting file path' do 224 | package = Package.new 225 | expect(package.send(:uri_without_credentials, url_with_credentials)).to eq('file:///fake_uri.tgz') 226 | end 227 | end 228 | end 229 | end 230 | describe '#copy_buildpack_to_temp_dir' do 231 | context 'with full manifest specified' do 232 | let(:manifest_path) { 'manifest-including-unsupported.yml' } 233 | 234 | before do 235 | allow(FileUtils).to receive(:mv) 236 | allow(FileUtils).to receive(:cp_r) 237 | allow(FileUtils).to receive(:cp) 238 | allow(FileUtils).to receive(:rm) 239 | allow(File).to receive(:open) 240 | allow(YAML).to receive(:load_file) 241 | end 242 | 243 | it 'replaces the default manifest with the full manifest' do 244 | expect(YAML).to receive(:load_file).with('manifest-including-unsupported.yml') { {} } 245 | packager.copy_buildpack_to_temp_dir('hello_dir') 246 | end 247 | end 248 | end 249 | 250 | describe '#build_dependencies' do 251 | let(:mode) { :cached } 252 | 253 | before do 254 | allow(FileUtils).to receive(:mkdir_p) 255 | allow(packager).to receive(:download_dependencies) 256 | end 257 | 258 | context 'when cache_dir is provided' do 259 | let(:local_cache_dir) { 'local_cache_dir' } 260 | 261 | it 'creates the provided cache dir' do 262 | expect(FileUtils).to receive(:mkdir_p).with(local_cache_dir) 263 | packager.build_dependencies('hello_dir') 264 | end 265 | 266 | it 'creates the dependency dir' do 267 | expect(FileUtils).to receive(:mkdir_p).with(File.join('hello_dir', 'dependencies')) 268 | packager.build_dependencies('hello_dir') 269 | end 270 | 271 | it 'calls download_dependencies with right arguments' do 272 | expect(packager).to receive(:download_dependencies).with([dependency], local_cache_dir, File.join('hello_dir', 'dependencies')) 273 | packager.build_dependencies('hello_dir') 274 | end 275 | end 276 | 277 | context 'when cache_dir is NOT provided' do 278 | it 'creates the default cache dir' do 279 | expect(FileUtils).to receive(:mkdir_p).with(File.join(ENV['HOME'], '.buildpack-packager', 'cache')) 280 | packager.build_dependencies('hello_dir') 281 | end 282 | end 283 | end 284 | 285 | describe '#build_zip_file' do 286 | before do 287 | allow(packager).to receive(:buildpack_version).and_return('1.0.0') 288 | allow(FileUtils).to receive(:rm_rf) 289 | allow(packager).to receive(:zip_files) 290 | allow(packager).to receive(:zip_file_path) 291 | .and_return(File.join('root_dir', 'fake_language_buildpack-v1.0.0.zip')) 292 | end 293 | 294 | it 'removes the file at the zip file path' do 295 | zip_file_path = File.join('root_dir', 'fake_language_buildpack-v1.0.0.zip') 296 | expect(FileUtils).to receive(:rm_rf).with(zip_file_path) 297 | packager.build_zip_file('hello_dir') 298 | end 299 | 300 | it 'zips up the temp directory to the zip file path without the excluded files' do 301 | zip_file_path = File.join('root_dir', 'fake_language_buildpack-v1.0.0.zip') 302 | expect(packager).to receive(:zip_files).with('hello_dir', zip_file_path, ['.DS_Store', '.gitignore']) 303 | packager.build_zip_file('hello_dir') 304 | end 305 | end 306 | 307 | describe '#run_pre_package' do 308 | context 'when manifest has pre_package set' do 309 | let(:manifest) do 310 | { pre_package: 'scripts/build.sh' } 311 | end 312 | 313 | context 'when the pre package script succeeds' do 314 | it 'does not raise an error' do 315 | allow(Kernel).to receive(:system) 316 | .with('scripts/build.sh') 317 | .and_return(true) 318 | 319 | packager.run_pre_package 320 | expect(Kernel).to have_received(:system) 321 | end 322 | end 323 | 324 | context 'when the pre package script fails' do 325 | it 'raises an error' do 326 | allow(Kernel).to receive(:system) 327 | .with('scripts/build.sh') 328 | .and_return(false) 329 | expect { packager.run_pre_package }.to raise_error('Failed to run pre_package script: scripts/build.sh') 330 | end 331 | end 332 | end 333 | 334 | context 'when manifest does not have pre_package set' do 335 | it 'does nothing' do 336 | expect(Kernel).not_to receive(:system) 337 | 338 | packager.run_pre_package 339 | end 340 | end 341 | end 342 | end 343 | end 344 | end 345 | --------------------------------------------------------------------------------