├── .ruby-version ├── .gitignore ├── spec ├── spec_helper.rb ├── runner_execution_spec.rb ├── git_fastclone_url_helper_spec.rb └── git_fastclone_runner_spec.rb ├── lib ├── git-fastclone │ └── version.rb ├── runner_execution.rb └── git-fastclone.rb ├── kochiku.yml ├── Gemfile ├── Rakefile ├── script ├── spec_demo_tool.sh └── ci ├── BUG-BOUNTY.md ├── .rubocop.yml ├── LICENSE ├── bin └── git-fastclone ├── .github └── workflows │ └── build.yml ├── .rubocop_todo.yml ├── git-fastclone.gemspec └── README.md /.ruby-version: -------------------------------------------------------------------------------- 1 | 3.3.9 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *:lock 2 | *.gem 3 | Gemfile.lock 4 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'rspec/core' 4 | require 'rspec/mocks' 5 | -------------------------------------------------------------------------------- /lib/git-fastclone/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # Version string for git-fastclone 4 | module GitFastCloneVersion 5 | VERSION = '1.6.1' 6 | end 7 | -------------------------------------------------------------------------------- /kochiku.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | ruby: 3 | - 2.1.5 4 | - 2.2.3 5 | 6 | test_command: 'script/ci' 7 | 8 | targets: 9 | - type: unit 10 | glob: 'spec/**/*_spec.rb' 11 | workers: 1 12 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source 'https://rubygems.org' 4 | 5 | group :development do 6 | gem 'bundler' 7 | gem 'rake' 8 | gem 'rubocop', install_if: -> { RUBY_VERSION >= '3.2' } 9 | end 10 | 11 | gem 'logger' 12 | gem 'rspec' 13 | 14 | gemspec 15 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'bundler/setup' 4 | 5 | task default: %w[spec] 6 | 7 | require 'rspec/core/rake_task' 8 | RSpec::Core::RakeTask.new 9 | 10 | begin 11 | require 'rubocop/rake_task' 12 | RuboCop::RakeTask.new 13 | task default: :rubocop 14 | rescue LoadError => e 15 | raise unless e.path == 'rubocop/rake_task' 16 | end 17 | -------------------------------------------------------------------------------- /script/spec_demo_tool.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | 3 | # This script is a sample script used in integration tests that exits with the code passed as the first argument 4 | # Also, it prints all extra arguments 5 | 6 | exit_code="$1" 7 | 8 | if [ $# -gt 1 ]; then 9 | # Skip first argument, which is the exit code 10 | shift 11 | echo "$@" 12 | fi 13 | 14 | exit $exit_code 15 | -------------------------------------------------------------------------------- /BUG-BOUNTY.md: -------------------------------------------------------------------------------- 1 | Serious about security 2 | ====================== 3 | 4 | Square recognizes the important contributions the security research community 5 | can make. We therefore encourage reporting security issues with the code 6 | contained in this repository. 7 | 8 | If you believe you have discovered a security vulnerability, please follow the 9 | guidelines at https://hackerone.com/square-open-source 10 | 11 | -------------------------------------------------------------------------------- /script/ci: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | REQUIRED_BUNDLER_VERSION=">= 1.6.2" 6 | 7 | function install_bundler() { 8 | gem install bundler --conservative --version "$REQUIRED_BUNDLER_VERSION" 9 | } 10 | 11 | function install_gems() { 12 | bundle check || bundle 13 | } 14 | 15 | function run_specs() { 16 | bundle exec rake 17 | } 18 | 19 | install_bundler 20 | install_gems 21 | run_specs 22 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | inherit_from: .rubocop_todo.yml 2 | 3 | AllCops: 4 | TargetRubyVersion: 3.2 5 | NewCops: enable 6 | 7 | Naming/FileName: 8 | Exclude: 9 | - 'bin/git-fastclone' 10 | - 'lib/git-fastclone.rb' 11 | 12 | Lint/EmptyBlock: 13 | Exclude: ['spec/*'] 14 | 15 | Metrics/ClassLength: 16 | Max: 10000 17 | 18 | Metrics/AbcSize: 19 | Enabled: false 20 | 21 | Metrics/MethodLength: 22 | Max: 2000 23 | 24 | Metrics/BlockLength: 25 | Exclude: 26 | - 'spec/**/*' 27 | 28 | Metrics/CyclomaticComplexity: 29 | Max: 15 30 | 31 | Metrics/PerceivedComplexity: 32 | Max: 15 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The Apache License (Apache) 2 | 3 | Copyright 2015 Square Inc. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | -------------------------------------------------------------------------------- /bin/git-fastclone: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | # Copyright 2015 Square Inc. 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | $LOAD_PATH.unshift(File.expand_path("#{File.dirname(__FILE__)}/../lib")) 19 | 20 | require 'git-fastclone' 21 | 22 | GitFastClone::Runner.new.run 23 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | permissions: 4 | contents: read 5 | 6 | on: 7 | push: 8 | branches: [master] 9 | pull_request: 10 | branches: [master] 11 | 12 | jobs: 13 | test: 14 | runs-on: ${{ matrix.runner }} 15 | strategy: 16 | fail-fast: false 17 | matrix: 18 | ruby-version: ["3.2", "3.3", "3.4", "3.5.0-preview1"] 19 | runner: [macos-latest, ubuntu-latest] 20 | steps: 21 | - uses: actions/checkout@v5 22 | - name: Set up Ruby 23 | # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby, 24 | # change this to (see https://github.com/ruby/setup-ruby#versioning): 25 | # uses: ruby/setup-ruby@v1 26 | uses: ruby/setup-ruby@v1 27 | with: 28 | ruby-version: ${{ matrix.ruby-version }} 29 | bundler-cache: true # runs 'bundle install' and caches installed gems automatically 30 | - name: Run rake 31 | run: bundle exec rake 32 | -------------------------------------------------------------------------------- /.rubocop_todo.yml: -------------------------------------------------------------------------------- 1 | # This configuration was generated by 2 | # `rubocop --auto-gen-config` 3 | # on 2019-01-17 09:51:29 -0800 using RuboCop version 0.63.0. 4 | # The point is for the user to remove these configuration records 5 | # one by one as the offenses are removed from the code base. 6 | # Note that changes in the inspected code, or installation of new 7 | # versions of RuboCop, may require this file to be generated again. 8 | 9 | # Offense count: 1 10 | Lint/AmbiguousBlockAssociation: 11 | Exclude: 12 | - 'spec/git_fastclone_runner_spec.rb' 13 | 14 | # Offense count: 5 15 | Metrics/AbcSize: 16 | Max: 25 17 | 18 | # Offense count: 4 19 | # Configuration parameters: CountComments, ExcludedMethods. 20 | # ExcludedMethods: refine 21 | Metrics/BlockLength: 22 | Max: 250 23 | 24 | # Offense count: 1 25 | Metrics/CyclomaticComplexity: 26 | Max: 7 27 | 28 | # Offense count: 1 29 | Metrics/PerceivedComplexity: 30 | Max: 8 31 | 32 | # Offense count: 6 33 | # Configuration parameters: . 34 | # SupportedStyles: inline, group 35 | Style/AccessModifierDeclarations: 36 | EnforcedStyle: inline 37 | -------------------------------------------------------------------------------- /git-fastclone.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # Copyright 2015 Square Inc. 4 | 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | require 'date' 18 | 19 | $LOAD_PATH.push File.expand_path('lib', __dir__) 20 | require 'git-fastclone/version' 21 | 22 | Gem::Specification.new do |gem| 23 | gem.name = 'git-fastclone' 24 | gem.version = GitFastCloneVersion::VERSION 25 | gem.summary = 'git-clone --recursive on steroids!' 26 | gem.description = 'A git command that uses reference repositories and threading to quickly' \ 27 | 'and recursively clone repositories with many nested submodules' 28 | gem.authors = ['Michael Tauraso', 'James Chang'] 29 | gem.email = ['mtauraso@squareup.com', 'jchang@squareup.com'] 30 | gem.files = Dir['Rakefile', '{bin,lib,man,test,spec}/**/*', 'README*', 'LICENSE*'] & 31 | `git ls-files -z`.split("\0") 32 | gem.executables = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) } 33 | gem.require_paths = ['lib'] 34 | gem.homepage = 'http://square.github.io/git-fastclone/' 35 | gem.license = 'Apache' 36 | 37 | gem.required_ruby_version = '>= 3.2' 38 | 39 | gem.add_dependency 'colorize' 40 | gem.metadata['rubygems_mfa_required'] = 'true' 41 | end 42 | -------------------------------------------------------------------------------- /spec/runner_execution_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # Copyright 2023 Square Inc. 4 | 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | require 'spec_helper' 18 | require 'git-fastclone' 19 | 20 | # Integration tests use real demo_tool.sh to inspect the E2E behavior 21 | describe RunnerExecution do 22 | subject { described_class } 23 | let(:external_tool) { "#{__dir__}/../script/spec_demo_tool.sh" } 24 | let(:logger) { double('logger') } 25 | 26 | before do 27 | allow($stdout).to receive(:puts) 28 | allow(logger).to receive(:info) 29 | allow(logger).to receive(:debug) 30 | allow(logger).to receive(:warn) 31 | allow(RunnerExecution).to receive(:logger).and_return(logger) 32 | end 33 | 34 | describe '.fail_on_error' do 35 | it 'should log failure info on command error' do 36 | expect(logger).to receive(:info).with("My error output\n") 37 | 38 | expect do 39 | described_class.fail_on_error(external_tool, '1', 'My error output', quiet: true, 40 | print_on_failure: true) 41 | end.to raise_error(RunnerExecution::RunnerExecutionRuntimeError) 42 | end 43 | 44 | it 'should not log failure output on command success' do 45 | expect($stdout).not_to receive(:info) 46 | 47 | described_class.fail_on_error(external_tool, '0', 'My success output', quiet: true, 48 | print_on_failure: true) 49 | end 50 | 51 | it 'should not log failure output when not in the quiet mode' do 52 | expect($stdout).not_to receive(:info) 53 | 54 | described_class.fail_on_error(external_tool, '0', 'My success output', quiet: false, 55 | print_on_failure: true) 56 | end 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /spec/git_fastclone_url_helper_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # Copyright 2015 Square Inc. 4 | 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | require 'spec_helper' 18 | require 'git-fastclone' 19 | 20 | describe GitFastClone::UrlHelper do 21 | let(:test_url_valid) { 'ssh://git@git.com/git-fastclone.git' } 22 | let(:test_url_invalid) { 'ssh://git@git.com/git-fastclone' } 23 | let(:test_reference_dir) { 'test_reference_dir' } 24 | let(:submodule_str) do 25 | "Submodule 'TestModule' (https://github.com/TestModule1/TestModule2) registered for path 26 | 'TestModule'" 27 | end 28 | 29 | describe '.path_from_git_url' do 30 | let(:tail) { 'git-fastclone' } 31 | 32 | context 'with a valid path' do 33 | it 'should get the tail' do 34 | expect(subject.path_from_git_url(test_url_valid)).to eq(tail) 35 | end 36 | end 37 | 38 | context 'with an invalid path' do 39 | it 'should still get the tail' do 40 | expect(subject.path_from_git_url(test_url_invalid)).to eq(tail) 41 | end 42 | end 43 | end 44 | 45 | describe '.parse_update_info' do 46 | it 'should parse correctly' do 47 | expect(subject.parse_update_info(submodule_str)) 48 | .to eq(['TestModule', 'https://github.com/TestModule1/TestModule2']) 49 | end 50 | end 51 | 52 | describe '.reference_repo_name' do 53 | let(:expected_result) { 'git.com-git-fastclone.git' } 54 | 55 | it 'should come up with a unique repo name' do 56 | expect(subject.reference_repo_name(test_url_valid)).to eq(expected_result) 57 | end 58 | end 59 | 60 | describe '.reference_repo_dir' do 61 | it 'should join correctly' do 62 | allow(subject).to receive(:reference_repo_name) { test_reference_dir } 63 | 64 | expect(subject.reference_repo_dir(test_url_valid, test_reference_dir, false)) 65 | .to eq("#{test_reference_dir}/#{test_reference_dir}") 66 | end 67 | end 68 | 69 | describe '.reference_repo_submodule_file' do 70 | it 'should return the right string' do 71 | allow(subject).to receive(:reference_repo_dir) { test_reference_dir } 72 | 73 | expect(subject.reference_repo_submodule_file(test_url_valid, test_reference_dir, false)) 74 | .to eq('test_reference_dir:submodules.txt') 75 | end 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | git-fastclone 2 | ============= 3 | 4 | [![Twitter: @longboardcat13](https://img.shields.io/badge/contact-@longboardcat13-blue.svg?style=flat)](https://twitter.com/longboardcat13) 5 | [![License](https://img.shields.io/badge/license-Apache-green.svg?style=flat)](https://github.com/square/git-fastclone/blob/master/LICENSE) 6 | [![Build Status](https://travis-ci.org/square/git-fastclone.svg?branch=master)](https://travis-ci.org/square/git-fastclone) 7 | [![Gem Version](https://badge.fury.io/rb/git-fastclone.svg)](http://badge.fury.io/rb/git-fastclone) 8 | 9 | git-fastclone is git clone --recursive on steroids. 10 | 11 | 12 | Why fastclone? 13 | -------------- 14 | Doing lots of repeated checkouts on a specific machine? 15 | 16 | | Repository | 1st Fastclone | 2nd Fastclone | git clone | cp -R | 17 | | ---------- | ------------- | ------------- | --------- | ----- | 18 | | angular.js | 8s | 3s | 6s | 0.5s | 19 | | bootstrap | 26s | 3s | 11s | 0.2s | 20 | | gradle | 25s | 9s | 19s | 6.2s | 21 | | linux | 4m 53s | 1m 6s | 3m 51s | 29s | 22 | | react.js | 18s | 3s | 8s | 0.5s | 23 | | tensorflow | 19s | 4s | 8s | 1.5s | 24 | 25 | Above times captured using `time` without verbose mode. 26 | 27 | 28 | What does it do? 29 | ---------------- 30 | It creates a reference repo with `git clone --mirror` in `/var/tmp/git-fastclone/reference` for each 31 | repository and git submodule linked in the main repo. You can control where it puts these by 32 | changing the `REFERENCE_REPO_DIR` environment variable. 33 | 34 | It aggressively updates these mirrors from origin and then clones from the mirrors into the 35 | directory of your choosing. It always works recursively and multithreaded to get your checkout up as 36 | fast as possible. 37 | 38 | Detailed explanation [here](https://developer.squareup.com/blog/introducing-git-fastclone/). 39 | 40 | 41 | Usage 42 | ----- 43 | gem install git-fastclone 44 | git fastclone [options] 45 | 46 | -b, --branch BRANCH Checkout this branch rather than the default 47 | -v, --verbose Verbose mode 48 | --print_git_errors Print git output if a command fails 49 | -c, --color Display colored output 50 | --config CONFIG Git config applied to the cloned repo 51 | --lock-timeout N Timeout in seconds to acquire a lock on any reference repo. 52 | Default is 0 which waits indefinitely. 53 | --pre-clone-hook command An optional command that should be invoked before cloning mirror repo 54 | --sparse-paths PATHS Comma-separated list of paths for sparse checkout. 55 | Enables sparse checkout mode using git sparse-checkout. 56 | 57 | Change the default `REFERENCE_REPO_DIR` environment variable if necessary. 58 | 59 | Cygwin users need to add `~/bin` to PATH. 60 | 61 | 62 | Hooks 63 | ----- 64 | 65 | - `pre-clone-hook` is invoked right before cloning a new mirror repo, which gives a change to prepopulate git's mirror from a different source. 66 | The hook is invoked with given arguments: 67 | 1. cloning repo url 68 | 1. path to the repo mirror location 69 | 1. attempt number, 0-indexed 70 | 71 | Sparse checkout support 72 | ----------------------- 73 | 74 | In passing `--sparse-paths`, git-fastclone will instead perform a sparse checkout, where the passed list of paths will be set up as patterns. This can be useful if you're interested only in a subset of paths in the repository. 75 | 76 | How to test? 77 | ------------ 78 | Manual testing: 79 | 80 | ruby -Ilib bin/git-fastclone 81 | 82 | Compatible with Travis and Kochiku. 83 | 84 | 85 | Contributing 86 | ------------ 87 | If you would like to contribute to git-fastclone, you can fork the repository and send us pull 88 | requests. 89 | 90 | When submitting code, please make every effort to follow existing conventions and style in order to 91 | keep the code as readable as possible. 92 | 93 | Before accepting any pull requests, we need you to sign an [Individual Contributor Agreement](https://docs.google.com/a/squareup.com/forms/d/13WR8m5uZ2nAkJH41k7GdVBXAAbzDk00vxtEYjd6Imzg/viewform?formkey=dDViT2xzUHAwRkI3X3k5Z0lQM091OGc6MQ&ndplr=1) 94 | (Google form). 95 | 96 | Once landed, please reach out to any owner listed in https://rubygems.org/gems/git-fastclone and ask them to help publish the new version. 97 | 98 | 99 | Acknowledgements 100 | ---------------- 101 | * [robolson](https://github.com/robolson) 102 | * [ianchesal](https://github.com/ianchesal) 103 | * [mtauraso](https://github.com/mtauraso) 104 | * [chriseckhardt](https://github.com/chriseckhardt) 105 | 106 | 107 | License 108 | ------- 109 | Copyright 2015 Square Inc. 110 | 111 | Licensed under the Apache License, Version 2.0 (the "License"); 112 | you may not use this file except in compliance with the License. 113 | You may obtain a copy of the License at 114 | 115 | http://www.apache.org/licenses/LICENSE-2.0 116 | 117 | Unless required by applicable law or agreed to in writing, software 118 | distributed under the License is distributed on an "AS IS" BASIS, 119 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 120 | See the License for the specific language governing permissions and 121 | limitations under the License. 122 | -------------------------------------------------------------------------------- /lib/runner_execution.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | # rubocop:disable all 3 | 4 | require 'open3' 5 | require 'logger' 6 | 7 | # Execution primitives that force explicit error handling and never call the shell. 8 | # Cargo-culted from internal BuildExecution code on top of public version: https://github.com/square/build_execution 9 | module RunnerExecution 10 | class RunnerExecutionRuntimeError < RuntimeError 11 | attr_reader :status, :exitstatus, :command, :output 12 | 13 | def initialize(status, command, output = nil) 14 | @status = status 15 | @exitstatus = status.exitstatus 16 | @command = command 17 | @output = output 18 | 19 | super "#{status.inspect}\n#{command.inspect}" 20 | end 21 | end 22 | 23 | # Runs a command that fails on error. 24 | # Uses popen2e wrapper. Handles bad statuses with potential for retries. 25 | def fail_on_error(*cmd, stdin_data: nil, binmode: false, quiet: false, print_on_failure: false, **opts) 26 | print_command('Running Shell Safe Command:', [cmd]) unless quiet 27 | shell_safe_cmd = shell_safe(cmd) 28 | retry_times = opts[:retry] || 0 29 | opts.delete(:retry) 30 | 31 | while retry_times >= 0 32 | output, status = popen2e_wrapper(*shell_safe_cmd, stdin_data: stdin_data, binmode: binmode, 33 | quiet: quiet, **opts) 34 | 35 | break unless status.exitstatus != 0 36 | 37 | logger.debug("Command failed with exit status #{status.exitstatus}, retrying #{retry_times} more time(s).") if retry_times > 0 38 | retry_times -= 1 39 | end 40 | 41 | # Get out with the status, good or bad. 42 | # When quiet, we don't need to print the output, as it is already streamed from popen2e_wrapper 43 | needs_print_on_failure = quiet && print_on_failure 44 | exit_on_status(output, [shell_safe_cmd], [status], quiet: quiet, print_on_failure: needs_print_on_failure) 45 | end 46 | module_function :fail_on_error 47 | 48 | # Wrapper around open3.popen2e 49 | # 50 | # We emulate open3.capture2e with the following changes in behavior: 51 | # 1) The command is printed to stdout before execution. 52 | # 2) Attempts to use the shell implicitly are blocked. 53 | # 3) Nonzero return codes result in the process exiting. 54 | # 4) Combined stdout/stderr goes to callers stdout 55 | # (continuously streamed) and is returned as a string 56 | # 57 | # If you're looking for more process/stream control read the spawn 58 | # documentation, and pass options directly here 59 | def popen2e_wrapper(*shell_safe_cmd, stdin_data: nil, binmode: false, 60 | quiet: false, **opts) 61 | 62 | env = opts.delete(:env) { {} } 63 | raise ArgumentError, "The :env option must be a hash, not #{env.inspect}" if !env.is_a?(Hash) 64 | 65 | # Most of this is copied from Open3.capture2e in ruby/lib/open3.rb 66 | _output, _status = Open3.popen2e(env, *shell_safe_cmd, opts) do |i, oe, t| 67 | if binmode 68 | i.binmode 69 | oe.binmode 70 | end 71 | 72 | outerr_reader = Thread.new do 73 | if quiet 74 | oe.read 75 | else 76 | # Instead of oe.read, we redirect. Output from command goes to stdout 77 | # and also is returned for processing if necessary. 78 | tee(oe, STDOUT) 79 | end 80 | end 81 | 82 | if stdin_data 83 | begin 84 | i.write stdin_data 85 | rescue Errno::EPIPE 86 | end 87 | end 88 | 89 | i.close 90 | [outerr_reader.value, t.value] 91 | end 92 | end 93 | module_function :popen2e_wrapper 94 | 95 | # Look at a cmd list intended for spawn. 96 | # determine if spawn will call the shell implicitly, fail in that case. 97 | def shell_safe(cmd) 98 | # Take the first string and change it to a list of [executable,argv0] 99 | # This syntax for calling popen2e (and eventually spawn) avoids 100 | # the shell in all cases 101 | shell_safe_cmd = Array.new(cmd) 102 | if shell_safe_cmd[0].class == String 103 | shell_safe_cmd[0] = [shell_safe_cmd[0], shell_safe_cmd[0]] 104 | end 105 | shell_safe_cmd 106 | end 107 | module_function :shell_safe 108 | 109 | def debug_print_cmd_list(cmd_list) 110 | # Take a list of command argument lists like you'd sent to open3.pipeline or 111 | # fail_on_error_pipe and print out a string that would do the same thing when 112 | # entered at the shell. 113 | # 114 | # This is a converter from our internal representation of commands to a subset 115 | # of bash that can be executed directly. 116 | # 117 | # Note this has problems if you specify env or opts 118 | # TODO: make this remove those command parts 119 | "\"" + 120 | cmd_list.map do |cmd| 121 | cmd.map do |arg| 122 | arg.gsub("\"", "\\\"") # Escape all double quotes in command arguments 123 | end.join("\" \"") # Fully quote all command parts, beginning and end. 124 | end.join("\" | \"") + "\"" # Pipe commands to one another. 125 | end 126 | module_function :debug_print_cmd_list 127 | 128 | # Prints a formatted string with command 129 | def print_command(message, cmd) 130 | logger.debug("#{message} #{debug_print_cmd_list(cmd)}\n") 131 | end 132 | module_function :print_command 133 | 134 | # Takes in an input stream and an output stream 135 | # Redirects data from one to the other until the input stream closes. 136 | # Returns all data that passed through on return. 137 | def tee(in_stream, out_stream) 138 | alldata = '' 139 | loop do 140 | begin 141 | data = in_stream.read_nonblock(4096) 142 | alldata += data 143 | out_stream.write(data) 144 | out_stream.flush 145 | rescue IO::WaitReadable 146 | IO.select([in_stream]) 147 | retry 148 | rescue IOError 149 | break 150 | end 151 | end 152 | alldata 153 | end 154 | module_function :tee 155 | 156 | # If any of the statuses are bad, exits with the 157 | # return code of the first one. 158 | # 159 | # Otherwise returns first argument (output) 160 | def exit_on_status(output, cmd_list, status_list, quiet: false, print_on_failure: false) 161 | status_list.each_index do |index| 162 | status = status_list[index] 163 | cmd = cmd_list[index] 164 | check_status(cmd, status, output: output, quiet: quiet, print_on_failure: print_on_failure) 165 | end 166 | 167 | output 168 | end 169 | module_function :exit_on_status 170 | 171 | def check_status(cmd, status, output: nil, quiet: false, print_on_failure: false) 172 | return if status.exited? && status.exitstatus == 0 173 | 174 | logger.info(output) if print_on_failure 175 | # If we exited nonzero or abnormally, print debugging info and explode. 176 | if status.exited? 177 | logger.debug("Process Exited normally. Exit status:#{status.exitstatus}") unless quiet 178 | else 179 | # This should only get executed if we're stopped or signaled 180 | logger.debug("Process exited abnormally:\nProcessStatus: #{status.inspect}\n" \ 181 | "Raw POSIX Status: #{status.to_i}\n") unless quiet 182 | end 183 | 184 | raise RunnerExecutionRuntimeError.new(status, cmd, output) 185 | end 186 | module_function :check_status 187 | 188 | DEFAULT_LOGGER = Logger.new(STDOUT) 189 | private_constant :DEFAULT_LOGGER 190 | 191 | def logger 192 | DEFAULT_LOGGER 193 | end 194 | module_function :logger 195 | end 196 | # rubocop:enable all 197 | -------------------------------------------------------------------------------- /lib/git-fastclone.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # Copyright 2015 Square Inc. 4 | 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | require 'optparse' 18 | require 'fileutils' 19 | require 'timeout' 20 | require_relative 'runner_execution' 21 | 22 | # Contains helper module UrlHelper and execution class GitFastClone::Runner 23 | module GitFastClone 24 | # Helper methods for fastclone url operations 25 | module UrlHelper 26 | def path_from_git_url(url) 27 | File.basename(url, '.git') 28 | end 29 | module_function :path_from_git_url 30 | 31 | def parse_update_info(line) 32 | [line.strip.match(/'([^']*)'$/)[1], line.strip.match(/\(([^)]*)\)/)[1]] 33 | end 34 | module_function :parse_update_info 35 | 36 | def reference_repo_name(url) 37 | url.gsub(%r{^.*://}, '').gsub(/^[^@]*@/, '').tr('/', '-').tr(':', '-').to_s 38 | end 39 | module_function :reference_repo_name 40 | 41 | def reference_repo_dir(url, reference_dir, using_local_repo) 42 | if using_local_repo 43 | File.join(reference_dir, "local#{reference_repo_name(url)}") 44 | else 45 | File.join(reference_dir, reference_repo_name(url)) 46 | end 47 | end 48 | module_function :reference_repo_dir 49 | 50 | def reference_filename(filename) 51 | separator = if RbConfig::CONFIG['host_os'] =~ /mswin|msys|mingw|cygwin|bccwin|wince|emc/ 52 | '__' 53 | else 54 | ':' 55 | end 56 | "#{separator}#{filename}" 57 | end 58 | module_function :reference_filename 59 | 60 | def reference_repo_submodule_file(url, reference_dir, using_local_repo) 61 | "#{reference_repo_dir(url, reference_dir, using_local_repo)}#{reference_filename('submodules.txt')}" 62 | end 63 | module_function :reference_repo_submodule_file 64 | 65 | def reference_repo_lock_file(url, reference_dir, using_local_repo) 66 | lock_file_name = "#{reference_repo_dir(url, reference_dir, using_local_repo)}#{reference_filename('lock')}" 67 | File.open(lock_file_name, File::RDWR | File::CREAT, 0o644) 68 | end 69 | module_function :reference_repo_lock_file 70 | end 71 | 72 | # Spawns one thread per submodule, and updates them in parallel. They will be 73 | # cached in the reference directory (see DEFAULT_REFERENCE_REPO_DIR), and their 74 | # index will be incrementally updated. This prevents a large amount of data 75 | # copying. 76 | class Runner 77 | require 'colorize' 78 | 79 | include GitFastClone::UrlHelper 80 | include RunnerExecution 81 | 82 | DEFAULT_REFERENCE_REPO_DIR = '/var/tmp/git-fastclone/reference' 83 | 84 | DEFAULT_GIT_ALLOW_PROTOCOL = 'file:git:http:https:ssh' 85 | 86 | attr_accessor :reference_dir, :prefetch_submodules, :reference_updated, :reference_mutex, 87 | :options, :abs_clone_path, :using_local_repo, :verbose, :print_git_errors, :color, 88 | :flock_timeout_secs, :sparse_paths 89 | 90 | def initialize 91 | # Prefetch reference repos for submodules we've seen before 92 | # Keep our own reference accounting of module dependencies. 93 | self.prefetch_submodules = true 94 | 95 | # Thread-level locking for reference repos 96 | # TODO: Add flock-based locking if we want to avoid conflicting with 97 | # ourselves. 98 | self.reference_mutex = Hash.new { |hash, key| hash[key] = Mutex.new } 99 | 100 | # Only update each reference repo once per run. 101 | # TODO: May want to update this so we don't duplicate work with other copies 102 | # of ourself. Perhaps a last-updated-time and a timeout per reference repo. 103 | self.reference_updated = Hash.new { |hash, key| hash[key] = false } 104 | 105 | self.options = {} 106 | 107 | self.abs_clone_path = Dir.pwd 108 | 109 | self.using_local_repo = false 110 | 111 | self.verbose = false 112 | 113 | self.print_git_errors = false 114 | 115 | self.color = false 116 | 117 | self.flock_timeout_secs = 0 118 | 119 | self.sparse_paths = nil 120 | end 121 | 122 | def run 123 | url, path, options = parse_inputs 124 | 125 | require_relative 'git-fastclone/version' 126 | msg = "git-fastclone #{GitFastCloneVersion::VERSION}" 127 | if color 128 | puts msg.yellow 129 | else 130 | puts msg 131 | end 132 | 133 | puts "Cloning #{path_from_git_url(url)} to #{File.join(abs_clone_path, path)}" 134 | ENV['GIT_ALLOW_PROTOCOL'] ||= DEFAULT_GIT_ALLOW_PROTOCOL 135 | clone(url, options[:branch], path, options[:config]) 136 | end 137 | 138 | def parse_options 139 | # One option --branch= We're not as brittle as clone. That branch 140 | # can be a sha or tag and we're still okay. 141 | OptionParser.new do |opts| 142 | opts.banner = usage 143 | options[:branch] = nil 144 | 145 | opts.on('-b', '--branch BRANCH', 'Checkout this branch rather than the default') do |branch| 146 | options[:branch] = branch 147 | end 148 | 149 | opts.on('-v', '--verbose', 'Verbose mode') do 150 | puts '--print_git_errors is redundant when using --verbose' if print_git_errors 151 | self.verbose = true 152 | end 153 | 154 | opts.on('--print_git_errors', 'Print git output if a command fails') do 155 | puts '--print_git_errors is redundant when using --verbose' if verbose 156 | self.print_git_errors = true 157 | end 158 | 159 | opts.on('-c', '--color', 'Display colored output') do 160 | self.color = true 161 | end 162 | 163 | opts.on('--config CONFIG', 'Git config applied to the cloned repo') do |config| 164 | options[:config] = config 165 | end 166 | 167 | opts.on('--lock-timeout N', 'Timeout in seconds to acquire a lock on any reference repo.', 168 | 'Default is 0 which waits indefinitely.') do |timeout_secs| 169 | self.flock_timeout_secs = timeout_secs.to_i 170 | end 171 | 172 | opts.on('--pre-clone-hook script_file', 173 | 'An optional file that should be invoked before cloning mirror repo', 174 | 'No-op when a file is missing') do |script_file| 175 | options[:pre_clone_hook] = script_file 176 | end 177 | 178 | opts.on('--sparse-paths PATHS', 179 | 'Comma-separated list of paths for sparse checkout.', 180 | 'Enables sparse checkout mode using git sparse-checkout.') do |paths| 181 | self.sparse_paths = paths.split(',').map(&:strip) 182 | end 183 | end.parse! 184 | end 185 | 186 | def parse_inputs 187 | parse_options 188 | 189 | unless ARGV[0] 190 | warn usage 191 | exit(129) 192 | end 193 | 194 | if Dir.exist?(ARGV[0]) 195 | url = File.expand_path ARGV[0] 196 | self.using_local_repo = true 197 | else 198 | url = ARGV[0] 199 | end 200 | 201 | path = ARGV[1] || path_from_git_url(url) 202 | 203 | if Dir.exist?(path) 204 | msg = "Clone destination #{File.join(abs_clone_path, path)} already exists!" 205 | raise msg.red if color 206 | 207 | raise msg 208 | end 209 | 210 | # Validate that --branch is specified when using --sparse-paths 211 | if sparse_paths && !options[:branch] 212 | msg = "Error: --branch is required when using --sparse-paths\n" \ 213 | "Sparse checkouts need an explicit branch/revision to checkout.\n" \ 214 | 'Usage: git-fastclone --sparse-paths --branch ' 215 | raise msg.red if color 216 | 217 | raise msg 218 | end 219 | 220 | self.reference_dir = ENV['REFERENCE_REPO_DIR'] || DEFAULT_REFERENCE_REPO_DIR 221 | FileUtils.mkdir_p(reference_dir) 222 | 223 | [url, path, options] 224 | end 225 | 226 | def clear_clone_dest_if_needed(attempt_number, clone_dest) 227 | return unless attempt_number.positive? 228 | 229 | dest_with_dotfiles = Dir.glob("#{clone_dest}/*", File::FNM_DOTMATCH) 230 | dest_files = dest_with_dotfiles.reject { |f| %w[. ..].include?(File.basename(f)) } 231 | return if dest_files.empty? 232 | 233 | clear_clone_dest(dest_files) 234 | end 235 | 236 | def clear_clone_dest(dest_files) 237 | puts 'Non-empty clone directory found, clearing its content now.' 238 | FileUtils.rm_rf(dest_files) 239 | end 240 | 241 | # Checkout to SOURCE_DIR. Update all submodules recursively. Use reference 242 | # repos everywhere for speed. 243 | def clone(url, rev, src_dir, config) 244 | clone_dest = File.join(abs_clone_path, src_dir).to_s 245 | initial_time = Time.now 246 | 247 | if Dir.exist?(clone_dest) && !Dir.empty?(clone_dest) 248 | raise "Can't clone into an existing non-empty path: #{clone_dest}" 249 | end 250 | 251 | with_git_mirror(url) do |mirror, attempt_number| 252 | clear_clone_dest_if_needed(attempt_number, clone_dest) 253 | 254 | clone_commands = ['git', 'clone', verbose ? '--verbose' : '--quiet'] 255 | # For sparse checkouts, clone directly from the local mirror and skip the actual checkout process 256 | # --shared is included so that the checkout remains fast even if the reference and destination directories 257 | # live on different filesystem volumes. 258 | # For normal clones, use --reference and clone from the remote URL 259 | if sparse_paths 260 | clone_commands.push('--no-checkout', '--shared') 261 | clone_commands << mirror.to_s << clone_dest 262 | else 263 | clone_commands << '--reference' << mirror.to_s << url.to_s << clone_dest 264 | end 265 | clone_commands << '--config' << config.to_s unless config.nil? 266 | fail_on_error(*clone_commands, quiet: !verbose, print_on_failure: print_git_errors) 267 | 268 | # Configure sparse checkout if enabled 269 | perform_sparse_checkout(clone_dest, rev) if sparse_paths 270 | end 271 | 272 | # Only checkout if we're changing branches to a non-default branch (for non-sparse clones) 273 | if !sparse_paths && rev 274 | fail_on_error('git', 'checkout', '--quiet', rev.to_s, quiet: !verbose, 275 | print_on_failure: print_git_errors, 276 | chdir: File.join(abs_clone_path, src_dir)) 277 | end 278 | 279 | update_submodules(src_dir, url) 280 | 281 | final_time = Time.now 282 | 283 | msg = "Checkout of #{src_dir} took #{final_time - initial_time}s" 284 | if color 285 | puts msg.green 286 | else 287 | puts msg 288 | end 289 | end 290 | 291 | def perform_sparse_checkout(clone_dest, rev) 292 | puts 'Configuring sparse checkout...' if verbose 293 | 294 | # Initialize sparse checkout with cone mode 295 | fail_on_error('git', 'sparse-checkout', 'init', '--cone', 296 | quiet: !verbose, print_on_failure: print_git_errors, chdir: clone_dest) 297 | 298 | # Set the sparse paths 299 | fail_on_error('git', 'sparse-checkout', 'set', *sparse_paths, 300 | quiet: !verbose, print_on_failure: print_git_errors, chdir: clone_dest) 301 | 302 | # Checkout the specified branch/revision 303 | fail_on_error('git', 'checkout', '--quiet', rev.to_s, 304 | quiet: !verbose, print_on_failure: print_git_errors, chdir: clone_dest) 305 | end 306 | 307 | def update_submodules(pwd, url) 308 | return unless File.exist?(File.join(abs_clone_path, pwd, '.gitmodules')) 309 | 310 | puts 'Updating submodules...' if verbose 311 | 312 | threads = [] 313 | submodule_url_list = [] 314 | output = fail_on_error('git', 'submodule', 'init', quiet: !verbose, 315 | print_on_failure: print_git_errors, 316 | chdir: File.join(abs_clone_path, pwd)) 317 | 318 | output.split("\n").each do |line| 319 | submodule_path, submodule_url = parse_update_info(line) 320 | submodule_url_list << submodule_url 321 | 322 | thread_update_submodule(submodule_url, submodule_path, threads, pwd) 323 | end 324 | 325 | update_submodule_reference(url, submodule_url_list) 326 | threads.each(&:join) 327 | end 328 | 329 | def thread_update_submodule(submodule_url, submodule_path, threads, pwd) 330 | threads << Thread.new do 331 | with_git_mirror(submodule_url) do |mirror, _| 332 | cmd = ['git', 'submodule', 333 | verbose ? nil : '--quiet', 'update', '--reference', mirror.to_s, submodule_path.to_s].compact 334 | fail_on_error(*cmd, quiet: !verbose, print_on_failure: print_git_errors, 335 | chdir: File.join(abs_clone_path, pwd)) 336 | end 337 | 338 | update_submodules(File.join(pwd, submodule_path), submodule_url) 339 | end 340 | end 341 | 342 | def with_reference_repo_lock(url, &) 343 | # Sane POSIX implementations remove exclusive flocks when a process is terminated or killed 344 | # We block here indefinitely. Waiting for other git-fastclone processes to release the lock. 345 | # With the default timeout of 0 we will wait forever, this can be overridden on the command line. 346 | lockfile = reference_repo_lock_file(url, reference_dir, using_local_repo) 347 | Timeout.timeout(flock_timeout_secs) { lockfile.flock(File::LOCK_EX) } 348 | with_reference_repo_thread_lock(url, &) 349 | ensure 350 | # Not strictly necessary to do this unlock as an ensure. If ever exception is caught outside this 351 | # primitive, ensure protection may come in handy. 352 | lockfile.flock(File::LOCK_UN) 353 | lockfile.close 354 | end 355 | 356 | def with_reference_repo_thread_lock(url, &) 357 | # We also need thread level locking because pre-fetch means multiple threads can 358 | # attempt to update the same repository from a single git-fastclone process 359 | # file locks in posix are tracked per process, not per userland thread. 360 | # This gives us the equivalent of pthread_mutex around these accesses. 361 | reference_mutex[reference_repo_name(url)].synchronize(&) 362 | end 363 | 364 | def update_submodule_reference(url, submodule_url_list) 365 | return if submodule_url_list.empty? || prefetch_submodules.nil? 366 | 367 | with_reference_repo_lock(url) do 368 | # Write the dependency file using submodule list 369 | File.open(reference_repo_submodule_file(url, reference_dir, using_local_repo), 'w') do |f| 370 | submodule_url_list.each { |submodule_url| f.write("#{submodule_url}\n") } 371 | end 372 | end 373 | end 374 | 375 | # Fail_hard indicates whether the update is considered a failure of the 376 | # overall checkout or not. When we pre-fetch based off of cached information, 377 | # fail_hard is false. When we fetch based off info in a repository directly, 378 | # fail_hard is true. 379 | def update_reference_repo(url, fail_hard, attempt_number) 380 | repo_name = reference_repo_name(url) 381 | mirror = reference_repo_dir(url, reference_dir, using_local_repo) 382 | 383 | with_reference_repo_lock(url) do 384 | # we've created this to track submodules' history 385 | submodule_file = reference_repo_submodule_file(url, reference_dir, using_local_repo) 386 | 387 | # if prefetch is on, then grab children immediately to frontload network requests 388 | prefetch(submodule_file, attempt_number) if File.exist?(submodule_file) && prefetch_submodules 389 | 390 | # Store the fact that our repo has been updated if necessary 391 | store_updated_repo(url, mirror, repo_name, fail_hard, attempt_number) unless reference_updated[repo_name] 392 | end 393 | end 394 | 395 | # Grab the children in the event of a prefetch 396 | def prefetch(submodule_file, attempt_number) 397 | File.readlines(submodule_file).each do |line| 398 | # We don't join these threads explicitly 399 | Thread.new { update_reference_repo(line.strip, false, attempt_number) } 400 | end 401 | end 402 | 403 | # Creates or updates the mirror repo then stores an indication 404 | # that this repo has been updated on this run of fastclone 405 | def store_updated_repo(url, mirror, repo_name, fail_hard, attempt_number) 406 | trigger_pre_clone_hook_if_needed(url, mirror, attempt_number) 407 | # If pre_clone_hook correctly creates a mirror directory, we don't want to clone, but just update it 408 | unless Dir.exist?(mirror) 409 | fail_on_error('git', 'clone', verbose ? '--verbose' : '--quiet', '--mirror', url.to_s, mirror.to_s, 410 | quiet: !verbose, print_on_failure: print_git_errors) 411 | end 412 | 413 | cmd = ['git', 'remote', verbose ? '--verbose' : nil, 'update', '--prune'].compact 414 | fail_on_error(*cmd, quiet: !verbose, print_on_failure: print_git_errors, chdir: mirror) 415 | 416 | reference_updated[repo_name] = true 417 | rescue RunnerExecutionRuntimeError => e 418 | # To avoid corruption of the cache, if we failed to update or check out we remove 419 | # the cache directory entirely. This may cause the current clone to fail, but if the 420 | # underlying error from git is transient it will not affect future clones. 421 | # 422 | # The only exception to this is authentication failures, because they are transient, 423 | # usually due to either a remote server outage or a local credentials config problem. 424 | clear_cache(mirror, url) unless auth_error?(e.output) 425 | raise e if fail_hard 426 | end 427 | 428 | def auth_error?(error) 429 | error.to_s =~ /.*^fatal: Authentication failed/m 430 | end 431 | 432 | def retriable_error?(error) 433 | error_strings = [ 434 | /^fatal: missing blob object/, 435 | /^fatal: remote did not send all necessary objects/, 436 | /^fatal: packed object [a-z0-9]+ \(stored in .*?\) is corrupt/, 437 | /^fatal: pack has \d+ unresolved delta/, 438 | /^error: unable to read sha1 file of /, 439 | /^fatal: did not receive expected object/, 440 | /^fatal: unable to read tree [a-z0-9]+\n^warning: Clone succeeded, but checkout failed/, 441 | /^fatal: Authentication failed/ 442 | ] 443 | error.to_s =~ /.*#{Regexp.union(error_strings)}/m 444 | end 445 | 446 | def print_formatted_error(error) 447 | indented_error = error.to_s.split("\n").map { |s| "> #{s}\n" }.join 448 | puts "[INFO] Encountered a retriable error:\n#{indented_error}\n" 449 | end 450 | 451 | # To avoid corruption of the cache, if we failed to update or check out we remove 452 | # the cache directory entirely. This may cause the current clone to fail, but if the 453 | # underlying error from git is transient it will not affect future clones. 454 | def clear_cache(dir, url) 455 | puts "[WARN] Removing the fastclone cache at #{dir}" 456 | FileUtils.remove_entry_secure(dir, force: true) 457 | reference_updated.delete(reference_repo_name(url)) 458 | end 459 | 460 | # This command will create and bring the mirror up-to-date on-demand, 461 | # blocking any code passed in while the mirror is brought up-to-date 462 | # 463 | # In future we may need to synchronize with flock here if we run multiple 464 | # builds at once against the same reference repos. One build per slave at the 465 | # moment means we only need to synchronize our own threads in case a single 466 | # submodule url is included twice via multiple dependency paths 467 | def with_git_mirror(url) 468 | retries_allowed ||= 1 469 | attempt_number ||= 0 470 | 471 | update_reference_repo(url, true, attempt_number) 472 | dir = reference_repo_dir(url, reference_dir, using_local_repo) 473 | 474 | # Sometimes remote updates involve re-packing objects on a different thread 475 | # We grab the reference repo lock here just to make sure whatever thread 476 | # ended up doing the update is done with its housekeeping. 477 | # This makes sure we have control and unlock when the block returns: 478 | with_reference_repo_lock(url) do 479 | yield dir, attempt_number 480 | end 481 | rescue RunnerExecutionRuntimeError => e 482 | if retriable_error?(e.output) 483 | print_formatted_error(e.output) 484 | clear_cache(dir, url) 485 | 486 | if attempt_number < retries_allowed 487 | attempt_number += 1 488 | retry 489 | end 490 | end 491 | 492 | raise e 493 | end 494 | 495 | def usage 496 | 'Usage: git fastclone [options] [path]' 497 | end 498 | 499 | private def trigger_pre_clone_hook_if_needed(url, mirror, attempt_number) 500 | return if Dir.exist?(mirror) || !options.include?(:pre_clone_hook) 501 | 502 | hook_command = options[:pre_clone_hook] 503 | unless File.exist?(File.expand_path(hook_command)) 504 | puts 'pre_clone_hook script is missing' if verbose 505 | return 506 | end 507 | 508 | popen2e_wrapper(hook_command, url.to_s, mirror.to_s, attempt_number.to_s, quiet: !verbose) 509 | end 510 | end 511 | end 512 | -------------------------------------------------------------------------------- /spec/git_fastclone_runner_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # Copyright 2015 Square Inc. 4 | 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | require 'spec_helper' 18 | require 'git-fastclone' 19 | 20 | describe GitFastClone::Runner do 21 | let(:test_url_valid) { 'ssh://git@git.com/git-fastclone.git' } 22 | let(:test_url_invalid) { 'ssh://git@git.com/git-fastclone' } 23 | let(:test_reference_dir) { 'test_reference_dir' } 24 | let(:test_reference_repo_dir) { '/var/tmp/git-fastclone/reference/test_reference_dir' } 25 | let(:placeholder_arg) { 'PH' } 26 | 27 | def create_lockfile_double 28 | lockfile = double 29 | expect(lockfile).to receive(:flock).with(File::LOCK_EX).once 30 | expect(lockfile).to receive(:flock).with(File::LOCK_UN).once 31 | expect(lockfile).to receive(:close).once 32 | lockfile 33 | end 34 | 35 | let(:lockfile) { create_lockfile_double } 36 | 37 | before do 38 | stub_const('ARGV', ['ssh://git@git.com/git-fastclone.git', 'test_reference_dir']) 39 | allow($stdout).to receive(:puts) 40 | end 41 | 42 | let(:yielded) { [] } 43 | 44 | describe '.initialize' do 45 | it 'should initialize properly' do 46 | stub_const('GitFastClone::DEFAULT_REFERENCE_REPO_DIR', 'new_dir') 47 | 48 | expect(GitFastClone::DEFAULT_REFERENCE_REPO_DIR).to eq('new_dir') 49 | expect(subject.prefetch_submodules).to eq(true) 50 | expect(subject.reference_mutex).to eq({}) 51 | expect(subject.reference_updated).to eq({}) 52 | expect(subject.options).to eq({}) 53 | end 54 | end 55 | 56 | describe '.run' do 57 | let(:options) { { branch: placeholder_arg } } 58 | 59 | it 'should run with the correct args' do 60 | allow(subject).to receive(:parse_inputs) { [placeholder_arg, placeholder_arg, options, nil] } 61 | expect(subject).to receive(:clone).with(placeholder_arg, placeholder_arg, placeholder_arg, nil) 62 | 63 | subject.run 64 | end 65 | 66 | describe 'with custom configs' do 67 | let(:options) { { branch: placeholder_arg, config: 'conf' } } 68 | 69 | it 'should clone correctly' do 70 | allow(subject).to receive(:parse_inputs) { [placeholder_arg, placeholder_arg, options, 'conf'] } 71 | expect(subject).to receive(:clone).with(placeholder_arg, placeholder_arg, placeholder_arg, 'conf') 72 | 73 | subject.run 74 | end 75 | end 76 | end 77 | 78 | describe '.parse_inputs' do 79 | it 'should print the proper inputs' do 80 | subject.reference_dir = test_reference_dir 81 | subject.options = {} 82 | allow(FileUtils).to receive(:mkdir_p) {} 83 | 84 | expect(subject.parse_inputs).to eq([test_url_valid, test_reference_dir, { branch: nil }]) 85 | end 86 | end 87 | 88 | describe '.clone' do 89 | let(:runner_execution_double) { double('runner_execution') } 90 | before(:each) do 91 | allow(runner_execution_double).to receive(:fail_on_error) {} 92 | allow(Dir).to receive(:pwd) { '/pwd' } 93 | allow(subject).to receive(:with_git_mirror).and_yield('/cache', 0) 94 | expect(subject).to receive(:clear_clone_dest_if_needed).once {} 95 | end 96 | 97 | it 'should clone correctly' do 98 | expect(subject).to receive(:fail_on_error).with( 99 | 'git', 'checkout', '--quiet', 'PH', 100 | { chdir: '/pwd/.', print_on_failure: false, quiet: true } 101 | ) { runner_execution_double } 102 | expect(subject).to receive(:fail_on_error).with( 103 | 'git', 'clone', '--quiet', '--reference', '/cache', 'PH', '/pwd/.', 104 | { quiet: true, print_on_failure: false } 105 | ) { runner_execution_double } 106 | 107 | subject.clone(placeholder_arg, placeholder_arg, '.', nil) 108 | end 109 | 110 | it 'should clone correctly with verbose mode on' do 111 | subject.verbose = true 112 | expect(subject).to receive(:fail_on_error).with( 113 | 'git', 'checkout', '--quiet', 'PH', 114 | { chdir: '/pwd/.', print_on_failure: false, quiet: false } 115 | ) { runner_execution_double } 116 | expect(subject).to receive(:fail_on_error).with( 117 | 'git', 'clone', '--verbose', '--reference', '/cache', 'PH', '/pwd/.', 118 | { quiet: false, print_on_failure: false } 119 | ) { runner_execution_double } 120 | 121 | subject.clone(placeholder_arg, placeholder_arg, '.', nil) 122 | end 123 | 124 | it 'should clone correctly with custom configs' do 125 | expect(subject).to receive(:fail_on_error).with( 126 | 'git', 'clone', '--quiet', '--reference', '/cache', 'PH', '/pwd/.', '--config', 'config', 127 | { quiet: true, print_on_failure: false } 128 | ) { runner_execution_double } 129 | 130 | subject.clone(placeholder_arg, nil, '.', 'config') 131 | end 132 | 133 | context 'with printing errors' do 134 | before(:each) do 135 | subject.print_git_errors = true 136 | end 137 | 138 | it 'prints failures' do 139 | expect(subject).to receive(:fail_on_error).with( 140 | 'git', 'clone', '--quiet', '--reference', '/cache', 'PH', '/pwd/.', '--config', 'config', 141 | { quiet: true, print_on_failure: true } 142 | ) { runner_execution_double } 143 | 144 | subject.clone(placeholder_arg, nil, '.', 'config') 145 | end 146 | end 147 | 148 | context 'with sparse checkout' do 149 | before(:each) do 150 | subject.sparse_paths = %w[path1 path2] 151 | end 152 | 153 | it 'should clone with --no-checkout and --shared flags' do 154 | expect(subject).to receive(:fail_on_error).with( 155 | 'git', 'clone', '--quiet', '--no-checkout', '--shared', '/cache', '/pwd/.', 156 | { quiet: true, print_on_failure: false } 157 | ) { runner_execution_double } 158 | expect(subject).to receive(:perform_sparse_checkout).with('/pwd/.', 'PH') 159 | 160 | subject.clone(placeholder_arg, 'PH', '.', nil) 161 | end 162 | 163 | it 'should clone with verbose mode and --shared flag' do 164 | subject.verbose = true 165 | expect(subject).to receive(:fail_on_error).with( 166 | 'git', 'clone', '--verbose', '--no-checkout', '--shared', '/cache', '/pwd/.', 167 | { quiet: false, print_on_failure: false } 168 | ) { runner_execution_double } 169 | expect(subject).to receive(:perform_sparse_checkout).with('/pwd/.', 'PH') 170 | 171 | subject.clone(placeholder_arg, 'PH', '.', nil) 172 | end 173 | 174 | it 'should not perform regular checkout when sparse checkout is enabled' do 175 | expect(subject).to receive(:fail_on_error).with( 176 | 'git', 'clone', '--quiet', '--no-checkout', '--shared', '/cache', '/pwd/.', 177 | { quiet: true, print_on_failure: false } 178 | ) { runner_execution_double } 179 | expect(subject).to receive(:perform_sparse_checkout).with('/pwd/.', 'PH') 180 | expect(subject).not_to receive(:fail_on_error).with( 181 | 'git', 'checkout', '--quiet', 'PH', 182 | anything 183 | ) 184 | 185 | subject.clone(placeholder_arg, 'PH', '.', nil) 186 | end 187 | end 188 | 189 | context 'with pre-clone-hook' do 190 | let(:pre_clone_hook) { '/some/command' } 191 | before(:each) do 192 | subject.options[:pre_clone_hook] = pre_clone_hook 193 | subject.reference_dir = placeholder_arg 194 | allow(File).to receive(:exist?).and_call_original 195 | allow(File).to receive(:exist?).with(pre_clone_hook).and_return(true) 196 | allow(subject).to receive(:with_git_mirror).and_call_original 197 | allow(subject).to receive(:with_reference_repo_lock) do |_url, &block| 198 | block.call 199 | end 200 | end 201 | 202 | it 'invokes hook command' do 203 | allow(subject).to receive(:fail_on_error) 204 | expect(subject).to receive(:popen2e_wrapper).with( 205 | pre_clone_hook, 'PH', 'PH/PH', '0', 206 | { quiet: true } 207 | ) { runner_execution_double } 208 | 209 | subject.clone(placeholder_arg, nil, '.', 'config') 210 | end 211 | 212 | it 'does not call clone if hook creates mirror' do 213 | allow(subject).to receive(:popen2e_wrapper).with( 214 | pre_clone_hook, 'PH', 'PH/PH', '0', 215 | { quiet: true } 216 | ) do 217 | # Emulate creating mirror dir 218 | allow(Dir).to receive(:exist?).with('PH/PH').and_return(true) 219 | end 220 | allow(subject).to receive(:fail_on_error) 221 | 222 | subject.clone(placeholder_arg, nil, '.', 'config') 223 | end 224 | 225 | it 'does not call pre-clone hook if mirror is already created' do 226 | # Emulate already created mirror dir 227 | allow(Dir).to receive(:exist?).and_call_original 228 | allow(Dir).to receive(:exist?).with('PH/PH').and_return(true) 229 | expect(subject).not_to receive(:popen2e_wrapper).with( 230 | pre_clone_hook, 'PH', 'PH/PH', '0', 231 | { quiet: true } 232 | ) 233 | allow(subject).to receive(:fail_on_error) 234 | 235 | subject.clone(placeholder_arg, nil, '.', 'config') 236 | end 237 | 238 | context 'non-existing script' do 239 | before(:each) do 240 | allow(File).to receive(:exist?).with(pre_clone_hook).and_return(false) 241 | end 242 | 243 | it 'does not invoke hook command' do 244 | allow(subject).to receive(:fail_on_error) 245 | expect(subject).not_to receive(:popen2e_wrapper).with( 246 | pre_clone_hook, 'PH', 'PH/PH', '0', 247 | { quiet: true } 248 | ) 249 | 250 | subject.clone(placeholder_arg, nil, '.', 'config') 251 | end 252 | end 253 | end 254 | end 255 | 256 | describe '.clear_clone_dest_if_needed' do 257 | it 'does not clear on first attempt' do 258 | expect(Dir).not_to receive(:glob) 259 | expect(subject).not_to receive(:clear_clone_dest) 260 | subject.clear_clone_dest_if_needed(0, '/some/path') 261 | end 262 | 263 | it 'does not clear if the directory is only FNM_DOTMATCH self and parent refs' do 264 | expect(Dir).to receive(:glob).and_return(%w[. ..]) 265 | expect(subject).not_to receive(:clear_clone_dest) 266 | subject.clear_clone_dest_if_needed(1, '/some/path') 267 | end 268 | 269 | it 'does clear if the directory is not empty' do 270 | expect(Dir).to receive(:glob).and_return(%w[. .. /some/path/file.txt]) 271 | expect(subject).to receive(:clear_clone_dest) {} 272 | subject.clear_clone_dest_if_needed(1, '/some/path') 273 | end 274 | end 275 | 276 | describe '.update_submodules' do 277 | it 'should return if no submodules' do 278 | subject.update_submodules(placeholder_arg, placeholder_arg) 279 | allow(File).to receive(:exist?) { false } 280 | 281 | expect(Thread).not_to receive(:new) 282 | end 283 | 284 | it 'should correctly update submodules' do 285 | expect(subject).to receive(:update_submodule_reference) 286 | 287 | allow(File).to receive(:exist?) { true } 288 | subject.update_submodules('.', placeholder_arg) 289 | end 290 | end 291 | 292 | describe '.thread_update_submodule' do 293 | it 'should update correctly' do 294 | pending('need to figure out how to test this') 295 | raise 296 | end 297 | end 298 | 299 | describe '.with_reference_repo_lock' do 300 | it 'should acquire a lock' do 301 | allow(Mutex).to receive(:synchronize) 302 | expect(Mutex).to respond_to(:synchronize) 303 | expect(subject).to receive(:reference_repo_lock_file).and_return(lockfile) 304 | 305 | subject.with_reference_repo_lock(test_url_valid) do 306 | yielded << test_url_valid 307 | end 308 | 309 | expect(yielded).to eq([test_url_valid]) 310 | end 311 | it 'should un-flock on thrown exception' do 312 | allow(Mutex).to receive(:synchronize) 313 | expect(Mutex).to respond_to(:synchronize) 314 | expect(subject).to receive(:reference_repo_lock_file).and_return(lockfile) 315 | 316 | expect do 317 | subject.with_reference_repo_lock(test_url_valid) do 318 | raise placeholder_arg 319 | end 320 | end.to raise_error(placeholder_arg) 321 | end 322 | end 323 | 324 | describe '.update_submodule_reference' do 325 | context 'when we have an empty submodule list' do 326 | it 'should return' do 327 | expect(subject).not_to receive(:with_reference_repo_lock) 328 | 329 | subject.prefetch_submodules = true 330 | subject.update_submodule_reference(placeholder_arg, []) 331 | end 332 | end 333 | 334 | context 'with a populated submodule list' do 335 | it 'should write to a file' do 336 | allow(File).to receive(:open) {} 337 | allow(File).to receive(:write) {} 338 | allow(subject).to receive(:reference_repo_name) {} 339 | allow(subject).to receive(:reference_repo_submodule_file) {} 340 | expect(File).to receive(:open) 341 | expect(subject).to receive(:reference_repo_lock_file).and_return(lockfile) 342 | 343 | subject.update_submodule_reference(placeholder_arg, [placeholder_arg, placeholder_arg]) 344 | end 345 | end 346 | end 347 | 348 | describe '.update_reference_repo' do 349 | context 'when prefetch is on' do 350 | it 'should grab the children immediately and then store' do 351 | expect(subject).to receive(:prefetch).once 352 | expect(subject).to receive(:store_updated_repo).once 353 | expect(subject).to receive(:reference_repo_lock_file).and_return(lockfile) 354 | 355 | allow(File).to receive(:exist?) { true } 356 | subject.prefetch_submodules = true 357 | subject.reference_dir = placeholder_arg 358 | subject.update_reference_repo(test_url_valid, false, 0) 359 | end 360 | end 361 | 362 | context 'when prefetch is off' do 363 | it 'should store the updated repo' do 364 | expect(subject).not_to receive(:prefetch) 365 | expect(subject).to receive(:store_updated_repo).once 366 | expect(subject).to receive(:reference_repo_lock_file).and_return(lockfile) 367 | 368 | allow(File).to receive(:exist?) { true } 369 | subject.prefetch_submodules = false 370 | subject.reference_dir = placeholder_arg 371 | subject.update_reference_repo(placeholder_arg, false, 0) 372 | end 373 | end 374 | 375 | let(:placeholder_hash) { {} } 376 | 377 | context 'when already have a hash' do 378 | it 'should not store' do 379 | placeholder_hash[placeholder_arg] = true 380 | expect(subject).not_to receive(:store_updated_repo) 381 | 382 | allow(subject).to receive(:reference_repo_name) { placeholder_arg } 383 | allow(subject).to receive(:reference_repo_dir) { placeholder_arg } 384 | subject.reference_updated = placeholder_hash 385 | subject.prefetch_submodules = false 386 | subject.update_reference_repo(placeholder_arg, false, 0) 387 | end 388 | end 389 | 390 | context 'when do not have a hash' do 391 | it 'should store' do 392 | placeholder_hash[placeholder_arg] = false 393 | expect(subject).to receive(:store_updated_repo) 394 | expect(subject).to receive(:reference_repo_lock_file).and_return(lockfile) 395 | 396 | allow(subject).to receive(:reference_repo_name) { placeholder_arg } 397 | subject.reference_updated = placeholder_hash 398 | subject.reference_dir = placeholder_arg 399 | subject.prefetch_submodules = false 400 | subject.update_reference_repo(placeholder_arg, false, 0) 401 | end 402 | end 403 | end 404 | 405 | describe '.prefetch' do 406 | it 'should go through the submodule file properly' do 407 | expect(Thread).to receive(:new).exactly(3).times 408 | 409 | allow(File).to receive(:readlines) { %w[1 2 3] } 410 | subject.prefetch_submodules = true 411 | subject.prefetch(placeholder_arg, 0) 412 | end 413 | end 414 | 415 | describe '.store_updated_repo' do 416 | context 'when fail_hard is true' do 417 | it 'should raise a Runtime error and clear cache if there were no authentication errors' do 418 | status = double('status') 419 | allow(status).to receive(:exitstatus).and_return(1) 420 | ex = RunnerExecution::RunnerExecutionRuntimeError.new(status, 'cmd') 421 | allow(subject).to receive(:fail_on_error) { raise ex } 422 | expect(FileUtils).to receive(:remove_entry_secure).with(placeholder_arg, force: true) 423 | expect do 424 | subject.store_updated_repo(placeholder_arg, placeholder_arg, placeholder_arg, true, 0) 425 | end.to raise_error(ex) 426 | end 427 | 428 | it 'should raise a Runtime error and skip clearing the cache if there were authentication errors' do 429 | status = double('status') 430 | allow(status).to receive(:exitstatus).and_return(1) 431 | ex = RunnerExecution::RunnerExecutionRuntimeError.new(status, 'cmd') 432 | allow(ex).to receive(:output).and_return('fatal: Authentication failed') 433 | allow(subject).to receive(:fail_on_error) { raise ex } 434 | expect(FileUtils).to_not receive(:remove_entry_secure).with(placeholder_arg, force: true) 435 | expect do 436 | subject.store_updated_repo(placeholder_arg, placeholder_arg, placeholder_arg, true, 0) 437 | end.to raise_error(ex) 438 | end 439 | end 440 | 441 | context 'when fail_hard is false' do 442 | it 'should not raise a Runtime error but clear cache' do 443 | status = double('status') 444 | allow(status).to receive(:exitstatus).and_return(1) 445 | ex = RunnerExecution::RunnerExecutionRuntimeError.new(status, 'cmd') 446 | allow(subject).to receive(:fail_on_error) { raise ex } 447 | expect(FileUtils).to receive(:remove_entry_secure).with(placeholder_arg, force: true) 448 | expect do 449 | subject.store_updated_repo(placeholder_arg, placeholder_arg, placeholder_arg, false, 0) 450 | end.to_not raise_error 451 | end 452 | end 453 | 454 | let(:placeholder_hash) { {} } 455 | 456 | it 'should correctly update the hash' do 457 | allow(subject).to receive(:fail_on_error) 458 | 459 | subject.reference_updated = placeholder_hash 460 | subject.store_updated_repo(placeholder_arg, placeholder_arg, placeholder_arg, false, 0) 461 | expect(subject.reference_updated).to eq(placeholder_arg => true) 462 | end 463 | end 464 | 465 | describe '.with_git_mirror' do 466 | def retriable_error 467 | %( 468 | fatal: bad object ee35b1e14e7c3a53dcc14d82606e5b872f6a05a7 469 | fatal: remote did not send all necessary objects 470 | ).strip.split("\n").map(&:strip).join("\n") 471 | end 472 | 473 | def try_with_git_mirror(responses, results) 474 | lambdas = responses.map do |response| 475 | if response == true 476 | # Simulate successful response 477 | ->(url) { url } 478 | else 479 | # Simulate failed error response 480 | lambda { |_url| 481 | status = double('status') 482 | allow(status).to receive(:exitstatus).and_return(1) 483 | raise RunnerExecution::RunnerExecutionRuntimeError.new(status, 'cmd', response) 484 | } 485 | end 486 | end 487 | 488 | subject.with_git_mirror(test_url_valid) do |url, attempt| 489 | raise 'Not enough responses were provided!' if lambdas.empty? 490 | 491 | yielded << [lambdas.shift.call(url), attempt] 492 | end 493 | 494 | expect(lambdas).to be_empty 495 | expect(yielded).to eq(results) 496 | end 497 | 498 | let(:expected_commands) { [] } 499 | 500 | before(:each) do 501 | allow(subject).to receive(:fail_on_error) { |*params| 502 | # last one is an argument `quiet:` 503 | command = params.first(params.size - 1) 504 | expect(expected_commands.length).to be > 0 505 | expected_command = expected_commands.shift 506 | expect(command).to eq(expected_command) 507 | } 508 | 509 | allow(subject).to receive(:print_formatted_error) {} 510 | allow(subject).to receive(:reference_repo_dir).and_return(test_reference_repo_dir) 511 | allow(subject).to receive(:reference_repo_lock_file) { create_lockfile_double } 512 | end 513 | 514 | after(:each) do 515 | expect(expected_commands).to be_empty 516 | end 517 | 518 | def clone_cmds(verbose: false) 519 | [ 520 | ['git', 'clone', verbose ? '--verbose' : '--quiet', '--mirror', test_url_valid, 521 | test_reference_repo_dir], 522 | ['git', 'remote', verbose ? '--verbose' : nil, 'update', 523 | '--prune'].compact 524 | ] 525 | end 526 | 527 | context 'expecting 1 clone attempt' do 528 | context 'with verbose mode on' do 529 | before { subject.verbose = true } 530 | let(:expected_commands) { clone_cmds(verbose: true) } 531 | 532 | it 'should succeed with a successful clone' do 533 | expect(subject).not_to receive(:clear_cache) 534 | try_with_git_mirror([true], [[test_reference_repo_dir, 0]]) 535 | end 536 | 537 | it 'should fail after a non-retryable clone error' do 538 | expect(subject).not_to receive(:clear_cache) 539 | expect do 540 | try_with_git_mirror(['Some unexpected error message'], []) 541 | end.to raise_error(RunnerExecution::RunnerExecutionRuntimeError) 542 | end 543 | end 544 | 545 | context 'with verbose mode off' do 546 | let(:expected_commands) { clone_cmds } 547 | 548 | it 'should succeed with a successful clone' do 549 | expect(subject).not_to receive(:clear_cache) 550 | try_with_git_mirror([true], [[test_reference_repo_dir, 0]]) 551 | end 552 | 553 | it 'should fail after a non-retryable clone error' do 554 | expect(subject).not_to receive(:clear_cache) 555 | expect do 556 | try_with_git_mirror(['Some unexpected error message'], []) 557 | end.to raise_error(RunnerExecution::RunnerExecutionRuntimeError) 558 | end 559 | end 560 | end 561 | 562 | context 'expecting 2 clone attempts' do 563 | let(:expected_commands) { clone_cmds + clone_cmds } 564 | let(:expected_commands_args) { clone_args + clone_args } 565 | 566 | it 'should succeed after a single retryable clone failure' do 567 | expect(subject).to receive(:clear_cache).and_call_original 568 | try_with_git_mirror([retriable_error, true], [[test_reference_repo_dir, 1]]) 569 | end 570 | 571 | it 'should fail after two retryable clone failures' do 572 | expect(subject).to receive(:clear_cache).twice.and_call_original 573 | expect do 574 | try_with_git_mirror([retriable_error, retriable_error], []) 575 | end.to raise_error(RunnerExecution::RunnerExecutionRuntimeError) 576 | end 577 | end 578 | end 579 | 580 | describe '.retriable_error?' do 581 | def format_error(error) 582 | error_wrapper = error.to_s 583 | error_wrapper.strip.lines.map(&:strip).join("\n") 584 | end 585 | 586 | it 'not for a random error message' do 587 | error = format_error 'random error message' 588 | 589 | expect(subject.retriable_error?(error)).to be_falsey 590 | end 591 | 592 | it 'when the cache looks corrupted' do 593 | error = format_error <<-ERROR 594 | fatal: bad object ee35b1e14e7c3a53dcc14d82606e5b872f6a05a7 595 | fatal: remote did not send all necessary objects 596 | ERROR 597 | 598 | expect(subject.retriable_error?(error)).to be_truthy 599 | end 600 | 601 | it 'when the clone succeeds but checkout fails with corrupt packed object' do 602 | error = format_error <<-ERROR 603 | fatal: packed object 7c4d79704f8adf701f38a7bfb3e33ec5342542f1 (stored in /private/var/tmp/git-fastclone/reference/some-repo.git/objects/pack/pack-d37d7ed3e88d6e5f0ac141a7b0a2b32baf6e21a0.pack) is corrupt 604 | warning: Clone succeeded, but checkout failed. 605 | You can inspect what was checked out with 'git status' and retry with 'git restore --source=HEAD :/' 606 | ERROR 607 | 608 | expect(subject.retriable_error?(error)).to be_truthy 609 | end 610 | 611 | it 'when the clone succeeds but checkout fails with unable to read tree' do 612 | error = format_error <<-ERROR 613 | error: Could not read 92cf57b8f07df010ab5f607b109c325e30e46235 614 | fatal: unable to read tree 0c32c0521d3b0bfb4e74e4a39b97a84d1a3bb9a1 615 | warning: Clone succeeded, but checkout failed. 616 | You can inspect what was checked out with 'git status' 617 | and retry with 'git restore --source=HEAD :/' 618 | ERROR 619 | 620 | expect(subject.retriable_error?(error)).to be_truthy 621 | end 622 | 623 | it 'when one delta is missing' do 624 | error = format_error <<-ERROR 625 | error: Could not read f7fad86d06fee0678f9af7203b6031feabb40c3e 626 | fatal: pack has 1 unresolved delta 627 | fatal: index-pack failed 628 | ERROR 629 | 630 | expect(subject.retriable_error?(error)).to be_truthy 631 | end 632 | 633 | it 'when deltas are missing' do 634 | error = format_error <<-ERROR 635 | error: Could not read f7fad86d06fee0678f9af7203b6031feabb40c3e 636 | fatal: pack has 138063 unresolved deltas 637 | fatal: index-pack failed 638 | ERROR 639 | 640 | expect(subject.retriable_error?(error)).to be_truthy 641 | end 642 | 643 | it 'when the cache errors with unable to read sha1 file' do 644 | error = format_error <<-ERROR 645 | error: unable to read sha1 file of sqiosbuild/lib/action/action.rb (6113b739af82d8b07731de8a58d6e233301f80ab) 646 | fatal: unable to checkout working tree 647 | warning: Clone succeeded, but checkout failed. 648 | You can inspect what was checked out with 'git status' 649 | and retry with 'git restore --source=HEAD :/' 650 | ERROR 651 | 652 | expect(subject.retriable_error?(error)).to be_truthy 653 | end 654 | 655 | it 'when the cache errors with did not receive expected object' do 656 | error = format_error <<-ERROR 657 | error: Could not read 6682dfe81f66656436e60883dd795e7ec6735153 658 | error: Could not read 0cd3703c23fa44c0043d97fbc26356a23939f31b 659 | fatal: did not receive expected object 3c64c9dd49c79bd09aa13d4b05ac18263ca29ccd 660 | fatal: index-pack failed 661 | ERROR 662 | 663 | expect(subject.retriable_error?(error)).to be_truthy 664 | end 665 | end 666 | end 667 | --------------------------------------------------------------------------------