├── .ruby-version ├── .gitignore ├── .bin ├── run-tests ├── run-test ├── accept_svn_ssl_certificate ├── string_encoder └── check_scm_version ├── spec ├── scm_fixtures │ ├── bzr.tgz │ ├── cvs.tgz │ ├── git.tgz │ ├── hg.tgz │ ├── svn.tgz │ ├── git_svn.tgz │ ├── git_walk.tgz │ ├── hg_large.tgz │ ├── hg_walk.tgz │ ├── bzr_colon.tgz │ ├── bzr_large.tgz │ ├── svn_subdir.tgz │ ├── git_with_mv.tgz │ ├── bzr_with_branch.tgz │ ├── git_dupe_delete.tgz │ ├── hg_dupe_delete.tgz │ ├── bzr_with_authors.tgz │ ├── git_with_master_tag.tgz │ ├── git_with_null_merge.tgz │ ├── git_with_submodules.tgz │ ├── bzr_with_subdirectories.tgz │ ├── bzr_with_invalid_encoding.tgz │ ├── bzr_with_nested_branches.tgz │ ├── git_with_invalid_encoding.tgz │ ├── git_with_multiple_branch.tgz │ └── hg_with_invalid_encoding.tgz ├── raw_fixtures │ ├── sample-content │ ├── invalid-utf-word │ ├── simultaneous_checkins_2.rlog │ ├── simultaneous_checkins.rlog │ ├── basic.rlog │ ├── simple.svn_log │ ├── file_created_on_branch.rlog │ ├── simple.ohlog │ └── multiple_revisions.rlog ├── helpers │ ├── generic_helper.rb │ ├── system_helper.rb │ ├── commit_tokens_helper.rb │ ├── assert_scm_attr_helper.rb │ └── repository_helper.rb ├── ohloh_scm │ ├── version_spec.rb │ ├── hg │ │ ├── status_spec.rb │ │ ├── scm_spec.rb │ │ └── validation_spec.rb │ ├── factory_spec.rb │ ├── bzr │ │ ├── scm_spec.rb │ │ └── validation_spec.rb │ ├── git_svn │ │ ├── scm_spec.rb │ │ └── activity_spec.rb │ ├── activity_spec.rb │ ├── git │ │ ├── status_spec.rb │ │ ├── validation_spec.rb │ │ └── scm_spec.rb │ ├── system_spec.rb │ ├── parser │ │ ├── array_writer_spec.rb │ │ ├── cvs_parser_spec.rb │ │ ├── git_parser_spec.rb │ │ └── branch_number_spec.rb │ ├── svn │ │ ├── activity_spec.rb │ │ ├── scm_spec.rb │ │ └── validation_spec.rb │ ├── cvs │ │ ├── activity_spec.rb │ │ ├── validation_spec.rb │ │ └── scm_spec.rb │ └── svn_parser_spec.rb ├── .rubocop.yml ├── spec_helper.rb ├── string_encoder_spec.rb └── benchmarks │ ├── hg_bzr_bash_vs_py_api.rb │ └── process_spawn_benchmark.rb ├── lib ├── ohloh_scm │ ├── bzr │ │ ├── status.rb │ │ ├── validation.rb │ │ ├── scm.rb │ │ └── activity.rb │ ├── hg │ │ ├── status.rb │ │ ├── validation.rb │ │ └── scm.rb │ ├── svn │ │ ├── status.rb │ │ ├── validation.rb │ │ ├── scm.rb │ │ └── activity.rb │ ├── git_svn │ │ ├── status.rb │ │ ├── validation.rb │ │ ├── activity.rb │ │ └── scm.rb │ ├── parser │ │ ├── hg_style │ │ ├── array_writer.rb │ │ ├── hg_verbose_style │ │ ├── branch_number.rb │ │ ├── hg_parser.rb │ │ ├── svn_parser.rb │ │ ├── git_parser.rb │ │ ├── bzr_xml_parser.rb │ │ └── cvs_parser.rb │ ├── py_bridge.rb │ ├── cvs.rb │ ├── git.rb │ ├── string_extensions.rb │ ├── svn.rb │ ├── version.rb │ ├── git_svn.rb │ ├── hg.rb │ ├── bzr.rb │ ├── cvs │ │ ├── status.rb │ │ ├── validation.rb │ │ ├── scm.rb │ │ └── activity.rb │ ├── factory.rb │ ├── git │ │ ├── validation.rb │ │ ├── status.rb │ │ └── scm.rb │ ├── status.rb │ ├── scm.rb │ ├── py_bridge │ │ ├── bzr_client.rb │ │ ├── hg_client.rb │ │ ├── py_client.rb │ │ ├── hg_server.py │ │ └── bzr_server.py │ ├── activity.rb │ ├── parser.rb │ ├── core.rb │ ├── data │ │ └── git_ignore_list.rb │ ├── system.rb │ ├── diff.rb │ ├── commit.rb │ └── validation.rb └── ohloh_scm.rb ├── Rakefile ├── .travis.yml ├── Gemfile ├── .rubocop.yml ├── .github └── workflows │ └── ci.yml ├── .git_hooks └── pre-commit ├── ohloh_scm.gemspec ├── Gemfile.lock ├── Dockerfile ├── security.md └── README.md /.ruby-version: -------------------------------------------------------------------------------- 1 | 3.1.7 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .byebug* 2 | coverage 3 | -------------------------------------------------------------------------------- /.bin/run-tests: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | bundle exec ruby -Ispec $@ 4 | -------------------------------------------------------------------------------- /.bin/run-test: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | bundle exec ruby -Ispec $1 --name /.*$2.*/ 4 | -------------------------------------------------------------------------------- /spec/scm_fixtures/bzr.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/bzr.tgz -------------------------------------------------------------------------------- /spec/scm_fixtures/cvs.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/cvs.tgz -------------------------------------------------------------------------------- /spec/scm_fixtures/git.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/git.tgz -------------------------------------------------------------------------------- /spec/scm_fixtures/hg.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/hg.tgz -------------------------------------------------------------------------------- /spec/scm_fixtures/svn.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/svn.tgz -------------------------------------------------------------------------------- /spec/scm_fixtures/git_svn.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/git_svn.tgz -------------------------------------------------------------------------------- /spec/scm_fixtures/git_walk.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/git_walk.tgz -------------------------------------------------------------------------------- /spec/scm_fixtures/hg_large.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/hg_large.tgz -------------------------------------------------------------------------------- /spec/scm_fixtures/hg_walk.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/hg_walk.tgz -------------------------------------------------------------------------------- /spec/raw_fixtures/sample-content: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/raw_fixtures/sample-content -------------------------------------------------------------------------------- /spec/scm_fixtures/bzr_colon.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/bzr_colon.tgz -------------------------------------------------------------------------------- /spec/scm_fixtures/bzr_large.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/bzr_large.tgz -------------------------------------------------------------------------------- /spec/scm_fixtures/svn_subdir.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/svn_subdir.tgz -------------------------------------------------------------------------------- /spec/raw_fixtures/invalid-utf-word: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/raw_fixtures/invalid-utf-word -------------------------------------------------------------------------------- /spec/scm_fixtures/git_with_mv.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/git_with_mv.tgz -------------------------------------------------------------------------------- /spec/scm_fixtures/bzr_with_branch.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/bzr_with_branch.tgz -------------------------------------------------------------------------------- /spec/scm_fixtures/git_dupe_delete.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/git_dupe_delete.tgz -------------------------------------------------------------------------------- /spec/scm_fixtures/hg_dupe_delete.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/hg_dupe_delete.tgz -------------------------------------------------------------------------------- /spec/scm_fixtures/bzr_with_authors.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/bzr_with_authors.tgz -------------------------------------------------------------------------------- /spec/scm_fixtures/git_with_master_tag.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/git_with_master_tag.tgz -------------------------------------------------------------------------------- /spec/scm_fixtures/git_with_null_merge.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/git_with_null_merge.tgz -------------------------------------------------------------------------------- /spec/scm_fixtures/git_with_submodules.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/git_with_submodules.tgz -------------------------------------------------------------------------------- /spec/scm_fixtures/bzr_with_subdirectories.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/bzr_with_subdirectories.tgz -------------------------------------------------------------------------------- /spec/scm_fixtures/bzr_with_invalid_encoding.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/bzr_with_invalid_encoding.tgz -------------------------------------------------------------------------------- /spec/scm_fixtures/bzr_with_nested_branches.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/bzr_with_nested_branches.tgz -------------------------------------------------------------------------------- /spec/scm_fixtures/git_with_invalid_encoding.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/git_with_invalid_encoding.tgz -------------------------------------------------------------------------------- /spec/scm_fixtures/git_with_multiple_branch.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/git_with_multiple_branch.tgz -------------------------------------------------------------------------------- /spec/scm_fixtures/hg_with_invalid_encoding.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blackducksoftware/ohloh_scm/HEAD/spec/scm_fixtures/hg_with_invalid_encoding.tgz -------------------------------------------------------------------------------- /spec/helpers/generic_helper.rb: -------------------------------------------------------------------------------- 1 | module GenericHelper 2 | def tmpdir(prefix = 'oh_scm_repo_', &block) 3 | Dir.mktmpdir(prefix, &block) 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /lib/ohloh_scm/bzr/status.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | module Bzr 5 | class Status < OhlohScm::Status 6 | end 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /lib/ohloh_scm/hg/status.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | module Hg 5 | class Status < OhlohScm::Status 6 | end 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /lib/ohloh_scm/svn/status.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | module Svn 5 | class Status < OhlohScm::Status 6 | end 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /lib/ohloh_scm/git_svn/status.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | module GitSvn 5 | class Status < OhlohScm::Status 6 | end 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /lib/ohloh_scm/parser/hg_style: -------------------------------------------------------------------------------- 1 | changeset = '__BEGIN_COMMIT__\nchangeset: {node}\nuser: {author}\ndate: {date}\n__BEGIN_COMMENT__\n{desc}\n__END_COMMENT__\n__END_COMMIT__\n' 2 | -------------------------------------------------------------------------------- /lib/ohloh_scm/git_svn/validation.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | module GitSvn 5 | class Validation < OhlohScm::Validation 6 | end 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /lib/ohloh_scm/py_bridge.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'py_bridge/hg_client' 4 | require_relative 'py_bridge/bzr_client' 5 | 6 | module OhlohScm 7 | module PyBridge 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /.bin/accept_svn_ssl_certificate: -------------------------------------------------------------------------------- 1 | #!/usr/bin/expect -f 2 | 3 | set timeout 10 4 | 5 | eval spawn [lrange $argv 0 end] 6 | 7 | expect { 8 | "(R)eject, accept (t)emporarily or accept (p)ermanently? " { send "p\r"; } 9 | } 10 | -------------------------------------------------------------------------------- /spec/ohloh_scm/version_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe 'Version' do 6 | it 'must return the version string' do 7 | assert OhlohScm::Version::STRING.is_a?(String) 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'rake/testtask' 4 | 5 | ENV['SIMPLECOV_START'] = 'true' 6 | task default: :test 7 | 8 | Rake::TestTask.new do |task| 9 | task.libs << 'spec' 10 | task.pattern = 'spec/**/*_spec.rb' 11 | end 12 | -------------------------------------------------------------------------------- /lib/ohloh_scm/cvs.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'cvs/scm' 4 | require_relative 'cvs/status' 5 | require_relative 'cvs/activity' 6 | require_relative 'cvs/validation' 7 | 8 | module OhlohScm 9 | module Cvs 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /lib/ohloh_scm/git.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'git/scm' 4 | require_relative 'git/status' 5 | require_relative 'git/activity' 6 | require_relative 'git/validation' 7 | 8 | module OhlohScm 9 | module Git 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /lib/ohloh_scm/string_extensions.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | module StringExtensions 5 | refine String do 6 | def camelize 7 | split('_').map(&:capitalize).join 8 | end 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /lib/ohloh_scm/svn.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'svn/scm' 4 | require_relative 'svn/status' 5 | require_relative 'svn/activity' 6 | require_relative 'svn/validation' 7 | 8 | module OhlohScm 9 | module Svn 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /lib/ohloh_scm/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | module Version 5 | STRING = '5.0.0' 6 | GIT = '2.34.1' 7 | SVN = '1.14.1' 8 | CVSNT = '1.12.13' 9 | HG = '4.9.1' 10 | BZR = '2.7.0' 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/ohloh_scm/git_svn.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'git_svn/scm' 4 | require_relative 'git_svn/status' 5 | require_relative 'git_svn/activity' 6 | require_relative 'git_svn/validation' 7 | 8 | module OhlohScm 9 | module GitSvn 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /lib/ohloh_scm/hg.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'parser/hg_parser' 4 | require_relative 'hg/scm' 5 | require_relative 'hg/status' 6 | require_relative 'hg/activity' 7 | require_relative 'hg/validation' 8 | 9 | module OhlohScm 10 | module Hg 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: generic 2 | services: 3 | - docker 4 | before_install: 5 | - docker pull ohdeployer/ohloh_scm:ubuntu18 6 | script: 7 | - docker run -P -v $(pwd):/home/app/ohloh_scm -ti ohdeployer/ohloh_scm:ubuntu18 /bin/sh -c "/etc/init.d/ssh start; rubocop && LANG=en_US.UTF-8 rake test 2> /dev/null" 8 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source 'https://rubygems.org' 4 | 5 | group :development do 6 | gem 'byebug' 7 | gem 'minitest' 8 | gem 'mocha' 9 | gem 'nokogiri' 10 | gem 'rake' 11 | gem 'rubocop', '>= 1.12.0' 12 | gem 'rubocop-performance' 13 | gem 'simplecov' 14 | end 15 | -------------------------------------------------------------------------------- /lib/ohloh_scm/bzr.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'parser/bzr_xml_parser' 4 | require_relative 'bzr/scm' 5 | require_relative 'bzr/status' 6 | require_relative 'bzr/activity' 7 | require_relative 'bzr/validation' 8 | 9 | module OhlohScm 10 | module Bzr 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/ohloh_scm/parser/array_writer.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | class ArrayWriter 5 | attr_accessor :buffer 6 | 7 | def initialize(buffer = []) 8 | @buffer = buffer 9 | end 10 | 11 | def write_commit(commit) 12 | @buffer << commit 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /spec/ohloh_scm/hg/status_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe 'Hg::Status' do 4 | it 'exist? must check if repo exists' do 5 | hg_repo = nil 6 | with_hg_repository('hg') do |hg| 7 | hg_repo = hg 8 | assert hg_repo.status.exist? 9 | end 10 | refute hg_repo.status.exist? 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/ohloh_scm/parser/hg_verbose_style: -------------------------------------------------------------------------------- 1 | changeset = '__BEGIN_COMMIT__\nchangeset: {node}\nuser: {author}\ndate: {date}\n__BEGIN_COMMENT__\n{desc}\n__END_COMMENT__\n__BEGIN_FILES__\n{file_mods}{file_adds}{file_dels}__END_FILES__\n__END_COMMIT__\n' 2 | file_mod = 'M {file_mod}\n' 3 | file_add = 'A {file_add}\n' 4 | file_del = 'D {file_del}\n' 5 | 6 | -------------------------------------------------------------------------------- /spec/.rubocop.yml: -------------------------------------------------------------------------------- 1 | inherit_from: ../.rubocop.yml 2 | 3 | Style/FrozenStringLiteralComment: 4 | Enabled: false 5 | 6 | Layout/LineLength: 7 | Max: 120 8 | 9 | Metrics/AbcSize: 10 | Enabled: false 11 | 12 | Metrics/ClassLength: 13 | Enabled: false 14 | 15 | Metrics/MethodLength: 16 | Enabled: false 17 | 18 | Metrics/BlockLength: 19 | Enabled: false 20 | -------------------------------------------------------------------------------- /.bin/string_encoder: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ruby 2 | # Replaces invalid utf-8 characters with �. 3 | # 4 | # Usage: 5 | # $ cat some_file | string_encoder 6 | 7 | while input = gets 8 | input = input.to_s.force_encoding('UTF-8') 9 | if input.valid_encoding? 10 | puts input 11 | else 12 | puts input.encode('UTF-8', 'binary', invalid: :replace, undef: :replace) 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /lib/ohloh_scm/cvs/status.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | module Cvs 5 | class Status < OhlohScm::Status 6 | def lock? 7 | run "timeout 2m cvsnt -q -d #{scm.url} rlog '#{scm.branch_name}'" 8 | false 9 | rescue StandardError => e 10 | raise 'CVS lock has been found' if /waiting for.*lock in/.match?(e.message) 11 | end 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | require: rubocop-performance 2 | 3 | AllCops: 4 | TargetRubyVersion: 3.0 5 | NewCops: enable 6 | SuggestExtensions: false 7 | 8 | Layout/LineLength: 9 | Max: 99 10 | 11 | Style/Documentation: 12 | Enabled: false 13 | 14 | Style/RegexpLiteral: 15 | Enabled: false 16 | 17 | Metrics/ClassLength: 18 | Enabled: false 19 | 20 | Lint/MissingSuper: 21 | Enabled: false 22 | 23 | Style/OptionalBooleanParameter: 24 | Enabled: false 25 | 26 | Metrics/AbcSize: 27 | Enabled: true 28 | -------------------------------------------------------------------------------- /lib/ohloh_scm/factory.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | module Factory 5 | module_function 6 | 7 | def get_core(opts = {}) 8 | scm_type = opts.fetch(:scm_type, :git) 9 | url = opts.fetch(:url) { raise ArgumentError, 'URL is required' } 10 | branch_name = opts[:branch_name] 11 | username = opts[:username] 12 | password = opts[:password] 13 | 14 | OhlohScm::Core.new(scm_type, url, branch_name, username, password) 15 | end 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /lib/ohloh_scm/hg/validation.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | module Hg 5 | class Validation < OhlohScm::Validation 6 | private 7 | 8 | def validate_server_connection 9 | msg = "The server did not respond to the 'hg id' command. Is the URL correct?" 10 | @errors << [:failed, msg] unless status.exist? 11 | end 12 | 13 | def public_url_regex 14 | %r{^(http|https)://(\w+@)?[\w\-.]+(:\d+)?/[\w\-./~+]*$} 15 | end 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /lib/ohloh_scm/git/validation.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | module Git 5 | class Validation < OhlohScm::Validation 6 | private 7 | 8 | def validate_server_connection 9 | msg = "The server did not respond to the 'git-ls-remote' command. Is the URL correct?" 10 | @errors << [:failed, msg] unless status.exist? 11 | end 12 | 13 | def public_url_regex 14 | %r{^(http|https|git)://(\w+@)?[\w\-.]+(:\d+)?/[\w\-./~+]*$} 15 | end 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /lib/ohloh_scm/bzr/validation.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | module Bzr 5 | class Validation < OhlohScm::Validation 6 | private 7 | 8 | def validate_server_connection 9 | msg = "The server did not respond to the 'bzr revno' command. Is the URL correct?" 10 | @errors << [:failed, msg] unless status.exist? 11 | end 12 | 13 | def public_url_regex 14 | %r{^(((http|https|bzr)://(\w+@)?[\w\-.]+(:\d+)?/)|(lp:[\w\-.~]))[\w\-./~+]*$} 15 | end 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /spec/ohloh_scm/factory_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe 'Factory' do 6 | it 'must provide access to scm, activity and status functions' do 7 | url = 'https://foobar.git' 8 | core = OhlohScm::Factory.get_core(scm_type: :git, url: url) 9 | 10 | assert core.status.scm.is_a?(OhlohScm::Git::Scm) 11 | assert_equal core.scm.url, url 12 | assert core.activity.method(:commits) 13 | assert core.status.method(:exist?) 14 | assert core.validation.method(:validate) 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /spec/ohloh_scm/bzr/scm_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe 'Bzr::Scm' do 6 | it 'must pull the repository correctly' do 7 | with_bzr_repository('bzr') do |src| 8 | tmpdir do |dest_dir| 9 | core = OhlohScm::Factory.get_core(scm_type: :bzr, url: dest_dir) 10 | refute core.status.scm_dir_exist? 11 | 12 | core.scm.pull(src.scm, TestCallback.new) 13 | assert core.status.scm_dir_exist? 14 | assert core.status.exist? 15 | end 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI Pipeline 2 | 3 | on: 4 | pull_request: 5 | types: 6 | - opened 7 | - synchronize 8 | push: 9 | branches: [main] 10 | 11 | jobs: 12 | ci_task: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v3 16 | - name: Run rubocop and tests 17 | run: | 18 | docker pull ohdeployer/ohloh_scm:latest 19 | cmd='/etc/init.d/ssh start; bundle exec rubocop; rake test 2> /dev/null' 20 | docker run --rm -P -v $(pwd):/home/app/ohloh_scm -i ohdeployer/ohloh_scm:latest /bin/sh -c "$cmd" 21 | -------------------------------------------------------------------------------- /lib/ohloh_scm/git/status.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | module Git 5 | class Status < OhlohScm::Status 6 | def branch?(name = scm.branch_name) 7 | return false unless scm_dir_exist? 8 | 9 | activity.branches.include?(name) 10 | end 11 | 12 | def default_branch 13 | return scm.branch_name_or_default unless exist? 14 | 15 | name = run("git remote show '#{scm.url}' | grep 'HEAD branch' | awk '{print $3}'").strip 16 | name.to_s.empty? ? scm.branch_name_or_default : name 17 | end 18 | end 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /lib/ohloh_scm/status.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | class Status 5 | include OhlohScm::System 6 | extend Forwardable 7 | def_delegators :@core, :scm, :activity 8 | attr_reader :errors 9 | 10 | def initialize(core) 11 | @core = core 12 | end 13 | 14 | def exist? 15 | !activity.head_token.to_s.empty? 16 | rescue RuntimeError 17 | logger.debug { $ERROR_INFO.inspect } 18 | false 19 | end 20 | 21 | def scm_dir_exist? 22 | Dir.exist?(scm.vcs_path) 23 | end 24 | 25 | def default_branch; end 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /spec/helpers/system_helper.rb: -------------------------------------------------------------------------------- 1 | # We intend to keep System functions protected. 2 | class SystemPrivateAccessor 3 | extend OhlohScm::System 4 | 5 | class << self 6 | def run_private(cmd) 7 | run(cmd) 8 | end 9 | 10 | def run_with_error_private(cmd) 11 | run_with_err(cmd) 12 | end 13 | end 14 | end 15 | 16 | module SystemHelper 17 | # Cannot use the name `run` since it conflicts with Minitest#run. 18 | def run_p(cmd) 19 | SystemPrivateAccessor.run_private(cmd) 20 | end 21 | 22 | def run_with_error_p(cmd) 23 | SystemPrivateAccessor.run_with_error_private(cmd) 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /spec/raw_fixtures/simultaneous_checkins_2.rlog: -------------------------------------------------------------------------------- 1 | 2 | RCS file: /cvsroot/wikipedia/extensions/icpagent/Makefile,v 3 | head: 1.2 4 | locks: strict 5 | access list: 6 | symbolic names: 7 | keyword substitution: kv 8 | total revisions: 2; selected revisions: 2 9 | description: 10 | ---------------------------- 11 | revision 1.1 12 | date: 2005/01/02 20:07:30; author: midom; state: Exp; 13 | Initial revision 14 | ---------------------------- 15 | revision 1.2 16 | date: 2005/01/02 20:07:30; author: foo_author; state: Exp; lines: +0 -0 17 | I have the same timestamp as my predecessor 18 | ============================================================================= 19 | -------------------------------------------------------------------------------- /.git_hooks/pre-commit: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | echo-color() { echo " $@ "; } 4 | modified_files=$(git diff --cached --name-only | grep '.*.rb$') 5 | 6 | # Run Rubocop 7 | [ -z "$modified_files" ] && exit 0 8 | echo-color 'Running rubocop on cached ruby files:' 9 | bundle exec rubocop $modified_files || exit 1 10 | 11 | test_files=() 12 | for filepath in $modified_files; do 13 | testpath=$(echo $filepath | sed 's/^lib/spec/' | sed 's/\.rb$/_spec.rb/') 14 | [ -e $testpath ] && test_files+=($testpath) 15 | done 16 | 17 | # Run tests 18 | [ -z "$test_files" ] && exit 0 19 | echo-color 'Running related tests for cached ruby files:' 20 | .bin/run-tests $test_files 21 | -------------------------------------------------------------------------------- /lib/ohloh_scm/bzr/scm.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | module Bzr 5 | class Scm < OhlohScm::Scm 6 | def pull(from, callback) 7 | callback.update(0, 1) 8 | 9 | if status.exist? 10 | run "cd '#{url}' && bzr revert && bzr pull --overwrite '#{from.url}'" 11 | else 12 | run "rm -rf '#{url}'" 13 | run "bzr branch '#{from.url}' '#{url}'" 14 | end 15 | 16 | callback.update(1, 1) 17 | end 18 | 19 | def vcs_path 20 | "#{url}/.bzr" 21 | end 22 | 23 | def checkout_files(_names) 24 | # Files are already checked out. 25 | end 26 | end 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /spec/helpers/commit_tokens_helper.rb: -------------------------------------------------------------------------------- 1 | class CommitTokensHelper 2 | def initialize(core, commit_labels, trunk_only: false) 3 | @core = core 4 | @trunk_only = trunk_only 5 | @commit_labels = commit_labels 6 | end 7 | 8 | def between(from, to) 9 | to_labels(@core.activity.commit_tokens(after: from_label(from), up_to: from_label(to), 10 | trunk_only: @trunk_only)) 11 | end 12 | 13 | private 14 | 15 | def to_label(sha1) 16 | @commit_labels.invert[sha1.to_s] 17 | end 18 | 19 | def to_labels(sha1s) 20 | sha1s.map { |sha1| to_label(sha1) } 21 | end 22 | 23 | def from_label(label) 24 | @commit_labels[label] 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /spec/ohloh_scm/git_svn/scm_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe 'GitSvn::Scm' do 4 | it 'must convert to git Repository' do 5 | with_svn_repository('svn') do |src| 6 | tmpdir do |local_dir| 7 | git_svn = get_core(:git_svn, url: local_dir) 8 | git_svn.scm.pull(src.scm, TestCallback.new) 9 | assert git_svn.status.exist? 10 | 11 | assert_equal git_svn.activity.commit_count, 5 12 | end 13 | end 14 | end 15 | 16 | it 'must fetch the repo' do 17 | OhlohScm::GitSvn::Scm.any_instance.expects(:run).times(3) 18 | with_git_svn_repository('git_svn') do |git_svn| 19 | git_svn.scm.pull(git_svn.scm, TestCallback.new) 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /lib/ohloh_scm/scm.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | class Scm 5 | include OhlohScm::System 6 | extend Forwardable 7 | def_delegators :@core, :status, :activity 8 | attr_reader :url, :username, :password 9 | attr_accessor :branch_name 10 | 11 | def initialize(core:, url:, branch_name: nil, username: nil, password: nil) 12 | @core = core 13 | @url = url.strip if url 14 | @branch_name = branch_name&.strip 15 | @branch_name = nil if branch_name&.empty? 16 | @username = username&.strip 17 | @password = password&.strip 18 | end 19 | 20 | def normalize; end 21 | 22 | def pull(_, _); end 23 | 24 | def vcs_path; end 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /spec/ohloh_scm/activity_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe 'Activity' do 6 | describe 'log_filename' do 7 | it 'should return system tmp dir path' do 8 | ENV['OHLOH_SCM_TEMP_FOLDER_PATH'] = nil 9 | core = get_core(:git) 10 | scm = OhlohScm::Activity.new(core) 11 | assert_equal scm.log_filename, "#{Dir.tmpdir}/foobar.log" 12 | end 13 | 14 | it 'should return temp folder path' do 15 | ENV['OHLOH_SCM_TEMP_FOLDER_PATH'] = '/test' 16 | core = get_core(:git) 17 | scm = OhlohScm::Activity.new(core) 18 | assert_equal scm.log_filename, '/test/foobar.log' 19 | ENV['OHLOH_SCM_TEMP_FOLDER_PATH'] = '' 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /lib/ohloh_scm/py_bridge/bzr_client.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'py_client' 4 | 5 | module OhlohScm 6 | module PyBridge 7 | class BzrClient < PyClient 8 | def initialize(repository_url) 9 | @repository_url = repository_url 10 | @py_script = "#{__dir__}/bzr_server.py" 11 | end 12 | 13 | def cat_file(revision, file) 14 | send_command("CAT_FILE|#{revision}|#{file}") 15 | end 16 | 17 | def parent_tokens(revision) 18 | send_command("PARENT_TOKENS|#{revision}").split('|') 19 | end 20 | 21 | private 22 | 23 | def open_repository 24 | send_command("REPO_OPEN|#{@repository_url}") 25 | end 26 | end 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /spec/raw_fixtures/simultaneous_checkins.rlog: -------------------------------------------------------------------------------- 1 | 2 | RCS file: /cvsroot/wikipedia/extensions/icpagent/Makefile,v 3 | head: 1.1 4 | branch: 1.1.1 5 | locks: strict 6 | access list: 7 | symbolic names: 8 | start: 1.1.1.1 9 | dammit: 1.1.1 10 | keyword substitution: kv 11 | total revisions: 2; selected revisions: 2 12 | description: 13 | ---------------------------- 14 | revision 1.1 15 | date: 2005/01/02 20:07:30; author: midom; state: Exp; 16 | branches: 1.1.1; 17 | Initial revision 18 | ---------------------------- 19 | revision 1.1.1.1 20 | date: 2005/01/02 20:07:30; author: midom; state: Exp; lines: +0 -0 21 | ICP agent, for delayed ICP responses on loaded systems... 22 | ============================================================================= 23 | -------------------------------------------------------------------------------- /lib/ohloh_scm/activity.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | class Activity 5 | include OhlohScm::System 6 | extend Forwardable 7 | def_delegators :@core, :scm, :status 8 | def_delegators :scm, :url 9 | 10 | def initialize(core) 11 | @core = core 12 | end 13 | 14 | def log_filename 15 | File.join(temp_folder, "#{url.gsub(/\W/, '')}.log") 16 | end 17 | 18 | def tags; end 19 | 20 | def export; end 21 | 22 | def export_tag; end 23 | 24 | def head_token; end 25 | 26 | def each_commit; end 27 | 28 | def commits; end 29 | 30 | def commit_tokens; end 31 | 32 | def commit_count; end 33 | 34 | def diffs; end 35 | 36 | def cleanup; end 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /lib/ohloh_scm/parser.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'parser/array_writer' 4 | require_relative 'parser/branch_number' 5 | 6 | module OhlohScm 7 | class Parser 8 | class << self 9 | def parse(buffer = '', opts = {}) 10 | buffer = StringIO.new(buffer) if buffer.is_a?(String) 11 | writer = ArrayWriter.new unless block_given? 12 | 13 | internal_parse(buffer, opts) do |commit| 14 | if commit 15 | yield commit if block_given? 16 | writer&.write_commit(commit) 17 | end 18 | end 19 | 20 | writer&.buffer 21 | end 22 | 23 | def internal_parse; end 24 | end 25 | end 26 | end 27 | 28 | require_relative 'parser/git_parser' 29 | require_relative 'parser/cvs_parser' 30 | require_relative 'parser/svn_parser' 31 | -------------------------------------------------------------------------------- /lib/ohloh_scm.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | end 5 | 6 | require 'time' 7 | require 'tmpdir' 8 | require 'forwardable' 9 | require 'nokogiri' 10 | 11 | require 'ohloh_scm/string_extensions' 12 | require 'ohloh_scm/version' 13 | require 'ohloh_scm/system' 14 | require 'ohloh_scm/diff' 15 | require 'ohloh_scm/commit' 16 | require 'ohloh_scm/parser' 17 | require 'ohloh_scm/core' 18 | require 'ohloh_scm/scm' 19 | require 'ohloh_scm/activity' 20 | require 'ohloh_scm/status' 21 | require 'ohloh_scm/validation' 22 | require 'ohloh_scm/py_bridge' 23 | 24 | require 'ohloh_scm/hg' 25 | require 'ohloh_scm/git' 26 | require 'ohloh_scm/bzr' 27 | require 'ohloh_scm/cvs' 28 | require 'ohloh_scm/svn' 29 | require 'ohloh_scm/git_svn' 30 | 31 | require 'ohloh_scm/factory' 32 | 33 | `#{File.expand_path('../.bin/check_scm_version', __dir__)}` 34 | -------------------------------------------------------------------------------- /lib/ohloh_scm/py_bridge/hg_client.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'py_client' 4 | 5 | module OhlohScm 6 | module PyBridge 7 | class HgClient < PyClient 8 | def initialize(repository_url) 9 | @repository_url = repository_url 10 | @py_script = "#{__dir__}/hg_server.py" 11 | end 12 | 13 | def cat_file(revision, file) 14 | send_command("CAT_FILE\t#{revision}\t#{file}") 15 | rescue RuntimeError => e 16 | raise unless /not found in manifest/.match?(e.message) # File does not exist. 17 | end 18 | 19 | def parent_tokens(revision) 20 | send_command("PARENT_TOKENS\t#{revision}").split("\t") 21 | end 22 | 23 | private 24 | 25 | def open_repository 26 | send_command("REPO_OPEN\t#{@repository_url}") 27 | end 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /lib/ohloh_scm/core.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | using OhlohScm::StringExtensions 4 | 5 | module OhlohScm 6 | class Core 7 | extend Forwardable 8 | def_delegators :validation, :validate, :errors 9 | attr_reader :scm, :activity, :status, :validation 10 | 11 | def initialize(scm_type, url, branch_name, username, password) 12 | scm_opts = { core: self, url: url, branch_name: branch_name, 13 | username: username, password: password } 14 | 15 | scm_class_name = scm_type.to_s.camelize 16 | 17 | @scm = OhlohScm.const_get(scm_class_name)::Scm.new(**scm_opts) 18 | @activity = OhlohScm.const_get(scm_class_name)::Activity.new(self) 19 | @status = OhlohScm.const_get(scm_class_name)::Status.new(self) 20 | @validation = OhlohScm.const_get(scm_class_name)::Validation.new(self) 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | $LOAD_PATH << File.expand_path('../lib', __dir__) 4 | 5 | if ENV['SIMPLECOV_START'] 6 | require 'simplecov' 7 | SimpleCov.start { add_filter '/spec/' } 8 | SimpleCov.minimum_coverage 94 9 | end 10 | 11 | require 'ohloh_scm' 12 | require 'minitest' 13 | require 'mocha/minitest' 14 | require 'minitest/autorun' 15 | require 'helpers/repository_helper' 16 | require 'helpers/system_helper' 17 | require 'helpers/generic_helper' 18 | require 'helpers/commit_tokens_helper' 19 | require 'helpers/assert_scm_attr_helper' 20 | 21 | FIXTURES_DIR = File.expand_path('raw_fixtures', __dir__) 22 | 23 | module Minitest 24 | class Test 25 | include RepositoryHelper 26 | include SystemHelper 27 | include GenericHelper 28 | include AssertScmAttrHelper 29 | end 30 | end 31 | 32 | class TestCallback; def update(_, _); end; end 33 | -------------------------------------------------------------------------------- /spec/ohloh_scm/git/status_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe 'Git::Status' do 4 | it 'branch?' do 5 | with_git_repository('git') do |git| 6 | assert_equal git.activity.send(:branches), %w[develop master] 7 | assert git.status.branch?('master') 8 | assert git.status.branch?('develop') 9 | end 10 | end 11 | 12 | describe 'default_branch' do 13 | it 'must return default branch when repository doesnt exist' do 14 | git = OhlohScm::Factory.get_core(scm_type: :git, url: 'foobar') 15 | git.status.stubs(:exist?) 16 | assert_equal git.status.default_branch, 'master' 17 | end 18 | 19 | it 'must return default branch when no HEAD branch is found in remote' do 20 | git = OhlohScm::Factory.get_core(scm_type: :git, url: 'foobar') 21 | git.status.stubs(:exist?).returns(true) 22 | assert_equal git.status.default_branch, 'master' 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /lib/ohloh_scm/data/git_ignore_list.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | GIT_IGNORE_LIST = ['.svn', 4 | 'CVS', 5 | '*.jar', 6 | '*.tar', 7 | '*.gz', 8 | '*.tgz', 9 | '*.zip', 10 | '*.gif', 11 | '*.jpg', 12 | '*.jpeg', 13 | '*.bmp', 14 | '*.png', 15 | '*.tif', 16 | '*.tiff', 17 | '*.ogg', 18 | '*.aiff', 19 | '*.wav', 20 | '*.mp3', 21 | '*.au', 22 | '*.ra', 23 | '*.m4a', 24 | '*.pdf', 25 | '*.mpg', 26 | '*.mov', 27 | '*.qt', 28 | '*.avi', 29 | '*.xbm', 30 | '*.nfs*'].freeze 31 | -------------------------------------------------------------------------------- /spec/raw_fixtures/basic.rlog: -------------------------------------------------------------------------------- 1 | 2 | RCS file: /shared/data/ccvs/repository/intelliglue/UML/intelliglue.zuml,v 3 | head: 1.1 4 | branch: 5 | locks: strict 6 | access list: 7 | symbolic names: 8 | keyword substitution: kv 9 | total revisions: 1; selected revisions: 1 10 | description: 11 | ---------------------------- 12 | revision 1.1 13 | date: 2005/07/25 17:11:06; author: pizzandre; state: Exp; 14 | Addin UNL file with using example- 15 | ============================================================================= 16 | 17 | RCS file: /shared/data/ccvs/repository/intelliglue/articles/IoC_Regras.pdf,v 18 | head: 1.1 19 | branch: 20 | locks: strict 21 | access list: 22 | symbolic names: 23 | keyword substitution: b 24 | total revisions: 1; selected revisions: 1 25 | description: 26 | ---------------------------- 27 | revision 1.1 28 | date: 2005/07/25 17:09:59; author: pizzandre; state: Exp; 29 | *** empty log message *** 30 | ============================================================================= 31 | -------------------------------------------------------------------------------- /spec/string_encoder_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe 'string_encoder' do 6 | before do 7 | @object = Object.new 8 | @object.extend(OhlohScm::System) 9 | def @object.string_encoder 10 | string_encoder_path 11 | end 12 | end 13 | 14 | it 'preserve length of translated content' do 15 | file_path = "#{FIXTURES_DIR}/sample-content" 16 | original_content_length = File.size(file_path) 17 | original_content_lines = File.readlines(file_path).size 18 | 19 | output = `cat #{file_path} | #{@object.string_encoder}` 20 | 21 | assert_equal original_content_length, output.length 22 | assert_equal original_content_lines, output.split("\n").length 23 | end 24 | 25 | it 'must convert invalid characters' do 26 | invalid_utf8_word_path = "#{FIXTURES_DIR}/invalid-utf-word" 27 | 28 | string = `cat #{invalid_utf8_word_path} | #{@object.string_encoder}` 29 | 30 | assert_equal true, string.valid_encoding? 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /.bin/check_scm_version: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ruby 2 | $LOAD_PATH.push File.expand_path('../../lib', __FILE__) 3 | 4 | require 'open3' 5 | require_relative '../lib/ohloh_scm/version' 6 | 7 | SCM_CLI_LIST = %w[git svn cvsnt hg bzr].freeze 8 | 9 | def match_scm_cli(scm_type) 10 | scm_version = get_scm_version(scm_type) 11 | major_version = Object.const_get('OhlohScm::Version::' + scm_type.upcase) 12 | return if scm_version.nil? || major_version.to_i == scm_version.to_i 13 | STDERR.puts "warning: Ohloh SCM is compatible with #{scm_type} v#{major_version.to_i}.x.x. "\ 14 | "The installed #{scm_type} version is #{scm_version}." 15 | end 16 | 17 | def get_scm_version(scm_type) 18 | out, _err, _status = Open3.capture3("#{scm_type} --version --quiet") 19 | out, _err, _status = Open3.capture3("#{scm_type} --version") if out.empty? 20 | out.match(/(\d+\.)(\d+\.)(\d+)/).to_s 21 | rescue => exception 22 | STDERR.puts exception 23 | nil 24 | end 25 | 26 | SCM_CLI_LIST.each do |scm_type| 27 | match_scm_cli(scm_type) 28 | end 29 | -------------------------------------------------------------------------------- /lib/ohloh_scm/system.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'logger' 4 | require 'open3' 5 | 6 | module OhlohScm 7 | module System 8 | protected 9 | 10 | def run(cmd) 11 | out, err, status = Open3.capture3(cmd) 12 | raise "#{cmd} failed: #{out}\n#{err}" unless status.success? 13 | 14 | out 15 | end 16 | 17 | def run_with_err(cmd) 18 | logger.debug { cmd } 19 | out, err, status = Open3.capture3(cmd) 20 | [out, err, status] 21 | end 22 | 23 | def string_encoder_path 24 | File.expand_path('../../.bin/string_encoder', __dir__) 25 | end 26 | 27 | def logger 28 | System.logger 29 | end 30 | 31 | def temp_folder 32 | ENV['OHLOH_SCM_TEMP_FOLDER_PATH'] || Dir.tmpdir 33 | end 34 | 35 | class << self 36 | # Use a single logger instance. 37 | def logger 38 | @logger ||= Logger.new($stderr).tap do |log_obj| 39 | log_obj.level = ENV['SCM_LOG_LEVEL'].to_i 40 | end 41 | end 42 | end 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /spec/ohloh_scm/system_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe 'System' do 4 | describe 'run' do 5 | it 'must run a command succesfully' do 6 | run_p('ls /tmp') 7 | end 8 | 9 | it 'must raise an exception when command fails' do 10 | assert_raises(Exception) { run_p('ls /tmp/foobartest') } 11 | end 12 | end 13 | 14 | describe 'run_with_error' do 15 | it 'must provide error and exitstatus' do 16 | cmd = %q(ruby -e" t = 'Hello World'; STDOUT.puts t; STDERR.puts t ") 17 | stdout, stderr, status = run_with_error_p(cmd) 18 | assert_equal status.exitstatus, 0 19 | assert_equal stdout, "Hello World\n" 20 | assert_equal stderr, "Hello World\n" 21 | end 22 | end 23 | 24 | describe 'logger' do 25 | it 'must allow setting logger level' do 26 | level = (1..5).to_a.sample 27 | OhlohScm::System.logger.level = level 28 | core = OhlohScm::Factory.get_core(scm_type: :git, url: 'foo') 29 | assert_equal core.scm.send(:logger).level, level 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /spec/ohloh_scm/parser/array_writer_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe 'ArrayWriter' do 4 | it 'must work' do 5 | log = <<-LOG.gsub(/^ {6}/, '') 6 | __BEGIN_COMMIT__ 7 | Commit: 1df547800dcd168e589bb9b26b4039bff3a7f7e4 8 | Author: Jason Allen 9 | AuthorEmail: jason@ohloh.net 10 | Date: Fri, 14 Jul 2006 16:07:15 -0700 11 | __BEGIN_COMMENT__ 12 | moving COPYING 13 | 14 | __END_COMMENT__ 15 | 16 | :000000 100755 0000000000000000000000000000000000000000 a7b13ff050aed1191c45d7a5db9a50edcdc5755f A COPYING 17 | LOG 18 | 19 | commits = OhlohScm::GitParser.parse(log) 20 | assert_equal commits.size, 1 21 | commit = commits.first 22 | assert_equal commit.token, '1df547800dcd168e589bb9b26b4039bff3a7f7e4' 23 | assert_equal commit.author_name, 'Jason Allen' 24 | assert_equal commit.author_email, 'jason@ohloh.net' 25 | assert_equal commit.message, "moving COPYING\n" 26 | assert_equal commit.author_date, Time.utc(2006, 7, 14, 23, 7, 15) 27 | assert_equal commit.diffs.size, 1 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /spec/helpers/assert_scm_attr_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module AssertScmAttrHelper 4 | def get_core(scm_type, opts = {}) 5 | OhlohScm::Factory.get_core({ scm_type: scm_type, url: 'foobar' }.merge(opts)) 6 | end 7 | 8 | def assert_url_error(scm_type, *urls) 9 | urls.each do |url| 10 | core = get_core(scm_type, url: url) 11 | refute_empty core.validation.send(:url_errors) 12 | end 13 | end 14 | 15 | def assert_url_valid(scm_type, url) 16 | core = get_core(scm_type, url: url) 17 | assert_nil core.validation.send(:url_errors) 18 | end 19 | 20 | def assert_branch_name_error(scm_type, *branches) 21 | branches.each do |branch_name| 22 | core = get_core(scm_type, url: ':pserver:cvs:cvs@cvs.test.org:/test', branch_name: branch_name) 23 | refute_empty core.validation.send(:branch_name_errors) 24 | end 25 | end 26 | 27 | def assert_branch_name_valid(scm_type, branch_name) 28 | core = get_core(scm_type, url: ':pserver:cvs:cvs@cvs.test.org:/test', branch_name: branch_name) 29 | assert_nil core.validation.send(:branch_name_errors) 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /lib/ohloh_scm/py_bridge/py_client.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | class PyClient 5 | def start 6 | @stdin, @stdout, @stderr, wait_thr = Open3.popen3 "python #{@py_script}" 7 | @pid = wait_thr[:pid] 8 | open_repository 9 | end 10 | 11 | def shutdown 12 | send_command('QUIT') 13 | [@stdin, @stdout, @stderr].reject(&:closed?).each(&:close) 14 | Process.waitpid(@pid, Process::WNOHANG) 15 | end 16 | 17 | private 18 | 19 | def send_command(cmd) 20 | # send the command 21 | @stdin.puts cmd 22 | @stdin.flush 23 | return if cmd == 'QUIT' 24 | 25 | # get status on stderr, first letter indicates state, 26 | # remaing value indicates length of the file content 27 | status = @stderr.read(10) 28 | flag = status[0, 1] 29 | size = status[1, 9].to_i 30 | return if flag == 'F' 31 | 32 | raise_subprocess_error(flag) 33 | 34 | # read content from stdout 35 | @stdout.read(size) 36 | end 37 | 38 | def raise_subprocess_error(flag) 39 | return unless flag == 'E' 40 | 41 | error = @stdout.read(size) 42 | raise "Exception in server process\n#{error}" 43 | end 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /ohloh_scm.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | $LOAD_PATH << File.expand_path('lib', __dir__) 4 | require 'ohloh_scm/version' 5 | 6 | Gem::Specification.new do |gem| 7 | gem.name = 'ohloh_scm' 8 | gem.version = OhlohScm::Version::STRING 9 | gem.authors = ['OpenHub Team at Synopsys'] 10 | gem.email = ['info@openhub.net'] 11 | gem.summary = 'Source Control Management' 12 | gem.description = 'The OpenHub source control management library for \ 13 | interacting with Git, SVN, CVS, Hg and Bzr repositories.' 14 | gem.homepage = 'https://github.com/blackducksoftware/ohloh_scm/' 15 | gem.license = 'GPL-2.0' 16 | 17 | gem.files = `git ls-files -z`.split("\x0") 18 | gem.require_paths = %w[lib] 19 | gem.required_ruby_version = '>= 3.0.0' 20 | gem.post_install_message = "Ohloh SCM is depending on Git #{OhlohScm::Version::GIT}, " \ 21 | "SVN #{OhlohScm::Version::SVN}, CVSNT #{OhlohScm::Version::CVSNT}, " \ 22 | "Mercurial #{OhlohScm::Version::HG} and Bazaar " \ 23 | "#{OhlohScm::Version::BZR}. If the installed version is different, " \ 24 | 'Ohloh SCM may not operate as expected.' 25 | end 26 | -------------------------------------------------------------------------------- /spec/raw_fixtures/simple.svn_log: -------------------------------------------------------------------------------- 1 | ------------------------------------------------------------------------ 2 | r5 | jason | 2006-07-14 16:07:15 -0700 (Fri, 14 Jul 2006) | 1 line 3 | Changed paths: 4 | D /COPYING 5 | A /trunk/COPYING (from /COPYING:4) 6 | 7 | moving COPYING 8 | ------------------------------------------------------------------------ 9 | r4 | jason | 2006-07-14 15:17:08 -0700 (Fri, 14 Jul 2006) | 1 line 10 | Changed paths: 11 | A /COPYING 12 | 13 | added bs COPYING to catch global licenses 14 | ------------------------------------------------------------------------ 15 | r3 | robin | 2006-06-11 11:34:17 -0700 (Sun, 11 Jun 2006) | 1 line 16 | Changed paths: 17 | A /trunk/README 18 | M /trunk/helloworld.c 19 | 20 | added some documentation and licensing info 21 | ------------------------------------------------------------------------ 22 | r2 | robin | 2006-06-11 11:32:13 -0700 (Sun, 11 Jun 2006) | 1 line 23 | Changed paths: 24 | A /trunk/makefile 25 | 26 | added makefile 27 | ------------------------------------------------------------------------ 28 | r1 | robin | 2006-06-11 11:28:00 -0700 (Sun, 11 Jun 2006) | 2 lines 29 | Changed paths: 30 | A /branches 31 | A /tags 32 | A /trunk 33 | A /trunk/helloworld.c 34 | 35 | Initial Checkin 36 | 37 | ------------------------------------------------------------------------ 38 | -------------------------------------------------------------------------------- /spec/ohloh_scm/hg/scm_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe 'Hg::Scm' do 4 | it 'must pull hg repository and clean up non .hg files' do 5 | with_hg_repository('hg') do |src| 6 | tmpdir do |dir| 7 | dest = OhlohScm::Factory.get_core(scm_type: :hg, url: dir) 8 | assert !dest.status.exist? 9 | 10 | dest.scm.pull(src.scm, TestCallback.new) 11 | assert dest.status.exist? 12 | assert_equal Dir.entries(dir).sort, ['.', '..', '.hg'] 13 | 14 | # Commit some new code on the original and pull again 15 | run_p "cd '#{src.scm.url}' && touch foo && hg add foo && hg commit -u test -m test" 16 | assert_equal src.activity.commits.last.message, "test\n" 17 | 18 | dest.scm.pull(src.scm, TestCallback.new) 19 | assert_equal Dir.entries(dir).sort, ['.', '..', '.hg'] 20 | end 21 | end 22 | end 23 | 24 | it 'must checkout_files matching given names' do 25 | with_git_repository('hg') do |src_core| 26 | dir = src_core.scm.url 27 | core = OhlohScm::Factory.get_core(scm_type: :hg, url: dir) 28 | 29 | core.scm.checkout_files(['Gemfile.lock', 'package.json', 'Godeps.json', 'doesnt-exist']) 30 | 31 | assert system("ls #{dir}/Gemfile.lock > /dev/null") 32 | assert system("ls #{dir}/nested/nested_again/package.json > /dev/null") 33 | assert system("ls #{dir}/Godeps/Godeps.json > /dev/null") 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /spec/benchmarks/hg_bzr_bash_vs_py_api.rb: -------------------------------------------------------------------------------- 1 | # NOTE: Setup before running benchmark. Replace hg with bzr in the following commands for bzr benchmarks. 2 | # cd spec/scm_fixtures 3 | # tar xf hg_large.tgz && cd hg_large 4 | # bash create-large-file.sh 14 # Create a file as per given factor # 14=19MB # 15=38MB # 16=76MB # 17=151MB 5 | # hg add && hg commit -m 'temp' 6 | # cd ../../../ 7 | # ruby -I lib spec/benchmarks/hg_bzr_bash_vs_py_api.rb hg 8 | # hg rollback # bzr uncommit # revert last commit to try different file sizes. 9 | 10 | require_relative '../../lib/ohloh_scm' 11 | require 'benchmark' 12 | 13 | OhlohScm::System.logger.level = :error 14 | 15 | scm = ARGV[0] || :hg 16 | repo_path = File.expand_path("../scm_fixtures/#{scm}_large", __dir__) 17 | 18 | puts 'Benchmarks for `cat_file`' 19 | 20 | activity = OhlohScm::Factory.get_core(scm_type: scm, url: repo_path).activity 21 | commit = OhlohScm::Commit.new(token: '1') 22 | diff = OhlohScm::Diff.new(path: 'large.php') 23 | 24 | puts `du -sh #{repo_path}/large.php` 25 | 26 | Benchmark.bmbm 20 do |reporter| 27 | reporter.report("#{scm}[bash api] ") do 28 | if scm.to_s == 'hg' 29 | `cd #{activity.url} && hg cat -r #{commit.token} #{diff.path}` 30 | else 31 | token = commit.token.to_i + 1 32 | `cd #{activity.url} && bzr cat --name-from-revision -r #{token} '#{diff.path}'` 33 | end 34 | end 35 | 36 | reporter.report("#{scm}[python api]") do 37 | activity.cat_file(commit, diff) 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /spec/helpers/repository_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module RepositoryHelper 4 | %w[svn git_svn cvs hg bzr].each do |scm_type| 5 | define_method("with_#{scm_type}_repository") do |name, branch_name = nil, &block| 6 | with_repository(scm_type, name, branch_name) { |core| block.call(core) } 7 | end 8 | end 9 | 10 | def with_git_repository(name, branch_name = 'master', &block) 11 | with_repository('git', name, branch_name, &block) 12 | end 13 | 14 | private 15 | 16 | def with_repository(scm_type, name, branch_name = nil) 17 | source_path = get_fixture_folder_path(name) 18 | Dir.mktmpdir('oh_scm_fixture_') do |dir_path| 19 | setup_repository_archive(source_path, dir_path) 20 | path_prefix = scm_type == 'svn' ? 'file://' : '' 21 | yield OhlohScm::Factory.get_core(scm_type: scm_type, url: "#{path_prefix}#{File.join(dir_path, name)}", 22 | branch_name: branch_name) 23 | end 24 | end 25 | 26 | def setup_repository_archive(source_path, dir_path) 27 | if Dir.exist?(source_path) 28 | `cp -R #{source_path} #{dir_path}` 29 | elsif File.exist?("#{source_path}.tgz") 30 | `tar xzf #{source_path}.tgz --directory #{dir_path}` 31 | else 32 | raise "Repository archive #{source_path} not found." 33 | end 34 | end 35 | 36 | def get_fixture_folder_path(name) 37 | fixture_dir = File.expand_path('../scm_fixtures', __dir__) 38 | File.join(fixture_dir, name) 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /spec/ohloh_scm/hg/validation_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe 'Hg::Validation' do 4 | describe 'validate_server_connection' do 5 | it 'must handle non existent remote source' do 6 | core = OhlohScm::Factory.get_core(scm_type: :hg, url: 'http://www.selenic.com/repo/foobar') 7 | core.validate 8 | refute_empty core.errors 9 | end 10 | end 11 | 12 | describe 'validate url' do 13 | it 'must have errors for invalid urls' do 14 | assert_url_error(:hg, nil, '', 'foo', 'http:/', 'http:://', 'http://', 'http://a') 15 | assert_url_error(:hg, 'www.selenic.com/repo/hello') # missing a protool prefix 16 | assert_url_error(:hg, 'http://www.selenic.com/repo/hello%20world') # no encoded strings allowed 17 | assert_url_error(:hg, 'http://www.selenic.com/repo/hello world') # no spaces allowed 18 | assert_url_error(:hg, 'git://www.selenic.com/repo/hello') # git protocol not allowed 19 | assert_url_error(:hg, 'svn://www.selenic.com/repo/hello') # svn protocol not allowed 20 | assert_url_error(:hg, '/home/robin/hg') 21 | assert_url_error(:hg, 'file:///home/robin/hg') 22 | assert_url_error(:hg, 'ssh://robin@localhost/home/robin/hg') 23 | assert_url_error(:hg, 'ssh://localhost/home/robin/hg') 24 | end 25 | 26 | it 'wont have errors for valid urls' do 27 | assert_url_valid(:hg, 'http://www.selenic.com/repo/hello') 28 | assert_url_valid(:hg, 'http://www.selenic.com:80/repo/hello') 29 | assert_url_valid(:hg, 'https://www.selenic.com/repo/hello') 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /spec/raw_fixtures/file_created_on_branch.rlog: -------------------------------------------------------------------------------- 1 | 2 | RCS file: /Users/robin/cvs_repo//simple/new_file.rb,v 3 | head: 1.2 4 | branch: 5 | locks: strict 6 | access list: 7 | symbolic names: 8 | another_branch: 1.2.0.2 9 | my_branch: 1.1.0.2 10 | keyword substitution: kv 11 | total revisions: 5; selected revisions: 5 12 | description: 13 | ---------------------------- 14 | revision 1.2 15 | date: 2006/06/29 18:14:47; author: robin; state: Exp; lines: +1 -0; kopt: kv; commitid: c3944a41896430e; mergepoint: 1.1.2.1; filename: new_file.rb; 16 | merged new_file.rb from branch onto the HEAD 17 | ---------------------------- 18 | revision 1.1 19 | date: 2006/06/29 18:05:27; author: robin; state: dead; kopt: kv; commitid: bd144a41667e765; filename: new_file.rb; 20 | branches: 1.1.2; 21 | file new_file.rb was initially added on branch my_branch. 22 | ---------------------------- 23 | revision 1.1.2.3 24 | date: 2006/06/29 18:40:38; author: robin; state: dead; lines: +0 -0; kopt: kv; commitid: d4e44a41ea6477e; filename: new_file.rb; 25 | removed new_file.rb from the branch only 26 | ---------------------------- 27 | revision 1.1.2.2 28 | date: 2006/06/29 18:17:49; author: robin; state: Exp; lines: +1 -0; kopt: kv; commitid: c7744a4194cefc8; filename: new_file.rb; 29 | modifed new_file.rb on the branch only 30 | ---------------------------- 31 | revision 1.1.2.1 32 | date: 2006/06/29 18:05:27; author: robin; state: Exp; lines: +1 -0; kopt: kv; commitid: bd144a41667e765; filename: new_file.rb; 33 | added new_file.rb on the branch 34 | ============================================================================= 35 | -------------------------------------------------------------------------------- /spec/raw_fixtures/simple.ohlog: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | moving COPYING 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | added bs COPYING to catch global licenses 14 | 15 | 16 | 17 | 18 | 19 | 20 | added some documentation and licensing info 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | added makefile 29 | 30 | 31 | 32 | 33 | 34 | 35 | Initial Checkin 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /lib/ohloh_scm/hg/scm.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | module Hg 5 | class Scm < OhlohScm::Scm 6 | def pull(remote_scm, callback) 7 | err_msg = "Cannot pull remote_scm #{remote_scm.inspect}" 8 | raise ArgumentError, err_msg unless remote_scm.is_a?(Hg::Scm) 9 | 10 | clone_or_fetch(remote_scm, callback) 11 | end 12 | 13 | def branch_name_or_default 14 | branch_name || :default 15 | end 16 | 17 | def vcs_path 18 | "#{url}/.hg" 19 | end 20 | 21 | def checkout_files(names) 22 | pattern = "(#{names.join('|')})" 23 | run "cd #{url} && hg revert $(hg manifest | grep -P '#{pattern}')" 24 | end 25 | 26 | private 27 | 28 | def clone_or_fetch(remote_scm, callback) 29 | callback.update(0, 1) 30 | 31 | status.exist? ? revert_and_pull(remote_scm) : clone_repository(remote_scm) 32 | 33 | clean_up_disk 34 | 35 | callback.update(1, 1) 36 | end 37 | 38 | def clone_repository(remote_scm) 39 | run "rm -rf '#{url}'" 40 | run "hg clone '#{remote_scm.url}' '#{url}'" 41 | end 42 | 43 | def revert_and_pull(remote_scm) 44 | branch_opts = "-r #{remote_scm.branch_name}" if branch_name 45 | run "cd '#{url}' && hg revert --all && hg pull #{branch_opts} -u -y '#{remote_scm.url}'" 46 | end 47 | 48 | def clean_up_disk 49 | return unless FileTest.exist?(url) 50 | 51 | run "cd #{url} && " \ 52 | "find . -maxdepth 1 -not -name .hg -not -name '*.nfs*' -not -name . -print0 " \ 53 | '| xargs -0 rm -rf --' 54 | end 55 | end 56 | end 57 | end 58 | -------------------------------------------------------------------------------- /spec/ohloh_scm/bzr/validation_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe 'Bzr::Validation' do 6 | describe 'validate_server_connection' do 7 | it 'must handle non existent remote source' do 8 | core = OhlohScm::Factory.get_core(scm_type: :bzr, url: 'lp:foobar') 9 | core.validate 10 | refute_empty core.errors 11 | end 12 | end 13 | 14 | describe 'validate url' do 15 | it 'must have errors for invalid urls' do 16 | assert_url_error(:bzr, nil, '', 'foo', 'http:/', 'http:://', 'http://', 'http://a') 17 | assert_url_error(:bzr, 'http://www.selenic.com/repo/hello%20world') # no encoded strings allowed 18 | assert_url_error(:bzr, 'http://www.selenic.com/repo/hello world') # no spaces allowed 19 | assert_url_error(:bzr, 'git://www.selenic.com/repo/hello') # git protocol not allowed 20 | assert_url_error(:bzr, 'svn://www.selenic.com/repo/hello') # svn protocol not allowed 21 | assert_url_error(:bzr, 'lp://foobar') # lp requires no '//' after colon 22 | assert_url_error(:bzr, 'file:///home/test/bzr') 23 | assert_url_error(:bzr, '/home/test/bzr') 24 | assert_url_error(:bzr, 'bzr+ssh://test@localhost/home/test/bzr') 25 | assert_url_error(:bzr, 'bzr+ssh://localhost/home/test/bzr') 26 | end 27 | 28 | it 'wont have errors for valid urls' do 29 | assert_url_valid(:bzr, 'http://www.selenic.com/repo/hello') 30 | assert_url_valid(:bzr, 'http://www.selenic.com:80/repo/hello') 31 | assert_url_valid(:bzr, 'https://www.selenic.com/repo/hello') 32 | assert_url_valid(:bzr, 'bzr://www.selenic.com/repo/hello') 33 | assert_url_valid(:bzr, 'lp:foobar') 34 | assert_url_valid(:bzr, 'lp:~foobar/bar') 35 | end 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | ast (2.4.3) 5 | byebug (12.0.0) 6 | docile (1.4.1) 7 | json (2.12.2) 8 | language_server-protocol (3.17.0.5) 9 | lint_roller (1.1.0) 10 | mini_portile2 (2.8.9) 11 | minitest (5.25.5) 12 | mocha (2.7.1) 13 | ruby2_keywords (>= 0.0.5) 14 | nokogiri (1.18.8) 15 | mini_portile2 (~> 2.8.2) 16 | racc (~> 1.4) 17 | parallel (1.27.0) 18 | parser (3.3.8.0) 19 | ast (~> 2.4.1) 20 | racc 21 | prism (1.4.0) 22 | racc (1.8.1) 23 | rainbow (3.1.1) 24 | rake (13.2.1) 25 | regexp_parser (2.10.0) 26 | rubocop (1.75.8) 27 | json (~> 2.3) 28 | language_server-protocol (~> 3.17.0.2) 29 | lint_roller (~> 1.1.0) 30 | parallel (~> 1.10) 31 | parser (>= 3.3.0.2) 32 | rainbow (>= 2.2.2, < 4.0) 33 | regexp_parser (>= 2.9.3, < 3.0) 34 | rubocop-ast (>= 1.44.0, < 2.0) 35 | ruby-progressbar (~> 1.7) 36 | unicode-display_width (>= 2.4.0, < 4.0) 37 | rubocop-ast (1.44.1) 38 | parser (>= 3.3.7.2) 39 | prism (~> 1.4) 40 | rubocop-performance (1.10.2) 41 | rubocop (>= 0.90.0, < 2.0) 42 | rubocop-ast (>= 0.4.0) 43 | ruby-progressbar (1.13.0) 44 | ruby2_keywords (0.0.5) 45 | simplecov (0.22.0) 46 | docile (~> 1.1) 47 | simplecov-html (~> 0.11) 48 | simplecov_json_formatter (~> 0.1) 49 | simplecov-html (0.13.1) 50 | simplecov_json_formatter (0.1.4) 51 | unicode-display_width (3.1.4) 52 | unicode-emoji (~> 4.0, >= 4.0.4) 53 | unicode-emoji (4.0.4) 54 | 55 | PLATFORMS 56 | ruby 57 | 58 | DEPENDENCIES 59 | byebug 60 | minitest 61 | mocha 62 | nokogiri 63 | rake 64 | rubocop (>= 1.12.0) 65 | rubocop-performance 66 | simplecov 67 | 68 | BUNDLED WITH 69 | 2.6.9 70 | -------------------------------------------------------------------------------- /lib/ohloh_scm/svn/validation.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | module Svn 5 | class Validation < OhlohScm::Validation 6 | private 7 | 8 | def validate_server_connection 9 | @errors ||= [] 10 | @errors << head_token_error 11 | @errors << url_error 12 | @errors.compact! 13 | rescue StandardError 14 | @errors << server_connection_error 15 | end 16 | 17 | def public_url_regex 18 | /^(http|https|svn):\/\/[A-Za-z0-9_\-.]+(:\d+)?(\/[A-Za-z0-9_\-.\/+%^~ ]*)?$/ 19 | end 20 | 21 | # Subversion usernames have been relaxed from the abstract rules. 22 | # We allow email names as usernames. 23 | def username_errors 24 | return if scm.username.to_s.empty? 25 | 26 | if scm.username.length > 32 27 | [:username, 'The username must not be longer than 32 characters.'] 28 | elsif !scm.username.match?(/^\w[\w@.+\-]*$/) 29 | [:username, 'The username contains illegal characters.'] 30 | end 31 | end 32 | 33 | def head_token_error 34 | return if activity.head_token 35 | 36 | [:failed, "The server did not respond to a 'svn info' command. Is the URL correct?"] 37 | end 38 | 39 | def url_error 40 | root_path = activity.root 41 | 42 | if scm.url[0..root_path.length - 1] != root_path 43 | [:failed, "The URL did not match the Subversion root #{root_path}. Is the URL correct?"] 44 | elsif scm.recalc_branch_name && activity.ls.nil? 45 | [:failed, "The server did not respond to a 'svn ls' command. Is the URL correct?"] 46 | end 47 | end 48 | 49 | def server_connection_error 50 | logger.error { $ERROR_INFO.inspect } 51 | [:failed, 52 | 'An error occured connecting to the server. Check the URL, username, and password.'] 53 | end 54 | end 55 | end 56 | end 57 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:22.04 2 | LABEL maintainer="OpenHub " 3 | 4 | ENV HOME=/home 5 | ENV LC_ALL=en_US.UTF-8 6 | ENV APP_HOME=$HOME/app/ohloh_scm 7 | ENV DEBIAN_FRONTEND=noninteractive 8 | ENV PATH=$HOME/.rbenv/shims:$HOME/.rbenv/bin:$HOME/.rbenv/plugins/ruby-build/bin:$PATH 9 | 10 | RUN apt-get update 11 | RUN apt-get install -y build-essential software-properties-common locales ragel \ 12 | libxml2-dev libpcre3 libpcre3-dev swig gperf openssh-server expect libreadline-dev \ 13 | zlib1g-dev git git-svn subversion cvs ca-certificates 14 | 15 | RUN apt-get install -y python2.7 python2-dev python-pip \ 16 | && ln -s /usr/bin/python2.7 /usr/local/bin/python 17 | 18 | RUN locale-gen en_US.UTF-8 19 | 20 | RUN cd $HOME \ 21 | && git clone https://github.com/rbenv/rbenv.git $HOME/.rbenv \ 22 | && git clone https://github.com/sstephenson/ruby-build.git $HOME/.rbenv/plugins/ruby-build \ 23 | && echo 'eval "$(rbenv init -)"' >> $HOME/.bashrc \ 24 | && echo 'gem: --no-rdoc --no-ri' >> $HOME/.gemrc \ 25 | && echo 'export PATH="$HOME/.rbenv/shims:$HOME/.rbenv/bin:/home/.rbenv/plugins/ruby-build/bin:$PATH"' >> $HOME/.bash_profile \ 26 | && rbenv install 3.1.7 && rbenv global 3.1.7 27 | 28 | RUN git config --global --add safe.directory '*' 29 | 30 | RUN ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa \ 31 | && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys \ 32 | && echo 'StrictHostKeyChecking no' >> /root/.ssh/config 33 | 34 | RUN pip2 install bzr "mercurial==4.9.1" 35 | RUN mkdir -p ~/.bazaar/plugins \ 36 | && cd ~/.bazaar/plugins \ 37 | && bzr branch lp:bzr-xmloutput ~/.bazaar/plugins/xmloutput 38 | 39 | RUN ln -s /usr/bin/cvs /usr/bin/cvsnt 40 | 41 | # Run bundle install before copying source to keep this step cached. 42 | RUN mkdir -p $APP_HOME 43 | COPY Gemfile* $APP_HOME/ 44 | RUN gem install rake bundler:2.6.9 \ 45 | && bundle config --global silence_root_warning 1 \ 46 | && cd $APP_HOME && bundle install 47 | 48 | ADD . $APP_HOME 49 | WORKDIR $APP_HOME 50 | -------------------------------------------------------------------------------- /lib/ohloh_scm/diff.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | # A +Diff+ represents a change to a single file. It can represent the addition or 5 | # deletion of a file, or it can represent a modification of the file contents. 6 | # 7 | # OpenHub does not track filename changes. If a file is renamed, OpenHub treats this 8 | # as the deletion of one file and the creation of another. 9 | # 10 | # OpenHub does not track directories, only the files within directories. 11 | # 12 | # Don't confuse our use of the word "Diff" with a patch file or the output of the 13 | # console tool 'diff'. This object doesn't have anything to do the actual contents 14 | # of the file; it's better to think of this object as representing a single line 15 | # item from a source control log. 16 | class Diff 17 | # The filename of the changed file, relative to the root of the repository. 18 | attr_accessor :path 19 | 20 | # An action code describing the type of change made to the file. 21 | # Action codes are copied directly from the Git standard. 22 | # The action code can be... 23 | # 'A' added 24 | # 'M' modified 25 | # 'D' deleted 26 | attr_accessor :action 27 | 28 | # The SHA1 hash of the file contents both before and after the change. 29 | # These must be computed using the same method as Git. 30 | attr_accessor :parent_sha1, :sha1 31 | 32 | # For Subversion only, a path may be reported as copied from another location. 33 | # These attributes store the path and revision number of the source of the copy. 34 | attr_accessor :from_path, :from_revision 35 | 36 | def initialize(params = {}) 37 | params.each { |k, v| send("#{k}=", v) if respond_to?("#{k}=") } 38 | end 39 | 40 | # eql?() and hash() are implemented so that [].uniq() will work on an array of Diffs. 41 | def eql?(other) 42 | @action.eql?(other.action) && @path.eql?(other.path) && 43 | @sha1.eql?(other.sha1) && @parent_sha1.eql?(other.parent_sha1) 44 | end 45 | 46 | def hash 47 | "#{action}|#{path}|#{sha1}|#{parent_sha1}".hash 48 | end 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /security.md: -------------------------------------------------------------------------------- 1 | ## Security 2 | The Black Duck Vulnerability Disclosure Process is executed by the Product Security Incident Response Team (PSIRT). The Black Duck process is based on well-known industry standards, such as NIST-SP-800-61, ISO 29147, and ISO 30111. 3 | 4 | The Black Duck PSIRT coordinates the response and, if necessary, disclosure of security incidents related to Black Duck products and associated software. Black Duck PSIRT's primary objective is to minimize the risks associated with security incidents in a timely, secure, and responsible manner. 5 | 6 | Black Duck will investigate all reports for Black Duck products/platforms that are currently supported; accepted reports will be prioritized based on severity and other environmental factors.  7 | 8 | If you believe you have found a security vulnerability in any repository that meets Black duck's definition of a security vulnerability, please report it to us as described below. 9 | 10 | ## Reporting Security Issues 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Black Duck PSIRT team through email psirt@blackduck.com. 14 | 15 | Contact Black Duck within 24 hours if you encounter any end user data. Do not view, alter, save, store, transfer, or otherwise access the data, and immediately purge any local information upon reporting the vulnerability to Black Duck. 16 | 17 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 18 | 19 | - Affected Product/Platform and Version 20 | - Technical description of the issue 21 | - Detailed steps to reproduce and/or sample code used to exploit the vulnerability 22 | - Contact information and optional name for acknowledgments 23 | - Proposed disclosure plans 24 | This information will help us triage your report more quickly. 25 | 26 | ## Preferred Languages 27 | We prefer all communications to be in English. 28 | 29 | ## Policy 30 | Black Duck follows the principle of [Coordinated Vulnerability Disclosure.](https://www.blackduck.com/company/legal/vulnerability-disclosure-policy.html) 31 | -------------------------------------------------------------------------------- /spec/raw_fixtures/multiple_revisions.rlog: -------------------------------------------------------------------------------- 1 | 2 | RCS file: /shared/data/ccvs/repository/intelliglue/www/index.html,v 3 | head: 1.9 4 | branch: 5 | locks: strict 6 | access list: 7 | symbolic names: 8 | keyword substitution: kv 9 | total revisions: 9; selected revisions: 9 10 | description: 11 | ---------------------------- 12 | revision 1.9 13 | date: 2005/07/26 20:39:16; author: pizzandre; state: Exp; lines: +2 -2 14 | Issue number: 15 | Obtained from: 16 | Submitted by: 17 | Reviewed by: 18 | Completing and fixing milestones texts 19 | ---------------------------- 20 | revision 1.8 21 | date: 2005/07/26 20:35:13; author: pizzandre; state: Exp; lines: +2 -2 22 | Issue number: 23 | Obtained from: 24 | Submitted by: 25 | Reviewed by: 26 | Adding current milestones- 27 | ---------------------------- 28 | revision 1.7 29 | date: 2005/07/26 20:30:41; author: pizzandre; state: Exp; lines: +1 -1 30 | Issue number:pizzandre 31 | Obtained from:pizzandre 32 | Submitted by:pizzandre 33 | Reviewed by:pizzandre 34 | Adding link to release 1.0 35 | ---------------------------- 36 | revision 1.6 37 | date: 2005/07/26 20:16:13; author: pizzandre; state: Exp; lines: +1 -1 38 | Issue number: 1 39 | Obtained from: pizzandre 40 | Submitted by: pizzandre 41 | Reviewed by: pizzandre 42 | 43 | Adding milestone 1 info 44 | ---------------------------- 45 | revision 1.5 46 | date: 2005/07/15 17:16:15; author: pizzandre; state: Exp; lines: +7 -26 47 | *** empty log message *** 48 | ---------------------------- 49 | revision 1.4 50 | date: 2005/07/15 16:46:35; author: pizzandre; state: Exp; lines: +4 -4 51 | *** empty log message *** 52 | ---------------------------- 53 | revision 1.3 54 | date: 2005/07/15 16:40:17; author: pizzandre; state: Exp; lines: +1 -282 55 | *** empty log message *** 56 | ---------------------------- 57 | revision 1.2 58 | date: 2005/07/15 16:33:54; author: pizzandre; state: Exp; lines: +349 -41 59 | *** empty log message *** 60 | ---------------------------- 61 | revision 1.1 62 | date: 2005/07/15 11:53:30; author: httpd; state: Exp; 63 | Initial data for the intelliglue project 64 | ============================================================================= 65 | -------------------------------------------------------------------------------- /lib/ohloh_scm/parser/branch_number.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | class BranchNumber 5 | def initialize(branch_number) 6 | @num = branch_number.split('.').collect(&:to_i) 7 | # Accomodate CVS magic branch numbers by swapping the magic zero 8 | # That is, 1.1.0.2 => 1.1.2.0 9 | @num[-1], @num[-2] = @num[-2], @num[-1] if (@num.size > 2) && @num[-2].zero? 10 | end 11 | 12 | # Returns true if is an ancestor of this object, 13 | # or if this object follows on the same line. 14 | def on_same_line?(branch_number) 15 | b = branch_number.to_a 16 | 17 | # b has been branched more times than this object. 18 | return false if b.size > @num.size 19 | if b.size == @num.size 20 | # b and a have the same number of branch events. 21 | # If either one inherits from the other then they 22 | # are on the same line. 23 | return inherits_from?(branch_number) || branch_number.inherits_from?(self) 24 | end 25 | 26 | # b has not been branched as often as this object. 27 | # That's OK if b is an ancestor of this object. 28 | inherits_from?(branch_number) if b.size < @num.size 29 | end 30 | 31 | def to_a 32 | @num 33 | end 34 | 35 | protected 36 | 37 | # Returns true if is an ancestor of this object. 38 | # Also returns true if is the same as this object. 39 | # rubocop:disable Metrics/AbcSize 40 | def inherits_from?(branch_number) 41 | b = branch_number.to_a 42 | 43 | return false if b.size > @num.size 44 | 45 | return false if b.size == 2 && descendant?(b) 46 | 47 | unless b.size == 2 48 | 0.upto(b.size - 2) do |i| 49 | return false if b[i] != @num[i] 50 | end 51 | return false if b[-1] > @num[b.size - 1] 52 | end 53 | true 54 | end 55 | # rubocop:enable Metrics/AbcSize 56 | 57 | private 58 | 59 | def descendant?(branch_number) 60 | true if branch_number[0] > @num[0] || 61 | ((branch_number[0] == @num[0]) && (branch_number[1] > @num[1])) 62 | end 63 | end 64 | end 65 | -------------------------------------------------------------------------------- /spec/ohloh_scm/svn/activity_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'mocha' 3 | 4 | describe 'Svn::Activity' do 5 | describe 'cat' do 6 | let(:commit1) { OhlohScm::Commit.new(token: 1) } 7 | let(:hello_diff) { OhlohScm::Diff.new(path: 'helloworld.c') } 8 | 9 | it 'must export data correctly' do 10 | with_svn_repository('svn') do |svn| 11 | tmpdir do |dir| 12 | svn.activity.export(dir) 13 | assert_equal Dir.entries(dir).sort, %w[. .. branches tags trunk] 14 | end 15 | end 16 | end 17 | 18 | it 'must export tags correctly' do 19 | with_svn_repository('svn', 'trunk') do |svn| 20 | tmpdir do |svn_working_folder| 21 | tmpdir('oh_scm_out_dir_') do |dir| 22 | root_path = svn.activity.root 23 | folder_name = root_path.slice(/[^\/]+\/?\Z/) 24 | cmd = "cd #{svn_working_folder} && svn co #{root_path} && cd #{folder_name} " \ 25 | "&& mkdir -p #{root_path.gsub(/^file:../, '')}/db/transactions " \ 26 | "&& svn copy trunk tags/2.0 && svn commit -m 'v2.0' && svn update" 27 | svn.activity.send :run, cmd 28 | 29 | svn.activity.export_tag(dir, '2.0') 30 | 31 | assert_equal Dir.entries(dir).sort, %w[. .. COPYING README helloworld.c makefile] 32 | end 33 | end 34 | end 35 | end 36 | 37 | it 'must get tags correctly' do 38 | with_svn_repository('svn', 'trunk') do |svn| 39 | tmpdir do |svn_working_folder| 40 | root_path = svn.activity.root 41 | folder_name = root_path.slice(/[^\/]+\/?\Z/) 42 | cmd = "cd #{svn_working_folder} && svn co #{root_path} && cd #{folder_name} " \ 43 | "&& mkdir -p #{root_path.gsub(/^file:../, '')}/db/transactions " \ 44 | "&& svn copy trunk tags/2.0 && svn commit -m 'v2.0' && svn update" 45 | svn.activity.send :run, cmd 46 | 47 | assert_equal svn.activity.tags.first[0..1], ['2.0', '6'] 48 | # Avoid millisecond comparision. 49 | assert_equal svn.activity.tags.first[-1].strftime('%F %R'), Time.now.utc.strftime('%F %R') 50 | end 51 | end 52 | end 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /lib/ohloh_scm/commit.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | # A commit is a collection of diffs united by a single timestamp, author, and 5 | # message. 6 | # 7 | # OpenHub's internal data model assumes that commits are immutable, and exist 8 | # in a singly-linked list. That is, commits can be nicely numbered a la 9 | # Subversion, and new commits are always added to the end of the list. 10 | # 11 | # This works for CVS and Subversion, but unfortunately, does not at all map 12 | # to the DAG used by Git, which allows a commit to have multiple parents and 13 | # children, and which allows new commits to appear during a pull which have 14 | # timestamps older than previously known commits. 15 | # 16 | # This means that OpenHub's support for systems like Git is crude at best. For 17 | # the near future, it is the job of the adapter to make the Git commit chain 18 | # appear as much like a single array as possible. 19 | # 20 | class Commit 21 | # This object supports the idea of distinct authors and committers, a la 22 | # Git. However, OpenHub will retain only one of them in its database. It 23 | # prefers author, but will fall back to committer if no author is given. 24 | attr_accessor :author_name, :author_email, :author_date, 25 | :committer_name, :committer_email, :committer_date 26 | 27 | attr_accessor :message, :diffs 28 | 29 | # The token is used to uniquely identify a commit, and can be any type of 30 | # adapter-specific data. 31 | # 32 | # For Subversion, the token is the revision number. 33 | # For Git, the token is the commit SHA1 hash. 34 | # For CVS, which does not support atomic commits with unique IDs, we use 35 | # the approximate timestamp of the change. 36 | attr_accessor :token 37 | 38 | # A pointer back to the adapter that contains this commit. 39 | attr_accessor :scm 40 | 41 | # Hack. To optimize CVS updates, we will store the names of all the 42 | # directories that require updating during this commit. OpenHub itself never 43 | # actually sees this. 44 | attr_accessor :directories 45 | 46 | def initialize(params = {}) 47 | params.each { |k, v| send("#{k}=", v) if respond_to?("#{k}=") } 48 | end 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /lib/ohloh_scm/validation.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'timeout' 4 | 5 | module OhlohScm 6 | class Validation 7 | include OhlohScm::System 8 | extend Forwardable 9 | def_delegators :@core, :scm, :activity, :status 10 | attr_reader :errors 11 | 12 | def initialize(core) 13 | @core = core 14 | end 15 | 16 | def validate 17 | validate_attributes 18 | Timeout.timeout(timeout_interval) { validate_server_connection } if @errors.none? 19 | end 20 | 21 | private 22 | 23 | def validate_server_connection; end 24 | 25 | # rubocop:disable Metrics/AbcSize 26 | def validate_attributes 27 | @errors = [] 28 | @errors << url_errors 29 | @errors << branch_name_errors unless scm.branch_name.to_s.empty? 30 | @errors << username_errors if scm.username 31 | @errors << password_errors if scm.password 32 | @errors.compact! 33 | end 34 | # rubocop:enable Metrics/AbcSize 35 | 36 | def url_errors 37 | error = if scm.url.nil? || scm.url.empty? 38 | "The URL can't be blank." 39 | elsif scm.url.length > 200 40 | 'The URL must not be longer than 200 characters.' 41 | elsif !scm.url.match?(public_url_regex) 42 | 'The URL does not appear to be a valid server connection string.' 43 | end 44 | 45 | [:url, error] if error 46 | end 47 | 48 | def branch_name_errors 49 | if scm.branch_name.length > 80 50 | [:branch_name, 'The branch name must not be longer than 80 characters.'] 51 | elsif !scm.branch_name.match?(/^[\w^\-+.\/\ ]+$/) 52 | [:branch_name, "The branch name may contain only letters, numbers, \ 53 | spaces, and the special characters '_', '-', '+', '/', '^', and '.'"] 54 | end 55 | end 56 | 57 | def username_errors 58 | if scm.username.length > 32 59 | [:username, 'The username must not be longer than 32 characters.'] 60 | elsif !scm.username.match?(/^\w*$/) 61 | [:username, 'The username may contain only A-Z, a-z, 0-9, and underscore (_)'] 62 | end 63 | end 64 | 65 | def password_errors 66 | if scm.password.length > 32 67 | [:password, 'The password must not be longer than 32 characters.'] 68 | elsif !scm.password.match?(/^[\w!@\#$%^&*(){}\[\];?|+\-=]*$/) 69 | [:password, 'The password contains illegal characters'] 70 | end 71 | end 72 | 73 | def timeout_interval 74 | ENV['SCM_URL_VALIDATION_TIMEOUT']&.to_i || 60 75 | end 76 | 77 | def public_url_regex; end 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /lib/ohloh_scm/git_svn/activity.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | module GitSvn 5 | class Activity < OhlohScm::Activity 6 | def_delegators :scm, :url 7 | 8 | def commit_count(opts = {}) 9 | cmd = "#{after_revision(opts)} | grep -E -e '^r[0-9]+.*lines$' | wc -l" 10 | git_svn_log(cmd: cmd, oneline: false).to_i 11 | end 12 | 13 | def commits(opts = {}) 14 | parsed_commits = [] 15 | open_log_file(opts) do |io| 16 | parsed_commits = OhlohScm::SvnParser.parse(io) 17 | end 18 | parsed_commits 19 | end 20 | 21 | def commit_tokens(opts = {}) 22 | cmd = "#{after_revision(opts)} | #{extract_revision_number}" 23 | git_svn_log(cmd: cmd, oneline: false).split.map(&:to_i) 24 | end 25 | 26 | def each_commit(opts = {}, &block) 27 | commits(opts).each(&block) 28 | end 29 | 30 | def head_token 31 | cmd = "--limit=1 | #{extract_revision_number}" 32 | git_svn_log(cmd: cmd, oneline: false) 33 | end 34 | 35 | def cat_file(commit, diff) 36 | cat(git_commit(commit), diff.path) 37 | end 38 | 39 | def cat_file_parent(commit, diff) 40 | cat("#{git_commit(commit)}^", diff.path) 41 | end 42 | 43 | def git_commit(commit) 44 | run("cd #{url} && git svn find-rev r#{commit.token}").strip 45 | end 46 | 47 | private 48 | 49 | def open_log_file(opts = {}, &block) 50 | cmd = "-v #{after_revision(opts)} | #{string_encoder_path} > #{log_filename}" 51 | git_svn_log(cmd: cmd, oneline: false) 52 | File.open(log_filename, 'r', &block) 53 | end 54 | 55 | def log_filename 56 | File.join(temp_folder, "#{url.gsub(/\W/, '')}.log") 57 | end 58 | 59 | def after_revision(opts) 60 | next_token = opts[:after].to_i + 1 61 | next_head_token = head_token.to_i + 1 62 | "-r#{next_token}:#{next_head_token}" 63 | end 64 | 65 | def extract_revision_number 66 | "grep '^r[0-9].*|' | awk -F'|' '{print $1}' | cut -c 2-" 67 | end 68 | 69 | def git_svn_log(cmd:, oneline:) 70 | oneline_flag = '--oneline' if oneline 71 | run("#{git_svn_log_cmd} #{oneline_flag} #{cmd}").strip 72 | end 73 | 74 | def cat(revision, file_path) 75 | file_path = file_path.to_s 76 | run("cd #{url} && git show #{revision}:#{file_path.shellescape} | #{string_encoder_path}") 77 | .strip 78 | end 79 | 80 | def git_svn_log_cmd 81 | "cd #{url} && git svn log" 82 | end 83 | end 84 | end 85 | end 86 | -------------------------------------------------------------------------------- /lib/ohloh_scm/py_bridge/hg_server.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import time 3 | import traceback 4 | 5 | from mercurial import ui, hg 6 | 7 | class HglibPipeServer: 8 | def __init__(self, repository_url): 9 | self.ui = ui.ui() 10 | self.repository = hg.repository(self.ui, repository_url) 11 | 12 | def get_file_content(self, filename, revision): 13 | c = self.changectx(revision) 14 | fc = c[filename] 15 | contents = fc.data() 16 | return contents 17 | 18 | def changectx(self, revision): 19 | if hasattr(self.repository, 'changectx'): 20 | return self.repository.changectx(revision) 21 | else: 22 | rev_no = self.repository.revs(revision).first() 23 | return self.repository[rev_no] 24 | 25 | def get_parent_tokens(self, revision): 26 | c = self.changectx(revision) 27 | parents = [p.hex() for p in c.parents() if p.hex() != '0000000000000000000000000000000000000000'] 28 | return parents 29 | 30 | class Command: 31 | def __init__(self, line): 32 | self.args = line.rstrip().split('\t') 33 | 34 | def get_action(self): 35 | return self.args[0] 36 | 37 | def get_arg(self, num): 38 | return self.args[num] 39 | 40 | def send_status(code, data_len): 41 | sys.stderr.write('%s%09d' % (code, data_len)) 42 | sys.stderr.flush() 43 | 44 | def send_success(data_len=0): 45 | send_status('T', data_len) 46 | 47 | def send_failure(data_len=0): 48 | send_status('F', data_len) 49 | 50 | def send_error(data_len=0): 51 | send_status('E', data_len) 52 | 53 | def send_data(result): 54 | sys.stdout.write(result) 55 | sys.stdout.flush() 56 | 57 | def command_loop(): 58 | while True: 59 | s = sys.stdin.readline() 60 | cmd = Command(s) 61 | if s == '' or cmd.get_action() == 'QUIT': 62 | sys.exit(0) 63 | elif cmd.get_action() == 'REPO_OPEN': 64 | commander = HglibPipeServer(cmd.get_arg(1)) 65 | send_success() 66 | elif cmd.get_action() == 'CAT_FILE': 67 | try: 68 | content = commander.get_file_content(cmd.get_arg(2), cmd.get_arg(1)) 69 | send_success(len(content)) 70 | send_data(content) 71 | except Exception: 72 | send_failure() # Assume file not found 73 | elif cmd.get_action() == 'PARENT_TOKENS': 74 | tokens = commander.get_parent_tokens(cmd.get_arg(1)) 75 | tokens = '\t'.join(tokens) 76 | send_success(len(tokens)) 77 | send_data(tokens) 78 | else: 79 | error = "Invalid Command - %s" % cmd.get_action() 80 | send_error(len(error)) 81 | send_data(error) 82 | sys.exit(1) 83 | 84 | if __name__ == "__main__": 85 | try: 86 | command_loop() 87 | except Exception: 88 | exc_trace = traceback.format_exc() 89 | send_error(len(exc_trace)) 90 | send_data(exc_trace) 91 | sys.exit(1) 92 | -------------------------------------------------------------------------------- /spec/ohloh_scm/git/validation_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe 'Git::Validation' do 4 | it 'wont have errors for valid url' do 5 | core = OhlohScm::Factory.get_core(scm_type: :git, url: 'https://github.com/ruby/ruby') 6 | core.validation.send(:validate_attributes) 7 | assert_empty core.errors 8 | end 9 | 10 | it 'must have errors for invalid branch_name' do 11 | refute_empty get_core(:git, branch_name: 'x' * 81).validation.send(:branch_name_errors) 12 | refute_empty get_core(:git, branch_name: 'foo@bar').validation.send(:branch_name_errors) 13 | end 14 | 15 | it 'must have errors for invalid username' do 16 | refute_empty get_core(:git, username: 'x' * 33).validation.send(:username_errors) 17 | refute_empty get_core(:git, username: 'foo@bar').validation.send(:username_errors) 18 | end 19 | 20 | it 'must have errors for invalid password' do 21 | refute_empty get_core(:git, password: 'x' * 33).validation.send(:password_errors) 22 | refute_empty get_core(:git, password: 'escape').validation.send(:password_errors) 23 | end 24 | 25 | describe 'validate url' do 26 | it 'must have errors for invalid urls' do 27 | assert_url_error(:git, nil, '', 'foo', 'http:/', 'http:://', 'http://', 'http://a') 28 | assert_url_error(:git, 'kernel.org/linux/linux.git') # missing a protocol prefix 29 | assert_url_error(:git, 'http://kernel.org/linux/lin%32ux.git') # no encoded strings allowed 30 | assert_url_error(:git, 'http://kernel.org/linux/linux.git malicious code') # no spaces allowed 31 | assert_url_error(:git, 'svn://svn.mythtv.org/svn/trunk') # svn protocol is not allowed 32 | assert_url_error(:git, '/home/robin/cvs') # local file paths not allowed 33 | assert_url_error(:git, 'file:///home/robin/cvs') # file protocol is not allowed 34 | # pserver is just wrong 35 | assert_url_error(:git, ':pserver:anonymous:@juicereceiver.cvs.sourceforge.net:/cvsroot/juicereceiver') 36 | end 37 | 38 | it 'wont have errors for valid urls' do 39 | assert_url_valid(:git, 'http://kernel.org/pub/scm/git/git.git') 40 | assert_url_valid(:git, 'git://kernel.org/pub/scm/git/git.git') 41 | assert_url_valid(:git, 'https://kernel.org/pub/scm/git/git.git') 42 | assert_url_valid(:git, 'https://kernel.org:8080/pub/scm/git/git.git') 43 | assert_url_valid(:git, 'git://kernel.org/~foo/git.git') 44 | assert_url_valid(:git, 'http://git.onerussian.com/pub/deb/impose+.git') 45 | assert_url_valid(:git, 'https://Person@github.com/Person/some_repo.git') 46 | assert_url_valid(:git, 'http://Person@github.com/Person/some_repo.git') 47 | assert_url_valid(:git, 'https://github.com/Person/some_repo') 48 | assert_url_valid(:git, 'http://github.com/Person/some_repo') 49 | end 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /lib/ohloh_scm/parser/hg_parser.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | # This parser processes Mercurial logs which have been generated using a custom style. 5 | # This custom style provides additional information required by Ohloh. 6 | class HgParser < Parser 7 | class << self 8 | # Use when you want to include diffs 9 | def verbose_style_path 10 | File.expand_path("#{__dir__}/hg_verbose_style") 11 | end 12 | 13 | # Use when you do not want to include diffs 14 | def style_path 15 | File.expand_path("#{__dir__}/hg_style") 16 | end 17 | 18 | # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/BlockLength 19 | # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity 20 | def internal_parse(buffer, _) 21 | e = nil 22 | state = :data 23 | 24 | buffer.each_line do |line| 25 | next_state = state 26 | case state 27 | when :data 28 | case line 29 | when /^changeset:\s+([0-9a-f]+)/ 30 | e = build_commit(Regexp.last_match(1)) 31 | when /^user:\s+(.+?)(\s+<(.+)>)?$/ 32 | e.committer_name = Regexp.last_match(1) 33 | e.committer_email = Regexp.last_match(3) 34 | when /^date:\s+([\d.]+)/ 35 | time = Regexp.last_match(1) 36 | e.committer_date = Time.at(time.to_f).utc 37 | when "__BEGIN_FILES__\n" 38 | next_state = :files 39 | when "__BEGIN_COMMENT__\n" 40 | next_state = :long_comment 41 | when "__END_COMMIT__\n" 42 | yield e if block_given? 43 | e = nil 44 | end 45 | 46 | when :files 47 | case line 48 | when "__END_FILES__\n" 49 | next_state = :data 50 | when /^([MAD]) (.+)$/ 51 | e.diffs << OhlohScm::Diff.new(action: Regexp.last_match(1), 52 | path: Regexp.last_match(2)) 53 | end 54 | 55 | when :long_comment 56 | if line == "__END_COMMENT__\n" 57 | next_state = :data 58 | elsif e.message 59 | e.message << line 60 | else 61 | e.message = line 62 | end 63 | end 64 | state = next_state 65 | end 66 | end 67 | # rubocop:enable Metrics/AbcSize, Metrics/MethodLength, Metrics/BlockLength 68 | # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity 69 | 70 | private 71 | 72 | def build_commit(token) 73 | OhlohScm::Commit.new.tap do |commit| 74 | commit.diffs = [] 75 | commit.token = token 76 | end 77 | end 78 | end 79 | end 80 | end 81 | -------------------------------------------------------------------------------- /lib/ohloh_scm/git_svn/scm.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | module GitSvn 5 | class Scm < OhlohScm::Scm 6 | def initialize(core:, url:, branch_name:, username:, password:) 7 | super 8 | @branch_name = branch_name || :master 9 | end 10 | 11 | def pull(source_scm, callback) 12 | @source_scm = source_scm 13 | convert_to_git(callback) 14 | end 15 | 16 | def accept_ssl_certificate_cmd 17 | File.expand_path('../../../.bin/accept_svn_ssl_certificate', __dir__) 18 | end 19 | 20 | def vcs_path 21 | "#{url}/.git" 22 | end 23 | 24 | def checkout_files(names) 25 | filenames = names.map { |name| "*#{name}" }.join(' ') 26 | run "cd #{url} && git checkout $(git ls-files #{filenames})" 27 | end 28 | 29 | private 30 | 31 | def convert_to_git(callback) 32 | callback.update(0, 1) 33 | if FileTest.exist?(git_path) 34 | accept_certificate_if_prompted 35 | fetch 36 | else 37 | clone 38 | end 39 | 40 | clean_up_disk 41 | callback.update(1, 1) 42 | end 43 | 44 | def git_path 45 | File.join(url, '/.git') 46 | end 47 | 48 | def clone 49 | prepare_dest_dir 50 | accept_certificate_if_prompted 51 | 52 | cmd = "#{password_prompt} git svn clone --quiet #{username_opts} " \ 53 | "'#{@source_scm.url}' '#{url}'" 54 | run(cmd) 55 | end 56 | 57 | def accept_certificate_if_prompted 58 | # git svn does not support non iteractive and serv-certificate options 59 | # Permanently accept svn certificate when it prompts 60 | opts = username_and_password_opts 61 | run "#{accept_ssl_certificate_cmd} svn info #{opts} '#{@source_scm.url}'" 62 | end 63 | 64 | def username_and_password_opts 65 | username = username.to_s.empty? ? '' : "--username #{@source_scm.username}" 66 | password = password.to_s.empty? ? '' : "--password='#{@source_scm.password}'" 67 | "#{username} #{password}" 68 | end 69 | 70 | def password_prompt 71 | password.to_s.empty? ? '' : "echo #{password} |" 72 | end 73 | 74 | def username_opts 75 | username.to_s.empty? ? '' : "--username #{username}" 76 | end 77 | 78 | def prepare_dest_dir 79 | FileUtils.mkdir_p(url) 80 | FileUtils.rm_rf(url) 81 | end 82 | 83 | def fetch 84 | cmd = "cd #{url} && git svn fetch" 85 | run(cmd) 86 | end 87 | 88 | def clean_up_disk 89 | return unless File.exist?(url) 90 | 91 | run "cd #{url} && " \ 92 | "find . -maxdepth 1 -not -name .git -not -name '*.nfs*' -not -name . -print0 " \ 93 | '| xargs -0 rm -rf --' 94 | end 95 | end 96 | end 97 | end 98 | -------------------------------------------------------------------------------- /spec/ohloh_scm/svn/scm_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe 'Svn::Scm' do 4 | it 'must prefix file: to local path' do 5 | assert_nil get_core(:svn, url: '').scm.send(:prefix_file_for_local_path, '') 6 | assert_equal get_core(:svn, url: '/home/test').scm.send(:prefix_file_for_local_path, '/home/test'), 7 | 'file:///home/test' 8 | end 9 | 10 | it 'must require https for sourceforge' do 11 | OhlohScm::Svn::Scm.any_instance.stubs(:recalc_branch_name) 12 | 13 | url = '://svn.code.sf.net/p/gallery/code/trunk/gallery2' 14 | assert_equal get_core(:svn, url: "http#{url}").scm.normalize.url, "https#{url}" 15 | assert_equal get_core(:svn, url: "https#{url}").scm.normalize.url, "https#{url}" 16 | 17 | url = 'https://github.com/blackducksw/ohloh_scm/trunk' 18 | assert_equal get_core(:svn, url: url).scm.normalize.url, url 19 | end 20 | 21 | it 'must recalc branch name' do 22 | with_svn_repository('svn') do |svn_core| 23 | svn_scm = get_core(:svn, url: svn_core.scm.url, branch_name: '').scm 24 | assert_nil svn_scm.branch_name 25 | assert_empty svn_scm.send(:recalc_branch_name) 26 | assert_empty svn_scm.branch_name 27 | 28 | svn_scm = get_core(:svn, url: svn_core.scm.url, branch_name: '/').scm 29 | assert_empty svn_scm.send(:recalc_branch_name) 30 | assert_empty svn_scm.branch_name 31 | 32 | svn_scm = get_core(:svn, url: "#{svn_core.scm.url}/trunk").scm 33 | OhlohScm::Svn::Activity.any_instance.stubs(:root).returns(svn_core.scm.url) 34 | svn_scm.send(:recalc_branch_name) 35 | assert_equal svn_scm.branch_name, '/trunk' 36 | 37 | svn_scm = get_core(:svn, url: "#{svn_core.scm.url}/trunk", branch_name: nil).scm 38 | OhlohScm::Svn::Activity.any_instance.stubs(:root).returns(svn_core.scm.url) 39 | assert_equal svn_scm.normalize.branch_name, '/trunk' 40 | end 41 | end 42 | 43 | describe 'restrict_url_to_trunk' do 44 | it 'must return url when url ends with trunk' do 45 | svn_scm = get_core(:svn, url: 'svn:foobar/trunk').scm 46 | assert_equal svn_scm.restrict_url_to_trunk, svn_scm.url 47 | end 48 | 49 | it 'must append trunk to url and set branch_name when trunk folder is present' do 50 | with_svn_repository('svn') do |svn_core| 51 | scm = svn_core.scm 52 | assert_equal scm.url, svn_core.activity.root 53 | assert_nil scm.branch_name 54 | 55 | scm.restrict_url_to_trunk 56 | 57 | assert_equal scm.url, "#{svn_core.activity.root}/trunk" 58 | assert_equal scm.branch_name, '/trunk' 59 | end 60 | end 61 | 62 | it 'must update url and branch_name when repo has a single subfolder' do 63 | with_svn_repository('svn_subdir') do |svn_core| 64 | scm = svn_core.scm 65 | assert_equal scm.url, svn_core.activity.root 66 | assert_nil scm.branch_name 67 | 68 | scm.restrict_url_to_trunk 69 | 70 | assert_equal scm.url, "#{svn_core.activity.root}/subdir/trunk" 71 | assert_equal scm.branch_name, '/subdir/trunk' 72 | end 73 | end 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /spec/ohloh_scm/git_svn/activity_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'mocha' 3 | 4 | describe 'GitSvn::Activity' do 5 | it 'must return all commit tokens' do 6 | with_git_svn_repository('git_svn') do |git_svn| 7 | assert_equal git_svn.activity.commit_tokens, [1, 2, 3, 5] 8 | assert_equal git_svn.activity.commit_tokens(after: 2), [3, 5] 9 | end 10 | end 11 | 12 | it 'must return commits' do 13 | with_git_svn_repository('git_svn') do |git_svn| 14 | assert_equal git_svn.activity.commits.map(&:token), [1, 2, 3, 5] 15 | assert_equal git_svn.activity.commits(after: 2).map(&:token), [3, 5] 16 | assert_equal git_svn.activity.commits(after: 7).map(&:token), [] 17 | end 18 | end 19 | 20 | it 'must iterate each commit' do 21 | with_git_svn_repository('git_svn') do |git_svn| 22 | commits = [] 23 | git_svn.activity.each_commit { |c| commits << c } 24 | assert_equal git_svn.activity.commits.map(&:token), [1, 2, 3, 5] 25 | end 26 | end 27 | 28 | it 'must return total commit count' do 29 | with_git_svn_repository('git_svn') do |git_svn| 30 | assert_equal git_svn.activity.commit_count, 4 31 | assert_equal git_svn.activity.commit_count(after: 2), 2 32 | end 33 | end 34 | 35 | describe 'cat' do 36 | let(:commit1) { OhlohScm::Commit.new(token: 1) } 37 | let(:hello_diff) { OhlohScm::Diff.new(path: 'helloworld.c') } 38 | 39 | it 'cat_file' do 40 | with_git_svn_repository('git_svn') do |git_svn| 41 | expected = <<-EXPECTED.gsub(/^\s+/, '') 42 | /* Hello, World! */ 43 | #include 44 | main() 45 | { 46 | printf("Hello, World!\\n"); 47 | } 48 | EXPECTED 49 | 50 | assert_equal git_svn.activity.cat_file(commit1, hello_diff) 51 | .delete("\t").strip, expected.strip 52 | end 53 | end 54 | 55 | it 'cat_file_with_non_existent_token' do 56 | with_git_svn_repository('git_svn') do |git_svn| 57 | assert git_svn.activity.cat_file(OhlohScm::Commit.new(token: 999), hello_diff) 58 | end 59 | end 60 | 61 | it 'cat_file_with_invalid_filename' do 62 | with_git_svn_repository('git_svn') do |git_svn| 63 | assert_empty git_svn.activity.cat_file(commit1, OhlohScm::Diff.new(path: 'invalid')) 64 | end 65 | end 66 | 67 | it 'cat_file_parent' do 68 | with_git_svn_repository('git_svn') do |git_svn| 69 | expected = <<-EXPECTED.gsub(/^\s+/, '') 70 | /* Hello, World! */ 71 | #include 72 | main() 73 | { 74 | printf("Hello, World!\\n"); 75 | } 76 | EXPECTED 77 | 78 | commit = OhlohScm::Commit.new(token: 2) 79 | assert_equal git_svn.activity.cat_file_parent(commit, hello_diff).delete("\t"), expected.strip 80 | end 81 | end 82 | 83 | it 'cat_file_parent_with_first_token' do 84 | with_git_svn_repository('git_svn') do |git_svn| 85 | assert git_svn.activity.cat_file(commit1, hello_diff) 86 | end 87 | end 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /lib/ohloh_scm/py_bridge/bzr_server.py: -------------------------------------------------------------------------------- 1 | from bzrlib.branch import Branch 2 | from bzrlib.revisionspec import RevisionSpec 3 | import os 4 | import sys 5 | import time 6 | import traceback 7 | import logging 8 | 9 | class BzrPipeServer: 10 | def __init__(self, repository_url): 11 | self.branch = Branch.open(repository_url) 12 | self.branch.lock_read() 13 | 14 | def get_file_content(self, filename, revision): 15 | rev_spec = RevisionSpec.from_string(revision) 16 | tree = rev_spec.as_tree(self.branch) 17 | file_id = tree.path2id(unicode(filename, 'utf8')) 18 | if file_id == None: 19 | return None 20 | content = tree.get_file_text(file_id) 21 | return content 22 | 23 | def get_parent_tokens(self, revision): 24 | rev_spec = RevisionSpec.from_string(revision) 25 | tree = rev_spec.as_tree(self.branch) 26 | parents = tree.get_parent_ids() 27 | return parents 28 | 29 | def cleanup(self): 30 | self.branch.unlock() 31 | 32 | class Command: 33 | def __init__(self, line): 34 | self.args = line.rstrip().split('|') 35 | 36 | def get_action(self): 37 | return self.args[0] 38 | 39 | def get_arg(self, num): 40 | return self.args[num] 41 | 42 | def send_status(code, data_len): 43 | sys.stderr.write('%s%09d' % (code, data_len)) 44 | sys.stderr.flush() 45 | 46 | def send_success(data_len=0): 47 | send_status('T', data_len) 48 | 49 | def send_failure(data_len=0): 50 | send_status('F', data_len) 51 | 52 | def send_error(data_len=0): 53 | send_status('E', data_len) 54 | 55 | def send_data(result): 56 | sys.stdout.write(result) 57 | sys.stdout.flush() 58 | 59 | def exit_delayed(status, delay=1): 60 | time.sleep(delay) 61 | sys.exit(status) 62 | 63 | def command_loop(): 64 | while True: 65 | cmd = Command(sys.stdin.readline()) 66 | if cmd.get_action() == 'REPO_OPEN': 67 | commander = BzrPipeServer(cmd.get_arg(1)) 68 | send_success() 69 | elif cmd.get_action() == 'CAT_FILE': 70 | content = commander.get_file_content(cmd.get_arg(2), cmd.get_arg(1)) 71 | if content == None: 72 | send_failure() 73 | else: 74 | send_success(len(content)) 75 | send_data(content) 76 | elif cmd.get_action() == 'PARENT_TOKENS': 77 | tokens = commander.get_parent_tokens(cmd.get_arg(1)) 78 | tokens = '|'.join(tokens) 79 | send_success(len(tokens)) 80 | send_data(tokens) 81 | elif cmd.get_action() == 'QUIT': 82 | commander.cleanup() 83 | send_success() 84 | exit_delayed(status=0) 85 | else: 86 | error = "Invalid Command - %s" % cmd.get_action() 87 | send_error(len(error)) 88 | send_data(error) 89 | exit_delayed(status=1) 90 | 91 | if __name__ == "__main__": 92 | try: 93 | handler = logging.FileHandler(os.devnull) 94 | logging.getLogger('bzr').addHandler(handler) 95 | command_loop() 96 | except: 97 | exc_trace = traceback.format_exc() 98 | send_error(len(exc_trace)) 99 | send_data(exc_trace) 100 | exit_delayed(status=1) 101 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Ohloh SCM on OpenHub](https://www.openhub.net/p/ohloh_scm/widgets/project_partner_badge.gif)](https://www.openhub.net/p/ohloh_scm) 2 | ![Coverity Scan Build](https://github.com/blackducksoftware/ohloh_scm/actions/workflows/coverity.yml/badge.svg?branch=main) 3 | ![Build Status](https://github.com/blackducksoftware/ohloh_scm/actions/workflows/ci.yml/badge.svg?branch=main) 4 | 5 | # Ohloh SCM 6 | 7 | The OpenHub source control management library 8 | 9 | ## Overview 10 | 11 | Ohloh SCM is an abstraction layer for source control management systems, 12 | allowing an application to interoperate with various SCMs using a 13 | single interface. 14 | 15 | It was originally developed at OpenHub, and is used to generate 16 | the reports at www.openhub.net. 17 | 18 | ## Using Docker 19 | 20 | One may use Docker to run Ohloh SCM and test changes. 21 | 22 | ```sh 23 | $ git clone https://github.com/blackducksoftware/ohloh_scm 24 | $ cd ohloh_scm 25 | $ docker build -t ohloh_scm:foobar . 26 | 27 | # To run all tests, we need to start the ssh server and set UTF-8 locale for encoding tests. 28 | $ cmd='/etc/init.d/ssh start; LANG=en_US.UTF-8 rake test 2> /dev/null' 29 | $ docker run --rm -P -v $(pwd):/home/app/ohloh_scm -ti ohloh_scm:foobar /bin/sh -c "$cmd" 30 | # This mounts the current folder into the docker container; 31 | # hence any edits made in ohloh_scm on the host machine would reflect in the container. 32 | ``` 33 | 34 | ## Development Setup 35 | 36 | Besides docker, one could setup OhlohScm locally on Ubuntu with the following commands: 37 | 38 | ```sh 39 | sudo apt-get update 40 | sudo apt-get install -y build-essential software-properties-common 41 | 42 | sudo apt-add-repository -y ppa:brightbox/ruby-ng 43 | sudo apt-get update 44 | sudo apt-get install -y ruby2.5 ruby2.5-dev 45 | 46 | sudo apt-get install -y ragel libxml2-dev libpcre3 libpcre3-dev swig gperf 47 | sudo apt-get install -y git git-svn subversion cvs mercurial bzr 48 | sudo ln -s /usr/bin/cvs /usr/bin/cvsnt 49 | 50 | mkdir -p ~/.bazaar/plugins 51 | cd ~/.bazaar/plugins 52 | bzr branch lp:bzr-xmloutput ~/.bazaar/plugins/xmloutput 53 | 54 | gem install bundler 55 | bundle install 56 | 57 | # For running tests 58 | sudo apt-get install -y openssh-server expect locales 59 | sudo locale-gen en_US.UTF-8 60 | ``` 61 | 62 | OhlohScm is currently tested on the following versions: 63 | Git 2.17.1, SVN 1.9.7, CVSNT 2.5.04, Mercurial 4.5.3 and Bazaar 2.8.0 64 | 65 | OhlohScm has been tested with other linux distros and MacOSx. The installation instructions will differ. 66 | Let us know if you need help installing OhlohScm on other distros. 67 | 68 | ## Running tests 69 | 70 | ```sh 71 | $ rake test 72 | $ ./bin/run-test spec/ohloh_scm/version_spec.rb foobar # run a single test matching 'foobar' # Used as /.*foobar.*/. 73 | $ ./bin/run-tests version_spec.rb foobar_spec.rb # run multiple tests. 74 | ``` 75 | 76 | ## Auto check for rubocop compliance and test failures 77 | 78 | This will prevent a git commit if the files being committed fail rubocop or their related tests. 79 | ```sh 80 | $ git config core.hooksPath .git_hooks/ 81 | ``` 82 | ```sh 83 | # Skip hooks when committing temporary code that breaks rubocop/tests. 84 | $ git commit -m 'temp' --no-verify 85 | ``` 86 | 87 | ## Contact OpenHub 88 | 89 | You can reach OpenHub via email at: 90 | [info@openhub.net](mailto:info@openhub.net) 91 | -------------------------------------------------------------------------------- /lib/ohloh_scm/svn/scm.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | module Svn 5 | class Scm < OhlohScm::Scm 6 | def normalize 7 | url = prefix_file_for_local_path(@url) 8 | @url = force_https_if_sourceforge(url) 9 | if branch_name 10 | clean_branch_name 11 | else 12 | @branch_name = recalc_branch_name 13 | end 14 | self 15 | end 16 | 17 | # From the given URL, determine which part of it is the root and 18 | # which part of it is the branch_name. The current branch_name is overwritten. 19 | def recalc_branch_name 20 | @branch_name = url ? url[activity.root.length..] : branch_name 21 | rescue RuntimeError => e 22 | pattern = /(svn:*is not a working copy|Unable to open an ra_local session to URL)/ 23 | @branch_name = '' if e.message&.match?(pattern) # we have a file system 24 | ensure 25 | clean_branch_name 26 | branch_name 27 | end 28 | 29 | def accept_ssl_certificate_cmd 30 | File.expand_path('../../../.bin/accept_svn_ssl_certificate', __dir__) 31 | end 32 | 33 | # Does some simple searching through the server's directory tree for a 34 | # good canditate for the trunk. Basically, we are looking for a trunk 35 | # in order to avoid the heavy lifting of processing all the branches and tags. 36 | # 37 | # There are two simple rules to the search: 38 | # (1) If the current directory contains a subdirectory named 'trunk', go there. 39 | # (2) If the current directory is empty except for a single subdirectory, go there. 40 | # Repeat until neither rule is satisfied. 41 | # 42 | # The url and branch_name of this object will be updated with the selected location. 43 | # The url will be unmodified if there is a problem connecting to the server. 44 | def restrict_url_to_trunk 45 | return url if url.match?(%r{/trunk/?$}) 46 | 47 | list = activity.ls 48 | return url unless list 49 | 50 | if list.include? 'trunk/' 51 | update_url_and_branch_with_trunk 52 | elsif list.size == 1 && list.first[-1..] == '/' 53 | update_url_and_branch_with_subdir(list) 54 | return restrict_url_to_trunk 55 | end 56 | url 57 | end 58 | 59 | private 60 | 61 | def update_url_and_branch_with_trunk 62 | @url = File.join(url, 'trunk') 63 | @branch_name = File.join(branch_name.to_s, 'trunk') 64 | end 65 | 66 | def update_url_and_branch_with_subdir(list) 67 | folder_name = list.first[0..-2] 68 | @url = File.join(url, folder_name) 69 | @branch_name = File.join(branch_name.to_s, folder_name) 70 | end 71 | 72 | def clean_branch_name 73 | return unless branch_name 74 | 75 | @branch_name.chop! if branch_name.to_s.end_with?('/') 76 | end 77 | 78 | def force_https_if_sourceforge(url) 79 | return url unless url =~ /http(:\/\/.*svn\.(sourceforge|code\.sf)\.net.*)/ 80 | 81 | # SourceForge requires https for svnsync 82 | "https#{Regexp.last_match(1)}" 83 | end 84 | 85 | # If the URL is a simple directory path, make sure it is prefixed by file:// 86 | def prefix_file_for_local_path(path) 87 | return if path.empty? 88 | 89 | %r{://}.match?(url) ? url : "file://#{File.expand_path(path)}" 90 | end 91 | end 92 | end 93 | end 94 | -------------------------------------------------------------------------------- /lib/ohloh_scm/parser/svn_parser.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | class SvnParser < Parser 5 | class << self 6 | # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/BlockLength 7 | # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity 8 | def internal_parse(buffer, _opts) 9 | e = nil 10 | state = :data 11 | previous_state = nil 12 | previous_line = nil 13 | 14 | buffer.each_line do |l| 15 | l.chomp! 16 | next_state = state 17 | case state 18 | when :data 19 | if l =~ /^r(\d+) \| (.*) \| (\d+-\d+-\d+ .*) \(.*\) \| .*/ 20 | yield e if e && block_given? 21 | 22 | e = OhlohScm::Commit.new 23 | e.token = Regexp.last_match(1).to_i 24 | e.committer_name = Regexp.last_match(2) 25 | e.committer_date = Time.parse(Regexp.last_match(3)).utc 26 | elsif l == 'Changed paths:' 27 | next_state = :diffs 28 | elsif l.empty? 29 | next_state = :comment 30 | elsif previous_state == :comment 31 | next_state = :comment 32 | e.message ||= '' 33 | e.message << "\n" 34 | e.message << previous_line 35 | e.message << "\n" 36 | e.message << l 37 | end 38 | 39 | when :diffs 40 | if l =~ /^ (\w) ([^()]+)( \(from (.+):(\d+)\))?$/ 41 | e.diffs ||= [] 42 | e.diffs << OhlohScm::Diff.new(action: Regexp.last_match(1), 43 | path: Regexp.last_match(2), 44 | from_path: Regexp.last_match(4), 45 | from_revision: Regexp.last_match(5).to_i) 46 | else 47 | next_state = :comment 48 | end 49 | 50 | # The :log_embedded_within_comment state is special-case code to fix the Wireshark 51 | # project, which includes fragments of svn logs within its comment blocks, which really 52 | # confuses the parser. I am not sure whether only Wireshark does this, but I suspect it 53 | # happens because there is a tool out there somewhere to generate 54 | # these embedded log comments. 55 | when :log_embedded_within_comment 56 | e.message << "\n" 57 | e.message << l 58 | next_state = :comment if /============================ .* log end =+/.match?(l) 59 | 60 | when :comment 61 | if /------------------------------------------------------------------------/.match?(l) 62 | next_state = :data 63 | elsif /============================ .* log start =+/.match?(l) 64 | e.message << "\n" 65 | e.message << l 66 | next_state = :log_embedded_within_comment 67 | elsif e.message 68 | e.message << "\n" 69 | e.message << l 70 | else 71 | e.message = l 72 | end 73 | end 74 | previous_state = state 75 | state = next_state 76 | previous_line = l 77 | end 78 | yield e if block_given? 79 | end 80 | # rubocop:enable Metrics/AbcSize, Metrics/MethodLength, Metrics/BlockLength 81 | # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity 82 | end 83 | end 84 | end 85 | -------------------------------------------------------------------------------- /spec/ohloh_scm/cvs/activity_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe 'Cvs::Activity' do 6 | it 'must return the host' do 7 | activity = get_core(:cvs, url: ':ext:anonymous:@moodle.cvs.sourceforge.net:/cvsroot/moodle', 8 | branch_name: 'contrib').activity 9 | assert_equal activity.send(:host), 'moodle.cvs.sourceforge.net' 10 | end 11 | 12 | it 'must return the protocol' do 13 | activity = get_core(:cvs, url: ':pserver:foo:@foo.com:/cvsroot/a', branch_name: 'b').activity 14 | assert_equal activity.send(:protocol), :pserver 15 | 16 | activity = get_core(:cvs, url: ':ext:foo:@foo.com:/cvsroot/a', branch_name: 'b').activity 17 | assert_equal activity.send(:protocol), :ext 18 | 19 | activity = get_core(:cvs, url: ':pserver:ext:@foo.com:/cvsroot/a', branch_name: 'b').activity 20 | assert_equal activity.send(:protocol), :pserver 21 | end 22 | 23 | it 'must test tags' do 24 | with_cvs_repository('cvs', 'simple') do |cvs| 25 | assert_equal cvs.activity.tags, [['simple_release_tag', '1.1.1.1'], ['simple_vendor_tag', '1.1.1']] 26 | end 27 | end 28 | 29 | it 'must test export_tag' do 30 | with_cvs_repository('cvs', 'simple') do |cvs| 31 | Dir.mktmpdir('oh_scm_tag_') do |dir| 32 | cvs.activity.export_tag(dir, 'simple_release_tag') 33 | assert_equal Dir.entries(dir).sort, ['.', '..', 'foo.rb'] 34 | end 35 | end 36 | end 37 | 38 | it 'must test commits' do 39 | with_cvs_repository('cvs', 'simple') do |cvs| 40 | assert_equal cvs.activity.commits.collect(&:token), ['2006-06-29 16:21:07', 41 | '2006-06-29 18:14:47', 42 | '2006-06-29 18:45:29', 43 | '2006-06-29 18:48:54', 44 | '2006-06-29 18:52:23'] 45 | 46 | # Make sure we are date format agnostic (2008/01/01 is the same as 2008-01-01) 47 | assert_equal cvs.activity.commits(after: '2006/06/29 18:45:29').collect(&:token), 48 | ['2006-06-29 18:48:54', '2006-06-29 18:52:23'] 49 | 50 | assert_equal cvs.activity.commits(after: '2006-06-29 18:45:29') 51 | .collect(&:token), ['2006-06-29 18:48:54', 52 | '2006-06-29 18:52:23'] 53 | 54 | assert_empty cvs.activity.commits(after: '2006/06/29 18:52:23').collect(&:token) 55 | end 56 | end 57 | 58 | it 'must correctly convert commits to git' do 59 | with_cvs_repository('cvs', 'simple') do |cvs| 60 | tmpdir do |tmp_dir| 61 | git_core = OhlohScm::Factory.get_core(url: tmp_dir, branch_name: 'master') 62 | git_core.scm.pull(cvs.scm, TestCallback.new) 63 | utc_dates = ['2006-06-29 16:21:07 UTC', '2006-06-29 18:14:47 UTC', 64 | '2006-06-29 18:45:29 UTC', '2006-06-29 18:48:54 UTC', 65 | '2006-06-29 18:52:23 UTC'] 66 | assert_equal git_core.activity.commits.map(&:author_date).map(&:to_s), utc_dates 67 | end 68 | end 69 | end 70 | 71 | it 'must test commits sets scm' do 72 | with_cvs_repository('cvs', 'simple') do |cvs| 73 | cvs.activity.commits.each do |c| 74 | assert_equal cvs.activity.scm, c.scm 75 | end 76 | end 77 | end 78 | 79 | it 'must test open log file encoding' do 80 | with_cvs_repository('cvs', 'invalid_utf8') do |cvs| 81 | cvs.activity.send(:open_log_file) do |io| 82 | assert_equal io.read.valid_encoding?, true 83 | end 84 | end 85 | end 86 | 87 | it 'commits must work with invalid_encoding' do 88 | with_cvs_repository('cvs', 'invalid_utf8') do |cvs| 89 | cvs.activity.commits 90 | end 91 | end 92 | end 93 | -------------------------------------------------------------------------------- /spec/ohloh_scm/parser/cvs_parser_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe 'CvsParser' do 4 | describe 'parse' do 5 | it 'must return empty array' do 6 | assert_empty OhlohScm::CvsParser.parse('') 7 | end 8 | 9 | it 'must parse the log' do 10 | revisions = OhlohScm::CvsParser.parse File.read("#{FIXTURES_DIR}/basic.rlog") 11 | 12 | assert_equal revisions.size, 2 13 | 14 | assert_equal revisions[0].token, '2005/07/25 17:09:59' 15 | assert_equal revisions[0].committer_name, 'pizzandre' 16 | assert_equal Time.utc(2005, 0o7, 25, 17, 9, 59), revisions[0].committer_date 17 | assert_equal revisions[0].message, '*** empty log message ***' 18 | 19 | assert_equal revisions[1].token, '2005/07/25 17:11:06' 20 | assert_equal revisions[1].committer_name, 'pizzandre' 21 | assert_equal Time.utc(2005, 0o7, 25, 17, 11, 6), revisions[1].committer_date 22 | assert_equal revisions[1].message, 'Addin UNL file with using example-' 23 | end 24 | 25 | # One file with several revisions 26 | it 'must test multiple revisions' do 27 | revisions = OhlohScm::CvsParser.parse File.read("#{FIXTURES_DIR}/multiple_revisions.rlog") 28 | 29 | # There are 9 revisions in the rlog, but some of them are close together with the same message. 30 | # Therefore we bin them together into only 7 revisions. 31 | assert_equal revisions.size, 7 32 | 33 | assert_equal revisions[0].token, '2005/07/15 11:53:30' 34 | assert_equal revisions[0].committer_name, 'httpd' 35 | assert_equal revisions[0].message, 'Initial data for the intelliglue project' 36 | 37 | assert_equal revisions[1].token, '2005/07/15 16:40:17' 38 | assert_equal revisions[1].committer_name, 'pizzandre' 39 | assert_equal revisions[1].message, '*** empty log message ***' 40 | 41 | assert_equal revisions[5].token, '2005/07/26 20:35:13' 42 | assert_equal revisions[5].committer_name, 'pizzandre' 43 | assert_equal "Issue number:\nObtained from:\nSubmitted by:\nReviewed by:\nAdding current milestones-", 44 | revisions[5].message 45 | 46 | assert_equal revisions[6].token, '2005/07/26 20:39:16' 47 | assert_equal revisions[6].committer_name, 'pizzandre' 48 | assert_equal "Issue number:\nObtained from:\nSubmitted by:\nReviewed by:\nCompleting and fixing milestones texts", 49 | revisions[6].message 50 | end 51 | 52 | # A file is created and modified on the branch, then merged to the trunk, then deleted from the branch. 53 | # From the trunk's point of view, we should see only the merge event. 54 | it 'must test file created on branch as seen from trunk' do 55 | revisions = OhlohScm::CvsParser.parse File.read("#{FIXTURES_DIR}/file_created_on_branch.rlog") 56 | assert_equal revisions.size, 1 57 | assert_equal revisions[0].message, 'merged new_file.rb from branch onto the HEAD' 58 | end 59 | 60 | # A file is created on the vender branch. This causes a simultaneous checkin on HEAD 61 | # with a different message ('Initial revision') but same committer_name name and timestamp. 62 | # We should only pick up one of these checkins. 63 | it 'must test simultaneous checkins' do 64 | revisions = OhlohScm::CvsParser.parse File.read("#{FIXTURES_DIR}/simultaneous_checkins.rlog") 65 | assert_equal revisions.size, 1 66 | assert_equal revisions[0].message, 'Initial revision' 67 | end 68 | 69 | # Two different authors check in with two different messages at the exact same moment. 70 | # How this happens is a mystery, but I have seen it in rlogs. 71 | # We arbitrarily choose the first one if so. 72 | it 'must test simultaneous checkins_2' do 73 | revisions = OhlohScm::CvsParser.parse File.read("#{FIXTURES_DIR}/simultaneous_checkins_2.rlog") 74 | assert_equal revisions.size, 1 75 | end 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /spec/ohloh_scm/parser/git_parser_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe 'GitParser' do 4 | describe 'parse' do 5 | it 'must be empty for blank string' do 6 | assert_empty OhlohScm::GitParser.parse('') 7 | end 8 | 9 | it 'must return epoch time for log with no date' do 10 | sample_log = <<-SAMPLE.gsub(/^ {8}/, '') 11 | __BEGIN_COMMIT__ 12 | Commit: 1df547800dcd168e589bb9b26b4039bff3a7f7e4 13 | Author: Jason Allen 14 | AuthorEmail: jason@ohloh.net 15 | Date:#{' '} 16 | __BEGIN_COMMENT__ 17 | moving COPYING 18 | 19 | __END_COMMENT__ 20 | SAMPLE 21 | commits = OhlohScm::GitParser.parse(sample_log) 22 | assert_equal commits.size, 1 23 | assert_equal commits[0].author_date, Time.utc(1970, 1, 1, 0, 0, 0) 24 | end 25 | 26 | it 'must return epoch time for log with invalid date' do 27 | sample_log = <<-SAMPLE.gsub(/^ {8}/, '') 28 | __BEGIN_COMMIT__ 29 | Commit: 1df547800dcd168e589bb9b26b4039bff3a7f7e4 30 | Author: Jason Allen 31 | AuthorEmail: jason@ohloh.net 32 | Date: Mon, Jan 01 2012 05:00:00 -0500 33 | __BEGIN_COMMENT__ 34 | moving COPYING 35 | 36 | __END_COMMENT__ 37 | SAMPLE 38 | 39 | commits = OhlohScm::GitParser.parse(sample_log) 40 | assert_equal commits.size, 1 41 | assert_equal commits[0].author_date, Time.utc(1970, 1, 1, 0, 0, 0) 42 | end 43 | 44 | it 'must parse a log correctly' do 45 | sample_log = <<-SAMPLE.gsub(/^ {8}/, '') 46 | __BEGIN_COMMIT__ 47 | Commit: 1df547800dcd168e589bb9b26b4039bff3a7f7e4 48 | Author: Jason Allen 49 | AuthorEmail: jason@ohloh.net 50 | Date: Fri, 14 Jul 2006 16:07:15 -0700 51 | __BEGIN_COMMENT__ 52 | moving COPYING 53 | 54 | __END_COMMENT__ 55 | 56 | :000000 100755 0000000000000000000000000000000000000000 a7b13ff050aed1191c45d7a5db9a50edcdc5755f A COPYING 57 | 58 | __BEGIN_COMMIT__ 59 | Commit: 2e9366dd7a786fdb35f211fff1c8ea05c51968b1 60 | Author: Robin Luckey 61 | AuthorEmail: robin@ohloh.net 62 | Date: Sun, 11 Jun 2006 11:34:17 -0700 63 | __BEGIN_COMMENT__ 64 | added some documentation and licensing info 65 | 66 | __END_COMMENT__ 67 | 68 | :100644 100644 d4a46caf1891fccebb726504f34794a0ca5d2e42 41dc0d12cb9eaa30e57aa7126b1227ba320ad297 M README 69 | :100644 000000 41dc0d12cb9eaa30e57aa7126b1227ba320ad297 0000000000000000000000000000000000000000 D helloworld.c 70 | SAMPLE 71 | 72 | commits = OhlohScm::GitParser.parse(sample_log) 73 | 74 | assert_equal commits.size, 2 75 | 76 | assert_equal commits[0].token, '1df547800dcd168e589bb9b26b4039bff3a7f7e4' 77 | assert_equal commits[0].author_name, 'Jason Allen' 78 | assert_equal commits[0].author_email, 'jason@ohloh.net' 79 | assert_equal commits[0].message, "moving COPYING\n" 80 | assert_equal commits[0].author_date, Time.utc(2006, 7, 14, 23, 7, 15) 81 | assert_equal commits[0].diffs.size, 1 82 | 83 | assert_equal commits[0].diffs[0].action, 'A' 84 | assert_equal commits[0].diffs[0].path, 'COPYING' 85 | 86 | assert_equal commits[1].token, '2e9366dd7a786fdb35f211fff1c8ea05c51968b1' 87 | assert_equal commits[1].author_name, 'Robin Luckey' 88 | assert_equal commits[1].author_email, 'robin@ohloh.net' 89 | assert_equal commits[1].message, "added some documentation and licensing info\n" 90 | assert_equal commits[1].author_date, Time.utc(2006, 6, 11, 18, 34, 17) 91 | assert_equal commits[1].diffs.size, 2 92 | 93 | assert_equal commits[1].diffs[0].action, 'M' 94 | assert_equal commits[1].diffs[0].path, 'README' 95 | assert_equal commits[1].diffs[1].action, 'D' 96 | assert_equal commits[1].diffs[1].path, 'helloworld.c' 97 | end 98 | end 99 | end 100 | -------------------------------------------------------------------------------- /lib/ohloh_scm/svn/activity.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | module Svn 5 | class Activity < OhlohScm::Activity 6 | def_delegators :scm, :url 7 | 8 | def root 9 | Regexp.last_match(1) if info =~ /^Repository Root: (.+)$/ 10 | end 11 | 12 | def username_and_password_opts(source_scm = scm) 13 | username = source_scm.username.to_s.empty? ? '' : "--username #{source_scm.username}" 14 | password = source_scm.password.to_s.empty? ? '' : "--password='#{source_scm.password}'" 15 | "#{username} #{password}" 16 | end 17 | 18 | def ls(path = nil, revision = 'HEAD') 19 | stdout = run "svn ls --trust-server-cert --non-interactive -r #{revision} " \ 20 | "#{username_and_password_opts} " \ 21 | "'#{uri_encode(File.join(root.to_s, scm.branch_name.to_s, 22 | path.to_s))}@#{revision}'" 23 | collect_files(stdout) 24 | rescue StandardError => e 25 | logger.error(e.message) && nil 26 | end 27 | 28 | def export(dest_dir, commit_id = 'HEAD') 29 | FileUtils.mkdir_p(File.dirname(dest_dir)) 30 | run 'svn export --trust-server-cert --non-interactive --ignore-externals --force ' \ 31 | "-r #{commit_id} '#{uri_encode(File.join(root.to_s, scm.branch_name.to_s))}' " \ 32 | "'#{dest_dir}'" 33 | end 34 | 35 | def export_tag(dest_dir, tag_name) 36 | tag_url = "#{base_path}/tags/#{tag_name}" 37 | run 'svn export --trust-server-cert --non-interactive --ignore-externals --force ' \ 38 | "'#{tag_url}' '#{dest_dir}'" 39 | end 40 | 41 | # Svn root is not usable here since several projects are nested in subfolders. 42 | # e.g. https://svn.apache.org/repos/asf/openoffice/ooo-site/trunk/ 43 | # http://svn.apache.org/repos/asf/httpd/httpd/trunk 44 | # http://svn.apache.org/repos/asf/maven/plugin-testing/trunk 45 | # all have the same root value(https://svn.apache.org/repos/asf) 46 | def tags 47 | doc = Nokogiri::XML(`svn ls --xml #{base_path}/tags`) 48 | doc.xpath('//lists/list/entry').map do |entry| 49 | tag_name = entry.xpath('name').text 50 | revision = entry.xpath('commit').attr('revision').text 51 | commit_time = entry.xpath('commit/date').text 52 | date_string = Time.parse(commit_time) unless commit_time.to_s.strip.empty? 53 | [tag_name, revision, date_string] 54 | end 55 | end 56 | 57 | def head_token 58 | return unless info =~ /^Revision: (\d+)$/ 59 | 60 | Regexp.last_match(1).to_i 61 | end 62 | 63 | private 64 | 65 | def collect_files(stdout) 66 | stdout.split("\n").map do |line| 67 | # CVSROOT/ is found in cvs repos converted to svn. 68 | line.chomp unless line.chomp.empty? || line == 'CVSROOT/' 69 | end.compact.sort 70 | end 71 | 72 | def info(path = nil, revision = 'HEAD') 73 | @info ||= {} 74 | uri = path ? File.join(root, scm.branch_name.to_s, path) : url 75 | @info[[path, revision]] ||= 76 | run 'svn info --trust-server-cert --non-interactive -r ' \ 77 | "#{revision} #{username_and_password_opts} '#{uri_encode(uri)}@#{revision}'" 78 | end 79 | 80 | # Because uri(with branch) may have characters(e.g. space) that break the shell command. 81 | def uri_encode(uri) 82 | # URI.encode is declared obsolete, however we couldn't find an alternative. 83 | # CGI.escape('foo bar') => foo+bar # `svn log svn://...foo+bar` won't work. 84 | # URI.encode('foo bar') => foo%20bar # `svn log svn://...foo%20bar` works in ruby 2.x 85 | # URI::DEFAULT_PARSER.escape('foo bar') => foo%20bar # This works in ruby 3.x 86 | URI::DEFAULT_PARSER.escape(uri) 87 | end 88 | 89 | def base_path 90 | url.sub(/(.*)(branches|trunk|tags)(.*)/, '\1').chomp('/') 91 | end 92 | end 93 | end 94 | end 95 | -------------------------------------------------------------------------------- /spec/benchmarks/process_spawn_benchmark.rb: -------------------------------------------------------------------------------- 1 | # Usage: ruby process_spawn_benchmark.rb [-n ] [-m ] 2 | # Run posix-spawn (Ruby extension) benchmarks and report to standard output. 3 | # 4 | # Options: 5 | # -n, --count=NUM total number of processes to spawn. 6 | # -m, --mem-size=MB RES size to bloat to before performing benchmarks. 7 | # 8 | # Benchmarks run with -n 500 -m 100 by default. 9 | require 'optparse' 10 | require 'posix-spawn' 11 | require 'benchmark' 12 | require 'digest/sha1' 13 | require 'open3' 14 | 15 | class ProcessSpawnBenchmark 16 | include Benchmark 17 | 18 | def initialize(allocate, iterations) 19 | bloat_main_process_memory(allocate) 20 | repo_path = setup_test_repository 21 | @cmd = "cd #{repo_path} && git log" 22 | @iterations = iterations 23 | benchmark_all 24 | end 25 | 26 | private 27 | 28 | def bloat_main_process_memory(allocate) 29 | _ = 'x' * allocate 30 | memory_used = `ps aux | grep #{File.basename(__FILE__)} | grep -v grep | awk '{print $6/1000}'`.strip.to_i 31 | puts "Parent process: #{memory_used}MB RESidual memory" 32 | end 33 | 34 | def setup_test_repository 35 | repo_name = 'git' 36 | repo_path = File.expand_path("../scm_fixtures/#{repo_name}.tgz", __dir__) 37 | dest_path = "/tmp/#{repo_name}" 38 | system("rm -rf #{dest_path} && tar xvf #{repo_path} -C /tmp/ > /dev/null") 39 | dest_path 40 | end 41 | 42 | def benchmark_all 43 | puts "Benchmarking: #{@iterations} child processes spawned for each" 44 | bmbm 40 do |reporter| 45 | @reporter = reporter 46 | benchmark_open3 47 | benchmark_posix_spawn_child 48 | benchmark_posix_spawn 49 | benchmark_popen 50 | benchmark_system 51 | benchmark_spawn if Process.respond_to?(:spawn) 52 | benchmark_fork_exec 53 | end 54 | end 55 | 56 | def benchmark_open3 57 | @reporter.report('Open3.capture3 => ') do 58 | @iterations.times do 59 | stdout, = Open3.capture3(@cmd) 60 | verify_output(stdout, __method__) 61 | end 62 | end 63 | end 64 | 65 | def benchmark_posix_spawn_child 66 | @reporter.report('POSIX::Spawn::Child => ') do 67 | @iterations.times do 68 | child = POSIX::Spawn::Child.new(@cmd) 69 | verify_output(child.out, __method__) 70 | end 71 | end 72 | end 73 | 74 | def benchmark_posix_spawn 75 | @reporter.report('pspawn (posix_spawn) => ') do 76 | @iterations.times do 77 | pid = POSIX::Spawn.pspawn("#{@cmd} > /dev/null") 78 | Process.wait(pid) 79 | end 80 | end 81 | end 82 | 83 | def benchmark_popen 84 | @reporter.report('IO.popen:') do 85 | @iterations.times do 86 | IO.popen(@cmd).each { |_line| } # consume output to avoid Lint/EmptyBlock 87 | end 88 | end 89 | end 90 | 91 | def benchmark_system 92 | @reporter.report('``:') do 93 | @iterations.times do 94 | stdout = `#{@cmd}` 95 | verify_output(stdout, __method__) 96 | end 97 | end 98 | end 99 | 100 | def benchmark_spawn 101 | @reporter.report('spawn (native):') do 102 | @iterations.times do 103 | pid = Process.spawn("#{@cmd} > /dev/null") 104 | Process.wait(pid) 105 | end 106 | end 107 | end 108 | 109 | def benchmark_fork_exec 110 | @reporter.report('fspawn (fork/exec):') do 111 | @iterations.times do 112 | pid = POSIX::Spawn.fspawn("#{@cmd} > /dev/null") 113 | Process.wait(pid) 114 | end 115 | end 116 | end 117 | 118 | def verify_output(stdout, method_name) 119 | return if Digest::SHA1.hexdigest(stdout) == 'df31df68785baa8725e40e2a2583bb8f7e9dd3c5' 120 | 121 | raise "Git log output did not match for #{method_name.slice(/benchmark_(.+)$/, 1)}" 122 | end 123 | end 124 | 125 | allocate = 100 * (1024**2) 126 | iterations = 500 127 | ARGV.options do |o| 128 | o.on('-n', '--count=num') { |val| iterations = val.to_i } 129 | o.on('-m', '--mem-size=MB') { |val| allocate = val.to_i * (1024**2) } 130 | o.parse! 131 | end 132 | 133 | ProcessSpawnBenchmark.new(allocate, iterations) 134 | -------------------------------------------------------------------------------- /lib/ohloh_scm/cvs/validation.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | module Cvs 5 | class Validation < OhlohScm::Validation 6 | include OhlohScm::System 7 | 8 | private 9 | 10 | def validate_server_connection 11 | return if ls 12 | 13 | @errors << [:failed, "The cvs server did not respond to an 'ls' command. 14 | Are the URL and branch name correct?"] 15 | end 16 | 17 | def branch_name_errors 18 | if scm.branch_name.to_s.empty? 19 | [:branch_name, "The branch name can't be blank."] 20 | elsif scm.branch_name.length > 200 21 | [:branch_name, 'The branch name must not be longer than 200 characters.'] 22 | elsif !scm.branch_name.match?(/^[\w\-+.\/\ ]+$/) 23 | [:branch_name, "The branch name may contain only letters, 24 | numbers, spaces, and the special characters '_', '-', '+', '/', and '.'"] 25 | end 26 | end 27 | 28 | def public_url_regex 29 | /^:(pserver|ext):[\w\-+_]*(:[\w\-+_]*)?@[\w\-+.]+:[0-9]*\/[\w\-+.\/]*$/ 30 | end 31 | 32 | # Returns an array of file and directory names from the remote server. 33 | # Directory names will end with a trailing '/' character. 34 | # 35 | # Directories named "CVSROOT" are always ignored, and thus never returned. 36 | # 37 | # An empty array means that the call succeeded, but the remote directory is empty. 38 | # A nil result means that the call failed and the remote server could not be queried. 39 | def ls(path = nil) 40 | path = File.join(scm.branch_name, path.to_s) 41 | 42 | cmd = "cvsnt -q -d #{scm.url} ls -e '#{path}'" 43 | 44 | activity.ensure_host_key 45 | 46 | stdout, stderr = run_with_err(cmd) 47 | files = get_filenames(stdout) 48 | 49 | error_handled = handle_error(stderr, files, path, cmd) 50 | return unless error_handled 51 | 52 | files.sort 53 | end 54 | 55 | def get_filenames(output) 56 | files = [] 57 | output.each_line do |s| 58 | s.strip! 59 | s = "#{Regexp.last_match(1)}/" if s =~ /^D\/(.*)\/\/\/\/$/ 60 | s = Regexp.last_match(1) if s =~ /^\/(.*)\/.*\/.*\/.*\/$/ 61 | next if s == 'CVSROOT/' 62 | 63 | files << s if s && !s.empty? 64 | end 65 | files 66 | end 67 | 68 | # rubocop:disable Metrics/MethodLength 69 | def handle_error(stderr, files, path, cmd) 70 | # Some of the cvs 'errors' are just harmless problems with some directories. 71 | # If we recognize all the error messages, then nothing is really wrong. 72 | # If some error messages go unhandled, then there really is an error. 73 | stderr.each_line do |error| 74 | error.strip! 75 | error_handled = error.empty? 76 | 77 | if error =~ /cvs server: New directory `(#{Regexp.escape(path.to_s)}\/)?(.*)' -- ignored/ 78 | files << "#{Regexp.last_match(2)}/" 79 | error_handled = true 80 | end 81 | 82 | ignored_error_handled(error, path) 83 | logger.warn { "'#{cmd}' resulted in unhandled error '#{error}'" } unless error_handled 84 | break unless error_handled 85 | end 86 | end 87 | # rubocop:enable Metrics/MethodLength 88 | 89 | def ignored_error_handled(error, path) 90 | ignored_error_messages = [ 91 | /Listing modules on server/, /Listing module: #{Regexp.escape(path.to_s)}/, 92 | /-m wrapper option is not supported remotely; ignored/, 93 | /cannot open directory .* No such file or directory/, 94 | /ignoring module/, /skipping directory/, 95 | /existing repository .* does not match/, 96 | /nothing known about/, 97 | 98 | # The signal 11 error should not really be ignored, but many CVS servers 99 | # including dev.eclipse.org return it at the end of every ls. 100 | /Terminated with fatal signal 11/ 101 | ] 102 | ignored_error_messages.any? { |msg| error.match?(msg) } 103 | end 104 | end 105 | end 106 | end 107 | -------------------------------------------------------------------------------- /spec/ohloh_scm/svn/validation_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe 'Svn::Status' do 4 | it 'should validate usernames' do 5 | [nil, '', 'joe_36', 'a' * 32, 'robin@ohloh.net'].each do |username| 6 | assert !get_core(:svn, username: username).validation.send(:username_errors) 7 | end 8 | end 9 | 10 | it 'should validate rejected urls' do 11 | [nil, '', 'foo', 'http:/', 'http:://', 'http://', 12 | 'sourceforge.net/svn/project/trunk', # missing a protocol prefix 13 | 'http://robin@svn.sourceforge.net/', # must not include a username with the url 14 | '/home/robin/cvs', # local file paths not allowed 15 | 'git://kernel.org/whatever/linux.git', # git protocol is not allowed 16 | ':pserver:anonymous:@juicereceiver.cvs.sourceforge.net:/cvsroot/juicereceiver', # pserver is just wrong 17 | 'svn://svn.gajim.org:/gajim/trunk', # invalid port number 18 | 'svn://svn.gajim.org:abc/gajim/trunk', # invalid port number 19 | 'svn log https://svn.sourceforge.net/svnroot/myserver/trunk'].each do |url| 20 | # Rejected for both internal and public use 21 | assert get_core(:svn, url: url).validation.send(:url_errors) 22 | end 23 | end 24 | 25 | it 'should validate urls' do 26 | [ 27 | 'https://svn.sourceforge.net/svnroot/opende/trunk', # https protocol OK 28 | 'svn://svn.gajim.org/gajim/trunk', # svn protocol OK 29 | 'http://svn.mythtv.org/svn/trunk/mythtv', # http protocol OK 30 | 'https://svn.sourceforge.net/svnroot/vienna-rss/trunk/2.0.0', # periods, numbers and dashes OK 31 | 'svn://svn.gajim.org:3690/gajim/trunk', # port number OK 32 | 'http://svn.mythtv.org:80/svn/trunk/mythtv', # port number OK 33 | 'http://svn.gnome.org/svn/gtk+/trunk', # + character OK 34 | 'http://svn.gnome.org', # no path, no trailing /, just a domain name is OK 35 | 'http://brlcad.svn.sourceforge.net/svnroot/brlcad/rt^3/trunk', # a caret ^ is allowed 36 | 'http://www.thus.ch/~patrick/svn/pvalsecc', # ~ is allowed 37 | 'http://franklinmath.googlecode.com/svn/trunk/Franklin Math' # space is allowed in path 38 | ].each do |url| 39 | # Accepted for both internal and public use 40 | assert !get_core(:svn, url: url).validation.send(:url_errors) 41 | end 42 | end 43 | 44 | # These urls are not available to the public 45 | it 'should reject public urls' do 46 | ['file:///home/robin/svn'].each do |url| 47 | assert get_core(:svn, url: url).validation.send(:url_errors) 48 | end 49 | end 50 | 51 | it 'should validate_server_connection' do 52 | with_svn_repository('svn') do |svn| 53 | svn.validation.send(:validate_server_connection) 54 | assert_empty svn.validation.errors 55 | end 56 | end 57 | 58 | it 'should strip trailing whitespace in branch_name' do 59 | assert_equal get_core(:svn, branch_name: '/trunk/').scm.normalize.branch_name, '/trunk' 60 | end 61 | 62 | it 'should catch exception when validating server connection' do 63 | git_svn = get_core(:svn) 64 | git_svn.validation.instance_variable_set('@errors', nil) 65 | git_svn.validation.send :validate_server_connection 66 | msg = 'An error occured connecting to the server. Check the URL, username, and password.' 67 | assert_equal git_svn.validation.errors, [[:failed, msg]] 68 | end 69 | 70 | it 'should validate head token when validating server connection' do 71 | git_svn = get_core(:svn) 72 | git_svn.validation.instance_variable_set('@errors', nil) 73 | OhlohScm::Svn::Activity.any_instance.stubs(:head_token).returns(nil) 74 | git_svn.validation.expects(:url_error) 75 | git_svn.validation.send :validate_server_connection 76 | msg = "The server did not respond to a 'svn info' command. Is the URL correct?" 77 | assert_equal git_svn.validation.errors, [[:failed, msg]] 78 | end 79 | 80 | it 'should validate url when validating server connection' do 81 | git_svn = get_core(:svn) 82 | git_svn.validation.instance_variable_set('@errors', nil) 83 | OhlohScm::Svn::Activity.any_instance.stubs(:head_token).returns('') 84 | OhlohScm::Svn::Activity.any_instance.stubs(:root).returns('tt') 85 | git_svn.validation.send :validate_server_connection 86 | assert_equal git_svn.validation.errors, 87 | [[:failed, 'The URL did not match the Subversion root tt. Is the URL correct?']] 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /spec/ohloh_scm/cvs/validation_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe 'Cvs::Validation' do 4 | describe 'validate_server_connection' do 5 | it 'must handle non existent remote source' do 6 | url = ':pserver:anonymous:@foobar.xyz_example.org:/cvsroot' 7 | core = OhlohScm::Factory.get_core(scm_type: :cvs, url: url, branch_name: 'foo') 8 | core.validate 9 | refute_empty core.errors 10 | end 11 | end 12 | 13 | it 'must have errors for invalid branch_name' do 14 | assert_nil get_core(:cvs, branch_name: 'x' * 81).validation.send(:branch_name_errors) 15 | refute_empty get_core(:cvs, branch_name: 'x' * 201).validation.send(:branch_name_errors) 16 | refute_empty get_core(:cvs, branch_name: 'foo@bar').validation.send(:branch_name_errors) 17 | end 18 | 19 | it 'must test rejected urls' do 20 | assert_url_error(:cvs, nil, '', 'foo', 'http:/', 'http:://', 'http://', 'http://a') 21 | assert_url_error(:cvs, ':pserver') # that's not enough 22 | assert_url_error(:cvs, ':pserver:anonymous') # still not enough 23 | assert_url_error(:cvs, ':pserver:anonymous:@ipodder.cvs.sourceforge.net') # missing the path 24 | assert_url_error(:cvs, ':pserver:anonymous:::@ipodder.cvs.sourceforge.net:/cvsroot/ipodder') # too many colons 25 | assert_url_error(:cvs, ':pserver@ipodder.cvs.sourceforge.net:/cvsroot/ipodder') # not enough colons 26 | # hostname and path not separated by colon 27 | assert_url_error(:cvs, ':pserver:anonymous:@ipodder.cvs.sourceforge.net/cvsroot/ipodder') 28 | assert_url_error(:cvs, ':pserver:anonymous:@ipodder.cvs.source/forge.net:/cvsroot/ipodder') # slash in hostname 29 | assert_url_error(:cvs, ':pserver:anonymous:ipodder.cvs.sourceforge.net:/cvsroot/ipodder') # missing @ 30 | # path does not begin at root 31 | assert_url_error(:cvs, ':pserver:anonymous:@ipodder.cvs.sourceforge.net:cvsroot/ipodder') 32 | # no encoded chars allowed 33 | assert_url_error(:cvs, ':pserver:anonymous:@ipodder.cvs.sourceforge.net:/cvsr%23oot/ipodder') 34 | assert_url_error(:cvs, ':pserver:anonymous:@ipodder.cvs.sourceforge.net:/cvsroot/ipodder;asdf') # no ; in url 35 | # spaces not allowed 36 | assert_url_error(:cvs, ':pserver:anonymous:@ipodder.cvs.sourceforge.net:/cvsroot/ipodder malicious code') 37 | assert_url_error(:cvs, 'sourceforge.net/svn/project/trunk') # missing a protocol prefix 38 | assert_url_error(:cvs, 'file:///home/robin/cvs') # file protocol is not allowed 39 | assert_url_error(:cvs, 'http://svn.sourceforge.net') # http protocol is not allowed 40 | assert_url_error(:cvs, 'git://kernel.org/whatever/linux.git') # git protocol is not allowed 41 | assert_url_error(:cvs, 'ext@kernel.org/whatever/linux.git') 42 | end 43 | 44 | it 'must test accepted urls' do 45 | assert_url_valid(:cvs, ':pserver:anonymous:@ipodder.cvs.sourceforge.net:/cvsroot/ipodder') 46 | assert_url_valid(:cvs, ':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot') 47 | assert_url_valid(:cvs, ':pserver:anonymous:@cvs-mirror.mozilla.org:/cvsroot') 48 | assert_url_valid(:cvs, ':pserver:guest:@cvs.dev.java.net:/shared/data/ccvs/repository') 49 | assert_url_valid(:cvs, ':pserver:anoncvs:password@anoncvs.postgresql.org:/projects/cvsroot') 50 | assert_url_valid(:cvs, ':pserver:anonymous:@rubyeclipse.cvs.sourceforge.net:/cvsroot/rubyeclipse') 51 | assert_url_valid(:cvs, ':pserver:cvs:cvs@cvs.winehq.org:/home/wine') 52 | assert_url_valid(:cvs, ':pserver:tcpdump:anoncvs@cvs.tcpdump.org:/tcpdump/master') 53 | assert_url_valid(:cvs, ':pserver:anonymous:@user-mode-linux.cvs.sourceforge.net:/cvsroot/user-mode-linux') 54 | assert_url_valid(:cvs, ':pserver:anonymous:@sc2.cvs.sourceforge.net:/cvsroot/sc2') 55 | # Hyphen should be OK in username 56 | assert_url_valid(:cvs, ':pserver:cool-dev:@sc2.cvs.sourceforge.net:/cvsroot/sc2') 57 | # Underscores should be ok in path 58 | assert_url_valid(:cvs, ':pserver:cvs_anon:@cvs.scms.waikato.ac.nz:/usr/local/global-cvs/ml_cvs') 59 | # Pluses should be OK 60 | assert_url_valid(:cvs, ':pserver:anonymous:freefem++@idared.ann.jussieu.fr:/Users/pubcvs/cvs') 61 | assert_url_valid(:cvs, ':ext:anoncvs@opensource.conformal.com:/anoncvs/scrotwm') 62 | end 63 | 64 | it 'must test rejected branch_names' do 65 | assert_branch_name_error(:cvs, nil, '', '%', ';', '&', "\n", "\t") 66 | end 67 | 68 | it 'must test accepted branch_names' do 69 | assert_branch_name_valid(:cvs, 'myproject') 70 | assert_branch_name_valid(:cvs, 'my/project') 71 | assert_branch_name_valid(:cvs, 'my/project/2.0') 72 | assert_branch_name_valid(:cvs, 'my_project') 73 | assert_branch_name_valid(:cvs, '0') 74 | assert_branch_name_valid(:cvs, 'My .Net Module') 75 | assert_branch_name_valid(:cvs, 'my-module') 76 | assert_branch_name_valid(:cvs, 'my-module++') 77 | end 78 | end 79 | -------------------------------------------------------------------------------- /spec/ohloh_scm/cvs/scm_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe 'Cvs::Scm' do 4 | it 'must test symlink fixup' do 5 | scm = get_core(:cvs, url: ':pserver:anoncvs:@cvs.netbeans.org:/cvs').scm 6 | scm.normalize 7 | assert_equal scm.url, ':pserver:anoncvs:@cvs.netbeans.org:/shared/data/ccvs/repository' 8 | 9 | scm = get_core(:cvs, url: ':pserver:anoncvs:@cvs.dev.java.net:/cvs').scm 10 | scm.normalize 11 | assert_equal scm.url, ':pserver:anoncvs:@cvs.dev.java.net:/shared/data/ccvs/repository' 12 | 13 | scm = get_core(:cvs, url: ':PSERVER:ANONCVS:@CVS.DEV.JAVA.NET:/cvs').scm 14 | scm.normalize 15 | assert_equal scm.url, ':PSERVER:ANONCVS:@CVS.DEV.JAVA.NET:/shared/data/ccvs/repository' 16 | 17 | scm = get_core(:cvs, url: ':pserver:anonymous:@cvs.gna.org:/cvs/eagleusb').scm 18 | scm.normalize 19 | assert_equal scm.url, ':pserver:anonymous:@cvs.gna.org:/var/cvs/eagleusb' 20 | end 21 | 22 | it 'must test sync_pserver_username_password' do 23 | # Pull username only from url 24 | scm = get_core(:cvs, url: ':pserver:guest:@ohloh.net:/test').scm 25 | scm.normalize 26 | assert_equal scm.url, ':pserver:guest:@ohloh.net:/test' 27 | assert_equal scm.username, 'guest' 28 | assert_equal scm.password, '' 29 | 30 | # Pull username and password from url 31 | scm = get_core(:cvs, url: ':pserver:guest:secret@ohloh.net:/test').scm 32 | scm.normalize 33 | 34 | assert_equal scm.url, ':pserver:guest:secret@ohloh.net:/test' 35 | assert_equal scm.username, 'guest' 36 | assert_equal scm.password, 'secret' 37 | 38 | # Apply username and password to url 39 | scm = get_core(:cvs, url: ':pserver::@ohloh.net:/test', username: 'guest', password: 'secret').scm 40 | scm.normalize 41 | assert_equal scm.url, ':pserver:guest:secret@ohloh.net:/test' 42 | assert_equal scm.username, 'guest' 43 | assert_equal scm.password, 'secret' 44 | 45 | # Passwords disagree, use :password attribute 46 | scm = get_core(:cvs, url: ':pserver:guest:old@ohloh.net:/test', username: 'guest', password: 'new').scm 47 | scm.normalize 48 | assert_equal scm.url, ':pserver:guest:new@ohloh.net:/test' 49 | assert_equal scm.username, 'guest' 50 | assert_equal scm.password, 'new' 51 | end 52 | 53 | it 'must test guess_forge' do 54 | scm = get_core(:cvs, url: nil).scm 55 | assert_nil scm.send(:guess_forge) 56 | 57 | scm = get_core(:cvs, url: 'garbage_in_garbage_out').scm 58 | assert_nil scm.send(:guess_forge) 59 | 60 | scm = get_core(:cvs, url: ':pserver:anonymous:@boost.cvs.sourceforge.net:/cvsroot/boost').scm 61 | assert_equal scm.send(:guess_forge), 'sourceforge.net' 62 | 63 | scm = get_core(:cvs, url: ':pserver:guest:@cvs.dev.java.net:/cvs').scm 64 | assert_equal scm.send(:guess_forge), 'java.net' 65 | 66 | scm = get_core(:cvs, url: ':PSERVER:ANONCVS:@CVS.DEV.JAVA.NET:/cvs').scm 67 | assert_equal scm.send(:guess_forge), 'java.net' 68 | 69 | scm = get_core(:cvs, url: ':pserver:guest:@colorchooser.dev.java.net:/cvs').scm 70 | assert_equal scm.send(:guess_forge), 'java.net' 71 | end 72 | 73 | it 'must test local directory trim' do 74 | scm = get_core(:cvs, url: '/Users/robin/cvs_repo/', branch_name: 'simple').scm 75 | assert_equal scm.send(:trim_directory, '/Users/robin/cvs_repo/simple/foo.rb'), '/Users/robin/cvs_repo/simple/foo.rb' 76 | end 77 | 78 | it 'must test remote directory trim' do 79 | scm = get_core(:cvs, url: ':pserver:anonymous:@moodle.cvs.sourceforge.net:/cvsroot/moodle', 80 | branch_name: 'contrib').scm 81 | assert_equal scm.send(:trim_directory, '/cvsroot/moodle/contrib/foo.rb'), 'foo.rb' 82 | end 83 | 84 | it 'must test remote directory trim with port number' do 85 | scm = get_core(:cvs, url: ':pserver:anoncvs:anoncvs@libvirt.org:2401/data/cvs', branch_name: 'libvirt').scm 86 | assert_equal scm.send(:trim_directory, '/data/cvs/libvirt/docs/html/Attic'), 'docs/html/Attic' 87 | end 88 | 89 | it 'must test ordered directory list' do 90 | scm = get_core(:cvs, url: ':pserver:anonymous:@moodle.cvs.sourceforge.net:/cvsroot/moodle', 91 | branch_name: 'contrib').scm 92 | 93 | list = scm.send(:build_ordered_directory_list, %i[/cvsroot/moodle/contrib/foo/bar 94 | /cvsroot/moodle/contrib 95 | /cvsroot/moodle/contrib/hello 96 | /cvsroot/moodle/contrib/hello]) 97 | assert_equal list.size, 4 98 | assert_equal list, ['', 'foo', 'hello', 'foo/bar'] 99 | end 100 | 101 | it 'must test ordered directory list ignores Attic' do 102 | scm = get_core(:cvs, url: ':pserver:anonymous:@moodle.cvs.sourceforge.net:/cvsroot/moodle', 103 | branch_name: 'contrib').scm 104 | 105 | list = scm.send(:build_ordered_directory_list, %i[/cvsroot/moodle/contrib/foo/bar 106 | /cvsroot/moodle/contrib/Attic 107 | /cvsroot/moodle/contrib/hello/Attic]) 108 | 109 | assert_equal list.size, 4 110 | assert_equal list, ['', 'foo', 'hello', 'foo/bar'] 111 | end 112 | end 113 | -------------------------------------------------------------------------------- /lib/ohloh_scm/parser/git_parser.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | # This parser processes Git whatchanged generated using a custom style. 5 | # This custom style provides additional information required by OpenHub. 6 | class GitParser < Parser 7 | class << self 8 | def whatchanged 9 | "git whatchanged --root -m --abbrev=40 --max-count=1 --always --pretty=#{format}" 10 | end 11 | 12 | def format 13 | "format:'__BEGIN_COMMIT__%nCommit: %H%nAuthor: %an%nAuthorEmail: " \ 14 | "%ae%nDate: %aD%n__BEGIN_COMMENT__%n%s%n%b%n__END_COMMENT__%n'" 15 | end 16 | 17 | ANONYMOUS = '(no author)' unless defined?(ANONYMOUS) 18 | 19 | # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/BlockLength 20 | # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity 21 | def internal_parse(io, _) 22 | e = nil 23 | state = :key_values 24 | io.each do |line| 25 | line.chomp! 26 | 27 | # Kind of a hack: the diffs section is not always present. 28 | # If we are expecting a line of diffs, but instead find a line 29 | # starting with "Commit: ", that means the diffs section 30 | # is missing for this commit, and we need to fix up our state. 31 | state = :key_values if state == :diffs && line =~ /^Commit: ([a-z0-9]+)$/ 32 | 33 | case state 34 | when :key_values 35 | case line 36 | when /^Commit: ([a-z0-9]+)$/ 37 | sha1 = Regexp.last_match(1) 38 | yield e if e 39 | e = build_commit(sha1) 40 | when /^Author: (.+)$/ 41 | e.author_name = Regexp.last_match(1) 42 | when /^Date: (.*)$/ 43 | # MUST be RFC2822 format to parse properly, else defaults to epoch time 44 | e.author_date = parse_date(Regexp.last_match(1)) 45 | when '__BEGIN_COMMENT__' 46 | state = :message 47 | when /^AuthorEmail: (.+)$/ 48 | e.author_email = Regexp.last_match(1) 49 | # In the rare case that the Git repository does not contain any names, 50 | # we use the email instead (see OpenEmbedded for example). 51 | if e.author_name.to_s.empty? || e.author_name == ANONYMOUS 52 | e.author_name = Regexp.last_match(1) 53 | end 54 | end 55 | 56 | when :message 57 | if line == '__END_COMMENT__' 58 | state = :diffs 59 | elsif line != '' 60 | if e.message 61 | e.message << "\n" << line 62 | else 63 | e.message = line 64 | end 65 | end 66 | 67 | when :diffs 68 | case line 69 | when '__BEGIN_COMMIT__' 70 | state = :key_values 71 | # Ref: https://git-scm.com/docs/git-diff-index#Documentation/git-diff-index.txt-git-diff-filesltpatterngt82308203 72 | when /:([0-9]+) ([0-9]+) ([a-z0-9]+) ([a-z0-9]+) ([A-Z])\t"?(.+)"?$/ 73 | add_generic_diff(e, Regexp.last_match) 74 | when /:([0-9]+) ([0-9]+) ([a-z0-9]+) ([a-z0-9]+) (R[0-9]+)\t"?(.+)"?$/ 75 | add_rename_edit_diff(e, Regexp.last_match) 76 | end 77 | else 78 | raise RuntimeError("Unknown parser state #{state}") 79 | end 80 | end 81 | 82 | yield e if e 83 | end 84 | # rubocop:enable Metrics/AbcSize, Metrics/MethodLength, Metrics/BlockLength 85 | # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity 86 | 87 | private 88 | 89 | def build_commit(sha1) 90 | commit = OhlohScm::Commit.new 91 | commit.diffs = [] 92 | commit.token = sha1 93 | commit.author_name = ANONYMOUS 94 | commit 95 | end 96 | 97 | def add_generic_diff(commit, match_data) 98 | src_mode, dst_mode, parent_sha1, sha1, action, path = match_data[1..] 99 | 100 | return if path == '.gitmodules' # contains submodule path config. 101 | # Submodules have a file mode of '160000'(gitlink). We ignore submodules completely. 102 | return if src_mode == '160000' || dst_mode == '160000' 103 | 104 | commit.diffs << OhlohScm::Diff.new(action: action, path: path, 105 | sha1: sha1, parent_sha1: parent_sha1) 106 | end 107 | 108 | def add_rename_edit_diff(commit, match_data) 109 | src_mode, dst_mode, parent_sha1, sha1, _, path = match_data[1..] 110 | 111 | return if src_mode == '160000' || dst_mode == '160000' 112 | 113 | old_path, new_path = path.split("\t") 114 | commit.diffs << OhlohScm::Diff.new(action: 'D', path: old_path, 115 | sha1: null_sha1, parent_sha1: parent_sha1) 116 | commit.diffs << OhlohScm::Diff.new(action: 'A', path: new_path, 117 | sha1: sha1, parent_sha1: null_sha1) 118 | end 119 | 120 | def null_sha1 121 | OhlohScm::Git::Activity::NULL_SHA1 122 | end 123 | 124 | def parse_date(date) 125 | Time.rfc2822(date).utc 126 | rescue ArgumentError 127 | Time.at(0) 128 | end 129 | end 130 | end 131 | end 132 | -------------------------------------------------------------------------------- /lib/ohloh_scm/git/scm.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | module Git 5 | class Scm < OhlohScm::Scm 6 | def initialize(core:, url:, branch_name:, username:, password:) 7 | super 8 | @branch_name = branch_name 9 | end 10 | 11 | # == Example: 12 | # remote_core = OhlohScm::Factory.get_core(url: 'https://github.com/ruby/ruby') 13 | # local_core = OhlohScm::Factory.get_core(url: '/tmp/ruby-src') 14 | # local_core.scm.pull(remote_core.scm) 15 | def pull(from, callback) 16 | case from 17 | when Cvs::Scm then convert_to_git(from, callback) 18 | else clone_or_fetch(from, callback) 19 | end 20 | end 21 | 22 | def vcs_path 23 | "#{url}/.git" 24 | end 25 | 26 | def checkout_files(names) 27 | filenames = names.map { |name| "*#{name}" }.join(' ') 28 | run "cd #{url} && git checkout $(git ls-files #{filenames})" 29 | end 30 | 31 | def branch_name_or_default 32 | branch_name || 'master' 33 | end 34 | 35 | private 36 | 37 | def clone_or_fetch(remote_scm, callback) 38 | callback.update(0, 1) 39 | if status.exist? && status.branch?(branch_name) 40 | clean_and_checkout_branch # must be on correct branch, but we want to be careful. 41 | fetch_new_commits(remote_scm) 42 | else 43 | clone_and_create_tracking_branch(remote_scm) 44 | end 45 | clean_up_disk 46 | callback.update(1, 1) 47 | end 48 | 49 | def fetch_new_commits(remote_scm) 50 | run "cd '#{url}' && git fetch --tags --force --update-head-ok " \ 51 | "'#{remote_scm.url}' #{branch_name}:#{branch_name}" 52 | end 53 | 54 | def clone_and_create_tracking_branch(remote_scm) 55 | unless status.scm_dir_exist? || status.exist? 56 | run "rm -rf '#{url}'" 57 | run "git clone -q -n '#{remote_scm.url}' '#{url}'" 58 | end 59 | create_tracking_branch(remote_scm.branch_name) # ensure the correct branch exists locally 60 | clean_and_checkout_branch # switch to the correct branch 61 | end 62 | 63 | # We need very high reliability and this sequence gets the job done every time. 64 | def clean_and_checkout_branch 65 | return unless status.scm_dir_exist? 66 | 67 | run "cd '#{url}' && git clean -f -d -x --exclude='*.nfs*'" 68 | return unless status.branch?(branch_name) 69 | 70 | run "cd '#{url}' && git reset --hard HEAD --" 71 | run "cd '#{url}' && git checkout #{branch_name} --" 72 | run "cd '#{url}' && git reset --hard heads/#{branch_name} --" 73 | end 74 | 75 | def create_tracking_branch(branch_name) 76 | return if branch_name.to_s.empty? 77 | return if activity.branches.include?(branch_name) 78 | 79 | run("cd '#{url}' && git remote update && " \ 80 | "git branch -f #{branch_name} origin/#{branch_name}") 81 | end 82 | 83 | # Deletes everything but the *.git* folder in the working directory. 84 | def clean_up_disk 85 | return unless Dir.exist?(url) 86 | 87 | run "cd #{url} && " \ 88 | "find . -maxdepth 1 -not -name .git -not -name '*.nfs*' -not -name . -print0 " \ 89 | '| xargs -0 rm -rf --' 90 | end 91 | 92 | def convert_to_git(remote_scm, callback) 93 | callback.update(0, 1) 94 | 95 | commits = remote_scm.activity.commits(after: activity.read_token) 96 | check_empty_repository(commits) 97 | 98 | if commits && !commits.empty? 99 | setup_dir_and_convert_commits(commits, callback) 100 | else 101 | logger.info { 'Already up-to-date.' } 102 | end 103 | end 104 | 105 | def setup_dir_and_convert_commits(commits, callback) 106 | set_up_working_directory 107 | convert(commits, callback) 108 | callback.update(commits.size, commits.size) 109 | end 110 | 111 | def convert(commits, callback) 112 | commits.each_with_index do |r, i| 113 | callback.update(i, commits.size) 114 | create_git_commit(r, i, commits.size) 115 | end 116 | end 117 | 118 | def check_empty_repository(commits) 119 | raise 'Empty repository' if !activity.read_token && commits.empty? 120 | end 121 | 122 | def set_up_working_directory 123 | # Start by making sure we are in a known good state. Set up our working directory. 124 | clean_up_disk 125 | clean_and_checkout_branch 126 | end 127 | 128 | def handle_checkout_error(commit) 129 | logger.error { $ERROR_INFO.inspect } 130 | # If we fail to checkout, it's often because there is junk of some kind 131 | # in our working directory. 132 | logger.info { 'Checkout failed. Cleaning and trying again...' } 133 | clean_up_disk 134 | commit.scm.checkout(commit, url) 135 | end 136 | 137 | def create_git_commit(commit, index, size) 138 | logger.info { "Downloading revision #{commit.token} (#{index + 1} of #{size})... " } 139 | checkout(commit) 140 | logger.debug { "Committing revision #{commit.token} (#{index + 1} of #{size})... " } 141 | activity.commit_all(commit) 142 | end 143 | 144 | def checkout(commit) 145 | commit.scm.checkout(commit, url) 146 | rescue StandardError 147 | handle_checkout_error(commit) 148 | end 149 | end 150 | end 151 | end 152 | -------------------------------------------------------------------------------- /lib/ohloh_scm/parser/bzr_xml_parser.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'rexml/document' 4 | require 'rexml/streamlistener' 5 | require 'time' 6 | 7 | module OhlohScm 8 | class BazaarListener 9 | include REXML::StreamListener 10 | attr_accessor :callback, :commit, :diff 11 | attr_writer :text 12 | 13 | def initialize(callback) 14 | @callback = callback 15 | @merge_commit = [] 16 | @state = :none 17 | @authors = [] 18 | end 19 | 20 | # rubocop:disable Metrics/MethodLength 21 | def tag_start(name, attrs) 22 | case name 23 | when 'log' 24 | @commit = OhlohScm::Commit.new 25 | @commit.diffs = [] 26 | when 'affected-files' 27 | @diffs = [] 28 | when 'added', 'modified', 'removed', 'renamed' 29 | @action = name 30 | @state = :collect_files 31 | when 'file' 32 | @before_path = attrs['oldpath'] 33 | when 'merge' 34 | # This is a merge commit, save it and pop it after all branch commits 35 | @merge_commit.push(@commit) 36 | when 'authors' 37 | @state = :collect_authors 38 | @authors = [] 39 | end 40 | end 41 | # rubocop:enable Metrics/MethodLength 42 | 43 | # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity 44 | def tag_end(name) 45 | case name 46 | when 'log' 47 | @callback.call(@commit) 48 | when 'revisionid' 49 | @commit.token = @text 50 | when 'message' 51 | @commit.message = @cdata 52 | when 'committer' 53 | committer = BzrXmlParser.capture_name(@text) 54 | @commit.committer_name = committer[0] 55 | @commit.committer_email = committer[1] 56 | when 'author' 57 | author = BzrXmlParser.capture_name(@text) 58 | @authors << { author_name: author[0], author_email: author[1] } 59 | when 'timestamp' 60 | @commit.committer_date = Time.parse(@text) 61 | when 'file' 62 | @diffs.concat(parse_diff(@action, @text, @before_path)) if @state == :collect_files 63 | @before_path = nil 64 | @text = nil 65 | when 'added', 'modified', 'removed', 'renamed' 66 | @state = :none 67 | when 'affected-files' 68 | @commit.diffs = remove_dupes(@diffs) 69 | when 'merge' 70 | @commit = @merge_commit.pop 71 | when 'authors' 72 | @commit.author_name = @authors[0][:author_name] 73 | @commit.author_email = @authors[0][:author_email] 74 | @authors.clear 75 | end 76 | end 77 | # rubocop:enable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity 78 | 79 | # # Cannot use attr_writer; we need cdata not cdata=. 80 | def cdata(data) 81 | @cdata = data 82 | end 83 | 84 | def text(text) 85 | @text = text 86 | end 87 | 88 | private 89 | 90 | # rubocop:disable Metrics/MethodLength 91 | # Parse one single diff 92 | def parse_diff(action, path, before_path) 93 | diffs = [] 94 | case action 95 | # A rename action requires two diffs: one to remove the old filename, 96 | # another to add the new filename. 97 | # 98 | # Note that is possible to be renamed to the empty string! 99 | # This happens when a subdirectory is moved to become the root. 100 | when 'renamed' 101 | diffs = [OhlohScm::Diff.new(action: 'D', path: before_path), 102 | OhlohScm::Diff.new(action: 'A', path: path || '')] 103 | when 'added' 104 | diffs = [OhlohScm::Diff.new(action: 'A', path: path)] 105 | when 'modified' 106 | diffs = [OhlohScm::Diff.new(action: 'M', path: path)] 107 | when 'removed' 108 | diffs = [OhlohScm::Diff.new(action: 'D', path: path)] 109 | end 110 | diffs.each do |d| 111 | d.path = strip_trailing_asterisk(d.path) 112 | end 113 | diffs 114 | end 115 | # rubocop:enable Metrics/MethodLength 116 | 117 | def strip_trailing_asterisk(path) 118 | path[-1..] == '*' ? path[0..-2] : path 119 | end 120 | 121 | def remove_dupes(diffs) 122 | BzrXmlParser.remove_dupes(diffs) 123 | end 124 | end 125 | 126 | class BzrXmlParser < Parser 127 | NAME_REGEX = /^(.+?)(\s+<(.+)>\s*)?$/ 128 | def self.internal_parse(buffer, _) 129 | buffer = '' if buffer.is_a?(StringIO) && buffer.length < 2 130 | REXML::Document.parse_stream(buffer, 131 | BazaarListener.new(proc { |c| yield c if block_given? })) 132 | rescue EOFError => e 133 | puts e.message 134 | end 135 | 136 | def self.scm 137 | 'bzr' 138 | end 139 | 140 | def self.remove_dupes(diffs) 141 | remove_modified_after_added!(diffs) 142 | # Bazaar may report that a file was both deleted and added in a single commit. 143 | # Reduce these cases to a single 'M' action. 144 | diffs.each do |d| 145 | d.action = 'M' if diffs.count { |x| x.path == d.path } > 1 146 | end.uniq 147 | end 148 | 149 | # Bazaar may report that a file was added and modified in a single commit. 150 | # Reduce these cases to a single 'A' action. 151 | def self.remove_modified_after_added!(diffs) 152 | diffs.delete_if do |d| 153 | d.action == 'M' && diffs.any? { |x| x.path == d.path && x.action == 'A' } 154 | end 155 | end 156 | 157 | # Bazaar expects committer/author to be specified in this format 158 | # Name , or John Doe 159 | # However, we find many variations in the real world including 160 | # ones where only email is specified as name. 161 | def self.capture_name(text) 162 | parts = text.match(NAME_REGEX).to_a 163 | name = parts[1] || parts[0] 164 | email = parts[3] 165 | [name, email] 166 | end 167 | end 168 | end 169 | -------------------------------------------------------------------------------- /spec/ohloh_scm/git/scm_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe 'Git::Scm' do 4 | it 'must pull git repository' do 5 | with_git_repository('git') do |src_core| 6 | tmpdir do |dest_dir| 7 | core = OhlohScm::Factory.get_core(scm_type: :git, url: dest_dir) 8 | refute core.status.scm_dir_exist? 9 | 10 | core.scm.pull(src_core.scm, TestCallback.new) 11 | assert core.status.scm_dir_exist? 12 | assert core.status.exist? 13 | end 14 | end 15 | end 16 | 17 | it 'must pull git repository with multiple branches' do 18 | # This should not change current/default branch(e.g. master) to point to the branch commit being pulled 19 | # In this case master should not point to test branch commit 20 | with_git_repository('git_with_multiple_branch', 'test') do |src_core| 21 | tmpdir do |dest_dir| 22 | core = OhlohScm::Factory.get_core(scm_type: :git, url: dest_dir, branch_name: 'test') 23 | refute core.status.scm_dir_exist? 24 | core.scm.pull(src_core.scm, TestCallback.new) 25 | 26 | remote_master_branch_sha = `cd #{dest_dir} && git rev-parse origin/master` 27 | master_branch_sha = `cd #{dest_dir} && git rev-parse master` 28 | test_branch_sha = `cd #{dest_dir} && git rev-parse test` 29 | 30 | refute_equal master_branch_sha, test_branch_sha 31 | assert_equal master_branch_sha, remote_master_branch_sha 32 | end 33 | end 34 | end 35 | 36 | it 'must checkout a branch which is behind the default' do 37 | with_git_repository('git_with_multiple_branch') do |src_core| 38 | tmpdir do |dest_dir| 39 | core = OhlohScm::Factory.get_core(scm_type: :git, url: dest_dir, branch_name: 'master') 40 | core.scm.pull(src_core.scm, TestCallback.new) 41 | 42 | src_core.scm.branch_name = 'test' 43 | core = OhlohScm::Factory.get_core(scm_type: :git, url: dest_dir, branch_name: 'test') 44 | core.scm.pull(src_core.scm, TestCallback.new) 45 | 46 | remote_master_branch_sha = `cd #{dest_dir} && git rev-parse origin/master` 47 | remote_test_branch_sha = `cd #{dest_dir} && git rev-parse origin/test` 48 | master_branch_sha = `cd #{dest_dir} && git rev-parse master` 49 | test_branch_sha = `cd #{dest_dir} && git rev-parse test` 50 | 51 | assert_equal master_branch_sha, remote_master_branch_sha 52 | assert_equal test_branch_sha, remote_test_branch_sha 53 | end 54 | end 55 | end 56 | 57 | it 'must handle file changes in multi branch directory' do 58 | with_git_repository('git_with_multiple_branch', 'test') do |src_core| 59 | tmpdir do |dest_dir| 60 | core = OhlohScm::Factory.get_core(scm_type: :git, url: dest_dir, branch_name: 'test') 61 | refute core.status.scm_dir_exist? 62 | core.scm.pull(src_core.scm, TestCallback.new) 63 | 64 | `cd #{dest_dir} && git checkout master` 65 | `echo 'new change' >> #{dest_dir}/hello.rb` 66 | 67 | core.scm.pull(src_core.scm, TestCallback.new) 68 | end 69 | end 70 | end 71 | 72 | it 'must update branches in local copy' do 73 | test_branch_name = 'test' # consider that 'test' is the current *main* branch. 74 | 75 | with_git_repository('git_with_multiple_branch', test_branch_name) do |src_core| 76 | tmpdir do |dest_dir| 77 | core = OhlohScm::Factory.get_core(scm_type: :git, url: dest_dir, branch_name: test_branch_name) 78 | core.scm.pull(src_core.scm, TestCallback.new) 79 | 80 | # Emulate a scenario where the local copy doesn't have the current *main* branch. 81 | `cd #{dest_dir} && git checkout master && git branch -D test` 82 | 83 | local_branch_cmd = "cd #{dest_dir} && git branch | grep '*' | sed 's/^* //'" 84 | assert_equal `#{local_branch_cmd}`.chomp, 'master' 85 | 86 | # On doing a refetch, our local copy will now have the updated *main* branch. 87 | core.scm.pull(src_core.scm, TestCallback.new) 88 | assert_equal `#{local_branch_cmd}`.chomp, test_branch_name 89 | end 90 | end 91 | end 92 | 93 | it 'must test the basic conversion to git' do 94 | with_cvs_repository('cvs', 'simple') do |src_core| 95 | tmpdir do |dest_dir| 96 | core = OhlohScm::Factory.get_core(scm_type: :git, branch_name: 'master', url: dest_dir) 97 | refute core.status.scm_dir_exist? 98 | core.scm.pull(src_core.scm, TestCallback.new) 99 | assert core.status.scm_dir_exist? 100 | assert core.status.exist? 101 | 102 | dest_commits = core.activity.commits 103 | src_core.activity.commits.each_with_index do |c, i| 104 | # Because CVS does not track authors (only committers), 105 | # the CVS committer becomes the Git author. 106 | assert_equal c.committer_date, dest_commits[i].author_date 107 | assert_equal c.committer_name, dest_commits[i].author_name 108 | 109 | # Depending upon version of Git used, we may or may not have a trailing \n. 110 | # We don't really care, so just compare the stripped versions. 111 | assert_equal c.message.strip, dest_commits[i].message.strip 112 | end 113 | end 114 | end 115 | end 116 | 117 | it 'must checkout_files matching given names' do 118 | with_git_repository('git') do |src_core| 119 | dir = src_core.scm.url 120 | core = OhlohScm::Factory.get_core(scm_type: :git, url: dir) 121 | 122 | core.scm.checkout_files(['Gemfile.lock', 'package.json', 'Godeps.json', 'doesnt-exist']) 123 | 124 | assert system("ls #{dir}/Gemfile.lock > /dev/null") 125 | assert system("ls #{dir}/nested/nested_again/package.json > /dev/null") 126 | assert system("ls #{dir}/Godeps/Godeps.json > /dev/null") 127 | end 128 | end 129 | 130 | it 'must return master when branch_name is null' do 131 | core = OhlohScm::Factory.get_core(scm_type: :git, url: 'foobar') 132 | assert_equal core.scm.branch_name_or_default, 'master' 133 | end 134 | end 135 | -------------------------------------------------------------------------------- /spec/ohloh_scm/svn_parser_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | DATA_DIR = File.expand_path(File.join(File.dirname(__FILE__), '../raw_fixtures')) 4 | 5 | describe 'SvnParser' do 6 | it 'test_empty_array' do 7 | assert_predicate OhlohScm::SvnParser.parse(''), :empty? 8 | end 9 | 10 | it 'test_yield_instead_of_writer' do 11 | commits = [] 12 | result = OhlohScm::SvnParser.parse(File.read("#{DATA_DIR}/simple.svn_log")) do |commit| 13 | commits << commit.token 14 | end 15 | assert_nil result 16 | assert_equal commits, [5, 4, 3, 2, 1] 17 | end 18 | 19 | it 'test_log_parser' do 20 | sample_log = <<~SAMPLE 21 | ------------------------------------------------------------------------ 22 | r1 | robin | 2006-06-11 11:28:00 -0700 (Sun, 11 Jun 2006) | 2 lines 23 | 24 | Initial Checkin 25 | 26 | ------------------------------------------------------------------------ 27 | r2 | jason | 2006-06-11 11:32:13 -0700 (Sun, 11 Jun 2006) | 1 line 28 | 29 | added makefile 30 | ------------------------------------------------------------------------ 31 | r3 | robin | 2006-06-11 11:34:17 -0700 (Sun, 11 Jun 2006) | 1 line 32 | 33 | added some documentation and licensing info 34 | ------------------------------------------------------------------------ 35 | SAMPLE 36 | 37 | revs = OhlohScm::SvnParser.parse(sample_log) 38 | 39 | assert revs 40 | assert_equal revs.size, 3 41 | 42 | assert_equal revs[0].token, 1 43 | assert_equal revs[0].committer_name, 'robin' 44 | assert_equal revs[0].message, "Initial Checkin\n" # Note \n at end of comment 45 | assert_equal revs[0].committer_date, Time.utc(2006, 6, 11, 18, 28, 0o0) 46 | 47 | assert_equal revs[1].token, 2 48 | assert_equal revs[1].committer_name, 'jason' 49 | assert_equal revs[1].message, 'added makefile' # Note no \n at end of comment 50 | assert_equal revs[1].committer_date, Time.utc(2006, 6, 11, 18, 32, 13) 51 | 52 | assert_equal revs[2].token, 3 53 | assert_equal revs[2].committer_name, 'robin' 54 | assert_equal revs[2].message, 'added some documentation and licensing info' 55 | assert_equal revs[2].committer_date, Time.utc(2006, 6, 11, 18, 34, 17) 56 | end 57 | 58 | # This is an excerpt from the log for Wireshark. It includes Subversion log excerpts in 59 | # its comments, which really screwed us up. This test confirms that I've fixed the 60 | # parser to ignore log excerpts in the comments. 61 | it 'test_log_embedded_in_comments' do 62 | log = <<~LOG 63 | ------------------------------------------------------------------------ 64 | r21932 | jmayer | 2007-05-25 01:34:15 -0700 (Fri, 25 May 2007) | 22 lines 65 | 66 | Update from samba tree revision 23054 to 23135 67 | ============================ Samba log start ============ 68 | ------------------------------------------------------------------------ 69 | r23069 | metze | 2007-05-22 13:23:36 +0200 (Tue, 22 May 2007) | 3 lines 70 | Changed paths: 71 | M /branches/SAMBA_4_0/source/pidl/tests/Util.pm 72 | 73 | print out the command, to find out the problem on host 'tridge' 74 | 75 | metze 76 | ------------------------------------------------------------------------ 77 | r23071 | metze | 2007-05-22 14:45:58 +0200 (Tue, 22 May 2007) | 3 lines 78 | Changed paths: 79 | M /branches/SAMBA_4_0/source/pidl/tests/Util.pm 80 | 81 | print the command on failure only 82 | 83 | metze 84 | ------------------------------------------------------------------------ 85 | ------------------------------------------------------------------------ 86 | ============================ Samba log end ============== 87 | 88 | ------------------------------------------------------------------------ 89 | r21931 | kukosa | 2007-05-24 23:54:39 -0700 (Thu, 24 May 2007) | 2 lines 90 | 91 | UMTS RRC updated to 3GPP TS 25.331 V7.4.0 (2007-03) and moved to one directory 92 | 93 | ------------------------------------------------------------------------ 94 | LOG 95 | revs = OhlohScm::SvnParser.parse(log) 96 | 97 | assert revs 98 | assert_equal revs.size, 2 99 | 100 | assert_equal revs[0].token, 21_932 101 | assert_equal revs[1].token, 21_931 102 | 103 | comment = <<~COMMENT 104 | Update from samba tree revision 23054 to 23135 105 | ============================ Samba log start ============ 106 | ------------------------------------------------------------------------ 107 | r23069 | metze | 2007-05-22 13:23:36 +0200 (Tue, 22 May 2007) | 3 lines 108 | Changed paths: 109 | M /branches/SAMBA_4_0/source/pidl/tests/Util.pm 110 | 111 | print out the command, to find out the problem on host 'tridge' 112 | 113 | metze 114 | ------------------------------------------------------------------------ 115 | r23071 | metze | 2007-05-22 14:45:58 +0200 (Tue, 22 May 2007) | 3 lines 116 | Changed paths: 117 | M /branches/SAMBA_4_0/source/pidl/tests/Util.pm 118 | 119 | print the command on failure only 120 | 121 | metze 122 | ------------------------------------------------------------------------ 123 | ------------------------------------------------------------------------ 124 | ============================ Samba log end ============== 125 | COMMENT 126 | assert_equal revs[0].message, comment 127 | end 128 | 129 | it 'test_svn_copy' do 130 | log = <<~LOG 131 | ------------------------------------------------------------------------ 132 | r8 | robin | 2009-02-05 05:40:46 -0800 (Thu, 05 Feb 2009) | 1 line 133 | Changed paths: 134 | A /trunk (from /branches/development:7) 135 | 136 | the branch becomes the new trunk 137 | LOG 138 | 139 | commits = OhlohScm::SvnParser.parse(log) 140 | assert_equal commits.size, 1 141 | assert_equal commits.first.diffs.size, 1 142 | assert_equal commits.first.diffs.first.path, '/trunk' 143 | assert_equal commits.first.diffs.first.from_path, '/branches/development' 144 | assert_equal commits.first.diffs.first.from_revision, 7 145 | end 146 | end 147 | -------------------------------------------------------------------------------- /lib/ohloh_scm/bzr/activity.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | module Bzr 5 | class Activity < OhlohScm::Activity 6 | # rubocop:disable Metrics/MethodLength 7 | def tags 8 | tag_strings.map do |tag_string| 9 | parse_tag_names_and_revision = tag_string.split(/\s+/) 10 | if parse_tag_names_and_revision.size > 1 11 | tag_name = parse_tag_names_and_revision[0..-2].join(' ') 12 | rev = parse_tag_names_and_revision.last 13 | else 14 | tag_name = parse_tag_names_and_revision.first 15 | rev = nil 16 | end 17 | next if rev == '?' || tag_name == '....' 18 | 19 | [tag_name, rev, Time.parse(time_string(rev))] 20 | end.compact 21 | end 22 | # rubocop:enable Metrics/MethodLength 23 | 24 | def export_tag(dest_dir, tag_name) 25 | run "cd '#{url}' && bzr export -r #{tag_name} #{dest_dir}" 26 | end 27 | 28 | def export(dest_dir, token = head_token) 29 | # Unlike other SCMs, Bzr doesn't simply place the contents into dest_dir. 30 | # It actually *creates* dest_dir. Since it should already exist at this point, 31 | # first we have to delete it. 32 | FileUtils.rm_f(dest_dir) 33 | 34 | run "cd '#{url}' && bzr export --format=dir -r #{to_rev_param(token)} '#{dest_dir}'" 35 | end 36 | 37 | # Returns a list of shallow commits (i.e., the diffs are not populated). 38 | # Not including the diffs is meant to be a memory savings when 39 | # we encounter massive repositories. If you need all commits 40 | # including diffs, you should use the each_commit() iterator, 41 | # which only holds one commit in memory at a time. 42 | def commits(opts = {}) 43 | after = opts[:after] 44 | log = run("#{rev_list_command(opts)} | cat") 45 | a = OhlohScm::BzrXmlParser.parse(log) 46 | 47 | if after && (i = a.index { |commit| commit.token == after }) 48 | a[(i + 1)..] 49 | else 50 | a 51 | end 52 | end 53 | 54 | # Return the list of commit tokens following +after+. 55 | def commit_tokens(opts = {}) 56 | commits(opts).map(&:token) 57 | end 58 | 59 | # Return the number of commits in the repository following +after+. 60 | def commit_count(opts = {}) 61 | commit_tokens(opts).size 62 | end 63 | 64 | def each_commit(opts = {}) 65 | after = opts[:after] 66 | skip_commits = !after.nil? # Don't emit any commits until the 'after' resume point passes 67 | 68 | safe_open_log_file(opts) do |io| 69 | OhlohScm::BzrXmlParser.parse(io) do |commit| 70 | yield remove_directories(commit) if block_given? && !skip_commits 71 | skip_commits = false if commit.token == after 72 | end 73 | end 74 | end 75 | 76 | def head_token 77 | run("bzr log --limit 1 --show-id #{url} 2> /dev/null " \ 78 | "| grep ^revision-id | cut -f2 -d' '").strip 79 | end 80 | 81 | def head 82 | verbose_commit(head_token) 83 | end 84 | 85 | def parent_tokens(commit) 86 | bzr_client.parent_tokens(commit.token) 87 | end 88 | 89 | def parents(commit) 90 | parent_tokens(commit).collect { |token| verbose_commit(token) } 91 | end 92 | 93 | def cat_file(commit, diff) 94 | cat(commit.token, diff.path) 95 | end 96 | 97 | def cat_file_parent(commit, diff) 98 | first_parent_token = parent_tokens(commit).first 99 | cat(first_parent_token, diff.path) if first_parent_token 100 | end 101 | 102 | def cleanup 103 | bzr_client.shutdown 104 | end 105 | 106 | private 107 | 108 | # Returns a single commit, including its diffs 109 | def verbose_commit(token) 110 | cmd = "cd '#{url}' && bzr xmllog --show-id -v --limit 1 -c #{to_rev_param(token)}" 111 | log = run(cmd) 112 | OhlohScm::BzrXmlParser.parse(log).first 113 | end 114 | 115 | def to_rev_param(rev = nil) 116 | case rev 117 | when nil 118 | 1 119 | when Integer 120 | rev.to_s 121 | when /^\d+$/ 122 | rev 123 | else 124 | "'revid:#{rev}'" 125 | end 126 | end 127 | 128 | def rev_list_command(opts = {}) 129 | after = opts[:after] 130 | trunk_only = opts[:trunk_only] ? '--levels=1' : '--include-merges' 131 | "cd '#{url}' && bzr xmllog --show-id --forward #{trunk_only} -r #{to_rev_param(after)}.." 132 | end 133 | 134 | # Returns a file handle to the log. 135 | # In our standard, the log should include everything AFTER 136 | # +after+. However, bzr doesn't work that way; it returns 137 | # everything after and INCLUDING +after+. Therefore, consumers 138 | # of this file should check for and reject the duplicate commit. 139 | def safe_open_log_file(opts = {}, &block) 140 | return '' if opts[:after] && opts[:after] == head_token 141 | 142 | open_log_file(opts, &block) 143 | end 144 | 145 | def open_log_file(opts = {}, &block) 146 | cmd = "#{rev_list_command(opts)} -v > #{log_filename}" 147 | run cmd 148 | File.open(log_filename, 'r', &block) 149 | ensure 150 | FileUtils.rm_f(log_filename) 151 | end 152 | 153 | # Ohloh tracks only files, not directories. This function removes directories 154 | # from the commit diffs. 155 | def remove_directories(commit) 156 | commit.diffs.delete_if { |d| d.path[-1..] == '/' } 157 | commit 158 | end 159 | 160 | def cat(revision, path) 161 | bzr_client.cat_file(revision, path) 162 | end 163 | 164 | # Bzr doesn't like it when the filename includes a colon 165 | # Also, fix the case where the filename includes a single quote 166 | def escape(path) 167 | path.gsub(':') { |c| "\\#{c}" }.gsub("'", "''") 168 | end 169 | 170 | def tag_strings 171 | run("cd '#{url}' && bzr tags").split("\n") 172 | end 173 | 174 | def time_string(rev) 175 | run("cd '#{url}' && bzr log -r #{rev} | grep 'timestamp:' | sed 's/timestamp://'") 176 | end 177 | 178 | def bzr_client 179 | @bzr_client ||= setup_bzr_client 180 | end 181 | 182 | def setup_bzr_client 183 | bzr_client = PyBridge::BzrClient.new(url) 184 | bzr_client.start 185 | bzr_client 186 | end 187 | end 188 | end 189 | end 190 | -------------------------------------------------------------------------------- /lib/ohloh_scm/parser/cvs_parser.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | class CvsParser < Parser 5 | class << self 6 | # Given an IO to a CVS rlog, returns a list of 7 | # commits (developer/date/message). 8 | # rubocop:disable Metrics/AbcSize, Metrics/MethodLength 9 | # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity 10 | def internal_parse(io, _opts, &block) 11 | commits = {} 12 | 13 | read_files(io) do |c| 14 | # As commits are yielded by the parser, we sort them into bins. 15 | # 16 | # The 'bins' are arrays of timestamps. We keep a separate array of 17 | # timestamps for each developer/message combination. 18 | # 19 | # If a commit lies near in time to another commit with the same 20 | # developer/message combination, then we merge them and store only 21 | # the later of the two timestamps. 22 | # 23 | # Typically, we end up with only a single timestamp for each developer/message 24 | # combination. However, if a developer repeatedly uses the same message 25 | # a number of separate times, we may end up with several timestamps for 26 | # that combination. 27 | 28 | key = "#{c.committer_name}:#{c.message}" 29 | if commits.key? key 30 | # We have already seen this developer/message combination 31 | match = false 32 | commits[key].each_index do |i| 33 | # Does the new commit lie near in time to a known one in our list? 34 | next unless near?(commits[key][i].committer_date, c.committer_date) 35 | 36 | match = true 37 | # Yes. Choose the most recent timestamp, and add the new 38 | # directory name to our list. 39 | if commits[key][i].committer_date < c.committer_date 40 | commits[key][i].committer_date = c.committer_date 41 | commits[key][i].token = c.token 42 | end 43 | unless commits[key][i].directories.include? c.directories[0] 44 | commits[key][i].directories << c.directories[0] 45 | end 46 | break 47 | end 48 | # This commit lies a long time away from any one we know. 49 | # Add it to the list as a new checkin event. 50 | commits[key] << c unless match 51 | else 52 | # We have never seen this developer/message combination. Start a new list. 53 | commits[key] = [c] 54 | end 55 | end 56 | # Pull all of the commits out of the hash and return them as a single sorted list. 57 | result = commits.values.flatten.sort! { |a, b| a.committer_date <=> b.committer_date } 58 | 59 | # If we have two commits with identical timestamps, arbitrarily choose the first 60 | (result.size - 1).downto(1) do |i| 61 | result.delete_at(i) if result[i].committer_date == result[i - 1].committer_date 62 | end 63 | 64 | result.each(&block) if block 65 | end 66 | # rubocop:enable Metrics/AbcSize, Metrics/MethodLength 67 | # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity 68 | 69 | private 70 | 71 | # Accepts two dates and 72 | # determines wether they are close enough together to consider simultaneous. 73 | def near?(date1, date2) 74 | ((date1 - date2).abs < 30 * 60) # Less than 30 minutes counts as 'near' 75 | end 76 | 77 | def read_files(io, &block) 78 | io.each_line do |l| 79 | if l =~ /^RCS file: (.*),.$/ 80 | filename = Regexp.last_match(1) 81 | read_file(io, filename, &block) 82 | end 83 | end 84 | end 85 | 86 | def read_file(io, filename, &block) 87 | branch_number = nil 88 | io.each_line do |l| 89 | case l 90 | when /^head: ([\d.]+)/ 91 | branch_number = BranchNumber.new(Regexp.last_match(1)) 92 | when /^----------------------------/ 93 | read_commits(io, branch_number, filename, &block) 94 | end 95 | end 96 | end 97 | 98 | def read_commits(io, branch_number, filename, &block) 99 | should_yield = nil 100 | io.each_line do |l| 101 | break if /^\s$/.match?(l) 102 | 103 | l =~ /^revision ([\d.]+)/ 104 | commit_number = Regexp.last_match(1) 105 | should_yield = branch_number&.on_same_line?(BranchNumber.new(commit_number)) 106 | read_commit(io, filename, commit_number, should_yield, &block) 107 | end 108 | end 109 | 110 | def read_commit(io, filename, commit_number, should_yield) 111 | io.each_line do |l| 112 | next unless l =~ /^date: (.*); author: ([^;]+); state: (\w+);/ 113 | 114 | state = Regexp.last_match(3) 115 | # CVS creates a "phantom" dead file at 1.1 on the head if a file 116 | # is created on a branch. Ignore this file. 117 | should_yield = false if (commit_number == '1.1') && (state == 'dead') 118 | message = read_message(io) 119 | if should_yield 120 | yield build_commit(Regexp.last_match(1), Regexp.last_match(2), message, filename) 121 | end 122 | break 123 | end 124 | end 125 | 126 | def build_commit(committer_date, committer_name, message, filename) 127 | commit = OhlohScm::Commit.new 128 | commit.token = committer_date[0..18] 129 | commit.committer_date = Time.parse("#{committer_date[0..18]} +0000").utc 130 | commit.committer_name = committer_name 131 | commit.message = message 132 | commit.directories = [File.dirname(filename).intern] 133 | commit 134 | end 135 | 136 | # rubocop:disable Metrics/MethodLength 137 | def read_message(io) 138 | message = '' 139 | first_line = true 140 | io.each_line do |l| 141 | unless l =~ /^branches: / && first_line # the first line might be 'branches:', skip it. 142 | l.chomp! 143 | return message if separator?(l) 144 | 145 | message += "\n" unless message.empty? 146 | message += l 147 | end 148 | first_line = false 149 | end 150 | message 151 | end 152 | # rubocop:enable Metrics/MethodLength 153 | 154 | def separator?(line) 155 | %w[============================================================================= 156 | ----------------------------].include?(line) 157 | end 158 | end 159 | end 160 | end 161 | -------------------------------------------------------------------------------- /lib/ohloh_scm/cvs/scm.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | module Cvs 5 | class Scm < OhlohScm::Scm 6 | # rubocop:disable Metrics/AbcSize, Metrics/MethodLength 7 | # rubocop:disable Metrics/PerceivedComplexity 8 | def checkout(rev, local_directory) 9 | opt_d = rev.token ? "-D'#{rev.token}Z'" : '' 10 | 11 | activity.ensure_host_key 12 | if File.exist?("#{local_directory}/CVS/Root") 13 | # We already have a local enlistment, so do a quick update. 14 | if rev.directories.empty? 15 | # Brute force: get all updates 16 | logger.warn("Revision #{rev.token} did not contain any directories. 17 | Using brute force update of entire module.") 18 | run "cd #{local_directory} && cvsnt update -d -R -C #{opt_d}" 19 | else 20 | build_ordered_directory_list(rev.directories).each do |d| 21 | if d.empty? 22 | run "cd #{local_directory} && cvsnt update -d -l -C #{opt_d} ." 23 | else 24 | run "cd #{local_directory} && cvsnt update -d -l -C #{opt_d} '#{d}'" 25 | end 26 | end 27 | end 28 | else 29 | # We do not have a local enlistment, so do a slow checkout to create one. 30 | # Silly cvsnt won't accept an absolute path. 31 | # We'll have to play some games and cd to the parent directory. 32 | parent_path, checkout_dir = File.split(local_directory) 33 | FileUtils.mkdir_p(parent_path) 34 | run "cd #{parent_path} && 35 | cvsnt -d #{url} checkout #{opt_d} -A -d'#{checkout_dir}' '#{branch_name}'" 36 | end 37 | end 38 | # rubocop:enable Metrics/AbcSize, Metrics/MethodLength 39 | # rubocop:enable Metrics/PerceivedComplexity 40 | 41 | def normalize 42 | # Some CVS forges publish an URL which is actually a symlink, which causes CVSNT to crash. 43 | # For some forges, we can work around this by using an alternate directory. 44 | case guess_forge 45 | when 'java.net', 'netbeans.org' 46 | url.gsub!(/:\/cvs\/?$/, ':/shared/data/ccvs/repository') 47 | when 'gna.org' 48 | url.gsub!(/:\/cvs\b/, ':/var/cvs') 49 | end 50 | 51 | sync_pserver_username_password 52 | 53 | self 54 | end 55 | 56 | private 57 | 58 | # A revision can contain an arbitrary collection of directories. 59 | # We need to ensure that for every directory we want to fetch, 60 | # we also have its parent directories. 61 | # rubocop:disable Metrics/AbcSize, Metrics/MethodLength 62 | # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity 63 | def build_ordered_directory_list(directories) 64 | # Integration Test Limitation 65 | # cvsnt has problems with absolute path names, so we are stuck with 66 | # using cvs modules that are only a single directory deep when testing. 67 | # We'll check if the url begins with '/' to detect an integration test, 68 | # then return an empty string (ie, the default root directory) if so. 69 | return [''] if /^\//.match?(url) 70 | 71 | list = [] 72 | directories = directories.collect { |a| trim_directory(a.to_s).to_s } 73 | directories.each do |d| 74 | # We always ignore Attic directories, which just contain deleted files 75 | # Update the parent directory of the Attic instead. 76 | if d =~ /^(.*)Attic$/ 77 | d = Regexp.last_match(1) 78 | d = d[0..-2] if !d.empty? && (d[-1, 1] == '/') 79 | end 80 | 81 | next if list.include? d 82 | 83 | list << d 84 | # We also need to include every parent directory of the directory 85 | # we are interested in, all the way up to the root. 86 | while d.rindex('/')&.positive? 87 | d = File.dirname(d) 88 | break if list.include? d 89 | 90 | list << d 91 | end 92 | end 93 | 94 | # Sort the list by length because we need to update parent directories before children 95 | list.sort! { |a, b| a.length <=> b.length } 96 | end 97 | # rubocop:enable Metrics/AbcSize, Metrics/MethodLength 98 | # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity 99 | 100 | def trim_directory(dir) 101 | # If we are connecting to a remote server (basically anytime we are not 102 | # running the integration test) then we need to create a relative path 103 | # by trimming the prefix from the directory. 104 | # The prefix can be determined by examining the url and the module name. 105 | # For example, if url = ':pserver:anonymous:@moodle.cvs.sourceforge.net:/cvsroot/moodle' 106 | # and module = 'contrib', then the directory prefix = '/cvsroot/moodle/contrib/' 107 | # If not remote, just leave the directory name as-is 108 | root ? dir[root.length..] : dir 109 | end 110 | 111 | def root 112 | return unless url =~ /^:(pserver|ext):.*@[^:]+:(\d+)?(\/.*)$/ 113 | 114 | "#{Regexp.last_match(3)}/#{branch_name}/" 115 | end 116 | 117 | # This bit of code patches up any inconsistencies that may arise because there 118 | # is both a @password attribute and a password embedded in the :pserver: url. 119 | # This method guarantees that they are both the same. 120 | # 121 | # It's assumed that if the user specified a @password attribute, then that is 122 | # the preferred value and it should take precedence over any password found 123 | # in the :pserver: url. 124 | # 125 | # If the user did not specify a @password attribute, then the value 126 | # found in the :pserver: url is assigned to both. 127 | def sync_pserver_username_password 128 | # Do nothing unless pserver connection string is well-formed. 129 | return unless url =~ /:pserver:([\w\-_]*)(:([\w\-_]*))?@(.*)$/ 130 | 131 | pserver_username = Regexp.last_match(1) 132 | pserver_password = Regexp.last_match(3) 133 | pserver_remainder = Regexp.last_match(4) 134 | 135 | @username = pserver_username if @username.to_s.empty? 136 | @password = pserver_password if @password.to_s.empty? 137 | 138 | @url = ":pserver:#{@username}:#{password}@#{pserver_remainder}" 139 | end 140 | 141 | # Based on the URL, take a guess about which forge this code is hosted on. 142 | def guess_forge 143 | return unless url =~ /.*(pserver|ext).*@(([^.]+\.)?(cvs|dev)\.)?([^:]+):\//i 144 | 145 | Regexp.last_match(5).downcase 146 | end 147 | end 148 | end 149 | end 150 | -------------------------------------------------------------------------------- /lib/ohloh_scm/cvs/activity.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module OhlohScm 4 | module Cvs 5 | class Activity < OhlohScm::Activity 6 | def tags 7 | cmd = "cvs -Q -d #{url} rlog -h #{scm.branch_name} | awk -F\"[.:]\" '/^\\t/&&$(NF-1)!=0'" 8 | run(cmd).split("\n").map do |tag_string| 9 | tag_name, version = tag_string.split(':') 10 | [tag_name.delete("\t"), version.strip] 11 | end 12 | end 13 | 14 | def commits(opts = {}) 15 | result = fetch_commits(opts) 16 | # Nothing found; we're done here. || We requested everything, so just return everything. 17 | return result if result.empty? || opts[:after].to_s.empty? 18 | 19 | filter_commits(result, opts[:after]) 20 | end 21 | 22 | def export_tag(dest_dir, tag_name = 'HEAD') 23 | run "cvsnt -d #{url} export -d'#{dest_dir}' -r #{tag_name} '#{scm.branch_name}'" 24 | end 25 | 26 | # using :ext (ssh) protocol might trigger ssh to confirm accepting the host's 27 | # ssh key. This causes the UI to hang asking for manual confirmation. To avoid 28 | # this we pre-populate the ~/.ssh/known_hosts file with the host's key. 29 | def ensure_host_key 30 | return if protocol != :ext 31 | 32 | ensure_key_file = "#{File.dirname(__FILE__)}/../../../../bin/ensure_key" 33 | cmd = "#{ensure_key_file} '#{host}'" 34 | run_with_err(cmd) 35 | end 36 | 37 | private 38 | 39 | def fetch_commits(opts) 40 | OhlohScm::CvsParser.parse(open_log_file(opts)).tap do |result| 41 | # Git converter needs a backpointer to the scm for each commit 42 | result.each { |c| c.scm = scm } 43 | end 44 | end 45 | 46 | def filter_commits(commits, after) 47 | return commits if first_commit_newer?(commits, after) 48 | 49 | # Walk the list of commits to find the first new one, throwing away all of the old ones. 50 | 51 | # I want to string-compare timestamps without converting to dates objects. 52 | # Some CVS servers print dates as 2006/01/02 03:04:05, others as 2006-01-02 03:04:05. 53 | # To work around this, we'll build a regex that matches either date format. 54 | timestamp_regex = create_timestamp_regex(after) 55 | match_index = find_match_index(commits, timestamp_regex) 56 | 57 | extract_new_commits(commits, match_index) 58 | end 59 | 60 | def first_commit_newer?(commits, after) 61 | # We must now remove any duplicates caused by timestamp fudge factors, 62 | # and only return commits with timestamp > after. 63 | 64 | # If the first commit is newer than after, 65 | # then the whole list is new and we can simply return. 66 | parse_time(commits.first.token) > parse_time(after) 67 | end 68 | 69 | def create_timestamp_regex(timestamp) 70 | Regexp.new(timestamp.gsub(/[\/-]/, '.')) 71 | end 72 | 73 | def find_match_index(commits, timestamp_regex) 74 | commits.index { |commit| commit.token&.match?(timestamp_regex) } 75 | end 76 | 77 | def extract_new_commits(commits, match_index) 78 | case match_index 79 | when nil 80 | # Something bad is going on: 'after' does not match any timestamp in the rlog. 81 | # This is very rare, but it can happen. 82 | # 83 | # Often this means that the *last* time we ran commits(), there was some kind of 84 | # undetected problem (CVS was in an intermediate state?) so the list of timestamps we 85 | # calculated last time does not match the list of timestamps we calculated this time. 86 | # 87 | # There's no work around for this condition here in the code, but there are some things 88 | # you can try manually to fix the problem. Typically, you can try throwing way the 89 | # commit associated with 'after' and fetching it again (git reset --hard HEAD^). 90 | raise 'token not found in rlog.' 91 | when commits.size - 1 92 | [] 93 | else 94 | commits[match_index + 1..] 95 | end 96 | end 97 | 98 | # Gets the rlog of the repository and saves it in a temporary file. 99 | # If you pass a timestamp token, then only commits after the timestamp will be returned. 100 | # 101 | # Warning! 102 | # 103 | # CVS servers are apparently unreliable when you truncate 104 | # the log by timestamp -- perhaps round-off error? 105 | # In any case, to be sure not to miss any commits, 106 | # this method subtracts 10 seconds from the provided timestamp. 107 | # This means that the returned log might actually contain a few revisions 108 | # that predate the requested time. 109 | # That's better than missing revisions completely! Just be sure to check for duplicates. 110 | def open_log_file(opts = {}, &block) 111 | ensure_host_key 112 | status.lock? 113 | run "cvsnt -d #{url} rlog #{opt_branch} #{opt_time(opts[:after])} '#{scm.branch_name}' " \ 114 | "| #{string_encoder_path} > #{rlog_filename}" 115 | File.open(rlog_filename, 'r', &block) 116 | ensure 117 | FileUtils.rm_f(rlog_filename) 118 | end 119 | 120 | def opt_time(after = nil) 121 | return '' unless after 122 | 123 | most_recent_time = parse_time(after) - 10 124 | # rubocop:disable Layout/LineLength 125 | " -d '#{most_recent_time.strftime('%Y-%m-%d %H:%M:%S')}Z<#{Time.now.utc.strftime('%Y-%m-%d %H:%M:%S')}Z' " 126 | # rubocop:enable Layout/LineLength 127 | end 128 | 129 | def rlog_filename 130 | File.join(temp_folder, "#{(url + scm.branch_name.to_s).gsub(/\W/, '')}.rlog") 131 | end 132 | 133 | # Converts a CVS time string to a Ruby Time object 134 | def parse_time(token) 135 | case token 136 | when /(\d\d\d\d).(\d\d).(\d\d) (\d\d):(\d\d):(\d\d)/ 137 | Time.gm(Regexp.last_match(1).to_i, Regexp.last_match(2).to_i, Regexp.last_match(3).to_i, 138 | Regexp.last_match(4).to_i, Regexp.last_match(5).to_i, Regexp.last_match(6).to_i) 139 | end 140 | end 141 | 142 | # returns the host this adapter is connecting to 143 | def host 144 | @host ||= begin 145 | url =~ /@([^:]*):/ 146 | Regexp.last_match(1) 147 | end 148 | end 149 | 150 | # returns the protocol this adapter connects with 151 | def protocol 152 | @protocol ||= case url 153 | when /^:pserver/ then :pserver 154 | when /^:ext/ then :ext 155 | end 156 | end 157 | 158 | def opt_branch 159 | '-b -r1:' 160 | end 161 | end 162 | end 163 | end 164 | -------------------------------------------------------------------------------- /spec/ohloh_scm/parser/branch_number_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe 'BranchNumber' do 4 | it 'must test basic' do 5 | assert_equal OhlohScm::BranchNumber.new('1.1').to_a, [1, 1] 6 | assert_equal OhlohScm::BranchNumber.new('1234.1234').to_a, [1234, 1234] 7 | assert_equal OhlohScm::BranchNumber.new('1.2.3.4').to_a, [1, 2, 3, 4] 8 | end 9 | 10 | it 'must test simple inherits_from' do 11 | b = OhlohScm::BranchNumber.new('1.3') 12 | 13 | assert b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.2')) 14 | assert b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.1')) 15 | assert b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.3')) 16 | 17 | refute b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.4')) 18 | refute b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.1.2.1')) 19 | refute b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.2.2.1')) 20 | refute b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.3.2.1')) 21 | end 22 | 23 | it 'must test complex inherits_from' do 24 | b = OhlohScm::BranchNumber.new('1.3.6.3.2.3') 25 | 26 | assert b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.2')) 27 | assert b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.1')) 28 | assert b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.3')) 29 | assert b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.3.6.1')) 30 | assert b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.3.6.2')) 31 | assert b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.3.6.3')) 32 | assert b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.3.6.3.2.1')) 33 | assert b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.3.6.3.2.2')) 34 | assert b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.3.6.3.2.3')) 35 | 36 | refute b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.4')) 37 | refute b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.1.2.1')) 38 | refute b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.2.2.1')) 39 | refute b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.3.2.1')) 40 | refute b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.3.4.1')) 41 | refute b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.3.6.1.2.1')) 42 | refute b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.3.6.4')) 43 | refute b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.3.6.3.4.1')) 44 | refute b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.3.6.3.2.2.2.1')) 45 | refute b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.3.6.3.2.4')) 46 | end 47 | 48 | it 'must test primary revision number change' do 49 | b = OhlohScm::BranchNumber.new('2.3') 50 | 51 | assert b.send(:inherits_from?, OhlohScm::BranchNumber.new('2.2')) 52 | assert b.send(:inherits_from?, OhlohScm::BranchNumber.new('2.1')) 53 | assert b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.1')) 54 | assert b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.9999')) 55 | 56 | refute b.send(:inherits_from?, OhlohScm::BranchNumber.new('2.4')) 57 | refute b.send(:inherits_from?, OhlohScm::BranchNumber.new('3.1')) 58 | end 59 | 60 | it 'must test complex primary revision number change' do 61 | b = OhlohScm::BranchNumber.new('2.3.2.1') 62 | 63 | assert b.send(:inherits_from?, OhlohScm::BranchNumber.new('2.3')) 64 | assert b.send(:inherits_from?, OhlohScm::BranchNumber.new('2.2')) 65 | assert b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.1')) 66 | assert b.send(:inherits_from?, OhlohScm::BranchNumber.new('1.9999')) 67 | 68 | refute b.send(:inherits_from?, OhlohScm::BranchNumber.new('3.1')) 69 | end 70 | 71 | it 'must test simple on_same_line' do 72 | b = OhlohScm::BranchNumber.new('1.3') 73 | 74 | assert b.on_same_line?(OhlohScm::BranchNumber.new('1.2')) 75 | assert b.on_same_line?(OhlohScm::BranchNumber.new('1.1')) 76 | assert b.on_same_line?(OhlohScm::BranchNumber.new('1.3')) 77 | assert b.on_same_line?(OhlohScm::BranchNumber.new('1.4')) 78 | 79 | refute b.on_same_line?(OhlohScm::BranchNumber.new('1.1.2.1')) 80 | refute b.on_same_line?(OhlohScm::BranchNumber.new('1.2.2.1')) 81 | refute b.on_same_line?(OhlohScm::BranchNumber.new('1.3.2.1')) 82 | end 83 | 84 | it 'must test complex on_same_line' do 85 | b = OhlohScm::BranchNumber.new('1.3.6.3.2.3') 86 | 87 | assert b.on_same_line?(OhlohScm::BranchNumber.new('1.1')) 88 | assert b.on_same_line?(OhlohScm::BranchNumber.new('1.2')) 89 | assert b.on_same_line?(OhlohScm::BranchNumber.new('1.3')) 90 | assert b.on_same_line?(OhlohScm::BranchNumber.new('1.3.6.1')) 91 | assert b.on_same_line?(OhlohScm::BranchNumber.new('1.3.6.2')) 92 | assert b.on_same_line?(OhlohScm::BranchNumber.new('1.3.6.3')) 93 | assert b.on_same_line?(OhlohScm::BranchNumber.new('1.3.6.3.2.1')) 94 | assert b.on_same_line?(OhlohScm::BranchNumber.new('1.3.6.3.2.2')) 95 | assert b.on_same_line?(OhlohScm::BranchNumber.new('1.3.6.3.2.3')) 96 | assert b.on_same_line?(OhlohScm::BranchNumber.new('1.3.6.3.2.4')) 97 | assert b.on_same_line?(OhlohScm::BranchNumber.new('1.3.6.3.2.99')) 98 | 99 | refute b.on_same_line?(OhlohScm::BranchNumber.new('1.4')) 100 | refute b.on_same_line?(OhlohScm::BranchNumber.new('1.1.2.1')) 101 | refute b.on_same_line?(OhlohScm::BranchNumber.new('1.2.2.1')) 102 | refute b.on_same_line?(OhlohScm::BranchNumber.new('1.3.2.1')) 103 | refute b.on_same_line?(OhlohScm::BranchNumber.new('1.3.4.1')) 104 | refute b.on_same_line?(OhlohScm::BranchNumber.new('1.3.6.1.2.1')) 105 | refute b.on_same_line?(OhlohScm::BranchNumber.new('1.3.6.4')) 106 | refute b.on_same_line?(OhlohScm::BranchNumber.new('1.3.6.3.4.1')) 107 | refute b.on_same_line?(OhlohScm::BranchNumber.new('1.3.6.3.2.2.2.1')) 108 | refute b.on_same_line?(OhlohScm::BranchNumber.new('1.3.6.3.2.99.2.1')) 109 | end 110 | 111 | # Crazy CVS inserts a zero before the last piece of a branch number 112 | it 'must test magic branch numbers' do 113 | assert OhlohScm::BranchNumber.new('1.1.2.1').on_same_line?(OhlohScm::BranchNumber.new('1.1.0.2')) 114 | assert OhlohScm::BranchNumber.new('1.1.2.1.2.1').on_same_line?(OhlohScm::BranchNumber.new('1.1.0.2')) 115 | 116 | assert OhlohScm::BranchNumber.new('1.1.0.2').on_same_line?(OhlohScm::BranchNumber.new('1.1')) 117 | refute OhlohScm::BranchNumber.new('1.1.0.2').on_same_line?(OhlohScm::BranchNumber.new('1.2')) 118 | assert OhlohScm::BranchNumber.new('1.1.0.2').on_same_line?(OhlohScm::BranchNumber.new('1.1.2.1')) 119 | 120 | assert OhlohScm::BranchNumber.new('1.1.0.4').on_same_line?(OhlohScm::BranchNumber.new('1.1')) 121 | refute OhlohScm::BranchNumber.new('1.1.0.4').on_same_line?(OhlohScm::BranchNumber.new('1.1.2.1')) 122 | refute OhlohScm::BranchNumber.new('1.1.0.4').on_same_line?(OhlohScm::BranchNumber.new('1.1.0.2')) 123 | assert OhlohScm::BranchNumber.new('1.1.0.4').on_same_line?(OhlohScm::BranchNumber.new('1.1.4.1')) 124 | refute OhlohScm::BranchNumber.new('1.1.0.4').on_same_line?(OhlohScm::BranchNumber.new('1.1.0.6')) 125 | end 126 | end 127 | --------------------------------------------------------------------------------