├── .coveralls.yml ├── .github ├── funding.yml ├── release-drafter.yml ├── CODEOWNERS ├── dependabot.yml ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── no-response.yml ├── settings.yml ├── stale.yml └── config.yml ├── spec ├── fixtures │ ├── git-repo │ │ ├── HEAD │ │ ├── refs │ │ │ └── heads │ │ │ │ └── master │ │ ├── description │ │ ├── config │ │ ├── objects │ │ │ ├── 9f │ │ │ │ └── b613b48d8f2d96e993cd2f58f2018aa6d2c56e │ │ │ ├── e6 │ │ │ │ └── 9de29bb2d1d6434b8b29ae775ad8c2e48c5391 │ │ │ └── f9 │ │ │ │ └── 3e3a1a1525fb5b91020da86e44810c87a2d7bc │ │ └── info │ │ │ └── exclude │ ├── orgs │ │ ├── balter-test-org │ │ │ ├── teams.json │ │ │ └── repos.json │ │ └── balter-test-org.json │ ├── teams │ │ ├── 1 │ │ │ ├── members.json │ │ │ └── repos.json │ │ └── 1.json │ ├── users │ │ └── benbalter.json │ └── repos │ │ └── balter-test-org │ │ ├── some-repo │ │ ├── issues │ │ │ ├── 1 │ │ │ │ └── comments.json │ │ │ ├── comments │ │ │ │ └── 1.json │ │ │ └── 1.json │ │ └── issues.json │ │ └── some-repo.json ├── github_records_archiver │ ├── git_repository_spec.rb │ ├── user_spec.rb │ ├── data_helper_spec.rb │ ├── wiki_spec.rb │ ├── organization_spec.rb │ ├── repository_spec.rb │ ├── comment_spec.rb │ ├── cli_spec.rb │ ├── team_spec.rb │ └── issue_spec.rb ├── github_records_archiver_spec.rb └── spec_helper.rb ├── .rspec ├── script ├── bootstrap ├── console └── cibuild ├── .gitignore ├── lib ├── github_records_archiver │ ├── version.rb │ ├── user.rb │ ├── wiki.rb │ ├── comment.rb │ ├── data_helper.rb │ ├── organization.rb │ ├── repository.rb │ ├── team.rb │ ├── git_repository.rb │ ├── issue.rb │ └── cli.rb └── github_records_archiver.rb ├── docs ├── SECURITY.md ├── CODE_OF_CONDUCT.md └── CONTRIBUTING.md ├── Rakefile ├── .travis.yml ├── bin └── github-records-archiver ├── Gemfile ├── .rubocop.yml ├── LICENSE.md ├── github_records_archiver.gemspec └── README.md /.coveralls.yml: -------------------------------------------------------------------------------- 1 | service_name: travis-ci 2 | -------------------------------------------------------------------------------- /.github/funding.yml: -------------------------------------------------------------------------------- 1 | patreon: benbalter 2 | -------------------------------------------------------------------------------- /spec/fixtures/git-repo/HEAD: -------------------------------------------------------------------------------- 1 | ref: refs/heads/master 2 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --format documentation 2 | --color 3 | --require spec_helper 4 | -------------------------------------------------------------------------------- /script/bootstrap: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ex 4 | 5 | bundle install 6 | -------------------------------------------------------------------------------- /.github/release-drafter.yml: -------------------------------------------------------------------------------- 1 | template: | 2 | ## What's Changed 3 | 4 | $CHANGES 5 | -------------------------------------------------------------------------------- /spec/fixtures/git-repo/refs/heads/master: -------------------------------------------------------------------------------- 1 | 9fb613b48d8f2d96e993cd2f58f2018aa6d2c56e 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.env 2 | /archive 3 | /coverage/ 4 | /Gemfile.lock 5 | /spec/examples.txt 6 | /*.gem 7 | -------------------------------------------------------------------------------- /script/console: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ex 4 | 5 | bundle exec pry -r './lib/github_records_archiver' 6 | -------------------------------------------------------------------------------- /lib/github_records_archiver/version.rb: -------------------------------------------------------------------------------- 1 | module GitHubRecordsArchiver 2 | VERSION = '0.3.1'.freeze 3 | end 4 | -------------------------------------------------------------------------------- /spec/fixtures/git-repo/description: -------------------------------------------------------------------------------- 1 | Unnamed repository; edit this file 'description' to name the repository. 2 | -------------------------------------------------------------------------------- /docs/SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | To report a security vulnerability, please email [ben@balter.com](mailto:ben@balter.com). 4 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'bundler/gem_tasks' 2 | require 'rspec/core/rake_task' 3 | 4 | RSpec::Core::RakeTask.new(:spec) 5 | 6 | task default: :spec 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | rvm: 2 | - 2.4 3 | before_install: gem install bundler 4 | langauage: ruby 5 | script: script/cibuild 6 | sudo: false 7 | cache: bundler 8 | -------------------------------------------------------------------------------- /bin/github-records-archiver: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require_relative '../lib/github_records_archiver' 4 | 5 | GitHubRecordsArchiver::CLI.start(ARGV) 6 | -------------------------------------------------------------------------------- /script/cibuild: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ex 4 | 5 | bundle exec rspec 6 | bundle exec rubocop 7 | bundle exec gem build github_records_archiver.gemspec 8 | -------------------------------------------------------------------------------- /spec/fixtures/git-repo/config: -------------------------------------------------------------------------------- 1 | [core] 2 | repositoryformatversion = 0 3 | filemode = true 4 | bare = true 5 | ignorecase = true 6 | precomposeunicode = true 7 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } 4 | 5 | gemspec 6 | 7 | gem 'coveralls', require: false 8 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Require @benbalter's :+1: for changes to the .github repo-config files 2 | # mainly due to https://github.com/probot/settings privilege escalation 3 | .github/* @benbalter 4 | -------------------------------------------------------------------------------- /spec/fixtures/git-repo/objects/9f/b613b48d8f2d96e993cd2f58f2018aa6d2c56e: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/benbalter/github_records_archiver/HEAD/spec/fixtures/git-repo/objects/9f/b613b48d8f2d96e993cd2f58f2018aa6d2c56e -------------------------------------------------------------------------------- /spec/fixtures/git-repo/objects/e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/benbalter/github_records_archiver/HEAD/spec/fixtures/git-repo/objects/e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391 -------------------------------------------------------------------------------- /spec/fixtures/git-repo/objects/f9/3e3a1a1525fb5b91020da86e44810c87a2d7bc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/benbalter/github_records_archiver/HEAD/spec/fixtures/git-repo/objects/f9/3e3a1a1525fb5b91020da86e44810c87a2d7bc -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: bundler 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | time: "10:00" 8 | timezone: US/Eastern 9 | open-pull-requests-limit: 99 10 | -------------------------------------------------------------------------------- /spec/fixtures/git-repo/info/exclude: -------------------------------------------------------------------------------- 1 | # git ls-files --others --exclude-from=.git/info/exclude 2 | # Lines that start with '#' are comments. 3 | # For a project mostly in C, the following would be a good set of 4 | # exclude patterns (uncomment them if you want to use them): 5 | # *.[oa] 6 | # *~ 7 | -------------------------------------------------------------------------------- /spec/fixtures/orgs/balter-test-org/teams.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": 1, 4 | "url": "https://api.github.com/teams/1", 5 | "name": "Justice League", 6 | "slug": "justice-league", 7 | "description": "A great team.", 8 | "privacy": "closed", 9 | "permission": "admin", 10 | "members_url": "https://api.github.com/teams/1/members{/member}", 11 | "repositories_url": "https://api.github.com/teams/1/repos", 12 | "parent": null 13 | } 14 | ] 15 | -------------------------------------------------------------------------------- /lib/github_records_archiver/user.rb: -------------------------------------------------------------------------------- 1 | module GitHubRecordsArchiver 2 | class User 3 | attr_reader :login 4 | alias name login 5 | 6 | include DataHelper 7 | 8 | KEYS = %i[login site_admin type].freeze 9 | 10 | def initialize(login_or_hash) 11 | if login_or_hash.is_a? String 12 | @login = login_or_hash 13 | else 14 | @login = login_or_hash[:login] 15 | @data = login_or_hash.to_h 16 | end 17 | end 18 | 19 | def data 20 | @data ||= GitHubRecordsArchiver.client.user login 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | 5 | --- 6 | 7 | ### Describe the bug 8 | 9 | A clear and concise description of what the bug is. 10 | 11 | ### Steps to reproduce the behavior 12 | 13 | 1. Go to '...' 14 | 2. Click on '....' 15 | 3. Scroll down to '....' 16 | 4. See error 17 | 18 | ### Expected behavior 19 | 20 | A clear and concise description of what you expected to happen. 21 | 22 | ### Screenshots 23 | 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | ### Additional context 27 | 28 | Add any other context about the problem here. 29 | -------------------------------------------------------------------------------- /lib/github_records_archiver/wiki.rb: -------------------------------------------------------------------------------- 1 | module GitHubRecordsArchiver 2 | class Wiki < GitRepository 3 | attr_accessor :repository 4 | 5 | include DataHelper 6 | 7 | def initialize(repository) 8 | @repository = if repository.is_a?(String) 9 | Repository.new(repository) 10 | else 11 | repository 12 | end 13 | end 14 | 15 | def repo_dir 16 | @repo_dir ||= File.join repository.repo_dir, 'wiki' 17 | end 18 | 19 | private 20 | 21 | def clone_url 22 | @clone_url ||= repository.clone_url.gsub(/\.git\z/, '.wiki.git') 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | AllCops: 2 | Exclude: 3 | - archive/**/* 4 | - vendor/**/* 5 | 6 | Style/Documentation: 7 | Enabled: false 8 | 9 | Metrics/BlockLength: 10 | Exclude: 11 | - spec/**/* 12 | - '*.gemspec' 13 | - lib/github_records_archiver/cli.rb 14 | 15 | Metrics/LineLength: 16 | Exclude: 17 | - spec/**/* 18 | - lib/github_records_archiver/cli.rb 19 | 20 | Metrics/MethodLength: 21 | Exclude: 22 | - lib/github_records_archiver/cli.rb 23 | - spec/spec_helper.rb 24 | 25 | Metrics/AbcSize: 26 | Exclude: 27 | - lib/github_records_archiver/cli.rb 28 | 29 | Metrics/CyclomaticComplexity: 30 | Exclude: 31 | - spec/spec_helper.rb 32 | 33 | Layout/IndentHeredoc: 34 | Enabled: false 35 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | 5 | --- 6 | 7 | ### Is your feature request related to a problem? Please describe the problem you're trying to solve. 8 | 9 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 10 | 11 | ### Describe the solution you'd like 12 | 13 | A clear and concise description of what you want to happen. 14 | 15 | ### Describe alternatives you've considered 16 | 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | ### Additional context 20 | 21 | Add any other context or screenshots about the feature request here. 22 | -------------------------------------------------------------------------------- /spec/github_records_archiver/git_repository_spec.rb: -------------------------------------------------------------------------------- 1 | 2 | RSpec.describe GitHubRecordsArchiver::GitRepository do 3 | class GitRepositorySpec < described_class 4 | def repo_dir 5 | File.join GitHubRecordsArchiver.dest_dir, 'git-repo' 6 | end 7 | 8 | private 9 | 10 | def clone_url 11 | File.join fixture_dir, 'git-repo' 12 | end 13 | end 14 | 15 | before do 16 | FileUtils.rm_rf subject.repo_dir 17 | end 18 | 19 | subject { GitRepositorySpec.new } 20 | 21 | it 'clones' do 22 | subject.clone 23 | expect(Dir.exist?(subject.repo_dir)).to be_truthy 24 | end 25 | 26 | it 'pulls' do 27 | subject.clone 28 | subject.clone 29 | expect(Dir.exist?(subject.repo_dir)).to be_truthy 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /lib/github_records_archiver/comment.rb: -------------------------------------------------------------------------------- 1 | module GitHubRecordsArchiver 2 | class Comment 3 | attr_reader :repository 4 | attr_reader :id 5 | 6 | include DataHelper 7 | 8 | def initialize(repo, id) 9 | repo = Repository.new(repo) if repo.is_a? String 10 | @repository = repo 11 | @id = id 12 | end 13 | 14 | def self.from_hash(repo, hash) 15 | comment = Comment.new(repo, hash[:number]) 16 | comment.instance_variable_set '@data', hash.to_h 17 | comment 18 | end 19 | 20 | def data 21 | @data ||= begin 22 | GitHubRecordsArchiver.client.issue_comment(repository.full_name, id) 23 | end 24 | end 25 | 26 | def to_s 27 | output = "@#{user[:login]} at #{created_at} wrote:\n\n" 28 | output << body 29 | output 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /.github/no-response.yml: -------------------------------------------------------------------------------- 1 | # Configuration for probot-no-response - https://github.com/probot/no-response 2 | # Note: Please Don't edit this file directly. 3 | # Edit https://github.com/benbalter/shared-community-files instead. 4 | 5 | # Number of days of inactivity before an Issue is closed for lack of response 6 | daysUntilClose: 14 7 | # Label requiring a response 8 | responseRequiredLabel: more-information-needed 9 | # Comment to post when closing an Issue for lack of response. Set to `false` to disable 10 | closeComment: > 11 | This issue has been automatically closed because there has been no response 12 | to our request for more information from the original author. With only the 13 | information that is currently in the issue, we don't have enough information 14 | to take action. Please reach out if you have or find the answers we need so 15 | that we can investigate further. 16 | -------------------------------------------------------------------------------- /spec/github_records_archiver/user_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe GitHubRecordsArchiver::User do 2 | let(:login) { 'benbalter' } 3 | subject { described_class.new(login) } 4 | 5 | before do 6 | stub_api_request 'users/benbalter' 7 | end 8 | 9 | it 'stores the login' do 10 | expect(subject.login).to eql(login) 11 | expect(subject.name).to eql(login) 12 | end 13 | 14 | context 'when given a hash' do 15 | let(:hash) do 16 | { 17 | login: login 18 | } 19 | end 20 | 21 | subject { described_class.new(hash) } 22 | 23 | it 'stores the login' do 24 | expect(subject.login).to eql(login) 25 | end 26 | 27 | it 'stores the data' do 28 | expect(subject.data).to eql(hash) 29 | end 30 | end 31 | 32 | it 'retrieves metadata' do 33 | expect(subject.data).to be_a(Sawyer::Resource) 34 | expect(subject.data.login).to eql(login) 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /.github/settings.yml: -------------------------------------------------------------------------------- 1 | # Repository settings set via https://github.com/probot/settings 2 | # Note: Please Don't edit this file directly. 3 | # Edit https://github.com/benbalter/shared-community-files instead. 4 | 5 | repository: 6 | has_issues: true 7 | has_wiki: false 8 | has_projects: false 9 | has_downloads: false 10 | 11 | labels: 12 | - name: help wanted 13 | oldname: help-wanted 14 | color: 0e8a16 15 | - name: more-information-needed 16 | color: d93f0b 17 | - name: bug 18 | color: b60205 19 | - name: feature 20 | color: 1d76db 21 | - name: good first issue 22 | color: "5319e7" 23 | 24 | # Not currently implemented by probot/settings, but manually implemented in script/deploy 25 | branch_protection: 26 | restrictions: null 27 | enforce_admins: false 28 | required_status_checks: 29 | strict: true 30 | contexts: 31 | - "continuous-integration/travis-ci" 32 | required_pull_request_reviews: 33 | require_code_owner_reviews: true 34 | -------------------------------------------------------------------------------- /lib/github_records_archiver/data_helper.rb: -------------------------------------------------------------------------------- 1 | module GitHubRecordsArchiver 2 | module DataHelper 3 | attr_writer :data 4 | 5 | def method_missing(method_sym, *arguments, &block) 6 | return data[method_sym] if data_key?(method_sym) 7 | if method_sym.to_s.end_with? '?' 8 | !send(non_predicate_method(method_sym)).to_s.empty? 9 | else 10 | super 11 | end 12 | end 13 | 14 | def respond_to_missing?(method_sym, include_private = false) 15 | if data_key? non_predicate_method(method_sym) 16 | true 17 | else 18 | super 19 | end 20 | end 21 | 22 | def to_h 23 | data.to_h 24 | end 25 | alias as_json to_h 26 | 27 | def to_json 28 | as_json.to_json 29 | end 30 | 31 | def data 32 | raise 'Not implemented' 33 | end 34 | 35 | private 36 | 37 | def data_key?(key) 38 | data && data.key?(key) 39 | end 40 | 41 | def non_predicate_method(method_sym) 42 | method_sym.to_s.gsub(/\?\z/, '').to_sym 43 | end 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /spec/github_records_archiver/data_helper_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe GitHubRecordsArchiver::DataHelper do 2 | class DataHelperSpec 3 | include GitHubRecordsArchiver::DataHelper 4 | 5 | def data 6 | { 7 | foo: 'bar' 8 | } 9 | end 10 | end 11 | 12 | subject { DataHelperSpec.new } 13 | 14 | it 'returns values' do 15 | expect(subject).to respond_to(:foo) 16 | expect(subject.foo).to eql('bar') 17 | end 18 | 19 | it 'responds to data' do 20 | expect(subject.data).to eql(foo: 'bar') 21 | end 22 | 23 | it 'responds to predicate methods' do 24 | expect(subject).to respond_to('foo?') 25 | expect(subject.foo?).to be_truthy 26 | end 27 | 28 | it "doesn't respond to invalid methods" do 29 | expect(subject).to_not respond_to(:bar) 30 | expect(subject).to_not respond_to(:bar?) 31 | end 32 | 33 | it 'responds to #as_json' do 34 | expect(subject.as_json).to eql(subject.data.to_h) 35 | end 36 | 37 | it 'responds to #to_json' do 38 | expect(subject.to_json).to eql(subject.data.to_json) 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /.github/stale.yml: -------------------------------------------------------------------------------- 1 | # Configuration for probot-stale - https://github.com/probot/stale 2 | # Note: Please Don't edit this file directly. 3 | # Edit https://github.com/benbalter/shared-community-files instead. 4 | 5 | # Number of days of inactivity before an Issue or Pull Request becomes stale 6 | daysUntilStale: 60 7 | 8 | # Number of days of inactivity before a stale Issue or Pull Request is closed 9 | daysUntilClose: 7 10 | 11 | # Issues or Pull Requests with these labels will never be considered stale 12 | exemptLabels: 13 | - pinned 14 | - security 15 | 16 | # Label to use when marking as stale 17 | staleLabel: wontfix 18 | 19 | # Comment to post when marking as stale. Set to `false` to disable 20 | markComment: > 21 | This issue has been automatically marked as stale because it has not had 22 | recent activity. It will be closed if no further activity occurs. Thank you 23 | for your contributions. 24 | 25 | # Comment to post when closing a stale Issue or Pull Request. Set to `false` to disable 26 | closeComment: false 27 | 28 | # Limit to only `issues` or `pulls` 29 | # only: issues 30 | -------------------------------------------------------------------------------- /spec/github_records_archiver_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe GitHubRecordsArchiver do 2 | before do 3 | %w[client dest_dir token].each do |option| 4 | described_class.public_send "#{option}=", nil 5 | end 6 | end 7 | 8 | after do 9 | %w[client dest_dir].each do |option| 10 | described_class.public_send "#{option}=", nil 11 | end 12 | described_class.token = ENV['GITHUB_TOKEN'] = 'TEST_TOKEN' 13 | end 14 | 15 | it 'returns the token' do 16 | with_env 'GITHUB_TOKEN', 'TOKEN' do 17 | expect(described_class.token).to eql('TOKEN') 18 | end 19 | end 20 | 21 | it 'returns the Octokit client' do 22 | with_env 'GITHUB_TOKEN', 'TOKEN' do 23 | expect(described_class.client).to be_a(Octokit::Client) 24 | expect(described_class.client.access_token).to eql('TOKEN') 25 | end 26 | end 27 | 28 | it 'sets the default destination directory' do 29 | path = File.expand_path '../archive', __dir__ 30 | expect(described_class.dest_dir).to eql(path) 31 | end 32 | 33 | it 'exposes the version' do 34 | expect(described_class::VERSION).to match(/\d\.\d\.\d/) 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Ben Balter 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /spec/fixtures/teams/1/members.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "login": "balter-test-org", 4 | "id": 1, 5 | "avatar_url": "https://github.com/images/error/balter-test-org_happy.gif", 6 | "gravatar_id": "", 7 | "url": "https://api.github.com/users/balter-test-org", 8 | "html_url": "https://github.com/balter-test-org", 9 | "followers_url": "https://api.github.com/users/balter-test-org/followers", 10 | "following_url": "https://api.github.com/users/balter-test-org/following{/other_user}", 11 | "gists_url": "https://api.github.com/users/balter-test-org/gists{/gist_id}", 12 | "starred_url": "https://api.github.com/users/balter-test-org/starred{/owner}{/repo}", 13 | "subscriptions_url": "https://api.github.com/users/balter-test-org/subscriptions", 14 | "organizations_url": "https://api.github.com/users/balter-test-org/orgs", 15 | "repos_url": "https://api.github.com/users/balter-test-org/repos", 16 | "events_url": "https://api.github.com/users/balter-test-org/events{/privacy}", 17 | "received_events_url": "https://api.github.com/users/balter-test-org/received_events", 18 | "type": "User", 19 | "site_admin": false 20 | } 21 | ] 22 | -------------------------------------------------------------------------------- /spec/github_records_archiver/wiki_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe GitHubRecordsArchiver::Wiki do 2 | let(:repo) { 'balter-test-org/some-repo' } 3 | subject { described_class.new(repo) } 4 | 5 | before do 6 | stub_api_request('repos/balter-test-org/some-repo') 7 | end 8 | 9 | it 'stores the repo' do 10 | expect(subject.repository).to be_a(GitHubRecordsArchiver::Repository) 11 | expect(subject.repository.name).to eql(repo) 12 | end 13 | 14 | context 'when initialized with a Repository object' do 15 | subject do 16 | repository = GitHubRecordsArchiver::Repository.new(repo) 17 | described_class.new(repository) 18 | end 19 | 20 | it 'stores the repo' do 21 | expect(subject.repository).to be_a(GitHubRecordsArchiver::Repository) 22 | expect(subject.repository.name).to eql(repo) 23 | end 24 | end 25 | 26 | it 'builds the repo dir' do 27 | path = File.expand_path "../../archive/#{repo}/wiki", __dir__ 28 | expect(subject.repo_dir).to eql(path) 29 | end 30 | 31 | it 'builds the clone URL' do 32 | expected = "https://TEST_TOKEN:x-oauth-basic@github.com/#{repo}.wiki.git" 33 | expect(subject.send(:clone_url)).to eql(expected) 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /spec/fixtures/teams/1.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": 1, 3 | "url": "https://api.github.com/teams/1", 4 | "name": "Justice League", 5 | "slug": "justice-league", 6 | "description": "A great team.", 7 | "privacy": "closed", 8 | "permission": "admin", 9 | "members_url": "https://api.github.com/teams/1/members{/member}", 10 | "repositories_url": "https://api.github.com/teams/1/repos", 11 | "parent": null, 12 | "members_count": 3, 13 | "repos_count": 10, 14 | "created_at": "2017-07-14T16:53:42Z", 15 | "updated_at": "2017-08-17T12:37:15Z", 16 | "organization": { 17 | "login": "github", 18 | "id": 1, 19 | "url": "https://api.github.com/orgs/github", 20 | "repos_url": "https://api.github.com/orgs/github/repos", 21 | "events_url": "https://api.github.com/orgs/github/events", 22 | "hooks_url": "https://api.github.com/orgs/github/hooks", 23 | "issues_url": "https://api.github.com/orgs/github/issues", 24 | "members_url": "https://api.github.com/orgs/github/members{/member}", 25 | "public_members_url": "https://api.github.com/orgs/github/public_members{/member}", 26 | "avatar_url": "https://github.com/images/error/balter-test-org_happy.gif", 27 | "description": "A great organization" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /lib/github_records_archiver/organization.rb: -------------------------------------------------------------------------------- 1 | module GitHubRecordsArchiver 2 | class Organization 3 | attr_reader :name 4 | 5 | include DataHelper 6 | 7 | def initialize(name) 8 | @name = name 9 | end 10 | 11 | def data 12 | @data ||= GitHubRecordsArchiver.client.organization name 13 | end 14 | 15 | def repositories 16 | @repositories ||= begin 17 | repos = GitHubRecordsArchiver.client.organization_repositories(name) 18 | repos.map { |hash| Repository.new(hash) } 19 | end 20 | end 21 | alias repos repositories 22 | 23 | def teams 24 | @teams ||= begin 25 | teams = GitHubRecordsArchiver.client.organization_teams(name) 26 | teams.map { |h| Team.from_hash(self, h) } 27 | rescue Octokit::Forbidden 28 | [] 29 | end 30 | end 31 | 32 | def archive_dir 33 | @archive_dir ||= begin 34 | dir = File.expand_path name, GitHubRecordsArchiver.dest_dir 35 | FileUtils.mkdir_p(dir) unless Dir.exist?(dir) 36 | dir 37 | end 38 | end 39 | 40 | def teams_dir 41 | @teams_dir ||= begin 42 | dir = File.expand_path 'teams', archive_dir 43 | FileUtils.mkdir_p(dir) unless Dir.exist?(dir) 44 | dir 45 | end 46 | end 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /spec/fixtures/users/benbalter.json: -------------------------------------------------------------------------------- 1 | { 2 | "login": "benbalter", 3 | "id": 1, 4 | "avatar_url": "https://github.com/images/error/benbalter_happy.gif", 5 | "gravatar_id": "", 6 | "url": "https://api.github.com/users/benbalter", 7 | "html_url": "https://github.com/benbalter", 8 | "followers_url": "https://api.github.com/users/benbalter/followers", 9 | "following_url": "https://api.github.com/users/benbalter/following{/other_user}", 10 | "gists_url": "https://api.github.com/users/benbalter/gists{/gist_id}", 11 | "starred_url": "https://api.github.com/users/benbalter/starred{/owner}{/repo}", 12 | "subscriptions_url": "https://api.github.com/users/benbalter/subscriptions", 13 | "organizations_url": "https://api.github.com/users/benbalter/orgs", 14 | "repos_url": "https://api.github.com/users/benbalter/repos", 15 | "events_url": "https://api.github.com/users/benbalter/events{/privacy}", 16 | "received_events_url": "https://api.github.com/users/benbalter/received_events", 17 | "type": "User", 18 | "site_admin": false, 19 | "name": "monalisa benbalter", 20 | "company": "GitHub", 21 | "blog": "https://github.com/blog", 22 | "location": "San Francisco", 23 | "email": "benbalter@github.com", 24 | "hireable": false, 25 | "bio": "There once was...", 26 | "public_repos": 2, 27 | "public_gists": 1, 28 | "followers": 20, 29 | "following": 0, 30 | "created_at": "2008-01-14T04:33:35Z", 31 | "updated_at": "2008-01-14T04:33:35Z" 32 | } 33 | -------------------------------------------------------------------------------- /spec/github_records_archiver/organization_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe GitHubRecordsArchiver::Organization do 2 | let(:org_name) { 'balter-test-org' } 3 | subject { described_class.new(org_name) } 4 | 5 | before do 6 | stub_api_request('orgs/balter-test-org') 7 | stub_api_request('orgs/balter-test-org/repos', per_page: 100) 8 | stub_api_request('orgs/balter-test-org/teams', per_page: 100) 9 | end 10 | 11 | it 'stores the name' do 12 | expect(subject.name).to eql(org_name) 13 | end 14 | 15 | it 'retrieves metadata' do 16 | expect(subject.data).to be_a(Sawyer::Resource) 17 | expect(subject.login).to eql(org_name) 18 | end 19 | 20 | it 'returns repos' do 21 | expect(subject.repositories).to be_an(Array) 22 | repo = subject.repositories.first 23 | expect(repo).to be_a(GitHubRecordsArchiver::Repository) 24 | end 25 | 26 | it 'returns teams' do 27 | expect(subject.teams).to be_an(Array) 28 | expect(subject.teams.first).to be_a(GitHubRecordsArchiver::Team) 29 | end 30 | 31 | it 'returns the archive dir' do 32 | path = File.expand_path "../../archive/#{org_name}", __dir__ 33 | expect(subject.archive_dir).to eql(path) 34 | expect(Dir.exist?(subject.archive_dir)).to be_truthy 35 | end 36 | 37 | it 'returns the teams dir' do 38 | path = File.expand_path "../../archive/#{org_name}/teams", __dir__ 39 | expect(subject.teams_dir).to eql(path) 40 | expect(Dir.exist?(subject.teams_dir)).to be_truthy 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /lib/github_records_archiver/repository.rb: -------------------------------------------------------------------------------- 1 | module GitHubRecordsArchiver 2 | class Repository < GitRepository 3 | attr_reader :name 4 | include DataHelper 5 | 6 | KEYS = %i[ 7 | name full_name description private fork homepage 8 | forks_count stargazers_count watchers_count size 9 | ].freeze 10 | 11 | def initialize(name_or_hash) 12 | if name_or_hash.is_a?(String) 13 | @name = name_or_hash 14 | else 15 | @data = name_or_hash.to_h 16 | @name = @data[:full_name] 17 | end 18 | end 19 | 20 | def data 21 | @data ||= GitHubRecordsArchiver.client.repository(name) 22 | end 23 | 24 | def wiki 25 | @wiki ||= Wiki.new(self) if has_wiki? 26 | end 27 | 28 | def issues 29 | @issues ||= begin 30 | issues = GitHubRecordsArchiver.client.list_issues name, state: 'all' 31 | issues.map { |i| Issue.from_hash(self, i) } 32 | end 33 | end 34 | 35 | def issues_dir 36 | @issues_dir ||= begin 37 | dir = File.expand_path 'issues', repo_dir 38 | FileUtils.mkdir_p(dir) unless Dir.exist?(dir) 39 | dir 40 | end 41 | end 42 | 43 | def clone_url 44 | replacement = "https://#{GitHubRecordsArchiver.token}:x-oauth-basic@" 45 | data[:clone_url].gsub(%r{https?://}, replacement) 46 | end 47 | 48 | def repo_dir 49 | @repo_dir ||= File.expand_path full_name, GitHubRecordsArchiver.dest_dir 50 | end 51 | end 52 | end 53 | -------------------------------------------------------------------------------- /spec/fixtures/repos/balter-test-org/some-repo/issues/comments/1.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": 1, 3 | "url": "https://api.github.com/repos/balter-test-org/some-repo/issues/comments/1", 4 | "html_url": "https://github.com/balter-test-org/some-repo/issues/1347#issuecomment-1", 5 | "body": "Me too", 6 | "user": { 7 | "login": "balter-test-org", 8 | "id": 1, 9 | "avatar_url": "https://github.com/images/error/balter-test-org_happy.gif", 10 | "gravatar_id": "", 11 | "url": "https://api.github.com/users/balter-test-org", 12 | "html_url": "https://github.com/balter-test-org", 13 | "followers_url": "https://api.github.com/users/balter-test-org/followers", 14 | "following_url": "https://api.github.com/users/balter-test-org/following{/other_user}", 15 | "gists_url": "https://api.github.com/users/balter-test-org/gists{/gist_id}", 16 | "starred_url": "https://api.github.com/users/balter-test-org/starred{/owner}{/repo}", 17 | "subscriptions_url": "https://api.github.com/users/balter-test-org/subscriptions", 18 | "organizations_url": "https://api.github.com/users/balter-test-org/orgs", 19 | "repos_url": "https://api.github.com/users/balter-test-org/repos", 20 | "events_url": "https://api.github.com/users/balter-test-org/events{/privacy}", 21 | "received_events_url": "https://api.github.com/users/balter-test-org/received_events", 22 | "type": "User", 23 | "site_admin": false 24 | }, 25 | "created_at": "2011-04-14T16:00:49Z", 26 | "updated_at": "2011-04-14T16:00:49Z" 27 | } 28 | -------------------------------------------------------------------------------- /spec/fixtures/repos/balter-test-org/some-repo/issues/1/comments.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": 1, 4 | "url": "https://api.github.com/repos/balter-test-org/some-repo/issues/comments/1", 5 | "html_url": "https://github.com/balter-test-org/some-repo/issues/1347#issuecomment-1", 6 | "body": "Me too", 7 | "user": { 8 | "login": "balter-test-org", 9 | "id": 1, 10 | "avatar_url": "https://github.com/images/error/balter-test-org_happy.gif", 11 | "gravatar_id": "", 12 | "url": "https://api.github.com/users/balter-test-org", 13 | "html_url": "https://github.com/balter-test-org", 14 | "followers_url": "https://api.github.com/users/balter-test-org/followers", 15 | "following_url": "https://api.github.com/users/balter-test-org/following{/other_user}", 16 | "gists_url": "https://api.github.com/users/balter-test-org/gists{/gist_id}", 17 | "starred_url": "https://api.github.com/users/balter-test-org/starred{/owner}{/repo}", 18 | "subscriptions_url": "https://api.github.com/users/balter-test-org/subscriptions", 19 | "organizations_url": "https://api.github.com/users/balter-test-org/orgs", 20 | "repos_url": "https://api.github.com/users/balter-test-org/repos", 21 | "events_url": "https://api.github.com/users/balter-test-org/events{/privacy}", 22 | "received_events_url": "https://api.github.com/users/balter-test-org/received_events", 23 | "type": "User", 24 | "site_admin": false 25 | }, 26 | "created_at": "2011-04-14T16:00:49Z", 27 | "updated_at": "2011-04-14T16:00:49Z" 28 | } 29 | ] 30 | -------------------------------------------------------------------------------- /spec/fixtures/orgs/balter-test-org.json: -------------------------------------------------------------------------------- 1 | { 2 | "login": "balter-test-org", 3 | "id": 1, 4 | "url": "https://api.github.com/orgs/github", 5 | "repos_url": "https://api.github.com/orgs/balter-test-org/repos", 6 | "events_url": "https://api.github.com/orgs/balter-test-org/events", 7 | "hooks_url": "https://api.github.com/orgs/balter-test-org/hooks", 8 | "issues_url": "https://api.github.com/orgs/balter-test-org/issues", 9 | "members_url": "https://api.github.com/orgs/balter-test-org/members{/member}", 10 | "public_members_url": "https://api.github.com/orgs/balter-test-org/public_members{/member}", 11 | "avatar_url": "https://github.com/images/error/balter-test-org_happy.gif", 12 | "description": "A great organization", 13 | "name": "balter-test-org", 14 | "company": "GitHub", 15 | "blog": "https://github.com/blog", 16 | "location": "San Francisco", 17 | "email": "balter-test-org@github.com", 18 | "has_organization_projects": true, 19 | "has_repository_projects": true, 20 | "public_repos": 2, 21 | "public_gists": 1, 22 | "followers": 20, 23 | "following": 0, 24 | "html_url": "https://github.com/balter-test-org", 25 | "created_at": "2008-01-14T04:33:35Z", 26 | "type": "Organization", 27 | "total_private_repos": 100, 28 | "owned_private_repos": 100, 29 | "private_gists": 81, 30 | "disk_usage": 10000, 31 | "collaborators": 8, 32 | "billing_email": "support@github.com", 33 | "plan": { 34 | "name": "Medium", 35 | "space": 400, 36 | "private_repos": 20 37 | }, 38 | "default_repository_settings": "read", 39 | "members_can_create_repositories": true 40 | } 41 | -------------------------------------------------------------------------------- /github_records_archiver.gemspec: -------------------------------------------------------------------------------- 1 | 2 | lib = File.expand_path('lib', __dir__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'github_records_archiver/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = 'github_records_archiver' 8 | spec.version = GitHubRecordsArchiver::VERSION 9 | spec.authors = ['Ben Balter'] 10 | spec.email = ['ben.balter@github.com'] 11 | 12 | spec.summary = <<-SUMMARY 13 | Backs up a GitHub organization's repositories and all their associated 14 | information for archival purposes 15 | SUMMARY 16 | 17 | spec.homepage = 'https://github.com/benbalter/github_records_archiver' 18 | spec.license = 'MIT' 19 | 20 | spec.files = `git ls-files -z`.split("\x0").reject do |f| 21 | f.match(%r{^(test|spec|features)/}) 22 | end 23 | spec.bindir = 'bin' 24 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 25 | spec.require_paths = ['lib'] 26 | 27 | spec.add_dependency 'dotenv', '~> 2.0' 28 | spec.add_dependency 'octokit', '~> 4.0' 29 | spec.add_dependency 'parallel', '~> 1.10' 30 | spec.add_dependency 'ruby-progressbar', '~> 1.0' 31 | spec.add_dependency 'thor', '~> 0.19' 32 | spec.add_development_dependency 'addressable', '~> 2.5' 33 | spec.add_development_dependency 'bundler', '~> 1.16' 34 | spec.add_development_dependency 'pry', '~> 0.10' 35 | spec.add_development_dependency 'rake', '~> 10.0' 36 | spec.add_development_dependency 'rspec', '~> 3.0' 37 | spec.add_development_dependency 'rubocop', '~> 0.50' 38 | spec.add_development_dependency 'webmock', '~> 3.0' 39 | end 40 | -------------------------------------------------------------------------------- /lib/github_records_archiver/team.rb: -------------------------------------------------------------------------------- 1 | module GitHubRecordsArchiver 2 | class Team 3 | attr_reader :id 4 | attr_reader :organization 5 | 6 | include DataHelper 7 | 8 | KEYS = %i[name slug description privacy permission].freeze 9 | 10 | def initialize(org, id) 11 | org = Organization.new(org) if org.is_a? String 12 | @organization = org 13 | @id = id 14 | end 15 | 16 | def self.from_hash(org, hash) 17 | team = Team.new(org, hash[:id]) 18 | team.instance_variable_set '@data', hash.to_h 19 | team 20 | end 21 | 22 | def data 23 | @data ||= GitHubRecordsArchiver.client.team id 24 | end 25 | 26 | def repositories 27 | @repositories ||= begin 28 | repos = GitHubRecordsArchiver.client.team_repositories id 29 | repos.map { |r| Repository.new(r) } 30 | end 31 | end 32 | 33 | def members 34 | @members ||= begin 35 | members = GitHubRecordsArchiver.client.team_members id 36 | members.map { |m| User.new(m) } 37 | end 38 | end 39 | 40 | def archive 41 | File.write(path, to_s) 42 | end 43 | 44 | def to_s 45 | meta_for_markdown.to_yaml 46 | end 47 | 48 | def as_json 49 | data.to_h.merge(repositories: repositories.map(&:name)) 50 | end 51 | 52 | private 53 | 54 | def path 55 | File.expand_path "#{data[:slug]}.md", organization.teams_dir 56 | end 57 | 58 | def meta_for_markdown 59 | meta = {} 60 | KEYS.each { |key| meta[key.to_s] = data[key] } 61 | meta['repositories'] = repositories.map(&:name) 62 | meta['members'] = members.map(&:name) 63 | meta 64 | end 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /spec/github_records_archiver/repository_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe GitHubRecordsArchiver::Repository do 2 | let(:name) { 'balter-test-org/some-repo' } 3 | subject { described_class.new(name) } 4 | 5 | before do 6 | stub_api_request('repos/balter-test-org/some-repo') 7 | stub_api_request('repos/balter-test-org/some-repo/issues', per_page: 100, state: 'all') 8 | end 9 | 10 | it 'stores the name' do 11 | expect(subject.name).to eql(name) 12 | end 13 | 14 | context 'when initialized with a data hash' do 15 | subject { described_class.new(full_name: name) } 16 | 17 | it 'stores the name' do 18 | expect(subject.name).to eql(name) 19 | end 20 | end 21 | 22 | it 'retrieves metadata' do 23 | expect(subject.data).to be_a(Sawyer::Resource) 24 | expect(subject.name).to eql(name) 25 | end 26 | 27 | it 'retrieves wikis' do 28 | expect(subject.wiki).to be_a(GitHubRecordsArchiver::Wiki) 29 | end 30 | 31 | it 'retrieves issues' do 32 | expect(subject.issues).to be_a(Array) 33 | expect(subject.issues.first).to be_a(GitHubRecordsArchiver::Issue) 34 | end 35 | 36 | it 'returns the issues dir' do 37 | path = File.expand_path "../../archive/#{name}/issues", __dir__ 38 | expect(subject.issues_dir).to eql(path) 39 | expect(Dir.exist?(subject.issues_dir)).to be_truthy 40 | end 41 | 42 | it 'builds the repo dir' do 43 | path = File.expand_path "../../archive/#{name}", __dir__ 44 | expect(subject.send(:repo_dir)).to eql(path) 45 | end 46 | 47 | it 'builds the clone URL' do 48 | expected = 'https://TEST_TOKEN:x-oauth-basic@github.com/' 49 | expected << 'balter-test-org/some-repo.git' 50 | expect(subject.send(:clone_url)).to eql(expected) 51 | end 52 | end 53 | -------------------------------------------------------------------------------- /spec/github_records_archiver/comment_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe GitHubRecordsArchiver::Comment do 2 | let(:repo) { 'balter-test-org/some-repo' } 3 | let(:id) { 1 } 4 | subject { described_class.new(repo, id) } 5 | 6 | before do 7 | stub_api_request('repos/balter-test-org/some-repo') 8 | stub_api_request('repos/balter-test-org/some-repo/issues/comments/1', per_page: 100) 9 | end 10 | 11 | it 'stores the repo' do 12 | expect(subject.repository).to be_a(GitHubRecordsArchiver::Repository) 13 | expect(subject.repository.full_name).to eql(repo) 14 | end 15 | 16 | it 'stores the ID' do 17 | expect(subject.id).to eql(id) 18 | end 19 | 20 | context 'when given a repository' do 21 | subject do 22 | repository = GitHubRecordsArchiver::Repository.new(repo) 23 | described_class.new(repository, id) 24 | end 25 | 26 | it 'stores the repo' do 27 | expect(subject.repository).to be_a(GitHubRecordsArchiver::Repository) 28 | expect(subject.repository.full_name).to eql(repo) 29 | end 30 | end 31 | 32 | context '#from_hash' do 33 | let(:hash) do 34 | { number: id } 35 | end 36 | 37 | subject { described_class.from_hash(repo, hash) } 38 | 39 | it 'store the number' do 40 | expect(subject.number).to eql(id) 41 | end 42 | 43 | it 'stores the data' do 44 | expect(subject.data[:number]).to eql(id) 45 | end 46 | end 47 | 48 | it 'returns metadata' do 49 | expect(subject.data).to be_a(Sawyer::Resource) 50 | expect(subject.data.body).to eql('Me too') 51 | end 52 | 53 | it 'converts to a string' do 54 | expected = "@balter-test-org at 2011-04-14 16:00:49 UTC wrote:\n\nMe too" 55 | expect(subject.to_s).to eql(expected) 56 | end 57 | end 58 | -------------------------------------------------------------------------------- /lib/github_records_archiver.rb: -------------------------------------------------------------------------------- 1 | $LOAD_PATH.unshift(File.dirname(__FILE__)) 2 | 3 | require 'yaml' 4 | require 'json' 5 | require 'logger' 6 | require 'fileutils' 7 | require 'open3' 8 | require 'thor' 9 | require 'octokit' 10 | require 'parallel' 11 | require 'dotenv/load' 12 | 13 | Octokit.auto_paginate = true 14 | 15 | module GitHubRecordsArchiver 16 | autoload :DataHelper, 'github_records_archiver/data_helper' 17 | autoload :Comment, 'github_records_archiver/comment' 18 | autoload :CLI, 'github_records_archiver/cli' 19 | autoload :GitRepository, 'github_records_archiver/git_repository' 20 | autoload :Issue, 'github_records_archiver/issue' 21 | autoload :Organization, 'github_records_archiver/organization' 22 | autoload :Repository, 'github_records_archiver/repository' 23 | autoload :Team, 'github_records_archiver/team' 24 | autoload :User, 'github_records_archiver/user' 25 | autoload :VERSION, 'github_records_archiver/version' 26 | autoload :Wiki, 'github_records_archiver/wiki' 27 | 28 | class << self 29 | attr_writer :token, :dest_dir, :verbose, :shell, :client 30 | 31 | def token 32 | @token ||= ENV['GITHUB_TOKEN'] 33 | end 34 | 35 | def client 36 | @client ||= Octokit::Client.new access_token: token 37 | end 38 | 39 | def dest_dir 40 | @dest_dir ||= File.expand_path('./archive', Dir.pwd) 41 | end 42 | 43 | def verbose 44 | @verbose ||= false 45 | end 46 | alias verbose? verbose 47 | 48 | def shell 49 | @shell ||= Thor::Base.shell.new 50 | end 51 | 52 | def verbose_status(status, message, color = :white) 53 | return unless verbose? 54 | shell.say_status status, remove_token(message), color 55 | end 56 | 57 | def remove_token(string) 58 | string.gsub(GitHubRecordsArchiver.token, '') 59 | end 60 | end 61 | end 62 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'coveralls' 2 | Coveralls.wear! 3 | 4 | require 'webmock' 5 | require 'webmock/rspec' 6 | require 'addressable/uri' 7 | require_relative '../lib/github_records_archiver' 8 | 9 | ENV['GITHUB_TOKEN'] = 'TEST_TOKEN' 10 | 11 | RSpec.configure do |config| 12 | config.example_status_persistence_file_path = 'spec/examples.txt' 13 | config.disable_monkey_patching! 14 | config.warnings = true 15 | config.order = :random 16 | 17 | config.default_formatter = 'doc' if config.files_to_run.one? 18 | 19 | Kernel.srand config.seed 20 | WebMock.disable_net_connect! 21 | end 22 | 23 | def with_env(key, value) 24 | old_env = ENV[key] 25 | ENV[key] = value 26 | yield 27 | ENV[key] = old_env 28 | end 29 | 30 | def fixture_dir 31 | File.join(__dir__, 'fixtures') 32 | end 33 | 34 | def fixture_path(fixture) 35 | File.join(fixture_dir, "#{fixture}.json") 36 | end 37 | 38 | def fixture_contents(fixture) 39 | path = fixture_path(fixture) 40 | raise "Missing fixture for '#{fixture}'" unless File.exist?(path) 41 | File.read(path) 42 | end 43 | 44 | def stub_api_request(fixture, args = nil) 45 | uri = Addressable::URI.join('https://api.github.com', fixture) 46 | uri.query_values = args if args 47 | stub_request(:get, uri).to_return( 48 | status: 200, 49 | body: fixture_contents(fixture), 50 | headers: { 'Content-Type' => 'application/json' } 51 | ) 52 | end 53 | 54 | def capture(stream) 55 | case stream 56 | when :stdout 57 | $stdout = StringIO.new 58 | when :stderr 59 | $stderr = StringIO.new 60 | end 61 | 62 | yield 63 | 64 | output = case stream 65 | when :stdout 66 | $stdout.string 67 | when :stderr 68 | $stderr.string 69 | end 70 | 71 | output 72 | ensure 73 | case stream 74 | when :stdout 75 | $stdout = STDOUT 76 | when :stderr 77 | $stderr = STDERR 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /.github/config.yml: -------------------------------------------------------------------------------- 1 | # Behaviorbot config. See https://github.com/behaviorbot/ for more information. 2 | # Note: Please Don't edit this file directly. 3 | # Edit https://github.com/benbalter/shared-community-files instead. 4 | 5 | # Configuration for update-docs - https://github.com/behaviorbot/update-docs 6 | updateDocsComment: "Thanks for the pull request! If you are making any changes to the user-facing functionality, please be sure to update the documentation in the `README` or `docs/` folder alongside your change. :heart:" 7 | 8 | # Configuration for request-info - https://github.com/behaviorbot/request-info 9 | requestInfoReplyComment: Thanks for this. Do you mind providing a bit more information about what problem you're trying to solve? 10 | requestInfoLabelToAdd: more-information-needed 11 | 12 | # Configuration for new-issue-welcome - https://github.com/behaviorbot/new-issue-welcome 13 | #newIssueWelcomeComment: > 14 | # Welcome! 15 | 16 | # Configuration for new-pr-welcome - https://github.com/behaviorbot/new-pr-welcome 17 | newPRWelcomeComment: Welcome! Congrats on your first pull request to GitHub Records Archiver. If you haven't already, please be sure to check out [the contributing guidelines](https://github.com/benbalter/github-records-archiver/blob/master/docs/CONTRIBUTING.md). 18 | 19 | # Configuration for first-pr-merge - https://github.com/behaviorbot/first-pr-merge 20 | firstPRMergeComment: "Congrats on getting your first pull request to GitHub Records Archiver merged! Without amazing humans like you submitting pull requests, we couldn’t run this project. You rock! :tada:

If you're interested in tackling another bug or feature, take a look at [the open issues](https://github.com/benbalter/github-records-archiver/issues), especially those [labeled `help wanted`](https://github.com/benbalter/github-records-archiver/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)." 21 | 22 | # Bug workaround 23 | contact_links: [] 24 | -------------------------------------------------------------------------------- /lib/github_records_archiver/git_repository.rb: -------------------------------------------------------------------------------- 1 | module GitHubRecordsArchiver 2 | class GitError < StandardError; end 3 | 4 | class GitRepository 5 | def clone 6 | # Repo already exists, just pull new objects 7 | if Dir.exist? File.join(repo_dir, '.git') 8 | Dir.chdir repo_dir do 9 | git 'pull' 10 | end 11 | else 12 | # Clone Git content from scratch 13 | git 'clone', clone_url, repo_dir 14 | end 15 | end 16 | 17 | def repo_dir 18 | raise 'Not implemented' 19 | end 20 | 21 | private 22 | 23 | def clone_url 24 | raise 'Not implemented' 25 | end 26 | 27 | # There's a bug, whereby if you attempt to clone a wiki that's enabled 28 | # but has not yet been initialized, GitHub returns a remote error 29 | # Rather than let this break the export, capture the error and continue 30 | def wiki_does_not_exist?(output) 31 | expected = '^fatal: remote error: access denied or repository not ' 32 | expected << "exported: .*?\.wiki\.git$" 33 | output =~ /#{expected}/ 34 | end 35 | 36 | # Attempting to clone an empty repo will rightfulyl fail at the Git level 37 | # But we shouldn't let that fail the archive operation 38 | def empty_repo?(output) 39 | expected = 'Your configuration specifies to merge with the ref ' 40 | expected << "'refs/heads/master'\n" 41 | expected << 'from the remote, but no such ref was fetched.' 42 | output =~ Regexp.new(expected) 43 | end 44 | 45 | # Run a git command, piping output to stdout 46 | def git(*args) 47 | output, status = Open3.capture2e('git', *args) 48 | cmd = "git #{args.join(' ')}" 49 | cmd << " in #{Dir.pwd}" if args == ['pull'] 50 | GitHubRecordsArchiver.verbose_status 'Git command:', cmd 51 | return false if empty_repo?(output) || wiki_does_not_exist?(output) 52 | if status.exitstatus != 0 53 | output = GitHubRecordsArchiver.remove_token(output) 54 | GitHubRecordsArchiver.shell.say_status 'Git Error', output, :red 55 | end 56 | output 57 | end 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /lib/github_records_archiver/issue.rb: -------------------------------------------------------------------------------- 1 | module GitHubRecordsArchiver 2 | class Issue 3 | attr_reader :repository 4 | attr_reader :number 5 | 6 | include DataHelper 7 | 8 | KEYS = %i[title number state html_url created_at closed_at].freeze 9 | 10 | def initialize(repository: nil, number: nil) 11 | repository = Repository.new(repository) if repository.is_a? String 12 | @repository = repository 13 | @number = number 14 | end 15 | 16 | def self.from_hash(repo, hash) 17 | issue = Issue.new(repository: repo, number: hash[:number]) 18 | issue.instance_variable_set('@data', hash.to_h) 19 | issue 20 | end 21 | 22 | def data 23 | @data ||= GitHubRecordsArchiver.client.issue repository.name, number 24 | end 25 | 26 | def comments 27 | @comments ||= begin 28 | return [] if data[:comments].nil? || data[:comments].zero? 29 | client = GitHubRecordsArchiver.client 30 | comments = client.issue_comments repository.full_name, number 31 | comments.map { |hash| Comment.from_hash(repository, hash) } 32 | end 33 | end 34 | 35 | def to_s 36 | md = meta_for_markdown.to_yaml + "---\n\n# #{title}\n\n" 37 | md << body unless body.to_s.empty? 38 | md << comments_string unless comments.nil? 39 | md 40 | end 41 | 42 | def as_json 43 | data.to_h.merge('comments' => comments.map(&:as_json)) 44 | end 45 | 46 | def archive 47 | File.write(path('md'), to_s) 48 | File.write(path('json'), to_json) 49 | end 50 | 51 | private 52 | 53 | def path(ext = 'md') 54 | File.expand_path "#{number}.#{ext}", repository.issues_dir 55 | end 56 | 57 | def meta_for_markdown 58 | meta = {} 59 | KEYS.each { |key| meta[key.to_s] = data[key] } 60 | meta['user'] = user[:login] 61 | meta['assignee'] = assignee[:login] unless assignee.nil? 62 | meta['tags'] = labels.map { |tag| tag[:name] } 63 | meta 64 | end 65 | 66 | def comments_string 67 | "\n\n---\n" + comments.map(&:to_s).join("\n\n---\n") 68 | end 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /lib/github_records_archiver/cli.rb: -------------------------------------------------------------------------------- 1 | module GitHubRecordsArchiver 2 | class CLI < Thor 3 | package_name 'GitHub Records Archiver' 4 | class_option :dest_dir, type: :string, required: false, 5 | desc: 'The destination directory for the archive', default: GitHubRecordsArchiver.dest_dir 6 | class_option :token, type: :string, required: false, desc: 'A GitHub personal access token with repo scope' 7 | class_option :verbose, type: :boolean, desc: 'Display verbose output while archiving', default: false 8 | 9 | desc 'version', 'Outputs the GitHubRecordsArchiver version' 10 | def version 11 | say "GitHub Records Archiver v#{GitHubRecordsArchiver::VERSION}" 12 | end 13 | 14 | desc 'delete [ORGANIZATION]', 'Deletes all archives, or the archive for an organization' 15 | option :force, type: :boolean, desc: 'Delete without prompting', default: false 16 | def delete(org_name = nil) 17 | path = GitHubRecordsArchiver.dest_dir 18 | path = File.join path, org_name if org_name 19 | FileUtils.rm_rf(path) if options[:force] || yes?("Are you sure? Remove #{path}?") 20 | end 21 | 22 | desc 'archive ORGANIZATION', 'Create or update archive for the given organization' 23 | def archive(org_name) 24 | start_time # Memoize start time for comparison 25 | @org_name = org_name 26 | 27 | GitHubRecordsArchiver.shell = shell 28 | %i[token dest_dir verbose].each do |option| 29 | next unless options[option] 30 | GitHubRecordsArchiver.public_send "#{option}=".to_sym, options[option] 31 | end 32 | 33 | say "Starting archive for @#{org.name} in #{org.archive_dir}" 34 | shell.indent(2) do 35 | archive_teams 36 | archive_repos 37 | end 38 | say "Done in #{Time.now - start_time} seconds.", :green 39 | end 40 | 41 | private 42 | 43 | def start_time 44 | @start_time ||= Time.now 45 | end 46 | 47 | def organization 48 | @organization ||= GitHubRecordsArchiver::Organization.new @org_name 49 | end 50 | alias org organization 51 | 52 | def archive_teams 53 | say_status 'Teams found:', org.teams.count, :white 54 | Parallel.each(org.teams, progress: 'Archiving teams', &:archive) 55 | end 56 | 57 | def archive_repos 58 | say_status 'Repositories found:', org.repos.count, :white 59 | 60 | Parallel.each(org.repos, progress: 'Archiving repos') do |repo| 61 | begin 62 | repo.clone 63 | repo.wiki.clone if repo.has_wiki? 64 | Parallel.each(repo.issues, &:archive) 65 | rescue GitHubRecordsArchiver::GitError => e 66 | say "Failed to archive #{repo.name}", :red 67 | say e.message, :red 68 | end 69 | end 70 | end 71 | end 72 | end 73 | -------------------------------------------------------------------------------- /spec/github_records_archiver/cli_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe GitHubRecordsArchiver::CLI do 2 | let(:command) { :help } 3 | let(:args) { [] } 4 | let(:output) { capture(:stdout) { subject.public_send(command, *args) } } 5 | 6 | it 'displays help output' do 7 | expect(output).to match('GitHub Records Archiver commands:') 8 | end 9 | 10 | context 'version' do 11 | let(:command) { :version } 12 | 13 | it 'displays the version' do 14 | expect(output).to match(/GitHub Records Archiver v\d\.\d\.\d/) 15 | end 16 | end 17 | 18 | context 'delete' do 19 | let(:command) { :delete } 20 | let(:dest_dir) { GitHubRecordsArchiver.dest_dir } 21 | 22 | before do 23 | FileUtils.mkdir dest_dir unless Dir.exist?(dest_dir) 24 | subject.options = subject.options.merge(force: true) 25 | end 26 | 27 | it 'deletes the archive' do 28 | expect(Dir.exist?(dest_dir)).to be_truthy 29 | subject.public_send(command, *args) 30 | expect(Dir.exist?(dest_dir)).to be_falsy 31 | end 32 | 33 | context 'when given an organization' do 34 | let(:args) { ['org'] } 35 | let(:path) { File.join(dest_dir, 'org') } 36 | before do 37 | FileUtils.mkdir path unless Dir.exist?(path) 38 | end 39 | 40 | it 'deletes the org' do 41 | expect(Dir.exist?(path)).to be_truthy 42 | subject.public_send(command, *args) 43 | expect(Dir.exist?(path)).to be_falsy 44 | expect(Dir.exist?(dest_dir)).to be_truthy 45 | end 46 | end 47 | 48 | context 'archiving' do 49 | before do 50 | stub_api_request('orgs/balter-test-org') 51 | stub_api_request('orgs/balter-test-org/repos', per_page: 100) 52 | stub_api_request('orgs/balter-test-org/teams', per_page: 100) 53 | stub_api_request('teams/1') 54 | stub_api_request('teams/1/repos', per_page: 100) 55 | stub_api_request('teams/1/members', per_page: 100) 56 | stub_api_request('repos/balter-test-org/some-repo') 57 | stub_api_request('repos/balter-test-org/some-repo/issues/1') 58 | stub_api_request('repos/balter-test-org/some-repo/issues/1/comments', per_page: 100) 59 | stub_api_request('repos/balter-test-org/some-repo/issues/comments/1', per_page: 100) 60 | stub_api_request('repos/balter-test-org/some-repo/issues', per_page: 100, state: 'all') 61 | stub_api_request 'users/benbalter' 62 | end 63 | 64 | let(:args) { ['balter-test-org'] } 65 | let(:command) { :archive } 66 | 67 | it 'archives' do 68 | expect(output).to match(/^Done/) 69 | files = [ 70 | 'balter-test-org/some-repo/issues/1347.json', 71 | 'balter-test-org/some-repo/issues/1347.md', 72 | 'balter-test-org/teams/justice-league.md' 73 | ] 74 | 75 | files.each do |file| 76 | path = File.join dest_dir, file 77 | expect(File.exist?(path)).to be_truthy, "Expected #{file} to exist" 78 | end 79 | end 80 | end 81 | end 82 | end 83 | -------------------------------------------------------------------------------- /spec/github_records_archiver/team_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe GitHubRecordsArchiver::Team do 2 | let(:id) { 1 } 3 | let(:org) { 'balter-test-org' } 4 | let(:slug) { 'justice-league' } 5 | let(:path) { subject.send(:path) } 6 | subject { described_class.new(org, id) } 7 | 8 | before do 9 | stub_api_request('orgs/balter-test-org') 10 | stub_api_request('teams/1') 11 | stub_api_request('teams/1/repos', per_page: 100) 12 | stub_api_request('teams/1/members', per_page: 100) 13 | end 14 | 15 | it 'stores the ID' do 16 | expect(subject.id).to eql(id) 17 | end 18 | 19 | it 'stores the org' do 20 | organization = GitHubRecordsArchiver::Organization.new(org) 21 | expect(subject.organization.id).to eql(organization.id) 22 | end 23 | 24 | context 'from a hash' do 25 | let(:hash) do 26 | { 27 | id: id 28 | } 29 | end 30 | 31 | subject { described_class.from_hash(org, hash) } 32 | 33 | it 'stores the ID' do 34 | expect(subject.id).to eql(id) 35 | end 36 | 37 | it 'store the org' do 38 | organization = GitHubRecordsArchiver::Organization.new(org) 39 | expect(subject.organization.id).to eql(organization.id) 40 | end 41 | 42 | it 'stores the metadata' do 43 | expect(subject.data).to eql(hash) 44 | end 45 | end 46 | 47 | it 'retrieves metadata' do 48 | expect(subject.data).to be_a(Sawyer::Resource) 49 | expect(subject.data.id).to eql(id) 50 | end 51 | 52 | it 'retrieves repos' do 53 | expect(subject.repositories).to be_an(Array) 54 | repo = subject.repositories.first 55 | expect(repo).to be_a(GitHubRecordsArchiver::Repository) 56 | end 57 | 58 | it 'retrives members' do 59 | expect(subject.members).to be_an(Array) 60 | user = subject.members.first 61 | expect(user).to be_a(GitHubRecordsArchiver::User) 62 | end 63 | 64 | it 'converts to a string' do 65 | expected = <