├── .editorconfig ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .travis.yml ├── .travis └── setup-database.sh ├── DATA_MODEL.md ├── LICENSE ├── Makefile ├── README.md ├── TODO.md ├── db ├── extension.sql ├── migrations │ ├── 1_initial_schema.sql │ ├── 20191012163928_add_column_shards.archived_at.sql │ ├── 20191102100059_upgrade_activity_log.sql │ ├── 20191106093828_add_constraints_for_repos.shard_id.sql │ ├── 20191115142944_change_releases.position_not_null.sql │ ├── 20191122122940_create_table_files.sql │ ├── 20200503132444_create_table_owners.sql │ └── 20200506215505_add_shards_merged_with.sql ├── schema-overview.svg └── schema.sql ├── scripts └── build_libgit2.sh ├── shard.lock ├── shard.yml ├── spec ├── catalog │ └── duplication_spec.cr ├── catalog_spec.cr ├── dependency_spec.cr ├── ext │ └── shards │ │ └── git_spec.cr ├── fetchers │ └── github_api_spec.cr ├── fixtures │ └── repo_metadata-github-graphql-response.json ├── owner_spec.cr ├── repo_metadata_spec.cr ├── repo_ref_spec.cr ├── service │ ├── create_owner_spec.cr │ ├── create_shard_spec.cr │ ├── fetch_metadata_spec.cr │ ├── import_catalog_spec.cr │ ├── import_categories_spec.cr │ ├── import_shard_spec.cr │ ├── order_releases_spec.cr │ ├── sync_dependencies_spec.cr │ ├── sync_release_spec.cr │ ├── sync_repo_spec.cr │ ├── update_dependencies_spec.cr │ ├── update_owner_metrics_spec.cr │ └── update_shard_metrics_spec.cr ├── shard_spec.cr ├── spec_helper.cr └── support │ ├── db.cr │ ├── factory.cr │ ├── fetcher_mocks.cr │ ├── mock_resolver.cr │ ├── raven.cr │ └── tempdir.cr └── src ├── catalog.cr ├── catalog ├── category.cr ├── duplication.cr ├── entry.cr └── mirror.cr ├── category.cr ├── db.cr ├── dependency.cr ├── ext ├── git │ └── repo.cr ├── shards │ └── resolvers │ │ └── git.cr └── yaml │ └── any.cr ├── fetchers ├── error.cr ├── github_api-owner_info.graphql ├── github_api-repo_metadata.graphql └── github_api.cr ├── log_activity.cr ├── raven.cr ├── release.cr ├── repo.cr ├── repo ├── owner.cr ├── ref.cr └── resolver.cr ├── service ├── create_owner.cr ├── create_shard.cr ├── fetch_metadata.cr ├── import_catalog.cr ├── import_categories.cr ├── import_shard.cr ├── order_releases.cr ├── sync_dependencies.cr ├── sync_release.cr ├── sync_repo.cr ├── sync_repos.cr ├── update_owner_metrics.cr ├── update_shard.cr ├── update_shard_metrics.cr └── worker_loop.cr ├── shard.cr ├── util └── software_version.cr └── worker.cr /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*.cr] 4 | charset = utf-8 5 | end_of_line = lf 6 | insert_final_newline = true 7 | indent_style = space 8 | indent_size = 2 9 | trim_trailing_whitespace = true 10 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | pull_request: 6 | branches: 7 | - master 8 | # Branches from forks have the form 'user:branch-name' so we only run 9 | # this job on pull_request events for branches that look like fork 10 | # branches. Without this we would end up running this job twice for non 11 | # forked PRs, once for the push and then once for opening the PR. 12 | - "**:**" 13 | schedule: 14 | - cron: '0 6 * * 1' # Every monday 6 AM 15 | 16 | jobs: 17 | test: 18 | strategy: 19 | fail-fast: false 20 | matrix: 21 | crystal: [latest, nightly] 22 | runs-on: ubuntu-latest 23 | 24 | services: 25 | postgres: 26 | image: postgres:12 27 | env: 28 | POSTGRES_PASSWORD: postgres 29 | POSTGRES_HOST_AUTH_METHOD: trust 30 | options: >- 31 | --health-cmd pg_isready 32 | --health-interval 10s 33 | --health-timeout 5s 34 | --health-retries 5 35 | ports: 36 | - 5432:5432 37 | env: 38 | TEST_DATABASE_URL: "postgres://postgres:postgres@localhost:5432/shardbox" 39 | PGHOST: localhost 40 | PGUSER: postgres 41 | DBMATE: "vendor/bin/dbmate" 42 | SHARDS_OPTS: "--ignore-crystal-version" 43 | 44 | steps: 45 | - name: Download source 46 | uses: actions/checkout@v2 47 | - name: Install Crystal 48 | uses: oprypin/install-crystal@v1 49 | with: 50 | crystal: ${{ matrix.crystal }} 51 | - name: Install dependencies 52 | run: | 53 | make vendor/bin/dbmate 54 | sudo apt-get install libgit2-dev 55 | - name: Configure git user 56 | run: | 57 | git config --global user.email "ci@shardbox.org" 58 | git config --global user.name "GitHub Actions" 59 | - name: Run unit tests 60 | run: make test 61 | - name: Run migration tests 62 | run: make test/migration TEST_DATABASE_URL=${TEST_DATABASE_URL}?sslmode=disable 63 | - name: Build worker 64 | run: make bin/worker 65 | - name: Run integration test 66 | run: bin/worker sync_repos && bin/worker update_metrics 67 | env: 68 | DATABASE_URL: "${{ env.TEST_DATABASE_URL }}?sslmode=disable" 69 | GITHUB_TOKEN: "${{ secrets.GithubToken }}" 70 | 71 | format: 72 | runs-on: ubuntu-latest 73 | steps: 74 | - name: Download source 75 | uses: actions/checkout@v2 76 | - name: Install Crystal 77 | uses: oprypin/install-crystal@v1 78 | with: 79 | crystal: latest 80 | - name: Check formatting 81 | run: crystal tool format; git diff --exit-code 82 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /docs/ 2 | /lib/ 3 | /bin/ 4 | /.shards/ 5 | *.dwarf 6 | .env 7 | /catalog/* 8 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: bionic 2 | language: crystal 3 | 4 | build_addons: &build_addons 5 | postgresql: "12" 6 | apt: 7 | packages: 8 | - postgresql-12 9 | - postgresql-client-12 10 | - libgit2-dev 11 | 12 | crystal: 13 | - latest 14 | 15 | cache: 16 | - shards 17 | - directories: 18 | - vendor/ 19 | 20 | env: 21 | global: 22 | - PGPORT: 5433 23 | - TEST_DATABASE_URL: "postgres://postgres:@localhost:5433/shardbox" 24 | - DBMATE: "vendor/bin/dbmate" 25 | - SHARDS_OPTS: "--ignore-crystal-version" 26 | 27 | jobs: 28 | include: 29 | - stage: test 30 | name: unit tests 31 | services: 32 | - postgresql 33 | addons: *build_addons 34 | before_script: ./.travis/setup-database.sh 35 | script: make test 36 | - stage: test 37 | name: migration test 38 | addons: *build_addons 39 | before_script: ./.travis/setup-database.sh 40 | script: make test_db test/migration 41 | - stage: test 42 | name: integration test 43 | addons: *build_addons 44 | before_script: ./.travis/setup-database.sh 45 | script: 46 | - make test_db bin/worker 47 | - | 48 | export DATABASE_URL=$TEST_DATABASE_URL 49 | bin/worker sync_repos && bin/worker update_metrics 50 | - stage: test 51 | name: format 52 | script: crystal tool format --check src spec 53 | -------------------------------------------------------------------------------- /.travis/setup-database.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | set -e 3 | 4 | sudo sed -i -e '/local.*peer/s/postgres/all/' -e 's/peer\|md5/trust/g' /etc/postgresql/*/main/pg_hba.conf 5 | # for some reason this command returns failure code but still succeeds... 6 | sudo service postgresql restart || true 7 | psql -U postgres -c 'SELECT version()' 8 | 9 | make vendor/bin/dbmate 10 | -------------------------------------------------------------------------------- /DATA_MODEL.md: -------------------------------------------------------------------------------- 1 | # ShardsDB Data Model 2 | 3 | All relevant information is stored in a PostgreSQL database. 4 | 5 | The database is the single source of truth and contains a pronounced data model 6 | which ensures consistency of values and references inside the database. Thus the 7 | database is independent of validations provided by ORM. Constraints and triggers 8 | are in place to make sure the data is inherently consistent, regardless of how it 9 | is accessed. 10 | 11 | ## Shards 12 | 13 | ``` 14 | Table "public.shards" 15 | Column | Type | Collation | Nullable | Default 16 | -------------+--------------------------+-----------+----------+---------------------------------- 17 | id | bigint | | not null | generated by default as identity 18 | name | citext | | not null | 19 | qualifier | citext | | not null | ''::citext 20 | description | text | | | 21 | created_at | timestamp with time zone | | not null | now() 22 | updated_at | timestamp with time zone | | not null | now() 23 | categories | bigint[] | | not null | '{}'::bigint[] 24 | archived_at | timestamp with time zone | | | 25 | Check constraints: 26 | "shards_name_check" CHECK (name ~ '^[A-Za-z0-9_\-.]{1,100}$'::text) 27 | "shards_qualifier_check" CHECK (qualifier ~ '^[A-Za-z0-9_\-.]{0,100}$'::citext) 28 | ``` 29 | 30 | ### Simple, fault-tolerant, global naming schema for shards 31 | 32 | Shards are generally called by their name. The name is defined in `shard.yml` and also 33 | applied when used as a dependency both in `dependencies:` mapping as well as the folder 34 | name in `./lib`. 35 | 36 | Because of shard's decentralized design, name clashes can't be ruled out. 37 | 38 | #### Mirrors 39 | 40 | Name clashes are often caused by mirrors of a shard's repository. This is usually 41 | not that problematic because they can be viewed as alternate repositories of the same shard. 42 | 43 | Example: kemal 44 | 45 | * Main repo location: `github:kemalcr/kemal`. This is considered the canonical repo. 46 | * Old repo location: `github:sdogruyol/kemal` 47 | * There are also development forks like `github:straight-shoota/kemal` 48 | 49 | They all reference the same shard. Mirrors could be considered as 50 | individual instances but unless they have separate releases, they are considered 51 | the same shard, just provided at a different location. When a mirror releases 52 | independently, it must be considered a fork and a separate shard. 53 | 54 | Shardbox doesn't take care of this automatically. Mirrors need to be listed as 55 | mirrors belonging to the canonical repository in the catalog. Shardbox only every 56 | synchronizes releases from the canonical repository. To separate a mirror to form its 57 | own shard, it simply needs to be removed from the mirrors list. 58 | 59 | ##### Homonymous Shards 60 | 61 | There might also be unrelated shards which share the same name. 62 | This is obviously not ideal, but can't really be avoided when there is no 63 | centralized registry that assigns names. 64 | 65 | This problem is approached as follows: 66 | 67 | * Shards are generally identified by their name as specified in `shard.yml` (e.g. `kemal`). 68 | * When there are multiple shards of the same name (related or unrelated) in the database, an 69 | additional qualifier is used to tell them apart (e.g. `kemalcr` or `straight-shoota`). 70 | This qualifier is interpreted as `#{qualifier}'s version of #{name}` 71 | * Qualifiers can be omitted when there is no ambiguity. 72 | * Name and qualifier combined form a slug which could look like `kemal` (no qualifier) or 73 | `kemal~straight-shoota`. 74 | * Avoids `/` as delimiter for easier use in HTTP routes and to distinguish from 75 | `/` scheme typically used for source code hosting. 76 | 77 | Shardbox automatically assigns qualifiers derived from the URL. 78 | Qualifiers are only assigned when a shard name has already been taken 79 | (first come – first serve). 80 | There is currently no mechanism for modifying qualifiers (but might be implemented in the future). 81 | 82 | ## Repos 83 | 84 | ``` 85 | Table "public.repos" 86 | Column | Type | Collation | Nullable | Default 87 | ----------------+--------------------------+-----------+----------+---------------------------------- 88 | id | bigint | | not null | generated by default as identity 89 | shard_id | bigint | | | 90 | resolver | repo_resolver | | not null | 91 | url | citext | | not null | 92 | role | repo_role | | not null | 'canonical'::repo_role 93 | metadata | jsonb | | not null | '{}'::jsonb 94 | sync_failed_at | timestamp with time zone | | | 95 | synced_at | timestamp with time zone | | | 96 | created_at | timestamp with time zone | | not null | now() 97 | updated_at | timestamp with time zone | | not null | now() 98 | Check constraints: 99 | "repos_obsolete_role_shard_id_null" CHECK (role <> 'obsolete'::repo_role OR shard_id IS NULL) 100 | "repos_resolvers_service_url" CHECK (NOT (resolver = ANY (ARRAY['github'::repo_resolver, 'gitlab'::repo_resolver, 'bitbucket'::repo_resolver])) OR url ~ '^[A-Za-z0-9_\-.]{1,100}/[A-Za-z0-9_\-.]{1,100}$'::citext AND url !~~ '%.git'::citext) 101 | "repos_shard_id_null_role" CHECK (shard_id IS NOT NULL OR role = 'canonical'::repo_role OR role = 'obsolete'::repo_role) 102 | ``` 103 | 104 | NOTES: 105 | 106 | * `role` specifies the role of this repo for the shard (defaults to `canonical`). Other values are `mirror`, `legacy` and `obsolete`. Thus, multiple repositories can be linked to the same shard. This is important for example to preserve continuity when a repository is transferred to a different location (for example `github:sdogruyol/kemal` to `github:kemalcr/kemal`) and to map mirrors of the same shard. 107 | 108 | ### Releases 109 | 110 | ``` 111 | Table "public.releases" 112 | Column | Type | Collation | Nullable | Default 113 | ---------------+--------------------------+-----------+----------+---------------------------------- 114 | id | bigint | | not null | generated by default as identity 115 | shard_id | bigint | | not null | 116 | version | character varying | | not null | 117 | revision_info | jsonb | | not null | 118 | spec | jsonb | | not null | 119 | position | integer | | not null | 120 | latest | boolean | | | 121 | released_at | timestamp with time zone | | not null | 122 | yanked_at | timestamp with time zone | | | 123 | created_at | timestamp with time zone | | not null | now() 124 | updated_at | timestamp with time zone | | not null | now() 125 | Check constraints: 126 | "releases_latest_check" CHECK (latest <> false) 127 | "releases_version_check" CHECK (version::text ~ '^[0-9]+(\.[0-9a-zA-Z]+)*(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$'::text OR version::text = 'HEAD'::text) 128 | ``` 129 | 130 | NOTES: 131 | 132 | * Releases are bound to a shard (`shard_id`), not an individual repo because repo locations may change. We consider each shard to have a unique release history determined by the releases provided by the canonical repo. 133 | * `position` is a utility column used to sort versions because PostgreSQL doesn't provide proper comparison operator for version strings. There is a `semver` extension, but it requires versions to follow SEMVER, which is not enforced by shards. So we need to implement sort order externally using `Service::OrderReleases`. As a benefit, result sorting is simple integer comparison. 134 | * If a release has been deleted from the repo (i.e. the tag was removed) it is marked as `yanked`. This procedure needs refinement. Yanked releases should still be addressable. 135 | * When a tag is changed to point to a different commit, it is simply updated. This also needs refinement. 136 | 137 | ### Dependencies 138 | 139 | ``` 140 | Table "public.dependencies" 141 | Column | Type | Collation | Nullable | Default 142 | ------------+--------------------------+-----------+----------+--------- 143 | release_id | bigint | | not null | 144 | name | citext | | not null | 145 | repo_id | bigint | | | 146 | spec | jsonb | | not null | 147 | scope | dependency_scope | | not null | 148 | created_at | timestamp with time zone | | not null | now() 149 | updated_at | timestamp with time zone | | not null | now() 150 | ``` 151 | 152 | NOTES: 153 | 154 | * `repo_id` points to the repo referenced as dependency. This is only `NULL` if it cannot be resolved every (`path` scheme). 155 | In all other cases, `repo_id` points to a repository record, but might not be actually resolvable (when the repository is not available). 156 | * The dependent shard is available through joining `releases` on `release_id`. 157 | * Scope is either `runtime` or `dependency`. 158 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018-2019 Johannes Müller 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | -include Makefile.local # for optional local options 2 | 3 | BUILD_TARGET := bin/worker 4 | 5 | # The dbmate command to use 6 | DBMATE ?= dbmate 7 | # The shards command to use 8 | SHARDS ?= shards 9 | # The crystal command to use 10 | CRYSTAL ?= crystal 11 | 12 | SRC_SOURCES := $(shell find src -name '*.cr' 2>/dev/null) 13 | LIB_SOURCES := $(shell find lib -name '*.cr' 2>/dev/null) 14 | SPEC_SOURCES := $(shell find spec -name '*.cr' 2>/dev/null) 15 | 16 | DATABASE_NAME := $(shell echo $(DATABASE_URL) | grep -o -P '[^/]+$$') 17 | TEST_DATABASE_NAME := $(shell echo $(TEST_DATABASE_URL) | grep -o -P '[^/]+$$') 18 | 19 | .PHONY: build 20 | build: ## Build the application binary 21 | build: $(BUILD_TARGET) 22 | 23 | $(BUILD_TARGET): $(SRC_SOURCES) $(LIB_SOURCES) lib 24 | mkdir -p $(shell dirname $(@)) 25 | $(CRYSTAL) build src/worker.cr -o $(@) 26 | 27 | .PHONY: test 28 | test: ## Run the test suite 29 | test: lib test_db 30 | $(CRYSTAL) spec 31 | 32 | .PHONY: format 33 | format: ## Apply source code formatting 34 | format: $(SRC_SOURCES) $(SPEC_SOURCES) 35 | $(CRYSTAL) tool format src spec 36 | 37 | docs: ## Generate API docs 38 | docs: $(SRC_SOURCES) lib 39 | $(CRYSTAL) docs -o docs 40 | 41 | lib: shard.lock 42 | $(SHARDS) install 43 | 44 | shard.lock: shard.yml 45 | $(SHARDS) update 46 | 47 | .PHONY: DATABASE_URL 48 | DATABASE_URL: 49 | @test "${$@}" || (echo "$@ is undefined" && false) 50 | 51 | .PHONY: TEST_DATABASE_URL 52 | TEST_DATABASE_URL: 53 | @test "${$@}" || (echo "$@ is undefined" && false) 54 | 55 | .PHONY: test_db 56 | test_db: TEST_DATABASE_URL 57 | @psql $(TEST_DATABASE_NAME) -c "SELECT 1" > /dev/null 2>&1 || \ 58 | (createdb $(TEST_DATABASE_NAME) && psql $(TEST_DATABASE_NAME) < db/extension.sql && psql $(TEST_DATABASE_URL) < db/schema.sql) 59 | 60 | .PHONY: db 61 | db: DATABASE_URL 62 | @psql $(DATABASE_NAME) -c "SELECT 1" > /dev/null 2>&1 || \ 63 | createdb $(DATABASE_NAME) 64 | psql $(DATABASE_NAME) -c "CREATE EXTENSION IF NOT EXISTS citext WITH SCHEMA public;" 65 | $(DBMATE) up 66 | 67 | .PHONY: db/dump 68 | db/dump: DATABASE_URL 69 | pg_dump -d $(DATABASE_NAME) -a -Tschema_migrations --disable-triggers > db/dump/$(shell date +'%Y-%m-%d-%H%M').sql 70 | 71 | .PHONY: db/dump_schema 72 | db/dump_schema: DATABASE_URL 73 | pg_dump -s $(DATABASE_NAME) > db/schema.sql 74 | 75 | .PHONY: test_db/drop_sync 76 | test_db/drop_sync: test_db/drop 77 | createdb $(TEST_DATABASE_NAME) 2> /dev/null 78 | pg_dump -s $(DATABASE_NAME) | psql $(TEST_DATABASE_NAME) -q 79 | 80 | .PHONY: test_db/drop 81 | test_db/drop: 82 | dropdb $(TEST_DATABASE_NAME) || true 83 | 84 | .PHONY: test/migration 85 | test/migration: 86 | git add db/schema.sql 87 | make test_db/rollback test_db/migrate 88 | git diff --exit-code db/schema.sql 89 | 90 | .PHONY: test_db/migrate 91 | test_db/migrate: 92 | $(DBMATE) -e TEST_DATABASE_URL migrate 93 | 94 | .PHONY: test_db/rollback 95 | test_db/rollback: 96 | $(DBMATE) -e TEST_DATABASE_URL rollback 97 | 98 | .PHONY: db/migrate 99 | db/migrate: 100 | $(DBMATE) -e DATABASE_URL migrate 101 | 102 | .PHONY: db/rollback 103 | db/rollback: 104 | $(DBMATE) -e DATABASE_URL rollback 105 | 106 | vendor/bin/dbmate: 107 | mkdir -p vendor/bin 108 | wget -qO "$@" https://github.com/amacneil/dbmate/releases/download/v1.7.0/dbmate-linux-amd64 109 | chmod +x "$@" 110 | 111 | .PHONY: clean 112 | clean: ## Remove application binary 113 | clean: 114 | @rm -f $(BUILD_TARGET) 115 | 116 | .PHONY: help 117 | help: ## Show this help 118 | @echo 119 | @printf '\033[34mtargets:\033[0m\n' 120 | @grep -hE '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) |\ 121 | sort |\ 122 | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-15s\033[0m %s\n", $$1, $$2}' 123 | @echo 124 | @printf '\033[34moptional variables:\033[0m\n' 125 | @grep -hE '^[a-zA-Z_-]+ \?=.*?## .*$$' $(MAKEFILE_LIST) |\ 126 | sort |\ 127 | awk 'BEGIN {FS = " \\?=.*?## "}; {printf " \033[36m%-15s\033[0m %s\n", $$1, $$2}' 128 | @echo 129 | @printf '\033[34mrecipes:\033[0m\n' 130 | @grep -hE '^##.*$$' $(MAKEFILE_LIST) |\ 131 | awk 'BEGIN {FS = "## "}; /^## [a-zA-Z_-]/ {printf " \033[36m%s\033[0m\n", $$2}; /^## / {printf " %s\n", $$2}' 132 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # shardbox-core 2 | 3 | Shards database, collecting shard repositories, releases and dependencies. 4 | 5 | ## Installation 6 | 7 | ### Prerequisites 8 | 9 | * Crystal (1.0.0 or later) 10 | * PostgreSQL database (version 12) 11 | * [dbmate](https://github.com/amacneil/dbmate) for database migrations 12 | 13 | #### Database 14 | PostgreSQL databases needs to be created manually. 15 | 16 | Connection configuration is read from environment variables. 17 | 18 | * development database: `DATABASE_URL` 19 | * testing database: `TEST_DATABASE_URL` (only for running tests) 20 | 21 | Install database schema: 22 | 23 | * `make db` - development database 24 | * `make test_db` - test database 25 | 26 | ### Shard Dependencies 27 | 28 | Run `shards install` to install dependencies. 29 | 30 | ## Usage 31 | 32 | Run `make bin/worker` to build the worker executable. 33 | 34 | * `bin/worker import_catalog https://github.com/shardbox/catalog`: 35 | Reads catalog description from the catalog repository and imports shards to the database. 36 | * `bin/worker sync_repos`: Synchronizes data from git repositories to database for all 37 | repositories not synced in the last 24 hours or never synced at all. 38 | Recommended to be run every hour. 39 | * `bin/worker updated_metrics`: Updates shard metrics in the database. Should be run once per day. 40 | * `bin/worker loop` starts a worker loop which schedules regular `sync_repos` and `update_metrics` 41 | jobs. It also listens to notifications sent through PostgreSQL notify channels in order 42 | to trigger catalog import. 43 | 44 | A web application for browsing the database is available at https://github.com/shardbox/shardbox-web 45 | 46 | ## Development 47 | 48 | Run `make test` to execute the spec suite. 49 | 50 | ## Contributing 51 | 52 | 1. Fork it () 53 | 2. Create your feature branch (`git checkout -b my-new-feature`) 54 | 3. Commit your changes (`git commit -am 'Add some feature'`) 55 | 4. Push to the branch (`git push origin my-new-feature`) 56 | 5. Create a new Pull Request 57 | 58 | ## Contributors 59 | 60 | * [Johannes Müller](https://github.com/straight-shoota) - creator and maintainer 61 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | # TODO 2 | 3 | * Resolve repo redirects: URLs (including Github etc. are not necessarily normalized). 4 | Example: https://github.com/luckyframework/cli -> https://github.com/luckyframework/lucky_cli 5 | 6 | ## Name 7 | 8 | * Shardarium 9 | * Shardotheque 10 | * Shard Collection 11 | 12 | # Open Questions 13 | 14 | 15 | ## Simple, fault-tolerant, global naming schema for shards. 16 | 17 | Shards should generally be known by their name. But name clashes need to be resolved. 18 | 19 | Example: kemal 20 | 21 | * Main repo: github:kemalcr/kemal 22 | * Alternative old repo: github:sdogruyol/kemal 23 | * There are also forks referenced as dependencies (for example github:matomo/kemal) 24 | 25 | Suggestion: 26 | 27 | * Shards are identified by name (e.g. `kemal`) and qualifier (e.g. `kemalcr`, `matomo`) 28 | * Qualifier can be omitted for "main" shard (usually the first one registered). This is either implemented by a flag or nilable qualifier. 29 | * Slug could look like `kemal` (main shard `kemalcr/kemal`) and `kemal~matomo` (fork `matomo/kemal`) 30 | * Avoids `/` as delimiter for easier use in HTTP routes and to distinguish from github `/` scheme. 31 | * Need to determine if this nomenclature should only be used for forks or also for entirely different shards, just sharing the same name. This is obviously bad by itself, but can't be avoided without a centralized registry. It's probably hard to tell these two cases apart. In case of unrelated name clas, we might consider to always require a qualifier. 32 | 33 | ## How to express unresolvable dependencies (i.e. local `path` dependencies?) 34 | 35 | Some publicly hosted gems have local `path` dependencies. They can't be resolved. 36 | 37 | Dependencies should be listed -> needs special entries for dependencies table. 38 | 39 | Shards should be marked as having non-resolvable dependencies. 40 | -------------------------------------------------------------------------------- /db/extension.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Name: citext; Type: EXTENSION; Schema: -; Owner: - 3 | -- 4 | 5 | CREATE EXTENSION IF NOT EXISTS citext WITH SCHEMA public; 6 | 7 | 8 | -- 9 | -- Name: EXTENSION citext; Type: COMMENT; Schema: -; Owner: - 10 | -- 11 | 12 | COMMENT ON EXTENSION citext IS 'data type for case-insensitive character strings'; 13 | -------------------------------------------------------------------------------- /db/migrations/20191012163928_add_column_shards.archived_at.sql: -------------------------------------------------------------------------------- 1 | -- migrate:up 2 | ALTER TABLE public.shards ADD COLUMN archived_at timestamptz; 3 | 4 | -- migrate:down 5 | ALTER TABLE public.shards DROP COLUMN archived_at; 6 | -------------------------------------------------------------------------------- /db/migrations/20191102100059_upgrade_activity_log.sql: -------------------------------------------------------------------------------- 1 | -- migrate:up 2 | ALTER TABLE sync_log RENAME TO activity_log; 3 | 4 | ALTER TABLE activity_log 5 | ADD COLUMN shard_id bigint REFERENCES shards(id), 6 | ALTER COLUMN repo_id DROP NOT NULL; 7 | 8 | ALTER SEQUENCE sync_log_id_seq RENAME TO activity_log_id_seq; 9 | 10 | ALTER TABLE activity_log RENAME CONSTRAINT sync_log_pkey TO activity_log_pkey; 11 | ALTER TABLE activity_log RENAME CONSTRAINT sync_log_repo_id_fkey TO activity_log_repo_id_fkey; 12 | 13 | UPDATE activity_log 14 | SET 15 | event = 'sync_repo:' || event 16 | ; 17 | 18 | -- migrate:down 19 | 20 | UPDATE activity_log 21 | SET 22 | event = substring(event FROM 11) 23 | WHERE 24 | starts_with(event, 'sync_repo:') 25 | ; 26 | 27 | ALTER TABLE activity_log RENAME CONSTRAINT activity_log_repo_id_fkey TO sync_log_repo_id_fkey; 28 | ALTER TABLE activity_log RENAME CONSTRAINT activity_log_pkey TO sync_log_pkey; 29 | 30 | ALTER SEQUENCE activity_log_id_seq RENAME TO sync_log_id_seq; 31 | 32 | ALTER TABLE activity_log 33 | ALTER COLUMN repo_id SET NOT NULL, 34 | DROP COLUMN shard_id; 35 | 36 | ALTER TABLE activity_log RENAME TO sync_log; 37 | -------------------------------------------------------------------------------- /db/migrations/20191106093828_add_constraints_for_repos.shard_id.sql: -------------------------------------------------------------------------------- 1 | -- migrate:up 2 | ALTER TABLE repos 3 | ADD CONSTRAINT repos_shard_id_null_role CHECK (shard_id IS NOT NULL OR role = 'canonical' OR role = 'obsolete'), 4 | ADD CONSTRAINT repos_obsolete_role_shard_id_null CHECK (role <> 'obsolete' OR shard_id IS NULL); 5 | 6 | -- migrate:down 7 | ALTER TABLE repos 8 | DROP CONSTRAINT repos_shard_id_null_role, 9 | DROP CONSTRAINT repos_obsolete_role_shard_id_null; 10 | 11 | -------------------------------------------------------------------------------- /db/migrations/20191115142944_change_releases.position_not_null.sql: -------------------------------------------------------------------------------- 1 | -- migrate:up 2 | UPDATE releases 3 | SET 4 | position = COALESCE((SELECT MAX(position) + 1 FROM releases AS r WHERE r.shard_id = releases.shard_id), 0) 5 | WHERE 6 | position IS NULL; 7 | ALTER TABLE releases ALTER COLUMN position SET NOT NULL; 8 | 9 | -- migrate:down 10 | ALTER TABLE releases ALTER COLUMN position DROP NOT NULL; 11 | -------------------------------------------------------------------------------- /db/migrations/20191122122940_create_table_files.sql: -------------------------------------------------------------------------------- 1 | -- migrate:up 2 | CREATE TABLE files ( 3 | id bigint GENERATED BY DEFAULT AS IDENTITY, 4 | release_id bigint NOT NULL REFERENCES releases(id), 5 | path text NOT NULL, 6 | content text 7 | ); 8 | 9 | CREATE UNIQUE INDEX files_release_id_path_idx ON public.files USING btree (release_id, path); 10 | ALTER TABLE files 11 | ADD CONSTRAINT files_release_id_path_uniq UNIQUE USING INDEX files_release_id_path_idx; 12 | 13 | -- migrate:down 14 | DROP TABLE files; 15 | -------------------------------------------------------------------------------- /db/migrations/20200503132444_create_table_owners.sql: -------------------------------------------------------------------------------- 1 | -- migrate:up 2 | CREATE TABLE public.owners ( 3 | id bigint NOT NULL PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, 4 | resolver public.repo_resolver NOT NULL, 5 | slug public.citext NOT NULL, 6 | name text, 7 | description text, 8 | extra jsonb DEFAULT '{}'::jsonb NOT NULL, 9 | shards_count integer, 10 | dependents_count integer, 11 | transitive_dependents_count integer, 12 | dev_dependents_count integer, 13 | transitive_dependencies_count integer, 14 | dev_dependencies_count integer, 15 | dependencies_count integer, 16 | popularity real, 17 | created_at timestamp with time zone DEFAULT now() NOT NULL, 18 | updated_at timestamp with time zone DEFAULT now() NOT NULL 19 | ); 20 | 21 | CREATE TRIGGER set_timestamp BEFORE UPDATE ON public.owners FOR EACH ROW EXECUTE PROCEDURE public.trigger_set_timestamp(); 22 | 23 | CREATE UNIQUE INDEX owners_resolver_slug_idx ON public.owners USING btree (resolver, slug); 24 | ALTER TABLE owners 25 | ADD CONSTRAINT owners_resolver_slug_uniq UNIQUE USING INDEX owners_resolver_slug_idx; 26 | 27 | ALTER TABLE repos 28 | ADD COLUMN owner_id bigint REFERENCES owners(id); 29 | 30 | CREATE TABLE public.owner_metrics ( 31 | id bigint NOT NULL PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, 32 | owner_id bigint NOT NULL, 33 | shards_count integer NOT NULL, 34 | dependents_count integer NOT NULL, 35 | transitive_dependents_count integer NOT NULL, 36 | dev_dependents_count integer NOT NULL, 37 | transitive_dependencies_count integer NOT NULL, 38 | dev_dependencies_count integer NOT NULL, 39 | dependencies_count integer NOT NULL, 40 | popularity real NOT NULL, 41 | created_at timestamp with time zone DEFAULT now() NOT NULL 42 | ); 43 | 44 | ALTER TABLE ONLY public.owner_metrics 45 | ADD CONSTRAINT owner_metrics_owner_id_fkey FOREIGN KEY (owner_id) REFERENCES public.owners(id) ON DELETE CASCADE DEFERRABLE; 46 | 47 | CREATE OR REPLACE FUNCTION public.owner_metrics_calculate(curr_owner_id bigint) RETURNS void 48 | LANGUAGE plpgsql 49 | AS $$ 50 | DECLARE 51 | aggregated_popularity real; 52 | local_dev_dependencies_count integer; 53 | BEGIN 54 | CREATE TEMPORARY TABLE owned_shards 55 | AS 56 | SELECT 57 | shard_id AS id 58 | FROM repos 59 | WHERE owner_id = curr_owner_id 60 | AND repos.role = 'canonical' 61 | ; 62 | 63 | CREATE TEMPORARY TABLE dependents 64 | AS 65 | SELECT 66 | d.shard_id, depends_on, scope 67 | FROM 68 | shard_dependencies d 69 | JOIN owned_shards 70 | ON depends_on = owned_shards.id 71 | ; 72 | CREATE TEMPORARY TABLE tmp_dependencies 73 | AS 74 | SELECT 75 | d.shard_id, depends_on, scope 76 | FROM 77 | shard_dependencies d 78 | JOIN owned_shards 79 | ON shard_id = owned_shards.id 80 | ; 81 | 82 | SELECT 83 | SUM(popularity) INTO aggregated_popularity 84 | FROM 85 | shard_metrics_current 86 | JOIN owned_shards 87 | ON owned_shards.id = shard_metrics_current.shard_id 88 | ; 89 | 90 | SELECT 91 | COUNT(DISTINCT depends_on) INTO local_dev_dependencies_count 92 | FROM 93 | tmp_dependencies 94 | WHERE 95 | scope <> 'runtime' 96 | ; 97 | 98 | UPDATE owners 99 | SET 100 | shards_count = ( 101 | SELECT 102 | COUNT(*) 103 | FROM 104 | owned_shards 105 | ), 106 | dependents_count = ( 107 | SELECT 108 | COUNT(DISTINCT shard_id) 109 | FROM 110 | dependents 111 | WHERE 112 | scope = 'runtime' 113 | ), 114 | dev_dependents_count = ( 115 | SELECT 116 | COUNT(DISTINCT shard_id) 117 | FROM 118 | dependents 119 | WHERE 120 | scope <> 'runtime' 121 | ), 122 | transitive_dependents_count = tdc.transitive_dependents_count, 123 | dependencies_count = ( 124 | SELECT 125 | COUNT(DISTINCT depends_on) 126 | FROM 127 | tmp_dependencies 128 | WHERE 129 | scope = 'runtime' 130 | ), 131 | dev_dependencies_count = local_dev_dependencies_count, 132 | transitive_dependencies_count = ( 133 | WITH RECURSIVE transitive_dependencies AS ( 134 | SELECT 135 | shard_id, depends_on 136 | FROM 137 | tmp_dependencies 138 | WHERE 139 | scope = 'runtime' 140 | UNION 141 | SELECT 142 | d.shard_id, d.depends_on 143 | FROM 144 | shard_dependencies d 145 | INNER JOIN 146 | transitive_dependencies ON transitive_dependencies.depends_on = d.shard_id AND d.scope = 'runtime' 147 | ) 148 | SELECT 149 | COUNT(*) 150 | FROM 151 | ( 152 | SELECT DISTINCT 153 | depends_on 154 | FROM 155 | transitive_dependencies 156 | ) AS d 157 | ), 158 | popularity = POWER( 159 | POWER(COALESCE(tdc.transitive_dependents_count, 0) + 1, 1.2) * 160 | POWER(COALESCE(local_dev_dependencies_count, 0) + 1, 0.6) * 161 | POWER(COALESCE(aggregated_popularity, 0) + 1, 1.2), 162 | 1.0/3.0 163 | ) 164 | FROM 165 | ( 166 | WITH RECURSIVE transitive_dependents AS ( 167 | SELECT 168 | shard_id, depends_on 169 | FROM 170 | dependents 171 | WHERE 172 | scope = 'runtime' 173 | UNION 174 | SELECT 175 | d.shard_id, d.depends_on 176 | FROM 177 | shard_dependencies d 178 | INNER JOIN 179 | transitive_dependents ON transitive_dependents.shard_id = d.depends_on AND d.scope = 'runtime' 180 | ) 181 | SELECT 182 | COUNT(*) AS transitive_dependents_count 183 | FROM 184 | ( 185 | SELECT DISTINCT 186 | shard_id 187 | FROM 188 | transitive_dependents 189 | ) AS d 190 | ) AS tdc 191 | WHERE 192 | id = curr_owner_id 193 | ; 194 | 195 | INSERT INTO owner_metrics 196 | ( 197 | owner_id, 198 | shards_count, 199 | dependents_count, dev_dependents_count, transitive_dependents_count, 200 | dependencies_count, dev_dependencies_count, transitive_dependencies_count, 201 | popularity 202 | ) 203 | SELECT 204 | id, 205 | shards_count, 206 | dependents_count, dev_dependents_count, transitive_dependents_count, 207 | dependencies_count, dev_dependencies_count, transitive_dependencies_count, 208 | popularity 209 | FROM 210 | owners 211 | WHERE id = curr_owner_id; 212 | 213 | DROP TABLE dependents; 214 | DROP TABLE tmp_dependencies; 215 | DROP TABLE owned_shards; 216 | END; 217 | $$; 218 | 219 | -- migrate:down 220 | DROP FUNCTION public.owner_metrics_calculate; 221 | 222 | ALTER TABLE repos DROP COLUMN owner_id; 223 | 224 | DROP TABLE owner_metrics; 225 | 226 | DROP TABLE owners; 227 | -------------------------------------------------------------------------------- /db/migrations/20200506215505_add_shards_merged_with.sql: -------------------------------------------------------------------------------- 1 | -- migrate:up 2 | 3 | ALTER TABLE shards 4 | ADD COLUMN merged_with bigint, 5 | ADD CONSTRAINT shards_merged_with_fk FOREIGN KEY (merged_with) REFERENCES shards(id), 6 | ADD CONSTRAINT shards_merged_with_archived_at CHECK (merged_with IS NULL OR (archived_at IS NOT NULL AND categories = '{}')), 7 | DROP CONSTRAINT shards_name_unique, 8 | ADD CONSTRAINT shards_name_unique UNIQUE (name, qualifier) DEFERRABLE INITIALLY IMMEDIATE 9 | ; 10 | 11 | -- Update all existing archived shards to point to the shard they were merged into 12 | UPDATE shards 13 | SET 14 | merged_with = log.shard_id 15 | FROM activity_log log 16 | WHERE log.event = 'import_catalog:mirror:switched' 17 | AND log.metadata->>'old_role' = 'canonical' 18 | AND log.metadata->'old_shard_id' <> 'null' 19 | AND shards.id = (metadata->'old_shard_id')::bigint 20 | ; 21 | 22 | -- Remove dependencies on the merged shard (they should be picked up by the merge target) 23 | DELETE FROM shard_dependencies 24 | USING shards 25 | WHERE shards.id = shard_id 26 | AND shards.merged_with IS NOT NULL 27 | ; 28 | 29 | -- Switch qualifiers if a merged shard has the empty qualifier 30 | SET CONSTRAINTS shards_name_unique DEFERRED; 31 | UPDATE shards 32 | SET qualifier = main.qualifier 33 | FROM shards main 34 | WHERE shards.merged_with = main.id 35 | AND shards.name = main.name 36 | AND shards.qualifier = '' 37 | ; 38 | UPDATE shards 39 | SET qualifier = '' 40 | FROM shards merged 41 | WHERE shards.id = merged.merged_with 42 | AND shards.name = merged.name 43 | AND shards.qualifier = merged.qualifier 44 | ; 45 | 46 | -- migrate:down 47 | 48 | ALTER TABLE shards 49 | DROP COLUMN merged_with, 50 | DROP CONSTRAINT shards_name_unique, 51 | ADD CONSTRAINT shards_name_unique UNIQUE (name, qualifier) 52 | ; 53 | -------------------------------------------------------------------------------- /scripts/build_libgit2.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | install_dir=$1 4 | branch=$2 5 | 6 | git clone --depth=1 -b $branch https://github.com/libgit2/libgit2.git 7 | 8 | mkdir libgit2/build 9 | cd libgit2/build 10 | 11 | cmake .. -DCMAKE_INSTALL_PREFIX=$install_dir -DBUILD_CLAR=OFF 12 | cmake --build . --target install 13 | -------------------------------------------------------------------------------- /shard.lock: -------------------------------------------------------------------------------- 1 | version: 2.0 2 | shards: 3 | any_hash: 4 | git: https://github.com/sija/any_hash.cr.git 5 | version: 0.2.5 6 | 7 | backtracer: 8 | git: https://github.com/sija/backtracer.cr.git 9 | version: 1.2.1 10 | 11 | db: 12 | git: https://github.com/crystal-lang/crystal-db.git 13 | version: 0.10.1 14 | 15 | git: 16 | git: https://github.com/smacker/libgit2.cr.git 17 | version: 0.1.0+git.commit.6267162e7d9e16edace62f942d08565518bc73d9 18 | 19 | github-cr: 20 | git: https://github.com/arnavb/github-cr.git 21 | version: 0.1.0+git.commit.119df4c747a81260b37af0878b24eab9345a94f4 22 | 23 | molinillo: 24 | git: https://github.com/crystal-lang/crystal-molinillo.git 25 | version: 0.2.0 26 | 27 | pg: 28 | git: https://github.com/will/crystal-pg.git 29 | version: 0.24.0 30 | 31 | raven: 32 | git: https://github.com/sija/raven.cr.git 33 | version: 1.9.1 34 | 35 | shards: 36 | git: https://github.com/crystal-lang/shards.git 37 | version: 0.16.0 38 | 39 | webmock: 40 | git: https://github.com/manastech/webmock.cr.git 41 | version: 0.14.0+git.commit.42b347cdd64e13193e46167a03593944ae2b3d20 42 | 43 | -------------------------------------------------------------------------------- /shard.yml: -------------------------------------------------------------------------------- 1 | name: shardbox-core 2 | version: 0.2.0 3 | 4 | authors: 5 | - Johannes Müller 6 | 7 | targets: 8 | worker: 9 | main: src/worker.cr 10 | 11 | crystal: ">= 1.0.0" 12 | 13 | license: MIT 14 | 15 | dependencies: 16 | shards: 17 | github: crystal-lang/shards 18 | pg: 19 | github: will/crystal-pg 20 | raven: 21 | github: Sija/raven.cr 22 | git: 23 | github: smacker/libgit2.cr 24 | github-cr: 25 | github: arnavb/github-cr 26 | 27 | development_dependencies: 28 | webmock: 29 | github: manastech/webmock.cr 30 | branch: master 31 | -------------------------------------------------------------------------------- /spec/catalog/duplication_spec.cr: -------------------------------------------------------------------------------- 1 | require "spec" 2 | require "../../src/catalog" 3 | require "file_utils" 4 | 5 | private def with_tempdir(name, &) 6 | path = File.join(Dir.tempdir, name) 7 | FileUtils.mkdir_p(path) 8 | 9 | begin 10 | yield path 11 | ensure 12 | FileUtils.rm_r(path) if File.exists?(path) 13 | end 14 | end 15 | 16 | describe "duplicate repos" do 17 | it "same entry" do 18 | with_tempdir("catalog-mirrors") do |catalog_path| 19 | File.write(File.join(catalog_path, "category.yml"), <<-YAML) 20 | name: Category 21 | shards: 22 | - git: foo/foo 23 | mirrors: 24 | - git: foo/foo 25 | YAML 26 | expect_raises(Catalog::Error, "category: duplicate mirror git:foo/foo also in category") do 27 | Catalog.read(catalog_path) 28 | end 29 | end 30 | end 31 | 32 | it "both mirrors" do 33 | with_tempdir("catalog-mirrors") do |catalog_path| 34 | File.write(File.join(catalog_path, "category.yml"), <<-YAML) 35 | name: Category 36 | shards: 37 | - git: foo/foo 38 | mirrors: 39 | - git: foo/bar 40 | - git: foo/baz 41 | mirrors: 42 | - git: foo/bar 43 | YAML 44 | expect_raises(Catalog::Error, "category: duplicate mirror git:foo/bar also on git:foo/foo in category") do 45 | Catalog.read(catalog_path) 46 | end 47 | end 48 | end 49 | 50 | it "mirror and canonical" do 51 | with_tempdir("catalog-mirrors") do |catalog_path| 52 | File.write(File.join(catalog_path, "category.yml"), <<-YAML) 53 | name: Category 54 | shards: 55 | - git: foo/foo 56 | mirrors: 57 | - git: foo/bar 58 | - git: foo/bar 59 | YAML 60 | expect_raises(Catalog::Error, "category: duplicate repo git:foo/bar also on git:foo/foo in category") do 61 | Catalog.read(catalog_path) 62 | end 63 | end 64 | end 65 | 66 | it "canonical and mirror" do 67 | with_tempdir("catalog-mirrors") do |catalog_path| 68 | File.write(File.join(catalog_path, "category.yml"), <<-YAML) 69 | name: Category 70 | shards: 71 | - git: foo/bar 72 | - git: foo/foo 73 | mirrors: 74 | - git: foo/bar 75 | YAML 76 | expect_raises(Catalog::Error, "category: duplicate mirror git:foo/bar also in category") do 77 | Catalog.read(catalog_path) 78 | end 79 | end 80 | end 81 | 82 | it "canonical with different descriptions" do 83 | with_tempdir("catalog-mirrors") do |catalog_path| 84 | File.write(File.join(catalog_path, "category.yml"), <<-YAML) 85 | name: Category 86 | shards: 87 | - git: foo/bar 88 | description: A 89 | - git: foo/bar 90 | description: B 91 | YAML 92 | expect_raises(Catalog::Error, "category: duplicate repo git:foo/bar also in category") do 93 | Catalog.read(catalog_path) 94 | end 95 | end 96 | end 97 | 98 | it "canonical with same descriptions" do 99 | with_tempdir("catalog-mirrors") do |catalog_path| 100 | File.write(File.join(catalog_path, "category.yml"), <<-YAML) 101 | name: Category 102 | shards: 103 | - git: foo/bar 104 | description: A 105 | - git: foo/bar 106 | description: A 107 | YAML 108 | expect_raises(Catalog::Error, "category: duplicate repo git:foo/bar also in category") do 109 | Catalog.read(catalog_path) 110 | end 111 | end 112 | end 113 | 114 | it "canonical with nil description" do 115 | with_tempdir("catalog-mirrors") do |catalog_path| 116 | File.write(File.join(catalog_path, "category.yml"), <<-YAML) 117 | name: Category 118 | shards: 119 | - git: foo/bar 120 | description: A 121 | - git: foo/bar 122 | YAML 123 | Catalog.read(catalog_path) 124 | end 125 | end 126 | 127 | it "canonical with both nil description" do 128 | with_tempdir("catalog-mirrors") do |catalog_path| 129 | File.write(File.join(catalog_path, "category.yml"), <<-YAML) 130 | name: Category 131 | shards: 132 | - git: foo/bar 133 | - git: foo/bar 134 | YAML 135 | Catalog.read(catalog_path) 136 | end 137 | end 138 | end 139 | -------------------------------------------------------------------------------- /spec/catalog_spec.cr: -------------------------------------------------------------------------------- 1 | require "spec" 2 | require "../src/catalog" 3 | require "file_utils" 4 | 5 | private def with_tempdir(name, &) 6 | path = File.join(Dir.tempdir, name) 7 | FileUtils.mkdir_p(path) 8 | 9 | begin 10 | yield path 11 | ensure 12 | FileUtils.rm_r(path) if File.exists?(path) 13 | end 14 | end 15 | 16 | describe Catalog do 17 | describe ".read" do 18 | it "reads" do 19 | with_tempdir("catalog-mirrors") do |catalog_path| 20 | File.write(File.join(catalog_path, "category.yml"), <<-YAML) 21 | name: Category 22 | shards: 23 | - git: foo/foo 24 | mirrors: 25 | - git: bar/foo 26 | - git: baz/foo 27 | role: legacy 28 | - git: qux/foo 29 | role: legacy 30 | - git: bar/bar 31 | mirrors: 32 | - git: foo/bar 33 | role: legacy 34 | YAML 35 | catalog = Catalog.read(catalog_path) 36 | catalog.categories.keys.should eq ["category"] 37 | end 38 | end 39 | end 40 | end 41 | 42 | describe Catalog::Category do 43 | describe ".from_yaml" do 44 | it "reads category" do 45 | io = IO::Memory.new <<-YAML 46 | name: Foo 47 | description: Foo category 48 | shards: 49 | - github: foo/foo 50 | - git: https://example.com/foo.git 51 | description: Another foo 52 | - git: https://github.com/bar/foo.git 53 | description: Triple the foo 54 | YAML 55 | category = Catalog::Category.from_yaml(io) 56 | category.name.should eq "Foo" 57 | category.description.should eq "Foo category" 58 | category.shards.should eq [ 59 | Catalog::Entry.new(Repo::Ref.new("github", "foo/foo")), 60 | Catalog::Entry.new(Repo::Ref.new("git", "https://example.com/foo.git"), "Another foo"), 61 | Catalog::Entry.new(Repo::Ref.new("github", "bar/foo"), "Triple the foo"), 62 | ] 63 | end 64 | end 65 | 66 | it "fails for unknown attributes" do 67 | expect_raises(YAML::ParseException, "Unknown yaml attribute: baar") do 68 | Catalog::Category.from_yaml <<-YAML 69 | description: foo 70 | baar: bar 71 | YAML 72 | end 73 | end 74 | end 75 | -------------------------------------------------------------------------------- /spec/dependency_spec.cr: -------------------------------------------------------------------------------- 1 | require "spec" 2 | require "../src/dependency" 3 | 4 | describe Dependency do 5 | it "#version_reference" do 6 | Dependency.new("foo", JSON.parse(%({"version": "1.0"}))).version_reference.should eq({"version", "1.0"}) 7 | Dependency.new("foo", JSON.parse(%({"branch": "master"}))).version_reference.should eq({"branch", "master"}) 8 | Dependency.new("foo", JSON.parse(%({"commit": "0123456789abcdef"}))).version_reference.should eq({"commit", "0123456789abcdef"}) 9 | Dependency.new("foo", JSON.parse(%({"tag": "foo"}))).version_reference.should eq({"tag", "foo"}) 10 | Dependency.new("foo", JSON.parse(%({}))).version_reference.should be_nil 11 | 12 | Dependency.new("foo", JSON.parse(%({"commit": "0123456789abcdef", "tag": "foo"}))).version_reference.should eq({"commit", "0123456789abcdef"}) 13 | Dependency.new("foo", JSON.parse(%({"tag": "foo", "version": "1.0"}))).version_reference.should eq({"tag", "foo"}) 14 | Dependency.new("foo", JSON.parse(%({"version": "1.0", "branch": "master"}))).version_reference.should eq({"version", "1.0"}) 15 | end 16 | 17 | it "#repo_ref" do 18 | Dependency.new("foo", JSON.parse(%({"git": "http://example.com/foo.git"}))).repo_ref.should eq Repo::Ref.new("git", "http://example.com/foo.git") 19 | Dependency.new("foo", JSON.parse(%({"github": "foo/foo"}))).repo_ref.should eq Repo::Ref.new("github", "foo/foo") 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /spec/ext/shards/git_spec.cr: -------------------------------------------------------------------------------- 1 | require "spec" 2 | require "shards/logger" 3 | require "shards/dependency" 4 | # FIXME: shards/package is only a workaround, should be shards/resolvers/resolver 5 | require "shards/package" 6 | require "shards/resolvers/git" 7 | require "../../../src/ext/shards/resolvers/git" 8 | require "shards/spec/support/factories" 9 | require "file_utils" 10 | 11 | private def create_git_tag(project, version, message) 12 | Dir.cd(git_path(project)) do 13 | run "git tag -m '#{message}' #{version}" 14 | end 15 | end 16 | 17 | describe Shards::GitResolver do 18 | describe "#revision_info" do 19 | it "handles special characters" do 20 | create_git_repository("repo1") 21 | create_git_commit("repo1", "foo\nbar") 22 | create_git_tag("repo1", "v0.1", "bar\nfoo") 23 | create_git_commit("repo1", "foo\"bar") 24 | create_git_tag("repo1", "v0.2", "bar\"foo") 25 | resolver = Shards::GitResolver.find_resolver("git", "repo1", git_url("repo1")).as(Shards::GitResolver) 26 | 27 | revision_info = resolver.revision_info(Shards::Version.new("0.1")) 28 | revision_info.commit.message.should eq "foo\nbar" 29 | revision_info.tag.not_nil!.message.should eq "bar\nfoo" 30 | 31 | revision_info = resolver.revision_info(Shards::Version.new("0.2")) 32 | revision_info.commit.message.should eq "foo\"bar" 33 | revision_info.tag.not_nil!.message.should eq "bar\"foo" 34 | ensure 35 | FileUtils.rm_rf(git_path("repo1")) 36 | end 37 | 38 | it "resolves HEAD" do 39 | create_git_repository("repo2") 40 | create_git_commit("repo2", "foo bar") 41 | resolver = Shards::GitResolver.find_resolver("git", "repo2", git_url("repo2")).as(Shards::GitResolver) 42 | 43 | revision_info = resolver.revision_info(Shards::GitHeadRef.new) 44 | revision_info.commit.message.should eq "foo bar" 45 | revision_info.tag.should be_nil 46 | ensure 47 | FileUtils.rm_rf(git_path("repo2")) 48 | end 49 | 50 | it "resolves symbolic reference" do 51 | create_git_repository("repo3") 52 | create_git_commit("repo3", "foo bar") 53 | create_git_tag("repo3", "v0.1", "bar foo") 54 | Dir.cd(git_path("repo3")) do 55 | run "git tag v0.2 v0.1" 56 | end 57 | resolver = Shards::GitResolver.find_resolver("git", "repo3", git_url("repo3")).as(Shards::GitResolver) 58 | 59 | revision_info = resolver.revision_info(Shards::Version.new("0.2")) 60 | revision_info.commit.message.should eq "foo bar" 61 | revision_info.tag.not_nil!.message.should eq "bar foo" 62 | ensure 63 | FileUtils.rm_rf(git_path("repo3")) 64 | end 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /spec/fetchers/github_api_spec.cr: -------------------------------------------------------------------------------- 1 | require "spec" 2 | require "../../src/fetchers/github_api" 3 | require "webmock" 4 | 5 | describe Shardbox::GitHubAPI do 6 | describe "#fetch_repo_metadata" do 7 | it "queries github graphql" do 8 | WebMock.wrap do 9 | WebMock.stub(:post, "https://api.github.com/graphql").to_return do |request| 10 | body = request.body.not_nil!.gets_to_end 11 | body.should contain %("variables":{"owner":"foo","name":"bar"}) 12 | 13 | HTTP::Client::Response.new(:ok, <<-JSON) 14 | { 15 | "data": { 16 | "repository": { 17 | "description": "foo bar baz" 18 | } 19 | } 20 | } 21 | JSON 22 | end 23 | api = Shardbox::GitHubAPI.new("") 24 | repo_info = api.fetch_repo_metadata("foo", "bar") 25 | 26 | repo_info.should eq Repo::Metadata.new(description: "foo bar baz") 27 | end 28 | end 29 | 30 | it "raises when endpoint unavailable" do 31 | WebMock.wrap do 32 | WebMock.stub(:post, "https://api.github.com/graphql").to_return(status: 401) 33 | 34 | api = Shardbox::GitHubAPI.new("") 35 | 36 | expect_raises(Shardbox::FetchError, "Repository unavailable") do 37 | api.fetch_repo_metadata("foo", "bar") 38 | end 39 | end 40 | end 41 | 42 | it "raises when response is invalid" do 43 | WebMock.wrap do 44 | WebMock.stub(:post, "https://api.github.com/graphql").to_return(status: 200) 45 | 46 | api = Shardbox::GitHubAPI.new("") 47 | 48 | expect_raises(Shardbox::FetchError, "Invalid response") do 49 | api.fetch_repo_metadata("foo", "bar") 50 | end 51 | end 52 | end 53 | end 54 | 55 | describe "#fetch_repository_owner" do 56 | it "queries from GraphQL" do 57 | WebMock.wrap do 58 | WebMock.stub(:post, "https://api.github.com/graphql").to_return(<<-JSON, status: 200) 59 | { 60 | "data": { 61 | "repositoryOwner": { 62 | "description": "foo" 63 | } 64 | } 65 | } 66 | JSON 67 | 68 | api = Shardbox::GitHubAPI.new("") 69 | 70 | api.fetch_owner_info("foo").should eq({ 71 | "description" => JSON::Any.new("foo"), 72 | }) 73 | end 74 | end 75 | 76 | it "handles repositoryOwner: null" do 77 | WebMock.wrap do 78 | WebMock.stub(:post, "https://api.github.com/graphql").to_return(<<-JSON, status: 200) 79 | { 80 | "data": { 81 | "repositoryOwner": null 82 | } 83 | } 84 | JSON 85 | 86 | api = Shardbox::GitHubAPI.new("") 87 | 88 | api.fetch_owner_info("foo").should be_nil 89 | end 90 | end 91 | end 92 | end 93 | -------------------------------------------------------------------------------- /spec/fixtures/repo_metadata-github-graphql-response.json: -------------------------------------------------------------------------------- 1 | { 2 | "nameWithOwner": "crystal-lang/crystal", 3 | "forks": { 4 | "totalCount": 964 5 | }, 6 | "stargazers": { 7 | "totalCount": 13060 8 | }, 9 | "watchers": { 10 | "totalCount": 479 11 | }, 12 | "createdAt": "2012-11-27T17:32:32Z", 13 | "description": "The Crystal Programming Language", 14 | "hasIssuesEnabled": true, 15 | "hasWikiEnabled": true, 16 | "homepageUrl": "https://crystal-lang.org", 17 | "isArchived": false, 18 | "isFork": false, 19 | "isMirror": false, 20 | "licenseInfo": { 21 | "key": "other" 22 | }, 23 | "primaryLanguage": { 24 | "name": "Crystal" 25 | }, 26 | "pushedAt": "2019-04-08T23:53:17Z", 27 | "closedIssues": { 28 | "totalCount": 3587 29 | }, 30 | "openIssues": { 31 | "totalCount": 717 32 | }, 33 | "closedPullRequests": { 34 | "totalCount": 673 35 | }, 36 | "openPullRequests": { 37 | "totalCount": 154 38 | }, 39 | "mergedPullRequests": { 40 | "totalCount": 2408 41 | }, 42 | "repositoryTopics": { 43 | "nodes": [ 44 | { 45 | "topic": { 46 | "name": "crystal" 47 | } 48 | }, 49 | { 50 | "topic": { 51 | "name": "language" 52 | } 53 | }, 54 | { 55 | "topic": { 56 | "name": "efficiency" 57 | } 58 | } 59 | ] 60 | }, 61 | "codeOfConduct": { 62 | "name": "Contributor Covenant", 63 | "url": "https://github.com/crystal-lang/crystal/blob/master/CODE_OF_CONDUCT.md" 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /spec/owner_spec.cr: -------------------------------------------------------------------------------- 1 | require "spec" 2 | require "../src/repo/owner" 3 | 4 | describe Repo::Owner do 5 | describe ".from_repo_ref" do 6 | it "returns nil for git" do 7 | Repo::Owner.from_repo_ref(Repo::Ref.new("git", "foo/bar")).should be_nil 8 | end 9 | 10 | it "creates owner for github" do 11 | Repo::Owner.from_repo_ref(Repo::Ref.new("github", "foo/bar")).should eq Repo::Owner.new("github", "foo") 12 | end 13 | 14 | it "creates owner for gitlab" do 15 | Repo::Owner.from_repo_ref(Repo::Ref.new("gitlab", "foo/bar")).should eq Repo::Owner.new("gitlab", "foo") 16 | end 17 | 18 | it "creates owner for bitbucket" do 19 | Repo::Owner.from_repo_ref(Repo::Ref.new("bitbucket", "foo/bar")).should eq Repo::Owner.new("bitbucket", "foo") 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /spec/repo_metadata_spec.cr: -------------------------------------------------------------------------------- 1 | require "spec" 2 | require "../src/repo" 3 | 4 | describe Repo::Metadata do 5 | describe "for Github graphql" do 6 | it ".from_github_graphql" do 7 | File.open("#{__DIR__}/fixtures/repo_metadata-github-graphql-response.json", "r") do |file| 8 | metadata = Repo::Metadata.from_json(file) 9 | metadata.should eq Repo::Metadata.new( 10 | forks_count: 964, 11 | stargazers_count: 13060, 12 | watchers_count: 479, 13 | created_at: Time.utc(2012, 11, 27, 17, 32, 32), 14 | description: "The Crystal Programming Language", 15 | issues_enabled: true, 16 | wiki_enabled: true, 17 | homepage_url: "https://crystal-lang.org", 18 | archived: false, 19 | fork: false, 20 | mirror: false, 21 | license: "other", 22 | primary_language: "Crystal", 23 | pushed_at: Time.utc(2019, 4, 8, 23, 53, 17), 24 | closed_issues_count: 3587, 25 | open_issues_count: 717, 26 | closed_pull_requests_count: 673, 27 | open_pull_requests_count: 154, 28 | merged_pull_requests_count: 2408, 29 | topics: ["crystal", "language", "efficiency"] 30 | ) 31 | end 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /spec/repo_ref_spec.cr: -------------------------------------------------------------------------------- 1 | require "spec" 2 | require "../src/repo" 3 | 4 | describe Repo::Ref do 5 | describe ".new" do 6 | describe "with URI" do 7 | it "defaults to git resolver" do 8 | repo_ref = Repo::Ref.new("file:///repo.git") 9 | repo_ref.resolver.should eq "git" 10 | repo_ref.url.should eq "file:///repo.git" 11 | end 12 | 13 | it "identifies service providers" do 14 | {"github", "gitlab", "bitbucket"}.each do |provider| 15 | Repo::Ref.new("https://#{provider}.com/foo/foo").should eq Repo::Ref.new(provider, "foo/foo") 16 | Repo::Ref.new("https://#{provider}.com/foo/foo.git").should eq Repo::Ref.new(provider, "foo/foo") 17 | Repo::Ref.new("https://#{provider}.com/foo/foo/").should eq Repo::Ref.new(provider, "foo/foo") 18 | Repo::Ref.new("https://#{provider}.com/foo/foo.git/").should eq Repo::Ref.new(provider, "foo/foo") 19 | Repo::Ref.new("https://www.#{provider}.com/foo/foo").should eq Repo::Ref.new(provider, "foo/foo") 20 | end 21 | end 22 | 23 | it "raises for invalid git URL" do 24 | expect_raises(Exception, %(Invalid url for resolver git: "http://example.com")) do 25 | Repo::Ref.new("http://example.com") 26 | end 27 | end 28 | end 29 | end 30 | 31 | it "#name" do 32 | Repo::Ref.new("file:///repo.git").name.should eq "repo" 33 | Repo::Ref.new("file:///repo").name.should eq "repo" 34 | Repo::Ref.new("https://example.com/foo/bar.git").name.should eq "bar" 35 | Repo::Ref.new("https://example.com/foo/bar.git/").name.should eq "bar" 36 | Repo::Ref.new("https://example.com/foo/bar/").name.should eq "bar" 37 | Repo::Ref.new("github", "foo/bar").name.should eq "bar" 38 | end 39 | 40 | it "#to_uri" do 41 | Repo::Ref.new("file:///repo.git").to_uri.should eq URI.parse("file:///repo.git") 42 | Repo::Ref.new("file:///repo").to_uri.should eq URI.parse("file:///repo") 43 | Repo::Ref.new("https://example.com/foo/bar.git").to_uri.should eq URI.parse("https://example.com/foo/bar.git") 44 | Repo::Ref.new("https://example.com/foo/bar.git/").to_uri.should eq URI.parse("https://example.com/foo/bar.git/") 45 | Repo::Ref.new("github", "foo/bar").to_uri.should eq URI.parse("https://github.com/foo/bar") 46 | end 47 | 48 | it "#base_url_source" do 49 | Repo::Ref.new("github", "foo/bar").base_url_source.should eq URI.parse("https://github.com/foo/bar/tree/master/") 50 | Repo::Ref.new("github", "foo/bar").base_url_source("HEAD").should eq URI.parse("https://github.com/foo/bar/tree/master/") 51 | Repo::Ref.new("github", "foo/bar").base_url_source("12345").should eq URI.parse("https://github.com/foo/bar/tree/12345/") 52 | 53 | Repo::Ref.new("gitlab", "foo/bar").base_url_source.should eq URI.parse("https://gitlab.com/foo/bar/-/tree/master/") 54 | Repo::Ref.new("gitlab", "foo/bar").base_url_source("HEAD").should eq URI.parse("https://gitlab.com/foo/bar/-/tree/master/") 55 | Repo::Ref.new("gitlab", "foo/bar").base_url_source("12345").should eq URI.parse("https://gitlab.com/foo/bar/-/tree/12345/") 56 | 57 | Repo::Ref.new("bitbucket", "foo/bar").base_url_source.should eq URI.parse("https://bitbucket.com/foo/bar/src/master/") 58 | Repo::Ref.new("bitbucket", "foo/bar").base_url_source("HEAD").should eq URI.parse("https://bitbucket.com/foo/bar/src/master/") 59 | Repo::Ref.new("bitbucket", "foo/bar").base_url_source("12345").should eq URI.parse("https://bitbucket.com/foo/bar/src/12345/") 60 | 61 | Repo::Ref.new("git", "foo/bar").base_url_source.should be_nil 62 | Repo::Ref.new("git", "foo/bar").base_url_source("12345").should be_nil 63 | end 64 | 65 | it "#base_url_raw" do 66 | Repo::Ref.new("github", "foo/bar").base_url_raw.should eq URI.parse("https://github.com/foo/bar/raw/master/") 67 | Repo::Ref.new("github", "foo/bar").base_url_raw("HEAD").should eq URI.parse("https://github.com/foo/bar/raw/master/") 68 | Repo::Ref.new("github", "foo/bar").base_url_raw("12345").should eq URI.parse("https://github.com/foo/bar/raw/12345/") 69 | 70 | Repo::Ref.new("gitlab", "foo/bar").base_url_raw.should eq URI.parse("https://gitlab.com/foo/bar/-/raw/master/") 71 | Repo::Ref.new("gitlab", "foo/bar").base_url_raw("HEAD").should eq URI.parse("https://gitlab.com/foo/bar/-/raw/master/") 72 | Repo::Ref.new("gitlab", "foo/bar").base_url_raw("12345").should eq URI.parse("https://gitlab.com/foo/bar/-/raw/12345/") 73 | 74 | Repo::Ref.new("bitbucket", "foo/bar").base_url_raw.should eq URI.parse("https://bitbucket.com/foo/bar/raw/master/") 75 | Repo::Ref.new("bitbucket", "foo/bar").base_url_raw("HEAD").should eq URI.parse("https://bitbucket.com/foo/bar/raw/master/") 76 | Repo::Ref.new("bitbucket", "foo/bar").base_url_raw("12345").should eq URI.parse("https://bitbucket.com/foo/bar/raw/12345/") 77 | 78 | Repo::Ref.new("git", "foo/bar").base_url_raw.should be_nil 79 | Repo::Ref.new("git", "foo/bar").base_url_raw("12345").should be_nil 80 | end 81 | 82 | it "#nice_url" do 83 | Repo::Ref.new("file:///repo.git").nice_url.should eq "file:///repo.git" 84 | Repo::Ref.new("file:///repo").nice_url.should eq "file:///repo" 85 | Repo::Ref.new("https://example.com/foo/bar").nice_url.should eq "example.com/foo/bar" 86 | Repo::Ref.new("https://example.com/foo/bar.git/").nice_url.should eq "example.com/foo/bar" 87 | Repo::Ref.new("github", "foo/bar").nice_url.should eq "foo/bar" 88 | Repo::Ref.new("gitlab", "foo/bar").nice_url.should eq "foo/bar" 89 | Repo::Ref.new("bitbucket", "foo/bar").nice_url.should eq "foo/bar" 90 | end 91 | 92 | it "#slug" do 93 | Repo::Ref.new("file:///repo.git").slug.should eq "file:///repo.git" 94 | Repo::Ref.new("file:///repo").slug.should eq "file:///repo" 95 | Repo::Ref.new("https://example.com/foo/bar").slug.should eq "example.com/foo/bar" 96 | Repo::Ref.new("https://example.com/foo/bar.git/").slug.should eq "example.com/foo/bar" 97 | Repo::Ref.new("github", "foo/bar").slug.should eq "github.com/foo/bar" 98 | Repo::Ref.new("gitlab", "foo/bar").slug.should eq "gitlab.com/foo/bar" 99 | Repo::Ref.new("bitbucket", "foo/bar").slug.should eq "bitbucket.com/foo/bar" 100 | end 101 | 102 | it "#<=>" do 103 | Repo::Ref.new("git", "bar").should be < Repo::Ref.new("git", "baz") 104 | (Repo::Ref.new("git", "bar") <=> Repo::Ref.new("git", "bar")).should eq 0 105 | Repo::Ref.new("git", "bar").should be > Repo::Ref.new("git", "Bar") 106 | (Repo::Ref.new("github", "goo/bar") <=> Repo::Ref.new("github", "goo/bar")).should eq 0 107 | (Repo::Ref.new("github", "goo/bar") <=> Repo::Ref.new("github", "goo/Bar")).should eq 0 108 | (Repo::Ref.new("github", "goo/bar") <=> Repo::Ref.new("github", "Goo/bar")).should eq 0 109 | Repo::Ref.new("github", "goo/bar").should be < Repo::Ref.new("github", "foo/baz") 110 | Repo::Ref.new("github", "goo/Bar").should be < Repo::Ref.new("github", "foo/baz") 111 | Repo::Ref.new("github", "goo/bar").should be < Repo::Ref.new("github", "foo/Baz") 112 | Repo::Ref.new("github", "goo/bar").should be < Repo::Ref.new("gitlab", "foo/baz") 113 | Repo::Ref.new("github", "goo/bar").should be < Repo::Ref.new("bitbucket", "foo/baz") 114 | Repo::Ref.new("github", "goo/bar").should be > Repo::Ref.new("bitbucket", "foo/bar") 115 | Repo::Ref.new("github", "goo/bar").should be > Repo::Ref.new("bitbucket", "goo/bar") 116 | 117 | Repo::Ref.new("git", "https://foo/bar").should be < Repo::Ref.new("github", "foo/bar") 118 | end 119 | end 120 | -------------------------------------------------------------------------------- /spec/service/create_owner_spec.cr: -------------------------------------------------------------------------------- 1 | require "spec" 2 | require "../support/db" 3 | require "../../src/service/create_owner" 4 | require "../support/fetcher_mocks" 5 | 6 | describe Service::CreateOwner do 7 | describe "#perform" do 8 | it "creates owner" do 9 | transaction do |db| 10 | repo_ref = Repo::Ref.new("github", "foo/bar") 11 | service = Service::CreateOwner.new(db, repo_ref) 12 | service.skip_owner_info = true 13 | 14 | db.get_owner?("github", "foo").should be_nil 15 | db.get_owner?(repo_ref).should be_nil 16 | service.perform 17 | db.get_owner?("github", "foo").should eq Repo::Owner.new("github", "foo", shards_count: 0) 18 | db.get_owner?(repo_ref).should be_nil 19 | end 20 | end 21 | 22 | it "assigns owner to repo" do 23 | transaction do |db| 24 | repo_ref = Repo::Ref.new("github", "foo/bar") 25 | db.create_repo(Repo.new(repo_ref, shard_id: nil)) 26 | db.get_owner?(repo_ref).should be_nil 27 | 28 | service = Service::CreateOwner.new(db, repo_ref) 29 | service.skip_owner_info = true 30 | service.perform 31 | 32 | db.get_owner?(repo_ref).should eq Repo::Owner.new("github", "foo", shards_count: 1) 33 | end 34 | end 35 | 36 | it "picks up existing owner" do 37 | transaction do |db| 38 | repo_ref = Repo::Ref.new("github", "foo/bar") 39 | db.create_repo(Repo.new(repo_ref, shard_id: nil)) 40 | db.create_owner(Repo::Owner.new("github", "foo")) 41 | db.get_owner?(repo_ref).should be_nil 42 | 43 | service = Service::CreateOwner.new(db, repo_ref) 44 | service.skip_owner_info = true 45 | service.perform 46 | 47 | db.get_owner?(repo_ref).should eq Repo::Owner.new("github", "foo", shards_count: 1) 48 | end 49 | end 50 | 51 | it "sets shards count" do 52 | transaction do |db| 53 | repo_ref = Repo::Ref.new("github", "foo/bar") 54 | db.create_repo(Repo.new(repo_ref, shard_id: nil)) 55 | 56 | service = Service::CreateOwner.new(db, repo_ref) 57 | service.skip_owner_info = true 58 | service.perform 59 | 60 | db.get_owner?(repo_ref).should eq Repo::Owner.new("github", "foo", shards_count: 1) 61 | 62 | repo_ref_baz = Repo::Ref.new("github", "foo/baz") 63 | db.create_repo(Repo.new(repo_ref_baz, shard_id: nil)) 64 | owner = Service::CreateOwner.new(db, repo_ref_baz).perform 65 | owner = owner.not_nil! 66 | 67 | db.get_owned_repos(owner.id).map(&.ref).should eq [repo_ref, repo_ref_baz] 68 | db.get_owner?(repo_ref).should eq Repo::Owner.new("github", "foo", shards_count: 2) 69 | end 70 | end 71 | end 72 | 73 | describe ".fetch_owner_info_github" do 74 | it "do" do 75 | api = Shardbox::GitHubAPI.new("") 76 | api.mock_owner_info = Hash(String, JSON::Any).from_json(<<-JSON) 77 | { 78 | "bio": "verbum domini manet in eternum", 79 | "company": "RKK", 80 | "createdAt": "2020-05-03T16:50:55Z", 81 | "email": "boni@boni.org", 82 | "location": "Germany", 83 | "name": "Bonifatius", 84 | "websiteUrl": "boni.org" 85 | } 86 | JSON 87 | 88 | owner = Repo::Owner.new("github", "boni") 89 | 90 | Service::CreateOwner.fetch_owner_info_github(owner, api) 91 | 92 | owner.should eq Repo::Owner.new("github", "boni", 93 | name: "Bonifatius", 94 | description: "verbum domini manet in eternum", 95 | extra: { 96 | "location" => JSON::Any.new("Germany"), 97 | "email" => JSON::Any.new("boni@boni.org"), 98 | "website_url" => JSON::Any.new("boni.org"), 99 | "company" => JSON::Any.new("RKK"), 100 | "created_at" => JSON::Any.new("2020-05-03T16:50:55Z"), 101 | } 102 | ) 103 | end 104 | end 105 | 106 | it "do" do 107 | api = Shardbox::GitHubAPI.new("") 108 | api.mock_owner_info = Hash(String, JSON::Any).from_json(<<-JSON) 109 | { 110 | "description": null, 111 | "name": null 112 | } 113 | JSON 114 | 115 | owner = Repo::Owner.new("github", "boni") 116 | 117 | Service::CreateOwner.fetch_owner_info_github(owner, api) 118 | 119 | owner.should eq Repo::Owner.new("github", "boni") 120 | end 121 | end 122 | -------------------------------------------------------------------------------- /spec/service/create_shard_spec.cr: -------------------------------------------------------------------------------- 1 | require "spec" 2 | require "../support/raven" 3 | require "../../src/service/create_shard" 4 | require "../support/db" 5 | 6 | private def find_qualifier(url, name, &) 7 | result = {"~", nil} 8 | transaction do |db| 9 | if url.is_a?(String) 10 | repo_ref = Repo::Ref.parse(url) 11 | else 12 | repo_ref = url 13 | end 14 | repo_id = Factory.create_repo(db, repo_ref) 15 | repo = db.get_repo(repo_id) 16 | 17 | yield db 18 | 19 | result = Service::CreateShard.new(db, repo, name).find_qualifier 20 | end 21 | result 22 | end 23 | 24 | describe Service::CreateShard do 25 | describe "#find_qualifier" do 26 | context "with git resolver" do 27 | it "detects username/repo" do 28 | find_qualifier("mock://example.com/user/test.git", "test") { }.should eq({"", nil}) 29 | end 30 | end 31 | 32 | it "detects username/repo" do 33 | find_qualifier("mock://example.com/user/test.git", "test") do |db| 34 | Factory.create_shard(db, "test") 35 | end.should eq({"user", nil}) 36 | end 37 | 38 | it "users hostname" do 39 | find_qualifier("mock://example.com/user/sub/test.git", "test") do |db| 40 | Factory.create_shard(db, "test") 41 | end.should eq({"example.com", nil}) 42 | end 43 | end 44 | 45 | it "with github resolver" do 46 | find_qualifier("github:testorg/test", "test") do |db| 47 | Factory.create_shard(db, "test") 48 | end.should eq({"testorg", nil}) 49 | end 50 | 51 | it "with existing qualifier" do 52 | find_qualifier("mock://example.com/user/test.git", "test") do |db| 53 | Factory.create_shard(db, "test") 54 | Factory.create_shard(db, "test", qualifier: "example.com") 55 | end.should eq({"user", nil}) 56 | end 57 | 58 | it "with different providers" do 59 | find_qualifier("github:testorg/test", "test") do |db| 60 | Factory.create_shard(db, "test") 61 | Factory.create_shard(db, "test", qualifier: "testorg") 62 | end.should eq({"github", nil}) 63 | end 64 | 65 | it "with different providers" do 66 | find_qualifier("github:testorg/test", "test") do |db| 67 | Factory.create_shard(db, "test") 68 | Factory.create_shard(db, "test", qualifier: "testorg") 69 | Factory.create_shard(db, "test", qualifier: "github") 70 | end.should eq({"testorg-github", nil}) 71 | end 72 | 73 | it "with archived shard" do 74 | shard_id = nil 75 | find_qualifier("github:testorg/test", "test") do |db| 76 | shard_id = Factory.create_shard(db, "test", archived_at: Time.utc) 77 | end.should eq({"", shard_id}) 78 | 79 | find_qualifier("github:testorg/test", "test") do |db| 80 | shard_id = Factory.create_shard(db, "test", archived_at: Time.utc) 81 | Factory.create_shard(db, "test", qualifier: "testorg", archived_at: Time.utc) 82 | end.should eq({"", shard_id}) 83 | 84 | find_qualifier("github:testorg/test", "test") do |db| 85 | Factory.create_shard(db, "test") 86 | shard_id = Factory.create_shard(db, "test", qualifier: "testorg", archived_at: Time.utc) 87 | end.should eq({"testorg", shard_id}) 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /spec/service/fetch_metadata_spec.cr: -------------------------------------------------------------------------------- 1 | require "spec" 2 | require "../../src/service/fetch_metadata" 3 | 4 | struct Shardbox::GitHubAPI 5 | property mock_repo_metadata : Repo::Metadata? 6 | 7 | def fetch_repo_metadata(owner : String, name : String) 8 | if mock = mock_repo_metadata 9 | return mock 10 | else 11 | previous_def 12 | end 13 | end 14 | end 15 | 16 | describe Service::FetchMetadata do 17 | describe "#fetch_repo_metadata" do 18 | it "returns nil for git" do 19 | service = Service::FetchMetadata.new(Repo::Ref.new("git", "foo")) 20 | service.fetch_repo_metadata.should be_nil 21 | end 22 | 23 | it "queries github graphql" do 24 | metadata = Repo::Metadata.new(created_at: Time.utc) 25 | service = Service::FetchMetadata.new(Repo::Ref.new("github", "foo/bar")) 26 | api = Shardbox::GitHubAPI.new("") 27 | api.mock_repo_metadata = metadata 28 | service.github_api = api 29 | service.fetch_repo_metadata.should eq metadata 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /spec/service/import_categories_spec.cr: -------------------------------------------------------------------------------- 1 | require "spec" 2 | require "../../src/service/import_categories" 3 | require "../../src/catalog" 4 | require "../support/db" 5 | require "../support/raven" 6 | require "../support/tempdir" 7 | 8 | describe Service::ImportCategories do 9 | it "creates category" do 10 | with_tempdir("import_categories") do |catalog_path| 11 | transaction do |db| 12 | File.write(File.join(catalog_path, "bar.yml"), <<-YAML) 13 | name: Bar 14 | description: bardesc 15 | YAML 16 | 17 | catalog = Catalog.read(catalog_path) 18 | service = Service::ImportCategories.new(db, catalog) 19 | 20 | import_stats = service.perform 21 | db.all_categories.map { |cat| {cat.name, cat.description} }.should eq [{"Bar", "bardesc"}] 22 | 23 | import_stats.should eq({ 24 | "deleted_categories" => [] of String, 25 | "new_categories" => ["bar"], 26 | "updated_categories" => [] of String, 27 | }) 28 | end 29 | end 30 | end 31 | 32 | it "deletes category" do 33 | with_tempdir("import_categories") do |catalog_path| 34 | transaction do |db| 35 | db.create_category(Category.new("foo", "Foo", "Foo Description")) 36 | 37 | catalog = Catalog.new(catalog_path) 38 | service = Service::ImportCategories.new(db, catalog) 39 | 40 | import_stats = service.perform 41 | db.all_categories.should be_empty 42 | 43 | import_stats.should eq({ 44 | "deleted_categories" => ["foo"], 45 | "new_categories" => [] of String, 46 | "updated_categories" => [] of String, 47 | }) 48 | end 49 | end 50 | end 51 | 52 | it "is idempotent" do 53 | with_tempdir("import_categories") do |catalog_path| 54 | File.write(File.join(catalog_path, "foo.yml"), <<-YAML) 55 | name: Foo 56 | description: Foo Description 57 | YAML 58 | 59 | transaction do |db| 60 | db.create_category(Category.new("foo", "Foo", "Foo Description")) 61 | 62 | catalog = Catalog.read(catalog_path) 63 | service = Service::ImportCategories.new(db, catalog) 64 | 65 | import_stats = service.perform 66 | db.all_categories.map { |cat| {cat.name, cat.description} }.should eq [{"Foo", "Foo Description"}] 67 | 68 | import_stats.should eq({ 69 | "deleted_categories" => [] of String, 70 | "new_categories" => [] of String, 71 | "updated_categories" => [] of String, 72 | }) 73 | end 74 | end 75 | end 76 | 77 | it "updates category" do 78 | with_tempdir("import_categories") do |catalog_path| 79 | File.write(File.join(catalog_path, "foo.yml"), <<-YAML) 80 | name: New Foo 81 | description: New Foo Description 82 | YAML 83 | 84 | transaction do |db| 85 | db.create_category(Category.new("foo", "Foo", "Foo Description")) 86 | 87 | catalog = Catalog.read(catalog_path) 88 | service = Service::ImportCategories.new(db, catalog) 89 | 90 | import_stats = service.perform 91 | db.all_categories.map { |cat| {cat.name, cat.description} }.should eq [{"New Foo", "New Foo Description"}] 92 | 93 | import_stats.should eq({ 94 | "deleted_categories" => [] of String, 95 | "new_categories" => [] of String, 96 | "updated_categories" => ["foo"], 97 | }) 98 | end 99 | end 100 | end 101 | end 102 | -------------------------------------------------------------------------------- /spec/service/import_shard_spec.cr: -------------------------------------------------------------------------------- 1 | require "spec" 2 | require "../../src/service/import_shard" 3 | require "../../src/catalog" 4 | require "../support/raven" 5 | require "../support/db" 6 | require "../support/mock_resolver" 7 | 8 | private def shard_categorizations(db) 9 | db.connection.query_all <<-SQL, as: {String, String, Array(String)?} 10 | SELECT 11 | name::text, qualifier::text, 12 | (SELECT array_agg(categories.slug::text) FROM categories WHERE shards.categories @> ARRAY[categories.id]) 13 | FROM 14 | shards 15 | ORDER BY 16 | name, qualifier 17 | SQL 18 | end 19 | 20 | describe Service::ImportShard do 21 | it "creates new shard" do 22 | repo_ref = Repo::Ref.new("git", "mock:test") 23 | 24 | transaction do |db| 25 | repo_id = Factory.create_repo(db, repo_ref, nil) 26 | 27 | mock_resolver = MockResolver.new 28 | mock_resolver.register("0.1.0", Factory.build_revision_info, <<-SPEC) 29 | name: test 30 | description: test shard 31 | version: 0.1.0 32 | SPEC 33 | 34 | shard_id = Service::ImportShard.new(db, 35 | db.get_repo(repo_id), 36 | entry: Catalog::Entry.new(repo_ref, description: "foo description"), 37 | resolver: Repo::Resolver.new(mock_resolver, repo_ref), 38 | ).perform 39 | 40 | ShardsDBHelper.persisted_shards(db).should eq [{"test", "", "foo description"}] 41 | 42 | repo_id = db.get_repo(repo_ref).id 43 | db.repos_pending_sync.map(&.ref).should eq [repo_ref] 44 | 45 | shard_categorizations(db).should eq [ 46 | {"test", "", nil}, 47 | ] 48 | 49 | db.last_activities.map { |a| {a.event, a.repo_id, a.shard_id, a.metadata} }.should eq [ 50 | {"create_shard:created", repo_id, shard_id, nil}, 51 | ] 52 | end 53 | end 54 | 55 | it "normalizes shard name" do 56 | repo_ref = Repo::Ref.new("git", "mock:test") 57 | 58 | transaction do |db| 59 | repo_id = Factory.create_repo(db, repo_ref, nil) 60 | 61 | mock_resolver = MockResolver.new 62 | mock_resolver.register("0.1.0", Factory.build_revision_info, <<-SPEC) 63 | name: foo bar 64 | version: 0.1.0 65 | SPEC 66 | 67 | shard_id = Service::ImportShard.new(db, 68 | db.get_repo(repo_id), 69 | resolver: Repo::Resolver.new(mock_resolver, repo_ref), 70 | ).perform 71 | 72 | ShardsDBHelper.persisted_shards(db).empty?.should be_true 73 | end 74 | end 75 | 76 | it "uses existing repo" do 77 | repo_ref = Repo::Ref.new("git", "mock://example.com/git/test.git") 78 | 79 | transaction do |db| 80 | repo_id = Factory.create_repo(db, repo_ref, nil) 81 | 82 | mock_resolver = MockResolver.new 83 | mock_resolver.register("0.1.0", Factory.build_revision_info, <<-SPEC) 84 | name: test 85 | description: test shard 86 | version: 0.1.0 87 | SPEC 88 | 89 | shard_id = Service::ImportShard.new(db, 90 | db.get_repo(repo_id), 91 | resolver: Repo::Resolver.new(mock_resolver, repo_ref), 92 | ).perform 93 | 94 | ShardsDBHelper.persisted_shards(db).should eq [{"test", "", nil}] 95 | 96 | db.repos_pending_sync.map(&.ref).should eq [repo_ref] 97 | 98 | db.last_activities.map { |a| {a.event, a.repo_id, a.shard_id, a.metadata} }.should eq [ 99 | {"create_shard:created", repo_id, shard_id, nil}, 100 | ] 101 | end 102 | end 103 | end 104 | -------------------------------------------------------------------------------- /spec/service/order_releases_spec.cr: -------------------------------------------------------------------------------- 1 | require "spec" 2 | require "../../src/service/order_releases" 3 | require "../support/db" 4 | 5 | describe Service::OrderReleases do 6 | it "orders releases by version number" do 7 | transaction do |db| 8 | shard_id = Factory.create_shard(db) 9 | 10 | versions = {"0.1.0", "0.2.0", "5.333.1", "5.2.1", "0.2", "0.2.0.1", "5.8", "0.0.0.11"} 11 | 12 | versions.each_with_index do |version, index| 13 | Factory.create_release(db, shard_id, version, Time.utc, position: index) 14 | end 15 | 16 | Service::OrderReleases.new(db, shard_id).perform 17 | 18 | results = db.connection.query_all <<-SQL, shard_id, as: {String} 19 | SELECT version 20 | FROM releases 21 | WHERE shard_id = $1 22 | ORDER BY position 23 | SQL 24 | 25 | results.should eq ["0.0.0.11", "0.1.0", "0.2", "0.2.0", "0.2.0.1", "5.2.1", "5.8", "5.333.1"] 26 | 27 | results = db.connection.query_all <<-SQL, shard_id, as: {String} 28 | SELECT version 29 | FROM releases 30 | WHERE shard_id = $1 AND latest = true 31 | SQL 32 | 33 | results.should eq ["5.333.1"] 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /spec/service/sync_dependencies_spec.cr: -------------------------------------------------------------------------------- 1 | require "spec" 2 | require "../../src/service/sync_dependencies" 3 | require "../../src/repo" 4 | require "../../src/repo/resolver" 5 | require "../support/db" 6 | require "../support/mock_resolver" 7 | 8 | private def query_dependencies_with_repo(db) 9 | db.connection.query_all <<-SQL, as: {String, JSON::Any, String, Int64?, String, String, String} 10 | SELECT 11 | name::text, spec, scope::text, 12 | repos.shard_id, repos.resolver::text, repos.url::text, repos.role::text 13 | FROM 14 | dependencies 15 | JOIN 16 | repos ON repo_id = repos.id 17 | SQL 18 | end 19 | 20 | describe Service::SyncDependencies do 21 | describe "#add_dependency" do 22 | context "non-existing repo" do 23 | it "creates dependency and repo" do 24 | transaction do |db| 25 | shard_id = Factory.create_shard(db) 26 | release_version = "0.1.0" 27 | release_id = Factory.create_release(db, version: release_version, shard_id: shard_id) 28 | 29 | spec = JSON.parse(%({"git":"foo"})) 30 | 31 | service = Service::SyncDependencies.new(db, release_id) 32 | service.add_dependency(Dependency.new("foo", spec)) 33 | 34 | results = query_dependencies_with_repo(db) 35 | 36 | results.should eq [ 37 | {"foo", spec, "runtime", nil, "git", "foo", "canonical"}, 38 | ] 39 | 40 | db.repos_pending_sync.map(&.ref).should eq [Repo::Ref.new("git", "foo")] 41 | 42 | repo_id = db.get_repo_id("git", "foo") 43 | db.last_activities.map { |a| {a.event, a.repo_id, a.shard_id, a.metadata} }.should eq [ 44 | {"sync_dependencies:created", repo_id, shard_id, {"name" => "foo", "scope" => "runtime", "release" => release_version}}, 45 | ] 46 | end 47 | end 48 | end 49 | 50 | describe "#update_dependency" do 51 | it "overrides homonymous dependency name" do 52 | transaction do |db| 53 | shard_id = Factory.create_shard(db) 54 | release_version = "0.1.0" 55 | release_id = Factory.create_release(db, version: release_version, shard_id: shard_id) 56 | 57 | service = Service::SyncDependencies.new(db, release_id) 58 | 59 | spec_bar = JSON.parse(%({"git":"bar"})) 60 | service.add_dependency(Dependency.new("foo", spec_bar, :development)) 61 | spec_foo = JSON.parse(%({"git":"foo"})) 62 | service.add_dependency(Dependency.new("foo", spec_foo)) 63 | 64 | results = query_dependencies_with_repo(db) 65 | 66 | results.should eq [ 67 | {"foo", spec_foo, "runtime", nil, "git", "foo", "canonical"}, 68 | ] 69 | 70 | db.repos_pending_sync.map(&.ref).should eq [Repo::Ref.new("git", "bar"), Repo::Ref.new("git", "foo")] 71 | 72 | foo_repo_id = db.get_repo_id("git", "foo") 73 | bar_repo_id = db.get_repo_id("git", "bar") 74 | db.last_activities.map { |a| {a.event, a.repo_id, a.shard_id, a.metadata} }.should eq [ 75 | {"sync_dependencies:created", bar_repo_id, shard_id, {"name" => "foo", "scope" => "development", "release" => release_version}}, 76 | {"sync_dependencies:duplicate", foo_repo_id, shard_id, {"name" => "foo", "scope" => "runtime", "release" => release_version}}, 77 | {"sync_dependencies:updated", foo_repo_id, shard_id, {"name" => "foo", "scope" => "runtime", "release" => release_version}}, 78 | ] 79 | end 80 | end 81 | end 82 | 83 | context "existing repo" do 84 | it "creates dependency and links repo" do 85 | transaction do |db| 86 | shard_id = Factory.create_shard(db) 87 | release_id = Factory.create_release(db, shard_id: shard_id) 88 | bar_shard_id = Factory.create_shard(db, "bar") 89 | repo_foo = db.create_repo Repo.new(Repo::Ref.new("git", "foo"), nil, :canonical, synced_at: Time.utc) 90 | repo_bar = db.create_repo Repo.new(Repo::Ref.new("git", "bar"), bar_shard_id, :canonical, synced_at: Time.utc) 91 | 92 | service = Service::SyncDependencies.new(db, release_id) 93 | 94 | spec_foo = JSON.parse(%({"git":"foo"})) 95 | service.add_dependency(Dependency.new("foo", spec_foo)) 96 | spec_bar = JSON.parse(%({"git":"bar"})) 97 | service.add_dependency(Dependency.new("bar", spec_bar)) 98 | 99 | results = query_dependencies_with_repo(db) 100 | 101 | results.should eq [ 102 | {"foo", spec_foo, "runtime", nil, "git", "foo", "canonical"}, 103 | {"bar", spec_bar, "runtime", bar_shard_id, "git", "bar", "canonical"}, 104 | ] 105 | 106 | db.repos_pending_sync.should eq [] of Repo 107 | 108 | db.last_activities.map { |a| {a.event, a.repo_id, a.shard_id, a.metadata} }.should eq [ 109 | {"sync_dependencies:created", repo_foo, shard_id, {"name" => "foo", "scope" => "runtime", "release" => "0.1.0"}}, 110 | {"sync_dependencies:created", repo_bar, shard_id, {"name" => "bar", "scope" => "runtime", "release" => "0.1.0"}}, 111 | ] 112 | end 113 | end 114 | 115 | it "updates dependency" do 116 | transaction do |db| 117 | home_shard = Factory.create_shard(db, "qux") 118 | release_id = Factory.create_release(db, home_shard) 119 | shard_id = Factory.create_shard(db, "bar") 120 | repo_foo = db.create_repo Repo.new(Repo::Ref.new("git", "foo"), nil, :canonical, synced_at: Time.utc) 121 | repo_bar = db.create_repo Repo.new(Repo::Ref.new("git", "bar"), shard_id, :canonical, synced_at: Time.utc) 122 | 123 | service = Service::SyncDependencies.new(db, release_id) 124 | 125 | spec_foo = JSON.parse(%({"git":"foo"})) 126 | service.add_dependency(Dependency.new("foo", spec_foo)) 127 | spec_bar = JSON.parse(%({"git":"bar"})) 128 | service.update_dependency(Dependency.new("foo", spec_bar, :development)) 129 | 130 | results = query_dependencies_with_repo(db) 131 | results.should eq [ 132 | {"foo", spec_bar, "development", shard_id, "git", "bar", "canonical"}, 133 | ] 134 | 135 | db.repos_pending_sync.should eq [] of Repo 136 | 137 | db.last_activities.map { |a| {a.event, a.repo_id, a.shard_id, a.metadata} }.should eq [ 138 | {"sync_dependencies:created", repo_foo, home_shard, {"name" => "foo", "scope" => "runtime", "release" => "0.1.0"}}, 139 | {"sync_dependencies:updated", repo_bar, home_shard, {"name" => "foo", "scope" => "development", "release" => "0.1.0"}}, 140 | ] 141 | end 142 | end 143 | end 144 | end 145 | 146 | it "#sync_dependencies" do 147 | transaction do |db| 148 | shard_id = Factory.create_shard(db) 149 | release_id = Factory.create_release(db, shard_id: shard_id) 150 | 151 | run_spec = JSON.parse(%({"git":"run"})) 152 | dev_spec = JSON.parse(%({"git":"dev"})) 153 | dependencies = [ 154 | Dependency.new("run_dependency", run_spec), 155 | Dependency.new("dev_dependency", dev_spec, :development), 156 | ] 157 | 158 | service = Service::SyncDependencies.new(db, release_id) 159 | service.sync_dependencies(dependencies) 160 | 161 | results = query_dependencies_with_repo(db) 162 | 163 | results.should eq [ 164 | {"run_dependency", run_spec, "runtime", nil, "git", "run", "canonical"}, 165 | {"dev_dependency", dev_spec, "development", nil, "git", "dev", "canonical"}, 166 | ] 167 | 168 | run_spec2 = JSON.parse(%({"git":"run2"})) 169 | new_dependencies = [ 170 | Dependency.new("run_dependency", run_spec), 171 | Dependency.new("run_dependency2", run_spec2), 172 | ] 173 | 174 | service.sync_dependencies(new_dependencies) 175 | results = query_dependencies_with_repo(db) 176 | 177 | results.should eq [ 178 | {"run_dependency", run_spec, "runtime", nil, "git", "run", "canonical"}, 179 | {"run_dependency2", run_spec2, "runtime", nil, "git", "run2", "canonical"}, 180 | ] 181 | db.last_activities.map { |a| {a.event, a.repo_id, a.shard_id, a.metadata} }.should eq [ 182 | {"sync_dependencies:created", db.get_repo_id?("git", "run"), shard_id, {"name" => "run_dependency", "scope" => "runtime", "release" => "0.1.0"}}, 183 | {"sync_dependencies:created", db.get_repo_id?("git", "dev"), shard_id, {"name" => "dev_dependency", "scope" => "development", "release" => "0.1.0"}}, 184 | {"sync_dependencies:removed", db.get_repo_id?("git", "dev"), shard_id, {"name" => "dev_dependency", "scope" => "development", "release" => "0.1.0"}}, 185 | {"sync_dependencies:created", db.get_repo_id?("git", "run2"), shard_id, {"name" => "run_dependency2", "scope" => "runtime", "release" => "0.1.0"}}, 186 | ] 187 | end 188 | end 189 | end 190 | -------------------------------------------------------------------------------- /spec/service/sync_release_spec.cr: -------------------------------------------------------------------------------- 1 | require "spec" 2 | require "../../src/service/sync_release" 3 | require "../../src/repo" 4 | require "../../src/repo/resolver" 5 | require "../support/db" 6 | require "../support/mock_resolver" 7 | require "../support/raven" 8 | 9 | describe Service::SyncRelease do 10 | commit_1 = Factory.build_commit("12345678") 11 | revision_info_1 = Release::RevisionInfo.new Factory.build_tag("v0.1.0"), commit_1 12 | mock_resolver = MockResolver.new 13 | mock_resolver.register("0.1.0", revision_info_1, <<-SPEC) 14 | name: foo 15 | version: 0.1.0 16 | SPEC 17 | 18 | it "stores new release" do 19 | transaction do |db| 20 | shard_id = Factory.create_shard(db) 21 | # repo_id = Factory.create_repo(db, shard_id: shard_id) 22 | 23 | service = Service::SyncRelease.new(db, shard_id, "0.1.0") 24 | 25 | service.sync_release(Repo::Resolver.new(mock_resolver, Repo::Ref.new("git", "foo"))) 26 | 27 | results = db.connection.query_all <<-SQL, as: {Int64, String, Time, JSON::Any, JSON::Any, Int32?, Bool?, Time?} 28 | SELECT 29 | shard_id, version, released_at, spec, revision_info, position, latest, yanked_at 30 | FROM releases 31 | SQL 32 | 33 | results.size.should eq 1 34 | row = results.first 35 | row[0].should eq shard_id 36 | row[1].should eq "0.1.0" 37 | row[2].should eq commit_1.time 38 | row[3].should eq JSON.parse(%({"name":"foo","version":"0.1.0"})) 39 | row[4].should eq JSON.parse(revision_info_1.to_json) 40 | row[5].should eq 0 41 | row[6].should eq nil 42 | row[7].should eq nil 43 | 44 | db.last_activities.map { |a| {a.event, a.repo_id, a.shard_id, a.metadata} }.should eq [ 45 | {"sync_release:created", nil, shard_id, {"version" => "0.1.0"}}, 46 | ] 47 | end 48 | end 49 | 50 | it "updates existing release" do 51 | transaction do |db| 52 | shard_id = Factory.create_shard(db) 53 | # repo_id = Factory.create_repo(db, shard_id: shard_id) 54 | db.connection.exec <<-SQL, shard_id 55 | INSERT INTO releases (shard_id, version, released_at, spec, revision_info, position, latest, yanked_at) 56 | VALUES($1, '0.1.0', '2018-12-30 00:00:00 UTC', '{}', '{}', 1, true, NOW()) 57 | SQL 58 | 59 | service = Service::SyncRelease.new(db, shard_id, "0.1.0") 60 | 61 | service.sync_release(Repo::Resolver.new(mock_resolver, Repo::Ref.new("git", "foo"))) 62 | 63 | results = db.connection.query_all <<-SQL, as: {Int64, String, Time, JSON::Any, JSON::Any, Int32?, Bool?, Time?} 64 | SELECT 65 | shard_id, version, released_at, spec, revision_info, position, latest, yanked_at 66 | FROM releases 67 | SQL 68 | 69 | results.size.should eq 1 70 | row = results.first 71 | row[0].should eq shard_id 72 | row[1].should eq "0.1.0" 73 | row[2].should eq commit_1.time 74 | row[3].should eq JSON.parse(%({"name":"foo","version":"0.1.0"})) 75 | row[4].should eq JSON.parse(revision_info_1.to_json) 76 | row[5].should eq 1 77 | row[6].should eq true 78 | row[7].should eq nil 79 | 80 | db.last_activities.should eq [] of LogActivity 81 | end 82 | end 83 | 84 | it "handles missing spec" do 85 | commit_1 = Factory.build_commit("12345678") 86 | revision_info_1 = Release::RevisionInfo.new Factory.build_tag("v0.1.0"), commit_1 87 | mock_resolver = MockResolver.new 88 | mock_resolver.register("0.1.0", revision_info_1, nil) 89 | 90 | transaction do |db| 91 | shard_id = Factory.create_shard(db) 92 | service = Service::SyncRelease.new(db, shard_id, "0.1.0") 93 | resolver = Repo::Resolver.new(mock_resolver, Repo::Ref.new("git", "foo")) 94 | 95 | service.sync_release(resolver) 96 | 97 | results = db.connection.query_all <<-SQL, as: {Int64, Int64, String, Time, JSON::Any, JSON::Any, Int32?, Bool?, Time?} 98 | SELECT 99 | id, shard_id, version, released_at, spec, revision_info, position, latest, yanked_at 100 | FROM 101 | releases 102 | SQL 103 | 104 | results.size.should eq 1 105 | 106 | release_id, result_shard_id, version, released_at, spec, revision_info, position, latest, yanked_at = results.first 107 | result_shard_id.should eq shard_id 108 | version.should eq "0.1.0" 109 | released_at.should eq commit_1.time 110 | spec.should eq JSON.parse(%({})) 111 | revision_info.should eq JSON.parse(revision_info_1.to_json) 112 | position.should eq 0 113 | latest.should be_nil 114 | yanked_at.should be_nil 115 | 116 | results = db.connection.query_one(<<-SQL, release_id, as: {Int64}).should eq 0 117 | SELECT 118 | COUNT(*) 119 | FROM 120 | dependencies 121 | WHERE 122 | release_id = $1 123 | SQL 124 | 125 | db.last_activities.map { |a| {a.event, a.repo_id, a.shard_id, a.metadata} }.should eq [ 126 | {"sync_release:created", nil, shard_id, {"version" => "0.1.0"}}, 127 | ] 128 | end 129 | end 130 | 131 | describe "#sync_files" do 132 | it "stores README" do 133 | commit_1 = Factory.build_commit("12345678") 134 | revision_info_1 = Release::RevisionInfo.new Factory.build_tag("v0.1.0"), commit_1 135 | mock_resolver = MockResolver.new 136 | version = mock_resolver.register("0.1.0", revision_info_1, <<-SPEC) 137 | name: foo 138 | version: 0.1.0 139 | SPEC 140 | transaction do |db| 141 | shard_id = Factory.create_shard(db) 142 | release_id = Factory.create_release(db, shard_id, "0.1.0") 143 | 144 | service = Service::SyncRelease.new(db, shard_id, "0.1.0") 145 | 146 | version.files["README.md"] = "Hello World!" 147 | service.sync_files(release_id, Repo::Resolver.new(mock_resolver, Repo::Ref.new("git", "foo"))) 148 | db.fetch_file(release_id, "README.md").should eq "Hello World!" 149 | 150 | version.files["README.md"] = "Hello Foo!" 151 | service.sync_files(release_id, Repo::Resolver.new(mock_resolver, Repo::Ref.new("git", "foo"))) 152 | db.fetch_file(release_id, "README.md").should eq "Hello Foo!" 153 | 154 | version.files.delete("README.md") 155 | service.sync_files(release_id, Repo::Resolver.new(mock_resolver, Repo::Ref.new("git", "foo"))) 156 | db.fetch_file(release_id, "README.md").should be_nil 157 | 158 | version.files["Readme.md"] = "Hello Camel!" 159 | service.sync_files(release_id, Repo::Resolver.new(mock_resolver, Repo::Ref.new("git", "foo"))) 160 | db.fetch_file(release_id, "README.md").should eq "Hello Camel!" 161 | 162 | version.files["README.md"] = "Hello YELL!" 163 | service.sync_files(release_id, Repo::Resolver.new(mock_resolver, Repo::Ref.new("git", "foo"))) 164 | db.fetch_file(release_id, "README.md").should eq "Hello YELL!" 165 | end 166 | end 167 | end 168 | end 169 | -------------------------------------------------------------------------------- /spec/service/sync_repo_spec.cr: -------------------------------------------------------------------------------- 1 | require "spec" 2 | require "../../src/service/sync_repo" 3 | require "../support/db" 4 | require "../support/mock_resolver" 5 | require "../support/fetcher_mocks" 6 | require "../support/raven" 7 | 8 | describe Service::SyncRepo do 9 | describe "#sync_repo" do 10 | it "successfully syncs" do 11 | transaction do |db| 12 | repo_ref = Repo::Ref.new("git", "foo") 13 | shard_id = Factory.create_shard(db, "foo") 14 | repo_id = Factory.create_repo(db, repo_ref, role: :mirror, shard_id: shard_id) 15 | service = Service::SyncRepo.new(db, repo_ref) 16 | 17 | resolver = Repo::Resolver.new(MockResolver.new, repo_ref) 18 | 19 | db.last_repo_activity.should eq(nil) 20 | 21 | service.sync_repo(resolver) 22 | 23 | repo = db.get_repo(repo_ref) 24 | repo.sync_failed_at.should be_nil 25 | repo.synced_at.should_not be_nil 26 | 27 | db.last_repo_activity.should eq({repo_id, "sync_repo:synced"}) 28 | 29 | service.sync_repo(resolver) 30 | end 31 | end 32 | 33 | it "handles unresolvable repo" do 34 | transaction do |db| 35 | repo_ref = Repo::Ref.new("git", "foo") 36 | repo_id = Factory.create_repo(db, repo_ref) 37 | service = Service::SyncRepo.new(db, repo_ref) 38 | 39 | resolver = Repo::Resolver.new(MockResolver.unresolvable, repo_ref) 40 | service.sync_repo(resolver) 41 | 42 | repo = db.get_repo(repo_ref) 43 | repo.sync_failed_at.should_not be_nil 44 | repo.synced_at.should be_nil 45 | 46 | db.last_repo_activity.should eq({repo_id, "sync_repo:fetch_spec_failed"}) 47 | end 48 | end 49 | end 50 | 51 | describe "#sync_releases" do 52 | it "skips new invalid release" do 53 | transaction do |db| 54 | shard_id = Factory.create_shard(db) 55 | repo_ref = Repo::Ref.new("git", "foo") 56 | repo_id = Factory.create_repo(db, repo_ref, shard_id: shard_id) 57 | mock_resolver = MockResolver.new 58 | mock_resolver.register "0.1.0", Factory.build_revision_info, spec: "" 59 | resolver = Repo::Resolver.new(mock_resolver, repo_ref) 60 | 61 | service = Service::SyncRepo.new(db, repo_ref) 62 | service.sync_releases(resolver, shard_id) 63 | 64 | db.last_activities.map { |a| {a.event, a.repo_id, a.shard_id, a.metadata} }.should eq [ 65 | {"sync_repo:sync_release:failed", repo_id, shard_id, { 66 | "error_message" => "Expected DOCUMENT_START but was STREAM_END at line 1, column 1", 67 | "version" => "0.1.0", 68 | "repo_role" => "canonical", 69 | "exception" => "Shards::ParseError", 70 | }}, 71 | ] 72 | end 73 | end 74 | 75 | it "yanks existing invalid release" do 76 | transaction do |db| 77 | shard_id = Factory.create_shard(db, "foo") 78 | repo_ref = Repo::Ref.new("git", "foo") 79 | repo_id = Factory.create_repo(db, repo_ref, shard_id: shard_id) 80 | Factory.create_release(db, shard_id, "0.1.0") 81 | mock_resolver = MockResolver.new 82 | mock_resolver.register "0.1.0", Factory.build_revision_info, spec: "" 83 | mock_resolver.register "0.2.0", Factory.build_revision_info, spec: nil 84 | resolver = Repo::Resolver.new(mock_resolver, repo_ref) 85 | 86 | service = Service::SyncRepo.new(db, repo_ref) 87 | service.sync_releases(resolver, shard_id) 88 | 89 | db.all_releases(shard_id).map { |r| {r.version.to_s, r.yanked?} }.should eq [{"0.2.0", false}, {"0.1.0", true}] 90 | db.last_activities.map { |a| {a.event, a.repo_id, a.shard_id, a.metadata} }.should eq [ 91 | {"sync_repo:sync_release:failed", repo_id, shard_id, { 92 | "error_message" => "Expected DOCUMENT_START but was STREAM_END at line 1, column 1", 93 | "version" => "0.1.0", 94 | "repo_role" => "canonical", 95 | "exception" => "Shards::ParseError", 96 | }}, 97 | {"sync_release:created", nil, shard_id, {"version" => "0.2.0"}}, 98 | {"sync_repo:release:yanked", nil, shard_id, {"version" => "0.1.0"}}, 99 | ] 100 | end 101 | end 102 | 103 | it "uses HEAD when no releases tagged" do 104 | transaction do |db| 105 | shard_id = Factory.create_shard(db, "foo") 106 | repo_ref = Repo::Ref.new("git", "foo") 107 | repo_id = Factory.create_repo(db, repo_ref, shard_id: shard_id) 108 | mock_resolver = MockResolver.new 109 | mock_resolver.register "HEAD", Factory.build_revision_info(tag: nil), spec: nil 110 | resolver = Repo::Resolver.new(mock_resolver, repo_ref) 111 | 112 | service = Service::SyncRepo.new(db, repo_ref) 113 | service.sync_releases(resolver, shard_id) 114 | 115 | db.all_releases(shard_id).map { |r| {r.version.to_s, r.yanked?} }.should eq [{"HEAD", false}] 116 | db.last_activities.map { |a| {a.event, a.repo_id, a.shard_id, a.metadata} }.should eq [ 117 | {"sync_release:created", nil, shard_id, {"version" => "HEAD"}}, 118 | ] 119 | end 120 | end 121 | 122 | it "yanks HEAD when version found" do 123 | transaction do |db| 124 | shard_id = Factory.create_shard(db, "foo") 125 | repo_ref = Repo::Ref.new("git", "foo") 126 | repo_id = Factory.create_repo(db, repo_ref, shard_id: shard_id) 127 | Factory.create_release(db, shard_id, "HEAD") 128 | mock_resolver = MockResolver.new 129 | mock_resolver.register "0.1.0", Factory.build_revision_info, spec: nil 130 | resolver = Repo::Resolver.new(mock_resolver, repo_ref) 131 | 132 | service = Service::SyncRepo.new(db, repo_ref) 133 | service.sync_releases(resolver, shard_id) 134 | 135 | db.all_releases(shard_id).map { |r| {r.version.to_s, r.yanked?} }.should eq [{"HEAD", true}, {"0.1.0", false}] 136 | db.last_activities.map { |a| {a.event, a.repo_id, a.shard_id, a.metadata} }.should eq [ 137 | {"sync_release:created", nil, shard_id, {"version" => "0.1.0"}}, 138 | {"sync_repo:release:yanked", nil, shard_id, {"version" => "HEAD"}}, 139 | ] 140 | end 141 | end 142 | 143 | it "inserts HEAD when all versions yanked" do 144 | transaction do |db| 145 | shard_id = Factory.create_shard(db, "foo") 146 | repo_ref = Repo::Ref.new("git", "foo") 147 | repo_id = Factory.create_repo(db, repo_ref, shard_id: shard_id) 148 | Factory.create_release(db, shard_id, "0.1.0") 149 | 150 | mock_resolver = MockResolver.new 151 | mock_resolver.register "HEAD", Factory.build_revision_info, spec: nil 152 | resolver = Repo::Resolver.new(mock_resolver, repo_ref) 153 | 154 | service = Service::SyncRepo.new(db, repo_ref) 155 | service.sync_releases(resolver, shard_id) 156 | 157 | db.all_releases(shard_id).map { |r| {r.version.to_s, r.yanked?} }.should eq [{"HEAD", false}, {"0.1.0", true}] 158 | db.last_activities.map { |a| {a.event, a.repo_id, a.shard_id, a.metadata} }.should eq [ 159 | {"sync_release:created", nil, shard_id, {"version" => "HEAD"}}, 160 | {"sync_repo:release:yanked", nil, shard_id, {"version" => "0.1.0"}}, 161 | ] 162 | end 163 | end 164 | end 165 | 166 | it "yanks removed releases" do 167 | transaction do |db| 168 | shard_id = Factory.create_shard(db) 169 | db.connection.exec <<-SQL, shard_id 170 | INSERT INTO releases (shard_id, version, released_at, spec, revision_info, position, latest, yanked_at) 171 | VALUES ($1, '0.1.0', '2018-12-30 00:00:00 UTC', '{}', '{}', 4, NULL, NULL), 172 | ($1, '0.1.1', '2018-12-30 00:00:01 UTC', '{}', '{}', 1, NULL, NOW()), 173 | ($1, '0.1.2', '2018-12-30 00:00:00 UTC', '{}', '{}', 10, NULL, NULL), 174 | ($1, '0.1.3', '2018-12-30 00:00:02 UTC', '{}', '{}', 3, true, NULL) 175 | SQL 176 | 177 | service = Service::SyncRepo.new(db, Repo::Ref.new("git", "blank")) 178 | 179 | valid_versions = ["0.1.0", "0.1.2"] 180 | service.yank_releases_with_missing_versions(shard_id, valid_versions) 181 | 182 | results = db.connection.query_all <<-SQL, as: {String, Bool?, Bool} 183 | SELECT 184 | version, latest, yanked_at IS NULL 185 | FROM releases 186 | ORDER BY position 187 | SQL 188 | 189 | results.should eq [{"0.1.1", nil, false}, {"0.1.3", true, false}, {"0.1.0", nil, true}, {"0.1.2", nil, true}] 190 | 191 | db.last_activities.map { |a| {a.event, a.repo_id, a.shard_id, a.metadata} }.should eq [ 192 | {"sync_repo:release:yanked", nil, shard_id, {"version" => "0.1.3"}}, 193 | ] 194 | end 195 | end 196 | 197 | it "#sync_metadata" do 198 | transaction do |db| 199 | repo_id = db.connection.query_one <<-SQL, as: Int64 200 | INSERT INTO repos 201 | (resolver, url, synced_at, sync_failed_at) 202 | VALUES 203 | ('git', 'foo', NOW() - interval '1h', NOW()) 204 | RETURNING id 205 | SQL 206 | 207 | repo_ref = Repo::Ref.new("git", "foo") 208 | service = Service::SyncRepo.new(db, repo_ref) 209 | 210 | repo = Repo.new(repo_ref, nil, id: repo_id) 211 | service.sync_metadata(repo, fetch_service: MockFetchMetadata.new(nil)) 212 | results = db.connection.query_all <<-SQL, as: {JSON::Any, Bool, Bool} 213 | SELECT 214 | metadata, synced_at > NOW() - interval '1s', sync_failed_at IS NOT NULL 215 | FROM repos 216 | SQL 217 | 218 | results.should eq [{JSON.parse(%({})), false, true}] 219 | 220 | service.sync_metadata(repo, fetch_service: MockFetchMetadata.new(Repo::Metadata.new(forks_count: 42))) 221 | 222 | results = db.connection.query_all <<-SQL, as: {JSON::Any, Bool, Bool} 223 | SELECT 224 | metadata, synced_at > NOW() - interval '1s', sync_failed_at IS NOT NULL 225 | FROM repos 226 | SQL 227 | 228 | results.should eq [{JSON.parse(%({"forks_count": 42})), true, false}] 229 | end 230 | end 231 | 232 | it "#sync_owner" do 233 | transaction do |db| 234 | repo_id = db.connection.query_one <<-SQL, as: Int64 235 | INSERT INTO repos 236 | (resolver, url, synced_at, sync_failed_at) 237 | VALUES 238 | ('github', 'foo/bar', NOW() - interval '1h', NOW()) 239 | RETURNING id 240 | SQL 241 | 242 | repo_ref = Repo::Ref.new("github", "foo/bar") 243 | repo = Repo.new(repo_ref, nil, id: repo_id) 244 | 245 | api = Shardbox::GitHubAPI.new("") 246 | api.mock_owner_info = Hash(String, JSON::Any).from_json(<<-JSON) 247 | { 248 | "name": "Foo", 249 | "exxxtra": "big" 250 | } 251 | JSON 252 | 253 | create_owner = Service::CreateOwner.new(db, repo_ref) 254 | create_owner.github_api = api 255 | 256 | service = Service::SyncRepo.new(db, repo_ref) 257 | service.sync_owner(repo, service: create_owner) 258 | 259 | results = db.connection.query_all <<-SQL, as: {String, String, String?, JSON::Any} 260 | SELECT 261 | owners.resolver::text, slug::text, name, extra 262 | FROM owners 263 | JOIN repos 264 | ON repos.owner_id = owners.id 265 | SQL 266 | 267 | results.should eq [{"github", "foo", "Foo", JSON.parse(%({"exxxtra": "big"}))}] 268 | end 269 | end 270 | end 271 | -------------------------------------------------------------------------------- /spec/service/update_dependencies_spec.cr: -------------------------------------------------------------------------------- 1 | require "spec" 2 | require "../../src/service/sync_repos" 3 | require "../support/db" 4 | require "../support/factory" 5 | require "../support/raven" 6 | 7 | private def persisted_dependencies(db) 8 | db.connection.query_all <<-SQL, as: {Int64, Int64?, Int64, String} 9 | SELECT 10 | shard_id, depends_on, depends_on_repo_id, scope::text 11 | FROM 12 | shard_dependencies 13 | ORDER BY 14 | shard_id, depends_on 15 | SQL 16 | end 17 | 18 | private def dependents_stats(db) 19 | db.connection.query_all <<-SQL, as: {Int64, Int32?, Int32?, Int32?} 20 | SELECT 21 | shard_id, dependents_count, dev_dependents_count, transitive_dependents_count 22 | FROM 23 | shard_metrics_current 24 | WHERE 25 | dependents_count > 0 OR dev_dependents_count > 0 OR transitive_dependents_count > 0 26 | ORDER BY 27 | id 28 | SQL 29 | end 30 | 31 | private def dependencies_stats(db) 32 | db.connection.query_all <<-SQL, as: {Int64, Int32?, Int32?, Int32?} 33 | SELECT 34 | shard_id, dependencies_count, dev_dependencies_count, transitive_dependencies_count 35 | FROM 36 | shard_metrics_current 37 | WHERE 38 | dependencies_count > 0 OR dev_dependencies_count > 0 OR transitive_dependencies_count > 0 39 | ORDER BY 40 | shard_id 41 | SQL 42 | end 43 | 44 | def calculate_shard_metrics(db) 45 | ids = db.connection.query_all <<-SQL, as: Int64 46 | SELECT 47 | id 48 | FROM 49 | shards 50 | SQL 51 | ids.each do |id| 52 | db.connection.exec "SELECT shard_metrics_calculate($1)", id 53 | end 54 | end 55 | 56 | describe "Service::SyncRepos #update_shard_dependencies" do 57 | it "calcs shard dependencies" do 58 | transaction do |db| 59 | foo_id = Factory.create_shard(db, "foo") 60 | Factory.create_repo(db, Repo::Ref.new("git", "foo"), shard_id: foo_id) 61 | foo_release = Factory.create_release(db, foo_id, "0.0.0", latest: true) 62 | bar_id = Factory.create_shard(db, "bar") 63 | bar_repo_id = Factory.create_repo(db, Repo::Ref.new("git", "bar"), shard_id: bar_id) 64 | 65 | Factory.create_dependency(db, foo_release, "bar", repo_id: bar_repo_id) 66 | 67 | service = Service::SyncRepos.new(db) 68 | 69 | service.update_shard_dependencies 70 | 71 | persisted_dependencies(db).should eq [ 72 | {foo_id, bar_id, bar_repo_id, "runtime"}, 73 | ] 74 | 75 | calculate_shard_metrics(db) 76 | 77 | dependents_stats(db).should eq [ 78 | {bar_id, 1, 0, 1}, 79 | ] 80 | dependencies_stats(db).should eq [ 81 | {foo_id, 1, 0, 1}, 82 | ] 83 | end 84 | end 85 | 86 | it "calcs shard dev dependencies" do 87 | transaction do |db| 88 | foo_id = Factory.create_shard(db, "foo") 89 | Factory.create_repo(db, Repo::Ref.new("git", "foo"), shard_id: foo_id) 90 | foo_release = Factory.create_release(db, foo_id, "0.0.0", latest: true) 91 | bar_id = Factory.create_shard(db, "bar") 92 | bar_repo_id = Factory.create_repo(db, Repo::Ref.new("git", "bar"), shard_id: bar_id) 93 | 94 | Factory.create_dependency(db, foo_release, "bar", repo_id: bar_repo_id, scope: "development") 95 | 96 | service = Service::SyncRepos.new(db) 97 | 98 | service.update_shard_dependencies 99 | 100 | persisted_dependencies(db).should eq [ 101 | {foo_id, bar_id, bar_repo_id, "development"}, 102 | ] 103 | 104 | calculate_shard_metrics(db) 105 | 106 | dependents_stats(db).should eq [ 107 | {bar_id, 0, 1, 0}, 108 | ] 109 | dependencies_stats(db).should eq [ 110 | {foo_id, 0, 1, 0}, 111 | ] 112 | end 113 | end 114 | 115 | it "calcs shard dependencies with missing repo" do 116 | transaction do |db| 117 | foo_id = Factory.create_shard(db, "foo") 118 | Factory.create_repo(db, Repo::Ref.new("git", "foo"), shard_id: foo_id) 119 | foo_release = Factory.create_release(db, foo_id, "0.0.0", latest: true) 120 | missing_repo_id = Factory.create_repo(db, Repo::Ref.new("git", "missing"), shard_id: nil) 121 | 122 | Factory.create_dependency(db, foo_release, "missing", repo_id: missing_repo_id) 123 | 124 | service = Service::SyncRepos.new(db) 125 | 126 | service.update_shard_dependencies 127 | 128 | persisted_dependencies(db).should eq [ 129 | {foo_id, nil, missing_repo_id, "runtime"}, 130 | ] 131 | 132 | calculate_shard_metrics(db) 133 | 134 | dependents_stats(db).should be_empty 135 | 136 | dependencies_stats(db).should eq [ 137 | {foo_id, 1, 0, 1}, 138 | ] 139 | end 140 | end 141 | 142 | it "calcs shard dependencies with existing and missing repo" do 143 | transaction do |db| 144 | foo_id = Factory.create_shard(db, "foo") 145 | Factory.create_repo(db, Repo::Ref.new("git", "foo"), shard_id: foo_id) 146 | foo_release = Factory.create_release(db, foo_id, "0.0.0", latest: true) 147 | missing_repo_id = Factory.create_repo(db, Repo::Ref.new("git", "missing"), shard_id: nil) 148 | bar_id = Factory.create_shard(db, "bar") 149 | bar_repo_id = Factory.create_repo(db, Repo::Ref.new("git", "bar"), shard_id: bar_id) 150 | 151 | Factory.create_dependency(db, foo_release, "missing", repo_id: missing_repo_id) 152 | Factory.create_dependency(db, foo_release, "bar", repo_id: bar_repo_id) 153 | 154 | service = Service::SyncRepos.new(db) 155 | 156 | service.update_shard_dependencies 157 | 158 | persisted_dependencies(db).should eq [ 159 | {foo_id, bar_id, bar_repo_id, "runtime"}, 160 | {foo_id, nil, missing_repo_id, "runtime"}, 161 | ] 162 | 163 | calculate_shard_metrics(db) 164 | 165 | dependents_stats(db).should eq [ 166 | {bar_id, 1, 0, 1}, 167 | ] 168 | 169 | dependencies_stats(db).should eq [ 170 | {foo_id, 2, 0, 2}, 171 | ] 172 | end 173 | end 174 | 175 | it "calcs shard dependencies with multiple releases" do 176 | transaction do |db| 177 | foo_id = Factory.create_shard(db, "foo") 178 | Factory.create_repo(db, Repo::Ref.new("git", "foo"), shard_id: foo_id) 179 | foo_release1 = Factory.create_release(db, foo_id, "0.1.0") 180 | foo_release2 = Factory.create_release(db, foo_id, "0.2.0", latest: true) 181 | bar_id = Factory.create_shard(db, "bar") 182 | bar_repo_id = Factory.create_repo(db, Repo::Ref.new("git", "bar"), shard_id: bar_id) 183 | 184 | Factory.create_dependency(db, foo_release1, "bar", repo_id: bar_repo_id) 185 | Factory.create_dependency(db, foo_release2, "bar", repo_id: bar_repo_id) 186 | 187 | service = Service::SyncRepos.new(db) 188 | 189 | service.update_shard_dependencies 190 | 191 | persisted_dependencies(db).should eq [ 192 | {foo_id, bar_id, bar_repo_id, "runtime"}, 193 | ] 194 | 195 | calculate_shard_metrics(db) 196 | 197 | dependents_stats(db).should eq [ 198 | {bar_id, 1, 0, 1}, 199 | ] 200 | 201 | dependencies_stats(db).should eq [ 202 | {foo_id, 1, 0, 1}, 203 | ] 204 | end 205 | end 206 | 207 | it "calcs shard dependencies transitive" do 208 | transaction do |db| 209 | foo_id = Factory.create_shard(db, "foo") 210 | foo_release = Factory.create_release(db, foo_id, "0.1.0", latest: true) 211 | foo_repo_id = Factory.create_repo(db, Repo::Ref.new("git", "foo"), shard_id: foo_id) 212 | bar_id = Factory.create_shard(db, "bar") 213 | bar_release = Factory.create_release(db, bar_id, "0.1.0", latest: true) 214 | bar_repo_id = Factory.create_repo(db, Repo::Ref.new("git", "bar"), shard_id: bar_id) 215 | baz_id = Factory.create_shard(db, "baz") 216 | baz_release = Factory.create_release(db, baz_id, "0.1.0", latest: true) 217 | baz_repo_id = Factory.create_repo(db, Repo::Ref.new("git", "baz"), shard_id: baz_id) 218 | qux_id = Factory.create_shard(db, "qux") 219 | qux_repo_id = Factory.create_repo(db, Repo::Ref.new("git", "qux"), shard_id: qux_id) 220 | 221 | service = Service::SyncRepos.new(db) 222 | 223 | Factory.create_dependency(db, foo_release, "baz", repo_id: baz_repo_id) 224 | Factory.create_dependency(db, foo_release, "bar", repo_id: bar_repo_id) 225 | Factory.create_dependency(db, baz_release, "qux", repo_id: qux_repo_id) 226 | Factory.create_dependency(db, bar_release, "baz", repo_id: baz_repo_id) 227 | 228 | service.update_shard_dependencies 229 | 230 | persisted_dependencies(db).should eq [ 231 | {foo_id, bar_id, bar_repo_id, "runtime"}, 232 | {foo_id, baz_id, baz_repo_id, "runtime"}, 233 | {bar_id, baz_id, baz_repo_id, "runtime"}, 234 | {baz_id, qux_id, qux_repo_id, "runtime"}, 235 | ] 236 | 237 | calculate_shard_metrics(db) 238 | 239 | dependents_stats(db).should eq [ 240 | {bar_id, 1, 0, 1}, 241 | {baz_id, 2, 0, 2}, 242 | {qux_id, 1, 0, 3}, 243 | ] 244 | dependencies_stats(db).should eq [ 245 | {foo_id, 2, 0, 3}, 246 | {bar_id, 1, 0, 2}, 247 | {baz_id, 1, 0, 1}, 248 | ] 249 | end 250 | end 251 | end 252 | -------------------------------------------------------------------------------- /spec/service/update_owner_metrics_spec.cr: -------------------------------------------------------------------------------- 1 | require "spec" 2 | require "../../src/service/update_owner_metrics" 3 | require "../support/db" 4 | require "../support/factory" 5 | require "../support/raven" 6 | 7 | module Factory 8 | def self.create_shard_with_release(db, name) 9 | shard_id = create_shard(db, name) 10 | create_release(db, shard_id, latest: true) 11 | end 12 | 13 | def self.create_owner(db, name) 14 | db.create_owner(Repo::Owner.new("github", name)) 15 | end 16 | 17 | def self.add_dependencies(db, shard, deps, scope = Dependency::Scope::RUNTIME) 18 | shard_id = db.get_shard_id(shard) 19 | deps.each do |dep| 20 | db.connection.exec <<-SQL, shard_id, dep, scope 21 | INSERT INTO shard_dependencies 22 | SELECT 23 | $1 AS shard_id, 24 | ( 25 | SELECT 26 | id 27 | FROM shards 28 | WHERE name = $2 29 | ) AS depends_on, 30 | ( 31 | SELECT 32 | repos.id 33 | FROM repos 34 | JOIN shards 35 | ON shards.id = repos.shard_id 36 | AND repos.role = 'canonical' 37 | WHERE name = $2 38 | ) AS depends_on_repo_id, 39 | $3 AS scope 40 | SQL 41 | end 42 | end 43 | 44 | def self.set_owner(db, owner_id, repo_ids) 45 | repo_ids.each do |repo_id| 46 | db.connection.exec <<-SQL, owner_id, repo_id 47 | UPDATE repos 48 | SET 49 | owner_id = $1 50 | WHERE 51 | id = $2 52 | SQL 53 | end 54 | end 55 | end 56 | 57 | describe Service::UpdateOwnerMetrics do 58 | it "calcs owner dependents" do 59 | transaction do |db| 60 | myshard1_id = Factory.create_shard db, "myshard1" 61 | myshard1_repo_id = Factory.create_repo db, Repo::Ref.new("github", "me/myshard1"), myshard1_id 62 | myshard2_id = Factory.create_shard db, "myshard2" 63 | myshard2_repo_id = Factory.create_repo db, Repo::Ref.new("github", "me/myshard2"), myshard2_id 64 | 65 | depshard1_id = Factory.create_shard db, "depshard1" 66 | depshard1_repo_id = Factory.create_repo db, Repo::Ref.new("github", "other/depshard1"), depshard1_id 67 | depshard2_id = Factory.create_shard db, "depshard2" 68 | depdepshard_id = Factory.create_shard db, "depdepshard" 69 | depdepshard_repo_id = Factory.create_repo db, Repo::Ref.new("github", "foo/depdepshard"), depdepshard_id 70 | 71 | Factory.add_dependencies(db, "depshard1", ["myshard1", "myshard2"]) 72 | Factory.add_dependencies(db, "depshard2", ["myshard2"]) 73 | Factory.add_dependencies(db, "depdepshard", ["depshard1"]) 74 | 75 | owner_id = Factory.create_owner(db, "me") 76 | Factory.set_owner(db, owner_id, [myshard1_repo_id, myshard2_repo_id]) 77 | 78 | service = Service::UpdateOwnerMetrics.new(db) 79 | service.perform 80 | 81 | results = db.connection.query_all <<-SQL, as: {Int64, Int32, Int32, Int32, Int32, Int32, Int32, Int32} 82 | SELECT 83 | owner_id, 84 | shards_count, 85 | dependents_count, 86 | transitive_dependents_count, 87 | dev_dependents_count, 88 | dependencies_count, 89 | transitive_dependencies_count, 90 | dev_dependencies_count 91 | FROM 92 | owner_metrics 93 | SQL 94 | results.should eq [{owner_id, 2, 2, 3, 0, 0, 0, 0}] 95 | 96 | results = db.connection.query_all <<-SQL, as: {Int64, Int32, Int32, Int32, Int32, Int32, Int32, Int32} 97 | SELECT 98 | id, 99 | shards_count, 100 | dependents_count, 101 | transitive_dependents_count, 102 | dev_dependents_count, 103 | dependencies_count, 104 | transitive_dependencies_count, 105 | dev_dependencies_count 106 | FROM 107 | owners 108 | SQL 109 | results.should eq [{owner_id, 2, 2, 3, 0, 0, 0, 0}] 110 | end 111 | end 112 | end 113 | -------------------------------------------------------------------------------- /spec/service/update_shard_metrics_spec.cr: -------------------------------------------------------------------------------- 1 | require "spec" 2 | require "../../src/service/update_shard_metrics" 3 | require "../support/db" 4 | require "../support/factory" 5 | require "../support/raven" 6 | 7 | describe Service::UpdateShardMetrics do 8 | it "calcs shard dependencies" do 9 | transaction do |db| 10 | db.connection.on_notice do |notice| 11 | puts notice 12 | end 13 | 14 | service = Service::UpdateShardMetrics.new(db) 15 | service.perform 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /spec/shard_spec.cr: -------------------------------------------------------------------------------- 1 | require "spec" 2 | require "../src/shard" 3 | 4 | describe Shard do 5 | it ".new" do 6 | shard = Shard.new("jerusalem") 7 | shard.name.should eq "jerusalem" 8 | end 9 | 10 | it "#display_name" do 11 | Shard.new("foo").display_name.should eq "foo" 12 | Shard.new("foo", "bar").display_name.should eq "foo~bar" 13 | end 14 | 15 | it "#archived?" do 16 | Shard.new("foo", archived_at: nil).archived?.should be_false 17 | Shard.new("foo", archived_at: Time.utc).archived?.should be_true 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /spec/spec_helper.cr: -------------------------------------------------------------------------------- 1 | require "spec" 2 | -------------------------------------------------------------------------------- /spec/support/db.cr: -------------------------------------------------------------------------------- 1 | require "./factory" 2 | require "../../src/db" 3 | 4 | ShardsDB.database_url = ENV["TEST_DATABASE_URL"] 5 | 6 | def transaction(&) 7 | ShardsDB.transaction do |db, transaction| 8 | db.connection.on_notice do |notice| 9 | puts 10 | print "NOTICE from PG: " 11 | puts notice 12 | end 13 | 14 | yield db, transaction 15 | 16 | transaction.rollback 17 | end 18 | end 19 | 20 | class ShardsDB 21 | def last_repo_activity 22 | connection.query_one? <<-SQL, as: {Int64, String} 23 | SELECT 24 | repo_id, event, created_at 25 | FROM 26 | activity_log 27 | WHERE 28 | repo_id IS NOT NULL 29 | ORDER BY created_at DESC 30 | LIMIT 1 31 | SQL 32 | end 33 | end 34 | 35 | module ShardsDBHelper 36 | def self.persisted_shards(db) 37 | db.connection.query_all <<-SQL, as: {String, String, String?} 38 | SELECT 39 | name::text, qualifier::text, description::text 40 | FROM shards 41 | ORDER BY name, qualifier 42 | SQL 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /spec/support/factory.cr: -------------------------------------------------------------------------------- 1 | module Factory 2 | def self.create_repo(db, ref : Repo::Ref, shard_id = nil, role : Repo::Role = :canonical) 3 | db.create_repo Repo.new(ref, shard_id, role) 4 | end 5 | 6 | def self.create_shard(db, name = "shard", qualifier = "", description = nil, categories : Array(String)? = nil, archived_at : Time? = nil) 7 | shard_id = db.create_shard Shard.new(name, qualifier, description, archived_at) 8 | 9 | if categories 10 | db.connection.exec <<-SQL % db.sql_array(categories), shard_id 11 | UPDATE 12 | shards 13 | SET 14 | categories = coalesce((SELECT array_agg(id) FROM categories WHERE slug = ANY(ARRAY[%s]::text[])), ARRAY[]::bigint[]) 15 | WHERE 16 | id = $1 17 | SQL 18 | end 19 | 20 | shard_id 21 | end 22 | 23 | def self.create_release(db, shard_id = nil, version = "0.1.0", released_at = Time.utc, 24 | revision_info = nil, spec = {} of String => JSON::Any, 25 | yanked_at = nil, latest = false, position = nil) 26 | shard_id ||= create_shard(db) 27 | revision_info ||= build_revision_info("v#{version}") 28 | 29 | release = Release.new(version, released_at, revision_info, spec, yanked_at, latest) 30 | 31 | db.create_release(shard_id, release, position) 32 | end 33 | 34 | def self.create_dependency(db, release_id : Int64, name : String, spec : JSON::Any = JSON.parse("{}"), repo_id : Int64? = nil, scope = Dependency::Scope::RUNTIME) 35 | db.connection.exec <<-SQL, release_id, name, spec.to_json, repo_id, scope 36 | INSERT INTO dependencies 37 | (release_id, name, spec, repo_id, scope) 38 | VALUES 39 | ($1, $2, $3::jsonb, $4, $5) 40 | SQL 41 | end 42 | 43 | def self.build_tag(name, message = "tag #{name}", tagger = "mock tagger") 44 | tagger = Release::Signature.new(tagger, "", Time.utc.at_beginning_of_second) unless tagger.is_a?(Release::Signature) 45 | Release::Tag.new(name, message, tagger) 46 | end 47 | 48 | def self.build_commit(sha, time = Time.utc, author = "mock author", committer = "mock comitter", message = "commit #{sha}") 49 | author = Release::Signature.new(author, "", Time.utc.at_beginning_of_second) unless author.is_a?(Release::Signature) 50 | committer = Release::Signature.new(committer, "", Time.utc.at_beginning_of_second) unless committer.is_a?(Release::Signature) 51 | Release::Commit.new(sha, time.at_beginning_of_second, author, committer, message) 52 | end 53 | 54 | def self.build_revision_info(tag = "v0.0.0", hash = "12345678") 55 | unless tag.nil? 56 | tag = Factory.build_tag(tag) 57 | end 58 | Release::RevisionInfo.new tag, Factory.build_commit(hash) 59 | end 60 | 61 | def self.create_category(db, slug, name = slug) 62 | db.create_category(Category.new(slug, name)) 63 | end 64 | end 65 | -------------------------------------------------------------------------------- /spec/support/fetcher_mocks.cr: -------------------------------------------------------------------------------- 1 | struct Shardbox::GitHubAPI 2 | property mock_owner_info : Hash(String, JSON::Any)? 3 | 4 | def fetch_owner_info(login : String) 5 | if mock = mock_owner_info 6 | return mock 7 | else 8 | previous_def 9 | end 10 | end 11 | end 12 | 13 | struct Service::CreateOwner 14 | property skip_owner_info = false 15 | 16 | def fetch_owner_info(owner) 17 | unless skip_owner_info 18 | previous_def 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /spec/support/mock_resolver.cr: -------------------------------------------------------------------------------- 1 | class Repo 2 | class Resolver 3 | def initialize(@resolver : MockResolver, @repo_ref) 4 | end 5 | end 6 | end 7 | 8 | class MockResolver 9 | record MockEntry, 10 | spec : String?, 11 | revision_info : Release::RevisionInfo, 12 | files : Hash(String, String) = {} of String => String 13 | 14 | property? resolvable : Bool = true 15 | 16 | def self.new(versions : Hash(String, MockEntry) = {} of String => MockEntry) 17 | new(versions.transform_keys { |key| Shards::Version.new(key) }) 18 | end 19 | 20 | def initialize(@versions : Hash(Shards::Version, MockEntry)) 21 | end 22 | 23 | def self.unresolvable 24 | resolver = new 25 | resolver.resolvable = false 26 | resolver 27 | end 28 | 29 | def register(version : String, revision_info : Release::RevisionInfo, spec : String?) 30 | register(Shards::Version.new(version), revision_info, spec) 31 | end 32 | 33 | def register(version : Shards::Version, revision_info : Release::RevisionInfo, spec : String?) 34 | @versions[version] = MockEntry.new(spec, revision_info) 35 | end 36 | 37 | def available_releases : Array(Shards::Version) 38 | raise Repo::Resolver::RepoUnresolvableError.new unless resolvable? 39 | @versions.keys.compact.reject { |version| version.value == "HEAD" } 40 | end 41 | 42 | def spec(version : Shards::Version) 43 | raise Repo::Resolver::RepoUnresolvableError.new unless resolvable? 44 | Shards::Spec.from_yaml(read_spec(version)) 45 | end 46 | 47 | def read_spec!(version : Shards::Version) 48 | raise Repo::Resolver::RepoUnresolvableError.new unless resolvable? 49 | @versions[version].spec 50 | end 51 | 52 | def revision_info(version : Shards::Version) 53 | raise Repo::Resolver::RepoUnresolvableError.new unless resolvable? 54 | @versions[version].revision_info 55 | end 56 | 57 | def revision_info(version : Shards::GitHeadRef) 58 | revision_info(Shards::Version.new("HEAD")) 59 | end 60 | 61 | def latest_version_for_ref(ref) 62 | raise Repo::Resolver::RepoUnresolvableError.new unless resolvable? 63 | @versions.keys.last? 64 | end 65 | 66 | def fetch_file(version, path) 67 | raise Repo::Resolver::RepoUnresolvableError.new unless resolvable? 68 | @versions[version].files[path]? 69 | end 70 | end 71 | 72 | struct MockFetchMetadata 73 | def initialize(@metadata : Repo::Metadata?) 74 | end 75 | 76 | def fetch_repo_metadata 77 | if metadata = @metadata 78 | metadata 79 | else 80 | raise Shardbox::FetchError.new("Repo unavailable") 81 | end 82 | end 83 | end 84 | -------------------------------------------------------------------------------- /spec/support/raven.cr: -------------------------------------------------------------------------------- 1 | require "raven" 2 | 3 | Raven.configure do |config| 4 | config.current_environment = "dev" 5 | config.environments = ["production"] 6 | end 7 | -------------------------------------------------------------------------------- /spec/support/tempdir.cr: -------------------------------------------------------------------------------- 1 | require "file_utils" 2 | 3 | def with_tempdir(name, &) 4 | path = File.join(Dir.tempdir, name) 5 | FileUtils.mkdir_p(path) 6 | 7 | begin 8 | yield path 9 | ensure 10 | FileUtils.rm_r(path) if File.exists?(path) 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /src/catalog.cr: -------------------------------------------------------------------------------- 1 | require "yaml" 2 | require "json" 3 | require "./repo" 4 | require "./category" 5 | require "uri" 6 | 7 | class Catalog 8 | class Error < Exception 9 | end 10 | 11 | getter location : URI 12 | getter categories = Hash(String, ::Category).new 13 | @entries = {} of Repo::Ref => Entry 14 | 15 | def initialize(@location : URI) 16 | end 17 | 18 | def self.empty 19 | new URI.new 20 | end 21 | 22 | def self.new(location) 23 | new(URI.parse(location)) 24 | end 25 | 26 | def self.read(location) : self 27 | catalog = new(location) 28 | catalog.read 29 | catalog 30 | end 31 | 32 | def read 33 | duplication = Duplication.new 34 | 35 | each_category do |yaml_category| 36 | category = ::Category.new(yaml_category.slug, yaml_category.name, yaml_category.description) 37 | categories[category.slug] = category 38 | yaml_category.shards.each do |shard| 39 | if error = duplication.register(yaml_category.slug, shard) 40 | raise error 41 | end 42 | 43 | if stored_entry = @entries[shard.repo_ref]? 44 | stored_entry.mirrors.concat(shard.mirrors) 45 | stored_entry.categories << yaml_category.slug 46 | else 47 | shard.categories << yaml_category.slug 48 | @entries[shard.repo_ref] = shard 49 | end 50 | end 51 | end 52 | end 53 | 54 | def self.duplicate_repo?(shard, mirrors, all_entries) 55 | # (1) The entry's repo is already specified as a mirror of another entry 56 | return shard.repo_ref if mirrors.includes?(shard.repo_ref) 57 | 58 | # (2) The 59 | if shard.mirrors.any? { |mirror| mirror.repo_ref == shard.repo_ref } 60 | return shard.repo_ref 61 | end 62 | 63 | shard.mirrors.each do |mirror| 64 | if all_entries[mirror.repo_ref]? || !mirrors.add?(mirror.repo_ref) 65 | return mirror.repo_ref 66 | end 67 | end 68 | 69 | if other_entry = all_entries[shard.repo_ref]? 70 | if other_entry.description && shard.description 71 | p! other_entry.description, shard.description 72 | return shard.repo_ref 73 | end 74 | end 75 | 76 | nil 77 | end 78 | 79 | def entries 80 | @entries.values 81 | end 82 | 83 | def entry?(repo_ref : Repo::Ref) 84 | @entries[repo_ref]? 85 | end 86 | 87 | def find_canonical_entry(ref : Repo::Ref) 88 | entry?(ref) || begin 89 | @entries.find do |_, entry| 90 | entry.mirrors.find { |mirror| mirror.repo_ref == ref } 91 | end 92 | end 93 | end 94 | 95 | def each_category(&) 96 | local_path = check_out 97 | 98 | each_category(local_path) do |category| 99 | yield category 100 | end 101 | end 102 | 103 | def each_category(path : Path, &) 104 | unless File.directory?(path) 105 | raise Error.new "Can't read catalog at #{path}, directory does not exist." 106 | end 107 | found_a_file = false 108 | 109 | filenames = Dir.glob(path.join("*.yml").to_s).sort 110 | filenames.each do |filename| 111 | found_a_file = true 112 | File.open(filename) do |file| 113 | begin 114 | category = Category.from_yaml(file) 115 | rescue exc 116 | raise Error.new("Failure reading catalog #{filename}", cause: exc) 117 | end 118 | 119 | category.slug = File.basename(filename, ".yml") 120 | 121 | yield category 122 | end 123 | end 124 | unless found_a_file 125 | raise "Catalog at #{path} is empty." 126 | end 127 | end 128 | 129 | def check_out(checkout_path = "./catalog") 130 | if location.scheme == "file" || location.scheme.nil? 131 | return Path.new(location.path) 132 | end 133 | 134 | local_path = Path[checkout_path, "catalog"] 135 | if File.directory?(checkout_path) 136 | if Process.run("git", ["-C", checkout_path.to_s, "pull", location.to_s], output: :inherit, error: :inherit).success? 137 | return local_path 138 | else 139 | abort "Can't checkout catalog from #{location}: checkout path #{checkout_path.inspect} exists, but is not a git repository" 140 | end 141 | end 142 | 143 | Process.run("git", ["clone", location.to_s, checkout_path.to_s], output: :inherit, error: :inherit) 144 | 145 | local_path 146 | end 147 | end 148 | 149 | require "./catalog/*" 150 | -------------------------------------------------------------------------------- /src/catalog/category.cr: -------------------------------------------------------------------------------- 1 | class Catalog::Category 2 | include YAML::Serializable 3 | include YAML::Serializable::Strict 4 | 5 | getter name : String 6 | 7 | getter description : String? 8 | 9 | getter shards : Array(Entry) = [] of Catalog::Entry 10 | 11 | @[YAML::Field(ignore: true)] 12 | property! slug : String? 13 | 14 | def initialize(@name : String, @description : String? = nil, @slug = nil) 15 | @slug ||= @name 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /src/catalog/duplication.cr: -------------------------------------------------------------------------------- 1 | class Catalog::DuplicateRepoError < Catalog::Error 2 | def self.new(repo : Repo::Ref, category : String, existing_entry : Catalog::Entry, existing_category : String, *, mirror = false) 3 | message = String.build do |io| 4 | io << "duplicate " 5 | io << (mirror ? "mirror " : "repo ") 6 | io << repo 7 | io << " also " 8 | if existing_entry.repo_ref != repo 9 | io << "on " << existing_entry.repo_ref << " " 10 | end 11 | io << "in " << existing_category 12 | end 13 | 14 | new(message, repo, category, existing_entry, existing_category) 15 | end 16 | 17 | def initialize(message : String, @repo : Repo::Ref, @category : String, @existing_entry : Catalog::Entry, @existing_category : String) 18 | super(message) 19 | end 20 | 21 | def message 22 | super.not_nil! 23 | end 24 | 25 | def to_s(io : IO) 26 | io << @category << ": " 27 | io << message 28 | end 29 | end 30 | 31 | class Catalog::Duplication 32 | def initialize 33 | @all_repos = {} of Repo::Ref => {String, Entry} 34 | end 35 | 36 | def register(category : String, entry : Entry) 37 | if error = check_duplicate_repo(category, entry) 38 | return error 39 | end 40 | 41 | @all_repos[entry.repo_ref] = {category, entry} 42 | entry.mirrors.each do |mirror| 43 | @all_repos[mirror.repo_ref] = {category, entry} 44 | end 45 | 46 | nil 47 | end 48 | 49 | private def check_duplicate_repo(category, entry) 50 | if existing = @all_repos[entry.repo_ref]? 51 | existing_entry = existing[1] 52 | if existing_entry.repo_ref == entry.repo_ref 53 | # existing is canonical 54 | if existing_entry.description && entry.description 55 | return DuplicateRepoError.new(entry.repo_ref, category, existing_entry, existing[0]) 56 | elsif entry.description 57 | @all_repos[entry.repo_ref] = {category, entry} 58 | end 59 | else 60 | # existing is mirror 61 | return DuplicateRepoError.new(entry.repo_ref, category, existing_entry, existing[0]) 62 | end 63 | end 64 | 65 | entry.mirrors.each do |mirror| 66 | if mirror.repo_ref == entry.repo_ref 67 | return DuplicateRepoError.new(mirror.repo_ref, category, entry, category, mirror: true) 68 | end 69 | if existing = @all_repos[mirror.repo_ref]? 70 | return DuplicateRepoError.new(mirror.repo_ref, category, existing[1], existing[0], mirror: true) 71 | end 72 | end 73 | end 74 | 75 | def foo 76 | # (1) The entry's repo is already specified as a mirror of another entry 77 | return shard.repo_ref if mirrors.includes?(shard.repo_ref) 78 | 79 | # (2) The 80 | if shard.mirrors.any? { |mirror| mirror.repo_ref == shard.repo_ref } 81 | return shard.repo_ref 82 | end 83 | 84 | shard.mirrors.each do |mirror| 85 | if all_entries[mirror.repo_ref]? || !mirrors.add?(mirror.repo_ref) 86 | return mirror.repo_ref 87 | end 88 | end 89 | 90 | if other_entry = all_entries[shard.repo_ref]? 91 | if other_entry 92 | p! other_entry.description, shard.description 93 | return shard.repo_ref 94 | end 95 | end 96 | 97 | nil 98 | end 99 | end 100 | -------------------------------------------------------------------------------- /src/catalog/entry.cr: -------------------------------------------------------------------------------- 1 | struct Catalog::Entry 2 | include JSON::Serializable 3 | include Comparable(self) 4 | 5 | enum State 6 | ACTIVE 7 | ARCHIVED 8 | 9 | def to_s(io : IO) 10 | io << to_s 11 | end 12 | 13 | def to_s 14 | super.downcase 15 | end 16 | 17 | def to_yaml(builder : YAML::Nodes::Builder) 18 | builder.scalar to_s 19 | end 20 | end 21 | 22 | getter repo_ref : Repo::Ref 23 | property description : String? 24 | getter mirrors : Array(Mirror) 25 | getter categories : Array(String) 26 | property state : State 27 | 28 | def initialize(@repo_ref : Repo::Ref, @description : String? = nil, 29 | @mirrors = [] of Mirror, 30 | @state : State = :active, @categories = [] of String) 31 | end 32 | 33 | def archived? : Bool 34 | state.archived? 35 | end 36 | 37 | def mirror?(repo_ref : Repo::Ref) : Mirror? 38 | mirrors.find { |mirror| mirror.repo_ref == repo_ref } 39 | end 40 | 41 | def to_yaml(builder : YAML::Nodes::Builder) 42 | builder.mapping do 43 | builder.scalar repo_ref.resolver 44 | builder.scalar repo_ref.url.to_s 45 | 46 | if description = @description 47 | builder.scalar "description" 48 | builder.scalar description 49 | end 50 | 51 | unless mirrors.empty? 52 | builder.scalar "mirrors" 53 | mirrors.to_yaml builder 54 | end 55 | 56 | if archived? 57 | builder.scalar "state" 58 | state.to_yaml(builder) 59 | end 60 | end 61 | end 62 | 63 | def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) 64 | unless node.is_a?(YAML::Nodes::Mapping) 65 | node.raise "Expected mapping, not #{node.class}" 66 | end 67 | 68 | description = nil 69 | repo_ref = nil 70 | mirrors = [] of Mirror 71 | state = nil 72 | 73 | YAML::Schema::Core.each(node) do |key, value| 74 | key = String.new(ctx, key) 75 | case key 76 | when "description" 77 | description = String.new(ctx, value) 78 | when "mirrors" 79 | mirrors += Array(Mirror).new(ctx, value) 80 | when "mirror" 81 | # Legacy fields, use `mirrors` instead 82 | # TODO: Remove compatibility later 83 | mirrors += Array(Mirror).new(ctx, value) 84 | when "legacy" 85 | # Legacy fields, use `mirrors` instead 86 | # TODO: Remove compatibility later 87 | array = Array(Mirror).new(ctx, value) 88 | array.map! { |mirror| mirror.role = :LEGACY; mirror } 89 | mirrors += array 90 | when "state" 91 | unless value.is_a?(YAML::Nodes::Scalar) && (state = State.parse?(value.value)) 92 | raise %(unexpected value for key `state` in Category::Entry mapping, allowed values: #{State.values}) 93 | end 94 | else 95 | repo_ref = parse_repo_ref(ctx, key, value) 96 | 97 | unless repo_ref 98 | node.raise "unknown key: #{key} in Category::Entry mapping" 99 | end 100 | end 101 | end 102 | 103 | unless repo_ref 104 | node.raise "missing required repo reference" 105 | end 106 | 107 | state ||= State::ACTIVE 108 | new(repo_ref, description, mirrors, state) 109 | end 110 | 111 | def self.parse_repo_ref(ctx : YAML::ParseContext, key, value) 112 | if key == "git" 113 | # Special case "git" to resolve URLs pointing at named service providers (like https://github.com/foo/foo) 114 | Repo::Ref.new(String.new(ctx, value)) 115 | elsif Repo::RESOLVERS.includes?(key) 116 | Repo::Ref.new(key, String.new(ctx, value)) 117 | end 118 | end 119 | 120 | def <=>(other : self) 121 | repo_ref <=> other.repo_ref 122 | end 123 | end 124 | -------------------------------------------------------------------------------- /src/catalog/mirror.cr: -------------------------------------------------------------------------------- 1 | struct Catalog::Mirror 2 | include JSON::Serializable 3 | 4 | getter repo_ref : Repo::Ref 5 | getter role : Repo::Role = :mirror 6 | 7 | def initialize(@repo_ref : Repo::Ref, role : Repo::Role = :mirror) 8 | self.role = role 9 | end 10 | 11 | def legacy? 12 | role.legacy? 13 | end 14 | 15 | def role=(role : Repo::Role) 16 | raise "Invalid role for Catalog::Mirror: #{role}" if role.canonical? 17 | @role = role 18 | end 19 | 20 | def to_yaml(builder) 21 | builder.mapping do 22 | builder.scalar repo_ref.resolver 23 | builder.scalar repo_ref.url.to_s 24 | 25 | if legacy? 26 | builder.scalar "role" 27 | role.to_yaml(builder) 28 | end 29 | end 30 | end 31 | 32 | def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) 33 | unless node.is_a?(YAML::Nodes::Mapping) 34 | node.raise "Expected mapping, not #{node.class}" 35 | end 36 | 37 | repo_ref = nil 38 | role = nil 39 | 40 | YAML::Schema::Core.each(node) do |key, value| 41 | key = String.new(ctx, key) 42 | case key 43 | when "role" 44 | unless value.is_a?(YAML::Nodes::Scalar) && (role = Repo::Role.parse?(value.value)) && !role.canonical? 45 | raise %(unexpected value for key `role` in Category::Mirror mapping, allowed values: #{Repo::Role.values.delete(Repo::Role::CANONICAL)}) 46 | end 47 | else 48 | repo_ref = Entry.parse_repo_ref(ctx, key, value) 49 | 50 | unless repo_ref 51 | node.raise "unknown key: #{key} in Category::Mirror mapping" 52 | end 53 | end 54 | end 55 | 56 | unless repo_ref 57 | node.raise "missing required repo reference" 58 | end 59 | 60 | role ||= Repo::Role::MIRROR 61 | new(repo_ref, role) 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /src/category.cr: -------------------------------------------------------------------------------- 1 | class Category 2 | getter slug : String 3 | getter name : String 4 | getter description : String? 5 | getter entries_count : Int32 6 | getter! id : Int64 7 | 8 | def initialize( 9 | @slug : String, 10 | @name : String, 11 | @description : String? = nil, 12 | @entries_count : Int32 = 0, 13 | @id : Int64? = nil, 14 | ) 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /src/dependency.cr: -------------------------------------------------------------------------------- 1 | class Dependency 2 | enum Scope 3 | RUNTIME 4 | DEVELOPMENT 5 | 6 | def to_s(io : IO) 7 | io << to_s 8 | end 9 | 10 | def to_s : String 11 | super.downcase 12 | end 13 | end 14 | 15 | getter name : String 16 | getter spec : JSON::Any 17 | getter scope : Scope 18 | 19 | def initialize(@name : String, @spec : JSON::Any, @scope : Scope = :RUNTIME) 20 | end 21 | 22 | def repo_ref : Repo::Ref? 23 | if git = spec["git"]? 24 | # Treat git specially to detect URLs to registered resolvers like `git: https://github.com/crystal-lang/shards` 25 | return Repo::Ref.new(git.as_s) 26 | else 27 | Repo::RESOLVERS.each do |resolver| 28 | if url = spec[resolver]? 29 | return Repo::Ref.new(resolver, url.as_s) 30 | end 31 | end 32 | end 33 | # if dependency["path"]? 34 | # # Can't resolve path dependency 35 | # return nil 36 | # else 37 | # # Can't find resolver for #{dependency}" 38 | # end 39 | end 40 | 41 | def resolvable? : Bool 42 | !repo_ref.nil? 43 | end 44 | 45 | def version_reference 46 | {"commit", "tag", "version", "branch"}.each do |key| 47 | if value = spec[key]? 48 | return {key, value} 49 | end 50 | end 51 | end 52 | 53 | def_equals_and_hash name, spec, scope 54 | end 55 | -------------------------------------------------------------------------------- /src/ext/git/repo.cr: -------------------------------------------------------------------------------- 1 | class Git::Repository 2 | def ref?(name : String) 3 | if 0 == LibGit.reference_lookup(out ref, @value, name) 4 | Reference.new(ref) 5 | end 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /src/ext/shards/resolvers/git.cr: -------------------------------------------------------------------------------- 1 | require "git" 2 | require "../../../release" 3 | require "../../git/repo" 4 | 5 | class Shards::GitResolver 6 | def revision_info(version : Shards::GitHeadRef | Shards::Version) 7 | update_local_cache 8 | 9 | repo = Git::Repo.open(local_path) 10 | 11 | case version 12 | when GitHeadRef 13 | ref = repo.ref(version.to_git_ref) 14 | when Version 15 | ref = git_ref(version) 16 | ref = repo.ref?(ref.to_git_ref) 17 | 18 | raise "Can't find tag #{version}" unless ref 19 | else 20 | raise "unreachable" 21 | end 22 | 23 | # Resolve symbolic references 24 | ref = ref.resolve 25 | 26 | target = ref.target 27 | if target.is_a?(Git::Tag) 28 | target = repo.lookup_tag(target.target_id) 29 | end 30 | 31 | case target 32 | when Git::Tag::Annotation 33 | # annotated tag 34 | tag = target 35 | commit = repo.lookup_commit(tag.target_oid) 36 | tag_info = Release::Tag.new(tag.name, tag.message.strip, signature(tag.tagger)) 37 | when Git::Commit 38 | # lightweight tag 39 | commit = target 40 | tag_info = nil 41 | else 42 | raise "Unexpected target type #{target.class}" 43 | end 44 | 45 | commit_info = Release::Commit.new(commit.sha, commit.time, signature(commit.author), signature(commit.committer), commit.message.strip) 46 | 47 | Release::RevisionInfo.new tag_info, commit_info 48 | end 49 | 50 | private def signature(signature) 51 | Release::Signature.new(signature.name, signature.email, signature.time) 52 | end 53 | 54 | def read_spec!(version) 55 | if version.value == "HEAD" 56 | version = latest_version_for_ref(nil) 57 | raise Shards::Error.new("No version for HEAD") unless version 58 | end 59 | 60 | read_spec(version) 61 | end 62 | 63 | def fetch_file(version, path) 64 | # Tree#path is not yet implemented in libgit2.cr, falling back to CLI 65 | update_local_cache 66 | if version.value == "HEAD" 67 | ref = Shards::GitHeadRef.new 68 | else 69 | ref = git_ref(version) 70 | end 71 | 72 | if file_exists?(ref, path) 73 | capture("git show #{ref.to_git_ref}:#{path}") 74 | end 75 | end 76 | 77 | def repo_stats(version) 78 | tree = tree(version) 79 | file_count = tree.size_recursive 80 | end 81 | 82 | def tree(version) 83 | update_local_cache 84 | 85 | repo = Git::Repo.open(local_path) 86 | 87 | repo.lookup_tree(version) 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /src/ext/yaml/any.cr: -------------------------------------------------------------------------------- 1 | struct YAML::Any 2 | def to_json(builder : JSON::Builder) 3 | if (raw = self.raw).is_a?(Slice) 4 | raise "Can't serialize #{raw.class} to JSON" 5 | else 6 | raw.to_json(builder) 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /src/fetchers/error.cr: -------------------------------------------------------------------------------- 1 | class Shardbox::FetchError < Exception 2 | end 3 | -------------------------------------------------------------------------------- /src/fetchers/github_api-owner_info.graphql: -------------------------------------------------------------------------------- 1 | query RepositoryOwnerQuery($login : String!) { 2 | repositoryOwner(login: $login) { 3 | avatarUrl, 4 | ... on Organization { 5 | createdAt, 6 | description, 7 | email, 8 | location, 9 | name, 10 | websiteUrl 11 | } 12 | ... on User { 13 | bio, 14 | company, 15 | createdAt, 16 | email, 17 | location, 18 | name, 19 | websiteUrl 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/fetchers/github_api-repo_metadata.graphql: -------------------------------------------------------------------------------- 1 | query RepositoryDataQuery($owner: String!, $name: String!) { 2 | repository(owner: $owner, name: $name) { 3 | nameWithOwner 4 | forks { 5 | totalCount 6 | } 7 | stargazers { 8 | totalCount 9 | } 10 | watchers { 11 | totalCount 12 | } 13 | createdAt 14 | description 15 | hasIssuesEnabled 16 | hasWikiEnabled 17 | homepageUrl 18 | isArchived 19 | isFork 20 | isMirror 21 | licenseInfo { 22 | key 23 | } 24 | primaryLanguage { 25 | name 26 | } 27 | pushedAt 28 | closedIssues: issues(states: CLOSED) { 29 | totalCount 30 | } 31 | openIssues: issues(states: OPEN) { 32 | totalCount 33 | } 34 | closedPullRequests: pullRequests(states: CLOSED) { 35 | totalCount 36 | } 37 | openPullRequests: pullRequests(states: OPEN) { 38 | totalCount 39 | } 40 | mergedPullRequests: pullRequests(states: MERGED) { 41 | totalCount 42 | } 43 | repositoryTopics(first: 100) { 44 | nodes { 45 | topic { 46 | name 47 | } 48 | } 49 | } 50 | codeOfConduct { 51 | name 52 | url 53 | } 54 | } 55 | rateLimit { 56 | limit 57 | cost 58 | remaining 59 | resetAt 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/fetchers/github_api.cr: -------------------------------------------------------------------------------- 1 | require "json" 2 | require "http/client" 3 | require "../repo" 4 | require "./error" 5 | 6 | struct Shardbox::GitHubAPI 7 | getter graphql_client : HTTP::Client { HTTP::Client.new("api.github.com", 443, true) } 8 | 9 | private getter query_repo_metadata : String = begin 10 | {{ read_file("#{__DIR__}/github_api-repo_metadata.graphql") }} 11 | end 12 | 13 | private getter query_owner_info : String = begin 14 | {{ read_file("#{__DIR__}/github_api-owner_info.graphql") }} 15 | end 16 | 17 | def initialize(@api_token = ENV["GITHUB_TOKEN"]) 18 | end 19 | 20 | def query(query, variables) 21 | body = {query: query, variables: variables} 22 | response = graphql_client.post "/graphql", body: body.to_json, headers: HTTP::Headers{"Authorization" => "bearer #{@api_token}"} 23 | 24 | raise FetchError.new("Repository unavailable") unless response.status_code == 200 25 | 26 | response.body 27 | end 28 | 29 | def fetch_repo_metadata(owner : String, name : String) : Repo::Metadata? 30 | response = query(query_repo_metadata, {owner: owner, name: name}) 31 | 32 | metadata = parse_github_graphql(response, "repository") do |pull| 33 | Repo::Metadata.new(pull) 34 | end 35 | 36 | metadata 37 | end 38 | 39 | def fetch_owner_info(login : String) : Hash(String, JSON::Any)? 40 | response = query(query_owner_info, {login: login}) 41 | 42 | data = parse_github_graphql(response, "repositoryOwner") do |pull| 43 | Hash(String, JSON::Any).new(pull) 44 | end 45 | 46 | data 47 | end 48 | 49 | def parse_github_graphql(content, expected_key, &) 50 | pull = JSON::PullParser.new(content) 51 | 52 | pull.read_object do |key| 53 | case key 54 | when "data" 55 | pull.read_object do |key| 56 | if key == expected_key 57 | if pull.kind.null? 58 | return nil 59 | else 60 | return yield pull 61 | end 62 | else 63 | pull.skip 64 | end 65 | end 66 | when "errors" 67 | errors = [] of String 68 | pull.read_array do 69 | pull.on_key!("message") do 70 | errors << pull.read_string 71 | end 72 | end 73 | raise Shardbox::FetchError.new("Repository error: #{errors.join(", ")}") 74 | else 75 | pull.skip 76 | end 77 | end 78 | 79 | raise FetchError.new("Invalid response") 80 | rescue exc : JSON::ParseException 81 | raise FetchError.new("Invalid response", cause: exc) 82 | end 83 | end 84 | 85 | class Repo 86 | struct Metadata 87 | def initialize(github_pull pull : JSON::PullParser) 88 | pull.read_object do |key| 89 | case key 90 | when "forks" 91 | pull.on_key!("totalCount") do 92 | @forks_count = pull.read?(Int32) 93 | end 94 | when "stargazers" 95 | pull.on_key!("totalCount") do 96 | @stargazers_count = pull.read?(Int32) 97 | end 98 | when "watchers" 99 | pull.on_key!("totalCount") do 100 | @watchers_count = pull.read?(Int32) 101 | end 102 | when "createdAt" 103 | @created_at = Time.new(pull) 104 | when "description" 105 | @description = pull.read_string_or_null 106 | when "hasIssuesEnabled" 107 | @issues_enabled = pull.read_bool 108 | when "hasWikiEnabled" 109 | @wiki_enabled = pull.read_bool 110 | when "homepageUrl" 111 | @homepage_url = pull.read_string_or_null 112 | when "isArchived" 113 | @archived = pull.read_bool 114 | when "isFork" 115 | @fork = pull.read_bool 116 | when "isMirror" 117 | @mirror = pull.read_bool 118 | when "licenseInfo" 119 | pull.read_null_or do 120 | pull.on_key!("key") do 121 | @license = pull.read_string 122 | end 123 | end 124 | when "primaryLanguage" 125 | pull.read_null_or do 126 | pull.on_key!("name") do 127 | @primary_language = pull.read_string 128 | end 129 | end 130 | when "pushedAt" 131 | pull.read_null_or do 132 | @pushed_at = Time.new(pull) 133 | end 134 | when "closedIssues" 135 | pull.on_key!("totalCount") do 136 | @closed_issues_count = pull.read?(Int32) 137 | end 138 | when "openIssues" 139 | pull.on_key!("totalCount") do 140 | @open_issues_count = pull.read?(Int32) 141 | end 142 | when "closedPullRequests" 143 | pull.on_key!("totalCount") do 144 | @closed_pull_requests_count = pull.read?(Int32) 145 | end 146 | when "openPullRequests" 147 | pull.on_key!("totalCount") do 148 | @open_pull_requests_count = pull.read?(Int32) 149 | end 150 | when "mergedPullRequests" 151 | pull.on_key!("totalCount") do 152 | @merged_pull_requests_count = pull.read?(Int32) 153 | end 154 | when "repositoryTopics" 155 | topics = [] of String 156 | @topics = topics 157 | pull.on_key!("nodes") do 158 | pull.read_array do 159 | pull.on_key!("topic") do 160 | pull.on_key!("name") do 161 | topics << pull.read_string 162 | end 163 | end 164 | end 165 | end 166 | else 167 | pull.skip 168 | end 169 | end 170 | end 171 | end 172 | end 173 | -------------------------------------------------------------------------------- /src/log_activity.cr: -------------------------------------------------------------------------------- 1 | module Shardbox 2 | Log = ::Log.for(self) 3 | ActivityLog = Log.for("activity") 4 | end 5 | 6 | class LogActivity 7 | DB.mapping( 8 | id: Int64, 9 | event: String, 10 | repo_id: Int64?, 11 | shard_id: Int64?, 12 | metadata: JSON::Any, 13 | created_at: Time, 14 | ) 15 | end 16 | -------------------------------------------------------------------------------- /src/raven.cr: -------------------------------------------------------------------------------- 1 | require "raven" 2 | 3 | Raven.configure do |config| 4 | config.connect_timeout = 10.seconds 5 | 6 | config.current_environment = ENV["CRYSTAL_ENV"]? || "development" 7 | 8 | if env_var = ENV["SENTRY_DSN_VAR"]? 9 | config.dsn = ENV[env_var] 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /src/release.cr: -------------------------------------------------------------------------------- 1 | require "./dependency" 2 | require "json" 3 | 4 | class Release 5 | record RevisionInfo, tag : Tag?, commit : Commit do 6 | include JSON::Serializable 7 | end 8 | 9 | record Commit, sha : String, time : Time, author : Signature, committer : Signature, message : String do 10 | include JSON::Serializable 11 | end 12 | 13 | record Tag, name : String, message : String, tagger : Signature do 14 | include JSON::Serializable 15 | end 16 | 17 | record Signature, name : String, email : String, time : Time do 18 | include JSON::Serializable 19 | end 20 | 21 | property version : String 22 | property released_at : Time 23 | property! revision_info : RevisionInfo? 24 | property? yanked_at : Time? 25 | getter spec : Hash(String, JSON::Any) 26 | getter? latest : Bool 27 | property! id : Int64 28 | 29 | def self.new( 30 | version : String, 31 | revision_info : RevisionInfo, 32 | spec : Hash(String, JSON::Any) = {} of String => JSON::Any, 33 | yanked_at : Time? = nil, 34 | latest : Bool = false, 35 | ) 36 | new( 37 | version: version, 38 | released_at: revision_info.commit.time, 39 | revision_info: revision_info, 40 | spec: spec, 41 | yanked_at: yanked_at, 42 | latest: latest 43 | ) 44 | end 45 | 46 | def initialize( 47 | @version : String, 48 | @released_at : Time, 49 | @revision_info : RevisionInfo? = nil, 50 | @spec : Hash(String, JSON::Any) = {} of String => JSON::Any, 51 | @yanked_at : Time? = nil, 52 | @latest : Bool = false, 53 | @id : Int64? = nil, 54 | ) 55 | end 56 | 57 | def revision_identifier 58 | if tag = revision_info.tag 59 | tag.name 60 | else 61 | revision_info.commit.sha 62 | end 63 | end 64 | 65 | def license : String? 66 | spec["license"]?.try &.as_s 67 | end 68 | 69 | def description : String? 70 | spec["description"]?.try &.as_s 71 | end 72 | 73 | def crystal : String? 74 | if crystal = spec["crystal"]? 75 | crystal.as_s? || crystal.as_f?.try &.to_s # A version might have been encoded as a number in YAML 76 | end 77 | end 78 | 79 | def yanked? : Bool 80 | !yanked_at?.nil? 81 | end 82 | 83 | def_equals_and_hash version, revision_info, released_at, spec 84 | end 85 | -------------------------------------------------------------------------------- /src/repo.cr: -------------------------------------------------------------------------------- 1 | require "uri" 2 | require "json" 3 | 4 | class Repo 5 | RESOLVERS = {"git", "github", "gitlab", "bitbucket"} 6 | 7 | enum Role 8 | # Main repository of a shard. 9 | CANONICAL 10 | # A mirror of the main repository. 11 | MIRROR 12 | # A previously used repository, associated with the shard. 13 | LEGACY 14 | # An unused repository, not associated with a shard. 15 | OBSOLETE 16 | 17 | def to_s(io : IO) 18 | io << to_s 19 | end 20 | 21 | def to_s 22 | super.downcase 23 | end 24 | 25 | def to_yaml(builder : YAML::Nodes::Builder) 26 | builder.scalar to_s 27 | end 28 | 29 | def to_json(builder : JSON::Builder) 30 | builder.string to_s 31 | end 32 | end 33 | 34 | # Returns a reference to the shard hosted in this repo. 35 | getter shard_id : Int64? 36 | 37 | # Returns the identifier of this repo, consisting of resolver and url. 38 | getter ref : Ref 39 | 40 | # Returns the role of this repo for the shard (defaults to `canonical`). 41 | property role : Role 42 | 43 | getter metadata : Metadata 44 | 45 | def_equals_and_hash ref, role 46 | 47 | getter synced_at : Time? 48 | 49 | getter sync_failed_at : Time? 50 | 51 | property! id : Int64? 52 | 53 | def initialize( 54 | @ref : Ref, @shard_id : Int64?, 55 | @role : Role = :canonical, @metadata = Metadata.new, 56 | @synced_at : Time? = nil, @sync_failed_at : Time? = nil, 57 | @id : Int64? = nil, 58 | ) 59 | end 60 | 61 | def self.new( 62 | resolver : String, url : String, shard_id : Int64?, 63 | role : String = "canonical", metadata = Metadata.new, 64 | synced_at : Time? = nil, sync_failed_at : Time? = nil, 65 | id : Int64? = nil, 66 | ) 67 | new(Ref.new(resolver, url), shard_id, Role.parse(role), metadata, synced_at, sync_failed_at, id) 68 | end 69 | 70 | record Metadata, 71 | forks_count : Int32? = nil, 72 | stargazers_count : Int32? = nil, 73 | watchers_count : Int32? = nil, 74 | created_at : Time? = nil, 75 | description : String? = nil, 76 | issues_enabled : Bool? = nil, 77 | wiki_enabled : Bool? = nil, 78 | homepage_url : String? = nil, 79 | archived : Bool? = nil, 80 | fork : Bool? = nil, 81 | mirror : Bool? = nil, 82 | license : String? = nil, 83 | primary_language : String? = nil, 84 | pushed_at : Time? = nil, 85 | closed_issues_count : Int32? = nil, 86 | open_issues_count : Int32? = nil, 87 | closed_pull_requests_count : Int32? = nil, 88 | open_pull_requests_count : Int32? = nil, 89 | merged_pull_requests_count : Int32? = nil, 90 | topics : Array(String)? = nil do 91 | include JSON::Serializable 92 | end 93 | end 94 | 95 | require "./repo/ref" 96 | -------------------------------------------------------------------------------- /src/repo/owner.cr: -------------------------------------------------------------------------------- 1 | class Repo 2 | class Owner 3 | property resolver : String 4 | property slug : String 5 | property name : String? 6 | property description : String? 7 | property extra : Hash(String, JSON::Any) 8 | property shards_count : Int32? 9 | property! id : Int64 10 | 11 | def initialize(@resolver : String, @slug : String, 12 | @name : String? = nil, @description : String? = nil, 13 | @extra : Hash(String, JSON::Any) = Hash(String, JSON::Any).new, 14 | @shards_count : Int32? = nil, 15 | *, @id : Int64? = nil) 16 | end 17 | 18 | def self.from_repo_ref(repo_ref : Ref) : Owner? 19 | if owner = repo_ref.owner 20 | new(repo_ref.resolver, owner) 21 | end 22 | end 23 | 24 | def_equals_and_hash @resolver, @slug, @name, @description, @extra, @shards_count 25 | 26 | def website_url : String? 27 | extra["website_url"]?.try(&.as_s?) 28 | end 29 | 30 | record Metrics, 31 | shards_count : Int32, 32 | dependents_count : Int32, 33 | transitive_dependents_count : Int32, 34 | dev_dependents_count : Int32, 35 | transitive_dependencies_count : Int32, 36 | dev_dependencies_count : Int32, 37 | dependencies_count : Int32, 38 | popularity : Float32, 39 | created_at : Time? = nil 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /src/repo/ref.cr: -------------------------------------------------------------------------------- 1 | class Repo 2 | struct Ref 3 | include JSON::Serializable 4 | include Comparable(self) 5 | 6 | PROVIDER_RESOLVERS = {"github", "gitlab", "bitbucket"} 7 | 8 | getter resolver : String 9 | getter url : String 10 | 11 | def initialize(@resolver : String, @url : String) 12 | raise "Unknown resolver #{@resolver}" unless RESOLVERS.includes?(@resolver) 13 | if provider_resolver? 14 | raise "Invalid url for resolver #{@resolver}: #{@url.inspect}" unless @url =~ /^[A-Za-z0-9_\-.]{1,100}\/[A-Za-z0-9_\-.]{1,100}$/ 15 | end 16 | end 17 | 18 | def self.new(url : String) : self 19 | new URI.parse(url) 20 | end 21 | 22 | def self.new(uri : URI) : self 23 | case uri.host 24 | when "github.com", "www.github.com" 25 | if path = extract_org_repo_url(uri) 26 | return new("github", path) 27 | end 28 | when "gitlab.com", "www.gitlab.com" 29 | if path = extract_org_repo_url(uri) 30 | return new("gitlab", path) 31 | end 32 | when "bitbucket.com", "www.bitbucket.com" 33 | if path = extract_org_repo_url(uri) 34 | return new("bitbucket", path) 35 | end 36 | else 37 | # fall through 38 | end 39 | 40 | path = uri.path 41 | if path.nil? || path.empty? || path == "/" 42 | raise "Invalid url for resolver git: #{uri.to_s.inspect}" 43 | end 44 | 45 | new("git", uri.to_s) 46 | end 47 | 48 | def self.parse(string : String) 49 | PROVIDER_RESOLVERS.each do |resolver| 50 | if string.starts_with?(resolver) 51 | size = resolver.bytesize 52 | if string.byte_at(size) == ':'.ord 53 | size += 1 54 | return new(resolver, string.byte_slice(size, string.bytesize - size)) 55 | end 56 | end 57 | end 58 | 59 | new(string) 60 | end 61 | 62 | # Returns `true` if `resolver` is any of `PROVIDER_RESOLVER`. 63 | def provider_resolver? : Bool 64 | PROVIDER_RESOLVERS.includes?(@resolver) 65 | end 66 | 67 | def to_uri : URI 68 | if provider_resolver? 69 | # FIXME: Leading slash should not be needed 70 | URI.new("https", "#{resolver}.com", path: "/#{url}") 71 | else 72 | URI.parse(url) 73 | end 74 | end 75 | 76 | private def self.extract_org_repo_url(uri) 77 | path = uri.path.not_nil!.strip('/').rchop(".git") 78 | if path.count('/') == 1 79 | path 80 | end 81 | end 82 | 83 | def_equals_and_hash resolver, url 84 | 85 | def name 86 | uri = URI.parse(url) 87 | File.basename(uri.path).rchop('/').rchop(".git") 88 | end 89 | 90 | def owner 91 | if provider_resolver? 92 | Path.posix(url).dirname 93 | end 94 | end 95 | 96 | def nice_url 97 | return url if provider_resolver? || !resolvable? 98 | 99 | url.lstrip("https://").lstrip("http://").rstrip("/").rstrip(".git") 100 | end 101 | 102 | def slug 103 | if provider_resolver? 104 | "#{resolver}.com/#{url}" 105 | else 106 | nice_url 107 | end 108 | end 109 | 110 | def base_url_source(refname = nil) 111 | refname = normalize_refname(refname) 112 | 113 | case resolver 114 | when "bitbucket" 115 | url = to_uri 116 | url.path += "/src/#{refname}/" 117 | url 118 | when "github" 119 | url = to_uri 120 | url.path += "/tree/#{refname}/" 121 | url 122 | when "gitlab" 123 | # gitlab doesn't necessarily need the `-` component but they use it by default 124 | # and it seems reasonable to be safe of any ambiguities 125 | url = to_uri 126 | url.path += "/-/tree/#{refname}/" 127 | url 128 | else 129 | nil 130 | end 131 | end 132 | 133 | def base_url_raw(refname = nil) 134 | refname = normalize_refname(refname) 135 | 136 | case resolver 137 | when "github", "bitbucket" 138 | url = to_uri 139 | url.path += "/raw/#{refname}/" 140 | url 141 | when "gitlab" 142 | # gitlab doesn't necessarily need the `-` component but they use it by default 143 | # and it seems reasonable to be safe of any ambiguities 144 | url = to_uri 145 | url.path += "/-/raw/#{refname}/" 146 | url 147 | else 148 | nil 149 | end 150 | end 151 | 152 | private def normalize_refname(refname) 153 | case refname 154 | when Nil, "HEAD" 155 | "master" 156 | else 157 | refname 158 | end 159 | end 160 | 161 | def resolvable? 162 | provider_resolver? || url.starts_with?("http://") || url.starts_with?("https://") 163 | end 164 | 165 | def <=>(other : self) 166 | result = name.compare(other.name, case_insensitive: true) 167 | return result unless result == 0 168 | 169 | if provider_resolver? && other.provider_resolver? 170 | result = url.compare(other.url, case_insensitive: true) 171 | return result unless result == 0 172 | 173 | resolver <=> other.resolver 174 | else 175 | slug <=> other.slug 176 | end 177 | end 178 | 179 | def to_s(io : IO) 180 | io << resolver 181 | io << ":" 182 | @url.dump_unquoted(io) 183 | end 184 | 185 | def inspect(io : IO) 186 | io << "#" 189 | end 190 | end 191 | end 192 | -------------------------------------------------------------------------------- /src/repo/resolver.cr: -------------------------------------------------------------------------------- 1 | require "shards/logger" 2 | require "shards/resolvers/resolver" 3 | require "shards/package" 4 | require "../ext/shards/resolvers/git" 5 | require "../release" 6 | 7 | class Repo 8 | class Resolver 9 | class RepoUnresolvableError < Exception 10 | end 11 | 12 | getter repo_ref 13 | 14 | def initialize(@resolver : Shards::GitResolver, @repo_ref : Repo::Ref) 15 | end 16 | 17 | def self.new(repo_ref : Repo::Ref) 18 | new(resolver_instance(repo_ref), repo_ref) 19 | end 20 | 21 | def fetch_versions : Array(String) 22 | @resolver.available_releases.map(&.value) 23 | rescue exc : Shards::Error 24 | if exc.message.try &.starts_with?("Failed to clone") 25 | raise RepoUnresolvableError.new(cause: exc) 26 | else 27 | raise exc 28 | end 29 | end 30 | 31 | def fetch_raw_spec(version : String? = nil) : String? 32 | @resolver.read_spec!(Shards::Version.new(version)) 33 | rescue exc : Shards::Error 34 | if exc.message.try &.starts_with?("Failed to clone") 35 | raise RepoUnresolvableError.new(cause: exc) 36 | elsif exc.message =~ /Missing ".*:shard.yml" for/ 37 | return 38 | else 39 | raise exc 40 | end 41 | end 42 | 43 | def fetch_file(version : String?, path : String) 44 | @resolver.fetch_file(Shards::Version.new(version), path) 45 | end 46 | 47 | def revision_info(version : String = "HEAD") : Release::RevisionInfo 48 | if version == "HEAD" 49 | version = Shards::GitHeadRef.new 50 | else 51 | version = Shards::Version.new(version) 52 | end 53 | @resolver.revision_info(version) 54 | end 55 | 56 | def latest_version_for_ref(ref) : String? 57 | @resolver.latest_version_for_ref(ref).try &.value 58 | end 59 | 60 | def self.resolver_instance(repo_ref) 61 | resolver_class = Shards::Resolver.find_class(repo_ref.resolver) 62 | unless resolver_class 63 | raise RepoUnresolvableError.new("Can't find a resolver for #{repo_ref}") 64 | end 65 | resolver = resolver_class.find_resolver(repo_ref.resolver, repo_ref.name, repo_ref.url) 66 | unless resolver.is_a?(Shards::GitResolver) 67 | raise RepoUnresolvableError.new("Invalid resolver #{resolver} for #{repo_ref}") 68 | end 69 | resolver 70 | end 71 | end 72 | end 73 | -------------------------------------------------------------------------------- /src/service/create_owner.cr: -------------------------------------------------------------------------------- 1 | require "../repo/owner" 2 | require "../fetchers/github_api" 3 | require "./update_owner_metrics" 4 | 5 | struct Service::CreateOwner 6 | property github_api : Shardbox::GitHubAPI { Shardbox::GitHubAPI.new } 7 | 8 | def initialize(@db : ShardsDB, @repo_ref : Repo::Ref) 9 | end 10 | 11 | def perform 12 | owner = Repo::Owner.from_repo_ref(@repo_ref) 13 | 14 | return unless owner 15 | 16 | db_owner = @db.get_owner?(owner.resolver, owner.slug) 17 | 18 | if db_owner 19 | # owner already exists in the database 20 | owner = db_owner 21 | else 22 | # owner does not yet exist, need to insert a new entry 23 | fetch_owner_info(owner) 24 | owner.id = @db.create_owner(owner) 25 | UpdateOwnerMetrics.new(@db).update_owner_metrics(owner.id) 26 | end 27 | 28 | assign_owner(owner) 29 | 30 | owner 31 | end 32 | 33 | private def assign_owner(owner) 34 | @db.set_owner(@repo_ref, owner.id) 35 | @db.update_owner_shards_count(owner.id) 36 | end 37 | 38 | def fetch_owner_info(owner) 39 | case owner.resolver 40 | when "github" 41 | CreateOwner.fetch_owner_info_github(owner, github_api) 42 | else 43 | # skip 44 | end 45 | end 46 | 47 | def self.fetch_owner_info_github(owner, github_api) 48 | data = github_api.fetch_owner_info(owner.slug) 49 | unless data 50 | # Skip if data could not be determined (for example GitHub API returns null when the owner was renamed) 51 | 52 | Raven.send_event Raven::Event.new( 53 | level: :info, 54 | message: "GitHub API returned null for owner", 55 | tags: { 56 | owner: owner.slug, 57 | } 58 | ) 59 | 60 | return 61 | end 62 | 63 | data.each do |key, value| 64 | case key 65 | when "bio", "description" 66 | owner.description = value.as_s? 67 | when "name" 68 | owner.name = value.as_s? 69 | else 70 | owner.extra[key.underscore] = value 71 | end 72 | end 73 | end 74 | end 75 | -------------------------------------------------------------------------------- /src/service/create_shard.cr: -------------------------------------------------------------------------------- 1 | require "../catalog" 2 | 3 | struct Service::CreateShard 4 | def initialize(@db : ShardsDB, @repo : Repo, @name : String, @entry : Catalog::Entry? = nil) 5 | raise "Repo has already a shard associated" if @repo.shard_id 6 | raise "Repo is not canonical" unless @repo.role.canonical? 7 | end 8 | 9 | def perform 10 | qualifier, shard_id = find_qualifier 11 | 12 | if shard_id 13 | # Re-taking an archived shard 14 | # TODO: Maybe this should instead create a new shard (new id) with the 15 | # archived shard's qualifier in order to isolate from previous shard 16 | # instance, which might have been a different one altogether. 17 | Service::UpdateShard.new(@db, shard_id, @entry).perform 18 | else 19 | # Create a new shard 20 | shard = build_shard(@name, qualifier) 21 | shard_id = @db.create_shard(shard) 22 | @db.log_activity "create_shard:created", repo_id: @repo.id, shard_id: shard_id 23 | end 24 | 25 | @db.connection.exec <<-SQL, @repo.id, shard_id 26 | UPDATE 27 | repos 28 | SET 29 | shard_id = $2, 30 | role = 'canonical', 31 | sync_failed_at = NULL 32 | WHERE 33 | id = $1 AND shard_id IS NULL 34 | SQL 35 | 36 | shard_id 37 | end 38 | 39 | private def build_shard(shard_name, qualifier) 40 | archived_at = nil 41 | if @entry.try &.archived? 42 | archived_at = Time.utc 43 | end 44 | 45 | Shard.new(shard_name, qualifier, 46 | description: @entry.try(&.description), 47 | archived_at: archived_at) 48 | end 49 | 50 | # Try to find a suitable shard name + qualifier. 51 | # If there is already a shard by that name, the repo could reasonable be a 52 | # mirror, a fork or simply a homonymous shard. 53 | # This is impossible to reliably detect automatically. 54 | # In essence, both forks and distinct shards need a separate shard instance. 55 | # Mirrors should point to the same shard, but such a unification needs to 56 | # be manually reviewed. 57 | def find_qualifier : {String, Int64?} 58 | unavailable_qualifiers = [] of String 59 | qualifier, archived_shard_id = possible_qualifiers do |qualifier| 60 | next unless qualifier 61 | qualifier = normalize(qualifier) 62 | 63 | shard_id = @db.get_shard_id?(@name, qualifier) 64 | 65 | if shard_id 66 | shard = @db.get_shard(shard_id) 67 | if shard.archived_at 68 | # found an archived shard, re-taking it 69 | break qualifier, shard_id 70 | else 71 | unavailable_qualifiers << qualifier 72 | end 73 | else 74 | break qualifier, nil 75 | end 76 | end 77 | 78 | unless qualifier 79 | raise "No suitable qualifier found (#{unavailable_qualifiers.inspect})" 80 | end 81 | 82 | {qualifier, archived_shard_id} 83 | end 84 | 85 | # This method yields qualifiers for this shard in order of preference 86 | private def possible_qualifiers(&) 87 | # 1. No qualifier 88 | yield "" 89 | 90 | # 2. Find a username 91 | resolver = @repo.ref.resolver 92 | url = @repo.ref.url 93 | case resolver 94 | when "github", "gitlab", "bitbucket" 95 | org = File.dirname(url) 96 | yield org 97 | 98 | yield resolver 99 | 100 | yield "#{org}-#{resolver}" 101 | when "git" 102 | # If path looks like / pattern 103 | uri = URI.parse(url) 104 | parents = Path.posix(uri.path).parents 105 | if parents.size == 2 106 | yield parents[-1].to_s 107 | end 108 | 109 | yield uri.host 110 | else 111 | raise "unreachable" 112 | end 113 | 114 | # 3. Repo name (if different from shard name) 115 | repo_name = @repo.ref.name 116 | if repo_name != @name 117 | yield repo_name 118 | end 119 | 120 | {nil, nil} 121 | end 122 | 123 | private def normalize(string) 124 | string.gsub(/[^A-Za-z_\-.]+/, '-').strip('-') 125 | end 126 | end 127 | -------------------------------------------------------------------------------- /src/service/fetch_metadata.cr: -------------------------------------------------------------------------------- 1 | require "json" 2 | require "http/client" 3 | require "../fetchers/github_api" 4 | 5 | struct Service::FetchMetadata 6 | class FetchError < Exception 7 | end 8 | 9 | property github_api : Shardbox::GitHubAPI { Shardbox::GitHubAPI.new } 10 | 11 | def initialize(@repo_ref : Repo::Ref) 12 | end 13 | 14 | def fetch_repo_metadata 15 | case @repo_ref.resolver 16 | when "github" 17 | fetch_repo_metadata_github(@repo_ref) 18 | else 19 | nil 20 | end 21 | end 22 | 23 | def fetch_repo_metadata_github(repo_ref : Repo::Ref) 24 | owner = repo_ref.owner 25 | if owner.nil? 26 | raise FetchError.new("Invalid repo_ref #{repo_ref}") 27 | end 28 | github_api.fetch_repo_metadata(owner, repo_ref.name) 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /src/service/import_categories.cr: -------------------------------------------------------------------------------- 1 | struct Service::ImportCategories 2 | def initialize(@db : ShardsDB, @catalog : Catalog) 3 | end 4 | 5 | def perform 6 | category_stats = update_categories 7 | 8 | update_categorizations 9 | delete_obsolete_categorizations 10 | 11 | @db.log_activity "import_categories:finished", metadata: category_stats 12 | 13 | category_stats 14 | end 15 | 16 | def update_categories 17 | all_categories = @db.connection.query_all <<-SQL, @catalog.categories.keys, as: {String?, String?, String?, String?} 18 | SELECT 19 | categories.slug::text, target_categories.slug, name::text, description 20 | FROM 21 | categories 22 | FULL OUTER JOIN 23 | ( 24 | SELECT unnest($1::text[]) AS slug 25 | ) AS target_categories 26 | ON categories.slug = target_categories.slug 27 | SQL 28 | 29 | deleted_categories = [] of String 30 | new_categories = [] of String 31 | updated_categories = [] of String 32 | all_categories.each do |existing_slug, new_slug, name, description| 33 | if existing_slug.nil? 34 | category = @catalog.categories[new_slug] 35 | @db.create_category(category) 36 | new_categories << new_slug.not_nil! 37 | elsif new_slug.nil? 38 | @db.remove_category(existing_slug.not_nil!) 39 | deleted_categories << existing_slug.not_nil! 40 | else 41 | category = @catalog.categories[existing_slug] 42 | if category.name != name || category.description != description 43 | @db.update_category(category) 44 | updated_categories << category.slug 45 | end 46 | end 47 | end 48 | 49 | { 50 | "deleted_categories" => deleted_categories, 51 | "new_categories" => new_categories, 52 | "updated_categories" => updated_categories, 53 | } 54 | end 55 | 56 | def update_categorizations 57 | @catalog.entries.each do |entry| 58 | @db.update_categorization(entry.repo_ref, entry.categories) 59 | end 60 | end 61 | 62 | def delete_obsolete_categorizations 63 | @db.delete_categorizations(@catalog.entries.map &.repo_ref) 64 | end 65 | end 66 | -------------------------------------------------------------------------------- /src/service/import_shard.cr: -------------------------------------------------------------------------------- 1 | require "../db" 2 | require "../repo" 3 | require "../repo/resolver" 4 | require "./sync_repo" 5 | require "./update_shard" 6 | require "./create_shard" 7 | require "../shard" 8 | 9 | struct Service::ImportShard 10 | @resolver : Repo::Resolver 11 | 12 | class Error < Exception 13 | def initialize(@repo : Repo, cause = nil) 14 | super("import_shard failed for #{repo.ref}", cause: cause) 15 | end 16 | end 17 | 18 | def initialize(@db : ShardsDB, @repo : Repo, resolver : Repo::Resolver? = nil, @entry : Catalog::Entry? = nil) 19 | @resolver = resolver || Repo::Resolver.new(@repo.ref) 20 | end 21 | 22 | def perform 23 | import_shard 24 | rescue exc 25 | ShardsDB.transaction do |db| 26 | db.repo_sync_failed(@repo) 27 | end 28 | 29 | raise Error.new(@repo, cause: exc) 30 | end 31 | 32 | # Entry point for ImportCatalog 33 | def import_shard 34 | raise "Repo has already a shard associated" if @repo.shard_id 35 | 36 | spec = retrieve_spec 37 | return unless spec 38 | 39 | shard_name = spec.name 40 | unless Shard.valid_name?(shard_name) 41 | Log.notice &.emit("invalid shard name", repo: @repo.ref.to_s, shard_name: shard_name) 42 | SyncRepo.sync_failed(@db, @repo, "invalid shard name") 43 | return 44 | end 45 | 46 | CreateShard.new(@db, @repo, shard_name, @entry).perform 47 | end 48 | 49 | private def retrieve_spec 50 | begin 51 | version = @resolver.latest_version_for_ref(nil) 52 | unless version 53 | return 54 | end 55 | spec_raw = @resolver.fetch_raw_spec(version) 56 | rescue exc : Repo::Resolver::RepoUnresolvableError 57 | SyncRepo.sync_failed(@db, @repo, "fetch_spec_failed", exc.cause) 58 | 59 | return 60 | rescue exc : Shards::Error 61 | SyncRepo.sync_failed(@db, @repo, "shards_error", exc.cause) 62 | 63 | return 64 | end 65 | 66 | unless spec_raw 67 | SyncRepo.sync_failed(@db, @repo, "spec_missing") 68 | 69 | return 70 | end 71 | 72 | Shards::Spec.from_yaml(spec_raw) 73 | end 74 | end 75 | -------------------------------------------------------------------------------- /src/service/order_releases.cr: -------------------------------------------------------------------------------- 1 | require "../db" 2 | require "../release" 3 | require "../util/software_version" 4 | 5 | class Service::OrderReleases 6 | def initialize(@db : ShardsDB, @shard_id : Int64) 7 | end 8 | 9 | def perform 10 | order_releases 11 | end 12 | 13 | def order_releases 14 | # sort releases 15 | sorted_releases = find_releases.sort 16 | 17 | # Deferred constraint check lets us switch positions without triggering 18 | # unique index violations. 19 | @db.connection.exec "SET CONSTRAINTS releases_position_uniq DEFERRED" 20 | 21 | sorted_releases.each_with_index do |release, index| 22 | @db.connection.exec <<-SQL, release.id, index 23 | UPDATE 24 | releases 25 | SET 26 | position = $2 27 | WHERE 28 | id = $1 29 | SQL 30 | end 31 | 32 | # The unique index constraint can be checked now, should all be good. 33 | @db.connection.exec "SET CONSTRAINTS releases_position_uniq IMMEDIATE" 34 | 35 | set_latest_flag(sorted_releases) 36 | end 37 | 38 | def find_releases 39 | # NOTE: We exclude HEAD versions because they can't be sorted and when there 40 | # is a HEAD version it means there are no tags at all, so there is no need 41 | # to order anyway. 42 | sql = <<-SQL 43 | SELECT 44 | id, version, yanked_at IS NOT NULL 45 | FROM 46 | releases 47 | WHERE 48 | shard_id = $1 49 | SQL 50 | 51 | releases = [] of ReleaseInfo 52 | @db.connection.query_all sql, @shard_id do |result_set| 53 | releases << ReleaseInfo.new(*result_set.read(Int64, String, Bool)) 54 | end 55 | releases 56 | end 57 | 58 | def set_latest_flag(sorted_releases) 59 | latest = sorted_releases.reverse_each.find do |release| 60 | !release.yanked? && !release.version.try(&.prerelease?) 61 | end 62 | 63 | if latest 64 | @db.connection.exec <<-SQL, latest.id 65 | UPDATE releases 66 | SET 67 | latest = true 68 | WHERE 69 | id = $1 70 | SQL 71 | else 72 | @db.connection.exec <<-SQL, @shard_id 73 | UPDATE releases 74 | SET 75 | latest = NULL 76 | WHERE 77 | shard_id = $1 AND latest = true 78 | SQL 79 | end 80 | end 81 | 82 | struct ReleaseInfo 83 | include Comparable(ReleaseInfo) 84 | 85 | getter id 86 | getter version 87 | getter? yanked 88 | 89 | def initialize(@id : Int64, version : String, @yanked : Bool) 90 | @version = SoftwareVersion.parse?(version) 91 | end 92 | 93 | def <=>(other : self) 94 | if version = self.version 95 | if other_version = other.version 96 | version <=> other_version 97 | else 98 | -1 99 | end 100 | else 101 | other.version ? 1 : 0 102 | end 103 | end 104 | end 105 | end 106 | -------------------------------------------------------------------------------- /src/service/sync_dependencies.cr: -------------------------------------------------------------------------------- 1 | require "../db" 2 | require "../repo" 3 | 4 | # This service upserts a dependency. 5 | class Service::SyncDependencies 6 | @version : String? 7 | @shard_id : Int64? 8 | 9 | def initialize(@db : ShardsDB, @release_id : Int64) 10 | end 11 | 12 | def sync_dependencies(dependencies : Array(Dependency)) 13 | persisted_dependencies = query_dependencies 14 | 15 | persisted_dependencies.each do |persisted_dependency| 16 | new_dependency = dependencies.find { |d| d.name == persisted_dependency.name && d.scope == persisted_dependency.scope } 17 | if new_dependency 18 | if new_dependency != persisted_dependency 19 | update_dependency(new_dependency) 20 | end 21 | else 22 | remove_dependency(persisted_dependency) 23 | end 24 | end 25 | 26 | dependencies.each do |new_dependency| 27 | persisted_dependency = persisted_dependencies.find { |d| d.name == new_dependency.name && d.scope == new_dependency.scope } 28 | unless persisted_dependency 29 | add_dependency(new_dependency) 30 | end 31 | end 32 | end 33 | 34 | def add_dependency(dependency) 35 | repo_ref = dependency.repo_ref 36 | 37 | if repo_ref 38 | repo_id = @db.get_repo_id?(repo_ref) 39 | unless repo_id 40 | repo_id = @db.create_repo(Repo.new(repo_ref, shard_id: nil)) 41 | end 42 | else 43 | repo_id = nil 44 | end 45 | 46 | dep_id = @db.connection.query_one? <<-SQL, @release_id, dependency.name, dependency.spec.to_json, dependency.scope, repo_id, as: Int64 47 | INSERT INTO dependencies 48 | (release_id, name, spec, scope, repo_id) 49 | VALUES 50 | ($1, $2, $3::jsonb, $4, $5) 51 | ON CONFLICT ON CONSTRAINT dependencies_uniq DO NOTHING 52 | RETURNING release_id 53 | SQL 54 | 55 | if dep_id.nil? 56 | # Insertion failed because a dependency with the same name exists already 57 | # for this release. It is probably a duplication of dev and runtime 58 | # dependencies. 59 | 60 | log("sync_dependencies:duplicate", dependency, repo_id) 61 | 62 | # Override previous if this is runtime dependency 63 | if dependency.scope.runtime? 64 | update_dependency(dependency) 65 | end 66 | 67 | return 68 | end 69 | 70 | log("sync_dependencies:created", dependency, repo_id) 71 | end 72 | 73 | def remove_dependency(dependency) 74 | @db.connection.exec <<-SQL, @release_id, dependency.name 75 | DELETE FROM 76 | dependencies 77 | WHERE 78 | release_id = $1 AND name = $2 79 | SQL 80 | 81 | if repo_ref = dependency.repo_ref 82 | repo_id = @db.get_repo_id?(repo_ref) 83 | else 84 | repo_id = nil 85 | end 86 | 87 | log("sync_dependencies:removed", dependency, repo_id) 88 | end 89 | 90 | def update_dependency(dependency) 91 | if repo_ref = dependency.repo_ref 92 | repo_id = @db.get_repo_id(repo_ref) 93 | else 94 | repo_id = nil 95 | end 96 | 97 | @db.connection.exec <<-SQL, @release_id, dependency.name, dependency.spec.to_json, dependency.scope, repo_id 98 | UPDATE dependencies SET 99 | spec = $3::jsonb, 100 | scope = $4, 101 | repo_id = $5 102 | WHERE 103 | release_id = $1 AND name = $2 104 | SQL 105 | 106 | log("sync_dependencies:updated", dependency, repo_id) 107 | end 108 | 109 | private def log(event, dependency, repo_id, metadata = nil) 110 | meta = { 111 | "release" => version, 112 | "name" => dependency.name, 113 | "scope" => dependency.scope.to_s, 114 | } 115 | if repo_id.nil? 116 | meta["repo_ref"] = dependency.repo_ref.to_s 117 | end 118 | 119 | if metadata 120 | meta.merge! metadata 121 | end 122 | @db.log_activity(event, repo_id, shard_id, meta) 123 | end 124 | 125 | private def version : String 126 | unless version = @version 127 | version = fetch_version_and_shard_id[0] 128 | end 129 | 130 | version 131 | end 132 | 133 | private def shard_id : Int64 134 | unless shard_id = @shard_id 135 | shard_id = fetch_version_and_shard_id[1] 136 | end 137 | 138 | shard_id 139 | end 140 | 141 | private def fetch_version_and_shard_id 142 | result = @db.connection.query_one("SELECT version, shard_id FROM releases WHERE id = $1", @release_id, as: {String, Int64}) 143 | @version, @shard_id = result 144 | result 145 | end 146 | 147 | private def query_dependencies 148 | dependencies = @db.connection.query_all <<-SQL, @release_id, as: {String, JSON::Any, String} 149 | SELECT 150 | name::text, spec, scope::text 151 | FROM 152 | dependencies 153 | WHERE 154 | release_id = $1 155 | SQL 156 | 157 | dependencies.map do |name, spec, scope| 158 | Dependency.new(name, spec, Dependency::Scope.parse(scope)) 159 | end 160 | end 161 | end 162 | -------------------------------------------------------------------------------- /src/service/sync_release.cr: -------------------------------------------------------------------------------- 1 | require "../db" 2 | require "../ext/yaml/any" 3 | require "../repo/resolver" 4 | require "../dependency" 5 | require "../util/software_version" 6 | require "./sync_dependencies" 7 | require "./import_shard" 8 | 9 | # This service synchronizes the information about a release in the database. 10 | class Service::SyncRelease 11 | def initialize(@db : ShardsDB, @shard_id : Int64, @version : String) 12 | end 13 | 14 | def perform 15 | repo = @db.find_canonical_repo(@shard_id) 16 | resolver = Repo::Resolver.new(repo.ref) 17 | 18 | Raven.tags_context repo: repo.ref.to_s, version: @version 19 | 20 | sync_release(resolver) 21 | end 22 | 23 | def sync_release(resolver) 24 | release, spec = get_spec(resolver) 25 | 26 | release_id = upsert_release(@shard_id, release) 27 | 28 | sync_dependencies(release_id, spec, release.spec) 29 | sync_files(release_id, resolver) 30 | sync_repos_stats(release_id, resolver) 31 | end 32 | 33 | def get_spec(resolver) 34 | spec_raw = resolver.fetch_raw_spec(@version) 35 | 36 | if spec_raw 37 | spec = Shards::Spec.from_yaml(spec_raw) 38 | spec_json = JSON.parse(YAML.parse(spec_raw).to_json).as_h 39 | else 40 | # No `shard.yml` found, using mock spec 41 | spec = Shards::Spec.new(name: @db.get_shard(@shard_id).name, version: Shards::Version.new(@version)) 42 | spec_json = {} of String => JSON::Any 43 | end 44 | 45 | # We're always using the tagged version as identifier (@version), which 46 | # might be different from the version reported in the spec (spec.version). 47 | # This is certainly unexpected but actually not a huge issue, we can just 48 | # accept this. 49 | # These mismatching releases can be queried from the database: 50 | # SELECT version, spec->>'version' FROM releases WHERE version != spec->>'version' 51 | revision_info = resolver.revision_info(@version) 52 | release = Release.new(@version, revision_info, spec_json) 53 | 54 | return release, spec 55 | end 56 | 57 | def check_version_match(tag_version, spec_version) 58 | # quick check if versions match up 59 | return true if tag_version == spec_version 60 | 61 | # Accept any version in spec on HEAD (i.e. there is no tag version) 62 | return true if tag_version == "HEAD" 63 | 64 | # If versions are not identical, maybe they're noted differently. 65 | # Try parsing as SoftwareVersion 66 | return false unless SoftwareVersion.valid?(tag_version) && SoftwareVersion.valid?(spec_version) 67 | tag_version = SoftwareVersion.new(tag_version) 68 | spec_version = SoftwareVersion.new(spec_version) 69 | 70 | return true if tag_version == spec_version 71 | 72 | # We also accept versions tagged as pre-release having the release version 73 | # in the spec 74 | return true if tag_version.release == spec_version 75 | 76 | false 77 | end 78 | 79 | def upsert_release(shard_id : Int64, release : Release) 80 | release_id = @db.connection.query_one?(<<-SQL, shard_id, release.version, as: Int64) 81 | SELECT id FROM releases WHERE shard_id = $1 AND version = $2 82 | SQL 83 | 84 | if release_id 85 | # update 86 | release.id = release_id 87 | @db.update_release(release) 88 | else 89 | # insert 90 | release_id = @db.create_release(shard_id, release) 91 | @db.log_activity("sync_release:created", nil, shard_id, {"version" => release.version}) 92 | end 93 | 94 | release_id 95 | end 96 | 97 | def sync_dependencies(release_id, spec : Shards::Spec, raw_spec) 98 | dependencies = [] of Dependency 99 | dependencies_json = raw_spec["dependencies"]? 100 | spec.dependencies.each do |spec_dependency| 101 | dependencies << Dependency.new(spec_dependency.name, dependencies_json.not_nil![spec_dependency.name], :runtime) 102 | end 103 | 104 | development_dependencies_json = raw_spec["development_dependencies"]? 105 | spec.development_dependencies.each do |spec_dependency| 106 | dependencies << Dependency.new(spec_dependency.name, development_dependencies_json.not_nil![spec_dependency.name], :development) 107 | end 108 | 109 | SyncDependencies.new(@db, release_id).sync_dependencies(dependencies) 110 | end 111 | 112 | def sync_repos_stats(release_id, resolver) 113 | end 114 | 115 | README_NAMES = ["README.md", "Readme.md"] 116 | 117 | def sync_files(release_id, resolver) 118 | found = README_NAMES.each do |name| 119 | if content = resolver.fetch_file(@version, name) 120 | @db.put_file(release_id, README_NAMES.first, content) 121 | break true 122 | end 123 | end 124 | unless found 125 | @db.delete_file(release_id, README_NAMES.first) 126 | end 127 | end 128 | end 129 | -------------------------------------------------------------------------------- /src/service/sync_repo.cr: -------------------------------------------------------------------------------- 1 | require "../ext/yaml/any" 2 | require "../repo/resolver" 3 | require "./sync_release" 4 | require "./order_releases" 5 | require "./fetch_metadata" 6 | require "./create_owner" 7 | 8 | # This service synchronizes the information about a repository in the database. 9 | struct Service::SyncRepo 10 | def initialize(@db : ShardsDB, @repo_ref : Repo::Ref) 11 | end 12 | 13 | Log = Shardbox::Log.for("service.sync_repo") 14 | 15 | def perform 16 | Log.debug { "Sync repo #{@repo_ref}" } 17 | 18 | duration = Time.measure do 19 | resolver = Repo::Resolver.new(@repo_ref) 20 | 21 | sync_repo(resolver) 22 | end 23 | 24 | Log.debug { "Done sync repo #{@repo_ref} in #{duration}" } 25 | end 26 | 27 | def sync_repo(resolver : Repo::Resolver) 28 | Raven.tags_context repo: @repo_ref.to_s 29 | repo = @db.get_repo(resolver.repo_ref) 30 | shard_id = repo.shard_id 31 | 32 | unless shard_id 33 | shard_id = ImportShard.new(@db, repo, resolver).perform 34 | 35 | return unless shard_id 36 | end 37 | 38 | if repo.role.canonical? 39 | # We only track releases on canonical repos 40 | 41 | begin 42 | sync_releases(resolver, shard_id) 43 | rescue exc : Repo::Resolver::RepoUnresolvableError 44 | SyncRepo.sync_failed(@db, repo, "clone_failed", exc.cause) 45 | 46 | return 47 | rescue exc : Shards::ParseError 48 | SyncRepo.sync_failed(@db, repo, "spec_invalid", exc, tags: {"error_message" => exc.message}) 49 | 50 | return 51 | end 52 | end 53 | 54 | sync_metadata(repo) 55 | 56 | sync_owner(repo) 57 | end 58 | 59 | def sync_releases(resolver, shard_id) 60 | versions = resolver.fetch_versions 61 | 62 | if versions.empty? 63 | versions = ["HEAD"] 64 | end 65 | 66 | failed_versions = [] of String 67 | versions.each do |version| 68 | if !SoftwareVersion.valid?(version) && version != "HEAD" 69 | # TODO: What should happen when a version tag is invalid? 70 | # Ignoring this release for now and sending a note to sentry. 71 | 72 | Raven.send_event Raven::Event.new( 73 | level: :warning, 74 | message: "Invalid version, ignoring release.", 75 | tags: { 76 | repo: resolver.repo_ref.to_s, 77 | tag_version: version, 78 | } 79 | ) 80 | next 81 | end 82 | 83 | begin 84 | SyncRelease.new(@db, shard_id, version).sync_release(resolver) 85 | rescue exc : Shards::ParseError 86 | repo = @db.get_repo(resolver.repo_ref) 87 | SyncRepo.sync_failed(@db, repo, "sync_release:failed", exc, tags: {"error_message" => exc.message, "version" => version}) 88 | 89 | failed_versions << version 90 | end 91 | end 92 | 93 | versions -= failed_versions 94 | yank_releases_with_missing_versions(shard_id, versions) 95 | 96 | Service::OrderReleases.new(@db, shard_id).perform 97 | end 98 | 99 | def yank_releases_with_missing_versions(shard_id, versions) 100 | yanked = @db.connection.query_all <<-SQL, shard_id, versions, as: String 101 | SELECT 102 | version 103 | FROM 104 | releases 105 | WHERE 106 | shard_id = $1 AND yanked_at IS NULL AND version <> ALL($2) 107 | ORDER BY position 108 | SQL 109 | 110 | @db.connection.exec <<-SQL, yanked 111 | UPDATE 112 | releases 113 | SET 114 | yanked_at = NOW() 115 | WHERE 116 | version = ANY($1) 117 | SQL 118 | 119 | yanked.each do |version| 120 | @db.log_activity("sync_repo:release:yanked", nil, shard_id, {"version" => version}) 121 | end 122 | end 123 | 124 | def sync_metadata(repo : Repo, *, fetch_service = Service::FetchMetadata.new(repo.ref)) 125 | begin 126 | metadata = fetch_service.fetch_repo_metadata 127 | rescue exc : Shardbox::FetchError 128 | SyncRepo.sync_failed(@db, repo, "fetch_metadata_failed", exc) 129 | 130 | return 131 | end 132 | 133 | metadata ||= Repo::Metadata.new 134 | 135 | @db.connection.exec <<-SQL, repo.id, metadata.to_json 136 | UPDATE 137 | repos 138 | SET 139 | synced_at = NOW(), 140 | sync_failed_at = NULL, 141 | metadata = $2::jsonb 142 | WHERE 143 | id = $1 144 | SQL 145 | 146 | @db.log_activity "sync_repo:synced", repo_id: repo.id 147 | end 148 | 149 | def sync_owner(repo, *, service = CreateOwner.new(@db, repo.ref)) 150 | unless @db.get_owner?(repo.ref) 151 | service.perform 152 | end 153 | end 154 | 155 | def self.log_sync_failed(repo : Repo, event, exc = nil, metadata = nil) 156 | # Log failure in a separate connection because the main transaction 157 | # has already failed and won't be committed. 158 | ShardsDB.transaction do |db| 159 | log_sync_failed(db, repo, event, exc, metadata) 160 | end 161 | end 162 | 163 | def self.log_sync_failed(db, repo : Repo, event, exc = nil, metadata = nil) 164 | db.repo_sync_failed(repo) 165 | 166 | metadata ||= {} of String => String 167 | metadata["repo_role"] ||= repo.role.to_s 168 | db.log_activity "sync_repo:#{event}", repo_id: repo.id, shard_id: repo.shard_id, metadata: metadata, exc: exc 169 | rescue exc : PQ::PQError 170 | Shardbox::Log.trace(exception: exc) { "Secondary db error in log_sync_failed" } 171 | # ignore secondary DB error 172 | end 173 | 174 | def self.sync_failed(db, repo : Repo, event, exc = nil, tags = nil) 175 | log_sync_failed(db, repo, event, exc, tags) 176 | 177 | tags ||= {} of String => String 178 | tags["repo_role"] ||= repo.role.to_s 179 | 180 | if exc 181 | tags["exception"] ||= exc.class.to_s 182 | tags["error_message"] ||= exc.to_s 183 | end 184 | 185 | tags["repo"] ||= repo.ref.to_s 186 | tags["event"] ||= event 187 | 188 | Raven.send_event Raven::Event.new( 189 | level: :warning, 190 | message: "Failed to sync repository", 191 | tags: tags 192 | ) 193 | end 194 | end 195 | -------------------------------------------------------------------------------- /src/service/sync_repos.cr: -------------------------------------------------------------------------------- 1 | require "../db" 2 | require "./sync_repo" 3 | 4 | # This service synchronizes the information about a repository in the database. 5 | struct Service::SyncRepos 6 | Log = Shardbox::Log.for("service.sync_repos") 7 | 8 | def initialize(@db : ShardsDB, @older_than : Time, @ratio : Float32) 9 | @repo_refs_count = 0 10 | @failures_count = 0 11 | @pending_repos_count = 0 12 | end 13 | 14 | def self.new(db, age : Time::Span = 24.hours, ratio : Number = 0.1) 15 | new(db, age.ago, ratio.to_f32) 16 | end 17 | 18 | def perform 19 | elapsed = Time.measure do 20 | # 1. Sync repos that haven't been synced since @older_than 21 | sync_repos 22 | # 2. Sync new repos that have never been processed (newly discovered dependencies) 23 | sync_all_pending_repos 24 | # 3. Update dependency table 25 | update_shard_dependencies 26 | end 27 | 28 | @db.log_activity "sync_repos:finished", metadata: { 29 | older_than: @older_than, 30 | ratio: @ratio, 31 | elapsed_time: elapsed.to_s, 32 | repo_refs_count: @repo_refs_count, 33 | failures_count: @failures_count, 34 | pending_repos_count: @pending_repos_count, 35 | } 36 | end 37 | 38 | def sync_all_pending_repos(limit : Int32? = nil) 39 | iteration = 0 40 | loop do 41 | iteration += 1 42 | pending = @db.repos_pending_sync 43 | break if pending.empty? 44 | if limit && pending.size > limit 45 | pending = pending.first(limit) 46 | end 47 | Log.debug { "Syncing pending repos (#{iteration}): #{pending.size}" } 48 | pending.each do |repo| 49 | sync_repo(repo.ref) 50 | @pending_repos_count += 1 51 | end 52 | Log.debug { "Done syncing pending repos (#{iteration}): #{pending.size}" } 53 | if limit 54 | return 55 | end 56 | end 57 | end 58 | 59 | def sync_repos 60 | repo_refs = @db.connection.query_all <<-SQL, @older_than, @ratio, as: {String, String} 61 | WITH repos_update AS ( 62 | SELECT 63 | id, resolver, url, shard_id, synced_at, sync_failed_at 64 | FROM 65 | repos 66 | WHERE 67 | (synced_at IS NULL OR synced_at < $1) 68 | AND (sync_failed_at IS NULL OR sync_failed_at < $1) 69 | AND role <> 'obsolete' 70 | ) 71 | SELECT 72 | resolver::text, url::text 73 | FROM 74 | repos_update 75 | ORDER BY 76 | COALESCE(sync_failed_at, synced_at) ASC 77 | LIMIT (SELECT COUNT(*) FROM repos_update) * $2::real 78 | SQL 79 | 80 | Log.debug { "Syncing #{repo_refs.size} repos" } 81 | repo_refs.each do |repo_ref| 82 | repo_ref = Repo::Ref.new(*repo_ref) 83 | sync_repo(repo_ref) 84 | end 85 | Log.debug { "Done syncing repos" } 86 | @repo_refs_count = repo_refs.size 87 | end 88 | 89 | def sync_repo(repo_ref) 90 | ShardsDB.transaction do |db| 91 | begin 92 | Service::SyncRepo.new(db, repo_ref).perform 93 | rescue exc 94 | Raven.capture(exc) 95 | Log.error(exception: exc) { "Failure while syncing repo #{repo_ref}" } 96 | @failures_count += 1 97 | end 98 | end 99 | end 100 | 101 | def update_shard_dependencies 102 | @db.connection.exec <<-SQL 103 | SELECT shard_dependencies_materialize() 104 | SQL 105 | end 106 | end 107 | -------------------------------------------------------------------------------- /src/service/update_owner_metrics.cr: -------------------------------------------------------------------------------- 1 | require "../db" 2 | 3 | struct Service::UpdateOwnerMetrics 4 | def initialize(@db : ShardsDB) 5 | end 6 | 7 | def perform 8 | owner_ids = @db.connection.query_all <<-SQL, as: Int64 9 | SELECT 10 | id 11 | FROM 12 | owners 13 | SQL 14 | owner_ids.each do |id| 15 | update_owner_metrics(id) 16 | end 17 | end 18 | 19 | def update_owner_metrics(id) 20 | @db.connection.exec <<-SQL, id 21 | SELECT owner_metrics_calculate($1) 22 | SQL 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /src/service/update_shard.cr: -------------------------------------------------------------------------------- 1 | require "../db" 2 | 3 | struct Service::UpdateShard 4 | getter! entry 5 | 6 | def initialize(@db : ShardsDB, @shard_id : Int64, @entry : Catalog::Entry?) 7 | end 8 | 9 | def perform 10 | return unless @entry 11 | 12 | shard = @db.get_shard(@shard_id) 13 | 14 | if shard.archived? && !entry.archived? 15 | # unarchive 16 | shard.archived_at = nil 17 | @db.log_activity("update_shard:unarchived", nil, @shard_id) 18 | elsif !shard.archived? && entry.archived? 19 | # archive 20 | shard.archived_at = Time.utc 21 | @db.log_activity("update_shard:archived", nil, @shard_id) 22 | end 23 | 24 | if entry.description != shard.description 25 | @db.log_activity("update_shard:description_changed", nil, @shard_id, metadata: {"old_value": shard.description}) 26 | shard.description = entry.description 27 | end 28 | 29 | @db.connection.exec <<-SQL, @shard_id, shard.description, shard.archived_at 30 | UPDATE 31 | shards 32 | SET 33 | description = $2, 34 | archived_at = $3 35 | WHERE 36 | id = $1 37 | SQL 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /src/service/update_shard_metrics.cr: -------------------------------------------------------------------------------- 1 | require "../db" 2 | 3 | struct Service::UpdateShardMetrics 4 | def initialize(@db : ShardsDB) 5 | end 6 | 7 | def perform 8 | shard_ids = @db.connection.query_all <<-SQL, as: Int64 9 | SELECT 10 | id 11 | FROM 12 | shards 13 | SQL 14 | shard_ids.each do |id| 15 | update_shard_metrics(id) 16 | end 17 | end 18 | 19 | def update_shard_metrics(id) 20 | @db.connection.exec <<-SQL, id 21 | SELECT shard_metrics_calculate($1) 22 | SQL 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /src/service/worker_loop.cr: -------------------------------------------------------------------------------- 1 | require "../db" 2 | 3 | # This service runs in the background and schedules commands. 4 | struct Service::WorkerLoop 5 | getter sync_interval = 60 6 | getter metrics_schedule = 4 # hour of day 7 | 8 | getter? running : Bool = false 9 | 10 | @notify_connection : PG::ListenConnection? 11 | 12 | def initialize 13 | @channel = Channel(String).new 14 | @processes = [] of Process 15 | end 16 | 17 | def stop 18 | return unless running? 19 | 20 | puts "Shutting down." 21 | @running = false 22 | 23 | @notify_connection.try &.close 24 | 25 | @processes.each do |process| 26 | process.kill(Signal::INT) unless process.terminated? 27 | end 28 | 29 | @channel.send "shutdown" 30 | end 31 | 32 | def perform 33 | last_repo_sync = ShardsDB.connect do |db| 34 | db.last_repo_sync 35 | end 36 | 37 | p! last_repo_sync 38 | unless last_repo_sync 39 | last_repo_sync = Time.utc - sync_interval.minutes 40 | end 41 | 42 | @notify_connection = listen_for_notifications 43 | 44 | scheduled(last_repo_sync) do 45 | execute("sync_repos") 46 | 47 | last_metrics_calc = ShardsDB.connect do |db| 48 | db.last_metrics_calc 49 | end 50 | 51 | puts "metrics last performed at #{last_metrics_calc}" 52 | p! Time.utc.at_beginning_of_day + metrics_schedule.hours - 24.hours 53 | unless last_metrics_calc && last_metrics_calc > Time.utc.at_beginning_of_day + metrics_schedule.hours - 24.hours 54 | execute("update_metrics") 55 | end 56 | end 57 | end 58 | 59 | def scheduled(scheduled_time : Time, &) 60 | @running = true 61 | 62 | while running? 63 | scheduled_time = scheduled_time + sync_interval.minutes 64 | 65 | wait_seconds = scheduled_time - Time.utc 66 | if wait_seconds < Time::Span.zero 67 | wait_seconds = 0 68 | scheduled_time = Time.utc 69 | end 70 | 71 | puts "Next sync run at #{scheduled_time} (triggers in #{wait_seconds} seconds)..." 72 | sleep wait_seconds 73 | 74 | yield 75 | end 76 | end 77 | 78 | def execute(action) 79 | Process.run(PROGRAM_NAME, [action], output: :inherit, error: :inherit) 80 | end 81 | 82 | def listen_for_notifications 83 | ShardsDB.listen_for_jobs do |notification| 84 | if notification.payload == "import_catalog" 85 | execute("import_catalog") 86 | else 87 | puts "Received unrecognized notification: #{notification.payload}" 88 | end 89 | end 90 | end 91 | end 92 | -------------------------------------------------------------------------------- /src/shard.cr: -------------------------------------------------------------------------------- 1 | class Shard 2 | getter name : String 3 | getter qualifier : String 4 | property description : String? 5 | property! id : Int64 6 | property archived_at : Time? 7 | property merged_with : Int64? 8 | 9 | def self.valid_name?(name : String) : Bool 10 | name.matches?(/^[A-Za-z0-9_\-.]{1,100}$/) 11 | end 12 | 13 | def initialize(@name : String, @qualifier : String = "", @description : String? = nil, @archived_at : Time? = nil, @merged_with : Int64? = nil, @id : Int64? = nil) 14 | end 15 | 16 | def_equals_and_hash name, qualifier, description, archived_at 17 | 18 | def display_name 19 | if qualifier.empty? 20 | name 21 | else 22 | "#{name}~#{qualifier}" 23 | end 24 | end 25 | 26 | def slug 27 | display_name.downcase 28 | end 29 | 30 | def archived? 31 | !archived_at.nil? 32 | end 33 | 34 | def inspect(io : IO) 35 | to_s(io) 36 | end 37 | 38 | def to_s(io : IO) 39 | io << "#" 48 | end 49 | end 50 | -------------------------------------------------------------------------------- /src/util/software_version.cr: -------------------------------------------------------------------------------- 1 | # The `SoftwareVersion` type represents a version number. 2 | # 3 | # An instance can be created from a version string which consists of a series of 4 | # segments separated by periods. Each segment contains one ore more alpanumerical 5 | # ASCII characters. The first segment is expected to contain only digits. 6 | # 7 | # There may be one instance of a dash (`-`) which denotes the version as a 8 | # pre-release. It is otherwise equivalent to a period. 9 | # 10 | # Optional version metadata may be attached and is separated by a plus character (`+`). 11 | # All content following a `+` is considered metadata. 12 | # 13 | # This format is described by the regular expression: 14 | # `/[0-9]+(?>\.[0-9a-zA-Z]+)*(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-.]+)?/` 15 | # 16 | # This implementation is compatible to popular versioning schemes such as 17 | # [`SemVer`](https://semver.org/) and [`CalVer`](https://calver.org/) but 18 | # doesn't enforce any particular one. 19 | # 20 | # It behaves mostly equivalent to [`Gem::Version`](http://docs.seattlerb.org/rubygems/Gem/Version.html) from `rubygems`. 21 | # 22 | # ## Sort order 23 | # This wrapper type is mostly important for properly sorting version numbers, 24 | # because generic lexical sorting doesn't work: For instance, `3.10` is supposed 25 | # to be greater than `3.2`. 26 | # 27 | # Every set of consecutive digits anywhere in the string are interpreted as a 28 | # decimal number and numerically sorted. Letters are lexically sorted. 29 | # Periods (and dash) delimit numbers but don't affect sort order by themselves. 30 | # Thus `1.0a` is considered equal to `1.0.a`. 31 | # 32 | # ## Pre-release 33 | # If a version number contains a letter (`a-z`) then that version is considered 34 | # a pre-release. Pre-releases sort lower than the rest of the version prior to 35 | # the first letter (or dash). For instance `1.0-b` compares lower than `1.0` but 36 | # greater than `1.0-a`. 37 | struct SoftwareVersion 38 | include Comparable(self) 39 | 40 | # :nodoc: 41 | VERSION_PATTERN = /[0-9]+(?>\.[0-9a-zA-Z]+)*(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-.]+)?/ 42 | # :nodoc: 43 | ANCHORED_VERSION_PATTERN = /\A\s*(#{VERSION_PATTERN})\s*\z/ 44 | 45 | @string : String 46 | 47 | # Returns `true` if *string* is a valid version format. 48 | def self.valid?(string : String) : Bool 49 | !ANCHORED_VERSION_PATTERN.match(string).nil? 50 | end 51 | 52 | # Constructs a `Version` from *string*. 53 | protected def initialize(@string : String) 54 | end 55 | 56 | # Parses an instance from a string. 57 | # 58 | # A version string is a series of digits or ASCII letters separated by dots. 59 | # 60 | # Returns `nil` if *string* describes an invalid version (see `.valid?`). 61 | def self.parse?(string : String) : self? 62 | # If string is an empty string convert it to 0 63 | string = "0" if string =~ /\A\s*\Z/ 64 | 65 | return unless valid?(string) 66 | 67 | new(string.strip) 68 | end 69 | 70 | # Parses an instance from a string. 71 | # 72 | # A version string is a series of digits or ASCII letters separated by dots. 73 | # 74 | # Raises `ArgumentError` if *string* describes an invalid version (see `.valid?`). 75 | def self.parse(string : String) : self 76 | parse?(string) || raise ArgumentError.new("Malformed version string #{string.inspect}") 77 | end 78 | 79 | # Constructs a `Version` from the string representation of *version* number. 80 | def self.new(version : Number) 81 | new(version.to_s) 82 | end 83 | 84 | # Appends the string representation of this version to *io*. 85 | def to_s(io : IO) 86 | io << @string 87 | end 88 | 89 | # Returns the string representation of this version. 90 | def to_s : String 91 | @string 92 | end 93 | 94 | # Returns `true` if this version is a pre-release version. 95 | # 96 | # A version is considered pre-release if it contains an ASCII letter or `-`. 97 | # 98 | # ``` 99 | # SoftwareVersion.new("1.0.0").prerelease? # => false 100 | # SoftwareVersion.new("1.0.0-dev").prerelease? # => true 101 | # SoftwareVersion.new("1.0.0-1").prerelease? # => true 102 | # SoftwareVersion.new("1.0.0a1").prerelease? # => true 103 | # ``` 104 | def prerelease? : Bool 105 | @string.each_char do |char| 106 | if char.ascii_letter? || char == '-' 107 | return true 108 | elsif char == '+' 109 | # the following chars are metadata 110 | return false 111 | end 112 | end 113 | 114 | false 115 | end 116 | 117 | # Returns the metadata attached to this version or `nil` if no metadata available. 118 | # 119 | # ``` 120 | # SoftwareVersion.new("1.0.0").metadata # => nil 121 | # SoftwareVersion.new("1.0.0-rc1").metadata # => nil 122 | # SoftwareVersion.new("1.0.0+build1").metadata # => "build1" 123 | # SoftwareVersion.new("1.0.0-rc1+build1").metadata # => "build1" 124 | # ``` 125 | def metadata : String? 126 | if index = @string.byte_index('+'.ord) 127 | @string.byte_slice(index + 1, @string.bytesize - index - 1) 128 | end 129 | end 130 | 131 | # Returns version representing the release version associated with this version. 132 | # 133 | # If this version is a pre-release (see `#prerelease?`) a new instance will be created 134 | # with the same version string before the first ASCII letter or `-`. 135 | # 136 | # Version metadata (see `#metadata`) will be stripped. 137 | # 138 | # ``` 139 | # SoftwareVersion.new("1.0.0").release # => SoftwareVersion.new("1.0.0") 140 | # SoftwareVersion.new("1.0.0-dev").release # => SoftwareVersion.new("1.0.0") 141 | # SoftwareVersion.new("1.0.0-1").release # => SoftwareVersion.new("1.0.0") 142 | # SoftwareVersion.new("1.0.0a1").release # => SoftwareVersion.new("1.0.0") 143 | # SoftwareVersion.new("1.0.0+b1").release # => SoftwareVersion.new("1.0.0") 144 | # SoftwareVersion.new("1.0.0-rc1+b1").release # => SoftwareVersion.new("1.0.0") 145 | # ``` 146 | def release : self 147 | @string.each_char_with_index do |char, index| 148 | if char.ascii_letter? || char == '-' || char == '+' 149 | return self.class.new(@string.byte_slice(0, index)) 150 | end 151 | end 152 | 153 | self 154 | end 155 | 156 | # Compares this version with *other* returning -1, 0, or 1 if the 157 | # other version is larger, the same, or smaller than this one. 158 | def <=>(other : self) 159 | lstring = @string 160 | rstring = other.@string 161 | lindex = 0 162 | rindex = 0 163 | 164 | while true 165 | lchar = lstring.byte_at?(lindex).try &.chr 166 | rchar = rstring.byte_at?(rindex).try &.chr 167 | 168 | # Both strings have been entirely consumed, they're identical 169 | return 0 if lchar.nil? && rchar.nil? 170 | 171 | ldelimiter = {'.', '-'}.includes?(lchar) 172 | rdelimiter = {'.', '-'}.includes?(rchar) 173 | 174 | # Skip delimiters 175 | lindex += 1 if ldelimiter 176 | rindex += 1 if rdelimiter 177 | next if ldelimiter || rdelimiter 178 | 179 | # If one string is consumed, the other is either ranked higher (char is a digit) 180 | # or lower (char is letter, making it a pre-release tag). 181 | if lchar.nil? 182 | return rchar.not_nil!.ascii_letter? ? 1 : -1 183 | elsif rchar.nil? 184 | return lchar.ascii_letter? ? -1 : 1 185 | end 186 | 187 | # Try to consume consequitive digits into a number 188 | lnumber, new_lindex = consume_number(lstring, lindex) 189 | rnumber, new_rindex = consume_number(rstring, rindex) 190 | 191 | # Proceed depending on where a number was found on each string 192 | case {new_lindex, new_rindex} 193 | when {lindex, rindex} 194 | # Both strings have a letter at current position. 195 | # They are compared (lexical) and the algorithm only continues if they 196 | # are equal. 197 | ret = lchar <=> rchar 198 | return ret unless ret == 0 199 | 200 | lindex += 1 201 | rindex += 1 202 | when {_, rindex} 203 | # Left hand side has a number, right hand side a letter (and thus a pre-release tag) 204 | return -1 205 | when {lindex, _} 206 | # Right hand side has a number, left hand side a letter (and thus a pre-release tag) 207 | return 1 208 | else 209 | # Both strings have numbers at current position. 210 | # They are compared (numerical) and the algorithm only continues if they 211 | # are equal. 212 | ret = lnumber <=> rnumber 213 | return ret unless ret == 0 214 | 215 | # Move to the next position in both strings 216 | lindex = new_lindex 217 | rindex = new_rindex 218 | end 219 | end 220 | end 221 | 222 | # Helper method to read a sequence of digits from *string* starting at 223 | # position *index* into an integer number. 224 | # It returns the consumed number and index position. 225 | private def consume_number(string : String, index : Int32) 226 | number = 0 227 | while (byte = string.byte_at?(index)) && byte.chr.ascii_number? 228 | number *= 10 229 | number += byte 230 | index += 1 231 | end 232 | {number, index} 233 | end 234 | 235 | def self.compare(a : String, b : String) 236 | new(a) <=> new(b) 237 | end 238 | 239 | def matches_pessimistic_version_constraint?(constraint : String) 240 | constraint = self.class.new(constraint).release.to_s 241 | 242 | if last_period_index = constraint.rindex('.') 243 | constraint_lead = constraint.[0...last_period_index] 244 | else 245 | constraint_lead = constraint 246 | end 247 | last_period_index = constraint_lead.bytesize 248 | 249 | # Compare the leading part of the constraint up until the last period. 250 | # If it doesn't match, the constraint is not fulfilled. 251 | return false unless @string.starts_with?(constraint_lead) 252 | 253 | # The character following the constraint lead can't be a number, otherwise 254 | # `0.10` would match `0.1` because it starts with the same three characters 255 | next_char = @string.byte_at?(last_period_index).try &.chr 256 | return true unless next_char 257 | return false if next_char.ascii_number? 258 | 259 | # We've established that constraint is met up until the second-to-last 260 | # segment. 261 | # Now we only need to ensure that the last segment is actually bigger than 262 | # the constraint so that `0.1` doesn't match `~> 0.2`. 263 | # self >= constraint 264 | constraint_number, _ = consume_number(constraint, last_period_index + 1) 265 | own_number, _ = consume_number(@string, last_period_index + 1) 266 | 267 | own_number >= constraint_number 268 | end 269 | 270 | # Custom hash implementation which produces the same hash for `a` and `b` when `a <=> b == 0` 271 | def hash(hasher) 272 | string = @string 273 | index = 0 274 | 275 | while byte = string.byte_at?(index) 276 | if {'.'.ord, '-'.ord}.includes?(byte) 277 | index += 1 278 | next 279 | end 280 | 281 | number, new_index = consume_number(string, index) 282 | 283 | if new_index != index 284 | hasher.int(number) 285 | index = new_index 286 | else 287 | hasher.int(byte) 288 | end 289 | index += 1 290 | end 291 | 292 | hasher 293 | end 294 | end 295 | -------------------------------------------------------------------------------- /src/worker.cr: -------------------------------------------------------------------------------- 1 | require "./service/import_catalog" 2 | require "./service/sync_repos" 3 | require "./service/update_shard_metrics" 4 | require "./service/update_owner_metrics" 5 | require "./service/worker_loop" 6 | require "./raven" 7 | require "uri" 8 | require "log" 9 | 10 | Log.setup do |config| 11 | stdout = Log::IOBackend.new 12 | config.bind "*", :info, stdout 13 | config.bind "service.*", :debug, stdout 14 | config.bind "shardbox.activity", :info, stdout 15 | config.bind "db.*", :info, stdout 16 | 17 | raven = Raven::LogBackend.new( 18 | capture_exceptions: true, 19 | record_breadcrumbs: true, 20 | ) 21 | config.bind "*", :warn, raven 22 | end 23 | 24 | unless ENV.has_key?("GITHUB_TOKEN") 25 | puts "Missing environment variable GITHUB_TOKEN" 26 | exit 1 27 | end 28 | 29 | # Disable git asking for credentials when cloning a repository. It's probably been deleted. 30 | # TODO: Remove this workaround (probably use libgit2 bindings instead) 31 | ENV["GIT_ASKPASS"] = "/usr/bin/test" 32 | 33 | def show_help(io) 34 | io.puts "shards-toolbox worker" 35 | io.puts "commands:" 36 | io.puts " import_catalog [] import catalog data from (default: ./catalog)" 37 | io.puts " sync_repos [ []] syncs repos not updated in last ( 0.0-1.0)" 38 | io.puts " sync_repo sync a single repo identified by " 39 | io.puts " update_metrics update shard metrics (should be run once per day)" 40 | io.puts " loop run worker loop to schedule sync and update tasks" 41 | io.puts " help show usage instructions" 42 | end 43 | 44 | def sync_all_pending_repos 45 | Shardbox::Log.info { "Syncing pending repos" } 46 | ShardsDB.connect do |db| 47 | Service::SyncRepos.new(db).sync_all_pending_repos 48 | end 49 | end 50 | 51 | case command = ARGV.shift? 52 | when "import_catalog" 53 | catalog_path = ARGV.shift? || ENV["SHARDBOX_CATALOG"]? 54 | unless catalog_path 55 | abort "No catalog path specified. Either provide program argument or environment variable SHARDBOX_CATALOG" 56 | end 57 | ShardsDB.transaction do |db| 58 | service = Service::ImportCatalog.new(db, catalog_path) 59 | service.perform 60 | end 61 | 62 | sync_all_pending_repos 63 | when "sync_repos" 64 | hours = nil 65 | ratio = nil 66 | if arg = ARGV.shift? 67 | hours = arg.to_i 68 | if arg = ARGV.shift? 69 | ratio = arg.to_f 70 | end 71 | end 72 | hours ||= ENV["SHARDBOX_WORKER_SYNC_REPO_HOURS"]?.try(&.to_i) || 24 73 | ratio ||= ENV["SHARDBOX_WORKER_SYNC_REPO_RATIO"]?.try(&.to_f) || 10.0 / hours 74 | 75 | ShardsDB.transaction do |db| 76 | Service::SyncRepos.new(db, hours.hours, ratio).perform 77 | end 78 | when "help" 79 | show_help(STDOUT) 80 | when "sync_repo" 81 | arg = ARGV.shift 82 | 83 | ShardsDB.transaction do |db| 84 | Service::SyncRepo.new(db, Repo::Ref.parse(arg)).perform 85 | end 86 | 87 | sync_all_pending_repos 88 | when "sync_pending_repos" 89 | limit = ARGV.shift?.try(&.to_i) 90 | ShardsDB.connect do |db| 91 | Service::SyncRepos.new(db).sync_all_pending_repos(limit) 92 | end 93 | when "update_metrics" 94 | ShardsDB.transaction do |db| 95 | Service::UpdateShardMetrics.new(db).perform 96 | Service::UpdateOwnerMetrics.new(db).perform 97 | end 98 | when "loop" 99 | Service::WorkerLoop.new.perform 100 | else 101 | STDERR.puts "unknown command #{command.inspect}" 102 | show_help(STDERR) 103 | exit 1 104 | end 105 | --------------------------------------------------------------------------------