├── .dockerignore ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── workflows │ └── build-and-test.yaml ├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── agent ├── agentclient │ └── client.go ├── agentserver │ ├── server.go │ └── server_test.go ├── cmd │ ├── cmd.go │ └── config.go └── main.go ├── assets ├── architecture.svg ├── benchmark.svg ├── kraken-logo-color.ai ├── kraken-logo-color.svg ├── kraken_build_index.svg ├── kraken_core.svg └── visualization.gif ├── build-index ├── cmd │ ├── cmd.go │ └── config.go ├── main.go ├── tagclient │ ├── client.go │ ├── provider.go │ └── testing.go ├── tagmodels │ └── models.go ├── tagserver │ ├── config.go │ ├── server.go │ └── server_test.go ├── tagstore │ ├── config.go │ ├── store.go │ └── store_test.go └── tagtype │ ├── default_resolver.go │ ├── docker_resolver.go │ ├── map.go │ └── map_test.go ├── config ├── agent │ ├── base.yaml │ └── test.template ├── build-index │ ├── base.yaml │ └── test.template ├── origin │ ├── base.yaml │ └── test.template ├── proxy │ ├── base.yaml │ └── test.template └── tracker │ ├── base.yaml │ └── test.template ├── core ├── blobinfo.go ├── digest.go ├── digest_test.go ├── digester.go ├── digester_test.go ├── fixtures.go ├── infohash.go ├── infohash_test.go ├── metainfo.go ├── metainfo_test.go ├── peer_context.go ├── peer_context_test.go ├── peer_id.go ├── peer_id_test.go ├── peer_info.go ├── peer_info_test.go └── piece_hash.go ├── docker ├── agent │ └── Dockerfile ├── build-index │ └── Dockerfile ├── herd │ └── Dockerfile ├── origin │ └── Dockerfile ├── proxy │ └── Dockerfile ├── setup_nginx.sh ├── testfs │ └── Dockerfile └── tracker │ └── Dockerfile ├── docs ├── ARCHITECTURE.md ├── CONFIGURATION.md ├── CONTRIBUTING.md ├── ENDPOINTS.md ├── INTEGRATEWITHHARBOR.md ├── ROADMAP.md ├── STYLEGUIDE.md ├── assets │ ├── architecture.svg │ ├── benchmark.svg │ ├── kraken-logo-color.ai │ ├── kraken-logo-color.svg │ ├── kraken_build_index.svg │ ├── kraken_core.svg │ └── visualization.gif └── index.md ├── examples ├── devcluster │ ├── README.md │ ├── agent_one_param.sh │ ├── agent_one_start_container.sh │ ├── agent_two_param.sh │ ├── agent_two_start_container.sh │ ├── config │ │ ├── agent │ │ │ └── development.yaml │ │ ├── build-index │ │ │ └── development.yaml │ │ ├── origin │ │ │ └── development.yaml │ │ ├── proxy │ │ │ └── development.yaml │ │ └── tracker │ │ │ └── development.yaml │ ├── herd_param.sh │ ├── herd_start_container.sh │ └── herd_start_processes.sh └── k8s │ ├── README.md │ └── demo.json ├── gen └── go │ └── proto │ └── p2p │ └── p2p.pb.go ├── go.mod ├── go.sum ├── helm ├── .helmignore ├── Chart.yaml ├── config │ ├── agent.yaml │ ├── build-index.yaml │ ├── origin.yaml │ ├── proxy.yaml │ └── tracker.yaml ├── templates │ ├── _helpers.tpl │ ├── agents.yaml │ ├── build-index.yaml │ ├── config.yaml │ ├── origins.yaml │ ├── proxy.yaml │ ├── testfs.yaml │ └── trackers.yaml └── values.yaml ├── lib ├── backend │ ├── backenderrors │ │ └── errors.go │ ├── client.go │ ├── config.go │ ├── constants.go │ ├── fixtures.go │ ├── gcsbackend │ │ ├── client.go │ │ ├── client_test.go │ │ ├── config.go │ │ └── gcs.go │ ├── hdfsbackend │ │ ├── client.go │ │ ├── client_test.go │ │ ├── config.go │ │ └── webhdfs │ │ │ ├── client.go │ │ │ ├── client_test.go │ │ │ ├── config.go │ │ │ └── json.go │ ├── httpbackend │ │ ├── http.go │ │ └── http_test.go │ ├── manager.go │ ├── manager_test.go │ ├── namepath │ │ ├── pather.go │ │ └── pather_test.go │ ├── noop.go │ ├── options.go │ ├── registrybackend │ │ ├── blobclient.go │ │ ├── blobclient_test.go │ │ ├── config.go │ │ ├── config_test.go │ │ ├── security │ │ │ └── security.go │ │ ├── tagclient.go │ │ └── tagclient_test.go │ ├── results.go │ ├── s3backend │ │ ├── client.go │ │ ├── client_test.go │ │ ├── config.go │ │ └── s3.go │ ├── shadowbackend │ │ ├── README.md │ │ ├── client.go │ │ ├── client_test.go │ │ └── config.go │ ├── sqlbackend │ │ ├── README.md │ │ ├── benchmark │ │ │ └── client_test.go │ │ ├── client.go │ │ ├── client_test.go │ │ ├── config.go │ │ └── schema.go │ ├── testfs │ │ ├── client.go │ │ ├── client_test.go │ │ ├── config.go │ │ ├── server.go │ │ └── server_test.go │ └── throttle.go ├── blobrefresh │ ├── config.go │ ├── refresher.go │ └── refresher_test.go ├── containerruntime │ ├── .factory.go.swp │ ├── containerd │ │ ├── .config.go.swp │ │ ├── client.go │ │ └── config.go │ ├── dockerdaemon │ │ ├── .cli.go.swn │ │ ├── .cli.go.swo │ │ ├── cli.go │ │ └── config.go │ └── factory.go ├── dockerregistry │ ├── blobs.go │ ├── config.go │ ├── fixtures.go │ ├── manifests.go │ ├── metadata.go │ ├── metadata_test.go │ ├── paths.go │ ├── paths_test.go │ ├── storage_driver.go │ ├── storage_driver_test.go │ ├── testutils_test.go │ ├── transfer │ │ ├── errors.go │ │ ├── ro_transferer.go │ │ ├── ro_transferer_test.go │ │ ├── rw_transferer.go │ │ ├── rw_transferer_test.go │ │ ├── testing.go │ │ └── transferer.go │ └── uploads.go ├── hashring │ ├── config.go │ ├── passive_ring.go │ ├── passive_ring_test.go │ ├── ring.go │ ├── ring_test.go │ └── testing.go ├── healthcheck │ ├── checker.go │ ├── config.go │ ├── filter.go │ ├── filter_test.go │ ├── list.go │ ├── monitor.go │ ├── monitor_test.go │ ├── passive.go │ ├── passive_filter.go │ ├── passive_filter_test.go │ ├── passive_test.go │ ├── state.go │ ├── state_test.go │ └── testing.go ├── hostlist │ ├── config.go │ ├── fixtures.go │ ├── list.go │ └── list_test.go ├── hrw │ ├── fixtures.go │ ├── rendezvous.go │ ├── rendezvous_test.go │ └── testutils_test.go ├── metainfogen │ ├── config.go │ ├── config_test.go │ ├── fixtures.go │ ├── generator.go │ └── generator_test.go ├── middleware │ ├── middleware.go │ └── middleware_test.go ├── persistedretry │ ├── config.go │ ├── errors.go │ ├── interfaces.go │ ├── manager.go │ ├── manager_test.go │ ├── tagreplication │ │ ├── executor.go │ │ ├── executor_test.go │ │ ├── fixtures.go │ │ ├── remotes.go │ │ ├── remotes_test.go │ │ ├── store.go │ │ ├── store_test.go │ │ ├── task.go │ │ └── testing.go │ └── writeback │ │ ├── executor.go │ │ ├── executor_test.go │ │ ├── fixtures.go │ │ ├── query.go │ │ ├── store.go │ │ ├── store_test.go │ │ ├── task.go │ │ └── testing.go ├── store │ ├── base │ │ ├── const.go │ │ ├── errors.go │ │ ├── file_entry.go │ │ ├── file_entry_test.go │ │ ├── file_map.go │ │ ├── file_map_test.go │ │ ├── file_op.go │ │ ├── file_op_test.go │ │ ├── file_readwriter.go │ │ ├── file_store.go │ │ ├── fixtures.go │ │ └── testutils_test.go │ ├── ca_download_store.go │ ├── ca_download_store_test.go │ ├── ca_store.go │ ├── ca_store_test.go │ ├── cache_store.go │ ├── cleanup.go │ ├── cleanup_test.go │ ├── config.go │ ├── file.go │ ├── fixtures.go │ ├── metadata │ │ ├── last_access_time.go │ │ ├── last_access_time_test.go │ │ ├── metadata.go │ │ ├── persist.go │ │ ├── persist_test.go │ │ ├── torrentmeta.go │ │ └── torrentmeta_test.go │ ├── simple_store.go │ ├── simple_store_test.go │ ├── testing.go │ ├── upload_store.go │ └── utils.go ├── torrent │ ├── networkevent │ │ ├── config.go │ │ ├── events.go │ │ ├── producer.go │ │ ├── producer_test.go │ │ ├── testing.go │ │ └── util.go │ ├── scheduler │ │ ├── announcequeue │ │ │ ├── queue.go │ │ │ └── queue_test.go │ │ ├── announcer │ │ │ ├── announcer.go │ │ │ └── announcer_test.go │ │ ├── buckets.go │ │ ├── buckets_test.go │ │ ├── config.go │ │ ├── conn │ │ │ ├── config.go │ │ │ ├── conn.go │ │ │ ├── conn_test.go │ │ │ ├── fake_peer.go │ │ │ ├── fake_peer_test.go │ │ │ ├── fixtures.go │ │ │ ├── handshaker.go │ │ │ ├── handshaker_test.go │ │ │ └── message.go │ │ ├── connstate │ │ │ ├── config.go │ │ │ ├── state.go │ │ │ └── state_test.go │ │ ├── constructors.go │ │ ├── dispatch │ │ │ ├── config.go │ │ │ ├── dispatcher.go │ │ │ ├── dispatcher_test.go │ │ │ ├── peer.go │ │ │ ├── piecerequest │ │ │ │ ├── default_policy.go │ │ │ │ ├── manager.go │ │ │ │ ├── manager_test.go │ │ │ │ ├── policy.go │ │ │ │ └── rarest_first_policy.go │ │ │ ├── sync_bitfield.go │ │ │ ├── sync_bitfield_test.go │ │ │ └── torrent_access_watcher.go │ │ ├── events.go │ │ ├── events_test.go │ │ ├── reload.go │ │ ├── scheduler.go │ │ ├── scheduler_test.go │ │ ├── state.go │ │ ├── testutils_test.go │ │ └── torrentlog │ │ │ └── logger.go │ └── storage │ │ ├── agentstorage │ │ ├── fixtures.go │ │ ├── pieces.go │ │ ├── torrent.go │ │ ├── torrent_archive.go │ │ ├── torrent_archive_test.go │ │ └── torrent_test.go │ │ ├── fixtures.go │ │ ├── originstorage │ │ ├── torrent.go │ │ ├── torrent_archive.go │ │ ├── torrent_archive_test.go │ │ └── torrent_test.go │ │ ├── piecereader │ │ ├── buffer.go │ │ └── file.go │ │ ├── storage.go │ │ ├── torrent_info.go │ │ └── torrent_info_test.go └── upstream │ └── config.go ├── localdb ├── config.go ├── database.go ├── fixtures.go └── migrations │ ├── 00001_tagreplication_init.go │ └── 00002_writeback_init.go ├── metrics ├── config.go ├── disabled.go ├── m3.go ├── metrics.go └── statsd.go ├── mkdocs.yml ├── mocks ├── agent │ └── agentclient │ │ └── client.go ├── build-index │ ├── tagclient │ │ ├── client.go │ │ └── provider.go │ ├── tagstore │ │ ├── filestore.go │ │ └── store.go │ └── tagtype │ │ └── dependencyresolver.go ├── lib │ ├── backend │ │ ├── client.go │ │ ├── gcsbackend │ │ │ └── gcs.go │ │ ├── hdfsbackend │ │ │ └── webhdfs │ │ │ │ └── client.go │ │ └── s3backend │ │ │ └── s3.go │ ├── containerruntime │ │ ├── containerd │ │ │ └── client.go │ │ ├── dockerdaemon │ │ │ └── dockerclient.go │ │ └── factory.go │ ├── dockerregistry │ │ └── transfer │ │ │ └── imagetransferer.go │ ├── hashring │ │ ├── ring.go │ │ └── watcher.go │ ├── healthcheck │ │ ├── checker.go │ │ ├── filter.go │ │ └── passivefilter.go │ ├── hostlist │ │ └── list.go │ ├── persistedretry │ │ ├── executor.go │ │ ├── manager.go │ │ ├── store.go │ │ ├── tagreplication │ │ │ └── remotevalidator.go │ │ └── task.go │ ├── store │ │ └── filereadwriter.go │ └── torrent │ │ └── scheduler │ │ ├── reloadablescheduler.go │ │ └── scheduler.go ├── origin │ └── blobclient │ │ ├── client.go │ │ ├── clientresolver.go │ │ ├── clusterclient.go │ │ ├── clusterprovider.go │ │ └── provider.go ├── tracker │ ├── announceclient │ │ └── client.go │ ├── metainfoclient │ │ └── client.go │ ├── originstore │ │ └── store.go │ └── peerstore │ │ └── store.go └── utils │ ├── dedup │ ├── intervaltask.go │ └── taskrunner.go │ └── httputil │ └── roundtripper.go ├── nginx ├── config │ ├── agent.go │ ├── base.go │ ├── build-index.go │ ├── default.go │ ├── origin.go │ ├── proxy.go │ └── tracker.go └── nginx.go ├── origin ├── blobclient │ ├── client.go │ ├── cluster_client.go │ ├── errors.go │ ├── provider.go │ └── uploader.go ├── blobserver │ ├── cluster_client_test.go │ ├── config.go │ ├── server.go │ ├── server_test.go │ ├── testutils_test.go │ ├── uploader.go │ ├── utils.go │ └── utils_test.go ├── cmd │ ├── cmd.go │ └── config.go └── main.go ├── proto └── p2p │ └── p2p.proto ├── proxy ├── cmd │ ├── cmd.go │ └── config.go ├── main.go ├── proxyserver │ ├── config.go │ ├── prefetch.go │ ├── preheat.go │ ├── registry_events.go │ ├── server.go │ ├── server_test.go │ └── testutils_test.go └── registryoverride │ ├── config.go │ └── server.go ├── requirements-docs.txt ├── requirements-tests.txt ├── scripts └── mkdocs.sh ├── test ├── python │ ├── components.py │ ├── conftest.py │ ├── test_core.py │ ├── test_docker.py │ ├── test_replication.py │ ├── uploader.py │ └── utils.py └── tls │ ├── README.md │ ├── ca │ ├── passphrase │ ├── server.crt │ ├── server.csr │ ├── server.key │ └── server.srl │ └── client │ ├── client.crt │ ├── client.csr │ ├── client.key │ ├── client_decrypted.key │ └── passphrase ├── tools ├── bin │ ├── puller │ │ ├── main.go │ │ ├── notification.go │ │ └── pull.go │ ├── reload │ │ └── main.go │ ├── simulation │ │ ├── procedural_generated_graph.py │ │ └── random_regular_graph.py │ ├── testfs │ │ └── main.go │ └── visualization │ │ ├── main.go │ │ ├── server.go │ │ └── static │ │ ├── css │ │ └── app.css │ │ ├── html │ │ └── app.html │ │ └── js │ │ └── app.js └── lib │ ├── image │ └── image.go │ └── tlsutil.go ├── tracker ├── announceclient │ └── client.go ├── cmd │ ├── cmd.go │ └── config.go ├── main.go ├── metainfoclient │ ├── client.go │ └── testing.go ├── originstore │ ├── config.go │ ├── fixtures.go │ ├── store.go │ └── store_test.go ├── peerhandoutpolicy │ ├── completeness_policy.go │ ├── completeness_policy_test.go │ ├── config.go │ ├── default_policy.go │ ├── default_policy_test.go │ ├── fixtures.go │ ├── peerhandoutpolicy.go │ └── peerhandoutpolicy_test.go ├── peerstore │ ├── config.go │ ├── local.go │ ├── local_test.go │ ├── redis.go │ ├── redis_test.go │ ├── store.go │ └── testing.go └── trackerserver │ ├── announce.go │ ├── announce_test.go │ ├── config.go │ ├── fixtures.go │ ├── metainfo.go │ ├── metainfo_test.go │ ├── server.go │ └── testutils_test.go └── utils ├── bandwidth ├── limiter.go └── limiter_test.go ├── bitsetutil └── bitsetutil.go ├── configutil ├── config.go ├── config_test.go └── testdata │ ├── configFullD │ ├── base.yaml │ ├── production-zone2.yaml │ └── production.yaml │ ├── configFullE │ ├── base.yaml │ ├── production-zone2.yaml │ └── production.yaml │ ├── multiple │ ├── configA │ │ ├── base.yaml │ │ ├── production-zone2.yaml │ │ └── production.yaml │ ├── configB │ │ ├── base.yaml │ │ ├── production-zone2.yaml │ │ └── production.yaml │ ├── configC │ │ ├── base.yaml │ │ ├── production-zone2.yaml │ │ └── production.yaml │ ├── configF │ │ ├── base.yaml │ │ └── production-zone2.yaml │ └── configG │ │ ├── base.yaml │ │ ├── production-zone2.yaml │ │ └── production.yaml │ └── single │ ├── base.yaml │ ├── secrets.yaml │ └── test.yaml ├── dedup ├── interval_trap.go ├── interval_trap_test.go ├── limiter.go ├── limiter_test.go ├── request_cache.go └── request_cache_test.go ├── diskspaceutil ├── diskspaceutil.go └── diskspaceutil_test.go ├── dockerutil ├── dockerutil.go ├── dockerutil_test.go └── fixtures.go ├── errutil ├── errutil.go └── errutil_test.go ├── flagutil └── flagutil.go ├── handler └── handler.go ├── heap ├── priority_queue.go └── priority_queue_test.go ├── httputil ├── backoff.go ├── httputil.go ├── httputil_test.go ├── tls.go └── tls_test.go ├── listener ├── config.go └── listen.go ├── lockermap ├── map.go └── map_test.go ├── log ├── log.go └── logger.go ├── memsize ├── memsize.go └── memsize_test.go ├── mockutil ├── mockutil.go └── mockutil_test.go ├── netutil └── netutil.go ├── osutil └── osutil.go ├── randutil └── randutil.go ├── rwutil ├── cappedbuffer.go ├── cappedbuffer_test.go ├── rwutil.go └── rwutil_test.go ├── stringset └── stringset.go ├── syncutil ├── counters.go └── counters_test.go ├── testutil └── testutil.go └── timeutil ├── timer.go ├── timer_test.go ├── timeutil.go └── timeutil_test.go /.dockerignore: -------------------------------------------------------------------------------- 1 | docs 2 | env 3 | go-build 4 | tags 5 | vendor 6 | .git 7 | .tmp 8 | .tmptest 9 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 16 | 17 | **Expected behavior** 18 | A clear and concise description of what you expected to happen. 19 | 20 | **Screenshots** 21 | If applicable, add screenshots to help explain your problem. 22 | 23 | **Environments** 24 | e.g Kuberenetes, AWS, GCP, Data centers, laptop, etc. 25 | 26 | **Additional context** 27 | Add any other context about the problem here--environment configurations, logs, etc. 28 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/workflows/build-and-test.yaml: -------------------------------------------------------------------------------- 1 | name: Build and Test 2 | 3 | # Trigger the action on push and pull requests 4 | on: 5 | push: 6 | pull_request: 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | strategy: 12 | matrix: 13 | go-version: ['1.14'] 14 | steps: 15 | - uses: actions/checkout@v4 16 | - name: Set up Go ${{ matrix.go-version }} 17 | uses: actions/setup-go@v5 18 | with: 19 | go-version: ${{ matrix.go-version }} 20 | 21 | - name: Display Go version 22 | run: go version 23 | 24 | - name: Install dependencies 25 | run: go mod download 26 | 27 | # Add CGO_CFLAGS environment variable to suppress the warning 28 | - name: Build 29 | run: go build ./... 30 | env: 31 | CGO_CFLAGS: "-Wno-return-local-addr" 32 | 33 | # Run tests with CGO_CFLAGS set 34 | - name: Test with the Go CLI 35 | run: go test $(go list ./...) -cover -json | tee TestResults-${{ matrix.go-version }}.json 36 | env: 37 | CGO_CFLAGS: "-Wno-return-local-addr" 38 | 39 | - name: Upload Go test results 40 | uses: actions/upload-artifact@v4 41 | with: 42 | name: Go-results-${{ matrix.go-version }} 43 | path: TestResults-${{ matrix.go-version }}.json 44 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .cache 2 | .idea 3 | .tmp 4 | .vscode 5 | *.test 6 | *.go~ 7 | vendor/ 8 | 9 | # Fake source dir for jenkins and debian, which uses repo root as GOPATH 10 | ^bin/ 11 | pkg/ 12 | 13 | # Binary executable 14 | tracker/tracker 15 | agent/agent 16 | origin/origin 17 | proxy/proxy 18 | build-index/build-index 19 | tools/bin/puller/puller 20 | tools/bin/reload/reload 21 | tools/bin/reload-config/reload-config 22 | tools/bin/visualization/visualization 23 | tools/bin/testfs/testfs 24 | tools/bin/pexchange/pexchange 25 | tools/bin/trackerload/trackerload 26 | 27 | # tmp migration files 28 | [0-9]*_-dir.go 29 | 30 | *.xml 31 | *.pyc 32 | venv 33 | 34 | # Golang coverage files 35 | *.out 36 | # Golang test files 37 | *.test 38 | # Visual coverage report 39 | coverage.html 40 | # codecov coverage file 41 | coverage.txt 42 | 43 | # test db data 44 | db/data 45 | 46 | # Test configuratino generated from templates. 47 | config/*/test*.yaml 48 | 49 | # For vim users. 50 | tags 51 | 52 | # For engdoc. 53 | .DS_Store 54 | docs/_build/ 55 | env* 56 | *.pyc 57 | *.pickle 58 | 59 | # For integration tests. 60 | .tmptest/ 61 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | 3 | language: go 4 | 5 | go: 6 | - 1.14.x 7 | 8 | services: 9 | - docker 10 | 11 | env: 12 | - PATH=$HOME/protoc/bin:$PATH 13 | 14 | before_script: 15 | - "curl -H 'Cache-Control: no-cache' https://raw.githubusercontent.com/fossas/fossa-cli/master/install.sh | sudo bash" 16 | 17 | jobs: 18 | include: 19 | - stage: Tests 20 | name: Unit Tests 21 | script: 22 | - make unit-test 23 | - name: Integration Tests 24 | script: 25 | - make integration 26 | - stage: Compliance 27 | name: Fossa Check 28 | script: 29 | - "fossa --option allow-unresolved:true" 30 | 31 | after_success: 32 | - bash <(curl -s https://codecov.io/bash) 33 | -------------------------------------------------------------------------------- /agent/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package main 15 | 16 | import "github.com/uber/kraken/agent/cmd" 17 | 18 | func main() { 19 | cmd.Run(cmd.ParseFlags()) 20 | } 21 | -------------------------------------------------------------------------------- /assets/kraken-logo-color.ai: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber/kraken/604194a25ea5a4b79678ff5ba62515a0f7062e4c/assets/kraken-logo-color.ai -------------------------------------------------------------------------------- /assets/visualization.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber/kraken/604194a25ea5a4b79678ff5ba62515a0f7062e4c/assets/visualization.gif -------------------------------------------------------------------------------- /build-index/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package main 15 | 16 | import ( 17 | "github.com/uber/kraken/build-index/cmd" 18 | 19 | // Import all backend client packages to register them with backend manager. 20 | _ "github.com/uber/kraken/lib/backend/gcsbackend" 21 | _ "github.com/uber/kraken/lib/backend/hdfsbackend" 22 | _ "github.com/uber/kraken/lib/backend/httpbackend" 23 | _ "github.com/uber/kraken/lib/backend/registrybackend" 24 | _ "github.com/uber/kraken/lib/backend/s3backend" 25 | _ "github.com/uber/kraken/lib/backend/shadowbackend" 26 | _ "github.com/uber/kraken/lib/backend/sqlbackend" 27 | _ "github.com/uber/kraken/lib/backend/testfs" 28 | ) 29 | 30 | func main() { 31 | cmd.Run(cmd.ParseFlags()) 32 | } 33 | -------------------------------------------------------------------------------- /build-index/tagclient/provider.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package tagclient 15 | 16 | import ( 17 | "crypto/tls" 18 | ) 19 | 20 | // Provider maps addresses into Clients. 21 | type Provider interface { 22 | Provide(addr string) Client 23 | } 24 | 25 | type provider struct{ tls *tls.Config } 26 | 27 | // NewProvider creates a Provider which wraps NewSingleClient. 28 | func NewProvider(config *tls.Config) Provider { return provider{config} } 29 | 30 | func (p provider) Provide(addr string) Client { 31 | return NewSingleClient(addr, p.tls) 32 | } 33 | -------------------------------------------------------------------------------- /build-index/tagclient/testing.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package tagclient 15 | 16 | import "log" 17 | 18 | // TestProvider is a testing utility for mapping addresses to mock clients. 19 | type TestProvider struct { 20 | clients map[string]Client 21 | } 22 | 23 | // NewTestProvider creates a new TestProvider. 24 | func NewTestProvider() *TestProvider { 25 | return &TestProvider{make(map[string]Client)} 26 | } 27 | 28 | // Register sets c as the client of addr. 29 | func (p *TestProvider) Register(addr string, c Client) { 30 | p.clients[addr] = c 31 | } 32 | 33 | // Provide selects the registered client of addr. 34 | func (p *TestProvider) Provide(addr string) Client { 35 | c, ok := p.clients[addr] 36 | if !ok { 37 | log.Panicf("addr %s not found", addr) 38 | } 39 | return c 40 | } 41 | -------------------------------------------------------------------------------- /build-index/tagserver/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package tagserver 15 | 16 | import ( 17 | "time" 18 | 19 | "github.com/uber/kraken/utils/listener" 20 | ) 21 | 22 | // Config defines Server configuration. 23 | type Config struct { 24 | Listener listener.Config `yaml:"listener"` 25 | DuplicateReplicateStagger time.Duration `yaml:"duplicate_replicate_stagger"` 26 | DuplicatePutStagger time.Duration `yaml:"duplicate_put_stagger"` 27 | } 28 | 29 | func (c Config) applyDefaults() Config { 30 | if c.DuplicateReplicateStagger == 0 { 31 | c.DuplicateReplicateStagger = 20 * time.Minute 32 | } 33 | if c.DuplicatePutStagger == 0 { 34 | c.DuplicatePutStagger = 20 * time.Minute 35 | } 36 | return c 37 | } 38 | -------------------------------------------------------------------------------- /build-index/tagstore/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package tagstore 15 | 16 | // Config defines tag store configuration. 17 | type Config struct { 18 | WriteThrough bool `yaml:"write_through"` 19 | } 20 | -------------------------------------------------------------------------------- /build-index/tagtype/default_resolver.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package tagtype 15 | 16 | import "github.com/uber/kraken/core" 17 | 18 | type defaultResolver struct{} 19 | 20 | // Resolve always returns d as the sole dependency of tag. 21 | func (r *defaultResolver) Resolve(tag string, d core.Digest) (core.DigestList, error) { 22 | return core.DigestList{d}, nil 23 | } 24 | -------------------------------------------------------------------------------- /config/agent/base.yaml: -------------------------------------------------------------------------------- 1 | zap: 2 | level: info 3 | development: false 4 | encoding: console 5 | disableStacktrace: true 6 | encoderConfig: 7 | messageKey: message 8 | nameKey: logger_name 9 | levelKey: level 10 | timeKey: ts 11 | callerKey: caller 12 | stacktraceKey: stack 13 | levelEncoder: capital 14 | timeEncoder: iso8601 15 | durationEncoder: seconds 16 | callerEncoder: short 17 | outputPaths: 18 | - stdout 19 | - /var/log/kraken/kraken-agent/stdout.log 20 | errorOutputPaths: 21 | - stdout 22 | - /var/log/kraken/kraken-agent/stdout.log 23 | 24 | metrics: 25 | m3: 26 | service: kraken-agent 27 | 28 | scheduler: 29 | log: 30 | timeEncoder: iso8601 31 | torrentlog: 32 | service_name: kraken-agent 33 | path: /var/log/kraken/kraken-agent/torrent.log 34 | encoding: json 35 | timeEncoder: epoch 36 | dispatch: 37 | piece_request_policy: rarest_first 38 | conn: 39 | bandwidth: 40 | enable: true 41 | 42 | store: 43 | download_dir: /var/cache/kraken/kraken-agent/download/ 44 | cache_dir: /var/cache/kraken/kraken-agent/cache/ 45 | download_cleanup: 46 | ttl: 24h 47 | cache_cleanup: 48 | ttl: 24h 49 | 50 | registry: 51 | docker: 52 | version: 0.1 53 | log: 54 | level: error 55 | http: 56 | net: unix 57 | addr: /tmp/kraken-agent-registry.sock 58 | 59 | peer_id_factory: addr_hash 60 | 61 | # Allow agent to only serve localhost and Docker default bridge requests. 62 | allowed_cidrs: 63 | - 127.0.0.1 64 | - 172.17.0.1 65 | 66 | nginx: 67 | name: kraken-agent 68 | cache_dir: /var/cache/kraken/kraken-agent/nginx/ 69 | log_dir: /var/log/kraken/kraken-agent/ 70 | -------------------------------------------------------------------------------- /config/agent/test.template: -------------------------------------------------------------------------------- 1 | extends: base.yaml 2 | 3 | scheduler: 4 | connstate: 5 | blacklist_duration: 5s 6 | 7 | peer_id_factory: random 8 | 9 | tracker: 10 | hosts: 11 | static: {trackers} 12 | 13 | build_index: 14 | hosts: 15 | static: {build_indexes} 16 | 17 | tls: 18 | name: kraken 19 | cas: 20 | - path: /etc/kraken/tls/ca/server.crt 21 | server: 22 | disabled: true 23 | client: 24 | cert: 25 | path: /etc/kraken/tls/client/client.crt 26 | key: 27 | path: /etc/kraken/tls/client/client.key 28 | passphrase: 29 | path: /etc/kraken/tls/client/passphrase 30 | -------------------------------------------------------------------------------- /config/build-index/base.yaml: -------------------------------------------------------------------------------- 1 | localdb: 2 | source: /var/cache/kraken/kraken-build-index/index.db 3 | 4 | store: 5 | upload_dir: /var/cache/kraken/kraken-build-index/upload/ 6 | cache_dir: /var/cache/kraken/kraken-build-index/cache/ 7 | 8 | tag_store: 9 | write_through: true 10 | 11 | zap: 12 | level: info 13 | development: false 14 | encoding: console 15 | disableStacktrace: true 16 | encoderConfig: 17 | messageKey: message 18 | nameKey: logger_name 19 | levelKey: level 20 | timeKey: ts 21 | callerKey: caller 22 | stacktraceKey: stack 23 | levelEncoder: capital 24 | timeEncoder: iso8601 25 | durationEncoder: seconds 26 | callerEncoder: short 27 | outputPaths: 28 | - stdout 29 | - /var/log/kraken/kraken-build-index/stdout.log 30 | errorOutputPaths: 31 | - stdout 32 | - /var/log/kraken/kraken-build-index/stdout.log 33 | 34 | metrics: 35 | m3: 36 | service: kraken-build-index 37 | 38 | tagserver: 39 | listener: 40 | net: unix 41 | addr: /tmp/kraken-build-index.sock 42 | 43 | nginx: 44 | name: kraken-build-index 45 | cache_dir: /var/cache/kraken/kraken-build-index/nginx/ 46 | log_dir: /var/log/kraken/kraken-build-index/ 47 | -------------------------------------------------------------------------------- /config/build-index/test.template: -------------------------------------------------------------------------------- 1 | extends: base.yaml 2 | 3 | cluster: 4 | hosts: 5 | static: {cluster} 6 | 7 | backends: 8 | - namespace: .* 9 | backend: 10 | testfs: 11 | addr: {testfs} 12 | root: tags 13 | name_path: docker_tag 14 | 15 | origin: 16 | hosts: 17 | static: {origins} 18 | 19 | {remotes} 20 | 21 | tag_replication: 22 | retry_interval: 100ms 23 | poll_retries_interval: 250ms 24 | 25 | tagserver: 26 | cache: 27 | ttl: 10ms 28 | error_ttl: 10ms 29 | cleanup_interval: 5ms 30 | duplicate_replicate_stagger: 100ms 31 | 32 | tag_types: 33 | - namespace: .* 34 | type: docker 35 | 36 | tag_store: 37 | write_through: false 38 | 39 | writeback: 40 | retry_interval: 100ms 41 | poll_retries_interval: 250ms 42 | 43 | nginx: 44 | cache_dir: /tmp/kraken-build-index-nginx/ 45 | 46 | tls: 47 | name: kraken 48 | cas: 49 | - path: /etc/kraken/tls/ca/server.crt 50 | server: 51 | cert: 52 | path: /etc/kraken/tls/ca/server.crt 53 | key: 54 | path: /etc/kraken/tls/ca/server.key 55 | passphrase: 56 | path: /etc/kraken/tls/ca/passphrase 57 | client: 58 | cert: 59 | path: /etc/kraken/tls/client/client.crt 60 | key: 61 | path: /etc/kraken/tls/client/client.key 62 | passphrase: 63 | path: /etc/kraken/tls/client/passphrase 64 | -------------------------------------------------------------------------------- /config/origin/base.yaml: -------------------------------------------------------------------------------- 1 | zap: 2 | level: info 3 | development: false 4 | encoding: console 5 | disableStacktrace: true 6 | encoderConfig: 7 | messageKey: message 8 | nameKey: logger_name 9 | levelKey: level 10 | timeKey: ts 11 | callerKey: caller 12 | stacktraceKey: stack 13 | levelEncoder: capital 14 | timeEncoder: iso8601 15 | durationEncoder: seconds 16 | callerEncoder: short 17 | outputPaths: 18 | - stdout 19 | - /var/log/kraken/kraken-origin/stdout.log 20 | errorOutputPaths: 21 | - stdout 22 | - /var/log/kraken/kraken-origin/stdout.log 23 | 24 | metainfogen: 25 | piece_lengths: 26 | 0: 4MB # Use 4MB piece lengths for all file sizes (for now). 27 | 28 | peer_id_factory: addr_hash 29 | 30 | scheduler: 31 | log: 32 | timeEncoder: iso8601 33 | torrentlog: 34 | disable: true 35 | 36 | metrics: 37 | m3: 38 | service: kraken-origin 39 | 40 | localdb: 41 | source: /var/cache/kraken/kraken-origin/origin.db 42 | 43 | castore: 44 | upload_dir: /var/cache/kraken/kraken-origin/upload/ 45 | cache_dir: /var/cache/kraken/kraken-origin/cache/ 46 | 47 | blobserver: 48 | listener: 49 | net: unix 50 | addr: /tmp/kraken-origin.sock 51 | 52 | nginx: 53 | name: kraken-origin 54 | cache_dir: /var/cache/kraken/kraken-origin/nginx/ 55 | log_dir: /var/log/kraken/kraken-origin/ 56 | -------------------------------------------------------------------------------- /config/origin/test.template: -------------------------------------------------------------------------------- 1 | extends: base.yaml 2 | 3 | hashring: 4 | max_replica: 2 5 | 6 | cluster: 7 | static: {origins} 8 | 9 | metainfogen: 10 | piece_lengths: 11 | 0: 1MB 12 | 13 | # Need stable peer ids for the origin so we can safely restart them. 14 | peer_id_factory: addr_hash 15 | 16 | backends: 17 | - namespace: .* 18 | backend: 19 | testfs: 20 | addr: {testfs} 21 | root: blobs 22 | name_path: identity 23 | 24 | writeback: 25 | retry_interval: 100ms 26 | poll_retries_interval: 250ms 27 | 28 | tls: 29 | name: kraken 30 | cas: 31 | - path: /etc/kraken/tls/ca/server.crt 32 | server: 33 | cert: 34 | path: /etc/kraken/tls/ca/server.crt 35 | key: 36 | path: /etc/kraken/tls/ca/server.key 37 | passphrase: 38 | path: /etc/kraken/tls/ca/passphrase 39 | client: 40 | cert: 41 | path: /etc/kraken/tls/client/client.crt 42 | key: 43 | path: /etc/kraken/tls/client/client.key 44 | passphrase: 45 | path: /etc/kraken/tls/client/passphrase 46 | -------------------------------------------------------------------------------- /config/proxy/base.yaml: -------------------------------------------------------------------------------- 1 | castore: 2 | upload_dir: /var/cache/kraken/kraken-proxy/upload/ 3 | cache_dir: /var/cache/kraken/kraken-proxy/cache/ 4 | capacity: 1024 5 | 6 | zap: 7 | level: info 8 | development: false 9 | encoding: console 10 | disableStacktrace: true 11 | encoderConfig: 12 | messageKey: message 13 | nameKey: logger_name 14 | levelKey: level 15 | timeKey: ts 16 | callerKey: caller 17 | stacktraceKey: stack 18 | levelEncoder: capital 19 | timeEncoder: iso8601 20 | durationEncoder: seconds 21 | callerEncoder: short 22 | outputPaths: 23 | - stdout 24 | - /var/log/kraken/kraken-proxy/stdout.log 25 | errorOutputPaths: 26 | - stdout 27 | - /var/log/kraken/kraken-proxy/stdout.log 28 | 29 | metrics: 30 | m3: 31 | service: kraken-proxy 32 | 33 | registry: 34 | docker: 35 | version: 0.1 36 | log: 37 | level: error 38 | http: 39 | net: unix 40 | addr: /tmp/kraken-proxy-registry.sock 41 | 42 | registryoverride: 43 | listener: 44 | net: unix 45 | addr: /tmp/kraken-proxy-registry-override.sock 46 | 47 | nginx: 48 | name: kraken-proxy 49 | cache_dir: /var/cache/kraken/kraken-proxy/nginx/ 50 | log_dir: /var/log/kraken/kraken-proxy/ 51 | -------------------------------------------------------------------------------- /config/proxy/test.template: -------------------------------------------------------------------------------- 1 | extends: base.yaml 2 | 3 | origin: 4 | hosts: 5 | static: {origins} 6 | 7 | build_index: 8 | hosts: 9 | static: {build_indexes} 10 | 11 | nginx: 12 | cache_dir: /tmp/kraken-proxy-nginx/ 13 | 14 | tls: 15 | name: kraken 16 | cas: 17 | - path: /etc/kraken/tls/ca/server.crt 18 | server: 19 | disabled: true 20 | cert: 21 | path: /etc/kraken/tls/ca/server.crt 22 | key: 23 | path: /etc/kraken/tls/ca/server.key 24 | passphrase: 25 | path: /etc/kraken/tls/ca/passphrase 26 | client: 27 | cert: 28 | path: /etc/kraken/tls/client/client.crt 29 | key: 30 | path: /etc/kraken/tls/client/client.key 31 | passphrase: 32 | path: /etc/kraken/tls/client/passphrase 33 | -------------------------------------------------------------------------------- /config/tracker/base.yaml: -------------------------------------------------------------------------------- 1 | zap: 2 | level: info 3 | development: false 4 | disableStacktrace: true 5 | encoding: console 6 | encoderConfig: 7 | messageKey: message 8 | nameKey: logger_name 9 | levelKey: level 10 | timeKey: ts 11 | callerKey: caller 12 | stacktraceKey: stack 13 | levelEncoder: capital 14 | timeEncoder: iso8601 15 | durationEncoder: seconds 16 | callerEncoder: short 17 | outputPaths: 18 | - stdout 19 | - /var/log/kraken/kraken-tracker/stdout.log 20 | errorOutputPaths: 21 | - stdout 22 | - /var/log/kraken/kraken-tracker/stdout.log 23 | 24 | logging: 25 | level: info 26 | stdout: true 27 | sentry: 28 | enabled: false 29 | tags: 30 | deployment: default 31 | 32 | peerhandoutpolicy: 33 | priority: completeness 34 | 35 | metrics: 36 | m3: 37 | service: kraken 38 | 39 | trackerserver: 40 | listener: 41 | net: unix 42 | addr: /tmp/kraken-tracker.sock 43 | 44 | nginx: 45 | name: kraken-tracker 46 | cache_dir: /var/cache/kraken/kraken-tracker/nginx/ 47 | log_dir: /var/log/kraken/kraken-tracker/ 48 | -------------------------------------------------------------------------------- /config/tracker/test.template: -------------------------------------------------------------------------------- 1 | extends: base.yaml 2 | 3 | origin: 4 | hosts: 5 | static: {origins} 6 | 7 | trackerserver: 8 | announce_interval: 2s 9 | 10 | nginx: 11 | cache_dir: /tmp/kraken-tracker-nginx/ 12 | 13 | tls: 14 | name: kraken 15 | cas: 16 | - path: /etc/kraken/tls/ca/server.crt 17 | server: 18 | cert: 19 | path: /etc/kraken/tls/ca/server.crt 20 | key: 21 | path: /etc/kraken/tls/ca/server.key 22 | passphrase: 23 | path: /etc/kraken/tls/ca/passphrase 24 | client: 25 | cert: 26 | path: /etc/kraken/tls/client/client.crt 27 | key: 28 | path: /etc/kraken/tls/client/client.key 29 | passphrase: 30 | path: /etc/kraken/tls/client/passphrase 31 | -------------------------------------------------------------------------------- /core/blobinfo.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package core 15 | 16 | // BlobInfo contains metadata about a blob. 17 | type BlobInfo struct { 18 | Size int64 19 | } 20 | 21 | // NewBlobInfo creates a new BlobInfo. 22 | func NewBlobInfo(size int64) *BlobInfo { 23 | return &BlobInfo{size} 24 | } 25 | -------------------------------------------------------------------------------- /core/infohash_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package core 15 | 16 | import ( 17 | "testing" 18 | 19 | "github.com/stretchr/testify/require" 20 | ) 21 | 22 | func TestNewInfoHashFromHex(t *testing.T) { 23 | require := require.New(t) 24 | 25 | d, err := NewInfoHashFromHex("e3b0c44298fc1c149afbf4c8996fb92427ae41e4") 26 | require.NoError(err) 27 | require.Equal("e3b0c44298fc1c149afbf4c8996fb92427ae41e4", d.Hex()) 28 | require.Equal("e3b0c44298fc1c149afbf4c8996fb92427ae41e4", d.String()) 29 | } 30 | 31 | func TestNewInfoHashFromHexErrors(t *testing.T) { 32 | tests := []struct { 33 | desc string 34 | input string 35 | }{ 36 | {"empty", ""}, 37 | {"too long", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, 38 | {"invalid hex", "x3b0c44298fc1c149afbf4c8996fb92427ae41e4"}, 39 | } 40 | for _, test := range tests { 41 | t.Run(test.desc, func(t *testing.T) { 42 | _, err := NewInfoHashFromHex(test.input) 43 | require.Error(t, err) 44 | }) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /core/peer_info_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package core 15 | 16 | import ( 17 | "testing" 18 | 19 | "github.com/stretchr/testify/require" 20 | ) 21 | 22 | func TestSortedByPeerID(t *testing.T) { 23 | require := require.New(t) 24 | 25 | p1 := PeerInfoFixture() 26 | p2 := PeerInfoFixture() 27 | p3 := PeerInfoFixture() 28 | 29 | sorted := SortedByPeerID([]*PeerInfo{p1, p2, p3}) 30 | require.True(sorted[0].PeerID.LessThan(sorted[1].PeerID)) 31 | require.True(sorted[1].PeerID.LessThan(sorted[2].PeerID)) 32 | } 33 | -------------------------------------------------------------------------------- /core/piece_hash.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package core 15 | 16 | import ( 17 | "hash" 18 | "hash/crc32" 19 | ) 20 | 21 | // PieceHash returns the hash used to sum pieces. 22 | func PieceHash() hash.Hash32 { 23 | return crc32.NewIEEE() 24 | } 25 | -------------------------------------------------------------------------------- /docker/agent/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:10 2 | 3 | RUN apt-get update && apt-get install -y curl nginx 4 | 5 | RUN mkdir -p -m 777 /var/log/kraken/kraken-agent 6 | RUN mkdir -p -m 777 /var/cache/kraken/kraken-agent 7 | RUN mkdir -p -m 777 /var/run/kraken 8 | 9 | ARG USERNAME="root" 10 | ARG USERID="0" 11 | RUN if [ ${USERID} != "0" ]; then useradd --uid ${USERID} ${USERNAME}; fi 12 | 13 | COPY ./docker/setup_nginx.sh /tmp/setup_nginx.sh 14 | RUN /tmp/setup_nginx.sh ${USERNAME} 15 | 16 | USER ${USERNAME} 17 | 18 | COPY ./agent/agent /usr/bin/kraken-agent 19 | COPY ./config /etc/kraken/config 20 | COPY ./nginx/config /etc/kraken/nginx/config 21 | COPY ./test/tls /etc/kraken/tls 22 | 23 | WORKDIR /etc/kraken 24 | -------------------------------------------------------------------------------- /docker/build-index/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:10 2 | 3 | RUN apt-get update && apt-get install -y curl nginx 4 | 5 | RUN mkdir -p -m 777 /var/log/kraken/kraken-build-index 6 | RUN mkdir -p -m 777 /var/cache/kraken/kraken-build-index 7 | RUN mkdir -p -m 777 /var/run/kraken 8 | 9 | ARG USERNAME="root" 10 | ARG USERID="0" 11 | RUN if [ ${USERID} != "0" ]; then useradd --uid ${USERID} ${USERNAME}; fi 12 | 13 | COPY ./docker/setup_nginx.sh /tmp/setup_nginx.sh 14 | RUN /tmp/setup_nginx.sh ${USERNAME} 15 | 16 | USER ${USERNAME} 17 | 18 | COPY ./build-index/build-index /usr/bin/kraken-build-index 19 | COPY ./config /etc/kraken/config 20 | COPY ./nginx/config /etc/kraken-build-index/nginx/config 21 | COPY ./localdb/migrations /etc/kraken-build-index/localdb/migrations 22 | COPY ./test/tls /etc/kraken/tls 23 | 24 | WORKDIR /etc/kraken-build-index 25 | -------------------------------------------------------------------------------- /docker/origin/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:10 2 | 3 | RUN apt-get update && apt-get install -y curl sqlite3 nginx 4 | 5 | RUN mkdir -p -m 777 /var/log/kraken/kraken-origin 6 | RUN mkdir -p -m 777 /var/cache/kraken/kraken-origin 7 | RUN mkdir -p -m 777 /var/run/kraken 8 | 9 | ARG USERNAME="root" 10 | ARG USERID="0" 11 | RUN if [ ${USERID} != "0" ]; then useradd --uid ${USERID} ${USERNAME}; fi 12 | 13 | COPY ./docker/setup_nginx.sh /tmp/setup_nginx.sh 14 | RUN /tmp/setup_nginx.sh ${USERNAME} 15 | 16 | USER ${USERNAME} 17 | 18 | COPY ./origin/origin /usr/bin/kraken-origin 19 | COPY ./config /etc/kraken/config 20 | COPY ./nginx/config /etc/kraken/nginx/config 21 | COPY ./localdb/migrations /etc/kraken/localdb/migrations 22 | COPY ./test/tls /etc/kraken/tls 23 | 24 | WORKDIR /etc/kraken 25 | -------------------------------------------------------------------------------- /docker/proxy/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:10 2 | 3 | RUN apt-get update && apt-get install -y curl nginx 4 | 5 | RUN mkdir -p -m 777 /var/log/kraken/kraken-proxy 6 | RUN mkdir -p -m 777 /var/cache/kraken/kraken-proxy 7 | RUN mkdir -p -m 777 /var/run/kraken 8 | 9 | ARG USERNAME="root" 10 | ARG USERID="0" 11 | RUN if [ ${USERID} != "0" ]; then useradd --uid ${USERID} ${USERNAME}; fi 12 | 13 | COPY ./docker/setup_nginx.sh /tmp/setup_nginx.sh 14 | RUN /tmp/setup_nginx.sh ${USERNAME} 15 | 16 | USER ${USERNAME} 17 | 18 | COPY ./proxy/proxy /usr/bin/kraken-proxy 19 | COPY ./config /etc/kraken/config 20 | COPY ./nginx/config /etc/kraken/nginx/config 21 | COPY ./test/tls /etc/kraken/tls 22 | 23 | WORKDIR /etc/kraken 24 | -------------------------------------------------------------------------------- /docker/setup_nginx.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | for d in /tmp/nginx/ /var/lib/nginx/ /var/log/nginx/ /var/run/nginx/; do 6 | mkdir -p $d 7 | chown -R $1:$1 $d 8 | chmod 0755 $d 9 | done 10 | -------------------------------------------------------------------------------- /docker/testfs/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:10 2 | 3 | RUN apt-get update && apt-get install -y curl 4 | 5 | RUN mkdir -p -m 777 /var/log/kraken/kraken-testfs 6 | RUN mkdir -p -m 777 /var/cache/kraken/kraken-testfs 7 | 8 | ARG USERNAME="root" 9 | ARG USERID="0" 10 | RUN if [ ${USERID} != "0" ]; then useradd --uid ${USERID} ${USERNAME}; fi 11 | USER ${USERNAME} 12 | 13 | COPY tools/bin/testfs/testfs /usr/bin/kraken-testfs 14 | 15 | WORKDIR /etc/kraken 16 | -------------------------------------------------------------------------------- /docker/tracker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:10 2 | 3 | RUN apt-get update && apt-get install -y curl nginx 4 | 5 | RUN mkdir -p -m 777 /var/log/kraken/kraken-tracker 6 | RUN mkdir -p -m 777 /var/cache/kraken/kraken-tracker 7 | RUN mkdir -p -m 777 /var/run/kraken 8 | 9 | ARG USERNAME="root" 10 | ARG USERID="0" 11 | RUN if [ ${USERID} != "0" ]; then useradd --uid ${USERID} ${USERNAME}; fi 12 | 13 | COPY ./docker/setup_nginx.sh /tmp/setup_nginx.sh 14 | RUN /tmp/setup_nginx.sh ${USERNAME} 15 | 16 | USER ${USERNAME} 17 | 18 | COPY ./tracker/tracker /usr/bin/kraken-tracker 19 | COPY ./config /etc/kraken/config 20 | COPY ./nginx/config /etc/kraken/nginx/config 21 | COPY ./test/tls /etc/kraken/tls 22 | 23 | WORKDIR /etc/kraken 24 | -------------------------------------------------------------------------------- /docs/ARCHITECTURE.md: -------------------------------------------------------------------------------- 1 | # Kraken Core 2 | 3 | Central P2P components that's not specific to docker images: 4 | 5 | ![](../assets/kraken_core.svg) 6 | 7 | ## Agent 8 | - Deployed on every host 9 | - Implements Docker registry interface 10 | 11 | ## Origin 12 | - Dedicated seeders 13 | - Pluggable storage backend (e.g. S3) 14 | - Self-healing hash ring 15 | 16 | ## Tracker 17 | - Tracks peers and seeders, instructs them to form a sparse graph 18 | - Self-healing hash ring 19 | 20 | # Kraken Proxy and Build Index 21 | 22 | Components responsible for image tags and replication to other clusters: 23 | 24 | ![](../assets/kraken_build_index.svg) 25 | 26 | ## Proxy 27 | - Handles image upload and direct download 28 | 29 | ## Build Index 30 | - Mapping of human readable tag to blob hash (digest) 31 | - No consistency guarantees, client should use unique tags 32 | - Powers image replication between clusters. Simple duplicated queues with retry 33 | - Pluggable storage 34 | - Self-healing hash ring 35 | -------------------------------------------------------------------------------- /docs/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing To Kraken 2 | 3 | ## Issues 4 | 5 | Please feel free to submit new issues. 6 | 7 | ## Contributing 8 | 9 | Start by reading our [coding style guidelines](STYLEGUIDE.md). 10 | 11 | Please follow standard fork-and-pull workflow. 12 | 13 | - Fork the repo on GitHub 14 | - Clone the project locally 15 | - Commit changes to your own branch 16 | - Push the change back to your fork 17 | - Submit a Pull request. We will review and merge your change. 18 | 19 | ## Setup 20 | 21 | Most tests and scripts assumes the developer to have Docker installed locally. 22 | To install dependencies: 23 | ``` 24 | $ make vendor 25 | ``` 26 | To run unit tests: 27 | ``` 28 | $ make unit-test 29 | ``` 30 | To run integration tests: 31 | ``` 32 | $ make integration 33 | ``` 34 | To build docker images: 35 | ``` 36 | $ make images 37 | ``` 38 | -------------------------------------------------------------------------------- /docs/assets/kraken-logo-color.ai: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber/kraken/604194a25ea5a4b79678ff5ba62515a0f7062e4c/docs/assets/kraken-logo-color.ai -------------------------------------------------------------------------------- /docs/assets/visualization.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber/kraken/604194a25ea5a4b79678ff5ba62515a0f7062e4c/docs/assets/visualization.gif -------------------------------------------------------------------------------- /examples/devcluster/README.md: -------------------------------------------------------------------------------- 1 | ## 1. Build 2 | 3 | `$ make devcluster` 4 | 5 | This command creates a `kraken-agent` image containing an agent binary and a `kraken-herd` image containing build-index, origin, proxy, and tracker binaries. 6 | 7 | It starts 2 agent docker containers and 1 herd container. Docker-for-Mac is required for devcluster to work, because the development config files use host.docker.internal for address of all components. 8 | 9 | ## 2. Pulling from Docker Hub Library 10 | 11 | A simple registry storage backend is provided for read-only access to Docker registry. A library image can be pulled from agent. 12 | 13 | `$ docker pull localhost:16000/library/golang:1.14` 14 | 15 | Note, this backend is used for all `library/.*` repositories. `library` is the default namespace for Docker Hub's standard public repositories. 16 | 17 | For all the other repositories, a testfs storage backend is included in the `kraken-herd` image, which is a simple http server that supports file uploading and downloading via port `14000`. Testfs simply stores blobs and tags on filesystem. 18 | 19 | ## 3. Pushing a Test Image 20 | 21 | A test image can be pushed to the herd instance 22 | 23 | `$ docker push localhost:15000/:` 24 | 25 | ## 4. Pull the Test Image 26 | 27 | Pull the test image from agent: 28 | 29 | `$ docker pull localhost:16000/:` 30 | -------------------------------------------------------------------------------- /examples/devcluster/agent_one_param.sh: -------------------------------------------------------------------------------- 1 | # Define agent ports. 2 | AGENT_REGISTRY_PORT=16000 3 | AGENT_PEER_PORT=16001 4 | AGENT_SERVER_PORT=16002 5 | 6 | # Hostname for docker for mac. 7 | HOSTNAME=host.docker.internal 8 | 9 | # Container config. 10 | AGENT_CONTAINER_NAME=kraken-agent-one 11 | -------------------------------------------------------------------------------- /examples/devcluster/agent_one_start_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | source examples/devcluster/agent_one_param.sh 6 | 7 | # Start kraken agent. 8 | docker run -d \ 9 | -p ${AGENT_PEER_PORT}:${AGENT_PEER_PORT} \ 10 | -p ${AGENT_SERVER_PORT}:${AGENT_SERVER_PORT} \ 11 | -p ${AGENT_REGISTRY_PORT}:${AGENT_REGISTRY_PORT} \ 12 | -v $(pwd)/examples/devcluster/config/agent/development.yaml:/etc/kraken/config/agent/development.yaml \ 13 | --name ${AGENT_CONTAINER_NAME} \ 14 | kraken-agent:dev \ 15 | /usr/bin/kraken-agent --config=/etc/kraken/config/agent/development.yaml --peer-ip=${HOSTNAME} --peer-port=${AGENT_PEER_PORT} --agent-server-port=${AGENT_SERVER_PORT} --agent-registry-port=${AGENT_REGISTRY_PORT} 16 | -------------------------------------------------------------------------------- /examples/devcluster/agent_two_param.sh: -------------------------------------------------------------------------------- 1 | # Define agent ports. 2 | AGENT_REGISTRY_PORT=17000 3 | AGENT_PEER_PORT=17001 4 | AGENT_SERVER_PORT=17002 5 | 6 | # Hostname for docker for mac. 7 | HOSTNAME=host.docker.internal 8 | 9 | # Container config. 10 | AGENT_CONTAINER_NAME=kraken-agent-two 11 | -------------------------------------------------------------------------------- /examples/devcluster/agent_two_start_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | source examples/devcluster/agent_two_param.sh 6 | 7 | # Start kraken agent. 8 | docker run -d \ 9 | -p ${AGENT_PEER_PORT}:${AGENT_PEER_PORT} \ 10 | -p ${AGENT_SERVER_PORT}:${AGENT_SERVER_PORT} \ 11 | -p ${AGENT_REGISTRY_PORT}:${AGENT_REGISTRY_PORT} \ 12 | -v $(pwd)/examples/devcluster/config/agent/development.yaml:/etc/kraken/config/agent/development.yaml \ 13 | --name ${AGENT_CONTAINER_NAME} \ 14 | kraken-agent:dev \ 15 | /usr/bin/kraken-agent --config=/etc/kraken/config/agent/development.yaml --peer-ip=${HOSTNAME} --peer-port=${AGENT_PEER_PORT} --agent-server-port=${AGENT_SERVER_PORT} --agent-registry-port=${AGENT_REGISTRY_PORT} 16 | -------------------------------------------------------------------------------- /examples/devcluster/config/agent/development.yaml: -------------------------------------------------------------------------------- 1 | extends: base.yaml 2 | 3 | tracker: 4 | hosts: 5 | static: 6 | - host.docker.internal:15003 7 | 8 | build_index: 9 | hosts: 10 | static: 11 | - host.docker.internal:15004 12 | 13 | tls: 14 | name: kraken 15 | cas: 16 | - path: /etc/kraken/tls/ca/server.crt 17 | server: 18 | disabled: true 19 | cert: 20 | path: /etc/kraken/tls/ca/server.crt 21 | key: 22 | path: /etc/kraken/tls/ca/server.key 23 | passphrase: 24 | path: /etc/kraken/tls/ca/passphrase 25 | client: 26 | cert: 27 | path: /etc/kraken/tls/client/client.crt 28 | key: 29 | path: /etc/kraken/tls/client/client.key 30 | passphrase: 31 | path: /etc/kraken/tls/client/passphrase 32 | -------------------------------------------------------------------------------- /examples/devcluster/config/build-index/development.yaml: -------------------------------------------------------------------------------- 1 | extends: base.yaml 2 | 3 | backends: 4 | - namespace: library/.* 5 | backend: 6 | registry_tag: 7 | address: index.docker.io 8 | security: 9 | basic: 10 | username: "" 11 | password: "" 12 | - namespace: .* 13 | backend: 14 | testfs: 15 | addr: host.docker.internal:14000 16 | root: tags 17 | name_path: docker_tag 18 | 19 | cluster: 20 | hosts: 21 | static: 22 | - host.docker.internal:15004 23 | 24 | origin: 25 | hosts: 26 | static: 27 | - host.docker.internal:15002 28 | 29 | remotes: {} 30 | 31 | tag_replication: 32 | retry_interval: 100ms 33 | poll_retries_interval: 250ms 34 | 35 | tag_types: 36 | - namespace: .* 37 | type: docker 38 | root: tags 39 | 40 | tag_store: 41 | write_through: false 42 | 43 | writeback: 44 | retry_interval: 100ms 45 | poll_retries_interval: 250ms 46 | 47 | nginx: 48 | cache_dir: /tmp/kraken-build-index-nginx/ 49 | 50 | tls: 51 | name: kraken 52 | cas: 53 | - path: /etc/kraken/tls/ca/server.crt 54 | server: 55 | cert: 56 | path: /etc/kraken/tls/ca/server.crt 57 | key: 58 | path: /etc/kraken/tls/ca/server.key 59 | passphrase: 60 | path: /etc/kraken/tls/ca/passphrase 61 | client: 62 | cert: 63 | path: /etc/kraken/tls/client/client.crt 64 | key: 65 | path: /etc/kraken/tls/client/client.key 66 | passphrase: 67 | path: /etc/kraken/tls/client/passphrase 68 | -------------------------------------------------------------------------------- /examples/devcluster/config/origin/development.yaml: -------------------------------------------------------------------------------- 1 | extends: base.yaml 2 | 3 | backends: 4 | - namespace: library/.* 5 | backend: 6 | registry_blob: 7 | address: index.docker.io 8 | security: 9 | basic: 10 | username: "" 11 | password: "" 12 | - namespace: .* 13 | backend: 14 | testfs: 15 | addr: host.docker.internal:14000 16 | root: blobs 17 | name_path: identity 18 | 19 | cluster: 20 | static: 21 | - host.docker.internal:15002 22 | 23 | hashring: 24 | max_replica: 2 25 | 26 | writeback: 27 | retry_interval: 100ms 28 | poll_retries_interval: 250ms 29 | 30 | tls: 31 | name: kraken 32 | cas: 33 | - path: /etc/kraken/tls/ca/server.crt 34 | server: 35 | cert: 36 | path: /etc/kraken/tls/ca/server.crt 37 | key: 38 | path: /etc/kraken/tls/ca/server.key 39 | passphrase: 40 | path: /etc/kraken/tls/ca/passphrase 41 | client: 42 | cert: 43 | path: /etc/kraken/tls/client/client.crt 44 | key: 45 | path: /etc/kraken/tls/client/client.key 46 | passphrase: 47 | path: /etc/kraken/tls/client/passphrase 48 | -------------------------------------------------------------------------------- /examples/devcluster/config/proxy/development.yaml: -------------------------------------------------------------------------------- 1 | extends: base.yaml 2 | 3 | origin: 4 | hosts: 5 | static: 6 | - host.docker.internal:15002 7 | 8 | build_index: 9 | hosts: 10 | static: 11 | - host.docker.internal:15004 12 | 13 | nginx: 14 | cache_dir: /tmp/kraken-proxy-nginx/ 15 | 16 | tls: 17 | name: kraken 18 | cas: 19 | - path: /etc/kraken/tls/ca/server.crt 20 | server: 21 | disabled: true 22 | cert: 23 | path: /etc/kraken/tls/ca/server.crt 24 | key: 25 | path: /etc/kraken/tls/ca/server.key 26 | passphrase: 27 | path: /etc/kraken/tls/ca/passphrase 28 | client: 29 | cert: 30 | path: /etc/kraken/tls/client/client.crt 31 | key: 32 | path: /etc/kraken/tls/client/client.key 33 | passphrase: 34 | path: /etc/kraken/tls/client/passphrase 35 | -------------------------------------------------------------------------------- /examples/devcluster/config/tracker/development.yaml: -------------------------------------------------------------------------------- 1 | extends: base.yaml 2 | 3 | peerstore: 4 | redis: 5 | addr: 127.0.0.1:14001 6 | 7 | origin: 8 | hosts: 9 | static: 10 | - host.docker.internal:15002 11 | 12 | trackerserver: 13 | announce_interval: 3s 14 | 15 | nginx: 16 | cache_dir: /tmp/kraken-tracker-nginx/ 17 | 18 | tls: 19 | name: kraken 20 | cas: 21 | - path: /etc/kraken/tls/ca/server.crt 22 | server: 23 | cert: 24 | path: /etc/kraken/tls/ca/server.crt 25 | key: 26 | path: /etc/kraken/tls/ca/server.key 27 | passphrase: 28 | path: /etc/kraken/tls/ca/passphrase 29 | client: 30 | cert: 31 | path: /etc/kraken/tls/client/client.crt 32 | key: 33 | path: /etc/kraken/tls/client/client.key 34 | passphrase: 35 | path: /etc/kraken/tls/client/passphrase 36 | -------------------------------------------------------------------------------- /examples/devcluster/herd_param.sh: -------------------------------------------------------------------------------- 1 | # Define optional storage ports. 2 | TESTFS_PORT=14000 3 | REDIS_PORT=14001 4 | 5 | # Define herd ports. 6 | PROXY_PORT=15000 7 | ORIGIN_PEER_PORT=15001 8 | ORIGIN_SERVER_PORT=15002 9 | TRACKER_PORT=15003 10 | BUILD_INDEX_PORT=15004 11 | PROXY_SERVER_PORT=15005 12 | 13 | # Hostname for docker for mac. 14 | HOSTNAME=host.docker.internal 15 | -------------------------------------------------------------------------------- /examples/devcluster/herd_start_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | source examples/devcluster/herd_param.sh 6 | 7 | # Start kraken herd. 8 | docker run -d \ 9 | -p ${TESTFS_PORT}:${TESTFS_PORT} \ 10 | -p ${ORIGIN_SERVER_PORT}:${ORIGIN_SERVER_PORT} \ 11 | -p ${ORIGIN_PEER_PORT}:${ORIGIN_PEER_PORT} \ 12 | -p ${TRACKER_PORT}:${TRACKER_PORT} \ 13 | -p ${BUILD_INDEX_PORT}:${BUILD_INDEX_PORT} \ 14 | -p ${PROXY_PORT}:${PROXY_PORT} \ 15 | -p ${PROXY_SERVER_PORT}:${PROXY_SERVER_PORT} \ 16 | -v $(pwd)/examples/devcluster/config/origin/development.yaml:/etc/kraken/config/origin/development.yaml \ 17 | -v $(pwd)/examples/devcluster/config/tracker/development.yaml:/etc/kraken/config/tracker/development.yaml \ 18 | -v $(pwd)/examples/devcluster/config/build-index/development.yaml:/etc/kraken/config/build-index/development.yaml \ 19 | -v $(pwd)/examples/devcluster/config/proxy/development.yaml:/etc/kraken/config/proxy/development.yaml \ 20 | -v $(pwd)/examples/devcluster/herd_param.sh:/etc/kraken/herd_param.sh \ 21 | -v $(pwd)/examples/devcluster/herd_start_processes.sh:/etc/kraken/herd_start_processes.sh \ 22 | --name kraken-herd \ 23 | kraken-herd:dev ./herd_start_processes.sh 24 | -------------------------------------------------------------------------------- /examples/devcluster/herd_start_processes.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source /etc/kraken/herd_param.sh 4 | 5 | redis-server --port ${REDIS_PORT} & 6 | 7 | sleep 3 8 | 9 | /usr/bin/kraken-testfs \ 10 | --port=${TESTFS_PORT} \ 11 | &>/var/log/kraken/kraken-testfs/stdout.log & 12 | 13 | /usr/bin/kraken-origin \ 14 | --config=/etc/kraken/config/origin/development.yaml \ 15 | --blobserver-hostname=${HOSTNAME} \ 16 | --blobserver-port=${ORIGIN_SERVER_PORT} \ 17 | --peer-ip=${HOSTNAME} \ 18 | --peer-port=${ORIGIN_PEER_PORT} \ 19 | &>/var/log/kraken/kraken-origin/stdout.log & 20 | 21 | /usr/bin/kraken-tracker \ 22 | --config=/etc/kraken/config/tracker/development.yaml \ 23 | --port=${TRACKER_PORT} \ 24 | &>/var/log/kraken/kraken-tracker/stdout.log & 25 | 26 | /usr/bin/kraken-build-index \ 27 | --config=/etc/kraken/config/build-index/development.yaml \ 28 | --port=${BUILD_INDEX_PORT} \ 29 | &>/var/log/kraken/kraken-build-index/stdout.log & 30 | 31 | /usr/bin/kraken-proxy \ 32 | --config=/etc/kraken/config/proxy/development.yaml \ 33 | --port=${PROXY_PORT} \ 34 | --server-port=${PROXY_SERVER_PORT} \ 35 | &>/var/log/kraken/kraken-proxy/stdout.log & 36 | 37 | sleep 3 38 | 39 | # Poor man's supervisor. 40 | while : ; do 41 | for c in redis-server kraken-testfs kraken-origin kraken-tracker kraken-build-index kraken-proxy; do 42 | ps aux | grep $c | grep -q -v grep 43 | status=$? 44 | if [ $status -ne 0 ]; then 45 | echo "$c exited unexpectedly. Logs:" 46 | tail -100 /var/log/kraken/$c/stdout.log 47 | exit 1 48 | fi 49 | done 50 | sleep 30 51 | done 52 | -------------------------------------------------------------------------------- /examples/k8s/README.md: -------------------------------------------------------------------------------- 1 | ## 1. Run 2 | 3 | `$ helm install --name=kraken-demo ./helm` 4 | 5 | This command starts 3 trackers, origins and build-index pods, 1 proxy pod and an agent daemonset. 6 | 7 | Once deployed, every node will have a docker registry API exposed on port 30081. The port number is 8 | configurable. 9 | 10 | ## 2. Pulling from Docker Hub Library 11 | 12 | A simple registry storage backend is provided for read-only access to Docker registry. A library 13 | image can be pulled from Kraken agent by specifying `127.0.0.1:30081` in the image name in pod 14 | spec. For example spec, see [example](demo.json). 15 | 16 | Note, this backend is used only for all `library/.*` repositories. `library` is the default 17 | namespace for Docker Hub's standard public repositories. To use your own registry as the backend, 18 | please update origin and build-index configs accordingly. 19 | -------------------------------------------------------------------------------- /examples/k8s/demo.json: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "apps/v1", 3 | "kind": "Deployment", 4 | "metadata": { 5 | "name": "demo" 6 | }, 7 | "spec": { 8 | "replicas": 1, 9 | "selector": { 10 | "matchLabels": { 11 | "demo": "true" 12 | } 13 | }, 14 | "template": { 15 | "metadata": { 16 | "labels": { 17 | "demo": "true" 18 | } 19 | }, 20 | "spec": { 21 | "containers": [ 22 | { 23 | "name": "main", 24 | "image": "127.0.0.1:30081/library/busybox:latest", 25 | "command": [ 26 | "/bin/sh", "-c", 27 | "while true; do sleep 10; done" 28 | ] 29 | } 30 | ] 31 | } 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /helm/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | .git 6 | -------------------------------------------------------------------------------- /helm/Chart.yaml: -------------------------------------------------------------------------------- 1 | name: kraken 2 | description: P2P Docker registry capable of distributing TBs of data in seconds 3 | version: 0.2.0 4 | kubeVersion: ">=1.10.0" 5 | keywords: 6 | - http 7 | - docker 8 | - registry 9 | - p2p 10 | home: https://github.com/uber/kraken/helm 11 | maintainers: 12 | - name: Antoine Pourchet 13 | email: antoine.pourchet@gmail.com 14 | -------------------------------------------------------------------------------- /helm/config/agent.yaml: -------------------------------------------------------------------------------- 1 | extends: /etc/kraken/config/agent/base.yaml 2 | {{ include "tls" . }} 3 | {{ include "trackers" . }} 4 | {{ include "build-index" . }} 5 | -------------------------------------------------------------------------------- /helm/config/build-index.yaml: -------------------------------------------------------------------------------- 1 | extends: /etc/kraken/config/build-index/base.yaml 2 | {{ include "tls" . }} 3 | {{ include "build-index-cluster" . }} 4 | {{ include "origins" . }} 5 | backends: 6 | {{ with .Values.build_index.extraBackends }}{{ tpl . $ | indent 2 }}{{ end }} 7 | {{ if .Values.testfs.enabled }} 8 | - namespace: .* 9 | backend: 10 | testfs: 11 | addr: kraken-testfs:80 12 | root: tags 13 | name_path: docker_tag 14 | {{ end }} 15 | 16 | remotes: {} 17 | 18 | tag_replication: 19 | retry_interval: 100ms 20 | poll_retries_interval: 250ms 21 | 22 | tag_types: 23 | - namespace: .* 24 | type: docker 25 | root: tags 26 | 27 | tag_store: 28 | write_through: false 29 | 30 | writeback: 31 | retry_interval: 100ms 32 | poll_retries_interval: 250ms 33 | 34 | nginx: 35 | cache_dir: /tmp/kraken-build-index-nginx/ 36 | -------------------------------------------------------------------------------- /helm/config/origin.yaml: -------------------------------------------------------------------------------- 1 | extends: /etc/kraken/config/origin/base.yaml 2 | {{ include "tls" . }} 3 | {{ include "origin-cluster" . }} 4 | 5 | backends: 6 | {{ with .Values.origin.extraBackends }}{{ tpl . $ | indent 2 }}{{ end }} 7 | {{ if .Values.testfs.enabled }} 8 | - namespace: .* 9 | backend: 10 | testfs: 11 | addr: kraken-testfs:80 12 | root: blobs 13 | name_path: identity 14 | {{ end }} 15 | 16 | hashring: 17 | max_replica: 2 18 | 19 | writeback: 20 | retry_interval: 100ms 21 | poll_retries_interval: 250ms 22 | -------------------------------------------------------------------------------- /helm/config/proxy.yaml: -------------------------------------------------------------------------------- 1 | extends: /etc/kraken/config/proxy/base.yaml 2 | {{ include "tls" . }} 3 | {{ include "build-index" . }} 4 | {{ include "origins" . }} 5 | 6 | nginx: 7 | cache_dir: /tmp/kraken-proxy-nginx/ 8 | -------------------------------------------------------------------------------- /helm/config/tracker.yaml: -------------------------------------------------------------------------------- 1 | extends: /etc/kraken/config/tracker/base.yaml 2 | {{ include "tls" . }} 3 | {{ include "origins" . }} 4 | 5 | peerstore: 6 | redis: 7 | addr: 127.0.0.1:6379 8 | 9 | trackerserver: 10 | announce_interval: 3s 11 | 12 | nginx: 13 | cache_dir: /tmp/kraken-tracker-nginx/ 14 | -------------------------------------------------------------------------------- /helm/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | 3 | 4 | {{- define "trackers" -}} 5 | tracker: 6 | hosts: 7 | dns: kraken-tracker:80 8 | {{- end -}} 9 | 10 | 11 | 12 | {{- define "origins" -}} 13 | origin: 14 | hosts: 15 | dns: kraken-origin:80 16 | {{- end -}} 17 | 18 | {{- define "origin-cluster" -}} 19 | cluster: 20 | dns: kraken-origin:80 21 | {{- end -}} 22 | 23 | 24 | 25 | {{- define "build-index" -}} 26 | build_index: 27 | hosts: 28 | dns: kraken-build-index:80 29 | {{- end -}} 30 | 31 | {{- define "build-index-cluster" -}} 32 | cluster: 33 | hosts: 34 | dns: kraken-build-index:80 35 | {{- end -}} 36 | 37 | 38 | 39 | {{- define "tls" -}} 40 | tls: 41 | client: 42 | disabled: true 43 | server: 44 | disabled: true 45 | {{- end -}} 46 | -------------------------------------------------------------------------------- /helm/templates/config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: kraken 5 | data: 6 | agent.yaml: {{ tpl (.Files.Get "config/agent.yaml") . | toYaml | indent 2 }} 7 | build-index.yaml: {{ tpl (.Files.Get "config/build-index.yaml") . | toYaml | indent 2 }} 8 | origin.yaml: {{ tpl (.Files.Get "config/origin.yaml") . | toYaml | indent 2 }} 9 | proxy.yaml: {{ tpl (.Files.Get "config/proxy.yaml") . | toYaml | indent 2 }} 10 | tracker.yaml: {{ tpl (.Files.Get "config/tracker.yaml") . | toYaml | indent 2 }} 11 | -------------------------------------------------------------------------------- /helm/templates/testfs.yaml: -------------------------------------------------------------------------------- 1 | {{ if .Values.testfs.enabled }} 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: kraken-testfs 6 | spec: 7 | replicas: 1 8 | selector: 9 | matchLabels: 10 | app.kubernetes.io/name: kraken 11 | app.kubernetes.io/component: testfs 12 | app.kubernetes.io/instance: {{ .Release.Name }} 13 | template: 14 | metadata: 15 | labels: 16 | app.kubernetes.io/name: kraken 17 | app.kubernetes.io/component: testfs 18 | app.kubernetes.io/instance: {{ .Release.Name }} 19 | {{ with .Values.testfs.annotations -}} 20 | annotations: 21 | {{ tpl . $ | indent 8 }} 22 | {{- end }} 23 | spec: 24 | containers: 25 | - name: main 26 | image: {{ .Values.kraken.repository }}/kraken-testfs:{{ .Values.kraken.tag }} 27 | imagePullPolicy: {{ .Values.kraken.imagePullPolicy }} 28 | command: 29 | - /usr/bin/kraken-testfs 30 | - --port=80 31 | --- 32 | kind: Service 33 | apiVersion: v1 34 | metadata: 35 | name: kraken-testfs 36 | spec: 37 | selector: 38 | app.kubernetes.io/name: kraken 39 | app.kubernetes.io/component: testfs 40 | app.kubernetes.io/instance: {{ .Release.Name }} 41 | ports: 42 | - protocol: TCP 43 | port: 80 44 | targetPort: 80 45 | {{ end }} -------------------------------------------------------------------------------- /lib/backend/backenderrors/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package backenderrors 15 | 16 | import "errors" 17 | 18 | // ErrBlobNotFound is returned when a blob is not found in a storage backend. 19 | var ErrBlobNotFound = errors.New("blob not found") 20 | -------------------------------------------------------------------------------- /lib/backend/constants.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package backend 15 | 16 | import ( 17 | "github.com/uber/kraken/core" 18 | "github.com/uber/kraken/utils/memsize" 19 | 20 | "github.com/c2h5oh/datasize" 21 | ) 22 | 23 | const ( 24 | DefaultPartSize int64 = int64(64 * memsize.MB) 25 | DefaultBufferGuard datasize.ByteSize = 10 * datasize.MB 26 | DefaultConcurrency int = 10 27 | DefaultListMaxKeys int = 250 28 | ) 29 | 30 | var ( 31 | ReadinessCheckNamespace string = core.NamespaceFixture() 32 | ReadinessCheckName string = core.DigestFixture().Hex() 33 | ReadinessCheckDigest, _ = core.NewSHA256DigestFromHex(ReadinessCheckName) 34 | ) 35 | -------------------------------------------------------------------------------- /lib/backend/fixtures.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package backend 15 | 16 | import ( 17 | "github.com/uber-go/tally" 18 | ) 19 | 20 | // ManagerFixture returns a Manager with no clients for testing purposes. 21 | func ManagerFixture() *Manager { 22 | m, err := NewManager(ManagerConfig{}, nil, AuthConfig{}, tally.NoopScope) 23 | if err != nil { 24 | panic(err) 25 | } 26 | return m 27 | } 28 | -------------------------------------------------------------------------------- /lib/backend/gcsbackend/gcs.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package gcsbackend 15 | 16 | import ( 17 | "io" 18 | 19 | "cloud.google.com/go/storage" 20 | "google.golang.org/api/iterator" 21 | ) 22 | 23 | // GCS defines the operations we use in the GCS api. Useful for mocking. 24 | type GCS interface { 25 | ObjectAttrs(objectName string) (*storage.ObjectAttrs, error) 26 | Download(objectName string, w io.Writer) (int64, error) 27 | Upload(objectName string, r io.Reader) (int64, error) 28 | GetObjectIterator(prefix string) iterator.Pageable 29 | NextPage(pager *iterator.Pager) ([]string, string, error) 30 | } 31 | -------------------------------------------------------------------------------- /lib/backend/hdfsbackend/webhdfs/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package webhdfs 15 | 16 | import "github.com/c2h5oh/datasize" 17 | 18 | // Config defines Client configuration. 19 | type Config struct { 20 | // BufferSize is the transfer block size. 21 | BufferSize datasize.ByteSize `yaml:"buffer_size"` 22 | 23 | // BufferGuard protects upload from draining the src reader into an oversized 24 | // buffer when io.Seeker is not implemented. 25 | BufferGuard datasize.ByteSize `yaml:"buffer_guard"` 26 | } 27 | 28 | func (c *Config) applyDefaults() { 29 | if c.BufferSize == 0 { 30 | c.BufferSize = 64 * datasize.MB 31 | } 32 | if c.BufferGuard == 0 { 33 | c.BufferGuard = 10 * datasize.MB 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /lib/backend/hdfsbackend/webhdfs/json.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package webhdfs 15 | 16 | // FileStatus defines FILESTATUS response body. 17 | type FileStatus struct { 18 | PathSuffix string `json:"pathSuffix"` 19 | Type string `json:"type"` 20 | Length int64 `json:"length"` 21 | } 22 | 23 | type fileStatusResponse struct { 24 | FileStatus FileStatus `json:"FileStatus"` 25 | } 26 | 27 | type listStatusResponse struct { 28 | FileStatuses struct { 29 | FileStatus []FileStatus `json:"FileStatus"` 30 | } `json:"FileStatuses"` 31 | } 32 | -------------------------------------------------------------------------------- /lib/backend/registrybackend/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package registrybackend 15 | 16 | import ( 17 | "time" 18 | 19 | "github.com/uber/kraken/lib/backend/registrybackend/security" 20 | ) 21 | 22 | // Config defines the registry address, timeout and security options. 23 | type Config struct { 24 | Address string `yaml:"address"` 25 | Timeout time.Duration `yaml:"timeout"` 26 | Security security.Config `yaml:"security"` 27 | } 28 | 29 | // Set default configuration 30 | func (c Config) applyDefaults() Config { 31 | if c.Timeout == 0 { 32 | c.Timeout = 60 * time.Second 33 | } 34 | return c 35 | } 36 | -------------------------------------------------------------------------------- /lib/backend/registrybackend/config_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package registrybackend 16 | 17 | import "github.com/uber/kraken/lib/backend/registrybackend/security" 18 | 19 | func newTestConfig(addr string) Config { 20 | return Config{ 21 | Address: addr, 22 | Security: security.Config{ 23 | EnableHTTPFallback: true, 24 | }, 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /lib/backend/results.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package backend 15 | 16 | // ListResult defines the response from a client list operation. 17 | // The names of the entries found will always be populated, and the 18 | // continuation token will only be populated if pagination was enabled 19 | // and there are remaining entries to list. 20 | type ListResult struct { 21 | Names []string 22 | ContinuationToken string 23 | } 24 | -------------------------------------------------------------------------------- /lib/backend/s3backend/s3.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package s3backend 15 | 16 | import ( 17 | "io" 18 | 19 | "github.com/aws/aws-sdk-go/service/s3" 20 | "github.com/aws/aws-sdk-go/service/s3/s3iface" 21 | "github.com/aws/aws-sdk-go/service/s3/s3manager" 22 | ) 23 | 24 | // S3 defines the operations we use in the s3 api. Useful for mocking. 25 | type S3 interface { 26 | HeadObject(input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) 27 | 28 | Download( 29 | w io.WriterAt, 30 | input *s3.GetObjectInput, 31 | options ...func(*s3manager.Downloader)) (n int64, err error) 32 | 33 | Upload( 34 | input *s3manager.UploadInput, 35 | options ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error) 36 | 37 | ListObjectsV2Pages(input *s3.ListObjectsV2Input, fn func(*s3.ListObjectsV2Output, bool) bool) error 38 | } 39 | 40 | type join struct { 41 | s3iface.S3API 42 | *s3manager.Downloader 43 | *s3manager.Uploader 44 | } 45 | 46 | var _ S3 = (*join)(nil) 47 | -------------------------------------------------------------------------------- /lib/backend/shadowbackend/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2020 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package shadowbackend 15 | 16 | // Config is used to initialize the Shadow client 17 | type Config struct { 18 | ActiveClientConfig map[string]interface{} `yaml:"active_backend"` 19 | ShadowClientConfig map[string]interface{} `yaml:"shadow_backend"` 20 | } 21 | -------------------------------------------------------------------------------- /lib/backend/sqlbackend/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2020 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package sqlbackend 15 | 16 | // Config is used to initialize the SQL Backend Client 17 | type Config struct { 18 | DebugLogging bool `yaml:"debug_logging"` 19 | Dialect string `yaml:"dialect"` 20 | ConnectionString string `yaml:"connection_string"` 21 | Username string `yaml:"username"` 22 | } 23 | 24 | // UserAuthConfig defines authentication configuration overlayed by Langley/Vault. 25 | // Each key is the iam username of the credentials. 26 | type UserAuthConfig map[string]AuthConfig 27 | 28 | // SQL is a struct that holds credentials. This is declared here to make testing easier 29 | type SQL struct { 30 | User string `yaml:"user"` 31 | Password string `yaml:"password"` 32 | } 33 | 34 | // AuthConfig matches Langley format. 35 | type AuthConfig struct { 36 | SQL SQL `yaml:"sql"` 37 | } 38 | -------------------------------------------------------------------------------- /lib/backend/sqlbackend/schema.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2020 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package sqlbackend 15 | 16 | import "time" 17 | 18 | // Tag represents a Docker tag 19 | type Tag struct { 20 | ID uint64 `gorm:"primary_key;auto_increment:true"` 21 | Repository string `gorm:"not null;type:varchar(255);index:repository;unique_index:repository_tag"` 22 | Tag string `gorm:"not null;type:varchar(128);unique_index:repository_tag"` 23 | ImageID string `gorm:"not null;type:varchar(2056)"` 24 | CreatedAt time.Time 25 | UpdatedAt time.Time 26 | } 27 | -------------------------------------------------------------------------------- /lib/backend/testfs/client_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package testfs 15 | 16 | import ( 17 | "testing" 18 | 19 | "github.com/uber-go/tally" 20 | "github.com/uber/kraken/lib/backend/namepath" 21 | 22 | "github.com/stretchr/testify/require" 23 | "go.uber.org/zap" 24 | ) 25 | 26 | func TestClientFactory(t *testing.T) { 27 | require := require.New(t) 28 | 29 | config := Config{ 30 | Addr: "test", 31 | NamePath: namepath.Identity, 32 | } 33 | f := factory{} 34 | _, err := f.Create(config, nil, tally.NoopScope, zap.NewNop().Sugar()) 35 | require.NoError(err) 36 | } 37 | -------------------------------------------------------------------------------- /lib/backend/testfs/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package testfs 15 | 16 | // Config defines Client configuration. 17 | type Config struct { 18 | Addr string `yaml:"addr"` 19 | Root string `yaml:"root"` 20 | NamePath string `yaml:"name_path"` 21 | } 22 | -------------------------------------------------------------------------------- /lib/blobrefresh/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package blobrefresh 15 | 16 | import "github.com/c2h5oh/datasize" 17 | 18 | // Config defines Refresher configuration. 19 | type Config struct { 20 | // Limits the size of blobs which origin will accept. A 0 size limit means 21 | // blob size is unbounded. 22 | SizeLimit datasize.ByteSize `yaml:"size_limit"` 23 | } 24 | -------------------------------------------------------------------------------- /lib/containerruntime/.factory.go.swp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber/kraken/604194a25ea5a4b79678ff5ba62515a0f7062e4c/lib/containerruntime/.factory.go.swp -------------------------------------------------------------------------------- /lib/containerruntime/containerd/.config.go.swp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber/kraken/604194a25ea5a4b79678ff5ba62515a0f7062e4c/lib/containerruntime/containerd/.config.go.swp -------------------------------------------------------------------------------- /lib/containerruntime/containerd/client.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License.ackage containerd 14 | package containerd 15 | 16 | import ( 17 | "context" 18 | "fmt" 19 | 20 | "github.com/containerd/containerd" 21 | ) 22 | 23 | type Client interface { 24 | PullImage(context.Context, string, string, string) error 25 | } 26 | 27 | type Impl struct { 28 | config Config 29 | registry string 30 | } 31 | 32 | func New(config Config, registry string) *Impl { 33 | return &Impl{config.applyDefaults(), registry} 34 | } 35 | 36 | func (c *Impl) PullImage(ctx context.Context, ns, repo, tag string) error { 37 | client, err := containerd.New(c.config.Address, containerd.WithDefaultNamespace(ns)) 38 | if err != nil { 39 | return fmt.Errorf("new containerd client: %s", err) 40 | } 41 | defer client.Close() 42 | 43 | _, err = client.Pull(ctx, fmt.Sprintf("%s/%s:%s", c.registry, repo, tag), containerd.WithPullUnpack) 44 | if err != nil { 45 | return fmt.Errorf("containerd pull image: %s", err) 46 | } 47 | return nil 48 | } 49 | -------------------------------------------------------------------------------- /lib/containerruntime/containerd/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License.ackage containerd 14 | 15 | // Config contains configuration to connect to containerd. 16 | package containerd 17 | 18 | type Config struct { 19 | Address string `yaml:"address"` 20 | } 21 | 22 | func (c Config) applyDefaults() Config { 23 | if c.Address == "" { 24 | c.Address = "/run/containerd/containerd.sock" 25 | } 26 | return c 27 | } 28 | -------------------------------------------------------------------------------- /lib/containerruntime/dockerdaemon/.cli.go.swn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber/kraken/604194a25ea5a4b79678ff5ba62515a0f7062e4c/lib/containerruntime/dockerdaemon/.cli.go.swn -------------------------------------------------------------------------------- /lib/containerruntime/dockerdaemon/.cli.go.swo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber/kraken/604194a25ea5a4b79678ff5ba62515a0f7062e4c/lib/containerruntime/dockerdaemon/.cli.go.swo -------------------------------------------------------------------------------- /lib/containerruntime/dockerdaemon/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package dockerdaemon 15 | 16 | // Configs connecting to docker daemon. 17 | type Config struct { 18 | DockerHost string `yaml:"docker_host"` 19 | DockerScheme string `yaml:"docker_scheme"` 20 | DockerClientVersion string `yaml:"docker_version"` 21 | } 22 | 23 | func (c Config) applyDefaults() Config { 24 | if c.DockerHost == "" { 25 | c.DockerHost = "unix:///var/run/docker.sock" 26 | } 27 | if c.DockerScheme == "" { 28 | c.DockerScheme = "http" 29 | } 30 | if c.DockerClientVersion == "" { 31 | c.DockerClientVersion = "1.21" 32 | } 33 | return c 34 | } 35 | -------------------------------------------------------------------------------- /lib/dockerregistry/fixtures.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package dockerregistry 15 | 16 | import ( 17 | "github.com/uber/kraken/lib/dockerregistry/transfer" 18 | "github.com/uber/kraken/lib/store" 19 | 20 | "github.com/uber-go/tally" 21 | ) 22 | 23 | // StorageDriverFixture creates a storage driver for testing purposes. 24 | func StorageDriverFixture() (*KrakenStorageDriver, func()) { 25 | cas, cleanup := store.CAStoreFixture() 26 | sd := NewReadWriteStorageDriver( 27 | Config{}, cas, transfer.NewTestTransferer(cas), tally.NoopScope) 28 | return sd, cleanup 29 | } 30 | -------------------------------------------------------------------------------- /lib/dockerregistry/metadata_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package dockerregistry 15 | 16 | import ( 17 | "testing" 18 | "time" 19 | 20 | "github.com/stretchr/testify/require" 21 | ) 22 | 23 | func TestStartedAtMetadataSerialization(t *testing.T) { 24 | require := require.New(t) 25 | 26 | s := newStartedAtMetadata(time.Now()) 27 | b, err := s.Serialize() 28 | require.NoError(err) 29 | 30 | var result startedAtMetadata 31 | require.NoError(result.Deserialize(b)) 32 | require.Equal(s.time.Unix(), result.time.Unix()) 33 | } 34 | 35 | func TestHashState(t *testing.T) { 36 | require := require.New(t) 37 | 38 | h := newHashStateMetadata("sha256", "500") 39 | require.Equal(h.GetSuffix(), "_hashstates/sha256/500") 40 | } 41 | -------------------------------------------------------------------------------- /lib/dockerregistry/transfer/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package transfer 15 | 16 | import "errors" 17 | 18 | // ErrBlobNotFound is returned when a blob is not found by transferer. 19 | var ErrBlobNotFound = errors.New("blob not found") 20 | 21 | // ErrTagNotFound is returned when a tag is not found by transferer. 22 | var ErrTagNotFound = errors.New("tag not found") 23 | -------------------------------------------------------------------------------- /lib/dockerregistry/transfer/transferer.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package transfer 15 | 16 | import ( 17 | "github.com/uber/kraken/core" 18 | "github.com/uber/kraken/lib/store" 19 | ) 20 | 21 | // ImageTransferer defines an interface that transfers images 22 | type ImageTransferer interface { 23 | Stat(namespace string, d core.Digest) (*core.BlobInfo, error) 24 | Download(namespace string, d core.Digest) (store.FileReader, error) 25 | Upload(namespace string, d core.Digest, blob store.FileReader) error 26 | 27 | GetTag(tag string) (core.Digest, error) 28 | PutTag(tag string, d core.Digest) error 29 | ListTags(prefix string) ([]string, error) 30 | } 31 | -------------------------------------------------------------------------------- /lib/hashring/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package hashring 15 | 16 | import "time" 17 | 18 | // Config defines Ring configuration. 19 | type Config struct { 20 | // MaxReplica is the max number of hosts each blob will be replicated across. 21 | // If MaxReplica is >= the number of hosts in the ring, every host will own 22 | // every blob. 23 | MaxReplica int `yaml:"max_replica"` 24 | 25 | // RefreshInterval is the interval at which membership / health information 26 | // is refreshed during monitoring. 27 | RefreshInterval time.Duration `yaml:"refresh_interval"` 28 | } 29 | 30 | func (c *Config) applyDefaults() { 31 | if c.MaxReplica == 0 { 32 | c.MaxReplica = 3 33 | } 34 | if c.RefreshInterval == 0 { 35 | c.RefreshInterval = 10 * time.Second 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /lib/hashring/passive_ring.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package hashring 15 | 16 | import ( 17 | "github.com/uber/kraken/lib/healthcheck" 18 | "github.com/uber/kraken/lib/hostlist" 19 | ) 20 | 21 | // PassiveRing is a wrapper around Ring which supports passive health checks. 22 | // See healthcheck.PassiveFilter for passive health check documentation. 23 | type PassiveRing interface { 24 | Ring 25 | Failed(addr string) 26 | } 27 | 28 | type passiveRing struct { 29 | Ring 30 | passiveFilter healthcheck.PassiveFilter 31 | } 32 | 33 | // NewPassive creats a new PassiveRing. 34 | func NewPassive( 35 | config Config, 36 | cluster hostlist.List, 37 | passiveFilter healthcheck.PassiveFilter, 38 | opts ...Option) PassiveRing { 39 | 40 | return &passiveRing{ 41 | New(config, cluster, passiveFilter, opts...), 42 | passiveFilter, 43 | } 44 | } 45 | 46 | // Failed marks a request to addr as failed. 47 | func (p *passiveRing) Failed(addr string) { 48 | p.passiveFilter.Failed(addr) 49 | } 50 | -------------------------------------------------------------------------------- /lib/hashring/passive_ring_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package hashring 15 | 16 | import ( 17 | "testing" 18 | "time" 19 | 20 | "github.com/andres-erbsen/clock" 21 | "github.com/stretchr/testify/require" 22 | 23 | "github.com/uber/kraken/core" 24 | "github.com/uber/kraken/lib/healthcheck" 25 | "github.com/uber/kraken/lib/hostlist" 26 | ) 27 | 28 | func TestPassiveRingFailedAffectsRefreshFilter(t *testing.T) { 29 | require := require.New(t) 30 | 31 | r := NewPassive( 32 | Config{MaxReplica: 3}, 33 | hostlist.Fixture(addrsFixture(10)...), 34 | healthcheck.NewPassiveFilter(healthcheck.PassiveFilterConfig{ 35 | Fails: 3, 36 | FailTimeout: 5 * time.Second, 37 | }, clock.New())) 38 | 39 | d := core.DigestFixture() 40 | 41 | replicas := r.Locations(d) 42 | require.Len(replicas, 3) 43 | 44 | for i := 0; i < 4; i++ { 45 | r.Failed(replicas[0]) 46 | } 47 | r.Refresh() 48 | 49 | result := r.Locations(d) 50 | require.Equal(replicas[1:], result) 51 | } 52 | -------------------------------------------------------------------------------- /lib/hashring/testing.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package hashring 15 | 16 | import ( 17 | "github.com/uber/kraken/lib/healthcheck" 18 | "github.com/uber/kraken/lib/hostlist" 19 | ) 20 | 21 | // NoopPassiveRing returns a PassiveRing which never filters unhealthy hosts. 22 | func NoopPassiveRing(hosts hostlist.List) PassiveRing { 23 | return NewPassive(Config{}, hosts, healthcheck.IdentityFilter{}) 24 | } 25 | -------------------------------------------------------------------------------- /lib/healthcheck/checker.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package healthcheck 15 | 16 | import ( 17 | "context" 18 | "crypto/tls" 19 | "fmt" 20 | 21 | "github.com/uber/kraken/utils/httputil" 22 | ) 23 | 24 | // Checker runs a health check against an address. 25 | type Checker interface { 26 | Check(ctx context.Context, addr string) error 27 | } 28 | 29 | // Default returns a Checker which makes a GET request against /health. 30 | func Default(tls *tls.Config) Checker { 31 | return defaultChecker{tls} 32 | } 33 | 34 | type defaultChecker struct{ tls *tls.Config } 35 | 36 | func (c defaultChecker) Check(ctx context.Context, addr string) error { 37 | _, err := httputil.Get( 38 | fmt.Sprintf("http://%s/health", addr), 39 | httputil.SendContext(ctx), 40 | httputil.SendTLS(c.tls)) 41 | return err 42 | } 43 | -------------------------------------------------------------------------------- /lib/healthcheck/list.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package healthcheck 15 | 16 | import ( 17 | "github.com/uber/kraken/lib/hostlist" 18 | ) 19 | 20 | // List is a hostlist.List which can be passively health checked. 21 | type List interface { 22 | hostlist.List 23 | Failed(addr string) 24 | } 25 | 26 | type noopFailed struct { 27 | hostlist.List 28 | } 29 | 30 | func (f *noopFailed) Failed(addr string) {} 31 | 32 | // NoopFailed converts a hostlist.List to a List by making the Failed method 33 | // a no-op. Useful for using a Monitor in place of a Passive. 34 | func NoopFailed(list hostlist.List) List { 35 | return &noopFailed{list} 36 | } 37 | -------------------------------------------------------------------------------- /lib/healthcheck/monitor_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package healthcheck 15 | 16 | import ( 17 | "testing" 18 | "time" 19 | 20 | "github.com/uber/kraken/lib/hostlist" 21 | "github.com/uber/kraken/mocks/lib/healthcheck" 22 | "github.com/uber/kraken/utils/stringset" 23 | 24 | "github.com/golang/mock/gomock" 25 | "github.com/stretchr/testify/require" 26 | ) 27 | 28 | func TestActiveMonitor(t *testing.T) { 29 | require := require.New(t) 30 | 31 | ctrl := gomock.NewController(t) 32 | defer ctrl.Finish() 33 | 34 | x := "x:80" 35 | y := "y:80" 36 | 37 | filter := mockhealthcheck.NewMockFilter(ctrl) 38 | 39 | filter.EXPECT().Run(stringset.New(x, y)).Return(stringset.New(x)) 40 | 41 | m := NewMonitor( 42 | MonitorConfig{Interval: time.Second}, 43 | hostlist.Fixture(x, y), 44 | filter) 45 | defer m.Stop() 46 | 47 | require.Equal(stringset.New(x, y), m.Resolve()) 48 | 49 | time.Sleep(1250 * time.Millisecond) 50 | 51 | require.Equal(stringset.New(x), m.Resolve()) 52 | } 53 | -------------------------------------------------------------------------------- /lib/healthcheck/passive.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package healthcheck 15 | 16 | import ( 17 | "github.com/uber/kraken/lib/hostlist" 18 | "github.com/uber/kraken/utils/stringset" 19 | ) 20 | 21 | // Passive wraps a passive health check and can be used as a hostlist.List. See 22 | // PassiveFilter for passive health check documenation. 23 | type Passive struct { 24 | hosts hostlist.List 25 | filter PassiveFilter 26 | } 27 | 28 | // NewPassive returns a new Passive. 29 | func NewPassive(hosts hostlist.List, filter PassiveFilter) *Passive { 30 | return &Passive{hosts, filter} 31 | } 32 | 33 | // Resolve returns the latest healthy hosts. If all hosts are unhealthy, returns 34 | // all hosts. 35 | func (p *Passive) Resolve() stringset.Set { 36 | all := p.hosts.Resolve() 37 | healthy := p.filter.Run(all) 38 | if len(healthy) == 0 { 39 | return all 40 | } 41 | return healthy 42 | } 43 | 44 | // Failed marks a request to addr as failed. 45 | func (p *Passive) Failed(addr string) { 46 | p.filter.Failed(addr) 47 | } 48 | -------------------------------------------------------------------------------- /lib/hostlist/fixtures.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package hostlist 15 | 16 | // Fixture returns a static list of addrs for testing purposes. 17 | func Fixture(addrs ...string) List { 18 | l, err := New(Config{Static: addrs}) 19 | if err != nil { 20 | panic(err) 21 | } 22 | return l 23 | } 24 | -------------------------------------------------------------------------------- /lib/hrw/testutils_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package hrw 15 | 16 | import ( 17 | "testing" 18 | 19 | "github.com/stretchr/testify/assert" 20 | ) 21 | 22 | // assertKeyDistribution makes sure the ratio of keys falls in a particular 23 | // bucket conforms to general weight distribution within delta. 24 | func assertKeyDistribution( 25 | t *testing.T, rh *RendezvousHash, nodekeys NodeKeysTable, 26 | numKeys int, totalWeights float64, delta float64) { 27 | 28 | for name, v := range nodekeys { 29 | node, _ := rh.GetNode(name) 30 | assert.NotEqual(t, node, nil) 31 | 32 | assert.InDelta( 33 | t, float64(len(v))/float64(numKeys), float64(node.Weight)/totalWeights, delta) 34 | } 35 | } 36 | 37 | type ByScore []float64 38 | 39 | func (a ByScore) Len() int { return len(a) } 40 | func (a ByScore) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 41 | func (a ByScore) Less(i, j int) bool { return a[i] < a[j] } 42 | -------------------------------------------------------------------------------- /lib/metainfogen/config_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package metainfogen 15 | 16 | import ( 17 | "testing" 18 | 19 | "github.com/c2h5oh/datasize" 20 | "github.com/stretchr/testify/require" 21 | ) 22 | 23 | func TestPieceLengthConfig(t *testing.T) { 24 | require := require.New(t) 25 | 26 | plConfig, err := newPieceLengthConfig(map[datasize.ByteSize]datasize.ByteSize{ 27 | 0: datasize.MB, 28 | 2 * datasize.GB: 4 * datasize.MB, 29 | 4 * datasize.GB: 8 * datasize.MB, 30 | }) 31 | require.NoError(err) 32 | 33 | require.Equal(int64(datasize.MB), plConfig.get(int64(datasize.GB))) 34 | require.Equal(int64(4*datasize.MB), plConfig.get(int64(2*datasize.GB))) 35 | require.Equal(int64(4*datasize.MB), plConfig.get(int64(3*datasize.GB))) 36 | require.Equal(int64(8*datasize.MB), plConfig.get(int64(4*datasize.GB))) 37 | require.Equal(int64(8*datasize.MB), plConfig.get(int64(8*datasize.GB))) 38 | } 39 | -------------------------------------------------------------------------------- /lib/metainfogen/fixtures.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package metainfogen 15 | 16 | import ( 17 | "github.com/uber/kraken/lib/store" 18 | 19 | "github.com/c2h5oh/datasize" 20 | ) 21 | 22 | // Fixture returns a Generator which creates all metainfo with pieceLength for 23 | // testing purposes. 24 | func Fixture(cas *store.CAStore, pieceLength int) *Generator { 25 | g, err := New(Config{ 26 | PieceLengths: map[datasize.ByteSize]datasize.ByteSize{0: datasize.ByteSize(pieceLength)}, 27 | }, cas) 28 | if err != nil { 29 | panic(err) 30 | } 31 | return g 32 | } 33 | -------------------------------------------------------------------------------- /lib/persistedretry/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package persistedretry 15 | 16 | import "errors" 17 | 18 | // Store errors. 19 | var ( 20 | ErrTaskExists = errors.New("task already exists in store") 21 | ErrTaskNotFound = errors.New("task not found") 22 | ) 23 | -------------------------------------------------------------------------------- /lib/persistedretry/tagreplication/fixtures.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package tagreplication 15 | 16 | import ( 17 | "fmt" 18 | 19 | "github.com/uber/kraken/core" 20 | "github.com/uber/kraken/utils/randutil" 21 | ) 22 | 23 | // TaskFixture creates a fixture of tagreplication.Task. 24 | func TaskFixture() *Task { 25 | tag := core.TagFixture() 26 | d := core.DigestFixture() 27 | dest := fmt.Sprintf("build-index-%s", randutil.Hex(8)) 28 | return NewTask(tag, d, core.DigestListFixture(3), dest, 0) 29 | } 30 | -------------------------------------------------------------------------------- /lib/persistedretry/tagreplication/testing.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package tagreplication 15 | 16 | import ( 17 | "reflect" 18 | "time" 19 | ) 20 | 21 | // TaskMatcher is a gomock Matcher which matches two tasks. 22 | type TaskMatcher struct { 23 | task Task 24 | } 25 | 26 | // MatchTask returns a new TaskMatcher 27 | func MatchTask(task *Task) *TaskMatcher { 28 | return &TaskMatcher{*task} 29 | } 30 | 31 | // Matches compares two tasks. It ignores checking for time. 32 | func (m *TaskMatcher) Matches(x interface{}) bool { 33 | expected := m.task 34 | result := *(x.(*Task)) 35 | 36 | expected.CreatedAt = time.Time{} 37 | result.CreatedAt = time.Time{} 38 | expected.LastAttempt = time.Time{} 39 | result.LastAttempt = time.Time{} 40 | 41 | return reflect.DeepEqual(expected, result) 42 | } 43 | 44 | // String returns the name of the matcher. 45 | func (m *TaskMatcher) String() string { 46 | return "TaskMatcher" 47 | } 48 | -------------------------------------------------------------------------------- /lib/persistedretry/writeback/fixtures.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package writeback 15 | 16 | import ( 17 | "fmt" 18 | 19 | "github.com/uber/kraken/core" 20 | "github.com/uber/kraken/utils/randutil" 21 | ) 22 | 23 | // TaskFixture returns a randomly generated Task for testing purposes. 24 | func TaskFixture() *Task { 25 | return NewTask( 26 | fmt.Sprintf("namespace-%s", randutil.Hex(8)), 27 | core.DigestFixture().Hex(), 0) 28 | } 29 | -------------------------------------------------------------------------------- /lib/persistedretry/writeback/query.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package writeback 15 | 16 | // NameQuery queries writeback tasks which match a name. 17 | type NameQuery struct { 18 | name string 19 | } 20 | 21 | // NewNameQuery returns a new NameQuery. 22 | func NewNameQuery(name string) *NameQuery { 23 | return &NameQuery{name} 24 | } 25 | -------------------------------------------------------------------------------- /lib/persistedretry/writeback/testing.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package writeback 15 | 16 | import ( 17 | "reflect" 18 | "time" 19 | ) 20 | 21 | // TaskMatcher is a gomock Matcher which matches two tasks. 22 | type TaskMatcher struct { 23 | task Task 24 | } 25 | 26 | // MatchTask returns a new TaskMatcher 27 | func MatchTask(task *Task) *TaskMatcher { 28 | return &TaskMatcher{*task} 29 | } 30 | 31 | // Matches compares two tasks. It ignores checking for time. 32 | func (m *TaskMatcher) Matches(x interface{}) bool { 33 | expected := m.task 34 | result := *(x.(*Task)) 35 | 36 | expected.CreatedAt = time.Time{} 37 | result.CreatedAt = time.Time{} 38 | expected.LastAttempt = time.Time{} 39 | result.LastAttempt = time.Time{} 40 | 41 | return reflect.DeepEqual(expected, result) 42 | } 43 | 44 | // String returns the name of the matcher. 45 | func (m *TaskMatcher) String() string { 46 | return "TaskMatcher" 47 | } 48 | -------------------------------------------------------------------------------- /lib/store/base/const.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package base 15 | 16 | // DefaultShardIDLength is the number of bytes of file digest to be used for shard ID. 17 | // For every byte (2 HEX char), one more level of directories will be created. 18 | const DefaultShardIDLength = 2 19 | 20 | // DefaultDirPermission is the default permission for new directories. 21 | const DefaultDirPermission = 0775 22 | 23 | // DefaultDataFileName is the name of the actual blob data file. 24 | const DefaultDataFileName = "data" 25 | -------------------------------------------------------------------------------- /lib/store/base/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package base 15 | 16 | import "fmt" 17 | 18 | // FileStateError represents errors related to file state. 19 | // It's used when a file is not in the state it was supposed to be in. 20 | type FileStateError struct { 21 | Op string 22 | Name string 23 | State FileState 24 | Msg string 25 | } 26 | 27 | func (e *FileStateError) Error() string { 28 | return fmt.Sprintf("failed to perform \"%s\" on %s/%s: %s", 29 | e.Op, e.State.GetDirectory(), e.Name, e.Msg) 30 | } 31 | 32 | // IsFileStateError returns true if the param is of FileStateError type. 33 | func IsFileStateError(err error) bool { 34 | _, ok := err.(*FileStateError) 35 | return ok 36 | } 37 | -------------------------------------------------------------------------------- /lib/store/ca_download_store_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package store 15 | 16 | import ( 17 | "os" 18 | "sync" 19 | "testing" 20 | 21 | "github.com/uber/kraken/core" 22 | 23 | "github.com/stretchr/testify/require" 24 | ) 25 | 26 | func TestCADownloadStoreDownloadAndDeleteFiles(t *testing.T) { 27 | require := require.New(t) 28 | 29 | s, cleanup := CADownloadStoreFixture() 30 | defer cleanup() 31 | 32 | var names []string 33 | var wg sync.WaitGroup 34 | for i := 0; i < 100; i++ { 35 | name := core.DigestFixture().Hex() 36 | names = append(names, name) 37 | wg.Add(1) 38 | go func() { 39 | defer wg.Done() 40 | require.NoError(s.CreateDownloadFile(name, 1)) 41 | require.NoError(s.MoveDownloadFileToCache(name)) 42 | require.NoError(s.Cache().DeleteFile(name)) 43 | }() 44 | } 45 | wg.Wait() 46 | 47 | for _, name := range names { 48 | _, err := s.Cache().GetFileStat(name) 49 | require.True(os.IsNotExist(err)) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /lib/store/file.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package store 15 | 16 | import "github.com/uber/kraken/lib/store/base" 17 | 18 | // FileReadWriter is a readable, writable file. 19 | type FileReadWriter = base.FileReadWriter 20 | 21 | // FileReader is a read-only file. 22 | type FileReader = base.FileReader 23 | -------------------------------------------------------------------------------- /lib/store/metadata/last_access_time_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package metadata 15 | 16 | import ( 17 | "testing" 18 | "time" 19 | 20 | "github.com/stretchr/testify/require" 21 | ) 22 | 23 | func TestLastAccessTimeSerialization(t *testing.T) { 24 | require := require.New(t) 25 | 26 | lat := NewLastAccessTime(time.Now().Add(-time.Hour)) 27 | b, err := lat.Serialize() 28 | require.NoError(err) 29 | 30 | var newLat LastAccessTime 31 | require.NoError(newLat.Deserialize(b)) 32 | require.Equal(lat.Time.Unix(), newLat.Time.Unix()) 33 | } 34 | -------------------------------------------------------------------------------- /lib/store/metadata/metadata.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package metadata 15 | 16 | import "regexp" 17 | 18 | // Metadata defines types of matadata file. 19 | // All implementations of Metadata must register themselves. 20 | type Metadata interface { 21 | GetSuffix() string 22 | Movable() bool 23 | Serialize() ([]byte, error) 24 | Deserialize([]byte) error 25 | } 26 | 27 | var _factories = make(map[*regexp.Regexp]Factory) 28 | 29 | // Factory creates Metadata objects given suffix. 30 | type Factory interface { 31 | Create(suffix string) Metadata 32 | } 33 | 34 | // Register registers new Factory with corresponding suffix regexp. 35 | func Register(suffix *regexp.Regexp, factory Factory) { 36 | _factories[suffix] = factory 37 | } 38 | 39 | // CreateFromSuffix creates a Metadata obj based on suffix. 40 | // This is not a very efficient method; It's mostly used during reload. 41 | func CreateFromSuffix(suffix string) Metadata { 42 | for re, factory := range _factories { 43 | if re.MatchString(suffix) { 44 | return factory.Create(suffix) 45 | } 46 | } 47 | return nil 48 | } 49 | -------------------------------------------------------------------------------- /lib/store/metadata/persist_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package metadata 15 | 16 | import ( 17 | "strconv" 18 | "testing" 19 | 20 | "github.com/stretchr/testify/require" 21 | ) 22 | 23 | func TestPersistMetadataSerialization(t *testing.T) { 24 | for _, v := range []bool{true, false} { 25 | t.Run(strconv.FormatBool(v), func(t *testing.T) { 26 | require := require.New(t) 27 | 28 | p := NewPersist(v) 29 | b, err := p.Serialize() 30 | require.NoError(err) 31 | 32 | var result Persist 33 | require.NoError(result.Deserialize(b)) 34 | require.Equal(p.Value, result.Value) 35 | }) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /lib/store/metadata/torrentmeta_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package metadata 15 | 16 | import ( 17 | "testing" 18 | 19 | "github.com/uber/kraken/core" 20 | 21 | "github.com/stretchr/testify/require" 22 | ) 23 | 24 | func TestTorrentMetaSerialization(t *testing.T) { 25 | require := require.New(t) 26 | 27 | tm := NewTorrentMeta(core.MetaInfoFixture()) 28 | b, err := tm.Serialize() 29 | require.NoError(err) 30 | 31 | var result TorrentMeta 32 | require.NoError(result.Deserialize(b)) 33 | require.Equal(tm.MetaInfo, result.MetaInfo) 34 | } 35 | -------------------------------------------------------------------------------- /lib/store/simple_store_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package store 15 | 16 | import ( 17 | "bytes" 18 | "io/ioutil" 19 | "testing" 20 | 21 | "github.com/uber/kraken/core" 22 | 23 | "github.com/stretchr/testify/require" 24 | ) 25 | 26 | func TestSimpleStoreCreateCacheFile(t *testing.T) { 27 | require := require.New(t) 28 | 29 | s, cleanup := SimpleStoreFixture() 30 | defer cleanup() 31 | 32 | tag := core.TagFixture() 33 | d := core.DigestFixture().String() 34 | 35 | require.NoError(s.CreateCacheFile(tag, bytes.NewBufferString(d))) 36 | 37 | f, err := s.GetCacheFileReader(tag) 38 | require.NoError(err) 39 | result, err := ioutil.ReadAll(f) 40 | require.NoError(err) 41 | require.Equal(d, string(result)) 42 | } 43 | -------------------------------------------------------------------------------- /lib/torrent/networkevent/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package networkevent 15 | 16 | // Config defines network event configuration. 17 | type Config struct { 18 | LogPath string `yaml:"log_path"` 19 | Enabled bool `yaml:"enabled"` 20 | } 21 | -------------------------------------------------------------------------------- /lib/torrent/networkevent/testing.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package networkevent 15 | 16 | import "sync" 17 | 18 | // TestProducer records all produced events. 19 | type TestProducer struct { 20 | sync.Mutex 21 | events []*Event 22 | } 23 | 24 | // NewTestProducer returns a new TestProducer. 25 | func NewTestProducer() *TestProducer { 26 | return &TestProducer{} 27 | } 28 | 29 | // Produce records e. 30 | func (p *TestProducer) Produce(e *Event) { 31 | p.Lock() 32 | defer p.Unlock() 33 | 34 | p.events = append(p.events, e) 35 | } 36 | 37 | // Close noops. 38 | func (p *TestProducer) Close() error { return nil } 39 | 40 | // Events returns all currently recorded events. 41 | func (p *TestProducer) Events() []*Event { 42 | p.Lock() 43 | defer p.Unlock() 44 | 45 | res := make([]*Event, len(p.events)) 46 | copy(res, p.events) 47 | return res 48 | } 49 | -------------------------------------------------------------------------------- /lib/torrent/scheduler/conn/conn_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package conn 15 | 16 | import ( 17 | "sync" 18 | "testing" 19 | 20 | "github.com/stretchr/testify/require" 21 | ) 22 | 23 | func TestConnClose(t *testing.T) { 24 | require := require.New(t) 25 | 26 | c, cleanup := Fixture() 27 | defer cleanup() 28 | 29 | require.False(c.IsClosed()) 30 | 31 | var wg sync.WaitGroup 32 | for i := 0; i < 1000; i++ { 33 | wg.Add(1) 34 | go func() { 35 | defer wg.Done() 36 | c.Close() 37 | }() 38 | } 39 | wg.Wait() 40 | 41 | require.True(c.IsClosed()) 42 | } 43 | -------------------------------------------------------------------------------- /lib/torrent/scheduler/conn/fake_peer_test.go: -------------------------------------------------------------------------------- 1 | package conn 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | "github.com/uber/kraken/lib/torrent/storage" 9 | ) 10 | 11 | func TestFakePeer(t *testing.T) { 12 | require := require.New(t) 13 | 14 | p, err := NewFakePeer() 15 | require.NoError(err) 16 | defer p.Close() 17 | 18 | h := HandshakerFixture(ConfigFixture()) 19 | 20 | info := storage.TorrentInfoFixture(32, 4) 21 | 22 | res, err := h.Initialize(p.PeerID(), p.Addr(), info, nil, "noexist") 23 | require.NoError(err) 24 | 25 | require.Equal(p.PeerID(), res.Conn.PeerID()) 26 | require.Equal(info.InfoHash(), res.Conn.InfoHash()) 27 | require.Equal(info.Bitfield().Len(), res.Bitfield.Len()) 28 | require.False(res.Bitfield.Any()) 29 | } 30 | -------------------------------------------------------------------------------- /lib/torrent/scheduler/dispatch/piecerequest/policy.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package piecerequest 15 | 16 | import ( 17 | "github.com/uber/kraken/utils/syncutil" 18 | 19 | "github.com/willf/bitset" 20 | ) 21 | 22 | // pieceSelectionPolicy defines a policy for determining which pieces to request 23 | // given a set of candidates and relevant stats about them. 24 | // If 'valid' is not thread-safe, caller must handle locking. 25 | type pieceSelectionPolicy interface { 26 | selectPieces( 27 | limit int, 28 | valid func(int) bool, // whether the given piece is a valid selection or not 29 | candidates *bitset.BitSet, 30 | numPeersByPiece syncutil.Counters) ([]int, error) 31 | } 32 | -------------------------------------------------------------------------------- /lib/torrent/storage/fixtures.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package storage 15 | 16 | import ( 17 | "github.com/uber/kraken/core" 18 | "github.com/uber/kraken/utils/bitsetutil" 19 | "github.com/uber/kraken/utils/randutil" 20 | ) 21 | 22 | // TorrentInfoFixture returns a randomly generated TorrentInfo for testing purposes. 23 | func TorrentInfoFixture(size, pieceLength uint64) *TorrentInfo { 24 | mi := core.SizedBlobFixture(size, pieceLength).MetaInfo 25 | bitfield := bitsetutil.FromBools(randutil.Bools(mi.NumPieces())...) 26 | return NewTorrentInfo(mi, bitfield) 27 | } 28 | -------------------------------------------------------------------------------- /lib/torrent/storage/piecereader/buffer.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package piecereader 15 | 16 | import "bytes" 17 | 18 | // Buffer is a storage.PieceReader which reads a piece from an in-memory buffer. 19 | type Buffer struct { 20 | reader *bytes.Reader 21 | } 22 | 23 | // NewBuffer returns a new Buffer for b. 24 | func NewBuffer(b []byte) *Buffer { 25 | return &Buffer{bytes.NewReader(b)} 26 | } 27 | 28 | // Read reads a piece into p. 29 | func (b *Buffer) Read(p []byte) (int, error) { 30 | return b.reader.Read(p) 31 | } 32 | 33 | // Close noops. 34 | func (b *Buffer) Close() error { 35 | return nil 36 | } 37 | 38 | // Length returns the length of the piece. 39 | func (b *Buffer) Length() int { 40 | return b.reader.Len() 41 | } 42 | -------------------------------------------------------------------------------- /localdb/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package localdb 15 | 16 | // Config defines database configuration. 17 | type Config struct { 18 | Source string `yaml:"source"` 19 | } 20 | -------------------------------------------------------------------------------- /localdb/fixtures.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package localdb 15 | 16 | import ( 17 | "io/ioutil" 18 | "os" 19 | "path/filepath" 20 | 21 | "github.com/uber/kraken/utils/testutil" 22 | 23 | "github.com/jmoiron/sqlx" 24 | ) 25 | 26 | // Fixture returns a temporary test database for testing. 27 | func Fixture() (*sqlx.DB, func()) { 28 | var cleanup testutil.Cleanup 29 | defer cleanup.Recover() 30 | 31 | tmpdir, err := ioutil.TempDir(".", "test-db-") 32 | if err != nil { 33 | panic(err) 34 | } 35 | cleanup.Add(func() { os.RemoveAll(tmpdir) }) 36 | 37 | source := filepath.Join(tmpdir, "test.db") 38 | 39 | db, err := New(Config{Source: source}) 40 | if err != nil { 41 | panic(err) 42 | } 43 | 44 | return db, cleanup.Run 45 | } 46 | -------------------------------------------------------------------------------- /localdb/migrations/00001_tagreplication_init.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package migrations 15 | 16 | import ( 17 | "database/sql" 18 | 19 | "github.com/pressly/goose" 20 | ) 21 | 22 | func init() { 23 | goose.AddMigration(up00001, down00001) 24 | } 25 | 26 | func up00001(tx *sql.Tx) error { 27 | _, err := tx.Exec( 28 | `CREATE TABLE IF NOT EXISTS replicate_tag_task ( 29 | tag text NOT NULL, 30 | digest blob NOT NULL, 31 | dependencies blob NOT NULL, 32 | destination text NOT NULL, 33 | created_at timestamp DEFAULT CURRENT_TIMESTAMP, 34 | last_attempt timestamp NOT NULL, 35 | status text NOT NULL, 36 | failures integer NOT NULL, 37 | delay integer NOT NULL, 38 | PRIMARY KEY(tag, destination) 39 | );`) 40 | return err 41 | } 42 | 43 | func down00001(tx *sql.Tx) error { 44 | _, err := tx.Exec(`DROP TABLE replicate_tag_task;`) 45 | return err 46 | } 47 | -------------------------------------------------------------------------------- /localdb/migrations/00002_writeback_init.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package migrations 15 | 16 | import ( 17 | "database/sql" 18 | 19 | "github.com/pressly/goose" 20 | ) 21 | 22 | func init() { 23 | goose.AddMigration(up00002, down00002) 24 | } 25 | 26 | func up00002(tx *sql.Tx) error { 27 | _, err := tx.Exec(` 28 | CREATE TABLE IF NOT EXISTS writeback_task ( 29 | namespace text NOT NULL, 30 | name text NOT NULL, 31 | created_at timestamp DEFAULT CURRENT_TIMESTAMP, 32 | last_attempt timestamp NOT NULL, 33 | status text NOT NULL, 34 | failures integer NOT NULL, 35 | delay integer NOT NULL, 36 | PRIMARY KEY(namespace, name) 37 | ); 38 | `) 39 | return err 40 | } 41 | 42 | func down00002(tx *sql.Tx) error { 43 | _, err := tx.Exec(`DROP TABLE writeback_task;`) 44 | return err 45 | } 46 | -------------------------------------------------------------------------------- /metrics/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package metrics 15 | 16 | // Config defines metrics configuration. 17 | type Config struct { 18 | Backend string `yaml:"backend"` 19 | Statsd StatsdConfig `yaml:"statsd"` 20 | M3 M3Config `yaml:"m3"` 21 | } 22 | 23 | // StatsdConfig defines statsd configuration. 24 | type StatsdConfig struct { 25 | HostPort string `yaml:"host_port"` 26 | Prefix string `yaml:"prefix"` 27 | } 28 | 29 | // M3Config defines m3 configuration. 30 | type M3Config struct { 31 | HostPort string `yaml:"host_port"` 32 | Service string `yaml:"service"` 33 | Env string `yaml:"env"` 34 | } 35 | -------------------------------------------------------------------------------- /metrics/m3.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package metrics 15 | 16 | import ( 17 | "fmt" 18 | "io" 19 | "time" 20 | 21 | "github.com/uber-go/tally" 22 | "github.com/uber-go/tally/m3" 23 | ) 24 | 25 | func newM3Scope(config Config, cluster string) (tally.Scope, io.Closer, error) { 26 | if cluster == "" { 27 | return nil, nil, fmt.Errorf("cluster required for m3") 28 | } 29 | 30 | if config.M3.Service == "" { 31 | return nil, nil, fmt.Errorf("service required for m3") 32 | } 33 | 34 | // HostPort is required for m3 metrics but tally/m3 does not fail when HostPort is empty. 35 | if config.M3.HostPort == "" { 36 | return nil, nil, fmt.Errorf("host_port required for m3") 37 | } 38 | 39 | m3Config := m3.Configuration{ 40 | HostPort: config.M3.HostPort, 41 | Service: config.M3.Service, 42 | Env: cluster, 43 | } 44 | r, err := m3Config.NewReporter() 45 | if err != nil { 46 | return nil, nil, err 47 | } 48 | s, c := tally.NewRootScope(tally.ScopeOptions{ 49 | CachedReporter: r, 50 | }, time.Second) 51 | return s, c, nil 52 | } 53 | -------------------------------------------------------------------------------- /metrics/statsd.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package metrics 15 | 16 | import ( 17 | "io" 18 | "time" 19 | 20 | "github.com/cactus/go-statsd-client/statsd" 21 | "github.com/uber-go/tally" 22 | tallystatsd "github.com/uber-go/tally/statsd" 23 | ) 24 | 25 | const ( 26 | flushInterval = 100 * time.Millisecond 27 | flushBytes = 512 28 | sampleRate = 1.0 29 | ) 30 | 31 | func newStatsdScope(config Config, cluster string) (tally.Scope, io.Closer, error) { 32 | statter, err := statsd.NewBufferedClient( 33 | config.Statsd.HostPort, config.Statsd.Prefix, flushInterval, flushBytes) 34 | if err != nil { 35 | return nil, nil, err 36 | } 37 | r := tallystatsd.NewReporter(statter, tallystatsd.Options{ 38 | SampleRate: sampleRate, 39 | }) 40 | s, c := tally.NewRootScope(tally.ScopeOptions{ 41 | Reporter: r, 42 | }, time.Second) 43 | return s, c, nil 44 | } 45 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: Kraken 2 | nav: 3 | - Home: index.md 4 | - Architecture: ARCHITECTURE.md 5 | - Configuration: CONFIGURATIION.md 6 | - Endpoints: ENDPOINTS.md 7 | - 'Harbor Integration': INTEGRATEWITHHARBOR.md 8 | - Contributing: CONTRIBUTING.md 9 | - 'Style Guide': STYLEGUIDE.md 10 | - Roadmap: ROADMAP.md 11 | theme: readthedocs 12 | -------------------------------------------------------------------------------- /mocks/build-index/tagclient/provider.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/uber/kraken/build-index/tagclient (interfaces: Provider) 3 | 4 | // Package mocktagclient is a generated GoMock package. 5 | package mocktagclient 6 | 7 | import ( 8 | gomock "github.com/golang/mock/gomock" 9 | tagclient "github.com/uber/kraken/build-index/tagclient" 10 | reflect "reflect" 11 | ) 12 | 13 | // MockProvider is a mock of Provider interface 14 | type MockProvider struct { 15 | ctrl *gomock.Controller 16 | recorder *MockProviderMockRecorder 17 | } 18 | 19 | // MockProviderMockRecorder is the mock recorder for MockProvider 20 | type MockProviderMockRecorder struct { 21 | mock *MockProvider 22 | } 23 | 24 | // NewMockProvider creates a new mock instance 25 | func NewMockProvider(ctrl *gomock.Controller) *MockProvider { 26 | mock := &MockProvider{ctrl: ctrl} 27 | mock.recorder = &MockProviderMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (m *MockProvider) EXPECT() *MockProviderMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // Provide mocks base method 37 | func (m *MockProvider) Provide(arg0 string) tagclient.Client { 38 | m.ctrl.T.Helper() 39 | ret := m.ctrl.Call(m, "Provide", arg0) 40 | ret0, _ := ret[0].(tagclient.Client) 41 | return ret0 42 | } 43 | 44 | // Provide indicates an expected call of Provide 45 | func (mr *MockProviderMockRecorder) Provide(arg0 interface{}) *gomock.Call { 46 | mr.mock.ctrl.T.Helper() 47 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Provide", reflect.TypeOf((*MockProvider)(nil).Provide), arg0) 48 | } 49 | -------------------------------------------------------------------------------- /mocks/lib/containerruntime/containerd/client.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/uber/kraken/lib/containerruntime/containerd (interfaces: Client) 3 | 4 | // Package mockcontainerd is a generated GoMock package. 5 | package mockcontainerd 6 | 7 | import ( 8 | context "context" 9 | gomock "github.com/golang/mock/gomock" 10 | reflect "reflect" 11 | ) 12 | 13 | // MockClient is a mock of Client interface 14 | type MockClient struct { 15 | ctrl *gomock.Controller 16 | recorder *MockClientMockRecorder 17 | } 18 | 19 | // MockClientMockRecorder is the mock recorder for MockClient 20 | type MockClientMockRecorder struct { 21 | mock *MockClient 22 | } 23 | 24 | // NewMockClient creates a new mock instance 25 | func NewMockClient(ctrl *gomock.Controller) *MockClient { 26 | mock := &MockClient{ctrl: ctrl} 27 | mock.recorder = &MockClientMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (m *MockClient) EXPECT() *MockClientMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // PullImage mocks base method 37 | func (m *MockClient) PullImage(arg0 context.Context, arg1, arg2, arg3 string) error { 38 | m.ctrl.T.Helper() 39 | ret := m.ctrl.Call(m, "PullImage", arg0, arg1, arg2, arg3) 40 | ret0, _ := ret[0].(error) 41 | return ret0 42 | } 43 | 44 | // PullImage indicates an expected call of PullImage 45 | func (mr *MockClientMockRecorder) PullImage(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { 46 | mr.mock.ctrl.T.Helper() 47 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PullImage", reflect.TypeOf((*MockClient)(nil).PullImage), arg0, arg1, arg2, arg3) 48 | } 49 | -------------------------------------------------------------------------------- /mocks/lib/hashring/watcher.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/uber/kraken/lib/hashring (interfaces: Watcher) 3 | 4 | // Package mockhashring is a generated GoMock package. 5 | package mockhashring 6 | 7 | import ( 8 | gomock "github.com/golang/mock/gomock" 9 | stringset "github.com/uber/kraken/utils/stringset" 10 | reflect "reflect" 11 | ) 12 | 13 | // MockWatcher is a mock of Watcher interface 14 | type MockWatcher struct { 15 | ctrl *gomock.Controller 16 | recorder *MockWatcherMockRecorder 17 | } 18 | 19 | // MockWatcherMockRecorder is the mock recorder for MockWatcher 20 | type MockWatcherMockRecorder struct { 21 | mock *MockWatcher 22 | } 23 | 24 | // NewMockWatcher creates a new mock instance 25 | func NewMockWatcher(ctrl *gomock.Controller) *MockWatcher { 26 | mock := &MockWatcher{ctrl: ctrl} 27 | mock.recorder = &MockWatcherMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (m *MockWatcher) EXPECT() *MockWatcherMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // Notify mocks base method 37 | func (m *MockWatcher) Notify(arg0 stringset.Set) { 38 | m.ctrl.T.Helper() 39 | m.ctrl.Call(m, "Notify", arg0) 40 | } 41 | 42 | // Notify indicates an expected call of Notify 43 | func (mr *MockWatcherMockRecorder) Notify(arg0 interface{}) *gomock.Call { 44 | mr.mock.ctrl.T.Helper() 45 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Notify", reflect.TypeOf((*MockWatcher)(nil).Notify), arg0) 46 | } 47 | -------------------------------------------------------------------------------- /mocks/lib/healthcheck/checker.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/uber/kraken/lib/healthcheck (interfaces: Checker) 3 | 4 | // Package mockhealthcheck is a generated GoMock package. 5 | package mockhealthcheck 6 | 7 | import ( 8 | context "context" 9 | gomock "github.com/golang/mock/gomock" 10 | reflect "reflect" 11 | ) 12 | 13 | // MockChecker is a mock of Checker interface 14 | type MockChecker struct { 15 | ctrl *gomock.Controller 16 | recorder *MockCheckerMockRecorder 17 | } 18 | 19 | // MockCheckerMockRecorder is the mock recorder for MockChecker 20 | type MockCheckerMockRecorder struct { 21 | mock *MockChecker 22 | } 23 | 24 | // NewMockChecker creates a new mock instance 25 | func NewMockChecker(ctrl *gomock.Controller) *MockChecker { 26 | mock := &MockChecker{ctrl: ctrl} 27 | mock.recorder = &MockCheckerMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (m *MockChecker) EXPECT() *MockCheckerMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // Check mocks base method 37 | func (m *MockChecker) Check(arg0 context.Context, arg1 string) error { 38 | m.ctrl.T.Helper() 39 | ret := m.ctrl.Call(m, "Check", arg0, arg1) 40 | ret0, _ := ret[0].(error) 41 | return ret0 42 | } 43 | 44 | // Check indicates an expected call of Check 45 | func (mr *MockCheckerMockRecorder) Check(arg0, arg1 interface{}) *gomock.Call { 46 | mr.mock.ctrl.T.Helper() 47 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Check", reflect.TypeOf((*MockChecker)(nil).Check), arg0, arg1) 48 | } 49 | -------------------------------------------------------------------------------- /mocks/lib/healthcheck/filter.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/uber/kraken/lib/healthcheck (interfaces: Filter) 3 | 4 | // Package mockhealthcheck is a generated GoMock package. 5 | package mockhealthcheck 6 | 7 | import ( 8 | gomock "github.com/golang/mock/gomock" 9 | stringset "github.com/uber/kraken/utils/stringset" 10 | reflect "reflect" 11 | ) 12 | 13 | // MockFilter is a mock of Filter interface 14 | type MockFilter struct { 15 | ctrl *gomock.Controller 16 | recorder *MockFilterMockRecorder 17 | } 18 | 19 | // MockFilterMockRecorder is the mock recorder for MockFilter 20 | type MockFilterMockRecorder struct { 21 | mock *MockFilter 22 | } 23 | 24 | // NewMockFilter creates a new mock instance 25 | func NewMockFilter(ctrl *gomock.Controller) *MockFilter { 26 | mock := &MockFilter{ctrl: ctrl} 27 | mock.recorder = &MockFilterMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (m *MockFilter) EXPECT() *MockFilterMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // Run mocks base method 37 | func (m *MockFilter) Run(arg0 stringset.Set) stringset.Set { 38 | m.ctrl.T.Helper() 39 | ret := m.ctrl.Call(m, "Run", arg0) 40 | ret0, _ := ret[0].(stringset.Set) 41 | return ret0 42 | } 43 | 44 | // Run indicates an expected call of Run 45 | func (mr *MockFilterMockRecorder) Run(arg0 interface{}) *gomock.Call { 46 | mr.mock.ctrl.T.Helper() 47 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockFilter)(nil).Run), arg0) 48 | } 49 | -------------------------------------------------------------------------------- /mocks/lib/hostlist/list.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/uber/kraken/lib/hostlist (interfaces: List) 3 | 4 | // Package mockhostlist is a generated GoMock package. 5 | package mockhostlist 6 | 7 | import ( 8 | gomock "github.com/golang/mock/gomock" 9 | stringset "github.com/uber/kraken/utils/stringset" 10 | reflect "reflect" 11 | ) 12 | 13 | // MockList is a mock of List interface 14 | type MockList struct { 15 | ctrl *gomock.Controller 16 | recorder *MockListMockRecorder 17 | } 18 | 19 | // MockListMockRecorder is the mock recorder for MockList 20 | type MockListMockRecorder struct { 21 | mock *MockList 22 | } 23 | 24 | // NewMockList creates a new mock instance 25 | func NewMockList(ctrl *gomock.Controller) *MockList { 26 | mock := &MockList{ctrl: ctrl} 27 | mock.recorder = &MockListMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (m *MockList) EXPECT() *MockListMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // Resolve mocks base method 37 | func (m *MockList) Resolve() stringset.Set { 38 | m.ctrl.T.Helper() 39 | ret := m.ctrl.Call(m, "Resolve") 40 | ret0, _ := ret[0].(stringset.Set) 41 | return ret0 42 | } 43 | 44 | // Resolve indicates an expected call of Resolve 45 | func (mr *MockListMockRecorder) Resolve() *gomock.Call { 46 | mr.mock.ctrl.T.Helper() 47 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Resolve", reflect.TypeOf((*MockList)(nil).Resolve)) 48 | } 49 | -------------------------------------------------------------------------------- /mocks/origin/blobclient/provider.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/uber/kraken/origin/blobclient (interfaces: Provider) 3 | 4 | // Package mockblobclient is a generated GoMock package. 5 | package mockblobclient 6 | 7 | import ( 8 | gomock "github.com/golang/mock/gomock" 9 | blobclient "github.com/uber/kraken/origin/blobclient" 10 | reflect "reflect" 11 | ) 12 | 13 | // MockProvider is a mock of Provider interface 14 | type MockProvider struct { 15 | ctrl *gomock.Controller 16 | recorder *MockProviderMockRecorder 17 | } 18 | 19 | // MockProviderMockRecorder is the mock recorder for MockProvider 20 | type MockProviderMockRecorder struct { 21 | mock *MockProvider 22 | } 23 | 24 | // NewMockProvider creates a new mock instance 25 | func NewMockProvider(ctrl *gomock.Controller) *MockProvider { 26 | mock := &MockProvider{ctrl: ctrl} 27 | mock.recorder = &MockProviderMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (m *MockProvider) EXPECT() *MockProviderMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // Provide mocks base method 37 | func (m *MockProvider) Provide(arg0 string) blobclient.Client { 38 | m.ctrl.T.Helper() 39 | ret := m.ctrl.Call(m, "Provide", arg0) 40 | ret0, _ := ret[0].(blobclient.Client) 41 | return ret0 42 | } 43 | 44 | // Provide indicates an expected call of Provide 45 | func (mr *MockProviderMockRecorder) Provide(arg0 interface{}) *gomock.Call { 46 | mr.mock.ctrl.T.Helper() 47 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Provide", reflect.TypeOf((*MockProvider)(nil).Provide), arg0) 48 | } 49 | -------------------------------------------------------------------------------- /mocks/tracker/metainfoclient/client.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/uber/kraken/tracker/metainfoclient (interfaces: Client) 3 | 4 | // Package mockmetainfoclient is a generated GoMock package. 5 | package mockmetainfoclient 6 | 7 | import ( 8 | gomock "github.com/golang/mock/gomock" 9 | core "github.com/uber/kraken/core" 10 | reflect "reflect" 11 | ) 12 | 13 | // MockClient is a mock of Client interface 14 | type MockClient struct { 15 | ctrl *gomock.Controller 16 | recorder *MockClientMockRecorder 17 | } 18 | 19 | // MockClientMockRecorder is the mock recorder for MockClient 20 | type MockClientMockRecorder struct { 21 | mock *MockClient 22 | } 23 | 24 | // NewMockClient creates a new mock instance 25 | func NewMockClient(ctrl *gomock.Controller) *MockClient { 26 | mock := &MockClient{ctrl: ctrl} 27 | mock.recorder = &MockClientMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (m *MockClient) EXPECT() *MockClientMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // Download mocks base method 37 | func (m *MockClient) Download(arg0 string, arg1 core.Digest) (*core.MetaInfo, error) { 38 | m.ctrl.T.Helper() 39 | ret := m.ctrl.Call(m, "Download", arg0, arg1) 40 | ret0, _ := ret[0].(*core.MetaInfo) 41 | ret1, _ := ret[1].(error) 42 | return ret0, ret1 43 | } 44 | 45 | // Download indicates an expected call of Download 46 | func (mr *MockClientMockRecorder) Download(arg0, arg1 interface{}) *gomock.Call { 47 | mr.mock.ctrl.T.Helper() 48 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Download", reflect.TypeOf((*MockClient)(nil).Download), arg0, arg1) 49 | } 50 | -------------------------------------------------------------------------------- /mocks/tracker/originstore/store.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/uber/kraken/tracker/originstore (interfaces: Store) 3 | 4 | // Package mockoriginstore is a generated GoMock package. 5 | package mockoriginstore 6 | 7 | import ( 8 | gomock "github.com/golang/mock/gomock" 9 | core "github.com/uber/kraken/core" 10 | reflect "reflect" 11 | ) 12 | 13 | // MockStore is a mock of Store interface 14 | type MockStore struct { 15 | ctrl *gomock.Controller 16 | recorder *MockStoreMockRecorder 17 | } 18 | 19 | // MockStoreMockRecorder is the mock recorder for MockStore 20 | type MockStoreMockRecorder struct { 21 | mock *MockStore 22 | } 23 | 24 | // NewMockStore creates a new mock instance 25 | func NewMockStore(ctrl *gomock.Controller) *MockStore { 26 | mock := &MockStore{ctrl: ctrl} 27 | mock.recorder = &MockStoreMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (m *MockStore) EXPECT() *MockStoreMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // GetOrigins mocks base method 37 | func (m *MockStore) GetOrigins(arg0 core.Digest) ([]*core.PeerInfo, error) { 38 | m.ctrl.T.Helper() 39 | ret := m.ctrl.Call(m, "GetOrigins", arg0) 40 | ret0, _ := ret[0].([]*core.PeerInfo) 41 | ret1, _ := ret[1].(error) 42 | return ret0, ret1 43 | } 44 | 45 | // GetOrigins indicates an expected call of GetOrigins 46 | func (mr *MockStoreMockRecorder) GetOrigins(arg0 interface{}) *gomock.Call { 47 | mr.mock.ctrl.T.Helper() 48 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetOrigins", reflect.TypeOf((*MockStore)(nil).GetOrigins), arg0) 49 | } 50 | -------------------------------------------------------------------------------- /mocks/utils/dedup/intervaltask.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/uber/kraken/utils/dedup (interfaces: IntervalTask) 3 | 4 | // Package mockdedup is a generated GoMock package. 5 | package mockdedup 6 | 7 | import ( 8 | gomock "github.com/golang/mock/gomock" 9 | reflect "reflect" 10 | ) 11 | 12 | // MockIntervalTask is a mock of IntervalTask interface 13 | type MockIntervalTask struct { 14 | ctrl *gomock.Controller 15 | recorder *MockIntervalTaskMockRecorder 16 | } 17 | 18 | // MockIntervalTaskMockRecorder is the mock recorder for MockIntervalTask 19 | type MockIntervalTaskMockRecorder struct { 20 | mock *MockIntervalTask 21 | } 22 | 23 | // NewMockIntervalTask creates a new mock instance 24 | func NewMockIntervalTask(ctrl *gomock.Controller) *MockIntervalTask { 25 | mock := &MockIntervalTask{ctrl: ctrl} 26 | mock.recorder = &MockIntervalTaskMockRecorder{mock} 27 | return mock 28 | } 29 | 30 | // EXPECT returns an object that allows the caller to indicate expected use 31 | func (m *MockIntervalTask) EXPECT() *MockIntervalTaskMockRecorder { 32 | return m.recorder 33 | } 34 | 35 | // Run mocks base method 36 | func (m *MockIntervalTask) Run() { 37 | m.ctrl.T.Helper() 38 | m.ctrl.Call(m, "Run") 39 | } 40 | 41 | // Run indicates an expected call of Run 42 | func (mr *MockIntervalTaskMockRecorder) Run() *gomock.Call { 43 | mr.mock.ctrl.T.Helper() 44 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockIntervalTask)(nil).Run)) 45 | } 46 | -------------------------------------------------------------------------------- /mocks/utils/dedup/taskrunner.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/uber/kraken/utils/dedup (interfaces: TaskRunner) 3 | 4 | // Package mockdedup is a generated GoMock package. 5 | package mockdedup 6 | 7 | import ( 8 | gomock "github.com/golang/mock/gomock" 9 | reflect "reflect" 10 | time "time" 11 | ) 12 | 13 | // MockTaskRunner is a mock of TaskRunner interface 14 | type MockTaskRunner struct { 15 | ctrl *gomock.Controller 16 | recorder *MockTaskRunnerMockRecorder 17 | } 18 | 19 | // MockTaskRunnerMockRecorder is the mock recorder for MockTaskRunner 20 | type MockTaskRunnerMockRecorder struct { 21 | mock *MockTaskRunner 22 | } 23 | 24 | // NewMockTaskRunner creates a new mock instance 25 | func NewMockTaskRunner(ctrl *gomock.Controller) *MockTaskRunner { 26 | mock := &MockTaskRunner{ctrl: ctrl} 27 | mock.recorder = &MockTaskRunnerMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (m *MockTaskRunner) EXPECT() *MockTaskRunnerMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // Run mocks base method 37 | func (m *MockTaskRunner) Run(arg0 interface{}) (interface{}, time.Duration) { 38 | m.ctrl.T.Helper() 39 | ret := m.ctrl.Call(m, "Run", arg0) 40 | ret0, _ := ret[0].(interface{}) 41 | ret1, _ := ret[1].(time.Duration) 42 | return ret0, ret1 43 | } 44 | 45 | // Run indicates an expected call of Run 46 | func (mr *MockTaskRunnerMockRecorder) Run(arg0 interface{}) *gomock.Call { 47 | mr.mock.ctrl.T.Helper() 48 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockTaskRunner)(nil).Run), arg0) 49 | } 50 | -------------------------------------------------------------------------------- /nginx/config/agent.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package config 15 | 16 | // AgentTemplate is the default agent nginx tmpl. 17 | var AgentTemplate = ` 18 | upstream registry-backend { 19 | server {{.registry_server}}; 20 | {{if ne .registry_backup ""}} server {{.registry_backup}} backup; {{end}} 21 | } 22 | 23 | upstream agent-server { 24 | server {{.agent_server}}; 25 | } 26 | 27 | server { 28 | listen {{.port}}; 29 | 30 | {{range .allowed_cidrs}} 31 | allow {{.}}; 32 | {{end}} 33 | deny all; 34 | 35 | {{.client_verification}} 36 | 37 | access_log {{.access_log_path}}; 38 | error_log {{.error_log_path}}; 39 | 40 | gzip on; 41 | gzip_types text/plain test/csv application/json; 42 | 43 | location ~ ^/(health|readiness)$ { 44 | proxy_pass http://agent-server; 45 | } 46 | 47 | location / { 48 | proxy_pass http://registry-backend; 49 | proxy_next_upstream error timeout http_404 http_500; 50 | } 51 | } 52 | ` 53 | -------------------------------------------------------------------------------- /nginx/config/origin.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package config 15 | 16 | // OriginTemplate is the default origin nginx tmpl. 17 | const OriginTemplate = ` 18 | server { 19 | listen {{.port}}; 20 | 21 | {{.client_verification}} 22 | 23 | client_max_body_size 10G; 24 | 25 | access_log {{.access_log_path}} json; 26 | error_log {{.error_log_path}}; 27 | 28 | gzip on; 29 | gzip_types text/plain test/csv application/json; 30 | 31 | location / { 32 | proxy_pass http://{{.server}}; 33 | } 34 | } 35 | ` 36 | -------------------------------------------------------------------------------- /nginx/config/tracker.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package config 15 | 16 | // TrackerTemplate is the default tracker nginx tmpl. 17 | const TrackerTemplate = ` 18 | proxy_cache_path {{.cache_dir}}/metainfo levels=1:2 keys_zone=metainfo:10m max_size=256g; 19 | 20 | upstream tracker { 21 | server {{.server}}; 22 | } 23 | 24 | server { 25 | listen {{.port}}; 26 | 27 | {{.client_verification}} 28 | 29 | access_log {{.access_log_path}}; 30 | error_log {{.error_log_path}}; 31 | 32 | location / { 33 | proxy_pass http://tracker; 34 | } 35 | 36 | location ~* ^/namespace/.*/blobs/.*/metainfo$ { 37 | proxy_pass http://tracker; 38 | 39 | proxy_cache metainfo; 40 | proxy_cache_methods GET; 41 | proxy_cache_valid 200 5m; 42 | proxy_cache_valid any 1s; 43 | proxy_cache_lock on; 44 | } 45 | } 46 | ` 47 | -------------------------------------------------------------------------------- /origin/blobclient/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package blobclient 15 | 16 | import "errors" 17 | 18 | // ErrBlobNotFound is returned when a blob is not found on origin. 19 | var ErrBlobNotFound = errors.New("blob not found") 20 | -------------------------------------------------------------------------------- /origin/blobserver/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package blobserver 15 | 16 | import ( 17 | "time" 18 | 19 | "github.com/uber/kraken/utils/listener" 20 | ) 21 | 22 | // Config defines the configuration used by Origin cluster for hashing blob digests. 23 | type Config struct { 24 | Listener listener.Config `yaml:"listener"` 25 | DuplicateWriteBackStagger time.Duration `yaml:"duplicate_write_back_stagger"` 26 | } 27 | 28 | func (c Config) applyDefaults() Config { 29 | if c.DuplicateWriteBackStagger == 0 { 30 | c.DuplicateWriteBackStagger = 30 * time.Minute 31 | } 32 | return c 33 | } 34 | -------------------------------------------------------------------------------- /origin/blobserver/utils_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package blobserver 15 | 16 | import ( 17 | "net/http" 18 | "testing" 19 | 20 | "github.com/uber/kraken/utils/handler" 21 | "github.com/stretchr/testify/require" 22 | ) 23 | 24 | func TestParseContentRangeHeaderBadRequests(t *testing.T) { 25 | tests := []struct { 26 | description string 27 | value string 28 | }{ 29 | {"empty value", ""}, 30 | {"invalid format", "blah"}, 31 | {"invalid start", "blah-5"}, 32 | {"invalid end", "5-blah"}, 33 | } 34 | for _, test := range tests { 35 | t.Run(test.description, func(t *testing.T) { 36 | require := require.New(t) 37 | 38 | h := http.Header{} 39 | h.Add("Content-Range", test.value) 40 | start, end, err := parseContentRange(h) 41 | require.Error(err) 42 | require.Equal(http.StatusBadRequest, err.(*handler.Error).GetStatus()) 43 | require.Equal(int64(0), start) 44 | require.Equal(int64(0), end) 45 | }) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /origin/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package main 15 | 16 | import ( 17 | "github.com/uber/kraken/origin/cmd" 18 | 19 | // Import all backend client packages to register them with backend manager. 20 | _ "github.com/uber/kraken/lib/backend/hdfsbackend" 21 | _ "github.com/uber/kraken/lib/backend/httpbackend" 22 | _ "github.com/uber/kraken/lib/backend/registrybackend" 23 | _ "github.com/uber/kraken/lib/backend/s3backend" 24 | _ "github.com/uber/kraken/lib/backend/gcsbackend" 25 | _ "github.com/uber/kraken/lib/backend/testfs" 26 | ) 27 | 28 | func main() { 29 | cmd.Run(cmd.ParseFlags()) 30 | } 31 | -------------------------------------------------------------------------------- /proxy/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package main 15 | 16 | import "github.com/uber/kraken/proxy/cmd" 17 | 18 | func main() { 19 | cmd.Run(cmd.ParseFlags()) 20 | } 21 | -------------------------------------------------------------------------------- /proxy/proxyserver/config.go: -------------------------------------------------------------------------------- 1 | package proxyserver 2 | 3 | import "github.com/uber/kraken/utils/listener" 4 | 5 | type Config struct { 6 | Listener listener.Config `yaml:"listener"` 7 | } 8 | -------------------------------------------------------------------------------- /proxy/proxyserver/registry_events.go: -------------------------------------------------------------------------------- 1 | package proxyserver 2 | 3 | import "time" 4 | 5 | // Notification holds all events. refer to https://docs.docker.com/registry/notifications/. 6 | type Notification struct { 7 | Events []Event 8 | } 9 | 10 | // Event holds the details of a event. 11 | type Event struct { 12 | ID string `json:"Id"` 13 | TimeStamp time.Time 14 | Action string 15 | Target *Target 16 | } 17 | 18 | // Target holds information about the target of a event. 19 | type Target struct { 20 | MediaType string 21 | Digest string 22 | Repository string 23 | URL string `json:"Url"` 24 | Tag string 25 | } 26 | -------------------------------------------------------------------------------- /proxy/registryoverride/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package registryoverride 15 | 16 | import "github.com/uber/kraken/utils/listener" 17 | 18 | // Config defines Server configuration. 19 | type Config struct { 20 | Listener listener.Config `yaml:"listener"` 21 | } 22 | -------------------------------------------------------------------------------- /requirements-docs.txt: -------------------------------------------------------------------------------- 1 | alabaster==0.7.12 2 | attrs==19.1.0 3 | Babel==2.9.1 4 | certifi==2019.6.16 5 | chardet==3.0.4 6 | docutils==0.15.2 7 | idna==2.8 8 | imagesize==1.1.0 9 | Jinja2==3.1.5 10 | MarkupSafe==1.1.1 11 | packaging==19.1 12 | Pygments==2.4.2 13 | pyparsing==2.4.2 14 | pytz==2019.2 15 | requests==2.22.0 16 | six==1.12.0 17 | snowballstemmer==1.9.0 18 | Sphinx==2.2.0 19 | sphinxcontrib-applehelp==1.0.1 20 | sphinxcontrib-devhelp==1.0.1 21 | sphinxcontrib-htmlhelp==1.0.2 22 | sphinxcontrib-jsmath==1.0.1 23 | sphinxcontrib-qthelp==1.0.2 24 | sphinxcontrib-serializinghtml==1.1.3 25 | urllib3==1.25.3 26 | -------------------------------------------------------------------------------- /requirements-tests.txt: -------------------------------------------------------------------------------- 1 | pytest==3.0.6 2 | pytest-timeout==1.2.0 3 | requests 4 | setuptools<45.0.0 5 | -------------------------------------------------------------------------------- /scripts/mkdocs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | cd $(dirname "${BASH_SOURCE[0]}")/.. 5 | 6 | # Setup virtualenv if not exists 7 | if [ ! -d env ]; then 8 | virtualenv env 9 | fi 10 | 11 | # Active virtualenv 12 | . env/bin/activate 13 | 14 | # Install pip requirements for the deployment script 15 | pip install -q -r requirements-docs.txt 16 | 17 | # Generate docs/index.md file automatically from README.md 18 | cat README.md | sed 's/(docs\//(/g' > docs/index.md 19 | 20 | # Run the mkdocs tool 21 | echo "cmd is $@" 22 | mkdocs "$@" 23 | 24 | # Exit virtualenv 25 | deactivate 26 | -------------------------------------------------------------------------------- /test/tls/ca/passphrase: -------------------------------------------------------------------------------- 1 | password -------------------------------------------------------------------------------- /test/tls/ca/server.srl: -------------------------------------------------------------------------------- 1 | D317EDB87B0B76BB 2 | -------------------------------------------------------------------------------- /test/tls/client/passphrase: -------------------------------------------------------------------------------- 1 | password -------------------------------------------------------------------------------- /tools/bin/testfs/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package main 15 | 16 | import ( 17 | "flag" 18 | "fmt" 19 | "net/http" 20 | 21 | "github.com/uber/kraken/lib/backend/testfs" 22 | "github.com/uber/kraken/utils/log" 23 | ) 24 | 25 | func main() { 26 | port := flag.Int("port", 0, "port which testfs server listens on") 27 | flag.Parse() 28 | 29 | if *port == 0 { 30 | log.Fatal("-port required") 31 | } 32 | 33 | server := testfs.NewServer() 34 | defer server.Cleanup() 35 | 36 | addr := fmt.Sprintf(":%d", *port) 37 | log.Infof("Starting testfs server on %s", addr) 38 | log.Fatal(http.ListenAndServe(addr, server.Handler())) 39 | } 40 | -------------------------------------------------------------------------------- /tools/bin/visualization/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package main 15 | 16 | import ( 17 | "fmt" 18 | "log" 19 | "net/http" 20 | 21 | "github.com/alecthomas/kingpin" 22 | ) 23 | 24 | func main() { 25 | eventFile := kingpin.Arg("events", "Network event file").Required().File() 26 | port := kingpin.Flag("port", "listening port").Default("3000").Int() 27 | kingpin.Parse() 28 | 29 | s := newServer(*eventFile) 30 | addr := fmt.Sprintf("localhost:%d", *port) 31 | log.Printf("Listening on %s ...", addr) 32 | log.Fatal(http.ListenAndServe(addr, s.handler())) 33 | } 34 | -------------------------------------------------------------------------------- /tools/bin/visualization/static/css/app.css: -------------------------------------------------------------------------------- 1 | .header { 2 | margin-left: 8px; 3 | margin-top: 8px; 4 | } 5 | 6 | .node { 7 | stroke: #999999; 8 | } 9 | 10 | #highlight { 11 | stroke: red; 12 | } 13 | -------------------------------------------------------------------------------- /tools/bin/visualization/static/html/app.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 |
9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /tools/lib/tlsutil.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package lib 15 | 16 | import ( 17 | "crypto/tls" 18 | "fmt" 19 | "io/ioutil" 20 | 21 | "github.com/uber/kraken/utils/httputil" 22 | 23 | "gopkg.in/yaml.v2" 24 | ) 25 | 26 | // ReadTLSFile reads config file in path and returns *tls.Config. 27 | // It returns nil when path is nil. 28 | func ReadTLSFile(path *string) (*tls.Config, error) { 29 | if path == nil { 30 | return nil, nil 31 | } 32 | data, err := ioutil.ReadFile(*path) 33 | if err != nil { 34 | return nil, fmt.Errorf("read tls config: %s", err) 35 | } 36 | var config httputil.TLSConfig 37 | if err := yaml.Unmarshal(data, &config); err != nil { 38 | return nil, fmt.Errorf("unmarshal tls config: %s", err) 39 | } 40 | tls, err := config.BuildClient() 41 | if err != nil { 42 | return nil, fmt.Errorf("build tls client: %s", err) 43 | } 44 | return tls, nil 45 | } 46 | -------------------------------------------------------------------------------- /tracker/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package main 15 | 16 | import "github.com/uber/kraken/tracker/cmd" 17 | 18 | func main() { 19 | cmd.Run(cmd.ParseFlags()) 20 | } 21 | -------------------------------------------------------------------------------- /tracker/originstore/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package originstore 15 | 16 | import "time" 17 | 18 | // Config defines Store configuration. 19 | type Config struct { 20 | LocationsTTL time.Duration `yaml:"locations_ttl"` 21 | LocationsErrorTTL time.Duration `yaml:"locations_error_ttl"` 22 | OriginContextTTL time.Duration `yaml:"origin_context_ttl"` 23 | OriginUnavailableTTL time.Duration `yaml:"origin_unavailable_ttl"` 24 | } 25 | 26 | func (c *Config) applyDefaults() { 27 | if c.LocationsTTL == 0 { 28 | c.LocationsTTL = 10 * time.Second 29 | } 30 | if c.LocationsErrorTTL == 0 { 31 | c.LocationsErrorTTL = time.Second 32 | } 33 | if c.OriginContextTTL == 0 { 34 | c.OriginContextTTL = 10 * time.Second 35 | } 36 | if c.OriginUnavailableTTL == 0 { 37 | c.OriginUnavailableTTL = time.Minute 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /tracker/originstore/fixtures.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package originstore 15 | 16 | import "github.com/uber/kraken/core" 17 | 18 | type noopStore struct{} 19 | 20 | // NewNoopStore returns a Store which never returns origins. Useful for testing. 21 | func NewNoopStore() Store { 22 | return noopStore{} 23 | } 24 | 25 | func (s noopStore) GetOrigins(core.Digest) ([]*core.PeerInfo, error) { 26 | return nil, nil 27 | } 28 | -------------------------------------------------------------------------------- /tracker/peerhandoutpolicy/completeness_policy.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package peerhandoutpolicy 15 | 16 | import "github.com/uber/kraken/core" 17 | 18 | const _completenessPolicy = "completeness" 19 | 20 | // completenessAssignmentPolicy assigns priorities based on download completeness. 21 | // Peers who've completed downloading are highest, then origins, then other peers. 22 | type completenessAssignmentPolicy struct{} 23 | 24 | func newCompletenessAssignmentPolicy() assignmentPolicy { 25 | return &completenessAssignmentPolicy{} 26 | } 27 | 28 | func (p *completenessAssignmentPolicy) assignPriority(peer *core.PeerInfo) (int, string) { 29 | if peer.Origin { 30 | return 1, "origin" 31 | } 32 | if peer.Complete { 33 | return 0, "peer_seeder" 34 | } 35 | return 2, "peer_incomplete" 36 | } 37 | -------------------------------------------------------------------------------- /tracker/peerhandoutpolicy/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package peerhandoutpolicy 15 | 16 | // Config defines configuration for the peer handout policy. 17 | type Config struct { 18 | Priority string `yaml:"priority"` 19 | } 20 | -------------------------------------------------------------------------------- /tracker/peerhandoutpolicy/default_policy.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package peerhandoutpolicy 15 | 16 | import "github.com/uber/kraken/core" 17 | 18 | const _defaultPolicy = "default" 19 | 20 | // defaultAssignmentPolicy is a NO-OP policy that assigns all peers 21 | // the highest priority. 22 | type defaultAssignmentPolicy struct{} 23 | 24 | func newDefaultAssignmentPolicy() assignmentPolicy { 25 | return &defaultAssignmentPolicy{} 26 | } 27 | 28 | func (p *defaultAssignmentPolicy) assignPriority(peer *core.PeerInfo) (int, string) { 29 | return 0, "default" 30 | } 31 | -------------------------------------------------------------------------------- /tracker/peerhandoutpolicy/default_policy_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package peerhandoutpolicy 15 | 16 | import ( 17 | "testing" 18 | 19 | "github.com/uber/kraken/core" 20 | 21 | "github.com/stretchr/testify/require" 22 | "github.com/uber-go/tally" 23 | ) 24 | 25 | func TestDefaultPriorityPolicy(t *testing.T) { 26 | require := require.New(t) 27 | 28 | policy, err := NewPriorityPolicy(tally.NoopScope, _defaultPolicy) 29 | require.NoError(err) 30 | 31 | nPeers := 50 32 | 33 | peers := make([]*core.PeerInfo, nPeers) 34 | for k := 0; k < len(peers); k++ { 35 | peers[k] = core.PeerInfoFixture() 36 | } 37 | 38 | policy.SortPeers(core.PeerInfoFixture(), peers) 39 | require.Len(peers, nPeers) 40 | } 41 | -------------------------------------------------------------------------------- /tracker/peerhandoutpolicy/fixtures.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package peerhandoutpolicy 15 | 16 | import "github.com/uber-go/tally" 17 | 18 | // DefaultPriorityPolicyFixture returns the default peer handout policy for testing purposes. 19 | func DefaultPriorityPolicyFixture() *PriorityPolicy { 20 | p, err := NewPriorityPolicy(tally.NoopScope, "default") 21 | if err != nil { 22 | panic(err) 23 | } 24 | return p 25 | } 26 | -------------------------------------------------------------------------------- /tracker/peerhandoutpolicy/peerhandoutpolicy_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package peerhandoutpolicy 15 | 16 | import ( 17 | "testing" 18 | 19 | "github.com/uber/kraken/core" 20 | 21 | "github.com/stretchr/testify/require" 22 | ) 23 | 24 | func TestPriorityPolicyRemoveSource(t *testing.T) { 25 | require := require.New(t) 26 | 27 | policy := DefaultPriorityPolicyFixture() 28 | 29 | src := core.PeerInfoFixture() 30 | peers := make([]*core.PeerInfo, 10) 31 | for k := 0; k < len(peers); k++ { 32 | peers[k] = core.PeerInfoFixture() 33 | } 34 | peers = append(peers, src) 35 | 36 | sorted := policy.SortPeers(src, peers) 37 | require.Len(sorted, len(peers)-1) 38 | for k := 0; k < len(sorted); k++ { 39 | require.NotEqual(src, sorted[k]) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /tracker/trackerserver/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package trackerserver 15 | 16 | import ( 17 | "time" 18 | 19 | "github.com/uber/kraken/utils/listener" 20 | ) 21 | 22 | // Config defines configuration for the tracker service. 23 | type Config struct { 24 | // Limits the number of unique metainfo requests to origin per namespace/digest. 25 | GetMetaInfoLimit time.Duration `yaml:"get_metainfo_limit"` 26 | 27 | // Limits the number of peers returned on each announce. 28 | PeerHandoutLimit int `yaml:"announce_limit"` 29 | 30 | AnnounceInterval time.Duration `yaml:"announce_interval"` 31 | 32 | Listener listener.Config `yaml:"listener"` 33 | } 34 | 35 | func (c Config) applyDefaults() Config { 36 | if c.GetMetaInfoLimit == 0 { 37 | c.GetMetaInfoLimit = time.Second 38 | } 39 | if c.PeerHandoutLimit == 0 { 40 | c.PeerHandoutLimit = 50 41 | } 42 | if c.AnnounceInterval == 0 { 43 | c.AnnounceInterval = 3 * time.Second 44 | } 45 | return c 46 | } 47 | -------------------------------------------------------------------------------- /tracker/trackerserver/fixtures.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package trackerserver 15 | 16 | import ( 17 | "time" 18 | 19 | "github.com/uber-go/tally" 20 | 21 | "github.com/uber/kraken/tracker/originstore" 22 | "github.com/uber/kraken/tracker/peerhandoutpolicy" 23 | "github.com/uber/kraken/tracker/peerstore" 24 | ) 25 | 26 | // Fixture is a test utility which returns a tracker server with in-memory storage. 27 | func Fixture() *Server { 28 | policy := peerhandoutpolicy.DefaultPriorityPolicyFixture() 29 | config := Config{ 30 | AnnounceInterval: 250 * time.Millisecond, 31 | } 32 | return New( 33 | config, tally.NoopScope, policy, 34 | peerstore.NewTestStore(), originstore.NewNoopStore(), nil) 35 | } 36 | -------------------------------------------------------------------------------- /utils/bitsetutil/bitsetutil.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package bitsetutil 15 | 16 | import "github.com/willf/bitset" 17 | 18 | // FromBools returns a new BitSet from the given bools. 19 | func FromBools(bs ...bool) *bitset.BitSet { 20 | s := bitset.New(uint(len(bs))) 21 | for i, b := range bs { 22 | s.SetTo(uint(i), b) 23 | } 24 | return s 25 | } 26 | -------------------------------------------------------------------------------- /utils/configutil/testdata/configFullD/base.yaml: -------------------------------------------------------------------------------- 1 | listen_address: configFullDBase:01 2 | buffer_space: 1000 3 | servers: 4 | - configFullDBase:01 5 | - configFullDBase:02 6 | -------------------------------------------------------------------------------- /utils/configutil/testdata/configFullD/production-zone2.yaml: -------------------------------------------------------------------------------- 1 | extends: production.yaml 2 | listen_address: configFullDProdZone2:03 3 | buffer_space: 3000 4 | servers: 5 | - configFullDProdZone2:03 6 | - configFullDProdZone2:04 7 | -------------------------------------------------------------------------------- /utils/configutil/testdata/configFullD/production.yaml: -------------------------------------------------------------------------------- 1 | extends: base.yaml 2 | listen_address: configFullDProd:02 3 | buffer_space: 2000 4 | servers: 5 | - configFullDProd:02 6 | - configFullDProd:03 7 | -------------------------------------------------------------------------------- /utils/configutil/testdata/configFullE/base.yaml: -------------------------------------------------------------------------------- 1 | listen_address: configFullEBase:01 2 | buffer_space: 4000 3 | servers: 4 | - configFullEBase:01 5 | - configFullEBase:02 6 | -------------------------------------------------------------------------------- /utils/configutil/testdata/configFullE/production-zone2.yaml: -------------------------------------------------------------------------------- 1 | extends: production.yaml 2 | listen_address: configFullEProdZone2:03 3 | buffer_space: 6000 4 | servers: 5 | - configFullEProdZone2:03 6 | - configFullEProdZone2:04 7 | -------------------------------------------------------------------------------- /utils/configutil/testdata/configFullE/production.yaml: -------------------------------------------------------------------------------- 1 | extends: base.yaml 2 | listen_address: configFullEProd:02 3 | buffer_space: 5000 4 | servers: 5 | - configFullEProd:02 6 | - configFullEProd:03 7 | -------------------------------------------------------------------------------- /utils/configutil/testdata/multiple/configA/base.yaml: -------------------------------------------------------------------------------- 1 | listen_address: 127.0.0.1:1 2 | -------------------------------------------------------------------------------- /utils/configutil/testdata/multiple/configA/production-zone2.yaml: -------------------------------------------------------------------------------- 1 | extends: production.yaml 2 | servers: 3 | - 127.0.0.1:01 4 | nodes: 5 | configAProdZone2: nodeA 6 | -------------------------------------------------------------------------------- /utils/configutil/testdata/multiple/configA/production.yaml: -------------------------------------------------------------------------------- 1 | extends: base.yaml 2 | buffer_space: 1000 3 | -------------------------------------------------------------------------------- /utils/configutil/testdata/multiple/configB/base.yaml: -------------------------------------------------------------------------------- 1 | listen_address: 127.0.0.1:2 2 | -------------------------------------------------------------------------------- /utils/configutil/testdata/multiple/configB/production-zone2.yaml: -------------------------------------------------------------------------------- 1 | extends: production.yaml 2 | servers: 3 | - 127.0.0.1:02 4 | nodes: 5 | configBProdZone2: nodeB 6 | -------------------------------------------------------------------------------- /utils/configutil/testdata/multiple/configB/production.yaml: -------------------------------------------------------------------------------- 1 | extends: base.yaml 2 | buffer_space: 2000 3 | -------------------------------------------------------------------------------- /utils/configutil/testdata/multiple/configC/base.yaml: -------------------------------------------------------------------------------- 1 | listen_address: 127.0.0.1:3 2 | -------------------------------------------------------------------------------- /utils/configutil/testdata/multiple/configC/production-zone2.yaml: -------------------------------------------------------------------------------- 1 | extends: production.yaml 2 | servers: 3 | - 127.0.0.1:03 4 | nodes: 5 | configCProdZone2: nodeC 6 | -------------------------------------------------------------------------------- /utils/configutil/testdata/multiple/configC/production.yaml: -------------------------------------------------------------------------------- 1 | extends: base.yaml 2 | buffer_space: 3000 3 | -------------------------------------------------------------------------------- /utils/configutil/testdata/multiple/configF/base.yaml: -------------------------------------------------------------------------------- 1 | listen_address: 127.0.0.1:1 2 | -------------------------------------------------------------------------------- /utils/configutil/testdata/multiple/configF/production-zone2.yaml: -------------------------------------------------------------------------------- 1 | extends: ../configG/production.yaml 2 | servers: 3 | - 127.0.0.1:04 4 | nodes: 5 | configFProdZone2: nodeF 6 | -------------------------------------------------------------------------------- /utils/configutil/testdata/multiple/configG/base.yaml: -------------------------------------------------------------------------------- 1 | listen_address: 127.0.0.1:1 2 | -------------------------------------------------------------------------------- /utils/configutil/testdata/multiple/configG/production-zone2.yaml: -------------------------------------------------------------------------------- 1 | extends: production.yaml 2 | servers: 3 | - 127.0.0.1:05 4 | nodes: 5 | configGProdZone2: nodeG 6 | -------------------------------------------------------------------------------- /utils/configutil/testdata/multiple/configG/production.yaml: -------------------------------------------------------------------------------- 1 | extends: base.yaml 2 | buffer_space: 1000 3 | -------------------------------------------------------------------------------- /utils/configutil/testdata/single/base.yaml: -------------------------------------------------------------------------------- 1 | listen_address: 127.0.0.1:8080 2 | bufferSpace: 9 3 | -------------------------------------------------------------------------------- /utils/configutil/testdata/single/secrets.yaml: -------------------------------------------------------------------------------- 1 | secret: shh 2 | -------------------------------------------------------------------------------- /utils/configutil/testdata/single/test.yaml: -------------------------------------------------------------------------------- 1 | extends: base.yaml 2 | buffer_space: 9000 3 | servers: 4 | - 127.0.0.1:01 5 | - 127.0.0.1:02 6 | -------------------------------------------------------------------------------- /utils/diskspaceutil/diskspaceutil.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package diskspaceutil 15 | 16 | import ( 17 | "syscall" 18 | ) 19 | 20 | const path = "/" 21 | 22 | // Helper method to get disk util. 23 | func DiskSpaceUtil() (int, error) { 24 | fs := syscall.Statfs_t{} 25 | err := syscall.Statfs(path, &fs) 26 | if err != nil { 27 | return 0, err 28 | } 29 | 30 | diskAll := fs.Blocks * uint64(fs.Bsize) 31 | diskFree := fs.Bfree * uint64(fs.Bsize) 32 | diskUsed := diskAll - diskFree 33 | return int(diskUsed * 100 / diskAll), nil 34 | 35 | } 36 | -------------------------------------------------------------------------------- /utils/diskspaceutil/diskspaceutil_test.go: -------------------------------------------------------------------------------- 1 | package diskspaceutil_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | "github.com/uber/kraken/utils/diskspaceutil" 9 | ) 10 | 11 | func TestParseManifestV2List(t *testing.T) { 12 | util, err := diskspaceutil.DiskSpaceUtil() 13 | require.NoError(t, err) 14 | 15 | require.Equal(t, true, util > 0) 16 | require.Equal(t, true, util < 100) 17 | } 18 | -------------------------------------------------------------------------------- /utils/errutil/errutil.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package errutil 15 | 16 | import "bytes" 17 | 18 | // MultiError defines a list of multiple errors. Useful for type assertions and 19 | // inspecting individual errors. 20 | // 21 | // XXX: Don't initialize errors as MultiError unless you know what you're doing! 22 | // See https://golang.org/doc/faq#nil_error for more details. 23 | type MultiError []error 24 | 25 | func (e MultiError) Error() string { 26 | var b bytes.Buffer 27 | for i, err := range e { 28 | b.WriteString(err.Error()) 29 | if i < len(e)-1 { 30 | b.WriteString(", ") 31 | } 32 | } 33 | return b.String() 34 | } 35 | 36 | // Join converts errs into an error interface. 37 | func Join(errs []error) error { 38 | if errs == nil { 39 | return nil 40 | } 41 | return MultiError(errs) 42 | } 43 | -------------------------------------------------------------------------------- /utils/flagutil/flagutil.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package flagutil 15 | 16 | import ( 17 | "flag" 18 | "strconv" 19 | ) 20 | 21 | // Ints allows specifying a slice of ints via flags, where multiple flags of the 22 | // same name append to the slice. 23 | type Ints []int 24 | 25 | var _ flag.Value = (*Ints)(nil) 26 | 27 | func (i *Ints) String() string { 28 | return "list of ints" 29 | } 30 | 31 | // Set appends v to i. 32 | func (i *Ints) Set(v string) error { 33 | n, err := strconv.Atoi(v) 34 | if err != nil { 35 | return err 36 | } 37 | *i = append(*i, n) 38 | return nil 39 | } 40 | -------------------------------------------------------------------------------- /utils/heap/priority_queue_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package heap 15 | 16 | import ( 17 | "testing" 18 | 19 | "github.com/stretchr/testify/require" 20 | ) 21 | 22 | func TestPriorityQueue(t *testing.T) { 23 | require := require.New(t) 24 | items := []*Item{{"a", 3}, {"b", 2}, {"c", 4}} 25 | itemsCopy := []*Item{{"a", 3}, {"b", 2}, {"c", 4}} 26 | 27 | pq := NewPriorityQueue(items...) 28 | 29 | var item *Item 30 | var err error 31 | 32 | item, err = pq.Pop() 33 | require.NoError(err) 34 | require.Equal(itemsCopy[1], item) 35 | 36 | newItem := &Item{"d", 1} 37 | pq.Push(newItem) 38 | 39 | item, err = pq.Pop() 40 | require.NoError(err) 41 | require.Equal(newItem, item) 42 | 43 | item, err = pq.Pop() 44 | require.NoError(err) 45 | require.Equal(itemsCopy[0], item) 46 | 47 | item, err = pq.Pop() 48 | require.NoError(err) 49 | require.Equal(itemsCopy[2], item) 50 | 51 | _, err = pq.Pop() 52 | require.Error(err) 53 | } 54 | -------------------------------------------------------------------------------- /utils/httputil/backoff.go: -------------------------------------------------------------------------------- 1 | package httputil 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/cenkalti/backoff" 7 | ) 8 | 9 | // ExponentialBackOffConfig maps backoff settings into YAML config format. 10 | type ExponentialBackOffConfig struct { 11 | Enabled bool `yaml:"enabled"` 12 | InitialInterval time.Duration `yaml:"initial_interval"` 13 | RandomizationFactor float64 `yaml:"randomization_factor"` 14 | Multiplier float64 `yaml:"multiplier"` 15 | MaxInterval time.Duration `yaml:"max_interval"` 16 | MaxRetries uint64 `yaml:"max_retries"` 17 | } 18 | 19 | func (c *ExponentialBackOffConfig) applyDefaults() { 20 | if c.InitialInterval == 0 { 21 | c.InitialInterval = 2 * time.Second 22 | } 23 | if c.RandomizationFactor == 0 { 24 | c.RandomizationFactor = 0.05 25 | } 26 | if c.Multiplier == 0 { 27 | c.Multiplier = 2 28 | } 29 | if c.MaxInterval == 0 { 30 | c.MaxInterval = 30 * time.Second 31 | } 32 | if c.MaxRetries == 0 { 33 | c.MaxRetries = 5 34 | } 35 | } 36 | 37 | // Build creates a new ExponentialBackOff using c's settings (if enabled). 38 | func (c ExponentialBackOffConfig) Build() backoff.BackOff { 39 | if c.Enabled { 40 | c.applyDefaults() 41 | b := &backoff.ExponentialBackOff{ 42 | InitialInterval: c.InitialInterval, 43 | RandomizationFactor: c.RandomizationFactor, 44 | Multiplier: c.Multiplier, 45 | MaxInterval: c.MaxInterval, 46 | Clock: backoff.SystemClock, 47 | } 48 | return backoff.WithMaxRetries(b, c.MaxRetries) 49 | } 50 | return &backoff.StopBackOff{} 51 | } 52 | -------------------------------------------------------------------------------- /utils/listener/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package listener 15 | 16 | import "fmt" 17 | 18 | // Config defines listener configuration. 19 | type Config struct { 20 | // Net is the network to listen on, e.g. unix, tcp, etc. 21 | Net string `yaml:"net"` 22 | 23 | // Addr is the address to listen on. 24 | Addr string `yaml:"addr"` 25 | } 26 | 27 | func (c Config) String() string { 28 | return fmt.Sprintf("%s:%s", c.Net, c.Addr) 29 | } 30 | -------------------------------------------------------------------------------- /utils/listener/listen.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package listener 15 | 16 | import ( 17 | "net" 18 | "net/http" 19 | ) 20 | 21 | // Serve serves h on a listener configured by config. Useful for easily 22 | // swapping tcp / unix servers. 23 | func Serve(config Config, h http.Handler) error { 24 | l, err := net.Listen(config.Net, config.Addr) 25 | if err != nil { 26 | return err 27 | } 28 | return http.Serve(l, h) 29 | } 30 | -------------------------------------------------------------------------------- /utils/rwutil/cappedbuffer_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package rwutil 15 | 16 | import ( 17 | "testing" 18 | 19 | "bytes" 20 | 21 | "github.com/stretchr/testify/require" 22 | ) 23 | 24 | func TestCappedBuffer_write_drain_success(t *testing.T) { 25 | require := require.New(t) 26 | 27 | content := []byte("hello this is a stream of bytes") 28 | buffer := NewCappedBuffer(len(content)) 29 | buffer.WriteAt(content[7:], 7) 30 | buffer.WriteAt(content[:7], 0) 31 | 32 | var dst bytes.Buffer 33 | buffer.DrainInto(&dst) 34 | require.Equal(content, dst.Bytes()) 35 | } 36 | 37 | func TestCappedBuffer_write_drain_error(t *testing.T) { 38 | require := require.New(t) 39 | 40 | content := []byte("hello this is a stream of bytes") 41 | buffer := NewCappedBuffer(len(content) - 10) 42 | _, err := buffer.WriteAt(content[7:], 7) 43 | require.Error(err) 44 | 45 | _, err = buffer.WriteAt(content[:7], 0) 46 | require.NoError(err) 47 | } 48 | -------------------------------------------------------------------------------- /utils/rwutil/rwutil.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package rwutil 15 | 16 | import "io" 17 | 18 | // PlainReader provides an io.Reader for a bytes slice. It intentionally does 19 | // not provide any other methods. 20 | type PlainReader []byte 21 | 22 | // Read always reads the entire underlying byte slice. 23 | func (p PlainReader) Read(b []byte) (n int, err error) { 24 | copy(b, p) 25 | return len(p), io.EOF 26 | } 27 | 28 | // PlainWriter provides an io.Writer for a bytes slice. It intentionally does 29 | // not provide any other methods. Clients should initialize length with make. 30 | type PlainWriter []byte 31 | 32 | // Write writes all of b to p. 33 | func (p PlainWriter) Write(b []byte) (n int, err error) { 34 | copy(p, b) 35 | return len(p), nil 36 | } 37 | -------------------------------------------------------------------------------- /utils/rwutil/rwutil_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package rwutil 15 | 16 | import ( 17 | "bytes" 18 | "io" 19 | "testing" 20 | 21 | "github.com/uber/kraken/utils/randutil" 22 | "github.com/stretchr/testify/require" 23 | ) 24 | 25 | func TestPlainReader(t *testing.T) { 26 | require := require.New(t) 27 | 28 | data := randutil.Text(32) 29 | 30 | var result bytes.Buffer 31 | _, err := io.Copy(&result, PlainReader(data)) 32 | require.NoError(err) 33 | require.Equal(data, result.Bytes()) 34 | } 35 | 36 | func TestPlainWriter(t *testing.T) { 37 | require := require.New(t) 38 | 39 | data := randutil.Text(32) 40 | 41 | w := make(PlainWriter, len(data)) 42 | _, err := io.Copy(w, bytes.NewReader(data)) 43 | require.NoError(err) 44 | require.Equal(data, []byte(w)) 45 | } 46 | -------------------------------------------------------------------------------- /utils/timeutil/timeutil.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package timeutil 15 | 16 | import "time" 17 | 18 | // MostRecent returns the most recent Time of ts. 19 | func MostRecent(ts ...time.Time) time.Time { 20 | if len(ts) == 0 { 21 | return time.Time{} 22 | } 23 | max := ts[0] 24 | for i := 1; i < len(ts); i++ { 25 | if max.Before(ts[i]) { 26 | max = ts[i] 27 | } 28 | } 29 | return max 30 | } 31 | 32 | // MaxDuration returns the largest duration between a and b. 33 | func MaxDuration(a, b time.Duration) time.Duration { 34 | if a > b { 35 | return a 36 | } 37 | return b 38 | } 39 | -------------------------------------------------------------------------------- /utils/timeutil/timeutil_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2019 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package timeutil 15 | 16 | import ( 17 | "testing" 18 | "time" 19 | 20 | "github.com/stretchr/testify/require" 21 | ) 22 | 23 | func TestMostRecent(t *testing.T) { 24 | a := time.Now() 25 | b := a.Add(time.Second) 26 | c := b.Add(time.Second) 27 | 28 | for _, test := range []struct { 29 | description string 30 | ts []time.Time 31 | expected time.Time 32 | }{ 33 | {"empty list", []time.Time{}, time.Time{}}, 34 | {"single item", []time.Time{a}, a}, 35 | {"ascending order", []time.Time{a, b, c}, c}, 36 | {"descending order", []time.Time{c, b, a}, c}, 37 | } { 38 | t.Run(test.description, func(t *testing.T) { 39 | actual := MostRecent(test.ts...) 40 | require.Equal(t, test.expected, actual) 41 | }) 42 | } 43 | } 44 | --------------------------------------------------------------------------------