├── .clabot ├── .dockerignore ├── .github └── pull_request_template.md ├── .gitignore ├── .gitreview ├── CODE_OF_CONDUCT.md ├── Jenkinsfile ├── LICENSE ├── accesslogs ├── errors.go ├── processor.go ├── processor_test.go ├── uploader.go └── uploader_test.go ├── base58 ├── LICENSE ├── README.md ├── alphabet.go ├── base58.go ├── base58_test.go ├── base58bench_test.go ├── base58check.go ├── base58check_test.go ├── cov_report.sh ├── doc.go ├── example_test.go └── genalphabet.go ├── bloomfilter ├── filter.go └── filter_test.go ├── cfgstruct ├── bind.go ├── bind_test.go ├── case.go ├── case_test.go └── flags.go ├── context2 ├── cancel.go ├── cancel_test.go ├── nocancel.go └── nocancel_test.go ├── currency ├── amount.go └── amount_test.go ├── debug ├── common.go ├── crawlspace │ └── crawlspace.go ├── endpoint.go ├── endpoint_test.go ├── error.go ├── panel.go ├── pcap_linux.go ├── pcap_other.go ├── prometheus.go ├── server.go ├── server_test.go ├── top.go ├── trace_disabled.go └── trace_enabled.go ├── encryption ├── aesgcm.go ├── aesgcm_test.go ├── bits.go ├── bits_test.go ├── common.go ├── encryption.go ├── encryption_test.go ├── examples_test.go ├── pad.go ├── pad_test.go ├── password.go ├── password_test.go ├── path.go ├── path_test.go ├── prefix.go ├── prefix_test.go ├── secretbox.go ├── secretbox_test.go ├── store.go ├── store_test.go ├── transform.go └── transform_test.go ├── errs2 ├── collect.go ├── collect_test.go ├── doc.go ├── group.go ├── group_test.go ├── ignore.go ├── ignore_test.go └── rpc.go ├── eventstat ├── counter.go ├── doc.go ├── publisher.go ├── publisher_test.go ├── registry.go └── registry_test.go ├── experiment ├── common.go ├── common_test.go ├── export.go └── import.go ├── fpath ├── atomic.go ├── atomic_test.go ├── doc.go ├── editor.go ├── os.go ├── path.go ├── path_test.go ├── path_unix_test.go ├── path_windows_test.go └── temp_data.go ├── go.mod ├── go.sum ├── grant ├── access.go ├── access_test.go ├── internal │ └── pb │ │ ├── doc.go │ │ ├── encryption.pico.go │ │ ├── encryption_access.pico.go │ │ └── scope.pico.go ├── restrict.go └── restrict_test.go ├── http └── requestid │ ├── requestid.go │ └── requestid_test.go ├── identity ├── certificate_authority.go ├── certificate_authority_test.go ├── common.go ├── doc.go ├── generate.go ├── identity.go ├── identity_test.go ├── testidentity │ ├── V0_identities_table.go │ ├── V0_signed_identities_table.go │ ├── doc.go │ ├── gen_identities.go │ ├── identities.go │ ├── identity.go │ └── pregenerated_identities_test.go └── utils.go ├── internal ├── hmacsha512 │ ├── GO_LICENSE │ ├── doc.go │ ├── hmac.go │ ├── hmac_fuzz_test.go │ ├── hmac_test.go │ ├── sha512.go │ ├── sha512_test.go │ ├── sha512block.go │ ├── sha512block_amd64.go │ ├── sha512block_amd64.s │ ├── sha512block_arm64.go │ ├── sha512block_arm64.s │ ├── sha512block_decl.go │ ├── sha512block_generic.go │ ├── sha512block_ppc64x.s │ ├── sha512block_s390x.go │ └── sha512block_s390x.s └── vendor.go ├── leak ├── ref_norace.go ├── ref_race.go ├── ref_test.go ├── resource.go └── resource_test.go ├── macaroon ├── apikey.go ├── apikey_test.go ├── caveat.go ├── doc.go ├── json_test.go ├── macaroon.go ├── macaroon_test.go ├── serialize.go ├── types.pico.go └── types.proto ├── memory ├── doc.go ├── size.go ├── size_test.go ├── sizes.go └── string.go ├── metrics ├── server.go ├── server_test.go └── testdata │ ├── README.md │ ├── ca-client.crt │ ├── ca-client.key │ ├── ca.crt │ ├── ca.key │ ├── client1.crt │ ├── client1.key │ ├── gen-ca.sh │ ├── gen-client-tls.sh │ ├── gen-host-tls.sh │ ├── server.crt │ └── server.key ├── netutil ├── common.go ├── timeout_linux.go ├── timeout_other.go └── tracking.go ├── nodetag ├── authority.go ├── sign.go └── sign_test.go ├── paths ├── doc.go ├── path.go └── path_test.go ├── pb ├── alias.go ├── certificate.pb.go ├── certificate.proto ├── certificate_drpc.pb.go ├── compatibility_test.go ├── contact.pb.go ├── contact.proto ├── contact_drpc.pb.go ├── debug.pb.go ├── debug.proto ├── debug_drpc.pb.go ├── doc.go ├── edgeauth.pb.go ├── edgeauth.proto ├── edgeauth_drpc.pb.go ├── encryption.go ├── encryption.pb.go ├── encryption.proto ├── encryption_access.go ├── encryption_access.pb.go ├── encryption_access.proto ├── gen.go ├── gogo.proto ├── gracefulexit.pb.go ├── gracefulexit.proto ├── gracefulexit_drpc.pb.go ├── hashing.go ├── heldamount.pb.go ├── heldamount.proto ├── heldamount_drpc.pb.go ├── json_test.go ├── meta.pb.go ├── meta.proto ├── metainfo.pb.go ├── metainfo.proto ├── metainfo_aliases.go ├── metainfo_drpc.pb.go ├── node.pb.go ├── node.proto ├── nodestats.pb.go ├── nodestats.proto ├── nodestats_drpc.pb.go ├── nodetags.pb.go ├── nodetags.proto ├── noise.pb.go ├── noise.proto ├── orders.pb.go ├── orders.proto ├── orders_drpc.pb.go ├── pico.proto ├── piecestore2.pb.go ├── piecestore2.proto ├── piecestore2_drpc.pb.go ├── pointerdb.pb.go ├── pointerdb.proto ├── replay.go ├── scope.pb.go ├── scope.proto ├── streams.pb.go ├── streams.proto ├── types.go ├── userinfo.pb.go ├── userinfo.proto ├── userinfo_drpc.pb.go ├── utils.go └── utils_test.go ├── peertls ├── doc.go ├── extensions │ ├── doc.go │ ├── extensions.go │ ├── extensions_test.go │ ├── fuzz_test.go │ ├── gob.go │ └── revocations.go ├── peertls.go ├── peertls_test.go ├── templates.go ├── testpeertls │ ├── certificates.go │ ├── certificates_test.go │ ├── doc.go │ └── revocations.go ├── tlsopts │ ├── cert.go │ ├── config.go │ ├── doc.go │ ├── options.go │ ├── options_internal_test.go │ ├── tls.go │ ├── tls_helpers_test.go │ └── tls_test.go └── utils.go ├── pkcrypto ├── common.go ├── doc.go ├── encoding.go ├── hashing.go ├── signing.go └── signing_test.go ├── process ├── config.go ├── debug.go ├── decorate_notwindows.go ├── decorate_windows.go ├── eventkitbq │ └── destination.go ├── exec.go ├── exec_conf.go ├── exec_conf_test.go ├── gcloudlogging │ ├── gcloudlogging.go │ ├── gcloudlogging_test.go │ ├── httprequest.go │ ├── httprequest_test.go │ ├── operation.go │ └── operation_test.go ├── googleprofiler │ └── init.go ├── logging.go ├── logging_test.go ├── metrics.go ├── profiler.go ├── publisher.go └── tracing.go ├── processgroup ├── doc.go ├── kill_fallback.go ├── kill_test.go ├── kill_unix.go ├── kill_unix_nodeath.go └── kill_windows.go ├── proto.lock ├── ranger ├── common.go ├── doc.go ├── file.go ├── file_test.go ├── force.go ├── httpranger │ ├── content.go │ ├── content_test.go │ ├── doc.go │ ├── http.go │ └── http_test.go ├── prefetch.go ├── reader.go ├── reader_test.go ├── readerat.go ├── readerat_test.go ├── thunk.go └── thunkreader.go ├── readcloser ├── doc.go ├── fatal.go ├── lazy.go ├── limit.go └── multi.go ├── rpc ├── common.go ├── conn.go ├── connector.go ├── dial.go ├── dial_test.go ├── dial_unit_test.go ├── doc.go ├── hybrid.go ├── hybrid_expose_test.go ├── hybrid_test.go ├── known_ids.go ├── known_ids_test.go ├── lookup.go ├── lookup_test.go ├── multidial │ ├── conn.go │ ├── dialer.go │ ├── multidial_test.go │ ├── setup.go │ └── utils.go ├── noise │ ├── noise.go │ └── noise_test.go ├── quic │ ├── common.go │ ├── conn.go │ ├── conn_noquic.go │ ├── connector.go │ ├── connector_noquic.go │ ├── errorhandling.go │ ├── errorhandling_windows.go │ ├── init.go │ ├── listener.go │ ├── listener_noquic.go │ └── listener_test.go ├── quic_rollout.go ├── rpccache │ ├── cache.go │ └── cache_test.go ├── rpcpeer │ └── peer.go ├── rpcpool │ ├── common.go │ ├── conn.go │ ├── debug.go │ ├── pool.go │ ├── pool_test.go │ ├── wrapper.go │ └── wrapper_test.go ├── rpcstatus │ ├── status.go │ ├── status_test.go │ └── statuscode_string.go ├── rpctest │ ├── assertions.go │ ├── assertions_test.go │ ├── doc.go │ ├── latency.go │ ├── recorder.go │ ├── recorder_test.go │ ├── stub.go │ ├── stub_test.go │ ├── wrappers.go │ └── wrappers_test.go ├── rpctimeout │ └── timeout.go └── rpctracing │ ├── common.go │ ├── excluded.go │ ├── handler.go │ └── tracing.go ├── scripts ├── arm-staticcheck.sh ├── check-cross-compile.sh └── check-dependencies.sh ├── signing ├── disabled.go ├── disabled_test.go ├── doc.go ├── encode.go ├── peers.go ├── sign.go ├── sign_test.go ├── verify.go └── verify_test.go ├── socket ├── dscp.go ├── extended.go ├── extended_darwin.go ├── extended_linux.go ├── extended_other.go └── extended_windows.go ├── storj ├── default.go ├── deleteobjects.go ├── doc.go ├── encryption.go ├── encryption_test.go ├── identity_version.go ├── identity_version_test.go ├── location │ ├── continent.go │ ├── country.go │ ├── countrycode.go │ ├── countrycode_test.go │ ├── doc.go │ ├── gen │ │ └── generate.go │ ├── region.go │ ├── set.go │ └── set_test.go ├── node.go ├── node_test.go ├── nodeurl.go ├── nodeurl_test.go ├── noise.go ├── path.go ├── path_test.go ├── pieceid.go ├── pieceid_deriver.go ├── pieceid_deriver_test.go ├── pieceid_test.go ├── piecekey.go ├── piecekey_test.go ├── placement.go ├── placement_test.go ├── redundancy.go ├── redundancy_test.go ├── retention.go ├── segmentid.go ├── serialnumber.go ├── serialnumber_test.go ├── streamid.go └── streamid_test.go ├── strictcsv ├── common.go ├── common_test.go ├── marshal.go ├── marshal_test.go ├── unmarshal.go └── unmarshal_test.go ├── sync2 ├── conc.go ├── conc_test.go ├── cooldown.go ├── cooldown_test.go ├── copy.go ├── copy_test.go ├── cycle.go ├── cycle_test.go ├── doc.go ├── event.go ├── event_test.go ├── fence.go ├── fence_test.go ├── limiter.go ├── limiter_test.go ├── mpscqueue │ ├── LICENSE │ ├── queue.go │ └── queue_test.go ├── nocopy.go ├── race2 │ ├── doc.go │ ├── race.go │ └── race0.go ├── rcchan.go ├── rcchan_test.go ├── readcache.go ├── readcache_test.go ├── semaphore.go ├── sleep.go ├── sleep_test.go ├── success_threshold.go ├── success_threshold_test.go ├── tee.go ├── tee_buffer.go ├── tee_test.go ├── throttle.go ├── throttle_test.go ├── timeout.go ├── timeout_test.go ├── workgroup.go ├── workgroup_test.go ├── workplace.go └── workplace_test.go ├── telemetry ├── client.go ├── client_test.go ├── common.go ├── example_test.go ├── reporter.go ├── server.go ├── server_test.go ├── tm_test.go ├── udp.go ├── utils.go └── utils_test.go ├── testcontext ├── compile.go ├── compile_norace.go ├── compile_race.go ├── compile_test.go ├── context.go ├── context_test.go └── testdata │ ├── hello │ └── main.go │ └── hellomod │ ├── go.mod │ └── main.go ├── testrand ├── rand.go └── rand_test.go ├── testtrace ├── trace.go └── trace_test.go ├── time2 ├── clock.go ├── clock_test.go ├── context.go ├── doc.go ├── machine.go └── machine_test.go ├── traces └── traces.go ├── tracing └── excluded.go ├── useragent ├── encode.go ├── encode_test.go ├── fuzz_test.go ├── info.go ├── info_test.go ├── parse.go ├── parse_internal_test.go └── parse_test.go ├── uuid ├── db.go ├── db_test.go ├── fuzz_test.go ├── uuid.go └── uuid_test.go └── version ├── build.go ├── buildinfo ├── build.go ├── build_test.go └── testbuild │ ├── go.mod │ ├── go.sum │ └── main.go ├── info.go ├── info_1.17.go ├── stats.go ├── stats_other.go ├── stats_test.go ├── stats_windows.go ├── version.go └── version_test.go /.clabot: -------------------------------------------------------------------------------- 1 | { 2 | "contributors": [ 3 | "aleitner", 4 | "aligeti", 5 | "Barterio", 6 | "brimstone", 7 | "bryanchriswhite", 8 | "cam-a", 9 | "coyle", 10 | "crawter", 11 | "dylanlott", 12 | "egonelbre", 13 | "fadila82", 14 | "iglesiasbrandon", 15 | "jenlij", 16 | "jhagans3", 17 | "jtolds", 18 | "kaloyan-raev", 19 | "littleskunk", 20 | "mniewrzal", 21 | "mobyvb", 22 | "navillasa", 23 | "nfarah86", 24 | "NikolaiYurchenko", 25 | "phutchins", 26 | "rikysya", 27 | "stefanbenten", 28 | "thepaul", 29 | "wthorp", 30 | "RichardLitt", 31 | "fuskovic", 32 | "keleffew", 33 | "oCroso", 34 | "pgerbes1", 35 | "JessicaGreben", 36 | "benjaminsirb", 37 | "simongui", 38 | "ifraixedes", 39 | "VinozzZ", 40 | "zeebo", 41 | "barlock", 42 | "sndrr", 43 | "BlackDuck888", 44 | "3bl3gamer", 45 | "ethanadams", 46 | "ReneSmeekes", 47 | "VitaliiShpital", 48 | "isaachess", 49 | "azdagron", 50 | "phthano", 51 | "nerdatwork", 52 | "kmozurkewich", 53 | "TopperDEL", 54 | "kristaxox", 55 | "calebcase", 56 | "mbouzi", 57 | "AlexeyALeonov", 58 | "Qweder93", 59 | "cpustejovsky", 60 | "grafael", 61 | "ihaid" 62 | ] 63 | } 64 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | /*.swp 2 | docker-compose.yaml 3 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | 2 | What: 3 | 4 | Why: 5 | 6 | Please describe the tests: 7 | - Test 1: 8 | - Test 2: 9 | 10 | Please describe the performance impact: 11 | 12 | ## Code Review Checklist (to be filled out by reviewer) 13 | - [ ] NEW: Are there any Satellite database migrations? Are they forwards _and_ backwards compatible? 14 | - [ ] Does the PR describe what changes are being made? 15 | - [ ] Does the PR describe why the changes are being made? 16 | - [ ] Does the code follow [our style guide](https://github.com/storj/docs/blob/main/code/Style.md)? 17 | - [ ] Does the code follow [our testing guide](https://github.com/storj/docs/blob/main/code/Testing.md)? 18 | - [ ] Is the PR appropriately sized? (If it could be broken into smaller PRs it should be) 19 | - [ ] Does the new code have enough tests? (*every* PR should have tests or justification otherwise. Bug-fix PRs especially) 20 | - [ ] Does the new code have enough documentation that answers "how do I use it?" and "what does it do?"? (both source documentation and [higher level](https://github.com/storj/docs), diagrams?) 21 | - [ ] Does any documentation need updating? 22 | - [ ] Do the database access patterns make sense? 23 | 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Mac OS X files 2 | .DS_Store 3 | 4 | # Binaries for programs and plugins 5 | *.exe 6 | *.dll 7 | *.so 8 | *.dylib 9 | *.db 10 | 11 | # Test binary, build with `go test -c` 12 | *.test 13 | *.prof 14 | 15 | # Output of the go coverage tool, specifically when used with LiteIDE 16 | *.out 17 | 18 | # VSCode 19 | /.vscode 20 | 21 | # Visual Studio 22 | /.vs 23 | 24 | # Jetbrains 25 | .idea/ 26 | 27 | *.coverprofile 28 | *.log 29 | 30 | /release/ 31 | *.swp 32 | /bin 33 | resource.syso 34 | 35 | *.resource.go 36 | 37 | .build 38 | -------------------------------------------------------------------------------- /.gitreview: -------------------------------------------------------------------------------- 1 | [gerrit] 2 | host=review.dev.storj.tools 3 | project=storj/common 4 | defaultbranch=main -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020 Storj Labs, Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /accesslogs/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2024 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package accesslogs 5 | 6 | import ( 7 | "errors" 8 | 9 | "github.com/zeebo/errs" 10 | ) 11 | 12 | var ( 13 | // Error is the error class for this package. 14 | Error = errs.Class("accesslogs") 15 | // ErrClosed means that the peer has already been closed. 16 | ErrClosed = errors.New("closed") 17 | // ErrTooLarge means that the provided payload is too large. 18 | ErrTooLarge = errors.New("entry too large") 19 | // ErrQueueLimit means that the upload queue limit has been reached. 20 | ErrQueueLimit = errors.New("upload queue limit reached") 21 | ) 22 | -------------------------------------------------------------------------------- /base58/LICENSE: -------------------------------------------------------------------------------- 1 | ISC License 2 | 3 | Copyright (c) 2013-2017 The btcsuite developers 4 | Copyright (c) 2016-2017 The Lightning Network Developers 5 | 6 | Permission to use, copy, modify, and distribute this software for any 7 | purpose with or without fee is hereby granted, provided that the above 8 | copyright notice and this permission notice appear in all copies. 9 | 10 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 | -------------------------------------------------------------------------------- /base58/README.md: -------------------------------------------------------------------------------- 1 | This package is vendored from github.com/btcsuite/btcd/btcutil/base58. -------------------------------------------------------------------------------- /base58/base58bench_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013-2014 The btcsuite developers 2 | // Use of this source code is governed by an ISC 3 | // license that can be found in the LICENSE file. 4 | 5 | package base58_test 6 | 7 | import ( 8 | "bytes" 9 | "testing" 10 | 11 | "storj.io/common/base58" 12 | ) 13 | 14 | var ( 15 | raw5k = bytes.Repeat([]byte{0xff}, 5000) 16 | raw100k = bytes.Repeat([]byte{0xff}, 100*1000) 17 | encoded5k = base58.Encode(raw5k) 18 | encoded100k = base58.Encode(raw100k) 19 | ) 20 | 21 | func BenchmarkBase58Encode_5K(b *testing.B) { 22 | b.SetBytes(int64(len(raw5k))) 23 | for i := 0; i < b.N; i++ { 24 | base58.Encode(raw5k) 25 | } 26 | } 27 | 28 | func BenchmarkBase58Encode_100K(b *testing.B) { 29 | b.SetBytes(int64(len(raw100k))) 30 | for i := 0; i < b.N; i++ { 31 | base58.Encode(raw100k) 32 | } 33 | } 34 | 35 | func BenchmarkBase58Decode_5K(b *testing.B) { 36 | b.SetBytes(int64(len(encoded5k))) 37 | for i := 0; i < b.N; i++ { 38 | base58.Decode(encoded5k) 39 | } 40 | } 41 | 42 | func BenchmarkBase58Decode_100K(b *testing.B) { 43 | b.SetBytes(int64(len(encoded100k))) 44 | for i := 0; i < b.N; i++ { 45 | base58.Decode(encoded100k) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /base58/cov_report.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # This script uses gocov to generate a test coverage report. 4 | # The gocov tool my be obtained with the following command: 5 | # go get github.com/axw/gocov/gocov 6 | # 7 | # It will be installed to $GOPATH/bin, so ensure that location is in your $PATH. 8 | 9 | # Check for gocov. 10 | type gocov >/dev/null 2>&1 11 | if [ $? -ne 0 ]; then 12 | echo >&2 "This script requires the gocov tool." 13 | echo >&2 "You may obtain it with the following command:" 14 | echo >&2 "go get github.com/axw/gocov/gocov" 15 | exit 1 16 | fi 17 | gocov test | gocov report 18 | -------------------------------------------------------------------------------- /base58/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014 The btcsuite developers 2 | // Use of this source code is governed by an ISC 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package base58 provides an API for working with modified base58 and Base58Check 7 | encodings. 8 | 9 | # Modified Base58 Encoding 10 | 11 | Standard base58 encoding is similar to standard base64 encoding except, as the 12 | name implies, it uses a 58 character alphabet which results in an alphanumeric 13 | string and allows some characters which are problematic for humans to be 14 | excluded. Due to this, there can be various base58 alphabets. 15 | 16 | The modified base58 alphabet used by Bitcoin, and hence this package, omits the 17 | 0, O, I, and l characters that look the same in many fonts and are therefore 18 | hard to humans to distinguish. 19 | 20 | # Base58Check Encoding Scheme 21 | 22 | The Base58Check encoding scheme is primarily used for Bitcoin addresses at the 23 | time of this writing, however it can be used to generically encode arbitrary 24 | byte arrays into human-readable strings along with a version byte that can be 25 | used to differentiate the same payload. For Bitcoin addresses, the extra 26 | version is used to differentiate the network of otherwise identical public keys 27 | which helps prevent using an address intended for one network on another. 28 | 29 | This package is vendored from github.com/btcsuite/btcd/tree/master/btcutil/base58. 30 | */ 31 | package base58 32 | -------------------------------------------------------------------------------- /cfgstruct/case.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package cfgstruct 5 | 6 | import ( 7 | "strings" 8 | "unicode" 9 | ) 10 | 11 | func hyphenate(val string) string { 12 | return strings.ReplaceAll(val, "_", "-") 13 | } 14 | 15 | func snakeCase(val string) string { 16 | // don't you think this function should be in the standard library? 17 | // seems useful 18 | if len(val) <= 1 { 19 | return strings.ToLower(val) 20 | } 21 | runes := []rune(val) 22 | rv := make([]rune, 0, len(runes)) 23 | for i := range runes { 24 | rv = append(rv, unicode.ToLower(runes[i])) 25 | if i < len(runes)-1 && 26 | unicode.IsLower(runes[i]) && 27 | unicode.IsUpper(runes[i+1]) { 28 | // lower-to-uppercase case 29 | rv = append(rv, '_') 30 | } else if i < len(runes)-2 && 31 | unicode.IsUpper(runes[i]) && 32 | unicode.IsUpper(runes[i+1]) && 33 | unicode.IsLower(runes[i+2]) { 34 | // end-of-acronym case 35 | rv = append(rv, '_') 36 | } 37 | } 38 | return string(rv) 39 | } 40 | -------------------------------------------------------------------------------- /cfgstruct/case_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package cfgstruct 5 | 6 | import ( 7 | "testing" 8 | ) 9 | 10 | func TestSnakeCase(t *testing.T) { 11 | for _, test := range []struct { 12 | input, expected string 13 | }{ 14 | {"CoolBeans", "cool_beans"}, 15 | {"coolBeans", "cool_beans"}, 16 | {"JSONBeans", "json_beans"}, 17 | {"JSONBeAns", "json_be_ans"}, 18 | {"JSONBeANS", "json_be_ans"}, 19 | {"coolbeans", "coolbeans"}, 20 | {"COOLBEANS", "coolbeans"}, 21 | {"CoolJSON", "cool_json"}, 22 | {"CoolJSONBeans", "cool_json_beans"}, 23 | } { 24 | actual := snakeCase(test.input) 25 | if actual != test.expected { 26 | t.Logf("expected %#v but got %#v", test.expected, actual) 27 | t.Fail() 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /cfgstruct/flags.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package cfgstruct 5 | 6 | import ( 7 | "time" 8 | 9 | "github.com/spf13/pflag" 10 | ) 11 | 12 | // FlagSet is an interface that matches *pflag.FlagSet. 13 | type FlagSet interface { 14 | BoolVar(p *bool, name string, value bool, usage string) 15 | IntVar(p *int, name string, value int, usage string) 16 | Int64Var(p *int64, name string, value int64, usage string) 17 | UintVar(p *uint, name string, value uint, usage string) 18 | Uint64Var(p *uint64, name string, value uint64, usage string) 19 | DurationVar(p *time.Duration, name string, value time.Duration, usage string) 20 | Float64Var(p *float64, name string, value float64, usage string) 21 | StringVar(p *string, name string, value string, usage string) 22 | StringArrayVar(p *[]string, name string, value []string, usage string) 23 | StringSliceVar(p *[]string, name string, value []string, usage string) 24 | Var(val pflag.Value, name string, usage string) 25 | MarkHidden(name string) error 26 | } 27 | 28 | var _ FlagSet = (*pflag.FlagSet)(nil) 29 | -------------------------------------------------------------------------------- /context2/cancel_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package context2_test 5 | 6 | import ( 7 | "context" 8 | "io" 9 | "testing" 10 | "time" 11 | 12 | "github.com/stretchr/testify/require" 13 | 14 | "storj.io/common/context2" 15 | "storj.io/common/testcontext" 16 | ) 17 | 18 | func TestCustomCancel(t *testing.T) { 19 | t.Parallel() 20 | ctx := testcontext.New(t) 21 | 22 | sub, cancel := context2.WithCustomCancel(ctx) 23 | 24 | require.NoError(t, sub.Err()) 25 | 26 | go func() { 27 | cancel(io.EOF) 28 | }() 29 | <-sub.Done() 30 | 31 | require.Equal(t, io.EOF, sub.Err()) 32 | } 33 | 34 | func TestCustomCancel_ParentCancel(t *testing.T) { 35 | t.Parallel() 36 | ctx := testcontext.New(t) 37 | 38 | tx, c := context.WithTimeout(ctx, 50*time.Millisecond) 39 | defer c() 40 | 41 | sub, c2 := context2.WithCustomCancel(tx) 42 | defer c2(nil) 43 | 44 | time.Sleep(100 * time.Millisecond) 45 | require.Error(t, sub.Err()) 46 | } 47 | -------------------------------------------------------------------------------- /context2/nocancel.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package context2 contains utilities for contexts. 5 | package context2 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | "time" 11 | ) 12 | 13 | // WithoutCancellation returns a context that does not propagate Done message 14 | // down to children. However, Values are propagated. 15 | func WithoutCancellation(ctx context.Context) context.Context { 16 | return noCancelContext{ctx} 17 | } 18 | 19 | type noCancelContext struct { 20 | ctx context.Context 21 | } 22 | 23 | // Deadline returns the time when work done on behalf of this context 24 | // should be canceled. 25 | func (noCancelContext) Deadline() (deadline time.Time, ok bool) { 26 | return time.Time{}, false 27 | } 28 | 29 | // Done returns empty channel. 30 | func (noCancelContext) Done() <-chan struct{} { 31 | return nil 32 | } 33 | 34 | // Err always returns nil. 35 | func (noCancelContext) Err() error { 36 | return nil 37 | } 38 | 39 | // String returns string. 40 | func (ctx noCancelContext) String() string { 41 | return fmt.Sprintf("no cancel (%s)", ctx.ctx) 42 | } 43 | 44 | // Value returns the value associated with this context for key, or nil 45 | // if no value is associated with key. Successive calls to Value with 46 | // the same key returns the same result. 47 | func (ctx noCancelContext) Value(key any) any { 48 | return ctx.ctx.Value(key) 49 | } 50 | 51 | // WithRetimeout allows to create a new timeout for a potentially cancelled context. 52 | func WithRetimeout(ctx context.Context, duration time.Duration) (context.Context, context.CancelFunc) { 53 | return context.WithTimeout(WithoutCancellation(ctx), duration) 54 | } 55 | -------------------------------------------------------------------------------- /context2/nocancel_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package context2_test 5 | 6 | import ( 7 | "context" 8 | "testing" 9 | "time" 10 | 11 | "github.com/stretchr/testify/require" 12 | 13 | "storj.io/common/context2" 14 | "storj.io/common/testcontext" 15 | ) 16 | 17 | func TestWithoutCancellation(t *testing.T) { 18 | t.Parallel() 19 | ctx := testcontext.New(t) 20 | 21 | parent, cancel := context.WithCancel(ctx) 22 | cancel() 23 | 24 | without := context2.WithoutCancellation(parent) 25 | require.Equal(t, error(nil), without.Err()) 26 | require.Equal(t, (<-chan struct{})(nil), without.Done()) 27 | } 28 | 29 | func TestWithRetimeout(t *testing.T) { 30 | t.Parallel() 31 | ctx := testcontext.New(t) 32 | 33 | parent, cancel := context.WithCancel(ctx) 34 | cancel() 35 | 36 | start := time.Now() 37 | subctx, subcancel := context2.WithRetimeout(parent, 3*time.Second) 38 | <-subctx.Done() 39 | finish := time.Now() 40 | subcancel() 41 | 42 | require.Greater(t, finish.Sub(start), time.Second) 43 | } 44 | -------------------------------------------------------------------------------- /debug/common.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package debug 5 | 6 | import "github.com/zeebo/errs" 7 | 8 | // Error is default error class for debug package. 9 | var Error = errs.Class("debug") 10 | -------------------------------------------------------------------------------- /debug/crawlspace/crawlspace.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2024 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package crawlspace adds a way for other packages to inject values into 5 | // crawlspace sessions. 6 | package crawlspace 7 | 8 | import ( 9 | "maps" 10 | "reflect" 11 | "sync" 12 | 13 | "github.com/jtolio/crawlspace/reflectlang" 14 | ) 15 | 16 | var ( 17 | mu sync.Mutex 18 | globalEnv = reflectlang.Environment{} 19 | ) 20 | 21 | // Register adds this type to crawlspace environments that use Apply. 22 | func Register(name string, val any) { 23 | mu.Lock() 24 | defer mu.Unlock() 25 | globalEnv[name] = reflect.ValueOf(val) 26 | } 27 | 28 | // Apply adds all registered things to a crawlspace environment. 29 | func Apply(env reflectlang.Environment) { 30 | mu.Lock() 31 | defer mu.Unlock() 32 | maps.Copy(env, globalEnv) 33 | } 34 | -------------------------------------------------------------------------------- /debug/error.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package debug 5 | 6 | import ( 7 | "github.com/spacemonkeygo/monkit/v3" 8 | 9 | "storj.io/common/rpc/rpcstatus" 10 | ) 11 | 12 | func init() { 13 | monkit.AddErrorNameHandler(func(err error) (string, bool) { 14 | var code uint64 15 | forLoop: 16 | for range 100 { 17 | if v, ok := err.(interface{ Name() (string, bool) }); ok { 18 | if cls, ok := v.Name(); ok && cls != "" { 19 | return cls, true 20 | } 21 | } 22 | if v, ok := err.(interface{ Code() uint64 }); ok { 23 | if code == 0 { 24 | code = v.Code() 25 | } 26 | } 27 | switch v := err.(type) { //nolint: errorlint // this is a custom unwrap loop 28 | case interface{ Cause() error }: 29 | err = v.Cause() 30 | case interface{ Unwrap() error }: 31 | err = v.Unwrap() 32 | case interface{ Unwrap() []error }: 33 | errs := v.Unwrap() 34 | if len(errs) == 0 { 35 | break 36 | } 37 | err = errs[0] 38 | default: 39 | break forLoop 40 | } 41 | } 42 | if code != 0 { 43 | return "drpc_" + rpcstatus.StatusCode(code).String(), true 44 | } 45 | return "", false 46 | }) 47 | } 48 | -------------------------------------------------------------------------------- /debug/pcap_other.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2024 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | //go:build !linux 5 | 6 | package debug 7 | 8 | import ( 9 | "context" 10 | "sync/atomic" 11 | ) 12 | 13 | func capturePackets(ctx context.Context, stop *atomic.Bool) {} 14 | -------------------------------------------------------------------------------- /debug/top.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package debug 5 | 6 | import ( 7 | "bytes" 8 | "fmt" 9 | "net/http" 10 | "sync" 11 | "time" 12 | 13 | "storj.io/common/eventstat" 14 | ) 15 | 16 | // Top is a specific registry exposed over the /top endpoint. 17 | // can be used to publish any high-cardinality event on local HTTP. 18 | var Top eventstat.Registry 19 | 20 | var lastReset time.Time 21 | 22 | var mu sync.Mutex 23 | 24 | // ServeTop returns all the counters from memory since the last (global) request. 25 | func ServeTop(w http.ResponseWriter, r *http.Request) { 26 | mu.Lock() 27 | defer mu.Unlock() 28 | buf := bytes.NewBuffer([]byte{}) 29 | buf.WriteString(fmt.Sprintf("since ~ %s\n", lastReset.Format(time.RFC3339))) 30 | lastReset = time.Now() 31 | Top.PublishAndReset(func(name string, tags eventstat.Tags, value float64) { 32 | buf.WriteString(fmt.Sprintf("%s %s %f\n", name, tags.String(), value)) 33 | }) 34 | 35 | _, err := w.Write(buf.Bytes()) 36 | if err != nil { 37 | w.WriteHeader(http.StatusInternalServerError) 38 | _, _ = w.Write([]byte(err.Error())) 39 | } 40 | w.WriteHeader(http.StatusOK) 41 | w.Header().Add("Content-Type", "text/plain") 42 | } 43 | -------------------------------------------------------------------------------- /debug/trace_disabled.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2024 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | //go:build !go1.22 5 | 6 | package debug 7 | 8 | const traceEnabled = false 9 | -------------------------------------------------------------------------------- /debug/trace_enabled.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2024 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | //go:build go1.22 5 | 6 | package debug 7 | 8 | const traceEnabled = true 9 | -------------------------------------------------------------------------------- /encryption/aesgcm_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package encryption 5 | 6 | import ( 7 | "bytes" 8 | "io" 9 | "testing" 10 | 11 | "storj.io/common/testrand" 12 | ) 13 | 14 | func TestAesGcm(t *testing.T) { 15 | key := testrand.Key() 16 | var firstNonce AESGCMNonce 17 | testrand.Read(firstNonce[:]) 18 | 19 | encrypter, err := NewAESGCMEncrypter(&key, &firstNonce, 4*1024) 20 | if err != nil { 21 | t.Fatal(err) 22 | } 23 | 24 | data := testrand.BytesInt(encrypter.InBlockSize() * 10) 25 | encrypted := TransformReader(io.NopCloser(bytes.NewReader(data)), encrypter, 0) 26 | decrypter, err := NewAESGCMDecrypter(&key, &firstNonce, 4*1024) 27 | if err != nil { 28 | t.Fatal(err) 29 | } 30 | decrypted := TransformReader(encrypted, decrypter, 0) 31 | data2, err := io.ReadAll(decrypted) 32 | if err != nil { 33 | t.Fatal(err) 34 | } 35 | 36 | if !bytes.Equal(data, data2) { 37 | t.Fatalf("encryption/decryption failed") 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /encryption/bits.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package encryption 5 | 6 | // incrementBytes takes a byte slice buf and treats it like a little-endian 7 | // encoded unsigned integer. it adds amount to it (which must be nonnegative) 8 | // in place. if rollover happens (the most significant bytes don't fit 9 | // anymore), truncated is true. 10 | func incrementBytes(buf []byte, amount int64) (truncated bool, err error) { 11 | if amount < 0 { 12 | return false, Error.New("amount was negative") 13 | } 14 | 15 | idx := 0 16 | for amount > 0 && idx < len(buf) { 17 | var inc, prev byte 18 | inc, amount = byte(amount), amount>>8 19 | 20 | prev = buf[idx] 21 | buf[idx] += inc 22 | if buf[idx] < prev { 23 | amount++ 24 | } 25 | 26 | idx++ 27 | } 28 | 29 | return amount != 0, nil 30 | } 31 | -------------------------------------------------------------------------------- /encryption/common.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package encryption collects common cryptographic primitives needed for path and data encryption. 5 | package encryption 6 | 7 | import ( 8 | "github.com/zeebo/errs" 9 | ) 10 | 11 | // Error is the default encryption errs class. 12 | var Error = errs.Class("encryption") 13 | 14 | // ErrDecryptFailed is the errs class when the decryption fails. 15 | var ErrDecryptFailed = errs.Class("decryption failed, check encryption key") 16 | 17 | // ErrInvalidConfig is the errs class for invalid configuration. 18 | var ErrInvalidConfig = errs.Class("invalid encryption configuration") 19 | -------------------------------------------------------------------------------- /encryption/password.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package encryption 5 | 6 | import ( 7 | "crypto/hmac" 8 | "crypto/sha256" 9 | 10 | "github.com/zeebo/errs" 11 | "golang.org/x/crypto/argon2" 12 | 13 | "storj.io/common/memory" 14 | "storj.io/common/storj" 15 | "storj.io/common/sync2/race2" 16 | ) 17 | 18 | func sha256hmac(key, data []byte) ([]byte, error) { 19 | h := hmac.New(sha256.New, key) 20 | if _, err := h.Write(data); err != nil { 21 | return nil, err 22 | } 23 | return h.Sum(nil), nil 24 | } 25 | 26 | // DeriveRootKey derives a root key for some path using the salt for the bucket and 27 | // a password from the user. See the password key derivation design doc. 28 | func DeriveRootKey(password, salt []byte, path storj.Path, argon2Threads uint8) (*storj.Key, error) { 29 | race2.ReadSlice(password) 30 | race2.ReadSlice(salt) 31 | 32 | mixedSalt, err := sha256hmac(password, salt) 33 | if err != nil { 34 | return nil, err 35 | } 36 | 37 | pathSalt, err := sha256hmac(mixedSalt, []byte(path)) 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | // use a time of 1, 64MB of ram, and all of the cores. 43 | keyData := argon2.IDKey(password, pathSalt, 1, uint32(64*memory.MiB/memory.KiB), argon2Threads, 32) 44 | if len(keyData) != len(storj.Key{}) { 45 | return nil, errs.New("invalid output from argon2id") 46 | } 47 | 48 | var key storj.Key 49 | copy(key[:], keyData) 50 | return &key, nil 51 | } 52 | -------------------------------------------------------------------------------- /encryption/password_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package encryption 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestDeriveRootKey(t *testing.T) { 13 | // ensure that we can derive with no errors 14 | _, err := DeriveRootKey([]byte("password"), []byte("salt"), "", 8) 15 | assert.NoError(t, err) 16 | _, err = DeriveRootKey([]byte("password"), []byte("salt"), "any/path", 8) 17 | assert.NoError(t, err) 18 | } 19 | -------------------------------------------------------------------------------- /encryption/secretbox_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package encryption 5 | 6 | import ( 7 | "bytes" 8 | "io" 9 | "testing" 10 | 11 | "storj.io/common/testrand" 12 | ) 13 | 14 | func TestSecretbox(t *testing.T) { 15 | key := testrand.Key() 16 | firstNonce := testrand.Nonce() 17 | 18 | encrypter, err := NewSecretboxEncrypter(&key, &firstNonce, 4*1024) 19 | if err != nil { 20 | t.Fatal(err) 21 | } 22 | 23 | data := testrand.BytesInt(encrypter.InBlockSize() * 10) 24 | 25 | encrypted := TransformReader(io.NopCloser(bytes.NewReader(data)), encrypter, 0) 26 | 27 | decrypter, err := NewSecretboxDecrypter(&key, &firstNonce, 4*1024) 28 | if err != nil { 29 | t.Fatal(err) 30 | } 31 | decrypted := TransformReader(encrypted, decrypter, 0) 32 | 33 | data2, err := io.ReadAll(decrypted) 34 | if err != nil { 35 | t.Fatal(err) 36 | } 37 | if !bytes.Equal(data, data2) { 38 | t.Fatalf("encryption/decryption failed") 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /errs2/collect.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package errs2 5 | 6 | import ( 7 | "time" 8 | 9 | "github.com/zeebo/errs" 10 | ) 11 | 12 | // Collect returns first error from channel and all errors that happen within duration. 13 | func Collect(errch chan error, duration time.Duration) error { 14 | errch = discardNil(errch) 15 | errlist := []error{<-errch} 16 | timeout := time.After(duration) 17 | for { 18 | select { 19 | case err := <-errch: 20 | errlist = append(errlist, err) 21 | case <-timeout: 22 | return errs.Combine(errlist...) 23 | } 24 | } 25 | } 26 | 27 | // discard nil errors that are returned from services. 28 | func discardNil(ch chan error) chan error { 29 | r := make(chan error) 30 | go func() { 31 | for err := range ch { 32 | if err == nil { 33 | continue 34 | } 35 | r <- err 36 | } 37 | close(r) 38 | }() 39 | return r 40 | } 41 | -------------------------------------------------------------------------------- /errs2/collect_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package errs2_test 5 | 6 | import ( 7 | "testing" 8 | "time" 9 | 10 | "github.com/stretchr/testify/assert" 11 | "github.com/zeebo/errs" 12 | 13 | "storj.io/common/errs2" 14 | ) 15 | 16 | func TestCollectSingleError(t *testing.T) { 17 | errchan := make(chan error) 18 | defer close(errchan) 19 | 20 | go func() { 21 | errchan <- errs.New("error") 22 | }() 23 | 24 | err := errs2.Collect(errchan, 1*time.Second) 25 | assert.Error(t, err) 26 | assert.Equal(t, err.Error(), "error") 27 | } 28 | 29 | func TestCollectMultipleError(t *testing.T) { 30 | errchan := make(chan error) 31 | defer close(errchan) 32 | 33 | go func() { 34 | errchan <- errs.New("error1") 35 | errchan <- errs.New("error2") 36 | errchan <- errs.New("error3") 37 | }() 38 | 39 | err := errs2.Collect(errchan, 1*time.Second) 40 | assert.Error(t, err) 41 | assert.Equal(t, err.Error(), "error1; error2; error3") 42 | } 43 | -------------------------------------------------------------------------------- /errs2/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package errs2 collects common error handling functions. 5 | package errs2 6 | -------------------------------------------------------------------------------- /errs2/group.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package errs2 5 | 6 | import "sync" 7 | 8 | // Group is a collection of goroutines working on subtasks that are part of 9 | // the same overall task. 10 | type Group struct { 11 | wg sync.WaitGroup 12 | mu sync.Mutex 13 | errors []error 14 | } 15 | 16 | // Go calls the given function in a new goroutine. 17 | func (group *Group) Go(f func() error) { 18 | group.wg.Add(1) 19 | 20 | go func() { 21 | defer group.wg.Done() 22 | 23 | if err := f(); err != nil { 24 | group.mu.Lock() 25 | defer group.mu.Unlock() 26 | 27 | group.errors = append(group.errors, err) 28 | } 29 | }() 30 | } 31 | 32 | // Wait blocks until all function calls from the Go method have returned, then 33 | // returns all errors (if any) from them. 34 | func (group *Group) Wait() []error { 35 | group.wg.Wait() 36 | 37 | return group.errors 38 | } 39 | -------------------------------------------------------------------------------- /errs2/group_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package errs2_test 5 | 6 | import ( 7 | "errors" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/require" 11 | 12 | "storj.io/common/errs2" 13 | ) 14 | 15 | func TestGroup(t *testing.T) { 16 | group := errs2.Group{} 17 | group.Go(func() error { 18 | return errors.New("first") 19 | }) 20 | group.Go(func() error { 21 | return nil 22 | }) 23 | group.Go(func() error { 24 | return errors.New("second") 25 | }) 26 | group.Go(func() error { 27 | return errors.New("third") 28 | }) 29 | 30 | allErrors := group.Wait() 31 | require.Len(t, allErrors, 3) 32 | } 33 | -------------------------------------------------------------------------------- /errs2/ignore.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package errs2 5 | 6 | import ( 7 | "context" 8 | 9 | "github.com/zeebo/errs" 10 | 11 | "storj.io/common/rpc/rpcstatus" 12 | ) 13 | 14 | // IsCanceled returns true, when the error is a cancellation. 15 | func IsCanceled(err error) bool { 16 | return errs.IsFunc(err, func(err error) bool { 17 | return err == context.Canceled || //nolint:errorlint, goerr113, err113 18 | rpcstatus.Code(err) == rpcstatus.Canceled 19 | }) 20 | } 21 | 22 | // IgnoreCanceled returns nil, when the operation was about canceling. 23 | func IgnoreCanceled(err error) error { 24 | if IsCanceled(err) { 25 | return nil 26 | } 27 | return err 28 | } 29 | -------------------------------------------------------------------------------- /errs2/rpc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package errs2 5 | 6 | import ( 7 | "github.com/zeebo/errs" 8 | 9 | "storj.io/common/rpc/rpcstatus" 10 | ) 11 | 12 | // IsRPC checks if err contains an RPC error with the given status code. 13 | func IsRPC(err error, code rpcstatus.StatusCode) bool { 14 | return errs.IsFunc(err, func(err error) bool { 15 | return rpcstatus.Code(err) == code 16 | }) 17 | } 18 | -------------------------------------------------------------------------------- /eventstat/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package eventstat contains helper to create statistics about events with unbounded cardinality. 5 | package eventstat 6 | -------------------------------------------------------------------------------- /experiment/common.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package experiment implements feature flag propagation. 5 | package experiment 6 | 7 | import ( 8 | "context" 9 | "slices" 10 | "strings" 11 | ) 12 | 13 | type key int 14 | 15 | const ( 16 | contextKey key = iota 17 | drpcKey string = "experiment" 18 | ) 19 | 20 | // With registers the feature flag of an ongoing experiment. 21 | func With(ctx context.Context, exp string) context.Context { 22 | existingValue := ctx.Value(contextKey) 23 | if existingValue != nil { 24 | if s, ok := existingValue.(string); ok { 25 | return context.WithValue(ctx, contextKey, s+","+exp) 26 | } 27 | } 28 | return context.WithValue(ctx, contextKey, exp) 29 | } 30 | 31 | // Get returns the registered feature flags. 32 | func Get(ctx context.Context) []string { 33 | value := ctx.Value(contextKey) 34 | if value == nil { 35 | return []string{} 36 | } 37 | if s, ok := value.(string); ok { 38 | return strings.Split(s, ",") 39 | } 40 | return []string{} 41 | } 42 | 43 | // Has checks if the experiment registered to the comma separated list. 44 | func Has(ctx context.Context, exp string) bool { 45 | value := ctx.Value(contextKey) 46 | if value == nil { 47 | return false 48 | } 49 | if s, ok := value.(string); ok { 50 | if slices.Contains(strings.Split(s, ","), exp) { 51 | return true 52 | } 53 | } 54 | return false 55 | } 56 | -------------------------------------------------------------------------------- /experiment/common_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package experiment 5 | 6 | import ( 7 | "context" 8 | "sort" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestGetExperimentInContext(t *testing.T) { 15 | ctx := context.Background() 16 | require.False(t, Has(ctx, "first")) 17 | 18 | ctx = With(ctx, "first") 19 | require.True(t, Has(ctx, "first")) 20 | require.False(t, Has(ctx, "second")) 21 | 22 | ctx = With(ctx, "second") 23 | require.True(t, Has(ctx, "first")) 24 | require.True(t, Has(ctx, "second")) 25 | 26 | exps := Get(ctx) 27 | sort.Strings(exps) 28 | require.Equal(t, []string{"first", "second"}, exps) 29 | 30 | } 31 | -------------------------------------------------------------------------------- /experiment/export.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package experiment 5 | 6 | import ( 7 | "context" 8 | 9 | "storj.io/common/rpc/rpcpool" 10 | "storj.io/drpc" 11 | "storj.io/drpc/drpcmetadata" 12 | ) 13 | 14 | // Wrapper wraps a Conn with experimental feature flag information. 15 | type Wrapper struct { 16 | rpcpool.Conn 17 | } 18 | 19 | // NewConnWrapper creates a new instance of the wrapper. 20 | func NewConnWrapper(conn rpcpool.Conn) *Wrapper { 21 | return &Wrapper{ 22 | conn, 23 | } 24 | } 25 | 26 | // Invoke implements drpc.Conn's Invoke method with feature flag information injected into the context. 27 | func (c *Wrapper) Invoke(ctx context.Context, rpc string, enc drpc.Encoding, in drpc.Message, out drpc.Message) (err error) { 28 | return c.Conn.Invoke(c.trace(ctx), rpc, enc, in, out) 29 | } 30 | 31 | // NewStream implements drpc.Conn's NewStream method with experiment feature flag injected into the context. 32 | func (c *Wrapper) NewStream(ctx context.Context, rpc string, enc drpc.Encoding) (_ drpc.Stream, err error) { 33 | return c.Conn.NewStream(c.trace(ctx), rpc, enc) 34 | } 35 | 36 | // trace injects tracing related information into the context. 37 | func (c *Wrapper) trace(ctx context.Context) context.Context { 38 | if exp := ctx.Value(contextKey); exp != nil { 39 | if exps, ok := exp.(string); ok { 40 | return drpcmetadata.Add(ctx, drpcKey, exps) 41 | } 42 | } 43 | return ctx 44 | } 45 | -------------------------------------------------------------------------------- /experiment/import.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package experiment 5 | 6 | import ( 7 | "context" 8 | 9 | "storj.io/drpc" 10 | "storj.io/drpc/drpcmetadata" 11 | ) 12 | 13 | type streamWrapper struct { 14 | drpc.Stream 15 | ctx context.Context 16 | } 17 | 18 | func (s *streamWrapper) GetStream() drpc.Stream { return s.Stream } 19 | func (s *streamWrapper) Context() context.Context { return s.ctx } 20 | 21 | // Handler implements drpc handler interface to extract experiment feature flag. 22 | type Handler struct { 23 | handler drpc.Handler 24 | } 25 | 26 | // NewHandler returns a new instance of Handler. 27 | func NewHandler(handler drpc.Handler) *Handler { 28 | return &Handler{ 29 | handler: handler, 30 | } 31 | } 32 | 33 | // HandleRPC copies experiment feature flag from drpcmeta to context. 34 | func (handler *Handler) HandleRPC(stream drpc.Stream, rpc string) (err error) { 35 | streamCtx := stream.Context() 36 | 37 | metadata, ok := drpcmetadata.Get(streamCtx) 38 | if ok { 39 | if exp, found := metadata[drpcKey]; found { 40 | streamCtx = context.WithValue(streamCtx, contextKey, exp) 41 | } 42 | } 43 | return handler.handler.HandleRPC(&streamWrapper{Stream: stream, ctx: streamCtx}, rpc) 44 | } 45 | -------------------------------------------------------------------------------- /fpath/atomic.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package fpath 5 | 6 | import ( 7 | "os" 8 | "path/filepath" 9 | 10 | "github.com/zeebo/errs" 11 | ) 12 | 13 | // AtomicWriteFile is a helper to atomically write the data to the outfile. 14 | func AtomicWriteFile(outfile string, data []byte, _ os.FileMode) (err error) { 15 | // TODO: provide better atomicity guarantees, like fsyncing the parent 16 | // directory and, on windows, using MoveFileEx with MOVEFILE_WRITE_THROUGH. 17 | 18 | fh, err := os.CreateTemp(filepath.Dir(outfile), filepath.Base(outfile)) 19 | if err != nil { 20 | return errs.Wrap(err) 21 | } 22 | needsClose, needsRemove := true, true 23 | 24 | defer func() { 25 | if needsClose { 26 | err = errs.Combine(err, errs.Wrap(fh.Close())) 27 | } 28 | if needsRemove { 29 | err = errs.Combine(err, errs.Wrap(os.Remove(fh.Name()))) 30 | } 31 | }() 32 | 33 | if _, err := fh.Write(data); err != nil { 34 | return errs.Wrap(err) 35 | } 36 | 37 | needsClose = false 38 | if err := fh.Close(); err != nil { 39 | return errs.Wrap(err) 40 | } 41 | 42 | if err := os.Rename(fh.Name(), outfile); err != nil { 43 | return errs.Wrap(err) 44 | } 45 | needsRemove = false 46 | 47 | return nil 48 | } 49 | -------------------------------------------------------------------------------- /fpath/atomic_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package fpath_test 5 | 6 | import ( 7 | "os" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/require" 11 | 12 | "storj.io/common/fpath" 13 | "storj.io/common/testcontext" 14 | ) 15 | 16 | func TestAtomicWriteFile(t *testing.T) { 17 | ctx := testcontext.New(t) 18 | defer ctx.Cleanup() 19 | 20 | err := fpath.AtomicWriteFile(ctx.File("example.txt"), []byte{1, 2, 3}, 0600) 21 | require.NoError(t, err) 22 | 23 | data, err := os.ReadFile(ctx.File("example.txt")) 24 | require.NoError(t, err) 25 | require.Equal(t, []byte{1, 2, 3}, data) 26 | } 27 | -------------------------------------------------------------------------------- /fpath/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package fpath implements cross-platform file and object path handling. 5 | package fpath 6 | -------------------------------------------------------------------------------- /fpath/path_unix_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | //go:build linux || darwin 5 | 6 | package fpath 7 | 8 | import ( 9 | "testing" 10 | ) 11 | 12 | func TestLocalPathUnix(t *testing.T) { 13 | for i, tt := range []struct { 14 | url string 15 | base string 16 | }{ 17 | { 18 | url: "/", 19 | base: "/", 20 | }, 21 | { 22 | url: "//", 23 | base: "/", 24 | }, 25 | { 26 | url: "/home/user/folder", 27 | base: "folder", 28 | }, 29 | { 30 | url: "/home/user/folder/", 31 | base: "folder", 32 | }, 33 | { 34 | url: "/home/user/folder/file.sh", 35 | base: "file.sh", 36 | }, 37 | { 38 | url: "//home//user//folder//file.sh", 39 | base: "file.sh", 40 | }, 41 | } { 42 | testLocalPath(t, tt.url, tt.base, i) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /fpath/temp_data.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // TODO maybe there is better place for this 5 | 6 | package fpath 7 | 8 | import "context" 9 | 10 | // The key type is unexported to prevent collisions with context keys defined in 11 | // other packages. 12 | type key int 13 | 14 | // temp is the context key for temp struct. 15 | const tempKey key = 0 16 | 17 | type temp struct { 18 | inmemory bool 19 | directory string 20 | } 21 | 22 | // WithTempData creates context with information how store temporary data, in memory or on disk. 23 | func WithTempData(ctx context.Context, directory string, inmemory bool) context.Context { 24 | temp := temp{ 25 | inmemory: inmemory, 26 | directory: directory, 27 | } 28 | return context.WithValue(ctx, tempKey, temp) 29 | } 30 | 31 | // GetTempData returns if temporary data should be stored in memory or on disk. 32 | func GetTempData(ctx context.Context) (string, bool, bool) { 33 | tempValue, ok := ctx.Value(tempKey).(temp) 34 | if !ok { 35 | return "", false, false 36 | } 37 | return tempValue.directory, tempValue.inmemory, ok 38 | } 39 | -------------------------------------------------------------------------------- /grant/internal/pb/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package pb contains protobuf definitions for Storj peers. 5 | // Run protolock in the storj.io/common module root to update the lock file. 6 | // E.g. `protolock commit`, no other parameters necessary. 7 | package pb 8 | 9 | //go:generate protoc --lint_out=. --pico_out=paths=source_relative:. -I=../../../pb ../../../pb/encryption.proto ../../../pb/encryption_access.proto ../../../pb/scope.proto 10 | -------------------------------------------------------------------------------- /grant/internal/pb/encryption.pico.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-pico. DO NOT EDIT. 2 | // source: encryption.proto 3 | // 4 | // versions: 5 | // protoc-gen-pico: v0.0.4 6 | // protoc: v6.30.2 7 | 8 | package pb 9 | 10 | import ( 11 | picobuf "storj.io/picobuf" 12 | strconv "strconv" 13 | ) 14 | 15 | type CipherSuite int32 16 | 17 | const ( 18 | CipherSuite_ENC_UNSPECIFIED CipherSuite = 0 19 | CipherSuite_ENC_NULL CipherSuite = 1 20 | CipherSuite_ENC_AESGCM CipherSuite = 2 21 | CipherSuite_ENC_SECRETBOX CipherSuite = 3 22 | ) 23 | 24 | func (m CipherSuite) String() string { 25 | switch m { 26 | case CipherSuite_ENC_UNSPECIFIED: 27 | return "ENC_UNSPECIFIED" 28 | case CipherSuite_ENC_NULL: 29 | return "ENC_NULL" 30 | case CipherSuite_ENC_AESGCM: 31 | return "ENC_AESGCM" 32 | case CipherSuite_ENC_SECRETBOX: 33 | return "ENC_SECRETBOX" 34 | default: 35 | return "CipherSuite(" + strconv.Itoa(int(m)) + ")" 36 | } 37 | } 38 | 39 | type EncryptionParameters struct { 40 | CipherSuite CipherSuite `json:"cipher_suite,omitempty"` 41 | BlockSize int64 `json:"block_size,omitempty"` 42 | } 43 | 44 | func (m *EncryptionParameters) Encode(c *picobuf.Encoder) bool { 45 | if m == nil { 46 | return false 47 | } 48 | c.Int32(1, (*int32)(&m.CipherSuite)) 49 | c.Int64(2, &m.BlockSize) 50 | return true 51 | } 52 | 53 | func (m *EncryptionParameters) Decode(c *picobuf.Decoder) { 54 | if m == nil { 55 | return 56 | } 57 | c.Int32(1, (*int32)(&m.CipherSuite)) 58 | c.Int64(2, &m.BlockSize) 59 | } 60 | -------------------------------------------------------------------------------- /grant/internal/pb/scope.pico.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-pico. DO NOT EDIT. 2 | // source: scope.proto 3 | // 4 | // versions: 5 | // protoc-gen-pico: v0.0.4 6 | // protoc: v6.30.2 7 | 8 | package pb 9 | 10 | import ( 11 | picobuf "storj.io/picobuf" 12 | ) 13 | 14 | type Scope struct { 15 | SatelliteAddr string `json:"satellite_addr,omitempty"` 16 | ApiKey []byte `json:"api_key,omitempty"` 17 | EncryptionAccess *EncryptionAccess `json:"encryption_access,omitempty"` 18 | } 19 | 20 | func (m *Scope) Encode(c *picobuf.Encoder) bool { 21 | if m == nil { 22 | return false 23 | } 24 | c.String(1, &m.SatelliteAddr) 25 | c.Bytes(2, &m.ApiKey) 26 | c.Message(3, m.EncryptionAccess.Encode) 27 | return true 28 | } 29 | 30 | func (m *Scope) Decode(c *picobuf.Decoder) { 31 | if m == nil { 32 | return 33 | } 34 | c.String(1, &m.SatelliteAddr) 35 | c.Bytes(2, &m.ApiKey) 36 | c.Message(3, func(c *picobuf.Decoder) { 37 | if m.EncryptionAccess == nil { 38 | m.EncryptionAccess = new(EncryptionAccess) 39 | } 40 | m.EncryptionAccess.Decode(c) 41 | }) 42 | } 43 | -------------------------------------------------------------------------------- /identity/common.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package identity 5 | 6 | import ( 7 | "github.com/spacemonkeygo/monkit/v3" 8 | "github.com/zeebo/errs" 9 | ) 10 | 11 | var ( 12 | mon = monkit.Package() 13 | 14 | // Error is a identity error. 15 | Error = errs.Class("identity") 16 | ) 17 | -------------------------------------------------------------------------------- /identity/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package identity implements CA and Peer identity management and generation. 5 | package identity 6 | -------------------------------------------------------------------------------- /identity/testidentity/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package testidentity contains pregenerated identities for testing. 5 | package testidentity 6 | -------------------------------------------------------------------------------- /identity/testidentity/pregenerated_identities_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package testidentity 5 | 6 | import ( 7 | "crypto/x509" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/require" 12 | 13 | "storj.io/common/identity" 14 | "storj.io/common/peertls" 15 | "storj.io/common/storj" 16 | ) 17 | 18 | func TestPregeneratedIdentity(t *testing.T) { 19 | IdentityVersionsTest(t, func(t *testing.T, version storj.IDVersion, ident *identity.FullIdentity) { 20 | assert.Equal(t, version.Number, ident.ID.Version().Number) 21 | 22 | caVersion, err := storj.IDVersionFromCert(ident.CA) 23 | require.NoError(t, err) 24 | assert.Equal(t, version.Number, caVersion.Number) 25 | 26 | chains := identity.ToChains(ident.Chain()) 27 | err = peertls.VerifyPeerCertChains(nil, chains) 28 | assert.NoError(t, err) 29 | }) 30 | } 31 | 32 | func TestPregeneratedSignedIdentity(t *testing.T) { 33 | SignedIdentityVersionsTest(t, func(t *testing.T, version storj.IDVersion, ident *identity.FullIdentity) { 34 | assert.Equal(t, version.Number, ident.ID.Version().Number) 35 | 36 | caVersion, err := storj.IDVersionFromCert(ident.CA) 37 | require.NoError(t, err) 38 | assert.Equal(t, version.Number, caVersion.Number) 39 | 40 | chains := identity.ToChains(ident.Chain()) 41 | err = peertls.VerifyPeerCertChains(nil, chains) 42 | assert.NoError(t, err) 43 | 44 | signer := NewPregeneratedSigner(ident.ID.Version()) 45 | err = peertls.VerifyCAWhitelist([]*x509.Certificate{signer.Cert})(nil, chains) 46 | assert.NoError(t, err) 47 | }) 48 | } 49 | -------------------------------------------------------------------------------- /internal/hmacsha512/GO_LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2009 The Go Authors. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /internal/hmacsha512/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package hmacsha512 contains an inlined an optimized version of hmac+sha512. 5 | // Unfortunately, this requires exposing some of the details from crypto/sha512. 6 | package hmacsha512 7 | 8 | // Currently vendored crypto/sha512 version is go1.22.2 9 | -------------------------------------------------------------------------------- /internal/hmacsha512/hmac.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package hmacsha512 5 | 6 | // Partial is an in-progress HMAC calculation. 7 | type Partial struct { 8 | outer digest 9 | inner digest 10 | isave digest 11 | osave digest 12 | } 13 | 14 | // New creates a new HMAC-SHA512 calculator. 15 | // It only supports keys that are smaller than sha512.BlockSize. 16 | func New(key []byte) Partial { 17 | p := Partial{} 18 | p.Init(key) 19 | return p 20 | } 21 | 22 | // Init initializes the state with the specified key. 23 | func (hm *Partial) Init(key []byte) { 24 | if len(key) > BlockSize { 25 | hm.outer.Reset() 26 | hm.outer.Write(key) 27 | newKey := hm.outer.FinishAndSum() 28 | key = newKey[:] 29 | } 30 | 31 | hm.outer.Reset() 32 | hm.inner.Reset() 33 | 34 | var ipad [BlockSize]byte 35 | var opad [BlockSize]byte 36 | copy(ipad[:], key) 37 | copy(opad[:], key) 38 | 39 | for i := range ipad { 40 | ipad[i] ^= 0x36 41 | } 42 | for i := range opad { 43 | opad[i] ^= 0x5c 44 | } 45 | 46 | hm.inner.Write(ipad[:]) 47 | hm.outer.Write(opad[:]) 48 | 49 | hm.isave = hm.inner 50 | hm.osave = hm.outer 51 | } 52 | 53 | // Write appends message to the HMAC calculation. 54 | func (hm *Partial) Write(p []byte) { 55 | hm.inner.Write(p) 56 | } 57 | 58 | // SumAndReset calculates the sum so far and resets the state. 59 | func (hm *Partial) SumAndReset() [Size]byte { 60 | in := hm.inner.FinishAndSum() 61 | hm.inner = hm.isave 62 | hm.outer = hm.osave 63 | hm.outer.Write(in[:]) 64 | return hm.outer.FinishAndSum() 65 | } 66 | -------------------------------------------------------------------------------- /internal/hmacsha512/hmac_fuzz_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | //go:build go1.18 5 | // +build go1.18 6 | 7 | package hmacsha512_test 8 | 9 | import ( 10 | "crypto/hmac" 11 | "crypto/sha512" 12 | "testing" 13 | 14 | "github.com/stretchr/testify/require" 15 | 16 | "storj.io/common/internal/hmacsha512" 17 | ) 18 | 19 | func FuzzPartial(f *testing.F) { 20 | pid := PieceID{} 21 | f.Add([]byte{}, []byte{}) 22 | f.Add(pid[:], []byte{1, 2, 3, 4}) 23 | 24 | f.Fuzz(func(t *testing.T, key []byte, data []byte) { 25 | var local hmacsha512.Partial 26 | local.Init(key) 27 | local.Write(data) 28 | actual1 := local.SumAndReset() 29 | local.Write(data) 30 | actual2 := local.SumAndReset() 31 | 32 | slow := hmac.New(sha512.New, key) 33 | _, _ = slow.Write(data) 34 | expected := slow.Sum(nil) 35 | 36 | require.Equal(t, expected, actual1[:]) 37 | require.Equal(t, expected, actual2[:]) 38 | }) 39 | } 40 | -------------------------------------------------------------------------------- /internal/hmacsha512/sha512_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See GO_LICENSE for copying information. 3 | 4 | package hmacsha512 5 | 6 | import "testing" 7 | 8 | func TestGenericBlock(t *testing.T) { 9 | // This is here to avoid linters complaining about blockGeneric being unused. 10 | var d digest 11 | d.Reset() 12 | var block [BlockSize]byte 13 | blockGeneric(&d, block[:]) 14 | } 15 | -------------------------------------------------------------------------------- /internal/hmacsha512/sha512block_amd64.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the GO_LICENSE file. 4 | 5 | //go:build amd64 6 | 7 | package hmacsha512 8 | 9 | import "golang.org/x/sys/cpu" 10 | 11 | //go:noescape 12 | func blockAVX2(dig *digest, p []byte) 13 | 14 | //go:noescape 15 | func blockAMD64(dig *digest, p []byte) 16 | 17 | var useAVX2 = cpu.X86.HasAVX2 && cpu.X86.HasBMI1 && cpu.X86.HasBMI2 18 | 19 | func block(dig *digest, p []byte) { 20 | if useAVX2 { 21 | blockAVX2(dig, p) 22 | } else { 23 | blockAMD64(dig, p) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /internal/hmacsha512/sha512block_arm64.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the GO_LICENSE file. 4 | 5 | package hmacsha512 6 | 7 | import "golang.org/x/sys/cpu" 8 | 9 | func block(dig *digest, p []byte) { 10 | if cpu.ARM64.HasSHA512 { 11 | blockAsm(dig, p) 12 | return 13 | } 14 | blockGeneric(dig, p) 15 | } 16 | 17 | //go:noescape 18 | func blockAsm(dig *digest, p []byte) 19 | -------------------------------------------------------------------------------- /internal/hmacsha512/sha512block_decl.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the GO_LICENSE file. 4 | 5 | //go:build s390x || ppc64le || ppc64 6 | 7 | package hmacsha512 8 | 9 | //go:noescape 10 | func block(dig *digest, p []byte) 11 | -------------------------------------------------------------------------------- /internal/hmacsha512/sha512block_generic.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the GO_LICENSE file. 4 | 5 | //go:build !amd64 && !arm64 && !s390x && !ppc64le && !ppc64 6 | 7 | package hmacsha512 8 | 9 | func block(dig *digest, p []byte) { 10 | blockGeneric(dig, p) 11 | } 12 | -------------------------------------------------------------------------------- /internal/hmacsha512/sha512block_s390x.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the GO_LICENSE file. 4 | 5 | package hmacsha512 6 | 7 | import "golang.org/x/sys/cpu" 8 | 9 | var useAsm = cpu.S390X.HasSHA512 10 | -------------------------------------------------------------------------------- /internal/hmacsha512/sha512block_s390x.s: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the GO_LICENSE file. 4 | 5 | #include "textflag.h" 6 | 7 | // func block(dig *digest, p []byte) 8 | TEXT ·block(SB), NOSPLIT|NOFRAME, $0-32 9 | MOVBZ ·useAsm(SB), R4 10 | LMG dig+0(FP), R1, R3 // R2 = &p[0], R3 = len(p) 11 | MOVBZ $3, R0 // SHA-512 function code 12 | CMPBEQ R4, $0, generic 13 | 14 | loop: 15 | KIMD R0, R2 // compute intermediate message digest (KIMD) 16 | BVS loop // continue if interrupted 17 | RET 18 | 19 | generic: 20 | BR ·blockGeneric(SB) 21 | -------------------------------------------------------------------------------- /leak/ref_norace.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2023 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | //go:build !race 5 | 6 | package leak 7 | 8 | import "context" 9 | 10 | // Ref implements a [Resource] tracker that is only enabled 11 | // when compiling with `-race`. 12 | type Ref struct { 13 | } 14 | 15 | // Root returns a root reference to a tracker. 16 | func Root(skipCallers int) Ref { 17 | return Ref{} 18 | } 19 | 20 | // Child returns a child ref. 21 | func (ref Ref) Child(name string, skipCallers int) Ref { 22 | return Ref{} 23 | } 24 | 25 | // StartStack returns formatted stack where the resource was created. 26 | func (ref Ref) StartStack() string { 27 | return "" 28 | } 29 | 30 | // Close closes the ref and checks whether all the children have been closed. 31 | // 32 | // The caller should close the ref with [Ref.Close]. 33 | func (ref Ref) Close() error { 34 | return nil 35 | } 36 | 37 | // WithContext attaches a root context that handles tracking. 38 | func WithContext(parent context.Context) (Ref, context.Context) { 39 | return Ref{}, parent 40 | } 41 | 42 | // FromContext returns the attached root Ref. 43 | func FromContext(parent context.Context) Ref { 44 | return Ref{} 45 | } 46 | -------------------------------------------------------------------------------- /leak/ref_race.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2023 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | //go:build race 5 | 6 | package leak 7 | 8 | import "context" 9 | 10 | // Ref implements a [Resource] tracker that is only enabled 11 | // when compiling with `-race`. 12 | type Ref struct { 13 | r *Resource 14 | } 15 | 16 | // Root returns a root reference to a tracker. 17 | func Root(skipCallers int) Ref { 18 | return Ref{ 19 | r: RootResource(skipCallers + 1), 20 | } 21 | } 22 | 23 | // Child returns a child ref. 24 | func (ref Ref) Child(name string, skipCallers int) Ref { 25 | if ref.r == nil { 26 | return Ref{} 27 | } 28 | return Ref{ 29 | r: ref.r.Child(name, skipCallers+1), 30 | } 31 | } 32 | 33 | // StartStack returns formatted stack where the resource was created. 34 | func (ref Ref) StartStack() string { 35 | if ref.r == nil { 36 | return "" 37 | } 38 | return ref.r.StartStack() 39 | } 40 | 41 | // Close closes the ref and checks whether all the children have been closed. 42 | func (ref Ref) Close() error { 43 | if ref.r == nil { 44 | return nil 45 | } 46 | return ref.r.Close() 47 | } 48 | 49 | type contextRefKey struct{} 50 | 51 | // WithContext attaches a root context that handles tracking. 52 | // 53 | // The caller should close the ref with [Ref.Close]. 54 | func WithContext(parent context.Context) (Ref, context.Context) { 55 | ref := Root(1) 56 | ctx := context.WithValue(parent, contextRefKey{}, ref) 57 | return ref, ctx 58 | } 59 | 60 | // FromContext returns the attached root Ref. 61 | func FromContext(parent context.Context) Ref { 62 | ref, ok := parent.Value(contextRefKey{}).(Ref) 63 | if !ok { 64 | return Ref{} 65 | } 66 | return ref 67 | } 68 | -------------------------------------------------------------------------------- /leak/ref_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2023 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | //go:build race 5 | 6 | package leak 7 | 8 | import ( 9 | "context" 10 | "testing" 11 | 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | func TestRef_Ok(t *testing.T) { 16 | root := Root(0) 17 | child1 := root.Child("alpha", 0) 18 | child2 := root.Child("beta", 0) 19 | require.NoError(t, child1.Close()) 20 | require.NoError(t, child2.Close()) 21 | require.NoError(t, root.Close()) 22 | } 23 | 24 | func TestRef_Nested(t *testing.T) { 25 | root := Root(0) 26 | child1 := root.Child("alpha", 0) 27 | leak2 := root.Child("beta", 0) 28 | require.NoError(t, child1.Close()) 29 | _ = leak2 30 | require.Error(t, root.Close()) 31 | } 32 | 33 | func TestRef_Context(t *testing.T) { 34 | bg := context.Background() 35 | 36 | ref, ctx := WithContext(bg) 37 | 38 | root := FromContext(ctx) 39 | require.Equal(t, root, ref) 40 | 41 | child := root.Child("alpha", 0) 42 | require.NoError(t, child.Close()) 43 | 44 | require.NoError(t, root.Close()) 45 | } 46 | -------------------------------------------------------------------------------- /leak/resource_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2023 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package leak 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestResource_Ok(t *testing.T) { 13 | root := RootResource(0) 14 | child1 := root.Child("alpha", 0) 15 | child2 := root.Child("beta", 0) 16 | require.NoError(t, child1.Close()) 17 | require.NoError(t, child2.Close()) 18 | require.NoError(t, root.Close()) 19 | } 20 | 21 | func TestResource_Nested(t *testing.T) { 22 | root := RootResource(0) 23 | child1 := root.Child("alpha", 0) 24 | leak2 := root.Child("beta", 0) 25 | require.NoError(t, child1.Close()) 26 | _ = leak2 27 | require.Error(t, root.Close()) 28 | } 29 | -------------------------------------------------------------------------------- /macaroon/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package macaroon implements contextual caveats and authorization. 5 | package macaroon 6 | 7 | //go:generate protoc -I../pb -I. --pico_out=paths=source_relative:. types.proto 8 | //go:generate goimports -local storj.io -w . 9 | -------------------------------------------------------------------------------- /macaroon/json_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package macaroon 5 | 6 | import ( 7 | "encoding/json" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestMarshalJSON(t *testing.T) { 14 | bs, err := json.Marshal(Caveat_Path{ 15 | Bucket: []byte("bucket"), 16 | EncryptedPathPrefix: []byte("prefix"), 17 | }) 18 | require.NoError(t, err) 19 | require.Equal(t, string(bs), `{"bucket":"YnVja2V0","encrypted_path_prefix":"cHJlZml4"}`) 20 | } 21 | -------------------------------------------------------------------------------- /memory/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package memory contains byte size types and manipulation. 5 | package memory 6 | -------------------------------------------------------------------------------- /memory/sizes.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package memory 5 | 6 | import "strings" 7 | 8 | // Sizes implements flag.Value for collecting memory size. 9 | type Sizes struct { 10 | Default []Size 11 | Custom []Size 12 | } 13 | 14 | // Sizes returns the loaded values. 15 | func (sizes Sizes) Sizes() []Size { 16 | if len(sizes.Custom) > 0 { 17 | return sizes.Custom 18 | } 19 | return sizes.Default 20 | } 21 | 22 | // String converts values to a string. 23 | func (sizes Sizes) String() string { 24 | sz := sizes.Sizes() 25 | xs := make([]string, len(sz)) 26 | for i, size := range sz { 27 | xs[i] = size.String() 28 | } 29 | return strings.Join(xs, " ") 30 | } 31 | 32 | // Set adds values from byte values. 33 | func (sizes *Sizes) Set(s string) error { 34 | for _, x := range strings.Fields(s) { 35 | var size Size 36 | if err := size.Set(x); err != nil { 37 | return err 38 | } 39 | sizes.Custom = append(sizes.Custom, size) 40 | } 41 | return nil 42 | } 43 | -------------------------------------------------------------------------------- /memory/string.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package memory 5 | 6 | // FormatBytes converts number of bytes to appropriately sized string. 7 | func FormatBytes(bytes int64) string { 8 | return Size(bytes).String() 9 | } 10 | 11 | // ParseString converts string to number of bytes. 12 | func ParseString(s string) (int64, error) { 13 | var size Size 14 | err := size.Set(s) 15 | return size.Int64(), err 16 | } 17 | -------------------------------------------------------------------------------- /metrics/testdata/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/storj/common/70ca83b6228edab750747b350ceb98781ea84348/metrics/testdata/README.md -------------------------------------------------------------------------------- /metrics/testdata/gen-ca.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | NAME=$1 3 | openssl genrsa -out $1.key 4096 4 | openssl req -new -x509 -days 3650 -key $1.key -out $1.crt 5 | -------------------------------------------------------------------------------- /metrics/testdata/gen-client-tls.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | if [ "$#" -ne 1 ]; then 3 | echo "Usage: ./gen-client.tls.sh " 4 | exit 1 5 | fi 6 | 7 | NAME=$1 8 | IP=$2 9 | rm $NAME.key || true 10 | rm $NAME.csr || true 11 | rm $NAME.crt || true 12 | openssl genrsa -out $NAME.key 4096 13 | openssl req -new -key $NAME.key -out $NAME.csr -subj "/CN=$NAME" 14 | openssl x509 -req \ 15 | -days 3650 \ 16 | -copy_extensions copy \ 17 | -in $NAME.csr \ 18 | -CA ca-client.crt \ 19 | -CAkey ca-client.key \ 20 | -CAcreateserial \ 21 | -out $NAME.crt 22 | -------------------------------------------------------------------------------- /metrics/testdata/gen-host-tls.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | if [ "$#" -ne 2 ]; then 3 | echo "Usage: ./gen-host-tls.sh " 4 | exit 1 5 | fi 6 | HOST=$1 7 | IP=$2 8 | rm $HOST.csr 9 | rm $HOST.crt 10 | EXT="subjectAltName=DNS:$HOST,DNS:localhost,IP:127.0.0.1,IP:$IP" 11 | openssl genrsa -out $HOST.key 4096 12 | openssl req -new -key $HOST.key -out $HOST.csr \ 13 | -subj "/CN=$HOST" \ 14 | -addext "$EXT" 15 | openssl x509 -req \ 16 | -days 3650 \ 17 | -copy_extensions copy \ 18 | -in $HOST.csr \ 19 | -CA ca.crt \ 20 | -CAkey ca.key \ 21 | -CAcreateserial \ 22 | -out $HOST.crt 23 | -------------------------------------------------------------------------------- /netutil/common.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package netutil 5 | 6 | import "github.com/spacemonkeygo/monkit/v3" 7 | 8 | var mon = monkit.Package() 9 | -------------------------------------------------------------------------------- /netutil/timeout_other.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | //go:build !linux 5 | 6 | package netutil 7 | 8 | import ( 9 | "net" 10 | "time" 11 | ) 12 | 13 | // SetUserTimeout sets the TCP_USER_TIMEOUT setting on the provided conn. 14 | func SetUserTimeout(conn *net.TCPConn, timeout time.Duration) error { 15 | return nil 16 | } 17 | -------------------------------------------------------------------------------- /netutil/tracking.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package netutil 5 | 6 | import ( 7 | "net" 8 | "runtime" 9 | ) 10 | 11 | // closeTrackingConn wraps a net.Conn and keeps track of if it was closed 12 | // or if it was leaked (and closes it if it was leaked). 13 | type closeTrackingConn struct { 14 | net.Conn 15 | } 16 | 17 | // TrackClose wraps the conn and sets a finalizer on the returned value to 18 | // close the conn and monitor that it was leaked. 19 | func TrackClose(conn net.Conn) net.Conn { 20 | tracked := &closeTrackingConn{Conn: conn} 21 | runtime.SetFinalizer(tracked, (*closeTrackingConn).finalize) 22 | return tracked 23 | } 24 | 25 | // Close clears the finalizer and closes the connection. 26 | func (c *closeTrackingConn) Close() error { 27 | runtime.SetFinalizer(c, nil) 28 | mon.Event("connection_closed") 29 | return c.Conn.Close() 30 | } 31 | 32 | // finalize monitors that a connection was leaked and closes the connection. 33 | func (c *closeTrackingConn) finalize() { 34 | mon.Event("connection_leaked") 35 | _ = c.Conn.Close() 36 | } 37 | 38 | // NetConn returns the underlying conn, like *tls.Conn does. 39 | func (c *closeTrackingConn) NetConn() net.Conn { 40 | return c.Conn 41 | } 42 | -------------------------------------------------------------------------------- /nodetag/authority.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2023 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package nodetag 5 | 6 | import ( 7 | "bytes" 8 | "context" 9 | 10 | "github.com/zeebo/errs" 11 | 12 | "storj.io/common/pb" 13 | "storj.io/common/signing" 14 | ) 15 | 16 | var ( 17 | // UnknownSignee is returned when the public key is not available for NodeID to check the signature. 18 | UnknownSignee = errs.Class("node tag signee is unknown") 19 | ) 20 | 21 | // Authority contains all possible signee. 22 | type Authority []signing.Signee 23 | 24 | // Verify checks if any of the storage signee can validate the signature. 25 | func (a Authority) Verify(ctx context.Context, tags *pb.SignedNodeTagSet) (*pb.NodeTagSet, error) { 26 | for _, signee := range a { 27 | if bytes.Equal(signee.ID().Bytes(), tags.SignerNodeId) { 28 | return Verify(ctx, tags, signee) 29 | } 30 | } 31 | return nil, UnknownSignee.New("no certificate for signer nodeID: %x", tags.SignerNodeId) 32 | } 33 | -------------------------------------------------------------------------------- /paths/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package paths implements wrappers for handling encrypted and unencrypted 5 | // paths safely. 6 | package paths 7 | -------------------------------------------------------------------------------- /pb/alias.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package pb 5 | 6 | import proto "github.com/gogo/protobuf/proto" 7 | 8 | // Unmarshal is an alias for proto.Unmarshal. 9 | func Unmarshal(buf []byte, pb proto.Message) error { return proto.Unmarshal(buf, pb) } 10 | 11 | // Marshal is an alias for proto.Marshal. 12 | func Marshal(pb proto.Message) ([]byte, error) { return proto.Marshal(pb) } 13 | 14 | // Size is an alias for proto.Size. 15 | func Size(pb proto.Message) int { return proto.Size(pb) } 16 | -------------------------------------------------------------------------------- /pb/certificate.proto: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | syntax = "proto3"; 5 | option go_package = "storj.io/common/pb"; 6 | 7 | package node; 8 | 9 | service Certificates { 10 | rpc Sign(SigningRequest) returns (SigningResponse); 11 | } 12 | 13 | message SigningRequest { 14 | string auth_token = 1; 15 | int64 timestamp = 2; 16 | } 17 | 18 | message SigningResponse { 19 | repeated bytes chain = 1; 20 | } 21 | -------------------------------------------------------------------------------- /pb/compatibility_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package pb_test 5 | 6 | import ( 7 | fmt "fmt" 8 | "reflect" 9 | "sort" 10 | "strings" 11 | "testing" 12 | 13 | "github.com/stretchr/testify/assert" 14 | 15 | "storj.io/common/pb" 16 | ) 17 | 18 | func TestCompatibility(t *testing.T) { 19 | // when these fail, the X and XSigning definitions are out of sync 20 | // remember to update the conversions in auth/signing 21 | check(t, pb.OrderLimit{}, pb.OrderLimitSigning{}) 22 | check(t, pb.Order{}, pb.OrderSigning{}) 23 | check(t, pb.PieceHash{}, pb.PieceHashSigning{}) 24 | } 25 | 26 | func check(t *testing.T, a, b any) { 27 | afields := fields(a) 28 | bfields := fields(b) 29 | assert.Equal(t, afields, bfields, fmt.Sprintf("%T and %T definitions don't match", a, b)) 30 | } 31 | 32 | type Field struct { 33 | Name string 34 | Type string 35 | Index string 36 | } 37 | 38 | func fields(v any) []Field { 39 | t := reflect.ValueOf(v).Type() 40 | if t.Kind() != reflect.Struct { 41 | panic(t.Kind()) 42 | } 43 | 44 | var fields []Field 45 | for i := 0; i < t.NumField(); i++ { 46 | f := t.Field(i) 47 | 48 | tag := f.Tag.Get("protobuf") 49 | if tag == "" { 50 | continue 51 | } 52 | tags := strings.Split(tag, ",") 53 | fields = append(fields, Field{ 54 | Name: f.Name, 55 | Type: tags[0], 56 | Index: tags[1], 57 | }) 58 | } 59 | 60 | sort.Slice(fields, func(i, k int) bool { 61 | return fields[i].Name < fields[k].Name 62 | }) 63 | 64 | return fields 65 | } 66 | -------------------------------------------------------------------------------- /pb/debug.proto: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2024 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | syntax = "proto3"; 5 | option go_package = "storj.io/common/pb"; 6 | 7 | package debug; 8 | 9 | service Debug { 10 | rpc CollectRuntimeTraces(CollectRuntimeTracesRequest) returns (stream CollectRuntimeTracesResponse); 11 | rpc CollectRuntimeTraces2(stream CollectRuntimeTracesRequest) returns (stream CollectRuntimeTracesResponse); 12 | } 13 | 14 | message CollectRuntimeTracesRequest { 15 | bool done = 1; 16 | } 17 | 18 | message CollectRuntimeTracesResponse { 19 | bytes data = 1; 20 | } 21 | -------------------------------------------------------------------------------- /pb/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package pb contains protobuf definitions for Storj peers. 5 | // Run protolock in the storj.io/common module root to update the lock file. 6 | // E.g. `protolock commit`, no other parameters necessary. 7 | package pb 8 | 9 | //go:generate go run gen.go 10 | -------------------------------------------------------------------------------- /pb/edgeauth.proto: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | syntax = "proto3"; 5 | option go_package = "storj.io/common/pb"; 6 | 7 | import "gogo.proto"; 8 | import "google/protobuf/timestamp.proto"; 9 | 10 | // Service which stores credentials for Linksharing and the Multi-Tenant S3 Gateway. 11 | // Production location at auth.[eu1|us1|ap1].storjshare.io:443 12 | // The REST interface has additional functionality. 13 | service EdgeAuth { 14 | // Request credentials for the Multi-Tenant S3 Gateway. 15 | // Usually called through Uplink. 16 | rpc RegisterAccess(EdgeRegisterAccessRequest) returns (EdgeRegisterAccessResponse); 17 | } 18 | 19 | message EdgeRegisterAccessRequest { 20 | // Serialized access in the usual base58 format. 21 | // Specifies the bucket, prefix, satellite and encryption key. 22 | string access_grant = 1; 23 | 24 | // Enable access without providing the secret. 25 | bool public = 2; 26 | } 27 | 28 | // Credentials, compatible with S3 clients. 29 | message EdgeRegisterAccessResponse { 30 | // Alphanumeric string. 31 | // AWS specifies this should be between 16 and 128 characters. 32 | // This is also used for linksharing in the url path. 33 | string access_key_id = 1; 34 | 35 | // Alphanumeric string. 36 | string secret_key = 2; 37 | 38 | // HTTP(S) URL to the Gateway. 39 | string endpoint = 3; 40 | 41 | // Restricted expiration date of the access grant. 42 | // It is set if the original expiration date surpassed the free-tier limit. 43 | google.protobuf.Timestamp free_tier_restricted_expiration = 4 [(gogoproto.stdtime) = true]; 44 | } 45 | -------------------------------------------------------------------------------- /pb/encryption.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package pb 5 | 6 | import "encoding/json" 7 | 8 | // MarshalJSON implements the json.Marshaler interface. 9 | func (cs CipherSuite) MarshalJSON() ([]byte, error) { 10 | return json.Marshal(cs.String()) 11 | } 12 | -------------------------------------------------------------------------------- /pb/encryption.proto: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | syntax = "proto3"; 5 | option go_package = "storj.io/common/pb"; 6 | 7 | package encryption; 8 | 9 | message EncryptionParameters { 10 | CipherSuite cipher_suite = 1; 11 | int64 block_size = 2; 12 | } 13 | 14 | enum CipherSuite { 15 | ENC_UNSPECIFIED = 0; 16 | ENC_NULL = 1; 17 | ENC_AESGCM = 2; 18 | ENC_SECRETBOX = 3; 19 | } 20 | -------------------------------------------------------------------------------- /pb/encryption_access.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package pb 5 | 6 | import ( 7 | "encoding/base64" 8 | "encoding/json" 9 | 10 | "storj.io/common/encryption" 11 | "storj.io/common/storj" 12 | ) 13 | 14 | type encryptionAccessStoreEntryMarshal struct { 15 | Bucket string `json:"bucket,omitempty"` 16 | UnencryptedPath string `json:"unencrypted_path,omitempty"` 17 | EncryptedPath string `json:"encrypted_path,omitempty"` 18 | Key string `json:"key,omitempty"` 19 | PathCipher CipherSuite `json:"path_cipher,omitempty"` 20 | EncryptionParameters *EncryptionParameters `json:"encryption_parameters,omitempty"` 21 | } 22 | 23 | // MarshalJSON implements the json.Marshaler interface. 24 | func (se *EncryptionAccess_StoreEntry) MarshalJSON() ([]byte, error) { 25 | key, err := storj.NewKey([]byte{}) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | path, err := encryption.DecryptPathRaw(string(se.EncryptedPath), storj.EncNullBase64URL, key) 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | return json.Marshal(encryptionAccessStoreEntryMarshal{ 36 | Bucket: string(se.Bucket), 37 | UnencryptedPath: string(se.UnencryptedPath), 38 | EncryptedPath: path, 39 | Key: base64.URLEncoding.EncodeToString(se.Key), 40 | PathCipher: se.PathCipher, 41 | EncryptionParameters: se.EncryptionParameters, 42 | }) 43 | } 44 | -------------------------------------------------------------------------------- /pb/encryption_access.proto: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | syntax = "proto3"; 5 | 6 | option go_package = "storj.io/common/pb"; 7 | 8 | package encryption_access; 9 | 10 | import "encryption.proto"; 11 | 12 | message EncryptionAccess { 13 | message StoreEntry { 14 | bytes bucket = 1; 15 | bytes unencrypted_path = 2; 16 | bytes encrypted_path = 3; 17 | bytes key = 4; 18 | 19 | encryption.CipherSuite path_cipher = 5; 20 | encryption.EncryptionParameters encryption_parameters = 6; 21 | } 22 | 23 | bytes default_key = 1; 24 | repeated StoreEntry store_entries = 2; 25 | encryption.CipherSuite default_path_cipher = 3; 26 | encryption.EncryptionParameters default_encryption_parameters = 4; 27 | } 28 | -------------------------------------------------------------------------------- /pb/hashing.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information.package pb 3 | 4 | package pb 5 | 6 | import ( 7 | "hash" 8 | 9 | "github.com/zeebo/blake3" 10 | 11 | "storj.io/common/pkcrypto" 12 | ) 13 | 14 | // NewHashFromAlgorithm creates the hash function based on hash algorithm. 15 | func NewHashFromAlgorithm(algorithm PieceHashAlgorithm) hash.Hash { 16 | switch algorithm { 17 | case PieceHashAlgorithm_BLAKE3: 18 | return blake3.New() 19 | default: 20 | return pkcrypto.NewHash() 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /pb/json_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package pb 5 | 6 | import ( 7 | "encoding/json" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestMarshalJSON(t *testing.T) { 14 | t.Run("CipherSuite", func(t *testing.T) { 15 | cs := CipherSuite_ENC_AESGCM 16 | bs, err := json.Marshal(cs) 17 | require.NoError(t, err) 18 | require.Equal(t, string(bs), `"ENC_AESGCM"`) 19 | }) 20 | 21 | t.Run("EncryptionAccess_StoreEntry", func(t *testing.T) { 22 | bs, err := json.Marshal(&EncryptionAccess_StoreEntry{ 23 | Bucket: []byte("bucket"), 24 | UnencryptedPath: []byte("unenc"), 25 | EncryptedPath: []byte("enc"), 26 | Key: []byte("key"), 27 | PathCipher: CipherSuite_ENC_AESGCM, 28 | }) 29 | require.NoError(t, err) 30 | require.Equal(t, string(bs), `{"bucket":"bucket","unencrypted_path":"unenc","encrypted_path":"ZW5j","key":"a2V5","path_cipher":"ENC_AESGCM"}`) 31 | }) 32 | } 33 | -------------------------------------------------------------------------------- /pb/meta.proto: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | syntax = "proto3"; 5 | option go_package = "storj.io/common/pb"; 6 | 7 | package objects; 8 | 9 | // SerializableMeta is the object metadata that will be stored serialized. 10 | message SerializableMeta { 11 | string content_type = 1; 12 | map user_defined = 2; 13 | } -------------------------------------------------------------------------------- /pb/nodetags.proto: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | syntax = "proto3"; 5 | option go_package = "storj.io/common/pb"; 6 | 7 | package node; 8 | 9 | // Tag represents single key, value. 10 | message Tag { 11 | string name = 1; 12 | bytes value = 2; // optional, representation dependent on name. 13 | } 14 | 15 | // NodeTagSet 16 | message NodeTagSet { 17 | // must always be set. this is the node the signer is signing for. 18 | bytes node_id = 1; 19 | repeated Tag tags = 2; 20 | // must always be set. this makes sure the signature is signing the 21 | // timestamp inside. 22 | int64 signed_at = 3; 23 | } 24 | 25 | // SignedNodeTagSet is a tag set with additional signature. 26 | message SignedNodeTagSet { 27 | // this is the serialized form of TagSet, serialized so that 28 | // the signature process has something stable to work with. 29 | bytes serialized_tag = 1; 30 | // this is who signed (could be self signed, could be well known) 31 | // public key supposed to be available on satellite side based on this id. 32 | bytes signer_node_id = 3; 33 | 34 | // the signature of serialized_tag. 35 | bytes signature = 4; 36 | } 37 | 38 | // SignedNodeTagSets 39 | message SignedNodeTagSets { 40 | repeated SignedNodeTagSet tags = 1; 41 | } -------------------------------------------------------------------------------- /pb/pico.proto: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | syntax = "proto3"; 5 | option go_package = "storj.io/picobuf;main"; 6 | 7 | package pico; 8 | 9 | import "google/protobuf/descriptor.proto"; 10 | 11 | message MessageOptions { 12 | bool always_present = 1; 13 | } 14 | 15 | extend google.protobuf.MessageOptions { 16 | MessageOptions message = 28980; 17 | } 18 | 19 | message FieldOptions { 20 | bool always_present = 1; 21 | string custom_type = 2; 22 | string custom_cast = 3; 23 | } 24 | 25 | extend google.protobuf.FieldOptions { 26 | FieldOptions field = 28980; 27 | } 28 | -------------------------------------------------------------------------------- /pb/scope.proto: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | syntax = "proto3"; 5 | 6 | option go_package = "storj.io/common/pb"; 7 | 8 | package scope; 9 | 10 | import "encryption_access.proto"; 11 | 12 | message Scope { 13 | string satellite_addr = 1; 14 | 15 | bytes api_key = 2; 16 | 17 | encryption_access.EncryptionAccess encryption_access = 3; 18 | } 19 | -------------------------------------------------------------------------------- /pb/streams.proto: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | syntax = "proto3"; 5 | option go_package = "storj.io/common/pb"; 6 | 7 | package streams; 8 | 9 | message SegmentMeta { 10 | bytes encrypted_key = 1; 11 | bytes key_nonce = 2; 12 | } 13 | 14 | message StreamInfo { 15 | int64 deprecated_number_of_segments = 1; 16 | int64 segments_size = 2; 17 | int64 last_segment_size = 3; 18 | bytes metadata = 4; 19 | } 20 | 21 | message StreamMeta { 22 | bytes encrypted_stream_info = 1; 23 | int32 encryption_type = 2; 24 | int32 encryption_block_size = 3; 25 | SegmentMeta last_segment_meta = 4; 26 | int64 number_of_segments = 5; 27 | } 28 | -------------------------------------------------------------------------------- /pb/types.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package pb 5 | 6 | import "storj.io/common/storj" 7 | 8 | // Path represents a object path. 9 | type Path = storj.Path 10 | 11 | // NodeID is an alias to storj.NodeID for use in generated protobuf code. 12 | type NodeID = storj.NodeID 13 | 14 | // NodeIDList is an alias to storj.NodeIDList for use in generated protobuf code. 15 | type NodeIDList = storj.NodeIDList 16 | 17 | // PieceID is an alias to storj.PieceID for use in generated protobuf code. 18 | type PieceID = storj.PieceID 19 | 20 | // PiecePublicKey is an alias to storj.PiecePublicKey for use in generated protobuf code. 21 | type PiecePublicKey = storj.PiecePublicKey 22 | 23 | // PiecePrivateKey is an alias to storj.PiecePrivateKey for use in generated protobuf code. 24 | type PiecePrivateKey = storj.PiecePrivateKey 25 | 26 | // SerialNumber is an alias to storj.SerialNumber for use in generated protobuf code. 27 | type SerialNumber = storj.SerialNumber 28 | 29 | // StreamID is an alias to storj.StreamID for use in generated protobuf code. 30 | type StreamID = storj.StreamID 31 | 32 | // Nonce is an alias to storj.Nonce for use in generated protobuf code. 33 | type Nonce = storj.Nonce 34 | 35 | // SegmentID is an alias to storj.SegmentID for use in generated protobuf code. 36 | type SegmentID = storj.SegmentID 37 | -------------------------------------------------------------------------------- /pb/userinfo.proto: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | syntax = "proto3"; 5 | option go_package = "storj.io/common/pb"; 6 | 7 | import "metainfo.proto"; 8 | 9 | package userinfo; 10 | 11 | service UserInfo { 12 | rpc Get(GetUserInfoRequest) returns (GetUserInfoResponse); 13 | } 14 | 15 | message GetUserInfoRequest { 16 | metainfo.RequestHeader header = 1; 17 | } 18 | 19 | message GetUserInfoResponse { 20 | bool paid_tier = 1; 21 | } 22 | -------------------------------------------------------------------------------- /pb/utils_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2023 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package pb_test 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | 11 | "storj.io/common/pb" 12 | "storj.io/common/storj" 13 | ) 14 | 15 | func TestCopyNode(t *testing.T) { 16 | node := &pb.Node{ 17 | Id: storj.NodeID{1}, 18 | Address: &pb.NodeAddress{ 19 | Address: "localhost:1234", 20 | NoiseInfo: &pb.NoiseInfo{ 21 | Proto: pb.NoiseProtocol_NOISE_IK_25519_AESGCM_BLAKE2B, 22 | PublicKey: []byte{1, 2, 3}, 23 | }, 24 | }, 25 | } 26 | 27 | copy := pb.CopyNode(node) 28 | require.EqualValues(t, node, copy) 29 | } 30 | -------------------------------------------------------------------------------- /peertls/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package peertls manages TLS configuration for peers. 5 | package peertls 6 | -------------------------------------------------------------------------------- /peertls/extensions/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package extensions contains extensions to TLS certificate handling. 5 | package extensions 6 | -------------------------------------------------------------------------------- /peertls/templates.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package peertls 5 | 6 | import ( 7 | "crypto/x509" 8 | "crypto/x509/pkix" 9 | ) 10 | 11 | // CATemplate returns x509.Certificate template for certificate authority. 12 | func CATemplate() (*x509.Certificate, error) { 13 | serialNumber, err := newSerialNumber() 14 | if err != nil { 15 | return nil, ErrTLSTemplate.Wrap(err) 16 | } 17 | 18 | template := &x509.Certificate{ 19 | SerialNumber: serialNumber, 20 | KeyUsage: x509.KeyUsageCertSign, 21 | BasicConstraintsValid: true, 22 | IsCA: true, 23 | Subject: pkix.Name{Organization: []string{"Storj"}}, 24 | } 25 | 26 | return template, nil 27 | } 28 | 29 | // LeafTemplate returns x509.Certificate template for signing and encrypting. 30 | func LeafTemplate() (*x509.Certificate, error) { 31 | serialNumber, err := newSerialNumber() 32 | if err != nil { 33 | return nil, ErrTLSTemplate.Wrap(err) 34 | } 35 | 36 | template := &x509.Certificate{ 37 | SerialNumber: serialNumber, 38 | KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, 39 | ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, 40 | BasicConstraintsValid: true, 41 | IsCA: false, 42 | Subject: pkix.Name{Organization: []string{"Storj"}}, 43 | } 44 | 45 | return template, nil 46 | } 47 | -------------------------------------------------------------------------------- /peertls/testpeertls/certificates_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package testpeertls 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | 12 | "storj.io/common/identity" 13 | "storj.io/common/identity/testidentity" 14 | "storj.io/common/peertls" 15 | "storj.io/common/pkcrypto" 16 | "storj.io/common/storj" 17 | ) 18 | 19 | func TestNewCertChain(t *testing.T) { 20 | testidentity.CompleteIdentityVersionsTest(t, func(t *testing.T, version storj.IDVersion, ident *identity.FullIdentity) { 21 | for length := 2; length < 4; length++ { 22 | t.Logf("length: %d", length) 23 | keys, chain, err := NewCertChain(length, version.Number) 24 | require.NoError(t, err) 25 | 26 | assert.Len(t, chain, length) 27 | assert.Len(t, keys, length) 28 | 29 | pubKey, err := pkcrypto.PublicKeyFromPrivate(keys[peertls.CAIndex]) 30 | require.NoError(t, err) 31 | assert.Equal(t, pubKey, chain[peertls.CAIndex].PublicKey) 32 | 33 | pubKey, err = pkcrypto.PublicKeyFromPrivate(keys[peertls.LeafIndex]) 34 | require.NoError(t, err) 35 | assert.Equal(t, pubKey, chain[peertls.LeafIndex].PublicKey) 36 | 37 | err = peertls.VerifyPeerCertChains(nil, identity.ToChains(chain)) 38 | assert.NoError(t, err) 39 | 40 | assert.True(t, chain[peertls.CAIndex].IsCA) 41 | assert.False(t, chain[peertls.LeafIndex].IsCA) 42 | } 43 | }) 44 | } 45 | -------------------------------------------------------------------------------- /peertls/testpeertls/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package testpeertls implements testing utilities for peertls. 5 | package testpeertls 6 | -------------------------------------------------------------------------------- /peertls/tlsopts/cert.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package tlsopts 5 | 6 | const ( 7 | // DefaultPeerCAWhitelist includes the production Storj network CAs. 8 | DefaultPeerCAWhitelist = `-----BEGIN CERTIFICATE----- 9 | MIIBWzCCAQGgAwIBAgIRAK7f/E+PDEvB/TrUSaHxOEYwCgYIKoZIzj0EAwIwEDEO 10 | MAwGA1UEChMFU3RvcmowIhgPMDAwMTAxMDEwMDAwMDBaGA8wMDAxMDEwMTAwMDAw 11 | MFowEDEOMAwGA1UEChMFU3RvcmowWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATr 12 | sDBAh7sr9eVZJUIFb79WK2qTcSKw/sP95JF5rCIJ5FvvwA/cx70VdW6IQjVhIaDY 13 | llQONAD90PeoOpqSyo+iozgwNjAOBgNVHQ8BAf8EBAMCAgQwEwYDVR0lBAwwCgYI 14 | KwYBBQUHAwEwDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAgNIADBFAiEAzPdn 15 | 5ZK9hIUm+0b7iBHfk1T/O7gpwGTmsSLps4cF6KgCIDhgQ4g2givMj5Khmuhnr/e7 16 | z6HlDVf3PJOQv1yZqg7W 17 | -----END CERTIFICATE-----` 18 | ) 19 | -------------------------------------------------------------------------------- /peertls/tlsopts/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package tlsopts 5 | 6 | import ( 7 | "storj.io/common/peertls/extensions" 8 | ) 9 | 10 | // Config holds tls configuration parameters. 11 | type Config struct { 12 | RevocationDBURL string `default:"bolt://$CONFDIR/revocations.db" help:"url for revocation database (e.g. bolt://some.db OR redis://127.0.0.1:6378?db=2&password=abc123)"` 13 | PeerCAWhitelistPath string `help:"path to the CA cert whitelist (peer identities must be signed by one these to be verified). this will override the default peer whitelist"` 14 | UsePeerCAWhitelist bool `devDefault:"false" releaseDefault:"true" help:"if true, uses peer ca whitelist checking"` 15 | PeerIDVersions string `default:"latest" help:"identity version(s) the server will be allowed to talk to"` 16 | Extensions extensions.Config 17 | } 18 | -------------------------------------------------------------------------------- /peertls/tlsopts/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package tlsopts handles TLS server options. 5 | package tlsopts 6 | -------------------------------------------------------------------------------- /peertls/tlsopts/options_internal_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package tlsopts 5 | 6 | import ( 7 | "crypto/x509" 8 | "errors" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/require" 12 | 13 | "storj.io/common/peertls" 14 | ) 15 | 16 | func TestRemoveNils(t *testing.T) { 17 | e1 := errors.New("error 1") 18 | f1 := peertls.PeerCertVerificationFunc(func([][]byte, [][]*x509.Certificate) error { return e1 }) 19 | e2 := errors.New("error 2") 20 | f2 := peertls.PeerCertVerificationFunc(func([][]byte, [][]*x509.Certificate) error { return e2 }) 21 | 22 | l := removeNils([]peertls.PeerCertVerificationFunc{f1, nil, nil, f2}) 23 | require.Equal(t, len(l), 2) 24 | require.Equal(t, l[0](nil, nil), e1) 25 | require.Equal(t, l[1](nil, nil), e2) 26 | } 27 | -------------------------------------------------------------------------------- /peertls/tlsopts/tls_helpers_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package tlsopts 5 | 6 | var VerifyIdentity = verifyIdentity 7 | -------------------------------------------------------------------------------- /pkcrypto/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | /* 5 | Package pkcrypto contains a set of helper functions and constants to perform 6 | common cryptographic operations like: 7 | 8 | * Signing and verification 9 | 10 | * Public and private key generation 11 | 12 | * Certification generation 13 | */ 14 | package pkcrypto 15 | -------------------------------------------------------------------------------- /pkcrypto/hashing.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package pkcrypto 5 | 6 | import ( 7 | "crypto/sha256" 8 | "hash" 9 | 10 | "storj.io/common/sync2/race2" 11 | ) 12 | 13 | // NewHash returns default hash in storj. 14 | func NewHash() hash.Hash { 15 | return sha256.New() 16 | } 17 | 18 | // SHA256Hash calculates the SHA256 hash of the input data. 19 | func SHA256Hash(data []byte) []byte { 20 | race2.ReadSlice(data) 21 | sum := sha256.Sum256(data) 22 | return sum[:] 23 | } 24 | -------------------------------------------------------------------------------- /process/debug.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package process 5 | 6 | import ( 7 | "context" 8 | "net" 9 | 10 | "github.com/spacemonkeygo/monkit/v3" 11 | "github.com/spf13/pflag" 12 | "go.uber.org/zap" 13 | 14 | "storj.io/common/cfgstruct" 15 | "storj.io/common/debug" 16 | ) 17 | 18 | var debugConfig struct { 19 | Debug debug.Config 20 | } 21 | 22 | func init() { 23 | cfgstruct.Bind(pflag.CommandLine, &debugConfig) 24 | } 25 | 26 | func initDebug(log *zap.Logger, r *monkit.Registry, atomicLevel *zap.AtomicLevel) (err error) { 27 | if debugConfig.Debug.Addr == "" { 28 | return nil 29 | } 30 | 31 | ln, err := net.Listen("tcp", debugConfig.Debug.Addr) 32 | if err != nil { 33 | return err 34 | } 35 | 36 | go func() { 37 | server := debug.NewServerWithAtomicLevel(log, ln, r, debugConfig.Debug, atomicLevel) 38 | log.Debug("debug server listening on " + ln.Addr().String()) 39 | err := server.Run(context.TODO()) 40 | if err != nil { 41 | log.Error("debug server died", zap.Error(err)) 42 | } 43 | }() 44 | 45 | return nil 46 | } 47 | -------------------------------------------------------------------------------- /process/decorate_notwindows.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | //go:build !windows 5 | 6 | package process 7 | 8 | import ( 9 | "go.uber.org/zap/zapcore" 10 | ) 11 | 12 | const ( 13 | colorReset = "\x1b[0m" 14 | colorRed = "\x1b[31m" 15 | colorGreen = "\x1b[32m" 16 | colorBlue = "\x1b[34m" 17 | colorMagenta = "\x1b[35m" 18 | ) 19 | 20 | func levelDecorate(level zapcore.Level, message string) string { 21 | return levelColor[level] + message + colorReset 22 | } 23 | 24 | var ( 25 | levelColor = map[zapcore.Level]string{ 26 | zapcore.DebugLevel: colorGreen, 27 | zapcore.InfoLevel: colorBlue, 28 | zapcore.WarnLevel: colorMagenta, 29 | zapcore.ErrorLevel: colorRed, 30 | zapcore.DPanicLevel: colorRed, 31 | zapcore.PanicLevel: colorRed, 32 | zapcore.FatalLevel: colorRed, 33 | } 34 | ) 35 | -------------------------------------------------------------------------------- /process/decorate_windows.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package process 5 | 6 | import ( 7 | "go.uber.org/zap/zapcore" 8 | ) 9 | 10 | func levelDecorate(level zapcore.Level, message string) string { 11 | return message 12 | } 13 | -------------------------------------------------------------------------------- /process/eventkitbq/destination.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2024 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package eventkitbq 5 | 6 | import ( 7 | "context" 8 | 9 | "go.uber.org/zap" 10 | 11 | "storj.io/eventkit" 12 | "storj.io/eventkit/bigquery" 13 | ) 14 | 15 | // BQDestination initializes the BQ destination. 16 | // Context should be cancelled to stop internal goroutines. 17 | func BQDestination(ctx context.Context, log *zap.Logger, destConfig string, eventRegistry *eventkit.Registry, appName string, instanceID string) { 18 | c, err := bigquery.CreateDestination(ctx, destConfig) 19 | if err != nil { 20 | log.Error("Eventkit BQ destination couldn't be initialized", zap.Error(err)) 21 | return 22 | } 23 | 24 | eventRegistry.AddDestination(c) 25 | go c.Run(ctx) 26 | } 27 | -------------------------------------------------------------------------------- /process/exec.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package process // import "storj.io/storj/pkg/process" 5 | 6 | import ( 7 | "fmt" 8 | "log" 9 | "os" 10 | 11 | "github.com/spf13/cobra" 12 | 13 | // We use a blank import here to get the side effects from the init function in version. 14 | _ "storj.io/common/version" 15 | ) 16 | 17 | func init() { 18 | cobra.MousetrapHelpText = "This is a command line tool.\n\n" + 19 | "This needs to be run from a Command Prompt.\n" 20 | 21 | // Figure out the executable name. 22 | exe, err := os.Executable() 23 | if err == nil { 24 | cobra.MousetrapHelpText += fmt.Sprintf( 25 | "Try running \"%s help\" for more information\n", exe) 26 | } 27 | } 28 | 29 | // fileExists checks whether file exists, handle error correctly if it doesn't. 30 | func fileExists(path string) bool { 31 | _, err := os.Stat(path) 32 | if err != nil { 33 | if os.IsNotExist(err) { 34 | return false 35 | } 36 | log.Fatalf("failed to check for file existence: %v", err) 37 | } 38 | return true 39 | } 40 | -------------------------------------------------------------------------------- /process/gcloudlogging/operation.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2023 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package gcloudlogging 5 | 6 | import ( 7 | "go.uber.org/zap" 8 | "go.uber.org/zap/zapcore" 9 | ) 10 | 11 | // Operation represents LogEntryOperation field. See: 12 | // https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#LogEntryOperation 13 | type Operation struct { 14 | ID string 15 | Producer string 16 | First bool 17 | Last bool 18 | } 19 | 20 | // MarshalLogObject implements zapcore.ObjectMarshaler. 21 | // All fields are optional. 22 | func (op *Operation) MarshalLogObject(enc zapcore.ObjectEncoder) error { 23 | if op.ID != "" { 24 | enc.AddString("id", op.ID) 25 | } 26 | if op.Producer != "" { 27 | enc.AddString("producer", op.Producer) 28 | } 29 | // note: GCP logging treats bool false as empty. 30 | if op.First { 31 | enc.AddBool("first", true) 32 | } 33 | if op.Last { 34 | enc.AddBool("last", true) 35 | } 36 | 37 | return nil 38 | } 39 | 40 | // LogOperation returns a zapcore.field for Operation. 41 | func LogOperation(op *Operation) zapcore.Field { 42 | return zap.Object("logging.googleapis.com/operation", op) 43 | } 44 | -------------------------------------------------------------------------------- /process/gcloudlogging/operation_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2023 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package gcloudlogging_test 5 | 6 | import ( 7 | "testing" 8 | "time" 9 | 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/require" 12 | "go.uber.org/zap" 13 | "go.uber.org/zap/zapcore" 14 | 15 | "storj.io/common/process/gcloudlogging" 16 | ) 17 | 18 | func TestOperationEncoding(t *testing.T) { 19 | op := &gcloudlogging.Operation{ 20 | ID: "foo", 21 | Producer: "github.com/storj/gateway-mt", 22 | First: true, 23 | Last: false, 24 | } 25 | 26 | enc := gcloudlogging.NewEncoder(gcloudlogging.NewEncoderConfig()) 27 | 28 | buf, err := enc.EncodeEntry(zapcore.Entry{ 29 | Level: zapcore.InfoLevel, 30 | Message: "test", 31 | Time: time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC), 32 | }, []zapcore.Field{ 33 | gcloudlogging.LogOperation(op), 34 | zap.String("something", "else"), 35 | }) 36 | require.NoError(t, err) 37 | 38 | assert.JSONEq(t, `{ 39 | "logging.googleapis.com/operation": { 40 | "first": true, 41 | "id": "foo", 42 | "producer": "github.com/storj/gateway-mt" 43 | }, 44 | "logging.googleapis.com/labels": { 45 | "something": "else" 46 | }, 47 | "severity": "INFO", 48 | "message": "test", 49 | "timestamp": { 50 | "nanos": 0, 51 | "seconds": 1.257894e+09 52 | } 53 | }`, buf.String()) 54 | } 55 | -------------------------------------------------------------------------------- /process/googleprofiler/init.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package googleprofiler attaches google cloud profiler to private/process. 5 | package googleprofiler 6 | 7 | import ( 8 | "flag" 9 | 10 | "cloud.google.com/go/profiler" 11 | "github.com/zeebo/errs" 12 | "go.uber.org/zap" 13 | 14 | "storj.io/common/process" 15 | "storj.io/common/version" 16 | ) 17 | 18 | var ( 19 | errorClass = errs.Class("initializing profiler") 20 | serviceName = flag.String("debug.profilername", "", "provide the name of the peer to enable continuous cpu/mem profiling for") 21 | projectID = flag.String("debug.profilerproject", "", "provide the google project id for continuous profiling (required only for non-k8s environments") 22 | ) 23 | 24 | func init() { 25 | process.SetProfiler(func(log *zap.Logger) error { 26 | name := *serviceName 27 | if name != "" { 28 | info := version.Build 29 | config := profiler.Config{ 30 | Service: name, 31 | ServiceVersion: info.Version.String(), 32 | } 33 | if *projectID != "" { 34 | config.ProjectID = *projectID 35 | } 36 | if err := profiler.Start(config); err != nil { 37 | return errorClass.Wrap(err) 38 | } 39 | log.Debug("success debug profiler init") 40 | } 41 | return nil 42 | }) 43 | } 44 | -------------------------------------------------------------------------------- /process/logging_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2024 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package process 5 | 6 | import ( 7 | "bytes" 8 | "strings" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/require" 12 | "go.uber.org/zap" 13 | "go.uber.org/zap/zapcore" 14 | ) 15 | 16 | func TestNamed(t *testing.T) { 17 | ptr := func(val string) *string { 18 | return &val 19 | } 20 | 21 | customLevel = ptr("log1=WARN") 22 | defer func() { 23 | customLevel = ptr("") 24 | }() 25 | 26 | out := bytes.NewBuffer([]byte{}) 27 | logger := zap.New(zapcore.NewCore(zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()), zapcore.AddSync(out), zap.LevelEnablerFunc(func(level zapcore.Level) bool { 28 | return true 29 | }))) 30 | 31 | NamedLog(logger, "asd").Info("ahoj") 32 | require.True(t, strings.Contains(out.String(), "ahoj")) 33 | out.Reset() 34 | 35 | NamedLog(logger, "log1").Info("ahoj") 36 | require.False(t, strings.Contains(out.String(), "ahoj")) 37 | out.Reset() 38 | 39 | NamedLog(logger, "log1").Warn("ahoj") 40 | require.True(t, strings.Contains(out.String(), "ahoj")) 41 | out.Reset() 42 | 43 | customLevel = ptr("") 44 | NamedLog(logger, "log1").Warn("ahoj") 45 | require.False(t, strings.Contains(out.String(), "Invalid")) 46 | out.Reset() 47 | } 48 | -------------------------------------------------------------------------------- /process/profiler.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package process 5 | 6 | import ( 7 | "go.uber.org/zap" 8 | ) 9 | 10 | var initProfiler func(log *zap.Logger) error 11 | 12 | // SetProfiler sets the profiler for process package. 13 | // 14 | // It panics on multiple calls. 15 | func SetProfiler(fn func(log *zap.Logger) error) { 16 | if initProfiler != nil { 17 | panic("profiler already set") 18 | } 19 | initProfiler = fn 20 | } 21 | -------------------------------------------------------------------------------- /processgroup/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information 3 | 4 | // Package processgroup implements process-grouping commands. 5 | // This is helpful when killing the parent process the child processes should be killed as well. 6 | package processgroup 7 | -------------------------------------------------------------------------------- /processgroup/kill_fallback.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information 3 | 4 | //go:build !windows && !linux && !darwin && !netbsd && !freebsd && !openbsd 5 | 6 | package processgroup 7 | 8 | import ( 9 | "os" 10 | "os/exec" 11 | ) 12 | 13 | // Setup sets up exec.Cmd such that it can be properly terminated. 14 | func Setup(c *exec.Cmd) {} 15 | 16 | // Kill tries to forcefully kill the process. 17 | func Kill(cmd *exec.Cmd) { 18 | proc := cmd.Process 19 | if proc == nil { 20 | return 21 | } 22 | _ = proc.Signal(os.Interrupt) 23 | _ = proc.Signal(os.Kill) 24 | } 25 | -------------------------------------------------------------------------------- /processgroup/kill_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information 3 | 4 | package processgroup_test 5 | 6 | import ( 7 | "io" 8 | "os" 9 | "os/exec" 10 | "testing" 11 | "time" 12 | 13 | "github.com/stretchr/testify/require" 14 | 15 | "storj.io/common/processgroup" 16 | "storj.io/common/testcontext" 17 | ) 18 | 19 | func TestProcessGroup(t *testing.T) { 20 | ctx := testcontext.New(t) 21 | 22 | source := ctx.File("main.go") 23 | binary := ctx.File("main.exe") 24 | err := os.WriteFile(source, []byte(code), 0644) 25 | require.NoError(t, err) 26 | 27 | { 28 | /* #nosec G204 */ // This is a test and both parameters' values are controlled 29 | cmd := exec.Command("go", "build", "-o", binary, source) 30 | cmd.Dir = ctx.Dir() 31 | 32 | _, err := cmd.CombinedOutput() 33 | require.NoError(t, err) 34 | } 35 | 36 | { 37 | 38 | /* #nosec G204 */ // This is a test and the parameter's values is controlled 39 | cmd := exec.Command(binary) 40 | cmd.Dir = ctx.Dir() 41 | cmd.Stdout, cmd.Stderr = io.Discard, io.Discard 42 | processgroup.Setup(cmd) 43 | 44 | started := time.Now() 45 | err := cmd.Start() 46 | require.NoError(t, err) 47 | processgroup.Kill(cmd) 48 | 49 | _ = cmd.Wait() // since we kill it, we might get an error 50 | duration := time.Since(started) 51 | 52 | require.Truef(t, duration < 10*time.Second, "completed in %s", duration) 53 | } 54 | } 55 | 56 | const code = `package main 57 | 58 | import "time" 59 | 60 | func main() { 61 | time.Sleep(20*time.Second) 62 | } 63 | ` 64 | -------------------------------------------------------------------------------- /processgroup/kill_unix.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information 3 | 4 | //go:build linux || freebsd 5 | 6 | package processgroup 7 | 8 | import ( 9 | "os" 10 | "os/exec" 11 | "syscall" 12 | ) 13 | 14 | // Setup sets up exec.Cmd such that it can be properly terminated. 15 | func Setup(c *exec.Cmd) { 16 | c.SysProcAttr = &syscall.SysProcAttr{ 17 | Setpgid: true, 18 | Pdeathsig: syscall.SIGKILL, 19 | } 20 | } 21 | 22 | // Kill tries to forcefully kill the process. 23 | func Kill(cmd *exec.Cmd) { 24 | proc := cmd.Process 25 | if proc == nil { 26 | return 27 | } 28 | 29 | pgid, err := syscall.Getpgid(proc.Pid) 30 | if err == nil { 31 | _ = syscall.Kill(-pgid, syscall.SIGTERM) 32 | } 33 | 34 | // just in case 35 | _ = proc.Signal(os.Interrupt) 36 | _ = proc.Signal(os.Kill) 37 | } 38 | -------------------------------------------------------------------------------- /processgroup/kill_unix_nodeath.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information 3 | 4 | //go:build darwin || netbsd || openbsd 5 | 6 | package processgroup 7 | 8 | import ( 9 | "os" 10 | "os/exec" 11 | "syscall" 12 | ) 13 | 14 | // Setup sets up exec.Cmd such that it can be properly terminated. 15 | func Setup(c *exec.Cmd) { 16 | c.SysProcAttr = &syscall.SysProcAttr{ 17 | Setpgid: true, 18 | } 19 | } 20 | 21 | // Kill tries to forcefully kill the process. 22 | func Kill(cmd *exec.Cmd) { 23 | proc := cmd.Process 24 | if proc == nil { 25 | return 26 | } 27 | 28 | pgid, err := syscall.Getpgid(proc.Pid) 29 | if err == nil { 30 | _ = syscall.Kill(-pgid, syscall.SIGTERM) 31 | } 32 | 33 | // just in case 34 | _ = proc.Signal(os.Interrupt) 35 | _ = proc.Signal(os.Kill) 36 | } 37 | -------------------------------------------------------------------------------- /processgroup/kill_windows.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information 3 | 4 | //go:build windows 5 | 6 | package processgroup 7 | 8 | import ( 9 | "os" 10 | "os/exec" 11 | "strconv" 12 | "syscall" 13 | ) 14 | 15 | // Setup sets up exec.Cmd such that it can be properly terminated. 16 | func Setup(c *exec.Cmd) { 17 | c.SysProcAttr = &syscall.SysProcAttr{ 18 | CreationFlags: syscall.CREATE_NEW_PROCESS_GROUP, 19 | } 20 | } 21 | 22 | // Kill tries to forcefully kill the process. 23 | func Kill(cmd *exec.Cmd) { 24 | proc := cmd.Process 25 | if proc == nil { 26 | return 27 | } 28 | 29 | _ = exec.Command("taskkill", "/f", "/pid", strconv.Itoa(proc.Pid)).Run() 30 | 31 | // just in case 32 | forcekill(proc.Pid) 33 | _ = proc.Signal(os.Interrupt) 34 | _ = proc.Signal(os.Kill) 35 | } 36 | 37 | func forcekill(pid int) { 38 | handle, err := syscall.OpenProcess(syscall.PROCESS_TERMINATE, true, uint32(pid)) 39 | if err != nil { 40 | return 41 | } 42 | 43 | _ = syscall.TerminateProcess(handle, 0) 44 | _ = syscall.CloseHandle(handle) 45 | } 46 | -------------------------------------------------------------------------------- /ranger/common.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package ranger 5 | 6 | import ( 7 | "github.com/spacemonkeygo/monkit/v3" 8 | "github.com/zeebo/errs" 9 | ) 10 | 11 | // Error is the errs class of standard Ranger errors. 12 | var Error = errs.Class("ranger") 13 | 14 | var mon = monkit.Package() 15 | -------------------------------------------------------------------------------- /ranger/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package ranger implements lazy io.Reader and io.Writer interfaces. 5 | package ranger 6 | -------------------------------------------------------------------------------- /ranger/force.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2023 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package ranger 5 | 6 | import ( 7 | "io" 8 | ) 9 | 10 | type forcedReader struct { 11 | buf []byte 12 | err error 13 | r io.ReadCloser 14 | } 15 | 16 | func (r *forcedReader) Close() error { 17 | return r.r.Close() 18 | } 19 | 20 | func (r *forcedReader) Read(p []byte) (n int, err error) { 21 | if len(r.buf) > 0 { 22 | n = copy(p, r.buf) 23 | r.buf = r.buf[n:] 24 | if len(r.buf) == 0 { 25 | r.buf = nil 26 | } 27 | return n, nil 28 | } 29 | if r.err != nil { 30 | return 0, r.err 31 | } 32 | return r.r.Read(p) 33 | } 34 | 35 | func forceReads(r io.ReadCloser) io.ReadCloser { 36 | var buf [4096]byte 37 | n, err := r.Read(buf[:]) 38 | return &forcedReader{ 39 | buf: buf[:n], 40 | err: err, 41 | r: r, 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /ranger/httpranger/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package httpranger implements lazy io.Reader and io.Writer interfaces. 5 | package httpranger 6 | 7 | import ( 8 | "github.com/spacemonkeygo/monkit/v3" 9 | "github.com/zeebo/errs" 10 | ) 11 | 12 | // Error is the errs class of standard Ranger errors. 13 | var Error = errs.Class("ranger") 14 | 15 | var mon = monkit.Package() 16 | -------------------------------------------------------------------------------- /ranger/readerat.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package ranger 5 | 6 | import ( 7 | "context" 8 | "io" 9 | ) 10 | 11 | type readerAtRanger struct { 12 | r io.ReaderAt 13 | size int64 14 | } 15 | 16 | // ReaderAtRanger converts a ReaderAt with a given size to a Ranger. 17 | func ReaderAtRanger(r io.ReaderAt, size int64) Ranger { 18 | return &readerAtRanger{ 19 | r: r, 20 | size: size, 21 | } 22 | } 23 | 24 | func (r *readerAtRanger) Size() int64 { 25 | return r.size 26 | } 27 | 28 | type readerAtReader struct { 29 | r io.ReaderAt 30 | offset, length int64 31 | } 32 | 33 | func (r *readerAtRanger) Range(ctx context.Context, offset, length int64) (_ io.ReadCloser, err error) { 34 | defer mon.Task()(&ctx)(&err) 35 | if offset < 0 { 36 | return nil, Error.New("negative offset") 37 | } 38 | if length < 0 { 39 | return nil, Error.New("negative length") 40 | } 41 | if offset+length > r.size { 42 | return nil, Error.New("buffer runoff") 43 | } 44 | return &readerAtReader{r: r.r, offset: offset, length: length}, nil 45 | } 46 | 47 | func (r *readerAtReader) Read(p []byte) (n int, err error) { 48 | if r.length == 0 { 49 | return 0, io.EOF 50 | } 51 | if int64(len(p)) > r.length { 52 | p = p[:r.length] 53 | } 54 | n, err = r.r.ReadAt(p, r.offset) 55 | r.offset += int64(n) 56 | r.length -= int64(n) 57 | return n, err 58 | } 59 | 60 | func (r *readerAtReader) Close() error { 61 | return nil 62 | } 63 | -------------------------------------------------------------------------------- /ranger/readerat_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package ranger 5 | 6 | import ( 7 | "context" 8 | "fmt" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestRange(t *testing.T) { 15 | for i, tt := range []struct { 16 | offset int64 17 | length int64 18 | size int64 19 | }{ 20 | {offset: -2, size: 0}, 21 | {offset: 2, length: -1, size: 0}, 22 | } { 23 | tag := fmt.Sprintf("#%d. %+v", i, tt) 24 | 25 | rr := readerAtRanger{size: tt.size} 26 | closer, err := rr.Range(context.Background(), tt.offset, tt.length) 27 | assert.Nil(t, closer, tag) 28 | assert.NotNil(t, err, tag) 29 | } 30 | } 31 | 32 | func TestClose(t *testing.T) { 33 | rr := readerAtReader{length: 0} 34 | assert.Nil(t, rr.Close()) 35 | } 36 | -------------------------------------------------------------------------------- /readcloser/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package readcloser implements utilities for io.ReadClosers. 5 | package readcloser 6 | -------------------------------------------------------------------------------- /readcloser/fatal.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package readcloser 5 | 6 | import "io" 7 | 8 | // FatalReadCloser returns a ReadCloser that always fails with err. 9 | func FatalReadCloser(err error) io.ReadCloser { 10 | return &fatalReadCloser{Err: err} 11 | } 12 | 13 | type fatalReadCloser struct { 14 | Err error 15 | } 16 | 17 | func (f *fatalReadCloser) Read(p []byte) (n int, err error) { 18 | return 0, f.Err 19 | } 20 | 21 | func (f *fatalReadCloser) Close() error { 22 | return nil 23 | } 24 | -------------------------------------------------------------------------------- /readcloser/lazy.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package readcloser 5 | 6 | import "io" 7 | 8 | // LazyReadCloser returns an ReadCloser that doesn't initialize the backing 9 | // Reader until the first Read. 10 | func LazyReadCloser(reader func() (io.ReadCloser, error)) io.ReadCloser { 11 | return &lazyReadCloser{fn: reader} 12 | } 13 | 14 | type lazyReadCloser struct { 15 | fn func() (io.ReadCloser, error) 16 | r io.ReadCloser 17 | } 18 | 19 | func (l *lazyReadCloser) Read(p []byte) (n int, err error) { 20 | if l.r == nil { 21 | l.r, err = l.fn() 22 | if err != nil { 23 | return 0, err 24 | } 25 | l.fn = nil 26 | } 27 | return l.r.Read(p) 28 | } 29 | 30 | func (l *lazyReadCloser) Close() error { 31 | if l.r != nil { 32 | return l.r.Close() 33 | } 34 | return nil 35 | } 36 | -------------------------------------------------------------------------------- /readcloser/limit.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package readcloser 5 | 6 | import "io" 7 | 8 | // LimitReadCloser is a LimitReader extension that returns a ReadCloser 9 | // that reads from r but stops with EOF after n bytes. 10 | func LimitReadCloser(r io.ReadCloser, n int64) io.ReadCloser { 11 | return &limitedReadCloser{io.LimitReader(r, n), r} 12 | } 13 | 14 | type limitedReadCloser struct { 15 | R io.Reader 16 | C io.Closer 17 | } 18 | 19 | func (l *limitedReadCloser) Read(p []byte) (n int, err error) { 20 | return l.R.Read(p) 21 | } 22 | 23 | func (l *limitedReadCloser) Close() error { 24 | return l.C.Close() 25 | } 26 | -------------------------------------------------------------------------------- /rpc/conn.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package rpc 5 | 6 | import ( 7 | "github.com/zeebo/errs" 8 | 9 | "storj.io/common/identity" 10 | "storj.io/common/rpc/rpcpool" 11 | ) 12 | 13 | // Conn is a wrapper around a drpc client connection. 14 | type Conn struct { 15 | rpcpool.Conn 16 | } 17 | 18 | // PeerIdentity returns the peer identity on the other end of the connection. 19 | func (c *Conn) PeerIdentity() (*identity.PeerIdentity, error) { 20 | state := c.Conn.State() 21 | if state == nil { 22 | return nil, errs.New("unknown identity: need to communicate first") 23 | } 24 | return identity.PeerIdentityFromChain(state.PeerCertificates) 25 | } 26 | -------------------------------------------------------------------------------- /rpc/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package rpc implements dialing on Storj Network. 5 | package rpc 6 | -------------------------------------------------------------------------------- /rpc/hybrid_expose_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2023 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package rpc 5 | 6 | // Connectors returns the list of connectors for testing. 7 | func (c *HybridConnector) Connectors() []candidateConnector { //nolint: revive // for testing 8 | return c.connectors 9 | } 10 | 11 | // Connector returns the actual connector for testing. 12 | func (c candidateConnector) Connector() Connector { 13 | return c.connector 14 | } 15 | -------------------------------------------------------------------------------- /rpc/hybrid_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package rpc_test 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | 11 | "storj.io/common/memory" 12 | "storj.io/common/rpc" 13 | _ "storj.io/common/rpc/quic" // register quic connector 14 | ) 15 | 16 | func TestSetTransferRate(t *testing.T) { 17 | hybridConnector := rpc.NewHybridConnector() 18 | hybridConnector.SetTransferRate(memory.GB) 19 | var n int 20 | for _, candidate := range hybridConnector.Connectors() { 21 | if connector, ok := candidate.Connector().(*rpc.TCPConnector); ok { 22 | assert.Equal(t, memory.GB, connector.TransferRate) 23 | n++ 24 | } 25 | } 26 | assert.Greater(t, n, 0, "expected at least one *TCPConnector") 27 | } 28 | 29 | func TestSetSendDRPCMuxHeader(t *testing.T) { 30 | hybridConnector := rpc.NewHybridConnector() 31 | hybridConnector.SetSendDRPCMuxHeader(false) 32 | var n int 33 | for _, candidate := range hybridConnector.Connectors() { 34 | if connector, ok := candidate.Connector().(*rpc.TCPConnector); ok { 35 | assert.False(t, connector.SendDRPCMuxHeader) 36 | n++ 37 | } 38 | } 39 | assert.Greater(t, n, 0, "expected at least one *TCPConnector") 40 | } 41 | -------------------------------------------------------------------------------- /rpc/known_ids_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package rpc_test 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | 11 | "storj.io/common/rpc" 12 | ) 13 | 14 | func TestKnownID(t *testing.T) { 15 | id, ok := rpc.KnownNodeID("us-central-1.tardigrade.io:7777") 16 | require.True(t, ok) 17 | require.Equal(t, id.String(), "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S") 18 | _, ok = rpc.KnownNodeID("non-existent.example.com:7777") 19 | require.False(t, ok) 20 | 21 | id, ok = rpc.KnownNodeID("us-central-1.tardigrade.io:10000") 22 | require.True(t, ok) 23 | require.Equal(t, id.String(), "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S") 24 | 25 | id, ok = rpc.KnownNodeID("us-central-1.tardigrade.io") 26 | require.True(t, ok) 27 | require.Equal(t, id.String(), "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S") 28 | } 29 | -------------------------------------------------------------------------------- /rpc/lookup.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package rpc 5 | 6 | import ( 7 | "context" 8 | "net" 9 | ) 10 | 11 | // LookupNodeAddress resolves a storage node address to the first IP address resolved. 12 | // If an IP address is accidentally provided it is returned back. This function 13 | // is used to resolve storage node IP addresses so that uplinks can use 14 | // IP addresses directly without resolving many hosts. 15 | func LookupNodeAddress(ctx context.Context, nodeAddress string) string { 16 | host, port, err := net.SplitHostPort(nodeAddress) 17 | if err != nil { 18 | // If there was an error parsing out the port we just use a plain host. 19 | host = nodeAddress 20 | port = "" 21 | } 22 | 23 | // We check if the address is an IP address to decide if we need to resolve it or not. 24 | ip := net.ParseIP(host) 25 | // nodeAddress is already an IP, so we can use that. 26 | if ip != nil { 27 | return nodeAddress 28 | } 29 | 30 | // We have a hostname not an IP address so we should resolve the IP address 31 | // to give back to the uplink client. 32 | addresses, err := net.DefaultResolver.LookupHost(ctx, host) 33 | if err != nil || len(addresses) == 0 { 34 | // We ignore the error because if this fails for some reason we can just 35 | // re-use the hostname, it just won't be as fast for the uplink to dial. 36 | return nodeAddress 37 | } 38 | 39 | // We return the first address found because some DNS servers already do 40 | // round robin load balancing and we would be messing with their behaviour 41 | // if we tried to get smart here. 42 | first := addresses[0] 43 | 44 | if port == "" { 45 | return first 46 | } 47 | return net.JoinHostPort(first, port) 48 | } 49 | -------------------------------------------------------------------------------- /rpc/lookup_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package rpc_test 5 | 6 | import ( 7 | "context" 8 | "net" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/assert" 12 | 13 | "storj.io/common/rpc" 14 | "storj.io/common/testcontext" 15 | ) 16 | 17 | func TestLookupNodeAddress_Host(t *testing.T) { 18 | // When we provide a host to LookupHostFirstAddress we should get a valid IP address back. 19 | address := rpc.LookupNodeAddress(context.Background(), "google.com") 20 | 21 | // Verify we get a properly formatted IP address back. 22 | ip := net.ParseIP(address) 23 | assert.NotNil(t, ip) 24 | } 25 | 26 | func TestLookupNodeAddress_HostAndPort(t *testing.T) { 27 | // When we provide a host to LookupHostFirstAddress we should get a valid IP address and port back. 28 | address := rpc.LookupNodeAddress(context.Background(), "google.com:8888") 29 | 30 | // Verify we get a properly formatted IP address back. 31 | host, port, err := net.SplitHostPort(address) 32 | assert.NoError(t, err) 33 | assert.Equal(t, "8888", port) 34 | assert.NotNil(t, net.ParseIP(host)) 35 | } 36 | 37 | func TestLookupNodeAddress_IP(t *testing.T) { 38 | ctx := testcontext.New(t) 39 | 40 | tests := []string{ 41 | "8.8.8.8", 42 | "2001:4860:4860::8888", 43 | "192.168.0.1:8888", 44 | "[2001:4860:4860::8888]:8888", 45 | } 46 | for _, test := range tests { 47 | address := rpc.LookupNodeAddress(ctx, test) 48 | assert.Equal(t, test, address) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /rpc/multidial/utils.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2023 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package multidial 5 | 6 | import ( 7 | "net" 8 | "sync/atomic" 9 | ) 10 | 11 | type addr struct { 12 | network string 13 | address string 14 | } 15 | 16 | func (a addr) Network() string { return a.network } 17 | func (a addr) String() string { return a.address } 18 | 19 | type atomicConn struct { 20 | conn atomic.Value 21 | } 22 | 23 | func (c *atomicConn) Store(conn net.Conn) { 24 | c.conn.Store(conn) 25 | } 26 | 27 | func (c *atomicConn) Load() (net.Conn, bool) { 28 | conn, ok := c.conn.Load().(net.Conn) 29 | return conn, ok 30 | } 31 | -------------------------------------------------------------------------------- /rpc/quic/common.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | //go:build go1.18 5 | 6 | package quic 7 | 8 | import ( 9 | "github.com/spacemonkeygo/monkit/v3" 10 | "github.com/zeebo/errs" 11 | ) 12 | 13 | var ( 14 | mon = monkit.Package() 15 | 16 | // Error is a pkg/quic error. 17 | Error = errs.Class("quic") 18 | 19 | // ErrQuicDisabled indicates QUIC has been disabled at build time. 20 | ErrQuicDisabled = Error.New("disabled at build time") 21 | ) 22 | -------------------------------------------------------------------------------- /rpc/quic/connector_noquic.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | //go:build !go1.20 || noquic 5 | 6 | package quic 7 | 8 | import ( 9 | "context" 10 | "crypto/tls" 11 | 12 | "storj.io/common/memory" 13 | "storj.io/common/rpc" 14 | ) 15 | 16 | // Connector implements a stub dialer that always fails. 17 | type Connector struct{} 18 | 19 | // NewDefaultConnector returns a stub connector that always fails. 20 | func NewDefaultConnector(quicConfig interface{}) Connector { 21 | return Connector{} 22 | } 23 | 24 | // DialContext returns a failure. 25 | func (c Connector) DialContext(ctx context.Context, tlsConfig *tls.Config, address string) (_ rpc.ConnectorConn, err error) { 26 | return nil, ErrQuicDisabled 27 | } 28 | 29 | // SetTransferRate has no effect. 30 | func (c *Connector) SetTransferRate(rate memory.Size) {} 31 | 32 | // TransferRate returns zero. 33 | func (c Connector) TransferRate() memory.Size { 34 | return 0 35 | } 36 | -------------------------------------------------------------------------------- /rpc/quic/errorhandling.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2023 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | //go:build !windows 5 | 6 | package quic 7 | 8 | func isMsgSizeErr(err error) bool { 9 | // *nix doesn't return a size error from Accept. 10 | return false 11 | } 12 | -------------------------------------------------------------------------------- /rpc/quic/errorhandling_windows.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2023 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | //go:build windows 5 | 6 | package quic 7 | 8 | import ( 9 | "errors" 10 | 11 | "golang.org/x/sys/windows" 12 | ) 13 | 14 | func isMsgSizeErr(err error) bool { 15 | return errors.Is(err, windows.WSAEMSGSIZE) 16 | } 17 | -------------------------------------------------------------------------------- /rpc/quic/init.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | //go:build go1.20 && !noquic 5 | 6 | package quic 7 | 8 | import ( 9 | "storj.io/common/rpc" 10 | ) 11 | 12 | const quicConnectorPriority = 20 13 | 14 | func init() { 15 | rpc.RegisterCandidateConnectorType("quic", func() rpc.Connector { 16 | return NewDefaultConnector(nil) 17 | }, quicConnectorPriority) 18 | } 19 | -------------------------------------------------------------------------------- /rpc/quic/listener_noquic.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | //go:build !go1.20 || noquic 5 | 6 | package quic 7 | 8 | import ( 9 | "crypto/tls" 10 | "net" 11 | "sync" 12 | ) 13 | 14 | // Ignore unused warnings when building without quic. 15 | var _ = isMsgSizeErr 16 | var _ = mon 17 | 18 | // Listener implements a stub/noop listener. 19 | type Listener struct { 20 | isClosed bool 21 | closedMutex sync.Mutex 22 | closedCond *sync.Cond 23 | } 24 | 25 | // NewListener returns a new stub/noop listener. It will never return a connection. 26 | func NewListener(conn *net.UDPConn, tlsConfig *tls.Config, quicConfig interface{}) (net.Listener, error) { 27 | l := &Listener{} 28 | l.closedCond = sync.NewCond(&l.closedMutex) 29 | return l, nil 30 | } 31 | 32 | // Accept simply blocks until the listener is closed. 33 | func (l *Listener) Accept() (net.Conn, error) { 34 | l.closedMutex.Lock() 35 | defer l.closedMutex.Unlock() 36 | for !l.isClosed { 37 | l.closedCond.Wait() 38 | } 39 | return nil, net.ErrClosed 40 | } 41 | 42 | // Close closes the listener. 43 | func (l *Listener) Close() (err error) { 44 | l.closedMutex.Lock() 45 | l.isClosed = true 46 | l.closedMutex.Unlock() 47 | l.closedCond.Broadcast() 48 | return nil 49 | } 50 | 51 | type dummyAddr struct{} 52 | 53 | func (d dummyAddr) Network() string { 54 | return "dummy" 55 | } 56 | 57 | func (d dummyAddr) String() string { 58 | return "dummy" 59 | } 60 | 61 | // Addr returns the local network addr that the server is listening on. 62 | func (l *Listener) Addr() net.Addr { 63 | return dummyAddr{} 64 | } 65 | -------------------------------------------------------------------------------- /rpc/rpcpool/common.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package rpcpool implements connection pooling for rpc. 5 | package rpcpool 6 | 7 | import ( 8 | "github.com/spacemonkeygo/monkit/v3" 9 | "github.com/zeebo/errs" 10 | ) 11 | 12 | var mon = monkit.Package() 13 | 14 | // Error is the class of errors returned by this package. 15 | var Error = errs.Class("rpcpool") 16 | -------------------------------------------------------------------------------- /rpc/rpcpool/wrapper.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package rpcpool 5 | 6 | import "context" 7 | 8 | type dialerWrapper struct{} 9 | 10 | // DialerWrapper can create enhanced functionality. 11 | type DialerWrapper = func(ctx context.Context, dialer Dialer) Dialer 12 | 13 | // WithDialerWrapper creates context with DialerWrapper used by dial.Invoke. 14 | func WithDialerWrapper(ctx context.Context, wrapper DialerWrapper) context.Context { 15 | return context.WithValue(ctx, dialerWrapper{}, wrapper) 16 | } 17 | 18 | // GetWrapper returns with the dialerWrapper if registered. 19 | func GetWrapper(ctx context.Context) (DialerWrapper, bool) { 20 | wrapper, ok := ctx.Value(dialerWrapper{}).(DialerWrapper) 21 | return wrapper, ok 22 | } 23 | 24 | // WrapDialer returns with dialed which may be wrapped if wrapper is registered. 25 | func WrapDialer(ctx context.Context, dialer Dialer) Dialer { 26 | wrapper, found := GetWrapper(ctx) 27 | if !found { 28 | return dialer 29 | } 30 | return wrapper(ctx, dialer) 31 | } 32 | -------------------------------------------------------------------------------- /rpc/rpcstatus/status_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package rpcstatus 5 | 6 | import ( 7 | "context" 8 | "errors" 9 | "fmt" 10 | "strings" 11 | "testing" 12 | 13 | "github.com/stretchr/testify/assert" 14 | 15 | "storj.io/drpc/drpcerr" 16 | ) 17 | 18 | var allCodes = []StatusCode{ 19 | Unknown, 20 | OK, 21 | Canceled, 22 | InvalidArgument, 23 | DeadlineExceeded, 24 | NotFound, 25 | AlreadyExists, 26 | PermissionDenied, 27 | ResourceExhausted, 28 | FailedPrecondition, 29 | Aborted, 30 | OutOfRange, 31 | Unimplemented, 32 | Internal, 33 | Unavailable, 34 | DataLoss, 35 | Unauthenticated, 36 | MethodNotAllowed, 37 | } 38 | 39 | func TestStatus(t *testing.T) { 40 | for _, code := range allCodes { 41 | err := Error(code, "") 42 | assert.Equal(t, Code(err), code) 43 | assert.Equal(t, drpcerr.Code(err), uint64(code)) 44 | } 45 | 46 | assert.Equal(t, Code(nil), OK) 47 | assert.Equal(t, Code(context.Canceled), Canceled) 48 | assert.Equal(t, Code(context.DeadlineExceeded), DeadlineExceeded) 49 | } 50 | 51 | func TestStatus_WrapFormatting(t *testing.T) { 52 | err := Wrap(Internal, errors.New("test")) 53 | assert.True(t, strings.Count(fmt.Sprintf("%+v", err), "\n") > 0) 54 | } 55 | -------------------------------------------------------------------------------- /rpc/rpctest/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package rpctest holds helpers for development/testing. 5 | // please check the example of the rpc/rpcpool/wrapper for a real word usage. 6 | package rpctest 7 | -------------------------------------------------------------------------------- /rpc/rpctest/latency.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package rpctest 5 | 6 | import ( 7 | "time" 8 | 9 | "storj.io/common/rpc/rpcpool" 10 | "storj.io/drpc" 11 | ) 12 | 13 | // ConnectionWithLatency wraps the original connection and add certain latency to it. 14 | func ConnectionWithLatency(conn rpcpool.RawConn, duration time.Duration) rpcpool.RawConn { 15 | return &MessageInterceptor{ 16 | delegate: conn, 17 | ResponseHook: func(rpc string, message drpc.Message, err error) { 18 | time.Sleep(duration) 19 | }, 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /rpc/rpctest/recorder_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package rpctest_test 5 | 6 | import ( 7 | "context" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/require" 11 | 12 | "storj.io/common/rpc/rpctest" 13 | "storj.io/drpc" 14 | ) 15 | 16 | func TestCounter(t *testing.T) { 17 | type helloMessage struct { 18 | Message string 19 | } 20 | 21 | // the original service with fake answer 22 | original := rpctest.NewStubConnection() 23 | original.RegisterHandler("func1", func(ctx context.Context, rpc string, enc drpc.Encoding, in, out drpc.Message) error { 24 | out.(*helloMessage).Message = "foobar" 25 | return nil 26 | }) 27 | 28 | callRecorder := rpctest.NewCallRecorder() 29 | wrapper := callRecorder.Attach(&original) 30 | 31 | in := helloMessage{Message: "hello"} 32 | out := helloMessage{} 33 | err := wrapper.Invoke(context.TODO(), "func1", nil, &in, &out) 34 | require.Nil(t, err) 35 | require.Equal(t, "foobar", out.Message) 36 | require.Equal(t, 1, callRecorder.CountOf("func1")) 37 | } 38 | 39 | func TestCounterAssert(t *testing.T) { 40 | c := rpctest.NewCallRecorder() 41 | c.RecordCall("first") 42 | c.RecordCall("first") 43 | c.RecordCall("second") 44 | 45 | // be sure we have copy, not reference 46 | c.History()[0] = "x" 47 | 48 | require.Equal(t, c.History(), []string{"first", "first", "second"}) 49 | } 50 | -------------------------------------------------------------------------------- /rpc/rpctest/wrappers_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package rpctest 5 | 6 | import ( 7 | "context" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/require" 11 | 12 | "storj.io/drpc" 13 | ) 14 | 15 | type message struct { 16 | content string 17 | } 18 | 19 | func TestMessageInterceptor(t *testing.T) { 20 | requestCounter := 0 21 | responseCounter := 0 22 | 23 | // the original service with fake answer 24 | original := NewStubConnection() 25 | original.RegisterHandler("test", func(ctx context.Context, rpc string, enc drpc.Encoding, in, out drpc.Message) error { 26 | out.(*message).content = "foobar" 27 | return nil 28 | }) 29 | 30 | // the wrapper, counts the invocations 31 | wrapper := NewMessageInterceptor(&original) 32 | wrapper.RequestHook = func(rpc string, message drpc.Message, err error) { 33 | requestCounter++ 34 | } 35 | wrapper.ResponseHook = func(rpc string, message drpc.Message, err error) { 36 | responseCounter++ 37 | } 38 | 39 | in := message{content: "hello"} 40 | out := message{} 41 | 42 | // sync call 43 | err := wrapper.Invoke(context.TODO(), "test", nil, &in, &out) 44 | require.Nil(t, err) 45 | require.Equal(t, "foobar", out.content) 46 | 47 | out = message{} 48 | 49 | // async call 50 | stream, err := wrapper.NewStream(context.TODO(), "test", nil) 51 | require.Nil(t, err) 52 | 53 | // send it 54 | err = stream.MsgSend(&in, nil) 55 | require.Nil(t, err) 56 | 57 | // wait for the response 58 | err = stream.MsgRecv(&out, nil) 59 | require.Nil(t, err) 60 | require.Equal(t, "foobar", out.content) 61 | 62 | require.Equal(t, 2, requestCounter) 63 | require.Equal(t, 2, responseCounter) 64 | 65 | } 66 | -------------------------------------------------------------------------------- /rpc/rpctimeout/timeout.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package rpctimeout provides helpers to have timeouts on rpc streams. 5 | package rpctimeout 6 | 7 | import ( 8 | "context" 9 | "time" 10 | ) 11 | 12 | // Run runs the provided function with a context that will be canceled after 13 | // the provided duration or when the provided function returns. It returns either 14 | // the error from the context or the error from the function. It runs the function 15 | // in its own goroutine and DOES NOT wait for it to exit. This is on purpose to get 16 | // around some grpc brain damage with respect to canceling operations on server streams. 17 | func Run(ctx context.Context, timeout time.Duration, fn func(ctx context.Context) error) error { 18 | ctx, cancel := context.WithTimeout(ctx, timeout) 19 | defer cancel() 20 | 21 | errch := make(chan error, 1) 22 | go func() { errch <- fn(ctx) }() 23 | 24 | select { 25 | case <-ctx.Done(): 26 | return ctx.Err() 27 | case err := <-errch: 28 | return err 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /rpc/rpctracing/common.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package rpctracing implements tracing for rpc. 5 | package rpctracing 6 | 7 | import ( 8 | "github.com/spacemonkeygo/monkit/v3" 9 | ) 10 | 11 | const ( 12 | // TraceID is the key we use to store trace id value into context. 13 | TraceID = "trace-id" 14 | // ParentID is the key we use to store parent's span id value into context. 15 | ParentID = "parent-id" 16 | // Sampled is the key we use to store sampled flag into context. 17 | Sampled = "sampled" 18 | // TraceHost is the host to send the traces to. If unprovided, the default 19 | // is used. 20 | TraceHost = "trace-host" 21 | ) 22 | 23 | var mon = monkit.Package() 24 | -------------------------------------------------------------------------------- /rpc/rpctracing/excluded.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package rpctracing 5 | 6 | import ( 7 | "context" 8 | 9 | "github.com/spacemonkeygo/monkit/v3" 10 | 11 | "storj.io/common/tracing" 12 | ) 13 | 14 | // WithoutDistributedTracing disables distributed tracing for the current span. 15 | // Deprecated: use tracing.WithoutDistributedTracing. 16 | func WithoutDistributedTracing(ctx context.Context) context.Context { 17 | return tracing.WithoutDistributedTracing(ctx) 18 | } 19 | 20 | // IsExcluded check if span shouldn't be reported to remote location. 21 | // Deprecated: use tracing.IsExcluded. 22 | func IsExcluded(span *monkit.Span) bool { 23 | return tracing.IsExcluded(span) 24 | } 25 | -------------------------------------------------------------------------------- /scripts/arm-staticcheck.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -x 3 | export GOOS=linux 4 | export GOARCH=arm 5 | find -name "*.go" | awk -F / '{print $2}' | sort | uniq | grep -v -E '(http|sqlite)' | xargs -IDIR staticcheck storj.io/common/DIR 6 | -------------------------------------------------------------------------------- /scripts/check-cross-compile.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | find -name "*.go" | awk -F / '{print $2}' | sort | uniq | grep -v -E '(process|metrics|db)' | xargs -IDIR check-cross-compile -compiler "go,go.min" storj.io/common/DIR/... 3 | -------------------------------------------------------------------------------- /scripts/check-dependencies.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -uo pipefail 3 | set +x 4 | 5 | # This script verifies that we don't accidentally import specific packages. 6 | 7 | if ! check-dependency -ignore "cockroachutil;dbutil" -check "github.com/(lib/pq|jackc/pg)" ./...; then 8 | echo "we shouldn't import postgres outside of database packages"; 9 | exit -1; 10 | fi 11 | 12 | if ! check-dependency -check "redis;bolt" ./...; then 13 | echo "common must not have a dependency to redis or bolt"; 14 | exit -1; 15 | fi 16 | 17 | if ! check-dependency -ignore "test;common/process;common/rpc/quic" -check "test" -except "internal/testlog" ./...; then 18 | echo "packages not related to test functionality should not bring in testing related things"; 19 | exit -1; 20 | fi 21 | 22 | if ! check-dependency -ignore "accesslogs;cfgstruct;dbutil;debug;httpranger;metrics;migrate;process;requestid;storj/location/gen;tagsql;test;traces;version" -check "net/http" ./...; then 23 | echo "net/http is a huge dependency that we don't want to accidentally introduce into uplink"; 24 | exit -1; 25 | fi 26 | -------------------------------------------------------------------------------- /signing/disabled.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2023 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package signing 5 | 6 | import ( 7 | "context" 8 | "os" 9 | "strconv" 10 | ) 11 | 12 | var ( 13 | disabledGlobally, _ = strconv.ParseBool(os.Getenv("STORJ_DISABLE_SIGNING")) 14 | 15 | // Takes the place of signatures when they are disabled. The presence 16 | // of a "signature" is required because some components check that the 17 | // signature field has len > 0. 18 | disabledSignature = []byte("DISABLED-SIGNATURE") 19 | ) 20 | 21 | type signaturesDisabledForTestKey struct{} 22 | 23 | // Disabled returns true if signatures are disabled. If disabled, signatures 24 | // are set to "DISABLED-SIGNATURE" and are ignored during verification. 25 | func Disabled() bool { 26 | return disabledGlobally 27 | } 28 | 29 | func areSignaturesDisabled(ctx context.Context) bool { 30 | if disabledGlobally { 31 | return true 32 | } 33 | _, disabledForTest := ctx.Value(signaturesDisabledForTestKey{}).(struct{}) 34 | return disabledForTest 35 | } 36 | 37 | func withSignaturesDisabledForTest(ctx context.Context) context.Context { 38 | return context.WithValue(ctx, signaturesDisabledForTestKey{}, struct{}{}) 39 | } 40 | -------------------------------------------------------------------------------- /signing/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package signing implements consistent signing and verifying protobuf messages. 5 | package signing 6 | -------------------------------------------------------------------------------- /socket/dscp.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package socket 5 | 6 | // A DSCP is a Differentiated Services Code Point. 7 | // 8 | //nolint:unused 9 | type dscp byte 10 | 11 | // See https://tools.ietf.org/html/rfc4594#section-2.3 for the definitions 12 | // of the below Differentiated Services Code Points. 13 | // 14 | //nolint:deadcode,varcheck,unused 15 | const ( 16 | dscpDF dscp = 0 17 | dscpCS6 dscp = 0b110000 18 | dscpEF dscp = 0b101110 19 | dscpCS5 dscp = 0b101000 20 | dscpAF41 dscp = 0b100010 21 | dscpAF42 dscp = 0b100100 22 | dscpAF43 dscp = 0b100110 23 | dscpCS4 dscp = 0b100000 24 | dscpAF31 dscp = 0b011010 25 | dscpAF32 dscp = 0b011100 26 | dscpAF33 dscp = 0b011110 27 | dscpCS3 dscp = 0b011000 28 | dscpAF21 dscp = 0b010010 29 | dscpAF22 dscp = 0b010100 30 | dscpAF23 dscp = 0b010110 31 | dscpCS2 dscp = 0b010000 32 | dscpAF11 dscp = 0b001010 33 | dscpAF12 dscp = 0b001100 34 | dscpAF13 dscp = 0b001110 35 | dscpCS1 dscp = 0b001000 36 | dscpLE dscp = 0b000001 37 | ) 38 | -------------------------------------------------------------------------------- /socket/extended_darwin.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package socket 5 | 6 | // TCPFastOpenConnectSupported is true if TCPFastOpenConnect is supported on this platform. 7 | const TCPFastOpenConnectSupported = false 8 | 9 | func setLowPrioCongestionController(fd uintptr) error { 10 | // TODO: https://stackoverflow.com/questions/8532372/how-to-load-a-different-congestion-control-algorithm-in-mac-os-x 11 | return nil 12 | } 13 | 14 | func setLowEffortQoS(fd uintptr) error { 15 | // TODO 16 | return nil 17 | } 18 | 19 | func setTCPFastOpenConnect(fd uintptr) error { 20 | // TODO 21 | return nil 22 | } 23 | -------------------------------------------------------------------------------- /socket/extended_linux.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package socket 5 | 6 | import ( 7 | "os" 8 | "strconv" 9 | "syscall" 10 | ) 11 | 12 | // TCPFastOpenConnectSupported is true if TCPFastOpenConnect is supported on this platform. 13 | var TCPFastOpenConnectSupported = func() bool { 14 | if envVar := os.Getenv("STORJ_SOCKET_TCP_FASTOPEN_CONNECT"); envVar != "" { 15 | if supported, err := strconv.ParseBool(envVar); err == nil { 16 | return supported 17 | } 18 | } 19 | // default 20 | return true 21 | }() 22 | 23 | var linuxLowPrioCongController = os.Getenv("STORJ_SOCKET_LOWPRIO_CTL") 24 | 25 | const tcpFastOpenConnect = 30 26 | 27 | func setLowPrioCongestionController(fd uintptr) error { 28 | if linuxLowPrioCongController != "" { 29 | return syscall.SetsockoptString(int(fd), syscall.IPPROTO_TCP, syscall.TCP_CONGESTION, linuxLowPrioCongController) 30 | } 31 | return nil 32 | } 33 | 34 | func setLowEffortQoS(fd uintptr) error { 35 | return syscall.SetsockoptByte(int(fd), syscall.SOL_IP, syscall.IP_TOS, byte(dscpLE)<<2) 36 | } 37 | 38 | func setTCPFastOpenConnect(fd uintptr) error { 39 | return syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, tcpFastOpenConnect, 1) 40 | } 41 | -------------------------------------------------------------------------------- /socket/extended_other.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | //go:build !linux && !darwin && !windows 5 | 6 | package socket 7 | 8 | // TCPFastOpenConnectSupported is true if TCPFastOpenConnect is supported on this platform. 9 | const TCPFastOpenConnectSupported = false 10 | 11 | func setLowPrioCongestionController(fd uintptr) error { return nil } 12 | 13 | func setLowEffortQoS(fd uintptr) error { return nil } 14 | 15 | func setTCPFastOpenConnect(fd uintptr) error { return nil } 16 | -------------------------------------------------------------------------------- /socket/extended_windows.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package socket 5 | 6 | // TCPFastOpenConnectSupported is true if TCPFastOpenConnect is supported on this platform. 7 | const TCPFastOpenConnectSupported = false 8 | 9 | func setLowPrioCongestionController(fd uintptr) error { 10 | // TODO: Evidently some Windowses come with LEDBAT? A hint: 11 | // https://deploymentresearch.com/setup-low-extra-delay-background-transport-ledbat-for-configmgr/ 12 | return nil 13 | } 14 | 15 | func setLowEffortQoS(fd uintptr) error { 16 | // TODO 17 | return nil 18 | } 19 | 20 | func setTCPFastOpenConnect(fd uintptr) error { 21 | // TODO: can we use standard tcp fast open without a send call, or 22 | // does windows have a version of TCP_FASTOPEN_CONNECT? 23 | return nil 24 | } 25 | -------------------------------------------------------------------------------- /storj/default.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package storj 5 | 6 | import "encoding/base32" 7 | 8 | // base32Encoding is base32 without padding. 9 | var base32Encoding = base32.StdEncoding.WithPadding(base32.NoPadding) 10 | -------------------------------------------------------------------------------- /storj/deleteobjects.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2025 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package storj 5 | 6 | // DeleteObjectsStatus represents the success or failure status of an individual DeleteObjects deletion. 7 | type DeleteObjectsStatus int 8 | 9 | const ( 10 | // DeleteObjectsStatusInternalError indicates that the deletion was not processed due to an internal error. 11 | DeleteObjectsStatusInternalError = DeleteObjectsStatus(0) 12 | 13 | // DeleteObjectsStatusUnauthorized indicates that the deletion was not processed due to insufficient privileges. 14 | DeleteObjectsStatusUnauthorized = DeleteObjectsStatus(1) 15 | 16 | // DeleteObjectsStatusNotFound indicates that the object could not be deleted because it did not exist. 17 | DeleteObjectsStatusNotFound = DeleteObjectsStatus(2) 18 | 19 | // DeleteObjectsStatusOK indicates that the object was successfully deleted. 20 | DeleteObjectsStatusOK = DeleteObjectsStatus(3) 21 | 22 | // DeleteObjectsStatusLocked indicates that the object's Object Lock configuration prevented its deletion. 23 | DeleteObjectsStatusLocked = DeleteObjectsStatus(4) 24 | ) 25 | -------------------------------------------------------------------------------- /storj/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package storj contains the types which represent the main entities of the Storj domain. 5 | package storj 6 | -------------------------------------------------------------------------------- /storj/location/countrycode_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package location 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestCountryCode_String(t *testing.T) { 13 | require.Equal(t, CountryCode(countryCodeZZ), ToCountryCode("ZZ")) 14 | require.Equal(t, "HU", ToCountryCode("HU").String()) 15 | require.Equal(t, "DE", ToCountryCode("DE").String()) 16 | require.Equal(t, "XX", ToCountryCode("XX").String()) 17 | require.Equal(t, "ZZ", ToCountryCode("ZZ").String()) 18 | require.Equal(t, "", None.String()) 19 | } 20 | 21 | func TestCountryCode_SQLConversion(t *testing.T) { 22 | p := Cyprus 23 | value, err := p.Value() 24 | require.NoError(t, err) 25 | 26 | res := new(CountryCode) 27 | err = res.Scan(value) 28 | require.NoError(t, err) 29 | require.Equal(t, Cyprus, *res) 30 | 31 | err = res.Scan(nil) 32 | require.NoError(t, err) 33 | require.Equal(t, None, *res) 34 | 35 | err = res.Scan(123) 36 | require.Error(t, err) 37 | } 38 | 39 | var sink string 40 | 41 | var countriesForTesting = []CountryCode{ 42 | Austria, Belgium, Bulgaria, Croatia, Cyprus, Czechia, Denmark, Estonia, 43 | Finland, France, Germany, Greece, Hungary, Ireland, Italy, Lithuania, 44 | Latvia, Luxembourg, Malta, TheNetherlands, Poland, Portugal, Romania, Slovenia, 45 | Slovakia, Spain, Sweden, 46 | } 47 | 48 | func BenchmarkCountryCode_String(b *testing.B) { 49 | for i := 0; i < b.N; i++ { 50 | code := countriesForTesting[i%len(countriesForTesting)] 51 | sink = code.String() 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /storj/location/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package location contains geolocation related constants and variables. 5 | package location 6 | 7 | //go:generate go run ./gen/generate.go 8 | -------------------------------------------------------------------------------- /storj/location/region.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package location 5 | 6 | // EuCountries defines the 27 member country of European Union. 7 | var EuCountries = NewSet( 8 | Austria, 9 | Belgium, 10 | Bulgaria, 11 | Croatia, 12 | Cyprus, 13 | Czechia, 14 | Denmark, 15 | Estonia, 16 | Finland, 17 | France, 18 | Germany, 19 | Greece, 20 | Hungary, 21 | Ireland, 22 | Italy, 23 | Lithuania, 24 | Latvia, 25 | Luxembourg, 26 | Malta, 27 | TheNetherlands, 28 | Poland, 29 | Portugal, 30 | Romania, 31 | Slovenia, 32 | Slovakia, 33 | Spain, 34 | Sweden, 35 | ) 36 | 37 | // EeaCountries defined the EEA countries. 38 | var EeaCountries = EuCountries.With( 39 | Iceland, 40 | Liechtenstein, 41 | Norway, 42 | ) 43 | -------------------------------------------------------------------------------- /storj/location/set_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2023 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package location 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestSet(t *testing.T) { 13 | var set Set 14 | set.Include(Belgium) 15 | set.Include(Hungary) 16 | set.Include(TheNetherlands) 17 | set.Include(ToCountryCode("ZZ")) 18 | 19 | require.True(t, set.Contains(Belgium)) 20 | require.True(t, set.Contains(Hungary)) 21 | require.True(t, set.Contains(TheNetherlands)) 22 | require.True(t, set.Contains(ToCountryCode("ZZ"))) 23 | 24 | require.False(t, set.Contains(Estonia)) 25 | require.False(t, set.Contains(Austria)) 26 | 27 | require.Equal(t, 4, set.Count()) 28 | 29 | set.Remove(Hungary) 30 | // removing non-existent things should be fine 31 | set.Remove(Estonia) 32 | set.Remove(Austria) 33 | require.Equal(t, 3, set.Count()) 34 | } 35 | 36 | func TestSet_Full(t *testing.T) { 37 | var set Set 38 | for c := CountryCode(0); int(c) < len(CountryISOCode); c++ { 39 | set.Include(c) 40 | } 41 | require.Equal(t, len(CountryISOCode), set.Count()) 42 | 43 | for c := CountryCode(0); int(c) < len(CountryISOCode); c++ { 44 | set.Remove(c) 45 | } 46 | require.Equal(t, 0, set.Count()) 47 | } 48 | -------------------------------------------------------------------------------- /storj/noise.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2023 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package storj 5 | 6 | import ( 7 | "net/url" 8 | "strconv" 9 | 10 | "storj.io/common/base58" 11 | ) 12 | 13 | // NoiseProto represents different possible Noise handshake and cipher suite 14 | // selections. 15 | type NoiseProto int 16 | 17 | const ( 18 | // NoiseProto_Unset is an unset protocol. 19 | NoiseProto_Unset = 0 20 | // NoiseProto_IK_25519_ChaChaPoly_BLAKE2b is a Noise protocol. 21 | NoiseProto_IK_25519_ChaChaPoly_BLAKE2b = 1 22 | // NoiseProto_IK_25519_AESGCM_BLAKE2b is a Noise protocol. 23 | NoiseProto_IK_25519_AESGCM_BLAKE2b = 2 24 | ) 25 | 26 | // NoiseInfo represents the information needed to dial a remote Noise peer. 27 | type NoiseInfo struct { 28 | Proto NoiseProto 29 | PublicKey string // byte representation 30 | } 31 | 32 | // IsZero returns whether it contains any information. 33 | func (info *NoiseInfo) IsZero() bool { 34 | return info.Proto == NoiseProto_Unset && info.PublicKey == "" 35 | } 36 | 37 | // WriteTo assists in serializing a NoiseInfo to a NodeURL. 38 | func (info *NoiseInfo) WriteTo(values url.Values) { 39 | if info.Proto != NoiseProto_Unset { 40 | values.Set("noise_proto", strconv.Itoa(int(info.Proto))) 41 | } 42 | if info.PublicKey != "" { 43 | values.Set("noise_pub", base58.CheckEncode([]byte(info.PublicKey), 0)) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /storj/path.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package storj 5 | 6 | import ( 7 | "strings" 8 | ) 9 | 10 | // Path represents a object path. 11 | type Path = string 12 | 13 | // SplitPath splits path into a slice of path components. 14 | func SplitPath(path Path) []string { 15 | return strings.Split(path, "/") 16 | } 17 | 18 | // JoinPaths concatenates paths to a new single path. 19 | func JoinPaths(paths ...Path) Path { 20 | return strings.Join(paths, "/") 21 | } 22 | -------------------------------------------------------------------------------- /storj/path_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package storj 5 | 6 | import ( 7 | "fmt" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestSplitPath(t *testing.T) { 14 | for i, tt := range []struct { 15 | path string 16 | comps []string 17 | }{ 18 | {"", []string{""}}, 19 | {"/", []string{"", ""}}, 20 | {"//", []string{"", "", ""}}, 21 | {" ", []string{" "}}, 22 | {"a", []string{"a"}}, 23 | {"/a/", []string{"", "a", ""}}, 24 | {"a/b/c/d", []string{"a", "b", "c", "d"}}, 25 | {"///a//b////c/d///", []string{"", "", "", "a", "", "b", "", "", "", "c", "d", "", "", ""}}, 26 | } { 27 | errTag := fmt.Sprintf("Test case #%d", i) 28 | assert.Equal(t, tt.comps, SplitPath(tt.path), errTag) 29 | } 30 | } 31 | 32 | func TestJoinPaths(t *testing.T) { 33 | for i, tt := range []struct { 34 | comps []string 35 | path string 36 | }{ 37 | {[]string{}, ""}, 38 | {[]string{""}, ""}, 39 | {[]string{"", ""}, "/"}, 40 | {[]string{"/", ""}, "//"}, 41 | {[]string{"/", "/"}, "///"}, 42 | {[]string{"", "", ""}, "//"}, 43 | {[]string{" "}, " "}, 44 | {[]string{"a"}, "a"}, 45 | {[]string{"", "a", ""}, "/a/"}, 46 | {[]string{"a", "b", "c", "d"}, "a/b/c/d"}, 47 | {[]string{"a/b", "c/d"}, "a/b/c/d"}, 48 | {[]string{"a/b/", "c/d"}, "a/b//c/d"}, 49 | {[]string{"", "", "", "a", "", "b", "", "", "", "c", "d", "", "", ""}, "///a//b////c/d///"}, 50 | } { 51 | errTag := fmt.Sprintf("Test case #%d", i) 52 | assert.Equal(t, tt.path, JoinPaths(tt.comps...), errTag) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /storj/pieceid_deriver.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package storj 5 | 6 | import ( 7 | "encoding/binary" 8 | 9 | "storj.io/common/internal/hmacsha512" 10 | ) 11 | 12 | // PieceIDDeriver can be used to for multiple derivation from the same PieceID 13 | // without need to initialize mac for each Derive call. 14 | type PieceIDDeriver struct { 15 | mac hmacsha512.Partial 16 | } 17 | 18 | // Deriver creates piece ID dervier for multiple derive operations. 19 | func (id PieceID) Deriver() PieceIDDeriver { 20 | return PieceIDDeriver{ 21 | mac: hmacsha512.New(id[:]), 22 | } 23 | } 24 | 25 | // Derive a new PieceID from the piece ID, the given storage node ID and piece number. 26 | // Initial mac is created from piece ID once while creating PieceDeriver and just 27 | // reset to initial state at the beginning of each call. 28 | func (pd PieceIDDeriver) Derive(storagenodeID NodeID, pieceNum int32) PieceID { 29 | pd.mac.Write(storagenodeID[:]) // on hash.Hash write never returns an error 30 | var num [4]byte 31 | binary.BigEndian.PutUint32(num[:], uint32(pieceNum)) 32 | pd.mac.Write(num[:]) // on hash.Hash write never returns an error 33 | var derived PieceID 34 | sum := pd.mac.SumAndReset() 35 | copy(derived[:], sum[:]) 36 | return derived 37 | } 38 | -------------------------------------------------------------------------------- /storj/placement_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Storj Labs, Inc. 2 | // See LICENSE for copying information 3 | 4 | package storj 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestPlacement_SQLConversion(t *testing.T) { 13 | p := PlacementConstraint(2) 14 | value, err := p.Value() 15 | require.NoError(t, err) 16 | 17 | res := new(PlacementConstraint) 18 | err = res.Scan(value) 19 | require.NoError(t, err) 20 | require.Equal(t, PlacementConstraint(2), *res) 21 | 22 | err = res.Scan(nil) 23 | require.NoError(t, err) 24 | require.Equal(t, DefaultPlacement, *res) 25 | 26 | err = res.Scan("") 27 | require.Error(t, err) 28 | } 29 | -------------------------------------------------------------------------------- /storj/retention.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2024 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package storj 5 | 6 | // RetentionMode represents the retention mode of an object version. 7 | type RetentionMode uint8 8 | 9 | const ( 10 | // NoRetention signifies that a retention period has not been set on an object version. 11 | NoRetention RetentionMode = 0 12 | 13 | // ComplianceMode signifies that an object version is locked in compliance mode 14 | // and cannot be deleted or modified until the retention period expires. 15 | ComplianceMode RetentionMode = 1 16 | 17 | // GovernanceMode signifies that an object version is locked in governance mode 18 | // and cannot be deleted or modified until the retention period expires or the lock is removed. 19 | GovernanceMode RetentionMode = 2 20 | ) 21 | -------------------------------------------------------------------------------- /storj/serialnumber_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package storj_test 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | 11 | "storj.io/common/storj" 12 | "storj.io/common/testrand" 13 | ) 14 | 15 | func TestSerialNumber_Encode(t *testing.T) { 16 | _, err := storj.SerialNumberFromString("likn43kilfzd") 17 | assert.Error(t, err) 18 | 19 | _, err = storj.SerialNumberFromBytes([]byte{1, 2, 3, 4, 5}) 20 | assert.Error(t, err) 21 | 22 | for range 10 { 23 | serialNumber := testrand.SerialNumber() 24 | 25 | fromString, err := storj.SerialNumberFromString(serialNumber.String()) 26 | assert.NoError(t, err) 27 | fromBytes, err := storj.SerialNumberFromBytes(serialNumber.Bytes()) 28 | assert.NoError(t, err) 29 | 30 | assert.Equal(t, serialNumber, fromString) 31 | assert.Equal(t, serialNumber, fromBytes) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /storj/streamid_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package storj_test 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | 11 | "storj.io/common/storj" 12 | "storj.io/common/testrand" 13 | ) 14 | 15 | func TestStreamID_Encode(t *testing.T) { 16 | for range 10 { 17 | expectedSize := testrand.Intn(255) 18 | streamID := testrand.StreamID(expectedSize) 19 | 20 | fromString, err := storj.StreamIDFromString(streamID.String()) 21 | require.NoError(t, err) 22 | require.Equal(t, streamID.String(), fromString.String()) 23 | 24 | fromBytes, err := storj.StreamIDFromBytes(streamID.Bytes()) 25 | require.NoError(t, err) 26 | require.Equal(t, streamID.Bytes(), fromBytes.Bytes()) 27 | 28 | require.Equal(t, streamID, fromString) 29 | require.Equal(t, expectedSize, fromString.Size()) 30 | require.Equal(t, streamID, fromBytes) 31 | require.Equal(t, expectedSize, fromBytes.Size()) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /strictcsv/common.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package strictcsv 5 | 6 | import "github.com/zeebo/errs" 7 | 8 | var ( 9 | // Error is an error class for the package. 10 | Error = errs.Class("strictcsv") 11 | ) 12 | -------------------------------------------------------------------------------- /sync2/conc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Storj Labs, Inc. 2 | // See LICENSE for copying information 3 | 4 | package sync2 5 | 6 | import ( 7 | "sync" 8 | 9 | "storj.io/common/errs2" 10 | ) 11 | 12 | // Concurrently runs fns concurrently and returns the non-nil errors. 13 | func Concurrently(fns ...func() error) []error { 14 | var g errs2.Group 15 | for _, fn := range fns { 16 | g.Go(fn) 17 | } 18 | return g.Wait() 19 | } 20 | 21 | // Go runs fns concurrently and returns a func to wait for them to complete. 22 | // 23 | // See also Concurrently and errs2.Group. 24 | func Go(fns ...func()) (wait func()) { 25 | var wg sync.WaitGroup 26 | wg.Add(len(fns)) 27 | for _, fn := range fns { 28 | go func() { 29 | defer wg.Done() 30 | fn() 31 | }() 32 | } 33 | return wg.Wait 34 | } 35 | -------------------------------------------------------------------------------- /sync2/conc_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2023 Storj Labs, Inc. 2 | // See LICENSE for copying information 3 | 4 | package sync2_test 5 | 6 | import ( 7 | "sync/atomic" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/require" 11 | 12 | "storj.io/common/sync2" 13 | ) 14 | 15 | func TestGo(t *testing.T) { 16 | var a int32 17 | wait := sync2.Go( 18 | func() { atomic.AddInt32(&a, 1) }, 19 | func() { atomic.AddInt32(&a, 1) }, 20 | ) 21 | wait() 22 | require.Equal(t, int32(2), a) 23 | wait() 24 | require.Equal(t, int32(2), a) 25 | } 26 | -------------------------------------------------------------------------------- /sync2/copy.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information 3 | 4 | package sync2 5 | 6 | import ( 7 | "context" 8 | "io" 9 | 10 | "github.com/spacemonkeygo/monkit/v3" 11 | ) 12 | 13 | var mon = monkit.Package() 14 | 15 | type readerFunc func(p []byte) (n int, err error) 16 | 17 | func (rf readerFunc) Read(p []byte) (n int, err error) { return rf(p) } 18 | 19 | // Copy implements copying with cancellation. 20 | func Copy(ctx context.Context, dst io.Writer, src io.Reader) (written int64, err error) { 21 | defer mon.Task()(&ctx)(&err) 22 | written, err = io.Copy(dst, readerFunc(func(p []byte) (int, error) { 23 | select { 24 | case <-ctx.Done(): 25 | return 0, ctx.Err() 26 | default: 27 | return src.Read(p) 28 | } 29 | })) 30 | 31 | return written, err 32 | } 33 | -------------------------------------------------------------------------------- /sync2/copy_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information 3 | 4 | package sync2_test 5 | 6 | import ( 7 | "context" 8 | "io" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/assert" 12 | 13 | "storj.io/common/memory" 14 | "storj.io/common/sync2" 15 | "storj.io/common/testrand" 16 | ) 17 | 18 | func TestCopy(t *testing.T) { 19 | t.Parallel() 20 | 21 | ctx, cancel := context.WithCancel(context.Background()) 22 | defer cancel() 23 | 24 | r := io.LimitReader(testrand.Reader(), 32*memory.KiB.Int64()) 25 | 26 | n, err := sync2.Copy(ctx, io.Discard, r) 27 | 28 | assert.NoError(t, err) 29 | assert.Equal(t, n, 32*memory.KiB.Int64()) 30 | } 31 | 32 | func TestCopy_Cancel(t *testing.T) { 33 | t.Parallel() 34 | 35 | ctx, cancel := context.WithCancel(context.Background()) 36 | cancel() 37 | 38 | r := io.LimitReader(testrand.Reader(), 32*memory.KiB.Int64()) 39 | 40 | n, err := sync2.Copy(ctx, io.Discard, r) 41 | 42 | assert.EqualError(t, err, context.Canceled.Error()) 43 | assert.EqualValues(t, n, 0) 44 | } 45 | -------------------------------------------------------------------------------- /sync2/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information 3 | 4 | /* 5 | Package sync2 provides a set of functions and types for: 6 | 7 | - Having context aware functionalities which aren't present in the standard 8 | library. 9 | - For offloading memory through the file system. 10 | - To control execution of tasks which can run repetitively, concurrently or 11 | asynchronously. 12 | */ 13 | package sync2 14 | -------------------------------------------------------------------------------- /sync2/fence.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information 3 | 4 | package sync2 5 | 6 | import ( 7 | "context" 8 | "sync" 9 | ) 10 | 11 | // Fence allows to wait for something to happen. 12 | type Fence struct { 13 | noCopy noCopy //nolint:structcheck 14 | 15 | setup sync.Once 16 | release sync.Once 17 | done chan struct{} 18 | } 19 | 20 | // init sets up the initial lock into wait. 21 | func (fence *Fence) init() { 22 | fence.setup.Do(func() { 23 | fence.done = make(chan struct{}) 24 | }) 25 | } 26 | 27 | // Release releases everyone from Wait. 28 | func (fence *Fence) Release() { 29 | fence.init() 30 | fence.release.Do(func() { close(fence.done) }) 31 | } 32 | 33 | // Wait waits for wait to be unlocked. 34 | // Returns true when it was successfully released. 35 | func (fence *Fence) Wait(ctx context.Context) bool { 36 | fence.init() 37 | 38 | select { 39 | case <-fence.done: 40 | return true 41 | default: 42 | select { 43 | case <-ctx.Done(): 44 | return false 45 | case <-fence.done: 46 | return true 47 | } 48 | } 49 | } 50 | 51 | // Released returns whether the fence has been released. 52 | func (fence *Fence) Released() bool { 53 | fence.init() 54 | 55 | select { 56 | case <-fence.done: 57 | return true 58 | default: 59 | return false 60 | } 61 | } 62 | 63 | // Done returns channel that will be closed when the fence is released. 64 | func (fence *Fence) Done() chan struct{} { 65 | fence.init() 66 | return fence.done 67 | } 68 | -------------------------------------------------------------------------------- /sync2/limiter.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information 3 | 4 | package sync2 5 | 6 | import ( 7 | "context" 8 | "sync" 9 | ) 10 | 11 | // Limiter implements concurrent goroutine limiting. 12 | // 13 | // After calling Wait or Close, no new goroutines are allowed 14 | // to start. 15 | type Limiter struct { 16 | noCopy noCopy //nolint:structcheck 17 | 18 | limit chan struct{} 19 | close sync.Once 20 | closed chan struct{} 21 | } 22 | 23 | // NewLimiter creates a new limiter with limit set to n. 24 | func NewLimiter(n int) *Limiter { 25 | return &Limiter{ 26 | limit: make(chan struct{}, n), 27 | closed: make(chan struct{}), 28 | } 29 | } 30 | 31 | // Go tries to start fn as a goroutine. 32 | // When the limit is reached it will wait until it can run it 33 | // or the context is canceled. 34 | func (limiter *Limiter) Go(ctx context.Context, fn func()) bool { 35 | if ctx.Err() != nil { 36 | return false 37 | } 38 | 39 | select { 40 | case limiter.limit <- struct{}{}: 41 | case <-limiter.closed: 42 | return false 43 | case <-ctx.Done(): 44 | return false 45 | } 46 | 47 | go func() { 48 | defer func() { <-limiter.limit }() 49 | fn() 50 | }() 51 | 52 | return true 53 | } 54 | 55 | // Wait waits for all running goroutines to finish and 56 | // disallows new goroutines to start. 57 | func (limiter *Limiter) Wait() { limiter.Close() } 58 | 59 | // Close waits for all running goroutines to finish and 60 | // disallows new goroutines to start. 61 | func (limiter *Limiter) Close() { 62 | limiter.close.Do(func() { 63 | close(limiter.closed) 64 | // ensure all goroutines are finished 65 | for i := 0; i < cap(limiter.limit); i++ { 66 | limiter.limit <- struct{}{} 67 | } 68 | }) 69 | } 70 | -------------------------------------------------------------------------------- /sync2/limiter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information 3 | 4 | package sync2_test 5 | 6 | import ( 7 | "context" 8 | "sync/atomic" 9 | "testing" 10 | "time" 11 | 12 | "storj.io/common/sync2" 13 | ) 14 | 15 | func TestLimiterLimiting(t *testing.T) { 16 | t.Parallel() 17 | 18 | const N, Limit = 1000, 10 19 | ctx := context.Background() 20 | limiter := sync2.NewLimiter(Limit) 21 | counter := int32(0) 22 | for range N { 23 | limiter.Go(ctx, func() { 24 | if atomic.AddInt32(&counter, 1) > Limit { 25 | panic("limit exceeded") 26 | } 27 | time.Sleep(time.Millisecond) 28 | atomic.AddInt32(&counter, -1) 29 | }) 30 | } 31 | limiter.Close() 32 | } 33 | 34 | func TestLimiterCanceling(t *testing.T) { 35 | t.Parallel() 36 | 37 | const N, Limit = 1000, 10 38 | limiter := sync2.NewLimiter(Limit) 39 | 40 | ctx, cancel := context.WithCancel(context.Background()) 41 | 42 | counter := int32(0) 43 | 44 | waitForCancel := make(chan struct{}, N) 45 | block := make(chan struct{}) 46 | allreturned := make(chan struct{}) 47 | 48 | go func() { 49 | for range N { 50 | limiter.Go(ctx, func() { 51 | if atomic.AddInt32(&counter, 1) > Limit { 52 | panic("limit exceeded") 53 | } 54 | 55 | waitForCancel <- struct{}{} 56 | <-block 57 | }) 58 | } 59 | close(allreturned) 60 | }() 61 | 62 | for range Limit { 63 | <-waitForCancel 64 | } 65 | cancel() 66 | <-allreturned 67 | close(block) 68 | 69 | limiter.Close() 70 | if counter > Limit { 71 | t.Fatal("too many times run") 72 | } 73 | 74 | started := limiter.Go(context.Background(), func() { 75 | panic("should not start") 76 | }) 77 | if started { 78 | t.Fatal("should not start") 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /sync2/mpscqueue/LICENSE: -------------------------------------------------------------------------------- 1 | Design based on code in http://www.1024cores.net/home/lock-free-algorithms/queues/non-intrusive-mpsc-node-based-queue, which was licensed under Simplified BSD License and by Apache License, Version 2.0. 2 | 3 | --- 4 | 5 | Copyright (c) 2010-2011 Dmitry Vyukov. All rights reserved. 6 | 7 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this list of 10 | conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, this list 13 | of conditions and the following disclaimer in the documentation and/or other materials 14 | provided with the distribution. 15 | 16 | THIS SOFTWARE IS PROVIDED BY DMITRY VYUKOV "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DMITRY VYUKOV OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 17 | 18 | The views and conclusions contained in the software and documentation are those of the authors and should not be interpreted as representing official policies, either expressed or implied, of Dmitry Vyukov. -------------------------------------------------------------------------------- /sync2/nocopy.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information 3 | 4 | package sync2 5 | 6 | // noCopy is used to ensure that we don't copy things that shouldn't 7 | // be copied. 8 | // 9 | // See https://golang.org/issues/8005#issuecomment-190753527. 10 | // 11 | // Currently users of noCopy must use "//nolint:structcheck", 12 | // because golint-ci does not handle this correctly. 13 | type noCopy struct{} 14 | 15 | func (noCopy) Lock() {} 16 | -------------------------------------------------------------------------------- /sync2/race2/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2023 Storj Labs, Inc. 2 | // See LICENSE for copying information 3 | 4 | // Package race2 exposes race detector API such that some assembly code can be 5 | // manually instrumented for race detector. 6 | package race2 7 | -------------------------------------------------------------------------------- /sync2/race2/race.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2023 Storj Labs, Inc. 2 | // See LICENSE for copying information 3 | 4 | //go:build race 5 | 6 | package race2 7 | 8 | import ( 9 | "runtime" 10 | "unsafe" 11 | ) 12 | 13 | // Read marks addr as being read for the race detector. 14 | func Read(addr unsafe.Pointer) { 15 | runtime.RaceRead(addr) 16 | } 17 | 18 | // Write marks addr as being written to for the race detector. 19 | func Write(addr unsafe.Pointer) { 20 | runtime.RaceWrite(addr) 21 | } 22 | 23 | // ReadRange marks [addr:addr+len] as being read for the race detector. 24 | func ReadRange(addr unsafe.Pointer, len int) { 25 | runtime.RaceReadRange(addr, len) 26 | } 27 | 28 | // WriteRange marks [addr:addr+len] as being written to for the race detector. 29 | func WriteRange(addr unsafe.Pointer, len int) { 30 | runtime.RaceWriteRange(addr, len) 31 | } 32 | 33 | // ReadSlice marks data slice as being read for the race detector. 34 | func ReadSlice[T any](data []T) { 35 | if len(data) == 0 { 36 | return 37 | } 38 | runtime.RaceReadRange(unsafe.Pointer(&data[0]), len(data)*int(unsafe.Sizeof(data[0]))) 39 | } 40 | 41 | // WriteSlice marks [addr:addr+len] as being written to for the race detector. 42 | func WriteSlice[T any](data []T) { 43 | if len(data) == 0 { 44 | return 45 | } 46 | runtime.RaceWriteRange(unsafe.Pointer(&data[0]), len(data)*int(unsafe.Sizeof(data[0]))) 47 | } 48 | -------------------------------------------------------------------------------- /sync2/race2/race0.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2023 Storj Labs, Inc. 2 | // See LICENSE for copying information 3 | 4 | //go:build !race 5 | 6 | package race2 7 | 8 | import "unsafe" 9 | 10 | // Read marks addr as being read for the race detector. 11 | func Read(addr unsafe.Pointer) {} 12 | 13 | // Write marks addr as being written to for the race detector. 14 | func Write(addr unsafe.Pointer) {} 15 | 16 | // ReadRange marks [addr:addr+len] as being read for the race detector. 17 | func ReadRange(addr unsafe.Pointer, len int) {} 18 | 19 | // WriteRange marks [addr:addr+len] as being written to for the race detector. 20 | func WriteRange(addr unsafe.Pointer, len int) {} 21 | 22 | // ReadSlice marks data slice as being read for the race detector. 23 | func ReadSlice[T any](data []T) {} 24 | 25 | // WriteSlice marks [addr:addr+len] as being written to for the race detector. 26 | func WriteSlice[T any](data []T) {} 27 | -------------------------------------------------------------------------------- /sync2/semaphore.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information 3 | 4 | package sync2 5 | 6 | import ( 7 | "context" 8 | 9 | "golang.org/x/sync/semaphore" 10 | ) 11 | 12 | // Semaphore implements a closable semaphore. 13 | type Semaphore struct { 14 | noCopy noCopy //nolint:structcheck 15 | 16 | ctx context.Context 17 | close func() 18 | sema *semaphore.Weighted 19 | } 20 | 21 | // NewSemaphore creates a semaphore with the specified size. 22 | func NewSemaphore(size int) *Semaphore { 23 | sema := &Semaphore{} 24 | sema.Init(size) 25 | return sema 26 | } 27 | 28 | // Init initializes semaphore to the specified size. 29 | func (sema *Semaphore) Init(size int) { 30 | sema.ctx, sema.close = context.WithCancel(context.Background()) 31 | sema.sema = semaphore.NewWeighted(int64(size)) 32 | } 33 | 34 | // Close closes the semaphore from further use. 35 | func (sema *Semaphore) Close() { 36 | sema.close() 37 | } 38 | 39 | // Lock locks the semaphore. 40 | func (sema *Semaphore) Lock() bool { 41 | return sema.sema.Acquire(sema.ctx, 1) == nil 42 | } 43 | 44 | // Unlock unlocks the semaphore. 45 | func (sema *Semaphore) Unlock() { 46 | sema.sema.Release(1) 47 | } 48 | -------------------------------------------------------------------------------- /sync2/sleep.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information 3 | 4 | package sync2 5 | 6 | import ( 7 | "context" 8 | "time" 9 | 10 | "storj.io/common/time2" 11 | ) 12 | 13 | // Sleep implements sleeping with cancellation. 14 | func Sleep(ctx context.Context, duration time.Duration) bool { 15 | return time2.Sleep(ctx, duration) 16 | } 17 | -------------------------------------------------------------------------------- /sync2/sleep_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information 3 | 4 | package sync2_test 5 | 6 | import ( 7 | "context" 8 | "testing" 9 | "time" 10 | 11 | "storj.io/common/sync2" 12 | "storj.io/common/time2" 13 | ) 14 | 15 | func TestSleep(t *testing.T) { 16 | t.Parallel() 17 | 18 | t.Run("against the real clock", func(t *testing.T) { 19 | const sleepError = time.Second / 2 // should be larger than most system error with regards to sleep 20 | 21 | ctx, cancel := context.WithCancel(context.Background()) 22 | defer cancel() 23 | 24 | start := time.Now() 25 | if !sync2.Sleep(ctx, time.Second) { 26 | t.Error("expected true as result") 27 | } 28 | if time.Since(start) < time.Second-sleepError { 29 | t.Error("sleep took too little time") 30 | } 31 | }) 32 | 33 | t.Run("against a fake clock", func(t *testing.T) { 34 | ctx, timeMachine := time2.WithNewMachine(context.Background()) 35 | 36 | defer sync2.Go(func() { timeMachine.BlockThenAdvance(ctx, 1, time.Second) })() 37 | 38 | start := timeMachine.Now() 39 | if !sync2.Sleep(ctx, time.Second) { 40 | t.Error("expected true as result") 41 | } 42 | if timeMachine.Since(start) != time.Second { 43 | t.Error("sleep took too little time") 44 | } 45 | }) 46 | } 47 | 48 | func TestSleep_Cancel(t *testing.T) { 49 | t.Parallel() 50 | 51 | ctx, cancel := context.WithCancel(context.Background()) 52 | cancel() 53 | 54 | start := time.Now() 55 | if sync2.Sleep(ctx, 5*time.Second) { 56 | t.Error("expected false as result") 57 | } 58 | if time.Since(start) > time.Second { 59 | t.Error("sleep took too long") 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /sync2/timeout.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package sync2 5 | 6 | import ( 7 | "time" 8 | ) 9 | 10 | // WithTimeout calls `do` and when the timeout is reached and `do` 11 | // has not finished, it'll call `onTimeout` concurrently. 12 | // 13 | // When WithTimeout returns it's guaranteed to not call onTimeout. 14 | func WithTimeout(timeout time.Duration, do, onTimeout func()) { 15 | c := make(chan struct{}) 16 | t := time.AfterFunc(timeout, func() { 17 | defer close(c) 18 | onTimeout() 19 | }) 20 | defer func() { 21 | if !t.Stop() { 22 | <-c 23 | } 24 | }() 25 | do() 26 | } 27 | -------------------------------------------------------------------------------- /sync2/workgroup_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information 3 | 4 | package sync2_test 5 | 6 | import ( 7 | "testing" 8 | "time" 9 | 10 | "github.com/stretchr/testify/require" 11 | 12 | "storj.io/common/sync2" 13 | ) 14 | 15 | func TestWaitGroup(t *testing.T) { 16 | t.Parallel() 17 | 18 | const Wait = 2 * time.Second 19 | const TimeError = time.Second / 2 20 | 21 | var group sync2.WorkGroup 22 | 23 | require.True(t, group.Start()) 24 | go func() { 25 | defer group.Done() 26 | time.Sleep(Wait) 27 | }() 28 | 29 | require.True(t, group.Go(func() { 30 | time.Sleep(Wait) 31 | })) 32 | 33 | start := time.Now() 34 | group.Wait() 35 | duration := time.Since(start) 36 | 37 | if duration < Wait-TimeError || duration > Wait+TimeError { 38 | t.Fatalf("waited %s instead of %s", duration, Wait) 39 | } 40 | } 41 | 42 | func TestWaitGroupClose(t *testing.T) { 43 | t.Parallel() 44 | 45 | const Wait = 2 * time.Second 46 | const LongWait = 10 * time.Second 47 | const TimeError = time.Second / 2 48 | 49 | var group sync2.WorkGroup 50 | 51 | require.True(t, group.Go(func() { 52 | time.Sleep(Wait) 53 | })) 54 | 55 | group.Close() 56 | 57 | require.False(t, group.Go(func() { 58 | time.Sleep(LongWait) 59 | })) 60 | 61 | start := time.Now() 62 | group.Wait() 63 | duration := time.Since(start) 64 | 65 | if duration < Wait-TimeError || duration > LongWait-TimeError { 66 | t.Fatalf("waited %s instead of %s", duration, Wait) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /telemetry/common.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package telemetry 5 | 6 | import ( 7 | "github.com/zeebo/errs" 8 | ) 9 | 10 | // Error is the default telemetry errs class. 11 | var Error = errs.Class("telemetry") 12 | -------------------------------------------------------------------------------- /telemetry/example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package telemetry_test 5 | 6 | import ( 7 | "context" 8 | "errors" 9 | "fmt" 10 | "log" 11 | "time" 12 | 13 | "github.com/spacemonkeygo/monkit/v3" 14 | "github.com/zeebo/admission/v3/admproto" 15 | "github.com/zeebo/errs" 16 | "golang.org/x/sync/errgroup" 17 | 18 | "storj.io/common/telemetry" 19 | ) 20 | 21 | // Example is an example of a receiver and sender. 22 | func Example() { 23 | ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 24 | defer cancel() 25 | 26 | var group errgroup.Group 27 | 28 | receiver, err := telemetry.Listen("127.0.0.1:0") 29 | if err != nil { 30 | log.Println(err) 31 | return 32 | } 33 | 34 | // receiver 35 | group.Go(func() (err error) { 36 | defer func() { err = errs.Combine(err, receiver.Close()) }() 37 | err = receiver.Serve(ctx, telemetry.HandlerFunc( 38 | func(application, instance string, key []byte, val float64) { 39 | fmt.Printf("receive %s %s %s %v\n", application, instance, string(key), val) 40 | }, 41 | )) 42 | if errors.Is(err, context.Canceled) { 43 | err = nil 44 | } 45 | return err 46 | }) 47 | 48 | // sender 49 | group.Go(func() error { 50 | client, err := telemetry.NewClient(receiver.Addr(), telemetry.ClientOpts{ 51 | Interval: time.Second, 52 | Application: "example", 53 | Instance: telemetry.DefaultInstanceID(), 54 | Registry: monkit.Default, 55 | FloatEncoding: admproto.Float32Encoding, 56 | }) 57 | if err != nil { 58 | return err 59 | } 60 | 61 | client.Run(ctx) 62 | return nil 63 | }) 64 | 65 | if err := group.Wait(); err != nil { 66 | fmt.Println(err) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /telemetry/reporter.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package telemetry 5 | 6 | import ( 7 | "context" 8 | "sync" 9 | "time" 10 | 11 | "storj.io/common/sync2" 12 | ) 13 | 14 | // Reporter calls a function to report metrics periodically. 15 | type Reporter struct { 16 | interval time.Duration 17 | mu sync.Mutex 18 | cancel context.CancelFunc 19 | stopped bool 20 | 21 | send func(ctx context.Context) error 22 | } 23 | 24 | // NewReporter creates a reporter which calls send function once in each interval. 25 | func NewReporter(interval time.Duration, send func(ctx context.Context) error) (rv *Reporter, err error) { 26 | return &Reporter{ 27 | interval: interval, 28 | send: send, 29 | }, nil 30 | } 31 | 32 | // Run calls Report roughly every Interval. 33 | func (c *Reporter) Run(ctx context.Context) { 34 | c.mu.Lock() 35 | if c.stopped { 36 | c.mu.Unlock() 37 | return 38 | } 39 | ctx, c.cancel = context.WithCancel(ctx) 40 | c.mu.Unlock() 41 | 42 | for { 43 | sync2.Sleep(ctx, jitter(c.interval)) 44 | if ctx.Err() != nil { 45 | return 46 | } 47 | 48 | _ = c.Publish(ctx) 49 | } 50 | } 51 | 52 | // Publish bundles up all the current stats and writes them out as UDP packets. 53 | func (c *Reporter) Publish(ctx context.Context) (err error) { 54 | defer mon.Task()(&ctx)(&err) 55 | return c.send(ctx) 56 | } 57 | -------------------------------------------------------------------------------- /telemetry/server_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package telemetry 5 | 6 | import ( 7 | "context" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestListen_NilOnBadAddress(t *testing.T) { 14 | server, errListen := Listen("11") 15 | defer func() { 16 | if server != nil { 17 | assert.NoError(t, server.Close()) 18 | } 19 | }() 20 | 21 | assert.Nil(t, server) 22 | assert.Error(t, errListen) 23 | } 24 | 25 | func TestServe_ReturnErrorOnConnFail(t *testing.T) { 26 | server, _ := Listen("127.0.0.1:0") 27 | defer func() { 28 | if server != nil && server.conn != nil { 29 | assert.NoError(t, server.Close()) 30 | } 31 | }() 32 | 33 | assert.NoError(t, server.conn.Close()) 34 | server.conn = nil 35 | 36 | errServe := server.Serve(context.Background(), nil) 37 | 38 | assert.EqualError(t, errServe, "telemetry: invalid conn: ") 39 | } 40 | 41 | func TestListenAndServe_ReturnErrorOnListenFails(t *testing.T) { 42 | err := ListenAndServe(context.Background(), "1", nil) 43 | assert.Error(t, err) 44 | } 45 | -------------------------------------------------------------------------------- /telemetry/utils.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package telemetry 5 | 6 | import ( 7 | "math/rand" 8 | "net" 9 | "time" 10 | ) 11 | 12 | // UnknownInstanceID is returned when no instance ID can be returned. 13 | const UnknownInstanceID = "unknown" 14 | 15 | // DefaultInstanceID will return the first non-nil mac address if possible, unknown otherwise. 16 | func DefaultInstanceID() string { 17 | ifaces, err := net.Interfaces() 18 | if err != nil { 19 | return UnknownInstanceID 20 | } 21 | for _, iface := range ifaces { 22 | if iface.HardwareAddr != nil { 23 | return iface.HardwareAddr.String() 24 | } 25 | } 26 | return UnknownInstanceID 27 | } 28 | 29 | func jitter(t time.Duration) time.Duration { 30 | nanos := rand.NormFloat64()*float64(t/4) + float64(t) 31 | if nanos <= 0 { 32 | nanos = 1 33 | } 34 | return time.Duration(nanos) 35 | } 36 | -------------------------------------------------------------------------------- /telemetry/utils_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package telemetry 5 | 6 | import ( 7 | "testing" 8 | "time" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestJitter_NegativeDuration(t *testing.T) { 14 | duration := time.Duration(-1) 15 | expected := time.Duration(1) 16 | 17 | actual := jitter(duration) 18 | 19 | assert.Equal(t, expected, actual) 20 | } 21 | -------------------------------------------------------------------------------- /testcontext/compile_norace.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | //go:build !race 5 | 6 | package testcontext 7 | 8 | const raceEnabled = false 9 | -------------------------------------------------------------------------------- /testcontext/compile_race.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | //go:build race 5 | 6 | package testcontext 7 | 8 | const raceEnabled = true 9 | -------------------------------------------------------------------------------- /testcontext/compile_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package testcontext_test 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | 11 | "storj.io/common/testcontext" 12 | ) 13 | 14 | func TestCompile(t *testing.T) { 15 | ctx := testcontext.New(t) 16 | 17 | exe := ctx.Compile("storj.io/common/testcontext/testdata/hello") 18 | assert.NotEmpty(t, exe) 19 | 20 | exemod := ctx.CompileAt("./testdata/hellomod", "test/hello") 21 | assert.NotEmpty(t, exemod) 22 | } 23 | -------------------------------------------------------------------------------- /testcontext/testdata/hello/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package main 5 | 6 | func main() { println("hello") } 7 | -------------------------------------------------------------------------------- /testcontext/testdata/hellomod/go.mod: -------------------------------------------------------------------------------- 1 | module test/hello 2 | 3 | go 1.16 4 | -------------------------------------------------------------------------------- /testcontext/testdata/hellomod/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package main 5 | 6 | func main() { println("hello") } 7 | -------------------------------------------------------------------------------- /testrand/rand_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package testrand_test 5 | 6 | import ( 7 | "regexp" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/require" 11 | 12 | "storj.io/common/testrand" 13 | ) 14 | 15 | var stringLength = 50 16 | 17 | func TestNoPanic(t *testing.T) { 18 | t.Log("URLPath", testrand.URLPath()) 19 | t.Log("URLPathNonFolder", testrand.URLPathNonFolder()) 20 | t.Log("Path", testrand.Path()) 21 | isNumeric := regexp.MustCompile(`^[0-9]+$`).MatchString 22 | randomNumericString := string(testrand.RandNumeric(stringLength)) 23 | t.Log("Numeric String", randomNumericString) 24 | require.True(t, isNumeric(randomNumericString)) 25 | require.Equal(t, stringLength, len(randomNumericString)) 26 | isAlphaNumeric := regexp.MustCompile(`^[A-Za-z0-9]+$`).MatchString 27 | randomAlphaNumericString := string(testrand.RandAlphaNumeric(stringLength)) 28 | t.Log("AlphaNumeric String", randomAlphaNumericString) 29 | require.True(t, isAlphaNumeric(randomAlphaNumericString)) 30 | require.Equal(t, stringLength, len(randomAlphaNumericString)) 31 | } 32 | -------------------------------------------------------------------------------- /testtrace/trace_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package testtrace_test 5 | 6 | import ( 7 | "context" 8 | "runtime/pprof" 9 | "testing" 10 | "time" 11 | 12 | "github.com/stretchr/testify/require" 13 | 14 | "storj.io/common/testtrace" 15 | ) 16 | 17 | func TestLabels(t *testing.T) { 18 | const label = "LABEL IS VISIBLE" 19 | 20 | ctx, cancel := context.WithCancel(context.Background()) 21 | t.Cleanup(cancel) 22 | 23 | pprof.Do(ctx, pprof.Labels("name", label), func(c context.Context) { 24 | for range 3 { 25 | go func() { <-c.Done() }() 26 | } 27 | }) 28 | 29 | time.Sleep(time.Millisecond) 30 | 31 | trace, err := testtrace.Summary() 32 | require.NoError(t, err) 33 | t.Log("\n" + trace) 34 | 35 | require.Contains(t, trace, label) 36 | } 37 | 38 | func TestFilter(t *testing.T) { 39 | ctx, cancel := context.WithCancel(context.Background()) 40 | t.Cleanup(cancel) 41 | 42 | pprof.Do(ctx, pprof.Labels("name", "alpha", "user", "ALPHA"), func(c context.Context) { 43 | go func() { <-c.Done() }() 44 | }) 45 | pprof.Do(ctx, pprof.Labels("name", "beta", "user", "BETA"), func(c context.Context) { 46 | go func() { <-c.Done() }() 47 | }) 48 | 49 | time.Sleep(time.Millisecond) 50 | 51 | trace, err := testtrace.Summary("name", "alpha") 52 | require.NoError(t, err) 53 | t.Log("\n" + trace) 54 | 55 | require.Contains(t, trace, "ALPHA") 56 | require.NotContains(t, trace, "alpha") 57 | require.NotContains(t, trace, "BETA") 58 | } 59 | -------------------------------------------------------------------------------- /time2/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information 3 | 4 | // Package time2 provides time related functionality that can be manipulated 5 | // to facilite deterministic testing. 6 | package time2 7 | -------------------------------------------------------------------------------- /time2/machine_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information 3 | 4 | package time2_test 5 | 6 | import ( 7 | "context" 8 | "testing" 9 | "time" 10 | 11 | "github.com/stretchr/testify/require" 12 | 13 | "storj.io/common/time2" 14 | ) 15 | 16 | func TestMachineWithTimeAt(t *testing.T) { 17 | expected := time.Now().Add(time.Hour) 18 | tm := time2.NewMachine(time2.WithTimeAt(expected)) 19 | require.Equal(t, expected, tm.Now()) 20 | } 21 | 22 | func TestMachineBlockReturnsTrueWhenBlocked(t *testing.T) { 23 | tm := time2.NewMachine() 24 | tm.Clock().NewTimer(time.Second) 25 | require.True(t, tm.Block(context.Background(), 1)) 26 | } 27 | 28 | func TestMachineBlockRespondsToContextCancellation(t *testing.T) { 29 | ctx, cancel := context.WithCancel(context.Background()) 30 | cancel() 31 | 32 | tm := time2.NewMachine() 33 | require.False(t, tm.Block(ctx, 1)) 34 | } 35 | -------------------------------------------------------------------------------- /traces/traces.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package traces 5 | 6 | import ( 7 | "context" 8 | "log" 9 | "os" 10 | "path/filepath" 11 | "strconv" 12 | 13 | tracetagger "github.com/jtolds/tracetagger/v2" 14 | monkit "github.com/spacemonkeygo/monkit/v3" 15 | "github.com/spacemonkeygo/monkit/v3/collect" 16 | ) 17 | 18 | var ( 19 | // TagDB tags a trace as hitting the database. 20 | TagDB = tracetagger.NewTagRef() 21 | ) 22 | 23 | // CollectTraces starts storing all known tagged traces on disk, 24 | // until cancel is called. 25 | func CollectTraces() (cancel func()) { 26 | path := filepath.Join(os.TempDir(), "storj-traces", strconv.Itoa(os.Getpid())) 27 | disable := TagDB.Enable() 28 | cancelCollect := tracetagger.TracesWithTag(TagDB, 10000, func(spans []*collect.FinishedSpan, capped bool) { 29 | name := tracetagger.TracePathPrefix(spans, capped) 30 | err := tracetagger.SaveTrace(spans, capped, filepath.Join(path, "unfiltered", name)) 31 | if err != nil { 32 | log.Print(err) 33 | } 34 | spans = tracetagger.JustTaggedSpans(spans, TagDB) 35 | err = tracetagger.SaveTrace(spans, capped, filepath.Join(path, "filtered", name)) 36 | if err != nil { 37 | log.Print(err) 38 | } 39 | }) 40 | return func() { 41 | cancelCollect() 42 | disable() 43 | } 44 | } 45 | 46 | // Tag tags a trace with the given tag. 47 | func Tag(ctx context.Context, tag *tracetagger.TagRef) { 48 | tracetagger.Tag(ctx, tag) 49 | } 50 | 51 | // TagScope tags all functions on an entire monkit Scope with a given tag. 52 | func TagScope(tag *tracetagger.TagRef, scope *monkit.Scope) { 53 | tracetagger.TagScope(tag, scope) 54 | } 55 | -------------------------------------------------------------------------------- /tracing/excluded.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package tracing 5 | 6 | import ( 7 | "context" 8 | 9 | "github.com/spacemonkeygo/monkit/v3" 10 | ) 11 | 12 | type contextKey int 13 | 14 | var ( 15 | excludeFromTracing contextKey = 1 16 | ) 17 | 18 | // WithoutDistributedTracing disables distributed tracing for the current span. 19 | func WithoutDistributedTracing(ctx context.Context) context.Context { 20 | return context.WithValue(ctx, excludeFromTracing, true) 21 | } 22 | 23 | // IsExcluded check if span shouldn't be reported to remote location. 24 | func IsExcluded(span *monkit.Span) bool { 25 | val, ok := span.Value(excludeFromTracing).(bool) 26 | return ok && val 27 | } 28 | -------------------------------------------------------------------------------- /useragent/encode.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package useragent 5 | 6 | import ( 7 | "fmt" 8 | "strings" 9 | ) 10 | 11 | // EncodeEntries encodes all entries. 12 | func EncodeEntries(entries []Entry) ([]byte, error) { 13 | parts := make([]string, len(entries)) 14 | for i, entry := range entries { 15 | if entry.Product != "" { 16 | ok := isToken([]byte(entry.Product)) 17 | if !ok { 18 | return nil, fmt.Errorf("product token is not valid: %s", entry.Product) 19 | } 20 | 21 | parts[i] = entry.Product 22 | if entry.Version != "" { 23 | ok := isToken([]byte(entry.Version)) 24 | if !ok { 25 | return nil, fmt.Errorf("version token is not valid: %s", entry.Version) 26 | } 27 | 28 | parts[i] += "/" + entry.Version 29 | } 30 | } 31 | if entry.Comment != "" { 32 | if entry.Product != "" { 33 | parts[i] += " " 34 | } 35 | parts[i] += "(" + entry.Comment + ")" 36 | } 37 | } 38 | return []byte(strings.Join(parts, " ")), nil 39 | } 40 | 41 | func isToken(data []byte) bool { 42 | // token is a sequence of token characters. 43 | // See tchar for the allowed characters. 44 | for i := range data { 45 | if !istchar(data[i]) { 46 | return false 47 | } 48 | } 49 | return true 50 | } 51 | -------------------------------------------------------------------------------- /useragent/fuzz_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | //go:build go1.18 5 | 6 | package useragent_test 7 | 8 | import ( 9 | "testing" 10 | 11 | "storj.io/common/useragent" 12 | ) 13 | 14 | func FuzzParseEntries(f *testing.F) { 15 | f.Add([]byte("")) 16 | f.Add([]byte("storj.io-common/v0.0.0-00010101000000-000000000000")) 17 | f.Add([]byte("storj.io-common/v0.0.0-00010101000000")) 18 | f.Add([]byte("storj.io-common/v9.0.0")) 19 | f.Add([]byte("Mozilla")) 20 | f.Add([]byte("Mozilla/5.0")) 21 | f.Add([]byte("Mozilla/5.0 (Linux; U; Android 4.4.3;)")) 22 | f.Add([]byte("storj.io-uplink/v0.0.1 storj.io-drpc/v5.0.0+123+123 Mozilla/5.0 (Linux; U; Android 4.4.3;) AppleWebkit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30 Opera News/1.0")) 23 | f.Add([]byte("!#$%&'*+-.^_`|~/!#$%&'*+-.^_`|~")) 24 | 25 | f.Fuzz(func(t *testing.T, data []byte) { 26 | _, _ = useragent.ParseEntries(data) 27 | }) 28 | } 29 | -------------------------------------------------------------------------------- /useragent/info.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | // Package useragent implements parts of 5 | // https://tools.ietf.org/html/rfc7231#section-5.5 and 6 | // https://tools.ietf.org/html/rfc2616#section-14.43 7 | package useragent 8 | 9 | import ( 10 | "strings" 11 | 12 | "github.com/zeebo/errs" 13 | ) 14 | 15 | // Error is the default error class. 16 | var Error = errs.Class("useragent") 17 | 18 | // Info contains parsed user agent. 19 | type Info struct { 20 | Product Product 21 | 22 | Full string 23 | } 24 | 25 | // Product is an user agent product. 26 | type Product struct { 27 | Name string 28 | Version string 29 | } 30 | 31 | // Parse parses user agent string to information. 32 | func Parse(s string) (Info, error) { 33 | if s == "" { 34 | return Info{}, nil 35 | } 36 | s = strings.TrimSpace(s) 37 | 38 | info := Info{} 39 | info.Full = s 40 | 41 | parts := strings.SplitN(s, " ", 2) 42 | productTokens := strings.SplitN(parts[0], "/", 2) 43 | 44 | if len(productTokens) >= 1 { 45 | info.Product.Name = productTokens[0] 46 | } 47 | if len(productTokens) >= 2 { 48 | info.Product.Version = productTokens[1] 49 | } 50 | 51 | return info, nil 52 | } 53 | -------------------------------------------------------------------------------- /useragent/info_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package useragent_test 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | 11 | "storj.io/common/useragent" 12 | ) 13 | 14 | func TestUserAgent(t *testing.T) { 15 | type testcase struct { 16 | in string 17 | info useragent.Info 18 | } 19 | 20 | var tests = []testcase{ 21 | {"Hello", useragent.Info{useragent.Product{"Hello", ""}, "Hello"}}, 22 | {"Hello/1.0", useragent.Info{useragent.Product{"Hello", "1.0"}, "Hello/1.0"}}, 23 | {"Hello/1.0+version#123", useragent.Info{useragent.Product{"Hello", "1.0+version#123"}, "Hello/1.0+version#123"}}, 24 | } 25 | 26 | for _, test := range tests { 27 | info, err := useragent.Parse(test.in) 28 | require.NoError(t, err) 29 | require.Equal(t, test.info, info) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /useragent/parse_internal_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2020 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package useragent 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestParseToken(t *testing.T) { 13 | type test struct { 14 | in string 15 | token string 16 | next int 17 | ok bool 18 | } 19 | 20 | tests := []test{ 21 | {``, ``, 0, false}, 22 | {` `, ``, 0, false}, 23 | {`(`, ``, 0, false}, 24 | {`)`, ``, 0, false}, 25 | {`a`, `a`, 1, true}, 26 | {`a `, `a`, 1, true}, 27 | {`a b`, `a`, 1, true}, 28 | {`a/x b`, `a`, 1, true}, 29 | } 30 | 31 | for _, test := range tests { 32 | token, next, ok := parseToken([]byte(test.in), 0) 33 | assert.Equal(t, test.token, token, test.in) 34 | assert.Equal(t, test.next, next, test.in) 35 | assert.Equal(t, test.ok, ok, test.in) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /uuid/fuzz_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | //go:build go1.18 5 | 6 | package uuid_test 7 | 8 | import ( 9 | "testing" 10 | 11 | "storj.io/common/uuid" 12 | ) 13 | 14 | func FuzzFromString(f *testing.F) { 15 | f.Add("") 16 | f.Add("6b\xff\xff\xff\u007f10-9dad-11d1-80b4-0c04fd430c8") 17 | f.Add("6ba7b810-$dad-11d1-80b4-0c04fd430c8") 18 | f.Add("6ba7b810-9dad-1 d1-.0b4-0c04fd430cA") 19 | f.Add("6ba7b810-9dad-11d1-80b4-00c04fd430c8") 20 | f.Add("6ba7b810-9dad-11d1-80b4-00c04fd430c8") 21 | f.Add("6ba7b810-9dad-11d1-80b4-0c04fd430c8") 22 | f.Add("6ba7b810-9dad-11d1-8>b4-0c04fd430c8") 23 | f.Add("6ba7b890-9dad-11d1-80b4-0c04fd430cF") 24 | f.Add("6Da7b890-9dad-11d1-80b4-0c04fd430cF") 25 | f.Add("6Da7b8D0-9dad-11d1-80b4-0c04CB430cF") 26 | f.Add("6Da7b8D0-9dad-11d1-80b4-0c04Cd430cF") 27 | f.Add("6Da7b8D0-9dad-11d1-80b4-0c04fd430cF") 28 | f.Add("6Da7b8D0-9dad-11d1-80b6-0c0FABSE\x10cF") 29 | f.Add("6Da7b8D0-9dad-11d1-80b6-0c0FALSE0cF") 30 | f.Add("6Da7b8D0-9dad-11d1-8FAB-0c0FABSE\x10cF") 31 | f.Add("6Da7b8D0-9dad-91d1-80cF-0c0FABSE\x10cF") 32 | f.Add("6Fa7b890-9dad-11d\xd7-80b4-0c04fd43/cb") 33 | f.Add("ba7b810-9dad-11d1-80b4-00c04fd430c8") 34 | f.Add("EDa7b8D0-9dad-11d1-8FAB-0c0FABSE\x10cF") 35 | 36 | f.Fuzz(func(t *testing.T, data string) { 37 | _, _ = uuid.FromString(data) 38 | }) 39 | } 40 | -------------------------------------------------------------------------------- /version/build.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package version 5 | 6 | import ( 7 | "storj.io/common/version/buildinfo" 8 | ) 9 | 10 | // FromBuild returns version string for a module. 11 | // Deprecated: use buildinfo package, which doesn't have any 3rd party dependencies. 12 | func FromBuild(modname string) (string, error) { 13 | return buildinfo.FromBuild(modname) 14 | } 15 | -------------------------------------------------------------------------------- /version/buildinfo/build.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package buildinfo 5 | 6 | import ( 7 | "runtime/debug" 8 | 9 | "github.com/zeebo/errs" 10 | ) 11 | 12 | // Error is common error for version package. 13 | var Error = errs.Class("version") 14 | 15 | // FromBuild returns version string for a module. 16 | // 17 | // This does not work inside tests. 18 | func FromBuild(modname string) (string, error) { 19 | info, ok := debug.ReadBuildInfo() 20 | if !ok { 21 | return "", Error.New("unable to read build info") 22 | } 23 | 24 | findmodule := func(modname string) *debug.Module { 25 | if info.Main.Path == modname { 26 | return &info.Main 27 | } 28 | for _, mod := range info.Deps { 29 | if mod.Path == modname { 30 | return mod 31 | } 32 | } 33 | return nil 34 | } 35 | 36 | mod := findmodule(modname) 37 | if mod == nil { 38 | return "", Error.New("unable to find module %q", modname) 39 | } 40 | 41 | return mod.Version, nil 42 | } 43 | -------------------------------------------------------------------------------- /version/buildinfo/build_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package buildinfo_test 5 | 6 | import ( 7 | "os/exec" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/require" 11 | 12 | "storj.io/common/testcontext" 13 | ) 14 | 15 | func TestFromBuild(t *testing.T) { 16 | ctx := testcontext.New(t) 17 | defer ctx.Cleanup() 18 | 19 | cmd := exec.Command("go", "run", ".") 20 | cmd.Dir = "testbuild" 21 | 22 | data, err := cmd.CombinedOutput() 23 | require.NoError(t, err) 24 | 25 | require.Equal(t, `"v0.0.0-00010101000000-000000000000"`, string(data)) 26 | } 27 | -------------------------------------------------------------------------------- /version/buildinfo/testbuild/go.mod: -------------------------------------------------------------------------------- 1 | module storj.test/buildinfo 2 | 3 | go 1.23.0 4 | 5 | replace storj.io/common => ../../../ 6 | 7 | require storj.io/common v0.0.0-00010101000000-000000000000 8 | 9 | require github.com/zeebo/errs v1.3.0 // indirect 10 | -------------------------------------------------------------------------------- /version/buildinfo/testbuild/go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= 2 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 h1:hR7/MlvK23p6+lIw9SN1TigNLn9ZnF3W4SYRKq2gAHs= 4 | github.com/google/pprof v0.0.0-20230602150820-91b7bce49751/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= 5 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= 6 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 7 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 8 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 9 | github.com/zeebo/errs v1.3.0 h1:hmiaKqgYZzcVgRL1Vkc1Mn2914BbzB0IBxs+ebeutGs= 10 | github.com/zeebo/errs v1.3.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= 11 | golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= 12 | golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= 13 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 14 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 15 | -------------------------------------------------------------------------------- /version/buildinfo/testbuild/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package main 5 | 6 | import ( 7 | "fmt" 8 | "os" 9 | 10 | "storj.io/common/version/buildinfo" 11 | ) 12 | 13 | func main() { 14 | versionstr, err := buildinfo.FromBuild("storj.io/common") 15 | if err != nil { 16 | fmt.Fprintln(os.Stderr, err.Error()) 17 | os.Exit(1) 18 | } 19 | 20 | fmt.Printf("%#v", versionstr) 21 | } 22 | -------------------------------------------------------------------------------- /version/info.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | //go:build go1.18 5 | 6 | package version 7 | 8 | import ( 9 | "runtime/debug" 10 | "strconv" 11 | "time" 12 | ) 13 | 14 | func init() { 15 | i := getInfoFromBuildTags() 16 | undefinedTime := time.Time{} 17 | info, ok := debug.ReadBuildInfo() 18 | if ok { 19 | for _, s := range info.Settings { 20 | switch s.Key { 21 | case "vcs.revision": 22 | if i.CommitHash == "" { 23 | i.CommitHash = s.Value 24 | } 25 | case "vcs.time": 26 | if i.Timestamp == undefinedTime { 27 | i.Timestamp, _ = time.Parse(time.RFC3339Nano, s.Value) 28 | } 29 | case "vcs.modified": 30 | modified, err := strconv.ParseBool(s.Value) 31 | if err == nil { 32 | i.Modified = i.Modified || modified 33 | } 34 | } 35 | } 36 | 37 | } 38 | Build = i 39 | } 40 | -------------------------------------------------------------------------------- /version/info_1.17.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | //go:build !go1.18 5 | 6 | package version 7 | 8 | func init() { 9 | Build = getInfoFromBuildTags() 10 | } 11 | -------------------------------------------------------------------------------- /version/stats_other.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2024 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | //go:build !windows 5 | 6 | package version 7 | 8 | func osversion() (version int64, ok bool) { 9 | return 0, false 10 | } 11 | -------------------------------------------------------------------------------- /version/stats_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2024 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package version 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/spacemonkeygo/monkit/v3" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestOsVersion(t *testing.T) { 14 | t.Log(osversion()) 15 | } 16 | 17 | func TestCommitHashCRC(t *testing.T) { 18 | info := Info{ 19 | CommitHash: "0e7695f391df6c2ea03d2dec02b9059ccaffb9c9", 20 | } 21 | 22 | getCommitHash := func() (ret float64) { 23 | info.Stats(func(key monkit.SeriesKey, field string, val float64) { 24 | if key.Measurement == "version_info" && field == "commit" { 25 | ret = val 26 | } 27 | }) 28 | return ret 29 | } 30 | 31 | // first time the value is calculated, second time cached value is used. 32 | // Let's test both. 33 | require.Equal(t, float64(868966439), getCommitHash()) 34 | require.Equal(t, float64(868966439), getCommitHash()) 35 | } 36 | -------------------------------------------------------------------------------- /version/stats_windows.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2024 Storj Labs, Inc. 2 | // See LICENSE for copying information. 3 | 4 | package version 5 | 6 | import ( 7 | "golang.org/x/sys/windows" 8 | ) 9 | 10 | func osversion() (version int64, ok bool) { 11 | info := windows.RtlGetVersion() 12 | if info == nil { 13 | return 0, false 14 | } 15 | 16 | // Current maximum minor version is 3, 17 | // so the following computation should be fine. 18 | 19 | return int64(info.MajorVersion)*10 + int64(info.MinorVersion), true 20 | } 21 | --------------------------------------------------------------------------------