├── bridge.log ├── docker.aws ├── DOCKER_VERSION ├── bin │ ├── stop.blobber.sh │ ├── build.blobber.sh │ ├── start.blobber.sh │ ├── init.setup.sh │ └── start.blobber.detach.sh ├── build.blobber │ ├── .env │ ├── docker-compose.yml │ └── Dockerfile └── build.validator │ ├── .env │ ├── docker-compose.yml │ └── Dockerfile ├── code └── go │ └── 0chain.net │ ├── blobber │ ├── .gitignore │ ├── prepare_noop.go │ ├── filestore.go │ ├── logging.go │ ├── prepare_conductor_testing.go │ ├── grpc.go │ ├── datastore.go │ ├── config.go │ ├── worker.go │ └── main.go │ ├── conductor │ ├── build.go │ ├── config │ │ ├── config.go │ │ └── byzantine.go │ └── conductrpc │ │ ├── server.go │ │ ├── resolve.go │ │ ├── file_meta.go │ │ └── client.go │ ├── blobbercore │ ├── handler │ │ ├── storage_handler_bench_test.go │ │ ├── helper.go │ │ ├── handler_hashnode.go │ │ ├── grpc_handler_main.go │ │ ├── grpc_handler_integration_tests.go │ │ ├── grpc_commit_handler.go │ │ ├── grpc_getallocation_component_test.go │ │ ├── handler_middlewares.go │ │ ├── object_operation_grpc_handler_main.go │ │ ├── handler_main.go │ │ ├── object_operation_grpc_handler_integration_tests.go │ │ ├── file_command.go │ │ ├── object_operation_grpc_handler.go │ │ ├── grpc_getobjectpath_component_test.go │ │ ├── grpc_getreferencepath_component_test.go │ │ ├── auth_ticket.go │ │ └── health.go │ ├── reference │ │ ├── vars.go │ │ ├── ds_test.go │ │ ├── entity.go │ │ ├── hashnode.go │ │ └── object.go │ ├── stats │ │ ├── dbstats.go │ │ ├── infrastats.go │ │ └── allocationstats.go │ ├── writemarker │ │ ├── write_lock.go │ │ ├── protocol_main.go │ │ ├── entity_test.go │ │ ├── writemarker.go │ │ └── protocol_integration_tests.go │ ├── datastore │ │ ├── model.go │ │ ├── blob_type.go │ │ ├── memory.go │ │ ├── README.md │ │ └── mock_store.go │ ├── mock │ │ ├── buf.go │ │ ├── allocation.go │ │ └── ctx.go │ ├── filestore │ │ ├── tree_validation_main.go │ │ ├── misc.go │ │ ├── file_manager.go │ │ └── tree_validation_integration_tests.go │ ├── allocation │ │ ├── renamefilechange_main.go │ │ ├── file_changer_upload_main.go │ │ ├── dao.go │ │ ├── file_changer.go │ │ ├── renamefilechange_integration.go │ │ ├── file_changer_upload_integration.go │ │ ├── connection_bench_test.go │ │ └── rollback.go │ ├── challenge │ │ └── stats.go │ ├── seqpriorityqueue │ │ └── seqpriorityqueue_test.go │ ├── zcn │ │ └── query.go │ ├── blobbergrpc │ │ ├── proto │ │ │ └── google │ │ │ │ └── api │ │ │ │ └── annotations.proto │ │ └── README.md │ ├── util │ │ └── json.go │ └── readmarker │ │ └── protocol.go │ ├── core │ ├── build │ │ └── info.go │ ├── common │ │ ├── types.go │ │ ├── blobber_registered.go │ │ ├── lookup.go │ │ ├── errors.go │ │ ├── aws.go │ │ ├── time.go │ │ ├── request_form.go │ │ ├── admin.go │ │ ├── pagination.go │ │ ├── handler │ │ │ └── worker.go │ │ ├── lock.go │ │ ├── constants.go │ │ └── context.go │ ├── cache │ │ ├── cache.go │ │ ├── lfu.go │ │ └── lru.go │ ├── config │ │ └── config.go │ ├── node │ │ └── context.go │ ├── transaction │ │ ├── vars.go │ │ ├── type.go │ │ └── http.go │ ├── lock │ │ ├── lock_test.go │ │ └── lock.go │ ├── util │ │ └── secure_value.go │ └── chain │ │ └── entity.go │ ├── validator │ ├── prepare_main.go │ └── prepare_integration_tests.go │ ├── tools │ └── tool.go │ ├── docs │ └── doc.go │ ├── dev │ ├── miner │ │ └── init.go │ ├── sharder │ │ └── init.go │ └── server.go │ └── validatorcore │ └── storage │ ├── challenge_handler_main.go │ ├── context.go │ ├── handler_main.go │ ├── handler_integration_tests.go │ ├── handler.go │ └── challenge_handler_integration_tests.go ├── .dockerignore ├── docker.local ├── bin │ ├── sync_clock.sh │ ├── blobber.init.setup-mac.sh │ ├── aws.build.blobber.sh │ ├── clean.sh │ ├── docker-clean.sh │ ├── blobber.start_github.sh │ ├── blobber.init.setup.sh │ ├── stop_all.sh │ ├── blobber.start.sh │ ├── blobber.stop_bls.sh │ ├── p0blobber.start.sh │ ├── build.blobber.dev.sh │ ├── blobber.start_bls.sh │ ├── build.validator.sh │ ├── build.base.sh │ ├── build.blobber.sh │ ├── build.blobber-integration-tests.sh │ └── test.swagger.sh ├── keys_config │ ├── minio_config.txt │ ├── b0vnode1_keys.txt │ ├── b0vnode2_keys.txt │ ├── b0vnode3_keys.txt │ ├── b0vnode4_keys.txt │ ├── b0vnode5_keys.txt │ ├── b0vnode6_keys.txt │ ├── b0bnode2_keys.txt │ ├── b0bnode3_keys.txt │ ├── b0bnode4_keys.txt │ ├── b0bnode5_keys.txt │ ├── b0bnode6_keys.txt │ ├── bnode2_keys.txt │ ├── bnode3_keys.txt │ ├── bnode4_keys.txt │ ├── bnode5_keys.txt │ ├── bnode6_keys.txt │ ├── b0bnode1_keys.txt │ ├── bnode1_keys.txt │ ├── b0bnode1_keys.txt.json │ ├── b0bnode2_keys.txt.json │ ├── b0bnode3_keys.txt.json │ ├── b0bnode4_keys.txt.json │ ├── b0bnode5_keys.txt.json │ ├── b0bnode6_keys.txt.json │ ├── b0vnode1_keys.txt.json │ ├── b0vnode2_keys.txt.json │ ├── b0vnode3_keys.txt.json │ ├── b0vnode4_keys.txt.json │ ├── b0vnode5_keys.txt.json │ └── b0vnode6_keys.txt.json ├── docker-clean │ ├── Dockerfile │ ├── docker-clean.sh │ └── docker-clean-compose.yml ├── .dockerignore ├── sql_init │ ├── 000-init-db.sh │ └── 00-init.sql ├── base.Dockerfile ├── b0docker-compose-github.yml ├── ValidatorDockerfile ├── IntegrationTestsValidator.Dockerfile ├── IntegrationTestsBlobber.Dockerfile ├── ValidatorDockerfile.dev ├── validator.Dockerfile ├── Dockerfile.dev ├── blobber.Dockerfile ├── Dockerfile ├── Dockerfile.swagger └── conductor-config │ └── 0chain_validator.yaml ├── docs ├── cicd │ └── blobber.png └── src │ ├── out │ ├── repair │ │ └── repair.png │ └── challenge │ │ └── challenge.png │ ├── share_revoke.plantuml │ ├── share.plantuml │ ├── proxy_encryption_download_flow.plantuml │ ├── download_flow.plantuml │ ├── upload_flow.puml │ ├── upload_flow.plantuml │ └── repair.plantuml ├── test.sh ├── .github ├── CODEOWNERS ├── workflows │ ├── automate-add-issue-in-project.yaml │ ├── benchmark.yml │ ├── cicd.yml │ ├── dev-1.yml │ ├── gosdk.yml │ └── config_change_alert.yaml └── dependabot.yml ├── goose ├── migrations │ ├── 1705084186_idle_timeout.sql │ ├── 1705414624_close_txn_nonce.sql │ ├── 1710413946_filestore_version.sql │ ├── 1731740380_encryption__version.sql │ ├── 1742282974_cleanup_connections.sql │ ├── 1718188301_change_idx.sql │ ├── 1716455063_ref_name_length.sql │ ├── 1718391849_ref_index.sql │ ├── 1717416291_change_lookuphash.sql │ ├── 1707996797_chain_wm.sql │ ├── 1718355236_terms_index.sql │ ├── 1730098482_session_key.sql │ └── 1725515413_storage_version.sql └── migrator.go ├── buf.gen.yaml ├── .gitignore ├── config ├── localhost.key ├── localhost.crt └── 0chain_validator.yaml ├── generate_migration.sh ├── buf.yaml ├── bin └── postgres-entrypoint.sh ├── PULL_REQUEST_TEMPLATE.md ├── https ├── docker-compose.yml └── conf.d │ └── nginx.conf ├── LICENSE.txt └── dev.local └── README.md /bridge.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docker.aws/DOCKER_VERSION: -------------------------------------------------------------------------------- 1 | 0.1.11 2 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobber/.gitignore: -------------------------------------------------------------------------------- 1 | blobber 2 | /config -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | awsnet 3 | docker.local/blobber* 4 | **/pkg 5 | 6 | -------------------------------------------------------------------------------- /code/go/0chain.net/conductor/build.go: -------------------------------------------------------------------------------- 1 | package conductor 2 | 3 | func init() {} 4 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/handler/storage_handler_bench_test.go: -------------------------------------------------------------------------------- 1 | package handler 2 | -------------------------------------------------------------------------------- /docker.local/bin/sync_clock.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | docker run --rm --privileged alpine hwclock -s -------------------------------------------------------------------------------- /docs/cicd/blobber.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0chain/blobber/HEAD/docs/cicd/blobber.png -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | cd code/go/0chain.net; CGO_ENABLED=1 go test ./...; -------------------------------------------------------------------------------- /docs/src/out/repair/repair.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0chain/blobber/HEAD/docs/src/out/repair/repair.png -------------------------------------------------------------------------------- /docs/src/out/challenge/challenge.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0chain/blobber/HEAD/docs/src/out/challenge/challenge.png -------------------------------------------------------------------------------- /code/go/0chain.net/core/build/info.go: -------------------------------------------------------------------------------- 1 | package build 2 | 3 | // BuildTag - the git commit for the build 4 | var BuildTag string 5 | -------------------------------------------------------------------------------- /docker.aws/bin/stop.blobber.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | docker-compose -p zchain -f docker.aws/build.blobber/docker-compose.yml stop 3 | 4 | -------------------------------------------------------------------------------- /docker.aws/bin/build.blobber.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | docker-compose -p zchain -f docker.aws/build.blobber/docker-compose.yml build --force-rm 3 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/reference/vars.go: -------------------------------------------------------------------------------- 1 | package reference 2 | 3 | const ( 4 | TableNameReferenceObjects = "reference_objects" 5 | ) 6 | -------------------------------------------------------------------------------- /docker.local/keys_config/minio_config.txt: -------------------------------------------------------------------------------- 1 | play.min.io 2 | Q3AM3UQ867SPQQA43P2F 3 | zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG 4 | mytestbucket 5 | us-east-1 -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # The following users own all code in the blobber repo 2 | # Peter Yury Saswata Dayi 3 | * @peterlimg @dabasov @guruhubb @cnlangzi 4 | -------------------------------------------------------------------------------- /goose/migrations/1705084186_idle_timeout.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | SET idle_in_transaction_session_timeout = 180000; 4 | -- +goose StatementEnd 5 | -------------------------------------------------------------------------------- /docker.aws/bin/start.blobber.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | echo Starting blobber... 3 | docker-compose -p zchain -f /0chain/docker.aws/build.blobber/docker-compose.yml up 4 | 5 | -------------------------------------------------------------------------------- /code/go/0chain.net/validator/prepare_main.go: -------------------------------------------------------------------------------- 1 | //go:build !integration_tests 2 | // +build !integration_tests 3 | 4 | package main 5 | 6 | func prepare(id string) { 7 | 8 | } 9 | -------------------------------------------------------------------------------- /docker.local/bin/blobber.init.setup-mac.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | for i in $(seq 1 6) 4 | do 5 | mkdir -p docker.local/blobber$i 6 | mkdir -p docker.local/validator$i 7 | done 8 | -------------------------------------------------------------------------------- /goose/migrations/1705414624_close_txn_nonce.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | ALTER TABLE write_markers ADD COLUMN close_txn_nonce BIGINT; 4 | -- +goose StatementEnd -------------------------------------------------------------------------------- /code/go/0chain.net/core/common/types.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | type ProviderType byte 4 | 5 | const ( 6 | ProviderTypeBlobber ProviderType = iota 7 | ProviderTypeValidator 8 | ) 9 | -------------------------------------------------------------------------------- /docker.local/docker-clean/Dockerfile: -------------------------------------------------------------------------------- 1 | 2 | # 3 | # clean up blockchain without sudo (as 'docker' system group member) 4 | # 5 | 6 | FROM alpine:latest 7 | COPY docker-clean.sh ./docker-clean.sh 8 | -------------------------------------------------------------------------------- /goose/migrations/1710413946_filestore_version.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | ALTER TABLE reference_objects ADD COLUMN filestore_version INTEGER DEFAULT 0; 4 | -- +goose StatementEnd -------------------------------------------------------------------------------- /goose/migrations/1731740380_encryption__version.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | 4 | ALTER TABLE reference_objects ADD COLUMN encryption_version smallint; 5 | 6 | -- +goose StatementEnd -------------------------------------------------------------------------------- /goose/migrations/1742282974_cleanup_connections.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | 4 | DELETE FROM allocation_connections WHERE status = 2 OR status = 3; 5 | 6 | -- +goose StatementEnd -------------------------------------------------------------------------------- /code/go/0chain.net/core/cache/cache.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | type Cache interface { 4 | Add(key string, value interface{}) error 5 | Get(key string) (interface{}, error) 6 | Delete(key string) error 7 | } 8 | -------------------------------------------------------------------------------- /docker.local/.dockerignore: -------------------------------------------------------------------------------- 1 | passphrase.txt 2 | logs/ 3 | docs/ 4 | .git 5 | *.md 6 | .cache 7 | awsnet 8 | docker.local/blobber* 9 | docker.aws/* 10 | config/ 11 | docker-clean/ 12 | keys_config 13 | **/pkg -------------------------------------------------------------------------------- /goose/migrations/1718188301_change_idx.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | 4 | CREATE INDEX idx_allocation_changes_lookup_hash ON allocation_changes USING HASH(lookup_hash); 5 | -- +goose StatementEnd -------------------------------------------------------------------------------- /goose/migrations/1716455063_ref_name_length.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | ALTER TABLE reference_objects ALTER COLUMN name TYPE character varying(150), 4 | ALTER COLUMN name SET NOT NULL; 5 | -- +goose StatementEnd -------------------------------------------------------------------------------- /docker.local/bin/aws.build.blobber.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | docker-compose -p zchain -f docker.local/docker-compose.yml build --force-rm 4 | #docker-compose -p zchain -f docker.aws/build.miner/docker-compose.yml build --force-rm 5 | 6 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/stats/dbstats.go: -------------------------------------------------------------------------------- 1 | package stats 2 | 3 | import ( 4 | "database/sql" 5 | ) 6 | 7 | // DBStats contains database statistics. 8 | type DBStats struct { 9 | sql.DBStats 10 | Status string 11 | } 12 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/writemarker/write_lock.go: -------------------------------------------------------------------------------- 1 | package writemarker 2 | 3 | import "time" 4 | 5 | // WriteLock WriteMarker lock 6 | type WriteLock struct { 7 | ConnectionID string 8 | CreatedAt time.Time 9 | } 10 | -------------------------------------------------------------------------------- /docker.aws/bin/init.setup.sh: -------------------------------------------------------------------------------- 1 | 2 | mkdir -p /0chain/files 3 | mkdir -p /0chain/data/badgerdb/blobberstate 4 | mkdir -p /0chain/logs 5 | sudo chmod -R 777 /0chain/data 6 | sudo chmod -R 777 /0chain/files 7 | sudo chmod -R 777 /0chain/logs 8 | -------------------------------------------------------------------------------- /docker.local/keys_config/b0vnode1_keys.txt: -------------------------------------------------------------------------------- 1 | 0a82a096ff14334bd78f061d451c96913cef8741742e1bc2af8e0276dbe7641926e926629597b0d39c1cbb31b63390cfae21e6f357c25a1b9dc53d0d757afe8a 2 | f94d9ed96484e2bec0772d2187d488a42af4ef489ed697a4d603e9ce8d5b771b 3 | -------------------------------------------------------------------------------- /docker.local/keys_config/b0vnode2_keys.txt: -------------------------------------------------------------------------------- 1 | 0a47b0e582f70aff30c6143e03462b52b389c67dcc73feb4e6e6c58a71290c03d00d548967ed33254e93817e68d8a2bbfc5a35fa09b61c50e32503fee36c06a1 2 | dca0e71419aadf266f41f2d5569fbf6174010fe5453433c7569e8ce9d7d5e113 3 | -------------------------------------------------------------------------------- /docker.local/keys_config/b0vnode3_keys.txt: -------------------------------------------------------------------------------- 1 | daab4f68dcd5e68ef03465e17c05bc9ba4a669361a511c2be96ca2b6b5faad0d6b03184ecd59a5c6a6aab74ee878b34bd309a7324ea3dca18153fa2f1962f60d 2 | 546b31cd186094d6b079bf10d8b68b3126f1722b5088817703977d30c1e37a0d 3 | -------------------------------------------------------------------------------- /docker.local/keys_config/b0vnode4_keys.txt: -------------------------------------------------------------------------------- 1 | a76736a941846763a774bdbd0e63ad38ed229b3fd392112041daeefa33cc120c4566c3929ba3938a6f8fce4b89197a1b1083facf6db091a43756bf117ee2d687 2 | 808cc0a5ba55c505914fa03e09562058c941cf75dcf16ea800fa71798817001a 3 | -------------------------------------------------------------------------------- /docker.local/keys_config/b0vnode5_keys.txt: -------------------------------------------------------------------------------- 1 | 2adaa464eedc053716d36ebcc43fd7c653829ef6eee970fa6069c00b43a57f163d9e3170b132bb96f8c8cde31861c3fa3b41cb73b41f4f7ec863eab0aab6348c 2 | 92847e81af38f35b0a4152803eda38b540cfa714c2772b1346b9c7fb74a9d322 3 | -------------------------------------------------------------------------------- /docker.local/keys_config/b0vnode6_keys.txt: -------------------------------------------------------------------------------- 1 | b6bea180d777d7a11b15d1eddd94d705a49286f437b4d17b9d97ebaebf481a1e69862f92d9695c3b573ee3ae5854f343f3f22790a3c5fbab55ea202a84fa6881 2 | 208ad5bcbd6523ba7b185d8f09a9fd32f7759d5e99a7dcd84599b972abb22d01 3 | -------------------------------------------------------------------------------- /goose/migrations/1718391849_ref_index.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | DROP INDEX IF EXISTS idx_created_at,idx_updated_at,idx_lookup_hash,idx_path_gin_trgm,idx_name_gin,idx_allocation_changes_lookup_hash; 4 | -- +goose StatementEnd -------------------------------------------------------------------------------- /docker.local/keys_config/b0bnode2_keys.txt: -------------------------------------------------------------------------------- 1 | e4dc5262ed8e20583e3293f358cc21aa77c2308ea773bab8913670ffeb5aa30d7e2effbce51f323b5b228ad01f71dc587b923e4aab7663a573ece5506f2e3b0e 2 | 4b6d9c5f7b0386e36b212324ea52f5ff17a9ed1338ca901d7f7fa7637159a912 3 | localhost 4 | 5052 -------------------------------------------------------------------------------- /docker.local/keys_config/b0bnode3_keys.txt: -------------------------------------------------------------------------------- 1 | a2670e307b6a0647e63e5d99be77c71158811ba76ae3b1b0e1a179068f720315b1e6a3ed5d538748661f3dd921e2221b7b219ecd49635c27fb566b2179882712 2 | f306c2bfc07509ee653e548171e3be3eb1c2a7bc344b30128e82c99694370006 3 | localhost 4 | 5053 -------------------------------------------------------------------------------- /docker.local/keys_config/b0bnode4_keys.txt: -------------------------------------------------------------------------------- 1 | 1a97fb9074d9b46d99104bef4beebb56d1a3c47b14f7f87399219a32c2ea7609deaef7d63b33f7e7498df8e39efdb69e93f7b14266be626abe34ea4c5f869c19 2 | 81f34514f462e6348e79c9c37c150ebeb63b080f682dba8173115582540e3922 3 | localhost 4 | 5054 -------------------------------------------------------------------------------- /docker.local/keys_config/b0bnode5_keys.txt: -------------------------------------------------------------------------------- 1 | 6e1e0c5a44bd1ca958b73e4b3f778362b6965be7b5f1aadeef84d8408bbeda0628f7bf906aa12e43195cd52d98db6c328460564d8581d940bc561297d0c81015 2 | ef9a9ac556d58ede0cc9f50bee0a6e32de2c338bf23ef9d43b2ea8cd04e9c80a 3 | localhost 4 | 5055 -------------------------------------------------------------------------------- /docker.local/keys_config/b0bnode6_keys.txt: -------------------------------------------------------------------------------- 1 | ee5f1b9d3bd74c8ab9ea4f045a3500d3678f37aa0fc14181445c8286ccf80d0e1d0310ce834313fde2345a3006e432939e7009c7f0c1a233005bdbbf7d20b88a 2 | 5d519368d416a5a186e8681e07b72cec19039ff1519c210db8c94477425e8915 3 | localhost 4 | 5056 -------------------------------------------------------------------------------- /docker.local/keys_config/bnode2_keys.txt: -------------------------------------------------------------------------------- 1 | 4c9d54fb676ea3df7211d1e26771270ddf22e7fd5bd172ac59e282f561921cd9 2 | 7e169941cfc917ce05b3b6b552b4629825a04d2603d4ef6ef641edc297e60f6b4c9d54fb676ea3df7211d1e26771270ddf22e7fd5bd172ac59e282f561921cd9 3 | localhost 4 | 5052 -------------------------------------------------------------------------------- /docker.local/keys_config/bnode3_keys.txt: -------------------------------------------------------------------------------- 1 | cb0da66e7ba55f7e06b4e451ba57b6e535412e83b128454dba5bfae7fbc84b50 2 | 21e28803d2da40b8af43d7fbd4be68e73d649d982d202aa3241cb8cde4793055cb0da66e7ba55f7e06b4e451ba57b6e535412e83b128454dba5bfae7fbc84b50 3 | localhost 4 | 5053 -------------------------------------------------------------------------------- /docker.local/keys_config/bnode4_keys.txt: -------------------------------------------------------------------------------- 1 | c21a9f18f768b5dba10d8c6a5811293de0a47042f091eb153650bb6beee3f83f 2 | 15e9fb90ebd24ba197c488642714a0dad743ad5efa7765aff1ff750ea90c36bec21a9f18f768b5dba10d8c6a5811293de0a47042f091eb153650bb6beee3f83f 3 | localhost 4 | 5054 -------------------------------------------------------------------------------- /docker.local/keys_config/bnode5_keys.txt: -------------------------------------------------------------------------------- 1 | 5076d8e123b66fa814860cb26f62c072706bd3743cb94bea8737227c2592ecd3 2 | b656183e3a6fc53b6f53e74a76eef763c3b84b2a5abdc5c68140c57fa94276405076d8e123b66fa814860cb26f62c072706bd3743cb94bea8737227c2592ecd3 3 | localhost 4 | 5055 -------------------------------------------------------------------------------- /docker.local/keys_config/bnode6_keys.txt: -------------------------------------------------------------------------------- 1 | 28856ae67db05a79a7838e749be6d657a78806f849dcd158b3e8fb26162326d0 2 | ddd4a0b62952b8d7ad3c2f437cf0133ef047ad24b161cb6c99e0c9847d8e2e2328856ae67db05a79a7838e749be6d657a78806f849dcd158b3e8fb26162326d0 3 | localhost 4 | 5056 -------------------------------------------------------------------------------- /docker.local/keys_config/b0bnode1_keys.txt: -------------------------------------------------------------------------------- 1 | de52c0a51872d5d2ec04dbc15a6f0696cba22657b80520e1d070e72de64c9b04e19ce3223cae3c743a20184158457582ffe9c369ca9218c04bfe83a26a62d88d 2 | 17fa2ab0fb49249cb46dbc13e4e9e6853af8b1506e48d84c03e5e92f6348bb1d 3 | localhost 4 | 5051 5 | -------------------------------------------------------------------------------- /docker.local/keys_config/bnode1_keys.txt: -------------------------------------------------------------------------------- 1 | 91cb9bc0715190f35faddc2c4542585b2cda5d958cca5278dc10331a331a80dc 2 | 7e5d97d741cba5d771c922bc08bb01fbdf50af51d86872751aa5f5231b61fc2891cb9bc0715190f35faddc2c4542585b2cda5d958cca5278dc10331a331a80dc 3 | localhost 4 | 5051 5 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/datastore/model.go: -------------------------------------------------------------------------------- 1 | package datastore 2 | 3 | import "time" 4 | 5 | type ModelWithTS struct { 6 | CreatedAt time.Time `gorm:"column:created_at" json:"created_at"` 7 | UpdatedAt time.Time `gorm:"column:updated_at" json:"updated_at"` 8 | } 9 | -------------------------------------------------------------------------------- /docker.aws/bin/start.blobber.detach.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | echo Starting blobber ... 3 | #docker-compose -p zchain -f /0chain/docker.aws/build.blobber/docker-compose.yml up --detach 4 | docker-compose -f /0chain/docker.aws/build.blobber/docker-compose.yml up --detach 5 | 6 | -------------------------------------------------------------------------------- /docker.local/sql_init/000-init-db.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | mkdir -p $SLOW_TABLESPACE_PATH 5 | 6 | psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL 7 | create tablespace $SLOW_TABLESPACE location '$SLOW_TABLESPACE_PATH'; 8 | EOSQL 9 | -------------------------------------------------------------------------------- /docker.local/bin/clean.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | for i in $(seq 1 6) 4 | do 5 | sudo rm -rf /mnt/hdd/blobber$i 6 | sudo rm -rf /mnt/ssd/blobber$i 7 | sudo rm -rf /mnt/hdd/validator$i 8 | sudo rm -rf docker.local/blobber$i 9 | sudo rm -rf docker.local/validator$i 10 | done 11 | -------------------------------------------------------------------------------- /docker.local/bin/docker-clean.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | docker-compose \ 5 | -f ./docker.local/docker-clean/docker-clean-compose.yml \ 6 | up \ 7 | --build docker-clean 8 | -------------------------------------------------------------------------------- /docker.local/bin/blobber.start_github.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | PWD=`pwd` 5 | BLOBBER_DIR=`basename $PWD` 6 | BLOBBER_ID=`echo my directory $BLOBBER_DIR | sed -e 's/.*\(.\)$/\1/'` 7 | 8 | 9 | BLOBBER=$BLOBBER_ID docker-compose -p blobber$BLOBBER_ID -f ../b0docker-compose-github.yml up -d 10 | -------------------------------------------------------------------------------- /docker.local/bin/blobber.init.setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | for i in $(seq 1 6) 4 | do 5 | sudo mkdir -p docker.local/blobber$i 6 | sudo mkdir -p /mnt/ssd/blobber$i 7 | sudo mkdir -p /mnt/hdd/blobber$i 8 | sudo mkdir -p docker.local/validator$i 9 | sudo mkdir -p /mnt/hdd/validator$i 10 | done 11 | -------------------------------------------------------------------------------- /docker.aws/build.blobber/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=0chain_blobber 2 | IMAGE_NAME=0chain/blobber 3 | IMAGE_TAG=latest 4 | AGENT_PORT=5051 5 | AGENT_DIR=blobber 6 | AGENT_CONTAINER=blobber-b00 7 | DOCKER_CODEBASE=/0chain 8 | ZCHAIN_USER=2000 9 | ZCHAIN_GROUP=2000 10 | MAX_RETRY_ATTEMPTS=3 11 | RESTART_POLICY="no" 12 | -------------------------------------------------------------------------------- /docker.local/bin/stop_all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cd docker.local/blobber1 3 | ../bin/blobber.stop_bls.sh 4 | cd - 5 | cd docker.local/blobber2 6 | ../bin/blobber.stop_bls.sh 7 | cd - 8 | cd docker.local/blobber3 9 | ../bin/blobber.stop_bls.sh 10 | cd - 11 | cd docker.local/blobber4 12 | ../bin/blobber.stop_bls.sh 13 | -------------------------------------------------------------------------------- /goose/migrations/1717416291_change_lookuphash.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | ALTER TABLE allocation_changes ADD COLUMN lookup_hash character varying(64); 4 | 5 | -- CREATE UNIQUE INDEX idx_allocation_changes_lookup_hash ON allocation_changes USING HASH(lookup_hash,connection_id); 6 | -- +goose StatementEnd -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/mock/buf.go: -------------------------------------------------------------------------------- 1 | package mock 2 | 3 | import "crypto/rand" 4 | 5 | func GenerateRandomBytes(n int) []byte { 6 | b := make([]byte, n) 7 | _, err := rand.Read(b) 8 | // Note that err == nil only if we read len(b) bytes. 9 | if err != nil { 10 | return nil 11 | } 12 | 13 | return b 14 | } 15 | -------------------------------------------------------------------------------- /docker.aws/build.validator/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=0chain_validator 2 | IMAGE_NAME=0chain/validator 3 | IMAGE_TAG=latest 4 | AGENT_PORT=5061 5 | AGENT_DIR=validator 6 | AGENT_CONTAINER=validator-v00 7 | DOCKER_CODEBASE=/0chain 8 | ZCHAIN_USER=2000 9 | ZCHAIN_GROUP=2000 10 | MAX_RETRY_ATTEMPTS=3 11 | RESTART_POLICY="no" 12 | -------------------------------------------------------------------------------- /buf.gen.yaml: -------------------------------------------------------------------------------- 1 | version: v1beta1 2 | plugins: 3 | - name: go 4 | out: proto 5 | opt: paths=source_relative 6 | - name: go-grpc 7 | out: proto 8 | opt: paths=source_relative,require_unimplemented_servers=false 9 | - name: grpc-gateway 10 | out: proto 11 | opt: paths=source_relative,allow_delete_body=true -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/writemarker/protocol_main.go: -------------------------------------------------------------------------------- 1 | //go:build !integration_tests 2 | // +build !integration_tests 3 | 4 | package writemarker 5 | 6 | import "context" 7 | 8 | func (wme *WriteMarkerEntity) RedeemMarker(ctx context.Context, startSeq int64) error { 9 | return wme.redeemMarker(ctx, startSeq) 10 | } 11 | -------------------------------------------------------------------------------- /code/go/0chain.net/validator/prepare_integration_tests.go: -------------------------------------------------------------------------------- 1 | //go:build integration_tests 2 | // +build integration_tests 3 | 4 | package main 5 | 6 | import ( 7 | crpc "github.com/0chain/blobber/code/go/0chain.net/conductor/conductrpc" // integration tests 8 | ) 9 | 10 | func prepare(id string) { 11 | crpc.Init(id) 12 | } 13 | -------------------------------------------------------------------------------- /goose/migrations/1707996797_chain_wm.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | ALTER TABLE write_markers 4 | ADD COLUMN chain_hash character varying(64), 5 | ADD COLUMN chain_size BIGINT, 6 | ADD COLUMN chain_length integer; 7 | 8 | ALTER TABLE allocations ADD COLUMN last_redeemed_sequence BIGINT DEFAULT 0; 9 | -- +goose StatementEnd -------------------------------------------------------------------------------- /docker.local/bin/blobber.start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | PWD=`pwd` 3 | BLOBBER_DIR=`basename $PWD` 4 | BLOBBER_ID=`echo my directory $BLOBBER_DIR | sed -e 's/.*\(.\)$/\1/'` 5 | 6 | 7 | echo Starting blobber$BLOBBER_ID ... 8 | 9 | # echo blobber$i 10 | 11 | BLOBBER=$BLOBBER_ID docker-compose -p blobber$BLOBBER_ID -f ../docker-compose.yml up 12 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobber/prepare_noop.go: -------------------------------------------------------------------------------- 1 | //go:build !integration_tests 2 | // +build !integration_tests 3 | 4 | package main 5 | 6 | // the following prepare functions are noop 7 | 8 | func prepareBlobber(id string) {} 9 | 10 | func prepareBlobberShutdown() {} //nolint:unused,deadcode // looks like it is being used in integration test 11 | -------------------------------------------------------------------------------- /docker.local/bin/blobber.stop_bls.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | PWD=`pwd` 3 | BLOBBER_DIR=`basename $PWD` 4 | BLOBBER_ID=`echo my directory $BLOBBER_DIR | sed -e 's/.*\(.\)$/\1/'` 5 | 6 | 7 | echo Stopping blobber$BLOBBER_ID ... 8 | 9 | # echo blobber$i 10 | 11 | BLOBBER=$BLOBBER_ID docker-compose -p blobber$BLOBBER_ID -f ../b0docker-compose.yml down 12 | -------------------------------------------------------------------------------- /code/go/0chain.net/tools/tool.go: -------------------------------------------------------------------------------- 1 | // +build tools 2 | 3 | package tools 4 | 5 | import ( 6 | _ "github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway" 7 | _ "github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2" 8 | _ "google.golang.org/grpc/cmd/protoc-gen-go-grpc" 9 | _ "google.golang.org/protobuf/cmd/protoc-gen-go" 10 | ) 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /data 2 | coverage.txt 3 | .DS_Store 4 | .idea/ 5 | .vscode/ 6 | __debug_bin 7 | **/client 8 | !**/client/main.go 9 | **/docker.local/blobber* 10 | **/blobber/blobber/blobber 11 | **/blobber/blobber/logs 12 | **/blobber/blobber/files 13 | **/blobber/blobber/data 14 | **/pkg/ 15 | dev.local/data 16 | out/ 17 | **/tmp/ 18 | *.tar.gz 19 | **/descriptor.proto -------------------------------------------------------------------------------- /code/go/0chain.net/core/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | /*Config - all the config options passed from the command line*/ 4 | type Config struct { 5 | Host string 6 | Port int 7 | ChainID string 8 | Capacity int64 9 | DeploymentMode byte 10 | SignatureScheme string 11 | } 12 | 13 | var Configuration Config 14 | -------------------------------------------------------------------------------- /code/go/0chain.net/conductor/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // common types 4 | type ( 5 | NodeName string // node name used in configurations 6 | NodeID string // a NodeID is ID of a miner or a sharder 7 | Round int64 // a Round number 8 | RoundName string // round name (remember round, round next after VC) 9 | Number int64 // magic block number 10 | ) 11 | -------------------------------------------------------------------------------- /code/go/0chain.net/docs/doc.go: -------------------------------------------------------------------------------- 1 | 2 | // Package classification Züs Blobber API. 3 | // 4 | // Documentation of the blobber API. 5 | // 6 | // Schemes: https 7 | // BasePath: / 8 | // Version: 1.12.0 9 | // Consumes: 10 | // - application/json 11 | // - application/x-www-form-urlencoded 12 | // - multipart/form-data 13 | // 14 | // swagger:meta 15 | package docs -------------------------------------------------------------------------------- /docker.local/docker-clean/docker-clean.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo "cleaning 6 blobbers..." 4 | for i in $(seq 1 6) 5 | do 6 | echo "deleting blobber$i logs" 7 | rm -rf ./blobber$i/log/* 8 | echo "deleting blobber$i postgresql data" 9 | rm -rf ./blobber$i/data/postgresql/* 10 | echo "deleting blobber$i files" 11 | rm -rf ./blobber$i/data/files/* 12 | done 13 | 14 | echo "cleaned up" 15 | -------------------------------------------------------------------------------- /docker.local/base.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.22.0-alpine3.18 as blobber_base 2 | 3 | LABEL zchain="blobber" 4 | 5 | RUN apk add --update --no-cache linux-headers build-base git cmake bash perl grep 6 | 7 | # Install Herumi's cryptography 8 | WORKDIR /tmp 9 | 10 | RUN apk upgrade 11 | RUN apk del libstdc++ gmp-dev openssl-dev vips-dev 12 | RUN apk add --update --no-cache libstdc++ gmp-dev openssl-dev vips-dev -------------------------------------------------------------------------------- /config/localhost.key: -------------------------------------------------------------------------------- 1 | -----BEGIN EC PARAMETERS----- 2 | BgUrgQQAIg== 3 | -----END EC PARAMETERS----- 4 | -----BEGIN EC PRIVATE KEY----- 5 | MIGkAgEBBDCyVug6uG+OfbkNxOtdK55M2+G143LeVb2j7z5L7comr9as3Ff10PGv 6 | Dv89Z4e1V8GgBwYFK4EEACKhZANiAAT+88OMeYPMz9uwBAWbw2Or4PCU31WPvdlT 7 | Q+A8YeCkmJy9ROmXsvKtuaUY8kvyDANmfemSkk33ToRHpjzMaodQBPFZ/h/rTXj7 8 | MOgSOuhoNoVOXV0ixdfwEKBxcQwpDcU= 9 | -----END EC PRIVATE KEY----- 10 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/handler/helper.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/0chain/blobber/code/go/0chain.net/core/common" 7 | ) 8 | 9 | func checkValidDate(s, dateLayOut string) error { 10 | if s != "" { 11 | _, err := time.Parse(dateLayOut, s) 12 | if err != nil { 13 | return common.NewError("invalid_parameters", err.Error()) 14 | } 15 | } 16 | return nil 17 | } 18 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/filestore/tree_validation_main.go: -------------------------------------------------------------------------------- 1 | //go:build !integration_tests 2 | // +build !integration_tests 3 | 4 | package filestore 5 | 6 | import "io" 7 | 8 | func (v *validationTreeProof) GetMerkleProofOfMultipleIndexes(r io.ReadSeeker, nodesSize int64, startInd, endInd int) ( 9 | [][][]byte, [][]int, error) { 10 | return v.getMerkleProofOfMultipleIndexes(r, nodesSize, startInd, endInd) 11 | } 12 | -------------------------------------------------------------------------------- /docker.local/sql_init/00-init.sql: -------------------------------------------------------------------------------- 1 | CREATE EXTENSION IF NOT EXISTS "pg_trgm"; 2 | SET log_statement = 'all'; 3 | CREATE DATABASE blobber_meta; 4 | \connect blobber_meta; 5 | CREATE USER blobber_user WITH ENCRYPTED PASSWORD 'blobber'; 6 | GRANT ALL PRIVILEGES ON DATABASE blobber_meta TO blobber_user; 7 | GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO blobber_user; 8 | GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO blobber_user; -------------------------------------------------------------------------------- /docker.local/bin/p0blobber.start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | PWD=`pwd` 3 | BLOBBER_DIR=`basename $PWD` 4 | BLOBBER_ID=`echo my directory $BLOBBER_DIR | sed -e 's/.*\(.\)$/\1/'` 5 | 6 | SSD_PATH="${1:-.}" 7 | HDD_PATH="${2:-.}" 8 | 9 | echo Starting blobber$BLOBBER_ID ... 10 | 11 | # echo blobber$i 12 | 13 | BLOBBER=$BLOBBER_ID SSD_PATH=$PROJECT_ROOT_SSD HDD_PATH=$PROJECT_ROOT_HDD docker-compose -p blobber$BLOBBER_ID -f ../p0docker-compose.yml up -d 14 | -------------------------------------------------------------------------------- /goose/migrations/1718355236_terms_index.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | DELETE FROM terms WHERE allocation_id NOT IN (SELECT id FROM allocations); 4 | ALTER TABLE ONLY terms DROP CONSTRAINT fk_terms_allocation; 5 | ALTER TABLE ONLY terms ADD CONSTRAINT fk_terms_allocation foreign key (allocation_id) references allocations(id) ON DELETE CASCADE; 6 | CREATE INDEX idx_terms_allocation_id ON terms USING HASH(allocation_id); 7 | -- +goose StatementEnd -------------------------------------------------------------------------------- /code/go/0chain.net/blobber/filestore.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" 7 | ) 8 | 9 | func setupFileStore() (err error) { 10 | fmt.Print("> setup file store") 11 | fs := &filestore.FileStore{} 12 | 13 | err = fs.Initialize() 14 | if err != nil { 15 | return 16 | } 17 | 18 | filestore.SetFileStore(fs) 19 | 20 | fmt.Print(" [OK]\n") 21 | return nil 22 | } 23 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/handler/handler_hashnode.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" 5 | ) 6 | 7 | // LoadRootHashnode load root node with its descendant nodes 8 | func LoadRootHashnode(ctx *Context) (interface{}, error) { 9 | 10 | root, err := reference.LoadRootHashnode(ctx, ctx.AllocationId) 11 | if err != nil { 12 | return nil, err 13 | } 14 | return root, nil 15 | } 16 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/mock/allocation.go: -------------------------------------------------------------------------------- 1 | package mock 2 | 3 | import ( 4 | gomocket "github.com/selvatico/go-mocket" 5 | ) 6 | 7 | func MockGetAllocationByID(allocationID string, allocation map[string]interface{}) { 8 | gomocket.Catcher.NewMock(). 9 | WithQuery(`SELECT * FROM "allocations" WHERE "allocations"."tx" = $1 ORDER BY "allocations"."id" LIMIT 1`). 10 | WithArgs(allocationID). 11 | WithReply([]map[string]interface{}{allocation}) 12 | } 13 | -------------------------------------------------------------------------------- /code/go/0chain.net/dev/miner/init.go: -------------------------------------------------------------------------------- 1 | package miner 2 | 3 | import ( 4 | "github.com/gorilla/mux" 5 | ) 6 | 7 | func RegisterHandlers(s *mux.Router) { 8 | // s.HandleFunc("/v1/file/upload/{allocation}", uploadAndUpdateFile).Methods(http.MethodPut, http.MethodPost) 9 | // s.HandleFunc("/v1/file/referencepath/{allocation}", getReference).Methods(http.MethodGet) 10 | // s.HandleFunc("/v1/connection/commit/{allocation}", commitWrite).Methods(http.MethodPost) 11 | } 12 | -------------------------------------------------------------------------------- /code/go/0chain.net/dev/sharder/init.go: -------------------------------------------------------------------------------- 1 | package sharder 2 | 3 | import ( 4 | "github.com/gorilla/mux" 5 | ) 6 | 7 | func RegisterHandlers(s *mux.Router) { 8 | // s.HandleFunc("/v1/file/upload/{allocation}", uploadAndUpdateFile).Methods(http.MethodPut, http.MethodPost) 9 | // s.HandleFunc("/v1/file/referencepath/{allocation}", getReference).Methods(http.MethodGet) 10 | // s.HandleFunc("/v1/connection/commit/{allocation}", commitWrite).Methods(http.MethodPost) 11 | } 12 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/datastore/blob_type.go: -------------------------------------------------------------------------------- 1 | package datastore 2 | 3 | import ( 4 | "database/sql/driver" 5 | "errors" 6 | ) 7 | 8 | type JSONString []byte 9 | 10 | func (data *JSONString) Scan(value interface{}) error { 11 | if b, ok := value.(string); ok { 12 | *data = []byte(b) 13 | return nil 14 | } 15 | return errors.New("String expected") 16 | } 17 | 18 | func (data JSONString) Value() (driver.Value, error) { 19 | return string(data), nil 20 | } 21 | -------------------------------------------------------------------------------- /generate_migration.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | MIGRATION_DIRECTORY="goose/migrations" 3 | ts=$(date +%s) 4 | fname=$1 5 | 6 | if [ -z "$fname" ]; then 7 | echo "Usage: $0 " 8 | exit 1 9 | fi 10 | 11 | 12 | # Write boilerplate of the migration file 13 | echo "-- +goose Up 14 | -- +goose StatementBegin 15 | 16 | -- +goose StatementEnd 17 | 18 | -- +goose Down 19 | -- +goose StatementBegin 20 | 21 | -- +goose StatementEnd" > "$MIGRATION_DIRECTORY/""$ts""_$fname.sql" -------------------------------------------------------------------------------- /goose/migrations/1730098482_session_key.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | ALTER TABLE allocations ADD COLUMN owner_signing_public_key character varying(512); 4 | 5 | ALTER TABLE reference_objects ADD COLUMN signature_version smallint; 6 | 7 | ALTER TABLE reference_objects ALTER COLUMN actual_file_hash_signature TYPE character varying(128); 8 | 9 | ALTER TABLE reference_objects ALTER COLUMN validation_root_signature TYPE character varying(128); 10 | 11 | -- +goose StatementEnd -------------------------------------------------------------------------------- /goose/migrator.go: -------------------------------------------------------------------------------- 1 | package goose 2 | 3 | import ( 4 | "database/sql" 5 | "embed" 6 | 7 | "github.com/pressly/goose/v3" 8 | ) 9 | 10 | //go:embed migrations/*.sql 11 | var embedMigrations embed.FS 12 | 13 | func Init() { 14 | goose.SetBaseFS(embedMigrations) 15 | 16 | if err := goose.SetDialect("postgres"); err != nil { 17 | panic(err) 18 | } 19 | } 20 | 21 | func Migrate(db *sql.DB) { 22 | if err := goose.Up(db, "migrations"); err != nil { 23 | panic(err) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/datastore/memory.go: -------------------------------------------------------------------------------- 1 | package datastore 2 | 3 | import ( 4 | "gorm.io/driver/sqlite" 5 | "gorm.io/gorm" 6 | ) 7 | 8 | // UseInMemory set the DB instance to an in-memory DB using SQLite. 9 | func UseInMemory() (*gorm.DB, error) { 10 | gdb, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{}) 11 | if err != nil { 12 | return nil, err 13 | } 14 | 15 | instance = &postgresStore{ 16 | db: gdb, 17 | } 18 | 19 | return gdb, nil 20 | } 21 | -------------------------------------------------------------------------------- /docs/src/share_revoke.plantuml: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | title Buyer revokes share for client 4 | actor Seller 5 | 6 | Seller -> Seller : Revoke share using **client id**,\n\ 7 | **allocation id** and **remote path** 8 | Seller --> Blobber : Revoke share using provided parameters 9 | Blobber --> Blobber : Find path share in **marketplace_share_info** \n\ 10 | table and set **revoked** flag to true 11 | Seller --> Seller : return consensus status, \n\ 12 | we need 100% consensus for success 13 | @enduml 14 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/writemarker/entity_test.go: -------------------------------------------------------------------------------- 1 | package writemarker 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestWriteMarkerEntity_OnChain(t *testing.T) { 9 | assert.False(t, (&WriteMarkerEntity{Status: Accepted}).OnChain()) 10 | assert.False(t, (&WriteMarkerEntity{Status: Failed}).OnChain()) 11 | assert.True(t, (&WriteMarkerEntity{Status: Committed}).OnChain()) 12 | assert.False(t, (&WriteMarkerEntity{}).OnChain()) // unspecified 13 | } 14 | -------------------------------------------------------------------------------- /code/go/0chain.net/conductor/conductrpc/server.go: -------------------------------------------------------------------------------- 1 | package conductrpc 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/0chain/blobber/code/go/0chain.net/conductor/config" 7 | ) 8 | 9 | var ErrShutdown = errors.New("server shutdown") 10 | 11 | // type aliases 12 | type ( 13 | NodeID = config.NodeID 14 | NodeName = config.NodeName 15 | Round = config.Round 16 | RoundName = config.RoundName 17 | Number = config.Number 18 | ) 19 | 20 | type ValidtorTicket struct { 21 | ValidatorId string 22 | } 23 | -------------------------------------------------------------------------------- /.github/workflows/automate-add-issue-in-project.yaml: -------------------------------------------------------------------------------- 1 | name: Add Issue to Backend Issues Project Board 2 | 3 | on: 4 | issues: 5 | types: 6 | - opened 7 | 8 | jobs: 9 | add-to-project: 10 | name: Add Issue to Backend Issues Project Board 11 | runs-on: arc-runner 12 | steps: 13 | - uses: actions/add-to-project@v0.4.0 14 | with: 15 | project-url: https://github.com/orgs/0chain/projects/${{ secrets.PROJECT_NUMBER }} 16 | github-token: ${{ secrets.SVC_ACCOUNT_SECRET }} -------------------------------------------------------------------------------- /docker.local/bin/build.blobber.dev.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | GIT_COMMIT=$(git rev-list -1 HEAD) 5 | echo $GIT_COMMIT 6 | 7 | docker build --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/ValidatorDockerfile.dev .. -t validator 8 | docker build --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/Dockerfile.dev .. -t blobber 9 | 10 | for i in $(seq 1 6); 11 | do 12 | BLOBBER=$i docker-compose -p blobber$i -f docker.local/docker-compose.yml build --force-rm 13 | done 14 | 15 | docker.local/bin/sync_clock.sh 16 | -------------------------------------------------------------------------------- /code/go/0chain.net/validatorcore/storage/challenge_handler_main.go: -------------------------------------------------------------------------------- 1 | //go:build !integration_tests 2 | // +build !integration_tests 3 | 4 | package storage 5 | 6 | import ( 7 | "context" 8 | "net/http" 9 | ) 10 | 11 | func ChallengeHandler(ctx context.Context, r *http.Request) (interface{}, error) { 12 | res, err := challengeHandler(ctx, r) 13 | 14 | if len(Last5Transactions) >= 5 { 15 | Last5Transactions = Last5Transactions[1:] 16 | } 17 | Last5Transactions = append(Last5Transactions, res) 18 | 19 | return res, err 20 | } 21 | -------------------------------------------------------------------------------- /docker.local/bin/blobber.start_bls.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | PWD=`pwd` 5 | BLOBBER_DIR=`basename $PWD` 6 | BLOBBER_ID=`echo my directory $BLOBBER_DIR | sed -e 's/.*\(.\)$/\1/'` 7 | 8 | if [[ "$*" == *"--debug"* ]] 9 | then 10 | echo Starting blobber$BLOBBER_ID in debug mode... 11 | BLOBBER=$BLOBBER_ID docker-compose -p blobber$BLOBBER_ID -f ../b0docker-compose-debug.yml up 12 | else 13 | echo Starting blobber$BLOBBER_ID ... 14 | BLOBBER=$BLOBBER_ID docker-compose -p blobber$BLOBBER_ID -f ../b0docker-compose.yml up 15 | fi -------------------------------------------------------------------------------- /docs/src/share.plantuml: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | title Buyer shares encrypted file 4 | actor Seller 5 | actor Buyer 6 | 7 | Buyer -> Seller: Client Id, encryption public key 8 | 9 | Seller -> Seller : Generate auth ticket for file / folder using \n\ 10 | buyer client id as well as encryption public key 11 | Seller --> Blobber : Upload auth ticket 12 | Blobber --> Blobber : Save auth ticket to **marketplace_share_info** \n\ 13 | table 14 | Seller --> Seller : Check upload consensus status 15 | Seller --> Seller : Return auth ticket 16 | 17 | @enduml 18 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/stats/infrastats.go: -------------------------------------------------------------------------------- 1 | package stats 2 | 3 | type contextKey string 4 | 5 | func (c contextKey) String() string { 6 | return string(c) 7 | } 8 | 9 | const ( 10 | HealthDataKey = contextKey("health") 11 | FailedChallengeRequestDataKey = contextKey("fcrd") 12 | AllocationListRequestDataKey = contextKey("alrd") 13 | ) 14 | 15 | type InfraStats struct { 16 | CPUs int 17 | NumberOfGoroutines int 18 | HeapAlloc int64 19 | HeapSys int64 20 | ActiveOnChain string 21 | } 22 | -------------------------------------------------------------------------------- /code/go/0chain.net/core/node/context.go: -------------------------------------------------------------------------------- 1 | package node 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/0chain/gosdk/constants" 7 | ) 8 | 9 | const SELF_NODE constants.ContextKey = "SELF_NODE" 10 | 11 | /*GetNodeContext - setup a context with the self node */ 12 | func GetNodeContext() context.Context { 13 | return context.WithValue(context.Background(), SELF_NODE, Self) 14 | } 15 | 16 | /*GetSelfNode - given a context, return the self node associated with it */ 17 | func GetSelfNode(ctx context.Context) *SelfNode { 18 | return ctx.Value(SELF_NODE).(*SelfNode) 19 | } 20 | -------------------------------------------------------------------------------- /docker.local/docker-clean/docker-clean-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | docker-clean: 4 | build: 5 | context: . 6 | dockerfile: Dockerfile 7 | volumes: 8 | # 1-6 blobbers 9 | - ../blobber1/:/blobber1 10 | - ../blobber2/:/blobber2 11 | - ../blobber3/:/blobber3 12 | - ../blobber4/:/blobber4 13 | - ../blobber5/:/blobber5 14 | - ../blobber6/:/blobber6 15 | command: /bin/sh docker-clean.sh 16 | 17 | volumes: 18 | blobber1: 19 | blobber2: 20 | blobber3: 21 | blobber4: 22 | blobber5: 23 | blobber6: 24 | -------------------------------------------------------------------------------- /docs/src/proxy_encryption_download_flow.plantuml: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | actor Client 4 | 5 | 6 | loop till all of requested file chunks are completely downloaded 7 | Client -> Blobber : Request the file in range of blocks x..y 8 | Blobber --> Database : Get reencryption key from postgres db table \n\ 9 | for the client 10 | Blobber --> Blobber : Iterate through all chunks and \n\ 11 | encrypt every chunk using re-encryption key 12 | Blobber --> Client : Return Re-encrypted message 13 | Client --> Client : Re-decrypt the received message using private key 14 | end 15 | 16 | @enduml 17 | -------------------------------------------------------------------------------- /docker.local/keys_config/b0bnode1_keys.txt.json: -------------------------------------------------------------------------------- 1 | { 2 | "client_id": "f65af5d64000c7cd2883f4910eb69086f9d6e6635c744e62afcfab58b938ee25", 3 | "client_key": "de52c0a51872d5d2ec04dbc15a6f0696cba22657b80520e1d070e72de64c9b04e19ce3223cae3c743a20184158457582ffe9c369ca9218c04bfe83a26a62d88d", 4 | "keys": [ 5 | { 6 | "public_key": "de52c0a51872d5d2ec04dbc15a6f0696cba22657b80520e1d070e72de64c9b04e19ce3223cae3c743a20184158457582ffe9c369ca9218c04bfe83a26a62d88d", 7 | "private_key": "17fa2ab0fb49249cb46dbc13e4e9e6853af8b1506e48d84c03e5e92f6348bb1d" 8 | } 9 | ], 10 | "version": "1.0" 11 | } 12 | -------------------------------------------------------------------------------- /docker.local/keys_config/b0bnode2_keys.txt.json: -------------------------------------------------------------------------------- 1 | { 2 | "client_id": "7a90e6790bcd3d78422d7a230390edc102870fe58c15472073922024985b1c7d", 3 | "client_key": "e4dc5262ed8e20583e3293f358cc21aa77c2308ea773bab8913670ffeb5aa30d7e2effbce51f323b5b228ad01f71dc587b923e4aab7663a573ece5506f2e3b0e", 4 | "keys": [ 5 | { 6 | "public_key": "e4dc5262ed8e20583e3293f358cc21aa77c2308ea773bab8913670ffeb5aa30d7e2effbce51f323b5b228ad01f71dc587b923e4aab7663a573ece5506f2e3b0e", 7 | "private_key": "4b6d9c5f7b0386e36b212324ea52f5ff17a9ed1338ca901d7f7fa7637159a912" 8 | } 9 | ], 10 | "version": "1.0" 11 | } 12 | -------------------------------------------------------------------------------- /docker.local/keys_config/b0bnode3_keys.txt.json: -------------------------------------------------------------------------------- 1 | { 2 | "client_id": "2f051ca6447d8712a020213672bece683dbd0d23a81fdf93ff273043a0764d18", 3 | "client_key": "a2670e307b6a0647e63e5d99be77c71158811ba76ae3b1b0e1a179068f720315b1e6a3ed5d538748661f3dd921e2221b7b219ecd49635c27fb566b2179882712", 4 | "keys": [ 5 | { 6 | "public_key": "a2670e307b6a0647e63e5d99be77c71158811ba76ae3b1b0e1a179068f720315b1e6a3ed5d538748661f3dd921e2221b7b219ecd49635c27fb566b2179882712", 7 | "private_key": "f306c2bfc07509ee653e548171e3be3eb1c2a7bc344b30128e82c99694370006" 8 | } 9 | ], 10 | "version": "1.0" 11 | } 12 | -------------------------------------------------------------------------------- /docker.local/keys_config/b0bnode4_keys.txt.json: -------------------------------------------------------------------------------- 1 | { 2 | "client_id": "2a4d5a5c6c0976873f426128d2ff23a060ee715bccf0fd3ca5e987d57f25b78e", 3 | "client_key": "1a97fb9074d9b46d99104bef4beebb56d1a3c47b14f7f87399219a32c2ea7609deaef7d63b33f7e7498df8e39efdb69e93f7b14266be626abe34ea4c5f869c19", 4 | "keys": [ 5 | { 6 | "public_key": "1a97fb9074d9b46d99104bef4beebb56d1a3c47b14f7f87399219a32c2ea7609deaef7d63b33f7e7498df8e39efdb69e93f7b14266be626abe34ea4c5f869c19", 7 | "private_key": "81f34514f462e6348e79c9c37c150ebeb63b080f682dba8173115582540e3922" 8 | } 9 | ], 10 | "version": "1.0" 11 | } 12 | -------------------------------------------------------------------------------- /docker.local/keys_config/b0bnode5_keys.txt.json: -------------------------------------------------------------------------------- 1 | { 2 | "client_id": "429c05b461f60a8db4329750810d3d40a49c0b4d95dd414bac1926f263321fe0", 3 | "client_key": "6e1e0c5a44bd1ca958b73e4b3f778362b6965be7b5f1aadeef84d8408bbeda0628f7bf906aa12e43195cd52d98db6c328460564d8581d940bc561297d0c81015", 4 | "keys": [ 5 | { 6 | "public_key": "6e1e0c5a44bd1ca958b73e4b3f778362b6965be7b5f1aadeef84d8408bbeda0628f7bf906aa12e43195cd52d98db6c328460564d8581d940bc561297d0c81015", 7 | "private_key": "ef9a9ac556d58ede0cc9f50bee0a6e32de2c338bf23ef9d43b2ea8cd04e9c80a" 8 | } 9 | ], 10 | "version": "1.0" 11 | } 12 | -------------------------------------------------------------------------------- /docker.local/keys_config/b0bnode6_keys.txt.json: -------------------------------------------------------------------------------- 1 | { 2 | "client_id": "f34e784da180ac6097bdcd243548f6c192599968c93e0cbf4c565d78cbcf8933", 3 | "client_key": "ee5f1b9d3bd74c8ab9ea4f045a3500d3678f37aa0fc14181445c8286ccf80d0e1d0310ce834313fde2345a3006e432939e7009c7f0c1a233005bdbbf7d20b88a", 4 | "keys": [ 5 | { 6 | "public_key": "ee5f1b9d3bd74c8ab9ea4f045a3500d3678f37aa0fc14181445c8286ccf80d0e1d0310ce834313fde2345a3006e432939e7009c7f0c1a233005bdbbf7d20b88a", 7 | "private_key": "5d519368d416a5a186e8681e07b72cec19039ff1519c210db8c94477425e8915" 8 | } 9 | ], 10 | "version": "1.0" 11 | } 12 | -------------------------------------------------------------------------------- /docker.local/keys_config/b0vnode1_keys.txt.json: -------------------------------------------------------------------------------- 1 | { 2 | "client_id": "41313b795d2c057b6277801e9ed277b444770c2af75f5209afd00bd07c72cc0b", 3 | "client_key": "0a82a096ff14334bd78f061d451c96913cef8741742e1bc2af8e0276dbe7641926e926629597b0d39c1cbb31b63390cfae21e6f357c25a1b9dc53d0d757afe8a", 4 | "keys": [ 5 | { 6 | "public_key": "0a82a096ff14334bd78f061d451c96913cef8741742e1bc2af8e0276dbe7641926e926629597b0d39c1cbb31b63390cfae21e6f357c25a1b9dc53d0d757afe8a", 7 | "private_key": "f94d9ed96484e2bec0772d2187d488a42af4ef489ed697a4d603e9ce8d5b771b" 8 | } 9 | ], 10 | "version": "1.0" 11 | } 12 | -------------------------------------------------------------------------------- /docker.local/keys_config/b0vnode2_keys.txt.json: -------------------------------------------------------------------------------- 1 | { 2 | "client_id": "ab549edb7cea822dab0b460f65dcde85f698c1e97d730e3ffc6b0f8b576b65bd", 3 | "client_key": "0a47b0e582f70aff30c6143e03462b52b389c67dcc73feb4e6e6c58a71290c03d00d548967ed33254e93817e68d8a2bbfc5a35fa09b61c50e32503fee36c06a1", 4 | "keys": [ 5 | { 6 | "public_key": "0a47b0e582f70aff30c6143e03462b52b389c67dcc73feb4e6e6c58a71290c03d00d548967ed33254e93817e68d8a2bbfc5a35fa09b61c50e32503fee36c06a1", 7 | "private_key": "dca0e71419aadf266f41f2d5569fbf6174010fe5453433c7569e8ce9d7d5e113" 8 | } 9 | ], 10 | "version": "1.0" 11 | } 12 | -------------------------------------------------------------------------------- /docker.local/keys_config/b0vnode3_keys.txt.json: -------------------------------------------------------------------------------- 1 | { 2 | "client_id": "86cf791f03f01e3e4d318b1ca009a51c91dd43f7cf3c87a32f531b609cc5044b", 3 | "client_key": "daab4f68dcd5e68ef03465e17c05bc9ba4a669361a511c2be96ca2b6b5faad0d6b03184ecd59a5c6a6aab74ee878b34bd309a7324ea3dca18153fa2f1962f60d", 4 | "keys": [ 5 | { 6 | "public_key": "daab4f68dcd5e68ef03465e17c05bc9ba4a669361a511c2be96ca2b6b5faad0d6b03184ecd59a5c6a6aab74ee878b34bd309a7324ea3dca18153fa2f1962f60d", 7 | "private_key": "546b31cd186094d6b079bf10d8b68b3126f1722b5088817703977d30c1e37a0d" 8 | } 9 | ], 10 | "version": "1.0" 11 | } 12 | -------------------------------------------------------------------------------- /docker.local/keys_config/b0vnode4_keys.txt.json: -------------------------------------------------------------------------------- 1 | { 2 | "client_id": "823cb45de27dfe739b320dcf6449e5fdea35c60804fd81d6f22c005042cfb337", 3 | "client_key": "a76736a941846763a774bdbd0e63ad38ed229b3fd392112041daeefa33cc120c4566c3929ba3938a6f8fce4b89197a1b1083facf6db091a43756bf117ee2d687", 4 | "keys": [ 5 | { 6 | "public_key": "a76736a941846763a774bdbd0e63ad38ed229b3fd392112041daeefa33cc120c4566c3929ba3938a6f8fce4b89197a1b1083facf6db091a43756bf117ee2d687", 7 | "private_key": "808cc0a5ba55c505914fa03e09562058c941cf75dcf16ea800fa71798817001a" 8 | } 9 | ], 10 | "version": "1.0" 11 | } 12 | -------------------------------------------------------------------------------- /docker.local/keys_config/b0vnode5_keys.txt.json: -------------------------------------------------------------------------------- 1 | { 2 | "client_id": "0c1748cb83e7417a1399b334efcf325fcfedce66ba57123e581627a8e681fb95", 3 | "client_key": "2adaa464eedc053716d36ebcc43fd7c653829ef6eee970fa6069c00b43a57f163d9e3170b132bb96f8c8cde31861c3fa3b41cb73b41f4f7ec863eab0aab6348c", 4 | "keys": [ 5 | { 6 | "public_key": "2adaa464eedc053716d36ebcc43fd7c653829ef6eee970fa6069c00b43a57f163d9e3170b132bb96f8c8cde31861c3fa3b41cb73b41f4f7ec863eab0aab6348c", 7 | "private_key": "92847e81af38f35b0a4152803eda38b540cfa714c2772b1346b9c7fb74a9d322" 8 | } 9 | ], 10 | "version": "1.0" 11 | } 12 | -------------------------------------------------------------------------------- /docker.local/keys_config/b0vnode6_keys.txt.json: -------------------------------------------------------------------------------- 1 | { 2 | "client_id": "109a99cbaff4b180abd9cace80afc27946e9a64030197914e61b2a2a04e3c02f", 3 | "client_key": "b6bea180d777d7a11b15d1eddd94d705a49286f437b4d17b9d97ebaebf481a1e69862f92d9695c3b573ee3ae5854f343f3f22790a3c5fbab55ea202a84fa6881", 4 | "keys": [ 5 | { 6 | "public_key": "b6bea180d777d7a11b15d1eddd94d705a49286f437b4d17b9d97ebaebf481a1e69862f92d9695c3b573ee3ae5854f343f3f22790a3c5fbab55ea202a84fa6881", 7 | "private_key": "208ad5bcbd6523ba7b185d8f09a9fd32f7759d5e99a7dcd84599b972abb22d01" 8 | } 9 | ], 10 | "version": "1.0" 11 | } 12 | -------------------------------------------------------------------------------- /code/go/0chain.net/core/common/blobber_registered.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import "sync" 4 | 5 | // Private variables 6 | var ( 7 | blobberRegisteredMutex sync.Mutex 8 | blobberRegistered bool 9 | ) 10 | 11 | // Getter for blobberRegistered 12 | func IsBlobberRegistered() bool { 13 | blobberRegisteredMutex.Lock() 14 | defer blobberRegisteredMutex.Unlock() 15 | return blobberRegistered 16 | } 17 | 18 | // Setter for blobberRegistered 19 | func SetBlobberRegistered(registered bool) { 20 | blobberRegisteredMutex.Lock() 21 | defer blobberRegisteredMutex.Unlock() 22 | blobberRegistered = registered 23 | } 24 | -------------------------------------------------------------------------------- /buf.yaml: -------------------------------------------------------------------------------- 1 | version: v1beta1 2 | name: buf.build/0chain/blobber 3 | deps: 4 | - buf.build/beta/googleapis 5 | build: 6 | roots: 7 | - code/go/0chain.net/blobbercore/blobbergrpc/proto 8 | lint: 9 | use: 10 | - DEFAULT 11 | enum_zero_value_suffix: _UNSPECIFIED 12 | rpc_allow_same_request_response: false 13 | rpc_allow_google_protobuf_empty_requests: false 14 | rpc_allow_google_protobuf_empty_responses: false 15 | service_suffix: Service 16 | ignore: 17 | - google/api 18 | except: 19 | - PACKAGE_AFFINITY 20 | - FILE_LAYOUT 21 | - PACKAGE_VERSION_SUFFIX 22 | breaking: 23 | use: 24 | - FILE -------------------------------------------------------------------------------- /bin/postgres-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | psql=( psql --username "$POSTGRES_USER" --port "$POSTGRES_PORT" --host "$POSTGRES_HOST" ) 6 | 7 | until pg_isready -h $POSTGRES_HOST 8 | do 9 | echo "Sleep 1s and try again..." 10 | sleep 1 11 | done 12 | 13 | export PGPASSWORD="$POSTGRES_PASSWORD" 14 | 15 | for f in /blobber/sql/*; do 16 | case "$f" in 17 | *.sh) echo "$0: running $f"; . "$f" ;; 18 | *.sql) echo "$0: running $f"; "${psql[@]}" -f "$f"; echo ;; 19 | *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "${psql[@]}"; echo ;; 20 | *) echo "$0: ignoring $f" ;; 21 | esac 22 | echo 23 | done 24 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/allocation/renamefilechange_main.go: -------------------------------------------------------------------------------- 1 | //go:build !integration_tests 2 | // +build !integration_tests 3 | 4 | package allocation 5 | 6 | import ( 7 | "context" 8 | 9 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" 10 | "github.com/0chain/blobber/code/go/0chain.net/core/common" 11 | ) 12 | 13 | func (rf *RenameFileChange) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, 14 | allocationRoot string, ts common.Timestamp, _ map[string]string) (*reference.Ref, error) { 15 | 16 | return rf.applyChange(ctx, rootRef, change, allocationRoot, ts, nil) 17 | } 18 | -------------------------------------------------------------------------------- /code/go/0chain.net/core/transaction/vars.go: -------------------------------------------------------------------------------- 1 | package transaction 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | var ( 8 | 9 | // MinConfirmation minial confirmation from sharders 10 | MinConfirmation = 50 11 | ) 12 | 13 | var ( 14 | // ErrBadRequest bad request from sharder 15 | ErrBadRequest = errors.New("[sharder]bad request") 16 | 17 | // ErrNoAvailableSharder no any available sharder 18 | ErrNoAvailableSharder = errors.New("[txn] there is no any available sharder") 19 | 20 | // ErrTooLessConfirmation too less sharder to confirm transaction 21 | ErrTooLessConfirmation = errors.New("[txn] too less sharders to confirm it") 22 | ) 23 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go: -------------------------------------------------------------------------------- 1 | //go:build !integration_tests 2 | // +build !integration_tests 3 | 4 | package allocation 5 | 6 | import ( 7 | "context" 8 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" 9 | "github.com/0chain/blobber/code/go/0chain.net/core/common" 10 | ) 11 | 12 | func (nf *UploadFileChanger) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, 13 | allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) { 14 | return nf.applyChange(ctx, rootRef, change, allocationRoot, ts, fileIDMeta) 15 | } 16 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobber/logging.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" 7 | "github.com/0chain/blobber/code/go/0chain.net/core/logging" 8 | "github.com/0chain/gosdk/zcncore" 9 | ) 10 | 11 | func setupLogging() { 12 | fmt.Print("> init logging") 13 | 14 | if config.Development() { 15 | logging.InitLogging("development", logDir, "0chainBlobber.log") 16 | } else { 17 | logging.InitLogging("production", logDir, "0chainBlobber.log") 18 | } 19 | 20 | zcncore.SetLogFile(logDir+"/0chainBlobber-sdk.log", false) 21 | zcncore.SetLogLevel(3) 22 | fmt.Print(" [OK]\n") 23 | } 24 | -------------------------------------------------------------------------------- /code/go/0chain.net/validatorcore/storage/context.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | 7 | "github.com/0chain/blobber/code/go/0chain.net/core/common" 8 | "github.com/0chain/gosdk/constants" 9 | ) 10 | 11 | func SetupContext(handler common.JSONResponderF) common.JSONResponderF { 12 | return func(ctx context.Context, r *http.Request) (interface{}, error) { 13 | ctx = context.WithValue(ctx, constants.ContextKeyClient, r.Header.Get(common.ClientHeader)) 14 | ctx = context.WithValue(ctx, constants.ContextKeyClientKey, 15 | r.Header.Get(common.ClientKeyHeader)) 16 | res, err := handler(ctx, r) 17 | return res, err 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "gomod" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "daily" 12 | open-pull-requests-limit: 10 13 | reviewers: 14 | - cnlangzi 15 | labels: 16 | - pr/dependencies 17 | -------------------------------------------------------------------------------- /code/go/0chain.net/core/transaction/type.go: -------------------------------------------------------------------------------- 1 | package transaction 2 | 3 | const ( 4 | TxnTypeSend = 0 // A transaction to send tokens to another account, state is maintained by account 5 | 6 | TxnTypeLockIn = 2 // A transaction to lock tokens, state is maintained on the account and the parent lock in transaction 7 | 8 | // Any txn type that refers to a parent txn should have an odd value 9 | TxnTypeStorageWrite = 101 // A transaction to write data to the blobber 10 | TxnTypeStorageRead = 103 // A transaction to read data from the blobber 11 | 12 | TxnTypeData = 10 // A transaction to just store a piece of data on the block chain 13 | 14 | TxnTypeSmartContract = 1000 // A smart contract transaction type 15 | ) 16 | -------------------------------------------------------------------------------- /PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### Changes 2 | 3 | 4 | ### Fixes 5 | 6 | 7 | 8 | ### Tests 9 | Tasks to complete before merging PR: 10 | - [ ] Ensure system tests are passing. If not [Run them manually](https://github.com/0chain/blobber/actions/workflows/system_tests.yml) to check for any regressions :clipboard: 11 | - [ ] Do any new system tests need added to test this change? do any existing system tests need updated? If so create a PR at [0chain/system_test](https://github.com/0chain/system_test) 12 | - [ ] Merge your system tests PR to master AFTER merging this PR 13 | 14 | ### Associated PRs (Link as appropriate): 15 | - 0chain: 16 | - gosdk: 17 | - system_test: 18 | - zboxcli: 19 | - zwalletcli: 20 | - Other: ... 21 | -------------------------------------------------------------------------------- /.github/workflows/benchmark.yml: -------------------------------------------------------------------------------- 1 | name: Benchmark 2 | 3 | concurrency: 4 | group: "benchmark-${{ github.ref }}" 5 | cancel-in-progress: true 6 | 7 | on: 8 | push: 9 | branches: [ master, staging ] 10 | 11 | jobs: 12 | benchmark: 13 | runs-on: [self-hosted, conductor-test] 14 | steps: 15 | - name: Setup go 16 | uses: actions/setup-go@v3 17 | with: 18 | go-version: ^1.20 # The Go version to download (if necessary) and use. 19 | 20 | - name: Clone blobber 21 | uses: actions/checkout@v2 22 | 23 | - name: Run benchmark 24 | run: CGO_ENABLED=1 go test -tags bn256 -benchmem -run="BenchmarkUploadFile*" -bench="BenchmarkUploadFile*" ./... | tee benchmark.txt -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/writemarker/writemarker.go: -------------------------------------------------------------------------------- 1 | package writemarker 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" 7 | "github.com/0chain/blobber/code/go/0chain.net/core/logging" 8 | ) 9 | 10 | func startRedeemWorker(ctx context.Context) { 11 | logging.Logger.Info("Starting redeem worker") 12 | for i := 0; i < config.Configuration.WMRedeemNumWorkers; i++ { 13 | go redeemWorker(ctx) 14 | } 15 | } 16 | 17 | func redeemWorker(ctx context.Context) { 18 | for { 19 | select { 20 | case <-ctx.Done(): 21 | logging.Logger.Info("Stopping redeem worker") 22 | return 23 | case dm := <-writeMarkerChan: 24 | _ = redeemWriteMarker(dm) 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobber/prepare_conductor_testing.go: -------------------------------------------------------------------------------- 1 | //go:build integration_tests 2 | // +build integration_tests 3 | 4 | // Integration tests is also called conductor tests. 5 | // TODO. There seems to be missing setup for conductor as only prepareBlobber() is hooked. 6 | 7 | package main 8 | 9 | import ( 10 | "github.com/0chain/blobber/code/go/0chain.net/core/logging" 11 | 12 | crpc "github.com/0chain/blobber/code/go/0chain.net/conductor/conductrpc" // integration tests 13 | ) 14 | 15 | // start lock, where the miner is ready to connect to blockchain (BC) 16 | func prepareBlobber(id string) { 17 | logging.Logger.Info("integration tests") 18 | crpc.Init(id) 19 | } 20 | 21 | func prepareBlobberShutdown() { 22 | crpc.Shutdown() 23 | } 24 | -------------------------------------------------------------------------------- /docker.local/b0docker-compose-github.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | postgres: 4 | image: postgres:14 5 | environment: 6 | POSTGRES_PORT: 5432 7 | POSTGRES_HOST: postgres 8 | POSTGRES_USER: postgres 9 | POSTGRES_HOST_AUTH_METHOD: trust 10 | ports: 11 | - 5432:5432 12 | volumes: 13 | - ./blobber${BLOBBER}/data/postgresql:/var/lib/postgresql/data 14 | - ./sql_init:/docker-entrypoint-initdb.d 15 | networks: 16 | default: 17 | 18 | networks: 19 | default: 20 | driver: bridge 21 | testnet0: 22 | external: true 23 | # driver: bridge 24 | # ipam: 25 | # config: 26 | # - subnet: 198.18.0.0/15 27 | # - gateway: 198.18.0.255 28 | 29 | 30 | volumes: 31 | data: 32 | config: 33 | bin: 34 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobber/grpc.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // func startGRPCServer() { 4 | // r := mux.NewRouter() 5 | // initHandlers(r) 6 | 7 | // grpcServer := handler.NewGRPCServerWithMiddlewares(r) 8 | // reflection.Register(grpcServer) 9 | 10 | // if grpcPort <= 0 { 11 | // logging.Logger.Error("grpc port missing") 12 | // panic(errors.New("grpc port missing")) 13 | // } 14 | 15 | // logging.Logger.Info("started grpc server on to grpc requests on port - " + strconv.Itoa(grpcPort)) 16 | 17 | // lis, err := net.Listen("tcp", fmt.Sprintf(":%v", grpcPort)) 18 | // if err != nil { 19 | // log.Fatalf("failed to listen: %v", err) 20 | // panic(err) 21 | // } 22 | 23 | // fmt.Print("> starting grpc server [OK]\n") 24 | 25 | // log.Fatal(grpcServer.Serve(lis)) 26 | 27 | // } 28 | -------------------------------------------------------------------------------- /code/go/0chain.net/core/cache/lfu.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "github.com/koding/cache" 5 | ) 6 | 7 | type LFU struct { 8 | Cache cache.Cache 9 | } 10 | 11 | // NewLFUCache - create a new LFU cache object 12 | func NewLFUCache(size int) *LFU { 13 | c := &LFU{} 14 | c.Cache = cache.NewLFU(size) 15 | return c 16 | } 17 | 18 | // Add - add a given key and value 19 | func (c *LFU) Add(key string, value interface{}) error { 20 | return c.Cache.Set(key, value) 21 | } 22 | 23 | // Get - get the value associated with the key 24 | func (c *LFU) Get(key string) (interface{}, error) { 25 | value, err := c.Cache.Get(key) 26 | if err != nil { 27 | return nil, err 28 | } 29 | return value, err 30 | } 31 | 32 | func (c *LFU) Delete(key string) error { 33 | return c.Cache.Delete(key) 34 | } 35 | -------------------------------------------------------------------------------- /code/go/0chain.net/core/common/lookup.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | /*Lookup - used to track a code, value pair. code is used in the program and value is user friendly label */ 4 | type Lookup struct { 5 | Code string `json:"code"` 6 | Value string `json:"value"` 7 | } 8 | 9 | /*GetCode - get the code */ 10 | func (l *Lookup) GetCode() string { 11 | return l.Code 12 | } 13 | 14 | /*GetValue - get the value */ 15 | func (l *Lookup) GetValue() string { 16 | return l.Value 17 | } 18 | 19 | /*CreateLookups - given a code,value args, return an array of lookups */ 20 | func CreateLookups(arg ...string) []*Lookup { 21 | lookups := make([]*Lookup, 0, len(arg)/2) 22 | for i := 0; i < len(arg); i += 2 { 23 | lookups = append(lookups, &Lookup{Code: arg[i], Value: arg[i+1]}) 24 | } 25 | return lookups 26 | } 27 | -------------------------------------------------------------------------------- /code/go/0chain.net/core/cache/lru.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | lru "github.com/koding/cache" 5 | ) 6 | 7 | // LRU - LRU cache 8 | type LRU struct { 9 | Cache lru.Cache 10 | } 11 | 12 | // NewLRUCache - create a new LRU cache 13 | func NewLRUCache(size int) *LRU { 14 | c := &LRU{} 15 | c.Cache = lru.NewLRU(size) 16 | return c 17 | } 18 | 19 | // Add - add a key and a value 20 | func (c *LRU) Add(key string, value interface{}) error { 21 | return c.Cache.Set(key, value) 22 | } 23 | 24 | // Get - get the value associated with the key 25 | func (c *LRU) Get(key string) (interface{}, error) { 26 | value, err := c.Cache.Get(key) 27 | if err != nil { 28 | return nil, err 29 | } 30 | return value, nil 31 | } 32 | 33 | func (c *LRU) Delete(key string) error { 34 | return c.Cache.Delete(key) 35 | } 36 | -------------------------------------------------------------------------------- /code/go/0chain.net/core/lock/lock_test.go: -------------------------------------------------------------------------------- 1 | package lock 2 | 3 | import ( 4 | "strconv" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestLock(t *testing.T) { 11 | max := 100 12 | 13 | for i := 0; i < max; i++ { 14 | lock1 := GetMutex("testlock", strconv.Itoa(i)) 15 | 16 | lock1.Lock() 17 | 18 | require.Equal(t, 1, lock1.usedby) 19 | 20 | lock1.Unlock() 21 | 22 | require.Equal(t, 0, lock1.usedby) 23 | 24 | lock2 := GetMutex("testlock", strconv.Itoa(i)) 25 | lock2.Lock() 26 | 27 | require.Equal(t, 1, lock2.usedby) 28 | 29 | lock2.Unlock() 30 | 31 | require.Equal(t, 0, lock2.usedby) 32 | } 33 | 34 | cleanUnusedMutexs() 35 | 36 | for i := 0; i < max; i++ { 37 | _, ok := lockPool["testlock:"+strconv.Itoa(i)] 38 | require.Equal(t, false, ok) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /code/go/0chain.net/conductor/conductrpc/resolve.go: -------------------------------------------------------------------------------- 1 | package conductrpc 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "os/exec" 7 | "strings" 8 | ) 9 | 10 | func Host(address string) (addr string, err error) { 11 | var host, port string 12 | if host, port, err = net.SplitHostPort(address); err != nil { 13 | return 14 | } 15 | if host != "host.docker.internal" { 16 | return address, nil // return the passed 17 | } 18 | var cmd = exec.Command("sh", "-c", 19 | "ip -4 route list match 0/0 | cut -d' ' -f3") 20 | var stdout []byte 21 | if stdout, err = cmd.Output(); err != nil { 22 | return 23 | } 24 | var ip = strings.TrimSpace(string(stdout)) 25 | if net.ParseIP(ip) == nil { 26 | return "", fmt.Errorf("invalid 'host.docker.internal' resolution: %s", 27 | ip) 28 | } 29 | return ip + ":" + port, nil // host 30 | } 31 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/writemarker/protocol_integration_tests.go: -------------------------------------------------------------------------------- 1 | //go:build integration_tests 2 | // +build integration_tests 3 | 4 | package writemarker 5 | 6 | import ( 7 | "context" 8 | "time" 9 | 10 | "github.com/0chain/blobber/code/go/0chain.net/conductor/conductrpc" 11 | "github.com/0chain/blobber/code/go/0chain.net/core/node" 12 | ) 13 | 14 | func (wme *WriteMarkerEntity) RedeemMarker(ctx context.Context, startSeq int64) error { 15 | for { 16 | state := conductrpc.Client().State() 17 | if state.StopWMCommit != nil && *state.StopWMCommit { 18 | time.Sleep(time.Second * 5) 19 | continue 20 | } 21 | break 22 | } 23 | err := wme.redeemMarker(ctx, startSeq) 24 | if err == nil { 25 | // send state to conductor server 26 | conductrpc.Client().BlobberCommitted(node.Self.ID) 27 | } 28 | return err 29 | } 30 | -------------------------------------------------------------------------------- /code/go/0chain.net/validatorcore/storage/handler_main.go: -------------------------------------------------------------------------------- 1 | //go:build !integration_tests 2 | // +build !integration_tests 3 | 4 | package storage 5 | 6 | import ( 7 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler" 8 | "github.com/0chain/blobber/code/go/0chain.net/core/common" 9 | "github.com/gorilla/mux" 10 | ) 11 | 12 | /* SetupHandlers sets up the necessary API end points */ 13 | func SetupHandlers(r *mux.Router) { 14 | ConfigureRateLimiter() 15 | r.Use(handler.UseRecovery, handler.UseCors) 16 | 17 | r.HandleFunc("/v1/storage/challenge/new", 18 | RateLimit(common.ToJSONResponse(SetupContext(ChallengeHandler)))) 19 | 20 | r.HandleFunc("/debug", common.ToJSONResponse(DumpGoRoutines)) 21 | 22 | r.HandleFunc("/_stats", statsHandler) 23 | 24 | r.HandleFunc("/_validator_info", common.ToJSONResponse(validatorInfoHandler)) 25 | } 26 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/mock/ctx.go: -------------------------------------------------------------------------------- 1 | package mock 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | 7 | "github.com/0chain/blobber/code/go/0chain.net/core/common" 8 | "github.com/0chain/gosdk/constants" 9 | ) 10 | 11 | func SetupHandlerContext(ctx context.Context, r *http.Request, allocation string) context.Context { 12 | ctx = context.WithValue(ctx, constants.ContextKeyClient, 13 | r.Header.Get(common.ClientHeader)) 14 | ctx = context.WithValue(ctx, constants.ContextKeyClientKey, 15 | r.Header.Get(common.ClientKeyHeader)) 16 | ctx = context.WithValue(ctx, constants.ContextKeyAllocation, 17 | allocation) 18 | // signature is not required for all requests, but if header is empty it won`t affect anything 19 | ctx = context.WithValue(ctx, constants.ContextKeyClientSignatureHeaderKey, r.Header.Get(common.ClientSignatureHeader)) 20 | return ctx 21 | } 22 | -------------------------------------------------------------------------------- /docs/src/download_flow.plantuml: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | actor Client 4 | 5 | 6 | Client -> Blobber : Request the latest read marker (if not cached) 7 | Blobber --> Client : Latest read marker with the read counter 8 | 9 | Client -> Blobber : request the meta data of a file for download 10 | Blobber --> Client : returns merkle root, size, hash and number of blocks 11 | 12 | loop till file is completely downloaded 13 | Client -> Blobber : Request the file and the 64 KB block number. \nPass readmarker (read_ctr - seq for blobber/client combination) 14 | Blobber --> Client : 64 KB block of the requested file 15 | end 16 | 17 | Blobber -> Blockchain : redeem the latest read marker for blobber / client combination 18 | note left 19 | async redeemption 20 | not blocking downloads 21 | end note 22 | 23 | Blockchain -> Blockchain : Tokens moved to blobber from client read pool 24 | 25 | @enduml -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/allocation/dao.go: -------------------------------------------------------------------------------- 1 | package allocation 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/0chain/blobber/code/go/0chain.net/core/common" 7 | "github.com/0chain/errors" 8 | "github.com/0chain/gosdk/constants" 9 | "gorm.io/gorm" 10 | ) 11 | 12 | // GetOrCreate, get allocation if it exists in db. if not, try to sync it from blockchain, and insert it in db. 13 | func GetOrCreate(ctx context.Context, allocationId string) (*Allocation, error) { 14 | if len(allocationId) == 0 { 15 | return nil, errors.Throw(constants.ErrInvalidParameter, "tx") 16 | } 17 | 18 | alloc, err := Repo.GetById(ctx, allocationId) 19 | 20 | if err == nil { 21 | return alloc, nil 22 | } 23 | if !errors.Is(err, gorm.ErrRecordNotFound) { 24 | 25 | return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) 26 | } 27 | 28 | return SyncAllocation(allocationId) 29 | 30 | } 31 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/allocation/file_changer.go: -------------------------------------------------------------------------------- 1 | package allocation 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | 7 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" 8 | ) 9 | 10 | // FileChanger file change processor 11 | type FileChanger interface { 12 | // ApplyChange process change and save them on reference_objects 13 | ApplyChange(ctx context.Context, 14 | change *AllocationChange, allocationRoot string) (*reference.Ref, error) 15 | // Marshal marshal change as JSON string 16 | Marshal() (string, error) 17 | // Unmarshal unmarshal change from JSON string 18 | Unmarshal(input string) error 19 | // DeleteTempFile delete temp file and thumbnail from disk 20 | DeleteTempFile() error 21 | // CommitToFileStore move temp file and thumbnail from temp dir to persistent folder 22 | CommitToFileStore(ctx context.Context, mut *sync.Mutex) error 23 | } 24 | -------------------------------------------------------------------------------- /code/go/0chain.net/core/common/errors.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | /*Error type for a new application error */ 8 | type Error struct { 9 | Code string `json:"code,omitempty"` 10 | Msg string `json:"msg"` 11 | } 12 | 13 | func (err *Error) Error() string { 14 | return fmt.Sprintf("%s: %s", err.Code, err.Msg) 15 | } 16 | 17 | /*NewError - create a new error */ 18 | func NewError(code, msg string) *Error { 19 | return &Error{Code: code, Msg: msg} 20 | } 21 | 22 | /*NewErrorf - create a new error with format */ 23 | func NewErrorf(code, format string, args ...interface{}) *Error { 24 | return &Error{Code: code, Msg: fmt.Sprintf(format, args...)} 25 | } 26 | 27 | /*InvalidRequest - create error messages that are needed when validating request input */ 28 | func InvalidRequest(msg string) error { 29 | return NewError("invalid_request", fmt.Sprintf("Invalid request (%v)", msg)) 30 | } 31 | -------------------------------------------------------------------------------- /code/go/0chain.net/core/common/aws.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aws/aws-sdk-go-v2/aws" 7 | "github.com/aws/aws-sdk-go-v2/config" 8 | "github.com/aws/aws-sdk-go-v2/service/secretsmanager" 9 | ) 10 | 11 | func GetSecretsFromAWS(secretName, region string) (string, error) { 12 | c, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(region)) 13 | if err != nil { 14 | return "", err 15 | } 16 | 17 | // Create Secrets Manager client 18 | svc := secretsmanager.NewFromConfig(c) 19 | 20 | input := &secretsmanager.GetSecretValueInput{ 21 | SecretId: aws.String(secretName), 22 | VersionStage: aws.String("AWSCURRENT"), // VersionStage defaults to AWSCURRENT if unspecified 23 | } 24 | 25 | result, err := svc.GetSecretValue(context.TODO(), input) 26 | if err != nil { 27 | return "", err 28 | } 29 | 30 | secretString := *result.SecretString 31 | return secretString, nil 32 | } -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/challenge/stats.go: -------------------------------------------------------------------------------- 1 | package challenge 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" 7 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" 8 | 9 | "gorm.io/gorm" 10 | ) 11 | 12 | func FileChallenged(ctx context.Context, refID int64, result ChallengeResult, challengeTxn string) { 13 | db := datastore.GetStore().GetTransaction(ctx) 14 | stats := &reference.FileStats{RefID: refID} 15 | if result == ChallengeSuccess { 16 | db.Table(stats.TableName()).Where(stats).Updates(map[string]interface{}{"num_of_challenges": gorm.Expr("num_of_challenges + ?", 1), "last_challenge_txn": challengeTxn}) 17 | } else if result == ChallengeFailure { 18 | db.Table(stats.TableName()).Where(stats).Updates(map[string]interface{}{"num_of_failed_challenges": gorm.Expr("num_of_failed_challenges + ?", 1), "last_challenge_txn": challengeTxn}) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /code/go/0chain.net/core/transaction/http.go: -------------------------------------------------------------------------------- 1 | package transaction 2 | 3 | import ( 4 | "github.com/0chain/blobber/code/go/0chain.net/core/common" 5 | "github.com/0chain/gosdk/core/client" 6 | ) 7 | 8 | const TXN_SUBMIT_URL = "v1/transaction/put" 9 | const TXN_VERIFY_URL = "v1/transaction/get/confirmation?hash=" 10 | const SC_REST_API_URL = "v1/screst/" 11 | const REGISTER_CLIENT = "v1/client/put" 12 | 13 | const ( 14 | SLEEP_FOR_TXN_CONFIRMATION = 1 15 | ) 16 | 17 | var ErrNoTxnDetail = common.NewError("missing_transaction_detail", "No transaction detail was found on any of the sharders") 18 | var MakeSCRestAPICall func(scAddress string, relativePath string, params map[string]string, options ...string) ([]byte, error) = MakeSCRestAPICallNoHandler 19 | 20 | func MakeSCRestAPICallNoHandler(address string, path string, params map[string]string, options ...string) ([]byte, error) { 21 | return client.MakeSCRestAPICall(address, path, params, options...) 22 | } 23 | -------------------------------------------------------------------------------- /https/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.8" 2 | services: 3 | web: 4 | image: nginx:latest 5 | restart: always 6 | volumes: 7 | - ./public:/var/www/html 8 | - ./conf.d:/etc/nginx/conf.d 9 | - ./certbot/conf:/etc/nginx/ssl 10 | - ./certbot/data:/var/www/certbot 11 | ports: 12 | - 80:80 13 | - 443:443 14 | networks: 15 | - default 16 | - testnet0 17 | certbot: 18 | image: certbot/certbot:latest 19 | command: certonly --webroot --webroot-path=/var/www/certbot --email --agree-tos --no-eff-email -d 20 | volumes: 21 | - ./certbot/conf:/etc/letsencrypt 22 | - ./certbot/logs:/var/log/letsencrypt 23 | - ./certbot/data:/var/www/certbot 24 | networks: 25 | - default 26 | - testnet0 27 | networks: 28 | default: 29 | driver: bridge 30 | testnet0: 31 | external: true -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/handler/grpc_handler_main.go: -------------------------------------------------------------------------------- 1 | //go:build !integration_tests 2 | // +build !integration_tests 3 | 4 | package handler 5 | 6 | import ( 7 | "context" 8 | "net/http" 9 | 10 | blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" 11 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/convert" 12 | ) 13 | 14 | func (b *blobberGRPCService) ListEntities(ctx context.Context, req *blobbergrpc.ListEntitiesRequest) (*blobbergrpc.ListEntitiesResponse, error) { 15 | r, err := http.NewRequest("", "", http.NoBody) 16 | if err != nil { 17 | return nil, err 18 | } 19 | httpRequestWithMetaData(r, getGRPCMetaDataFromCtx(ctx), req.Allocation) 20 | r.Form = map[string][]string{ 21 | "path": {req.Path}, 22 | "path_hash": {req.PathHash}, 23 | "auth_token": {req.AuthToken}, 24 | } 25 | 26 | resp, err := ListHandler(ctx, r) 27 | if err != nil { 28 | return nil, err 29 | } 30 | 31 | return convert.ListEntitesResponseCreator(resp), nil 32 | } 33 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/seqpriorityqueue/seqpriorityqueue_test.go: -------------------------------------------------------------------------------- 1 | package seqpriorityqueue 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestSeqPriorityQueue(t *testing.T) { 9 | pq := NewSeqPriorityQueue(21) 10 | 11 | go func() { 12 | for i := 19; i >= 0; i-- { 13 | j := i 14 | go func() { 15 | ud := UploadData{ 16 | Offset: int64(j), 17 | DataBytes: 1, 18 | } 19 | pq.Push(ud) 20 | }() 21 | } 22 | time.Sleep(100 * time.Millisecond) 23 | pq.Done(UploadData{ 24 | Offset: 20, 25 | DataBytes: 1, 26 | }, 21) 27 | }() 28 | expectedOffset := int64(0) 29 | for { 30 | ud := pq.Popup() 31 | if ud.Offset != expectedOffset { 32 | t.Errorf("expected offset %v, got %v", expectedOffset, ud.Offset) 33 | } 34 | if ud.Offset+ud.DataBytes == 21 { 35 | if expectedOffset+(21-ud.Offset) != 21 { 36 | t.Errorf("expected 21, got %v", expectedOffset+(21-ud.Offset)) 37 | } 38 | break 39 | } 40 | expectedOffset += ud.DataBytes 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/handler/grpc_handler_integration_tests.go: -------------------------------------------------------------------------------- 1 | //go:build integration_tests 2 | // +build integration_tests 3 | 4 | package handler 5 | 6 | import ( 7 | "context" 8 | "net/http" 9 | 10 | blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" 11 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/convert" 12 | ) 13 | 14 | func (b *blobberGRPCService) ListEntities(ctx context.Context, req *blobbergrpc.ListEntitiesRequest) (*blobbergrpc.ListEntitiesResponse, error) { 15 | r, err := http.NewRequest("", "", http.NoBody) 16 | if err != nil { 17 | return nil, err 18 | } 19 | httpRequestWithMetaData(r, getGRPCMetaDataFromCtx(ctx), req.Allocation) 20 | r.Form = map[string][]string{ 21 | "path": {req.Path}, 22 | "path_hash": {req.PathHash}, 23 | "auth_token": {req.AuthToken}, 24 | } 25 | 26 | resp, err, _ := ListHandler(ctx, r) 27 | if err != nil { 28 | return nil, err 29 | } 30 | 31 | return convert.ListEntitesResponseCreator(resp), nil 32 | } 33 | -------------------------------------------------------------------------------- /.github/workflows/cicd.yml: -------------------------------------------------------------------------------- 1 | name: CICD-01 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | image: 7 | description: 'image tag' 8 | required: true 9 | default: latest 10 | 11 | jobs: 12 | deploy: 13 | runs-on: [self-hosted] 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | - uses: azure/setup-helm@v1 18 | with: 19 | version: 'v3.2.2' 20 | - name: Setup helm repo 21 | run: | 22 | helm repo add 0chain-helm http://0chain-helm-chart.s3-website.us-east-2.amazonaws.com/helmCharts/ 23 | helm repo upgrade 24 | - name: Setup kubeconfig 25 | run: | 26 | mkdir -p ~/.kube 27 | echo "${{ secrets.CICD01KC }}" | base64 -d > ~/.kube/config 28 | - name: Setup chain 29 | run: | 30 | helm upgrade --install blobber -n helmtest --set blobber.blobberImage.tag=${{ github.event.inputs.image }} --set validator.validatorImage.tag=${{ github.event.inputs.image }} 0chain-helm/blobber 31 | rm -rf ~/.kube -------------------------------------------------------------------------------- /.github/workflows/dev-1.yml: -------------------------------------------------------------------------------- 1 | name: DEPLOY_TO_DEV-1 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | image: 7 | description: 'image tag' 8 | required: true 9 | default: latest 10 | 11 | jobs: 12 | deploy: 13 | runs-on: [self-hosted] 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | - uses: azure/setup-helm@v1 18 | with: 19 | version: 'v3.2.2' 20 | - name: Setup helm repo 21 | run: | 22 | helm repo add 0chain-helm http://0chain-helm-chart.s3-website.us-east-2.amazonaws.com/helmCharts/ 23 | helm repo upgrade 24 | - name: Setup kubeconfig 25 | run: | 26 | mkdir -p ~/.kube 27 | echo "${{ secrets.DEV1KC }}" | base64 -d > ~/.kube/config 28 | - name: Setup chain 29 | run: | 30 | helm upgrade --install blobber -n dev-1 --set blobber.blobberImage.tag=${{ github.event.inputs.image }} --set validator.validatorImage.tag=${{ github.event.inputs.image }} 0chain-helm/blobber 31 | rm -rf ~/.kube -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/allocation/renamefilechange_integration.go: -------------------------------------------------------------------------------- 1 | //go:build integration_tests 2 | // +build integration_tests 3 | 4 | package allocation 5 | 6 | import ( 7 | "context" 8 | "errors" 9 | 10 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" 11 | "github.com/0chain/blobber/code/go/0chain.net/conductor/conductrpc" 12 | "github.com/0chain/blobber/code/go/0chain.net/core/common" 13 | "github.com/0chain/blobber/code/go/0chain.net/core/node" 14 | ) 15 | 16 | func (rf *RenameFileChange) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, 17 | allocationRoot string, ts common.Timestamp, _ map[string]string) (*reference.Ref, error) { 18 | 19 | state := conductrpc.Client().State() 20 | if state.FailRenameCommit != nil { 21 | for _, nodeId := range state.FailRenameCommit { 22 | if nodeId == node.Self.ID { 23 | return nil, errors.New("error directed by conductor") 24 | } 25 | } 26 | } 27 | return rf.applyChange(ctx, rootRef, change, allocationRoot, ts, nil) 28 | } 29 | -------------------------------------------------------------------------------- /code/go/0chain.net/core/common/time.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | //DateTimeFormat - the format in which the date time fields should be displayed in the UI 8 | var DateTimeFormat = "2006-01-02T15:04:05+00:00" 9 | 10 | /*Timestamp - just a wrapper to control the json encoding */ 11 | type Timestamp int64 12 | 13 | /*Now - current datetime */ 14 | func Now() Timestamp { 15 | return Timestamp(time.Now().Unix()) 16 | } 17 | 18 | /*Within ensures a given timestamp is within certain number of seconds */ 19 | func Within(ts, seconds int64) bool { 20 | now := time.Now().Unix() 21 | return now > ts-seconds && now < ts+seconds 22 | } 23 | 24 | //ToTime - converts the common.Timestamp to time.Time 25 | func ToTime(ts Timestamp) time.Time { 26 | return time.Unix(int64(ts), 0) 27 | } 28 | 29 | // ValidateLt will validate timestamp if it is within (currentTime - duration) and currentTime 30 | func ValidateLt(ts Timestamp, d time.Duration) bool { 31 | now := Now() 32 | return ts < now && ts > (now-Timestamp(d.Seconds())) 33 | } 34 | -------------------------------------------------------------------------------- /code/go/0chain.net/dev/server.go: -------------------------------------------------------------------------------- 1 | // package dev providers tools for local development 2 | package dev 3 | 4 | import ( 5 | "net/http/httptest" 6 | 7 | "github.com/0chain/blobber/code/go/0chain.net/dev/miner" 8 | "github.com/0chain/blobber/code/go/0chain.net/dev/sharder" 9 | "github.com/gorilla/mux" 10 | ) 11 | 12 | // Server a local dev server to mock server APIs 13 | type Server struct { 14 | *httptest.Server 15 | *mux.Router 16 | } 17 | 18 | // NewServer create a local dev server 19 | func NewServer() *Server { 20 | router := mux.NewRouter() 21 | s := &Server{ 22 | Router: router, 23 | Server: httptest.NewServer(router), 24 | } 25 | 26 | return s 27 | } 28 | 29 | // NewSharderServer create a local dev sharder server 30 | func NewSharderServer() *Server { 31 | s := NewServer() 32 | 33 | sharder.RegisterHandlers(s.Router) 34 | 35 | return s 36 | } 37 | 38 | // NewMinerServer create a local dev miner server 39 | func NewMinerServer() *Server { 40 | s := NewServer() 41 | 42 | miner.RegisterHandlers(s.Router) 43 | 44 | return s 45 | } 46 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/allocation/file_changer_upload_integration.go: -------------------------------------------------------------------------------- 1 | //go:build integration_tests 2 | // +build integration_tests 3 | 4 | package allocation 5 | 6 | import ( 7 | "context" 8 | "errors" 9 | 10 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" 11 | "github.com/0chain/blobber/code/go/0chain.net/conductor/conductrpc" 12 | "github.com/0chain/blobber/code/go/0chain.net/core/common" 13 | "github.com/0chain/blobber/code/go/0chain.net/core/node" 14 | ) 15 | 16 | func (nf *UploadFileChanger) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, 17 | allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) { 18 | 19 | state := conductrpc.Client().State() 20 | if state.FailUploadCommit != nil { 21 | for _, nodeId := range state.FailUploadCommit { 22 | if nodeId == node.Self.ID { 23 | return nil, errors.New("error directed by conductor") 24 | } 25 | } 26 | } 27 | return nf.applyChange(ctx, rootRef, change, allocationRoot, ts, fileIDMeta) 28 | } 29 | -------------------------------------------------------------------------------- /config/localhost.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICvDCCAkKgAwIBAgIUdBMs5yZ7wQS8ZQFDGaIgURs7TdMwCgYIKoZIzj0EAwIw 3 | gZQxCzAJBgNVBAYTAklOMRAwDgYDVQQIDAdHdWphcmF0MQ4wDAYDVQQHDAVTdXJh 4 | dDEUMBIGA1UECgwLQWxmYUNyZWF0b3IxFDASBgNVBAsMC0RldmVsb3BtZW50MQ8w 5 | DQYDVQQDDAZQcmFkaXAxJjAkBgkqhkiG9w0BCQEWF3ByYWRpcEBhbGZhLWNyZWF0 6 | b3IuY29tMB4XDTIyMDExNDAyMDUwNVoXDTMyMDExMjAyMDUwNVowgZQxCzAJBgNV 7 | BAYTAklOMRAwDgYDVQQIDAdHdWphcmF0MQ4wDAYDVQQHDAVTdXJhdDEUMBIGA1UE 8 | CgwLQWxmYUNyZWF0b3IxFDASBgNVBAsMC0RldmVsb3BtZW50MQ8wDQYDVQQDDAZQ 9 | cmFkaXAxJjAkBgkqhkiG9w0BCQEWF3ByYWRpcEBhbGZhLWNyZWF0b3IuY29tMHYw 10 | EAYHKoZIzj0CAQYFK4EEACIDYgAE/vPDjHmDzM/bsAQFm8Njq+DwlN9Vj73ZU0Pg 11 | PGHgpJicvUTpl7LyrbmlGPJL8gwDZn3pkpJN906ER6Y8zGqHUATxWf4f6014+zDo 12 | EjroaDaFTl1dIsXX8BCgcXEMKQ3Fo1MwUTAdBgNVHQ4EFgQU047AQXH061UPttjb 13 | SIFmiAryHgIwHwYDVR0jBBgwFoAU047AQXH061UPttjbSIFmiAryHgIwDwYDVR0T 14 | AQH/BAUwAwEB/zAKBggqhkjOPQQDAgNoADBlAjEAypdZMUc/8OYXICh7zONovLpX 15 | d5OrKL7OXgeqTQ4GaLZvCB/mvRAyApONAiuL7EleAjB6uDoTI0JbOM2LQpECx7JE 16 | C0w0e34Usc6/7S+DciPHSeG2oF6P1yP/bQ+QPVSRCbo= 17 | -----END CERTIFICATE----- 18 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/stats/allocationstats.go: -------------------------------------------------------------------------------- 1 | package stats 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" 7 | ) 8 | 9 | // Timestamp that implements standard fmt.Stringer interface. 10 | type Timestamp int64 11 | 12 | // String implements standard fmt.Stringer interface. 13 | func (t Timestamp) String() string { 14 | return time.Unix(int64(t), 0).String() 15 | } 16 | 17 | type AllocationStats struct { 18 | AllocationID string `json:"allocation_id"` 19 | TempFolderSize uint64 `json:"-"` 20 | Stats 21 | Expiration Timestamp `json:"expiration_date" gorm:"column:expiration_date"` 22 | 23 | ReadMarkers *ReadMarkersStat `json:"read_markers"` 24 | WriteMarkers *WriteMarkersStat `json:"write_markers"` 25 | } 26 | 27 | func (aStat *AllocationStats) loadAllocationDiskUsageStats() error { 28 | aStat.DiskSizeUsed = filestore.GetFileStore().GetCommittedFileSizeOfAllocation(aStat.AllocationID) 29 | aStat.TempFolderSize = filestore.GetFileStore().GetTempFilesSizeOfAllocation(aStat.AllocationID) 30 | return nil 31 | } 32 | -------------------------------------------------------------------------------- /docker.local/bin/build.validator.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | GIT_COMMIT=$(git rev-list -1 HEAD) 5 | echo $GIT_COMMIT 6 | 7 | echo "1> set DOCKER_IMAGE & DOCKER_BUILD" 8 | if [ -z "$DOCKER_BUILD" ]; then 9 | if [ "x86_64" != "$(uname -m)" ]; then 10 | #docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use 11 | DOCKER_BUILD="buildx build --platform linux/arm64" 12 | else 13 | DOCKER_BUILD="build" 14 | fi 15 | fi 16 | 17 | if [ -z "$DOCKER_IMAGE_BASE" ]; then 18 | DOCKER_IMAGE_BASE="blobber_base" 19 | fi 20 | 21 | if [ -z "$DOCKER_IMAGE_VALIDATOR" ]; then 22 | DOCKER_IMAGE_VALIDATOR="-t validator" 23 | fi 24 | 25 | echo " DOCKER_BUILD=$DOCKER_BUILD" 26 | echo " DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE" 27 | echo " DOCKER_IMAGE_VALIDATOR=$DOCKER_IMAGE_VALIDATOR" 28 | 29 | echo "" 30 | echo "2> docker build validator" 31 | DOCKER_BUILDKIT=1 docker $DOCKER_BUILD --progress=plain --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE -f docker.local/validator.Dockerfile . $DOCKER_IMAGE_VALIDATOR -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/allocation/connection_bench_test.go: -------------------------------------------------------------------------------- 1 | package allocation 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/lithammer/shortuuid/v3" 8 | ) 9 | 10 | var table = []struct { 11 | input int 12 | }{ 13 | {input: 100}, 14 | {input: 1000}, 15 | {input: 10000}, 16 | } 17 | 18 | func BenchmarkConnectionObj(b *testing.B) { 19 | for _, v := range table { 20 | connectionIDs := make([]string, 0) 21 | for i := 0; i < v.input; i++ { 22 | connectionID := shortuuid.New() 23 | connectionIDs = append(connectionIDs, connectionID) 24 | UpdateConnectionObjSize(connectionID, int64(i)) 25 | } 26 | 27 | b.Run(fmt.Sprintf("input_size_%d", v.input), func(b *testing.B) { 28 | for i := 0; i < b.N; i++ { 29 | connectionID := connectionIDs[i%v.input] 30 | GetConnectionObjSize(connectionID) 31 | UpdateConnectionObjSize(connectionID, int64(v.input)) 32 | 33 | newConnectionID := shortuuid.New() 34 | go GetConnectionObjSize(newConnectionID) 35 | go UpdateConnectionObjSize(newConnectionID, int64(v.input)) 36 | 37 | } 38 | }) 39 | 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /docker.local/ValidatorDockerfile: -------------------------------------------------------------------------------- 1 | FROM blobber_base as validator_build 2 | 3 | LABEL zchain="validator" 4 | 5 | RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep 6 | 7 | ENV SRC_DIR=/blobber 8 | ENV GO111MODULE=on 9 | 10 | # Download the dependencies: 11 | # Will be cached if we don't change mod/sum files 12 | COPY ./go.mod ./go.sum $SRC_DIR/ 13 | RUN cd $SRC_DIR && go mod download 14 | 15 | #Add the source code 16 | ADD . $SRC_DIR 17 | 18 | WORKDIR $SRC_DIR/code/go/0chain.net/validator 19 | 20 | RUN CGO_ENABLED=1 go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" 21 | 22 | # Copy the build artifact into a minimal runtime image: 23 | FROM golang:1.19-alpine3.17 24 | RUN apk add gmp gmp-dev openssl-dev 25 | COPY --from=validator_build /usr/local/lib/libmcl*.so \ 26 | /usr/local/lib/libbls*.so \ 27 | /usr/local/lib/ 28 | ENV APP_DIR=/blobber 29 | WORKDIR $APP_DIR 30 | COPY --from=validator_build $APP_DIR/code/go/0chain.net/validator/validator $APP_DIR/bin/validator 31 | -------------------------------------------------------------------------------- /code/go/0chain.net/core/common/request_form.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/0chain/blobber/code/go/0chain.net/core/logging" 7 | "go.uber.org/zap" 8 | ) 9 | 10 | const ( 11 | FormFileParseMaxMemory = 32 * 1024 * 1024 12 | ) 13 | 14 | // TryParseForm try populates r.Form and r.PostForm. 15 | func TryParseForm(r *http.Request) { 16 | if r.Method == http.MethodPost || r.Method == http.MethodPut || r.Method == http.MethodPatch { 17 | ct := r.Header.Get("Content-Type") 18 | if ct == "application/x-www-form-urlencoded" { 19 | r.ParseForm() //nolint: errcheck 20 | } else { 21 | err := r.ParseMultipartForm(FormFileParseMaxMemory) //nolint: errcheck 22 | if err != nil { 23 | logging.Logger.Error("TryParseForm: ParseMultipartForm", zap.Error(err)) 24 | } 25 | } 26 | } 27 | } 28 | 29 | // GetField get field from form or query 30 | func GetField(r *http.Request, key string) (string, bool) { 31 | TryParseForm(r) 32 | 33 | v, ok := r.Form[key] 34 | if ok { 35 | return v[0], true 36 | } 37 | 38 | v, ok = r.URL.Query()[key] 39 | if ok { 40 | return v[0], true 41 | } 42 | 43 | return "", false 44 | } 45 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/reference/ds_test.go: -------------------------------------------------------------------------------- 1 | package reference 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" 8 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | // this is just a dummy snippet to connect to local database 13 | func TestMockDb(t *testing.T) { 14 | t.Skip("Fails as the data store is not mocked, so Open returns a dial error") 15 | config.Configuration.DBHost = "localhost" 16 | config.Configuration.DBName = "blobber_meta" 17 | config.Configuration.DBPort = "5431" 18 | config.Configuration.DBUserName = "blobber_user" 19 | config.Configuration.DBPassword = "" 20 | 21 | require.NoError(t, datastore.GetStore().Open()) 22 | _ = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { 23 | tx := datastore.GetStore().GetTransaction(ctx) 24 | ref := &Ref{} 25 | err := tx.Where(&Ref{AllocationID: "4f928c7857fabb5737347c42204eea919a4777f893f35724f563b932f64e2367", Path: "/hack.txt"}). 26 | First(ref). 27 | Error 28 | require.NoError(t, err) 29 | 30 | return nil 31 | }) 32 | } 33 | -------------------------------------------------------------------------------- /docker.local/IntegrationTestsValidator.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM blobber_base as validator_build 2 | 3 | LABEL zchain="validator" 4 | 5 | RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep 6 | 7 | ENV SRC_DIR=/validator 8 | ENV GO111MODULE=on 9 | 10 | # Download the dependencies: 11 | # Will be cached if we don't change mod/sum files 12 | COPY ./go.mod ./go.sum $SRC_DIR/ 13 | RUN cd $SRC_DIR && go mod download 14 | 15 | #Add the source code 16 | ADD . $SRC_DIR 17 | 18 | WORKDIR $SRC_DIR/code/go/0chain.net/validator 19 | 20 | RUN CGO_ENABLED=1 go build -v -tags "bn256 development integration_tests" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" 21 | 22 | # Copy the build artifact into a minimal runtime image: 23 | FROM golang:1.19-alpine3.17 24 | RUN apk add gmp gmp-dev openssl-dev 25 | COPY --from=validator_build /usr/local/lib/libmcl*.so \ 26 | /usr/local/lib/libbls*.so \ 27 | /usr/local/lib/ 28 | ENV APP_DIR=/validator 29 | WORKDIR $APP_DIR 30 | COPY --from=validator_build $APP_DIR/code/go/0chain.net/validator/validator $APP_DIR/bin/validator 31 | 32 | -------------------------------------------------------------------------------- /code/go/0chain.net/validatorcore/storage/handler_integration_tests.go: -------------------------------------------------------------------------------- 1 | //go:build integration_tests 2 | // +build integration_tests 3 | 4 | package storage 5 | 6 | import ( 7 | "context" 8 | "net/http" 9 | 10 | "github.com/0chain/blobber/code/go/0chain.net/core/common" 11 | "github.com/0chain/gosdk/constants" 12 | "github.com/gorilla/mux" 13 | ) 14 | 15 | /* SetupHandlers sets up the necessary API end points */ 16 | func SetupHandlers(r *mux.Router) { 17 | ConfigureRateLimiter() 18 | r.HandleFunc("/v1/storage/challenge/new", 19 | RateLimit(common.ToJSONOrNotResponse(setupContextNotRespond(ChallengeHandler)))) 20 | 21 | r.HandleFunc("/debug", common.ToJSONResponse(DumpGoRoutines)) 22 | } 23 | 24 | func setupContextNotRespond(handler common.JSONResponderOrNotF) common.JSONResponderOrNotF { 25 | return func(ctx context.Context, r *http.Request) (interface{}, error, bool) { 26 | ctx = context.WithValue(ctx, constants.ContextKeyClient, r.Header.Get(common.ClientHeader)) 27 | ctx = context.WithValue(ctx, constants.ContextKeyClientKey, 28 | r.Header.Get(common.ClientKeyHeader)) 29 | res, err, shouldRespond := handler(ctx, r) 30 | return res, err, shouldRespond 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /docker.aws/build.validator/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | 4 | validator: 5 | build: 6 | context: ${DOCKER_CODEBASE} 7 | dockerfile: ./docker.aws/build.validator/Dockerfile 8 | 9 | container_name: ${AGENT_CONTAINER} 10 | 11 | image: ${IMAGE_NAME}:${IMAGE_TAG} 12 | 13 | environment: 14 | - DOCKER= true 15 | 16 | user: ${ZCHAIN_USER}:${ZCHAIN_GROUP} 17 | 18 | command: /0chain/bin/validator --deployment_mode 0 --keys_file /0chain/config/validator.txt --nodes_file nodes 19 | 20 | ports: 21 | - ${AGENT_PORT}:${AGENT_PORT} 22 | 23 | volumes: 24 | - /0chain/${AGENT_DIR}/config:/0chain/config 25 | - /0chain/${AGENT_DIR}/log:/0chain/log 26 | - /0chain/${AGENT_DIR}/data:/0chain/data 27 | - /0chain/${AGENT_DIR}/data/badgerdb:/0chain/data/badgerdb 28 | - /0chain/${AGENT_DIR}/data/badgerdb/blobberstate:/0chain/data/badgerdb/blobberstate 29 | 30 | logging: 31 | driver: json-file 32 | options: 33 | max-file: "10" 34 | max-size: 10M 35 | 36 | networks: 37 | default: 38 | 39 | restart: ${RESTART_POLICY} 40 | 41 | networks: 42 | default: 43 | driver: bridge 44 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/zcn/query.go: -------------------------------------------------------------------------------- 1 | package zcn 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "github.com/0chain/gosdk/zboxcore/sdk" 8 | "sync" 9 | 10 | "github.com/0chain/gosdk/zcncore" 11 | ) 12 | 13 | var ErrBlobberNotFound = errors.New("blobber is not found on chain") 14 | 15 | // GetBlobber try to get blobber info from chain. 16 | func GetBlobber(blobberID string) (*sdk.Blobber, error) { 17 | var ( 18 | blobber *sdk.Blobber 19 | err error 20 | ) 21 | if blobber, err = sdk.GetBlobber(blobberID); err != nil { 22 | return nil, err 23 | } 24 | 25 | return blobber, nil 26 | 27 | } 28 | 29 | type getBlobberCallback struct { 30 | wg sync.WaitGroup 31 | Blobber *zcncore.Blobber 32 | Error error 33 | } 34 | 35 | func (cb *getBlobberCallback) OnInfoAvailable(op int, status int, info string, err string) { 36 | defer cb.wg.Done() 37 | if status != zcncore.StatusSuccess { 38 | cb.Error = errors.New(err) 39 | return 40 | } 41 | b := &zcncore.Blobber{} 42 | if err := json.Unmarshal([]byte(info), b); err != nil { 43 | cb.Error = fmt.Errorf("getBlobber:json %s %w", info, err) 44 | return 45 | } 46 | 47 | cb.Blobber = b 48 | 49 | } 50 | -------------------------------------------------------------------------------- /docker.local/IntegrationTestsBlobber.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM blobber_base as blobber_build 2 | 3 | RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep 4 | 5 | ENV SRC_DIR=/blobber 6 | ENV GO111MODULE=on 7 | 8 | # Download the dependencies: 9 | # Will be cached if we don't change mod/sum files 10 | COPY ./go.mod ./go.sum $SRC_DIR/ 11 | RUN cd $SRC_DIR && go mod download 12 | 13 | #Add the source code 14 | ADD ./goose $SRC_DIR/goose 15 | ADD ./code/go/0chain.net $SRC_DIR/code/go/0chain.net 16 | 17 | WORKDIR $SRC_DIR/code/go/0chain.net/blobber 18 | 19 | RUN CGO_ENABLED=1 go build -v -tags "bn256 development integration_tests" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" 20 | 21 | # Copy the build artifact into a minimal runtime image: 22 | FROM golang:1.19-alpine3.17 23 | RUN apk add gmp gmp-dev openssl-dev 24 | COPY --from=blobber_build /usr/local/lib/libmcl*.so \ 25 | /usr/local/lib/libbls*.so \ 26 | /usr/local/lib/ 27 | ENV APP_DIR=/blobber 28 | WORKDIR $APP_DIR 29 | COPY --from=blobber_build $APP_DIR/code/go/0chain.net/blobber/blobber $APP_DIR/bin/blobber -------------------------------------------------------------------------------- /docker.local/bin/build.base.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | GIT_COMMIT=$(git rev-list -1 HEAD) 5 | echo $GIT_COMMIT 6 | echo "1> set DOCKER_IMAGE & DOCKER_BUILD" 7 | if [ -z "$DOCKER_BUILD" ]; then 8 | if [ "x86_64" != "$(uname -m)" ]; then 9 | #docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use 10 | DOCKER_BUILD="buildx build --platform linux/arm64" 11 | else 12 | DOCKER_BUILD="build" 13 | fi 14 | fi 15 | 16 | if [ -z "$DOCKER_IMAGE_BASE" ]; then 17 | DOCKER_IMAGE_BASE="blobber_base" 18 | fi 19 | echo " DOCKER_BUILD=$DOCKER_BUILD" 20 | echo " DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE" 21 | 22 | echo "" 23 | echo "2> download herumi" 24 | 25 | [ ! -f ./docker.local/bin/mcl.tar.gz ] && wget -O ./docker.local/bin/mcl.tar.gz https://github.com/herumi/mcl/archive/refs/tags/v1.57.tar.gz 26 | 27 | [ ! -f ./docker.local/bin/bls.tar.gz ] && wget -O ./docker.local/bin/bls.tar.gz https://github.com/herumi/bls/archive/refs/tags/v1.22.tar.gz 28 | 29 | echo "3> docker build" 30 | DOCKER_BUILDKIT=1 docker $DOCKER_BUILD --progress=plain --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/base.Dockerfile . -t $DOCKER_IMAGE_BASE --network host 31 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/blobbergrpc/proto/google/api/annotations.proto: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package google.api; 18 | 19 | import "google/api/http.proto"; 20 | import "google/protobuf/descriptor.proto"; 21 | 22 | option go_package = "google.golang.org/genproto/googleapis/api/httpbody;httpbody"; 23 | option java_multiple_files = true; 24 | option java_outer_classname = "AnnotationsProto"; 25 | option java_package = "com.google.api"; 26 | option objc_class_prefix = "GAPI"; 27 | 28 | extend google.protobuf.MethodOptions { 29 | // See `HttpRule`. 30 | HttpRule http = 72295728; 31 | } -------------------------------------------------------------------------------- /docker.local/bin/build.blobber.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | GIT_COMMIT=$(git rev-list -1 HEAD) 5 | echo $GIT_COMMIT 6 | 7 | echo "1> set DOCKER_IMAGE & DOCKER_BUILD" 8 | if [ -z "$DOCKER_BUILD" ]; then 9 | if [ "x86_64" != "$(uname -m)" ]; then 10 | #docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use 11 | DOCKER_BUILD="buildx build --platform linux/arm64" 12 | else 13 | DOCKER_BUILD="build" 14 | fi 15 | fi 16 | 17 | if [ -z "$DOCKER_IMAGE_BASE" ]; then 18 | DOCKER_IMAGE_BASE="blobber_base" 19 | fi 20 | 21 | if [ -z "$DOCKER_IMAGE_BLOBBER" ]; then 22 | DOCKER_IMAGE_BLOBBER="-t blobber" 23 | fi 24 | 25 | echo " DOCKER_BUILD=$DOCKER_BUILD" 26 | echo " DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE" 27 | echo " DOCKER_IMAGE_SWAGGER=$DOCKER_IMAGE_SWAGGER" 28 | echo " DOCKER_IMAGE_BLOBBER=$DOCKER_IMAGE_BLOBBER" 29 | 30 | echo "" 31 | 32 | # echo "generating swagger" 33 | # docker.local/bin/test.swagger.sh 34 | 35 | echo "2> docker build blobber" 36 | DOCKER_BUILDKIT=1 docker $DOCKER_BUILD --progress=plain --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE -f docker.local/blobber.Dockerfile . $DOCKER_IMAGE_BLOBBER --network host 37 | -------------------------------------------------------------------------------- /docker.local/ValidatorDockerfile.dev: -------------------------------------------------------------------------------- 1 | FROM blobber_base as validator_build 2 | 3 | RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep 4 | 5 | ENV SRC_DIR=/blobber 6 | ENV GO111MODULE=on 7 | 8 | # Download the dependencies: 9 | # Will be cached if we don't change mod/sum files 10 | ADD ./gosdk /gosdk 11 | COPY ./blobber/code/go/0chain.net/go.mod ./blobber/code/go/0chain.net/go.sum $SRC_DIR/go/0chain.net/ 12 | RUN cd $SRC_DIR/go/0chain.net && go mod download 13 | 14 | #Add the source code 15 | ADD ./blobber/code/go/0chain.net $SRC_DIR/go/0chain.net 16 | 17 | WORKDIR $SRC_DIR/go/0chain.net/validator 18 | 19 | RUN CGO_ENABLED=1 go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" 20 | 21 | # Copy the build artifact into a minimal runtime image: 22 | FROM golang:1.19-alpine3.17 23 | RUN apk add gmp gmp-dev openssl-dev 24 | COPY --from=validator_build /usr/local/lib/libmcl*.so \ 25 | /usr/local/lib/libbls*.so \ 26 | /usr/local/lib/ 27 | ENV APP_DIR=/blobber 28 | WORKDIR $APP_DIR 29 | COPY --from=validator_build $APP_DIR/go/0chain.net/validator/validator $APP_DIR/bin/validator 30 | -------------------------------------------------------------------------------- /docker.local/validator.Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1 2 | ARG DOCKER_IMAGE_BASE 3 | FROM $DOCKER_IMAGE_BASE as validator_build 4 | 5 | LABEL zchain="validator" 6 | 7 | 8 | ENV SRC_DIR=/0chain 9 | ENV GO111MODULE=on 10 | #ENV GOPROXY=https://goproxy.cn,direct 11 | 12 | 13 | # Download the dependencies: 14 | # Will be cached if we don't change mod/sum files 15 | COPY . $SRC_DIR 16 | # COPY ./gosdk /gosdk 17 | 18 | RUN cd $SRC_DIR/ && go mod download 19 | 20 | WORKDIR $SRC_DIR/code/go/0chain.net/validator 21 | ARG GIT_COMMIT 22 | ENV GIT_COMMIT=$GIT_COMMIT 23 | RUN CGO_ENABLED=1 go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" 24 | 25 | # Copy the build artifact into a minimal runtime image: 26 | FROM alpine:3.18 27 | 28 | RUN apk add --update --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/main libstdc++ gmp openssl vips 29 | 30 | COPY --from=validator_build /usr/local/lib/libmcl*.so \ 31 | /usr/local/lib/libbls*.so \ 32 | /usr/local/lib/ 33 | ENV APP_DIR=/validator 34 | WORKDIR $APP_DIR 35 | COPY --from=validator_build /0chain/code/go/0chain.net/validator/validator $APP_DIR/bin/validator -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/handler/grpc_commit_handler.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "mime/multipart" 7 | "net/http" 8 | "strings" 9 | 10 | blobbergrpc "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc/proto" 11 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/convert" 12 | ) 13 | 14 | func (b *blobberGRPCService) Commit(ctx context.Context, req *blobbergrpc.CommitRequest) (*blobbergrpc.CommitResponse, error) { 15 | body := bytes.NewBuffer([]byte{}) 16 | writer := multipart.NewWriter(body) 17 | err := writer.WriteField("write_marker", req.WriteMarker) 18 | if err != nil { 19 | return nil, err 20 | } 21 | err = writer.WriteField("connection_id", req.ConnectionId) 22 | if err != nil { 23 | return nil, err 24 | } 25 | writer.Close() 26 | 27 | r, err := http.NewRequest("POST", "", strings.NewReader(body.String())) 28 | if err != nil { 29 | return nil, err 30 | } 31 | httpRequestWithMetaData(r, getGRPCMetaDataFromCtx(ctx), req.Allocation) 32 | r.Header.Set("Content-Type", writer.FormDataContentType()) 33 | 34 | resp, _, err := CommitHandler(ctx, r) 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | return convert.CommitWriteResponseCreator(resp), nil 40 | } 41 | -------------------------------------------------------------------------------- /docker.local/Dockerfile.dev: -------------------------------------------------------------------------------- 1 | FROM blobber_base as blobber_build 2 | 3 | RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep 4 | 5 | ENV SRC_DIR=/blobber 6 | ENV GO111MODULE=on 7 | 8 | # Download the dependencies: 9 | # Will be cached if we don't change mod/sum files 10 | ADD ./gosdk /gosdk 11 | COPY ./blobber/code/go/0chain.net/go.mod ./blobber/code/go/0chain.net/go.sum $SRC_DIR/go/0chain.net/ 12 | RUN cd $SRC_DIR/go/0chain.net && go mod download 13 | 14 | #Add the source code 15 | ADD ./blobber/code/go/0chain.net $SRC_DIR/go/0chain.net 16 | 17 | WORKDIR $SRC_DIR/go/0chain.net/blobber 18 | 19 | ARG GIT_COMMIT 20 | ENV GIT_COMMIT=$GIT_COMMIT 21 | RUN CGO_ENABLED=1 go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" 22 | 23 | # Copy the build artifact into a minimal runtime image: 24 | FROM golang:1.19-alpine3.17 25 | RUN apk add gmp gmp-dev openssl-dev 26 | COPY --from=blobber_build /usr/local/lib/libmcl*.so \ 27 | /usr/local/lib/libbls*.so \ 28 | /usr/local/lib/ 29 | ENV APP_DIR=/blobber 30 | WORKDIR $APP_DIR 31 | COPY --from=blobber_build $APP_DIR/go/0chain.net/blobber/blobber $APP_DIR/bin/blobber 32 | -------------------------------------------------------------------------------- /docker.local/blobber.Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1 2 | ARG DOCKER_IMAGE_BASE 3 | FROM $DOCKER_IMAGE_BASE as blobber_build 4 | LABEL zchain="blobber" 5 | 6 | ENV SRC_DIR=/0chain 7 | ENV GO111MODULE=on 8 | # ENV GOPROXY=http://10.10.10.100:3080 9 | 10 | # Download the dependencies: 11 | # Will be cached if we don't change mod/sum files 12 | COPY . $SRC_DIR 13 | # COPY ./gosdk /gosdk 14 | 15 | RUN cd $SRC_DIR/ && go mod download 16 | 17 | WORKDIR $SRC_DIR/code/go/0chain.net/blobber 18 | 19 | ARG GIT_COMMIT 20 | ENV GIT_COMMIT=$GIT_COMMIT 21 | RUN CGO_ENABLED=1 go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" 22 | 23 | # Copy the build artifact into a minimal runtime image: 24 | FROM alpine:3.18 25 | 26 | RUN apk add --update --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/main libstdc++ gmp openssl vips 27 | 28 | COPY --from=blobber_build /usr/local/lib/libmcl*.so \ 29 | /usr/local/lib/libbls*.so \ 30 | /usr/local/lib/ 31 | 32 | ENV APP_DIR=/blobber 33 | WORKDIR $APP_DIR 34 | COPY --from=blobber_build /0chain/code/go/0chain.net/blobber/blobber $APP_DIR/bin/blobber 35 | # COPY ./code/go/0chain.net/swagger.yaml /docs/swagger.yaml 36 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/util/json.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "strings" 7 | 8 | "github.com/0chain/gosdk/zboxcore/fileref" 9 | ) 10 | 11 | // Validate unmarshalled data with tag-based rules 12 | // Example: 13 | // struct { 14 | // Name string `json:"name" validation:"required"` 15 | // } 16 | func UnmarshalValidation(v interface{}) error { 17 | fields := reflect.ValueOf(v).Elem() 18 | 19 | for i := 0; i < fields.NumField(); i++ { 20 | validation := fields.Type().Field(i).Tag.Get("validation") 21 | if strings.Contains(validation, "required") && fields.Field(i).IsZero() { 22 | // todo: better try this first: 23 | // jsonFieldName := fields.Type().Field(i).Tag.Get("json") 24 | return fmt.Errorf("The '%s' field is required", fields.Type().Field(i).Name) 25 | } 26 | } 27 | 28 | return nil 29 | } 30 | 31 | func GetParentPathHashes(allocationTx, filePath string) []string { 32 | splitted := strings.Split(filePath, "/") 33 | pathHashes := []string{} 34 | 35 | for i := 0; i < len(splitted); i++ { 36 | path := strings.Join(splitted[:len(splitted)-i], "/") 37 | if path == "" { 38 | path = "/" 39 | } 40 | pathHash := fileref.GetReferenceLookup(allocationTx, path) 41 | pathHashes = append(pathHashes, pathHash) 42 | } 43 | return pathHashes 44 | } 45 | -------------------------------------------------------------------------------- /docker.local/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM blobber_base as blobber_build 2 | 3 | LABEL zchain="blobber" 4 | 5 | RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep 6 | 7 | ENV SRC_DIR=/blobber 8 | ENV GO111MODULE=on 9 | 10 | # Download the dependencies: 11 | # Will be cached if we don't change mod/sum files 12 | COPY ./go.mod ./go.sum $SRC_DIR/ 13 | RUN cd $SRC_DIR && go mod download 14 | 15 | #Add the source code 16 | ADD ./code/go/0chain.net $SRC_DIR/code/go/0chain.net 17 | 18 | WORKDIR $SRC_DIR/code/go/0chain.net/blobber 19 | 20 | ARG GIT_COMMIT 21 | ENV GIT_COMMIT=$GIT_COMMIT 22 | RUN CGO_ENABLED=1 go build -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" 23 | 24 | # Copy the build artifact into a minimal runtime image: 25 | FROM alpine:3.18 26 | RUN apk add --update --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/main libstdc++ gmp openssl vips 27 | #RUN apk add gmp gmp-dev openssl-dev 28 | COPY --from=blobber_build /usr/local/lib/libmcl*.so \ 29 | /usr/local/lib/libbls*.so \ 30 | /usr/local/lib/ 31 | 32 | ENV APP_DIR=/blobber 33 | WORKDIR $APP_DIR 34 | COPY --from=blobber_build $APP_DIR/code/go/0chain.net/blobber/blobber $APP_DIR/bin/blobber 35 | -------------------------------------------------------------------------------- /docker.local/Dockerfile.swagger: -------------------------------------------------------------------------------- 1 | ARG DOCKER_IMAGE_BASE=blobber_base 2 | FROM $DOCKER_IMAGE_BASE as cross 3 | 4 | ARG commit_hash="dev" 5 | ARG tag_name="dev" 6 | 7 | RUN git clone https://github.com/go-swagger/go-swagger /work 8 | WORKDIR /work 9 | 10 | RUN apk --no-cache add ca-certificates shared-mime-info mailcap git build-base binutils-gold 11 | 12 | RUN mkdir -p bin &&\ 13 | LDFLAGS="$LDFLAGS -X github.com/go-swagger/go-swagger/cmd/swagger/commands.Commit=${commit_hash}" &&\ 14 | LDFLAGS="$LDFLAGS -X github.com/go-swagger/go-swagger/cmd/swagger/commands.Version=${tag_name}" &&\ 15 | CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -tags osusergo,netgo -o bin/swagger -ldflags "$LDFLAGS" -a ./cmd/swagger 16 | 17 | FROM $DOCKER_IMAGE_BASE 18 | 19 | ENV SRC_DIR=/blobber 20 | ENV GO111MODULE=on 21 | 22 | RUN apk --no-cache add yq 23 | 24 | # Download the dependencies: 25 | # Will be cached if we don't change mod/sum files 26 | COPY ./go.mod ./go.sum $SRC_DIR/ 27 | RUN cd $SRC_DIR && go mod download 28 | 29 | #Add the source code 30 | ADD ./code/go/0chain.net $SRC_DIR/code/go/0chain.net 31 | 32 | WORKDIR $SRC_DIR/code/go/0chain.net/blobber 33 | 34 | COPY --from=cross /work/bin/swagger /usr/bin/swagger 35 | COPY --from=cross /work/generator/templates/contrib /templates/ 36 | 37 | WORKDIR $SRC_DIR 38 | -------------------------------------------------------------------------------- /docs/src/upload_flow.puml: -------------------------------------------------------------------------------- 1 | @startuml Upload / Delete Flow 2 | 3 | actor Client 4 | 5 | title Upload/Delete flow 6 | Client -> Blockchain : Make allocation request (erasure code, size, expiry) 7 | Blockchain -> Blockchain : Assigns the blobbers \ncreates the read/write pools for allocation 8 | Blockchain --> Client : Allocation status, if success, list of blobbers selected 9 | 10 | 11 | loop till done 12 | group upload 13 | Client -> Blobber : POST request - Upload the file (path, metadata, connection id) 14 | Blobber --> Client : Upload successful 15 | end 16 | group delete 17 | Client -> Blobber : DELETE request - Delete the file and pass delete token (path hash, file hash from list api) 18 | Blobber -> Blobber : Validate the delete token and stores with New state 19 | Blobber --> Client : delete successful 20 | end 21 | end 22 | Client -> Blobber : commit the connection and pass the writemarker (allocation root, prev allocation root, upload size) 23 | Blobber -> Blobber : validates the write marker, commits any delete tokens 24 | Blobber --> Client : commit success / failure 25 | 26 | 27 | 28 | Blobber -> Blockchain : redeem the write marker in the order received 29 | note right 30 | async redeemption 31 | not blocking uploads 32 | end note 33 | 34 | Blockchain -> Blockchain : Move tokens from write pool to challenge pool 35 | 36 | @enduml -------------------------------------------------------------------------------- /docs/src/upload_flow.plantuml: -------------------------------------------------------------------------------- 1 | @startuml Upload / Delete Flow 2 | 3 | actor Client 4 | 5 | title Upload/Delete flow 6 | Client -> Blockchain : Make allocation request (erasure code, size, expiry) 7 | Blockchain -> Blockchain : Assigns the blobbers \ncreates the read/write pools for allocation 8 | Blockchain --> Client : Allocation status, if success, list of blobbers selected 9 | 10 | 11 | loop till done 12 | group upload 13 | Client -> Blobber : POST request - Upload the file (path, metadata, connection id) 14 | Blobber --> Client : Upload successful 15 | end 16 | group delete 17 | Client -> Blobber : DELETE request - Delete the file and pass delete token (path hash, file hash from list api) 18 | Blobber -> Blobber : Validate the delete token and stores with New state 19 | Blobber --> Client : delete successful 20 | end 21 | end 22 | Client -> Blobber : commit the connection and pass the writemarker (allocation root, prev allocation root, upload size) 23 | Blobber -> Blobber : validates the write marker, commits any delete tokens 24 | Blobber --> Client : commit success / failure 25 | 26 | 27 | 28 | Blobber -> Blockchain : redeem the write marker in the order received 29 | note right 30 | async redeemption 31 | not blocking uploads 32 | end note 33 | 34 | Blockchain -> Blockchain : Move tokens from write pool to challenge pool 35 | 36 | @enduml -------------------------------------------------------------------------------- /code/go/0chain.net/core/common/admin.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "github.com/0chain/blobber/code/go/0chain.net/core/logging" 5 | "go.uber.org/zap" 6 | "net/http" 7 | 8 | "github.com/spf13/viper" 9 | ) 10 | 11 | // global username and password used to access endpoints only by admin 12 | var gUsername, gPassword string 13 | var isDevelopment bool 14 | var PublicKey0box string 15 | 16 | func SetAdminCredentials(devMode bool) { 17 | gUsername = viper.GetString("admin.username") 18 | gPassword = viper.GetString("admin.password") 19 | isDevelopment = devMode 20 | } 21 | 22 | func AuthenticateAdmin(handler ReqRespHandlerf) ReqRespHandlerf { 23 | return func(w http.ResponseWriter, r *http.Request) { 24 | if !isDevelopment { 25 | uname, passwd, ok := r.BasicAuth() 26 | if !ok { 27 | w.WriteHeader(http.StatusForbidden) 28 | w.Write([]byte("Admin only api")) // nolint 29 | return 30 | } 31 | 32 | if uname != gUsername || passwd != gPassword { 33 | w.WriteHeader(http.StatusForbidden) 34 | w.Write([]byte("Invalid username or password")) // nolint 35 | return 36 | } 37 | } 38 | 39 | handler(w, r) 40 | } 41 | } 42 | 43 | func Set0boxDetails() { 44 | logging.Logger.Info("Setting 0box details") 45 | PublicKey0box = viper.GetString("0box.public_key") 46 | logging.Logger.Info("0box public key", zap.Any("public_key", PublicKey0box)) 47 | } 48 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobber/datastore.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" 8 | "github.com/0chain/blobber/code/go/0chain.net/core/logging" 9 | "github.com/0chain/blobber/goose" 10 | "gorm.io/gorm" 11 | ) 12 | 13 | func setupDatabase() error { 14 | fmt.Print("\r> connect data store") 15 | // check for database connection 16 | var pgDB *gorm.DB 17 | var err error 18 | for i := 0; i < 600; i++ { 19 | if i > 0 { 20 | fmt.Printf("\r connect(%v) data store", i) 21 | } 22 | 23 | err = datastore.GetStore().Open() 24 | 25 | if err == nil { 26 | pgDB = datastore.GetStore().GetDB() 27 | break 28 | } 29 | 30 | if i == 599 { // no more attempts 31 | logging.Logger.Error("Failed to connect to the database. Shutting the server down") 32 | return err 33 | } 34 | 35 | time.Sleep(1 * time.Second) 36 | } 37 | if err := migrateDatabase(pgDB); err != nil { 38 | return fmt.Errorf("error while migrating schema: %v", err) 39 | } 40 | if err := datastore.OpenBlockStore(); err != nil { 41 | return fmt.Errorf("error while opening block store: %v", err) 42 | } 43 | 44 | return nil 45 | } 46 | 47 | func migrateDatabase(db *gorm.DB) error { 48 | goose.Init() 49 | 50 | sqlDB, err := db.DB() 51 | if err != nil { 52 | return err 53 | } 54 | 55 | goose.Migrate(sqlDB) 56 | return nil 57 | } 58 | -------------------------------------------------------------------------------- /code/go/0chain.net/validatorcore/storage/handler.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "os" 7 | "runtime/pprof" 8 | "time" 9 | 10 | "github.com/0chain/blobber/code/go/0chain.net/core/common" 11 | "github.com/didip/tollbooth/v6/limiter" 12 | "github.com/spf13/viper" 13 | ) 14 | 15 | const ( 16 | RequestPerSecond = 10 17 | DefualtExpirationTime = time.Minute * 5 18 | ) 19 | 20 | var lmt *limiter.Limiter 21 | 22 | func DumpGoRoutines(ctx context.Context, r *http.Request) (interface{}, error) { 23 | _ = pprof.Lookup("goroutine").WriteTo(os.Stdout, 1) 24 | return "success", nil 25 | } 26 | 27 | func ConfigureRateLimiter() { 28 | rps := viper.GetFloat64("rate_limiters.request_per_second") 29 | if rps <= 0 { 30 | rps = RequestPerSecond 31 | } 32 | 33 | tokenExpirettl := viper.GetDuration("rate_limiters.default_token_expire_duration") 34 | if tokenExpirettl <= 0 { 35 | tokenExpirettl = DefualtExpirationTime 36 | } 37 | 38 | isProxy := viper.GetBool("rate_limiters.proxy") 39 | ipLookups := []string{"RemoteAddr", "X-Forwarded-For", "X-Real-IP"} 40 | 41 | if isProxy { 42 | ipLookups = []string{"X-Forwarded-For", "RemoteAddr", "X-Real-IP"} 43 | } 44 | 45 | lmt = common.GetRateLimiter(rps, ipLookups, true, tokenExpirettl) 46 | } 47 | 48 | func RateLimit(handler common.ReqRespHandlerf) common.ReqRespHandlerf { 49 | return common.RateLimit(handler, lmt) 50 | } 51 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/datastore/README.md: -------------------------------------------------------------------------------- 1 | # How to auto migrate database schema 2 | 3 | ## what is `version`. how it works 4 | Given a version number TABLE.INDEX.COLUMN, increment the: 5 | 6 | - TABLE version when you add/drop any table, 7 | - INDEX version when you add/drop/update and index 8 | - COLUMN version when you add/drop/update any column 9 | 10 | NB: current schema that is created by sql scripts is versioned as `0.0.0`. 11 | 12 | ## How to add a new version 13 | 14 | ### Migrate table/column in gorm.AutoMigrate 15 | if migration works with gorm.AutoMigrate, please use it to migrate. 16 | - update your model 17 | - added your model in [AutoMigrate](postgres.go#L76)if it doesn't exists 18 | ``` 19 | db.AutoMigrate(&Migration{},&YourModel{}) 20 | ``` 21 | 22 | ### Migrate index/constraints manually if it is not easy to do in `AutoMigrate` 23 | - create a new `Migration` with scripts 24 | - append it in [releases](postgres_releases.go#L4) 25 | ``` 26 | var releases = []Migration{ 27 | { 28 | Version: "0.1.0", 29 | CreatedAt: time.Date(2021, 10, 15, 0, 0, 0, 0, time.UTC), 30 | Scripts: []string{ 31 | "CREATE INDEX idx_allocation_path ON reference_objects (allocation_id,path);", 32 | }, 33 | }, 34 | { 35 | Version: "0.1.1", 36 | CreatedAt: time.Date(2021, 10, 16, 0, 0, 0, 0, time.UTC), 37 | Scripts: []string{ 38 | "sql1", 39 | "sql2", 40 | }, 41 | }, 42 | } 43 | ``` -------------------------------------------------------------------------------- /docker.local/conductor-config/0chain_validator.yaml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | 3 | # delegate wallet (must be set) 4 | delegate_wallet: '1746b06bb09f55ee01b33b5e2e055d6cc7a900cb57c0a3a5eaabb8a0e7745802' 5 | # maximum allowed number of stake holders 6 | num_delegates: 50 7 | # service charge of related blobber 8 | service_charge: 0.10 9 | 10 | block_worker: http://198.18.0.98:9091 11 | 12 | rate_limiters: 13 | # Rate limiters will use this duration to clean unused token buckets. 14 | # If it is 0 then token will expire in 10 years. 15 | default_token_expire_duration: 5m 16 | # If blobber is behind some proxy eg. nginx, cloudflare, etc. 17 | proxy: true 18 | 19 | logging: 20 | level: "debug" 21 | console: true # printing log to console is only supported in development mode 22 | 23 | healthcheck: 24 | frequency: 60s # send healthcheck to miners every 60 seconds 25 | 26 | server_chain: 27 | id: "0afc093ffb509f059c55478bc1a60351cef7b4e9c008a53a6cc8241ca8617dfe" 28 | owner: "edb90b850f2e7e7cbd0a1fa370fdcc5cd378ffbec95363a7bc0e5a98b8ba5759" 29 | genesis_block: 30 | id: "ed79cae70d439c11258236da1dfa6fc550f7cc569768304623e8fbd7d70efae4" 31 | signature_scheme: "bls0chain" 32 | # integration tests related configurations 33 | integration_tests: 34 | # address of the server 35 | address: host.docker.internal:15210 36 | # lock_interval used by nodes to request server to connect to blockchain 37 | # after start 38 | lock_interval: 1s 39 | -------------------------------------------------------------------------------- /config/0chain_validator.yaml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | 3 | # delegate wallet (must be set) 4 | delegate_wallet: '9c693cb14f29917968d6e8c909ebbea3425b4c1bc64b6732cadc2a1869f49be9' 5 | # maximum allowed number of stake holders 6 | num_delegates: 1 7 | # service charge of related blobber 8 | service_charge: 1 #TODO CHECK IF 0 WORKS WELL 9 | 10 | block_worker: https://dev.0chain.net/dns 11 | 12 | rate_limiters: 13 | # Rate limiters will use this duration to clean unused token buckets. 14 | # If it is 0 then token will expire in 10 years. 15 | default_token_expire_duration: 5m 16 | # If blobber is behind some proxy eg. nginx, cloudflare, etc. 17 | proxy: true 18 | 19 | logging: 20 | level: "error" 21 | console: true # printing log to console is only supported in development mode 22 | 23 | healthcheck: 24 | frequency: 50m # send healthcheck to miners every 60 seconds 25 | 26 | server_chain: 27 | id: "0afc093ffb509f059c55478bc1a60351cef7b4e9c008a53a6cc8241ca8617dfe" 28 | owner: "edb90b850f2e7e7cbd0a1fa370fdcc5cd378ffbec95363a7bc0e5a98b8ba5759" 29 | genesis_block: 30 | id: "ed79cae70d439c11258236da1dfa6fc550f7cc569768304623e8fbd7d70efae4" 31 | signature_scheme: "bls0chain" 32 | # integration tests related configurations 33 | integration_tests: 34 | # address of the server 35 | address: host.docker.internal:15210 36 | # lock_interval used by nodes to request server to connect to blockchain 37 | # after start 38 | lock_interval: 1s 39 | -------------------------------------------------------------------------------- /code/go/0chain.net/core/common/pagination.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "net/url" 5 | "strconv" 6 | ) 7 | 8 | const DefaultQueryLimit = 20 9 | 10 | type Pagination struct { 11 | Offset int 12 | Limit int 13 | IsDescending bool 14 | } 15 | 16 | func GetOffsetLimitOrderParam(values url.Values) (Pagination, error) { 17 | var ( 18 | offsetString = values.Get("offset") 19 | limitString = values.Get("limit") 20 | sort = values.Get("sort") 21 | 22 | limit = DefaultQueryLimit 23 | offset = 0 24 | isDescending = false 25 | err error 26 | ) 27 | 28 | if offsetString != "" { 29 | offset, err = strconv.Atoi(offsetString) 30 | if err != nil { 31 | 32 | return Pagination{Limit: DefaultQueryLimit}, NewError("invalid_parameters", "offset parameter is not valid") 33 | } 34 | } 35 | 36 | if limitString != "" { 37 | limit, err = strconv.Atoi(limitString) 38 | if err != nil { 39 | return Pagination{Limit: DefaultQueryLimit}, NewError("invalid_parameters", "limit parameter is not valid") 40 | } 41 | 42 | if limit > DefaultQueryLimit { 43 | limit = DefaultQueryLimit 44 | } 45 | } 46 | 47 | if sort != "" { 48 | switch sort { 49 | case "desc": 50 | isDescending = true 51 | case "asc": 52 | isDescending = false 53 | default: 54 | return Pagination{Limit: DefaultQueryLimit}, err 55 | } 56 | } 57 | 58 | return Pagination{Offset: offset, Limit: limit, IsDescending: isDescending}, nil 59 | } 60 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/handler/grpc_getallocation_component_test.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | // func TestBlobberGRPCService_GetAllocation(t *testing.T) { 4 | // bClient, tdController := setupGrpcTests(t) 5 | 6 | // if err := tdController.AddGetAllocationTestData(); err != nil { 7 | // t.Fatal(err) 8 | // } 9 | 10 | // testCases := []struct { 11 | // name string 12 | // input *blobbergrpc.GetAllocationRequest 13 | // expectedTx string 14 | // expectingError bool 15 | // }{ 16 | // { 17 | // name: "Success", 18 | // input: &blobbergrpc.GetAllocationRequest{ 19 | // Id: "exampleTransaction", 20 | // }, 21 | // expectedTx: "exampleTransaction", 22 | // expectingError: false, 23 | // }, 24 | // { 25 | // name: "UnknownAllocation", 26 | // input: &blobbergrpc.GetAllocationRequest{ 27 | // Id: "exampleTransaction1", 28 | // }, 29 | // expectedTx: "", 30 | // expectingError: true, 31 | // }, 32 | // } 33 | 34 | // for _, tc := range testCases { 35 | // getAllocationResp, err := bClient.GetAllocation(context.Background(), tc.input) 36 | // if err != nil { 37 | // if !tc.expectingError { 38 | // t.Fatal(err) 39 | // } 40 | // continue 41 | // } 42 | 43 | // if tc.expectingError { 44 | // t.Fatal("expected error") 45 | // } 46 | 47 | // if getAllocationResp.Allocation.Tx != tc.expectedTx { 48 | // t.Fatal("response with wrong allocation transaction") 49 | // } 50 | // } 51 | // } 52 | -------------------------------------------------------------------------------- /docker.local/bin/build.blobber-integration-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | GIT_COMMIT=$(git rev-list -1 HEAD) 5 | echo $GIT_COMMIT 6 | 7 | echo "1> set DOCKER_IMAGE & DOCKER_BUILD" 8 | if [ -z "$DOCKER_BUILD" ]; then 9 | if [ "x86_64" != "$(uname -m)" ]; then 10 | #docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use 11 | DOCKER_BUILD="buildx build --platform linux/arm64" 12 | else 13 | DOCKER_BUILD="build" 14 | fi 15 | fi 16 | 17 | if [ -z "$DOCKER_IMAGE_BASE" ]; then 18 | DOCKER_IMAGE_BASE="blobber_base" 19 | fi 20 | 21 | if [ -z "$DOCKER_IMAGE_BLOBBER" ]; then 22 | DOCKER_IMAGE_BLOBBER="-t blobber" 23 | fi 24 | 25 | echo " DOCKER_BUILD=$DOCKER_BUILD" 26 | echo " DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE" 27 | echo " DOCKER_IMAGE_SWAGGER=$DOCKER_IMAGE_SWAGGER" 28 | echo " DOCKER_IMAGE_BLOBBER=$DOCKER_IMAGE_BLOBBER" 29 | 30 | echo "" 31 | 32 | DOCKER_BUILDKIT=1 docker $DOCKER_BUILD --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE -f docker.local/IntegrationTestsValidator.Dockerfile . -t validator --network host 33 | DOCKER_BUILDKIT=1 docker $DOCKER_BUILD --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE -f docker.local/IntegrationTestsBlobber.Dockerfile . -t blobber --network host 34 | 35 | for i in $(seq 1 6); 36 | do 37 | BLOBBER=$i docker-compose -p blobber$i -f docker.local/docker-compose.yml build --force-rm 38 | done 39 | 40 | docker.local/bin/sync_clock.sh 41 | -------------------------------------------------------------------------------- /code/go/0chain.net/core/common/handler/worker.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | blobConfig "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" 8 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler" 9 | valConfig "github.com/0chain/blobber/code/go/0chain.net/validatorcore/config" 10 | 11 | "github.com/0chain/blobber/code/go/0chain.net/core/common" 12 | "github.com/0chain/blobber/code/go/0chain.net/core/logging" 13 | "go.uber.org/zap" 14 | ) 15 | 16 | func StartHealthCheck(ctx context.Context, provider common.ProviderType) { 17 | 18 | var t time.Duration 19 | 20 | switch provider { 21 | case common.ProviderTypeBlobber: 22 | t = blobConfig.Configuration.HealthCheckWorkerFreq 23 | case common.ProviderTypeValidator: 24 | t = valConfig.Configuration.HealthCheckWorkerFreq 25 | 26 | } 27 | 28 | for { 29 | select { 30 | case <-ctx.Done(): 31 | return 32 | case <-time.After(t): 33 | go func() { 34 | start := time.Now() 35 | 36 | txnHash, err := handler.SendHealthCheck(provider) 37 | end := time.Now() 38 | if err == nil { 39 | logging.Logger.Info("success to send heartbeat", zap.String("txn_hash", txnHash), zap.Time("start", start), zap.Time("end", end), zap.Duration("duration", end.Sub(start))) 40 | } else { 41 | logging.Logger.Warn("failed to send heartbeat", zap.String("txn_hash", txnHash), zap.Time("start", start), zap.Time("end", end), zap.Duration("duration", end.Sub(start)), zap.Error(err)) 42 | } 43 | }() 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/handler/handler_middlewares.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | 7 | "github.com/0chain/blobber/code/go/0chain.net/core/logging" 8 | "go.uber.org/zap" 9 | ) 10 | 11 | func UseCors(h http.Handler) http.Handler { 12 | 13 | allowedMethods := []string{"GET", "HEAD", "POST", "PUT", 14 | "DELETE", "OPTIONS"} 15 | 16 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 17 | defer func() { 18 | if err := recover(); err != nil { 19 | escapedUrl := sanitizeString(r.URL.String()) 20 | logging.Logger.Error("[recover]http", zap.String("url", escapedUrl), zap.Any("err", err)) 21 | } 22 | }() 23 | 24 | w.Header().Add("Access-Control-Allow-Headers", "*") 25 | w.Header().Add("Access-Control-Allow-Origin", "*") 26 | w.Header().Add("Access-Control-Allow-Methods", strings.Join(allowedMethods, ", ")) 27 | 28 | // return directly for preflight request 29 | if r.Method == http.MethodOptions { 30 | w.Header().Add("Access-Control-Max-Age", "3600") 31 | w.WriteHeader(http.StatusNoContent) 32 | return 33 | } 34 | 35 | h.ServeHTTP(w, r) 36 | }) 37 | } 38 | 39 | func UseRecovery(h http.Handler) http.Handler { 40 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 41 | defer func() { 42 | if err := recover(); err != nil { 43 | escapedUrl := sanitizeString(r.URL.String()) 44 | logging.Logger.Error("[recover]http", zap.String("url", escapedUrl), zap.Any("err", err)) 45 | } 46 | }() 47 | 48 | h.ServeHTTP(w, r) 49 | }) 50 | } 51 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/filestore/misc.go: -------------------------------------------------------------------------------- 1 | package filestore 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" 7 | ) 8 | 9 | // For an allocation 4c9bad252272bc6e3969be637610d58f3ab2ff8ca336ea2fadd6171fc68fdd56, providing dirlevel [1,2] would 10 | // return string {mount_point}/4/c9/bad252272bc6e3969be637610d58f3ab2ff8ca336ea2fadd6171fc68fdd56 11 | // Similarly for dirlevel [1,2,3,4] it would return 12 | // {mount_point}/4/c9/bad/2522/72bc6e3969be637610d58f3ab2ff8ca336ea2fadd6171fc68fdd56 13 | var getDirLevelsForAllocations = func() []int { // TODO make return values uint 14 | return []int{2, 1} // default 15 | } 16 | 17 | var getDirLevelsForFiles = func() []int { 18 | return []int{2, 2, 1} // default 19 | } 20 | 21 | func validateDirLevels() error { 22 | if len(config.Configuration.AllocDirLevel) > 0 { 23 | getDirLevelsForAllocations = func() []int { return config.Configuration.AllocDirLevel } 24 | } 25 | 26 | if len(config.Configuration.FileDirLevel) > 0 { 27 | getDirLevelsForFiles = func() []int { return config.Configuration.FileDirLevel } 28 | } 29 | 30 | var s int 31 | for _, i := range getDirLevelsForAllocations() { 32 | s += i 33 | } 34 | if s >= 64 || s <= 0 { 35 | return errors.New("allocation directory levels sum should be in range 0= 64 || s <= 0 { 43 | return errors.New("files directory levels has sum should be in range 0 Blobber : List command on a directory 7 | Blobber --> Client : Directory meta data and children \n\ 8 | meta data has path hash 9 | Client -> OtherBlobbers : List command on same directory 10 | OtherBlobbers --> Client : Directory meta data and children \n\ 11 | meta data has path hash 12 | Client -> Client : checks for the path hash on meta data to be same 13 | Client -> Client : Merge outputs from all blobbers to get true structure \n\ 14 | Figure out the missing files in each Blobber 15 | 16 | loop each missing file 17 | alt file not present 18 | Client -> OtherBlobbers : Request for missing file 19 | OtherBlobbers --> Client : requested file 20 | end 21 | Client -> Blobber : File upload, commit, wm 22 | Blobber --> Client : Success 23 | Blobber -> Blockchain : Redeem wm 24 | end 25 | end 26 | 27 | group blobber lost data - after commit to Blockchain 28 | Blobber -> Blockchain : Request to opt out of allocation 29 | Blockchain -> Blockchain : Freeze challenges, \n\ 30 | fail all pending challenges immediately \n\ 31 | wipes blobber state for allocation \n\ 32 | gives back 60% allocation stake and goes to Client 33 | loop each missing file 34 | alt file not present 35 | Client -> OtherBlobbers : Request for missing file 36 | OtherBlobbers --> Client : requested file 37 | end 38 | Client -> Blobber : File upload 39 | Blobber --> Client : Success 40 | 41 | end 42 | note right 43 | This heavy lifting can be done by 44 | payer of the allocation (0Box) 45 | but writemarker should be from owner of the allocation 46 | end note 47 | end 48 | Client -> Blobber : Commit the repair, write marker 49 | Blobber -> Blockchain : Redeem writemarker 50 | @enduml -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/reference/hashnode.go: -------------------------------------------------------------------------------- 1 | package reference 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" 7 | "github.com/0chain/blobber/code/go/0chain.net/core/common" 8 | "github.com/0chain/errors" 9 | ) 10 | 11 | // LoadRootHashnode load root node with its descendant nodes 12 | func LoadRootHashnode(ctx context.Context, allocationID string) (*Hashnode, error) { 13 | 14 | tx := datastore.GetStore().GetTransaction(ctx) 15 | 16 | db := tx.Raw(` 17 | SELECT allocation_id, type, name, path, validation_root, fixed_merkle_root, actual_file_hash, chunk_size,size,actual_file_size, parent_path 18 | FROM reference_objects 19 | WHERE allocation_id = ? 20 | ORDER BY level desc, path`, allocationID) 21 | 22 | rows, err := db.Rows() 23 | if err != nil { 24 | return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) 25 | } 26 | 27 | defer rows.Close() 28 | 29 | nodes := make(map[string]*Hashnode) 30 | for rows.Next() { 31 | 32 | node := &Hashnode{} 33 | err = db.ScanRows(rows, node) 34 | if err != nil { 35 | return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) 36 | } 37 | 38 | _, ok := nodes[node.Path] 39 | if ok { 40 | return nil, common.ErrDuplicatedNode 41 | } 42 | 43 | nodes[node.Path] = node 44 | 45 | parent, ok := nodes[node.ParentPath] 46 | if ok { 47 | parent.AddChild(node) 48 | } 49 | 50 | } 51 | 52 | // create empty dir if root is missing 53 | if len(nodes) == 0 { 54 | return &Hashnode{AllocationID: allocationID, Type: DIRECTORY, Path: "/", Name: "/", ParentPath: ""}, nil 55 | } 56 | 57 | root, ok := nodes["/"] 58 | 59 | if ok { 60 | return root, nil 61 | } 62 | 63 | return nil, common.ErrMissingRootNode 64 | } 65 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Non-compete BSD-3-Clause 2 | 3 | Copyright © 2020 0Chain,LLC. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, 6 | are permitted provided that the following conditions are met: 7 | 8 | 1. Redistributions of source code must retain the above copyright notice, 9 | this list of conditions and the following disclaimer. 10 | 11 | 2. Redistributions in binary form must reproduce the above copyright notice, 12 | this list of conditions and the following disclaimer in the documentation 13 | and/or other materials provided with the distribution. 14 | 15 | 3. Neither the name of the copyright holder nor the names of its contributors may be used 16 | to endorse or promote products derived from this software without specific prior written permission. 17 | 18 | 4. It is not permitted to provide any offering that competes with the source code and binary form 19 | or any other offering the licensor provides using the source code and binary form. 20 | 21 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS 22 | OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY 23 | AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 24 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 27 | IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 28 | OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /https/conf.d/nginx.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen [::]:80; 3 | listen 80; 4 | 5 | location ~ /.well-known/acme-challenge { 6 | allow all; 7 | root /var/www/certbot; 8 | } 9 | # location /blobber51/ { 10 | # proxy_pass http://blobber1_blobber_1:5051/; 11 | # } 12 | # location /blobber52/ { 13 | # proxy_pass http://blobber2_blobber_1:5052/; 14 | # } 15 | # location /blobber53/ { 16 | # proxy_pass http://blobber3_blobber_1:5053/; 17 | # } 18 | # location /blobber54/ { 19 | # proxy_pass http://blobber4_blobber_1:5054/; 20 | # } 21 | # location /blobber55/ { 22 | # proxy_pass http://blobber5_blobber_1:5055/; 23 | # } 24 | # location /blobber56/ { 25 | # proxy_pass http://blobber6_blobber_1:5056/; 26 | # } 27 | } 28 | 29 | # server { 30 | # listen [::]:443 ssl http2; 31 | # listen 443 ssl http2; 32 | 33 | # server_name ; 34 | 35 | # # SSL code 36 | # ssl_certificate /etc/nginx/ssl/live//fullchain.pem; 37 | # ssl_certificate_key /etc/nginx/ssl/live//privkey.pem; 38 | 39 | # root /var/www/html; 40 | 41 | # location /blobber51/ { 42 | # proxy_pass http://blobber1_blobber_1:5051/; 43 | # } 44 | # location /blobber52/ { 45 | # proxy_pass http://blobber2_blobber_1:5052/; 46 | # } 47 | # location /blobber53/ { 48 | # proxy_pass http://blobber3_blobber_1:5053/; 49 | # } 50 | # location /blobber54/ { 51 | # proxy_pass http://blobber4_blobber_1:5054/; 52 | # } 53 | # location /blobber55/ { 54 | # proxy_pass http://blobber5_blobber_1:5055/; 55 | # } 56 | # location /blobber56/ { 57 | # proxy_pass http://blobber6_blobber_1:5056/; 58 | # } 59 | # } 60 | -------------------------------------------------------------------------------- /code/go/0chain.net/core/lock/lock.go: -------------------------------------------------------------------------------- 1 | package lock 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | ) 7 | 8 | var ( 9 | // MutexCleanInterval start to clean unused mutex at specified interval 10 | MutexCleanInterval = 10 * time.Minute 11 | ) 12 | 13 | var ( 14 | lockPool = make(map[string]*Mutex) 15 | lockMutex sync.Mutex 16 | ) 17 | 18 | // Mutex a mutual exclusion lock. 19 | type Mutex struct { 20 | // usedby how objects it is used by 21 | usedby int 22 | 23 | mu *sync.RWMutex 24 | } 25 | 26 | // Lock implements Locker.Lock 27 | func (m *Mutex) Lock() { 28 | m.mu.Lock() 29 | } 30 | 31 | func (m *Mutex) RLock() { 32 | m.mu.RLock() 33 | } 34 | 35 | func (m *Mutex) RUnlock() { 36 | lockMutex.Lock() 37 | defer lockMutex.Unlock() 38 | m.usedby-- 39 | m.mu.RUnlock() 40 | } 41 | 42 | // Unlock implements Locker.Unlock, and mark mutex as unlock object 43 | func (m *Mutex) Unlock() { 44 | lockMutex.Lock() 45 | defer lockMutex.Unlock() 46 | 47 | m.usedby-- 48 | m.mu.Unlock() 49 | } 50 | 51 | // GetMutex get mutex by table and key 52 | func GetMutex(tablename, key string) *Mutex { 53 | lockKey := tablename + ":" + key 54 | lockMutex.Lock() 55 | 56 | defer lockMutex.Unlock() 57 | if eLock, ok := lockPool[lockKey]; ok { 58 | eLock.usedby++ 59 | return eLock 60 | } 61 | 62 | m := &Mutex{ 63 | usedby: 1, 64 | mu: &sync.RWMutex{}, 65 | } 66 | 67 | lockPool[lockKey] = m 68 | 69 | return m 70 | } 71 | 72 | func init() { 73 | go startWorker() 74 | } 75 | 76 | func cleanUnusedMutexs() { 77 | lockMutex.Lock() 78 | 79 | for k, v := range lockPool { 80 | if v.usedby < 1 { 81 | delete(lockPool, k) 82 | } 83 | } 84 | 85 | lockMutex.Unlock() 86 | } 87 | 88 | func startWorker() { 89 | for { 90 | time.Sleep(MutexCleanInterval) 91 | cleanUnusedMutexs() 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /.github/workflows/gosdk.yml: -------------------------------------------------------------------------------- 1 | name: "Upgrade 0Chain GoSDK" 2 | 3 | concurrency: 4 | group: "gosdk-${{ github.ref }}" 5 | cancel-in-progress: true 6 | 7 | on: 8 | workflow_dispatch: 9 | inputs: 10 | gosdk: 11 | description: 'The version/branch of 0Chain GoSDK' 12 | required: true 13 | base: 14 | description: the base branch to which PR needs to be raised 15 | required: true 16 | 17 | jobs: 18 | create-pr: 19 | runs-on: [arc-runner] 20 | steps: 21 | - name: Setup go 1.21 22 | uses: actions/setup-go@v2 23 | with: 24 | go-version: '1.21' 25 | 26 | - name: Checkout 27 | uses: actions/checkout@v4 28 | with: 29 | ref: ${{ github.event.inputs.base }} 30 | 31 | - name: Upgrade 0Chain GoSDK 32 | run: | 33 | echo "BRANCH=$(echo ${{github.event.inputs.gosdk}} | sed 's/\//-/g')" >> $GITHUB_ENV 34 | 35 | go get github.com/0chain/gosdk@${{github.event.inputs.gosdk}} 36 | go mod tidy 37 | 38 | - name: Create Pull Request 39 | uses: peter-evans/create-pull-request@v3 40 | with: 41 | base: ${{ github.event.inputs.base }} 42 | token: ${{ secrets.SVC_ACCOUNT_SECRET }} 43 | commit-message: upgrade GoSDK to ${{ github.event.inputs.gosdk }} 44 | branch: gosdk-upgrade-${{ env.BRANCH }} 45 | delete-branch: true 46 | title: Upgrade 0Chain GoSDK to ${{ github.event.inputs.gosdk }} 47 | body: | 48 | 0Chain GoSDK `${{ github.event.inputs.gosdk }}` is released. 49 | see full changelog on https://github.com/0chain/gosdk/releases/tag/${{ github.event.inputs.gosdk }} 50 | draft: false 51 | reviewers: guruhubb 52 | labels: GoSDK 53 | -------------------------------------------------------------------------------- /code/go/0chain.net/conductor/conductrpc/client.go: -------------------------------------------------------------------------------- 1 | package conductrpc 2 | 3 | import ( 4 | "net/rpc" 5 | ) 6 | 7 | // client of the conductor RPC server. 8 | type client struct { 9 | address string // RPC server address 10 | client *rpc.Client // RPC client 11 | } 12 | 13 | // newClient creates new client will be interacting 14 | // with server with given address. 15 | func newClient(address string) (c *client, err error) { 16 | if address, err = Host(address); err != nil { 17 | return 18 | } 19 | c = new(client) 20 | if c.client, err = rpc.Dial("tcp", address); err != nil { 21 | return nil, err 22 | } 23 | c.address = address 24 | return 25 | } 26 | 27 | func (c *client) dial() (err error) { //nolint:unused,deadcode // might be used later? 28 | c.client, err = rpc.Dial("tcp", c.address) 29 | return 30 | } 31 | 32 | // Address of RPC server. 33 | func (c *client) Address() string { 34 | return c.address 35 | } 36 | 37 | // 38 | // miner SC RPC 39 | // 40 | 41 | // state requests current client state using long polling strategy. E.g. 42 | // when the state had updated, then the method returns. 43 | func (c *client) state(me NodeID) (state *State, err error) { 44 | err = c.client.Call("Server.State", me, &state) 45 | for err == rpc.ErrShutdown { 46 | err = c.client.Call("Server.State", me, &state) 47 | } 48 | return 49 | } 50 | 51 | func (c *client) blobberCommitted(blobberID string) (err error) { 52 | err = c.client.Call("Server.BlobberCommitted", blobberID, nil) 53 | return 54 | } 55 | 56 | func (c *client) validationTicketGenerated(ticket ValidtorTicket) (err error) { 57 | err = c.client.Call("Server.ValidatorTicket", &ticket, nil) 58 | return 59 | } 60 | 61 | func (c *client) sendFileMetaRoot(m map[string]string) (err error) { 62 | err = c.client.Call("Server.GetFileMetaRoot", m, nil) 63 | return 64 | } 65 | -------------------------------------------------------------------------------- /.github/workflows/config_change_alert.yaml: -------------------------------------------------------------------------------- 1 | name: config_change_alert 2 | 3 | on: 4 | push: 5 | branches: 6 | - staging 7 | paths: 8 | - config/0chain_blobber.yaml 9 | - config/0chain_validator.yaml 10 | 11 | pull_request: 12 | branches: 13 | - staging 14 | types: 15 | - closed 16 | paths: 17 | - config/0chain_blobber.yaml 18 | - config/0chain_validator.yaml 19 | 20 | jobs: 21 | test: 22 | runs-on: docker-builds 23 | steps: 24 | - uses: actions/checkout@v1 25 | with: 26 | fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. 27 | 28 | - name: Get changed files using defaults 29 | id: changed-files 30 | uses: tj-actions/changed-files@v18.4 31 | 32 | - name: Run step when a 0chain_blobber.yaml file changes 33 | if: contains(steps.changed-files.outputs.modified_files, 'config/0chain_blobber.yaml') 34 | run: | 35 | echo "Your 0chain_blobber.yaml file has been modified." 36 | curl -X POST -H 'Content-type: application/json' --data '{"text":" Your 0chain_blobber.yaml config file has been modified.\n LINK: https://github.com/0chain/blobber/blob/staging/config/0chain_blobber.yaml"}' ${{ secrets.DEVOPS_CHANNEL_WEBHOOK_URL }} 37 | 38 | - name: Run step when a 0chain_validator.yaml file changes 39 | if: contains(steps.changed-files.outputs.modified_files, 'config/0chain_validator.yaml') 40 | run: | 41 | echo "Your 0chain_validator.yaml file has been modified." 42 | curl -X POST -H 'Content-type: application/json' --data '{"text":" Your 0chain_validator.yaml config file has been modified.\n LINK: https://github.com/0chain/blobber/blob/staging/config/0chain_validator.yaml"}' ${{ secrets.DEVOPS_CHANNEL_WEBHOOK_URL }} 43 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/handler/grpc_getobjectpath_component_test.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | // func TestBlobberGRPCService_GetObjectPath(t *testing.T) { 4 | // bClient, tdController := setupGrpcTests(t) 5 | // allocationTx := randString(32) 6 | 7 | // pubKey, _, signScheme := GeneratePubPrivateKey(t) 8 | // clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) 9 | 10 | // err := tdController.AddGetObjectPathTestData(allocationTx, pubKey) 11 | // if err != nil { 12 | // t.Fatal(err) 13 | // } 14 | 15 | // testCases := []struct { 16 | // name string 17 | // context metadata.MD 18 | // input *blobbergrpc.GetObjectPathRequest 19 | // expectedPath string 20 | // expectingError bool 21 | // }{ 22 | // { 23 | // name: "Success", 24 | // context: metadata.New(map[string]string{ 25 | // common.ClientHeader: "exampleOwnerId", 26 | // common.ClientSignatureHeader: clientSignature, 27 | // }), 28 | // input: &blobbergrpc.GetObjectPathRequest{ 29 | // Allocation: allocationTx, 30 | // Path: "examplePath", 31 | // BlockNum: "0", 32 | // }, 33 | // expectedPath: "/", 34 | // expectingError: false, 35 | // }, 36 | // } 37 | 38 | // for _, tc := range testCases { 39 | // ctx := context.Background() 40 | // ctx = metadata.NewOutgoingContext(ctx, tc.context) 41 | // getObjectPathResp, err := bClient.GetObjectPath(ctx, tc.input) 42 | // if err != nil { 43 | // if !tc.expectingError { 44 | // t.Fatal(err) 45 | // } 46 | // continue 47 | // } 48 | 49 | // if tc.expectingError { 50 | // t.Fatal("expected error") 51 | // } 52 | 53 | // if getObjectPathResp.ObjectPath.Path.DirMetaData.Path != tc.expectedPath { 54 | // t.Fatal("unexpected root hash from GetObjectPath rpc") 55 | // } 56 | // } 57 | // } 58 | -------------------------------------------------------------------------------- /code/go/0chain.net/conductor/config/byzantine.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/mitchellh/mapstructure" 7 | ) 8 | 9 | // The Bad is common bad / only sending configuration. 10 | type Bad struct { 11 | // By these nodes. 12 | By []NodeName `json:"by" yaml:"by" mapstructure:"by"` 13 | // Good to these nodes. 14 | Good []NodeName `json:"good" yaml:"good" mapstructure:"good"` 15 | // Bad to these nodes. 16 | Bad []NodeName `json:"bad" yaml:"bad" mapstructure:"bad"` 17 | } 18 | 19 | // Unmarshal with given name and from given map[interface{}]interface{} 20 | // by mapstructure package. 21 | func (b *Bad) Unmarshal(name string, val interface{}) (err error) { 22 | if err = mapstructure.Decode(val, b); err != nil { 23 | return fmt.Errorf("invalid '%s' argument type: %T, "+ 24 | "decoding error: %v", name, val, err) 25 | } 26 | if len(b.By) == 0 { 27 | return fmt.Errorf("empty 'by' field of '%s'", name) 28 | } 29 | return 30 | } 31 | 32 | // Is given name in given names list. 33 | func isInList(ids []NodeName, id NodeName) bool { 34 | for _, x := range ids { 35 | if x == id { 36 | return true 37 | } 38 | } 39 | return false 40 | } 41 | 42 | type Namer interface { 43 | Name(NodeID) NodeName 44 | } 45 | 46 | // IsGood returns true if the Bad is nil or given name is in Good list. 47 | func (b *Bad) IsGood(state Namer, id string) bool { 48 | return b == nil || isInList(b.Good, state.Name(NodeID(id))) 49 | } 50 | 51 | // IsBad returns true if the Bad is not nil and given name is in Bad list. 52 | func (b *Bad) IsBad(state Namer, id string) bool { 53 | return b != nil && isInList(b.Bad, state.Name(NodeID(id))) 54 | } 55 | 56 | // IsBy returns true if given name is in By list. 57 | func (b *Bad) IsBy(state Namer, id string) bool { 58 | return b != nil && isInList(b.By, state.Name(NodeID(id))) 59 | } 60 | -------------------------------------------------------------------------------- /code/go/0chain.net/core/util/secure_value.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "encoding/hex" 5 | "strings" 6 | 7 | "github.com/0chain/blobber/code/go/0chain.net/core/encryption" 8 | ) 9 | 10 | /*Hashable - anything that can provide it's hash */ 11 | type Hashable interface { 12 | GetHash() string 13 | GetHashBytes() []byte 14 | } 15 | 16 | /*Serializable interface */ 17 | type Serializable interface { 18 | Encode() []byte 19 | Decode([]byte) error 20 | } 21 | 22 | /*HashStringToBytes - convert a hex hash string to bytes */ 23 | func HashStringToBytes(hash string) []byte { 24 | hashBytes, err := hex.DecodeString(hash) 25 | if err != nil { 26 | return nil 27 | } 28 | return hashBytes 29 | } 30 | 31 | /*SecureSerializableValueI an interface that makes a serializable value secure with hashing */ 32 | type SecureSerializableValueI interface { 33 | Serializable 34 | Hashable 35 | } 36 | 37 | /*SecureSerializableValue - a proxy persisted value that just tracks the encoded bytes of a persisted value */ 38 | type SecureSerializableValue struct { 39 | Buffer []byte 40 | } 41 | 42 | /*GetHash - implement interface */ 43 | func (spv *SecureSerializableValue) GetHash() string { 44 | return ToHex(spv.GetHashBytes()) 45 | } 46 | 47 | /*ToHex - converts a byte array to hex encoding with upper case */ 48 | func ToHex(buf []byte) string { 49 | return strings.ToUpper(hex.EncodeToString(buf)) 50 | } 51 | 52 | /*GetHashBytes - implement interface */ 53 | func (spv *SecureSerializableValue) GetHashBytes() []byte { 54 | return encryption.RawHash(spv.Buffer) 55 | } 56 | 57 | /*Encode - implement interface */ 58 | func (spv *SecureSerializableValue) Encode() []byte { 59 | return spv.Buffer 60 | } 61 | 62 | /*Decode - implement interface */ 63 | func (spv *SecureSerializableValue) Decode(buf []byte) error { 64 | spv.Buffer = buf 65 | return nil 66 | } 67 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/filestore/file_manager.go: -------------------------------------------------------------------------------- 1 | package filestore 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | "sync" 9 | 10 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" 11 | "github.com/0chain/blobber/code/go/0chain.net/core/common" 12 | "github.com/0chain/blobber/code/go/0chain.net/core/encryption" 13 | ) 14 | 15 | type FileStore struct { 16 | mp string // mount point 17 | mAllocs map[string]*allocation 18 | 19 | allocMu *sync.Mutex 20 | rwMU *sync.RWMutex 21 | 22 | diskCapacity uint64 23 | } 24 | 25 | var contentHashMapLock = common.GetNewLocker() 26 | 27 | func getKey(allocID, contentHash string) string { 28 | return encryption.Hash(allocID + contentHash) 29 | } 30 | 31 | func (fs *FileStore) Initialize() (err error) { 32 | defer func() { 33 | if r := recover(); r != nil { 34 | err = r.(error) 35 | } 36 | }() 37 | 38 | fs.mp = config.Configuration.MountPoint 39 | if !fs.isMountPoint() { 40 | return fmt.Errorf("%s is not mount point", fs.mp) 41 | } 42 | 43 | if err = validateDirLevels(); err != nil { 44 | return 45 | } 46 | 47 | fs.allocMu = &sync.Mutex{} 48 | fs.rwMU = &sync.RWMutex{} 49 | fs.mAllocs = make(map[string]*allocation) 50 | 51 | if err = fs.initMap(); err != nil { 52 | return 53 | } 54 | 55 | return nil 56 | } 57 | 58 | func (fs *FileStore) IterateObjects(allocationID string, handler FileObjectHandler) error { 59 | allocDir := fs.getAllocDir(allocationID) 60 | tmpPrefix := filepath.Join(allocDir, TempDir) 61 | return filepath.Walk(allocDir, func(path string, info os.FileInfo, err error) error { 62 | if err == nil && !info.IsDir() && !strings.HasPrefix(path, tmpPrefix) { 63 | p := strings.ReplaceAll(path, allocDir, "") 64 | handler(strings.ReplaceAll(p, "/", ""), info.Size()) 65 | } 66 | return nil 67 | }) 68 | } 69 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/handler/grpc_getreferencepath_component_test.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | // func TestBlobberGRPCService_GetReferencePath(t *testing.T) { 4 | // bClient, tdController := setupGrpcTests(t) 5 | // allocationTx := randString(32) 6 | 7 | // pubKey, _, signScheme := GeneratePubPrivateKey(t) 8 | // clientSignature, _ := signScheme.Sign(encryption.Hash(allocationTx)) 9 | 10 | // err := tdController.AddGetReferencePathTestData(allocationTx, pubKey) 11 | // if err != nil { 12 | // t.Fatal(err) 13 | // } 14 | 15 | // testCases := []struct { 16 | // name string 17 | // context metadata.MD 18 | // input *blobbergrpc.GetReferencePathRequest 19 | // expectedPath string 20 | // expectingError bool 21 | // }{ 22 | // { 23 | // name: "Success", 24 | // context: metadata.New(map[string]string{ 25 | // common.ClientHeader: "exampleOwnerId", 26 | // common.ClientSignatureHeader: clientSignature, 27 | // }), 28 | // input: &blobbergrpc.GetReferencePathRequest{ 29 | // Paths: "", 30 | // Path: "/", 31 | // Allocation: allocationTx, 32 | // }, 33 | // expectedPath: "/", 34 | // expectingError: false, 35 | // }, 36 | // } 37 | 38 | // for _, tc := range testCases { 39 | // ctx := context.Background() 40 | // ctx = metadata.NewOutgoingContext(ctx, tc.context) 41 | // getReferencePathResp, err := bClient.GetReferencePath(ctx, tc.input) 42 | // if err != nil { 43 | // if !tc.expectingError { 44 | // t.Fatal(err) 45 | // } 46 | // continue 47 | // } 48 | 49 | // if tc.expectingError { 50 | // t.Fatal("expected error") 51 | // } 52 | 53 | // if getReferencePathResp.ReferencePath.MetaData.DirMetaData.Path != tc.expectedPath { 54 | // t.Fatal("unexpected path from GetReferencePath rpc") 55 | // } 56 | // } 57 | // } 58 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/filestore/tree_validation_integration_tests.go: -------------------------------------------------------------------------------- 1 | //go:build integration_tests 2 | // +build integration_tests 3 | 4 | package filestore 5 | 6 | import ( 7 | "io" 8 | 9 | crpc "github.com/0chain/blobber/code/go/0chain.net/conductor/conductrpc" 10 | "github.com/0chain/blobber/code/go/0chain.net/core/logging" 11 | "go.uber.org/zap" 12 | ) 13 | 14 | func (v *validationTreeProof) GetMerkleProofOfMultipleIndexes(r io.ReadSeeker, nodesSize int64, startInd, endInd int) ( 15 | [][][]byte, [][]int, error) { 16 | nodes, indexes, err := v.getMerkleProofOfMultipleIndexes(r, nodesSize, startInd, endInd) 17 | if err != nil { 18 | return nodes, indexes, err 19 | } 20 | 21 | state := crpc.Client().State() 22 | if state == nil { 23 | return nodes, indexes, err 24 | } 25 | 26 | logging.Logger.Debug("[conductor] just after getMerkleProofOfMultipleIndexes", 27 | zap.Bool("miss_up_download", state.MissUpDownload), 28 | zap.Int("start_ind", startInd), 29 | zap.Int("end_ind", endInd), 30 | zap.Int64("nodes_size", nodesSize), 31 | zap.Int("output_nodes_size", len(nodes)), 32 | zap.Int("output_indexes_size", len(indexes)), 33 | zap.Any("nodes", nodes), 34 | zap.Any("indexes", indexes), 35 | ) 36 | 37 | if state.MissUpDownload { 38 | logging.Logger.Debug("miss up/download", 39 | zap.Bool("miss_up_download", state.MissUpDownload), 40 | zap.Int("start_ind", startInd), 41 | zap.Int("end_ind", endInd), 42 | zap.Int64("nodes_size", nodesSize), 43 | zap.Int("output_nodes_size", len(nodes)), 44 | zap.Int("output_indexes_size", len(indexes)), 45 | zap.Any("nodes", nodes), 46 | zap.Any("indexes", indexes), 47 | ) 48 | 49 | for i := range nodes { 50 | nodes[i][0] = []byte("wrong data") 51 | } 52 | 53 | for i := range indexes { 54 | indexes[i][0] = 0 55 | } 56 | } 57 | 58 | return nodes, indexes, err 59 | } -------------------------------------------------------------------------------- /docker.local/bin/test.swagger.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | echo "1> set DOCKER_IMAGE & DOCKER_BUILD" 6 | if [ -z "$DOCKER_BUILD" ]; then 7 | if [ "x86_64" != "$(uname -m)" ]; then 8 | #docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use 9 | DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64" 10 | else 11 | DOCKER_BUILD="build" 12 | fi 13 | fi 14 | 15 | if [ -z "$DOCKER_IMAGE_BASE" ]; then 16 | DOCKER_IMAGE_BASE="blobber_base" 17 | fi 18 | 19 | if [ -z "$DOCKER_IMAGE_SWAGGER" ]; then 20 | DOCKER_IMAGE_SWAGGER="swagger" 21 | fi 22 | 23 | # cmd="build --build-arg DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE" 24 | 25 | dockerfile="docker.local/Dockerfile.swagger" 26 | platform="" 27 | INTERACTIVE="" 28 | 29 | # for arg in "$@" 30 | # do 31 | # case $arg in 32 | # -m1|--m1|m1) 33 | # echo "The build will be performed for Apple M1 chip" 34 | # cmd="buildx build --platform linux/amd64" 35 | # dockerfile="docker.local/build.unit_test/Dockerfile.m1" 36 | # platform="--platform=linux/amd64" 37 | # shift 38 | # ;; 39 | # esac 40 | # done 41 | 42 | echo "2> Build swagger_test docker image." 43 | DOCKER_BUILDKIT=1 docker $DOCKER_BUILD --progress=plain --build-arg DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE -f $dockerfile . -t $DOCKER_IMAGE_SWAGGER --network host 44 | 45 | echo "3> Create swagger.yaml file." 46 | docker run $platform $INTERACTIVE -v $(pwd):/codecov $DOCKER_IMAGE_SWAGGER bash -c "\ 47 | cd /codecov/code/go/0chain.net/; \ 48 | swagger generate spec -w . -m -o swagger.yaml; \ 49 | sed -i \"s/in\:\ form/in\:\ formData/g\" ./swagger.yaml; \ 50 | yq -i '(.paths.*.*.parameters.[] | select(.in == \"formData\") | select(.type == \"object\")).type = \"file\"' swagger.yaml; \ 51 | swagger generate markdown -f swagger.yaml --output=swagger.md" 52 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/allocation/rollback.go: -------------------------------------------------------------------------------- 1 | package allocation 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" 7 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" 8 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" 9 | ) 10 | 11 | func ApplyRollback(ctx context.Context, allocationID string) error { 12 | 13 | db := datastore.GetStore().GetTransaction(ctx) 14 | 15 | // delete all is_precommit rows 16 | 17 | err := db.Model(&reference.Ref{}).Unscoped(). 18 | Delete(&reference.Ref{}, 19 | "allocation_id=? AND is_precommit=? AND deleted_at IS NULL", 20 | allocationID, true).Error 21 | if err != nil { 22 | return err 23 | } 24 | 25 | // err = db.Exec("UPDATE file_stats SET deleted_at=NULL WHERE ref_id IN (SELECT id FROM reference_objects WHERE allocation_id=? AND deleted_at IS NOT NULL)", allocationID).Error 26 | // revive soft deleted ref rows 27 | err = db.Exec("UPDATE reference_objects SET deleted_at=NULL,is_precommit=? WHERE allocation_id=? AND deleted_at IS NOT NULL", false, allocationID).Error 28 | return err 29 | } 30 | 31 | func ApplyRollbackV2(ctx context.Context, allocationID, allocationRoot string, timestamp int64) error { 32 | db := datastore.GetStore().GetTransaction(ctx) 33 | 34 | err := db.Model(&reference.Ref{}).Unscoped(). 35 | Delete(&reference.Ref{}, 36 | "allocation_id=? AND allocation_root=? AND created_at=? AND deleted_at is NULL", 37 | allocationID, allocationRoot, timestamp).Error 38 | if err != nil { 39 | return err 40 | } 41 | 42 | return db.Exec("UPDATE reference_objects SET deleted_at=NULL WHERE allocation_id=? AND deleted_at IS NOT NULL", allocationID).Error 43 | } 44 | 45 | func CommitRollback(allocationID string) error { 46 | 47 | err := filestore.GetFileStore().DeletePreCommitDir(allocationID) 48 | return err 49 | } 50 | -------------------------------------------------------------------------------- /docker.aws/build.blobber/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | postgres: 4 | image: postgres:14 5 | volumes: 6 | - /0chain/${AGENT_DIR}/data/postgresql:/var/lib/postgresql/data 7 | networks: 8 | default: 9 | 10 | container_name: ${AGENT_CONTAINER}-postgres 11 | 12 | postgres-post: 13 | image: postgres:14 14 | environment: 15 | POSTGRES_PORT: 5432 16 | POSTGRES_HOST: postgres 17 | POSTGRES_USER: postgres 18 | volumes: 19 | - /0chain/${AGENT_DIR}/bin:/blobber/bin 20 | - /0chain/${AGENT_DIR}/sql:/blobber/sql 21 | command: bash /blobber/bin/postgres-entrypoint.sh 22 | links: 23 | - postgres:postgres 24 | 25 | container_name: ${AGENT_CONTAINER}-postgres-post 26 | 27 | blobber: 28 | build: 29 | context: ${DOCKER_CODEBASE} 30 | dockerfile: ./docker.aws/build.blobber/Dockerfile 31 | 32 | 33 | container_name: ${AGENT_CONTAINER} 34 | 35 | environment: 36 | - DOCKER= true 37 | depends_on: 38 | - postgres-post 39 | links: 40 | - postgres-post:postgres-post 41 | volumes: 42 | - /0chain/${AGENT_DIR}/config:/0chain/config 43 | - /0chain/${AGENT_DIR}/files:/0chain/files 44 | - /0chain/${AGENT_DIR}/data:/0chain/data 45 | - /0chain/${AGENT_DIR}/log:/0chain/log 46 | 47 | ports: 48 | - ${AGENT_PORT}:${AGENT_PORT} 49 | 50 | command: /0chain/bin/blobber --deployment_mode 0 --keys_file /0chain/config/blobber.txt --nodes_file nodes --files_dir /0chain/files --log_dir /0chain/log --db_dir /0chain/data 51 | 52 | networks: 53 | default: 54 | 55 | logging: 56 | driver: json-file 57 | options: 58 | max-file: "10" 59 | max-size: 10M 60 | 61 | image: ${IMAGE_NAME}:${IMAGE_TAG} 62 | 63 | user: ${ZCHAIN_USER}:${ZCHAIN_GROUP} 64 | restart: ${RESTART_POLICY} 65 | 66 | networks: 67 | default: 68 | driver: bridge 69 | 70 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobber/config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" 8 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" 9 | "github.com/0chain/blobber/code/go/0chain.net/core/transaction" 10 | "github.com/spf13/viper" 11 | ) 12 | 13 | func setupConfig(configDir string, deploymentMode int) { 14 | fmt.Print("> load config") 15 | // setup default 16 | config.SetupDefaultConfig() 17 | 18 | // setup config file 19 | config.SetupConfig(configDir) 20 | 21 | if mountPoint != "" { 22 | config.Configuration.MountPoint = mountPoint 23 | } else { 24 | config.Configuration.MountPoint = viper.GetString("storage.files_dir") 25 | } 26 | 27 | if config.Configuration.MountPoint == "" { 28 | panic("Please specify mount point in flag or config file") 29 | } 30 | transaction.MinConfirmation = config.Configuration.MinConfirmation 31 | config.ReadConfig(deploymentMode) 32 | fmt.Print(" [OK]\n") 33 | } 34 | 35 | func reloadConfig() error { 36 | fmt.Print("> reload config") 37 | 38 | return datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { 39 | s, ok := config.Get(ctx, datastore.GetStore().GetDB()) 40 | if ok { 41 | if err := s.CopyTo(&config.Configuration); err != nil { 42 | return err 43 | } 44 | fmt.Print(" [OK]\n") 45 | return nil 46 | } 47 | 48 | config.Configuration.Capacity = viper.GetInt64("capacity") 49 | 50 | config.Configuration.NumDelegates = viper.GetInt("num_delegates") 51 | config.Configuration.ReadPrice = viper.GetFloat64("read_price") 52 | config.Configuration.ServiceCharge = viper.GetFloat64("service_charge") 53 | config.Configuration.WritePrice = viper.GetFloat64("write_price") 54 | 55 | if err := config.Update(ctx, datastore.GetStore().GetDB()); err != nil { 56 | return err 57 | } 58 | 59 | fmt.Print(" [OK]\n") 60 | return nil 61 | 62 | }) 63 | } 64 | -------------------------------------------------------------------------------- /code/go/0chain.net/validatorcore/storage/challenge_handler_integration_tests.go: -------------------------------------------------------------------------------- 1 | //go:build integration_tests 2 | // +build integration_tests 3 | 4 | package storage 5 | 6 | import ( 7 | "context" 8 | "fmt" 9 | "net/http" 10 | 11 | "github.com/0chain/blobber/code/go/0chain.net/conductor/conductrpc" 12 | "github.com/0chain/blobber/code/go/0chain.net/core/node" 13 | ) 14 | 15 | func ChallengeHandler(ctx context.Context, r *http.Request) (interface{}, error, bool) { 16 | state := conductrpc.Client().State() 17 | 18 | if state.AdversarialValidator.ID == node.Self.ID && state.AdversarialValidator.FailValidChallenge { 19 | challengeRequest, _, err := NewChallengeRequest(r) 20 | if err != nil { 21 | return nil, err, true 22 | } 23 | 24 | challengeObj, err := NewChallengeObj(ctx, challengeRequest) 25 | if err != nil { 26 | return nil, err, true 27 | } 28 | 29 | if len(challengeObj.Validators) > 2 { 30 | res, err := InvalidValidationTicket(challengeObj, fmt.Errorf("Challenge failed by adversarial validator")) 31 | return res, err, true 32 | } 33 | } else if state.AdversarialValidator.ID == node.Self.ID && state.AdversarialValidator.DenialOfService { 34 | return nil, nil, false 35 | } else if state.AdversarialValidator.ID == node.Self.ID && state.AdversarialValidator.PassAllChallenges { 36 | challengeRequest, challengeHash, err := NewChallengeRequest(r) 37 | if err != nil { 38 | return nil, err, true 39 | } 40 | 41 | challengeObj, err := NewChallengeObj(ctx, challengeRequest) 42 | if err != nil { 43 | return nil, err, true 44 | } 45 | 46 | res, err := ValidValidationTicket(challengeObj, challengeRequest.ChallengeID, challengeHash) 47 | return res, err, true 48 | } 49 | 50 | res, err := challengeHandler(ctx, r) 51 | 52 | if state.NotifyOnValidationTicketGeneration { 53 | conductrpc.Client().ValidatorTicket(conductrpc.ValidtorTicket{ 54 | ValidatorId: node.Self.ID, 55 | }) 56 | } 57 | 58 | return res, err, true 59 | } 60 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/handler/auth_ticket.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/0chain/blobber/code/go/0chain.net/core/encryption" 7 | "github.com/0chain/blobber/code/go/0chain.net/core/node" 8 | "github.com/0chain/common/core/common" 9 | "github.com/0chain/gosdk/zcncore" 10 | "net/http" 11 | ) 12 | 13 | // swagger:model AuthTicketResponse 14 | type AuthTicketResponse struct { 15 | AuthTicket string `json:"auth_ticket"` 16 | } 17 | 18 | // swagger:route GET /v1/auth/generate GetAuthTicket 19 | // Generate blobber authentication ticket. 20 | // 21 | // Generate and retrieve blobber authentication ticket signed by the blobber's signature. Used by restricted blobbers to enable users to use them to host allocations. 22 | // 23 | // parameters: 24 | // 25 | // +name: Zbox-Signature 26 | // in: header 27 | // type: string 28 | // description: Digital signature to verify that the sender is 0box service. 29 | // +name: client_id 30 | // type: string 31 | // in: query 32 | // description: Client ID is used as a payload to the token generated. The token represents a signed version of this string by the blobber's private key. 33 | // 34 | // responses: 35 | // 36 | // 200: AuthTicketResponse 37 | func GenerateAuthTicket(ctx context.Context, r *http.Request) (interface{}, error) { 38 | 39 | clientID := r.URL.Query().Get("client_id") 40 | if clientID == "" { 41 | return nil, common.NewError("missing_client_id", "client_id is required") 42 | } 43 | 44 | round := r.URL.Query().Get("round") 45 | 46 | payload := encryption.Hash(fmt.Sprintf("%s_%s", clientID, round)) 47 | 48 | if isActivated, err := zcncore.IsHardforkActivated("hermes"); err != nil || !isActivated { 49 | payload = clientID 50 | } 51 | 52 | signature, err := node.Self.Sign(payload) 53 | if err != nil { 54 | return nil, common.NewError("signature_failed", "signature failed") 55 | } 56 | 57 | return &AuthTicketResponse{ 58 | AuthTicket: signature, 59 | }, nil 60 | } 61 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/readmarker/protocol.go: -------------------------------------------------------------------------------- 1 | package readmarker 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "time" 7 | 8 | "github.com/0chain/blobber/code/go/0chain.net/core/common" 9 | "github.com/0chain/blobber/code/go/0chain.net/core/node" 10 | "github.com/0chain/blobber/code/go/0chain.net/core/transaction" 11 | "gorm.io/gorm" 12 | ) 13 | 14 | type ReadRedeem struct { 15 | ReadMarker *ReadMarker `json:"read_marker"` 16 | } 17 | 18 | func (r *ReadMarkerEntity) BeforeCreate(tx *gorm.DB) error { 19 | r.CreatedAt = time.Now() 20 | r.UpdatedAt = r.CreatedAt 21 | return nil 22 | } 23 | 24 | func (r *ReadMarkerEntity) BeforeSave(tx *gorm.DB) error { 25 | r.UpdatedAt = time.Now() 26 | return nil 27 | } 28 | 29 | // PendNumBlocks Return number of blocks pending to be redeemed 30 | func (rme *ReadMarkerEntity) PendNumBlocks() (pendNumBlocks int64, err error) { 31 | if rme.LatestRM == nil { 32 | return 0, common.NewErrorf("rme_pend_num_blocks", "missing latest read marker (nil)") 33 | } 34 | 35 | pendNumBlocks = rme.LatestRM.ReadCounter - rme.LatestRedeemedRC 36 | return 37 | } 38 | 39 | // RedeemReadMarker redeems the read marker. 40 | func (rme *ReadMarkerEntity) RedeemReadMarker(ctx context.Context) (err error) { 41 | // Depreciated 42 | return 43 | } 44 | 45 | func GetLatestReadMarkerEntityFromChain(clientID, allocID string) (*ReadMarker, error) { 46 | params := map[string]string{ 47 | "blobber": node.Self.ID, 48 | "client": clientID, 49 | "allocation": allocID, 50 | } 51 | 52 | latestRMBytes, err := transaction.MakeSCRestAPICall( 53 | transaction.STORAGE_CONTRACT_ADDRESS, "/latestreadmarker", params) 54 | 55 | if err != nil { 56 | return nil, err 57 | } 58 | latestRM := &ReadMarker{} 59 | err = json.Unmarshal(latestRMBytes, latestRM) 60 | if err != nil { 61 | return nil, err 62 | } 63 | if latestRM.ClientID == "" { // RMs are not yet redeemed and thus it is empty 64 | return nil, nil 65 | } 66 | return latestRM, nil 67 | } 68 | -------------------------------------------------------------------------------- /docker.aws/build.blobber/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG image_tag 2 | FROM golang:1.19-alpine3.17 as blobber_build 3 | 4 | RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep 5 | 6 | # Install Herumi's cryptography 7 | RUN apk add gmp gmp-dev openssl-dev && \ 8 | cd /tmp && \ 9 | wget -O - https://github.com/herumi/mcl/archive/master.tar.gz | tar xz && \ 10 | wget -O - https://github.com/herumi/bls/archive/master.tar.gz | tar xz && \ 11 | mv mcl* mcl && \ 12 | mv bls* bls && \ 13 | make -C mcl -j $(nproc) lib/libmclbn256.so install && \ 14 | cp mcl/lib/libmclbn256.so /usr/local/lib && \ 15 | make MCL_DIR=$(pwd)/mcl -C bls -j $(nproc) install && \ 16 | rm -R /tmp/mcl && \ 17 | rm -R /tmp/bls 18 | 19 | ENV SRC_DIR=/0chain 20 | ENV GO111MODULE=on 21 | 22 | # Download the dependencies: 23 | # Will be cached if we don't change mod/sum files 24 | COPY ./code/go/0chain.net/go.mod ./code/go/0chain.net/go.sum $SRC_DIR/go/0chain.net/ 25 | RUN cd $SRC_DIR/go/0chain.net && go mod download 26 | 27 | #Add the source code 28 | ADD ./code/go/0chain.net $SRC_DIR/go/0chain.net 29 | 30 | WORKDIR $SRC_DIR/go/0chain.net/blobber 31 | 32 | ARG image_tag 33 | ARG go_build_mode 34 | ARG go_bls_tag 35 | RUN CGO_ENABLED=1 go build -v -tags ${go_build_mode} -tags ${go_bls_tag} -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=${image_tag}" 36 | 37 | # Copy the build artifact into a minimal runtime image: 38 | FROM golang:1.19-alpine3.17 39 | RUN apk add gmp gmp-dev openssl-dev 40 | COPY --from=blobber_build /usr/local/lib/libmcl*.so \ 41 | /usr/local/lib/libbls*.so \ 42 | /usr/local/lib/ 43 | ENV APP_DIR=/0chain 44 | WORKDIR $APP_DIR 45 | COPY --from=blobber_build $APP_DIR/go/0chain.net/blobber/blobber $APP_DIR/bin/blobber 46 | 47 | #Store all files and run environment under 0chain userid. 48 | RUN addgroup -g 2000 -S 0chain && adduser -u 2000 -S 0chain -G 0chain 49 | USER 0chain:0chain 50 | -------------------------------------------------------------------------------- /docker.aws/build.validator/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG image_tag 2 | FROM golang:1.19-alpine3.17 as validator_build 3 | 4 | RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep 5 | 6 | # Install Herumi's cryptography 7 | RUN apk add gmp gmp-dev openssl-dev && \ 8 | cd /tmp && \ 9 | wget -O - https://github.com/herumi/mcl/archive/master.tar.gz | tar xz && \ 10 | wget -O - https://github.com/herumi/bls/archive/master.tar.gz | tar xz && \ 11 | mv mcl* mcl && \ 12 | mv bls* bls && \ 13 | make -C mcl -j $(nproc) lib/libmclbn256.so install && \ 14 | cp mcl/lib/libmclbn256.so /usr/local/lib && \ 15 | make MCL_DIR=$(pwd)/mcl -C bls -j $(nproc) install && \ 16 | rm -R /tmp/mcl && \ 17 | rm -R /tmp/bls 18 | 19 | ENV SRC_DIR=/0chain 20 | ENV GO111MODULE=on 21 | 22 | # Download the dependencies: 23 | # Will be cached if we don't change mod/sum files 24 | COPY ./code/go/0chain.net/go.mod ./code/go/0chain.net/go.sum $SRC_DIR/go/0chain.net/ 25 | RUN cd $SRC_DIR/go/0chain.net && go mod download 26 | 27 | #Add the source code 28 | ADD ./code/go/0chain.net $SRC_DIR/go/0chain.net 29 | 30 | WORKDIR $SRC_DIR/go/0chain.net/validator 31 | 32 | ARG image_tag 33 | ARG go_build_mode 34 | ARG go_bls_tag 35 | RUN CGO_ENABLED=1 go build -v -tags ${go_build_mode} -tags ${go_bls_tag} -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=${image_tag}" 36 | 37 | # Copy the build artifact into a minimal runtime image: 38 | FROM golang:1.19-alpine3.17 39 | RUN apk add gmp gmp-dev openssl-dev 40 | COPY --from=validator_build /usr/local/lib/libmcl*.so \ 41 | /usr/local/lib/libbls*.so \ 42 | /usr/local/lib/ 43 | ENV APP_DIR=/0chain 44 | WORKDIR $APP_DIR 45 | COPY --from=validator_build $APP_DIR/go/0chain.net/validator/validator $APP_DIR/bin/validator 46 | 47 | #Store all files and run environment under 0chain userid. 48 | RUN addgroup -g 2000 -S 0chain && adduser -u 2000 -S 0chain -G 0chain 49 | USER 0chain:0chain 50 | -------------------------------------------------------------------------------- /code/go/0chain.net/core/chain/entity.go: -------------------------------------------------------------------------------- 1 | package chain 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/0chain/blobber/code/go/0chain.net/core/common" 7 | "github.com/0chain/blobber/code/go/0chain.net/core/config" 8 | "github.com/spf13/viper" 9 | ) 10 | 11 | /*ServerChain - the chain object of the chain the server is responsible for */ 12 | var ServerChain *Chain 13 | 14 | /*SetServerChain - set the server chain object */ 15 | func SetServerChain(c *Chain) { 16 | ServerChain = c 17 | } 18 | 19 | /*GetServerChain - returns the chain object for the server chain */ 20 | func GetServerChain() *Chain { 21 | return ServerChain 22 | } 23 | 24 | /*Chain - data structure that holds the chain data*/ 25 | type Chain struct { 26 | ID string 27 | Version string 28 | CreationDate common.Timestamp 29 | OwnerID string 30 | ParentChainID string 31 | BlockWorker string 32 | ZauthServer string 33 | 34 | GenesisBlockHash string 35 | } 36 | 37 | /*Validate - implementing the interface */ 38 | func (c *Chain) Validate(ctx context.Context) error { 39 | if common.IsEmpty(c.ID) { 40 | return common.InvalidRequest("chain id is required") 41 | } 42 | if common.IsEmpty(c.OwnerID) { 43 | return common.InvalidRequest("owner id is required") 44 | } 45 | return nil 46 | } 47 | 48 | // NewChainFromConfig - create a new chain from config 49 | func NewChainFromConfig() *Chain { 50 | chain := Provider() 51 | chain.ID = common.ToKey(config.Configuration.ChainID) 52 | chain.OwnerID = viper.GetString("server_chain.owner") 53 | chain.BlockWorker = viper.GetString("block_worker") 54 | chain.ZauthServer = viper.GetString("zauth_server") 55 | return chain 56 | } 57 | 58 | /*Provider - entity provider for chain object */ 59 | func Provider() *Chain { 60 | c := &Chain{} 61 | c.Version = "1.0" 62 | c.InitializeCreationDate() 63 | return c 64 | } 65 | 66 | /*InitializeCreationDate - intializes the creation date for the chain */ 67 | func (c *Chain) InitializeCreationDate() { 68 | c.CreationDate = common.Now() 69 | } 70 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobber/worker.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" 8 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/challenge" 9 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" 10 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" 11 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler" 12 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/readmarker" 13 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" 14 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" 15 | "github.com/0chain/blobber/code/go/0chain.net/core/logging" 16 | 17 | "go.uber.org/zap" 18 | ) 19 | 20 | func setupWorkers(ctx context.Context) { 21 | handler.SetupWorkers(ctx) 22 | challenge.SetupWorkers(ctx) 23 | readmarker.SetupWorkers(ctx) 24 | writemarker.SetupWorkers(ctx) 25 | allocation.StartUpdateWorker(ctx, config.Configuration.UpdateAllocationsInterval) 26 | allocation.StartFinalizeWorker(ctx, config.Configuration.FinalizeAllocationsInterval) 27 | allocation.SetupWorkers(ctx) 28 | challenge.SetupChallengeCleanUpWorker(ctx) 29 | challenge.SetupChallengeTimingsCleanupWorker(ctx) 30 | stats.SetupStatsWorker(ctx) 31 | updateStorageScConfigWorker(ctx) 32 | } 33 | 34 | // startRefreshSettings sync settings from blockchain 35 | func startRefreshSettings(ctx context.Context) { 36 | const REPEAT_DELAY = 60 * 3 // 3 minutes 37 | for { 38 | select { 39 | case <-ctx.Done(): 40 | return 41 | case <-time.After(REPEAT_DELAY * time.Second): 42 | err := datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { 43 | _, e := config.ReloadFromChain(ctx, datastore.GetStore().GetDB()) 44 | return e 45 | }) 46 | if err != nil { 47 | logging.Logger.Warn("failed to refresh blobber settings from chain", zap.Error(err)) 48 | continue 49 | } 50 | 51 | logging.Logger.Info("success to refresh blobber settings from chain") 52 | 53 | } 54 | 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /code/go/0chain.net/core/common/context.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "os" 7 | "os/signal" 8 | "syscall" 9 | "time" 10 | 11 | "github.com/0chain/blobber/code/go/0chain.net/core/logging" 12 | 13 | "go.uber.org/zap" 14 | ) 15 | 16 | var ErrStop = NewError("stop_error", "Stop signal error") 17 | 18 | var rootContext context.Context 19 | var rootCancel context.CancelFunc 20 | 21 | /*SetupRootContext - sets up the root context that can be used to shutdown the node */ 22 | func SetupRootContext(nodectx context.Context) { 23 | rootContext, rootCancel = context.WithCancel(nodectx) 24 | // TODO: This go routine is not needed. Workaround for the "vet" error 25 | done := make(chan bool) 26 | go func() { 27 | select { //nolint:gosimple // need more time to verify 28 | case <-done: 29 | //Logger.Info("Shutting down all workers...") 30 | rootCancel() 31 | } 32 | }() 33 | } 34 | 35 | /*GetRootContext - get the root context for the server 36 | * This will be used to control shutting down the server but cleanup all the workers 37 | */ 38 | func GetRootContext() context.Context { 39 | return rootContext 40 | } 41 | 42 | /*Done - call this when the program needs to stop and notify all workers */ 43 | func Done() { 44 | // Logger.Info("Initiating shutdown...") 45 | rootCancel() 46 | // TODO: How do we ensure every worker is completed any shutdown sequence before we finally shut down 47 | // the server using server.Shutdown(ctx) 48 | } 49 | 50 | /*HandleShutdown - handles various shutdown signals */ 51 | func HandleShutdown(server *http.Server) { 52 | c := make(chan os.Signal, 1) 53 | signal.Notify(c, syscall.SIGINT, syscall.SIGQUIT) 54 | <-c 55 | ctx, cancelf := context.WithTimeout(context.Background(), 10*time.Second) 56 | if err := server.Shutdown(ctx); err != nil && err != http.ErrServerClosed { 57 | logging.Logger.Error("server failed to gracefully shuts down", zap.Error(err)) 58 | } 59 | cancelf() 60 | Done() 61 | // wait for 20 seconds to allow the workers and database to shutdown 62 | time.Sleep(20 * time.Second) 63 | } 64 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/reference/object.go: -------------------------------------------------------------------------------- 1 | package reference 2 | 3 | import ( 4 | "context" 5 | "path/filepath" 6 | 7 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" 8 | "github.com/0chain/blobber/code/go/0chain.net/core/common" 9 | ) 10 | 11 | func DeleteObject(ctx context.Context, rootRef *Ref, allocationID, objPath string, ts common.Timestamp) error { 12 | likePath := objPath + "/%" 13 | if objPath == "/" { 14 | likePath = "/%" 15 | } 16 | 17 | db := datastore.GetStore().GetTransaction(ctx) 18 | 19 | err := db.Exec("UPDATE reference_objects SET is_precommit=? WHERE allocation_id=? AND path != ? AND (path=? OR path LIKE ?)", true, allocationID, "/", objPath, likePath).Error 20 | if err != nil { 21 | return err 22 | } 23 | 24 | err = db.Delete(&Ref{}, "allocation_id=? AND path != ? AND (path=? OR path LIKE ?)", 25 | allocationID, "/", objPath, likePath).Error 26 | 27 | if err != nil { 28 | return err 29 | } 30 | if objPath == "/" { 31 | rootRef.Children = nil 32 | rootRef.HashToBeComputed = true 33 | rootRef.childrenLoaded = true 34 | rootRef.UpdatedAt = ts 35 | return nil 36 | } 37 | parentPath, deleteFileName := filepath.Split(objPath) 38 | rootRef.UpdatedAt = ts 39 | fields, err := common.GetPathFields(parentPath) 40 | if err != nil { 41 | return err 42 | } 43 | 44 | dirRef := rootRef 45 | 46 | for _, name := range fields { 47 | var found bool 48 | for _, ref := range dirRef.Children { 49 | if ref.Name == name { 50 | ref.HashToBeComputed = true 51 | ref.childrenLoaded = true 52 | ref.UpdatedAt = ts 53 | found = true 54 | dirRef = ref 55 | break 56 | } 57 | } 58 | 59 | if !found { 60 | return common.NewError("invalid_reference_path", "Reference path has invalid references") 61 | } 62 | } 63 | 64 | for i, child := range dirRef.Children { 65 | basePath := filepath.Base(child.Path) 66 | if basePath == deleteFileName || child.Path == objPath { 67 | dirRef.RemoveChild(i) 68 | break 69 | } 70 | } 71 | 72 | rootRef.HashToBeComputed = true 73 | rootRef.childrenLoaded = true 74 | return nil 75 | } 76 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobber/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" 5 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" 6 | "github.com/0chain/blobber/code/go/0chain.net/core/common" 7 | "github.com/0chain/blobber/code/go/0chain.net/core/logging" 8 | "github.com/0chain/blobber/code/go/0chain.net/core/node" 9 | ) 10 | 11 | func main() { 12 | 13 | parseFlags() 14 | 15 | setupConfig(configDir, deploymentMode) 16 | 17 | setupLogging() 18 | 19 | if err := setupDatabase(); err != nil { 20 | logging.Logger.Error("Error setting up data store" + err.Error()) 21 | panic(err) 22 | } 23 | 24 | if err := reloadConfig(); err != nil { 25 | logging.Logger.Error("Error reloading config" + err.Error()) 26 | panic(err) 27 | } 28 | 29 | if err := setupNode(); err != nil { 30 | logging.Logger.Error("Error setting up blobber node " + err.Error()) 31 | panic(err) 32 | } 33 | 34 | if err := setupServerChain(); err != nil { 35 | logging.Logger.Error("Error setting up server chain " + err.Error()) 36 | panic(err) 37 | } 38 | 39 | // Initialize after server chain is setup. 40 | if err := setupFileStore(); err != nil { 41 | logging.Logger.Error("Error setting up file store" + err.Error()) 42 | panic(err) 43 | } 44 | 45 | // prepare is to configure more. 46 | // when enabled "// +build integration_tests", this sets blobber for conductor tests. 47 | prepareBlobber(node.Self.ID) 48 | 49 | go func() { 50 | if err := registerOnChain(); err != nil { 51 | logging.Logger.Error("Error register on blockchain" + err.Error()) 52 | panic(err) 53 | } 54 | 55 | common.SetBlobberRegistered(true) 56 | }() 57 | 58 | if err := setStorageScConfigFromChain(); err != nil { 59 | logging.Logger.Error("Error setStorageScConfigFromChain" + err.Error()) 60 | panic(err) 61 | } 62 | 63 | if recoverTrie { 64 | logging.Logger.Info("Recovering trie") 65 | allocation.RecoverTrie(config.Configuration.RecoverAllocations) 66 | } 67 | 68 | // todo: activate this when gRPC functionalities are implemented 69 | // go startGRPCServer() 70 | 71 | startHttpServer() 72 | } 73 | -------------------------------------------------------------------------------- /code/go/0chain.net/blobbercore/handler/health.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "sync" 5 | 6 | coreTxn "github.com/0chain/gosdk/core/transaction" 7 | 8 | "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" 9 | "github.com/0chain/blobber/code/go/0chain.net/core/common" 10 | "github.com/0chain/blobber/code/go/0chain.net/core/logging" 11 | "github.com/0chain/blobber/code/go/0chain.net/core/transaction" 12 | "go.uber.org/zap" 13 | ) 14 | 15 | var ( 16 | // blobberHealthCheckError use it on stats page 17 | blobberHealthCheckError error 18 | blobberHealthCheckMutex sync.RWMutex 19 | ) 20 | 21 | func setBlobberHealthCheckError(err error) { 22 | blobberHealthCheckMutex.Lock() 23 | blobberHealthCheckError = err 24 | blobberHealthCheckMutex.Unlock() 25 | } 26 | 27 | func getBlobberHealthCheckError() error { 28 | blobberHealthCheckMutex.RLock() 29 | err := blobberHealthCheckError 30 | blobberHealthCheckMutex.RUnlock() 31 | return err 32 | } 33 | 34 | func BlobberHealthCheck() (string, error) { 35 | if config.Configuration.Capacity == 0 { 36 | 37 | setBlobberHealthCheckError(ErrBlobberHasRemoved) 38 | return "", ErrBlobberHasRemoved 39 | } 40 | 41 | _, _, _, txn, err := coreTxn.SmartContractTxn(transaction.STORAGE_CONTRACT_ADDRESS, coreTxn.SmartContractTxnData{ 42 | Name: transaction.BLOBBER_HEALTH_CHECK, 43 | InputArgs: common.Now(), 44 | }, true) 45 | if err != nil || txn == nil { 46 | logging.Logger.Error("Failed to health check blobber on the blockchain", 47 | zap.Error(err)) 48 | setBlobberHealthCheckError(err) 49 | 50 | return "", err 51 | } 52 | 53 | setBlobberHealthCheckError(nil) 54 | 55 | return txn.Hash, nil 56 | } 57 | 58 | func ValidatorHealthCheck() (string, error) { 59 | _, _, _, txn, err := coreTxn.SmartContractTxn(transaction.STORAGE_CONTRACT_ADDRESS, coreTxn.SmartContractTxnData{ 60 | Name: transaction.VALIDATOR_HEALTH_CHECK, 61 | InputArgs: common.Now(), 62 | }, true) 63 | 64 | if err != nil || txn == nil { 65 | logging.Logger.Error("Failed to health check validator on the blockchain", 66 | zap.Error(err)) 67 | return "", err 68 | } 69 | 70 | return txn.Hash, err 71 | } 72 | -------------------------------------------------------------------------------- /dev.local/README.md: -------------------------------------------------------------------------------- 1 | # blobber development guide 2 | 3 | 4 | 5 | 6 | ## install `postgres:14` in docker as shared database for blobbers and validators 7 | 8 | ``` 9 | ********************************************** 10 | Welcome to blobber/validator development CLI 11 | ********************************************** 12 | 13 | 14 | Please select which blobber/validator you will work on: 15 | 1) 1 16 | 2) 2 17 | 3) 3 18 | 4) clean all 19 | #? 1 20 | 21 | ********************************************** 22 | Blobber/Validator 1 23 | ********************************************** 24 | 25 | Please select what you will do: 26 | 1) install postgres 27 | 2) start blobber 28 | 3) start validator 29 | #? 1 30 | ``` 31 | 32 | 33 | It will check and install a `blobber_postgres` container as shared database for all blobbers and validators, and initialized database `blobber_meta$i` and user `blobber_user$i` 34 | 35 | ## run local validator instance 36 | 37 | ``` 38 | ********************************************** 39 | Welcome to blobber/validator development CLI 40 | ********************************************** 41 | 42 | 43 | Please select which blobber/validator you will work on: 44 | 1) 1 45 | 2) 2 46 | 3) 3 47 | 4) clean all 48 | #? 1 49 | 50 | ********************************************** 51 | Blobber/Validator 1 52 | ********************************************** 53 | 54 | Please select what you will do: 55 | 1) install postgres 3) start validator 56 | 2) start blobber 4) clean 57 | #? 3 58 | 59 | ``` 60 | 61 | 62 | ## run local blobber instance 63 | 64 | ``` 65 | ********************************************** 66 | Welcome to blobber/validator development CLI 67 | ********************************************** 68 | 69 | 70 | Please select which blobber/validator you will work on: 71 | 1) 1 72 | 2) 2 73 | 3) 3 74 | 4) clean all 75 | #? 1 76 | 77 | ********************************************** 78 | Blobber/Validator 1 79 | ********************************************** 80 | 81 | Please select what you will do: 82 | 1) install postgres 3) start validator 83 | 2) start blobber 4) clean 84 | #? 2 85 | 86 | ``` --------------------------------------------------------------------------------