├── worker ├── context.go ├── repair_object.go ├── replicate_object.go ├── delete_object.go ├── fetcher.go ├── storage_health.go └── base.go ├── config ├── config_q4m.go ├── config_redis.go └── config.go ├── cli ├── stf-worker-delete_object │ └── stf-worker-delete_object.go ├── stf-worker-repair_object │ └── stf-worker-repair_object.go ├── stf-worker-storage_health │ └── stf-worker-storage_health.go ├── stf-worker-replicate_object │ └── stf-worker-replicate_object.go ├── storage │ └── storage.go ├── stf-worker │ └── stf-worker.go └── dispatcher │ └── dispatcher.go ├── test.sh ├── api ├── api.go ├── queue.go ├── context_test.go ├── deleted_object.go ├── queue_redis.go ├── queue_q4m.go ├── bucket.go ├── storage.go ├── context.go ├── cluster.go ├── entity.go └── object.go ├── database_test.go ├── eg └── Procfile ├── etc └── config.gcfg ├── database.go ├── stf.go ├── data └── data.go ├── .travis.yml ├── murmurhash.go ├── stftest ├── queue_q4m_test.go ├── queue_redis_test.go ├── client_test.go └── all_test.go ├── drone ├── periodic.go ├── drone_test.go ├── minion.go └── drone.go ├── dispatcher ├── response.go ├── id_gen.go ├── bucket.go ├── dispatcher.go └── object.go ├── install.sh ├── README.md ├── debug.go ├── cache └── memcache.go ├── constants.go ├── server_storage.go └── stf.sql /worker/context.go: -------------------------------------------------------------------------------- 1 | package worker 2 | -------------------------------------------------------------------------------- /config/config_q4m.go: -------------------------------------------------------------------------------- 1 | // +build q4m 2 | 3 | package config 4 | 5 | type QueueConfig DatabaseConfig 6 | -------------------------------------------------------------------------------- /config/config_redis.go: -------------------------------------------------------------------------------- 1 | // +build redis 2 | 3 | package config 4 | 5 | import ( 6 | "github.com/vmihailenco/redis/v2" 7 | ) 8 | 9 | type QueueConfig redis.Options 10 | -------------------------------------------------------------------------------- /cli/stf-worker-delete_object/stf-worker-delete_object.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/stf-storage/go-stf-server/worker" 5 | ) 6 | 7 | func main() { 8 | worker.NewDeleteObjectWorker().Run() 9 | } 10 | -------------------------------------------------------------------------------- /cli/stf-worker-repair_object/stf-worker-repair_object.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/stf-storage/go-stf-server/worker" 5 | ) 6 | 7 | func main() { 8 | worker.NewRepairObjectWorker().Run() 9 | } 10 | -------------------------------------------------------------------------------- /cli/stf-worker-storage_health/stf-worker-storage_health.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/stf-storage/go-stf-server/worker" 5 | ) 6 | 7 | func main() { 8 | worker.NewStorageHealthWorker().Run() 9 | } 10 | -------------------------------------------------------------------------------- /cli/stf-worker-replicate_object/stf-worker-replicate_object.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/stf-storage/go-stf-server/worker" 5 | ) 6 | 7 | func main() { 8 | worker.NewReplicateObjectWorker().Run() 9 | } 10 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | if [ -z "$SKIP_BUILD" ]; then 5 | export SKIP_DEPS=1 6 | sh ./build.sh 7 | fi 8 | 9 | if [ -z "$STF_QUEUE_TYPE" ]; then 10 | STF_QUEUE_TYPE=q4m 11 | fi 12 | 13 | export STF_DEBUG="1:`pwd`/test.log" 14 | export STF_HOME=`pwd` 15 | exec go test -tags $STF_QUEUE_TYPE -run Basic -v github.com/stf-storage/go-stf-server/stftest -------------------------------------------------------------------------------- /api/api.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | type BaseApi struct { 4 | ctx ContextWithApi 5 | } 6 | 7 | type ApiHolder interface { 8 | BucketApi() *Bucket 9 | DeletedObjectApi() *DeletedObject 10 | EntityApi() *Entity 11 | ObjectApi() *Object 12 | QueueApi() QueueApiInterface 13 | StorageApi() *Storage 14 | StorageClusterApi() *StorageCluster 15 | } 16 | 17 | func (self *BaseApi) Ctx() ContextWithApi { return self.ctx } 18 | -------------------------------------------------------------------------------- /database_test.go: -------------------------------------------------------------------------------- 1 | package stf 2 | 3 | import ( 4 | "github.com/stf-storage/go-stf-server/config" 5 | "log" 6 | ) 7 | 8 | func ExampleDatabase() { 9 | config := &config.DatabaseConfig{ 10 | "mysql", 11 | "root", 12 | "password", 13 | "tcp(127.0.0.1:3306)", 14 | "stf", 15 | } 16 | 17 | db, err := ConnectDB(config) 18 | if err != nil { 19 | log.Fatalf("Failed to connecto database: %s", err) 20 | } 21 | 22 | db.QueryRow("SELECT ...") 23 | } 24 | -------------------------------------------------------------------------------- /api/queue.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "github.com/stf-storage/go-stf-server/config" 5 | ) 6 | 7 | type WorkerArg struct { 8 | Arg string 9 | CreatedAt int 10 | } 11 | 12 | type ContextForQueueApi interface { 13 | Config() *config.Config 14 | } 15 | 16 | type QueueApiInterface interface { 17 | Enqueue(string, string) error 18 | Dequeue(string, int) (*WorkerArg, error) 19 | } 20 | 21 | type BaseQueueApi struct { 22 | ctx ContextForQueueApi 23 | } 24 | -------------------------------------------------------------------------------- /eg/Procfile: -------------------------------------------------------------------------------- 1 | storage1: ../bin/storage --listen :9001 --root=storage1 2 | storage2: ../bin/storage --listen :9002 --root=storage2 3 | storage3: ../bin/storage --listen :9003 --root=storage3 4 | dispatcher1: ../bin/dispatcher --config ../etc/config.gcfg --id 1001 --listen :8081 5 | dispatcher2: ../bin/dispatcher --config ../etc/config.gcfg --id 1002 --listen :8082 6 | dispatcher3: ../bin/dispatcher --config ../etc/config.gcfg --id 1003 --listen :8083 7 | worker: ../bin/stf-worker --config ../etc/config.gcfg 8 | -------------------------------------------------------------------------------- /etc/config.gcfg: -------------------------------------------------------------------------------- 1 | [Global] 2 | Debug = true 3 | 4 | [Dispatcher] 5 | ServerId = 1 6 | Listen = :8080 7 | AccessLog = accesslog.%Y%m%d 8 | AccessLogLink = accesslog 9 | 10 | [MainDB] 11 | Username = root 12 | 13 | [QueueDB "1"] 14 | ConnectString = tcp(127.0.0.1:6379) 15 | 16 | ; [QueueDB "1"] 17 | ; ConnectString = tcp(127.0.0.1:3306) 18 | ; Dbname = stf_queue 19 | 20 | ; [QueueDB "2"] 21 | ; ConnectString = tcp(...) 22 | ; Dbname = stf_queue 23 | 24 | [Memcached] 25 | Servers = 127.0.0.1:11211 -------------------------------------------------------------------------------- /api/context_test.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "github.com/stf-storage/go-stf-server/config" 5 | "log" 6 | ) 7 | 8 | func ExampleScopedContext() { 9 | config, err := config.BootstrapConfig() 10 | if err != nil { 11 | log.Fatalf("Failed to bootstrap config: %s", err) 12 | } 13 | 14 | for { 15 | ctx := NewContext(config) 16 | rollback, err := ctx.TxnBegin() 17 | if err != nil { 18 | log.Fatalf("Failed to start transaction: %s", err) 19 | } 20 | defer rollback() 21 | 22 | // Do stuff... 23 | ctx.TxnCommit() 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /cli/storage/storage.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "github.com/stf-storage/go-stf-server" 6 | "log" 7 | "os" 8 | ) 9 | 10 | func main() { 11 | var listen string 12 | var root string 13 | 14 | pwd, err := os.Getwd() 15 | if err != nil { 16 | log.Fatalf("Could not determine current working directory") 17 | } 18 | 19 | flag.StringVar(&listen, "listen", ":9000", "Interface/port to listen to") 20 | flag.StringVar(&root, "root", pwd, "Path to store/fetch files to/from") 21 | flag.Parse() 22 | 23 | ss := stf.NewStorageServer(listen, root) 24 | ss.Start() 25 | } 26 | -------------------------------------------------------------------------------- /database.go: -------------------------------------------------------------------------------- 1 | package stf 2 | 3 | import ( 4 | "database/sql" 5 | "errors" 6 | "fmt" 7 | "github.com/stf-storage/go-stf-server/config" 8 | ) 9 | 10 | type DB struct{ sql.DB } 11 | 12 | func ConnectDB(config *config.DatabaseConfig) (*DB, error) { 13 | dsn, err := config.Dsn() 14 | if err != nil { 15 | return nil, err 16 | } 17 | 18 | Debugf("Connecting to database %s", dsn) 19 | db, err := sql.Open(config.Dbtype, dsn) 20 | if err != nil { 21 | return nil, errors.New( 22 | fmt.Sprintf("Failed to connect to database: %s", err), 23 | ) 24 | } 25 | 26 | return &DB{*db}, nil 27 | } 28 | -------------------------------------------------------------------------------- /stf.go: -------------------------------------------------------------------------------- 1 | package stf 2 | 3 | import ( 4 | "crypto/sha1" 5 | "fmt" 6 | "io" 7 | "log" 8 | "math/rand" 9 | "os" 10 | "strconv" 11 | "time" 12 | ) 13 | 14 | const VERSION = "0.0.1" 15 | 16 | func init() { 17 | rand.Seed(time.Now().UTC().UnixNano()) 18 | } 19 | 20 | func GenerateRandomId(hint string, length int) string { 21 | h := sha1.New() 22 | io.WriteString(h, hint) 23 | io.WriteString(h, strconv.FormatInt(time.Now().UTC().UnixNano(), 10)) 24 | return (fmt.Sprintf("%x", h.Sum(nil)))[0:length] 25 | } 26 | 27 | func GetHome() string { 28 | home := os.Getenv("STF_HOME") 29 | if home == "" { 30 | var err error 31 | home, err = os.Getwd() 32 | if err != nil { 33 | log.Fatalf("Failed to get home from env and Getwd: %s", err) 34 | } 35 | } 36 | return home 37 | } 38 | -------------------------------------------------------------------------------- /data/data.go: -------------------------------------------------------------------------------- 1 | package data 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type StfObject struct { 8 | Id uint64 9 | CreatedAt int 10 | UpdatedAt time.Time 11 | } 12 | 13 | type Object struct { 14 | StfObject 15 | BucketId uint64 16 | Name string 17 | InternalName string 18 | Size int64 19 | Status int 20 | } 21 | 22 | type Bucket struct { 23 | StfObject 24 | Name string 25 | } 26 | 27 | type StorageCluster struct { 28 | StfObject 29 | Name string 30 | Mode int 31 | SortHint uint32 32 | } 33 | 34 | type Storage struct { 35 | StfObject 36 | ClusterId uint64 37 | Uri string 38 | Mode int 39 | } 40 | 41 | type Entity struct { 42 | ObjectId uint64 43 | StorageId uint64 44 | Status int 45 | } 46 | 47 | type DeletedObject Object 48 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.1 4 | - 1.2 5 | - tip 6 | install: 7 | - sh build.sh 8 | script: 9 | - SKIP_BUILD=1 sh test.sh || (echo "=== test.log ===" ; cat `pwd`/test.log ; echo "=== worker.log ==="; cat `pwd`/worker.log; exit 1) 10 | after_success: 11 | - go get github.com/mattn/goveralls 12 | - go get code.google.com/p/go.tools/cmd/cover 13 | - export PATH="$TRAVIS_BUILD_DIR/bin:$HOME/gopath/bin:$PATH" 14 | - goveralls -v -repotoken $COVERALLS_TOKEN -coverprofile=c.out -tags $STF_QUEUE_TYPE -run Basic -v github.com/stf-storage/go-stf-server/stftest 15 | env: 16 | global: 17 | secure: "fhKD8yvdUoK5JJhSe4wte+spwUIg8C6VQqpYh31thL9DyfeuYvGC6JwWlEkm92XKTrKe+uJYICh5TlJSI+t3VNNwPV8CsQp+qlzn1MWjnfz9lj7/gfN62Hd4j9cQKeHlDQi2MdfGR0Eq3LcyB4yvZoLGtvoN18W9SQUSp7xJDvM=" 18 | matrix: 19 | - STF_QUEUE_TYPE=redis 20 | -------------------------------------------------------------------------------- /murmurhash.go: -------------------------------------------------------------------------------- 1 | package stf 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "log" 7 | ) 8 | 9 | // Imeplementation of murmurHash (ver 1) in go 10 | func MurmurHash(data []byte) uint32 { 11 | const m uint32 = 0x5bd1e995 12 | const r uint8 = 16 13 | var length uint32 = uint32(len(data)) 14 | var h uint32 = length * m 15 | 16 | nblocks := int(length / 4) 17 | buf := bytes.NewBuffer(data) 18 | for i := 0; i < nblocks; i++ { 19 | var x uint32 20 | err := binary.Read(buf, binary.LittleEndian, &x) 21 | if err != nil { 22 | log.Fatal("Failed to read from buffer") 23 | } 24 | h += x 25 | h *= m 26 | h ^= h >> r 27 | } 28 | 29 | tailIndex := nblocks * 4 30 | switch length & 3 { 31 | case 3: 32 | h += uint32(data[tailIndex+2]) << 16 33 | fallthrough 34 | case 2: 35 | h += uint32(data[tailIndex+1]) << 8 36 | fallthrough 37 | case 1: 38 | h += uint32(data[tailIndex]) 39 | h *= m 40 | h ^= h >> r 41 | } 42 | 43 | h *= m 44 | h ^= h >> 10 45 | h *= m 46 | h ^= h >> 17 47 | 48 | return h 49 | } 50 | -------------------------------------------------------------------------------- /api/deleted_object.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "github.com/stf-storage/go-stf-server" 5 | "github.com/stf-storage/go-stf-server/data" 6 | ) 7 | 8 | type DeletedObject struct { 9 | *BaseApi 10 | } 11 | 12 | func NewDeletedObject(ctx ContextWithApi) *DeletedObject { 13 | return &DeletedObject{&BaseApi{ctx}} 14 | } 15 | 16 | // No caching 17 | func (self *DeletedObject) Lookup(id uint64) (*data.DeletedObject, error) { 18 | ctx := self.Ctx() 19 | 20 | tx, err := ctx.Txn() 21 | if err != nil { 22 | return nil, err 23 | } 24 | 25 | row := tx.QueryRow("SELECT id, bucket_id, name, internal_name, size, status, created_at, updated_at FROM deleted_object WHERE id = ?", id) 26 | 27 | var o data.DeletedObject 28 | err = row.Scan( 29 | &o.Id, 30 | &o.BucketId, 31 | &o.Name, 32 | &o.InternalName, 33 | &o.Size, 34 | &o.Status, 35 | &o.CreatedAt, 36 | &o.UpdatedAt, 37 | ) 38 | 39 | if err != nil { 40 | stf.Debugf("Failed to execute query (Lookup): %s", err) 41 | return nil, err 42 | } 43 | 44 | return &o, nil 45 | } 46 | -------------------------------------------------------------------------------- /worker/repair_object.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "github.com/stf-storage/go-stf-server" 5 | "github.com/stf-storage/go-stf-server/api" 6 | "strconv" 7 | ) 8 | 9 | type RepairObjectWorker struct { 10 | *BaseWorker 11 | } 12 | 13 | func NewRepairObjectWorker() *RepairObjectWorker { 14 | f := NewQueueFetcher("queue_repair_object", 1) 15 | w := &RepairObjectWorker{NewBaseWorker("RepairObject", f)} 16 | w.WorkCb = w.Work 17 | return w 18 | } 19 | 20 | func (self *RepairObjectWorker) Work(arg *api.WorkerArg) (err error) { 21 | objectId, err := strconv.ParseUint(arg.Arg, 10, 64) 22 | if err != nil { 23 | return 24 | } 25 | defer func() { 26 | if err == nil { 27 | stf.Debugf("Processed object %d", objectId) 28 | } else { 29 | stf.Debugf("Failed to process object %d: %s", objectId, err) 30 | } 31 | }() 32 | 33 | ctx := self.ctx 34 | closer, err := ctx.TxnBegin() 35 | if err != nil { 36 | return 37 | } 38 | defer closer() 39 | 40 | objectApi := ctx.ObjectApi() 41 | err = objectApi.Repair(objectId) 42 | if err != nil { 43 | return 44 | } 45 | 46 | err = ctx.TxnCommit() 47 | if err != nil { 48 | return 49 | } 50 | 51 | return 52 | } 53 | -------------------------------------------------------------------------------- /stftest/queue_q4m_test.go: -------------------------------------------------------------------------------- 1 | // +build q4m 2 | 3 | package stftest 4 | 5 | import ( 6 | "fmt" 7 | "github.com/stf-storage/go-stf-server" 8 | "github.com/stf-storage/go-stf-server/config" 9 | "os" 10 | ) 11 | 12 | func (self *TestEnv) startQueue() { 13 | cfg := self.MysqlConfig 14 | self.QueueConfig = &config.QueueConfig{ 15 | cfg.Dbtype, 16 | cfg.Username, 17 | cfg.Password, 18 | cfg.ConnectString, 19 | "test_queue", 20 | } 21 | 22 | self.createQueue() 23 | } 24 | 25 | func (self *TestEnv) createQueue() { 26 | t := self.Test 27 | 28 | db, err := stf.ConnectDB(self.MysqlConfig) 29 | if err != nil { 30 | t.Errorf("Failed to connect to database: %s", err) 31 | t.FailNow() 32 | } 33 | 34 | _, err = db.Exec("CREATE DATABASE test_queue") 35 | if err != nil { 36 | t.Errorf("Failed to create database test_queue: %s", err) 37 | t.FailNow() 38 | } 39 | } 40 | 41 | func (self *TestEnv) writeQueueConfig(tempfile *os.File) { 42 | tempfile.WriteString(fmt.Sprintf( 43 | ` 44 | [QueueDB "1"] 45 | Username=%s 46 | ConnectString=%s 47 | Dbname=%s 48 | `, 49 | self.QueueConfig.Username, 50 | self.QueueConfig.ConnectString, 51 | self.QueueConfig.Dbname, 52 | )) 53 | } 54 | -------------------------------------------------------------------------------- /worker/replicate_object.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "github.com/stf-storage/go-stf-server" 5 | "github.com/stf-storage/go-stf-server/api" 6 | "strconv" 7 | ) 8 | 9 | type ReplicateObjectWorker struct { 10 | *BaseWorker 11 | } 12 | 13 | func NewReplicateObjectWorker() *ReplicateObjectWorker { 14 | f := NewQueueFetcher("queue_replicate", 1) 15 | w := &ReplicateObjectWorker{NewBaseWorker("ReplicateObject", f)} 16 | w.WorkCb = w.Work 17 | return w 18 | } 19 | 20 | func (self *ReplicateObjectWorker) Work(arg *api.WorkerArg) (err error) { 21 | objectId, err := strconv.ParseUint(arg.Arg, 10, 64) 22 | if err != nil { 23 | return 24 | } 25 | defer func() { 26 | if err == nil { 27 | stf.Debugf("Processed object %d", objectId) 28 | } else { 29 | stf.Debugf("Failed to process object %d: %s", objectId, err) 30 | } 31 | }() 32 | 33 | ctx := self.ctx 34 | closer, err := ctx.TxnBegin() 35 | if err != nil { 36 | return 37 | } 38 | defer closer() 39 | 40 | objectApi := ctx.ObjectApi() 41 | err = objectApi.Repair(objectId) 42 | if err != nil { 43 | return 44 | } 45 | 46 | err = ctx.TxnCommit() 47 | if err != nil { 48 | return 49 | } 50 | 51 | return 52 | } 53 | -------------------------------------------------------------------------------- /cli/stf-worker/stf-worker.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "github.com/stf-storage/go-stf-server/config" 6 | "github.com/stf-storage/go-stf-server/drone" 7 | "log" 8 | "os" 9 | "path" 10 | "path/filepath" 11 | "strings" 12 | ) 13 | 14 | func main() { 15 | var configFile string 16 | 17 | pwd, err := os.Getwd() 18 | if err != nil { 19 | log.Fatalf("Could not determine current working directory") 20 | } 21 | 22 | defaultConfig := path.Join(pwd, "etc", "config.gcfg") 23 | 24 | flag.StringVar( 25 | &configFile, 26 | "config", 27 | defaultConfig, 28 | "Path to config file", 29 | ) 30 | flag.Parse() 31 | 32 | os.Setenv("STF_CONFIG", configFile) 33 | 34 | config, err := config.BootstrapConfig() 35 | if err != nil { 36 | log.Fatalf("Could not load config: %s", err) 37 | } 38 | 39 | // Get our path, and add this to PATH, so other binaries 40 | // can safely be executed 41 | dir, err := filepath.Abs(filepath.Dir(os.Args[0])) 42 | if err != nil { 43 | log.Fatalf("Failed to find our directory name?!: %s", err) 44 | } 45 | 46 | p := os.Getenv("PATH") 47 | os.Setenv("PATH", strings.Join([]string{p, dir}, ":")) 48 | 49 | drone.NewDrone(config).Run() 50 | } 51 | -------------------------------------------------------------------------------- /worker/delete_object.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "github.com/stf-storage/go-stf-server" 5 | "github.com/stf-storage/go-stf-server/api" 6 | "strconv" 7 | ) 8 | 9 | type DeleteObjectWorker struct { 10 | *BaseWorker 11 | } 12 | 13 | func NewDeleteObjectWorker() *DeleteObjectWorker { 14 | f := NewQueueFetcher("queue_delete_object", 1) 15 | w := &DeleteObjectWorker{ 16 | NewBaseWorker("DeleteObject", f), 17 | } 18 | w.WorkCb = w.Work 19 | return w 20 | } 21 | 22 | func (self *DeleteObjectWorker) Work(arg *api.WorkerArg) (err error) { 23 | objectId, err := strconv.ParseUint(arg.Arg, 10, 64) 24 | if err != nil { 25 | return 26 | } 27 | defer func() { 28 | if err != nil { 29 | stf.Debugf("Failed to delete entities for object %d: %s", objectId, err) 30 | } else { 31 | stf.Debugf("Deleted object %d", objectId) 32 | } 33 | }() 34 | 35 | ctx := self.ctx 36 | closer, err := ctx.TxnBegin() 37 | if err != nil { 38 | return 39 | } 40 | defer closer() 41 | 42 | entityApi := ctx.EntityApi() 43 | err = entityApi.RemoveForDeletedObjectId(objectId) 44 | if err != nil { 45 | return 46 | } 47 | 48 | err = ctx.TxnCommit() 49 | if err != nil { 50 | return 51 | } 52 | return 53 | } 54 | -------------------------------------------------------------------------------- /drone/periodic.go: -------------------------------------------------------------------------------- 1 | package drone 2 | 3 | import ( 4 | "math/rand" 5 | "time" 6 | ) 7 | 8 | type PeriodicTask struct { 9 | cmdCh chan struct {} // XXX bogus data, just send signal for now 10 | maxInterval time.Duration 11 | randomize bool 12 | callback func(*PeriodicTask) 13 | } 14 | type PeriodicTaskList []*PeriodicTask 15 | 16 | func NewPeriodicTask(interval time.Duration, randomize bool, callback func(*PeriodicTask)) *PeriodicTask { 17 | return &PeriodicTask { 18 | make(chan struct{}, 1), 19 | interval, 20 | randomize, 21 | callback, 22 | } 23 | } 24 | 25 | func (pt *PeriodicTask) Stop() { 26 | pt.cmdCh <- struct{}{} 27 | } 28 | 29 | func (pt *PeriodicTask) Run() { 30 | for { 31 | // next fire time might need to be randomized 32 | interval := pt.maxInterval 33 | if pt.randomize { 34 | interval = time.Duration(rand.Int63n(int64(interval))) 35 | } 36 | 37 | timer := time.After(interval) 38 | for loop := true; loop; { 39 | select { 40 | case <-pt.cmdCh: 41 | // owner of this task may ask us to stop 42 | // bail out. 43 | return 44 | case <-timer: 45 | // next tick! 46 | pt.callback(pt) 47 | loop = false 48 | } 49 | } 50 | } 51 | } 52 | 53 | 54 | -------------------------------------------------------------------------------- /cli/dispatcher/dispatcher.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "github.com/stf-storage/go-stf-server/config" 6 | "github.com/stf-storage/go-stf-server/dispatcher" 7 | "log" 8 | "os" 9 | "path" 10 | ) 11 | 12 | func main() { 13 | var configFile string 14 | var dispatcherId uint64 15 | var listen string 16 | 17 | pwd, err := os.Getwd() 18 | if err != nil { 19 | log.Fatalf("Could not determine current working directory") 20 | } 21 | 22 | defaultConfig := path.Join(pwd, "etc", "config.gcfg") 23 | 24 | flag.Uint64Var( 25 | &dispatcherId, 26 | "id", 27 | 0, 28 | "Dispatcher ID, overrides config file settings", 29 | ) 30 | flag.StringVar( 31 | &listen, 32 | "listen", 33 | "", 34 | "host:port to listen on", 35 | ) 36 | flag.StringVar( 37 | &configFile, 38 | "config", 39 | defaultConfig, 40 | "Path to config file", 41 | ) 42 | flag.Parse() 43 | 44 | os.Setenv("STF_CONFIG", configFile) 45 | config, err := config.BootstrapConfig() 46 | if err != nil { 47 | log.Fatal(err) 48 | } 49 | 50 | if dispatcherId > 0 { 51 | config.Dispatcher.ServerId = dispatcherId 52 | } 53 | if listen != "" { 54 | config.Dispatcher.Listen = listen 55 | } 56 | 57 | d := dispatcher.New(config) 58 | d.Start() 59 | } 60 | -------------------------------------------------------------------------------- /dispatcher/response.go: -------------------------------------------------------------------------------- 1 | package dispatcher 2 | 3 | import ( 4 | "net/http" 5 | ) 6 | 7 | /* In our case, we don't really need a full-fledged response 8 | object as we never return a content-body ourselves 9 | */ 10 | type HTTPResponse struct { 11 | Code int 12 | Message string 13 | Header http.Header 14 | } 15 | 16 | func NewResponse(code int) *HTTPResponse { 17 | h := &HTTPResponse{ 18 | Code: code, 19 | Header: map[string][]string{}, 20 | } 21 | return h 22 | } 23 | 24 | func (self *HTTPResponse) Write(rw http.ResponseWriter) { 25 | hdrs := self.Header 26 | if hdrs != nil { 27 | if ct := hdrs.Get("Content-Type"); ct == "" { 28 | hdrs.Set("Content-Type", "text/plain") 29 | } 30 | } 31 | 32 | outHeader := rw.Header() 33 | for k, list := range self.Header { 34 | for _, v := range list { 35 | outHeader.Add(k, v) 36 | } 37 | } 38 | rw.WriteHeader(self.Code) 39 | 40 | if self.Message != "" { 41 | rw.Write([]byte(self.Message)) 42 | } 43 | } 44 | 45 | var HTTPCreated = NewResponse(201) 46 | var HTTPNoContent = NewResponse(204) 47 | var HTTPNotModified = NewResponse(304) 48 | var HTTPBadRequest = NewResponse(400) 49 | var HTTPNotFound = NewResponse(404) 50 | var HTTPMethodNotAllowed = NewResponse(405) 51 | var HTTPInternalServerError = NewResponse(500) 52 | -------------------------------------------------------------------------------- /drone/drone_test.go: -------------------------------------------------------------------------------- 1 | package drone 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestDrone(t *testing.T) { 9 | 10 | } 11 | 12 | func TestCommandDecl(t *testing.T) { 13 | thing := make(map[DroneCmd]struct {}) 14 | consts := []DroneCmd { 15 | CmdStopDrone, 16 | CmdAnnounce, 17 | CmdSpawnMinion, 18 | CmdReloadMinion, 19 | CmdCheckState, 20 | CmdElection, 21 | CmdRebalance, 22 | CmdExpireDrone, 23 | } 24 | 25 | for _, cmd := range consts { 26 | t.Logf("Checking command %s", cmd) 27 | if _, exists := thing[cmd]; exists { 28 | t.Errorf("command type %s already exists ?! Duplicate value?", cmd) 29 | } 30 | } 31 | 32 | } 33 | 34 | func TestPeriodicTask(t *testing.T) { 35 | var prev time.Time 36 | count := 0 37 | c := make(chan struct{}) 38 | task := NewPeriodicTask(time.Second, false, func (pt *PeriodicTask) { 39 | now := time.Now() 40 | if ! prev.IsZero() { 41 | diff := now.Sub(prev) 42 | if diff < time.Second { 43 | t.Errorf("Whoa, fired periodic task fired in %s!", diff) 44 | } 45 | } 46 | 47 | if count++; count > 5 { 48 | pt.Stop() 49 | c<-struct{}{} 50 | } 51 | }) 52 | 53 | go task.Run() 54 | 55 | <-c 56 | 57 | if count != 6 { 58 | t.Errorf("Expected 5 iterations to be executed, got %d", count) 59 | } 60 | } -------------------------------------------------------------------------------- /dispatcher/id_gen.go: -------------------------------------------------------------------------------- 1 | package dispatcher 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | const ( 8 | EPOCH_OFFSET = 946684800 9 | HOST_ID_BITS = 16 10 | SERIAL_BITS = 12 11 | SERIAL_SHIFT = 16 12 | TIME_BITS = 36 13 | TIME_SHIFT = 28 14 | ) 15 | 16 | type UUIDGen struct { 17 | seed uint64 18 | mutex chan int 19 | serialId int64 20 | timeId int64 21 | timeout int64 22 | } 23 | 24 | func NewIdGenerator(seed uint64) *UUIDGen { 25 | return &UUIDGen{ 26 | seed: seed, 27 | mutex: make(chan int, 1), 28 | serialId: 1, 29 | timeId: time.Now().Unix(), 30 | } 31 | } 32 | 33 | func (self *UUIDGen) CreateId() uint64 { 34 | /* Only one thread can enter this critical section, but it also must be 35 | guarded carefully so that we don't find this thread being blocked 36 | for an indefinite amount of time 37 | */ 38 | mutex := self.mutex 39 | mutex <- 1 40 | defer func() { <-mutex }() 41 | 42 | timeId := time.Now().Unix() 43 | serialId := self.serialId 44 | if self.timeId == 0 { 45 | self.timeId = timeId 46 | } 47 | 48 | if self.timeId == timeId { 49 | serialId++ 50 | } else { 51 | serialId = 1 52 | } 53 | 54 | if serialId >= (1< 0 { 35 | t := <-i.tickChan 36 | 37 | w.SendJob(&api.WorkerArg{ 38 | Arg: fmt.Sprintf("%d", t.UnixNano()), 39 | }) 40 | } 41 | } 42 | 43 | type QueueFetcher struct { 44 | queueName string 45 | queueTimeout int 46 | running bool 47 | } 48 | 49 | func NewQueueFetcher(queueName string, queueTimeout int) *QueueFetcher { 50 | return &QueueFetcher{ 51 | queueName, 52 | queueTimeout, 53 | false, 54 | } 55 | } 56 | 57 | func (q *QueueFetcher) SetRunning(b bool) { 58 | q.running = false 59 | } 60 | 61 | func (q *QueueFetcher) IsRunning() bool { 62 | return q.running 63 | } 64 | 65 | func (q *QueueFetcher) Loop(w Worker) { 66 | closer := stf.LogMark("[Fetcher:%s]", w.Name()) 67 | defer closer() 68 | 69 | defer q.SetRunning(false) 70 | q.SetRunning(true) 71 | 72 | ctx := w.Ctx() 73 | 74 | // We should only be dequeuing jobs if have slaves to consume 75 | // them. Otherwise we'd risk missing them in the air 76 | for w.ActiveSlaves() > 0 { 77 | api := ctx.QueueApi() 78 | arg, err := api.Dequeue(q.queueName, q.queueTimeout) 79 | if err != nil { 80 | continue 81 | } 82 | w.SendJob(arg) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | PROGNAME=$(basename $0) 3 | queue_type=q4m 4 | prefix=/usr/local/stf 5 | 6 | usage() { 7 | echo "Usage: $PROGNAME [OPTIONS]" 8 | echo " Installer script for go-stf-server" 9 | echo 10 | echo "Options:" 11 | echo " -h, --help" 12 | echo " -q, --queue [ q4m | redo ] (default: $queue_type)" 13 | echo " --prefix ARG (default: $prefix)" 14 | echo 15 | exit 1 16 | } 17 | 18 | 19 | for OPT in "$@" 20 | do 21 | case "$OPT" in 22 | '-h'|'--help') 23 | usage 24 | exit 1 25 | ;; 26 | '-q'|'--queue') 27 | queue_type=$2 28 | shift 2 29 | ;; 30 | '--prefix') 31 | prefix=$2 32 | shift 2 33 | ;; 34 | '--'|'-' ) 35 | shift 1 36 | param+=( "$@" ) 37 | break 38 | ;; 39 | -*) 40 | echo "$PROGNAME: illegal option -- '$(echo $1 | sed 's/^-*//')'" 1>&2 41 | exit 1 42 | ;; 43 | *) 44 | if [[ ! -z "$1" ]] && [[ ! "$1" =~ ^-+ ]]; then 45 | #param=( ${param[@]} "$1" ) 46 | param+=( "$1" ) 47 | shift 1 48 | fi 49 | ;; 50 | esac 51 | done 52 | 53 | # Check we have hg 54 | output=$(hg --version) 55 | if [ "$?" != "0" ]; then 56 | echo "hg is required to install go-stf-server's dependencies" 57 | exit 1 58 | fi 59 | 60 | # Check we have git 61 | output=$(git --version) 62 | if [ "$?" != "0" ]; then 63 | echo "git is required to install go-stf-server" 64 | exit 1 65 | fi 66 | 67 | dir=$(mktemp -d -t go-stf-server) 68 | if [ "$?" != "0" ]; then 69 | echo "failed to create temporary directory" 70 | exit 1 71 | fi 72 | 73 | export GOPATH=$dir 74 | go get -tags $queue_type -v github.com/stf-storage/go-stf-server/... 75 | 76 | CMDS="dispatcher storage stf-worker stf-worker-delete_object stf-worker-repair_object stf-worker-replicate_object stf-worker-storage_health" 77 | for cmd in $CMDS 78 | do 79 | echo "building $cmd..." 80 | go build -tags $queue_type -o $prefix/bin/$cmd github.com/stf-storage/go-stf-server/cli/$cmd 81 | done 82 | 83 | rm -rf $dir 84 | 85 | echo "Installed go-stf-server (queue: $queue_type) in $prefix" 86 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | go-stf-server 2 | ============= 3 | 4 | [![Build Status](https://travis-ci.org/stf-storage/go-stf-server.png?branch=master)](https://travis-ci.org/stf-storage/go-stf-server) 5 | 6 | [![Coverage Status](https://coveralls.io/repos/stf-storage/go-stf-server/badge.png)](https://coveralls.io/r/stf-storage/go-stf-server) 7 | 8 | Go Implementation of STF Distributed Object Store 9 | 10 | # Current Status 11 | 12 | STF is a simple, amazon S3-like distributed object store, aimed for people who want to run such systems on their own data centers, on commodity hardware. 13 | 14 | While the [Perl version](https://github.com/stf-storage/stf) is used in production for over 3 years handling billions of hits every day, this go version is still in heavy development. 15 | 16 | ## Features 17 | 18 | * Simple GET/POST/PUT/DELETE to perform CRUD operations 19 | * Multiple copies of the data are stored in the backend storage to prevent data-loss 20 | * Supports Q4M/Redis as queue backends (use '-tags redis' or '-tags q4m' when compiling) 21 | * Automatic storage failure detection 22 | 23 | ## Done, so far 24 | 25 | * Object creation works. 26 | * Object fetch works. 27 | * Object deletion works. 28 | * Workers: Auto-balancing via simple leader-election 29 | * Workers: RepairObject / ReplicateObject / DeleteObject done 30 | 31 | ## TODO 32 | 33 | * Move objects 34 | * Web UI 35 | 36 | ## Installation 37 | 38 | To install go-stf-server's binaries, you can simply use the following: 39 | 40 | ``` 41 | curl -L https://raw.githubusercontent.com/stf-storage/go-stf-server/master/install.sh | bash -s 42 | ``` 43 | 44 | This downloads and compiles go-stf-server's binary files into /usr/local/stf 45 | 46 | To change where files are installed, you can use the --prefix option: 47 | 48 | ``` 49 | curl -L https://raw.githubusercontent.com/stf-storage/go-stf-server/master/install.sh | bash -s --prefix /opt/local/stf 50 | ``` 51 | 52 | To change the queue type, you can use the --queue options: 53 | 54 | ``` 55 | curl -L https://raw.githubusercontent.com/stf-storage/go-stf-server/master/install.sh | bash -s --queue redis 56 | ``` 57 | 58 | 59 | -------------------------------------------------------------------------------- /stftest/client_test.go: -------------------------------------------------------------------------------- 1 | package stftest 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "os" 8 | "testing" 9 | ) 10 | 11 | type TestClient struct { 12 | t *testing.T 13 | server *TestDispatcherServer 14 | client *http.Client 15 | } 16 | 17 | func NewTestSTFClient(t *testing.T, server *TestDispatcherServer) *TestClient { 18 | return &TestClient{ 19 | t, 20 | server, 21 | &http.Client{}, 22 | } 23 | } 24 | 25 | func (c *TestClient) MakeRequest(method string, uri string, body io.Reader) *http.Request { 26 | req, err := http.NewRequest(method, uri, body) 27 | if err != nil { 28 | c.t.Fatalf("Failed to create request: %s", err) 29 | } 30 | return req 31 | } 32 | 33 | func (c *TestClient) SendRequestExpect(req *http.Request, expected int, title string) *http.Response { 34 | res, err := c.client.Do(req) 35 | if err != nil { 36 | c.t.Fatalf("Failed to send request: %s", err) 37 | } 38 | 39 | if res.StatusCode != expected { 40 | c.t.Logf(title) 41 | c.t.Fatalf(`Request "%s %s": Expected response status code %d, got %d`, req.Method, req.URL, expected, res.StatusCode) 42 | } 43 | 44 | return res 45 | } 46 | 47 | func (c *TestClient) BucketCreate(path string) *http.Response { 48 | uri := c.server.MakeURL(path) 49 | req := c.MakeRequest("PUT", uri, nil) 50 | res := c.SendRequestExpect(req, 201, "Create bucket should succeed") 51 | 52 | return res 53 | } 54 | 55 | func (c *TestClient) FilePut(path string, filename string) *http.Response { 56 | file, err := os.Open(filename) 57 | if err != nil { 58 | c.t.Fatalf("Failed to open %s: %s", filename, err) 59 | } 60 | fi, err := file.Stat() 61 | if err != nil { 62 | c.t.Fatalf("Failed to stat %s: %s", filename, err) 63 | } 64 | 65 | uri := c.server.MakeURL(path) 66 | req := c.MakeRequest("PUT", uri, file) 67 | req.ContentLength = fi.Size() 68 | res := c.SendRequestExpect(req, 201, fmt.Sprintf("File upload (%s) should success", filename)) 69 | 70 | return res 71 | } 72 | 73 | func (c *TestClient) ObjectGet(path string) *http.Response { 74 | return c.ObjectGetExpect(path, 200, "Fetch object should succeed") 75 | } 76 | 77 | func (c *TestClient) ObjectGetExpect(path string, expectedStatus int, title string) *http.Response { 78 | uri := c.server.MakeURL(path) 79 | req := c.MakeRequest("GET", uri, nil) 80 | res := c.SendRequestExpect(req, expectedStatus, title) 81 | return res 82 | } 83 | 84 | func (c *TestClient) ObjectDelete(path string) *http.Response { 85 | uri := c.server.MakeURL(path) 86 | req := c.MakeRequest("DELETE", uri, nil) 87 | res := c.SendRequestExpect(req, 204, "Delete object should succeed") 88 | return res 89 | } 90 | -------------------------------------------------------------------------------- /dispatcher/bucket.go: -------------------------------------------------------------------------------- 1 | package dispatcher 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/stf-storage/go-stf-server/api" 6 | "log" 7 | ) 8 | 9 | func (self *Dispatcher) CreateBucket(ctx *DispatcherContext, bucketName string, objectName string) *HTTPResponse { 10 | rollback, err := ctx.TxnBegin() 11 | if err != nil { 12 | ctx.Debugf("Failed to start transaction: %s", err) 13 | return HTTPInternalServerError 14 | } 15 | defer rollback() 16 | 17 | closer := ctx.LogMark("[Dispatcher.CreateBucket]") 18 | defer closer() 19 | 20 | if objectName != "" { 21 | return &HTTPResponse{Code: 400, Message: "Bad bucket name"} 22 | } 23 | 24 | bucketApi := ctx.BucketApi() 25 | 26 | id, err := bucketApi.LookupIdByName(bucketName) 27 | if err == nil { // No error, so we found a bucket 28 | ctx.Debugf("Bucket '%s' already exists (id = %d)", bucketName, id) 29 | return HTTPNoContent 30 | } else if err != sql.ErrNoRows { 31 | ctx.Debugf("Error while looking up bucket '%s': %s", bucketName, err) 32 | return HTTPInternalServerError 33 | } 34 | 35 | // If we got here, it's a new Bucket Create it 36 | id = self.IdGenerator().CreateId() 37 | log.Printf("id = %d", id) 38 | 39 | err = bucketApi.Create( 40 | id, 41 | bucketName, 42 | ) 43 | 44 | if err != nil { 45 | ctx.Debugf("Failed to create bucket '%s': %s", bucketName, err) 46 | return HTTPInternalServerError 47 | } 48 | 49 | if err = ctx.TxnCommit(); err != nil { 50 | ctx.Debugf("Failed to commit: %s", err) 51 | } 52 | 53 | return HTTPCreated 54 | } 55 | 56 | func (self *Dispatcher) DeleteBucket(ctx api.ContextWithApi, bucketName string) *HTTPResponse { 57 | rollback, err := ctx.TxnBegin() 58 | if err != nil { 59 | ctx.Debugf("Failed to start transaction: %s", err) 60 | return HTTPInternalServerError 61 | } 62 | defer rollback() 63 | 64 | bucketApi := ctx.BucketApi() 65 | id, err := bucketApi.LookupIdByName(bucketName) 66 | 67 | if err != nil { 68 | return &HTTPResponse{Code: 500, Message: "Failed to find bucket"} 69 | } 70 | 71 | err = bucketApi.MarkForDelete(id) 72 | if err != nil { 73 | ctx.Debugf("Failed to delete bucket %s", err) 74 | return &HTTPResponse{Code: 500, Message: "Failed to delete bucket"} 75 | } 76 | 77 | if err = ctx.TxnCommit(); err != nil { 78 | ctx.Debugf("Failed to commit: %s", err) 79 | } else { 80 | ctx.Debugf("Deleted bucket '%s' (id = %d)", bucketName, id) 81 | } 82 | 83 | return HTTPNoContent 84 | } 85 | 86 | // MOVE /bucket_name 87 | // X-STF-Move-Destination: /new_name 88 | func (self *Dispatcher) RenameBucket(ctx api.ContextWithApi, bucketName string, dest string) *HTTPResponse { 89 | return nil 90 | } 91 | -------------------------------------------------------------------------------- /api/queue_redis.go: -------------------------------------------------------------------------------- 1 | // +build redis 2 | 3 | package api 4 | 5 | import ( 6 | "errors" 7 | "github.com/vmihailenco/redis/v2" 8 | "math/rand" 9 | "time" 10 | ) 11 | 12 | type Redis struct { 13 | BaseApi 14 | currentQueue int 15 | RedisClients []*redis.Client 16 | } 17 | 18 | func NewRedis(ctx ContextWithApi) *Redis { 19 | // Find the number of queues, get a random queueIdx 20 | cfg := ctx.Config() 21 | max := len(cfg.QueueDBList) 22 | qidx := rand.Intn(max) 23 | 24 | return &Redis{BaseApi{ctx}, qidx, make([]*redis.Client, max, max)} 25 | } 26 | 27 | func (self *Redis) NumQueueDB() int { 28 | return len(self.ctx.Config().QueueDBList) 29 | } 30 | 31 | func (self *Redis) RedisDB(i int) (*redis.Client, error) { 32 | 33 | client := self.RedisClients[i] 34 | if client != nil { 35 | return client, nil 36 | } 37 | 38 | qc := self.ctx.Config().QueueDBList[i] 39 | rc := redis.Options(*qc) 40 | 41 | self.ctx.Debugf("Connecting to new Redis server %s", rc.Addr) 42 | 43 | client = redis.NewTCPClient(&rc) 44 | self.RedisClients[i] = client 45 | 46 | return client, nil 47 | } 48 | 49 | func NewQueue(ctx ContextWithApi) QueueApiInterface { 50 | return NewRedis(ctx) 51 | } 52 | 53 | func (self *Redis) Enqueue(qname string, data string) error { 54 | // Lpush 55 | max := self.NumQueueDB() 56 | for i := 0; i < max; i++ { 57 | qidx := self.currentQueue 58 | client, err := self.RedisDB(qidx) 59 | 60 | qidx++ 61 | if qidx >= max { 62 | qidx = 0 63 | } 64 | self.currentQueue = qidx 65 | 66 | if err != nil { 67 | continue 68 | } 69 | 70 | _, err = client.LPush(qname, data).Result() 71 | if err != nil { 72 | continue 73 | } 74 | 75 | return nil 76 | } 77 | return errors.New("Failed to enqueue into any queue") 78 | } 79 | 80 | func (self *Redis) Dequeue(qname string, timeout int) (*WorkerArg, error) { 81 | // Rpop 82 | max := self.NumQueueDB() 83 | for i := 0; i < max; i++ { 84 | qidx := self.currentQueue 85 | client, err := self.RedisDB(qidx) 86 | 87 | qidx++ 88 | if qidx >= max { 89 | qidx = 0 90 | } 91 | self.currentQueue = qidx 92 | 93 | if err != nil { 94 | continue 95 | } 96 | 97 | val, err := client.RPop(qname).Result() 98 | if err != nil { 99 | if err == redis.Nil { 100 | // sleep a bit 101 | time.Sleep(time.Duration(rand.Int63n(int64(5 * time.Second)))) 102 | } 103 | continue 104 | } 105 | 106 | self.Ctx().Debugf("val -> %s", val) 107 | return &WorkerArg{Arg: val}, nil 108 | 109 | } 110 | return nil, errors.New("Failed to dequeue from any queue") 111 | } 112 | -------------------------------------------------------------------------------- /debug.go: -------------------------------------------------------------------------------- 1 | package stf 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "log" 7 | "os" 8 | "strconv" 9 | "strings" 10 | ) 11 | 12 | func init() { 13 | dbgEnabled := false 14 | 15 | dbgOutput := os.Stderr 16 | if dbg := os.Getenv("STF_DEBUG"); dbg != "" { 17 | // STF_DEBUG='1:path/to/log' 18 | if strings.Index(dbg, ":") > -1 { 19 | list := strings.Split(dbg, ":") 20 | dbg = list[0] 21 | if len(list) > 1 { 22 | file, err := os.OpenFile(list[1], os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0755) 23 | if err != nil { 24 | log.Fatalf("Failed to open debug log output %s: %s", list[1], err) 25 | } 26 | dbgOutput = file 27 | } 28 | } 29 | x, err := strconv.ParseBool(dbg) 30 | if err == nil { 31 | dbgEnabled = x 32 | } 33 | } 34 | 35 | if dbgEnabled { 36 | global = NewDebugLog() 37 | global.SetOutput(dbgOutput) 38 | } 39 | } 40 | 41 | const INDENT_PATTERN string = " " 42 | 43 | type DebugLog struct { 44 | Prefix string 45 | Indent string 46 | out io.Writer 47 | } 48 | 49 | func NewDebugLog() *DebugLog { 50 | return &DebugLog{ 51 | "", 52 | "", 53 | os.Stderr, 54 | } 55 | } 56 | 57 | var global *DebugLog 58 | 59 | func (self *DebugLog) SetOutput(out io.Writer) { 60 | self.out = out 61 | } 62 | 63 | func (self *DebugLog) Printf(format string, args ...interface{}) { 64 | message := fmt.Sprintf(format, args...) 65 | log.Printf("%s %s %s", self.Prefix, self.Indent, message) 66 | } 67 | 68 | func (self *DebugLog) LogIndent() func() { 69 | self.Indent = INDENT_PATTERN + self.Indent 70 | return func() { 71 | self.Indent = strings.TrimPrefix(self.Indent, INDENT_PATTERN) 72 | } 73 | } 74 | 75 | func Debugf(format string, args ...interface{}) { 76 | if global != nil { 77 | global.Printf(format, args...) 78 | } 79 | } 80 | 81 | func LogIndent() { 82 | if global != nil { 83 | global.LogIndent() 84 | } 85 | } 86 | 87 | func LogMark(format string, args ...interface{}) func() { 88 | if global == nil { 89 | return func() {} 90 | } 91 | 92 | return global.LogMark(format, args...) 93 | } 94 | 95 | func DebugEnabled() bool { 96 | return global != nil 97 | } 98 | 99 | func (self *DebugLog) LogMark(format string, args ...interface{}) func() { 100 | marker := fmt.Sprintf(format, args...) 101 | 102 | self.Printf("%s START", marker) 103 | closer := self.LogIndent() 104 | return func() { 105 | err := recover() 106 | if err != nil { 107 | self.Printf("Encoundered panic during '%s': %s", marker, err) 108 | } 109 | closer() 110 | self.Printf("%s END", marker) 111 | if err != nil { 112 | panic(err) 113 | } 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /cache/memcache.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "crypto/md5" 5 | "errors" 6 | "fmt" 7 | "github.com/bradfitz/gomemcache/memcache" 8 | "github.com/stf-storage/go-stf-server" 9 | "github.com/vmihailenco/msgpack" 10 | "io" 11 | "log" 12 | "strings" 13 | ) 14 | 15 | type MemdClient struct { 16 | client *memcache.Client 17 | } 18 | 19 | type Int64Value struct { 20 | Value int64 21 | } 22 | 23 | type StringValue struct { 24 | Value string 25 | } 26 | 27 | func NewMemdClient(args ...string) *MemdClient { 28 | memd := memcache.New(args...) 29 | return &MemdClient{client: memd} 30 | } 31 | 32 | func (self *MemdClient) CacheKey(args ...string) string { 33 | // Prepend with our custom namespace 34 | parts := append([]string{"go-stf", fmt.Sprintf("version(%s)", stf.VERSION)}, args...) 35 | key := strings.Join(parts, ".") 36 | // Encode keys that are too long 37 | if len(key) > 250 { 38 | h := md5.New() 39 | io.WriteString(h, key) 40 | key = fmt.Sprintf("%x", h.Sum(nil)) 41 | } 42 | return key 43 | } 44 | 45 | func (self *MemdClient) Add(key string, value interface{}, expires int32) error { 46 | b, err := msgpack.Marshal(value) 47 | if err != nil { 48 | log.Fatalf("Failed to encode value: %s", err) 49 | } 50 | item := &memcache.Item{Key: key, Value: b, Flags: 4, Expiration: expires} 51 | return self.client.Add(item) 52 | } 53 | 54 | func (self *MemdClient) Set(key string, value interface{}, expires int32) error { 55 | b, err := msgpack.Marshal(value) 56 | if err != nil { 57 | log.Fatalf("Failed to encode value: %s", err) 58 | } 59 | item := &memcache.Item{Key: key, Value: b, Flags: 4, Expiration: expires} 60 | return self.client.Set(item) 61 | } 62 | 63 | func (self *MemdClient) GetMulti( 64 | keys []string, 65 | makeContainer func() interface{}, 66 | ) (map[string]interface{}, error) { 67 | items, err := self.client.GetMulti(keys) 68 | if err != nil { 69 | return nil, err 70 | } 71 | 72 | ret := make(map[string]interface{}) 73 | for k, item := range items { 74 | v := makeContainer() 75 | err := msgpack.Unmarshal(item.Value, v) 76 | if err != nil { 77 | continue 78 | } 79 | ret[k] = v 80 | } 81 | 82 | return ret, nil 83 | } 84 | 85 | func (self *MemdClient) Get(key string, v interface{}) error { 86 | item, err := self.client.Get(key) 87 | 88 | if err != nil { 89 | return err 90 | } 91 | 92 | if item.Flags&4 != 0 { 93 | err := msgpack.Unmarshal(item.Value, &v) 94 | if err != nil { 95 | err = errors.New(fmt.Sprintf("Failed to decode value: %s", err)) 96 | return err 97 | } 98 | return nil 99 | } 100 | 101 | v = item.Value 102 | return nil 103 | } 104 | 105 | func (self *MemdClient) Delete(key string) error { 106 | return self.client.Delete(key) 107 | } 108 | -------------------------------------------------------------------------------- /constants.go: -------------------------------------------------------------------------------- 1 | package stf 2 | 3 | const ( 4 | // These are currently in need of reconsideration. 5 | STORAGE_MODE_CRASH_RECOVERED = -4 6 | STORAGE_MODE_CRASH_RECOVER_NOW = -3 7 | STORAGE_MODE_CRASH = -2 8 | 9 | /* Denotes that the storage is temporarily down. Use it to stop 10 | * the dispatcher from accessing a storage for a short period 11 | * of time while you do minor maintenance work. No GET/PUT/DELETE 12 | * will be issued against this node while in this mode. 13 | * 14 | * Upon repair worker hitting this node, the entity is deemed 15 | * alive, and no new entities are created to replace it. 16 | * This is why you should only use this mode TEMPORARILY. 17 | */ 18 | STORAGE_MODE_TEMPORARILY_DOWN = -1 19 | 20 | /* Denotes that the storage is for read-only. No PUT/DELETE operations 21 | * will be issued against this node while in this mode. 22 | * 23 | * An entity residing on this node is deemed alive. 24 | */ 25 | STORAGE_MODE_READ_ONLY = 0 26 | 27 | /* Denotes that the storage is for read-write. 28 | * This is the default, and the most "normal" mode for a storage. 29 | */ 30 | STORAGE_MODE_READ_WRITE = 1 31 | 32 | /* Denotes that the storage has been retired. Marking a storage as 33 | * retired means that the storage is not to be put back again. 34 | * 35 | * Entities residing on this node are deemed dead. Upon repair, 36 | * the worker(s) will try to replace the missing entity with 37 | */ 38 | STORAGE_MODE_RETIRE = 2 39 | 40 | /* These are only used to denote that an automatic migration 41 | * is happening (XXX Unused? Legacy from a long time ago...) 42 | */ 43 | STORAGE_MODE_MIGRATE_NOW = 3 44 | STORAGE_MODE_MIGRATED = 4 45 | 46 | /* These storages are not crashed, they don't need to be 47 | * emptied out, they just need to be checked for repairments 48 | */ 49 | STORAGE_MODE_REPAIR = 5 50 | STORAGE_MODE_REPAIR_NOW = 6 51 | STORAGE_MODE_REPAIR_DONE = 7 52 | 53 | /* Denotes that the storage is a spare for the registered cluster. 54 | * Writes are performed, but reads do not happen. Upon a failure 55 | * you can either replace the broken storage with this one, or 56 | * use this to restore the broken storage. 57 | */ 58 | STORAGE_MODE_SPARE = 10 59 | 60 | STORAGE_CLUSTER_MODE_READ_ONLY = 0 61 | 62 | STORAGE_CLUSTER_MODE_READ_WRITE = 1 63 | STORAGE_CLUSTER_MODE_RETIRE = 2 64 | ) 65 | 66 | var WRITABLE_MODES = []int{ 67 | STORAGE_MODE_READ_WRITE, 68 | STORAGE_MODE_SPARE, 69 | } 70 | 71 | var WRITABLE_MODES_ON_REPAIR = []int{ 72 | STORAGE_MODE_READ_WRITE, 73 | STORAGE_MODE_SPARE, 74 | STORAGE_MODE_REPAIR, 75 | STORAGE_MODE_REPAIR_NOW, 76 | STORAGE_MODE_REPAIR_DONE, 77 | } 78 | 79 | var READABLE_MODES = []int{ 80 | STORAGE_MODE_READ_ONLY, 81 | STORAGE_MODE_READ_WRITE, 82 | } 83 | 84 | var READABLE_MODES_ON_REPAIR = []int{ 85 | STORAGE_MODE_READ_ONLY, 86 | STORAGE_MODE_READ_WRITE, 87 | STORAGE_MODE_REPAIR, 88 | STORAGE_MODE_REPAIR_NOW, 89 | STORAGE_MODE_REPAIR_DONE, 90 | } 91 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "code.google.com/p/gcfg" 5 | "errors" 6 | "fmt" 7 | "log" 8 | "os" 9 | "os/user" 10 | "path" 11 | "path/filepath" 12 | ) 13 | 14 | type GlobalConfig struct { 15 | Debug bool 16 | } 17 | 18 | type DispatcherConfig struct { 19 | ServerId uint64 20 | Listen string 21 | AccessLog string // /path/to/accesslog.%Y%m%d 22 | AccessLogLink string // /path/to/accesslog 23 | } 24 | 25 | type DatabaseConfig struct { 26 | Dbtype string 27 | Username string 28 | Password string 29 | // tcp(127.0.0.1:3306) 30 | // unix(/path/to/sock) 31 | ConnectString string 32 | Dbname string 33 | } 34 | 35 | type MemcachedConfig struct { 36 | Servers []string 37 | } 38 | 39 | type Config struct { 40 | FileName string 41 | Dispatcher DispatcherConfig 42 | Global GlobalConfig 43 | MainDB DatabaseConfig 44 | Memcached MemcachedConfig 45 | QueueDB map[string]*QueueConfig 46 | QueueDBList []*QueueConfig 47 | } 48 | 49 | func BootstrapConfig() (*Config, error) { 50 | home := os.Getenv("STF_HOME") 51 | if home == "" { 52 | var err error 53 | home, err = os.Getwd() 54 | if err != nil { 55 | log.Fatalf("Failed to get home from env and Getwd: %s", err) 56 | } 57 | } 58 | 59 | cfg := Config{} 60 | file := os.Getenv("STF_CONFIG") 61 | if file == "" { 62 | file = path.Join("etc", "config.gcfg") 63 | } 64 | if !filepath.IsAbs(file) { 65 | file = path.Join(home, file) 66 | } 67 | 68 | err := gcfg.ReadFileInto(&cfg, file) 69 | if err != nil { 70 | return nil, errors.New( 71 | fmt.Sprintf( 72 | "Failed to load config file '%s': %s", 73 | file, 74 | err, 75 | ), 76 | ) 77 | } 78 | 79 | cfg.FileName = file 80 | cfg.Prepare() 81 | return &cfg, nil 82 | } 83 | 84 | func (cfg *Config) Prepare() { 85 | l := len(cfg.QueueDB) 86 | list := make([]*QueueConfig, l) 87 | i := 0 88 | for _, v := range cfg.QueueDB { 89 | list[i] = v 90 | i++ 91 | } 92 | cfg.QueueDBList = list 93 | } 94 | 95 | func LoadConfig(home string) (*Config, error) { 96 | cfg := &Config{} 97 | file := os.Getenv("STF_CONFIG") 98 | if file == "" { 99 | file = path.Join("etc", "config.gcfg") 100 | } 101 | if !filepath.IsAbs(file) { 102 | file = path.Join(home, file) 103 | } 104 | log.Printf("Loading config file %s", file) 105 | 106 | err := gcfg.ReadFileInto(cfg, file) 107 | if err != nil { 108 | return nil, errors.New( 109 | fmt.Sprintf( 110 | "Failed to load config file '%s': %s", 111 | file, 112 | err, 113 | ), 114 | ) 115 | } 116 | 117 | cfg.FileName = file 118 | cfg.Prepare() 119 | 120 | return cfg, nil 121 | } 122 | 123 | func (config *DatabaseConfig) Dsn() (string, error) { 124 | if config.Dbtype == "" { 125 | config.Dbtype = "mysql" 126 | } 127 | 128 | if config.ConnectString == "" { 129 | switch config.Dbtype { 130 | case "mysql": 131 | config.ConnectString = "tcp(127.0.0.1:3306)" 132 | default: 133 | return "", errors.New( 134 | fmt.Sprintf( 135 | "No database connect string provided, and can't assign a default value for dbtype '%s'", 136 | config.Dbtype, 137 | ), 138 | ) 139 | } 140 | } 141 | 142 | if config.Username == "" { 143 | u, err := user.Current() 144 | if err == nil { 145 | config.Username = u.Username 146 | } else { 147 | config.Username = "root" 148 | } 149 | } 150 | 151 | if config.Dbname == "" { 152 | config.Dbname = "stf" 153 | } 154 | 155 | dsn := fmt.Sprintf( 156 | "%s:%s@%s/%s?parseTime=true", 157 | config.Username, 158 | config.Password, 159 | config.ConnectString, 160 | config.Dbname, 161 | ) 162 | 163 | return dsn, nil 164 | } 165 | -------------------------------------------------------------------------------- /api/queue_q4m.go: -------------------------------------------------------------------------------- 1 | // +build q4m 2 | 3 | package api 4 | 5 | import ( 6 | "errors" 7 | "fmt" 8 | "github.com/stf-storage/go-stf-server" 9 | "github.com/stf-storage/go-stf-server/config" 10 | "math/rand" 11 | ) 12 | 13 | type Q4M struct { 14 | BaseApi 15 | currentQueue int 16 | QueueDBPtrList []*stf.DB 17 | } 18 | 19 | func (self *Q4M) NumQueueDB() int { 20 | return len(self.QueueDBPtrList) 21 | } 22 | 23 | func NewQ4M(ctx ContextWithApi) *Q4M { 24 | // Find the number of queues, get a random queueIdx 25 | cfg := ctx.Config() 26 | max := len(cfg.QueueDBList) 27 | qidx := rand.Intn(max) 28 | 29 | return &Q4M{BaseApi{ctx}, qidx, nil} 30 | } 31 | 32 | // ctx.NumQueueDBCount = len(cfg.QueueDBList) 33 | // ctx.QueueDBPtrList = make([]*sql.DB, ctx.NumQueueDBCount) 34 | func NewQueue(ctx ContextWithApi) QueueApiInterface { 35 | return NewQ4M(ctx) 36 | } 37 | 38 | func ConnectQueue(cfg *config.QueueConfig) (*stf.DB, error) { 39 | return stf.ConnectDB(&config.DatabaseConfig{ 40 | cfg.Dbtype, 41 | cfg.Username, 42 | cfg.Password, 43 | cfg.ConnectString, 44 | cfg.Dbname, 45 | }) 46 | } 47 | 48 | // Gets the i-th Queue DB 49 | func (self *Q4M) QueueDB(i int) (*stf.DB, error) { 50 | if self.QueueDBPtrList[i] == nil { 51 | config := self.ctx.Config().QueueDBList[i] 52 | db, err := ConnectQueue(config) 53 | if err != nil { 54 | return nil, err 55 | } 56 | self.QueueDBPtrList[i] = db 57 | } 58 | return self.QueueDBPtrList[i], nil 59 | } 60 | 61 | func (self *Q4M) Ctx() ContextWithApi { 62 | return self.ctx 63 | } 64 | 65 | func (self *Q4M) Enqueue(queueName string, data string) error { 66 | ctx := self.Ctx() 67 | closer := ctx.LogMark("[Q4M.Enqueue]") 68 | defer closer() 69 | 70 | max := self.NumQueueDB() 71 | done := false 72 | 73 | sql := fmt.Sprintf("INSERT INTO %s (args, created_at) VALUES (?, UNIX_TIMESTAMP())", queueName) 74 | 75 | for i := 0; i < max; i++ { 76 | qidx := self.currentQueue 77 | self.currentQueue++ 78 | if self.currentQueue >= max { 79 | self.currentQueue = 0 80 | } 81 | 82 | db, err := self.QueueDB(qidx) 83 | if err != nil { 84 | continue 85 | } 86 | 87 | _, err = db.Exec(sql, data) 88 | if err != nil { 89 | continue 90 | } 91 | 92 | done = true 93 | break 94 | } 95 | 96 | if !done { 97 | return errors.New("Failed to insert to any of the queue databases") 98 | } 99 | 100 | return nil 101 | } 102 | 103 | var ErrNothingToDequeue = errors.New("Could not dequeue anything") 104 | var ErrNothingToDequeueDbErrors = errors.New("Could not dequeue anything (DB errors)") 105 | 106 | func (self *Q4M) Dequeue(queueName string, timeout int) (*WorkerArg, error) { 107 | ctx := self.Ctx() 108 | closer := ctx.LogMark("[Q4M.Dequeue]") 109 | defer closer() 110 | 111 | max := self.NumQueueDB() 112 | 113 | sql := fmt.Sprintf("SELECT args, created_at FROM %s WHERE queue_wait('%s', ?)", queueName, queueName) 114 | 115 | dberr := 0 116 | // try all queues 117 | for i := 0; i < max; i++ { 118 | qidx := self.currentQueue 119 | self.currentQueue++ 120 | if self.currentQueue >= max { 121 | self.currentQueue = 0 122 | } 123 | 124 | db, err := self.QueueDB(qidx) 125 | if err != nil { 126 | ctx.Debugf("Failed to retrieve QueueDB (%d): %s", qidx, err) 127 | // Ugh, try next one 128 | dberr++ 129 | continue 130 | } 131 | 132 | var arg WorkerArg 133 | row := db.QueryRow(sql, timeout) 134 | err = row.Scan(&arg.Arg, &arg.CreatedAt) 135 | 136 | // End it! End it! 137 | db.Exec("SELECT queue_end()") 138 | 139 | if err != nil { 140 | ctx.Debugf("Failed to fetch from queue on QueueDB (%d): %s", qidx, err) 141 | // Ugn, try next one 142 | dberr++ 143 | continue 144 | } 145 | 146 | ctx.Debugf("Fetched next job %+v", arg) 147 | return &arg, nil 148 | } 149 | 150 | // if we got here, we were not able to dequeue anything 151 | var err error 152 | if dberr > 0 { 153 | err = ErrNothingToDequeueDbErrors 154 | } else { 155 | err = ErrNothingToDequeue 156 | } 157 | ctx.Debugf("%s", err) 158 | return nil, err 159 | } 160 | -------------------------------------------------------------------------------- /worker/storage_health.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/stf-storage/go-stf-server" 7 | "github.com/stf-storage/go-stf-server/api" 8 | "github.com/stf-storage/go-stf-server/data" 9 | "log" 10 | "net/http" 11 | "strconv" 12 | "strings" 13 | "time" 14 | ) 15 | 16 | type StorageHealthWorker struct { 17 | *BaseWorker 18 | } 19 | 20 | func NewStorageHealthWorker() *StorageHealthWorker { 21 | f := NewIntervalFetcher(900 * time.Second) 22 | w := &StorageHealthWorker{NewBaseWorker("StorageHealth", f)} 23 | w.WorkCb = w.Work 24 | 25 | return w 26 | } 27 | 28 | func (self *StorageHealthWorker) Work(arg *api.WorkerArg) (err error) { 29 | ctx := self.ctx 30 | closer, err := ctx.TxnBegin() 31 | if err != nil { 32 | return 33 | } 34 | defer closer() 35 | 36 | sql := `SELECT id, uri FROM storage WHERE mode IN (?, ?)` 37 | db, err := ctx.MainDB() 38 | if err != nil { 39 | return 40 | } 41 | 42 | rows, err := db.Query(sql, stf.STORAGE_MODE_READ_ONLY, stf.STORAGE_MODE_READ_WRITE) 43 | if err != nil { 44 | return 45 | } 46 | 47 | var storages []*data.Storage 48 | for rows.Next() { 49 | var s data.Storage 50 | 51 | err = rows.Scan(&s.Id, &s.Uri) 52 | if err != nil { 53 | return 54 | } 55 | storages = append(storages, &s) 56 | } 57 | 58 | for _, s := range storages { 59 | if err = self.StorageIsAvailable(s); err != nil { 60 | log.Printf(` 61 | CRITICAL! FAILED TO PUT/HEAD/GET/DELETE to storage 62 | error : %s 63 | storage id : %d 64 | storage uri : %s 65 | GOING TO BRING DOWN THIS STORAGE! 66 | `, 67 | err, 68 | s.Id, 69 | s.Uri, 70 | ) 71 | if err = self.MarkStorageDown(ctx, s); err != nil { 72 | log.Printf("Failed to mark storage as down: %s", err) 73 | return 74 | } 75 | } 76 | } 77 | ctx.TxnCommit() 78 | 79 | return 80 | } 81 | 82 | func (self *StorageHealthWorker) StorageIsAvailable(s *data.Storage) (err error) { 83 | uri := strings.Join([]string{s.Uri, "health.txt"}, "/") 84 | content := stf.GenerateRandomId(uri, 40) 85 | client := &http.Client{} 86 | 87 | // Delete the object first, just in case 88 | // Note: No error checks here 89 | req, _ := http.NewRequest("DELETE", uri, nil) 90 | res, _ := client.Do(req) 91 | 92 | // Now do a successibe PUT/HEAD/GET/DELETE 93 | 94 | req, err = http.NewRequest("PUT", uri, strings.NewReader(content)) 95 | if err != nil { 96 | return 97 | } 98 | res, err = client.Do(req) 99 | if err != nil { 100 | return 101 | } 102 | if res.StatusCode != 201 { 103 | return errors.New(fmt.Sprintf("Failed to PUT %s: %s", uri, res.Status)) 104 | } 105 | 106 | req, err = http.NewRequest("HEAD", uri, nil) 107 | if err != nil { 108 | return 109 | } 110 | res, err = client.Do(req) 111 | if err != nil { 112 | return 113 | } 114 | if res.StatusCode != 200 { 115 | return errors.New(fmt.Sprintf("Failed to HEAD %s: %s", uri, res.Status)) 116 | } 117 | 118 | req, err = http.NewRequest("GET", uri, nil) 119 | if err != nil { 120 | return 121 | } 122 | res, err = client.Do(req) 123 | if err != nil { 124 | return 125 | } 126 | if res.StatusCode != 200 { 127 | return errors.New(fmt.Sprintf("Failed to GET %s: %s", uri, res.Status)) 128 | } 129 | 130 | req, err = http.NewRequest("DELETE", uri, nil) 131 | if err != nil { 132 | return 133 | } 134 | res, err = client.Do(req) 135 | if err != nil { 136 | return 137 | } 138 | if res.StatusCode != 200 { 139 | return errors.New(fmt.Sprintf("Failed to DELETE %s: %s", uri, res.Status)) 140 | } 141 | 142 | return 143 | } 144 | 145 | func (self *StorageHealthWorker) MarkStorageDown(ctx *api.Context, s *data.Storage) (err error) { 146 | db, err := ctx.MainDB() 147 | if err != nil { 148 | return 149 | } 150 | sql := `UPDATE storage SET mode = ?, updated_at = NOW() WHERE id = ?` 151 | _, err = db.Exec(sql, stf.STORAGE_MODE_TEMPORARILY_DOWN, s.Id) 152 | if err != nil { 153 | return 154 | } 155 | 156 | // Kill the cache 157 | cache := ctx.Cache() 158 | cacheKey := cache.CacheKey("storage", strconv.FormatUint(uint64(s.Id), 10)) 159 | cache.Delete(cacheKey) 160 | 161 | return 162 | } 163 | -------------------------------------------------------------------------------- /server_storage.go: -------------------------------------------------------------------------------- 1 | package stf 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | "log" 8 | "net/http" 9 | "os" 10 | "path" 11 | "strconv" 12 | "time" 13 | ) 14 | 15 | type StorageServer struct { 16 | root string 17 | fserver http.Handler 18 | tempdir string 19 | listen string 20 | } 21 | 22 | func NewStorageServer(listen string, root string) *StorageServer { 23 | pwd, err := os.Getwd() 24 | if err != nil { 25 | log.Fatalf("Could not determine current working directory") 26 | } 27 | 28 | if !path.IsAbs(root) { 29 | root = path.Join(pwd, root) 30 | } 31 | 32 | tempdir, err := ioutil.TempDir("", "stf-storage") 33 | if err != nil { 34 | log.Fatalf("Failed to create a temporary directory: %s", err) 35 | } 36 | 37 | ss := &StorageServer{ 38 | fserver: http.FileServer(http.Dir(root)), 39 | tempdir: tempdir, 40 | root: root, 41 | listen: listen, 42 | } 43 | 44 | return ss 45 | } 46 | 47 | func (self *StorageServer) Root() string { 48 | return self.root 49 | } 50 | 51 | func (self *StorageServer) Close() { 52 | // Make sure to clean up 53 | os.RemoveAll(self.tempdir) 54 | } 55 | 56 | func (self *StorageServer) Start() { 57 | server := &http.Server{ 58 | Addr: self.listen, 59 | Handler: self, 60 | } 61 | defer self.Close() 62 | 63 | log.Printf("Server starting at %s, files in %s", server.Addr, self.root) 64 | err := server.ListenAndServe() 65 | if err != nil { 66 | log.Fatal( 67 | fmt.Sprintf("Error from server's ListenAndServe: %s\n", err), 68 | ) 69 | } 70 | } 71 | 72 | func (self *StorageServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { 73 | switch r.Method { 74 | case "GET", "HEAD": 75 | self.fserver.ServeHTTP(w, r) 76 | case "PUT": 77 | self.PutObject(w, r) 78 | case "DELETE": 79 | self.DeleteObject(w, r) 80 | } 81 | } 82 | 83 | func (self *StorageServer) httpError(w http.ResponseWriter, message string) { 84 | w.Header().Set("Content-Type", "text/plain") 85 | w.WriteHeader(500) 86 | w.Write([]byte(message)) 87 | } 88 | 89 | func (self *StorageServer) PutObject(w http.ResponseWriter, r *http.Request) { 90 | // Final destination of this file 91 | dest := path.Join(self.root, r.URL.Path) 92 | dir := path.Dir(dest) 93 | if _, err := os.Stat(dir); err != nil { 94 | if !os.IsNotExist(err) { 95 | log.Printf("Failed to stat directory %s: %s", dir, err) 96 | self.httpError(w, "Unknown error") 97 | return 98 | } 99 | // If it doesn't exist, create this guy 100 | err = os.MkdirAll(dir, 0744) 101 | if err != nil { 102 | log.Printf("Failed to create directory: %s", err) 103 | self.httpError(w, "Failed to create directory") 104 | return 105 | } 106 | } 107 | 108 | // Slurp the contents to a temporary file 109 | tempdest, err := ioutil.TempFile(self.tempdir, "object") 110 | if err != nil { 111 | log.Printf("Failed to create tempfile: %s", err) 112 | self.httpError(w, "Failed to create file") 113 | return 114 | } 115 | 116 | defer func() { 117 | os.Remove(tempdest.Name()) 118 | }() 119 | 120 | if _, err = io.Copy(tempdest, r.Body); err != nil { 121 | log.Printf("Failed to copy contents of request to temporary file %s: %s", tempdest, err) 122 | self.httpError(w, "Failed to create file") 123 | return 124 | } 125 | 126 | // Now move this sucker to the actual location 127 | if err = os.Rename(tempdest.Name(), dest); err != nil { 128 | log.Printf("Failed to rename %s to %s: %s", tempdest.Name(), dest, err) 129 | self.httpError(w, "Failed to create file") 130 | return 131 | } 132 | 133 | if hdr := r.Header.Get("X-STF-Object-Timestamp"); hdr != "" { 134 | timestamp, err := strconv.ParseInt(hdr, 10, 64) 135 | if err != nil { 136 | log.Printf("Failed to parse timestamp (%s) of file %s (ignored): %s", hdr, dest, err) 137 | } else { 138 | t := time.Unix(timestamp, 0) 139 | err = os.Chtimes(dest, t, t) 140 | if err != nil { 141 | log.Printf("Failed to set timestamp (%s) of file %s (ignored): %s", t.String(), dest, err) 142 | } 143 | } 144 | } 145 | 146 | log.Printf("Successfully created %s", dest) 147 | 148 | w.WriteHeader(201) 149 | } 150 | 151 | func (self *StorageServer) DeleteObject(w http.ResponseWriter, r *http.Request) { 152 | dest := path.Join(self.root, r.URL.Path) 153 | if _, err := os.Stat(dest); err != nil { 154 | if !os.IsNotExist(err) { 155 | log.Printf("Failed to stat file %s: %s", dest, err) 156 | self.httpError(w, "Unknown error") 157 | return 158 | } 159 | w.WriteHeader(404) 160 | return 161 | } 162 | 163 | err := os.Remove(dest) 164 | if err != nil { 165 | log.Printf("Failed to remove file %s: %s", dest, err) 166 | self.httpError(w, "Failed to remove file") 167 | return 168 | } 169 | 170 | w.WriteHeader(204) 171 | } 172 | -------------------------------------------------------------------------------- /worker/base.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "flag" 5 | "github.com/stf-storage/go-stf-server" 6 | "github.com/stf-storage/go-stf-server/api" 7 | "github.com/stf-storage/go-stf-server/config" 8 | "log" 9 | "os" 10 | "path" 11 | "sync" 12 | ) 13 | 14 | type Fetcher interface { 15 | Loop(Worker) 16 | IsRunning() bool 17 | } 18 | 19 | type Worker interface { 20 | Name() string 21 | Loop() bool 22 | ActiveSlaves() int 23 | Ctx() *api.Context 24 | SendJob(*api.WorkerArg) 25 | } 26 | 27 | type BaseWorker struct { 28 | name string 29 | loop bool 30 | activeSlaves int 31 | maxSlaves int 32 | ctx *api.Context 33 | waiter *sync.WaitGroup 34 | fetcher Fetcher 35 | CmdChan chan WorkerCmd 36 | jobChan chan *api.WorkerArg 37 | WorkCb func(*api.WorkerArg) error 38 | } 39 | 40 | type WorkerCmd int 41 | 42 | var ( 43 | CmdWorkerSlaveSpawn = WorkerCmd(1) 44 | CmdWorkerSlaveStop = WorkerCmd(2) 45 | CmdWorkerSlaveStarted = WorkerCmd(3) 46 | CmdWorkerSlaveExited = WorkerCmd(4) 47 | CmdWorkerReload = WorkerCmd(5) 48 | CmdWorkerFetcherSpawn = WorkerCmd(6) 49 | CmdWorkerFetcherExited = WorkerCmd(7) 50 | ) 51 | 52 | func NewBaseWorker(name string, f Fetcher) *BaseWorker { 53 | return &BaseWorker{ 54 | name, 55 | true, 56 | 0, 57 | 0, 58 | nil, 59 | &sync.WaitGroup{}, 60 | f, 61 | make(chan WorkerCmd, 16), 62 | make(chan *api.WorkerArg, 16), 63 | nil, 64 | } 65 | } 66 | 67 | func (w *BaseWorker) Run() { 68 | closer := stf.LogMark("[Worker:%s]", w.name) 69 | defer closer() 70 | 71 | var configFile string 72 | 73 | pwd, err := os.Getwd() 74 | if err != nil { 75 | log.Fatalf("Could not determine current working directory") 76 | } 77 | 78 | defaultConfig := path.Join(pwd, "etc", "config.gcfg") 79 | 80 | flag.StringVar( 81 | &configFile, 82 | "config", 83 | defaultConfig, 84 | "Path to config file", 85 | ) 86 | flag.Parse() 87 | 88 | os.Setenv("STF_CONFIG", configFile) 89 | 90 | cfg, err := config.BootstrapConfig() 91 | if err != nil { 92 | stf.Debugf("Failed to load config: %s", err) 93 | return 94 | } 95 | 96 | w.ctx = api.NewContext(cfg) 97 | 98 | w.SendCmd(CmdWorkerSlaveSpawn) 99 | 100 | w.MainLoop() 101 | stf.Debugf("Worker exiting") 102 | } 103 | 104 | func (w *BaseWorker) SendCmd(cmd WorkerCmd) { 105 | if !w.Loop() { 106 | return 107 | } 108 | 109 | w.CmdChan <- cmd 110 | } 111 | 112 | func (w *BaseWorker) Name() string { 113 | return w.name 114 | } 115 | 116 | func (w *BaseWorker) Ctx() *api.Context { 117 | return w.ctx 118 | } 119 | 120 | func (w *BaseWorker) ActiveSlaves() int { 121 | return w.activeSlaves 122 | } 123 | 124 | func (w *BaseWorker) SendJob(job *api.WorkerArg) { 125 | if !w.Loop() { 126 | return 127 | } 128 | 129 | w.jobChan <- job 130 | } 131 | 132 | func (w *BaseWorker) Loop() bool { 133 | return w.loop 134 | } 135 | 136 | func (w *BaseWorker) MainLoop() { 137 | for { 138 | cmd := <-w.CmdChan 139 | w.HandleCommand(cmd) 140 | } 141 | } 142 | 143 | func (w *BaseWorker) HandleCommand(cmd WorkerCmd) { 144 | switch cmd { 145 | case CmdWorkerReload: 146 | w.ReloadConfig() 147 | case CmdWorkerFetcherSpawn: 148 | w.FetcherSpawn() 149 | case CmdWorkerFetcherExited: 150 | stf.Debugf("Fetcher exited") 151 | w.SendCmd(CmdWorkerFetcherSpawn) 152 | case CmdWorkerSlaveStop: 153 | w.SlaveStop() 154 | case CmdWorkerSlaveSpawn: 155 | w.SlaveSpawn() 156 | case CmdWorkerSlaveStarted: 157 | w.activeSlaves++ 158 | w.SendCmd(CmdWorkerFetcherSpawn) 159 | case CmdWorkerSlaveExited: 160 | w.activeSlaves-- 161 | w.SendCmd(CmdWorkerSlaveSpawn) 162 | } 163 | } 164 | 165 | func (w *BaseWorker) ReloadConfig() { 166 | // Get the number of slaves that we need to spawn. 167 | // If this number is different than the previous state, 168 | // send myself CmdWorkerSlaveStop or CmdWorkerStartSlave 169 | if !w.loop { 170 | return 171 | } 172 | 173 | oldMaxSlaves := w.maxSlaves 174 | 175 | diff := oldMaxSlaves - w.activeSlaves 176 | if diff > 0 { 177 | for i := 0; i < diff; i++ { 178 | w.SendCmd(CmdWorkerSlaveStop) 179 | } 180 | } else if diff < 0 { 181 | diff = diff * -1 182 | for i := 0; i < diff; i++ { 183 | w.SendCmd(CmdWorkerSlaveSpawn) 184 | } 185 | } 186 | } 187 | 188 | func (w *BaseWorker) FetcherSpawn() { 189 | if !w.loop { 190 | return 191 | } 192 | 193 | // do we already have a fetcher? 194 | if w.fetcher.IsRunning() { 195 | return 196 | } 197 | 198 | go func() { 199 | defer w.SendCmd(CmdWorkerFetcherExited) 200 | w.fetcher.Loop(w) 201 | }() 202 | } 203 | 204 | func (w *BaseWorker) SlaveStop() { 205 | if !w.loop { 206 | return 207 | } 208 | 209 | w.jobChan <- nil 210 | } 211 | 212 | func (w *BaseWorker) SlaveSpawn() { 213 | // Bail out if we're supposed to be exiting 214 | if !w.loop { 215 | return 216 | } 217 | 218 | go w.SlaveLoop() 219 | } 220 | 221 | func (w *BaseWorker) SlaveLoop() { 222 | w.SendCmd(CmdWorkerSlaveStarted) 223 | // if we exited, spawn a new one 224 | defer func() { w.SendCmd(CmdWorkerSlaveExited) }() 225 | 226 | stf.Debugf("New slave starting...") 227 | for { 228 | job := <-w.jobChan 229 | if job == nil { 230 | break // Bail out of loop 231 | } 232 | w.WorkCb(job) 233 | } 234 | } 235 | -------------------------------------------------------------------------------- /api/bucket.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "errors" 5 | "github.com/stf-storage/go-stf-server/data" 6 | "strconv" 7 | ) 8 | 9 | type Bucket struct { 10 | *BaseApi 11 | } 12 | 13 | func NewBucket(ctx ContextWithApi) *Bucket { 14 | return &Bucket{&BaseApi{ctx}} 15 | } 16 | 17 | func (self *Bucket) LookupIdByName(name string) (uint64, error) { 18 | ctx := self.Ctx() 19 | 20 | closer := ctx.LogMark("[Bucket.LookupIdByName]") 21 | defer closer() 22 | 23 | ctx.Debugf("Looking for bucket '%s'", name) 24 | 25 | tx, err := ctx.Txn() 26 | if err != nil { 27 | return 0, err 28 | } 29 | 30 | row := tx.QueryRow("SELECT id FROM bucket WHERE name = ?", name) 31 | 32 | var id uint64 33 | err = row.Scan(&id) 34 | if err != nil { 35 | return 0, err 36 | } 37 | 38 | ctx.Debugf("Found id '%d' for bucket '%s'", id, name) 39 | return id, nil 40 | } 41 | 42 | func (self *Bucket) LookupFromDB( 43 | id uint64, 44 | ) (*data.Bucket, error) { 45 | ctx := self.Ctx() 46 | 47 | closer := ctx.LogMark("[Bucket.LookupFromDB]") 48 | defer closer() 49 | 50 | var b data.Bucket 51 | tx, err := ctx.Txn() 52 | if err != nil { 53 | return nil, err 54 | } 55 | 56 | row := tx.QueryRow("SELECT id, name FROM bucket WHERE id = ?", id) 57 | err = row.Scan(&b.Id, &b.Name) 58 | if err != nil { 59 | ctx.Debugf("Failed to scan query: %s", err) 60 | return nil, err 61 | } 62 | return &b, nil 63 | } 64 | 65 | func (self *Bucket) Lookup(id uint64) (*data.Bucket, error) { 66 | ctx := self.Ctx() 67 | closer := ctx.LogMark("[Bucket.Lookup]") 68 | defer closer() 69 | 70 | var b data.Bucket 71 | cache := ctx.Cache() 72 | cacheKey := cache.CacheKey("bucket", strconv.FormatUint(id, 10)) 73 | err := cache.Get(cacheKey, &b) 74 | 75 | if err == nil { 76 | ctx.Debugf("Cache HIT. Loaded from Memcached") 77 | return &b, nil 78 | } 79 | 80 | ctx.Debugf("Cache MISS. Loading from database") 81 | 82 | bptr, err := self.LookupFromDB(id) 83 | if err != nil { 84 | return nil, err 85 | } 86 | ctx.Debugf("Successfully looked up bucket '%d' from DB", b.Id) 87 | cache.Set(cacheKey, *bptr, 600) 88 | 89 | return bptr, nil 90 | } 91 | 92 | func (self *Bucket) Create(id uint64, name string) error { 93 | ctx := self.Ctx() 94 | 95 | closer := ctx.LogMark("[Bucket.Create]") 96 | defer closer() 97 | 98 | tx, err := ctx.Txn() 99 | if err != nil { 100 | return err 101 | } 102 | 103 | _, err = tx.Exec( 104 | "INSERT INTO bucket (id, name, created_at, updated_at) VALUES (?, ?, UNIX_TIMESTAMP(), NOW())", 105 | id, 106 | name, 107 | ) 108 | 109 | if err != nil { 110 | return err 111 | } 112 | 113 | ctx.Debugf("Created bucket '%s' (id = %d)", name, id) 114 | 115 | return nil 116 | } 117 | 118 | func (self *Bucket) MarkForDelete(id uint64) error { 119 | ctx := self.Ctx() 120 | 121 | closer := ctx.LogMark("[Bucket.MarkForDelete]") 122 | defer closer() 123 | 124 | tx, err := ctx.Txn() 125 | if err != nil { 126 | return err 127 | } 128 | 129 | res, err := tx.Exec("REPLACE INTO deleted_bucket SELECT * FROM bucket WHERE id = ?", id) 130 | 131 | if err != nil { 132 | ctx.Debugf("Failed to execute query (REPLACE into deleted_bucket): %s", err) 133 | return err 134 | } 135 | 136 | if count, _ := res.RowsAffected(); count <= 0 { 137 | // Grr, we failed to insert to deleted_bucket table 138 | err = errors.New("Failed to insert bucket into deleted bucket queue") 139 | ctx.Debugf("%s", err) 140 | return err 141 | } 142 | 143 | res, err = tx.Exec("DELETE FROM bucket WHERE id = ?", id) 144 | if err != nil { 145 | ctx.Debugf("Failed to execute query (DELETE from bucket): %s", err) 146 | return err 147 | } 148 | 149 | if count, _ := res.RowsAffected(); count <= 0 { 150 | err = errors.New("Failed to delete bucket") 151 | ctx.Debugf("%s", err) 152 | return err 153 | } 154 | 155 | cache := ctx.Cache() 156 | cache.Delete(cache.CacheKey("bucket", strconv.FormatUint(id, 10))) 157 | 158 | return nil 159 | } 160 | 161 | func (self *Bucket) DeleteObjects(id uint64) error { 162 | ctx := self.Ctx() 163 | 164 | closer := ctx.LogMark("[Bucket.LookupIdByName]") 165 | defer closer() 166 | 167 | tx, err := ctx.Txn() 168 | if err != nil { 169 | return err 170 | } 171 | 172 | rows, err := tx.Query("SELECT id FROM object WHERE bucket_id = ?", id) 173 | if err != nil { 174 | ctx.Debugf("Failed to execute query: %s", err) 175 | return err 176 | } 177 | 178 | var objectId uint64 179 | queueApi := ctx.QueueApi() 180 | for rows.Next() { 181 | err = rows.Scan(&objectId) 182 | if err != nil { 183 | ctx.Debugf("Failed to scan from query: %s", err) 184 | return err 185 | } 186 | 187 | err = queueApi.Enqueue("delete_object", strconv.FormatUint(objectId, 10)) 188 | if err != nil { 189 | ctx.Debugf("Failed to insert object ID in delete_object queue: %s", err) 190 | } 191 | } 192 | 193 | _, err = tx.Exec("DELETE FROM deleted_bucket WHERE id = ?", id) 194 | if err != nil { 195 | ctx.Debugf("Failed to delete bucket from deleted_bucket: %s", err) 196 | } 197 | 198 | return nil 199 | } 200 | 201 | func (self *Bucket) Delete(id uint64, recursive bool) error { 202 | ctx := self.Ctx() 203 | 204 | closer := ctx.LogMark("[Bucket.Delete]") 205 | defer closer() 206 | 207 | if recursive { 208 | err := self.DeleteObjects(id) 209 | if err != nil { 210 | return err 211 | } 212 | } 213 | return nil 214 | } 215 | -------------------------------------------------------------------------------- /dispatcher/dispatcher.go: -------------------------------------------------------------------------------- 1 | package dispatcher 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/braintree/manners" 7 | "github.com/lestrrat/go-apache-logformat" 8 | "github.com/lestrrat/go-file-rotatelogs" 9 | "github.com/lestrrat/go-server-starter-listener" 10 | "github.com/stf-storage/go-stf-server/api" 11 | "github.com/stf-storage/go-stf-server/config" 12 | "log" 13 | "net/http" 14 | "os" 15 | "regexp" 16 | "runtime" 17 | "runtime/debug" 18 | "strings" 19 | ) 20 | 21 | type Dispatcher struct { 22 | config *config.Config 23 | Address string 24 | Ctx *api.Context 25 | ResponseWriter *http.ResponseWriter 26 | Request *http.Request 27 | logger *apachelog.ApacheLog 28 | idgen *UUIDGen 29 | } 30 | 31 | type DispatcherContext struct { 32 | *api.Context 33 | ResponseWriter http.ResponseWriter 34 | request *http.Request 35 | } 36 | 37 | type DispatcherContextWithApi interface { 38 | api.ContextWithApi 39 | Request() *http.Request 40 | } 41 | 42 | func init() { 43 | if os.Getenv("GOMAXPROCS") == "" { 44 | old := runtime.GOMAXPROCS(-1) 45 | cpu := runtime.NumCPU() 46 | log.Printf("Automatically setting GOMAXPROCS to %d (was %d)", cpu, old) 47 | runtime.GOMAXPROCS(cpu) 48 | } 49 | } 50 | 51 | func New(config *config.Config) *Dispatcher { 52 | d := &Dispatcher{ 53 | config: config, 54 | idgen: NewIdGenerator(config.Dispatcher.ServerId), 55 | } 56 | 57 | d.logger = apachelog.CombinedLog.Clone() 58 | 59 | if filename := config.Dispatcher.AccessLog; filename != "" { 60 | rl := rotatelogs.NewRotateLogs(filename) 61 | if linkname := config.Dispatcher.AccessLogLink; linkname != "" { 62 | rl.LinkName = linkname 63 | } 64 | d.logger.SetOutput(rl) 65 | } 66 | 67 | return d 68 | } 69 | 70 | func (ctx *DispatcherContext) Request() *http.Request { 71 | return ctx.request 72 | } 73 | 74 | func (self *Dispatcher) IdGenerator() *UUIDGen { 75 | return self.idgen 76 | } 77 | 78 | func (self *Dispatcher) Start() { 79 | ctx := api.NewContext(self.config) 80 | 81 | // Work with Server::Stareter 82 | baseListener, err := ss.NewListenerOrDefault("tcp", self.config.Dispatcher.Listen) 83 | if err != nil { 84 | panic(fmt.Sprintf("Failed to listen at %s: %s", self.config.Dispatcher.Listen, err)) 85 | } 86 | ctx.Debugf("Listening on %s", baseListener.Addr()) 87 | 88 | s := manners.NewServer() 89 | l := manners.NewListener(baseListener, s) 90 | err = http.Serve(l, self) 91 | if err != nil { 92 | log.Fatal( 93 | fmt.Sprintf("Error from server's ListenAndServe: %s\n", err), 94 | ) 95 | } 96 | } 97 | 98 | func (self *Dispatcher) ServeHTTP(w http.ResponseWriter, r *http.Request) { 99 | ctx := &DispatcherContext{ 100 | api.NewContext(self.config), 101 | w, 102 | r, 103 | } 104 | closer := ctx.LogMark("[%s %s]", r.Method, r.URL.Path) 105 | defer closer() 106 | 107 | defer apachelog.NewLoggingWriter(w, r, self.logger).EmitLog() 108 | 109 | // Generic catch-all handler 110 | defer func() { 111 | if err := recover(); err != nil { 112 | debug.PrintStack() 113 | ctx.Debugf("Error while serving request: %s", err) 114 | http.Error(w, http.StatusText(500), 500) 115 | } 116 | }() 117 | 118 | // First see if we have a proper URL that STF understands 119 | bucketName, objectName, err := parseObjectPath(r.URL.Path) 120 | if err != nil { 121 | http.Error(w, http.StatusText(404), 404) 122 | return 123 | } 124 | ctx.Debugf( 125 | "Parsed bucketName = '%s', objectName = '%s'\n", 126 | bucketName, 127 | objectName, 128 | ) 129 | 130 | var resp *HTTPResponse 131 | switch r.Method { 132 | case "GET": 133 | resp = self.FetchObject(ctx, bucketName, objectName) 134 | case "DELETE": 135 | if objectName == "" { 136 | resp = self.DeleteBucket(ctx, bucketName) 137 | } else { 138 | resp = self.DeleteObject(ctx, bucketName, objectName) 139 | } 140 | case "PUT": 141 | // If the Content-Length is 0, then attempt to create the 142 | // Bucket Otherwise, try the bucket 143 | if cl := r.ContentLength; cl > 0 { 144 | resp = self.CreateObject(ctx, bucketName, objectName) 145 | } else { 146 | resp = self.CreateBucket(ctx, bucketName, objectName) 147 | } 148 | case "POST": 149 | resp = self.ModifyObject(ctx, bucketName, objectName) 150 | case "MOVE": 151 | dest := r.Header.Get("X-STF-Move-Destination") 152 | if objectName == "" { 153 | resp = self.RenameBucket(ctx, bucketName, dest) 154 | } else { 155 | resp = self.RenameObject(ctx, bucketName, objectName, dest) 156 | } 157 | default: 158 | resp = HTTPMethodNotAllowed 159 | return 160 | } 161 | 162 | if resp == nil { 163 | panic("Did not get a response object?!") 164 | } 165 | resp.Write(ctx.ResponseWriter) 166 | } 167 | 168 | func parseObjectPath(path string) (string, string, error) { 169 | precedingSlashRegexp := regexp.MustCompile(`^/`) 170 | 171 | // The path starts with a "/", but it's really not necessary 172 | path = precedingSlashRegexp.ReplaceAllLiteralString(path, "") 173 | 174 | len := len(path) 175 | index := strings.Index(path, "/") 176 | 177 | if index == 0 { 178 | // Whoa, found a slash as the first thing? 179 | return "", "", errors.New("No bucket name could be extracted") 180 | } 181 | 182 | if index == -1 { 183 | // No slash? is this all bucket names? 184 | if len > 1 { 185 | return path, "", nil 186 | } else { 187 | return "", "", errors.New("No bucket name could be extracted") 188 | } 189 | } 190 | 191 | // If we got here, least 1 "/" was found. 192 | 193 | bucketName := path[0:index] 194 | objectName := path[index+1 : len] 195 | 196 | index = strings.Index(objectName, "/") 197 | if index == 0 { 198 | // Duplicate slashes. Fuck you 199 | return "", "", errors.New("Illegal object name") 200 | } 201 | 202 | return bucketName, objectName, nil 203 | } 204 | -------------------------------------------------------------------------------- /api/storage.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "fmt" 5 | "github.com/stf-storage/go-stf-server" 6 | "github.com/stf-storage/go-stf-server/data" 7 | "strconv" 8 | "strings" 9 | ) 10 | 11 | type Storage struct { 12 | *BaseApi 13 | } 14 | 15 | func NewStorage(ctx ContextWithApi) *Storage { 16 | return &Storage{&BaseApi{ctx}} 17 | } 18 | 19 | func (self *Storage) LookupFromDB(id uint64) (*data.Storage, error) { 20 | ctx := self.Ctx() 21 | 22 | closer := ctx.LogMark("[Storage.LookupFromDB]") 23 | defer closer() 24 | 25 | tx, err := ctx.Txn() 26 | if err != nil { 27 | return nil, err 28 | } 29 | row := tx.QueryRow("SELECT id, cluster_id, uri, mode, created_at, updated_at FROM storage WHERE id = ?", id) 30 | 31 | var s data.Storage 32 | err = row.Scan( 33 | &s.Id, 34 | &s.ClusterId, 35 | &s.Uri, 36 | &s.Mode, 37 | &s.CreatedAt, 38 | &s.UpdatedAt, 39 | ) 40 | 41 | if err != nil { 42 | ctx.Debugf("Failed to execute query (StorageLookup): %s", err) 43 | return &s, err 44 | } 45 | 46 | ctx.Debugf("Successfully loaded storage %d from database", id) 47 | 48 | return &s, nil 49 | } 50 | 51 | func (self *Storage) Lookup(id uint64) (*data.Storage, error) { 52 | ctx := self.Ctx() 53 | 54 | closer := ctx.LogMark("[Storage.StorageLookup]") 55 | defer closer() 56 | 57 | var s data.Storage 58 | cache := ctx.Cache() 59 | cacheKey := cache.CacheKey("storage", strconv.FormatUint(uint64(id), 10)) 60 | err := cache.Get(cacheKey, &s) 61 | if err == nil { 62 | ctx.Debugf("Cache HIT for storage %d, returning storage from cache", id) 63 | return &s, nil 64 | } 65 | 66 | ctx.Debugf("Cache MISS for '%s', fetching from database", cacheKey) 67 | 68 | sptr, err := self.LookupFromDB(id) 69 | if err != nil { 70 | return nil, err 71 | } 72 | 73 | cache.Set(cacheKey, *sptr, 3600) 74 | return sptr, nil 75 | } 76 | 77 | func (self *Storage) LookupFromSql(sql string, binds []interface{}) ([]*data.Storage, error) { 78 | ctx := self.Ctx() 79 | 80 | tx, err := ctx.Txn() 81 | if err != nil { 82 | return nil, err 83 | } 84 | 85 | rows, err := tx.Query(sql, binds...) 86 | if err != nil { 87 | return nil, err 88 | } 89 | 90 | var ids []uint64 91 | for rows.Next() { 92 | var sid uint64 93 | err = rows.Scan(&sid) 94 | if err != nil { 95 | return nil, err 96 | } 97 | ids = append(ids, sid) 98 | } 99 | 100 | return self.LookupMulti(ids) 101 | } 102 | 103 | func (self *Storage) LookupMulti(ids []uint64) ([]*data.Storage, error) { 104 | ctx := self.Ctx() 105 | 106 | closer := ctx.LogMark("[Storage.LookupMulti]") 107 | defer closer() 108 | 109 | cache := ctx.Cache() 110 | 111 | var keys []string 112 | for _, id := range ids { 113 | key := cache.CacheKey("storage", strconv.FormatUint(id, 10)) 114 | keys = append(keys, key) 115 | } 116 | 117 | var cached map[string]interface{} 118 | cached, err := cache.GetMulti(keys, func() interface{} { return &Storage{} }) 119 | if err != nil { 120 | ctx.Debugf("GetMulti failed: %s", err) 121 | return nil, err 122 | } 123 | 124 | var ret []*data.Storage 125 | misses := 0 126 | for _, id := range ids { 127 | key := cache.CacheKey("storage", strconv.FormatUint(id, 10)) 128 | st, ok := cached[key].(*data.Storage) 129 | 130 | var s *data.Storage 131 | if ok { 132 | s = st 133 | } else { 134 | ctx.Debugf("Cache MISS on key '%s'", key) 135 | misses++ 136 | s, err = self.Lookup(id) 137 | if err != nil { 138 | return nil, err 139 | } 140 | } 141 | ret = append(ret, s) 142 | } 143 | 144 | ctx.Debugf("Loaded %d storages (cache misses = %d)", len(ret), misses) 145 | return ret, nil 146 | } 147 | 148 | func (self *Storage) LoadInCluster(clusterId uint64) ([]*data.Storage, error) { 149 | ctx := self.Ctx() 150 | 151 | closer := ctx.LogMark("[Storage.LoadInCluster]") 152 | defer closer() 153 | 154 | sql := `SELECT id FROM storage WHERE cluster_id = ?` 155 | 156 | list, err := self.LookupFromSql(sql, []interface{}{clusterId}) 157 | if err != nil { 158 | return nil, err 159 | } 160 | 161 | ctx.Debugf("Loaded %d storages", len(list)) 162 | return list, nil 163 | } 164 | 165 | func (self *Storage) ReadableModes(isRepair bool) []int { 166 | var modes []int 167 | if isRepair { 168 | modes = stf.READABLE_MODES_ON_REPAIR 169 | } else { 170 | modes = stf.READABLE_MODES 171 | } 172 | return modes 173 | } 174 | 175 | func (self *Storage) WritableModes(isRepair bool) []int { 176 | var modes []int 177 | if isRepair { 178 | modes = stf.WRITABLE_MODES_ON_REPAIR 179 | } else { 180 | modes = stf.WRITABLE_MODES 181 | } 182 | return modes 183 | } 184 | 185 | func IsModeIn(s *data.Storage, modes []int) bool { 186 | for _, mode := range modes { 187 | if s.Mode == mode { 188 | return true 189 | } 190 | } 191 | return false 192 | } 193 | 194 | func (self *Storage) IsReadable(s *data.Storage, isRepair bool) bool { 195 | return IsModeIn(s, self.ReadableModes(isRepair)) 196 | } 197 | 198 | func (self *Storage) IsWritable(s *data.Storage, isRepair bool) bool { 199 | return IsModeIn(s, self.WritableModes(isRepair)) 200 | } 201 | 202 | func (self *Storage) LoadWritable(clusterId uint64, isRepair bool) ([]*data.Storage, error) { 203 | ctx := self.Ctx() 204 | 205 | closer := ctx.LogMark("[Storage.LoadWritable]") 206 | defer closer() 207 | 208 | placeholders := []string{} 209 | binds := []interface{}{clusterId} 210 | modes := self.WritableModes(isRepair) 211 | 212 | ctx.Debugf("Repair flag is '%v', using %+v for modes", isRepair, modes) 213 | 214 | for _, v := range modes { 215 | binds = append(binds, v) 216 | placeholders = append(placeholders, "?") 217 | } 218 | 219 | sql := fmt.Sprintf( 220 | "SELECT id FROM storage WHERE cluster_id = ? AND mode IN (%s)", 221 | strings.Join(placeholders, ", "), 222 | ) 223 | 224 | list, err := self.LookupFromSql(sql, binds) 225 | if err != nil { 226 | return nil, err 227 | } 228 | 229 | ctx.Debugf("Loaded %d storages", len(list)) 230 | return list, nil 231 | } 232 | -------------------------------------------------------------------------------- /api/context.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "database/sql" 5 | "errors" 6 | _ "github.com/go-sql-driver/mysql" 7 | "github.com/stf-storage/go-stf-server" 8 | "github.com/stf-storage/go-stf-server/cache" 9 | "github.com/stf-storage/go-stf-server/config" 10 | "io" 11 | "log" 12 | "os" 13 | "strconv" 14 | "strings" 15 | ) 16 | 17 | type TxnManager interface { 18 | TxnBegin() (func(), error) 19 | TxnCommit() error 20 | TxnRollback() 21 | } 22 | 23 | type ContextWithApi interface { 24 | Config() *config.Config 25 | Cache() *cache.MemdClient 26 | Txn() (*sql.Tx, error) 27 | LogMark(string, ...interface{}) func() 28 | Debugf(string, ...interface{}) 29 | ApiHolder 30 | TxnManager 31 | } 32 | 33 | type Context struct { 34 | debug *stf.DebugLog 35 | HomeStr string 36 | bucketapi *Bucket 37 | config *config.Config 38 | deletedobjectapi *DeletedObject 39 | entityapi *Entity 40 | objectapi *Object 41 | queueapi QueueApiInterface 42 | storageapi *Storage 43 | storageclusterapi *StorageCluster 44 | cache *cache.MemdClient 45 | maindb *stf.DB 46 | tx *sql.Tx 47 | } 48 | 49 | func (self *Context) Home() string { return self.HomeStr } 50 | func (ctx *Context) LoadConfig() (*config.Config, error) { 51 | return config.LoadConfig(ctx.Home()) 52 | } 53 | 54 | func NewContext(config *config.Config) *Context { 55 | return &Context{ 56 | config: config, 57 | debug: stf.NewDebugLog(), 58 | } 59 | } 60 | 61 | func (self *Context) NewScope() (*ScopedContext, error) { 62 | return NewScopedContext(self.Config()) 63 | } 64 | 65 | type ScopedContext struct { 66 | config *config.Config 67 | tx *sql.Tx 68 | maindb *stf.DB 69 | } 70 | 71 | func (self *Context) MainDB() (*stf.DB, error) { 72 | if db := self.maindb; db != nil { 73 | return db, nil 74 | } 75 | 76 | db, err := stf.ConnectDB(&self.Config().MainDB) 77 | if err != nil { 78 | return nil, err 79 | } 80 | 81 | self.maindb = db 82 | return db, nil 83 | } 84 | 85 | func (self *Context) Config() *config.Config { 86 | return self.config 87 | } 88 | 89 | func (self *ScopedContext) EndScope() {} 90 | func NewScopedContext(config *config.Config) (*ScopedContext, error) { 91 | return &ScopedContext{ 92 | config, 93 | nil, 94 | nil, 95 | }, nil 96 | } 97 | 98 | func (self *Context) TxnBegin() (func(), error) { 99 | db, err := self.MainDB() 100 | if err != nil { 101 | return nil, err 102 | } 103 | 104 | tx, err := db.Begin() 105 | if err != nil { 106 | return nil, err 107 | } 108 | 109 | self.tx = tx 110 | 111 | return self.TxnRollback, nil 112 | } 113 | 114 | func (self *Context) TxnRollback() { 115 | if tx, _ := self.Txn(); tx != nil { 116 | tx.Rollback() 117 | } 118 | } 119 | 120 | var ErrNoTxnInProgress = errors.New("No transaction in progress") 121 | 122 | func (self *Context) TxnCommit() error { 123 | if tx, _ := self.Txn(); tx != nil { 124 | if err := tx.Commit(); err != nil { 125 | return err 126 | } 127 | self.Debugf("Commited current transaction") 128 | self.tx = nil 129 | return nil 130 | } 131 | 132 | return ErrNoTxnInProgress 133 | } 134 | 135 | func (self *Context) Txn() (*sql.Tx, error) { 136 | if self.tx == nil { 137 | return nil, ErrNoTxnInProgress 138 | } 139 | return self.tx, nil 140 | } 141 | 142 | func BootstrapContext() (*Context, error) { 143 | ctx := NewContext(nil) 144 | 145 | cfg, err := ctx.LoadConfig() 146 | if err != nil { 147 | return nil, err 148 | } 149 | 150 | ctx.config = cfg 151 | 152 | var dbgOutput io.Writer = os.Stderr 153 | if dbg := os.Getenv("STF_DEBUG"); dbg != "" { 154 | // STF_DEBUG='1:path/to/log' 155 | if strings.Index(dbg, ":") > -1 { 156 | list := strings.Split(dbg, ":") 157 | dbg = list[0] 158 | if len(list) > 1 { 159 | file, err := os.OpenFile(list[1], os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0755) 160 | if err != nil { 161 | log.Fatalf("Failed to open debug log output %s: %s", list[1], err) 162 | } 163 | dbgOutput = file 164 | } 165 | } 166 | x, err := strconv.ParseBool(dbg) 167 | if err == nil { 168 | cfg.Global.Debug = x 169 | } 170 | } 171 | 172 | if cfg.Global.Debug { 173 | log.SetOutput(dbgOutput) 174 | ctx.debug = stf.NewDebugLog() 175 | ctx.debug.Prefix = "GLOBAL" 176 | ctx.debug.Printf("Starting debug log") 177 | } 178 | 179 | return ctx, nil 180 | } 181 | 182 | func (self *Context) Debugf(format string, args ...interface{}) { 183 | if dl := self.debug; dl != nil { 184 | dl.Printf(format, args...) 185 | } 186 | } 187 | 188 | func (self *Context) Cache() *cache.MemdClient { 189 | if self.cache == nil { 190 | config := self.Config() 191 | self.cache = cache.NewMemdClient(config.Memcached.Servers...) 192 | } 193 | return self.cache 194 | } 195 | 196 | func (ctx *Context) BucketApi() *Bucket { 197 | if b := ctx.bucketapi; b != nil { 198 | return b 199 | } 200 | ctx.bucketapi = NewBucket(ctx) 201 | return ctx.bucketapi 202 | } 203 | 204 | func (ctx *Context) DeletedObjectApi() *DeletedObject { 205 | if b := ctx.deletedobjectapi; b != nil { 206 | return b 207 | } 208 | ctx.deletedobjectapi = NewDeletedObject(ctx) 209 | return ctx.deletedobjectapi 210 | } 211 | 212 | func (ctx *Context) EntityApi() *Entity { 213 | if b := ctx.entityapi; b != nil { 214 | return b 215 | } 216 | ctx.entityapi = NewEntity(ctx) 217 | return ctx.entityapi 218 | } 219 | 220 | func (ctx *Context) ObjectApi() *Object { 221 | if b := ctx.objectapi; b != nil { 222 | return b 223 | } 224 | ctx.objectapi = NewObject(ctx) 225 | return ctx.objectapi 226 | } 227 | 228 | func (ctx *Context) QueueApi() QueueApiInterface { 229 | if b := ctx.queueapi; b != nil { 230 | return b 231 | } 232 | ctx.queueapi = NewQueue(ctx) 233 | return ctx.queueapi 234 | } 235 | 236 | func (ctx *Context) StorageApi() *Storage { 237 | if b := ctx.storageapi; b != nil { 238 | return b 239 | } 240 | ctx.storageapi = NewStorage(ctx) 241 | return ctx.storageapi 242 | } 243 | 244 | func (ctx *Context) StorageClusterApi() *StorageCluster { 245 | if b := ctx.storageclusterapi; b != nil { 246 | return b 247 | } 248 | ctx.storageclusterapi = NewStorageCluster(ctx) 249 | return ctx.storageclusterapi 250 | } 251 | 252 | func (ctx *Context) LogMark(format string, args ...interface{}) func() { 253 | if d := ctx.debug; d != nil { 254 | return d.LogMark(format, args...) 255 | } 256 | return func() {} 257 | } 258 | -------------------------------------------------------------------------------- /dispatcher/object.go: -------------------------------------------------------------------------------- 1 | package dispatcher 2 | 3 | import ( 4 | "bytes" 5 | "database/sql" 6 | "github.com/stf-storage/go-stf-server/api" 7 | "io/ioutil" 8 | "math/rand" 9 | "regexp" 10 | "strconv" 11 | ) 12 | 13 | func (self *Dispatcher) FetchObject(ctx DispatcherContextWithApi, bucketName string, objectName string) *HTTPResponse { 14 | lmc := ctx.LogMark("[Dispatcher.FetchObject]") 15 | defer lmc() 16 | 17 | rbc, err := ctx.TxnBegin() 18 | if err != nil { 19 | ctx.Debugf("%s", err) 20 | return HTTPInternalServerError 21 | } 22 | defer rbc() 23 | 24 | bucketApi := ctx.BucketApi() 25 | bucketId, err := bucketApi.LookupIdByName(bucketName) 26 | if err != nil { 27 | ctx.Debugf("Bucket %s does not exist", bucketName) 28 | return HTTPNotFound 29 | } 30 | bucketObj, err := bucketApi.Lookup(bucketId) 31 | if err != nil { 32 | return HTTPNotFound 33 | } 34 | 35 | objectApi := ctx.ObjectApi() 36 | objectId, err := objectApi.LookupIdByBucketAndPath(bucketObj, objectName) 37 | switch { 38 | case err == sql.ErrNoRows: 39 | ctx.Debugf("No entry found in database") 40 | // failed to lookup, 404 41 | return HTTPNotFound 42 | case err != nil: 43 | // Whatever error 44 | ctx.Debugf("Errored during lookup: %s", err) 45 | return HTTPInternalServerError 46 | } 47 | 48 | objectObj, err := objectApi.Lookup(objectId) 49 | if err != nil { 50 | return HTTPNotFound 51 | } 52 | 53 | ifModifiedSince := ctx.Request().Header.Get("If-Modified-Since") 54 | doHealthCheck := rand.Float64() < 0.001 55 | uri, err := objectApi.GetAnyValidEntityUrl( 56 | bucketObj, 57 | objectObj, 58 | doHealthCheck, 59 | ifModifiedSince, 60 | ) 61 | 62 | switch { 63 | case uri == "": 64 | return HTTPNotFound 65 | case err == api.ErrContentNotModified: 66 | // Special case 67 | return HTTPNotModified 68 | case err != nil: 69 | return HTTPInternalServerError 70 | } 71 | 72 | // something was found, return with a X-Reproxy-URL 73 | response := NewResponse(200) 74 | response.Header.Add("X-Reproxy-URL", uri) 75 | response.Header.Add("X-Accel-Redirect", "/redirect") 76 | 77 | if err = ctx.TxnCommit(); err != nil { 78 | ctx.Debugf("Failed to commit: %s", err) 79 | } 80 | 81 | return response 82 | } 83 | 84 | func (self *Dispatcher) DeleteObject(ctx api.ContextWithApi, bucketName string, objectName string) *HTTPResponse { 85 | rollback, err := ctx.TxnBegin() 86 | if err != nil { 87 | ctx.Debugf("Failed to start transaction: %s", err) 88 | return HTTPInternalServerError 89 | } 90 | defer rollback() 91 | 92 | bucketApi := ctx.BucketApi() 93 | bucketId, err := bucketApi.LookupIdByName(bucketName) 94 | if err != nil { 95 | return &HTTPResponse{Code: 500, Message: "Failed to find bucket"} 96 | } 97 | 98 | bucketObj, err := bucketApi.Lookup(bucketId) 99 | if err != nil { 100 | return HTTPNotFound 101 | } 102 | 103 | if objectName == "" { 104 | return &HTTPResponse{Code: 500, Message: "Could not extact object name"} 105 | } 106 | 107 | objectApi := ctx.ObjectApi() 108 | objectId, err := objectApi.LookupIdByBucketAndPath(bucketObj, objectName) 109 | if err != nil { 110 | ctx.Debugf("Failed to lookup object %s/%s", bucketName, objectName) 111 | return HTTPNotFound 112 | } 113 | 114 | err = objectApi.MarkForDelete(objectId) 115 | if err != nil { 116 | ctx.Debugf("Failed to mark object (%d) as deleted: %s", objectId, err) 117 | return &HTTPResponse{Code: 500, Message: "Failed to mark object as deleted"} 118 | } 119 | 120 | err = ctx.TxnCommit() 121 | if err != nil { 122 | ctx.Debugf("Failed to commit: %s", err) 123 | return HTTPInternalServerError 124 | } 125 | 126 | ctx.Debugf("Successfully deleted object %s/%s", bucketName, objectName) 127 | go func() { 128 | queueApi := ctx.QueueApi() 129 | queueApi.Enqueue("queue_delete_object", strconv.FormatUint(objectId, 10)) 130 | }() 131 | 132 | return HTTPNoContent 133 | } 134 | 135 | var reMatchSuffix = regexp.MustCompile(`\.([a-zA-Z0-9]+)$`) 136 | 137 | func (self *Dispatcher) CreateObject(ctx DispatcherContextWithApi, bucketName string, objectName string) *HTTPResponse { 138 | lmc := ctx.LogMark("[Dispatcher.CreateObject]") 139 | defer lmc() 140 | 141 | rollback, err := ctx.TxnBegin() 142 | if err != nil { 143 | ctx.Debugf("Failed to start transaction: %s", err) 144 | return HTTPInternalServerError 145 | } 146 | defer rollback() 147 | 148 | bucketApi := ctx.BucketApi() 149 | bucketId, err := bucketApi.LookupIdByName(bucketName) 150 | if err != nil { 151 | return &HTTPResponse{Code: 500, Message: "Failed to find bucket"} 152 | } 153 | 154 | bucketObj, err := bucketApi.Lookup(bucketId) 155 | if err != nil { 156 | return HTTPNotFound 157 | } 158 | 159 | if objectName == "" { 160 | return &HTTPResponse{Code: 500, Message: "Could not extact object name"} 161 | } 162 | 163 | objectApi := ctx.ObjectApi() 164 | oldObjectId, err := objectApi.LookupIdByBucketAndPath(bucketObj, objectName) 165 | switch { 166 | case err == sql.ErrNoRows: 167 | // Just means that this is a new object 168 | case err != nil: 169 | // Some unknown error occurred 170 | return HTTPInternalServerError 171 | default: 172 | // Found oldObjectId. Mark this old object to be deleted 173 | // Note: Don't send to the queue just yet 174 | ctx.Debugf( 175 | "Object '%s' on bucket '%s' already exists", 176 | objectName, 177 | bucketName, 178 | ) 179 | objectApi.MarkForDelete(oldObjectId) 180 | } 181 | 182 | matches := reMatchSuffix.FindStringSubmatch(ctx.Request().URL.Path) 183 | var suffix string 184 | if len(matches) < 2 { 185 | suffix = "dat" 186 | } else { 187 | suffix = matches[1] 188 | } 189 | 190 | objectId := self.IdGenerator().CreateId() 191 | 192 | // XXX Request.Body is an io.ReadCloser, which doesn't implment 193 | // a Seek() mechanism. I don't know if there's a better machanism 194 | // for this, but because we want to be using the body many times 195 | // we create a new Buffer 196 | // XXX Do we need to check for malicious requests where 197 | // ContentLength != Request.Body length? 198 | 199 | body, err := ioutil.ReadAll(ctx.Request().Body) 200 | if err != nil { 201 | ctx.Debugf("Failed to read request body: %s", err) 202 | return HTTPInternalServerError 203 | } 204 | 205 | buf := bytes.NewReader(body) 206 | 207 | err = objectApi.Store( 208 | objectId, 209 | bucketObj, 210 | objectName, 211 | ctx.Request().ContentLength, 212 | buf, 213 | suffix, 214 | false, // isRepair = false 215 | true, // force = true 216 | ) 217 | 218 | if err != nil { 219 | return HTTPInternalServerError 220 | } 221 | 222 | ctx.Debugf("Commiting changes") 223 | err = ctx.TxnCommit() 224 | if err != nil { 225 | ctx.Debugf("Failed to commit transaction: %s", err) 226 | return HTTPInternalServerError 227 | } 228 | 229 | ctx.Debugf("Successfully created object %s/%s", bucketName, objectName) 230 | go func() { 231 | queueApi := ctx.QueueApi() 232 | queueApi.Enqueue("queue_replicate", strconv.FormatUint(objectId, 10)) 233 | }() 234 | return HTTPCreated 235 | } 236 | 237 | func (self *Dispatcher) ModifyObject(ctx api.ContextWithApi, bucketName string, objectName string) *HTTPResponse { 238 | return nil 239 | } 240 | 241 | // MOVE /bucket_name 242 | // X-STF-Move-Destination: /new_name 243 | func (self *Dispatcher) RenameObject(ctx api.ContextWithApi, bucketName string, objectName string, dest string) *HTTPResponse { 244 | return nil 245 | } 246 | -------------------------------------------------------------------------------- /stf.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE config ( 2 | varname VARCHAR(127) NOT NULL PRIMARY KEY, 3 | varvalue TEXT 4 | ) ENGINE=InnoDB; 5 | 6 | REPLACE INTO config (varname, varvalue) 7 | VALUES ("stf.drone.AdaptiveThrottler.instances", 1); 8 | REPLACE INTO config (varname, varvalue) 9 | VALUES ("stf.drone.Replicate.instances", 8); 10 | REPLACE INTO config (varname, varvalue) 11 | VALUES ("stf.drone.RepairObject.instances", 4); 12 | REPLACE INTO config (varname, varvalue) 13 | VALUES ("stf.drone.DeleteBucket.instances", 2); 14 | REPLACE INTO config (varname, varvalue) 15 | VALUES ("stf.drone.DeleteObject.instances", 2); 16 | REPLACE INTO config (varname, varvalue) 17 | VALUES ("stf.drone.RepairStorage.instances", 1); 18 | REPLACE INTO config (varname, varvalue) 19 | VALUES ("stf.drone.ContinuousRepair.instances", 1); 20 | REPLACE INTO config (varname, varvalue) 21 | VALUES ("stf.drone.StorageHealth.instances", 1); 22 | REPLACE INTO config (varname, varvalue) 23 | VALUES ("stf.drone.Notify.instances", 1); 24 | 25 | REPLACE INTO config (varname, varvalue) 26 | VALUES ("stf.worker.RepairObject.throttle.auto_adapt", 1); 27 | REPLACE INTO config (varname, varvalue) 28 | VALUES ("stf.worker.RepairObject.throttle.threshold", 300); 29 | REPLACE INTO config (varname, varvalue) 30 | VALUES ("stf.worker.RepairObject.throttle.current_threshold", 0); 31 | 32 | CREATE TABLE storage_cluster ( 33 | id INT NOT NULL PRIMARY KEY, 34 | name VARCHAR(128), 35 | mode TINYINT NOT NULL DEFAULT 1, 36 | KEY (mode) 37 | ) ENGINE=InnoDB; 38 | 39 | CREATE TABLE storage ( 40 | id INT NOT NULL PRIMARY KEY, 41 | cluster_id INT, 42 | uri VARCHAR(100) NOT NULL, 43 | mode TINYINT NOT NULL DEFAULT 1, 44 | created_at INT NOT NULL, 45 | updated_at TIMESTAMP, 46 | FOREIGN KEY(cluster_id) REFERENCES storage_cluster (id) ON DELETE SET NULL, 47 | UNIQUE KEY(uri), 48 | KEY(mode) 49 | ) ENGINE=InnoDB; 50 | 51 | /* 52 | storage_meta - Used to store storage meta data. 53 | 54 | This is a spearate table because historically the 'storage' table 55 | was declared without a character set declaration, and things go 56 | badly when multibyte 'notes' are added. 57 | 58 | Make sure to place ONLY items that has nothing to do with the 59 | core STF functionality here. 60 | 61 | XXX Theoretically this table could be in a different database 62 | than the main mysql instance. 63 | */ 64 | CREATE TABLE storage_meta ( 65 | storage_id INT NOT NULL PRIMARY KEY, 66 | used BIGINT UNSIGNED DEFAULT 0, 67 | capacity BIGINT UNSIGNED DEFAULT 0, 68 | notes TEXT, 69 | /* XXX if we move this table to a different database, then 70 | this foreign key is moot. this is placed where because I'm 71 | too lazy to cleanup the database when we delete the storage 72 | */ 73 | FOREIGN KEY(storage_id) REFERENCES storage(id) ON DELETE CASCADE 74 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 75 | 76 | CREATE TABLE bucket ( 77 | id BIGINT NOT NULL PRIMARY KEY, 78 | name VARCHAR(255) NOT NULL, 79 | objects BIGINT UNSIGNED NOT NULL DEFAULT 0, 80 | created_at INT NOT NULL, 81 | updated_at TIMESTAMP, 82 | UNIQUE KEY(name) 83 | ) ENGINE=InnoDB; 84 | 85 | CREATE TABLE object ( 86 | id BIGINT NOT NULL PRIMARY KEY, 87 | bucket_id BIGINT NOT NULL, 88 | name VARCHAR(255) NOT NULL, 89 | internal_name VARCHAR(128) NOT NULL, 90 | size INT NOT NULL DEFAULT 0, 91 | num_replica INT NOT NULL DEFAULT 1, 92 | status TINYINT DEFAULT 1 NOT NULL, 93 | created_at INT NOT NULL, 94 | updated_at TIMESTAMP, 95 | UNIQUE KEY(bucket_id, name), 96 | UNIQUE KEY(internal_name) 97 | ) ENGINE=InnoDB; 98 | 99 | /* object_cluster_map 100 | maps objects to clusters 101 | */ 102 | CREATE TABLE object_cluster_map ( 103 | object_id BIGINT NOT NULL, 104 | cluster_id INT NOT NULL, 105 | PRIMARY KEY(object_id), 106 | FOREIGN KEY (object_id) REFERENCES object (id) ON DELETE CASCADE, 107 | FOREIGN KEY (cluster_id) REFERENCES storage_cluster (id) ON DELETE CASCADE 108 | ) ENGINE=InnoDB; 109 | 110 | /* object_meta - same caveats as storage_meta applies */ 111 | CREATE TABLE object_meta ( 112 | object_id BIGINT NOT NULL PRIMARY KEY, 113 | hash CHAR(32), 114 | FOREIGN KEY(object_id) REFERENCES object(id) ON DELETE CASCADE 115 | ) ENGINE=InnoDB; 116 | 117 | CREATE TABLE deleted_object ENGINE=InnoDB SELECT * FROM object LIMIT 0; 118 | ALTER TABLE deleted_object ADD PRIMARY KEY(id); 119 | -- ALTER TABLE deleted_object ADD UNIQUE KEY(internal_name); 120 | CREATE TABLE deleted_bucket ENGINE=InnoDB SELECT * FROM bucket LIMIT 0; 121 | ALTER TABLE deleted_bucket ADD PRIMARY KEY(id); 122 | 123 | CREATE TABLE entity ( 124 | object_id BIGINT NOT NULL, 125 | storage_id INT NOT NULL, 126 | status TINYINT DEFAULT 1 NOT NULL, 127 | created_at INT NOT NULL, 128 | updated_at TIMESTAMP, 129 | PRIMARY KEY id (object_id, storage_id), 130 | KEY(object_id, status), 131 | KEY(storage_id), 132 | FOREIGN KEY(storage_id) REFERENCES storage(id) ON DELETE RESTRICT 133 | ) ENGINE=InnoDB; 134 | 135 | CREATE TABLE worker_election ( 136 | id BIGINT AUTO_INCREMENT PRIMARY KEY, 137 | drone_id VARCHAR(255) NOT NULL, 138 | expires_at INT NOT NULL, 139 | UNIQUE KEY (drone_id), 140 | KEY (expires_at) 141 | ) ENGINE=InnoDB DEFAULT CHARACTER SET = 'utf8'; 142 | 143 | CREATE TABLE worker_instances ( 144 | drone_id VARCHAR(255) NOT NULL, 145 | worker_type VARCHAR(255) NOT NULL, 146 | instances INT NOT NULL DEFAULT 1, 147 | PRIMARY KEY(drone_id, worker_type), 148 | FOREIGN KEY(drone_id) REFERENCES worker_election (drone_id) ON DELETE CASCADE 149 | ) ENGINE=InnoDB DEFAULT CHARACTER SET = 'utf8'; 150 | 151 | CREATE TABLE notification ( 152 | id BIGINT AUTO_INCREMENT PRIMARY KEY, 153 | ntype CHAR(40) NOT NULL, 154 | /* source of this notification. should include file + linu num */ 155 | source TEXT NOT NULL, 156 | /* severity of this notification: critical, info ? anything else? */ 157 | severity VARCHAR(32) NOT NULL DEFAULT 'info', 158 | message TEXT NOT NULL, 159 | created_at INT NOT NULL, 160 | KEY(created_at), 161 | KEY(ntype) 162 | ) ENGINE=InnoDB DEFAULT CHARACTER SET = 'utf8'; 163 | 164 | CREATE TABLE notification_rule ( 165 | id BIGINT AUTO_INCREMENT PRIMARY KEY, 166 | 167 | /* status: 0 -> suspended, won't execute. status: 1 -> will execute */ 168 | status TINYINT NOT NULL DEFAULT 1, 169 | 170 | /* operation type, "eq", "ne", "==", "!=", "<=", ">=", "=~" */ 171 | operation CHAR(2) NOT NULL, 172 | /* which notificiation object field to apply thie operation against */ 173 | op_field VARCHAR(255) NOT NULL DEFAULT "ntype", 174 | /* user-defined operand. 175 | e.g., op_field = "ntype", operation = "=~", op_arg = "^foo" 176 | yields "ntype" =~ /^foo/ 177 | */ 178 | op_arg VARCHAR(255) NOT NULL, 179 | /* user-defined extra set of arguments that are required by 180 | the notifier to complete the notification. e.g. 181 | Ikachan notification requires "channel", email notification 182 | requires "to" address. 183 | encoded as JSON string 184 | */ 185 | extra_args TEXT, 186 | /* which notifier to invoke upon rule match. Must be able to 187 | look this up via container->get. e.g. API::Notification::Email 188 | */ 189 | notifier_name VARCHAR(255) NOT NULL, 190 | KEY(status) 191 | ) ENGINE=InnoDB DEFAULT CHARACTER SET = 'utf8'; 192 | 193 | -------------------------------------------------------------------------------- /api/cluster.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "bytes" 5 | "crypto/md5" 6 | "errors" 7 | "fmt" 8 | "github.com/stf-storage/go-stf-server" 9 | "github.com/stf-storage/go-stf-server/data" 10 | "io" 11 | "io/ioutil" 12 | "sort" 13 | "strconv" 14 | ) 15 | 16 | type StorageCluster struct { 17 | *BaseApi 18 | } 19 | 20 | func NewStorageCluster(ctx ContextWithApi) *StorageCluster { 21 | return &StorageCluster{&BaseApi{ctx}} 22 | } 23 | 24 | // These are defined to allow sorting via the sort package 25 | type ClusterCandidates []*data.StorageCluster 26 | 27 | func (self ClusterCandidates) Prepare(objectId uint64) { 28 | idStr := strconv.FormatUint(objectId, 10) 29 | for _, x := range self { 30 | key := strconv.FormatUint(uint64(x.Id), 10) + idStr 31 | x.SortHint = stf.MurmurHash([]byte(key)) 32 | } 33 | } 34 | func (self ClusterCandidates) Len() int { return len(self) } 35 | func (self ClusterCandidates) Swap(i, j int) { self[i], self[j] = self[j], self[i] } 36 | func (self ClusterCandidates) Less(i, j int) bool { 37 | return self[i].SortHint < self[j].SortHint 38 | } 39 | 40 | func (self *StorageCluster) LoadWritable() (ClusterCandidates, error) { 41 | ctx := self.Ctx() 42 | 43 | closer := ctx.LogMark("[Cluster.LoadWritable]") 44 | defer closer() 45 | 46 | tx, err := ctx.Txn() 47 | if err != nil { 48 | return nil, err 49 | } 50 | 51 | rows, err := tx.Query("SELECT id, name, mode FROM storage_cluster WHERE mode = ?", stf.STORAGE_CLUSTER_MODE_READ_WRITE) 52 | 53 | if err != nil { 54 | ctx.Debugf("Failed to execute query: %s", err) 55 | return nil, err 56 | } 57 | 58 | var list ClusterCandidates 59 | for rows.Next() { 60 | s := &data.StorageCluster{} 61 | err = rows.Scan(&s.Id, &s.Name, &s.Mode) 62 | if err != nil { 63 | return nil, err 64 | } 65 | 66 | list = append(list, s) 67 | } 68 | 69 | ctx.Debugf("Loaded %d clusters", len(list)) 70 | 71 | return list, nil 72 | } 73 | 74 | func (self *StorageCluster) LookupFromDB(id uint64) (*data.StorageCluster, error) { 75 | ctx := self.Ctx() 76 | 77 | closer := ctx.LogMark("[StorageCluster.LookupFromDB]") 78 | defer closer() 79 | 80 | tx, err := ctx.Txn() 81 | if err != nil { 82 | return nil, err 83 | } 84 | 85 | row := tx.QueryRow("SELECT name, mode FROM storage_cluster WHERE id = ?", id) 86 | 87 | c := data.StorageCluster{data.StfObject{Id: id}, "", 0, 0} 88 | err = row.Scan(&c.Name, &c.Mode) 89 | if err != nil { 90 | ctx.Debugf("Failed to execute query (LookupFromDB): %s", err) 91 | return nil, err 92 | } 93 | 94 | ctx.Debugf("Loaded storage cluster %d", id) 95 | 96 | return &c, nil 97 | } 98 | 99 | func (self *StorageCluster) Lookup(id uint64) (*data.StorageCluster, error) { 100 | ctx := self.Ctx() 101 | 102 | closer := ctx.LogMark("[StorageCluster.Lookup]") 103 | defer closer() 104 | 105 | var c data.StorageCluster 106 | cache := ctx.Cache() 107 | cacheKey := cache.CacheKey("storage_cluster", strconv.FormatUint(id, 10)) 108 | err := cache.Get(cacheKey, &c) 109 | 110 | if err == nil { 111 | ctx.Debugf("Cache HIT for cluster %d, returning cluster from cache", id) 112 | return &c, nil 113 | } 114 | 115 | cptr, err := self.LookupFromDB(id) 116 | if err != nil { 117 | return nil, err 118 | } 119 | 120 | ctx.Debugf("Successfully loaded cluster %d from database", id) 121 | cache.Set(cacheKey, *cptr, 3600) 122 | return cptr, nil 123 | } 124 | 125 | func (self *StorageCluster) LookupForObject(objectId uint64) (*data.StorageCluster, error) { 126 | ctx := self.Ctx() 127 | 128 | closer := ctx.LogMark("[StorageCluster.LookupForObject]") 129 | defer closer() 130 | 131 | tx, err := ctx.Txn() 132 | if err != nil { 133 | return nil, err 134 | } 135 | 136 | var cid uint64 137 | row := tx.QueryRow("SELECT cluster_id FROM object_cluster_map WHERE object_id = ?", objectId) 138 | err = row.Scan(&cid) 139 | if err != nil { 140 | return nil, err 141 | } 142 | 143 | return self.Lookup(cid) 144 | } 145 | 146 | func (self *StorageCluster) LoadCandidatesFor(objectId uint64) (ClusterCandidates, error) { 147 | ctx := self.Ctx() 148 | 149 | closer := ctx.LogMark("[StorageCluster.LoadCandidatesFor]") 150 | defer closer() 151 | 152 | list, err := self.LoadWritable() 153 | if err != nil { 154 | return nil, err 155 | } 156 | list.Prepare(objectId) 157 | sort.Sort(list) 158 | 159 | return list, nil 160 | } 161 | 162 | func calcMD5(input interface { 163 | Read([]byte) (int, error) 164 | }) []byte { 165 | h := md5.New() 166 | var buf []byte 167 | for { 168 | n, err := input.Read(buf) 169 | if n > 0 { 170 | io.WriteString(h, string(buf)) 171 | } 172 | if n == 0 || err == io.EOF { 173 | break 174 | } 175 | } 176 | 177 | return h.Sum(nil) 178 | } 179 | 180 | func (self *StorageCluster) Store( 181 | clusterId uint64, 182 | o *data.Object, 183 | input *bytes.Reader, 184 | minimumToStore int, 185 | isRepair bool, 186 | force bool, 187 | ) error { 188 | ctx := self.Ctx() 189 | 190 | closer := ctx.LogMark("[Cluster.Store]") 191 | defer closer() 192 | 193 | storageApi := ctx.StorageApi() 194 | storages, err := storageApi.LoadWritable(clusterId, isRepair) 195 | if err != nil { 196 | ctx.Debugf("Failed to load storage candidates for writing: %s", err) 197 | return err 198 | } 199 | 200 | if minimumToStore < 1 { 201 | // Give it a default value 202 | minimumToStore = 3 203 | } 204 | 205 | if len(storages) < minimumToStore { 206 | err = errors.New( 207 | fmt.Sprintf( 208 | "Only loaded %d storages (wanted %d) for cluster %d", 209 | len(storages), 210 | minimumToStore, 211 | clusterId, 212 | ), 213 | ) 214 | ctx.Debugf("%s", err) 215 | return err 216 | } 217 | 218 | var expected []byte 219 | if !force { 220 | // Micro-optimize 221 | expected = calcMD5(input) 222 | } 223 | 224 | entityApi := ctx.EntityApi() 225 | 226 | stored := 0 227 | for _, s := range storages { 228 | ctx.Debugf("Attempting to store to storage %s (id = %d)", s.Uri, s.Id) 229 | // Without the force flag, we fetch the object before storing to 230 | // avoid redundant writes. force should only be used when you KNOW 231 | // that this is a new entity 232 | var fetchedContent []byte 233 | var fetchedMD5 []byte 234 | fetchOK := false 235 | if !force { 236 | // Entity in database needs to exist 237 | if _, err := entityApi.Lookup(o.Id, s.Id); err == nil { 238 | // If it does, not check for the content 239 | if fetched, err := entityApi.FetchContent(o, s, isRepair); err == nil { 240 | if fetchedContent, err = ioutil.ReadAll(fetched); err == nil { 241 | fetchedMD5 = calcMD5(bytes.NewReader(fetchedContent)) 242 | fetchOK = true 243 | } 244 | } 245 | } 246 | } 247 | 248 | if fetchOK { 249 | // Find the MD5 checksum of the fetchedContent, and make sure that 250 | // this indeed matches what we want to store 251 | if bytes.Equal(fetchedMD5, expected) { 252 | // It's a match! 253 | stored++ 254 | ctx.Debugf( 255 | "Entity on storage %d exist, and md5 matches. Assume this is OK", 256 | s.Id, 257 | ) 258 | continue 259 | } 260 | } 261 | 262 | if _, err = input.Seek(0, 0); err != nil { 263 | err = errors.New(fmt.Sprintf("failed to seek: %s", err)) 264 | return err 265 | } 266 | 267 | err = entityApi.Store(s, o, input) 268 | if err == nil { 269 | stored++ 270 | if minimumToStore > 0 && stored >= minimumToStore { 271 | break 272 | } 273 | } 274 | } 275 | 276 | storedOK := minimumToStore == 0 || stored >= minimumToStore 277 | 278 | if !storedOK { 279 | return errors.New( 280 | fmt.Sprintf( 281 | "Only stored %d entities while we wanted %d entities", 282 | stored, 283 | minimumToStore, 284 | ), 285 | ) 286 | } 287 | 288 | ctx.Debugf( 289 | "Stored %d entities for object %d in cluster %d", 290 | stored, 291 | o.Id, 292 | clusterId, 293 | ) 294 | return nil 295 | } 296 | 297 | func (self *StorageCluster) RegisterForObject(clusterId uint64, objectId uint64) error { 298 | return nil 299 | } 300 | 301 | var ErrClusterNotWritable = errors.New("Cluster is not writable") 302 | var ErrNoStorageAvailable = errors.New("No storages available") 303 | 304 | func (self *StorageCluster) CheckEntityHealth( 305 | cluster *data.StorageCluster, 306 | o *data.Object, 307 | isRepair bool, 308 | ) error { 309 | ctx := self.Ctx() 310 | 311 | closer := ctx.LogMark("[StorageCluster.CheckEntityHealth]") 312 | defer closer() 313 | 314 | ctx.Debugf( 315 | "Checking entity health for object %d on cluster %d", 316 | o.Id, 317 | cluster.Id, 318 | ) 319 | 320 | // Short circuit. If the cluster mode is not rw or ro, then 321 | // we have a problem. 322 | if cluster.Mode != stf.STORAGE_CLUSTER_MODE_READ_WRITE && cluster.Mode != stf.STORAGE_CLUSTER_MODE_READ_ONLY { 323 | ctx.Debugf( 324 | "Cluster %d is not read-write or read-only, need to move object %d out of this cluster", 325 | cluster.Id, 326 | o.Id, 327 | ) 328 | return ErrClusterNotWritable 329 | } 330 | 331 | storageApi := ctx.StorageApi() 332 | storages, err := storageApi.LoadInCluster(cluster.Id) 333 | if err != nil { 334 | return ErrNoStorageAvailable 335 | } 336 | 337 | entityApi := ctx.EntityApi() 338 | for _, s := range storages { 339 | err = entityApi.CheckHealth(o, s, isRepair) 340 | if err != nil { 341 | ctx.Debugf("Health check for entity on object %d storage %d failed", o.Id, s.Id) 342 | return errors.New( 343 | fmt.Sprintf( 344 | "Entity for object %d on storage %d is unavailable: %s", 345 | o.Id, 346 | s.Id, 347 | err, 348 | ), 349 | ) 350 | } 351 | } 352 | return nil 353 | } 354 | -------------------------------------------------------------------------------- /drone/drone.go: -------------------------------------------------------------------------------- 1 | package drone 2 | 3 | import ( 4 | "fmt" 5 | "github.com/stf-storage/go-stf-server" 6 | "github.com/stf-storage/go-stf-server/api" 7 | "github.com/stf-storage/go-stf-server/cache" 8 | "github.com/stf-storage/go-stf-server/config" 9 | "log" 10 | "math/rand" 11 | "os" 12 | "os/exec" 13 | "os/signal" 14 | "sync" 15 | "syscall" 16 | "time" 17 | ) 18 | 19 | type Drone struct { 20 | id string 21 | ctx *api.Context 22 | loop bool 23 | leader bool 24 | tasks []*PeriodicTask 25 | minions []*Minion 26 | waiter *sync.WaitGroup 27 | CmdChan chan DroneCmd 28 | sigchan chan os.Signal 29 | lastElectionTime time.Time 30 | } 31 | 32 | type DroneCmd int 33 | 34 | func (dc DroneCmd) String() string { 35 | var name string 36 | switch dc { 37 | case CmdStopDrone: 38 | name = "StopDrone" 39 | case CmdAnnounce: 40 | name = "Account" 41 | case CmdSpawnMinion: 42 | name = "SpawnMinion" 43 | case CmdReloadMinion: 44 | name = "ReloadMinion" 45 | case CmdCheckState: 46 | name = "CheckState" 47 | case CmdElection: 48 | name = "Election" 49 | case CmdRebalance: 50 | name = "Rebalance" 51 | case CmdExpireDrone: 52 | name = "ExpireDrone" 53 | default: 54 | name = fmt.Sprintf("UnknownCmd(%d)", int(dc)) 55 | } 56 | return name 57 | } 58 | 59 | const ( 60 | CmdStopDrone = DroneCmd(iota) 61 | CmdAnnounce = DroneCmd(iota) 62 | CmdSpawnMinion = DroneCmd(iota) 63 | CmdReloadMinion = DroneCmd(iota) 64 | CmdCheckState = DroneCmd(iota) 65 | CmdElection = DroneCmd(iota) 66 | CmdRebalance = DroneCmd(iota) 67 | CmdExpireDrone = DroneCmd(iota) 68 | ) 69 | 70 | func NewDrone(config *config.Config) *Drone { 71 | host, err := os.Hostname() 72 | if err != nil { 73 | log.Fatalf("Failed to get hostname: %s", err) 74 | } 75 | pid := os.Getpid() 76 | id := fmt.Sprintf("%s.%d.%d", host, pid, rand.Int31()) 77 | return &Drone{ 78 | id: id, 79 | ctx: api.NewContext(config), 80 | loop: true, 81 | leader: false, 82 | tasks: nil, 83 | minions: nil, 84 | waiter: &sync.WaitGroup{}, 85 | CmdChan: make(chan DroneCmd, 1), 86 | sigchan: make(chan os.Signal, 1), 87 | lastElectionTime: time.Time{}, 88 | } 89 | } 90 | 91 | func (d *Drone) Loop() bool { 92 | return d.loop 93 | } 94 | 95 | func (d *Drone) Run() { 96 | defer func() { 97 | if list := d.tasks; list != nil { 98 | for _, task := range list { 99 | task.Stop() 100 | } 101 | } 102 | }() 103 | defer func() { 104 | d.Unregister() 105 | for _, m := range d.Minions() { 106 | m.Kill() 107 | } 108 | }() 109 | 110 | d.StartPeriodicTasks() 111 | 112 | go d.WaitSignal() 113 | 114 | // We need to kickstart: 115 | go d.SendCmd(CmdSpawnMinion) 116 | 117 | d.MainLoop() 118 | stf.Debugf("Drone %s exiting...", d.id) 119 | } 120 | 121 | func (d *Drone) WaitSignal() { 122 | sigchan := d.sigchan 123 | signal.Notify(sigchan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP) 124 | 125 | OUTER: 126 | for d.Loop() { 127 | sig, ok := <-sigchan 128 | if !ok { 129 | continue 130 | } 131 | 132 | stf.Debugf("Received signal %s", sig) 133 | 134 | switch sig { 135 | case syscall.SIGTERM, syscall.SIGINT: 136 | d.SendCmd(CmdStopDrone) 137 | signal.Stop(sigchan) 138 | break OUTER 139 | case syscall.SIGHUP: 140 | d.SendCmd(CmdReloadMinion) 141 | } 142 | } 143 | } 144 | 145 | func (d *Drone) makeAnnounceTask() *PeriodicTask { 146 | return NewPeriodicTask( 147 | 30 * time.Second, 148 | true, 149 | func(pt *PeriodicTask) { d.SendCmd(CmdAnnounce) }, 150 | ) 151 | } 152 | 153 | func (d *Drone) makeCheckStateTask() *PeriodicTask { 154 | return NewPeriodicTask( 155 | 5 * time.Second, 156 | true, 157 | func(pt *PeriodicTask) { d.SendCmd(CmdCheckState) }, 158 | ) 159 | } 160 | 161 | func (d *Drone) makeElectionTask() *PeriodicTask { 162 | return NewPeriodicTask( 163 | 300 * time.Second, 164 | false, 165 | func(pt *PeriodicTask) { d.SendCmd(CmdElection) }, 166 | ) 167 | } 168 | 169 | func (d *Drone) makeExpireTask() *PeriodicTask { 170 | return NewPeriodicTask( 171 | 60 * time.Second, 172 | true, 173 | func(pt *PeriodicTask) { d.SendCmd(CmdExpireDrone) }, 174 | ) 175 | } 176 | 177 | func (d *Drone) makeRebalanceTask() *PeriodicTask { 178 | return NewPeriodicTask( 179 | 60 * time.Second, 180 | true, 181 | func(pt *PeriodicTask) { d.SendCmd(CmdRebalance) }, 182 | ) 183 | } 184 | 185 | func (d *Drone) PeriodicTasks() []*PeriodicTask { 186 | if d.tasks != nil { 187 | return d.tasks 188 | } 189 | 190 | if d.leader { 191 | d.tasks = make([]*PeriodicTask, 5) 192 | } else { 193 | d.tasks = make([]*PeriodicTask, 4) 194 | } 195 | 196 | d.tasks[0] = d.makeAnnounceTask() 197 | d.tasks[1] = d.makeCheckStateTask() 198 | d.tasks[2] = d.makeElectionTask() 199 | d.tasks[3] = d.makeExpireTask() 200 | if d.leader { 201 | d.tasks[4] = d.makeRebalanceTask() 202 | } 203 | 204 | return d.tasks 205 | } 206 | 207 | func (d *Drone) Minions() []*Minion { 208 | if d.minions != nil { 209 | return d.minions 210 | } 211 | 212 | cmds := []string{ 213 | // "stf-worker-adaptive_throttler", 214 | // "stf-worker-delete_bucket", 215 | "stf-worker-delete_object", 216 | "stf-worker-replicate_object", 217 | "stf-worker-repair_object", 218 | "stf-worker-storage_health", 219 | } 220 | 221 | d.minions = make([]*Minion, len(cmds)) 222 | for i, x := range cmds { 223 | // XXX x is used as an alias, so for each iteration we'd be using 224 | // the same variable with new value if we bind callback with x 225 | // in order to avoid that, we need to create a new variable, cmdname 226 | cmdname := x 227 | callback := func() *exec.Cmd { 228 | return exec.Command(cmdname, "--config", d.ctx.Config().FileName) 229 | } 230 | d.minions[i] = &Minion{ 231 | drone: d, 232 | cmd: nil, 233 | makeCmd: callback, 234 | } 235 | } 236 | return d.minions 237 | } 238 | 239 | func (d *Drone) NotifyMinions() { 240 | for _, m := range d.Minions() { 241 | m.NotifyReload() 242 | } 243 | } 244 | 245 | func (d *Drone) StartPeriodicTasks() { 246 | for _, task := range d.PeriodicTasks() { 247 | go task.Run() 248 | } 249 | } 250 | 251 | func (d *Drone) MainLoop() { 252 | for d.Loop() { 253 | cmd, ok := <-d.CmdChan 254 | if ok { 255 | d.HandleCommand(cmd) 256 | } 257 | } 258 | } 259 | 260 | func (d *Drone) HandleCommand(cmd DroneCmd) { 261 | switch cmd { 262 | case CmdStopDrone: 263 | stf.Debugf("Hanlding StopDrone") 264 | d.loop = false 265 | case CmdAnnounce: 266 | d.Announce() 267 | case CmdSpawnMinion: 268 | d.SpawnMinions() 269 | case CmdElection: 270 | d.HoldElection() 271 | case CmdReloadMinion: 272 | d.NotifyMinions() 273 | case CmdCheckState: 274 | d.CheckState() 275 | case CmdExpireDrone: 276 | d.ExpireDrones() 277 | } 278 | } 279 | 280 | func (d *Drone) Announce() error { 281 | db, err := d.ctx.MainDB() 282 | if err != nil { 283 | return err 284 | } 285 | 286 | stf.Debugf("Annoucing %s", d.id) 287 | db.Exec(` 288 | INSERT INTO worker_election (drone_id, expires_at) 289 | VALUES (?, UNIX_TIMESTAMP() + 300) 290 | ON DUPLICATE KEY UPDATE expires_at = UNIX_TIMESTAMP() + 300`, 291 | d.id, 292 | ) 293 | 294 | mc := d.ctx.Cache() 295 | mc.Set("go-stf.worker.election", &cache.Int64Value{time.Now().Unix()}, 0) 296 | 297 | return nil 298 | } 299 | 300 | func (d *Drone) ExpireDrones() { 301 | db, err := d.ctx.MainDB() 302 | if err != nil { 303 | return 304 | } 305 | 306 | rows, err := db.Query(`SELECT drone_id FROM worker_election WHERE expires_at <= UNIX_TIMESTAMP()`) 307 | if err != nil { 308 | return 309 | } 310 | 311 | for rows.Next() { 312 | var id string 313 | err = rows.Scan(&id) 314 | if err != nil { 315 | return 316 | } 317 | stf.Debugf("Expiring drone %s", id) 318 | db.Exec(`DELETE FROM worker_election WHERE drone_id = ?`, id) 319 | db.Exec(`DELETE FROM worker_instances WHERE drone_id = ?`, id) 320 | } 321 | } 322 | 323 | func (d *Drone) Unregister() error { 324 | db, err := d.MainDB() 325 | if err != nil { 326 | return err 327 | } 328 | 329 | stf.Debugf("Unregistering drone %s from database", d.id) 330 | db.Exec(`DELETE FROM worker_election WHERE drone_id = ?`, d.id) 331 | db.Exec(`DELETE FROM worker_instances WHERE drone_id = ?`, d.id) 332 | 333 | mc := d.ctx.Cache() 334 | mc.Set("go-stf.worker.election", &cache.Int64Value{time.Now().Unix()}, 0) 335 | 336 | return nil 337 | } 338 | 339 | func (d *Drone) CheckState() { 340 | stf.Debugf("Checking state") 341 | 342 | // Have we run an election up to this point? 343 | lastElectionTime := d.lastElectionTime 344 | if lastElectionTime.IsZero() { 345 | stf.Debugf("First time!") 346 | // then we should just run the election, regardless 347 | go d.SendCmd(CmdElection) 348 | return 349 | } 350 | 351 | var v cache.Int64Value 352 | mc := d.ctx.Cache() 353 | err := mc.Get("go-stf.worker.election", &v) 354 | if err != nil { 355 | stf.Debugf("Failed to fetch election cache key: %s", err) 356 | return 357 | } 358 | 359 | tnano := time.Unix(v.Value, 0) 360 | now := time.Unix(time.Now().Unix(), 0) 361 | stf.Debugf("t.Now = %s", now) 362 | stf.Debugf("last election = %s", lastElectionTime) 363 | stf.Debugf("tnano = %s", tnano) 364 | if lastElectionTime.After(tnano) { 365 | stf.Debugf("last election > tnano, no election") 366 | return 367 | } 368 | 369 | if now.After(tnano) { 370 | go d.SendCmd(CmdElection) 371 | } 372 | } 373 | 374 | func (d *Drone) MainDB() (*stf.DB, error) { 375 | return d.ctx.MainDB() 376 | } 377 | 378 | func (d *Drone) HoldElection() error { 379 | db, err := d.MainDB() 380 | if err != nil { 381 | return err 382 | } 383 | 384 | stf.Debugf("Holding leader election") 385 | row := db.QueryRow(`SELECT drone_id FROM worker_election ORDER BY id ASC LIMIT 1`) 386 | 387 | var id string 388 | err = row.Scan(&id) 389 | if err != nil { 390 | return err 391 | } 392 | 393 | if d.id == id { 394 | stf.Debugf("Elected myself (%s) as leader", id) 395 | d.leader = true 396 | } else { 397 | stf.Debugf("Elected %s as leader", id) 398 | d.leader = false 399 | } 400 | d.lastElectionTime = time.Unix(time.Now().Unix(), 0) 401 | return nil 402 | } 403 | 404 | func (d *Drone) SendCmd(cmd DroneCmd) { 405 | if !d.Loop() { 406 | return 407 | } 408 | 409 | d.CmdChan <- cmd 410 | } 411 | 412 | func (d *Drone) SpawnMinions() { 413 | for _, m := range d.Minions() { 414 | m.Run() 415 | } 416 | } 417 | -------------------------------------------------------------------------------- /api/entity.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "bytes" 5 | "database/sql" 6 | "errors" 7 | "fmt" 8 | "github.com/stf-storage/go-stf-server" 9 | "github.com/stf-storage/go-stf-server/data" 10 | "io" 11 | "net/http" 12 | "strconv" 13 | "strings" 14 | ) 15 | 16 | type Entity struct { 17 | *BaseApi 18 | } 19 | 20 | func NewEntity(ctx ContextWithApi) *Entity { 21 | return &Entity{&BaseApi{ctx}} 22 | } 23 | 24 | func (self *Entity) Lookup(objectId uint64, storageId uint64) (*data.Entity, error) { 25 | ctx := self.Ctx() 26 | 27 | closer := ctx.LogMark("[Entity.Lookup]") 28 | defer closer() 29 | 30 | tx, err := ctx.Txn() 31 | 32 | row := tx.QueryRow( 33 | "SELECT status FROM entity WHERE object_id = ? AND storage_id = ?", 34 | objectId, 35 | storageId, 36 | ) 37 | 38 | e := data.Entity{objectId, storageId, 0} 39 | err = row.Scan(&e.Status) 40 | if err != nil { 41 | ctx.Debugf( 42 | "Failed to execute query (Entity.Lookup [%d, %d]): %s", 43 | objectId, 44 | storageId, 45 | err, 46 | ) 47 | return nil, err 48 | } 49 | 50 | ctx.Debugf("Successfully loaded entity for object %d storage %d", objectId, storageId) 51 | return &e, nil 52 | } 53 | 54 | func (self *Entity) LookupFromRows(rows *sql.Rows) ([]*data.Entity, error) { 55 | ctx := self.Ctx() 56 | 57 | closer := ctx.LogMark("[Entity.LookupFromRows]") 58 | defer closer() 59 | 60 | var list []*data.Entity 61 | for rows.Next() { 62 | e := &data.Entity{} 63 | err := rows.Scan(&e.ObjectId, &e.StorageId, &e.Status) 64 | if err != nil { 65 | return nil, err 66 | } 67 | list = append(list, e) 68 | } 69 | 70 | ctx.Debugf("Loaded %d entities", len(list)) 71 | return list, nil 72 | } 73 | 74 | func (self *Entity) LookupForObjectNotInCluster(objectId uint64, clusterId uint64) ([]*data.Entity, error) { 75 | ctx := self.Ctx() 76 | 77 | closer := ctx.LogMark("[Entity.LookupForObjectNotInCluster]") 78 | defer closer() 79 | 80 | ctx.Debugf("Looking for entities for object %d", objectId) 81 | 82 | tx, err := ctx.Txn() 83 | 84 | rows, err := tx.Query(` 85 | SELECT e.object_id, e.storage_id, e.status 86 | FROM entity e JOIN storage s ON e.storage_id = s.id 87 | WHERE object_id = ? AND s.cluster_id != ? 88 | `, 89 | objectId, 90 | clusterId, 91 | ) 92 | if err != nil { 93 | return nil, err 94 | } 95 | 96 | list, err := self.LookupFromRows(rows) 97 | if err != nil { 98 | return nil, err 99 | } 100 | 101 | ctx.Debugf( 102 | "Loaded %d entities for object %d (except cluster %d)", 103 | len(list), 104 | objectId, 105 | clusterId, 106 | ) 107 | return list, nil 108 | } 109 | 110 | func (self *Entity) LookupForObject(objectId uint64) ([]*data.Entity, error) { 111 | ctx := self.Ctx() 112 | 113 | closer := ctx.LogMark("[Entity.LookupForObject]") 114 | defer closer() 115 | 116 | ctx.Debugf("Looking for entities for object %d", objectId) 117 | 118 | tx, err := ctx.Txn() 119 | 120 | rows, err := tx.Query( 121 | `SELECT e.object_id, e.storage_id, e.status FROM entity e WHERE object_id = ?`, 122 | objectId, 123 | ) 124 | if err != nil { 125 | return nil, err 126 | } 127 | 128 | list, err := self.LookupFromRows(rows) 129 | if err != nil { 130 | return nil, err 131 | } 132 | 133 | ctx.Debugf("Loaded %d entities for object %d", len(list), objectId) 134 | return list, nil 135 | } 136 | 137 | func (self *Entity) Create( 138 | objectId uint64, 139 | storageId uint64, 140 | ) error { 141 | ctx := self.Ctx() 142 | 143 | closer := ctx.LogMark("[Entity.Create]") 144 | defer closer() 145 | 146 | tx, err := ctx.Txn() 147 | if err != nil { 148 | return err 149 | } 150 | 151 | _, err = tx.Exec("INSERT INTO entity (object_id, storage_id, status, created_at) VALUES (?, ?, 1, UNIX_TIMESTAMP())", objectId, storageId) 152 | if err != nil { 153 | ctx.Debugf("Failed to execute query: %s", err) 154 | return err 155 | } 156 | 157 | ctx.Debugf("Created entity entry for '%d', '%d'", objectId, storageId) 158 | return nil 159 | } 160 | 161 | func (self *Entity) FetchContent(o *data.Object, s *data.Storage, isRepair bool) (io.ReadCloser, error) { 162 | ctx := self.Ctx() 163 | 164 | closer := ctx.LogMark("[Entity.FetchContent]") 165 | defer closer() 166 | 167 | storageApi := ctx.StorageApi() 168 | if !storageApi.IsReadable(s, isRepair) { 169 | return nil, errors.New( 170 | fmt.Sprintf( 171 | "Storage %d is not readable", 172 | s.Id, 173 | ), 174 | ) 175 | } 176 | 177 | return self.FetchContentNocheck(o, s, isRepair) 178 | } 179 | 180 | func (self *Entity) FetchContentNocheck( 181 | o *data.Object, 182 | s *data.Storage, 183 | isRepair bool, 184 | ) (io.ReadCloser, error) { 185 | 186 | ctx := self.Ctx() 187 | 188 | closer := ctx.LogMark("[Entity.FetchContentNocheck]") 189 | defer closer() 190 | 191 | ctx.Debugf( 192 | "Fetching content from storage %d", 193 | s.Id, 194 | ) 195 | 196 | client := &http.Client{} 197 | 198 | uri := strings.Join([]string{s.Uri, o.InternalName}, "/") 199 | ctx.Debugf( 200 | "Sending GET %s (object = %d, storage = %d)", 201 | uri, 202 | o.Id, 203 | s.Id, 204 | ) 205 | 206 | // XXX Original perl version used to optimize the content fetch 207 | // here by writing the content into the file system in chunks. 208 | // Does go need/have such a mechanism? 209 | resp, err := client.Get(uri) 210 | if err != nil { 211 | return nil, err 212 | } 213 | 214 | var okStr string 215 | if resp.StatusCode == 200 { 216 | okStr = "OK" 217 | } else { 218 | okStr = "FAIL" 219 | } 220 | ctx.Debugf( 221 | " GET %s was %s (%s)", 222 | uri, 223 | okStr, 224 | resp.StatusCode, 225 | ) 226 | 227 | if resp.ContentLength != o.Size { 228 | ctx.Debugf( 229 | "Fetched content size for object %d does not match registered size?! (got %d, expected %d)", 230 | o.Id, 231 | resp.ContentLength, 232 | o.Size, 233 | ) 234 | return nil, errors.New("Content size mismatch") 235 | } 236 | 237 | ctx.Debugf( 238 | "Success fetching %s (object = %d, storage = %d)", 239 | uri, 240 | o.Id, 241 | s.Id, 242 | ) 243 | 244 | return resp.Body, nil 245 | } 246 | 247 | func (self *Entity) FetchContentFromStorageIds(o *data.Object, list []uint64, isRepair bool) (io.ReadCloser, error) { 248 | ctx := self.Ctx() 249 | 250 | closer := ctx.LogMark("[Entity.FetchContentFromStorageIds]") 251 | defer closer() 252 | 253 | storageApi := ctx.StorageApi() 254 | storages, err := storageApi.LookupMulti(list) 255 | if err != nil { 256 | return nil, err 257 | } 258 | 259 | for _, s := range storages { 260 | content, err := self.FetchContentNocheck(o, s, isRepair) 261 | if err == nil { 262 | return content, nil 263 | } 264 | } 265 | 266 | return nil, errors.New("Failed to fetch any content") 267 | } 268 | 269 | func (self *Entity) FetchContentFromAll(o *data.Object, isRepair bool) (io.ReadCloser, error) { 270 | ctx := self.Ctx() 271 | 272 | closer := ctx.LogMark("[Entity.FetchContentFromAll]") 273 | defer closer() 274 | 275 | sql := "SELECT s.id FROM storage s ORDER BY rand()" 276 | tx, err := ctx.Txn() 277 | if err != nil { 278 | return nil, err 279 | } 280 | 281 | rows, err := tx.Query(sql) 282 | 283 | var list []uint64 284 | for rows.Next() { 285 | var sid uint64 286 | err = rows.Scan(&sid) 287 | if err != nil { 288 | return nil, err 289 | } 290 | 291 | list = append(list, sid) 292 | } 293 | 294 | return self.FetchContentFromStorageIds(o, list, isRepair) 295 | } 296 | 297 | func (self *Entity) FetchContentFromAny(o *data.Object, isRepair bool) (io.ReadCloser, error) { 298 | ctx := self.Ctx() 299 | 300 | closer := ctx.LogMark("[Entity.FetchContentFromAny]") 301 | defer closer() 302 | 303 | sql := ` 304 | SELECT s.id 305 | FROM storage s JOIN entity e ON s.id = e.storage_id 306 | WHERE s.mode IN (?, ?) AND e.object_id = ? 307 | ORDER BY rand() 308 | ` 309 | 310 | tx, err := ctx.Txn() 311 | if err != nil { 312 | return nil, err 313 | } 314 | 315 | rows, err := tx.Query(sql, stf.STORAGE_MODE_READ_ONLY, stf.STORAGE_MODE_READ_WRITE, o.Id) 316 | if err != nil { 317 | return nil, err 318 | } 319 | 320 | var list []uint64 321 | for rows.Next() { 322 | var sid uint64 323 | err = rows.Scan(&sid) 324 | if err != nil { 325 | return nil, err 326 | } 327 | 328 | list = append(list, sid) 329 | } 330 | 331 | return self.FetchContentFromStorageIds(o, list, isRepair) 332 | } 333 | 334 | func (self *Entity) Store( 335 | storageObj *data.Storage, 336 | objectObj *data.Object, 337 | input *bytes.Reader, 338 | ) error { 339 | ctx := self.Ctx() 340 | 341 | closer := ctx.LogMark("[Entity.Store]") 342 | defer closer() 343 | 344 | uri := strings.Join([]string{storageObj.Uri, objectObj.InternalName}, "/") 345 | cl := input.Len() 346 | 347 | ctx.Debugf("Going to store %d bytes in %s", cl, uri) 348 | 349 | req, err := http.NewRequest("PUT", uri, input) 350 | if err != nil { 351 | ctx.Debugf("Failed to create request: %s", err) 352 | return err 353 | } 354 | 355 | // XXX Need to check if this vanilla http client is ok 356 | client := &http.Client{} 357 | resp, err := client.Do(req) 358 | if err != nil { 359 | ctx.Debugf("Failed to send PUT request to %s (storage = %d): %s", uri, storageObj.Id, err) 360 | return err 361 | } 362 | 363 | if resp.StatusCode != 201 { 364 | err = errors.New( 365 | fmt.Sprintf( 366 | "Expected response 201 for PUT request, but did not get it: %s", 367 | resp.Status, 368 | ), 369 | ) 370 | ctx.Debugf("Failed to store PUT request to %s (storage = %d): %s", uri, storageObj.Id, err) 371 | return err 372 | } 373 | 374 | ctx.Debugf("Successfully stored object in %s", uri) 375 | 376 | err = self.Create( 377 | objectObj.Id, 378 | storageObj.Id, 379 | ) 380 | 381 | if err != nil { 382 | return err 383 | } 384 | 385 | return nil 386 | } 387 | 388 | // Proceed with caution!!!! THIS WILL DELETE THE ENTIRE ENTITY SET! 389 | func (self *Entity) DeleteOrphansForObjectId(objectId uint64) error { 390 | ctx := self.Ctx() 391 | 392 | closer := ctx.LogMark("[Entity.DeletedOrphasForObjectId]") 393 | defer closer() 394 | 395 | tx, err := ctx.Txn() 396 | if err != nil { 397 | return err 398 | } 399 | 400 | _, err = tx.Exec("DELETE FROM entity WHERE object_id = ?", objectId) 401 | return err 402 | } 403 | 404 | func (self *Entity) RemoveForDeletedObjectId(objectId uint64) error { 405 | ctx := self.Ctx() 406 | 407 | closer := ctx.LogMark("[EntityRemoveForDeletedObjectId]") 408 | defer closer() 409 | 410 | // Find existing entities 411 | entities, err := self.LookupForObject(objectId) 412 | if err != nil { 413 | return err 414 | } 415 | for _, e := range entities { 416 | err = self.RemoveDeleted(e, true) 417 | if err != nil { 418 | return err 419 | } 420 | } 421 | 422 | return nil 423 | } 424 | 425 | func (self *Entity) CheckHealth(o *data.Object, s *data.Storage, isRepair bool) error { 426 | ctx := self.Ctx() 427 | 428 | closer := ctx.LogMark("[Entity.CheckHealth]") 429 | defer closer() 430 | 431 | ctx.Debugf("Checking entity health on object %d storage %d", o.Id, s.Id) 432 | 433 | _, err := self.Lookup(o.Id, s.Id) 434 | if err != nil { 435 | ctx.Debugf( 436 | "Entity on storage %d for object %d is not recorded.", 437 | s.Id, 438 | o.Id, 439 | ) 440 | return errors.New( 441 | fmt.Sprintf( 442 | "Could not find entity in database: %s", 443 | err, 444 | ), 445 | ) 446 | } 447 | 448 | // An entity in TEMPORARILY_DOWN node needs to be treated as alive 449 | if s.Mode == stf.STORAGE_MODE_TEMPORARILY_DOWN { 450 | ctx.Debugf( 451 | "Storage %d is temporarily down. Assuming this is intact.", 452 | s.Id, 453 | ) 454 | return nil 455 | } 456 | 457 | // If the mode is not in a readable state, then we've purposely 458 | // taken it out of the system, and needs to be repaired. Also, 459 | // if this were the case, we DO NOT issue an DELETE on the backend, 460 | // as it most likely will not properly respond. 461 | 462 | storageApi := ctx.StorageApi() 463 | if !storageApi.IsReadable(s, isRepair) { 464 | ctx.Debugf( 465 | "Storage %d is not reable. Adding to invalid list.", 466 | s.Id, 467 | ) 468 | return errors.New("Storage is down") 469 | } 470 | 471 | url := strings.Join([]string{s.Uri, o.InternalName}, "/") 472 | ctx.Debugf( 473 | "Going to check %s (object_id = %d, storage_id = %d)", 474 | url, 475 | o.Id, 476 | s.Id, 477 | ) 478 | 479 | client := &http.Client{} 480 | res, err := client.Get(url) 481 | 482 | var okStr string 483 | var st int 484 | if err != nil { 485 | okStr = "FAIL" 486 | st = 500 487 | } else if res.StatusCode != 200 { 488 | okStr = "FAIL" 489 | st = res.StatusCode 490 | } else { 491 | okStr = "OK" 492 | st = res.StatusCode 493 | } 494 | 495 | ctx.Debugf( 496 | "GET %s was %s (%d)", 497 | url, 498 | okStr, 499 | st, 500 | ) 501 | 502 | if err != nil { 503 | return errors.New("An error occurred while trying to fetch entity") 504 | } 505 | 506 | if res.StatusCode != 200 { 507 | return errors.New( 508 | fmt.Sprintf( 509 | "Failed to fetch entity: %s", 510 | res.Status, 511 | ), 512 | ) 513 | } 514 | 515 | if res.ContentLength != o.Size { 516 | ctx.Debugf( 517 | "Object %d sizes do not match (got %d, expected %d)", 518 | o.Id, 519 | res.ContentLength, 520 | o.Size, 521 | ) 522 | return errors.New("Object size mismatch") 523 | } 524 | 525 | return nil 526 | } 527 | 528 | func (self *Entity) SetStatus(e *data.Entity, st int) error { 529 | ctx := self.Ctx() 530 | 531 | closer := ctx.LogMark("[Entity.SetStatus]") 532 | defer closer() 533 | 534 | tx, err := ctx.Txn() 535 | if err != nil { 536 | return err 537 | } 538 | 539 | _, err = tx.Exec( 540 | "UPDATE entity SET status = ? WHERE object_id = ? AND storage_id = ?", 541 | st, 542 | e.ObjectId, 543 | e.StorageId, 544 | ) 545 | 546 | if err != nil { 547 | ctx.Debugf( 548 | "Failed to set status of entity (object %d storage %d) to %d)", 549 | e.ObjectId, 550 | e.StorageId, 551 | st, 552 | ) 553 | return err 554 | } 555 | 556 | ctx.Debugf( 557 | "Successfully set status of entity (object %d storage %d) to %d)", 558 | e.ObjectId, 559 | e.StorageId, 560 | st, 561 | ) 562 | return nil 563 | } 564 | 565 | func (self *Entity) Delete(objectId uint64, storageId uint64) error { 566 | ctx := self.Ctx() 567 | 568 | closer := ctx.LogMark("[Entity.Delete]") 569 | defer closer() 570 | 571 | tx, err := ctx.Txn() 572 | if err != nil { 573 | return err 574 | } 575 | 576 | _, err = tx.Exec( 577 | "DELETE FROM entity WHERE object_id = ? AND storage_id = ?", 578 | objectId, 579 | storageId, 580 | ) 581 | 582 | if err != nil { 583 | ctx.Debugf( 584 | "Failed to delete logical entity (object %d, storage %d): %s", 585 | objectId, 586 | storageId, 587 | err, 588 | ) 589 | return err 590 | } 591 | 592 | ctx.Debugf( 593 | "Successfully deleted logical entity (object %d, storage %d)", 594 | objectId, 595 | storageId, 596 | ) 597 | 598 | return nil 599 | } 600 | 601 | func (self *Entity) Remove(e *data.Entity, isRepair bool) error { 602 | return self.removeInternal( 603 | e, 604 | isRepair, 605 | false, // useDeletedObject: "no" 606 | ) 607 | } 608 | 609 | func (self *Entity) RemoveDeleted(e *data.Entity, isRepair bool) error { 610 | return self.removeInternal( 611 | e, 612 | isRepair, 613 | true, // useDeletedObject: "yes" 614 | ) 615 | } 616 | 617 | func (self *Entity) removeInternal(e *data.Entity, isRepair bool, useDeletedObject bool) error { 618 | ctx := self.Ctx() 619 | 620 | closer := ctx.LogMark("[Entity.Remove]") 621 | defer closer() 622 | 623 | self.Delete(e.ObjectId, e.StorageId) 624 | 625 | cache := ctx.Cache() 626 | cacheKey := cache.CacheKey( 627 | "storage", 628 | strconv.FormatUint(e.StorageId, 10), 629 | "http_accessible", 630 | ) 631 | var httpAccesibleFlg int64 632 | err := cache.Get(cacheKey, &httpAccesibleFlg) 633 | if err == nil && httpAccesibleFlg == -1 { 634 | ctx.Debugf( 635 | "Storage %d was previously unaccessible, skipping physical delete", 636 | e.StorageId, 637 | ) 638 | return errors.New("Storage is inaccessible (negative cache)") 639 | } 640 | 641 | storageApi := ctx.StorageApi() 642 | s, err := storageApi.Lookup(e.StorageId) 643 | if err != nil { 644 | return err 645 | } 646 | 647 | if !storageApi.IsWritable(s, isRepair) { 648 | ctx.Debugf("Storage %d is not writable (isRepair = %s)", s.Id, isRepair) 649 | return errors.New("Storage is not writable") 650 | } 651 | 652 | var internalName string 653 | if useDeletedObject { 654 | o, err := ctx.DeletedObjectApi().Lookup(e.ObjectId) 655 | if err != nil { 656 | return err 657 | } 658 | internalName = o.InternalName 659 | } else { 660 | o, err := ctx.ObjectApi().Lookup(e.ObjectId) 661 | if err != nil { 662 | return err 663 | } 664 | internalName = o.InternalName 665 | } 666 | 667 | uri := strings.Join([]string{s.Uri, internalName}, "/") 668 | req, err := http.NewRequest("DELETE", uri, nil) 669 | client := &http.Client{} 670 | res, err := client.Do(req) 671 | if err != nil { 672 | // If you got here, the 'error' is usually error in DNS resolution 673 | // or connection refused and such. Remember this incident via a 674 | // negative cache, so that we don't keep on 675 | cache.Set(cacheKey, -1, 300) 676 | return err 677 | } 678 | 679 | switch { 680 | case res.StatusCode == 404: 681 | ctx.Debugf("%s was not found while deleting (ignored)", uri) 682 | case res.StatusCode >= 200 && res.StatusCode < 300: 683 | ctx.Debugf("Successfully deleted %s", uri) 684 | default: 685 | ctx.Debugf("An error occurred while deleting %s: %s", uri, res.Status) 686 | } 687 | 688 | return nil 689 | } 690 | -------------------------------------------------------------------------------- /stftest/all_test.go: -------------------------------------------------------------------------------- 1 | package stftest 2 | 3 | import ( 4 | "fmt" 5 | "github.com/lestrrat/go-tcptest" 6 | "github.com/lestrrat/go-test-mysqld" 7 | "github.com/stf-storage/go-stf-server" 8 | "github.com/stf-storage/go-stf-server/api" 9 | "github.com/stf-storage/go-stf-server/config" 10 | "github.com/stf-storage/go-stf-server/dispatcher" 11 | "io" 12 | "io/ioutil" 13 | "net/http" 14 | "net/http/httptest" 15 | "net/url" 16 | "os" 17 | "os/exec" 18 | "path" 19 | "path/filepath" 20 | "runtime" 21 | "runtime/debug" 22 | "strings" 23 | "sync" 24 | "syscall" 25 | "testing" 26 | "time" 27 | ) 28 | 29 | type TestEnv struct { 30 | Test *testing.T 31 | Ctx *api.Context 32 | Guards []func() 33 | WorkDir string 34 | ConfigFile *os.File 35 | Mysqld *mysqltest.TestMysqld 36 | MysqlConfig *config.DatabaseConfig 37 | MemdPort int 38 | QueueConfig *config.QueueConfig 39 | StorageServers []*stf.StorageServer 40 | } 41 | 42 | type TestDatabase struct { 43 | Config *config.DatabaseConfig 44 | Socket string 45 | DataDir string 46 | PidFile string 47 | TmpDir string 48 | } 49 | 50 | func NewTestEnv(t *testing.T) *TestEnv { 51 | env := &TestEnv{} 52 | env.Test = t 53 | return env 54 | } 55 | 56 | func (self *TestEnv) Setup() { 57 | if home := os.Getenv("STF_HOME"); home != "" { 58 | oldpath := os.Getenv("PATH") 59 | newpath := path.Join(home, "bin") 60 | self.Test.Logf("Adding %s to path", newpath) 61 | os.Setenv("PATH", strings.Join([]string{newpath, oldpath}, ":")) 62 | } 63 | 64 | self.createTemporaryDir() 65 | self.startDatabase() 66 | self.startQueue() 67 | self.startMemcached() 68 | self.createTemporaryConfig() 69 | self.startTemporaryStorageServers() 70 | self.startWorkers() 71 | 72 | os.Setenv("STF_CONFIG", self.ConfigFile.Name()) 73 | config, err := config.BootstrapConfig() 74 | if err != nil { 75 | self.Test.Fatalf("%s", err) 76 | } 77 | 78 | self.Ctx = api.NewContext(config) 79 | } 80 | 81 | func (self *TestEnv) Release() { 82 | for _, f := range self.Guards { 83 | f() 84 | } 85 | } 86 | 87 | func (self *TestEnv) Errorf(format string, args ...interface{}) { 88 | self.Test.Errorf(format, args...) 89 | } 90 | 91 | func (self *TestEnv) FailNow(format string, args ...interface{}) { 92 | self.Test.Errorf(format, args...) 93 | debug.PrintStack() 94 | self.Test.FailNow() 95 | } 96 | 97 | func (self *TestEnv) Logf(format string, args ...interface{}) { 98 | self.Test.Logf(format, args...) 99 | } 100 | 101 | func AssertDir(dir string) { 102 | _, err := os.Stat(dir) 103 | if err == nil { 104 | return // XXX not checking if dir is a directory 105 | } 106 | 107 | if !os.IsNotExist(err) { 108 | panic(fmt.Sprintf("Error while asserting directory %s: %s", dir, err)) 109 | } 110 | 111 | err = os.MkdirAll(dir, 0777) 112 | if err != nil { 113 | panic(fmt.Sprintf("Failed to create directory %s: %s", dir, err)) 114 | } 115 | } 116 | 117 | func (self *TestEnv) AddGuard(cb func()) { 118 | self.Guards = append(self.Guards, cb) 119 | } 120 | 121 | func (self *TestEnv) startMemcached() { 122 | var cmd *exec.Cmd 123 | var server *tcptest.TCPTest 124 | var err error 125 | for i := 0; i < 5; i++ { 126 | server, err = tcptest.Start(func(port int) { 127 | out, err := os.OpenFile("memcached.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) 128 | 129 | cmd = exec.Command("memcached", "-vv", "-p", fmt.Sprintf("%d", port)) 130 | cmd.SysProcAttr = &syscall.SysProcAttr{ 131 | Setpgid: true, 132 | } 133 | stderrpipe, err := cmd.StderrPipe() 134 | if err != nil { 135 | self.FailNow("Failed to open pipe to stderr") 136 | } 137 | stdoutpipe, err := cmd.StdoutPipe() 138 | if err != nil { 139 | self.FailNow("Failed to open pipe to stdout") 140 | } 141 | 142 | go io.Copy(out, stderrpipe) 143 | go io.Copy(out, stdoutpipe) 144 | cmd.Run() 145 | }, time.Minute) 146 | if err == nil { 147 | break 148 | } 149 | self.Logf("Failed to start memcached: %s", err) 150 | } 151 | 152 | if server == nil { 153 | self.FailNow("Failed to start memcached") 154 | } 155 | 156 | self.MemdPort = server.Port() 157 | 158 | self.AddGuard(func() { 159 | if cmd != nil && cmd.Process != nil { 160 | self.Logf("Killing memcached") 161 | cmd.Process.Signal(syscall.SIGTERM) 162 | } 163 | server.Wait() 164 | }) 165 | } 166 | 167 | type backgroundproc struct { 168 | cmdname string 169 | args []string 170 | logfile string 171 | } 172 | 173 | func (self *TestEnv) startBackground(p *backgroundproc) { 174 | cmdname := p.cmdname 175 | args := p.args 176 | logfile := p.logfile 177 | path, err := exec.LookPath(cmdname) 178 | if err != nil { 179 | self.Test.Fatalf("Failed to find %s executable: %s", cmdname, err) 180 | } 181 | 182 | cmd := exec.Command(path, args...) 183 | 184 | stderrpipe, err := cmd.StderrPipe() 185 | if err != nil { 186 | self.FailNow("Failed to open pipe to stderr") 187 | } 188 | stdoutpipe, err := cmd.StdoutPipe() 189 | if err != nil { 190 | self.FailNow("Failed to open pipe to stdout") 191 | } 192 | 193 | self.Logf("Starting command %v", cmd.Args) 194 | err = cmd.Start() 195 | if err != nil { 196 | self.FailNow("Failed to start %s: %s", cmdname, err) 197 | } 198 | killed := false 199 | 200 | if logfile == "" { 201 | go io.Copy(os.Stdout, stdoutpipe) 202 | go io.Copy(os.Stderr, stderrpipe) 203 | } else { 204 | out, err := os.OpenFile(logfile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) 205 | if err != nil { 206 | self.FailNow("Could not open logfile: %s", err) 207 | } 208 | go io.Copy(out, stdoutpipe) 209 | go io.Copy(out, stderrpipe) 210 | } 211 | 212 | go func() { 213 | err := cmd.Wait() 214 | if !killed && err != nil { 215 | self.Logf("Failed to wait for %s: %s", cmdname, err) 216 | } 217 | }() 218 | 219 | self.AddGuard(func() { cmd.Process.Signal(syscall.SIGTERM); killed = true }) 220 | } 221 | 222 | func (self *TestEnv) startDatabase() { 223 | mycnf := mysqltest.NewConfig() 224 | mysqld, err := mysqltest.NewMysqld(mycnf) 225 | if err != nil { 226 | self.FailNow("Failed to start mysqld: %s", err) 227 | } 228 | self.Mysqld = mysqld 229 | 230 | self.MysqlConfig = &config.DatabaseConfig{ 231 | "mysql", 232 | "root", 233 | "", 234 | mysqld.ConnectString(0), 235 | "test", 236 | } 237 | 238 | self.Guards = append(self.Guards, func() { 239 | if mysqld := self.Mysqld; mysqld != nil { 240 | mysqld.Stop() 241 | } 242 | }) 243 | 244 | _, err = stf.ConnectDB(self.MysqlConfig) 245 | if err != nil { 246 | self.FailNow("Failed to connect to database: %s", err) 247 | } 248 | 249 | self.Logf("Database files in %s", mysqld.BaseDir()) 250 | self.createDatabase() 251 | } 252 | 253 | func (self *TestEnv) createDatabase() { 254 | self.Logf("Creating database...") 255 | 256 | // Read from DDL file, each statement (delimited by ";") 257 | // then execute each statement via db.Exec() 258 | db, err := stf.ConnectDB(self.MysqlConfig) 259 | if err != nil { 260 | self.FailNow("Failed to connect to database: %s", err) 261 | } 262 | 263 | file, err := os.Open("../stf.sql") 264 | if err != nil { 265 | self.FailNow("Failed to read DDL: %s", err) 266 | } 267 | 268 | fi, err := file.Stat() 269 | if err != nil { 270 | self.FailNow("Failed to stat file: %s", err) 271 | } 272 | 273 | buf := make([]byte, fi.Size()) 274 | _, err = io.ReadFull(file, buf) 275 | strbuf := string(buf) 276 | 277 | for { 278 | i := strings.Index(strbuf, ";") 279 | if i < 0 { 280 | break 281 | } 282 | stmt := strbuf[0:i] 283 | _, err = db.Exec(stmt) 284 | if err != nil { 285 | self.Logf("Failed to create database!") 286 | self.Logf(" SQL: %s", stmt) 287 | self.FailNow("Failed to create database SQL: %s", err) 288 | } 289 | strbuf = strbuf[i+1 : len(strbuf)-1] 290 | } 291 | } 292 | 293 | func (self *TestEnv) createTemporaryDir() { 294 | tempdir, err := ioutil.TempDir("", "stf-test") 295 | if err != nil { 296 | self.FailNow("Failed to create a temporary directory: %s", err) 297 | } 298 | 299 | self.WorkDir = tempdir 300 | self.Guards = append(self.Guards, func() { 301 | self.Logf("Releasing work dir %s", tempdir) 302 | os.RemoveAll(tempdir) 303 | }) 304 | } 305 | 306 | func (self *TestEnv) createTemporaryConfig() { 307 | tempfile, err := ioutil.TempFile(self.WorkDir, "test.gcfg") 308 | if err != nil { 309 | self.FailNow("Failed to create tempfile: %s", err) 310 | } 311 | 312 | tempfile.WriteString(fmt.Sprintf( 313 | ` 314 | [MainDB] 315 | Username=%s 316 | ConnectString=%s 317 | Dbname=%s 318 | 319 | [Memcached] 320 | Servers = 127.0.0.1:%d 321 | 322 | `, 323 | self.MysqlConfig.Username, 324 | self.MysqlConfig.ConnectString, 325 | self.MysqlConfig.Dbname, 326 | self.MemdPort, 327 | )) 328 | 329 | self.writeQueueConfig(tempfile) 330 | 331 | tempfile.Sync() 332 | 333 | self.Logf("Created config file %s", tempfile.Name()) 334 | 335 | self.ConfigFile = tempfile 336 | self.Guards = append(self.Guards, func() { 337 | self.Logf("Removing config file %s", tempfile.Name()) 338 | os.Remove(tempfile.Name()) 339 | }) 340 | } 341 | 342 | func (self *TestEnv) startTemporaryStorageServer(id int, dir string) *stf.StorageServer { 343 | ss := stf.NewStorageServer("dummy", dir) 344 | dts := httptest.NewServer(ss) 345 | 346 | // Register ourselves in the database 347 | db, err := stf.ConnectDB(self.MysqlConfig) 348 | if err != nil { 349 | self.Test.Fatalf("Failed to connect to database: %s", err) 350 | } 351 | _, err = db.Exec("INSERT INTO storage (id, uri, mode, cluster_id) VALUES (?, ?, 1, 1)", id, dts.URL) 352 | if err != nil { 353 | self.Test.Fatalf("Failed to insert storage into DB: %s", err) 354 | } 355 | 356 | self.Guards = append(self.Guards, func() { 357 | dts.Close() 358 | }) 359 | 360 | return ss 361 | } 362 | 363 | func (self *TestEnv) startTemporaryStorageServers() { 364 | // Register ourselves in the database 365 | db, err := stf.ConnectDB(self.MysqlConfig) 366 | if err != nil { 367 | self.Test.Fatalf("Failed to connect to database: %s", err) 368 | } 369 | _, err = db.Exec("INSERT INTO storage_cluster (id, name, mode) VALUES (1, 1, 1)") 370 | if err != nil { 371 | self.Test.Fatalf("Failed to create storage cluster: %s", err) 372 | } 373 | 374 | max := 3 375 | servers := make([]*stf.StorageServer, max) 376 | for i := 1; i <= max; i++ { 377 | mydir := filepath.Join(self.WorkDir, fmt.Sprintf("storage%03d", i)) 378 | servers[i-1] = self.startTemporaryStorageServer(i, mydir) 379 | } 380 | 381 | self.StorageServers = servers 382 | } 383 | 384 | func (self *TestEnv) startWorkers() { 385 | self.startBackground(&backgroundproc{ 386 | cmdname: "stf-worker", 387 | args: []string{fmt.Sprintf("--config=%s", self.ConfigFile.Name())}, 388 | logfile: "worker.log", 389 | }) 390 | } 391 | 392 | func TestBasic(t *testing.T) { 393 | env := NewTestEnv(t) 394 | defer env.Release() 395 | 396 | env.Setup() 397 | 398 | dts, err := env.startDispatcher() 399 | if err != nil { 400 | t.Fatalf("%s", err) 401 | } 402 | defer dts.Close() 403 | 404 | client := &http.Client{} 405 | 406 | t.Logf("Test server ready at %s", dts.URL) 407 | 408 | uri := dts.MakeURL("test", "test.txt") 409 | res, err := client.Get(uri) 410 | if err != nil { 411 | t.Fatalf("Request to '%s' failed: %s", uri, err) 412 | } 413 | if res.StatusCode != 404 { 414 | t.Errorf("GET on non-existent URL %s: want 404, got %d", uri, res.StatusCode) 415 | } 416 | 417 | stfclient := NewTestSTFClient(t, dts) 418 | 419 | stfclient.ObjectGetExpect("test/test.txt", 404, "Fetch before creating an object should be 404") 420 | env.Test.Logf("Create bucket:") 421 | stfclient.BucketCreate("test") 422 | 423 | env.Test.Logf("Create new object:") 424 | _, filename, _, _ := runtime.Caller(1) 425 | stfclient.FilePut("test/test.txt", filename) 426 | 427 | env.Test.Logf("Fetch after create new object:") 428 | res = stfclient.ObjectGet("test/test.txt") 429 | reproxy_uri := res.Header.Get("X-Reproxy-URL") 430 | if reproxy_uri == "" { 431 | t.Errorf("GET %s: want X-Reproxy-URL, got empty", uri) 432 | } 433 | 434 | time.Sleep(5 * time.Second) 435 | env.checkEntityCountForObject("test/test.txt", 3) 436 | 437 | parsedUri, err := url.Parse(reproxy_uri) 438 | if err != nil { 439 | t.Errorf("Failed to parse reproxy uri: %s", err) 440 | } 441 | internalName := parsedUri.Path 442 | internalName = strings.TrimPrefix(internalName, "/") 443 | 444 | // The object has already been deleted, but make sure that the entity 445 | // in the backend storage has been properly deleted 446 | for _, ss := range env.StorageServers { 447 | localPath := path.Join(ss.Root(), internalName) 448 | _, err := os.Stat(localPath) 449 | if err == nil { 450 | env.Test.Logf("Path %s properly created", localPath) 451 | } else { 452 | env.Test.Errorf("Path %s should have been created: %s", localPath, err) 453 | } 454 | } 455 | 456 | stfclient.ObjectDelete("test/test.txt") 457 | stfclient.ObjectGetExpect("test/test.txt", 404, "Fetch after DELETE should be 404") 458 | 459 | // Give it a few more seconds 460 | time.Sleep(5 * time.Second) 461 | 462 | // The object has already been deleted, but make sure that the entity 463 | // in the backend storage has been properly deleted 464 | for _, ss := range env.StorageServers { 465 | localPath := path.Join(ss.Root(), internalName) 466 | _, err := os.Stat(localPath) 467 | if err == nil { 468 | env.Test.Errorf("Path %s should have been deleted: %s", localPath, err) 469 | } else { 470 | env.Test.Logf("Path %s properly deleted", localPath) 471 | } 472 | } 473 | } 474 | 475 | func (self *TestEnv) checkEntityCountForObject(path string, expected int) { 476 | var bucketName string 477 | var objectName string 478 | i := strings.Index(path, "/") 479 | if i == -1 { 480 | self.Errorf("Failed to parse uri") 481 | return 482 | } else { 483 | bucketName = path[0:i] 484 | objectName = path[i+1 : len(path)] 485 | } 486 | 487 | rollback, err := self.Ctx.TxnBegin() 488 | if err != nil { 489 | self.Errorf("Failed to start transaction") 490 | return 491 | } 492 | defer rollback() 493 | 494 | bucketId, err := self.Ctx.BucketApi().LookupIdByName(bucketName) 495 | if err != nil { 496 | self.FailNow("Failed to find id for bucket %s: %s", bucketName, err) 497 | } 498 | bucket, err := self.Ctx.BucketApi().Lookup(bucketId) 499 | if err != nil { 500 | self.FailNow("Failed to load bucket %d: %s", bucketId, err) 501 | } 502 | 503 | objectId, err := self.Ctx.ObjectApi().LookupIdByBucketAndPath(bucket, objectName) 504 | if err != nil { 505 | self.Test.Fatalf("Failed to find id for object %s/%s: %s", bucketName, objectName, err) 506 | } 507 | 508 | // Find the entities mapped to this object 509 | entities, err := self.Ctx.EntityApi().LookupForObject(objectId) 510 | if err != nil { 511 | self.FailNow("Failed to find entities for object %d: %s", objectId, err) 512 | } 513 | if len(entities) != expected { 514 | self.Errorf("Expected entity count = %d, got = %d", expected, len(entities)) 515 | self.Logf("Note: if count != %d, then replicate worker isn't being fired", expected) 516 | } else { 517 | self.Logf("Entity count = %d, got %d", expected, len(entities)) 518 | } 519 | } 520 | 521 | type TestDispatcherServer struct { 522 | *httptest.Server 523 | } 524 | 525 | func (env *TestEnv) startDispatcher() (*TestDispatcherServer, error) { 526 | config := env.Ctx.Config() 527 | d := dispatcher.New(config) 528 | dts := httptest.NewServer(d) 529 | 530 | return &TestDispatcherServer{dts}, nil 531 | } 532 | 533 | func (t *TestDispatcherServer) MakeURL(args ...string) string { 534 | return fmt.Sprintf("%s/%s", t.URL, strings.Join(args, "/")) 535 | } 536 | 537 | func TestCreateID(t *testing.T) { 538 | env := NewTestEnv(t) 539 | defer env.Release() 540 | 541 | env.Setup() 542 | 543 | dts, err := env.startDispatcher() 544 | if err != nil { 545 | t.Fatalf("%s", err) 546 | } 547 | defer dts.Close() 548 | 549 | t.Logf("Test server ready at %s", dts.URL) 550 | bucketUrl := dts.MakeURL("test_id") 551 | 552 | client := &http.Client{} 553 | req, err := http.NewRequest("PUT", bucketUrl, nil) 554 | if err != nil { 555 | t.Fatalf("Failed to create new request: %s", err) 556 | } 557 | res, err := client.Do(req) 558 | if err != nil { 559 | t.Fatalf("Failed to send request to '%s': %s", bucketUrl, err) 560 | } 561 | 562 | if res.StatusCode != 201 { 563 | t.Fatalf("PUT %s: want 201, got %d", bucketUrl, res.StatusCode) 564 | } 565 | 566 | wg := &sync.WaitGroup{} 567 | ready := make(chan bool) 568 | 569 | _, filename, _, _ := runtime.Caller(1) 570 | 571 | for i := 0; i < 10; i++ { 572 | wg.Add(1) 573 | x := i 574 | go func(filename string, x int) { 575 | defer wg.Done() 576 | file, err := os.Open(filename) 577 | if err != nil { 578 | t.Fatalf("Failed to open %s: %s", filename, err) 579 | } 580 | fi, err := file.Stat() 581 | if err != nil { 582 | t.Errorf("Failed to stat %s: %s", filename, err) 583 | } 584 | uri := fmt.Sprintf("%s/test%03d.txt", bucketUrl, x) 585 | req, _ := http.NewRequest("PUT", uri, file) 586 | req.ContentLength = fi.Size() 587 | client := &http.Client{} 588 | 589 | <-ready 590 | 591 | res, err := client.Do(req) 592 | if err != nil { 593 | t.Logf("Failed to send request: %s", err) 594 | } else if res.StatusCode != 201 { 595 | t.Errorf("Failed to create %s: %s", uri, res.Status) 596 | } 597 | }(filename, x) 598 | } 599 | 600 | for i := 0; i < 10; i++ { 601 | ready <- true 602 | } 603 | 604 | wg.Wait() 605 | } 606 | 607 | /* 608 | func TestMove(t *testing.T) { 609 | env := NewTestEnv(t) 610 | defer env.Release() 611 | 612 | env.Setup() 613 | 614 | dts, err := env.startDispatcher() 615 | if err != nil { 616 | t.Fatalf("%s", err) 617 | } 618 | defer dts.Close() 619 | } 620 | */ 621 | -------------------------------------------------------------------------------- /api/object.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "bytes" 5 | "database/sql" 6 | "errors" 7 | "fmt" 8 | randbo "github.com/dustin/randbo" 9 | "github.com/stf-storage/go-stf-server" 10 | "github.com/stf-storage/go-stf-server/data" 11 | "io/ioutil" 12 | "log" 13 | "net/http" 14 | "strconv" 15 | "strings" 16 | ) 17 | 18 | type Object struct { 19 | *BaseApi 20 | } 21 | 22 | var ErrContentNotModified error = errors.New("Request Content Not Modified") 23 | 24 | func NewObject(ctx ContextWithApi) *Object { 25 | return &Object{&BaseApi{ctx}} 26 | } 27 | 28 | func (self *Object) LookupIdByBucketAndPath(bucketObj *data.Bucket, path string) (uint64, error) { 29 | ctx := self.Ctx() 30 | 31 | closer := ctx.LogMark("[Object.LookupIdByBucketAndPath]") 32 | defer closer() 33 | 34 | tx, err := ctx.Txn() 35 | if err != nil { 36 | return 0, err 37 | } 38 | 39 | ctx.Debugf("Looking for object where bucket_id = %d, name = %d", bucketObj.Id, path) 40 | 41 | row := tx.QueryRow("SELECT id FROM object WHERE bucket_id = ? AND name = ?", bucketObj.Id, path) 42 | 43 | var id uint64 44 | err = row.Scan(&id) 45 | switch { 46 | case err == sql.ErrNoRows: 47 | ctx.Debugf("Could not find any object for %s/%s", bucketObj.Name, path) 48 | return 0, sql.ErrNoRows 49 | case err != nil: 50 | return 0, errors.New(fmt.Sprintf("Failed to execute query (LookupByBucketAndPath): %s", err)) 51 | } 52 | 53 | ctx.Debugf("Loaded Object ID '%d' from %s/%s", id, bucketObj.Name, path) 54 | 55 | return id, nil 56 | } 57 | 58 | func (self *Object) LookupFromDB(id uint64) (*data.Object, error) { 59 | ctx := self.Ctx() 60 | 61 | tx, err := ctx.Txn() 62 | if err != nil { 63 | return nil, err 64 | } 65 | 66 | row := tx.QueryRow("SELECT id, bucket_id, name, internal_name, size, status, created_at, updated_at FROM object WHERE id = ?", id) 67 | 68 | var o data.Object 69 | err = row.Scan( 70 | &o.Id, 71 | &o.BucketId, 72 | &o.Name, 73 | &o.InternalName, 74 | &o.Size, 75 | &o.Status, 76 | &o.CreatedAt, 77 | &o.UpdatedAt, 78 | ) 79 | 80 | if err != nil { 81 | ctx.Debugf("Failed to execute query (LookupFromDB): %s", err) 82 | return nil, err 83 | } 84 | 85 | return &o, nil 86 | } 87 | 88 | func (self *Object) Lookup(id uint64) (*data.Object, error) { 89 | ctx := self.Ctx() 90 | 91 | closer := ctx.LogMark("[Object.Lookup]") 92 | defer closer() 93 | 94 | var o data.Object 95 | cache := ctx.Cache() 96 | cacheKey := cache.CacheKey("object", strconv.FormatUint(id, 10)) 97 | err := cache.Get(cacheKey, &o) 98 | if err == nil { 99 | ctx.Debugf("Cache HIT for object %d, returning object from cache", id) 100 | return &o, nil 101 | } 102 | 103 | optr, err := self.LookupFromDB(id) 104 | if err != nil { 105 | return nil, err 106 | } 107 | 108 | ctx.Debugf("Successfully loaded object %d from database", id) 109 | cache.Set(cacheKey, *optr, 3600) 110 | return optr, nil 111 | } 112 | 113 | func (self *Object) GetStoragesFor(objectObj *data.Object) ([]*data.Storage, error) { 114 | ctx := self.Ctx() 115 | closer := ctx.LogMark("[Object.GetStoragesFor]") 116 | defer closer() 117 | 118 | /* We cache 119 | * "storages_for.$object_id => [ storage_id, storage_id, ... ] 120 | */ 121 | cache := ctx.Cache() 122 | cacheKey := cache.CacheKey( 123 | "storages_for", 124 | strconv.FormatUint(objectObj.Id, 10), 125 | ) 126 | var storageIds []uint64 127 | var list []*data.Storage 128 | 129 | err := cache.Get(cacheKey, &storageIds) 130 | 131 | if err == nil { 132 | // Cache HIT. we need to check for the validity of the storages 133 | list, err = ctx.StorageApi().LookupMulti(storageIds) 134 | if err != nil { 135 | list = []*data.Storage{} 136 | } else { 137 | // Check each 138 | } 139 | } 140 | 141 | if len(list) == 0 { 142 | ctx.Debugf("Cache MISS for storages for object %d, loading from database", objectObj.Id) 143 | 144 | var storageIds []int64 145 | sql := ` 146 | SELECT s.id, s.uri, s.mode 147 | FROM object o JOIN entity e ON o.id = e.object_id 148 | JOIN storage s ON s.id = e.storage_id 149 | WHERE o.id = ? AND 150 | o.status = 1 AND 151 | e.status = 1 AND 152 | s.mode IN (?, ?) 153 | ` 154 | 155 | tx, err := ctx.Txn() 156 | if err != nil { 157 | return nil, err 158 | } 159 | 160 | rows, err := tx.Query(sql, objectObj.Id, stf.STORAGE_MODE_READ_ONLY, stf.STORAGE_MODE_READ_WRITE) 161 | if err != nil { 162 | return nil, err 163 | } 164 | 165 | for rows.Next() { 166 | s := data.Storage{} 167 | err = rows.Scan( 168 | &s.Id, 169 | &s.Uri, 170 | &s.Mode, 171 | ) 172 | storageIds = append(storageIds, int64(s.Id)) 173 | list = append(list, &s) 174 | } 175 | if len(storageIds) > 0 { 176 | cache.Set(cacheKey, storageIds, 600) 177 | } 178 | } 179 | ctx.Debugf("Loaded %d storages", len(list)) 180 | return list, nil 181 | } 182 | 183 | func (self *Object) EnqueueRepair( 184 | bucketObj *data.Bucket, 185 | objectObj *data.Object, 186 | ) { 187 | ctx := self.Ctx() 188 | 189 | go func() { 190 | // This operation does not have to complete succesfully, so 191 | // so we use defer() here to eat any panic conditions that we 192 | // may encounter 193 | if err := recover(); err != nil { 194 | ctx.Debugf( 195 | "Error while sending object %d (%s/%s) to repair (ignored): %s", 196 | objectObj.Id, 197 | bucketObj.Name, 198 | objectObj.Name, 199 | ) 200 | } 201 | }() 202 | 203 | ctx.Debugf( 204 | "Object %d (%s/%s) being sent to repair (harmless)", 205 | objectObj.Id, 206 | bucketObj.Name, 207 | objectObj.Name, 208 | ) 209 | 210 | queueApi := ctx.QueueApi() 211 | queueApi.Enqueue("queue_repair_object", strconv.FormatUint(objectObj.Id, 10)) 212 | 213 | // Putting this in memcached via Add() allows us from not sending 214 | // this object to repair repeatedly 215 | cache := ctx.Cache() 216 | cacheKey := cache.CacheKey( 217 | "repair_from_dispatcher", 218 | strconv.FormatUint(objectObj.Id, 10), 219 | ) 220 | cache.Add(cacheKey, 1, 3600) 221 | } 222 | 223 | func (self *Object) GetAnyValidEntityUrl( 224 | bucketObj *data.Bucket, 225 | objectObj *data.Object, 226 | doHealthCheck bool, // true if we want to run repair 227 | ifModifiedSince string, 228 | ) (string, error) { 229 | ctx := self.Ctx() 230 | 231 | closer := ctx.LogMark("[Object.GetAnyValidEntityUrl]") 232 | defer closer() 233 | 234 | // XXX We have to do this before we check the entities, because in 235 | // real-life applications many of the requests come with an IMS header 236 | // which short-circuits from this method, and never allows us to 237 | // reach in this enqueuing block 238 | if doHealthCheck { 239 | defer func() { 240 | go self.EnqueueRepair(bucketObj, objectObj) 241 | }() 242 | } 243 | 244 | storages, err := self.GetStoragesFor(objectObj) 245 | if err != nil { 246 | return "", err 247 | } 248 | 249 | client := &http.Client{} 250 | for _, storage := range storages { 251 | ctx.Debugf("Attempting to make a request to %s (id = %d)", storage.Uri, storage.Id) 252 | url := fmt.Sprintf("%s/%s", storage.Uri, objectObj.InternalName) 253 | request, err := http.NewRequest("HEAD", url, nil) 254 | // if this is errornous, we're in deep shit 255 | if err != nil { 256 | return "", err 257 | } 258 | 259 | if ifModifiedSince != "" { 260 | request.Header.Set("If-Modified-Since", ifModifiedSince) 261 | } 262 | resp, err := client.Do(request) 263 | if err != nil { 264 | ctx.Debugf("Failed to send request to %s: %s", url, err) 265 | continue 266 | } 267 | 268 | switch resp.StatusCode { 269 | case 200: 270 | ctx.Debugf("Request successs, returning URL '%s'", url) 271 | return url, nil 272 | case 304: 273 | // This is wierd, but this is how we're going to handle it 274 | ctx.Debugf("Request target is not modified, returning 304") 275 | return "", ErrContentNotModified 276 | default: 277 | ctx.Debugf("Request for %s failed: %s", url, resp.Status) 278 | // If we failed to fetch the object, send it to repair 279 | if !doHealthCheck { 280 | doHealthCheck = true 281 | defer func() { 282 | go self.EnqueueRepair(bucketObj, objectObj) 283 | }() 284 | } 285 | // nothing more to do, try our next candidate 286 | } 287 | } 288 | 289 | // if we fell through here, we're done for 290 | err = errors.New("Could not find a valid entity in any of the storages") 291 | ctx.Debugf("%s", err) 292 | return "", err 293 | } 294 | 295 | func (self *Object) MarkForDelete(id uint64) error { 296 | ctx := self.Ctx() 297 | 298 | closer := ctx.LogMark("[Object.MarkForDelete]") 299 | defer closer() 300 | 301 | tx, err := ctx.Txn() 302 | if err != nil { 303 | return err 304 | } 305 | 306 | res, err := tx.Exec("REPLACE INTO deleted_object SELECT * FROM object WHERE id = ?", id) 307 | 308 | if err != nil { 309 | ctx.Debugf("Failed to execute query (REPLACE into deleted_object): %s", err) 310 | return err 311 | } 312 | 313 | if count, _ := res.RowsAffected(); count <= 0 { 314 | // Grr, we failed to insert to deleted_object table 315 | err = errors.New("Failed to insert object into deleted object queue") 316 | ctx.Debugf("%s", err) 317 | return err 318 | } 319 | 320 | res, err = tx.Exec("DELETE FROM object WHERE id = ?", id) 321 | if err != nil { 322 | ctx.Debugf("Failed to execute query (DELETE from object): %s", err) 323 | return err 324 | } 325 | 326 | if count, _ := res.RowsAffected(); count <= 0 { 327 | err = errors.New("Failed to delete object") 328 | ctx.Debugf("%s", err) 329 | return err 330 | } 331 | 332 | cache := ctx.Cache() 333 | cacheKey := cache.CacheKey("object", strconv.FormatUint(id, 10)) 334 | err = cache.Delete(cacheKey) 335 | 336 | if err != nil && err.Error() != "memcache: cache miss" { 337 | ctx.Debugf("Failed to delete cache '%s': '%s'", cacheKey, err) 338 | return err 339 | } 340 | 341 | ctx.Debugf("Deleted object (%d), and placed it into deleted_object table", id) 342 | return nil 343 | } 344 | 345 | func (self *Object) Delete(id uint64) error { 346 | ctx := self.Ctx() 347 | tx, err := ctx.Txn() 348 | if err != nil { 349 | return err 350 | } 351 | 352 | _, err = tx.Exec("DELETE FROM object WHERE id = ?", id) 353 | if err != nil { 354 | return err 355 | } 356 | 357 | cache := ctx.Cache() 358 | cacheKey := cache.CacheKey("object", strconv.FormatUint(id, 10)) 359 | err = cache.Delete(cacheKey) 360 | if err != nil { 361 | return err 362 | } 363 | 364 | return nil 365 | } 366 | 367 | func (self *Object) Create( 368 | objectId uint64, 369 | bucketId uint64, 370 | objectName string, 371 | internalName string, 372 | size int64, 373 | ) error { 374 | ctx := self.Ctx() 375 | closer := ctx.LogMark("[Object.Create]") 376 | defer closer() 377 | tx, err := ctx.Txn() 378 | if err != nil { 379 | return err 380 | } 381 | 382 | _, err = tx.Exec("INSERT INTO object (id, bucket_id, name, internal_name, size, created_at) VALUES (?, ?, ?, ?, ?, UNIX_TIMESTAMP())", objectId, bucketId, objectName, internalName, size) 383 | 384 | if err != nil { 385 | ctx.Debugf("Failed to execute query: %s", err) 386 | return err 387 | } 388 | 389 | ctx.Debugf("Created object entry for '%d' (internal_name = '%s')", objectId, internalName) 390 | return nil 391 | } 392 | 393 | func (self *Object) AttemptCreate( 394 | objectId uint64, 395 | bucketId uint64, 396 | objectName string, 397 | internalName string, 398 | size int64, 399 | ) (err error) { 400 | defer func() { 401 | if v := recover(); v != nil { 402 | err = v.(error) // this becomes the return value. woot 403 | } 404 | }() 405 | 406 | err = self.Create( 407 | objectId, 408 | bucketId, 409 | objectName, 410 | internalName, 411 | size, 412 | ) 413 | 414 | return err 415 | } 416 | 417 | func createInternalName(suffix string) string { 418 | buf := make([]byte, 30) 419 | n, err := randbo.New().Read(buf) 420 | 421 | if err != nil { 422 | log.Fatalf("createInternalName failed: %s", err) 423 | } 424 | 425 | if n != len(buf) { 426 | log.Fatalf("createInternalName failed: (n = %d) != (len = %d)", n, len(buf)) 427 | } 428 | 429 | hex := fmt.Sprintf("%x.%s", buf, suffix) 430 | 431 | return strings.Join( 432 | []string{hex[0:1], hex[1:2], hex[2:3], hex[3:4], hex}, 433 | "/", 434 | ) 435 | } 436 | 437 | func (self *Object) Store( 438 | objectId uint64, 439 | bucketObj *data.Bucket, 440 | objectName string, 441 | size int64, 442 | input *bytes.Reader, 443 | suffix string, 444 | isRepair bool, 445 | force bool, 446 | ) error { 447 | ctx := self.Ctx() 448 | 449 | closer := ctx.LogMark("[Object.Store]") 450 | defer closer() 451 | 452 | done := false 453 | for i := 0; i < 10; i++ { 454 | internalName := createInternalName(suffix) 455 | err := self.AttemptCreate( 456 | objectId, 457 | bucketObj.Id, 458 | objectName, 459 | internalName, 460 | size, 461 | ) 462 | if err == nil { 463 | done = true 464 | break 465 | } else { 466 | ctx.Debugf("Failed to create object in DB: %s", err) 467 | } 468 | } 469 | 470 | if !done { // whoa, we fell through here w/o creating the object! 471 | err := errors.New("Failed to create object entry") 472 | ctx.Debugf("%s", err) 473 | return err 474 | } 475 | 476 | // After this point if something wicked happens and we bail out, 477 | // we don't want to keep this object laying around in a half-baked 478 | // state. So make sure to get rid of it 479 | done = false 480 | defer func() { 481 | if !done { 482 | ctx.Debugf("Something went wrong, deleting object to make sure") 483 | self.Delete(objectId) 484 | } 485 | }() 486 | 487 | objectObj, err := self.Lookup(objectId) 488 | if err != nil { 489 | ctx.Debugf("Failed to lookup up object from DB: %s", err) 490 | return err 491 | } 492 | 493 | // Load all possible clusters ordered by a consistent hash 494 | clusterApi := ctx.StorageClusterApi() 495 | clusters, err := clusterApi.LoadCandidatesFor(objectId) 496 | if err != nil { 497 | return err 498 | } 499 | 500 | if len(clusters) <= 0 { 501 | err := errors.New(fmt.Sprintf("No write candidate cluster found for object %d!", objectId)) 502 | ctx.Debugf("%s", err) 503 | return err 504 | } 505 | 506 | for _, clusterObj := range clusters { 507 | err := clusterApi.Store( 508 | clusterObj.Id, 509 | objectObj, 510 | input, 511 | 2, 512 | isRepair, 513 | force, 514 | ) 515 | if err == nil { // Success! 516 | ctx.Debugf("Successfully stored objects in cluster %d", clusterObj.Id) 517 | clusterApi.RegisterForObject( 518 | clusterObj.Id, 519 | objectId, 520 | ) 521 | // Set this to true so that the defered cleanup 522 | // doesn't get triggered 523 | done = true 524 | return nil 525 | } 526 | ctx.Debugf("Failed to store in cluster %d: %s", clusterObj.Id, err) 527 | } 528 | 529 | err = errors.New("Could not store in ANY clusters!") 530 | ctx.Debugf("%s", err) 531 | 532 | return err 533 | } 534 | 535 | var ErrNothingToRepair = errors.New("Nothing to repair") 536 | 537 | func (self *Object) Repair(objectId uint64) error { 538 | ctx := self.Ctx() 539 | 540 | closer := ctx.LogMark("[Object.Repair]") 541 | defer closer() 542 | 543 | ctx.Debugf( 544 | "Repairing object %d", 545 | objectId, 546 | ) 547 | 548 | entityApi := ctx.EntityApi() 549 | o, err := self.Lookup(objectId) 550 | if err != nil { 551 | ctx.Debugf("No matching object %d", objectId) 552 | 553 | entities, err := entityApi.LookupForObject(objectId) 554 | if err != nil { 555 | return ErrNothingToRepair 556 | } 557 | 558 | if stf.DebugEnabled() { 559 | ctx.Debugf("Removing orphaned entities in storages:") 560 | for _, e := range entities { 561 | ctx.Debugf(" + %d", e.StorageId) 562 | } 563 | } 564 | entityApi.DeleteOrphansForObjectId(objectId) 565 | return ErrNothingToRepair 566 | } 567 | 568 | ctx.Debugf( 569 | "Fetching master content for object %d from any of the known entities", 570 | o.Id, 571 | ) 572 | masterContent, err := entityApi.FetchContentFromAny(o, true) 573 | if err != nil { 574 | // One more shot. See if we can recover the content from ANY 575 | // of the available storages 576 | masterContent, err = entityApi.FetchContentFromAll(o, true) 577 | if err != nil { 578 | return errors.New( 579 | fmt.Sprintf( 580 | "PANIC: No content for %d could be fetched!! Cannot proceed with repair.", 581 | objectId, 582 | ), 583 | ) 584 | } 585 | } 586 | 587 | ctx.Debugf("Successfully fetched master content") 588 | 589 | clusterApi := ctx.StorageClusterApi() 590 | clusters, err := clusterApi.LoadCandidatesFor(objectId) 591 | if err != nil || len(clusters) < 1 { 592 | return errors.New( 593 | fmt.Sprintf( 594 | "Could not find any storage candidate for object %d", 595 | objectId, 596 | ), 597 | ) 598 | } 599 | 600 | // Keep this empty until successful completion of the next 601 | // `if err == nil {...} else {...} block. It serves as a marker that 602 | // the object is stored in this cluster 603 | var designatedCluster *data.StorageCluster 604 | 605 | // The object SHOULD be stored in the first instance 606 | ctx.Debugf( 607 | "Checking entity health on cluster %d", 608 | clusters[0].Id, 609 | ) 610 | err = clusterApi.CheckEntityHealth(clusters[0], o, true) 611 | needsRepair := err != nil 612 | 613 | if !needsRepair { 614 | // No need to repair. Just make sure object -> cluster mapping 615 | // is intact 616 | ctx.Debugf( 617 | "Object %d is correctly stored in cluster %d. Object does not need repair", 618 | objectId, 619 | clusters[0].Id, 620 | ) 621 | 622 | designatedCluster = clusters[0] 623 | currentCluster, _ := clusterApi.LookupForObject(objectId) 624 | if currentCluster == nil || designatedCluster.Id != currentCluster.Id { 625 | // Ignore errors. No harm done 626 | clusterApi.RegisterForObject(designatedCluster.Id, objectId) 627 | } 628 | } else { 629 | // Need to repair 630 | ctx.Debugf("Object %d needs repair", objectId) 631 | 632 | // If it got here, either the object was not properly in designatedCluster 633 | // (i.e., some/all of the storages in the cluster did not have this 634 | // object stored) or it was in a different cluster 635 | contentBuf, err := ioutil.ReadAll(masterContent) 636 | if err != nil { 637 | return errors.New( 638 | fmt.Sprintf( 639 | "Failed to read from content handle: %s", 640 | err, 641 | ), 642 | ) 643 | } 644 | contentReader := bytes.NewReader(contentBuf) 645 | for _, cluster := range clusters { 646 | ctx.Debugf( 647 | "Attempting to store object %d on cluster %d", 648 | o.Id, 649 | cluster.Id, 650 | ) 651 | err = clusterApi.Store( 652 | cluster.Id, 653 | o, 654 | contentReader, 655 | 0, // minimumTosTore 656 | true, // isRepair 657 | false, // force 658 | ) 659 | if err == nil { 660 | designatedCluster = cluster 661 | ctx.Debugf( 662 | "Successfully stored object %d on cluster %d", 663 | o.Id, 664 | cluster.Id, 665 | ) 666 | break 667 | } 668 | ctx.Debugf( 669 | "Failed to store object %d on cluster %d: %s", 670 | o.Id, 671 | cluster.Id, 672 | err, 673 | ) 674 | } 675 | 676 | if designatedCluster == nil { 677 | return errors.New( 678 | fmt.Sprintf( 679 | "PANIC: Failed to repair object %d to any cluster!", 680 | objectId, 681 | ), 682 | ) 683 | } 684 | } 685 | 686 | // Object is now properly stored in designatedCluster. Find which storages 687 | // map to this cluster, and remove any other entities, if available. 688 | // This may happen if we added new clusters and rebalancing ocurred 689 | entities, err := entityApi.LookupForObjectNotInCluster(objectId, designatedCluster.Id) 690 | 691 | // Cache needs to be invalidated regardless, but we should be careful 692 | // about the timing 693 | cache := ctx.Cache() 694 | cacheKey := cache.CacheKey("storages_for", strconv.FormatUint(objectId, 10)) 695 | 696 | cacheInvalidator := func() { 697 | ctx.Debugf("Invalidating cache %s", cacheKey) 698 | cache.Delete(cacheKey) 699 | } 700 | 701 | if needsRepair { 702 | defer cacheInvalidator() 703 | } 704 | 705 | // Note: this err is from entityApi.LookupForObject 706 | if err != nil { 707 | ctx.Debugf("Failed to fetch entities for object %d: %s", objectId, err) 708 | } else if entities != nil && len(entities) > 0 { 709 | ctx.Debugf("Extra entities found: dropping status flag, then proceeding to remove %d entities", len(entities)) 710 | for _, e := range entities { 711 | entityApi.SetStatus(e, 0) 712 | } 713 | 714 | // Make sure to invalidate the cache here, because we don't want 715 | // the dispatcher to pick the entities with status = 0 716 | cacheInvalidator() 717 | 718 | for _, e := range entities { 719 | entityApi.Remove(e, true) 720 | } 721 | } 722 | 723 | ctx.Debugf("Done repair for object %d", objectId) 724 | return nil 725 | } 726 | --------------------------------------------------------------------------------