├── LICENSE ├── README.md ├── go.mod ├── go.sum ├── lib ├── es │ ├── es5 │ │ └── es.go │ └── es7 │ │ └── es.go ├── objectStream │ ├── get.go │ ├── put.go │ └── temp.go ├── rabbitmq │ ├── rabbitmq.go │ └── rabbitmq_test.go ├── rs │ ├── common.go │ ├── decoder.go │ ├── encoder.go │ ├── get.go │ ├── put.go │ ├── resumable_get.go │ └── resumable_put.go ├── types │ └── types.go └── utils │ └── utils.go ├── part1 ├── go.mod ├── objects │ ├── get.go │ ├── handler.go │ └── put.go └── server.go ├── part2 ├── apiServer │ ├── apiServer.go │ ├── heartbeat │ │ ├── choose.go │ │ └── heartbeat.go │ ├── locate │ │ ├── handler.go │ │ └── locate.go │ └── objects │ │ ├── get.go │ │ ├── get_stream.go │ │ ├── handler.go │ │ ├── put.go │ │ ├── put_stream.go │ │ └── store.go ├── dataServer │ ├── dataServer.go │ ├── heartbeat │ │ └── heartbeat.go │ ├── locate │ │ └── locate.go │ └── objects │ │ ├── get.go │ │ ├── handler.go │ │ └── put.go └── test │ ├── 流程.txt │ └── 错误修正.txt ├── part3 ├── apiServer │ ├── apiServer.go │ ├── heartbeat │ │ ├── choose.go │ │ └── heartbeat.go │ ├── locate │ │ ├── handler.go │ │ └── locate.go │ ├── objects │ │ ├── del.go │ │ ├── get.go │ │ ├── get_stream.go │ │ ├── handler.go │ │ ├── put.go │ │ ├── put_stream.go │ │ └── store.go │ └── version │ │ └── handler.go ├── dataServer │ ├── dataServer.go │ ├── heartbeat │ │ └── heartbeat.go │ ├── locate │ │ └── locate.go │ └── objects │ │ ├── get.go │ │ ├── handler.go │ │ └── put.go └── test │ ├── 流程.txt │ └── 错误修正.txt ├── part4 ├── apiServer │ ├── apiServer.go │ ├── heartbeat │ │ ├── choose.go │ │ └── heartbeat.go │ ├── locate │ │ ├── handler.go │ │ └── locate.go │ ├── objects │ │ ├── del.go │ │ ├── get.go │ │ ├── get_stream.go │ │ ├── handler.go │ │ ├── put.go │ │ ├── put_stream.go │ │ └── store.go │ └── version │ │ └── handler.go ├── dataServer │ ├── dataServer.go │ ├── heartbeat │ │ └── heartbeat.go │ ├── locate │ │ └── locate.go │ ├── objects │ │ ├── get.go │ │ ├── get_object.go │ │ ├── handler.go │ │ └── send.go │ └── temp │ │ ├── commit.go │ │ ├── del.go │ │ ├── handler.go │ │ ├── patch.go │ │ ├── post.go │ │ └── put.go └── test │ ├── 命令汇总.txt │ └── 流程.txt ├── part5 ├── apiServer │ ├── apiServer.go │ ├── heartbeat │ │ ├── choose.go │ │ └── heartbeat.go │ ├── locate │ │ ├── handler.go │ │ └── locate.go │ ├── objects │ │ ├── del.go │ │ ├── get.go │ │ ├── get_stream.go │ │ ├── handler.go │ │ ├── put.go │ │ ├── put_stream.go │ │ └── store.go │ └── versions │ │ └── handler.go ├── dataServer │ ├── dataServer.go │ ├── heartbeat │ │ └── heartbeat.go │ ├── locate │ │ └── locate.go │ ├── objects │ │ ├── get.go │ │ ├── get_object.go │ │ ├── handler.go │ │ └── send.go │ └── temp │ │ ├── commit.go │ │ ├── del.go │ │ ├── handler.go │ │ ├── patch.go │ │ ├── post.go │ │ └── put.go └── test │ └── 流程.txt ├── part6 ├── apiServer │ ├── apiServer.go │ ├── heartbeat │ │ ├── choose.go │ │ └── heartbeat.go │ ├── locate │ │ ├── handler.go │ │ └── locate.go │ ├── objects │ │ ├── del.go │ │ ├── get.go │ │ ├── get_stream.go │ │ ├── handler.go │ │ ├── post.go │ │ ├── put.go │ │ ├── put_stream.go │ │ └── store.go │ ├── temp │ │ ├── handler.go │ │ ├── head.go │ │ └── put.go │ └── versions │ │ └── handler.go ├── dataServer │ ├── dataServer.go │ ├── heartbeat │ │ └── heartbeat.go │ ├── locate │ │ └── locate.go │ ├── objects │ │ ├── get.go │ │ ├── get_object.go │ │ ├── handler.go │ │ └── send.go │ └── temp │ │ ├── commit.go │ │ ├── del.go │ │ ├── get.go │ │ ├── handler.go │ │ ├── head.go │ │ ├── patch.go │ │ ├── post.go │ │ └── put.go └── test │ ├── 流程.txt │ └── 错误修正.txt ├── part7 ├── apiServer │ ├── apiServer.go │ ├── heartbeat │ │ ├── choose.go │ │ └── heartbeat.go │ ├── locate │ │ ├── handler.go │ │ └── locate.go │ ├── objects │ │ ├── del.go │ │ ├── get.go │ │ ├── get_stream.go │ │ ├── handler.go │ │ ├── post.go │ │ ├── put.go │ │ ├── put_stream.go │ │ └── store.go │ ├── temp │ │ ├── handler.go │ │ ├── head.go │ │ └── put.go │ └── versions │ │ └── handler.go ├── dataServer │ ├── dataServer.go │ ├── heartbeat │ │ └── heartbeat.go │ ├── locate │ │ └── locate.go │ ├── objects │ │ ├── get.go │ │ ├── get_object.go │ │ ├── handler.go │ │ └── send.go │ └── temp │ │ ├── commit.go │ │ ├── del.go │ │ ├── get.go │ │ ├── handler.go │ │ ├── head.go │ │ ├── patch.go │ │ ├── post.go │ │ └── put.go └── test │ └── 流程.txt └── part8 ├── apiServer ├── apiServer.go ├── heartbeat │ ├── choose.go │ └── heartbeat.go ├── locate │ ├── handler.go │ └── locate.go ├── objects │ ├── del.go │ ├── get.go │ ├── get_stream.go │ ├── handler.go │ ├── post.go │ ├── put.go │ ├── put_stream.go │ └── store.go ├── temp │ ├── handler.go │ ├── head.go │ └── put.go └── versions │ └── handler.go ├── dataServer ├── dataServer.go ├── heartbeat │ └── heartbeat.go ├── locate │ └── locate.go ├── objects │ ├── del.go │ ├── get.go │ ├── get_object.go │ ├── handler.go │ └── send.go └── temp │ ├── commit.go │ ├── del.go │ ├── get.go │ ├── handler.go │ ├── head.go │ ├── patch.go │ ├── post.go │ └── put.go ├── deleteOldMetadata └── deleteOldMetaData.go ├── deleteOrphanObject └── deleteOrphanObject.go ├── objectScanner └── objectScanner.go └── test └── 流程.txt /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 白瑾 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # object-storage-service-golang 2 | 使用Go语言搭建一个简单的OSS对象存储服务器，依据书ISBN978-7-115-48055-2编写，修复书上一些bug，并提供构建的完整视频过程 3 | 4 | 构建过程视频：https://b23.tv/recEf3 5 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module storage 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/klauspost/reedsolomon v1.9.13 7 | github.com/streadway/amqp v1.0.0 8 | ) 9 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/klauspost/cpuid/v2 v2.0.6 h1:dQ5ueTiftKxp0gyjKSx5+8BtPWkyQbd95m8Gys/RarI= 2 | github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= 3 | github.com/klauspost/reedsolomon v1.9.13 h1:Xr0COKf7F0ACTXUNnz2ZFCWlUKlUTAUX3y7BODdUxqU= 4 | github.com/klauspost/reedsolomon v1.9.13/go.mod h1:eqPAcE7xar5CIzcdfwydOEdcmchAKAP/qs14y4GCBOk= 5 | github.com/streadway/amqp v1.0.0 h1:kuuDrUJFZL1QYL9hUNuCxNObNzB0bV/ZG5jV3RWAQgo= 6 | github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= 7 | -------------------------------------------------------------------------------- /lib/es/es5/es.go: -------------------------------------------------------------------------------- 1 | package es5 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | url2 "net/url" 9 | "os" 10 | "strings" 11 | ) 12 | 13 | type Metadata struct { 14 | Name string 15 | Version int 16 | Size int64 17 | Hash string 18 | } 19 | 20 | type hit struct { 21 | Source Metadata `json:"_source"` 22 | } 23 | 24 | type searchResult struct { 25 | Hits struct { 26 | Total int 27 | Hits []hit 28 | } 29 | } 30 | 31 | func getMetadata(name string, versionId int) (meta Metadata, err error) { 32 | url := fmt.Sprintf("http://%s/metadata/objects/%s_%d/_source", os.Getenv("ES_SERVER"), name, versionId) 33 | result, err := http.Get(url) 34 | if err != nil { 35 | return 36 | } 37 | if result.StatusCode != http.StatusOK { 38 | err = fmt.Errorf("fail to get %s_%d:%d", name, versionId, result.StatusCode) 39 | return 40 | } 41 | result2, _ := ioutil.ReadAll(result.Body) 42 | json.Unmarshal(result2, &meta) 43 | return 44 | } 45 | 46 | func SearchLatestVersion(name string) (meta Metadata, err error) { 47 | url := fmt.Sprintf("http://%s//metadata/_search?q=name:%s&size=1&sort=version:desc", os.Getenv("ES_SERVER"), url2.PathEscape(name)) 48 | result, err := http.Get(url) 49 | if err != nil { 50 | return 51 | } 52 | if result.StatusCode != http.StatusOK { 53 | err = fmt.Errorf("fail to search latest metadata:%s", result.StatusCode) 54 | return 55 | } 56 | result2, _ := ioutil.ReadAll(result.Body) 57 | var sr searchResult 58 | json.Unmarshal(result2, &sr) 59 | if len(sr.Hits.Hits) != 0 { 60 | meta = sr.Hits.Hits[0].Source 61 | } 62 | return 63 | } 64 | 65 | func GetMetadata(name string, version int) (Metadata, error) { 66 | if version == 0 { 67 | return SearchLatestVersion(name) 68 | } 69 | return getMetadata(name, version) 70 | } 71 | 72 | func PutMetadata(name string, version int, size int64, hash string) error { 73 | document := fmt.Sprintf(`{"name":"%s","version":%d,"size":%d,"hash":"%s"}`, name, version, size, hash) 74 | client := http.Client{} 75 | url := fmt.Sprintf("http://%s/metadata/objects/%s_%d?op_type=create", os.Getenv("ES_SERVER"), name, version) 76 | request, _ := http.NewRequest("PUT", url, strings.NewReader(document)) 77 | request.Header.Set("Content-Type", "application/json") 78 | result, err := client.Do(request) 79 | if err != nil { 80 | return err 81 | } 82 | if result.StatusCode == http.StatusConflict { 83 | return PutMetadata(name, version+1, size, hash) 84 | } 85 | if result.StatusCode != http.StatusCreated { 86 | result2, _ := ioutil.ReadAll(result.Body) 87 | return fmt.Errorf("fail to put metadata:%d %s", result.StatusCode, string(result2)) 88 | } 89 | return nil 90 | } 91 | 92 | func AddVersion(name, hash string, size int64) error { 93 | version, err := SearchLatestVersion(name) 94 | if err != nil { 95 | return err 96 | } 97 | return PutMetadata(name, version.Version+1, size, hash) 98 | } 99 | 100 | func SearchAllVersions(name string, from, size int) ([]Metadata, error) { 101 | url := fmt.Sprintf("http://%s/metadata/_search?sort=name,version&from=%d&size=%d", os.Getenv("ES_SERVER"), from, size) 102 | if name != "" { 103 | url += "&q=name:" + name 104 | } 105 | result, err := http.Get(url) 106 | if err != nil { 107 | return nil, err 108 | } 109 | metas := make([]Metadata, 0) 110 | result2, _ := ioutil.ReadAll(result.Body) 111 | var sr searchResult 112 | json.Unmarshal(result2, &sr) 113 | for i := range sr.Hits.Hits { 114 | metas = append(metas, sr.Hits.Hits[i].Source) 115 | } 116 | return metas, nil 117 | } 118 | 119 | func DelMetadata(name string, version int) { 120 | url := fmt.Sprintf("http://%s/metadata/objects/%s_%d", os.Getenv("ES_SERVER"), name, version) 121 | client := http.Client{} 122 | request, _ := http.NewRequest("DELETE", url, nil) 123 | client.Do(request) 124 | } 125 | 126 | type Bucket struct { 127 | Key string 128 | DocCount int 129 | MinVersion struct { 130 | Value float32 131 | } 132 | } 133 | 134 | type aggregateResult struct { 135 | Aggregations struct { 136 | GroupByName struct { 137 | Buckets []Bucket 138 | } 139 | } 140 | } 141 | 142 | func SearchVersionStatus(minDocCount int) ([]Bucket, error) { 143 | url := fmt.Sprintf("http://%s/metadata/_search", os.Getenv("ES_SERVER")) 144 | body := fmt.Sprintf(` 145 | { 146 | "size": 0, 147 | "aggs": { 148 | "group_by_name": { 149 | "terms": { 150 | "field": "name", 151 | "min_doc_count": %d 152 | }, 153 | "aggs": { 154 | "min_version": { 155 | "min": { 156 | "field": "version" 157 | } 158 | } 159 | } 160 | } 161 | } 162 | }`, minDocCount) 163 | client := http.Client{} 164 | request, _ := http.NewRequest("GET", url, strings.NewReader(body)) 165 | request.Header.Set("Content-Type", "application/json") 166 | result, err := client.Do(request) 167 | if err != nil { 168 | return nil, err 169 | } 170 | responseBody, _ := ioutil.ReadAll(result.Body) 171 | var ar aggregateResult 172 | json.Unmarshal(responseBody, &ar) 173 | return ar.Aggregations.GroupByName.Buckets, nil 174 | } 175 | 176 | func HasHash(hash string) (bool, error) { 177 | url := fmt.Sprintf("http://%s/metadata/_search?q=hash:%s&size=0", os.Getenv("ES_SERVER"), hash) 178 | result, err := http.Get(url) 179 | if err != nil { 180 | return false, err 181 | } 182 | body, _ := ioutil.ReadAll(result.Body) 183 | var sr searchResult 184 | json.Unmarshal(body, &sr) 185 | return sr.Hits.Total != 0, nil 186 | } 187 | 188 | func SearchHashSize(hash string) (size int64, err error) { 189 | url := fmt.Sprintf("http://%s/metadata/_search?q=hash:%s&size=1", os.Getenv("ES_SERVER"), hash) 190 | result, err := http.Get(url) 191 | if err != nil { 192 | return 193 | } 194 | if result.StatusCode != http.StatusOK { 195 | err = fmt.Errorf("fail to search hash size:%d", result.StatusCode) 196 | return 197 | } 198 | body, _ := ioutil.ReadAll(result.Body) 199 | var sr searchResult 200 | json.Unmarshal(body, &sr) 201 | if len(sr.Hits.Hits) != 0 { 202 | size = sr.Hits.Hits[0].Source.Size 203 | } 204 | return 205 | } 206 | -------------------------------------------------------------------------------- /lib/es/es7/es.go: -------------------------------------------------------------------------------- 1 | package es7 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | url2 "net/url" 9 | "os" 10 | "strings" 11 | ) 12 | 13 | type Metadata struct { 14 | Name string 15 | Version int 16 | Size int64 17 | Hash string 18 | } 19 | 20 | type hit struct { 21 | Source Metadata `json:"_source"` 22 | } 23 | 24 | type searchResult struct { 25 | Hits struct { 26 | Total struct { 27 | Value int 28 | Relation string 29 | } 30 | Hits []hit 31 | } 32 | } 33 | 34 | func getMetadata(name string, versionId int) (meta Metadata, err error) { 35 | url := fmt.Sprintf("http://%s/metadata/_doc/%s_%d/_source", os.Getenv("ES_SERVER"), name, versionId) 36 | result, err := http.Get(url) 37 | if err != nil { 38 | return 39 | } 40 | if result.StatusCode != http.StatusOK { 41 | err = fmt.Errorf("fail to get %s_%d:%d", name, versionId, result.StatusCode) 42 | return 43 | } 44 | result2, _ := ioutil.ReadAll(result.Body) 45 | json.Unmarshal(result2, &meta) 46 | return 47 | } 48 | 49 | func SearchLatestVersion(name string) (meta Metadata, err error) { 50 | url := fmt.Sprintf("http://%s/metadata/_search?q=name:%s&size=1&sort=version:desc", os.Getenv("ES_SERVER"), url2.PathEscape(name)) 51 | result, err := http.Get(url) 52 | if err != nil { 53 | fmt.Println(err) 54 | return 55 | } 56 | if result.StatusCode != http.StatusOK { 57 | err = fmt.Errorf("fail to search latest metadata:%s", result.StatusCode) 58 | return 59 | } 60 | result2, err := ioutil.ReadAll(result.Body) 61 | if err != nil { 62 | fmt.Println(err) 63 | } 64 | var sr searchResult 65 | json.Unmarshal(result2, &sr) 66 | if len(sr.Hits.Hits) != 0 { 67 | meta = sr.Hits.Hits[0].Source 68 | } 69 | return 70 | } 71 | 72 | func GetMetadata(name string, version int) (Metadata, error) { 73 | if version == 0 { 74 | return SearchLatestVersion(name) 75 | } 76 | return getMetadata(name, version) 77 | } 78 | 79 | func PutMetadata(name string, version int, size int64, hash string) error { 80 | document := fmt.Sprintf(`{"name":"%s","version":%d,"size":%d,"hash":"%s"}`, name, version, size, hash) 81 | client := http.Client{} 82 | url := fmt.Sprintf("http://%s/metadata/_doc/%s_%d?op_type=create", os.Getenv("ES_SERVER"), name, version) 83 | request, _ := http.NewRequest("PUT", url, strings.NewReader(document)) 84 | request.Header.Set("Content-Type", "application/json") 85 | result, err := client.Do(request) 86 | if err != nil { 87 | return err 88 | } 89 | if result.StatusCode == http.StatusConflict { 90 | return PutMetadata(name, version+1, size, hash) 91 | } 92 | if result.StatusCode != http.StatusCreated { 93 | result2, _ := ioutil.ReadAll(result.Body) 94 | return fmt.Errorf("fail to put metadata:%d %s", result.StatusCode, string(result2)) 95 | } 96 | return nil 97 | } 98 | 99 | func AddVersion(name, hash string, size int64) error { 100 | version, err := SearchLatestVersion(name) 101 | if err != nil { 102 | return err 103 | } 104 | return PutMetadata(name, version.Version+1, size, hash) 105 | } 106 | 107 | func SearchAllVersions(name string, from, size int) ([]Metadata, error) { 108 | url := fmt.Sprintf("http://%s/metadata/_search?sort=name,version&from=%d&size=%d", os.Getenv("ES_SERVER"), from, size) 109 | if name != "" { 110 | url += "&q=name:" + name 111 | } 112 | result, err := http.Get(url) 113 | if err != nil { 114 | return nil, err 115 | } 116 | metas := make([]Metadata, 0) 117 | result2, _ := ioutil.ReadAll(result.Body) 118 | var sr searchResult 119 | json.Unmarshal(result2, &sr) 120 | for i := range sr.Hits.Hits { 121 | metas = append(metas, sr.Hits.Hits[i].Source) 122 | } 123 | return metas, nil 124 | } 125 | 126 | func DelMetadata(name string, version int) { 127 | url := fmt.Sprintf("http://%s/metadata/_doc/%s_%d", os.Getenv("ES_SERVER"), name, version) 128 | client := http.Client{} 129 | request, _ := http.NewRequest("DELETE", url, nil) 130 | client.Do(request) 131 | } 132 | 133 | type Bucket struct { 134 | Key string `json:"key"` 135 | DocCount int `json:"doc_count"` 136 | MinVersion struct { 137 | Value float32 `json:"value"` 138 | } `json:"min_version"` 139 | } 140 | 141 | type aggregateResult struct { 142 | Aggregations struct { 143 | Group_by_name struct { 144 | Buckets []Bucket `json:"buckets"` 145 | } `json:"group_by_name"` 146 | } `json:"aggregations"` 147 | } 148 | 149 | func SearchVersionStatus(minDocCount int) ([]Bucket, error) { 150 | url := fmt.Sprintf("http://%s/metadata/_search", os.Getenv("ES_SERVER")) 151 | body := fmt.Sprintf(` 152 | { 153 | "size": 0, 154 | "aggs": { 155 | "group_by_name": { 156 | "terms": { 157 | "field": "name", 158 | "min_doc_count": %d 159 | }, 160 | "aggs": { 161 | "min_version": { 162 | "min": { 163 | "field": "version" 164 | } 165 | } 166 | } 167 | } 168 | } 169 | }`, minDocCount) 170 | client := http.Client{} 171 | request, _ := http.NewRequest("GET", url, strings.NewReader(body)) 172 | request.Header.Set("Content-Type", "application/json") 173 | result, err := client.Do(request) 174 | if err != nil { 175 | return nil, err 176 | } 177 | responseBody, _ := ioutil.ReadAll(result.Body) 178 | var ar aggregateResult 179 | err = json.Unmarshal(responseBody, &ar) 180 | return ar.Aggregations.Group_by_name.Buckets, nil 181 | } 182 | 183 | func HasHash(hash string) (bool, error) { 184 | url := fmt.Sprintf("http://%s/metadata/_search?q=hash:%s&size=0", os.Getenv("ES_SERVER"), hash) 185 | result, err := http.Get(url) 186 | if err != nil { 187 | return false, err 188 | } 189 | body, _ := ioutil.ReadAll(result.Body) 190 | var sr searchResult 191 | json.Unmarshal(body, &sr) 192 | return sr.Hits.Total.Value != 0, nil 193 | } 194 | 195 | func SearchHashSize(hash string) (size int64, err error) { 196 | url := fmt.Sprintf("http://%s/metadata/_search?q=hash:%s&size=1", os.Getenv("ES_SERVER"), hash) 197 | result, err := http.Get(url) 198 | if err != nil { 199 | return 200 | } 201 | if result.StatusCode != http.StatusOK { 202 | err = fmt.Errorf("fail to search hash size:%d", result.StatusCode) 203 | return 204 | } 205 | body, _ := ioutil.ReadAll(result.Body) 206 | var sr searchResult 207 | json.Unmarshal(body, &sr) 208 | if len(sr.Hits.Hits) != 0 { 209 | size = sr.Hits.Hits[0].Source.Size 210 | } 211 | return 212 | } 213 | -------------------------------------------------------------------------------- /lib/objectStream/get.go: -------------------------------------------------------------------------------- 1 | package objectStream 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | ) 8 | 9 | type GetStream struct { 10 | reader io.Reader 11 | } 12 | 13 | func newGetStream(url string) (*GetStream, error) { 14 | fmt.Println("rpc:get stream from:", url) 15 | result, err := http.Get(url) 16 | if err != nil { 17 | return nil, err 18 | } 19 | if result.StatusCode != http.StatusOK { 20 | fmt.Println("URL:", url) 21 | return nil, fmt.Errorf("dataServer return http code %d", result.StatusCode) 22 | } 23 | return &GetStream{result.Body}, nil 24 | } 25 | 26 | func NewGetStream(server, object string) (*GetStream, error) { 27 | if server == "" || object == "" { 28 | return nil, fmt.Errorf("invalid server %s object %s", server, object) 29 | } 30 | return newGetStream("http://" + server + "/objects/" + object) 31 | } 32 | 33 | func (r *GetStream) Read(p []byte) (n int, err error) { 34 | return r.reader.Read(p) 35 | } 36 | -------------------------------------------------------------------------------- /lib/objectStream/put.go: -------------------------------------------------------------------------------- 1 | package objectStream 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | ) 8 | 9 | type PutStream struct { 10 | writer *io.PipeWriter 11 | c chan error 12 | } 13 | 14 | func NewPutStream(server, object string) *PutStream { 15 | reader, writer := io.Pipe() 16 | channel := make(chan error) 17 | go func() { 18 | request, _ := http.NewRequest("PUT", "http://"+server+"/objects/"+object, reader) 19 | client := http.Client{} 20 | result, err := client.Do(request) 21 | if err == nil && result.StatusCode != http.StatusOK { 22 | err = fmt.Errorf("dataServer return http code %d", result.StatusCode) 23 | } 24 | channel <- err 25 | }() 26 | return &PutStream{writer: writer, c: channel} 27 | } 28 | 29 | func (w *PutStream) Write(p []byte) (n int, err error) { 30 | return w.writer.Write(p) 31 | } 32 | 33 | func (w *PutStream) Close() error { 34 | w.writer.Close() 35 | return <-w.c 36 | } 37 | -------------------------------------------------------------------------------- /lib/objectStream/temp.go: -------------------------------------------------------------------------------- 1 | package objectStream 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | "strings" 8 | ) 9 | 10 | type TempPutStream struct { 11 | Server string 12 | Uuid string 13 | } 14 | 15 | func NewTempPutStream(server, object string, size int64) (*TempPutStream, error) { 16 | request, err := http.NewRequest("POST", "http://"+server+"/temp/"+object, nil) 17 | if err != nil { 18 | return nil, err 19 | } 20 | request.Header.Set("size", fmt.Sprintf("%d", size)) 21 | client := http.Client{} 22 | response, err := client.Do(request) 23 | if err != nil { 24 | return nil, err 25 | } 26 | uuid, err := ioutil.ReadAll(response.Body) 27 | if err != nil { 28 | return nil, err 29 | } 30 | return &TempPutStream{server, string(uuid)}, nil 31 | } 32 | 33 | func (w *TempPutStream) Write(p []byte) (n int, err error) { 34 | request, err := http.NewRequest("PATCH", "http://"+w.Server+"/temp/"+w.Uuid, strings.NewReader(string(p))) 35 | if err != nil { 36 | return 0, err 37 | } 38 | client := http.Client{} 39 | response, err := client.Do(request) 40 | if err != nil { 41 | return 0, err 42 | } 43 | if response.StatusCode != http.StatusOK { 44 | return 0, fmt.Errorf("dataServer return http code %d", response.StatusCode) 45 | } 46 | return len(p), nil 47 | } 48 | 49 | func (w *TempPutStream) Commit(good bool) { 50 | method := http.MethodDelete 51 | if good { 52 | method = http.MethodPut 53 | } 54 | request, _ := http.NewRequest(method, "http://"+w.Server+"/temp/"+w.Uuid, nil) 55 | client := http.Client{} 56 | client.Do(request) 57 | } 58 | 59 | func NewTempGetStream(server, uuid string) (*GetStream, error) { 60 | return newGetStream("http://" + server + "/temp/" + uuid) 61 | } 62 | -------------------------------------------------------------------------------- /lib/rabbitmq/rabbitmq.go: -------------------------------------------------------------------------------- 1 | package rabbitmq 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/streadway/amqp" 6 | ) 7 | 8 | type RabbitMQ struct { 9 | channel *amqp.Channel 10 | conn *amqp.Connection 11 | Name string 12 | exchange string 13 | } 14 | 15 | func New(s string) *RabbitMQ { 16 | connection, err := amqp.Dial(s) 17 | if err != nil { 18 | panic(err) 19 | } 20 | channel, err := connection.Channel() 21 | if err != nil { 22 | panic(err) 23 | } 24 | queue, err := channel.QueueDeclare( 25 | "", //name 26 | false, //durable 27 | true, //delete when unused 28 | false, //exclusive 29 | false, //no-wait 30 | nil) //arguments 31 | if err != nil { 32 | panic(err) 33 | } 34 | mq := new(RabbitMQ) 35 | mq.channel = channel 36 | mq.conn = connection 37 | mq.Name = queue.Name 38 | return mq 39 | } 40 | 41 | func (q *RabbitMQ) Bind(exchange string) { 42 | err := q.channel.QueueBind( 43 | q.Name, 44 | "", 45 | exchange, 46 | false, 47 | nil) 48 | if err != nil { 49 | panic(err) 50 | } 51 | q.exchange = exchange 52 | } 53 | 54 | func (q *RabbitMQ) Send(queue string, body interface{}) { 55 | str, err := json.Marshal(body) 56 | if err != nil { 57 | panic(err) 58 | } 59 | err = q.channel.Publish("", queue, false, false, amqp.Publishing{ 60 | ReplyTo: q.Name, 61 | Body: []byte(str), 62 | }) 63 | if err != nil { 64 | panic(err) 65 | } 66 | } 67 | 68 | func (q *RabbitMQ) Publish(exchange string, body interface{}) { 69 | str, err := json.Marshal(body) 70 | if err != nil { 71 | panic(err) 72 | } 73 | err = q.channel.Publish(exchange, 74 | "", 75 | false, 76 | false, 77 | amqp.Publishing{ 78 | ReplyTo: q.Name, 79 | Body: []byte(str), 80 | }) 81 | if err != nil { 82 | panic(err) 83 | } 84 | } 85 | 86 | func (q *RabbitMQ) Consume() <-chan amqp.Delivery { 87 | consume, err := q.channel.Consume(q.Name, 88 | "", 89 | true, 90 | false, 91 | false, 92 | false, 93 | nil) 94 | if err != nil { 95 | panic(err) 96 | } 97 | return consume 98 | } 99 | 100 | func (q *RabbitMQ) Close() { 101 | q.channel.Close() 102 | q.conn.Close() 103 | } 104 | -------------------------------------------------------------------------------- /lib/rabbitmq/rabbitmq_test.go: -------------------------------------------------------------------------------- 1 | package rabbitmq 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | ) 7 | 8 | const host = "amqp://test:test@192.168.110.134:5672" 9 | 10 | func TestPublish(t *testing.T) { 11 | queue := New(host) 12 | defer queue.Close() 13 | queue.Bind("test") 14 | 15 | queue2 := New(host) 16 | defer queue2.Close() 17 | queue2.Bind("test") 18 | 19 | queue3 := New(host) 20 | defer queue.Close() 21 | 22 | expect := "test" 23 | queue3.Publish("test2", "any") 24 | queue3.Publish("test", expect) 25 | 26 | consume := queue.Consume() 27 | message := <-consume 28 | var actual interface{} 29 | err := json.Unmarshal(message.Body, &actual) 30 | if err != nil { 31 | t.Error(err) 32 | } 33 | if actual != expect { 34 | t.Errorf("expected %s,actual %s", expect, actual) 35 | } 36 | if message.ReplyTo != queue3.Name { 37 | t.Error(message) 38 | } 39 | 40 | queue2.Send(message.ReplyTo, "test3") 41 | consume3 := queue3.Consume() 42 | message = <-consume3 43 | if string(message.Body) != `"test3"` { 44 | t.Error(string(message.Body)) 45 | } 46 | } 47 | 48 | func TestSend(t *testing.T) { 49 | queue := New(host) 50 | defer queue.Close() 51 | 52 | queue2 := New(host) 53 | defer queue2.Close() 54 | 55 | expect := "test" 56 | expect2 := "test2" 57 | queue2.Send(queue.Name, expect) 58 | queue2.Send(queue2.Name, expect2) 59 | 60 | consume := queue.Consume() 61 | message := <-consume 62 | var actual interface{} 63 | err := json.Unmarshal(message.Body, &actual) 64 | if err != nil { 65 | t.Error(err) 66 | } 67 | if actual != expect { 68 | t.Errorf("expected %s,actual %s", expect, actual) 69 | } 70 | consume2 := queue2.Consume() 71 | message = <-consume2 72 | err = json.Unmarshal(message.Body, &actual) 73 | if err != nil { 74 | t.Error(err) 75 | } 76 | if actual != expect2 { 77 | t.Errorf("expected %s,actual %s", expect2, actual) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /lib/rs/common.go: -------------------------------------------------------------------------------- 1 | package rs 2 | 3 | const ( 4 | DataShard = 4 5 | ParityShard = 2 6 | AllShard = DataShard + ParityShard 7 | BlockPerShard = 8000 8 | BlockSie = BlockPerShard * DataShard 9 | ) 10 | -------------------------------------------------------------------------------- /lib/rs/decoder.go: -------------------------------------------------------------------------------- 1 | package rs 2 | 3 | import ( 4 | "github.com/klauspost/reedsolomon" 5 | "io" 6 | ) 7 | 8 | type decoder struct { 9 | readers []io.Reader 10 | writers []io.Writer 11 | encode reedsolomon.Encoder 12 | size int64 13 | cache []byte 14 | cacheSize int 15 | total int64 16 | } 17 | 18 | func NewDecoder(readers []io.Reader, writers []io.Writer, size int64) *decoder { 19 | encode, _ := reedsolomon.New(DataShard, ParityShard) 20 | return &decoder{readers, writers, encode, size, nil, 0, 0} 21 | } 22 | 23 | func (d *decoder) Read(p []byte) (count int, err error) { 24 | if d.cacheSize == 0 { 25 | err = d.getData() 26 | if err != nil { 27 | return 0, err 28 | } 29 | } 30 | length := len(p) 31 | if d.cacheSize < length { 32 | length = d.cacheSize 33 | } 34 | d.cacheSize -= length 35 | copy(p, d.cache[:length]) 36 | d.cache = d.cache[length:] 37 | return length, nil 38 | } 39 | 40 | func (d *decoder) getData() error { 41 | if d.total == d.size { 42 | return io.EOF 43 | } 44 | shards := make([][]byte, AllShard) 45 | repairIds := make([]int, 0) 46 | for i := range shards { 47 | if d.readers[i] == nil { 48 | repairIds = append(repairIds, i) 49 | } else { 50 | shards[i] = make([]byte, BlockPerShard) 51 | count, err := io.ReadFull(d.readers[i], shards[i]) 52 | if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { 53 | shards[i] = nil 54 | } else if count != BlockPerShard { 55 | shards[i] = shards[i][:count] 56 | } 57 | } 58 | } 59 | err := d.encode.Reconstruct(shards) 60 | if err != nil { 61 | return err 62 | } 63 | for i := range repairIds { 64 | id := repairIds[i] 65 | d.writers[id].Write(shards[id]) 66 | } 67 | for i := 0; i < DataShard; i++ { 68 | shardSie := int64(len(shards[i])) 69 | if d.total+shardSie > d.size { 70 | shardSie -= d.total + shardSie - d.size 71 | } 72 | d.cache = append(d.cache, shards[i][:shardSie]...) 73 | d.cacheSize += int(shardSie) 74 | d.total += shardSie 75 | } 76 | return nil 77 | } 78 | -------------------------------------------------------------------------------- /lib/rs/encoder.go: -------------------------------------------------------------------------------- 1 | package rs 2 | 3 | import ( 4 | "github.com/klauspost/reedsolomon" 5 | "io" 6 | ) 7 | 8 | type encoder struct { 9 | writers []io.Writer 10 | encode reedsolomon.Encoder 11 | cache []byte 12 | } 13 | 14 | func NewEncoder(writers []io.Writer) *encoder { 15 | encode, _ := reedsolomon.New(DataShard, ParityShard) 16 | return &encoder{writers, encode, nil} 17 | } 18 | 19 | func (e *encoder) Write(p []byte) (count int, err error) { 20 | length := len(p) 21 | current := 0 22 | for length != 0 { 23 | next := BlockSie - len(e.cache) 24 | if next > length { 25 | next = length 26 | } 27 | e.cache = append(e.cache, p[current:current+next]...) 28 | if len(e.cache) == BlockSie { 29 | e.Flush() 30 | } 31 | current += next 32 | length -= next 33 | } 34 | return len(p), nil 35 | } 36 | 37 | func (e *encoder) Flush() { 38 | if len(e.cache) == 0 { 39 | return 40 | } 41 | shards, _ := e.encode.Split(e.cache) 42 | e.encode.Encode(shards) 43 | for i := range shards { 44 | e.writers[i].Write(shards[i]) 45 | } 46 | e.cache = []byte{} 47 | } 48 | -------------------------------------------------------------------------------- /lib/rs/get.go: -------------------------------------------------------------------------------- 1 | package rs 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "storage/lib/objectStream" 7 | ) 8 | 9 | type RSGetStream struct { 10 | *decoder 11 | } 12 | 13 | func NewRSGetStream(locateInfo map[int]string, dataServers []string, hash string, size int64) (*RSGetStream, error) { 14 | if len(locateInfo)+len(dataServers) != AllShard { 15 | return nil, fmt.Errorf("dataServers number mismatch") 16 | } 17 | readers := make([]io.Reader, AllShard) 18 | for i := 0; i < AllShard; i++ { 19 | server := locateInfo[i] 20 | if server == "" { 21 | locateInfo[i] = dataServers[0] 22 | dataServers = dataServers[1:] 23 | continue 24 | } 25 | reader, err := objectStream.NewGetStream(server, fmt.Sprintf("%s.%d", hash, i)) 26 | if err == nil { 27 | readers[i] = reader 28 | } 29 | } 30 | writers := make([]io.Writer, AllShard) 31 | perShard := (size + DataShard - 1) / DataShard 32 | var err error 33 | for i := range readers { 34 | if readers[i] == nil { 35 | writers[i], err = objectStream.NewTempPutStream(locateInfo[i], fmt.Sprintf("%s.%d", hash, i), perShard) 36 | if err != nil { 37 | return nil, err 38 | } 39 | } 40 | } 41 | decode := NewDecoder(readers, writers, size) 42 | return &RSGetStream{decode}, nil 43 | } 44 | 45 | func (s *RSGetStream) Close() { 46 | for i := range s.writers { 47 | if s.writers[i] != nil { 48 | s.writers[i].(*objectStream.TempPutStream).Commit(true) 49 | } 50 | } 51 | } 52 | 53 | func (s *RSGetStream) Seek(offset int64, whence int) (int64, error) { 54 | if whence != io.SeekCurrent { 55 | panic("only support SeekCurrent") 56 | } 57 | if offset < 0 { 58 | panic("only support forward seek") 59 | } 60 | for offset != 0 { 61 | length := int64(BlockSie) 62 | if offset < length { 63 | length = offset 64 | } 65 | buffer := make([]byte, length) 66 | io.ReadFull(s, buffer) 67 | offset -= length 68 | } 69 | return offset, nil 70 | } 71 | -------------------------------------------------------------------------------- /lib/rs/put.go: -------------------------------------------------------------------------------- 1 | package rs 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "storage/lib/objectStream" 7 | ) 8 | 9 | type RSPutStream struct { 10 | *encoder 11 | } 12 | 13 | func NewRSPutStream(dataServers []string, hash string, size int64) (*RSPutStream, error) { 14 | if len(dataServers) != AllShard { 15 | return nil, fmt.Errorf("dataServer number mismatch") 16 | } 17 | perShard := (size + DataShard - 1) / DataShard 18 | writers := make([]io.Writer, AllShard) 19 | var err error 20 | for i := range writers { 21 | writers[i], err = objectStream.NewTempPutStream(dataServers[i], fmt.Sprintf("%s.%d", hash, i), perShard) 22 | if err != nil { 23 | return nil, err 24 | } 25 | } 26 | encode := NewEncoder(writers) 27 | return &RSPutStream{encode}, nil 28 | } 29 | 30 | func (s *RSPutStream) Commit(success bool) { 31 | s.Flush() 32 | for i := range s.writers { 33 | s.writers[i].(*objectStream.TempPutStream).Commit(success) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /lib/rs/resumable_get.go: -------------------------------------------------------------------------------- 1 | package rs 2 | 3 | import ( 4 | "io" 5 | "storage/lib/objectStream" 6 | ) 7 | 8 | type RSResumableGetStream struct { 9 | *decoder 10 | } 11 | 12 | func NewRSResumableGetStream(dataServers, uuids []string, size int64) (*RSResumableGetStream, error) { 13 | readers := make([]io.Reader, AllShard) 14 | var err error 15 | for i := 0; i < AllShard; i++ { 16 | readers[i], err = objectStream.NewTempGetStream(dataServers[i], uuids[i]) 17 | if err != nil { 18 | return nil, err 19 | } 20 | } 21 | writers := make([]io.Writer, AllShard) 22 | decode := NewDecoder(readers, writers, size) 23 | return &RSResumableGetStream{decode}, nil 24 | } 25 | -------------------------------------------------------------------------------- /lib/rs/resumable_put.go: -------------------------------------------------------------------------------- 1 | package rs 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "log" 9 | "net/http" 10 | "storage/lib/objectStream" 11 | "storage/lib/utils" 12 | ) 13 | 14 | type resumableToken struct { 15 | Name string 16 | Size int64 17 | Hash string 18 | Servers []string 19 | Uuids []string 20 | } 21 | 22 | type RSResumablePutStream struct { 23 | *RSPutStream 24 | *resumableToken 25 | } 26 | 27 | func NewRSResumablePutStream(dataServers []string, name, hash string, size int64) (*RSResumablePutStream, error) { 28 | putStream, err := NewRSPutStream(dataServers, hash, size) 29 | if err != nil { 30 | return nil, err 31 | } 32 | uuids := make([]string, AllShard) 33 | for i := range uuids { 34 | uuids[i] = putStream.writers[i].(*objectStream.TempPutStream).Uuid 35 | } 36 | token := &resumableToken{name, size, hash, dataServers, uuids} 37 | return &RSResumablePutStream{putStream, token}, nil 38 | } 39 | 40 | func PutStreamFromToken(inputToken string) (*RSResumablePutStream, error) { 41 | bytes, err := base64.StdEncoding.DecodeString(inputToken) 42 | if err != nil { 43 | return nil, err 44 | } 45 | var token resumableToken 46 | err = json.Unmarshal(bytes, &token) 47 | if err != nil { 48 | return nil, err 49 | } 50 | writers := make([]io.Writer, AllShard) 51 | for i := range writers { 52 | writers[i] = &objectStream.TempPutStream{token.Servers[i], token.Uuids[i]} 53 | } 54 | encode := NewEncoder(writers) 55 | return &RSResumablePutStream{&RSPutStream{encode}, &token}, nil 56 | } 57 | 58 | func (s *RSResumablePutStream) ToToken() string { 59 | bytes, _ := json.Marshal(s) 60 | return base64.StdEncoding.EncodeToString(bytes) 61 | } 62 | 63 | func (s *RSResumablePutStream) CurrentSize() int64 { 64 | result, err := http.Head(fmt.Sprintf("http://%s/temp/%s", s.Servers[0], s.Uuids[0])) 65 | if err != nil { 66 | log.Println(err) 67 | return -1 68 | } 69 | if result.StatusCode != http.StatusOK { 70 | log.Println(result.StatusCode) 71 | return -1 72 | } 73 | size := utils.GetSizeFromHeader(result.Header) * DataShard 74 | if size > s.Size { 75 | size = s.Size 76 | } 77 | return size 78 | } 79 | -------------------------------------------------------------------------------- /lib/types/types.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | type LocateMessage struct { 4 | Address string 5 | Id int 6 | } 7 | -------------------------------------------------------------------------------- /lib/utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/base64" 6 | "io" 7 | "net/http" 8 | "strconv" 9 | "strings" 10 | ) 11 | 12 | func GetOffsetFromHeader(header http.Header) int64 { 13 | byteRange := header.Get("range") 14 | if len(byteRange) < 7 { 15 | 16 | return 0 17 | } 18 | if byteRange[:6] != "bytes=" { 19 | return 0 20 | } 21 | bytePos := strings.Split(byteRange[6:], "-") 22 | offset, _ := strconv.ParseInt(bytePos[0], 0, 64) 23 | return offset 24 | } 25 | 26 | func GetHashFromHeader(header http.Header) string { 27 | digest := header.Get("digest") 28 | if len(digest) < 9 { 29 | return "" 30 | } 31 | if digest[:8] != "SHA-256=" { 32 | return "" 33 | } 34 | return digest[8:] 35 | } 36 | 37 | func GetSizeFromHeader(header http.Header) int64 { 38 | size, _ := strconv.ParseInt(header.Get("content-length"), 0, 64) 39 | return size 40 | } 41 | 42 | func CalculateHash(reader io.Reader) string { 43 | hash := sha256.New() 44 | io.Copy(hash, reader) 45 | return base64.StdEncoding.EncodeToString(hash.Sum(nil)) 46 | } 47 | -------------------------------------------------------------------------------- /part1/go.mod: -------------------------------------------------------------------------------- 1 | module part1 2 | 3 | go 1.12 4 | -------------------------------------------------------------------------------- /part1/objects/get.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "os" 8 | "strings" 9 | ) 10 | 11 | func get(w http.ResponseWriter, r *http.Request) { 12 | file, err := os.Open(os.Getenv("STORAGE_ROOT") + "/objects/" + 13 | strings.Split(r.URL.EscapedPath(), "/")[2]) 14 | if err != nil { 15 | fmt.Println(err) 16 | w.WriteHeader(http.StatusNotFound) 17 | return 18 | } 19 | defer file.Close() 20 | io.Copy(w, file) 21 | } 22 | -------------------------------------------------------------------------------- /part1/objects/handler.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import "net/http" 4 | 5 | func Handler(w http.ResponseWriter, r *http.Request) { 6 | method := r.Method 7 | if method == http.MethodPut { 8 | put(w, r) 9 | return 10 | } 11 | if method == http.MethodGet { 12 | get(w, r) 13 | return 14 | } 15 | w.WriteHeader(http.StatusMethodNotAllowed) 16 | } 17 | -------------------------------------------------------------------------------- /part1/objects/put.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "os" 8 | "strings" 9 | ) 10 | 11 | func put(w http.ResponseWriter, r *http.Request) { 12 | file, err := os.Create(os.Getenv("STORAGE_ROOT") + "/objects/" + 13 | strings.Split(r.URL.EscapedPath(), "/")[2]) 14 | if err != nil { 15 | fmt.Println(err) 16 | w.WriteHeader(http.StatusInternalServerError) 17 | return 18 | } 19 | defer file.Close() 20 | io.Copy(file, r.Body) 21 | } 22 | -------------------------------------------------------------------------------- /part1/server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "os" 7 | "part1/objects" 8 | ) 9 | 10 | func main() { 11 | http.HandleFunc("/objects/", objects.Handler) 12 | log.Fatal(http.ListenAndServe(os.Getenv("LISTEN_ADDRESS"), nil)) 13 | } 14 | -------------------------------------------------------------------------------- /part2/apiServer/apiServer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "os" 7 | "storage/part2/apiServer/heartbeat" 8 | "storage/part2/apiServer/locate" 9 | "storage/part2/apiServer/objects" 10 | ) 11 | 12 | func main() { 13 | go heartbeat.ListenHeartBeat() 14 | http.HandleFunc("/objects/", objects.Handler) 15 | http.HandleFunc("/locate/", locate.Handler) 16 | log.Fatal(http.ListenAndServe(os.Getenv("LISTEN_ADDRESS"), nil)) 17 | } 18 | -------------------------------------------------------------------------------- /part2/apiServer/heartbeat/choose.go: -------------------------------------------------------------------------------- 1 | package heartbeat 2 | 3 | import ( 4 | "math/rand" 5 | ) 6 | 7 | func ChooseRandomDataServer() string { 8 | dataServers := GetDataServers() 9 | count := len(dataServers) 10 | if count == 0 { 11 | return "" 12 | } 13 | return dataServers[rand.Intn(count)] 14 | } 15 | -------------------------------------------------------------------------------- /part2/apiServer/heartbeat/heartbeat.go: -------------------------------------------------------------------------------- 1 | package heartbeat 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "storage/lib/rabbitmq" 7 | "strconv" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | var dataServers = make(map[string]time.Time) 13 | var mutex sync.Mutex 14 | 15 | func ListenHeartBeat() { 16 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 17 | defer queue.Close() 18 | queue.Bind("apiServers") 19 | consume := queue.Consume() 20 | go removeExpiredDataServer() 21 | for message := range consume { 22 | dataServer, err := strconv.Unquote(string(message.Body)) 23 | if err != nil { 24 | fmt.Println(err) 25 | } 26 | mutex.Lock() 27 | dataServers[dataServer] = time.Now() 28 | mutex.Unlock() 29 | } 30 | } 31 | 32 | func removeExpiredDataServer() { 33 | for { 34 | time.Sleep(5 * time.Second) 35 | mutex.Lock() 36 | for server, timer := range dataServers { 37 | if timer.Add(10 * time.Second).Before(time.Now()) { 38 | delete(dataServers, server) 39 | } 40 | } 41 | mutex.Unlock() 42 | } 43 | } 44 | 45 | func GetDataServers() []string { 46 | mutex.Lock() 47 | defer mutex.Unlock() 48 | dataServer := make([]string, 0) 49 | for server, _ := range dataServers { 50 | dataServer = append(dataServer, server) 51 | } 52 | return dataServer 53 | } 54 | -------------------------------------------------------------------------------- /part2/apiServer/locate/handler.go: -------------------------------------------------------------------------------- 1 | package locate 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "strings" 7 | ) 8 | 9 | func Handler(w http.ResponseWriter, r *http.Request) { 10 | method := r.Method 11 | if method != http.MethodGet { 12 | w.WriteHeader(http.StatusMethodNotAllowed) 13 | return 14 | } 15 | info := Locate(strings.Split(r.URL.EscapedPath(), "/")[2]) 16 | if len(info) == 0 { 17 | w.WriteHeader(http.StatusNotFound) 18 | return 19 | } 20 | by, _ := json.Marshal(info) 21 | w.Write(by) 22 | } 23 | -------------------------------------------------------------------------------- /part2/apiServer/locate/locate.go: -------------------------------------------------------------------------------- 1 | package locate 2 | 3 | import ( 4 | "os" 5 | "storage/lib/rabbitmq" 6 | "strconv" 7 | "time" 8 | ) 9 | 10 | func Locate(name string) string { 11 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 12 | queue.Publish("dataServers", name) 13 | consume := queue.Consume() 14 | go func() { 15 | time.Sleep(time.Second) 16 | queue.Close() 17 | }() 18 | message := <-consume 19 | str, _ := strconv.Unquote(string(message.Body)) 20 | return str 21 | } 22 | 23 | func Exist(name string) bool { 24 | return Locate(name) != "" 25 | } 26 | -------------------------------------------------------------------------------- /part2/apiServer/objects/get.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "io" 5 | "log" 6 | "net/http" 7 | "strings" 8 | ) 9 | 10 | func get(w http.ResponseWriter, r *http.Request) { 11 | object := strings.Split(r.URL.EscapedPath(), "/")[2] 12 | stream, err := getStream(object) 13 | if err != nil { 14 | log.Println(err) 15 | w.WriteHeader(http.StatusNotFound) 16 | return 17 | } 18 | io.Copy(w, stream) 19 | } 20 | -------------------------------------------------------------------------------- /part2/apiServer/objects/get_stream.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "storage/lib/objectStream" 7 | "storage/part2/apiServer/locate" 8 | ) 9 | 10 | func getStream(object string) (io.Reader, error) { 11 | server := locate.Locate(object) 12 | if server == "" { 13 | return nil, fmt.Errorf("object %s locate fail", object) 14 | } 15 | return objectStream.NewGetStream(server, object) 16 | } 17 | -------------------------------------------------------------------------------- /part2/apiServer/objects/handler.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import "net/http" 4 | 5 | func Handler(w http.ResponseWriter, r *http.Request) { 6 | method := r.Method 7 | if method == http.MethodPut { 8 | put(w, r) 9 | return 10 | } 11 | if method == http.MethodGet { 12 | get(w, r) 13 | return 14 | } 15 | w.WriteHeader(http.StatusMethodNotAllowed) 16 | } 17 | -------------------------------------------------------------------------------- /part2/apiServer/objects/put.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "strings" 7 | ) 8 | 9 | func put(w http.ResponseWriter, r *http.Request) { 10 | object := strings.Split(r.URL.EscapedPath(), "/")[2] 11 | c, err := storeObject(r.Body, object) 12 | if err != nil { 13 | log.Println(err) 14 | } 15 | w.WriteHeader(c) 16 | } 17 | -------------------------------------------------------------------------------- /part2/apiServer/objects/put_stream.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "storage/lib/objectStream" 6 | "storage/part2/apiServer/heartbeat" 7 | ) 8 | 9 | func putStream(object string) (*objectStream.PutStream, error) { 10 | server := heartbeat.ChooseRandomDataServer() 11 | if server == "" { 12 | return nil, fmt.Errorf("cannot find any dataServer") 13 | } 14 | return objectStream.NewPutStream(server, object), nil 15 | } 16 | -------------------------------------------------------------------------------- /part2/apiServer/objects/store.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "io" 5 | "net/http" 6 | ) 7 | 8 | func storeObject(r io.Reader, object string) (int, error) { 9 | stream, err := putStream(object) 10 | if err != nil { 11 | return http.StatusServiceUnavailable, err 12 | } 13 | io.Copy(stream, r) 14 | err = stream.Close() 15 | if err != nil { 16 | return http.StatusInternalServerError, err 17 | } 18 | return http.StatusOK, nil 19 | } 20 | -------------------------------------------------------------------------------- /part2/dataServer/dataServer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "os" 7 | "storage/part2/dataServer/heartbeat" 8 | "storage/part2/dataServer/locate" 9 | "storage/part2/dataServer/objects" 10 | ) 11 | 12 | func main() { 13 | go heartbeat.StartHeartBeat() 14 | go locate.StartLocate() 15 | http.HandleFunc("/objects/", objects.Handler) 16 | log.Fatal(http.ListenAndServe(os.Getenv("LISTEN_ADDRESS"), nil)) 17 | } 18 | -------------------------------------------------------------------------------- /part2/dataServer/heartbeat/heartbeat.go: -------------------------------------------------------------------------------- 1 | package heartbeat 2 | 3 | import ( 4 | "os" 5 | "storage/lib/rabbitmq" 6 | "time" 7 | ) 8 | 9 | func StartHeartBeat() { 10 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 11 | defer queue.Close() 12 | for { 13 | queue.Publish("apiServers", os.Getenv("LISTEN_ADDRESS")) 14 | time.Sleep(5 * time.Second) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /part2/dataServer/locate/locate.go: -------------------------------------------------------------------------------- 1 | package locate 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "storage/lib/rabbitmq" 7 | "strconv" 8 | ) 9 | 10 | func Locate(name string) bool { 11 | _, err := os.Stat(name) 12 | return !os.IsNotExist(err) 13 | } 14 | 15 | func StartLocate() { 16 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 17 | defer queue.Close() 18 | queue.Bind("dataServers") 19 | consume := queue.Consume() 20 | for message := range consume { 21 | object, err := strconv.Unquote(string(message.Body)) 22 | if err != nil { 23 | panic(err) 24 | } 25 | if Locate(os.Getenv("STORAGE_ROOT") + "/objects/" + object) { 26 | fmt.Println("locate success,prepare send message to ", message.ReplyTo, os.Getenv("LISTEN_ADDRESS")) 27 | queue.Send(message.ReplyTo, os.Getenv("LISTEN_ADDRESS")) 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /part2/dataServer/objects/get.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "os" 8 | "strings" 9 | ) 10 | 11 | func get(w http.ResponseWriter, r *http.Request) { 12 | file, err := os.Open(os.Getenv("STORAGE_ROOT") + "/objects/" + 13 | strings.Split(r.URL.EscapedPath(), "/")[2]) 14 | if err != nil { 15 | fmt.Println(err) 16 | w.WriteHeader(http.StatusNotFound) 17 | return 18 | } 19 | defer file.Close() 20 | io.Copy(w, file) 21 | } 22 | -------------------------------------------------------------------------------- /part2/dataServer/objects/handler.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import "net/http" 4 | 5 | func Handler(w http.ResponseWriter, r *http.Request) { 6 | method := r.Method 7 | if method == http.MethodPut { 8 | put(w, r) 9 | return 10 | } 11 | if method == http.MethodGet { 12 | get(w, r) 13 | return 14 | } 15 | w.WriteHeader(http.StatusMethodNotAllowed) 16 | } 17 | -------------------------------------------------------------------------------- /part2/dataServer/objects/put.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "os" 8 | "strings" 9 | ) 10 | 11 | func put(w http.ResponseWriter, r *http.Request) { 12 | file, err := os.Create(os.Getenv("STORAGE_ROOT") + "/objects/" + 13 | strings.Split(r.URL.EscapedPath(), "/")[2]) 14 | if err != nil { 15 | fmt.Println(err) 16 | w.WriteHeader(http.StatusInternalServerError) 17 | return 18 | } 19 | defer file.Close() 20 | io.Copy(file, r.Body) 21 | } 22 | -------------------------------------------------------------------------------- /part2/test/流程.txt: -------------------------------------------------------------------------------- 1 | 1.初始化网络环境 2 | 1.1先使用ifconfig查看ens，本机ens33 3 | sudo ifconfig ens33:1 10.29.1.1/16 4 | sudo ifconfig ens33:2 10.29.1.2/16 5 | sudo ifconfig ens33:3 10.29.1.3/16 6 | sudo ifconfig ens33:4 10.29.1.4/16 7 | sudo ifconfig ens33:5 10.29.1.5/16 8 | sudo ifconfig ens33:6 10.29.1.6/16 9 | sudo ifconfig ens33:7 10.29.2.1/16 10 | sudo ifconfig ens33:8 10.29.2.2/16 11 | 12 | 1.2 13 | 第一种：本地配置RabbitMQ环境： 14 | sudo apt-get install rabbitmq-server 15 | sudo rabbitmq-plugins enable rabbitmq_management 16 | 17 | wget localhost:15672/cli/rabbitmqadmin 18 | 19 | 创建exchange 20 | rabbitmqadmin declare exchange name=apiServers type=fanout 21 | rabbitmqadmin declare exchange name=dataServers type=fanout 22 | 23 | 添加用户 24 | sudo rabbitmqctl add_user test 25 | 26 | 授权 27 | sudo rabbitmqctl set_permissions -p / test ".*" ".*" ".*" 28 | 29 | 第二种：使用docker，在这里不做介绍，按照网上的教程映射端口即可 30 | 31 | 2.初始化存储路径 32 | for i in `seq 1 6`;do mkdir -p /tmp/$i/objects; done 33 | 34 | 3.设置环境变量 35 | export RABBITMQ_SERVER=amqp://test:test@192.168.110.134:5672 36 | 将ip改为本机ip即可 37 | 38 | 4.启动数据服务器 39 | LISTEN_ADDRESS=10.29.1.1:8800 STORAGE_ROOT=/tmp/1 go run dataServer.go & 40 | LISTEN_ADDRESS=10.29.1.2:8800 STORAGE_ROOT=/tmp/2 go run dataServer.go & 41 | LISTEN_ADDRESS=10.29.1.3:8800 STORAGE_ROOT=/tmp/3 go run dataServer.go & 42 | LISTEN_ADDRESS=10.29.1.4:8800 STORAGE_ROOT=/tmp/4 go run dataServer.go & 43 | LISTEN_ADDRESS=10.29.1.5:8800 STORAGE_ROOT=/tmp/5 go run dataServer.go & 44 | LISTEN_ADDRESS=10.29.1.6:8800 STORAGE_ROOT=/tmp/6 go run dataServer.go & 45 | 46 | 5.启动接口服务器 47 | LISTEN_ADDRESS=10.29.2.1:8800 go run apiServer.go & 48 | LISTEN_ADDRESS=10.29.2.2:8800 go run apiServer.go & 49 | 50 | 注意端口号冒号 51 | 52 | 53 | 6.测试并验证 54 | curl -v 10.29.2.1:8800/objects/test2 -XPUT -d"This is object test2" 55 | 56 | curl 10.29.2.1:8800/locate/test2 57 | 58 | curl 10.29.2.1:8800/objects/test2 59 | 60 | curl 10.29.2.2:8800/objects/test2 -------------------------------------------------------------------------------- /part2/test/错误修正.txt: -------------------------------------------------------------------------------- 1 | lib/objectstream/get.go文件存在错误，修改为 2 | func NewGetStream(server, object string) (*GetStream, error) { 3 | if server == "" || object == "" { 4 | return nil, fmt.Errorf("invalid server %s object %s", server, object) 5 | } 6 | return newGetStream("http://" + server + "/objects/" + object) 7 | } -------------------------------------------------------------------------------- /part3/apiServer/apiServer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "os" 7 | "storage/part3/apiServer/heartbeat" 8 | "storage/part3/apiServer/locate" 9 | "storage/part3/apiServer/objects" 10 | "storage/part3/apiServer/version" 11 | ) 12 | 13 | func main() { 14 | go heartbeat.ListenHeartBeat() 15 | http.HandleFunc("/objects/", objects.Handler) 16 | http.HandleFunc("/locate/", locate.Handler) 17 | http.HandleFunc("/versions/", version.Handler) 18 | log.Fatal(http.ListenAndServe(os.Getenv("LISTEN_ADDRESS"), nil)) 19 | 20 | } 21 | -------------------------------------------------------------------------------- /part3/apiServer/heartbeat/choose.go: -------------------------------------------------------------------------------- 1 | package heartbeat 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | ) 7 | 8 | func ChooseRandomDataServer() string { 9 | dataServers := GetDataServers() 10 | count := len(dataServers) 11 | fmt.Println("dataServer.size=", len(dataServers)) 12 | if count == 0 { 13 | return "" 14 | } 15 | return dataServers[rand.Intn(count)] 16 | } 17 | -------------------------------------------------------------------------------- /part3/apiServer/heartbeat/heartbeat.go: -------------------------------------------------------------------------------- 1 | package heartbeat 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "storage/lib/rabbitmq" 7 | "strconv" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | var dataServers = make(map[string]time.Time) 13 | var mutex sync.Mutex 14 | 15 | func ListenHeartBeat() { 16 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 17 | defer queue.Close() 18 | queue.Bind("apiServers") 19 | consume := queue.Consume() 20 | go removeExpiredDataServer() 21 | for message := range consume { 22 | dataServer, err := strconv.Unquote(string(message.Body)) 23 | if err != nil { 24 | fmt.Println(err) 25 | } 26 | mutex.Lock() 27 | dataServers[dataServer] = time.Now() 28 | mutex.Unlock() 29 | } 30 | } 31 | 32 | func removeExpiredDataServer() { 33 | for { 34 | time.Sleep(5 * time.Second) 35 | mutex.Lock() 36 | for server, timer := range dataServers { 37 | if timer.Add(10 * time.Second).Before(time.Now()) { 38 | delete(dataServers, server) 39 | } 40 | } 41 | mutex.Unlock() 42 | } 43 | } 44 | 45 | func GetDataServers() []string { 46 | mutex.Lock() 47 | defer mutex.Unlock() 48 | dataServer := make([]string, 0) 49 | for server, _ := range dataServers { 50 | dataServer = append(dataServer, server) 51 | } 52 | return dataServer 53 | } 54 | -------------------------------------------------------------------------------- /part3/apiServer/locate/handler.go: -------------------------------------------------------------------------------- 1 | package locate 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "strings" 7 | ) 8 | 9 | func Handler(w http.ResponseWriter, r *http.Request) { 10 | method := r.Method 11 | if method != http.MethodGet { 12 | w.WriteHeader(http.StatusMethodNotAllowed) 13 | return 14 | } 15 | info := Locate(strings.Split(r.URL.EscapedPath(), "/")[2]) 16 | if len(info) == 0 { 17 | w.WriteHeader(http.StatusNotFound) 18 | return 19 | } 20 | by, _ := json.Marshal(info) 21 | w.Write(by) 22 | } 23 | -------------------------------------------------------------------------------- /part3/apiServer/locate/locate.go: -------------------------------------------------------------------------------- 1 | package locate 2 | 3 | import ( 4 | "os" 5 | "storage/lib/rabbitmq" 6 | "strconv" 7 | "time" 8 | ) 9 | 10 | func Locate(name string) string { 11 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 12 | queue.Publish("dataServers", name) 13 | consume := queue.Consume() 14 | go func() { 15 | time.Sleep(time.Second) 16 | queue.Close() 17 | }() 18 | message := <-consume 19 | str, _ := strconv.Unquote(string(message.Body)) 20 | return str 21 | } 22 | 23 | func Exist(name string) bool { 24 | return Locate(name) != "" 25 | } 26 | -------------------------------------------------------------------------------- /part3/apiServer/objects/del.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "storage/lib/es/es7" 7 | "strings" 8 | ) 9 | 10 | func del(w http.ResponseWriter, r *http.Request) { 11 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 12 | version, err := es7.SearchLatestVersion(name) 13 | if err != nil { 14 | log.Println(err) 15 | w.WriteHeader(http.StatusInternalServerError) 16 | return 17 | } 18 | err = es7.PutMetadata(name, version.Version+1, 0, "") 19 | if err != nil { 20 | log.Println(err) 21 | w.WriteHeader(http.StatusInternalServerError) 22 | return 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /part3/apiServer/objects/get.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "io" 5 | "log" 6 | "net/http" 7 | "net/url" 8 | "storage/lib/es/es7" 9 | "strconv" 10 | "strings" 11 | ) 12 | 13 | func get(w http.ResponseWriter, r *http.Request) { 14 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 15 | versionId := r.URL.Query()["version"] 16 | version := 0 17 | var err error 18 | if len(versionId) != 0 { 19 | version, err = strconv.Atoi(versionId[0]) 20 | if err != nil { 21 | log.Println(err) 22 | w.WriteHeader(http.StatusBadRequest) 23 | return 24 | } 25 | } 26 | meta, err := es7.GetMetadata(name, version) 27 | if err != nil { 28 | log.Println(err) 29 | w.WriteHeader(http.StatusInternalServerError) 30 | return 31 | } 32 | if meta.Hash == "" { 33 | w.WriteHeader(http.StatusNotFound) 34 | return 35 | } 36 | object := url.PathEscape(meta.Hash) 37 | stream, err := getStream(object) 38 | if err != nil { 39 | log.Println(err) 40 | w.WriteHeader(http.StatusNotFound) 41 | return 42 | } 43 | io.Copy(w, stream) 44 | } 45 | -------------------------------------------------------------------------------- /part3/apiServer/objects/get_stream.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "storage/lib/objectStream" 7 | "storage/part3/apiServer/locate" 8 | ) 9 | 10 | func getStream(object string) (io.Reader, error) { 11 | server := locate.Locate(object) 12 | if server == "" { 13 | return nil, fmt.Errorf("object %s locate fail", object) 14 | } 15 | return objectStream.NewGetStream(server, object) 16 | } 17 | -------------------------------------------------------------------------------- /part3/apiServer/objects/handler.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import "net/http" 4 | 5 | func Handler(w http.ResponseWriter, r *http.Request) { 6 | method := r.Method 7 | if method == http.MethodPut { 8 | put(w, r) 9 | return 10 | } 11 | if method == http.MethodGet { 12 | get(w, r) 13 | return 14 | } 15 | if method == http.MethodDelete { 16 | del(w, r) 17 | return 18 | } 19 | w.WriteHeader(http.StatusMethodNotAllowed) 20 | 21 | } 22 | -------------------------------------------------------------------------------- /part3/apiServer/objects/put.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "net/url" 7 | "storage/lib/es/es7" 8 | "storage/lib/utils" 9 | "strings" 10 | ) 11 | 12 | func put(w http.ResponseWriter, r *http.Request) { 13 | hash := utils.GetHashFromHeader(r.Header) 14 | if hash == "" { 15 | log.Println("missing object hash in digest header") 16 | w.WriteHeader(http.StatusBadRequest) 17 | return 18 | } 19 | code, err := storeObject(r.Body, url.PathEscape(hash)) 20 | if err != nil { 21 | log.Println(err) 22 | w.WriteHeader(code) 23 | return 24 | } 25 | if code != http.StatusOK { 26 | w.WriteHeader(code) 27 | return 28 | } 29 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 30 | size := utils.GetSizeFromHeader(r.Header) 31 | err = es7.AddVersion(name, hash, size) 32 | if err != nil { 33 | log.Println(err) 34 | w.WriteHeader(http.StatusInternalServerError) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /part3/apiServer/objects/put_stream.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "storage/lib/objectStream" 6 | "storage/part3/apiServer/heartbeat" 7 | ) 8 | 9 | func putStream(object string) (*objectStream.PutStream, error) { 10 | server := heartbeat.ChooseRandomDataServer() 11 | if server == "" { 12 | return nil, fmt.Errorf("cannot find any dataServer") 13 | } 14 | return objectStream.NewPutStream(server, object), nil 15 | } 16 | -------------------------------------------------------------------------------- /part3/apiServer/objects/store.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "io" 5 | "net/http" 6 | ) 7 | 8 | func storeObject(r io.Reader, object string) (int, error) { 9 | stream, err := putStream(object) 10 | if err != nil { 11 | return http.StatusServiceUnavailable, err 12 | } 13 | io.Copy(stream, r) 14 | err = stream.Close() 15 | if err != nil { 16 | return http.StatusInternalServerError, err 17 | } 18 | return http.StatusOK, nil 19 | } 20 | -------------------------------------------------------------------------------- /part3/apiServer/version/handler.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | "storage/lib/es/es7" 8 | "strings" 9 | ) 10 | 11 | func Handler(w http.ResponseWriter, r *http.Request) { 12 | method := r.Method 13 | if method != http.MethodGet { 14 | w.WriteHeader(http.StatusMethodNotAllowed) 15 | return 16 | } 17 | from := 0 18 | size := 1000 19 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 20 | for { 21 | metas, err := es7.SearchAllVersions(name, from, size) 22 | if err != nil { 23 | log.Println(err) 24 | w.WriteHeader(http.StatusInternalServerError) 25 | return 26 | } 27 | for i := range metas { 28 | body, _ := json.Marshal(metas[i]) 29 | w.Write(body) 30 | w.Write([]byte("\n")) 31 | } 32 | if len(metas) != size { 33 | return 34 | } 35 | from += size 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /part3/dataServer/dataServer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "os" 7 | "storage/part3/dataServer/heartbeat" 8 | "storage/part3/dataServer/locate" 9 | "storage/part3/dataServer/objects" 10 | ) 11 | 12 | func main() { 13 | go heartbeat.StartHeartBeat() 14 | go locate.StartLocate() 15 | http.HandleFunc("/objects/", objects.Handler) 16 | log.Fatal(http.ListenAndServe(os.Getenv("LISTEN_ADDRESS"), nil)) 17 | } 18 | -------------------------------------------------------------------------------- /part3/dataServer/heartbeat/heartbeat.go: -------------------------------------------------------------------------------- 1 | package heartbeat 2 | 3 | import ( 4 | "os" 5 | "storage/lib/rabbitmq" 6 | "time" 7 | ) 8 | 9 | func StartHeartBeat() { 10 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 11 | defer queue.Close() 12 | for { 13 | queue.Publish("apiServers", os.Getenv("LISTEN_ADDRESS")) 14 | time.Sleep(5 * time.Second) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /part3/dataServer/locate/locate.go: -------------------------------------------------------------------------------- 1 | package locate 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "storage/lib/rabbitmq" 7 | "strconv" 8 | ) 9 | 10 | func Locate(name string) bool { 11 | _, err := os.Stat(name) 12 | return !os.IsNotExist(err) 13 | } 14 | 15 | func StartLocate() { 16 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 17 | defer queue.Close() 18 | queue.Bind("dataServers") 19 | consume := queue.Consume() 20 | for message := range consume { 21 | object, err := strconv.Unquote(string(message.Body)) 22 | if err != nil { 23 | panic(err) 24 | } 25 | if Locate(os.Getenv("STORAGE_ROOT") + "/objects/" + object) { 26 | fmt.Println("locate success,prepare send message to ", message.ReplyTo, os.Getenv("LISTEN_ADDRESS")) 27 | queue.Send(message.ReplyTo, os.Getenv("LISTEN_ADDRESS")) 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /part3/dataServer/objects/get.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "os" 8 | "strings" 9 | ) 10 | 11 | func get(w http.ResponseWriter, r *http.Request) { 12 | file, err := os.Open(os.Getenv("STORAGE_ROOT") + "/objects/" + 13 | strings.Split(r.URL.EscapedPath(), "/")[2]) 14 | if err != nil { 15 | fmt.Println(err) 16 | w.WriteHeader(http.StatusNotFound) 17 | return 18 | } 19 | defer file.Close() 20 | io.Copy(w, file) 21 | } 22 | -------------------------------------------------------------------------------- /part3/dataServer/objects/handler.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import "net/http" 4 | 5 | func Handler(w http.ResponseWriter, r *http.Request) { 6 | method := r.Method 7 | if method == http.MethodPut { 8 | put(w, r) 9 | return 10 | } 11 | if method == http.MethodGet { 12 | get(w, r) 13 | return 14 | } 15 | w.WriteHeader(http.StatusMethodNotAllowed) 16 | } 17 | -------------------------------------------------------------------------------- /part3/dataServer/objects/put.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "os" 8 | "strings" 9 | ) 10 | 11 | func put(w http.ResponseWriter, r *http.Request) { 12 | file, err := os.Create(os.Getenv("STORAGE_ROOT") + "/objects/" + 13 | strings.Split(r.URL.EscapedPath(), "/")[2]) 14 | if err != nil { 15 | fmt.Println(err) 16 | w.WriteHeader(http.StatusInternalServerError) 17 | return 18 | } 19 | defer file.Close() 20 | io.Copy(file, r.Body) 21 | } 22 | -------------------------------------------------------------------------------- /part3/test/流程.txt: -------------------------------------------------------------------------------- 1 | 1.配置elasticsearch环境 2 | 方式一 本机配置： 3 | sudo apt-get install elasticsearch 4 | 5 | 方式二 使用docker配置 6 | 设置max_map_count 7 | 先查看max_map_count，设置为262144 8 | cat /proc/sys/vm/max_map_count 9 | sudo sysctl -w vm.max_map_count=262144 10 | 11 | 下载镜像并运行 12 | docker pull elasticsearch:7.7.0 13 | docker run --name=elasticsearch-storage -d -e ES_JAVA_OPTS="-Xms512m -Xmx512m" -e "discovery.type=single-node" -p 9200:9200 -p 9300:9300 elasticsearch:7.7.0 14 | 15 | 访问localhost:9200，返回一条JSON表示成功 16 | 17 | 安装elasticsearch-head 18 | docker pull mobz/elasticsearch-head:5 19 | 创建容器 20 | docker create --name=elasticsearch-head-storage -p 9100:9100 mobz/elasticsearch-head:5 21 | 启动容器 22 | docker start elasticsearch-head-storage 23 | 访问localhost:9100，发现上方提示未连接，是因为跨域访问的问题，需要在服务端做CORS配置 24 | 25 | 进入elasticsearch容器 26 | docker exec -it elasticsearch-storage /bin/bash 27 | 编辑配置文件 28 | vi config/elasticsearch.yml 29 | 添加两行： 30 | http.cors.enabled: true 31 | http.cors.allow-origin: "*" 32 | 使用wq退出 33 | exit退出容器 34 | 重启容器 35 | docker restart elasticsearch-storage 36 | 37 | 38 | 修改vendor.js 39 | 从head中复制vendor.js 40 | docker cp elasticsearch-head-storage:/usr/src/app/_site/vendor.js /home/lxs/ 41 | 编辑vendor.js 42 | 在6886行：修改为 contentType: "application/json;charset=UTF-8" 43 | 在7573行：修改为 var inspectData = s.contentType === "application/json;charset=UTF-8" && 44 | 复制回容器 45 | docker cp /home/lxs/vendor.js elasticsearch-head-storage:/usr/src/app/_site 46 | 重启容器 47 | docker restart elasticsearch-head-storage 48 | 2.元数据服务启动以后，还需要在es上创建metadata索引以及objects类型的映射 49 | curl -H"Content-Type: application/json" 192.168.110.134:9200/metadata -XPUT -d'{"mappings":{"properties":{"name":{"type":"text","index":false},"version":{"type":"integer"},"size":{"type":"integer"},"hash":{"type":"text"}}}}' 50 | 这里由于提前创建过了所以提示错误，但是命令是有效的 51 | 52 | 3.测试 53 | 初始化网络环境，同part2 54 | 55 | 初始化存储目录，同part2 56 | 57 | 启动容器 58 | docker start rabbitmq-server-storage 59 | docker start elasticsearch-storage 60 | docker start elasticsearch-head-storage 61 | 62 | 设置环境变量 63 | export RABBITMQ_SERVER=amqp://test:test@192.168.110.134:5672 64 | export ES_SERVER=192.168.110.134:9200 65 | 66 | 启动程序，同part2 67 | 68 | 尝试发送不带有Hash的put请求，由于没有Hash，返回BadRequest 69 | curl -v 10.29.2.2:8800/objects/test3 -XPUT -d"this is object test3" 70 | 71 | 计算上传目标的Hash，并添加到Header中上传 72 | echo -n "this is object test3" | openssl dgst -sha256 -binary | base64 73 | GYqqAdFPt+CScnUDc0/Gcu3kwcWmOADKNYpiZtdbgsM= 74 | 发送数据 75 | curl -v 10.29.2.2:8800/objects/test3 -XPUT -d"this is object test3" -H"Digest: SHA-256=GYqqAdFPt+CScnUDc0/Gcu3kwcWmOADKNYpiZtdbgsM=" 76 | 正常上传，返回200 77 | 78 | 查看test3的最新版本的数据 79 | curl 10.29.2.1:8800/objects/test3 80 | 81 | 上传第二个test3版本 82 | echo -n "this is object test3 version 2" | openssl dgst -sha256 -binary | base64cWmOADKNYpiZtdbgsM=" 83 | cAPvsxZe1PR54zIESQy0BaxC1pYJIvaHSF3qEOZYYIo= 84 | curl -v 10.29.2.2:8800/objects/test3 -XPUT -d"this is object test3 version 2" -H"Digest: SHA-256=cAPvsxZe1PR54zIESQy0BaxC1pYJIvaHSF3qEOZYYIo=" 85 | 正常上传，返回200 86 | 87 | 查看test3的最新版本的数据 88 | curl 10.29.2.1:8800/objects/test3 89 | 90 | 定位文件所在数据节点 91 | curl 10.29.2.1:8800/locate/cAPvsxZe1PR54zIESQy0BaxC1pYJIvaHSF3qEOZYYIo= 92 | 93 | 查看文件版本 94 | curl 10.29.2.1:8800/versions/test3 95 | 两个版本，正常 96 | 97 | 查看指定版本的文件 98 | curl 10.29.2.1:8800/objects/test3?version=2 99 | 正常获取数据 100 | 101 | 删除文件（逻辑删除） 102 | curl -v 10.29.2.1:8800/objects/test3 -XDELETE 103 | 删除后会将Hash标记为""，正常删除，返回200 104 | 105 | 查看文件版本发现已删除 106 | curl 10.29.2.1:8800/versions/test3 107 | 108 | 109 | 指定版本查看仍然可以查看到，但直接获取文件会返回404 110 | 111 | -------------------------------------------------------------------------------- /part3/test/错误修正.txt: -------------------------------------------------------------------------------- 1 | 之前es的代码是ES5版本的，与现在使用的ES7不兼容，需要修改部分接口，原es.go中存在一个拼写错误，HashHash->HasHash 2 | 3 | ES7建立索引时，不需要建立objects类型的映射，同时删除了string类型，所以需要修改建立索引的JSON 4 | 5 | { 6 | “mappings”:{ 7 | "properties":{ 8 | "name":{ 9 | "type":"text", 10 | "index":"true", 11 | "fileddata":true 12 | }, 13 | "version":{"type":"integer"}, 14 | "size":{"type":"integer"}, 15 | "hash":{"type":"text"} 16 | } 17 | } 18 | } 19 | 将这个JSON用PUT发送到/metadata即可完成索引的建立 20 | 21 | ES7的驱动修改完毕，重新引入ES7的驱动即可 22 | 23 | 24 | -------------------------------------------------------------------------------- /part4/apiServer/apiServer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "os" 7 | "storage/part4/apiServer/heartbeat" 8 | "storage/part4/apiServer/locate" 9 | "storage/part4/apiServer/objects" 10 | "storage/part4/apiServer/version" 11 | ) 12 | 13 | func main() { 14 | go heartbeat.ListenHeartBeat() 15 | http.HandleFunc("/objects/", objects.Handler) 16 | http.HandleFunc("/locate/", locate.Handler) 17 | http.HandleFunc("/versions/", version.Handler) 18 | log.Fatal(http.ListenAndServe(os.Getenv("LISTEN_ADDRESS"), nil)) 19 | 20 | } 21 | -------------------------------------------------------------------------------- /part4/apiServer/heartbeat/choose.go: -------------------------------------------------------------------------------- 1 | package heartbeat 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | ) 7 | 8 | func ChooseRandomDataServer() string { 9 | dataServers := GetDataServers() 10 | count := len(dataServers) 11 | fmt.Println("dataServer.size=", len(dataServers)) 12 | if count == 0 { 13 | return "" 14 | } 15 | return dataServers[rand.Intn(count)] 16 | } 17 | -------------------------------------------------------------------------------- /part4/apiServer/heartbeat/heartbeat.go: -------------------------------------------------------------------------------- 1 | package heartbeat 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "storage/lib/rabbitmq" 7 | "strconv" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | var dataServers = make(map[string]time.Time) 13 | var mutex sync.Mutex 14 | 15 | func ListenHeartBeat() { 16 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 17 | defer queue.Close() 18 | queue.Bind("apiServers") 19 | consume := queue.Consume() 20 | go removeExpiredDataServer() 21 | for message := range consume { 22 | dataServer, err := strconv.Unquote(string(message.Body)) 23 | if err != nil { 24 | fmt.Println(err) 25 | } 26 | mutex.Lock() 27 | dataServers[dataServer] = time.Now() 28 | mutex.Unlock() 29 | } 30 | } 31 | 32 | func removeExpiredDataServer() { 33 | for { 34 | time.Sleep(5 * time.Second) 35 | mutex.Lock() 36 | for server, timer := range dataServers { 37 | if timer.Add(10 * time.Second).Before(time.Now()) { 38 | delete(dataServers, server) 39 | } 40 | } 41 | mutex.Unlock() 42 | } 43 | } 44 | 45 | func GetDataServers() []string { 46 | mutex.Lock() 47 | defer mutex.Unlock() 48 | dataServer := make([]string, 0) 49 | for server, _ := range dataServers { 50 | dataServer = append(dataServer, server) 51 | } 52 | return dataServer 53 | } 54 | -------------------------------------------------------------------------------- /part4/apiServer/locate/handler.go: -------------------------------------------------------------------------------- 1 | package locate 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "strings" 7 | ) 8 | 9 | func Handler(w http.ResponseWriter, r *http.Request) { 10 | method := r.Method 11 | if method != http.MethodGet { 12 | w.WriteHeader(http.StatusMethodNotAllowed) 13 | return 14 | } 15 | info := Locate(strings.Split(r.URL.EscapedPath(), "/")[2]) 16 | if len(info) == 0 { 17 | w.WriteHeader(http.StatusNotFound) 18 | return 19 | } 20 | by, _ := json.Marshal(info) 21 | w.Write(by) 22 | } 23 | -------------------------------------------------------------------------------- /part4/apiServer/locate/locate.go: -------------------------------------------------------------------------------- 1 | package locate 2 | 3 | import ( 4 | "os" 5 | "storage/lib/rabbitmq" 6 | "strconv" 7 | "time" 8 | ) 9 | 10 | func Locate(name string) string { 11 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 12 | queue.Publish("dataServers", name) 13 | consume := queue.Consume() 14 | go func() { 15 | time.Sleep(time.Second) 16 | queue.Close() 17 | }() 18 | message := <-consume 19 | str, _ := strconv.Unquote(string(message.Body)) 20 | return str 21 | } 22 | 23 | func Exist(name string) bool { 24 | return Locate(name) != "" 25 | } 26 | -------------------------------------------------------------------------------- /part4/apiServer/objects/del.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "storage/lib/es/es7" 7 | "strings" 8 | ) 9 | 10 | func del(w http.ResponseWriter, r *http.Request) { 11 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 12 | version, err := es7.SearchLatestVersion(name) 13 | if err != nil { 14 | log.Println(err) 15 | w.WriteHeader(http.StatusInternalServerError) 16 | return 17 | } 18 | err = es7.PutMetadata(name, version.Version+1, 0, "") 19 | if err != nil { 20 | log.Println(err) 21 | w.WriteHeader(http.StatusInternalServerError) 22 | return 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /part4/apiServer/objects/get.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "io" 5 | "log" 6 | "net/http" 7 | "net/url" 8 | "storage/lib/es/es7" 9 | "strconv" 10 | "strings" 11 | ) 12 | 13 | func get(w http.ResponseWriter, r *http.Request) { 14 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 15 | versionId := r.URL.Query()["version"] 16 | version := 0 17 | var err error 18 | if len(versionId) != 0 { 19 | version, err = strconv.Atoi(versionId[0]) 20 | if err != nil { 21 | log.Println(err) 22 | w.WriteHeader(http.StatusBadRequest) 23 | return 24 | } 25 | } 26 | meta, err := es7.GetMetadata(name, version) 27 | if err != nil { 28 | log.Println(err) 29 | w.WriteHeader(http.StatusInternalServerError) 30 | return 31 | } 32 | if meta.Hash == "" { 33 | w.WriteHeader(http.StatusNotFound) 34 | return 35 | } 36 | object := url.PathEscape(meta.Hash) 37 | stream, err := getStream(object) 38 | if err != nil { 39 | log.Println(err) 40 | w.WriteHeader(http.StatusNotFound) 41 | return 42 | } 43 | io.Copy(w, stream) 44 | } 45 | -------------------------------------------------------------------------------- /part4/apiServer/objects/get_stream.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "storage/lib/objectStream" 7 | "storage/part4/apiServer/locate" 8 | ) 9 | 10 | func getStream(object string) (io.Reader, error) { 11 | server := locate.Locate(object) 12 | if server == "" { 13 | return nil, fmt.Errorf("object %s locate fail", object) 14 | } 15 | return objectStream.NewGetStream(server, object) 16 | } 17 | -------------------------------------------------------------------------------- /part4/apiServer/objects/handler.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import "net/http" 4 | 5 | func Handler(w http.ResponseWriter, r *http.Request) { 6 | method := r.Method 7 | if method == http.MethodPut { 8 | put(w, r) 9 | return 10 | } 11 | if method == http.MethodGet { 12 | get(w, r) 13 | return 14 | } 15 | if method == http.MethodDelete { 16 | del(w, r) 17 | return 18 | } 19 | w.WriteHeader(http.StatusMethodNotAllowed) 20 | 21 | } 22 | -------------------------------------------------------------------------------- /part4/apiServer/objects/put.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "storage/lib/es/es7" 7 | "storage/lib/utils" 8 | "strings" 9 | ) 10 | 11 | func put(w http.ResponseWriter, r *http.Request) { 12 | hash := utils.GetHashFromHeader(r.Header) 13 | if hash == "" { 14 | log.Println("missing object hash in digest header") 15 | w.WriteHeader(http.StatusBadRequest) 16 | return 17 | } 18 | size := utils.GetSizeFromHeader(r.Header) 19 | code, err := storeObject(r.Body, hash, size) 20 | if err != nil { 21 | log.Println(err) 22 | w.WriteHeader(code) 23 | return 24 | } 25 | if code != http.StatusOK { 26 | w.WriteHeader(code) 27 | return 28 | } 29 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 30 | err = es7.AddVersion(name, hash, size) 31 | if err != nil { 32 | log.Println(err) 33 | w.WriteHeader(http.StatusInternalServerError) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /part4/apiServer/objects/put_stream.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "storage/lib/objectStream" 6 | "storage/part4/apiServer/heartbeat" 7 | ) 8 | 9 | func putStream(hash string, size int64) (*objectStream.TempPutStream, error) { 10 | server := heartbeat.ChooseRandomDataServer() 11 | if server == "" { 12 | return nil, fmt.Errorf("cannot find any dataServer") 13 | } 14 | return objectStream.NewTempPutStream(server, hash, size) 15 | } 16 | -------------------------------------------------------------------------------- /part4/apiServer/objects/store.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "net/url" 8 | "storage/lib/utils" 9 | "storage/part4/apiServer/locate" 10 | ) 11 | 12 | func storeObject(r io.Reader, hash string, size int64) (int, error) { 13 | if locate.Exist(url.PathEscape(hash)) { 14 | return http.StatusOK, nil 15 | } 16 | stream, err := putStream(url.PathEscape(hash), size) 17 | if err != nil { 18 | return http.StatusInternalServerError, err 19 | } 20 | reader := io.TeeReader(r, stream) 21 | caculate := utils.CalculateHash(reader) 22 | if caculate != hash { 23 | stream.Commit(false) 24 | return http.StatusBadRequest, fmt.Errorf("object hash mismatch,caculated=%s,requestd=%s", caculate, hash) 25 | } 26 | stream.Commit(true) 27 | return http.StatusOK, nil 28 | } 29 | -------------------------------------------------------------------------------- /part4/apiServer/version/handler.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | "storage/lib/es/es7" 8 | "strings" 9 | ) 10 | 11 | func Handler(w http.ResponseWriter, r *http.Request) { 12 | method := r.Method 13 | if method != http.MethodGet { 14 | w.WriteHeader(http.StatusMethodNotAllowed) 15 | return 16 | } 17 | from := 0 18 | size := 1000 19 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 20 | for { 21 | metas, err := es7.SearchAllVersions(name, from, size) 22 | if err != nil { 23 | log.Println(err) 24 | w.WriteHeader(http.StatusInternalServerError) 25 | return 26 | } 27 | for i := range metas { 28 | body, _ := json.Marshal(metas[i]) 29 | w.Write(body) 30 | w.Write([]byte("\n")) 31 | } 32 | if len(metas) != size { 33 | return 34 | } 35 | from += size 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /part4/dataServer/dataServer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "os" 7 | "storage/part4/dataServer/heartbeat" 8 | "storage/part4/dataServer/locate" 9 | "storage/part4/dataServer/objects" 10 | "storage/part4/dataServer/temp" 11 | ) 12 | 13 | func main() { 14 | locate.CollectObject() 15 | go heartbeat.StartHeartBeat() 16 | go locate.StartLocate() 17 | http.HandleFunc("/objects/", objects.Handler) 18 | http.HandleFunc("/temp/", temp.Handler) 19 | log.Fatal(http.ListenAndServe(os.Getenv("LISTEN_ADDRESS"), nil)) 20 | } 21 | -------------------------------------------------------------------------------- /part4/dataServer/heartbeat/heartbeat.go: -------------------------------------------------------------------------------- 1 | package heartbeat 2 | 3 | import ( 4 | "os" 5 | "storage/lib/rabbitmq" 6 | "time" 7 | ) 8 | 9 | func StartHeartBeat() { 10 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 11 | defer queue.Close() 12 | for { 13 | queue.Publish("apiServers", os.Getenv("LISTEN_ADDRESS")) 14 | time.Sleep(5 * time.Second) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /part4/dataServer/locate/locate.go: -------------------------------------------------------------------------------- 1 | package locate 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "storage/lib/rabbitmq" 8 | "strconv" 9 | "sync" 10 | ) 11 | 12 | var objects = make(map[string]int) 13 | var mutex sync.Mutex 14 | 15 | func Locate(hash string) bool { 16 | mutex.Lock() 17 | _, ok := objects[hash] 18 | mutex.Unlock() 19 | return ok 20 | } 21 | 22 | func Add(hash string) { 23 | mutex.Lock() 24 | objects[hash] = 1 25 | mutex.Unlock() 26 | } 27 | 28 | func Del(hash string) { 29 | mutex.Lock() 30 | delete(objects, hash) 31 | mutex.Unlock() 32 | } 33 | 34 | func StartLocate() { 35 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 36 | defer queue.Close() 37 | queue.Bind("dataServers") 38 | consume := queue.Consume() 39 | for message := range consume { 40 | hash, err := strconv.Unquote(string(message.Body)) 41 | if err != nil { 42 | fmt.Println(err) 43 | continue 44 | } 45 | exist := Locate(hash) 46 | if exist { 47 | queue.Send(message.ReplyTo, os.Getenv("LISTEN_ADDRESS")) 48 | } 49 | 50 | } 51 | } 52 | 53 | func CollectObject() { 54 | files, _ := filepath.Glob(os.Getenv("STORAGE_ROOT") + "/objects/*") 55 | for i := range files { 56 | hash := filepath.Base(files[i]) 57 | objects[hash] = 1 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /part4/dataServer/objects/get.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | ) 7 | 8 | func get(w http.ResponseWriter, r *http.Request) { 9 | file := getFile(strings.Split(r.URL.EscapedPath(), "/")[2]) 10 | if file == "" { 11 | w.WriteHeader(http.StatusNotFound) 12 | return 13 | } 14 | sendFile(w, file) 15 | } 16 | -------------------------------------------------------------------------------- /part4/dataServer/objects/get_object.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/url" 7 | "os" 8 | "storage/lib/utils" 9 | "storage/part4/dataServer/locate" 10 | ) 11 | 12 | func getFile(hash string) string { 13 | filePath := os.Getenv("STORAGE_ROOT") + "/objects/" + hash 14 | file, _ := os.Open(filePath) 15 | document := url.PathEscape(utils.CalculateHash(file)) 16 | file.Close() 17 | fmt.Println("hash:", hash, "-----document: ", document) 18 | if document != hash { 19 | log.Println("object hash mismatch,remove,", filePath) 20 | locate.Del(hash) 21 | os.Remove(filePath) 22 | return "" 23 | } 24 | return filePath 25 | } 26 | -------------------------------------------------------------------------------- /part4/dataServer/objects/handler.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import "net/http" 4 | 5 | func Handler(w http.ResponseWriter, r *http.Request) { 6 | method := r.Method 7 | if method == http.MethodGet { 8 | get(w, r) 9 | return 10 | } 11 | w.WriteHeader(http.StatusMethodNotAllowed) 12 | } 13 | -------------------------------------------------------------------------------- /part4/dataServer/objects/send.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "io" 5 | "os" 6 | ) 7 | 8 | func sendFile(w io.Writer, filePath string) { 9 | file, _ := os.Open(filePath) 10 | defer file.Close() 11 | io.Copy(w, file) 12 | 13 | } 14 | -------------------------------------------------------------------------------- /part4/dataServer/temp/commit.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "os" 5 | "storage/part4/dataServer/locate" 6 | ) 7 | 8 | func commitTempObject(dataFile string, tempInfo *tempInfo) { 9 | os.Rename(dataFile, os.Getenv("STORAGE_ROOT")+"/objects/"+tempInfo.Name) 10 | locate.Add(tempInfo.Name) 11 | } 12 | -------------------------------------------------------------------------------- /part4/dataServer/temp/del.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "net/http" 5 | "os" 6 | "strings" 7 | ) 8 | 9 | func del(w http.ResponseWriter, r *http.Request) { 10 | uuid := strings.Split(r.URL.EscapedPath(), "/")[2] 11 | infoFile := os.Getenv("STORAGE_ROOT") + "/temp/" + uuid 12 | dataFile := infoFile + ".dat" 13 | os.Remove(infoFile) 14 | os.Remove(dataFile) 15 | } 16 | -------------------------------------------------------------------------------- /part4/dataServer/temp/handler.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import "net/http" 4 | 5 | func Handler(w http.ResponseWriter, r *http.Request) { 6 | method := r.Method 7 | if method == http.MethodPut { 8 | put(w, r) 9 | return 10 | } 11 | if method == http.MethodPatch { 12 | patch(w, r) 13 | return 14 | } 15 | if method == http.MethodPost { 16 | post(w, r) 17 | return 18 | } 19 | if method == http.MethodDelete { 20 | del(w, r) 21 | return 22 | } 23 | w.WriteHeader(http.StatusMethodNotAllowed) 24 | } 25 | -------------------------------------------------------------------------------- /part4/dataServer/temp/patch.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | "io/ioutil" 7 | "log" 8 | "net/http" 9 | "os" 10 | "strings" 11 | ) 12 | 13 | func patch(w http.ResponseWriter, r *http.Request) { 14 | uuid := strings.Split(r.URL.EscapedPath(), "/")[2] 15 | temp_info, err := readFromFile(uuid) 16 | if err != nil { 17 | log.Println(err) 18 | w.WriteHeader(http.StatusNotFound) 19 | return 20 | } 21 | filePath := os.Getenv("STORAGE_ROOT") + "/temp/" + uuid 22 | dataFile := filePath + ".dat" 23 | file, err := os.OpenFile(dataFile, os.O_WRONLY|os.O_APPEND, 0) 24 | if err != nil { 25 | log.Println(err) 26 | w.WriteHeader(http.StatusInternalServerError) 27 | return 28 | } 29 | defer file.Close() 30 | _, err = io.Copy(file, r.Body) 31 | if err != nil { 32 | log.Println(err) 33 | w.WriteHeader(http.StatusInternalServerError) 34 | } 35 | info, err := file.Stat() 36 | if err != nil { 37 | log.Println(err) 38 | w.WriteHeader(http.StatusInternalServerError) 39 | return 40 | } 41 | actual := info.Size() 42 | if actual > temp_info.Size { 43 | os.Remove(dataFile) 44 | os.Remove(filePath) 45 | log.Println("actual size,", actual, " exceeds ", temp_info.Size) 46 | w.WriteHeader(http.StatusInternalServerError) 47 | } 48 | } 49 | 50 | func readFromFile(uuid string) (*tempInfo, error) { 51 | file, err := os.Open(os.Getenv("STORAGE_ROOT") + "/temp/" + uuid) 52 | if err != nil { 53 | return nil, err 54 | } 55 | defer file.Close() 56 | bytes, _ := ioutil.ReadAll(file) 57 | var info tempInfo 58 | json.Unmarshal(bytes, &info) 59 | return &info, nil 60 | } 61 | -------------------------------------------------------------------------------- /part4/dataServer/temp/post.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | "os" 8 | "os/exec" 9 | "strconv" 10 | "strings" 11 | ) 12 | 13 | type tempInfo struct { 14 | Uuid string 15 | Name string 16 | Size int64 17 | } 18 | 19 | func post(w http.ResponseWriter, r *http.Request) { 20 | output, _ := exec.Command("uuidgen").Output() 21 | uuid := strings.TrimSuffix(string(output), "\n") 22 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 23 | size, err := strconv.ParseInt(r.Header.Get("size"), 0, 64) 24 | if err != nil { 25 | log.Println(err) 26 | w.WriteHeader(http.StatusInternalServerError) 27 | return 28 | } 29 | temp := tempInfo{uuid, name, size} 30 | err = temp.writeToFile() 31 | if err != nil { 32 | log.Println(err) 33 | w.WriteHeader(http.StatusInternalServerError) 34 | return 35 | } 36 | os.Create(os.Getenv("STORAGE_ROOT") + "/temp/" + temp.Uuid + ".dat") 37 | w.Write([]byte(uuid)) 38 | } 39 | 40 | func (t *tempInfo) writeToFile() error { 41 | file, err := os.Create(os.Getenv("STORAGE_ROOT") + "/temp/" + t.Uuid) 42 | if err != nil { 43 | return err 44 | } 45 | defer file.Close() 46 | bytes, _ := json.Marshal(t) 47 | file.Write(bytes) 48 | return nil 49 | } 50 | -------------------------------------------------------------------------------- /part4/dataServer/temp/put.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "os" 7 | "strings" 8 | ) 9 | 10 | func put(w http.ResponseWriter, r *http.Request) { 11 | uuid := strings.Split(r.URL.EscapedPath(), "/")[2] 12 | temp_info, err := readFromFile(uuid) 13 | if err != nil { 14 | log.Println(err) 15 | w.WriteHeader(http.StatusNotFound) 16 | return 17 | } 18 | infoFile := os.Getenv("STORAGE_ROOT") + "/temp/" + uuid 19 | dataFile := infoFile + ".dat" 20 | file, err := os.Open(dataFile) 21 | if err != nil { 22 | log.Println(err) 23 | w.WriteHeader(http.StatusInternalServerError) 24 | return 25 | } 26 | defer file.Close() 27 | info, err := file.Stat() 28 | if err != nil { 29 | log.Println(err) 30 | w.WriteHeader(http.StatusInternalServerError) 31 | return 32 | } 33 | actual := info.Size() 34 | os.Remove(infoFile) 35 | if actual != temp_info.Size { 36 | os.Remove(dataFile) 37 | log.Println("actual size mismatch,expect ", temp_info.Size, " actual ", actual) 38 | w.WriteHeader(http.StatusInternalServerError) 39 | return 40 | } 41 | commitTempObject(dataFile, temp_info) 42 | } 43 | -------------------------------------------------------------------------------- /part4/test/命令汇总.txt: -------------------------------------------------------------------------------- 1 | sudo ifconfig ens33:1 10.29.1.1/16 2 | sudo ifconfig ens33:2 10.29.1.2/16 3 | sudo ifconfig ens33:3 10.29.1.3/16 4 | sudo ifconfig ens33:4 10.29.1.4/16 5 | sudo ifconfig ens33:5 10.29.1.5/16 6 | sudo ifconfig ens33:6 10.29.1.6/16 7 | sudo ifconfig ens33:7 10.29.2.1/16 8 | sudo ifconfig ens33:8 10.29.2.2/16 9 | for i in `seq 1 6`;do mkdir -p /tmp/$i/objects; done 10 | for i in `seq 1 6`;do mkdir -p /tmp/$i/temp;done 11 | 12 | docker start rabbitmq-server-storage 13 | docker start elasticsearch-storage 14 | docker start elasticsearch-head-storage 15 | 16 | export ES_SERVER=192.168.110.134:9200 17 | export RABBITMQ_SERVER=amqp://test:test@192.168.110.134:5672 18 | 19 | LISTEN_ADDRESS=10.29.1.1:8800 STORAGE_ROOT=/tmp/1 go run dataServer.go & 20 | LISTEN_ADDRESS=10.29.1.2:8800 STORAGE_ROOT=/tmp/2 go run dataServer.go & 21 | LISTEN_ADDRESS=10.29.1.3:8800 STORAGE_ROOT=/tmp/3 go run dataServer.go & 22 | LISTEN_ADDRESS=10.29.1.4:8800 STORAGE_ROOT=/tmp/4 go run dataServer.go & 23 | LISTEN_ADDRESS=10.29.1.5:8800 STORAGE_ROOT=/tmp/5 go run dataServer.go & 24 | LISTEN_ADDRESS=10.29.1.6:8800 STORAGE_ROOT=/tmp/6 go run dataServer.go & 25 | 26 | LISTEN_ADDRESS=10.29.2.1:8800 go run apiServer.go & 27 | LISTEN_ADDRESS=10.29.2.2:8800 go run apiServer.go & -------------------------------------------------------------------------------- /part4/test/流程.txt: -------------------------------------------------------------------------------- 1 | 1.初始化网络环境 2 | sudo ifconfig ens33:1 10.29.1.1/16 3 | sudo ifconfig ens33:2 10.29.1.2/16 4 | sudo ifconfig ens33:3 10.29.1.3/16 5 | sudo ifconfig ens33:4 10.29.1.4/16 6 | sudo ifconfig ens33:5 10.29.1.5/16 7 | sudo ifconfig ens33:6 10.29.1.6/16 8 | sudo ifconfig ens33:7 10.29.2.1/16 9 | sudo ifconfig ens33:8 10.29.2.2/16 10 | 11 | 2.初始化存储路径 12 | for i in `seq 1 6`;do mkdir -p /tmp/$i/objects; done 13 | for i in `seq 1 6`;do mkdir -p /tmp/$i/temp;done 14 | 15 | 3.启动容器 16 | docker start rabbitmq-server-storage 17 | docker start elasticsearch-storage 18 | docker start elasticsearch-head-storage 19 | 20 | 4.设置环境变量 21 | export ES_SERVER=192.168.110.134:9200 22 | export RABBITMQ_SERVER=amqp://test:test@192.168.110.134:5672 23 | 24 | 5.启动程序 25 | LISTEN_ADDRESS=10.29.1.1:8800 STORAGE_ROOT=/tmp/1 go run dataServer.go & 26 | LISTEN_ADDRESS=10.29.1.2:8800 STORAGE_ROOT=/tmp/2 go run dataServer.go & 27 | LISTEN_ADDRESS=10.29.1.3:8800 STORAGE_ROOT=/tmp/3 go run dataServer.go & 28 | LISTEN_ADDRESS=10.29.1.4:8800 STORAGE_ROOT=/tmp/4 go run dataServer.go & 29 | LISTEN_ADDRESS=10.29.1.5:8800 STORAGE_ROOT=/tmp/5 go run dataServer.go & 30 | LISTEN_ADDRESS=10.29.1.6:8800 STORAGE_ROOT=/tmp/6 go run dataServer.go & 31 | 32 | LISTEN_ADDRESS=10.29.2.1:8800 go run apiServer.go & 33 | LISTEN_ADDRESS=10.29.2.2:8800 go run apiServer.go & 34 | 35 | 6.测试 36 | 先计算目标Hash 37 | echo -n "this object will have only 1 instance" | openssl dgst -sha256 -binary | base64 38 | aWKQ2BipX94sb+h3xdTbWYAu1yzjn5vyFG2SOwUQIXY= 39 | 40 | 先上传一个包含错误Hash的请求 41 | curl -v 10.29.2.1:8800/objects/test4_1 -XPUT -d"this object will have only 1 instance" -H"Digest: SHA-256=incorrectHash" 42 | 由于Hash错误，返回BadRequest 43 | 44 | 上传包含正确Hash的请求 45 | curl -v 10.29.2.1:8800/objects/test4_1 -XPUT -d"this object will have only 1 instance" -H"Digest: SHA-256=aWKQ2BipX94sb+h3xdTbWYAu1yzjn5vyFG2SOwUQIXY=" 46 | 返回200 47 | 48 | 上传包含正确Hash，但是名字不同的请求 49 | curl -v 10.29.2.1:8800/objects/test4_2 -XPUT -d"this object will have only 1 instance" -H"Digest: SHA-256=aWKQ2BipX94sb+h3xdTbWYAu1yzjn5vyFG2SOwUQIXY=" 50 | 返回200 51 | 52 | 定位文件所在节点 53 | curl 10.29.2.1:8800/locate/aWKQ2BipX94sb+h3xdTbWYAu1yzjn5vyFG2SOwUQIXY= 54 | 定位成功 55 | 56 | 查看对象的数据 57 | curl 10.29.2.1:8800/objects/test4_2 58 | 59 | 查看名字不同但Hash相同的对象数据 60 | curl 10.29.2.1:8800/objects/test4_1 61 | 62 | 查看版本信息 63 | curl 10.29.2.1:8800/version/test4_1 64 | 65 | 查看版本信息 66 | curl 10.29.2.1:8800/version/test4_2 67 | 68 | 两个版本信息一致，说明映射的是一个文件 69 | 70 | 多次上传文件，查看数据节点中文件数量，仅有一个副本 71 | 72 | 至此，简单的数据校验与去重完成 -------------------------------------------------------------------------------- /part5/apiServer/apiServer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "os" 7 | "storage/part5/apiServer/heartbeat" 8 | "storage/part5/apiServer/locate" 9 | "storage/part5/apiServer/objects" 10 | "storage/part5/apiServer/versions" 11 | ) 12 | 13 | func main() { 14 | go heartbeat.ListenHeartBeat() 15 | http.HandleFunc("/objects/", objects.Handler) 16 | http.HandleFunc("/locate/", locate.Handler) 17 | http.HandleFunc("/versions/", version.Handler) 18 | log.Fatal(http.ListenAndServe(os.Getenv("LISTEN_ADDRESS"), nil)) 19 | 20 | } 21 | -------------------------------------------------------------------------------- /part5/apiServer/heartbeat/choose.go: -------------------------------------------------------------------------------- 1 | package heartbeat 2 | 3 | import "math/rand" 4 | 5 | func ChooseRandomDataServers(n int, exclude map[int]string) (dataServers []string) { 6 | candidates := make([]string, 0) 7 | reverseExcludeMap := make(map[string]int) 8 | for id, address := range exclude { 9 | reverseExcludeMap[address] = id 10 | } 11 | servers := GetDataServers() 12 | for i := range servers { 13 | server := servers[i] 14 | _, exclude := reverseExcludeMap[server] 15 | if !exclude { 16 | candidates = append(candidates, server) 17 | } 18 | } 19 | length := len(candidates) 20 | if length < n { 21 | return 22 | } 23 | part := rand.Perm(length) 24 | for i := 0; i < n; i++ { 25 | dataServers = append(dataServers, candidates[part[i]]) 26 | } 27 | return 28 | } 29 | -------------------------------------------------------------------------------- /part5/apiServer/heartbeat/heartbeat.go: -------------------------------------------------------------------------------- 1 | package heartbeat 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "storage/lib/rabbitmq" 7 | "strconv" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | var dataServers = make(map[string]time.Time) 13 | var mutex sync.Mutex 14 | 15 | func ListenHeartBeat() { 16 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 17 | defer queue.Close() 18 | queue.Bind("apiServers") 19 | consume := queue.Consume() 20 | go removeExpiredDataServer() 21 | for message := range consume { 22 | dataServer, err := strconv.Unquote(string(message.Body)) 23 | if err != nil { 24 | fmt.Println(err) 25 | } 26 | mutex.Lock() 27 | dataServers[dataServer] = time.Now() 28 | mutex.Unlock() 29 | } 30 | } 31 | 32 | func removeExpiredDataServer() { 33 | for { 34 | time.Sleep(5 * time.Second) 35 | mutex.Lock() 36 | for server, timer := range dataServers { 37 | if timer.Add(10 * time.Second).Before(time.Now()) { 38 | delete(dataServers, server) 39 | } 40 | } 41 | mutex.Unlock() 42 | } 43 | } 44 | 45 | func GetDataServers() []string { 46 | mutex.Lock() 47 | defer mutex.Unlock() 48 | dataServer := make([]string, 0) 49 | for server, _ := range dataServers { 50 | dataServer = append(dataServer, server) 51 | } 52 | return dataServer 53 | } 54 | -------------------------------------------------------------------------------- /part5/apiServer/locate/handler.go: -------------------------------------------------------------------------------- 1 | package locate 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "strings" 7 | ) 8 | 9 | func Handler(w http.ResponseWriter, r *http.Request) { 10 | method := r.Method 11 | if method != http.MethodGet { 12 | w.WriteHeader(http.StatusMethodNotAllowed) 13 | return 14 | } 15 | info := Locate(strings.Split(r.URL.EscapedPath(), "/")[2]) 16 | if len(info) == 0 { 17 | w.WriteHeader(http.StatusNotFound) 18 | return 19 | } 20 | by, _ := json.Marshal(info) 21 | w.Write(by) 22 | } 23 | -------------------------------------------------------------------------------- /part5/apiServer/locate/locate.go: -------------------------------------------------------------------------------- 1 | package locate 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | "storage/lib/rabbitmq" 8 | "storage/lib/rs" 9 | "storage/lib/types" 10 | "time" 11 | ) 12 | 13 | func Locate(name string) (locateInfo map[int]string) { 14 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 15 | queue.Publish("dataServers", name) 16 | consume := queue.Consume() 17 | go func() { 18 | time.Sleep(time.Second) 19 | queue.Close() 20 | }() 21 | locateInfo = make(map[int]string) 22 | for i := 0; i < rs.AllShard; i++ { 23 | message := <-consume 24 | fmt.Println("receive message:", message.Body) 25 | if len(message.Body) == 0 { 26 | return 27 | } 28 | var info types.LocateMessage 29 | json.Unmarshal(message.Body, &info) 30 | fmt.Println("locate message:", info.Address) 31 | locateInfo[info.Id] = info.Address 32 | } 33 | return 34 | } 35 | 36 | func Exist(name string) bool { 37 | return len(Locate(name)) >= rs.DataShard 38 | } 39 | -------------------------------------------------------------------------------- /part5/apiServer/objects/del.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "storage/lib/es/es7" 7 | "strings" 8 | ) 9 | 10 | func del(w http.ResponseWriter, r *http.Request) { 11 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 12 | version, err := es7.SearchLatestVersion(name) 13 | if err != nil { 14 | log.Println(err) 15 | w.WriteHeader(http.StatusInternalServerError) 16 | return 17 | } 18 | err = es7.PutMetadata(name, version.Version+1, 0, "") 19 | if err != nil { 20 | log.Println(err) 21 | w.WriteHeader(http.StatusInternalServerError) 22 | return 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /part5/apiServer/objects/get.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "io" 5 | "log" 6 | "net/http" 7 | "net/url" 8 | "storage/lib/es/es7" 9 | "strconv" 10 | "strings" 11 | ) 12 | 13 | func get(w http.ResponseWriter, r *http.Request) { 14 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 15 | versionId := r.URL.Query()["version"] 16 | version := 0 17 | var err error 18 | if len(versionId) != 0 { 19 | version, err = strconv.Atoi(versionId[0]) 20 | if err != nil { 21 | log.Println(err) 22 | w.WriteHeader(http.StatusBadRequest) 23 | return 24 | } 25 | } 26 | meta, err := es7.GetMetadata(name, version) 27 | if err != nil { 28 | log.Println(err) 29 | w.WriteHeader(http.StatusInternalServerError) 30 | return 31 | } 32 | if meta.Hash == "" { 33 | w.WriteHeader(http.StatusNotFound) 34 | } 35 | hash := url.PathEscape(meta.Hash) 36 | stream, err := GetStream(hash, meta.Size) 37 | if err != nil { 38 | log.Println(err) 39 | w.WriteHeader(http.StatusNotFound) 40 | return 41 | } 42 | _, err = io.Copy(w, stream) 43 | if err != nil { 44 | log.Println(err) 45 | w.WriteHeader(http.StatusNotFound) 46 | return 47 | } 48 | stream.Close() 49 | } 50 | -------------------------------------------------------------------------------- /part5/apiServer/objects/get_stream.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "storage/lib/rs" 6 | "storage/part5/apiServer/heartbeat" 7 | "storage/part5/apiServer/locate" 8 | ) 9 | 10 | func GetStream(hash string, size int64) (*rs.RSGetStream, error) { 11 | locateInfo := locate.Locate(hash) 12 | if len(locateInfo) < rs.DataShard { 13 | return nil, fmt.Errorf("object %s locate fail,result %v", hash, locateInfo) 14 | } 15 | dataServers := make([]string, 0) 16 | if len(locateInfo) != rs.AllShard { 17 | dataServers = heartbeat.ChooseRandomDataServers(rs.AllShard-len(locateInfo), locateInfo) 18 | } 19 | return rs.NewRSGetStream(locateInfo, dataServers, hash, size) 20 | } 21 | -------------------------------------------------------------------------------- /part5/apiServer/objects/handler.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import "net/http" 4 | 5 | func Handler(w http.ResponseWriter, r *http.Request) { 6 | method := r.Method 7 | if method == http.MethodPut { 8 | put(w, r) 9 | return 10 | } 11 | if method == http.MethodGet { 12 | get(w, r) 13 | return 14 | } 15 | if method == http.MethodDelete { 16 | del(w, r) 17 | return 18 | } 19 | w.WriteHeader(http.StatusMethodNotAllowed) 20 | 21 | } 22 | -------------------------------------------------------------------------------- /part5/apiServer/objects/put.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "storage/lib/es/es7" 7 | "storage/lib/utils" 8 | "strings" 9 | ) 10 | 11 | func put(w http.ResponseWriter, r *http.Request) { 12 | hash := utils.GetHashFromHeader(r.Header) 13 | if hash == "" { 14 | log.Println("missing object hash in digest header") 15 | w.WriteHeader(http.StatusBadRequest) 16 | return 17 | } 18 | size := utils.GetSizeFromHeader(r.Header) 19 | code, err := storeObject(r.Body, hash, size) 20 | if err != nil { 21 | log.Println(err) 22 | w.WriteHeader(code) 23 | return 24 | } 25 | if code != http.StatusOK { 26 | w.WriteHeader(code) 27 | return 28 | } 29 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 30 | err = es7.AddVersion(name, hash, size) 31 | if err != nil { 32 | log.Println(err) 33 | w.WriteHeader(http.StatusInternalServerError) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /part5/apiServer/objects/put_stream.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "storage/lib/rs" 6 | "storage/part5/apiServer/heartbeat" 7 | ) 8 | 9 | func putStream(hash string, size int64) (*rs.RSPutStream, error) { 10 | servers := heartbeat.ChooseRandomDataServers(rs.AllShard, nil) 11 | if len(servers) != rs.AllShard { 12 | return nil, fmt.Errorf("cannot find enough dataServer") 13 | } 14 | return rs.NewRSPutStream(servers, hash, size) 15 | } 16 | -------------------------------------------------------------------------------- /part5/apiServer/objects/store.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "net/url" 8 | "storage/lib/utils" 9 | "storage/part5/apiServer/locate" 10 | ) 11 | 12 | func storeObject(r io.Reader, hash string, size int64) (int, error) { 13 | if locate.Exist(url.PathEscape(hash)) { 14 | return http.StatusOK, nil 15 | } 16 | stream, err := putStream(url.PathEscape(hash), size) 17 | if err != nil { 18 | return http.StatusInternalServerError, err 19 | } 20 | reader := io.TeeReader(r, stream) 21 | caculate := utils.CalculateHash(reader) 22 | if caculate != hash { 23 | stream.Commit(false) 24 | return http.StatusBadRequest, fmt.Errorf("object hash mismatch,caculated=%s,requestd=%s", caculate, hash) 25 | } 26 | stream.Commit(true) 27 | return http.StatusOK, nil 28 | } 29 | -------------------------------------------------------------------------------- /part5/apiServer/versions/handler.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | "storage/lib/es/es7" 8 | "strings" 9 | ) 10 | 11 | func Handler(w http.ResponseWriter, r *http.Request) { 12 | method := r.Method 13 | if method != http.MethodGet { 14 | w.WriteHeader(http.StatusMethodNotAllowed) 15 | return 16 | } 17 | from := 0 18 | size := 1000 19 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 20 | for { 21 | metas, err := es7.SearchAllVersions(name, from, size) 22 | if err != nil { 23 | log.Println(err) 24 | w.WriteHeader(http.StatusInternalServerError) 25 | return 26 | } 27 | for i := range metas { 28 | body, _ := json.Marshal(metas[i]) 29 | w.Write(body) 30 | w.Write([]byte("\n")) 31 | } 32 | if len(metas) != size { 33 | return 34 | } 35 | from += size 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /part5/dataServer/dataServer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "os" 7 | "storage/part5/dataServer/heartbeat" 8 | "storage/part5/dataServer/locate" 9 | "storage/part5/dataServer/objects" 10 | "storage/part5/dataServer/temp" 11 | ) 12 | 13 | func main() { 14 | locate.CollectObjects() 15 | go heartbeat.StartHeartBeat() 16 | go locate.StartLocate() 17 | http.HandleFunc("/objects/", objects.Handler) 18 | http.HandleFunc("/temp/", temp.Handler) 19 | log.Fatal(http.ListenAndServe(os.Getenv("LISTEN_ADDRESS"), nil)) 20 | } 21 | -------------------------------------------------------------------------------- /part5/dataServer/heartbeat/heartbeat.go: -------------------------------------------------------------------------------- 1 | package heartbeat 2 | 3 | import ( 4 | "os" 5 | "storage/lib/rabbitmq" 6 | "time" 7 | ) 8 | 9 | func StartHeartBeat() { 10 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 11 | defer queue.Close() 12 | for { 13 | queue.Publish("apiServers", os.Getenv("LISTEN_ADDRESS")) 14 | time.Sleep(5 * time.Second) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /part5/dataServer/locate/locate.go: -------------------------------------------------------------------------------- 1 | package locate 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "path/filepath" 8 | "storage/lib/rabbitmq" 9 | "storage/lib/types" 10 | "strconv" 11 | "strings" 12 | "sync" 13 | ) 14 | 15 | var objects = make(map[string]int) 16 | var mutex sync.Mutex 17 | 18 | func Locate(hash string) int { 19 | mutex.Lock() 20 | id, ok := objects[hash] 21 | mutex.Unlock() 22 | if !ok { 23 | return -1 24 | } 25 | return id 26 | } 27 | 28 | func Add(hash string, id int) { 29 | mutex.Lock() 30 | objects[hash] = id 31 | mutex.Unlock() 32 | } 33 | 34 | func Del(hash string) { 35 | mutex.Lock() 36 | delete(objects, hash) 37 | mutex.Unlock() 38 | } 39 | 40 | func StartLocate() { 41 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 42 | defer queue.Close() 43 | queue.Bind("dataServers") 44 | consume := queue.Consume() 45 | for message := range consume { 46 | hash, err := strconv.Unquote(string(message.Body)) 47 | if err != nil { 48 | log.Println(err) 49 | break 50 | } 51 | fmt.Println("prepare locate:", hash) 52 | id := Locate(hash) 53 | if id != -1 { 54 | queue.Send(message.ReplyTo, types.LocateMessage{Address: os.Getenv("LISTEN_ADDRESS"), Id: id}) 55 | } 56 | } 57 | } 58 | 59 | func CollectObjects() { 60 | files, _ := filepath.Glob(os.Getenv("STORAGE_ROOT") + "/objects/*") 61 | for i := range files { 62 | file := strings.Split(filepath.Base(files[i]), ".") 63 | if len(file) != 3 { 64 | panic(files[i]) 65 | } 66 | hash := file[0] 67 | id, err := strconv.Atoi(file[1]) 68 | if err != nil { 69 | panic(err) 70 | } 71 | objects[hash] = id 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /part5/dataServer/objects/get.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | ) 7 | 8 | func get(w http.ResponseWriter, r *http.Request) { 9 | file := getFile(strings.Split(r.URL.EscapedPath(), "/")[2]) 10 | if file == "" { 11 | w.WriteHeader(http.StatusNotFound) 12 | return 13 | } 14 | sendFile(w, file) 15 | } 16 | -------------------------------------------------------------------------------- /part5/dataServer/objects/get_object.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/base64" 6 | "log" 7 | "net/url" 8 | "os" 9 | "path/filepath" 10 | "storage/part4/dataServer/locate" 11 | "strings" 12 | ) 13 | 14 | func getFile(name string) string { 15 | files, _ := filepath.Glob(os.Getenv("STORAGE_ROOT") + "/objects/" + name + ".*") 16 | if len(files) != 1 { 17 | return "" 18 | } 19 | file := files[0] 20 | hash := sha256.New() 21 | sendFile(hash, file) 22 | calculated := url.PathEscape(base64.StdEncoding.EncodeToString(hash.Sum(nil))) 23 | hash2 := strings.Split(file, ".")[2] 24 | if calculated != hash2 { 25 | log.Println("object hash mismatch,removed,", file) 26 | locate.Del(hash2) 27 | os.Remove(file) 28 | return "" 29 | } 30 | return file 31 | } 32 | -------------------------------------------------------------------------------- /part5/dataServer/objects/handler.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import "net/http" 4 | 5 | func Handler(w http.ResponseWriter, r *http.Request) { 6 | method := r.Method 7 | if method == http.MethodGet { 8 | get(w, r) 9 | return 10 | } 11 | w.WriteHeader(http.StatusMethodNotAllowed) 12 | } 13 | -------------------------------------------------------------------------------- /part5/dataServer/objects/send.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "io" 5 | "os" 6 | ) 7 | 8 | func sendFile(w io.Writer, filePath string) { 9 | file, _ := os.Open(filePath) 10 | defer file.Close() 11 | io.Copy(w, file) 12 | 13 | } 14 | -------------------------------------------------------------------------------- /part5/dataServer/temp/commit.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "net/url" 5 | "os" 6 | "storage/lib/utils" 7 | "storage/part5/dataServer/locate" 8 | "strconv" 9 | "strings" 10 | ) 11 | 12 | func (t *tempInfo) hash() string { 13 | str := strings.Split(t.Name, ".") 14 | return str[0] 15 | } 16 | 17 | func (t *tempInfo) id() int { 18 | str := strings.Split(t.Name, ".") 19 | id, _ := strconv.Atoi(str[1]) 20 | return id 21 | } 22 | 23 | func commitTempObject(dataFile string, info *tempInfo) { 24 | file, _ := os.Open(dataFile) 25 | calculated := url.PathEscape(utils.CalculateHash(file)) 26 | file.Close() 27 | os.Rename(dataFile, os.Getenv("STORAGE_ROOT")+"/objects/"+info.Name+"."+calculated) 28 | locate.Add(info.hash(), info.id()) 29 | 30 | } 31 | -------------------------------------------------------------------------------- /part5/dataServer/temp/del.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "net/http" 5 | "os" 6 | "strings" 7 | ) 8 | 9 | func del(w http.ResponseWriter, r *http.Request) { 10 | uuid := strings.Split(r.URL.EscapedPath(), "/")[2] 11 | infoFile := os.Getenv("STORAGE_ROOT") + "/temp/" + uuid 12 | dataFile := infoFile + ".dat" 13 | os.Remove(infoFile) 14 | os.Remove(dataFile) 15 | } 16 | -------------------------------------------------------------------------------- /part5/dataServer/temp/handler.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import "net/http" 4 | 5 | func Handler(w http.ResponseWriter, r *http.Request) { 6 | method := r.Method 7 | if method == http.MethodPut { 8 | put(w, r) 9 | return 10 | } 11 | if method == http.MethodPatch { 12 | patch(w, r) 13 | return 14 | } 15 | if method == http.MethodPost { 16 | post(w, r) 17 | return 18 | } 19 | if method == http.MethodDelete { 20 | del(w, r) 21 | return 22 | } 23 | w.WriteHeader(http.StatusMethodNotAllowed) 24 | } 25 | -------------------------------------------------------------------------------- /part5/dataServer/temp/patch.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | "io/ioutil" 7 | "log" 8 | "net/http" 9 | "os" 10 | "strings" 11 | ) 12 | 13 | func patch(w http.ResponseWriter, r *http.Request) { 14 | uuid := strings.Split(r.URL.EscapedPath(), "/")[2] 15 | temp_info, err := readFromFile(uuid) 16 | if err != nil { 17 | log.Println(err) 18 | w.WriteHeader(http.StatusNotFound) 19 | return 20 | } 21 | filePath := os.Getenv("STORAGE_ROOT") + "/temp/" + uuid 22 | dataFile := filePath + ".dat" 23 | file, err := os.OpenFile(dataFile, os.O_WRONLY|os.O_APPEND, 0) 24 | if err != nil { 25 | log.Println(err) 26 | w.WriteHeader(http.StatusInternalServerError) 27 | return 28 | } 29 | defer file.Close() 30 | _, err = io.Copy(file, r.Body) 31 | if err != nil { 32 | log.Println(err) 33 | w.WriteHeader(http.StatusInternalServerError) 34 | } 35 | info, err := file.Stat() 36 | if err != nil { 37 | log.Println(err) 38 | w.WriteHeader(http.StatusInternalServerError) 39 | return 40 | } 41 | actual := info.Size() 42 | if actual > temp_info.Size { 43 | os.Remove(dataFile) 44 | os.Remove(filePath) 45 | log.Println("actual size,", actual, " exceeds ", temp_info.Size) 46 | w.WriteHeader(http.StatusInternalServerError) 47 | } 48 | } 49 | 50 | func readFromFile(uuid string) (*tempInfo, error) { 51 | file, err := os.Open(os.Getenv("STORAGE_ROOT") + "/temp/" + uuid) 52 | if err != nil { 53 | return nil, err 54 | } 55 | defer file.Close() 56 | bytes, _ := ioutil.ReadAll(file) 57 | var info tempInfo 58 | json.Unmarshal(bytes, &info) 59 | return &info, nil 60 | } 61 | -------------------------------------------------------------------------------- /part5/dataServer/temp/post.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | "os" 8 | "os/exec" 9 | "strconv" 10 | "strings" 11 | ) 12 | 13 | type tempInfo struct { 14 | Uuid string 15 | Name string 16 | Size int64 17 | } 18 | 19 | func post(w http.ResponseWriter, r *http.Request) { 20 | output, _ := exec.Command("uuidgen").Output() 21 | uuid := strings.TrimSuffix(string(output), "\n") 22 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 23 | size, err := strconv.ParseInt(r.Header.Get("size"), 0, 64) 24 | if err != nil { 25 | log.Println(err) 26 | w.WriteHeader(http.StatusInternalServerError) 27 | return 28 | } 29 | temp := tempInfo{uuid, name, size} 30 | err = temp.writeToFile() 31 | if err != nil { 32 | log.Println(err) 33 | w.WriteHeader(http.StatusInternalServerError) 34 | return 35 | } 36 | os.Create(os.Getenv("STORAGE_ROOT") + "/temp/" + temp.Uuid + ".dat") 37 | w.Write([]byte(uuid)) 38 | } 39 | 40 | func (t *tempInfo) writeToFile() error { 41 | file, err := os.Create(os.Getenv("STORAGE_ROOT") + "/temp/" + t.Uuid) 42 | if err != nil { 43 | return err 44 | } 45 | defer file.Close() 46 | bytes, _ := json.Marshal(t) 47 | file.Write(bytes) 48 | return nil 49 | } 50 | -------------------------------------------------------------------------------- /part5/dataServer/temp/put.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "os" 7 | "strings" 8 | ) 9 | 10 | func put(w http.ResponseWriter, r *http.Request) { 11 | uuid := strings.Split(r.URL.EscapedPath(), "/")[2] 12 | temp_info, err := readFromFile(uuid) 13 | if err != nil { 14 | log.Println(err) 15 | w.WriteHeader(http.StatusNotFound) 16 | return 17 | } 18 | infoFile := os.Getenv("STORAGE_ROOT") + "/temp/" + uuid 19 | dataFile := infoFile + ".dat" 20 | file, err := os.Open(dataFile) 21 | if err != nil { 22 | log.Println(err) 23 | w.WriteHeader(http.StatusInternalServerError) 24 | return 25 | } 26 | defer file.Close() 27 | info, err := file.Stat() 28 | if err != nil { 29 | log.Println(err) 30 | w.WriteHeader(http.StatusInternalServerError) 31 | return 32 | } 33 | actual := info.Size() 34 | os.Remove(infoFile) 35 | if actual != temp_info.Size { 36 | os.Remove(dataFile) 37 | log.Println("actual size mismatch,expect ", temp_info.Size, " actual ", actual) 38 | w.WriteHeader(http.StatusInternalServerError) 39 | return 40 | } 41 | commitTempObject(dataFile, temp_info) 42 | } 43 | -------------------------------------------------------------------------------- /part5/test/流程.txt: -------------------------------------------------------------------------------- 1 | 1.初始化网络环境 2 | sudo ifconfig ens33:1 10.29.1.1/16 3 | sudo ifconfig ens33:2 10.29.1.2/16 4 | sudo ifconfig ens33:3 10.29.1.3/16 5 | sudo ifconfig ens33:4 10.29.1.4/16 6 | sudo ifconfig ens33:5 10.29.1.5/16 7 | sudo ifconfig ens33:6 10.29.1.6/16 8 | sudo ifconfig ens33:7 10.29.2.1/16 9 | sudo ifconfig ens33:8 10.29.2.2/16 10 | 11 | 2.初始化存储路径 12 | for i in `seq 1 6`;do mkdir -p /tmp/$i/objects; done 13 | for i in `seq 1 6`;do mkdir -p /tmp/$i/temp;done 14 | 15 | 3.启动容器 16 | docker start rabbitmq-server-storage 17 | docker start elasticsearch-storage 18 | docker start elasticsearch-head-storage 19 | 20 | 4.设置环境变量 21 | export ES_SERVER=192.168.110.134:9200 22 | export RABBITMQ_SERVER=amqp://test:test@192.168.110.134:5672 23 | 24 | 5.启动程序 25 | LISTEN_ADDRESS=10.29.1.1:8800 STORAGE_ROOT=/tmp/1 go run dataServer.go & 26 | LISTEN_ADDRESS=10.29.1.2:8800 STORAGE_ROOT=/tmp/2 go run dataServer.go & 27 | LISTEN_ADDRESS=10.29.1.3:8800 STORAGE_ROOT=/tmp/3 go run dataServer.go & 28 | LISTEN_ADDRESS=10.29.1.4:8800 STORAGE_ROOT=/tmp/4 go run dataServer.go & 29 | LISTEN_ADDRESS=10.29.1.5:8800 STORAGE_ROOT=/tmp/5 go run dataServer.go & 30 | LISTEN_ADDRESS=10.29.1.6:8800 STORAGE_ROOT=/tmp/6 go run dataServer.go & 31 | 32 | LISTEN_ADDRESS=10.29.2.1:8800 go run apiServer.go & 33 | LISTEN_ADDRESS=10.29.2.2:8800 go run apiServer.go & 34 | 35 | 6.可能遇到的状况： 36 | cpuid这个库存在错误，无法通过编译，解决方案： 37 | 先查看gopath，进入gopath 38 | 进入$GOPATH/pkg/mod/github.com/klauspost/cpuid/v2@v2.0.6 39 | 编辑cpuid.go,如果无权限可使用sudo 40 | 查看报错原因与行数： 41 | cpuid.go:443:31: invalid operation: 1 << (feat & flagMask) (shift count type FeatureID, must be unsigned integer) 42 | cpuid.go:447:29: invalid operation: 1 << (feat & flagMask) (shift count type FeatureID, must be unsigned integer) 43 | cpuid.go:454:33: invalid operation: 1 << (offset & flagMask) (shift count type FeatureID, must be unsigned integer) 44 | cpuid.go:460:17: invalid operation: 1 << (offset & flagMask) (shift count type FeatureID, must be unsigned integer) 45 | 得知这是一个语法错误，是库的问题，那么就修改库，将FeatureID转换为unsigned integer 46 | 47 | 查看定义：type FeatureID int可知其本身就是个int 48 | 编写函数转换类型 49 | func convertFeatureIDToUint(feat FeatureID)uint{ 50 | var num int=int(feat) 51 | return uint(num) 52 | } 53 | 在报错的地方使用即可 54 | 55 | 修改库以后可以顺利通过编译 56 | 57 | 7.测试 58 | 首先计算上传对象的Hash 59 | echo -n "this object will be separate to 4+2 shards" | openssl dgst -sha256 -binary | base64 60 | MBMxWHrPMsuOBaVYHkwScZQRyTRMQyiKp2oelpLZza8= 61 | 62 | 上传对象，附带头信息 63 | curl -v 10.29.2.1:8800/objects/test5 -XPUT -d"this object will be separate to 4+2 shards" -H"Digest: SHA-256=MBMxWHrPMsuOBaVYHkwScZQRyTRMQyiKp2oelpLZza8=" 64 | 上传成功返回200 65 | 66 | 查看数据分片位置 67 | ls -ltr /tmp/?/objects 68 | 可以看到共有4+2个分片，分片id从0~5 69 | 70 | 获取对象，验证是否可以正常合并出正确的数据 71 | curl 10.29.2.1:8800/objects/test5 72 | 可以获得数据 73 | 74 | 尝试定位文件 75 | curl 10.29.2.1:8800/locate/MBMxWHrPMsuOBaVYHkwScZQRyTRMQyiKp2oelpLZza8= 76 | 可以获得6个分片所在的位置信息 77 | 78 | 删除一个分片 79 | rm /tmp/1/objects/MBMxWHrPMsuOBaVYHkwScZQRyTRMQyiKp2oelpLZza8=.* 80 | 81 | 破坏一个分片 82 | echo some_data > /tmp/2/objects/MBMxWHrPMsuOBaVYHkwScZQRyTRMQyiKp2oelpLZza8=.* 83 | 84 | 查看数据分片位置 85 | ls -ltr /tmp/?/objects 86 | 可以看到共有5个分片，缺失一个分片 87 | 88 | 尝试获取对象，并在获取对象时修复分片 89 | curl 10.29.2.1:8800/objects/test5 90 | 可以看到在部分分片丢失、错误的情况下还可以得到正确的数据 91 | 92 | 查看数据分片是否被恢复 93 | ls -ltr /tmp/?/objects 94 | 可以看到分片数量恢复到6个 95 | 96 | 至此，数据校验与即时修复完成 97 | -------------------------------------------------------------------------------- /part6/apiServer/apiServer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "os" 7 | "storage/part6/apiServer/heartbeat" 8 | "storage/part6/apiServer/locate" 9 | "storage/part6/apiServer/objects" 10 | "storage/part6/apiServer/temp" 11 | "storage/part6/apiServer/versions" 12 | ) 13 | 14 | func main() { 15 | go heartbeat.ListenHeartBeat() 16 | http.HandleFunc("/objects/", objects.Handler) 17 | http.HandleFunc("/locate/", locate.Handler) 18 | http.HandleFunc("/versions/", version.Handler) 19 | http.HandleFunc("/temp/", temp.Handler) 20 | log.Fatal(http.ListenAndServe(os.Getenv("LISTEN_ADDRESS"), nil)) 21 | } 22 | -------------------------------------------------------------------------------- /part6/apiServer/heartbeat/choose.go: -------------------------------------------------------------------------------- 1 | package heartbeat 2 | 3 | import "math/rand" 4 | 5 | func ChooseRandomDataServers(n int, exclude map[int]string) (dataServers []string) { 6 | candidates := make([]string, 0) 7 | reverseExcludeMap := make(map[string]int) 8 | for id, address := range exclude { 9 | reverseExcludeMap[address] = id 10 | } 11 | servers := GetDataServers() 12 | for i := range servers { 13 | server := servers[i] 14 | _, exclude := reverseExcludeMap[server] 15 | if !exclude { 16 | candidates = append(candidates, server) 17 | } 18 | } 19 | length := len(candidates) 20 | if length < n { 21 | return 22 | } 23 | part := rand.Perm(length) 24 | for i := 0; i < n; i++ { 25 | dataServers = append(dataServers, candidates[part[i]]) 26 | } 27 | return 28 | } 29 | -------------------------------------------------------------------------------- /part6/apiServer/heartbeat/heartbeat.go: -------------------------------------------------------------------------------- 1 | package heartbeat 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "storage/lib/rabbitmq" 7 | "strconv" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | var dataServers = make(map[string]time.Time) 13 | var mutex sync.Mutex 14 | 15 | func ListenHeartBeat() { 16 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 17 | defer queue.Close() 18 | queue.Bind("apiServers") 19 | consume := queue.Consume() 20 | go removeExpiredDataServer() 21 | for message := range consume { 22 | dataServer, err := strconv.Unquote(string(message.Body)) 23 | if err != nil { 24 | fmt.Println(err) 25 | } 26 | mutex.Lock() 27 | dataServers[dataServer] = time.Now() 28 | mutex.Unlock() 29 | } 30 | } 31 | 32 | func removeExpiredDataServer() { 33 | for { 34 | time.Sleep(5 * time.Second) 35 | mutex.Lock() 36 | for server, timer := range dataServers { 37 | if timer.Add(10 * time.Second).Before(time.Now()) { 38 | delete(dataServers, server) 39 | } 40 | } 41 | mutex.Unlock() 42 | } 43 | } 44 | 45 | func GetDataServers() []string { 46 | mutex.Lock() 47 | defer mutex.Unlock() 48 | dataServer := make([]string, 0) 49 | for server, _ := range dataServers { 50 | dataServer = append(dataServer, server) 51 | } 52 | return dataServer 53 | } 54 | -------------------------------------------------------------------------------- /part6/apiServer/locate/handler.go: -------------------------------------------------------------------------------- 1 | package locate 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "strings" 7 | ) 8 | 9 | func Handler(w http.ResponseWriter, r *http.Request) { 10 | method := r.Method 11 | if method != http.MethodGet { 12 | w.WriteHeader(http.StatusMethodNotAllowed) 13 | return 14 | } 15 | info := Locate(strings.Split(r.URL.EscapedPath(), "/")[2]) 16 | if len(info) == 0 { 17 | w.WriteHeader(http.StatusNotFound) 18 | return 19 | } 20 | by, _ := json.Marshal(info) 21 | w.Write(by) 22 | } 23 | -------------------------------------------------------------------------------- /part6/apiServer/locate/locate.go: -------------------------------------------------------------------------------- 1 | package locate 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | "storage/lib/rabbitmq" 8 | "storage/lib/rs" 9 | "storage/lib/types" 10 | "time" 11 | ) 12 | 13 | func Locate(name string) (locateInfo map[int]string) { 14 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 15 | queue.Publish("dataServers", name) 16 | consume := queue.Consume() 17 | go func() { 18 | time.Sleep(time.Second) 19 | queue.Close() 20 | }() 21 | locateInfo = make(map[int]string) 22 | for i := 0; i < rs.AllShard; i++ { 23 | message := <-consume 24 | fmt.Println("receive message:", message.Body) 25 | if len(message.Body) == 0 { 26 | return 27 | } 28 | var info types.LocateMessage 29 | json.Unmarshal(message.Body, &info) 30 | fmt.Println("locate message:", info.Address) 31 | locateInfo[info.Id] = info.Address 32 | } 33 | return 34 | } 35 | 36 | func Exist(name string) bool { 37 | return len(Locate(name)) >= rs.DataShard 38 | } 39 | -------------------------------------------------------------------------------- /part6/apiServer/objects/del.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "storage/lib/es/es7" 7 | "strings" 8 | ) 9 | 10 | func del(w http.ResponseWriter, r *http.Request) { 11 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 12 | version, err := es7.SearchLatestVersion(name) 13 | if err != nil { 14 | log.Println(err) 15 | w.WriteHeader(http.StatusInternalServerError) 16 | return 17 | } 18 | err = es7.PutMetadata(name, version.Version+1, 0, "") 19 | if err != nil { 20 | log.Println(err) 21 | w.WriteHeader(http.StatusInternalServerError) 22 | return 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /part6/apiServer/objects/get.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "log" 7 | "net/http" 8 | "net/url" 9 | "storage/lib/es/es7" 10 | "storage/lib/utils" 11 | "strconv" 12 | "strings" 13 | ) 14 | 15 | func get(w http.ResponseWriter, r *http.Request) { 16 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 17 | versionId := r.URL.Query()["version"] 18 | version := 0 19 | var err error 20 | if len(versionId) != 0 { 21 | version, err = strconv.Atoi(versionId[0]) 22 | if err != nil { 23 | log.Println(err) 24 | w.WriteHeader(http.StatusBadRequest) 25 | return 26 | } 27 | } 28 | meta, err := es7.GetMetadata(name, version) 29 | if err != nil { 30 | log.Println(err) 31 | w.WriteHeader(http.StatusInternalServerError) 32 | return 33 | } 34 | if meta.Hash == "" { 35 | w.WriteHeader(http.StatusNotFound) 36 | return 37 | } 38 | hash := url.PathEscape(meta.Hash) 39 | stream, err := GetStream(hash, meta.Size) 40 | if err != nil { 41 | log.Println(err) 42 | w.WriteHeader(http.StatusNotFound) 43 | return 44 | } 45 | offset := utils.GetOffsetFromHeader(r.Header) 46 | if offset != 0 { 47 | stream.Seek(offset, io.SeekCurrent) 48 | w.Header().Set("content-range", fmt.Sprintf("bytes %d-%d/%d", offset, meta.Size-1, meta.Size)) 49 | w.WriteHeader(http.StatusPartialContent) 50 | } 51 | io.Copy(w, stream) 52 | stream.Close() 53 | } 54 | -------------------------------------------------------------------------------- /part6/apiServer/objects/get_stream.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "storage/lib/rs" 6 | "storage/part6/apiServer/heartbeat" 7 | "storage/part6/apiServer/locate" 8 | ) 9 | 10 | func GetStream(hash string, size int64) (*rs.RSGetStream, error) { 11 | locateInfo := locate.Locate(hash) 12 | if len(locateInfo) < rs.DataShard { 13 | return nil, fmt.Errorf("object %s locate fail,result %v", hash, locateInfo) 14 | } 15 | dataServers := make([]string, 0) 16 | if len(locateInfo) != rs.AllShard { 17 | dataServers = heartbeat.ChooseRandomDataServers(rs.AllShard-len(locateInfo), locateInfo) 18 | } 19 | return rs.NewRSGetStream(locateInfo, dataServers, hash, size) 20 | } 21 | -------------------------------------------------------------------------------- /part6/apiServer/objects/handler.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import "net/http" 4 | 5 | func Handler(w http.ResponseWriter, r *http.Request) { 6 | method := r.Method 7 | if method == http.MethodPost { 8 | post(w, r) 9 | return 10 | } 11 | if method == http.MethodPut { 12 | put(w, r) 13 | return 14 | } 15 | if method == http.MethodGet { 16 | get(w, r) 17 | return 18 | } 19 | if method == http.MethodDelete { 20 | del(w, r) 21 | return 22 | } 23 | w.WriteHeader(http.StatusMethodNotAllowed) 24 | } 25 | -------------------------------------------------------------------------------- /part6/apiServer/objects/post.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "net/url" 7 | "storage/lib/es/es7" 8 | "storage/lib/rs" 9 | "storage/lib/utils" 10 | "storage/part6/apiServer/heartbeat" 11 | "storage/part6/apiServer/locate" 12 | "strconv" 13 | "strings" 14 | ) 15 | 16 | func post(w http.ResponseWriter, r *http.Request) { 17 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 18 | size, err := strconv.ParseInt(r.Header.Get("size"), 0, 64) 19 | if err != nil { 20 | log.Println(err) 21 | w.WriteHeader(http.StatusForbidden) 22 | return 23 | } 24 | hash := utils.GetHashFromHeader(r.Header) 25 | if hash == "" { 26 | log.Println("missing object hash in digest header") 27 | w.WriteHeader(http.StatusBadRequest) 28 | return 29 | } 30 | if locate.Exist(url.PathEscape(hash)) { 31 | err = es7.AddVersion(name, hash, size) 32 | if err != nil { 33 | log.Println(err) 34 | w.WriteHeader(http.StatusInternalServerError) 35 | } else { 36 | w.WriteHeader(http.StatusOK) 37 | } 38 | return 39 | } 40 | dataServers := heartbeat.ChooseRandomDataServers(rs.AllShard, nil) 41 | if len(dataServers) != rs.AllShard { 42 | log.Println("cannot find enough dataServer") 43 | w.WriteHeader(http.StatusServiceUnavailable) 44 | return 45 | } 46 | stream, err := rs.NewRSResumablePutStream(dataServers, name, url.PathEscape(hash), size) 47 | if err != nil { 48 | log.Println(err) 49 | w.WriteHeader(http.StatusInternalServerError) 50 | return 51 | } 52 | w.Header().Set("location", "/temp/"+url.PathEscape(stream.ToToken())) 53 | w.WriteHeader(http.StatusCreated) 54 | } 55 | -------------------------------------------------------------------------------- /part6/apiServer/objects/put.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "storage/lib/es/es7" 7 | "storage/lib/utils" 8 | "strings" 9 | ) 10 | 11 | func put(w http.ResponseWriter, r *http.Request) { 12 | hash := utils.GetHashFromHeader(r.Header) 13 | if hash == "" { 14 | log.Println("missing object hash in digest header") 15 | w.WriteHeader(http.StatusBadRequest) 16 | return 17 | } 18 | size := utils.GetSizeFromHeader(r.Header) 19 | code, err := storeObject(r.Body, hash, size) 20 | if err != nil { 21 | log.Println(err) 22 | w.WriteHeader(code) 23 | return 24 | } 25 | if code != http.StatusOK { 26 | w.WriteHeader(code) 27 | return 28 | } 29 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 30 | err = es7.AddVersion(name, hash, size) 31 | if err != nil { 32 | log.Println(err) 33 | w.WriteHeader(http.StatusInternalServerError) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /part6/apiServer/objects/put_stream.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "storage/lib/rs" 6 | "storage/part6/apiServer/heartbeat" 7 | ) 8 | 9 | func putStream(hash string, size int64) (*rs.RSPutStream, error) { 10 | servers := heartbeat.ChooseRandomDataServers(rs.AllShard, nil) 11 | if len(servers) != rs.AllShard { 12 | return nil, fmt.Errorf("cannot find enough dataServer") 13 | } 14 | return rs.NewRSPutStream(servers, hash, size) 15 | } 16 | -------------------------------------------------------------------------------- /part6/apiServer/objects/store.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "net/url" 8 | "storage/lib/utils" 9 | "storage/part6/apiServer/locate" 10 | ) 11 | 12 | func storeObject(r io.Reader, hash string, size int64) (int, error) { 13 | if locate.Exist(url.PathEscape(hash)) { 14 | return http.StatusOK, nil 15 | } 16 | stream, err := putStream(url.PathEscape(hash), size) 17 | if err != nil { 18 | return http.StatusInternalServerError, err 19 | } 20 | reader := io.TeeReader(r, stream) 21 | caculate := utils.CalculateHash(reader) 22 | if caculate != hash { 23 | stream.Commit(false) 24 | return http.StatusBadRequest, fmt.Errorf("object hash mismatch,caculated=%s,requestd=%s", caculate, hash) 25 | } 26 | stream.Commit(true) 27 | return http.StatusOK, nil 28 | } 29 | -------------------------------------------------------------------------------- /part6/apiServer/temp/handler.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import "net/http" 4 | 5 | func Handler(w http.ResponseWriter, r *http.Request) { 6 | method := r.Method 7 | if method == http.MethodGet { 8 | head(w, r) 9 | } 10 | if method == http.MethodPut { 11 | put(w, r) 12 | return 13 | } 14 | w.WriteHeader(http.StatusMethodNotAllowed) 15 | } 16 | -------------------------------------------------------------------------------- /part6/apiServer/temp/head.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/http" 7 | "storage/lib/rs" 8 | "strings" 9 | ) 10 | 11 | func head(w http.ResponseWriter, r *http.Request) { 12 | token := strings.Split(r.URL.EscapedPath(), "/")[2] 13 | stream, err := rs.PutStreamFromToken(token) 14 | if err != nil { 15 | log.Println(err) 16 | w.WriteHeader(http.StatusForbidden) 17 | return 18 | } 19 | current := stream.CurrentSize() 20 | if current == -1 { 21 | w.WriteHeader(http.StatusNotFound) 22 | return 23 | } 24 | w.Header().Set("content-length", fmt.Sprintf("%d", current)) 25 | } 26 | -------------------------------------------------------------------------------- /part6/apiServer/temp/put.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "log" 7 | "net/http" 8 | "net/url" 9 | "storage/lib/es/es7" 10 | "storage/lib/rs" 11 | "storage/lib/utils" 12 | "storage/part4/apiServer/locate" 13 | "strings" 14 | ) 15 | 16 | func put(w http.ResponseWriter, r *http.Request) { 17 | token := strings.Split(r.URL.EscapedPath(), "/")[2] 18 | stream, err := rs.PutStreamFromToken(token) 19 | if err != nil { 20 | log.Println(err) 21 | w.WriteHeader(http.StatusForbidden) 22 | return 23 | } 24 | current := stream.CurrentSize() 25 | fmt.Println("current:", current) 26 | if current == -1 { 27 | w.WriteHeader(http.StatusNotFound) 28 | return 29 | } 30 | offset := utils.GetOffsetFromHeader(r.Header) 31 | fmt.Println("offset:", offset) 32 | if current != offset { 33 | w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) 34 | return 35 | } 36 | bytes := make([]byte, rs.BlockSie) 37 | for { 38 | content, err := io.ReadFull(r.Body, bytes) 39 | if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { 40 | log.Println(err) 41 | w.WriteHeader(http.StatusInternalServerError) 42 | return 43 | } 44 | current += int64(content) 45 | if current > stream.Size { 46 | stream.Commit(false) 47 | log.Println("resumable put exceed size") 48 | w.WriteHeader(http.StatusForbidden) 49 | } 50 | if content != rs.BlockSie && current != stream.Size { 51 | return 52 | } 53 | stream.Write(bytes[:content]) 54 | if current == stream.Size { 55 | stream.Flush() 56 | getStream, _ := rs.NewRSResumableGetStream(stream.Servers, stream.Uuids, stream.Size) 57 | hash := url.PathEscape(utils.CalculateHash(getStream)) 58 | if hash != stream.Hash { 59 | stream.Commit(false) 60 | log.Println("resumable put done but hash mismatch") 61 | w.WriteHeader(http.StatusForbidden) 62 | return 63 | } 64 | if locate.Exist(url.PathEscape(hash)) { 65 | stream.Commit(false) 66 | } else { 67 | stream.Commit(true) 68 | } 69 | err = es7.AddVersion(stream.Name, stream.Hash, stream.Size) 70 | if err != nil { 71 | log.Println(err) 72 | w.WriteHeader(http.StatusInternalServerError) 73 | } 74 | return 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /part6/apiServer/versions/handler.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | "storage/lib/es/es7" 8 | "strings" 9 | ) 10 | 11 | func Handler(w http.ResponseWriter, r *http.Request) { 12 | method := r.Method 13 | if method != http.MethodGet { 14 | w.WriteHeader(http.StatusMethodNotAllowed) 15 | return 16 | } 17 | from := 0 18 | size := 1000 19 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 20 | for { 21 | metas, err := es7.SearchAllVersions(name, from, size) 22 | if err != nil { 23 | log.Println(err) 24 | w.WriteHeader(http.StatusInternalServerError) 25 | return 26 | } 27 | for i := range metas { 28 | body, _ := json.Marshal(metas[i]) 29 | w.Write(body) 30 | w.Write([]byte("\n")) 31 | } 32 | if len(metas) != size { 33 | return 34 | } 35 | from += size 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /part6/dataServer/dataServer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "os" 7 | "storage/part6/dataServer/heartbeat" 8 | "storage/part6/dataServer/locate" 9 | "storage/part6/dataServer/objects" 10 | "storage/part6/dataServer/temp" 11 | ) 12 | 13 | func main() { 14 | locate.CollectObjects() 15 | go heartbeat.StartHeartBeat() 16 | go locate.StartLocate() 17 | http.HandleFunc("/objects/", objects.Handler) 18 | http.HandleFunc("/temp/", temp.Handler) 19 | log.Fatal(http.ListenAndServe(os.Getenv("LISTEN_ADDRESS"), nil)) 20 | } 21 | -------------------------------------------------------------------------------- /part6/dataServer/heartbeat/heartbeat.go: -------------------------------------------------------------------------------- 1 | package heartbeat 2 | 3 | import ( 4 | "os" 5 | "storage/lib/rabbitmq" 6 | "time" 7 | ) 8 | 9 | func StartHeartBeat() { 10 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 11 | defer queue.Close() 12 | for { 13 | queue.Publish("apiServers", os.Getenv("LISTEN_ADDRESS")) 14 | time.Sleep(5 * time.Second) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /part6/dataServer/locate/locate.go: -------------------------------------------------------------------------------- 1 | package locate 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "path/filepath" 8 | "storage/lib/rabbitmq" 9 | "storage/lib/types" 10 | "strconv" 11 | "strings" 12 | "sync" 13 | ) 14 | 15 | var objects = make(map[string]int) 16 | var mutex sync.Mutex 17 | 18 | func Locate(hash string) int { 19 | mutex.Lock() 20 | id, ok := objects[hash] 21 | mutex.Unlock() 22 | if !ok { 23 | return -1 24 | } 25 | return id 26 | } 27 | 28 | func Add(hash string, id int) { 29 | mutex.Lock() 30 | objects[hash] = id 31 | mutex.Unlock() 32 | } 33 | 34 | func Del(hash string) { 35 | mutex.Lock() 36 | delete(objects, hash) 37 | mutex.Unlock() 38 | } 39 | 40 | func StartLocate() { 41 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 42 | defer queue.Close() 43 | queue.Bind("dataServers") 44 | consume := queue.Consume() 45 | for message := range consume { 46 | hash, err := strconv.Unquote(string(message.Body)) 47 | if err != nil { 48 | log.Println(err) 49 | break 50 | } 51 | fmt.Println("prepare locate:", hash) 52 | id := Locate(hash) 53 | if id != -1 { 54 | queue.Send(message.ReplyTo, types.LocateMessage{Address: os.Getenv("LISTEN_ADDRESS"), Id: id}) 55 | } 56 | } 57 | } 58 | 59 | func CollectObjects() { 60 | files, _ := filepath.Glob(os.Getenv("STORAGE_ROOT") + "/objects/*") 61 | for i := range files { 62 | file := strings.Split(filepath.Base(files[i]), ".") 63 | if len(file) != 3 { 64 | panic(files[i]) 65 | } 66 | hash := file[0] 67 | id, err := strconv.Atoi(file[1]) 68 | if err != nil { 69 | panic(err) 70 | } 71 | objects[hash] = id 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /part6/dataServer/objects/get.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | ) 7 | 8 | func get(w http.ResponseWriter, r *http.Request) { 9 | file := getFile(strings.Split(r.URL.EscapedPath(), "/")[2]) 10 | if file == "" { 11 | w.WriteHeader(http.StatusNotFound) 12 | return 13 | } 14 | sendFile(w, file) 15 | } 16 | -------------------------------------------------------------------------------- /part6/dataServer/objects/get_object.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/base64" 6 | "fmt" 7 | "log" 8 | "net/url" 9 | "os" 10 | "path/filepath" 11 | "storage/part6/dataServer/locate" 12 | "strings" 13 | ) 14 | 15 | func getFile(name string) string { 16 | fmt.Println("objects try getFile:", name) 17 | files, _ := filepath.Glob(os.Getenv("STORAGE_ROOT") + "/objects/" + name + ".*") 18 | if len(files) != 1 { 19 | return "" 20 | } 21 | file := files[0] 22 | hash := sha256.New() 23 | sendFile(hash, file) 24 | calculated := url.PathEscape(base64.StdEncoding.EncodeToString(hash.Sum(nil))) 25 | hash2 := strings.Split(file, ".")[2] 26 | if calculated != hash2 { 27 | log.Println("object hash mismatch,removed,", file) 28 | locate.Del(hash2) 29 | os.Remove(file) 30 | return "" 31 | } 32 | return file 33 | } 34 | -------------------------------------------------------------------------------- /part6/dataServer/objects/handler.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import "net/http" 4 | 5 | func Handler(w http.ResponseWriter, r *http.Request) { 6 | method := r.Method 7 | if method == http.MethodGet { 8 | get(w, r) 9 | return 10 | } 11 | w.WriteHeader(http.StatusMethodNotAllowed) 12 | } 13 | -------------------------------------------------------------------------------- /part6/dataServer/objects/send.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "io" 5 | "os" 6 | ) 7 | 8 | func sendFile(w io.Writer, filePath string) { 9 | file, _ := os.Open(filePath) 10 | defer file.Close() 11 | io.Copy(w, file) 12 | 13 | } 14 | -------------------------------------------------------------------------------- /part6/dataServer/temp/commit.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "net/url" 5 | "os" 6 | "storage/lib/utils" 7 | "storage/part5/dataServer/locate" 8 | "strconv" 9 | "strings" 10 | ) 11 | 12 | func (t *tempInfo) hash() string { 13 | str := strings.Split(t.Name, ".") 14 | return str[0] 15 | } 16 | 17 | func (t *tempInfo) id() int { 18 | str := strings.Split(t.Name, ".") 19 | id, _ := strconv.Atoi(str[1]) 20 | return id 21 | } 22 | 23 | func commitTempObject(dataFile string, info *tempInfo) { 24 | file, _ := os.Open(dataFile) 25 | calculated := url.PathEscape(utils.CalculateHash(file)) 26 | file.Close() 27 | os.Rename(dataFile, os.Getenv("STORAGE_ROOT")+"/objects/"+info.Name+"."+calculated) 28 | locate.Add(info.hash(), info.id()) 29 | 30 | } 31 | -------------------------------------------------------------------------------- /part6/dataServer/temp/del.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "net/http" 5 | "os" 6 | "strings" 7 | ) 8 | 9 | func del(w http.ResponseWriter, r *http.Request) { 10 | uuid := strings.Split(r.URL.EscapedPath(), "/")[2] 11 | infoFile := os.Getenv("STORAGE_ROOT") + "/temp/" + uuid 12 | dataFile := infoFile + ".dat" 13 | os.Remove(infoFile) 14 | os.Remove(dataFile) 15 | } 16 | -------------------------------------------------------------------------------- /part6/dataServer/temp/get.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "io" 5 | "log" 6 | "net/http" 7 | "os" 8 | "strings" 9 | ) 10 | 11 | func get(w http.ResponseWriter, r *http.Request) { 12 | uuid := strings.Split(r.URL.EscapedPath(), "/")[2] 13 | file, err := os.Open(os.Getenv("STORAGE_ROOT") + "/temp/" + uuid + ".dat") 14 | if err != nil { 15 | log.Println(err) 16 | w.WriteHeader(http.StatusNotFound) 17 | return 18 | } 19 | defer file.Close() 20 | io.Copy(w, file) 21 | } 22 | -------------------------------------------------------------------------------- /part6/dataServer/temp/handler.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import "net/http" 4 | 5 | func Handler(w http.ResponseWriter, r *http.Request) { 6 | method := r.Method 7 | if method == http.MethodGet { 8 | get(w, r) 9 | return 10 | } 11 | if method == http.MethodHead { 12 | head(w, r) 13 | return 14 | } 15 | 16 | if method == http.MethodPut { 17 | put(w, r) 18 | return 19 | } 20 | if method == http.MethodPatch { 21 | patch(w, r) 22 | return 23 | } 24 | if method == http.MethodPost { 25 | post(w, r) 26 | return 27 | } 28 | if method == http.MethodDelete { 29 | del(w, r) 30 | return 31 | } 32 | w.WriteHeader(http.StatusMethodNotAllowed) 33 | } 34 | -------------------------------------------------------------------------------- /part6/dataServer/temp/head.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/http" 7 | "os" 8 | "strings" 9 | ) 10 | 11 | func head(w http.ResponseWriter, r *http.Request) { 12 | uuid := strings.Split(r.URL.EscapedPath(), "/")[2] 13 | file, err := os.Open(os.Getenv("STORAGE_ROOT") + "/temp/" + uuid + ".dat") 14 | if err != nil { 15 | log.Println(err) 16 | w.WriteHeader(http.StatusNotFound) 17 | return 18 | } 19 | defer file.Close() 20 | info, err := file.Stat() 21 | if err != nil { 22 | log.Println(err) 23 | w.WriteHeader(http.StatusInternalServerError) 24 | return 25 | } 26 | w.Header().Set("content-length", fmt.Sprintf("%d", info.Size())) 27 | } 28 | -------------------------------------------------------------------------------- /part6/dataServer/temp/patch.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | "io/ioutil" 7 | "log" 8 | "net/http" 9 | "os" 10 | "strings" 11 | ) 12 | 13 | func patch(w http.ResponseWriter, r *http.Request) { 14 | uuid := strings.Split(r.URL.EscapedPath(), "/")[2] 15 | temp_info, err := readFromFile(uuid) 16 | if err != nil { 17 | log.Println(err) 18 | w.WriteHeader(http.StatusNotFound) 19 | return 20 | } 21 | filePath := os.Getenv("STORAGE_ROOT") + "/temp/" + uuid 22 | dataFile := filePath + ".dat" 23 | file, err := os.OpenFile(dataFile, os.O_WRONLY|os.O_APPEND, 0) 24 | if err != nil { 25 | log.Println(err) 26 | w.WriteHeader(http.StatusInternalServerError) 27 | return 28 | } 29 | defer file.Close() 30 | _, err = io.Copy(file, r.Body) 31 | if err != nil { 32 | log.Println(err) 33 | w.WriteHeader(http.StatusInternalServerError) 34 | } 35 | info, err := file.Stat() 36 | if err != nil { 37 | log.Println(err) 38 | w.WriteHeader(http.StatusInternalServerError) 39 | return 40 | } 41 | actual := info.Size() 42 | if actual > temp_info.Size { 43 | os.Remove(dataFile) 44 | os.Remove(filePath) 45 | log.Println("actual size,", actual, " exceeds ", temp_info.Size) 46 | w.WriteHeader(http.StatusInternalServerError) 47 | } 48 | } 49 | 50 | func readFromFile(uuid string) (*tempInfo, error) { 51 | file, err := os.Open(os.Getenv("STORAGE_ROOT") + "/temp/" + uuid) 52 | if err != nil { 53 | return nil, err 54 | } 55 | defer file.Close() 56 | bytes, _ := ioutil.ReadAll(file) 57 | var info tempInfo 58 | json.Unmarshal(bytes, &info) 59 | return &info, nil 60 | } 61 | -------------------------------------------------------------------------------- /part6/dataServer/temp/post.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | "os" 8 | "os/exec" 9 | "strconv" 10 | "strings" 11 | ) 12 | 13 | type tempInfo struct { 14 | Uuid string 15 | Name string 16 | Size int64 17 | } 18 | 19 | func post(w http.ResponseWriter, r *http.Request) { 20 | output, _ := exec.Command("uuidgen").Output() 21 | uuid := strings.TrimSuffix(string(output), "\n") 22 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 23 | size, err := strconv.ParseInt(r.Header.Get("size"), 0, 64) 24 | if err != nil { 25 | log.Println(err) 26 | w.WriteHeader(http.StatusInternalServerError) 27 | return 28 | } 29 | temp := tempInfo{uuid, name, size} 30 | err = temp.writeToFile() 31 | if err != nil { 32 | log.Println(err) 33 | w.WriteHeader(http.StatusInternalServerError) 34 | return 35 | } 36 | os.Create(os.Getenv("STORAGE_ROOT") + "/temp/" + temp.Uuid + ".dat") 37 | w.Write([]byte(uuid)) 38 | } 39 | 40 | func (t *tempInfo) writeToFile() error { 41 | file, err := os.Create(os.Getenv("STORAGE_ROOT") + "/temp/" + t.Uuid) 42 | if err != nil { 43 | return err 44 | } 45 | defer file.Close() 46 | bytes, _ := json.Marshal(t) 47 | file.Write(bytes) 48 | return nil 49 | } 50 | -------------------------------------------------------------------------------- /part6/dataServer/temp/put.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "os" 7 | "strings" 8 | ) 9 | 10 | func put(w http.ResponseWriter, r *http.Request) { 11 | uuid := strings.Split(r.URL.EscapedPath(), "/")[2] 12 | temp_info, err := readFromFile(uuid) 13 | if err != nil { 14 | log.Println(err) 15 | w.WriteHeader(http.StatusNotFound) 16 | return 17 | } 18 | infoFile := os.Getenv("STORAGE_ROOT") + "/temp/" + uuid 19 | dataFile := infoFile + ".dat" 20 | file, err := os.Open(dataFile) 21 | if err != nil { 22 | log.Println(err) 23 | w.WriteHeader(http.StatusInternalServerError) 24 | return 25 | } 26 | defer file.Close() 27 | info, err := file.Stat() 28 | if err != nil { 29 | log.Println(err) 30 | w.WriteHeader(http.StatusInternalServerError) 31 | return 32 | } 33 | actual := info.Size() 34 | os.Remove(infoFile) 35 | if actual != temp_info.Size { 36 | os.Remove(dataFile) 37 | log.Println("actual size mismatch,expect ", temp_info.Size, " actual ", actual) 38 | w.WriteHeader(http.StatusInternalServerError) 39 | return 40 | } 41 | commitTempObject(dataFile, temp_info) 42 | } 43 | -------------------------------------------------------------------------------- /part6/test/流程.txt: -------------------------------------------------------------------------------- 1 | 1.初始化网络环境 2 | sudo ifconfig ens33:1 10.29.1.1/16 3 | sudo ifconfig ens33:2 10.29.1.2/16 4 | sudo ifconfig ens33:3 10.29.1.3/16 5 | sudo ifconfig ens33:4 10.29.1.4/16 6 | sudo ifconfig ens33:5 10.29.1.5/16 7 | sudo ifconfig ens33:6 10.29.1.6/16 8 | sudo ifconfig ens33:7 10.29.2.1/16 9 | sudo ifconfig ens33:8 10.29.2.2/16 10 | 11 | 2.初始化存储路径 12 | for i in `seq 1 6`;do mkdir -p /tmp/$i/objects; done 13 | for i in `seq 1 6`;do mkdir -p /tmp/$i/temp;done 14 | 15 | 3.启动容器 16 | docker start rabbitmq-server-storage 17 | docker start elasticsearch-storage 18 | docker start elasticsearch-head-storage 19 | 20 | 4.设置环境变量 21 | export ES_SERVER=192.168.110.134:9200 22 | export RABBITMQ_SERVER=amqp://test:test@192.168.110.134:5672 23 | 24 | 5.启动程序 25 | LISTEN_ADDRESS=10.29.1.1:8800 STORAGE_ROOT=/tmp/1 go run dataServer.go & 26 | LISTEN_ADDRESS=10.29.1.2:8800 STORAGE_ROOT=/tmp/2 go run dataServer.go & 27 | LISTEN_ADDRESS=10.29.1.3:8800 STORAGE_ROOT=/tmp/3 go run dataServer.go & 28 | LISTEN_ADDRESS=10.29.1.4:8800 STORAGE_ROOT=/tmp/4 go run dataServer.go & 29 | LISTEN_ADDRESS=10.29.1.5:8800 STORAGE_ROOT=/tmp/5 go run dataServer.go & 30 | LISTEN_ADDRESS=10.29.1.6:8800 STORAGE_ROOT=/tmp/6 go run dataServer.go & 31 | 32 | LISTEN_ADDRESS=10.29.2.1:8800 go run apiServer.go & 33 | LISTEN_ADDRESS=10.29.2.2:8800 go run apiServer.go & 34 | 35 | 6.准备要上传的文件 36 | dd if=/dev/urandom of=/tmp/file bs=1000 count=100 37 | 38 | openssl dgst -sha256 -binary /tmp/file | base64 39 | 40 | dd if=/tmp/file of=/tmp/first bs=1000 count=50 41 | 42 | dd if=/tmp/file of=/tmp/second bs=1000 skip=32 count=68 43 | 44 | 7.测试 45 | 参数：$hash$表示计算出来的hash，$token$表示上传请求获得的token地址 46 | 47 | 获取token 48 | curl -v 10.29.2.1:8800/objects/test6 -XPOST -H"Digest: SHA-256=$hash$" -H"Size: 100000" 49 | 返回201 50 | 51 | 上传第一部分 52 | curl -v -XPUT --data-binary @/tmp/first 10.29.2.1:8800/$token$ 53 | 返回continue与ok 54 | 55 | 上传第二部分 56 | curl -v -XPUT --data-binary @/tmp/second -H"range: bytes=32000-" 10.29.2.1:8800/$token$ 57 | 返回continue与ok 58 | 59 | 获取上传的数据 60 | curl 10.29.2.1:8800/objects/test6 61 | 62 | 如果Hash或者Token中有转义字符，那么不能得到结果，但从各数据节点的分片情况看，断点续传是正常的 63 | 若解决本程序的问题，将REST的RPC变成使用JSON传递参数即可 -------------------------------------------------------------------------------- /part6/test/错误修正.txt: -------------------------------------------------------------------------------- 1 | 修改utils.GetOffsetFromHeader 2 | bytePos := strings.Split(byteRange[:6], "-")改为 3 | bytePos := strings.Split(byteRange[6:], "-") -------------------------------------------------------------------------------- /part7/apiServer/apiServer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "os" 7 | "storage/part7/apiServer/heartbeat" 8 | "storage/part7/apiServer/locate" 9 | "storage/part7/apiServer/objects" 10 | "storage/part7/apiServer/temp" 11 | "storage/part7/apiServer/versions" 12 | ) 13 | 14 | func main() { 15 | go heartbeat.ListenHeartBeat() 16 | http.HandleFunc("/objects/", objects.Handler) 17 | http.HandleFunc("/locate/", locate.Handler) 18 | http.HandleFunc("/versions/", version.Handler) 19 | http.HandleFunc("/temp/", temp.Handler) 20 | log.Fatal(http.ListenAndServe(os.Getenv("LISTEN_ADDRESS"), nil)) 21 | } 22 | -------------------------------------------------------------------------------- /part7/apiServer/heartbeat/choose.go: -------------------------------------------------------------------------------- 1 | package heartbeat 2 | 3 | import "math/rand" 4 | 5 | func ChooseRandomDataServers(n int, exclude map[int]string) (dataServers []string) { 6 | candidates := make([]string, 0) 7 | reverseExcludeMap := make(map[string]int) 8 | for id, address := range exclude { 9 | reverseExcludeMap[address] = id 10 | } 11 | servers := GetDataServers() 12 | for i := range servers { 13 | server := servers[i] 14 | _, exclude := reverseExcludeMap[server] 15 | if !exclude { 16 | candidates = append(candidates, server) 17 | } 18 | } 19 | length := len(candidates) 20 | if length < n { 21 | return 22 | } 23 | part := rand.Perm(length) 24 | for i := 0; i < n; i++ { 25 | dataServers = append(dataServers, candidates[part[i]]) 26 | } 27 | return 28 | } 29 | -------------------------------------------------------------------------------- /part7/apiServer/heartbeat/heartbeat.go: -------------------------------------------------------------------------------- 1 | package heartbeat 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "storage/lib/rabbitmq" 7 | "strconv" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | var dataServers = make(map[string]time.Time) 13 | var mutex sync.Mutex 14 | 15 | func ListenHeartBeat() { 16 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 17 | defer queue.Close() 18 | queue.Bind("apiServers") 19 | consume := queue.Consume() 20 | go removeExpiredDataServer() 21 | for message := range consume { 22 | dataServer, err := strconv.Unquote(string(message.Body)) 23 | if err != nil { 24 | fmt.Println(err) 25 | } 26 | mutex.Lock() 27 | dataServers[dataServer] = time.Now() 28 | mutex.Unlock() 29 | } 30 | } 31 | 32 | func removeExpiredDataServer() { 33 | for { 34 | time.Sleep(5 * time.Second) 35 | mutex.Lock() 36 | for server, timer := range dataServers { 37 | if timer.Add(10 * time.Second).Before(time.Now()) { 38 | delete(dataServers, server) 39 | } 40 | } 41 | mutex.Unlock() 42 | } 43 | } 44 | 45 | func GetDataServers() []string { 46 | mutex.Lock() 47 | defer mutex.Unlock() 48 | dataServer := make([]string, 0) 49 | for server, _ := range dataServers { 50 | dataServer = append(dataServer, server) 51 | } 52 | return dataServer 53 | } 54 | -------------------------------------------------------------------------------- /part7/apiServer/locate/handler.go: -------------------------------------------------------------------------------- 1 | package locate 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "strings" 7 | ) 8 | 9 | func Handler(w http.ResponseWriter, r *http.Request) { 10 | method := r.Method 11 | if method != http.MethodGet { 12 | w.WriteHeader(http.StatusMethodNotAllowed) 13 | return 14 | } 15 | info := Locate(strings.Split(r.URL.EscapedPath(), "/")[2]) 16 | if len(info) == 0 { 17 | w.WriteHeader(http.StatusNotFound) 18 | return 19 | } 20 | by, _ := json.Marshal(info) 21 | w.Write(by) 22 | } 23 | -------------------------------------------------------------------------------- /part7/apiServer/locate/locate.go: -------------------------------------------------------------------------------- 1 | package locate 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | "storage/lib/rabbitmq" 8 | "storage/lib/rs" 9 | "storage/lib/types" 10 | "time" 11 | ) 12 | 13 | func Locate(name string) (locateInfo map[int]string) { 14 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 15 | queue.Publish("dataServers", name) 16 | consume := queue.Consume() 17 | go func() { 18 | time.Sleep(time.Second) 19 | queue.Close() 20 | }() 21 | locateInfo = make(map[int]string) 22 | for i := 0; i < rs.AllShard; i++ { 23 | message := <-consume 24 | fmt.Println("receive message:", message.Body) 25 | if len(message.Body) == 0 { 26 | return 27 | } 28 | var info types.LocateMessage 29 | json.Unmarshal(message.Body, &info) 30 | fmt.Println("locate message:", info.Address) 31 | locateInfo[info.Id] = info.Address 32 | } 33 | return 34 | } 35 | 36 | func Exist(name string) bool { 37 | return len(Locate(name)) >= rs.DataShard 38 | } 39 | -------------------------------------------------------------------------------- /part7/apiServer/objects/del.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "storage/lib/es/es7" 7 | "strings" 8 | ) 9 | 10 | func del(w http.ResponseWriter, r *http.Request) { 11 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 12 | version, err := es7.SearchLatestVersion(name) 13 | if err != nil { 14 | log.Println(err) 15 | w.WriteHeader(http.StatusInternalServerError) 16 | return 17 | } 18 | err = es7.PutMetadata(name, version.Version+1, 0, "") 19 | if err != nil { 20 | log.Println(err) 21 | w.WriteHeader(http.StatusInternalServerError) 22 | return 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /part7/apiServer/objects/get.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "compress/gzip" 5 | "fmt" 6 | "io" 7 | "log" 8 | "net/http" 9 | "net/url" 10 | "storage/lib/es/es7" 11 | "storage/lib/utils" 12 | "strconv" 13 | "strings" 14 | ) 15 | 16 | func get(w http.ResponseWriter, r *http.Request) { 17 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 18 | versionId := r.URL.Query()["version"] 19 | version := 0 20 | var err error 21 | if len(versionId) != 0 { 22 | version, err = strconv.Atoi(versionId[0]) 23 | if err != nil { 24 | log.Println(err) 25 | w.WriteHeader(http.StatusBadRequest) 26 | return 27 | } 28 | } 29 | meta, err := es7.GetMetadata(name, version) 30 | if err != nil { 31 | log.Println(err) 32 | w.WriteHeader(http.StatusInternalServerError) 33 | return 34 | } 35 | if meta.Hash == "" { 36 | w.WriteHeader(http.StatusNotFound) 37 | return 38 | } 39 | hash := url.PathEscape(meta.Hash) 40 | stream, err := GetStream(hash, meta.Size) 41 | if err != nil { 42 | log.Println(err) 43 | w.WriteHeader(http.StatusNotFound) 44 | return 45 | } 46 | offset := utils.GetOffsetFromHeader(r.Header) 47 | if offset != 0 { 48 | stream.Seek(offset, io.SeekCurrent) 49 | w.Header().Set("content-range", fmt.Sprintf("bytes %d-%d/%d", offset, meta.Size-1, meta.Size)) 50 | w.WriteHeader(http.StatusPartialContent) 51 | } 52 | acceptGzip := false 53 | encoding := r.Header["Accept-Encoding"] 54 | for i := range encoding { 55 | if encoding[i] == "gzip" { 56 | acceptGzip = true 57 | break 58 | } 59 | } 60 | if acceptGzip { 61 | w.Header().Set("content-encoding", "gzip") 62 | w2 := gzip.NewWriter(w) 63 | io.Copy(w2, stream) 64 | w2.Close() 65 | } else { 66 | io.Copy(w, stream) 67 | } 68 | stream.Close() 69 | } 70 | -------------------------------------------------------------------------------- /part7/apiServer/objects/get_stream.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "storage/lib/rs" 6 | "storage/part7/apiServer/heartbeat" 7 | "storage/part7/apiServer/locate" 8 | ) 9 | 10 | func GetStream(hash string, size int64) (*rs.RSGetStream, error) { 11 | locateInfo := locate.Locate(hash) 12 | if len(locateInfo) < rs.DataShard { 13 | return nil, fmt.Errorf("object %s locate fail,result %v", hash, locateInfo) 14 | } 15 | dataServers := make([]string, 0) 16 | if len(locateInfo) != rs.AllShard { 17 | dataServers = heartbeat.ChooseRandomDataServers(rs.AllShard-len(locateInfo), locateInfo) 18 | } 19 | return rs.NewRSGetStream(locateInfo, dataServers, hash, size) 20 | } 21 | -------------------------------------------------------------------------------- /part7/apiServer/objects/handler.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import "net/http" 4 | 5 | func Handler(w http.ResponseWriter, r *http.Request) { 6 | method := r.Method 7 | if method == http.MethodPost { 8 | post(w, r) 9 | return 10 | } 11 | if method == http.MethodPut { 12 | put(w, r) 13 | return 14 | } 15 | if method == http.MethodGet { 16 | get(w, r) 17 | return 18 | } 19 | if method == http.MethodDelete { 20 | del(w, r) 21 | return 22 | } 23 | w.WriteHeader(http.StatusMethodNotAllowed) 24 | } 25 | -------------------------------------------------------------------------------- /part7/apiServer/objects/post.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "net/url" 7 | "storage/lib/es/es7" 8 | "storage/lib/rs" 9 | "storage/lib/utils" 10 | "storage/part7/apiServer/heartbeat" 11 | "storage/part7/apiServer/locate" 12 | "strconv" 13 | "strings" 14 | ) 15 | 16 | func post(w http.ResponseWriter, r *http.Request) { 17 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 18 | size, err := strconv.ParseInt(r.Header.Get("size"), 0, 64) 19 | if err != nil { 20 | log.Println(err) 21 | w.WriteHeader(http.StatusForbidden) 22 | return 23 | } 24 | hash := utils.GetHashFromHeader(r.Header) 25 | if hash == "" { 26 | log.Println("missing object hash in digest header") 27 | w.WriteHeader(http.StatusBadRequest) 28 | return 29 | } 30 | if locate.Exist(url.PathEscape(hash)) { 31 | err = es7.AddVersion(name, hash, size) 32 | if err != nil { 33 | log.Println(err) 34 | w.WriteHeader(http.StatusInternalServerError) 35 | } else { 36 | w.WriteHeader(http.StatusOK) 37 | } 38 | return 39 | } 40 | dataServers := heartbeat.ChooseRandomDataServers(rs.AllShard, nil) 41 | if len(dataServers) != rs.AllShard { 42 | log.Println("cannot find enough dataServer") 43 | w.WriteHeader(http.StatusServiceUnavailable) 44 | return 45 | } 46 | stream, err := rs.NewRSResumablePutStream(dataServers, name, url.PathEscape(hash), size) 47 | if err != nil { 48 | log.Println(err) 49 | w.WriteHeader(http.StatusInternalServerError) 50 | return 51 | } 52 | w.Header().Set("location", "/temp/"+url.PathEscape(stream.ToToken())) 53 | w.WriteHeader(http.StatusCreated) 54 | } 55 | -------------------------------------------------------------------------------- /part7/apiServer/objects/put.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "storage/lib/es/es7" 7 | "storage/lib/utils" 8 | "strings" 9 | ) 10 | 11 | func put(w http.ResponseWriter, r *http.Request) { 12 | hash := utils.GetHashFromHeader(r.Header) 13 | if hash == "" { 14 | log.Println("missing object hash in digest header") 15 | w.WriteHeader(http.StatusBadRequest) 16 | return 17 | } 18 | size := utils.GetSizeFromHeader(r.Header) 19 | code, err := storeObject(r.Body, hash, size) 20 | if err != nil { 21 | log.Println(err) 22 | w.WriteHeader(code) 23 | return 24 | } 25 | if code != http.StatusOK { 26 | w.WriteHeader(code) 27 | return 28 | } 29 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 30 | err = es7.AddVersion(name, hash, size) 31 | if err != nil { 32 | log.Println(err) 33 | w.WriteHeader(http.StatusInternalServerError) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /part7/apiServer/objects/put_stream.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "storage/lib/rs" 6 | "storage/part7/apiServer/heartbeat" 7 | ) 8 | 9 | func putStream(hash string, size int64) (*rs.RSPutStream, error) { 10 | servers := heartbeat.ChooseRandomDataServers(rs.AllShard, nil) 11 | if len(servers) != rs.AllShard { 12 | return nil, fmt.Errorf("cannot find enough dataServer") 13 | } 14 | return rs.NewRSPutStream(servers, hash, size) 15 | } 16 | -------------------------------------------------------------------------------- /part7/apiServer/objects/store.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "net/url" 8 | "storage/lib/utils" 9 | "storage/part7/apiServer/locate" 10 | ) 11 | 12 | func storeObject(r io.Reader, hash string, size int64) (int, error) { 13 | if locate.Exist(url.PathEscape(hash)) { 14 | return http.StatusOK, nil 15 | } 16 | stream, err := putStream(url.PathEscape(hash), size) 17 | if err != nil { 18 | return http.StatusInternalServerError, err 19 | } 20 | reader := io.TeeReader(r, stream) 21 | caculate := utils.CalculateHash(reader) 22 | if caculate != hash { 23 | stream.Commit(false) 24 | return http.StatusBadRequest, fmt.Errorf("object hash mismatch,caculated=%s,requestd=%s", caculate, hash) 25 | } 26 | stream.Commit(true) 27 | return http.StatusOK, nil 28 | } 29 | -------------------------------------------------------------------------------- /part7/apiServer/temp/handler.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import "net/http" 4 | 5 | func Handler(w http.ResponseWriter, r *http.Request) { 6 | method := r.Method 7 | if method == http.MethodGet { 8 | head(w, r) 9 | } 10 | if method == http.MethodPut { 11 | put(w, r) 12 | return 13 | } 14 | w.WriteHeader(http.StatusMethodNotAllowed) 15 | } 16 | -------------------------------------------------------------------------------- /part7/apiServer/temp/head.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/http" 7 | "storage/lib/rs" 8 | "strings" 9 | ) 10 | 11 | func head(w http.ResponseWriter, r *http.Request) { 12 | token := strings.Split(r.URL.EscapedPath(), "/")[2] 13 | stream, err := rs.PutStreamFromToken(token) 14 | if err != nil { 15 | log.Println(err) 16 | w.WriteHeader(http.StatusForbidden) 17 | return 18 | } 19 | current := stream.CurrentSize() 20 | if current == -1 { 21 | w.WriteHeader(http.StatusNotFound) 22 | return 23 | } 24 | w.Header().Set("content-length", fmt.Sprintf("%d", current)) 25 | } 26 | -------------------------------------------------------------------------------- /part7/apiServer/temp/put.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "log" 7 | "net/http" 8 | "net/url" 9 | "storage/lib/es/es7" 10 | "storage/lib/rs" 11 | "storage/lib/utils" 12 | "storage/part7/apiServer/locate" 13 | "strings" 14 | ) 15 | 16 | func put(w http.ResponseWriter, r *http.Request) { 17 | token := strings.Split(r.URL.EscapedPath(), "/")[2] 18 | stream, err := rs.PutStreamFromToken(token) 19 | if err != nil { 20 | log.Println(err) 21 | w.WriteHeader(http.StatusForbidden) 22 | return 23 | } 24 | current := stream.CurrentSize() 25 | fmt.Println("current:", current) 26 | if current == -1 { 27 | w.WriteHeader(http.StatusNotFound) 28 | return 29 | } 30 | offset := utils.GetOffsetFromHeader(r.Header) 31 | fmt.Println("offset:", offset) 32 | if current != offset { 33 | w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) 34 | return 35 | } 36 | bytes := make([]byte, rs.BlockSie) 37 | for { 38 | content, err := io.ReadFull(r.Body, bytes) 39 | if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { 40 | log.Println(err) 41 | w.WriteHeader(http.StatusInternalServerError) 42 | return 43 | } 44 | current += int64(content) 45 | if current > stream.Size { 46 | stream.Commit(false) 47 | log.Println("resumable put exceed size") 48 | w.WriteHeader(http.StatusForbidden) 49 | } 50 | if content != rs.BlockSie && current != stream.Size { 51 | return 52 | } 53 | stream.Write(bytes[:content]) 54 | if current == stream.Size { 55 | stream.Flush() 56 | getStream, _ := rs.NewRSResumableGetStream(stream.Servers, stream.Uuids, stream.Size) 57 | hash := url.PathEscape(utils.CalculateHash(getStream)) 58 | if hash != stream.Hash { 59 | stream.Commit(false) 60 | log.Println("resumable put done but hash mismatch") 61 | w.WriteHeader(http.StatusForbidden) 62 | return 63 | } 64 | if locate.Exist(url.PathEscape(hash)) { 65 | stream.Commit(false) 66 | } else { 67 | stream.Commit(true) 68 | } 69 | err = es7.AddVersion(stream.Name, stream.Hash, stream.Size) 70 | if err != nil { 71 | log.Println(err) 72 | w.WriteHeader(http.StatusInternalServerError) 73 | } 74 | return 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /part7/apiServer/versions/handler.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | "storage/lib/es/es7" 8 | "strings" 9 | ) 10 | 11 | func Handler(w http.ResponseWriter, r *http.Request) { 12 | method := r.Method 13 | if method != http.MethodGet { 14 | w.WriteHeader(http.StatusMethodNotAllowed) 15 | return 16 | } 17 | from := 0 18 | size := 1000 19 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 20 | for { 21 | metas, err := es7.SearchAllVersions(name, from, size) 22 | if err != nil { 23 | log.Println(err) 24 | w.WriteHeader(http.StatusInternalServerError) 25 | return 26 | } 27 | for i := range metas { 28 | body, _ := json.Marshal(metas[i]) 29 | w.Write(body) 30 | w.Write([]byte("\n")) 31 | } 32 | if len(metas) != size { 33 | return 34 | } 35 | from += size 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /part7/dataServer/dataServer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "os" 7 | "storage/part7/dataServer/heartbeat" 8 | "storage/part7/dataServer/locate" 9 | "storage/part7/dataServer/objects" 10 | "storage/part7/dataServer/temp" 11 | ) 12 | 13 | func main() { 14 | locate.CollectObjects() 15 | go heartbeat.StartHeartBeat() 16 | go locate.StartLocate() 17 | http.HandleFunc("/objects/", objects.Handler) 18 | http.HandleFunc("/temp/", temp.Handler) 19 | log.Fatal(http.ListenAndServe(os.Getenv("LISTEN_ADDRESS"), nil)) 20 | } 21 | -------------------------------------------------------------------------------- /part7/dataServer/heartbeat/heartbeat.go: -------------------------------------------------------------------------------- 1 | package heartbeat 2 | 3 | import ( 4 | "os" 5 | "storage/lib/rabbitmq" 6 | "time" 7 | ) 8 | 9 | func StartHeartBeat() { 10 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 11 | defer queue.Close() 12 | for { 13 | queue.Publish("apiServers", os.Getenv("LISTEN_ADDRESS")) 14 | time.Sleep(5 * time.Second) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /part7/dataServer/locate/locate.go: -------------------------------------------------------------------------------- 1 | package locate 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "path/filepath" 8 | "storage/lib/rabbitmq" 9 | "storage/lib/types" 10 | "strconv" 11 | "strings" 12 | "sync" 13 | ) 14 | 15 | var objects = make(map[string]int) 16 | var mutex sync.Mutex 17 | 18 | func Locate(hash string) int { 19 | mutex.Lock() 20 | id, ok := objects[hash] 21 | mutex.Unlock() 22 | if !ok { 23 | return -1 24 | } 25 | return id 26 | } 27 | 28 | func Add(hash string, id int) { 29 | mutex.Lock() 30 | objects[hash] = id 31 | mutex.Unlock() 32 | } 33 | 34 | func Del(hash string) { 35 | mutex.Lock() 36 | delete(objects, hash) 37 | mutex.Unlock() 38 | } 39 | 40 | func StartLocate() { 41 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 42 | defer queue.Close() 43 | queue.Bind("dataServers") 44 | consume := queue.Consume() 45 | for message := range consume { 46 | hash, err := strconv.Unquote(string(message.Body)) 47 | if err != nil { 48 | log.Println(err) 49 | break 50 | } 51 | fmt.Println("prepare locate:", hash) 52 | id := Locate(hash) 53 | if id != -1 { 54 | queue.Send(message.ReplyTo, types.LocateMessage{Address: os.Getenv("LISTEN_ADDRESS"), Id: id}) 55 | } 56 | } 57 | } 58 | 59 | func CollectObjects() { 60 | files, _ := filepath.Glob(os.Getenv("STORAGE_ROOT") + "/objects/*") 61 | for i := range files { 62 | file := strings.Split(filepath.Base(files[i]), ".") 63 | if len(file) != 3 { 64 | panic(files[i]) 65 | } 66 | hash := file[0] 67 | id, err := strconv.Atoi(file[1]) 68 | if err != nil { 69 | panic(err) 70 | } 71 | objects[hash] = id 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /part7/dataServer/objects/get.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | ) 7 | 8 | func get(w http.ResponseWriter, r *http.Request) { 9 | file := getFile(strings.Split(r.URL.EscapedPath(), "/")[2]) 10 | if file == "" { 11 | w.WriteHeader(http.StatusNotFound) 12 | return 13 | } 14 | sendFile(w, file) 15 | } 16 | -------------------------------------------------------------------------------- /part7/dataServer/objects/get_object.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/base64" 6 | "fmt" 7 | "log" 8 | "net/url" 9 | "os" 10 | "path/filepath" 11 | "storage/part6/dataServer/locate" 12 | "strings" 13 | ) 14 | 15 | func getFile(name string) string { 16 | fmt.Println("objects try getFile:", name) 17 | files, _ := filepath.Glob(os.Getenv("STORAGE_ROOT") + "/objects/" + name + ".*") 18 | if len(files) != 1 { 19 | return "" 20 | } 21 | file := files[0] 22 | hash := sha256.New() 23 | sendFile(hash, file) 24 | calculated := url.PathEscape(base64.StdEncoding.EncodeToString(hash.Sum(nil))) 25 | hash2 := strings.Split(file, ".")[2] 26 | if calculated != hash2 { 27 | log.Println("object hash mismatch,removed,", file) 28 | locate.Del(hash2) 29 | os.Remove(file) 30 | return "" 31 | } 32 | return file 33 | } 34 | -------------------------------------------------------------------------------- /part7/dataServer/objects/handler.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import "net/http" 4 | 5 | func Handler(w http.ResponseWriter, r *http.Request) { 6 | method := r.Method 7 | if method == http.MethodGet { 8 | get(w, r) 9 | return 10 | } 11 | w.WriteHeader(http.StatusMethodNotAllowed) 12 | } 13 | -------------------------------------------------------------------------------- /part7/dataServer/objects/send.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "compress/gzip" 5 | "io" 6 | "log" 7 | "os" 8 | ) 9 | 10 | func sendFile(w io.Writer, filePath string) { 11 | file, err := os.Open(filePath) 12 | if err != nil { 13 | log.Println(err) 14 | return 15 | } 16 | defer file.Close() 17 | gzipStream, err := gzip.NewReader(file) 18 | if err != nil { 19 | log.Println(err) 20 | return 21 | } 22 | io.Copy(w, gzipStream) 23 | gzipStream.Close() 24 | } 25 | -------------------------------------------------------------------------------- /part7/dataServer/temp/commit.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "compress/gzip" 5 | "io" 6 | "net/url" 7 | "os" 8 | "storage/lib/utils" 9 | "storage/part7/dataServer/locate" 10 | "strconv" 11 | "strings" 12 | ) 13 | 14 | func (t *tempInfo) hash() string { 15 | str := strings.Split(t.Name, ".") 16 | return str[0] 17 | } 18 | 19 | func (t *tempInfo) id() int { 20 | str := strings.Split(t.Name, ".") 21 | id, _ := strconv.Atoi(str[1]) 22 | return id 23 | } 24 | 25 | func commitTempObject(dataFile string, info *tempInfo) { 26 | file, _ := os.Open(dataFile) 27 | defer file.Close() 28 | calculated := url.PathEscape(utils.CalculateHash(file)) 29 | file.Seek(0, io.SeekStart) 30 | path, _ := os.Create(os.Getenv("STORAGE_ROOT") + "/objects/" + info.Name + "." + calculated) 31 | writer := gzip.NewWriter(path) 32 | io.Copy(writer, file) 33 | writer.Close() 34 | os.Remove(dataFile) 35 | locate.Add(info.hash(), info.id()) 36 | 37 | } 38 | -------------------------------------------------------------------------------- /part7/dataServer/temp/del.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "net/http" 5 | "os" 6 | "strings" 7 | ) 8 | 9 | func del(w http.ResponseWriter, r *http.Request) { 10 | uuid := strings.Split(r.URL.EscapedPath(), "/")[2] 11 | infoFile := os.Getenv("STORAGE_ROOT") + "/temp/" + uuid 12 | dataFile := infoFile + ".dat" 13 | os.Remove(infoFile) 14 | os.Remove(dataFile) 15 | } 16 | -------------------------------------------------------------------------------- /part7/dataServer/temp/get.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "io" 5 | "log" 6 | "net/http" 7 | "os" 8 | "strings" 9 | ) 10 | 11 | func get(w http.ResponseWriter, r *http.Request) { 12 | uuid := strings.Split(r.URL.EscapedPath(), "/")[2] 13 | file, err := os.Open(os.Getenv("STORAGE_ROOT") + "/temp/" + uuid + ".dat") 14 | if err != nil { 15 | log.Println(err) 16 | w.WriteHeader(http.StatusNotFound) 17 | return 18 | } 19 | defer file.Close() 20 | io.Copy(w, file) 21 | } 22 | -------------------------------------------------------------------------------- /part7/dataServer/temp/handler.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import "net/http" 4 | 5 | func Handler(w http.ResponseWriter, r *http.Request) { 6 | method := r.Method 7 | if method == http.MethodGet { 8 | get(w, r) 9 | return 10 | } 11 | if method == http.MethodHead { 12 | head(w, r) 13 | return 14 | } 15 | 16 | if method == http.MethodPut { 17 | put(w, r) 18 | return 19 | } 20 | if method == http.MethodPatch { 21 | patch(w, r) 22 | return 23 | } 24 | if method == http.MethodPost { 25 | post(w, r) 26 | return 27 | } 28 | if method == http.MethodDelete { 29 | del(w, r) 30 | return 31 | } 32 | w.WriteHeader(http.StatusMethodNotAllowed) 33 | } 34 | -------------------------------------------------------------------------------- /part7/dataServer/temp/head.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/http" 7 | "os" 8 | "strings" 9 | ) 10 | 11 | func head(w http.ResponseWriter, r *http.Request) { 12 | uuid := strings.Split(r.URL.EscapedPath(), "/")[2] 13 | file, err := os.Open(os.Getenv("STORAGE_ROOT") + "/temp/" + uuid + ".dat") 14 | if err != nil { 15 | log.Println(err) 16 | w.WriteHeader(http.StatusNotFound) 17 | return 18 | } 19 | defer file.Close() 20 | info, err := file.Stat() 21 | if err != nil { 22 | log.Println(err) 23 | w.WriteHeader(http.StatusInternalServerError) 24 | return 25 | } 26 | w.Header().Set("content-length", fmt.Sprintf("%d", info.Size())) 27 | } 28 | -------------------------------------------------------------------------------- /part7/dataServer/temp/patch.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | "io/ioutil" 7 | "log" 8 | "net/http" 9 | "os" 10 | "strings" 11 | ) 12 | 13 | func patch(w http.ResponseWriter, r *http.Request) { 14 | uuid := strings.Split(r.URL.EscapedPath(), "/")[2] 15 | temp_info, err := readFromFile(uuid) 16 | if err != nil { 17 | log.Println(err) 18 | w.WriteHeader(http.StatusNotFound) 19 | return 20 | } 21 | filePath := os.Getenv("STORAGE_ROOT") + "/temp/" + uuid 22 | dataFile := filePath + ".dat" 23 | file, err := os.OpenFile(dataFile, os.O_WRONLY|os.O_APPEND, 0) 24 | if err != nil { 25 | log.Println(err) 26 | w.WriteHeader(http.StatusInternalServerError) 27 | return 28 | } 29 | defer file.Close() 30 | _, err = io.Copy(file, r.Body) 31 | if err != nil { 32 | log.Println(err) 33 | w.WriteHeader(http.StatusInternalServerError) 34 | } 35 | info, err := file.Stat() 36 | if err != nil { 37 | log.Println(err) 38 | w.WriteHeader(http.StatusInternalServerError) 39 | return 40 | } 41 | actual := info.Size() 42 | if actual > temp_info.Size { 43 | os.Remove(dataFile) 44 | os.Remove(filePath) 45 | log.Println("actual size,", actual, " exceeds ", temp_info.Size) 46 | w.WriteHeader(http.StatusInternalServerError) 47 | } 48 | } 49 | 50 | func readFromFile(uuid string) (*tempInfo, error) { 51 | file, err := os.Open(os.Getenv("STORAGE_ROOT") + "/temp/" + uuid) 52 | if err != nil { 53 | return nil, err 54 | } 55 | defer file.Close() 56 | bytes, _ := ioutil.ReadAll(file) 57 | var info tempInfo 58 | json.Unmarshal(bytes, &info) 59 | return &info, nil 60 | } 61 | -------------------------------------------------------------------------------- /part7/dataServer/temp/post.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | "os" 8 | "os/exec" 9 | "strconv" 10 | "strings" 11 | ) 12 | 13 | type tempInfo struct { 14 | Uuid string 15 | Name string 16 | Size int64 17 | } 18 | 19 | func post(w http.ResponseWriter, r *http.Request) { 20 | output, _ := exec.Command("uuidgen").Output() 21 | uuid := strings.TrimSuffix(string(output), "\n") 22 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 23 | size, err := strconv.ParseInt(r.Header.Get("size"), 0, 64) 24 | if err != nil { 25 | log.Println(err) 26 | w.WriteHeader(http.StatusInternalServerError) 27 | return 28 | } 29 | temp := tempInfo{uuid, name, size} 30 | err = temp.writeToFile() 31 | if err != nil { 32 | log.Println(err) 33 | w.WriteHeader(http.StatusInternalServerError) 34 | return 35 | } 36 | os.Create(os.Getenv("STORAGE_ROOT") + "/temp/" + temp.Uuid + ".dat") 37 | w.Write([]byte(uuid)) 38 | } 39 | 40 | func (t *tempInfo) writeToFile() error { 41 | file, err := os.Create(os.Getenv("STORAGE_ROOT") + "/temp/" + t.Uuid) 42 | if err != nil { 43 | return err 44 | } 45 | defer file.Close() 46 | bytes, _ := json.Marshal(t) 47 | file.Write(bytes) 48 | return nil 49 | } 50 | -------------------------------------------------------------------------------- /part7/dataServer/temp/put.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "os" 7 | "strings" 8 | ) 9 | 10 | func put(w http.ResponseWriter, r *http.Request) { 11 | uuid := strings.Split(r.URL.EscapedPath(), "/")[2] 12 | temp_info, err := readFromFile(uuid) 13 | if err != nil { 14 | log.Println(err) 15 | w.WriteHeader(http.StatusNotFound) 16 | return 17 | } 18 | infoFile := os.Getenv("STORAGE_ROOT") + "/temp/" + uuid 19 | dataFile := infoFile + ".dat" 20 | file, err := os.Open(dataFile) 21 | if err != nil { 22 | log.Println(err) 23 | w.WriteHeader(http.StatusInternalServerError) 24 | return 25 | } 26 | defer file.Close() 27 | info, err := file.Stat() 28 | if err != nil { 29 | log.Println(err) 30 | w.WriteHeader(http.StatusInternalServerError) 31 | return 32 | } 33 | actual := info.Size() 34 | os.Remove(infoFile) 35 | if actual != temp_info.Size { 36 | os.Remove(dataFile) 37 | log.Println("actual size mismatch,expect ", temp_info.Size, " actual ", actual) 38 | w.WriteHeader(http.StatusInternalServerError) 39 | return 40 | } 41 | commitTempObject(dataFile, temp_info) 42 | } 43 | -------------------------------------------------------------------------------- /part7/test/流程.txt: -------------------------------------------------------------------------------- 1 | 1.初始化网络环境 2 | sudo ifconfig ens33:1 10.29.1.1/16 3 | sudo ifconfig ens33:2 10.29.1.2/16 4 | sudo ifconfig ens33:3 10.29.1.3/16 5 | sudo ifconfig ens33:4 10.29.1.4/16 6 | sudo ifconfig ens33:5 10.29.1.5/16 7 | sudo ifconfig ens33:6 10.29.1.6/16 8 | sudo ifconfig ens33:7 10.29.2.1/16 9 | sudo ifconfig ens33:8 10.29.2.2/16 10 | 11 | 2.初始化存储路径 12 | for i in `seq 1 6`;do mkdir -p /tmp/$i/objects; done 13 | for i in `seq 1 6`;do mkdir -p /tmp/$i/temp;done 14 | 15 | 3.启动容器 16 | docker start rabbitmq-server-storage 17 | docker start elasticsearch-storage 18 | docker start elasticsearch-head-storage 19 | 20 | 4.设置环境变量 21 | export ES_SERVER=192.168.110.134:9200 22 | export RABBITMQ_SERVER=amqp://test:test@192.168.110.134:5672 23 | 24 | 5.启动程序 25 | LISTEN_ADDRESS=10.29.1.1:8800 STORAGE_ROOT=/tmp/1 go run dataServer.go & 26 | LISTEN_ADDRESS=10.29.1.2:8800 STORAGE_ROOT=/tmp/2 go run dataServer.go & 27 | LISTEN_ADDRESS=10.29.1.3:8800 STORAGE_ROOT=/tmp/3 go run dataServer.go & 28 | LISTEN_ADDRESS=10.29.1.4:8800 STORAGE_ROOT=/tmp/4 go run dataServer.go & 29 | LISTEN_ADDRESS=10.29.1.5:8800 STORAGE_ROOT=/tmp/5 go run dataServer.go & 30 | LISTEN_ADDRESS=10.29.1.6:8800 STORAGE_ROOT=/tmp/6 go run dataServer.go & 31 | 32 | LISTEN_ADDRESS=10.29.2.1:8800 go run apiServer.go & 33 | LISTEN_ADDRESS=10.29.2.2:8800 go run apiServer.go & 34 | 35 | 6.准备文件 36 | dd if=/dev/zero of=/tmp/file bs=1M count=100 37 | 38 | openssl dgst -sha256 -binary /tmp/file | base64 39 | 40 | IEkqTQ2E+L6xdn9mFiKfhdRMKCe2S9v7Jg7hL6EQng4= 41 | 42 | 7.测试 43 | 44 | 上传文件 45 | curl -v 10.29.2.1:8800/objects/test7 -XPUT --data-binary @/tmp/file -H"Digest: SHA-256=IEkqTQ2E+L6xdn9mFiKfhdRMKCe2S9v7Jg7hL6EQng4=" 46 | 47 | 获取上传的文件 48 | curl -v 10.29.2.1:8800/objects/test7 -o /tmp/output 49 | 50 | 对比两个文件 51 | diff -s /tmp/output /tmp/file，可以看到两个文件相同 52 | 53 | 查看数据分片位置 54 | ls -ltr /tmp/?/objects，看到6个分片 55 | 56 | 使用gzip下载数据 57 | curl -v 10.29.2.1:8800/objects/test7 -H"Accept-Encoding: gzip" -o /tmp/output2.gz 58 | 59 | 解压缩 60 | gunzip /tmp/output2.gz 61 | 62 | 对比两个文件 63 | diff -s /tmp/output2 /tmp/file -------------------------------------------------------------------------------- /part8/apiServer/apiServer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "os" 7 | "storage/part7/apiServer/heartbeat" 8 | "storage/part7/apiServer/locate" 9 | "storage/part7/apiServer/objects" 10 | "storage/part7/apiServer/temp" 11 | "storage/part7/apiServer/versions" 12 | ) 13 | 14 | func main() { 15 | go heartbeat.ListenHeartBeat() 16 | http.HandleFunc("/objects/", objects.Handler) 17 | http.HandleFunc("/locate/", locate.Handler) 18 | http.HandleFunc("/versions/", version.Handler) 19 | http.HandleFunc("/temp/", temp.Handler) 20 | log.Fatal(http.ListenAndServe(os.Getenv("LISTEN_ADDRESS"), nil)) 21 | } 22 | -------------------------------------------------------------------------------- /part8/apiServer/heartbeat/choose.go: -------------------------------------------------------------------------------- 1 | package heartbeat 2 | 3 | import "math/rand" 4 | 5 | func ChooseRandomDataServers(n int, exclude map[int]string) (dataServers []string) { 6 | candidates := make([]string, 0) 7 | reverseExcludeMap := make(map[string]int) 8 | for id, address := range exclude { 9 | reverseExcludeMap[address] = id 10 | } 11 | servers := GetDataServers() 12 | for i := range servers { 13 | server := servers[i] 14 | _, exclude := reverseExcludeMap[server] 15 | if !exclude { 16 | candidates = append(candidates, server) 17 | } 18 | } 19 | length := len(candidates) 20 | if length < n { 21 | return 22 | } 23 | part := rand.Perm(length) 24 | for i := 0; i < n; i++ { 25 | dataServers = append(dataServers, candidates[part[i]]) 26 | } 27 | return 28 | } 29 | -------------------------------------------------------------------------------- /part8/apiServer/heartbeat/heartbeat.go: -------------------------------------------------------------------------------- 1 | package heartbeat 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "storage/lib/rabbitmq" 7 | "strconv" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | var dataServers = make(map[string]time.Time) 13 | var mutex sync.Mutex 14 | 15 | func ListenHeartBeat() { 16 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 17 | defer queue.Close() 18 | queue.Bind("apiServers") 19 | consume := queue.Consume() 20 | go removeExpiredDataServer() 21 | for message := range consume { 22 | dataServer, err := strconv.Unquote(string(message.Body)) 23 | if err != nil { 24 | fmt.Println(err) 25 | } 26 | mutex.Lock() 27 | dataServers[dataServer] = time.Now() 28 | mutex.Unlock() 29 | } 30 | } 31 | 32 | func removeExpiredDataServer() { 33 | for { 34 | time.Sleep(5 * time.Second) 35 | mutex.Lock() 36 | for server, timer := range dataServers { 37 | if timer.Add(10 * time.Second).Before(time.Now()) { 38 | delete(dataServers, server) 39 | } 40 | } 41 | mutex.Unlock() 42 | } 43 | } 44 | 45 | func GetDataServers() []string { 46 | mutex.Lock() 47 | defer mutex.Unlock() 48 | dataServer := make([]string, 0) 49 | for server, _ := range dataServers { 50 | dataServer = append(dataServer, server) 51 | } 52 | return dataServer 53 | } 54 | -------------------------------------------------------------------------------- /part8/apiServer/locate/handler.go: -------------------------------------------------------------------------------- 1 | package locate 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "strings" 7 | ) 8 | 9 | func Handler(w http.ResponseWriter, r *http.Request) { 10 | method := r.Method 11 | if method != http.MethodGet { 12 | w.WriteHeader(http.StatusMethodNotAllowed) 13 | return 14 | } 15 | info := Locate(strings.Split(r.URL.EscapedPath(), "/")[2]) 16 | if len(info) == 0 { 17 | w.WriteHeader(http.StatusNotFound) 18 | return 19 | } 20 | by, _ := json.Marshal(info) 21 | w.Write(by) 22 | } 23 | -------------------------------------------------------------------------------- /part8/apiServer/locate/locate.go: -------------------------------------------------------------------------------- 1 | package locate 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | "storage/lib/rabbitmq" 8 | "storage/lib/rs" 9 | "storage/lib/types" 10 | "time" 11 | ) 12 | 13 | func Locate(name string) (locateInfo map[int]string) { 14 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 15 | queue.Publish("dataServers", name) 16 | consume := queue.Consume() 17 | go func() { 18 | time.Sleep(time.Second) 19 | queue.Close() 20 | }() 21 | locateInfo = make(map[int]string) 22 | for i := 0; i < rs.AllShard; i++ { 23 | message := <-consume 24 | fmt.Println("receive message:", message.Body) 25 | if len(message.Body) == 0 { 26 | return 27 | } 28 | var info types.LocateMessage 29 | json.Unmarshal(message.Body, &info) 30 | fmt.Println("locate message:", info.Address) 31 | locateInfo[info.Id] = info.Address 32 | } 33 | return 34 | } 35 | 36 | func Exist(name string) bool { 37 | return len(Locate(name)) >= rs.DataShard 38 | } 39 | -------------------------------------------------------------------------------- /part8/apiServer/objects/del.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "storage/lib/es/es7" 7 | "strings" 8 | ) 9 | 10 | func del(w http.ResponseWriter, r *http.Request) { 11 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 12 | version, err := es7.SearchLatestVersion(name) 13 | if err != nil { 14 | log.Println(err) 15 | w.WriteHeader(http.StatusInternalServerError) 16 | return 17 | } 18 | err = es7.PutMetadata(name, version.Version+1, 0, "") 19 | if err != nil { 20 | log.Println(err) 21 | w.WriteHeader(http.StatusInternalServerError) 22 | return 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /part8/apiServer/objects/get.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "compress/gzip" 5 | "fmt" 6 | "io" 7 | "log" 8 | "net/http" 9 | "net/url" 10 | "storage/lib/es/es7" 11 | "storage/lib/utils" 12 | "strconv" 13 | "strings" 14 | ) 15 | 16 | func get(w http.ResponseWriter, r *http.Request) { 17 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 18 | versionId := r.URL.Query()["version"] 19 | version := 0 20 | var err error 21 | if len(versionId) != 0 { 22 | version, err = strconv.Atoi(versionId[0]) 23 | if err != nil { 24 | log.Println(err) 25 | w.WriteHeader(http.StatusBadRequest) 26 | return 27 | } 28 | } 29 | meta, err := es7.GetMetadata(name, version) 30 | if err != nil { 31 | log.Println(err) 32 | w.WriteHeader(http.StatusInternalServerError) 33 | return 34 | } 35 | if meta.Hash == "" { 36 | w.WriteHeader(http.StatusNotFound) 37 | return 38 | } 39 | hash := url.PathEscape(meta.Hash) 40 | stream, err := GetStream(hash, meta.Size) 41 | if err != nil { 42 | log.Println(err) 43 | w.WriteHeader(http.StatusNotFound) 44 | return 45 | } 46 | offset := utils.GetOffsetFromHeader(r.Header) 47 | if offset != 0 { 48 | stream.Seek(offset, io.SeekCurrent) 49 | w.Header().Set("content-range", fmt.Sprintf("bytes %d-%d/%d", offset, meta.Size-1, meta.Size)) 50 | w.WriteHeader(http.StatusPartialContent) 51 | } 52 | acceptGzip := false 53 | encoding := r.Header["Accept-Encoding"] 54 | for i := range encoding { 55 | if encoding[i] == "gzip" { 56 | acceptGzip = true 57 | break 58 | } 59 | } 60 | if acceptGzip { 61 | w.Header().Set("content-encoding", "gzip") 62 | w2 := gzip.NewWriter(w) 63 | io.Copy(w2, stream) 64 | w2.Close() 65 | } else { 66 | io.Copy(w, stream) 67 | } 68 | stream.Close() 69 | } 70 | -------------------------------------------------------------------------------- /part8/apiServer/objects/get_stream.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "storage/lib/rs" 6 | "storage/part7/apiServer/heartbeat" 7 | "storage/part7/apiServer/locate" 8 | ) 9 | 10 | func GetStream(hash string, size int64) (*rs.RSGetStream, error) { 11 | locateInfo := locate.Locate(hash) 12 | if len(locateInfo) < rs.DataShard { 13 | return nil, fmt.Errorf("object %s locate fail,result %v", hash, locateInfo) 14 | } 15 | dataServers := make([]string, 0) 16 | if len(locateInfo) != rs.AllShard { 17 | dataServers = heartbeat.ChooseRandomDataServers(rs.AllShard-len(locateInfo), locateInfo) 18 | } 19 | return rs.NewRSGetStream(locateInfo, dataServers, hash, size) 20 | } 21 | -------------------------------------------------------------------------------- /part8/apiServer/objects/handler.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import "net/http" 4 | 5 | func Handler(w http.ResponseWriter, r *http.Request) { 6 | method := r.Method 7 | if method == http.MethodPost { 8 | post(w, r) 9 | return 10 | } 11 | if method == http.MethodPut { 12 | put(w, r) 13 | return 14 | } 15 | if method == http.MethodGet { 16 | get(w, r) 17 | return 18 | } 19 | if method == http.MethodDelete { 20 | del(w, r) 21 | return 22 | } 23 | w.WriteHeader(http.StatusMethodNotAllowed) 24 | } 25 | -------------------------------------------------------------------------------- /part8/apiServer/objects/post.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "net/url" 7 | "storage/lib/es/es7" 8 | "storage/lib/rs" 9 | "storage/lib/utils" 10 | "storage/part7/apiServer/heartbeat" 11 | "storage/part7/apiServer/locate" 12 | "strconv" 13 | "strings" 14 | ) 15 | 16 | func post(w http.ResponseWriter, r *http.Request) { 17 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 18 | size, err := strconv.ParseInt(r.Header.Get("size"), 0, 64) 19 | if err != nil { 20 | log.Println(err) 21 | w.WriteHeader(http.StatusForbidden) 22 | return 23 | } 24 | hash := utils.GetHashFromHeader(r.Header) 25 | if hash == "" { 26 | log.Println("missing object hash in digest header") 27 | w.WriteHeader(http.StatusBadRequest) 28 | return 29 | } 30 | if locate.Exist(url.PathEscape(hash)) { 31 | err = es7.AddVersion(name, hash, size) 32 | if err != nil { 33 | log.Println(err) 34 | w.WriteHeader(http.StatusInternalServerError) 35 | } else { 36 | w.WriteHeader(http.StatusOK) 37 | } 38 | return 39 | } 40 | dataServers := heartbeat.ChooseRandomDataServers(rs.AllShard, nil) 41 | if len(dataServers) != rs.AllShard { 42 | log.Println("cannot find enough dataServer") 43 | w.WriteHeader(http.StatusServiceUnavailable) 44 | return 45 | } 46 | stream, err := rs.NewRSResumablePutStream(dataServers, name, url.PathEscape(hash), size) 47 | if err != nil { 48 | log.Println(err) 49 | w.WriteHeader(http.StatusInternalServerError) 50 | return 51 | } 52 | w.Header().Set("location", "/temp/"+url.PathEscape(stream.ToToken())) 53 | w.WriteHeader(http.StatusCreated) 54 | } 55 | -------------------------------------------------------------------------------- /part8/apiServer/objects/put.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "storage/lib/es/es7" 7 | "storage/lib/utils" 8 | "strings" 9 | ) 10 | 11 | func put(w http.ResponseWriter, r *http.Request) { 12 | hash := utils.GetHashFromHeader(r.Header) 13 | if hash == "" { 14 | log.Println("missing object hash in digest header") 15 | w.WriteHeader(http.StatusBadRequest) 16 | return 17 | } 18 | size := utils.GetSizeFromHeader(r.Header) 19 | code, err := storeObject(r.Body, hash, size) 20 | if err != nil { 21 | log.Println(err) 22 | w.WriteHeader(code) 23 | return 24 | } 25 | if code != http.StatusOK { 26 | w.WriteHeader(code) 27 | return 28 | } 29 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 30 | err = es7.AddVersion(name, hash, size) 31 | if err != nil { 32 | log.Println(err) 33 | w.WriteHeader(http.StatusInternalServerError) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /part8/apiServer/objects/put_stream.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "storage/lib/rs" 6 | "storage/part7/apiServer/heartbeat" 7 | ) 8 | 9 | func putStream(hash string, size int64) (*rs.RSPutStream, error) { 10 | servers := heartbeat.ChooseRandomDataServers(rs.AllShard, nil) 11 | if len(servers) != rs.AllShard { 12 | return nil, fmt.Errorf("cannot find enough dataServer") 13 | } 14 | return rs.NewRSPutStream(servers, hash, size) 15 | } 16 | -------------------------------------------------------------------------------- /part8/apiServer/objects/store.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "net/url" 8 | "storage/lib/utils" 9 | "storage/part7/apiServer/locate" 10 | ) 11 | 12 | func storeObject(r io.Reader, hash string, size int64) (int, error) { 13 | if locate.Exist(url.PathEscape(hash)) { 14 | return http.StatusOK, nil 15 | } 16 | stream, err := putStream(url.PathEscape(hash), size) 17 | if err != nil { 18 | return http.StatusInternalServerError, err 19 | } 20 | reader := io.TeeReader(r, stream) 21 | caculate := utils.CalculateHash(reader) 22 | if caculate != hash { 23 | stream.Commit(false) 24 | return http.StatusBadRequest, fmt.Errorf("object hash mismatch,caculated=%s,requestd=%s", caculate, hash) 25 | } 26 | stream.Commit(true) 27 | return http.StatusOK, nil 28 | } 29 | -------------------------------------------------------------------------------- /part8/apiServer/temp/handler.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import "net/http" 4 | 5 | func Handler(w http.ResponseWriter, r *http.Request) { 6 | method := r.Method 7 | if method == http.MethodGet { 8 | head(w, r) 9 | } 10 | if method == http.MethodPut { 11 | put(w, r) 12 | return 13 | } 14 | w.WriteHeader(http.StatusMethodNotAllowed) 15 | } 16 | -------------------------------------------------------------------------------- /part8/apiServer/temp/head.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/http" 7 | "storage/lib/rs" 8 | "strings" 9 | ) 10 | 11 | func head(w http.ResponseWriter, r *http.Request) { 12 | token := strings.Split(r.URL.EscapedPath(), "/")[2] 13 | stream, err := rs.PutStreamFromToken(token) 14 | if err != nil { 15 | log.Println(err) 16 | w.WriteHeader(http.StatusForbidden) 17 | return 18 | } 19 | current := stream.CurrentSize() 20 | if current == -1 { 21 | w.WriteHeader(http.StatusNotFound) 22 | return 23 | } 24 | w.Header().Set("content-length", fmt.Sprintf("%d", current)) 25 | } 26 | -------------------------------------------------------------------------------- /part8/apiServer/temp/put.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "log" 7 | "net/http" 8 | "net/url" 9 | "storage/lib/es/es7" 10 | "storage/lib/rs" 11 | "storage/lib/utils" 12 | "storage/part7/apiServer/locate" 13 | "strings" 14 | ) 15 | 16 | func put(w http.ResponseWriter, r *http.Request) { 17 | token := strings.Split(r.URL.EscapedPath(), "/")[2] 18 | stream, err := rs.PutStreamFromToken(token) 19 | if err != nil { 20 | log.Println(err) 21 | w.WriteHeader(http.StatusForbidden) 22 | return 23 | } 24 | current := stream.CurrentSize() 25 | fmt.Println("current:", current) 26 | if current == -1 { 27 | w.WriteHeader(http.StatusNotFound) 28 | return 29 | } 30 | offset := utils.GetOffsetFromHeader(r.Header) 31 | fmt.Println("offset:", offset) 32 | if current != offset { 33 | w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) 34 | return 35 | } 36 | bytes := make([]byte, rs.BlockSie) 37 | for { 38 | content, err := io.ReadFull(r.Body, bytes) 39 | if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { 40 | log.Println(err) 41 | w.WriteHeader(http.StatusInternalServerError) 42 | return 43 | } 44 | current += int64(content) 45 | if current > stream.Size { 46 | stream.Commit(false) 47 | log.Println("resumable put exceed size") 48 | w.WriteHeader(http.StatusForbidden) 49 | } 50 | if content != rs.BlockSie && current != stream.Size { 51 | return 52 | } 53 | stream.Write(bytes[:content]) 54 | if current == stream.Size { 55 | stream.Flush() 56 | getStream, _ := rs.NewRSResumableGetStream(stream.Servers, stream.Uuids, stream.Size) 57 | hash := url.PathEscape(utils.CalculateHash(getStream)) 58 | if hash != stream.Hash { 59 | stream.Commit(false) 60 | log.Println("resumable put done but hash mismatch") 61 | w.WriteHeader(http.StatusForbidden) 62 | return 63 | } 64 | if locate.Exist(url.PathEscape(hash)) { 65 | stream.Commit(false) 66 | } else { 67 | stream.Commit(true) 68 | } 69 | err = es7.AddVersion(stream.Name, stream.Hash, stream.Size) 70 | if err != nil { 71 | log.Println(err) 72 | w.WriteHeader(http.StatusInternalServerError) 73 | } 74 | return 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /part8/apiServer/versions/handler.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | "storage/lib/es/es7" 8 | "strings" 9 | ) 10 | 11 | func Handler(w http.ResponseWriter, r *http.Request) { 12 | method := r.Method 13 | if method != http.MethodGet { 14 | w.WriteHeader(http.StatusMethodNotAllowed) 15 | return 16 | } 17 | from := 0 18 | size := 1000 19 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 20 | for { 21 | metas, err := es7.SearchAllVersions(name, from, size) 22 | if err != nil { 23 | log.Println(err) 24 | w.WriteHeader(http.StatusInternalServerError) 25 | return 26 | } 27 | for i := range metas { 28 | body, _ := json.Marshal(metas[i]) 29 | w.Write(body) 30 | w.Write([]byte("\n")) 31 | } 32 | if len(metas) != size { 33 | return 34 | } 35 | from += size 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /part8/dataServer/dataServer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "os" 7 | "storage/part8/dataServer/heartbeat" 8 | "storage/part8/dataServer/locate" 9 | "storage/part8/dataServer/objects" 10 | "storage/part8/dataServer/temp" 11 | ) 12 | 13 | func main() { 14 | locate.CollectObjects() 15 | go heartbeat.StartHeartBeat() 16 | go locate.StartLocate() 17 | http.HandleFunc("/objects/", objects.Handler) 18 | http.HandleFunc("/temp/", temp.Handler) 19 | log.Fatal(http.ListenAndServe(os.Getenv("LISTEN_ADDRESS"), nil)) 20 | } 21 | -------------------------------------------------------------------------------- /part8/dataServer/heartbeat/heartbeat.go: -------------------------------------------------------------------------------- 1 | package heartbeat 2 | 3 | import ( 4 | "os" 5 | "storage/lib/rabbitmq" 6 | "time" 7 | ) 8 | 9 | func StartHeartBeat() { 10 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 11 | defer queue.Close() 12 | for { 13 | queue.Publish("apiServers", os.Getenv("LISTEN_ADDRESS")) 14 | time.Sleep(5 * time.Second) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /part8/dataServer/locate/locate.go: -------------------------------------------------------------------------------- 1 | package locate 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "path/filepath" 8 | "storage/lib/rabbitmq" 9 | "storage/lib/types" 10 | "strconv" 11 | "strings" 12 | "sync" 13 | ) 14 | 15 | var objects = make(map[string]int) 16 | var mutex sync.Mutex 17 | 18 | func Locate(hash string) int { 19 | mutex.Lock() 20 | id, ok := objects[hash] 21 | mutex.Unlock() 22 | if !ok { 23 | return -1 24 | } 25 | return id 26 | } 27 | 28 | func Add(hash string, id int) { 29 | mutex.Lock() 30 | objects[hash] = id 31 | mutex.Unlock() 32 | } 33 | 34 | func Del(hash string) { 35 | mutex.Lock() 36 | delete(objects, hash) 37 | mutex.Unlock() 38 | } 39 | 40 | func StartLocate() { 41 | queue := rabbitmq.New(os.Getenv("RABBITMQ_SERVER")) 42 | defer queue.Close() 43 | queue.Bind("dataServers") 44 | consume := queue.Consume() 45 | for message := range consume { 46 | hash, err := strconv.Unquote(string(message.Body)) 47 | if err != nil { 48 | log.Println(err) 49 | break 50 | } 51 | fmt.Println("prepare locate:", hash) 52 | id := Locate(hash) 53 | if id != -1 { 54 | queue.Send(message.ReplyTo, types.LocateMessage{Address: os.Getenv("LISTEN_ADDRESS"), Id: id}) 55 | } 56 | } 57 | } 58 | 59 | func CollectObjects() { 60 | files, _ := filepath.Glob(os.Getenv("STORAGE_ROOT") + "/objects/*") 61 | for i := range files { 62 | file := strings.Split(filepath.Base(files[i]), ".") 63 | if len(file) != 3 { 64 | panic(files[i]) 65 | } 66 | hash := file[0] 67 | id, err := strconv.Atoi(file[1]) 68 | if err != nil { 69 | panic(err) 70 | } 71 | objects[hash] = id 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /part8/dataServer/objects/del.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "net/http" 5 | "os" 6 | "path/filepath" 7 | "storage/part4/dataServer/locate" 8 | "strings" 9 | ) 10 | 11 | func del(w http.ResponseWriter, r *http.Request) { 12 | hash := strings.Split(r.URL.EscapedPath(), "/")[2] 13 | files, _ := filepath.Glob(os.Getenv("STORAGE_ROOT") + "/objects/" + hash + ".*") 14 | if len(files) != 1 { 15 | return 16 | } 17 | locate.Del(hash) 18 | os.Rename(files[0], os.Getenv("STORAGE_ROOT")+"/garbage/"+filepath.Base(files[0])) 19 | } 20 | -------------------------------------------------------------------------------- /part8/dataServer/objects/get.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | ) 7 | 8 | func get(w http.ResponseWriter, r *http.Request) { 9 | file := getFile(strings.Split(r.URL.EscapedPath(), "/")[2]) 10 | if file == "" { 11 | w.WriteHeader(http.StatusNotFound) 12 | return 13 | } 14 | sendFile(w, file) 15 | } 16 | -------------------------------------------------------------------------------- /part8/dataServer/objects/get_object.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/base64" 6 | "fmt" 7 | "log" 8 | "net/url" 9 | "os" 10 | "path/filepath" 11 | "storage/part8/dataServer/locate" 12 | "strings" 13 | ) 14 | 15 | func getFile(name string) string { 16 | fmt.Println("objects try getFile:", name) 17 | files, _ := filepath.Glob(os.Getenv("STORAGE_ROOT") + "/objects/" + name + ".*") 18 | if len(files) != 1 { 19 | return "" 20 | } 21 | file := files[0] 22 | hash := sha256.New() 23 | sendFile(hash, file) 24 | calculated := url.PathEscape(base64.StdEncoding.EncodeToString(hash.Sum(nil))) 25 | hash2 := strings.Split(file, ".")[2] 26 | if calculated != hash2 { 27 | log.Println("object hash mismatch,removed,", file) 28 | locate.Del(hash2) 29 | os.Remove(file) 30 | return "" 31 | } 32 | return file 33 | } 34 | -------------------------------------------------------------------------------- /part8/dataServer/objects/handler.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import "net/http" 4 | 5 | func Handler(w http.ResponseWriter, r *http.Request) { 6 | method := r.Method 7 | if method == http.MethodGet { 8 | get(w, r) 9 | return 10 | } 11 | if method == http.MethodDelete { 12 | del(w, r) 13 | return 14 | } 15 | w.WriteHeader(http.StatusMethodNotAllowed) 16 | } 17 | -------------------------------------------------------------------------------- /part8/dataServer/objects/send.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "compress/gzip" 5 | "io" 6 | "log" 7 | "os" 8 | ) 9 | 10 | func sendFile(w io.Writer, filePath string) { 11 | file, err := os.Open(filePath) 12 | if err != nil { 13 | log.Println(err) 14 | return 15 | } 16 | defer file.Close() 17 | gzipStream, err := gzip.NewReader(file) 18 | if err != nil { 19 | log.Println(err) 20 | return 21 | } 22 | io.Copy(w, gzipStream) 23 | gzipStream.Close() 24 | } 25 | -------------------------------------------------------------------------------- /part8/dataServer/temp/commit.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "compress/gzip" 5 | "io" 6 | "net/url" 7 | "os" 8 | "storage/lib/utils" 9 | "storage/part8/dataServer/locate" 10 | "strconv" 11 | "strings" 12 | ) 13 | 14 | func (t *tempInfo) hash() string { 15 | str := strings.Split(t.Name, ".") 16 | return str[0] 17 | } 18 | 19 | func (t *tempInfo) id() int { 20 | str := strings.Split(t.Name, ".") 21 | id, _ := strconv.Atoi(str[1]) 22 | return id 23 | } 24 | 25 | func commitTempObject(dataFile string, info *tempInfo) { 26 | file, _ := os.Open(dataFile) 27 | defer file.Close() 28 | calculated := url.PathEscape(utils.CalculateHash(file)) 29 | file.Seek(0, io.SeekStart) 30 | path, _ := os.Create(os.Getenv("STORAGE_ROOT") + "/objects/" + info.Name + "." + calculated) 31 | writer := gzip.NewWriter(path) 32 | io.Copy(writer, file) 33 | writer.Close() 34 | os.Remove(dataFile) 35 | locate.Add(info.hash(), info.id()) 36 | 37 | } 38 | -------------------------------------------------------------------------------- /part8/dataServer/temp/del.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "net/http" 5 | "os" 6 | "strings" 7 | ) 8 | 9 | func del(w http.ResponseWriter, r *http.Request) { 10 | uuid := strings.Split(r.URL.EscapedPath(), "/")[2] 11 | infoFile := os.Getenv("STORAGE_ROOT") + "/temp/" + uuid 12 | dataFile := infoFile + ".dat" 13 | os.Remove(infoFile) 14 | os.Remove(dataFile) 15 | } 16 | -------------------------------------------------------------------------------- /part8/dataServer/temp/get.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "io" 5 | "log" 6 | "net/http" 7 | "os" 8 | "strings" 9 | ) 10 | 11 | func get(w http.ResponseWriter, r *http.Request) { 12 | uuid := strings.Split(r.URL.EscapedPath(), "/")[2] 13 | file, err := os.Open(os.Getenv("STORAGE_ROOT") + "/temp/" + uuid + ".dat") 14 | if err != nil { 15 | log.Println(err) 16 | w.WriteHeader(http.StatusNotFound) 17 | return 18 | } 19 | defer file.Close() 20 | io.Copy(w, file) 21 | } 22 | -------------------------------------------------------------------------------- /part8/dataServer/temp/handler.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import "net/http" 4 | 5 | func Handler(w http.ResponseWriter, r *http.Request) { 6 | method := r.Method 7 | if method == http.MethodGet { 8 | get(w, r) 9 | return 10 | } 11 | if method == http.MethodHead { 12 | head(w, r) 13 | return 14 | } 15 | 16 | if method == http.MethodPut { 17 | put(w, r) 18 | return 19 | } 20 | if method == http.MethodPatch { 21 | patch(w, r) 22 | return 23 | } 24 | if method == http.MethodPost { 25 | post(w, r) 26 | return 27 | } 28 | if method == http.MethodDelete { 29 | del(w, r) 30 | return 31 | } 32 | w.WriteHeader(http.StatusMethodNotAllowed) 33 | } 34 | -------------------------------------------------------------------------------- /part8/dataServer/temp/head.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/http" 7 | "os" 8 | "strings" 9 | ) 10 | 11 | func head(w http.ResponseWriter, r *http.Request) { 12 | uuid := strings.Split(r.URL.EscapedPath(), "/")[2] 13 | file, err := os.Open(os.Getenv("STORAGE_ROOT") + "/temp/" + uuid + ".dat") 14 | if err != nil { 15 | log.Println(err) 16 | w.WriteHeader(http.StatusNotFound) 17 | return 18 | } 19 | defer file.Close() 20 | info, err := file.Stat() 21 | if err != nil { 22 | log.Println(err) 23 | w.WriteHeader(http.StatusInternalServerError) 24 | return 25 | } 26 | w.Header().Set("content-length", fmt.Sprintf("%d", info.Size())) 27 | } 28 | -------------------------------------------------------------------------------- /part8/dataServer/temp/patch.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | "io/ioutil" 7 | "log" 8 | "net/http" 9 | "os" 10 | "strings" 11 | ) 12 | 13 | func patch(w http.ResponseWriter, r *http.Request) { 14 | uuid := strings.Split(r.URL.EscapedPath(), "/")[2] 15 | temp_info, err := readFromFile(uuid) 16 | if err != nil { 17 | log.Println(err) 18 | w.WriteHeader(http.StatusNotFound) 19 | return 20 | } 21 | filePath := os.Getenv("STORAGE_ROOT") + "/temp/" + uuid 22 | dataFile := filePath + ".dat" 23 | file, err := os.OpenFile(dataFile, os.O_WRONLY|os.O_APPEND, 0) 24 | if err != nil { 25 | log.Println(err) 26 | w.WriteHeader(http.StatusInternalServerError) 27 | return 28 | } 29 | defer file.Close() 30 | _, err = io.Copy(file, r.Body) 31 | if err != nil { 32 | log.Println(err) 33 | w.WriteHeader(http.StatusInternalServerError) 34 | } 35 | info, err := file.Stat() 36 | if err != nil { 37 | log.Println(err) 38 | w.WriteHeader(http.StatusInternalServerError) 39 | return 40 | } 41 | actual := info.Size() 42 | if actual > temp_info.Size { 43 | os.Remove(dataFile) 44 | os.Remove(filePath) 45 | log.Println("actual size,", actual, " exceeds ", temp_info.Size) 46 | w.WriteHeader(http.StatusInternalServerError) 47 | } 48 | } 49 | 50 | func readFromFile(uuid string) (*tempInfo, error) { 51 | file, err := os.Open(os.Getenv("STORAGE_ROOT") + "/temp/" + uuid) 52 | if err != nil { 53 | return nil, err 54 | } 55 | defer file.Close() 56 | bytes, _ := ioutil.ReadAll(file) 57 | var info tempInfo 58 | json.Unmarshal(bytes, &info) 59 | return &info, nil 60 | } 61 | -------------------------------------------------------------------------------- /part8/dataServer/temp/post.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | "os" 8 | "os/exec" 9 | "strconv" 10 | "strings" 11 | ) 12 | 13 | type tempInfo struct { 14 | Uuid string 15 | Name string 16 | Size int64 17 | } 18 | 19 | func post(w http.ResponseWriter, r *http.Request) { 20 | output, _ := exec.Command("uuidgen").Output() 21 | uuid := strings.TrimSuffix(string(output), "\n") 22 | name := strings.Split(r.URL.EscapedPath(), "/")[2] 23 | size, err := strconv.ParseInt(r.Header.Get("size"), 0, 64) 24 | if err != nil { 25 | log.Println(err) 26 | w.WriteHeader(http.StatusInternalServerError) 27 | return 28 | } 29 | temp := tempInfo{uuid, name, size} 30 | err = temp.writeToFile() 31 | if err != nil { 32 | log.Println(err) 33 | w.WriteHeader(http.StatusInternalServerError) 34 | return 35 | } 36 | os.Create(os.Getenv("STORAGE_ROOT") + "/temp/" + temp.Uuid + ".dat") 37 | w.Write([]byte(uuid)) 38 | } 39 | 40 | func (t *tempInfo) writeToFile() error { 41 | file, err := os.Create(os.Getenv("STORAGE_ROOT") + "/temp/" + t.Uuid) 42 | if err != nil { 43 | return err 44 | } 45 | defer file.Close() 46 | bytes, _ := json.Marshal(t) 47 | file.Write(bytes) 48 | return nil 49 | } 50 | -------------------------------------------------------------------------------- /part8/dataServer/temp/put.go: -------------------------------------------------------------------------------- 1 | package temp 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "os" 7 | "strings" 8 | ) 9 | 10 | func put(w http.ResponseWriter, r *http.Request) { 11 | uuid := strings.Split(r.URL.EscapedPath(), "/")[2] 12 | temp_info, err := readFromFile(uuid) 13 | if err != nil { 14 | log.Println(err) 15 | w.WriteHeader(http.StatusNotFound) 16 | return 17 | } 18 | infoFile := os.Getenv("STORAGE_ROOT") + "/temp/" + uuid 19 | dataFile := infoFile + ".dat" 20 | file, err := os.Open(dataFile) 21 | if err != nil { 22 | log.Println(err) 23 | w.WriteHeader(http.StatusInternalServerError) 24 | return 25 | } 26 | defer file.Close() 27 | info, err := file.Stat() 28 | if err != nil { 29 | log.Println(err) 30 | w.WriteHeader(http.StatusInternalServerError) 31 | return 32 | } 33 | actual := info.Size() 34 | os.Remove(infoFile) 35 | if actual != temp_info.Size { 36 | os.Remove(dataFile) 37 | log.Println("actual size mismatch,expect ", temp_info.Size, " actual ", actual) 38 | w.WriteHeader(http.StatusInternalServerError) 39 | return 40 | } 41 | commitTempObject(dataFile, temp_info) 42 | } 43 | -------------------------------------------------------------------------------- /part8/deleteOldMetadata/deleteOldMetaData.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "storage/lib/es/es7" 6 | ) 7 | 8 | const MinVersionCount = 5 9 | 10 | func main() { 11 | buckets, err := es7.SearchVersionStatus(MinVersionCount + 1) 12 | if err != nil { 13 | log.Println(err) 14 | return 15 | } 16 | for i := range buckets { 17 | bucket := buckets[i] 18 | for v := 0; v < bucket.DocCount-MinVersionCount; v++ { 19 | es7.DelMetadata(bucket.Key, v+int(bucket.MinVersion.Value)) 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /part8/deleteOrphanObject/deleteOrphanObject.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "os" 7 | "path/filepath" 8 | "storage/lib/es/es7" 9 | "strings" 10 | ) 11 | 12 | func main() { 13 | files, _ := filepath.Glob(os.Getenv("STORAGE_ROOT") + "/objects/*") 14 | for i := range files { 15 | hash := strings.Split(filepath.Base(files[i]), ".")[0] 16 | hashInMetaData, err := es7.HasHash(hash) 17 | if err != nil { 18 | log.Println(err) 19 | return 20 | } 21 | if !hashInMetaData { 22 | del(hash) 23 | } 24 | } 25 | } 26 | 27 | func del(hash string) { 28 | log.Println("delete:", hash) 29 | url := "http://" + os.Getenv("LISTEN_ADDRESS") + "/objects/" + hash 30 | request, _ := http.NewRequest("DELETE", url, nil) 31 | client := http.Client{} 32 | client.Do(request) 33 | } 34 | -------------------------------------------------------------------------------- /part8/objectScanner/objectScanner.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "path/filepath" 7 | "storage/lib/es/es7" 8 | "storage/lib/utils" 9 | "storage/part8/apiServer/objects" 10 | "strings" 11 | ) 12 | 13 | func main() { 14 | files, _ := filepath.Glob(os.Getenv("STORAGE_ROOT") + "/objects/*") 15 | for i := range files { 16 | hash := strings.Split(filepath.Base(files[i]), ".")[0] 17 | verify(hash) 18 | } 19 | } 20 | 21 | func verify(hash string) { 22 | log.Println("verify:", hash) 23 | size, err := es7.SearchHashSize(hash) 24 | if err != nil { 25 | log.Println(err) 26 | return 27 | } 28 | stream, err := objects.GetStream(hash, size) 29 | if err != nil { 30 | log.Println(err) 31 | return 32 | } 33 | document := utils.CalculateHash(stream) 34 | if document != hash { 35 | log.Printf("object hash mismatch,calculated=%s,request=%s", document, hash) 36 | } 37 | stream.Close() 38 | } 39 | -------------------------------------------------------------------------------- /part8/test/流程.txt: -------------------------------------------------------------------------------- 1 | 1.初始化网络环境 2 | sudo ifconfig ens33:1 10.29.1.1/16 3 | sudo ifconfig ens33:2 10.29.1.2/16 4 | sudo ifconfig ens33:3 10.29.1.3/16 5 | sudo ifconfig ens33:4 10.29.1.4/16 6 | sudo ifconfig ens33:5 10.29.1.5/16 7 | sudo ifconfig ens33:6 10.29.1.6/16 8 | sudo ifconfig ens33:7 10.29.2.1/16 9 | sudo ifconfig ens33:8 10.29.2.2/16 10 | 11 | 2.初始化存储路径 12 | for i in `seq 1 6`;do mkdir -p /tmp/$i/objects; done 13 | for i in `seq 1 6`;do mkdir -p /tmp/$i/temp;done 14 | for i in `seq 1 6`;do mkdir -p /tmp/$i/garbage;done 15 | 16 | 3.启动容器 17 | docker start rabbitmq-server-storage 18 | docker start elasticsearch-storage 19 | docker start elasticsearch-head-storage 20 | 21 | 4.设置环境变量 22 | export ES_SERVER=192.168.110.134:9200 23 | export RABBITMQ_SERVER=amqp://test:test@192.168.110.134:5672 24 | 25 | 5.启动程序 26 | LISTEN_ADDRESS=10.29.1.1:8800 STORAGE_ROOT=/tmp/1 go run dataServer.go & 27 | LISTEN_ADDRESS=10.29.1.2:8800 STORAGE_ROOT=/tmp/2 go run dataServer.go & 28 | LISTEN_ADDRESS=10.29.1.3:8800 STORAGE_ROOT=/tmp/3 go run dataServer.go & 29 | LISTEN_ADDRESS=10.29.1.4:8800 STORAGE_ROOT=/tmp/4 go run dataServer.go & 30 | LISTEN_ADDRESS=10.29.1.5:8800 STORAGE_ROOT=/tmp/5 go run dataServer.go & 31 | LISTEN_ADDRESS=10.29.1.6:8800 STORAGE_ROOT=/tmp/6 go run dataServer.go & 32 | 33 | LISTEN_ADDRESS=10.29.2.1:8800 go run apiServer.go & 34 | LISTEN_ADDRESS=10.29.2.2:8800 go run apiServer.go & 35 | 36 | 6.测试 37 | 创建六个版本的对象并上传 38 | echo -n "this is object test8 version 1" | openssl dgst -sha256 -binary | base64 39 | curl 10.29.2.1:8800/objects/test8 -XPUT -d"" -H"Digest: SHA-256=" 40 | 41 | 获取对象版本信息与最新版本 42 | curl 10.29.2.1:8800/versions/test8 43 | curl 10.29.2.1:8800/objects/test8 44 | 45 | 检查数据分片 46 | ls -l /tmp/?/objects 47 | 每个目录下都有6个分片 48 | 49 | 清除旧版本数据 50 | go run deleteOldMetaData.go 51 | curl 10.29.2.1:8800/versions/test8 52 | 检查版本信息，发现只有五个版本 53 | 54 | 再次检查数据分片 55 | ls -l /tmp/?/objects 56 | 删除无效的对象分片 57 | STORAGE_ROOT=/tmp/1 LISTEN_ADDRESS=10.29.1.1:8800 go run deleteOrphanObject.go 58 | STORAGE_ROOT=/tmp/2 LISTEN_ADDRESS=10.29.1.2:8800 go run deleteOrphanObject.go 59 | STORAGE_ROOT=/tmp/3 LISTEN_ADDRESS=10.29.1.3:8800 go run deleteOrphanObject.go 60 | STORAGE_ROOT=/tmp/4 LISTEN_ADDRESS=10.29.1.4:8800 go run deleteOrphanObject.go 61 | STORAGE_ROOT=/tmp/5 LISTEN_ADDRESS=10.29.1.5:8800 go run deleteOrphanObject.go 62 | STORAGE_ROOT=/tmp/6 LISTEN_ADDRESS=10.29.1.6:8800 go run deleteOrphanObject.go 63 | 检查数据分片 64 | ls -l /tmp/?/objects 65 | 检查回收站 66 | ls -l /tmp/?/garbage 67 | 删除失败的原因是没有建立对应的目录，重新删除 68 | 69 | 建立目录后重新删除，发现object里面的数据分片变成5片，garbage里面出现删除的数据分片 70 | 71 | 移除一个数据分片，破坏一个数据分片 72 | ULH%2FlCWEp7WpC%2FRE%2FrSob%2FCSXwVZMDBzzVSUn268KO0= 73 | rm /tmp/1/objects/ULH%2FlCWEp7WpC%2FRE%2FrSob%2FCSXwVZMDBzzVSUn268KO0=.* 74 | echo some_garbage > /tmp/2/objects/ULH%2FlCWEp7WpC%2FRE%2FrSob%2FCSXwVZMDBzzVSUn268KO0=.* 75 | 76 | 检查数据分片 77 | 一个分片已经被删除 78 | 79 | 扫描对象列表，修复对象 80 | STORAGE_ROOT=/tmp/2 go run objectScanner.go 81 | 检查分片 82 | 分片恢复正常 --------------------------------------------------------------------------------