├── .gitignore ├── LICENSE ├── UDPush └── UDPush.go ├── backup └── old_client.go ├── boxtools ├── boxtools.go └── boxtools_test.go ├── client ├── api │ ├── api.go │ ├── api.go.orig │ └── api_test.go ├── client.go ├── client_test.go ├── shelltest.sh └── watcher │ └── watcher.go ├── cmd └── gobox-client │ ├── .gitignore │ ├── fs_watcher.go │ ├── main.go │ ├── muxer.go │ ├── spec_test.go │ ├── udp_watcher.go │ └── watcher_test.go ├── fsnotifytest └── fsnotifytest.go ├── main_test.go ├── readme.md ├── server ├── api │ ├── api.go │ └── api_test.go ├── model │ └── model.go ├── s3 │ ├── s3.go │ └── s3_test.go ├── server.go └── templates │ ├── index.html │ └── upload.html └── structs └── structs.go /.gitignore: -------------------------------------------------------------------------------- 1 | tmp/ 2 | localMeta.gob 3 | test/ 4 | 5 | .*.*.sw* 6 | .GoBox 7 | .bashrc 8 | sandbox 9 | 10 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 11 | *.o 12 | *.a 13 | *.so 14 | 15 | # Folders 16 | _obj 17 | _test 18 | 19 | # Architecture specific extensions/prefixes 20 | *.[568vq] 21 | [568vq].out 22 | 23 | *.cgo1.go 24 | *.cgo2.c 25 | _cgo_defun.c 26 | _cgo_gotypes.go 27 | _cgo_export.* 28 | 29 | _testmain.go 30 | 31 | *.exe 32 | *.test 33 | *.prof 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Jon Poler, Marín Alcaraz Córdova and Max McDonnell 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /UDPush/UDPush.go: -------------------------------------------------------------------------------- 1 | /* 2 | ** Observer.go 3 | ** Author: Marin Alcaraz 4 | ** Mail 5 | ** Started on Mon Feb 09 14:36:00 2015 Marin Alcaraz 6 | ** Last update Thu Feb 19 15:45:08 2015 Marin Alcaraz 7 | */ 8 | 9 | package UDPush 10 | 11 | import ( 12 | "fmt" 13 | "net" 14 | ) 15 | 16 | // Constants 17 | 18 | // MAX NUMBER PER WATCHER ENGINE 19 | 20 | const maxClients = 10 21 | 22 | type update struct { 23 | status bool 24 | ownerID int 25 | } 26 | 27 | // NotificationEngine interface for the notification system 28 | // Defines the requirements to create a gobox 29 | type NotificationEngine interface { 30 | Initialize(id string) 31 | Attach(Watcher) error 32 | Detach(Watcher) bool 33 | Notify() 34 | } 35 | 36 | // WatcherEngine Interface for watcher (Observer) system 37 | // Defines the requirements to create a gobox 38 | // notification watcher. 39 | type WatcherEngine interface { 40 | Update() 41 | } 42 | 43 | //Pusher struct that satisfies the NotificationEngine interface 44 | type Pusher struct { 45 | ServerID string 46 | BindedTo uint 47 | Watchers map[string]Watcher 48 | Pending bool 49 | } 50 | 51 | // Watcher Struct that satisfies the WatcherEngine 52 | // This type requires an auth mecanism in order 53 | // to work in a safe way 54 | type Watcher struct { 55 | OwnerID int 56 | ClientID int 57 | SessionKey string 58 | Action bool 59 | Connection net.Conn 60 | } 61 | 62 | // Methods for struct to satisfy the notificationEngine interface 63 | 64 | //Initialize is a 'constructor' for the pusher struct 65 | func (e *Pusher) Initialize(id string) { 66 | e.ServerID = id 67 | e.Watchers = make(map[string]Watcher, maxClients) 68 | } 69 | 70 | //Attach Add a new Watcher to the notification slice 71 | func (e *Pusher) Attach(w Watcher) (err error) { 72 | //Check if Watchers is full 73 | if len(e.Watchers) == maxClients { 74 | return fmt.Errorf("[!] Error: Not enough space for new client") 75 | } 76 | //Check if element already exists 77 | if _, k := e.Watchers[w.SessionKey]; k { 78 | return fmt.Errorf("[!] Warning: client already monitored, skipping addition") 79 | } 80 | fmt.Println("Client registered ", w.SessionKey) 81 | e.Watchers[w.SessionKey] = w 82 | return nil 83 | } 84 | 85 | //Detach Remove a watcher from the notification slice 86 | func (e *Pusher) Detach(w Watcher) (err error) { 87 | //Check if element already exists 88 | if item, ok := e.Watchers[w.SessionKey]; ok { 89 | item.Connection.Close() 90 | delete(e.Watchers, w.SessionKey) 91 | return nil 92 | } 93 | return fmt.Errorf("[!] Error: client doesn't exist") 94 | } 95 | 96 | //Notify Tell the watcher {clientID} to update 97 | func (e *Pusher) Notify(sessionkey string) { 98 | for _, k := range e.Watchers { 99 | if k.SessionKey != sessionkey { 100 | k.Update() 101 | k.Action = true 102 | } 103 | } 104 | } 105 | 106 | //Utilities for pusher 107 | 108 | //ShowWatchers Print current watchers in pusher 109 | func (e *Pusher) ShowWatchers() { 110 | for _, k := range e.Watchers { 111 | fmt.Println("Watcher: ", k) 112 | } 113 | } 114 | 115 | // Methods for satisfiying the interface 116 | 117 | // Update Get update from pusher... Golint forces me to do this 118 | // http://tinyurl.com/lhzjvmm 119 | func (w *Watcher) Update() { 120 | w.Action = true 121 | fits := w.SessionKey[:2] 122 | fmt.Println("[!] Attempting to update watcher: %s", fits) 123 | writen, err := w.Connection.Write([]byte("Y")) 124 | if writen != len([]byte("Y")) { 125 | fmt.Println("[!]Error writting: unable to write") 126 | } 127 | if err != nil { 128 | fmt.Printf("%s", err) 129 | } 130 | 131 | } 132 | 133 | //Network related methods 134 | 135 | func getPendingUpdates() update { 136 | return update{status: true, 137 | ownerID: 1} 138 | } 139 | 140 | //InitUDPush 'constructs' the UDP notification engine 141 | //The e on the reciever stands for event 142 | func (e *Pusher) InitUDPush() error { 143 | //Initialize the map 144 | e.Watchers = make(map[string]Watcher, maxClients) 145 | connectionString := fmt.Sprintf("%s:%d", e.ServerID, e.BindedTo) 146 | ln, err := net.Listen("tcp", connectionString) 147 | if err != nil { 148 | return fmt.Errorf("Error at initUDPush: %s", err) 149 | } 150 | fmt.Println("[+] UDP Listening on: ", connectionString) 151 | for { 152 | conn, err := ln.Accept() 153 | fmt.Println("Host connected: ", ln.Addr()) 154 | if err != nil { 155 | return fmt.Errorf("Error at initUDPush: %s", err) 156 | } 157 | session := make([]byte, 64) 158 | conn.Read(session) 159 | e.Attach(Watcher{ 160 | SessionKey: string(session), 161 | Connection: conn, 162 | }) 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /backup/old_client.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "crypto/sha256" 6 | "encoding/hex" 7 | "encoding/json" 8 | "fmt" 9 | "io" 10 | "io/ioutil" 11 | "log" 12 | "mime/multipart" 13 | "net/http" 14 | "os" 15 | "path/filepath" 16 | "strconv" 17 | "time" 18 | 19 | "github.com/golangbox/gobox/structs" 20 | ) 21 | 22 | const ( 23 | goBoxDirectory = "." 24 | goBoxDataDirectory = ".GoBox" 25 | serverEndpoint = "http://requestb.in/1mv9fa41" 26 | // serverEndpoint = "http://www.google.com" 27 | filesystemCheckFrequency = 5 28 | ) 29 | 30 | func main() { 31 | fmt.Println("Running GoBox...") 32 | 33 | createGoBoxLocalDirectory() 34 | 35 | go monitorFiles() 36 | 37 | select {} 38 | 39 | } 40 | 41 | func createGoBoxLocalDirectory() { 42 | 43 | _, err := os.Stat(goBoxDataDirectory) 44 | if err != nil { 45 | fmt.Println(err) 46 | fmt.Println("Making directory") 47 | err := os.Mkdir(goBoxDataDirectory, 0777) 48 | if err != nil { 49 | log.Fatal(err) 50 | } 51 | } 52 | } 53 | 54 | func monitorFiles() { 55 | 56 | var newfileInfos = make(map[string]structs.File) 57 | var fileInfos = make(map[string]structs.File) 58 | 59 | data, err := ioutil.ReadFile(goBoxDataDirectory + "/data") 60 | if err != nil { 61 | fmt.Println(err) 62 | } 63 | if data != nil { 64 | err = json.Unmarshal(data, &fileInfos) 65 | if err != nil { 66 | fmt.Println(err) 67 | } 68 | } 69 | for _ = range time.Tick(filesystemCheckFrequency * time.Second) { 70 | fmt.Println("Checking to see if there are any filesystem changes.") 71 | newfileInfos, err = findFilesInDirectory(goBoxDirectory) 72 | if err != nil { 73 | fmt.Println(err) 74 | } 75 | err := compareFileInfos(fileInfos, newfileInfos) 76 | if err != nil { 77 | fmt.Println(err) 78 | fmt.Println("Error uploading file changes, skipping this upload cycle") 79 | } else { 80 | err = writefileInfosToLocalFile(newfileInfos) 81 | if err != nil { 82 | fmt.Println(err) 83 | } else { 84 | // fmt.Println("Saved data locally") 85 | } 86 | fileInfos = newfileInfos 87 | } 88 | } 89 | } 90 | 91 | func writefileInfosToLocalFile(fileInfos map[string]structs.File) error { 92 | jsonBytes, err := json.Marshal(fileInfos) 93 | if err != nil { 94 | return err 95 | } 96 | 97 | err = ioutil.WriteFile(goBoxDataDirectory+"/data", jsonBytes, 0644) 98 | return err 99 | } 100 | 101 | func handleFileChange(isCreate bool, file structs.File) (err error) { 102 | infoToSend := structs.FileAction{ 103 | IsCreate: isCreate, 104 | File: file, 105 | } 106 | resp, err := uploadMetadata(infoToSend) 107 | if err != nil { 108 | return 109 | } 110 | if resp.StatusCode != 200 { 111 | err = fmt.Errorf("Error uploading metadata, status code: %d", resp.StatusCode) 112 | return 113 | } 114 | 115 | return 116 | } 117 | 118 | func uploadMetadata(uploadinfo structs.FileAction) (resp *http.Response, err error) { 119 | fmt.Println("Uploading metadata: (" + uploadinfo.File.Name + ")") 120 | jsonBytes, err := json.Marshal(uploadinfo) 121 | if err != nil { 122 | return 123 | } 124 | resp, err = http.Post(serverEndpoint+"/meta", "application/json", bytes.NewBuffer(jsonBytes)) 125 | fmt.Println(resp) 126 | if err != nil { 127 | return resp, err 128 | } 129 | if uploadinfo.IsCreate == true { 130 | var contents []byte 131 | contents, err = ioutil.ReadAll(resp.Body) 132 | err := resp.Body.Close() 133 | if err != nil { 134 | return resp, err 135 | } 136 | fmt.Println(string(contents)) 137 | if string(contents) == "true" { 138 | resp, err = uploadFile(uploadinfo.File.Path) 139 | if err != nil { 140 | return resp, err 141 | } 142 | } else { 143 | fmt.Println("no need for upload") 144 | } 145 | } 146 | if err != nil { 147 | return 148 | } 149 | return 150 | } 151 | 152 | func compareFileInfos(fileInfos map[string]structs.File, 153 | newfileInfos map[string]structs.File) (err error) { 154 | 155 | for key, value := range newfileInfos { 156 | // http://stackoverflow.com/questions/2050391/how-to-test-key-existence-in-a-map 157 | if _, exists := fileInfos[key]; !exists { 158 | fmt.Println("Need to Upload:", value.Name) 159 | err = handleFileChange(true, value) 160 | if err != nil { 161 | return err 162 | } 163 | } 164 | } 165 | for key, value := range fileInfos { 166 | if _, exists := newfileInfos[key]; !exists { 167 | fmt.Println("Need to delete:", key) 168 | err = handleFileChange(false, value) 169 | if err != nil { 170 | return err 171 | } 172 | } 173 | } 174 | return 175 | } 176 | 177 | // http://matt.aimonetti.net/posts/2013/07/01/golang-multipart-file-upload-example/ 178 | func newfileUploadRequest(uri string, params map[string]string, 179 | paramName, path string) (*http.Request, error) { 180 | file, err := os.Open(path) 181 | if err != nil { 182 | return nil, err 183 | } 184 | defer file.Close() 185 | 186 | body := &bytes.Buffer{} 187 | writer := multipart.NewWriter(body) 188 | part, err := writer.CreateFormFile(paramName, filepath.Base(path)) 189 | if err != nil { 190 | return nil, err 191 | } 192 | _, err = io.Copy(part, file) 193 | 194 | for key, val := range params { 195 | _ = writer.WriteField(key, val) 196 | } 197 | err = writer.Close() 198 | if err != nil { 199 | return nil, err 200 | } 201 | 202 | req, err := http.NewRequest("POST", uri, body) 203 | req.Header.Add("Content-Type", writer.FormDataContentType()) 204 | 205 | return req, err 206 | } 207 | 208 | func getSha256FromFilename(filename string) (sha256String string, 209 | err error) { 210 | file, err := ioutil.ReadFile(filename) 211 | if err != nil { 212 | return "", fmt.Errorf("Error reading file for sha256: %s", err) 213 | } 214 | h := sha256.New() 215 | _, err = h.Write(file) 216 | if err != nil { 217 | return "", fmt.Errorf("Error writing file to hash for sha256: %s", err) 218 | } 219 | byteString := h.Sum(nil) 220 | 221 | sha256String = hex.EncodeToString(byteString) 222 | 223 | return sha256String, nil 224 | } 225 | 226 | func uploadFile(name string) (*http.Response, error) { 227 | // filename := "main.go" 228 | file, _ := os.Stat(name) 229 | s, err := getSha256FromFilename(name) 230 | if err != nil { 231 | fmt.Println(err) 232 | } 233 | 234 | size := strconv.Itoa(int(file.Size())) 235 | extraParams := map[string]string{ 236 | "Name": file.Name(), 237 | "Hash": s, 238 | "Size": size, 239 | } 240 | 241 | // http://requestb.in/19w82ne1 242 | //"http://10.0.7.205:4242/upload" 243 | request, err := newfileUploadRequest("http://10.0.7.205:4242/upload", extraParams, "FileName", name) 244 | if err != nil { 245 | return nil, err 246 | } 247 | client := &http.Client{} 248 | // resp, err := client.Do(request) 249 | return client.Do(request) 250 | } 251 | 252 | func mapKeyValue(path string, sha256 string) (key string) { 253 | return path + "-" + sha256 254 | } 255 | 256 | func findFilesInDirectoryHelper(directory string, 257 | fileInfos map[string]structs.File) (outputfileInfos map[string]structs.File, err error) { 258 | files, err := ioutil.ReadDir(directory) 259 | if err != nil { 260 | return nil, fmt.Errorf("Unable to read directory: %s", err) 261 | } 262 | 263 | for _, f := range files { 264 | name := f.Name() 265 | path := directory + "/" + name 266 | 267 | // fmt.Println("Scanning file: " + path) 268 | if f.IsDir() { 269 | if f.Name() != goBoxDataDirectory { 270 | fileInfos, err = findFilesInDirectoryHelper(path, fileInfos) 271 | } 272 | } else { 273 | sha256, err := getSha256FromFilename(path) 274 | if err != nil { 275 | fmt.Println(err) 276 | } 277 | 278 | fileInfos[mapKeyValue(path, sha256)] = structs.File{ 279 | Name: name, 280 | Hash: sha256, 281 | Size: f.Size(), 282 | Path: path, 283 | Modified: f.ModTime(), 284 | } 285 | } 286 | } 287 | 288 | return fileInfos, err 289 | } 290 | 291 | func findFilesInDirectory(directory string) (outputfileInfos map[string]structs.File, err error) { 292 | emptyfileInfos := make(map[string]structs.File) 293 | return findFilesInDirectoryHelper(directory, emptyfileInfos) 294 | } 295 | -------------------------------------------------------------------------------- /boxtools/boxtools.go: -------------------------------------------------------------------------------- 1 | package boxtools 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/hex" 6 | "encoding/json" 7 | "errors" 8 | "fmt" 9 | "io" 10 | "io/ioutil" 11 | "log" 12 | "math/rand" 13 | "os" 14 | "path/filepath" 15 | "time" 16 | 17 | "github.com/golangbox/gobox/server/model" 18 | "github.com/golangbox/gobox/structs" 19 | "github.com/jinzhu/gorm" 20 | 21 | "code.google.com/p/go.crypto/bcrypt" 22 | ) 23 | 24 | var salt []byte 25 | 26 | func init() { 27 | f, err := os.Open("/dev/random") 28 | if err != nil { 29 | panic("Unable to open /dev/random") 30 | } 31 | salt = make([]byte, 8) 32 | n, err := f.Read(salt) 33 | if n != 8 || err != nil { 34 | panic("Couldn't read from dev/random") 35 | } 36 | f.Close() 37 | 38 | rand.Seed(time.Now().Unix()) 39 | 40 | } 41 | 42 | func RandomString(n int) string { 43 | s := "" 44 | for i := 0; i < n; i++ { 45 | s += string('a' + rand.Intn(26)) 46 | } 47 | return s 48 | } 49 | 50 | func GenerateFilePathFromRoot(root string, depth int) string { 51 | s := root 52 | for i := 0; i < depth; i++ { 53 | 54 | s += string(filepath.Separator) 55 | s += RandomString(rand.Intn(10) + 1) 56 | } 57 | return s 58 | } 59 | 60 | func GenerateRandomBytes(n int) []byte { 61 | slc := make([]byte, n) 62 | for i := 0; i < n; i++ { 63 | slc[i] = 'a' + byte(rand.Intn(26)) 64 | } 65 | return slc 66 | } 67 | 68 | func WriteRandomFileContent(path string) (err error) { 69 | bytes := GenerateRandomBytes(rand.Intn(1000)) 70 | f, err := os.Create(path) 71 | if err != nil { 72 | return 73 | } 74 | _, err = f.Write(bytes) 75 | return 76 | } 77 | 78 | func sortPair(a int, b int) (less int, more int) { 79 | if a < b { 80 | return a, b 81 | } 82 | return b, a 83 | } 84 | 85 | func changeRandomPartOfFile(path string) (err error) { 86 | fi, err := os.Stat(path) 87 | if err != nil { 88 | return 89 | } 90 | if fi.IsDir() { 91 | return errors.New("Cannot alter a dir") 92 | } 93 | flen := fi.Size() 94 | f, err := os.Open(path) 95 | if err != nil { 96 | return 97 | } 98 | bytes := make([]byte, flen) 99 | n, err := f.Read(bytes) 100 | if err != nil { 101 | return 102 | } 103 | start, stop := sortPair(rand.Intn(n), rand.Intn(n)) 104 | for i := start; i < stop; i++ { 105 | bytes[i] = (((bytes[i] - 'a') + 1) % 26) + 'a' 106 | } 107 | _, err = f.Write(bytes) 108 | return 109 | } 110 | 111 | func SimulateFilesystemChanges(dir string, inserts int, alters int, deletes int) { 112 | counter := 0 113 | inserted := make([]string, 0) 114 | for i := 0; i < inserts; i++ { 115 | path := GenerateFilePathFromRoot(dir, 1) 116 | err := WriteRandomFileContent(path) 117 | if err == nil { 118 | counter++ 119 | inserted = append(inserted, path) 120 | } 121 | } 122 | fmt.Println(len(inserted)) 123 | for i := 0; i < alters; i++ { 124 | changeRandomPartOfFile(inserted[i]) 125 | counter++ 126 | } 127 | for i := 0; i < deletes; i++ { 128 | err := os.Remove(inserted[i]) 129 | counter++ 130 | if err != nil { 131 | fmt.Println("Failed to remove a file in simulateFilesystemChanges") 132 | } 133 | } 134 | fmt.Println(counter) 135 | } 136 | 137 | func CleanTestFolder(path string, ignores map[string]bool, rootDir bool) (err error) { 138 | ignored := 0 139 | infos, err := ioutil.ReadDir(path) 140 | if err != nil { 141 | return 142 | } 143 | for _, fi := range infos { 144 | if fi.Name() == "." || fi.Name() == ".." { 145 | continue 146 | } 147 | 148 | fileName := filepath.Join(path, fi.Name()) 149 | fmt.Println(fileName) 150 | if _, found := ignores[fi.Name()]; found { 151 | ignored++ 152 | continue 153 | } 154 | if fi.IsDir() { 155 | err = CleanTestFolder(fileName, ignores, false) 156 | if err != nil { 157 | return 158 | } 159 | err = os.Remove(fileName) 160 | continue 161 | } 162 | err = os.Remove(fileName) 163 | if err != nil { 164 | return 165 | } 166 | } 167 | if ignored != 0 && !rootDir { 168 | err = errors.New("Cannot delete a non-empty folder") 169 | } 170 | return 171 | } 172 | 173 | func GenerateRandomFile(user_id int) (file structs.File, err error) { 174 | path := GenerateFilePathFromRoot("/path", 3) 175 | basename := filepath.Base(path) 176 | h, err := GenerateRandomSha256() 177 | if err != nil { 178 | return file, err 179 | } 180 | return structs.File{ 181 | UserId: int64(user_id), 182 | Name: basename, 183 | Hash: h, 184 | Size: rand.Int63(), 185 | Modified: time.Now(), 186 | Path: path, 187 | CreatedAt: time.Now(), 188 | }, err 189 | 190 | } 191 | 192 | func GenerateRandomFileAction(client_id int, user_id int, isCreate bool) (fileAction structs.FileAction, err error) { 193 | file, err := GenerateRandomFile(user_id) 194 | if err != nil { 195 | return fileAction, err 196 | } 197 | _ = isCreate 198 | return structs.FileAction{ 199 | ClientId: int64(client_id), 200 | IsCreate: true, 201 | CreatedAt: time.Now(), 202 | File: file, 203 | }, err 204 | } 205 | 206 | func GenerateSliceOfRandomFileActions(user_id int, clients int, actions int) (fileActions []structs.FileAction, err error) { 207 | fileActions = make([]structs.FileAction, actions) 208 | for i := 0; i < int(actions); i++ { 209 | isAction := rand.Intn(2) == 1 210 | action, err := GenerateRandomFileAction(rand.Intn(clients)+1, user_id, isAction) 211 | if err != nil { 212 | return fileActions, err 213 | } 214 | fileActions[i] = action 215 | } 216 | return fileActions, err 217 | } 218 | 219 | func GenerateNoisyAndNonNoisyFileActions(user_id int, clients int, totalNonNoisyActions int, createPairs int) (nonNoisyActions []structs.FileAction, 220 | noisyActions []structs.FileAction, err error) { 221 | numNoisyActions := totalNonNoisyActions + createPairs 222 | nonNoisyActions, err = GenerateSliceOfRandomFileActions(user_id, clients, totalNonNoisyActions) 223 | if err != nil { 224 | return 225 | } 226 | noisyActions = make([]structs.FileAction, numNoisyActions) 227 | copy(noisyActions, nonNoisyActions) 228 | offset := len(nonNoisyActions) 229 | for i := 0; i < createPairs; i++ { 230 | new := nonNoisyActions[i] 231 | new.IsCreate = !new.IsCreate 232 | noisyActions[i+offset] = new 233 | } 234 | return 235 | 236 | } 237 | 238 | func NewUser(email string, password string) (user structs.User, err error) { 239 | hash, err := hashPassword(password) 240 | if err != nil { 241 | return 242 | } 243 | user = structs.User{ 244 | Email: email, 245 | HashedPassword: hash, 246 | } 247 | query := model.DB.Create(&user) 248 | if query.Error != nil { 249 | return user, query.Error 250 | } 251 | client, err := NewClient(user, "Server", true) 252 | _ = client 253 | if err != nil { 254 | return 255 | } 256 | return 257 | } 258 | 259 | func NewClient(user structs.User, name string, isServer bool) (client structs.Client, err error) { 260 | // calculate key if we need a key? 261 | newKey, err := GenerateRandomSha256() 262 | if err != nil { 263 | return 264 | } 265 | client = structs.Client{ 266 | UserId: user.Id, 267 | SessionKey: newKey, 268 | IsServer: isServer, 269 | Name: name, 270 | } 271 | query := model.DB.Create(&client) 272 | if query.Error != nil { 273 | return client, query.Error 274 | } 275 | return 276 | } 277 | 278 | func GenerateRandomSha256() (s string, err error) { 279 | h := sha256.New() 280 | h.Write(salt) 281 | io.WriteString(h, time.Now().String()) 282 | bytes := h.Sum(nil) 283 | s = hex.EncodeToString(bytes) 284 | return s, err 285 | } 286 | 287 | func ValidateUserPassword(email, password string) (user structs.User, err error) { 288 | model.DB.Where("email = ?", email).First(&user) 289 | bytePassword := []byte(password) 290 | byteHash := []byte(user.HashedPassword) 291 | err = bcrypt.CompareHashAndPassword(byteHash, bytePassword) 292 | return user, err 293 | } 294 | 295 | func clear(b []byte) { 296 | for i := 0; i < len(b); i++ { 297 | b[i] = 0 298 | } 299 | } 300 | 301 | func hashPassword(password string) (hash string, err error) { 302 | bytePassword := []byte(password) 303 | defer clear(bytePassword) 304 | byteHash, err := bcrypt.GenerateFromPassword(bytePassword, bcrypt.DefaultCost) 305 | return string(byteHash), err 306 | } 307 | 308 | func ConvertJsonStringToFileActionsStruct(jsonFileAction string, client structs.Client) (fileAction structs.FileAction, err error) { 309 | // {"Id":0,"ClientId":0,"IsCreate":true,"CreatedAt":"0001-01-01T00:00:00Z","File":{"Id":0,"UserId":0,"Name":"client.go","Hash":"f953d35b6d8067bf2bd9c46017c554b73aa28a549fac06ba747d673b2da5bfe0","Size":6622,"Modified":"2015-02-09T14:39:22-05:00","Path":"./client.go","CreatedAt":"0001-01-01T00:00:00Z"}} 310 | data := []byte(jsonFileAction) 311 | var unmarshalledFileAction structs.FileAction 312 | err = json.Unmarshal(data, &unmarshalledFileAction) 313 | if err != nil { 314 | err = fmt.Errorf("Error unmarshaling json to structs.FileAction: %s", err) 315 | return 316 | } 317 | return 318 | } 319 | 320 | func ConvertFileActionStructToJsonString(fileActionStruct structs.FileAction) (fileActionJson string, err error) { 321 | jsonBytes, err := json.Marshal(fileActionStruct) 322 | if err != nil { 323 | return 324 | } 325 | fileActionJson = string(jsonBytes) 326 | return 327 | } 328 | 329 | func RemoveRedundancyFromFileActions(fileActions []structs.FileAction) (simplifiedFileActions []structs.FileAction) { 330 | // removing redundancy 331 | // if a file is created, and then deleted remove 332 | 333 | var actionMap = make(map[string]int) 334 | 335 | var isCreateMap = make(map[bool]string) 336 | isCreateMap[true] = "1" 337 | isCreateMap[false] = "0" 338 | 339 | for _, action := range fileActions { 340 | // create a map of the fileaction 341 | // key values are task+path+hash 342 | // hash map value is the number of occurences 343 | actionKey := isCreateMap[action.IsCreate] + action.File.Path + action.File.Hash 344 | value, exists := actionMap[actionKey] 345 | if exists { 346 | actionMap[actionKey] = value + 1 347 | } else { 348 | actionMap[actionKey] = 1 349 | } 350 | } 351 | 352 | for _, action := range fileActions { 353 | // for each action value, check if a pair exists in the map 354 | // if it does remove one iteration of that value 355 | // if it doesn't write that to the simplified array 356 | 357 | // This ends up removing pairs twice, once for each matching value 358 | var opposingTask string 359 | if action.IsCreate == true { 360 | opposingTask = isCreateMap[false] 361 | } else { 362 | opposingTask = isCreateMap[true] 363 | } 364 | opposingMapKey := opposingTask + action.File.Path + action.File.Hash 365 | value, exists := actionMap[opposingMapKey] 366 | if exists == true && value > 0 { 367 | actionMap[opposingMapKey] = value - 1 368 | } else { 369 | simplifiedFileActions = append(simplifiedFileActions, action) 370 | } 371 | } 372 | return 373 | } 374 | 375 | func ComputeFilesFromFileActions(fileActions []structs.FileAction) (files []structs.File) { 376 | simplifiedFileActions := RemoveRedundancyFromFileActions(fileActions) 377 | for _, value := range simplifiedFileActions { 378 | files = append(files, value.File) 379 | } 380 | return 381 | } 382 | 383 | func WriteFileActionsToDatabase(fileActions []structs.FileAction, 384 | client structs.Client) (outPutFileActions []structs.FileAction, 385 | err error) { 386 | var user structs.User 387 | query := model.DB.Model(&client).Related(&user) 388 | if query.Error != nil { 389 | err = query.Error 390 | return outPutFileActions, err 391 | } 392 | for _, fileAction := range fileActions { 393 | fileAction.ClientId = client.Id 394 | fileAction.File.UserId = user.Id 395 | file, err := FindFile(fileAction.File.Hash, fileAction.File.Path, user) 396 | if err != nil { 397 | return outPutFileActions, err 398 | } 399 | if file.Id != 0 { 400 | // if the file exists, assign an id 401 | // otherwise GORM automatically creates 402 | // the file, so make sure to clear the File 403 | // struct 404 | fileAction.FileId = file.Id 405 | fileAction.File = structs.File{} 406 | } 407 | query = model.DB.Create(&fileAction) 408 | outPutFileActions = append( 409 | outPutFileActions, 410 | fileAction, 411 | ) 412 | if query.Error != nil { 413 | return outPutFileActions, query.Error 414 | } 415 | } 416 | return outPutFileActions, nil 417 | } 418 | 419 | func FindFile(hash string, path string, user structs.User) (file structs.File, err error) { 420 | query := model.DB.Where(&structs.File{ 421 | UserId: user.Id, 422 | Path: path, 423 | Hash: hash, 424 | }).First(&file) 425 | if query.Error != nil { 426 | if query.Error != gorm.RecordNotFound { 427 | return file, query.Error 428 | } else { 429 | return 430 | } 431 | } 432 | return //this should never happen 433 | } 434 | 435 | func ApplyFileActionsToFileSystemFileTable(fileActions []structs.FileAction, user structs.User) (errs []error) { 436 | for _, fileAction := range fileActions { 437 | // get file for fileaction 438 | var file structs.File 439 | model.DB.First(&file, fileAction.FileId) 440 | if fileAction.IsCreate == true { 441 | err := deleteFileSystemFileAtPath(file.Path, user) 442 | if err != nil { 443 | log.Fatal(err) 444 | errs = append(errs, err) 445 | } 446 | newFileSystemFile := structs.FileSystemFile{ 447 | UserId: user.Id, 448 | FileId: fileAction.FileId, 449 | Path: file.Path, 450 | } 451 | query := model.DB.Create(&newFileSystemFile) 452 | if query.Error != nil { 453 | fmt.Println(query.Error) 454 | } 455 | } else { 456 | err := deleteFileSystemFileAtPath(file.Path, user) 457 | if err != nil { 458 | log.Fatal(err) 459 | errs = append(errs, err) 460 | } 461 | } 462 | } 463 | return 464 | } 465 | 466 | func deleteFileSystemFileAtPath(path string, user structs.User) (err error) { 467 | query := model.DB. 468 | Where("path = ?", path). 469 | Where("user_id = ?", user.Id). 470 | Delete(structs.FileSystemFile{}) 471 | if query.Error != nil { 472 | return query.Error 473 | } 474 | return 475 | } 476 | -------------------------------------------------------------------------------- /boxtools/boxtools_test.go: -------------------------------------------------------------------------------- 1 | package boxtools 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/golangbox/gobox/server/model" 8 | "github.com/golangbox/gobox/structs" 9 | "github.com/jinzhu/gorm" 10 | _ "github.com/lib/pq" 11 | ) 12 | 13 | const ( 14 | email = "max.t.mcdonnell@gmail.com" 15 | password = "password" 16 | ) 17 | 18 | var user structs.User 19 | var client structs.Client 20 | 21 | func init() { 22 | var err error 23 | 24 | model.DB, err = gorm.Open("postgres", "dbname=goboxtest sslmode=disable") 25 | 26 | model.DB.DropTableIfExists(&structs.User{}) 27 | model.DB.DropTableIfExists(&structs.Client{}) 28 | model.DB.DropTableIfExists(&structs.FileAction{}) 29 | model.DB.DropTableIfExists(&structs.File{}) 30 | model.DB.DropTableIfExists(&structs.FileSystemFile{}) 31 | model.DB.AutoMigrate(&structs.User{}, &structs.Client{}, &structs.FileAction{}, &structs.File{}, &structs.FileSystemFile{}) 32 | 33 | if err != nil { 34 | fmt.Println(err) 35 | } 36 | } 37 | 38 | func TestUserCreation(t *testing.T) { 39 | var err error 40 | user, err = NewUser(email, password) 41 | if err != nil { 42 | t.Error(err) 43 | } 44 | if user.Email != email { 45 | t.Fail() 46 | } 47 | if user.HashedPassword == "" { 48 | t.Fail() 49 | } 50 | } 51 | 52 | func TestClientCreation(t *testing.T) { 53 | var user structs.User 54 | model.DB.Where("email = ?", email).Find(&user) 55 | 56 | client, err := NewClient(user, "test", false) 57 | 58 | if err != nil { 59 | t.Error(err) 60 | } 61 | user = structs.User{} //nil user 62 | 63 | //testing relation 64 | model.DB.Model(&client).Related(&user) 65 | if user.Email != email { 66 | t.Fail() 67 | } 68 | } 69 | 70 | func TestPasswordValidation(t *testing.T) { 71 | user, err := ValidateUserPassword(email, password) 72 | if err != nil { 73 | t.Error(err) 74 | } 75 | if user.Email != email { 76 | t.Fail() 77 | } 78 | // clean up created user 79 | model.DB.Where("email = ?", email).Delete(structs.User{}) 80 | } 81 | 82 | func TestJsonMetaConversion(t *testing.T) { 83 | } 84 | 85 | func TestRemoveRedundancy(t *testing.T) { 86 | _, noisy, err := GenerateNoisyAndNonNoisyFileActions(1, 4, 10, 10) 87 | if err != nil { 88 | t.Log("Could not generate file actions successfully") 89 | t.FailNow() 90 | } 91 | result := RemoveRedundancyFromFileActions(noisy) 92 | if 0 != len(result) { 93 | t.Log("Result should be empty") 94 | t.FailNow() 95 | } 96 | _, noisy, err = GenerateNoisyAndNonNoisyFileActions(1, 4, 10, 5) 97 | result = RemoveRedundancyFromFileActions(noisy) 98 | if 5 != len(result) { 99 | t.Log("Result of cleaning should be length 5") 100 | t.FailNow() 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /client/api/api.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "log" 9 | "net/http" 10 | "net/url" 11 | "strconv" 12 | "time" 13 | 14 | "github.com/golangbox/gobox/structs" 15 | ) 16 | 17 | const ( 18 | ApiEndpoint = "http://127.0.0.1:8000/" 19 | UDPEndpoint = "127.0.0.1:4242" 20 | ) 21 | 22 | type Api struct { 23 | SessionKey string 24 | } 25 | 26 | func New(SessionKey string) (c Api) { 27 | resp, err := http.Get(ApiEndpoint + "login/") 28 | if err != nil { 29 | log.Fatal(err) 30 | } 31 | keyBytes, err := ioutil.ReadAll(resp.Body) 32 | if err != nil { 33 | log.Fatal(err) 34 | } 35 | c.SessionKey = string(keyBytes) 36 | return 37 | } 38 | 39 | func (c *Api) apiRequest(endpoint string, body []byte, 40 | fileType string) (*http.Response, error) { 41 | return http.Post( 42 | ApiEndpoint+endpoint+"/?SessionKey="+c.SessionKey, 43 | "application/json", 44 | bytes.NewBuffer(body), 45 | ) 46 | } 47 | 48 | func (c *Api) SendFileActionsToServer( 49 | fileActions []structs.FileAction) ( 50 | filesToUpload []string, err error) { 51 | 52 | jsonBytes, err := json.Marshal(fileActions) 53 | if err != nil { 54 | fmt.Println("[+]") 55 | return 56 | } 57 | 58 | resp, err := c.apiRequest( 59 | "file-actions", 60 | jsonBytes, 61 | "application/json", 62 | ) 63 | 64 | //if err != nil { 65 | //fmt.Println("[!] API") 66 | //return 67 | //} 68 | 69 | contents, _ := ioutil.ReadAll(resp.Body) 70 | if err != nil { 71 | return 72 | } 73 | 74 | if resp.StatusCode != http.StatusOK { 75 | return 76 | } 77 | err = json.Unmarshal(contents, &filesToUpload) 78 | if err != nil { 79 | return 80 | } 81 | return 82 | } 83 | 84 | func (c *Api) UploadFileToServer(fileBody []byte) (err error) { 85 | resp, err := c.apiRequest( 86 | "upload", 87 | fileBody, 88 | "", 89 | ) 90 | if err != nil { 91 | return 92 | } 93 | if resp.StatusCode != http.StatusOK { 94 | contents, err := ioutil.ReadAll(resp.Body) 95 | if err != nil { 96 | return err 97 | } 98 | err = fmt.Errorf(string(contents)) 99 | return err 100 | } 101 | return 102 | } 103 | 104 | func (c *Api) DownloadFileFromServer( 105 | hash string) (s3_url string, err error) { 106 | for { 107 | resp, err := http.PostForm( 108 | ApiEndpoint+"download/", 109 | url.Values{ 110 | "SessionKey": {c.SessionKey}, 111 | "fileHash": {hash}, 112 | }, 113 | ) 114 | if err != nil { 115 | return "", err 116 | } 117 | fmt.Println("HASH: ", hash) 118 | contents, err := ioutil.ReadAll(resp.Body) 119 | fmt.Println(string(contents)) 120 | if resp.StatusCode == http.StatusInternalServerError { 121 | err = fmt.Errorf(string(contents)) 122 | return "", err 123 | } else if resp.StatusCode == http.StatusOK { 124 | s3_url = string(contents) 125 | return s3_url, err 126 | } else { 127 | time.Sleep(time.Second * 10) 128 | } 129 | } 130 | return 131 | } 132 | 133 | func (c *Api) DownloadClientFileActions(lastId int64) ( 134 | clientFileActionsResponse structs.ClientFileActionsResponse, err error) { 135 | var lastIdString string 136 | lastIdString = strconv.FormatInt(lastId, 32) 137 | resp, err := http.PostForm( 138 | ApiEndpoint+"clients/", 139 | url.Values{ 140 | "SessionKey": {c.SessionKey}, 141 | "lastId": {lastIdString}, 142 | }, 143 | ) 144 | if err != nil { 145 | fmt.Println("[!]") 146 | return 147 | } 148 | contents, err := ioutil.ReadAll(resp.Body) 149 | if resp.StatusCode != http.StatusOK { 150 | err = fmt.Errorf(string(contents)) 151 | fmt.Println(resp.StatusCode) 152 | return 153 | } 154 | 155 | err = json.Unmarshal(contents, &clientFileActionsResponse) 156 | if err != nil { 157 | fmt.Println("[+]") 158 | return 159 | } 160 | return 161 | } 162 | -------------------------------------------------------------------------------- /client/api/api.go.orig: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "log" 9 | "net/http" 10 | "net/url" 11 | "strconv" 12 | "time" 13 | 14 | "github.com/golangbox/gobox/structs" 15 | ) 16 | 17 | const ( 18 | <<<<<<< HEAD 19 | ApiEndpoint = "http://127.0.0.1:8000/" 20 | UDPEndpoint = "127.0.0.1:4242" 21 | ======= 22 | ApiEndpoint = "http://localhost:8000/" 23 | UDPEndpoint = "localhost:4242" 24 | >>>>>>> b75353a8a44313b9ce2acf63c3a3f29513e45772 25 | ) 26 | 27 | type Api struct { 28 | SessionKey string 29 | } 30 | 31 | func New(SessionKey string) (c Api) { 32 | resp, err := http.Get(ApiEndpoint + "login/") 33 | if err != nil { 34 | log.Fatal(err) 35 | } 36 | keyBytes, err := ioutil.ReadAll(resp.Body) 37 | if err != nil { 38 | log.Fatal(err) 39 | } 40 | c.SessionKey = string(keyBytes) 41 | return 42 | } 43 | 44 | func (c *Api) apiRequest(endpoint string, body []byte, 45 | fileType string) (*http.Response, error) { 46 | return http.Post( 47 | ApiEndpoint+endpoint+"/?SessionKey="+c.SessionKey, 48 | "application/json", 49 | bytes.NewBuffer(body), 50 | ) 51 | } 52 | 53 | func (c *Api) SendFileActionsToServer( 54 | fileActions []structs.FileAction) ( 55 | filesToUpload []string, err error) { 56 | 57 | jsonBytes, err := json.Marshal(fileActions) 58 | if err != nil { 59 | return 60 | } 61 | 62 | resp, err := c.apiRequest( 63 | "file-actions", 64 | jsonBytes, 65 | "application/json", 66 | ) 67 | 68 | if err != nil { 69 | return 70 | } 71 | 72 | contents, err := ioutil.ReadAll(resp.Body) 73 | if err != nil { 74 | return 75 | } 76 | 77 | if resp.StatusCode != http.StatusOK { 78 | err = fmt.Errorf(string(contents)) 79 | return 80 | } 81 | 82 | err = json.Unmarshal(contents, &filesToUpload) 83 | if err != nil { 84 | return 85 | } 86 | return 87 | } 88 | 89 | func (c *Api) UploadFileToServer(fileBody []byte) (err error) { 90 | resp, err := c.apiRequest( 91 | "upload", 92 | fileBody, 93 | "", 94 | ) 95 | if err != nil { 96 | return 97 | } 98 | if resp.StatusCode != http.StatusOK { 99 | contents, err := ioutil.ReadAll(resp.Body) 100 | if err != nil { 101 | return err 102 | } 103 | err = fmt.Errorf(string(contents)) 104 | return err 105 | } 106 | return 107 | } 108 | 109 | func (c *Api) DownloadFileFromServer( 110 | hash string) (s3_url string, err error) { 111 | for { 112 | resp, err := http.PostForm( 113 | ApiEndpoint+"download/", 114 | url.Values{ 115 | "SessionKey": {c.SessionKey}, 116 | "fileHash": {hash}, 117 | }, 118 | ) 119 | if err != nil { 120 | return "", err 121 | } 122 | 123 | contents, err := ioutil.ReadAll(resp.Body) 124 | if resp.StatusCode == http.StatusInternalServerError { 125 | err = fmt.Errorf(string(contents)) 126 | return "", err 127 | } else if resp.StatusCode == http.StatusOK { 128 | s3_url = string(contents) 129 | return "", err 130 | } else { 131 | time.Sleep(time.Second * 10) 132 | } 133 | } 134 | return 135 | } 136 | 137 | func (c *Api) DownloadClientFileActions(lastId int64) ( 138 | clientFileActionsResponse structs.ClientFileActionsResponse, err error) { 139 | var lastIdString string 140 | lastIdString = strconv.FormatInt(lastId, 32) 141 | resp, err := http.PostForm( 142 | ApiEndpoint+"/clients/", 143 | url.Values{ 144 | "SessionKey": {c.SessionKey}, 145 | "lastID": {lastIdString}, 146 | }, 147 | ) 148 | if err != nil { 149 | return 150 | } 151 | contents, err := ioutil.ReadAll(resp.Body) 152 | if resp.StatusCode != http.StatusOK { 153 | err = fmt.Errorf(string(contents)) 154 | return 155 | } 156 | 157 | err = json.Unmarshal(contents, &clientFileActionsResponse) 158 | if err != nil { 159 | return 160 | } 161 | return 162 | } 163 | -------------------------------------------------------------------------------- /client/api/api_test.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | "testing" 8 | 9 | "github.com/golangbox/gobox/boxtools" 10 | server_api "github.com/golangbox/gobox/server/api" 11 | "github.com/golangbox/gobox/server/model" 12 | "github.com/golangbox/gobox/structs" 13 | "github.com/jinzhu/gorm" 14 | ) 15 | 16 | var user structs.User 17 | var client structs.Client 18 | var apiClient Api 19 | 20 | func init() { 21 | model.DB, _ = gorm.Open("postgres", "dbname=goboxtest sslmode=disable") 22 | 23 | model.DB.DropTableIfExists(&structs.User{}) 24 | model.DB.DropTableIfExists(&structs.Client{}) 25 | model.DB.DropTableIfExists(&structs.FileAction{}) 26 | model.DB.DropTableIfExists(&structs.File{}) 27 | model.DB.DropTableIfExists(&structs.FileSystemFile{}) 28 | model.DB.AutoMigrate(&structs.User{}, &structs.Client{}, &structs.FileAction{}, &structs.File{}, &structs.FileSystemFile{}) 29 | 30 | user, _ = boxtools.NewUser("max.t.mcdonnell@gmail", "password") 31 | 32 | var err error 33 | client, err = boxtools.NewClient(user, "test", false) 34 | if err != nil { 35 | fmt.Println(err) 36 | } 37 | 38 | apiClient = New(client.SessionKey) 39 | 40 | go server_api.ServeServerRoutes("8000") 41 | } 42 | 43 | func TestSendFileActionsToServer(t *testing.T) { 44 | fileActions, _ := boxtools.GenerateSliceOfRandomFileActions(1, 1, 1) 45 | 46 | var hashesToUpload []string 47 | var err error 48 | hashesToUpload, err = apiClient.SendFileActionsToServer(fileActions) 49 | if err != nil { 50 | t.Error(err) 51 | } 52 | 53 | if fileActions[0].File.Hash != hashesToUpload[0] { 54 | t.Error(fmt.Errorf("Wrong hash returned")) 55 | } 56 | } 57 | 58 | func TestDownloadFileFromServer(t *testing.T) { 59 | testFile := structs.File{ 60 | Hash: "fc45acaffc35a3aa674f7c0d5a03d22350b4f2ff4bf45ccebad077e5af80e512", 61 | UserId: user.Id, 62 | } 63 | model.DB.Create(&testFile) 64 | 65 | url, err := apiClient.DownloadFileFromServer("fc45acaffc35a3aa674f7c0d5a03d22350b4f2ff4bf45ccebad077e5af80e512") 66 | if err != nil { 67 | t.Error(err) 68 | } 69 | _ = url 70 | // futher tested to confirm functioanlity in 71 | // TestUploadFileToServer 72 | } 73 | 74 | func TestUploadFileToServer(t *testing.T) { 75 | file := []byte("this is a file") 76 | err := apiClient.UploadFileToServer(file) 77 | if err != nil { 78 | t.Error(err) 79 | } 80 | 81 | expectedHash := "fc45acaffc35a3aa674f7c0d5a03d22350b4f2ff4bf45ccebad077e5af80e512" 82 | testFile := structs.File{ 83 | Hash: expectedHash, 84 | UserId: user.Id, 85 | } 86 | model.DB.Create(&testFile) 87 | 88 | s3_url, err := apiClient.DownloadFileFromServer( 89 | expectedHash, 90 | ) 91 | if err != nil { 92 | t.Error(err) 93 | t.FailNow() 94 | } 95 | resp, err := http.Get(s3_url) 96 | if err != nil { 97 | t.Error(err) 98 | } 99 | contents, err := ioutil.ReadAll(resp.Body) 100 | if err != nil { 101 | t.Error(err) 102 | } 103 | 104 | if string(contents) != "this is a file" { 105 | t.Error(fmt.Errorf("S3 file contents don't match")) 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /client/client.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/hex" 6 | "encoding/json" 7 | "fmt" 8 | "io/ioutil" 9 | "log" 10 | "net" 11 | "net/http" 12 | "os" 13 | "path/filepath" 14 | "reflect" 15 | "regexp" 16 | "time" 17 | 18 | "github.com/golangbox/gobox/client/api" 19 | "github.com/golangbox/gobox/client/watcher" 20 | "github.com/golangbox/gobox/structs" 21 | ) 22 | 23 | // TODO 24 | // get client/server sessions set up 25 | // sync a damn file 26 | // work on download branch 27 | 28 | // PROBLEMS: No way to tell if a remove event was dir or a file because it can't be os.Stat'ed 29 | // Can't remove that dir from a watch because Watcher.watches isn't exposed 30 | 31 | var client api.Api 32 | 33 | const ( 34 | dataDirectoryBasename = ".Gobox" 35 | serverEndpoint = "http://requestb.in/1mv9fa41" 36 | ) 37 | 38 | func writeError(err error, change structs.StateChange, function string) { 39 | change.Error <- structs.ErrorMessage{ 40 | Error: err, 41 | File: change.File, 42 | Function: function, 43 | } 44 | close(change.Done) 45 | } 46 | 47 | // it may be easier to just have this function make a FileAction, but ignore for now 48 | func writeDone(change structs.StateChange, fa structs.FileAction) { 49 | change.Done <- fa 50 | close(change.Error) 51 | } 52 | 53 | func findChangedFilesOnInit( 54 | fileSystemStateFile string, 55 | goboxDirectoryPath string, 56 | watcherInitScanDone chan<- struct{}, 57 | serverActionsInitScanDone chan<- struct{}) ( 58 | out chan structs.StateChange, err error) { 59 | 60 | out = make(chan structs.StateChange) 61 | fileSystemState, err := fetchFileSystemState(fileSystemStateFile) 62 | if err != nil { 63 | return 64 | } 65 | go func() { 66 | err = filepath.Walk(goboxDirectoryPath, func(fp string, fi os.FileInfo, errIn error) (errOut error) { 67 | if errIn != nil { 68 | return errIn 69 | } 70 | if filepath.HasPrefix(fp, ".") || filepath.HasPrefix(fp, "..") { 71 | return 72 | } 73 | matched, errOut := regexp.MatchString(".Gobox(/.*)*", fp) 74 | if errOut != nil { 75 | return 76 | } 77 | if matched { 78 | return 79 | } 80 | f, found := fileSystemState[fp] 81 | if !found { 82 | change, err := watcher.CreateLocalStateChange(fp, 83 | watcher.CREATE) 84 | if err != nil { 85 | return 86 | } 87 | out <- change 88 | return 89 | } 90 | h, err := getSha256FromFilename(fp) 91 | if err != nil { 92 | return 93 | } 94 | if f.Hash != h { 95 | change, err := watcher.CreateLocalStateChange(fp, watcher.MODIFY) 96 | if err != nil { 97 | return 98 | } 99 | out <- change 100 | delete(fileSystemState, fp) 101 | return 102 | } 103 | return 104 | }) 105 | if err != nil { 106 | return 107 | } 108 | for fp := range fileSystemState { 109 | // may need to change structs so that PreviousHash is in the File struct 110 | change, err := watcher.CreateLocalStateChange(fp, watcher.DELETE) 111 | if err != nil { 112 | return 113 | } 114 | out <- change 115 | } 116 | watcherInitScanDone <- struct{}{} 117 | serverActionsInitScanDone <- struct{}{} 118 | return 119 | }() 120 | return 121 | } 122 | 123 | func startWatcher(dir string, initScanDone <-chan struct{}) ( 124 | out chan structs.StateChange, err error) { 125 | rw, err := watcher.NewRecursiveWatcher(dir) 126 | if err != nil { 127 | return out, err 128 | 129 | } 130 | rw.Run(initScanDone, false) 131 | return rw.Files, err 132 | } 133 | 134 | func createServerStateChange(fa structs.FileAction) (change structs.StateChange) { 135 | change.File = fa.File 136 | change.IsCreate = fa.IsCreate 137 | change.IsLocal = false 138 | change.PreviousHash = fa.PreviousHash 139 | return 140 | } 141 | 142 | // ah shoot! is this the only function that needs to know about the last fileaction ID state? 143 | // the following function is hypothetical and non-functional as of now 144 | 145 | func serverActions(UDPing <-chan bool, fileActionIdPath string, 146 | initScanDone <-chan struct{}) (out chan structs.StateChange, 147 | errorChan chan interface{}, err error) { 148 | out = make(chan structs.StateChange) 149 | fileActionId, err := fetchFileActionID(fileActionIdPath) 150 | if err != nil { 151 | panic("Couldn't properly read fil eActionId from given path") 152 | } 153 | 154 | go func() { 155 | <-initScanDone 156 | for { 157 | <-UDPing 158 | fmt.Println("-----------------PING RECIEVED--------------------") 159 | // these return values are obviously wrong right now 160 | fmt.Println("File aCTION ID", fileActionId) 161 | clientFileActionResponse, err := client.DownloadClientFileActions( 162 | fileActionId) 163 | // need to rethink errors, assumption of a statechange is invalid 164 | if err != nil { 165 | fmt.Println(err) 166 | writeError(err, structs.StateChange{}, "serverActions") 167 | } 168 | fileActionId = clientFileActionResponse.LastId 169 | for _, fileAction := range clientFileActionResponse.FileActions { 170 | change := createServerStateChange(fileAction) 171 | fmt.Println("In server actions hash: ", change.File.Hash) 172 | out <- change 173 | fmt.Println("loop") 174 | } 175 | err = writeFileActionIDToLocalFile(fileActionId, fileActionIdPath) 176 | if err != nil { 177 | fmt.Println("Couldn't write fileActionId to path : ", 178 | fileActionIdPath) 179 | } 180 | } 181 | }() 182 | return 183 | } 184 | 185 | func initUDPush(sessionKey string) (notification chan bool, err error) { 186 | notification = make(chan bool) 187 | go func() { 188 | conn, err := net.Dial("tcp", api.UDPEndpoint) 189 | // defer conn.Close() 190 | if err != nil { 191 | fmt.Println(fmt.Errorf("%s", err)) 192 | return 193 | } 194 | sessionKeyBytes := []byte(sessionKey) 195 | _, err = conn.Write(sessionKeyBytes) 196 | if err != nil { 197 | return 198 | } 199 | response := make([]byte, 21) 200 | for { 201 | read, err := conn.Read(response) 202 | if err != nil { 203 | fmt.Println(err) 204 | } 205 | fmt.Println("Message read from socket: ", read, string(response)) 206 | notification <- true 207 | fmt.Println("YO") 208 | } 209 | }() 210 | return 211 | 212 | } 213 | 214 | func fanActionsIn(initActions <-chan structs.StateChange, 215 | watcherActions <-chan structs.StateChange, 216 | serverActions <-chan structs.StateChange) chan structs.StateChange { 217 | out := make(chan structs.StateChange) 218 | go func() { 219 | for { 220 | fmt.Println("FanActions") 221 | select { 222 | case stateChange := <-initActions: 223 | fmt.Println("fanActions inside the stateChange := <-initActions") 224 | out <- stateChange 225 | case stateChange := <-watcherActions: 226 | fmt.Println("fanActions inside the stateChange := <-WatcherActions") 227 | out <- stateChange 228 | case stateChange := <-serverActions: 229 | fmt.Println("fanActions inside the stateChange := <-serverActions") 230 | fmt.Println("Fanin: ", stateChange.File.Hash) 231 | out <- stateChange 232 | } 233 | } 234 | }() 235 | return out 236 | } 237 | 238 | func createGoboxLocalDirectory(path string) { 239 | if _, err := os.Stat(path); err != nil { 240 | fmt.Println(err.Error()) 241 | if os.IsNotExist(err) { 242 | fmt.Println("Making directory") 243 | err := os.Mkdir(path, 0777) 244 | if err != nil { 245 | log.Fatal(err) 246 | } 247 | } 248 | } 249 | } 250 | 251 | func writeFileSystemStateToLocalFile(fileSystemState map[string]structs.File, path string) error { 252 | jsonBytes, err := json.Marshal(fileSystemState) 253 | if err != nil { 254 | return err 255 | } 256 | 257 | err = ioutil.WriteFile(path, jsonBytes, 0644) 258 | return err 259 | } 260 | 261 | func fetchFileSystemState(path string) (fileSystemState map[string]structs.File, err error) { 262 | if _, err := os.Stat(path); err != nil { 263 | emptyState := make(map[string]structs.File) 264 | writeFileSystemStateToLocalFile(emptyState, path) 265 | } 266 | 267 | data, err := ioutil.ReadFile(path) 268 | if err != nil { 269 | fmt.Println(err.Error()) 270 | return 271 | } 272 | if data != nil { 273 | err = json.Unmarshal(data, &fileSystemState) 274 | } 275 | return 276 | 277 | } 278 | 279 | func writeFileActionIDToLocalFile(id int64, path string) (err error) { 280 | jsonBytes, err := json.Marshal(id) 281 | if err != nil { 282 | return 283 | } 284 | err = ioutil.WriteFile(path, jsonBytes, 0644) 285 | return 286 | } 287 | 288 | func fetchFileActionID(path string) (id int64, err error) { 289 | if _, err := os.Stat(path); err != nil { 290 | defaultId := int64(1) 291 | err = writeFileActionIDToLocalFile(defaultId, path) 292 | if err != nil { 293 | return id, err 294 | } 295 | } 296 | data, err := ioutil.ReadFile(path) 297 | if err != nil { 298 | return 299 | } 300 | if data != nil { 301 | err = json.Unmarshal(data, &id) 302 | } 303 | return 304 | } 305 | 306 | func getSha256FromFilename(filename string) (sha256String string, 307 | err error) { 308 | file, err := ioutil.ReadFile(filename) 309 | if err != nil { 310 | return "", fmt.Errorf("Error reading file for sha256: %s", err) 311 | } 312 | h := sha256.New() 313 | _, err = h.Write(file) 314 | if err != nil { 315 | return "", fmt.Errorf("Error writing file to hash for sha256: %s", err) 316 | } 317 | byteString := h.Sum(nil) 318 | 319 | sha256String = hex.EncodeToString(byteString) 320 | 321 | return sha256String, nil 322 | } 323 | 324 | func makeFileAction(change structs.StateChange) (fa structs.FileAction) { 325 | fa.IsCreate = change.IsCreate 326 | fa.CreatedAt = change.File.CreatedAt 327 | fa.FileId = change.File.Id 328 | fa.File = change.File 329 | fa.PreviousHash = change.PreviousHash 330 | return 331 | } 332 | 333 | func fileActionSender(change structs.StateChange) { 334 | select { 335 | case <-change.Quit: 336 | gracefulQuit(change) 337 | return 338 | default: 339 | fileActions := make([]structs.FileAction, 1) 340 | fileActions[0] = makeFileAction(change) 341 | 342 | needed, err := client.SendFileActionsToServer(fileActions) 343 | if err != nil { 344 | writeError(err, change, "fileActionSender") 345 | return 346 | } 347 | // if len needed 0, need to do cleanup by signaling done 348 | if len(needed) == 0 { 349 | writeDone(change, fileActions[0]) 350 | return 351 | } 352 | // need to fix this to just get responses for one file 353 | go uploader(change.File.Path, change, fileActions[0]) 354 | 355 | } 356 | return 357 | } 358 | 359 | func uploader(path string, change structs.StateChange, fa structs.FileAction) { 360 | select { 361 | case <-change.Quit: 362 | gracefulQuit(change) 363 | return 364 | default: 365 | buf, err := ioutil.ReadFile(path) 366 | if err != nil { 367 | writeError(err, change, "uploader") 368 | return 369 | } 370 | err = client.UploadFileToServer(buf) 371 | if err != nil { 372 | writeError(err, change, "uploader") 373 | return 374 | } 375 | change.Done <- fa 376 | close(change.Error) 377 | } 378 | return 379 | } 380 | 381 | func gracefulQuit(change structs.StateChange) { 382 | close(change.Done) 383 | close(change.Error) 384 | } 385 | 386 | func hasher(change structs.StateChange) { 387 | select { 388 | case <-change.Quit: 389 | gracefulQuit(change) 390 | return 391 | default: 392 | h, err := getSha256FromFilename(change.File.Path) 393 | if err != nil { 394 | writeError(err, change, "hasher") 395 | return 396 | } 397 | change.File.Hash = h 398 | go fileActionSender(change) 399 | } 400 | return 401 | } 402 | 403 | // potential for deadly embrace when stephen tries to send a new channel while this func 404 | // blocks on sending on out 405 | 406 | // fix was to add a go func to write to out. I don't love this solution becuase it makes order 407 | // of stuff coming out of the out channel non-deterministic, so any other ideas are invited. 408 | func arbitraryFanIn(newChannels <-chan chan interface{}, out chan<- interface{}, removeOnEvent bool) { 409 | go func() { 410 | var ch <-chan interface{} 411 | chans := make([]reflect.SelectCase, 0) 412 | timeout := time.Tick(10 * time.Millisecond) 413 | chans = append(chans, reflect.SelectCase{ 414 | Dir: reflect.SelectRecv, 415 | Chan: reflect.ValueOf(timeout), 416 | }) 417 | for { 418 | select { 419 | case ch = <-newChannels: 420 | chans = append(chans, reflect.SelectCase{ 421 | Dir: reflect.SelectRecv, 422 | Chan: reflect.ValueOf(ch), 423 | }) 424 | default: 425 | chosen, value, ok := reflect.Select(chans) 426 | if chosen == 0 { 427 | continue 428 | } 429 | if removeOnEvent || !ok { 430 | lastIndex := len(chans) - 1 431 | chans[chosen] = chans[lastIndex] 432 | chans = chans[:lastIndex] 433 | } 434 | if ok { 435 | go func() { out <- value.Interface() }() 436 | } 437 | } 438 | } 439 | }() 440 | } 441 | 442 | func serverDeleter(change structs.StateChange) { 443 | _, err := os.Stat(change.File.Path) 444 | if err != nil { 445 | if os.IsNotExist(err) { 446 | writeDone(change, makeFileAction(change)) 447 | return 448 | } 449 | writeError(err, change, "deleter") 450 | return 451 | } 452 | err = os.Remove(change.File.Path) 453 | if err != nil { 454 | writeError(err, change, "deleter") 455 | return 456 | } 457 | writeDone(change, makeFileAction(change)) 458 | } 459 | 460 | func downloader(change structs.StateChange) { 461 | fmt.Println("Change: ", change.File.Hash) 462 | select { 463 | case <-change.Quit: 464 | gracefulQuit(change) 465 | return 466 | default: 467 | // this could take a long time 468 | s3_url, err := client.DownloadFileFromServer(change.File.Hash) 469 | if err != nil { 470 | writeError(err, change, "downloader") 471 | } 472 | resp, err := http.Get(s3_url) 473 | if err != nil { 474 | writeError(err, change, "downloader") 475 | } 476 | contents, err := ioutil.ReadAll(resp.Body) 477 | fmt.Println(contents) 478 | if err != nil { 479 | writeError(err, change, "downloader") 480 | } 481 | //tmpFilename := filepath.Join(".Gobox/tmp/", change.File.Hash) 482 | err = ioutil.WriteFile(change.File.Path, contents, 0644) 483 | if err != nil { 484 | writeError(err, change, "downloader") 485 | } 486 | //select { 487 | //case <-change.Quit: 488 | //gracefulQuit(change) 489 | //return 490 | //default: 491 | //err = os.Rename(tmpFilename, change.File.Path) 492 | //if err != nil { 493 | //writeError(err, change, "downloader") 494 | //} 495 | //} 496 | } 497 | fmt.Println("look") 498 | return 499 | } 500 | 501 | func localDeleter(change structs.StateChange) { 502 | go fileActionSender(change) 503 | } 504 | 505 | func stephen(goboxFileSystemStateFile string, stateChanges <-chan structs.StateChange, 506 | inputErrChans []chan interface{}) { 507 | // spin up a goroutine that will fan in error messages using reflect.select 508 | // hand it an error channel, and add this to the main select statement 509 | // do the same thing for done, so I can write a generic fan-n-in function 510 | fileSystemState, err := fetchFileSystemState(goboxFileSystemStateFile) 511 | if err != nil { 512 | panic("Could not properly retrieve fileSystemState") 513 | } 514 | 515 | quitChannels := make(map[string]structs.CurrentAction) 516 | 517 | newErrors := make((chan (chan interface{}))) 518 | errors := make(chan interface{}) 519 | go arbitraryFanIn(newErrors, errors, true) 520 | 521 | newDones := make((chan (chan interface{}))) 522 | dones := make(chan interface{}) 523 | go arbitraryFanIn(newDones, dones, true) 524 | 525 | for _, ch := range inputErrChans { 526 | newErrors <- ch 527 | } 528 | 529 | writeFileSystemStateCounter := 0 530 | for { 531 | if writeFileSystemStateCounter > 5 { 532 | writeFileSystemStateToLocalFile( 533 | fileSystemState, 534 | goboxFileSystemStateFile, 535 | ) 536 | if err != nil { 537 | fmt.Println( 538 | "Error while writing fileSystemState to: ", 539 | goboxFileSystemStateFile) 540 | } 541 | writeFileSystemStateCounter = 0 542 | 543 | } 544 | // maybe it would be better to combine errors and dones into the same multiplexor? 545 | select { 546 | case e := <-errors: 547 | msg := e.(structs.ErrorMessage) 548 | fmt.Println("Experienced an error with ", msg.File.Path) 549 | fmt.Println("In function: ", msg.Function) 550 | fmt.Println("Error: ", msg.Error) 551 | fmt.Println("File: ", msg.File) 552 | delete(quitChannels, msg.File.Path) 553 | case d := <-dones: 554 | fa := d.(structs.FileAction) 555 | delete(quitChannels, fa.File.Path) 556 | fileSystemState[fa.File.Path] = fa.File 557 | case change := <-stateChanges: 558 | if currentAction, found := quitChannels[change.File.Path]; found { 559 | // tell goroutine branch to quit 560 | if currentAction.IsLocal == false { 561 | continue 562 | } 563 | currentAction.Quit <- true 564 | close(currentAction.Quit) 565 | } 566 | f, found := fileSystemState[change.File.Path] 567 | if change.IsLocal { 568 | if found { 569 | change.PreviousHash = f.Hash 570 | } else { 571 | change.PreviousHash = "" 572 | } 573 | } 574 | 575 | fmt.Println(change) 576 | quitChan := make(chan bool, 1) 577 | doneChan := make(chan interface{}, 1) 578 | newDones <- doneChan 579 | errChan := make(chan interface{}, 1) 580 | newErrors <- errChan 581 | quitChannels[change.File.Path] = structs.CurrentAction{ 582 | Quit: quitChan, 583 | IsCreate: change.IsCreate, 584 | IsLocal: change.IsLocal, 585 | } 586 | change.Quit = quitChan 587 | change.Done = doneChan 588 | change.Error = errChan 589 | if change.IsCreate { 590 | if change.IsLocal { 591 | go hasher(change) 592 | } else { 593 | fmt.Println("[!]Attempting download...") 594 | go downloader(change) 595 | } 596 | } else { 597 | if found { 598 | if change.IsLocal { 599 | go localDeleter(change) 600 | } else { 601 | if f.Hash == change.PreviousHash { 602 | go serverDeleter(change) 603 | delete(fileSystemState, change.File.Path) 604 | } 605 | 606 | } 607 | } 608 | 609 | } 610 | } 611 | writeFileSystemStateCounter++ 612 | } 613 | } 614 | func run(path string) { 615 | errChans := make([]chan interface{}, 0) 616 | watcherInitScanDone := make(chan struct{}) 617 | serverActionsInitScanDone := make(chan struct{}) 618 | client = api.New("") 619 | err := os.Chdir(path) 620 | if err != nil { 621 | fmt.Println("unable to change dir, quitting") 622 | return 623 | } 624 | goboxDirectory := "." 625 | goboxDataDirectory := filepath.Join(goboxDirectory, dataDirectoryBasename) 626 | goboxFileSystemStateFile := filepath.Join(goboxDataDirectory, "fileSystemState") 627 | goboxFileActionIdFile := filepath.Join(goboxDataDirectory, "fileActionId") 628 | 629 | createGoboxLocalDirectory(goboxDataDirectory) 630 | initActions, err := findChangedFilesOnInit( 631 | goboxFileSystemStateFile, 632 | goboxDirectory, 633 | watcherInitScanDone, 634 | serverActionsInitScanDone, 635 | ) 636 | if err != nil { 637 | panic("Could not start init scan") 638 | } 639 | watcherActions, err := startWatcher(goboxDirectory, watcherInitScanDone) 640 | if err != nil { 641 | panic("Could not start watcher") 642 | } 643 | UDPNotification, err := initUDPush(client.SessionKey) 644 | if err != nil { 645 | panic("Could not start UDP socket") 646 | } 647 | // fix this to get correct fileActionID 648 | remoteActions, errChan, err := serverActions(UDPNotification, 649 | goboxFileActionIdFile, serverActionsInitScanDone) 650 | errChans = append(errChans, errChan) 651 | if err != nil { 652 | panic("Could not properly start remote actions") 653 | } 654 | 655 | actions := fanActionsIn(initActions, watcherActions, remoteActions) 656 | stephen(goboxFileSystemStateFile, actions, errChans) 657 | } 658 | 659 | func main() { 660 | if len(os.Args) != 2 { 661 | fmt.Println("usage: ./gobox_client PATH_TO_GOBOX_DIRECTORY") 662 | return 663 | } 664 | fi, err := os.Stat(os.Args[1]) 665 | if err != nil { 666 | fmt.Println("Error reading gobox directory") 667 | return 668 | } 669 | if !fi.IsDir() { 670 | fmt.Println("Provided path is not a directory") 671 | return 672 | } 673 | 674 | fmt.Println("Running : ", os.Args[1]) 675 | run(os.Args[1]) 676 | } 677 | -------------------------------------------------------------------------------- /client/client_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/golangbox/gobox/boxtools" 8 | "github.com/golangbox/gobox/structs" 9 | 10 | "path/filepath" 11 | "reflect" 12 | "testing" 13 | ) 14 | 15 | const ( 16 | sandboxDir = "../sandbox" 17 | ) 18 | 19 | func TestStartWatcher(t *testing.T) { 20 | _, err := startWatcher("/billybob") 21 | if err == nil { 22 | t.Log("startWatcher must fail on invalid directory") 23 | t.FailNow() 24 | } 25 | 26 | ch, err := startWatcher(sandboxDir) 27 | if err != nil { 28 | t.Log("startWatcher didn't work on a valid directory") 29 | t.FailNow() 30 | } 31 | 32 | if reflect.ValueOf(ch).Kind() != reflect.Chan { 33 | t.Log("Return value from startWatcher must be a channel") 34 | t.FailNow() 35 | } 36 | go boxtools.SimulateFilesystemChanges(sandboxDir, 10, 5, 0) 37 | for i := 0; i < 18; i++ { 38 | fmt.Println(<-ch) 39 | fmt.Println(i) 40 | 41 | } 42 | ignores := make(map[string]bool) 43 | ignores[".Gobox"] = true 44 | abspath, err := filepath.Abs(sandboxDir) 45 | if err != nil { 46 | t.Log("Could not clean up properly") 47 | t.FailNow() 48 | } 49 | err = boxtools.CleanTestFolder(abspath, ignores, true) 50 | if err != nil { 51 | t.Log(err.Error()) 52 | t.FailNow() 53 | } 54 | return 55 | } 56 | 57 | func TestServerActions(t *testing.T) { 58 | 59 | } 60 | 61 | func TestFanActionsIn(t *testing.T) { 62 | 63 | ch1, ch2 := make(chan structs.StateChange), make(chan structs.StateChange) 64 | out := fanActionsIn(ch1, ch2) 65 | numRead := 0 66 | go func() { ch1 <- structs.StateChange{} }() 67 | go func() { ch2 <- structs.StateChange{} }() 68 | timeout := time.Tick(1000 * time.Millisecond) 69 | timedOut := false 70 | for !timedOut { 71 | select { 72 | case <-out: 73 | numRead++ 74 | case <-timeout: 75 | timedOut = true 76 | } 77 | } 78 | if numRead != 2 { 79 | t.FailNow() 80 | } 81 | 82 | } 83 | 84 | func TestArbitraryFanIn(t *testing.T) { 85 | validNums := make(map[int]bool) 86 | newChans := make(chan chan interface{}) 87 | out := make(chan interface{}) 88 | arbitraryFanIn(newChans, out, true) 89 | chans := make([]chan interface{}, 0) 90 | for i := 0; i < 10; i++ { 91 | chans = append(chans, make(chan interface{}, 1)) 92 | newChans <- chans[i] 93 | } 94 | 95 | for i := 0; i < 5; i++ { 96 | chans[i] <- interface{}(i) 97 | validNums[i] = true 98 | } 99 | 100 | for i := 10; i < 20; i++ { 101 | chans = append(chans, make(chan interface{}, 1)) 102 | newChans <- chans[i] 103 | chans[i] <- interface{}(i) 104 | validNums[i] = true 105 | } 106 | 107 | timeout := time.Tick(1000 * time.Microsecond) 108 | timedOut := false 109 | for !timedOut { 110 | select { 111 | case v := <-out: 112 | fmt.Println(v, v.(int)) 113 | if _, found := validNums[v.(int)]; !found { 114 | fmt.Println(found) 115 | t.FailNow() 116 | } 117 | case <-timeout: 118 | timedOut = true 119 | } 120 | } 121 | } 122 | 123 | func TestStephen(t *testing.T) { 124 | run(sandboxDir) 125 | go boxtools.SimulateFilesystemChanges(sandboxDir, 10, 5, 0) 126 | for { 127 | time.Sleep(1000) 128 | } 129 | } 130 | 131 | func TestHasherQuitsProperly(t *testing.T) { 132 | 133 | } 134 | -------------------------------------------------------------------------------- /client/shelltest.sh: -------------------------------------------------------------------------------- 1 | GOBOX_PATH=$(pwd)/../test 2 | CLIENT_PATH=$(pwd)/client.go 3 | 4 | echo $GOBOX_PATH 5 | 6 | go run $CLIENT_PATH $GOBOX_PATH & 7 | 8 | CLIENT_PROCESS=$! 9 | 10 | echo $CLIENT_PROCESS 11 | 12 | cd $GOBOX_PATH 13 | 14 | touch f1 15 | echo "foo" >> f1 16 | rm f1 17 | 18 | kill $CLIENT_PROCESS 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /client/watcher/watcher.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "log" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | 11 | "github.com/go-fsnotify/fsnotify" 12 | "github.com/golangbox/gobox/structs" 13 | ) 14 | 15 | const ( 16 | CREATE = 0 17 | MODIFY = 1 18 | DELETE = 2 19 | ) 20 | 21 | type RecursiveWatcher struct { 22 | *fsnotify.Watcher 23 | Files chan structs.StateChange 24 | Folders chan string 25 | } 26 | 27 | func NewRecursiveWatcher(path string) (*RecursiveWatcher, error) { 28 | folders := Subfolders(path) 29 | fmt.Println(folders) 30 | if len(folders) == 0 { 31 | return nil, errors.New("No folders to watch.") 32 | } 33 | 34 | watcher, err := fsnotify.NewWatcher() 35 | if err != nil { 36 | return nil, err 37 | } 38 | rw := &RecursiveWatcher{Watcher: watcher} 39 | 40 | rw.Files = make(chan structs.StateChange, 10) 41 | rw.Folders = make(chan string, len(folders)) 42 | 43 | for _, folder := range folders { 44 | rw.AddFolder(folder) 45 | } 46 | return rw, nil 47 | } 48 | 49 | func (watcher *RecursiveWatcher) AddFolder(folder string) { 50 | err := watcher.Add(folder) 51 | if err != nil { 52 | log.Println("Error watching: ", folder, err) 53 | } 54 | // watcher.Folders <- folder 55 | } 56 | 57 | func CreateLocalStateChange(path string, eventType int) (change structs.StateChange, err error) { 58 | fi, err := os.Stat(path) 59 | if err != nil { 60 | if !(eventType == DELETE && os.IsNotExist(err)) { 61 | fmt.Println("returning") 62 | return 63 | } 64 | err = nil 65 | } 66 | change.IsCreate = (eventType != DELETE) 67 | change.IsLocal = true 68 | change.File.Path = path 69 | if eventType != DELETE { 70 | change.File.Name = fi.Name() 71 | change.File.Size = fi.Size() 72 | change.File.Modified = fi.ModTime() 73 | // hmmm, what do we do if the file wasn't created? os.Stat doesn't provide created 74 | if eventType == CREATE { 75 | change.File.CreatedAt = fi.ModTime() 76 | } 77 | } 78 | return 79 | } 80 | 81 | func (watcher *RecursiveWatcher) Run(initScanDone <-chan struct{}, debug bool) { 82 | go func() { 83 | <-initScanDone 84 | fmt.Println("recieved init scan done signal") 85 | for { 86 | select { 87 | case event := <-watcher.Events: 88 | if ext := filepath.Ext(event.Name); ext == ".tmp" { 89 | continue 90 | } 91 | if strings.HasPrefix(event.Name, ".Gobox") { 92 | continue 93 | } 94 | // absPath, err := filepath.Abs(event.Name) 95 | // if err == nil { 96 | // event.Name = absPath 97 | // } 98 | 99 | // create a file/directory 100 | if event.Op&fsnotify.Create == fsnotify.Create { 101 | fi, err := os.Stat(event.Name) 102 | if err != nil { 103 | // eg. stat .subl513.tmp : no such file or directory 104 | if debug { 105 | // DebugError(err) 106 | } 107 | } else if fi.IsDir() { 108 | if debug { 109 | // DebugMessage("Detected new directory %s", event.Name) 110 | } 111 | if !shouldIgnoreFile(filepath.Base(event.Name)) { 112 | fmt.Println("adding folder: ", event.Name) 113 | watcher.AddFolder(event.Name) 114 | } 115 | } else { 116 | if debug { 117 | // DebugMessage("Detected new file %s", event.Name) 118 | } 119 | 120 | change, err := CreateLocalStateChange(event.Name, CREATE) 121 | if err != nil { 122 | continue 123 | } 124 | watcher.Files <- change 125 | } 126 | } 127 | 128 | if event.Op&fsnotify.Write == fsnotify.Write { 129 | // modified a file, assuming that you don't modify folders 130 | if debug { 131 | // DebugMessage("Detected file modification %s", event.Name) 132 | } 133 | 134 | change, err := CreateLocalStateChange(event.Name, MODIFY) 135 | if err != nil { 136 | continue 137 | } 138 | watcher.Files <- change 139 | } 140 | if event.Op&fsnotify.Remove == fsnotify.Remove { 141 | fmt.Println("Got the remove") 142 | change, err := CreateLocalStateChange(event.Name, DELETE) 143 | if err != nil { 144 | fmt.Println("error") 145 | continue 146 | } 147 | watcher.Files <- change 148 | 149 | } 150 | 151 | case err := <-watcher.Errors: 152 | log.Println("error", err) 153 | } 154 | } 155 | }() 156 | } 157 | 158 | // Subfolders returns a slice of subfolders (recursive), including the folder provided. 159 | func Subfolders(path string) (paths []string) { 160 | filepath.Walk(path, func(newPath string, info os.FileInfo, err error) error { 161 | if err != nil { 162 | return err 163 | } 164 | 165 | if info.IsDir() { 166 | name := info.Name() 167 | // skip folders that begin with a dot 168 | if shouldIgnoreFile(name) && name != "." && name != ".." { 169 | return filepath.SkipDir 170 | } 171 | paths = append(paths, newPath) 172 | } 173 | return nil 174 | }) 175 | return paths 176 | } 177 | 178 | // shouldIgnoreFile determines if a file should be ignored. 179 | // File names that begin with "." or "_" are ignored by the go tool. 180 | func shouldIgnoreFile(name string) bool { 181 | return strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") 182 | } 183 | -------------------------------------------------------------------------------- /cmd/gobox-client/.gitignore: -------------------------------------------------------------------------------- 1 | gobox-client 2 | -------------------------------------------------------------------------------- /cmd/gobox-client/fs_watcher.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // Interface used by the Fs Watcher 4 | type FsEventCollector interface { 5 | NewFsEvent(stateChange) 6 | } 7 | 8 | // Called by Watcher 9 | func (s collector) NewFsEvent(c stateChange) { 10 | s.collect <- c 11 | } 12 | 13 | // TODO move to tests 14 | var _ FsEventCollector = collector{} 15 | -------------------------------------------------------------------------------- /cmd/gobox-client/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | func main() { 4 | 5 | } 6 | -------------------------------------------------------------------------------- /cmd/gobox-client/muxer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "log" 4 | 5 | type stateChange interface{} 6 | 7 | type collectHaltRequest struct { 8 | hasHalted <-chan struct{} 9 | } 10 | 11 | type collector struct { 12 | // Changes sent to this channel are collected 13 | collect chan<- stateChange 14 | 15 | // TODO needs to stop 16 | } 17 | 18 | // Starts a go routine that will collect state changes 19 | // and send them out the channel returned. 20 | func (s *collector) startCollecting() (changes <-chan stateChange) { 21 | collectCh := make(chan stateChange) 22 | s.collect = collectCh 23 | var newChange <-chan stateChange = collectCh 24 | 25 | changesOutCh := make(chan stateChange) 26 | changes = changesOutCh 27 | var fifo chan<- stateChange 28 | 29 | go func() { 30 | // buffer to collect changes in 31 | var changes []stateChange 32 | 33 | // first change to be sent out 34 | var change stateChange 35 | 36 | for { 37 | if change == nil && len(changes) > 0 { 38 | change = changes[0] 39 | changes = changes[1:] 40 | } 41 | 42 | if change == nil { 43 | select { 44 | case c := <-newChange: 45 | changes = append(changes, c) 46 | } 47 | 48 | } else { 49 | select { 50 | case c := <-newChange: 51 | changes = append(changes, c) 52 | 53 | case fifo <- change: 54 | change = nil 55 | } 56 | } 57 | } 58 | }() 59 | 60 | return changes 61 | } 62 | 63 | type changeId int 64 | 65 | type executionError struct { 66 | change executableChange 67 | err error 68 | } 69 | 70 | type executableChange interface { 71 | Id() changeId 72 | Exec(errCh chan<- executionError, doneCh chan<- executableChange) 73 | Halt() 74 | } 75 | 76 | type changeHaltRequest struct { 77 | hasHalted <-chan executableChange 78 | } 79 | 80 | type upload struct { 81 | id changeId 82 | stateChange 83 | 84 | requestHalt <-chan changeHaltRequest 85 | } 86 | 87 | func (c upload) Id() changeId { return c.id } 88 | func (c upload) Exec(errCh chan<- executionError, doneCh chan<- executableChange) { 89 | go func() { 90 | // TODO Do upload 91 | doneCh <- c 92 | }() 93 | } 94 | func (upload) Halt() { 95 | } 96 | 97 | type download struct { 98 | id changeId 99 | stateChange 100 | 101 | requestHalt <-chan changeHaltRequest 102 | } 103 | 104 | func (c download) Id() changeId { return c.id } 105 | func (c download) Exec(errCh chan<- executionError, doneCh chan<- executableChange) { 106 | go func() { 107 | // TODO Do download 108 | doneCh <- c 109 | }() 110 | } 111 | func (download) Halt() { 112 | } 113 | 114 | func newExecutableChange(id changeId, sc stateChange) executableChange { 115 | // TODO Process change, sc 116 | // - Is it an upload? 117 | // - Is it a download? 118 | 119 | // TODO return the correct type of exec change 120 | return upload{id, sc, nil} 121 | } 122 | 123 | type executorHaltRequest struct { 124 | hasHalted chan<- struct{} 125 | } 126 | 127 | type executor struct { 128 | // Used by the Halt() method to stop the executor 129 | requestHalt chan<- executorHaltRequest 130 | } 131 | 132 | func (e *executor) executeFrom(changesIn <-chan stateChange) { 133 | haltCh := make(chan executorHaltRequest) 134 | e.requestHalt = haltCh 135 | var haltRequested <-chan executorHaltRequest = haltCh 136 | 137 | go func() { 138 | errCh := make(chan executionError) 139 | var executionErr <-chan executionError = errCh 140 | 141 | doneCh := make(chan executableChange) 142 | var executionComplete <-chan executableChange = doneCh 143 | 144 | nextId := func() func() changeId { 145 | var nextId changeId = 0 146 | return func() changeId { 147 | defer func() { nextId++ }() 148 | id := nextId 149 | return id 150 | } 151 | }() 152 | 153 | runningChanges := make(map[changeId]executableChange) 154 | 155 | for { 156 | select { 157 | case c := <-changesIn: 158 | ec := newExecutableChange(nextId(), c) 159 | // TODO Should I cancel any running changes because of ec? 160 | 161 | // Non blocking 162 | ec.Exec(errCh, doneCh) 163 | 164 | runningChanges[ec.Id()] = ec 165 | 166 | case e := <-executionErr: 167 | log.Println(e.err) 168 | 169 | case c := <-executionComplete: 170 | delete(runningChanges, c.Id()) 171 | 172 | case r := <-haltRequested: 173 | // TODO Cleanup 174 | r.hasHalted <- struct{}{} 175 | } 176 | } 177 | }() 178 | } 179 | 180 | func (e executor) halt() { 181 | hasHalted := make(chan struct{}) 182 | e.requestHalt <- executorHaltRequest{hasHalted} 183 | <-hasHalted 184 | } 185 | -------------------------------------------------------------------------------- /cmd/gobox-client/spec_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/ghthor/gospec" 7 | ) 8 | 9 | func TestUnitSpecs(t *testing.T) { 10 | r := gospec.NewRunner() 11 | 12 | r.AddSpec(DescribeReaderWatcher) 13 | 14 | gospec.MainGoTest(r, t) 15 | } 16 | -------------------------------------------------------------------------------- /cmd/gobox-client/udp_watcher.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | ) 7 | 8 | // Interface used by the UDP Listener 9 | type RemoteEventCollector interface { 10 | NewRemoteEvent(stateChange) 11 | } 12 | 13 | // Called by Server Notifications 14 | func (s collector) NewRemoteEvent(c stateChange) { 15 | s.collect <- c 16 | } 17 | 18 | // TODO move to tests 19 | var _ RemoteEventCollector = collector{} 20 | 21 | type readerWatcher struct { 22 | // TODO There should probly be some communication if 23 | // watcher encounters an error while it's running 24 | 25 | // Stores the error that caused watcher to exit 26 | err error 27 | } 28 | 29 | func (w *readerWatcher) watchFrom(r io.Reader, sink RemoteEventCollector) { 30 | go func() { 31 | scanner := bufio.NewScanner(r) 32 | scanner.Split(bufio.ScanLines) 33 | 34 | for scanner.Scan() { 35 | sink.NewRemoteEvent(stateChange(scanner.Text())) 36 | } 37 | 38 | w.err = scanner.Err() 39 | }() 40 | } 41 | -------------------------------------------------------------------------------- /cmd/gobox-client/watcher_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/ghthor/gospec" 7 | . "github.com/ghthor/gospec" 8 | ) 9 | 10 | type mockRemoteEventCollector struct { 11 | changes chan stateChange 12 | } 13 | 14 | func (c *mockRemoteEventCollector) NewRemoteEvent(sc stateChange) { 15 | c.changes <- sc 16 | } 17 | 18 | func DescribeReaderWatcher(c gospec.Context) { 19 | c.Specify("a io.Reader watcher", func() { 20 | c.Specify("generates state changes", func() { 21 | watcher := readerWatcher{} 22 | collector := mockRemoteEventCollector{make(chan stateChange)} 23 | 24 | watcher.watchFrom(strings.NewReader("update\n"), &collector) 25 | change := <-collector.changes 26 | 27 | c.Expect(change.(string), Equals, "update") 28 | c.Expect(watcher.err, IsNil) 29 | }) 30 | }) 31 | } 32 | -------------------------------------------------------------------------------- /fsnotifytest/fsnotifytest.go: -------------------------------------------------------------------------------- 1 | package recursive 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/go-fsnotify/fsnotify" 7 | "log" 8 | "os" 9 | "path/filepath" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | type RecursiveWatcher struct { 15 | *fsnotify.Watcher 16 | Files chan string 17 | Folders chan string 18 | } 19 | 20 | func NewRecursiveWatcher(path string) (*RecursiveWatcher, error) { 21 | folders := Subfolders(path) 22 | if len(folders) == 0 { 23 | return nil, errors.New("No folders to watch.") 24 | } 25 | 26 | watcher, err := fsnotify.NewWatcher() 27 | if err != nil { 28 | return nil, err 29 | } 30 | rw := &RecursiveWatcher{Watcher: watcher} 31 | 32 | rw.Files = make(chan string, 10) 33 | rw.Folders = make(chan string, len(folders)) 34 | 35 | for _, folder := range folders { 36 | rw.AddFolder(folder) 37 | } 38 | return rw, nil 39 | } 40 | 41 | func (watcher *RecursiveWatcher) AddFolder(folder string) { 42 | err := watcher.Add(folder) 43 | if err != nil { 44 | log.Println("Error watching: ", folder, err) 45 | } 46 | watcher.Folders <- folder 47 | } 48 | 49 | func (watcher *RecursiveWatcher) Run(debug bool) { 50 | go func() { 51 | for { 52 | select { 53 | case event := <-watcher.Events: 54 | // create a file/directory 55 | if event.Op&fsnotify.Create == fsnotify.Create { 56 | fi, err := os.Stat(event.Name) 57 | if err != nil { 58 | // eg. stat .subl513.tmp : no such file or directory 59 | if debug { 60 | // DebugError(err) 61 | } 62 | } else if fi.IsDir() { 63 | if debug { 64 | // DebugMessage("Detected new directory %s", event.Name) 65 | } 66 | if !shouldIgnoreFile(filepath.Base(event.Name)) { 67 | watcher.AddFolder(event.Name) 68 | } 69 | } else { 70 | if debug { 71 | // DebugMessage("Detected new file %s", event.Name) 72 | } 73 | watcher.Files <- event.Name // created a file 74 | } 75 | } 76 | 77 | if event.Op&fsnotify.Write == fsnotify.Write { 78 | // modified a file, assuming that you don't modify folders 79 | if debug { 80 | // DebugMessage("Detected file modification %s", event.Name) 81 | } 82 | watcher.Files <- event.Name 83 | } 84 | if event.Op&fsnotify.Remove == fsnotify.Remove { 85 | watcher.Files <- event.Name 86 | } 87 | 88 | case err := <-watcher.Errors: 89 | log.Println("error", err) 90 | } 91 | } 92 | }() 93 | } 94 | 95 | // Subfolders returns a slice of subfolders (recursive), including the folder provided. 96 | func Subfolders(path string) (paths []string) { 97 | filepath.Walk(path, func(newPath string, info os.FileInfo, err error) error { 98 | if err != nil { 99 | return err 100 | } 101 | 102 | if info.IsDir() { 103 | name := info.Name() 104 | // skip folders that begin with a dot 105 | if shouldIgnoreFile(name) && name != "." && name != ".." { 106 | return filepath.SkipDir 107 | } 108 | paths = append(paths, newPath) 109 | } 110 | return nil 111 | }) 112 | return paths 113 | } 114 | 115 | // shouldIgnoreFile determines if a file should be ignored. 116 | // File names that begin with "." or "_" are ignored by the go tool. 117 | func shouldIgnoreFile(name string) bool { 118 | return strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") 119 | } 120 | 121 | func main() { 122 | dir, err := filepath.Abs("/home/jdp/go/src/github.com/golangbox/gobox/test") 123 | if err != nil { 124 | log.Fatal(err) 125 | } 126 | rw, err := NewRecursiveWatcher(dir) 127 | if err != nil { 128 | log.Println(err.Error()) 129 | log.Fatal("Couldn't start a recursive watcher") 130 | 131 | } 132 | rw.Run(false) 133 | go func() { 134 | for { 135 | fileEv := <-rw.Files 136 | fmt.Println(fileEv) 137 | } 138 | }() 139 | go func() { 140 | for { 141 | foldEv := <-rw.Folders 142 | fmt.Println(foldEv) 143 | } 144 | }() 145 | 146 | for { 147 | time.Sleep(1000) 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "os/exec" 6 | "testing" 7 | "time" 8 | 9 | // "github.com/golangbox/gobox/boxtools" 10 | "github.com/golangbox/gobox/boxtools" 11 | "github.com/golangbox/gobox/server" 12 | "github.com/golangbox/gobox/server/model" 13 | "github.com/golangbox/gobox/structs" 14 | "github.com/jinzhu/gorm" 15 | ) 16 | 17 | func init() { 18 | model.DB, _ = gorm.Open("postgres", "dbname=goboxtest sslmode=disable") 19 | 20 | model.DB.DropTableIfExists(&structs.User{}) 21 | model.DB.DropTableIfExists(&structs.Client{}) 22 | model.DB.DropTableIfExists(&structs.FileAction{}) 23 | model.DB.DropTableIfExists(&structs.File{}) 24 | model.DB.DropTableIfExists(&structs.FileSystemFile{}) 25 | model.DB.AutoMigrate( 26 | &structs.User{}, 27 | &structs.Client{}, 28 | &structs.FileAction{}, 29 | &structs.File{}, 30 | &structs.FileSystemFile{}, 31 | ) 32 | 33 | } 34 | func TestEverything(t *testing.T) { 35 | go server.Run() 36 | time.Sleep(time.Second * 2) 37 | paths := []string{ 38 | "sandbox/client1/", 39 | "sandbox/client2/", 40 | } 41 | ignores := make(map[string]bool) 42 | for _, value := range paths { 43 | err := boxtools.CleanTestFolder(value, ignores, true) 44 | if err != nil { 45 | panic("Could not delete folder contents") 46 | } 47 | go func(value string) { 48 | cmd := exec.Command( 49 | "go", 50 | "run", 51 | "client/client.go", 52 | value) 53 | cmd.Stdout = os.Stdout 54 | cmd.Stderr = os.Stderr 55 | cmd.Run() 56 | }(value) 57 | } 58 | select {} 59 | } 60 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # GoBox 2 | 3 | GoBox is a Dropbox clone written in Go. 4 | 5 | The design is a client/server architecture using HTTP endpoints to communicate file change events between clients. A global journal of all file changes is kept on the server in a Postgres database. 6 | 7 | Local file changes are hashed and sent to the server, which discerns whether or not it has a file under that hash already. If not, the client uploads the file to the server, where the file is hashed to check for integrity, and if valid uploaded to an Amazon S3 instance. All other clients are alerted that a change has been made through a UDP socket, and then the other clients request the necessary changes through an HTTP endpoint. Clients then get the necessary changes directly from the Amazon S3 instance through an S3 signed URL. 8 | 9 | ## Notes 10 | - Must have `GOBOX_AWS_ACCESS_KEY_ID` and `GOBOX_AWS_SECRET_ACCESS_KEY` set for aws client. 11 | - os.FileMode struct has all the information me need to handle files. symlink, permission, directory, etc.... 12 | - https://blogs.dropbox.com/tech/2014/07/streaming-file-synchronization/ 13 | - https://www.youtube.com/watch?v=PE4gwstWhmc 14 | 15 | ## Api 16 | 17 | #### Server Endpoints: 18 | 19 | ##### POST: /file-actions/ 20 | 21 | ##### POST: /upload/ 22 | 23 | ##### POST: /download/ 24 | 25 | ##### POST: /clients/ 26 | 27 | ## Resources 28 | -------------------------------------------------------------------------------- /server/api/api.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/hex" 6 | "encoding/json" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "log" 11 | "net/http" 12 | "strconv" 13 | "text/template" 14 | 15 | "github.com/golangbox/gobox/UDPush" 16 | "github.com/golangbox/gobox/boxtools" 17 | "github.com/golangbox/gobox/server/model" 18 | "github.com/golangbox/gobox/server/s3" 19 | "github.com/golangbox/gobox/structs" 20 | "github.com/gorilla/mux" 21 | "github.com/jinzhu/gorm" 22 | ) 23 | 24 | var T *template.Template 25 | 26 | func RenderTemplate(w http.ResponseWriter, tmpl string, data interface{}) { 27 | T.ExecuteTemplate(w, tmpl+".html", data) 28 | } 29 | 30 | var Pusher *UDPush.Pusher 31 | 32 | func ServeServerRoutes(port string, pusher *UDPush.Pusher) { 33 | Pusher = pusher 34 | var err error 35 | T, err = template.ParseGlob("server/templates/*") 36 | _ = err 37 | r := mux.NewRouter() 38 | r.StrictSlash(true) 39 | 40 | // public 41 | r.HandleFunc("/", IndexHandler) 42 | r.HandleFunc("/login/", LoginHandler).Methods("GET") 43 | r.HandleFunc("/sign-up/", SignUpHandler).Methods("POST") 44 | r.HandleFunc("/file-data/{email}", FilesHandler).Methods("POST") 45 | r.HandleFunc("/download/{id}/{filename}", DownloadHandler).Methods("GET") 46 | 47 | // require client authentication 48 | r.HandleFunc("/file-actions/", sessionValidate(FileActionsHandler)).Methods("POST") 49 | r.HandleFunc("/upload/", sessionValidate(UploadHandler)).Methods("POST") 50 | r.HandleFunc("/download/", sessionValidate(FileDownloadHandler)).Methods("POST") 51 | r.HandleFunc("/clients/", sessionValidate(ClientsFileActionsHandler)).Methods("POST") 52 | 53 | // static files? (css, js, etc...) 54 | // r.PathPrefix("/").Handler(http.FileServer(http.Dir("./public/"))) 55 | 56 | http.Handle("/", r) 57 | 58 | fmt.Println("Serving api on port :" + port) 59 | http.ListenAndServe(":"+port, nil) 60 | } 61 | 62 | type httpError struct { 63 | err error 64 | code int 65 | responseWriter http.ResponseWriter 66 | } 67 | 68 | func (h *httpError) check() bool { 69 | if h.err != nil { 70 | h.responseWriter.WriteHeader(h.code) 71 | h.responseWriter.Write([]byte(h.err.Error())) 72 | log.Fatal(h.err) 73 | return true 74 | } else { 75 | return false 76 | } 77 | } 78 | 79 | func sessionValidate(fn func(http.ResponseWriter, *http.Request, structs.Client)) http.HandlerFunc { 80 | return func(w http.ResponseWriter, r *http.Request) { 81 | log.Printf("%s %s %s", r.RemoteAddr, r.Method, r.URL) 82 | client, err := verifyAndReturnClient(r) 83 | if err != nil { 84 | w.WriteHeader(http.StatusUnauthorized) 85 | w.Write([]byte(err.Error())) 86 | return 87 | } 88 | fn(w, r, client) 89 | } 90 | } 91 | 92 | func verifyAndReturnClient(req *http.Request) (client structs.Client, err error) { 93 | sessionKey := req.FormValue("SessionKey") 94 | if sessionKey == "" { 95 | err = fmt.Errorf("No session key with request") 96 | return 97 | } 98 | query := model.DB.Where("session_key = ?", sessionKey).First(&client) 99 | if query.Error != nil { 100 | err = query.Error 101 | return 102 | } 103 | if client.Id == 0 { 104 | err = fmt.Errorf("No client matching this session key") 105 | return 106 | } 107 | return 108 | } 109 | 110 | func FileActionsHandler(w http.ResponseWriter, req *http.Request, 111 | client structs.Client) { 112 | httpError := httpError{responseWriter: w} 113 | 114 | var contents []byte 115 | contents, httpError.err = ioutil.ReadAll(req.Body) 116 | httpError.code = http.StatusInternalServerError 117 | if httpError.check() { 118 | return 119 | } 120 | 121 | var fileActions []structs.FileAction 122 | httpError.err = json.Unmarshal(contents, &fileActions) 123 | httpError.code = http.StatusNotAcceptable 124 | if httpError.check() { 125 | return 126 | } 127 | 128 | for _, value := range fileActions { 129 | value.ClientId = client.Id 130 | } 131 | 132 | fileActions, httpError.err = boxtools.WriteFileActionsToDatabase(fileActions, client) 133 | httpError.code = http.StatusInternalServerError 134 | if httpError.check() { 135 | return 136 | } 137 | 138 | var user structs.User 139 | query := model.DB.Model(&client).Related(&user) 140 | httpError.err = query.Error 141 | if httpError.check() { 142 | return 143 | } 144 | 145 | Pusher.Notify(client.SessionKey) 146 | 147 | errs := boxtools.ApplyFileActionsToFileSystemFileTable(fileActions, user) 148 | if len(errs) != 0 { 149 | fmt.Println(errs) 150 | } 151 | 152 | var hashesThatNeedToBeUploaded []string 153 | hashMap := make(map[string]bool) 154 | // write to a map to remove any duplicate hashes 155 | for _, value := range fileActions { 156 | if value.IsCreate == true { 157 | hashMap[value.File.Hash] = true 158 | } 159 | } 160 | for key, _ := range hashMap { 161 | var exists bool 162 | exists, httpError.err = s3.TestKeyExistence(key) 163 | httpError.code = http.StatusInternalServerError 164 | if httpError.check() { 165 | return 166 | } 167 | if exists == false { 168 | hashesThatNeedToBeUploaded = append( 169 | hashesThatNeedToBeUploaded, 170 | key, 171 | ) 172 | } 173 | } 174 | 175 | var jsonBytes []byte 176 | jsonBytes, httpError.err = json.Marshal(hashesThatNeedToBeUploaded) 177 | httpError.code = http.StatusInternalServerError 178 | if httpError.check() { 179 | return 180 | } 181 | w.WriteHeader(http.StatusOK) 182 | w.Write(jsonBytes) 183 | } 184 | func UploadHandler(w http.ResponseWriter, req *http.Request, 185 | client structs.Client) { 186 | httpError := httpError{responseWriter: w} 187 | 188 | var contents []byte 189 | contents, httpError.err = ioutil.ReadAll(req.Body) 190 | httpError.code = http.StatusUnauthorized 191 | if httpError.check() { 192 | return 193 | } 194 | 195 | h := sha256.New() 196 | _, httpError.err = h.Write(contents) 197 | httpError.code = http.StatusInternalServerError 198 | if httpError.check() { 199 | return 200 | } 201 | byteString := h.Sum(nil) 202 | sha256String := hex.EncodeToString(byteString) 203 | 204 | // we have the hash, so we might as well check if it 205 | // exists again before we upload 206 | var exists bool 207 | exists, httpError.err = s3.TestKeyExistence(sha256String) 208 | if httpError.check() { 209 | return 210 | } 211 | if exists == false { 212 | httpError.err = s3.UploadFile(sha256String, contents) 213 | if httpError.check() { 214 | return 215 | } 216 | } 217 | w.WriteHeader(http.StatusOK) 218 | } 219 | 220 | func FileDownloadHandler(w http.ResponseWriter, req *http.Request, 221 | client structs.Client) { 222 | httpError := httpError{responseWriter: w} 223 | httpError.code = http.StatusInternalServerError 224 | 225 | var user structs.User 226 | query := model.DB.Model(&client).Related(&user) 227 | httpError.err = query.Error 228 | if httpError.check() { 229 | return 230 | } 231 | 232 | fileHash := req.FormValue("fileHash") 233 | if fileHash == "" { 234 | w.WriteHeader(http.StatusNotAcceptable) 235 | return 236 | } 237 | 238 | var file structs.File 239 | query = model.DB.Where( 240 | &structs.File{ 241 | UserId: user.Id, 242 | Hash: fileHash, 243 | }, 244 | ).First(&file) 245 | if query.Error != nil { 246 | if query.Error == gorm.RecordNotFound { 247 | w.WriteHeader(http.StatusUnauthorized) 248 | } else { 249 | w.WriteHeader(http.StatusInternalServerError) 250 | w.Write([]byte(query.Error.Error())) 251 | } 252 | return 253 | } 254 | 255 | var exists bool 256 | exists, httpError.err = s3.TestKeyExistence(fileHash) 257 | if httpError.check() { 258 | return 259 | } 260 | 261 | if exists != true { 262 | w.WriteHeader(http.StatusNoContent) 263 | return 264 | } 265 | 266 | var url string 267 | url, httpError.err = s3.GenerateSignedUrl(fileHash) 268 | if httpError.check() { 269 | return 270 | } 271 | w.WriteHeader(http.StatusOK) 272 | w.Write([]byte(url)) 273 | } 274 | 275 | func ClientsFileActionsHandler(w http.ResponseWriter, req *http.Request, 276 | client structs.Client) { 277 | httpError := httpError{responseWriter: w} 278 | httpError.code = http.StatusInternalServerError 279 | 280 | lastIdString := req.FormValue("lastId") 281 | if lastIdString == "" { 282 | httpError.err = fmt.Errorf("Need last fileAction Id.") 283 | if httpError.check() { 284 | return 285 | } 286 | } 287 | 288 | var user structs.User 289 | query := model.DB.Model(&client).Related(&user) 290 | httpError.err = query.Error 291 | if httpError.check() { 292 | return 293 | } 294 | 295 | var clients []structs.Client 296 | query = model.DB.Model(&user).Not("Id = ?", client.Id).Related(&clients, "clients") 297 | httpError.err = query.Error 298 | if httpError.check() { 299 | return 300 | } 301 | 302 | var clientIds []int64 303 | for _, value := range clients { 304 | clientIds = append(clientIds, value.Id) 305 | } 306 | 307 | var fileActions []structs.FileAction 308 | query = model.DB.Where("client_id in (?)", clientIds). 309 | Where("Id > ?", lastIdString). 310 | Find(&fileActions) 311 | httpError.err = query.Error 312 | if httpError.check() { 313 | return 314 | } 315 | 316 | var highestId int64 317 | for _, value := range fileActions { 318 | if value.Id > highestId { 319 | highestId = value.Id 320 | } 321 | } 322 | 323 | fileActions = boxtools.RemoveRedundancyFromFileActions(fileActions) 324 | 325 | for key, value := range fileActions { 326 | var file structs.File 327 | _ = model.DB.First(&file, value.FileId) 328 | fileActions[key].File = file 329 | } 330 | 331 | responseStruct := structs.ClientFileActionsResponse{ 332 | LastId: highestId, 333 | FileActions: fileActions, 334 | } 335 | 336 | var responseJsonBytes []byte 337 | responseJsonBytes, httpError.err = json.Marshal(responseStruct) 338 | if httpError.check() { 339 | return 340 | } 341 | w.Write(responseJsonBytes) 342 | 343 | model.DB.Save(&client) 344 | 345 | } 346 | 347 | func FilesHandler(w http.ResponseWriter, req *http.Request) { 348 | vars := mux.Vars(req) 349 | email := vars["email"] 350 | 351 | var user structs.User 352 | query := model.DB.Where("email = ?", email).First(&user) 353 | 354 | var files []structs.FileSystemFile 355 | query = model.DB.Where("user_id = ?", user.Id).Find(&files) 356 | 357 | for i, value := range files { 358 | query = model.DB.First(&files[i].File, value.FileId) 359 | } 360 | 361 | if query.Error != nil { 362 | panic(query.Error) 363 | } 364 | jsonBytes, _ := json.Marshal(files) 365 | w.Write(jsonBytes) 366 | } 367 | 368 | func DownloadHandler(w http.ResponseWriter, req *http.Request) { 369 | vars := mux.Vars(req) 370 | id := vars["id"] 371 | intId, _ := strconv.Atoi(id) 372 | int64Id := int64(intId) 373 | 374 | var file structs.File 375 | query := model.DB.First(&file, int64Id) 376 | _ = query 377 | url, err := s3.GenerateSignedUrl(file.Hash) 378 | resp, err := http.Get(url) 379 | // keyBytes, err := ioutil.ReadAll(resp.Body) 380 | w.Header().Add("Content-Type", "application/octet-stream") 381 | io.Copy(w, resp.Body) 382 | _ = err 383 | // w.Write(keyBytes) 384 | 385 | // w.Write() 386 | // fmt.Println(query.Error) 387 | } 388 | 389 | func IndexHandler(w http.ResponseWriter, req *http.Request) { 390 | RenderTemplate(w, "index", nil) 391 | } 392 | 393 | func SignUpHandler(w http.ResponseWriter, req *http.Request) { 394 | // wants username and pass1 and pass2 posted as a form? 395 | // returns 200 or some sort of error to client? 396 | 397 | } 398 | 399 | func LoginHandler(w http.ResponseWriter, req *http.Request) { 400 | httpError := httpError{responseWriter: w} 401 | httpError.code = http.StatusInternalServerError 402 | 403 | var user structs.User 404 | model.DB.First(&user) 405 | client, err := boxtools.NewClient(user, "stephen", false) 406 | httpError.err = err 407 | if httpError.check() { 408 | return 409 | } 410 | 411 | w.Write([]byte(client.SessionKey)) 412 | } 413 | -------------------------------------------------------------------------------- /server/api/api_test.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "bytes" 5 | "crypto/sha256" 6 | "encoding/hex" 7 | "encoding/json" 8 | "fmt" 9 | "io/ioutil" 10 | "net/http" 11 | "net/url" 12 | "testing" 13 | 14 | "github.com/golangbox/gobox/boxtools" 15 | "github.com/golangbox/gobox/server/model" 16 | "github.com/golangbox/gobox/server/s3" 17 | "github.com/golangbox/gobox/structs" 18 | "github.com/jinzhu/gorm" 19 | ) 20 | 21 | var user structs.User 22 | var client structs.Client 23 | 24 | func init() { 25 | model.DB, _ = gorm.Open("postgres", "dbname=goboxtest sslmode=disable") 26 | 27 | model.DB.DropTableIfExists(&structs.User{}) 28 | model.DB.DropTableIfExists(&structs.Client{}) 29 | model.DB.DropTableIfExists(&structs.FileAction{}) 30 | model.DB.DropTableIfExists(&structs.File{}) 31 | model.DB.DropTableIfExists(&structs.FileSystemFile{}) 32 | model.DB.AutoMigrate( 33 | &structs.User{}, 34 | &structs.Client{}, 35 | &structs.FileAction{}, 36 | &structs.File{}, 37 | &structs.FileSystemFile{}, 38 | ) 39 | 40 | user, _ = boxtools.NewUser("max.t.mcdonnell@gmail", "password") 41 | 42 | var err error 43 | client, err = boxtools.NewClient(user, "test", false) 44 | if err != nil { 45 | fmt.Println(err) 46 | } 47 | 48 | go ServeServerRoutes("8000") 49 | } 50 | 51 | // func httpErrorCheck(err error, statusCode int, w http.ResponseWriter) 52 | 53 | func TestClientsFileActionsHandler(t *testing.T) { 54 | _, _ = boxtools.NewClient(user, "test", false) 55 | fileActions, _ := boxtools.GenerateSliceOfRandomFileActions(1, 1, 10) 56 | for _, value := range fileActions { 57 | model.DB.Create(&value) 58 | } 59 | resp, _ := http.PostForm( 60 | "http://localhost:8000/clients/", 61 | url.Values{ 62 | "sessionKey": {client.SessionKey}, 63 | "lastId": {"0"}, 64 | }, 65 | ) 66 | contents, _ := ioutil.ReadAll(resp.Body) 67 | if resp.StatusCode != 200 { 68 | t.Fail() 69 | } 70 | var incomingFileActions structs.ClientFileActionsResponse 71 | json.Unmarshal(contents, &incomingFileActions) 72 | 73 | if len(incomingFileActions.FileActions) != 10 { 74 | t.Fail() 75 | } 76 | } 77 | 78 | func TestUploadHandler(t *testing.T) { 79 | file := []byte("These are the file contents") 80 | resp, _ := http.Post( 81 | "http://localhost:8000/upload/?sessionKey="+client.SessionKey, 82 | "", 83 | bytes.NewBuffer(file), 84 | ) 85 | contents, _ := ioutil.ReadAll(resp.Body) 86 | if resp.StatusCode != 200 { 87 | err := fmt.Errorf(string(contents)) 88 | t.Error(err) 89 | } 90 | h := sha256.New() 91 | h.Write(file) 92 | byteString := h.Sum(nil) 93 | sha256String := hex.EncodeToString(byteString) 94 | 95 | url, _ := s3.GenerateSignedUrl(sha256String) 96 | 97 | resp, _ = http.Get(url) 98 | 99 | contents, _ = ioutil.ReadAll(resp.Body) 100 | 101 | if string(contents) != string(file) { 102 | t.Fail() 103 | } 104 | } 105 | 106 | func TestApiCallWithWrongAndNoAuth(t *testing.T) { 107 | resp, err := http.PostForm( 108 | "http://localhost:8000/file-actions/", 109 | url.Values{"sessionKey": {"nope"}}, 110 | ) 111 | if err != nil { 112 | t.Error(err) 113 | } 114 | if resp.StatusCode != 401 { 115 | t.Error("Wrong status code") 116 | } 117 | 118 | resp, err = http.Post( 119 | "http://localhost:8000/file-actions/", 120 | "text", 121 | bytes.NewBuffer([]byte("wheee")), 122 | ) 123 | if err != nil { 124 | t.Error(err) 125 | } 126 | if resp.StatusCode != 401 { 127 | t.Error("Wrong status code") 128 | } 129 | } 130 | 131 | func TestFileActionsHandler(t *testing.T) { 132 | fileActions, _ := boxtools.GenerateSliceOfRandomFileActions(1, 1, 2) 133 | var bothfileActions []structs.FileAction 134 | for _, value := range fileActions { 135 | bothfileActions = append(bothfileActions, value) 136 | bothfileActions = append(bothfileActions, value) 137 | } 138 | jsonBytes, _ := json.Marshal(bothfileActions) 139 | resp, _ := http.Post( 140 | "http://localhost:8000/file-actions/?sessionKey="+client.SessionKey, 141 | "application/json", 142 | bytes.NewBuffer(jsonBytes), 143 | ) 144 | contents, _ := ioutil.ReadAll(resp.Body) 145 | if resp.StatusCode != http.StatusOK { 146 | fmt.Println(resp) 147 | t.Fail() 148 | } 149 | _ = contents 150 | // var responseSlice []string 151 | // json.Unmarshal(contents, &responseSlice) 152 | // if len(responseSlice) != 2 { 153 | // fmt.Println(len(responseSlice)) 154 | // fmt.Println(responseSlice) 155 | // t.Fail() 156 | // } 157 | } 158 | 159 | func TestFileDownloadHandler(t *testing.T) { 160 | file, _ := boxtools.GenerateRandomFile(1) 161 | model.DB.Create(&file) 162 | resp, err := http.PostForm( 163 | "http://localhost:8000/download/", 164 | url.Values{"sessionKey": {client.SessionKey}, "fileHash": {file.Hash}}, 165 | ) 166 | if err != nil { 167 | t.Error(err) 168 | } 169 | contents, _ := ioutil.ReadAll(resp.Body) 170 | if resp.StatusCode != 200 { 171 | // t.Fail() 172 | } 173 | url := string(contents) 174 | _ = url 175 | //not sure how we check to see if the url is valid 176 | } 177 | -------------------------------------------------------------------------------- /server/model/model.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "github.com/jinzhu/gorm" 5 | _ "github.com/lib/pq" 6 | ) 7 | 8 | var DB gorm.DB 9 | 10 | func main() { 11 | // var err error 12 | // DB, err = gorm.Open("postgres", "dbname=gobox sslmode=disable") 13 | // if err != nil { 14 | // fmt.Println(err) 15 | // } 16 | // query := DB.AutoMigrate(&User{}, &Client{}) 17 | // if query.Error != nil { 18 | // fmt.Println(query.Error) 19 | // } 20 | // fmt.Println(query) 21 | 22 | // meta, err := convertJsonStringToMetaStruct(testJsonString) 23 | // fmt.Println(meta) 24 | // file, err := convertMetaStructToFileStruct(meta) 25 | // fmt.Println(file) 26 | } 27 | -------------------------------------------------------------------------------- /server/s3/s3.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | import ( 4 | "os" 5 | "time" 6 | 7 | "github.com/golangbox/goamz/aws" 8 | "github.com/golangbox/goamz/s3" 9 | ) 10 | 11 | var client *s3.S3 12 | var bucket *s3.Bucket 13 | 14 | func init() { 15 | key := os.Getenv("GOBOX_AWS_ACCESS_KEY_ID") 16 | secret := os.Getenv("GOBOX_AWS_SECRET_ACCESS_KEY") 17 | auth := aws.Auth{AccessKey: key, SecretKey: secret} 18 | client = s3.New(auth, aws.Regions["us-west-2"]) 19 | bucket = client.Bucket("gobox") 20 | } 21 | 22 | func TestKeyExistence(hash string) (exists bool, err error) { 23 | // this is expensive, both in terms of time and $ 24 | // maybe store the s3 values in a db? 25 | exists, err = bucket.Exists(hash) 26 | return exists, err 27 | } 28 | 29 | func GenerateSignedUrl(hash string) (url string, err error) { 30 | return bucket.SignedURL(hash, time.Now().Add(time.Duration(time.Minute*10))) 31 | } 32 | 33 | func UploadFile(hash string, fileBody []byte) error { 34 | var options s3.Options 35 | return bucket.Put(hash, fileBody, "", s3.PublicRead, options) 36 | } 37 | -------------------------------------------------------------------------------- /server/s3/s3_test.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | import ( 4 | "io/ioutil" 5 | "net/http" 6 | "testing" 7 | ) 8 | 9 | const ( 10 | validS3Key = "test" 11 | validS3KeyContent = "asdfasdfasdfasldkjfhalskdjhfalsjkdhflaksjdhflasjkdfha\ndsfa\nsdfhlasjdhflaskjdhflasjhdflkjh" 12 | ) 13 | 14 | func TestTestKeyExistence(t *testing.T) { 15 | var exists bool 16 | var err error 17 | 18 | exists, err = TestKeyExistence(validS3Key) 19 | if exists != true { 20 | t.Fail() 21 | } 22 | if err != nil { 23 | t.Error(err) 24 | } 25 | exists, err = TestKeyExistence("notvalid") 26 | if exists == true { 27 | t.Fail() 28 | } 29 | if err != nil { 30 | t.Error(err) 31 | } 32 | } 33 | 34 | func TestGenerateSignedUrl(t *testing.T) { 35 | 36 | url, err := GenerateSignedUrl(validS3Key) 37 | _ = url 38 | if err != nil { 39 | t.Error(err) 40 | } 41 | 42 | resp, err := http.Get(url) 43 | contents, err := ioutil.ReadAll(resp.Body) 44 | if validS3KeyContent != string(contents) { 45 | t.Fail() 46 | } 47 | } 48 | 49 | func TestFileUpload(t *testing.T) { 50 | testString := "file thing" 51 | file := []byte(testString) 52 | err := UploadFile("test2", file) 53 | if err != nil { 54 | t.Error(err) 55 | } 56 | getByte, err := bucket.Get("test2") 57 | if err != nil { 58 | t.Error(err) 59 | } 60 | if string(file) != string(getByte) { 61 | t.Fail() 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | 7 | "github.com/golangbox/gobox/UDPush" 8 | "github.com/golangbox/gobox/boxtools" 9 | "github.com/golangbox/gobox/server/api" 10 | ) 11 | 12 | type services struct { 13 | s3 bool 14 | api bool 15 | udpush bool 16 | } 17 | 18 | type server struct { 19 | services 20 | name string 21 | ip string 22 | port uint 23 | clientLimit uint 24 | status func() bool 25 | display func() string 26 | } 27 | 28 | func (s *server) checkStatus() bool { 29 | if s.services.api == true && 30 | s.services.s3 == true && 31 | s.services.udpush == true { 32 | return true 33 | } 34 | return false 35 | } 36 | 37 | func createDummyUser() error { 38 | user, err := boxtools.NewUser("gobox@gmail.com", "password") 39 | if err != nil { 40 | return err 41 | } 42 | _ = user 43 | return nil 44 | } 45 | 46 | //Run creates all the structures to make or project work 47 | func Run() { 48 | 49 | //Launch API 50 | 51 | s := server{ 52 | name: "Elvis", 53 | ip: "127.0.0.1", 54 | port: 4242, 55 | clientLimit: 10, 56 | } 57 | 58 | // model.DB, _ = gorm.Open( 59 | // "postgres", 60 | // "dbname=gobox sslmode=disable", 61 | // ) 62 | // model.DB.AutoMigrate( 63 | // &structs.User{}, 64 | // &structs.Client{}, 65 | // &structs.FileAction{}, 66 | // &structs.File{}, 67 | // &structs.FileSystemFile{}, 68 | // ) 69 | 70 | err := createDummyUser() 71 | if err != nil { 72 | log.Fatal(err) 73 | } 74 | ////Launch UDP notification service 75 | ////Define the Subject (The guy who is goin to hold all the clients) 76 | 77 | pusher := &UDPush.Pusher{ 78 | ServerID: s.ip, 79 | BindedTo: s.port, 80 | } 81 | 82 | go func() { 83 | err = pusher.InitUDPush() 84 | if err != nil { 85 | fmt.Println(err) 86 | } 87 | }() 88 | api.ServeServerRoutes("8000", pusher) 89 | } 90 | -------------------------------------------------------------------------------- /server/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 37 |
38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 |
NameFile IdSize
50 |
51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /server/templates/upload.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | File Upload Demo 5 | 6 | 7 | 8 |
9 |

File Upload Demo

10 |
{{.}}
11 | 17 |
18 | 19 | -------------------------------------------------------------------------------- /structs/structs.go: -------------------------------------------------------------------------------- 1 | package structs 2 | 3 | import "time" 4 | 5 | type StateChange struct { 6 | File File 7 | IsCreate bool 8 | IsLocal bool 9 | Quit <-chan bool 10 | Done chan<- interface{} 11 | Error chan<- interface{} 12 | PreviousHash string 13 | } 14 | 15 | type CurrentAction struct { 16 | Quit chan<- bool 17 | IsCreate bool 18 | IsLocal bool 19 | } 20 | 21 | type ClientFileActionsResponse struct { 22 | LastId int64 23 | FileActions []FileAction 24 | } 25 | 26 | type ErrorMessage struct { 27 | Error error 28 | File File 29 | Function string 30 | } 31 | 32 | type FileSystemState struct { 33 | FileActionId int64 34 | State map[string]File 35 | } 36 | 37 | type User struct { 38 | Id int64 39 | Email string `sql:"type:text;"` 40 | HashedPassword string 41 | CreatedAt time.Time 42 | UpdatedAt time.Time 43 | DeletedAt time.Time 44 | } 45 | 46 | type Client struct { 47 | Id int64 48 | UserId int64 49 | SessionKey string 50 | Name string 51 | IsServer bool 52 | LastSynchedFileActionId int64 53 | CreatedAt time.Time 54 | UpdatedAt time.Time 55 | DeletedAt time.Time 56 | } 57 | 58 | type FileAction struct { 59 | Id int64 60 | ClientId int64 61 | IsCreate bool 62 | CreatedAt time.Time 63 | PreviousHash string 64 | File File 65 | FileId int64 66 | } 67 | 68 | type File struct { 69 | Id int64 70 | UserId int64 71 | Name string 72 | Hash string 73 | Size int64 74 | Modified time.Time 75 | Path string `sql:"type:text;"` 76 | CreatedAt time.Time 77 | } 78 | 79 | type FileSystemFile struct { 80 | Id int64 81 | UserId int64 82 | FileId int64 83 | Path string `sql:"type:text;"` 84 | File File 85 | } 86 | --------------------------------------------------------------------------------