├── LICENSE ├── README.md └── main.go /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 daedaleanai 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Rsync Custom Transfer Agent for Git LFS 2 | 3 | The rsync [git-lfs](https://git-lfs.github.com/) custom transfer agent allows 4 | transferring the data through rsync, for example using SSH authentication. 5 | 6 | Natively, git-lfs allows transferring data through HTTPS only. 7 | This means to use git-lfs with your standard git repository, 8 | you need a separate git-lfs server for storing the large files. 9 | Unfortunately there is no official server implementation. 10 | 11 | Luckily git-lfs has a Custom Transfer Adapter which allows using [Custom Transfer](https://github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md) 12 | Agents for transferring the data! 13 | 14 | To use it, add to your `.lfsconfig` 15 | [config file](https://github.com/git-lfs/git-lfs/blob/master/docs/man/git-lfs-config.5.ronn) 16 | something like: 17 | ``` 18 | $ git config --replace-all lfs.concurrenttransfers 8 19 | $ git config --replace-all lfs.standalonetransferagent rsync 20 | $ git config --replace-all lfs.customtransfer.rsync.path git-lfs-rsync-agent 21 | $ git config --replace-all lfs.customtransfer.rsync.concurrent true 22 | $ git config --replace-all lfs.customtransfer.rsync.args SERVER:PATH 23 | ``` 24 | 25 | When pushing a branch to the origin repo, git-lfs tries to contact the inexistent 26 | git-lfs server and dies because it can't. If this happens, make sure the locks 27 | verify feature is disabled: 28 | ``` 29 | $ git config --replace-all lfs.locksverify false 30 | 31 | ``` 32 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // rsync custom agent for git-lfs 2 | 3 | package main 4 | 5 | import ( 6 | "bufio" 7 | "encoding/json" 8 | "fmt" 9 | "io/ioutil" 10 | "os" 11 | "os/exec" 12 | "strings" 13 | "time" 14 | ) 15 | 16 | var remote string 17 | 18 | func main() { 19 | scanner := bufio.NewScanner(os.Stdin) 20 | writer := bufio.NewWriter(os.Stdout) 21 | errWriter := bufio.NewWriter(os.Stderr) 22 | 23 | for scanner.Scan() { 24 | line := scanner.Text() 25 | var req request 26 | if err := json.Unmarshal([]byte(line), &req); err != nil { 27 | writeToStderr(fmt.Sprintf("Unable to parse request: %v\n", line), errWriter) 28 | continue 29 | } 30 | 31 | switch req.Event { 32 | case "init": 33 | writeToStderr(fmt.Sprintf("Initialising rsync agent for: %s\n", req.Operation), errWriter) 34 | initAgent(writer, errWriter) 35 | case "download": 36 | writeToStderr(fmt.Sprintf("Received download request for: %s\n", req.Oid), errWriter) 37 | performDownload(req.Oid, req.Size, req.Action, writer, errWriter) 38 | case "upload": 39 | writeToStderr(fmt.Sprintf("Received upload request for: %s\n", req.Oid), errWriter) 40 | performUpload(req.Oid, req.Size, req.Action, req.Path, writer, errWriter) 41 | case "terminate": 42 | writeToStderr("Terminating rsync agent gracefully.\n", errWriter) 43 | // Nothing to do. 44 | } 45 | } 46 | } 47 | 48 | func writeToStderr(msg string, errWriter *bufio.Writer) { 49 | if !strings.HasSuffix(msg, "\n") { 50 | msg = msg + "\n" 51 | } 52 | errWriter.WriteString(msg) 53 | errWriter.Flush() 54 | } 55 | 56 | func sendResponse(r interface{}, writer, errWriter *bufio.Writer) error { 57 | b, err := json.Marshal(r) 58 | if err != nil { 59 | return err 60 | } 61 | // Line oriented JSON 62 | b = append(b, '\n') 63 | _, err = writer.Write(b) 64 | if err != nil { 65 | return err 66 | } 67 | writer.Flush() 68 | writeToStderr(fmt.Sprintf("Sent message %v", string(b)), errWriter) 69 | return nil 70 | } 71 | 72 | func sendTransferError(oid string, code int, message string, writer, errWriter *bufio.Writer) { 73 | resp := &transferResponse{"complete", oid, "", &operationError{code, message}} 74 | err := sendResponse(resp, writer, errWriter) 75 | if err != nil { 76 | writeToStderr(fmt.Sprintf("Unable to send transfer error: %v\n", err), errWriter) 77 | } 78 | } 79 | 80 | func sendProgress(oid string, bytesSoFar int64, bytesSinceLast int, writer, errWriter *bufio.Writer) { 81 | resp := &progressResponse{"progress", oid, bytesSoFar, bytesSinceLast} 82 | err := sendResponse(resp, writer, errWriter) 83 | if err != nil { 84 | writeToStderr(fmt.Sprintf("Unable to send progress update: %v\n", err), errWriter) 85 | } 86 | } 87 | 88 | func initAgent(writer, errWriter *bufio.Writer) { 89 | // Make sure we have a remote. 90 | remote = os.Args[1] 91 | if remote == "" { 92 | resp := &initResponse{&operationError{3, "No remote specified when launching the process"}} 93 | sendResponse(resp, writer, errWriter) 94 | return 95 | } 96 | // Success! 97 | resp := &initResponse{} 98 | sendResponse(resp, writer, errWriter) 99 | } 100 | 101 | func performDownload(oid string, size int64, a *action, writer, errWriter *bufio.Writer) { 102 | dlFile, err := ioutil.TempFile("", "rsync-agent") 103 | if err != nil { 104 | sendTransferError(oid, 3, err.Error(), writer, errWriter) 105 | return 106 | } 107 | defer dlFile.Close() 108 | dlfilename := dlFile.Name() 109 | 110 | if err = rsync(remoteFile(oid), dlfilename); err != nil { 111 | sendTransferError(oid, 4, err.Error(), writer, errWriter) 112 | return 113 | } 114 | 115 | complete := &transferResponse{"complete", oid, dlfilename, nil} 116 | err = sendResponse(complete, writer, errWriter) 117 | if err != nil { 118 | writeToStderr(fmt.Sprintf("Unable to send completion message: %v\n", err), errWriter) 119 | } 120 | } 121 | 122 | func performUpload(oid string, size int64, a *action, fromPath string, writer, errWriter *bufio.Writer) { 123 | var err error 124 | for i := 0; i < 5; i++ { 125 | err = rsync("--chmod=g+r", fromPath, remoteFile(oid)) 126 | if err == nil { 127 | break 128 | } 129 | } 130 | if err != nil { 131 | sendTransferError(oid, 5, err.Error(), writer, errWriter) 132 | return 133 | } 134 | 135 | complete := &transferResponse{"complete", oid, "", nil} 136 | if err := sendResponse(complete, writer, errWriter); err != nil { 137 | writeToStderr(fmt.Sprintf("Unable to send completion message: %v\n", err), errWriter) 138 | } 139 | } 140 | 141 | func remoteFile(oid string) string { 142 | return fmt.Sprintf("%s/%s", remote, oid) 143 | } 144 | 145 | func rsync(args ...string) error { 146 | cmd := exec.Command("rsync", args...) 147 | out, err := cmd.CombinedOutput() 148 | if err != nil { 149 | return fmt.Errorf("Error while running `rsync %s`: %v\n%s", args, err, out) 150 | } 151 | return nil 152 | } 153 | 154 | type header struct { 155 | Key string `json:"key"` 156 | Value string `json:"value"` 157 | } 158 | type action struct { 159 | Href string `json:"href"` 160 | Header map[string]string `json:"header,omitempty"` 161 | ExpiresAt time.Time `json:"expires_at,omitempty"` 162 | } 163 | type operationError struct { 164 | Code int `json:"code"` 165 | Message string `json:"message"` 166 | } 167 | 168 | // Combined request struct which can accept anything 169 | type request struct { 170 | Event string `json:"event"` 171 | Operation string `json:"operation"` 172 | Concurrent bool `json:"concurrent"` 173 | ConcurrentTransfers int `json:"concurrenttransfers"` 174 | Oid string `json:"oid"` 175 | Size int64 `json:"size"` 176 | Path string `json:"path"` 177 | Action *action `json:"action"` 178 | } 179 | 180 | type initResponse struct { 181 | Error *operationError `json:"error,omitempty"` 182 | } 183 | type transferResponse struct { 184 | Event string `json:"event"` 185 | Oid string `json:"oid"` 186 | Path string `json:"path,omitempty"` // always blank for upload 187 | Error *operationError `json:"error,omitempty"` 188 | } 189 | type progressResponse struct { 190 | Event string `json:"event"` 191 | Oid string `json:"oid"` 192 | BytesSoFar int64 `json:"bytesSoFar"` 193 | BytesSinceLast int `json:"bytesSinceLast"` 194 | } 195 | --------------------------------------------------------------------------------