├── README.md ├── chunk-upload.go ├── go.mod └── main.go /README.md: -------------------------------------------------------------------------------- 1 | # Example Chunk Upload In Go 2 | 3 | This is the accompanying code to this [blog post](https://www.threeaccents.com/posts/handling-large-file-uploads-in-go/). 4 | -------------------------------------------------------------------------------- /chunk-upload.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "mime/multipart" 9 | "net/http" 10 | "os" 11 | "sort" 12 | "strconv" 13 | ) 14 | 15 | const ( 16 | maxChunkSize = int64(5 << 20) // 5MB 17 | 18 | uploadDir = "./data/chunks" 19 | ) 20 | 21 | // Chunk is a chunk of a file. 22 | // It contains information to be able to put the full file back together 23 | // when all file chunks have been uploaded. 24 | type Chunk struct { 25 | UploadID string // unique id for the current upload. 26 | ChunkNumber int32 27 | TotalChunks int32 28 | TotalFileSize int64 // in bytes 29 | Filename string 30 | Data io.Reader 31 | UploadDir string 32 | } 33 | 34 | // ProcessChunk will parse the chunk data from the request and store in a file on disk. 35 | func ProcessChunk(r *http.Request) error { 36 | chunk, err := ParseChunk(r) 37 | if err != nil { 38 | return fmt.Errorf("failed to parse chunk %w", err) 39 | } 40 | 41 | // Let's create the dir to store the file chunks. 42 | if err := os.MkdirAll(chunk.UploadID, 02750); err != nil { 43 | return err 44 | } 45 | 46 | if err := StoreChunk(chunk); err != nil { 47 | return err 48 | } 49 | 50 | return nil 51 | } 52 | 53 | // CompleteChunk rebulds the file chunks into the original full file. 54 | // It then stores the file on disk. 55 | func CompleteChunk(uploadID, filename string) error { 56 | uploadDir := fmt.Sprintf("%s/%s", uploadDir, uploadID) 57 | 58 | f, err := RebuildFile(uploadDir) 59 | if err != nil { 60 | return fmt.Errorf("failed to rebuild file %w", err) 61 | } 62 | // we might also want to delete the temp file from disk. 63 | // It would be handy to create a struct with a close method that closes and deletes the temp file. 64 | defer f.Close() 65 | 66 | // here we can just keep our file on disk 67 | // or do any processing we want such as resizing, tagging, storing in a cloud storage. 68 | // to keep this simple we'll just store the file on disk. 69 | 70 | newFile, err := os.Create(filename) 71 | if err != nil { 72 | return fmt.Errorf("failed creating file %w", err) 73 | } 74 | defer newFile.Close() 75 | 76 | if _, err := io.Copy(newFile, f); err != nil { 77 | return fmt.Errorf("failed copying file contents %w", err) 78 | } 79 | 80 | return nil 81 | } 82 | 83 | // ParseChunk parse the request body and creates our chunk struct. It expects the data to be sent in a 84 | // specific order and handles validating the order. 85 | func ParseChunk(r *http.Request) (*Chunk, error) { 86 | var chunk Chunk 87 | 88 | buf := new(bytes.Buffer) 89 | 90 | reader, err := r.MultipartReader() 91 | if err != nil { 92 | return nil, err 93 | } 94 | 95 | // start readings parts 96 | // 1. upload id 97 | // 2. chunk number 98 | // 3. total chunks 99 | // 4. total file size 100 | // 5. file name 101 | // 6. chunk data 102 | 103 | // 1 104 | if err := getPart("upload_id", reader, buf); err != nil { 105 | return nil, err 106 | } 107 | 108 | chunk.UploadID = buf.String() 109 | buf.Reset() 110 | 111 | // dir to where we store our chunk 112 | chunk.UploadDir = fmt.Sprintf("%s/%s", uploadDir, chunk.UploadID) 113 | 114 | // 2 115 | if err := getPart("chunk_number", reader, buf); err != nil { 116 | return nil, err 117 | } 118 | 119 | parsedChunkNumber, err := strconv.ParseInt(buf.String(), 10, 32) 120 | if err != nil { 121 | return nil, err 122 | } 123 | 124 | chunk.ChunkNumber = int32(parsedChunkNumber) 125 | buf.Reset() 126 | 127 | // 3 128 | if err := getPart("total_chunks", reader, buf); err != nil { 129 | return nil, err 130 | } 131 | 132 | parsedTotalChunksNumber, err := strconv.ParseInt(buf.String(), 10, 32) 133 | if err != nil { 134 | return nil, err 135 | } 136 | 137 | chunk.TotalChunks = int32(parsedTotalChunksNumber) 138 | buf.Reset() 139 | 140 | // 4 141 | if err := getPart("total_file_size", reader, buf); err != nil { 142 | return nil, err 143 | } 144 | 145 | parsedTotalFileSizeNumber, err := strconv.ParseInt(buf.String(), 10, 64) 146 | if err != nil { 147 | return nil, err 148 | } 149 | 150 | chunk.TotalFileSize = parsedTotalFileSizeNumber 151 | buf.Reset() 152 | 153 | // 5 154 | if err := getPart("file_name", reader, buf); err != nil { 155 | return nil, err 156 | } 157 | 158 | chunk.Filename = buf.String() 159 | buf.Reset() 160 | 161 | // 6 162 | part, err := reader.NextPart() 163 | if err != nil { 164 | return nil, fmt.Errorf("failed reading chunk part %w", err) 165 | } 166 | 167 | chunk.Data = part 168 | 169 | return &chunk, nil 170 | } 171 | 172 | // StoreChunk stores the chunk on disk for it to later be processed when all other file chunks have been uploaded. 173 | func StoreChunk(chunk *Chunk) error { 174 | chunkFile, err := os.Create(fmt.Sprintf("%s/%d", chunk.UploadDir, chunk.ChunkNumber)) 175 | if err != nil { 176 | return err 177 | } 178 | 179 | if _, err := io.CopyN(chunkFile, chunk.Data, maxChunkSize); err != nil && err != io.EOF { 180 | return err 181 | } 182 | 183 | return nil 184 | } 185 | 186 | // ByChunk is a helper type to sort the files by name. Since the name of the file is it's chunk number 187 | // it makes rebuilding the file a trivial task. 188 | type ByChunk []os.FileInfo 189 | 190 | func (a ByChunk) Len() int { return len(a) } 191 | func (a ByChunk) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 192 | func (a ByChunk) Less(i, j int) bool { 193 | ai, _ := strconv.Atoi(a[i].Name()) 194 | aj, _ := strconv.Atoi(a[j].Name()) 195 | return ai < aj 196 | } 197 | 198 | // RebuildFile grabs all the files from the directory passed on concantinates them to build the original file. 199 | // It stores the file contents in a temp file and returns it. 200 | func RebuildFile(dir string) (*os.File, error) { 201 | fileInfos, err := ioutil.ReadDir(uploadDir) 202 | if err != nil { 203 | return nil, err 204 | } 205 | 206 | fullFile, err := ioutil.TempFile("", "fullfile-") 207 | if err != nil { 208 | return nil, err 209 | } 210 | 211 | sort.Sort(ByChunk(fileInfos)) 212 | for _, fs := range fileInfos { 213 | if err := appendChunk(uploadDir, fs, fullFile); err != nil { 214 | return nil, err 215 | } 216 | } 217 | 218 | if err := os.RemoveAll(uploadDir); err != nil { 219 | return nil, err 220 | } 221 | 222 | return fullFile, nil 223 | } 224 | 225 | func appendChunk(uploadDir string, fs os.FileInfo, fullFile *os.File) error { 226 | src, err := os.Open(uploadDir + "/" + fs.Name()) 227 | if err != nil { 228 | return err 229 | } 230 | defer src.Close() 231 | if _, err := io.Copy(fullFile, src); err != nil { 232 | return err 233 | } 234 | 235 | return nil 236 | } 237 | 238 | func getPart(expectedPart string, reader *multipart.Reader, buf *bytes.Buffer) error { 239 | part, err := reader.NextPart() 240 | if err != nil { 241 | return fmt.Errorf("failed reading %s part %w", expectedPart, err) 242 | } 243 | 244 | if part.FormName() != expectedPart { 245 | return fmt.Errorf("invalid form name for part. Expected %s got %s", expectedPart, part.FormName()) 246 | } 247 | 248 | if _, err := io.Copy(buf, part); err != nil { 249 | return fmt.Errorf("failed copying %s part %w", expectedPart, err) 250 | } 251 | 252 | return nil 253 | } 254 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/threeaccents/large-file-upload-example 2 | 3 | go 1.14 4 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | ) 8 | 9 | func main() { 10 | http.Handle("/upload-chunk", handleUploadChunk()) 11 | http.Handle("/completed-chunks", handleCompletedChunk()) 12 | 13 | log.Fatal(http.ListenAndServe(":8080", nil)) 14 | } 15 | 16 | func handleUploadChunk() http.Handler { 17 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 18 | if err := ProcessChunk(r); err != nil { 19 | http.Error(w, err.Error(), http.StatusInternalServerError) 20 | return 21 | } 22 | 23 | w.Write([]byte("chunk processed")) 24 | }) 25 | } 26 | func handleCompletedChunk() http.Handler { 27 | type request struct { 28 | UploadID string `json:"uploadId"` 29 | Filename string `json:"filename"` 30 | } 31 | 32 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 33 | var payload request 34 | if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { 35 | http.Error(w, err.Error(), http.StatusBadRequest) 36 | return 37 | } 38 | 39 | // validate payload 40 | 41 | if err := CompleteChunk(payload.UploadID, payload.Filename); err != nil { 42 | http.Error(w, err.Error(), http.StatusInternalServerError) 43 | return 44 | } 45 | 46 | w.Write([]byte("file processed")) 47 | }) 48 | } 49 | --------------------------------------------------------------------------------