├── .codecov.yml ├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── auth_test.go ├── config.go ├── db.go ├── filedropd ├── .gitignore ├── filedropd.example.yml ├── filedropd.service └── main.go ├── global_limits_test.go ├── go.mod ├── go.sum ├── mysql.go ├── perfile_limits_test.go ├── postgresql.go ├── server.go ├── server_test.go ├── sqlite3.go └── utils_test.go /.codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | status: 3 | project: 4 | default: 5 | target: auto 6 | threshold: 5 7 | base: auto 8 | # advanced 9 | branches: null 10 | if_no_uploads: error 11 | if_not_found: success 12 | if_ci_failed: error 13 | only_pulls: false 14 | flags: null 15 | paths: null 16 | patch: off 17 | comment: 18 | layout: "diff, files" 19 | behavior: default 20 | require_changes: false # if true: only post the comment if coverage changes 21 | require_base: no # [yes :: must have a base report to post] 22 | require_head: yes # [yes :: must have a head report to post] 23 | branches: null # branch names that can post comment 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | coverage.txt 2 | .idea/ 3 | *.db 4 | *.exe 5 | filedropd.yml 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: go 3 | 4 | go: 5 | - "1.x" 6 | 7 | matrix: 8 | include: 9 | - env: GO111MODULE=on TEST_DB=sqlite3 TEST_DSN=":memory:" 10 | - env: GO111MODULE=on TEST_DB=postgres TEST_DSN="user=postgres dbname=filedrop_test sslmode=disable" 11 | services: 12 | - postgresql 13 | before_install: 14 | - psql -c 'create database filedrop_test;' -U postgres 15 | 16 | script: 17 | - go test -race -coverprofile=coverage.txt -covermode=atomic -tags $TEST_DB 18 | 19 | after_success: 20 | - bash <(curl -s https://codecov.io/bash) 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright © Max Mazurov (fox.cpp) 2018 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 7 | of the Software, and to permit persons to whom the Software is furnished to do 8 | so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | filedrop 2 | ========== 3 | 4 | [![Travis CI](https://img.shields.io/travis/com/foxcpp/filedrop.svg?style=flat-square&logo=Linux)](https://travis-ci.com/foxcpp/filedrop) 5 | [![CodeCov](https://img.shields.io/codecov/c/github/foxcpp/filedrop.svg?style=flat-square)](https://codecov.io/gh/foxcpp/filedrop) 6 | [![Issues](https://img.shields.io/github/issues-raw/foxcpp/filedrop.svg?style=flat-square)](https://github.com/foxcpp/filedrop/issues) 7 | [![License](https://img.shields.io/github/license/foxcpp/filedrop.svg?style=flat-square)](https://github.com/foxcpp/filedrop/blob/master/LICENSE) 8 | 9 | Lightweight file storage server with HTTP API. 10 | 11 | ### Features 12 | - Painless configuration! You don't even have to `rewrite` requests on your reverse proxy! 13 | - Limits support! Link usage count, file size and storage time. 14 | - Embeddable! Can run as part of your application. 15 | 16 | You can use filedrop either as a standalone server or as a part of your application. 17 | In former case you want to check `filedropd` subpackage, in later case just 18 | import `filedrop` package and pass config stucture to `filedrop.New`, returned 19 | object implements `http.Handler` so you can use it how you like. 20 | 21 | ### Installation 22 | 23 | This repository uses Go 1.11 modules. Things may work with old `GOPATH` 24 | approach but we don't support it so don't report cryptic compilation errors 25 | caused by wrong dependency version. 26 | 27 | `master` branch contains code from latest (pre-)release. `dev` branch 28 | contains bleeding-edge code. You probably want to use one of [tagged 29 | releases](https://github.com/foxcpp/filedrop/releases). 30 | 31 | #### SQL drivers 32 | 33 | filedrop uses SQL database as a meta-information storage so you need a 34 | SQL driver for it to use. 35 | 36 | When building standalone server you may want to enable one of the 37 | supported SQL DBMS using build tags: 38 | * `postgres` for PostgreSQL 39 | * `sqlite3` for SQLite3 40 | * `mysql` for MySQL 41 | 42 | **Note:** No MS SQL Server support is planned. However if you would like 43 | to see it - PRs are welcome. 44 | 45 | When using filedrop as a library you are given more freedom. Just make 46 | sure that you import driver you use. 47 | 48 | #### Library 49 | 50 | Just use `github.com/foxcpp/filedrop` as any other library. Documentation 51 | is here: [godoc.org](https://godoc.org/github.com/foxcpp/filedrop). 52 | 53 | #### Standalone server 54 | 55 | See `fildropd` subdirectory. To start server you need a configuration 56 | file. See example [here](filedropd/filedropd.example.yml). It should be pretty 57 | straightforward. Then just pass path to configuration file in 58 | command-line arguments. 59 | 60 | ``` 61 | filedropd /etc/filedropd.yml 62 | ``` 63 | 64 | systemd unit file is included for your convenience. 65 | 66 | ### HTTP API 67 | 68 | POST single file to any endpoint to save it. 69 | For example: 70 | ``` 71 | POST /filedrop 72 | Content-Type: image/png 73 | Content-Length: XXXX 74 | ``` 75 | 76 | You will get response with full file URL (endpoint used to POST + UUID), like this one: 77 | ``` 78 | http://example.com/filedrop/41a8f78c-ce06-11e8-b2ed-b083fe9824ac 79 | ``` 80 | 81 | You can add anything as last component to URL to give it human-understandable meaning: 82 | ``` 83 | http://example.com/filedrop/41a8f78c-ce06-11e8-b2ed-b083fe9824ac/amazing-screenshot.png 84 | ``` 85 | However you can't add more than one component: 86 | ``` 87 | http://example.com/filedrop/41a8f78c-ce06-11e8-b2ed-b083fe9824ac/invalid/in/filedrop 88 | ``` 89 | 90 | You can specify `max-uses` and `store-time-secs` to override default settings 91 | from server configuration (however you can't set value higher then configured). 92 | 93 | ``` 94 | POST /filedrop/screenshot.png?max-uses=5&store-secs=3600 95 | ``` 96 | Following request will store file screenshot.png for one hour (3600 seconds) 97 | and allow it to be downloaded not more than 10 times. 98 | 99 | **Note** To get `https` scheme in URLs downstream server should set header 100 | `X-HTTPS-Downstream` to `1` (or you can also set HTTPSDownstream config option) 101 | 102 | ### Authorization 103 | 104 | When using filedrop as a library you can setup custom callbacks 105 | for access control. 106 | 107 | See `filedrop.AuthConfig` documentation. 108 | -------------------------------------------------------------------------------- /auth_test.go: -------------------------------------------------------------------------------- 1 | package filedrop_test 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/foxcpp/filedrop" 10 | ) 11 | 12 | var authDB = map[string]bool{ 13 | "foo": true, 14 | "bar": true, 15 | "baz": false, 16 | } 17 | 18 | func authCallback(r *http.Request) bool { 19 | return authDB[r.URL.Query().Get("authToken")] 20 | } 21 | 22 | func TestAccessDenied(t *testing.T) { 23 | conf := filedrop.Default 24 | conf.UploadAuth.Callback = authCallback 25 | conf.DownloadAuth.Callback = authCallback 26 | serv := initServ(conf) 27 | ts := httptest.NewServer(serv) 28 | defer cleanServ(serv) 29 | defer ts.Close() 30 | c := ts.Client() 31 | 32 | if !t.Run("upload (fail)", func(t *testing.T) { 33 | doPOSTFail(t, c, ts.URL+"/filedrop?authToken=baz", "text/plain", strings.NewReader(file)) 34 | }) { 35 | t.FailNow() 36 | } 37 | 38 | // Access check should be done before existence check to deter scanning. 39 | if !t.Run("download (fail)", func(t *testing.T) { 40 | doGETFail(t, c, ts.URL+"/filedrop/AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA/meow.txt?authToken=baz") 41 | }) { 42 | t.FailNow() 43 | } 44 | } 45 | 46 | func TestUploadAuth(t *testing.T) { 47 | conf := filedrop.Default 48 | conf.UploadAuth.Callback = authCallback 49 | conf.DownloadAuth.Callback = authCallback 50 | serv := initServ(conf) 51 | ts := httptest.NewServer(serv) 52 | defer cleanServ(serv) 53 | defer ts.Close() 54 | c := ts.Client() 55 | 56 | if !t.Run("upload", func(t *testing.T) { 57 | doPOST(t, c, ts.URL+"/filedrop?authToken=foo", "text/plain", strings.NewReader(file)) 58 | }) { 59 | t.FailNow() 60 | } 61 | 62 | if !t.Run("download (fail)", func(t *testing.T) { 63 | doGETFail(t, c, ts.URL+"/filedrop/AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA?authToken=baz") 64 | }) { 65 | t.FailNow() 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package filedrop 2 | 3 | import "net/http" 4 | 5 | type LimitsConfig struct { 6 | // MaxUses is how much much times file can be accessed. Note that it also counts HEAD requests 7 | // and incomplete downloads (byte-range requests). 8 | // Per-file max-uses parameter can't exceed this value but can be smaller. 9 | MaxUses uint `yaml:"max_uses"` 10 | 11 | // MaxStoreSecs specifies max time for which files will be stored on 12 | // filedrop server. Per-file store-secs parameter can't exceed this value but 13 | // can be smaller. 14 | MaxStoreSecs uint `yaml:"max_store_secs"` 15 | 16 | // MaxFileSize is a maximum file size in bytes that can uploaded to filedrop. 17 | MaxFileSize uint `yaml:"max_file_size"` 18 | } 19 | 20 | type DBConfig struct { 21 | // Driver is a database/sql driver name. 22 | Driver string `yaml:"driver"` 23 | 24 | // Data Source Name. 25 | DSN string `yaml:"dsn"` 26 | } 27 | 28 | type AuthConfig struct { 29 | // Callback is called to check access before processing any request. 30 | // If Callback is null, no check will be performed. 31 | Callback func(*http.Request) bool `yaml:"omitempty"` 32 | } 33 | 34 | type Config struct { 35 | // ListenOn specifies endpoint to listen on in format ADDR:PORT. Used only by filedropd. 36 | ListenOn string `yaml:"listen_on"` 37 | 38 | Limits LimitsConfig `yaml:"limits"` 39 | DB DBConfig `yaml:"db"` 40 | DownloadAuth AuthConfig `yaml:"download_auth"` 41 | UploadAuth AuthConfig `yaml:"upload_auth"` 42 | 43 | // StorageDir is where files will be saved on disk. 44 | StorageDir string `yaml:"storage_dir"` 45 | 46 | // HTTPSDownstream specifies whether filedrop should return links with https scheme or not. 47 | // Overridden by X-HTTPS-Downstream header. 48 | HTTPSDownstream bool `yaml:"https_downstream"` 49 | 50 | // AllowedOrigins specifies Access-Control-Allow-Origin header. 51 | AllowedOrigins string `yaml:"allowed_origins"` 52 | 53 | // Internal, used only for testing. Always 60 secs in production. 54 | CleanupIntervalSecs int `yaml:"-"` 55 | } 56 | 57 | var Default Config 58 | -------------------------------------------------------------------------------- /db.go: -------------------------------------------------------------------------------- 1 | package filedrop 2 | 3 | import ( 4 | "database/sql" 5 | "strconv" 6 | "strings" 7 | "time" 8 | ) 9 | 10 | type db struct { 11 | *sql.DB 12 | 13 | Driver, DSN string 14 | 15 | addFile *sql.Stmt 16 | remFile *sql.Stmt 17 | contentType *sql.Stmt 18 | 19 | addUse *sql.Stmt 20 | shouldDelete *sql.Stmt 21 | removeStaleFiles *sql.Stmt 22 | staleFiles *sql.Stmt 23 | } 24 | 25 | func openDB(driver, dsn string) (*db, error) { 26 | if driver == "sqlite3" { 27 | // We apply some tricks for SQLite to avoid "database is locked" errors. 28 | 29 | if !strings.HasPrefix(dsn, "file:") { 30 | dsn = "file:" + dsn 31 | } 32 | if !strings.Contains(dsn, "?") { 33 | dsn = dsn + "?" 34 | } 35 | dsn = dsn + "cache=shared&_journal=WAL&_busy_timeout=5000" 36 | } 37 | 38 | db := new(db) 39 | var err error 40 | db.DB, err = sql.Open(driver, dsn) 41 | if err != nil { 42 | return nil, err 43 | } 44 | 45 | db.Driver = driver 46 | db.DSN = dsn 47 | 48 | if driver == "mysql" { 49 | db.Exec(`SET SESSION TRANSACTION ISOLATION LEVEL SERIALIZABLE`) 50 | } 51 | if driver == "sqlite3" { 52 | // Also some optimizations for SQLite to make it FAA-A-A-AST. 53 | db.Exec(`PRAGMA auto_vacuum = INCREMENTAL`) 54 | db.Exec(`PRAGMA journal_mode = WAL`) 55 | db.Exec(`PRAGMA synchronous = NORMAL`) 56 | db.Exec(`PRAGMA cache_size = 5000`) 57 | } 58 | 59 | db.initSchema() 60 | db.initStmts() 61 | return db, nil 62 | } 63 | 64 | func (db *db) initSchema() { 65 | _, err := db.Exec(`CREATE TABLE IF NOT EXISTS filedrop ( 66 | uuid CHAR(36) PRIMARY KEY NOT NULL, 67 | contentType VARCHAR(255) DEFAULT NULL, 68 | uses INTEGER NOT NULL DEFAULT 0, 69 | maxUses INTEGER DEFAULT NULL, 70 | storeUntil BIGINT DEFAULT NULL 71 | )`) 72 | if err != nil { 73 | panic(err) 74 | } 75 | } 76 | 77 | func (db *db) reformatBindvars(raw string) (res string) { 78 | // THIS IS VERY LIMITED IMPLEMENTATION. 79 | // If someday this will become not enough - just switch to https://github.com/jmoiron/sqlx. 80 | res = raw 81 | 82 | // sqlite3 supports both $N and ?. 83 | // mysql supports only ?. 84 | // postgresql supports only $1 (SHOWFLAKE!!!). 85 | 86 | if db.Driver == "postgres" { 87 | varCount := strings.Count(raw, "?") 88 | for i := 1; i <= varCount; i++ { 89 | res = strings.Replace(res, "?", "$"+strconv.Itoa(i), 1) 90 | } 91 | } 92 | return 93 | } 94 | 95 | func (db *db) Prepare(query string) (*sql.Stmt, error) { 96 | return db.DB.Prepare(db.reformatBindvars(query)) 97 | } 98 | 99 | func (db *db) initStmts() { 100 | var err error 101 | db.addFile, err = db.Prepare(`INSERT INTO filedrop(uuid, contentType, maxUses, storeUntil) VALUES (?, ?, ?, ?)`) 102 | if err != nil { 103 | panic(err) 104 | } 105 | db.remFile, err = db.Prepare(`DELETE FROM filedrop WHERE uuid = ?`) 106 | if err != nil { 107 | panic(err) 108 | } 109 | db.contentType, err = db.Prepare(`SELECT contentType FROM filedrop WHERE uuid = ?`) 110 | if err != nil { 111 | panic(err) 112 | } 113 | db.shouldDelete, err = db.Prepare(`SELECT EXISTS(SELECT uuid FROM filedrop WHERE uuid = ? AND (storeUntil < ? OR maxUses = uses))`) 114 | if err != nil { 115 | panic(err) 116 | } 117 | db.addUse, err = db.Prepare(`UPDATE filedrop SET uses = uses + 1 WHERE uuid = ?`) 118 | if err != nil { 119 | panic(err) 120 | } 121 | db.staleFiles, err = db.Prepare(`SELECT uuid FROM filedrop WHERE storeUntil < ? OR maxUses = uses`) 122 | if err != nil { 123 | panic(err) 124 | } 125 | db.removeStaleFiles, err = db.Prepare(`DELETE FROM filedrop WHERE storeUntil < ? OR maxUses = uses`) 126 | if err != nil { 127 | panic(err) 128 | } 129 | } 130 | 131 | func (db *db) AddFile(tx *sql.Tx, uuid string, contentType string, maxUses uint, storeUntil time.Time) error { 132 | maxUsesN := sql.NullInt64{Int64: int64(maxUses), Valid: maxUses != 0} 133 | storeUntilN := sql.NullInt64{Int64: storeUntil.Unix(), Valid: !storeUntil.IsZero()} 134 | contentTypeN := sql.NullString{String: contentType, Valid: contentType != ""} 135 | 136 | if tx != nil { 137 | _, err := tx.Stmt(db.addFile).Exec(uuid, contentTypeN, maxUsesN, storeUntilN) 138 | return err 139 | } else { 140 | _, err := db.addFile.Exec(uuid, contentTypeN, maxUsesN, storeUntilN) 141 | return err 142 | } 143 | } 144 | 145 | func (db *db) RemoveFile(tx *sql.Tx, uuid string) error { 146 | if tx != nil { 147 | _, err := tx.Stmt(db.remFile).Exec(uuid) 148 | return err 149 | } else { 150 | _, err := db.remFile.Exec(uuid) 151 | return err 152 | } 153 | } 154 | 155 | func (db *db) ShouldDelete(tx *sql.Tx, uuid string) bool { 156 | var row *sql.Row 157 | if tx != nil { 158 | row = tx.Stmt(db.shouldDelete).QueryRow(uuid, time.Now().Unix()) 159 | } else { 160 | row = db.shouldDelete.QueryRow(uuid, time.Now().Unix()) 161 | } 162 | if db.Driver != "postgres" { 163 | res := 0 164 | if err := row.Scan(&res); err != nil { 165 | return false 166 | } 167 | return res == 1 168 | } else { 169 | res := false 170 | if err := row.Scan(&res); err != nil { 171 | return false 172 | } 173 | return res 174 | } 175 | } 176 | 177 | func (db *db) AddUse(tx *sql.Tx, uuid string) error { 178 | if tx != nil { 179 | _, err := tx.Stmt(db.addUse).Exec(uuid) 180 | return err 181 | } else { 182 | _, err := db.addUse.Exec(uuid, uuid) 183 | return err 184 | } 185 | } 186 | 187 | func (db *db) ContentType(tx *sql.Tx, fileUUID string) (string, error) { 188 | var row *sql.Row 189 | if tx != nil { 190 | row = tx.Stmt(db.contentType).QueryRow(fileUUID) 191 | } else { 192 | row = db.contentType.QueryRow(fileUUID) 193 | } 194 | 195 | res := sql.NullString{} 196 | return res.String, row.Scan(&res) 197 | } 198 | 199 | func (db *db) StaleFiles(tx *sql.Tx, now time.Time) ([]string, error) { 200 | uuids := []string{} 201 | var rows *sql.Rows 202 | var err error 203 | if tx != nil { 204 | rows, err = tx.Stmt(db.staleFiles).Query(now.Unix()) 205 | } else { 206 | rows, err = db.staleFiles.Query(now.Unix()) 207 | } 208 | if err != nil { 209 | return uuids, err 210 | } 211 | for rows.Next() { 212 | uuid := "" 213 | if err := rows.Scan(&uuid); err != nil { 214 | return uuids, err 215 | } 216 | uuids = append(uuids, uuid) 217 | } 218 | return uuids, nil 219 | } 220 | 221 | func (db *db) RemoveStaleFiles(tx *sql.Tx, now time.Time) error { 222 | if tx != nil { 223 | _, err := tx.Stmt(db.removeStaleFiles).Exec(now.Unix()) 224 | return err 225 | } else { 226 | _, err := db.removeStaleFiles.Exec(now.Unix()) 227 | return err 228 | } 229 | } 230 | -------------------------------------------------------------------------------- /filedropd/.gitignore: -------------------------------------------------------------------------------- 1 | filedropd 2 | filedropd.yml 3 | test.yml 4 | *.db* 5 | filedrop 6 | -------------------------------------------------------------------------------- /filedropd/filedropd.example.yml: -------------------------------------------------------------------------------- 1 | # IP:PORT to listen on. 2 | # Use 0.0.0.0 to listen on all interfaces, however we recommend using 3 | # reverse proxy for caching and stuff. 4 | listen_on: "127.0.0.1:8000" 5 | 6 | limits: 7 | # How much much times file can be accessed. Note that it also counts HEAD requests 8 | # and incomplete downloads (byte-range requests). 9 | # Per-file max-uses parameter can't exceed this value but can be smaller. 10 | max_uses: 60 11 | 12 | # Max time for which files will be stored on filedrop server. Per-file store-secs 13 | # parameter can't exceed this value but can be smaller. 14 | max_store_secs: 3600 15 | 16 | # Maximum size of file which can be uploaded to filedrop, in bytes. 17 | max_file_size: 1073741824 18 | 19 | db: 20 | # Driver to use for SQL DB (same as build tag you used to enable it). 21 | driver: sqlite3 22 | 23 | # Data Source Name, see underlying driver documentation for exact format you should use: 24 | # - PostgreSQL https://godoc.org/github.com/lib/pq 25 | # TLDR: `postgres://user:password@address/dbname` 26 | # - MySQL https://github.com/go-sql-driver/mysql 27 | # TLDR: `username:password@protocol(address)/dbname` 28 | # - SQLite3 https://github.com/mattn/go-sqlite3 29 | # TLDR: `filepath` 30 | dsn: /var/lib/filedrop/index.db 31 | 32 | # Where files will be saved on disk. 33 | storage_dir: /var/lib/filedrop 34 | 35 | # Specifies whether filedrop should return links with https scheme or not. 36 | # Overridden by X-HTTPS-Downstream header. 37 | https_downstream: true 38 | 39 | # Specifies Access-Control-Allow-Origin header. 40 | allowed_origins: "*" 41 | -------------------------------------------------------------------------------- /filedropd/filedropd.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=filedrop standalone server 3 | After=network.target 4 | 5 | [Service] 6 | Type=simple 7 | ExecStart=/usr/bin/filedropd /etc/filedropd.yml 8 | Restart=on-failure 9 | DynamicUser=true 10 | StateDirectory=filedrop 11 | 12 | [Install] 13 | WantedBy=multi-user.target 14 | -------------------------------------------------------------------------------- /filedropd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "log" 7 | "net/http" 8 | "os" 9 | "os/signal" 10 | "syscall" 11 | 12 | "github.com/foxcpp/filedrop" 13 | "gopkg.in/yaml.v2" 14 | ) 15 | 16 | func main() { 17 | if len(os.Args) != 2 { 18 | fmt.Println("Usage:", os.Args[0], "") 19 | os.Exit(1) 20 | } 21 | 22 | confBlob, err := ioutil.ReadFile(os.Args[1]) 23 | if err != nil { 24 | log.Fatalln("Failed to read config file:", err) 25 | } 26 | 27 | config := filedrop.Config{} 28 | if err := yaml.Unmarshal(confBlob, &config); err != nil { 29 | log.Fatalln("Failed to parse config file:", err) 30 | } 31 | 32 | serv, err := filedrop.New(config) 33 | if err != nil { 34 | log.Fatalln("Failed to start server:", err) 35 | } 36 | 37 | go func() { 38 | log.Println("Listening on", config.ListenOn+"...") 39 | if err := http.ListenAndServe(config.ListenOn, serv); err != nil { 40 | log.Println("Failed to listen:", err) 41 | os.Exit(1) 42 | } 43 | }() 44 | 45 | sig := make(chan os.Signal, 1) 46 | signal.Notify(sig, os.Interrupt, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP) 47 | <-sig 48 | } 49 | -------------------------------------------------------------------------------- /global_limits_test.go: -------------------------------------------------------------------------------- 1 | package filedrop_test 2 | 3 | import ( 4 | "net/http/httptest" 5 | "strconv" 6 | "strings" 7 | "testing" 8 | "time" 9 | 10 | "github.com/foxcpp/filedrop" 11 | ) 12 | 13 | func TestGlobalMaxUses(t *testing.T) { 14 | conf := filedrop.Default 15 | conf.Limits.MaxUses = 2 16 | serv := initServ(conf) 17 | ts := httptest.NewServer(serv) 18 | defer cleanServ(serv) 19 | defer ts.Close() 20 | c := ts.Client() 21 | 22 | url := string(doPOST(t, c, ts.URL+"/filedrop", "text/plain", strings.NewReader(file))) 23 | 24 | t.Run("1 use", func(t *testing.T) { 25 | doGET(t, c, url) 26 | }) 27 | t.Run("2 use", func(t *testing.T) { 28 | doGET(t, c, url) 29 | }) 30 | t.Run("3 use (fail)", func(t *testing.T) { 31 | if code := doGETFail(t, c, url); code != 404 { 32 | t.Error("GET: HTTP", code) 33 | t.FailNow() 34 | } 35 | }) 36 | } 37 | 38 | func TestGlobalMaxFileSize(t *testing.T) { 39 | conf := filedrop.Default 40 | conf.Limits.MaxFileSize = uint(len(file) - 20) 41 | serv := initServ(conf) 42 | ts := httptest.NewServer(serv) 43 | defer cleanServ(serv) 44 | defer ts.Close() 45 | c := ts.Client() 46 | 47 | t.Log("Max size:", conf.Limits.MaxFileSize, "bytes") 48 | if !t.Run("submit with size "+strconv.Itoa(len(file)), func(t *testing.T) { 49 | doPOSTFail(t, c, ts.URL+"/filedrop", "text/plain", strings.NewReader(file)) 50 | }) { 51 | t.FailNow() 52 | } 53 | 54 | strippedFile := file[:25] 55 | if !t.Run("submit with size "+strconv.Itoa(len(strippedFile)), func(t *testing.T) { 56 | doPOST(t, c, ts.URL+"/filedrop", "text/plain", strings.NewReader(strippedFile)) 57 | }) { 58 | t.FailNow() 59 | } 60 | } 61 | 62 | func TestGlobalMaxStoreTime(t *testing.T) { 63 | conf := filedrop.Default 64 | conf.Limits.MaxStoreSecs = 3 65 | serv := initServ(conf) 66 | ts := httptest.NewServer(serv) 67 | defer cleanServ(serv) 68 | defer ts.Close() 69 | c := ts.Client() 70 | 71 | var url string 72 | if !t.Run("submit", func(t *testing.T) { 73 | url = string(doPOST(t, c, ts.URL+"/filedrop", "text/plain", strings.NewReader(file))) 74 | }) { 75 | t.FailNow() 76 | } 77 | 78 | time.Sleep(1 * time.Second) 79 | 80 | if !t.Run("get after 1 second", func(t *testing.T) { 81 | doGET(t, c, url) 82 | }) { 83 | t.FailNow() 84 | } 85 | 86 | time.Sleep(3 * time.Second) 87 | 88 | if !t.Run("get after 3 seconds (fail)", func(t *testing.T) { 89 | if code := doGETFail(t, c, url); code != 404 { 90 | t.Error("GET: HTTP", code) 91 | t.FailNow() 92 | } 93 | }) { 94 | t.FailNow() 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/foxcpp/filedrop 2 | 3 | require ( 4 | github.com/go-sql-driver/mysql v1.4.0 5 | github.com/gofrs/uuid v3.2.0+incompatible 6 | github.com/lib/pq v1.0.0 7 | github.com/mattn/go-sqlite3 v1.9.0 8 | github.com/pkg/errors v0.8.0 9 | google.golang.org/appengine v1.2.0 // indirect 10 | gopkg.in/yaml.v2 v2.2.1 11 | ) 12 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= 2 | github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= 3 | github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= 4 | github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= 5 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 6 | github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= 7 | github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 8 | github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= 9 | github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= 10 | github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= 11 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 12 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 13 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 14 | google.golang.org/appengine v1.2.0 h1:S0iUepdCWODXRvtE+gcRDd15L+k+k1AiHlMiMjefH24= 15 | google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 16 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 17 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 18 | gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= 19 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 20 | -------------------------------------------------------------------------------- /mysql.go: -------------------------------------------------------------------------------- 1 | // +build mysql 2 | 3 | package filedrop 4 | 5 | import _ "github.com/go-sql-driver/mysql" 6 | -------------------------------------------------------------------------------- /perfile_limits_test.go: -------------------------------------------------------------------------------- 1 | package filedrop_test 2 | 3 | import ( 4 | "net/http/httptest" 5 | "strings" 6 | "testing" 7 | "time" 8 | 9 | "github.com/foxcpp/filedrop" 10 | ) 11 | 12 | func TestPerFileMaxUses(t *testing.T) { 13 | conf := filedrop.Default 14 | conf.Limits.MaxUses = 2 15 | serv := initServ(conf) 16 | ts := httptest.NewServer(serv) 17 | defer cleanServ(serv) 18 | defer ts.Close() 19 | c := ts.Client() 20 | 21 | if !t.Run("submit with max-uses=WRONG (fail)", func(t *testing.T) { 22 | doPOSTFail(t, c, ts.URL+"/filedrop?max-uses=WRONG", "text/plain", strings.NewReader(file)) 23 | }) { 24 | t.FailNow() 25 | } 26 | 27 | if !t.Run("submit with max-uses=3 (fail)", func(t *testing.T) { 28 | doPOSTFail(t, c, ts.URL+"/filedrop?max-uses=3", "text/plain", strings.NewReader(file)) 29 | }) { 30 | t.FailNow() 31 | } 32 | 33 | var url string 34 | if !t.Run("submit with max-uses=1", func(t *testing.T) { 35 | url = string(doPOST(t, c, ts.URL+"/filedrop?max-uses=1", "text/plain", strings.NewReader(file))) 36 | }) { 37 | t.FailNow() 38 | } 39 | if !t.Run("1 use", func(t *testing.T) { 40 | doGET(t, c, url) 41 | }) { 42 | t.FailNow() 43 | } 44 | if !t.Run("2 use", func(t *testing.T) { 45 | doGET(t, c, url) 46 | }) { 47 | t.FailNow() 48 | } 49 | } 50 | 51 | func TestPerFileStoreTime(t *testing.T) { 52 | conf := filedrop.Default 53 | conf.Limits.MaxStoreSecs = 5 54 | serv := initServ(conf) 55 | ts := httptest.NewServer(serv) 56 | defer cleanServ(serv) 57 | defer ts.Close() 58 | c := ts.Client() 59 | 60 | if !t.Run("submit with store-secs=WRONG (fail)", func(t *testing.T) { 61 | doPOSTFail(t, c, ts.URL+"/filedrop?store-secs=WRONG", "text/plain", strings.NewReader(file)) 62 | }) { 63 | t.FailNow() 64 | } 65 | 66 | if !t.Run("submit with store-secs=15 (fail)", func(t *testing.T) { 67 | doPOSTFail(t, c, ts.URL+"/filedrop?store-secs=15", "text/plain", strings.NewReader(file)) 68 | }) { 69 | t.FailNow() 70 | } 71 | 72 | var url string 73 | if !t.Run("submit with store-secs=5", func(t *testing.T) { 74 | url = string(doPOST(t, c, ts.URL+"/filedrop?store-secs=5", "text/plain", strings.NewReader(file))) 75 | }) { 76 | t.FailNow() 77 | } 78 | 79 | time.Sleep(1 * time.Second) 80 | 81 | if !t.Run("get after 1 second", func(t *testing.T) { 82 | doGET(t, c, url) 83 | }) { 84 | t.FailNow() 85 | } 86 | 87 | time.Sleep(5 * time.Second) 88 | 89 | if !t.Run("get after 6 seconds (fail)", func(t *testing.T) { 90 | if code := doGETFail(t, c, url); code != 404 { 91 | t.Error("GET: HTTP", code) 92 | t.FailNow() 93 | } 94 | }) { 95 | t.FailNow() 96 | } 97 | } 98 | 99 | func TestUnboundedFileLimits(t *testing.T) { 100 | // Setting no limits in file should allow us to set them to any value in arguments. 101 | conf := filedrop.Default 102 | serv := initServ(conf) 103 | ts := httptest.NewServer(serv) 104 | defer cleanServ(serv) 105 | defer ts.Close() 106 | c := ts.Client() 107 | 108 | doPOST(t, c, ts.URL+"/filedrop?store-secs=999999999", "text/plain", strings.NewReader(file)) 109 | doPOST(t, c, ts.URL+"/filedrop?max-uses=999999999", "text/plain", strings.NewReader(file)) 110 | } 111 | -------------------------------------------------------------------------------- /postgresql.go: -------------------------------------------------------------------------------- 1 | // +build postgres 2 | 3 | package filedrop 4 | 5 | import _ "github.com/lib/pq" 6 | -------------------------------------------------------------------------------- /server.go: -------------------------------------------------------------------------------- 1 | package filedrop 2 | 3 | import ( 4 | "bytes" 5 | "database/sql" 6 | "fmt" 7 | "io" 8 | "log" 9 | "net/http" 10 | "net/url" 11 | "os" 12 | "path/filepath" 13 | "strconv" 14 | "strings" 15 | "time" 16 | 17 | "github.com/gofrs/uuid" 18 | "github.com/pkg/errors" 19 | ) 20 | 21 | var ErrFileDoesntExists = errors.New("file doesn't exists") 22 | 23 | // Main filedrop server structure, implements http.Handler. 24 | type Server struct { 25 | DB *db 26 | Conf Config 27 | Logger *log.Logger 28 | DebugLogger *log.Logger 29 | 30 | fileCleanerStopChan chan bool 31 | } 32 | 33 | // Create and initialize new server instance using passed configuration. 34 | // 35 | // serv.Logger will be redirected to os.Stderr by default. 36 | // Created instances should be closed by using serv.Close. 37 | func New(conf Config) (*Server, error) { 38 | s := new(Server) 39 | var err error 40 | 41 | s.Conf = conf 42 | 43 | if err := os.MkdirAll(conf.StorageDir, os.ModePerm); err != nil { 44 | return nil, err 45 | } 46 | if err := s.testPerms(); err != nil { 47 | return nil, err 48 | } 49 | 50 | s.fileCleanerStopChan = make(chan bool) 51 | s.Logger = log.New(os.Stderr, "filedrop ", log.LstdFlags) 52 | s.DB, err = openDB(conf.DB.Driver, conf.DB.DSN) 53 | 54 | go s.fileCleaner() 55 | 56 | return s, err 57 | } 58 | 59 | func (s *Server) dbgLog(v ...interface{}) { 60 | if s.DebugLogger != nil { 61 | s.DebugLogger.Output(2, fmt.Sprintln(v...)) 62 | } 63 | } 64 | 65 | func (s *Server) testPerms() error { 66 | testPath := filepath.Join(s.Conf.StorageDir, "test_file") 67 | 68 | // Check write permissions. 69 | f, err := os.Create(testPath) 70 | if err != nil { 71 | return err 72 | } 73 | f.Close() 74 | 75 | // Check read permissions. 76 | f, err = os.Open(testPath) 77 | if err != nil { 78 | return err 79 | } 80 | f.Close() 81 | 82 | // Check remove permissions. 83 | return os.Remove(testPath) 84 | } 85 | 86 | // AddFile adds file to storage and returns assigned UUID which can be directly 87 | // substituted into URL. 88 | func (s *Server) AddFile(contents io.Reader, contentType string, maxUses uint, storeUntil time.Time) (string, error) { 89 | fileUUID, err := uuid.NewV4() 90 | if err != nil { 91 | return "", errors.Wrap(err, "UUID generation") 92 | } 93 | outLocation := filepath.Join(s.Conf.StorageDir, fileUUID.String()) 94 | 95 | _, err = os.Stat(outLocation) 96 | if err == nil { 97 | s.Logger.Println("UUID collision detected:", fileUUID) 98 | return "", errors.New("UUID collision detected") 99 | } 100 | 101 | file, err := os.Create(outLocation) 102 | if err != nil { 103 | s.Logger.Printf("File create failure (%v): %v\n", fileUUID, err) 104 | return "", errors.Wrap(err, "file open") 105 | } 106 | if _, err := io.Copy(file, contents); err != nil { 107 | s.Logger.Printf("File write failure (%v): %v\n", fileUUID, err) 108 | return "", errors.Wrap(err, "file write") 109 | } 110 | if err := s.DB.AddFile(nil, fileUUID.String(), contentType, maxUses, storeUntil); err != nil { 111 | os.Remove(outLocation) 112 | s.Logger.Printf("DB add failure (%v, %v, %v, %v): %v\n", fileUUID, contentType, maxUses, storeUntil, err) 113 | return "", errors.Wrap(err, "db add") 114 | } 115 | 116 | return fileUUID.String(), nil 117 | } 118 | 119 | // RemoveFile removes file from database and underlying storage. 120 | func (s *Server) RemoveFile(fileUUID string) error { 121 | return s.removeFile(nil, fileUUID) 122 | } 123 | 124 | func (s *Server) removeFile(tx *sql.Tx, fileUUID string) error { 125 | fileLocation := filepath.Join(s.Conf.StorageDir, fileUUID) 126 | 127 | // Just to check validity. 128 | _, err := uuid.FromString(fileUUID) 129 | if err != nil { 130 | return errors.Wrap(err, "uuid parse") 131 | } 132 | 133 | if err := s.DB.RemoveFile(tx, fileUUID); err != nil { 134 | s.Logger.Printf("DB remove failure (%v): %v\n", fileUUID, err) 135 | return errors.Wrap(err, "db remove") 136 | } 137 | 138 | if err := os.Remove(fileLocation); err != nil { 139 | // TODO: Recover DB entry? 140 | s.Logger.Printf("File remove failure (%v): %v\n", fileUUID, err) 141 | return errors.Wrap(err, "file remove") 142 | } 143 | return nil 144 | } 145 | 146 | // OpenFile opens file for reading without any other side-effects 147 | // applied (such as "link" usage counting). 148 | func (s *Server) OpenFile(fileUUID string) (io.ReadSeeker, error) { 149 | // Just to check validity. 150 | _, err := uuid.FromString(fileUUID) 151 | if err != nil { 152 | return nil, errors.Wrap(err, "uuid parse") 153 | } 154 | 155 | fileLocation := filepath.Join(s.Conf.StorageDir, fileUUID) 156 | file, err := os.Open(fileLocation) 157 | if err != nil { 158 | if os.IsNotExist(err) { 159 | // Clean up the DB entry if the file was removed by an external program. 160 | if err := s.DB.RemoveFile(nil, fileUUID); err != nil { 161 | s.Logger.Printf("DB remove failure (%v): %v\n", fileUUID, err) 162 | } 163 | return nil, ErrFileDoesntExists 164 | } 165 | return nil, err 166 | } 167 | return file, nil 168 | } 169 | 170 | // GetFile opens file for reading. 171 | // 172 | // Note that access using this function is equivalent to access 173 | // through HTTP API, so it will count against usage count, for example. 174 | // To avoid this use OpenFile(fileUUID). 175 | func (s *Server) GetFile(fileUUID string) (r io.ReadSeeker, contentType string, err error) { 176 | // Just to check validity. 177 | _, err = uuid.FromString(fileUUID) 178 | if err != nil { 179 | return nil, "", ErrFileDoesntExists 180 | } 181 | 182 | tx, err := s.DB.Begin() 183 | if err != nil { 184 | return nil, "", errors.Wrap(err, "tx begin") 185 | } 186 | defer tx.Rollback() // rollback is no-op after commit 187 | 188 | s.dbgLog("Serving file", fileUUID) 189 | 190 | if s.DB.ShouldDelete(tx, fileUUID) { 191 | s.dbgLog("File removed just before getting, UUID:", fileUUID) 192 | if err := s.removeFile(tx, fileUUID); err != nil { 193 | s.Logger.Println("Error while trying to remove file", fileUUID+":", err) 194 | } 195 | if err := tx.Commit(); err != nil { 196 | return nil, "", err 197 | } 198 | return nil, "", ErrFileDoesntExists 199 | } 200 | if err := s.DB.AddUse(tx, fileUUID); err != nil { 201 | return nil, "", errors.Wrap(err, "add use") 202 | } 203 | 204 | fileLocation := filepath.Join(s.Conf.StorageDir, fileUUID) 205 | r, err = os.Open(fileLocation) 206 | if err != nil { 207 | if os.IsNotExist(err) { 208 | return nil, "", ErrFileDoesntExists 209 | } 210 | // Clean up the DB entry if the file was removed by an external program. 211 | if err := s.DB.RemoveFile(tx, fileUUID); err != nil { 212 | s.Logger.Printf("DB remove failure (%v): %v\n", fileUUID, err) 213 | } 214 | return nil, "", err 215 | } 216 | if err := tx.Commit(); err != nil { 217 | return nil, "", errors.Wrap(err, "tx commit") 218 | } 219 | 220 | ttype, err := s.DB.ContentType(nil, fileUUID) 221 | if err != nil { 222 | return nil, "", errors.Wrap(err, "content type query") 223 | } 224 | 225 | return r, ttype, nil 226 | } 227 | 228 | func (s *Server) acceptFile(w http.ResponseWriter, r *http.Request) { 229 | if s.Conf.UploadAuth.Callback != nil && !s.Conf.UploadAuth.Callback(r) { 230 | s.Logger.Printf("Authentication failure (URL %v, IP %v)", r.URL.String(), r.RemoteAddr) 231 | s.writeErr(w, r, http.StatusForbidden, "forbidden") 232 | return 233 | } 234 | 235 | if s.Conf.Limits.MaxFileSize != 0 && r.ContentLength > int64(s.Conf.Limits.MaxFileSize) { 236 | s.Logger.Printf("Too big file (URL %v, IP %v)", r.URL.String(), r.RemoteAddr) 237 | s.writeErr(w, r, http.StatusRequestEntityTooLarge, "too big file") 238 | return 239 | } 240 | 241 | storeUntil := time.Time{} 242 | if r.URL.Query().Get("store-secs") == "" && s.Conf.Limits.MaxStoreSecs != 0 { 243 | storeUntil = time.Now().Add(time.Duration(s.Conf.Limits.MaxStoreSecs) * time.Second) 244 | } else if r.URL.Query().Get("store-secs") != "" { 245 | secs, err := strconv.Atoi(r.URL.Query().Get("store-secs")) 246 | if err != nil { 247 | s.Logger.Printf("Invalid store-secs (URL %v, IP %v)", r.URL.String(), r.RemoteAddr) 248 | s.writeErr(w, r, http.StatusBadRequest, "invalid store-secs value") 249 | return 250 | } 251 | if s.Conf.Limits.MaxStoreSecs != 0 && uint(secs) > s.Conf.Limits.MaxStoreSecs { 252 | s.Logger.Printf("Too big store-secs (URL %v, IP %v)", r.URL.String(), r.RemoteAddr) 253 | s.writeErr(w, r, http.StatusBadRequest, "too big store-secs value") 254 | return 255 | } 256 | storeUntil = time.Now().Add(time.Duration(secs) * time.Second) 257 | } 258 | var maxUses uint 259 | if r.URL.Query().Get("max-uses") == "" && s.Conf.Limits.MaxUses != 0 { 260 | maxUses = s.Conf.Limits.MaxUses 261 | } else if r.URL.Query().Get("max-uses") != "" { 262 | var err error 263 | maxUses, err := strconv.Atoi(r.URL.Query().Get("max-uses")) 264 | if err != nil { 265 | s.Logger.Printf("Invalid max-uses store-secs (URL %v, IP %v)", r.URL.String(), r.RemoteAddr) 266 | s.writeErr(w, r, http.StatusBadRequest, "invalid max-uses value") 267 | return 268 | } 269 | if s.Conf.Limits.MaxUses != 0 && uint(maxUses) > s.Conf.Limits.MaxUses { 270 | s.Logger.Printf("Too big max-uses store-secs (URL %v, IP %v)", r.URL.String(), r.RemoteAddr) 271 | s.writeErr(w, r, http.StatusBadRequest, "too big max-uses value") 272 | return 273 | } 274 | } 275 | 276 | fileUUID, err := s.AddFile(r.Body, r.Header.Get("Content-Type"), maxUses, storeUntil) 277 | if err != nil { 278 | s.Logger.Println("Error while serving", r.RequestURI+":", err) 279 | s.writeErr(w, r, http.StatusInternalServerError, "internal server error") 280 | return 281 | } 282 | 283 | s.dbgLog("Accepted file, assigned UUID is", fileUUID) 284 | 285 | // Smart logic to convert request's URL into absolute result URL. 286 | resURL := url.URL{} 287 | if r.Header.Get("X-HTTPS-Downstream") == "1" { 288 | resURL.Scheme = "https" 289 | } else if r.Header.Get("X-HTTPS-Downstream") == "0" { 290 | resURL.Scheme = "http" 291 | } else if s.Conf.HTTPSDownstream { 292 | resURL.Scheme = "https" 293 | } else { 294 | resURL.Scheme = "http" 295 | } 296 | resURL.Host = r.Host 297 | splittenPath := strings.Split(r.URL.Path, "/") 298 | if r.URL.Path == "/" { 299 | splittenPath = nil 300 | } 301 | splittenPath = append(splittenPath, fileUUID) 302 | resURL.Path = strings.Join(splittenPath, "/") 303 | 304 | w.Header().Add("Content-Type", `text/plain; charset="us-ascii"`) 305 | w.WriteHeader(http.StatusCreated) 306 | if _, err := w.Write([]byte(resURL.String())); err != nil { 307 | s.Logger.Printf("I/O error (URL %v, IP %v): %v", r.URL.String(), r.RemoteAddr, err) 308 | } 309 | } 310 | 311 | func (s *Server) writeErr(w http.ResponseWriter, r *http.Request, code int, replyText string) { 312 | w.Header().Add("Content-Type", `text/plain; charset="us-ascii"`) 313 | w.WriteHeader(code) 314 | _, err := io.WriteString(w, strconv.Itoa(code)+" "+replyText) 315 | if err != nil { 316 | s.Logger.Printf("I/O error (URL %v, IP %v): %v", r.URL.String(), r.RemoteAddr, err) 317 | } 318 | } 319 | 320 | func (s *Server) serveFile(w http.ResponseWriter, r *http.Request) { 321 | if s.Conf.DownloadAuth.Callback != nil && !s.Conf.DownloadAuth.Callback(r) { 322 | s.Logger.Printf("Authentication failure (URL %v, IP %v)", r.URL.String(), r.RemoteAddr) 323 | s.writeErr(w, r, http.StatusForbidden, "forbidden") 324 | return 325 | } 326 | 327 | splittenPath := strings.Split(r.URL.Path, "/") 328 | if len(splittenPath) < 2 { 329 | s.writeErr(w, r, http.StatusNotFound, "not found") 330 | return 331 | } 332 | fileUUID := splittenPath[len(splittenPath)-1] 333 | if _, err := uuid.FromString(fileUUID); err != nil { 334 | // Probably last component is fake "filename". 335 | if len(splittenPath) == 1 { 336 | s.writeErr(w, r, http.StatusNotFound, "not found") 337 | return 338 | } 339 | fileUUID = splittenPath[len(splittenPath)-2] 340 | } 341 | reader, ttype, err := s.GetFile(fileUUID) 342 | if err != nil { 343 | if err == ErrFileDoesntExists { 344 | s.writeErr(w, r, http.StatusNotFound, "not found") 345 | } else { 346 | s.Logger.Println("Error while serving", r.RequestURI+":", err) 347 | s.writeErr(w, r, http.StatusInternalServerError, "internal server error") 348 | } 349 | return 350 | } 351 | if ttype != "" { 352 | w.Header().Set("Content-Type", ttype) 353 | } 354 | w.Header().Set("ETag", fileUUID) 355 | w.Header().Set("Cache-Control", "public, immutable, max-age=31536000") 356 | if r.Method == http.MethodOptions { 357 | reader = bytes.NewReader([]byte{}) 358 | } 359 | http.ServeContent(w, r, fileUUID, time.Time{}, reader) 360 | } 361 | 362 | // ServeHTTP implements http.Handler for filedrop.Server. 363 | // 364 | // Note that filedrop code is URL prefix-agnostic, so request URI doesn't 365 | // matters much. 366 | func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { 367 | w.Header().Set("Access-Control-Allow-Origin", s.Conf.AllowedOrigins) 368 | if r.Method == http.MethodPost { 369 | s.acceptFile(w, r) 370 | } else if r.Method == http.MethodGet || 371 | r.Method == http.MethodHead { 372 | 373 | s.serveFile(w, r) 374 | } else if r.Method == http.MethodOptions { 375 | w.Header().Set("Access-Control-Allow-Methods", "HEAD, GET, POST, DELETE") 376 | w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length") 377 | w.WriteHeader(http.StatusNoContent) 378 | } else { 379 | s.writeErr(w, r, http.StatusMethodNotAllowed, "method not allowed") 380 | } 381 | } 382 | 383 | func (s *Server) Close() error { 384 | // don't close DB if "cleaner" is doing something, wait for it to finish 385 | s.fileCleanerStopChan <- true 386 | <-s.fileCleanerStopChan 387 | 388 | return s.DB.Close() 389 | } 390 | 391 | func (s *Server) fileCleaner() { 392 | if s.Conf.CleanupIntervalSecs == 0 { 393 | s.Conf.CleanupIntervalSecs = 60 394 | } 395 | tick := time.NewTicker(time.Duration(s.Conf.CleanupIntervalSecs) * time.Second) 396 | for { 397 | select { 398 | case <-s.fileCleanerStopChan: 399 | s.fileCleanerStopChan <- true 400 | return 401 | case <-tick.C: 402 | s.cleanupFiles() 403 | } 404 | } 405 | } 406 | 407 | func (s *Server) cleanupFiles() { 408 | tx, err := s.DB.Begin() 409 | if err != nil { 410 | s.Logger.Println("Failed to begin transaction for clean-up:", err) 411 | return 412 | } 413 | defer tx.Rollback() // rollback is no-op after commit 414 | 415 | now := time.Now() 416 | 417 | uuids, err := s.DB.StaleFiles(tx, now) 418 | if err != nil { 419 | s.Logger.Println("Failed to get list of files pending removal:", err) 420 | return 421 | } 422 | 423 | if len(uuids) != 0 { 424 | s.dbgLog(len(uuids), "file to be removed") 425 | } 426 | 427 | for _, fileUUID := range uuids { 428 | if err := os.Remove(filepath.Join(s.Conf.StorageDir, fileUUID)); err != nil { 429 | s.Logger.Println("Failed to remove file during clean-up:", err) 430 | } 431 | } 432 | 433 | if err := s.DB.RemoveStaleFiles(tx, now); err != nil { 434 | s.Logger.Println("Failed to remove stale files from DB:", err) 435 | return 436 | } 437 | 438 | if err := tx.Commit(); err != nil { 439 | s.Logger.Println("Failed to begin transaction for clean-up:", err) 440 | } 441 | } 442 | -------------------------------------------------------------------------------- /server_test.go: -------------------------------------------------------------------------------- 1 | package filedrop_test 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/hex" 6 | "io/ioutil" 7 | "net/http" 8 | "net/http/httptest" 9 | "os" 10 | "path/filepath" 11 | "reflect" 12 | "strings" 13 | "testing" 14 | "time" 15 | 16 | "github.com/foxcpp/filedrop" 17 | _ "github.com/mattn/go-sqlite3" 18 | ) 19 | 20 | func TestBasicSubmit(t *testing.T) { 21 | serv := initServ(filedrop.Default) 22 | ts := httptest.NewServer(serv) 23 | defer cleanServ(serv) 24 | defer ts.Close() 25 | c := ts.Client() 26 | 27 | url := string(doPOST(t, c, ts.URL+"/filedrop", "text/plain", strings.NewReader(file))) 28 | 29 | t.Log("File URL:", url) 30 | 31 | fileBody := doGET(t, c, url) 32 | if string(fileBody) != file { 33 | t.Log("Got different file!") 34 | sentHash := sha256.Sum256([]byte(file)) 35 | t.Log("Sent:", hex.EncodeToString(sentHash[:])) 36 | recvHash := sha256.Sum256(fileBody) 37 | t.Log("Received:", hex.EncodeToString(recvHash[:])) 38 | t.FailNow() 39 | } 40 | } 41 | 42 | func TestBasicSubmit_NoPath(t *testing.T) { 43 | serv := initServ(filedrop.Default) 44 | ts := httptest.NewServer(serv) 45 | defer cleanServ(serv) 46 | defer ts.Close() 47 | c := ts.Client() 48 | 49 | url := string(doPOST(t, c, ts.URL+"", "text/plain", strings.NewReader(file))) 50 | 51 | t.Log("File URL:", url) 52 | 53 | // First match is in http:// 54 | if strings.Count(url, "//") > 1 { 55 | t.Error("URL path includes empty components") 56 | } 57 | 58 | fileBody := doGET(t, c, url) 59 | if string(fileBody) != file { 60 | t.Log("Got different file!") 61 | sentHash := sha256.Sum256([]byte(file)) 62 | t.Log("Sent:", hex.EncodeToString(sentHash[:])) 63 | recvHash := sha256.Sum256(fileBody) 64 | t.Log("Received:", hex.EncodeToString(recvHash[:])) 65 | t.FailNow() 66 | } 67 | } 68 | 69 | func TestHeadAndGet(t *testing.T) { 70 | serv := initServ(filedrop.Default) 71 | ts := httptest.NewServer(serv) 72 | defer cleanServ(serv) 73 | defer ts.Close() 74 | c := ts.Client() 75 | 76 | url := string(doPOST(t, c, ts.URL+"/filedrop", "text/plain", strings.NewReader(file))) 77 | 78 | t.Log("File URL:", url) 79 | 80 | respHead, err := c.Head(url) 81 | if err != nil { 82 | t.Error("GET:", err) 83 | t.FailNow() 84 | } 85 | respGet, err := c.Get(url) 86 | if err != nil { 87 | t.Error("HEAD:", err) 88 | t.FailNow() 89 | } 90 | defer respHead.Body.Close() 91 | defer respGet.Body.Close() 92 | 93 | if !reflect.DeepEqual(respHead.Header, respGet.Header) { 94 | t.Error("Headers are different!") 95 | t.FailNow() 96 | } 97 | } 98 | 99 | func TestFakeFilename(t *testing.T) { 100 | conf := filedrop.Default 101 | serv := initServ(conf) 102 | ts := httptest.NewServer(serv) 103 | defer cleanServ(serv) 104 | defer ts.Close() 105 | c := ts.Client() 106 | 107 | fileUrl := string(doPOST(t, c, ts.URL+"/filedrop", "text/plain", strings.NewReader(file))) 108 | t.Log("File URL:", fileUrl) 109 | 110 | t.Run("without fake filename", func(t *testing.T) { 111 | doGET(t, c, fileUrl) 112 | }) 113 | t.Run("with fake filename (meow.txt)", func(t *testing.T) { 114 | doGET(t, c, fileUrl+"/meow.txt") 115 | }) 116 | } 117 | 118 | func TestNonExistent(t *testing.T) { 119 | // Non-existent file should correctly return 404 code. 120 | serv := initServ(filedrop.Default) 121 | ts := httptest.NewServer(serv) 122 | defer cleanServ(serv) 123 | defer ts.Close() 124 | c := ts.Client() 125 | 126 | t.Run("non-existent UUID in path", func(t *testing.T) { 127 | code := doGETFail(t, c, ts.URL+"/filedrop/AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA") 128 | if code != 404 { 129 | t.Error("GET: HTTP", code) 130 | t.FailNow() 131 | } 132 | }) 133 | 134 | t.Run("no UUID in path", func(t *testing.T) { 135 | code := doGETFail(t, c, ts.URL+"/filedrop") 136 | if code != 404 { 137 | t.Error("GET: HTTP", code) 138 | t.FailNow() 139 | } 140 | }) 141 | 142 | t.Run("invalid UUID in path", func(t *testing.T) { 143 | code := doGETFail(t, c, ts.URL+"/filedrop/IAMINVALIDUUID") 144 | if code != 404 { 145 | t.Error("GET: HTTP", code) 146 | t.FailNow() 147 | } 148 | }) 149 | } 150 | 151 | func TestContentTypePreserved(t *testing.T) { 152 | serv := initServ(filedrop.Default) 153 | ts := httptest.NewServer(serv) 154 | defer cleanServ(serv) 155 | defer ts.Close() 156 | c := ts.Client() 157 | 158 | url := string(doPOST(t, c, ts.URL+"/filedrop", "text/kitteh", strings.NewReader(file))) 159 | 160 | t.Log("File URL:", url) 161 | 162 | resp, err := c.Get(url) 163 | if err != nil { 164 | t.Error("GET:", err) 165 | t.FailNow() 166 | } 167 | defer resp.Body.Close() 168 | body, err := ioutil.ReadAll(resp.Body) 169 | if err != nil { 170 | t.Error("ioutil.ReadAll:", err) 171 | t.FailNow() 172 | } 173 | if resp.StatusCode/100 != 2 { 174 | t.Error("GET: HTTP", resp.Status) 175 | t.Error("Body:", string(body)) 176 | t.FailNow() 177 | } 178 | if resp.Header.Get("Content-Type") != "text/kitteh" { 179 | t.Log("Mismatched content type:") 180 | t.Log("\tWanted: 'text/kitteh'") 181 | t.Log("\tGot:", "'"+resp.Header.Get("Content-Type")+"'") 182 | t.Fail() 183 | } 184 | } 185 | 186 | func TestNoContentType(t *testing.T) { 187 | serv := initServ(filedrop.Default) 188 | ts := httptest.NewServer(serv) 189 | defer cleanServ(serv) 190 | defer ts.Close() 191 | c := ts.Client() 192 | 193 | url := string(doPOST(t, c, ts.URL+"/filedrop", "", strings.NewReader(file))) 194 | 195 | t.Log("File URL:", url) 196 | 197 | resp, err := c.Get(url) 198 | if err != nil { 199 | t.Error("GET:", err) 200 | t.FailNow() 201 | } 202 | defer resp.Body.Close() 203 | body, err := ioutil.ReadAll(resp.Body) 204 | if err != nil { 205 | t.Error("ioutil.ReadAll:", err) 206 | t.FailNow() 207 | } 208 | if resp.StatusCode/100 != 2 { 209 | t.Error("GET: HTTP", resp.Status) 210 | t.Error("Body:", string(body)) 211 | t.FailNow() 212 | } 213 | t.Log("Got:", "'"+resp.Header.Get("Content-Type")+"'") 214 | } 215 | 216 | func TestHTTPSDownstream(t *testing.T) { 217 | serv := initServ(filedrop.Default) 218 | ts := httptest.NewServer(serv) 219 | defer cleanServ(serv) 220 | defer ts.Close() 221 | c := ts.Client() 222 | 223 | t.Run("X-HTTPS-Downstream=1", func(t *testing.T) { 224 | req, err := http.NewRequest("POST", ts.URL, strings.NewReader(file)) 225 | if err != nil { 226 | t.Error(err) 227 | t.FailNow() 228 | } 229 | req.Header.Set("X-HTTPS-Downstream", "1") 230 | resp, err := c.Do(req) 231 | if err != nil { 232 | t.Error("POST:", err) 233 | t.FailNow() 234 | } 235 | defer resp.Body.Close() 236 | body, err := ioutil.ReadAll(resp.Body) 237 | if err != nil { 238 | t.Error("ioutil.ReadAll:", err) 239 | t.FailNow() 240 | } 241 | if resp.StatusCode/100 != 2 { 242 | t.Error("POST: HTTP", resp.StatusCode, resp.Status) 243 | t.Error("Body:", string(body)) 244 | t.FailNow() 245 | } 246 | if !strings.HasPrefix(string(body), "https") { 247 | t.Error("Got non-HTTPS URl with X-HTTPS-Downstream=1") 248 | t.FailNow() 249 | } 250 | }) 251 | t.Run("X-HTTPS-Downstream=0", func(t *testing.T) { 252 | req, err := http.NewRequest("POST", ts.URL, strings.NewReader(file)) 253 | if err != nil { 254 | t.Error(err) 255 | t.FailNow() 256 | } 257 | req.Header.Set("X-HTTPS-Downstream", "0") 258 | resp, err := c.Do(req) 259 | if err != nil { 260 | t.Error("POST:", err) 261 | t.FailNow() 262 | } 263 | defer resp.Body.Close() 264 | body, err := ioutil.ReadAll(resp.Body) 265 | if err != nil { 266 | t.Error("ioutil.ReadAll:", err) 267 | t.FailNow() 268 | } 269 | if resp.StatusCode/100 != 2 { 270 | t.Error("POST: HTTP", resp.StatusCode, resp.Status) 271 | t.Error("Body:", string(body)) 272 | t.FailNow() 273 | } 274 | if !strings.HasPrefix(string(body), "http") { 275 | t.Error("Got non-HTTP URL with X-HTTPS-Downstream=0") 276 | t.FailNow() 277 | } 278 | }) 279 | } 280 | 281 | func testWithPrefix(t *testing.T, ts *httptest.Server, c *http.Client, prefix string) { 282 | var URL string 283 | t.Run("submit with prefix "+prefix, func(t *testing.T) { 284 | URL = string(doPOST(t, c, ts.URL+prefix+"/meow.txt", "text/plain", strings.NewReader(file))) 285 | }) 286 | 287 | if !strings.Contains(URL, prefix) { 288 | t.Errorf("Result URL doesn't contain prefix %v: %v", prefix, URL) 289 | t.FailNow() 290 | } 291 | 292 | if URL != "" { 293 | t.Run("get with "+prefix, func(t *testing.T) { 294 | body := doGET(t, c, URL) 295 | if string(body) != file { 296 | t.Error("Got different file!") 297 | t.FailNow() 298 | } 299 | }) 300 | } 301 | } 302 | 303 | func TestPrefixAgnostic(t *testing.T) { 304 | // Server should be able to handle requests independently 305 | // from full URL. 306 | serv := initServ(filedrop.Default) 307 | ts := httptest.NewServer(serv) 308 | defer cleanServ(serv) 309 | defer ts.Close() 310 | c := ts.Client() 311 | 312 | testWithPrefix(t, ts, c, "/a/b/c/d/e/f/g") 313 | testWithPrefix(t, ts, c, "/a/f%20oo/g") 314 | testWithPrefix(t, ts, c, "") 315 | } 316 | 317 | func TestCleanup(t *testing.T) { 318 | conf := filedrop.Default 319 | conf.CleanupIntervalSecs = 1 320 | serv := initServ(conf) 321 | ts := httptest.NewServer(serv) 322 | defer cleanServ(serv) 323 | defer ts.Close() 324 | c := ts.Client() 325 | 326 | URL := string(doPOST(t, c, ts.URL+"/filedrop?store-secs=1", "text/plain", strings.NewReader(file))) 327 | splittenURL := strings.Split(URL, "/") 328 | UUID := splittenURL[len(splittenURL)-1] 329 | time.Sleep(4 * time.Second) 330 | 331 | _, err := os.Stat(filepath.Join(serv.Conf.StorageDir, UUID)) 332 | if err == nil || !os.IsNotExist(err) { 333 | t.Error("Wanted 'no such file or directory', got:", err) 334 | t.FailNow() 335 | } 336 | } 337 | -------------------------------------------------------------------------------- /sqlite3.go: -------------------------------------------------------------------------------- 1 | // +build sqlite3 2 | 3 | package filedrop 4 | 5 | import _ "github.com/mattn/go-sqlite3" 6 | -------------------------------------------------------------------------------- /utils_test.go: -------------------------------------------------------------------------------- 1 | package filedrop_test 2 | 3 | import ( 4 | "io" 5 | "io/ioutil" 6 | "log" 7 | "net/http" 8 | "os" 9 | "path/filepath" 10 | "testing" 11 | 12 | "github.com/foxcpp/filedrop" 13 | _ "github.com/mattn/go-sqlite3" 14 | ) 15 | 16 | var TestDB = os.Getenv("TEST_DB") 17 | var TestDSN = os.Getenv("TEST_DSN") 18 | 19 | var file = `Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow 20 | Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow 21 | Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow Meow` 22 | 23 | func initServ(conf filedrop.Config) *filedrop.Server { 24 | tempDir, err := ioutil.TempDir("", "filedrop-tests-") 25 | if err != nil { 26 | panic(err) 27 | } 28 | conf.StorageDir = tempDir 29 | 30 | conf.DB.Driver = TestDB 31 | conf.DB.DSN = TestDSN 32 | 33 | // This is meant for DB debugging. 34 | if TestDB == "" || TestDSN == "" { 35 | log.Println("Using sqlite3 DB in temporary directory.") 36 | conf.DB.Driver = "sqlite3" 37 | conf.DB.DSN = filepath.Join(tempDir, "index.db") 38 | } 39 | 40 | serv, err := filedrop.New(conf) 41 | if err != nil { 42 | panic(err) 43 | } 44 | if testing.Verbose() { 45 | serv.DebugLogger = log.New(os.Stderr, "filedrop/debug ", log.Lshortfile) 46 | } 47 | return serv 48 | } 49 | 50 | func cleanServ(serv *filedrop.Server) { 51 | if _, err := serv.DB.Exec(`DROP TABLE filedrop`); err != nil { 52 | panic(err) 53 | } 54 | serv.Close() 55 | os.Remove(serv.Conf.StorageDir) 56 | } 57 | 58 | func doPOST(t *testing.T, c *http.Client, url string, contentType string, reqBody io.Reader) []byte { 59 | t.Helper() 60 | 61 | resp, err := c.Post(url, contentType, reqBody) 62 | if err != nil { 63 | t.Error("POST:", err) 64 | t.FailNow() 65 | } 66 | defer resp.Body.Close() 67 | body, err := ioutil.ReadAll(resp.Body) 68 | if err != nil { 69 | t.Error("ioutil.ReadAll:", err) 70 | t.FailNow() 71 | } 72 | if resp.StatusCode/100 != 2 { 73 | t.Error("POST: HTTP", resp.StatusCode, resp.Status) 74 | t.Error("Body:", string(body)) 75 | t.FailNow() 76 | } 77 | return body 78 | } 79 | 80 | func doPOSTFail(t *testing.T, c *http.Client, url string, contentType string, reqBody io.Reader) int { 81 | t.Helper() 82 | 83 | resp, err := c.Post(url, contentType, reqBody) 84 | if err != nil { 85 | t.Error("POST:", err) 86 | t.FailNow() 87 | } 88 | defer resp.Body.Close() 89 | if resp.StatusCode/100 == 2 { 90 | t.Error("POST: HTTP", resp.StatusCode, resp.Status) 91 | t.FailNow() 92 | } 93 | return resp.StatusCode 94 | } 95 | 96 | func doGET(t *testing.T, c *http.Client, url string) []byte { 97 | t.Helper() 98 | 99 | resp, err := c.Get(url) 100 | if err != nil { 101 | t.Error("GET:", err) 102 | t.FailNow() 103 | } 104 | defer resp.Body.Close() 105 | body, err := ioutil.ReadAll(resp.Body) 106 | if err != nil { 107 | t.Error("ioutil.ReadAll:", err) 108 | t.FailNow() 109 | } 110 | if resp.StatusCode/100 != 2 { 111 | t.Error("GET: HTTP", resp.Status) 112 | t.Error("Body:", string(body)) 113 | t.FailNow() 114 | } 115 | return body 116 | } 117 | 118 | func doGETFail(t *testing.T, c *http.Client, url string) int { 119 | t.Helper() 120 | 121 | resp, err := c.Get(url) 122 | if err != nil { 123 | t.Error("GET:", err) 124 | t.FailNow() 125 | } 126 | defer resp.Body.Close() 127 | if resp.StatusCode/100 == 2 { 128 | t.Error("GET: HTTP", resp.StatusCode, resp.Status) 129 | t.FailNow() 130 | } 131 | return resp.StatusCode 132 | } 133 | --------------------------------------------------------------------------------