├── .dockerignore ├── .editorconfig ├── .github └── workflows │ ├── docker-image.yml │ └── go.yml ├── .gitignore ├── .travis.yml ├── API.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── api ├── api.go ├── config │ └── config.go ├── data │ ├── data.go │ ├── db.go │ ├── jobs.go │ ├── presets.go │ ├── settings.go │ └── users.go ├── encoder │ ├── encoder.go │ ├── ffmpeg.go │ └── ffprobe.go ├── helpers │ ├── crypto.go │ └── util.go ├── logging │ └── logging.go ├── machine │ ├── cloudinit.go │ ├── digitalocean.go │ └── machine.go ├── net │ ├── download.go │ ├── ftp.go │ ├── net.go │ ├── progress.go │ ├── s3.go │ └── upload.go ├── notify │ └── slack.go ├── server │ ├── handlers.go │ ├── jobs.go │ ├── jwt.go │ ├── machines.go │ ├── presets.go │ ├── routes.go │ ├── server.go │ ├── settings.go │ ├── status.go │ ├── storage.go │ ├── users.go │ └── workers.go ├── types │ ├── job.go │ ├── preset.go │ ├── setting.go │ └── user.go └── worker │ ├── encode_worker.go │ ├── job.go │ └── worker.go ├── cmd ├── root.go ├── server.go ├── version.go └── worker.go ├── config └── default.yml ├── docker-compose-letsencrypt.yml ├── docker-compose-production.yml ├── docker-compose.yml ├── go.mod ├── go.sum ├── main.go ├── screenshot.png ├── scripts ├── 00_setup.sh ├── 10_schema.sql ├── 20_settings_options.sql ├── 21_settings_defaults.sql ├── 30_presets.sql ├── 40_users.sql └── 50_jobs.sql └── web ├── .browserslistrc ├── .editorconfig ├── .eslintrc.js ├── .gitignore ├── README.md ├── babel.config.js ├── package-lock.json ├── package.json ├── postcss.config.js ├── public ├── img │ ├── icons │ │ ├── android-icon-144x144.png │ │ ├── android-icon-192x192.png │ │ ├── android-icon-36x36.png │ │ ├── android-icon-48x48.png │ │ ├── android-icon-72x72.png │ │ ├── android-icon-96x96.png │ │ ├── apple-icon-114x114.png │ │ ├── apple-icon-120x120.png │ │ ├── apple-icon-144x144.png │ │ ├── apple-icon-152x152.png │ │ ├── apple-icon-180x180.png │ │ ├── apple-icon-57x57.png │ │ ├── apple-icon-60x60.png │ │ ├── apple-icon-72x72.png │ │ ├── apple-icon-76x76.png │ │ ├── apple-icon-precomposed.png │ │ ├── apple-icon.png │ │ ├── favicon-16x16.png │ │ ├── favicon-32x32.png │ │ ├── favicon-96x96.png │ │ ├── favicon.ico │ │ ├── ms-icon-144x144.png │ │ ├── ms-icon-150x150.png │ │ ├── ms-icon-310x310.png │ │ └── ms-icon-70x70.png │ └── logo.png ├── index.html ├── manifest.json └── robots.txt ├── src ├── App.vue ├── api.js ├── assets │ └── logo.png ├── auth.js ├── components │ ├── FileBrowser.vue │ ├── JobForm.vue │ ├── JobsTable.vue │ ├── LoginForm.vue │ ├── MachinesForm.vue │ ├── MachinesTable.vue │ ├── PresetForm.vue │ ├── PresetsTable.vue │ ├── QueueTable.vue │ ├── RegisterForm.vue │ ├── SettingsForm.vue │ ├── Status.vue │ ├── UpdatePasswordForm.vue │ ├── UserProfile.vue │ ├── UsersTable.vue │ └── WorkersTable.vue ├── cookie.js ├── main.js ├── registerServiceWorker.js ├── router.js ├── store.js └── views │ ├── Encode.vue │ ├── Jobs.vue │ ├── Login.vue │ ├── Machines.vue │ ├── Presets.vue │ ├── PresetsCreate.vue │ ├── Queue.vue │ ├── Register.vue │ ├── Settings.vue │ ├── Status.vue │ ├── UpdatePassword.vue │ ├── UserProfile.vue │ ├── Users.vue │ └── Workers.vue └── vue.config.js /.dockerignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.mp4 3 | *.exe 4 | *.exe~ 5 | *.db 6 | .env 7 | bin/ 8 | media/ 9 | progress-log.txt 10 | vendor/ 11 | .vscode/ 12 | examples/ 13 | web/node_modules/ -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | ; http://editorconfig.org/ 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | charset = utf-8 9 | trim_trailing_whitespace = false 10 | insert_final_newline = false 11 | end_of_line = lf 12 | 13 | [{Makefile,go.mod,go.sum,*.go}] 14 | indent_style = tab 15 | indent_size = 4 -------------------------------------------------------------------------------- /.github/workflows/docker-image.yml: -------------------------------------------------------------------------------- 1 | # name: Docker Image Push 2 | 3 | # on: 4 | # release: 5 | # types: [created] 6 | 7 | # jobs: 8 | # build: 9 | # runs-on: ubuntu-latest 10 | # steps: 11 | # - uses: actions/checkout@master 12 | # - name: Build the Docker image 13 | # run: | 14 | # docker build --build-arg BUILD_VERSION=${GITHUB_REF/refs\/tags\//} . --file Dockerfile --tag openencoder:latest 15 | # docker tag openencoder alfg/openencoder:latest 16 | # docker login docker.pkg.github.com --username alfg --password ${{ secrets.TOKEN }} 17 | # docker tag openencoder docker.pkg.github.com/alfg/openencoder/openencoder:${GITHUB_REF/refs\/tags\//} 18 | # docker push docker.pkg.github.com/alfg/openencoder/openencoder:${GITHUB_REF/refs\/tags\//} 19 | 20 | name: Publish Docker image 21 | 22 | on: 23 | release: 24 | types: [published] 25 | 26 | jobs: 27 | push_to_registry: 28 | name: Push Docker image to Docker Hub 29 | runs-on: ubuntu-latest 30 | steps: 31 | - name: Check out the repo 32 | uses: actions/checkout@v2 33 | 34 | - name: Log in to Docker Hub 35 | uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 36 | with: 37 | username: ${{ secrets.DOCKER_USERNAME }} 38 | password: ${{ secrets.DOCKER_PASSWORD }} 39 | 40 | - name: Extract metadata (tags, labels) for Docker 41 | id: meta 42 | uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 43 | with: 44 | images: alfg/openencoder 45 | 46 | - name: Build and push Docker image 47 | uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc 48 | with: 49 | context: . 50 | push: true 51 | tags: ${{ steps.meta.outputs.tags }} 52 | labels: ${{ steps.meta.outputs.labels }} 53 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | on: [push] 3 | jobs: 4 | 5 | build: 6 | name: Build 7 | runs-on: ubuntu-latest 8 | steps: 9 | 10 | - name: Set up Go 1.14 11 | uses: actions/setup-go@v1 12 | with: 13 | go-version: 1.14 14 | id: go 15 | 16 | - name: Check out code into the Go module directory 17 | uses: actions/checkout@master 18 | 19 | - name: Get dependencies 20 | run: | 21 | go get -v -t -d ./... 22 | if [ -f Gopkg.toml ]; then 23 | curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh 24 | dep ensure 25 | fi 26 | - name: Build 27 | run: go build -v . 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .env 3 | *.mp4 4 | *.exe 5 | *.exe~ 6 | *.db 7 | bin/ 8 | media/ 9 | progress-log.txt 10 | vendor/ 11 | .vscode/ 12 | examples/ 13 | data/ -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - "1.11" 5 | - "1.12" 6 | - "1.13" 7 | - "1.14" 8 | - tip -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.14-alpine AS builder 2 | 3 | # Create the user and group files that will be used in the running container to 4 | # run the process as an unprivileged user. 5 | RUN mkdir /user && \ 6 | echo 'nobody:x:65534:65534:nobody:/:' > /user/passwd && \ 7 | echo 'nobody:x:65534:' > /user/group 8 | 9 | # Outside GOPATH since we're using modules. 10 | WORKDIR /src 11 | 12 | # Required for fetching dependencies. 13 | RUN apk add --update --no-cache ca-certificates git nodejs nodejs-npm 14 | 15 | # Fetch dependencies to cache. 16 | COPY go.mod go.sum ./ 17 | RUN go mod download 18 | 19 | # Copy project source files. 20 | COPY . . 21 | 22 | # Build static web project. 23 | RUN cd web && npm install && npm run build 24 | 25 | # Build. 26 | RUN CGO_ENABLED=0 GOOS=linux go build -installsuffix 'static' -v -o /app . 27 | 28 | # Final release image. 29 | FROM alfg/ffmpeg:latest 30 | 31 | # Set version from CI build. 32 | ARG BUILD_VERSION=${BUILD_VERSION} 33 | ENV VERSION=$BUILD_VERSION 34 | 35 | # Import the user and group files from the first stage. 36 | COPY --from=builder /user/group /user/passwd /etc/ 37 | 38 | # Import the Certificate-Authority certificates for enabling HTTPS. 39 | COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ 40 | 41 | # Import the project web & executable. 42 | COPY --from=builder /src/web/dist /web/dist 43 | COPY --from=builder /app /app 44 | COPY --from=builder /src/config/default.yml /config/default.yml 45 | 46 | EXPOSE 8080 47 | 48 | # Perform any further action as an unpriviledged user. 49 | USER nobody:nobody 50 | 51 | # Run binary. 52 | ENTRYPOINT ["/app"] 53 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Alfred Gutierrez 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | BINARY=openencoder 2 | 3 | .PHONY: all 4 | 5 | all: build 6 | 7 | build: 8 | CGO_ENABLED=0 GOOS=linux go build -installsuffix 'static' -v -o ${BINARY} . -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 |

openencoder

3 |

Open Source Cloud Encoder for FFmpeg

4 |

A distributed and scalable video encoding pipeline to be used 5 | as an API or web interface using your own hosted or cloud infrastructure 6 | and FFmpeg encoding presets. 7 |

8 |

⚠️ Currently functional, but a work-in-progress! Check back for updates!

9 |

10 | 11 | Build Status 12 | 13 | 14 | GoDoc 15 | 16 | 17 | Go Report Card 18 | 19 | 20 | Docker Automated build 21 | 22 | 23 | Docker Pulls 24 | 25 |

26 |
27 | 28 | 29 | ## Features 30 | * HTTP API for submitting jobs to a redis-backed FFmpeg worker 31 | * FTP and S3 storage (AWS, Digital Ocean Spaces and Custom S3 Providers supported) 32 | * Web Dashboard UI for managing encode jobs, workers, users and settings 33 | * Machines UI/API for scaling cloud worker instances in a VPC 34 | * Database stored FFmpeg encoding presets 35 | * User accounts and roles 36 | 37 | 38 | ## Preview 39 | ![Screenshot](screenshot.png) 40 | 41 | 42 | ## Development 43 | 44 | #### Requirements 45 | * Docker 46 | * Go 1.11+ 47 | * NodeJS 8+ (For web dashboard) 48 | * FFmpeg 49 | * Postgres 50 | * S3 API Credentials & Bucket (AWS or Digital Ocean) 51 | * Digital Ocean API Key (only required for Machines API) 52 | 53 | Docker is optional, but highly recommended for this setup. This guide assumes you are using Docker. 54 | 55 | 56 | #### Setup 57 | * Start Redis and Postgres in Docker: 58 | ``` 59 | docker-compose up -d redis 60 | docker-compose up -d db 61 | ``` 62 | 63 | When the database container runs for the first time, it will create a persistent volume as `/var/lib/postgresql/data`. It will also run the scripts in `scripts/` to create the database, schema, settings, presets, and an admin user. 64 | 65 | * Build & start API server: 66 | ``` 67 | go build -v && ./openencoder server 68 | ``` 69 | 70 | * Start the worker: 71 | ``` 72 | ./openencoder worker 73 | ``` 74 | 75 | * Start Web Dashboard for development: 76 | ``` 77 | cd static && npm run serve 78 | ``` 79 | 80 | * Open `http://localhost:8081/dashboard` in the browser and login with `admin/password`. 81 | 82 | 83 | See [Quick-Setup-Guide](https://github.com/alfg/openencoder/wiki/Quick-Setup-Guide-%5Bfor-development%5D) for full development setup guide. 84 | 85 | ## API 86 | See: [API.md](/API.md) 87 | 88 | 89 | ## Scaling 90 | You can scale workers by adding more machines via the Web UI or API. 91 | 92 | Currently only `Digital Ocean` is supported. More providers are planned. 93 | 94 | See: [API.md](/API.md) for Machines API documentation. 95 | 96 | 97 | ## Documentation 98 | See: [wiki](https://github.com/alfg/openencoder/wiki) for more documentation. 99 | 100 | 101 | ## Roadmap 102 | See: [Development Project](https://github.com/alfg/openencoder/projects/1) for current development tasks and status. 103 | 104 | 105 | ## License 106 | MIT 107 | -------------------------------------------------------------------------------- /api/api.go: -------------------------------------------------------------------------------- 1 | package api 2 | -------------------------------------------------------------------------------- /api/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "encoding/hex" 5 | "fmt" 6 | 7 | "github.com/spf13/viper" 8 | ) 9 | 10 | // C is a config instance available as a public config object. 11 | var C Config 12 | 13 | // Config defines the main configuration object. 14 | type Config struct { 15 | Port string `mapstructure:"server_port"` 16 | JWTKey string `mapstructure:"jwt_key"` 17 | Keyseed string `mapstructure:"keyseed"` 18 | RedisHost string `mapstructure:"redis_host"` 19 | RedisPort int `mapstructure:"redis_port"` 20 | RedisMaxActive int `mapstructure:"redis_max_active"` 21 | RedisMaxIdle int `mapstructure:"redis_max_idle"` 22 | DatabaseHost string `mapstructure:"database_host"` 23 | DatabasePort int `mapstructure:"database_port"` 24 | DatabaseUser string `mapstructure:"database_user"` 25 | DatabasePassword string `mapstructure:"database_password"` 26 | DatabaseName string `mapstructure:"database_name"` 27 | WorkerNamespace string `mapstructure:"worker_namespace"` 28 | WorkerJobName string `mapstructure:"worker_job_name"` 29 | WorkerConcurrency uint `mapstructure:"worker_concurrency"` 30 | WorkDirectory string `mapstructure:"work_dir"` 31 | 32 | CloudinitRedisHost string `mapstructure:"cloudinit_redis_host"` 33 | CloudinitRedisPort int `mapstructure:"cloudinit_redis_port"` 34 | CloudinitDatabaseHost string `mapstructure:"cloudinit_database_host"` 35 | CloudinitDatabasePort int `mapstructure:"cloudinit_database_port"` 36 | CloudinitDatabaseUser string `mapstructure:"cloudinit_database_user"` 37 | CloudinitDatabasePassword string `mapstructure:"cloudinit_database_password"` 38 | CloudinitDatabaseName string `mapstructure:"cloudinit_database_name"` 39 | CloudinitWorkerImage string `mapstructure:"cloudinit_worker_image"` 40 | } 41 | 42 | // LoadConfig loads up the configuration struct. 43 | func LoadConfig(file string) { 44 | viper.SetConfigType("yaml") 45 | viper.SetConfigName(file) 46 | viper.AddConfigPath(".") 47 | viper.AddConfigPath("config") 48 | err := viper.ReadInConfig() 49 | 50 | viper.AutomaticEnv() 51 | err = viper.Unmarshal(&C) 52 | if err != nil { 53 | panic(fmt.Errorf("fatal error config file: %s", err)) 54 | } 55 | } 56 | 57 | // Get gets the current config. 58 | func Get() *Config { 59 | return &C 60 | } 61 | 62 | // Keyseed gets the keyseed in a byte array. 63 | func Keyseed() []byte { 64 | ks, _ := hex.DecodeString(Get().Keyseed) 65 | return ks 66 | } 67 | -------------------------------------------------------------------------------- /api/data/data.go: -------------------------------------------------------------------------------- 1 | package data 2 | 3 | // Data represents the available database tables. 4 | type Data struct { 5 | Presets Presets 6 | Settings Settings 7 | Jobs Jobs 8 | Users Users 9 | } 10 | 11 | // New creates a new database instance. 12 | func New() *Data { 13 | return &Data{ 14 | Presets: &PresetsOp{}, 15 | Settings: &SettingsOp{}, 16 | Jobs: &JobsOp{}, 17 | Users: &UsersOp{}, 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /api/data/db.go: -------------------------------------------------------------------------------- 1 | package data 2 | 3 | import ( 4 | _ "database/sql" // Database. 5 | "fmt" 6 | 7 | "github.com/alfg/openencoder/api/config" 8 | "github.com/alfg/openencoder/api/logging" 9 | "github.com/jmoiron/sqlx" 10 | 11 | _ "github.com/lib/pq" // Postgres driver. 12 | ) 13 | 14 | const ( 15 | driverName = "postgres" 16 | ) 17 | 18 | var ( 19 | connectionString string 20 | conn *sqlx.DB 21 | log = logging.Log 22 | connectionFormat = "host=%s port=%d user=%s password=%s dbname=%s sslmode=disable" 23 | ) 24 | 25 | // ConnectDB Connects to postgres database 26 | func ConnectDB() (*sqlx.DB, error) { 27 | var err error 28 | if connectionString == "" { 29 | log.Info("setting database connectionString.") 30 | var ( 31 | host = config.Get().DatabaseHost 32 | port = config.Get().DatabasePort 33 | user = config.Get().DatabaseUser 34 | password = config.Get().DatabasePassword 35 | dbname = config.Get().DatabaseName 36 | ) 37 | connectionString = fmt.Sprintf(connectionFormat, host, port, user, password, dbname) 38 | } 39 | 40 | if conn, err = sqlx.Connect(driverName, connectionString); err != nil { 41 | log.Panic(err) 42 | } 43 | return conn, err 44 | } 45 | -------------------------------------------------------------------------------- /api/data/presets.go: -------------------------------------------------------------------------------- 1 | package data 2 | 3 | import ( 4 | "github.com/alfg/openencoder/api/types" 5 | ) 6 | 7 | // Presets represents the Presets database operations. 8 | type Presets interface { 9 | GetPresets(offset, count int) *[]types.Preset 10 | GetPresetByID(id int) (*types.Preset, error) 11 | GetPresetByName(name string) (*types.Preset, error) 12 | GetPresetsCount() int 13 | CreatePreset(user types.Preset) (*types.Preset, error) 14 | UpdatePresetByID(id int, preset types.Preset) *types.Preset 15 | UpdatePresetStatusByID(id int, active bool) error 16 | } 17 | 18 | // PresetsOp represents the users operations. 19 | type PresetsOp struct { 20 | u *Presets 21 | } 22 | 23 | var _ Presets = &PresetsOp{} 24 | 25 | // GetPresets Gets all presets. 26 | func (p PresetsOp) GetPresets(offset, count int) *[]types.Preset { 27 | const query = ` 28 | SELECT * FROM presets 29 | ORDER BY id DESC 30 | LIMIT $1 OFFSET $2` 31 | 32 | db, _ := ConnectDB() 33 | presets := []types.Preset{} 34 | err := db.Select(&presets, query, count, offset) 35 | if err != nil { 36 | log.Fatal(err) 37 | } 38 | db.Close() 39 | return &presets 40 | } 41 | 42 | // GetPresetByID Gets a preset by ID. 43 | func (p PresetsOp) GetPresetByID(id int) (*types.Preset, error) { 44 | const query = ` 45 | SELECT * 46 | FROM presets 47 | WHERE id = $1` 48 | 49 | db, _ := ConnectDB() 50 | preset := types.Preset{} 51 | err := db.Get(&preset, query, id) 52 | if err != nil { 53 | log.Fatal(err) 54 | return &preset, err 55 | } 56 | db.Close() 57 | return &preset, nil 58 | } 59 | 60 | // GetPresetByName Gets a preset by name. 61 | func (p PresetsOp) GetPresetByName(name string) (*types.Preset, error) { 62 | const query = ` 63 | SELECT * 64 | FROM presets 65 | WHERE name = $1` 66 | 67 | db, _ := ConnectDB() 68 | preset := types.Preset{} 69 | err := db.Get(&preset, query, name) 70 | if err != nil { 71 | log.Fatal(err) 72 | return &preset, err 73 | } 74 | db.Close() 75 | return &preset, nil 76 | } 77 | 78 | // GetPresetsCount Gets a count of all presets. 79 | func (p PresetsOp) GetPresetsCount() int { 80 | var count int 81 | const query = `SELECT COUNT(*) FROM presets` 82 | 83 | db, _ := ConnectDB() 84 | err := db.Get(&count, query) 85 | if err != nil { 86 | log.Fatal(err) 87 | } 88 | db.Close() 89 | return count 90 | } 91 | 92 | // CreatePreset creates a preset. 93 | func (p PresetsOp) CreatePreset(preset types.Preset) (*types.Preset, error) { 94 | const query = ` 95 | INSERT INTO 96 | presets (name,description,data,active,output) 97 | VALUES (:name,:description,:data,:active,:output) 98 | RETURNING id` 99 | 100 | db, _ := ConnectDB() 101 | tx := db.MustBegin() 102 | stmt, err := tx.PrepareNamed(query) 103 | if err != nil { 104 | log.Fatal(err) 105 | } 106 | 107 | var id int64 // Returned ID. 108 | err = stmt.QueryRowx(&preset).Scan(&id) 109 | if err != nil { 110 | log.Fatal(err.Error()) 111 | return nil, err 112 | } 113 | tx.Commit() 114 | preset.ID = id 115 | 116 | db.Close() 117 | return &preset, nil 118 | } 119 | 120 | // UpdatePresetByID Update job by ID. 121 | func (p PresetsOp) UpdatePresetByID(id int, preset types.Preset) *types.Preset { 122 | const query = ` 123 | UPDATE presets 124 | SET name = :name, description = :description, data = :data, active = :active 125 | WHERE id = :id` 126 | 127 | db, _ := ConnectDB() 128 | tx := db.MustBegin() 129 | _, err := tx.NamedExec(query, &preset) 130 | if err != nil { 131 | log.Fatal(err) 132 | } 133 | tx.Commit() 134 | 135 | db.Close() 136 | return &preset 137 | } 138 | 139 | // UpdatePresetStatusByID Update preset status by ID. 140 | func (p PresetsOp) UpdatePresetStatusByID(id int, active bool) error { 141 | const query = `UPDATE presets SET active = $2 WHERE id = $1` 142 | 143 | db, _ := ConnectDB() 144 | tx := db.MustBegin() 145 | _, err := tx.Exec(query, active, id) 146 | if err != nil { 147 | log.Fatal(err) 148 | return err 149 | } 150 | tx.Commit() 151 | 152 | db.Close() 153 | return nil 154 | } 155 | -------------------------------------------------------------------------------- /api/data/settings.go: -------------------------------------------------------------------------------- 1 | package data 2 | 3 | import ( 4 | "encoding/hex" 5 | "fmt" 6 | 7 | "github.com/alfg/openencoder/api/config" 8 | "github.com/alfg/openencoder/api/helpers" 9 | "github.com/alfg/openencoder/api/types" 10 | ) 11 | 12 | // Settings represents the Settings database operations. 13 | type Settings interface { 14 | GetSetting(key string) (*types.Setting, error) 15 | GetSettings() []types.Setting 16 | GetSettingsOptions() []types.SettingsOption 17 | CreateSetting(setting types.Setting) *types.Setting 18 | CreateOrUpdateSetting(key, value string) 19 | UpdateSettings(setting map[string]string) error 20 | UpdateSetting(setting types.Setting) *types.Setting 21 | SettingExists(optionID int64) bool 22 | } 23 | 24 | // SettingsOp represents the settings. 25 | type SettingsOp struct { 26 | s *Settings 27 | } 28 | 29 | var _ Settings = &SettingsOp{} 30 | 31 | // GetSetting Gets a setting. 32 | func (s SettingsOp) GetSetting(key string) (*types.Setting, error) { 33 | const query = ` 34 | SELECT 35 | settings.*, 36 | settings_option.id "settings_option.id", 37 | settings_option.name "settings_option.name", 38 | settings_option.title "settings_option.title", 39 | settings_option.description "settings_option.description", 40 | settings_option.secure "settings_option.secure" 41 | FROM settings 42 | JOIN settings_option ON settings.settings_option_id = settings_option.id 43 | WHERE settings_option.name = $1 44 | ORDER BY id DESC` 45 | 46 | db, _ := ConnectDB() 47 | setting := types.Setting{} 48 | err := db.Get(&setting, query, key) 49 | if err != nil { 50 | log.Error(err) 51 | return nil, err 52 | } 53 | db.Close() 54 | 55 | if setting.Secure { 56 | enc, _ := hex.DecodeString(setting.Value) 57 | plaintext, _ := helpers.Decrypt(enc, config.Keyseed()) 58 | setting.Value = string(plaintext) 59 | } 60 | return &setting, nil 61 | } 62 | 63 | // GetSettings Gets settings. 64 | func (s SettingsOp) GetSettings() []types.Setting { 65 | const query = ` 66 | SELECT 67 | settings.*, 68 | settings_option.id "settings_option.id", 69 | settings_option.name "settings_option.name", 70 | settings_option.title "settings_option.title", 71 | settings_option.description "settings_option.description", 72 | settings_option.secure "settings_option.secure" 73 | FROM settings 74 | JOIN settings_option ON settings.settings_option_id = settings_option.id 75 | ORDER BY id DESC` 76 | 77 | db, _ := ConnectDB() 78 | settings := []types.Setting{} 79 | err := db.Select(&settings, query) 80 | if err != nil { 81 | log.Error(err) 82 | } 83 | db.Close() 84 | 85 | for i := range settings { 86 | if settings[i].Secure { 87 | enc, _ := hex.DecodeString(settings[i].Value) 88 | plaintext, _ := helpers.Decrypt(enc, config.Keyseed()) 89 | settings[i].Value = string(plaintext) 90 | } 91 | } 92 | return settings 93 | } 94 | 95 | // GetSettingsOptions Gets all available setting options. 96 | func (s SettingsOp) GetSettingsOptions() []types.SettingsOption { 97 | const query = "SELECT * FROM settings_option ORDER BY id ASC" 98 | 99 | db, _ := ConnectDB() 100 | options := []types.SettingsOption{} 101 | err := db.Select(&options, query) 102 | if err != nil { 103 | log.Error(err) 104 | } 105 | db.Close() 106 | return options 107 | } 108 | 109 | // CreateSetting Creates a setting. 110 | func (s SettingsOp) CreateSetting(setting types.Setting) *types.Setting { 111 | const query = ` 112 | INSERT INTO 113 | settings (settings_option_id,value,encrypted) 114 | VALUES (:settings_option_id,:value,:encrypted) 115 | RETURNING id` 116 | 117 | db, _ := ConnectDB() 118 | tx := db.MustBegin() 119 | stmt, err := tx.PrepareNamed(query) 120 | if err != nil { 121 | log.Error(err.Error()) 122 | } 123 | 124 | var id int64 // Returned ID. 125 | err = stmt.QueryRowx(&setting).Scan(&id) 126 | if err != nil { 127 | log.Error(err.Error()) 128 | } 129 | tx.Commit() 130 | 131 | // Set to Job type response. 132 | setting.SettingsOptionID = id 133 | 134 | db.Close() 135 | return &setting 136 | } 137 | 138 | // CreateOrUpdateSetting Runs an "upsert"-like transaction for a setting. 139 | func (s SettingsOp) CreateOrUpdateSetting(key, value string) { 140 | availableSettings := s.GetSettingsOptions() 141 | k := getOptionKeyID(availableSettings, key) 142 | isSecure := isSecure(availableSettings, key) 143 | exists := s.SettingExists(k) 144 | 145 | se := types.Setting{ 146 | SettingsOptionID: k, 147 | Value: value, 148 | Encrypted: false, 149 | } 150 | 151 | if isSecure { 152 | ciphertext, _ := helpers.Encrypt([]byte(value), config.Keyseed()) 153 | se.Value = fmt.Sprintf("%x", ciphertext) 154 | se.Encrypted = true 155 | } 156 | 157 | if exists { 158 | s.UpdateSetting(se) 159 | } else { 160 | s.CreateSetting(se) 161 | } 162 | } 163 | 164 | // UpdateSettings Updates settings. 165 | func (s SettingsOp) UpdateSettings(setting map[string]string) error { 166 | 167 | // Run insert or update for each setting. 168 | for k, v := range setting { 169 | s.CreateOrUpdateSetting(k, v) 170 | } 171 | return nil 172 | } 173 | 174 | // UpdateSetting updates an existing setting. 175 | func (s SettingsOp) UpdateSetting(setting types.Setting) *types.Setting { 176 | const query = ` 177 | UPDATE settings 178 | SET value = :value, encrypted = :encrypted 179 | WHERE settings_option_id = :settings_option_id` 180 | 181 | db, _ := ConnectDB() 182 | tx := db.MustBegin() 183 | _, err := tx.NamedExec(query, &setting) 184 | if err != nil { 185 | log.Error(err) 186 | } 187 | tx.Commit() 188 | 189 | db.Close() 190 | return &setting 191 | } 192 | 193 | // SettingExists Queries a setting exists. 194 | func (s SettingsOp) SettingExists(optionID int64) bool { 195 | const query = ` 196 | SELECT EXISTS 197 | (SELECT id 198 | FROM settings 199 | WHERE settings_option_id = $1)` 200 | 201 | var exists bool 202 | db, _ := ConnectDB() 203 | err := db.QueryRow(query, optionID).Scan(&exists) 204 | if err != nil { 205 | log.Error(err) 206 | } 207 | db.Close() 208 | return exists 209 | } 210 | 211 | func getOptionKeyID(s []types.SettingsOption, key string) int64 { 212 | for _, a := range s { 213 | if a.Name == key { 214 | return a.ID 215 | } 216 | } 217 | return -1 218 | } 219 | 220 | func isSecure(s []types.SettingsOption, key string) bool { 221 | for _, a := range s { 222 | if a.Secure && key == a.Name { 223 | return true 224 | } 225 | } 226 | return false 227 | } 228 | -------------------------------------------------------------------------------- /api/data/users.go: -------------------------------------------------------------------------------- 1 | package data 2 | 3 | import ( 4 | "github.com/alfg/openencoder/api/types" 5 | ) 6 | 7 | // Users represents the Users database operations. 8 | type Users interface { 9 | GetUsers(offset, count int) *[]types.User 10 | GetUserByID(id int) (*types.User, error) 11 | GetUsersCount() int 12 | GetUserByUsername(username string) (*types.User, error) 13 | GetUserID(username string) int64 14 | CreateUser(user types.User) (*types.User, error) 15 | UpdateUserByID(id int, user *types.User) (*types.User, error) 16 | UpdateUserPasswordByID(id int64, user *types.User) (*types.User, error) 17 | } 18 | 19 | // UsersOp represents the users operations. 20 | type UsersOp struct { 21 | u *Users 22 | } 23 | 24 | var _ Users = &UsersOp{} 25 | 26 | // GetUsers gets a list of users with an offset and count. 27 | func (u UsersOp) GetUsers(offset, count int) *[]types.User { 28 | const query = ` 29 | SELECT * 30 | FROM users 31 | ORDER BY id DESC 32 | LIMIT $1 OFFSET $2` 33 | 34 | db, _ := ConnectDB() 35 | users := []types.User{} 36 | err := db.Select(&users, query, count, offset) 37 | if err != nil { 38 | log.Fatal(err) 39 | } 40 | db.Close() 41 | return &users 42 | } 43 | 44 | // GetUserByID Gets a user by ID. 45 | func (u UsersOp) GetUserByID(id int) (*types.User, error) { 46 | const query = ` 47 | SELECT * 48 | FROM users 49 | WHERE id = $1` 50 | 51 | db, _ := ConnectDB() 52 | user := types.User{} 53 | err := db.Get(&user, query, id) 54 | if err != nil { 55 | return &user, err 56 | } 57 | db.Close() 58 | return &user, nil 59 | } 60 | 61 | // GetUsersCount Gets a count of all users. 62 | func (u UsersOp) GetUsersCount() int { 63 | var count int 64 | const query = `SELECT COUNT(*) FROM users` 65 | 66 | db, _ := ConnectDB() 67 | err := db.Get(&count, query) 68 | if err != nil { 69 | log.Fatal(err) 70 | } 71 | db.Close() 72 | return count 73 | } 74 | 75 | // GetUserByUsername Gets a user by username. 76 | func (u UsersOp) GetUserByUsername(username string) (*types.User, error) { 77 | const query = ` 78 | SELECT 79 | users.* 80 | FROM users 81 | WHERE users.username = $1` 82 | 83 | db, _ := ConnectDB() 84 | user := types.User{} 85 | err := db.Get(&user, query, username) 86 | if err != nil { 87 | log.Fatal(err) 88 | return &user, err 89 | } 90 | db.Close() 91 | return &user, nil 92 | } 93 | 94 | // GetUserID Gets a user ID by username. 95 | func (u UsersOp) GetUserID(username string) int64 { 96 | const query = "SELECT id FROM users WHERE username = $1" 97 | 98 | var id int64 99 | 100 | db, _ := ConnectDB() 101 | err := db.QueryRow(query, username).Scan(&id) 102 | if err != nil { 103 | log.Fatal(err) 104 | } 105 | return id 106 | } 107 | 108 | // CreateUser creates a user. 109 | func (u UsersOp) CreateUser(user types.User) (*types.User, error) { 110 | const query = ` 111 | INSERT INTO 112 | users (username,password,role) 113 | VALUES (:username,:password,:role) 114 | RETURNING id` 115 | 116 | db, _ := ConnectDB() 117 | tx := db.MustBegin() 118 | stmt, err := tx.PrepareNamed(query) 119 | if err != nil { 120 | log.Fatal(err) 121 | } 122 | 123 | var id int64 // Returned ID. 124 | err = stmt.QueryRowx(&user).Scan(&id) 125 | if err != nil { 126 | log.Fatal(err.Error()) 127 | return nil, err 128 | } 129 | tx.Commit() 130 | user.ID = id 131 | 132 | db.Close() 133 | return &user, nil 134 | } 135 | 136 | // UpdateUserByID Update user by ID. 137 | func (u UsersOp) UpdateUserByID(id int, user *types.User) (*types.User, error) { 138 | const query = ` 139 | UPDATE users 140 | SET username = :username, password = :password, active = :active, role = :role 141 | WHERE id = :id` 142 | 143 | db, _ := ConnectDB() 144 | tx := db.MustBegin() 145 | _, err := tx.NamedExec(query, &user) 146 | if err != nil { 147 | log.Fatal(err) 148 | return user, err 149 | } 150 | tx.Commit() 151 | 152 | db.Close() 153 | return user, nil 154 | } 155 | 156 | // UpdateUserPasswordByID Update user password by ID and reset force_password_reset. 157 | func (u UsersOp) UpdateUserPasswordByID(id int64, user *types.User) (*types.User, error) { 158 | const query = ` 159 | UPDATE users 160 | SET password = :password, force_password_reset = false 161 | WHERE id = :id` 162 | 163 | db, _ := ConnectDB() 164 | tx := db.MustBegin() 165 | _, err := tx.NamedExec(query, &user) 166 | if err != nil { 167 | log.Fatal(err) 168 | return user, err 169 | } 170 | tx.Commit() 171 | 172 | db.Close() 173 | return user, nil 174 | } 175 | -------------------------------------------------------------------------------- /api/encoder/encoder.go: -------------------------------------------------------------------------------- 1 | package encoder 2 | 3 | import "github.com/alfg/openencoder/api/logging" 4 | 5 | var log = logging.Log 6 | -------------------------------------------------------------------------------- /api/encoder/ffprobe.go: -------------------------------------------------------------------------------- 1 | package encoder 2 | 3 | import ( 4 | "encoding/json" 5 | "os/exec" 6 | ) 7 | 8 | const ffprobeCmd = "ffprobe" 9 | 10 | // FFProbe struct. 11 | type FFProbe struct{} 12 | 13 | // Run runs an FFProbe command. 14 | func (f FFProbe) Run(input string) *FFProbeResponse { 15 | args := []string{ 16 | "-i", input, 17 | "-show_streams", 18 | "-print_format", "json", 19 | "-v", "quiet", 20 | } 21 | 22 | // Execute command. 23 | cmd := exec.Command(ffprobeCmd, args...) 24 | log.Info("Running FFprobe...") 25 | stdout, err := cmd.CombinedOutput() 26 | if err != nil { 27 | log.Error(err.Error()) 28 | } 29 | // log.Info((string(stdout)) 30 | 31 | dat := &FFProbeResponse{} 32 | if err := json.Unmarshal([]byte(stdout), &dat); err != nil { 33 | panic(err) 34 | } 35 | return dat 36 | } 37 | 38 | // FFProbeResponse defines the response from ffprobe. 39 | type FFProbeResponse struct { 40 | Streams []stream `json:"streams"` 41 | } 42 | 43 | type stream struct { 44 | Index int `json:"index"` 45 | CodecName string `json:"codec_name"` 46 | CodecLongName string `json:"codec_long_name"` 47 | Profile string `json:"profile"` 48 | CodecType string `json:"codec_type"` 49 | CodecTimeBase string `json:"codec_time_base"` 50 | CodecTagString string `json:"codec_tag_string"` 51 | CodecTag string `json:"codec_tag"` 52 | Width int `json:"width"` 53 | Height int `json:"height"` 54 | CodedWidth int `json:"coded_width"` 55 | CodedHeight int `json:"coded_height"` 56 | HasBFrames int `json:"has_b_frames"` 57 | SampleAspectRatio string `json:"sample_aspect_ratio"` 58 | DisplayAspectRatio string `json:"display_aspect_ratio"` 59 | PixFmt string `json:"pix_fmt"` 60 | Level int `json:"level"` 61 | ChromaLocation string `json:"chroma_location"` 62 | Refs int `json:"refs"` 63 | IsAVC string `json:"is_avc"` 64 | NalLengthSize string `json:"nal_length_size"` 65 | RFrameRate string `json:"r_frame_rate"` 66 | AvgFrameRate string `json:"avg_frame_rate"` 67 | TimeBase string `json:"time_base"` 68 | StartPts int `json:"start_pts"` 69 | StartTime string `json:"start_time"` 70 | DurationTS int `json:"duration_ts"` 71 | Duration string `json:"duration"` 72 | BitRate string `json:"bit_rate"` 73 | BitsPerRawSample string `json:"bits_per_raw_sample"` 74 | NbFrames string `json:"nb_frames"` 75 | Disposition disposition `json:"disposition"` 76 | Tags tags `json:"tags"` 77 | } 78 | 79 | type disposition struct { 80 | Default int `json:"default"` 81 | Dub int `json:"dub"` 82 | Original int `json:"original"` 83 | Comment int `json:"comment"` 84 | Lyrics int `json:"lyrics"` 85 | Karoake int `json:"karaoke"` 86 | Forced int `json:"forced"` 87 | HearingImpaired int `json:"hearing_impaired"` 88 | VisualImpaired int `json:"visual_empaired"` 89 | CleanEffects int `json:"clean_effects"` 90 | AttachedPic int `json:"attached_pic"` 91 | TimedThumbnails int `json:"timed_thumbnails"` 92 | } 93 | 94 | type tags struct { 95 | Language string `json:"language"` 96 | HandlerName string `json:"handler_name"` 97 | } 98 | -------------------------------------------------------------------------------- /api/helpers/crypto.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | import ( 4 | "crypto/aes" 5 | "crypto/cipher" 6 | "crypto/rand" 7 | "errors" 8 | "io" 9 | ) 10 | 11 | func Encrypt(plaintext []byte, key []byte) ([]byte, error) { 12 | c, err := aes.NewCipher(key) 13 | if err != nil { 14 | return nil, err 15 | } 16 | 17 | gcm, err := cipher.NewGCM(c) 18 | if err != nil { 19 | return nil, err 20 | } 21 | 22 | nonce := make([]byte, gcm.NonceSize()) 23 | if _, err = io.ReadFull(rand.Reader, nonce); err != nil { 24 | return nil, err 25 | } 26 | 27 | return gcm.Seal(nonce, nonce, plaintext, nil), nil 28 | } 29 | 30 | func Decrypt(ciphertext []byte, key []byte) ([]byte, error) { 31 | c, err := aes.NewCipher(key) 32 | if err != nil { 33 | return nil, err 34 | } 35 | 36 | gcm, err := cipher.NewGCM(c) 37 | if err != nil { 38 | return nil, err 39 | } 40 | 41 | nonceSize := gcm.NonceSize() 42 | if len(ciphertext) < nonceSize { 43 | return nil, errors.New("ciphertext too short") 44 | } 45 | 46 | nonce, ciphertext := ciphertext[:nonceSize], ciphertext[nonceSize:] 47 | return gcm.Open(nil, nonce, ciphertext, nil) 48 | } 49 | -------------------------------------------------------------------------------- /api/helpers/util.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | import ( 4 | "crypto/rand" 5 | "io" 6 | "os" 7 | "path" 8 | ) 9 | 10 | func CreateLocalSourcePath(workDir string, src string, ID string) string { 11 | // Get local destination path. 12 | tmpDir := workDir + "/" + ID + "/" 13 | os.MkdirAll(tmpDir, 0700) 14 | os.MkdirAll(tmpDir+"src", 0700) 15 | os.MkdirAll(tmpDir+"dst", 0700) 16 | return tmpDir + path.Base(src) 17 | } 18 | 19 | func GetTmpPath(workDir string, ID string) string { 20 | tmpDir := workDir + "/" + ID + "/" 21 | return tmpDir 22 | } 23 | 24 | func GenerateRandomKey(length int) []byte { 25 | k := make([]byte, length) 26 | if _, err := io.ReadFull(rand.Reader, k); err != nil { 27 | return nil 28 | } 29 | return k 30 | } 31 | -------------------------------------------------------------------------------- /api/logging/logging.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/sirupsen/logrus" 7 | ) 8 | 9 | // Log exports the configured logger. 10 | var Log *logrus.Logger 11 | 12 | func init() { 13 | log := logrus.New() 14 | 15 | // Set JSON formatter if production. 16 | if os.Getenv("GIN_MODE") == "release" { 17 | log.SetFormatter(&logrus.JSONFormatter{}) 18 | } 19 | log.SetOutput(os.Stdout) 20 | 21 | Log = log 22 | } 23 | -------------------------------------------------------------------------------- /api/machine/cloudinit.go: -------------------------------------------------------------------------------- 1 | package machine 2 | 3 | import ( 4 | "bytes" 5 | "html/template" 6 | 7 | "github.com/alfg/openencoder/api/config" 8 | ) 9 | 10 | const userDataTmpl = ` 11 | #cloud-config 12 | package_upgrade: false 13 | write_files: 14 | - path: "/opt/.env" 15 | content: | 16 | REDIS_HOST={{.CloudinitRedisHost}} 17 | REDIS_PORT={{.CloudinitRedisPort}} 18 | DATABASE_HOST={{.CloudinitDatabaseHost}} 19 | DATABASE_PORT={{.CloudinitDatabasePort}} 20 | DATABASE_USER={{.CloudinitDatabaseUser}} 21 | DATABASE_PASSWORD={{.CloudinitDatabasePassword}} 22 | DATABASE_NAME={{.CloudinitDatabaseName}} 23 | runcmd: 24 | - docker run -d --env-file /opt/.env --rm {{.CloudinitWorkerImage}} worker 25 | ` 26 | 27 | // UserData defines the userdata used for cloud-init. 28 | type UserData struct { 29 | CloudinitRedisHost string 30 | CloudinitRedisPort int 31 | CloudinitDatabaseHost string 32 | CloudinitDatabasePort int 33 | CloudinitDatabaseUser string 34 | CloudinitDatabasePassword string 35 | CloudinitDatabaseName string 36 | CloudinitWorkerImage string 37 | } 38 | 39 | func createUserData() string { 40 | data := &UserData{ 41 | CloudinitRedisHost: config.Get().CloudinitRedisHost, 42 | CloudinitRedisPort: config.Get().CloudinitRedisPort, 43 | CloudinitDatabaseHost: config.Get().CloudinitDatabaseHost, 44 | CloudinitDatabasePort: config.Get().CloudinitDatabasePort, 45 | CloudinitDatabaseUser: config.Get().CloudinitDatabaseUser, 46 | CloudinitDatabasePassword: config.Get().CloudinitDatabasePassword, 47 | CloudinitDatabaseName: config.Get().CloudinitDatabaseName, 48 | CloudinitWorkerImage: config.Get().CloudinitWorkerImage, 49 | } 50 | 51 | var tpl bytes.Buffer 52 | t := template.Must(template.New("userdata").Parse(userDataTmpl)) 53 | 54 | if err := t.Execute(&tpl, data); err != nil { 55 | log.Println(err) 56 | } 57 | 58 | result := tpl.String() 59 | return result 60 | } 61 | -------------------------------------------------------------------------------- /api/machine/digitalocean.go: -------------------------------------------------------------------------------- 1 | package machine 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/digitalocean/godo" 7 | "golang.org/x/oauth2" 8 | ) 9 | 10 | // TokenSource defines an access token for oauth2.TokenSource. 11 | type TokenSource struct { 12 | AccessToken string 13 | } 14 | 15 | // Token creates a token for TokenSource. 16 | func (t *TokenSource) Token() (*oauth2.Token, error) { 17 | token := &oauth2.Token{ 18 | AccessToken: t.AccessToken, 19 | } 20 | return token, nil 21 | } 22 | 23 | // DigitalOcean client. 24 | type DigitalOcean struct { 25 | client *godo.Client 26 | } 27 | 28 | // NewDigitalOceanClient creates a Digital Ocean client. 29 | func NewDigitalOceanClient(token string) (*DigitalOcean, error) { 30 | tokenSource := &TokenSource{ 31 | AccessToken: token, 32 | } 33 | 34 | oauthClient := oauth2.NewClient(context.Background(), tokenSource) 35 | client := godo.NewClient(oauthClient) 36 | 37 | return &DigitalOcean{ 38 | client: client, 39 | }, nil 40 | } 41 | 42 | // ListDropletByTag lists the droplets for a digitalocean account. 43 | func (do *DigitalOcean) ListDropletByTag(ctx context.Context, tag string) ([]Machine, error) { 44 | list := []Machine{} 45 | 46 | opt := &godo.ListOptions{} 47 | for { 48 | droplets, resp, err := do.client.Droplets.ListByTag(ctx, tag, opt) 49 | if err != nil { 50 | return nil, err 51 | } 52 | 53 | for _, d := range droplets { 54 | list = append(list, Machine{ 55 | ID: d.ID, 56 | Name: d.Name, 57 | Status: d.Status, 58 | SizeSlug: d.SizeSlug, 59 | Created: d.Created, 60 | Region: d.Region.Name, 61 | Tags: d.Tags, 62 | Provider: digitalOceanProviderName, 63 | }) 64 | } 65 | 66 | if resp.Links == nil || resp.Links.IsLastPage() { 67 | break 68 | } 69 | 70 | page, err := resp.Links.CurrentPage() 71 | if err != nil { 72 | return nil, err 73 | } 74 | opt.Page = page + 1 75 | } 76 | return list, nil 77 | } 78 | 79 | // CreateDroplets creates a new DigitalOcean droplet. 80 | func (do *DigitalOcean) CreateDroplets(ctx context.Context, region, size, vpc string, count int) ([]CreatedResponse, error) { 81 | 82 | var ( 83 | ipv6 = true 84 | tags = []string{tagName, workerTagName} 85 | monitoring = true 86 | privateNetworking = true 87 | ) 88 | 89 | var names []string 90 | for i := 0; i < count; i++ { 91 | names = append(names, workerTagName) 92 | } 93 | 94 | createRequest := &godo.DropletMultiCreateRequest{ 95 | Names: names, 96 | Region: region, 97 | Size: size, 98 | Image: godo.DropletCreateImage{ 99 | Slug: dockerImageName, 100 | }, 101 | // SSHKeys: []godo.DropletCreateSSHKey{ 102 | // godo.DropletCreateSSHKey{ID: 107149}, 103 | // }, 104 | UserData: createUserData(), 105 | Tags: tags, 106 | Monitoring: monitoring, 107 | IPv6: ipv6, 108 | PrivateNetworking: privateNetworking, 109 | VPCUUID: vpc, 110 | } 111 | 112 | droplets, _, err := do.client.Droplets.CreateMultiple(ctx, createRequest) 113 | if err != nil { 114 | return nil, err 115 | } 116 | 117 | list := []CreatedResponse{} 118 | for _, d := range droplets { 119 | list = append(list, CreatedResponse{ 120 | ID: d.ID, 121 | Provider: digitalOceanProviderName, 122 | }) 123 | } 124 | return list, nil 125 | } 126 | 127 | // DeleteDropletByID deletes a DigitalOcean droplet by ID. 128 | func (do *DigitalOcean) DeleteDropletByID(ctx context.Context, ID int) (*DeletedResponse, error) { 129 | _, err := do.client.Droplets.Delete(ctx, ID) 130 | if err != nil { 131 | return nil, err 132 | } 133 | 134 | deleted := &DeletedResponse{ 135 | ID: ID, 136 | Provider: digitalOceanProviderName, 137 | } 138 | return deleted, nil 139 | } 140 | 141 | // DeleteDropletByTag deletes a DigitalOcean droplet by Tag. 142 | func (do *DigitalOcean) DeleteDropletByTag(ctx context.Context, tag string) error { 143 | _, err := do.client.Droplets.DeleteByTag(ctx, tag) 144 | return err 145 | } 146 | 147 | // ListRegions gets a list of DigitalOcean regions. 148 | func (do *DigitalOcean) ListRegions(ctx context.Context) ([]Region, error) { 149 | opt := &godo.ListOptions{ 150 | Page: 1, 151 | PerPage: 200, 152 | } 153 | 154 | regions, _, err := do.client.Regions.List(ctx, opt) 155 | if err != nil { 156 | return nil, err 157 | } 158 | 159 | list := []Region{} 160 | for _, d := range regions { 161 | list = append(list, Region{ 162 | Name: d.Name, 163 | Slug: d.Slug, 164 | Sizes: d.Sizes, 165 | Available: d.Available, 166 | }) 167 | } 168 | 169 | return list, err 170 | } 171 | 172 | // ListSizes gets a list of DigitalOcean sizes. 173 | func (do *DigitalOcean) ListSizes(ctx context.Context) ([]Size, error) { 174 | opt := &godo.ListOptions{ 175 | Page: 1, 176 | PerPage: 200, 177 | } 178 | 179 | sizes, _, err := do.client.Sizes.List(ctx, opt) 180 | if err != nil { 181 | return nil, err 182 | } 183 | 184 | list := []Size{} 185 | for _, d := range sizes { 186 | // if contains(sizesLimiter, d.Slug) { 187 | list = append(list, Size{ 188 | Slug: d.Slug, 189 | Available: d.Available, 190 | PriceMonthly: d.PriceMonthly, 191 | PriceHourly: d.PriceHourly, 192 | }) 193 | // } 194 | } 195 | return list, err 196 | } 197 | 198 | // ListVPCs gets a list of DigitalOcean VPCs. 199 | func (do *DigitalOcean) ListVPCs(ctx context.Context) ([]VPC, error) { 200 | opt := &godo.ListOptions{ 201 | Page: 1, 202 | PerPage: 200, 203 | } 204 | 205 | vpcs, _, err := do.client.VPCs.List(ctx, opt) 206 | if err != nil { 207 | return nil, err 208 | } 209 | 210 | list := []VPC{} 211 | for _, d := range vpcs { 212 | list = append(list, VPC{ 213 | ID: d.ID, 214 | Name: d.Name, 215 | }) 216 | } 217 | return list, err 218 | } 219 | 220 | // GetCurrentPricing gets the current pricing data of running machines. 221 | func (do *DigitalOcean) GetCurrentPricing(ctx context.Context, tag string) (*Pricing, error) { 222 | 223 | // Get sizes first. 224 | opt := &godo.ListOptions{ 225 | Page: 1, 226 | PerPage: 200, 227 | } 228 | 229 | sizes, _, err := do.client.Sizes.List(ctx, opt) 230 | if err != nil { 231 | return nil, err 232 | } 233 | 234 | sizeList := []Size{} 235 | for _, d := range sizes { 236 | sizeList = append(sizeList, Size{ 237 | Slug: d.Slug, 238 | Available: d.Available, 239 | PriceMonthly: d.PriceMonthly, 240 | PriceHourly: d.PriceHourly, 241 | }) 242 | } 243 | 244 | // Get current running machines. 245 | machines := []Machine{} 246 | 247 | opt = &godo.ListOptions{} 248 | for { 249 | droplets, resp, err := do.client.Droplets.ListByTag(ctx, tag, opt) 250 | if err != nil { 251 | return nil, err 252 | } 253 | 254 | for _, d := range droplets { 255 | machines = append(machines, Machine{ 256 | ID: d.ID, 257 | Name: d.Name, 258 | Status: d.Status, 259 | SizeSlug: d.SizeSlug, 260 | Created: d.Created, 261 | Region: d.Region.Name, 262 | Tags: d.Tags, 263 | Provider: digitalOceanProviderName, 264 | }) 265 | } 266 | 267 | if resp.Links == nil || resp.Links.IsLastPage() { 268 | break 269 | } 270 | 271 | page, err := resp.Links.CurrentPage() 272 | if err != nil { 273 | return nil, err 274 | } 275 | opt.Page = page + 1 276 | } 277 | 278 | // Calculate pricing. 279 | var running = len(machines) 280 | var priceHourly float64 281 | var priceMonthly float64 282 | 283 | for _, m := range machines { 284 | slug := m.SizeSlug 285 | for _, sl := range sizeList { 286 | if sl.Slug == slug { 287 | priceHourly += sl.PriceHourly 288 | priceMonthly += sl.PriceMonthly 289 | } 290 | } 291 | } 292 | 293 | p := &Pricing{ 294 | Count: running, 295 | PriceHourly: priceHourly, 296 | PriceMonthly: priceMonthly, 297 | } 298 | 299 | return p, nil 300 | } 301 | 302 | func contains(slice []string, item string) bool { 303 | for _, s := range slice { 304 | if s == item { 305 | return true 306 | } 307 | } 308 | return false 309 | } 310 | -------------------------------------------------------------------------------- /api/machine/machine.go: -------------------------------------------------------------------------------- 1 | package machine 2 | 3 | import "github.com/alfg/openencoder/api/logging" 4 | 5 | var log = logging.Log 6 | 7 | const ( 8 | digitalOceanProviderName = "digitalocean" 9 | workerTagName = "openencoder-worker" 10 | tagName = "openencoder" 11 | dockerImageName = "docker-18-04" 12 | ) 13 | 14 | // var ( 15 | // sizesLimiter = []string{"s-1vcpu-1gb", "s-1vcpu-2gb"} 16 | // ) 17 | 18 | // Machine defines a machine struct from a provider. 19 | type Machine struct { 20 | ID int `json:"id"` 21 | Name string `json:"name"` 22 | Status string `json:"status"` 23 | SizeSlug string `json:"size_slug"` 24 | Created string `json:"created_at"` 25 | Region string `json:"region"` 26 | Tags []string `json:"tags"` 27 | 28 | Provider string `json:"provider"` 29 | } 30 | 31 | // CreatedResponse defines the response for creating a machine. 32 | type CreatedResponse struct { 33 | ID int `json:"id"` 34 | Provider string `json:"provider"` 35 | } 36 | 37 | // DeletedResponse defines the response for deleted a machine. 38 | type DeletedResponse struct { 39 | ID int `json:"id"` 40 | Provider string `json:"provider"` 41 | } 42 | 43 | // Region defines the response for listing regions. 44 | type Region struct { 45 | Name string `json:"name"` 46 | Slug string `json:"slug"` 47 | Sizes []string `json:"sizes"` 48 | Available bool `json:"available"` 49 | } 50 | 51 | // Size defines the response for listing sizes. 52 | type Size struct { 53 | Slug string `json:"slug"` 54 | Available bool `json:"available"` 55 | PriceMonthly float64 `json:"price_monthly"` 56 | PriceHourly float64 `json:"price_hourly"` 57 | } 58 | 59 | // Pricing defines the response for listing pricing. 60 | type Pricing struct { 61 | Count int `json:"count"` 62 | PriceHourly float64 `json:"price_hourly"` 63 | PriceMonthly float64 `json:"price_monthly"` 64 | } 65 | 66 | // VPC defines the response for listing VPCs. 67 | type VPC struct { 68 | ID string `json:"id"` 69 | Name string `json:"name"` 70 | } 71 | -------------------------------------------------------------------------------- /api/net/download.go: -------------------------------------------------------------------------------- 1 | package net 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/alfg/openencoder/api/data" 7 | "github.com/alfg/openencoder/api/types" 8 | ) 9 | 10 | // Download downloads a job source based on the driver setting. 11 | func Download(job types.Job) error { 12 | db := data.New() 13 | setting, err := db.Settings.GetSetting(types.StorageDriver) 14 | if err != nil { 15 | return err 16 | } 17 | driver := setting.Value 18 | 19 | if driver == "s3" { 20 | if err := s3Download(job); err != nil { 21 | return err 22 | } 23 | return nil 24 | } else if driver == "ftp" { 25 | if err := ftpDownload(job); err != nil { 26 | return err 27 | } 28 | return nil 29 | } 30 | return errors.New("no driver set") 31 | } 32 | 33 | // GetPresignedURL gets a presigned URL from S3. 34 | func GetPresignedURL(job types.Job) (string, error) { 35 | db := data.New() 36 | settings := db.Settings.GetSettings() 37 | 38 | config := S3Config{ 39 | AccessKey: types.GetSetting(types.S3AccessKey, settings), 40 | SecretKey: types.GetSetting(types.S3SecretKey, settings), 41 | Provider: types.GetSetting(types.S3Provider, settings), 42 | Region: types.GetSetting(types.S3OutboundBucketRegion, settings), 43 | InboundBucket: types.GetSetting(types.S3InboundBucket, settings), 44 | OutboundBucket: types.GetSetting(types.S3OutboundBucket, settings), 45 | } 46 | s3 := NewS3(config) 47 | str, err := s3.GetPresignedURL(job) 48 | if err != nil { 49 | return str, err 50 | } 51 | return str, nil 52 | } 53 | 54 | // S3Download sets the download function. 55 | func s3Download(job types.Job) error { 56 | db := data.New() 57 | settings := db.Settings.GetSettings() 58 | 59 | // Get job data. 60 | j, err := db.Jobs.GetJobByGUID(job.GUID) 61 | if err != nil { 62 | log.Error(err) 63 | return err 64 | } 65 | encodeID := j.EncodeID 66 | 67 | config := S3Config{ 68 | AccessKey: types.GetSetting(types.S3AccessKey, settings), 69 | SecretKey: types.GetSetting(types.S3SecretKey, settings), 70 | Provider: types.GetSetting(types.S3Provider, settings), 71 | Region: types.GetSetting(types.S3OutboundBucketRegion, settings), 72 | InboundBucket: types.GetSetting(types.S3InboundBucket, settings), 73 | OutboundBucket: types.GetSetting(types.S3OutboundBucket, settings), 74 | } 75 | s3 := NewS3(config) 76 | 77 | // Download with progress updates. 78 | go trackTransferProgress(encodeID, s3) 79 | err = s3.Download(job) 80 | close(progressCh) 81 | 82 | return err 83 | } 84 | 85 | // FTPDownload sets the FTP download function. 86 | func ftpDownload(job types.Job) error { 87 | db := data.New() 88 | settings := db.Settings.GetSettings() 89 | 90 | addr := types.GetSetting(types.FTPAddr, settings) 91 | user := types.GetSetting(types.FTPUsername, settings) 92 | pass := types.GetSetting(types.FTPPassword, settings) 93 | 94 | f := NewFTP(addr, user, pass) 95 | err := f.Download(job) 96 | return err 97 | } 98 | -------------------------------------------------------------------------------- /api/net/ftp.go: -------------------------------------------------------------------------------- 1 | package net 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "net/textproto" 7 | "net/url" 8 | "os" 9 | "path" 10 | "path/filepath" 11 | "time" 12 | 13 | "github.com/alfg/openencoder/api/types" 14 | "github.com/jlaffaye/ftp" 15 | ) 16 | 17 | const ( 18 | // ErrorFileExists error return from FTP client. 19 | ErrorFileExists = "Can't create directory: File exists" 20 | ) 21 | 22 | // FTP connection details. 23 | type FTP struct { 24 | Addr string 25 | Username string 26 | Password string 27 | Timeout time.Duration 28 | } 29 | 30 | // NewFTP creates a new FTP instance. 31 | func NewFTP(addr string, username string, password string) *FTP { 32 | return &FTP{ 33 | Addr: addr, 34 | Username: username, 35 | Password: password, 36 | Timeout: 5, 37 | } 38 | } 39 | 40 | // Download download a file from an FTP connection. 41 | func (f *FTP) Download(job types.Job) error { 42 | log.Info("downloading from FTP: ", job.Source) 43 | 44 | // Create FTP connection. 45 | c, err := ftp.Dial(f.Addr, ftp.DialWithTimeout(f.Timeout*time.Second)) 46 | if err != nil { 47 | log.Error(err) 48 | return err 49 | } 50 | 51 | // Login. 52 | err = c.Login(f.Username, f.Password) 53 | if err != nil { 54 | log.Error(err) 55 | return err 56 | } 57 | 58 | resp, err := c.Retr(job.Source) 59 | if err != nil { 60 | log.Error(err) 61 | return err 62 | } 63 | defer resp.Close() 64 | 65 | outputFile, _ := os.OpenFile(job.LocalSource, os.O_WRONLY|os.O_CREATE, 0644) 66 | defer outputFile.Close() 67 | 68 | reader := bufio.NewReader(resp) 69 | p := make([]byte, 1024*4) 70 | 71 | for { 72 | n, err := reader.Read(p) 73 | if err == io.EOF { 74 | break 75 | } 76 | outputFile.Write(p[:n]) 77 | } 78 | 79 | // Quit connection. 80 | if err := c.Quit(); err != nil { 81 | log.Error(err) 82 | return err 83 | } 84 | return err 85 | } 86 | 87 | // Upload uploads a file to FTP. 88 | func (f *FTP) Upload(job types.Job) error { 89 | log.Info("uploading files to FTP: ", job.Destination) 90 | defer log.Info("upload complete") 91 | 92 | // Get list of files in output dir. 93 | filelist := []string{} 94 | filepath.Walk(path.Dir(job.LocalSource)+"/dst", func(path string, f os.FileInfo, err error) error { 95 | if isDirectory(path) { 96 | return nil 97 | } 98 | filelist = append(filelist, path) 99 | return nil 100 | }) 101 | 102 | f.uploadDir(filelist, job) 103 | return nil 104 | } 105 | 106 | func (f *FTP) uploadDir(filelist []string, job types.Job) { 107 | for _, file := range filelist { 108 | f.uploadFile(file, job) 109 | } 110 | } 111 | 112 | // UploadFile uploads a file from an FTP connection. 113 | func (f *FTP) uploadFile(path string, job types.Job) error { 114 | // Create FTP connection. 115 | c, err := ftp.Dial(f.Addr, ftp.DialWithTimeout(f.Timeout*time.Second)) 116 | if err != nil { 117 | log.Error(err) 118 | return err 119 | } 120 | 121 | // Login. 122 | err = c.Login(f.Username, f.Password) 123 | if err != nil { 124 | log.Error(err) 125 | return err 126 | } 127 | 128 | file, err := os.Open(path) 129 | defer file.Close() 130 | if err != nil { 131 | return err 132 | } 133 | reader := bufio.NewReader(file) 134 | 135 | // Set destination path. 136 | parsedURL, _ := url.Parse(job.Destination) 137 | key := parsedURL.Path + filepath.Base(path) 138 | 139 | // Create directory. 140 | err = c.MakeDir(parsedURL.Path) 141 | if err != nil && err.(*textproto.Error).Msg != ErrorFileExists { 142 | log.Error(err) 143 | return err 144 | } 145 | 146 | err = c.Stor(key, reader) 147 | if err != nil { 148 | log.Error(err) 149 | return err 150 | } 151 | return nil 152 | } 153 | 154 | // ListFiles lists FTP files for a given prefix. 155 | func (f *FTP) ListFiles(prefix string) ([]*ftp.Entry, error) { 156 | c, err := ftp.Dial(f.Addr, ftp.DialWithTimeout(f.Timeout*time.Second)) 157 | if err != nil { 158 | log.Error(err) 159 | return nil, err 160 | } 161 | 162 | err = c.Login(f.Username, f.Password) 163 | if err != nil { 164 | log.Error(err) 165 | return nil, err 166 | } 167 | 168 | entries, err := c.List(prefix) 169 | 170 | if err := c.Quit(); err != nil { 171 | log.Error(err) 172 | return nil, err 173 | } 174 | return entries, nil 175 | } 176 | -------------------------------------------------------------------------------- /api/net/net.go: -------------------------------------------------------------------------------- 1 | package net 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/alfg/openencoder/api/logging" 7 | ) 8 | 9 | var log = logging.Log 10 | 11 | // Settings that S3 uses. 12 | const ( 13 | EndpointAmazonAWS = ".amazonaws.com" 14 | EndpointDigitalOceanSpaces = ".digitaloceanspaces.com" 15 | PresignedDuration = 72 * time.Hour // 3 days. 16 | ProgressInterval = time.Second * 5 17 | ) 18 | 19 | // S3 Provider Endpoints with region. 20 | var ( 21 | EndpointDigitalOceanSpacesRegion = func(region string) string { return region + EndpointDigitalOceanSpaces } 22 | EndpointAmazonAWSRegion = func(region string) string { return "s3." + region + EndpointAmazonAWS } 23 | progressCh chan struct{} 24 | ) 25 | -------------------------------------------------------------------------------- /api/net/progress.go: -------------------------------------------------------------------------------- 1 | package net 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "sync/atomic" 8 | 9 | "github.com/aws/aws-sdk-go/aws" 10 | "github.com/aws/aws-sdk-go/service/s3" 11 | ) 12 | 13 | // ProgressWriter tracks the download progress. 14 | type ProgressWriter struct { 15 | written int64 16 | writer io.WriterAt 17 | size int64 18 | } 19 | 20 | func (pw *ProgressWriter) WriteAt(p []byte, off int64) (int, error) { 21 | atomic.AddInt64(&pw.written, int64(len(p))) 22 | // percentageDownloaded := float32(pw.written*100) / float32(pw.size) 23 | // fmt.Printf("File size:%d downloaded:%d percentage:%.2f%%\r", pw.size, pw.written, percentageDownloaded) 24 | return pw.writer.WriteAt(p, off) 25 | } 26 | 27 | func byteCountDecimal(b int64) string { 28 | const unit = 1000 29 | if b < unit { 30 | return fmt.Sprintf("%d B", b) 31 | } 32 | div, exp := int64(unit), 0 33 | for n := b / unit; n >= unit; n /= unit { 34 | div *= unit 35 | exp++ 36 | } 37 | return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "kMGTPE"[exp]) 38 | } 39 | 40 | func getFileSize(svc *s3.S3, bucket, prefix string) (filesize int64, error error) { 41 | params := &s3.HeadObjectInput{ 42 | Bucket: aws.String(bucket), 43 | Key: aws.String(prefix), 44 | } 45 | 46 | resp, err := svc.HeadObject(params) 47 | if err != nil { 48 | return 0, err 49 | } 50 | return *resp.ContentLength, nil 51 | } 52 | 53 | // ProgressReader for uploading progress. 54 | type ProgressReader struct { 55 | fp *os.File 56 | size int64 57 | read int64 58 | Progress int 59 | } 60 | 61 | func (r *ProgressReader) Read(p []byte) (int, error) { 62 | return r.fp.Read(p) 63 | } 64 | 65 | func (r *ProgressReader) ReadAt(p []byte, off int64) (int, error) { 66 | n, err := r.fp.ReadAt(p, off) 67 | if err != nil { 68 | return n, err 69 | } 70 | atomic.AddInt64(&r.read, int64(n)) 71 | // fmt.Printf("total read:%d progress:%d%%\r", r.read/2, int(float32(r.read*100/2)/float32(r.size))) 72 | return n, err 73 | } 74 | 75 | func (r *ProgressReader) Seek(offset int64, whence int) (int64, error) { 76 | return r.fp.Seek(offset, whence) 77 | } 78 | -------------------------------------------------------------------------------- /api/net/upload.go: -------------------------------------------------------------------------------- 1 | package net 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/alfg/openencoder/api/data" 7 | "github.com/alfg/openencoder/api/types" 8 | ) 9 | 10 | // Upload uploads a job based on the driver setting. 11 | func Upload(job types.Job) error { 12 | db := data.New() 13 | driver, err := db.Settings.GetSetting(types.StorageDriver) 14 | if err != nil { 15 | return errors.New("no driver set") 16 | } 17 | 18 | if driver.Value == "s3" { 19 | if err := s3Upload(job); err != nil { 20 | return err 21 | } 22 | return nil 23 | } else if driver.Value == "ftp" { 24 | if err := ftpUpload(job); err != nil { 25 | return err 26 | } 27 | return nil 28 | } 29 | return errors.New("no driver set") 30 | } 31 | 32 | // GetUploader gets the upload function. 33 | func s3Upload(job types.Job) error { 34 | // Get credentials from settings. 35 | db := data.New() 36 | settings := db.Settings.GetSettings() 37 | 38 | config := S3Config{ 39 | AccessKey: types.GetSetting(types.S3AccessKey, settings), 40 | SecretKey: types.GetSetting(types.S3SecretKey, settings), 41 | Provider: types.GetSetting(types.S3Provider, settings), 42 | Region: types.GetSetting(types.S3OutboundBucketRegion, settings), 43 | InboundBucket: types.GetSetting(types.S3InboundBucket, settings), 44 | OutboundBucket: types.GetSetting(types.S3OutboundBucket, settings), 45 | } 46 | 47 | // Get job data. 48 | j, err := db.Jobs.GetJobByGUID(job.GUID) 49 | if err != nil { 50 | log.Error(err) 51 | return err 52 | } 53 | encodeID := j.EncodeID 54 | 55 | s3 := NewS3(config) 56 | go trackTransferProgress(encodeID, s3) 57 | err = s3.Upload(job) 58 | close(progressCh) 59 | 60 | return err 61 | } 62 | 63 | // GetFTPUploader sets the FTP upload function. 64 | func ftpUpload(job types.Job) error { 65 | db := data.New() 66 | settings := db.Settings.GetSettings() 67 | 68 | addr := types.GetSetting(types.FTPAddr, settings) 69 | user := types.GetSetting(types.FTPUsername, settings) 70 | pass := types.GetSetting(types.FTPPassword, settings) 71 | 72 | f := NewFTP(addr, user, pass) 73 | err := f.Upload(job) 74 | return err 75 | } 76 | -------------------------------------------------------------------------------- /api/notify/slack.go: -------------------------------------------------------------------------------- 1 | package notify 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "io/ioutil" 7 | "net/http" 8 | ) 9 | 10 | // SendSlackMessage sends a slack webhook post with a message. 11 | // url is the webhook. 12 | func SendSlackMessage(url string, message string) error { 13 | payload, _ := json.Marshal(map[string]interface{}{ 14 | "attachments": []map[string]string{ 15 | map[string]string{ 16 | "text": message, 17 | "color": "good", 18 | }, 19 | }, 20 | }) 21 | req, err := http.NewRequest("POST", url, bytes.NewBuffer(payload)) 22 | req.Header.Set("Content-Type", "application/json") 23 | 24 | client := &http.Client{} 25 | resp, err := client.Do(req) 26 | if err != nil { 27 | return err 28 | } 29 | defer resp.Body.Close() 30 | 31 | _, err = ioutil.ReadAll(resp.Body) 32 | return err 33 | } 34 | -------------------------------------------------------------------------------- /api/server/handlers.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | 7 | "github.com/alfg/openencoder/api/config" 8 | "github.com/alfg/openencoder/api/data" 9 | "github.com/gin-gonic/gin" 10 | "github.com/gocraft/work" 11 | "github.com/gomodule/redigo/redis" 12 | ) 13 | 14 | func indexHandler(c *gin.Context) { 15 | c.JSON(http.StatusOK, gin.H{ 16 | "name": ProjectName, 17 | "version": ProjectVersion, 18 | "github": ProjectGithub, 19 | "docs": ProjectDocs, 20 | "wiki": ProjectWiki, 21 | }) 22 | } 23 | 24 | func healthHandler(c *gin.Context) { 25 | var ( 26 | dbHealth = OK 27 | redisHealth = OK 28 | ) 29 | 30 | // Check database health. 31 | db, err := data.ConnectDB() 32 | if err != nil { 33 | dbHealth = NOK 34 | } 35 | err = db.Ping() 36 | if err != nil { 37 | dbHealth = NOK 38 | } 39 | defer db.Close() 40 | 41 | // Check Redis health. 42 | conn, err := redis.Dial("tcp", fmt.Sprintf("%s:%d", config.Get().RedisHost, config.Get().RedisPort)) 43 | if err != nil { 44 | redisHealth = NOK 45 | } 46 | defer conn.Close() 47 | 48 | // Check workers heartbeat. 49 | client := work.NewClient(config.Get().WorkerNamespace, redisPool) 50 | workerHeartbeats, _ := client.WorkerPoolHeartbeats() 51 | workers := len(workerHeartbeats) 52 | 53 | c.JSON(200, gin.H{ 54 | "api": OK, 55 | "db": dbHealth, 56 | "redis": redisHealth, 57 | "workers": workers, 58 | }) 59 | } 60 | -------------------------------------------------------------------------------- /api/server/jobs.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "database/sql" 5 | "net/http" 6 | "strconv" 7 | "sync" 8 | 9 | "github.com/alfg/openencoder/api/config" 10 | "github.com/alfg/openencoder/api/data" 11 | "github.com/alfg/openencoder/api/types" 12 | "github.com/gin-gonic/gin" 13 | "github.com/gocraft/work" 14 | "github.com/rs/xid" 15 | ) 16 | 17 | type request struct { 18 | Preset string `json:"preset" binding:"required"` 19 | Source string `json:"source" binding:"required"` 20 | Destination string `json:"dest" binding:"required"` 21 | } 22 | 23 | type updateRequest struct { 24 | Status string `json:"status"` 25 | } 26 | 27 | type response struct { 28 | Message string `json:"message"` 29 | Status int `json:"status"` 30 | Job *types.Job `json:"job"` 31 | } 32 | 33 | func createJobHandler(c *gin.Context) { 34 | user, _ := c.Get(JwtIdentityKey) 35 | 36 | // Role check. 37 | if !isAdminOrOperator(user) { 38 | c.JSON(http.StatusUnauthorized, gin.H{"message": "unauthorized"}) 39 | return 40 | } 41 | 42 | // Decode json. 43 | var json request 44 | if err := c.ShouldBindJSON(&json); err != nil { 45 | c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) 46 | return 47 | } 48 | 49 | // Create Job and push the work to work queue. 50 | job := types.Job{ 51 | GUID: xid.New().String(), 52 | Preset: json.Preset, 53 | Source: json.Source, 54 | Destination: json.Destination, 55 | Status: types.JobQueued, // Status queued. 56 | } 57 | 58 | // Send to work queue. 59 | _, err := enqueuer.Enqueue(config.Get().WorkerJobName, work.Q{ 60 | "guid": job.GUID, 61 | "preset": job.Preset, 62 | "source": job.Source, 63 | "destination": job.Destination, 64 | }) 65 | if err != nil { 66 | log.Info(err) 67 | } 68 | 69 | db := data.New() 70 | created := db.Jobs.CreateJob(job) 71 | 72 | // Create the encode relationship. 73 | ed := types.Encode{ 74 | JobID: created.ID, 75 | Progress: types.NullFloat64{ 76 | NullFloat64: sql.NullFloat64{ 77 | Float64: 0, 78 | Valid: true, 79 | }, 80 | }, 81 | Probe: types.NullString{ 82 | NullString: sql.NullString{ 83 | String: "{}", 84 | Valid: true, 85 | }, 86 | }, 87 | Options: types.NullString{ 88 | NullString: sql.NullString{ 89 | String: "{}", 90 | Valid: true, 91 | }, 92 | }, 93 | } 94 | edCreated := db.Jobs.CreateEncode(ed) 95 | created.EncodeID = edCreated.EncodeID 96 | 97 | // Create response. 98 | resp := response{ 99 | Message: "Job created", 100 | Status: 200, 101 | Job: created, 102 | } 103 | c.JSON(http.StatusCreated, resp) 104 | } 105 | 106 | func getJobsHandler(c *gin.Context) { 107 | page := c.DefaultQuery("page", "1") 108 | count := c.DefaultQuery("count", "10") 109 | pageInt, _ := strconv.Atoi(page) 110 | countInt, _ := strconv.Atoi(count) 111 | 112 | if page == "0" { 113 | pageInt = 1 114 | } 115 | 116 | var wg sync.WaitGroup 117 | var jobs *[]types.Job 118 | var jobsCount int 119 | 120 | db := data.New() 121 | wg.Add(1) 122 | go func() { 123 | jobs = db.Jobs.GetJobs((pageInt-1)*countInt, countInt) 124 | wg.Done() 125 | }() 126 | 127 | wg.Add(1) 128 | go func() { 129 | jobsCount = db.Jobs.GetJobsCount() 130 | wg.Done() 131 | }() 132 | wg.Wait() 133 | 134 | c.JSON(http.StatusOK, gin.H{ 135 | "count": jobsCount, 136 | "items": jobs, 137 | }) 138 | } 139 | 140 | func getJobsByIDHandler(c *gin.Context) { 141 | id := c.Param("id") 142 | jobInt, _ := strconv.Atoi(id) 143 | 144 | db := data.New() 145 | job, err := db.Jobs.GetJobByID(int64(jobInt)) 146 | if err != nil { 147 | c.JSON(http.StatusNotFound, gin.H{ 148 | "status": http.StatusNotFound, 149 | "message": "Job does not exist", 150 | }) 151 | return 152 | } 153 | 154 | c.JSON(http.StatusOK, gin.H{ 155 | "status": http.StatusOK, 156 | "job": job, 157 | }) 158 | } 159 | 160 | func updateJobByIDHandler(c *gin.Context) { 161 | id, _ := strconv.Atoi(c.Param("id")) 162 | user, _ := c.Get(JwtIdentityKey) 163 | 164 | // Role check. 165 | if !isAdminOrOperator(user) { 166 | c.JSON(http.StatusUnauthorized, gin.H{"message": "unauthorized"}) 167 | return 168 | } 169 | 170 | // Decode json. 171 | var json updateRequest 172 | if err := c.ShouldBindJSON(&json); err != nil { 173 | c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) 174 | return 175 | } 176 | 177 | db := data.New() 178 | job, err := db.Jobs.GetJobByID(int64(id)) 179 | if err != nil { 180 | c.JSON(http.StatusNotFound, gin.H{ 181 | "status": http.StatusNotFound, 182 | "message": "Job does not exist", 183 | }) 184 | return 185 | } 186 | 187 | if json.Status != "" { 188 | job.Status = json.Status 189 | } 190 | 191 | updatedJob := db.Jobs.UpdateJobByID(id, *job) 192 | c.JSON(http.StatusOK, updatedJob) 193 | } 194 | 195 | func getJobStatusByIDHandler(c *gin.Context) { 196 | id, _ := strconv.Atoi(c.Param("id")) 197 | 198 | // Update status. 199 | db := data.New() 200 | status, _ := db.Jobs.GetJobStatusByID(int64(id)) 201 | 202 | c.JSON(http.StatusOK, gin.H{ 203 | "status": http.StatusOK, 204 | "job_status": status, 205 | }) 206 | } 207 | 208 | func cancelJobByIDHandler(c *gin.Context) { 209 | id, _ := strconv.Atoi(c.Param("id")) 210 | user, _ := c.Get(JwtIdentityKey) 211 | 212 | // Role check. 213 | if !isAdminOrOperator(user) { 214 | c.JSON(http.StatusUnauthorized, gin.H{"message": "unauthorized"}) 215 | return 216 | } 217 | 218 | // Update status. 219 | db := data.New() 220 | db.Jobs.UpdateJobStatusByID(id, types.JobCancelled) 221 | 222 | c.JSON(http.StatusOK, gin.H{ 223 | "status": http.StatusOK, 224 | }) 225 | } 226 | 227 | func restartJobByIDHandler(c *gin.Context) { 228 | id, _ := strconv.Atoi(c.Param("id")) 229 | user, _ := c.Get(JwtIdentityKey) 230 | 231 | // Role check. 232 | if !isAdminOrOperator(user) { 233 | c.JSON(http.StatusUnauthorized, gin.H{"message": "unauthorized"}) 234 | return 235 | } 236 | 237 | // Update status. 238 | db := data.New() 239 | db.Jobs.UpdateJobStatusByID(id, types.JobRestarting) 240 | 241 | job, _ := db.Jobs.GetJobByID(int64(id)) 242 | 243 | // Send back to work queue. 244 | _, err := enqueuer.Enqueue(config.Get().WorkerJobName, work.Q{ 245 | "guid": job.GUID, 246 | "preset": job.Preset, 247 | "source": job.Source, 248 | "destination": job.Destination, 249 | }) 250 | if err != nil { 251 | log.Info(err) 252 | } 253 | 254 | c.JSON(http.StatusOK, gin.H{ 255 | "status": http.StatusOK, 256 | }) 257 | } 258 | -------------------------------------------------------------------------------- /api/server/jwt.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "errors" 5 | "time" 6 | 7 | "github.com/alfg/openencoder/api/config" 8 | "github.com/alfg/openencoder/api/data" 9 | "github.com/alfg/openencoder/api/helpers" 10 | "github.com/alfg/openencoder/api/types" 11 | jwt "github.com/appleboy/gin-jwt/v2" 12 | "github.com/gin-gonic/gin" 13 | "golang.org/x/crypto/bcrypt" 14 | ) 15 | 16 | type login struct { 17 | Username string `form:"username" json:"username" binding:"required"` 18 | Password string `form:"password" json:"password" binding:"required"` 19 | } 20 | 21 | var jwtKey []byte 22 | 23 | func jwtMiddleware() *jwt.GinJWTMiddleware { 24 | 25 | // Set the JWT Key if provided in config. Otherwise, generate a random one. 26 | key := config.Get().JWTKey 27 | if key == "" { 28 | jwtKey = helpers.GenerateRandomKey(16) 29 | } else { 30 | jwtKey = []byte(key) 31 | } 32 | 33 | authMiddleware, err := jwt.New(&jwt.GinJWTMiddleware{ 34 | Realm: JwtRealm, 35 | Key: jwtKey, 36 | Timeout: JwtTimeout, 37 | MaxRefresh: JwtMaxRefresh, 38 | IdentityKey: JwtIdentityKey, 39 | 40 | PayloadFunc: func(data interface{}) jwt.MapClaims { 41 | if v, ok := data.(*types.User); ok { 42 | return jwt.MapClaims{ 43 | JwtIdentityKey: v.Username, 44 | JwtRoleKey: v.Role, 45 | } 46 | } 47 | return jwt.MapClaims{} 48 | }, 49 | 50 | IdentityHandler: func(c *gin.Context) interface{} { 51 | claims := jwt.ExtractClaims(c) 52 | return &types.User{ 53 | Username: claims["id"].(string), 54 | Role: claims["role"].(string), 55 | } 56 | }, 57 | 58 | Authenticator: func(c *gin.Context) (interface{}, error) { 59 | var loginVals login 60 | if err := c.ShouldBind(&loginVals); err != nil { 61 | return "", jwt.ErrMissingLoginValues 62 | } 63 | userID := loginVals.Username 64 | password := loginVals.Password 65 | 66 | db := data.New() 67 | user, err := db.Users.GetUserByUsername(userID) 68 | if err != nil { 69 | return nil, jwt.ErrFailedAuthentication 70 | } 71 | 72 | // Check the encrypted password. 73 | err = bcrypt.CompareHashAndPassword([]byte(user.Password), []byte(password)) 74 | if err != nil { 75 | return nil, jwt.ErrFailedAuthentication 76 | } 77 | 78 | // Error with 403 if password needs to be reset. 79 | if user.ForcePasswordReset { 80 | return nil, errors.New("require password reset") 81 | } 82 | 83 | // Log-in the user. 84 | return &types.User{ 85 | Username: user.Username, 86 | Role: user.Role, 87 | }, nil 88 | }, 89 | 90 | Authorizator: func(data interface{}, c *gin.Context) bool { 91 | // Only authorize if user has the following roles. 92 | if v, ok := data.(*types.User); ok && 93 | (v.Role == "guest" || v.Role == "operator" || v.Role == "admin") { 94 | return true 95 | } 96 | return false 97 | }, 98 | 99 | Unauthorized: func(c *gin.Context, code int, message string) { 100 | c.JSON(code, gin.H{ 101 | "code": code, 102 | "message": message, 103 | }) 104 | }, 105 | 106 | LoginResponse: func(c *gin.Context, code int, message string, time time.Time) { 107 | c.JSON(code, gin.H{ 108 | "code": code, 109 | "token": message, 110 | "expire": time, 111 | }) 112 | }, 113 | 114 | TokenLookup: "header: Authorization, query: token, cookie: jwt", 115 | TokenHeadName: "Bearer", 116 | TimeFunc: time.Now, 117 | }) 118 | 119 | if err != nil { 120 | log.Error("JWT Error:" + err.Error()) 121 | } 122 | return authMiddleware 123 | } 124 | 125 | func isAdminOrOperator(user interface{}) bool { 126 | role := user.(*types.User).Role 127 | if role != RoleOperator && role != RoleAdmin { 128 | return false 129 | } 130 | return true 131 | } 132 | 133 | func isOperator(user interface{}) bool { 134 | role := user.(*types.User).Role 135 | if role != RoleOperator { 136 | return false 137 | } 138 | return true 139 | } 140 | 141 | func isAdmin(user interface{}) bool { 142 | role := user.(*types.User).Role 143 | if role != RoleAdmin { 144 | return false 145 | } 146 | return true 147 | } 148 | -------------------------------------------------------------------------------- /api/server/presets.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "net/http" 5 | "strconv" 6 | "sync" 7 | 8 | "github.com/alfg/openencoder/api/data" 9 | "github.com/alfg/openencoder/api/types" 10 | "github.com/gin-gonic/gin" 11 | ) 12 | 13 | type createPresetRequest struct { 14 | Name string `json:"name" binding:"required"` 15 | Description string `json:"description" binding:"required"` 16 | Output string `json:"output" binding:"required"` 17 | Data string `json:"data" binding:"required"` 18 | Active *bool `json:"active" binding:"required"` 19 | } 20 | 21 | type presetUpdateRequest struct { 22 | Name string `json:"name" binding:"required"` 23 | Description string `json:"description" binding:"required"` 24 | Output string `json:"output" binding:"required"` 25 | Data string `json:"data" binding:"required"` 26 | Active *bool `json:"active" binding:"required"` 27 | } 28 | 29 | func createPresetHandler(c *gin.Context) { 30 | user, _ := c.Get(JwtIdentityKey) 31 | 32 | // Role check. 33 | if !isAdminOrOperator(user) { 34 | c.JSON(http.StatusUnauthorized, gin.H{"message": "unauthorized"}) 35 | return 36 | } 37 | 38 | // Decode json. 39 | var json createPresetRequest 40 | if err := c.ShouldBindJSON(&json); err != nil { 41 | c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) 42 | return 43 | } 44 | 45 | // Create Preset. 46 | preset := types.Preset{ 47 | Name: json.Name, 48 | Description: json.Description, 49 | Output: json.Output, 50 | Data: json.Data, 51 | Active: json.Active, 52 | } 53 | 54 | db := data.New() 55 | created, err := db.Presets.CreatePreset(preset) 56 | if err != nil { 57 | log.Error(err) 58 | } 59 | 60 | // Create response. 61 | c.JSON(http.StatusCreated, gin.H{ 62 | "status": http.StatusCreated, 63 | "preset": created, 64 | }) 65 | } 66 | 67 | func getPresetsHandler(c *gin.Context) { 68 | page := c.DefaultQuery("page", "1") 69 | count := c.DefaultQuery("count", "10") 70 | pageInt, _ := strconv.Atoi(page) 71 | countInt, _ := strconv.Atoi(count) 72 | 73 | if page == "0" { 74 | pageInt = 1 75 | } 76 | 77 | var wg sync.WaitGroup 78 | var presets *[]types.Preset 79 | var presetsCount int 80 | 81 | db := data.New() 82 | wg.Add(1) 83 | go func() { 84 | presets = db.Presets.GetPresets((pageInt-1)*countInt, countInt) 85 | wg.Done() 86 | }() 87 | 88 | wg.Add(1) 89 | go func() { 90 | presetsCount = db.Presets.GetPresetsCount() 91 | wg.Done() 92 | }() 93 | wg.Wait() 94 | 95 | c.JSON(http.StatusOK, gin.H{ 96 | "count": presetsCount, 97 | "presets": presets, 98 | }) 99 | } 100 | 101 | func getPresetByIDHandler(c *gin.Context) { 102 | id := c.Param("id") 103 | presetInt, _ := strconv.Atoi(id) 104 | 105 | db := data.New() 106 | preset, err := db.Presets.GetPresetByID(presetInt) 107 | if err != nil { 108 | c.JSON(http.StatusNotFound, gin.H{ 109 | "status": http.StatusNotFound, 110 | "message": "Preset does not exist", 111 | }) 112 | return 113 | } 114 | 115 | c.JSON(http.StatusOK, gin.H{ 116 | "status": http.StatusOK, 117 | "preset": preset, 118 | }) 119 | } 120 | 121 | func updatePresetByIDHandler(c *gin.Context) { 122 | id, _ := strconv.Atoi(c.Param("id")) 123 | user, _ := c.Get(JwtIdentityKey) 124 | 125 | // Role check. 126 | if !isAdminOrOperator(user) { 127 | c.JSON(http.StatusUnauthorized, gin.H{"message": "unauthorized"}) 128 | return 129 | } 130 | 131 | // Decode json. 132 | var json presetUpdateRequest 133 | if err := c.ShouldBindJSON(&json); err != nil { 134 | c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) 135 | return 136 | } 137 | 138 | db := data.New() 139 | preset, err := db.Presets.GetPresetByID(id) 140 | if err != nil { 141 | c.JSON(http.StatusNotFound, gin.H{ 142 | "status": http.StatusNotFound, 143 | "message": "Preset does not exist", 144 | }) 145 | return 146 | } 147 | 148 | // Update struct with new data if provided. 149 | if json.Name != "" { 150 | preset.Name = json.Name 151 | } 152 | 153 | if json.Description != "" { 154 | preset.Description = json.Description 155 | } 156 | 157 | if json.Data != "" { 158 | preset.Data = json.Data 159 | } 160 | 161 | if json.Output != "" { 162 | preset.Output = json.Output 163 | } 164 | 165 | preset.Active = json.Active 166 | 167 | updatedPreset := db.Presets.UpdatePresetByID(id, *preset) 168 | c.JSON(http.StatusOK, updatedPreset) 169 | } 170 | -------------------------------------------------------------------------------- /api/server/routes.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import "github.com/gin-gonic/gin" 4 | 5 | func registerRoutes(r *gin.Engine) { 6 | 7 | // JWT middleware. 8 | authMiddlware := jwtMiddleware() 9 | r.POST("/api/register", registerHandler) 10 | r.POST("/api/login", authMiddlware.LoginHandler) 11 | r.GET("/api/refresh-token", authMiddlware.RefreshHandler) 12 | r.POST("/api/update-password", updatePasswordHandler) 13 | r.GET("/api/", indexHandler) 14 | r.GET("/api/health", healthHandler) 15 | 16 | // API routes. 17 | api := r.Group("/api") 18 | api.Use(authMiddlware.MiddlewareFunc()) 19 | { 20 | // User profile. 21 | api.GET("/me", getUserProfileHandler) 22 | api.PUT("/me", updateUserProfileHandler) 23 | 24 | // Storage. 25 | api.GET("/storage/list", storageListHandler) 26 | 27 | // Jobs. 28 | api.POST("/jobs", createJobHandler) 29 | api.GET("/jobs", getJobsHandler) 30 | api.GET("/jobs/:id", getJobsByIDHandler) 31 | api.PUT("/jobs/:id", updateJobByIDHandler) 32 | api.GET("/jobs/:id/status", getJobStatusByIDHandler) 33 | api.POST("/jobs/:id/cancel", cancelJobByIDHandler) 34 | api.POST("/jobs/:id/restart", restartJobByIDHandler) 35 | 36 | // Stats. 37 | api.GET("/stats", getStatsHandler) 38 | 39 | // Worker info. 40 | api.GET("/worker/queue", workerQueueHandler) 41 | api.GET("/worker/pools", workerPoolsHandler) 42 | api.GET("/worker/busy", workerBusyHandler) 43 | 44 | // Machines. 45 | api.GET("/machines", machinesHandler) 46 | api.POST("/machines", createMachineHandler) 47 | api.DELETE("/machines", deleteMachineByTagHandler) 48 | api.DELETE("/machines/:id", deleteMachineHandler) 49 | api.GET("/machines/regions", listMachineRegionsHandler) 50 | api.GET("/machines/sizes", listMachineSizesHandler) 51 | api.GET("/machines/pricing", getCurrentMachinePricing) 52 | api.GET("/machines/vpc", listVPCsHandler) 53 | 54 | // Presets. 55 | api.POST("/presets", createPresetHandler) 56 | api.GET("/presets", getPresetsHandler) 57 | api.GET("/presets/:id", getPresetByIDHandler) 58 | api.PUT("/presets/:id", updatePresetByIDHandler) 59 | 60 | // Users. 61 | api.GET("/users", getUsersHandler) 62 | api.PUT("/users/:id", updateUserByIDHandler) 63 | 64 | // Settings. 65 | api.GET("/settings", settingsHandler) 66 | api.PUT("/settings", updateSettingsHandler) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /api/server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "os" 7 | "time" 8 | 9 | "github.com/alfg/openencoder/api/logging" 10 | "github.com/gin-gonic/gin" 11 | "github.com/gocraft/work" 12 | "github.com/gomodule/redigo/redis" 13 | ) 14 | 15 | // Project constants. 16 | const ( 17 | ProjectName = "openencoder" 18 | ProjectGithub = "https://github.com/alfg/openencoder" 19 | ProjectDocs = "https://github.com/alfg/openencoder/blob/master/API.md" 20 | ProjectWiki = "https://github.com/alfg/openencoder/wiki" 21 | OK = "OK" 22 | NOK = "NOK" 23 | 24 | // Machines. 25 | WorkerTag = "openencoder-worker" 26 | 27 | // JWT settings. 28 | JwtRealm = "openencoder" 29 | JwtIdentityKey = "id" 30 | JwtRoleKey = "role" 31 | JwtTimeout = time.Hour // Duration a JWT is valid. 32 | JwtMaxRefresh = time.Hour // Duration a JWT can be refreshed. 33 | 34 | // User role types. 35 | RoleAdmin = "admin" 36 | RoleOperator = "operator" 37 | RoleGuest = "guest" 38 | ) 39 | 40 | var ( 41 | // ProjectVersion gets the current version set by the CI build. 42 | ProjectVersion = os.Getenv("VERSION") 43 | 44 | // Server settings. 45 | redisPool *redis.Pool 46 | enqueuer *work.Enqueuer 47 | log = logging.Log 48 | ) 49 | 50 | // Config defines configuration for creating a NewServer. 51 | type Config struct { 52 | ServerPort string 53 | RedisHost string 54 | RedisPort int 55 | Namespace string 56 | JobName string 57 | Concurrency uint 58 | } 59 | 60 | // NewServer creates a new server 61 | func NewServer(serverCfg Config) { 62 | // Setup redis queue. 63 | redisPool = &redis.Pool{ 64 | MaxActive: 5, 65 | MaxIdle: 5, 66 | Wait: true, 67 | Dial: func() (redis.Conn, error) { 68 | return redis.Dial("tcp", 69 | fmt.Sprintf("%s:%d", serverCfg.RedisHost, serverCfg.RedisPort)) 70 | }, 71 | } 72 | enqueuer = work.NewEnqueuer(serverCfg.Namespace, redisPool) 73 | 74 | // Setup server. 75 | r := gin.New() 76 | r.Use(gin.Logger()) 77 | r.Use(gin.Recovery()) 78 | 79 | // Default redirect to dashboard. 80 | r.GET("/", func(c *gin.Context) { 81 | c.Redirect(http.StatusMovedPermanently, "/dashboard") 82 | }) 83 | 84 | // Web dashboard. 85 | r.Static("/dashboard", "./web/dist") 86 | 87 | // Catch all fallback for HTML5 History Mode. 88 | // https://router.vuejs.org/guide/essentials/history-mode.html 89 | r.NoRoute(func(c *gin.Context) { 90 | c.File("./web/dist/index.html") 91 | }) 92 | 93 | registerRoutes(r) 94 | 95 | log.Info("started server on port: ", serverCfg.ServerPort) 96 | r.Run() 97 | } 98 | -------------------------------------------------------------------------------- /api/server/settings.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/alfg/openencoder/api/data" 7 | "github.com/alfg/openencoder/api/types" 8 | "github.com/gin-gonic/gin" 9 | ) 10 | 11 | type settingsUpdateRequest struct { 12 | StorageDriver string `json:"STORAGE_DRIVER" binding:"eq=s3|eq=ftp"` 13 | 14 | S3AccessKey string `json:"S3_ACCESS_KEY"` 15 | S3SecretKey string `json:"S3_SECRET_KEY"` 16 | S3InboundBucket string `json:"S3_INBOUND_BUCKET"` 17 | S3InboundBucketRegion string `json:"S3_INBOUND_BUCKET_REGION"` 18 | S3OutboundBucket string `json:"S3_OUTBOUND_BUCKET"` 19 | S3OutboundBucketRegion string `json:"S3_OUTBOUND_BUCKET_REGION"` 20 | S3Provider string `json:"S3_PROVIDER" binding:"eq=digitaloceanspaces|eq=amazonaws|eq=custom|eq="` 21 | S3Streaming string `json:"S3_STREAMING" binding:"eq=enabled|eq=disabled"` 22 | S3Endpoint string `json:"S3_ENDPOINT"` 23 | 24 | FTPAddr string `json:"FTP_ADDR"` 25 | FTPUsername string `json:"FTP_USERNAME"` 26 | FTPPassword string `json:"FTP_PASSWORD"` 27 | 28 | DigitalOceanEnabled string `json:"DIGITAL_OCEAN_ENABLED" binding:"eq=enabled|eq=disabled"` 29 | DigitalOceanAccessToken string `json:"DIGITAL_OCEAN_ACCESS_TOKEN"` 30 | DigitalOceanRegion string `json:"DIGITAL_OCEAN_REGION"` 31 | DigitalOceanVPC string `json:"DIGITAL_OCEAN_VPC"` 32 | SlackWebhook string `json:"SLACK_WEBHOOK"` 33 | } 34 | 35 | func settingsHandler(c *gin.Context) { 36 | user, _ := c.Get(JwtIdentityKey) 37 | 38 | // Role check. 39 | if !isAdmin(user) { 40 | c.JSON(http.StatusUnauthorized, gin.H{"message": "unauthorized"}) 41 | return 42 | } 43 | 44 | d := data.New() 45 | settings := d.Settings.GetSettings() 46 | settingOptions := d.Settings.GetSettingsOptions() 47 | 48 | // Get all settings for response and set blank defaults. 49 | var resp []types.SettingsForm 50 | for _, v := range settingOptions { 51 | s := types.SettingsForm{ 52 | Title: v.Title, 53 | Name: v.Name, 54 | Description: v.Description, 55 | Secure: v.Secure, 56 | } 57 | for _, j := range settings { 58 | if j.Name == v.Name { 59 | s.Value = j.Value 60 | } 61 | } 62 | resp = append(resp, s) 63 | } 64 | 65 | c.JSON(http.StatusOK, gin.H{ 66 | "settings": resp, 67 | }) 68 | } 69 | 70 | func updateSettingsHandler(c *gin.Context) { 71 | user, _ := c.Get(JwtIdentityKey) 72 | 73 | // Role check. 74 | if !isAdmin(user) { 75 | c.JSON(http.StatusUnauthorized, gin.H{"message": "unauthorized"}) 76 | return 77 | } 78 | 79 | // Decode json. 80 | var json settingsUpdateRequest 81 | if err := c.ShouldBindJSON(&json); err != nil { 82 | c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) 83 | return 84 | } 85 | 86 | s := map[string]string{ 87 | types.StorageDriver: json.StorageDriver, 88 | 89 | types.S3AccessKey: json.S3AccessKey, 90 | types.S3SecretKey: json.S3SecretKey, 91 | types.S3InboundBucket: json.S3InboundBucket, 92 | types.S3InboundBucketRegion: json.S3InboundBucketRegion, 93 | types.S3OutboundBucket: json.S3OutboundBucket, 94 | types.S3OutboundBucketRegion: json.S3OutboundBucketRegion, 95 | types.S3Provider: json.S3Provider, 96 | types.S3Streaming: json.S3Streaming, 97 | types.S3Endpoint: json.S3Endpoint, 98 | 99 | types.FTPAddr: json.FTPAddr, 100 | types.FTPUsername: json.FTPUsername, 101 | types.FTPPassword: json.FTPPassword, 102 | 103 | types.DigitalOceanEnabled: json.DigitalOceanEnabled, 104 | types.DigitalOceanAccessToken: json.DigitalOceanAccessToken, 105 | types.DigitalOceanRegion: json.DigitalOceanRegion, 106 | types.DigitalOceanVPC: json.DigitalOceanVPC, 107 | types.SlackWebhook: json.SlackWebhook, 108 | } 109 | 110 | db := data.New() 111 | // userID := db.Users.GetUserID(username) 112 | 113 | err := db.Settings.UpdateSettings(s) 114 | if err != nil { 115 | c.JSON(http.StatusBadRequest, gin.H{ 116 | "message": "error updating settings", 117 | }) 118 | } 119 | 120 | c.JSON(http.StatusOK, gin.H{ 121 | "settings": "updated", 122 | }) 123 | } 124 | -------------------------------------------------------------------------------- /api/server/status.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/alfg/openencoder/api/data" 7 | "github.com/gin-gonic/gin" 8 | ) 9 | 10 | func getStatsHandler(c *gin.Context) { 11 | db := data.New() 12 | stats, err := db.Jobs.GetJobsStats() 13 | if err != nil { 14 | c.JSON(http.StatusNotFound, gin.H{ 15 | "status": http.StatusNotFound, 16 | "message": "Job does not exist", 17 | }) 18 | return 19 | } 20 | 21 | c.JSON(http.StatusOK, gin.H{ 22 | "status": http.StatusOK, 23 | "stats": gin.H{ 24 | "jobs": stats, 25 | }, 26 | }) 27 | } 28 | -------------------------------------------------------------------------------- /api/server/storage.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/alfg/openencoder/api/data" 7 | "github.com/alfg/openencoder/api/net" 8 | "github.com/alfg/openencoder/api/types" 9 | "github.com/gin-gonic/gin" 10 | "github.com/jlaffaye/ftp" 11 | ) 12 | 13 | // Storage drivers available. 14 | const ( 15 | StorageS3 = "s3" 16 | StorageFTP = "ftp" 17 | ) 18 | 19 | type storageListResponse struct { 20 | Folders []string `json:"folders"` 21 | Files []file `json:"files"` 22 | } 23 | 24 | type file struct { 25 | Name string `json:"name"` 26 | Size int64 `json:"size"` 27 | } 28 | 29 | type s3ListResponse struct { 30 | Folders []string `json:"folders"` 31 | Files []file `json:"files"` 32 | } 33 | 34 | func storageListHandler(c *gin.Context) { 35 | prefix := c.DefaultQuery("prefix", "") 36 | 37 | db := data.New() 38 | driver, err := db.Settings.GetSetting(types.StorageDriver) 39 | if err != nil { 40 | log.Error(err) 41 | c.JSON(http.StatusUnauthorized, gin.H{ 42 | "status": http.StatusUnauthorized, 43 | "message": "storage not configured", 44 | }) 45 | } 46 | 47 | files := getFileList(driver.Value, prefix) 48 | 49 | c.JSON(200, gin.H{ 50 | "data": files, 51 | }) 52 | } 53 | 54 | func getFileList(driver string, prefix string) *storageListResponse { 55 | resp := &storageListResponse{} 56 | if driver == StorageS3 { 57 | resp, _ = getS3FileList(prefix) 58 | } else if driver == StorageFTP { 59 | resp, _ = getFTPFileList(prefix) 60 | } 61 | return resp 62 | } 63 | 64 | func getS3FileList(prefix string) (*storageListResponse, error) { 65 | db := data.New() 66 | settings := db.Settings.GetSettings() 67 | 68 | config := net.S3Config{ 69 | AccessKey: types.GetSetting(types.S3AccessKey, settings), 70 | SecretKey: types.GetSetting(types.S3SecretKey, settings), 71 | Provider: types.GetSetting(types.S3Provider, settings), 72 | Region: types.GetSetting(types.S3OutboundBucketRegion, settings), 73 | InboundBucket: types.GetSetting(types.S3InboundBucket, settings), 74 | OutboundBucket: types.GetSetting(types.S3OutboundBucket, settings), 75 | } 76 | 77 | s3 := net.NewS3(config) 78 | 79 | resp := &storageListResponse{} 80 | files, err := s3.S3ListFiles(prefix) 81 | if err != nil { 82 | return nil, err 83 | } 84 | 85 | for _, item := range files.CommonPrefixes { 86 | resp.Folders = append(resp.Folders, *item.Prefix) 87 | } 88 | 89 | for _, item := range files.Contents { 90 | var obj file 91 | obj.Name = *item.Key 92 | obj.Size = *item.Size 93 | resp.Files = append(resp.Files, obj) 94 | } 95 | return resp, nil 96 | } 97 | 98 | func getFTPFileList(prefix string) (*storageListResponse, error) { 99 | db := data.New() 100 | settings := db.Settings.GetSettings() 101 | 102 | addr := types.GetSetting(types.FTPAddr, settings) 103 | user := types.GetSetting(types.FTPUsername, settings) 104 | pass := types.GetSetting(types.FTPPassword, settings) 105 | 106 | f := net.NewFTP(addr, user, pass) 107 | files, err := f.ListFiles(prefix) 108 | if err != nil { 109 | return nil, err 110 | } 111 | 112 | resp := &storageListResponse{} 113 | 114 | for _, item := range files { 115 | if item.Type != ftp.EntryTypeFolder { 116 | var obj file 117 | obj.Name = item.Name 118 | obj.Size = int64(item.Size) 119 | resp.Files = append(resp.Files, obj) 120 | } else { 121 | resp.Folders = append(resp.Folders, item.Name+"/") 122 | } 123 | } 124 | return resp, nil 125 | } 126 | -------------------------------------------------------------------------------- /api/server/workers.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "github.com/alfg/openencoder/api/config" 5 | "github.com/gin-gonic/gin" 6 | "github.com/gocraft/work" 7 | ) 8 | 9 | func workerQueueHandler(c *gin.Context) { 10 | client := work.NewClient(config.Get().WorkerNamespace, redisPool) 11 | 12 | queues, err := client.Queues() 13 | if err != nil { 14 | log.Error(err) 15 | } 16 | c.JSON(200, queues) 17 | } 18 | 19 | func workerPoolsHandler(c *gin.Context) { 20 | client := work.NewClient(config.Get().WorkerNamespace, redisPool) 21 | 22 | resp, err := client.WorkerPoolHeartbeats() 23 | if err != nil { 24 | log.Error(err) 25 | } 26 | c.JSON(200, resp) 27 | } 28 | 29 | func workerBusyHandler(c *gin.Context) { 30 | client := work.NewClient(config.Get().WorkerNamespace, redisPool) 31 | 32 | observations, err := client.WorkerObservations() 33 | if err != nil { 34 | log.Error(err) 35 | } 36 | 37 | var busyObservations []*work.WorkerObservation 38 | for _, ob := range observations { 39 | if ob.IsBusy { 40 | busyObservations = append(busyObservations, ob) 41 | } 42 | } 43 | c.JSON(200, busyObservations) 44 | } 45 | -------------------------------------------------------------------------------- /api/types/job.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "database/sql" 5 | "encoding/json" 6 | ) 7 | 8 | // Job status types. 9 | const ( 10 | JobQueued = "queued" 11 | JobDownloading = "downloading" 12 | JobProbing = "probing" 13 | JobEncoding = "encoding" 14 | JobUploading = "uploading" 15 | JobCompleted = "completed" 16 | JobError = "error" 17 | JobCancelled = "cancelled" 18 | JobRestarting = "restarting" 19 | ) 20 | 21 | // JobStatuses All job status types. 22 | var JobStatuses = []string{ 23 | JobQueued, 24 | JobDownloading, 25 | JobProbing, 26 | JobEncoding, 27 | JobUploading, 28 | JobCompleted, 29 | JobError, 30 | JobCancelled, 31 | JobRestarting, 32 | } 33 | 34 | // Job describes the job info. 35 | type Job struct { 36 | ID int64 `db:"id" json:"id"` 37 | GUID string `db:"guid" json:"guid"` 38 | Preset string `db:"preset" json:"preset"` 39 | CreatedDate string `db:"created_date" json:"created_date"` 40 | Status string `db:"status" json:"status"` 41 | Source string `db:"source" json:"source"` 42 | Destination string `db:"destination" json:"destination"` 43 | 44 | // EncodeData. 45 | Encode `db:"encode"` 46 | 47 | LocalSource string `json:"local_source,omitempty"` 48 | LocalDestination string `json:"local_destination,omitempty"` 49 | Streaming bool `json:"streaming"` 50 | } 51 | 52 | // Encode describes the encode data. 53 | type Encode struct { 54 | EncodeID int64 `db:"id" json:"-"` 55 | JobID int64 `db:"job_id" json:"-"` 56 | Probe NullString `db:"probe" json:"probe,omitempty"` 57 | Options NullString `db:"options" json:"options,omitempty"` 58 | Progress NullFloat64 `db:"progress" json:"progress,omitempty"` 59 | Speed NullString `db:"speed" json:"speed"` 60 | FPS NullFloat64 `db:"fps" json:"fps"` 61 | } 62 | 63 | // NullString is an alias for sql.NullString data type 64 | type NullString struct { 65 | sql.NullString 66 | } 67 | 68 | // MarshalJSON for NullString 69 | func (ns *NullString) MarshalJSON() ([]byte, error) { 70 | if !ns.Valid { 71 | return []byte("null"), nil 72 | } 73 | return json.Marshal(ns.String) 74 | } 75 | 76 | // NullInt64 is an alias for sql.NullInt64 data type 77 | type NullInt64 struct { 78 | sql.NullInt64 79 | } 80 | 81 | // MarshalJSON for NullInt64 82 | func (ni *NullInt64) MarshalJSON() ([]byte, error) { 83 | if !ni.Valid { 84 | return []byte("null"), nil 85 | } 86 | return json.Marshal(ni.Int64) 87 | } 88 | 89 | // NullFloat64 is an alias for sql.NullFloat64 data type 90 | type NullFloat64 struct { 91 | sql.NullFloat64 92 | } 93 | 94 | // MarshalJSON for NullFloat64 95 | func (nf *NullFloat64) MarshalJSON() ([]byte, error) { 96 | if !nf.Valid { 97 | return []byte("null"), nil 98 | } 99 | return json.Marshal(nf.Float64) 100 | } 101 | -------------------------------------------------------------------------------- /api/types/preset.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | // Preset contains user models. 4 | type Preset struct { 5 | ID int64 `db:"id" json:"id,omitempty"` 6 | Name string `db:"name" json:"name"` 7 | Description string `db:"description" json:"description"` 8 | Data string `db:"data" json:"data"` 9 | Output string `db:"output" json:"output"` 10 | Active *bool `db:"active" json:"active"` 11 | } 12 | -------------------------------------------------------------------------------- /api/types/setting.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | // Settings types. 4 | const ( 5 | StorageDriver = "STORAGE_DRIVER" 6 | 7 | S3AccessKey = "S3_ACCESS_KEY" 8 | S3SecretKey = "S3_SECRET_KEY" 9 | S3InboundBucket = "S3_INBOUND_BUCKET" 10 | S3InboundBucketRegion = "S3_INBOUND_BUCKET_REGION" 11 | S3OutboundBucket = "S3_OUTBOUND_BUCKET" 12 | S3OutboundBucketRegion = "S3_OUTBOUND_BUCKET_REGION" 13 | S3Provider = "S3_PROVIDER" 14 | S3Endpoint = "S3_ENDPOINT" 15 | S3Streaming = "S3_STREAMING" 16 | 17 | FTPAddr = "FTP_ADDR" 18 | FTPUsername = "FTP_USERNAME" 19 | FTPPassword = "FTP_PASSWORD" 20 | 21 | DigitalOceanEnabled = "DIGITAL_OCEAN_ENABLED" 22 | DigitalOceanAccessToken = "DIGITAL_OCEAN_ACCESS_TOKEN" 23 | DigitalOceanRegion = "DIGITAL_OCEAN_REGION" 24 | DigitalOceanVPC = "DIGITAL_OCEAN_VPC" 25 | SlackWebhook = "SLACK_WEBHOOK" 26 | 27 | DigitalOceanSpaces = "DIGITALOCEANSPACES" 28 | AmazonAWS = "AMAZONAWS" 29 | Custom = "CUSTOM" 30 | ) 31 | 32 | // Setting defines a setting for a user. 33 | type Setting struct { 34 | ID int64 `db:"id" json:"-"` 35 | UserID int64 `db:"user_id" json:"-"` 36 | 37 | SettingsOptionID int64 `db:"settings_option_id" json:"-"` 38 | SettingsOption `db:"settings_option"` 39 | 40 | Value string `db:"value" json:"value"` 41 | Encrypted bool `db:"encrypted" json:"encrypted"` 42 | } 43 | 44 | // SettingsOption defines a setting option. 45 | type SettingsOption struct { 46 | ID int64 `db:"id" json:"-"` 47 | Name string `db:"name" json:"name"` 48 | Title string `db:"title" json:"title"` 49 | Description string `db:"description" json:"description"` 50 | Secure bool `db:"secure" json:"secure"` 51 | } 52 | 53 | // SettingsForm defines the setting form options. 54 | type SettingsForm struct { 55 | Name string `json:"name"` 56 | Title string `json:"title"` 57 | Value string `json:"value"` 58 | Description string `json:"description"` 59 | Secure bool `json:"secure"` 60 | } 61 | 62 | // GetSetting gets a setting value by key from a slice of Setting. 63 | func GetSetting(s string, settings []Setting) string { 64 | for _, v := range settings { 65 | if s == v.Name { 66 | return v.Value 67 | } 68 | } 69 | return "" 70 | } 71 | -------------------------------------------------------------------------------- /api/types/user.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | // User contains user models. 4 | type User struct { 5 | ID int64 `db:"id" json:"id,omitempty"` 6 | Username string `db:"username" json:"username" valid:"required"` 7 | Password string `db:"password" json:"-" valid:"password,required"` 8 | Role string `db:"role" json:"role" valid:"required"` 9 | ForcePasswordReset bool `db:"force_password_reset" json:"-"` 10 | Active bool `db:"active" json:"active"` 11 | } 12 | -------------------------------------------------------------------------------- /api/worker/encode_worker.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/signal" 7 | 8 | "github.com/alfg/openencoder/api/config" 9 | "github.com/alfg/openencoder/api/data" 10 | "github.com/alfg/openencoder/api/logging" 11 | "github.com/alfg/openencoder/api/types" 12 | "github.com/gocraft/work" 13 | "github.com/gomodule/redigo/redis" 14 | ) 15 | 16 | var log = logging.Log 17 | 18 | // Context defines the job context to be passed to the worker. 19 | type Context struct { 20 | GUID string 21 | Preset string 22 | Source string 23 | Destination string 24 | } 25 | 26 | // Log worker middleware for logging job. 27 | func (c *Context) Log(job *work.Job, next work.NextMiddlewareFunc) error { 28 | log.Infof("worker: starting job %s\n", job.Name) 29 | return next() 30 | } 31 | 32 | // FindJob worker middleware for setting job context from job arguments. 33 | func (c *Context) FindJob(job *work.Job, next work.NextMiddlewareFunc) error { 34 | if _, ok := job.Args["guid"]; ok { 35 | c.GUID = job.ArgString("guid") 36 | if err := job.ArgError(); err != nil { 37 | return err 38 | } 39 | } 40 | if _, ok := job.Args["preset"]; ok { 41 | c.Preset = job.ArgString("preset") 42 | if err := job.ArgError(); err != nil { 43 | return err 44 | } 45 | } 46 | if _, ok := job.Args["source"]; ok { 47 | c.Source = job.ArgString("source") 48 | if err := job.ArgError(); err != nil { 49 | return err 50 | } 51 | } 52 | if _, ok := job.Args["destination"]; ok { 53 | c.Destination = job.ArgString("destination") 54 | if err := job.ArgError(); err != nil { 55 | return err 56 | } 57 | } 58 | return next() 59 | } 60 | 61 | // SendJob worker handler for running job. 62 | func (c *Context) SendJob(job *work.Job) error { 63 | guid := job.ArgString("guid") 64 | preset := job.ArgString("preset") 65 | source := job.ArgString("source") 66 | destination := job.ArgString("destination") 67 | 68 | j := types.Job{ 69 | GUID: guid, 70 | Preset: preset, 71 | Source: source, 72 | Destination: destination, 73 | } 74 | 75 | // Check if job is cancelled. 76 | db := data.New() 77 | jobStatus, _ := db.Jobs.GetJobStatusByGUID(guid) 78 | if jobStatus == types.JobCancelled { 79 | return nil 80 | } 81 | 82 | // Start job. 83 | runEncodeJob(j) 84 | log.Infof("worker: completed %s!\n", j.Preset) 85 | return nil 86 | } 87 | 88 | func startJob(id int, j types.Job) { 89 | log.Infof("worker: started %s\n", j.Preset) 90 | 91 | // runWorkflow(j) 92 | log.Infof("worker: completed %s!\n", j.Preset) 93 | } 94 | 95 | // func (c *Context) Export(job *work.Job) error { 96 | // return nil 97 | // } 98 | 99 | // Config defines configuration for creating a NewWorker. 100 | type Config struct { 101 | Host string 102 | Port int 103 | Namespace string 104 | JobName string 105 | Concurrency uint 106 | MaxActive int 107 | MaxIdle int 108 | } 109 | 110 | // NewWorker creates a new worker instance to listen and process jobs in the queue. 111 | func NewWorker(workerCfg Config) { 112 | 113 | // Make a redis pool 114 | redisPool := &redis.Pool{ 115 | MaxActive: workerCfg.MaxActive, 116 | MaxIdle: workerCfg.MaxIdle, 117 | Wait: true, 118 | Dial: func() (redis.Conn, error) { 119 | return redis.Dial("tcp", 120 | fmt.Sprintf("%s:%d", workerCfg.Host, workerCfg.Port)) 121 | }, 122 | } 123 | 124 | // Make a new pool. 125 | pool := work.NewWorkerPool(Context{}, 126 | workerCfg.Concurrency, workerCfg.Namespace, redisPool) 127 | 128 | // Add middleware that will be executed for each job 129 | pool.Middleware((*Context).Log) 130 | pool.Middleware((*Context).FindJob) 131 | 132 | // Map the name of jobs to handler functions 133 | pool.Job(config.Get().WorkerJobName, (*Context).SendJob) 134 | 135 | // Customize options: 136 | // pool.JobWithOptions("export", work.JobOptions{Priority: 10, MaxFails: 1}, (*Context).Export) 137 | 138 | // Start processing jobs 139 | pool.Start() 140 | 141 | // Wait for a signal to quit: 142 | signalChan := make(chan os.Signal, 1) 143 | signal.Notify(signalChan, os.Interrupt, os.Kill) 144 | <-signalChan 145 | 146 | // Stop the pool 147 | pool.Stop() 148 | } 149 | -------------------------------------------------------------------------------- /api/worker/job.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "math" 7 | "os" 8 | "path" 9 | "strconv" 10 | "time" 11 | 12 | "github.com/alfg/openencoder/api/config" 13 | "github.com/alfg/openencoder/api/data" 14 | "github.com/alfg/openencoder/api/encoder" 15 | "github.com/alfg/openencoder/api/helpers" 16 | "github.com/alfg/openencoder/api/net" 17 | "github.com/alfg/openencoder/api/notify" 18 | "github.com/alfg/openencoder/api/types" 19 | ) 20 | 21 | var progressCh chan struct{} 22 | 23 | func generatePresignedURL(job types.Job) (string, error) { 24 | log.Info("generating a presigned URL") 25 | 26 | // Update status. 27 | db := data.New() 28 | db.Jobs.UpdateJobStatusByGUID(job.GUID, types.JobDownloading) 29 | 30 | // Get presigned URL. 31 | str, err := net.GetPresignedURL(job) 32 | if err != nil { 33 | log.Error(err) 34 | } 35 | return str, nil 36 | } 37 | 38 | func download(job types.Job, storageDriver string) error { 39 | log.Info("running download task for: ", storageDriver) 40 | 41 | // Update status. 42 | db := data.New() 43 | db.Jobs.UpdateJobStatusByGUID(job.GUID, types.JobDownloading) 44 | 45 | // Get job data. 46 | j, err := db.Jobs.GetJobByGUID(job.GUID) 47 | if err != nil { 48 | log.Error(err) 49 | return err 50 | } 51 | encodeID := j.EncodeID 52 | 53 | if err := net.Download(job); err != nil { 54 | log.Error(err) 55 | return err 56 | } 57 | 58 | // Set progress to 100. 59 | db.Jobs.UpdateTransferProgressByID(encodeID, 100) 60 | return err 61 | } 62 | 63 | func probe(job types.Job) (*encoder.FFProbeResponse, error) { 64 | log.Info("running probe task") 65 | 66 | // Update status. 67 | db := data.New() 68 | db.Jobs.UpdateJobStatusByGUID(job.GUID, types.JobProbing) 69 | 70 | // Run FFProbe. 71 | f := encoder.FFProbe{} 72 | probeData := f.Run(job.Source) 73 | 74 | // Add probe data to DB. 75 | b, err := json.Marshal(probeData) 76 | if err != nil { 77 | log.Error(err) 78 | } 79 | j, _ := db.Jobs.GetJobByGUID(job.GUID) 80 | db.Jobs.UpdateEncodeProbeByID(j.EncodeID, string(b)) 81 | 82 | return probeData, nil 83 | } 84 | 85 | func encode(job types.Job, probeData *encoder.FFProbeResponse) error { 86 | log.Info("running encode task") 87 | 88 | // Update status. 89 | db := data.New() 90 | db.Jobs.UpdateJobStatusByGUID(job.GUID, types.JobEncoding) 91 | 92 | p, err := db.Presets.GetPresetByName(job.Preset) 93 | if err != nil { 94 | return err 95 | } 96 | dest := path.Dir(job.LocalSource) + "/dst/" + p.Output 97 | 98 | // Get job data. 99 | j, _ := db.Jobs.GetJobByGUID(job.GUID) 100 | 101 | // Update encode options in DB. 102 | db.Jobs.UpdateEncodeOptionsByID(j.EncodeID, p.Data) 103 | 104 | // Run FFmpeg. 105 | f := &encoder.FFmpeg{} 106 | go trackEncodeProgress(j.GUID, j.EncodeID, probeData, f) 107 | err = f.Run(job.Source, dest, p.Data) 108 | if err != nil { 109 | close(progressCh) 110 | return err 111 | } 112 | close(progressCh) 113 | return err 114 | } 115 | 116 | func upload(job types.Job) error { 117 | log.Info("running upload task") 118 | 119 | // Update status. 120 | db := data.New() 121 | db.Jobs.UpdateJobStatusByGUID(job.GUID, types.JobUploading) 122 | 123 | // Get job data. 124 | j, err := db.Jobs.GetJobByGUID(job.GUID) 125 | if err != nil { 126 | log.Error(err) 127 | return err 128 | } 129 | encodeID := j.EncodeID 130 | 131 | if err := net.Upload(job); err != nil { 132 | log.Error(err) 133 | return err 134 | } 135 | 136 | // Set progress to 100. 137 | db.Jobs.UpdateTransferProgressByID(encodeID, 100) 138 | return err 139 | } 140 | 141 | func cleanup(job types.Job) error { 142 | log.Info("running cleanup task") 143 | 144 | tmpPath := helpers.GetTmpPath(config.Get().WorkDirectory, job.GUID) 145 | err := os.RemoveAll(tmpPath) 146 | if err != nil { 147 | return err 148 | } 149 | return nil 150 | } 151 | 152 | func completed(job types.Job) error { 153 | log.Info("job completed") 154 | 155 | // Update status. 156 | db := data.New() 157 | db.Jobs.UpdateJobStatusByGUID(job.GUID, types.JobCompleted) 158 | return nil 159 | } 160 | 161 | func sendAlert(job types.Job) error { 162 | log.Info("sending alert") 163 | 164 | db := data.New() 165 | webhook, err := db.Settings.GetSetting(types.SlackWebhook) 166 | if err != nil { 167 | return err 168 | } 169 | 170 | // Only send Slack alert if configured. 171 | if webhook.Value != "" { 172 | message := fmt.Sprintf(AlertMessageFormat, job.GUID, job.Preset, job.Source, job.Destination) 173 | err := notify.SendSlackMessage(webhook.Value, message) 174 | if err != nil { 175 | return err 176 | } 177 | } 178 | return nil 179 | } 180 | 181 | func runEncodeJob(job types.Job) { 182 | // Set local src path. 183 | job.LocalSource = helpers.CreateLocalSourcePath( 184 | config.Get().WorkDirectory, job.Source, job.GUID) 185 | 186 | db := data.New() 187 | storageDriver, err := db.Settings.GetSetting(types.StorageDriver) 188 | if err != nil { 189 | log.Error(err) 190 | return 191 | } 192 | 193 | // If STREAMING setting is enabled, get a presigned URL and update 194 | // the job.Source. 195 | s3Streaming, err := db.Settings.GetSetting(types.S3Streaming) 196 | if err != nil { 197 | log.Error(err) 198 | return 199 | } 200 | 201 | if s3Streaming.Value == "enabled" && storageDriver.Value == "s3" { 202 | // 1a. Get presigned URL. 203 | presigned, err := generatePresignedURL(job) 204 | if err != nil { 205 | log.Error(err) 206 | db.Jobs.UpdateJobStatusByGUID(job.GUID, types.JobError) 207 | return 208 | } 209 | 210 | // Update source with presigned URL. 211 | job.Source = presigned 212 | 213 | } else { 214 | // 1b. Download. 215 | err := download(job, storageDriver.Value) 216 | if err != nil { 217 | log.Error(err) 218 | db.Jobs.UpdateJobStatusByGUID(job.GUID, types.JobError) 219 | return 220 | } 221 | 222 | job.Source = job.LocalSource 223 | } 224 | 225 | // 2. Probe data. 226 | probeData, err := probe(job) 227 | if err != nil { 228 | log.Error(err) 229 | db.Jobs.UpdateJobStatusByGUID(job.GUID, types.JobError) 230 | return 231 | } 232 | 233 | // 3. Encode. 234 | err = encode(job, probeData) 235 | if err != nil { 236 | log.Error(err) 237 | if err := cleanup(job); err != nil { 238 | log.Error("cleanup err", err) 239 | } 240 | 241 | // Set job to 'cancelled' if it was cancelled. 242 | if err.Error() == types.JobCancelled { 243 | db.Jobs.UpdateJobStatusByGUID(job.GUID, types.JobCancelled) 244 | return 245 | } 246 | db.Jobs.UpdateJobStatusByGUID(job.GUID, types.JobError) 247 | return 248 | } 249 | 250 | // 4. Upload. 251 | err = upload(job) 252 | if err != nil { 253 | log.Error(err) 254 | db.Jobs.UpdateJobStatusByGUID(job.GUID, types.JobError) 255 | return 256 | } 257 | 258 | // 5. Cleanup. 259 | err = cleanup(job) 260 | if err != nil { 261 | log.Error(err) 262 | db.Jobs.UpdateJobStatusByGUID(job.GUID, types.JobError) 263 | return 264 | } 265 | 266 | // 6. Done 267 | completed(job) 268 | if err != nil { 269 | log.Error(err) 270 | } 271 | 272 | // 7. Alert 273 | sendAlert(job) 274 | if err != nil { 275 | log.Error(err) 276 | } 277 | } 278 | 279 | func trackEncodeProgress(guid string, encodeID int64, p *encoder.FFProbeResponse, f *encoder.FFmpeg) { 280 | db := data.New() 281 | progressCh = make(chan struct{}) 282 | ticker := time.NewTicker(ProgressInterval) 283 | 284 | for { 285 | select { 286 | case <-progressCh: 287 | ticker.Stop() 288 | return 289 | case <-ticker.C: 290 | currentFrame := f.Progress.Frame 291 | totalFrames, _ := strconv.Atoi(p.Streams[0].NbFrames) 292 | speed := f.Progress.Speed 293 | fps := f.Progress.FPS 294 | 295 | // Check cancel. 296 | status, _ := db.Jobs.GetJobStatusByGUID(guid) 297 | if status == types.JobCancelled { 298 | f.Cancel() 299 | } 300 | 301 | // Only track progress if we know the total frames. 302 | if totalFrames != 0 { 303 | pct := (float64(currentFrame) / float64(totalFrames)) * 100 304 | 305 | // Update DB with progress. 306 | pct = math.Round(pct*100) / 100 307 | log.Infof("progress: %d / %d - %0.2f%%", currentFrame, totalFrames, pct) 308 | db.Jobs.UpdateEncodeProgressByID(encodeID, pct, speed, fps) 309 | } 310 | } 311 | } 312 | } 313 | -------------------------------------------------------------------------------- /api/worker/worker.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import "time" 4 | 5 | // Worker constants. 6 | const ( 7 | ProgressInterval = time.Second * 5 8 | ) 9 | 10 | // Worker variables. 11 | var ( 12 | AlertMessageFormat = ` 13 | *Encode Successful!* :tada:\n 14 | "*Job ID*: %s:\n" 15 | "*Preset*: %s\n" 16 | "*Source*: %s\n" 17 | "*Destination*: %s\n\n" 18 | ` 19 | ) 20 | -------------------------------------------------------------------------------- /cmd/root.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/alfg/openencoder/api/config" 8 | "github.com/spf13/cobra" 9 | ) 10 | 11 | var cfgFile string 12 | 13 | var rootCmd = &cobra.Command{ 14 | Use: "openencoder", 15 | Short: "Open Source Cloud Encoder.", 16 | Run: func(cmd *cobra.Command, args []string) { 17 | // Do Stuff Here 18 | }, 19 | } 20 | 21 | func init() { 22 | rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "default", "Config YAML") 23 | 24 | config.LoadConfig(cfgFile) 25 | } 26 | 27 | // Execute starts cmd. 28 | func Execute() { 29 | if err := rootCmd.Execute(); err != nil { 30 | fmt.Println(err) 31 | os.Exit(1) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /cmd/server.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | 7 | "github.com/alfg/openencoder/api/config" 8 | "github.com/alfg/openencoder/api/server" 9 | "github.com/spf13/cobra" 10 | ) 11 | 12 | func init() { 13 | rootCmd.AddCommand(serverCmd) 14 | } 15 | 16 | var serverCmd = &cobra.Command{ 17 | Use: "server", 18 | Short: "Start the server.", 19 | Run: func(cmd *cobra.Command, args []string) { 20 | fmt.Println("Starting server...") 21 | startServer() 22 | }, 23 | } 24 | 25 | func configRuntime() { 26 | numCPU := runtime.NumCPU() 27 | runtime.GOMAXPROCS(numCPU) 28 | fmt.Printf("Running with %d CPUs\n", numCPU) 29 | } 30 | func startServer() { 31 | 32 | // Server config. 33 | serverCfg := &server.Config{ 34 | ServerPort: config.Get().Port, 35 | RedisHost: config.Get().RedisHost, 36 | RedisPort: config.Get().RedisPort, 37 | Namespace: config.Get().WorkerNamespace, 38 | JobName: config.Get().WorkerJobName, 39 | Concurrency: config.Get().WorkerConcurrency, 40 | } 41 | 42 | // Create HTTP Server. 43 | configRuntime() 44 | server.NewServer(*serverCfg) 45 | } 46 | -------------------------------------------------------------------------------- /cmd/version.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | func init() { 11 | rootCmd.AddCommand(versionCmd) 12 | } 13 | 14 | var versionCmd = &cobra.Command{ 15 | Use: "version", 16 | Short: " Print the version.", 17 | Run: func(cmd *cobra.Command, args []string) { 18 | fmt.Println(os.Getenv("VERSION")) 19 | }, 20 | } 21 | -------------------------------------------------------------------------------- /cmd/worker.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/alfg/openencoder/api/config" 7 | "github.com/alfg/openencoder/api/worker" 8 | "github.com/spf13/cobra" 9 | ) 10 | 11 | func init() { 12 | rootCmd.AddCommand(workerCmd) 13 | } 14 | 15 | var workerCmd = &cobra.Command{ 16 | Use: "worker", 17 | Short: "Start the worker.", 18 | Run: func(cmd *cobra.Command, args []string) { 19 | fmt.Println("Starting worker...") 20 | startWorkers() 21 | }, 22 | } 23 | 24 | func startWorkers() { 25 | 26 | // Worker config. 27 | workerCfg := &worker.Config{ 28 | Host: config.Get().RedisHost, 29 | Port: config.Get().RedisPort, 30 | Namespace: config.Get().WorkerNamespace, 31 | JobName: config.Get().WorkerJobName, 32 | Concurrency: config.Get().WorkerConcurrency, 33 | MaxActive: config.Get().RedisMaxActive, 34 | MaxIdle: config.Get().RedisMaxIdle, 35 | } 36 | 37 | // Create Workers. 38 | worker.NewWorker(*workerCfg) 39 | } 40 | -------------------------------------------------------------------------------- /config/default.yml: -------------------------------------------------------------------------------- 1 | server_port: 8080 2 | jwt_key: secretkey 3 | keyseed: deadbeefdeadbeefdeadbeefdeadbeef 4 | redis_host: localhost 5 | redis_port: 6379 6 | redis_max_active: 5 7 | redis_max_idle: 5 8 | database_host: localhost 9 | database_port: 5432 10 | database_user: postgres 11 | database_password: postgres 12 | database_name: openencoder 13 | worker_namespace: openencoder 14 | worker_job_name: encode 15 | worker_concurrency: 1 16 | work_dir: /tmp 17 | 18 | cloudinit_redis_host: dev.openencode.com 19 | cloudinit_redis_port: 6379 20 | cloudinit_database_host: dev.openencode.com 21 | cloudinit_database_port: 5432 22 | cloudinit_database_user: postgres 23 | cloudinit_database_password: postgres 24 | cloudinit_database_name: openencoder 25 | cloudinit_worker_image: alfg/openencoder:latest -------------------------------------------------------------------------------- /docker-compose-letsencrypt.yml: -------------------------------------------------------------------------------- 1 | services: 2 | nginx-proxy: 3 | image: jwilder/nginx-proxy 4 | ports: 5 | - "80:80" 6 | - "443:443" 7 | volumes: 8 | - conf:/etc/nginx/conf.d 9 | - vhost:/etc/nginx/vhost.d 10 | - html:/usr/share/nginx/html 11 | - dhparam:/etc/nginx/dhparam 12 | - certs:/etc/nginx/certs:ro 13 | - /var/run/docker.sock:/tmp/docker.sock:ro 14 | environment: 15 | - DHPARAM_GENERATION=false 16 | 17 | nginx-proxy-letsencrypt: 18 | image: jrcs/letsencrypt-nginx-proxy-companion 19 | volumes_from: 20 | - nginx-proxy 21 | volumes: 22 | - certs:/etc/nginx/certs:rw 23 | - /var/run/docker.sock:/var/run/docker.sock:ro 24 | 25 | openencoder-web: 26 | image: alfg/openencoder:latest 27 | environment: 28 | - GIN_MODE=release 29 | - DATABASE_HOST=db 30 | - REDIS_HOST=redis 31 | - CLOUDINIT_REDIS_HOST=priv.net.ip.addr 32 | - CLOUDINIT_DATABASE_HOST=priv.net.ip.addr 33 | - VIRTUAL_HOST=dev.openencode.com 34 | - LETSENCRYPT_HOST=dev.openencode.com 35 | links: 36 | - redis 37 | - db 38 | ports: 39 | - "8080:8080" 40 | entrypoint: ["/app", "server"] 41 | 42 | redis: 43 | image: "redis:alpine" 44 | ports: 45 | - priv.net.ip.addr:6379:6379 46 | volumes: 47 | - /data 48 | 49 | db: 50 | image: postgres 51 | ports: 52 | - priv.net.ip.addr:5432:5432 53 | environment: 54 | POSTGRES_PASSWORD: 'postgres' 55 | volumes: 56 | - /var/lib/postgresql/data 57 | 58 | volumes: 59 | conf: 60 | vhost: 61 | html: 62 | dhparam: 63 | certs: -------------------------------------------------------------------------------- /docker-compose-production.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | 3 | services: 4 | nginx-proxy: 5 | image: jwilder/nginx-proxy 6 | ports: 7 | - "80:80" 8 | volumes: 9 | - /var/run/docker.sock:/tmp/docker.sock:ro 10 | - ./htpasswd:/etc/nginx/htpasswd 11 | 12 | openencoder-web: 13 | image: alfg/openencoder:latest 14 | environment: 15 | - GIN_MODE=release 16 | - DATABASE_HOST=db 17 | - REDIS_HOST=redis 18 | - VIRTUAL_HOST=dev.openencode.com 19 | - CLOUDINIT_REDIS_HOST=priv.net.ip.addr 20 | - CLOUDINIT_DATABASE_HOST=priv.net.ip.addr 21 | - CLOUDINIT_WORKER_IMAGE=alfg/openencoder:latest 22 | links: 23 | - redis 24 | - db 25 | ports: 26 | - "8080:8080" 27 | entrypoint: ["/app", "server"] 28 | 29 | redis: 30 | image: "redis:alpine" 31 | ports: 32 | - priv.net.ip.addr:6379:6379 33 | volumes: 34 | - /data 35 | 36 | db: 37 | image: postgres 38 | ports: 39 | - priv.net.ip.addr:5432:5432 40 | environment: 41 | POSTGRES_PASSWORD: 'postgres' 42 | volumes: 43 | - /var/lib/postgresql/data -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | server: 4 | build: . 5 | environment: 6 | - GIN_MODE=release 7 | - DATABASE_HOST=db 8 | - REDIS_HOST=redis 9 | links: 10 | - redis 11 | - db 12 | ports: 13 | - "8080:8080" 14 | entrypoint: ["/app", "server"] 15 | 16 | worker: 17 | build: . 18 | environment: 19 | - DATABASE_HOST=db 20 | - REDIS_HOST=redis 21 | links: 22 | - redis 23 | - db 24 | entrypoint: ["/app", "worker"] 25 | 26 | redis: 27 | image: "redis:alpine" 28 | ports: 29 | - "6379:6379" 30 | 31 | db: 32 | image: postgres 33 | ports: 34 | - 5432:5432 35 | environment: 36 | POSTGRES_PASSWORD: 'postgres' 37 | POSTGRES_DB: 'openencoder' 38 | volumes: 39 | - /var/lib/postgresql/data 40 | - ./scripts:/docker-entrypoint-initdb.d 41 | 42 | # ftpd: 43 | # image: stilliard/pure-ftpd 44 | # container_name: pure-ftpd 45 | # ports: 46 | # - "21:21" 47 | # - "30000-30009:30000-30009" 48 | # volumes: 49 | # - "./data/data:/home/username/" 50 | # - "./data/passwd:/etc/pure-ftpd/passwd" 51 | # environment: 52 | # PUBLICHOST: "localhost" 53 | # FTP_USER_NAME: username 54 | # FTP_USER_PASS: mypass 55 | # FTP_USER_HOME: /home/username -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/alfg/openencoder 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/appleboy/gin-jwt/v2 v2.6.2 7 | github.com/aws/aws-sdk-go v1.20.15 8 | github.com/digitalocean/godo v1.42.0 9 | github.com/gin-gonic/gin v1.6.3 10 | github.com/gocraft/work v0.5.1 11 | github.com/gomodule/redigo v2.0.0+incompatible 12 | github.com/jlaffaye/ftp v0.0.0-20200720194710-13949d38913e 13 | github.com/jmoiron/sqlx v1.2.0 14 | github.com/lib/pq v1.1.1 15 | github.com/robfig/cron v1.2.0 // indirect 16 | github.com/rs/xid v1.2.1 17 | github.com/sirupsen/logrus v1.2.0 18 | github.com/spf13/cobra v0.0.5 19 | github.com/spf13/viper v1.4.0 20 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 21 | golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d 22 | ) 23 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/alfg/openencoder/cmd" 4 | 5 | func main() { 6 | cmd.Execute() 7 | } 8 | -------------------------------------------------------------------------------- /screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/screenshot.png -------------------------------------------------------------------------------- /scripts/00_setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL 5 | CREATE USER docker; 6 | GRANT ALL PRIVILEGES ON DATABASE openencoder TO docker; 7 | EOSQL -------------------------------------------------------------------------------- /scripts/10_schema.sql: -------------------------------------------------------------------------------- 1 | -- auto-generated definition 2 | create table jobs 3 | ( 4 | id serial not null, 5 | guid varchar(128) not null 6 | constraint jobs_pk 7 | primary key, 8 | preset varchar(128) not null, 9 | created_date timestamp default CURRENT_TIMESTAMP, 10 | status varchar(64), 11 | source varchar(128), 12 | destination varchar(128) 13 | ); 14 | 15 | alter table jobs 16 | owner to postgres; 17 | 18 | create unique index jobs_id_uindex 19 | on jobs (id); 20 | 21 | create unique index jobs_guid_uindex 22 | on jobs (guid); 23 | 24 | create index jobs_status_index 25 | on jobs (status); 26 | 27 | -- auto-generated definition 28 | create table encode 29 | ( 30 | id serial not null 31 | constraint encode_pkey 32 | primary key, 33 | probe json, 34 | progress double precision default 0, 35 | job_id integer 36 | constraint encode_jobs_id_fk 37 | references jobs (id), 38 | speed varchar(64), 39 | fps double precision default 0, 40 | options json 41 | ); 42 | 43 | alter table encode 44 | owner to postgres; 45 | 46 | create unique index encode_id_uindex 47 | on encode (id); 48 | 49 | 50 | -- auto-generated definition 51 | create table users 52 | ( 53 | id serial not null, 54 | username varchar(128) not null 55 | constraint user_pkey 56 | primary key, 57 | password varchar(128) not null, 58 | role varchar(64), 59 | force_password_reset boolean default false, 60 | active boolean default true 61 | ); 62 | 63 | alter table users 64 | owner to postgres; 65 | 66 | create unique index user_id_uindex 67 | on users (id); 68 | 69 | create unique index user_username_uindex 70 | on users (username); 71 | 72 | -- auto-generated definition 73 | create table settings_option 74 | ( 75 | id serial not null, 76 | name varchar(64), 77 | description varchar(1024), 78 | title varchar(64), 79 | secure boolean default false 80 | ); 81 | 82 | alter table settings_option 83 | owner to postgres; 84 | 85 | create unique index settings_option_id_uindex 86 | on settings_option (id); 87 | 88 | 89 | -- auto-generated definition 90 | create table settings 91 | ( 92 | settings_option_id integer not null 93 | constraint settings_settings_option_id_fk 94 | references settings_option (id), 95 | value varchar(256), 96 | id serial not null 97 | constraint settings_pk 98 | primary key, 99 | encrypted boolean default false 100 | ); 101 | 102 | alter table settings 103 | owner to postgres; 104 | 105 | create unique index settings_id_uindex 106 | on settings (id); 107 | 108 | 109 | -- auto-generated definition 110 | create table presets 111 | ( 112 | id serial not null 113 | constraint presets_pk 114 | primary key, 115 | name varchar(128), 116 | description varchar, 117 | data varchar, 118 | active boolean default false, 119 | output varchar(128) 120 | ); 121 | 122 | alter table presets 123 | owner to postgres; 124 | 125 | create unique index presets_id_uindex 126 | on presets (id); 127 | 128 | -------------------------------------------------------------------------------- /scripts/20_settings_options.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO public.settings_option (id, name, description, title, secure) VALUES (1, 'S3_ACCESS_KEY', 'S3 Access Key', 'S3 Access Key', true); 2 | INSERT INTO public.settings_option (id, name, description, title, secure) VALUES (2, 'S3_SECRET_KEY', 'S3 Secret Key', 'S3 Secret Key', true); 3 | INSERT INTO public.settings_option (id, name, description, title, secure) VALUES (3, 'DIGITAL_OCEAN_ACCESS_TOKEN', 'Digital Ocean Access Token (Required for Machines)', 'Digital Ocean Access Token', true); 4 | INSERT INTO public.settings_option (id, name, description, title, secure) VALUES (4, 'SLACK_WEBHOOK', 'Slack Webhook for notifications', 'Slack Webhook', true); 5 | INSERT INTO public.settings_option (id, name, description, title, secure) VALUES (5, 'S3_INBOUND_BUCKET', 'S3 Inbound Bucket', 'S3 Inbound Bucket', false); 6 | INSERT INTO public.settings_option (id, name, description, title, secure) VALUES (6, 'S3_OUTBOUND_BUCKET', 'S3 Outbound Bucket', 'S3 Outbound Bucket', false); 7 | INSERT INTO public.settings_option (id, name, description, title, secure) VALUES (7, 'S3_OUTBOUND_BUCKET_REGION', 'S3 Outbound Bucket Region', 'S3 Outbound Bucket Region', false); 8 | INSERT INTO public.settings_option (id, name, description, title, secure) VALUES (8, 'S3_INBOUND_BUCKET_REGION', 'S3 Inbound Bucket Region', 'S3 Inbound Bucket Region', false); 9 | INSERT INTO public.settings_option (id, name, description, title, secure) VALUES (9, 'S3_PROVIDER', 'S3 Provider', 'S3 Provider', false); 10 | INSERT INTO public.settings_option (id, name, description, title, secure) VALUES (10, 'S3_STREAMING', 'Enable this setting to enable streaming directly to FFmpeg from a pre-signed S3 URL, instead of downloading the file first if disk space is a concern. Please note this setting can impact performance.', 'Stream Encode from S3', false); 11 | INSERT INTO public.settings_option (id, name, description, title, secure) VALUES (11, 'FTP_ADDR', 'FTP Connection', 'FTP Address', false); 12 | INSERT INTO public.settings_option (id, name, description, title, secure) VALUES (12, 'FTP_USERNAME', 'FTP Username', 'FTP Username', true); 13 | INSERT INTO public.settings_option (id, name, description, title, secure) VALUES (13, 'FTP_PASSWORD', 'FTP Password', 'FTP Password', true); 14 | INSERT INTO public.settings_option (id, name, description, title, secure) VALUES (14, 'STORAGE_DRIVER', 'Storage Driver for input and output', 'Storage Driver', false); 15 | INSERT INTO public.settings_option (id, name, description, title, secure) VALUES (15, 'S3_ENDPOINT', 'Provide a custom endpoint if using a service provider that uses the S3 protocol.', 'S3 Custom Endpoint', false); 16 | INSERT INTO public.settings_option (id, name, description, title, secure) VALUES (16, 'DIGITAL_OCEAN_REGION', 'Digital Ocean Machines Region (Required for Machines)', 'Digital Ocean Region', false); 17 | INSERT INTO public.settings_option (id, name, description, title, secure) VALUES (17, 'DIGITAL_OCEAN_ENABLED', 'Enable Digital Ocean Machines', 'Digital Ocean Machines', false); 18 | INSERT INTO public.settings_option (id, name, description, title, secure) VALUES (18, 'DIGITAL_OCEAN_VPC', 'Enable Digital Ocean Machines VPC', 'Digital Ocean Machines VPC', false); 19 | 20 | SELECT setval('settings_option_id_seq', max(id)) FROM settings_option; -------------------------------------------------------------------------------- /scripts/21_settings_defaults.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO public.settings (id, value, settings_option_id, encrypted) VALUES (1, 'digitaloceanspaces', 9, false); 2 | INSERT INTO public.settings (id, value, settings_option_id, encrypted) VALUES (2, 'sfo2', 8, false); 3 | INSERT INTO public.settings (id, value, settings_option_id, encrypted) VALUES (3, 'outbound', 6, false); 4 | INSERT INTO public.settings (id, value, settings_option_id, encrypted) VALUES (4, 'inbound', 5, false); 5 | INSERT INTO public.settings (id, value, settings_option_id, encrypted) VALUES (5, 'sfo2', 7, false); 6 | INSERT INTO public.settings (id, value, settings_option_id, encrypted) VALUES (6, 'disabled', 10, false); 7 | INSERT INTO public.settings (id, value, settings_option_id, encrypted) VALUES (7, 's3', 14, false); 8 | 9 | SELECT setval('settings_id_seq', max(id)) FROM settings; -------------------------------------------------------------------------------- /scripts/30_presets.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO public.presets (id, name, description, data, active, output) VALUES (5, 'webm_vp9_720p_3000', 'webm_vp9_720p_3000', '{"raw":["-vf scale=-2:720","-c:v libvpx-vp9","-level:v 4.0","-b:v 3000k","-pix_fmt yuv420p","-f webm","-y"]}', true, 'webm_vp9_3000_720.webm'); 2 | INSERT INTO public.presets (id, name, description, data, active, output) VALUES (4, 'h264_main_1080p_6000', 'h264_main_1080p_6000', '{"raw":["-vf scale=-2:1080","-c:v libx264","-profile:v main","-level:v 4.2","-x264opts scenecut=0:open_gop=0:min-keyint=72:keyint=72","-minrate 6000k","-maxrate 6000k","-bufsize 6000k","-b:v 6000k","-y"]}', true, 'h264_main_1080p_6000.mp4'); 3 | INSERT INTO public.presets (id, name, description, data, active, output) VALUES (3, 'h264_main_720p_3000', 'h264_main_720p_3000', '{"raw":["-vf scale=-2:720","-c:v libx264","-profile:v main","-level:v 4.0","-x264opts scenecut=0:open_gop=0:min-keyint=72:keyint=72","-minrate 3000k","-maxrate 3000k","-bufsize 3000k","-b:v 3000k","-y"]}', true, 'h264_main_720p_3000.mp4'); 4 | INSERT INTO public.presets (id, name, description, data, active, output) VALUES (2, 'h264_main_480p_1000', 'h264_main_480p_1000', '{"raw":["-vf scale=-2:480","-c:v libx264","-profile:v main","-level:v 3.1","-x264opts scenecut=0:open_gop=0:min-keyint=72:keyint=72","-minrate 1000k","-maxrate 1000k","-bufsize 1000k","-b:v 1000k","-y"]}', true, 'h264_main_480p_1000.mp4'); 5 | INSERT INTO public.presets (id, name, description, data, active, output) VALUES (1, 'h264_baseline_360p_600', 'h264_baseline_360p_600', '{"raw":["-vf scale=-2:360","-c:v libx264","-profile:v baseline","-level:v 3.0","-x264opts scenecut=0:open_gop=0:min-keyint=72:keyint=72","-minrate 600k","-maxrate 600k","-bufsize 600k","-b:v 600k","-y"]}', true, 'h264_baseline_360p_600.mp4'); 6 | 7 | SELECT setval('presets_id_seq', max(id)) FROM presets; -------------------------------------------------------------------------------- /scripts/40_users.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO public.users (id, username, password, role, force_password_reset, active) VALUES (1, 'admin', '$2a$04$FxaVhOgeUazmjfhe4eGrgeFx/Dm3nyw0/so4k.pPSsVDj.7lZmJDW', 'admin', true, true); 2 | 3 | SELECT setval('users_id_seq', max(id)) FROM users; -------------------------------------------------------------------------------- /scripts/50_jobs.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO public.jobs (id, guid, preset, created_date, status, source, destination) VALUES (1, 'bu95hhb5bidi4l552rhg', 'h264_baseline_360p_600', '2020-10-23 04:15:02.008245', 'completed', 'hello-world.mp4', 'hello-world/'); 2 | INSERT INTO public.encode (id, probe, progress, job_id, speed, fps, options) VALUES (1, '{"streams":[{"index":0,"codec_name":"h264","codec_long_name":"H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10","profile":"High","codec_type":"video","codec_time_base":"1/48","codec_tag_string":"avc1","codec_tag":"0x31637661","width":1920,"height":800,"coded_width":1920,"coded_height":800,"has_b_frames":2,"sample_aspect_ratio":"1:1","display_aspect_ratio":"12:5","pix_fmt":"yuv420p","level":40,"chroma_location":"left","refs":1,"is_avc":"true","nal_length_size":"4","r_frame_rate":"24/1","avg_frame_rate":"24/1","time_base":"1/12288","start_pts":0,"start_time":"0.000000","duration_ts":26112,"duration":"2.125000","bit_rate":"9155858","bits_per_raw_sample":"8","nb_frames":"59","disposition":{"default":1,"dub":0,"original":0,"comment":0,"lyrics":0,"karaoke":0,"forced":0,"hearing_impaired":0,"visual_empaired":0,"clean_effects":0,"attached_pic":0,"timed_thumbnails":0},"tags":{"language":"eng","handler_name":"VideoHandler"}},{"index":1,"codec_name":"aac","codec_long_name":"AAC (Advanced Audio Coding)","profile":"LC","codec_type":"audio","codec_time_base":"1/44100","codec_tag_string":"mp4a","codec_tag":"0x6134706d","width":0,"height":0,"coded_width":0,"coded_height":0,"has_b_frames":0,"sample_aspect_ratio":"","display_aspect_ratio":"","pix_fmt":"","level":0,"chroma_location":"","refs":0,"is_avc":"","nal_length_size":"","r_frame_rate":"0/0","avg_frame_rate":"0/0","time_base":"1/44100","start_pts":0,"start_time":"0.000000","duration_ts":88421,"duration":"2.005011","bit_rate":"189437","bits_per_raw_sample":"","nb_frames":"107","disposition":{"default":1,"dub":0,"original":0,"comment":0,"lyrics":0,"karaoke":0,"forced":0,"hearing_impaired":0,"visual_empaired":0,"clean_effects":0,"attached_pic":0,"timed_thumbnails":0},"tags":{"language":"eng","handler_name":"SoundHandler"}}]}', 100, 1, null, 0, '{"raw":["-vf scale=-2:360","-c:v libx264","-profile:v baseline","-level:v 3.0","-x264opts scenecut=0:open_gop=0:min-keyint=72:keyint=72","-minrate 600k","-maxrate 600k","-bufsize 600k","-b:v 600k","-y"]}'); 3 | 4 | SELECT setval('jobs_id_seq', max(id)) FROM jobs; 5 | SELECT setval('encode_id_seq', max(id)) FROM encode; -------------------------------------------------------------------------------- /web/.browserslistrc: -------------------------------------------------------------------------------- 1 | > 1% 2 | last 2 versions 3 | -------------------------------------------------------------------------------- /web/.editorconfig: -------------------------------------------------------------------------------- 1 | [*.{js,jsx,ts,tsx,vue}] 2 | indent_style = space 3 | indent_size = 2 4 | end_of_line = lf 5 | trim_trailing_whitespace = true 6 | insert_final_newline = true 7 | max_line_length = 100 8 | -------------------------------------------------------------------------------- /web/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | env: { 4 | node: true, 5 | }, 6 | extends: [ 7 | 'plugin:vue/essential', 8 | '@vue/airbnb', 9 | ], 10 | rules: { 11 | 'no-console': process.env.NODE_ENV === 'production' ? 'warn' : 'off', 12 | 'no-debugger': process.env.NODE_ENV === 'production' ? 'error' : 'off', 13 | 'import/no-unresolved': 0, 14 | }, 15 | parserOptions: { 16 | parser: 'babel-eslint', 17 | }, 18 | }; 19 | -------------------------------------------------------------------------------- /web/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules 3 | /dist 4 | 5 | # local env files 6 | .env.local 7 | .env.*.local 8 | 9 | # Log files 10 | npm-debug.log* 11 | yarn-debug.log* 12 | yarn-error.log* 13 | 14 | # Editor directories and files 15 | .idea 16 | .vscode 17 | *.suo 18 | *.ntvs* 19 | *.njsproj 20 | *.sln 21 | *.sw? 22 | -------------------------------------------------------------------------------- /web/README.md: -------------------------------------------------------------------------------- 1 | # static 2 | 3 | ## Project setup 4 | ``` 5 | npm install 6 | ``` 7 | 8 | ### Compiles and hot-reloads for development 9 | ``` 10 | npm run serve 11 | ``` 12 | 13 | ### Compiles and minifies for production 14 | ``` 15 | npm run build 16 | ``` 17 | 18 | ### Run your tests 19 | ``` 20 | npm run test 21 | ``` 22 | 23 | ### Lints and fixes files 24 | ``` 25 | npm run lint 26 | ``` 27 | 28 | ### Customize configuration 29 | See [Configuration Reference](https://cli.vuejs.org/config/). 30 | -------------------------------------------------------------------------------- /web/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [ 3 | '@vue/cli-plugin-babel/preset', 4 | ], 5 | }; 6 | -------------------------------------------------------------------------------- /web/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "static", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "serve": "vue-cli-service serve", 7 | "build": "vue-cli-service build", 8 | "lint": "vue-cli-service lint" 9 | }, 10 | "dependencies": { 11 | "bootstrap": "^4.3.1", 12 | "bootstrap-vue": "^2.1.0", 13 | "core-js": "^3.4.3", 14 | "js-cookie": "^2.2.1", 15 | "jsoneditor": "^7.2.1", 16 | "jwt-decode": "^2.2.0", 17 | "register-service-worker": "^1.6.2", 18 | "vue": "^2.6.10", 19 | "vue-moment": "^4.1.0", 20 | "vue-resource": "^1.5.1", 21 | "vue-router": "^3.1.2" 22 | }, 23 | "devDependencies": { 24 | "@vue/cli-plugin-babel": "^4.1.1", 25 | "@vue/cli-plugin-eslint": "^4.1.1", 26 | "@vue/cli-plugin-pwa": "^4.1.1", 27 | "@vue/cli-service": "^4.4.1", 28 | "@vue/eslint-config-airbnb": "^4.0.1", 29 | "babel-eslint": "^10.0.3", 30 | "eslint": "^6.2.2", 31 | "eslint-plugin-vue": "^5.2.3", 32 | "vue-template-compiler": "^2.6.10" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /web/postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | autoprefixer: {}, 4 | }, 5 | }; 6 | -------------------------------------------------------------------------------- /web/public/img/icons/android-icon-144x144.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/icons/android-icon-144x144.png -------------------------------------------------------------------------------- /web/public/img/icons/android-icon-192x192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/icons/android-icon-192x192.png -------------------------------------------------------------------------------- /web/public/img/icons/android-icon-36x36.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/icons/android-icon-36x36.png -------------------------------------------------------------------------------- /web/public/img/icons/android-icon-48x48.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/icons/android-icon-48x48.png -------------------------------------------------------------------------------- /web/public/img/icons/android-icon-72x72.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/icons/android-icon-72x72.png -------------------------------------------------------------------------------- /web/public/img/icons/android-icon-96x96.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/icons/android-icon-96x96.png -------------------------------------------------------------------------------- /web/public/img/icons/apple-icon-114x114.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/icons/apple-icon-114x114.png -------------------------------------------------------------------------------- /web/public/img/icons/apple-icon-120x120.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/icons/apple-icon-120x120.png -------------------------------------------------------------------------------- /web/public/img/icons/apple-icon-144x144.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/icons/apple-icon-144x144.png -------------------------------------------------------------------------------- /web/public/img/icons/apple-icon-152x152.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/icons/apple-icon-152x152.png -------------------------------------------------------------------------------- /web/public/img/icons/apple-icon-180x180.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/icons/apple-icon-180x180.png -------------------------------------------------------------------------------- /web/public/img/icons/apple-icon-57x57.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/icons/apple-icon-57x57.png -------------------------------------------------------------------------------- /web/public/img/icons/apple-icon-60x60.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/icons/apple-icon-60x60.png -------------------------------------------------------------------------------- /web/public/img/icons/apple-icon-72x72.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/icons/apple-icon-72x72.png -------------------------------------------------------------------------------- /web/public/img/icons/apple-icon-76x76.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/icons/apple-icon-76x76.png -------------------------------------------------------------------------------- /web/public/img/icons/apple-icon-precomposed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/icons/apple-icon-precomposed.png -------------------------------------------------------------------------------- /web/public/img/icons/apple-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/icons/apple-icon.png -------------------------------------------------------------------------------- /web/public/img/icons/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/icons/favicon-16x16.png -------------------------------------------------------------------------------- /web/public/img/icons/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/icons/favicon-32x32.png -------------------------------------------------------------------------------- /web/public/img/icons/favicon-96x96.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/icons/favicon-96x96.png -------------------------------------------------------------------------------- /web/public/img/icons/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/icons/favicon.ico -------------------------------------------------------------------------------- /web/public/img/icons/ms-icon-144x144.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/icons/ms-icon-144x144.png -------------------------------------------------------------------------------- /web/public/img/icons/ms-icon-150x150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/icons/ms-icon-150x150.png -------------------------------------------------------------------------------- /web/public/img/icons/ms-icon-310x310.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/icons/ms-icon-310x310.png -------------------------------------------------------------------------------- /web/public/img/icons/ms-icon-70x70.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/icons/ms-icon-70x70.png -------------------------------------------------------------------------------- /web/public/img/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/public/img/logo.png -------------------------------------------------------------------------------- /web/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Open Encoder - Dashboard 9 | 10 | 11 | 14 |
15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /web/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "static", 3 | "short_name": "static", 4 | "icons": [ 5 | { 6 | "src": "./img/icons/favicon-120x120.png", 7 | "sizes": "120x120", 8 | "type": "image/png" 9 | } 10 | ], 11 | "start_url": "./index.html", 12 | "display": "standalone", 13 | "background_color": "#000000", 14 | "theme_color": "#4DBA87" 15 | } 16 | -------------------------------------------------------------------------------- /web/public/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Disallow: 3 | -------------------------------------------------------------------------------- /web/src/App.vue: -------------------------------------------------------------------------------- 1 | 48 | 49 | 94 | 95 | 132 | -------------------------------------------------------------------------------- /web/src/api.js: -------------------------------------------------------------------------------- 1 | import auth from './auth'; 2 | 3 | const root = '/api'; 4 | 5 | const Endpoints = { 6 | Version: `${root}/`, 7 | Stats: `${root}/stats`, 8 | Health: `${root}/health`, 9 | Pricing: `${root}/machines/pricing`, 10 | 11 | JobsList: page => `${root}/jobs?page=${page}`, 12 | Jobs: `${root}/jobs`, 13 | JobsCancel: id => `${root}/jobs/${id}/cancel`, 14 | JobsRestart: id => `${root}/jobs/${id}/restart`, 15 | 16 | Machines: `${root}/machines`, 17 | MachinesRegions: `${root}/machines/regions`, 18 | MachinesSizes: `${root}/machines/sizes`, 19 | MachinesVPCs: `${root}/machines/vpc`, 20 | MachinesId: id => `${root}/machines/${id}`, 21 | 22 | PresetsList: page => `${root}/presets?page=${page}`, 23 | Presets: `${root}/presets`, 24 | PresetsId: id => `${root}/presets/${id}`, 25 | 26 | FileList: prefix => `${root}/storage/list?prefix=${prefix}`, 27 | 28 | WorkerQueue: `${root}/worker/queue`, 29 | WorkerPools: `${root}/worker/pools`, 30 | 31 | Users: `${root}/users`, 32 | UsersId: id => `${root}/users/${id}`, 33 | 34 | Settings: `${root}/settings`, 35 | 36 | CurrentUser: `${root}/me`, 37 | }; 38 | 39 | function get(context, url, callback) { 40 | context.$http.get(url, { 41 | headers: auth.getAuthHeader(), 42 | }) 43 | .then(response => ( 44 | response.json() 45 | )) 46 | .then((json) => { 47 | callback(null, json); 48 | }) 49 | .catch((err) => { 50 | callback(err); 51 | }); 52 | } 53 | 54 | function post(context, url, data, callback) { 55 | context.$http.post(url, data, { 56 | headers: auth.getAuthHeader(), 57 | }) 58 | .then(response => ( 59 | response.json() 60 | )) 61 | .then((json) => { 62 | callback(null, json); 63 | }) 64 | .catch((err) => { 65 | callback(err); 66 | }); 67 | } 68 | 69 | function update(context, url, data, callback) { 70 | context.$http.put(url, data, { 71 | headers: auth.getAuthHeader(), 72 | }) 73 | .then(response => ( 74 | response.json() 75 | )) 76 | .then((json) => { 77 | callback(null, json); 78 | }) 79 | .catch((err) => { 80 | callback(err); 81 | }); 82 | } 83 | 84 | function del(context, url, callback) { 85 | context.$http.delete(url, { 86 | headers: auth.getAuthHeader(), 87 | }) 88 | .then(response => ( 89 | response.json() 90 | )) 91 | .then((json) => { 92 | callback(null, json); 93 | }) 94 | .catch((err) => { 95 | callback(err); 96 | }); 97 | } 98 | 99 | export default { 100 | getVersion(context, callback) { 101 | return get(context, Endpoints.Version, callback); 102 | }, 103 | 104 | getStats(context, callback) { 105 | return get(context, Endpoints.Stats, callback); 106 | }, 107 | 108 | getHealth(context, callback) { 109 | return get(context, Endpoints.Health, callback); 110 | }, 111 | 112 | getPricing(context, callback) { 113 | return get(context, Endpoints.Pricing, callback); 114 | }, 115 | 116 | getJobs(context, page, callback) { 117 | return get(context, Endpoints.JobsList(page), callback); 118 | }, 119 | 120 | createJob(context, data, callback) { 121 | return post(context, Endpoints.Jobs, data, callback); 122 | }, 123 | 124 | cancelJob(context, id, callback) { 125 | return post(context, Endpoints.JobsCancel(id), {}, callback); 126 | }, 127 | 128 | restartJob(context, id, callback) { 129 | return post(context, Endpoints.JobsRestart(id), {}, callback); 130 | }, 131 | 132 | getMachines(context, callback) { 133 | return get(context, Endpoints.Machines, callback); 134 | }, 135 | 136 | getMachineRegions(context, callback) { 137 | return get(context, Endpoints.MachinesRegions, callback); 138 | }, 139 | 140 | getMachineSizes(context, callback) { 141 | return get(context, Endpoints.MachinesSizes, callback); 142 | }, 143 | 144 | getMachineVPCs(context, callback) { 145 | return get(context, Endpoints.MachinesVPCs, callback); 146 | }, 147 | 148 | createMachine(context, data, callback) { 149 | return post(context, Endpoints.Machines, data, callback); 150 | }, 151 | 152 | deleteMachine(context, id, callback) { 153 | return del(context, Endpoints.MachinesId(id), callback); 154 | }, 155 | 156 | deleteAllMachines(context, callback) { 157 | return del(context, Endpoints.Machines, callback); 158 | }, 159 | 160 | getPresets(context, page, callback) { 161 | return get(context, Endpoints.PresetsList(page), callback); 162 | }, 163 | 164 | createPreset(context, data, callback) { 165 | return post(context, Endpoints.Presets, data, callback); 166 | }, 167 | 168 | updatePreset(context, data, callback) { 169 | return update(context, Endpoints.PresetsId(data.id), data, callback); 170 | }, 171 | 172 | getFileList(context, prefix, callback) { 173 | return get(context, Endpoints.FileList(prefix), callback); 174 | }, 175 | 176 | getWorkerQueue(context, callback) { 177 | return get(context, Endpoints.WorkerQueue, callback); 178 | }, 179 | 180 | getWorkerPools(context, callback) { 181 | return get(context, Endpoints.WorkerPools, callback); 182 | }, 183 | 184 | getUsers(context, callback) { 185 | return get(context, Endpoints.Users, callback); 186 | }, 187 | 188 | updateUser(context, data, callback) { 189 | return update(context, Endpoints.UsersId(data.id), data, callback); 190 | }, 191 | 192 | getSettings(context, callback) { 193 | return get(context, Endpoints.Settings, callback); 194 | }, 195 | 196 | updateSettings(context, data, callback) { 197 | return update(context, Endpoints.Settings, data, callback); 198 | }, 199 | 200 | getCurrentUser(context, callback) { 201 | return get(context, Endpoints.CurrentUser, callback); 202 | }, 203 | 204 | updateCurrentUser(context, data, callback) { 205 | return update(context, Endpoints.CurrentUser, data, callback); 206 | }, 207 | }; 208 | -------------------------------------------------------------------------------- /web/src/assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alfg/openencoder/7bf61495953b5e19dd64b6dfff05bce2b9b49cb1/web/src/assets/logo.png -------------------------------------------------------------------------------- /web/src/auth.js: -------------------------------------------------------------------------------- 1 | import jwtDecode from 'jwt-decode'; 2 | import cookie from './cookie'; 3 | import store from './store'; 4 | 5 | const LOGIN_URL = '/api/login'; 6 | const REGISTER_URL = '/api/register'; 7 | const UPDATE_PASSWORD_URL = '/api/update-password'; 8 | 9 | export default { 10 | 11 | user: { 12 | username: null, 13 | role: null, 14 | authenticated: false, 15 | }, 16 | 17 | login(context, creds, redirect, callback) { 18 | context.$http.post(LOGIN_URL, creds).then((data) => { 19 | cookie.set('token', data.body.token); 20 | store.setTokenAction(data.body.token); 21 | 22 | this.user.authenticated = true; 23 | this.user.username = jwtDecode(data.body.token).id; 24 | 25 | if (redirect) { 26 | context.$router.push({ name: redirect }); 27 | context.$router.go(); 28 | } 29 | }, (err) => { 30 | // If password needs to be updated. 31 | if (err && err.body.message === 'require password reset') { 32 | context.$router.push({ name: 'update-password' }); 33 | context.$router.go(); 34 | } 35 | callback(err); 36 | }); 37 | }, 38 | 39 | register(context, creds, redirect, callback) { 40 | context.$http.post(REGISTER_URL, creds).then(() => { 41 | // TODO: Authenticate after registration? 42 | // cookie.set('token', data.body.token); 43 | // store.setTokenAction(data.body.token); 44 | 45 | // this.user.authenticated = true; 46 | // this.user.username = jwtDecode(data.body.token).id; 47 | 48 | if (redirect) { 49 | context.$router.push({ name: redirect }); 50 | } 51 | }, (err) => { 52 | callback(err); 53 | }); 54 | }, 55 | 56 | updatePassword(context, creds, redirect, callback) { 57 | context.$http.post(UPDATE_PASSWORD_URL, creds).then(() => { 58 | if (redirect) { 59 | context.$router.push({ name: redirect }); 60 | } 61 | }, (err) => { 62 | callback(err); 63 | }); 64 | }, 65 | 66 | 67 | logout(context) { 68 | cookie.remove('token'); 69 | this.user.authenticated = false; 70 | context.$router.push({ name: 'login' }); 71 | context.$router.go(); 72 | }, 73 | 74 | checkAuth(context) { 75 | const jwt = cookie.get('token'); 76 | 77 | if (context.$route.name === 'register' || context.$route.name === 'update-password') { 78 | return; 79 | } 80 | 81 | // Check if token exists from cookie and set the store. 82 | // If not, then redirect to the login page to get a new token. 83 | if (jwt && !this.isExpired(jwt)) { 84 | store.setTokenAction(jwt); 85 | this.user.authenticated = true; 86 | this.user.username = jwtDecode(jwt).id; 87 | this.user.role = jwtDecode(jwt).role; 88 | } else if (context.$route.name !== 'login') { 89 | context.$router.push({ name: 'login' }); 90 | } 91 | }, 92 | 93 | isExpired(jwt) { 94 | return Date.now() >= jwtDecode(jwt).exp * 1000; 95 | }, 96 | 97 | getAuthHeader() { 98 | return { 99 | Authorization: `Bearer ${store.state.token}`, 100 | }; 101 | }, 102 | }; 103 | -------------------------------------------------------------------------------- /web/src/components/FileBrowser.vue: -------------------------------------------------------------------------------- 1 | 15 | 16 | 85 | -------------------------------------------------------------------------------- /web/src/components/JobForm.vue: -------------------------------------------------------------------------------- 1 | 50 | 51 | 135 | -------------------------------------------------------------------------------- /web/src/components/JobsTable.vue: -------------------------------------------------------------------------------- 1 | 121 | 122 | 211 | 212 | 225 | -------------------------------------------------------------------------------- /web/src/components/LoginForm.vue: -------------------------------------------------------------------------------- 1 | 41 | 42 | 78 | -------------------------------------------------------------------------------- /web/src/components/MachinesForm.vue: -------------------------------------------------------------------------------- 1 | 61 | 62 | 162 | 163 | 172 | -------------------------------------------------------------------------------- /web/src/components/MachinesTable.vue: -------------------------------------------------------------------------------- 1 | 20 | 21 | 64 | -------------------------------------------------------------------------------- /web/src/components/PresetForm.vue: -------------------------------------------------------------------------------- 1 | 60 | 61 | 188 | 189 | 194 | -------------------------------------------------------------------------------- /web/src/components/PresetsTable.vue: -------------------------------------------------------------------------------- 1 | 84 | 85 | 182 | -------------------------------------------------------------------------------- /web/src/components/QueueTable.vue: -------------------------------------------------------------------------------- 1 | 7 | 8 | 31 | -------------------------------------------------------------------------------- /web/src/components/RegisterForm.vue: -------------------------------------------------------------------------------- 1 | 51 | 52 | 87 | -------------------------------------------------------------------------------- /web/src/components/Status.vue: -------------------------------------------------------------------------------- 1 | 69 | 70 | 151 | -------------------------------------------------------------------------------- /web/src/components/UpdatePasswordForm.vue: -------------------------------------------------------------------------------- 1 | 60 | 61 | 96 | -------------------------------------------------------------------------------- /web/src/components/UserProfile.vue: -------------------------------------------------------------------------------- 1 | 74 | 75 | 156 | -------------------------------------------------------------------------------- /web/src/components/UsersTable.vue: -------------------------------------------------------------------------------- 1 | 67 | 68 | 145 | -------------------------------------------------------------------------------- /web/src/components/WorkersTable.vue: -------------------------------------------------------------------------------- 1 | 7 | 8 | 31 | -------------------------------------------------------------------------------- /web/src/cookie.js: -------------------------------------------------------------------------------- 1 | import Cookies from 'js-cookie'; 2 | 3 | export default { 4 | set(key, value) { 5 | const options = { 6 | expires: 7, 7 | }; 8 | Cookies.set(key, value, options); 9 | }, 10 | 11 | get(key) { 12 | return Cookies.get(key); 13 | }, 14 | 15 | remove(key) { 16 | Cookies.remove(key); 17 | }, 18 | }; 19 | -------------------------------------------------------------------------------- /web/src/main.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue'; 2 | import App from './App.vue'; 3 | import router from './router'; 4 | import './registerServiceWorker'; 5 | 6 | Vue.config.productionTip = false; 7 | 8 | new Vue({ 9 | router, 10 | render: h => h(App), 11 | }).$mount('#app'); 12 | -------------------------------------------------------------------------------- /web/src/registerServiceWorker.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | 3 | import { register } from 'register-service-worker'; 4 | 5 | if (process.env.NODE_ENV === 'production') { 6 | register(`${process.env.BASE_URL}service-worker.js`, { 7 | ready() { 8 | console.log( 9 | 'App is being served from cache by a service worker.\n' 10 | + 'For more details, visit https://goo.gl/AFskqB', 11 | ); 12 | }, 13 | registered() { 14 | console.log('Service worker has been registered.'); 15 | }, 16 | cached() { 17 | console.log('Content has been cached for offline use.'); 18 | }, 19 | updatefound() { 20 | console.log('New content is downloading.'); 21 | }, 22 | updated() { 23 | console.log('New content is available; please refresh.'); 24 | }, 25 | offline() { 26 | console.log('No internet connection found. App is running in offline mode.'); 27 | }, 28 | error(error) { 29 | console.error('Error during service worker registration:', error); 30 | }, 31 | }); 32 | } 33 | -------------------------------------------------------------------------------- /web/src/router.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue'; 2 | import Router from 'vue-router'; 3 | import VueResource from 'vue-resource'; 4 | import Moment from 'vue-moment'; 5 | import BootstrapVue from 'bootstrap-vue'; 6 | import Login from './views/Login.vue'; 7 | import Register from './views/Register.vue'; 8 | import UpdatePassword from './views/UpdatePassword.vue'; 9 | 10 | import cookie from './cookie'; 11 | 12 | import 'bootstrap/dist/css/bootstrap.css'; 13 | import 'bootstrap-vue/dist/bootstrap-vue.css'; 14 | 15 | Vue.use(Router); 16 | Vue.use(BootstrapVue); 17 | Vue.use(VueResource); 18 | Vue.use(Moment); 19 | 20 | Vue.http.headers.common.Authorization = `Bearer ${cookie.get('token')}`; 21 | 22 | export default new Router({ 23 | mode: 'history', 24 | base: process.env.BASE_URL, 25 | routes: [ 26 | { 27 | path: '/', 28 | redirect: '/status', 29 | }, 30 | { 31 | path: '/status', 32 | name: 'home', 33 | component: () => import(/* webpackChunkName: "status" */ './views/Status.vue'), 34 | }, 35 | { 36 | path: '/jobs', 37 | name: 'jobs', 38 | component: () => import(/* webpackChunkName: "jobs" */ './views/Jobs.vue'), 39 | }, 40 | { 41 | path: '/encode', 42 | name: 'encode', 43 | component: () => import(/* webpackChunkName: "encode" */ './views/Encode.vue'), 44 | }, 45 | { 46 | path: '/queue', 47 | name: 'queue', 48 | component: () => import(/* webpackChunkName: "queue" */ './views/Queue.vue'), 49 | }, 50 | { 51 | path: '/workers', 52 | name: 'workers', 53 | component: () => import(/* webpackChunkName: "workers" */ './views/Workers.vue'), 54 | }, 55 | { 56 | path: '/machines', 57 | name: 'machines', 58 | component: () => import(/* webpackChunkName: "machines" */ './views/Machines.vue'), 59 | }, 60 | { 61 | path: '/presets', 62 | name: 'presets', 63 | component: () => import(/* webpackChunkName: "presets" */ './views/Presets.vue'), 64 | }, 65 | { 66 | path: '/presets/create', 67 | name: 'presets-create', 68 | component: () => import(/* webpackChunkName: "presets" */ './views/PresetsCreate.vue'), 69 | }, 70 | { 71 | path: '/users', 72 | name: 'users', 73 | component: () => import(/* webpackChunkName: "users" */ './views/Users.vue'), 74 | }, 75 | { 76 | path: '/settings', 77 | name: 'settings', 78 | component: () => import(/* webpackChunkName: "settings" */ './views/Settings.vue'), 79 | }, 80 | { 81 | path: '/profile', 82 | name: 'profile', 83 | component: () => import(/* webpackChunkName: "profile" */ './views/UserProfile.vue'), 84 | }, 85 | { 86 | path: '/login', 87 | name: 'login', 88 | component: Login, 89 | meta: { hideNavigation: true }, 90 | }, 91 | { 92 | path: '/register', 93 | name: 'register', 94 | component: Register, 95 | meta: { hideNavigation: true }, 96 | }, 97 | { 98 | path: '/update-password', 99 | name: 'update-password', 100 | component: UpdatePassword, 101 | meta: { hideNavigation: true }, 102 | }, 103 | ], 104 | }); 105 | -------------------------------------------------------------------------------- /web/src/store.js: -------------------------------------------------------------------------------- 1 | export default { 2 | state: { 3 | token: null, 4 | }, 5 | 6 | setTokenAction(newValue) { 7 | this.state.token = newValue; 8 | }, 9 | }; 10 | -------------------------------------------------------------------------------- /web/src/views/Encode.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 17 | -------------------------------------------------------------------------------- /web/src/views/Jobs.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 17 | -------------------------------------------------------------------------------- /web/src/views/Login.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 17 | -------------------------------------------------------------------------------- /web/src/views/Machines.vue: -------------------------------------------------------------------------------- 1 | 7 | 8 | 20 | -------------------------------------------------------------------------------- /web/src/views/Presets.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 17 | -------------------------------------------------------------------------------- /web/src/views/PresetsCreate.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 17 | -------------------------------------------------------------------------------- /web/src/views/Queue.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 17 | -------------------------------------------------------------------------------- /web/src/views/Register.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 17 | -------------------------------------------------------------------------------- /web/src/views/Settings.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 17 | -------------------------------------------------------------------------------- /web/src/views/Status.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 17 | -------------------------------------------------------------------------------- /web/src/views/UpdatePassword.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 17 | -------------------------------------------------------------------------------- /web/src/views/UserProfile.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 17 | -------------------------------------------------------------------------------- /web/src/views/Users.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 17 | -------------------------------------------------------------------------------- /web/src/views/Workers.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 17 | -------------------------------------------------------------------------------- /web/vue.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | publicPath: '/dashboard/', 3 | devServer: { 4 | port: 8081, 5 | proxy: { 6 | '/api': { 7 | target: 'http://localhost:8080', 8 | ws: true, 9 | changeOrigin: true, 10 | }, 11 | }, 12 | }, 13 | }; 14 | --------------------------------------------------------------------------------