├── .gitignore ├── README.md ├── api ├── Dockerfile-streaming ├── Dockerfile-transcoder ├── Dockerfile-video-backend ├── backend │ └── backend_api_server.go ├── common │ ├── database │ │ ├── operation.go │ │ └── orm.go │ └── entity │ │ ├── ffmpeg.go │ │ ├── queue.go │ │ └── video.go ├── streaming │ └── streaming_api_server.go └── transcode │ ├── transcode_api_server.go │ └── transcoder.go ├── build_docker_images.sh ├── build_helm_packages.sh ├── build_kubernetes_resources.sh ├── database └── helm │ └── database-config.yml ├── kubernetes └── minikube │ ├── queue-consumer-job.yml │ ├── secrets │ └── redis-queue-info.yml │ ├── streaming-api-deployment.yml │ ├── streaming-api-service.yml │ ├── transcoder-api-deployment.yml │ ├── transcoder-api-service.yml │ ├── video-api-deployment.yml │ ├── video-api-service.yml │ ├── video-upload-minikube-persistent-volume-claim.yml │ └── video-upload-minikube-persistent-volume.yml └── task_queue ├── client ├── Dockerfile-consumer ├── consumer.go └── task.go └── helm └── queue-storage-config.yml /.gitignore: -------------------------------------------------------------------------------- 1 | # Golang 2 | # ---------------------------------------- 3 | 4 | # Binaries for programs and plugins 5 | *.exe 6 | *.dll 7 | *.so 8 | *.dylib 9 | 10 | # Test binary, build with `go test -c` 11 | *.test 12 | 13 | # Output of the go coverage tool, specifically when used with LiteIDE 14 | *.out 15 | 16 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 17 | .glide/ 18 | 19 | # ---------------------------------------- 20 | 21 | # JS 22 | # ---------------------------------------- 23 | 24 | # Logs 25 | logs 26 | *.log* 27 | 28 | # Runtime data 29 | pids 30 | *.pid 31 | *.seed 32 | 33 | # Directory for instrumented libs generated by jscoverage/JSCover 34 | lib-cov 35 | 36 | # Coverage directory used by tools like istanbul 37 | coverage 38 | 39 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 40 | .grunt 41 | 42 | # node-waf configuration 43 | .lock-wscript 44 | 45 | # Compiled binary addons (http://nodejs.org/api/addons.html) 46 | build/Release 47 | 48 | # ---------------------------------------- 49 | 50 | # Dependency directory 51 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git 52 | node_modules 53 | # ---------------------------------------- 54 | 55 | #dist folder 56 | dist 57 | 58 | # IDEA/Webstorm project files 59 | .idea 60 | *.iml 61 | 62 | #VSCode metadata 63 | .vscode 64 | 65 | # Mac files 66 | .DS_Store 67 | 68 | # dotenv files 69 | .*.env 70 | 71 | # Web UI (TBD) 72 | web_client 73 | 74 | # ---------------------------------------- 75 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Video-transcode-queue 2 | A sample infrastructure for processing video upload & transcoding. 3 | * React web UI 4 | * REST API in Golang 5 | * PostgreSQL 6 | * Nginx proxy 7 | * Redis task queue storage 8 | * Redis task consumer in Golang 9 | * Redis task producer in Golang 10 | * Video transcoder in Golang 11 | 12 | ## Powered by Kubernetes Helm packages 13 | 14 | ### How to run locally 15 | 1. Install Docker, Kubernetes, Minikube, and helm package manager 16 | 2. Install ffmpeg && codecs (If you want to troubleshoot ffmpeg commands outside of docker) 17 | * `brew install ffmpeg --with-chromaprint --with-fdk-aac --with-fontconfig --with-freetype --with-frei0r --with-game-music-emu --with-libass --with-libbluray --with-libbs2b --with-libcaca --with-libebur128 --with-libgsm --with-libmodplug --with-libsoxr --with-libssh --with-libvidstab --with-libvorbis --with-libvpx --with-opencore-amr --with-openh264 --with-openjpeg --with-openssl --with-opus --with-rtmpdump --with-rubberband --with-schroedinger --with-sdl2 --with-snappy --with-speex --with-tesseract --with-theora --with-tools --with-two-lame --with-wavpack --with-webp --with-x265 --with-xz --with-zeromq --with-zimg` 18 | 2. Expose FFMPEG C libraries (If you want to troubleshoot ffmpeg commands outside of docker) 19 | * `export FFMPEG_ROOT=export FFMPEG_ROOT=/usr/local/Cellar/ffmpeg/3.3.1 20 | export CGO_LDFLAGS="-L$FFMPEG_ROOT/lib/ -lavcodec -lavformat -lavutil -lswscale -lswresample -lavdevice -lavfilter" 21 | export CGO_CFLAGS="-I$FFMPEG_ROOT/include" 22 | export LD_LIBRARY_PATH=$HOME/ffmpeg/lib` 23 | 2. `helm init && helm repo update` 24 | 3. Install helm packages 25 | * `bash build_helm_packages.sh` 26 | 4. Build docker images 27 | - `eval $(minikube docker-env)` 28 | - `bash build_docker_images.sh` 29 | 5. Run Kubernetes resources 30 | - `bash build_kubernetes_resources.sh` 31 | 6. Run minikube and kubectl proxy 32 | * `minikube start` 33 | * `kubectl proxy` 34 | 7. Access minikube external url 35 | * `minikube service video-api --url` or `minikube service streaming-api --url` 36 | -------------------------------------------------------------------------------- /api/Dockerfile-streaming: -------------------------------------------------------------------------------- 1 | FROM golang:1.8.3-alpine 2 | 3 | ENV GOBIN=/go/bin 4 | 5 | RUN apk update && apk upgrade && \ 6 | apk add --no-cache git openssh 7 | 8 | RUN go get -u github.com/satori/go.uuid 9 | RUN go get -u github.com/joho/godotenv 10 | RUN go get -u go.uber.org/zap 11 | RUN go get -u github.com/gin-gonic/gin 12 | RUN go get -u github.com/gin-contrib/static 13 | RUN go get -u github.com/gin-gonic/autotls 14 | RUN go get -u github.com/gin-contrib/cors 15 | RUN go get -u github.com/jinzhu/gorm 16 | RUN go get -u github.com/jinzhu/gorm/dialects/postgres 17 | RUN go get -u gopkg.in/redis.v3 18 | RUN go get -u github.com/adjust/rmq 19 | 20 | ADD common /go/src/github.com/n1207n/video-transcode-queue/api/common 21 | ADD streaming /go/src/github.com/n1207n/video-transcode-queue/api/streaming 22 | 23 | WORKDIR /go/src/github.com/n1207n/video-transcode-queue/api/streaming 24 | 25 | RUN go build 26 | RUN go install 27 | 28 | ENTRYPOINT /go/bin/streaming 29 | 30 | EXPOSE 8880 31 | -------------------------------------------------------------------------------- /api/Dockerfile-transcoder: -------------------------------------------------------------------------------- 1 | FROM golang:1.8.3-alpine 2 | 3 | # Building ffmpeg 3.3 alpine binaries 4 | # --------------------------------- 5 | # ffmpeg - http://ffmpeg.org/download.html 6 | # 7 | # https://hub.docker.com/r/jrottenberg/ffmpeg/ 8 | # 9 | # 10 | # FROM alpine:3.4 11 | # MAINTAINER Julien Rottenberg 12 | 13 | WORKDIR /tmp/workdir 14 | 15 | ENV FFMPEG_VERSION=3.3.1 \ 16 | FDKAAC_VERSION=0.1.5 \ 17 | LAME_VERSION=3.99.5 \ 18 | OGG_VERSION=1.3.2 \ 19 | OPENCOREAMR_VERSION=0.1.4 \ 20 | OPUS_VERSION=1.1.4 \ 21 | THEORA_VERSION=1.1.1 \ 22 | VORBIS_VERSION=1.3.5 \ 23 | VPX_VERSION=1.6.1 \ 24 | X264_VERSION=20170226-2245-stable \ 25 | X265_VERSION=2.3 \ 26 | XVID_VERSION=1.3.4 \ 27 | PKG_CONFIG_PATH=/usr/local/lib/pkgconfig \ 28 | SRC=/usr/local 29 | 30 | ARG OGG_SHA256SUM="e19ee34711d7af328cb26287f4137e70630e7261b17cbe3cd41011d73a654692 libogg-1.3.2.tar.gz" 31 | ARG OPUS_SHA256SUM="9122b6b380081dd2665189f97bfd777f04f92dc3ab6698eea1dbb27ad59d8692 opus-1.1.4.tar.gz" 32 | ARG VORBIS_SHA256SUM="6efbcecdd3e5dfbf090341b485da9d176eb250d893e3eb378c428a2db38301ce libvorbis-1.3.5.tar.gz" 33 | ARG THEORA_SHA256SUM="40952956c47811928d1e7922cda3bc1f427eb75680c3c37249c91e949054916b libtheora-1.1.1.tar.gz" 34 | ARG XVID_SHA256SUM="4e9fd62728885855bc5007fe1be58df42e5e274497591fec37249e1052ae316f xvidcore-1.3.4.tar.gz" 35 | ARG FFMPEG_KEY="D67658D8" 36 | 37 | RUN buildDeps="autoconf \ 38 | automake \ 39 | bash \ 40 | binutils \ 41 | bzip2 \ 42 | cmake \ 43 | curl \ 44 | coreutils \ 45 | g++ \ 46 | gcc \ 47 | gnupg \ 48 | libtool \ 49 | make \ 50 | python \ 51 | openssl-dev \ 52 | tar \ 53 | yasm \ 54 | zlib-dev" && \ 55 | export MAKEFLAGS="-j$(($(grep -c ^processor /proc/cpuinfo) + 1))" && \ 56 | apk add --update ${buildDeps} libgcc libstdc++ ca-certificates 57 | 58 | RUN \ 59 | ## opencore-amr https://sourceforge.net/projects/opencore-amr/ 60 | DIR=$(mktemp -d) && cd ${DIR} && \ 61 | curl -sL https://downloads.sf.net/project/opencore-amr/opencore-amr/opencore-amr-${OPENCOREAMR_VERSION}.tar.gz | \ 62 | tar -zx --strip-components=1 && \ 63 | ./configure --prefix="${SRC}" --bindir="${SRC}/bin" --enable-shared --datadir=${DIR} && \ 64 | make && \ 65 | make install && \ 66 | rm -rf ${DIR} 67 | RUN \ 68 | ## x264 http://www.videolan.org/developers/x264.html 69 | DIR=$(mktemp -d) && cd ${DIR} && \ 70 | curl -sL https://ftp.videolan.org/pub/videolan/x264/snapshots/x264-snapshot-${X264_VERSION}.tar.bz2 | \ 71 | tar -jx --strip-components=1 && \ 72 | ./configure --prefix="${SRC}" --bindir="${SRC}/bin" --enable-pic --enable-shared --disable-cli && \ 73 | make && \ 74 | make install && \ 75 | rm -rf ${DIR} 76 | RUN \ 77 | ## x265 http://x265.org/ 78 | DIR=$(mktemp -d) && cd ${DIR} && \ 79 | curl -sL https://download.videolan.org/pub/videolan/x265/x265_${X265_VERSION}.tar.gz | \ 80 | tar -zx && \ 81 | cd x265_${X265_VERSION}/build/linux && \ 82 | ./multilib.sh && \ 83 | make -C 8bit install && \ 84 | rm -rf ${DIR} 85 | RUN \ 86 | ## libogg https://www.xiph.org/ogg/ 87 | DIR=$(mktemp -d) && cd ${DIR} && \ 88 | curl -sLO http://downloads.xiph.org/releases/ogg/libogg-${OGG_VERSION}.tar.gz && \ 89 | echo ${OGG_SHA256SUM} | sha256sum --check && \ 90 | tar -zx --strip-components=1 -f libogg-${OGG_VERSION}.tar.gz && \ 91 | ./configure --prefix="${SRC}" --bindir="${SRC}/bin" --disable-static --datarootdir=${DIR} && \ 92 | make && \ 93 | make install && \ 94 | rm -rf ${DIR} 95 | RUN \ 96 | ## libopus https://www.opus-codec.org/ 97 | DIR=$(mktemp -d) && cd ${DIR} && \ 98 | curl -sLO http://downloads.xiph.org/releases/opus/opus-${OPUS_VERSION}.tar.gz && \ 99 | echo ${OPUS_SHA256SUM} | sha256sum --check && \ 100 | tar -zx --strip-components=1 -f opus-${OPUS_VERSION}.tar.gz && \ 101 | autoreconf -fiv && \ 102 | ./configure --prefix="${SRC}" --disable-static --datadir="${DIR}" && \ 103 | make && \ 104 | make install && \ 105 | rm -rf ${DIR} 106 | RUN \ 107 | ## libvorbis https://xiph.org/vorbis/ 108 | DIR=$(mktemp -d) && cd ${DIR} && \ 109 | curl -sLO http://downloads.xiph.org/releases/vorbis/libvorbis-${VORBIS_VERSION}.tar.gz && \ 110 | echo ${VORBIS_SHA256SUM} | sha256sum --check && \ 111 | tar -zx --strip-components=1 -f libvorbis-${VORBIS_VERSION}.tar.gz && \ 112 | ./configure --prefix="${SRC}" --with-ogg="${SRC}" --bindir="${SRC}/bin" \ 113 | --disable-static --datadir="${DIR}" && \ 114 | make && \ 115 | make install && \ 116 | rm -rf ${DIR} 117 | RUN \ 118 | ## libtheora http://www.theora.org/ 119 | DIR=$(mktemp -d) && cd ${DIR} && \ 120 | curl -sLO http://downloads.xiph.org/releases/theora/libtheora-${THEORA_VERSION}.tar.gz && \ 121 | echo ${THEORA_SHA256SUM} | sha256sum --check && \ 122 | tar -zx --strip-components=1 -f libtheora-${THEORA_VERSION}.tar.gz && \ 123 | ./configure --prefix="${SRC}" --with-ogg="${SRC}" --bindir="${SRC}/bin" \ 124 | --disable-static --datadir="${DIR}" && \ 125 | make && \ 126 | make install && \ 127 | rm -rf ${DIR} 128 | RUN \ 129 | ## libvpx https://www.webmproject.org/code/ 130 | DIR=$(mktemp -d) && cd ${DIR} && \ 131 | curl -sL https://codeload.github.com/webmproject/libvpx/tar.gz/v${VPX_VERSION} | \ 132 | tar -zx --strip-components=1 && \ 133 | ./configure --prefix="${SRC}" --enable-vp8 --enable-vp9 --enable-pic --disable-debug --disable-examples --disable-docs --disable-install-bins --enable-shared && \ 134 | make && \ 135 | make install && \ 136 | rm -rf ${DIR} 137 | RUN \ 138 | ## libmp3lame http://lame.sourceforge.net/ 139 | DIR=$(mktemp -d) && cd ${DIR} && \ 140 | curl -sL https://downloads.sf.net/project/lame/lame/${LAME_VERSION%.*}/lame-${LAME_VERSION}.tar.gz | \ 141 | tar -zx --strip-components=1 && \ 142 | ./configure --prefix="${SRC}" --bindir="${SRC}/bin" --disable-static --enable-nasm --datarootdir="${DIR}" && \ 143 | make && \ 144 | make install && \ 145 | rm -rf ${DIR} 146 | RUN \ 147 | ## xvid https://www.xvid.com/ 148 | DIR=$(mktemp -d) && cd ${DIR} && \ 149 | curl -sLO http://downloads.xvid.org/downloads/xvidcore-${XVID_VERSION}.tar.gz && \ 150 | echo ${XVID_SHA256SUM} | sha256sum --check && \ 151 | tar -zx -f xvidcore-${XVID_VERSION}.tar.gz && \ 152 | cd xvidcore/build/generic && \ 153 | ./configure --prefix="${SRC}" --bindir="${SRC}/bin" --datadir="${DIR}" --disable-static --enable-shared && \ 154 | make && \ 155 | make install && \ 156 | rm -rf ${DIR} 157 | RUN \ 158 | ## fdk-aac https://github.com/mstorsjo/fdk-aac 159 | DIR=$(mktemp -d) && cd ${DIR} && \ 160 | curl -sL https://github.com/mstorsjo/fdk-aac/archive/v${FDKAAC_VERSION}.tar.gz | \ 161 | tar -zx --strip-components=1 && \ 162 | autoreconf -fiv && \ 163 | ./configure --prefix="${SRC}" --disable-static --datadir="${DIR}" && \ 164 | make && \ 165 | make install && \ 166 | make distclean && \ 167 | rm -rf ${DIR} 168 | RUN \ 169 | ## ffmpeg https://ffmpeg.org/ 170 | DIR=$(mktemp -d) && cd ${DIR} && \ 171 | gpg --keyserver hkp://pool.sks-keyservers.net:80 --recv-keys ${FFMPEG_KEY} && \ 172 | curl -sLO http://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.gz && \ 173 | curl -sLO http://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.gz.asc && \ 174 | gpg --batch --verify ffmpeg-${FFMPEG_VERSION}.tar.gz.asc ffmpeg-${FFMPEG_VERSION}.tar.gz && \ 175 | tar -zx --strip-components=1 -f ffmpeg-${FFMPEG_VERSION}.tar.gz && \ 176 | ./configure \ 177 | --bindir="${SRC}/bin" \ 178 | --disable-debug \ 179 | --disable-doc \ 180 | --disable-ffplay \ 181 | --disable-static \ 182 | --enable-avresample \ 183 | --enable-gpl \ 184 | --enable-libopencore-amrnb \ 185 | --enable-libopencore-amrwb \ 186 | --enable-libfdk_aac \ 187 | --enable-libmp3lame \ 188 | --enable-libopus \ 189 | --enable-libtheora \ 190 | --enable-libvorbis \ 191 | --enable-libvpx \ 192 | --enable-libx264 \ 193 | --enable-libx265 \ 194 | --enable-libxvid \ 195 | --enable-nonfree \ 196 | --enable-openssl \ 197 | --enable-postproc \ 198 | --enable-shared \ 199 | --enable-small \ 200 | --enable-version3 \ 201 | --extra-cflags="-I${SRC}/include" \ 202 | --extra-ldflags="-L${SRC}/lib" \ 203 | --extra-libs=-ldl \ 204 | --prefix="${SRC}" && \ 205 | make && \ 206 | make install && \ 207 | make distclean && \ 208 | hash -r && \ 209 | cd tools && \ 210 | make qt-faststart && \ 211 | cp qt-faststart ${SRC}/bin && \ 212 | rm -rf ${DIR} 213 | 214 | RUN \ 215 | # cleanup 216 | cd && \ 217 | apk del ${buildDeps} && \ 218 | rm -rf /var/cache/apk/* && \ 219 | ffmpeg -buildconf 220 | 221 | # Let's make sure the app built correctly 222 | # Convenient to verify on https://hub.docker.com/r/jrottenberg/ffmpeg/builds/ console output 223 | # --------------------------------- 224 | 225 | ENV GOBIN=/go/bin 226 | 227 | RUN apk add --no-cache git jpeg-dev zlib-dev libpng-dev 228 | 229 | RUN git clone https://github.com/gpac/gpac.git 230 | RUN gpac/configure --static-mp4box 231 | RUN make -j4 232 | RUN make install 233 | RUN cp /tmp/workdir/bin/gcc/MP4Box /bin/MP4Box 234 | 235 | RUN go get -u github.com/satori/go.uuid 236 | RUN go get -u github.com/gin-gonic/gin 237 | RUN go get -u github.com/nareix/joy4 238 | RUN go get -u github.com/nareix/bits 239 | RUN go get -u github.com/jinzhu/gorm 240 | RUN go get -u github.com/jinzhu/gorm/dialects/postgres 241 | RUN go get -u go.uber.org/zap 242 | 243 | RUN mkdir -p /home/dev/lib 244 | 245 | ENV CGO_LDFLAGS="-L${SRC}/lib -lavcodec -lavformat -lavutil -lswscale -lswresample -lavdevice -lavfilter" 246 | ENV CGO_CFLAGS="-I${SRC}/include" 247 | ENV LD_LIBRARY_PATH=/home/dev/lib 248 | 249 | ADD common /go/src/github.com/n1207n/video-transcode-queue/api/common 250 | ADD transcode /go/src/github.com/n1207n/video-transcode-queue/api/transcode 251 | 252 | WORKDIR /go/src/github.com/n1207n/video-transcode-queue/api/transcode 253 | 254 | RUN go build 255 | RUN go install 256 | 257 | ENTRYPOINT /go/bin/transcode 258 | 259 | EXPOSE 8800 260 | -------------------------------------------------------------------------------- /api/Dockerfile-video-backend: -------------------------------------------------------------------------------- 1 | FROM golang:1.8.3-alpine 2 | 3 | ENV GOBIN=/go/bin 4 | 5 | RUN apk update && apk upgrade && \ 6 | apk add --no-cache git openssh 7 | 8 | RUN go get -u github.com/satori/go.uuid 9 | RUN go get -u github.com/joho/godotenv 10 | RUN go get -u go.uber.org/zap 11 | RUN go get -u github.com/gin-gonic/gin 12 | RUN go get -u github.com/gin-gonic/autotls 13 | RUN go get -u github.com/jinzhu/gorm 14 | RUN go get -u github.com/jinzhu/gorm/dialects/postgres 15 | RUN go get -u gopkg.in/redis.v3 16 | RUN go get -u github.com/adjust/rmq 17 | 18 | ADD common /go/src/github.com/n1207n/video-transcode-queue/api/common 19 | ADD backend /go/src/github.com/n1207n/video-transcode-queue/api/backend 20 | 21 | WORKDIR /go/src/github.com/n1207n/video-transcode-queue/api/backend 22 | 23 | RUN go build 24 | RUN go install 25 | 26 | ENTRYPOINT /go/bin/backend 27 | 28 | EXPOSE 8080 29 | -------------------------------------------------------------------------------- /api/backend/backend_api_server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "os" 9 | "strconv" 10 | "time" 11 | 12 | "go.uber.org/zap" 13 | 14 | "github.com/adjust/rmq" 15 | "github.com/gin-gonic/gin" 16 | "github.com/n1207n/video-transcode-queue/api/common/database" 17 | "github.com/n1207n/video-transcode-queue/api/common/entity" 18 | 19 | redis "gopkg.in/redis.v3" 20 | ) 21 | 22 | var ( 23 | pgDb, pgUser, pgPassword, pgHost string 24 | uploadFolderPath string 25 | redisURL, redisPort, redisPassword, redisTopic string 26 | redisProtocol = "tcp" 27 | redisNetworkTag = "transcode_task_consume" 28 | logger *zap.SugaredLogger 29 | ) 30 | 31 | func main() { 32 | loadEnvironmentVariables() 33 | database.CreateSchemas(pgUser, pgPassword, pgHost, pgDb) 34 | startBackendAPIServer() 35 | } 36 | 37 | // loadEnvironmentVariables loads PostgreSQL 38 | // information from dotenv 39 | func loadEnvironmentVariables() { 40 | pgDb = os.Getenv("PGDB") 41 | if len(pgDb) == 0 { 42 | panic("No pgDB environment variable") 43 | } 44 | 45 | pgUser = os.Getenv("PGUSER") 46 | if len(pgUser) == 0 { 47 | panic("No pgUSER environment variable") 48 | } 49 | 50 | pgPassword = os.Getenv("PGPASSWORD") 51 | if len(pgPassword) == 0 { 52 | panic("No pgPASSWORD environment variable") 53 | } 54 | 55 | pgHost = os.Getenv("PGHOST") 56 | if len(pgHost) == 0 { 57 | panic("No pgHOST environment variable") 58 | } 59 | 60 | uploadFolderPath = os.Getenv("UPLOAD_FOLDER_PATH") 61 | if len(uploadFolderPath) == 0 { 62 | panic("No UPLOAD_FOLDER_PATH environment variable") 63 | } 64 | 65 | redisURL = os.Getenv("REDIS_URL") 66 | if len(redisURL) == 0 { 67 | panic("No REDIS_URL environment variable") 68 | } 69 | 70 | redisPort = os.Getenv("REDIS_PORT") 71 | if len(redisPort) == 0 { 72 | panic("No REDIS_PORT environment variable") 73 | } 74 | 75 | redisPassword = os.Getenv("REDIS_PASSWORD") 76 | if len(redisPassword) == 0 { 77 | panic("No REDIS_PASSWORD environment variable") 78 | } 79 | 80 | redisTopic = os.Getenv("REDIS_TOPIC") 81 | if len(redisTopic) == 0 { 82 | panic("No REDIS_TOPIC environment variable") 83 | } 84 | } 85 | 86 | // openTaskQueue connects to redis and return a Queue interface 87 | func openTaskQueue() rmq.Queue { 88 | redisClient := redis.NewClient(&redis.Options{ 89 | Network: redisProtocol, 90 | Addr: fmt.Sprintf("%s:%s", redisURL, redisPort), 91 | DB: int64(1), 92 | Password: redisPassword, 93 | }) 94 | 95 | connection := rmq.OpenConnectionWithRedisClient(redisNetworkTag, redisClient) 96 | 97 | logger.Infof("Connected to Redis task queue: %s\n", connection.Name) 98 | 99 | return connection.OpenQueue(redisTopic) 100 | } 101 | 102 | func startBackendAPIServer() { 103 | log, _ := zap.NewProduction() 104 | defer log.Sync() 105 | 106 | logger = log.Sugar() 107 | logger.Info("Starting video backend API server") 108 | 109 | // Creates a gin router with default middleware: 110 | // logger and recovery (crash-free) middleware 111 | router := gin.Default() 112 | 113 | v1 := router.Group("/api/v1") 114 | { 115 | v1.GET("/videos", getVideoList) 116 | v1.GET("/videos/:id", getVideoDetail) 117 | v1.DELETE("/videos/:id", deleteVideo) 118 | v1.POST("/videos", createVideo) 119 | v1.POST("/video-upload", uploadVideoFile) 120 | } 121 | 122 | // By default it serves on :8080 123 | router.Run() 124 | } 125 | 126 | func getVideoList(c *gin.Context) { 127 | connection := database.GetConnection(pgUser, pgPassword, pgHost, pgDb) 128 | count, videos, err := database.GetVideoObjects(connection) 129 | 130 | if err != nil { 131 | c.JSON(http.StatusBadRequest, gin.H{ 132 | "error": err.Error(), 133 | }) 134 | } else { 135 | c.JSON(http.StatusOK, gin.H{ 136 | "count": count, 137 | "results": videos, 138 | }) 139 | } 140 | } 141 | 142 | func getVideoDetail(c *gin.Context) { 143 | videoID, err := strconv.Atoi(c.Param("id")) 144 | 145 | connection := database.GetConnection(pgUser, pgPassword, pgHost, pgDb) 146 | video, err := database.GetVideoObject(videoID, connection) 147 | if err != nil { 148 | c.JSON(http.StatusBadRequest, gin.H{ 149 | "error": err.Error(), 150 | }) 151 | } else { 152 | c.JSON(http.StatusOK, gin.H{ 153 | "data": video, 154 | }) 155 | } 156 | } 157 | 158 | func deleteVideo(c *gin.Context) { 159 | videoID, err := strconv.Atoi(c.Param("id")) 160 | 161 | connection := database.GetConnection(pgUser, pgPassword, pgHost, pgDb) 162 | video, err := database.GetVideoObject(videoID, connection) 163 | if err != nil { 164 | c.JSON(http.StatusBadRequest, gin.H{ 165 | "error": err.Error(), 166 | }) 167 | 168 | return 169 | } 170 | 171 | connection = database.GetConnection(pgUser, pgPassword, pgHost, pgDb) 172 | video, err = database.DeleteVideoObject(video, connection) 173 | if err != nil { 174 | c.JSON(http.StatusBadRequest, gin.H{ 175 | "error": err.Error(), 176 | }) 177 | } else { 178 | c.JSON(http.StatusNoContent, gin.H{ 179 | "data": video, 180 | }) 181 | } 182 | } 183 | 184 | func createVideo(c *gin.Context) { 185 | var videoSerializer entity.Video 186 | 187 | if err := c.BindJSON(&videoSerializer); err != nil { 188 | c.JSON(http.StatusBadRequest, gin.H{ 189 | "error": err.Error(), 190 | }) 191 | 192 | return 193 | } 194 | 195 | connection := database.GetConnection(pgUser, pgPassword, pgHost, pgDb) 196 | videoSerializer, err := database.CreateVideoObject(videoSerializer, connection) 197 | if err != nil { 198 | c.JSON(http.StatusBadRequest, gin.H{ 199 | "error": err.Error(), 200 | }) 201 | 202 | return 203 | } 204 | 205 | c.JSON(http.StatusOK, gin.H{ 206 | "title": videoSerializer.Title, 207 | "message": "Object created. Please upload the file for this Video.", 208 | }) 209 | } 210 | 211 | func uploadVideoFile(c *gin.Context) { 212 | c.Request.ParseMultipartForm(64 << 25) 213 | 214 | videoID := c.PostForm("video_id") 215 | if videoID == "" { 216 | c.JSON(http.StatusBadRequest, gin.H{ 217 | "message": "video_id is required", 218 | }) 219 | 220 | return 221 | } 222 | 223 | file, header, err := c.Request.FormFile("upload") 224 | if err != nil { 225 | c.JSON(http.StatusBadRequest, gin.H{ 226 | "error": err.Error(), 227 | "message": "Please upload file with 'upload' form field key.", 228 | }) 229 | 230 | return 231 | } 232 | 233 | filename := header.Filename 234 | 235 | videoFolderPath := uploadFolderPath + videoID 236 | os.MkdirAll(videoFolderPath, os.ModePerm) 237 | 238 | videoFullPath := fmt.Sprintf("%s/%s", videoFolderPath, filename) 239 | 240 | outFile, err := os.Create(videoFullPath) 241 | if err != nil { 242 | logger.Fatal("Failed to write filesystem:", err) 243 | 244 | c.JSON(http.StatusBadRequest, gin.H{ 245 | "error": err.Error(), 246 | "message": "File upload is having issues right now. Please try later.", 247 | }) 248 | 249 | return 250 | } 251 | 252 | defer outFile.Close() 253 | 254 | _, err = io.Copy(outFile, file) 255 | if err != nil { 256 | logger.Fatal("Failed to copy video file:", err) 257 | 258 | c.JSON(http.StatusBadRequest, gin.H{ 259 | "error": err.Error(), 260 | "message": "File upload is having issues right now. Please try later.", 261 | }) 262 | 263 | return 264 | } 265 | 266 | taskQueue := openTaskQueue() 267 | task := entity.Task{ID: videoID, Timestamp: time.Now(), FilePath: videoFullPath} 268 | 269 | queueDataBytes, err := json.Marshal(task) 270 | taskQueue.PublishBytes(queueDataBytes) 271 | logger.Info("Queue task created...:", task) 272 | 273 | c.JSON(http.StatusOK, gin.H{ 274 | "message": fmt.Sprintf("Video file uploaded. Transcoding now: %s", videoID), 275 | }) 276 | } 277 | -------------------------------------------------------------------------------- /api/common/database/operation.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/jinzhu/gorm" 7 | _ "github.com/jinzhu/gorm/dialects/postgres" 8 | "github.com/n1207n/video-transcode-queue/api/common/entity" 9 | ) 10 | 11 | // GetConnection returns an instance 12 | // as a database connection 13 | func GetConnection(user string, password string, host string, db string) *gorm.DB { 14 | connection, err := gorm.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable", user, password, host, db)) 15 | 16 | if err != nil { 17 | panic(err) 18 | } 19 | 20 | return connection 21 | } 22 | 23 | // CreateSchemas creates a set of database tables 24 | // from Go struct classes 25 | func CreateSchemas(user string, password string, host string, db string) { 26 | connection := GetConnection(user, password, host, db) 27 | 28 | defer connection.Close() 29 | 30 | connection.AutoMigrate(&entity.Video{}, &entity.VideoRendering{}) 31 | 32 | connection.Model(&entity.VideoRendering{}).AddForeignKey("video_id", "videos(id)", "CASCADE", "CASCADE") 33 | 34 | connection.Model(&entity.VideoRendering{}).AddIndex("idx_video_id", "video_id") 35 | 36 | } 37 | -------------------------------------------------------------------------------- /api/common/database/orm.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/jinzhu/gorm" 7 | "github.com/n1207n/video-transcode-queue/api/common/entity" 8 | ) 9 | 10 | // GetVideoObjects returns a list of Video objects and its count from database 11 | func GetVideoObjects(connection *gorm.DB) (uint, []entity.Video, error) { 12 | var videos []entity.Video 13 | var count uint 14 | var dbError error 15 | 16 | defer connection.Close() 17 | 18 | connection.Preload("Renderings").Find(&videos) 19 | if connection.Error != nil { 20 | dbError = connection.Error 21 | } 22 | 23 | return count, videos, dbError 24 | } 25 | 26 | // GetVideoRenderingObjects returns a list of VideoRendering objects linked to Video ID and its count from database 27 | func GetVideoRenderingObjects(video entity.Video, connection *gorm.DB) (uint, []entity.VideoRendering, error) { 28 | var renderings []entity.VideoRendering 29 | var count uint 30 | var dbError error 31 | 32 | defer connection.Close() 33 | 34 | connection.Model(&video).Related(&renderings).Count(&count) 35 | if connection.Error != nil { 36 | dbError = connection.Error 37 | } 38 | 39 | return count, renderings, dbError 40 | } 41 | 42 | // GetVideoObject returns a Video object from given id from database 43 | func GetVideoObject(videoID int, connection *gorm.DB) (entity.Video, error) { 44 | var video entity.Video 45 | var dbError error 46 | 47 | defer connection.Close() 48 | 49 | connection.Where(map[string]interface{}{"id": videoID}).Preload("Renderings").First(&video) 50 | if connection.Error != nil { 51 | dbError = connection.Error 52 | } 53 | 54 | if video.ID == 0 { 55 | dbError = errors.New("no video found") 56 | } 57 | 58 | return video, dbError 59 | } 60 | 61 | // CreateVideoObject pushes Video object to database 62 | func CreateVideoObject(videoSerializer entity.Video, connection *gorm.DB) (entity.Video, error) { 63 | var dbError error 64 | 65 | defer connection.Close() 66 | 67 | connection.NewRecord(videoSerializer) 68 | connection.Create(&videoSerializer) 69 | if connection.Error != nil { 70 | dbError = connection.Error 71 | } 72 | 73 | return videoSerializer, dbError 74 | } 75 | 76 | // UpdateVideoObject updates Video object to database 77 | func UpdateVideoObject(updatedVideo entity.Video, connection *gorm.DB) (entity.Video, error) { 78 | var dbError error 79 | 80 | defer connection.Close() 81 | 82 | connection.Save(&updatedVideo) 83 | if connection.Error != nil { 84 | dbError = connection.Error 85 | } 86 | 87 | return updatedVideo, dbError 88 | } 89 | 90 | // DeleteVideoObject deletes Video object in database 91 | func DeleteVideoObject(video entity.Video, connection *gorm.DB) (entity.Video, error) { 92 | var dbError error 93 | 94 | defer connection.Close() 95 | 96 | connection.Delete(&video) 97 | if connection.Error != nil { 98 | dbError = connection.Error 99 | } 100 | 101 | return video, dbError 102 | } 103 | 104 | // CreateVideoRenderingObject pushes VideoRendering object to database 105 | func CreateVideoRenderingObject(videoRendering entity.VideoRendering, connection *gorm.DB) (entity.VideoRendering, error) { 106 | var dbError error 107 | 108 | defer connection.Close() 109 | 110 | connection.NewRecord(videoRendering) 111 | connection.Create(&videoRendering) 112 | if connection.Error != nil { 113 | dbError = connection.Error 114 | } 115 | 116 | return videoRendering, dbError 117 | } 118 | -------------------------------------------------------------------------------- /api/common/entity/ffmpeg.go: -------------------------------------------------------------------------------- 1 | package entity 2 | 3 | import "time" 4 | 5 | // FFProbeStreamData represents JSON format for each stream 6 | type FFProbeStreamData struct { 7 | Index int `json:"index"` 8 | CodecName string `json:"codec_name"` 9 | CodecLongName string `json:"codec_long_name"` 10 | Profile int `json:"profile,string"` 11 | CodecType string `json:"codec_type"` 12 | CodecTimeBase string `json:"codec_time_base"` 13 | CodecTagString string `json:"codec_tag_string"` 14 | CodecTag string `json:"codec_tag"` 15 | Width *int `json:"width,omitempty"` 16 | Height *int `json:"height,omitempty"` 17 | CodedWidth *int `json:"coded_width,omitempty"` 18 | CodedHeight *int `json:"coded_height,omitempty"` 19 | HasBFrames *int `json:"has_b_frames,omitempty"` 20 | SampleAspectRatio *string `json:"sample_aspect_ratio,omitempty"` 21 | DisplayAspectRatio *string `json:"display_aspect_ratio,omitempty"` 22 | PixFmt *string `json:"pix_fmt,omitempty"` 23 | Level *int `json:"level,omitempty"` 24 | ColorRange *string `json:"color_range,omitempty"` 25 | ColorSpace *string `json:"color_space,omitempty"` 26 | ColorTransfer *string `json:"color_transfer,omitempty"` 27 | ColorPrimaries *string `json:"color_primaries,omitempty"` 28 | ChromaLocation *string `json:"chroma_location,omitempty"` 29 | Refs *int `json:"refs,omitempty"` 30 | IsAVC *bool `json:"is_avc,string,omitempty"` 31 | NalLengthSize *int `json:"nal_length_size,string,omitempty"` 32 | RFrameRate string `json:"r_frame_rate"` 33 | AVGFrameRate string `json:"avg_frame_rate"` 34 | TimeBase string `json:"time_base"` 35 | StartPTS int `json:"start_pts"` 36 | StartTime float64 `json:"start_time,string"` 37 | DurationTS int `json:"duration_ts"` 38 | Duration float64 `json:"duration,string"` 39 | BitRate int `json:"bit_rate,string"` 40 | BitsPerRawSample *int `json:"bits_per_raw_sample,string,omitempty"` 41 | NBFrames int `json:"nb_frames,string"` 42 | SampleFMT *string `json:"sample_fmt,omitempty"` 43 | SampleRate *int `json:"sample_rate,string,omitempty"` 44 | Channels *int `json:"channels,omitempty"` 45 | ChannelLayout *string `json:"channel_layout,omitempty"` 46 | BitsPerSample *int `json:"bits_per_sample,omitempty"` 47 | Disposition map[string]int `json:"disposition"` 48 | Tags map[string]string `json:"tags"` 49 | } 50 | 51 | // StartTimeDuration represents 52 | // FFProbeStreamData's StartTime field as Duration object 53 | func (f FFProbeStreamData) StartTimeDuration() time.Duration { 54 | return time.Duration(f.StartTime * float64(time.Second)) 55 | } 56 | 57 | // DurationAsObject represents 58 | // FFProbeStreamData's Duration field as Duration object 59 | func (f FFProbeStreamData) DurationAsObject() time.Duration { 60 | return time.Duration(f.Duration * float64(time.Second)) 61 | } 62 | 63 | // ProbeData represents ffprobe info as JSON struct 64 | type ProbeData struct { 65 | Stream []FFProbeStreamData `json:"streams,omitempty"` 66 | } 67 | -------------------------------------------------------------------------------- /api/common/entity/queue.go: -------------------------------------------------------------------------------- 1 | package entity 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // Task represents a job task to transcode a video file 8 | type Task struct { 9 | ID string 10 | FilePath string 11 | Timestamp time.Time 12 | } 13 | -------------------------------------------------------------------------------- /api/common/entity/video.go: -------------------------------------------------------------------------------- 1 | package entity 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | // Video represents an uploaded video instance 9 | // Relation: 10 | // - has many VideoRendering 11 | type Video struct { 12 | ID uint `gorm:"primary_key" json:"id"` 13 | CreatedAt time.Time `gorm:"not null" json:"created_at"` 14 | UpdatedAt time.Time `gorm:"not null" json:"updated_at"` 15 | 16 | Title string `gorm:"not null" json:"title" binding:"required"` 17 | IsReadyToServe bool `sql:"DEFAULT:false" json:"is_ready_to_serve"` 18 | StreamFilePath string `json:"stream_file_path"` 19 | 20 | Renderings []VideoRendering `gorm:"ForeignKey:VideoID"` 21 | } 22 | 23 | func (v Video) String() string { 24 | return fmt.Sprintf("Video: %d - %s", v.ID, v.Title) 25 | } 26 | 27 | // VideoRendering represents each rendering variant from original 28 | type VideoRendering struct { 29 | ID uint `gorm:"primary_key" json:"id"` 30 | CreatedAt time.Time `gorm:"not null" json:"created_at"` 31 | UpdatedAt time.Time `gorm:"not null" json:"updated_at"` 32 | RenderingTitle string `gorm:"not null" json:"rendering_title" binding:"required"` 33 | 34 | FilePath string `gorm:"not null" json:"file_path"` 35 | URL string `gorm:"not null" json:"url"` 36 | Width uint `gorm:"not null" json:"width"` 37 | Height uint `gorm:"not null" json:"height"` 38 | 39 | VideoID uint `json:"video_id"` 40 | } 41 | 42 | func (vr VideoRendering) String() string { 43 | return fmt.Sprintf("VideoRendering: %d - %s", vr.ID, vr.RenderingTitle) 44 | } 45 | -------------------------------------------------------------------------------- /api/streaming/streaming_api_server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "go.uber.org/zap" 7 | 8 | "github.com/gin-contrib/cors" 9 | "github.com/gin-contrib/static" 10 | "github.com/gin-gonic/gin" 11 | ) 12 | 13 | var ( 14 | pgDb, pgUser, pgPassword, pgHost string 15 | uploadFolderPath string 16 | logger *zap.SugaredLogger 17 | ) 18 | 19 | func main() { 20 | loadEnvironmentVariables() 21 | startStreamingAPIServer() 22 | } 23 | 24 | // loadEnvironmentVariables loads PostgreSQL 25 | // information from dotenv 26 | func loadEnvironmentVariables() { 27 | pgDb = os.Getenv("PGDB") 28 | if len(pgDb) == 0 { 29 | panic("No pgDB environment variable") 30 | } 31 | 32 | pgUser = os.Getenv("PGUSER") 33 | if len(pgUser) == 0 { 34 | panic("No pgUSER environment variable") 35 | } 36 | 37 | pgPassword = os.Getenv("PGPASSWORD") 38 | if len(pgPassword) == 0 { 39 | panic("No pgPASSWORD environment variable") 40 | } 41 | 42 | pgHost = os.Getenv("PGHOST") 43 | if len(pgHost) == 0 { 44 | panic("No pgHOST environment variable") 45 | } 46 | 47 | uploadFolderPath = os.Getenv("UPLOAD_FOLDER_PATH") 48 | if len(uploadFolderPath) == 0 { 49 | panic("No UPLOAD_FOLDER_PATH environment variable") 50 | } 51 | } 52 | 53 | func startStreamingAPIServer() { 54 | log, _ := zap.NewProduction() 55 | defer log.Sync() 56 | 57 | logger = log.Sugar() 58 | logger.Info("Starting streaming API server") 59 | 60 | // Creates a gin router with default middleware: 61 | // logger and recovery (crash-free) middleware 62 | router := gin.Default() 63 | 64 | corsConfig := cors.DefaultConfig() 65 | corsConfig.AllowAllOrigins = true 66 | corsConfig.AllowHeaders = []string{"Origin", "Content-Length", "Content-Type", "Range"} 67 | 68 | router.Use(cors.New(corsConfig)) 69 | 70 | // TODO: Use uploadFolderPath later for better security 71 | router.Use(static.Serve("/contents", static.LocalFile("/", false))) 72 | 73 | // By default it serves on :8080 74 | router.Run(":8880") 75 | } 76 | -------------------------------------------------------------------------------- /api/transcode/transcode_api_server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/http" 5 | "os" 6 | "strconv" 7 | "strings" 8 | "sync" 9 | 10 | "github.com/gin-gonic/gin" 11 | 12 | "go.uber.org/zap" 13 | ) 14 | 15 | var ( 16 | pgDb, pgUser, pgPassword, pgHost string 17 | uploadFolderPath string 18 | logger *zap.SugaredLogger 19 | ) 20 | 21 | func main() { 22 | loadEnvironmentVariables() 23 | startTranscodeAPIServer() 24 | } 25 | 26 | // loadEnvironmentVariables loads PostgreSQL 27 | // information from dotenv 28 | func loadEnvironmentVariables() { 29 | pgDb = os.Getenv("PGDB") 30 | if len(pgDb) == 0 { 31 | panic("No pgDB environment variable") 32 | } 33 | 34 | pgUser = os.Getenv("PGUSER") 35 | if len(pgUser) == 0 { 36 | panic("No pgUSER environment variable") 37 | } 38 | 39 | pgPassword = os.Getenv("PGPASSWORD") 40 | if len(pgPassword) == 0 { 41 | panic("No pgPASSWORD environment variable") 42 | } 43 | 44 | pgHost = os.Getenv("PGHOST") 45 | if len(pgHost) == 0 { 46 | panic("No pgHOST environment variable") 47 | } 48 | 49 | uploadFolderPath = os.Getenv("UPLOAD_FOLDER_PATH") 50 | if len(uploadFolderPath) == 0 { 51 | panic("No UPLOAD_FOLDER_PATH environment variable") 52 | } 53 | } 54 | 55 | func startTranscodeAPIServer() { 56 | log, _ := zap.NewProduction() 57 | defer log.Sync() 58 | 59 | logger = log.Sugar() 60 | logger.Info("Starting transcode API server") 61 | 62 | // Creates a gin router with default middleware: 63 | // logger and recovery (crash-free) middleware 64 | router := gin.Default() 65 | 66 | v1 := router.Group("/api/v1") 67 | { 68 | v1.POST("/video-transcode", transcodeVideo) 69 | } 70 | 71 | // By default it serves on :8080 72 | router.Run(":8800") 73 | } 74 | 75 | func transcodeVideo(c *gin.Context) { 76 | var request TranscodeRequest 77 | 78 | if c.BindJSON(&request) == nil { 79 | // TODO: Check if video id exists in PostgreSQL 80 | 81 | performTranscoding(request, c) 82 | } 83 | } 84 | 85 | // TranscodeRequest represents a JSON POST data for video-transcode API 86 | type TranscodeRequest struct { 87 | Path string `json:"path" binding:"required"` 88 | VideoID string `json:"video_id" binding:"required"` 89 | } 90 | 91 | func performTranscoding(request TranscodeRequest, c *gin.Context) (transcodedFilePaths []string, transcodeError error) { 92 | splitStringPaths := strings.Split(request.Path, "/") 93 | fileFolderPath := strings.Join(splitStringPaths[:len(splitStringPaths)-1], "/") 94 | filename := splitStringPaths[len(splitStringPaths)-1] 95 | 96 | // Strip the file extension and convert any reverse subsequent dots to underscore 97 | splitFilenameCharacters := strings.Split(filename, ".") 98 | videoName := strings.Join(splitFilenameCharacters[:len(splitFilenameCharacters)-1], "_") 99 | 100 | videoID, _ := strconv.Atoi(request.VideoID) 101 | 102 | var waitGroup sync.WaitGroup 103 | 104 | _, height, err := GetVideoDimensionInfo(filename, fileFolderPath, logger) 105 | if err != nil { 106 | logger.Errorf("Error from getting video dimension info: %s\n", err.Error()) 107 | 108 | c.JSON(http.StatusBadRequest, gin.H{"video_id": request.VideoID, "status": "Failed to get video metadata. Corrupted file?"}) 109 | 110 | // TODO: Delete the video file 111 | } else { 112 | var targets []int 113 | dbConnectionInfo := map[string]string{ 114 | "pgDb": pgDb, 115 | "pgUser": pgUser, 116 | "pgPassword": pgPassword, 117 | "pgHost": pgHost, 118 | } 119 | 120 | if height >= 720 { 121 | targets = append(targets, 720) 122 | } 123 | 124 | if height >= 540 { 125 | targets = append(targets, 540) 126 | } 127 | 128 | if height >= 360 { 129 | targets = append(targets, 360) 130 | } 131 | 132 | if height < 360 { 133 | targets = append(targets, 360) 134 | } 135 | 136 | waitGroup.Add(len(targets)) 137 | 138 | for _, target := range targets { 139 | switch target { 140 | case 720: 141 | go TranscodeToHD720P(videoName, videoID, filename, fileFolderPath, dbConnectionInfo, &waitGroup, logger) 142 | case 540: 143 | go TranscodeToSD540P(videoName, videoID, filename, fileFolderPath, dbConnectionInfo, &waitGroup, logger) 144 | case 360: 145 | go TranscodeToSD360P(videoName, videoID, filename, fileFolderPath, dbConnectionInfo, &waitGroup, logger) 146 | default: 147 | go TranscodeToSD360P(videoName, videoID, filename, fileFolderPath, dbConnectionInfo, &waitGroup, logger) 148 | } 149 | } 150 | 151 | waitGroup.Wait() 152 | 153 | c.JSON(http.StatusOK, gin.H{"video_id": request.VideoID, "status": "In progress"}) 154 | 155 | logger.Infof("Constructing MPD for %s", videoName) 156 | 157 | ConstructMPD(videoName, videoID, filename, fileFolderPath, targets, dbConnectionInfo, logger) 158 | } 159 | 160 | return 161 | } 162 | -------------------------------------------------------------------------------- /api/transcode/transcoder.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "os/exec" 8 | "strings" 9 | "sync" 10 | "time" 11 | 12 | "github.com/n1207n/video-transcode-queue/api/common/database" 13 | "github.com/n1207n/video-transcode-queue/api/common/entity" 14 | 15 | "go.uber.org/zap" 16 | ) 17 | 18 | // ExecuteCLI executes constructed command string by os.exec.Command 19 | func ExecuteCLI(commandString string, returnOutput bool) ([]byte, error) { 20 | commandArguments := strings.Fields(commandString) 21 | head, commandArguments := commandArguments[0], commandArguments[1:] 22 | 23 | cmd := exec.Command(head, commandArguments...) 24 | outputBytes, err := cmd.Output() 25 | 26 | if err != nil { 27 | return outputBytes, err 28 | } 29 | 30 | return outputBytes, nil 31 | } 32 | 33 | // GetVideoDimensionInfo extracts video width and height values 34 | func GetVideoDimensionInfo(filename string, folderPath string, logger *zap.SugaredLogger) (int, int, error) { 35 | logger.Infof("Getting video resolution info: %s/%s\n", folderPath, filename) 36 | 37 | ffprobeCommand := fmt.Sprintf("ffprobe -show_streams -print_format json -v quiet %s/%s", folderPath, filename) 38 | 39 | width, height := -1, -1 40 | 41 | outputBytes, err := ExecuteCLI(ffprobeCommand, true) 42 | if err != nil { 43 | logger.Errorf("Error during command execution: %s\nError: %s", ffprobeCommand, err.Error()) 44 | 45 | return width, height, err 46 | } 47 | 48 | var probeData entity.ProbeData 49 | err = json.Unmarshal(outputBytes, &probeData) 50 | 51 | if err != nil { 52 | logger.Errorf("ffprobe JSON parse error: %s\n", err.Error()) 53 | 54 | return width, height, err 55 | } 56 | 57 | for index := 0; index < len(probeData.Stream); index++ { 58 | stream := probeData.Stream[index] 59 | 60 | if stream.Width != nil { 61 | width = *probeData.Stream[0].Width 62 | height = *probeData.Stream[0].Height 63 | break 64 | } 65 | } 66 | 67 | if width == -1 { 68 | return width, height, errors.New("no video stream found from file") 69 | } 70 | 71 | return width, height, nil 72 | } 73 | 74 | // TranscodeToSD360P transcodes video file to 360P 75 | func TranscodeToSD360P(videoName string, videoID int, filename string, folderPath string, dbConnectionInfo map[string]string, waitGroup *sync.WaitGroup, logger *zap.SugaredLogger) { 76 | logger.Infof("Transcoding to SD 360P: %s\n", videoName) 77 | 78 | defer waitGroup.Done() 79 | 80 | transcodedFileName := fmt.Sprintf("%s/%s_360.mp4", folderPath, videoName) 81 | 82 | ffmpegCommand360P := fmt.Sprintf("ffmpeg -y -i %s/%s -c:a libfdk_aac -ac 2 -ab 128k -preset slow -c:v libx264 -x264opts keyint=24:min-keyint=24:no-scenecut -b:v 400k -maxrate 400k -bufsize 400k -vf scale=-1:360 -pass 1 %s", folderPath, filename, transcodedFileName) 83 | 84 | _, err := ExecuteCLI(ffmpegCommand360P, false) 85 | if err != nil { 86 | logger.Errorf("Error during command execution: %s\nError: %s", ffmpegCommand360P, err.Error()) 87 | return 88 | } 89 | 90 | logger.Infof("Transcoded to SD 360P: %s\n", videoName) 91 | 92 | width, height, err := GetVideoDimensionInfo(videoName+"_360.mp4", folderPath, logger) 93 | if err != nil { 94 | logger.Errorf("Error from getting video dimension info: %s\n", err.Error()) 95 | return 96 | } 97 | 98 | videoRendering := entity.VideoRendering{ 99 | CreatedAt: time.Now(), 100 | UpdatedAt: time.Now(), 101 | RenderingTitle: fmt.Sprintf("%s_360", videoName), 102 | FilePath: fmt.Sprintf("%s/%s_360.mp4", folderPath, videoName), 103 | URL: fmt.Sprintf("%s/%s_360.mp4", folderPath, videoName), 104 | Width: uint(width), 105 | Height: uint(height), 106 | VideoID: uint(videoID), 107 | } 108 | 109 | pgDb := dbConnectionInfo["pgDb"] 110 | pgUser := dbConnectionInfo["pgUser"] 111 | pgPassword := dbConnectionInfo["pgPassword"] 112 | pgHost := dbConnectionInfo["pgHost"] 113 | 114 | connection := database.GetConnection(pgUser, pgPassword, pgHost, pgDb) 115 | database.CreateVideoRenderingObject(videoRendering, connection) 116 | 117 | logger.Infof("Added DB record for SD 360P: %s\n", videoName) 118 | } 119 | 120 | // TranscodeToSD540P transcodes video file to 540P 121 | func TranscodeToSD540P(videoName string, videoID int, filename string, folderPath string, dbConnectionInfo map[string]string, waitGroup *sync.WaitGroup, logger *zap.SugaredLogger) { 122 | logger.Infof("Transcoding to SD 540P: %s\n", videoName) 123 | 124 | defer waitGroup.Done() 125 | 126 | transcodedFileName := fmt.Sprintf("%s/%s_540.mp4", folderPath, videoName) 127 | 128 | ffmpegCommand540P := fmt.Sprintf("ffmpeg -y -i %s/%s -c:a libfdk_aac -ac 2 -ab 128k -preset slow -c:v libx264 -x264opts keyint=24:min-keyint=24:no-scenecut -b:v 800k -maxrate 800k -bufsize 500k -vf scale=-1:540 -pass 1 %s", folderPath, filename, transcodedFileName) 129 | 130 | _, err := ExecuteCLI(ffmpegCommand540P, false) 131 | if err != nil { 132 | logger.Errorf("Error during command execution: %s\nError: %s", ffmpegCommand540P, err.Error()) 133 | return 134 | } 135 | 136 | logger.Infof("Transcoded to SD 540P: %s\n", videoName) 137 | 138 | width, height, err := GetVideoDimensionInfo(videoName+"_540.mp4", folderPath, logger) 139 | if err != nil { 140 | logger.Errorf("Error from getting video dimension info: %s\n", err.Error()) 141 | return 142 | } 143 | 144 | videoRendering := entity.VideoRendering{ 145 | CreatedAt: time.Now(), 146 | UpdatedAt: time.Now(), 147 | RenderingTitle: fmt.Sprintf("%s_540", videoName), 148 | FilePath: fmt.Sprintf("%s/%s_540.mp4", folderPath, videoName), 149 | URL: fmt.Sprintf("%s/%s_540.mp4", folderPath, videoName), 150 | Width: uint(width), 151 | Height: uint(height), 152 | VideoID: uint(videoID), 153 | } 154 | 155 | pgDb := dbConnectionInfo["pgDb"] 156 | pgUser := dbConnectionInfo["pgUser"] 157 | pgPassword := dbConnectionInfo["pgPassword"] 158 | pgHost := dbConnectionInfo["pgHost"] 159 | 160 | connection := database.GetConnection(pgUser, pgPassword, pgHost, pgDb) 161 | database.CreateVideoRenderingObject(videoRendering, connection) 162 | 163 | logger.Infof("Added DB record for SD 540P: %s\n", videoName) 164 | } 165 | 166 | // TranscodeToHD720P transcodes video file to 720P 167 | func TranscodeToHD720P(videoName string, videoID int, filename string, folderPath string, dbConnectionInfo map[string]string, waitGroup *sync.WaitGroup, logger *zap.SugaredLogger) { 168 | logger.Infof("Transcoding to HD 720P: %s\n", videoName) 169 | 170 | defer waitGroup.Done() 171 | 172 | transcodedFileName := fmt.Sprintf("%s/%s_720.mp4", folderPath, videoName) 173 | 174 | ffmpegCommand720P := fmt.Sprintf("ffmpeg -y -i %s/%s -c:a libfdk_aac -ac 2 -ab 128k -preset slow -c:v libx264 -x264opts keyint=24:min-keyint=24:no-scenecut -b:v 1500k -maxrate 1500k -bufsize 1000k -vf scale=-1:720 -pass 1 %s", folderPath, filename, transcodedFileName) 175 | 176 | _, err := ExecuteCLI(ffmpegCommand720P, false) 177 | if err != nil { 178 | logger.Errorf("Error during command execution: %s\nError: %s", ffmpegCommand720P, err.Error()) 179 | return 180 | } 181 | 182 | logger.Infof("Transcoded to HD 720P: %s\n", videoName) 183 | 184 | width, height, err := GetVideoDimensionInfo(videoName+"_720.mp4", folderPath, logger) 185 | if err != nil { 186 | logger.Errorf("Error from getting video dimension info: %s\n", err.Error()) 187 | return 188 | } 189 | 190 | videoRendering := entity.VideoRendering{ 191 | CreatedAt: time.Now(), 192 | UpdatedAt: time.Now(), 193 | RenderingTitle: fmt.Sprintf("%s_720", videoName), 194 | FilePath: fmt.Sprintf("%s/%s_720.mp4", folderPath, videoName), 195 | URL: fmt.Sprintf("%s/%s_720.mp4", folderPath, videoName), 196 | Width: uint(width), 197 | Height: uint(height), 198 | VideoID: uint(videoID), 199 | } 200 | 201 | pgDb := dbConnectionInfo["pgDb"] 202 | pgUser := dbConnectionInfo["pgUser"] 203 | pgPassword := dbConnectionInfo["pgPassword"] 204 | pgHost := dbConnectionInfo["pgHost"] 205 | 206 | connection := database.GetConnection(pgUser, pgPassword, pgHost, pgDb) 207 | database.CreateVideoRenderingObject(videoRendering, connection) 208 | 209 | logger.Infof("Added DB record for HD 720P: %s\n", videoName) 210 | } 211 | 212 | // ConstructMPD creates MPD file for DASH streaming 213 | func ConstructMPD(videoName string, videoID int, filename string, folderPath string, transcodeTargets []int, dbConnectionInfo map[string]string, logger *zap.SugaredLogger) { 214 | logger.Infof("Constructing MPD file: %s\n", videoName) 215 | 216 | filePath := fmt.Sprintf("%s/%s", folderPath, videoName) 217 | 218 | mp4boxCommand := fmt.Sprintf("MP4Box -dash 3000 -frag 3000 -rap -profile dashavc264:onDemand -out %s.mpd", filePath) 219 | 220 | // Appending video streams for each transcoded size 221 | for _, resize := range transcodeTargets { 222 | mp4boxCommand += fmt.Sprintf(" %s_%d.mp4#video", filePath, resize) 223 | } 224 | 225 | // Appending audio streams for each transcoded size 226 | for _, resize := range transcodeTargets { 227 | mp4boxCommand += fmt.Sprintf(" %s_%d.mp4#audio", filePath, resize) 228 | } 229 | 230 | _, err := ExecuteCLI(mp4boxCommand, false) 231 | if err != nil { 232 | logger.Errorf("Error during command execution: %s\nError: %s", mp4boxCommand, err.Error()) 233 | } else { 234 | pgDb := dbConnectionInfo["pgDb"] 235 | pgUser := dbConnectionInfo["pgUser"] 236 | pgPassword := dbConnectionInfo["pgPassword"] 237 | pgHost := dbConnectionInfo["pgHost"] 238 | 239 | connection := database.GetConnection(pgUser, pgPassword, pgHost, pgDb) 240 | object, err := database.GetVideoObject(videoID, connection) 241 | if err != nil { 242 | logger.Errorw("Video object GET failed for updating:", err.Error()) 243 | return 244 | } 245 | 246 | object.UpdatedAt = time.Now() 247 | object.StreamFilePath = fmt.Sprintf("%s.mpd", filePath) 248 | object.IsReadyToServe = true 249 | 250 | connection = database.GetConnection(pgUser, pgPassword, pgHost, pgDb) 251 | _, renderings, renderingListErr := database.GetVideoRenderingObjects(object, connection) 252 | if renderingListErr != nil { 253 | logger.Errorw("Video rendering objects GET failed for updating:", renderingListErr.Error()) 254 | } 255 | 256 | object.Renderings = renderings 257 | 258 | connection = database.GetConnection(pgUser, pgPassword, pgHost, pgDb) 259 | _, updateErr := database.UpdateVideoObject(object, connection) 260 | if updateErr != nil { 261 | logger.Errorw("Video object Update failed:", updateErr.Error()) 262 | } 263 | } 264 | } 265 | -------------------------------------------------------------------------------- /build_docker_images.sh: -------------------------------------------------------------------------------- 1 | docker build -f api/Dockerfile-video-backend -t n1207n/video-transcode-queue/video_backend_api:dev api 2 | 3 | docker build -f api/Dockerfile-streaming -t n1207n/video-transcode-queue/streaming_service:dev api 4 | 5 | docker build -f api/Dockerfile-transcoder -t n1207n/video-transcode-queue/transcoder_service:dev api 6 | 7 | docker build -t n1207n/video-transcode-queue/task_queue/consumer:dev -f task_queue/client/Dockerfile-consumer task_queue/client 8 | -------------------------------------------------------------------------------- /build_helm_packages.sh: -------------------------------------------------------------------------------- 1 | helm install -f task_queue/helm/queue-storage-config.yml --name=queue-storage stable/redis 2 | 3 | helm install -f database/helm/database-config.yml --name=app-database stable/postgresql 4 | -------------------------------------------------------------------------------- /build_kubernetes_resources.sh: -------------------------------------------------------------------------------- 1 | kubectl create -f kubernetes/minikube/secrets/redis-queue-info.yml 2 | 3 | kubectl create -f kubernetes/minikube/video-upload-minikube-persistent-volume.yml 4 | kubectl create -f kubernetes/minikube/video-upload-minikube-persistent-volume-claim.yml 5 | 6 | kubectl create -f kubernetes/minikube/video-api-deployment.yml 7 | # kubectl create -f kubernetes/minikube/video-api-service.yml 8 | kubectl expose deployment video-api --type=NodePort 9 | 10 | kubectl create -f kubernetes/minikube/streaming-api-deployment.yml 11 | kubectl expose deployment streaming-api --type=NodePort 12 | # kubectl create -f kubernetes/minikube/streaming-api-service.yml 13 | 14 | kubectl create -f kubernetes/minikube/transcoder-api-deployment.yml 15 | kubectl create -f kubernetes/minikube/transcoder-api-service.yml 16 | 17 | kubectl create -f kubernetes/minikube/queue-consumer-job.yml 18 | -------------------------------------------------------------------------------- /database/helm/database-config.yml: -------------------------------------------------------------------------------- 1 | ## NOTES: 2 | ## PostgreSQL can be accessed via port 5432 on the following DNS name from within your cluster: app-database-postgresql.default.svc.cluster.local 3 | 4 | ## To get your user password run: 5 | 6 | ## PGPASSWORD=$(kubectl get secret --namespace default app-database-postgresql -o jsonpath="{.data.postgres-password}" | base64 --decode; echo) 7 | 8 | ## To connect to your database run the following command (using the env variable from above): 9 | 10 | ## kubectl run app-database-postgresql-client --rm --tty -i --image postgres --env "PGPASSWORD=$PGPASSWORD" --command -- psql -U video_app -h app-database-postgresql video_app 11 | 12 | ## postgres image repository 13 | image: "postgres" 14 | ## postgres image version 15 | ## ref: https://hub.docker.com/r/library/postgres/tags/ 16 | ## 17 | imageTag: "9.6.2" 18 | 19 | ## Specify a imagePullPolicy 20 | ## 'Always' if imageTag is 'latest', else set to 'IfNotPresent' 21 | ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images 22 | ## 23 | imagePullPolicy: "Always" 24 | 25 | ## Create a database user 26 | ## Default: postgres 27 | postgresUser: "video_app" 28 | ## Default: random 10 character string 29 | postgresPassword: "video_app" 30 | 31 | ## Create a database 32 | ## Default: the postgres user 33 | postgresDatabase: "video_app" 34 | 35 | ## Specify initdb arguments, e.g. --data-checksums 36 | ## ref: https://github.com/docker-library/docs/blob/master/postgres/content.md#postgres_initdb_args 37 | ## ref: https://www.postgresql.org/docs/current/static/app-initdb.html 38 | # postgresInitdbArgs: 39 | 40 | ## Persist data to a persitent volume 41 | persistence: 42 | enabled: true 43 | 44 | ## A manually managed Persistent Volume and Claim 45 | ## Requires persistence.enabled: true 46 | ## If defined, PVC must be created manually before volume will be bound 47 | # existingClaim: 48 | 49 | ## If defined, volume.beta.kubernetes.io/storage-class: 50 | ## Default: volume.alpha.kubernetes.io/storage-class: default 51 | ## 52 | # storageClass: 53 | accessMode: ReadWriteOnce 54 | size: 8Gi 55 | subPath: "app-db" 56 | 57 | metrics: 58 | enabled: false 59 | image: wrouesnel/postgres_exporter 60 | imageTag: v0.1.1 61 | imagePullPolicy: IfNotPresent 62 | resources: 63 | requests: 64 | memory: 256Mi 65 | cpu: 100m 66 | 67 | ## Configure resource requests and limits 68 | ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ 69 | ## 70 | resources: 71 | requests: 72 | memory: 256Mi 73 | cpu: 100m 74 | 75 | service: 76 | port: 5432 77 | externalIPs: [] 78 | -------------------------------------------------------------------------------- /kubernetes/minikube/queue-consumer-job.yml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1 2 | kind: Job 3 | metadata: 4 | name: job-task-queue-consumer-worker-2 5 | spec: 6 | parallelism: 2 7 | template: 8 | metadata: 9 | name: job-task-queue-consumer-worker-2 10 | spec: 11 | volumes: 12 | - name: video-upload-minikube-pv-volume 13 | persistentVolumeClaim: 14 | claimName: video-upload-minikube-pv-volume-claim 15 | 16 | containers: 17 | - name: c 18 | image: n1207n/video-transcode-queue/task_queue/consumer:dev 19 | imagePullPolicy: Never 20 | 21 | volumeMounts: 22 | - mountPath: "/data/video_uploads/" 23 | name: video-upload-minikube-pv-volume 24 | 25 | env: 26 | - name: GET_HOSTS_FROM 27 | value: dns 28 | - name: REDIS_URL 29 | value: "queue-storage-redis" 30 | - name: REDIS_PORT 31 | value: "6379" 32 | - name: REDIS_TOPIC 33 | valueFrom: 34 | secretKeyRef: 35 | name: redis-queue-info 36 | key: queue-topic 37 | - name: REDIS_PASSWORD 38 | valueFrom: 39 | secretKeyRef: 40 | name: queue-storage-redis 41 | key: redis-password 42 | - name: UPLOAD_FOLDER_PATH 43 | value: /data/video_uploads/ 44 | 45 | restartPolicy: OnFailure 46 | -------------------------------------------------------------------------------- /kubernetes/minikube/secrets/redis-queue-info.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | 4 | metadata: 5 | name: redis-queue-info 6 | 7 | type: Opaque 8 | 9 | data: 10 | queue-topic: dHJhbnNjb2RlX3ZpZGVvCg== 11 | -------------------------------------------------------------------------------- /kubernetes/minikube/streaming-api-deployment.yml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | 3 | kind: Deployment 4 | 5 | metadata: 6 | name: streaming-api 7 | labels: 8 | name: streaming-api 9 | tier: backend 10 | 11 | spec: 12 | replicas: 1 13 | 14 | template: 15 | metadata: 16 | labels: 17 | name: streaming-api 18 | tier: backend 19 | 20 | spec: 21 | volumes: 22 | - name: video-upload-minikube-pv-volume 23 | persistentVolumeClaim: 24 | claimName: video-upload-minikube-pv-volume-claim 25 | 26 | containers: 27 | - name: streaming-api 28 | image: n1207n/video-transcode-queue/streaming_service:dev 29 | imagePullPolicy: Never 30 | 31 | volumeMounts: 32 | - mountPath: "/data/video_uploads/" 33 | name: video-upload-minikube-pv-volume 34 | 35 | command: ["ash"] 36 | args: ["-c", "/go/bin/streaming"] 37 | 38 | ports: 39 | - containerPort: 8880 40 | name: streaming-api 41 | 42 | env: 43 | - name: GET_HOSTS_FROM 44 | value: dns 45 | 46 | - name: PGDB 47 | value: video_app 48 | - name: PGUSER 49 | value: video_app 50 | - name: PGPASSWORD 51 | valueFrom: 52 | secretKeyRef: 53 | name: app-database-postgresql 54 | key: postgres-password 55 | - name: PGHOST 56 | value: app-database-postgresql:5432 57 | - name: UPLOAD_FOLDER_PATH 58 | value: /data/video_uploads/ 59 | -------------------------------------------------------------------------------- /kubernetes/minikube/streaming-api-service.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | 4 | metadata: 5 | name: streaming-api 6 | labels: 7 | name: streaming-api 8 | tier: backend 9 | 10 | spec: 11 | ports: 12 | - port: 8880 13 | targetPort: 8880 14 | protocol: TCP 15 | selector: 16 | name: streaming-api 17 | tier: backend 18 | -------------------------------------------------------------------------------- /kubernetes/minikube/transcoder-api-deployment.yml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | 3 | kind: Deployment 4 | 5 | metadata: 6 | name: transcoder-api 7 | labels: 8 | name: transcoder-api 9 | tier: backend 10 | 11 | spec: 12 | replicas: 1 13 | 14 | template: 15 | metadata: 16 | labels: 17 | name: transcoder-api 18 | tier: backend 19 | 20 | spec: 21 | volumes: 22 | - name: video-upload-minikube-pv-volume 23 | persistentVolumeClaim: 24 | claimName: video-upload-minikube-pv-volume-claim 25 | 26 | containers: 27 | - name: transcoder-api 28 | image: n1207n/video-transcode-queue/transcoder_service:dev 29 | imagePullPolicy: Never 30 | 31 | volumeMounts: 32 | - mountPath: "/data/video_uploads/" 33 | name: video-upload-minikube-pv-volume 34 | 35 | command: ["ash"] 36 | args: ["-c", "/go/bin/transcode"] 37 | 38 | ports: 39 | - containerPort: 8800 40 | name: transcoder-api 41 | 42 | env: 43 | - name: GET_HOSTS_FROM 44 | value: dns 45 | 46 | - name: PGDB 47 | value: video_app 48 | - name: PGUSER 49 | value: video_app 50 | - name: PGPASSWORD 51 | valueFrom: 52 | secretKeyRef: 53 | name: app-database-postgresql 54 | key: postgres-password 55 | - name: PGHOST 56 | value: app-database-postgresql:5432 57 | - name: UPLOAD_FOLDER_PATH 58 | value: /data/video_uploads/ 59 | -------------------------------------------------------------------------------- /kubernetes/minikube/transcoder-api-service.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | 4 | metadata: 5 | name: transcoder-api 6 | labels: 7 | name: transcoder-api 8 | tier: backend 9 | 10 | spec: 11 | ports: 12 | - port: 8800 13 | targetPort: 8800 14 | protocol: TCP 15 | selector: 16 | name: transcoder-api 17 | tier: backend 18 | -------------------------------------------------------------------------------- /kubernetes/minikube/video-api-deployment.yml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | 3 | kind: Deployment 4 | 5 | metadata: 6 | name: video-api 7 | labels: 8 | name: video-api 9 | tier: backend 10 | 11 | spec: 12 | replicas: 1 13 | 14 | template: 15 | metadata: 16 | labels: 17 | name: video-api 18 | tier: backend 19 | 20 | spec: 21 | volumes: 22 | - name: video-upload-minikube-pv-volume 23 | persistentVolumeClaim: 24 | claimName: video-upload-minikube-pv-volume-claim 25 | 26 | containers: 27 | - name: video-api 28 | image: n1207n/video-transcode-queue/video_backend_api:dev 29 | imagePullPolicy: Never 30 | 31 | volumeMounts: 32 | - mountPath: "/data/video_uploads/" 33 | name: video-upload-minikube-pv-volume 34 | 35 | command: ["ash"] 36 | args: ["-c", "/go/bin/backend"] 37 | 38 | ports: 39 | - containerPort: 8080 40 | name: video-api 41 | 42 | env: 43 | - name: GET_HOSTS_FROM 44 | value: dns 45 | - name: PGDB 46 | value: video_app 47 | - name: PGUSER 48 | value: video_app 49 | - name: PGPASSWORD 50 | valueFrom: 51 | secretKeyRef: 52 | name: app-database-postgresql 53 | key: postgres-password 54 | - name: PGHOST 55 | value: app-database-postgresql:5432 56 | - name: REDIS_URL 57 | value: "queue-storage-redis" 58 | - name: REDIS_PORT 59 | value: "6379" 60 | - name: REDIS_TOPIC 61 | valueFrom: 62 | secretKeyRef: 63 | name: redis-queue-info 64 | key: queue-topic 65 | - name: REDIS_PASSWORD 66 | valueFrom: 67 | secretKeyRef: 68 | name: queue-storage-redis 69 | key: redis-password 70 | - name: UPLOAD_FOLDER_PATH 71 | value: /data/video_uploads/ 72 | -------------------------------------------------------------------------------- /kubernetes/minikube/video-api-service.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | 4 | metadata: 5 | name: video-api 6 | labels: 7 | name: video-api 8 | tier: backend 9 | 10 | spec: 11 | ports: 12 | - port: 8080 13 | targetPort: 8080 14 | protocol: TCP 15 | selector: 16 | name: video-api 17 | tier: backend 18 | -------------------------------------------------------------------------------- /kubernetes/minikube/video-upload-minikube-persistent-volume-claim.yml: -------------------------------------------------------------------------------- 1 | kind: PersistentVolumeClaim 2 | apiVersion: v1 3 | 4 | metadata: 5 | name: video-upload-minikube-pv-volume-claim 6 | 7 | spec: 8 | storageClassName: standard 9 | accessModes: 10 | - ReadWriteOnce 11 | resources: 12 | requests: 13 | storage: 3Gi 14 | volumeName: video-upload-minikube-pv-volume 15 | -------------------------------------------------------------------------------- /kubernetes/minikube/video-upload-minikube-persistent-volume.yml: -------------------------------------------------------------------------------- 1 | kind: PersistentVolume 2 | apiVersion: v1 3 | 4 | metadata: 5 | name: video-upload-minikube-pv-volume 6 | labels: 7 | type: local 8 | 9 | spec: 10 | storageClassName: standard 11 | capacity: 12 | storage: 10Gi 13 | accessModes: 14 | - ReadWriteOnce 15 | hostPath: 16 | path: "/data/video_uploads" 17 | -------------------------------------------------------------------------------- /task_queue/client/Dockerfile-consumer: -------------------------------------------------------------------------------- 1 | FROM golang:1.8.3-alpine 2 | 3 | ENV GOBIN=/go/bin 4 | 5 | RUN apk update && apk upgrade && \ 6 | apk add --no-cache git openssh 7 | 8 | RUN go get -u github.com/adjust/rmq 9 | RUN go get -u github.com/golang/glog 10 | RUN go get -u gopkg.in/redis.v3 11 | 12 | ADD . /go/src/github.com/n1207n/video-transcode-queue/task_queue 13 | 14 | WORKDIR /go/src/github.com/n1207n/video-transcode-queue/task_queue 15 | 16 | RUN go build 17 | 18 | RUN go install 19 | 20 | ENTRYPOINT /go/bin/task_queue 21 | 22 | EXPOSE 6379 23 | -------------------------------------------------------------------------------- /task_queue/client/consumer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "os" 10 | "time" 11 | 12 | "github.com/adjust/rmq" 13 | "github.com/golang/glog" 14 | "gopkg.in/redis.v3" 15 | ) 16 | 17 | var ( 18 | redisURL, redisPort, redisPassword, redisTopic string 19 | redisProtocol = "tcp" 20 | redisNetworkTag = "transcode_task_consume" 21 | transcodeServiceHost, transcodeServicePort string 22 | 23 | // TODO: Make below variables as a CLI argument 24 | queueFetchInterval = 10 25 | queueFetchIntervalMeasurement = time.Second 26 | ) 27 | 28 | func main() { 29 | loadEnvironmentVariables() 30 | 31 | taskQueue := openTaskQueue() 32 | glog.Infof("Queue accessed: %s\n", redisTopic) 33 | 34 | taskQueue.StartConsuming(queueFetchInterval, queueFetchIntervalMeasurement) 35 | glog.Infoln("Queue consumption started...") 36 | 37 | taskConsumer := &TaskConsumer{} 38 | taskQueue.AddConsumer("Task consumer 1", taskConsumer) 39 | 40 | select {} 41 | } 42 | 43 | // TaskConsumer represents the Redis topic consumer 44 | type TaskConsumer struct { 45 | name string 46 | count int 47 | lastAccessed time.Time 48 | } 49 | 50 | // TranscodeRequest represents a JSON POST data for video-transcode API 51 | type TranscodeRequest struct { 52 | Path string `json:"path"` 53 | VideoID string `json:"video_id"` 54 | } 55 | 56 | // Consume method implements TaskConsumer struct 57 | // to be registered on Queue. 58 | // It handles actual data handling from Queue 59 | func (tc *TaskConsumer) Consume(delivery rmq.Delivery) { 60 | var task Task 61 | 62 | tc.count++ 63 | 64 | if err := json.Unmarshal([]byte(delivery.Payload()), &task); err != nil { 65 | glog.Errorf("Failed to read task message: %s\n", err) 66 | delivery.Reject() 67 | return 68 | } 69 | 70 | glog.Infof("Processed task message: Transcoding %s\n", task.FilePath) 71 | 72 | // TODO: Call Go subroutine to call go binding of ffmpeg 73 | transcodeRequest := &TranscodeRequest{ 74 | Path: task.FilePath, 75 | VideoID: task.ID, 76 | } 77 | 78 | b := new(bytes.Buffer) 79 | json.NewEncoder(b).Encode(transcodeRequest) 80 | 81 | url := fmt.Sprintf("http://%s:%s/api/v1/video-transcode", transcodeServiceHost, transcodeServicePort) 82 | 83 | request, err := http.NewRequest("POST", url, b) 84 | if err != nil { 85 | glog.Warningf("Failed to trigger transcode API: %s\n", err) 86 | delivery.Reject() 87 | return 88 | } 89 | 90 | client := &http.Client{} 91 | response, err := client.Do(request) 92 | if err != nil { 93 | glog.Warningf("Unsuccessful transcode request: %s\n", err) 94 | delivery.Reject() 95 | return 96 | } 97 | 98 | responseBuffer := new(bytes.Buffer) 99 | io.Copy(responseBuffer, response.Body) 100 | 101 | glog.Infof("Successful transcode request: %s\n", responseBuffer) 102 | delivery.Ack() 103 | } 104 | 105 | // loadEnvironmentVariables loads Redis 106 | // information from environment variables 107 | func loadEnvironmentVariables() { 108 | redisURL = os.Getenv("REDIS_URL") 109 | if len(redisURL) == 0 { 110 | panic("No REDIS_URL environment variable") 111 | } 112 | 113 | redisPort = os.Getenv("REDIS_PORT") 114 | if len(redisPort) == 0 { 115 | panic("No REDIS_PORT environment variable") 116 | } 117 | 118 | redisTopic = os.Getenv("REDIS_TOPIC") 119 | if len(redisTopic) == 0 { 120 | panic("No REDIS_TOPIC environment variable") 121 | } 122 | 123 | redisPassword = os.Getenv("REDIS_PASSWORD") 124 | if len(redisPassword) == 0 { 125 | panic("No REDIS_PASSWORD environment variable") 126 | } 127 | 128 | redisTopic = os.Getenv("REDIS_TOPIC") 129 | if len(redisTopic) == 0 { 130 | panic("No REDIS_TOPIC environment variable") 131 | } 132 | 133 | transcodeServiceHost = os.Getenv("TRANSCODER_API_SERVICE_HOST") 134 | if len(transcodeServiceHost) == 0 { 135 | panic("No TRANSCODER_API_SERVICE_HOST environment variable") 136 | } 137 | 138 | transcodeServicePort = os.Getenv("TRANSCODER_API_SERVICE_PORT") 139 | if len(transcodeServicePort) == 0 { 140 | panic("No TRANSCODER_API_SERVICE_PORT environment variable") 141 | } 142 | } 143 | 144 | // openTaskQueue connects to redis and return a Queue interface 145 | func openTaskQueue() rmq.Queue { 146 | redisClient := redis.NewClient(&redis.Options{ 147 | Network: redisProtocol, 148 | Addr: fmt.Sprintf("%s:%s", redisURL, redisPort), 149 | DB: int64(1), 150 | Password: redisPassword, 151 | }) 152 | 153 | connection := rmq.OpenConnectionWithRedisClient(redisNetworkTag, redisClient) 154 | 155 | glog.Infof("Connected to Redis task queue: %s\n", connection.Name) 156 | 157 | return connection.OpenQueue(redisTopic) 158 | } 159 | -------------------------------------------------------------------------------- /task_queue/client/task.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "time" 4 | 5 | // Task represents a job task to transcode a video file 6 | type Task struct { 7 | ID string 8 | FilePath string 9 | Timestamp time.Time 10 | } 11 | -------------------------------------------------------------------------------- /task_queue/helm/queue-storage-config.yml: -------------------------------------------------------------------------------- 1 | ## Usage: helm install -f queue_storage_config.yml --name=queue-storage stable/redis 2 | 3 | ## NOTES: Redis can be accessed via port 6379 on the following DNS name from within your cluster: 4 | ## 5 | ## queue-storage-redis 6 | 7 | ## To get your password run: 8 | 9 | ## REDIS_PASSWORD=$(kubectl get secret queue-storage-redis -o jsonpath="{.data.redis-password}" | base64 --decode) 10 | 11 | ## To connect to your Redis server: 12 | 13 | ## 1. Run a Redis pod that you can use as a client: 14 | 15 | ## kubectl run queue-storage-redis-client --rm --tty -i --env REDIS_PASSWORD=$REDIS_PASSWORD --image bitnami/redis:3.2.9-r0 -- bash 16 | 17 | ## 2. Connect using the Redis CLI: 18 | 19 | ## redis-cli -h queue-storage-redis -a $REDIS_PASSWORD 20 | 21 | ## 22 | ## Bitnami Redis image version 23 | ## ref: https://hub.docker.com/r/bitnami/redis/tags/ 24 | ## 25 | image: bitnami/redis:3.2.9-r0 26 | 27 | ## Specify a imagePullPolicyå 28 | ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images 29 | ## 30 | imagePullPolicy: IfNotPresent 31 | 32 | ## Redis password 33 | ## Defaults to a random 10-character alphanumeric string if not set 34 | ## ref: https://github.com/bitnami/bitnami-docker-redis#setting-the-server-password-on-first-run 35 | ## 36 | redisPassword: xtCWzoNXsC 37 | 38 | ## Enable persistence using Persistent Volume Claims 39 | ## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ 40 | ## 41 | persistence: 42 | enabled: true 43 | 44 | ## A manually managed Persistent Volume and Claim 45 | ## Requires persistence.enabled: true 46 | ## If defined, PVC must be created manually before volume will be bound 47 | # existingClaim: 48 | 49 | ## If defined, volume.beta.kubernetes.io/storage-class: 50 | ## Default: volume.alpha.kubernetes.io/storage-class: default 51 | ## 52 | # storageClass: 53 | accessMode: ReadWriteOnce 54 | size: 8Gi 55 | 56 | ## Configure resource requests and limits 57 | ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ 58 | ## 59 | resources: 60 | requests: 61 | memory: 256Mi 62 | cpu: 100m 63 | --------------------------------------------------------------------------------