├── .gitignore
├── README.md
├── celeryconfig.py.dist
├── dirConfig.py.dist
├── queueFiles.py
└── videoTasks.py


/.gitignore:
--------------------------------------------------------------------------------
  1 | # Byte-compiled / optimized / DLL files
  2 | __pycache__/
  3 | *.py[cod]
  4 | *$py.class
  5 | 
  6 | # C extensions
  7 | *.so
  8 | 
  9 | # Distribution / packaging
 10 | .Python
 11 | env/
 12 | build/
 13 | develop-eggs/
 14 | dist/
 15 | downloads/
 16 | eggs/
 17 | .eggs/
 18 | lib/
 19 | lib64/
 20 | parts/
 21 | sdist/
 22 | var/
 23 | wheels/
 24 | *.egg-info/
 25 | .installed.cfg
 26 | *.egg
 27 | 
 28 | # PyInstaller
 29 | #  Usually these files are written by a python script from a template
 30 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 31 | *.manifest
 32 | *.spec
 33 | 
 34 | # Installer logs
 35 | pip-log.txt
 36 | pip-delete-this-directory.txt
 37 | 
 38 | # Unit test / coverage reports
 39 | htmlcov/
 40 | .tox/
 41 | .coverage
 42 | .coverage.*
 43 | .cache
 44 | nosetests.xml
 45 | coverage.xml
 46 | *.cover
 47 | .hypothesis/
 48 | 
 49 | # Translations
 50 | *.mo
 51 | *.pot
 52 | 
 53 | # Django stuff:
 54 | *.log
 55 | local_settings.py
 56 | 
 57 | # Flask stuff:
 58 | instance/
 59 | .webassets-cache
 60 | 
 61 | # Scrapy stuff:
 62 | .scrapy
 63 | 
 64 | # Sphinx documentation
 65 | docs/_build/
 66 | 
 67 | # PyBuilder
 68 | target/
 69 | 
 70 | # Jupyter Notebook
 71 | .ipynb_checkpoints
 72 | 
 73 | # pyenv
 74 | .python-version
 75 | 
 76 | # celery beat schedule file
 77 | celerybeat-schedule
 78 | 
 79 | # SageMath parsed files
 80 | *.sage.py
 81 | 
 82 | # dotenv
 83 | .env
 84 | 
 85 | # virtualenv
 86 | .venv
 87 | venv/
 88 | ENV/
 89 | 
 90 | # Spyder project settings
 91 | .spyderproject
 92 | .spyproject
 93 | 
 94 | # Rope project settings
 95 | .ropeproject
 96 | 
 97 | # mkdocs documentation
 98 | /site
 99 | 
100 | # mypy
101 | .mypy_cache/
102 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # python-videoencoder
 2 | A Python Celery application for queuing and processing video encoding jobs
 3 | 
 4 | ## Overview
 5 | 
 6 | `queueFiles.py` will scan a directory defined in `dirConfig.py` and move/queue all video files ready for processing. The destination directories are also defined in `dirConfig.py`.
 7 | 
 8 | `videoTasks.py` defines the encode task. This takes a source file (including full path) and a destination directory to output the encoded file to. It uses ffmpeg, and creates a mp4 h264 file according to the settings in the file. 
 9 | 
10 | 
11 | ## Making it go
12 | 
13 | Check out this repo to a system with python3. You'll need a RabbitMQ instance somewhere, and whichever machine runs the workers will need access to ffmpeg. 
14 | 
15 | Configure the RabbitMQ settings in `celeryconfig.py`.
16 | 
17 | One machine needs to run `queueFiles.py` to process and queue videos in a given directory. 
18 | 
19 | Start a celery worker either on the same machine, or on 1 or more other machines. You'll need celery installed using `pip install celery`.
20 | 
21 | If more than one machine is used, then there must be shared storage between them, e.g by using NFS, and the video folders mounted to identical paths.
22 | 
23 | Start a worker like so:
24 | ```
25 | cd /path/to/git/checkout
26 | celery worker -A videoTasks -l info -Ofair
27 | ```
28 | 
29 | The `-l` and `-O` arguments are optional, change them if you like.
30 | 
31 | The worker process should pick up any messages in the queue and begin processing.
32 | 


--------------------------------------------------------------------------------
/celeryconfig.py.dist:
--------------------------------------------------------------------------------
1 | broker_url = 'amqp://user:pass@rabbitserver//'
2 | worker_concurrency = 1


--------------------------------------------------------------------------------
/dirConfig.py.dist:
--------------------------------------------------------------------------------
1 | srcDir = "/tmp/to_encode/"
2 | encDir = "/tmp/encoding/"
3 | outDir = "/tmp/encoded/"
4 | 


--------------------------------------------------------------------------------
/queueFiles.py:
--------------------------------------------------------------------------------
 1 | import shutil
 2 | from os import listdir
 3 | from os.path  import isfile, join
 4 | from os import walk
 5 | import videoTasks
 6 | from dirConfig import srcDir, encDir, outDir
 7 | 
 8 | 
 9 | f = []
10 | for (dirpath, dirnames, filenames) in walk (srcDir):
11 | 	f.extend(filenames)
12 | 
13 | for file in f:
14 | 	shutil.move(srcDir+file, encDir+file)
15 | 	srcFile = encDir+file
16 | 	videoTasks.encode.delay(srcFile, outDir)


--------------------------------------------------------------------------------
/videoTasks.py:
--------------------------------------------------------------------------------
 1 | from celery import Celery
 2 | import celeryconfig
 3 | import os.path
 4 | from subprocess import call
 5 | 
 6 | app = Celery()
 7 | app.config_from_object('celeryconfig')
 8 | 
 9 | @app.task
10 | def encode(srcFile, dstFolder):
11 |     outFileName = dstFolder+os.path.splitext(os.path.basename(srcFile))[0]+".mkv"
12 |     argsStr =  " -i \"%s\" -max_muxing_queue_size 1000 -map 0 -c copy -c:v libx264 -crf 17 -y \"%s\" " % (srcFile, outFileName)
13 |     result = call("ffmpeg" + argsStr, shell=True)
14 |     return result
15 | 


--------------------------------------------------------------------------------