├── .gitignore ├── README.md ├── build-centos7.sh ├── fabfile.py ├── opt └── openresty │ ├── etc │ ├── cloudflare.conf │ ├── common_config.conf │ ├── mime.types │ ├── nginx.conf │ └── variables.conf │ └── nginx │ └── lua │ └── fix_chunked_content_length.lua └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .coverage.* 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | *,cover 45 | 46 | # Translations 47 | *.mo 48 | *.pot 49 | 50 | # Django stuff: 51 | *.log 52 | 53 | # Sphinx documentation 54 | docs/_build/ 55 | 56 | # PyBuilder 57 | target/ 58 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HLS/DASH MP4 Proxy Server 2 | ## Basic usage 3 | ####Request a Master Playlist 4 | ``` 5 | GET http://video.streamboat.tv/hls/master.m3u8?source=http://example.com/video.mp4 6 | ``` 7 | ``` 8 | #EXTM3U 9 | #EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=1283488,RESOLUTION=640x360,CODECS="avc1.42c01e,mp4a.40.2" 10 | http://video.streamboat.tv/hls/index-v1-a1.m3u8?source=http://example.com/video.mp4 11 | ``` 12 | This video only has a single variant. 13 | ``` 14 | GET http://video.streamboat.tv/hls/index-v1-a1.m3u8?source=http://example.com/video.mp4 15 | ``` 16 | ``` 17 | #EXTM3U 18 | #EXT-X-TARGETDURATION:10 19 | #EXT-X-ALLOW-CACHE:YES 20 | #EXT-X-PLAYLIST-TYPE:VOD 21 | #EXT-X-VERSION:3 22 | #EXT-X-MEDIA-SEQUENCE:1 23 | #EXTINF:10.000, 24 | http://video.streamboat.tv/hls/seg-1-v1-a1.ts?source=http://example.com/video.mp4 25 | #EXTINF:10.000, 26 | http://video.streamboat.tv/hls/seg-2-v1-a1.ts?source=http://example.com/video.mp4 27 | #EXTINF:10.000, 28 | http://video.streamboat.tv/hls/seg-3-v1-a1.ts?source=http://example.com/video.mp4 29 | #EXTINF:10.000, 30 | http://video.streamboat.tv/hls/seg-4-v1-a1.ts?source=http://example.com/video.mp4 31 | #EXTINF:4.821, 32 | http://video.streamboat.tv/hls/seg-5-v1-a1.ts?source=http://example.com/video.mp4 33 | #EXT-X-ENDLIST 34 | ``` 35 | -------------------------------------------------------------------------------- /build-centos7.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # 4 | # This script bootstraps a new CentOS 7 server for video serving 5 | yum check-update 6 | yum update -y 7 | yum install -y epel-release 8 | yum groupinstall -y "Development Tools" 9 | yum install -y cmake lua pcre-devel openssl-devel \ 10 | gd-devel curl-devel nasm iptables-services 11 | 12 | 13 | # OpenResty 14 | groupadd -r openresty 15 | useradd -r -g openresty -s /sbin/nologin -d /opt/openresty -c "openresty user" openresty 16 | cd /usr/src 17 | wget -O- https://github.com/kaltura/nginx-vod-module/archive/1.2.tar.gz | tar zxv 18 | wget -O- https://github.com/yaoweibin/ngx_http_substitutions_filter_module/archive/v0.6.4.tar.gz | tar zxv 19 | wget -O- https://openresty.org/download/ngx_openresty-1.9.3.1.tar.gz | tar zxv 20 | cd ngx_openresty-1.9.3.1 21 | ./configure --prefix=/opt/openresty \ 22 | --sbin-path=/opt/openresty/sbin/nginx \ 23 | --conf-path=/opt/openresty/etc/nginx.conf \ 24 | --error-log-path=/opt/openresty/log/error.log \ 25 | --http-log-path=/opt/openresty/log/access.log \ 26 | --pid-path=/opt/openresty/run/nginx.pid \ 27 | --lock-path=/opt/openresty/run/nginx.lock \ 28 | --http-client-body-temp-path=/opt/openresty/cache/client_temp \ 29 | --http-proxy-temp-path=/opt/openresty/cache/proxy_temp \ 30 | --http-fastcgi-temp-path=/opt/openresty/cache/fastcgi_temp \ 31 | --http-uwsgi-temp-path=/opt/openresty/cache/uwsgi_temp \ 32 | --http-scgi-temp-path=/opt/openresty/cache/scgi_temp \ 33 | --user=openresty \ 34 | --group=openresty \ 35 | --with-http_addition_module \ 36 | --with-http_ssl_module \ 37 | --with-http_realip_module \ 38 | --with-http_addition_module \ 39 | --with-http_sub_module \ 40 | --with-http_dav_module \ 41 | --with-http_flv_module \ 42 | --with-http_mp4_module \ 43 | --with-http_gunzip_module \ 44 | --with-http_gzip_static_module \ 45 | --with-http_random_index_module \ 46 | --with-http_secure_link_module \ 47 | --with-http_stub_status_module \ 48 | --with-http_auth_request_module \ 49 | --with-http_image_filter_module \ 50 | --with-pcre-jit \ 51 | --with-file-aio \ 52 | --with-ipv6 \ 53 | --with-http_spdy_module \ 54 | --with-luajit \ 55 | --with-lua51=/usr \ 56 | --with-cc-opt='-O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -m64 -mtune=generic' \ 57 | -j2 \ 58 | --add-module=../nginx-vod-module-1.2 \ 59 | --add-module=../ngx_http_substitutions_filter_module-0.6.4 60 | make -j2 61 | make install 62 | mkdir /opt/openresty/cache 63 | 64 | # OpenResty service 65 | cat << "EOF" > /usr/lib/systemd/system/openresty.service 66 | [Unit] 67 | Description=openresty-nginx - high performance web application server 68 | Documentation=http://openresty.org 69 | After=network.target remote-fs.target nss-lookup.target 70 | 71 | [Service] 72 | Type=forking 73 | PIDFile=/opt/openresty/run/nginx.pid 74 | ExecStartPre=/opt/openresty/sbin/nginx -t -c /opt/openresty/etc/nginx.conf 75 | ExecStart=/opt/openresty/sbin/nginx -c /opt/openresty/etc/nginx.conf 76 | ExecReload=/bin/kill -s HUP $MAINPID 77 | ExecStop=/bin/kill -s QUIT $MAINPID 78 | PrivateTmp=true 79 | 80 | [Install] 81 | WantedBy=multi-user.target 82 | EOF 83 | systemctl enable openresty.service 84 | 85 | # Logs 86 | cat <<- "EOF" > /etc/logrotate.d/openresty 87 | /opt/openresty/log/*.log { 88 | daily 89 | missingok 90 | rotate 366 91 | compress 92 | delaycompress 93 | notifempty 94 | create 640 openresty adm 95 | sharedscripts 96 | postrotate 97 | [ -f /opt/openresty/run/nginx.pid ] && kill -USR1 `cat /opt/openresty/run/nginx.pid` 98 | endscript 99 | } 100 | EOF 101 | 102 | # Firewall 103 | cat << "EOF" > /etc/sysconfig/iptables 104 | *filter 105 | :INPUT ACCEPT [0:0] 106 | :FORWARD ACCEPT [0:0] 107 | :OUTPUT ACCEPT [0:0] 108 | -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT 109 | -A INPUT -p icmp -j ACCEPT 110 | -A INPUT -i lo -j ACCEPT 111 | -A INPUT -m state --state NEW -m tcp -p tcp --dport 22 -j ACCEPT -m comment --comment "ssh" 112 | -A INPUT -m state --state NEW -m tcp -p tcp --dport 80 -j ACCEPT -m comment --comment "http" 113 | -A INPUT -m state --state NEW -m tcp -p tcp --dport 443 -j ACCEPT -m comment --comment "https" 114 | -A INPUT -j REJECT --reject-with icmp-host-prohibited 115 | -A FORWARD -j REJECT --reject-with icmp-host-prohibited 116 | COMMIT 117 | EOF 118 | 119 | systemctl stop firewalld 120 | systemctl mask firewalld 121 | systemctl enable iptables 122 | systemctl restart iptables 123 | 124 | -------------------------------------------------------------------------------- /fabfile.py: -------------------------------------------------------------------------------- 1 | from os.path import dirname, abspath, join as pathjoin 2 | from fabric.contrib.project import rsync_project 3 | from fabric.api import env, run 4 | 5 | env.use_ssh_config = True 6 | env.appdir = abspath(dirname(__file__)) 7 | env.confdir = pathjoin(env.appdir, "opt/openresty") 8 | 9 | 10 | def sync(): 11 | srcdir = env.confdir + "/" 12 | dstdir = "/opt/openresty/" 13 | rsync_project(local_dir=srcdir, remote_dir=dstdir) 14 | run("systemctl restart openresty.service") 15 | -------------------------------------------------------------------------------- /opt/openresty/etc/cloudflare.conf: -------------------------------------------------------------------------------- 1 | set_real_ip_from 199.27.128.0/21; 2 | set_real_ip_from 173.245.48.0/20; 3 | set_real_ip_from 103.21.244.0/22; 4 | set_real_ip_from 103.22.200.0/22; 5 | set_real_ip_from 103.31.4.0/22; 6 | set_real_ip_from 141.101.64.0/18; 7 | set_real_ip_from 108.162.192.0/18; 8 | set_real_ip_from 190.93.240.0/20; 9 | set_real_ip_from 188.114.96.0/20; 10 | set_real_ip_from 197.234.240.0/22; 11 | set_real_ip_from 198.41.128.0/17; 12 | set_real_ip_from 162.158.0.0/15; 13 | set_real_ip_from 104.16.0.0/12; 14 | set_real_ip_from 172.64.0.0/13; 15 | set_real_ip_from 2400:cb00::/32; 16 | set_real_ip_from 2606:4700::/32; 17 | set_real_ip_from 2803:f800::/32; 18 | set_real_ip_from 2405:b500::/32; 19 | set_real_ip_from 2405:8100::/32; 20 | real_ip_header CF-Connecting-IP; -------------------------------------------------------------------------------- /opt/openresty/etc/common_config.conf: -------------------------------------------------------------------------------- 1 | client_header_timeout 3m; 2 | client_body_timeout 3m; 3 | send_timeout 3m; 4 | 5 | connection_pool_size 256; 6 | request_pool_size 4k; 7 | 8 | client_header_buffer_size 1k; 9 | large_client_header_buffers 4 4k; 10 | 11 | output_buffers 1 32k; 12 | postpone_output 1460; 13 | 14 | sendfile on; 15 | sendfile_max_chunk 128k; 16 | tcp_nopush on; 17 | tcp_nodelay on; 18 | 19 | keepalive_timeout 75 20; 20 | 21 | lingering_time 30; 22 | lingering_timeout 10; 23 | reset_timedout_connection on; 24 | 25 | gzip_min_length 256; 26 | gzip_proxied any; 27 | gzip_vary on; 28 | gzip_buffers 4 8k; 29 | gzip_proxied expired no-cache no-store private auth; 30 | 31 | gzip_types application/atom+xml application/javascript application/json 32 | application/ld+json application/manifest+json application/rdf+xml 33 | application/rss+xml application/schema+json application/vnd.geo+json 34 | application/vnd.ms-fontobject application/x-font-ttf application/x-javascript 35 | application/x-web-app-manifest+json application/xhtml+xml application/xml 36 | font/eot font/opentype image/bmp image/svg+xml image/vnd.microsoft.icon 37 | image/x-icon text/cache-manifest text/css text/javascript text/plain 38 | text/vcard text/vnd.rim.location.xloc text/vtt text/x-component 39 | text/x-cross-domain-policy text/xml; 40 | 41 | proxy_set_header Host $host; 42 | proxy_set_header X-Real-IP $remote_addr; 43 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 44 | 45 | open_file_cache max=1000 inactive=20s; 46 | open_file_cache_valid 30s; 47 | open_file_cache_min_uses 2; 48 | open_file_cache_errors on; 49 | 50 | client_max_body_size 4096m; 51 | client_body_buffer_size 128k; 52 | 53 | proxy_connect_timeout 90; 54 | proxy_send_timeout 90; 55 | proxy_read_timeout 90; 56 | 57 | proxy_buffer_size 4k; 58 | proxy_buffers 4 32k; 59 | proxy_busy_buffers_size 64k; 60 | proxy_temp_file_write_size 64k; 61 | proxy_ignore_client_abort on; 62 | 63 | resolver 8.8.8.8 8.8.4.4 valid=30s; -------------------------------------------------------------------------------- /opt/openresty/etc/mime.types: -------------------------------------------------------------------------------- 1 | types { 2 | # Borrowed (and slightly fixed) from good people at https://github.com/h5bp/server-configs-nginx 3 | # Data interchange 4 | 5 | application/atom+xml atom; 6 | application/json json map topojson; 7 | application/ld+json jsonld; 8 | application/rss+xml rss; 9 | application/vnd.geo+json geojson; 10 | application/xml rdf xml; 11 | 12 | 13 | # JavaScript 14 | 15 | # Normalize to standard type. 16 | # https://tools.ietf.org/html/rfc4329#section-7.2 17 | application/javascript js; 18 | 19 | 20 | # Manifest files 21 | 22 | application/x-web-app-manifest+json webapp; 23 | text/cache-manifest appcache; 24 | 25 | 26 | # Media files 27 | 28 | audio/midi mid midi kar; 29 | audio/mp4 aac f4a f4b m4a; 30 | audio/mpeg mp3; 31 | audio/ogg oga ogg opus; 32 | audio/x-realaudio ra; 33 | audio/x-wav wav; 34 | image/bmp bmp; 35 | image/gif gif; 36 | image/jpeg jpeg jpg; 37 | image/png png; 38 | image/svg+xml svg svgz; 39 | image/tiff tif tiff; 40 | image/vnd.wap.wbmp wbmp; 41 | image/webp webp; 42 | image/x-jng jng; 43 | video/3gpp 3gpp 3gp; 44 | video/mp4 f4v f4p m4v mp4; 45 | video/mpeg mpeg mpg; 46 | video/ogg ogv; 47 | video/quicktime mov; 48 | video/webm webm; 49 | video/x-flv flv; 50 | video/x-mng mng; 51 | video/x-ms-asf asx asf; 52 | video/x-ms-wmv wmv; 53 | video/x-msvideo avi; 54 | 55 | # Serving `.ico` image files with a different media type 56 | # prevents Internet Explorer from displaying then as images: 57 | # https://github.com/h5bp/html5-boilerplate/commit/37b5fec090d00f38de64b591bcddcb205aadf8ee 58 | 59 | image/x-icon cur ico; 60 | 61 | 62 | # Microsoft Office 63 | 64 | application/msword doc; 65 | application/vnd.ms-excel xls; 66 | application/vnd.ms-powerpoint ppt; 67 | application/vnd.openxmlformats-officedocument.wordprocessingml.document docx; 68 | application/vnd.openxmlformats-officedocument.spreadsheetml.sheet xlsx; 69 | application/vnd.openxmlformats-officedocument.presentationml.presentation pptx; 70 | 71 | 72 | # Web fonts 73 | 74 | application/font-woff woff; 75 | application/font-woff2 woff2; 76 | application/vnd.ms-fontobject eot; 77 | 78 | # Browsers usually ignore the font media types and simply sniff 79 | # the bytes to figure out the font type. 80 | # https://mimesniff.spec.whatwg.org/#matching-a-font-type-pattern 81 | # 82 | # However, Blink and WebKit based browsers will show a warning 83 | # in the console if the following font types are served with any 84 | # other media types. 85 | 86 | application/x-font-ttf ttc ttf; 87 | font/opentype otf; 88 | 89 | 90 | # Other 91 | 92 | application/java-archive jar war ear; 93 | application/mac-binhex40 hqx; 94 | application/octet-stream bin deb dll dmg exe img iso msi msm msp safariextz; 95 | application/pdf pdf; 96 | application/postscript ps eps ai; 97 | application/rtf rtf; 98 | application/vnd.google-earth.kml+xml kml; 99 | application/vnd.google-earth.kmz kmz; 100 | application/vnd.wap.wmlc wmlc; 101 | application/x-7z-compressed 7z; 102 | application/x-bb-appworld bbaw; 103 | application/x-bittorrent torrent; 104 | application/x-chrome-extension crx; 105 | application/x-cocoa cco; 106 | application/x-java-archive-diff jardiff; 107 | application/x-java-jnlp-file jnlp; 108 | application/x-makeself run; 109 | application/x-opera-extension oex; 110 | application/x-perl pl pm; 111 | application/x-pilot prc pdb; 112 | application/x-rar-compressed rar; 113 | application/x-redhat-package-manager rpm; 114 | application/x-sea sea; 115 | application/x-shockwave-flash swf; 116 | application/x-stuffit sit; 117 | application/x-tcl tcl tk; 118 | application/x-x509-ca-cert der pem crt; 119 | application/x-xpinstall xpi; 120 | application/xhtml+xml xhtml; 121 | application/xslt+xml xsl; 122 | application/zip zip; 123 | text/css css; 124 | text/html html htm shtml; 125 | text/mathml mml; 126 | text/plain txt; 127 | text/vcard vcard vcf; 128 | text/vnd.rim.location.xloc xloc; 129 | text/vnd.sun.j2me.app-descriptor jad; 130 | text/vnd.wap.wml wml; 131 | text/vtt vtt; 132 | text/x-component htc; 133 | 134 | } -------------------------------------------------------------------------------- /opt/openresty/etc/nginx.conf: -------------------------------------------------------------------------------- 1 | user openresty; 2 | worker_processes auto; 3 | worker_rlimit_nofile 300000; 4 | pcre_jit on; 5 | 6 | error_log /opt/openresty/log/error.log info; 7 | pid /opt/openresty/run/nginx.pid; 8 | 9 | events { 10 | worker_connections 8192; 11 | use epoll; 12 | multi_accept on; 13 | } 14 | 15 | http { 16 | include mime.types; 17 | default_type application/octet-stream; 18 | lua_package_path '/opt/openresty/etc/lua/?.lua;;/opt/openresty/nginx/lua/?.lua;;'; 19 | 20 | log_format main '$remote_addr - $remote_user [$time_local] "$request" ' 21 | '$status $body_bytes_sent "$http_referer" ' 22 | '"$http_user_agent" "$http_x_forwarded_for" "$http_x_sbimg_host" ' 23 | '"$http_x_forwarded_proto" "$http_cf_connecting_ip" $request_time'; 24 | 25 | access_log /opt/openresty/log/access.log main; 26 | 27 | # Control expiration based on mime type 28 | map $sent_http_content_type $cache_expires { 29 | default 0; 30 | 31 | application/vnd.apple.mpegurl "16d"; 32 | video/MP2T "16d"; 33 | } 34 | 35 | # Server wide optimizations 36 | include common_config.conf; 37 | 38 | # User variables 39 | include variables.conf; 40 | 41 | 42 | # Proxy config 43 | # Place file-cache on tmpfs for speed, we do not care if it is lost 44 | proxy_cache_path /dev/shm/nginx-cache-sbvid levels=1:2 keys_zone=SBVID:100m inactive=24h; 45 | proxy_cache_key "$scheme$request_method$host$request_uri"; 46 | 47 | # Use stale cache if upstream is in troubles 48 | proxy_cache_use_stale updating error timeout invalid_header http_500; 49 | 50 | # Prevent pile-up effect and control inflow of traffic to upstream 51 | proxy_cache_lock on; 52 | proxy_cache_lock_age 5s; 53 | proxy_cache_lock_timeout 3s; 54 | 55 | # Cache regardless of downstream controls 56 | proxy_ignore_headers X-Accel-Expires Cache-Control Expires Set-Cookie; 57 | 58 | # MP4 Proxy Upstream 59 | upstream mp4_proxy { 60 | server 127.0.0.1:8080; 61 | } 62 | 63 | server { 64 | listen 8080; 65 | 66 | location ~ /hlsify/(?https?):/+(?[^/]+)/(?.+) { 67 | 68 | if ($origin_host_allowed = 0) { 69 | return 403 "Origin not allowed"; 70 | } 71 | 72 | proxy_pass $origin_scheme://$origin_host/$origin_path; 73 | proxy_set_header Host $origin_host; 74 | proxy_set_header Cookie ""; 75 | proxy_set_header Authorization ""; 76 | 77 | # https://github.com/kaltura/nginx-vod-module/issues/131 78 | header_filter_by_lua_file lua/fix_chunked_content_length.lua; 79 | } 80 | } 81 | 82 | server { 83 | listen 80 default_server; 84 | server_name _; 85 | 86 | location ~ /hls/(?.+)\.m3u8$ { 87 | rewrite ^ /hlsify/$arg_source/${playlist}.m3u8?source=$arg_source last; 88 | } 89 | 90 | location ~ /hls/(?.+)\.ts$ { 91 | rewrite ^ /hlsify/$arg_source/${segment}.ts?source=$arg_source last; 92 | } 93 | 94 | location /hlsify { 95 | vod hls; 96 | vod_mode remote; 97 | vod_child_request_path /child; 98 | vod_upstream mp4_proxy; 99 | subs_filter_types application/vnd.apple.mpegurl; 100 | subs_filter "hlsify/.+/([^/\n]+)" "hls/$1?source=$arg_source" r; 101 | 102 | # Required for web players that request playlists/chunks via XHR 103 | add_header Access-Control-Allow-Origin "*"; 104 | } 105 | 106 | location /child { 107 | internal; 108 | vod_child_request; 109 | } 110 | } 111 | 112 | } -------------------------------------------------------------------------------- /opt/openresty/etc/variables.conf: -------------------------------------------------------------------------------- 1 | 2 | # This map controls what hosts are OK to proxy for, keep it tight ... 3 | map $origin_host $origin_host_allowed { 4 | hostnames; 5 | default 0; 6 | 7 | .streamable.com 1; 8 | } -------------------------------------------------------------------------------- /opt/openresty/nginx/lua/fix_chunked_content_length.lua: -------------------------------------------------------------------------------- 1 | -- https://github.com/kaltura/nginx-vod-module/issues/131 2 | if not ngx.header.content_length and ngx.header.content_range then 3 | local start_byte, end_byte = ngx.header.content_range:match("bytes (%d+)-(%d+)/%d+") 4 | if start_byte and end_byte then 5 | ngx.header.Content_Length = (end_byte - start_byte) + 1 6 | end 7 | end -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Fabric==1.10.2 2 | ecdsa==0.13 3 | paramiko==1.15.2 4 | pycrypto==2.6.1 5 | wsgiref==0.1.2 6 | --------------------------------------------------------------------------------