├── .gitignore ├── LICENSE ├── README.md ├── f4v.py ├── libmako.py ├── mako ├── mako_key_extractor ├── README.md ├── asset.asasm ├── extract.py ├── main.asasm └── object.asasm └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Itay Perl 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | mako-dl 2 | ======= 3 | 4 | Download episodes from the Mako VOD service 5 | 6 | Dependencies 7 | ------------ 8 | * libmms is a dependency for some streams (non-HDS streams). 9 | * Python dependencies can be installed by running `pip install -r requirements.txt`. 10 | 11 | Example usage 12 | ------------- 13 | 14 | ``` 15 | # List shows 16 | mako -l 17 | 18 | # List episodes of a show 19 | mako -l /show/url 20 | 21 | # Download season 1 of a show (see `mako --help` for the syntax of the `-s` option) 22 | mako -o DIRNAME -s 1: /show/url 23 | ``` 24 | 25 | TODO 26 | ---- 27 | * libmako is not script-friendly at all 28 | * XBMC plugin 29 | * The code requires some cleanup and documentation 30 | -------------------------------------------------------------------------------- /f4v.py: -------------------------------------------------------------------------------- 1 | from bs4 import BeautifulSoup 2 | import progressbar 3 | import multiprocessing.dummy as multiprocessing 4 | import argparse 5 | import requests 6 | import urlparse 7 | import httplib 8 | import struct 9 | import subprocess 10 | import itertools 11 | 12 | # Specs: 13 | # http://download.macromedia.com/f4v/video_file_format_spec_v10_1.pdf 14 | # http://sourceforge.net/apps/mediawiki/osmf.adobe/index.php?title=Flash_Media_Manifest_(F4M)_File_Format 15 | 16 | def get_xml_document(url, session): 17 | resp = session.get(url, params=session.params) 18 | resp.raise_for_status() 19 | return BeautifulSoup(resp.content, 'xml') 20 | 21 | def get_box_data(buf, name): 22 | box_offset = buf.find(name) - 4 23 | size, = struct.unpack('>L', buf[box_offset:box_offset+4]) 24 | header_size = 8 25 | if size == 1: 26 | # Extended box size 27 | size, = struct.unpack('>Q', buf[box_offset+8:box_offset+16]) 28 | header_size += 8 29 | 30 | return buf[box_offset + header_size:box_offset + size] 31 | 32 | def get_fragment_urls(manifest_url, session): 33 | manifest = get_xml_document(manifest_url, session) 34 | 35 | def fix_url(url): 36 | baseurl_tag = manifest.find('baseURL') 37 | base_url = baseurl_tag.string if baseurl_tag else manifest_url 38 | return urlparse.urljoin(base_url, url) 39 | 40 | # Get the stream with highest bitrate (likely the highest quality) 41 | media = max(manifest.find_all('media'), key=lambda tag: int(tag.get('bitrate', 0))) 42 | 43 | if 'href' in media.attrs: 44 | # Multi-level format (F4M 2.0) 45 | manifest_url = fix_url(media['href']) 46 | manifest = get_xml_document(manifest_url, session) 47 | # Assuming there is a single media element and that the second manifest 48 | # is not multi-level 49 | media = manifest.find('media') 50 | 51 | media_url = fix_url(media['url']) 52 | 53 | # get max fragment ID from bootstrap info 54 | bootstrap_info = manifest.find('bootstrapInfo', id=media['bootstrapInfoId']).string.decode('base64') 55 | 56 | asrt = get_box_data(bootstrap_info, 'asrt') 57 | _, QualityEntryCount, SegmentRunEntryCount, FirstSegment, FragmentsPerSegment = struct.unpack('>LBLLL', asrt) 58 | assert QualityEntryCount == 0 and SegmentRunEntryCount == 1 and FirstSegment == 1 59 | 60 | for frag_id in xrange(1, FragmentsPerSegment + 1): 61 | yield '%sSeg1-Frag%d' % (media_url, frag_id) 62 | 63 | def download_fragment(frag_url, session): 64 | status = None 65 | attempts = 3 66 | while status != httplib.OK and attempts > 0: 67 | resp = session.get(frag_url, params=session.params, stream=True) 68 | status = resp.status_code 69 | attempts -= 1 70 | 71 | resp.raise_for_status() 72 | 73 | fragment_data = bytearray() 74 | content_it = resp.iter_content(4096) 75 | for chunk in content_it: 76 | fragment_data.extend(chunk) 77 | mdat_pos = fragment_data.find('mdat') 78 | # Make sure the entire mdat header was read 79 | if 0 <= mdat_pos and mdat_pos + 12 < len(fragment_data): 80 | break 81 | 82 | box_offset = mdat_pos - 4 83 | size, = struct.unpack('>L', str(fragment_data[box_offset:box_offset+4])) 84 | header_size = 8 85 | if size == 1: 86 | # Extended box size 87 | size, = struct.unpack('>Q', str(fragment_data[box_offset+8:box_offset+16])) 88 | header_size += 8 89 | 90 | payload = fragment_data[box_offset + header_size:] 91 | size -= len(payload) 92 | yield payload 93 | 94 | for chunk in content_it: 95 | if size == 0: 96 | break 97 | payload = chunk[:size] 98 | size -= len(payload) 99 | yield payload 100 | 101 | def download(manifest_url, out_filename, reindex=True, session=None, parallel=20, progress=False): 102 | FLV_HEADER = '464c5601050000000900000000'.decode('hex') 103 | USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36' 104 | 105 | if session is None: 106 | session = requests.Session() 107 | if session.headers['User-Agent'] == requests.utils.default_user_agent(): 108 | session.headers['User-Agent'] = USER_AGENT 109 | 110 | if progress: 111 | pbar_widgets = ['Downloading: ', progressbar.Percentage(), ' ', progressbar.Bar(), 112 | ' ', progressbar.ETA(), ] 113 | else: 114 | pbar_widgets = [] 115 | 116 | stream_urls = list(get_fragment_urls(manifest_url, session)) 117 | pb = progressbar.ProgressBar(widgets=pbar_widgets, maxval=len(stream_urls)).start() 118 | 119 | with open(out_filename, 'w') as outfile: 120 | pool = multiprocessing.Pool(parallel) 121 | 122 | outfile.write(FLV_HEADER) 123 | for frag in pool.imap(lambda url: download_fragment(url, session), stream_urls): 124 | pb.update(pb.currval + 1) 125 | for chunk in frag: 126 | outfile.write(chunk) 127 | 128 | pb.finish() 129 | 130 | # The downloaded FLV is playable by itself, but will have extremely 131 | # slow seeking without reindexing. 132 | if reindex: 133 | subprocess.call(['index-flv', '-rU', out_filename]) 134 | 135 | 136 | def main(): 137 | parser = argparse.ArgumentParser() 138 | parser.add_argument('manifest_url') 139 | parser.add_argument('outfile') 140 | parser.add_argument('-t', '--ticket') 141 | parser.add_argument('-p', '--parallel', type=int, default=20, help='Number of parallel connections.') 142 | 143 | args = parser.parse_args() 144 | 145 | session = requests.Session() 146 | session.params = args.ticket 147 | 148 | download(args.manifest_url, args.outfile, session=session, parallel=args.parallel, progress=True) 149 | 150 | if __name__ == '__main__': 151 | main() 152 | -------------------------------------------------------------------------------- /libmako.py: -------------------------------------------------------------------------------- 1 | from Crypto.Cipher import AES 2 | from bs4 import BeautifulSoup 3 | import argparse 4 | import sys 5 | import subprocess 6 | import progressbar 7 | import re 8 | import collections 9 | import os 10 | import requests 11 | import json 12 | import uuid 13 | import urllib 14 | import urlparse 15 | import logging 16 | 17 | import f4v 18 | 19 | BASE_URL = 'http://www.mako.co.il' 20 | USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36' 21 | PLAYER_CONFIG = {} 22 | VOD_CONFIG = {} 23 | 24 | PLAYLIST_KEY = 'LTf7r/zM2VndHwP+4So6bw==' 25 | PAYMENT_SERVICE_KEY = 'Ad4NIXQN4y6HyPp1qoT1H1==' 26 | 27 | logger = logging.getLogger('mako') 28 | 29 | def fix_asx(asx): 30 | """Fix unescaped ampersands in //ref/@href""" 31 | def fix_href(m): 32 | before, href, after = m.groups() 33 | return before + re.sub('&(?!amp;)', '&', href) + after 34 | 35 | return re.sub(r'(]*href\s*=\s*")([^"]*)(")', fix_href, asx) 36 | 37 | def load_config(): 38 | VOD_CONFIG_URL = 'http://www.mako.co.il/html/flash_swf/VODConfig.xml' 39 | # The player now loads configNew.xml which is identical 40 | PLAYER_CONFIG_URL = 'http://rcs.mako.co.il/flash_swf/players/makoPlayer/configNew.xml' 41 | 42 | vod_config_raw = requests.get(VOD_CONFIG_URL).content 43 | # Fix broken XML 44 | def fix_href(m): 45 | before, href, after = m.groups() 46 | return before + re.sub('&(?!amp;)', '&', href) + after 47 | 48 | vod_config = BeautifulSoup(re.sub(r'(<[^>]+Url>)([^<]+)(>sys.stderr, 'No video variable in episode page. Skipping.' 204 | continue 205 | 206 | do_video(vod_json['video'], download, dl_dir, silent=True) 207 | 208 | def process_url(url, selection, output='.', download=True): 209 | fragment = urlparse.urlparse(url).fragment 210 | if fragment.startswith('/'): 211 | url = urlparse.urljoin(BASE_URL, fragment) 212 | else: 213 | url = urlparse.urljoin(BASE_URL, url) 214 | 215 | json_vars = collect_json(url) 216 | logger.debug('Main URL "%s" has JSON vars: %r', url, json_vars.keys()) 217 | if json_vars['pageType'] == 'Programs': 218 | show_programs(json_vars['allPrograms']) 219 | elif json_vars['pageType'] == 'ProgramPage': 220 | do_episodes(json_vars['programData'], selection, download, output) 221 | elif json_vars['pageType'] == 'ViewPage': 222 | do_video(json_vars['video'], download, output) 223 | 224 | class Selection(object): 225 | Entry = collections.namedtuple('Entry', ('seasons', 'episodes')) 226 | Range = collections.namedtuple('Range', ('start', 'end')) 227 | INFINITY = float('inf') 228 | 229 | def __init__(self): 230 | self._entries = set() 231 | 232 | def __contains__(self, item): 233 | # The default selection accepts all episodes 234 | if len(self._entries) == 0: 235 | return True 236 | 237 | season, episode = item 238 | return any(any(s.start <= season <= s.end for s in entry.seasons) and 239 | any(e.start <= episode <= e.end for e in entry.episodes) 240 | for entry in self._entries) 241 | 242 | def __repr__(self): 243 | return repr(self._entries) 244 | 245 | @staticmethod 246 | def _validate_string(s): 247 | RANGE_RE = r'\d*(?:-\d*)?' 248 | SELECTION_RE = r'{0}(?:,{0})*'.format(RANGE_RE) 249 | FULL_RE = r'^(?:{0}:)?{0}$'.format(SELECTION_RE) 250 | if s == '' or re.match(FULL_RE, s) is None: 251 | raise argparse.ArgumentTypeError('Invalid selection string') 252 | 253 | def add_from_string(self, s): 254 | self._validate_string(s) 255 | 256 | def make_range(range_str): 257 | if range_str == '': 258 | range_str = '-' 259 | if '-' not in range_str: 260 | return self.Range(start=int(range_str), end=int(range_str)) 261 | start, end = range_str.split('-') 262 | start = 1 if start == '' else int(start) 263 | end = self.INFINITY if end == '' else int(end) 264 | return self.Range(start, end) 265 | 266 | if ':' not in s: 267 | s = '1:%s' % (s, ) 268 | 269 | seasons, episodes = [ tuple(map(make_range, x.split(','))) for x in s.split(':') ] 270 | self._entries.add(self.Entry(seasons=seasons, episodes=episodes)) 271 | return self 272 | 273 | def add_selection_option(parser, *names): 274 | selection = Selection() 275 | parser.add_argument(*names, help='Select episodes to download in the form s1-sN:ep1-epN (the option can be repeated, and any number can be omitted, e.g. : means all episodes of all seasons, 1-4,7 means episodes 1,2,3,4,7 of the first season', type=selection.add_from_string) 276 | -------------------------------------------------------------------------------- /mako: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import logging 3 | import libmako 4 | import argparse 5 | from libmako import logger 6 | 7 | def setup_logging(level): 8 | log_handler = logging.StreamHandler() 9 | log_handler.setLevel(level) 10 | logger.addHandler(log_handler) 11 | logger.setLevel(level) 12 | 13 | def main(): 14 | parser = argparse.ArgumentParser() 15 | parser.add_argument('url', help='Mako VOD URL (program index, program page or episode page)', nargs='?', default='/mako-vod-index') 16 | parser.add_argument('-d', '--debug', default='error', choices=['debug', 'info', 'warn', 'error'], 17 | help='Enable debug output') 18 | libmako.add_selection_option(parser, '-s', '--select') 19 | parser.add_argument('-l', '--list', action='store_true', help="Don't download anything -- only display a list") 20 | parser.add_argument('-o', '--output', metavar='DIR', help='Output directory', default='.') 21 | args = parser.parse_args() 22 | 23 | setup_logging(getattr(logging, args.debug.upper())) 24 | libmako.process_url(args.url, selection=args.select, output=args.output, download=not args.list) 25 | 26 | if __name__ == '__main__': 27 | main() 28 | -------------------------------------------------------------------------------- /mako_key_extractor/README.md: -------------------------------------------------------------------------------- 1 | This is a script that extracts AES keys from the mako VOD flash player. 2 | 3 | There are two separate keys: 4 | * Payment service key: the service that generates tokens for Akamai HDS. Its 5 | responses are encrypted. 6 | * Playlist key: for some reason the playlist XML containing the stream URL is 7 | also encrypted, and using a different key. 8 | 9 | Dependencies (all executables are assumed to be in `PATH`): 10 | * RABCDasm 11 | * swftools 12 | * redtamarin (redshell) 13 | -------------------------------------------------------------------------------- /mako_key_extractor/asset.asasm: -------------------------------------------------------------------------------- 1 | ; This class is a fake ByteArrayAsset, since redtamarin can't read a real asset. 2 | ; The class is a simple ByteArray subclass. Its constructor decodes a *hardcoded* 3 | ; base64 string and fills the ByteArray with the result. 4 | ; Template variables: {name} - class name. 5 | ; {data} - Base64-encoded binary data. 6 | 7 | script 8 | sinit 9 | refid "{name}/init" 10 | body 11 | maxstack 2 12 | localcount 1 13 | initscopedepth 1 14 | maxscopedepth 4 15 | code 16 | getlocal0 17 | pushscope 18 | 19 | ; Initialize a ByteArray subclass 20 | getscopeobject 0 21 | getlex QName(PackageNamespace(""), "Object") 22 | pushscope 23 | 24 | getlex QName(PackageNamespace("flash.utils"), "ByteArray") 25 | pushscope 26 | 27 | getlex QName(PackageNamespace("flash.utils"), "ByteArray") 28 | newclass "{name}" 29 | popscope 30 | popscope 31 | initproperty QName(PackageNamespace(""), "{name}") 32 | 33 | returnvoid 34 | end ; code 35 | end ; body 36 | end ; method 37 | trait class QName(PackageNamespace(""), "{name}") slotid 1 38 | class 39 | refid "{name}" 40 | instance QName(PackageNamespace(""), "{name}") 41 | extends QName(PackageNamespace("flash.utils"), "ByteArray") 42 | flag SEALED 43 | flag PROTECTEDNS 44 | protectedns ProtectedNamespace("{name}") 45 | iinit 46 | refid "{name}/instance/init" 47 | body 48 | maxstack 4 49 | localcount 3 50 | initscopedepth 6 51 | maxscopedepth 7 52 | code 53 | ; Instance constructor: 54 | getlocal0 55 | constructsuper 0 56 | ; Decode base64 string into a new ByteArray 57 | getlex QName(PackageNamespace("com.hurlant.util"), "Base64") 58 | pushstring "{data}" 59 | callproperty QName(PackageNamespace(""), "decodeToByteArray"), 1 60 | setlocal1 61 | ; this.writeBytes(decoded) 62 | getlocal0 63 | getlocal1 64 | callpropvoid Multiname("writeBytes", [PackageNamespace("")]), 1 65 | ; Rewind 66 | getlocal0 67 | pushint 0 68 | setproperty QName(PackageNamespace(""), "position") 69 | 70 | returnvoid 71 | end ; code 72 | end ; body 73 | end ; method 74 | end ; instance 75 | cinit 76 | refid "{name}/class/init" 77 | body 78 | maxstack 4 79 | localcount 3 80 | initscopedepth 5 81 | maxscopedepth 6 82 | code 83 | getlocal0 84 | pushscope 85 | 86 | returnvoid 87 | end ; code 88 | end ; body 89 | end ; method 90 | end ; class 91 | 92 | end ; trait 93 | end ; script 94 | 95 | -------------------------------------------------------------------------------- /mako_key_extractor/extract.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | import argparse 4 | import tempfile 5 | from contextlib import contextmanager 6 | import shutil 7 | import os 8 | import logging 9 | import urlparse 10 | import re 11 | import base64 12 | import subprocess 13 | 14 | @contextmanager 15 | def tempdir(delete=True): 16 | dirname = tempfile.mkdtemp() 17 | try: 18 | yield dirname 19 | finally: 20 | if delete: 21 | shutil.rmtree(dirname) 22 | 23 | def download_swf(dest): 24 | VOD_CONFIG_URL = 'http://www.mako.co.il/html/flash_swf/VODConfig.xml' 25 | BASE_URL = 'http://rcs.mako.co.il' 26 | 27 | logging.info('Downloading VideoPlayer SWF.') 28 | vod_config_raw = requests.get(VOD_CONFIG_URL).content 29 | # Fix broken XML 30 | def fix_href(m): 31 | before, href, after = m.groups() 32 | return before + re.sub('&(?!amp;)', '&', href) + after 33 | 34 | vod_config = BeautifulSoup(re.sub(r'(<[^>]+Url>)([^<]+)(