├── README.md
└── xiaoet.py


/README.md:
--------------------------------------------------------------------------------
 1 | # Download Xiaoet
 2 | > 小鹅通资源下载工具
 3 | 
 4 | ## 一、安装
 5 | 
 6 | #### 1. 安装python 依赖
 7 | ```
 8 | sudo pip3 install ffmpy m3u8 beautifulsoup4
 9 | ```
10 | #### 2. 安装ffmpeg
11 | ```
12 | CentOS
13 | https://download1.rpmfusion.org/free/el/updates/7/x86_64/repoview/letter_f.group.html
14 | 
15 | Ubuntu
16 | https://launchpad.net/ubuntu/+source/ffmpeg
17 | ```
18 | 
19 | ## 二、使用方法示例     
20 | 
21 | #### 1. 下载单独视频/音频
22 | ```
23 | python3 xiaoet.py <店铺ID> -d <ResourceID>
24 | ```
25 | #### 2. 下载一个专栏所有视频/音频
26 | ```
27 | python3 xiaoet.py <店铺ID> -d <ProductID>
28 | ```
29 | #### 3. 列出一个店铺所有专栏(部分商铺可能失效)
30 | ```
31 | python3 xiaoet.py <店铺ID> -pl
32 | ```
33 | #### 4. 列出该专栏下所有视频/音频
34 | ```
35 | python3 xiaoet.py <店铺ID> -rl <ProductID>
36 | ```
37 | #### 5. 列出视频/音频所在专栏ID
38 | ```
39 | python3 xiaoet.py <店铺ID> -r2p <ResourceID>
40 | ```
41 | #### 5. ffmpeg合成视频
42 | ```
43 | python3 xiaoet.py <店铺ID> -tc <ResourceID>
44 | ```
45 | 
46 | 备注：
47 | 1. 登录需要微信扫码登录，session时效性为4小时，更换店铺需要重新扫码登录
48 | 2. 默认下载目录为同级download目录下，下载完成后视频为分段，将自动合成；音频不需要合成。
49 | 3. 店铺为`appxxxx`,专栏ID(ProductID)为`p_xxxx_xxx`,资源ID分为视频与音频分别为`v_xxx_xxx`、`a_xxx_xxx`


--------------------------------------------------------------------------------
/xiaoet.py:
--------------------------------------------------------------------------------
  1 | import argparse
  2 | import ast
  3 | import base64
  4 | import ffmpy
  5 | import m3u8
  6 | import os
  7 | import json
  8 | import requests
  9 | import subprocess
 10 | import time
 11 | import sys
 12 | 
 13 | from m3u8.model import SegmentList, Segment, find_key
 14 | from bs4 import BeautifulSoup
 15 | 
 16 | class Xet(object):
 17 |     def __init__(self, appid, re_login=False):
 18 |         self.appid = appid
 19 |         self.configs = self.config('r') or {}
 20 |         self.session = self.login(re_login)
 21 |         self.download_dir = 'download'
 22 | 
 23 |     def config(self, mode):
 24 |         try:
 25 |             if mode == 'r':
 26 |                 with open("config.json", "r") as config_file:
 27 |                     return json.load(config_file)
 28 |             elif mode == 'w':
 29 |                 with open("config.json", "w") as config_file:
 30 |                     json.dump(self.configs, config_file)
 31 |                     return True
 32 |         except:
 33 |             return
 34 | 
 35 |     def openfile(self, filepath):
 36 |         if sys.platform.startswith('win'):
 37 |             return subprocess.run(['call', filepath], shell=True)
 38 |         else:
 39 |             return subprocess.run(['open', filepath])
 40 |             
 41 |     def login(self, re_login=False):
 42 |         session = requests.Session()
 43 |         if not re_login and self.configs.get('last_appid') == self.appid and (time.time() - self.configs.get('cookies_time')) < 14400: # 4小时
 44 |             for key, value in self.configs['cookies'].items():
 45 |                 session.cookies[key] = value
 46 |         else:
 47 |             headers = {
 48 |                 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',
 49 |                 'Referer': '',
 50 |                 'Origin': 'https://pc-shop.xiaoe-tech.com',
 51 |                 'Content-Type': 'application/x-www-form-urlencoded'
 52 |             }
 53 |             html = session.get('https://pc-shop.xiaoe-tech.com/{appid}/login'.format(appid=self.appid), headers=headers).text
 54 |             soup = BeautifulSoup(html, 'lxml')
 55 |             initdata = json.loads(soup.find(name='input', id='initData')['value'])
 56 |             with open('qrcode.png', 'wb') as file:
 57 |                 file.write(base64.b64decode(initdata['qrcodeImg']))
 58 |             self.openfile('qrcode.png')
 59 |             # Wait for QRcode to be scanned
 60 |             islogin = False
 61 |             for _ in range(300):
 62 |                 res = json.loads(session.post('https://pc-shop.xiaoe-tech.com/{appid}/checkIfUserHasLogin'.format(appid=self.appid), data={'code': initdata['code']}).text)
 63 |                 if not res['code'] and res['data']['code'] == 1:
 64 |                     islogin = True
 65 |                     break
 66 |                 else:
 67 |                     time.sleep(1)
 68 |             if islogin:
 69 |                 os.remove('qrcode.png')
 70 |                 session.get('https://pc-shop.xiaoe-tech.com/{appid}/pcLogin/0?code={code}'.format(appid=self.appid, code=initdata['code']))
 71 |                 self.configs['last_appid'] = self.appid
 72 |                 self.configs['cookies_time'] = time.time()
 73 |                 self.configs['cookies'] = requests.utils.dict_from_cookiejar(session.cookies)
 74 |                 self.config('w')
 75 |             else:
 76 |                 print('Log in timeout')
 77 |                 exit(1)
 78 |         return session
 79 | 
 80 |     def get_product_list(self):
 81 |         url = 'https://pc-shop.xiaoe-tech.com/{appid}/open/column.all.get/2.0'.format(appid=self.appid)
 82 |         body = {
 83 |             'data[page_index]': '0',
 84 |             'data[page_size]': '1000',
 85 |             'data[order_by]': 'start_at:desc',
 86 |             'data[state]': '0',
 87 |         }
 88 |         self.session.headers.update(
 89 |             {'Referer': 'https://pc-shop.xiaoe-tech.com/{appid}/'.format(appid=self.appid)})
 90 |         res = self.session.post(url, data=body)
 91 |         if res.status_code == 200:
 92 |             content = ast.literal_eval(res.content.decode("UTF-8"))
 93 |             if not content['code']:
 94 |                 for product in content['data']:
 95 |                     print('name: {} price: {} productid: {}'.format(product['title'], int(product['price']) / 100, product['id']))
 96 |                 return content['data']
 97 |             else:
 98 |                 print('status: {} msg: {}'.format(content['code'], content['msg']))
 99 |         return
100 | 
101 |     def get_resource_list(self, productid):
102 |         url = 'https://pc-shop.xiaoe-tech.com/{appid}/open/column.resourcelist.get/2.0'.format(appid=self.appid)
103 |         body = {
104 |             'data[page_index]': '0',
105 |             'data[page_size]': '1000',
106 |             'data[order_by]': 'start_at:desc',
107 |             'data[resource_id]': productid,
108 |             'data[state]': '0'
109 |         }
110 |         self.session.headers.update({'Referer': 'https://pc-shop.xiaoe-tech.com/{appid}/'.format(appid=self.appid)})
111 |         res = self.session.post(url, data=body)
112 |         if res.status_code == 200:
113 |             content = ast.literal_eval(res.content.decode("UTF-8"))
114 |             if not content['code']:
115 |                 for resource in content['data']:
116 |                     print('name: {} resourceid: {}'.format(resource['title'], resource['id']))
117 |                 return content['data']
118 |             else:
119 |                 print('status: {} msg: {}'.format(content['code'], content['msg']))
120 |         return
121 | 
122 |     def transform_type(self, id):
123 |         transform_box = {'a': 'audio', 'v': 'video', 'p': 'product'}
124 |         type = transform_box.get(id[0], None)
125 |         if type:
126 |             return type
127 |         else:
128 |             print('Invalid id. None suitable type')
129 |             exit (1)
130 | 
131 |     def get_resource(self, resourceid):
132 |         resourcetype = self.transform_type(resourceid)
133 |         url = 'https://pc-shop.xiaoe-tech.com/{appid}/open/{resourcetype}.detail.get/1.0'.format(appid=self.appid,
134 |                                                                                   resourcetype=resourcetype)
135 |         body = {
136 |             'data[resource_id]': resourceid
137 |         }
138 |         self.session.headers.update({'Referer': 'https://pc-shop.xiaoe-tech.com/{appid}/{resourcetype}_details?id={resourceid}'.format(
139 |             appid=self.appid, resourcetype=resourcetype, resourceid=resourceid)})
140 |         res = self.session.post(url, data=body)
141 |         if res.status_code == 200:
142 |             content = ast.literal_eval(res.content.decode("UTF-8"))
143 |             if not content['code']:
144 |                 return content['data']
145 |             else:
146 |                 print('status: {} msg: {}'.format(content['code'], content['msg']))
147 |         return {'id': resourceid}
148 | 
149 |     def get_productid(self, resourceid):
150 |         res = self.get_resource(resourceid)
151 |         if res.get('products'):
152 |             print (res['products'][0]['product_id'])
153 |         return
154 | 
155 |     def download_video(self, download_dir, resource, nocache=False):
156 |         resource_dir = os.path.join(download_dir, resource['id'])
157 |         os.makedirs(resource_dir, exist_ok=True)
158 | 
159 |         url = resource['video_hls'].replace('\\', '')
160 |         self.session.headers.update({'Referer': 'https://pc-shop.xiaoe-tech.com/{appid}/video_details?id={resourceid}'.format(
161 |                 appid=self.appid, resourceid=resource['id'])})
162 |         media = m3u8.loads(self.session.get(url).text)
163 |         url_prefix, segments, changed, complete = url.split('v.f230')[0], SegmentList(), False, True
164 | 
165 |         print('Total: {} part'.format(len(media.data['segments'])))
166 |         for index, segment in enumerate(media.data['segments']):
167 |             ts_file = os.path.join(resource_dir, 'v_{}.ts'.format(index))
168 |             if not nocache and os.path.exists(ts_file):
169 |                 print('Already Downloaded: {title} {file}'.format(title=resource['title'], file=ts_file))
170 |             else:
171 |                 url = url_prefix + segment.get('uri')
172 |                 res = self.session.get(url)
173 |                 if res.status_code == 200:
174 |                     with open(ts_file + '.tmp', 'wb') as ts:
175 |                         ts.write(res.content)
176 |                     os.rename(ts_file + '.tmp', ts_file)
177 |                     changed = True
178 |                     print('Download Successful: {title} {file}'.format(title=resource['title'], file=ts_file))
179 |                 else:
180 |                     print('Download Failed: {title} {file}'.format(title=resource['title'], file=ts_file))
181 |                     complete = False
182 |             segment['uri'] = 'v_{}.ts'.format(index)
183 |             segments.append(Segment(base_uri=None, keyobject=find_key(segment.get('key', {}), media.keys), **segment))
184 | 
185 |         m3u8_file = os.path.join(resource_dir, 'video.m3u8')
186 |         if changed or not os.path.exists(m3u8_file):
187 |             media.segments = segments
188 |             with open(m3u8_file, 'w', encoding='utf8') as f:
189 |                 f.write(media.dumps())
190 |         metadata = {'title': resource['title'], 'complete': complete}
191 |         with open(os.path.join(download_dir, resource['id'], 'metadata'), 'w') as f:
192 |             json.dump(metadata, f)
193 |         return
194 | 
195 |     def download_audio(self, download_dir, resource, nocache=False):
196 |         url = resource['audio_url'].replace('\\', '')
197 |         audio_file = os.path.join(download_dir, '{title}.{suffix}'.format(title=resource['title'], suffix=os.path.basename(url).split('.')[-1]))
198 |         if not nocache and os.path.exists(audio_file):
199 |             print('Already Downloaded: {title} {file}'.format(title=resource['title'], file=audio_file))
200 |         else:
201 |             self.session.headers.update(
202 |                 {'Referer': 'https://pc-shop.xiaoe-tech.com/{appid}/audio_details?id={resourceid}'.format(
203 |                     appid=self.appid, resourceid=resource['id'])})
204 |             res = self.session.get(url, stream=True)
205 |             if res.status_code == 200:
206 |                 with open(audio_file, 'wb') as f:
207 |                     f.write(res.content)
208 |                 print('Download Successful: {title} {file}'.format(title=resource['title'], file=audio_file))
209 |             else:
210 |                 print('Download Failed: {title} {file}'.format(title=resource['title'], file=audio_file))
211 |         return
212 | 
213 |     def transcode(self, resourceid):
214 |         resource_dir = os.path.join(self.download_dir, resourceid)
215 |         if os.path.exists(resource_dir) and os.path.exists(os.path.join(resource_dir, 'metadata')):
216 |             with open(os.path.join(resource_dir, 'metadata')) as f:
217 |                 metadata = json.load(f)
218 |             if metadata['complete']:
219 |                 ff = ffmpy.FFmpeg(inputs={os.path.join(resource_dir, 'video.m3u8'): ['-protocol_whitelist', 'crypto,file,http,https,tcp,tls']}, outputs={os.path.join(self.download_dir, metadata['title'] + '.mp4'): None})
220 |                 print(ff.cmd)
221 |                 ff.run()
222 |         return
223 | 
224 |     def download(self, id, nocahce=False):
225 |         os.makedirs(self.download_dir, exist_ok=True)
226 |         if self.transform_type(id) == 'product':
227 |             resource_list = [self.get_resource(resource['id']) for resource in self.get_resource_list(id)]
228 |         else:
229 |             resource_list = [self.get_resource(id)]
230 | 
231 |         for resource in resource_list:
232 |             if resource.get('is_available') == 1:
233 |                 if self.transform_type(resource['id']) == 'audio':
234 |                     self.download_audio(self.download_dir, resource, nocahce)
235 |                 elif self.transform_type(resource['id']) == 'video':
236 |                     self.download_video(self.download_dir, resource, nocahce)
237 |                     self.transcode(resource['id'])
238 |             elif resource.get('is_available') == 0:
239 |                 print('Not purchased. name: {} resourceid: {}'.format(resource['title'], resource['id']))
240 |             else:
241 |                 print('Not Found. resourceid: {}'.format(resource['id']))
242 |         return
243 | 
244 | def parse_args():
245 |     parser = argparse.ArgumentParser(description='''Download tools for Xiaoe-tech''')
246 |     parser.add_argument("appid", type=str,
247 |                         help='''Shop ID of xiaoe-tech.''')
248 |     parser.add_argument("-d", type=str, metavar='ID', help='''Download resources by Resource ID or Product ID.''')
249 |     parser.add_argument("-rl", type=str, metavar='Product ID', help='''Display All resources of the Product ID''')
250 |     parser.add_argument("-pl", action='store_true', help='''Display All products of the Shop''')
251 |     parser.add_argument("-r2p", type=str, metavar='Resource ID', help='''Get Product ID from Resource ID''')
252 |     parser.add_argument("-tc", type=str, metavar='Resource ID', help='''Combine and transcode the video''')
253 |     parser.add_argument("--nocache", action='store_true', help='''Download without cache''')
254 |     parser.add_argument("--login", action='store_true', help='''Force to re-login''')
255 |     return parser.parse_args()
256 | 
257 | def main():
258 |     args = parse_args()
259 |     xet = Xet(args.appid, args.login)
260 |     if args.d:
261 |         xet.download(args.d, args.nocache)
262 |     if args.rl:
263 |         xet.get_resource_list(args.rl)
264 |     if args.pl:
265 |         xet.get_product_list()
266 |     if args.r2p:
267 |         xet.get_productid(args.r2p)
268 |     if args.tc:
269 |         xet.transcode(args.tc)
270 | 
271 | if __name__ == '__main__':
272 |     main()


--------------------------------------------------------------------------------