├── README.md ├── douyin_spider.py ├── huoshan_spider.py ├── images └── res01.png ├── kuaishou_spider.py ├── settings.py └── utils ├── __init__.py ├── __pycache__ ├── __init__.cpython-36.pyc ├── download.cpython-36.pyc ├── name_manager.cpython-36.pyc └── verify_params.cpython-36.pyc ├── download.py ├── name_manager.py ├── out.mp4 └── verify_params.py /README.md: -------------------------------------------------------------------------------- 1 | ## 抖音 2 | 3 | ### 1. 获取播放`url`的`json`数据 4 | 5 | 1. 请求的`url` 6 | 7 | `https://aweme-hl.snssdk.com/aweme/v1/feed/` 8 | 9 | 2. 请求的方式 10 | 11 | `get` 12 | 13 | 2. `headers` 14 | 15 | ```python 16 | headers = { 17 | "User-Agent": "okhttp/3.10.0.1", 18 | } 19 | ``` 20 | 21 | 3. `query string` 22 | 23 | ```python 24 | params = { 25 | "type": "0", 26 | "max_cursor": "0", 27 | "min_cursor": "0", 28 | "count": "6", 29 | "aweme_id": "id", 30 | "volume": "0.7333333333333333", 31 | "pull_type": "0", 32 | "need_relieve_aweme": "0", 33 | "filter_warn": "0", 34 | "req_from": "enter_auto", 35 | "cached_item_num": "0", 36 | "last_ad_show_interval": "-1", 37 | "ts": "1585487675", 38 | "app_type": "lite", 39 | "os_api": "23", 40 | "device_type": "MI 5s", 41 | "device_platform": "android", 42 | "ssmix": "a", 43 | "iid": "104392158277", 44 | "manifest_version_code": "290", 45 | "dpi": "270", 46 | "uuid": "490000000085156", 47 | "version_code": "290", 48 | "app_name": "douyin_lite", 49 | "cdid": "b20c68ce-2e2f-4b3e-a3b8-74b2e490df41", 50 | "version_name": "2.9.0", 51 | "openudid": "b8db9c78de7c82b5", 52 | "device_id": "70837664152", 53 | "resolution": "810*1440", 54 | "os_version": "6.0.1", 55 | "language": "zh", 56 | "device_brand": "Xiaomi", 57 | "ac": "wifi", 58 | "update_version_code": "2900", 59 | "aid": "2329", 60 | "channel": "tengxun", 61 | "_rticket": "1585487676509", 62 | "as": "a111111111111111111111", 63 | "cp": "a000000000000000000000", 64 | "mas": "", 65 | } 66 | ``` 67 | 68 | ## 火山小视频 69 | 70 | ### 1. 获得了一个视频`url`的`json`请求 71 | 72 | 通过这个`url`只能得到一个视频的`url`. 73 | 74 | 1. 请求的`url` 75 | 76 | `https://hotsoon-hl.snssdk.com/hotsoon/item/video/_get/` 77 | 78 | 2. 请求的方式 79 | 80 | `get` 81 | 82 | 3. 请求头 83 | 84 | ```python 85 | headers = { 86 | "User-Agent": "ttnet okhttp/3.10.0.2", 87 | } 88 | ``` 89 | 90 | 4. 请求的参数: 经过分析只需要三个参数 91 | 92 | ```python 93 | params = { 94 | "item_id": "6800228635925171471", # 请求的视频的id 95 | # "live_sdk_version": "827", 96 | # "iid": "104447577985", 97 | # "device_id": "70837664152", 98 | # "ac": "wifi", # 连接类型 99 | # "channel": "tengxun_new", 100 | # "aid": "1112", 101 | "app_name": "live_stream", # app 名字 102 | # "version_code": "827", # 版本代码 103 | # "version_name": "8.2.7", # 版本名字 104 | # "device_platform": "android", # 设备平台 105 | # "ssmix": "a", 106 | "device_type": "MI+5s", # 设备类型 107 | # "device_brand": "Xiaomi", # 设备品牌 108 | # "language": "zh", # 语言 109 | # "os_api": "23", # 操作系统api 110 | # "os_version": "6.0.1", # 操作系统版本 111 | # "uuid": "490000000085156", 112 | # "openudid": "b8db9c78de7c82b5", 113 | # "manifest_version_code": "827", # 清单版本代码 114 | # "resolution": "810*1440", # 像素比例 115 | # "dpi": "270", # 图像分辨率 116 | # "update_version_code": "8270", # 更新版本代码 117 | # "_rticket": "1585566002578", # 请求 ticket 118 | # "ab_version": "1413809%2C1244214%2C889330%2C1138752%2C1589082%2C1063522%2C1480776%2C1377092%2C1380327%2C1582436%2C1588775%2C1167795%2C1476946%2C1404472%2C1517651%2C1354483%2C1479194%2C1258912%2C1264664%2C1521584%2C955276%2C1589846%2C947985%2C1548003%2C1182061%2C1480948%2C1435640%2C1477984%2C929432%2C1490515%2C1432944%2C1555350%2C1590375%2C1541253%2C1540549%2C1428670%2C1048435%2C1168129%2C1396601%2C1582073%2C1549345%2C1396899%2C1096187%2C1104584%2C1478759%2C1419023%2C1538832%2C1548270%2C1565149%2C1496674%2C1550828%2C1568912%2C1574488%2C1581133%2C1580160%2C1320817%2C1133591%2C692223%2C1169771%2C956107%2C1247692%2C1019139%2C682009%2C1032070%2C1165214%2C1265052%2C1584527%2C1072545%2C1317441%2C1562047%2C1069233%2C1583612%2C1143559%2C1544623%2C1337822%2C1293405%2C1347260%2C1046183%2C1354701%2C1143672%2C1498072%2C1143730%2C1417290%2C1491283%2C1165209%2C1576837%2C1376626%2C1572549%2C1409058%2C1502675%2C1578552%2C1050089", # 版本 119 | # "client_version_code": "827", 120 | # "jssdk_version": "1.37.1.2", # js sdk 版本 121 | # "mcc_mnc": "46005", 122 | # "cdid": "64d624fa-4514-401c-99d5-95e8a05b0c83", 123 | # "new_nav": "1", 124 | # "ws_status": "CONNECTED", # 状态 125 | # "settings_version": "19", # 配置版本 126 | # "last_update_time": "1585565111754", # 上一次的更新时间 127 | # "ts": "1585566002", # 请求的时间 128 | } 129 | ``` 130 | 131 | ### 2. 主页的视频数据请求接口 132 | 133 | 1. 请求的`url` 134 | 135 | `https://hotsoon-hl.snssdk.com/hotsoon/feed、` 136 | 137 | 2. 请求的方式 138 | 139 | `get` 140 | 141 | 3. 请求头 142 | 143 | ```python 144 | headers = { 145 | "User-Agent": "ttnet okhttp/3.10.0.2", 146 | } 147 | ``` 148 | 149 | 4. 请求参数 150 | 151 | ```python 152 | params = { 153 | "type": "video", 154 | "tab_id": "5", 155 | "front_ids": "6809869449601109263%2C6809893253329095944%2C6808726733139889421%2C6801421644104617216%2C1634668034797582%2C6800306668589698304%2C6799054861905317135%2C6806037711485553924", 156 | "last_ad_items": "%5B%7B%22id%22%3A6803159557326654724%2C%22show_time%22%3A1585565114%7D%2C%7B%22id%22%3A6800667492667772173%2C%22show_time%22%3A1585565114%7D%2C%7B%22id%22%3A1662299439581208%2C%22show_time%22%3A1585565114%7D%2C%7B%22id%22%3A4760%2C%22show_time%22%3A1585565114%7D%2C%7B%22id%22%3A6805071744987188480%2C%22show_time%22%3A1585565181%7D%2C%7B%22id%22%3A6805102455383280908%2C%22show_time%22%3A1585565181%7D%2C%7B%22id%22%3A6804693165258460424%2C%22show_time%22%3A1585565182%7D%2C%7B%22id%22%3A6804840893229714695%2C%22show_time%22%3A1585565182%7D%2C%7B%22id%22%3A1662462880442372%2C%22show_time%22%3A1585567006%7D%5D", 157 | "n_viewed": "0", 158 | "offset": "18", 159 | "diff_stream": "1", 160 | "ad_user_agent": "com.ss.android.ugc.live%2F827+%28Linux%3B+U%3B+Android+6.0.1%3B+zh_CN%3B+MI+5s%3B+Build%2FV417IR%3B+Chrome%29", 161 | "req_from": "feed_loadmore", 162 | "count": "10", 163 | "secs_video_watching": "1811", 164 | "n_skipped": "0", 165 | "minor_control_status": "0", 166 | "feed_video_gap": "140", 167 | "max_time": "1585565187347", 168 | "live_sdk_version": "827", 169 | "iid": "104447577985", 170 | "device_id": "70837664152", 171 | "ac": "wifi", 172 | "channel": "tengxun_new", 173 | "aid": "1112", 174 | "app_name": "live_stream", 175 | "version_code": "827", 176 | "version_name": "8.2.7", 177 | "device_platform": "android", 178 | "ssmix": "a", 179 | "device_type": "MI+5s", 180 | "device_brand": "Xiaomi", 181 | "language": "zh", 182 | "os_api": "23", 183 | "os_version": "6.0.1", 184 | "uuid": "490000000085156", 185 | "openudid": "b8db9c78de7c82b5", 186 | "manifest_version_code": "827", 187 | "resolution": "810*1440", 188 | "dpi": "270", 189 | "update_version_code": "8270", 190 | "_rticket": "1585567014625", 191 | "ab_version": "1413809%2C1244214%2C889330%2C1138752%2C1589082%2C1063522%2C1480776%2C1377092%2C1380327%2C1582436%2C1588775%2C1167795%2C1476946%2C1404472%2C1517651%2C1354483%2C1479194%2C1258912%2C1264664%2C1521584%2C955276%2C1589846%2C947985%2C1548003%2C1182061%2C1480948%2C1435640%2C1477984%2C929432%2C1490515%2C1432944%2C1555350%2C1590375%2C1541253%2C1540549%2C1428670%2C1048435%2C1168129%2C1396601%2C1582073%2C1549345%2C1396899%2C1096187%2C1104584%2C1478759%2C1419023%2C1538832%2C1548270%2C1565149%2C1496674%2C1550828%2C1568912%2C1574488%2C1581133%2C1580160%2C1320817%2C1133591%2C692223%2C1169771%2C956107%2C1247692%2C1019139%2C682009%2C1032070%2C1165214%2C1265052%2C1584527%2C1072545%2C1317441%2C1562047%2C1069233%2C1583612%2C1143559%2C1544623%2C1337822%2C1293405%2C1347260%2C1046183%2C1354701%2C1143672%2C1498072%2C1143730%2C1417290%2C1491283%2C1165209%2C1576837%2C1376626%2C1572549%2C1409058%2C1502675%2C1578552%2C1050089", 192 | "client_version_code": "827", 193 | "jssdk_version": "1.37.1.2", 194 | "mcc_mnc": "46005", 195 | "cdid": "64d624fa-4514-401c-99d5-95e8a05b0c83", 196 | "new_nav": "1", 197 | "ws_status": "CONNECTED", 198 | "settings_version": "19", 199 | "last_update_time": "1585565111754", 200 | "mac_address": "08%3A00%3A27%3A3E%3A23%3A0D", 201 | "ts": "1585567014", 202 | } 203 | ``` 204 | 205 | ## 快手 206 | 207 | ### 1. 获取播放`url`的`json`数据 208 | 209 | 1. 请求的`url` 210 | 211 | `https://apissl.gifshow.com/rest/n/feed/hot` 212 | 213 | 2. 请求方式 214 | 215 | `post` 216 | 217 | 3. 请求头 218 | 219 | ```python 220 | headers = { 221 | "User-Agent": "kwai-android aegon/1.10.2-curl", 222 | } 223 | ``` 224 | 225 | 4. 请求的参数 226 | 227 | ```python 228 | params = { 229 | "mod": "vivo(vivo X9Plus)", 230 | "lon": "121.492379", 231 | "country_code": "cn", 232 | "extId": "bc70d48c39d1f18a8713e4cb3f8a800c", 233 | "kpn": "KUAISHOU", 234 | "oc": "360APP,1", 235 | "egid": "DFP058657B907ED8C6D9F0E270AAD8E19C6DF413211D9C0E53D8F775812D4F1F", 236 | "hotfix_ver": "", 237 | "sh": "1920", 238 | "appver": "7.2.2.12969", 239 | "socName": "UNKNOWN", 240 | "max_memory": "192", 241 | "isp": "CMCC", 242 | "kcv": "188", 243 | "browseType": "1", 244 | "kpf": "ANDROID_PHONE", 245 | "did": "ANDROID_c50c87e4562bf3f9", 246 | "net": "WIFI", 247 | "app": "0", 248 | "ud": "0", 249 | "c": "360APP,1", 250 | "sys": "ANDROID_5.1.1", 251 | "sw": "1080", 252 | "ftt": "", 253 | "ll": "CTTVk/lHPz9AES6PNSODX15A", 254 | "language": "zh-cn", 255 | "iuid": "", 256 | "lat": "31.247192", 257 | "did_gt": "1585622393761", 258 | "ver": "7.2", 259 | } 260 | ``` 261 | 262 | 5. `post`参数 263 | 264 | ```python 265 | form = { 266 | "type": "7", 267 | "page": "2", 268 | "coldStart": "false", 269 | "count": "20", 270 | "pv": "false", 271 | "id": "13", 272 | "refreshTimes": "1", 273 | "pcursor": "1", 274 | "source": "1", 275 | "extInfo": "TXURYODnxuqz8vLON/bBsj+X/NITG1FIT5ubnS+Bf39hSC0TLguH3XERnIrzRNPlxqxQcdIb58jND7AJFA6LgYnjjyDXznlJwi/wK8LxROT5tDiTQleU9pi/0VobR39E552kiUZ/aIMYbudUt7J/UQ==", 276 | "needInterestTag": "false", 277 | "seid": "f40304fb-d089-498a-91ad-ee6031186711", 278 | "volume": "0.27", 279 | "backRefresh": "false", 280 | "pageCount": "2", 281 | "adChannel": '{"type":1,"data":""}', 282 | "passThrough": "0", 283 | "thanosSpring": "false", 284 | "newUserRefreshTimes": "15", 285 | "newUserAction": '{"click":[5189835650000717127,5243315891402845228],"follow":[],"like":[]}', 286 | "cellList": '[{"ci":53185,"lac":6311,"mcc":460,"mnc":0,"radio":"gsm","rssi":0}]', 287 | "__NS_sig3": "2202972399d8693c1e9e2b081d7e7f5bc60c03c6dc", 288 | "client_key": "3c2cd3f3", 289 | "os": "android", 290 | "sig": "d695f07ae9d37816fdf4de698e659ab6", 291 | } 292 | ``` 293 | 294 | ## 总结 295 | 296 | 这三个移动端的小视频平台，请求的内容，在参数是相同的情况下，每次都会发生变化。具体的实现（比如：这次的请求内容和上次的请求内容，如何不重复的原理，还没有搞懂） 297 | 298 | ## 代码 299 | 300 | 欢迎来到我的 [github](https://github.com/DAJINZI01/movile_video) 301 | 302 | ## 运行截图 303 | 304 | ![res01.png](images/res01.png) -------------------------------------------------------------------------------- /douyin_spider.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | from settings import BASE_DIR 3 | from utils import download 4 | 5 | import requests 6 | import os 7 | import json 8 | 9 | 10 | class DouYin(object): 11 | def __init__(self): 12 | self.headers = { 13 | # "X-SS-REQ-TICKET": "1585487676447", 14 | # "sdk-version": "1", 15 | # "Cookie": "odin_tt=9c7ef2bb926863a0f8fcfb6d06532d569a6eb5cbfae6186b76f1046e269479630288340487398d82b9d4c346eefd7ecca0c1ae41aed65a51d1b92ea7cb381838; install_id=104392158277; ttreq=1$34c948237e9c5447e4a79e2b2a196db4e7845adb; sid_guard=3d396bfeb06b92101fc3d29c127172d9%7C1585486945%7C5184000%7CThu%2C+28-May-2020+13%3A02%3A25+GMT; uid_tt=acc78696ba27990a477b092ae95e900b; sid_tt=3d396bfeb06b92101fc3d29c127172d9; sessionid=3d396bfeb06b92101fc3d29c127172d9; qh[360]=1", 16 | # "x-tt-token": "003d396bfeb06b92101fc3d29c127172d9b13a081789d0d9f368f9890562fa422c1f58b29bf7c80257820fbb7382e8411f39", 17 | # "X-Gorgon": "0401c0d2400101345a666d88d3a4bf821513bbbee98c700d5efc", 18 | # "X-Khronos": "1585487676", 19 | # "Host": "aweme-hl.snssdk.com", 20 | "User-Agent": "okhttp/3.10.0.1", # 经过测试只需要这个 user-agent 头 21 | } 22 | self.downloader = download.MyDownloader(bar_symbol="*") 23 | 24 | def get_info(self): 25 | """ 26 | 获取播放数据的接口，很容易找到。 27 | 返回的数据，如下: 28 | { 29 | "status_code":0, 30 | "min_cursor":0, 31 | "max_cursor":0, 32 | "has_more":1, 33 | "aweme_list":Array[6], 34 | "home_model":1, 35 | "refresh_clear":1, 36 | "extra":Object{...}, 37 | "log_pb":Object{...}, 38 | "preload_ads":Array[0], 39 | "preload_awemes":null 40 | } 41 | :return: 字典 42 | """ 43 | url = "https://aweme-hl.snssdk.com/aweme/v1/feed/" 44 | params = { 45 | # "type": "0", 46 | # "max_cursor": "0", 47 | # "min_cursor": "0", 48 | # "count": "20", 49 | # "aweme_id": "id", 50 | # "volume": "0.7333333333333333", 51 | # "pull_type": "0", 52 | # "need_relieve_aweme": "0", 53 | # "filter_warn": "0", 54 | # "req_from": "enter_auto", 55 | "cached_item_num": "4", 56 | # "last_ad_show_interval": "-1", 57 | # "ts": "1585487675", 58 | # "app_type": "lite", 59 | # "os_api": "23", 60 | "device_type": "MI 5s", 61 | "device_platform": "android", 62 | # "ssmix": "a", 63 | # "iid": "104392158277", 64 | # "manifest_version_code": "290", 65 | # "dpi": "270", 66 | # "uuid": "490000000085156", 67 | "version_code": "290", 68 | "app_name": "douyin_lite", 69 | # "cdid": "b20c68ce-2e2f-4b3e-a3b8-74b2e490df41", 70 | # "version_name": "2.9.0", 71 | # "openudid": "b8db9c78de7c82b5", 72 | # "device_id": "70837664152", 73 | # "resolution": "810*1440", 74 | "os_version": "6.0.1", 75 | # "language": "zh", 76 | # "device_brand": "Xiaomi", 77 | # "ac": "wifi", 78 | # "update_version_code": "2900", 79 | # "aid": "2329", 80 | "channel": "tengxun", 81 | # "_rticket": "1585487676509", 82 | # "as": "a111111111111111111111", 83 | # "cp": "a000000000000000000000", 84 | # "mas": "", 85 | } 86 | return requests.get(url, params=params, headers=self.headers).json() 87 | 88 | def get_video_list(self): 89 | """ 90 | 获取播放视频的列表，每一个元素为 {"desc": "xxx", "play_addr": "xxx"} 91 | :return: 列表 [{"desc": "xxx", "play_addr": "xxx"}, ...] 92 | """ 93 | aweme_list = self.get_info()["aweme_list"] 94 | video_list = [] 95 | for aweme in aweme_list: 96 | item = { 97 | "desc": aweme["desc"], 98 | "play_addr": aweme["video"]["play_addr"]["url_list"][0], 99 | } 100 | video_list.append(item) 101 | return video_list 102 | 103 | def download_video_list(self, folder="douyin"): 104 | """ 105 | 从视频列表中下载视频 106 | :param video_list: 视频列表 [{"desc": "xxx", "play_addr": "xxx"}, ...] 107 | :param folder: 下载存放的位置, 默认为当前目录下的data文件夹 108 | :return: 109 | """ 110 | video_list = self.get_video_list() 111 | # 1. 创建文件夹 112 | video_folder = "%s/%s" % (BASE_DIR, folder) 113 | if not os.path.exists(video_folder): 114 | os.mkdir(video_folder) 115 | # 2. 遍历视频列表下载文件 116 | for video in video_list: 117 | file_name = "%s/%s.mp4" % (video_folder, video["desc"]) 118 | # 3. 当文件不存在的时候下载文件 119 | if not os.path.exists(file_name): 120 | self.downloader.download(video["play_addr"], file_name) 121 | 122 | 123 | if __name__ == "__main__": 124 | d = DouYin() 125 | # print(json.dumps(d.get_info(), ensure_ascii=False)) 126 | # print(d.get_video_list()) 127 | for _ in range(10): 128 | d.download_video_list() -------------------------------------------------------------------------------- /huoshan_spider.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | from settings import BASE_DIR 3 | from utils import download, name_manager 4 | 5 | import requests 6 | import json 7 | import os 8 | 9 | 10 | class HuoShan(object): 11 | def __init__(self): 12 | self.headers = { 13 | "User-Agent": "ttnet okhttp/3.10.0.2", 14 | } 15 | self.downloader = download.MyDownloader(bar_symbol="■") 16 | 17 | def get_item_info(self): 18 | url = "https://hotsoon-hl.snssdk.com/hotsoon/item/video/_get/" 19 | params = { 20 | "item_id": "6800228635925171471", 21 | # "live_sdk_version": "827", 22 | # "iid": "104447577985", 23 | # "device_id": "70837664152", 24 | # "ac": "wifi", # 连接类型 25 | # "channel": "tengxun_new", 26 | # "aid": "1112", 27 | "app_name": "live_stream", # app 名字 28 | # "version_code": "827", # 版本代码 29 | # "version_name": "8.2.7", # 版本名字 30 | # "device_platform": "android", # 设备平台 31 | # "ssmix": "a", 32 | "device_type": "MI+5s", # 设备类型 33 | # "device_brand": "Xiaomi", # 设备品牌 34 | # "language": "zh", # 语言 35 | # "os_api": "23", # 操作系统api 36 | # "os_version": "6.0.1", # 操作系统版本 37 | # "uuid": "490000000085156", 38 | # "openudid": "b8db9c78de7c82b5", 39 | # "manifest_version_code": "827", # 清单版本代码 40 | # "resolution": "810*1440", # 像素比例 41 | # "dpi": "270", # 图像分辨率 42 | # "update_version_code": "8270", # 更新版本代码 43 | # "_rticket": "1585566002578", # 请求 ticket 44 | # "ab_version": "1413809%2C1244214%2C889330%2C1138752%2C1589082%2C1063522%2C1480776%2C1377092%2C1380327%2C1582436%2C1588775%2C1167795%2C1476946%2C1404472%2C1517651%2C1354483%2C1479194%2C1258912%2C1264664%2C1521584%2C955276%2C1589846%2C947985%2C1548003%2C1182061%2C1480948%2C1435640%2C1477984%2C929432%2C1490515%2C1432944%2C1555350%2C1590375%2C1541253%2C1540549%2C1428670%2C1048435%2C1168129%2C1396601%2C1582073%2C1549345%2C1396899%2C1096187%2C1104584%2C1478759%2C1419023%2C1538832%2C1548270%2C1565149%2C1496674%2C1550828%2C1568912%2C1574488%2C1581133%2C1580160%2C1320817%2C1133591%2C692223%2C1169771%2C956107%2C1247692%2C1019139%2C682009%2C1032070%2C1165214%2C1265052%2C1584527%2C1072545%2C1317441%2C1562047%2C1069233%2C1583612%2C1143559%2C1544623%2C1337822%2C1293405%2C1347260%2C1046183%2C1354701%2C1143672%2C1498072%2C1143730%2C1417290%2C1491283%2C1165209%2C1576837%2C1376626%2C1572549%2C1409058%2C1502675%2C1578552%2C1050089", # 版本 45 | # "client_version_code": "827", 46 | # "jssdk_version": "1.37.1.2", # js sdk 版本 47 | # "mcc_mnc": "46005", 48 | # "cdid": "64d624fa-4514-401c-99d5-95e8a05b0c83", 49 | # "new_nav": "1", 50 | # "ws_status": "CONNECTED", # 状态 51 | # "settings_version": "19", # 配置版本 52 | # "last_update_time": "1585565111754", # 上一次的更新时间 53 | # "ts": "1585566002", # 请求的时间 54 | } 55 | data = requests.get(url, params=params, headers=self.headers).json() 56 | print(json.dumps(data, ensure_ascii=False)) 57 | 58 | def get_info(self): 59 | url = "https://hotsoon-hl.snssdk.com/hotsoon/feed/" 60 | params = { 61 | "type": "video", 62 | # "tab_id": "5", 63 | # "front_ids": "6809869449601109263%2C6809893253329095944%2C6808726733139889421%2C6801421644104617216%2C1634668034797582%2C6800306668589698304%2C6799054861905317135%2C6806037711485553924", 64 | # "last_ad_items": "%5B%7B%22id%22%3A6803159557326654724%2C%22show_time%22%3A1585565114%7D%2C%7B%22id%22%3A6800667492667772173%2C%22show_time%22%3A1585565114%7D%2C%7B%22id%22%3A1662299439581208%2C%22show_time%22%3A1585565114%7D%2C%7B%22id%22%3A4760%2C%22show_time%22%3A1585565114%7D%2C%7B%22id%22%3A6805071744987188480%2C%22show_time%22%3A1585565181%7D%2C%7B%22id%22%3A6805102455383280908%2C%22show_time%22%3A1585565181%7D%2C%7B%22id%22%3A6804693165258460424%2C%22show_time%22%3A1585565182%7D%2C%7B%22id%22%3A6804840893229714695%2C%22show_time%22%3A1585565182%7D%2C%7B%22id%22%3A1662462880442372%2C%22show_time%22%3A1585567006%7D%5D", 65 | # "n_viewed": "0", 66 | # "offset": "18", 67 | # "diff_stream": "1", 68 | # "ad_user_agent": "com.ss.android.ugc.live%2F827+%28Linux%3B+U%3B+Android+6.0.1%3B+zh_CN%3B+MI+5s%3B+Build%2FV417IR%3B+Chrome%29", # 添加 user-agent 69 | # "req_from": "feed_loadmore", # 请求来源 70 | # "count": "10", 71 | # "secs_video_watching": "1811", 72 | # "n_skipped": "0", # 跳过的数量 73 | # "minor_control_status": "0", # 控制状态 74 | # "feed_video_gap": "140", # 输出视频间隔 75 | # "max_time": "1585565187347", 76 | # "live_sdk_version": "827", 77 | # "iid": "104447577985", 78 | # "device_id": "70837664152", 79 | # "ac": "wifi", 80 | # "channel": "tengxun_new", 81 | "aid": "1112", # 这个参数似乎很重要 82 | # "app_name": "live_stream", 83 | # "version_code": "827", 84 | # "version_name": "8.2.7", 85 | # "device_platform": "android", # 设备平台 86 | # "ssmix": "a", 87 | # "device_type": "MI+5s", # 设备类型 88 | # "device_brand": "Xiaomi", # 设备品牌 89 | # "language": "zh", # 语言 90 | # "os_api": "23", 91 | # "os_version": "6.0.1", 92 | # "uuid": "490000000085156", 93 | # "openudid": "b8db9c78de7c82b5", 94 | # "manifest_version_code": "827", 95 | # "resolution": "810*1440", 96 | # "dpi": "270", 97 | # "update_version_code": "8270", 98 | # "_rticket": "1585567014625", 99 | # "ab_version": "1413809%2C1244214%2C889330%2C1138752%2C1589082%2C1063522%2C1480776%2C1377092%2C1380327%2C1582436%2C1588775%2C1167795%2C1476946%2C1404472%2C1517651%2C1354483%2C1479194%2C1258912%2C1264664%2C1521584%2C955276%2C1589846%2C947985%2C1548003%2C1182061%2C1480948%2C1435640%2C1477984%2C929432%2C1490515%2C1432944%2C1555350%2C1590375%2C1541253%2C1540549%2C1428670%2C1048435%2C1168129%2C1396601%2C1582073%2C1549345%2C1396899%2C1096187%2C1104584%2C1478759%2C1419023%2C1538832%2C1548270%2C1565149%2C1496674%2C1550828%2C1568912%2C1574488%2C1581133%2C1580160%2C1320817%2C1133591%2C692223%2C1169771%2C956107%2C1247692%2C1019139%2C682009%2C1032070%2C1165214%2C1265052%2C1584527%2C1072545%2C1317441%2C1562047%2C1069233%2C1583612%2C1143559%2C1544623%2C1337822%2C1293405%2C1347260%2C1046183%2C1354701%2C1143672%2C1498072%2C1143730%2C1417290%2C1491283%2C1165209%2C1576837%2C1376626%2C1572549%2C1409058%2C1502675%2C1578552%2C1050089", 100 | # "client_version_code": "827", 101 | # "jssdk_version": "1.37.1.2", 102 | # "mcc_mnc": "46005", 103 | # "cdid": "64d624fa-4514-401c-99d5-95e8a05b0c83", 104 | # "new_nav": "1", 105 | # "ws_status": "CONNECTED", 106 | # "settings_version": "19", 107 | # "last_update_time": "1585565111754", 108 | # "mac_address": "08%3A00%3A27%3A3E%3A23%3A0D", 109 | # "ts": "1585567014", 110 | } 111 | data = requests.get(url, params=params, headers=self.headers).json() 112 | return data 113 | 114 | def get_video_list(self): 115 | data_list = self.get_info()["data"] 116 | video_list = [] 117 | for data in data_list: 118 | item = { 119 | "url": data["data"]["video"]["download_url"][0], 120 | "title": data["data"]["title"] 121 | } 122 | video_list.append(item) 123 | return video_list 124 | 125 | def download_video_list(self, folder="huoshan"): 126 | """ 127 | 下载视频列表 128 | :param folder: 存放的文件夹, 默认为 ./BASE_DIR/huoshan 129 | :return: 130 | """ 131 | # 1. 判断文件夹是否存在，不存在则创建 132 | video_folder = "%s/%s" % (BASE_DIR, folder) 133 | if not os.path.exists(video_folder): 134 | os.mkdir(video_folder) 135 | video_list = self.get_video_list() 136 | for video in video_list: 137 | file_name = "%s/%s.mp4" % (video_folder, video["title"]) 138 | # 过滤文件名中的无效字符 139 | file_name = name_manager.filter_name(file_name) 140 | # 当文件不存在的时候进行下载 141 | if not os.path.exists(file_name): 142 | self.downloader.download(video["url"], file_name) 143 | 144 | if __name__ == '__main__': 145 | h = HuoShan() 146 | # h.get_item_info() 147 | # print(json.dumps(h.get_info(), ensure_ascii=False)) 148 | # print(h.get_video_list()) 149 | for _ in range(10): 150 | h.download_video_list() 151 | -------------------------------------------------------------------------------- /images/res01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAJINZI01/mobile_video/4a224c5b66e191bbf084a115286b51ca2b0b264a/images/res01.png -------------------------------------------------------------------------------- /kuaishou_spider.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | from settings import BASE_DIR 3 | from utils import download, name_manager 4 | 5 | import requests 6 | import os 7 | import json 8 | 9 | 10 | class KuaiShou(object): 11 | def __init__(self): 12 | self.headers = { 13 | "User-Agent": "kwai-android aegon/1.10.2-curl", 14 | "Host": "apissl.gifshow.com", 15 | "Cookie": "region_ticket=RT_B5890E16417BC4E5595F7EDB6E188572806692681C8F352D1488433228783", 16 | "X-REQUESTID": "158562636964822384", 17 | } 18 | self.downloader = download.MyDownloader(bar_symbol="c") 19 | 20 | def get_info(self): 21 | """ 22 | 根据接口，获取所有的详细信息 23 | :return: 24 | """ 25 | url = "https://apissl.gifshow.com/rest/n/feed/hot" 26 | params = { 27 | "mod": "vivo(vivo X9Plus)", 28 | "lon": "121.492379", 29 | "country_code": "cn", 30 | "extId": "bc70d48c39d1f18a8713e4cb3f8a800c", 31 | "kpn": "KUAISHOU", 32 | "oc": "360APP,1", 33 | "egid": "DFP058657B907ED8C6D9F0E270AAD8E19C6DF413211D9C0E53D8F775812D4F1F", 34 | "hotfix_ver": "", 35 | "sh": "1920", 36 | "appver": "7.2.2.12969", 37 | "socName": "UNKNOWN", 38 | "max_memory": "192", 39 | "isp": "CMCC", 40 | "kcv": "188", 41 | "browseType": "1", 42 | "kpf": "ANDROID_PHONE", 43 | "did": "ANDROID_c50c87e4562bf3f9", 44 | "net": "WIFI", 45 | "app": "0", 46 | "ud": "0", 47 | "c": "360APP,1", 48 | "sys": "ANDROID_5.1.1", 49 | "sw": "1080", 50 | "ftt": "", 51 | "ll": "CTTVk/lHPz9AES6PNSODX15A", 52 | "language": "zh-cn", 53 | "iuid": "", 54 | "lat": "31.247192", 55 | "did_gt": "1585622393761", 56 | "ver": "7.2", 57 | } 58 | form = { 59 | "type": "7", 60 | "page": "2", 61 | "coldStart": "false", 62 | "count": "20", 63 | "pv": "false", 64 | "id": "13", 65 | "refreshTimes": "1", 66 | "pcursor": "1", 67 | "source": "1", 68 | "extInfo": "TXURYODnxuqz8vLON/bBsj+X/NITG1FIT5ubnS+Bf39hSC0TLguH3XERnIrzRNPlxqxQcdIb58jND7AJFA6LgYnjjyDXznlJwi/wK8LxROT5tDiTQleU9pi/0VobR39E552kiUZ/aIMYbudUt7J/UQ==", 69 | "needInterestTag": "false", 70 | "seid": "f40304fb-d089-498a-91ad-ee6031186711", 71 | "volume": "0.27", 72 | "backRefresh": "false", 73 | "pageCount": "2", 74 | "adChannel": '{"type":1,"data":""}', 75 | "passThrough": "0", 76 | "thanosSpring": "false", 77 | "newUserRefreshTimes": "15", 78 | "newUserAction": '{"click":[5189835650000717127,5243315891402845228],"follow":[],"like":[]}', 79 | "cellList": '[{"ci":53185,"lac":6311,"mcc":460,"mnc":0,"radio":"gsm","rssi":0}]', 80 | "__NS_sig3": "2202972399d8693c1e9e2b081d7e7f5bc60c03c6dc", 81 | "client_key": "3c2cd3f3", 82 | "os": "android", 83 | "sig": "d695f07ae9d37816fdf4de698e659ab6", 84 | } 85 | return requests.post(url, params=params, data=form, headers=self.headers).json() 86 | 87 | def get_video_list(self): 88 | """ 89 | 获取视频列表 90 | :return: 列表 [{"caption": "xxx", "url": "xxx"}, ...] 91 | """ 92 | data_list = self.get_info()["feeds"] 93 | video_list = [] 94 | for data in data_list: 95 | if data.get("main_mv_urls"): # 有的没有视频链接 96 | item = { 97 | "caption": data["caption"], 98 | "url": data["main_mv_urls"][0]["url"], 99 | } 100 | video_list.append(item) 101 | return video_list 102 | 103 | def download_video_list(self, folder="kuaishou"): 104 | """ 105 | 下载视频列表 106 | :param folder: 下载存放的文件夹 107 | :return: 108 | """ 109 | # 1. 创建文件夹 110 | video_folder = "%s/%s" % (BASE_DIR, folder) 111 | if not os.path.exists(video_folder): 112 | os.mkdir(video_folder) 113 | # 2. 遍历视频列表，下载视频 114 | video_list = self.get_video_list() 115 | for video in video_list: 116 | file_name = "%s/%s.mp4" % (video_folder, video["caption"]) 117 | file_name = name_manager.filter_name(file_name) 118 | # 3. 如果视频文件不存在则下载视频 119 | if not os.path.exists(file_name): 120 | self.downloader.download(video["url"], file_name) 121 | 122 | 123 | if __name__ == '__main__': 124 | k = KuaiShou() 125 | # print(json.dumps(k.get_info(), ensure_ascii=False)) 126 | # print(k.get_video_list()) 127 | k.download_video_list() 128 | -------------------------------------------------------------------------------- /settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | BASE_DIR = "./data" 4 | if not os.path.exists(BASE_DIR): 5 | os.mkdir(BASE_DIR) -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAJINZI01/mobile_video/4a224c5b66e191bbf084a115286b51ca2b0b264a/utils/__init__.py -------------------------------------------------------------------------------- /utils/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAJINZI01/mobile_video/4a224c5b66e191bbf084a115286b51ca2b0b264a/utils/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /utils/__pycache__/download.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAJINZI01/mobile_video/4a224c5b66e191bbf084a115286b51ca2b0b264a/utils/__pycache__/download.cpython-36.pyc -------------------------------------------------------------------------------- /utils/__pycache__/name_manager.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAJINZI01/mobile_video/4a224c5b66e191bbf084a115286b51ca2b0b264a/utils/__pycache__/name_manager.cpython-36.pyc -------------------------------------------------------------------------------- /utils/__pycache__/verify_params.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAJINZI01/mobile_video/4a224c5b66e191bbf084a115286b51ca2b0b264a/utils/__pycache__/verify_params.cpython-36.pyc -------------------------------------------------------------------------------- /utils/download.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | from datetime import datetime, timedelta 3 | import requests 4 | 5 | 6 | class MyDownloader(object): 7 | def __init__(self, headers=None, progress_bar_len=30, bar_symbol="="): 8 | """ 9 | 下载器的初始化方法 10 | :param headers: 可以添加，下载要求的请求头，没有的话，使用自带的 user-agent 头 11 | :param progress_bar_len: 进度条的长度, 默认为 30 12 | :param bar_symbol: 进度条显示的符号，默认为 = 13 | """ 14 | if not headers: 15 | self.headers = { 16 | "user-agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36", 17 | } 18 | self.progress_bar_len = progress_bar_len 19 | self.bar_symbol = bar_symbol 20 | 21 | def show_line(self, current_size, total_size, start_time): 22 | """ 23 | 显示一行的下载信息 24 | :param current_size: 当前下载的大小 25 | :param total_size: 总的文件大小 26 | :param start_time: 下载的开始时间 27 | :return: 28 | """ 29 | # 进度条 30 | progress_percent = current_size / total_size # 进度百分比 31 | progress_bar = "[%s%s]" % (self.bar_symbol*int((self.progress_bar_len*progress_percent)), 32 | " "*int((self.progress_bar_len*(1-progress_percent)))) 33 | # 百分比 34 | percent_bar = "%.2fM/%.2fM" % (current_size/1024/1024, total_size/1024/1024) 35 | # 时间 36 | # time_bar = time.strftime("%H:%M:%S", time.localtime(time.time()-start_time)) 37 | time_bar = str(datetime.now() - start_time) 38 | print("\r%s %s %s" % (progress_bar, percent_bar, time_bar), end="") 39 | 40 | def download(self, url, file_name="out"): 41 | """ 42 | 下载器 43 | :param url: 下载的链接 44 | :param file_name: 下载的文件名, 如果没有传，使用out 45 | :return: 46 | """ 47 | # 1. 发送请求 48 | response = requests.get(url, headers=self.headers, stream=True) 49 | # 2. 获取文件大小 50 | file_size = int(response.headers["Content-Length"]) 51 | current_size = 0 52 | # 3. 读取文件写入到本地 53 | f = open(file_name, "wb") 54 | now = datetime.now() 55 | print("%s download..." % file_name) 56 | for chunk in response.iter_content(chunk_size=1024): 57 | current_size += f.write(chunk) 58 | self.show_line(current_size, file_size, now) 59 | f.close() 60 | print("[ok]") 61 | 62 | 63 | if __name__ == '__main__': 64 | d = MyDownloader() 65 | # d.show_line(10000, 100000, datetime.now() - timedelta(seconds=10)) 66 | url = "http://v3-default.ixigua.com/77d725434b95f0fede386b5d20cf7cab/5e81c988/video/tos/cn/tos-cn-v-0015c002/05464977975a46dabc084efb64806651/?a=0&br=0&bt=9960&cr=3&cs=0&dr=0&ds=3&er=&l=202003301727010101440620382F08CFD7&lr=all&qs=13&rc=anBxbGV5O2xzdDMzN2kzM0ApPHYzaHg1ZWQzZTMzOTM1eWc2cC1ici9fXi1fLS0zLS9zc2Rqa2ZfczVmbW4tLTFjLS06Yw%3D%3D&vl=&vr=" 67 | d.download(url) 68 | 69 | -------------------------------------------------------------------------------- /utils/name_manager.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | import re 3 | 4 | def filter_name(name): 5 | """ 6 | 过滤文件名中的无效字符 7 | :param name: 8 | :return: 9 | """ 10 | return re.sub(r"[\n]", "-", name) 11 | -------------------------------------------------------------------------------- /utils/out.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAJINZI01/mobile_video/4a224c5b66e191bbf084a115286b51ca2b0b264a/utils/out.mp4 -------------------------------------------------------------------------------- /utils/verify_params.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | import requests 3 | 4 | 5 | def verify(url, params, haskey): 6 | """ 7 | 验证请求的url参数，取出必要参数 8 | :param url: 要验证的url 9 | :param params: 要验证的参数 10 | :param haskey: 验证的属性路径，匹配是否有这个路径下的属性 11 | :return: 12 | """ 13 | verify_params = {} 14 | # 1. 解析 haskey 属性路径 15 | key_list = haskey.split(".") 16 | # 2. 循环比较 删除不影响判断的key 17 | keys = list(params.keys()) 18 | for k in keys: 19 | v = params.pop(k) 20 | try: 21 | # 发送请求进行验证 22 | data = requests.get(url, params=params).json() 23 | # 沿着属性路径查找 24 | for key in key_list: 25 | if key.isdigit(): 26 | key = int(key) 27 | data = data[key] 28 | except: 29 | # 出现异常表示，这个参数不能剔除，是必须的参数 30 | params[k] = v 31 | verify_params[k] = v 32 | return verify_params 33 | 34 | if __name__ == '__main__': 35 | url = "https://hotsoon-hl.snssdk.com/hotsoon/feed/" 36 | params = { 37 | "type": "video", 38 | "tab_id": "5", 39 | "front_ids": "6809869449601109263%2C6809893253329095944%2C6808726733139889421%2C6801421644104617216%2C1634668034797582%2C6800306668589698304%2C6799054861905317135%2C6806037711485553924", 40 | "last_ad_items": "%5B%7B%22id%22%3A6803159557326654724%2C%22show_time%22%3A1585565114%7D%2C%7B%22id%22%3A6800667492667772173%2C%22show_time%22%3A1585565114%7D%2C%7B%22id%22%3A1662299439581208%2C%22show_time%22%3A1585565114%7D%2C%7B%22id%22%3A4760%2C%22show_time%22%3A1585565114%7D%2C%7B%22id%22%3A6805071744987188480%2C%22show_time%22%3A1585565181%7D%2C%7B%22id%22%3A6805102455383280908%2C%22show_time%22%3A1585565181%7D%2C%7B%22id%22%3A6804693165258460424%2C%22show_time%22%3A1585565182%7D%2C%7B%22id%22%3A6804840893229714695%2C%22show_time%22%3A1585565182%7D%2C%7B%22id%22%3A1662462880442372%2C%22show_time%22%3A1585567006%7D%5D", 41 | "n_viewed": "0", 42 | "offset": "18", 43 | "diff_stream": "1", 44 | "ad_user_agent": "com.ss.android.ugc.live%2F827+%28Linux%3B+U%3B+Android+6.0.1%3B+zh_CN%3B+MI+5s%3B+Build%2FV417IR%3B+Chrome%29", 45 | "req_from": "feed_loadmore", 46 | "count": "10", 47 | "secs_video_watching": "1811", 48 | "n_skipped": "0", 49 | "minor_control_status": "0", 50 | "feed_video_gap": "140", 51 | "max_time": "1585565187347", 52 | "live_sdk_version": "827", 53 | "iid": "104447577985", 54 | "device_id": "70837664152", 55 | "ac": "wifi", 56 | "channel": "tengxun_new", 57 | "aid": "1112", 58 | "app_name": "live_stream", 59 | "version_code": "827", 60 | "version_name": "8.2.7", 61 | "device_platform": "android", 62 | "ssmix": "a", 63 | "device_type": "MI+5s", 64 | "device_brand": "Xiaomi", 65 | "language": "zh", 66 | "os_api": "23", 67 | "os_version": "6.0.1", 68 | "uuid": "490000000085156", 69 | "openudid": "b8db9c78de7c82b5", 70 | "manifest_version_code": "827", 71 | "resolution": "810*1440", 72 | "dpi": "270", 73 | "update_version_code": "8270", 74 | "_rticket": "1585567014625", 75 | "ab_version": "1413809%2C1244214%2C889330%2C1138752%2C1589082%2C1063522%2C1480776%2C1377092%2C1380327%2C1582436%2C1588775%2C1167795%2C1476946%2C1404472%2C1517651%2C1354483%2C1479194%2C1258912%2C1264664%2C1521584%2C955276%2C1589846%2C947985%2C1548003%2C1182061%2C1480948%2C1435640%2C1477984%2C929432%2C1490515%2C1432944%2C1555350%2C1590375%2C1541253%2C1540549%2C1428670%2C1048435%2C1168129%2C1396601%2C1582073%2C1549345%2C1396899%2C1096187%2C1104584%2C1478759%2C1419023%2C1538832%2C1548270%2C1565149%2C1496674%2C1550828%2C1568912%2C1574488%2C1581133%2C1580160%2C1320817%2C1133591%2C692223%2C1169771%2C956107%2C1247692%2C1019139%2C682009%2C1032070%2C1165214%2C1265052%2C1584527%2C1072545%2C1317441%2C1562047%2C1069233%2C1583612%2C1143559%2C1544623%2C1337822%2C1293405%2C1347260%2C1046183%2C1354701%2C1143672%2C1498072%2C1143730%2C1417290%2C1491283%2C1165209%2C1576837%2C1376626%2C1572549%2C1409058%2C1502675%2C1578552%2C1050089", 76 | "client_version_code": "827", 77 | "jssdk_version": "1.37.1.2", 78 | "mcc_mnc": "46005", 79 | "cdid": "64d624fa-4514-401c-99d5-95e8a05b0c83", 80 | "new_nav": "1", 81 | "ws_status": "CONNECTED", 82 | "settings_version": "19", 83 | "last_update_time": "1585565111754", 84 | "mac_address": "08%3A00%3A27%3A3E%3A23%3A0D", 85 | "ts": "1585567014", 86 | } 87 | print(verify(url, params, "data.0.data.video.download_url")) 88 | --------------------------------------------------------------------------------