├── README.md
├── ximalaya_search.py
└── ximalaya_vip.py
/README.md:
--------------------------------------------------------------------------------
1 | # XimalayaFM
2 |
3 | python爬取喜马拉雅音频
4 |
5 | ## TODO
6 |
7 | * 写一个UI界面
8 | * 提供多种爬取选项
9 |
10 | ## 2019-10-12 19:10
11 |
12 | [CSDN](https://blog.csdn.net/weixin_42050513/article/details/101224552)上有人评论说`xm-sign`规则改了，于是去看了看，发现实际只改了一个字母，整体流程任可以看下方正文
13 |
14 | 把`hashlib.md5("ximalaya-{}".format(servertime).encode()`中的`ximalaya`替换成`himalaya`即可
15 |
16 | 改动点如下图
17 |
18 | 
19 |
20 | 
21 |
22 | 改动不大，可能是发现了有人在大量爬取，先小地方改动，随后可能会有较大的规则改动，教程写出来的目的是学习、测试，切勿过度爬取！
23 |
24 | ## 2020-02-02
25 |
26 | 完成VIP音频下载功能
27 |
28 | 使用方法：
29 |
30 | * 首先你已经开通了喜马会员
31 | * 该音频属于会员或者付费可听
32 | * 运行程序，选择`VIP`选项，然后输入音频集的albumID，以及你的token，点击运行即可
33 |
34 | token在下图这里复制
35 |
36 | 
37 |
38 | 就是`1&_token=xxx`，这一部分，不用加`;`
39 |
40 | 代码主要是通过`Scapy`实现的功能，对，`不是Scrapy`，Scapy具有模拟发送数据包、监听解析数据包、互联网协议解析、数据挖掘等多种用处
41 |
42 | 然后发现scapy-http这个模块，二者配合使用后，可以解析抓到的包的url等参数
43 |
44 | ### 安装工具
45 | ``` bash
46 | pip3 install scapy
47 |
48 | pip3 install scapy-http
49 | ```
50 |
51 | 还要安装winpcap软件，为监控网卡提供接口，[下载地址](https://www.winpcap.org/install/default.htm)
52 |
53 | 注意替换代码中的iface：[iface 参数为你要监听的网卡的名称，参考这里](https://blog.csdn.net/luanpeng825485697/article/details/78379154)
54 |
55 | ## Stargazers over time
56 |
57 | [](https://starchart.cc/joelYing/XimalayaFM)
58 |
--------------------------------------------------------------------------------
/ximalaya_search.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | # -*- coding:utf-8 -*-
3 | # author:joel 19-9-22
4 |
5 | import hashlib
6 | import json
7 | import os
8 | import re
9 | import time
10 | import random
11 | import requests
12 |
13 | """
14 | 注意运行前请修改 make_dir() 中的下载路径！不要过度爬取，仅供测试学习！
15 | """
16 |
17 |
18 | class XiMa(object):
19 | def __init__(self):
20 | self.base_url = 'https://www.ximalaya.com'
21 | self.base_api = 'https://www.ximalaya.com/revision/play/album?albumId={}&pageNum={}&sort=0&pageSize=30'
22 | self.time_api = 'https://www.ximalaya.com/revision/time'
23 | self.header = {
24 | 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:63.0) Gecko/20100101 Firefox/63.0'
25 | }
26 | self.s = requests.session()
27 |
28 | def get_time(self):
29 | """
30 | 获取服务器时间戳
31 | :return:
32 | """
33 | r = self.s.get(self.time_api, headers=self.header)
34 | return r.text
35 |
36 | def get_sign(self):
37 | """
38 | 获取sign： md5(ximalaya-服务器时间戳)(100以内随机数)服务器时间戳(100以内随机数)现在时间戳
39 | :return: xm_sign
40 | """
41 | nowtime = str(round(time.time() * 1000))
42 | servertime = self.get_time()
43 | sign = str(hashlib.md5("himalaya-{}".format(servertime).encode()).hexdigest()) + "({})".format(
44 | str(round(random.random() * 100))) + servertime + "({})".format(str(round(random.random() * 100))) + nowtime
45 | self.header["xm-sign"] = sign
46 | # print(sign)
47 | # return sign
48 |
49 | def index_choose(self):
50 | c_num = input(u'请输入对应操作的选项：\n'
51 | u'1、下载整部有声书\n'
52 | u'2、下载单个音源\n'
53 | u'3、返回\n')
54 | if c_num == '1':
55 | xm_id = input(u'请输入要获取喜马拉雅节目的ID：')
56 | xima.get_fm(xm_id)
57 | self.index_choose()
58 | elif c_num == '2':
59 | xm_id = input(u'请输入要获取的音源：')
60 | print(xm_id)
61 | self.index_choose()
62 | elif c_num == '3':
63 | print('结束')
64 | else:
65 | pass
66 |
67 | @staticmethod
68 | def make_dir(xm_fm_id):
69 | # 保存路径，请自行修改，这里是以有声书ID作为文件夹的路径
70 | fm_path = 'F:\\{}\\'.format(xm_fm_id)
71 | f = os.path.exists(fm_path)
72 | if not f:
73 | os.makedirs(fm_path)
74 | print('make file success...')
75 | else:
76 | print('file already exists...')
77 | return fm_path
78 |
79 | def get_fm(self, xm_fm_id):
80 | # 根据有声书ID构造url
81 | fm_url = self.base_url + '/youshengshu/{}'.format(xm_fm_id)
82 | print(fm_url)
83 | r_fm_url = self.s.get(fm_url, headers=self.header)
84 | fm_title = re.findall('(.*?)
', r_fm_url.text, re.S)[0]
85 | print('书名：' + fm_title)
86 | # 新建有声书ID的文件夹
87 | fm_path = self.make_dir(xm_fm_id)
88 | # 取最大页数
89 | max_page = re.findall(r'', r_fm_url.text, re.S)
91 | if max_page and max_page[0]:
92 | for page in range(1, int(max_page[0]) + 1):
93 | print('第' + str(page) + '页')
94 | self.get_sign()
95 | r = self.s.get(self.base_api.format(xm_fm_id, page), headers=self.header)
96 | # print(json.loads(r.text))
97 | r_json = json.loads(r.text)
98 | for audio in r_json['data']['tracksAudioPlay']:
99 | audio_title = str(audio['trackName']).replace(' ', '')
100 | audio_src = audio['src']
101 | self.get_detail(audio_title, audio_src, fm_path)
102 | # 每爬取1页，30个音频，休眠3秒
103 | time.sleep(3)
104 | else:
105 | print(os.error)
106 |
107 | def get_detail(self, title, src, path):
108 | r_audio_src = self.s.get(src, headers=self.header)
109 | m4a_path = path + title + '.m4a'
110 | if not os.path.exists(m4a_path):
111 | with open(m4a_path, 'wb') as f:
112 | f.write(r_audio_src.content)
113 | print(title + '保存完毕...')
114 | else:
115 | print(title + 'm4a已存在')
116 |
117 |
118 | if __name__ == '__main__':
119 | xima = XiMa()
120 | xima.index_choose()
121 |
--------------------------------------------------------------------------------
/ximalaya_vip.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | # -*- coding:utf-8 -*-
3 | # author:joel 18-6-5
4 |
5 | import sys
6 | import hashlib
7 | import json
8 | import math
9 | import os
10 | import random
11 | import time
12 | import requests
13 | from scapy.layers.inet import TCP
14 | from scapy.all import sniff
15 | from scapy_http import http
16 | from selenium import webdriver
17 | from selenium.webdriver.chrome.options import Options
18 | from PyQt5.QtWidgets import QFileDialog, QMainWindow, QApplication
19 | from PyQt5 import QtCore, QtGui, QtWidgets
20 |
21 |
22 | class Ui_XiMaDownloader(object):
23 | def setupUi(self, XiMaDownloader):
24 | XiMaDownloader.setObjectName("XiMaDownloader")
25 | XiMaDownloader.resize(600, 563)
26 | self.centralwidget = QtWidgets.QWidget(XiMaDownloader)
27 | self.centralwidget.setObjectName("centralwidget")
28 | self.title_label = QtWidgets.QLabel(self.centralwidget)
29 | self.title_label.setGeometry(QtCore.QRect(20, 20, 191, 41))
30 | self.title_label.setLineWidth(1)
31 | self.title_label.setScaledContents(False)
32 | self.title_label.setAlignment(QtCore.Qt.AlignCenter)
33 | self.title_label.setWordWrap(True)
34 | self.title_label.setIndent(-1)
35 | self.title_label.setObjectName("title_label")
36 | self.verticalGroupBox = QtWidgets.QGroupBox(self.centralwidget)
37 | self.verticalGroupBox.setGeometry(QtCore.QRect(0, 0, 601, 71))
38 | self.verticalGroupBox.setAutoFillBackground(True)
39 | self.verticalGroupBox.setStyleSheet("")
40 | self.verticalGroupBox.setObjectName("verticalGroupBox")
41 | self.verticalLayout = QtWidgets.QVBoxLayout(self.verticalGroupBox)
42 | self.verticalLayout.setObjectName("verticalLayout")
43 | self.choose_label = QtWidgets.QLabel(self.centralwidget)
44 | self.choose_label.setGeometry(QtCore.QRect(10, 80, 201, 31))
45 | self.choose_label.setObjectName("choose_label")
46 | self.free_fm = QtWidgets.QCheckBox(self.centralwidget)
47 | self.free_fm.setGeometry(QtCore.QRect(20, 140, 91, 16))
48 | self.free_fm.setObjectName("free_fm")
49 | self.vip_fm = QtWidgets.QCheckBox(self.centralwidget)
50 | self.vip_fm.setGeometry(QtCore.QRect(150, 140, 111, 16))
51 | self.vip_fm.setObjectName("vip_fm")
52 | self.single_fm = QtWidgets.QCheckBox(self.centralwidget)
53 | self.single_fm.setGeometry(QtCore.QRect(290, 140, 71, 16))
54 | self.single_fm.setObjectName("single_fm")
55 | self.id_label = QtWidgets.QLabel(self.centralwidget)
56 | self.id_label.setGeometry(QtCore.QRect(20, 190, 171, 41))
57 | self.id_label.setObjectName("id_label")
58 | self.id_input_line = QtWidgets.QLineEdit(self.centralwidget)
59 | self.id_input_line.setGeometry(QtCore.QRect(200, 200, 133, 20))
60 | self.id_input_line.setObjectName("id_input_line")
61 | self.token_label = QtWidgets.QLabel(self.centralwidget)
62 | self.token_label.setGeometry(QtCore.QRect(20, 250, 161, 41))
63 | self.token_label.setObjectName("token_label")
64 | self.token_input_line = QtWidgets.QLineEdit(self.centralwidget)
65 | self.token_input_line.setGeometry(QtCore.QRect(200, 260, 361, 20))
66 | self.token_input_line.setObjectName("token_input_line")
67 | self.d_type_box = QtWidgets.QGroupBox(self.centralwidget)
68 | self.d_type_box.setGeometry(QtCore.QRect(10, 120, 581, 51))
69 | self.d_type_box.setObjectName("d_type_box")
70 | self.path_input_line = QtWidgets.QLineEdit(self.centralwidget)
71 | self.path_input_line.setGeometry(QtCore.QRect(200, 230, 133, 20))
72 | self.path_input_line.setObjectName("path_input_line")
73 | self.path_label = QtWidgets.QLabel(self.centralwidget)
74 | self.path_label.setGeometry(QtCore.QRect(20, 220, 171, 41))
75 | self.path_label.setObjectName("path_label")
76 | self.d_config_box = QtWidgets.QGroupBox(self.centralwidget)
77 | self.d_config_box.setGeometry(QtCore.QRect(10, 180, 581, 111))
78 | self.d_config_box.setObjectName("d_config_box")
79 | self.choose_file_button = QtWidgets.QPushButton(self.d_config_box)
80 | self.choose_file_button.setGeometry(QtCore.QRect(340, 50, 75, 23))
81 | self.choose_file_button.setObjectName("choose_file_button")
82 | self.run_button = QtWidgets.QPushButton(self.centralwidget)
83 | self.run_button.setGeometry(QtCore.QRect(510, 320, 75, 23))
84 | self.run_button.setObjectName("run_button")
85 | self.cancel_button = QtWidgets.QPushButton(self.centralwidget)
86 | self.cancel_button.setGeometry(QtCore.QRect(510, 360, 75, 23))
87 | self.cancel_button.setObjectName("cancel_button")
88 | self.output_text = QtWidgets.QTextBrowser(self.centralwidget)
89 | self.output_text.setGeometry(QtCore.QRect(10, 300, 491, 241))
90 | self.output_text.setObjectName("output_text")
91 | self.d_config_box.raise_()
92 | self.d_type_box.raise_()
93 | self.verticalGroupBox.raise_()
94 | self.title_label.raise_()
95 | self.choose_label.raise_()
96 | self.free_fm.raise_()
97 | self.vip_fm.raise_()
98 | self.single_fm.raise_()
99 | self.id_label.raise_()
100 | self.id_input_line.raise_()
101 | self.token_label.raise_()
102 | self.token_input_line.raise_()
103 | self.path_input_line.raise_()
104 | self.path_label.raise_()
105 | self.run_button.raise_()
106 | self.cancel_button.raise_()
107 | self.output_text.raise_()
108 | XiMaDownloader.setCentralWidget(self.centralwidget)
109 | self.statusbar = QtWidgets.QStatusBar(XiMaDownloader)
110 | self.statusbar.setObjectName("statusbar")
111 | XiMaDownloader.setStatusBar(self.statusbar)
112 |
113 | self.retranslateUi(XiMaDownloader)
114 | self.cancel_button.clicked.connect(XiMaDownloader.close)
115 | QtCore.QMetaObject.connectSlotsByName(XiMaDownloader)
116 |
117 | def retranslateUi(self, XiMaDownloader):
118 | _translate = QtCore.QCoreApplication.translate
119 | XiMaDownloader.setWindowTitle(_translate("XiMaDownloader", "XiMaFM下载器"))
120 | self.title_label.setText(_translate("XiMaDownloader", "