歌曲搜素
网易云音乐网址为：/
思路是进入后输入一个歌曲名，点击搜索按钮，通过开发者调试工具捕获搜索请求，捕获到的数据信息如下：
所有的歌曲相关信息都在result里面，每个a标签（超链接）里面携带着大量信息，包括歌曲名，id号，演唱者等等，这里最需要的最关键的一个信息就是它的id号，因为歌曲名作者可能会重复，但是id是唯一的，可以通过id来获取该歌曲其他信息。
再查看该请求的URL，/weapi/cloudsearch/get/web?csrf_token=，可以通过此URL来获取歌曲的id
获取到id后如何搜索呢，以第一首歌曲为例，进入到详细页面：
发现网址后面只需要填写id号再做一些字符串的拼接即可
那这样一来就可以实现了，搜索歌曲的id：
import requestsfrom Netease_cloud_comment_capture.Encrypt import Encryptedclass search():'''跟歌单直接下载的不同之处，1.就是headers的referer2.加密的text内容不一样！3.搜索的URL也是不一样的输入搜索内容，可以获取歌曲ID'''def __init__(self):self.headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36','Host': '','Referer': '/search/'}self.main_url='/'self.session = requests.Session()self.session.headers=self.headersself.ep = Encrypted()def search_song(self, search_content, search_type=1, limit=9):"""根据音乐名搜索:params search_content: 音乐名:params search_type: 不知:params limit: 返回结果数量return: 可以得到id 再进去歌曲具体的url"""url = '/weapi/cloudsearch/get/web?csrf_token='text = {'s': search_content, 'type': search_type, 'offset': 0, 'sub': 'false', 'limit': limit}data = self.ep.search(text)resp = self.session.post(url, data=data)result = resp.json()if result['result']['songCount']<= 0:print('搜不到！！')else:songs = result['result']['songs']for song in songs:song_id = song['id']return song_id
内容解析
在歌曲详情页面里，可以继续利用调试工具抓取评论对应的数据包，可以发现要抓取的评论在data里面的comments放着
那这样就可以通过表单获取评论内容了
但是通过数据包头发现，表单的数据是加密的，params和encSecKey需要进行解码。
查了一下是AES加密，现在以及有很多解密方法了，那加密问题也破解了：
import base64import jsonimport osfrom binascii import hexlifyfrom Crypto.Cipher import AESsecond_param = "010001"third_param = "00e0b509f6259df8642dbc35662901477df22677ec152b5ff68ace615bb7b725152b3ab17a876aea8a5aa76d2e417629ec4ee341f56135fccf695280104e0312ecbda92557c93870114af6c9d05c4f7f0c3685b7a46bee255932575cce10b424d813cfe4875d3e82047b97ddef52741d546b8e289dc6935b3ece0462db0a22b8e7"forth_param = "0CoJUm6Qyw8W8jud"# 加密解密class Encrypted():'''传入歌曲的ID，加密生成'params'、'encSecKey 返回'''def __init__(self):self.pub_key = second_paramself.modulus = third_paramself.nonce = forth_paramdef create_secret_key(self, size):return hexlify(os.urandom(size))[:16].decode('utf-8')def aes_encrypt(self, text, key):iv = '0102030405060708'pad = 16 - len(text) % 16text = text + pad * chr(pad)encryptor = AES.new(key.encode('utf-8'), AES.MODE_CBC, iv.encode('utf-8'))result = encryptor.encrypt(text.encode('utf-8'))result_str = base64.b64encode(result).decode('utf-8')return result_strdef rsa_encrpt(self, text, pubKey, modulus):text = text[::-1]rs = pow(int(hexlify(text.encode('utf-8')), 16), int(pubKey, 16), int(modulus, 16))return format(rs, 'x').zfill(256)def search(self, text):text = json.dumps(text)i = self.create_secret_key(16)encText = self.aes_encrypt(text, self.nonce)encText = self.aes_encrypt(encText, i)encSecKey = self.rsa_encrpt(i, self.pub_key, self.modulus)data = {'params': encText, 'encSecKey': encSecKey}return data
import requests, base64from Crypto.Cipher import AESfrom Netease_cloud_comment_capture.Encrypt import forth_paramdef get_params(page): # page为传入页数iv = "0102030405060708"first_key = forth_paramsecond_key = 16 * 'F'if(page == 1): # 如果为第一页first_param = '{rid:"", offset:"0", total:"true", limit:"20", csrf_token:""}'h_encText = AES_encrypt(first_param, first_key, iv)else:offset = str((page - 1) * 20)first_param = '{rid:"", offset:"%s", total:"%s", limit:"20", csrf_token:""}' %(offset, 'false')h_encText = AES_encrypt(first_param, first_key, iv)h_encText = AES_encrypt(h_encText, second_key, iv)return h_encTextdef get_encSecKey():encSecKey = "257348aecb5e556c066de214e531faadd1c55d814f9be95fd06d6bff9f4c7a41f831f6394d5a3fd2e3881736d94a02ca919d952872e7d0a50ebfa1769a7a62d512f5f1ca21aec60bc3819a9c3ffca5eca9a0dba6d6f7249b06f5965ecfff3695b54e1c28f3f624750ed39e7de08fc8493242e26dbc4484a01c76f739e135637c"return encSecKeydef AES_encrypt(text, key, iv):if type(text) == type(b'123'):text = text.decode('utf-8')# text=text.decode('utf-8')pad = 16 - len(text) % 16text = text + pad * chr(pad)iv = iv.encode('utf-8')key = key.encode('utf-8')encryptor = AES.new((key), AES.MODE_CBC, (iv))text = text.encode('utf-8')encrypt_text = encryptor.encrypt(text)encrypt_text = base64.b64encode(encrypt_text)return encrypt_text# 构造函数获取歌手信息def get_comments_json(url, data):headers={'Accept': 'text/html, application/xhtml+xml, application/xml;q=0.9, image/webp, image/apng, */*;q=0.8','Accept-Encoding': 'gzip, deflate','Accept-Language': 'zh-CN, zh;q=0.9','Connection': 'keep-alive','Cookie': 'WM_TID=36fj4OhQ7NdU9DhsEbdKFbVmy9tNk1KM; _iuqxldmzr_=32; _ntes_nnid=26fc3120577a92f179a3743269d8d0d9, 1536048184013; _ntes_nuid=26fc3120577a92f179a3743269d8d0d9; __utmc=94650624; __utmz=94650624.1536199016.26.8.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr=(not%20provided); WM_NI=2Uy%2FbtqzhAuF6WR544z5u96yPa%2BfNHlrtTBCGhkg7oAHeZje7SJiXAoA5YNCbyP6gcJ5NYTs5IAJHQBjiFt561sfsS5Xg%2BvZx1OW9mPzJ49pU7Voono9gXq9H0RpP5HTclE%3D; WM_NIKE=9ca17ae2e6ffcda170e2e6eed5cb8085b2ab83ee7b87ac8c87cb60f78da2dac5439b9ca4b1d621f3e900b4b82af0fea7c3b92af28bb7d0e180b3a6a8a2f84ef6899ed6b740baebbbdab57394bfe587cd44b0aebcb5c14985b8a588b6658398abbbe96ff58d868adb4bad9ffbbacd49a2a7a0d7e6698aeb82bad779f7978fabcb5b82b6a7a7f73ff6efbd87f259f788a9ccf552bcef81b8bc6794a686d5bc7c97e99a90ee66ade7a9b9f4338cf09e91d33f8c8cad8dc837e2a3; JSESSIONID-WYYY=G%5CSvabx1X1F0JTg8HK5Z%2BIATVQdgwh77oo%2BDOXuG2CpwvoKPnNTKOGH91AkCHVdm0t6XKQEEnAFP%2BQ35cF49Y%2BAviwQKVN04%2B6ZbeKc2tNOeeC5vfTZ4Cme%2BwZVk7zGkwHJbfjgp1J9Y30o1fMKHOE5rxyhwQw%2B%5CDH6Md%5CpJZAAh2xkZ%3A1536204296617; __utma=94650624.105654.1536048185.1536199016.1536203113.27; __utmb=94650624.12.10.1536203113','Host': '','Referer': '/','Upgrade-Insecure-Requests': '1','User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ''Chrome/66.0.3359.181 Safari/537.36'}try:r = requests.post(url, headers=headers, data=data)r.encoding = "utf-8"if r.status_code == 200: # 状态码 200代表服务器正常响应# 返回json格式的数据return r.json()except:print("爬取失败!")
评论抓取
评论分为两种，一种是精彩评论即我们所说的热评，另一种就是最新评论，按时间排序的maybe
接下来只需要解析网页内容，两种评论只是所在的路径不同，就可以获取到评论了
在获取评论后，利用tkinter写个GUI界面，用于显示爬取的内容，同时将爬取的评论也写入文件中存储
import mathimport tkinterfrom tkinter import *from Netease_cloud_comment_capture.searchMusic import search# 搜索歌曲名称from Netease_cloud_comment_capture.tool import get_params, get_comments_json, get_encSecKeydef get_music_name():d = search()song_id = d.search_song(entry.get())text.insert(END, '解析到歌曲的id为：{}\n'.format(song_id))text.update()# 歌曲名字songname = entry.get()# 文件存储路径filepath = songname + ".txt"page = 1params = get_params(1)encSecKey = get_encSecKey()url = '/weapi/v1/resource/comments/R_SO_4_' + str(song_id) + '?csrf_token='data = {'params': params, 'encSecKey': encSecKey}# 获取第一页评论html = get_comments_json(url, data)# 评论总数total = html['total']# 总页数pages = math.ceil(total / 20)if(total>5):pages = 5else:pages = totalhotcomments(html, songname, page, pages, total, filepath)comments(html, songname, page, pages, total, filepath)# 开始获取歌曲的全部评论page = 2while page <= pages:params = get_params(page)encSecKey = get_encSecKey()data = {'params': params, 'encSecKey': encSecKey}html = get_comments_json(url, data)# 从第二页开始获取评论comments(html, songname, page, pages, total, filepath)page += 1tkinter.messagebox.showinfo('提示', '评论抓取完成，请查看！')########def hotcomments(html, songname, i, pages, total, filepath):text.insert(END, '加载中，请稍等！\n')text.update()text.after(100)# 写入文件with open(filepath, 'a', encoding='utf-8') as f:f.write("正在获取歌曲{}的第{}页评论, 总共有{}页{}条评论！\n\n".format(songname, i, pages, total))text.insert(END, "正在获取歌曲{}的第{}页评论, 总共有{}页{}条评论！\n\n".format(songname, i, pages, total))text.update()text.after(100)# 精彩评论m = 1# 键在字典中则返回True, 否则返回Falseif 'hotComments' in html:for item in html['hotComments']:# 提取发表热门评论的用户名user = item['user']# 写入文件text.insert(END, " 热门评论{} 用户名：{} 点赞次数: {}\n\n".format(m, user['nickname'],item['likedCount']))text.insert(END, " 评论内容：{}\n\n".format(item['content']))text.update()text.after(100)with open(filepath, 'a', encoding='utf-8') as f:f.write(" 热门评论{} 用户名：{} 点赞次数: {}\n\n".format(m, user['nickname'], item['likedCount']))f.write(" 评论内容：{}\n\n".format(item['content']))text.insert(END, "\n\n")# 回复评论if len(item['beReplied']) != 0:for reply in item['beReplied']:# 提取发表回复评论的用户名replyuser = reply['user']text.insert(END, " 回复：{} : {}".format(replyuser['nickname'], reply['content']))text.insert(END, "\n\n")text.update()text.after(100)f.write(" 回复：{} : {}\n".format(replyuser['nickname'], reply['content']))m += 1def comments(html, songname, i, pages, total, filepath):with open(filepath, 'a', encoding='utf-8') as f:f.write("\n\n正在获取歌曲{}的第{}页评论, 总共有{}页{}条评论！\n".format(songname, i, pages, total))text.insert(END, "\n\n正在获取歌曲{}的第{}页评论, 总共有{}页{}条评论！\n".format(songname, i, pages, total))text.update()text.after(100)# 全部评论j = 1for item in html['comments']:# 提取发表评论的用户名user = item['user']text.insert(END, " 最新评论{} 用户名：{} 点赞次数: {}\n\n".format(j, user['nickname'],item['likedCount']))text.insert(END, " 评论内容：{}\n\n".format(item['content']))text.insert(END, "\n\n")text.update()text.after(10)with open(filepath, 'a', encoding='utf-8') as f:f.write(" 最新评论{} 用户名：{} 点赞次数: {}\n\n".format(j, user['nickname'], item['likedCount']))f.write(" 评论内容：{}\n\n".format(item['content']))text.insert(END, "\n\n")# 回复评论if len(item['beReplied']) != 0:for reply in item['beReplied']:# 提取发表回复评论的用户名replyuser = reply['user']text.insert(END, " 回复：{} : {}".format(replyuser['nickname'], reply['content']))text.insert(END, "\n\n")text.update()text.after(10)f.write(" 回复：{} : {}\n".format(replyuser['nickname'], reply['content']))j += 1# 创建界面root = Tk()# 标题root.title("网易云评论爬取脚本")# 设置窗口大小root.geometry('1123x410')root.configure(bg="#FFFFDF")# 标签控件label = Label(root, text='请输入要爬取的歌曲名称：', font=('幼圆', 15,), bg='#FAF4FF')# 标签定位label.grid(sticky=W)# 输入框entry = Entry(root, font=('幼圆', 15), bg='#ECECFF')entry.grid(row=0, column=1, sticky=W)# 抓取按钮button = Button(root, text='抓取评论', font=('幼圆', 15), command=get_music_name, bg='#CEFFCE')# 左对齐button.grid(row=0, column=2, sticky=W)# 列表框text = Listbox(root, font=('幼圆', 16), width=100, height=20, bg='#E6E6F2')text.grid(row=1, columnspan=4)# 退出按钮button1 = Button(root, text='退出', font=('幼圆', 15), command=root.quit, bg='#CAFFFF')button1.grid(row=0, column=3, sticky=E)# 显示界面root.mainloop()
