最近在学Python爬虫，看了Blibili爬取网易云音乐评论的视频，视频中是将一首歌的评论存入json文件，我在此代码的基础上扩展了三点：
1.爬取热歌榜200首歌曲的精彩评论；
2.解析评论，爬取评论的内容、时间、用户、歌曲名字等数据；**
3.将爬取的数据存入txt文本文件或者mysql数据库中。
参考代码
存入txt文本文件代码存入Mysql数据库代码下面是我的一小点分析，代码附在最后
下面是参考代码，逻辑有点乱，一些功能没有用函数封装，大家可以继续优化。
存入txt文本文件代码
# -*- coding: utf-8 -*-# @Time : /11/10 11:21# @Author : MuTutuMumei# @File : Comment_txt.py# 爬取每首歌的ID和名称，改变参数ID获得不同歌曲的评论# 爬取评论信息内容的步骤：#1.找到未加密的参数 #window.arsea(参数，xxx,xxx,xxx)#2.想办法把参数进行加密（必须参考网易的逻辑），params=>encText，encSecKey=>encSecKey#3.请求到网易，拿到评论信息from Crypto.Cipher import AESfrom base64 import b64encodeimport requests,jsonimport timeimport refrom lxml import etreeif __name__ == "__main__":headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36 Edg/95.0.1020.40'}# 服务于d的f = "00e0b509f6259df8642dbc35662901477df22677ec152b5ff68ace615bb7b725152b3ab17a876aea8a5aa76d2e417629ec4ee341f56135fccf695280104e0312ecbda92557c93870114af6c9d05c4f7f0c3685b7a46bee255932575cce10b424d813cfe4875d3e82047b97ddef52741d546b8e289dc6935b3ece0462db0a22b8e7"g = "0CoJUm6Qyw8W8jud"e = "010001"i = "oStLTrr0FaYKC6IE" # 手动固定的，网易是随机的def get_encSecKey(): # 因为i是固定的，所以这也是固定的return "bf0c12e363f8fca9793ddab0cf6e1ea72b5d1c84c136a11147ce1b8b25da43ba385a7df0b5c97f0f9fe8904a23e318757cef5b8fe9da78ba447bfffc89a3fd3fc99db34631b69dffabe2bf6849961452f751fc71302bc3259177f70b4fdbf9ae19a5bd58b1d8422f4f0c1319f32099cbf6bf871bf59e459f05247c009f0f41f5"#把参数进行加密def get_params(data): #默认这里收到的是字符串first = enc_params(data,g)second = enc_params(first, i)return second #返回的就是pramas#转化成16的倍数，为下方的加密算法服务def to_16(data):pad = 16 - len(data)%16data += chr(pad)*padreturn data#加密过程def enc_params(data,key):iv = "0102030405060708"data = to_16(data)aes = AES.new(key = key.encode("utf-8"),IV = iv.encode('utf-8'),mode = AES.MODE_CBC)bs = aes.encrypt(data.encode("utf-8")) #加密，加密的长度是16的倍数return str(b64encode(bs),"utf-8") #转化成字符串返回#处理加密过程'''function a(a) { a=16var d, e, b = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", c = "";for (d = 0; a > d; d += 1)e = Math.random() * b.length, #生成随机数e = Math.floor(e), #取整c += b.charAt(e); #取出b中对应位置的字符return c}function b(a, b) {var c = CryptoJS.enc.Utf8.parse(b), d = CryptoJS.enc.Utf8.parse("0102030405060708"), e = CryptoJS.enc.Utf8.parse(a) #e是数据, f = CryptoJS.AES.encrypt(e, c, { #c就是加密密钥 iv: d, #iv是偏移量mode: CryptoJS.mode.CBC # 模式：CBC加密});return f.toString()}function c(a, b, c) {var d, e;return setMaxDigits(131),d = new RSAKeyPair(b,"",c),e = encryptedString(d, a)}function d(d, e, f, g) { d：数据json串 e:"010001" f: g = "0CoJUm6Qyw8W8jud"var h = {}, i = a(16); #16位随机值return h.encText = b(d, g), g是密钥h.encText = b(h.encText, i), #返回的就是params i是密钥h.encSecKey = c(i, e, f), #返回的是encSecKey e和f定死，能产生变数的只能是ih}'''#获取歌曲ID的列表,得到的是列表，从列表中正则解析出ID，再添加到一个列表中url_song = '/discover/toplist?id=3778678'resp = requests.get(url=url_song, headers=headers).texttree = etree.HTML(resp)song = tree.xpath('//div[@data-key="song_toplist-3778678"]/ul[@class="f-hide"]/li/a/@href')#歌曲名称在首页获取song_name = tree.xpath('//div[@data-key="song_toplist-3778678"]/ul[@class="f-hide"]/li/a/text()')obj = pile(r'\d+', re.S)songID_list = []for it in song:song_num = obj.findall(it)song_num_str = ''.join(song_num)songID_list.append(song_num_str)#print(song_list)Comment_mum = int(input('请输入每首歌要爬取的热评数量(不能超过15首)：'))song_mum = int(input('请输入你要爬取评论的top歌曲数量(不能超过200首)：'))print('开始爬虫！！！')song_count = 0 #这里记录第几首歌曲，获取歌曲名称的时候使用#request请求到歌曲详情页评论的地址for it in songID_list[0:song_mum]:url_comment = "/weapi/comment/resource/comments/get?csrf_token="#请求方式是POSTdata = {'csrf_token': "",'cursor': "-1",'offset': "0",'orderType': "1",'pageNo': "1",'pageSize': "1", #因为精彩热评只在第一页'rid': "R_SO_4_" + it, #组合获得不同歌曲的评论'threadId': "R_SO_4_" + it}response = requests.post(url_comment,data={"params":get_params(json.dumps(data)),"encSecKey":get_encSecKey()},headers=headers)result = json.loads(response.content.decode('utf-8'))#注意文件打开的方式fp = open('./网易云评论.txt', 'a+', encoding='utf-8')fp.write('hotComments' + ' ' + '\n')#爬取hotComments并写入txt文本文件#字典镶嵌列表，列表镶嵌字典，注意如何使用 /jiba/p/14945833.htmlfor hot in range(0,Comment_mum):fp.write('昵称：' + result['data']['hotComments'][hot]['user']['nickname'] + '\n')#昵称fp.write('歌曲名称：' + song_name[song_count] + '\n') # 昵称fp.write('评论：' + result['data']['hotComments'][hot]['content'] + '\n')#评论# "time": 1561597817178# 13位的数字串是毫秒级别的时间戳，通过下边的代码转换为表转格式timeNum = result['data']['hotComments'][hot]['time']timeTemp = float(timeNum / 1000)tupTime = time.localtime(timeTemp)stadardTime = time.strftime("%Y-%m-%d %H:%M:%S", tupTime)fp.write('评论时间：' + stadardTime + '\n')#时间if result['data']['hotComments'][hot]['user']['vipRights'] == None: #判断是否是VIPfp.write('vip:yes' + '\n')else:fp.write('vip:no' + '\n')fp.write('点赞数：' + str(result['data']['hotComments'][hot]['likedCount']) + '\n')#点赞数fp.write('-------------------------------------' + '\n')song_count = song_count + 1 #递增歌曲的名称fp.close()print('爬取完毕！！！')
代码运行结果：
存入Mysql数据库代码
# -*- coding: utf-8 -*-# @Time : /11/10 23:34# @Author : MuTutuMumei# @File : Comment_MYSQL.pyfrom Crypto.Cipher import AESfrom base64 import b64encodeimport requests,jsonimport timeimport refrom lxml import etreeimport pymysqlif __name__ == "__main__":#数据库参数设置db = pymysql.connect(host="localhost",port=3306,user="root",passwd="123456",database="wayimusic_comment",charset="utf8",autocommit=True) #在connect() 中加上参数autocommit并且给它赋值True这样就能自动检查是否真的入库cursor = db.cursor()headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36 Edg/95.0.1020.40'}# 关于解密参数的一些数据，服务于d的f = "00e0b509f6259df8642dbc35662901477df22677ec152b5ff68ace615bb7b725152b3ab17a876aea8a5aa76d2e417629ec4ee341f56135fccf695280104e0312ecbda92557c93870114af6c9d05c4f7f0c3685b7a46bee255932575cce10b424d813cfe4875d3e82047b97ddef52741d546b8e289dc6935b3ece0462db0a22b8e7"g = "0CoJUm6Qyw8W8jud"e = "010001"i = "oStLTrr0FaYKC6IE" # 手动固定的，网易是随机的def get_encSecKey(): # 因为i是固定的，所以这也是固定的return "bf0c12e363f8fca9793ddab0cf6e1ea72b5d1c84c136a11147ce1b8b25da43ba385a7df0b5c97f0f9fe8904a23e318757cef5b8fe9da78ba447bfffc89a3fd3fc99db34631b69dffabe2bf6849961452f751fc71302bc3259177f70b4fdbf9ae19a5bd58b1d8422f4f0c1319f32099cbf6bf871bf59e459f05247c009f0f41f5"#把参数进行加密def get_params(data): #默认这里收到的是字符串first = enc_params(data,g)second = enc_params(first, i)return second #返回的就是pramas#转化成16的倍数，为下方的加密算法服务def to_16(data):pad = 16 - len(data)%16data += chr(pad)*padreturn data#加密过程def enc_params(data,key):iv = "0102030405060708"data = to_16(data)aes = AES.new(key = key.encode("utf-8"),IV = iv.encode('utf-8'),mode = AES.MODE_CBC)bs = aes.encrypt(data.encode("utf-8")) #加密，加密的长度是16的倍数return str(b64encode(bs),"utf-8") #转化成字符串返回#获取歌曲ID的列表,得到的是列表，从列表中正则解析出ID，再添加到一个列表中url_song = '/discover/toplist?id=3778678'resp = requests.get(url=url_song, headers=headers).texttree = etree.HTML(resp)song = tree.xpath('//div[@data-key="song_toplist-3778678"]/ul[@class="f-hide"]/li/a/@href')#歌曲名称在首页获取song_name = tree.xpath('//div[@data-key="song_toplist-3778678"]/ul[@class="f-hide"]/li/a/text()')obj = pile(r'\d+', re.S)songID_list = []for it in song:song_num = obj.findall(it)song_num_str = ''.join(song_num)songID_list.append(song_num_str)#print(song_list)Comment_mum = int(input('请输入每首歌要爬取的热评数量(不能超过15首)：'))song_mum = int(input('请输入你要爬取评论的top歌曲数量(不能超过200首)：'))print('开始爬虫！！！')song_count1 = 0 #这里记录第几首歌曲，获取歌曲名称的时候使用song_count2 = 0comment_count = 0 #记录数据库的评论数量#request请求到歌曲详情页评论的地址for it in songID_list[0:song_mum]:url_comment = "/weapi/comment/resource/comments/get?csrf_token="#请求方式是POSTdata = {'csrf_token': "",'cursor': "-1",'offset': "0",'orderType': "1",'pageNo': "1",'pageSize': "1", #因为精彩热评只在第一页'rid': "R_SO_4_" + it, #组合获得不同歌曲的评论'threadId': "R_SO_4_" + it}response = requests.post(url_comment,data={"params":get_params(json.dumps(data)),"encSecKey":get_encSecKey()},headers=headers)result = json.loads(response.content.decode('utf-8'))#将评论信息存入MYSQL数据库for hot in range(0,Comment_mum):nickname = result['data']['hotComments'][hot]['user']['nickname'] #昵称songName = song_name[song_count1] # 昵称hot_comment = result['data']['hotComments'][hot]['content'] #评论# "time": 1561597817178# 13位的数字串是毫秒级别的时间戳，通过下边的代码转换为表转格式timeNum = result['data']['hotComments'][hot]['time']timeTemp = float(timeNum / 1000)tupTime = time.localtime(timeTemp)sta_Time = time.strftime("%Y-%m-%d %H:%M:%S", tupTime)if result['data']['hotComments'][hot]['user']['vipRights'] == None: #判断是否是VIPVip = 0else:Vip = 1likedCount = result['data']['hotComments'][hot]['likedCount'] #点赞数sql = "insert into comment(ID,nickName,songName,comment,`datatime`,VIP,likedNum) " \"values(%d,'%s','%s','%s','%s',%d,%d)"%(comment_count,nickname,songName,hot_comment,sta_Time,Vip,likedCount)# "insert into '表名'(字段名) values(值)"try:cursor.execute(sql)mit() # 都插入成功，将结果提交给数据库print(str(comment_count)+' 歌曲：“'+songName+'”的评论信息'+'已经存入MYSQL数据库！！！')except Exception as e:db.rollback() # 如果提交失败，结果回退到上一次提交的结果print("执行MySQL时出错：%s" %e)comment_count = comment_count + 1song_count1 = song_count1 + 1 # 递增歌曲的名称cursor.close()db.close()
Mysql数据库的表结构：
运行结果：
数据表结果：
