一、需求
Python爬取网易云音乐所有歌手的名称和链接
二、代码实现
import requestsfrom lxml import etreefrom urllib import requestimport timeclass WangYiYun:def __init__(self, base_url):# 初始化treeself.html = self.request_url(base_url)self.parse_html()def request_url(self, base_url):# 请求访问 生成树结构headers = {'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'}self.html = requests.get(url=base_url, headers=headers).content.decode('utf-8')self.tree = etree.HTML(self.html)return self.treedef parse_html(self):# 解析各类歌手urlgroup_list = self.html.xpath('//div[@class = "blk"]')for group in group_list:# 连接分类href_list = group.xpath('.//a/@href')# 歌手分类group_name_list = group.xpath('.//a/text()')# print(group_name_list)for href, group_name in zip(href_list, group_name_list):full_href = request.urljoin('', href)# print(full_href,group_name)html1 = requests.get(full_href).content.decode('utf-8')tree1 = etree.HTML(html1)# print(tree1)# 歌手分类对应的歌手列表页链接url_list = tree1.xpath('//ul[@class="n-ltlst f-cb"]/li[position()>1]/a/@href')# print(url_list)# 歌手列表页链接遍历for url in url_list:full_url = request.urljoin('', url)single_letter = requests.get(full_url).content.decode('utf-8')#single_letter_tree = etree.HTML(single_letter)# 歌手列表页获取歌手信息li_list = single_letter_tree.xpath('//ul[@class="m-cvrlst m-cvrlst-5 f-cb"]/li')for li in li_list:# 从歌手列表页中遍历获取名字列表及对应的链接列表name_list = li.xpath('//a[@class="nm nm-icn f-thide s-fc0"]/text()')link_list = li.xpath('//a[@class="nm nm-icn f-thide s-fc0"]/@href')for name, link in zip(name_list, link_list):full_link = '' + linkprint(name, full_link)time.sleep(0.5)if __name__ == '__main__':base_url = '/discover/artist'WangYiYun(base_url)
