from bs4 import BeautifulSoupimport requestsimport jsonimport timefrom requests.exceptions import RequestException#获取响应内容def get_html(url):try:headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'}response = requests.get(url,headers=headers)if response.status_code == 200:return response.textreturn Noneexcept RequestException:return None#单个页面抓取def parse_one_page(html):soup = BeautifulSoup(html, "lxml")items = soup.find_all(name='dd')for item in items:yield {'index': item.i.string,'image': item.find(class_='board-img')['data-src'],'title': item.find(class_='name').string,'actor': item.find(class_='star').string.strip()[3:],'time': item.find(class_='releasetime').string,'score': item.find(class_='score').find(class_='integer').string.strip() + item.find(class_='score').find(class_='fraction').string.strip()}#写入文件def write_to_file(content):with open('test.txt','a',encoding='utf-8') as f:f.write(json.dumps(content,ensure_ascii=False)+'\n')def main(offset):url = '/board/4?offset='+str(offset)html = get_html(url)for item in parse_one_page(html):print(item)#写入文件#write_to_file(item)if __name__ == '__main__':for i in range(10):main(offset=i*10)time.sleep(1)
