最近在做网页分析需要爬取很多的网页，然后就使用selenium进行了一系列的操作，最后可以保存网页的首页截图，整体截图（包含所有滚动区域），HTML源文件和MHTML源文件，整理后的代码如下：
邮箱,qq邮箱,/邮箱,阿里邮箱,/alimail/auth/login邮箱,163邮箱,/邮箱,新浪邮箱,/搜索引擎,百度,/搜索引擎,搜狗,/搜索引擎,bing,/商城,淘宝,/商城,小米商城,/shop商城,京东,/商城,唯品会,/
整体代码如下，可以根据自己的功能留下需要的部分
from selenium import webdriverfrom selenium.webdriver.chrome.options import Optionsimport csv# 构造webdriverdriver_path = r"C:\Program Files\Google\Chrome\Application\chromedriver.exe"options = Options()options.add_argument('--headless')driver = webdriver.Chrome(driver_path, options=options)def save_page(kind, name, url):driver.get(url)save_path = f"../source/{kind}/{name}/{name}"dir_name = os.path.dirname(path)if not os.path.exists(dir_name):os.makedirs(dir_name)# 仅首页图片(未滚动) 1920x1080driver.set_window_size(1920, 1680)driver.get_screenshot_as_file(save_path + "_short.png")# 整体截图(带滚动)width = driver.execute_script("return document.documentElement.scrollWidth")height = driver.execute_script("return document.documentElement.scrollHeight")driver.set_window_size(width, height)driver.get_screenshot_as_file(save_path + "_full.png")# 保存为htmlsource_code = driver.page_sourcewith open(save_path + ".html", mode='w', encoding='utf-8') as html_file:html_file.write(source_code)# 保存为mhtmlres = driver.execute_cdp_cmd('Page.captureSnapshot', {})# 2. write file locallywith open(save_path + ".mhtml", 'w', newline='') as sf:sf.write(res['data'])if __name__ == '__main__':# 打开所有网页列表with open("pagelist.txt", mode='r', encoding='utf-8') as f:csv_reader = csv.reader(f)for line in csv_reader:print(line)save_page(line[0], line[1], line[2])driver.quit()
