python爬虫——用selenium爬取京东商品信息
1.先附上效果图(我偷懒只爬了4页)
2.京东的网址/
3.我这里是不加载图片，加快爬取速度，也可以用Headless无弹窗模式
options.add_experimental_option('prefs', {'profile.managed_default_content_settings.images': 2}) #不加载图片 browser = webdriver.Chrome(options=options) wait =WebDriverWait(browser,50)#设置等待时间 url = '/' data_list = []#设置全局变量用来存储数据 keyword="python爬虫"#关键词options = webdriver.ChromeOptions()
1
2
3
4
5
6
7
8
4.先找到搜索框并用selenium模拟点击(这里发现京东不需要登录就能看到商品信息)
browser.get('/') try: input = wait.until( EC.presence_of_all_elements_located((By.CSS_SELECTOR, "#key")) ) #等到搜索框加载出来 submit = wait.until( EC.element_to_be_clickable((By.CSS_SELECTOR, "#search > div > div.form > button")) )#等到搜索按钮可以被点击 input[0].send_keys(keyword)#向搜索框内输入关键词 submit.click()#点击 total = wait.until( EC.presence_of_all_elements_located( (By.CSS_SELECTOR, '#J_bottomPage > span.p-skip > em:nth-child(1) > b') ) )#记录一下总页码,等到总页码加载出来 html = browser.page_source#获取网页信息 prase_html(html)#调用提取数据的函数(后面才写到) return total[0].text except TimeoutError: search()def search():
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
5.进入了第一页，先写好翻页的函数，需要滑动到底部才能加载后30个商品，总共有60个商品
def next_page(page_number): try: # 滑动到底部 browser.execute_script("window.scrollTo(0, document.body.scrollHeight);") time.sleep(random.randint(1, 3))#设置随机延迟 button = wait.until( EC.element_to_be_clickable((By.CSS_SELECTOR, '#J_bottomPage > span.p-num > a.pn-next > em')) )#翻页按钮 button.click()# 翻页动作 wait.until( EC.presence_of_all_elements_located((By.CSS_SELECTOR, "#J_goodsList > ul > li:nth-child(30)")) )#等到30个商品都加载出来 # 滑动到底部，加载出后三十个货物信息 browser.execute_script("window.scrollTo(0, document.body.scrollHeight);") wait.until( EC.presence_of_all_elements_located((By.CSS_SELECTOR, "#J_goodsList > ul > li:nth-child(60)")) )#等到60个商品都加载出来 wait.until( EC.text_to_be_present_in_element((By.CSS_SELECTOR, "#J_bottomPage > span.p-num > a.curr"), str(page_number)) )# 判断翻页成功,高亮的按钮数字与设置的页码一样 html = browser.page_source#获取网页信息 prase_html(html)#调用提取数据的函数 except TimeoutError: return next_page(page_number)
