python爬虫爬取京东店铺商品价格数据(更新版)

主要使用的库：

requests:爬虫请求并获取源码

re：使用正则表达式提取数据

json:使用JSON提取数据

pandas：使用pandans存储数据

##sqlalchemy ：备用方案，上传数据到mysql

以下是源代码：

# -*- coding:utf-8 -*-import requestsimport reimport randomimport timeimport jsonimport pymysqlfrom sqlalchemy import create_enginefrom requests.packages.urllib3.exceptions import InsecureRequestWarningimport pandas as pdrequests.packages.urllib3.disable_warnings(InsecureRequestWarning) ###禁止提醒SSL警告class jd(object):def __init__(self):self.s = requests.session() ## 创建一个session对象headers = {'accept':'application/json, text/javascript, */*; q=0.01','accept-encoding':'gzip, deflate, br','User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1',}self.s.headers.update(headers) ### 设置请求头#self.engine = create_engine('mysql+pymysql://root:123456@192.168.0.1:3306/jd') ##存储到SQLdef getdata(self,url,name):getdate=time.strftime("%Y-%m-%d",time.localtime())self.shopid=re.search('index-(.*?).html',url).group(1) ###获取店铺ID号self.s.get('https://shop./search/search?shopId='+str(self.shopid))for i in range(1,10000): ###爬取页数范围 没有找到商品后会自动退出循环wareId_list = []wname_list = []jdPrice_list = []time.sleep(random.random()) ##随机延时0-1秒t = int(time.time() * 1000)## /search/searchjson?datatype=1&page=2&pagesize=40&merge_sku=yes&qp_disable=yes&key=ids%2C%2C121614&_=1537524375713&sceneval=2&g_login_type=1&callback=jsonpCBKQ&g_ty=lssearchurl = '/search/searchjson?datatype=1&page={}&pagesize=40&merge_sku=yes&qp_disable=yes&key=ids%2C%2C{}&_={}&sceneval=2&g_login_type=1&callback=jsonpCBKA&g_ty=ls'.format(i,self.shopid,t) ##请求数据网址print(searchurl)req=self.s.get(url=searchurl,verify=False).text ###获取数据print(req)print(name,i)wareId=re.findall('"wareid": "(.*?)",',req) ##获取商品IDwname=re.findall('"warename": "(.*?)",',req) ###获取商品名称jdPrice=re.findall('"dredisprice": "(.*?)",',req) ###获取商品价格if wareId==[]: ###如果没有找到ID退出循环break#####处理数据wareId_list.extend(wareId)wname_list.extend(wname)jdPrice_list.extend(jdPrice)wareId_l=len(wareId_list)name_list=[]name_list.append(name)name_list.extend(name_list*(wareId_l-1))getdate_list = []getdate_list.append(getdate)getdate_list.extend(getdate_list * (wareId_l - 1))jddata={'name':name_list,'wareId':wareId_list,'wname':wname_list,'jdPrice':jdPrice_list,'update': getdate_list}df = pd.DataFrame(data=jddata)df.to_csv(r'e:\jdmall.csv', index=False, encoding="GB18030")###保存csv文件#df.to_sql('店铺前端', con=self.engine, if_exists='append', index=False) ##上传到数据库if __name__ == '__main__':j=jd()url='/index-1000000693.html'nm='intel'j.getdata(url,nm)

【腾讯云】11.11 云上盛惠，云产品限时抢购，1核2G云服务器首年88元