2-5.多进程 多线程 异步携程

多线程

1. threading

import threading

thread = threading.Thread(target=target, args=[i])

def targets(second):

print(f’Threading {threading.current_thread().name} is running’)

print(f’Threading {threading.current_thread().name} sleep {second}s’)

time.sleep(second)

print(f’Threading {threading.current_thread().name} is ended’)

print(f’Threading {threading.current_thread().name} is running’)

t = []

for i in [1, 5]:

thread = threading.Thread(target=targets, args=[i])

t.append(thread)

thread.start()

for i in t:

i.join()

print(f’Threading {threading.current_thread().name} is ended’)

2. 线程池

1名员工 10双鞋

10名员工 100双

10名员工 半天 50双

5名员工 一天 50双

给线程一个数量 只有这么多人

from concurrent.futures import ThreadPoolExecutordef crawl(url):print(url)if __name__ == '__main__':base_url = '/pachongkaifa/p{}/'with ThreadPoolExecutor(10) as f:for i in range(1,15):f.submit(crawl,url=base_url.format(i))

多线程采集案例

1、 先提取json文件 获取英雄的Id

2、根据iD 找到英雄的详情页地址

3、从详情页里面提取头像地址 构造 皮肤地址

import requestsimport osimport jsonimport threadingfrom lxml import etreeimport timeh=[]s=time.time()#1.先提取json文件 获取英雄的Iddef duo():'''处理多任务:return:'''response=requests.get('/web05/js/herolist.json')data=json.loads(response.text)#print(data)for j in data:t=threading.Thread(target=pa,args=(j,))t.start()h.append(t)for k in h:k.join()#2.根据iD 找到英雄的详情页地址def pa(j):num = j['ename'] #从data中获取ename的值name = j['cname']res2 = requests.get("/web05/herodetail/{}.shtml".format(num))res2_decode = res2.content.decode('gbk') # 返回相应的html页面，字符串格式，解码为utf-8_element = etree.HTML(res2_decode) # 将html转换为_Element对象，可以方便的使用getparent()、remove()、xpath()等方法element_img = _element.xpath('//div[@class="pic-pf"]/ul/@data-imgname')#print(element_img)name_img = element_img[0].split('|') # 去掉字符串中的|字符,并分割#print(name_img)for i in range(0,10):res1=requests.get("/images/yxzj/img06/skin/hero-info/{0}/{0}-bigskin-{1}.jpg".format(num,i+1)) #返回响应包if res1.status_code == 200:aa=name_img[i].find('&')#print(aa)bb=name_img[i][:aa]res_img=res1.content #把相应包内容转换为2进制a = './王者荣耀/' + str(name)b='./王者荣耀/'+str(name)+'/'+bb+'.jpg'if not os.path.exists('./王者荣耀/'):os.mkdir('./王者荣耀/')if not os.path.exists(a):os.mkdir(a)#3.从详情页里面提取头像地址 构造 皮肤地址with open(b,"wb") as f: #创建一个名为1.jpg的图片f.write(res_img) #把响应包2进制内容写入到1.jpg中print(name, bb)else:breakif __name__=='__main__':duo()g=time.time()print("用时：",g-s,"秒")

多进程

multiprocessing

from multiprocessing import Pool #进程池

import multiprocessingfrom multiprocessing import Poolimport requestsdef process(index):print(f'Proess:{index}')def scrape(url):try:requests.get(url)print(f'URL {url} Scraped')except requests.ConnectionError:print(f'URL {url} not Scraped')if __name__ == '__main__':# for i in range(5):#p = multiprocessing.Process(target=process,args=(i,))#p.start()pool = Pool(processes=3)urls = ['','/','/','']pool.map(scrape, urls)pool.close()

异步携程

#异步函数声明 async

实例

import asyncioimport timeimport httpxasync def req(client, i):res = await client.get('')print(f'第{i + 1}次请求，status_code = {res.status_code}')return resasync def main():async with httpx.AsyncClient() as client:task_list = [] # 任务列表for i in range(50):res = req(client, i)task = asyncio.create_task(res) # 创建任务task_list.append(task)#await 耗时任务给他挂起await asyncio.gather(*task_list) # 收集任务if __name__ == '__main__':start = time.time()asyncio.run(main())end = time.time()print(f'异步发送50次请求，耗时：{end - start}')