python多线程爬今日头条

import threading

import time

from lxmlimport etree

import requests

from queueimport Queue

import re

"""

其实 函数参数中的冒号是参数的类型建议符，告诉程序员希望传入的实参的类型 。 函数后面跟着的箭头是函数返回值的类型建议符

，用来说明该函数返回的值是什么类型 。 更官方的解释：此为type hints，是Python 3.5新加的"""

headers = {

'user-agent':"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36 Edg/99.0.1150.55",

    'refer':'https://so.toutiao.com/search?keyword=%E8%A1%97%E6%8B%8D%E7%BE%8E%E5%A5%B3&pd=information&source=pagination&dvpf=pc&aid=4916&page_num={page}&search_id=202203201951110102120460753D61F421'

}

cookie={'cookie':'tt_webid=7077014656070960676; _S_DPR=1.5; _S_IPAD=0; MONITOR_WEB_ID=7077014656070960676; FRM=new; PIXIEL_RATIO=1.5; WIN_WH=1280_648; ttcid=c9720ff1a7c146a790a1c113c8f8e91f39; tt_scid=0LEZtCnx3EI6WHPLeDb9vGDK-P-YwQDee.cVi7b6xfkeANJ9dxJUAUIHgJiGKGF08d18; ttwid=1%7C5pSenh2ZSIAoAPAnZmcHQ2xN3TykA8ikp6iCucnETC0%7C1648709296%7Ce85f11b922972468dc1718583be6152a42d5113842d03386a442d9afccd8b8aa; _S_WIN_WH=1280_648'}

class Producer(threading.Thread):

def __init__(self, page_queue,img_queue, *args, **kwargs):

super(Producer, self).__init__(*args, **kwargs)

self.page_queue= page_queue

self.img_queue = img_queue

def run(self):

while True:

if self.page_queue.empty():

break

            else:

url =self.page_queue.get()

self.parse_page(url)

def parse_page(self, url):

response = requests.get(url=url, headers=headers,cookies=cookie)

text = response.text

html = etree.HTML(text)

img_list = html.xpath('//div[@class="cs-view cs-view-block cs-card-content"]')

for liin img_list:

href = li.xpath('./a/@href')

if href:

m = re.findall(r'.*(\d{19}).*', href[0])

a ='https://www.toutiao.com/a' + m[0] +'/?channel=&source=search_tab'

                self.parser2(a)

def parser2(self,a):

response = requests.get(url=a, headers=headers,cookies=cookie )

m = etree.HTML(response.text)

s = m.xpath('//article[@class="syl-article-base tt-article-content syl-page-article syl-device-pc"]')[0]

x = s.xpath('./div[@class="pgc-img"]')

for iin x:

ll = i.xpath('./img/@src')

if ll:

self.img_queue.put(ll[0])

class Consumer(threading.Thread):

def __init__(self, page_queue, img_queue, *args, **kwargs):

super(Consumer, self).__init__(*args, **kwargs)

self.page_queue = page_queue

self.img_queue = img_queue

print('af',self.img_queue.get())

def run(self):

while True:

if self.page_queue.empty()and self.img_queue.empty():

break

            else:

imgurl=self.img_queue.get()

name = imgurl[-11:-8]

path =r"C:\Users\29258\Desktop\er\\" + name +'.jpg'

                response = requests.get(url=imgurl, headers=headers,)

with open(path, 'wb')as fp:

fp.write(response.content)

print(name,'下载好了')

def main():

page_queue = Queue(50)# 存储页码链接

    img_queue = Queue(10)# 存储解析出来的图片链接

    # 想要爬取前10页的数据

    for pagein range(50):

url =f'https://so.toutiao.com/search?keyword=%E8%A1%97%E6%8B%8D%E7%BE%8E%E5%A5%B3&pd=information&source=pagination&dvpf=pc&aid=4916&page_num={page}&search_id=202203201951110102120460753D61F421'

        print(url)

page_queue.put(url)# 将10页的页码链接加入到了page_queue

    for xin range(3):

t = Producer(page_queue, img_queue)

t.start()

# 开启三个消费者

    for xin range(5):

t = Consumer(page_queue, img_queue)

t.start()

time.sleep(3)

if __name__=='__main__':

main()

# 开启三个生产者

    # for x in range(3):

#    t = Producer(page_queue, img_queue)

#    t.start()

  # # 开启三个消费者

    # for x in range(3):

#    t = Consumer(page_queue, img_queue)

#    t.start()

原创