多线程爬某小说网

jaaks

[Python] 纯文本查看 复制代码

from bs4 import BeautifulSoup
import os,requests,re,threading,time,json
url_list = []
headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"
        }
directory = "txt"  # 相对路径，将在当前工作目录下创建txt目录
if not os.path.exists(directory):
    os.makedirs(directory)
def get_list(bookid):#获取章节列表
    data = {"bookId": bookid}
    r = requests.post("https://bookapi.zongheng.com/api/chapter/getChapterList", data=data, headers=headers)
    response_data = json.loads(r.text)
    # print(response_data["result"]["chapterList"]["chapterViewList"]["chapterId"])
    chapter_list = response_data["result"]["chapterList"]
    for chapter in chapter_list:
        for chapte in chapter["chapterViewList"]:
            chapterId = chapte["chapterId"]
            url_list.append(f"https://read.zongheng.com/chapter/{bookid}/{chapterId}.html")

    return True
def get_text(url,Lock:threading.Lock):#访问正文
        p_text = ""
        for ur in url:
            #Lock.acquire()  # 锁
            r = requests.get(ur,headers=headers)
            #Lock.release()
            soup = BeautifulSoup(r.text, 'html.parser')
            name = soup.find(class_="title_txtbox").text    #标题
            contents = soup.find('div', class_="content")   #正文
            content = contents.find_all("p")
            for conten in content:
                p_text += conten.text+"\n\n"
            name = re.sub('[?|&]',"",name.strip())    #正则过滤内容
            #将标题和内容写进去
            file_name = os.path.join("txt",name+".txt")
            sava_file(file_name,p_text)
            time.sleep(1)
            print(name)
def sava_file(name,text):
    with open(name,"w",encoding="utf8") as f:
        f.write(text)
Chapter = get_list("1249806") #访问章节
Lock = threading.Lock() #设置线程锁
print("长度："+str(len(url_list)))
if Chapter:
    # 计算每个子列表的长度
    num = int(input("输入线程数："))    #线程数
    Length = len(url_list) // num
    urls = [url_list[i:i+num] for i in range(0,len(url_list),num)]  #对列表进行切片为子列表
    for url in urls:
        threading.Thread(target=get_text, args=(url,Lock)).start()

有一点我不是很明白，我测试使用线程池并发CPU直接占满，但是使用threading多线程并发却不会

还有一点保存下来的文件排序也不好排，如果设置线程锁，确实能按照排序，但是其他线程阻塞导致就像单线程一样慢

wkdxz

试了下，我电脑线程30时速度比较快，具体线程可自行测试。

[Asm] 纯文本查看 复制代码

★  maxWorkers：30        time：10.26秒

[Python] 纯文本查看 复制代码

# -*- coding: utf-8 -*-

import concurrent.futures
import json
import os
import re
import time

import requests
from bs4 import BeautifulSoup

url_list = []
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"
}
directory = "txt"  # 相对路径，将在当前工作目录下创建txt目录
if not os.path.exists(directory):
    os.makedirs(directory)


def time_it(func):
    def wrapper(*args, **kwargs):
        start_time = time.time()
        result = func(*args, **kwargs)
        end_time = time.time()
        print(f"time：{end_time - start_time:.2f}秒")
        return result

    return wrapper


def get_list(bookid):  # 获取章节列表
    data = {"bookId": bookid}
    r = requests.post(
        "https://bookapi.zongheng.com/api/chapter/getChapterList",
        data=data,
        headers=headers,
    )
    response_data = json.loads(r.text)
    chapter_list = response_data["result"]["chapterList"]
    for chapter in chapter_list:
        for chapte in chapter["chapterViewList"]:
            chapterId = chapte["chapterId"]
            url_list.append(
                f"https://read.zongheng.com/chapter/{bookid}/{chapterId}.html"
            )

    return True


def get_text(url):
    p_text = ""
    r = requests.get(url, headers=headers)
    soup = BeautifulSoup(r.text, "html.parser")
    name = soup.find(class_="title_txtbox").text  # 标题
    contents = soup.find("div", class_="content")  # 正文
    content = contents.find_all("p")
    for conten in content:
        p_text += conten.text + "\n\n"
    name = re.sub("[?|&]", "", name.strip())  # 正则过滤内容
    file_name = os.path.join("txt", name + ".txt")
    sava_file(file_name, p_text)
    # print(name)


def sava_file(name, text):
    with open(name, "w", encoding="utf8") as f:
        f.write(text)


@time_it
def main(maxWorkers):
    print(f"★  maxWorkers：{maxWorkers}", end="\t ")
    Chapter = get_list("1249806")  # 访问章节
    # print("长度：" + str(len(url_list)))
    if Chapter:
        with concurrent.futures.ThreadPoolExecutor(maxWorkers) as executor:
            executor.map(get_text, url_list)


if __name__ == "__main__":
    main(30)  # 线程数

zhangxu0529

[Python] 纯文本查看 复制代码

headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"
        }

如果换成代{过}{滤}理，这么该怎么填？

MyModHeaven

看这个发布时间，楼主这是早起学习？

MyModHeaven

我用threading多线程的时候，CPU会占满。没用过线程池

MyModHeaven

num = int(input("输入线程数："))，是不是因为线程数不够多？网站不禁IP的话，我就不怎么限制线程数量

ysjd22

学习一下代码

baliao

感谢分享! 学习了.

yswd

多谢分享，学习了。

KevinF

感谢分享

吖力锅

我试一下网站就不给我下了