【爬虫】爬取CCHO历年试卷及解析

icer233

目标网站：https://ccho.eduzhixin.com/archives/tag/csst
网站总共有3页：https://ccho.eduzhixin.com/archives/tag/csst/page/1
问题：个别详情页面结构不同，大部分的答案是pdf，个别的是图片或者没有，我直接跳过了，程序会提示哪个页面不行，需要手动处理
源码如下

[Python] 纯文本查看 复制代码

# -*- coding:utf-8 -*-
import requests
from lxml import etree
import os
from multiprocessing.dummy import Pool

# 创建储存目录
if not os.path.exists('./ccho'):
    os.makedirs('./ccho')

url = 'https://ccho.eduzhixin.com/archives/tag/csst/page/'
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 Edg/128.0.0.0'
}

for i in range(1,4):
    # 创建文件目录实例
    purl = url + str(i)
    response = requests.get(url=purl, headers=headers).text
    tree = etree.HTML(response)
    # 解析div列表
    div_list = tree.xpath('/html/body/div[1]/div[4]/div/div/div[1]/div')
    def down(div):
        detail_url = div.xpath('./article/a[1]/@href')[0] # 获取详情页链接
        name = div.xpath('./article/a[1]/@title')[0]  # 获取文件名
        ans_path = './ccho/' + name +'.pdf' # 生成文件路径
        detail_page = requests.get(url=detail_url, headers=headers).text
        detail_tree = etree.HTML(detail_page)
        try:
            down_url = detail_tree.xpath('//div[@class="row"]/div/article/div/div/div//strong/a/@href')[0]
            data = requests.get(url=down_url, headers=headers).content
            with open(ans_path, 'wb') as fp:
                fp.write(data)
            print(name, 'is downloaded')
        except:
            print('Unable to download', name, ' , url:', detail_url)
    pool = Pool(5)
    pool.map(down, div_list)
    pool.close()
    pool.join()

欢迎大家指正错误

weixinzr99968

# -*- coding:utf-8 -*-
import requests
from lxml import etree
import os
import re
from multiprocessing.dummy import Pool

# 创建储存目录
if not os.path.exists('./ccho'):
    os.makedirs('./ccho')

url = 'https://ccho.eduzhixin.com/archives/tag/csst/page/'
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 Edg/128.0.0.0'
}

def down(div):
    try:
        detail_url = div.xpath('./article/a[1]/@href')[0]  # 获取详情页链接
        name = div.xpath('./article/a[1]/@title')[0]  # 获取文件名
        name = re.sub(r'[\/:*?"<>|]', '-', name)  # 替换掉非法字符
        ans_path = './ccho/' + name + '.pdf'  # 生成文件路径
        
        detail_page = requests.get(url=detail_url, headers=headers).text
        detail_tree = etree.HTML(detail_page)
        
        down_url = detail_tree.xpath('//div[@class="row"]/div/article/div/div/div//strong/a/@href')
        if down_url:
            down_url = down_url[0]
        else:
            print(f'No download link found for {name}, url: {detail_url}')
            return
        
        data = requests.get(url=down_url, headers=headers).content
        with open(ans_path, 'wb') as fp:
            fp.write(data)
        print(name, 'is downloaded')
   
    except Exception as e:
        print(f'Error downloading {name}, url: {detail_url}, error: {e}')

for i in range(1, 4):
    # 拼接页面 URL
    purl = url + str(i)
    try:
        response = requests.get(url=purl, headers=headers)
        response.raise_for_status()  # 检查请求是否成功
        response_text = response.text
    except requests.exceptions.RequestException as e:
        print(f"Error fetching page {purl}: {e}")
        continue

    tree = etree.HTML(response_text)
   
    # 解析 div 列表
    div_list = tree.xpath('/html/body/div[1]/div[4]/div/div/div[1]/div')
   
    # 使用多线程池并发下载
    pool = Pool(5)
    pool.map(down, div_list)
    pool.close()
    pool.join()

weixinzr99968

weixinzr99968 发表于 2024-9-24 17:56
# -*- coding:utf-8 -*-
import requests
from lxml import etree

        1.        修复了 url 拼接错误，确保循环时请求正确的页面。
        2.        添加了非法字符过滤，确保文件名不会因为非法字符而报错。
        3.        提取下载链接时增加了检查，避免 IndexError。
        4.        为 requests.get() 添加了异常处理，避免程序崩溃。
        5.        在多线程下载中改进了错误处理和日志输出。

icer233

爬虫思路见https://www.52pojie.cn/thread-1957905-1-1.html

baihedengge

学习了，真厉害。

woshizhj0451

本来是想学习下爬虫编程的，结果发现这个网站是真不错，在上面答了一会儿题

msmvc

虽然试题都作不出来，但是我看透了本质：如果我掌握了X，我就会作Y

W8826

完蛋  看不懂啊  小白一个

Stars666

感谢大佬分享

aiguohou

感谢分享

xiaoyiyongji

爬虫技术天下无敌！