Python获取人人电影网的链接

鸠山一茶

平时有收藏影视作品的爱好，很久之前接触到人人电影网（https://www.rrdynb.com/）
但是一个一个的保存非常耗费时间，于是使用Python写了两个脚本用来实现保存链接的目的
首先来对网站进行分析
这是一个很简单的网页，可以直接使用URL进行请求
https://www.rrdynb.com/dianshiju/2023/0607/35586.html

于是就可以实现下面的代码

[Python] 纯文本查看 复制代码

import requests
import pandas as pd
import os
import logging
import json
import random
import time
from bs4 import BeautifulSoup

# Logger setup
logging.basicConfig(filename='extractor.log',encoding="utf-8", level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger()

DOWNLOAD_METHODS = ["阿里网盘", "夸克网盘", "迅雷云盘", "百度网盘"]

USER_AGENTS = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3 Edge/16.16299",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
    # 可根据需要添加更多的User Agent
]


def extract_html_content_with_class(url, class_name):

    user_agent = random.choice(USER_AGENTS)
    headers = {"User-Agent": user_agent}

    try:
        response = requests.get(url,headers=headers)
        response.raise_for_status()
        soup = BeautifulSoup(response.text, 'html.parser')
        return soup.find(class_=class_name)
    except Exception as e:
        logger.error(f"获取网址 {url} 的内容时出现错误：{e}")
        return None


def modified_extract_links(html_content):
    soup = BeautifulSoup(html_content, 'html.parser')
    resources = soup.find_all('a', href=True)

    links_dict = {
        "阿里网盘链接": None,
        "夸克网盘链接": None,
        "迅雷云盘链接": None,
        "百度网盘链接": None
    }

    for resource in resources:
        link = resource['href']
        if "aliyundrive.com" in link:
            links_dict["阿里网盘链接"] = link
        elif "pan.quark.cn" in link:
            links_dict["夸克网盘链接"] = link
        elif "pan.xunlei.com" in link:
            links_dict["迅雷云盘链接"] = link
        elif "pan.baidu.com" in link:
            links_dict["百度网盘链接"] = link

    return [links_dict.get(method + "链接", None) for method in DOWNLOAD_METHODS]


def write_links_to_csv(filename, movie_name, data):
    data_row = [movie_name] + data
    df = pd.DataFrame([data_row], columns=['网址链接', '阿里云盘链接', '夸克网盘链接', '迅雷云盘链接', '百度网盘链接'])
    try:
        df.to_csv(filename, index=False, encoding='utf-8-sig', mode='a', header=(not os.path.exists(filename)))
    except Exception as e:
        logger.error(f"写入CSV文件时出现错误：{e}")


if __name__ == "__main__":
    input_file = input("请输入JSON文件路径：")
    csv_filename = os.path.splitext(os.path.basename(input_file))[0] + ".csv"

    start_index = 0
    if os.path.exists("breakpoint.txt"):
        with open("breakpoint.txt", "r") as bp:
            start_index = int(bp.read().strip())

    with open(input_file, 'r') as file:
        urls = json.load(file)

    for index, url in enumerate(urls[start_index:]):
        logger.info(f"开始处理网址：{url}")
        movie_txt_content = extract_html_content_with_class(url, 'movie-txt')
        if movie_txt_content:
            download_links = modified_extract_links(str(movie_txt_content))
            if download_links:
                write_links_to_csv(csv_filename, url, download_links)
            else:
                logger.warning("未找到下载链接。")
        else:
            logger.warning("未找到带有'movie-txt' class的HTML内容。")

        # 随机暂停
        if random.randint(1, 10) <= 3:  # 30% chance to pause after a URL
            sleep_time = random.randint(5, 15)
            logger.info(f"随机暂停{sleep_time}秒")
            time.sleep(sleep_time)

        # 断点再续文件
        with open("breakpoint.txt", "w") as bp:
            bp.write(str(index + start_index))

但是我们发现这个URL没有什么规律，这对于我们全自动的抓取就很不友好
经过对网站的挖掘发现，网站一个提供了4个列表对全部的网页进行展示

所以就可以根据这四个页面写一个脚本，先把全部的网页链接抓取下来，然后再使用脚本进行抓取，下面展示内容

[Python] 纯文本查看 复制代码

import requests
import random
import time
import json
import logging
import os
from bs4 import BeautifulSoup
from datetime import datetime

USER_AGENTS = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3 Edge/16.16299",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
    # 可根据需要添加更多的User Agent
]

# 设置日志模块使用utf-8编码
log_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'crawler.log')
logging.basicConfig(filename=log_file_path, level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', encoding='utf-8')

def get_target_links(category, listnum, start_page=1):
    category_list = [
        ("dongman", "13"),
        ("movie", "2"),
        ("dianshiju", "6"),
        ("zongyi", "10"),
    ]

    # Check if the provided category and listnum are within valid range
    if not (1 <= category <= len(category_list)):
        logging.warning("无效的选项。")
        return []

    if not (1 <= listnum <= len(category_list)):
        logging.warning("无效的选择。")
        return []

    # Get the selected category and listnum
    selected_category, selected_listnum = category_list[category - 1]

    base_url = f"https://www.rrdynb.com/{selected_category}/list_{selected_listnum}_"
    page_number = start_page
    target_links = []

    while True:
        url = f"{base_url}{page_number}.html"

        # 随机选择一个User Agent
        user_agent = random.choice(USER_AGENTS)
        headers = {"User-Agent": user_agent}

        logging.info(f"正在获取第{page_number}页...")
        response = requests.get(url, headers=headers)

        if response.status_code == 200:
            logging.info(f"成功连接，正在抓取...")
            soup = BeautifulSoup(response.text, "html.parser")
            target_link = soup.find("a", class_="movie-thumbnails")["href"]
            target_links.append(target_link)
            page_number += 1
            time.sleep(3)  # 暂停
        elif response.status_code == 404:
            logging.info(f"已获取所有页面，共{page_number - 1}页。")
            break
        else:
            logging.warning(f"获取第{page_number}页失败。5秒后重试...")
            time.sleep(5)  # 5秒后重试
            continue

    return target_links

def save_to_json(target_links, category):
    categories = {
        1: "动漫",
        2: "电影",
        3: "电视剧",
        4: "老电影",
    }

    if category not in categories:
        logging.warning("无效的选项。")
        return

    today = datetime.now().strftime("%Y%m%d")
    file_name = f"{categories[category]}_{today}.json"
    file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), file_name)
    with open(file_path, "w", encoding="utf-8") as json_file:
        json.dump(target_links, json_file, ensure_ascii=False, indent=4)

if __name__ == "__main__":
    print("1. 动漫")
    print("2. 电影")
    print("3. 电视剧")
    print("4. 老电影")

    selected_category = int(input("请选择一个选项："))

    logging.info(f"用户选择了选项 {selected_category}")

    checkpoint_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'checkpoint.txt')
    try:
        with open(checkpoint_file_path, 'r') as checkpoint_file:
            start_page = int(checkpoint_file.readline())
    except FileNotFoundError:
        start_page = 1

    target_links = get_target_links(selected_category, start_page)
    if target_links:
        today = datetime.now().strftime("%Y%m%d")
        with open(checkpoint_file_path, 'w') as checkpoint_file:
            checkpoint_file.write(f"{len(target_links) + start_page - 1}_{today}")
            logging.info("断点已保存。")

        logging.info("目标链接列表：")
        for link in target_links:
            logging.info(link)

            # 保存当前页的链接至JSON文件并添加https://www.rrdynb.com前缀
            current_page_links = [f"https://www.rrdynb.com{link}" for link in target_links]
            save_to_json(current_page_links, selected_category)
            logging.info("当前页链接已保存至对应的JSON文件。")

        logging.info(f"{len(target_links)}页获取成功已保存到JSON文件当中。")
    else:
        logging.warning("未找到目标链接。")

于是就可以成功的抓取到内容

下面是部分抓取到的内容展示

我已经抓取到了相关的文件，为了给网站方服务器不造成影响，下面将内容打包放在下面
https://www.123pan.com/s/oNv9-7yJ2.html提取码:52pj

52soft

好好学习

吖力锅

哇哇，这代码不少哇

5cfsz

谢谢楼主分享

Blackberry01

感谢分享

nanqian

用心了，感谢分享

kabin

感谢楼主分享，都打包了，真方便，谢谢

skzhaixing

学习一下  不错的思路

Ylvan

收藏了，有空学习！

acs

代码学习了