报纸下载

FeiyuYip · 发表于 2023-12-16 19:02

本帖最后由 FeiyuYip 于 2023-12-16 19:09 编辑

勾选报纸种类、选择日期，然后开始下载即可
目前仅添加了三种的报纸种类，理论还可以扩展

一、主界面
主界面是QTDesigner画的，再转成py代码

[Python] 纯文本查看 复制代码

# -*- coding: utf-8 -*-

# Form implementation generated from reading ui file 'UI.ui'
#
# Created by: PyQt5 UI code generator 5.15.9
#
# WARNING: Any manual changes made to this file will be lost when pyuic5 is
# run again.  Do not edit this file unless you know what you are doing.


from PyQt5 import QtCore, QtGui, QtWidgets


class Ui_MainWindow(object):
    def setupUi(self, MainWindow):
        screen = QtWidgets.QDesktopWidget().screenGeometry()  # 自动适应屏幕宽高
        width = int(screen.width() / 4)
        height = int(screen.height() / 2.5)

        MainWindow.setObjectName("MainWindow")
        MainWindow.setEnabled(True)
        MainWindow.resize(width, height)
        MainWindow.setMinimumSize(QtCore.QSize(415, 515))
        MainWindow.setAcceptDrops(True)
        icon = QtGui.QIcon()
        icon.addPixmap(QtGui.QPixmap(resource_path('paper.ico')), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        MainWindow.setWindowIcon(icon)
        MainWindow.setLayoutDirection(QtCore.Qt.LeftToRight)
        MainWindow.setLocale(QtCore.QLocale(QtCore.QLocale.Chinese, QtCore.QLocale.China))
        self.centralwidget = QtWidgets.QWidget(MainWindow)
        self.centralwidget.setMinimumSize(QtCore.QSize(415, 550))
        self.centralwidget.setObjectName("centralwidget")
        self.verticalLayout = QtWidgets.QVBoxLayout(self.centralwidget)
        self.verticalLayout.setObjectName("verticalLayout")
        self.horizontalLayout = QtWidgets.QHBoxLayout()
        self.horizontalLayout.setObjectName("horizontalLayout")
        self.verticalLayout_2 = QtWidgets.QVBoxLayout()
        self.verticalLayout_2.setObjectName("verticalLayout_2")
        self.label_2 = QtWidgets.QLabel(self.centralwidget)
        self.label_2.setAlignment(QtCore.Qt.AlignCenter)
        self.label_2.setObjectName("label_2")
        self.verticalLayout_2.addWidget(self.label_2, 0, QtCore.Qt.AlignHCenter | QtCore.Qt.AlignVCenter)
        spacerItem = QtWidgets.QSpacerItem(20, 40, QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Expanding)
        self.verticalLayout_2.addItem(spacerItem)
        self.checkBox = QtWidgets.QCheckBox(self.centralwidget)
        self.checkBox.setChecked(True)
        self.checkBox.setTristate(False)
        self.checkBox.setObjectName("checkBox")
        self.verticalLayout_2.addWidget(self.checkBox)
        spacerItem1 = QtWidgets.QSpacerItem(20, 40, QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Expanding)
        self.verticalLayout_2.addItem(spacerItem1)
        self.checkBox_3 = QtWidgets.QCheckBox(self.centralwidget)
        self.checkBox_3.setChecked(True)
        self.checkBox_3.setObjectName("checkBox_3")
        self.verticalLayout_2.addWidget(self.checkBox_3)
        spacerItem2 = QtWidgets.QSpacerItem(20, 40, QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Expanding)
        self.verticalLayout_2.addItem(spacerItem2)
        self.checkBox_2 = QtWidgets.QCheckBox(self.centralwidget)
        self.checkBox_2.setChecked(True)
        self.checkBox_2.setObjectName("checkBox_2")
        self.verticalLayout_2.addWidget(self.checkBox_2)
        spacerItem3 = QtWidgets.QSpacerItem(20, 40, QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Expanding)
        self.verticalLayout_2.addItem(spacerItem3)
        self.horizontalLayout.addLayout(self.verticalLayout_2)
        self.verticalLayout_4 = QtWidgets.QVBoxLayout()
        self.verticalLayout_4.setObjectName("verticalLayout_4")
        self.label = QtWidgets.QLabel(self.centralwidget)
        self.label.setObjectName("label")
        self.verticalLayout_4.addWidget(self.label, 0, QtCore.Qt.AlignHCenter | QtCore.Qt.AlignVCenter)
        self.calendarWidget = QtWidgets.QCalendarWidget(self.centralwidget)
        self.calendarWidget.setMinimumSize(QtCore.QSize(314, 244))
        self.calendarWidget.setGridVisible(True)
        self.calendarWidget.setSelectionMode(QtWidgets.QCalendarWidget.SingleSelection)
        self.calendarWidget.setHorizontalHeaderFormat(QtWidgets.QCalendarWidget.ShortDayNames)
        self.calendarWidget.setVerticalHeaderFormat(QtWidgets.QCalendarWidget.NoVerticalHeader)
        self.calendarWidget.setDateEditEnabled(False)
        self.calendarWidget.setObjectName("calendarWidget")
        self.verticalLayout_4.addWidget(self.calendarWidget)
        self.horizontalLayout.addLayout(self.verticalLayout_4)
        self.verticalLayout.addLayout(self.horizontalLayout)
        self.horizontalLayout_2 = QtWidgets.QHBoxLayout()
        self.horizontalLayout_2.setObjectName("horizontalLayout_2")
        self.pushButton = QtWidgets.QPushButton(self.centralwidget)
        self.pushButton.setObjectName("pushButton")
        self.horizontalLayout_2.addWidget(self.pushButton)
        spacerItem4 = QtWidgets.QSpacerItem(40, 20, QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Minimum)
        self.horizontalLayout_2.addItem(spacerItem4)
        self.pushButton_2 = QtWidgets.QPushButton(self.centralwidget)
        self.pushButton_2.setObjectName("pushButton_2")
        self.horizontalLayout_2.addWidget(self.pushButton_2)
        self.verticalLayout.addLayout(self.horizontalLayout_2)
        self.horizontalLayout_5 = QtWidgets.QHBoxLayout()
        self.horizontalLayout_5.setObjectName("horizontalLayout_5")
        self.textEdit = QtWidgets.QTextEdit(self.centralwidget)
        self.textEdit.setEnabled(True)
        self.textEdit.setMinimumSize(QtCore.QSize(395, 230))
        self.textEdit.setObjectName("textEdit")
        self.horizontalLayout_5.addWidget(self.textEdit)
        self.verticalLayout.addLayout(self.horizontalLayout_5)
        MainWindow.setCentralWidget(self.centralwidget)

        self.retranslateUi(MainWindow)
        QtCore.QMetaObject.connectSlotsByName(MainWindow)

    def retranslateUi(self, MainWindow):
        _translate = QtCore.QCoreApplication.translate
        MainWindow.setWindowTitle(_translate("MainWindow", "报纸下载器"))
        self.label_2.setText(_translate("MainWindow", "报纸种类"))
        self.checkBox.setText(_translate("MainWindow", "人民日报"))
        self.checkBox_3.setText(_translate("MainWindow", "江西日报"))
        self.checkBox_2.setText(_translate("MainWindow", "赣南日报"))
        self.label.setText(_translate("MainWindow", "选择日期"))
        self.pushButton.setText(_translate("MainWindow", "开始下载"))
        self.pushButton_2.setText(_translate("MainWindow", "查看下载"))


def resource_path(relative):
    import os, sys
    if hasattr(sys, "_MEIPASS"):
        absolute_path = os.path.join(sys._MEIPASS, relative)
    else:
        absolute_path = os.path.join(relative)
    return absolute_path

二、逻辑部分
1.RMRB下载

[Python] 纯文本查看 复制代码

from get_page import get_page, make_dir, header, merge_pdf, handle_string
from lxml import etree
import os
import requests
import time


def download(url, date, year, month, day, date_path, paper_name):
    response = get_page(url)

    seclector = etree.HTML(response)
    chapter_urls = seclector.xpath('//div[@class="swiper-slide"]/a/@href')
    chapter_names = seclector.xpath('//div[@class="swiper-slide"]/a/text()')

    print(f'{paper_name}（{date}） 共有 {len(chapter_urls)} 个版面')

    for chapter_name, chapter_url in zip(chapter_names, chapter_urls):
        chapter_name = handle_string(chapter_name)
        # print(chapter_name)
        if os.path.exists(f'{date_path}/{chapter_name}.pdf'):
            pass
        else:
            chapter_url = f'http://paper.people.com.cn/rmrb/html/{year}-{month}/{day}/nbs.D110000renmrb_{str(chapter_urls.index(chapter_url) + 1).zfill(2)}.htm'
            chapter_response = get_page(chapter_url)
            chapter_seclector = etree.HTML(chapter_response)
            chapter_pdf_dowload_url = chapter_seclector.xpath('//p[@class="right btn"]/a/@href')[0]
            chapter_pdf_dowload_url = [x for x in chapter_pdf_dowload_url.split('/')[-5:]]
            # # http://paper.people.com.cn/rmrb/images/2023-11/13/01/rmrb2023111301.pdf
            chapter_pdf_dowload_url = 'http://paper.people.com.cn/rmrb/' + '/'.join(chapter_pdf_dowload_url)

            #
            pdf_response = requests.get(chapter_pdf_dowload_url, headers=header)
            if pdf_response.headers['Content-Type'] == 'application/pdf':
                with open(f'{date_path}/{chapter_name}.pdf', 'wb') as f:
                    f.write(pdf_response.content)
            print(f'{paper_name}（{date}） {chapter_name}  保存完毕!!!')

    time.sleep(1)
    merge_pdf(date, date_path, paper_name)


def main(paper_name, date):
    year = date.split('-')[0]
    month = date.split('-')[1]
    day = date.split('-')[2]

    date_path = f'{paper_name}/{date}'
    make_dir(date_path)

    url = f'http://paper.people.com.cn/rmrb/html/{year}-{month}/{day}/nbs.D110000renmrb_01.htm'

    download(url, date, year, month, day, date_path, paper_name)


if __name__ == '__main__':
    paper_name = '人民日报'
    date = '2023-12-12'
    main(paper_name, date)

2.JXRB下载

[Python] 纯文本查看 复制代码

import requests
import os
import time
from lxml import etree

from get_page import get_page, make_dir, header, merge_pdf,handle_string


def download(url, date, year, month, day, date_path, paper_name):
    response = get_page(url)
    seclector = etree.HTML(response)
    chapter_urls = seclector.xpath('//td[@class="szb_text_color"]/a/@href')
    chapter_names = seclector.xpath('//td[@class="szb_text_color"]/a/text()')

    print(f'{paper_name}（{date}） 共有 {len(chapter_urls)} 个版面')

    for chapter_name, chapter_url in zip(chapter_names, chapter_urls):
        chapter_name = handle_string(chapter_name)
        # print(chapter_name)
        if os.path.exists(f'{date_path}/{chapter_name}.pdf'):
            pass
        else:
            chapter_url = f'http://epaper.jxxw.com.cn/html/{year}-{month}/{day}/{chapter_url}'
            chapter_response = get_page(chapter_url)
            chapter_seclector = etree.HTML(chapter_response)
            chapter_pdf_dowload_url = chapter_seclector.xpath('//a[@id="bigbmshowpdf"]/@href')[0]
            chapter_pdf_dowload_url = [x for x in chapter_pdf_dowload_url.split('/')[-3:]]
            chapter_pdf_dowload_url = 'http://epaper.jxxw.com.cn/resfile/' + '/'.join(chapter_pdf_dowload_url)

            pdf_response = requests.get(chapter_pdf_dowload_url, headers=header)
            if pdf_response.headers['Content-Type'] == 'application/pdf':
                with open(f'{date_path}/{chapter_name}.pdf', 'wb') as f:
                    f.write(pdf_response.content)
            print(f'{paper_name}（{date}） {chapter_name}  保存完毕!!!')

    time.sleep(1)
    merge_pdf(date, date_path, paper_name)


def main(paper_name, date):
    year = date.split('-')[0]
    month = date.split('-')[1]
    day = date.split('-')[2]

    date_path = f'{paper_name}/{date}'
    make_dir(date_path)

    url = f'http://epaper.jxxw.com.cn/html/{year}-{month}/{day}/index_{date}.htm'
    download(url, date, year, month, day, date_path, paper_name)


if __name__ == '__main__':
    paper_name = '江西日报'
    date = '2023-12-12'
    main(paper_name, date)

3.GNRB下载

[Python] 纯文本查看 复制代码

import requests
import os
import time
from lxml import etree

from get_page import get_page, make_dir, header, merge_pdf,handle_string


def download(url, date, year, month, day, date_path, paper_name):
    response = get_page(url)
    # print(response)
    seclector = etree.HTML(response)
    chapter_urls = seclector.xpath('//td[@class="szb_text_color"]/a/@href')
    chapter_names = seclector.xpath('//td[@class="szb_text_color"]/a/text()')

    print(f'{paper_name}（{date}） 共有 {len(chapter_urls)} 个版面')

    for chapter_name, chapter_url in zip(chapter_names, chapter_urls):
        chapter_name = handle_string(chapter_name)
        if os.path.exists(f'{date_path}/{chapter_name}.pdf'):
            print(f'{paper_name}（{date}） {chapter_name}  保存完毕~~~')
        else:
            # https://szb.gnrbs.cn/html/2023-12/16/node_95762.htm
            chapter_url = f'https://szb.gnrbs.cn/html/{year}-{month}/{day}/{chapter_url}'
            chapter_response = get_page(chapter_url)
            chapter_seclector = etree.HTML(chapter_response)
            chapter_pdf_dowload_url = chapter_seclector.xpath('//a[@id="bigbmshowpdf"]/@href')[0]
            chapter_pdf_dowload_url = [x for x in chapter_pdf_dowload_url.split('/')[-3:]]

            # https://szb.gnrbs.cn/resfile/2023-12-16/01/gnrb-20231216-001.pdf
            chapter_pdf_dowload_url = 'https://szb.gnrbs.cn/resfile/' + '/'.join(chapter_pdf_dowload_url)
            # print(chapter_pdf_dowload_url, chapter_name)

            pdf_response = requests.get(chapter_pdf_dowload_url, headers=header)
            if pdf_response.headers['Content-Type'] == 'application/pdf':
                with open(f'{date_path}/{chapter_name}.pdf', 'wb') as f:
                    f.write(pdf_response.content)
                print(f'{paper_name}（{date}） {chapter_name}  保存完毕!!!')

    time.sleep(1)
    #
    print(f'{paper_name}（{date}） 版面下载完毕，开始合并，请稍候……')
    merge_pdf(date, date_path, paper_name)


def main(paper_name, date):

    year = date.split('-')[0]
    month = date.split('-')[1]
    day = date.split('-')[2]

    date_path = f'{paper_name}/{date}'
    make_dir(date_path)

    url = f'https://szb.gnrbs.cn/html/{year}-{month}/{day}/index_{year}-{month}-{day}.htm'
    download(url, date, year, month, day, date_path, paper_name)


if __name__ == '__main__':
    main()

4.调用逻辑

[Python] 纯文本查看 复制代码

from PyQt5 import QtWidgets, QtCore, QtGui
from UI import Ui_MainWindow

from PyQt5.QtWidgets import *
from PyQt5.QtCore import *
from PyQt5.QtGui import *

import rmrb_downloader
import jxrb_downloader
import gnrb_downloader

import os


# 發射信號
class Stream(QObject):
    newText = pyqtSignal(str)

    def write(self, text):
        self.newText.emit(str(text))
        QApplication.processEvents()


class Thread(QThread):
    def __init__(self, paper_selector_list=None, select_date=None, parent=None):
        super(Thread, self).__init__(parent)

        self.paper_selector_list = paper_selector_list
        self.select_date = select_date

    def __del__(self):
        self.wait()

    def run(self):
        print('--------分--------隔--------线--------')
        print(f'选择的报纸种类有：{self.paper_selector_list}')
        print(f'选择的下载日期是：{self.select_date}')
        try:
            for self.paper_name in self.paper_selector_list:
                if self.paper_name == '人民日报':
                    print('--------分--------隔--------线--------')
                    print(f'开始下载 {self.select_date} {self.paper_name}')
                    rmrb_downloader.main(self.paper_name, self.select_date)
                if self.paper_name == '江西日报':
                    print('--------分--------隔--------线--------')
                    print(f'开始下载 {self.select_date} {self.paper_name}')
                    jxrb_downloader.main(self.paper_name, self.select_date)
                if self.paper_name == '赣南日报':
                    print('--------分--------隔--------线--------')
                    print(f'开始下载 {self.select_date} {self.paper_name}')
                    gnrb_downloader.main(self.paper_name, self.select_date)
                print('--------分--------隔--------线--------')

        except Exception as e:
            print(e)


class MainWindow(QtWidgets.QMainWindow, Ui_MainWindow):
    def __init__(self, parent=None):
        super(MainWindow, self).__init__(parent)
        self.setupUi(self)
        self.center()

        # 发射信息
        sys.stdout = Stream(newText=self.onUpdateText)
        sys.stderr = Stream(newText=self.onUpdateText)

        # 开始下载
        self.pushButton.clicked.connect(self.start_download)
        # 查看下载
        self.pushButton_2.clicked.connect(self.show_download)

    def start_download(self):
        # 选择的日期
        self.select_date = self.get_date()

        # 选择的报纸种类
        self.paper_selector_list = self.get_paper_name()

        self.thread = Thread()
        self.thread.paper_selector_list = self.paper_selector_list
        self.thread.select_date = self.select_date
        self.thread.start()

    def show_download(self):
        try:
            os.startfile(f'{os.getcwd()}')
        except Exception as e:
            print(e)

    def get_paper_name(self):
        paper_selector_list = []
        if self.checkBox.isChecked():  # 人民日报
            paper_selector_list.append(self.checkBox.text())
        if self.checkBox_3.isChecked():  # 赣南日报
            paper_selector_list.append(self.checkBox_3.text())
        if self.checkBox_2.isChecked():  # 江西日报
            paper_selector_list.append(self.checkBox_2.text())

        return paper_selector_list

    def get_date(self):
        date = QtCore.QDate(self.calendarWidget.selectedDate())
        year = date.year()
        month = str(date.month()).zfill(2)  # 补齐2位
        day = str(date.day()).zfill(2)  # 补齐2位
        select_date = f'{year}-{month}-{day}'
        return select_date

    def onUpdateText(self, text):
        """Write console output to text widget."""
        cursor = self.textEdit.textCursor()
        cursor.movePosition(QTextCursor.End)
        cursor.insertText(text)
        self.textEdit.setTextCursor(cursor)
        self.textEdit.ensureCursorVisible()

    def center(self):
        qr = self.frameGeometry()
        cp = QDesktopWidget().availableGeometry().center()
        qr.moveCenter(cp)
        self.move(qr.topLeft())

    def closeEvent(self, event):
        reply = QMessageBox.question(self, '退出提示',
                                     "您确定要退出吗？", QMessageBox.Yes |
                                     QMessageBox.No, QMessageBox.No)

        if reply == QMessageBox.Yes:
            event.accept()
        elif reply == QMessageBox.No:
            event.ignore()


def resource_path(relative):
    import os, sys
    if hasattr(sys, "_MEIPASS"):
        absolute_path = os.path.join(sys._MEIPASS, relative)
    else:
        absolute_path = os.path.join(relative)
    return absolute_path


if __name__ == '__main__':
    import sys

    app = QtWidgets.QApplication(sys.argv)
    mainWindow = MainWindow()
    mainWindow.show()
    sys.exit(app.exec_())

5.其他代码，放到一个py文件里了

[Python] 纯文本查看 复制代码

import requests
import os
from pypdf import PdfMerger, PdfReader
import shutil
import time
import re

header = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36'
}


def handle_string(string):
    string = re.sub(r'\r\n *', '', str(string))
    return string


def merge_pdf(date, date_path, paper_name):
    pdf_list = os.listdir(date_path)
    pdf_merge = PdfMerger()
    pdf_list.sort()

    # 记录每次合并后的总页数
    pdf_page_num = 0
    for pdf in pdf_list:
        pdf_path = os.path.join(date_path, pdf)
        pdf_in = PdfReader(pdf_path, strict=False)
        # 每张报纸的标题
        pdf_title = pdf.split('.')[0]

        pdf_merge.append(pdf_path)
        pdf_merge.add_outline_item(pdf_title, pdf_page_num, None)
        pdf_page_num += len(pdf_in.pages)

    pdf_merge.write(f'{paper_name}/{paper_name}（{date}）.pdf')
    print(f'{paper_name}（{date}).pdf下载完成')
    pdf_merge.close()

    time.sleep(1)
    # 删除文件夹
    shutil.rmtree(date_path)


def make_dir(path):
    if os.path.exists(path):
        pass
    else:
        os.makedirs(path)


def get_page(url):
    response = requests.get(url, headers=header)
    response.encoding = 'utf-8'
    if response.status_code == 200:
        return response.text

三、界面展示
微信截图_20231216190016.png

四、不足之处
1.三个下载逻辑相似，可以集成到一个函数中调用
2.使用pypdf库合并pdf，会出现“Multiple definitions in dictionary at byte 0x13866c for key /Ascent”的信息，不过不影响使用
有知道怎样解决的大佬还望帮助解决

五、声明
此仅供学习研究使用，请勿用于其他用途

FeiyuYip · 发表于 2023-12-16 20:48

wazct 发表于 2023-12-16 20:45
问一下这个get_page模块怎么下载啊，我找不到get_page库
[mw_shl_code=python,true]from get_page import ...

就是第5项，我随便起的名

知心 · 发表于 2023-12-16 22:15

wazct 发表于 2023-12-16 20:45
问一下这个get_page模块怎么下载啊，我找不到get_page库
[mw_shl_code=python,true]from get_page import ...

自己写的，第5部分的代码就是。你自己新建一个get_page.py的文件，把代码拷贝进去就行

hacker23 · 发表于 2023-12-16 20:27

能添加一个电脑报吗，万分感谢

wangfs111 · 发表于 2023-12-17 10:24

非常开心，已经成功运行下载！
注意5个文件名：报纸下载器（调用文件），UI，get_page（其他文件）, rmrb_downloader，jxrb_downloader，gnrb_downloader

52soft · 发表于 2023-12-16 19:17

高手的技术就是牛

hanchao2021 · 发表于 2023-12-16 19:27

虽然不用，还是点赞

cbkxh · 发表于 2023-12-16 19:27

有没有成品软件呢

光年之外 · 发表于 2023-12-16 19:52

感谢分享~

jxsdxxf · 发表于 2023-12-16 20:27

这年头还有人看报纸？？今日头条不香吗。。。。

qsr0554 · 发表于 2023-12-16 20:31

能发一个打包好的嘛

wen198710 · 发表于 2023-12-16 20:39

好牛啊1987年的能下载不

wazct · 发表于 2023-12-16 20:45

问一下这个get_page模块怎么下载啊，我找不到get_page库

[Python] 纯文本查看 复制代码

from get_page import get_page, make_dir, header, merge_pdf,handle_string

用pip安装时候出现如下报错

[Python] 纯文本查看 复制代码

ERROR: Could not find a version that satisfies the requirement get_page (from versions: none)
ERROR: No matching distribution found for get_page

不然代码运行不了

[Python] 纯文本查看 复制代码

from get_page import get_page, make_dir, header, merge_pdf, handle_string

ModuleNotFoundError: No module named 'get_page'

帐号		自动登录	找回密码
密码			注册[Register]

[Python 原创] 报纸下载

免费评分

本帖被以下淘专辑推荐:

免费评分

个人中心