前言
这套源码只包含python部分，vue部分是编译后的。前端用的是vue和vueify还有axios。后端是Flask，楼主也是业余选手一个。代码不是很好看，目的还是分享一下学习的过程。这篇帖子，教程向较少，因为楼主文笔不是很好，有啥问题或者建议可以评论区交流。
首先先上后端部分代码
Python部分
爬虫代码
import requests
from bs4 import BeautifulSoup
from urllib.parse import quote
from multiprocessing.dummy import Pool as ThreadPool
def get_ip():
resp = requests.get(
'代{过}{滤}理api地址')
return resp.text
class Spider:
def __init__(self):
ip = get_ip()
self.proxies = {
'http': 'http://' + ip,
'https': 'http://' + ip
}
self.base_url = 'https://www.btbtt11.com'
self.headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36'
}
def search(self, keyword):
keyword = quote(keyword)
search_url = f'{self.base_url}/search-index-keyword-{keyword}.htm'
resp = requests.get(search_url, headers=self.headers, proxies=self.proxies)
soup = BeautifulSoup(resp.text, 'lxml')
all_search_result = soup.find_all('table')
search_list = []
url_list = []
for i in all_search_result:
filter_one = i.find(attrs={'class': 'subject_link thread-old'})
movie_name, movie_url = '', '',
if filter_one is not None:
movie_name = filter_one.text
movie_url = filter_one.get('href')
url_list.append(movie_url)
search_list.append({'name': movie_name, 'url': movie_url}) # , 'bt_list': bt_list
if search_list:
pool = ThreadPool(30)
results = pool.map(self.get_down_url, url_list)
# close the pool and wait for the work to finish
pool.close()
pool.join()
for k in range(len(search_list)):
search_list[k]['bt_list'] = results[k]
return search_list[1:]
else:
return None
def get_down_url(self, url):
get_url = f'{self.base_url}/{url}'
resp = requests.get(get_url, headers=self.headers, proxies=self.proxies)
soup = BeautifulSoup(resp.text, 'lxml')
try:
all_bt_resource = soup.find(attrs={'class': 'attachlist'}).find_all('a')
except AttributeError:
return None
resource_list = []
for i in all_bt_resource:
bt_name = i.text
bt_url = i.get('href').replace('dialog', 'download')
resource_list.append({'name': bt_name, 'url': self.base_url + '/' + bt_url})
return resource_list
由于网站有反爬机制，爬取过多时会报错，所以加了个代{过}{滤}理，这个大家自己用可以直接注释掉。由于一次性需要爬取的数据有点多，如果不用并行的话，会很慢。所以用了一个map。其他没什么特别的地方。
flask部分
from flask import Flask, request, jsonify
from spider.bt_spider import Spider
from flask_cors import CORS
app = Flask(__name__)
CORS(app, supports_credentials=True)
@app.route('/<path:fallback>')
def fallback(fallback):
if fallback.startswith('css/') or fallback.startswith('js/') \
or fallback.startswith('img/') or fallback == 'favicon.ico':
return app.send_static_file(fallback)
else:
return app.send_static_file('index.html')
@app.route('/')
def hello_world():
return app.send_static_file('index.html')
@app.route('/api/search')
def search_movie():
search_keyword = request.args.get('keyword')
search_resluts = Spider().search(search_keyword)
if search_resluts is None:
return jsonify({'status': 'error', 'reslut': 'No Result'})
else:
return jsonify({'status': 'ok', 'reslut': search_resluts})
@app.route('/api/download')
def download():
down_url = request.args.get('url')
down_resluts = Spider().get_down_url(down_url)
if down_resluts is None:
return jsonify({'status': 'error', 'reslut': 'No Result'})
else:
return jsonify({'status': 'ok', 'reslut': down_resluts})
if __name__ == '__main__':
app.run()
CROS是为了解决跨域问题，也很简单其他没什么可说的。最后展示一下成品
另外我把这套代码部署到了服务器，大家可以在线体验下。不保证稳定性，也应该不会怎么维护。
http://129.226.40.25/
| 
[复制链接]
发表于 2022-5-9 23:20
|
发表于 2022-5-9 23:47
