# 爬虫部分
# 导入必要的包
from bs4 import BeautifulSoup
import pandas as pd
from tqdm import tqdm
import math
import requests
import lxml
import re
import time
# 构造url字典
area_dic = {#'罗湖区':'luohuqu',
#'福田区':'futianqu',
'南山区':'nanshanqu',
#'盐田区':'yantianqu',
#'宝安区':'baoanqu',
#'龙岗区':'longgangqu',
#'龙华区':'longhuaqu',
#'坪山区':'pingshanqu'
}
# 当正则表达式匹配失败时，返回默认值（errif）
def re_match(re_pattern, string, errif=None):
try:
return re.findall(re_pattern, string)[0].strip()
except IndexError:
return errif
# 主函数部分，
# 通过request获取源码，
# 通过正则表达式提取相应的字段，
# 通过BeautifulSoup包获取房子的信息，
# DataFrame存储信息
data = pd.DataFrame()
for key_, value_ in area_dic.items():
# 加个header进行伪装
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36',
'Referer': 'https://sz.lianjia.com/ershoufang/'}
# 新建一个会话
sess = requests.session()
sess.get('https://sz.lianjia.com/ershoufang/', headers=headers)
# url示例：https://sz.lianjia.com/ershoufang/luohuqu/pg2/
url = 'https://sz.lianjia.com/ershoufang/{}/pg{}/'
# 获取该行政区下房源记录数
start_url = 'https://sz.lianjia.com/ershoufang/{}/'.format(value_)
html = sess.get(start_url).text
# print(html[:100])
print(re.findall('共找到<span> (.*?) </span>套.*二手房', html))
house_num = re.findall('共找到<span> (.*?) </span>套.*二手房', html)[0].strip()
print('{}: 二手房源共计{}套'.format(key_, house_num))
time.sleep(1)
# 页面限制 每个行政区只能获取最多100页共计3000条房源信息
total_page = int(math.ceil(min(3000, int(house_num)) / 30.0))
for i in tqdm(range(total_page), desc=key_):
html = sess.get(url.format(value_, i+1)).text
soup = BeautifulSoup(html, 'lxml')
info_collect = soup.find_all(class_="info clear")
for info in info_collect:
info_dic = {}
# 行政区
info_dic['area'] = key_
# 房源的标题
info_dic['title'] = re_match('target="_blank">(.*?)</a><!--', str(info))
# 小区名
info_dic['community'] = re_match('xiaoqu.*?target="_blank">(.*?)</a>', str(info))
# 位置
info_dic['position'] = re_match('<a href.*?target="_blank">(.*?)</a>.*?class="address">', str(info))
# 税相关，如房本满5年
info_dic['tax'] = re_match('class="taxfree">(.*?)</span>', str(info))
# 总价
info_dic['total_price'] = float(re_match('class="totalPrice"><span>(.*?)</span>万', str(info)))
# 单价
info_dic['unit_price'] = float(re_match('data-price="(.*?)"', str(info)))
# 匹配房源标签信息，通过|切割
# 包括面积，朝向，装修等信息
icons = re.findall('class="houseIcon"></span>(.*?)</div>', str(info))[0].strip().split('|')
info_dic['hourseType'] = icons[0].strip()
info_dic['hourseSize'] = float(icons[1].replace('平米', ''))
info_dic['direction'] = icons[2].strip()
info_dic['fitment'] = icons[3].strip()
# 存入DataFrame
if data.empty:
data = pd.DataFrame(info_dic,index=[0])
else:
data = data.append(info_dic,ignore_index=True)
# 去掉面积10000+平米的房源记录（离群值），查看我们爬取到的信息
data = data[data['hourseSize'] < 10000]
data.head()
#数据可视化分析
from pyecharts.charts import *
from pyecharts import options as opts
from pyecharts.commons.utils import JsCode
from jieba import posseg as psg
import collections
#价格和面积的散点图
scatter = (Scatter(init_opts=opts.InitOpts(theme='dark'))
.add_xaxis(data['hourseSize'])
.add_yaxis("房价", data['total_price'])
.set_series_opts(label_opts=opts.LabelOpts(is_show=False),
markpoint_opts=opts.MarkPointOpts(data=[opts.MarkPointItem(type_="max", name="最大值"),]))
.set_global_opts(
legend_opts=opts.LegendOpts(is_show=False),
title_opts=opts.TitleOpts(title="深圳二手房 总价-面积 散点图"),
xaxis_opts=opts.AxisOpts(
name='面积',
# 设置坐标轴为数值类型
type_="value",
# 不显示分割线
splitline_opts=opts.SplitLineOpts(is_show=False)),
yaxis_opts=opts.AxisOpts(
name='总价',
name_location='middle',
# 设置坐标轴为数值类型
type_="value",
# 默认为False表示起始为0
is_scale=True,
splitline_opts=opts.SplitLineOpts(is_show=False),),
visualmap_opts=opts.VisualMapOpts(is_show=True, type_='color', min_=100, max_=1000)
))
scatter.load_javascript()
scatter.render_notebook()
#二手房均价地图图
temp = data.groupby(['community'])['unit_price'].agg(['mean', 'count']).reset_index()
# 该小区内至少3套在售房源才统计
data_pair = sorted([(row['community'], round(row['mean']/10000, 1)) if row['count']>=3 else (0, 0)
for _, row in temp.iterrows()], key=lambda x: x[1], reverse=True)[:10]
bar = (Bar(init_opts=opts.InitOpts(theme='dark'))
.add_xaxis([x[0] for x in data_pair[::-1]])
.add_yaxis('二手房均价', [x[1] for x in data_pair[::-1]])
.set_series_opts(label_opts=opts.LabelOpts(is_show=True,
position='insideRight',
font_style='italic'),
itemstyle_opts=opts.ItemStyleOpts(
color=JsCode("""new echarts.graphic.LinearGradient(1, 0, 0, 0,
[{
offset: 0,
color: 'rgb(0,206,209)'
}, {
offset: 1,
color: 'rgb(218,165,32)'
}])"""))
)
.set_global_opts(
title_opts=opts.TitleOpts(title="深圳二手房均价TOP 10小区"),
legend_opts=opts.LegendOpts(is_show=False),
tooltip_opts=opts.TooltipOpts(formatter='{b}:{c}万元'),
xaxis_opts=opts.AxisOpts(min_=14),
)
.reversal_axis()
)
bar.render_notebook()
#均价top10小区条形图
temp = data.groupby(['community'])['unit_price'].agg(['mean', 'count']).reset_index()
# 该小区内至少3套在售房源才统计
data_pair = sorted([(row['community'], round(row['mean']/10000, 1)) if row['count']>=3 else (0, 0)
for _, row in temp.iterrows()], key=lambda x: x[1], reverse=True)[:10]
bar = (Bar(init_opts=opts.InitOpts(theme='dark'))
.add_xaxis([x[0] for x in data_pair[::-1]])
.add_yaxis('二手房均价', [x[1] for x in data_pair[::-1]])
.set_series_opts(label_opts=opts.LabelOpts(is_show=True,
position='insideRight',
font_style='italic'),
itemstyle_opts=opts.ItemStyleOpts(
color=JsCode("""new echarts.graphic.LinearGradient(1, 0, 0, 0,
[{
offset: 0,
color: 'rgb(0,206,209)'
}, {
offset: 1,
color: 'rgb(218,165,32)'
}])"""))
)
.set_global_opts(
title_opts=opts.TitleOpts(title="深圳二手房均价TOP 10小区"),
legend_opts=opts.LegendOpts(is_show=False),
tooltip_opts=opts.TooltipOpts(formatter='{b}:{c}万元'),
xaxis_opts=opts.AxisOpts(min_=14),
)
.reversal_axis()
)
bar.render_notebook()
#均价top10地段
temp = data.groupby(['position'])['unit_price'].mean().reset_index()
data_pair = sorted([(row['position'], round(row['unit_price']/10000, 1))
for _, row in temp.iterrows()], key=lambda x: x[1], reverse=True)[:10]
bar = (Bar(init_opts=opts.InitOpts(theme='dark'))
.add_xaxis([x[0] for x in data_pair])
.add_yaxis('二手房均价', [x[1] for x in data_pair])
.set_series_opts(label_opts=opts.LabelOpts(is_show=True, font_style='italic'),
itemstyle_opts=opts.ItemStyleOpts(
color=JsCode("""new echarts.graphic.LinearGradient(0, 1, 0, 0,
[{
offset: 0,
color: 'rgb(0,206,209)'
}, {
offset: 1,
color: 'rgb(218,165,32)'
}])"""))
)
.set_global_opts(
title_opts=opts.TitleOpts(title="深圳二手房均价TOP 10地段"),
legend_opts=opts.LegendOpts(is_show=False),
tooltip_opts=opts.TooltipOpts(formatter='{b}:{c}万元'))
)
bar.render_notebook()
# 户型分布图
temp = data.groupby(['hourseType'])['area'].count().reset_index()
data_pair = sorted([(row['hourseType'], row['area'])
for _, row in temp.iterrows()], key=lambda x: x[1], reverse=True)[:10]
pie = (Pie(init_opts=opts.InitOpts(theme='dark'))
.add('', data_pair,
radius=["30%", "75%"],
rosetype="radius")
.set_global_opts(title_opts=opts.TitleOpts(title="深圳二手房 户型分布"),
legend_opts=opts.LegendOpts(is_show=False),)
.set_series_opts(label_opts=opts.LabelOpts(formatter="{b}: {d}%"))
)
pie.render_notebook()
#房型描述词云图
word_list = []
stop_words = ['花园','业主','出售']
string = str(''.join([i for i in data['title'] if isinstance(i, str)]))
words = psg.cut(string)
for x in words:
if len(x.word)==1:
pass
elif x.flag in ('m', 'x'):
pass
elif x.word in stop_words:
pass
else:
word_list.append(x.word)
data_pair = collections.Counter(word_list).most_common(100)
wc = (WordCloud()
.add("", data_pair, word_size_range=[20, 100], shape='triangle')
.set_global_opts(title_opts=opts.TitleOpts(title="房源描述词云图"))
)
wc.render_notebook()
[复制链接]
发表于 2020-12-4 10:16
|
发表于 2020-12-4 16:43
又一个学习案例