使用爬虫获取南京地铁一号线周边的房源信息并作统计

.libPaths("d://R_package/R.4.2.1/")
# 加载必要的库
library(httr)
library(rvest)
library(stringr)
library(dplyr)
library(ggplot2)
library(ggprism)
library(purrr)
library(tidyr)
library(forcats)
library(plyr)

# 函数来获取页面内容
get_page_content <- function(url) {
  Sys.sleep(5)  # 添加延迟以避免被封IP
  response <- GET(url, user_agent("Mozilla/5.0"))
  content(response, "text")
}

# 函数来解析页面内容
parse_page <- function(html_content) {
  page <- read_html(html_content)
  
  title <- page %>% html_nodes('li.house-cell.realverify div.des h2 a') %>% html_text(trim = TRUE)
  url <- page %>% html_nodes('li.house-cell.realverify div.des h2 a') %>% html_attr("href")
  size <- page %>% html_nodes('li.house-cell.realverify div.des p.room') %>% html_text(trim = TRUE)
  info <- page %>% html_nodes('li.house-cell.realverify div.des p.infor') %>% html_text(trim = TRUE)
  price <- page %>% html_nodes('li.house-cell.realverify div.money') %>% html_text(trim = TRUE)
  author <- page %>% html_nodes('li.house-cell.realverify div.jjr') %>% html_text(trim = TRUE)
  
  data.frame(title = title, url = url, size = size, info = info, price = price, author = author)
}

# 获取所有页面的数据
zhengzu_info <- map_df(1:70, function(page_num) {
  url <- paste0("https://nj.58.com/zufang/sub/l47/pn", page_num, "/")
  html_content <- get_page_content(url)
  df <- parse_page(html_content)
  print(paste("完成第", page_num, "页信息提取，共获得", nrow(df), "条信息", sep = ""))
  df
})

# 保存原始数据
saveRDS(zhengzu_info, "zhengzu.rds")

zhengzu_info_raw<-readRDS("zhengzu.rds")

# 数据清洗
yihaoxianzhan <- c("迈皋桥站", "红山动物园站", "南京站站", "新模范马路站", 
                   "玄武门站", "鼓楼站", "珠江路站", "新街口站", 
                   "张府园站", "三山街站", "中华门站", "安德门站", 
                   "天隆寺站", "软件大道站", "花神庙站", "南京南站站", 
                   "双龙大道站", "河定桥站", "胜太路站", "百家湖站", 
                   "小龙湾站", "竹山路站", "天印大道站", "龙眠大道站", 
                   "南医大站", "南京交院站", "中国药科大学站")

zhengzu_info <- zhengzu_info_raw %>%
  mutate(
    title = str_remove_all(title, " "),
    num =str_split(size, " {1,}", simplify = TRUE)%>%.[,1],
    area = str_split(size, " {1,}", simplify = TRUE)%>%.[,2],
    info = str_remove_all(info, " {1,}"),
    dir1 = str_split(info, "\\n{1,}", simplify = TRUE)[,1],
    dir2 = str_split(info, "\\n{1,}", simplify = TRUE)[,3],
    dir_tmp = str_split(info, "\\n{1,}", simplify = TRUE)[,4],
    dir3 = str_remove(str_extract(dir_tmp, "距1号线-[\\p{Han}]{1,}"), "距1号线-"),
    dir4 = as.numeric(str_remove(str_extract(dir_tmp, "[0-9]{1,}m"), "m")),
    author = str_remove_all(author, " {1,}"),
    author1 = str_split(author, "\\n{1,}", simplify = TRUE)[,1],
    author2 = str_split(author, "\\n{1,}", simplify = TRUE)[,2],
    author3 = str_split(author, "\\n{1,}", simplify = TRUE)[,3]
  ) %>%
  select(title, num, area, dir1, dir2, dir3, dir4, price, author1, author2, author3, url) %>%
  mutate(
    area = as.numeric(str_extract(area, "[0-9\\.]{1,}")),
    price = as.numeric(str_remove(price, "元/月"))
  )

yihaoxianzhan <- intersect(yihaoxianzhan, unique(paste0(zhengzu_info$dir3,"站")))
zhengzu_info$dir3 <- factor(paste0(zhengzu_info$dir3,"站"), levels = yihaoxianzhan)

# 保存清洗后的数据
saveRDS(zhengzu_info, "zhengzu_info_cleandata.rds")  

# 去除重复项
zhengzu_info <- zhengzu_info[!duplicated(zhengzu_info$title),]

#统计
#1、计算均房价
zhengzu_summary<-zhengzu_info%>%
  mutate(ave_price=price/area)%>%
  filter(!is.na(dir3))%>%
  mutate(dir3=factor(dir3,levels = yihaoxianzhan))
zhengzu_num<-ddply(zhengzu_summary,.(dir3),nrow)
ggplot(zhengzu_summary%>%filter(!is.na(dir3)),aes(x=dir3,y=ave_price))+
  geom_boxplot()+
  geom_text(data = zhengzu_num,aes(x=dir3,y=120,label=paste0("n=",V1)))+
  #geom_jitter()+
  coord_flip()+
  scale_y_continuous(limits = c(15,150),breaks = seq(0,120,10))+
  labs(x="",y="租房均价/m^2/月")

#根据均房价高低排序
zhengzu_summary_mean=zhengzu_summary%>%
  ddply(.,.(dir3),function(x){mean(x$ave_price)})%>%
  arrange(V1)
zhengzu_summary%>%
  mutate(dir3=factor(dir3,levels = zhengzu_summary_mean$dir3))%>%
  ggplot(aes(x=dir3,y=ave_price))+
  geom_boxplot()+
  geom_text(data = zhengzu_num,aes(x=dir3,y=120,label=paste0("n=",V1)))+
  #geom_jitter()+
  coord_flip()+
  scale_y_continuous(limits = c(15,150),breaks = seq(0,120,10))+
  labs(x="",y="租房均价/m^2/月")

#距离地铁站的距离与均价的关系
zhengzu_summary%>%
  filter(!is.na(dir4))%>%
  mutate(dir5=ifelse(dir4<500,"500米以内",ifelse(dir4>=500&dir4<1000,"500-1000米",ifelse(dir4>=1000&dir4<1500,"1000-1500米","大于1500米"))))%>%
  mutate(dir5=factor(dir5,levels=c("500米以内","500-1000米","1000-1500米","大于1500米")))%>%
  ggplot(aes(x=dir5,y=ave_price,color=dir3))+
  geom_boxplot()+
  scale_y_continuous(limits = c(15,150))+
  theme(legend.position = "bottom")+
  labs(x="到1号线站点的距离",y="租房均价/m^2/月")

zhengzu_summary%>%
  ggplot(aes(x=area,y=price,color=dir3))+
  geom_point()+
  scale_x_continuous(limits = c(0,200))+
  scale_y_continuous(limits = c(0,10000))

zhengzu_summary_classmean<-zhengzu_summary%>%
  filter(!is.na(dir4))%>%
  filter(!is.na(dir3))%>%
  mutate(dir5=ifelse(dir4<500,"A",ifelse(dir4>=500&dir4<1000,"B",ifelse(dir4>=1000&dir4<1500,"C","D"))))%>%
  ddply(.,.(dir5,dir3),function(x){
    nrow(x)
  })%>%
  mutate(y=sapply(dir5,function(y){switch(y,"A"=25,"B"=20,"C"=15,"D"=10)}))

zhengzu_summary%>%
  filter(!is.na(dir4))%>%
  filter(!is.na(dir3))%>%
  mutate(dir5=ifelse(dir4<500,"A",ifelse(dir4>=500&dir4<1000,"B",ifelse(dir4>=1000&dir4<1500,"C","D"))))%>%
  ddply(.,.(dir3,dir5),function(x){mean(x$ave_price)})%>%
  ggplot(aes(x=dir3,y=V1,color=dir5))+
  geom_line(aes(color=dir5,group=dir5),size=2)+
  geom_point(aes(color=dir5),size=5)+
  geom_text(data = zhengzu_summary_classmean,aes(x=dir3,y=y,label=V1,color=dir5))+
  #geom_text_repel(aes(label=round(V1,2)),max.overlaps = 100)+
  scale_y_continuous(limits = c(5,90),breaks = seq(0,75,5))+
  scale_color_discrete(label=c("0-500m","500-1000m","1000-1500m","1500m以上"),name="到地铁站距离")+
  theme_prism()+
  theme(axis.text.x = element_text(angle = 45,hjust = 1,vjust = 1),
        legend.title = element_text())+
  #facet_wrap(~factor(dir5),scales = "free_y")+
  labs(y="租房均价/m^2",x="")