1、酒店评论数据

library(pacman)
p_load(dplyr)
df <- read.csv("./data_set/review.csv",header = T,stringsAsFactors = F) %>% 
  tbl_df() %>% 
  select(id=ReviewID,date=RatingDate,review=ReviewText,rate=Obs_Avg_Rating) %>% 
  mutate(sentiment=ifelse(rate<3,0,1)) 

df <- df %>% select(-rate)
df$date <- as.Date(df$date)
nrow(df)

## [1] 11005

# 切割为5个文件
N_files <- 5
# 每个文件多少行数据
chunk_len <- nrow(df) / N_files

# 创建临时文件
files <- sapply(1:N_files,function(x) tempfile())

# 将文件分块
chunks <- split(df,rep(1:N_files,each=chunk_len))

# 将分块写入到对应的文件
for (i in 1:N_files) {
  write.table(chunks[[i]],files[[i]],quote=T,row.names=F,
              col.names = T,sep = "|")
}

str(df,strict.width="cut")

## Classes 'tbl_df', 'tbl' and 'data.frame':    11005 obs. of  4 variables:
##  $ id       : int  116455519 116885145 117395588 126717496 132233722..
##  $ date     : Date, format: "2011-08-08" ...
##  $ review   : chr  "I gave this place 2 because the staff was very "..
##  $ sentiment: num  0 0 0 0 0 1 0 1 1 1 ...

2、构造text2vec读取文件函数

p_load(data.table,text2vec)

reader <- function(x, ...) {
  # 读取数据
  chunk <- data.table::fread(x, header = T, sep = "|")
  # 选择列
  res <- chunk$review
  # 加入ids信息
  names(res) <- chunk$id
  res
}

# 创建迭代器
it_files <- ifiles(files, reader = reader)
# 
it_tokens = itoken(it_files,
                   preprocessor = tolower,
                   tokenizer = word_tokenizer,
                   progressbar = FALSE)

vocab <- create_vocabulary(it_tokens)

3、使用构造的函数创建DTM

请注意，DTM具有文档id。它们继承自我们在reader函数中分配的文档名称。 在处理文件时，这是分配文档id的方便方法。

dtm <- create_dtm(it_tokens,vectorizer = vocab_vectorizer(vocab))
str(dtm, list.len = 5)

## Formal class 'dgCMatrix' [package "Matrix"] with 6 slots
##   ..@ i       : int [1:734787] 9012 558 10744 2736 1950 10921 9345 983 1874 758 ...
##   ..@ p       : int [1:26935] 0 1 2 3 4 5 6 7 8 9 ...
##   ..@ Dim     : int [1:2] 11005 26934
##   ..@ Dimnames:List of 2
##   .. ..$ : chr [1:11005] "116455519" "116885145" "117395588" "126717496" ...
##   .. ..$ : chr [1:26934] "contar" "reacting" "veriy" "cranny's" ...
##   ..@ x       : num [1:734787] 1 1 1 1 1 1 1 1 1 1 ...
##   .. [list output truncated]

4、使用多核读取内存中的数据

library(doParallel)

# 查看系统核心数
N_WORKERS <- detectCores()

# 创建集群
cl <- makeCluster(N_WORKERS-1)

# 注册parallel后端
registerDoParallel(cl)

it_token_par <- itoken_parallel(df$review,
                                preprocessor = tolower,
                                tokenizer = word_tokenizer,
                                ids = df$id,
                                # 可以自己控制n_chunks参数
                                n_chunks = 8)
vocab <- create_vocabulary(it_token_par)
v_vectorizer <- vocab_vectorizer(vocab)
dtm <- create_dtm(it_token_par,v_vectorizer)

# 停止集群
stopCluster(cl)

str(dtm)

## Formal class 'dgCMatrix' [package "Matrix"] with 6 slots
##   ..@ i       : int [1:734787] 558 1146 1481 2583 2588 2651 2741 2796 2991 3457 ...
##   ..@ p       : int [1:26935] 0 1 53 55 56 58 59 60 68 69 ...
##   ..@ Dim     : int [1:2] 11005 26934
##   ..@ Dimnames:List of 2
##   .. ..$ : chr [1:11005] "116455519" "116885145" "117395588" "126717496" ...
##   .. ..$ : chr [1:26934] "reacting" "downside" "impacted" "janitorial" ...
##   ..@ x       : num [1:734787] 1 1 1 1 1 1 1 1 1 1 ...
##   ..@ factors : list()

5、读取磁盘中的数据

library(doParallel)

# 查看系统核心数
N_WORKERS <- detectCores()

# 创建集群
cl <- makeCluster(N_WORKERS-1)

# 注册parallel后端
registerDoParallel(cl)

it_files_par <- ifiles_parallel(file_paths = files)

it_token_par <- itoken_parallel(it_files_par,
                                preprocessor = tolower,
                                tokenizer = word_tokenizer)
vocab <- create_vocabulary(it_token_par)

# 词向量DTM
v_vectorizer <- vocab_vectorizer(vocab)
dtm_v <- create_dtm(it_token_par,v_vectorizer)
str(dtm_v)

## Formal class 'dgCMatrix' [package "Matrix"] with 6 slots
##   ..@ i       : int [1:788756] 559 3900 5157 2473 2738 935 1951 984 2086 6899 ...
##   ..@ p       : int [1:37943] 0 1 3 4 5 6 7 8 10 11 ...
##   ..@ Dim     : int [1:2] 11010 37942
##   ..@ Dimnames:List of 2
##   .. ..$ : chr [1:11010] "filec082b5c5c_1" "filec082b5c5c_2" "filec082b5c5c_3" "filec082b5c5c_4" ...
##   .. ..$ : chr [1:37942] "reacting" "infact" "193827036" "cranny's" ...
##   ..@ x       : num [1:788756] 1 1 1 1 1 1 1 1 1 1 ...
##   ..@ factors : list()

# hash向量DTM
h_vectorizer <- hash_vectorizer()
dtm_h <- create_dtm(it_token_par,h_vectorizer)
str(dtm_h)

## Formal class 'dgCMatrix' [package "Matrix"] with 6 slots
##   ..@ i       : int [1:788604] 6402 1093 8357 559 630 815 894 957 1347 2872 ...
##   ..@ p       : int [1:262145] 0 0 0 0 0 0 0 0 0 0 ...
##   ..@ Dim     : int [1:2] 11010 262144
##   ..@ Dimnames:List of 2
##   .. ..$ : chr [1:11010] "filec089266d1a_1" "filec089266d1a_2" "filec089266d1a_3" "filec089266d1a_4" ...
##   .. ..$ : NULL
##   ..@ x       : num [1:788604] 1 1 1 1 1 1 1 1 1 1 ...
##   ..@ factors : list()

# 词共现矩阵
tcm <- create_tcm(it_token_par,
                  vectorizer = v_vectorizer,
                  skip_grams_window = 5)
str(tcm)

## Formal class 'dgTMatrix' [package "Matrix"] with 6 slots
##   ..@ i       : int [1:1107832] 216 285 193 292 494 422 235 180 416 361 ...
##   ..@ j       : int [1:1107832] 216 349 406 416 494 526 538 585 603 614 ...
##   ..@ Dim     : int [1:2] 37942 37942
##   ..@ Dimnames:List of 2
##   .. ..$ : chr [1:37942] "reacting" "infact" "193827036" "cranny's" ...
##   .. ..$ : chr [1:37942] "reacting" "infact" "193827036" "cranny's" ...
##   ..@ x       : num [1:1107832] 0.2 1 0.25 0.25 0.5 ...
##   ..@ factors : list()

# 停止集群
stopCluster(cl)