########FINAL STUFF############

library("tm")
library("SnowballC")
library("ggplot2")
library("wordcloud")




###SENTENCE###

setwd("C:/Users/AI/Downloads")
ctclnb <- read.csv("ctsampleinds_cleaned_nobool.csv", header = F)


setwd("C:/Users/AI/Downloads/ClusterSentFinal1")

n <- 1
for (i in 1:nrow(ctclnb)) {
  for (j in 1:ncol(ctclnb)) {
    if (!is.na(ctclnb[i,j])) {
      if (ctclnb[i,j] != "") {
        n <- as.character(n)
        str <- paste("sent", n, ".txt", sep = "")
        write.table(as.data.frame(ctclnb[i,j]),file = str, row.names = F, col.names = F)
        n <- as.numeric(n)
        n <- n + 1
      }
    }
  }
}


sentdocs <- Corpus(DirSource("C:/Users/AI/Downloads/ClusterSentFinal1"))
writeLines(as.character(sentdocs[[25]]))

sentdocs <- tm_map(sentdocs,content_transformer(tolower))
toSpace <- content_transformer(function(x, pattern) { return (gsub(pattern, " ", x))})
sentdocs <- tm_map(sentdocs, toSpace, '"')
sentdocs <- tm_map(sentdocs, removeWords, stopwords("english"))
sentdocs <- tm_map(sentdocs, stripWhitespace)
sentdocs <- tm_map(sentdocs,stemDocument)
sentdtm <- DocumentTermMatrix(sentdocs)
sentm <- as.matrix(sentdtm)
rownames(sentm) <- paste(substring(rownames(sentm),1,3),rep("..",nrow(sentm)), substring(rownames(sentm), nchar(rownames(sentm))-12,nchar(rownames(sentm))-4))
sentd <- dist(sentm)
setwd("C:/Users/AI/Downloads")
write.csv(as.matrix(sentd), file ="distmatclustersentfinal1.csv")
sentgroups <- hclust(sentd,method="ward.D")
sentgroupsplot <- as.dendrogram(sentgroups)

plot(sentgroups, hang=-1)
par(mfrow=c(3,1))

plot(sentgroupsplot, main="Main")
plot(cut(sentgroupsplot, h=4000)$upper, 
     main="Upper tree of cut at h=1000")
plot(cut(sentgroupsplot, h=4000)$lower[[2]], 
     main="Second branch of lower tree with cut at h=3000")
library("ggplot2")
library("ggdendro")

ggdendrogram(sentgroups)
sentdata <- dendro_data(sentgroupsplot, type = "rectangle")
head(sentdata$labels)

sentdatalabel<- sentdata[["labels"]][["label"]]

sentdatalabeldf <- as.data.frame(sentdatalabel)
sentdatalabeldf <- as.data.frame(cbind(sentdatalabeldf, rep("n/a", times = nrow(sentdatalabeldf))))
gsubsen <- function(x) {gsub("sen .. ", "", x)}
sentdatalabeldf$sentdatalabel <- lapply(sentdatalabeldf$sentdatalabel, gsubsen)
setwd("C:/Users/AI/Downloads/ClusterSentFinal1")
colnames(sentdatalabeldf)[c(1,2)] <- c("file", "text")
sentdatalabeldf[] <- lapply(sentdatalabeldf, as.character)
test1 <- as.character(read.table(paste(sentdatalabeldf[1,1],".txt",sep=""), header = F)[1,1])
for (i in 1:nrow(sentdatalabeldf)) {
  sentdatalabeldf[i,2] <- as.character(read.table(paste(sentdatalabeldf[i,1],".txt",sep=""), header = F)[1,1])
  
} 

setwd("C:/Users/AI/Downloads")

write.csv(sentdatalabeldf, file = "sentclusterresultsclean.csv")

pdf("pdfgraphsentfinal2.pdf", width=40, height=15)
plot(sentgroups, hang=-1)
dev.off()