{"cells":[{"metadata":{"trusted":true,"_uuid":"37784d8c56c0a742cc824534b85c393c2947f53d"},"cell_type":"code","source":"library(tm)\nlibrary(ctv)\nlibrary(NLP)\nlibrary(wordcloud)\nlibrary(ggplot2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8871a7fd327eab6a6816df945efdde055a251f37","scrolled":true},"cell_type":"code","source":"#library(tm)\nlibrary(RTextTools)\nlibrary(e1071)\nlibrary(dplyr)\nlibrary(caret)\n# Library for parallel processing\nlibrary(doParallel)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3478103e931822873a5413f67e0310281c510b7e","_execution_state":"idle","trusted":true},"cell_type":"code","source":"Data1 = read.csv(\"../input/train.csv\")\nData1 = Data1[-1]\nhead(Data1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e24d033d11db8a6bfaf615150ce41cf9a285c68"},"cell_type":"code","source":"glimpse(Data1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7c5eebf53704dccb66889fd9574de41d2b2500e5"},"cell_type":"code","source":"# randomize so that pos and neg are mixed up\nset.seed(1)\nData1 <- Data1[sample(nrow(Data1)), ]\nData1 <- Data1[sample(nrow(Data1)), ]\ndplyr::glimpse(Data1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b214bf39f49e4c58b430ea51a85f17fe0fc583f6"},"cell_type":"code","source":"# Creating Corpus\n#------------------------------------------------------------------------\ncorpus <- tm::Corpus(tm::VectorSource(Data1$question_text))\n# Inspect the corpus\n#corpus\ninspect(corpus[1:3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d1e80752e56f9d6805dd62ad928209308bf59f8"},"cell_type":"code","source":"#Transformation:- Means we have to repalce some charecter with space Reguler Expression.\n# This function \ntoSpace <- content_transformer(function (x , pattern ) gsub(pattern, \"\", x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e6d98bf9fbe07391ad6fb92f38e136c03c7a7585"},"cell_type":"code","source":"# or we can remove Panctuation:-\nremove_punctuation  <- function(x, pattern){\n    return(gsub(pattern,\"\", x))\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8a602aa35e230a859db370e162827fea15005d4"},"cell_type":"code","source":"# remove numbers\n# remove [](){}\n# any other symbol like #,$,%,&\n# Remove punctuation and space\ncorpus = tm_map(corpus, toSpace,\"/\")\ncorpus = tm_map(corpus, toSpace,\"@\")\ncorpus = tm_map(corpus, toSpace,\"\\\\|\")\ncorpus = tm_map(corpus, toSpace,\"\\\\.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"104ddf6bd556ed753d2b2ed40771aeacb5ba893e"},"cell_type":"code","source":"inspect(corpus[1:3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3ecefa072e9126bcdea1f5837bc8080af4ecfde7"},"cell_type":"code","source":"#2. Conversion to lower case\ncorpus <- tm_map(corpus, content_transformer(tolower))\ninspect(corpus[1:3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47565c6865a5170390b2dccbc492b7cb63d46e78"},"cell_type":"code","source":"#3. Remove numbers\n# removeNumbers() from TM\ncorpus <- tm_map(corpus, removeNumbers)\ninspect(corpus[1:3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf90cb3e2cbedab6c961f6c417905144cd016ad9"},"cell_type":"code","source":"#4. Remove punctuation\ncorpus <- tm_map(corpus, removePunctuation)\ninspect(corpus[1:3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a71774b70bde6c150951030a9714a1ee01747c8d"},"cell_type":"code","source":"#5.Remove english stop words\n# removeWords(doc, c(\"A\",\"by\",\"to\",\"the\"))\n#SW <- tm::stopwords()\n#corpus <- tm_map(corpus, removeWords, SW)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"92ad3f0ce144c4a9f757f8b262b9ca88148e0bd9"},"cell_type":"code","source":"#7.Strip whitespace\ncorpus <- tm_map(corpus, stripWhitespace)\ninspect(corpus[1:5])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"382018903a54afd75e36dc4c7f0083f112af8304"},"cell_type":"code","source":"#5.Remove english stop words\n# removeWords(doc, c(\"A\",\"by\",\"to\",\"the\"))\nSW <- tm::stopwords()\ncorpus <- tm_map(corpus, removeWords, SW)\ninspect(corpus[1:5])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47ba2552860395ea190c28d9dbfbfbacd2bb2643"},"cell_type":"code","source":"dtm <- tm::DocumentTermMatrix(corpus[1:100])\nx <- as.matrix(dtm)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b28774b2d1b5498c50037789c20c939d23cdcfb"},"cell_type":"code","source":"#Step 7 : Explore the corpus\n# Get terms with > 7 frequency    \nfindFreqTerms(dtm, lowfreq= 7 , highfreq = Inf)\nd<-as.matrix(dtm)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d56cf2edaad1ab10df5521dd817cb3d075547235"},"cell_type":"code","source":"# Find terms that are related to a certain term\n#findAssocs(dtm, \"arsenal\", corlimit=0.5)\n\nfreq <- sort(colSums(as.matrix(dtm)), decreasing=TRUE)\nd <- data.frame(word=names(freq), freq=freq)  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b978eb95db8e18b056976a7ef742d6177d430ac"},"cell_type":"code","source":"#require(ggplot2)\np <- ggplot(subset(d, freq>5), aes(word, freq))\np <- p+ geom_bar(stat=\"identity\")\np <- p+ theme(axis.text.x=element_text(angle=45, hjust=1))\np     \nbarplot(freq)        \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c52cb7dc0022810b7c5f7be1a0ce663628256af"},"cell_type":"code","source":"#Step 8 : Remove Sparse Terms: Sparse terms - Terms occurring only in very few documents\ninspect(removeSparseTerms(dtm, 0.2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"921d930bd6c4ba15829e871545376e33d976a838"},"cell_type":"code","source":"#Step 9 : Generate a word cloud\n#library(wordcloud)\n#wordcloud::wordcloud(d$word,d$freq,min.freq=3)\nwordcloud(d$word,d$freq,min.freq=3, colors=brewer.pal(6,\"Dark2\"),random.order=FALSE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"74d1f37f9f06d01844e3c12e3714b56b53b06cca"},"cell_type":"code","source":"#------------------------------------------------------------------------\n# Load required libraries\n#------------------------------------------------------------------------\nlibrary(tm)\nlibrary(RTextTools)\nlibrary(e1071)\nlibrary(dplyr)\nlibrary(caret)\n# Library for parallel processing\nlibrary(doParallel)\ndoParallel::registerDoParallel(cores=detectCores())  # Use all available cores","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3fbd24be6de0a8c8fdcf808502a065622fd6923","scrolled":true},"cell_type":"code","source":"Data1 = read.csv(\"../input/train.csv\")\nData1 = Data1[-1]\nhead(Data1)\nglimpse(Data1)\n# randomize so that pos and neg are mixed up\nset.seed(1)\nData1 <- Data1[sample(nrow(Data1)), ]\nData1 <- Data1[sample(nrow(Data1)), ]\ndplyr::glimpse(Data1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd7f57e43608598de518cc14f863f2a50aee33ab"},"cell_type":"code","source":"nrow(Data1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a8b1cf8b36cb654f72815c6c5a3dad204a969807"},"cell_type":"code","source":"# class to be converted to factor\nData1$target <- as.factor(Data1$target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e0c336179f2fbb1c808acc2aa98ae82c51bdc95e"},"cell_type":"code","source":"#------------------------------------------------------------------------\n# Creating Corpus\n#------------------------------------------------------------------------\ncorpus <- tm::Corpus(tm::VectorSource(Data1$question_text))\n# Inspect the corpus\n#corpus\ninspect(corpus[1:3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"130ca73d4df776e26dc507ff0991acfe8b43a983"},"cell_type":"code","source":"#------------------------------------------------------------------------\n# Cleaning\n#------------------------------------------------------------------------\ncorpus.clean <- corpus %>%\n  tm_map(content_transformer(tolower)) %>% \n  tm_map(removePunctuation) %>%\n  tm_map(removeNumbers) %>%\n  tm_map(removeWords, stopwords(kind=\"en\")) %>%\n  tm_map(stripWhitespace)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"efc1133b2c385102128abfbd54f36b0169bde896"},"cell_type":"code","source":"#------------------------------------------------------------------------\n# BoW - Doc Term Matrix\n#------------------------------------------------------------------------\ndtm <- DocumentTermMatrix(corpus.clean)\n# Inspect the dtm\ninspect(dtm[1:10, ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea106a431ece22881c4e425f38e6aaf9a0eb1f4a"},"cell_type":"code","source":"#inspect(corpus.clean[1:1306122])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da544fa9bb5d98188cc5cf9ec2eded931d576257"},"cell_type":"code","source":"#------------------------------------------------------------------------\n# Test and Train\n#------------------------------------------------------------------------\n#df.train <- Data1[1:53061,]\n#df.test <- Data1[53062:73061,]\n\n#dtm.train <- dtm[1:53061,]\n#dtm.test <- dtm[53062:73061,]\n\n#corpus.clean.train <- corpus.clean[1:53061]\n#corpus.clean.test <- corpus.clean[53062:73061]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f162ab6e06b774fd5d7a0ebfd18e8d07b22142c4"},"cell_type":"code","source":"#fivefreq <- findFreqTerms(dtm.train, 10)\n#length((fivefreq))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2a59a8e67c7d60e69c9926db8132ec86705af799"},"cell_type":"code","source":"# Use only 5 most frequent words (fivefreq) to build the DTM\n\n#dtm.train.nb <- DocumentTermMatrix(corpus.clean.train, control=list(dictionary = fivefreq))\n\n#dim(dtm.train.nb)\n\n#dtm.test.nb <- DocumentTermMatrix(corpus.clean.test, control=list(dictionary = fivefreq))\n\n#dim(dtm.test.nb)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b5f0feeaaccec34123dc831dbe4619f4f93161ca"},"cell_type":"code","source":"#convert_count <- function(x) {\n # y <- ifelse(x > 0,1,0)\n  #y <- factor(y, levels=c(0,1), labels=c(\"No\", \"Yes\"))\n  #y\n#}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"20281b332c82834e5a22c51ec7bf0f8ad63c8ed6"},"cell_type":"code","source":"#trainNB <- apply(dtm.train.nb,2,convert_count)\n#testNB <- apply(dtm.test.nb, 2, convert_count)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6edcd9a19bcc1691f25d8a35efe3ce847419e3f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"R","language":"R","name":"ir"},"language_info":{"mimetype":"text/x-r-source","name":"R","pygments_lexer":"r","version":"3.4.2","file_extension":".r","codemirror_mode":"r"}},"nbformat":4,"nbformat_minor":1}