# This R environment comes with all of CRAN preinstalled, as well as many other helpful packages
# The environment is defined by the kaggle/rstats docker image: https://github.com/kaggle/docker-rstats
# For example, here's several helpful packages to load in 

library(ggplot2) # Data visualization
library(readr) # CSV file I/O, e.g. the read_csv function

# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

library(tidyverse)
library(ggplot2)
library(pryr) # pryr::object_size(data1,data2,data3...) - provides combined memory of all objects in the list
library(data.table)
library(tidytext)
library(stringr)
library(rnn)
library(caret)
library(text2vec)

#library(mxnet)

#options(na.action = na.warn)

set.seed(100)

######################################
##
##
## Custom Functions - Start
##
##
######################################


######################################
##
##
## Custom Functions - Start
##
##
######################################

# removes all html tags
delete_htmltags <- function(htmlString) {
    return(gsub("<.*?>", "", htmlString))
}

remove_linebreaks <- function(htmlString) {
	return(gsub("\r?\n|\r", " ", htmlString))
}

clean_string <- function(string) {
	tempString <- tolower(string)
	tempString <- str_replace_all(tempString, "[^[:alnum:]]", " ") # remove all special characters
	tempString <- gsub('[[:digit:]]+', '', tempString)
	tempString <- gsub("([><-])|[[:punct:]]", "\\1", tempString)
	# remove numbers
	# remove newline characters

}


# Reference
# http://stackoverflow.com/questions/15253798/r-remove-stopwords-from-a-character-vector-using-in

rm_words <- function(string, words) {
  stopifnot(is.character(string), is.character(words))
  spltted <- strsplit(string, " ", fixed = TRUE) # fixed = TRUE for speedup
  vapply(spltted, function(x) paste(x[!tolower(x) %in% words], collapse = " "), character(1))
}

######################################
##
##
## Custom Functions - End
##
##
######################################

# Get and set working directory
#cur_dir <- getwd()
#setwd("Y:/MachineLearning/Quora")

start.time <- proc.time()

# Read input data
data <- read_csv("../input/train.csv", col_names=TRUE)

# Split data into test and cross validation
trainIndex <- createDataPartition(y= data$is_duplicate, p=0.7, list= FALSE)

train <- data[trainIndex,]
cv <- data[-trainIndex,]

#dim(train)
#dim(cv)
# Column Names for reference
# "id"  "qid1"  "qid2"  "question1" "question2" "is_duplicate"

# Removing all ids
#train$id = NULL
train$qid1 = NULL
train$qid2 = NULL

train$question1 <- delete_htmltags(train$question1)
train$question2 <- delete_htmltags(train$question2)

train$question1 <- remove_linebreaks(train$question1)
train$question2 <- remove_linebreaks(train$question2)

train$question1 <- clean_string(train$question1)
train$question2 <- clean_string(train$question2)

#head(train$question1)
#head(train$question2)

# Remove stop words from question 1 and 2
custom_stop_words <- stop_words

train$question1 <- rm_words(train$question1, custom_stop_words$word)
train$question2 <- rm_words(train$question2, custom_stop_words$word)
#head(train)

#pryr::object_size(train)

q1_cols <- c("id","question1")
q2_cols <- c("id","question2")

q1_train <- train[,q1_cols]
q2_train <- train[,q2_cols]

doc_set_1 <- itoken(q1_train$question1, progressbar = FALSE)
doc_set_2 <- itoken(q2_train$question2, progressbar = FALSE)

train$question <- paste0(train$question1, train$question2)

train_set <- itoken(train$question, progressbar = FALSE)

v = create_vocabulary(doc_set_2) %>% prune_vocabulary(doc_proportion_max = 0.1, term_count_min = 5)
vectorizer = vocab_vectorizer(v)

dtm1 <- create_dtm(doc_set_1, vectorizer)
dtm2 <- create_dtm(doc_set_2, vectorizer)

d1_d2_cos_sim <- psim2(dtm1, dtm2, method = "cosine", norm = "l2")
head(d1_d2_cos_sim)

head(train$is_duplicate)