library(tidyverse)
library(tidytext)
library(stopwords)
library(magrittr)
library(caret)
library(e1071)
library(MLmetrics)
library(tictoc)

library(doParallel)
library(foreach)

# load --------------------------------------------------------------------

tic("load data")
tr <- read_csv("../input/train.csv")
test <- read_csv("../input/test.csv")
subm <- read_csv("../input/sample_submission.csv")
toc()

# preproc -----------------------------------------------------------------

tic("bind...")
tr_te <- tr %>%
  bind_rows(test) %>%
  mutate(group = ifelse(is.na(target), "Test", "Train") %>% factor)
toc()

misspells <-
  tribble(
    ~bad, ~good,
    "aren't", "are not",
    "can't" , "cannot",
    "couldn't" , "could not",
    "didn't" , "did not",
    "doesn't" , "does not",
    "don't" , "do not",
    "hadn't" , "had not",
    "hasn't" , "has not",
    "haven't" , "have not",
    "he'd" , "he would",
    "he'll" , "he will",
    "he's" , "he is",
    "i'd" , "i would",
    "i'll" , "i will",
    "i'm" , "i am",
    "isn't" , "is not",
    "it's" , "it is",
    "it'll","it will",
    "i've" , "i have",
    "let's" , "let us",
    "mightn't" , "might not",
    "mustn't" , "must not",
    "shan't" , "shall not",
    "she'd" , "she would",
    "she'll" , "she will",
    "she's" , "she is",
    "shouldn't" , "should not",
    "that's" , "that is",
    "there's" , "there is",
    "they'd" , "they would",
    "they'll" , "they will",
    "they're" , "they are",
    "they've" , "they have",
    "we'd" , "we would",
    "we're" , "we are",
    "weren't" , "were not",
    "we've" , "we have",
    "what'll" , "what will",
    "what're" , "what are",
    "what's" , "what is",
    "what've" , "what have",
    "where's" , "where is",
    "who'd" , "who would",
    "who'll" , "who will",
    "who're" , "who are",
    "who's" , "who is",
    "who've" , "who have",
    "won't" , "will not",
    "wouldn't" , "would not",
    "you'd" , "you would",
    "you'll" , "you will",
    "you're" , "you are",
    "you've" , "you have",
    "'re", " are",
    "wasn't", "was not",
    "we'll"," will",
    "didn't", "did not",
    "tryin'","trying",
    "gey","gay",
    "fck","fuck",
    "gayyy","gay",
    "mem","meme",
    "mom","mother",
    "dont","do not",
    "wont","would not",
    "cant","could not",
    "colour", "color",
    "centre", "center",
    "favourite", "favorite",
    "travelling", "traveling",
    "counselling", "counseling",
    "theatre", "theater",
    "cancelled", "canceled",
    "labour", "labor",
    "organisation", "organization",
    "wwii", "world war 2",
    "citicise", "criticize",
    "youtu ", "youtube ",
    "qoura", "quora",
    "sallary", "salary",
    "whta", "what",
    "narcisist", "narcissist",
    "howdo", "how do",
    "whatare", "what are",
    "howcan", "how can",
    "howmuch", "how much",
    "howmany", "how many",
    "whydo", "why do",
    "doI", "do i",
    "theBest", "the best",
    "howdoes", "how does",
    "mastrubation", "masturbation",
    "mastrubate", "masturbate",
    "mastrubating", "masturbating",
    "pennis", "penis",
    "Etherium", "Ethereum",
    "narcissit", "narcissist",
    "bigdata", "big data",
    "qouta", "quota",
    "ex boyfriend", "exboyfriend",
    "air hostess", "airhostess",
    "whst", "what",
    "watsapp", "whatsapp",
    "demonitisation", "demonetization",
    "demonitization", "demonetization",
    "demonetisation", "demonetization",
    "[^[:alpha:][:space:]]+", ""
  )

tic("Preprocessing tokens...")
for (i in seq_along(1:nrow(misspells))) {
  bad <- misspells[i,][[1]]
  good <- misspells[i,][[2]]
  tr_te %<>%
    mutate(question_text = str_replace_all(question_text, bad, good))
}
tokens <- tr_te %>% unnest_tokens(word, question_text)
toc()

tic("Anti join...")
tokens %<>%
  anti_join(stop_words, by = "word") %>%
  filter(nchar(word) > 1)
toc()

# train/test/valid --------------------------------------------------------

tic("train/test/valid")
set.seed(0)
inTrain <- createDataPartition(y = tr$target, p = .85, list = F) %>% c()
train <- tr[ inTrain, ]
valid <- tr[-inTrain, ]
toc()

tokens.train <-
  tokens %>%
  left_join(train, by = "qid") %>%
  rename(target = target.x) %>%
  select(-target.y, -group) %>%
  na.omit() %>%
  select(-question_text)

rm(tr, tr_te); invisible(gc())

# naive bayes -------------------------------------------------------------

registerDoParallel(cores = 4)  # because it's a kernel competition

m <- naiveBayes(as.factor(target) ~ word,
                tokens.train %>% select(target, word),
                laplace = 1)

# score -------------------------------------------------------------------

tokens.valid <-
  tokens %>%
  left_join(valid, by = "qid") %>%
  rename(target = target.x) %>%
  select(-target.y, -group) %>%
  na.omit() %>%
  select(-question_text)

score <- function(threshold = .1, lap = .25) {
  m <- naiveBayes(as.factor(target) ~ word,
                  tokens.train %>% select(target, word),
                  laplace = lap)

  p <-
    tibble(qid = tokens.valid$qid,
           pred =
             predict(m,
                     tokens.valid %>% select(target, word),
                     type = "raw")[, "1"]
           )

  p %<>%
    group_by(qid) %>%
    summarise(pred = ifelse(mean(pred) > threshold, 1, 0))

  truth <-
    tokens.valid %>%
    group_by(qid) %>%
    summarise(truth = ifelse(sum(target) > 0, 1, 0))

  F1_Score(truth$truth, p$pred, positive = "1")
}

search_threshold <-
  foreach(threshold = seq(.1, .9, .05)) %dopar%
    score(threshold = threshold)
#search_threshold

# submit ------------------------------------------------------------------

tokens.subm <-
  tokens %>%
  left_join(test, by = "qid") %>%
  filter(group == "Test")

p <- predict(m, tokens.subm %>% select(target, word), type = "raw")

tokens.subm %<>%
  mutate(pred = p[, "1"])

pred <-
  tokens.subm %>%
  group_by(qid) %>%
  summarise(target = ifelse(mean(pred) > .20, 1, 0))

subm %<>%
  left_join(pred, by = "qid") %<>%
  select(-prediction) %>%
  rename(prediction = target)
subm[is.na(subm)] <- 0

subm %>%
  write_csv("submission.csv")