## I haven't been able to find a clear answer whether hard coded stuff is allowed/ not allowed.
## that is why i am publishing my data cleaning/ missspellings/ special characters.
## nothing special! 
## have fun ;)

library(tidyverse)
library(tidytext)
library(data.table)
library(Matrix)
library(qdapRegex)
library(keras)
library(reticulate)

tr <- fread('../input/train.csv', data.table = T, encoding = "UTF-8")
te <- fread('../input/test.csv', data.table = T, encoding = "UTF-8")
tri <- 1:nrow(tr)

tr[,question_text:= str_replace_all(question_text, "aren't", "are not") ] 
tr[,question_text:= str_replace_all(question_text, "can't" , "cannot") ] 
tr[,question_text:= str_replace_all(question_text, "couldn't" , "could not") ] 
tr[,question_text:= str_replace_all(question_text, "didn't" , "did not") ] 
tr[,question_text:= str_replace_all(question_text, "doesn't" , "does not") ] 
tr[,question_text:= str_replace_all(question_text, "don't" , "do not") ] 
tr[,question_text:= str_replace_all(question_text, "hadn't" , "had not") ] 
tr[,question_text:= str_replace_all(question_text, "hasn't" , "has not") ] 
tr[,question_text:= str_replace_all(question_text, "haven't" , "have not") ] 
tr[,question_text:= str_replace_all(question_text, "he'd" , "he would") ] 
tr[,question_text:= str_replace_all(question_text, "he'll" , "he will") ] 
tr[,question_text:= str_replace_all(question_text, "he's" , "he is") ] 
tr[,question_text:= str_replace_all(question_text, "i'd" , "I would") ] 
tr[,question_text:= str_replace_all(question_text, "i'll" , "I will") ] 
tr[,question_text:= str_replace_all(question_text, "i'm" , "I am") ] 
tr[,question_text:= str_replace_all(question_text, "isn't" , "is not") ] 
tr[,question_text:= str_replace_all(question_text, "it's" , "it is") ] 
tr[,question_text:= str_replace_all(question_text, "it'll","it will") ] 
tr[,question_text:= str_replace_all(question_text, "i've" , "I have") ] 
tr[,question_text:= str_replace_all(question_text, "let's" , "let us") ] 
tr[,question_text:= str_replace_all(question_text, "mightn't" , "might not") ] 
tr[,question_text:= str_replace_all(question_text, "mustn't" , "must not") ] 
tr[,question_text:= str_replace_all(question_text, "shan't" , "shall not") ] 
tr[,question_text:= str_replace_all(question_text, "she'd" , "she would") ] 
tr[,question_text:= str_replace_all(question_text, "she'll" , "she will") ] 
tr[,question_text:= str_replace_all(question_text, "she's" , "she is") ] 
tr[,question_text:= str_replace_all(question_text, "shouldn't" , "should not") ] 
tr[,question_text:= str_replace_all(question_text, "that's" , "that is") ] 
tr[,question_text:= str_replace_all(question_text, "there's" , "there is") ] 
tr[,question_text:= str_replace_all(question_text, "they'd" , "they would") ] 
tr[,question_text:= str_replace_all(question_text, "they'll" , "they will") ] 
tr[,question_text:= str_replace_all(question_text, "they're" , "they are") ] 
tr[,question_text:= str_replace_all(question_text, "they've" , "they have") ] 
tr[,question_text:= str_replace_all(question_text, "we'd" , "we would") ] 
tr[,question_text:= str_replace_all(question_text, "we're" , "we are") ] 
tr[,question_text:= str_replace_all(question_text, "weren't" , "were not") ] 
tr[,question_text:= str_replace_all(question_text, "we've" , "we have") ] 
tr[,question_text:= str_replace_all(question_text, "what'll" , "what will") ] 
tr[,question_text:= str_replace_all(question_text, "what're" , "what are") ] 
tr[,question_text:= str_replace_all(question_text, "what's" , "what is") ] 
tr[,question_text:= str_replace_all(question_text, "what've" , "what have") ] 
tr[,question_text:= str_replace_all(question_text, "where's" , "where is") ] 
tr[,question_text:= str_replace_all(question_text, "who'd" , "who would") ] 
tr[,question_text:= str_replace_all(question_text, "who'll" , "who will") ] 
tr[,question_text:= str_replace_all(question_text, "who're" , "who are") ] 
tr[,question_text:= str_replace_all(question_text, "who's" , "who is") ] 
tr[,question_text:= str_replace_all(question_text, "who've" , "who have") ] 
tr[,question_text:= str_replace_all(question_text, "won't" , "will not") ] 
tr[,question_text:= str_replace_all(question_text, "wouldn't" , "would not") ] 
tr[,question_text:= str_replace_all(question_text, "you'd" , "you would") ] 
tr[,question_text:= str_replace_all(question_text, "you'll" , "you will") ] 
tr[,question_text:= str_replace_all(question_text, "you're" , "you are") ] 
tr[,question_text:= str_replace_all(question_text, "you've" , "you have") ] 
tr[,question_text:= str_replace_all(question_text, "'re", " are") ] 
tr[,question_text:= str_replace_all(question_text, "wasn't", "was not") ] 
tr[,question_text:= str_replace_all(question_text, "we'll"," will") ] 
tr[,question_text:= str_replace_all(question_text, "didn't", "did not") ] 
tr[,question_text:= str_replace_all(question_text, "tryin'","trying")]
tr[,question_text:= str_replace_all(question_text, "gey","gay")]
tr[,question_text:= str_replace_all(question_text, "fck","fuck")]
tr[,question_text:= str_replace_all(question_text, "gayyy","gay")]
tr[,question_text:= str_replace_all(question_text, "mem","mother")]
tr[,question_text:= str_replace_all(question_text, "dont","do not")]
tr[,question_text:= str_replace_all(question_text, "wont","would not")]
tr[,question_text:= str_replace_all(question_text, "cant","could not")]

tr[,question_text:= str_replace_all(question_text,'colour', 'color')] 
tr[,question_text:= str_replace_all(question_text,'centre', 'center')] 
tr[,question_text:= str_replace_all(question_text,'favourite', 'favorite')] 
tr[,question_text:= str_replace_all(question_text,'travelling', 'traveling')] 
tr[,question_text:= str_replace_all(question_text,'counselling', 'counseling')] 
tr[,question_text:= str_replace_all(question_text,'theatre', 'theater')]
tr[,question_text:= str_replace_all(question_text,'cancelled', 'canceled')] 
tr[,question_text:= str_replace_all(question_text,'labour', 'labor')]
tr[,question_text:= str_replace_all(question_text,'organisation', 'organization')]
tr[,question_text:= str_replace_all(question_text,'wwii', 'world war 2')]
tr[,question_text:= str_replace_all(question_text,'citicise', 'criticize')] 
tr[,question_text:= str_replace_all(question_text,'youtu ', 'youtube ')]
tr[,question_text:= str_replace_all(question_text,'Qoura', 'Quora')]
tr[,question_text:= str_replace_all(question_text,'sallary', 'salary')] 
tr[,question_text:= str_replace_all(question_text,'Whta', 'What')]
tr[,question_text:= str_replace_all(question_text,'narcisist', 'narcissist')]
tr[,question_text:= str_replace_all(question_text,'howdo', 'how do')]
tr[,question_text:= str_replace_all(question_text,'whatare', 'what are')]
tr[,question_text:= str_replace_all(question_text,'howcan', 'how can')]
tr[,question_text:= str_replace_all(question_text,'howmuch', 'how much')] 
tr[,question_text:= str_replace_all(question_text,'howmany', 'how many')]
tr[,question_text:= str_replace_all(question_text,'whydo', 'why do')]
tr[,question_text:= str_replace_all(question_text,'doI', 'do I')]
tr[,question_text:= str_replace_all(question_text,'theBest', 'the best')]
tr[,question_text:= str_replace_all(question_text,'howdoes', 'how does')]
tr[,question_text:= str_replace_all(question_text,'mastrubation', 'masturbation')] 
tr[,question_text:= str_replace_all(question_text,'mastrubate', 'masturbate')]
tr[,question_text:= str_replace_all(question_text,"mastrubating", 'masturbating')] 
tr[,question_text:= str_replace_all(question_text,'pennis', 'penis')]
tr[,question_text:= str_replace_all(question_text,'Etherium', 'Ethereum')] 
tr[,question_text:= str_replace_all(question_text,'narcissit', 'narcissist')] 
tr[,question_text:= str_replace_all(question_text,'bigdata', 'big data')]
tr[,question_text:= str_replace_all(question_text,'2k17', '2017')]
tr[,question_text:= str_replace_all(question_text,'2k18', '2018')]
tr[,question_text:= str_replace_all(question_text,'qouta', 'quota')] 
tr[,question_text:= str_replace_all(question_text,'exboyfriend', 'ex boyfriend')] 
tr[,question_text:= str_replace_all(question_text,'airhostess', 'air hostess')]
tr[,question_text:= str_replace_all(question_text,"whst", 'what')] 
tr[,question_text:= str_replace_all(question_text,'watsapp', 'whatsapp')] 
tr[,question_text:= str_replace_all(question_text,'demonitisation', 'demonetization')]
tr[,question_text:= str_replace_all(question_text,'demonitization', 'demonetization')]
tr[,question_text:= str_replace_all(question_text,'demonetisation', 'demonetization')]

print("train done")

te[,question_text:= str_replace_all(question_text, "aren't", "are not") ] 
te[,question_text:= str_replace_all(question_text, "can't" , "cannot") ] 
te[,question_text:= str_replace_all(question_text, "couldn't" , "could not") ] 
te[,question_text:= str_replace_all(question_text, "didn't" , "did not") ] 
te[,question_text:= str_replace_all(question_text, "doesn't" , "does not") ] 
te[,question_text:= str_replace_all(question_text, "don't" , "do not") ] 
te[,question_text:= str_replace_all(question_text, "hadn't" , "had not") ] 
te[,question_text:= str_replace_all(question_text, "hasn't" , "has not") ] 
te[,question_text:= str_replace_all(question_text, "haven't" , "have not") ] 
te[,question_text:= str_replace_all(question_text, "he'd" , "he would") ] 
te[,question_text:= str_replace_all(question_text, "he'll" , "he will") ] 
te[,question_text:= str_replace_all(question_text, "he's" , "he is") ] 
te[,question_text:= str_replace_all(question_text, "i'd" , "I would") ] 
te[,question_text:= str_replace_all(question_text, "i'll" , "I will") ] 
te[,question_text:= str_replace_all(question_text, "i'm" , "I am") ] 
te[,question_text:= str_replace_all(question_text, "isn't" , "is not") ] 
te[,question_text:= str_replace_all(question_text, "it's" , "it is") ] 
te[,question_text:= str_replace_all(question_text, "it'll","it will") ] 
te[,question_text:= str_replace_all(question_text, "i've" , "I have") ] 
te[,question_text:= str_replace_all(question_text, "let's" , "let us") ] 
te[,question_text:= str_replace_all(question_text, "mightn't" , "might not") ] 
te[,question_text:= str_replace_all(question_text, "mustn't" , "must not") ] 
te[,question_text:= str_replace_all(question_text, "shan't" , "shall not") ] 
te[,question_text:= str_replace_all(question_text, "she'd" , "she would") ] 
te[,question_text:= str_replace_all(question_text, "she'll" , "she will") ] 
te[,question_text:= str_replace_all(question_text, "she's" , "she is") ] 
te[,question_text:= str_replace_all(question_text, "shouldn't" , "should not") ] 
te[,question_text:= str_replace_all(question_text, "that's" , "that is") ] 
te[,question_text:= str_replace_all(question_text, "there's" , "there is") ] 
te[,question_text:= str_replace_all(question_text, "they'd" , "they would") ] 
te[,question_text:= str_replace_all(question_text, "they'll" , "they will") ] 
te[,question_text:= str_replace_all(question_text, "they're" , "they are") ] 
te[,question_text:= str_replace_all(question_text, "they've" , "they have") ] 
te[,question_text:= str_replace_all(question_text, "we'd" , "we would") ] 
te[,question_text:= str_replace_all(question_text, "we're" , "we are") ] 
te[,question_text:= str_replace_all(question_text, "weren't" , "were not") ] 
te[,question_text:= str_replace_all(question_text, "we've" , "we have") ] 
te[,question_text:= str_replace_all(question_text, "what'll" , "what will") ] 
te[,question_text:= str_replace_all(question_text, "what're" , "what are") ] 
te[,question_text:= str_replace_all(question_text, "what's" , "what is") ] 
te[,question_text:= str_replace_all(question_text, "what've" , "what have") ] 
te[,question_text:= str_replace_all(question_text, "where's" , "where is") ] 
te[,question_text:= str_replace_all(question_text, "who'd" , "who would") ] 
te[,question_text:= str_replace_all(question_text, "who'll" , "who will") ] 
te[,question_text:= str_replace_all(question_text, "who're" , "who are") ] 
te[,question_text:= str_replace_all(question_text, "who's" , "who is") ] 
te[,question_text:= str_replace_all(question_text, "who've" , "who have") ] 
te[,question_text:= str_replace_all(question_text, "won't" , "will not") ] 
te[,question_text:= str_replace_all(question_text, "wouldn't" , "would not") ] 
te[,question_text:= str_replace_all(question_text, "you'd" , "you would") ] 
te[,question_text:= str_replace_all(question_text, "you'll" , "you will") ] 
te[,question_text:= str_replace_all(question_text, "you're" , "you are") ] 
te[,question_text:= str_replace_all(question_text, "you've" , "you have") ] 
te[,question_text:= str_replace_all(question_text, "'re", " are") ] 
te[,question_text:= str_replace_all(question_text, "wasn't", "was not") ] 
te[,question_text:= str_replace_all(question_text, "we'll"," will") ] 
te[,question_text:= str_replace_all(question_text, "didn't", "did not") ] 
te[,question_text:= str_replace_all(question_text, "tryin'","trying")]
te[,question_text:= str_replace_all(question_text, "gey","gay")]
te[,question_text:= str_replace_all(question_text, "fck","fuck")]
te[,question_text:= str_replace_all(question_text, "gayyy","gay")]
te[,question_text:= str_replace_all(question_text, "mem","mother")]
te[,question_text:= str_replace_all(question_text, "dont","do not")]
te[,question_text:= str_replace_all(question_text, "wont","would not")]
te[,question_text:= str_replace_all(question_text, "cant","could not")]

te[,question_text:= str_replace_all(question_text,'colour', 'color')] 
te[,question_text:= str_replace_all(question_text,'centre', 'center')] 
te[,question_text:= str_replace_all(question_text,'favourite', 'favorite')] 
te[,question_text:= str_replace_all(question_text,'travelling', 'traveling')] 
te[,question_text:= str_replace_all(question_text,'counselling', 'counseling')] 
te[,question_text:= str_replace_all(question_text,'theatre', 'theater')]
te[,question_text:= str_replace_all(question_text,'cancelled', 'canceled')] 
te[,question_text:= str_replace_all(question_text,'labour', 'labor')]
te[,question_text:= str_replace_all(question_text,'organisation', 'organization')]
te[,question_text:= str_replace_all(question_text,'wwii', 'world war 2')]
te[,question_text:= str_replace_all(question_text,'citicise', 'criticize')] 
te[,question_text:= str_replace_all(question_text,'youtu ', 'youtube ')]
te[,question_text:= str_replace_all(question_text,'Qoura', 'Quora')]
te[,question_text:= str_replace_all(question_text,'sallary', 'salary')] 
te[,question_text:= str_replace_all(question_text,'Whta', 'What')]
te[,question_text:= str_replace_all(question_text,'narcisist', 'narcissist')]
te[,question_text:= str_replace_all(question_text,'howdo', 'how do')]
te[,question_text:= str_replace_all(question_text,'whatare', 'what are')]
te[,question_text:= str_replace_all(question_text,'howcan', 'how can')]
te[,question_text:= str_replace_all(question_text,'howmuch', 'how much')] 
te[,question_text:= str_replace_all(question_text,'howmany', 'how many')]
te[,question_text:= str_replace_all(question_text,'whydo', 'why do')]
te[,question_text:= str_replace_all(question_text,'doI', 'do I')]
te[,question_text:= str_replace_all(question_text,'theBest', 'the best')]
te[,question_text:= str_replace_all(question_text,'howdoes', 'how does')]
te[,question_text:= str_replace_all(question_text,'mastrubation', 'masturbation')] 
te[,question_text:= str_replace_all(question_text,'mastrubate', 'masturbate')]
te[,question_text:= str_replace_all(question_text,"mastrubating", 'masturbating')] 
te[,question_text:= str_replace_all(question_text,'pennis', 'penis')]
te[,question_text:= str_replace_all(question_text,'Etherium', 'Ethereum')] 
te[,question_text:= str_replace_all(question_text,'narcissit', 'narcissist')] 
te[,question_text:= str_replace_all(question_text,'bigdata', 'big data')]
te[,question_text:= str_replace_all(question_text,'2k17', '2017')]
te[,question_text:= str_replace_all(question_text,'2k18', '2018')]
te[,question_text:= str_replace_all(question_text,'qouta', 'quota')] 
te[,question_text:= str_replace_all(question_text,'exboyfriend', 'ex boyfriend')] 
te[,question_text:= str_replace_all(question_text,'airhostess', 'air hostess')]
te[,question_text:= str_replace_all(question_text,"whst", 'what')] 
te[,question_text:= str_replace_all(question_text,'watsapp', 'whatsapp')] 
te[,question_text:= str_replace_all(question_text,'demonitisation', 'demonetization')]
te[,question_text:= str_replace_all(question_text,'demonitization', 'demonetization')]
te[,question_text:= str_replace_all(question_text,'demonetisation', 'demonetization')]

print("test done")

puncts <- c(
  '\u002C','\u002E','\u0022','\u003A','\u0029','\u0028','\u002D','\u0021','\u003B','\u0027','\u0024','\u0026','\u002F','\u005B','\u005D','\u003E','\u0025','\u003D','\u0023','\u002A','\u002B','\u005C',
  '\u2022','\u007E','\u0040','\u00A3','\u00B7','\u005F','\u007B','\u007D','\u00A9','\u005E','\u00AE','\u0060','\u003C','\u2192','\u00B0','\u20AC','\u2122','\u203A','\u2665','\u2190','\u00D7','\u00A7','\u2033',
  '\u2032','\u00C2','\u2588','\u00BD','\u00E0','\u2026','\u201C','\u2605','\u201D','\u2013','\u25CF','\u00E2','\u25BA','\u2212','\u00A2','\u00B2','\u00AC','\u2591','\u00B6','\u2191','\u00B1','\u00BF','\u25BE',
  '\u2550','\u00A6','\u2551','\u2015','\u00A5','\u2593','\u2014','\u2039','\u2500','\u2592','\uFF1A','\u00BC','\u2295','\u25BC','\u25AA','\u2020','\u25A0','\u2019','\u2580','\u00A8','\u2584','\u266B','\u2606',
  '\u00E9','\u00AF','\u2666','\u00A4','\u25B2','\u00E8','\u00B8','\u00BE','\u00C3','\u22C5','\u2018','\u221E','\u2219','\uFF09','\u2193','\u3001','\u2502','\u00BB','\uFF0C','\uFF08','\u266A','\u2569','\u00B3',
  '\u30FB','\u2566','\u2563','\u2554','\u2557','\u25AC','\u2764','\u00EF','\u00D8','\u00B9','\u2264','\u221A','\u2021','\u5350','\u534D','\u003F', '\u002A','\u007C'
)

replace_special_chars <- function(sentence) {
  stringi::stri_replace_all_fixed(sentence, puncts, paste0(" ", puncts, " "), vectorize_all = F)
}

preprocess_sentence <- compose(
  #add_tokens, ## removed
  replace_special_chars
  #replace_nations, ## removed
  #replace_years, ## removed
  #replace_gods ## removed
)