{"cells":[{"metadata":{"_uuid":"01f361ddc47e0b386595316fe3d7f4dabbd260db"},"cell_type":"markdown","source":"# 26th Place Kernel\nLast version was cleaned for public sharing. The previous one is the one that was submitted.\n\n*Code is a bit dirty, it was my first experience with PyTorch, and with Kaggle competitons in general. \nA lot of improvements can be made.*"},{"metadata":{},"cell_type":"markdown","source":"# Initialization"},{"metadata":{},"cell_type":"markdown","source":"### Imports"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import re\nimport gc\nimport os\nimport time\nimport random\nimport operator\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom tqdm import tqdm\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\ntqdm.pandas()\n\nfrom nltk import word_tokenize\nfrom collections import Counter\nfrom sklearn.utils import shuffle\nfrom tqdm import tqdm_notebook as tqdm\nfrom nltk.tokenize import TweetTokenizer\nfrom multiprocessing import Pool, cpu_count\nfrom keras.preprocessing.text import Tokenizer\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LinearRegression\nfrom keras.preprocessing.sequence import pad_sequences\nfrom sklearn.metrics import roc_curve, precision_recall_curve, f1_score\nfrom sklearn.model_selection import train_test_split, StratifiedKFold, RepeatedStratifiedKFold\n\nimport torch\nimport torch.nn as nn\nimport torch.utils.data\n\nimport psutil\nimport multiprocessing\nimport markovify as mk\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Seeding\n- For reproductibility"},{"metadata":{"trusted":true,"_uuid":"d4b9e0c513495041b334d2cd9f315a1f2aa8192f","_kg_hide-input":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    tf.set_random_seed(seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e784eedbc8234704b41156808743993c9da13d9f"},"cell_type":"code","source":"seed = 5583\nseed_everything(seed)\n\nbegin = time.time()\nsns.set_style('whitegrid')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Embeddings paths"},{"metadata":{"trusted":true,"_uuid":"273f8d56af9ac2e322bb273b1c3153e6600e7e31","scrolled":true,"_kg_hide-input":false},"cell_type":"code","source":"GLOVE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\nPARAGRAM = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\nFASTTEXT = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"63530632439404a85540565eb31c6390bddb33e9"},"cell_type":"markdown","source":"# Text Data"},{"metadata":{"_uuid":"e36041fbc5188f706645603c140cd4f09ec789d0"},"cell_type":"markdown","source":"## Loading"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \",train_df.shape)\nprint(\"Test shape : \",test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6ead9fc944079798e110e4a2334ae2aee51a4ebd"},"cell_type":"markdown","source":"## Augmenting\n- No augmentation was applied, but I tried augmenting with Markov Chains"},{"metadata":{"trusted":true,"_uuid":"4fb9557b8b4d4b2c9276db295b9674a9f622f769"},"cell_type":"code","source":"augment_insincere_count = 0","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"ba0221aa8c758bf10107b10f07370f41a4fab8af"},"cell_type":"code","source":"class MarkovifyTextAugmentor(object):\n    def __init__(self, docs, gen_max_len, filter_min_len=10):\n        \"\"\"\n        Build Markov models from large corpora of text, and generating random sentences from that.\n        \"\"\"\n        self.mk_text_model = mk.Text(docs)\n        self.gen_max_len = gen_max_len\n        self.filter_min_len = filter_min_len\n    \n    def genenerate(self, count):\n        texts = []\n        for _ in range(count):\n            text = self.mk_text_model.make_short_sentence(self.gen_max_len)\n            if text and len(text.split(' ')) > self.filter_min_len:\n                texts.append(text)\n        return texts","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"cbaeeba88b8fdfa3913caab6ba219a8018317111"},"cell_type":"code","source":"insincere_df = train_df[train_df['target'] == 1]\ninsincere_texts = insincere_df['question_text'].apply(\" \".join)\nmedian_length = int(insincere_texts.str.len().median())\n\nmk_augmentor = MarkovifyTextAugmentor(insincere_texts, median_length, filter_min_len=6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33bd41ca9d96547c34139aea9c3bcf2ba01b73b4","_kg_hide-input":false},"cell_type":"code","source":"%%time\ndef gen_text_process(count):\n    return mk_augmentor.genenerate(count)\n\nnum_cores = psutil.cpu_count()  # number of cores on your machine\npool = multiprocessing.Pool(processes=num_cores)\n\napply_results = []\nprocess_counts = augment_insincere_count // num_cores\nfor _ in range(num_cores):\n    apply_results.append(pool.apply_async(gen_text_process, (process_counts,)))\n\npool.close()\npool.join()\n\ngenerated_texts = []\nfor res in apply_results:\n    generated_texts.extend(res.get())\n    \ngen_df = pd.DataFrame({'question_text': generated_texts, 'target': [1] * len(generated_texts)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6195a0d455e6caebd03fa55c3bd5956f9e2a482d"},"cell_type":"code","source":"print('Generated {} insincere questions'.format(len(generated_texts)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"539712b2305112282055e1f20978f39cd0affc73"},"cell_type":"code","source":"if augment_insincere_count > 0:\n    train_df = pd.concat((train_df, gen_df), axis=0, sort=True)\n    train_df = shuffle(train_df, random_state=seed)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6da1c76e8ffd757223b2cdd79b542d3a0cb60772"},"cell_type":"markdown","source":"## Features\n> Features are taken from one of last year's Jigsaw competiton top scoring kernels\n\n- **Toxic word ratio :** The proportion of words in the sentence that are in a list of words labelled as toxic\n- **Total length :** Length of the sentence as a string\n- **Capital letters ratio :** PROPORTION OF LETTERS WRITTEN IN CAPS.\n- **Unique word ratio :** Proportion of words in the sentence that appear only once\n- **Average word frequency :** Frequency of the words in the sentence in the overall corpus\n"},{"metadata":{"_uuid":"7c0d3dbcc83db73425f6e920931eeaf3fb2bdb49"},"cell_type":"markdown","source":"### Toxic word ratio"},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"ae109a86d56bcd0106cfe1a8f025cd09aabc0ff8"},"cell_type":"code","source":"toxic_words = ['4r5e', '5h1t', '5hit', 'a55', 'anal', 'anus', 'ar5e', 'arrse', 'arse', 'ass', 'ass-fucker', 'asses', 'assfucker', 'assfukka', 'asshole', 'assholes', 'asswhole', 'a_s_s', 'b!tch', 'b00bs', 'b17ch', 'b1tch', 'ballbag', 'balls', 'ballsack', 'bastard', 'beastial', 'beastiality', 'bellend', 'bestial', 'bestiality', 'bi+ch', 'biatch', 'bitch', 'bitcher', 'bitchers', 'bitches', 'bitchin', 'bitching', 'bloody', 'blow', 'job', 'blowjob', 'blowjobs', 'boiolas', 'bollock', 'bollok', 'boner', 'boob', 'boobs', 'booobs', 'boooobs', 'booooobs', 'booooooobs', 'breasts', 'buceta', 'bugger', 'bum', 'bunny', 'fucker', 'butt', 'butthole', 'buttmuch', 'buttplug', 'c0ck', 'c0cksucker', 'carpet', 'muncher', 'cawk', 'chink', 'cipa', 'cl1t', 'clit', 'clitoris', 'clits', 'cnut', 'cock', 'cock-sucker', 'cockface', 'cockhead', 'cockmunch', 'cockmuncher', 'cocks', 'cocksuck', 'cocksucked', 'cocksucker', 'cocksucking', 'cocksucks', 'cocksuka', 'cocksukka', 'cok', 'cokmuncher', 'coksucka', 'coon', 'cox', 'crap', 'cum', 'cummer', 'cumming', 'cums', 'cumshot', 'cunilingus', 'cunillingus', 'cunnilingus', 'cunt', 'cuntlick', 'cuntlicker', 'cuntlicking', 'cunts', 'cyalis', 'cyberfuc', 'cyberfuck', 'cyberfucked', 'cyberfucker', 'cyberfuckers', 'cyberfucking', 'd1ck', 'damn', 'dick', 'dickhead', 'dildo', 'dildos', 'dink', 'dinks', 'dirsa', 'dlck', 'dog-fucker', 'doggin', 'dogging', 'donkeyribber', 'doosh', 'duche', 'dyke', 'ejaculate', 'ejaculated', 'ejaculates', 'ejaculating', 'ejaculatings', 'ejaculation', 'ejakulate', 'f', 'u', 'c', 'k', 'f', 'u', 'c', 'k', 'e', 'r', 'f4nny', 'fag', 'fagging', 'faggitt', 'faggot', 'faggs', 'fagot', 'fagots', 'fags', 'fanny', 'fannyflaps', 'fannyfucker', 'fanyy', 'fatass', 'fcuk', 'fcuker', 'fcuking', 'feck', 'fecker', 'felching', 'fellate', 'fellatio', 'fingerfuck', 'fingerfucked', 'fingerfucker', 'fingerfuckers', 'fingerfucking', 'fingerfucks', 'fistfuck', 'fistfucked', 'fistfucker', 'fistfuckers', 'fistfucking', 'fistfuckings', 'fistfucks', 'flange', 'fook', 'fooker', 'fuck', 'fucka', 'fucked', 'fucker', 'fuckers', 'fuckhead', 'fuckheads', 'fuckin', 'fucking', 'fuckings', 'fuckingshitmotherfucker', 'fuckme', 'fucks', 'fuckwhit', 'fuckwit', 'fudge', 'packer', 'fudgepacker', 'fuk', 'fuker', 'fukker', 'fukkin', 'fuks', 'fukwhit', 'fukwit', 'fux', 'fux0r', 'f_u_c_k', 'gangbang', 'gangbanged', 'gangbangs', 'gaylord', 'gaysex', 'goatse', 'God', 'god-dam', 'god-damned', 'goddamn', 'goddamned', 'hardcoresex', 'hell', 'heshe', 'hoar', 'hoare', 'hoer', 'homo', 'hore', 'horniest', 'horny', 'hotsex', 'jack-off', 'jackoff', 'jap', 'jerk-off', 'jism', 'jiz', 'jizm', 'jizz', 'kawk', 'knob', 'knobead', 'knobed', 'knobend', 'knobhead', 'knobjocky', 'knobjokey', 'kock', 'kondum', 'kondums', 'kum', 'kummer', 'kumming', 'kums', 'kunilingus', 'l3i+ch', 'l3itch', 'labia', 'lmfao', 'lust', 'lusting', 'm0f0', 'm0fo', 'm45terbate', 'ma5terb8', 'ma5terbate', 'masochist', 'master-bate', 'masterb8', 'masterbat*', 'masterbat3', 'masterbate', 'masterbation', 'masterbations', 'masturbate', 'mo-fo', 'mof0', 'mofo', 'mothafuck', 'mothafucka', 'mothafuckas', 'mothafuckaz', 'mothafucked', 'mothafucker', 'mothafuckers', 'mothafuckin', 'mothafucking', 'mothafuckings', 'mothafucks', 'mother', 'fucker', 'motherfuck', 'motherfucked', 'motherfucker', 'motherfuckers', 'motherfuckin', 'motherfucking', 'motherfuckings', 'motherfuckka', 'motherfucks', 'muff', 'mutha', 'muthafecker', 'muthafuckker', 'muther', 'mutherfucker', 'n1gga', 'n1gger', 'nazi', 'nigg3r', 'nigg4h', 'nigga', 'niggah', 'niggas', 'niggaz', 'nigger', 'niggers', 'nob', 'nob', 'jokey', 'nobhead', 'nobjocky', 'nobjokey', 'numbnuts', 'nutsack', 'orgasim', 'orgasims', 'orgasm', 'orgasms', 'p0rn', 'pawn', 'pecker', 'penis', 'penisfucker', 'phonesex', 'phuck', 'phuk', 'phuked', 'phuking', 'phukked', 'phukking', 'phuks', 'phuq', 'pigfucker', 'pimpis', 'piss', 'pissed', 'pisser', 'pissers', 'pisses', 'pissflaps', 'pissin', 'pissing', 'pissoff', 'poop', 'porn', 'porno', 'pornography', 'pornos', 'prick', 'pricks', 'pron', 'pube', 'pusse', 'pussi', 'pussies', 'pussy', 'pussys', 'rectum', 'retard', 'rimjaw', 'rimming', 's', 'hit', 's.o.b.', 'sadist', 'schlong', 'screwing', 'scroat', 'scrote', 'scrotum', 'semen', 'sex', 'sh!+', 'sh!t', 'sh1t', 'shag', 'shagger', 'shaggin', 'shagging', 'shemale', 'shi+', 'shit', 'shitdick', 'shite', 'shited', 'shitey', 'shitfuck', 'shitfull', 'shithead', 'shiting', 'shitings', 'shits', 'shitted', 'shitter', 'shitters', 'shitting', 'shittings', 'shitty', 'skank', 'slut', 'sluts', 'smegma', 'smut', 'snatch', 'son-of-a-bitch', 'spac', 'spunk', 's_h_i_t', 't1tt1e5', 't1tties', 'teets', 'teez', 'testical', 'testicle', 'tit', 'titfuck', 'tits', 'titt', 'tittie5', 'tittiefucker', 'titties', 'tittyfuck', 'tittywank', 'titwank', 'tosser', 'turd', 'tw4t', 'twat', 'twathead', 'twatty', 'twunt', 'twunter', 'v14gra', 'v1gra', 'vagina', 'viagra', 'vulva', 'w00se', 'wang', 'wank', 'wanker', 'wanky', 'whoar', 'whore', 'willies', 'willy', 'xrated', 'xxx']","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"45a31dad03352e0a1167e87307322b00e861155a"},"cell_type":"code","source":"def toxic_words_ratio(text, toxic_words=toxic_words):\n    count = 0\n    text = word_tokenize(text)\n    for word in text:\n        count += int(word.lower() in toxic_words)\n    return count / len(text)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ae3a258e1e051090d7fd8f0a37a288d4b58def81"},"cell_type":"markdown","source":"### Frequency Ratio"},{"metadata":{"trusted":true,"_uuid":"3e40eafbbbd8276f3dcd486c909b3de3cda92f2b"},"cell_type":"code","source":"def build_vocab(texts):\n    vocab = {}\n    for sentence in texts:\n        for word in sentence.split(' '):\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n    return vocab","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2796c9ad0ae0091f4a9e22b18ca8f8dab7714435"},"cell_type":"code","source":"word_count = build_vocab(list(train_df['question_text']) + list(test_df['question_text']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa3dbfa18d8e0a705e74d195c6a021e9121795ba"},"cell_type":"code","source":"def freq_count(text):\n    text = text.split(\" \")\n    all_count = 0\n    for word in text:\n        all_count += word_count[word]\n    return len(text) / all_count","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"514627a14937dc57d75759a0ad4f3475e5676a8e"},"cell_type":"markdown","source":"### Making all features"},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"ed9e86df78f0a854a0d9c1de0eced96bf9071f51"},"cell_type":"code","source":"def make_features(df):    \n    df['total_length'] = df['question_text'].apply(len)\n    df['capitals'] = df['question_text'].apply(lambda comment: sum(1 for c in comment if c.isupper()))\n    df['caps_vs_length'] = df.apply(lambda row: float(row['capitals'])/float(row['total_length']), axis=1)\n    df['num_words'] = df.question_text.str.count('\\S+')\n    df['num_unique_words'] = df['question_text'].apply(lambda comment: len(set(w for w in comment.split())))\n    df['words_vs_unique'] = df['num_unique_words'] / df['num_words']\n    df['toxic_ratio'] = df['question_text'].apply(toxic_words_ratio)\n    df['word_freq'] = df['question_text'].apply(freq_count)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = ['caps_vs_length', 'words_vs_unique', 'toxic_ratio', 'total_length', 'word_freq']\nnb_features = len(features)\n\nprint(f'Generated {nb_features} features :', ', '.join(features))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2aed2a8b64d51da90821fa5705c423d00bebe4ce"},"cell_type":"code","source":"%%time\nmake_features(train_df)\nmake_features(test_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Filling NaNs and scaling"},{"metadata":{"trusted":true,"_uuid":"8c336b14c8b69e9b87ea27e2bea1a8721d39dd76","scrolled":true},"cell_type":"code","source":"features_train = train_df[features].fillna(0)\nfeatures_test = test_df[features].fillna(0)\n\nss = StandardScaler()\nss.fit(np.vstack((features_train, features_test)))\nfeatures_train = ss.transform(features_train)\nfeatures_test = ss.transform(features_test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9032f827658e0209171a738add697ce7776c3e7a"},"cell_type":"markdown","source":"## Preprocessing\n- Treating apostrophes\n- Substituting with dic (contractions, misspels, some punctuation)\n- Removing 's and lone '\n- Cleaning numbers\n- Cleaning special characters\n- Removing extra spaces\n- Clean latex tags"},{"metadata":{"_uuid":"18133bc32f2dbfa94e7628390d2df61910b211d1"},"cell_type":"markdown","source":"### Data"},{"metadata":{"trusted":true,"_uuid":"ecc0051767a2183c858e80538e6b77eeb3b8502d","_kg_hide-input":true},"cell_type":"code","source":"# All appearing special characters\nuseful_punct = ['_', '☹', '＞', '½', '△', '¿', '¼', '∆', '≥', '⇒', '¬', '∨', '＾', 'µ', '?', '!', '.', ',', '\"', '#', '$', '%', '\\\\', \"'\", '(', ')', '*', '+', '-', '/', ':', ';', '<', '=', '>', '@', '[', ']', '^', '{', '|', '}', '~', '’', '′', '‘', '°', '→', '£', 'ø', '´', '↑', 'θ', '±', '≤', '≠', '...', '⟨', '⟩', '∖', 'ⁿ', '⅔', '❤', '✌', '✅', '✓', '∘', '¨', '″', 'ύ', 'ኤ', 'ል', 'ሮ', 'ኢ', 'የ', 'ኝ', 'ን', 'አ', 'ሁ', '≅', 'ϕ', '‑', '', 'ֿ', 'か', 'く', 'れ', '－', 'ș', 'ן', '∪', 'φ', 'ψ', '⊨', 'β', '∠', '«', '»', 'ம', '≈', '⁰', '⁷', '،', '＝', '（', '）', 'ə', 'ɑ', 'ː', '¹', '⅓', 'ř', '《', '》', 'ρ', '∅', '&', '·', '©', '¥', '：', '⋅', '↓', '、', '│', '，', '・', '•', '®', '`', '€', '™', '›', '♥', '←', '×', '§', 'Â', '█', 'à', '…', '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '░', '¶', '▾', '═', '¦', '║', '―', '▓', '—', '‹', '─', '▒', '⊕', '▼', '▪', '†', '■', '▀', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '∞', '∙', '♪', '╩', '╚', '³', '╦', '╣', '╔', '╗', '▬', 'ï', 'Ø', '‡', '√']\nuseless_punct = ['च', '不', 'ঢ়', '平', 'ᠠ', '錯', '判', '∙', '言', 'ς', 'ل', '្', 'ジ', 'あ', '得', '水', 'ь', '◦', '创', '康', '華', 'ḵ', '☺', '支', '就', '„', '」', '어', '谈', '陈', '团', '腻', '权', '年', '业', 'マ', 'य', 'ا', '売', '甲', '拼', '˂', 'ὤ', '贯', '亚', 'ि', '放', 'ʻ', 'ទ', 'ʖ', '點', '્', '発', '青', '能', '木', 'д', '微', '藤', '̃', '僕', '妒', '͜', 'ន', 'ध', '이', '希', '特', 'ड', '¢', '滢', 'ส', '나', '女', 'క', '没', '什', 'з', '天', '南', 'ʿ', 'ค', 'も', '凰', '步', '籍', '西', 'ำ', '−', 'л', 'ڤ', 'ៃ', '號', 'ص', 'स', '®', 'ʋ', '批', 'រ', '치', '谢', '生', '道', '═', '下', '俄', 'ɖ', '觀', 'வ', '—', 'ی', '您', '♥', '一', 'や', '⊆', 'ʌ', '語', 'ี', '兴', '惶', '瀛', '狐', '⁴', 'प', '臣', 'ద', '―', 'ì', 'ऌ', 'ీ', '自', '信', '健', '受', 'ɨ', '시', 'י', 'ছ', '嬛', '湾', '吃', 'ち', 'ड़', '反', '红', '有', '配', 'ে', 'ឯ', '宮', 'つ', 'μ', '記', '口', '℅ι', 'ो', '狸', '奇', 'о', 'ट', '聖', '蘭', '読', 'ū', '標', '要', 'ត', '识', 'で', '汤', 'ま', 'ʀ', '局', 'リ', '्', 'ไ', '呢', '工', 'ल', '沒', 'τ', 'ិ', 'ö', 'せ', '你', 'ん', 'ュ', '枚', '部', '大', '罗', 'হ', 'て', '表', '报', '攻', 'ĺ', 'ฉ', '∩', '宝', '对', '字', '文', '这', '∑', '髪', 'り', '่', '능', '罢', '내', '阻', '为', '菲', 'ي', 'न', 'ί', 'ɦ', '開', '†', '茹', '做', '東', 'ত', 'に', 'ت', '晓', '키', '悲', 'સ', '好', '›', '上', '存', '없', '하', '知', 'ធ', '斯', ' ', '授', 'ł', '傳', '兰', '封', 'ோ', 'و', 'х', 'だ', '人', '太', '品', '毒', 'ᡳ', '血', '席', '剔', 'п', '蛋', '王', '那', '梦', 'ី', '彩', '甄', 'и', '柏', 'ਨ', '和', '坊', '⌚', '广', '依', '∫', 'į', '故', 'ś', 'ऊ', '几', '日', 'ک', '音', '×', '”', '▾', 'ʊ', 'ज', 'ด', 'ठ', 'उ', 'る', '清', 'ग', 'ط', 'δ', 'ʏ', '官', '∛', '়', '้', '男', '骂', '复', '∂', 'ー', '过', 'য', '以', '短', '翻', 'র', '教', '儀', 'ɛ', '‹', 'へ', '¾', '合', '学', 'ٌ', '학', '挑', 'ष', '比', '体', 'م', 'س', 'អ', 'ת', '訓', '∀', '迎', 'វ', 'ɔ', '٨', '▒', '化', 'చ', '‛', 'প', 'º', 'น', '업', '说', 'ご', '¸', '₹', '儿', '︠', '게', '骨', 'ท', 'ऋ', 'ホ', '茶', '는', 'જ', 'ุ', '羡', '節', 'ਮ', 'উ', '番', 'ড়', '讲', 'ㅜ', '등', '伟', 'จ', '我', 'ล', 'す', 'い', 'ញ', '看', 'ċ', '∧', 'भ', 'ઘ', 'ั', 'ម', '街', 'ય', '还', '鰹', 'ខ', 'ు', '訊', 'म', 'ю', '復', '杨', 'ق', 'त', '金', '味', 'ব', '风', '意', '몇', '佬', '爾', '精', '¶', 'ం', '乱', 'χ', '교', 'ה', '始', 'ᠰ', '了', '个', '克', '্', 'ห', '已', 'ʃ', 'わ', '新', '译', '︡', '本', 'ง', 'б', 'け', 'ి', '明', '¯', '過', 'ك', 'ῥ', 'ف', 'ß', '서', '进', 'ដ', '样', '乐', '寧', '€', 'ณ', 'ル', '乡', '子', 'fi', 'ج', '慕', '–', 'ᡵ', 'Ø', '͡', '제', 'Ω', 'ប', '絕', '눈', 'फ', 'ম', 'గ', '他', 'α', 'ξ', '§', 'ஜ', '黎', 'ね', '복', 'π', 'ú', '鸡', '话', '会', 'ক', '八', '之', '북', 'ن', '¦', '가', 'ו', '恋', '地', 'ῆ', '許', '产', 'ॡ', 'ش', '़', '野', 'ή', 'ɒ', '啧', 'យ', '᠌', 'ᠨ', 'ب', '皎', '老', '公', '☆', 'व', 'ি', 'ល', 'ر', 'គ', '행', 'ង', 'ο', '让', 'ំ', 'λ', 'خ', 'ἰ', '家', 'ট', 'ब', '理', '是', 'め', 'र', '√', '기', 'ν', '玉', '한', '入', 'ד', '别', 'د', 'ะ', '电', 'ા', '♫', 'ع', 'ં', '堵', '嫉', '伊', 'う', '千', '관', '篇', 'क', '非', '荣', '粵', '瑜', '英', '를', '美', '条', '`', '宋', '←', '수', '後', '•', '³', 'ी', '고', '肉', '℃', 'し', '漢', '싱', 'ϵ', '送', 'ه', '落', 'న', 'ក', 'க', 'ℇ', 'た', 'ះ', '中', '射', '♪', '符', 'ឃ', '谷', '分', '酱', 'び', 'থ', 'ة', 'г', 'σ', 'と', '楚', '胡', '饭', 'み', '禮', '主', '直', '÷', '夢', 'ɾ', 'চ', '⃗', '統', '高', '顺', '据', 'ら', '頭', 'よ', '最', 'ా', 'ੁ', '亲', 'ស', '花', '≡', '眼', '病', '…', 'の', '發', 'ா', '汝', '★', '氏', 'ร', '景', 'ᡠ', '读', '件', '仲', 'শ', 'お', 'っ', 'پ', 'ᡤ', 'ч', '♭', '悠', 'ं', '六', '也', 'ռ', 'য়', '恐', 'ह', '可', '啊', '莫', '书', '总', 'ষ', 'ք', '̂', '간', 'な', '此', '愛', 'ర', 'ใ', '陳', 'Ἀ', 'ण', '望', 'द', '请', '油', '露', '니', 'ş', '宗', 'ʍ', '鳳', 'अ', '邋', '的', 'ព', '火', 'ा', 'ก', '約', 'ட', '章', '長', '商', '台', '勢', 'さ', '국', 'Î', '簡', 'ई', '∈', 'ṭ', '經', '族', 'ु', '孫', '身', '坑', 'স', '么', 'ε', '失', '殺', 'ž', 'ર', 'が', '手', 'ា', '心', 'ਾ', '로', '朝', '们', '黒', '欢', '早', '️', 'া', 'आ', 'ɸ', '常', '快', '民', 'ﷺ', 'ូ', '遢', 'η', '国', '无', '江', 'ॠ', '「', 'ন', '™', 'ើ', 'ζ', '紫', 'ె', 'я', '“', '♨', '國', 'े', 'อ', '∞']\n\n# Mapping special letters\nletter_mapping = {'\\u200b':' ', 'ũ': \"u\", 'ẽ': 'e', 'é': \"e\", 'á': \"a\", 'ķ': 'k', 'ï': 'i', 'Ź': 'Z', 'Ż': 'Z', 'Š': 'S', 'Π': ' pi ', 'Ö': 'O', 'É': 'E', 'Ñ': 'N', 'Ž': 'Z', 'ệ': 'e', '²': '2', 'Å': 'A', 'Ā': 'A', 'ế': 'e', 'ễ': 'e', 'ộ': 'o', '⧼': '<', '⧽': '>', 'Ü': 'U', 'Δ': 'delta', 'ợ': 'o', 'İ': 'I', 'Я': 'R', 'О': 'O', 'Č': 'C', 'П': 'pi', 'В': 'B', 'Φ': 'phi', 'ỵ': 'y', 'օ': 'o', 'Ľ': 'L', 'ả': 'a', 'Γ': 'theta', 'Ó': 'O', 'Í': 'I', 'ấ': 'a', 'ụ': 'u', 'Ō': 'O', 'Ο': 'O', 'Σ': 'sigma', 'Â': 'A', 'Ã': 'A', 'ᗯ': 'w', 'ᕼ': \"h\", \"ᗩ\": \"a\", \"ᖇ\": \"r\", \"ᗯ\": \"w\", \"O\": \"o\", \"ᗰ\": \"m\", \"ᑎ\": \"n\", \"ᐯ\": \"v\", \"н\": \"h\", \"м\": \"m\", \"o\": \"o\", \"т\": \"t\", \"в\": \"b\", \"υ\": \"u\",  \"ι\": \"i\",\"н\": \"h\", \"č\": \"c\", \"š\": \"s\", \"ḥ\": \"h\", \"ā\": \"a\", \"ī\": \"i\", \"à\": \"a\", \"ý\": \"y\", \"ò\": \"o\", \"è\": \"e\", \"ù\": \"u\", \"â\": \"a\", \"ğ\": \"g\", \"ó\": \"o\", \"ê\": \"e\", \"ạ\": \"a\", \"ü\": \"u\", \"ä\": \"a\", \"í\": \"i\", \"ō\": \"o\", \"ñ\": \"n\", \"ç\": \"c\", \"ã\": \"a\", \"ć\": \"c\", \"ô\": \"o\", \"с\": \"c\", \"ě\": \"e\", \"æ\": \"ae\", \"î\": \"i\", \"ő\": \"o\", \"å\": \"a\", \"Ä\": \"A\", }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3826897f4f4bd456926ce0238cccc8e3e2a7ee53","_kg_hide-input":true},"cell_type":"code","source":"mispell_dict = {\"trimp\": \"trump\", \"wanket\": \"wanker\",'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'qoura': 'quora', 'sallary': 'salary', 'whta': 'what', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'etherium': 'ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization',}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e0db95a02a685497b072d2749832755523a87e3e"},"cell_type":"markdown","source":"#### All substitutions"},{"metadata":{"trusted":true,"_uuid":"f747e2b8793a0ecb1e0b5c2a8fb35e7142084a2b","_kg_hide-input":false},"cell_type":"code","source":"def add_maj(dic):\n    d = dic.copy()\n    for w in dic.keys():\n        try:\n            d[w[0].upper() + w[1:]] = dic[w][0].upper() + dic[w][1:]\n        except:\n            d[w[0].upper()] = dic[w][0].upper()\n    return d","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"393f97923716a1210d1a614712129a4916ba86fb","_kg_hide-input":true},"cell_type":"code","source":"substitution_dic = {}\nsubstitution_dic.update(mispell_dict)\nsubstitution_dic = add_maj(substitution_dic)\nsubstitution_dic.update(letter_mapping)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6e07bec25e2aaa775681535973ff5b6ad9a794e2"},"cell_type":"markdown","source":"#### Substitutions"},{"metadata":{"trusted":true,"_uuid":"76d7f363e592f975b6f2e022180e6df822988d13"},"cell_type":"code","source":"def _get_substitution(sub_dic):\n    sub_re = re.compile('(%s)' % '|'.join(sub_dic.keys()))\n    return sub_dic, sub_re\n\nsubstitutions, substitutions_re = _get_substitution(substitution_dic)\n\ndef replace_substitution(text):\n    def replace(match):\n        return substitutions[match.group(0)]\n    return substitutions_re.sub(replace, text)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d933a46addd54cbf5604e015f2068ad1c3e6607d"},"cell_type":"markdown","source":"#### Apostrophes"},{"metadata":{"trusted":true,"_uuid":"afb88e1b29c7f811426b9031cfc58382a6690ffa"},"cell_type":"code","source":"def clean_apostrophes(x):\n    apostrophes = [\"’\", \"‘\", \"´\", \"`\"]\n    for s in apostrophes:\n        x = re.sub(s, \"'\", x)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"434f5d0ce954bc74bd5fa6ff73deb41e93cfbeaf"},"cell_type":"markdown","source":"#### 's"},{"metadata":{"trusted":true,"_uuid":"9a169221495370131b024ca0fd1943834998dec9"},"cell_type":"code","source":"def remove_s(x): \n    if len(x) > 2:\n        return re.sub(\"('$ |'$|'s |'s)\", ' ', x)\n    else:\n        return x","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"aea95aa6cac820d7b2696526f198c0f74188749d"},"cell_type":"markdown","source":"#### Spaces"},{"metadata":{"trusted":true,"_uuid":"1535526d3abbc69d6621948bcd02f52d9b797b9b"},"cell_type":"code","source":"spaces = ['\\u200b', '\\u200e', '\\u202a', '\\u202c', '\\ufeff', '\\uf0d8', '\\u2061', '\\x10', '\\x7f', '\\x9d', '\\xad', '\\xa0']\ndef clean_spaces(text):\n    for space in spaces:\n        text = text.replace(space, ' ')\n    text = text.strip()\n    text = re.sub('\\s+', ' ', text)\n    return text","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bef654848609e2342eadfc9ae744caf80b58ca65"},"cell_type":"markdown","source":"#### Contractions"},{"metadata":{"trusted":true,"_uuid":"deb633646949aa4365ab3a14a53212e88771ea49"},"cell_type":"code","source":"def decontract(text):\n    text = re.sub(r\"(W|w)on(\\'|\\’)t \", \"will not \", text)\n    text = re.sub(r\"(C|c)an(\\'|\\’)t \", \"can not \", text)\n    text = re.sub(r\"(Y|y)(\\'|\\’)all \", \"you all \", text)\n    text = re.sub(r\"(Y|y)a(\\'|\\’)ll \", \"you all \", text)\n    text = re.sub(r\"(I|i)(\\'|\\’)m \", \"i am \", text)\n    text = re.sub(r\"(A|a)isn(\\'|\\’)t \", \"is not \", text)\n    text = re.sub(r\"n(\\'|\\’)t \", \" not \", text)\n    text = re.sub(r\"(\\'|\\’)re \", \" are \", text)\n    text = re.sub(r\"(\\'|\\’)s \", \" is \", text)\n    text = re.sub(r\"(\\'|\\’)d \", \" would \", text)\n    text = re.sub(r\"(\\'|\\’)ll \", \" will \", text)\n    text = re.sub(r\"(\\'|\\’)t \", \" not \", text)\n    text = re.sub(r\"(\\'|\\’)ve \", \" have \", text)\n    return text","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"305679ae8304397b069f4fcb21210623d9ee1a56"},"cell_type":"markdown","source":"#### Numbers"},{"metadata":{"trusted":true,"_uuid":"1e8e6ddf58ef800f8629986af057bd662e0998ae"},"cell_type":"code","source":"def clean_numbers(text):\n    text = re.sub(r'(\\d+)([a-zA-Z])', '\\g<1> \\g<2>', text)\n    text = re.sub(r'(\\d+) (th|st|nd|rd) ', '\\g<1>\\g<2> ', text)\n    text = re.sub(r'(\\d+),(\\d+)', '\\g<1>\\g<2>', text)\n    return text","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5d807c0b2ba1570a06d34a02ace1d4a93ac4dd73"},"cell_type":"markdown","source":"#### Special characters"},{"metadata":{"trusted":true,"_uuid":"2304a6cb62d38bd040d77e298be2e68ffc18f7f2"},"cell_type":"code","source":"def clean_special_chars(text, punct=useful_punct):\n    for p in punct:\n        text = text.replace(p, f' {p} ')\n    return text","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3327c1f0f99a3364072019c3711a955599b77188"},"cell_type":"markdown","source":"#### Latex Tag"},{"metadata":{"trusted":true,"_uuid":"b5d9a972aec0a9bdfb5829a96420f1ca6c9ef988"},"cell_type":"code","source":"def clean_latex_tag(text):\n    corr_t = []\n    for t in text.split(\" \"):\n        t = t.strip()\n        if t != '':\n            corr_t.append(t)\n    text = ' '.join(corr_t)\n    text = re.sub('(\\[ math \\]).+(\\[ / math \\])', 'mathematical formula', text)\n    return text","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4c09c6d749df5f9153a231033307586863f1337a"},"cell_type":"markdown","source":"### Applying stuff"},{"metadata":{"trusted":true,"_uuid":"bbdaa69579a9b0441ea8a1d05fc06cc757497695","_kg_hide-input":true},"cell_type":"code","source":"def build_vocab(texts):\n    vocab = {}\n    for sentence in texts:\n        for word in sentence.split(' '):\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n    return vocab","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c018dc156854979b6a15c4f6dcb8df2dce65675"},"cell_type":"code","source":"from multiprocessing import Pool, cpu_count\nprint(\"Number of available cpu cores: {}\".format(cpu_count()))\n\ndef process_in_parallel(function, list_):\n    with Pool(cpu_count()) as p:\n        tmp = p.map(function, list_)\n    return tmp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c69deec8fc27d3e72acadbf2ded5e1b1bbe87602"},"cell_type":"code","source":"def treat_texts(texts):\n    texts = process_in_parallel(clean_apostrophes, texts)\n    texts = process_in_parallel(replace_substitution, texts)\n    texts = process_in_parallel(decontract, texts)\n    texts = process_in_parallel(remove_s, texts)\n    texts = process_in_parallel(clean_numbers, texts)\n    texts = process_in_parallel(clean_special_chars, texts)\n    texts = process_in_parallel(clean_spaces, texts)\n    texts = process_in_parallel(clean_latex_tag, texts)   \n    return texts","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d515dc065cd07136a44935c0c66bff0c54b8804f"},"cell_type":"code","source":"%%time\ntrain_df[\"question_text\"] = treat_texts(train_df[\"question_text\"])\ntest_df[\"question_text\"] = treat_texts(test_df[\"question_text\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e20598cb5df70d93c62f4289f73d7a449ebbd87"},"cell_type":"code","source":"for q in test_df[\"question_text\"][:5]:\n    print(q)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a502a71499813490b3c369413df52c61f75ce988"},"cell_type":"code","source":"vocab = build_vocab(list(train_df[\"question_text\"]) + list(test_df[\"question_text\"]))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"77496cf68d2a85c0878ffd396de6f50d21673eb2"},"cell_type":"markdown","source":"## Text Input\n- Tokenize & pad"},{"metadata":{"trusted":true,"_uuid":"9a3f78378d46733734d412cf40d069a6b4817436"},"cell_type":"code","source":"len_voc = None\nmax_len = 70","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc092308b5a26ecb8d6299a704eac1e0a824cf9e"},"cell_type":"code","source":"def make_input_data(X_train, X_test):\n    t = Tokenizer(num_words=len_voc, filters='', lower=False)\n    t.fit_on_texts((np.concatenate((X_train, X_test), axis=0)))\n    X_train = t.texts_to_sequences(X_train)\n    X_test = t.texts_to_sequences(X_test)\n    X_train = pad_sequences(X_train, maxlen=max_len, padding='post', truncating='post')\n    X_test = pad_sequences(X_test, maxlen=max_len, padding='post', truncating='post')\n\n    return X_train, X_test, t.word_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"68847d69312b393b5b81da8c3409b4ea5841dfdb"},"cell_type":"code","source":"%%time\nX_train, X_test, word_index = make_input_data(train_df['question_text'], test_df['question_text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a1db7061c715cfe802a3405203606bcbd4fe8f4"},"cell_type":"code","source":"len_voc = len(word_index) + 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Target"},{"metadata":{"trusted":true,"_uuid":"ba5a1b8109dee2c9fbc628d5da4a7c3447d42fb8"},"cell_type":"code","source":"y_train = train_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(y_train)\nplt.title('Target Repartition', size=15)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"57533e8b1621dd25eff1196e9a183e2ef94a13bd"},"cell_type":"markdown","source":"# Embedding matrices\n- Concatenation of Glove, FastText & Paragram"},{"metadata":{"trusted":true,"_uuid":"0f838586cdf876df076b2a1232f2f9b65df9650d","_kg_hide-input":true},"cell_type":"code","source":"def make_embed_index(file, word_index, vocab=[], exceptions={}):\n    def get_coefs(word,*arr): \n        return word, np.asarray(arr, dtype='float32')\n    \n    if file == FASTTEXT:\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file) if len(o)>100 and o.split(\" \")[0] in word_index )\n    if file == PARAGRAM:\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding=\"utf8\", errors='ignore') if len(o)>100 and o.split(\" \")[0] in word_index)\n    else: #GloVe\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file) if o.split(\" \")[0] in word_index)\n    \n    for word in vocab:\n        try:\n            _ = embeddings_index[word]\n        except:\n            try:\n                embeddings_index[word] = embeddings_index[word.lower()]\n            except:\n                try:\n                    embeddings_index[word] = embeddings_index[word.upper()]\n                except:\n                    try:\n                        embeddings_index[word] = embeddings_index[word[0].upper() + word[1:].lower()]\n                    except:\n                        pass\n            \n    return embeddings_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_uuid":"83c10d5e2279e7af93f827344a8fd6fd08b975b7"},"cell_type":"code","source":"def make_embed_mat(embeddings_index):\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (len(word_index)+1, embed_size))\n    for word, i in word_index.items():\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: \n            embedding_matrix[i] = embedding_vector\n\n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"03cce1bc5525b77a3d6833d12e3a68ef1e453c19"},"cell_type":"code","source":"%%time\nglove_index = make_embed_index(GLOVE, word_index, vocab)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4dffc0ae5f8fb23d47f561d21bb4e10a902e3ef9","scrolled":true},"cell_type":"code","source":"%%time\nfasttext_index = make_embed_index(FASTTEXT, word_index, vocab)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9238b8e858ef1cd58b08ac763e751e2d31802562"},"cell_type":"code","source":"%%time\npara_index = make_embed_index(PARAGRAM, word_index, vocab)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"edcd0e3c4eb462fe1f127e70953fde13089dc48b"},"cell_type":"markdown","source":"### Coverage"},{"metadata":{"trusted":true,"_kg_hide-input":true,"_uuid":"ed858c2cf711cfcc38fb604e52dd57fab016502d"},"cell_type":"code","source":"def check_coverage(vocab, embeddings_index, word_index=None):\n    known_words = {}\n    unknown_words = {}\n    nb_known_words = 0\n    nb_unknown_words = 0\n    for word in vocab.keys():\n        try:\n            known_words[word] = embeddings_index[word]\n            nb_known_words += vocab[word]\n        except:\n            unknown_words[word] = vocab[word]\n            nb_unknown_words += vocab[word]\n            pass\n\n    print('Found embeddings for {:.2%} of vocab'.format(len(known_words) / len(vocab)))\n    print('Found embeddings for  {:.2%} of all text'.format(nb_known_words / (nb_known_words + nb_unknown_words)))\n    unknown_words = sorted(unknown_words.items(), key=operator.itemgetter(1))[::-1]\n\n    return unknown_words","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"23a721b9f2a3bf37ce17fd9c76c3f6f913b9eeb1"},"cell_type":"code","source":"print(\"Glove : \")\noov_glove = check_coverage(vocab, glove_index, word_index)\nprint(\"Paragram : \")\noov_paragram = check_coverage(vocab, para_index, word_index)\nprint(\"FastText : \")\noov_fasttext = check_coverage(vocab, fasttext_index, word_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eea5619a84da9f20ec60f59a6e1a215207d1003f"},"cell_type":"code","source":"embed_concat = np.concatenate((make_embed_mat(fasttext_index), make_embed_mat(glove_index), make_embed_mat(para_index)), axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3da561869a0a2b5e9487c9ac567baaa616c9e55f"},"cell_type":"code","source":"t_init = time.time()\nprint(f\"Initialized in {(t_init - begin) // 60} minutes\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"99b48dca7930787a6eb9c7c98271b2c4099d9792"},"cell_type":"markdown","source":"# Modeling"},{"metadata":{},"cell_type":"markdown","source":"## Tools"},{"metadata":{"_uuid":"32eae56339a52e449b87be5b2a0cec3b6b4887b0"},"cell_type":"markdown","source":"### Noise"},{"metadata":{"trusted":true,"_uuid":"ac110c73d7a974925b75cad798dd106e432c5a14","_kg_hide-input":true},"cell_type":"code","source":"class Noise(nn.Module):\n    def __init__(self, mean=0.0, stddev=0.1):\n        super(Noise, self).__init__()\n        self.mean = mean\n        self.stddev = stddev\n\n    def forward(self, input):\n        noise = input.clone().normal_(self.mean, self.stddev)\n        return input + noise","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eddf0df677a9ea7ab043d3e78acb58c0c17d6e0e"},"cell_type":"markdown","source":"### Attention Layer"},{"metadata":{"trusted":true,"_uuid":"ab443919ba40594af45a68e00c4581dd1274299a","_kg_hide-input":true},"cell_type":"code","source":"class Attention(nn.Module):\n    def __init__(self, feature_dim, step_dim, bias=True, **kwargs):\n        super(Attention, self).__init__(**kwargs)\n        self.supports_masking = True\n        self.bias = bias\n        self.feature_dim = feature_dim\n        self.step_dim = step_dim\n        self.features_dim = 0\n        weight = torch.zeros(feature_dim, 1)\n        nn.init.xavier_uniform_(weight)\n        self.weight = nn.Parameter(weight)\n        if bias:\n            self.b = nn.Parameter(torch.zeros(step_dim))\n        \n    def forward(self, x, mask=None):\n        feature_dim = self.feature_dim\n        step_dim = self.step_dim\n        eij = torch.mm(x.contiguous().view(-1, feature_dim), self.weight).view(-1, step_dim)\n        if self.bias:\n            eij = eij + self.b  \n        eij = torch.tanh(eij)\n        a = torch.exp(eij)\n        if mask is not None: \n            a = a * mask\n        a = a / torch.sum(a, 1, keepdim=True) + 1e-10\n        weighted_input = x * torch.unsqueeze(a, -1)\n        return torch.sum(weighted_input, 1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5c79a9136f41cf4ad76051632f8152a0f8208336"},"cell_type":"markdown","source":"## Model"},{"metadata":{"trusted":true,"_uuid":"c0815521236eed44c8b76de313e1ff6a60024f58"},"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self, embedding_matrix):\n        super(Model, self).__init__()\n        \n        h1 = 64\n        h2 = 32\n        hd = 32\n        \n        self.embedding = nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1])\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        self.embedding.weight.requires_grad = False\n        self.noise = Noise(stddev=0.05)\n        \n        self.lstm = nn.LSTM(embedding_matrix.shape[1], 2*h1, bidirectional=False, batch_first=True)\n        self.gru = nn.GRU(2*h1, 2*h2, bidirectional=False, batch_first=True)\n        \n        self.lstm_att = Attention(2*h1, max_len)\n        self.gru_att = Attention(2*h2, max_len)\n        \n        self.dropout = nn.Dropout(0.1)\n        self.linear = nn.Linear(6*(h1+h2) + nb_features, hd)\n        self.relu = nn.ReLU()\n        self.bn = nn.BatchNorm1d(hd, momentum=0.5)\n        self.out = nn.Linear(hd, 1)\n    \n    def forward(self, x):\n        embed = self.embedding(x[0])\n        \n        if self.training:\n            embed = torch.squeeze(self.noise(torch.unsqueeze(embed, 0)))\n        \n        lstm, _ = self.lstm(embed)\n        gru, _ = self.gru(lstm)\n        \n        att1 = self.lstm_att(lstm)\n        att2 = self.gru_att(gru)\n        \n        avg_pool1 = torch.mean(lstm, 1)\n        avg_pool2 = torch.mean(gru, 1)\n        max_pool1, _ = torch.max(lstm, 1)\n        max_pool2, _ = torch.max(gru, 1)\n        \n        conc = torch.cat((att1, avg_pool1, max_pool1, att2, avg_pool2, max_pool2, x[1]), 1)\n        conc = self.dropout(conc)\n        conc = self.bn(self.relu(self.linear(conc)))\n        conc = self.dropout(conc)\n        out = self.out(conc)\n        \n        return out","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"80580a6745b44b8b63ab725672c8ffafbab57e75"},"cell_type":"markdown","source":"# Training"},{"metadata":{"_uuid":"a7a163f4704046ecd6b0d1ad0668bd21186d48b6"},"cell_type":"markdown","source":"### Sigmoid"},{"metadata":{"trusted":true,"_uuid":"4a2aa9c20b2d5e6e7aeb18d637b56b281818d7df","_kg_hide-input":true},"cell_type":"code","source":"def sigmoid(x):\n    return 1 / (1 + np.exp(-x))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6ef6ada77fc3f425e1b68544aa35862c0ee94127"},"cell_type":"markdown","source":"### Learning rate"},{"metadata":{"trusted":true,"_uuid":"6a4f4f82796d3cb04219187c4b5a2872f75376f6","_kg_hide-input":true},"cell_type":"code","source":"def get_lr(epoch):\n    if epoch <= 3:\n        return 0.001\n    else:\n        return 0.0005","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0bf57b15f1c9efb94ac1e8dedc685d5de5820102"},"cell_type":"markdown","source":"### Learning curves"},{"metadata":{"trusted":true,"_uuid":"76f2aa64a1cee4ec64d90941b6b1c25127268400","_kg_hide-input":true},"cell_type":"code","source":"def plot_history(history, title='Learning Curves'):\n    plt.plot(history['loss'], label='Train loss')\n    try : plt.plot(history['val_loss'], label='Test loss')\n    except: pass\n    plt.title(title, size=15)\n    plt.legend()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"748abb0219f4f6c2a46947a9ccac9bd51fcbce75"},"cell_type":"markdown","source":"### Tweak Threshold"},{"metadata":{"trusted":true,"_uuid":"5db4fe3e720364fe2452a196d4803bcab86efe7d","_kg_hide-input":true},"cell_type":"code","source":"def tweak_threshold(train_preds, y_train):\n    tmp = [0,0,0] # idx, cur, max\n    delta = 0\n    for tmp[0] in np.arange(0.1, 0.501, 0.01):\n        tmp[1] = f1_score(y_train, np.array(train_preds)>tmp[0])\n        if tmp[1] > tmp[2]:\n            delta = tmp[0]\n            tmp[2] = tmp[1]\n    return tmp[2], delta","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"886ac42e27a107d63d29777618a60a8ae5ff1d37"},"cell_type":"markdown","source":"### Predict"},{"metadata":{"trusted":true,"_uuid":"7185f2ee89813b7c279567439fb6448adc988e6e","_kg_hide-input":true},"cell_type":"code","source":"def predict(X, f,model, batch_size=1024):\n    y = np.array([])\n    X = torch.tensor(X, dtype=torch.long).cuda()\n    f = torch.tensor(f, dtype=torch.float32).cuda()\n    Xf = torch.utils.data.TensorDataset(X, f)\n    loader = torch.utils.data.DataLoader(Xf, batch_size=batch_size, shuffle=False)\n    for i, (x, f) in enumerate(loader):\n        y = np.concatenate((y, sigmoid(model([x, f]).detach().cpu().numpy())[:, 0]))\n    return y","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"18a67b151f99a9f3243c3ed96cf6a2d52961e2b3"},"cell_type":"markdown","source":"### Fitting"},{"metadata":{"trusted":true,"_uuid":"c37d78d7fa35b10f306805babcfa76e918c97529","_kg_hide-input":true},"cell_type":"code","source":"def fit(model, X_train, f_train, y_train, X_val=None, f_val=None, y_val=None, epochs=5, batch_size=512):\n    history = {\"loss\":[], \"val_loss\": []}\n    best_loss = 10\n    model.cuda()\n    \n    loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum')\n    optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=0.001)\n        \n    X_train = torch.tensor(X_train, dtype=torch.long).cuda()\n    f_train = torch.tensor(f_train, dtype=torch.float32).cuda()\n    y_train = torch.tensor(y_train[:, np.newaxis], dtype=torch.float32).cuda()\n    train = torch.utils.data.TensorDataset(X_train, f_train, y_train)    \n    train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n    \n    X_val = torch.tensor(X_val, dtype=torch.long).cuda()\n    f_val = torch.tensor(f_val, dtype=torch.float32).cuda()\n    y_val = torch.tensor(y_val[:, np.newaxis], dtype=torch.float32).cuda()\n    val = torch.utils.data.TensorDataset(X_val, f_val, y_val)\n    val_loader = torch.utils.data.DataLoader(val, batch_size=batch_size, shuffle=False)\n\n    for epoch in range(epochs):\n        model.train()\n        avg_loss = 0\n        start_time = time.time()  \n            \n        lr = get_lr(epoch)\n        for param_group in optimizer.param_groups:\n            param_group['lr'] = lr\n        \n        for x_batch, f_batch, y_batch in train_loader:\n            optimizer.zero_grad()\n            y_pred = model([x_batch, f_batch])\n            loss = loss_fn(y_pred, y_batch)\n            loss.backward()\n            optimizer.step()\n            avg_loss += loss.item() / X_train.shape[0]\n    \n        model.eval()\n        avg_val_loss = 0.\n        for i, (x_batch, f_batch, y_batch) in enumerate(val_loader):\n            y_pred = model([x_batch, f_batch]).detach()\n            avg_val_loss += loss_fn(y_pred, y_batch).item() / X_val.shape[0]\n        \n        history['loss'].append(avg_loss)\n        history['val_loss'].append(avg_val_loss)\n        elapsed_time = time.time() - start_time \n        print('Epoch {}/{} \\t lr={} \\t loss={:.4f} \\t val_loss={:.4f} \\t time={:.2f}s'.format(epoch + 1, epochs, lr, avg_loss, avg_val_loss, elapsed_time))\n        \n    return history","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5388fcfcb1e485f77cb725f0ca32612adad05905"},"cell_type":"markdown","source":"### $k$-fold"},{"metadata":{"trusted":true,"_uuid":"0d826f60f049a0ccad48948d5dbdec65168d8699","_kg_hide-input":true},"cell_type":"code","source":"def k_fold(model_class, embedding_matrix, X, f, y, X_test, f_test, k=5, batch_size=512, epochs=5, seed=seed):\n    splits = list(StratifiedKFold(n_splits=k, shuffle=True, random_state=420).split(X, y))\n    pred_test = np.zeros(X_test.shape[0])\n    pred_oof = np.zeros(y.shape[0])\n    histories = []\n    \n    for i, (train_idx, val_idx) in enumerate(splits):\n        print(f\"-------------   Fold {i+1}  ------------- \\n\")\n        seed_everything(seed + i)\n        \n        cp_path = f\"{i}_weights.pth.tar\"\n        start_time = time.time()\n        \n        model = model_class(embedding_matrix)\n\n        history = fit(model, X[train_idx], f[train_idx], y[train_idx], X[val_idx], f[val_idx], y[val_idx], epochs=epochs, batch_size=batch_size)\n        histories.append(history)\n\n        pred_oof[val_idx] = predict(X[val_idx], f[val_idx], model)\n        pred_test += predict(X_test, f_test, model) / k\n        \n        score, threshold = tweak_threshold(pred_oof[val_idx], y[val_idx])\n        \n        print(f\"\\n Scored {score :.4f} for threshold {threshold :.3f} on validation data\")\n        print(f\"\\n    Done in {(time.time() - start_time) / 60 :.1f} minutes \\n\")\n        \n    return pred_test, pred_oof, histories","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a504bdc5306bd29b9b5f4ee453e515ecc1d1ffb4"},"cell_type":"markdown","source":"## Training"},{"metadata":{"trusted":true,"_uuid":"cda94967ed5749b9ffa2c918f2924826ab42d61e"},"cell_type":"code","source":"pred_test, pred_oof, histories = k_fold(Model, embed_concat, \n                                        X_train, features_train, y_train, X_test, features_test,\n                                        k=5, batch_size=512, epochs=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"score, threshold = tweak_threshold(pred_oof, y_train)\nprint(f\"Local CV : {score:.4f} for threshold {threshold:.3f}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"103cb64bac9dd4645893687d05368acb458de34f","scrolled":false},"cell_type":"code","source":"plt.figure(figsize=(15, 10))\nfor i in range(len(histories)):\n    plt.subplot(3, 2, i+1)\n    plot_history(histories[i], \"Fold \" + str(i+1))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5870f654f01d91c646900df1e00b98fc8f7b446a"},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true,"_uuid":"52c1b020bf4a928e8bf0c4956b47d9e75b113ef5"},"cell_type":"code","source":"label_test = (pred_test > threshold).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c90fb4a4ef1b3b2ea06563a6901deac1b38822f3"},"cell_type":"code","source":"output = pd.DataFrame({\"qid\": test_df[\"qid\"].values})\noutput['prediction'] = label_test\noutput.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"60641a068ca9abe28ff6f5d16d38afe3d6d01636"},"cell_type":"code","source":"output.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf23080c7d4f6508617b98043a972f0f12c64c37"},"cell_type":"code","source":"print(f\"Ended in {(time.time() - begin) / 60 :.1f} minutes\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}