{"cells":[{"metadata":{"trusted":false,"_uuid":"604607f330f5a92fe8adf91f4f68ece19c6ff313"},"cell_type":"code","source":"# basics\nimport os\nimport time\nimport re\nimport regex\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom pathlib import Path\nimport random\nimport matplotlib.pyplot as plt\nfrom multiprocessing import Pool\n\n# machine learning\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import f1_score, roc_curve, precision_recall_curve\nfrom sklearn.preprocessing import StandardScaler\n\n# nn\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data\n\n# use only for tokenizer and padding\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d982491328b41a40d29fae639d601048c4e9c861"},"cell_type":"code","source":"cuda_idx = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"6ee63e341817aa3cd56335ada059a834c998d2b6"},"cell_type":"code","source":"all_start = time.time()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"8233e45f9bf4c813f82005702ee277b594405166"},"cell_type":"code","source":"def timer(fn):\n    def func(*args):\n        start = time.time()\n        res = fn(*args)\n        elapsed = time.time() - start\n        print('timer: {:.3f} elapsed by {}'.format(elapsed, fn))\n        return res\n        \n    return func","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"03f5bcf651c3eda56488f9a78797ed758358677f"},"cell_type":"code","source":"def seed_torch(seed=1019):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nSEED = 1019\nseed_torch(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e25e07ae494f36d76c37dad5e39a06d85768ea1d"},"cell_type":"code","source":"# model parameters\nclass Config:\n    num_epochs = 6\n    num_embedding_learn_epochs = 1\n    embedding_noise_var = 0.01\n    batch_size = [256, 512, 512, 1024, 1024, 1536]\n    test_batch_size = 512\n    vocab_size = 120000\n    max_length = 72\n    embedding_size = 300\n    hidden_size = 96\n    num_layers = 1\n    num_gru_layers = 1\n    embedding_dropout = 0.3\n    layer_dropout = 0.1\n    dense_size = [hidden_size*2*4+3, int(hidden_size/4)] # depend on concat num\n    output_size = 1\n    num_cv_splits = 5\n    num_routings = 4\n    num_capsules = 5\n    dim_capsules = 5\n    learning_rate = 0.001\n    max_learning_rate = 0.003\n    clr_step_size = 300\n    clr_gamma = 0.9999\n    clip_grad = 5.0\n    embeddings = ['glove', 'paragram', 'fasttext']  # ['word2vec', 'glove', 'paragram', 'fasttext']\n    # datadir = Path('../../../data/quora_insincere')\n    datadir = Path('../input') # for kernel\n\nc = Config()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"9eb12bfb150b984cc50ca58c67bfa050800398eb"},"cell_type":"code","source":"puncts = [\n    ',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&',\n    '/', '[', ']', '%', '=', '#', '*', '+', '\\\\', '•', '~', '@', '£',\n    '·', '_', '{', '}', '©', '^', '®', '`', '→', '°', '€', '™', '›',\n    '♥', '←', '×', '§', '″', '′', 'Â', '█', 'à', '…', '“', '★', '”',\n    '–', '●', 'â', '►', '−', '¢', '¬', '░', '¶', '↑', '±',  '▾',\n    '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '⊕', '▼',\n    '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲',\n    'è', '¸', 'Ã', '⋅', '‘', '∞', '∙', '）', '↓', '、', '│', '（', '»',\n    '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø',\n    '¹', '≤', '‡', '₹', '´'\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a4bff9dce9999894a3cafdf33fae900aaf5d9024"},"cell_type":"code","source":"abbreviations = {\n    \"ain't\": \"is not\",\n    \"aren't\": \"are not\",\n    \"can't\": \"cannot\",\n    \"'cause\": \"because\",\n    \"could've\": \"could have\",\n    \"couldn't\": \"could not\",\n    \"didn't\": \"did not\",\n    \"doesn't\": \"does not\",\n    \"don't\": \"do not\",\n    \"hadn't\": \"had not\",\n    \"hasn't\": \"has not\",\n    \"haven't\": \"have not\",\n    \"he'd\": \"he would\",\n    \"he'll\": \"he will\",\n    \"he's\": \"he is\",\n    \"how'd\": \"how did\",\n    \"how'd'y\": \"how do you\",\n    \"how'll\": \"how will\",\n    \"how's\": \"how is\",\n    \"I'd\": \"I would\",\n    \"I'd've\": \"I would have\",\n    \"I'll\": \"I will\",\n    \"I'll've\": \"I will have\",\n    \"I'm\": \"I am\",\n    \"I've\": \"I have\",\n    \"i'd\": \"i would\",\n    \"i'd've\": \"i would have\",\n    \"i'll\": \"i will\",\n    \"i'll've\": \"i will have\",\n    \"i'm\": \"i am\",\n    \"i've\": \"i have\",\n    \"isn't\": \"is not\",\n    \"it'd\": \"it would\",\n    \"it'd've\": \"it would have\",\n    \"it'll\": \"it will\",\n    \"it'll've\": \"it will have\",\n    \"it's\": \"it is\",\n    \"let's\": \"let us\",\n    \"ma'am\": \"madam\",\n    \"mayn't\": \"may not\",\n    \"might've\": \"might have\",\n    \"mightn't\": \"might not\",\n    \"mightn't've\": \"might not have\",\n    \"must've\": \"must have\",\n    \"mustn't\": \"must not\",\n    \"mustn't've\": \"must not have\",\n    \"needn't\": \"need not\",\n    \"needn't've\": \"need not have\",\n    \"o'clock\": \"of the clock\",\n    \"oughtn't\": \"ought not\",\n    \"oughtn't've\": \"ought not have\",\n    \"shan't\": \"shall not\",\n    \"sha'n't\": \"shall not\",\n    \"shan't've\": \"shall not have\",\n    \"she'd\": \"she would\",\n    \"she'd've\": \"she would have\",\n    \"she'll\": \"she will\",\n    \"she'll've\": \"she will have\",\n    \"she's\": \"she is\",\n    \"should've\": \"should have\",\n    \"shouldn't\": \"should not\",\n    \"shouldn't've\": \"should not have\",\n    \"so've\": \"so have\",\n    \"so's\": \"so as\",\n    \"this's\": \"this is\",\n    \"that'd\": \"that would\",\n    \"that'd've\": \"that would have\",\n    \"that's\": \"that is\",\n    \"there'd\": \"there would\",\n    \"there'd've\": \"there would have\",\n    \"there's\": \"there is\",\n    \"here's\": \"here is\",\n    \"they'd\": \"they would\",\n    \"they'd've\": \"they would have\",\n    \"they'll\": \"they will\",\n    \"they'll've\": \"they will have\",\n    \"they're\": \"they are\",\n    \"they've\": \"they have\",\n    \"to've\": \"to have\",\n    \"wasn't\": \"was not\",\n    \"we'd\": \"we would\",\n    \"we'd've\": \"we would have\",\n    \"we'll\": \"we will\",\n    \"we'll've\": \"we will have\",\n    \"we're\": \"we are\",\n    \"we've\": \"we have\",\n    \"weren't\": \"were not\",\n    \"what'll\": \"what will\",\n    \"what'll've\": \"what will have\",\n    \"what're\": \"what are\",\n    \"what's\": \"what is\",\n    \"what've\": \"what have\",\n    \"when's\": \"when is\",\n    \"when've\": \"when have\",\n    \"where'd\": \"where did\",\n    \"where's\": \"where is\",\n    \"where've\": \"where have\",\n    \"who'll\": \"who will\",\n    \"who'll've\": \"who will have\",\n    \"who's\": \"who is\",\n    \"who've\": \"who have\",\n    \"why's\": \"why is\",\n    \"why've\": \"why have\",\n    \"will've\": \"will have\",\n    \"won't\": \"will not\",\n    \"won't've\": \"will not have\",\n    \"would've\": \"would have\",\n    \"wouldn't\": \"would not\",\n    \"wouldn't've\": \"would not have\",\n    \"y'all\": \"you all\",\n    \"y'all'd\": \"you all would\",\n    \"y'all'd've\": \"you all would have\",\n    \"y'all're\": \"you all are\",\n    \"y'all've\": \"you all have\",\n    \"you'd\": \"you would\",\n    \"you'd've\": \"you would have\",\n    \"you'll\": \"you will\",\n    \"you'll've\": \"you will have\",\n    \"you're\": \"you are\",\n    \"you've\": \"you have\",\n    \"who'd\": \"who would\",\n    \"who're\": \"who are\",\n    \"'re\": \" are\",\n    \"tryin'\": \"trying\",\n    \"doesn'\": \"does not\",\n    'howdo': 'how do',\n    'whatare': 'what are',\n    'howcan': 'how can',\n    'howmuch': 'how much',\n    'howmany': 'how many',\n    'whydo': 'why do',\n    'doI': 'do I',\n    'theBest': 'the best',\n    'howdoes': 'how does',\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"5dceeab1a8b45bfe1aaa3259a6845fa7e75c9c24"},"cell_type":"code","source":"spells = {\n    'colour': 'color',\n    'centre': 'center',\n    'favourite': 'favorite',\n    'travelling': 'traveling',\n    'counselling': 'counseling',\n    'theatre': 'theater',\n    'cancelled': 'canceled',\n    'labour': 'labor',\n    'organisation': 'organization',\n    'wwii': 'world war 2',\n    'citicise': 'criticize',\n    'youtu.be': 'youtube',\n    'youtu ': 'youtube ',\n    'qoura': 'quora',\n    'sallary': 'salary',\n    'Whta': 'what',\n    'whta': 'what',\n    'narcisist': 'narcissist',\n    'mastrubation': 'masturbation',\n    'mastrubate': 'masturbate',\n    \"mastrubating\": 'masturbating',\n    'pennis': 'penis',\n    'Etherium': 'ethereum',\n    'etherium': 'ethereum',\n    'narcissit': 'narcissist',\n    'bigdata': 'big data',\n    '2k17': '2017',\n    '2k18': '2018',\n    'qouta': 'quota',\n    'exboyfriend': 'ex boyfriend',\n    'exgirlfriend': 'ex girlfriend',\n    'airhostess': 'air hostess',\n    \"whst\": 'what',\n    'watsapp': 'whatsapp',\n    'demonitisation': 'demonetization',\n    'demonitization': 'demonetization',\n    'demonetisation': 'demonetization',\n    'quorans': 'quora user',\n    'quoran': 'quora user',\n    'pokémon': 'pokemon',\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"9ee11b84cae015ab244c4322eef88b86c09a80bc"},"cell_type":"code","source":"@timer\ndef load_data(datadir):\n    train_df = pd.read_csv(datadir / 'train.csv')\n    test_df = pd.read_csv(datadir / 'test.csv')\n    print(\"Train shape : \", train_df.shape)\n    print(\"Test shape : \", test_df.shape)\n    return train_df, test_df\n\ndef clean(df):\n    df = clean_lower(df)\n    df = clean_unicode(df)\n    df = clean_math(df)\n    df = clean_abbreviation(df, abbreviations)\n    df = clean_spells(df, spells)\n    df = clean_language(df)\n    df = clean_puncts(df, puncts)\n    df = clean_space(df)\n    return df\n\ndef clean_unicode(df):\n    codes = ['\\x7f', '\\u200b', '\\xa0', '\\ufeff', '\\u200e', '\\u202a', '\\u202c', '\\u2060', '\\uf0d8', '\\ue019', '\\uf02d', '\\u200f', '\\u2061', '\\ue01b']\n    df[\"question_text\"] = df[\"question_text\"].apply(lambda x: _clean_unicode(x, codes))\n    return df\n\ndef _clean_unicode(x, codes):\n    for u in codes:\n        if u in x:\n            x = x.replace(u, '')\n    return x\n\ndef clean_language(df):\n    langs1 = r'[\\p{Katakana}\\p{Hiragana}\\p{Han}]' # regex\n    langs2 = r'[ஆய்தஎழுத்துஆயுதஎழுத்துशुषछछशुषدوउसशुष북한내제តើបងប្អូនមានមធ្យបាយអ្វីខ្លះដើម្បីរកឃើញឯកសារអំពីប្រវត្តិស្ត្រនៃប្រាសាទអង្គរវट्टरौरआदસંઘરાજ્યपीतऊनअहএকটিবাড়িএকটিখামারএরঅধীনেপদেরবাছাইপরীক্ষাএরপ্রশ্নওউত্তরসহকোথায়পেতেপারিص、。Емелядуракلكلمقاممقال수능ί서로가를행복하게기乡국고등학교는몇시간업니《》싱관없어나이रचा키کپڤ」मिलगईकलेजेकोठंडकऋॠऌॡर]'\n    compiled_langs1 = regex.compile(langs1)\n    compiled_langs2 = re.compile(langs2)\n    df['question_text'] = df['question_text'].apply(lambda x: _clean_language(x, compiled_langs1))\n    df['question_text'] = df['question_text'].apply(lambda x: _clean_language(x, compiled_langs2))\n    return df\n\ndef _clean_language(x, compiled_re):\n    return compiled_re.sub(' <lang> ', x)\n\ndef clean_bitcoin(df, bitcoins):\n    compiled_bitcoins = re.compile('(%s)' % '|'.join(bitcoins))\n    df['question_text'] = df['question_text'].apply(lambda x: _clean_language(x, compiled_bitcoins))\n    return df\n\ndef _clean_bitcoin(x, compiled_re):\n    return compiled_re.sub(' Bitcoin ', x)\n\ndef clean_math(df):\n    math_puncts = 'θπα÷⁴≠β²¾∫≥⇒¬∠＝∑Φ√½¼'\n    math_puncts_long = [r'\\\\frac', r'\\[math\\]', r'\\[/math\\]', r'\\\\lim']\n    compiled_math = re.compile('(%s)' % '|'.join(math_puncts))\n    compiled_math_long = re.compile('(%s)' % '|'.join(math_puncts_long))\n    df['question_text'] = df['question_text'].apply(lambda x: _clean_math(x, compiled_math_long))\n    df['question_text'] = df['question_text'].apply(lambda x: _clean_math(x, compiled_math))\n    return df\n\ndef _clean_math(x, compiled_re):\n    return compiled_re.sub(' <math> ', x)\n\ndef clean_lower(df):\n    df[\"question_text\"] = df[\"question_text\"].apply(lambda x: x.lower())\n    return df\n\ndef clean_puncts(df, puncts):\n    df['question_text'] = df['question_text'].apply(lambda x: _clean_puncts(x, puncts))\n    return df\n    \ndef _clean_puncts(x, puncts):\n    x = str(x)\n    # added space around puncts after replace\n    for punct in puncts:\n        if punct in x:\n            x = x.replace(punct, f' {punct} ')\n    return x\n\ndef clean_bad_case_words(df, bad_case_words):\n    compiled_bad_case_words = re.compile('(%s)' % '|'.join(bad_case_words.keys()))\n    def replace(match):\n        return bad_case_words[match.group(0)]\n    df['question_text'] = df[\"question_text\"].apply(\n        lambda x: _clean_bad_case_words(x, compiled_bad_case_words, replace)\n    )\n    return df\n    \ndef _clean_bad_case_words(x, compiled_re, replace):\n    return compiled_re.sub(replace, x)\n\ndef clean_spells(df, spells):\n    compiled_spells = re.compile('(%s)' % '|'.join(spells.keys()))\n    def replace(match):\n        return spells[match.group(0)]\n    df['question_text'] = df[\"question_text\"].apply(\n        lambda x: _clean_spells(x, compiled_spells, replace)\n    )\n    return df\n    \ndef _clean_spells(x, compiled_re, replace):\n    return compiled_re.sub(replace, x)\n\ndef clean_abbreviation(df, abbreviations):\n    compiled_abbreviation = re.compile('(%s)' % '|'.join(abbreviations.keys()))\n    def replace(match):\n        return abbreviations[match.group(0)]\n    df['question_text'] = df[\"question_text\"].apply(\n        lambda x: _clean_abreviation(x, compiled_abbreviation, replace)\n    )\n    return df\n    \ndef _clean_abreviation(x, compiled_re, replace):\n    return compiled_re.sub(replace, x)\n\ndef clean_space(df):\n    compiled_re = re.compile(r\"\\s+\")\n    df['question_text'] = df[\"question_text\"].apply(lambda x: _clean_space(x, compiled_re))\n    return df\n\ndef _clean_space(x, compiled_re):\n    return compiled_re.sub(\" \", x)\n        \ndef prepare_tokenizer(texts, max_words):\n    tokenizer = Tokenizer(num_words=max_words, filters='', oov_token='<unk>')\n    tokenizer.fit_on_texts(list(texts))\n    return tokenizer\n\ndef tokenize_and_padding(texts, tokenizer, max_length):\n    texts = tokenizer.texts_to_sequences(texts)\n    texts = pad_sequences(texts, maxlen=max_length)\n    return texts\n\ndef get_all_vocabs(texts):\n    sentences = texts.apply(lambda x: x.split()).values\n    vocab = {}\n    for sentence in sentences:\n        for word in sentence:\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n    return vocab","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"175f677f3ec60cf59a63bea09d2af715094eb5e2"},"cell_type":"code","source":"def add_features(df):\n    df['question_text'] = df['question_text'].apply(lambda x:str(x))\n    df['lower_question_text'] = df['question_text'].apply(lambda x: x.lower())\n    df['total_length'] = df['question_text'].apply(len)\n    df['capitals'] = df['question_text'].apply(lambda comment: sum(1 for c in comment if c.isupper()))\n    df['caps_vs_length'] = df.apply(lambda row: float(row['capitals'])/float(row['total_length']), axis=1)\n    df['num_words'] = df['question_text'].str.count('\\S+')\n    df['num_unique_words'] = df['question_text'].apply(lambda comment: len(set(comment.split())))\n    df['words_vs_unique'] = df['num_unique_words'] / df['num_words'] \n    return df[['caps_vs_length', 'words_vs_unique', 'num_words']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"353b0a23684fe89ac67a380de0b21ec1ab698f7d"},"cell_type":"code","source":"class Embeddings(nn.Module):\n    \n    def __init__(self, config: Config, tokenizer, all_vocabs, embedding_weights = None):\n        super(Embeddings, self).__init__()\n        \n        self.embedding_map = {\n            'fasttext': self._load_fasttext,\n            'glove': self._load_glove,\n            'paragram': self._load_paragram\n        }\n        self.c = config\n        self.tokenizer = tokenizer\n        self.all_vocabs = all_vocabs\n        \n        if embedding_weights is None:\n            embedding_weights = self._load_embeddings(self.c.embeddings)\n            \n        self.original_embedding_weights = embedding_weights\n        self.embeddings = nn.Embedding(self.c.vocab_size + 1, self.c.embedding_size, padding_idx=0)\n        self.embeddings.weight = nn.Parameter(embedding_weights)\n        self.embeddings.weight.requires_grad = False\n        self.embedding_dropout = nn.Dropout2d(self.c.embedding_dropout)\n        \n    def forward(self, x):\n        embedding = self.embeddings(x)\n        if self.training:\n            embedding += torch.randn_like(embedding) * self.c.embedding_noise_var\n        return self.embedding_dropout(embedding.permute(0, 2, 1)).permute(0, 2, 1)\n    \n    def reset_weights(self):\n        self.embeddings.weight = nn.Parameter(self.original_embedding_weights)\n        self.embeddings.weight.requires_grad = False\n    \n    def _load_embeddings(self, embedding_list: list):\n        embedding_weights = np.zeros((self.c.vocab_size, self.c.embedding_size))\n        pool = Pool(num_cores)\n        embedding_weights = np.mean(pool.map(self._load_an_embedding, embedding_list), 0)\n        pool.close()\n        pool.join()\n        return torch.tensor(embedding_weights, dtype=torch.float32)\n\n    def _load_an_embedding(self, emb):\n        return self.embedding_map[emb](self.tokenizer.word_index)\n        \n    def _get_embeddings_pair(self, word, *arr): \n        return word, np.asarray(arr, dtype='float32')\n        \n    def _make_embeddings(self, embeddings_index, word_index, emb_mean, emb_std):\n        nb_words = min(self.c.vocab_size, len(word_index))\n        embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, self.c.embedding_size))\n        embedding_matrix[0] = np.zeros(self.c.embedding_size)\n        for word, i in word_index.items():\n            if i >= self.c.vocab_size:\n                continue\n            embedding_vector = embeddings_index.get(word)\n            if embedding_vector is not None:\n                embedding_matrix[i] = embedding_vector\n\n        return embedding_matrix\n    \n    def _load_glove(self, word_index):\n        print('loading glove')\n        filepath = self.c.datadir / 'embeddings/glove.840B.300d/glove.840B.300d.txt'\n        embeddings_index = dict(\n            self._get_embeddings_pair(*o.split(\" \"))\n            for o in open(filepath)\n            if o.split(\" \")[0] in word_index\n        )\n        emb_mean, emb_std = -0.005838499, 0.48782197\n        return self._make_embeddings(embeddings_index, word_index, emb_mean, emb_std)\n    \n    def _load_fasttext(self, word_index):    \n        print('loading fasttext')\n        filepath = self.c.datadir / 'embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n        embeddings_index = dict(\n            self._get_embeddings_pair(*o.split(\" \"))\n            for o in open(filepath)\n            if len(o) > 100 and o.split(\" \")[0] in word_index\n        )\n        emb_mean, emb_std = -0.0033469985, 0.109855495\n        return self._make_embeddings(embeddings_index, word_index, emb_mean, emb_std)\n\n    def _load_paragram(self, word_index):\n        print('loading paragram')\n        filepath = self.c.datadir / 'embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n        embeddings_index = dict(\n            self._get_embeddings_pair(*o.split(\" \"))\n            for o in open(filepath, encoding=\"utf8\", errors='ignore')\n            if len(o) > 100 and o.split(\" \")[0] in word_index\n        )\n        emb_mean, emb_std = -0.0053247833, 0.49346462\n        return self._make_embeddings(embeddings_index, word_index, emb_mean, emb_std)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a13d20cddd55b6315e19c00644c093ecc7a54b5a"},"cell_type":"code","source":"num_cores = 2\n@timer\ndef df_parallelize_run(df, func, num_cores=2):\n    df_split = np.array_split(df, num_cores)\n    pool = Pool(num_cores)\n    df = pd.concat(pool.map(func, df_split))\n    pool.close()\n    pool.join()\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f5934c5b5c379da1e788d1f85106a328fe1f846a"},"cell_type":"code","source":"train_df, test_df = load_data(c.datadir)\ntrain_features = df_parallelize_run(train_df, add_features)\ntest_features = df_parallelize_run(test_df, add_features)\ntrain_df = df_parallelize_run(train_df, clean)\ntest_df = df_parallelize_run(test_df, clean)\ntrain_x, train_y = train_df['question_text'].values, train_df['target'].values\ntest_x = test_df['question_text'].values\ntokenizer = prepare_tokenizer(train_x, c.vocab_size)\ntrain_x = tokenize_and_padding(train_x, tokenizer, c.max_length)\ntest_x = tokenize_and_padding(test_x, tokenizer, c.max_length)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"0a902a6466ea112ba05980738c211b406b01300c"},"cell_type":"code","source":"ss = StandardScaler()\nss.fit(np.vstack((train_features, test_features)))\ntrain_features = ss.transform(train_features)\ntest_features = ss.transform(test_features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"125ca9f2b0d1bf416bd7e4be033b4549dfd9ac04"},"cell_type":"code","source":"start = time.time()\nall_vocabs = get_all_vocabs(train_df['question_text'])\nprint('all_vocabs: ', len(all_vocabs))\nembeddings = Embeddings(c, tokenizer, all_vocabs)\nprint(time.time() - start)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"61b662563ed5550ae333897a9b455dbe91a3f094"},"cell_type":"code","source":"class GRULayer(nn.Module):\n    def __init__(self, input_size, hidden_size, num_layers, dropout_rate):\n        super(GRULayer, self).__init__()\n        \n        self.gru = nn.GRU(input_size=input_size,\n                          hidden_size=hidden_size,\n                          num_layers=num_layers,\n                          bias=False,\n                          bidirectional=True,\n                          batch_first=True)\n        self.dropout = nn.Dropout(dropout_rate)\n        \n        self.init_weights()\n        \n    def init_weights(self):\n        ih = (param.data for name, param in self.named_parameters() if 'weight_ih' in name)\n        hh = (param.data for name, param in self.named_parameters() if 'weight_hh' in name)\n        b = (param.data for name, param in self.named_parameters() if 'bias' in name)\n        for k in ih:\n            nn.init.xavier_uniform_(k)\n        for k in hh:\n            nn.init.orthogonal_(k)\n        for k in b:\n            nn.init.constant_(k, 0)\n\n    def forward(self, x):\n        gru_outputs, gru_state = self.gru(x)\n        return self.dropout(gru_outputs), gru_state\n            ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"3f715453058dac059a25e1b540f2e8900022b933"},"cell_type":"code","source":"class LSTMLayer(nn.Module):\n    def __init__(self, input_size, hidden_size, num_layers, dropout_rate):\n        super(LSTMLayer, self).__init__()\n        \n        self.lstm = nn.LSTM(input_size=input_size,\n                            hidden_size=hidden_size,\n                            num_layers=num_layers,\n                            bias=False,\n                            bidirectional=True,\n                            batch_first=True)\n        self.dropout = nn.Dropout(dropout_rate)\n        \n        self.init_weights()\n        \n    def init_weights(self):\n        ih = (param.data for name, param in self.named_parameters() if 'weight_ih' in name)\n        hh = (param.data for name, param in self.named_parameters() if 'weight_hh' in name)\n        b = (param.data for name, param in self.named_parameters() if 'bias' in name)\n        for k in ih:\n            nn.init.xavier_uniform_(k)\n        for k in hh:\n            nn.init.orthogonal_(k)\n        for k in b:\n            nn.init.constant_(k, 0)\n\n    def forward(self, x):\n        lstm_outputs, (lstm_states, _) = self.lstm(x)\n        return self.dropout(lstm_outputs), lstm_states","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"7f1a543a38b797fa2be568e0685f12ed889129e9"},"cell_type":"code","source":"# https://github.com/anandsaha/pytorch.cyclic.learning.rate/blob/master/cls.py\nclass CyclicLR(object):\n    def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,\n                 step_size=2000, mode='triangular', gamma=1.,\n                 scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):\n\n        if not isinstance(optimizer, torch.optim.Optimizer):\n            raise TypeError('{} is not an Optimizer'.format(\n                type(optimizer).__name__))\n        self.optimizer = optimizer\n\n        if isinstance(base_lr, list) or isinstance(base_lr, tuple):\n            if len(base_lr) != len(optimizer.param_groups):\n                raise ValueError(\"expected {} base_lr, got {}\".format(\n                    len(optimizer.param_groups), len(base_lr)))\n            self.base_lrs = list(base_lr)\n        else:\n            self.base_lrs = [base_lr] * len(optimizer.param_groups)\n\n        if isinstance(max_lr, list) or isinstance(max_lr, tuple):\n            if len(max_lr) != len(optimizer.param_groups):\n                raise ValueError(\"expected {} max_lr, got {}\".format(\n                    len(optimizer.param_groups), len(max_lr)))\n            self.max_lrs = list(max_lr)\n        else:\n            self.max_lrs = [max_lr] * len(optimizer.param_groups)\n\n        self.step_size = step_size\n\n        if mode not in ['triangular', 'triangular2', 'exp_range'] \\\n                and scale_fn is None:\n            raise ValueError('mode is invalid and scale_fn is None')\n\n        self.mode = mode\n        self.gamma = gamma\n\n        if scale_fn is None:\n            if self.mode == 'triangular':\n                self.scale_fn = self._triangular_scale_fn\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = self._triangular2_scale_fn\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = self._exp_range_scale_fn\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n\n        self.batch_step(last_batch_iteration + 1)\n        self.last_batch_iteration = last_batch_iteration\n\n    def batch_step(self, batch_iteration=None):\n        if batch_iteration is None:\n            batch_iteration = self.last_batch_iteration + 1\n        self.last_batch_iteration = batch_iteration\n        for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):\n            param_group['lr'] = lr\n\n    def _triangular_scale_fn(self, x):\n        return 1.\n\n    def _triangular2_scale_fn(self, x):\n        return 1 / (2. ** (x - 1))\n\n    def _exp_range_scale_fn(self, x):\n        return self.gamma**(x)\n\n    def get_lr(self):\n        step_size = float(self.step_size)\n        cycle = np.floor(1 + self.last_batch_iteration / (2 * step_size))\n        x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)\n\n        lrs = []\n        param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)\n        for param_group, base_lr, max_lr in param_lrs:\n            base_height = (max_lr - base_lr) * np.maximum(0, (1 - x))\n            if self.scale_mode == 'cycle':\n                lr = base_lr + base_height * self.scale_fn(cycle)\n            else:\n                lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)\n            lrs.append(lr)\n        return lrs","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"0ded3151e8a5a8884358e050b8c14f21287229a1"},"cell_type":"code","source":"class SimpleRNN(nn.Module):\n    def __init__(self, config: Config, embeddings):\n        super(SimpleRNN, self).__init__()\n        self.c = config\n        \n        self.embedding = embeddings\n        self.lstm = LSTMLayer(input_size=self.c.embedding_size,\n                              hidden_size=self.c.hidden_size,\n                              num_layers=self.c.num_layers,\n                              dropout_rate=self.c.layer_dropout)\n        self.gru = GRULayer(input_size=self.c.hidden_size*2,\n                            hidden_size=self.c.hidden_size,\n                            num_layers=self.c.num_gru_layers,\n                            dropout_rate=self.c.layer_dropout)\n        \n        self.cell_dropout = nn.Dropout(self.c.layer_dropout)\n        self.linear = nn.Linear(self.c.dense_size[0], self.c.dense_size[1])\n        self.batch_norm = torch.nn.BatchNorm1d(self.c.dense_size[1])\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(self.c.layer_dropout)\n        self.out = nn.Linear(self.c.dense_size[1], self.c.output_size)\n        \n    def forward(self, x, features):\n        h_embedding = self.embedding(x)\n        o_lstm, h_lstm = self.lstm(h_embedding)\n        o_gru, h_gru = self.gru(o_lstm)\n        \n        avg_pool = torch.mean(o_gru, 1)\n        max_pool, _ = torch.max(o_gru, 1)\n        \n        h_lstm = self.cell_dropout(torch.cat(h_lstm.split(1, 0), -1).squeeze(0))\n        h_gru = self.cell_dropout(torch.cat(h_gru.split(1, 0), -1).squeeze(0))\n\n        concat = torch.cat([h_lstm, h_gru, avg_pool, max_pool, features], 1)\n        concat = self.linear(concat)\n        concat = self.batch_norm(concat)\n        concat = self.relu(concat)\n        concat = self.dropout(concat)\n        out = self.out(concat)\n        \n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d4266e902fd607371dee028600ff3411848cb826"},"cell_type":"code","source":"def sigmoid(x):\n    return 1 / (1 + np.exp(-x))\n\n@timer\ndef threshold_search(y_true, y_proba, plot=False):\n    precision, recall, thresholds = precision_recall_curve(y_true, y_proba)\n    thresholds = np.append(thresholds, 1.001) \n    F = 2 / (1/precision + 1/recall)\n    best_score = np.max(F)\n    best_th = thresholds[np.argmax(F)]\n    if plot:\n        plt.plot(thresholds, F, '-b')\n        plt.plot([best_th], [best_score], '*r')\n        plt.show()\n    search_result = {'threshold': best_th , 'f1': best_score}\n    return search_result ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"c29014ec263ef47df87a8997f5a415403aafcfcb"},"cell_type":"code","source":"def cut_length(data, mask):\n    max_length = data.shape[1]\n    transposed = torch.transpose(data, 1, 0)\n    res = (transposed == mask).all(1)\n    for i, r in enumerate(res):\n        if r == 0:\n            break\n    data = data[:, -(max_length - i):]\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"23c10c8c2167826da7b8c890cd305c9eb8221910"},"cell_type":"code","source":"@timer\ndef training(train_x, train_f, train_y, test_x, test_f, c, embeddings):\n    splits = list(StratifiedKFold(n_splits=c.num_cv_splits, shuffle=True, random_state=SEED).split(train_x, train_y))\n    x_test_cuda = torch.tensor(test_x, dtype=torch.long).cuda(cuda_idx)\n    f_test_cuda = torch.tensor(test_f, dtype=torch.float).cuda(cuda_idx)\n    test = torch.utils.data.TensorDataset(x_test_cuda, f_test_cuda)\n    test_loader = torch.utils.data.DataLoader(test, batch_size=c.test_batch_size, shuffle=False)\n    train_preds = np.zeros((len(train_x)))\n    test_preds = np.zeros((len(test_x)))\n    \n    mask = torch.zeros((c.max_length, 1), dtype=torch.long).cuda(cuda_idx)\n    \n    for i, (train_idx, valid_idx) in enumerate(splits):\n        x_train_fold = torch.tensor(train_x[train_idx], dtype=torch.long).cuda(cuda_idx)\n        f_train_fold = torch.tensor(train_f[train_idx], dtype=torch.float).cuda(cuda_idx)\n        y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32).cuda(cuda_idx)\n        x_val_fold = torch.tensor(train_x[valid_idx], dtype=torch.long).cuda(cuda_idx)\n        f_val_fold = torch.tensor(train_f[valid_idx], dtype=torch.float).cuda(cuda_idx)\n        y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32).cuda(cuda_idx)\n\n        embeddings.reset_weights()\n        model = SimpleRNN(c, embeddings)\n        model.cuda(cuda_idx)\n        model.embedding.original_embedding_weights.cuda(cuda_idx)\n\n        loss_fn = torch.nn.BCEWithLogitsLoss()\n        optimizer = torch.optim.Adam(model.parameters(), lr=c.learning_rate)\n        scheduler = CyclicLR(optimizer, base_lr=c.learning_rate, max_lr=c.max_learning_rate,\n                             step_size=c.clr_step_size, mode='exp_range', gamma=0.9999)\n\n        train = torch.utils.data.TensorDataset(x_train_fold, f_train_fold, y_train_fold)\n        valid = torch.utils.data.TensorDataset(x_val_fold, f_val_fold, y_val_fold)\n        valid_loader = torch.utils.data.DataLoader(valid, batch_size=c.test_batch_size, shuffle=False)\n\n        print(f'Fold {i + 1}')\n\n        for epoch in range(c.num_epochs):\n            train_loader = torch.utils.data.DataLoader(train, batch_size=c.batch_size[epoch], shuffle=True)\n            start_time = time.time()\n            if epoch >= c.num_epochs - c.num_embedding_learn_epochs:\n                model.embedding.embeddings.weight.requires_grad = True\n\n            model.train()\n            avg_loss = 0.\n            for x_batch, f_batch, y_batch in tqdm(train_loader, disable=True):\n                x_batch = cut_length(x_batch, mask)\n                y_pred = model(x_batch, f_batch)\n                scheduler.batch_step()\n                loss = loss_fn(y_pred, y_batch)\n                optimizer.zero_grad()\n                loss.backward()\n                nn.utils.clip_grad_norm_(model.parameters(), c.clip_grad)\n                optimizer.step()\n                avg_loss += loss.item() / len(train_loader)\n\n            model.eval()\n            valid_preds_fold = np.zeros((x_val_fold.size(0)))\n            test_preds_fold = np.zeros(len(test_x))\n            avg_val_loss = 0.\n\n            # validation prediction\n            for i, (x_batch, f_batch, y_batch) in enumerate(valid_loader):\n                x_batch = cut_length(x_batch, mask)\n                y_pred = model(x_batch, f_batch).detach()\n                avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n                valid_preds_fold[i * c.test_batch_size:(i+1) * c.test_batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n\n            for ps in optimizer.param_groups:\n                current_learning_rate = ps['lr']\n            elapsed_time = time.time() - start_time \n            print('Epoch {}/{}  loss={:.4f}  val_loss={:.4f}  time={:.2f}s  lr={:.6f}'.format(\n                epoch + 1, c.num_epochs, avg_loss, avg_val_loss, elapsed_time, current_learning_rate))\n\n        # test prediction\n        for i, (x_batch, f_batch) in enumerate(test_loader):\n            x_batch = cut_length(x_batch, mask)\n            y_pred = model(x_batch, f_batch).detach()\n\n            test_preds_fold[i * c.test_batch_size:(i+1) * c.test_batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n\n        train_preds[valid_idx] = valid_preds_fold\n        test_preds += test_preds_fold / len(splits)\n    return train_preds, test_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"7796e10a4469986ac08e894f6f4b26b5d190852e"},"cell_type":"code","source":"train_preds, test_preds = training(train_x, train_features, train_y, test_x, test_features, c, embeddings)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"aebb12981d32d1c289d176fe685681663232f829"},"cell_type":"code","source":"search_result = threshold_search(train_y, train_preds)\nsearch_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"921c90ee8a15cf793bc24331e9e6d76a57c09409"},"cell_type":"code","source":"all_elapsed = time.time() - all_start\nall_elapsed","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"9aeb93a0e922142e6d82af5a31ca4d08e34aef04"},"cell_type":"code","source":"# kernel only\nsubmission = test_df[['qid']].copy()\nsubmission['prediction'] = (test_preds > search_result['threshold']).astype(int)\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}