{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import time\nimport random\nimport pandas as pd\nimport numpy as np\nimport gc\nimport re\nimport torch\nfrom torchtext import data\nimport spacy\nfrom tqdm import tqdm_notebook, tnrange\nfrom tqdm.auto import tqdm\n\ntqdm.pandas(desc='Progress')\nfrom collections import Counter\nfrom textblob import TextBlob\nfrom nltk import word_tokenize\nfrom nltk.corpus import stopwords, wordnet\nfrom nltk.tag import pos_tag\n\nfrom torch import normal\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\nfrom torch.nn import Parameter\nfrom torch.autograd import Variable\nfrom torchtext.data import Example\nfrom sklearn.metrics import f1_score\nimport torchtext\nimport os \n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\n# cross validation and metrics\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import f1_score\nfrom torch.optim.optimizer import Optimizer\nfrom unidecode import unidecode","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"embed_size = 300 # how big is each word vector\nmax_features = 120000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 70 # max number of words in a question to use\nbatch_size = 512 # how many samples to process at once\nn_epochs = 5 # how many times to iterate over all samples\nn_splits = 5 # Number of K-fold Splits\n\nSEED = 1029","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"212ca4dc7fc5abfde5e6c132360821ae9000d19b"},"cell_type":"code","source":"def seed_everything(seed=1029):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\nseed_everything()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"027a86615ece47914032f919af8c6573d8238676"},"cell_type":"code","source":"## FUNCTIONS TAKEN FROM https://www.kaggle.com/gmhost/gru-capsule\n\ndef load_glove(word_index):\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')[:300]\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n    \n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = -0.005838499,0.48782197\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n            \n    return embedding_matrix \n    \ndef load_fasttext(word_index):    \n    EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n\n    return embedding_matrix\n\ndef load_para(word_index):\n    EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = -0.0053247833,0.49346462\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n    \n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b2028d9f352cde4532d4b7a853a4611996f27d4"},"cell_type":"code","source":"df_train = pd.read_csv(\"../input/train.csv\")\ndf_test = pd.read_csv(\"../input/test.csv\")\ndf = pd.concat([df_train ,df_test],sort=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ec5206b792bc7a0ca7c1bad54d18139bc17482c9"},"cell_type":"code","source":"def build_vocab(texts):\n    sentences = texts.apply(lambda x: x.split()).values\n    vocab = {}\n    for sentence in sentences:\n        for word in sentence:\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n    return vocab\nvocab = build_vocab(df['question_text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7def06ded2f51e904adb0835d22374534f43e657"},"cell_type":"code","source":"sin = len(df_train[df_train[\"target\"]==0])\ninsin = len(df_train[df_train[\"target\"]==1])\npersin = (sin/(sin+insin))*100\nperinsin = (insin/(sin+insin))*100            \nprint(\"# Sincere questions: {:,}({:.2f}%) and # Insincere questions: {:,}({:.2f}%)\".format(sin,persin,insin,perinsin))\n# print(\"Sinsere:{}% Insincere: {}%\".format(round(persin,2),round(perinsin,2)))\nprint(\"# Test samples: {:,}({:.3f}% of train samples)\".format(len(df_test),len(df_test)/len(df_train)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a80abd1f9c3a33b23275f4c20a7d86e35181a2c"},"cell_type":"code","source":"def build_vocab(texts):\n    sentences = texts.apply(lambda x: x.split()).values\n    vocab = {}\n    for sentence in sentences:\n        for word in sentence:\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n    return vocab\n\ndef known_contractions(embed):\n    known = []\n    for contract in contraction_mapping:\n        if contract in embed:\n            known.append(contract)\n    return known\n\ndef clean_contractions(text, mapping):\n    specials = [\"’\", \"‘\", \"´\", \"`\"]\n    for s in specials:\n        if s in text:\n            text = text.replace(s, \"'\")\n    text = ' '.join([mapping[t] if t in mapping else t for t in text.split(\" \")])\n    return text\ndef correct_spelling(x, dic):\n    for word in dic.keys():\n        if word in dic:\n            x = x.replace(word, dic[word])\n    return x\ndef unknown_punct(embed, punct):\n    unknown = ''\n    for p in punct:\n        if p not in embed:\n            unknown += p\n            unknown += ' '\n    return unknown\n\ndef clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x\n\ndef clean_special_chars(text, punct, mapping):\n    for p in mapping:\n        if p in text:\n            text = text.replace(p, mapping[p])\n    \n    for p in punct:\n        if p in text:\n            text = text.replace(p, f' {p} ')\n    \n    specials = {'\\u200b': ' ', '…': ' ... ', '\\ufeff': '', 'करना': '', 'है': ''}  # Other special characters that I have to deal with in last\n    for s in specials:\n        if s in text:\n            text = text.replace(s, specials[s])\n    \n    return text\n\ndef replace(word, pos=None):\n    \"\"\" Creates a set of all antonyms for the word and if there is only one antonym, it returns it \"\"\"\n    antonyms = set()\n    for syn in wordnet.synsets(word, pos=pos):\n        for lemma in syn.lemmas():\n            for antonym in lemma.antonyms():\n                antonyms.add(antonym.name())\n    if len(antonyms) == 1:\n        return antonyms.pop()\n    else:\n        return None\n\ndef replace_negations(text):\n    \"\"\" Finds \"not\" and antonym for the next word and if found, replaces not and the next word with the antonym \"\"\"\n    i, l = 0, len(text)\n    words = []\n    while i < l:\n        word = text[i]\n        if word == 'not' and i+1 < l:\n            ant = replace(text[i+1])\n            if ant:\n                words.append(ant)\n                i += 2\n                continue\n        words.append(word)\n        i += 1\n    return words\n\n\"\"\"\" def replace_elongated_word(word): \"\"\"\n\"\"\" Replaces an elongated word with its basic form, unless the word exists in the lexicon \"\"\"\n\n\"\"\" repeat_regexp = re.compile(r'(\\w*)(\\w)\\2(\\w*)')\n    repl = r'\\1\\2\\3'\n    if wordnet.synsets(word):\n        return word\n    repl_word = repeat_regexp.sub(repl, word)\n    if repl_word != word:      \n        return replaceElongated(repl_word)\n    else:       \n        return repl_word\n    \ndef replace_elongated(text):\n    final_tokens = []\n    tokens = word_tokenize(text)\n    for w in tokens:\n        final_tokens.append(replace_elongated_word(w))\n    text = \" \".join(final_tokens)\n    return text \"\"\"\n\ndef add_lower(embedding, vocab):\n    count = 0\n    for word in vocab:\n        if word in embedding and word.lower() not in embedding:  \n            embedding[word.lower()] = embedding[word]\n            count += 1\n    print(f\"Added {count} words to embedding\") ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4cbcfb5e9584156f820f853cd9c1abbd67a733f2"},"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n\ndef clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        if punct in x:\n            x = x.replace(punct, f' {punct} ')\n    return x\n\ndef clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x\n\nmispell_dict = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization', 'oher':'other', 'fuckin':'fucking', }\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n    return mispellings_re.sub(replace, text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"92fcf4746dbf1703f3e24db70ca1873384e3a323"},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\ndef add_features(df):\n    \n    df['question_text'] = df['question_text'].progress_apply(lambda x:str(x))\n    df['total_length'] = df['question_text'].progress_apply(len)\n    df['capitals'] = df['question_text'].progress_apply(lambda comment: sum(1 for c in comment if c.isupper()))\n    df['caps_vs_length'] = df.progress_apply(lambda row: float(row['capitals'])/float(row['total_length']),\n                                axis=1)\n    df['num_words'] = df.question_text.str.count('\\S+')\n    df['num_unique_words'] = df['question_text'].progress_apply(lambda comment: len(set(w for w in comment.split())))\n    df['words_vs_unique'] = df['num_unique_words'] / df['num_words']  \n    \n    # df['has_why_them'] = df['question_text'].apply(lambda x: float((('Why' in x) or ('why' in x) or ('Why do' in x) or ('why do' in x)) and (('Them' in x) or ('them' in x) or ('They' in x) or ('they' in x))))\n    # df['has_country'] = df['question_text'].apply(lambda x: float('Americans' in x or 'americans' in x  or 'Jew' in x or 'jew' in x or 'Arab' in x or 'arab' in x or 'Europe' in x or 'europe' in x or 'Canad' in x or 'canad' in x or 'Brit' in x or 'brit' in x or 'Indians' in x or 'indians' in x or 'indian people' in x))\n    # df['has_country_and_why'] = df['question_text'].apply(lambda x : float(('Americans' in x or 'americans' in x  or 'Jew' in x or 'jew' in x or 'Arab' in x or 'arab' in x or 'Europe' in x or 'europe' in x or 'Canad' in x or 'canad' in x or 'Brit' in x or 'brit' in x or 'Indians' in x or 'indians' in x or 'indian people' in x) and (('Why' in x) or ('why' in x) or ('Why do' in x) or ('why do' in x))))\n    # df['has_country_and_why_them'] = df['question_text'].apply(lambda x : float(('Americans' in x or 'americans' in x  or 'Jew' in x or 'jew' in x or 'Arab' in x or 'arab' in x or 'Europe' in x or 'europe' in x or 'Canad' in x or 'canad' in x or 'Brit' in x or 'brit' in x or 'Indians' in x or 'indians' in x or 'indian people' in x) and (('Why' in x) or ('why' in x) or ('Why do' in x) or ('why do' in x)) and (('Them' in x) or ('them' in x) or ('They' in x) or ('they' in x))))\n\n    return df\n\ndef load_and_prec():\n    train_df_original = pd.read_csv(\"../input/train.csv\")\n    test_df_original = pd.read_csv(\"../input/test.csv\")\n    print(\"Train shape : \",train_df_original.shape)\n    print(\"Test shape : \",test_df_original.shape)\n    \n    train_df = train_df_original\n    test_df = test_df_original\n    \n    # lower\n    train_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: x.lower())\n    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: x.lower())\n    \n    # Clean the text\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_text(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: clean_text(x))\n    \n    # Clean numbers\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: clean_numbers(x))\n    \n    # Clean spellings\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: replace_typical_misspell(x))\n    \n    ## fill up the missing values\n    train_X = train_df[\"question_text\"].fillna(\"_##_\").values\n    test_X = test_df[\"question_text\"].fillna(\"_##_\").values\n\n\n    \n    ###################### Add Features ###############################\n    #  https://github.com/wongchunghang/toxic-comment-challenge-lstm/blob/master/toxic_comment_9872_model.ipynb\n    train = add_features(train_df_original)\n    test = add_features(test_df_original)\n\n    features = train[['caps_vs_length', 'words_vs_unique']].fillna(0) # 'has_why_them', 'has_country', 'has_country_and_why_them', 'has_country_and_why'\n    test_features = test[['caps_vs_length', 'words_vs_unique']].fillna(0) # 'has_why_them', 'has_country', 'has_country_and_why_them', 'has_country_and_why'\n\n    ss = StandardScaler()\n    ss.fit(np.vstack((features, test_features)))\n    features = ss.transform(features)\n    test_features = ss.transform(test_features)\n    ###########################################################################\n    \n    # Remove negations and add antonyms\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: replace_negations(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: replace_negations(x))\n    \n    # Remove stopwords\n    # train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: remove_stopwords(x))\n    # test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: remove_stopwords(x))\n    \n    # Replace elongated words\n    # train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: replace_elongated(x))\n    # test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: replace_elongated(x))\n\n    ## Tokenize the sentences\n    tokenizer = Tokenizer(num_words=max_features)\n    tokenizer.fit_on_texts(list(train_X))\n    train_X = tokenizer.texts_to_sequences(train_X)\n    test_X = tokenizer.texts_to_sequences(test_X)\n\n    ## Pad the sentences \n    train_X = pad_sequences(train_X, maxlen=maxlen)\n    test_X = pad_sequences(test_X, maxlen=maxlen)\n\n    ## Get the target values\n    train_y = train_df['target'].values\n    \n#     # Splitting to training and a final test set    \n#     train_X, x_test_f, train_y, y_test_f = train_test_split(list(zip(train_X,features)), train_y, test_size=0.2, random_state=SEED)    \n#     train_X, features = zip(*train_X)\n#     x_test_f, features_t = zip(*x_test_f)    \n    \n    #shuffling the data\n    np.random.seed(SEED)\n    trn_idx = np.random.permutation(len(train_X))\n\n    train_X = train_X[trn_idx]\n    train_y = train_y[trn_idx]\n    \n    return train_X, test_X, train_y, features, test_features, tokenizer.word_index\n#     return train_X, test_X, train_y, x_test_f,y_test_f,features, test_features, features_t, tokenizer.word_index\n#     return train_X, test_X, train_y, tokenizer.word_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ff9385fc67468715bb373a0b9e04657c8d4cd6b","scrolled":false},"cell_type":"code","source":"# fill up the missing values\n# x_train, x_test, y_train, word_index = load_and_prec()\nx_train, x_test, y_train, features, test_features, word_index = load_and_prec() \n# x_train, x_test, y_train, x_test_f,y_test_f,features, test_features,features_t, word_index = load_and_prec() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"681971bf48fbfa12fcf26051a8ff264f71690fbe"},"cell_type":"code","source":"questions = list(df_train['question_text'])\ntargets = list(df_train['target'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf27e1cfc166d1f7d2acc22e09bf8c22810d76b4"},"cell_type":"code","source":"np.save(\"x_train\",x_train)\nnp.save(\"x_test\",x_test)\nnp.save(\"y_train\",y_train)\n\nnp.save(\"features\",features)\nnp.save(\"test_features\",test_features)\nnp.save(\"word_index.npy\",word_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca6c443cf3d7658590dd64516735c10cda4554c4"},"cell_type":"code","source":"x_train = np.load(\"x_train.npy\")\nx_test = np.load(\"x_test.npy\")\ny_train = np.load(\"y_train.npy\")\nfeatures = np.load(\"features.npy\")\ntest_features = np.load(\"test_features.npy\")\nword_index = np.load(\"word_index.npy\").item()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6de0424b344d2fe49cc3fc785ce1cee98d770f21"},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ec76d6490549bcdcd9821f5a677275226ecee2c3"},"cell_type":"code","source":"# missing entries in the embedding are set using np.random.normal so we have to seed here too\nseed_everything()\n\nglove_embeddings = load_glove(word_index)\nparagram_embeddings = load_para(word_index)\n\nembedding_matrix = np.mean([glove_embeddings, paragram_embeddings], axis=0)\n\n# vocab = build_vocab(df['question_text'])\n# add_lower(embedding_matrix, vocab)\ndel glove_embeddings, paragram_embeddings\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"83c0be90322fe673522e333910f88200e8a673bc"},"cell_type":"code","source":"splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED).split(x_train, y_train))\nsplits[:3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d75c25af82bce454f143a68cca53ef3c99c76d5d"},"cell_type":"code","source":"# code inspired from: https://github.com/anandsaha/pytorch.cyclic.learning.rate/blob/master/cls.py\nclass CyclicLR(object):\n    def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,\n                 step_size=2000, mode='triangular', gamma=1.,\n                 scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):\n\n        if not isinstance(optimizer, Optimizer):\n            raise TypeError('{} is not an Optimizer'.format(\n                type(optimizer).__name__))\n        self.optimizer = optimizer\n\n        if isinstance(base_lr, list) or isinstance(base_lr, tuple):\n            if len(base_lr) != len(optimizer.param_groups):\n                raise ValueError(\"expected {} base_lr, got {}\".format(\n                    len(optimizer.param_groups), len(base_lr)))\n            self.base_lrs = list(base_lr)\n        else:\n            self.base_lrs = [base_lr] * len(optimizer.param_groups)\n\n        if isinstance(max_lr, list) or isinstance(max_lr, tuple):\n            if len(max_lr) != len(optimizer.param_groups):\n                raise ValueError(\"expected {} max_lr, got {}\".format(\n                    len(optimizer.param_groups), len(max_lr)))\n            self.max_lrs = list(max_lr)\n        else:\n            self.max_lrs = [max_lr] * len(optimizer.param_groups)\n\n        self.step_size = step_size\n\n        if mode not in ['triangular', 'triangular2', 'exp_range'] \\\n                and scale_fn is None:\n            raise ValueError('mode is invalid and scale_fn is None')\n\n        self.mode = mode\n        self.gamma = gamma\n\n        if scale_fn is None:\n            if self.mode == 'triangular':\n                self.scale_fn = self._triangular_scale_fn\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = self._triangular2_scale_fn\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = self._exp_range_scale_fn\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n\n        self.batch_step(last_batch_iteration + 1)\n        self.last_batch_iteration = last_batch_iteration\n\n    def batch_step(self, batch_iteration=None):\n        if batch_iteration is None:\n            batch_iteration = self.last_batch_iteration + 1\n        self.last_batch_iteration = batch_iteration\n        for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):\n            param_group['lr'] = lr\n\n    def _triangular_scale_fn(self, x):\n        return 1.\n\n    def _triangular2_scale_fn(self, x):\n        return 1 / (2. ** (x - 1))\n\n    def _exp_range_scale_fn(self, x):\n        return self.gamma**(x)\n\n    def get_lr(self):\n        step_size = float(self.step_size)\n        cycle = np.floor(1 + self.last_batch_iteration / (2 * step_size))\n        x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)\n\n        lrs = []\n        param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)\n        for param_group, base_lr, max_lr in param_lrs:\n            base_height = (max_lr - base_lr) * np.maximum(0, (1 - x))\n            if self.scale_mode == 'cycle':\n                lr = base_lr + base_height * self.scale_fn(cycle)\n            else:\n                lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)\n            lrs.append(lr)\n        return lrs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e1eae34b623f49e68b04f672ada73410f29b656"},"cell_type":"code","source":"import torch as t\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nembedding_dim = 300\nembedding_path = '../save/embedding_matrix.npy'  # or False, not use pre-trained-matrix\nuse_pretrained_embedding = True\n\nhidden_size = 60\ngru_len = hidden_size\n\nRoutings = 4 #5\nNum_capsule = 5\nDim_capsule = 5#16\ndropout_p = 0.25\nrate_drop_dense = 0.28\nLR = 0.001\nT_epsilon = 1e-7\nnum_classes = 30\n\n\nclass Embed_Layer(nn.Module):\n    def __init__(self, embedding_matrix=None, vocab_size=None, embedding_dim=300):\n        super(Embed_Layer, self).__init__()\n        self.encoder = nn.Embedding(vocab_size + 1, embedding_dim)\n        if use_pretrained_embedding:\n            # self.encoder.weight.data.copy_(t.from_numpy(np.load(embedding_path))) # 方法一，加载np.save的npy文件\n            self.encoder.weight.data.copy_(t.from_numpy(embedding_matrix))  # 方法二\n\n    def forward(self, x, dropout_p=0.25):\n        return nn.Dropout(p=dropout_p)(self.encoder(x))\n\n\nclass GRU_Layer(nn.Module):\n    def __init__(self):\n        super(GRU_Layer, self).__init__()\n        self.gru = nn.GRU(input_size=300,\n                          hidden_size=gru_len,\n                          bidirectional=True)\n        '''\n        自己修改GRU里面的激活函数及加dropout和recurrent_dropout\n        如果要使用，把rnn_revised import进来，但好像是使用cpu跑的，比较慢\n       '''\n        # # if you uncomment /*from rnn_revised import * */, uncomment following code aswell\n        # self.gru = RNNHardSigmoid('GRU', input_size=300,\n        #                           hidden_size=gru_len,\n        #                           bidirectional=True)\n\n    # 这步很关键，需要像keras一样用glorot_uniform和orthogonal_uniform初始化参数\n    def init_weights(self):\n        ih = (param.data for name, param in self.named_parameters() if 'weight_ih' in name)\n        hh = (param.data for name, param in self.named_parameters() if 'weight_hh' in name)\n        b = (param.data for name, param in self.named_parameters() if 'bias' in name)\n        for k in ih:\n            nn.init.xavier_uniform_(k)\n        for k in hh:\n            nn.init.orthogonal_(k)\n        for k in b:\n            nn.init.constant_(k, 0)\n\n    def forward(self, x):\n        return self.gru(x)\n\n\n# core caps_layer with squash func\nclass Caps_Layer(nn.Module):\n    def __init__(self, input_dim_capsule=gru_len * 2, num_capsule=Num_capsule, dim_capsule=Dim_capsule, \\\n                 routings=Routings, kernel_size=(9, 1), share_weights=True,\n                 activation='default', **kwargs):\n        super(Caps_Layer, self).__init__(**kwargs)\n\n        self.num_capsule = num_capsule\n        self.dim_capsule = dim_capsule\n        self.routings = routings\n        self.kernel_size = kernel_size  # 暂时没用到\n        self.share_weights = share_weights\n        if activation == 'default':\n            self.activation = self.squash\n        else:\n            self.activation = nn.ReLU(inplace=True)\n\n        if self.share_weights:\n            self.W = nn.Parameter(\n                nn.init.xavier_normal_(t.empty(1, input_dim_capsule, self.num_capsule * self.dim_capsule)))\n        else:\n            self.W = nn.Parameter(\n                t.randn(BATCH_SIZE, input_dim_capsule, self.num_capsule * self.dim_capsule))  # 64即batch_size\n\n    def forward(self, x):\n\n        if self.share_weights:\n            u_hat_vecs = t.matmul(x, self.W)\n        else:\n            print('add later')\n\n        batch_size = x.size(0)\n        input_num_capsule = x.size(1)\n        u_hat_vecs = u_hat_vecs.view((batch_size, input_num_capsule,\n                                      self.num_capsule, self.dim_capsule))\n        u_hat_vecs = u_hat_vecs.permute(0, 2, 1, 3)  # 转成(batch_size,num_capsule,input_num_capsule,dim_capsule)\n        b = t.zeros_like(u_hat_vecs[:, :, :, 0])  # (batch_size,num_capsule,input_num_capsule)\n\n        for i in range(self.routings):\n            b = b.permute(0, 2, 1)\n            c = F.softmax(b, dim=2)\n            c = c.permute(0, 2, 1)\n            b = b.permute(0, 2, 1)\n            outputs = self.activation(t.einsum('bij,bijk->bik', (c, u_hat_vecs)))  # batch matrix multiplication\n            # outputs shape (batch_size, num_capsule, dim_capsule)\n            if i < self.routings - 1:\n                b = t.einsum('bik,bijk->bij', (outputs, u_hat_vecs))  # batch matrix multiplication\n        return outputs  # (batch_size, num_capsule, dim_capsule)\n\n    # text version of squash, slight different from original one\n    def squash(self, x, axis=-1):\n        s_squared_norm = (x ** 2).sum(axis, keepdim=True)\n        scale = t.sqrt(s_squared_norm + T_epsilon)\n        return x / scale\n    \nclass Capsule_Main(nn.Module):\n    def __init__(self, embedding_matrix=None, vocab_size=None):\n        super(Capsule_Main, self).__init__()\n        self.embed_layer = Embed_Layer(embedding_matrix, vocab_size)\n        self.gru_layer = GRU_Layer()\n        # 【重要】初始化GRU权重操作，这一步非常关键，acc上升到0.98，如果用默认的uniform初始化则acc一直在0.5左右\n        self.gru_layer.init_weights()\n        self.caps_layer = Caps_Layer()\n        self.dense_layer = Dense_Layer()\n\n    def forward(self, content):\n        content1 = self.embed_layer(content)\n        content2, _ = self.gru_layer(\n            content1)  # 这个输出是个tuple，一个output(seq_len, batch_size, num_directions * hidden_size)，一个hn\n        content3 = self.caps_layer(content2)\n        output = self.dense_layer(content3)\n        return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4f057759c2c35b3698568a14fbeb8193194ae7fc"},"cell_type":"code","source":"class BackHook(torch.nn.Module):\n    def __init__(self, hook):\n        super(BackHook, self).__init__()\n        self._hook = hook\n        self.register_backward_hook(self._backward)\n\n    def forward(self, *inp):\n        return inp\n\n    @staticmethod\n    def _backward(self, grad_in, grad_out):\n        self._hook()\n        return None\n\n\nclass WeightDrop(torch.nn.Module):\n    \"\"\"\n    Implements drop-connect, as per Merity et al https://arxiv.org/abs/1708.02182\n    \"\"\"\n    def __init__(self, module, weights, dropout=0, variational=False):\n        super(WeightDrop, self).__init__()\n        self.module = module\n        self.weights = weights\n        self.dropout = dropout\n        self.variational = variational\n        self._setup()\n        self.hooker = BackHook(lambda: self._backward())\n\n    def _setup(self):\n        for name_w in self.weights:\n            w = getattr(self.module, name_w)\n            self.register_parameter(name_w + '_raw', Parameter(w.data))\n\n    def _setweights(self):\n        for name_w in self.weights:\n            raw_w = getattr(self, name_w + '_raw')\n            if self.training:\n                mask = raw_w.new_ones((raw_w.size(0), 1))\n                mask = torch.nn.functional.dropout(mask, p=self.dropout, training=self.training)\n                w = mask.expand_as(raw_w) * raw_w\n                setattr(self, name_w + \"_mask\", mask)\n            else:\n                w = raw_w\n            rnn_w = getattr(self.module, name_w)\n            rnn_w.data.copy_(w)\n\n    def _backward(self):\n        # transfer gradients from embeddedRNN to raw params\n        for name_w in self.weights:\n            raw_w = getattr(self, name_w + '_raw')\n            rnn_w = getattr(self.module, name_w)\n            raw_w.grad = rnn_w.grad * getattr(self, name_w + \"_mask\")\n\n    def forward(self, *args):\n        self._setweights()\n        return self.module(*self.hooker(*args))\n    \nclass WeightDropLinear(torch.nn.Module):\n    \"\"\"\n    Implements drop-connect, as per Merity et al https://arxiv.org/abs/1708.02182\n    \"\"\"\n    def __init__(self, module, weights, dropout=0, variational=False):\n        super(WeightDropLinear, self).__init__()\n        self.module = module\n        self.weights = weights\n        self.dropout = dropout\n        self.variational = variational\n        self._setup()\n        self.hooker = BackHook(lambda: self._backward())\n\n    def _setup(self):\n        for name_w in self.weights:\n            w = getattr(self.module.linear, name_w)\n            self.register_parameter(name_w + '_raw', Parameter(w.data))\n\n    def _setweights(self):\n        for name_w in self.weights:\n            raw_w = getattr(self, name_w + '_raw')\n            if self.training:\n                mask = raw_w.new_ones((raw_w.size(0), 1))\n                mask = torch.nn.functional.dropout(mask, p=self.dropout, training=self.training)\n                w = mask.expand_as(raw_w) * raw_w\n                setattr(self, name_w + \"_mask\", mask)\n            else:\n                w = raw_w\n            rnn_w = getattr(self.module.linear, name_w)\n            rnn_w.data.copy_(w)\n\n    def _backward(self):\n        # transfer gradients from embeddedRNN to raw params\n        for name_w in self.weights:\n            raw_w = getattr(self, name_w + '_raw')\n            rnn_w = getattr(self.module.linear, name_w)\n            raw_w.grad = rnn_w.grad * getattr(self, name_w + \"_mask\")\n\n    def forward(self, *args):\n        self._setweights()\n        return self.module(*self.hooker(*args))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8d92f2376597ad7122b0d30ea115b381ec16236"},"cell_type":"code","source":"class New_GRU_Layer(nn.Module):\n    def __init__(self):\n        super(New_GRU_Layer, self).__init__()\n        self.gru = nn.GRU(input_size=hidden_size*2,\n                          hidden_size=hidden_size,\n                          bidirectional=True,\n                          batch_first=True)\n        '''\n        自己修改GRU里面的激活函数及加dropout和recurrent_dropout\n        如果要使用，把rnn_revised import进来，但好像是使用cpu跑的，比较慢\n       '''\n        # # if you uncomment /*from rnn_revised import * */, uncomment following code aswell\n        # self.gru = RNNHardSigmoid('GRU', input_size=300,\n        #                           hidden_size=gru_len,\n        #                           bidirectional=True)\n\n    # 这步很关键，需要像keras一样用glorot_uniform和orthogonal_uniform初始化参数\n    def init_weights(self):\n        ih = (param.data for name, param in self.named_parameters() if 'weight_ih' in name)\n        hh = (param.data for name, param in self.named_parameters() if 'weight_hh' in name)\n        b = (param.data for name, param in self.named_parameters() if 'bias' in name)\n        for k in ih:\n            nn.init.xavier_uniform_(k)\n        for k in hh:\n            nn.init.orthogonal_(k)\n        for k in b:\n            nn.init.constant_(k, 0)\n\n    def forward(self, x):\n        return self.gru(x)\n    \nclass New_LSTM_Layer(nn.Module):\n    def __init__(self):\n        super(New_LSTM_Layer, self).__init__()\n        self.lstm = nn.LSTM(input_size=embed_size,\n                            hidden_size=hidden_size,\n                            bidirectional=True,\n                            batch_first=True)\n        '''\n        自己修改GRU里面的激活函数及加dropout和recurrent_dropout\n        如果要使用，把rnn_revised import进来，但好像是使用cpu跑的，比较慢\n       '''\n        # # if you uncomment /*from rnn_revised import * */, uncomment following code aswell\n        # self.gru = RNNHardSigmoid('GRU', input_size=300,\n        #                           hidden_size=gru_len,\n        #                           bidirectional=True)\n\n    # 这步很关键，需要像keras一样用glorot_uniform和orthogonal_uniform初始化参数\n    def init_weights(self):\n        ih = (param.data for name, param in self.named_parameters() if 'weight_ih' in name)\n        hh = (param.data for name, param in self.named_parameters() if 'weight_hh' in name)\n        b = (param.data for name, param in self.named_parameters() if 'bias' in name)\n        for k in ih:\n            nn.init.xavier_uniform_(k)\n        for k in hh:\n            nn.init.orthogonal_(k)\n        for k in b:\n            nn.init.constant_(k, 0)\n\n    def forward(self, x):\n        return self.lstm(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b2216a5ad3733fe1448d820073712809d4845a7"},"cell_type":"code","source":"class Linear(nn.Module):\n    def __init__(self):\n        super(Linear, self).__init__()\n        fc_layer1 = 16\n        self.linear = nn.Linear(hidden_size * 8 + 3, fc_layer1)\n        self.relu = nn.ReLU()\n        \n    def forward(self, x):\n        return self.relu(self.linear(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c4558d17a8b021e9a4436bdcf01093016329119"},"cell_type":"code","source":"class Attention(nn.Module):\n    def __init__(self, feature_dim, step_dim, bias=True, **kwargs):\n        super(Attention, self).__init__(**kwargs)\n        \n        self.supports_masking = True\n\n        self.bias = bias\n        self.feature_dim = feature_dim\n        self.step_dim = step_dim\n        self.features_dim = 0\n        \n        weight = torch.zeros(feature_dim, 1)\n        nn.init.xavier_uniform_(weight)\n        self.weight = nn.Parameter(weight)\n        \n        if bias:\n            self.b = nn.Parameter(torch.zeros(step_dim))\n        \n    def forward(self, x, mask=None):\n        feature_dim = self.feature_dim\n        step_dim = self.step_dim\n\n        eij = torch.mm(\n            x.contiguous().view(-1, feature_dim), \n            self.weight\n        ).view(-1, step_dim)\n        \n        if self.bias:\n            eij = eij + self.b\n            \n        eij = torch.tanh(eij)\n        a = torch.exp(eij)\n        \n        if mask is not None:\n            a = a * mask\n\n        a = a / torch.sum(a, 1, keepdim=True) + 1e-10\n\n        weighted_input = x * torch.unsqueeze(a, -1)\n        return torch.sum(weighted_input, 1)\n    \nclass NeuralNet(nn.Module):\n    def __init__(self):\n        super(NeuralNet, self).__init__()\n        \n        fc_layer = 16\n        fc_layer1 = 16\n\n        self.embedding = nn.Embedding(max_features, embed_size)\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        self.embedding.weight.requires_grad = False\n        \n        self.embedding_dropout = nn.Dropout2d(0.1)\n        self.lstm = WeightDrop(nn.LSTM(input_size=embed_size, hidden_size=hidden_size, bidirectional=True, batch_first=True), ['weight_hh_l0'], dropout=0.045)\n        self.gru = WeightDrop(nn.GRU(input_size=hidden_size*2, hidden_size=hidden_size, bidirectional=True, batch_first=True), ['weight_hh_l0'], dropout=0.045) \n\n        self.lstm2 = nn.LSTM(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)\n\n        self.lstm_attention = Attention(hidden_size * 2, maxlen)\n        self.gru_attention = Attention(hidden_size * 2, maxlen)\n        \n        self.linear = nn.Linear(hidden_size * 8 + 3, fc_layer1) # WeightDropLinear(Linear(), ['weight'], dropout=0.09) #643:80 - 483:60 - 323:40\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(0.0925)\n        self.fc = nn.Linear(fc_layer**2,fc_layer)\n        self.out = nn.Linear(fc_layer1, 1) # nn.Linear(fc_layer, 1)\n        self.lincaps = nn.Linear(Num_capsule * Dim_capsule, 1)\n        self.caps_layer = Caps_Layer()\n    \n    def forward(self, x):\n        \n#         Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True)(x)\n\n        h_embedding = self.embedding(x[0])\n        h_embedding = torch.squeeze(\n            self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))\n        \n        h_lstm, _ = self.lstm(h_embedding)\n        h_gru, _ = self.gru(h_lstm)\n        \n        ##Capsule Layer        \n        content3 = self.caps_layer(h_gru)\n        content3 = self.dropout(content3)\n        batch_size = content3.size(0)\n        content3 = content3.view(batch_size, -1)\n        content3 = self.relu(self.lincaps(content3))\n\n        ##Attention Layer\n        h_lstm_atten = self.lstm_attention(h_lstm)\n        h_gru_atten = self.gru_attention(h_gru)\n        \n        # global average pooling\n        avg_pool = torch.mean(h_gru, 1)\n        # global max pooling\n        max_pool, _ = torch.max(h_gru, 1)\n        \n        f = torch.tensor(x[1], dtype=torch.float).cuda()\n\n                #[512,160]\n        conc = torch.cat((h_lstm_atten, h_gru_atten, content3, avg_pool, max_pool, f), 1) # add f (features)\n        conc = self.relu(self.linear(conc))\n        conc = self.dropout(conc)\n        out = self.out(conc)\n        \n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f3f6a51cfb845cd49eb3a5e9521e5456e012d26"},"cell_type":"code","source":"class Padam(optim.Optimizer):\n    \"\"\"Implements Partially adaptive momentum estimation (Padam) algorithm.\n        params (iterable): iterable of parameters to optimize or dicts defining\n            parameter groups\n        lr (float, optional): learning rate (default: 1e-1)\n        betas (Tuple[float, float], optional): coefficients used for computing\n            running averages of gradient and its square (default: (0.9, 0.999))\n        eps (float, optional): term added to the denominator to improve\n            numerical stability (default: 1e-8)\n        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)\n        partial (float, optional): partially adaptive parameter\n    \"\"\"\n\n    def __init__(self, params, lr=1e-1, betas=(0.9, 0.999), eps=1e-8, weight_decay=0, amsgrad=True, partial = 1/4):\n        if not 0.0 <= betas[0] < 1.0:\n            raise ValueError(\"Invalid beta parameter at index 0: {}\".format(betas[0]))\n        if not 0.0 <= betas[1] < 1.0:\n            raise ValueError(\"Invalid beta parameter at index 1: {}\".format(betas[1]))\n        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, amsgrad=amsgrad, partial = partial)\n        super(Padam, self).__init__(params, defaults)\n\n    def step(self, closure=None):\n        \"\"\"Performs a single optimization step.\n        Arguments:\n            closure (callable, optional): A closure that reevaluates the model\n                and returns the loss.\n        \"\"\"\n        loss = None\n        if closure is not None:\n            loss = closure()\n\n        for group in self.param_groups:\n            for p in group['params']:\n                if p.grad is None:\n                    continue\n                grad = p.grad.data\n                if grad.is_sparse:\n                    raise RuntimeError('Adam does not support sparse gradients, please consider SparseAdam instead')\n                amsgrad = group['amsgrad']\n                partial = group['partial']\n\n                state = self.state[p]\n\n                # State initialization\n                if len(state) == 0:\n                    state['step'] = 0\n                    # Exponential moving average of gradient values\n                    state['exp_avg'] = torch.zeros_like(p.data)\n                    # Exponential moving average of squared gradient values\n                    state['exp_avg_sq'] = torch.zeros_like(p.data)\n                    if amsgrad:\n                        # Maintains max of all exp. moving avg. of sq. grad. values\n                        state['max_exp_avg_sq'] = torch.zeros_like(p.data)\n\n                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n                if amsgrad:\n                    max_exp_avg_sq = state['max_exp_avg_sq']\n                beta1, beta2 = group['betas']\n\n                state['step'] += 1\n\n                if group['weight_decay'] != 0:\n                    grad = grad.add(group['weight_decay'], p.data)\n\n                # Decay the first and second moment running average coefficient\n                exp_avg.mul_(beta1).add_(1 - beta1, grad)\n                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)\n                if amsgrad:\n                    # Maintains the maximum of all 2nd moment running avg. till now\n                    torch.max(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq)\n                    # Use the max. for normalizing running avg. of gradient\n                    denom = max_exp_avg_sq.sqrt().add_(group['eps'])\n                else:\n                    denom = exp_avg_sq.sqrt().add_(group['eps'])\n\n                bias_correction1 = 1 - beta1 ** state['step']\n                bias_correction2 = 1 - beta2 ** state['step']\n                step_size = group['lr'] * np.sqrt(bias_correction2) / bias_correction1\n\n                p.data.addcdiv_(-step_size, exp_avg, denom**(partial*2))\n                \n        return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ab806c0831fd937403a9a3b2e9604616c20006d"},"cell_type":"code","source":"def sigmoid(x):\n    return 1 / (1 + np.exp(-x))\n    #return np.log(1+np.exp(x))\n    #return np.tanh(x)\n    #return np.maximum(0.0, x)\n    \n# matrix for the out-of-fold predictions\ntrain_preds = np.zeros((len(x_train)))\n# matrix for the predictions on the test set\ntest_preds = np.zeros((len(df_test)))\n\n# always call this before training for deterministic results\nseed_everything()\n\n# x_test_cuda_f = torch.tensor(x_test_f, dtype=torch.long).cuda()\n# test_f = torch.utils.data.TensorDataset(x_test_cuda_f)\n# test_loader_f = torch.utils.data.DataLoader(test_f, batch_size=batch_size, shuffle=False)\n\n\"\"\"\nx_test = list(x_test)\nfor _ in range(6):\n    x_test.extend(x_test)\nx_test = np.array(x_test)\n\"\"\"\n\nx_test_cuda = torch.tensor(x_test, dtype=torch.long).cuda()\ntest = torch.utils.data.TensorDataset(x_test_cuda)\ntest_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)\n\navg_losses_f = []\navg_val_losses_f = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2ee1ecf044dfdd5f0effe6c99229e948b137616"},"cell_type":"code","source":"class MyDataset(Dataset):\n    def __init__(self,dataset):\n        self.dataset = dataset\n\n    def __getitem__(self, index):\n        data, target = self.dataset[index]\n\n        return data, target, index\n    \n    def __len__(self):\n        return len(self.dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2dcce97b1170cdd2d5764ec92e3d985258fee620"},"cell_type":"code","source":"for i, (train_idx, valid_idx) in enumerate(splits):    \n    # split data in train / validation according to the KFold indeces\n    # also, convert them to a torch tensor and store them on the GPU (done with .cuda())\n    x_train = np.array(x_train)\n    y_train = np.array(y_train)\n    features = np.array(features)\n\n    x_train_fold = torch.tensor(x_train[train_idx.astype(int)], dtype=torch.long).cuda()\n    y_train_fold = torch.tensor(y_train[train_idx.astype(int), np.newaxis], dtype=torch.float32).cuda()\n    \n    kfold_X_features = features[train_idx.astype(int)]\n    kfold_X_valid_features = features[valid_idx.astype(int)]\n    x_val_fold = torch.tensor(x_train[valid_idx.astype(int)], dtype=torch.long).cuda()\n    y_val_fold = torch.tensor(y_train[valid_idx.astype(int), np.newaxis], dtype=torch.float32).cuda()\n    \n#     model = BiLSTM(lstm_layer=2,hidden_dim=40,dropout=DROPOUT).cuda()\n    model = NeuralNet()\n\n    # make sure everything in the model is running on the GPU\n    model.cuda()\n\n    # define binary cross entropy loss\n    # note that the model returns logit to take advantage of the log-sum-exp trick \n    # for numerical stability in the loss\n    loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum')\n\n    step_size = 300\n    base_lr, max_lr = 1e-3, 3e-3 # base_lr, max_lr = 1e-3 or 2e-3, 3e-3 or 4e-3\n    optimizer = Padam(filter(lambda p: p.requires_grad, model.parameters()),    # combinations : base_lr, max_lr, lr = 1e-3, 2e-3, 3e-3  \"\"OR\"\" 2e-3, 3e-3, 4e-3\n                      lr=2e-3, partial=0.4, eps=2e-8) # lr=2e-3 or 3e-3\n    \n    ################################################################################################\n    scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,\n               step_size=step_size, mode='exp_range',\n               gamma=0.999999999)\n                #gamma=1.5)\n    ###############################################################################################\n\n    train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)\n    valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)\n    \n    train = MyDataset(train)\n    valid = MyDataset(valid)\n    \n    # _weights = [1.]\n    # sampler = torch.utils.data.sampler.WeightedRandomSampler(_weights, num_samples=batch_size)\n    \n    train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n    valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)\n\n    print(f'Fold {i + 1}')\n    for epoch in range(n_epochs):\n        # set train mode of the model. This enables operations which are only applied during training like dropout\n        start_time = time.time()\n        model.train()\n        avg_loss = 0.  \n        for i, (x_batch, y_batch, index) in enumerate(train_loader):\n            # Forward pass: compute predicted y by passing x to the model.\n            ################################################################################################            \n            f = kfold_X_features[index]\n            y_pred = model([x_batch,f])\n            ################################################################################################\n\n            if scheduler:\n                scheduler.batch_step()\n            ################################################################################################\n\n\n            # Compute and print loss.\n            loss = loss_fn(y_pred, y_batch)\n\n            # Before the backward pass, use the optimizer object to zero all of the\n            # gradients for the Tensors it will update (which are the learnable weights\n            # of the model)\n            optimizer.zero_grad()\n\n            # Backward pass: compute gradient of the loss with respect to model parameters\n            loss.backward()\n\n            # Calling the step function on an Optimizer makes an update to its parameters\n            optimizer.step()\n            avg_loss += loss.item() / len(train_loader)\n            \n        # set evaluation mode of the model. This disabled operations which are only applied during training like dropout\n        model.eval()\n        \n        # predict all the samples in y_val_fold batch per batch\n        valid_preds_fold = np.zeros((x_val_fold.size(0)))\n        test_preds_fold = np.zeros((len(df_test)))\n        \n        avg_val_loss = 0.\n        for i, (x_batch, y_batch, index) in enumerate(valid_loader):\n            f = kfold_X_valid_features[index]            \n            y_pred = model([x_batch,f]).detach()\n            \n            avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n            valid_preds_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n        \n        elapsed_time = time.time() - start_time \n        print('Epoch {}/{} \\t loss={:.4f} \\t val_loss={:.4f} \\t time={:.2f}s'.format(\n            epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))\n    avg_losses_f.append(avg_loss)\n    avg_val_losses_f.append(avg_val_loss) \n    # predict all samples in the test set batch per batch\n    for i, (x_batch,) in enumerate(test_loader):\n        f = test_features[i * batch_size:(i+1) * batch_size]\n        y_pred = model([x_batch,f]).detach()\n\n        test_preds_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n        \n    train_preds[valid_idx] = valid_preds_fold\n    test_preds += test_preds_fold / len(splits)\n\nprint('All \\t loss={:.4f} \\t val_loss={:.4f} \\t '.format(np.average(avg_losses_f),np.average(avg_val_losses_f)))\n\n# x_train, x_test_f, y_train, y_test_f","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2a13a8be57bc9ac6e27e6c890a491feb97a0f238"},"cell_type":"code","source":"def bestThresshold(y_train,train_preds):\n    tmp = [0,0,0] # idx, cur, max\n    delta = 0\n    for tmp[0] in tqdm(np.arange(0.2, 0.402, 0.001)):\n        tmp[1] = f1_score(y_train, np.array(train_preds)>tmp[0])\n        if tmp[1] > tmp[2]:\n            delta = tmp[0]\n            tmp[2] = tmp[1]\n    print('best threshold is {:.4f} with F1 score: {:.4f}'.format(delta, tmp[2]))\n    return delta\ndelta = bestThresshold(y_train,train_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"359a4816c2203d1e1956e1685ad16ecf8ba5434d"},"cell_type":"code","source":"submission = df_test[['qid']].copy()\nsubmission['prediction'] = (test_preds > delta).astype(int)\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2add08ab46cb2bec5d0dea6156e08c802e66479"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}