{"cells":[{"metadata":{"_uuid":"710ed17d0c57bd287be0ee3b2782a53a54510561"},"cell_type":"markdown","source":"## IMPORTS "},{"metadata":{"_uuid":"abb7e3c30b8a412a50c6b451c49939e3cf4bc11b","scrolled":true,"trusted":false},"cell_type":"code","source":"import time\nimport random\nimport pandas as pd\nimport numpy as np\nimport gc\nimport re\nimport torch\nfrom torchtext import data\nimport spacy\nfrom tqdm import tqdm_notebook, tnrange\nfrom tqdm.auto import tqdm\n\ntqdm.pandas(desc='Progress')\nfrom collections import Counter\nfrom textblob import TextBlob\nfrom nltk import word_tokenize\n\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\nfrom torch.autograd import Variable\nfrom torchtext.data import Example\nfrom sklearn.metrics import f1_score\nimport torchtext\nimport os \n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\n# cross validation and metrics\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import f1_score\nfrom torch.optim.optimizer import Optimizer\nfrom unidecode import unidecode","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9a4ff5590a6f152dc1bec5aeca79aef10218f7de"},"cell_type":"markdown","source":"### Basic Parameters"},{"metadata":{"_uuid":"deee49df5ca1c4413f71677939e26aa1ff784e44","scrolled":true,"trusted":false},"cell_type":"code","source":"embed_size = 300 # how big is each word vector\nmax_features = 120000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 70 # max number of words in a question to use\nbatch_size = 512 # how many samples to process at once\nn_epochs = 5 # how many times to iterate over all samples\nn_splits = 5 # Number of K-fold Splits\n\nSEED = 1029","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"654cbe3c8a1f2a618a2441afe00df3b4a89e0a58"},"cell_type":"markdown","source":"### Ensure determinism in the results\n\nA common headache in this competition is the lack of determinism in the results due to cudnn. The following Kernel has a solution in Pytorch.\n\nSee https://www.kaggle.com/hengzheng/pytorch-starter. "},{"metadata":{"_uuid":"58bbf87335799247586aaed16531f4d28d10ed4a","scrolled":true,"trusted":false},"cell_type":"code","source":"def seed_everything(seed=1029):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c890692644acce2dc4f6e2f929d6d294faca4ad2"},"cell_type":"markdown","source":"### Code for Loading Embeddings\n\nFunctions taken from the kernel:https://www.kaggle.com/gmhost/gru-capsule\n"},{"metadata":{"_uuid":"7026ee1d913f54dd4b560f654efdb9f833581cd3","scrolled":true,"trusted":false},"cell_type":"code","source":"## FUNCTIONS TAKEN FROM https://www.kaggle.com/gmhost/gru-capsule\n\ndef load_glove(word_index):\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')[:300]\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n    \n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = -0.005838499,0.48782197\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    # Why random embedding for OOV? what if use mean?\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    #embedding_matrix = np.random.normal(emb_mean, 0, (nb_words, embed_size)) # std 0\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n            \n    return embedding_matrix \n    \ndef load_fasttext(word_index):    \n    EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    #embedding_matrix = np.random.normal(emb_mean, 0, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n\n    return embedding_matrix\n\ndef load_para(word_index):\n    EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = -0.0053247833,0.49346462\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    #embedding_matrix = np.random.normal(emb_mean, 0, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n    \n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ea10c8e218a1280faa9802bcb7f1117c89ec96f9"},"cell_type":"markdown","source":"## LOAD PROCESSED TRAINING DATA FROM DISK"},{"metadata":{"_uuid":"173753f0178464d2ba26baf22899884d76d1c83d","scrolled":true,"trusted":false},"cell_type":"code","source":"df_train = pd.read_csv(\"../input/train.csv\")\ndf_test = pd.read_csv(\"../input/test.csv\")\ndf = pd.concat([df_train ,df_test],sort=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0f75559b6fa28c27ecbf145121309378afc884ee","scrolled":true,"trusted":false},"cell_type":"code","source":"def build_vocab(texts):\n    sentences = texts.apply(lambda x: x.split()).values\n    vocab = {}\n    for sentence in sentences:\n        for word in sentence:\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n    return vocab\nvocab = build_vocab(df['question_text'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5cb425ffbf1f79c1edc4cad3da15a1c1aa53edca","scrolled":true,"trusted":false},"cell_type":"code","source":"sin = len(df_train[df_train[\"target\"]==0])\ninsin = len(df_train[df_train[\"target\"]==1])\npersin = (sin/(sin+insin))*100\nperinsin = (insin/(sin+insin))*100            \nprint(\"# Sincere questions: {:,}({:.2f}%) and # Insincere questions: {:,}({:.2f}%)\".format(sin,persin,insin,perinsin))\n# print(\"Sinsere:{}% Insincere: {}%\".format(round(persin,2),round(perinsin,2)))\nprint(\"# Test samples: {:,}({:.3f}% of train samples)\".format(len(df_test),len(df_test)/len(df_train)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"07e9890ec0b490cef57565f7dff953aa56ebd3dc"},"cell_type":"markdown","source":"## Normalization\n\nBorrowed from:\n* How to: Preprocessing when using embeddings\nhttps://www.kaggle.com/christofhenkel/how-to-preprocessing-when-using-embeddings\n* Improve your Score with some Text Preprocessing https://www.kaggle.com/theoviel/improve-your-score-with-some-text-preprocessing"},{"metadata":{"_uuid":"28ebb28ba78972bb8d4fee9b53437045542d20fb","scrolled":true,"trusted":false},"cell_type":"code","source":"def build_vocab(texts):\n    sentences = texts.apply(lambda x: x.split()).values\n    vocab = {}\n    for sentence in sentences:\n        for word in sentence:\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n    return vocab\n\ndef known_contractions(embed):\n    known = []\n    for contract in contraction_mapping:\n        if contract in embed:\n            known.append(contract)\n    return known\ndef clean_contractions(text, mapping):\n    specials = [\"’\", \"‘\", \"´\", \"`\"]\n    for s in specials:\n        text = text.replace(s, \"'\")\n    text = ' '.join([mapping[t] if t in mapping else t for t in text.split(\" \")])\n    return text\ndef correct_spelling(x, dic):\n    for word in dic.keys():\n        x = x.replace(word, dic[word])\n    return x\ndef unknown_punct(embed, punct):\n    unknown = ''\n    for p in punct:\n        if p not in embed:\n            unknown += p\n            unknown += ' '\n    return unknown\n\ndef clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x\n\ndef clean_special_chars(text, punct, mapping):\n    for p in mapping:\n        text = text.replace(p, mapping[p])\n    \n    for p in punct:\n        text = text.replace(p, f' {p} ')\n    \n    specials = {'\\u200b': ' ', '…': ' ... ', '\\ufeff': '', 'करना': '', 'है': ''}  # Other special characters that I have to deal with in last\n    for s in specials:\n        text = text.replace(s, specials[s])\n    \n    return text\ndef add_lower(embedding, vocab):\n    count = 0\n    for word in vocab:\n        if word in embedding and word.lower() not in embedding:  \n            embedding[word.lower()] = embedding[word]\n            count += 1\n    print(f\"Added {count} words to embedding\")    ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"abeab4c80d6829cf2eae706bfa7929e2871af81f","scrolled":true,"trusted":false},"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n\ndef clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        x = x.replace(punct, f' {punct} ')\n    return x\n\ndef clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x\n\nmispell_dict = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n    return mispellings_re.sub(replace, text)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3c09d981ae674e6a373189a04dba8d0932b0765b"},"cell_type":"markdown","source":"Extra feature part taken from https://github.com/wongchunghang/toxic-comment-challenge-lstm/blob/master/toxic_comment_9872_model.ipynb"},{"metadata":{"_uuid":"63cb21525251b060aeb309e7be4b48772f8720f5","scrolled":true,"trusted":false},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\n\ndef add_features(df):\n    \n    df['question_text'] = df['question_text'].progress_apply(lambda x:str(x))\n    df['total_length'] = df['question_text'].progress_apply(len)\n    df['capitals'] = df['question_text'].progress_apply(lambda comment: sum(1 for c in comment if c.isupper()))\n    df['caps_vs_length'] = df.progress_apply(lambda row: float(row['capitals'])/float(row['total_length']),\n                                axis=1)\n    df['num_words'] = df.question_text.str.count('\\S+')\n    df['num_unique_words'] = df['question_text'].progress_apply(lambda comment: len(set(w for w in comment.split())))\n    df['words_vs_unique'] = df['num_unique_words'] / df['num_words']  \n\n    return df\n\ndef load_and_prec():\n    train_df = pd.read_csv(\"../input/train.csv\")\n    test_df = pd.read_csv(\"../input/test.csv\")\n    print(\"Train shape : \",train_df.shape)\n    print(\"Test shape : \",test_df.shape)\n    \n    # lower\n    train_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: x.lower())\n    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: x.lower())\n\n    # Clean the text\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_text(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: clean_text(x))\n    \n    # Clean numbers\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: clean_numbers(x))\n    \n    # Clean speelings\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: replace_typical_misspell(x))\n    \n    ## fill up the missing values\n    train_X = train_df[\"question_text\"].fillna(\"_##_\").values\n    test_X = test_df[\"question_text\"].fillna(\"_##_\").values\n\n\n    \n    ###################### Add Features ###############################\n    #  https://github.com/wongchunghang/toxic-comment-challenge-lstm/blob/master/toxic_comment_9872_model.ipynb\n    train = add_features(train_df)\n    test = add_features(test_df)\n\n    features = train[['caps_vs_length', 'words_vs_unique']].fillna(0)\n    test_features = test[['caps_vs_length', 'words_vs_unique']].fillna(0)\n\n    ss = StandardScaler()\n    ss.fit(np.vstack((features, test_features)))\n    features = ss.transform(features)\n    test_features = ss.transform(test_features)\n    ###########################################################################\n\n    ## Tokenize the sentences\n    tokenizer = Tokenizer(num_words=max_features)\n    tokenizer.fit_on_texts(list(train_X))\n    train_X = tokenizer.texts_to_sequences(train_X)\n    test_X = tokenizer.texts_to_sequences(test_X)\n\n    ## Pad the sentences \n    train_X = pad_sequences(train_X, maxlen=maxlen)\n    test_X = pad_sequences(test_X, maxlen=maxlen)\n\n    ## Get the target values\n    train_y = train_df['target'].values\n    \n#     # Splitting to training and a final test set    \n#     train_X, x_test_f, train_y, y_test_f = train_test_split(list(zip(train_X,features)), train_y, test_size=0.2, random_state=SEED)    \n#     train_X, features = zip(*train_X)\n#     x_test_f, features_t = zip(*x_test_f)    \n    \n    #shuffling the data\n    np.random.seed(SEED)\n    trn_idx = np.random.permutation(len(train_X))\n\n    train_X = train_X[trn_idx]\n    train_y = train_y[trn_idx]\n    features = features[trn_idx]\n    \n    return train_X, test_X, train_y, features, test_features, tokenizer.word_index\n#     return train_X, test_X, train_y, x_test_f,y_test_f,features, test_features, features_t, tokenizer.word_index\n#     return train_X, test_X, train_y, tokenizer.word_index","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3c72fcddb4f680879e231c3dbfc0c71e27fc424c","scrolled":true,"trusted":false},"cell_type":"code","source":"\n# fill up the missing values\n# x_train, x_test, y_train, word_index = load_and_prec()\nx_train, x_test, y_train, features, test_features, word_index = load_and_prec() \n# x_train, x_test, y_train, x_test_f,y_test_f,features, test_features,features_t, word_index = load_and_prec() \n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5264b1a511613e0cb9cffc11e93e906f932be191"},"cell_type":"markdown","source":"### SAVE DATASET TO DISK"},{"metadata":{"_uuid":"7dc2704001695c0d691fec74cc111308da3fc340","scrolled":true,"trusted":false},"cell_type":"code","source":"np.save(\"x_train\",x_train)\nnp.save(\"x_test\",x_test)\nnp.save(\"y_train\",y_train)\n\nnp.save(\"features\",features)\nnp.save(\"test_features\",test_features)\nnp.save(\"word_index.npy\",word_index)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ef622350c3ac00bcbea516ccf7dffbeca7b4cc39"},"cell_type":"markdown","source":"### LOAD DATASET FROM DISK"},{"metadata":{"_uuid":"6e1f33f0d744e86cfbcc21b8e696bb568ba1c454","scrolled":true,"trusted":false},"cell_type":"code","source":"x_train = np.load(\"x_train.npy\")\nx_test = np.load(\"x_test.npy\")\ny_train = np.load(\"y_train.npy\")\nfeatures = np.load(\"features.npy\")\ntest_features = np.load(\"test_features.npy\")\nword_index = np.load(\"word_index.npy\").item()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6db7b1d3f22a6e35699d9f0dbb51343e249d9ee7","scrolled":true,"trusted":false},"cell_type":"code","source":"features.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e5c51a8329d569d13b9f0369ebb98ca8e2e55440"},"cell_type":"markdown","source":"### Load Embeddings\n\nTwo embedding matrices have been used. Glove, and paragram. The mean of the two is used as the final embedding matrix"},{"metadata":{"_uuid":"6a5f4502324d369ff6faa3692accee4f8a233005","scrolled":true,"trusted":false},"cell_type":"code","source":"# missing entries in the embedding are set using np.random.normal so we have to seed here too\nseed_everything()\n\nglove_embeddings = load_glove(word_index)\nparagram_embeddings = load_para(word_index)\nfasttext_embeddings = load_fasttext(word_index)\n\nembedding_matrix = np.mean([glove_embeddings, paragram_embeddings, fasttext_embeddings], axis=0)\n\n# vocab = build_vocab(df['question_text'])\n# add_lower(embedding_matrix, vocab)\ndel glove_embeddings, paragram_embeddings, fasttext_embeddings\ngc.collect()\n\nnp.shape(embedding_matrix)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2484f29f159af679af7d7745e522221e31e42ce0","trusted":false},"cell_type":"code","source":"np.shape(embedding_matrix)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d3b707858357eec18403f97368a40c43987b40e8"},"cell_type":"markdown","source":"### Use Stratified K Fold to improve results"},{"metadata":{"_uuid":"fa55c64890964220ded1762236e091e59e502e53","scrolled":true,"trusted":false},"cell_type":"code","source":"splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED).split(x_train, y_train))\nsplits[:3]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8098bea0cee9117ff9dc4e11feba53e49b80cb55"},"cell_type":"markdown","source":"### Cyclic CLR\nCode taken from https://www.kaggle.com/dannykliu/lstm-with-attention-clr-in-pytorch"},{"metadata":{"_uuid":"9d531a7454923f90d0e7443b1ed1373d008c2e88","scrolled":true,"trusted":false},"cell_type":"code","source":"# code inspired from: https://github.com/anandsaha/pytorch.cyclic.learning.rate/blob/master/cls.py\nclass CyclicLR(object):\n    def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,\n                 step_size=2000, mode='triangular', gamma=1.,\n                 scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):\n\n        if not isinstance(optimizer, Optimizer):\n            raise TypeError('{} is not an Optimizer'.format(\n                type(optimizer).__name__))\n        self.optimizer = optimizer\n\n        if isinstance(base_lr, list) or isinstance(base_lr, tuple):\n            if len(base_lr) != len(optimizer.param_groups):\n                raise ValueError(\"expected {} base_lr, got {}\".format(\n                    len(optimizer.param_groups), len(base_lr)))\n            self.base_lrs = list(base_lr)\n        else:\n            self.base_lrs = [base_lr] * len(optimizer.param_groups)\n\n        if isinstance(max_lr, list) or isinstance(max_lr, tuple):\n            if len(max_lr) != len(optimizer.param_groups):\n                raise ValueError(\"expected {} max_lr, got {}\".format(\n                    len(optimizer.param_groups), len(max_lr)))\n            self.max_lrs = list(max_lr)\n        else:\n            self.max_lrs = [max_lr] * len(optimizer.param_groups)\n\n        self.step_size = step_size\n\n        if mode not in ['triangular', 'triangular2', 'exp_range'] \\\n                and scale_fn is None:\n            raise ValueError('mode is invalid and scale_fn is None')\n\n        self.mode = mode\n        self.gamma = gamma\n\n        if scale_fn is None:\n            if self.mode == 'triangular':\n                self.scale_fn = self._triangular_scale_fn\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = self._triangular2_scale_fn\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = self._exp_range_scale_fn\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n\n        self.batch_step(last_batch_iteration + 1)\n        self.last_batch_iteration = last_batch_iteration\n\n    def batch_step(self, batch_iteration=None):\n        if batch_iteration is None:\n            batch_iteration = self.last_batch_iteration + 1\n        self.last_batch_iteration = batch_iteration\n        for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):\n            param_group['lr'] = lr\n\n    def _triangular_scale_fn(self, x):\n        return 1.\n\n    def _triangular2_scale_fn(self, x):\n        return 1 / (2. ** (x - 1))\n\n    def _exp_range_scale_fn(self, x):\n        return self.gamma**(x)\n\n    def get_lr(self):\n        step_size = float(self.step_size)\n        cycle = np.floor(1 + self.last_batch_iteration / (2 * step_size))\n        x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)\n\n        lrs = []\n        param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)\n        for param_group, base_lr, max_lr in param_lrs:\n            base_height = (max_lr - base_lr) * np.maximum(0, (1 - x))\n            if self.scale_mode == 'cycle':\n                lr = base_lr + base_height * self.scale_fn(cycle)\n            else:\n                lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)\n            lrs.append(lr)\n        return lrs\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b8caecb207a12d4a5524fe16e4524b31c7da8bac"},"cell_type":"markdown","source":"### Model Architecture\n\nBinary LSTM with an attention layer and an additional fully connected layer. Also added extra features taken from a winning kernel of the toxic comments competition. Also using CLR and a capsule Layer. Blended together in concatentation.\n\nInitial idea borrowed from: https://www.kaggle.com/ziliwang/baseline-pytorch-bilstm"},{"metadata":{"_uuid":"8f7e1d15451201efbac2741b03d4b0dfd3b9bfe0","trusted":false},"cell_type":"code","source":"import torch as t\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nembedding_dim = 300\nembedding_path = '../save/embedding_matrix.npy'  # or False, not use pre-trained-matrix\nuse_pretrained_embedding = True\n\nhidden_size = 60\ngru_len = hidden_size\n\nRoutings = 4 #5\nNum_capsule = 5\nDim_capsule = 5#16\ndropout_p = 0.25\nrate_drop_dense = 0.28\nLR = 0.001\nT_epsilon = 1e-7\nnum_classes = 30\n\n\nclass Embed_Layer(nn.Module):\n    def __init__(self, embedding_matrix=None, vocab_size=None, embedding_dim=300):\n        super(Embed_Layer, self).__init__()\n        self.encoder = nn.Embedding(vocab_size + 1, embedding_dim)\n        if use_pretrained_embedding:\n            # self.encoder.weight.data.copy_(t.from_numpy(np.load(embedding_path))) # 方法一，加载np.save的npy文件\n            self.encoder.weight.data.copy_(t.from_numpy(embedding_matrix))  # 方法二\n\n    def forward(self, x, dropout_p=0.25):\n        return nn.Dropout(p=dropout_p)(self.encoder(x))\n\n\nclass GRU_Layer(nn.Module):\n    def __init__(self):\n        super(GRU_Layer, self).__init__()\n        self.gru = nn.GRU(input_size=300,\n                          hidden_size=gru_len,\n                          bidirectional=True)\n        '''\n        自己修改GRU里面的激活函数及加dropout和recurrent_dropout\n        如果要使用，把rnn_revised import进来，但好像是使用cpu跑的，比较慢\n       '''\n        # # if you uncomment /*from rnn_revised import * */, uncomment following code aswell\n        # self.gru = RNNHardSigmoid('GRU', input_size=300,\n        #                           hidden_size=gru_len,\n        #                           bidirectional=True)\n\n    # 这步很关键，需要像keras一样用glorot_uniform和orthogonal_uniform初始化参数\n    def init_weights(self):\n        ih = (param.data for name, param in self.named_parameters() if 'weight_ih' in name)\n        hh = (param.data for name, param in self.named_parameters() if 'weight_hh' in name)\n        b = (param.data for name, param in self.named_parameters() if 'bias' in name)\n        for k in ih:\n            nn.init.xavier_uniform_(k)\n        for k in hh:\n            nn.init.orthogonal_(k)\n        for k in b:\n            nn.init.constant_(k, 0)\n\n    def forward(self, x):\n        return self.gru(x)\n\n\n# core caps_layer with squash func\nclass Caps_Layer(nn.Module):\n    def __init__(self, input_dim_capsule=gru_len * 2, num_capsule=Num_capsule, dim_capsule=Dim_capsule, \\\n                 routings=Routings, kernel_size=(9, 1), share_weights=True,\n                 activation='default', **kwargs):\n        super(Caps_Layer, self).__init__(**kwargs)\n\n        self.num_capsule = num_capsule\n        self.dim_capsule = dim_capsule\n        self.routings = routings\n        self.kernel_size = kernel_size  # 暂时没用到\n        self.share_weights = share_weights\n        if activation == 'default':\n            self.activation = self.squash\n        else:\n            self.activation = nn.ReLU(inplace=True)\n\n        if self.share_weights:\n            self.W = nn.Parameter(\n                nn.init.xavier_normal_(t.empty(1, input_dim_capsule, self.num_capsule * self.dim_capsule)))\n        else:\n            self.W = nn.Parameter(\n                t.randn(BATCH_SIZE, input_dim_capsule, self.num_capsule * self.dim_capsule))  # 64即batch_size\n\n    def forward(self, x):\n\n        if self.share_weights:\n            u_hat_vecs = t.matmul(x, self.W)\n        else:\n            print('add later')\n\n        batch_size = x.size(0)\n        input_num_capsule = x.size(1)\n        u_hat_vecs = u_hat_vecs.view((batch_size, input_num_capsule,\n                                      self.num_capsule, self.dim_capsule))\n        u_hat_vecs = u_hat_vecs.permute(0, 2, 1, 3)  # 转成(batch_size,num_capsule,input_num_capsule,dim_capsule)\n        b = t.zeros_like(u_hat_vecs[:, :, :, 0])  # (batch_size,num_capsule,input_num_capsule)\n\n        for i in range(self.routings):\n            b = b.permute(0, 2, 1)\n            c = F.softmax(b, dim=2)\n            c = c.permute(0, 2, 1)\n            b = b.permute(0, 2, 1)\n            outputs = self.activation(t.einsum('bij,bijk->bik', (c, u_hat_vecs)))  # batch matrix multiplication\n            # outputs shape (batch_size, num_capsule, dim_capsule)\n            if i < self.routings - 1:\n                b = t.einsum('bik,bijk->bij', (outputs, u_hat_vecs))  # batch matrix multiplication\n        return outputs  # (batch_size, num_capsule, dim_capsule)\n\n    # text version of squash, slight different from original one\n    def squash(self, x, axis=-1):\n        s_squared_norm = (x ** 2).sum(axis, keepdim=True)\n        scale = t.sqrt(s_squared_norm + T_epsilon)\n        return x / scale\n    \nclass Capsule_Main(nn.Module):\n    def __init__(self, embedding_matrix=None, vocab_size=None):\n        super(Capsule_Main, self).__init__()\n        self.embed_layer = Embed_Layer(embedding_matrix, vocab_size)\n        self.gru_layer = GRU_Layer()\n        # 【重要】初始化GRU权重操作，这一步非常关键，acc上升到0.98，如果用默认的uniform初始化则acc一直在0.5左右\n        self.gru_layer.init_weights()\n        self.caps_layer = Caps_Layer()\n        self.dense_layer = Dense_Layer()\n\n    def forward(self, content):\n        content1 = self.embed_layer(content)\n        content2, _ = self.gru_layer(\n            content1)  # 这个输出是个tuple，一个output(seq_len, batch_size, num_directions * hidden_size)，一个hn\n        content3 = self.caps_layer(content2)\n        output = self.dense_layer(content3)\n        return output\n    \n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"170b27f66c7e546d8e84c79357d40f86c6b1ec42","scrolled":true,"trusted":false},"cell_type":"code","source":"class Attention(nn.Module):\n    def __init__(self, feature_dim, step_dim, bias=True, **kwargs):\n        super(Attention, self).__init__(**kwargs)\n        \n        self.supports_masking = True\n\n        self.bias = bias\n        self.feature_dim = feature_dim\n        self.step_dim = step_dim\n        self.features_dim = 0\n        \n        weight = torch.zeros(feature_dim, 1)\n        nn.init.xavier_uniform_(weight)\n        self.weight = nn.Parameter(weight)\n        \n        if bias:\n            self.b = nn.Parameter(torch.zeros(step_dim))\n        \n    def forward(self, x, mask=None):\n        feature_dim = self.feature_dim\n        step_dim = self.step_dim\n\n        eij = torch.mm(\n            x.contiguous().view(-1, feature_dim), \n            self.weight\n        ).view(-1, step_dim)\n        \n        if self.bias:\n            eij = eij + self.b\n            \n        eij = torch.tanh(eij)\n        a = torch.exp(eij)\n        \n        if mask is not None:\n            a = a * mask\n\n        a = a / torch.sum(a, 1, keepdim=True) + 1e-10\n\n        weighted_input = x * torch.unsqueeze(a, -1)\n        return torch.sum(weighted_input, 1)\n    \nclass NeuralNet(nn.Module):\n    def __init__(self):\n        super(NeuralNet, self).__init__()\n        \n        fc_layer = 16\n        fc_layer1 = 16\n\n        self.embedding = nn.Embedding(max_features, embed_size)\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        self.embedding.weight.requires_grad = False\n        \n        self.embedding_dropout = nn.Dropout2d(0.1)\n        self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)\n        self.gru = nn.GRU(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)\n\n        self.lstm2 = nn.LSTM(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)\n\n        self.lstm_attention = Attention(hidden_size * 2, maxlen)\n        self.gru_attention = Attention(hidden_size * 2, maxlen)\n        self.bn = nn.BatchNorm1d(16, momentum=0.5)\n        self.linear = nn.Linear(hidden_size*8+3, fc_layer1) #643:80 - 483:60 - 323:40\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(0.1)\n        self.fc = nn.Linear(fc_layer**2,fc_layer)\n        self.out = nn.Linear(fc_layer, 1)\n        self.lincaps = nn.Linear(Num_capsule * Dim_capsule, 1)\n        self.caps_layer = Caps_Layer()\n    \n    def forward(self, x):\n        \n#         Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True)(x)\n\n        h_embedding = self.embedding(x[0])\n        h_embedding = torch.squeeze(\n            self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))\n        \n        h_lstm, _ = self.lstm(h_embedding)\n        h_gru, _ = self.gru(h_lstm)\n\n        ##Capsule Layer        \n        content3 = self.caps_layer(h_gru)\n        content3 = self.dropout(content3)\n        batch_size = content3.size(0)\n        content3 = content3.view(batch_size, -1)\n        content3 = self.relu(self.lincaps(content3))\n\n        ##Attention Layer\n        h_lstm_atten = self.lstm_attention(h_lstm)\n        h_gru_atten = self.gru_attention(h_gru)\n        \n        # global average pooling\n        avg_pool = torch.mean(h_gru, 1)\n        # global max pooling\n        max_pool, _ = torch.max(h_gru, 1)\n        \n        f = torch.tensor(x[1], dtype=torch.float).cuda()\n\n                #[512,160]\n        conc = torch.cat((h_lstm_atten, h_gru_atten,content3, avg_pool, max_pool,f), 1)\n        conc = self.relu(self.linear(conc))\n        conc = self.bn(conc)\n        conc = self.dropout(conc)\n\n        out = self.out(conc)\n        \n        return out","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e4e47597cde552a41cdd8ec2531aa6a861e491ae"},"cell_type":"markdown","source":"### Training\n\nThe method for training is borrowed from https://www.kaggle.com/hengzheng/pytorch-starter"},{"metadata":{"_uuid":"704fcb82f5f25d6349a487409f519384df352375","trusted":false},"cell_type":"code","source":"class MyDataset(Dataset):\n    def __init__(self,dataset):\n        self.dataset = dataset\n\n    def __getitem__(self, index):\n        data, target = self.dataset[index]\n\n        return data, target, index\n    def __len__(self):\n        return len(self.dataset)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0c6339acdc14e6688ed47e9f439e81cd7cfca57f","trusted":false},"cell_type":"code","source":"def sigmoid(x):\n    return 1 / (1 + np.exp(-x))\n\n# matrix for the out-of-fold predictions\ntrain_preds = np.zeros((len(x_train)))\n# matrix for the predictions on the test set\ntest_preds = np.zeros((len(df_test)))\n\n# always call this before training for deterministic results\nseed_everything()\n\n# x_test_cuda_f = torch.tensor(x_test_f, dtype=torch.long).cuda()\n# test_f = torch.utils.data.TensorDataset(x_test_cuda_f)\n# test_loader_f = torch.utils.data.DataLoader(test_f, batch_size=batch_size, shuffle=False)\n\n\nx_test_cuda = torch.tensor(x_test, dtype=torch.long).cuda()\ntest = torch.utils.data.TensorDataset(x_test_cuda)\ntest_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)\n\navg_losses_f = []\navg_val_losses_f = []","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2cfdd46b117a11af54ed01f1f9511782e50a00fa","scrolled":true,"trusted":false},"cell_type":"code","source":"for i, (train_idx, valid_idx) in enumerate(splits):    \n    # split data in train / validation according to the KFold indeces\n    # also, convert them to a torch tensor and store them on the GPU (done with .cuda())\n    x_train = np.array(x_train)\n    y_train = np.array(y_train)\n    features = np.array(features)\n\n    x_train_fold = torch.tensor(x_train[train_idx.astype(int)], dtype=torch.long).cuda()\n    y_train_fold = torch.tensor(y_train[train_idx.astype(int), np.newaxis], dtype=torch.float32).cuda()\n    \n    kfold_X_features = features[train_idx.astype(int)]\n    kfold_X_valid_features = features[valid_idx.astype(int)]\n    x_val_fold = torch.tensor(x_train[valid_idx.astype(int)], dtype=torch.long).cuda()\n    y_val_fold = torch.tensor(y_train[valid_idx.astype(int), np.newaxis], dtype=torch.float32).cuda()\n    \n#     model = BiLSTM(lstm_layer=2,hidden_dim=40,dropout=DROPOUT).cuda()\n    model = NeuralNet()\n\n    # make sure everything in the model is running on the GPU\n    model.cuda()\n\n    # define binary cross entropy loss\n    # note that the model returns logit to take advantage of the log-sum-exp trick \n    # for numerical stability in the loss\n    loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum')\n\n    step_size = 300\n    base_lr, max_lr = 0.001, 0.003   \n    optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), \n                             lr=max_lr)\n    \n    ################################################################################################\n    scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,\n               step_size=step_size, mode='exp_range',\n               gamma=0.99994)\n    ###############################################################################################\n\n    train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)\n    valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)\n    \n    train = MyDataset(train)\n    valid = MyDataset(valid)\n\n    ##No need to shuffle the data again here. Shuffling happens when splitting for kfolds.\n    train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n    \n    valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)\n\n    print(f'Fold {i + 1}')\n    for epoch in range(n_epochs):\n        # set train mode of the model. This enables operations which are only applied during training like dropout\n        start_time = time.time()\n        model.train()\n\n        avg_loss = 0.  \n        for i, (x_batch, y_batch, index) in enumerate(train_loader):\n            # Forward pass: compute predicted y by passing x to the model.\n            ################################################################################################            \n            f = kfold_X_features[index]\n            y_pred = model([x_batch,f])\n            ################################################################################################\n\n            ################################################################################################\n\n            if scheduler:\n                scheduler.batch_step()\n            ################################################################################################\n\n\n            # Compute and print loss.\n            loss = loss_fn(y_pred, y_batch)\n\n            # Before the backward pass, use the optimizer object to zero all of the\n            # gradients for the Tensors it will update (which are the learnable weights\n            # of the model)\n            optimizer.zero_grad()\n\n            # Backward pass: compute gradient of the loss with respect to model parameters\n            loss.backward()\n\n            # Calling the step function on an Optimizer makes an update to its parameters\n            optimizer.step()\n            avg_loss += loss.item() / len(train_loader)\n            \n        # set evaluation mode of the model. This disabled operations which are only applied during training like dropout\n        model.eval()\n        \n        # predict all the samples in y_val_fold batch per batch\n        valid_preds_fold = np.zeros((x_val_fold.size(0)))\n        test_preds_fold = np.zeros((len(df_test)))\n        \n        avg_val_loss = 0.\n        for i, (x_batch, y_batch, index) in enumerate(valid_loader):\n            f = kfold_X_valid_features[index]\n            y_pred = model([x_batch,f]).detach()\n            \n            avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n            valid_preds_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n        \n        elapsed_time = time.time() - start_time \n        print('Epoch {}/{} \\t loss={:.4f} \\t val_loss={:.4f} \\t time={:.2f}s'.format(\n            epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))\n    avg_losses_f.append(avg_loss)\n    avg_val_losses_f.append(avg_val_loss) \n    # predict all samples in the test set batch per batch\n    for i, (x_batch,) in enumerate(test_loader):\n        f = test_features[i * batch_size:(i+1) * batch_size]\n        y_pred = model([x_batch,f]).detach()\n\n        test_preds_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n        \n    train_preds[valid_idx] = valid_preds_fold\n    test_preds += test_preds_fold / len(splits)\n\nprint('All \\t loss={:.4f} \\t val_loss={:.4f} \\t '.format(np.average(avg_losses_f),np.average(avg_val_losses_f)))\n\n# x_train, x_test_f, y_train, y_test_f","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1d2187d4bbf48350eaf365b6dd8b027d5be69e0c"},"cell_type":"markdown","source":"### Find final Thresshold\n\nBorrowed from: https://www.kaggle.com/ziliwang/baseline-pytorch-bilstm"},{"metadata":{"_uuid":"dc4a4c681294ba06526fed4e871cfe8639cf25e4","scrolled":true,"trusted":false},"cell_type":"code","source":"def bestThresshold(y_train,train_preds):\n    tmp = [0,0,0] # idx, cur, max\n    delta = 0\n    for tmp[0] in tqdm(np.arange(0.1, 0.501, 0.01)):\n        tmp[1] = f1_score(y_train, np.array(train_preds)>tmp[0])\n        if tmp[1] > tmp[2]:\n            delta = tmp[0]\n            tmp[2] = tmp[1]\n    print('best threshold is {:.4f} with F1 score: {:.4f}'.format(delta, tmp[2]))\n    return delta\ndelta = bestThresshold(y_train,train_preds)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d17dd7b0a92ec98134bf8996fd210edaadf7bed6","scrolled":true,"trusted":false},"cell_type":"code","source":"submission = df_test[['qid']].copy()\nsubmission['prediction'] = (test_preds > delta).astype(int)\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a1a07a65e47b5dcefaac040f3f633dc077e8f61e","scrolled":true,"trusted":false},"cell_type":"code","source":"!head submission.csv","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"06b7a7eda89cc6520497f1ec6582f506b8c2f2fd","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.4"}},"nbformat":4,"nbformat_minor":4}