{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import re\nimport time\nimport gc\nimport random\nimport os\nimport math\n\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold\nfrom sklearn.metrics import f1_score, precision_recall_fscore_support\n\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\n\nimport torch\nimport torch.nn as nn\nimport torch.utils.data\nimport torch.nn.functional as F","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def seed_torch(seed=1130):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = False\n\nSEED = 1130\nseed_torch(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85ee6b26944da2cd972c5038fa63c78b763184d3"},"cell_type":"code","source":"embed_size = 300 # how big is each word vector\nmax_features = 196534 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 72 # max number of words in a question to use\n\nlog_interval = 250\nbatch_size = 512","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b9e9328b13999617e31a2bd9f137149131c4704c"},"cell_type":"code","source":"import psutil\nfrom multiprocessing import Pool\n\nnum_partitions = 20  # number of partitions to split dataframe\nnum_cores = psutil.cpu_count()  # number of cores on your machine\n\nprint('number of cores:', num_cores)\ndef df_parallelize_run(df, func):\n    df_split = np.array_split(df, num_partitions)\n    pool = Pool(num_cores)\n    df = pd.concat(pool.map(func, df_split))\n    pool.close()\n    pool.join()\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5013e3945f2454854337ff3ee7eee98efa79fd25"},"cell_type":"code","source":"puncts = ['!', '?', '.', ',', '-', \"'\", '\"', ':', ')', '(', '|', ';', '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', '÷', 'π',\n 'है', 'है',           \n         ]\n\npuncts_str = ''.join(puncts)[5:]\n\npuncts = ['!', '?', '.', ',', '-']\ndef clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        if punct in x:\n            x = x.replace(punct, f' {punct} ')\n    return x\n\ndef clean_quotation(x):\n    x = str(x)\n    if \"'s\" in x:\n        x = x.replace(\"'s\", \"\")\n    if \"s'\" in x:\n        x = x.replace(\"s'\", \"\")\n    if \"n't\" in x:\n        x = x.replace(\"n't\", \" not\")\n    if \"'ve\" in x:\n        x = x.replace(\"'ve\", \" have\")\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cced5a1c352bf835e3667d3fa08f2c68b5a18d12"},"cell_type":"code","source":"mispell_dict = {\n    \"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\",\n    \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\",\n    \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\",\n    \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\", \"i'd\": \"i would\", \"i'd've\": \"i would have\",\n    \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\",\n    \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\",\n    \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\",\n    \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\",\n    \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\",\n    \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\",\n    \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\",\n    \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\n    \"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\",\n    \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\n    \"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\",\n    \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\",\n    \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\",\n    \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",\n    \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \n    \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \n    \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\",\n    \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \n    \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\n    \"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\",\n    \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\", 'colour': 'color', 'centre': 'center',\n    'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater',\n    'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize',\n    'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', \n    'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many',\n    'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation',\n    'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', \n    'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', \n    'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', \n    'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization', \n    'quoran': 'quora', 'quorans': 'quora', 'brexit': 'british exit', 'quoras':'quora', 'fortnite': 'video game',\n    'pubg': 'video game', 'redmi': 'cell phone', 'jinping': 'chinese president', 'lyft': 'uber', 'θ': 'theta', 'σ': 'sigma'\n}\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n    return mispellings_re.sub(replace, text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"02128bf327b2bf0e0aada9dcebc950cba664de17"},"cell_type":"code","source":"def preprocess(text):\n    \"\"\"\n    preprocess text main steps\n    \"\"\"\n    text = text.lower()\n    text = clean_text(text)\n    text = replace_typical_misspell(text)\n    text = clean_quotation(text)\n    \n    return text\n\ndef text_clean_wrapper(df):\n    df[\"question_text\"] = df[\"question_text\"].apply(preprocess)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87a5fb67cdd061388568a1cd27250cecf5fb24a7"},"cell_type":"code","source":"def load_and_prec():\n    train_df = pd.read_csv(\"../input/train.csv\")\n    test_df = pd.read_csv(\"../input/test.csv\")\n    print(\"Train shape : \",train_df.shape)\n    print(\"Test shape : \",test_df.shape)\n    \n#     # lower\n#     train_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: x.lower())\n#     test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: x.lower())\n    \n#     # Clean the text\n#     train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_text(x))\n#     test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: clean_text(x))\n    \n#     # Clean spellings\n#     train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\n#     test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\n    \n#     # Clean belong\n#     train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_quotation(x))\n#     test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: clean_quotation(x))\n\n    train_df = df_parallelize_run(train_df, text_clean_wrapper)\n    test_df = df_parallelize_run(test_df, text_clean_wrapper)\n    \n    ## fill up the missing values\n    train_X = train_df[\"question_text\"].fillna(\"_##_\").values\n    test_X = test_df[\"question_text\"].fillna(\"_##_\").values\n\n    ## Tokenize the sentences\n    tokenizer = Tokenizer(num_words=max_features, filters=puncts_str)\n    tokenizer.fit_on_texts(list(train_X))\n    train_X = tokenizer.texts_to_sequences(train_X)\n    test_X = tokenizer.texts_to_sequences(test_X)\n\n    ## Pad the sentences \n    train_X = pad_sequences(train_X, maxlen=maxlen)\n    test_X = pad_sequences(test_X, maxlen=maxlen)\n\n    ## Get the target values\n    train_y = train_df['target'].values\n    \n    #shuffling the data\n    np.random.seed(SEED)\n    trn_idx = np.random.permutation(len(train_X))\n\n    train_X = train_X[trn_idx]\n    train_y = train_y[trn_idx]\n    \n    return train_X, test_X, train_y, tokenizer.word_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9fa4b3016d2edf2c14340de1f23988b086850851"},"cell_type":"code","source":"def load_glove(word_index):\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    emb_mean, emb_std = -0.005838499, 0.48782197\n    \n    embedding_matrix = np.random.normal(emb_mean, emb_std, (len(word_index), embed_size))\n    with open(EMBEDDING_FILE, 'r', encoding=\"utf8\") as f:\n        for line in f:\n            word, vec = line.split(' ', 1)\n            if word not in word_index:\n                continue\n            i = word_index[word]\n            if i >= len(word_index):\n                continue\n            embedding_vector = np.asarray(vec.split(' '), dtype='float32')[:300]\n            if len(embedding_vector) == 300:\n                embedding_matrix[i] = embedding_vector\n    return embedding_matrix\n\ndef load_para(word_index):\n    EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n    emb_mean, emb_std = -0.0053247833, 0.49346462\n    \n    embedding_matrix = np.random.normal(emb_mean, emb_std, (max_features, embed_size))\n    with open(EMBEDDING_FILE, 'r', encoding=\"utf8\", errors='ignore') as f:\n        for line in f:\n            word, vec = line.split(' ', 1)\n            if word not in word_index:\n                continue\n            i = word_index[word]\n            if i >= max_features:\n                continue\n            embedding_vector = np.asarray(vec.split(' '), dtype='float32')[:300]\n            if len(embedding_vector) == 300:\n                embedding_matrix[i] = embedding_vector\n    return embedding_matrix\n\ndef load_fasttext(word_index):\n    EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n    emb_mean, emb_std = -0.0033469985, 0.109855495\n    \n    embedding_matrix = np.random.normal(emb_mean, emb_std, (len(word_index), embed_size))\n    with open(EMBEDDING_FILE, 'r', encoding=\"utf8\") as f:\n        for line in f:\n            word, vec = line.split(' ', 1)\n            if word not in word_index:\n                continue\n            i = word_index[word]\n            if i >= len(word_index):\n                continue\n            embedding_vector = np.asarray(vec.split(' '), dtype='float32')[:300]\n            if len(embedding_vector) == 300:\n                embedding_matrix[i] = embedding_vector\n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06944041d5cad5617c641b73f513c4840eb502fc"},"cell_type":"code","source":"from tqdm import tqdm\ntqdm.pandas()\n\nstart_time = time.time()\n\ntrain_X, test_X, train_y, word_index = load_and_prec()\nembedding_matrix_1 = load_glove(word_index)\nembedding_matrix_2 = load_para(word_index)\nembedding_matrix_3 = load_fasttext(word_index)\n\ntotal_time = (time.time() - start_time) / 60\nprint(\"Took {:.2f} minutes\".format(total_time))\n\n\n# embedding_matrix = np.concatenate((embedding_matrix_1, embedding_matrix_2), axis=1)\n# embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_2], axis=0)\nprint(np.shape(embedding_matrix_1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71eeb24155b28b82e08b89becf8ba2cf3e207d67"},"cell_type":"code","source":"class RNNModel(nn.Module):\n    def __init__(self, rnn_type, input_size, hidden_size):\n        super(RNNModel, self).__init__()\n        self.rnn = getattr(nn, rnn_type)(input_size, hidden_size, bidirectional=True, batch_first=True)\n    \n    def init_weights(self):\n        ih = (param.data for name, param in self.named_parameters() if 'weight_ih' in name)\n        hh = (param.data for name, param in self.named_parameters() if 'weight_hh' in name)\n        b = (param.data for name, param in self.named_parameters() if 'bias' in name)\n        for k in ih:\n            nn.init.xavier_uniform_(k)\n        for k in hh:\n            nn.init.orthogonal_(k)\n        for k in b:\n            nn.init.constant_(k, 0)\n\n    def forward(self, x):\n        return self.rnn(x)\n    \nclass LSTM_TextCNN(nn.Module):\n    def __init__(self, hidden_size, embedding_matrix, embed_size, embedding_dropout=0.5, initialization=True):\n        super(LSTM_TextCNN, self).__init__()\n        \n        self.embedding = nn.Embedding(len(word_index), embed_size, padding_idx=0)\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        self.embedding.weight.requires_grad = False\n        \n        self.embedding_dropout = nn.Dropout2d(embedding_dropout)\n        \n        self.lstm = RNNModel(\"LSTM\", embed_size, hidden_size)\n        self.lstm.init_weights()\n        \n        self.conv1 = nn.Conv1d(hidden_size*2, hidden_size, 1, bias=False)\n        self.bn1 = nn.BatchNorm1d(hidden_size)\n        self.conv2 = nn.Conv1d(hidden_size*2, hidden_size, 2, bias=False)\n        self.bn2 = nn.BatchNorm1d(hidden_size)\n        self.conv3 = nn.Conv1d(hidden_size*2, hidden_size, 3, bias=False)\n        self.bn3 = nn.BatchNorm1d(hidden_size)\n        self.conv4 = nn.Conv1d(hidden_size*2, hidden_size, 4, bias=False)\n        self.bn4 = nn.BatchNorm1d(hidden_size)\n        self.relu = nn.ReLU(inplace=True)\n        \n        self.merge = nn.Linear(hidden_size*4, hidden_size)\n        self.dropout = nn.Dropout(0.1)\n        self.bn_merge = nn.BatchNorm1d(hidden_size)\n        self.out = nn.Linear(hidden_size, 1)\n        \n        if initialization:\n            self.init_weights()\n            \n    def init_weights(self):\n        self.out.bias.data.fill_(0)\n        nn.init.kaiming_normal_(self.out.weight)\n        \n    def forward(self, x):\n        embed = self.embedding(x)\n        out = torch.squeeze(\n            self.embedding_dropout(torch.unsqueeze(embed, 0)))\n        out, _ = self.lstm(embed)\n        \n        out = out.transpose(1, 2)\n        \n        out1 = self.bn1(self.relu(self.conv1(out)))\n        out1 = F.adaptive_max_pool1d(out1, 1)\n        \n        out2 = self.bn2(self.relu(self.conv2(out)))\n        out2 = F.adaptive_max_pool1d(out2, 1)\n        \n        out3 = self.bn3(self.relu(self.conv3(out)))\n        out3 = F.adaptive_max_pool1d(out3, 1)\n        \n        out4 = self.bn4(self.relu(self.conv4(out)))\n        out4 = F.adaptive_max_pool1d(out4, 1)\n        \n        conc = torch.squeeze(torch.cat((out1, out2, out3, out4), 1)) \n        conc = self.relu(self.merge(conc))\n        conc = self.bn_merge(self.dropout(conc))\n        out = self.out(conc)\n        \n        return out\n    \nclass LSTM_GRU(nn.Module):\n    def __init__(self, hidden_size, embedding_matrix, embed_size, embedding_dropout=0.1):\n        super(LSTM_GRU, self).__init__()\n        \n        self.hidden_size = hidden_size\n        self.embedding_matrix = embedding_matrix\n        \n        self.embedding = nn.Embedding(len(word_index), embed_size)\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        self.embedding.weight.requires_grad = False\n        \n        self.embedding_dropout = nn.Dropout2d(embedding_dropout)\n        self.lstm = RNNModel(\"LSTM\", embed_size, hidden_size)\n        self.lstm.init_weights()\n        self.gru = RNNModel(\"GRU\", hidden_size*2, hidden_size)\n        self.gru.init_weights()\n        \n        self.linear = nn.Linear(4*hidden_size, 16)\n        self.relu = nn.ReLU(inplace=True)\n        self.dropout = nn.Dropout(0.0)\n        self.bn = nn.BatchNorm1d(16)\n        self.out = nn.Linear(16, 1)\n        \n    def forward(self, x):\n        h_embedding = self.embedding(x)\n        h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))\n        \n        h_lstm, _ = self.lstm(h_embedding)\n        h_gru, _ = self.gru(h_lstm)\n        \n        avg_pool = torch.mean(h_gru, 1)\n        max_pool, _ = torch.max(h_gru, 1)\n\n        conc = torch.cat((avg_pool, max_pool), 1)\n        conc = self.relu(self.linear(conc))\n        conc = self.bn(self.dropout(conc))\n        out = self.out(conc)\n        \n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4bc17062b48b74557b5e4d7b94014dd3a5f87e48"},"cell_type":"code","source":"splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=SEED).split(train_X, train_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6c179e5478f1ac92b69020fb875bf3d8154bff3"},"cell_type":"code","source":"def sigmoid(x):\n    return 1 / (1 + np.exp(-x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ad4bf11f982e199ff5d64004eeedd9f3d3d454c"},"cell_type":"code","source":"def threshold_search_submit(y_proba, ratio):\n    best_threshold = 0\n    best_score = np.Inf\n    for threshold in tqdm([i * 0.001 for i in range(1000)]):\n        score = np.abs((y_proba > threshold).mean() - ratio)\n        if score < best_score:\n            best_threshold = threshold\n            best_score = score\n    search_result = {'threshold': best_threshold, 'dist': best_score, 'ratio': ratio}\n    return search_result\n\nimport matplotlib.pyplot as plt\n\ndef threshold_search(y_proba, y_true):\n    best_threshold = 0\n    best_score = 0\n    scores = []\n    for threshold in tqdm([i * 0.01 for i in range(100)]):\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n        scores.append(score)\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    \n    plt.figure(figsize=(12,9))\n    plt.plot([i * 0.01 for i in range(100)], scores)\n    plt.plot(best_threshold, best_score, \"xr\", label=\"Best threshold\")\n    plt.show()\n    return search_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c7dc58d062c0aa2f935c3d42c393a18d1d7be653"},"cell_type":"code","source":"from torch.optim.optimizer import Optimizer\nfrom torch.optim.lr_scheduler import _LRScheduler\n\nclass CosineLRWithRestarts():\n    \"\"\"Decays learning rate with cosine annealing, normalizes weight decay\n    hyperparameter value, implements restarts.\n    https://arxiv.org/abs/1711.05101\n    Args:\n        optimizer (Optimizer): Wrapped optimizer.\n        batch_size: minibatch size\n        epoch_size: training samples per epoch\n        restart_period: epoch count in the first restart period\n        t_mult: multiplication factor by which the next restart period will extend/shrink\n    Example:\n        >>> scheduler = CosineLRWithRestarts(optimizer, 32, 1024, restart_period=5, t_mult=1.2)\n        >>> for epoch in range(100):\n        >>>     scheduler.step()\n        >>>     train(...)\n        >>>         ...\n        >>>         optimizer.zero_grad()\n        >>>         loss.backward()\n        >>>         optimizer.step()\n        >>>         scheduler.batch_step()\n        >>>     validate(...)\n    \"\"\"\n\n    def __init__(self, optimizer, batch_size, epoch_size, restart_period=100,\n                 t_mult=2, last_epoch=-1, eta_threshold=1000, verbose=False):\n        if not isinstance(optimizer, Optimizer):\n            raise TypeError('{} is not an Optimizer'.format(\n                type(optimizer).__name__))\n        self.optimizer = optimizer\n        if last_epoch == -1:\n            for group in optimizer.param_groups:\n                group.setdefault('initial_lr', group['lr'])\n        else:\n            for i, group in enumerate(optimizer.param_groups):\n                if 'initial_lr' not in group:\n                    raise KeyError(\"param 'initial_lr' is not specified \"\n                                   \"in param_groups[{}] when resuming an\"\n                                   \" optimizer\".format(i))\n        self.base_lrs = list(map(lambda group: group['initial_lr'],\n                                 optimizer.param_groups))\n\n        self.last_epoch = last_epoch\n        self.batch_size = batch_size\n        self.epoch_size = epoch_size\n        self.eta_threshold = eta_threshold\n        self.t_mult = t_mult\n        self.verbose = verbose\n        self.base_weight_decays = list(map(lambda group: group['weight_decay'],\n                                           optimizer.param_groups))\n        self.restart_period = restart_period\n        self.restarts = 0\n        self.t_epoch = -1\n\n    def _schedule_eta(self):\n        \"\"\"\n        Threshold value could be adjusted to shrink eta_min and eta_max values.\n        \"\"\"\n        eta_min = 0\n        eta_max = 1\n        if self.restarts <= self.eta_threshold:\n            return eta_min, eta_max\n        else:\n            d = self.restarts - self.eta_threshold\n            k = d * 0.09\n            return (eta_min + k, eta_max - k)\n\n    def get_lr(self, t_cur):\n        eta_min, eta_max = self._schedule_eta()\n\n        eta_t = (eta_min + 0.5 * (eta_max - eta_min)\n                 * (1. + math.cos(math.pi *\n                                  (t_cur / self.restart_period))))\n\n        weight_decay_norm_multi = math.sqrt(self.batch_size /\n                                            (self.epoch_size *\n                                             self.restart_period))\n        lrs = [base_lr * eta_t for base_lr in self.base_lrs]\n        weight_decays = [base_weight_decay * eta_t * weight_decay_norm_multi\n                         for base_weight_decay in self.base_weight_decays]\n\n        if self.t_epoch % self.restart_period < self.t_epoch:\n            if self.verbose:\n                print(\"Restart at epoch {}\".format(self.last_epoch))\n            self.restart_period *= self.t_mult\n            self.restarts += 1\n            self.t_epoch = 0\n\n        return zip(lrs, weight_decays)\n\n    def _set_batch_size(self):\n        d, r = divmod(self.epoch_size, self.batch_size)\n        batches_in_epoch = d + 2 if r > 0 else d + 1\n        self.batch_increment = iter(torch.linspace(0, 1, batches_in_epoch))\n\n    def step(self):\n        self.last_epoch += 1\n        self.t_epoch += 1\n        self._set_batch_size()\n        self.batch_step()\n\n    def batch_step(self):\n        t_cur = self.t_epoch + next(self.batch_increment)\n        for param_group, (lr, weight_decay) in zip(self.optimizer.param_groups,\n                                                   self.get_lr(t_cur)):\n            param_group['lr'] = lr\n            param_group['weight_decay'] = weight_decay","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9789b20059489a40da82e8da1f1af646b9153b79"},"cell_type":"code","source":"def f1_loss(logits, labels):\n    __small_value = 1e-6\n    beta = 1\n    batch_size = logits.size()[0]\n    p = torch.sigmoid(logits)\n    l = labels\n    num_pos = torch.sum(p, 1) + __small_value\n    num_pos_hat = torch.sum(l, 1) + __small_value\n    tp = torch.sum(l * p, 1)\n    precise = tp / num_pos\n    recall = tp / num_pos_hat\n    fs = (1 + beta * beta) * precise * recall / (beta * beta * precise + recall + __small_value)\n    loss = fs.sum() / batch_size\n    return (1 - loss)\n\nclass FocalLoss(nn.Module):\n    def __init__(self, gamma=2):\n        super().__init__()\n        self.gamma = gamma\n\n    def forward(self, logit, target):\n        max_val = (-logit).clamp(min=0)\n        loss = logit - logit * target + max_val + ((-max_val).exp() + (-logit - max_val).exp()).log()\n\n        invprobs = F.logsigmoid(-logit * (target * 2.0 - 1.0))\n        loss = (invprobs * self.gamma).exp() * loss\n        if len(loss.size())==2:\n            loss = loss.sum(dim=1)\n        return loss.mean()\n\ndef lovasz_grad(gt_sorted):\n    \"\"\"\n    Computes gradient of the Lovasz extension w.r.t sorted errors\n    See Alg. 1 in paper\n    \"\"\"\n    p = len(gt_sorted)\n    gts = gt_sorted.sum()\n    intersection = gts - gt_sorted.float().cumsum(0)\n    union = gts + (1 - gt_sorted).float().cumsum(0)\n    jaccard = 1. - intersection / union\n    if p > 1: # cover 1-pixel case\n        jaccard[1:p] = jaccard[1:p] - jaccard[0:-1]\n    return jaccard    \n\ndef lovasz_hinge_flat(logits, labels):\n    \"\"\"\n    Binary Lovasz hinge loss\n      logits: [P] Variable, logits at each prediction (between -\\infty and +\\infty)\n      labels: [P] Tensor, binary ground truth labels (0 or 1)\n      ignore: label to ignore\n    \"\"\"\n    logits = logits.squeeze()\n    labels = labels.squeeze()\n    \n    if len(labels) == 0:\n        # only void pixels, the gradients should be 0\n        return logits.sum() * 0.\n    signs = 2. * labels.float() - 1.\n    errors = (1. - logits * Variable(signs))\n    errors_sorted, perm = torch.sort(errors, dim=0, descending=True) # bug\n    perm = perm.data\n    gt_sorted = labels[perm]\n    grad = lovasz_grad(gt_sorted)\n    # loss = torch.dot(F.relu(errors_sorted), Variable(grad))\n    loss = torch.dot(F.elu(errors_sorted) + 1, Variable(grad))\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"92ed04a1263f211df138623c2dd878eb26336881"},"cell_type":"code","source":"train_epochs = 4\ntrain_preds = np.zeros((len(train_X)))\ntest_preds = np.zeros((len(test_X)))\n\nx_test_cuda = torch.tensor(test_X, dtype=torch.long).cuda()\ntest = torch.utils.data.TensorDataset(x_test_cuda)\ntest_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4d4e3f2aebdbf569c8e777ad81eed0de95d1926"},"cell_type":"code","source":"for i, (train_idx, valid_idx) in enumerate(splits):\n    x_train_fold = torch.tensor(train_X[train_idx], dtype=torch.long).cuda()\n    y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32).cuda()\n    x_val_fold = torch.tensor(train_X[valid_idx], dtype=torch.long).cuda()\n    y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32).cuda()\n    \n    model1 = LSTM_TextCNN(64, np.concatenate((embedding_matrix_1, embedding_matrix_3), axis=1), embed_size*2, embedding_dropout=0.5)\n    model2 = LSTM_GRU(64, np.mean([embedding_matrix_1, embedding_matrix_2], axis=0), embed_size, embedding_dropout=0.1)\n    \n    model1.cuda()\n    model2.cuda()\n    \n    loss_fn1 = torch.nn.BCEWithLogitsLoss()\n    loss_fn2 = f1_loss\n    \n    optimizer1 = torch.optim.Adam(model1.parameters(), lr=0.0035)\n    scheduler1 = CosineLRWithRestarts(optimizer1, batch_size, len(x_train_fold), restart_period=4, t_mult=1, verbose=True)\n    optimizer2 = torch.optim.Adam(model2.parameters(), lr=0.0035)\n    scheduler2 = CosineLRWithRestarts(optimizer2, batch_size, len(x_train_fold), restart_period=4, t_mult=1, verbose=True)\n    \n    train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)\n    valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)\n    \n    train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n    valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)\n    \n    print(f'Fold {i + 1}')\n    \n    for epoch in range(train_epochs):\n        start_time = time.time()\n        scheduler1.step()\n        scheduler2.step()\n        \n        model1.train()\n        model2.train()\n        \n        for batch_idx, (x_batch, y_batch) in enumerate(train_loader):\n            y_pred1 = model1(x_batch)\n            y_pred2 = model2(x_batch)\n            \n            loss1 = loss_fn1(y_pred1, y_batch) + loss_fn2(y_pred1, y_batch)\n            loss2 = loss_fn1(y_pred2, y_batch) + loss_fn2(y_pred2, y_batch)\n            \n            optimizer1.zero_grad()\n            optimizer2.zero_grad()\n            \n            loss1.backward()\n            loss2.backward()\n            \n            optimizer1.step()\n            optimizer2.step()\n            \n            scheduler1.batch_step()\n            scheduler2.batch_step()\n            \n        model1.eval()\n        model2.eval()\n        \n        valid_preds_fold = np.zeros((x_val_fold.size(0)))\n        test_preds_fold = np.zeros(len(test_X))\n        \n        for i, (x_batch, y_batch) in enumerate(valid_loader):\n            y_pred1 = model1(x_batch).detach()\n            y_pred2 = model2(x_batch).detach()\n            valid_preds_fold[i * batch_size:(i+1) * batch_size] = (\n                sigmoid(y_pred1.cpu().numpy())[:, 0] + \n                sigmoid(y_pred2.cpu().numpy())[:, 0]\n            ) / 2\n        \n        search_result = threshold_search(valid_preds_fold, train_y[valid_idx])\n        valid_yhat_fold = (valid_preds_fold > search_result['threshold']).astype(int)\n        valid_precision, valid_recall, valid_fscore, _ = precision_recall_fscore_support(train_y[valid_idx], valid_yhat_fold, average='binary')\n        \n        elapsed_time = time.time() - start_time \n        print('Epoch {}/{} | val_precision={:.5f} | val_recall={:.5f} | val_f1={:.5f} | threshold={:.2f} | time={:.2f}s'.format(\n            epoch, train_epochs - 1, valid_precision, valid_recall, valid_fscore, search_result['threshold'], elapsed_time))\n        \n    for i, (x_batch,) in enumerate(test_loader):\n        y_pred1 = model1(x_batch).detach()\n        y_pred2 = model2(x_batch).detach()\n        test_preds_fold[i * batch_size:(i+1) * batch_size] = (\n            sigmoid(y_pred1.cpu().numpy())[:, 0] +\n            sigmoid(y_pred2.cpu().numpy())[:, 0]\n        ) / 2\n    \n    train_preds[valid_idx] = valid_preds_fold\n    test_preds += test_preds_fold / len(splits)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3543044c4e3b9f37de8e31af18f990078e0f36bf"},"cell_type":"code","source":"search_result = threshold_search(train_preds, train_y)\ntrain_yhat = (train_preds > search_result['threshold']).astype(int)\ntrain_precision, train_recall, train_fscore, _ = precision_recall_fscore_support(train_y, train_yhat, average='binary')\nprint('oof_precision={:.5f} | oof_recall={:.5f} | oof_f1={:.5f} | threshold={:.2f}'.format(train_precision, train_recall, train_fscore, search_result['threshold']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"27d312bb54c49c5c937817c82067162d53de6b01"},"cell_type":"code","source":"ratio = (train_yhat.mean() + (test_preds > search_result['threshold']).mean()) / 2\n\nsearch_result = threshold_search_submit(test_preds, ratio)\nprint(search_result)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c3581f74ae694eb07182e2a23c9db8f01a78f1ba"},"cell_type":"code","source":"sub = pd.read_csv('../input/sample_submission.csv')\nsub.prediction = test_preds > search_result['threshold']\nsub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}