{"cells":[{"metadata":{"trusted":true,"_uuid":"a7fe0e6e0364e1e7e0570b0563dc5be69632430a"},"cell_type":"code","source":"FOLD = 0\n\nimport os\nimport random\nimport glob\nimport gc\n\nimport numpy as np\nimport pandas as pd\n\nimport mlcrate as mlc\n\nimport os\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import f1_score\n\nfrom tqdm import tqdm, tqdm_notebook\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom torch import optim\nimport torch.utils.checkpoint as checkpoint\n\nimport torchtext\nimport torchtext.data as data\n\nfrom torch.nn.utils.rnn import pad_sequence\n\nimport spacy\nfrom spacy.lang.en import English\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nSEED = 1337\n\nNOTIFY_EACH_EPOCH = False\n\nWORKERS = 0\nBATCH_SIZE = 512\n\nN_SPLITS = 5\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\ndef get_n_params(model):\n    pp=0\n    for p in list(model.parameters()):\n        nn=1\n        for s in list(p.size()):\n            nn = nn*s\n        pp += nn\n    return pp\n\n# from https://github.com/floydhub/save-and-resume\ndef save_checkpoint(state):\n    \"\"\"Save checkpoint if a new best is achieved\"\"\"\n    print (\" Saving checkpoint\")\n\n    filename = f'./checkpoint-{state[\"epoch\"]}.pt.tar'\n    torch.save(state, filename)\n    \ndef initialize(model, path=None, optimizer=None):   \n    if path == None:\n        checkpoints = glob.glob('./*.pt.tar')\n        path = checkpoints[np.argmax([int(checkpoint.split('checkpoint-')[1].split('.')[0]) for checkpoint in checkpoints])]\n\n    checkpoint = torch.load(path)\n\n    model.load_state_dict(checkpoint['model'])\n\n    print(f' Loaded checkpoint {path} | Trained for {checkpoint[\"epoch\"] + 1} epochs')\n    \n    if optimizer:\n        optimizer.load_state_dict(checkpoint['optimizer'])\n          \n        epoch = checkpoint['epoch'] + 1\n        train_iteration = checkpoint['train_iteration']\n        val_iteration = checkpoint['val_iteration']\n\n        return model, optimizer, epoch, train_iteration, val_iteration\n    else:\n        return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"321593e69c05e023da01c4019576ba09e34cafd1"},"cell_type":"code","source":"sample_submission = pd.read_csv('../input/sample_submission.csv')\ntrain = pd.read_csv('../input/train.csv')#.loc[:1000]\ntest = pd.read_csv('../input/test.csv')#.loc[:1000]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e50a1c48d9631c9a12d0a8ae8e9b8b2902a63ebc"},"cell_type":"code","source":"# From https://www.kaggle.com/spirosrap/bilstm-attention-kfold-clr-extra-features-capsule/notebook\ndef add_features(df):\n    df2 = df.copy(deep=True)\n    df2['question_text'] = df2['question_text'].apply(lambda x:str(x))\n    df2['total_length'] = df2['question_text'].apply(len)\n    df2['capitals'] = df2['question_text'].apply(lambda comment: sum(1 for c in comment if c.isupper()))\n    df2['caps_vs_length'] = df2.apply(lambda row: float(row['capitals'])/float(row['total_length']), axis=1)\n    df2['num_words'] = df2.question_text.str.count('\\S+')\n    df2['num_unique_words'] = df2['question_text'].apply(lambda comment: len(set(w for w in comment.split())))\n    df2['words_vs_unique'] = df2['num_unique_words'] / df2['num_words']  \n\n    return np.concatenate((df2['caps_vs_length'].values.reshape(-1, 1), df2['words_vs_unique'].values.reshape(-1, 1)), axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b9ed2b5d49999dd93a312ea591ef38030b8a86bc"},"cell_type":"code","source":"%%time\ntest = train.iloc[:56000+376000]\ntrain = train.iloc[56000+376000:]\ntest_1 = test.iloc[:56000]\ntest_2 = test.iloc[56000:]\n# test = train.iloc[:560+376]\n# train = train.iloc[560+376:1500]\n# test_1 = test.iloc[:560]\n# test_2 = test.iloc[560:]\n\ntrain_meta = add_features(train)\ntest_1_meta = add_features(test_1)\ntest_2_meta = add_features(test_2)\n\ntrain = train.reset_index()\ntest_1 = test_1.reset_index()\ntest_2 = test_2.reset_index()\n\ntrain.to_csv('train.csv')\ntest_1.to_csv('test_1.csv')\ntest_2.to_csv('test_2.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da32027e482bc113c9feac0e363bc5f705e55ada"},"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\','•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4c7e4854b60b1ff433abe2c2082542f58acb43a5"},"cell_type":"code","source":"nlp = English()\ndef tokenize(sentence):\n#     sentence = sentence.split()\n#     return sentence\n\n    sentence = str(sentence)\n    for punct in puncts:\n        sentence = sentence.replace(punct, f' {punct} ')\n        \n    x = nlp(sentence)\n    return [token.text for token in x]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d52cc97f05e431151f7f488fa39a3ae014610b88","scrolled":false},"cell_type":"code","source":"%%time\n# from http://mlexplained.com/2018/02/08/a-comprehensive-tutorial-to-torchtext\nindex_field = data.Field(sequential=False, use_vocab=False, batch_first=True)\nquestion_field = data.Field(tokenize=tokenize, lower=True, batch_first=True, include_lengths=True)\ntarget_field = data.Field(sequential=False, use_vocab=False, batch_first=True)\n\ntrain_fields = [\n    ('id', index_field),\n    ('index', None),\n    ('qid', None),\n    ('question_text', question_field),\n    ('target', target_field)\n]\n\ntest_fields = [\n    ('id', index_field),\n    ('index', None),\n    ('qid', None),\n    ('question_text', question_field)\n]\n\ntotal_train_dataset = data.TabularDataset('./train.csv', format='CSV', skip_header=True, fields=train_fields)\ntest_1_dataset = data.TabularDataset('./test_1.csv', format='CSV', skip_header=True, fields=train_fields)\ntest_2_dataset = data.TabularDataset('./test_2.csv', format='CSV', skip_header=True, fields=train_fields)\n\nquestion_field.build_vocab(total_train_dataset, max_size=95000)\n\ntest_1_dataloader = data.BucketIterator(test_1_dataset, 512, sort=False, shuffle=False)\ntest_2_dataloader = data.BucketIterator(test_2_dataset, 512, sort=False, shuffle=False)\n\nprint(f'Total Train Dataset: {len(total_train_dataset)}')\nprint(f'Test 1 Dataset: {len(test_1_dataset)}')\nprint(f'Test 2 Dataset: {len(test_2_dataset)}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dda7d32355775e045d56ae70f793e55bacbc0651"},"cell_type":"code","source":"# from https://discuss.pytorch.org/t/self-attention-on-words-and-masking/5671/4\nclass SelfAttention(nn.Module):\n    def __init__(self, hidden_size, batch_first=False):\n        super(SelfAttention, self).__init__()\n\n        self.hidden_size = hidden_size\n        self.batch_first = batch_first\n\n        self.att_weights = nn.Parameter(torch.Tensor(1, hidden_size), requires_grad=True)\n        nn.init.xavier_uniform_(self.att_weights.data)\n\n    def get_mask(self):\n        pass\n\n    def forward(self, inputs, lengths):\n        if self.batch_first:\n            batch_size, max_len = inputs.size()[:2]\n        else:\n            max_len, batch_size = inputs.size()[:2]\n            \n        # apply attention layer\n        weights = torch.bmm(inputs,\n                            self.att_weights  # (1, hidden_size)\n                            .permute(1, 0)  # (hidden_size, 1)\n                            .unsqueeze(0)  # (1, hidden_size, 1)\n                            .repeat(batch_size, 1, 1) # (batch_size, hidden_size, 1)\n                            )\n    \n        attentions = torch.softmax(F.relu(weights.squeeze()), dim=-1)\n        \n        # create mask based on the sentence lengths\n        mask = torch.ones(attentions.size(), requires_grad=True).cuda()\n        for i, l in enumerate(lengths):  # skip the first sentence\n            if l < max_len:\n                mask[i, l:] = 0\n\n        # apply mask and renormalize attention scores (weights)\n        masked = attentions * mask\n        _sums = masked.sum(-1).unsqueeze(-1)  # sums per row\n        \n        attentions = masked.div(_sums)\n\n        # apply attention weights\n        weighted = torch.mul(inputs, attentions.unsqueeze(-1).expand_as(inputs))\n\n        # get the final fixed vector representations of the sentences\n        representations = weighted.sum(1).squeeze()\n\n        return representations, attentions\n\nclass net(nn.Module):\n    def __init__(self, embedding):\n        super(net, self).__init__()\n                \n        self.embedding = nn.Embedding.from_pretrained(embedding)\n        \n        self.skip_lstm = nn.LSTM(input_size=300, hidden_size=128, num_layers=1, batch_first=True, bidirectional=True)        \n        self.skip_attention = SelfAttention(128*2, batch_first=True)\n\n        self.lstm = nn.LSTM(input_size=300, hidden_size=128, num_layers=1, batch_first=True, bidirectional=True)\n        self.attention = SelfAttention(128*2, batch_first=True)\n        \n        self.avg_pool = nn.AdaptiveAvgPool1d(1)\n        self.max_pool = nn.AdaptiveMaxPool1d(1)\n        \n        self.fc = nn.Linear(128*2*4+2, 1)\n        self.logit = nn.Linear(1, 1)\n\n    def forward(self, x, x_meta, x_len):\n        x = self.embedding(x)\n        x = nn.utils.rnn.pack_padded_sequence(x, x_len, batch_first=True)\n\n        skip_lstm, _ = self.skip_lstm(x)\n        lstm, _ = self.lstm(x)\n        \n        lstm, lstm_lengths = nn.utils.rnn.pad_packed_sequence(lstm, batch_first=True)\n        skip_lstm, skip_lstm_lengths = nn.utils.rnn.pad_packed_sequence(skip_lstm, batch_first=True)\n        \n        skip_attention, _ = self.skip_attention(skip_lstm, skip_lstm_lengths)\n        attention, _ = self.attention(lstm, lstm_lengths)\n        \n        avg_pool = self.avg_pool(lstm.transpose(1, 2))\n        max_pool = self.max_pool(lstm.transpose(1, 2))\n                                \n        x = torch.cat([\n            skip_attention.view(-1, 128*2),\n            attention.view(-1, 128*2),\n            avg_pool.view(-1, 128*2),\n            max_pool.view(-1, 128*2),\n            x_meta.view(-1, 2)\n        ], dim=1)\n                \n        x = self.fc(x)\n        x = self.logit(x).view(-1)\n        \n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"deae11480f02a2f9907cc77248ecf3e6630e25d9"},"cell_type":"code","source":"def choose_threshold(val_preds, y_val):\n    thresholds = np.arange(0.1, 0.501, 0.01)\n\n    val_scores = []\n    for threshold in thresholds:\n        threshold = np.round(threshold, 2)\n        f1 = f1_score(y_val, (val_preds > threshold).astype(int))\n        val_scores.append(f1)\n\n    best_val_f1 = np.max(val_scores)\n    best_threshold = np.round(thresholds[np.argmax(val_scores)], 2)\n    \n    return best_threshold, best_val_f1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f2258a11577dce6342a7af9a271751d40f8fdfb9"},"cell_type":"code","source":"def test(net, question_field, test_dataloader, test_meta):\n    model = net(question_field.vocab.vectors).to(device)\n    model = initialize(model)\n\n    preds = []\n    y_test = []\n\n    model.eval()\n    with torch.no_grad():\n        for i, batch in tqdm_notebook(enumerate(test_dataloader)):\n            (questions, lengths), index, label = batch.question_text, batch.id, batch.target\n\n            # sort questions and lengths in descending order\n            indices = torch.argsort(lengths, descending=True)\n\n            questions = questions[indices]\n            index = index[indices]\n            lengths = lengths[indices]\n\n            questions = questions.to(device)\n            meta = torch.from_numpy(test_meta[index]).float().to(device)\n\n            out = model(questions, meta, lengths)\n            \n            # unsort outputs\n            indices = torch.argsort(indices)\n            out = out[indices]\n\n            out = torch.sigmoid(out)\n            pred = out.detach().cpu().numpy()\n            \n            preds.append(pred)\n            y_test.append(label.detach().cpu().numpy())\n\n    preds = np.concatenate(preds, axis=0).reshape(-1, 1)\n    y_test = np.concatenate(y_test, axis=0).reshape(-1, 1)\n\n    return preds, y_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"afe61e636a4e3adb7d31efa1b3fb76c09c09ea42","scrolled":false},"cell_type":"code","source":"def train(train_dataset, val_dataset, train_dataloader, val_dataloader, train_meta, net, question_field, vectors, stoi):\n    best_train_loss = 1e10\n    best_val_loss = 1e10\n\n    best_epoch = 0\n    epochs = 4\n\n    timer = mlc.time.Timer()\n    logger = mlc.LinewiseCSVWriter('train_log.csv', header=['epoch', 'lr', 'train_loss', 'val_loss'])\n\n    question_field.vocab.set_vectors(stoi, vectors, 300)\n    \n    model = net(question_field.vocab.vectors).to(device)\n\n    optimizer = optim.Adam(model.parameters())\n    criterion = nn.BCEWithLogitsLoss()\n\n    for epoch in range(epochs):\n        print(f'\\nStarting Epoch {epoch}')\n\n        train_loss = 0\n        val_loss = 0\n\n        y_train = []\n        train_preds = []\n\n        timer.add(epoch)\n\n        model.train()\n        for i, batch in tqdm_notebook(enumerate(train_dataloader), total=(int(len(train_dataset) / BATCH_SIZE))):\n            (question, length), index, label = batch.question_text, batch.id, batch.target\n\n            question = question.to(device)\n            meta = torch.from_numpy(train_meta[index]).float().to(device)\n            label = label.to(device).float()\n            \n            out = model(question, meta, length)\n\n            loss = criterion(out, label)\n\n            train_loss += loss.item()\n\n            optimizer.zero_grad()\n\n            loss.backward()\n            torch.nn.utils.clip_grad_norm_(model.parameters(), 5.0)\n\n            optimizer.step()\n\n            y_train.append(label.detach())\n            train_preds.append(out.detach())\n\n        model.eval()\n        with torch.no_grad():\n\n            y_val = []\n            val_preds = []\n\n            for j, batch in tqdm_notebook(enumerate(val_dataloader), total=(int(len(val_dataset) / BATCH_SIZE))):\n                (question, length), index, label = batch.question_text,batch.id, batch.target\n                \n                question = question.to(device)\n                meta = torch.from_numpy(train_meta[index]).float().to(device)\n                label = label.to(device).float()\n                \n                out = model(question, meta, length)\n\n                loss = criterion(out, label)\n\n                val_loss += loss.item()\n\n                optimizer.zero_grad()\n\n                y_val.append(label.detach())\n                val_preds.append(out.detach())\n\n        train_loss /= (i + 1)\n        val_loss /= (j + 1)\n\n        y_train = torch.cat(y_train, dim=0).reshape(-1, 1)\n        y_val = torch.cat(y_val, dim=0).reshape(-1, 1)\n\n        train_preds = torch.cat(train_preds, dim=0).reshape(-1, 1)\n        val_preds = torch.cat(val_preds, dim=0).reshape(-1, 1)\n\n        logger.write([epoch, optimizer.param_groups[0]['lr'], train_loss, val_loss])\n\n        print(f'{timer.fsince(epoch)} | End of Epoch {epoch} | Train Loss: {train_loss} | Val Loss: {val_loss}')\n\n        if val_loss < best_val_loss:\n            best_epoch = epoch\n\n            best_train_loss = train_loss\n            best_val_loss = val_loss\n\n            best_val_preds = val_preds\n\n            save_checkpoint({\n                'model': model.state_dict(),\n                'optimizer': optimizer.state_dict(),\n                'epoch': epoch\n            })\n            \n    message = f'Training Finished | Best Epoch {best_epoch} | Best Train Loss: {best_train_loss} | Best Val Loss: {best_val_loss}'\n                        \n    return best_val_preds, y_val, best_epoch, best_train_loss, best_val_loss, message","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b897df33986e8dd051625f93a9d5f06e169f36af"},"cell_type":"code","source":"#CV kernels with meta features use a single `train_meta` array since the torchtext dataset is defined at the top of the kernel","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2a50e01449e45323199c58958e999f8bc3f30579","scrolled":false},"cell_type":"code","source":"def cv(total_train_dataset, test_1_dataset, test_2_dataset, test_1_dataloader, test_2_dataloader, train_meta, test_1_meta, test_2_meta, net, question_field, mean_vectors):\n    fold = 0\n\n    best_epochs = []\n    best_train_losses = []\n    best_val_losses = []\n    best_val_f1s = []\n\n    train_df = pd.read_csv('./train.csv')\n\n    preds = np.zeros(len(total_train_dataset))\n    labels = np.zeros(len(total_train_dataset))\n    \n    total_test_1_preds = np.zeros(len(test_1_dataset))\n    total_test_2_preds = np.zeros(len(test_2_dataset))\n\n    train_examples = np.array(total_train_dataset.examples)\n\n    kfold = KFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)\n\n    for train_idx, val_idx in kfold.split(train_df):\n        print('\\n')\n        print('=' * 50)\n        print(f'Fold: {fold}\\n')\n\n        # Get data for current fold\n        x_train, x_val = train_examples[train_idx], train_examples[val_idx]\n\n        train_dataset = data.Dataset(x_train, train_fields)\n        val_dataset = data.Dataset(x_val, train_fields)\n        \n        train_dataloader, val_dataloader = data.BucketIterator.splits((train_dataset, val_dataset), (BATCH_SIZE, BATCH_SIZE), sort_key=lambda x: len(x.question_text), sort_within_batch=True)\n\n        # Train model \n        # See above markdown cell for reason behind single train_meta variable\n        val_preds, y_val, best_epoch, best_train_loss, best_val_loss, message = train(train_dataset, val_dataset, train_dataloader, val_dataloader, train_meta, net, question_field, mean_vectors, question_field.vocab.stoi)\n        print(message)\n        \n        log = pd.read_csv('train_log.csv')\n        plt.plot(log['epoch'], log['train_loss'], log['val_loss'])\n        plt.show()\n\n        # Find best threshold\n        best_threshold, best_val_f1 = choose_threshold(torch.sigmoid(val_preds).cpu().numpy(), y_val.cpu().numpy())\n\n        # Get test preds\n        test_1_preds, test_1_y_val = test(net, question_field, test_1_dataloader, test_1_meta)\n        test_2_preds, test_2_y_val = test(net, question_field, test_2_dataloader, test_2_meta)\n\n        preds[val_idx] = val_preds.cpu().numpy().reshape(-1)\n        labels[val_idx] = y_val.cpu().numpy().reshape(-1)\n\n        total_test_1_preds += test_1_preds.reshape(-1)\n        total_test_2_preds += test_2_preds.reshape(-1)\n\n        # Remove all saved model files\n        for file in os.listdir('./'):\n            if file.endswith('.pt.tar'):\n                os.remove(f'./{file}')\n\n        for file in os.listdir('./.vector_cache/'):\n            os.remove(f'./.vector_cache/{file}')\n\n        # Log stats\n        best_epochs.append(best_epoch)\n        best_train_losses.append(best_train_loss)\n        best_val_losses.append(best_val_loss)\n        best_val_f1s.append(best_val_f1)\n\n        fold += 1\n\n    threshold, best_val_f1 = choose_threshold(torch.sigmoid(torch.from_numpy(preds)).numpy(), labels)\n\n    total_test_1_preds /= N_SPLITS\n    total_test_2_preds /= N_SPLITS\n\n    total_test_1_preds = (total_test_1_preds > threshold).astype(int)\n    total_test_2_preds = (total_test_2_preds > threshold).astype(int)\n    \n    test_1_f1 = f1_score(test_1_y_val, total_test_1_preds)\n    test_2_f1 = f1_score(test_2_y_val, total_test_2_preds)\n\n    return test_1_f1, test_2_f1, threshold, best_val_f1, best_epochs, best_train_losses, best_val_losses, best_val_f1s","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"274c2b9ac490fd589e80437f9cf5068e2eb8be68","_kg_hide-output":true},"cell_type":"code","source":"glove_vectors = torchtext.vocab.Vectors('../input/embeddings/glove.840B.300d/glove.840B.300d.txt')#, max_vectors=1000)\n\nfor file in os.listdir('./.vector_cache/'):\n    os.remove(f'./.vector_cache/{file}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0364480c546bea27dbb13bd5b9a77f5b6df44273","_kg_hide-output":true},"cell_type":"code","source":"paragram_vectors = torchtext.vocab.Vectors('../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt')#, max_vectors=1000)\n\nfor file in os.listdir('./.vector_cache/'):\n    os.remove(f'./.vector_cache/{file}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"570c9cf153d2dfa8df0b081274823780b31aeb18","scrolled":false},"cell_type":"code","source":"%%time\nmean_vectors = torch.zeros((len(question_field.vocab.stoi), 300))\n\nfor word, i in tqdm_notebook(question_field.vocab.stoi.items(), total=len(question_field.vocab.stoi)):\n    glove_vector = glove_vectors[word]\n    paragram_vector = paragram_vectors[word]\n    \n    vector = torch.stack([glove_vector, paragram_vector])\n    vector = torch.sum(vector, dim=0).reshape(1, -1) / 2\n    \n    mean_vectors[i] = vector\n    \ndel glove_vectors, paragram_vectors\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"scrolled":true,"_uuid":"67ddce7eb42beb196688d8e080befd0cc870529a"},"cell_type":"code","source":"logger = mlc.LinewiseCSVWriter('shakeup.csv', header=['seed', 'val_f1', 'stage_1_test_f1', 'stage_2_test_f1', 'threshold'])\n\nfor i in range(5):\n    seed = np.random.randint(1000, high=9999)\n\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\n    test_1_f1, test_2_f1, best_threshold, best_val_f1, _, _, _, _ = cv(total_train_dataset, test_1_dataset, test_2_dataset, test_1_dataloader, test_2_dataloader, train_meta, test_1_meta, test_2_meta, net, question_field, mean_vectors)\n    \n    logger.write([seed, best_val_f1, test_1_f1, test_2_f1, best_threshold])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d59f88346b656da045444cc2987571bbd0f1892e"},"cell_type":"code","source":"log = pd.read_csv('shakeup.csv')\nlog['stage_2_shakeup'] = log['stage_2_test_f1'] - log['val_f1']\nlog['stage_1_shakeup'] = log['stage_1_test_f1'] - log['val_f1']\nlog['stage_1_2_shakeup'] = log['stage_2_test_f1'] - log['stage_1_test_f1']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"d9b8a7663db0aa050433765cfd032eab7d096317"},"cell_type":"code","source":"plt.hist(log['val_f1'])\nplt.show()\nsns.kdeplot(log['val_f1'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a7bec20388e01c098da2f29c1f0fbd897ab476c"},"cell_type":"code","source":"plt.hist(log['stage_1_test_f1'])\nplt.show()\nsns.kdeplot(log['stage_1_test_f1'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af24ceea50ab470bc567e992f9e07c133165385f"},"cell_type":"code","source":"plt.hist(log['stage_2_test_f1'])\nplt.show()\nsns.kdeplot(log['stage_2_test_f1'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"68a40c01d088f17c92053ca4b7345491f4535432"},"cell_type":"code","source":"plt.hist(log['threshold'])\nplt.show()\nsns.kdeplot(log['threshold'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"28a5c380330bc3967d2b7629250445bfd1b0942f"},"cell_type":"code","source":"plt.hist(log['stage_1_shakeup'])\nplt.show()\nsns.kdeplot(log['stage_1_shakeup'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ae3ff7b4900944854727e4021a0c4c11391ddd15"},"cell_type":"code","source":"plt.hist(log['stage_2_shakeup'])\nplt.show()\nsns.kdeplot(log['stage_2_shakeup'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"17bac48c068e4a2064c62c0ba18118141a50edd7"},"cell_type":"code","source":"plt.hist(log['stage_1_2_shakeup'])\nplt.show()\nsns.kdeplot(log['stage_1_2_shakeup'])","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"\n\n\n\n\n\n\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}