{"cells":[{"metadata":{"trusted":true,"_uuid":"a7fe0e6e0364e1e7e0570b0563dc5be69632430a"},"cell_type":"code","source":"FOLD = 0\n\nimport os\nimport time\nimport math\nimport requests\nimport glob\n\nimport ast\n\nimport numpy as np\nimport pandas as pd\n\nimport mlcrate as mlc\n\nimport os\n\nimport cv2\n\nfrom sklearn.model_selection import train_test_split, KFold, StratifiedKFold\nfrom sklearn.metrics import fbeta_score, accuracy_score, precision_score, recall_score, f1_score\n\nfrom skimage.transform import resize\n\nfrom PIL import Image, ImageDraw\n\nfrom tqdm import tqdm, tqdm_notebook\n\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom torch import optim\nfrom torch.optim import Optimizer\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.utils.checkpoint as checkpoint\n\nimport torchvision\nfrom torchvision import transforms, utils\n\nimport torchtext\nimport torchtext.data as data\n\nfrom torch.nn.utils.rnn import pad_sequence\n\nimport spacy\nfrom spacy.lang.en import English\n\nSEED = 1337\n\nNOTIFY_EACH_EPOCH = False\n\nWORKERS = 0\nBATCH_SIZE = 512\n\nN_SPLITS = 10\n\nnp.random.seed(SEED)\n# torch.manual_seed(SEED)\n# torch.cuda.manual_seed(SEED)\n# torch.backends.cudnn.deterministic = True\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\ndef get_n_params(model):\n    pp=0\n    for p in list(model.parameters()):\n        nn=1\n        for s in list(p.size()):\n            nn = nn*s\n        pp += nn\n    return pp\n\n# from https://github.com/floydhub/save-and-resume\ndef save_checkpoint(state):\n    \"\"\"Save checkpoint if a new best is achieved\"\"\"\n    print (\" Saving checkpoint\")\n\n    filename = f'./checkpoint-{state[\"epoch\"]}.pt.tar'\n    torch.save(state, filename)\n    \ndef initialize(model, path=None, optimizer=None):   \n    if path == None:\n        checkpoints = glob.glob('./*.pt.tar')\n        path = checkpoints[np.argmax([int(checkpoint.split('checkpoint-')[1].split('.')[0]) for checkpoint in checkpoints])]\n\n    checkpoint = torch.load(path)\n\n    model.load_state_dict(checkpoint['model'])\n\n    print(f' Loaded checkpoint {path} | Trained for {checkpoint[\"epoch\"] + 1} epochs')\n    \n    if optimizer:\n        optimizer.load_state_dict(checkpoint['optimizer'])\n          \n        epoch = checkpoint['epoch'] + 1\n        train_iteration = checkpoint['train_iteration']\n        val_iteration = checkpoint['val_iteration']\n\n        return model, optimizer, epoch, train_iteration, val_iteration\n    else:\n        return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"321593e69c05e023da01c4019576ba09e34cafd1"},"cell_type":"code","source":"sample_submission = pd.read_csv('../input/sample_submission.csv')\ntrain = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4e3d795b5e36576dc98d5319faae7887b4a4d26"},"cell_type":"code","source":"kfold = KFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)\n\ntrain_idx, val_idx = list(kfold.split(train))[FOLD]\nx_train, x_val = train.iloc[train_idx], train.iloc[val_idx]\n\nx_train.to_csv('train.csv')\nx_val.to_csv('val.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4c7e4854b60b1ff433abe2c2082542f58acb43a5"},"cell_type":"code","source":"nlp = English()\ndef tokenize(sentence):\n    x = nlp(sentence)\n    return [token.text for token in x]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d52cc97f05e431151f7f488fa39a3ae014610b88","scrolled":true},"cell_type":"code","source":"# from http://mlexplained.com/2018/02/08/a-comprehensive-tutorial-to-torchtext\nquestion_field = data.Field(tokenize=tokenize, lower=True, batch_first=True, include_lengths=True)\ntarget_field = data.Field(sequential=False, use_vocab=False, batch_first=True)\n\ntrain_fields = [\n    ('id', None),\n    ('qid', None),\n    ('question_text', question_field),\n    ('target', target_field)\n]\n\ntest_fields = [\n    ('qid', None),\n    ('question_text', question_field)\n]\n\ntrain_dataset, val_dataset = data.TabularDataset.splits('./', train='train.csv', validation='val.csv', format='CSV', skip_header=True, fields=train_fields)\ntest_dataset = data.TabularDataset('../input/test.csv', format='CSV', skip_header=True, fields=test_fields)\n\nvectors = torchtext.vocab.Vectors('../input/embeddings/glove.840B.300d/glove.840B.300d.txt')#, max_vectors=1000)\n\nquestion_field.build_vocab(train_dataset, max_size=95000)\nquestion_field.vocab.set_vectors(vectors.stoi, vectors.vectors, vectors.dim)\npretrained_embedding = question_field.vocab.vectors\n\ntrain_dataloader, val_dataloader = data.BucketIterator.splits((train_dataset, val_dataset), (BATCH_SIZE, BATCH_SIZE), sort_key=lambda x: len(x.question_text), sort_within_batch=True)\ntest_dataloader = data.BucketIterator(test_dataset, 1, sort=False, shuffle=False)\n\nprint(f'Train Dataset: {len(train_dataset)}')\nprint(f'Val Dataset: {len(val_dataset)}')\nprint(f'Test Dataset: {len(test_dataset)}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"52ef5b659e612995b903ab699d0ecda3fb077d0f"},"cell_type":"code","source":"len(question_field.vocab.itos)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dda7d32355775e045d56ae70f793e55bacbc0651"},"cell_type":"code","source":"# from https://discuss.pytorch.org/t/self-attention-on-words-and-masking/5671/4\nclass SelfAttention(nn.Module):\n    def __init__(self, hidden_size, batch_first=False):\n        super(SelfAttention, self).__init__()\n\n        self.hidden_size = hidden_size\n        self.batch_first = batch_first\n\n        self.att_weights = nn.Parameter(torch.Tensor(1, hidden_size), requires_grad=True)\n\n        stdv = 1.0 / math.sqrt(self.hidden_size)\n        for weight in self.att_weights:\n            nn.init.uniform_(weight, -stdv, stdv)\n\n    def get_mask(self):\n        pass\n\n    def forward(self, inputs, lengths):\n        if self.batch_first:\n            batch_size, max_len = inputs.size()[:2]\n        else:\n            max_len, batch_size = inputs.size()[:2]\n            \n        # apply attention layer\n        weights = torch.bmm(inputs,\n                            self.att_weights  # (1, hidden_size)\n                            .permute(1, 0)  # (hidden_size, 1)\n                            .unsqueeze(0)  # (1, hidden_size, 1)\n                            .repeat(batch_size, 1, 1) # (batch_size, hidden_size, 1)\n                            )\n    \n        attentions = torch.softmax(F.relu(weights.squeeze()), dim=-1)\n\n        # create mask based on the sentence lengths\n        mask = torch.ones(attentions.size(), requires_grad=True).cuda()\n        for i, l in enumerate(lengths):  # skip the first sentence\n            if l < max_len:\n                mask[i, l:] = 0\n\n        # apply mask and renormalize attention scores (weights)\n        masked = attentions * mask\n        _sums = masked.sum(-1).unsqueeze(-1)  # sums per row\n        \n        attentions = masked.div(_sums)\n\n        # apply attention weights\n        weighted = torch.mul(inputs, attentions.unsqueeze(-1).expand_as(inputs))\n\n        # get the final fixed vector representations of the sentences\n        representations = weighted.sum(1).squeeze()\n\n        return representations, attentions\n\nclass BaselineLSTM(nn.Module):\n    def __init__(self, embedding):\n        super(BaselineLSTM, self).__init__()\n                \n        self.embedding = nn.Embedding.from_pretrained(embedding)\n        \n        self.lstm = nn.LSTM(input_size=300, hidden_size=128, num_layers=2, batch_first=True, bidirectional=True)\n        \n        self.attention = SelfAttention(128*2, batch_first=True)\n        \n        self.fc = nn.Linear(128*2, 1)\n        self.logit = nn.Linear(1, 1)\n\n    def forward(self,x, x_len):\n        x = self.embedding(x)\n        x = nn.utils.rnn.pack_padded_sequence(x, x_len, batch_first=True)\n\n        out, (hidden, _) = self.lstm(x)\n        \n        x, lengths = nn.utils.rnn.pad_packed_sequence(out, batch_first=True)\n        \n        x, _ = self.attention(x, lengths) \n        \n        x = self.fc(x)\n        x = self.logit(x).view(-1)\n        \n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2a50e01449e45323199c58958e999f8bc3f30579","scrolled":false},"cell_type":"code","source":"start_epoch = 0\nepochs = 7\nearly_stopping = 10\n\ntrain_iteration = 0\nval_iteration = 0\n\nthreshold = 0.35\n\nmodel = BaselineLSTM(pretrained_embedding).to(device)\n\noptimizer = optim.Adam(model.parameters())\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=10, verbose=True)\n\ncriterion = nn.BCEWithLogitsLoss()\n\nget_n_params(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"afe61e636a4e3adb7d31efa1b3fb76c09c09ea42","scrolled":false},"cell_type":"code","source":"best_train_loss = 1e10\nbest_val_loss = 1e10\n\nbest_train_f1 = 0\nbest_val_f1 = 0\n\nbest_epoch = 0\n\ntimer = mlc.time.Timer()\nlogger = mlc.LinewiseCSVWriter('train_log.csv', header=['epoch', 'lr', 'train_loss', 'val_loss', 'train_f1', 'val_f1'])\n\nfor epoch in range(start_epoch, epochs):\n    print(f'\\n Starting Epoch {epoch} | LR: {optimizer.param_groups[0][\"lr\"]}')\n        \n    train_loss = 0\n    val_loss = 0\n\n    y_train = []\n    train_preds = []\n    \n    timer.add(epoch)\n\n    model.train()\n    for i, batch in tqdm_notebook(enumerate(train_dataloader), total=(int(len(train_dataset) / BATCH_SIZE))):\n        (question, length), label = batch.question_text, batch.target.to(device).float()\n        question = question.to(device)\n        \n        out = model(question, length)\n\n        loss = criterion(out, label)\n\n        train_loss += loss.item()\n\n        optimizer.zero_grad()\n        \n        loss.backward()\n        torch.nn.utils.clip_grad_norm_(model.parameters(), 5.0)\n\n        optimizer.step()\n        \n        y_train.append(label.detach())\n        train_preds.append(out.detach())\n            \n        train_iteration += 1\n          \n    model.eval()\n    with torch.no_grad():\n        \n        y_val = []\n        val_preds = []\n\n        for j, batch in tqdm_notebook(enumerate(val_dataloader), total=(int(len(val_dataset) / BATCH_SIZE))):\n            (question, length), label = batch.question_text, batch.target.to(device).float()\n            question = question.to(device)\n\n            out = model(question, length)\n            \n            loss = criterion(out, label)\n\n            val_loss += loss.item()\n\n            optimizer.zero_grad()\n            \n            y_val.append(label.detach())\n            val_preds.append(out.detach())\n\n            val_iteration += 1\n    \n    train_loss /= (i + 1)\n    val_loss /= (j + 1)\n\n    y_train = torch.cat(y_train, dim=0).reshape(-1, 1)\n    y_val = torch.cat(y_val, dim=0).reshape(-1, 1)\n\n    train_preds = torch.cat(train_preds, dim=0).reshape(-1, 1)\n    val_preds = torch.cat(val_preds, dim=0).reshape(-1, 1)\n    \n    train_f1 = f1_score(y_train, (train_preds > threshold))\n    val_f1 = f1_score(y_val, (val_preds > threshold))\n    \n    logger.write([epoch, optimizer.param_groups[0]['lr'], train_loss, val_loss, train_f1, val_f1])\n            \n    print(f' {timer.fsince(epoch)} | End of Epoch {epoch} | Train Loss: {train_loss} | Val Loss: {val_loss} | Train F1: {round(train_f1, 4)} | Val F1: {round(val_f1, 4)}')\n          \n    scheduler.step(val_loss)\n          \n    if val_loss < best_val_loss:\n        best_epoch = epoch\n\n        best_train_loss = train_loss\n        best_val_loss = val_loss\n\n        best_train_f1 = train_f1\n        best_val_f1 = val_f1\n          \n        save_checkpoint({\n            'model': model.state_dict(),\n            'optimizer': optimizer.state_dict(),\n            'epoch': epoch,\n            'train_iteration': train_iteration,\n            'val_iteration': val_iteration\n        })\n\n    elif epoch - best_epoch > early_stopping:\n        print(f' Val loss has not decreased for {early_stopping} epochs, stopping training')\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"265663699f720cc3c80ce88d3f2df5f37d6fc823","scrolled":true},"cell_type":"code","source":"print(f' Training Finished | Best Epoch {best_epoch} | LR: {optimizer.param_groups[0][\"lr\"]} \\n Best Train Loss: {best_train_loss} | Best Val Loss: {best_val_loss} | Best Train F1: {round(best_train_f1, 4)} | Best Val F1: {round(best_val_f1, 4)}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b037b4b8cc1b76d6bf52f709a7e069b2e03cb539","scrolled":false},"cell_type":"code","source":"log = pd.read_csv('train_log.csv')\nplt.plot(log['epoch'], log['train_loss'], log['val_loss'])\nplt.show()\nplt.plot(log['epoch'], log['train_f1'], log['val_f1'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d93dfad029210f27c4a80af177d4a490db2445d","scrolled":true},"cell_type":"code","source":"thresholds = np.arange(0.1, 0.501, 0.01)\n\nval_scores = []\nfor threshold in thresholds:\n    threshold = np.round(threshold, 2)\n    f1 = f1_score(y_val.cpu().numpy(), (torch.sigmoid(val_preds).cpu().numpy() > threshold).astype(int))\n    val_scores.append(f1)\n\nbest_val_f1 = np.max(val_scores)\nbest_threshold = np.round(thresholds[np.argmax(val_scores)], 2)\n\nplt.plot(thresholds, val_scores)\n\nprint(f' Best threshold: {best_threshold} | Best Train F1: {f1_score(y_train, (torch.sigmoid(train_preds).cpu().numpy() > best_threshold).astype(int))} | Best Val F1: {best_val_f1}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab3b71a5ac16ee9e5c54b4e266d1de46ee0a0c6d"},"cell_type":"code","source":"model = BaselineLSTM(pretrained_embedding).to(device)\nmodel = initialize(model)\n\npreds = []\n\nmodel.eval()\nwith torch.no_grad():\n    for i, batch in tqdm(enumerate(test_dataloader), total=int(len(test_dataset) / BATCH_SIZE)):\n        (question, length) = batch.question_text\n        question = question.to(device)\n        \n        out = model(question, length)\n        out = torch.sigmoid(out)\n        pred = out.detach().cpu().numpy()\n        preds.append(pred)\n        \npreds = np.concatenate(preds, axis=0).reshape(-1, 1)\npreds = (preds > best_threshold).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f5c296d79520402657f33f8c237dd74df0423ac"},"cell_type":"code","source":"sample_submission['prediction'] = preds\nmlc.kaggle.save_sub(sample_submission, 'submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"49231a2c9b66beeabfed7d3cb448019d1566ec00"},"cell_type":"code","source":"sample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"\n\n\n\n\n\n\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}