{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\n\nfrom torchtext import data\nfrom torchtext import datasets\nfrom torchtext import vocab\n\nimport random\nimport time\n\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.metrics import f1_score\n\n# To run experiments deterministically\nSEED = 42\n\ntorch.manual_seed(SEED)\ntorch.cuda.manual_seed(SEED)\ntorch.backends.cudnn.deterministic = True","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"# For kaggle\nimport os\nprint(os.listdir(\"../input\"))\nTRAIN_PATH = '../input/train.csv'\nTEST_PATH = '../input/test.csv'\nGLOVE_PATH = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\nGLOVE = 'glove.840B.300d.txt'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"988da3eb41844113ad05c83996b90c1040bec7d2","trusted":false},"cell_type":"code","source":"print(os.listdir(\"../input/embeddings/glove.840B.300d\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dd377d43bf3ab1b99e5b5595b301f2ddf985ed58"},"cell_type":"markdown","source":"**Hyperparameters**  \nLet's keep track of the various hyperparameters here."},{"metadata":{"trusted":false,"_uuid":"d2ef93f8232acf2e0e0b9dced6f2bda43c88f667"},"cell_type":"code","source":"# Pretrained embedding to use\nEMBEDDING_PATH = GLOVE_PATH\n# Should we limit the vocab size?\n# (A) 120000, 95000\nMAX_SIZE = 120000\n# (A) Should we limit number of words in a sentence?\nMAX_LEN = 70\n\n# Split ratio for test/valid\nSPLIT_RATIO = 0.9\n\n# (A)\nBATCH_SIZE = 512\n\n# Model parameters\n# (A) Could be lesser I think.\nHIDDEN_DIM = 32\n# (A)\nN_LAYERS = 2\nBIDIRECTIONAL = True\n# (C)\nDROPOUT = 0.5","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c9c6a62ec86f96fb8d1935097fba292b9745b110"},"cell_type":"markdown","source":"# Data Preprocessing"},{"metadata":{"_uuid":"2a53d55710f4f4d30950299688e2f9cb7845935b"},"cell_type":"markdown","source":"## Creating Train / Validation / Test Datasets"},{"metadata":{"_uuid":"98f7ac5711353dd4c2a13bbc0eb5c357ebe883fd","trusted":false},"cell_type":"code","source":"# Defining the Fields for our dataset\n# Skipping the id column\nID = data.Field()\nTEXT = data.Field(tokenize='spacy')\nTARGET = data.LabelField(dtype=torch.float)\n\ntrain_fields = [('id', None), ('text', TEXT), ('target', TARGET)]\ntest_fields = [('id', ID), ('text', TEXT)]\n\n# Creating our train and test data\ntrain_data = data.TabularDataset(\n    path=TRAIN_PATH,\n    format='csv',\n    skip_header=True,\n    fields=train_fields\n)\n\ntest_data = data.TabularDataset(\n    path=TEST_PATH,\n    format='csv',\n    skip_header=True,\n    fields=test_fields\n)\n\n# Create validation dataset (default 70:30 split)\ntrain_data, valid_data = train_data.split(split_ratio=SPLIT_RATIO, random_state=random.seed(SEED))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3fc42a37bc362e54a256901e9988cd6bfca43d72","trusted":false},"cell_type":"code","source":"print(f'Number of training examples: {len(train_data)}')\nprint(f'Number of validation examples: {len(valid_data)}')\nprint(f'Number of test examples: {len(test_data)}')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b69cad669ae912e6f970428be25bc0383a92cbb6","trusted":false},"cell_type":"code","source":"# One training example\nvars(train_data.examples[0])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"549eab9da2ddefcb365f3d32117390a66ebc8f6f"},"cell_type":"markdown","source":"## Building the Vocabulary and Embeddings"},{"metadata":{"_uuid":"7483f8ca5c6a55ec701b54c5cf6c6ffed208580d","trusted":false},"cell_type":"code","source":"# Importing the pretrained embedding\nvec = vocab.Vectors(EMBEDDING_PATH)\n\n# Build the vocabulary using only the train dataset?,\n# and also by specifying the pretrained embedding\nTEXT.build_vocab(train_data, vectors=vec, max_size=MAX_SIZE)\nTARGET.build_vocab(train_data)\nID.build_vocab(test_data)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0e59c0db4a95d82744fa238937cac3ab76a9dc8b","trusted":false},"cell_type":"code","source":"print(f'Unique tokens in TEXT vocab: {len(TEXT.vocab)}')\nprint(f'Unique tokens in TARGET vocab: {len(TARGET.vocab)}')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8ea47255b3548e84d5ea75094a015a83cc80c794","trusted":false},"cell_type":"code","source":"TEXT.vocab.vectors.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e6ab4187277222860335d83dac61770384337d8c"},"cell_type":"markdown","source":"## Constructing the Iterator / Batching"},{"metadata":{"_uuid":"8ef0f828eb8489ee494361ac6bf3c0a408544abb","trusted":false},"cell_type":"code","source":"# Might have some confusion as to how the batch iterators are defined\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Automatically shuffles and buckets the input sequences into\n# sequences of similar length\ntrain_iter, valid_iter = data.BucketIterator.splits(\n    (train_data, valid_data),\n    sort_key=lambda x: len(x.text), # what function/field to use to group the data\n    batch_size=BATCH_SIZE,\n    device=device\n)\n\n# Don't want to shuffle test data, so use a standard iterator\ntest_iter = data.Iterator(\n    test_data,\n    batch_size=BATCH_SIZE,\n    device=device,\n    train=False,\n    sort=False,\n    sort_within_batch=False\n)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"346fba2b1a940f8441a040776c2daba6cb6206bf"},"cell_type":"markdown","source":"# Defining the Model\nThe RNN architecture will be\n- LSTM\n- Bidirectional\n- Multi-layer\n- With dropout"},{"metadata":{"_uuid":"c55fa8d01a7ea63c74d7b7cfa7be81dd9851bfcd","trusted":false},"cell_type":"code","source":"class RNN(nn.Module):\n    def __init__(self, vocab_size, embedding_dim, hidden_dim, output_dim,\n                 n_layers, bidirectional, dropout):\n        super().__init__()\n        \n        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n        self.rnn = nn.LSTM(embedding_dim, hidden_dim, num_layers=n_layers,\n                           bidirectional=bidirectional, dropout=dropout)\n        # Final hidden state has both forward and backward components\n        self.fc = nn.Linear(hidden_dim*2, output_dim)\n        self.dropout = nn.Dropout(dropout)\n        \n    def forward(self, x):\n        # x: [seq_length, batch_size]\n        embedded = self.dropout(self.embedding(x))\n        # embedded: [seq_length, batch_size, emb_dim]\n        \n        output, (hidden, cell) = self.rnn(embedded)\n        # output: [seq_length, batch_size, hid_dim * num_directions]\n        # hidden: [num_layers * num_directions, batch_size, hid_dim]\n        # cell:\n        \n        # Concat the final forward and backward hidden layers\n        hidden = self.dropout(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim=1))\n        # hidden: [batch_size, hid_dim * num_directions]\n        \n        return self.fc(hidden.squeeze(0))\n        # return: [batch_size, 1]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2ac4fd622ae3ad8408aba5fbf78d1f91eb208748","trusted":false},"cell_type":"code","source":"emb_shape = TEXT.vocab.vectors.shape\nINPUT_DIM = emb_shape[0]\nEMBEDDING_DIM = emb_shape[1]\nOUTPUT_DIM = 1\n\nmodel = RNN(INPUT_DIM, EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM, N_LAYERS, BIDIRECTIONAL, DROPOUT)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ff95ceb5395b64e0ba6fe88445254dc83c9b2d59"},"cell_type":"markdown","source":"## Transfering the pre-trained word embeddings"},{"metadata":{"_uuid":"f9e7dc2c950207e571273326a0df8374ae426661","trusted":false},"cell_type":"code","source":"pretrained_embeddings = TEXT.vocab.vectors\npretrained_embeddings.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6ac7c8da398747191ea823bd762648bccaf24f07","trusted":false},"cell_type":"code","source":"model.embedding.weight.data.copy_(pretrained_embeddings)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f8906b5b53e659332e67e4e4b580a4cf5a67b484"},"cell_type":"markdown","source":"# Training the Model"},{"metadata":{"_uuid":"38f28118411839b2cf102c06fe29e1a15cdc3f27","trusted":false},"cell_type":"code","source":"optimizer = optim.Adam(model.parameters())\ncriterion = nn.BCEWithLogitsLoss()\n\nmodel = model.to(device)\ncriterion = criterion.to(device)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c466868a2c02eff35cd11e42d5d389580e8a3147","trusted":false},"cell_type":"code","source":"def train(model, iterator, optimizer, criterion):\n    # Track the loss\n    epoch_loss = 0\n    \n    model.train()\n    \n    for batch in iterator:\n        optimizer.zero_grad()\n        \n        predictions = model(batch.text).squeeze(1)\n        loss = criterion(predictions, batch.target)\n        \n        loss.backward()\n        optimizer.step()\n        \n        epoch_loss += loss.item()\n        \n    return epoch_loss / len(iterator)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"79c5ea3462d1c8e5e4e5f7c157dc5fc88414de5b","trusted":false},"cell_type":"code","source":"def evaluate(model, iterator, criterion):\n    epoch_loss = 0\n    \n    model.eval()\n    \n    with torch.no_grad():\n        for batch in iterator:\n            predictions = model(batch.text).squeeze(1)\n            loss = criterion(predictions, batch.target)\n\n            epoch_loss += loss.item()\n        \n    return epoch_loss / len(iterator)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"aa36e8281963ae1fcccfe51ec9c11a8d95bd26fd"},"cell_type":"markdown","source":"## Training loop here"},{"metadata":{"_uuid":"00d7c9873bc0b59cc8cdd19a05291b0b64135aeb","trusted":false},"cell_type":"code","source":"N_EPOCHS = 6\n\n# Track time taken\nstart_time = time.time()\n\nfor epoch in range(N_EPOCHS):\n    epoch_start_time = time.time()\n    \n    train_loss = train(model, train_iter, optimizer, criterion)\n    valid_loss = evaluate(model, valid_iter, criterion)\n    \n    print(f'| Epoch: {epoch+1:02} '\n          f'| Train Loss: {train_loss:.3f} '\n          f'| Val. Loss: {valid_loss:.3f} '\n          f'| Time taken: {time.time() - epoch_start_time:.2f}s'\n          f'| Time elapsed: {time.time() - start_time:.2f}s')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"041c95a1b20a2f6598ba61c5043e8ad7d48054cc"},"cell_type":"markdown","source":"# Prediction"},{"metadata":{"_uuid":"da574beee8a890d882c1c50c34a3256378f35399"},"cell_type":"markdown","source":"## Determining probability threshold"},{"metadata":{"_uuid":"6b48889d29543b19cbc291f805c6cb0d3a06543b","trusted":false},"cell_type":"code","source":"# Use validation dataset\nvalid_pred = []\nvalid_truth = []\n    \nmodel.eval()\n    \nwith torch.no_grad():\n    for batch in valid_iter:\n        valid_truth += batch.target.cpu().numpy().tolist()\n        predictions = model(batch.text).squeeze(1)\n        valid_pred += torch.sigmoid(predictions).cpu().data.numpy().tolist()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4656b6f6266a63dc55008648aa9330b4e73692f9","trusted":false},"cell_type":"code","source":"tmp = [0,0,0] # idx, cur, max\ndelta = 0\nfor tmp[0] in np.arange(0.1, 0.501, 0.01):\n    tmp[1] = f1_score(valid_truth, np.array(valid_pred)>tmp[0])\n    if tmp[1] > tmp[2]:\n        delta = tmp[0]\n        tmp[2] = tmp[1]\nprint('best threshold is {:.4f} with F1 score: {:.4f}'.format(delta, tmp[2]))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3ab8a5544109703199460ba6ff6092bac4f36c84"},"cell_type":"markdown","source":"## Prediction and submission"},{"metadata":{"trusted":false,"_uuid":"651f8f0fd82c2b08c185d3fe968bafe27c7e1d3d"},"cell_type":"code","source":"test_pred = []\ntest_id = []\n\nmodel.eval()\n\nwith torch.no_grad():\n    for batch in test_iter:\n        predictions = model(batch.text).squeeze(1)\n        test_pred += torch.sigmoid(predictions).cpu().data.numpy().tolist()\n        test_id += batch.id.view(-1).cpu().numpy().tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"903d2712a96b09a0d8b1736d3289c4ba038894b3"},"cell_type":"code","source":"test_pred = (np.array(test_pred) >= delta).astype(int)\ntest_id = [ID.vocab.itos[i] for i in test_id]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f895f61ee19ef477c32e0de9dec0a7d08ffc8b78"},"cell_type":"code","source":"submission = pd.DataFrame({'qid': test_id, 'prediction': test_pred})","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"56ccf09a636f08ba832141c5966099ea0256287e"},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.1"},"toc-autonumbering":true},"nbformat":4,"nbformat_minor":1}