{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Pytorch Base Models\nThe following notebook contains base models for various tasks in vanilla Pytorch. The different models are covered;\n\n1. Simple Linear Regression \n2. Classification\n - Bi GRU Model\n - Bi LSTM/GRU Model with Attention\n - Multiclass BiLSTM/GRU models"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport time\nimport numpy as np\nimport pandas as pd \nimport matplotlib.pyplot as plt\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import f1_score,classification_report,accuracy_score,confusion_matrix,mean_absolute_error,mean_squared_error\nfrom tqdm import tqdm\n\n# Pytorch Imports\nimport torch\nimport torch.nn as nn\nimport torch.utils.data\nfrom torch.autograd import Variable\n\n# Deterministic behaviour when using GPUs\ndef fixing_seed(seed=1326):\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    print(f\">> Set Numpy/Torch/Cuda Seeds\\n>> Deterministic CUDNN : {torch.backends.cudnn.deterministic}\")\nfixing_seed()","execution_count":11,"outputs":[{"output_type":"stream","text":">> Set Numpy/Torch/Cuda Seeds\n>> Deterministic CUDNN : True\n","name":"stdout"}]},{"metadata":{},"cell_type":"markdown","source":"## Linear Regression Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_array = np.array([[3.3], [4.4], [5.5], [6.71], [6.93], [4.168], \n                    [9.779], [6.182], [7.59], [2.167], [7.042], \n                    [10.791], [5.313], [7.997], [3.1]], dtype=np.float32)\n\ny_array = np.array([[1.7], [2.76], [2.09], [3.19], [1.694], [1.573], \n                    [3.366], [2.596], [2.53], [1.221], [2.827], \n                    [3.465], [1.65], [2.904], [1.3]], dtype=np.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class LinearRegression(nn.Module):\n    \n    def __init__(self, input_dim, output_dim):\n        # Calling Super Class's constructor\n        super(LinearRegression, self).__init__() \n        # We can add more inbuilt layers or initialize custom layers here\n        self.linear = nn.Linear(input_dim, output_dim)\n        \n    def forward(self, x):\n        # Forward decides flow of data in the pytorch models\n        #  We can tweak and fine architecture in forward pass\n        out = self.linear(x)\n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"input_dim = X_array.shape[1]\noutput_dim = 1\nlearning_rate = 0.01\nnum_epochs = 500\n\n# Linear regression model\nmodel = LinearRegression(input_dim, output_dim).cuda()\ncriterion = nn.MSELoss() # Mean Squared Loss\noptimizer = torch.optim.Adam(model.parameters(), lr = learning_rate)\n\n# Train the model\nfor epoch in range(num_epochs):\n    # Convert numpy arrays to torch tensors\n    inputs = torch.from_numpy(X_array).cuda()\n    targets = torch.from_numpy(y_array).cuda()\n\n    # Forward pass\n    outputs = model(inputs)\n    loss = criterion(outputs, targets)\n    \n    # Backward and optimize\n    optimizer.zero_grad()\n    loss.backward()\n    optimizer.step()\n    \n    if (epoch+1) % 100 == 0:\n        print ('Epoch [{}/{}], Loss: {:.4f}'.format(epoch+1, num_epochs, loss.item()))\n\n# Plot the graph\npredicted = model(inputs).detach().cpu().numpy()\nplt.plot(X_array, y_array, 'ro', label='Original data')\nplt.plot(X_array, predicted, label='Fitted line')\nplt.legend()\nplt.show()\n# Save the model checkpoint\ntorch.save(model.state_dict(), 'model.ckpt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"Mean Absolute Error {mean_absolute_error(y_array,predicted)**.5}\")\nprint(f\"Mean Squared Error {mean_squared_error(y_array,predicted)}\")\nprint(f\"Root Mean Squared Error {mean_squared_error(y_array,predicted)**.5}\")","execution_count":12,"outputs":[{"output_type":"stream","text":"Mean Absolute Error 0.6016478539886916\nMean Squared Error 0.22421894967556\nRoot Mean Squared Error 0.4735176339647342\n","name":"stdout"}]},{"metadata":{},"cell_type":"markdown","source":"## Getting Started with Dataset\nLet us use Quora Question Insincere Compeitiotion dataset to make a simple and later complex NN models"},{"metadata":{"trusted":true},"cell_type":"code","source":"# DEBUG \nnrows  = 10000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embed_size = 300 \nmax_features = 5000 \nmaxlen = 70\n\ntrain_df = pd.read_csv(\"../input/quora-insincere-questions-classification/train.csv\", nrows= nrows)\ntest_df = pd.read_csv(\"../input/quora-insincere-questions-classification/test.csv\", nrows= nrows)\nprint('Train data dimension: ', train_df.shape)\nprint('Test data dimension: ', test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_fasttext(word_index):    \n    EMBEDDING_FILE = '../input/quora-insincere-questions-classification/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[\"question_text\"] = train_df[\"question_text\"].str.lower()\ntest_df[\"question_text\"] = test_df[\"question_text\"].str.lower()\n\n# fill up the missing values\nx_train = train_df[\"question_text\"].fillna(\"_##_\").values\nx_test = test_df[\"question_text\"].fillna(\"_##_\").values\n\n# Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(x_train))\nx_train = tokenizer.texts_to_sequences(x_train)\nx_test = tokenizer.texts_to_sequences(x_test)\n# Pad the sentences \nx_train = pad_sequences(x_train, maxlen=maxlen)\nx_test = pad_sequences(x_test, maxlen=maxlen)\n# Get the target values\ny_train = train_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Loading FastText Embedding Matrix"},{"metadata":{"trusted":true},"cell_type":"code","source":"fastText_embeddings = load_fasttext(tokenizer.word_index)\nembedding_matrix = fastText_embeddings","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Bidirectional GRU/LSTM Classifier"},{"metadata":{},"cell_type":"markdown","source":"### Training and Inference Function With K-Fold Split Validation (GPU enabled)"},{"metadata":{"trusted":true},"cell_type":"code","source":"def sigmoid(x):\n    return 1 / (1 + np.exp(-x))    \n\nclass BiGRUClassifier(nn.Module):  \n    def __init__(self):\n        hidden_size = 256\n        self.hidden_size = 300\n        self.batch_size = 512\n        super(BiGRUClassifier, self).__init__()\n        self.embedding = nn.Embedding(max_features, 300)\n        self.embedding.weight.data.copy_(torch.from_numpy(embedding_matrix))\n        self.gru = nn.GRU(input_size=300, hidden_size=hidden_size, \n                          bidirectional=True, batch_first=True)\n        #out \n        self.linear1 = nn.Linear(2 * hidden_size, hidden_size)\n        self.linear2 = nn.Linear(hidden_size, 1)\n        self.dropout = nn.Dropout(0.1)\n        \n    def forward(self, question, train=True):\n        batch = question.size(0)\n        question_embed = self.embedding(question) \n\n        gru_output, hidden = self.gru(question_embed) \n        hidden = hidden.transpose(0, 1).contiguous().view(batch, -1) \n        hidden = self.dropout(hidden)\n        hidden = torch.relu(self.linear1(hidden))  #batch x hidden_size\n        hidden = self.dropout(hidden)\n        return (self.linear2(hidden))  \n    \n    def init_hidden(self, batch_size):\n        return cuda_available(torch.zeros(2, batch_size, self.hidden_size))\n","execution_count":13,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def TrainingInference(model, batch_size=512, epochs=5):\n    n_epochs = epochs # how many times to iterate over all samples\n    # matrix for the out-of-fold predictions\n    train_preds = np.zeros((len(train_df)))\n    # matrix for the predictions on the test set\n    test_preds = np.zeros((len(test_df)))\n    x_test_cuda = torch.tensor(x_test, dtype=torch.long).cuda()\n    test = torch.utils.data.TensorDataset(x_test_cuda)\n    test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)\n    splits = list(StratifiedKFold(n_splits=3, shuffle=True, random_state=10).split(x_train, y_train))\n    for i, (train_idx, valid_idx) in enumerate(splits):    \n        # split data in train / validation according to the KFold indeces\n        # also, convert them to a torch tensor and store them on the GPU (done with .cuda())\n        x_train_fold = torch.tensor(x_train[train_idx], dtype=torch.long).cuda()\n        y_train_fold = torch.tensor(y_train[train_idx, np.newaxis], dtype=torch.float32).cuda()\n        x_val_fold = torch.tensor(x_train[valid_idx], dtype=torch.long).cuda()\n        y_val_fold = torch.tensor(y_train[valid_idx, np.newaxis], dtype=torch.float32).cuda()\n\n        # make sure everything in the model is running on the GPU\n        model.cuda()\n        # define binary cross entropy loss\n        # note that the model returns logit to take advantage of the log-sum-exp trick \n        # for numerical stability in the loss\n        loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum')\n        optimizer = torch.optim.Adam(model.parameters())\n\n        train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)\n        valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)\n\n        train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n        valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)\n\n        print(f'Fold {i + 1}')\n\n        for epoch in range(n_epochs):\n            # set train mode of the model. This enables operations which are only applied during training like dropout\n            start_time = time.time()\n            model.train()\n            avg_loss = 0.  \n            for x_batch, y_batch in tqdm(train_loader, disable=True):\n                # Forward pass: compute predicted y by passing x to the model.\n                y_pred = model(x_batch)\n\n                # Compute and print loss.\n                loss = loss_fn(y_pred, y_batch)\n\n                # Before the backward pass, use the optimizer object to zero all of the\n                # gradients for the Tensors it will update (which are the learnable weights\n                # of the model)\n                optimizer.zero_grad()\n\n                # Backward pass: compute gradient of the loss with respect to model parameters\n                loss.backward()\n\n                # Calling the step function on an Optimizer makes an update to its parameters\n                optimizer.step()\n                avg_loss += loss.item() / len(train_loader)\n\n            # set evaluation mode of the model. This disabled operations which are only applied during training like dropout\n            model.eval()\n\n            # predict all the samples in y_val_fold batch per batch\n            valid_preds_fold = np.zeros((x_val_fold.size(0)))\n            test_preds_fold = np.zeros((len(test_df)))\n\n            avg_val_loss = 0.\n            for i, (x_batch, y_batch) in enumerate(valid_loader):\n                y_pred = model(x_batch).detach()\n\n                avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n                valid_preds_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n\n            elapsed_time = time.time() - start_time \n            print('Epoch {}/{} \\t loss={:.4f} \\t val_loss={:.4f} \\t time={:.2f}s'.format(\n                epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))\n\n        # predict all samples in the test set batch per batch\n        for i, (x_batch,) in enumerate(test_loader):\n            y_pred = model(x_batch).detach()\n\n            test_preds_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n\n        train_preds[valid_idx] = valid_preds_fold\n        test_preds += test_preds_fold / len(splits)\n    return train_preds, test_preds","execution_count":16,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = BiGRUClassifier()\ntrain_oofpreds, test_oofpreds = TrainingInference(model)\npredictions = (train_oofpreds > .5).astype('uint')\nprint(f\"Confusion Matrix \\n{classification_report(y_train, predictions)}\")\nprint(f\"Accuracy {accuracy_score(y_train, predictions)}\")","execution_count":17,"outputs":[{"output_type":"stream","text":"Fold 1\nEpoch 1/5 \t loss=168.5845 \t val_loss=115.4214 \t time=0.93s\nEpoch 2/5 \t loss=112.6795 \t val_loss=107.6663 \t time=0.90s\nEpoch 3/5 \t loss=97.3574 \t val_loss=93.6429 \t time=0.89s\nEpoch 4/5 \t loss=63.3837 \t val_loss=85.9728 \t time=0.88s\nEpoch 5/5 \t loss=40.9739 \t val_loss=92.3152 \t time=1.03s\nFold 2\nEpoch 1/5 \t loss=63.3051 \t val_loss=35.2673 \t time=0.90s\nEpoch 2/5 \t loss=46.5313 \t val_loss=35.1847 \t time=0.90s\nEpoch 3/5 \t loss=36.0459 \t val_loss=40.4437 \t time=0.88s\nEpoch 4/5 \t loss=29.8616 \t val_loss=40.8318 \t time=0.88s\nEpoch 5/5 \t loss=19.7641 \t val_loss=44.6988 \t time=0.90s\nFold 3\nEpoch 1/5 \t loss=36.7159 \t val_loss=15.2788 \t time=0.91s\nEpoch 2/5 \t loss=24.3628 \t val_loss=18.4728 \t time=0.89s\nEpoch 3/5 \t loss=17.5533 \t val_loss=17.3202 \t time=0.89s\nEpoch 4/5 \t loss=10.7540 \t val_loss=19.0558 \t time=0.88s\nEpoch 5/5 \t loss=6.5301 \t val_loss=24.9545 \t time=0.90s\nConfusion Matrix \n              precision    recall  f1-score   support\n\n           0       0.97      0.99      0.98      9363\n           1       0.75      0.62      0.68       637\n\n   micro avg       0.96      0.96      0.96     10000\n   macro avg       0.86      0.80      0.83     10000\nweighted avg       0.96      0.96      0.96     10000\n\nAccuracy 0.9623\n","name":"stdout"}]},{"metadata":{},"cell_type":"markdown","source":"# Bidirectional LSTM + GRU + Attention"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Custom Layers\nclass Attention(nn.Module):\n    def __init__(self, feature_dim, step_dim, bias=True, **kwargs):\n        super(Attention, self).__init__(**kwargs)\n        \n        self.supports_masking = True\n\n        self.bias = bias\n        self.feature_dim = feature_dim\n        self.step_dim = step_dim\n        self.features_dim = 0\n        \n        weight = torch.zeros(feature_dim, 1)\n        nn.init.xavier_uniform_(weight)\n        self.weight = nn.Parameter(weight)\n        \n        if bias:\n            self.b = nn.Parameter(torch.zeros(step_dim))\n        \n    def forward(self, x, mask=None):\n        feature_dim = self.feature_dim\n        step_dim = self.step_dim\n\n        eij = torch.mm(\n            x.contiguous().view(-1, feature_dim), \n            self.weight\n        ).view(-1, step_dim)\n        \n        if self.bias:\n            eij = eij + self.b\n            \n        eij = torch.tanh(eij)\n        a = torch.exp(eij)\n        \n        if mask is not None:\n            a = a * mask\n\n        a = a / torch.sum(a, 1, keepdim=True) + 1e-10\n\n        weighted_input = x * torch.unsqueeze(a, -1)\n        return torch.sum(weighted_input, 1)","execution_count":29,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BiLSTMGRUAttention(nn.Module):\n    def __init__(self):\n        super(BiLSTMGRUAttention, self).__init__()\n        hidden_size = 512\n        self.embedding = nn.Embedding(max_features, embed_size)\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        self.embedding.weight.requires_grad = False      \n        self.embedding_dropout = nn.Dropout2d(0.1)\n        self.lstm = nn.LSTM(embed_size, 128, bidirectional=True, batch_first=True)\n        self.gru = nn.GRU(256, 256*2, bidirectional=True, batch_first=True)  \n        self.lstm_attention = Attention(128 * 2, maxlen)\n        self.gru_attention = Attention(256 * 4, maxlen)\n        self.linear = nn.Linear(3328,512)\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(0.1)\n        self.linear2 = nn.Linear(512, 256)\n        self.linear3 = nn.Linear(256, 64)\n        self.out = nn.Linear(64, 1)\n\n    def forward(self, x):\n        h_embedding = self.embedding(x)\n        h_embedding = torch.squeeze(\n            self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))\n        \n        h_lstm, _ = self.lstm(h_embedding)\n        h_gru, _ = self.gru(h_lstm)\n        \n        h_lstm_atten = self.lstm_attention(h_lstm)\n        h_gru_atten = self.gru_attention(h_gru)\n        \n        # global average pooling\n        avg_pool = torch.mean(h_gru, 1)\n        # global max pooling\n        max_pool, _ = torch.max(h_gru, 1)\n        \n        conc = torch.cat((h_lstm_atten, h_gru_atten, avg_pool, max_pool), 1)\n        conc = self.relu(self.linear(conc))\n        \n        conc = self.relu(self.linear2(conc))\n        conc = self.relu(self.linear3(conc))\n#         conc = self.dropout(conc)\n\n        out = self.out(conc)\n        return out","execution_count":30,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = BiLSTMGRUAttention()\ntrain_oofpreds, test_oofpreds = TrainingInference(model, epochs=15)","execution_count":null,"outputs":[{"output_type":"stream","text":"Fold 1\nEpoch 1/15 \t loss=179.8803 \t val_loss=111.1615 \t time=1.85s\nEpoch 2/15 \t loss=115.1646 \t val_loss=117.9021 \t time=1.72s\nEpoch 3/15 \t loss=106.5927 \t val_loss=104.0283 \t time=1.71s\nEpoch 4/15 \t loss=93.7740 \t val_loss=96.1409 \t time=1.71s\nEpoch 5/15 \t loss=85.9024 \t val_loss=79.0039 \t time=1.71s\nEpoch 6/15 \t loss=77.6390 \t val_loss=87.1034 \t time=1.72s\nEpoch 7/15 \t loss=73.2798 \t val_loss=74.6124 \t time=1.71s\nEpoch 8/15 \t loss=69.6234 \t val_loss=71.6010 \t time=1.71s\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = (train_oofpreds > .5).astype('uint')\nprint(f\"Confusion Matrix \\n{classification_report(y_train, predictions)}\")\nprint(f\"Accuracy {accuracy_score(y_train, predictions)}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Multiclass Classifications"},{"metadata":{},"cell_type":"markdown","source":"For multiclass problem let us take **Spooky Author Identification Data** and lets take only 10000 rows"},{"metadata":{"trusted":true},"cell_type":"code","source":"embed_size = 300 \nmax_features = 10000\nmaxlen = 500\n\n\ntrain_df = pd.read_csv(\"../input/spooky-author-identification/train.csv\", nrows= nrows)\ntest_df = pd.read_csv(\"../input/spooky-author-identification/test.csv\", nrows=nrows)\nprint('Train data dimension: ', train_df.shape)\nprint('Test data dimension: ', test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[\"text\"] = train_df[\"text\"].str.lower()\ntest_df[\"text\"] = test_df[\"text\"].str.lower()\n# fill up the missing values\nx_train = train_df[\"text\"].fillna(\"_##_\").values\nx_test = test_df[\"text\"].fillna(\"_##_\").values\n# Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(x_train))\nx_train = tokenizer.texts_to_sequences(x_train)\nx_test = tokenizer.texts_to_sequences(x_test)\n# Pad the sentences \nx_train = pad_sequences(x_train, maxlen=maxlen)\nx_test = pad_sequences(x_test, maxlen=maxlen)\n# Get the target values\ny_train = train_df['author'].values\ny_dumm = pd.get_dummies(train_df['author'].values).values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['author'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fastText_embeddings = load_fasttext(tokenizer.word_index)\nembedding_matrix = fastText_embeddings","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nle = LabelEncoder()\ny_train = le.fit_transform(y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BiLSTMGRUAttention(nn.Module):\n    def __init__(self):\n        super(BiLSTMGRUAttention, self).__init__()\n        hidden_size = 256\n        self.embedding = nn.Embedding(max_features, embed_size)\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        self.embedding.weight.requires_grad = False      \n        self.embedding_dropout = nn.Dropout2d(0.1)\n        self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)\n        self.gru = nn.GRU(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)  \n        self.lstm_attention = Attention(hidden_size * 2, maxlen)\n        self.gru_attention =  Attention(hidden_size * 2, maxlen)\n        self.linear = nn.Linear(1024, 16)\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(0.1)\n        self.out = nn.Linear(16, 3)\n    \n    def forward(self, x):\n        h_embedding = self.embedding(x)\n        h_embedding = torch.squeeze(\n            self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))\n        \n        h_lstm, _ = self.lstm(h_embedding)\n        h_gru, _ = self.gru(h_lstm)\n        \n        h_lstm_atten = self.lstm_attention(h_lstm)\n        h_gru_atten = self.gru_attention(h_gru)\n        \n        # global average pooling\n        avg_pool = torch.mean(h_gru, 1)\n        # global max pooling\n        max_pool, _ = torch.max(h_gru, 1)\n        \n        conc = torch.cat((h_lstm_atten, h_gru_atten, avg_pool, max_pool), 1)\n        conc = self.relu(self.linear(conc))\n        conc = self.dropout(conc)\n        out = self.out(conc)\n        return out","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We need to change our Inference Function as well and set loss to `loss_fn = torch.nn.CrossEntropyLoss(reduction='sum')` and other init changes to print and compile multiclass losses"},{"metadata":{"trusted":true},"cell_type":"code","source":"def MultiClassTrainingInference(model, batch_size=512, epochs=5):\n    n_epochs = epochs # how many times to iterate over all samples\n    # matrix for the out-of-fold predictions\n    train_preds = np.zeros((len(train_df)))\n    # matrix for the predictions on the test set\n    test_preds = np.zeros((len(test_df)))\n    x_test_cuda = torch.tensor(x_test, dtype=torch.long).cuda()\n    test = torch.utils.data.TensorDataset(x_test_cuda)\n    test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)\n    splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=10).split(x_train, y_train))\n    for i, (train_idx, valid_idx) in enumerate(splits):    \n        # split data in train / validation according to the KFold indeces\n        # also, convert them to a torch tensor and store them on the GPU (done with .cuda())\n        x_train_fold = torch.tensor(x_train[train_idx], dtype=torch.long).cuda()\n        y_train_fold = torch.tensor(y_dumm[train_idx, np.newaxis], dtype=torch.float32).cuda()\n        x_val_fold = torch.tensor(x_train[valid_idx], dtype=torch.long).cuda()\n        y_val_fold = torch.tensor(y_dumm[valid_idx, np.newaxis], dtype=torch.float32).cuda()\n\n        # make sure everything in the model is running on the GPU\n        model.cuda()\n        # define cross entropy loss\n        # for numerical stability in the loss\n        loss_fn = torch.nn.CrossEntropyLoss(reduction='mean')\n        optimizer = torch.optim.Adam(model.parameters())\n\n        train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)\n        valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)\n\n        train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n        valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)\n\n        print(f'Fold {i + 1}')\n\n        for epoch in range(n_epochs):\n            # set train mode of the model. This enables operations which are only applied during training like dropout\n            start_time = time.time()\n            model.train()\n            avg_loss = 0.  \n            for x_batch, y_batch in tqdm(train_loader, disable=True):\n                # Forward pass: compute predicted y by passing x to the model.\n                y_pred = model(x_batch)\n\n                # Compute and print loss.\n                loss = loss_fn(y_pred, y_batch.view(-1,3).max(1)[1])\n\n                # Before the backward pass, use the optimizer object to zero all of the\n                # gradients for the Tensors it will update (which are the learnable weights\n                # of the model)\n                optimizer.zero_grad()\n\n                # Backward pass: compute gradient of the loss with respect to model parameters\n                loss.backward()\n\n                # Calling the step function on an Optimizer makes an update to its parameters\n                optimizer.step()\n                avg_loss += loss.item() / len(train_loader)\n\n            # set evaluation mode of the model. This disabled operations which are only applied during training like dropout\n            model.eval()\n\n            # predict all the samples in y_val_fold batch per batch\n            valid_preds_fold = np.zeros((x_val_fold.size(0)))\n            test_preds_fold = np.zeros((len(test_df)))\n\n            avg_val_loss = 0.\n            for i, (x_batch, y_batch) in enumerate(valid_loader):\n                y_pred = model(x_batch).detach()\n\n                avg_val_loss += loss_fn(y_pred, y_batch.view(-1,3).max(1)[1]).item() / len(valid_loader)\n                \n                _, pred_label = torch.max(y_pred.cpu().data, 1)\n                \n                valid_preds_fold[i * batch_size:(i+1) * batch_size] = pred_label\n\n            elapsed_time = time.time() - start_time \n            print('Epoch {}/{} \\t loss={:.4f} \\t val_loss={:.4f} \\t time={:.2f}s'.format(\n                epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))\n\n        # predict all samples in the test set batch per batch\n        for i, (x_batch,) in enumerate(test_loader):\n            y_pred = model(x_batch).detach()\n            \n            _, pred_label = torch.max(y_pred.cpu().data, 1)\n\n\n            test_preds_fold[i * batch_size:(i+1) * batch_size] = pred_label\n\n        train_preds[valid_idx] = valid_preds_fold\n        test_preds += test_preds_fold / len(splits)\n    return train_preds, test_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# del loss_fn\ntorch.cuda.empty_cache() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = BiGRUClassifier()\ntrain_oofpreds, test_oofpreds = MultiClassTrainingInference(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = (train_oofpreds)\nprint(f\"Confusion Matrix \\n{classification_report(y_train, predictions)}\")\nprint(f\"Accuracy {accuracy_score(y_train, predictions)}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}