{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"print (os.listdir('../input'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import nltk\nimport operator\nimport re\nfrom sklearn.model_selection import train_test_split\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = '../input/quora-insincere-questions-classification/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Loading data\ntrain_df = pd.read_csv(data + 'train.csv')\nprint (train_df.shape)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df_len = train_df.shape[0]\nprint ('Train data length: ',train_df_len)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# statistics of target 1 vs target 0\nt0, t1 = len(train_df[train_df.target == 0]), len(train_df[train_df.target == 1])\nt0_pct, t1_pct = t0/train_df_len * 100, t1/train_df_len * 100\nprint (f'Target 0 vs Target 1 = {t0} vs {t1} ,{t0_pct:.2f}% vs {t1_pct:.2f}%')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(data + 'test.csv')\ntest_df_len = test_df.shape[0]\nprint ('Test data length: ',test_df_len)\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df = pd.read_csv(data + 'sample_submission.csv')\nprint ('sample sub length: ', sample_df.shape[0])\nsample_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del sample_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preprocessing\n#### From the reference, Paragram will be used as pre-trained embeddings.\n#### Preprocessing steps - \n1. lower\n2. clean contractions\n3. replace special characters\n4. tokenize\n5. remove stopwords\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# contraction corrections\ncontraction_mapping = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\" }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_contractions(text, contraction_mapping):\n    specials = [\"’\", \"‘\", \"´\", \"`\"]\n    for s in specials:\n        text = text.replace(s, \"'\")\n    text = ' '.join([contraction_mapping[word] if word in contraction_mapping else word for word in text.split(\" \")])\n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# example\ntext = \"I can`t go to work today. I'd rather stay home.\"\ntext = clean_contractions(text, contraction_mapping)\ntext","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# special characters\npunct = \"/-'?!.,#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~\" + '\"\"“”’' + '∞θ÷α•à−β∅³π‘₹´°£€\\×™√²—–&'\npunct","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"punct_mapping = {\n    \"‘\": \"'\",    \"₹\": \"e\",      \"´\": \"'\", \"°\": \"\",         \"€\": \"e\",\n    \"™\": \"tm\",   \"√\": \" sqrt \", \"×\": \"x\", \"²\": \"2\",        \"—\": \"-\",\n    \"–\": \"-\",    \"’\": \"'\",      \"_\": \"-\", \"`\": \"'\",        '“': '\"',\n    '”': '\"',    '“': '\"',      \"£\": \"e\", '∞': 'infinity', 'θ': 'theta',\n    '÷': '/',    'α': 'alpha',  '•': '.', 'à': 'a',        '−': '-',\n    'β': 'beta', '∅': '',       '³': '3', 'π': 'pi'\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_special_chars(text, punct, punct_mapping):\n    for p in punct_mapping:\n        text = text.replace(p, punct_mapping[p])\n    for p in punct:\n        text = text.replace(p, f' {p} ')\n    specials = {'\\u200b': ' ', '…': ' ... ', '\\ufeff': '', 'करना': '', 'है': ''}  # Other special characters that I have to deal with in last\n    for s in specials:\n        text = text.replace(s, specials[s])\n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# example\ntext = \"I have $20. So, I can buy an awesome watch!!\"\ntext = clean_special_chars(text,punct,punct_mapping)\ntext","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stopwords = nltk.corpus.stopwords.words('english')\nstopwords","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess(df, contraction_mapping, punct, punct_mapping):\n    texts = df.question_text\n    processed_texts = texts.apply(lambda x:x.lower())\n    processed_texts = processed_texts.apply(lambda x: clean_contractions(x, contraction_mapping))\n    processed_texts = processed_texts.apply(lambda x: clean_special_chars(x, punct, punct_mapping))\n    processed_texts = processed_texts.apply(lambda x: re.split('\\W+', x))\n    processed_texts = processed_texts.apply(lambda x: [token for token in x if token not in stopwords])\n    df['processed_text'] = processed_texts","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Choose data from trainset\nIn this training, some portion of the data will be used. The number of negative data is very small compared to positive.\nSince the test run with big portion of positive data made the result worse, the positive data is cut down to some portion. How many to read is a big question though.\n* test data: 56370\n* target 0/1 ratio: 93.81/6.19%, 1225312/80810 (very skewed)\n\nThe total number of data is set to become 10x of test data after train/validation split by 0.9 to 0.1","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_rows_t0 = 639190    # positive data\nsample_rows_t1 = 80810     # negative data\ndf_t0 = train_df[train_df.target == 0].sample(sample_rows_t0)\ndf_t1 = train_df[train_df.target == 1].sample(sample_rows_t1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print (f'df_t0 length : {df_t0.shape[0]}')\nprint (f'df_t1 length : {df_t1.shape[0]}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preprocess(df_t0, contraction_mapping, punct, punct_mapping)\ndf_t0.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preprocess(df_t1, contraction_mapping, punct, punct_mapping)\ndf_t1.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preprocess(test_df, contraction_mapping, punct, punct_mapping)\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Find Vocabulary\nMemory restriction is tight. Loading whole pretrained embeddings easily leads to memory exhaustion. To save memory, below just grabs vocabulary found in train and test data.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_vocab(texts, vocab):\n    for word in texts:\n        vocab.add(word)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab = set()\ndf_t1.processed_text.apply(lambda x:build_vocab(x,vocab))\ndf_t0.processed_text.apply(lambda x:build_vocab(x,vocab))\ntest_df.processed_text.apply(lambda x:build_vocab(x,vocab))\nprint (len(vocab))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loading Embeddings","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from zipfile import ZipFile\nimport codecs\nfile = ZipFile('../input/quora-insincere-questions-classification/embeddings.zip','r')\nprint (file.printdir())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# choosing paragram \nparagram = file.open(file.namelist()[8])     # since we want to use paragram as pretrained embeddings, hence index 8","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load embeddings\nword2vec = {}\ni = 0\ndef get_coefs(word, *arr):\n    return word, np.asarray(arr, dtype='float32')\nfor line in tqdm(codecs.iterdecode(paragram,'latin')):\n    word, coefs = get_coefs(*line.split(\" \"))\n    if word in vocab:\n        word2vec[word] = coefs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print ('Vocab length: ',len(vocab))\nprint ('Word2Vec length: ',len(word2vec))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Question Word statistics\nThe number of question in words varies. To deal with both long and short questions, we find the appropriate number of words.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"lens_t0 = list(map(len, df_t0.processed_text))\nlens_t1 = list(map(len, df_t1.processed_text))\nlens_test = list(map(len, test_df.processed_text))\n\nprint (f'For positive questions: Min words:{min(lens_t0)} vs Max words:{max(lens_t0)}')\nprint (f'For negative questions: Min words:{min(lens_t1)} vs Max words:{max(lens_t1)}')\nprint (f'For test questions: Min words:{min(lens_test)} vs Max words:{max(lens_test)}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def freq_stats(tag,counts, key, topk, total):\n    most_freqs = sorted(counts, key=key, reverse=True)[:topk]\n    freqs = [counts[freq] for freq in most_freqs]\n    \n    print (f'{tag}: best {topk} frequent word count: {most_freqs}')\n    print (f'freqs: {freqs}')\n    print (f'Covers: {sum(freqs)/total*100:.2f}%')\n    \n    return max(most_freqs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import Counter\n\ncounts_t0 = Counter(lens_t0) # counts words freq. Ex. How many 13 words questions are there?\ncounts_t1 = Counter(lens_t1)\ncounts_test = Counter(lens_test)\n\ntopk = 20  # pick top 20 freq of words\nmax_t0 = freq_stats('pos',counts_t0, counts_t0.get, topk, sample_rows_t0)\nmax_t1 = freq_stats('neg',counts_t1, counts_t1.get, topk, sample_rows_t1)\nmax_test = freq_stats('test',counts_test, counts_test.get, topk, test_df_len)\n\nprint (max_t0, max_t1, max_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seq_length = max(max_t0, max_t1, max_test)\nseq_length","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word2vec['india'].shape[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build Word Matrix","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_weights_matrix(word2vec):\n    word2idx = {}\n    weights_matrix = np.zeros((len(word2vec), 300))\n    for i, (k,v) in enumerate(word2vec.items()):\n        word2idx[k] = i\n        weights_matrix[i] = v\n    return word2idx, weights_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word2idx, weights_matrix = build_weights_matrix(word2vec)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weights_matrix.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def encode_question(word2idx, text, seq_length):\n    encoded = []\n    for word in text[:seq_length]:\n        try:\n            encoded.append(word2idx[word])\n        except KeyError:\n            # missing words in the table\n            continue\n    \n    return np.array(encoded, dtype='int_')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_padding(np_arr, seq_length):\n    curr_length = np_arr.shape[0]\n    if curr_length < seq_length:\n        padding = np.zeros((seq_length - curr_length, ), dtype = 'int_')\n        return np.concatenate((padding,np_arr))\n    else:\n        return np_arr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_dataset(texts, label, word2idx, seq_length):\n    texts_len = len(texts)\n    y = np.array([label]*texts_len, dtype='float')\n    X = []\n    for i, text in enumerate(texts):\n        text_array = encode_question(word2idx, text, seq_length)\n        text_array = add_padding(text_array, seq_length)\n        X.append(text_array)\n    return np.array(X), y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# split train data to train and validation\ntest_size = 0.1\ntrain_texts_t0, val_texts_t0 = train_test_split(df_t0.processed_text, test_size = test_size)\ntrain_texts_t1, val_texts_t1 = train_test_split(df_t1.processed_text, test_size = test_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X_t0, train_y_t0 = create_dataset(train_texts_t0, 0, word2idx, seq_length)\ntrain_X_t1, train_y_t1 = create_dataset(train_texts_t1, 1, word2idx, seq_length)\n\ntrain_X = np.concatenate((train_X_t0, train_X_t1))\ntrain_y = np.concatenate((train_y_t0, train_y_t1))\n\nprint (f'Shapes: train_X {train_X.shape}, train_y {train_y.shape}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_X_t0, val_y_t0 = create_dataset(val_texts_t0, 0, word2idx, seq_length)\nval_X_t1, val_y_t1 = create_dataset(val_texts_t1, 1, word2idx, seq_length)\n\nval_X = np.concatenate((val_X_t0, val_X_t1))\nval_y = np.concatenate((val_y_t0, val_y_t1))\n\nprint (f'Shapes: val_X {val_X.shape}, val_y {val_y.shape}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Pytorch - ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# importing libraries\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset, TensorDataset\nfrom torch.autograd import Variable","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# device config\ndevice = 'cuda:0' if torch.cuda.is_available() else 'cpu'\ndevice","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create tensor dataset\ntrain_set = TensorDataset(torch.from_numpy(train_X), torch.from_numpy(train_y))\nval_set = TensorDataset(torch.from_numpy(val_X), torch.from_numpy(val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create dataloader\nbatch_size = 200\n\ntrain_loader = DataLoader(train_set, shuffle=True, batch_size=batch_size)\nval_loader = DataLoader(val_set, shuffle=True, batch_size=batch_size)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Building Network Architecture\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class RNN(nn.Module):\n    def __init__(self, weights, output_size, hidden_size, n_layers, bidirectional=False, dropout=0.5,layer_dropout=0.3):\n        super(RNN, self).__init__()\n        \n        self.output_size = output_size\n        self.hidden_size = hidden_size\n        self.n_layers = n_layers\n        if bidirectional:\n            self.direction = 2\n        else:\n            self.direction = 1\n        \n        num_embeddings, embedding_dim = weights.shape\n        \n        # embedding layer\n        self.embedding = nn.Embedding(num_embeddings, embedding_dim)\n        self.embedding.weight.data.copy_(torch.from_numpy(weights))\n        self.embedding.requires_grad = False\n        \n        # LSTM layer\n        self.lstm = nn.LSTM(embedding_dim, hidden_size, n_layers, batch_first=True, dropout=dropout,bidirectional = bidirectional)\n        \n        # GRU layer\n        self.gru = nn.GRU(embedding_dim, hidden_size, n_layers, batch_first=True, dropout=dropout,bidirectional = bidirectional)\n            \n        # dropout layer\n        self.dropout = nn.Dropout(layer_dropout)\n        \n        # Fully Connected Layer\n        self.fc = nn.Linear((hidden_size*self.direction), output_size)\n        \n        # Sigmoid activation layer\n        self.sig = nn.Sigmoid()\n        \n    def forward(self, x, hidden):\n        batch_size = x.size(0)\n        seq_len = x.size(1)\n        lstm_hidden = hidden\n        \n        embeds = self.embedding(x)\n        \n        lstm_out, lstm_hidden = self.lstm(embeds, lstm_hidden)\n        lstm_out = lstm_out.contiguous().view(-1, self.hidden_size)\n        \n        out = self.dropout(lstm_out)\n        out = self.fc(out)\n        sig_out = self.sig(out)\n        sig_out = sig_out.view(batch_size, -1)\n        sig_out = sig_out[:, -1]   # get last batch of labels\n        \n        return  sig_out, hidden\n\n    def init_hidden(self, batch_size, bidirectional=False):\n        weight = next(self.parameters()).data\n        # for LSTM (initial_hidden_state, initial_cell_state)\n        lstm_hidden = (\n            weight.new(self.n_layers*self.direction, batch_size, self.hidden_size).zero_().to(device),\n            weight.new(self.n_layers*self.direction, batch_size, self.hidden_size).zero_().to(device)\n        )\n        # for GRU, initial_hidden_state\n        #gru_hidden = weight.new(self.n_layers*self.direction, batch_size, self.n_hidden).zero_().to(DEVICE)\n        return lstm_hidden\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Instantiate the Network\n# Hyperparams\noutput_size = 1\nhidden_size = 256\nn_layers = 2\n\n\nnet = RNN(weights_matrix, output_size, hidden_size, n_layers, bidirectional=False).to(device)\nprint(net)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Training params\nlr = 0.00001\nepochs = 10\nclip = 5  # gradient clipping","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# loss and optimizer functions\ncriterion = nn.BCELoss()\noptimizer = optim.Adam(net.parameters(), lr=lr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import datetime","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(net, criterion, optimizer, train_loader, clip, epoch, epochs, gru=False):\n    counter = 0\n    print_every = 500\n    train_length = len(train_loader)\n    \n    # init hidden state\n    h = net.init_hidden(batch_size)\n    \n    train_losses = []\n    net.train()\n    for inputs, labels in train_loader:\n        counter += 1\n        inputs, labels = inputs.to(device), labels.to(device)\n        \n        h = tuple([each.data for each in h])\n        \n        # zero accumulated gradients\n        net.zero_grad()\n        \n        # forward pass\n        outputs, h = net(inputs, h)\n        \n        # calculate loss and perform backprop\n        loss = criterion(outputs.squeeze(), labels.float())\n        loss.backward()\n        \n        # clip_grad_norm helps prevent exploding gradient\n        nn.utils.clip_grad_norm_(net.parameters(), clip)\n        optimizer.step()\n        \n        # loss stats\n        if counter % print_every == 0:\n            train_losses.append(loss.item())\n            print (f'Epoch: {epoch+1}/{epochs} \\t Step: {counter} \\t Train Loss: {np.mean(train_losses):.6f} \\t Time: {datetime.datetime.now()}')\n            ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# validation loss\ndef validate(net, criterion, optimizer, val_loader, epoch, epochs, gru=False):\n    # init hidden state\n    h = net.init_hidden(batch_size)\n    \n    val_losses = []\n    acc = 0.0\n    net.eval()\n    with torch.no_grad():\n        for inputs, labels in val_loader:\n            inputs, labels = inputs.to(device), labels.to(device)\n            \n            h = tuple([each.data for each in h])\n\n            # forward pass\n            outputs, h = net(inputs, h)\n\n            # calculate loss and perform backprop\n            val_loss = criterion(outputs.squeeze(), labels.float())\n            val_losses.append(val_loss.item())\n            \n            acc += torch.eq(labels.float(), torch.round(outputs.squeeze())).sum().item()\n            \n        print (f'Epoch: {epoch+1}/{epochs} \\t Val Loss: {np.mean(val_losses):.6f} \\t Acc: {(acc/(len(val_loader)*batch_size))*100:.2f}% \\t Time: {datetime.datetime.now()}')    \n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_train(net, criterion, optimizer, epochs, train_loader, val_loader, clip, gru=False):\n    for epoch in range(epochs):\n        print ('Running epoch {}...\\n'.format(epoch+1))\n        train(net, criterion, optimizer, train_loader ,clip, epoch, epochs, gru)\n        validate(net, criterion, optimizer, val_loader, epoch, epochs, gru)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"run_train(net, criterion, optimizer, epochs, train_loader, val_loader, clip,gru=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class QuoraTestDataset(Dataset):\n    def __init__(self, df, word2idx, seq_length):\n        self.word2idx = word2idx\n        self.seq_length = seq_length\n        self.data = df\n        self.data_len = len(df)\n        \n    def __len__(self):\n        return self.data_len\n\n    def __getitem__(self, idx):\n        if idx >= self.data_len:\n            idx %= self.data_len\n            \n        #preprocessed\n        tokens = self.data.iloc[idx].processed_text\n        \n        # encode to make array of indices\n        encoded = encode_question(word2idx, tokens, self.seq_length)\n        text_array = add_padding(encoded, self.seq_length)\n        return self.data.iloc[idx].qid, torch.from_numpy(text_array)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create dataset\ntest_set = QuoraTestDataset(test_df, word2idx, seq_length)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_batch_size = 41","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create dataloader\ntest_loader = DataLoader(test_set, shuffle=False, batch_size=test_batch_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def test(net, test_loader, batch_size=test_batch_size):\n    test_l_h = net.init_hidden(batch_size)\n    ret_qid = []\n    ret_pred = []\n    test_len = len(test_loader)\n    counter = 0\n    with torch.no_grad():\n        for qids, inputs in test_loader:\n            counter += 1\n            inputs = inputs.to(device)\n            \n            # for LSTM\n            test_l_h = tuple([each.data for each in test_l_h])\n\n            outputs, test_l_h = net(inputs, test_l_h)\n            \n            ret_qid.append(qids)\n            ret_pred.append(torch.round(outputs.squeeze()).cpu().numpy().astype(int))\n            \n            if counter % 300 == 0:\n                print('{}/{} done'.format(counter, test_len))\n\n    return ret_qid, ret_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ret_qid, ret_pred = test(net, test_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ret_qid = np.concatenate(ret_qid)\nret_pred = np.concatenate(ret_pred)\nprint (len(ret_qid))\nprint (len(ret_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit_df = pd.DataFrame({'qid': ret_qid, 'prediction': ret_pred})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit_df.to_csv(\"submission.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}