{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import datetime\nimport nltk\nimport operator \nimport re\nfrom sklearn.model_selection import train_test_split\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b6ba47f27a6de45263d2b6233784878820a03e4b"},"cell_type":"markdown","source":"### Loading data\nBelow loads train and test data and creates Pandas dataframes."},{"metadata":{"trusted":true,"_uuid":"892617bb25c0f6ccbf07b68ae5ed7397888e0d5f"},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntrain_df_len = train_df.shape[0]\nprint('train data length: {}'.format(train_df_len)) # 1306122\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d1be4e2f434b0b8738630427e439aae0d73ae9d6"},"cell_type":"code","source":"# statistis of target 0 and 1\nt0, t1 = len(train_df[train_df.target == 0]), len(train_df[train_df.target == 1])\nt0_pct, t1_pct = t0 / train_df_len * 100, t1 / train_df_len * 100\nprint('target 0 vs 1 = {} vs {}, {:.2f}% vs {:.2f}%'.format(t0, t1, t0_pct, t1_pct))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4eb5a6a8e12a387198a66ff38aeb8fde02ef0e4f"},"cell_type":"code","source":"test_df = pd.read_csv(\"../input/test.csv\")\nprint('test data length: {}'.format(test_df.shape[0]))\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"985f4ac502780cd3cf1cb66e1e49c873e923b89c"},"cell_type":"code","source":"sample_df = pd.read_csv('../input/sample_submission.csv')\nprint('sample submission length: {}'.format(sample_df.shape[0]))\nsample_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f4ff1de1e03f1adc3928c3a66bbd5f1714f5d5b4"},"cell_type":"code","source":"del sample_df","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8c6193382cf98237c883aba046b71da185d358f6"},"cell_type":"markdown","source":"#### Preprocessing\nReference: https://www.kaggle.com/theoviel/improve-your-score-with-some-text-preprocessing/notebook\nFrom the reference, Paragram will be used as pretrained embeddings.  Preprocessings are done in order described below.\n\n1. lower\n2. clean contractions\n3. replace special characters\n4. tokenize\n5. remove stopwords"},{"metadata":{"trusted":true,"_uuid":"c08844b52bda4cad61a19549661ab5f172dcd622"},"cell_type":"code","source":"# Contractions corrections\ncontraction_dict = {\n    \"ain't\": \"is not\", \"aren't\": \"are not\", \"can't\": \"cannot\",\n    \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\",\n    \"didn't\": \"did not\", \"doesn't\": \"does not\", \"don't\": \"do not\",\n    \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\",\n    \"he'd\": \"he would\", \"he'll\": \"he will\", \"he's\": \"he is\",\n    \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\",\n    \"how's\": \"how is\", \"I'd\": \"I would\", \"I'd've\": \"I would have\",\n    \"I'll\": \"I will\", \"I'll've\": \"I will have\", \"I'm\": \"I am\",\n    \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\",\n    \"i'll\": \"i will\",  \"i'll've\": \"i will have\", \"i'm\": \"i am\",\n    \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\",\n    \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\n    \"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\",\n    \"mayn't\": \"may not\", \"might've\": \"might have\", \"mightn't\": \"might not\",\n    \"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\",\n    \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\n    \"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\",\n    \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\",\n    \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\",\n    \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\",\n    \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\n    \"so's\": \"so as\", \"this's\": \"this is\", \"that'd\": \"that would\",\n    \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\",\n    \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\n    \"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\",\n    \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\",\n    \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\",\n    \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\",\n    \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\",\n    \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",\n    \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\",\n    \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\",\n    \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\",\n    \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\",\n    \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\",\n    \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\",\n    \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\n    \"y'all'd've\": \"you all would have\", \"y'all're\": \"you all are\", \"y'all've\": \"you all have\",\n    \"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\",\n    \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\"\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"27abf016e6803d8f9a74eed17642114974f8162d"},"cell_type":"code","source":"def clean_contractions(text, contraction_dict):\n    specials = [\"’\", \"‘\", \"´\", \"`\"]\n    for s in specials:\n        text = text.replace(s, \"'\")\n    text = ' '.join([contraction_dict[t] if t in contraction_dict else t for t in text.split(\" \")])\n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b4dfe004de5f954fa1fb64db79882cd04374f6f"},"cell_type":"code","source":"# special characters\npunct = \"/-'?!.,#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~\" + '\"\"“”’' + '∞θ÷α•à−β∅³π‘₹´°£€\\×™√²—–&'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"19f979f5b0310b93ad055f03703305eaebce47f5"},"cell_type":"code","source":"punct_dict = {\n    \"‘\": \"'\",    \"₹\": \"e\",      \"´\": \"'\", \"°\": \"\",         \"€\": \"e\",\n    \"™\": \"tm\",   \"√\": \" sqrt \", \"×\": \"x\", \"²\": \"2\",        \"—\": \"-\",\n    \"–\": \"-\",    \"’\": \"'\",      \"_\": \"-\", \"`\": \"'\",        '“': '\"',\n    '”': '\"',    '“': '\"',      \"£\": \"e\", '∞': 'infinity', 'θ': 'theta',\n    '÷': '/',    'α': 'alpha',  '•': '.', 'à': 'a',        '−': '-',\n    'β': 'beta', '∅': '',       '³': '3', 'π': 'pi'\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b1410b95fc13dcd0e1f70f394cb513f12fd5cd8e"},"cell_type":"code","source":"def clean_special_chars(text, punct, punct_dict):\n    for p in punct_dict:\n        text = text.replace(p, punct_dict[p])\n    \n    for p in punct:\n        text = text.replace(p, f' {p} ')\n    \n    specials = {'\\u200b': ' ', '…': ' ... ', '\\ufeff': '', 'करना': '', 'है': ''}  # Other special characters that I have to deal with in last\n    for s in specials:\n        text = text.replace(s, specials[s])\n    \n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d9ca0f418d218a10ed3229aa78b2aca14ad731eb"},"cell_type":"code","source":"stopwords = nltk.corpus.stopwords.words('english')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e2bf457dd2bf4fa4203a16cafac2794c0c05c998"},"cell_type":"code","source":"def preprocess(df, contraction_dict, punct, punct_dict):\n    texts = df.question_text\n    processed_texts = texts.apply(lambda x: x.lower())\n    processed_texts = processed_texts.apply(lambda x: clean_contractions(x, contraction_dict))\n    processed_texts = processed_texts.apply(lambda x: clean_special_chars(x, punct, punct_dict))\n    processed_texts = processed_texts.apply(lambda x: re.split('\\W+', x))\n    processed_texts = processed_texts.apply(lambda x: [token for token in x if token not in stopwords])\n    df['processed_text'] = processed_texts","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"807fb5b16eec9eb1dea25fb7b8358e82ee277fe9"},"cell_type":"markdown","source":"### Choose data from train dataset\nIn this training, some portion of train data will be used. The number of negative data is very small compared to positive. Since the test run with big portion of positive data made the result worse, the positive data is cut down to some portion. How many to read is a big question though.\n\n- test data: 56370\n- target 0/1 ratio: 93.81/6.19%, 1225312/80810 (very skewed)\n\nThe total number of data is set to become 10x of test data after train/validation split by 0.9 to 0.1"},{"metadata":{"trusted":true,"_uuid":"37cfebd620894ceb1b5afe2136a1713098223343"},"cell_type":"code","source":"#SAMPLE_ROWS_T0 = 575000\n#SAMPLE_ROWS_T0 = 1220000 # too many positive data makes test score worse\n#SAMPLE_ROWS_T1 = 80000\n#SAMPLE_ROWS_T0 = 273200 # too few\n#SAMPLE_ROWS_T0 = 547200\n#SAMPLE_ROWS_T1 = 80800\nSAMPLE_ROWS_T0 = 639190\nSAMPLE_ROWS_T1 = 80810\ndf_t0 = train_df[train_df.target==0].sample(SAMPLE_ROWS_T0)\ndf_t1 = train_df[train_df.target==1].sample(SAMPLE_ROWS_T1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6cd99eeba1ceda8aaa097deaa46bd3fa042801cd"},"cell_type":"code","source":"preprocess(df_t0, contraction_dict, punct, punct_dict)\ndf_t0.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef40ab7c0119436d070c52e3c6d90483b0785175"},"cell_type":"code","source":"preprocess(df_t1, contraction_dict, punct, punct_dict)\ndf_t1.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8f8d3a76ef601381274192888ebb210decaaecc"},"cell_type":"code","source":"preprocess(test_df, contraction_dict, punct, punct_dict)\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3dc47a842f2e480d4d02f5bbd51b2fcff1f9c78c"},"cell_type":"markdown","source":"### Find Vocabulary\nMemory restriction is tight. Loading whole pretrained embeddings easily leads to memory exhaustion. To save memory, below just grabs vocabulary found in train and test data."},{"metadata":{"trusted":true,"_uuid":"2cd2f24ea9f221a4c708294ccc78537f2687bb0e"},"cell_type":"code","source":"def build_vocab(texts, vocab):\n    for word in texts:\n        vocab.add(word)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b7bc2094f9def71c32ad3dd73ec523cb4c736831"},"cell_type":"code","source":"vocab = set()\ndf_t1.processed_text.apply(lambda x: build_vocab(x, vocab))\ndf_t0.processed_text.apply(lambda x: build_vocab(x, vocab))\ntest_df.processed_text.apply(lambda x: build_vocab(x, vocab))\nprint(len(vocab))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c4cae73690db8a3cb2af95ba671fff03ce52f6d5"},"cell_type":"markdown","source":"### Loading embeddings"},{"metadata":{"trusted":true,"_uuid":"2a2ac32570f36b927bc5d12034b58ef6ca1909ca"},"cell_type":"code","source":"def load_embed(filename, vocab):\n    word2vec = {}\n    def get_coefs(word,*arr): \n        return word, np.asarray(arr, dtype='float32')\n    f = open(filename, encoding='latin')\n    for line in tqdm(f):\n        word, coefs = get_coefs(*line.split(\" \"))\n        if word in vocab:\n            word2vec[word] = coefs\n    f.close()\n    return word2vec","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"81d229e65bca12c9b224197d305e6cb242912454"},"cell_type":"code","source":"#glove = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\nparagram =  '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n#wiki_news = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"54945a0180257eaae33db854e4cfd7efcbbdb833"},"cell_type":"code","source":"word2vec = load_embed(paragram, vocab)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"355ea1163f79f2ac0e4a23791c0820b66d706743"},"cell_type":"code","source":"len(word2vec), word2vec['add'].shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3f8e3661cf2ae2b6230e6f52bcc72bc2586a6369"},"cell_type":"markdown","source":"### Question Words Statistics\n\nThe number of words in a question varies. To deal with both short and long questions, below section finds the appropriate number of words."},{"metadata":{"trusted":true,"_uuid":"7a299f0e6079f630bf8b1196f15a7d96a74ae658"},"cell_type":"code","source":"# see the train data closely\n# min and max number of words in questions\nlens_t0 = list(map(len, df_t0.processed_text))\nlens_t1 = list(map(len, df_t1.processed_text))\nlens_test = list(map(len, test_df.processed_text))\nprint('min and max words in pos questions: {}, {}'.format(min(lens_t0), max(lens_t0)))\nprint('min and max words in neg questions: {}, {}'.format(min(lens_t1), max(lens_t1)))\nprint('min and max words in test questions: {}, {}'.format(min(lens_test), max(lens_test)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e2ece0e28f59b9d630dfa1f80919e0c8c24ac35"},"cell_type":"code","source":"def freq_stats(tag, counts, key, topk, total):\n    most_freqs = sorted(counts, key=key, reverse=True)[:topk]\n    freqs = [counts[freq] for freq in most_freqs]\n    print('{}: best {} frequent word count: {}, '.format(tag, topk, most_freqs),\n          'freqs: {}, '.format(freqs),\n          'covers: {:.2f}%'.format(sum(freqs)/total*100))\n    return max(most_freqs)\n\nfrom collections import Counter\ncounts_t0 = Counter(lens_t0)\ncounts_t1 = Counter(lens_t1)\ncounts_test = Counter(lens_test)\n#topk = 5 # vast majority of questions are covered, but may lose clues to classify correctly\ntopk = 20\nmax_t0 = freq_stats('pos', counts_t0, counts_t0.get, topk, SAMPLE_ROWS_T0)\nmax_t1 = freq_stats('neg', counts_t1, counts_t1.get, topk, SAMPLE_ROWS_T1)\nmax_test = freq_stats('test', counts_test, counts_test.get, topk, test_df.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3e8c0f46607b4bea547a97bc796c3fd88afcc24"},"cell_type":"code","source":"SEQ_LENGTH = max(max_t0, max_t1, max_test)\nSEQ_LENGTH","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"30afab99cdb811a914e2f064bbb73a9f3573bfeb"},"cell_type":"markdown","source":"### Build Word Matrix"},{"metadata":{"trusted":true,"_uuid":"d53cdc6da14de267eaae9d39b990e85552d207a7"},"cell_type":"code","source":"def build_weights_matrix(word2vec):\n    word_to_idx = {}\n    weights_matrix = np.zeros((len(word2vec), 300))\n    for i, (k, v) in enumerate(word2vec.items()):\n        word_to_idx[k] = i\n        weights_matrix[i] = v\n    return word_to_idx, weights_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4b165fc48286857455079a36a7be05e3ad41a5f3"},"cell_type":"code","source":"word_to_idx, weight_matrix = build_weights_matrix(word2vec)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e24db7499db57826f53964da7d69acd6dfdf610b"},"cell_type":"code","source":"# the length of word vector: seq_length\ndef encode_question(word_to_idx, text, seq_length):\n    encoded = []\n    for word in text[:seq_length]:\n        try:\n            encoded.append(word_to_idx[word])\n        except KeyError:\n            # missing words in the table such typos or created words\n            continue\n\n    return np.array(encoded, dtype='int_')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e6d1dbf8b7679aadf097b6f18a94a8cae5f59df5"},"cell_type":"code","source":"# adds padding\ndef add_padding(numpy_array, seq_length):\n    cur_length = numpy_array.shape[0]\n    if cur_length < seq_length:\n        padding = np.zeros((seq_length-cur_length, ), dtype='int_')\n        return np.concatenate((padding, numpy_array))\n    else:\n        return numpy_array","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cbe707f26eea82b596321738427db23ef2abf858"},"cell_type":"code","source":"def create_dataset(texts, label, word_to_idx, seq_length):\n    texts_len = len(texts)\n    y = np.array([label]*texts_len, dtype='float')\n    X = []\n    for i, text in enumerate(texts):\n        text_array = encode_question(word_to_idx, text, seq_length)\n        text_array = add_padding(text_array, seq_length)\n        X.append(text_array)\n    return np.array(X), y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d7b45cc63cf5bb8142fbde0162d43bf6094e99d9"},"cell_type":"code","source":"# splits train data to train and validation\nTEST_SIZE = 0.1\ntrain_texts_t0, val_texts_t0 = train_test_split(df_t0.processed_text, test_size=TEST_SIZE)\ntrain_texts_t1, val_texts_t1 = train_test_split(df_t1.processed_text, test_size=TEST_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d93312cbe2b552089daac7f7e6876332b5cff19"},"cell_type":"code","source":"train_X_t0, train_y_t0 = create_dataset(train_texts_t0, 0, word_to_idx, SEQ_LENGTH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"66db57ca5cb88e24d299ce0c7083c8ee6217a99a"},"cell_type":"code","source":"train_X_t1, train_y_t1 = create_dataset(train_texts_t1, 1, word_to_idx, SEQ_LENGTH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31a993eb8374373d66e68d2faee9b951292ef99d"},"cell_type":"code","source":"train_X = np.concatenate((train_X_t0, train_X_t1))\ntrain_y = np.concatenate((train_y_t0, train_y_t1))\nprint('shapes: train_X {}, train_y {}'.format(train_X.shape, train_y.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a5ac84709f23d253497025b9e2a0b9bd62ebb0b"},"cell_type":"code","source":"val_X_t0, val_y_t0 = create_dataset(val_texts_t0, 0, word_to_idx, SEQ_LENGTH)\nval_X_t1, val_y_t1 = create_dataset(val_texts_t1, 1, word_to_idx, SEQ_LENGTH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4118fbd38a0739a28f9c1c841a96ea2e49b9cb20"},"cell_type":"code","source":"val_X = np.concatenate((val_X_t0, val_X_t1))\nval_y = np.concatenate((val_y_t0, val_y_t1))\nprint('shapes: val_X {}, val_y {}'.format(val_X.shape, val_y.shape))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8e867e5ef012e2db3d15a64c91294ce5298c43dd"},"cell_type":"markdown","source":"### Create Network and Train - PyTorch"},{"metadata":{"trusted":true,"_uuid":"4f61d9592d2cfe3324ff5f53fed44704e098f74e"},"cell_type":"code","source":"import torch\nfrom torch.autograd import Variable\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset, TensorDataset\n\nprint(torch.__version__)\nDEVICE = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint(DEVICE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"256bff7b87695581c8a2fa546fb109a25e9eee4d"},"cell_type":"code","source":"# creates Tensor datasets\ntrain_set = TensorDataset(torch.from_numpy(train_X), torch.from_numpy(train_y))\nval_set = TensorDataset(torch.from_numpy(val_X), torch.from_numpy(val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9645ec8168bb09d5964c71284bf7ea2f37e275a3"},"cell_type":"code","source":"# creates dataloaders\n# hyperparameter for data loading\n#  - batch_size: size of one batch\nBATCH_SIZE = 200\n\n# make sure to SHUFFLE the training data\ntrain_loader = DataLoader(train_set, shuffle=True, batch_size=BATCH_SIZE)\nval_loader = DataLoader(val_set, shuffle=True, batch_size=BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"871ebd40472ebaa9a5e8feddcf1263781e951c1e"},"cell_type":"markdown","source":"### Define Network\n1. Embedding Layer from pretrained word2vec\n2. LSTM layer (may GRU?)\n3. Fully-connected output layer\n4. Sigmoid activation layer for final output"},{"metadata":{"trusted":true,"_uuid":"2e778643d84257a80b40ffc3ef16512000e59143"},"cell_type":"code","source":"# Only LSTM (2 or 3 layers) model suffered an overfitting problem.\n# To avoid the problem, GRU and average pooling layer were added.\n# The overfitting got better, but still the problem exists.\nclass SentimentRNN(nn.Module):\n    def __init__(self, weights, n_out, n_hidden, n_layers,\n                 bidirectional=False, dropout=0.5, layer_dropout=0.3):\n        super(SentimentRNN, self).__init__()\n\n        self.n_out = n_out\n        self.n_hidden = n_hidden\n        self.n_layers = n_layers\n        if bidirectional:\n            self.direction = 2\n        else:\n            self.direction = 1\n\n        num_embeddings, embedding_dim = weights.shape\n        \n        # embedding layer\n        self.embedding = nn.Embedding(num_embeddings, embedding_dim)\n        self.embedding.weight.data.copy_(torch.from_numpy(weights))\n        self.embedding.weight.requires_grad = False\n        # for some reason from_pretrained doesn't work\n        #self.embedding = nn.Embedding.from_pretrained(torch.from_numpy(weights))\n        \n        # LSTM layer\n        self.lstm = nn.LSTM(embedding_dim, n_hidden, n_layers,\n                            batch_first=True, dropout=dropout,\n                            bidirectional=bidirectional)\n        \n        # GRU layer\n        self.gru = nn.GRU(embedding_dim, n_hidden, n_layers,\n                          batch_first=True, dropout=dropout,\n                          bidirectional=bidirectional)\n        # Conv1d layer\n        self.conv1d = nn.Conv1d(n_hidden*self.direction, (n_hidden*self.direction)//2, 1)\n        # Average Pooling layer\n        self.avp = nn.AvgPool1d(2)\n        # Dropout layer\n        self.dropout = nn.Dropout(layer_dropout)\n        # Fully-conneted layer\n        self.fc = nn.Linear((n_hidden*self.direction)//4*2, n_out)\n        \n        # Sigmoid activation layer\n        self.sig = nn.Sigmoid()\n\n    def forward(self, x, hidden):\n        batch_size = x.size(0)\n        seq_len = x.size(1)\n        lstm_hidden, gru_hidden = hidden\n        \n        embeds = self.embedding(x)\n        \n        lstm_out, lstm_hidden = self.lstm(embeds, lstm_hidden)\n        lstm_out = lstm_out.contiguous().view(-1, self.n_hidden*self.direction, seq_len)\n        lstm_out = self.conv1d(lstm_out)\n        lstm_out = lstm_out.contiguous().view(-1, seq_len, (self.n_hidden*self.direction)//2)\n        lstm_out = self.avp(lstm_out)\n        \n        gru_out, gru_hidden = self.gru(embeds, gru_hidden)\n        gru_out = gru_out.contiguous().view(-1, self.n_hidden*self.direction, seq_len)\n        gru_out = self.conv1d(gru_out)\n        gru_out = gru_out.contiguous().view(-1, seq_len, (self.n_hidden*self.direction)//2)\n        gru_out = self.avp(gru_out)\n        \n        #out = (lstm_out + gru_out) / 2.0\n        out = torch.cat((lstm_out, gru_out), 2)\n        out = self.dropout(out)\n        out = self.dropout(out)\n        out = self.fc(out.float())\n        sig_out = self.sig(out)\n\n        sig_out = sig_out.view(batch_size, -1)\n        sig_out = sig_out[:, -1] # get only last labels\n        \n        return sig_out, (lstm_hidden, gru_hidden)\n    \n    def init_hidden(self, batch_size, bidirectional=False):\n        weight = next(self.parameters()).data\n        # for LSTM (initial_hidden_state, initial_cell_state)\n        lstm_hidden = (\n            weight.new(self.n_layers*self.direction, batch_size, self.n_hidden).zero_().to(DEVICE),\n            weight.new(self.n_layers*self.direction, batch_size, self.n_hidden).zero_().to(DEVICE)\n        )\n        # for GRU, initial_hidden_state\n        gru_hidden = weight.new(self.n_layers*self.direction, batch_size, self.n_hidden).zero_().to(DEVICE)\n        return lstm_hidden, gru_hidden","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4eff0eac91a06dd92d3f2baaaf626795e4ceedc4"},"cell_type":"markdown","source":"### Instantiate the network\nHyperparameters\n- n_out: output size. in this case, 1 since final label is 0 or 1\n- n_hidden: number of units in the hidden layers, common values are 128, 256 and 512\n- n_layers: number of LSTM layers in the network, typically 1 to 3"},{"metadata":{"trusted":true,"_uuid":"eebecbb332727b5a3571cbe271142cdb89c9b5e8"},"cell_type":"code","source":"# hyperparameters\nn_out = 1\n#n_hidden = 512\nn_hidden = 256\nn_layers = 3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4b51bf474da64a4844a4618ef6652ceeb30a03fc"},"cell_type":"code","source":"# instantiate the network\nnet = SentimentRNN(weight_matrix, n_out, n_hidden, n_layers, bidirectional=False).to(DEVICE)\nnet","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9795d61e5eccf0f77c561b8ed9c2a54cd4c7113e"},"cell_type":"markdown","source":"### Training"},{"metadata":{"trusted":true,"_uuid":"9af04065cc23c8f1fafd73dc1d0380ed98e75589"},"cell_type":"code","source":"# hyperparameters for training\n#  - lr: learning rate\n#  - epochs: number of epochs\nlr = 0.00008\nepochs = 10\nclip = 5 # gradient clipping","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c04b02baeb9b70ac70c9a32bcf8c77f383dac723"},"cell_type":"code","source":"# loss and optimizer functions\ncriterion = nn.BCELoss()\noptimizer = optim.Adam(net.parameters(), lr=lr)\n# for now, scheduler is not used. (has a bigger step_size than epochs)\nscheduler = optim.lr_scheduler.StepLR(optimizer, step_size=30, gamma=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"378200a76d2ef5e724b1487dbe0ade76803382ed"},"cell_type":"code","source":"def train(net, criterion, optimizer, train_loader, clip, epoch, epochs, gru=True):\n    # paramters for printing\n    counter = 0\n    print_every = 500\n\n    train_length = len(train_loader)\n    \n    # initialize hidden state\n    hidden = net.init_hidden(BATCH_SIZE)\n    \n    train_losses = []\n\n    # batch loop\n    for inputs, labels in train_loader:\n        counter += 1\n\n        inputs, labels = inputs.to(DEVICE), labels.to(DEVICE)\n        \n        if gru:\n            l_h, g_h = hidden\n            # for LSTM\n            l_h = tuple([each.data for each in l_h])\n            # for GRU\n            g_h = g_h.data\n            hidden = (l_h, g_h)\n        else:\n            hidden = tuple([each.data for each in hidden])\n        \n        # zero accumulated gradients\n        net.zero_grad()\n        \n        # get the output from the model\n        outputs, hidden = net(inputs, hidden)\n\n        # calcuate the loss and perform backprop\n        loss = criterion(outputs.squeeze(), labels.float())\n        loss.backward()\n\n        # `clip_grad_norm` helps prevent the exploding gradient probelm in RNNs/ LSTMs\n        nn.utils.clip_grad_norm_(net.parameters(), clip)\n        optimizer.step()\n\n        # loss stats\n        if counter % print_every == 0:\n            train_losses.append(loss.item())\n            print(\"Epoch: {}/{}...\".format(epoch+1, epochs),\n                  \"Step: {}...\".format(counter),\n                  \"Train Loss: {:.6f}...\".format(np.mean(train_losses)),\n                  \"Time: {}\".format(datetime.datetime.now()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3183478457887bd9deab03d935e0a6ab0461a145"},"cell_type":"code","source":"# get validation loss\nTHRESHOLD = 0.6\ndef validate(net, criterion, val_loader, epoch, epochs, gru=True):\n    hidden = net.init_hidden(BATCH_SIZE)\n    val_losses = []\n    with torch.no_grad():\n        for inputs, labels in val_loader:\n\n            inputs, labels = inputs.to(DEVICE), labels.to(DEVICE)\n            \n            if gru:\n                val_l_h, val_g_h = hidden\n                # for LSTM\n                val_l_h = tuple([each.data for each in val_l_h])\n                # for GRU\n                val_g_h = val_g_h.data\n                hidden = (val_l_h, val_g_h)\n            else:\n                hidden = tuple([each.data for each in hidden])\n\n            outputs, hidden = net(inputs, hidden)\n            val_loss = criterion(outputs.squeeze(), labels.float())\n            val_losses.append(val_loss.item())\n\n            acc = torch.eq(labels.float(), torch.round(outputs.squeeze())).sum().item()\n\n        print(\"Epoch: {}/{}...\".format(epoch+1, epochs),\n              \"Val Loss: {:.6f}\".format(np.mean(val_losses)),\n              \"Val Acc: {}/{}\".format(acc, BATCH_SIZE),\n              \"Time: {}\".format(datetime.datetime.now()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7fcddeb49754d03d75f32a4d2714e2799403be02"},"cell_type":"code","source":"def run_train(net,\n              criterion, optimizer, scheduler,\n              epochs, train_loader, val_loader,\n              clip, gru=True):\n    for epoch in range(epochs):\n        scheduler.step()\n        train(net, criterion, optimizer, train_loader, clip, epoch, epochs, gru)\n        validate(net, criterion, val_loader, epoch, epochs, gru)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4a07cd1613ed0eb1efa8e3f5aa734ef06a19d4b5"},"cell_type":"code","source":"run_train(net, criterion, optimizer, scheduler, epochs, train_loader, val_loader, clip, gru=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fd4db7eeafa06588f94526d3c1e659376fed9e94"},"cell_type":"markdown","source":"### Test\nTensorDataset is not a convenient class for test data as it doesn't have labels. A test data specific dataset is defined. here. This class does below when `__getitem__` method is called.\n\n1. pre-process\n2. create dataset\n    1. encode\n    2. padding\n3. create tensor dataset with qid"},{"metadata":{"trusted":true,"_uuid":"abfa93902f55d1ca3bf1924e47043eadbdb23ecc"},"cell_type":"code","source":"class QuoraTestDataset(Dataset):\n    def __init__(self, df, word_to_idx, seq_length):\n        self.word_to_idx = word_to_idx\n        self.seq_length = seq_length\n        self.data = df\n        self.data_len = len(df)\n\n    def __len__(self):\n        return self.data_len\n\n    def __getitem__(self, idx):\n        if idx >= self.data_len:\n            idx %= self.data_len\n        # pre-processed\n        tokens = self.data.iloc[idx].processed_text\n        # encode to make array of indices\n        encoded = encode_question(word_to_idx, tokens, self.seq_length) # numpy array of int\n        text_array = add_padding(encoded, self.seq_length)\n        return self.data.iloc[idx].qid, torch.from_numpy(text_array)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3ff5fe089ca3fe309ab67991062ba1cbf0dff8d5"},"cell_type":"code","source":"# create dataset\ntest_set = QuoraTestDataset(test_df, word_to_idx, SEQ_LENGTH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"86bf785214df92a33acb0b30616ad8270dfeea9b"},"cell_type":"code","source":"TEST_BATCH_SIZE = 30","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"56d15ca1e9d44cd2f8c5bfd1a61dacb5ac0727ae"},"cell_type":"code","source":"# create dataloader\ntest_loader = DataLoader(test_set, shuffle=False, batch_size=TEST_BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba5704b8779d1dfc16f990a5a8e57983efb2c0b8"},"cell_type":"code","source":"def test(net, test_loader, batch_size=TEST_BATCH_SIZE):\n    test_l_h, test_g_h = net.init_hidden(batch_size)\n    ret_qid = []\n    ret_pred = []\n    test_len = len(test_loader)\n    counter = 0\n    with torch.no_grad():\n        for qids, inputs in test_loader:\n            counter += 1\n            inputs = inputs.to(DEVICE)\n            \n            # for LSTM\n            test_l_h = tuple([each.data for each in test_l_h])\n            # for GRU\n            test_g_h = test_g_h.data\n\n            outputs, (test_l_h, test_g_h) = net(inputs, (test_l_h, test_g_h))\n            \n            ret_qid.append(qids)\n            ret_pred.append(torch.round(outputs.squeeze()).cpu().numpy().astype(int))\n            \n            if counter % 300 == 0:\n                print('{}/{} done'.format(counter, test_len))\n\n    return ret_qid, ret_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6339a77c2c1c0e6e4220a5e2e90c5126c9f09a90"},"cell_type":"code","source":"ret_qid, ret_pred = test(net, test_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f7446fe0761d0ef1d454daaee70ec2c76fae4df"},"cell_type":"code","source":"ret_qid, ret_pred = np.concatenate(ret_qid), np.concatenate(ret_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f507c1a0ee42d17139918264537a19ef60c38de"},"cell_type":"code","source":"submit_df = pd.DataFrame({\"qid\": ret_qid, \"prediction\": ret_pred})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2ddab18baef11f1a698978ddf5f8d048b740066"},"cell_type":"code","source":"submit_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2e401724eac9dd4e13eb4d9bd9c0b282c853743"},"cell_type":"code","source":"submit_df[-5:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d2d278d2da0e202966d9103afbc663aaf4621f3"},"cell_type":"code","source":"submit_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4db34edc81d0a46a94b5aa97219e4748b2a510a8"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}