{"cells":[{"metadata":{"_uuid":"51d190bcb6d9fb15dde6ed6f8ebb360e6768b580"},"cell_type":"markdown","source":"**SUMMARY**\n\n*LSTM:*\nF1 score: 0.6622\nTime elapsed: 1273.20\nAccuracy: 0.9562\n\n*FFM:*\nF1 score: 0.6049\nTime elapsed: 184.46\nAccuracy: 0.9473"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"                                                                                                                                                                                                                                                                                                # This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport math\nimport random\nimport time\nimport os\nprint(os.listdir(\"../input\"))\n\ndf = pd.read_csv(\"../input/train.csv\")\n# Any results you write to the current directory are saved as output.\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import *\nfrom keras.models import *\nfrom keras import initializers, regularizers, constraints, optimizers, layers\n\nimport torch\nimport torchtext\nfrom torch import nn\nimport torch.nn.functional as F\nfrom nltk import word_tokenize\nfrom torch import optim","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6473e5fc7308148002108a9de97e0061ea5b847b"},"cell_type":"code","source":"\nmax_len = 50\n\ntext = torchtext.data.Field(lower=True, batch_first=True, tokenize=word_tokenize, fix_length = max_len)\ntarget = torchtext.data.Field(sequential=False, use_vocab=False, is_target=True)\ntrain = torchtext.data.TabularDataset(path='../input/train.csv', format='csv',\n                                      fields={'question_text': ('text',text),\n                                              'target': ('target',target)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3f937efbd77d3f5c1832cba13932c18a8e0b938"},"cell_type":"code","source":"text.build_vocab(train, min_freq=1)\ntext.vocab.load_vectors(torchtext.vocab.Vectors(\"../input/embeddings/glove.840B.300d/glove.840B.300d.txt\"))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d469f339156fd3a4f32518f39c8724eac0b252ab"},"cell_type":"code","source":"random_state = random.getstate()\ntrain, val = train.split(split_ratio=0.8, random_state=random_state)\nbatch_size = 512\ntrain_iter = torchtext.data.BucketIterator(dataset=train,\n                                           batch_size=batch_size,\n                                           sort_key=lambda x: x.text.__len__(),\n                                           shuffle=True,\n                                           sort=False)\n\nval_iter = torchtext.data.BucketIterator(dataset=val,\n                                         batch_size=batch_size,\n                                         sort_key=lambda x: x.text.__len__(),\n                                         train=False,\n                                         sort=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eec22d0dfdc1ccc0d4de38b48e48b2d7009d5b54"},"cell_type":"code","source":"def training(epoch, model, loss_func, optimizer, train_iter, val_iter):\n    step = 0\n    train_record = []\n    val_record = []\n    losses = []\n    \n    for e in range(epoch):\n        train_iter.init_epoch()\n        for train_batch in iter(train_iter):\n            step += 1\n            model.train()\n            x = train_batch.text.cuda()\n            y = train_batch.target.type(torch.Tensor).cuda()\n            model.zero_grad()\n            pred = model.forward(x).view(-1)\n            #print('Pred:{}'.format(pred.shape))\n            #print('y:{}'.format(y.shape))\n            \n            loss = loss_function(pred, y)\n            loss_data = loss.cpu().data.numpy()\n            train_record.append(loss_data)\n            loss.backward()\n            optimizer.step()\n            if step % 1000 == 0:\n                print(\"Step: {:06}, loss {:.4f}\".format(step, loss_data))\n        model.eval()\n        model.zero_grad()\n        val_loss = []\n        for val_batch in iter(val_iter):\n            val_x = val_batch.text.cuda()\n            val_y = val_batch.target.type(torch.Tensor).cuda()\n            val_pred = model.forward(val_x).view(-1)\n            val_loss.append(loss_function(val_pred, val_y).cpu().data.numpy())\n        val_record.append({'step': step, 'loss': np.mean(val_loss)})\n        print('Epoch {:02} - step {:06} - train_loss {:.4f} - val_loss {:.4f} '.format(\n                    e, step, np.mean(train_record), val_record[-1]['loss']))\n        train_record = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0cd92322fc811b62968187aaceebebaa21854c95"},"cell_type":"code","source":"def results(m, t):\n    model = m\n    model.eval()\n    val_pred = []\n    val_true = []\n    val_iter.init_epoch()\n    for val_batch in iter(val_iter):\n        val_x = val_batch.text.cuda()\n        val_true += val_batch.target.data.numpy().tolist()\n        val_pred += torch.sigmoid(model.forward(val_x).view(-1)).cpu().data.numpy().tolist()\n\n    tmp = [0,0,0] # idx, cur, max\n    delta = 0\n    for tmp[0] in np.arange(0.1, 0.501, 0.01):\n        tmp[1] = metrics.f1_score(val_true, np.array(val_pred)>tmp[0])\n        if tmp[1] > tmp[2]:\n            delta = tmp[0]\n            tmp[2] = tmp[1]\n\n    tp = 0\n    fp = 0\n    tn = 0\n    fn = 0\n    total = len(val_pred)\n    for i in range(0,len(val_pred)):\n        pred = val_pred[i] > delta\n        if val_true[i] == 1:\n            if pred == 1:\n                tp += 1\n            else:\n                fp += 1\n        else:\n            if pred == 1:\n                fn += 1\n            else:\n                tn += 1\n\n    print('----TIME FOR SOME STATISCTICS!!!!----')\n    print('-------------{} MODEL--------------'.format(model.name))\n    print('best threshold is {:.4f} with F1 score: {:.4f}'.format(delta, tmp[2]))\n    print('Time elapsed: {:.2f}'.format(time.time() - t))\n    print('True Positive: {}'.format(tp))\n    print('False Positive: {}'.format(fp))\n    print('False Negative: {}'.format(fn))\n    print('True Negative: {}'.format(tn))\n    print('Accuracy: {:.4f}'.format((tp+tn)/float(total)))\n    print('Precision: {:.4f}'.format(tp/(float(tp+fp))))\n    print('False positive rate: {:.4f}'.format(fp/(float(tn+fp))))\n    print('Recall: {:.4f}'.format(tp/(float(tp+fn))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d5f0db8db8148126e3cf5d76a87ce93d465a4a1c"},"cell_type":"code","source":"class LSTM(nn.Module):\n    def __init__(self, pretrained_lm, padding_idx, hidden_dim = 128, static=True):\n        super(LSTM, self).__init__()\n        self.name = 'LSTM'\n        self.hidden_dim = hidden_dim\n        self.dropout = nn.Dropout(p=0.5)\n        self.embedding = nn.Embedding.from_pretrained(pretrained_lm)\n        self.embedding.padding_idx = padding_idx\n        if static:\n            self.embedding.weight.requires_grad = False\n        self.lstm = nn.LSTM(input_size=self.embedding.embedding_dim,\n                            hidden_size=hidden_dim,\n                            num_layers=2, \n                            dropout = 0.5)\n        self.hidden2label = nn.Linear(hidden_dim*2, 1)\n    \n    def forward(self, sents):\n        x = self.embedding(sents)\n        x = torch.transpose(x, dim0=1, dim1=0)\n        lstm_out, (h_n, c_n) = self.lstm(x)\n        y = self.hidden2label(self.dropout(torch.cat([c_n[i,:, :] for i in range(c_n.shape[0])], dim=1)))\n        return y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2390f7b5578016fba686ac81445ae7a14b230a3"},"cell_type":"code","source":"lstm = LSTM(text.vocab.vectors,\n                    padding_idx=text.vocab.stoi[text.pad_token], hidden_dim=128).cuda()\nloss_function = nn.BCEWithLogitsLoss()\noptimizer = optim.Adam(filter(lambda p: p.requires_grad, lstm.parameters()),lr=0.0001)\n\nt = time.time()\n\ntraining(model=lstm,\n         epoch=10,\n         loss_func=loss_function,\n         optimizer=optimizer,\n         train_iter=train_iter,\n         val_iter=val_iter)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b70ddfef4c47c38f0f7f55876818f6ac8b3f6483"},"cell_type":"code","source":"results(lstm,t)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c4012df384a7caa4ae3fa9a0cc1dcdf74f10001"},"cell_type":"code","source":"class CNN(nn.Module):\n    def __init__(self, pretrained_lm, padding_idx, static=True):\n        super(CNN, self).__init__()\n        self.name = 'CNN'\n        self.embedding = nn.Embedding.from_pretrained(pretrained_lm)\n        self.embedding.padding_idx = padding_idx\n        filter_sizes = [1,2,3,5]\n        if static:\n            self.embedding.weight.requires_grad = False\n        self.conv1 = nn.Sequential(\n            nn.Conv1d(in_channels = 300, out_channels = 1, kernel_size = filter_sizes[3]),\n            nn.MaxPool1d(kernel_size = 2)\n        )\n        self.lin = nn.Linear(23,64)\n        self.fc = nn.Linear(64,1)\n        \n        \n    def forward(self, sents):\n        x = self.embedding(sents)\n        x = torch.transpose(x, dim0=2, dim1=1)\n        c1 = self.conv1(x)\n        '''c2 = self.conv2(x)\n        c3 = self.conv3(x)\n        c4 = self.conv4(x)\n        x = torch.cat((c1,c2,c3,c4), dim=1)'''\n        x = c1\n        x = nn.Dropout()(x)\n        x = x.reshape(x.size(0), -1)\n        x = self.lin(x)\n        x = nn.Dropout()(x)\n        x = self.fc(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e2c3aed1748b19241525afdffe8c6d3d53b7e961","scrolled":true},"cell_type":"code","source":"cnn = CNN(text.vocab.vectors,\n                    padding_idx=text.vocab.stoi[text.pad_token]).cuda()\nloss_function = nn.BCEWithLogitsLoss()\noptimizer = optim.Adam(filter(lambda p: p.requires_grad, cnn.parameters()),lr=0.0001)\n\nt = time.time()\n\ntraining(model=cnn,\n         epoch=10,\n         loss_func=loss_function,\n         optimizer=optimizer,\n         train_iter=train_iter,\n         val_iter=val_iter)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e99606130856958268eb122495210b655139ff7a"},"cell_type":"code","source":"results(cnn, t)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c9d1b4661f3699cd4fef89d7e3769e17196c2e47"},"cell_type":"code","source":"class FFM(nn.Module):\n    def __init__(self, pretrained_lm, padding_idx, static=True):\n        super(FFM, self).__init__()\n        self.name = 'FFM'\n        self.embedding = nn.Embedding.from_pretrained(pretrained_lm)\n        self.embedding.padding_idx = padding_idx\n        if static:\n            self.embedding.weight.requires_grad = False\n        self.layer1 = nn.Sequential(\n            nn.Linear(15000,1000),\n            nn.ReLU()\n        )\n        self.layer2 = nn.Sequential(\n            nn.Linear(1000,600),\n            nn.ReLU()\n        )\n        self.layer3 = nn.Sequential(\n            nn.Linear(600,200),\n            nn.ReLU()\n        )\n        self.layer4 = nn.Sequential(\n            nn.Linear(200,90),\n            nn.ReLU()\n        )\n        self.layer5 = nn.Sequential(\n            nn.Linear(90,32),\n            nn.ReLU()\n        )\n        self.fc = nn.Linear(32,1)\n        \n        \n    def forward(self, sents):\n        x = self.embedding(sents)\n        x = torch.cat([x[:, i, :] for i in range(x.shape[1])], dim=1)\n        x = self.layer1(x)\n        x = nn.Dropout()(x)\n        x = self.layer2(x)\n        x = nn.Dropout()(x)\n        x = self.layer3(x)\n        x = nn.Dropout()(x)\n        x = self.layer4(x)\n        x = nn.Dropout()(x)\n        x = self.layer5(x)\n        x = nn.Dropout()(x)\n        x = self.fc(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"401bdf119acad0462ebddd3f23e5fdc5735f69be","scrolled":true},"cell_type":"code","source":"ffm = FFM(text.vocab.vectors,\n                    padding_idx=text.vocab.stoi[text.pad_token]).cuda()\nloss_function = nn.BCEWithLogitsLoss()\noptimizer = optim.Adam(filter(lambda p: p.requires_grad, ffm.parameters()),lr=0.0001)\n\nt = time.time()\n\ntraining(model=ffm,\n         epoch=2,\n         loss_func=loss_function,\n         optimizer=optimizer,\n         train_iter=train_iter,\n         val_iter=val_iter)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc5262368c365562b93685bfb407e3938a5f8012"},"cell_type":"code","source":"results(ffm, t)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b2734cfd90d8266fd630f96c1aad02cd9800098"},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \",train_df.shape)\nprint(\"Test shape : \",test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ac262eb8152a5feb77da97a09437573636021799"},"cell_type":"code","source":"train_df['len'] = train_df['question_text'].apply(lambda x: len(x.split(' ')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc37e182e626cd134a091f8ea370245f4587d14d"},"cell_type":"code","source":"train_df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fea36730fa7b67f680a005a3b8cb4fe01fdead67"},"cell_type":"code","source":"def SimpleFFModel:\n    model=Sequential() # Instantiate the Sequential class\n    model.add(Embedding(max_features, 300, input_length=maxlen,  weights=[embedding_matrix], trainable=False)) # Creat embedding layer as described above\n    model.add(layers.Flatten()) #Flatten the embedding layer as input to a Dense layer\n    model.add(layers.Dense(1000, activation='relu')) # Dense layer with relu activation\n    model.add(layers.Dropout(0.3))\n    model.add(layers.Dense(600, activation='relu')) # Dense layer with relu activation\n    model.add(layers.Dropout(0.3))\n    model.add(layers.Dense(200, activation='relu')) # Dense layer with relu activation\n    model.add(layers.Dropout(0.3))\n    model.add(layers.Dense(90, activation='relu')) # Dense layer with relu activation\n    model.add(layers.Dropout(0.3))\n    model.add(layers.Dense(32, activation='relu')) # Dense layer with relu activation\n    model.add(layers.Dropout(0.3))\n    model.add(layers.Dense(1,activation='sigmoid')) # Dense layer with sigmoid activation for binary target\n    model.compile(optimizer='rmsprop',loss='binary_crossentropy',metrics=['accuracy']) #binary cross entropy is used as the loss function and accuracy as the metric \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d0061082e52e7a76adb031f1a2590ea2363364c"},"cell_type":"code","source":"def ConvModel:\n    inp = Input(shape=(maxlen, ))\n    embed = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp)\n\n    #filter_sizes = [1,2,3,5]\n    filter_sizes = [5]\n    num_filters = 64\n\n    conv_0 = Conv1D(num_filters, filter_sizes[0], padding='valid', kernel_initializer='normal', activation='relu')(embed)\n    #conv_1 = Conv1D(num_filters, filter_sizes[1], padding='valid', kernel_initializer='normal', activation='relu')(embed)\n    #conv_2 = Conv1D(num_filters, filter_sizes[2], padding='valid', kernel_initializer='normal', activation='relu')(embed)\n    #conv_3 = Conv1D(num_filters, filter_sizes[3], padding='valid', kernel_initializer='normal', activation='relu')(embed)\n\n    maxpool_0 = MaxPool1D(pool_size=(maxlen - filter_sizes[0] + 1), strides=(1), padding='valid')(conv_0)\n    #maxpool_1 = MaxPool1D(pool_size=(maxlen - filter_sizes[1] + 1), strides=(1), padding='valid')(conv_1)\n    #maxpool_2 = MaxPool1D(pool_size=(maxlen - filter_sizes[2] + 1), strides=(1), padding='valid')(conv_2)\n    #maxpool_3 = MaxPool1D(pool_size=(maxlen - filter_sizes[3] + 1), strides=(1), padding='valid')(conv_3)\n\n    #concatenated_tensor = Concatenate(axis=1)([maxpool_0, maxpool_1, maxpool_2, maxpool_3])\n\n    #x = Flatten()(concatenated_tensor)\n    x = Flatten()(maxpool_0)\n    x = Dense(128, activation='relu')(x)\n    x = Dropout(0.3)(x)\n    outp = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy',\n                  optimizer='adam',\n                  metrics=['accuracy'])    \n    return model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e263aa29c9e9d7664db9df170a25f9a335f88b49"},"cell_type":"code","source":"from scipy import spatial\nfrom sklearn.metrics.pairwise import cosine_similarity\n\nprint(1 - spatial.distance.cosine(embs_index[\"White\"], embs_index[\"Black\"]))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}