{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":52,"outputs":[{"output_type":"stream","text":"['train.csv', 'test.csv', 'sample_submission.csv', 'embeddings']\n","name":"stdout"}]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \",train_df.shape)\nprint(\"Test shape : \",test_df.shape)","execution_count":53,"outputs":[{"output_type":"stream","text":"Train shape :  (1306122, 3)\nTest shape :  (375806, 2)\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\nmax_features = 100000\n\ntokenizer = Tokenizer(num_words=max_features, lower=True)\nfull_text = list(train_df['question_text'].values)\ntokenizer.fit_on_texts(full_text)\n\ntrain_tokenized = tokenizer.texts_to_sequences(train_df['question_text'].fillna('missing'))\ntest_tokenized = tokenizer.texts_to_sequences(test_df['question_text'].fillna('missing'))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":false,"trusted":true},"cell_type":"code","source":"max_len = 72\ntrain_X = pad_sequences(train_tokenized, maxlen=max_len)\ntest_X = pad_sequences(test_tokenized, maxlen=max_len)\ntrain_y = train_df.target.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from gensim.models import KeyedVectors\nfrom gensim.test.utils import datapath, get_tmpfile\nfrom gensim.scripts.glove2word2vec import glove2word2vec\n\nnews_path = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\nembeddings = KeyedVectors.load_word2vec_format(news_path, binary=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mean = np.mean(embeddings.vectors)\nstd = np.std(embeddings.vectors)\n","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":false,"trusted":true},"cell_type":"code","source":"_, size = embeddings.vectors.shape\nnum_words = min(len(tokenizer.word_index), max_features)\nembedding_matrix = np.random.normal(mean, std, (num_words, size))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":false},"cell_type":"code","source":"for word, index in tokenizer.word_index.items():\n    if index >= max_features: continue\n    try:\n        word_vector = embeddings.get_vector(word)\n        embedding_matrix[index] = word_vector\n    except:\n        pass\nprint(embedding_matrix.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport random\nfrom tqdm import  tqdm\ntqdm.pandas()\n\ndef seed_torch(seed=0):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nseed = 0\nseed_torch(seed)\n\ndef sigmoid(x):\n    return 1 / (1 + np.exp(-x))\n\nclass CNN(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.embedding = nn.Embedding(max_features, size)\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        self.embedding.weight.requires_grad = False\n        \n        self.conv_0 = nn.Conv2d(in_channels=1, out_channels=100, kernel_size=(3, size))\n        self.conv_1 = nn.Conv2d(in_channels=1, out_channels=100, kernel_size=(4, size))\n        self.conv_2 = nn.Conv2d(in_channels=1, out_channels=100, kernel_size=(5, size))\n        \n        self.lstm = nn.LSTM(100, 50)\n        self.fc = nn.Linear(50, 1)\n        self.dropout = nn.Dropout(0.1)\n        \n    def forward(self, text):\n        embedded = self.embedding(text)\n        embedded = embedded.unsqueeze(1)\n        conved_0 = F.relu(self.conv_0(embedded).squeeze(3))\n        conved_1 = F.relu(self.conv_1(embedded).squeeze(3))\n        conved_2 = F.relu(self.conv_2(embedded).squeeze(3))\n        # print('conved_0', conved_0.shape)\n        # print('conved_1', conved_1.shape)\n        # print('conved_2', conved_2.shape)\n        pooled_0 = F.adaptive_avg_pool1d(conved_0, 50).squeeze(2)\n        pooled_1 = F.adaptive_avg_pool1d(conved_1, 50).squeeze(2)\n        pooled_2 = F.adaptive_avg_pool1d(conved_2, 50).squeeze(2)\n        # print('pooled_0', pooled_0.shape)\n        # print('pooled_1', pooled_1.shape)\n        # print('pooled_2', pooled_2.shape)\n        cat = torch.cat((pooled_0, pooled_1, pooled_2), dim=2)\n        # print('cat',cat.shape)\n        cat = self.dropout(cat)\n        cat = cat.transpose(1,2).transpose(0,1)\n        # print('cat', cat.shape)\n        output, (hn, cn) = self.lstm(cat)\n        \n        return self.fc(hn.squeeze(0))\n        \n        \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nfrom torch.utils.data import DataLoader, TensorDataset\nimport time\nimport re\nimport random\nfrom sklearn.metrics import f1_score\n\nsplits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=0).split(train_X, train_y))\ntrain_epochs = 5\nbatch_size = 64\n\nprint(type(train_X), train_X.shape)\nprint(type(train_y), train_y.shape)\ntrain_X = torch.Tensor(train_X)\ntrain_y = torch.Tensor(train_y)\n\ntrain_preds = np.zeros((len(train_X)))\ntest_preds = np.zeros((len(test_X)))\n\nx_test_cuda = torch.tensor(test_X, dtype=torch.long).cuda()\ntest = torch.utils.data.TensorDataset(x_test_cuda)\ntest_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)\n\nfor i, (train_idx, valid_idx) in enumerate(splits):\n    print('Fold:{0}'.format(i))\n    x_train_fold = torch.tensor(train_X[train_idx], dtype=torch.long).cuda()\n    y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32).cuda()\n    x_val_fold = torch.tensor(train_X[valid_idx], dtype=torch.long).cuda()\n    y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32).cuda()\n    \n    train_data = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)\n    valid_data = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)\n    \n    train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, shuffle=True)\n    valid_loader = torch.utils.data.DataLoader(valid_data, batch_size=batch_size, shuffle=False)\n    \n    model = CNN()\n    model.cuda()\n    loss_function = nn.BCEWithLogitsLoss()\n    optimiser = optim.Adam(model.parameters())\n    \n    for epoch in range(train_epochs):\n        start_time = time.time()\n        model.train()\n        avg_loss = 0.0\n        # train  model\n        for x_batch, y_batch in tqdm(train_loader, disable=True):\n            y_predicted = model(x_batch)\n            loss = loss_function(y_predicted, y_batch)\n            optimiser.zero_grad()\n            loss.backward()\n            optimiser.step()\n            avg_loss += loss.item()/len(train_loader)\n\n        # evaluate model\n        model.eval()\n        val_predicted_fold = np.zeros(x_val_fold.size(0))\n        test_predicted_fold = np.zeros(len(test_X))\n        avg_val_loss =0.0\n\n        for i, (x_batch, y_batch) in enumerate(valid_loader):\n            y_predicted = model(x_batch).detach()\n            avg_val_loss += loss_function(y_predicted, y_batch).item()/len(valid_loader)\n            val_predicted_fold[i*batch_size:(i+1)*batch_size] = sigmoid(y_predicted.cpu().numpy())[:,0]   \n\n        elapsed_time = time.time() - start_time\n        print('Epoch {}/{} \\t loss={:.4f} \\t val_loss={:.4f} \\t time={:.2f}s'.format(epoch + 1,\n                                                        train_epochs, avg_loss, avg_val_loss, elapsed_time))\n    # predict testing dataset\n    test_predicted_fold = np.zeros(len(test_X))\n    for i, (x_batch,) in enumerate(test_loader):\n        y_pred = model(x_batch).detach()\n        test_predicted_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n    \n    train_preds[valid_idx] = val_predicted_fold\n    test_preds += test_predicted_fold / len(splits) # take average","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in tqdm([i * 0.01 for i in range(100)]):\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"search_result = threshold_search(train_y, train_preds)\nsearch_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('../input/sample_submission.csv')\nsub.prediction = test_preds > search_result['threshold']\nsub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}