{"cells":[{"metadata":{"ExecuteTime":{"end_time":"2019-01-04T17:51:25.862024Z","start_time":"2019-01-04T17:51:24.995204Z"},"_uuid":"b378958a9606ac48fe0dc54e24bed4cd503e0ac7","code_folding":[0],"trusted":true},"cell_type":"code","source":"# _*_ coding: utf-8 _*_\n# import\nimport numpy as np # linear algebra\nimport time\nimport torch\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.nn as nn\nimport numpy as np\nimport os\nfrom torchtext.data import Field, Dataset, Example\nimport pandas as pd\nfrom sklearn.metrics import f1_score\nfrom torchtext import data\nprint(os.listdir(\"../input/\"))\nfrom tqdm import tqdm\ntqdm.pandas()\nimport operator \nimport re \nimport pickle as pkl\nfrom sklearn.model_selection import train_test_split\ndevice = 'cuda'","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-01-04T17:51:28.055600Z","start_time":"2019-01-04T17:51:28.051260Z"},"_uuid":"6e432bb170a329c79f58f78527bbbe4e857b2c41","code_folding":[0],"trusted":false},"cell_type":"code","source":"# const\nMAX_Q_LEN = 128\nBATCH_SIZE = 128\nDOWNSAMPLE_RATE = 5 # ratio of label==0 to label==1","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-01-06T15:08:44.558226Z","start_time":"2019-01-06T15:08:41.476647Z"},"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","code_folding":[0],"trusted":false},"cell_type":"code","source":"# read dataframe\ntrain = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \",train.shape)\nprint(\"Test shape : \",test.shape)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-01-04T17:52:28.410363Z","start_time":"2019-01-04T17:51:31.244036Z"},"_uuid":"807e734e0ce617480c824f8bf26f0672f38397d5","code_folding":[0],"trusted":false},"cell_type":"code","source":"# load embedding\nfrom gensim.models import KeyedVectors\n\nnews_path = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\nembeddings_index = KeyedVectors.load_word2vec_format(news_path, binary=True)\nembeddings_weight = torch.FloatTensor(embeddings_index.vectors)\nembeddings_weight = torch.cat((embeddings_weight, (torch.sum(embeddings_weight, dim=0)/embeddings_weight.shape[0]).unsqueeze(0)), dim=0)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-01-06T15:17:51.323312Z","start_time":"2019-01-06T15:17:51.292813Z"},"_uuid":"f35a7213fc9a7e80a7c210d11b3a8094d3a8e07e","code_folding":[0,1,22,33,40,44,66],"trusted":false},"cell_type":"code","source":"# clean functions\ndef clean_text(x):\n\n    x = str(x)\n    for punct in \"/-'\":\n        x = x.replace(punct, ' ')\n    for punct in '&':\n        x = x.replace(punct, f' {punct} ')\n    for punct in '?!.,\"#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~' + '“”’':\n        x = x.replace(punct, '')\n    return x\n\ndef clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispell_dict = {'colour':'color',\n                'centre':'center',\n                'didnt':'did not',\n                'doesnt':'does not',\n                'isnt':'is not',\n                'shouldnt':'should not',\n                'favourite':'favorite',\n                'travelling':'traveling',\n                'counselling':'counseling',\n                'theatre':'theater',\n                'cancelled':'canceled',\n                'labour':'labor',\n                'organisation':'organization',\n                'wwii':'world war 2',\n                'citicise':'criticize',\n                'instagram': 'social medium',\n                'whatsapp': 'social medium',\n                'snapchat': 'social medium'\n\n                }\nmispellings, mispellings_re = _get_mispell(mispell_dict)\n\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n\n    return mispellings_re.sub(replace, text)\n\ndef remove_stopword(sentences):\n    to_remove = ['a','to','of','and']\n    return [[word for word in sentence if not word in to_remove] for sentence in tqdm(sentences)]","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-01-06T15:18:49.040165Z","start_time":"2019-01-06T15:17:53.640880Z"},"_uuid":"c887a0a6f7498724a2c377e9e29fac683ea3a59b","code_folding":[0],"collapsed":true,"trusted":false},"cell_type":"code","source":"# clean\ntrain[\"question_text\"] = train[\"question_text\"].progress_apply(lambda x: clean_text(x))\ntrain[\"question_text\"] = train[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\ntrain[\"question_text\"] = train[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\ntest[\"question_text\"] = test[\"question_text\"].progress_apply(lambda x: clean_text(x))\ntest[\"question_text\"] = test[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\ntest[\"question_text\"] = test[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\n# sentences = train[\"question_text\"].progress_apply(lambda x: x.split())\nto_remove = ['a','to','of','and']\n# sentences = [[word for word in sentence if not word in to_remove] for sentence in tqdm(sentences)]\n# vocab = build_vocab(sentences)\n\ntrain_txt = remove_stopword(train[\"question_text\"].progress_apply(lambda x: x.split()))\ntest_stc = remove_stopword(test[\"question_text\"].progress_apply(lambda x: x.split()))\nlabel = list(train[\"target\"].progress_apply(lambda x: x))","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-01-06T15:28:41.490049Z","start_time":"2019-01-06T15:28:41.478865Z"},"code_folding":[1,10],"trusted":false,"_uuid":"e28d1e2d1d6977af5588d1093f617947b87a233e"},"cell_type":"code","source":"# Dateset\ndef stc2idx(stc):\n    m = np.zeros((MAX_Q_LEN), dtype=np.int)\n    for i in range(len(stc)):\n        try:\n            m[i] = embeddings_index.vocab[stc[i]].index\n        except:\n            m[i] = embeddings_weight.shape[0]-1\n    return m, len(stc)\n                 \nclass TextDataset(torch.utils.data.Dataset):\n\n    def __init__(self, txt, label=None):\n        self.txt = np.zeros((len(txt),MAX_Q_LEN),dtype=np.int)\n        self.len = np.zeros((len(txt)), dtype=np.int)\n        for i, stc in enumerate(txt):\n            self.txt[i], self.len[i] = stc2idx(stc)\n        if label:\n            self.label = np.array(label, dtype=np.int)\n        else:\n            self.label = None\n                 \n    def __len__(self):\n        return self.txt.shape[0]\n\n    def __getitem__(self, idx):\n        try:\n            return self.txt[idx], self.len[idx], self.label[idx]\n        except:\n            return self.txt[idx], self.len[idx]","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-01-04T17:53:55.034037Z","start_time":"2019-01-04T17:53:48.505280Z"},"code_folding":[0],"trusted":false,"_uuid":"e30d60b3cbb32134b33c58e212e017a832f7270c"},"cell_type":"code","source":"# dataset\nlabel_1 = np.where(np.array(label) == 1)[0]\nlabel_0 = np.where(np.array(label) == 0)[0]\nlabel_0 = label_0[:min(DOWNSAMPLE_RATE*len(label_1), len(label_0))]\ntrain_idx = np.concatenate((label_1, label_0))\n\ntrain_stc = []\ntrain_label = []\nfor idx in train_idx:\n    train_stc.append(train_txt[idx])\n    train_label.append(label[idx])\ntrain_dataset = TextDataset(train_stc, train_label)\ntrain_dataloader = torch.utils.data.DataLoader(train_dataset, shuffle=True, batch_size=BATCH_SIZE, drop_last=True)    \n\ntest_dataset = TextDataset(test_stc)\ntest_dataloader = torch.utils.data.dataloader.DataLoader(test_dataset, batch_size=2, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-01-04T21:27:15.615090Z","start_time":"2019-01-04T21:27:15.600181Z"},"code_folding":[0],"trusted":false,"_uuid":"f7883d9476a0b6fd265f552ca9b9cff77bb9aa02"},"cell_type":"code","source":"class HighLinear(nn.Module):\n    def __init__(self, txt_length, embeddings_length):\n        super(HighLinear, self).__init__()\n        \n        self.txt_length = txt_length\n        self.embeddings_length = embeddings_length\n        self.embed = torch.nn.Embedding.from_pretrained(embeddings_weight, freeze=True)\n        self.W_0 = torch.randn((embeddings_length, txt_length), device=device, requires_grad=True)\n        self.b_0 = torch.randn((txt_length, txt_length), device=device, requires_grad=True)\n        self.act_0 = nn.Tanh()\n        self.avgpool = nn.AvgPool1d(kernel_size=txt_length)\n        self.W_1 = torch.randn((txt_length, 2), device=device, requires_grad=True)\n        self.b_1 = torch.randn((2), device=device, requires_grad=True)\n        self.act_1 = nn.Softmax(dim=1)\n    \n    def forward(self, stc):\n        tmp = self.act_0(self.embed(stc)@self.W_0+self.b_0)\n        tmp = self.avgpool(tmp.squeeze()).squeeze()\n#         print(tmp.shape)\n        tmp = self.act_1(tmp@self.W_1+self.b_1)\n        return tmp\n    \n    def parameters(self):\n        return self.W_0, self.b_0, self.W_1, self.b_1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"07019d3cd59b027f5ee748a9b4213515893b1bb8"},"cell_type":"code","source":"class CNN(nn.Module):\n    def __init__(self):\n        super(CNN, self).__init__()\n        channels = embeddings_weight.shape[-1]\n        kernel_size = 3 \n        dropout_rate = 0\n        num_layers = 1\n\n        self.embed = torch.nn.Embedding.from_pretrained(embeddings_weight, freeze=True)\n        self.CNN = torch.nn.Sequential(nn.Conv1d(in_channels=300, out_channels=300, kernel_size=1),\n                                       nn.ReLU(),\n                                       nn.MaxPool1d(128))\n        self.fc0 = nn.Linear(in_features=300, out_features=2)\n        self.softmax = nn.Softmax()\n        \n    def forward(self, stc):\n        tmp = self.embed(stc).permute(0,2,1)\n        tmp = self.CNN(tmp).squeeze()\n        tmp = self.fc0(tmp)\n        tmp = self.softmax(tmp)\n\n        return tmp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"057c06fdeb6ddf494b50996a5c3ba2f26e93b8d4"},"cell_type":"code","source":"class CNN_1(nn.Module):\n    def __init__(self):\n        super(CNN_1, self).__init__()\n        channels = embeddings_weight.shape[-1]\n        kernel_size = 3 \n        dropout_rate = 0\n        num_layers = 1\n\n        self.embed = torch.nn.Embedding.from_pretrained(embeddings_weight, freeze=True)\n        self.CNN = torch.nn.Sequential(nn.Conv1d(in_channels=300, out_channels=1200, kernel_size=1),\n                                       nn.ReLU(),\n                                       nn.MaxPool1d(128))\n        self.fc = torch.nn.Sequential(nn.Linear(in_features=1200, out_features=300),\n                                      nn.Sigmoid(),\n                                      nn.Linear(in_features=300, out_features=2))\n        self.softmax = nn.Softmax()\n        \n    def forward(self, stc):\n        tmp = self.embed(stc).permute(0,2,1)\n        tmp = self.CNN(tmp).squeeze()\n#         print(tmp.shape)\n        tmp = self.fc(tmp)\n#         tmp = self.fc1(tmp)\n        tmp = self.softmax(tmp)\n\n        return tmp","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-01-04T21:27:20.515401Z","start_time":"2019-01-04T21:27:19.177816Z"},"trusted":false,"_uuid":"9277f1656d0c8d05fae3504454e00f57b2322354"},"cell_type":"code","source":"model = CNN_1().to(device)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-01-04T23:13:33.715972Z","start_time":"2019-01-04T22:42:04.551965Z"},"code_folding":[0,16],"collapsed":true,"trusted":false,"_uuid":"c0d4e728fd1e7fc5c0b3863fe32b7bc7edf15f6b"},"cell_type":"code","source":"# train\nfrom torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\nCE = nn.CrossEntropyLoss()\nfactor = 10000\nlearning_rate = [1e-3]*3+[1e-4]*15+[1e-5]*2#+[1e-6]*factor+[1e-7]*factor+[1e-7]*factor\ntotal_loss = []\nstart_time = time.time()\n# model = nn.DataParallel(model)\n\nfor epoch in range(len(learning_rate)):\n    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate[epoch])\n    for stc, stc_len, target in train_dataloader:\n        start_time = time.time()\n        \n        stc, stc_len = stc.to(device), stc_len.to(device)\n        stc_len, sort_idx = torch.sort(stc_len, descending=True)\n        if stc_len[-1] == 0:\n            continue\n        stc, target = stc[sort_idx], target[sort_idx]\n#         pred = model(stc, stc_len)\n        pred = model(stc)\n        loss = CE(pred, target.to(device))\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n#         total_loss.append(loss.detach().data.item())\n#     print('time:', time.time()-start_time, ' epoch:', epoch, ' loss:',sum(total_loss)/len(total_loss))\n#     if (epoch+1) % 10 == 0:\n#         eval()\n#         model.train()","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-01-06T15:33:38.352516Z","start_time":"2019-01-06T15:33:23.598443Z"},"trusted":false,"_uuid":"81809f810e2179ed0beab771a4b836a2cf656a9e"},"cell_type":"code","source":"# answer\nans = []\nthreshold = 0.5\n\nfor stc, stc_len in test_dataloader:\n    stc = stc.to(device)\n    pred = model(stc)\n\n    ans += torch.argmax(pred, dim=1).detach().cpu().numpy().tolist()","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-01-06T15:34:19.642025Z","start_time":"2019-01-06T15:34:19.518556Z"},"trusted":false,"_uuid":"bd05848d2eb20f4f6a9fedf618872d888ffdb5d8"},"cell_type":"code","source":"# output\nthreshold = 0.5\ntmp_ans = (np.array(ans)>threshold).astype(np.int)\ntest_df = pd.read_csv(\"../input/test.csv\", usecols=[\"qid\"])\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = tmp_ans\nout_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}