{"cells":[{"metadata":{"ExecuteTime":{"end_time":"2019-01-04T17:51:25.862024Z","start_time":"2019-01-04T17:51:24.995204Z"},"_uuid":"b378958a9606ac48fe0dc54e24bed4cd503e0ac7","code_folding":[0],"trusted":true},"cell_type":"code","source":"# _*_ coding: utf-8 _*_\n# import\nimport numpy as np # linear algebra\nimport time\nimport torch\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.nn as nn\nimport numpy as np\nimport os\nfrom torchtext.data import Field, Dataset, Example\nimport pandas as pd\nfrom sklearn.metrics import f1_score\nfrom torchtext import data\nprint(os.listdir(\"../input/\"))\nfrom tqdm import tqdm\ntqdm.pandas()\nimport operator \nimport re \nimport pickle as pkl\nfrom sklearn.model_selection import train_test_split\ndevice = 'cuda'","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-01-04T17:51:28.055600Z","start_time":"2019-01-04T17:51:28.051260Z"},"_uuid":"6e432bb170a329c79f58f78527bbbe4e857b2c41","code_folding":[0],"trusted":true},"cell_type":"code","source":"# const\nMAX_Q_LEN = 128\nBATCH_SIZE = 128\nDOWNSAMPLE_RATE = 5 # ratio of label==0 to label==1","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-01-06T15:08:44.558226Z","start_time":"2019-01-06T15:08:41.476647Z"},"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","code_folding":[0],"trusted":true},"cell_type":"code","source":"# read dataframe\ntrain = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \",train.shape)\nprint(\"Test shape : \",test.shape)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-01-04T17:52:28.410363Z","start_time":"2019-01-04T17:51:31.244036Z"},"_uuid":"807e734e0ce617480c824f8bf26f0672f38397d5","code_folding":[0],"trusted":true},"cell_type":"code","source":"# load embedding\nfrom gensim.models import KeyedVectors\n\nnews_path = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\nembeddings_index = KeyedVectors.load_word2vec_format(news_path, binary=True)\nembeddings_weight = torch.FloatTensor(embeddings_index.vectors)\nembeddings_weight = torch.cat((embeddings_weight, (torch.sum(embeddings_weight, dim=0)/embeddings_weight.shape[0]).unsqueeze(0)), dim=0)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-01-06T15:17:51.323312Z","start_time":"2019-01-06T15:17:51.292813Z"},"_uuid":"f35a7213fc9a7e80a7c210d11b3a8094d3a8e07e","code_folding":[0,1,22,33,40,44,66],"trusted":true},"cell_type":"code","source":"# clean functions\ndef clean_text(x):\n\n    x = str(x)\n    for punct in \"/-'\":\n        x = x.replace(punct, ' ')\n    for punct in '&':\n        x = x.replace(punct, f' {punct} ')\n    for punct in '?!.,\"#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~' + '“”’':\n        x = x.replace(punct, '')\n    return x\n\ndef clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispell_dict = {'colour':'color',\n                'centre':'center',\n                'didnt':'did not',\n                'doesnt':'does not',\n                'isnt':'is not',\n                'shouldnt':'should not',\n                'favourite':'favorite',\n                'travelling':'traveling',\n                'counselling':'counseling',\n                'theatre':'theater',\n                'cancelled':'canceled',\n                'labour':'labor',\n                'organisation':'organization',\n                'wwii':'world war 2',\n                'citicise':'criticize',\n                'instagram': 'social medium',\n                'whatsapp': 'social medium',\n                'snapchat': 'social medium'\n\n                }\nmispellings, mispellings_re = _get_mispell(mispell_dict)\n\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n\n    return mispellings_re.sub(replace, text)\n\ndef remove_stopword(sentences):\n    to_remove = ['a','to','of','and']\n    return [[word for word in sentence if not word in to_remove] for sentence in tqdm(sentences)]","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-01-06T15:18:49.040165Z","start_time":"2019-01-06T15:17:53.640880Z"},"_uuid":"c887a0a6f7498724a2c377e9e29fac683ea3a59b","code_folding":[0],"trusted":true},"cell_type":"code","source":"# clean\ntrain[\"question_text\"] = train[\"question_text\"].progress_apply(lambda x: clean_text(x))\ntrain[\"question_text\"] = train[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\ntrain[\"question_text\"] = train[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\ntest[\"question_text\"] = test[\"question_text\"].progress_apply(lambda x: clean_text(x))\ntest[\"question_text\"] = test[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\ntest[\"question_text\"] = test[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\n# sentences = train[\"question_text\"].progress_apply(lambda x: x.split())\nto_remove = ['a','to','of','and']\n# sentences = [[word for word in sentence if not word in to_remove] for sentence in tqdm(sentences)]\n# vocab = build_vocab(sentences)\n\ntrain_txt = remove_stopword(train[\"question_text\"].progress_apply(lambda x: x.split()))\ntest_stc = remove_stopword(test[\"question_text\"].progress_apply(lambda x: x.split()))\nlabel = list(train[\"target\"].progress_apply(lambda x: x))","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-01-06T15:28:41.490049Z","start_time":"2019-01-06T15:28:41.478865Z"},"code_folding":[1,10],"trusted":true,"_uuid":"e28d1e2d1d6977af5588d1093f617947b87a233e"},"cell_type":"code","source":"# Dateset\ndef stc2idx(stc):\n    m = np.zeros((MAX_Q_LEN), dtype=np.int)\n    for i in range(len(stc)):\n        try:\n            m[i] = embeddings_index.vocab[stc[i]].index\n        except:\n            m[i] = embeddings_weight.shape[0]-1\n    return m, len(stc)\n                 \nclass TextDataset(torch.utils.data.Dataset):\n\n    def __init__(self, txt, label=None):\n        self.txt = np.zeros((len(txt),MAX_Q_LEN),dtype=np.int)\n        self.len = np.zeros((len(txt)), dtype=np.int)\n        for i, stc in enumerate(txt):\n            self.txt[i], self.len[i] = stc2idx(stc)\n        if label:\n            self.label = np.array(label, dtype=np.int)\n        else:\n            self.label = None\n                 \n    def __len__(self):\n        return self.txt.shape[0]\n\n    def __getitem__(self, idx):\n        try:\n            return self.txt[idx], self.len[idx], self.label[idx]\n        except:\n            return self.txt[idx], self.len[idx]","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-01-04T17:53:55.034037Z","start_time":"2019-01-04T17:53:48.505280Z"},"code_folding":[0],"trusted":true,"_uuid":"e30d60b3cbb32134b33c58e212e017a832f7270c"},"cell_type":"code","source":"# dataset\nlabel_1 = np.where(np.array(label) == 1)[0]\nlabel_0 = np.where(np.array(label) == 0)[0]\nlabel_0 = label_0[:min(DOWNSAMPLE_RATE*len(label_1), len(label_0))]\ntrain_idx = np.concatenate((label_1, label_0))\n\ntrain_stc = []\ntrain_label = []\nfor idx in train_idx:\n    train_stc.append(train_txt[idx])\n    train_label.append(label[idx])\ntrain_dataset = TextDataset(train_stc, train_label)\ntrain_dataloader = torch.utils.data.DataLoader(train_dataset, shuffle=True, batch_size=BATCH_SIZE, drop_last=True)    \n\ntest_dataset = TextDataset(test_stc)\ntest_dataloader = torch.utils.data.dataloader.DataLoader(test_dataset, batch_size=2, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"07019d3cd59b027f5ee748a9b4213515893b1bb8"},"cell_type":"code","source":"class RCNN_CNN(nn.Module):\n    def __init__(self, batch_size, hidden_size, embedding_length):\n        super(RCNN_CNN, self).__init__()\n        self.embedding_length = embedding_length\n        \n        self.batch_size = batch_size\n        self.hidden_size = hidden_size\n        self.embed = torch.nn.Embedding.from_pretrained(embeddings_weight, freeze=True)\n        self.dropout = 0.2\n        self.h_0 = torch.zeros(4, self.batch_size, self.hidden_size).to(device)\n        self.c_0 = torch.zeros(4, self.batch_size, self.hidden_size).to(device)\n        self.h_0_t = torch.zeros(4, 2, self.hidden_size).to(device)\n        self.c_0_t = torch.zeros(4, 2, self.hidden_size).to(device)\n        self.lstm = nn.LSTM(embedding_length, hidden_size, dropout=self.dropout, bidirectional=True, batch_first=True, num_layers=2)\n        self.cnn = nn.Conv1d(in_channels=2 * hidden_size + embedding_length, out_channels=600, kernel_size=1)\n        self.ReLU = nn.ReLU()\n        self.label = nn.Linear(600, 2)\n\n    def forward(self, stc, stc_len, test=False):        \n        stc = self.embed(stc)\n        stc = pack_padded_sequence(stc, stc_len, batch_first=True)\n        if test:\n            output, (_, _) = self.lstm(stc, (self.h_0_t, self.c_0_t))\n        else:\n            output, (_, _) = self.lstm(stc, (self.h_0, self.c_0))\n        output = pad_packed_sequence(output, batch_first=True)[0]\n        stc = pad_packed_sequence(stc, batch_first=True)[0]\n#         print(stc.shape, output.shape)\n        stc = torch.cat((output, stc), 2).permute(0, 2, 1)\n#         print(stc.shape)\n        final_encoding = self.ReLU(self.cnn(stc))\n        y = F.max_pool1d(final_encoding, kernel_size=final_encoding.shape[2]).squeeze()\n#         print(y.shape)\n        logits = self.label(y)\n        \n        return logits","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-01-04T21:27:20.515401Z","start_time":"2019-01-04T21:27:19.177816Z"},"trusted":true,"_uuid":"9277f1656d0c8d05fae3504454e00f57b2322354"},"cell_type":"code","source":"model = RCNN_CNN(batch_size=BATCH_SIZE, hidden_size=256, embedding_length=embeddings_weight.shape[1]).to(device)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-01-04T23:13:33.715972Z","start_time":"2019-01-04T22:42:04.551965Z"},"code_folding":[0,16],"trusted":true,"_uuid":"c0d4e728fd1e7fc5c0b3863fe32b7bc7edf15f6b"},"cell_type":"code","source":"# train\nfrom torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\nCE = nn.CrossEntropyLoss()\nfactor = 10000\nlearning_rate = [1e-3]*2+[1e-4]*15+[1e-5]*7#+[1e-6]*factor+[1e-7]*factor+[1e-7]*factor\ntotal_loss = []\nstart_time = time.time()\n# model = nn.DataParallel(model)\n\nfor epoch in range(len(learning_rate)):\n    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate[epoch])\n    for stc, stc_len, target in train_dataloader:\n        \n        stc, stc_len = stc.to(device), stc_len.to(device)\n        stc_len, sort_idx = torch.sort(stc_len, descending=True)\n        if stc_len[-1] == 0:\n            continue\n        stc, target = stc[sort_idx], target[sort_idx]\n        pred = model(stc, stc_len)\n#         pred = model(stc)\n        loss = CE(pred, target.to(device))\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n    print('time:', time.time()-start_time)\n    start_time = time.time()","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-01-06T15:33:38.352516Z","start_time":"2019-01-06T15:33:23.598443Z"},"trusted":true,"_uuid":"81809f810e2179ed0beab771a4b836a2cf656a9e"},"cell_type":"code","source":"# answer\nans = []\nthreshold = 0.5\n\nfor stc, stc_len in test_dataloader:\n    stc, stc_len = stc.to(device), stc_len.to(device)\n    stc_len, sort_idx = torch.sort(stc_len, descending=True)\n    stc, target = stc[sort_idx], target[sort_idx]\n    pred = model(stc, stc_len, True)\n    ans += torch.argmax(pred, dim=1).detach().cpu().numpy().tolist()","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-01-06T15:34:19.642025Z","start_time":"2019-01-06T15:34:19.518556Z"},"trusted":true,"_uuid":"bd05848d2eb20f4f6a9fedf618872d888ffdb5d8"},"cell_type":"code","source":"# output\nthreshold = 0.5\ntmp_ans = (np.array(ans)>threshold).astype(np.int)\ntest_df = pd.read_csv(\"../input/test.csv\", usecols=[\"qid\"])\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = tmp_ans\nout_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}