{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader, TensorDataset\n\nfrom keras_preprocessing.text import Tokenizer\nfrom keras_preprocessing.sequence import pad_sequences\n\nimport math\nimport datetime\nimport string, re","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af5818175d21927f1618d6ef79f7110989630251"},"cell_type":"code","source":"#show all columns\n#pd.set_option('display.max_columns', None)\n#show all rows\n#pd.set_option('display.max_rows', None)\n#show textarea length \n#pd.set_option('max_colwidth',100)\n#some config values\nembedding_size = 300\nhidden_size = 256\nwords_size = 40000\nlr = 0.0001\nbatch_size = 128   \nepoch = 5\ndropout = 0.1","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"#define some help function\ndef cuda_available(tensor):\n    if torch.cuda.is_available:\n        return tensor.cuda()\n    return tensor\n\ndef changeTime(allTime):  \n    day = 24*60*60  \n    hour = 60*60  \n    min = 60  \n    if allTime <60:          \n        return  \"%d sec\"%math.ceil(allTime)  \n    elif  allTime > day:  \n        days = divmod(allTime,day)   \n        return \"%d days, %s\"%(int(days[0]),changeTime(days[1]))  \n    elif allTime > hour:  \n        hours = divmod(allTime,hour)  \n        return '%d hours, %s'%(int(hours[0]),changeTime(hours[1]))  \n    else:  \n        mins = divmod(allTime,min)  \n        return \"%d mins, %d sec\"%(int(mins[0]),math.ceil(mins[1]))\n    \ndef clean(text): \n    ## Remove puncuation\n    text = text.translate(string.punctuation)\n    ## Convert words to lower case and split them\n    text = text.lower()\n    ## Remove stop words\n    #text = text.split()\n    #stops = set(stopwords.words(\"english\"))\n    #text = [w for w in text if not w in stops and len(w) >= 3]\n    #text = \" \".join(text)\n\n    # Clean the text\n    text = re.sub(r\"[^A-Za-z0-9^,!.\\/'+-=]\", \" \", text)\n    text = re.sub(r\"what's\", \"what is \", text)\n    text = re.sub(r\"\\'s\", \" \", text)\n    text = re.sub(r\"\\'ve\", \" have \", text)\n    text = re.sub(r\"n't\", \" not \", text)\n    text = re.sub(r\"i'm\", \"i am \", text)\n    text = re.sub(r\"\\'re\", \" are \", text)\n    text = re.sub(r\"\\'d\", \" would \", text)\n    text = re.sub(r\"\\'ll\", \" will \", text)\n    text = re.sub(r\",\", \" \", text)\n    text = re.sub(r\"\\.\", \" \", text)\n    text = re.sub(r\"!\", \" ! \", text)\n    text = re.sub(r\"\\/\", \" \", text)\n    text = re.sub(r\"\\^\", \" ^ \", text)\n    text = re.sub(r\"\\+\", \" + \", text)\n    text = re.sub(r\"\\-\", \" - \", text)\n    text = re.sub(r\"\\=\", \" = \", text)\n    text = re.sub(r\"'\", \" \", text)\n    #text = re.sub(r\"(\\d+)(k)\", r\"\\g<1>000\", text)\n    text = re.sub(r\":\", \" : \", text)\n    text = re.sub(r\" e g \", \" eg \", text)\n    text = re.sub(r\" b g \", \" bg \", text)\n    text = re.sub(r\" u s \", \" american \", text)\n    text = re.sub(r\"\\0s\", \"0\", text)\n    text = re.sub(r\" 9 11 \", \"911\", text)\n    text = re.sub(r\"e - mail\", \"email\", text)\n    text = re.sub(r\"j k\", \"jk\", text)\n    text = re.sub(r\"\\s{2,}\", \" \", text)\n    #text = re.sub('[^a-zA-Z]',' ', text)\n    text = re.sub('  +',' ',text)\n    #text = text.split()\n    #stemmer = SnowballStemmer('english')\n    #stemmed_words = [stemmer.stem(word) for word in text]\n    #text = \" \".join(stemmed_words)\n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94ab8e1ecbe409ae157a6a78c944db930c81800f"},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\nsubmission = pd.read_csv('../input/sample_submission.csv')\nprint(\"Train shape : \",train_df.shape)\nprint(\"Test shape : \",test_df.shape)\nprint(\"submission shape :\", submission.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4a3b1ae78d467d72a17fdbc1bde790776cf1cf01"},"cell_type":"code","source":"train_df['clean_text'] = train_df['question_text'].apply(clean)\ntest_df['clean_text'] = test_df['question_text'].apply(clean)\n#split to train and val\ntrain_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)\n\n#fill up the missing values\ntrain_X = train_df['clean_text'].fillna('_na_').values  #1175509\nval_X = val_df['clean_text'].fillna('_na_').values  #130613\ntest_X = test_df['clean_text'].fillna('_na_').values  #563","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2bddd0451ea839616261c71aea2f3a4143bdac68"},"cell_type":"code","source":"#Tokenize the sequences\ntokenizer = Tokenizer(num_words=words_size)\ntokenizer.fit_on_texts(list(train_X) + list(test_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X =tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\n#Get the target values\ntrain_y = train_df['target'].values  #1175509\nval_y = val_df['target'].values  #130613","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b7deb5baaecdbc63a18fc2b34043f65bcb28ae0"},"cell_type":"code","source":"#convert to tensor\ntrain_tensor_X = [torch.tensor(x) for x in train_X]\nval_tensor_X = [torch.tensor(x) for x in val_X]\ntest_tensor_X = [torch.tensor(x) for x in test_X]\ntrain_tensor_y = torch.from_numpy(train_y)\nval_tensor_y = torch.from_numpy(val_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca6ec55b378d63f147d4ef78237cf031a55f6473"},"cell_type":"code","source":"n_batchs = int(len(train_tensor_X) / batch_size) + 1\nprint(n_batchs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18ceb437a7b0acbefcf977451f80d9881c871f11"},"cell_type":"code","source":"#load glove\nEMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))  \n\nword_index = tokenizer.word_index\nnb_words = len(word_index)\nembedding_matrix = np.zeros((nb_words, embedding_size))\nfor word, i in word_index.items():\n    #1 <= i <= 187159\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i-1] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d9147b849d92c37791ee786f48c89391dcbef64"},"cell_type":"code","source":"#use glove\nclass GRU_Classifier(nn.Module):  \n    def __init__(self, input_size, pretrained_embeddings, embedding_size=300, hidden_size=100,dropout=0.5):\n        self.hidden_size = hidden_size\n        self.batch_size = batch_size\n        super(GRU_Classifier, self).__init__()\n        self.embedding = nn.Embedding(input_size, embedding_size)\n        self.embedding.weight.data.copy_(torch.from_numpy(pretrained_embeddings))\n        #self.embedding.weight.requires_grad = False\n        self.gru = nn.GRU(input_size=embedding_size, hidden_size=hidden_size, \n                          bidirectional=True, batch_first=True)\n        #out \n        self.linear1 = nn.Linear(2 * hidden_size, hidden_size)\n        self.linear2 = nn.Linear(hidden_size, 1)\n        self.dropout = nn.Dropout(dropout)\n        \n    def forward(self, question, train=True):\n        batch = question.size(0)\n        question_embed = self.embedding(question)  #bacth x max_len x embedding_size\n        #gru_output: batch x max_len x (2 x hidden_size), hidden: num_directions(2) x batch x hidden_size\n        hidden = self.init_hidden(batch)\n        gru_output, hidden = self.gru(question_embed, hidden) \n        hidden = hidden.transpose(0, 1).contiguous().view(batch, -1)  #batch x (2 x hidden_size)   \n        if train:\n            hidden = self.dropout(hidden)\n        hidden = torch.relu(self.linear1(hidden))  #batch x hidden_size\n        if train:\n            hidden = self.dropout(hidden)\n        return torch.sigmoid(self.linear2(hidden))  \n    \n    def init_hidden(self, batch_size):\n        return cuda_available(torch.zeros(2, batch_size, self.hidden_size))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5655e514a233e12f702f23a50d84f8a490812003"},"cell_type":"code","source":"#use glove\nmodel_gru = GRU_Classifier(nb_words, embedding_matrix, embedding_size=embedding_size, \n                           hidden_size=hidden_size, dropout=dropout).cuda()\ncriterion = nn.BCELoss() \noptimizer = torch.optim.Adam(model_gru.parameters(), lr=lr)  #non-static\n#optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model_gru.parameters()), lr=lr)  #static","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba2c22136af37ff66cdc29cf8b854f5572b691b7"},"cell_type":"code","source":"#train process\ntrain_loss_array = []  #keep total loss\nstarttime = datetime.datetime.now()\nprint(\"Training for %d epochs...\" % epoch)\nfor i in range(epoch):\n    train_loss = 0\n    for i in range(n_batchs):\n        optimizer.zero_grad()\n        X = train_tensor_X[i * batch_size:(i + 1) * batch_size]\n        y = cuda_available(train_tensor_y[i * batch_size:(i + 1) * batch_size])\n        X_lengths = torch.LongTensor([x for x in map(len, X)]).cuda()\n        X_tensor = cuda_available(torch.zeros((len(X), X_lengths.max()))).long()\n        for idx, (seq, seqlen) in enumerate(zip(X, X_lengths)):\n            X_tensor[idx, :seqlen] = seq\n        output = model_gru(X_tensor, train=True).squeeze(1)\n        loss = criterion(output, y.float())\n        train_loss += loss.item()\n        loss.backward()\n        optimizer.step()\n        train_loss_array.append(train_loss)\n        endtime = datetime.datetime.now()\n    print(\"epoch is %d, train_loss is %.4f, batch is %d, cost time is about %s\" % \n          (i, train_loss, batch_size, changeTime((endtime - starttime).seconds)))\nprint(\"train finish!\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa327c33953b6cced05b087fe1be3f9a890b46af"},"cell_type":"code","source":"n_batchs = int(len(val_tensor_X) / batch_size) + 1\nprint(n_batchs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63cfd4ccb41d19e50249d12fc7178e40c94c8880"},"cell_type":"code","source":"#test gru on val_set\nbest_F1 = 0\nbest_threshold = 0\nfor thresh in np.arange(0.1, 0.701, 0.01):\n    thresh = np.round(thresh, 2)\n    TP, TN, FP, FN = 0, 0, 0, 0\n    for i in range(n_batchs):\n        X = val_tensor_X[i * batch_size:(i + 1) * batch_size]\n        y = cuda_available(val_tensor_y[i * batch_size:(i + 1) * batch_size])\n        X_lengths = torch.LongTensor([x for x in map(len, X)]).cuda()\n        X_tensor = cuda_available(torch.zeros((len(X), X_lengths.max()))).long()\n        for idx, (seq, seqlen) in enumerate(zip(X, X_lengths)):\n            X_tensor[idx, :seqlen] = seq\n        output = model_gru(X_tensor, train=False)\n        predict1 = (output.squeeze(1) > thresh).long()\n        TP += ((predict1 == 1) & (y == 1)).sum()\n        TN += ((predict1 == 0) & (y == 0)).sum()\n        FN += ((predict1 == 0) & (y == 1)).sum()\n        FP += ((predict1 == 1) & (y == 0)).sum()\n    p = TP.item() / (TP + FP).item()\n    r = TP.item() / (TP + FN).item()\n    F1 = 2 * r * p / (r + p)\n    acc = (TP + TN).item() / (TP + TN + FP + FN).item()\n    if F1 > best_F1:\n        best_F1 = F1\n        best_threshold = thresh\n    print(\"threshold {0}:F1 score={1}, Acc={2}\".format(thresh, F1, acc))\nprint('----------------------------------------------------------')\nprint('the best_F1 score={0} at threshold {1}'.format(best_F1, best_threshold))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a5d74b7bc8ee45c6fcc4400b5253edfd02300baf"},"cell_type":"code","source":"n_batchs = int(len(test_tensor_X) / batch_size) + 1\nprint(n_batchs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a60ad52b8e7bf1e9936241c12699f152a0dc708c"},"cell_type":"code","source":"#predict\npredict = []\nfor i in range(n_batchs):\n    X = test_tensor_X[i * batch_size:(i + 1) * batch_size]\n    X_lengths = torch.LongTensor([x for x in map(len, X)]).cuda()\n    X_tensor = cuda_available(torch.zeros((len(X), X_lengths.max()))).long()\n    for idx, (seq, seqlen) in enumerate(zip(X, X_lengths)):\n        X_tensor[idx, :seqlen] = seq\n    output = model_gru(X_tensor, train=False).squeeze(1)\n    result = (output > best_threshold)\n    for j in range(len(X)):\n        predict.append(result[j].item())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf494b0e181660bb9555b75a3842c24c4620ea9e"},"cell_type":"code","source":"#just see\npredict[40:50]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c497312d1927ec48a17c746ddbf997f4ed47dc7"},"cell_type":"code","source":"#submit\nsubmission['prediction'] = predict\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}