{"cells":[{"metadata":{"_uuid":"3cad80fb17e89e86273882ea81b5c5094636395b"},"cell_type":"markdown","source":"- I referred below helpful kernels. Thanks for the authors! If you think this kernel is helpful, please upvote them!\n- https://www.kaggle.com/yekenot/2dcnn-textclassifier\n- https://www.kaggle.com/applecer/use-f1-to-select-model-lstm-based"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score\nfrom keras.models import Model\nfrom keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D\nfrom keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D\nfrom keras.preprocessing import text, sequence\nfrom keras.callbacks import Callback\nimport warnings\n\nLEN_WORDS = 30\nLEN_EMBEDDING = 300","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3e939c29a585088b26988d48f2070246b078973e"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ntrain = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')\nsubmission = pd.read_csv('../input/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a44e03d472ebb35ed9e38cb49a1a6351d8edbebb"},"cell_type":"code","source":"X_train = train[\"question_text\"].fillna(\"fillna\").values\ny_train = train[\"target\"].values\nX_test = test[\"question_text\"].fillna(\"fillna\").values\n\nmax_features = 30000\nmaxlen = 40\nembed_size = 300\n\ntokenizer = text.Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(X_train) + list(X_test))\nX_train = tokenizer.texts_to_sequences(X_train)\nX_test = tokenizer.texts_to_sequences(X_test)\nx_train = sequence.pad_sequences(X_train, maxlen=maxlen)\nx_test = sequence.pad_sequences(X_test, maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75073a7cc2fe4fb43c5a3254124605628aee485e"},"cell_type":"code","source":"def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(EMBEDDING_FILE))\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.zeros((nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43641fa6dccea92c101ea6a64d36770cae5af6e6"},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import CuDNNLSTM, Dense, Bidirectional, Dropout\nfrom keras.callbacks import Callback\nfrom sklearn.metrics import confusion_matrix, f1_score, precision_score, recall_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d1e0220535c024f3a30035dedcba86b052cf2a2f"},"cell_type":"code","source":"from keras import backend as K\n\ndef f1(y_true, y_pred):\n    def recall(y_true, y_pred):\n        \"\"\"Recall metric.\n\n        Only computes a batch-wise average of recall.\n\n        Computes the recall, a metric for multi-label classification of\n        how many relevant items are selected.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        \"\"\"Precision metric.\n\n        Only computes a batch-wise average of precision.\n\n        Computes the precision, a metric for multi-label classification of\n        how many selected items are relevant.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))\n\nclass Metrics(Callback):\n    def on_train_begin(self, logs={}):\n        self.val_f1s = []\n        self.val_recalls = []\n        self.val_precisions = []\n\n    def on_epoch_end(self, epoch, logs={}):\n        val_predict = (np.asarray(self.model.predict(self.model.validation_data[0]))).round()\n        val_targ = self.model.validation_data[1]\n        _val_f1 = f1_score(val_targ, val_predict)\n        _val_recall = recall_score(val_targ, val_predict)\n        _val_precision = precision_score(val_targ, val_predict)\n        self.val_f1s.append(_val_f1)\n        self.val_recalls.append(_val_recall)\n        self.val_precisions.append(_val_precision)\n        print(\" — val_f1: %f — val_precision: %f — val_recall %f\" %(_val_f1, _val_precision, _val_recall))\n        return \n \nmy_metrics = Metrics()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6901101cb37fa060f39e9afa453c5fb6177d1d91"},"cell_type":"code","source":"from keras.models import Model\nfrom keras.layers import Conv1D, Input, MaxPooling1D, Flatten, Dense, BatchNormalization, concatenate\nfrom keras.layers import LeakyReLU, Activation\nfrom keras.layers import LeakyReLU\n\nclass LeakyReLU(LeakyReLU):\n    def __init__(self, **kwargs):\n        self.__name__ = \"LeakyReLU\"\n        super(LeakyReLU, self).__init__(**kwargs)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0057a9c55d4ee1ed2ee7be5daec85c37167b17dc"},"cell_type":"code","source":"STRIDE_1 = 2\nSTRIDE_2 = 4\nSTRIDE_3 = 8\n\nFILTER_1 = 64\nFILTER_2 = 64\nFILTER_3 = 64","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"edb2d66913c24990c689e08c98349189ef59788e"},"cell_type":"markdown","source":"# Conv1D line"},{"metadata":{"trusted":true,"_uuid":"43a181d2d060bb37fd8f9f20d382f11d7ba949fe"},"cell_type":"code","source":"inp = Input(shape=(maxlen, ))\nembed_layer1 = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\nembed_layer1 = SpatialDropout1D(0.4)(embed_layer1)\n\n# line1 = BatchNormalization()(Input_layer)\nline1 = Conv1D(FILTER_1, STRIDE_1)(embed_layer1)\nline1 = Activation(LeakyReLU())(line1)\nline1 = MaxPooling1D(STRIDE_1)(line1)\nline1 = Conv1D(FILTER_1, STRIDE_1)(line1)\nline1 = Activation(LeakyReLU())(line1)\nline1 = MaxPooling1D(STRIDE_1)(line1)\nline1 = Conv1D(FILTER_1, STRIDE_1)(line1)\nline1 = Activation(LeakyReLU())(line1)\nline1 = MaxPooling1D(STRIDE_1*2)(line1)  # global max pooling\nline1 = Flatten()(line1)\n\n# line2 = BatchNormalization()(Input_layer)\nline2 = Conv1D(FILTER_2, STRIDE_1)(embed_layer1)\nline2 = Activation(LeakyReLU())(line2)\nline2 = MaxPooling1D(STRIDE_1)(line2)\nline2 = Conv1D(FILTER_2, STRIDE_1)(line2)\nline2 = Activation(LeakyReLU())(line2)\nline2 = MaxPooling1D(STRIDE_1)(line2)\nline2 = Conv1D(FILTER_2, STRIDE_1)(line2)\nline2 = Activation(LeakyReLU())(line2)\nline2 = MaxPooling1D(STRIDE_1*2)(line2)  # global max pooling\nline2 = Flatten()(line2)\n\n# line3 = BatchNormalization()(Input_layer)\nline3 = Conv1D(FILTER_3, STRIDE_1)(embed_layer1)\nline3 = Activation(LeakyReLU())(line3)\nline3 = MaxPooling1D(STRIDE_1)(line3)\nline3 = Conv1D(FILTER_3, STRIDE_1)(line3)\nline3 = Activation(LeakyReLU())(line3)\nline3 = MaxPooling1D(STRIDE_1)(line3)\nline3 = Conv1D(FILTER_3, STRIDE_1)(line3)\nline3 = Activation(LeakyReLU())(line3)\nline3 = MaxPooling1D(STRIDE_1*2)(line3)  # global max pooling\nline3 = Flatten()(line3)\n\nconcat_layer = concatenate([line1, line2, line3])\n\nconv1d_dense = Dense(1024, activation='relu')(concat_layer)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f0d2275c584e030f67fc4358d2714c7e2b3af379"},"cell_type":"markdown","source":"# LSTM line"},{"metadata":{"trusted":true,"_uuid":"ea9fa8077ab2691e9e5161cc5819bbb56bcf61ef"},"cell_type":"code","source":"embed_layer2 = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\nembed_layer2 = SpatialDropout1D(0.4)(embed_layer2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2a296cf070451deaf59186cc8dda43d01697700"},"cell_type":"code","source":"rnn_line = Bidirectional(CuDNNLSTM(32, return_sequences=True), input_shape=(maxlen, embed_size))(embed_layer2)\nrnn_line = Bidirectional(CuDNNLSTM(32))(rnn_line)\nrnn_dense = Dense(1024, activation='relu')(rnn_line)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4de047537c3485bc4fd8127ac221709e6087ab99"},"cell_type":"code","source":"total = concatenate([conv1d_dense, rnn_dense])\npreds = Dense(1, activation='sigmoid')(total)\n\nmodel = Model(inputs=inp, outputs=preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d771cdefff846b265675f6b8a4c7b6664fd7f349"},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"32f3c0f36a67778d64eefa5ead70458441e594af"},"cell_type":"code","source":"from keras.utils import plot_model\nplot_model(model, to_file='model.png')\n\nfrom IPython.display import SVG\nfrom keras.utils.vis_utils import model_to_dot\n\nSVG(model_to_dot(model).create(prog='dot', format='svg'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"211707aa75749d4e0b967d95d1bed052126921c7"},"cell_type":"code","source":"from keras import backend as K","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"42fb5248f8897fb13e849efb33087032c475299b"},"cell_type":"code","source":"def f1_loss(y_true, y_pred):\n    \"\"\"Custom f1 loss for bicategorical\n    y must be of shape where y.shape[-1] == 2\n    y[..., 0] must be the category for true\n    y[..., 1] must be the category for false\n    \"\"\"\n    true_truth = K.dot(y_true, K.constant([1., 0.], dtype='float32', shape=(2, 1)))\n    true_false = K.dot(y_true, K.constant([0., 1.], dtype='float32', shape=(2, 1)))\n\n    y_false = K.constant(1., dtype='float32') - y_true\n\n    fake_truth = K.dot(y_false, K.constant([1., 0.], dtype='float32', shape=(2, 1)))\n    fake_false = K.dot(y_false, K.constant([0., 1.], dtype='float32', shape=(2, 1)))\n\n    TP_temp = K.sum(true_truth * y_pred)\n    FP_temp = K.sum(fake_truth * y_pred)\n    FN_temp = K.sum(fake_false * y_pred)\n\n    loss = (FP_temp + FN_temp) / (2 * TP_temp + FP_temp + FN_temp + K.epsilon())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4db2ba346388afd570539d11de1e6fb9b70e0381"},"cell_type":"code","source":"model.compile(loss='binary_crossentropy',\n              optimizer='adam',\n              metrics=['accuracy', f1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7cf97f1cc0256dc20345effc08b1ec5a62acbf70"},"cell_type":"code","source":"X_tra, X_val, y_tra, y_val = train_test_split(x_train, y_train, train_size=0.95,\n                                              random_state=1989)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"990aee9428493148f8cf9d039b1efa6ebf15e2a1"},"cell_type":"code","source":"class F1Evaluation(Callback):\n    def __init__(self, validation_data=(), interval=1):\n        super(Callback, self).__init__()\n\n        self.interval = interval\n        self.X_val, self.y_val = validation_data\n\n    def on_epoch_end(self, epoch, logs={}):\n        if epoch % self.interval == 0:\n            y_pred = self.model.predict(self.X_val, verbose=0)\n            y_pred = (y_pred > 0.5).astype(int)\n            score = f1_score(self.y_val, y_pred)\n            print(\"\\n F1 Score - epoch: %d - score: %.6f \\n\" % (epoch+1, score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5877ddc84fe56009be2a92e672e98b7f65c7ff11"},"cell_type":"code","source":"my_weights = '../working/2_channel.h5'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b34f62fc5f57bb1894383fadafc668380a36f5b"},"cell_type":"code","source":"try:\n    model.load_weights(my_weights)\n    print('Load weights')\nexcept:\n    pass","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b21c817f97955de7e07c9231eaa648180d57839a","scrolled":false},"cell_type":"code","source":"batch_size = 256\nepochs = 50\n\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\ncheck_point = ModelCheckpoint(my_weights, monitor=\"val_f1\", mode=\"max\",\n                              verbose=True, save_best_only=True)\nearly_stop = EarlyStopping(monitor=\"val_f1\", mode=\"max\", patience=8,verbose=True)\nF1_Score = F1Evaluation(validation_data=(X_val, y_val), interval=1)\n\nhist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs,\n                 validation_data=(X_val, y_val),\n                 callbacks=[early_stop, check_point, F1_Score], verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"fd7c41c108142a3cc5c4b10ef114f6b76dd64565"},"cell_type":"code","source":"from sklearn import metrics\n\npred_noemb_val_y = model.predict([X_val], batch_size=1024, verbose=1)\nscores_list = dict()\nfor thresh in np.arange(0.1, 0.6, 0.01):\n    thresh = np.round(thresh, 2)\n    temp_score = metrics.f1_score(y_val, (pred_noemb_val_y>thresh).astype(int))\n    scores_list[thresh] = temp_score\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(y_val, (pred_noemb_val_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"202603de6d8a72b23155c2500ef1ba7f3c554e26"},"cell_type":"code","source":"opt_threshold = max(scores_list, key=scores_list.get)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4722f73b5c17c7b764978d0b4358be27cd2424d"},"cell_type":"code","source":"y_pred = model.predict(x_test, batch_size=1024, verbose=1)\ny_pred = (y_pred > opt_threshold).astype(int)\nsubmission['prediction'] = y_pred\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e9fa79880b9e50742f34a0702c53152ed545271"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba196891cb9aa319fb6d03cd28c584dc19b7666c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"533d9dc0b55af9a28ea38058eeaa3f6a6ac87e04"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"59c2dde466dde7ceda59439ce280c1d3d1254b26"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f9126a960346bd569444c9a2a75e23bb17492398"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1384f6abeac2cb6825af3462245b5320c7ef5236"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d141cb544f71d280aebc2df6091664725905498"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"206b963498a5ebd11abc71aec7be10f14a0a0f49"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}