{"cells":[{"metadata":{"_uuid":"639b324929911978233c5562fbacb37131626d46"},"cell_type":"markdown","source":"Code forked from Vladimirs kernel https://www.kaggle.com/yekenot/2dcnn-textclassifier so all credit for the architecture goes to him. I just converted o Conv1D and added flipping augmentation."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nnp.random.seed(42)\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score\nfrom keras.models import Model\nfrom keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D\nfrom keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D\nfrom keras.preprocessing import text, sequence\nfrom keras.callbacks import Callback\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b60b0950ddf9fce1c088bf2f89d07ce04c3a82e1"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ntrain = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')\nsubmission = pd.read_csv('../input/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"784c915cb895f5c7586d4dd61d99d70a3c9a5651"},"cell_type":"code","source":"X_train = train[\"question_text\"].fillna(\"fillna\").values\ny_train = train[\"target\"].values\nX_test = test[\"question_text\"].fillna(\"fillna\").values\n\nmax_features = 40000\nmaxlen = 50\nembed_size = 300\n\ntokenizer = text.Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(X_train) + list(X_test))\nX_train = tokenizer.texts_to_sequences(X_train)\nX_test = tokenizer.texts_to_sequences(X_test)\nx_train = sequence.pad_sequences(X_train, maxlen=maxlen)\nx_test = sequence.pad_sequences(X_test, maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f2ee3631454ab235e7951a2313f11100bf7b7d97"},"cell_type":"code","source":"def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(EMBEDDING_FILE))\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.zeros((nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bfd80e14218851850452b086565d4f8414a0517d"},"cell_type":"code","source":"class F1Evaluation(Callback):\n    def __init__(self, validation_data=(), interval=1):\n        super(Callback, self).__init__()\n\n        self.interval = interval\n        self.X_val, self.y_val = validation_data\n\n    def on_epoch_end(self, epoch, logs={}):\n        if epoch % self.interval == 0:\n            y_pred = self.model.predict(self.X_val, verbose=0)\n            y_pred = (y_pred > 0.35).astype(int)\n            score = f1_score(self.y_val, y_pred)\n            print(\"\\n F1 Score - epoch: %d - score: %.6f \\n\" % (epoch+1, score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7a3fed14c968b22e18f6c6c855ab9c23b9f0cce3"},"cell_type":"code","source":"filter_sizes = [1,2,3,5]\nnum_filters = 36\nfrom keras.layers import Conv1D, MaxPool1D, BatchNormalization\ndef get_model():    \n    inp = Input(shape=(maxlen, ))\n    x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\n    x = SpatialDropout1D(0.4)(x)\n    #x = Reshape((maxlen, embed_size, 1))(x)\n    \n    conv_0 = Conv1D(num_filters, kernel_size=(filter_sizes[0]),\n                                 kernel_initializer='he_normal', activation='elu')(x)\n    conv_1 = Conv1D(num_filters, kernel_size=(filter_sizes[1]),\n                                 kernel_initializer='he_normal', activation='elu')(x)\n    conv_2 = Conv1D(num_filters, kernel_size=(filter_sizes[2]), \n                                 kernel_initializer='he_normal', activation='elu')(x)\n    conv_3 = Conv1D(num_filters, kernel_size=(filter_sizes[3]),\n                                 kernel_initializer='he_normal', activation='elu')(x)\n    \n    maxpool_0 = MaxPool1D(pool_size=(maxlen - filter_sizes[0] + 1))(conv_0)\n    maxpool_1 = MaxPool1D(pool_size=(maxlen - filter_sizes[1] + 1))(conv_1)\n    maxpool_2 = MaxPool1D(pool_size=(maxlen - filter_sizes[2] + 1))(conv_2)\n    maxpool_3 = MaxPool1D(pool_size=(maxlen - filter_sizes[3] + 1))(conv_3)\n        \n    z = Concatenate(axis=1)([maxpool_0, maxpool_1, maxpool_2, maxpool_3])   \n    z = Flatten()(z)\n    z = BatchNormalization()(z)\n        \n    outp = Dense(1, activation=\"sigmoid\")(z)\n    \n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy',\n                  optimizer='adam',\n                  metrics=['accuracy'])\n\n    return model\n\nmodel = get_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"51deacab40f77c038525faf1cfd2c8e820f7f01b"},"cell_type":"code","source":"batch_size = 1024\nepochs = 4\n\nX_tra, X_val, y_tra, y_val = train_test_split(x_train, y_train, train_size=0.95,\n                                              random_state=233)\nF1_Score = F1Evaluation(validation_data=(X_val, y_val), interval=1)\n\nhist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs,\n                 validation_data=(X_val, y_val),\n                 callbacks=[F1_Score], verbose=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d9522fbc1780725f81af8ed34cb2e804d6284e8"},"cell_type":"code","source":"filter_sizes = [1,2,3,5]\nnum_filters = 36\nfrom keras.layers import Conv1D, MaxPool1D, BatchNormalization, Lambda\nimport keras.backend as K\ndef get_model():    \n    inp = Input(shape=(maxlen, ))\n    x = Lambda(lambda x: K.reverse(x,axes=-1))(inp)\n    x = Embedding(max_features, embed_size, weights=[embedding_matrix])(x)\n    x = SpatialDropout1D(0.4)(x)\n    #x = Reshape((maxlen, embed_size, 1))(x)\n    \n    conv_0 = Conv1D(num_filters, kernel_size=(filter_sizes[0]),\n                                 kernel_initializer='he_normal', activation='elu')(x)\n    conv_1 = Conv1D(num_filters, kernel_size=(filter_sizes[1]),\n                                 kernel_initializer='he_normal', activation='elu')(x)\n    conv_2 = Conv1D(num_filters, kernel_size=(filter_sizes[2]), \n                                 kernel_initializer='he_normal', activation='elu')(x)\n    conv_3 = Conv1D(num_filters, kernel_size=(filter_sizes[3]),\n                                 kernel_initializer='he_normal', activation='elu')(x)\n    \n    maxpool_0 = MaxPool1D(pool_size=(maxlen - filter_sizes[0] + 1))(conv_0)\n    maxpool_1 = MaxPool1D(pool_size=(maxlen - filter_sizes[1] + 1))(conv_1)\n    maxpool_2 = MaxPool1D(pool_size=(maxlen - filter_sizes[2] + 1))(conv_2)\n    maxpool_3 = MaxPool1D(pool_size=(maxlen - filter_sizes[3] + 1))(conv_3)\n        \n    z = Concatenate(axis=1)([maxpool_0, maxpool_1, maxpool_2, maxpool_3])   \n    z = Flatten()(z)\n    z = BatchNormalization()(z)\n        \n    outp = Dense(1, activation=\"sigmoid\")(z)\n    \n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy',\n                  optimizer='adam',\n                  metrics=['accuracy'])\n\n    return model\n\nmodel_flip = get_model()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55798d500e19aeb88b96d964662630c8f0247da6"},"cell_type":"code","source":"hist_flip = model_flip.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs,\n                 validation_data=(X_val, y_val),\n                 callbacks=[F1_Score], verbose=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d84681f33bc0212621b7a88a11273081447e0bd"},"cell_type":"code","source":"val_y_pred1 = model.predict(X_val, batch_size=1024, verbose = True)\nval_y_pred2 = model_flip.predict(X_val, batch_size=1024, verbose = True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a0cc2c3ed519efb1ead8f5e6d76354e59e95ab6"},"cell_type":"code","source":"val_y_pred = np.mean([val_y_pred1,val_y_pred2],axis = 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9607e11c2591d51bb31da729c9934a9a69de854e"},"cell_type":"code","source":"y_pred1 = model.predict(x_test, batch_size=1024, verbose = True)\ny_pred2 = model_flip.predict(x_test, batch_size=1024, verbose = True)\ny_pred = np.mean([y_pred1,y_pred2],axis = 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b5b7a038d01b8350a4d66dd2d6641c8099d4fc54"},"cell_type":"code","source":"np.corrcoef([y_pred1[:,0],y_pred2[:,0]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d9ec4a9a930f338cb32589bfef39cbeddd8b2a9"},"cell_type":"code","source":"best_threshold = 0.01\nbest_score = 0.0\nfor threshold in range(1, 100):\n    threshold = threshold / 100\n    score = f1_score(y_val, val_y_pred > threshold)\n    if score > best_score:\n        best_threshold = threshold\n        best_score = score\nprint(\"Score at threshold=0.5 is {}\".format(f1_score(y_val, val_y_pred > 0.5)))\nprint(\"Optimal threshold is {} with a score of {}\".format(best_threshold, best_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d972f83b42f195c2c986e21a0fddf8e255c65fd"},"cell_type":"code","source":"y_pred = (y_pred > best_threshold).astype(int)\nsubmission['prediction'] = y_pred\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7685bcc3e67a411ac41c6f96ef393016cee83a1f"},"cell_type":"code","source":"submission.head(50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3004d6e29936f99e766a299e3ddb281e570f98c6"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}