{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\nnp.random.seed(42)\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score\nfrom keras.models import Model\nfrom keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D\nfrom keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D\nfrom keras.preprocessing import text, sequence\nfrom keras.callbacks import Callback\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(os.listdir(\"../input\"))\n\nEMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ntrain = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')\nsubmission = pd.read_csv('../input/sample_submission.csv')\n\nX_train = train[\"question_text\"].fillna(\"_na_\").values\ny_train = train[\"target\"].values\nX_test = test[\"question_text\"].fillna(\"_na_\").values\n\nmax_features = 40000\nmaxlen = 200\nembed_size = 300\n\ntokenizer = text.Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(X_train) + list(X_test))\n\nX_train = tokenizer.texts_to_sequences(X_train)\nX_test = tokenizer.texts_to_sequences(X_test)\nx_train = sequence.pad_sequences(X_train, maxlen=maxlen)\nx_test = sequence.pad_sequences(X_test, maxlen=maxlen)\n\n# embdedding setup\n# Source https://blog.keras.io/using-pre-trained-word-embeddings-in-a-keras-model.html\nfrom tqdm import tqdm\n\nembeddings_index = {}\nf = open(EMBEDDING_FILE, 'r', encoding='utf-8')\nfor line in tqdm(f):\n    values = line.split(\" \")\n    word = values[0]\n    coefs = np.asarray(values[1:], dtype='float32')\n    embeddings_index[word] = coefs\nf.close()\n\nprint('Found %s word vectors.' % len(embeddings_index))\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nprint(len(word_index), max_features)\nembedding_matrix = np.zeros((nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n        \nclass F1Evaluation(Callback):\n    def __init__(self, validation_data=(), interval=1):\n        super(Callback, self).__init__()\n\n        self.interval = interval\n        self.X_val, self.y_val = validation_data\n\n    def on_epoch_end(self, epoch, logs={}):\n        if epoch % self.interval == 0:\n            y_pred = self.model.predict(self.X_val, verbose=0)\n            y_pred = (y_pred > 0.5).astype(int)\n            score = f1_score(self.y_val, y_pred)\n            print(\"\\n F1 Score - epoch: %d - score: %.6f \\n\" % (epoch+1, score))     \n            \nfrom keras.models import Model\nfrom keras.layers import Input, Dropout, LSTM, Activation, Dense, SpatialDropout1D, concatenate, BatchNormalization\nfrom keras.layers.convolutional import Conv1D\nfrom keras.layers.pooling import GlobalMaxPooling1D, GlobalAveragePooling1D,MaxPooling1D\nfrom keras.layers.embeddings import Embedding\nfrom keras.preprocessing import sequence\nfrom keras.initializers import glorot_uniform\nnp.random.seed(1)\n\ndef get_model():    \n    inp = Input(shape=(maxlen, ))\n    x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp)\n    \n    X = SpatialDropout1D(0.1)(x)\n\n    \n    X1 = Conv1D(filters=256, kernel_size=3,activation='relu')(X)\n    BatchNormalization()(X1)\n    X2 = Conv1D(filters=256, kernel_size=4,activation='relu')(X)\n    BatchNormalization()(X2)\n    X3 = Conv1D(filters=256, kernel_size=5,activation='relu')(X)\n    BatchNormalization()(X3)\n    X4 = Conv1D(filters=256, kernel_size=6,activation='relu')(X)\n    BatchNormalization()(X4)   \n    X5 = Conv1D(filters=256, kernel_size=7,activation='relu')(X)\n    BatchNormalization()(X5)\n    \n    \n    avg_pool1 = GlobalAveragePooling1D()(X1)    \n    max_pool1 = GlobalMaxPooling1D()(X1)\n    avg_pool2 = GlobalAveragePooling1D()(X2)    \n    max_pool2 = GlobalMaxPooling1D()(X2)\n    avg_pool3 = GlobalAveragePooling1D()(X3)    \n    max_pool3 = GlobalMaxPooling1D()(X3)\n    avg_pool4 = GlobalAveragePooling1D()(X4)    \n    max_pool4 = GlobalMaxPooling1D()(X4) \n    avg_pool5 = GlobalAveragePooling1D()(X5)    \n    max_pool5 = GlobalMaxPooling1D()(X5)    \n    z = concatenate([avg_pool1, max_pool1, avg_pool2, max_pool2,  avg_pool3, max_pool3, avg_pool4, max_pool4, avg_pool5, max_pool5])\n          \n    z = Dropout(0.1)(z)        \n        \n    outp = Dense(1, activation=\"sigmoid\")(z)\n    \n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy',\n                  optimizer='adam',\n                  metrics=['accuracy'])\n\n    return model\n\nmodel = get_model()\n\nmodel.summary()\n\nfrom keras.callbacks import ModelCheckpoint\nfrom keras.models import load_model\nmodelname = \"submission_paralleleCNN-50seqlen.h5\"\n\ncheckpoint = ModelCheckpoint(filepath= modelname, monitor='val_acc', save_best_only=True, mode='max')\nbatch_size = 256\nepochs = 2\n\nX_tra, X_val, y_tra, y_val = train_test_split(x_train, y_train, train_size=0.95,\n                                              random_state=233)\nF1_Score = F1Evaluation(validation_data=(X_val, y_val), interval=1)\n\n\nhist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs,\n                 validation_data=(X_val, y_val),\n                 callbacks=[F1_Score, checkpoint], verbose=2)\n\nmaxvalue = max(hist.history['val_acc'])\nepoch = hist.history['val_acc'].index(maxvalue) +1\nprint(epoch, maxvalue)\n\nfrom keras.models import load_model\n\nmodelmax = load_model(modelname)\n\nfrom sklearn.metrics import confusion_matrix\nyprediction = modelmax.predict(X_val)\n\ny_pred = (yprediction > 0.5).astype(int)\ny_true = (y_val > 0.5).astype(int)\n\nscore = f1_score(y_val, y_pred)\nprint(\"F1_score: \", score)\n\n#confusion matrix\nimport itertools\nimport matplotlib.pyplot as plt\ndef plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    \n\nF1_score_lst = {}\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    score = f1_score(y_val, (yprediction>thresh).astype(int))\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, score))\n    F1_score_lst[thresh] = score\n    \nmaxF1 = max(F1_score_lst.values())\nprint(maxF1)\n\nthresh = list(F1_score_lst.keys())[list(F1_score_lst.values()).index(maxF1)]\nprint(thresh)\n\ny_pred_thresh = (yprediction > thresh).astype(int)\ny_true_thresh = (y_val > thresh).astype(int)\n\nconfusion_mtx = confusion_matrix(y_true_thresh, y_pred_thresh)\n#plot confusion matrix\nplot_confusion_matrix(confusion_mtx, classes = range(2))\n\n#prediction on test sample\ny_pred_test = modelmax.predict([x_test], batch_size = 1024, verbose = 1)\ny_pred_test_thresh = (y_pred_test > thresh).astype(int)\n\nout_df = pd.DataFrame({\"qid\":test[\"qid\"].values})\nout_df['prediction'] = y_pred_test_thresh\n\nout_df.to_csv(\"submission.csv\", index=False)\n\nprint(\"submission done\")\n\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}