{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport pandas as pd\nimport numpy as np\nimport string\nimport re\nimport keras\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import f1_score\n\nfrom keras.models import Sequential, Model\nfrom keras.layers import Dense\nfrom keras.layers import Dropout\nfrom keras.layers import CuDNNLSTM, CuDNNGRU\nfrom keras.layers import Bidirectional\nfrom keras.layers import Embedding\nfrom keras.layers import Flatten\nfrom keras.layers import SpatialDropout1D\nfrom keras.layers import Input, Conv1D, MaxPooling1D, concatenate\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras import backend as K\n\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def f1(y_true, y_pred):\n    # F1 function to be used while training the model\n    def recall(y_true, y_pred):\n        \"\"\"Recall metric.\n\n        Only computes a batch-wise average of recall.\n\n        Computes the recall, a metric for multi-label classification of\n        how many relevant items are selected.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        \"\"\"Precision metric.\n\n        Only computes a batch-wise average of precision.\n\n        Computes the precision, a metric for multi-label classification of\n        how many selected items are relevant.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))\n\ndef calc_ratio(p):\n    # p is the fraction of the resampled dataset that y = 1\n    # returns the ratio of y=1 : y=0\n    return p/(1-p)\n\ndef pretoken(X):\n    # processing the text\n    X = X.map(lambda x : re.sub(r'[^\\w\\s]','',x)) # remove punctuations\n    X = X.map(lambda x : x.lower()) # Lowercased\n    X = X.map(lambda x : x.split()) # split strings into list of strings\n    return X\n\ndef tokenize_padding(X, max_length, token):\n    X = token.texts_to_sequences(X)\n    X = pad_sequences(X, maxlen=max_length, padding='post')\n    return X\n    \ndef train_preproc(X, max_length):\n    # function to prepare the training data set\n    X = pretoken(X)\n    token = Tokenizer()\n    token.fit_on_texts(X_train)\n    X = tokenize_padding(X,max_length,token)\n    return X, token, len(token.word_index) + 1\n\ndef test_preproc(X, max_length, token):\n    # function to prepare the dev and test data sets\n    X = pretoken(X)\n    X = tokenize_padding(X,max_length,token)\n    return X\n\ndef cutoff(x, frac):\n    if x > frac:\n        return 1\n    else:\n        return 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a41ed568b4eb22a8b6b5a31aedc57fb9fdb8ef41"},"cell_type":"code","source":"# Importing the data\ntrainData = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"49c4c6d21adc1deb9f758698a7f5db9051e1d7af"},"cell_type":"code","source":"# Importing the glove word embeddings into a dict\ndef get_coefs(word, *args): \n    return word, np.asarray(args, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open('../input/embeddings/glove.840B.300d/glove.840B.300d.txt', encoding = 'utf8'))\n\n# Calculating the mean and std of the word embeddings\nglove_matrix = np.stack(embeddings_index.values())\nglove_mean = glove_matrix.mean()\nglove_std = glove_matrix.std()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"361d86bc395c090aab78a0b4ef5dc520ff304dd8"},"cell_type":"code","source":"# Splitting the train and dev data sets and pre-processing all train, dev and test\nX_train , X_dev, y_train, y_dev = train_test_split(trainData['question_text'], trainData['target'], test_size=0.2, random_state=93)\nmax_sen_length = 30 # arbitrary. to be tuned later\nX_train, token, vocab_size = train_preproc(X_train, max_sen_length)\nX_dev = test_preproc(X_dev, max_sen_length, token)\nX_test = test_preproc(test['question_text'], max_sen_length,token)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"777c6c3cef2859462de45096d2547e61670742e0"},"cell_type":"code","source":"# Create a weight matrix for words \nembedding_matrix = np.random.normal(glove_mean, glove_std, [vocab_size,300])\nfor word, i in token.word_index.items():\n    try:\n        embedding_matrix[i] = embeddings_index[word]\n    except:\n        continue","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e28949a383c069ea8cd558dfcf6950fa8cca7a6c"},"cell_type":"code","source":"# 2x LSTM Model \nmodel2 = Sequential()\nmodel2.add(Embedding(vocab_size, 300, weights = [embedding_matrix], input_length = max_sen_length, trainable = False))\nmodel2.add(SpatialDropout1D(0.2))\nmodel2.add(Bidirectional(CuDNNLSTM(100, return_sequences = True)))\nmodel2.add(Bidirectional(CuDNNLSTM(100, return_sequences = True)))\nmodel2.add(Flatten())\nmodel2.add(Dense(16, activation = 'relu'))\nmodel2.add(Dropout(0.5))\nmodel2.add(Dense(8, activation = 'relu'))\nmodel2.add(Dropout(0.5))\nmodel2.add(Dense(1, activation = 'sigmoid'))\nopt = keras.optimizers.Adam(lr = 1e-3, decay = 1e-5)\nmodel2.compile(loss = 'binary_crossentropy', optimizer = opt, metrics = [f1,'accuracy'])\nmodel2.fit(X_train, y_train, batch_size = 256, epochs = 5, validation_data = (X_dev,y_dev), verbose = 2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f2d08968195c5cf1ecdf0041444426d16fab3bc2"},"cell_type":"code","source":"# CNN model with various filters and max pooling\nfilter_size = [1,2,3,4]\nconv_lst = []\npool_lst = []\nflat_lst = []\n\nmodel_input = Input(shape=(max_sen_length,), dtype='int32', name = 'model_input')\nemb_layer = Embedding(vocab_size, 300, weights = [embedding_matrix], input_length = max_sen_length, trainable = False)(model_input)\nx = SpatialDropout1D(0.4)(emb_layer)\nfor i in range(len(filter_size)):\n    conv_lst += [Conv1D(filters = 64, kernel_size = filter_size[i], padding = 'same', name = 'conv'+str(filter_size[i]))(x)]\n    pool_lst += [MaxPooling1D(pool_size = max_sen_length+1-filter_size[i])(conv_lst[i])]\n    flat_lst += [Flatten()(pool_lst[i])]\nx = concatenate(flat_lst)\nx = Dense(16, activation = 'relu')(x)\nx = Dropout(0.5)(x)\nx = Dense(8, activation = 'relu')(flat_lst[i])\nmodel_output = Dense(1, activation = 'sigmoid')(x)\nopt = keras.optimizers.Adam(lr = 1e-3, decay = 1e-5)\n\nmodel = Model(inputs=model_input, outputs = model_output)\nmodel.compile(loss = 'binary_crossentropy', optimizer = opt, metrics = [f1,'accuracy'])\nmodel.fit(X_train, y_train, epochs = 3, batch_size=256, validation_data = (X_dev,y_dev), verbose = 2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4a6aa4e18d09bb582d9e8485e3a4d155f9b945be"},"cell_type":"code","source":"# ensemble the 2 models\nensemble_p = 0.3 # magic number first, will tune later on\ny_dev_predict = pd.DataFrame(ensemble_p*model.predict(X_dev) + (1-ensemble_p)*model2.predict(X_dev))\n\n# estimating the cut off level to determine if the question is insincere\nf1_lst = []\nfor i in range(1,7):\n    f1_lst += [f1_score(y_dev, y_dev_predict[0].map(lambda x : cutoff(x, i/10)))]\n\nensemble_cutoff = (np.argmax(f1_lst) + 1)/10","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a65f8f6fbfb708203be9c06f413a7f226bc11354"},"cell_type":"code","source":"# calculating the predictions for the test set\ny_pred = ensemble_p*model.predict(X_test) + (1-ensemble_p)*model2.predict(X_test)\ny_pred = pd.DataFrame(y_pred)\n\n# preparing submission file\nmy_submission = pd.DataFrame({'qid': test.qid, 'prediction': y_pred[0].map(lambda x : cutoff(x, ensemble_cutoff))})\nmy_submission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1ba2d340d1ec08bd830a9671a23b67f8743d013c"},"cell_type":"markdown","source":"This blended model does not perform (previous runs obtain a score of about 0.66) as well as the previous 2x bi-lstm with input and recurrent dropouts (public score of 0.672).\n\n**Next step**\n\n1. Implement attention mechnism for the lstm models.\n2. Blend the glove word embedding with the 3 other embeddings available. Two possible approaches: take the simple average of the embeddings or implement a method to learn the weights\n\nHyper parameter tuning can be done later. Current priority is to learn, build and test as many different models/architectures/mechanisms as possible."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}