{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom random import shuffle\n\nMAX_SENTENCE_WIDTH = 100\nEMBED_DIM = 300\nSUPER_BATCH_SIZE = 10000\n\ndef clean_str(string):\n    \"\"\"\n    Tokenization/string cleaning for all datasets except for SST.\n    Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py\n    \"\"\"\n    import re\n    string = re.sub(r\"[^A-Za-z0-9(),!?\\'\\`]\", \" \", string)\n    string = re.sub(r\"\\'s\", \" \\'s\", string)\n    string = re.sub(r\"\\'ve\", \" \\'ve\", string)\n    string = re.sub(r\"n\\'t\", \" n\\'t\", string)\n    string = re.sub(r\"\\'re\", \" \\'re\", string)\n    string = re.sub(r\"\\'d\", \" \\'d\", string)\n    string = re.sub(r\"\\'ll\", \" \\'ll\", string)\n    string = re.sub(r\",\", \" , \", string)\n    string = re.sub(r\"!\", \" ! \", string)\n    string = re.sub(r\"\\(\", \" \\( \", string)\n    string = re.sub(r\"\\)\", \" \\) \", string)\n    string = re.sub(r\"\\?\", \" \\? \", string)\n    string = re.sub(r\"\\s{2,}\", \" \", string)\n    return string.strip().lower()\n\ndef load_data_and_labels(df):\n    \"\"\"\n    Loads MR polarity data from files, splits the data into words and generates labels.\n    Returns split sentences and labels.\n    \"\"\"\n    # Load data from df\n    positive_examples = [row.question_text for row in df.itertuples() if row.target == 1]\n    positive_examples = [clean_str(s.strip()) for s in positive_examples]\n    negative_examples = [row.question_text for row in df.itertuples() if row.target == 0]\n    negative_examples = [clean_str(s.strip()) for s in negative_examples]\n    return positive_examples, negative_examples\n\ndef vocab(x_text):\n    return set(word for question in x_text for word in question.split(' '))\n\ndef get_embedding(fname, vocab):\n    embeddings = {}\n    for i, line in enumerate(open(fname, 'r')):\n        line_split = line.split(' ')\n        word = line_split[0]\n        if word in vocab:\n            vec = np.array([float(s) for s in line_split[1:]], dtype='float32')\n            embeddings[word] = vec\n    return embeddings\n    \ndef generate_features (question, embeddings):\n    feats = np.zeros((MAX_SENTENCE_WIDTH, EMBED_DIM))\n    for i, word in enumerate(question.split(\" \")):\n        if word in embeddings and i < MAX_SENTENCE_WIDTH:\n            feats[i, :] = embeddings[word]\n    return feats\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"920d2169a3bc4ac2de6c20975eca6bf8d4b618f8"},"cell_type":"code","source":"df = pd.read_csv(\"../input/train.csv\")\npositive_examples, negative_examples = load_data_and_labels(df)\ndel df\nvocab = vocab(positive_examples + negative_examples)\nfname = \"../input/embeddings/glove.840B.300d/glove.840B.300d.txt\"\nembeddings = get_embedding(fname, vocab)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b9558c304455fc26c53eb1bc749ab1c6ae35440"},"cell_type":"code","source":"num_pos_all = len(positive_examples)\nnum_neg_all = len(negative_examples)\ntrain_fraction = 0.99\n\ntrain_data = {\n    \"pos\": positive_examples[:int(train_fraction * num_pos_all)],\n    \"neg\": negative_examples[:int(train_fraction * num_neg_all)]\n}\n\ndev_data = {\n    \"pos\": positive_examples[int(train_fraction * num_pos_all):],\n    \"neg\": negative_examples[int(train_fraction * num_neg_all):]\n}\ndel positive_examples, negative_examples","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b30d24d6c5e8ae184a8079301cc98b6c4733820"},"cell_type":"code","source":"\nLABEL_FRACTION = 0.5\nnum_pos = int(LABEL_FRACTION * SUPER_BATCH_SIZE)\nnum_neg = SUPER_BATCH_SIZE - num_pos\n\nIDX = [i for i in range(SUPER_BATCH_SIZE)]\nshuffle(IDX)\n\npos_idx = 0\nneg_idx = 0\n\ndef generate_training_batch(pos_idx, neg_idx):\n\n    pos_idxs = [(pos_idx + i) % len(train_data[\"pos\"]) for i in range(num_pos)]\n    neg_idxs = [(neg_idx + i) % len(train_data[\"neg\"]) for i in range(num_neg)]\n    x = np.zeros(shape=(SUPER_BATCH_SIZE, MAX_SENTENCE_WIDTH, EMBED_DIM))\n    y = np.zeros(shape=(SUPER_BATCH_SIZE, 2))\n    for i, question in enumerate([train_data[\"pos\"][i] for i in pos_idxs]):\n        x[i, :, :] = generate_features(question, embeddings)\n        y[i, 1] = 1\n    for i, question in enumerate([train_data[\"neg\"][i] for i in neg_idxs]):\n        x[-i - 1, :, :] = generate_features(question, embeddings)\n        y[-i - 1, 0] = 1\n    return x[IDX, :, :], y[IDX, :]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"148203b1a7d204914b9e662cb179ce1aadc40df4"},"cell_type":"code","source":"def generate_val_data():\n    n = len(dev_data[\"pos\"]) + len(dev_data[\"neg\"])\n    x = np.zeros(shape=(n, MAX_SENTENCE_WIDTH, EMBED_DIM))\n    y = np.zeros(shape=(n, 2))\n    for i, question in enumerate(dev_data[\"pos\"]):\n        x[i, :, :] = generate_features(question, embeddings)\n        y[i, 1] = 1\n    for i, question in enumerate(dev_data[\"neg\"]):\n        x[-i - 1, :, :] = generate_features(question, embeddings)\n        y[-i - 1, 0] = 1\n    return x, y\n\nx_dev, y_dev = generate_val_data()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7cb491b6a16e0ba459e8eb084568f9048e9ed0e5"},"cell_type":"code","source":"\ndef define_model():\n    from keras.layers import Dense, Input, Flatten\n    from keras.layers import Reshape, Dropout, Concatenate\n    from keras.layers import Conv1D, MaxPool1D\n    from keras.models import Model\n    \n    inputs = Input(shape=(MAX_SENTENCE_WIDTH, EMBED_DIM), dtype='float32')\n\n    conv_0 = Conv1D(128, kernel_size=(3), padding='valid', kernel_initializer='normal', activation='relu')(inputs)\n    conv_1 = Conv1D(128, kernel_size=(4), padding='valid', kernel_initializer='normal', activation='relu')(inputs)\n    conv_2 = Conv1D(128, kernel_size=(5), padding='valid', kernel_initializer='normal', activation='relu')(inputs)\n    \n    maxpool_0 = MaxPool1D(pool_size=(5), strides=(3), padding='valid')(conv_0)\n    maxpool_1 = MaxPool1D(pool_size=(5), strides=(3), padding='valid')(conv_1)\n    maxpool_2 = MaxPool1D(pool_size=(5), strides=(3), padding='valid')(conv_2)\n    \n    concatenated_tensor = Concatenate(axis=1)([maxpool_0, maxpool_1, maxpool_2])\n    flatten = Flatten()(concatenated_tensor)\n    dropout = Dropout(0.5)(flatten)\n    preds = Dense(2, activation='softmax')(dropout)\n\n    # this creates a model that includes inputs and outputs\n    model = Model(inputs=inputs, outputs=preds)\n\n    model.compile(loss='categorical_crossentropy',\n              optimizer='adam',\n              metrics=['acc', 'mae'])\n\n    model.summary()\n    \n    return model\n    \n\nmodel = define_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ddded257bfeaa79bfa042425c4f9f5b58bae282"},"cell_type":"code","source":"\nival = pos_idx + neg_idx\nNUM_TRAIN_SAMPLES = num_pos_all + num_neg_all\n\nwhile ival * 1. / NUM_TRAIN_SAMPLES < 2.:\n    ival = pos_idx + neg_idx\n    x_train, y_train = generate_training_batch(pos_idx, neg_idx)\n    model.fit(x_train, y_train,\n          batch_size=128,\n          epochs=1,\n          validation_data=(x_dev, y_dev))\n    ival += SUPER_BATCH_SIZE\n    pos_idx += num_pos\n    neg_idx += num_neg\n    print(ival * 1. / NUM_TRAIN_SAMPLES)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"73ce702032b8b5b623194491f0f277dcd679e6bb"},"cell_type":"code","source":"y_val_pred = model.predict(x_dev)[:,1]\ny_val_bin = y_dev[:,1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ccca8c1ff9563500514f103ef755ef1962095ae7"},"cell_type":"code","source":"def precision(pred, true, thresh):\n    n_true = sum(1 for p, t in zip(pred, true) if p > thresh and t == 1)\n    n_fire = sum(1 for p, t in zip(pred, true) if p > thresh)\n    return n_true * 1. / n_fire if n_fire > 0 else 1.\n\ndef recall(pred, true, thresh):\n    n_true = sum(1 for p, t in zip(pred, true) if p > thresh and t == 1)\n    n_all = sum(1 for p, t in zip(pred, true) if t == 1)\n    return n_true * 1. / n_all if n_all > 0 else 1.\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d26d869a562a8163b8b1bde910056757669904e"},"cell_type":"code","source":"thresh = [i * 0.01 for i in range(101)]\nprecision_list = [precision(y_val_pred, y_val_bin, t) for t in thresh]\nrecall_list = [recall(y_val_pred, y_val_bin, t) for t in thresh]\nf_measure = [2. * p * r / (p + r) for p, r in zip(precision_list, recall_list)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"24e0ef9f63a8ef76ec67444ab1c9e929e6f546a2"},"cell_type":"code","source":"fval, tval = max(zip(f_measure, thresh))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"935fb62b968fbd68c4d33129d3341fdc5faa20ed"},"cell_type":"code","source":"fval, tval","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"df_test = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b5ea5205557a88b0fc25ad90940fde47825bfb9"},"cell_type":"code","source":"new_vocab = set(w for row in df_test.itertuples() for w in clean_str(row.question_text.strip()).split() if w not in embeddings)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"02ac6b6aaa6426d2c05f83a482f87f4382f4d9dc"},"cell_type":"code","source":"len(new_vocab)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8dbadfbb2b5b6d3ef1b9c58bf7ccc4558fd1a0f"},"cell_type":"code","source":"def add_to_embedding(fname, new_vocab, embeddings):\n    for i, line in enumerate(open(fname, 'r')):\n        line_split = line.split(' ')\n        word = line_split[0]\n        if word in new_vocab:\n            vec = np.array([float(s) for s in line_split[1:]], dtype='float32')\n            embeddings[word] = vec\n            \nadd_to_embedding(fname, new_vocab, embeddings)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8252628355d79e86143e489a61192bcdc728a709"},"cell_type":"code","source":"def predict(row):\n    qid = row.qid\n    question = clean_str(row.question_text.strip())\n    x = np.zeros((1, MAX_SENTENCE_WIDTH, EMBED_DIM))\n    x[0, :, :] = generate_features(question, embeddings)\n    y = 1 if model.predict(x)[0,1] > tval else 0\n    return (qid, y)\n\ntups = [predict(row) for row in df_test.itertuples()]\ndf_pred = pd.DataFrame(tups, columns=['qid', 'prediction'])\ndf_pred.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f67bd69a928fe0136b3aa6a4ec0b53e5dcde0245"},"cell_type":"code","source":"fval","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"017f04c6074415b04fd90e3e533c248890afb0cb"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}