{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport math \n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nimport re\nimport operator\nfrom tqdm import tqdm\n\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix, f1_score, precision_score, recall_score\n\nfrom keras.models import Sequential, Model\nfrom keras.layers import CuDNNLSTM, Dense, Bidirectional, Dropout, Input, Embedding, GlobalMaxPool1D\nfrom keras.callbacks import Callback","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dcd39c828a3650b6b68f4c5a7ed9576f51ab80b9"},"cell_type":"code","source":"# Read in data, and take a look\ntrain = pd.read_csv('../input/train.csv')\ntrain, val = train_test_split(train, test_size = 0.2, random_state = 369)\ntest = pd.read_csv('../input/test.csv')\nprint(train.shape)\nprint(test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f0d155670ce868291e7254b968fce0454dc0628"},"cell_type":"code","source":"# Initial implementation to user \"theoviel\"\ndef load_embed(file):\n    def get_coefs(word,*arr): \n        return word, np.asarray(arr, dtype='float32')\n    \n    if file == '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec':\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file) if len(o)>100)\n    else:\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding='latin'))\n        \n    return embeddings_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"388b932c3dc054201afef0396c194cb90fb6924d"},"cell_type":"code","source":"glove = load_embed (\"../input/embeddings/glove.840B.300d/glove.840B.300d.txt\")\nprint(\"Done extracting GloVe embedding\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e73500d24e3739127cce6eafc1e5b5cbe5bef1cd"},"cell_type":"code","source":"\ndef build_vocab(text):\n    sentences = text.apply(lambda x: x.split()).values\n    vocab = {}\n    for s in sentences:\n        for w in s:\n            try:\n                vocab[w] = vocab[w] + 1\n            except KeyError:\n                vocab[w] = 1\n    return vocab\n\n# evaluation of how much coverage there is in the embeddings\ndef check_coverage(vocab, embeddings_index):\n    known_words = {}\n    unknown_words = {}\n    nb_known_words = 0\n    nb_unknown_words = 0\n    for word in vocab.keys():\n        try:\n            known_words[word] = embeddings_index[word]\n            nb_known_words += vocab[word]\n        except:\n            unknown_words[word] = vocab[word]\n            nb_unknown_words += vocab[word]\n            pass\n\n    print('Found embeddings for {:.2%} of vocab'.format(len(known_words) / len(vocab)))\n    print('Found embeddings for {:.2%} of all text'.format(nb_known_words / (nb_known_words + nb_unknown_words)))\n    unknown_words = sorted(unknown_words.items(), key=operator.itemgetter(1))[::-1]\n\n    return unknown_words","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dae3cb1a8f181f77d782785a40fd902581191de5"},"cell_type":"code","source":"# We will apply 2 changes to improve our embedding accuracy:\n# 1) Change things to lower case\n# 2) Removal special characters and punctuation\n\npunct = \"/-'?!.,#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~\" + '\"\"“”’' + '∞θ÷α•à−β∅³π‘₹´°£€\\×™√²—–&'\n\npunct_mapping = {\"‘\": \"'\", \"₹\": \"e\", \"´\": \"'\", \"°\": \"\", \"€\": \"e\", \"™\": \"tm\", \"√\": \" sqrt \", \"×\": \"x\", \"²\": \"2\", \"—\": \"-\", \"–\": \"-\", \"’\": \"'\", \"_\": \"-\", \"`\": \"'\", '“': '\"', '”': '\"', '“': '\"', \"£\": \"e\", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }\n\ndef unknown_punct(embed, punct):\n    unknown = ''\n    for p in punct:\n        if p not in embed:\n            unknown += p\n            unknown += ' '\n    return unknown\n\ndef clean_special_chars(text, punct, mapping):\n    for p in mapping:\n        text = text.replace(p, mapping[p])\n    \n    for p in punct:\n        text = text.replace(p, f' {p} ')\n    \n    specials = {'\\u200b': ' ', '…': ' ... ', '\\ufeff': '', 'करना': '', 'है': ''}  # Other special characters that I have to deal with in last\n    for s in specials:\n        text = text.replace(s, specials[s])\n    \n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"99bed19188c2a6689a605eec3bc89cbaa8d663e8"},"cell_type":"code","source":"#train['processed_question'] = train['question_text'].apply(lambda x: x.lower())\n#train['processed_question'] = train['processed_question'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))\ntrain['question_text'] = train['question_text'].apply(lambda x: x.lower())\ntrain['question_text'] = train['question_text'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9bf783f0d08bf400cffcab45bca998bea9b866f1"},"cell_type":"code","source":"vocab = build_vocab(train['question_text'])\noov_glove = check_coverage(vocab, glove)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bceae829829c004067aec7f0c78840400b997c17"},"cell_type":"code","source":"# Convert values to embeddings\ndef text_to_array(text):\n    empyt_emb = np.zeros(300)\n    text = text[:-1].split()[:30]\n    embeds = [glove.get(x, empyt_emb) for x in text]\n    embeds+= [empyt_emb] * (30 - len(embeds))\n    return np.array(embeds)\n\n# train_vects = [text_to_array(X_text) for X_text in tqdm(train[\"question_text\"])]\nval_vects = np.array([text_to_array(X_text) for X_text in tqdm(val[\"question_text\"][:3000])])\nval_y = np.array(val[\"target\"][:3000])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cccea5a8b082c70d6790a4e4030d73bf330b21dc"},"cell_type":"code","source":"# Preprocessing adapted user sudalairajkumar\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\n# some config values \nmax_features = 50000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 80 # max number of words in a question to use\n\n# Fill missing values\ntrain_X = train[\"question_text\"].fillna(\"_na_\").values\nval_X = val[\"question_text\"].fillna(\"_na_\").values\ntest_X = test[\"question_text\"].fillna(\"_na_\").values\n\n# Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\n# Pad sentences \ntrain_X = pad_sequences(train_X, maxlen=maxlen)\nval_X = pad_sequences(val_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)\n\n# Get target values\ntrain_y = train['target'].values\nval_y = val['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab0606e241fdac1b7adcf1c807627288e0febb88"},"cell_type":"code","source":"# Model following: https://www.kaggle.com/nikhilroxtomar/embeddings-cnn-lstm-models-lb-0-683/notebook\n\nall_embs = np.stack(glove.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = glove.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n        \n        \ninp = Input(shape=(maxlen,))\nx = Embedding(max_features, 300, weights=[embedding_matrix])(inp)\nx = Bidirectional(CuDNNLSTM(64, return_sequences = True))(x)\nx = Bidirectional(CuDNNLSTM(64, return_sequences = True))(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(32, activation = \"relu\")(x)\nx = Dropout(0.3)(x)\n#out = Dense(1, activation=\"sigmoid\")(x)\nx = Dense(1, activation = \"sigmoid\")(x)\nmodel = Model(inputs = inp, outputs = x)\nmodel.compile(loss='binary_crossentropy',\n              optimizer='adam',\n              metrics=['accuracy'])\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6c4993e5ef13d9de08da5e3e0c1ea6c93ebe590d"},"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3bdec40c7608c7dba2ebf1b8201ee06d72693643"},"cell_type":"code","source":"pred_val_y = model.predict([val_X], batch_size=1024, verbose=1)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, f1_score(val_y, (pred_val_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b0a8e43006d1e4dd354f014e3f41a9fe514e78bf"},"cell_type":"code","source":"pred_test_y = model.predict([test_X], batch_size=1024, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90ea1d11c24af01d714ac7c2d4b61752937bbc6f"},"cell_type":"code","source":"pred_test_y = (pred_test_y>0.35).astype(int)\nout_df = pd.DataFrame({\"qid\":test[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}