{"cells":[{"metadata":{"trusted":true,"_uuid":"2ec728c59179c88d137def42f6be820f4a5af10b"},"cell_type":"markdown","source":"thanks https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings"},{"metadata":{"_uuid":"ee0909ee0e4552d7d07a8f6e88a3204b4e85e763"},"cell_type":"markdown","source":"**Import libraries** and **Data Frame**"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport time\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D,CuDNNLSTM\nfrom keras.layers import Bidirectional, GlobalMaxPool1D\nfrom keras.models import Model\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nimport re","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"072ab54f9d4af4799d22b96269f3d8777c6292c0"},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport gc\n\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.metrics import log_loss,confusion_matrix,classification_report,roc_curve,auc\nfrom sklearn.model_selection import train_test_split\nimport string\nimport nltk\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\nfrom scipy import sparse\n%matplotlib inline\nseed = 42","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \",train_df.shape)\nprint(\"Test shape : \",test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a07e6728c964a426d2c44a643c268a2dd0b82ca7"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f624bf3113f368aa31ddf7687c3091e4a8d3ca03"},"cell_type":"markdown","source":"## Data Cleaning"},{"metadata":{"trusted":true,"_uuid":"24bbd466cb7f80cbd7ffa23611f6a2eed69d2261"},"cell_type":"code","source":"\ndef clean_text(x):\n    #translator = str.maketrans('', '', string.punctuation)\n    x = x.lower()\n    #return x.translate(translator)\n    x = re.sub(r\"[^\\sa-zA-Z']\", \"\", x)\n    return x\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a9b8c9f3e74fafd712c74bb6b13abd4402efb990"},"cell_type":"code","source":"#train_df['question_text']= train_df['question_text'].apply(lambda x: clean_text(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e5a926cfb70a0f46786f3f8f397574ab62fac382"},"cell_type":"code","source":"#test_df['question_text']=test_df['question_text'].apply(lambda x: clean_text(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b130c138da217b8a446d9912ae7da70fb9a68148"},"cell_type":"code","source":"## split to train and val\ntrain_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)\n\n## some config values \nembed_size = 300 # how big is each word vector\nmax_features = 90000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 100 # max number of words in a question to use\n\n## fill up the missing values\ntrain_X = train_df[\"question_text\"].fillna(\"_na_\").values\nval_X = val_df[\"question_text\"].fillna(\"_na_\").values\ntest_X = test_df[\"question_text\"].fillna(\"_na_\").values\n\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\n## Pad the sentences \ntrain_X = pad_sequences(train_X, maxlen=maxlen)\nval_X = pad_sequences(val_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)\n\n## Get the target values\ntrain_y = train_df['target'].values\nval_y = val_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78a5a6fee5089403688556c6df14080d9c1bbf84"},"cell_type":"code","source":"#from imblearn.over_sampling import SMOTE\n#train_X, train_y = SMOTE().fit_resample(train_X, train_y)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef3f21140572f5b4481eb0426019c3920c12b829"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12a21a717660695ac0e5958e5b64dee5b4bed05b"},"cell_type":"code","source":"#val_X, val_y = SMOTE().fit_resample(val_X, val_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e859efe9ac4c504bf4ef72e0adfc8a2487c5ced"},"cell_type":"code","source":"inp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size)(inp)\nx = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(16, activation=\"relu\")(x)\nx = Dropout(0.1)(x)\nx = Dense(1, activation=\"sigmoid\")(x)\nmodel = Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n\nprint(model.summary())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2c9bf4b1098064f53ab4c7906b10f326908b180"},"cell_type":"code","source":"from sklearn.utils import class_weight","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a231f35a47be9c4e2e32dec7197a139ffcbdccb3"},"cell_type":"code","source":"class_weights = class_weight.compute_class_weight('balanced',np.unique(train_y),train_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aad1e679c3364e217339ca66d74f3595b28dfd4b"},"cell_type":"code","source":"class_weight_dict = dict(enumerate(class_weights))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"27d6bd017d3bdeff32c313b804482df84f1afe5a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b3fb0f545dd9e8e7af706a401fe462d1c810392"},"cell_type":"code","source":"history = model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))#, class_weight=class_weight_dict)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f4f1475f5919ac342eac80808f13a63b01ebfe7c"},"cell_type":"markdown","source":"Plot Model history"},{"metadata":{"trusted":true,"_uuid":"37c8fe2c355cade79980fdedfd00312260244707"},"cell_type":"code","source":"# Get training and test loss histories\ntraining_loss = history.history['loss']\ntest_loss = history.history['val_loss']\n\n# Create count of the number of epochs\nepoch_count = range(1, len(training_loss) + 1)\n\n# Visualize loss history\nplt.plot(epoch_count, training_loss, 'r--')\nplt.plot(epoch_count, test_loss, 'b-')\nplt.legend(['Training Loss', 'Test Loss'])\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c83e3d599209c200b65532e913045b66bcfc2315"},"cell_type":"code","source":"#val_X_n = val_df[\"question_text\"].fillna(\"_na_\").values\n#val_X_n = tokenizer.texts_to_sequences(val_X_n)\n#val_X_n = pad_sequences(val_X_n, maxlen=maxlen)\n#val_y_n = val_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"350a5ab8d29bbca4641463bd7624df077f175368"},"cell_type":"code","source":"pred_noemb_val_y = model.predict([val_X], batch_size=1024, verbose=1)\nfor thresh in np.arange(0.1, 0.701, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_noemb_val_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7a9f24b87fdff56b565da689c642871a5252efa4"},"cell_type":"code","source":"pred_nval_y = (pred_noemb_val_y>0.35).astype(int)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf7a24bec9a7b79180cf1881bb1aa71e61d0973b"},"cell_type":"code","source":"pred_nval_y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"66490f9530dd735da561197ee0852db8d0bb0579"},"cell_type":"code","source":"from sklearn import metrics","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47f80b7965dd0fd1f9775071ab651aecbc1eea14"},"cell_type":"code","source":"print(metrics.classification_report(pred_nval_y, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75fc404388c5fec526d8efa84f2617b548f45d10"},"cell_type":"code","source":"print(metrics.accuracy_score(pred_nval_y, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a03c42328ab060a3b6509accf5cdca733b20ea1"},"cell_type":"code","source":"print(metrics.f1_score(pred_nval_y, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc00faa86b79d005132550558c6a33171f4be711"},"cell_type":"code","source":"print(metrics.recall_score(pred_nval_y, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6954ba395846edeafb0224b895479b8a4f17ece8"},"cell_type":"code","source":"confusion_matrix= metrics.confusion_matrix(pred_nval_y, val_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7319793dc7188958ec06eaafecbf1aecd9e149ab"},"cell_type":"code","source":"confusion_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d6a62ea66fd45edf13cb421d36208c1ed68257d6"},"cell_type":"code","source":"sns.heatmap(confusion_matrix, annot=True, cmap=sns.color_palette(\"Paired\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"443cbc83e363e33ea36fb5e9f68fede5dcab468b"},"cell_type":"code","source":"pred_noemb_test_y = model.predict([test_X], batch_size=1024, verbose=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba2e015bca20c70c6f459a751175a678f7e67f25"},"cell_type":"code","source":"pred_test_y = (pred_noemb_test_y>0.35).astype(int)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2f58d8c5bb130bc4b62919e9abba1a5517124c6e"},"cell_type":"markdown","source":"## With Glove Embedding"},{"metadata":{"trusted":true,"_uuid":"4ae135963209f63b1ea6f354101c3a1dba8289f7"},"cell_type":"code","source":"!ls ../input/embeddings/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d041d9d33dc029be5e34869b2a2ce3ed24fcbe5"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"10dbe344a7cdcd98a5c698bc6ad93a957e12b510"},"cell_type":"code","source":"inp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\nx = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(16, activation=\"relu\")(x)\nx = Dropout(0.1)(x)\nx = Dense(1, activation=\"sigmoid\")(x)\nmodel = Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0e14cc2c7bf146332e620334ff2417cb94f1f37a"},"cell_type":"code","source":"history=model.fit(train_X, train_y, batch_size=512, epochs=5, validation_data=(val_X, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9796e8da34d7dd849726a31de9bbab8164fba091"},"cell_type":"code","source":"# Get training and test loss histories\ntraining_loss = history.history['loss']\ntest_loss = history.history['val_loss']\n\n# Create count of the number of epochs\nepoch_count = range(1, len(training_loss) + 1)\n\n# Visualize loss history\nplt.plot(epoch_count, training_loss, 'r--')\nplt.plot(epoch_count, test_loss, 'b-')\nplt.legend(['Training Loss', 'Test Loss'])\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3b00e4af8ec13cfaa26b5aded5f7127a58ac9a23"},"cell_type":"markdown","source":"change epoch to 2"},{"metadata":{"trusted":true,"_uuid":"ecd424753a4c7cae4765ca51815b826df2b894e6"},"cell_type":"code","source":"inp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\nx = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(16, activation=\"relu\")(x)\nx = Dropout(0.1)(x)\nx = Dense(1, activation=\"sigmoid\")(x)\nmodel = Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"92d1b138fbf4e8f77a923165af603e633fffa8a4"},"cell_type":"code","source":"history=model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"343e05dc42aaf787b8ae3318352f1857e68f641b"},"cell_type":"code","source":"# Get training and test loss histories\ntraining_loss = history.history['loss']\ntest_loss = history.history['val_loss']\n\n# Create count of the number of epochs\nepoch_count = range(1, len(training_loss) + 1)\n\n# Visualize loss history\nplt.plot(epoch_count, training_loss, 'r--')\nplt.plot(epoch_count, test_loss, 'b-')\nplt.legend(['Training Loss', 'Test Loss'])\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"04e5197c8839f8d1f8843f26d0ceacbe5720bbcd"},"cell_type":"code","source":"pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)\nfor thresh in np.arange(0.01, 0.501, 0.01):\n    thresh = np.round(thresh, 3)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_glove_val_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7a4ddfaad132fb11aebf7d6bd7e6e44a8e9a06e"},"cell_type":"code","source":"pred_glove_val_y = (pred_glove_val_y>0.37).astype(int)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d8fbf27d8937ca1dbb8cdcc9b56ffb5bbedf207"},"cell_type":"code","source":"print(metrics.classification_report(pred_glove_val_y, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1008f3ae3329a56e45deb1b36facb8b800636ca7"},"cell_type":"code","source":"print(metrics.accuracy_score(pred_glove_val_y, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be36944935b252d935507ecbda02bb46648b5eac"},"cell_type":"code","source":"print(metrics.f1_score(pred_glove_val_y, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4da14fad7766408c26097403ea8f772da2252c34"},"cell_type":"code","source":"print(metrics.recall_score(pred_glove_val_y, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e579264e601d267ef2b52c11e13f844eb55e7e23"},"cell_type":"code","source":"confusion_matrix_glove= metrics.confusion_matrix(pred_glove_val_y, val_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"04c56e9a0aea85b011cd470d758226aa0f5de36c"},"cell_type":"code","source":"sns.heatmap(confusion_matrix_glove, annot=True, cmap=sns.color_palette(\"Paired\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f13f5168b3f337a09c648923c8b496cf5c7efaf4"},"cell_type":"code","source":"pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aabe88c8f2a1a1e1b309543378aee5843e5babc2"},"cell_type":"code","source":"del model, inp, x\nimport gc; gc.collect()\ntime.sleep(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2a0e08d11358c991add71d1eb7e666acbf5f837e"},"cell_type":"code","source":"pred_glove_test_y = (pred_glove_test_y>0.33).astype(int)\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)\n\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}