{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nimport string\nfrom nltk.corpus import stopwords\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom nltk.stem.wordnet import WordNetLemmatizer\nfrom sklearn.model_selection import train_test_split\nprint(os.listdir(\"../input\"))\n\n\nstopword = stopwords.words(\"english\")\nlemma = WordNetLemmatizer()\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"424608b985a00403bbf84d16b8545727935472b1"},"cell_type":"code","source":"!pip install hyperas","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ffe3c6a235c70d7effe841f6590d2c0638e4088"},"cell_type":"code","source":"print(\"Train dataset shape: \", train_df.shape)\nprint(\"Test dataset shape: \", test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d1b390ff2952d86a247d34a8f3c6fedef597e4e4"},"cell_type":"code","source":"train_df.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"70462b1ebd83e87215a46c2137693bd1d1130135"},"cell_type":"code","source":"print(train_df.duplicated(subset=['question_text']).sum())\nprint(test_df.duplicated(subset=['question_text']).sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8afc765e7e6f812230c5d8718709263ad95bdec9"},"cell_type":"code","source":"(train_df['target'].value_counts()/train_df.shape[0]).plot(kind='bar')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f98a6ac6f12ee6b5539ceaa9c363c00488a11797"},"cell_type":"code","source":"train_df['qlen'] = train_df['question_text'].apply(lambda x: len(x.split()))\ntest_df['qlen'] = test_df['question_text'].apply(lambda x: len(x.split()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18304a4f6e1c273aac59695bd84cc45a13229649"},"cell_type":"code","source":"plt.figure(figsize=(15,8))\nsns.boxplot(y='qlen',x='target', data=train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cbb3445f8650fc7301b2b341c3a7c8d077cdee82"},"cell_type":"code","source":"#Remove the outlier and plot again to get better visualization\ntrain_df = train_df[train_df['qlen']<100]\n\nplt.figure(figsize=(15,8))\nsns.boxplot(y='qlen',x='target', data=train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"565222b6b7023271cb20765751b7cd7b0e1c3820"},"cell_type":"code","source":"from tqdm import tqdm\n\nglove_embedding = \"../input/embeddings/glove.840B.300d/glove.840B.300d.txt\"\nfile = open(glove_embedding)\nembedding_index={}\nfor line in tqdm(file):\n    split_values = line.split(\" \")\n    #print(line.split(\" \")[0], line.split()[1:])\n    embedding_index[split_values[0]] = np.asarray(split_values[1:], dtype='float32')\nfile.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b02f111567add120ad783d0e66f2fe63051e2f60"},"cell_type":"code","source":"def cleanup(text):\n    text = \"\".join([char for char in text if (ord(char)>=48 and ord(char)<=57) or (ord(char)>=65 and ord(char)<=90) or (ord(char)>=97 and ord(char)<=122) or (ord(char)==32) ])\n    #text = \" \".join([lemma.lemmatize(word.lower()) for word in text.split() if word.lower() not in stopword if len(word)>2])\n    text = \" \".join(word if (word in embedding_index) and (word.lower() not in embedding_index) else word.lower() for word in text.split())\n    return text\n\ntrain = train_df.copy()\ntrain['question_text'] = train['question_text'].apply(cleanup)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"678c3a82507cc8ac33eca2cb1a60ebf461bfb938"},"cell_type":"code","source":"train['qlen_wostopwords'] = train['question_text'].apply(lambda x: len(x.split()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd11ddb11ce98f4d85f96aa6bff0f952d3e63867"},"cell_type":"code","source":"plt.figure(figsize=(15,8))\nsns.boxplot(y='qlen_wostopwords',x='target', data=train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6c78a996ee37c6a3f6d394c5a2455b4b632c2209","scrolled":true},"cell_type":"code","source":"train.groupby(by=['target']).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"248e10ae04d381741c4f5770a48d80b4fcbf94c7"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de470034b032f0e77108f4cfd6fb64cede71bdd8"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d5726445545e602fa7a2d07ab0707c08448c0c7"},"cell_type":"code","source":"print('Max words in train dataset: ', train.qlen_wostopwords.max())\n#maxlen = train.qlen_wostopwords.max()+2\nmaxlen=100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3fd7fb74b26cc276c709b16bffbbbf4c496ddc93"},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(train['question_text'],train['target'], test_size=0.2, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"97dabbed52a0b47f0d2ac62839965d3e5bd02e1f"},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\n\ntokenizer = Tokenizer(num_words=50000)\ntokenizer.fit_on_texts(list(X_train))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c8909a8bcddbf959c2678fc5129167b10b6ef4c"},"cell_type":"code","source":"X_train = tokenizer.texts_to_sequences(X_train)\nX_test = tokenizer.texts_to_sequences(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ac0dbed779b73f5b8802469fef1157e4f05b81a"},"cell_type":"code","source":"from keras.preprocessing.sequence import pad_sequences\nX_train = pad_sequences(X_train,maxlen=maxlen)\nX_test = pad_sequences(X_test,maxlen=maxlen)\ny_train = y_train.values\ny_test = y_test.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31d54d02e4ef85ca28df59ca81a1b08bfa8422df"},"cell_type":"code","source":"#np.stack(embedding_index.values()).shape\nembedding_matrix = np.zeros((50000,300))\nfor word,i in tokenizer.word_index.items():\n    if i>=50000: continue\n    v = embedding_index.get(word)\n    if v is not None: embedding_matrix[i]=v","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bdb47bf2175572a1c967bb68e560a43aafb478ff"},"cell_type":"code","source":"from keras import backend as K\n\ndef f1(y_true, y_pred):\n    def recall(y_true, y_pred):\n        \"\"\"Recall metric.\n\n        Only computes a batch-wise average of recall.\n\n        Computes the recall, a metric for multi-label classification of\n        how many relevant items are selected.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        \"\"\"Precision metric.\n\n        Only computes a batch-wise average of precision.\n\n        Computes the precision, a metric for multi-label classification of\n        how many selected items are relevant.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c9ff377e39519c6be48e2eb7e4f9e7fed09a7ba5"},"cell_type":"code","source":"from keras.layers import Dense, Input, LSTM, Embedding, Dropout, CuDNNLSTM\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, GlobalAveragePooling1D, concatenate\nfrom keras.models import Model\n#from keras import initializers, regularizers, constraints, optimizers, layers\n\ninp = Input(shape=(maxlen,))\nx = Embedding(50000, 300, weights=[embedding_matrix])(inp)\nx = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)\nmax_pool = GlobalMaxPool1D()(x)\navg_pool = GlobalAveragePooling1D()(x)\nconc = concatenate([max_pool,avg_pool])\nx = Dense(16, activation=\"relu\")(conc)\nx = Dropout(0.1)(x)\nx = Dense(1, activation=\"sigmoid\")(x)\nmodel = Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=[f1])\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8f760ec1383652dce2e2a5ab0bf410cb8d4aef9"},"cell_type":"code","source":"model.fit(X_train,y_train,validation_data=(X_test,y_test),epochs=3,batch_size=1024)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90a013ed978b994364e45a87507651e2da0a3afa"},"cell_type":"code","source":"pred = model.predict(X_test)\n\nfrom sklearn.metrics import accuracy_score\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f2977f30ad4b470753781f01794a876aa548c55"},"cell_type":"code","source":"from sklearn.metrics import f1_score, recall_score\n\nthresh=0.1\nprev_score=0\nfor i in np.arange(0.1,0.41,0.01):\n    score = f1_score(y_test, (pred>i))\n    if score>prev_score: thresh=i\n    prev_score=score\n    print('threshold at ', i, score)\n#recall_score(y_test, (pred>0.3))\nprint('Best threshold:  ', thresh, ' f1score: ', f1_score(y_test, (pred>thresh)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2befe860be1cdfc4d64caf5fa2436aefdb2fbbe2"},"cell_type":"code","source":"test_df['question_text'] = test_df['question_text'].apply(cleanup)\ntest_X = tokenizer.texts_to_sequences(test_df['question_text'].values)\ntest_X = pad_sequences(test_X,maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"09e143f0f5e8606e0e17041c2f2135560629537f"},"cell_type":"code","source":"test_pred = model.predict(test_X)\ntest_pred = (test_pred>thresh).astype('int')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a9f3a4c73d40168498502c63cb2d6a7ed30030e6"},"cell_type":"code","source":"pd.DataFrame({'qid':test_df['qid'].values, 'prediction':np.squeeze(test_pred)}).to_csv('submission.csv',index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"38b09b14ebd3bdef62a2d9e828ae9d58085065c7"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}