{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.\nfrom keras.models import Sequential\nfrom keras.layers import LSTM,Dense,Dropout,Embedding,CuDNNLSTM,Bidirectional\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\nimport numpy as np\nimport pandas as pd\nfrom keras.models import Model\nfrom keras.layers.convolutional import Conv1D\nfrom keras.layers import MaxPooling1D , GlobalMaxPooling1D","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!wget http://nlp.stanford.edu/data/glove.6B.zip","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#unzip the file \n!unzip glove*.zip","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#store 300 size vector representation different words from the file to a disctionary\nembedding_index = {}\nf = open('glove.6B.300d.txt',encoding='utf-8')\nfor line in f:\n  value = line.split()\n  word = value[0]\n  coeffs = np.asarray(value[1:],dtype = 'float32')\n  embedding_index[word] = coeffs\nf.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(embedding_index)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Reading the dataset"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"df = pd.read_csv('../input/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Displaying the count of each class in Y label**"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(df['target'].value_counts())\nsns.countplot(df['target'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#to check the proportion of the ones to zeros\ntarget_count = df['target'].value_counts()\n\nprint('Class 0:', target_count[0])\nprint('Class 1:', target_count[1])\nprint('Proportion:', round(target_count[0] / target_count[1], 2), ': 1')\n\ntarget_count.plot(kind='bar', title='Count (target)');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The above graph depicts that we have a imbalanced classess"},{"metadata":{"trusted":true},"cell_type":"code","source":"x = df['question_text']\ny = df['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"token = Tokenizer()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nq_train , q_test = train_test_split(df, test_size=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"q_train.shape , q_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = q_train['question_text']\ny_train = q_train['target']\nx_test  = q_test['question_text']\ny_test  = q_test['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train.shape , y_train.shape , x_test.shape , y_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.utils import class_weight\nclass_weights = class_weight.compute_class_weight('balanced',\n                                                 np.unique(y_train),\n                                                 y_train)\nclass_weights","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Converting the text into sequence for processing in LSTM Layers**"},{"metadata":{"trusted":true},"cell_type":"code","source":"token.fit_on_texts(x)\nseq = token.texts_to_sequences(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pad_seq = pad_sequences(seq,maxlen=300)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab_size = len(token.word_index)+1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x = df['question_text']\ny = df['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing import sequence\nfrom keras.layers import LSTM,Activation,Dense,Input,Embedding,Dropout\nfrom keras.models import Model\nfrom nltk import word_tokenize\nimport nltk\nnltk.download('punkt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sent_lens=[len(word_tokenize(x)) for x in x_train]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"max(sent_lens)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.percentile(sent_lens,95)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"max_len=31 #taking the 95% quantile value of the sentence length\n\ntk=Tokenizer(char_level=False,split=' ') # tokenizing the sentence \n\ntk.fit_on_texts(x_train)\n\nseq_train=tk.texts_to_sequences(x_train) # create tokens on train\nseq_test=tk.texts_to_sequences(x_test) # create tokens on test\n\nvocab_size=len(tk.word_index)\n\nseq_train_matrix=sequence.pad_sequences(seq_train,maxlen=max_len) #padding the sentence with 0 for matching length\nseq_test_matrix=sequence.pad_sequences(seq_test,maxlen=max_len)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seq_train_matrix.shape , seq_test_matrix.shape , vocab_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seq_train_matrix[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# creating our own embedding matrix to bring down the size to 300\n# we'll use 300 D vector representation of the words from pretrained embedding index \n# that we downloaded \n\nembedding_matrix=np.zeros((vocab_size+1,300))\n\nfor word,i in tk.word_index.items():\n    embed_vector=embedding_index.get(word)\n    if embed_vector is not None:\n        embedding_matrix[i]=embed_vector\n# if there are specific words which are not present in pretrained embedding \n# their weights will remain 0. if there are too many such words \n# then you should probably not use pretrained embeddings","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Converting the words in our Vocabulary to their corresponding embeddings and placing them in a matrix.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"inputs=Input(name='text_input',shape=[max_len])\nembed=Embedding(vocab_size+1,300,input_length=max_len,mask_zero=True,\n                weights=[embedding_matrix],trainable=False)(inputs)\n\nGRU_layer=GRU(50)(embed)\n\ndense1=Dense(10,activation='relu')(GRU_layer)\ndrop=Dropout(0.2)(dense1)\n\nfinal_layer=Dense(1,activation='sigmoid')(drop)\n\nmodel_GRU=Model(inputs=inputs,outputs=final_layer)\nmodel_GRU.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_GRU.compile(optimizer='adam',loss='binary_crossentropy',metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import ModelCheckpoint\nimport os\noutputFolder = './content/Model_output/'\nif not os.path.exists(outputFolder):\n    os.makedirs(outputFolder)\nfilepath = outputFolder+\"/weights-{epoch:02d}-{val_acc:.4f}.h5\"\ncheckpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, \n                             save_best_only=False, save_weights_only=True, \n                             mode='auto', period=1)\n# this will save the weights every 10 epoch\n\nfrom keras.callbacks import EarlyStopping\nearlystop = EarlyStopping(monitor='val_acc', min_delta=0.0001, patience=3,\n                          verbose=1, mode='auto')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_GRU.fit(seq_train_matrix,y_train,validation_data=[seq_test_matrix,y_test],epochs=10,class_weight={0:0.53,1:8},\n          batch_size=10000,callbacks=[earlystop,checkpoint])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"p=model_GRU.predict(seq_test_matrix)\nfrom sklearn.metrics import roc_auc_score\nroc_auc_score(y_test,p)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import classification_report,f1_score\nprint(f1_score(y_test,p >.50))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(classification_report(y_test,p>.50))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}