{"cells":[{"metadata":{"_uuid":"4a030df26f181aa56a57533712d49ad19d3bed55"},"cell_type":"markdown","source":"# Using Randomized search CV with Keras"},{"metadata":{"_uuid":"bfc80f7f948992cb2c789301a15134c6648af9a1"},"cell_type":"markdown","source":"The objectif of this Kernel is to use Randomized search cv to test different parameter and architecture for a GRU  \n\nThis kernel is inspired by :\n* Miha Skalic kernel : LSTM is all you need! well, maybe embeddings also.\n* Shujian Liu Discussion topic :  3 Methods to combine embeddings\n* SRK kerne : A look at different embeddings.!"},{"metadata":{"_uuid":"fc355d72054822571c1d44f3ec5be7d675fdb4f2"},"cell_type":"markdown","source":"## Import of libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"scrolled":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport matplotlib as mp\nimport matplotlib.pyplot as plt\nimport os\nfrom sklearn.metrics import confusion_matrix,f1_score\nfrom sklearn.model_selection import GridSearchCV,RandomizedSearchCV,cross_validate,train_test_split,StratifiedKFold\nfrom datetime import datetime","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"881b898e2881f616e6f2d0a794d4b473ce3bd4ff"},"cell_type":"code","source":"#Import des librairies\nfrom keras.models import Sequential\nfrom keras.layers import *\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.models import Model\nfrom keras.wrappers.scikit_learn import KerasClassifier","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"449625e22659ebef3f222529c139d1777ff83e56"},"cell_type":"markdown","source":"## Data upload"},{"metadata":{"trusted":true,"_uuid":"89ef655f8e086eecc46e352732c93faa8a2e7147"},"cell_type":"code","source":"train=pd.read_csv(\"../input/train.csv\",sep=',')\ntest=pd.read_csv(\"../input/test.csv\",sep=',')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"335e82f49679d9744925a1a9b1055bab6d084d27"},"cell_type":"markdown","source":"##  Preprocessing"},{"metadata":{"trusted":true,"_uuid":"d6744f22e6fbc7837f1cbff299264eeda264f10a"},"cell_type":"code","source":"#Setting param\nvocabulary_size=50000 #number of word used in encoding\nvec_size=70 #size of encoded question\nemb_size=300 #size of emberdding","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9c47ae765d2cdd137bf14814b27f5d17b2b33bb4","scrolled":true},"cell_type":"code","source":"#Tokenizer hot encode the corpus \ntokenizer = Tokenizer(num_words=vocabulary_size)\ntokenizer.fit_on_texts(train.question_text)\n#Transforming question_text to a sequences of hot encoded word \nsequences = tokenizer.texts_to_sequences(train.question_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"032c8f8d62b50f3f754d7338c033cf0407859663"},"cell_type":"code","source":"#We can get the encoding of trump\ntokenizer.word_index.get('trump')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"348ca6cf7c405f8651d0f70567986b007f712850","scrolled":true},"cell_type":"code","source":"#Tokenizer can also give word counts \ntokenizer.word_counts.get('trump')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1bbdbcef79c713404d99a0df6252f4850fd98ec9"},"cell_type":"code","source":"#let see the encoding of the first question\nsequences[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8a4f458b65ac8beaaa0531c8ea90c9c7f6cb0d2b"},"cell_type":"code","source":"#pad_sequences set the size of question vector to vec_size\nX = pad_sequences(sequences, maxlen=vec_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c70575233d2ab68937bfa39c9f91730ce3089d19"},"cell_type":"code","source":"X[0]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e987f9e67677a24ce25335458d8b1433d969b532"},"cell_type":"markdown","source":"## Embedding"},{"metadata":{"trusted":true,"_uuid":"9fc738502a07e63c9cfe2b40672f9ec2fd40c344"},"cell_type":"code","source":"#get_emb_index is function how take the file path and get the embedding index\ndef get_emb_index (filepath) :\n    embeddings_index = dict()\n    f = open(filepath,errors='ignore')\n    for line in f:\n        if len(line)>100 :\n            values = line.split(\" \")\n            #the first value of the line is the word\n            word = values[0]\n            #the rest of values of the line are the coef\n            coefs = np.asarray(values[1:], dtype='float32')\n            embeddings_index[word] = coefs\n    f.close()\n    return embeddings_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"61e81dd129aceed29b4b4103193e22c646d4763e"},"cell_type":"code","source":"#get_emb_matrix is function how take the a embedding index and get a coef matrix for Tokenizer corpus\ndef get_emb_matrix(embeddings_index) :\n    #calculating the mean and the std of embedding index\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    emb_mean,emb_std\n    #random initialization of the matrix\n    embedding_matrix = np.random.normal(emb_mean, emb_std,(vocabulary_size, emb_size))\n    #for each word in tokenizer corpus with index < vocabulary index we search coef in the embedding index\n    for word, index in tokenizer.word_index.items():\n        if index > vocabulary_size - 1:\n            break\n        else:\n            embedding_vector = embeddings_index.get(word)\n            if embedding_vector is not None:\n                embedding_matrix[index] = embedding_vector\n    embedding_matrix= (embedding_matrix-emb_mean)/emb_std\n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8e4e9594497af636895407ef55d241202bf6a3c"},"cell_type":"code","source":"%%time\n#glove embeddding\nglove_emb_index=get_emb_index('../input/embeddings/glove.840B.300d/glove.840B.300d.txt')\nglove_emb_matrix= get_emb_matrix(glove_emb_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7128117c5e2237ae7a4a65976e48b8736f085fdb","scrolled":true},"cell_type":"code","source":"%%time \n#wikinews embedding\nwiki_emb_index=get_emb_index('../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec')\nwiki_emb_matrix= get_emb_matrix(wiki_emb_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"81582832906c4566949bee27a118cd2f5ab16d5d"},"cell_type":"code","source":"%%time\n#paragram embedding\nparagram_emb_index=get_emb_index('../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt')\nparagram_emb_matrix= get_emb_matrix(paragram_emb_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b760721038390d33846dd75213760233bfb4577"},"cell_type":"code","source":"#let see the coef of trump in different embedding\npd.DataFrame({'glove':glove_emb_index.get('trump'),'paragram' : paragram_emb_index.get('trump'),'wikiNews':wiki_emb_index.get('trump')}).T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dae44a2980a9712de9c9b1b11913fe70e7d7765b"},"cell_type":"code","source":"#average of embedding matrix\navg_emb_matrix =(glove_emb_matrix+paragram_emb_matrix+wiki_emb_matrix)/3","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3e309e14c41573d5690538f4ec7fa125eb4d857f"},"cell_type":"markdown","source":"### GRU model "},{"metadata":{"trusted":true,"_uuid":"bb66d0b5805dcc0a80ca50d437d9095de1b5e17d"},"cell_type":"code","source":"%%time\n#Our model : Embedding -> Spatialdropout -> CuDNNGRU -> GlobalMaxPool1D-> Dense -> Dropout -> Dense\ndef get_model_gru(dropout=0.2,gru_units=64,dense_units=16,kernel_initializer='he_normal', embedding='avg') :\n    inp = Input(shape=(vec_size,))\n    if embedding =='avg':\n        x = Embedding(vocabulary_size, emb_size, weights=[avg_emb_matrix])(inp)\n    elif embedding=='glove' :\n        x = Embedding(vocabulary_size, emb_size, weights=[glove_emb_matrix])(inp)\n    elif embedding=='paragram':\n        x = Embedding(vocabulary_size, emb_size, weights=[paragram_emb_matrix])(inp)\n    elif embedding =='wiki':\n        x = Embedding(vocabulary_size, emb_size, weights=[wiki_emb_matrix])(inp)\n    x = SpatialDropout1D(dropout)(x)\n    x = Bidirectional(CuDNNGRU(gru_units, return_sequences=True))(x)\n    x = GlobalMaxPool1D()(x)\n    x = Dense(dense_units,kernel_initializer=kernel_initializer, activation=\"relu\")(x)\n    x = Dropout(dropout)(x)\n    x = Dense(1, kernel_initializer=kernel_initializer,activation=\"sigmoid\")(x)\n    model = Model(inputs=inp, outputs=x)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cef5d3fff229c09ecd71a949c868af26143d87b8"},"cell_type":"markdown","source":"## Randomized search CV"},{"metadata":{"trusted":true,"_uuid":"9dca21c152e53780af103e1969234d6d51e726d8"},"cell_type":"code","source":"#The grid for Randomized search CV\nparam = {\n    'dropout': [0.1,0.2,0.5],\n    'gru_units': [16, 32, 64],\n    'dense_units': [16, 32, 64],\n    'kernel_initializer': ['he_normal','uniform'],\n    #'kernel_initializer': ['he_normal'],\n    'embedding': ['avg','glove','paragram','wiki']\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e36f76ebbf537e0adf82f42740187b7579d8928"},"cell_type":"code","source":"#Before using Randomized search CV we have to specify the number of epochs and batch_size\nmodel = KerasClassifier(build_fn=get_model_gru, epochs=3, batch_size=2000,verbose=False)\nrnn_model=RandomizedSearchCV(model,n_iter=10, param_distributions=param,cv=3,scoring='f1',return_train_score=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"400aa5f9bcb71d8872ca869625285e4618ac3741"},"cell_type":"code","source":"%%time\n#rnn_model.fit(X[:1000],train.target[:1000])\nrnn_model.fit(X,train.target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f2c0d0c174df8bd8a279ee2c327885dc35fe2660"},"cell_type":"code","source":"#let's see the result\npd.DataFrame(rnn_model.cv_results_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c7301b3675b141ccace5875b50c568ee27ccf2df"},"cell_type":"code","source":"pred_train=rnn_model.predict(X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a754ab508a675fe3b4755624e08adf90144c4595"},"cell_type":"code","source":"#f1 score and confusion matrix for threshold to\nto=0.50\nprint('F1 score :%s \\n'% f1_score(train.target,pred_train>to) )\nprint('Confusion matrix \\n%s'%confusion_matrix(train.target,pred_train))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"365800ef77c7c3a1d5a04aeaf1c85d05f730044b"},"cell_type":"code","source":"#Analizing best model's errors\ntrain[train.target!=pred_train[:,0]]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8187e551cb2c7c262d816634520ce0b4f5f27295"},"cell_type":"markdown","source":"## Test data transformation"},{"metadata":{"trusted":true,"_uuid":"5c13363864d08e117c53f262ebfa98d079eec33f"},"cell_type":"code","source":"sequences_t = tokenizer.texts_to_sequences(test.question_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ecacbfd2d052523a33c1740f06652c877ee956b"},"cell_type":"code","source":"X_t= pad_sequences(sequences_t, maxlen=vec_size)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"501ebbb2e238eec409ce667ab8908772cc87b941"},"cell_type":"markdown","source":"## Prediction on test data  and submission"},{"metadata":{"trusted":true,"_uuid":"cf5b9babe64c18257f78ac90a75158165564e4f2"},"cell_type":"code","source":"result=pd.read_csv(\"../input/sample_submission.csv\",sep=',')\npred_res=rnn_model.predict(X_t)>to\nresult['prediction']=pd.DataFrame(pred_res)\nresult.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}