{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from __future__ import division\nimport os\nimport random\nimport numpy as np\nimport tensorflow as tf\nfrom keras import backend as K\n\nimport pandas as pd\n\nfrom keras.layers import Input, Dropout, Dense, concatenate,  Embedding, Flatten, Activation, CuDNNLSTM,  Lambda\nfrom keras.layers import Conv1D, Bidirectional, SpatialDropout1D, BatchNormalization, multiply\nfrom keras.layers import GlobalMaxPooling1D, GlobalAveragePooling1D\nfrom keras import optimizers, callbacks, regularizers\nfrom keras.models import Model\n\n\nfrom collections import defaultdict\nfrom sklearn.feature_extraction.text import CountVectorizer\n\nfrom sklearn.metrics import log_loss\n\nimport re\n\nimport gc\nimport time\nimport nltk\n\nfrom nltk.stem import WordNetLemmatizer\nwordnet_lemmatizer = WordNetLemmatizer()\nfrom nltk.corpus import stopwords\nstop_words = set(stopwords.words('english'))\n\nfrom keras.preprocessing.text import Tokenizer\nfrom sklearn.model_selection import train_test_split\n\nimport matplotlib.pyplot as plt\n\nPATH = '../input/'\nEMBEDDINGS_PATH = '../input/embeddings/'\nWEIGHTS_PATH = './w0.h5'\nMAX_TEXT_LENGTH = 40\nEMBEDDING_SIZE  = 300\n\n\ndef embeddingNN(data, use_glove=True, trainable=True, seed=42):                                             \n    np.random.seed(seed)\n\n    emb_inpt = Input( shape=[data.shape[1]], name='emb_inpt')  \n    if use_glove:\n        x = Embedding(len( encoding_dc )+1, EMBEDDING_SIZE,\n                      weights=[embedding_weights], trainable=trainable) (emb_inpt)      \n    else:\n        x = Embedding(len( encoding_dc )+1, EMBEDDING_SIZE) (emb_inpt)      \n    \n    x = CuDNNLSTM(64, return_sequences=False) (x)   \n  \n    x= Dense(1)(x)\n    x = Activation('sigmoid')(x)\n    \n    model = Model([emb_inpt],x)\n\n    return model\n\n\ndef run_model(lr=1e-3, bs=2048, use_glove=False, trainable=True):    \n    predictions_test   = pd.DataFrame()\n    predictions_train  = pd.DataFrame()\n    for seed in range(3):\n        es = callbacks.EarlyStopping( patience=2 )\n        mc = callbacks.ModelCheckpoint( filepath=WEIGHTS_PATH, monitor='val_loss', mode='min', save_best_only=True )\n\n        model = embeddingNN(X_test_emb, use_glove=use_glove, trainable=trainable, seed=seed)\n        \n        optimizer = optimizers.Adam(lr=lr)\n        model.compile(loss=\"binary_crossentropy\", optimizer=optimizer)\n\n        model.fit(  X_train_emb, y_train, validation_data=(X_test_emb, y_test), callbacks=[es, mc],\n                     batch_size=bs, epochs=1000, verbose=2 )\n\n        model.load_weights(WEIGHTS_PATH)\n\n        p = model.predict(X_test_emb, batch_size=4096)\n        predictions_test[str(seed)] = p.flatten()\n\n        p = model.predict(X_train_emb, batch_size=4096)\n        predictions_train[str(seed)] = p.flatten()\n\n        print ( 'BAGGING SCORE Test: ' , log_loss(y_test,  predictions_test.mean(axis=1), eps = 1e-7) )\n        print ( 'BAGGING SCORE Train: ', log_loss(y_train, predictions_train.mean(axis=1), eps = 1e-7) )\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"full_data = pd.read_csv(PATH+'train.csv',  encoding='utf-8', engine='python')\nfull_data['question_text'].fillna(u'unknownstring', inplace=True)\n\nprint (full_data.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess( x ):\n    x = re.sub( u\"\\s+\", u\" \", x ).strip()\n    x = x.split(' ')[:MAX_TEXT_LENGTH]\n    return ' '.join(x)\n\n\nX_train, X_test, y_train, y_test = train_test_split(  full_data.question_text.values, full_data.target.values, \n                                                    shuffle =True, test_size=0.5, random_state=42)\n\nX_train = np.array( [preprocess(x) for x in X_train] )\nX_test  = np.array( [preprocess(x) for x in X_test] )\n\nprint ( X_train.shape, X_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word_frequency_dc=defaultdict(np.uint32)\ndef word_count(text):\n    text = text.split(' ')\n    for w in text:\n        word_frequency_dc[w]+=1\n\nfor x in X_train:\n    word_count(x) \n\nencoding_dc = dict()\nlabelencoder=1\nfor key in word_frequency_dc:\n    if word_frequency_dc[key]>1:\n        encoding_dc[key]=labelencoder\n        labelencoder+=1\n    \nprint (len(encoding_dc))\n\ndef preprocess_keras(text):\n    \n    def get_encoding(w):\n        if w in encoding_dc:\n            return encoding_dc[w]\n        return 0\n    \n    x = [ get_encoding(w) for w in text.split(' ') ]\n    x = x + (MAX_TEXT_LENGTH-len(x))*[0]\n    return x\nX_train_emb = np.array( [ preprocess_keras(x) for x in X_train ] )\nX_test_emb  = np.array( [ preprocess_keras(x) for x in X_test ]  )\nprint ( X_train_emb.shape, X_test_emb.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"In this kernel, we will use the transfer learning aproach.\n\nInstead of initializing the embedding weights randomly, we will use a set of pretrained weights that have been computed using a large corpus.\n\nIn this particular kernel, we will use GloVe embeddings, the GloVe algorithm has been trained of the Common Crawl Dataset (the web archive) https://nlp.stanford.edu/projects/glove/"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Function to load embeddings, returns a matrix (vovabulary size x embedding size)\ndef get_embeddings( word_index , method):    \n    EMBEDDING_FILE = EMBEDDINGS_PATH+'glove.840B.300d/glove.840B.300d.txt'\n    #each line of the file looks like : word dim1 dim2 .... dim300\n    embeddings = { o.split(\" \")[0]:np.asarray(o.split(\" \")[1:], dtype='float32') for o in open(EMBEDDING_FILE) if o.split(\" \")[0] in word_index }\n    \n    temp = np.stack(embeddings.values())\n    mean, std = temp.mean(), temp.std()\n    print ('mean and std of GloVe weights :', mean, std)\n    embedding_weights    = np.random.normal(mean, std, (len(word_index)+1,  EMBEDDING_SIZE ) ).astype(np.float32)\n\n    for word, i in word_index.items():\n        if (word in embeddings):\n            embedding_weights[i] = embeddings.get(word)\n    \n    return embedding_weights\n\nembedding_weights = get_embeddings(encoding_dc, method='glove')\nprint (embedding_weights.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\nFor each word in our vocabulary, we will assign a pretrained embedding weights from GloVe. But what about words for which we don't have a pretrained embedding?\n\nwe can initialize the weights using a normal distribution with 0 mean and some small std. Why 0 mean? because we can expect that the average of the final weights is close to zero (some will be positive and somme negative). But since we can extract those statistics from GloVe, we will use them instead\n \n"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We found previously some optimal values for batch size and learning rate\n\nlr = 1e-2 seems to be the best learning rate\nbs 512, 1024 and 2048 have similar performance, let's choose 2048 to make runtime faster.\n\nLet's first run the model without transfer learning to have a benchmark to compare with"},{"metadata":{"trusted":true},"cell_type":"code","source":"run_model( lr = 1e-2, bs=2048, use_glove=False )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We have a loss of  0.1144 which is better than the previous one (0.1152) without optimal parameters\n\nNext, we run the same model with the same parameters, but now, we will use Glove embeddings"},{"metadata":{"trusted":true},"cell_type":"code","source":"run_model( lr = 5e-3, bs=2048, use_glove=True )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"So we try to beat 0.1144 using transfer learning.\n\nWith the same parameters (lr=1e-2 and bs=2048), we obtain a better score of 0.1129.\n    \nCan we make transfer learning work better? Let's try smaller learning rates :\n\nlr=1e-2:     0.1129\n\nlr=5e-3:     0.1090 !!!!!!!!\n\nlr=1e-3:     0.1105\n\nlr=5e-4:     0.1120\n\nWe will discuss the results later.\n\n\n"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}