{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport time\nimport numpy as np \nimport pandas as pd \nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNLSTM, Conv1D,Dropout\nfrom keras.layers import Bidirectional, GlobalMaxPool1D,MaxPooling1D,BatchNormalization,Flatten,RepeatVector,Permute,concatenate\nfrom keras.models import Model","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba5a1b8109dee2c9fbc628d5da4a7c3447d42fb8"},"cell_type":"code","source":"train, dev = train_test_split(train, test_size=0.1, random_state=2018)\n\nembed_size = 300 \nmax_features = 50000 \nmaxlen = 40 \n\ntrain_X = train[\"question_text\"].fillna(\"_na_\").values\ndev_X = dev[\"question_text\"].fillna(\"_na_\").values\ntest_X = test[\"question_text\"].fillna(\"_na_\").values\n\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\ndev_X = tokenizer.texts_to_sequences(dev_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\ntrain_X = pad_sequences(train_X, maxlen=maxlen)\ndev_X = pad_sequences(dev_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)\n\ntrain_y = train['target'].values\ndev_y = dev['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f34fad42a4c6f25156881000a0a75b743c5a7b15"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7a9c233699ac6e99b1bbc6fab1605ae1886fdbde"},"cell_type":"code","source":"#import xgboost as xgb\n# read in data\n#exgb_classifier = xgb.XGBClassifier()\n#model=exgb_classifier.fit(train_X[:100000,:], train_y[:100000], sample_weight={0:0.4,1:1})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ac04cdf924af6714d8a97a0d43d2f5dce1e412a"},"cell_type":"code","source":"#ypred = model.predict(dev_X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e14a51061452e0f76b2c21ecbe1fe5a7a6a28aa5"},"cell_type":"code","source":"#metrics.f1_score(dev_y,ypred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3cfab26c6cced33ef7ab84f0d36997113131d530"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d603907622fe9f9fa6626915c7fe3d97b15ef92"},"cell_type":"code","source":"from keras import backend as K\n\ndef f1(y_true, y_pred):\n    def recall(y_true, y_pred):\n        \"\"\"Recall metric.\n\n        Only computes a batch-wise average of recall.\n\n        Computes the recall, a metric for multi-label classification of\n        how many relevant items are selected.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        \"\"\"Precision metric.\n\n        Only computes a batch-wise average of precision.\n\n        Computes the precision, a metric for multi-label classification of\n        how many selected items are relevant.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47edd83c6dfae9e063f64c4f43c88c04154dcfd6"},"cell_type":"code","source":"def net(input_shape):\n    sentence_indices = Input(input_shape, dtype='int32')\n    X = Embedding(max_features, embed_size,weights=[embedding_matrix])(sentence_indices)\n    activations = Bidirectional(CuDNNLSTM(128, return_sequences=True))(X)\n    activations = Dropout(0.1)(activations)\n    # compute importance for each step\n    attention = Dense(1, activation='tanh')(activations)\n    attention = Flatten()(attention)\n    attention = Activation('softmax')(attention)\n    attention = RepeatVector(128)(attention)\n    attention = Permute([2, 1])(attention)\n\n    X = concatenate([activations, attention])\n    activations = Bidirectional(CuDNNLSTM(128, return_sequences=True))(X)\n    activations = Dropout(0.1)(activations)\n    # compute importance for each step\n    attention = Dense(1, activation='tanh')(activations)\n    attention = Flatten()(attention)\n    attention = Activation('softmax')(attention)\n    attention = RepeatVector(128)(attention)\n    attention = Permute([2, 1])(attention)\n\n    X = concatenate([activations, attention])\n    X = Bidirectional(CuDNNLSTM(64, return_sequences=True))(X)\n    X = Dropout(0.1)(X)\n    X = GlobalMaxPool1D()(X)\n    X = Dense(16, activation=\"relu\")(X)\n    X = Dropout(0.1)(X)\n    X = Dense(1, activation=\"sigmoid\")(X)\n    model = Model(inputs=sentence_indices, outputs=X)\n    return model\nmodel=net((maxlen,))\n#model.layers[1].trainable=False\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy',f1])\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3af5590978531810849719e41a563eb02de84add"},"cell_type":"code","source":"weight={0:0.4,1:1}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47238831a4701c8a67dc7ecb130ac1402baf7bb2","scrolled":true},"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=512, epochs=3 , validation_data=(dev_X, dev_y),class_weight=weight)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba8538f22d044cac35ada3c142018cbae9ca99c9"},"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(dev_X, dev_y),class_weight=weight)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6d7ce765396fcf379e2a0f7fb5618314b0df66e"},"cell_type":"code","source":"f1=[]\npred_noemb_val_y = model.predict([dev_X], batch_size=1024, verbose=1)\nfor thresh in np.arange(0.4, 0.702, 0.01):\n    thresh = np.round(thresh, 2)\n    f1.append(metrics.f1_score(dev_y, (pred_noemb_val_y>thresh).astype(int)))\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(dev_y, (pred_noemb_val_y>thresh).astype(int))))\nf1=np.array(f1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6385fbf03227b63b00e0dc0e6fcc34f1ac3aa6ee"},"cell_type":"code","source":"thresh=np.argmax(f1)*0.01+0.4\nprint(thresh)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b5c34e50246e9743550c285a052c3f1536c375d"},"cell_type":"code","source":"pred_noemb_val_y1 = model.predict([test_X], verbose=1)\nlabel1=(pred_noemb_val_y1>thresh).astype(int)\nmodel=None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b7ab4100f723ad535528865b1edc7896bce80223"},"cell_type":"code","source":"#model2=net((60,))\n#model2.compile(loss='binary_crossentropy', optimizer=\"adam\", metrics=[\"accuracy\",f1])\n#history=model2.fit(train_X, train_y, epochs = 2,validation_data=(dev_X,dev_y),class_weight=weight,batch_size = 128, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e5caf0e8de6ba37b7c6d2bf226dd9e52169169dc"},"cell_type":"code","source":"#history=model2.fit(train_X, train_y, epochs = 2,validation_data=(dev_X,dev_y),class_weight=weight,batch_size = 128, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b8d6506cd458806d45af996d0b9b7cb850e897d"},"cell_type":"code","source":"#pred_noemb_val_y2 = model2.predict([test_X], verbose=1)\n#label2=(pred_noemb_val_y2>0.58).astype(int)\n#model2=None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b12fac67c8d7920a0b4148ea7c66573471616b7"},"cell_type":"code","source":"#model3=net((60,))\n#model3.compile(loss='binary_crossentropy', optimizer=\"adam\", metrics=[\"accuracy\",f1])\n#history=model3.fit(train_X, train_y, epochs = 2,validation_data=(dev_X,dev_y),class_weight=weight,batch_size = 128, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d9312a6f652e07b33306b07d19dc81b0247e0667"},"cell_type":"code","source":"#history=model3.fit(train_X, train_y, epochs = 2,validation_data=(dev_X,dev_y),class_weight=weight,batch_size = 128, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43804f22c9f7cc1a62d6248f40dedcfd1c5fa113"},"cell_type":"code","source":"#pred_noemb_val_y3 = model3.predict([test_X], verbose=1)\n#label3=(pred_noemb_val_y3>0.58).astype(int)\n#model3=None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3216362afb0f49579d287a06f13adf8cd7d8b0cf"},"cell_type":"code","source":"test_labels=label1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc188f2787ea7b98d3a40953a95a5fc09ff2764d"},"cell_type":"code","source":"submission=pd.read_csv(\"../input/sample_submission.csv\")\nsubmission[\"prediction\"]=test_labels\nsubmission.head()\nsubmission.to_csv(\"submission.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90c673c15289f026d1f3a008ff851d5f5f9eef6d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c535faa26a2d93026a6c967889f1d25ca2c5ecd"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"03b02731aedef3cf1a7eb1c6edea896b1dfc0343"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}