{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.model_selection import train_test_split\nfrom keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.layers import Reshape, Dropout, Input, Embedding, Dense, Flatten, Conv2D, MaxPool2D, Concatenate\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.optimizers import Adam\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\ntrain_input_path = \"../input/train.csv\"\ntest_input_path = \"../input/test.csv\"\nprint(train_input_path)\n\nprint(os.listdir(\"../input/embeddings\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Input processing and input split**"},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"train = pd.read_csv(train_input_path)\n\ntest = pd.read_csv(test_input_path)\ntest","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":false},"cell_type":"code","source":"max_features = 95000\nmaxlen = 70\nX = train.question_text\ny = train.target\ntest_X = test.question_text\n# Split into validation and training data\ntrain_X, val_X, train_y, val_y = train_test_split(X, y, random_state=1)\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\n\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\n## Pad the sentences \ntrain_X = pad_sequences(train_X, maxlen=maxlen)\nval_X = pad_sequences(val_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print( \"Length of train is {}\", len(train_X))\nprint( \"Length of validation is {}\", len(val_X))\nprint(\"Ratio of train is {}, val is {}\", len(train_X)/len(X), len(val_X)/len(X) )\nprint(\"Length of test: \", len(test_X))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Transfer learning**"},{"metadata":{"trusted":true},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\ndef get_coefs(word,*arr): \n    return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\nprint(\"Embeddings size: \", embed_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"inp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp)\nx = Reshape((maxlen, embed_size, 1))(x)\n\ncnn_1 = Conv2D(42, kernel_size=(7, embed_size), kernel_initializer='he_normal', activation='relu')(x)\npool_1 = MaxPool2D(pool_size=(7,1))(cnn_1)\n\ncnn_2 = Conv2D(42, kernel_size=(5, embed_size), kernel_initializer='he_normal', activation='relu')(x)\npool_2 = MaxPool2D(pool_size=(5,1))(cnn_2)\n\ncnn_3 = Conv2D(42, kernel_size=(3, embed_size), kernel_initializer='he_normal', activation='relu')(x)\npool_3 = MaxPool2D(pool_size=(3,1))(cnn_3)\n\ncnn_4 = Conv2D(42, kernel_size=(1, embed_size), kernel_initializer='he_normal', activation='relu')(x)\npool_4 = MaxPool2D(pool_size=(1,1))(cnn_4)\n\nx = Concatenate(axis=1)([pool_1,pool_2,pool_3,pool_4])\n\nx = Dense(64, activation='relu')(x)\nx = Flatten()(x)\nx = Dropout(0.1)(x)\noutp = Dense(1, activation='softmax')(x)\n\nmodel_1 = Model(inputs=inp, outputs=outp)\n\nmodel_1.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_1.compile(optimizer=Adam(lr=5e-3), loss='binary_crossentropy', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_1.fit(train_X, train_y, batch_size=1024, epochs=3, validation_data=(val_X, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import numpy as np\n# from sklearn import metrics\n\n# pred_val_y = model_1.predict([val_X], batch_size=1024, verbose=1)\n\n# def get_best_threshold():\n#     best_tresh = 0.33 # default\n#     best_f1_score = 0\n#     for thresh in np.arange(0.05, 0.501, 0.01):\n#         f1_score = metrics.f1_score(val_y, (pred_val_y >= thresh))\n#         print('F1 score at ',thresh,' is ', f1_score)\n#         if best_f1_score < f1_score :\n#             best_f1_score = f1_score\n#             best_tresh = thresh\n#     return best_tresh\n    \n# best_threshold = get_best_threshold()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_test_y = model_1.predict([test_X], batch_size=1024, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_threshold = 0.33\npred_test_y = (pred_test_y > best_threshold).astype(int)\nout_df = pd.DataFrame({\"qid\":test[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir(\".\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_test_y","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}