{"cells":[{"metadata":{"_uuid":"46db0dc6db28e6646805199db0f983c08470b872"},"cell_type":"markdown","source":"The model for this notebook is taken from: https://www.kaggle.com/michaelsnell/conv1d-dpcnn-in-keras\n\nDPCNN(Deep Pyramid Convolutional Neural Networks for Text Categorization): https://ai.tencent.com/ailab/media/publications/ACL3-Brady.pdf"},{"metadata":{"trusted":true,"_uuid":"83f6884eb49edb596bd3959d1f067de3bcd9aae0"},"cell_type":"code","source":"import os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport time\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\n\nfrom keras import backend as K\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.models import Model\nfrom keras.layers import Input, Dense, Embedding, MaxPooling1D, Conv1D, SpatialDropout1D\nfrom keras.layers import add, Dropout, PReLU, BatchNormalization, GlobalMaxPooling1D\nfrom keras.preprocessing import text, sequence\nfrom keras.callbacks import Callback\nfrom keras import optimizers\nfrom keras import initializers, regularizers, constraints, callbacks","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/train.csv')\ntest_df  = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"824e62c67cfe268a126b00e59607c4e07ec7d41c"},"cell_type":"code","source":"train_df, val_df = train_test_split(train_df, test_size=0.08, random_state=2018)\n\n## some config values \nembed_size = 300 # how big is each word vector\nmax_features = 60000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 100 # max number of words in a question to use\n\n\n## fill up the missing values\ntrain_X = train_df[\"question_text\"].fillna(\"_##_\").values\nval_X = val_df[\"question_text\"].fillna(\"_##_\").values\ntest_X = test_df[\"question_text\"].fillna(\"_##_\").values\n\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\n## Pad the sentences \ntrain_X = pad_sequences(train_X, maxlen=maxlen)\nval_X = pad_sequences(val_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)\n\n## Get the target values\ntrain_y = train_df['target'].values\nval_y = val_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ac522ee57949f9879a4125bcc2ea50a7b0b763c"},"cell_type":"code","source":"np.random.seed(2018)\ntrn_idx = np.random.permutation(len(train_X))\nval_idx = np.random.permutation(len(val_X))\n\ntrain_X = train_X[trn_idx]\nval_X = val_X[val_idx]\ntrain_y = train_y[trn_idx]\nval_y = val_y[val_idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c8c096310a916a5d7bed3347b344580e5707a81"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"0bbcf7cd4bb227df40755555312794ef8766a795"},"cell_type":"code","source":"filter_nr = 64\nfilter_size = 3\nmax_pool_size = 3\nmax_pool_strides = 2\ndense_nr = 256\nspatial_dropout = 0.2\ndense_dropout = 0.5\ntrain_embed = False\nconv_kern_reg = regularizers.l2(0.00001)\nconv_bias_reg = regularizers.l2(0.00001)\n\ninputs = Input(shape=(maxlen,))\nembedding = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=train_embed)(inputs)\nembedding = SpatialDropout1D(spatial_dropout)(embedding)\n\nblock1 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', \n            kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(embedding)\nblock1 = BatchNormalization()(block1)\nblock1 = PReLU()(block1)\nblock1 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', \n            kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block1)\nblock1 = BatchNormalization()(block1)\nblock1 = PReLU()(block1)\n\n#we pass embedded comment through conv1d with filter size 1 because it needs to have the same shape as block output\n#if you choose filter_nr = embed_size (300 in this case) you don't have to do this part and can add emb_comment directly to block1_output\nresize_emb = Conv1D(filter_nr, kernel_size=1, padding='same', activation='linear', \n            kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(embedding)\nresize_emb = PReLU()(resize_emb)\n    \nblock1_output = add([block1, resize_emb])\nblock1_output = MaxPooling1D(pool_size=max_pool_size, strides=max_pool_strides)(block1_output)\n\nblock2 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', \n            kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block1_output)\nblock2 = BatchNormalization()(block2)\nblock2 = PReLU()(block2)\nblock2 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', \n            kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block2)\nblock2 = BatchNormalization()(block2)\nblock2 = PReLU()(block2)\n    \nblock2_output = add([block2, block1_output])\nblock2_output = MaxPooling1D(pool_size=max_pool_size, strides=max_pool_strides)(block2_output)\n\nblock3 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', \n            kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block2_output)\nblock3 = BatchNormalization()(block3)\nblock3 = PReLU()(block3)\nblock3 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', \n            kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block3)\nblock3 = BatchNormalization()(block3)\nblock3 = PReLU()(block3)\n    \nblock3_output = add([block3, block2_output])\nblock3_output = MaxPooling1D(pool_size=max_pool_size, strides=max_pool_strides)(block3_output)\n\nblock4 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', \n            kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block3_output)\nblock4 = BatchNormalization()(block4)\nblock4 = PReLU()(block4)\nblock4 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', \n            kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block4)\nblock4 = BatchNormalization()(block4)\nblock4 = PReLU()(block4)\n\nblock4_output = add([block4, block3_output])\nblock4_output = MaxPooling1D(pool_size=max_pool_size, strides=max_pool_strides)(block4_output)\n\nblock5 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', \n            kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block4_output)\nblock5 = BatchNormalization()(block5)\nblock5 = PReLU()(block5)\nblock5 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', \n            kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block5)\nblock5 = BatchNormalization()(block5)\nblock5 = PReLU()(block5)\n\nblock5_output = add([block5, block4_output])\nblock5_output = MaxPooling1D(pool_size=max_pool_size, strides=max_pool_strides)(block5_output)\n\nblock6 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', \n            kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block5_output)\nblock6 = BatchNormalization()(block6)\nblock6 = PReLU()(block6)\nblock6 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear', \n            kernel_regularizer=conv_kern_reg, bias_regularizer=conv_bias_reg)(block6)\nblock6 = BatchNormalization()(block6)\nblock6 = PReLU()(block6)\n\nblock6_output = add([block6, block5_output])\noutput = GlobalMaxPooling1D()(block6_output)\n\noutput = Dense(dense_nr, activation='linear')(output)\noutput = BatchNormalization()(output)\noutput = PReLU()(output)\noutput = Dropout(dense_dropout)(output)\noutput = Dense(1, activation='sigmoid')(output)\n\nmodel = Model(inputs, output)\nprint(model.summary())\n\nmodel.compile(loss='binary_crossentropy', \n            optimizer=optimizers.Adam(),\n            metrics=['accuracy'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe08e7f61f73ac21afa42e6eb8a7fb5e796f8f3c"},"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=512, epochs=20, validation_data=(val_X, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b60c6cdc07328eab450de2a47b1d7757e7e0715"},"cell_type":"code","source":"pred_noemb_val_y = model.predict([val_X], batch_size=1024, verbose=1)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_noemb_val_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"38a1c3b4d0cdce0919a51cc1ac90f39ca5ef11cc"},"cell_type":"code","source":"predictions = model.predict([test_X], batch_size=1024, verbose=1)\npredictions = (predictions>0.28).astype(int)\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = predictions\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc788671ac90236740986014b40cce77a05b32aa"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"038499a4fe7f57c7b9c1c69e9c3282892634ab6d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}