{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# fork\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/train.csv').sample(frac=0.1) # downsample for speed\nX_train = train_df[\"question_text\"].fillna(\"dieter\").values\ntest_df = pd.read_csv('../input/test.csv')\nX_test = test_df[\"question_text\"].fillna(\"dieter\").values\ny = train_df[\"target\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d565c03e9b7b2f133e4e8f4eb0ed687c355f1fc4"},"cell_type":"code","source":"from keras.models import Model\nfrom keras.layers import Input, Dense, Embedding, concatenate\nfrom keras.layers import CuDNNGRU, Bidirectional, GlobalAveragePooling1D, GlobalMaxPooling1D, Conv1D\nfrom keras.layers import Add, BatchNormalization, Activation, CuDNNLSTM, Dropout\nfrom keras.layers import *\nfrom keras.models import *\nfrom keras.preprocessing import text, sequence\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import keras\nprint (keras.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca0edbb51d3a75ca4d667c5eca1dcdfb916a7fa7"},"cell_type":"code","source":"maxlen = 60\nmax_features = 30000\nembed_size = 200\n\ntokenizer = text.Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(X_train) + list(X_test))\n\nX_train = tokenizer.texts_to_sequences(X_train)\nX_test = tokenizer.texts_to_sequences(X_test)\n\nx_train = sequence.pad_sequences(X_train, maxlen=maxlen)\nx_test = sequence.pad_sequences(X_test, maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f5c8eee53cf45ccfd1ce56f4011f1df00186a5e"},"cell_type":"code","source":"def attention_3d_block(inputs, name):\n    # inputs.shape = (batch_size, time_steps, input_dim)\n    TIME_STEPS = inputs.shape[1].value\n    SINGLE_ATTENTION_VECTOR = False\n    \n    input_dim = int(inputs.shape[2])\n    a = Permute((2, 1))(inputs)\n    a = Reshape((input_dim, TIME_STEPS))(a) # this line is not useful. It's just to know which dimension is what.\n    a = Dense(TIME_STEPS, activation='softmax')(a)\n    if SINGLE_ATTENTION_VECTOR:\n        a = Lambda(lambda x: K.mean(x, axis=1))(a)\n        a = RepeatVector(input_dim)(a)\n    a_probs = Permute((2, 1), name=name)(a)\n    output_attention_mul = Multiply()([inputs, a_probs])\n    return output_attention_mul","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"56fbe4f2508fbaa615d4d817b46b1adea8f7e262"},"cell_type":"code","source":"embedding_index = dict()\nf = open('../input/embeddings/glove.840B.300d/glove.840B.300d.txt',encoding='utf8')\n\nfor line in f:\n    \n    values = line.split(\" \")\n    words = values[0]\n   \n    coefs = np.asarray(values[1:(embed_size+1)], dtype='float32')\n    embedding_index[words]= coefs\n    \nf.close()\nprint('Loaded %s word vectors.' % len(embedding_index))\n\nembedding_matrix = np.zeros((max_features, embed_size))\nfor word, index in tokenizer.word_index.items():\n    if index > max_features - 1:\n        break\n    else:\n        embedding_vector = embedding_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[index] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d536ec84108d3044fbf90a5394223848699d9f1d"},"cell_type":"code","source":"def model1(init):\n    x = init\n    x = Conv1D(64, 3,strides=2,padding='same',activation='relu')(x)\n    x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)\n    x = attention_3d_block(x, 'attention_vec_1')\n    x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)\n    x = attention_3d_block(x, 'attention_vec_2')\n    x = GlobalMaxPool1D()(x)\n    out = Dense(64, activation=\"relu\")(x)\n    return out\n\ndef m2_block(init, filter, kernel, pool):\n    x = init\n    \n    x = Conv1D(filter, kernel, padding='same', kernel_initializer='he_normal', activation='elu')(x)\n    skip = x\n    x = Conv1D(filter, kernel, padding='same', kernel_initializer='he_normal', activation='elu')(x)\n    x = Conv1D(filter, kernel, padding='same')(x)\n    x = BatchNormalization()(x)\n    x = Add()([x, skip])\n    x = Activation('relu')(x)\n    x = MaxPooling1D(pool)(x)\n    \n    x = Flatten()(x)\n    x = BatchNormalization()(x)\n    \n    return x\n\ndef model2(init):\n    #init = Reshape((maxlen, embed_size, 1))(init)\n    \n    # pool = maxlen - filter + 1\n    x0 = m2_block(init, 32, 1, maxlen - 1 + 1)\n    x1 = m2_block(init, 32, 2, maxlen - 2 + 1)\n    x2 = m2_block(init, 32, 3, maxlen - 3 + 1)\n    x3 = m2_block(init, 32, 5, maxlen - 5 + 1)\n    \n    x = concatenate([x0, x1, x2, x3])\n    x = Dropout(0.25)(x)\n    out = Dense(64, activation=\"relu\")(x)\n    return out\n\n\ndef get_model():\n    inp = Input(shape=(maxlen, ))\n    #x = Embedding(max_features, embed_size)(inp)\n    x = Embedding(input_dim=max_features, output_dim= embed_size , input_length=maxlen,weights=[embedding_matrix], trainable=False)(inp)\n    \n    out1 = model1(x)\n    out2 = model2(x)\n    \n    conc = concatenate([out1, out2])\n    \n    #conc = out1\n    x = Dropout(0.2)(conc)\n    x = Dense(64, activation='relu')(x)\n    x = Reshape((x.shape[1].value, 1))(x)\n    x = CuDNNLSTM(32)(x)\n    outp = Dense(1, activation=\"sigmoid\")(x)\n    \n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy',\n                  optimizer='adam',\n                  metrics=['accuracy'])    \n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"f07462c499d43887b355f683829862fcc245e817"},"cell_type":"code","source":"model = get_model()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"535881828c1f36da0373c5eb11d9bffab3bc4410"},"cell_type":"code","source":"batch_size = 256\nepochs = 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f14547280cb1ee50b34dc2d884d878520177073"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_tra, X_val, y_tra, y_val = train_test_split(x_train, y, test_size = 0.07, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ae9b027caa539ee520f4289b6a338dfdc750f14"},"cell_type":"code","source":"early_stopping = EarlyStopping(patience=4, verbose=1)\nmodel_checkpoint = ModelCheckpoint('./quora.model', save_best_only=True, verbose=1)\nreduce_lr = ReduceLROnPlateau(factor=0.5, patience=3, min_lr=0.0001, verbose=1)\n\n#model = load_model('./quora.model')\n\nhist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),\n                 verbose=True, callbacks = [early_stopping, model_checkpoint, reduce_lr])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"404a671dca09cebb0a014796063bd59d82f98714"},"cell_type":"code","source":"y_pred = model.predict(x_test, batch_size=1024, verbose=True)\ny_te = (y_pred[:,0] > 0.5).astype(np.int)\n\nsubmit_df = pd.DataFrame({\"qid\": test_df[\"qid\"], \"prediction\": y_te})\nsubmit_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3c861699e0ae612641cc3eb7d288500c2168cd1"},"cell_type":"code","source":"from IPython.display import HTML\nimport base64  \nimport pandas as pd  \n\ndef create_download_link( df, title = \"Download CSV file\", filename = \"data.csv\"):  \n    csv = df.to_csv(index =False)\n    b64 = base64.b64encode(csv.encode())\n    payload = b64.decode()\n    html = '<a download=\"{filename}\" href=\"data:text/csv;base64,{payload}\" target=\"_blank\">{title}</a>'\n    html = html.format(payload=payload,title=title,filename=filename)\n    return HTML(html)\n\ncreate_download_link(submit_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"48313b5e086fd1dd08d43e51190db353a689cbbc"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}