{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/train.csv')\nX_train = train_df[\"question_text\"].fillna(\"dieter\").values\ntest_df = pd.read_csv('../input/test.csv')\nX_test = test_df[\"question_text\"].fillna(\"dieter\").values\ny = train_df[\"target\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77841423e419b55163c9e4f741e6aa4316d46503","scrolled":true},"cell_type":"code","source":"from keras.models import Model\nfrom keras.layers import Input, Dense, Embedding, concatenate\nfrom keras.layers import CuDNNGRU, Bidirectional, GlobalAveragePooling1D, GlobalMaxPooling1D, Conv1D\nfrom keras.layers import Add, BatchNormalization, Activation, CuDNNLSTM, Dropout\nfrom keras.layers import *\nfrom keras.preprocessing import text, sequence\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"439a84cbe1a431755034a4c0d40693a789b16cad"},"cell_type":"code","source":"maxlen = 100\nmax_features = 30000\nembed_size=100\n\ntokenizer = text.Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(X_train) + list(X_test))\n\nX_train = tokenizer.texts_to_sequences(X_train)\nX_test = tokenizer.texts_to_sequences(X_test)\n\nx_train = sequence.pad_sequences(X_train, maxlen=maxlen)\nx_test = sequence.pad_sequences(X_test, maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"42cf8537db193d7e591068ae4e7e392663c4cfd9"},"cell_type":"code","source":"\ndef model1(init):\n    x = init\n    x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)\n    x = Bidirectional(CuDNNLSTM(64))(x)\n    out = Dense(64, activation=\"relu\")(x)\n    return out\n\ndef m2_block(init, filter, kernel, pool):\n    x = init\n    x = Conv1D(filter, kernel, padding='same')(x)\n    skip = x\n    x = BatchNormalization()(x)\n    x = Conv1D(filter, kernel, padding='same')(x)\n    x = BatchNormalization()(x)\n    x = Add()([x, skip])\n    x = Activation('relu')(x)\n    x = MaxPooling1D(pool)(x)\n    \n    x = Flatten()(x)\n    \n    return x\n\ndef model2(init):\n    #init = Reshape((maxlen, embed_size, 1))(init)\n    \n    x0 = m2_block(init, 32, 2, 2)\n    x1 = m2_block(init, 32, 3, 2)\n    x2 = m2_block(init, 32, 5, 2)\n    x3 = m2_block(init, 32, 7, 8)\n    \n    x = concatenate([x0, x1, x2, x3])\n    out = Dense(64, activation=\"relu\")(x)\n    return out\n\ndef get_model():\n    inp = Input(shape=(maxlen, ))\n    x = Embedding(max_features, embed_size)(inp)\n    \n    out1 = model1(x)\n    out2 = model2(x)\n    \n    conc = concatenate([out1, out2])\n    x = Dropout(0.5)(conc)\n    x = Dense(64, activation='relu')(x)\n    x = Reshape((x.shape[1].value, 1))(x)\n    x = CuDNNLSTM(32)(x)\n    outp = Dense(1, activation=\"sigmoid\")(x)\n    \n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy',\n                  optimizer='adam',\n                  metrics=['accuracy'])    \n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"c04798916f8f73c9573a1b5fc5304d55882ec97d"},"cell_type":"code","source":"model = get_model()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"68f56878015f3958e28a8c5b8bc962550cb9b2f1"},"cell_type":"code","source":"batch_size = 256\nepochs = 20","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"41e030a9eb996c96ad22b3c6655e097338453d62"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_tra, X_val, y_tra, y_val = train_test_split(x_train, y, test_size = 0.1, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"38d3db09f0c43416464eba4dd67cf80ba2fb176b"},"cell_type":"code","source":"early_stopping = EarlyStopping(patience=3, verbose=1)\nmodel_checkpoint = ModelCheckpoint('./quora.model', save_best_only=True, verbose=1)\nreduce_lr = ReduceLROnPlateau(factor=0.5, patience=3, min_lr=0.0001, verbose=1)\n\nhist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),\n                 verbose=True, callbacks = [early_stopping, model_checkpoint, reduce_lr])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c77dc65b5201e0efc0dabd72fcf64d6271b848d"},"cell_type":"code","source":"y_pred = model.predict(x_test, batch_size=1024, verbose=True)\ny_te = (y_pred[:,0] > 0.5).astype(np.int)\n\nsubmit_df = pd.DataFrame({\"qid\": test_df[\"qid\"], \"prediction\": y_te})\nsubmit_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7cd1d76a1021cd5edfd5ed78b25f95c2e6933b87"},"cell_type":"code","source":"from IPython.display import HTML\nimport base64  \nimport pandas as pd  \n\ndef create_download_link( df, title = \"Download CSV file\", filename = \"data.csv\"):  \n    csv = df.to_csv(index =False)\n    b64 = base64.b64encode(csv.encode())\n    payload = b64.decode()\n    html = '<a download=\"{filename}\" href=\"data:text/csv;base64,{payload}\" target=\"_blank\">{title}</a>'\n    html = html.format(payload=payload,title=title,filename=filename)\n    return HTML(html)\n\ncreate_download_link(submit_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a4598a50d72aa764ac81529bff1cce8d0c9c1f31"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"89949e34b628c1ea5b573112fd86aa7c1966a7a1"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}