{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import keras\nimport numpy as np\nimport pandas as pd\nimport re\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Embedding, LSTM, Conv1D, GlobalMaxPooling1D, Input, concatenate, Dropout, Reshape\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.models import Sequential, Model\nfrom keras import optimizers\nfrom keras.utils import to_categorical\nimport os\nfrom imblearn.under_sampling import RandomUnderSampler\nfrom keras.layers import Bidirectional","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"188d919e25f32e95fb6c90e14c37a843a58ab96b"},"cell_type":"code","source":"MAX_TIME = 30\nVOCAB_SIZE = 20000\nQUES_CLEANING_PATTERN = re.compile(\"[\\s\\n\\r\\t.,:;\\-_\\'\\\"?!#&()\\/%\\[\\]\\{\\}\\<\\>\\\\$@\\!\\*\\+\\=]\")\nLSTM_DIM = 256\nLSTM_DIMS = [512,256]\nNUM_FILTERS = 5\nFILTER_LENGTHS = [1,2,3,4,5,8,10,15,20,25]\nDROPOUT = 0.4\nLEARNING_RATE = 0.005\nNUM_EPOCHS = 10\nBATCH_SIZE = 2000\nNUM_UNDERSAMPLE = 3\nRUS_RATIO = 1.0/4.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"67bb95791036e8af4a2797ac73cd88e709212e08"},"cell_type":"code","source":"train_data = pd.read_csv('../input/train.csv')\ntest_data = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d53eac285dd4cdcd0157c9d6c68dcc893c3e26f7"},"cell_type":"code","source":"train_data = np.array(train_data)\ntest_data = np.array(test_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"48e6ea3e3a4ccb342a9ffc4068aaae419f74c62e"},"cell_type":"code","source":"ct_0 = 0\nc = 0\nfor d in train_data:\n    if d[2] == 0:\n        ct_0 += 1\n    if len(QUES_CLEANING_PATTERN.split(d[1])) > MAX_TIME:\n        c+=1\nprint(ct_0, len(train_data) - ct_0, c/len(train_data))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf211f2a57b2290fa302da37e04879b6284748fb"},"cell_type":"code","source":"#Preprocess\nNUM_TRAIN = int(len(train_data)*0.8)\nnp.random.shuffle(train_data)\ntrain_x = train_data[:NUM_TRAIN,1]\ntrain_y = train_data[:NUM_TRAIN,2]\nval_x = train_data[NUM_TRAIN:,1]\nval_y = train_data[NUM_TRAIN:,2]\ndel train_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4bbc6c9f307d28c3705d2f42552b1a4837a55f69"},"cell_type":"code","source":"test_x = test_data[:,1]\ntest_ids = test_data[:,0]\ndel test_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db820357624f66bc52c40325f98f6693f5e60a19"},"cell_type":"code","source":"tokenizer = Tokenizer(VOCAB_SIZE)\ntokenizer.fit_on_texts(train_x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc5aba38b5d155014cdeb5213327dae807c8e762"},"cell_type":"code","source":"train_x = tokenizer.texts_to_sequences(train_x)\nval_x = tokenizer.texts_to_sequences(val_x)\ntest_x = tokenizer.texts_to_sequences(test_x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1a2c2a569edbe00492c8d9b00573394ccca6b39"},"cell_type":"code","source":"train_y.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"53e8e3328aea8df92b4d74fc582d61c33d66cd33"},"cell_type":"code","source":"def getEmbeddingMatrix(wordIndex):\n    \"\"\"Populate an embedding matrix using a word-index. If the word \"happy\" has an index 19,\n       the 19th row in the embedding matrix should contain the embedding vector for the word \"happy\".\n    Input:\n        wordIndex : A dictionary of (word : index) pairs, extracted using a tokeniser\n    Output:\n        embeddingMatrix : A matrix where every row has 100 dimensional GloVe embedding\n    \"\"\"\n    embeddingsIndex = {}\n    # Load the embedding vectors from ther GloVe file\n    with open(\"../input/embeddings/glove.840B.300d/glove.840B.300d.txt\", encoding=\"utf8\") as f:\n        for line in f:\n            values = line.split(' ')\n            word = values[0]\n            try:\n                embeddingVector = np.asarray(values[1:], dtype='float32')\n            except:\n                print(values)\n                break\n            embeddingsIndex[word] = embeddingVector\n\n    print('Found %s word vectors.' % len(embeddingsIndex))\n\n    # Minimum word index of any word is 1.\n    embeddingMatrix = np.zeros((len(wordIndex) + 1, 300))\n    for word, i in wordIndex.items():\n        embeddingVector = embeddingsIndex.get(word)\n        if embeddingVector is not None:\n            # words not found in embedding index will be all-zeros.\n            embeddingMatrix[i] = embeddingVector\n    del embeddingsIndex\n    return embeddingMatrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77bb5985bfe5889503cf8e9dab4fa2ed0e11ac18"},"cell_type":"code","source":"wordIndex = tokenizer.word_index\nwI = {}\nfor k,v in wordIndex.items():\n    if v < VOCAB_SIZE:\n        wI[k] = v\nwordIndex = wI","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef9cfca0274c10d8c8d1f7e9d210000c6707c55a"},"cell_type":"code","source":"embeddingMatrix = getEmbeddingMatrix(wordIndex)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b20d04281375247aa3689810d81a9e6a5fa29ae8"},"cell_type":"code","source":"train_x = pad_sequences(train_x,maxlen=MAX_TIME,padding='post')\nval_x = pad_sequences(val_x,maxlen=MAX_TIME,padding='post')\ntest_x = pad_sequences(test_x,maxlen=MAX_TIME,padding='post')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"efd63adf474342c0291af3ce4e6dac12d748ddd9"},"cell_type":"code","source":"train_y = to_categorical(train_y)\n#train_y = train_y.astype('int')\nval_y = to_categorical(val_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f2d13ce559560298db8a4bee5fb5a7ae737bfeb"},"cell_type":"code","source":"train_y.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e31c8f82ae314f4d38c52f34b30bc3291be81f8"},"cell_type":"code","source":"def buildModel(embeddingMatrix):\n    embeddingLayer = Embedding(embeddingMatrix.shape[0],300,weights=[embeddingMatrix],\\\n                              input_length=MAX_TIME,trainable=False)\n    '''\n    model = Sequential()\n    model.add(embeddingLayer)\n    model.add(Bidirectional(LSTM(LSTM_DIM,dropout=DROPOUT)))\n    #model.add(LSTM(LSTM_DIMS[0],dropout = DROPOUT,return_sequences=True))\n    #model.add(LSTM(LSTM_DIMS[1],dropout=DROPOUT))\n    model.add(Dense(32,activation='relu'))\n    model.add(Dense(2,activation='sigmoid'))\n    '''\n    sent_inp = Input(shape=(MAX_TIME,))\n    sent = embeddingLayer(sent_inp)\n    \n    _, h1, c1 = LSTM(LSTM_DIM,dropout=DROPOUT,return_state=True)(sent)\n    h1 = Reshape([MAX_TIME,LSTM_DIM])(concatenate([h1]*MAX_TIME,1))\n    lstm_inp = concatenate([sent,h1],axis=2)\n    _, h2, c2 = LSTM(LSTM_DIM*2,dropout=DROPOUT,return_state=True)(lstm_inp)\n    probs = Dense(64,activation='relu')(h2)\n    probs = Dense(2,activation='sigmoid')(probs)\n    model = Model(inputs=sent_inp,outputs=probs)\n    rmsprop = optimizers.rmsprop(lr=LEARNING_RATE)\n    \n    model.compile(loss='categorical_crossentropy',\n                 optimizer=rmsprop,\n                 metrics=['acc'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6051faec7a4d81e1fcc8c286b37740ba4483a3dc"},"cell_type":"code","source":"model = buildModel(embeddingMatrix)\n#model = buildCNNModel(embeddingMatrix)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eec92b486f5ddfbe9b38fe217fd2a94a86c77c9a"},"cell_type":"code","source":"'''\nfor i in range(1):\n    rus = RandomUnderSampler(RUS_RATIO,random_state=i)\n    train_x_res, train_y_res = rus.fit_resample(train_x,train_y)\n    #train_x_res = np.reshape(train_x_res,(train_x_res.shape[0]*train_x_res.shape[1],-1))\n    #train_y_res = np.reshape(train_y_res,(train_y_res.shape[0]*train_y_res.shape[1],-1))\n    \n    #print(train_y_res.shape)\n    train_y_res = to_categorical(train_y_res)\n    #print(train_y_res.shape)\n    perm = np.random.permutation(len(train_y_res))\n    train_x_res = train_x_res[perm]\n    train_y_res = train_y_res[perm]\n    \n    #print(train_x_res.shape,train_y_res.shape)\n    #break\n    model.fit(train_x_res,train_y_res,epochs=NUM_EPOCHS,batch_size=BATCH_SIZE,verbose=1,validation_data=(val_x,val_y))\n'''\nmodel.fit(train_x,train_y,epochs=NUM_EPOCHS,batch_size=BATCH_SIZE,verbose=1,validation_data=(val_x,val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3dc974dbe84329a651cac8437c4488d4b7ab4f4a"},"cell_type":"code","source":"predictions = model.predict(test_x,batch_size=BATCH_SIZE)\npredictions = predictions.argmax(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b9456b52338353afe91beac8fd4d148480479c61"},"cell_type":"code","source":"np.sum(predictions==1)/len(test_x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"70b37804c0bbe670c29bfef3d664b5f1263fc858"},"cell_type":"code","source":"np.save(\"preds.npy\",predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c66be94560d1c94a67d1b7cc99717f906facb833"},"cell_type":"code","source":"import csv\nwith open('submission.csv','w') as f:\n    writer = csv.writer(f)\n    writer.writerow(['qid','prediction'])\n    for i,idx in enumerate(test_ids):\n        writer.writerow([idx,predictions[i]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"861caa2ee3bd05522d11d387b33cb4eb0054ef3b"},"cell_type":"code","source":"val_pred = model.predict(val_x,batch_size=BATCH_SIZE)\nval_pred = val_pred.argmax(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93a6519317482d90db44e8210116a27e4d95208e"},"cell_type":"code","source":"from sklearn.metrics import precision_recall_fscore_support as fscore","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7083d280892fcdf3de61494cc64a32aecf718057"},"cell_type":"code","source":"fs = fscore(val_y.argmax(axis=1),val_pred)\nfs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"67dad96d1fc32298d37e7b19aa7016eef8e59834"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}