{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport gc\nimport os\nfrom tqdm import tqdm\ntqdm.pandas()\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/train.csv')\ndf_test = pd.read_csv('../input/test.csv')\n\nprint(f'Train shape: {df_train.shape}')\nprint(f'Test shape: {df_test.shape}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.concat([df_train,df_test],sort=True)\n\ndf.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_embed(file):\n    def get_coefs(word,*arr): \n        return word, np.asarray(arr, dtype='float32')\n    \n    if file == '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec':\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file) if len(o)>100)\n    else:\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding='latin'))\n        \n    return embeddings_index","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's Load Glove Embedding"},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding = load_embed('../input/embeddings/glove.840B.300d/glove.840B.300d.txt')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now we should build our vocab keeping the frequency of each word in the vocab"},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import defaultdict\n\ndef build_vocab(sentences):\n    fd = defaultdict(int)\n    for sentence in tqdm(sentences):\n        for word in sentence:\n            fd[word]+=1\n    return fd    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's create a function to check how many words in our vocab are actually present in out embedding"},{"metadata":{"trusted":true},"cell_type":"code","source":"def embed_intersection(vocab,embedding):\n    temp = {}\n    oov = {}\n    i = 0\n    j = 0\n    \n    for word in vocab.keys():\n        try:\n            temp[word] = embedding[word]\n            i+=vocab[word]\n        except:\n            oov[word] = vocab[word]\n            j+=vocab[word]\n            pass\n    \n    print(f\"Found embeddings for {(len(temp)/len(vocab)*100):.3f}% of vocab\")\n    print(f\"Found embeddings for {(i/(i+j))*100:.3f}% of all text\")\n    \n    sorted_x = sorted(oov.items(), key = lambda x: x[1])[::-1]\n    return sorted_x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sentences = df['question_text'].progress_apply(lambda x: x.split()).values\nvocab = build_vocab(sentences)\noov = embed_intersection(vocab,embedding)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov[0:10]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Due to presence of capital letters some words might not be matching with the embeddings, we should convert them to lower case words"},{"metadata":{"trusted":true},"cell_type":"code","source":"df['lower_que'] = df['question_text'].apply(lambda x: x.lower())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"But making the words in lower case can lead to loss of information as there are words whose embeddings are present in upper case only.\n\nWe can fix this:\n* word.lower() takes the embedding of word if word.lower() doesn't have an embedding"},{"metadata":{"trusted":true},"cell_type":"code","source":"def fix_case(embedding,vocab):\n    count = 0\n    for word in vocab:\n        if word in embedding and word.lower() not in embedding:\n            embedding[word.lower()] = embedding[word]\n            count +=1\n    print(f'{count} no of words inserted into embedding')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov = embed_intersection(vocab,embedding)\nfix_case(embedding,vocab)\noov = embed_intersection(vocab,embedding)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()\noov[0:10]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can see that the punctuations and contractions are causing the mismatch between our vocab and embeddings"},{"metadata":{"trusted":true},"cell_type":"code","source":"contraction_mapping = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \n                       \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \n                       \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\",\n                       \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \n                       \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \n                       \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \n                       \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \n                       \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \n                       \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \n                       \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \n                       \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \n                       \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \n                       \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \n                       \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \n                       \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \n                       \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \n                       \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \n                       \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \n                       \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\",\n                       \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \n                       \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \n                       \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \n                       \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \n                       \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \n                       \"they'd've\": \"they would have\", \"they'll\": \"they will\", \n                       \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \n                       \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \n                       \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\",\n                       \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \n                       \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \n                       \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \n                       \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \n                       \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\",\n                       \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \n                       \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\",\n                       \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \n                       \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\n                       \"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \n                       \"you'd've\": \"you would have\", \"you'll\": \"you will\", \n                       \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\" }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def cont_map(embedding):\n    known = []\n    for cont in contraction_mapping:\n        if cont in embedding:\n            known.append(cont)\n    return known","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cont_map(embedding)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"It can be seen that not all contractions are present in the embeddings, we will now be replacing them"},{"metadata":{"trusted":true},"cell_type":"code","source":"def fix_cont(sentence,mapping):\n    sentence = str(sentence)\n    specials = [\"’\", \"‘\", \"´\", \"`\"]\n    for each in specials:\n        sentence = sentence.replace(each,\"'\")\n    sentence = \" \".join([mapping[word] if word in mapping else word for word in sentence.split(\" \")])\n    return sentence","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['fixed_question'] = df['lower_que'].apply(lambda x: fix_cont(x,contraction_mapping))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sentences = df['fixed_question'].progress_apply(lambda x: x.split()).values\nvocab = build_vocab(sentences)\noov = embed_intersection(vocab,embedding)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Now it's time to deal with puncuations and special characters**"},{"metadata":{"trusted":true},"cell_type":"code","source":"punct = \"/-'?!.,#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~\" + '\"\"“”’' + '∞θ÷α•à−β∅³π‘₹´°£€\\×™√²—–&'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's check which of the punct. are present in GloVe"},{"metadata":{"trusted":true},"cell_type":"code","source":"def unknown_punct(embed, punct):\n    unknown = []\n    for p in punct:\n        if p not in embed:\n            unknown.append(p)\n    return unknown","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Unknown Puctuations')\nprint(unknown_punct(embedding,punct))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1. Let's replace these with the known ones and for the known ones add a space between word and punctuation"},{"metadata":{"trusted":true},"cell_type":"code","source":"punct_mapping = {\"‘\": \"'\", \"₹\": \"e\", \"´\": \"'\", \"°\": \"\", \"€\": \"e\", \"™\": \"tm\", \"√\": \" sqrt \",\n                 \"×\": \"x\", \"²\": \"2\", \"—\": \"-\", \"–\": \"-\", \"’\": \"'\", \"_\": \"-\", \"`\": \"'\", \n                 '“': '\"', '”': '\"', '“': '\"', \"£\": \"e\", '∞': 'infinity', \n                 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', \n                 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }\n\ndef fix_punt(sentence,punct,mapping):\n    for p in mapping:\n        sentence = sentence.replace(p, mapping[p])\n    \n    for p in punct:\n        sentence = sentence.replace(p, f' {p} ')\n        \n    return sentence","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['fixed_question'] = df['fixed_question'].apply(lambda x: fix_punt(x,punct,punct_mapping))\n\nsentences = df['fixed_question'].progress_apply(lambda x: x.split()).values\nvocab = build_vocab(sentences)\noov = embed_intersection(vocab,embedding)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov[:50]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"What's still missing ?\n* Unknown words\n* Acronyms\n* Spelling mistake\n\nNow we will fix some of the common mispellings and acronyms to further improve our intersection of vocab with embeddings"},{"metadata":{},"cell_type":"markdown","source":"Some of the common mispellings"},{"metadata":{"trusted":true},"cell_type":"code","source":"mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', \n                'travelling': 'traveling', 'counselling': 'counseling', \n                'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', \n                'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', \n                'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', \n                'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', \n                'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much',\n                'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', \n                'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \n                \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', \n                'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018',\n                'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \n                \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', \n                'demonitization': 'demonetization', 'demonetisation': 'demonetization', \n                'pokémon': 'pokemon','redmi': 'company','oneplus':'company','bhakts':'Worshippers','…': '...'}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fix_spellings(sentence,mapping):\n    for word in mapping.keys():\n        sentence = sentence.replace(word,mapping[word])\n    return sentence","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['fixed_question'] = df['fixed_question'].apply(lambda x: fix_spellings(x,mispell_dict))\n\nsentences = df['fixed_question'].progress_apply(lambda x: x.split()).values\nvocab = build_vocab(sentences)\noov = embed_intersection(vocab,embedding)\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Applying all the changes to test and train set"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['fix_ques'] = df_train['question_text'].apply(lambda x: x.lower())\ndf_train['fix_ques'] = df_train['fix_ques'].apply(lambda x: fix_cont(x,contraction_mapping))\ndf_train['fix_ques'] = df_train['fix_ques'].apply(lambda x: fix_punt(x,punct,punct_mapping))\ndf_train['fix_ques'] = df_train['fix_ques'].apply(lambda x: fix_spellings(x,mispell_dict))\n\ndf_test['fix_ques'] = df_test['question_text'].apply(lambda x: x.lower())\ndf_test['fix_ques'] = df_test['fix_ques'].apply(lambda x: fix_cont(x,contraction_mapping))\ndf_test['fix_ques'] = df_test['fix_ques'].apply(lambda x: fix_punt(x,punct,punct_mapping))\ndf_test['fix_ques'] = df_test['fix_ques'].apply(lambda x: fix_spellings(x,mispell_dict))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now the embeddings covers almost all of our text data Yay!!\n\nIt's time to build the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"del df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab_size = len(vocab) + 1\nmax_len = 65","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def process_data(data):\n    t = Tokenizer(filters='')\n    t.fit_on_texts(data)\n    data = t.texts_to_sequences(data)\n    data = pad_sequences(data,maxlen = max_len)\n    return data, t.word_index,t","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X, word_index, tokenizer = process_data(df_train['fix_ques'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's make our train and CV sets"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ny = df_train['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.1,random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's make our embedding matrix"},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_embed_mat(embedding,word_index,vocab_size):\n    embds = np.stack(embedding.values())\n    emb_mean,emb_std = embds.mean(), embds.std()\n    embed_size = embds.shape[1]\n    word_index = word_index\n    embedding_matrix = np.random.normal(emb_mean,emb_std,(vocab_size,embed_size))\n    \n    for word,i in word_index.items():\n        if i>=vocab_size:\n            continue\n        embedding_vec = embedding.get(word)\n        if embedding_vec is not None:\n            embedding_matrix[i] = embedding_vec\n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embed_matrix = make_embed_mat(embedding,word_index,vocab_size)\ndel word_index\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embed_matrix.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import backend as K\n\ndef f1(y_true, y_pred):\n    def recall(y_true, y_pred):\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    \n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers import Dense, Embedding, CuDNNGRU, Bidirectional, GlobalAveragePooling1D\nfrom keras.layers import GlobalMaxPooling1D,concatenate,Input, Dropout\nfrom keras.optimizers import Adam\nfrom keras.models import Model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_model(embedding_matrix, embed_size=300, loss='binary_crossentropy'):\n    inp = Input(shape=(max_len,))\n    x = Embedding(input_dim=vocab_size,output_dim=embed_size,weights=[embedding_matrix],trainable=False)(inp)\n    x = Bidirectional(CuDNNGRU(128,return_sequences=True))(x)\n    x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)\n    avg_pl = GlobalAveragePooling1D()(x)\n    max_pl = GlobalMaxPooling1D()(x)\n    concat = concatenate([avg_pl,max_pl])\n    dense  = Dense(64, activation=\"relu\")(concat)\n    dense   = Dropout(rate = 0.7)(dense)\n    output = Dense(1, activation=\"sigmoid\")(dense)\n    \n    model = Model(inputs=inp, output=output)\n    model.compile(loss=loss,optimizer=Adam(lr=0.0001), metrics=['accuracy', f1])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = make_model(embed_matrix)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoints = ModelCheckpoint('model.h5',monitor='val_f1',mode='max',save_best_only='True',verbose=True)\nreduce_lr = ReduceLROnPlateau(monitor='val_f1', factor=0.1, patience=2, verbose=1, min_lr=0.000001)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 10\nbatch_size = 128\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(X_train, y_train, batch_size=batch_size, epochs=epochs, \n                    validation_data=[X_test, y_test], callbacks=[checkpoints, reduce_lr])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.figure(figsize=(12,8))\nplt.plot(history.history['acc'], label='Train Accuracy')\nplt.plot(history.history['val_acc'], label='Test Accuracy')\nplt.legend(('Train Acc', 'Val Acc'))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights('model.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred = model.predict(X_test,batch_size=512, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import f1_score\n\ndef tweak_threshold(pred, truth):\n    thresholds = []\n    scores = []\n    for thresh in np.arange(0.1, 0.501, 0.01):\n        thresh = np.round(thresh, 2)\n        thresholds.append(thresh)\n        score = f1_score(truth, (pred>thresh).astype(int))\n        scores.append(score)\n    return np.max(scores), thresholds[np.argmax(scores)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"score_val, threshold_val = tweak_threshold(pred, y_test)\n\nprint(f\"Scored {round(score_val, 4)} for threshold {threshold_val} with treated texts on validation data\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = tokenizer.texts_to_sequences(df_test['fix_ques'])\ntest = pad_sequences(test,maxlen = max_len)\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_test = model.predict(test,batch_size=512, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test['prediction'] = (pred_test>0.39).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = df_test.drop(labels=['question_text','fix_ques'],axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv(path_or_buf='submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}