{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Importations"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import re\nimport time\nimport gc\nimport random\nimport os\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nfrom tabulate import tabulate\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold\nfrom sklearn.metrics import f1_score, roc_auc_score\n\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\n\nimport torch\nimport torch.nn as nn\nimport torch.utils.data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#unzip embedding files\nimport zipfile\nimport shutil\nimport os\n\nif not os.path.exists('./embeddings') :\n    os.mkdir('./embeddings')\n    os.mkdir('./embeddings/glove.840B.300d/')\n    with zipfile.ZipFile('../input/quora-insincere-questions-classification/embeddings.zip', 'r') as z:\n        with z.open('glove.840B.300d/glove.840B.300d.txt') as zf, open('./embeddings/glove.840B.300d/glove.840B.300d.txt', 'wb') as f:\n            shutil.copyfileobj(zf, f)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"embed_size = 300 # how big is each word vector (given by the number of column in the embedding matrix)\nmax_features = 30000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 72 # max number of words in a question to use\n\nbatch_size = 512\ntrain_epochs = 5\n\nSEED = 1029\n\n\ndef seed_torch(seed=1029):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n\ndef clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        x = x.replace(punct, f' {punct} ')\n    return x\n\ndef clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    x = re.sub('[0-9]{1}', '#', x)\n    return x\n\nmispell_dict = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \n                \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \n                \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \n                \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \n                \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \n                \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\n                \"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\n                \"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \n                \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \n                \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \n                \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \n                \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \n                \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \n                \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \n                \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \n                \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \n                \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \n                \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \n                \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \n                \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\n                \"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \n                \"you've\": \"you have\", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', \n                'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', \n                'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', \n                'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', \n                'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', \n                'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', \n                'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\nspaces_re = re.compile('\\s')\nnumber_re, numbers_re = re.compile('(?<![0-9])[0-9]{1}(?![0-9])'), re.compile('[0-9]{2,}')\nCAPS_re = re.compile('[A-Z]{2,}')\npunct_re = re.compile('(\\?|!){2,}')\n\ndef prop_pattern(text, pattern_re):\n    return len(pattern_re.findall(text))/(len(spaces_re.findall(text))+1)\n\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n    return mispellings_re.sub(replace, text)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Statistic Analysis"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Downloading\n\ntrain_df = pd.read_csv(\"../input/quora-insincere-questions-classification/train.csv\")\ntest_df = pd.read_csv(\"../input/quora-insincere-questions-classification/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Check the proportion of sincere/insincere questions\n\ndescription = train_df['target'].value_counts(sort=False)\n\nprint(tabulate([['Count',description[0],description[1]], \n                ['Proportion',\"{0:.0%}\".format(description[0]/sum(description)),\"{0:.0%}\".format(description[1]/sum(description))]], \n                headers=['',description.index[0], description.index[1]]))\n\ndel description","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The dataset is strongly unbalanced <br>\nFor the rest of the code, we consider the test set to have the same distribution of sincere/insincere questions."},{"metadata":{"trusted":true},"cell_type":"code","source":"#Get a look at the number of words in each category\n\nwords_nb_df = train_df.copy()\nwords_nb_df['word_nb'] = words_nb_df[\"question_text\"].apply(lambda x : len(spaces_re.findall(x)))\n\nprint(tabulate([['Mean',words_nb_df[words_nb_df['target']==0]['word_nb'].mean(),words_nb_df[words_nb_df['target']==1]['word_nb'].mean()],\n               ['25%',words_nb_df[words_nb_df['target']==0]['word_nb'].quantile(0.25),words_nb_df[words_nb_df['target']==1]['word_nb'].quantile(0.25)],\n               ['Med',words_nb_df[words_nb_df['target']==0]['word_nb'].median(),words_nb_df[words_nb_df['target']==1]['word_nb'].median()],\n               ['75%',words_nb_df[words_nb_df['target']==0]['word_nb'].quantile(0.75),words_nb_df[words_nb_df['target']==1]['word_nb'].quantile(0.75)]], \n                headers=['','Sincere', 'Insincere']))\n\ndel words_nb_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The number of words is slightly different for sincere and insincere question. <br>\n_The average number of words is small compare to the padding._"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Get a look at the proportion of mispelled words, CAPS WORDS, punctuation, number in each category\n\npattern_list = [number_re, numbers_re, CAPS_re, punct_re, mispellings_re]\nlegends = [\"Number\", \"Numbers\", \"CAPS\", \"Punctuation\", \"Mispellings\"]\ndisplay = []\nsincere_df = train_df[train_df['target']==0][\"question_text\"].copy()\ninsincere_df = train_df[train_df['target']==1][\"question_text\"].copy()\n\nfor i in range(len(legends)) : \n    if legends[i] == \"Mispellings\" : \n        sincere_df = sincere_df.apply(lambda x: x.lower())\n        insincere_df = insincere_df.apply(lambda x: x.lower())\n        \n    metrics_sincere = sincere_df.apply(lambda x: prop_pattern(x, pattern_list[i]))\n    metrics_insincere = insincere_df.apply(lambda x: prop_pattern(x, pattern_list[i]))\n    display.append([legends[i] + \" - Prop(Words)\", \"{0:.1%}\".format(metrics_sincere.mean()),\"{0:.1%}\".format(metrics_insincere.mean())])\n    display.append([legends[i] + \" - Prop(Texts)\", \"{0:.1%}\".format(sum(metrics_sincere>0)/len(metrics_sincere)),\"{0:.1%}\".format(sum(metrics_insincere>0)/len(metrics_insincere))])\n    \nprint(tabulate(display, headers=['','Sincere', 'Insincere']))  \n\ndel sincere_df, insincere_df, metrics_sincere, metrics_insincere, train_df, test_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**The statistical analysis justifies/or not the following preprocess :**\n* Number cleaning : No significant differences\n* Numbers cleaning : No significant differences\n* Lower cleaning : No significant differences\n* Punctuation cleaning : Too few examples\n* Mispellings cleaning : Large gap between Sincere and Insincere categories. Cleaning mispellings could delete usefull information. For the basic version, we will not consider this effect."},{"metadata":{},"cell_type":"markdown","source":"## Pre-Processing"},{"metadata":{"trusted":true,"_uuid":"87a5fb67cdd061388568a1cd27250cecf5fb24a7"},"cell_type":"code","source":"def load_and_prec():\n    train_df = pd.read_csv(\"../input/quora-insincere-questions-classification/train.csv\")\n    test_df = pd.read_csv(\"../input/quora-insincere-questions-classification/test.csv\")\n    \n    # lower\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: x.lower())\n    test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: x.lower())\n    \n    # Clean the text\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_text(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: clean_text(x))\n    \n    # Clean numbers\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\n    \n    # Clean misspellings\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\n    \n    # Is ascii ?\n    \n    ## fill up the missing values\n    train_X = train_df[\"question_text\"].fillna(\"_##_\").values\n    test_X = test_df[\"question_text\"].fillna(\"_##_\").values\n    # Get rid of the missing values ?\n\n    ## Tokenize the sentences\n    tokenizer = Tokenizer(num_words=max_features)\n    tokenizer.fit_on_texts(list(train_X))\n    train_X = tokenizer.texts_to_sequences(train_X)\n    test_X = tokenizer.texts_to_sequences(test_X)\n\n    ## Pad the sentences \n    train_X = pad_sequences(train_X, maxlen=maxlen)\n    test_X = pad_sequences(test_X, maxlen=maxlen)\n\n    ## Get the target values\n    train_y = train_df['target'].values\n    \n    #shuffling the data (data is ordered)\n    np.random.seed(SEED)\n    trn_idx = np.random.permutation(len(train_X))\n\n    train_X = train_X[trn_idx]\n    train_y = train_y[trn_idx]\n    \n    return train_X, test_X, train_y, tokenizer.word_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pickle\nimport os\nfrom tqdm import tqdm\ntqdm.pandas()\n\nif not os.path.exists('./train_X.npy') : \n\n    start_time = time.time()\n\n    train_X, test_X, train_y, word_index = load_and_prec()\n\n    total_time = (time.time() - start_time) / 60\n\n    print(\"Took {:.2f} minutes\".format(total_time))\n    \n    np.save('./train_X', train_X, allow_pickle=True)\n    np.save('./test_X', test_X, allow_pickle=True)\n    np.save('./train_y', train_y, allow_pickle=True)\n\n    # saving\n    with open('./word_index.pickle', 'wb') as handle:\n        pickle.dump(word_index, handle, protocol=pickle.HIGHEST_PROTOCOL)\n\nelse : \n    train_X = np.load('./train_X.npy', allow_pickle=True)\n    test_X = np.load('./test_X.npy', allow_pickle=True)\n    train_y = np.load('./train_y.npy', allow_pickle=True)\n    \n    # loading\n    with open('./word_index.pickle', 'rb') as handle:\n        tokenizer = pickle.load(handle)\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Embeddings"},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_glove(word_index):\n    EMBEDDING_FILE = '../working/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index)+1)\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n            \n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not os.path.exists('./embedding_matrix.npy') :\n\n    start_time = time.time()\n\n    embedding_matrix = load_glove(word_index)\n    print(np.shape(embedding_matrix))\n\n    total_time = (time.time() - start_time) / 60\n\n    print(\"Took {:.2f} minutes\".format(total_time))\n    \n    np.save('./embedding_matrix', embedding_matrix, allow_pickle=True)\n\nelse : \n    \n    embedding_matrix = np.load('./embedding_matrix.npy', allow_pickle=True)\n    \ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Neural Networks"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Model Basic\n\nclass BasicNet(nn.Module):\n    def __init__(self):\n        super(BasicNet, self).__init__()\n        \n        #Parameters\n        hidden_size = 60\n        self.word_layer = 16\n        num_layers = 2\n        dropout = 0.1\n        \n        #Lookup table that stores words embeddings.\n        self.embedding = nn.Embedding(max_features, embed_size, padding_idx=0)\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        \n        #Neural Network\n        self.lstm = nn.LSTM(input_size=embed_size, hidden_size=hidden_size, num_layers=num_layers, dropout=dropout, bidirectional=True, batch_first=True)\n        self.sigmoid = nn.Sigmoid()\n        self.linear_word = nn.Linear(hidden_size*2, self.word_layer) #Bidirectionnel (*2)\n        self.linear_sentence = nn.Linear(self.word_layer*maxlen, 1)\n        self.dropout = nn.Dropout(dropout)\n        \n    def forward(self, x):\n        h_embedding = self.embedding(x)\n        h_lstm, _ = self.lstm(h_embedding)\n        h_lstm = self.sigmoid(h_lstm)\n        \n        result = self.sigmoid(self.linear_word(h_lstm))\n        result = self.dropout(result)\n        result = result.reshape((len(result),-1,))\n        \n        result = self.linear_sentence(result)\n        \n        return result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6c179e5478f1ac92b69020fb875bf3d8154bff3"},"cell_type":"code","source":"def sigmoid(x):\n    return 1 / (1 + np.exp(-x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ad4bf11f982e199ff5d64004eeedd9f3d3d454c"},"cell_type":"code","source":"#Search the best threshold for the training set\ndef threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in tqdm([i * 0.01 for i in range(100)]):\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Basic Model\n\ntrain_epochs = 5\n\nsplits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=SEED).split(train_X, train_y))\n\ntrain_preds = np.zeros((len(train_X)))\ntest_preds = np.zeros((len(test_X)))\n\nseed_torch(SEED)\n\nx_test_cuda = torch.tensor(test_X, dtype=torch.long).cuda()\ntest = torch.utils.data.TensorDataset(x_test_cuda)\ntest_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)\n\nfor i, (train_idx, valid_idx) in enumerate(splits):\n    x_train_fold = torch.tensor(train_X[train_idx], dtype=torch.long).cuda()\n    y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32).cuda()\n    x_val_fold = torch.tensor(train_X[valid_idx], dtype=torch.long).cuda()\n    y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32).cuda()\n    \n    model = BasicNet()\n    model.cuda()\n    \n    #Loss combines a Sigmoid layer and the Entropy Loss\n    loss_fn = torch.nn.BCEWithLogitsLoss(weight=None, reduction=\"sum\") \n    optimizer = torch.optim.Adam(model.parameters()) #Adam optimizes the gradient descent\n    \n    train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)\n    valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)\n    \n    train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n    valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)\n    \n    print(f'Fold {i + 1}')\n    \n    for epoch in range(train_epochs):\n        start_time = time.time()\n        \n        model.train() #Training mode\n        avg_loss = 0.\n        for x_batch, y_batch in tqdm(train_loader, disable=True):\n            y_pred = model(x_batch)\n            loss = loss_fn(y_pred, y_batch)\n            optimizer.zero_grad() #The optimizer needs to be reset\n            loss.backward() #Computes the direction to follow for minimizing the loss\n            optimizer.step() #Updates the parameters of the NN\n            avg_loss += loss.item() / len(train_loader)\n        \n        model.eval() # Evaluation mode\n        valid_preds_fold = np.zeros((x_val_fold.size(0)))\n        test_preds_fold = np.zeros(len(test_X))\n        avg_val_loss = 0.\n        \n        for i, (x_batch, y_batch) in enumerate(valid_loader):\n            y_pred = model(x_batch).detach() #Deep copy\n            avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n            valid_preds_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n        \n        elapsed_time = time.time() - start_time \n        print('Epoch {}/{} \\t loss={:.4f} \\t val_loss={:.4f} \\t time={:.2f}s'.format(\n            epoch + 1, train_epochs, avg_loss, avg_val_loss, elapsed_time))\n        \n    #Dataset is too large to be tested without batch\n    for j, (x_batch,) in enumerate(test_loader):\n        y_pred = model(x_batch).detach()\n\n        test_preds_fold[j * batch_size:(j+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n\n    train_preds[valid_idx] = valid_preds_fold\n    test_preds += test_preds_fold / len(splits) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission"},{"metadata":{"trusted":true,"_uuid":"f0c7655686bd19130f7f2c8074b2ec44b1451848"},"cell_type":"code","source":"search_result = threshold_search(train_y, train_preds)\nsearch_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c3581f74ae694eb07182e2a23c9db8f01a78f1ba"},"cell_type":"code","source":"sub = pd.read_csv('../input/quora-insincere-questions-classification/sample_submission.csv')\nsub.prediction = test_preds > search_result['threshold']\nsub['prediction'] = sub['prediction'].apply(lambda x : int(x))\nsub.to_csv(\"./submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"381fef40c02ac313759eda569394d7413f35e7fe"},"cell_type":"code","source":"!head ./submission.csv","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}