{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"### General information\n\nIn this kernel I rewrite my previous kernel:\n* changed preprocessing a bit;\n* wrote one single function for training the model;\n* use different algorithm for finding optimal threshold;\n* use three embeddings;\n* a bit different architecture"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\n\nfrom nltk.tokenize import TweetTokenizer\nimport datetime\nfrom scipy import stats\nfrom scipy.sparse import hstack, csr_matrix\nfrom sklearn.model_selection import train_test_split, cross_val_score\n\nfrom collections import Counter\nfrom nltk.corpus import stopwords\nfrom nltk.util import ngrams\nimport time\npd.set_option('max_colwidth',400)\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom sklearn.preprocessing import OneHotEncoder\nfrom tqdm import tqdm_notebook\nfrom sklearn.metrics import f1_score\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\nfrom torch.autograd import Variable\nimport torch.utils.data\nimport random\nimport warnings\nwarnings.filterwarnings(\"ignore\", message=\"F-score is ill-defined and being set to 0.0 due to no predicted samples.\")\nnp.seterr(divide='ignore')\nimport re\nimport os\nfrom torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR\n\nfrom sklearn.metrics import roc_curve, precision_recall_curve, f1_score\nfrom sklearn.model_selection import RepeatedStratifiedKFold, StratifiedKFold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bbbe7de9d9a8ac8bfed3ab4a5c4641b3edd3e51a"},"cell_type":"code","source":"path = '../input/'\ntrain = pd.read_csv(os.path.join(path,\"train.csv\"))\ntest = pd.read_csv(os.path.join(path,\"test.csv\"))\nsub = pd.read_csv(os.path.join(path,'sample_submission.csv'))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fc1d6001a78fe58d154a018096df0dab6f22c637"},"cell_type":"markdown","source":"## Data overview\n\nThis is a kernel competition, where we can't use external data. As a result we can use only train and test datasets as well as embeddings which were provided by organizers."},{"metadata":{"trusted":true,"_uuid":"5b99f028bbf2ecad3e6bd9a95f014e912e7af67b"},"cell_type":"code","source":"print('Available embeddings:',  os.listdir(os.path.join(path,\"embeddings/\")))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd4bbfb908d2e0c549b8b3054c60b8399ff34552"},"cell_type":"code","source":"train[\"target\"].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6ab6eee87656aff1d1d92487c85956fe55e66a93"},"cell_type":"markdown","source":"We have a seriuos disbalance - only ~6% of data are positive. No wonder the metric for the competition is f1-score."},{"metadata":{"trusted":true,"_uuid":"b07a44cc10e1a396ceac97e3af5fd7c266b17b28"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5b3eb9795553e4b810a3c0e17bd3459c0e7881cd"},"cell_type":"markdown","source":"In the dataset we have only texts of questions."},{"metadata":{"trusted":true,"_uuid":"54a553b7e92a2a0a3d491ccf92b437011b813c85"},"cell_type":"code","source":"print('Average word length of questions in train is {0:.0f}.'.format(np.mean(train['question_text'].apply(lambda x: len(x.split())))))\nprint('Average word length of questions in test is {0:.0f}.'.format(np.mean(test['question_text'].apply(lambda x: len(x.split())))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7861669f72f36145c25911926a51bc688f51d473"},"cell_type":"code","source":"print('Max word length of questions in train is {0:.0f}.'.format(np.max(train['question_text'].apply(lambda x: len(x.split())))))\nprint('Max word length of questions in test is {0:.0f}.'.format(np.max(test['question_text'].apply(lambda x: len(x.split())))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b1303137eb44cc0de3329921751c48209037562"},"cell_type":"code","source":"print('Average character length of questions in train is {0:.0f}.'.format(np.mean(train['question_text'].apply(lambda x: len(x)))))\nprint('Average character length of questions in test is {0:.0f}.'.format(np.mean(test['question_text'].apply(lambda x: len(x)))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b82f366ef8b46b0115e9940c7966849023545733"},"cell_type":"markdown","source":"As we can see on average questions in train and test datasets are similar, but there are quite long questions in train dataset."},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"4f1838e686d67670bb6429b8b43619a69803fde8"},"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n\ndef clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        x = x.replace(punct, f' {punct} ')\n    return x\n\ndef clean_numbers(x):\n    if bool(re.search(r'\\d', x)):\n        x = re.sub('[0-9]{5,}', '#####', x)\n        x = re.sub('[0-9]{4}', '####', x)\n        x = re.sub('[0-9]{3}', '###', x)\n        x = re.sub('[0-9]{2}', '##', x)\n    return x\n\nmispell_dict = {\"aren't\" : \"are not\",\n                \"can't\" : \"cannot\",\n                \"couldn't\" : \"could not\",\n                \"didn't\" : \"did not\",\n                \"doesn't\" : \"does not\",\n                \"don't\" : \"do not\",\n                \"hadn't\" : \"had not\",\n                \"hasn't\" : \"has not\",\n                \"haven't\" : \"have not\",\n                \"he'd\" : \"he would\",\n                \"he'll\" : \"he will\",\n                \"he's\" : \"he is\",\n                \"i'd\" : \"I would\",\n                \"i'd\" : \"I had\",\n                \"i'll\" : \"I will\",\n                \"i'm\" : \"I am\",\n                \"isn't\" : \"is not\",\n                \"it's\" : \"it is\",\n                \"it'll\":\"it will\",\n                \"i've\" : \"I have\",\n                \"let's\" : \"let us\",\n                \"mightn't\" : \"might not\",\n                \"mustn't\" : \"must not\",\n                \"shan't\" : \"shall not\",\n                \"she'd\" : \"she would\",\n                \"she'll\" : \"she will\",\n                \"she's\" : \"she is\",\n                \"shouldn't\" : \"should not\",\n                \"that's\" : \"that is\",\n                \"there's\" : \"there is\",\n                \"they'd\" : \"they would\",\n                \"they'll\" : \"they will\",\n                \"they're\" : \"they are\",\n                \"they've\" : \"they have\",\n                \"we'd\" : \"we would\",\n                \"we're\" : \"we are\",\n                \"weren't\" : \"were not\",\n                \"we've\" : \"we have\",\n                \"what'll\" : \"what will\",\n                \"what're\" : \"what are\",\n                \"what's\" : \"what is\",\n                \"what've\" : \"what have\",\n                \"where's\" : \"where is\",\n                \"who'd\" : \"who would\",\n                \"who'll\" : \"who will\",\n                \"who're\" : \"who are\",\n                \"who's\" : \"who is\",\n                \"who've\" : \"who have\",\n                \"won't\" : \"will not\",\n                \"wouldn't\" : \"would not\",\n                \"you'd\" : \"you would\",\n                \"you'll\" : \"you will\",\n                \"you're\" : \"you are\",\n                \"you've\" : \"you have\",\n                \"'re\": \" are\",\n                \"wasn't\": \"was not\",\n                \"we'll\":\" will\",\n                \"didn't\": \"did not\",\n                \"tryin'\":\"trying\",\n               '\\u200b': '',\n                '…': '',\n                '\\ufeff': '',\n                'करना': '',\n                'है': ''}\n\nfor coin in ['Litecoin', 'altcoin', 'altcoins', 'coinbase', 'litecoin', 'Unocoin', 'Dogecoin', 'cryptocoin', 'Altcoins', 'filecoin', 'Altcoin', 'cryptocoins',\n             'Altacoin', 'Dentacoin', 'Bytecoin', 'Siacoin', 'Onecoin', 'dogecoin', 'unocoin', 'siacoin', 'litecoins', 'Filecoin', 'Buyucoin', 'Litecoins',\n             'Laxmicoin', 'shtcoins', 'Sweatcoin', 'Skycoin', 'vitrocoin', 'Monacoin', 'Litcoin', 'reddcoin', 'freebitcoin', 'Namecoin', 'plexcoin', 'Onecoins',\n             'daikicoin', 'Gainbitcoin', 'Gatecoin', 'Plexcoin', 'peercoin', 'coinsecure', 'dogecoins', 'cointries', 'Zcoin', 'genxcoin', 'Frazcoin', 'frazcoin',\n             'coinify', 'Nagricoin', 'OKcoin', 'Presscoins', 'Dagcoin', 'batcoin', 'Spectrocoin', 'Travelflexcoin', 'ecoin', 'Minexcoin', 'Kashhcoin', 'coinone',\n             'octacoin', 'coinsides', 'zabercoin', 'ADZcoin', 'cyptocoin', 'bitecoin', 'Bitecoin', 'Emercoin', 'tegcoin', 'flipcoin', 'Gridcoin', 'Facecoin',\n             'Ravencoins', 'digicoin', 'bitcoincash', 'Vitrocoin', 'Livecoin', 'dashcoin', 'Fedcoin', 'litcoins', 'Webcoin', 'coinspot', 'bitoxycoin', 'peercoins',\n             'Ucoin', 'ALTcoins', 'coincidece', 'dagcoin', 'Giracoin', 'coincheck', 'Swisscoin', 'butcoin', 'neocoin', 'mintcoin', 'Myriadcoin', 'Viacoin', 'jiocoin',\n             'Potcoin', 'bibitcoin', 'gainbitcoin', 'altercoins', 'coinburn', 'Kodakcoin', 'Bcoin', 'Kucoin', 'Operacoin', 'Lomocoin', 'dentacoin', 'Nyancoin',\n             'Jiocoin', 'Indicoin', 'coinsidered', 'Vertcoin', 'Maidsafecoin', 'coindelta', 'coinfirm', 'coinvest', 'bixcoin', 'litcoin', 'Dogecoins', 'Unicoin',\n             'Rothscoin', 'localbitcoins', 'groestlcoin', 'sibcoin', 'Travelercoin', 'Vericoin', 'bytecoin', 'Bananacoin', 'PACcoin']:\n    mispell_dict[coin] = 'bitcoin'\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n    return mispellings_re.sub(replace, text)\n\n# Clean the text\ntrain[\"question_text\"] = train[\"question_text\"].apply(lambda x: clean_text(x.lower()))\ntest[\"question_text\"] = test[\"question_text\"].apply(lambda x: clean_text(x.lower()))\n\n# Clean numbers\ntrain[\"question_text\"] = train[\"question_text\"].apply(lambda x: clean_numbers(x))\ntest[\"question_text\"] = test[\"question_text\"].apply(lambda x: clean_numbers(x))\n\n# Clean speelings\ntrain[\"question_text\"] = train[\"question_text\"].apply(lambda x: replace_typical_misspell(x))\ntest[\"question_text\"] = test[\"question_text\"].apply(lambda x: replace_typical_misspell(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9d081b2f0d46faf01a943c309568c27f92462f94"},"cell_type":"code","source":"max_features = 120000\ntk = Tokenizer(lower = True, filters='', num_words=max_features)\nfull_text = list(train['question_text'].values) + list(test['question_text'].values)\ntk.fit_on_texts(full_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33929a60e1872b73e40424daa1178a3d8fbf8f5a"},"cell_type":"code","source":"train_tokenized = tk.texts_to_sequences(train['question_text'].fillna('missing'))\ntest_tokenized = tk.texts_to_sequences(test['question_text'].fillna('missing'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4330f6c01064b5fda4ca9661dc4f1cefb439cf75"},"cell_type":"code","source":"train['question_text'].apply(lambda x: len(x.split())).plot(kind='hist');\nplt.yscale('log');\nplt.title('Distribution of question text length in characters');","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"05c7e2c63afb343b64835432721a42f81dd53626"},"cell_type":"markdown","source":"We can see that most of the questions are 40 words long or shorter. Let's try having sequence length equal to 70 for now."},{"metadata":{"trusted":true,"_uuid":"15d7e3a2fb5fe5fb38da45ca3d0bd13c82b7eda5"},"cell_type":"code","source":"max_len = 72\nX_train = pad_sequences(train_tokenized, maxlen=max_len)\nX_test = pad_sequences(test_tokenized, maxlen=max_len)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"50ade6e2c4dab8ee4add2a9c080f132e012cedc7"},"cell_type":"markdown","source":"### Preparing data for Pytorch\n\nOne of main differences from Keras is preparing data.\nPytorch requires special dataloaders. I'll write a class for it.\n\nAt first I'll append padded texts to original DF."},{"metadata":{"trusted":true,"_uuid":"31839c56ed1d7945decf176c97b49f0205cc40af"},"cell_type":"code","source":"y_train = train['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"69d325ea77439697e53143adef3a0da58e7f31f2"},"cell_type":"code","source":"def sigmoid(x):\n    return 1 / (1 + np.exp(-x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cca23aa2dc1207a6f08212ad8d87f8182e567e0e"},"cell_type":"code","source":"splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=10).split(X_train, y_train))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6d336aa73a5b5646ae80ef2d0fed5710f476718e"},"cell_type":"markdown","source":"### Embeddings"},{"metadata":{"trusted":true,"_uuid":"744ee4a1fbc66cb47aae6a18f829dea284b860c9"},"cell_type":"code","source":"embed_size = 300\nembedding_path = \"../input/embeddings/glove.840B.300d/glove.840B.300d.txt\"\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembedding_index = dict(get_coefs(*o.split(\" \")) for o in open(embedding_path, encoding='utf-8', errors='ignore'))\n# all_embs = np.stack(embedding_index.values())\n# emb_mean,emb_std = all_embs.mean(), all_embs.std()\nemb_mean,emb_std = -0.005838499, 0.48782197\nword_index = tk.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words + 1, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embedding_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc553dd7ddbd3a81ed1a39084fdb4982b5887971"},"cell_type":"code","source":"embedding_path = \"../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt\"\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembedding_index = dict(get_coefs(*o.split(\" \")) for o in open(embedding_path, encoding='utf-8', errors='ignore') if len(o)>100)\n# all_embs = np.stack(embedding_index.values())\n# emb_mean,emb_std = all_embs.mean(), all_embs.std()\nemb_mean,emb_std = -0.0053247833, 0.49346462\nembedding_matrix1 = np.random.normal(emb_mean, emb_std, (nb_words + 1, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embedding_index.get(word)\n    if embedding_vector is not None: embedding_matrix1[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bbaa0c09fa7f128ad5e5b2b11254585f66b5dfe2"},"cell_type":"code","source":"embedding_path = \"../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec\"\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembedding_index = dict(get_coefs(*o.split(\" \")) for o in open(embedding_path, encoding='utf-8', errors='ignore') if len(o)>100)\n# all_embs = np.stack(embedding_index.values())\n# emb_mean,emb_std = all_embs.mean(), all_embs.std()\n# print(emb_mean,emb_std)\nemb_mean,emb_std = -0.0033469985, 0.109855495\nembedding_matrix2 = np.random.normal(emb_mean, emb_std, (nb_words + 1, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embedding_index.get(word)\n    if embedding_vector is not None: embedding_matrix2[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"333429eaa0c7c057769518c8f1b0fefbb68f1d1d"},"cell_type":"code","source":"embedding_matrix = np.mean([embedding_matrix, embedding_matrix1, embedding_matrix2], axis=0)\nprint(embedding_matrix.shape)\ndel embedding_matrix1, embedding_matrix2","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9b1165a1818467dadb501149b608e40630176a36"},"cell_type":"markdown","source":"### Model"},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"bc56c0802f5e9735a74bc9d4dcd04a05429f3684"},"cell_type":"code","source":"class Attention(nn.Module):\n    def __init__(self, feature_dim, step_dim, bias=True, **kwargs):\n        super(Attention, self).__init__(**kwargs)\n        \n        self.supports_masking = True\n\n        self.bias = bias\n        self.feature_dim = feature_dim\n        self.step_dim = step_dim\n        self.features_dim = 0\n        \n        weight = torch.zeros(feature_dim, 1)\n        nn.init.xavier_uniform_(weight)\n        self.weight = nn.Parameter(weight)\n        \n        if bias:\n            self.b = nn.Parameter(torch.zeros(step_dim))\n        \n    def forward(self, x, mask=None):\n        feature_dim = self.feature_dim\n        step_dim = self.step_dim\n\n        eij = torch.mm(\n            x.contiguous().view(-1, feature_dim), \n            self.weight\n        ).view(-1, step_dim)\n        \n        if self.bias:\n            eij = eij + self.b\n            \n        eij = torch.tanh(eij)\n        a = torch.exp(eij)\n        \n        if mask is not None:\n            a = a * mask\n\n        a = a / torch.sum(a, 1, keepdim=True) + 1e-10\n\n        weighted_input = x * torch.unsqueeze(a, -1)\n        return torch.sum(weighted_input, 1)\n    \nclass NeuralNet(nn.Module):\n    def __init__(self):\n        super(NeuralNet, self).__init__()\n        \n        hidden_size = 128\n        \n        self.embedding = nn.Embedding(max_features, embed_size)\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        self.embedding.weight.requires_grad = False\n        \n        self.embedding_dropout = nn.Dropout2d(0.1)\n        self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)\n        self.gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True)\n        \n        self.lstm_attention = Attention(hidden_size*2, max_len)\n        self.gru_attention = Attention(hidden_size*2, max_len)\n        \n        self.linear = nn.Linear(1536, 256)\n        self.linear1 = nn.Linear(256, 32)\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(0.1)\n        self.out = nn.Linear(32, 1)\n        \n    def forward(self, x):\n        h_embedding = self.embedding(x)\n        h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))\n        \n        h_lstm, _ = self.lstm(h_embedding)\n        h_gru, _ = self.gru(h_lstm)\n        \n        h_lstm_atten = self.lstm_attention(h_lstm)\n        h_gru_atten = self.gru_attention(h_gru)\n        \n        avg_pool_g = torch.mean(h_gru, 1)\n        max_pool_g, _ = torch.max(h_gru, 1)\n        \n        avg_pool_l = torch.mean(h_lstm, 1)\n        max_pool_l, _ = torch.max(h_lstm, 1)\n        \n        conc = torch.cat((h_lstm_atten, h_gru_atten, avg_pool_g, max_pool_g, avg_pool_l, max_pool_l), 1)\n        conc = self.relu(self.linear(conc))\n        conc = self.dropout(conc)\n        conc = self.relu(self.linear1(conc))\n        out = self.out(conc)\n        \n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8bd0f7c0d542d6cfe172b8bb880b50e5e17fb884"},"cell_type":"code","source":"m = NeuralNet()\nx_test_cuda = torch.tensor(X_test, dtype=torch.long).cuda()\ntest = torch.utils.data.TensorDataset(x_test_cuda)\nbatch_size = 512\ntest_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)\nseed=1029\ndef seed_everything(seed=1234):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything(seed)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0a726a4e04bd7167351f8d0405f228a23d0b14f1"},"cell_type":"markdown","source":"### Training fuction"},{"metadata":{"trusted":true,"_uuid":"92edb5957257d88fbc06bc36cebcdf61a25bdcbe","_kg_hide-input":true},"cell_type":"code","source":"def train_model_full(X_train=X_train, y_train=y_train, splits=splits, n_epochs=5, batch_size=batch_size, validate=False):\n    train_preds = np.zeros(len(X_train))\n    test_preds = np.zeros((len(test), len(splits)))\n    scores = []\n    for i, (train_idx, valid_idx) in enumerate(splits):\n        print(f'Fold {i + 1}. {time.ctime()}')\n        x_train_fold = torch.tensor(X_train[train_idx], dtype=torch.long).cuda()\n        y_train_fold = torch.tensor(y_train[train_idx, np.newaxis], dtype=torch.float32).cuda()\n        x_val_fold = torch.tensor(X_train[valid_idx], dtype=torch.long).cuda()\n        y_val_fold = torch.tensor(y_train[valid_idx, np.newaxis], dtype=torch.float32).cuda()\n        \n        seed_everything(seed + i)\n        model = NeuralNet()\n        model.cuda()\n        optimizer = torch.optim.Adam(model.parameters())\n        # scheduler = StepLR(optimizer, step_size=3, gamma=0.1)\n        \n        loss_fn = torch.nn.BCEWithLogitsLoss(reduction='mean').cuda()\n        \n        train_dataset = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)\n        valid_dataset = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)\n\n        train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n        valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)\n        \n        best_f1 = 0\n        best_model_name = ''\n        \n        for epoch in range(n_epochs):\n            print()\n            print(f'Epoch {epoch}. {time.ctime()}')\n            model.train()\n            avg_loss = 0.\n\n            for x_batch, y_batch in train_loader:\n                # print(x_batch.shape)\n                y_pred = model(x_batch)\n                loss = loss_fn(y_pred, y_batch)\n\n                optimizer.zero_grad()\n                loss.backward()\n                optimizer.step()\n                avg_loss += loss.item() / len(train_loader)\n\n            model.eval()\n\n            valid_preds = np.zeros((x_val_fold.size(0)))\n\n            if validate:\n                avg_val_loss = 0.\n                for j, (x_batch, y_batch) in enumerate(valid_loader):\n                    y_pred = model(x_batch).detach()\n\n                    avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n                    valid_preds[j * batch_size:(j+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n\n                best_th, score = scoring(y_val_fold.cpu().numpy(), valid_preds, verbose=True)\n\n#                 if score > best_f1:\n#                     best_f1 = score\n#                     torch.save(model.state_dict(), f'model_{epoch}.pt')\n#                     best_model_name = f'model_{epoch}.pt'\n#                 else:\n#                     print('Stopping training on this fold')\n#                     break\n        \n#         if score < best_f1:\n#             checkpoint = torch.load(best_model_name)\n#             model.load_state_dict(checkpoint)\n#             model.eval()\n\n        valid_preds = np.zeros((x_val_fold.size(0)))\n\n        avg_val_loss = 0.\n        for j, (x_batch, y_batch) in enumerate(valid_loader):\n            y_pred = model(x_batch).detach()\n\n            avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n            valid_preds[j * batch_size:(j+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n        best_th, score = scoring(y_val_fold.cpu().numpy(), valid_preds, verbose=True)\n\n        scores.append(score)\n\n        test_preds_fold = np.zeros((len(test_loader.dataset)))\n\n        for j, (x_batch,) in enumerate(test_loader):\n            y_pred = model(x_batch).detach()\n\n            test_preds_fold[j * batch_size:(j+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n\n        train_preds[valid_idx] = valid_preds\n        test_preds[:, i] = test_preds_fold\n    print(f'Finished training at {time.ctime()}')\n    print(f'Mean validation f1-score: {np.mean(scores)}. Std: {np.std(scores)}')\n    \n    return train_preds, test_preds","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e1993fc8a6da7b05a559e0c79897ed460100450f"},"cell_type":"markdown","source":"### Searching for optimal threshold"},{"metadata":{"trusted":true,"_uuid":"e54cb26ad88568cedc3b1ded5c3afc8e6c3766d4","_kg_hide-input":true},"cell_type":"code","source":"def scoring(y_true, y_proba, verbose=True):\n    # https://www.kaggle.com/c/quora-insincere-questions-classification/discussion/76391\n    \n    def threshold_search1(y_true, y_proba):\n        precision , recall, thresholds = precision_recall_curve(y_true, y_proba)\n        thresholds = np.append(thresholds, 1.001) \n        F = 2 / (1/precision + 1/recall)\n        best_score = np.max(F)\n        best_th = thresholds[np.argmax(F)]\n        return best_th \n\n    rkf = RepeatedStratifiedKFold(n_splits=5, n_repeats=10)\n    # rkf = StratifiedKFold(n_splits=5)\n\n    scores = []\n    ths = []\n    for train_index, test_index in rkf.split(y_true, y_true):\n        y_prob_train, y_prob_test = y_proba[train_index], y_proba[test_index]\n        y_true_train, y_true_test = y_true[train_index], y_true[test_index]\n\n        # determine best threshold on 'train' part \n        best_threshold = threshold_search1(y_true_train, y_prob_train)\n\n        # use this threshold on 'test' part for score \n        sc = f1_score(y_true_test, (y_prob_test >= best_threshold).astype(int))\n        scores.append(sc)\n        ths.append(best_threshold)\n\n    best_th = np.mean(ths)\n    score = np.mean(scores)\n\n    if verbose: print(f'Best threshold: {np.round(best_th, 4)}, Score: {np.round(score,5)}')\n\n    return best_th, score","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"_kg_hide-output":false,"_uuid":"51b02ac65e14751f73e6f0e8b74e5792b0cdfb22"},"cell_type":"markdown","source":"### Training"},{"metadata":{"trusted":true,"_uuid":"773da7218b17f950a1329a80bc2654f0b1b39e49","scrolled":true},"cell_type":"code","source":"train_preds, test_preds = train_model_full(X_train=X_train, y_train=y_train, splits=splits, n_epochs=5, batch_size=batch_size, validate=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d78af890ce743532cbe61e7dc67ef562f4aad4c1"},"cell_type":"code","source":"best_th, score = scoring(y_train, train_preds)\nsub['prediction'] = test_preds.mean(1) > best_th\nsub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}