{"cells":[{"metadata":{"trusted":true,"_uuid":"3b8db4a801e3eff5ba2f9266cff76cd73b19e0b2"},"cell_type":"code","source":"%reload_ext autoreload\n%autoreload 2","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"958216cee1e8f878260be62ccec71aa452871f16"},"cell_type":"markdown","source":"## Library"},{"metadata":{"trusted":true,"_uuid":"277abf3307bf4abe35ec4ae21650ab0cc430bdd8"},"cell_type":"code","source":"#from src.util import *\n#from src.data import *\n#from src.pmodel import *\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6054719f99c5cba8fd2130183642474cd8b1e4bb"},"cell_type":"code","source":"import gc\nimport os\nimport pickle\nimport random\nimport re\nimport sklearn.metrics\nimport sklearn.model_selection\nimport time\nimport warnings\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom collections import OrderedDict\nfrom tqdm import tqdm, tqdm_notebook\n\nKAGGLE_RUN = (not os.path.exists('/opt/conda/home/.history'))\nif KAGGLE_RUN: print('Kaggle run')\n\npd.options.display.float_format = '{:.6f}'.format\n\n\ndef tf_seed_everything(seed):\n    import tensorflow as tf\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.set_random_seed(seed)\n\ndef f1_curve(target, preds, t_min=0.01, t_max=0.99, steps=99):\n    curve = {}\n    for t in np.linspace(t_min, t_max, steps):\n        with warnings.catch_warnings():\n            warnings.simplefilter(\"ignore\")\n            curve[t] = sklearn.metrics.f1_score(target, preds >= t)\n    return pd.Series(curve).sort_index()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"517677d6446af0de10cb97877c87d6b0e8b9b155"},"cell_type":"code","source":"import functools\nimport multiprocessing\nimport os\nimport pickle\nimport random\nimport re\nimport sklearn.preprocessing\nimport unicodedata\n\nimport numpy as np\nimport pandas as pd\n\n\nclass QuoraData:\n    def __init__(self):\n        self.paths = {\n            'glove': '../input/embeddings/glove.840B.300d/glove.840B.300d.txt',\n            'news': '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin',\n            'paragram': '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt',\n            'wiki': '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec',\n        }\n\n    def glove(self): return QuoraEmbedding(self.paths['glove'])\n    def news(self): return QuoraEmbedding(self.paths['news'])\n    def paragram(self): return QuoraEmbedding(self.paths['paragram'])\n    def wiki(self): return QuoraEmbedding(self.paths['wiki'])\n\n    def convert_start(self, embeddings):\n        \"\"\"Start conversion of specified embedding names to .npy format in background.\"\"\"\n\n        self.convert_pids = []\n\n        for name in embeddings:\n            path = self.paths[name]\n            if path.endswith('.npy'): continue\n            if os.path.exists(path + '.npy'): continue\n            if os.path.exists(name + '.npy'): continue\n            if KAGGLE_RUN:\n                out_path = name + '.npy'\n            else:\n                out_path = path + '.npy'\n            print(f'Converting {path} -> {out_path}', flush=True)\n            pid = os.fork()\n            if pid == 0:\n                emb = QuoraEmbedding(path)\n                emb.save_npy(out_path)\n                os._exit(0)\n\n            self.convert_pids.append(pid)\n            self.paths[name] = out_path\n\n    def convert_wait(self):\n        \"\"\"Wait for .npy conversion to finish.\"\"\"\n        for pid in self.convert_pids:\n            try:\n                os.waitpid(pid, 0)\n            except:\n                pass\n\n    def read_train(self):\n        return pd.read_csv('../input/train.csv')\n\n    def read_test(self):\n        return pd.read_csv('../input/test.csv')\n\n    def read_input(self):\n        return pd.concat([self.read_train(), self.read_test()], axis=0, copy=False, sort=False, ignore_index=True)\n\n\nclass QuoraEmbedding:\n    \"\"\"Quora's pretrained embeddings loader.\"\"\"\n\n    def __init__(self, filename, vectors=None, vocab=None):\n        self.filename = filename\n        if vectors is None:\n            vectors, vocab = self._read(filename)\n        self.vectors = vectors\n        self.index2word = vocab\n        self.name = re.findall('^\\w+', os.path.basename(filename))[0]\n        self.shape = self.vectors.shape\n        self.num_words, self.dim = self.shape\n        assert len(self.index2word) == self.num_words\n        # On collisions, pick the earliest (more frequent) vector's index\n        self.word2index = { self.index2word[i]: i for i in reversed(range(self.num_words)) }\n        self.lword2index = { self.index2word[i].lower(): i for i in reversed(range(self.num_words)) }\n\n    def __repr__(self):\n        return f'QuoraEmbedding({self.name}, {self.shape[0]}x{self.shape[1]}, {self.vectors.nbytes/(1024**3):.1f}GiB)'\n\n    def lookup(self, word, lower=False):\n        if lower:\n            idx = self.lword2index.get(word.lower(), -1)\n        else:\n            idx = self.word2index.get(word, -1)\n        return self.vectors[idx] if idx != -1 else None\n\n    @functools.lru_cache(1)\n    def mean(self): return self.vectors.mean()\n\n    @functools.lru_cache(1)\n    def std(self): return self.vectors.std()\n\n    def save_npy(self, filename):\n        assert filename.endswith('.npy')\n        np.save(filename, self.vectors)\n        with open(filename[:-4] + '.vocab', 'wb') as fp:\n            pickle.dump(self.index2word, fp)\n\n    @classmethod\n    def _read(cls, filename):\n        \"\"\"Reads file, returns (weights matrix, vocabulary list).\"\"\"\n        if filename.endswith('.npy'):\n            return cls._read_npy(filename)\n        elif os.path.exists(filename + '.npy'):\n            return cls._read_npy(filename + '.npy')\n        elif os.path.exists(re.sub('[.]\\w+$', '.npy', filename)):\n            return cls._read_npy(re.sub('[.]\\w+$', '.npy', filename))\n        elif filename.endswith('.bin'):\n            return cls._read_bin(filename)\n        else:\n            return cls._read_txt(filename)\n\n    @classmethod\n    def _read_npy(cls, filename):\n        assert filename.endswith('.npy')\n        vectors = np.load(filename, 'r')\n        with open(filename[:-4] + '.vocab', 'rb') as fp:\n            vocab = pickle.load(fp)\n        return vectors, vocab\n\n    @classmethod\n    def _read_txt(cls, filename):\n        vectors = []\n        vocab = []\n        dim = -1\n\n        for line_num, line in enumerate(open(filename, 'rb')):\n            try:\n                line = line.decode('utf-8')\n            except:\n                line = line.decode('latin_1')\n                print(f'Bad word on line {line_num+1}: {line[:20]}...')\n\n            word, line = line.split(' ', maxsplit=1)\n            vec = np.fromstring(line, np.float32, count=dim, sep=' ')\n            if len(vec) == 1 and len(vectors) == 0:\n                dim = int(vec[0])\n                continue\n\n            if dim != -1 and dim != len(vec):\n                raise Exception(f'Mismatching vector lengths: {dim} vs {len(vec)}')\n            dim = len(vec)\n\n            vectors.append(vec)\n            vocab.append(word)\n\n        vectors = np.stack(vectors)\n        vocab = '\\n'.join(vocab).split('\\n')\n        return vectors, vocab\n\n    @classmethod\n    def _read_bin(cls, filename):\n        from gensim.models import KeyedVectors\n        kv = KeyedVectors.load_word2vec_format(filename, binary=True)\n        return kv.vectors, kv.index2word\n\n\nclass QuoraPreprocessor:\n    \"\"\"Preprocesses raw text before tokenization.\"\"\"\n\n    punct = r\"\"\"-!\"#$%^&*+,.'\\\\\\[\\]()/:;?@_{}|~`’:”“=…<>√£°₹×€—？÷र™−•¿→®一，¹²³⁴₂∞℅∫∆øΔ∈½·≠（）。»«ʻくº—\"\"\"\n    punct += \"\\xa0\"\n\n    specialLR = '|'.join([re.sub('([.()])', r'\\\\\\1', s) for s in r\"\"\"\n        's 'm 'd 'll 're 've n't 'em o'clock\n        i.e. e.g. vs. U.S. U.K. [A-Za-z]. a.m. p.m.\n        e-mail t-shirt\n        : :) :D :-) :) ;) :-) =) ;-)\n        [0-9]+,[0-9]{3}\n    \"\"\".split()])\n\n    specialL = '|'.join([re.sub('([.()])', r'\\\\\\1', s) for s in r\"\"\"\n        [A-Za-z]. Mr. Mrs. Dr. a.m. p.m.\n    \"\"\".split()])\n\n    special = '|'.join([s for s in r\"\"\"\n        \\.{3,5}  [?!]{1,3}\n    \"\"\".split()])\n\n    re_ws = re.compile(r'[\\t\\n ]+')\n    re_apos = re.compile(r\"\"\"['’`‘´`′′]\"\"\")\n    re_contractions = re.compile(r\"(\\w)('s|'m|'d|'ll|'re|'ve|n't|'em)( |[%s])\" % punct, re.UNICODE | re.I)\n    re_special = re.compile(fr\"(((?<= )({specialLR})(?= ))|((?<= )({specialL}))|{special}|[{punct}])\",\n                            re.UNICODE | re.I)\n\n    tests = [\n        (\"Don`t y'all thinkin' it's, like, we're getting... a\\xa0little ``too'' late?!?!\",\n         \"Do n't y ' all thinkin ' it 's , like , we 're getting ... a \\xa0 little ' ' too ' ' late ?!? !\"),\n    ]\n\n    def __init__(self):\n        for inp, exp in self.tests:\n            outp = self.transform_str(inp)\n            assert outp == exp, (outp, exp)\n\n    def transform_str(self, text):\n        text = ' ' + self.re_ws.sub(' ', text) + ' '\n        text = self.re_apos.sub(\"'\", text)\n        text = self.re_contractions.sub(r'\\1 \\2 \\3', text)\n        text = self.re_special.sub(r' \\1 ', text)\n        text = self.re_ws.sub(' ', text).strip()\n        return text\n\n    def transform(self, texts, n_jobs=None, chunksize=10000):\n        if type(texts) is str:\n            return self.transform_str(texts)\n        if len(texts) < chunksize:\n            return [self.transform_str(s) for s in texts]\n        if n_jobs is None:\n            n_jobs = min(20, multiprocessing.cpu_count())\n        with multiprocessing.Pool(n_jobs) as pool:\n            res = list(pool.imap(self.transform_str, texts, chunksize=chunksize))\n            return pd.Series(res, dtype='O')\n\n\nclass QuoraFeatureExtractor:\n    def __init__(self, num_words=120000, max_len=70, want_capstate=False, want_wide=False, preprocessor=None):\n        self.max_len = max_len\n        self.num_words = num_words\n        self.num_aux = 1\n        self.preprocessor = preprocessor\n        self.want_capstate = want_capstate\n        self.want_wide = want_wide\n\n    def fit_transform(self, texts, fit_mask=None):\n        if fit_mask is None:\n            fit_mask = np.full(len(texts), True, dtype='bool')\n\n        if self.preprocessor:\n            print('Preprocessing', flush=True)\n            texts = self.preprocessor.transform(texts)\n            fit_mask = np.array(fit_mask, dtype='bool')\n\n        print('Tokenizing', flush=True)\n        from keras.preprocessing.text import Tokenizer\n        self.tokenizer = Tokenizer(num_words=self.num_words, filters='', split=' ', oov_token='__')\n        self.tokenizer.fit_on_texts(texts[fit_mask])\n        self.word_index = self.tokenizer.word_index\n\n        from keras.preprocessing.sequence import pad_sequences\n        tokens = self.tokenizer.texts_to_sequences(texts)\n        tokens = pad_sequences(tokens, maxlen=self.max_len)\n        res = {'tokens': tokens}\n\n        if self.want_capstate:\n            print('Generating capstate', flush=True)\n            with multiprocessing.Pool(multiprocessing.cpu_count()) as pool:\n                capstate = np.stack(pool.imap(self.map_capstate, texts, chunksize=10000))\n                res['capstate'] = capstate\n\n        if self.want_wide:\n            print('Generating wide features', flush=True)\n            wide = self.gen_wide_features(texts)\n            self.ss = sklearn.preprocessing.RobustScaler()\n            self.ss.fit(wide[fit_mask].values)\n            wide = self.ss.transform(wide.values)\n            wide[wide > 3] = 3\n            res['wide'] = wide\n\n        return res\n\n    def map_capstate(self, text):\n        toks = text.split(' ')\n        res = np.zeros(self.max_len, dtype=np.int8)\n\n        for i in range(min(len(res), len(toks))):\n            tok = toks[len(toks) - 1 - i]\n            x = 0\n            if tok[0].isdigit():\n                x = 6\n            elif unicodedata.category(tok[0]).startswith('P'):\n                x = 5\n            elif tok == tok.lower():\n                x = 1\n            elif tok == tok.capitalize():\n                x = 2\n            elif tok == tok.upper():\n                x = 3\n            else:\n                x = 4\n            res[-i-1] = x\n        return res\n\n    def gen_wide_features(self, texts):\n        texts = pd.Series(texts)\n        wide = pd.DataFrame()\n        wide['num_chars'] = texts.str.len()\n        wide['num_caps'] = texts.str.count('[A-Z]')\n        wide['frac_caps'] = (wide.num_caps / wide.num_chars).fillna(0)\n        wide['num_dots'] = texts.str.count('[.]')\n        #wide['num_words'] = texts.str.count(' ')\n        #wide['num_unique_words'] = texts.str.split(' ', n=70, expand=True).nunique(axis=1)\n        #wide['num_unique_words'] = np.minimum(wide['num_words'], wide['num_unique_words'])\n        #wide['frac_unique'] = (wide.num_unique_words / wide.num_words).fillna(0)\n        return wide\n\n    def embedding_weights(self, emb, verbose=1):\n        \"\"\"Embeds tokens with given pretrained embedding, initializes missing with random vectors.\"\"\"\n\n        W = np.random.normal(emb.mean(), emb.std(), (self.num_words, emb.dim))\n        #W[0, :] = 0\n        miss = 0\n        for word, idx in self.word_index.items():\n            if idx >= self.num_words: continue\n            vec = emb.lookup(word, lower=True)\n            if vec is not None:\n                W[idx] = vec\n            else:\n                miss += 1\n        if verbose > 0:\n            print(f'{miss} words from tokenizer missing ({miss/self.num_words:.2f}%) in {emb.name}')\n        return W\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3426abd1335b8672a859ad85fa808dd3bbe56154"},"cell_type":"code","source":"import copy\nimport os\nimport random\nimport time\n\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data\n\nimport numpy as np\nimport pandas as pd\n\n\ndef torch_seed(seed):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\ndef torch_gc():\n    import gc\n    gc.collect()\n    torch.cuda.empty_cache()\n\ndef keras_rnn_init(m):\n    for name, param in m.named_parameters():\n        if 'weight_ih' in name: torch.nn.init.xavier_uniform_(param)\n        if 'weight_hh' in name: torch.nn.init.orthogonal_(param)\n        if 'bias_' in name: torch.nn.init.constant_(param, 0)\n\n\nclass Fold:\n    def __init__(self, data, X, seed=42, valid_frac=0.1, holdout_frac=0.0432, holdout_seed=42):\n        self.data = data\n        self.X = X\n        self.y = data.target.values.astype(np.float32)\n        self.seed = seed\n\n        tmask = data.target.notnull().values\n\n        self.all_idx = data.index.values.astype(np.int32)\n        self.test_idx = self.all_idx[~tmask]\n\n        # Split off holdout sample\n        trainval_idx, self.holdout_idx = train_test_split(\n            self.all_idx[tmask],\n            stratify=self.y[tmask],\n            test_size=holdout_frac,\n            random_state=holdout_seed,\n            shuffle=True)\n\n        self.train_idx, self.valid_idx = train_test_split(\n            trainval_idx,\n            stratify=self.y[trainval_idx],\n            test_size=valid_frac,\n            random_state=seed,\n            shuffle=True)\n\n        self.train_idx.sort()\n        self.valid_idx.sort()\n        self.holdout_idx.sort()\n        self.oob_idx = self.all_idx[~np.isin(self.all_idx, self.train_idx, assume_unique=True)]\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9f16c9f4cebf8a4da1abb4dd4b3a84d8220635f1"},"cell_type":"markdown","source":"## Run"},{"metadata":{"trusted":true,"_uuid":"26d9dc5ec332ad0be2045872eafc7341f8ba90a7"},"cell_type":"code","source":"%%time\ntorch_seed(42)\n\nqd = QuoraData()\nqd.convert_start(['glove', 'wiki'])\n\nprep = QuoraPreprocessor()\ninput_df = qd.read_input()\ninput_df['question_text'] = prep.transform(input_df.question_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94371f4b4f519bb9dea807f8f0142be1a76009c6"},"cell_type":"code","source":"%%time\nqfe = QuoraFeatureExtractor(num_words=95000, max_len=70)\ninput_X = qfe.fit_transform(input_df.question_text, fit_mask=input_df.target.notnull().values)\nprint({ k: v.shape for (k, v) in input_X.items() })","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c45722e5a16e0d02a3ba2da8678924ec583b4825"},"cell_type":"code","source":"%%time\nqd.convert_wait()\n\ntorch_seed(42)\nemb_glove = qfe.embedding_weights(qd.glove())\n\ntorch_seed(43)\nemb_wiki = qfe.embedding_weights(qd.wiki())\n\nemb_glovewiki = np.concatenate([emb_glove, emb_wiki], axis=1)\n\ngc.collect()\nos.system('rm -f glove.npy glove.vocab wiki.npy wiki.vocab')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c9cdebf6a10dc0bdd34f2038f80e942bbd63255f"},"cell_type":"code","source":"class Arch1b_LstmGru(torch.nn.Module):\n    def __init__(self, emb_weights):\n        super().__init__()\n        self.embedding = nn.Embedding(emb_weights.shape[0], emb_weights.shape[1])\n        self.embedding.weight = nn.Parameter(torch.Tensor(emb_weights), requires_grad=False)\n        self.lstm = nn.LSTM(self.embedding.embedding_dim, 128, bidirectional=True, batch_first=True)\n        self.gru = nn.GRU(self.lstm.hidden_size*2, 64, bidirectional=True, batch_first=True)\n        self.fc = nn.Linear(128, 1)\n\n    def dropout1d(self, x, p):\n        # x = [batch, time, channels]\n        x = x.permute(0, 2, 1)   # [batch, channels, time]\n        x = F.dropout2d(x, p, training=self.training)\n        x = x.permute(0, 2, 1)   # [batch, time, channels]\n        return x\n\n    def forward(self, x_tok):\n        x = self.embedding(x_tok)  # [batch, time, channels]\n        x = self.dropout1d(x, 0.2)\n        x = F.dropout2d(x, 0.05, training=self.training)\n        x, _ = self.lstm(x)\n        x, c = self.gru(x)\n        x = x.max(1)[0]\n        x = F.dropout(x, 0.1, training=self.training)\n        x = self.fc(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8749b81041bd41dc2c0bcf358f5eb2e8ac393c75"},"cell_type":"code","source":"class Learner:\n    def __init__(self, fold, model, seed=None, batch_size=512, grad_clip=1):\n        self.__dict__.update(fold.__dict__)\n        self.fold = fold\n        self.model = model\n\n        self.dataset = torch.utils.data.TensorDataset(\n            torch.LongTensor(self.X['tokens']),\n            #torch.FloatTensor(self.X['wide']),\n            torch.FloatTensor(self.y[:, np.newaxis]))\n        self.train_dl = self.make_loader(self.train_idx, batch_size=batch_size, shuffle=True)\n        self.valid_dl = self.make_loader(self.valid_idx)\n\n        opt_params = [p for p in self.model.parameters() if p.requires_grad]\n        self.opt = torch.optim.Adam(opt_params)\n        self.grad_clip = grad_clip\n        self.loss_fn = nn.BCEWithLogitsLoss(reduction='sum')\n        self.steps = 0\n        self.epochs = 0\n\n        params_sz = sum([p.detach().cpu().numpy().nbytes for p in opt_params])\n        print(f'Training on {len(self.train_dl.dataset)} examples, '\n              f'validating on {len(self.valid_dl.dataset)} examples. '\n              f'Parameters: {params_sz/1048576:.2f}MiB', flush=True)\n\n        self.epoch_params = [ self.get_params() ]\n        self.val_metrics = []\n\n    def make_loader(self, idx, batch_size=1024, shuffle=False):\n        ds = torch.utils.data.Subset(self.dataset, idx)\n        return torch.utils.data.DataLoader(\n            ds, batch_size=batch_size, shuffle=shuffle, pin_memory=True)\n\n    def get_params(self):\n        return { name: param.detach().cpu().numpy().copy()\n                 for (name, param) in self.model.named_parameters()\n                 if param.requires_grad }\n\n    def restore_params(self, prev):\n        for name, param in self.model.named_parameters():\n            if name in prev:\n                param.data.copy_(torch.from_numpy(prev[name]))\n\n    def restore_epoch(self, epoch):\n        self.restore_params(self.epoch_params[epoch])\n\n    def predict(self, idx=None, dl=None):\n        if dl is None:\n            dl = self.make_loader(idx, shuffle=False)\n\n        preds = []\n        with torch.no_grad():\n            self.model.eval()\n            for batch in dl:\n                batch = [t.cuda() for t in batch[:-1]]\n                pred = torch.sigmoid(self.model(*batch))\n                preds.append(pred.cpu().numpy().flatten())\n            self.model.train()\n\n        return np.concatenate(preds)\n\n    def eval(self):\n        y_pred = self.predict(dl=self.valid_dl)\n        y_true = self.y[self.valid_idx]\n        curve = f1_curve(y_true, y_pred)\n        metrics = {\n            'epochs': self.epochs,\n            'steps': self.steps,\n            'f1': curve.max(),\n            'roc-auc': sklearn.metrics.roc_auc_score(y_true, y_pred),\n            'pr-auc': sklearn.metrics.average_precision_score(y_true, y_pred),\n            'loss': sklearn.metrics.log_loss(y_true, y_pred),\n            'thresh': curve.idxmax(),\n        }\n        self.val_metrics.append(metrics)\n        print(' '.join('%s=%.6g' % (k, float(v)) for (k, v) in metrics.items()), flush=True)\n\n    def train(self, epochs=1, eval_steps=0):\n        self.model.cuda()\n        self.model.train()\n\n        for ep in range(epochs):\n            t_start = time.time()\n\n            if not KAGGLE_RUN and eval_steps > 0:\n                it = enumerate(tqdm_notebook(self.train_dl, leave=False))\n            else:\n                it = enumerate(self.train_dl)\n\n            for i, batch in it:\n                batch = [t.cuda() for t in batch]\n                y_batch = batch.pop()\n\n                self.opt.zero_grad()\n\n                y_pred = self.model(*batch)\n                loss = self.loss_fn(y_pred, y_batch)\n                loss.backward()\n\n                if self.grad_clip is not None:\n                    nn.utils.clip_grad_norm_(self.model.parameters(), self.grad_clip)\n\n                self.opt.step()\n                self.steps += 1\n\n                if eval_steps and i % eval_steps == 0 and i > 0: self.eval()\n\n            self.epoch_params.append(self.get_params())\n            self.epochs += 1\n\n            print('t=%.2fs' % (time.time() - t_start), end=' ', flush=True)\n            self.eval()\n\n        return self","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"96a1aa0c3c0ecc2b284e17aaa84a998db3d61e98"},"cell_type":"code","source":"%%time\nfold = Fold(input_df, input_X, seed=1005); torch_seed(fold.seed)\nlearn1 = Learner(fold, Arch1b_LstmGru(emb_glovewiki)).train(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7bc99e9ae7cd1d567d0fcfff04261e3f2151a13f"},"cell_type":"code","source":"%%time\nfold = Fold(input_df, input_X, seed=555); torch_seed(fold.seed)\nlearn2 = Learner(fold, Arch1b_LstmGru(emb_glovewiki)).train(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31ceca23335296ea83ff751863f1b096c36b39de"},"cell_type":"code","source":"%%time\nfold = Fold(input_df, input_X, seed=1111); torch_seed(fold.seed)\nlearn3 = Learner(fold, Arch1b_LstmGru(emb_glovewiki)).train(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31ceca23335296ea83ff751863f1b096c36b39de"},"cell_type":"code","source":"%%time\nfold = Fold(input_df, input_X, seed=3333); torch_seed(fold.seed)\nlearn4 = Learner(fold, Arch1b_LstmGru(emb_glovewiki)).train(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"68ed45e3c130456302f420f942dbee6a046582a5"},"cell_type":"markdown","source":"## Eval"},{"metadata":{"trusted":true,"_uuid":"7c015525f77a64d99d06f80c7f7633a5a81395c5"},"cell_type":"code","source":"%%time\ntest_idx = fold.test_idx\nhold_idx = fold.holdout_idx\nhold_y = input_df.loc[fold.holdout_idx, 'target']\n\nensemble_hold = []\nensemble_test = []\nfor li, learn in enumerate([learn1, learn2, learn3, learn4]):\n    for ep in [5]:\n        if ep >= len(learn.epoch_params): continue\n        learn.restore_epoch(ep)\n        ensemble_hold.append(learn.predict(idx=hold_idx))\n        f1 = f1_curve(hold_y, ensemble_hold[-1]).max()\n        print('learn%d ep%d %.6f' % (li+1, ep, f1))\n        ensemble_test.append(learn.predict(idx=test_idx))\n\nensemble_test_s = pd.Series(np.mean(ensemble_test, axis=0), index=fold.data.loc[test_idx, 'qid'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2c401d804c27e1e296bbf034b743960958c11f56"},"cell_type":"code","source":"curve = f1_curve(hold_y, np.mean(ensemble_hold, axis=0))\nthresh = curve.idxmax()\ncurve.max(), thresh","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1cc54d2b48d2b2a17622b9e7fc1c9a6d9e3b88f3"},"cell_type":"code","source":"curve.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e64f0b379d8a604a74d4ebff27da71c51686fe45"},"cell_type":"code","source":"submission_df = pd.read_csv('../input/sample_submission.csv')\nsubmission_df['prediction'] = (ensemble_test_s.loc[submission_df.qid] >= thresh).astype(int).values\nsubmission_df.to_csv('submission.csv', index=False)\nsubmission_df.prediction.mean()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}