{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"raw","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \nimport os\nimport random\nimport re\nimport time\nfrom collections import Counter\nfrom itertools import chain\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom sklearn.metrics import f1_score, roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold, KFold\nfrom sklearn.utils import shuffle\nfrom torch import optim\nfrom torch.utils.data import Dataset, Sampler, DataLoader\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# constants\nembedding_glove = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\nembedding_fasttext = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\nembedding_para = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\nembedding_w2v = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\ntrain_path = '../input/train.csv'\ntest_path = '../input/test.csv'\n\nmispell_dict = {\"ain't\": \"is not\", \"aren't\": \"are not\", \"can't\": \"cannot\", \"'cause\": \"because\",\n                \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\", \"doesn't\": \"does not\",\n                \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\",\n                \"he'd\": \"he would\", \"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\",\n                \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\", \"i'd\": \"i would\",\n                \"i'd've\": \"i would have\", \"i'll\": \"i will\", \"i'll've\": \"I will have\", \"i'm\": \"i am\",\n                \"i've\": \"I have\", \"isn't\": \"is not\", \"it'd\": \"it would\",\n                \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\", \"it's\": \"it is\",\n                \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\n                \"mightn't\": \"might not\", \"mightn't've\": \"might not have\", \"must've\": \"must have\",\n                \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\",\n                \"needn't've\": \"need not have\", \"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\",\n                \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\",\n                \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\",\n                \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\",\n                \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\",\n                \"so've\": \"so have\", \"so's\": \"so as\", \"this's\": \"this is\", \"that'd\": \"that would\",\n                \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\",\n                \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\n                \"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\",\n                \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\",\n                \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\",\n                \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\",\n                \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\",\n                \"what're\": \"what are\", \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\",\n                \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\",\n                \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\",\n                \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\",\n                \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\",\n                \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\n                \"y'all'd've\": \"you all would have\", \"y'all're\": \"you all are\", \"y'all've\": \"you all have\",\n                \"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\",\n                \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\", 'colour': 'color',\n                'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling',\n                'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor',\n                'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize',\n                'youtu ': 'youtube ', 'qoura': 'quora', 'sallary': 'salary', 'whta': 'what',\n                'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can',\n                'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doi': 'do I',\n                'thebest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation',\n                'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis',\n                'etherium': 'ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017',\n                '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess',\n                \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization',\n                'demonitization': 'demonetization', 'demonetisation': 'demonetization'}\n\npuncts = '\\'!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n'\npunct_mapping = {\"‘\": \"'\", \"₹\": \"e\", \"´\": \"'\", \"°\": \"\", \"€\": \"e\", \"™\": \"tm\", \"√\": \" sqrt \", \"×\": \"x\", \"²\": \"2\",\n                 \"—\": \"-\", \"–\": \"-\", \"’\": \"'\", \"_\": \"-\", \"`\": \"'\", '”': '\"', '“': '\"', \"£\": \"e\",\n                 '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta',\n                 '∅': '', '³': '3', 'π': 'pi', '\\u200b': ' ', '…': ' ... ', '\\ufeff': '', 'करना': '', 'है': ''}\nfor p in puncts:\n    punct_mapping[p] = ' %s ' % p\n\np = re.compile('(\\[ math \\]).+(\\[ / math \\])')\np_space = re.compile(r'[^\\x20-\\x7e]')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  seeding functions\ndef set_seed(seed):\n    np.random.seed(seed)\n    torch.manual_seed(seed + 1)\n    if torch.cuda.is_available(): torch.cuda.manual_seed_all(seed + 2)\n    random.seed(seed + 4)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#data loading & pre-processing\n\ndef clean_text(text):\n    # clean latex maths\n    text = p.sub(' [ math ] ', text)\n    # clean invisible chars\n    text = p_space.sub(r'', text)\n    # clean punctuations\n    for punct in punct_mapping:\n        if punct in text:\n            text = text.replace(punct, punct_mapping[punct])\n    tokens = []\n    for token in text.split():\n        # replace contractions & correct misspells\n        token = mispell_dict.get(token.lower(), token)\n        tokens.append(token)\n    text = ' '.join(tokens)\n    return text\n\ndef load_data(train_path=train_path, test_path=test_path, debug=False):\n    train_df = pd.read_csv(train_path)\n    test_df = pd.read_csv(test_path)\n    if debug:\n        train_df = train_df[:10000]\n        test_df = test_df[:10000]\n    s = time.time()\n    train_df['question_text'] = train_df['question_text'].apply(clean_text)\n    test_df['question_text'] = test_df['question_text'].apply(clean_text)\n    print('preprocssing {}s'.format(time.time() - s))\n    return train_df, test_df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# vocabulary functions\ndef build_counter(sents, splited=False):\n    counter = Counter()\n    for sent in tqdm(sents, ascii=True, desc='building conuter'):\n        if splited:\n            counter.update(sent)\n        else:\n            counter.update(sent.split())\n    return counter\n\n\ndef build_vocab(counter, max_vocab_size):\n    vocab = {'token2id': {'<PAD>': 0, '<UNK>': max_vocab_size + 1}}\n    vocab['token2id'].update(\n        {token: _id + 1 for _id, (token, count) in\n         tqdm(enumerate(counter.most_common(max_vocab_size)), desc='building vocab')})\n    vocab['id2token'] = {v: k for k, v in vocab['token2id'].items()}\n    return vocab\n\ndef tokens2ids(tokens, token2id):\n    seq = []\n    for token in tokens:\n        token_id = token2id.get(token, len(token2id) - 1)\n        seq.append(token_id)\n    return seq\n\n#  data set\nclass TextDataset(Dataset):\n    def __init__(self, df, vocab=None, num_max=None, max_seq_len=100,\n                 max_vocab_size=95000):\n        if num_max is not None:\n            df = df[:num_max]\n\n        self.src_sents = df['question_text'].tolist()\n        self.qids = df['qid'].values\n        if vocab is None:\n            src_counter = build_counter(self.src_sents)\n            vocab = build_vocab(src_counter, max_vocab_size)\n        self.vocab = vocab\n        if 'src_seqs' not in df.columns:\n            self.src_seqs = []\n            for sent in tqdm(self.src_sents, desc='tokenize'):\n                seq = tokens2ids(sent.split()[:max_seq_len], vocab['token2id'])\n                self.src_seqs.append(seq)\n        else:\n            self.src_seqs = df['src_seqs'].tolist()\n        if 'target' in df.columns:\n            self.targets = df['target'].values\n        else:\n            self.targets = np.random.randint(2, size=(len(self.src_sents),))\n        self.max_seq_len = max_seq_len\n\n    def __len__(self):\n        return len(self.src_sents)\n\n    # for bucket iterator\n    def get_keys(self):\n        lens = np.fromiter(\n            tqdm(((min(self.max_seq_len, len(c.split()))) for c in self.src_sents), desc='generate lens'),\n            dtype=np.int32)\n        return lens\n\n    def __getitem__(self, index):\n        return self.qids[index], self.src_sents[index], self.src_seqs[index], self.targets[index]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  dynamic padding\ndef _pad_sequences(seqs):\n    lens = [len(seq) for seq in seqs]\n    max_len = max(lens)\n\n    padded_seqs = torch.zeros(len(seqs), max_len).long()\n    for i, seq in enumerate(seqs):\n        end = lens[i]\n        padded_seqs[i, :end] = torch.LongTensor(seq)\n    return padded_seqs, lens\n\n\ndef collate_fn(data):\n    qids, src_sents, src_seqs, targets, = zip(*data)\n    src_seqs, src_lens = _pad_sequences(src_seqs)\n    return qids, src_sents, src_seqs, src_lens, torch.FloatTensor(targets)\n\n\n#  bucket iterator\ndef divide_chunks(l, n):\n    if n == len(l):\n        yield np.arange(len(l), dtype=np.int32), l\n    else:\n        # looping till length l\n        for i in range(0, len(l), n):\n            data = l[i:i + n]\n            yield np.arange(i, i + len(data), dtype=np.int32), data\n\n\ndef prepare_buckets(lens, bucket_size, batch_size, shuffle_data=True, indices=None):\n    lens = -lens\n    assert bucket_size % batch_size == 0 or bucket_size == len(lens)\n    if indices is None:\n        if shuffle_data:\n            indices = shuffle(np.arange(len(lens), dtype=np.int32))\n            lens = lens[indices]\n        else:\n            indices = np.arange(len(lens), dtype=np.int32)\n    new_indices = []\n    extra_batch = None\n    for chunk_index, chunk in (divide_chunks(lens, bucket_size)):\n        # sort indices in bucket by descending order of length\n        indices_sorted = chunk_index[np.argsort(chunk, axis=-1)]\n        batches = []\n        for _, batch in divide_chunks(indices_sorted, batch_size):\n            if len(batch) == batch_size:\n                batches.append(batch.tolist())\n            else:\n                assert extra_batch is None\n                assert batch is not None\n                extra_batch = batch\n        # shuffling batches within buckets\n        if shuffle_data:\n            batches = shuffle(batches)\n        for batch in batches:\n            new_indices.extend(batch)\n\n    if extra_batch is not None:\n        new_indices.extend(extra_batch)\n    return indices[new_indices]\n\n\nclass BucketSampler(Sampler):\n\n    def __init__(self, data_source, sort_keys, bucket_size=None, batch_size=1536, shuffle_data=True):\n        super().__init__(data_source)\n        self.shuffle = shuffle_data\n        self.batch_size = batch_size\n        self.sort_keys = sort_keys\n        self.bucket_size = bucket_size if bucket_size is not None else len(sort_keys)\n        if not shuffle_data:\n            self.index = prepare_buckets(self.sort_keys, bucket_size=self.bucket_size, batch_size=self.batch_size,\n                                         shuffle_data=self.shuffle)\n        else:\n            self.index = None\n        self.weights = None\n\n    def set_weights(self, w):\n        assert w >= 0\n        total = np.sum(w)\n        if total != 1:\n            w = w / total\n        self.weights = w\n\n    def __iter__(self):\n        indices = None\n        if self.weights is not None:\n            total = len(self.sort_keys)\n\n            indices = np.random.choice(total, (total,), p=self.weights)\n        if self.shuffle:\n            self.index = prepare_buckets(self.sort_keys, bucket_size=self.bucket_size, batch_size=self.batch_size,\n                                         shuffle_data=self.shuffle, indices=indices)\n        return iter(self.index)\n\n    def get_reverse_indexes(self):\n        indexes = np.zeros((len(self.index),), dtype=np.int32)\n        for i, j in enumerate(self.index):\n            indexes[j] = i\n        return indexes\n\n    def __len__(self):\n        return len(self.sort_keys)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# embedding stuffs\ndef read_embedding(embedding_file):\n    \"\"\"\n    read embedding file into a dictionary\n    each line of the embedding file should in the format like  word 0.13 0.22 ... 0.44\n    :param embedding_file: path of the embedding.\n    :return: a dictionary of word to its embedding (numpy array)\n    \"\"\"\n    if os.path.basename(embedding_file) != 'wiki-news-300d-1M.vec':\n        skip_head = None\n    else:\n        skip_head = 0\n    if os.path.basename(embedding_file) == 'paragram_300_sl999.txt':\n        encoding = 'latin'\n    else:\n        encoding = 'utf-8'\n    embeddings_index = {}\n    t_chunks = pd.read_csv(embedding_file, index_col=0, skiprows=skip_head, encoding=encoding, sep=' ', header=None,\n                           quoting=3,\n                           doublequote=False, quotechar=None, engine='c', na_filter=False, low_memory=True,\n                           chunksize=10000)\n    for t in t_chunks:\n        for k, v in zip(t.index.values, t.values):\n            embeddings_index[k] = v.astype(np.float32)\n    return embeddings_index\n\n\ndef get_emb(embedding_index, word, word_raw):\n    if word == word_raw:\n        return None\n    else:\n        return embedding_index.get(word, None)\n\n\ndef embedding2numpy(embedding_path, word_index, num_words, embed_size, emb_mean=0., emb_std=0.5,\n                    report_stats=False):\n    embedding_index = read_embedding(embedding_path)\n    num_words = min(num_words + 2, len(word_index))\n    if report_stats:\n        all_coefs = []\n        for v in embedding_index.values():\n            all_coefs.append(v.reshape([-1, 1]))\n        all_coefs = np.concatenate(all_coefs)\n        print(all_coefs.mean(), all_coefs.std(), np.linalg.norm(all_coefs, axis=-1).mean())\n    embedding_matrix = np.zeros((num_words, embed_size), dtype=np.float32)\n    oov = 0\n    oov_cap = 0\n    oov_upper = 0\n    oov_lower = 0\n    for word, i in word_index.items():\n        if i == 0:  # padding\n            continue\n        if i >= num_words:\n            continue\n        embedding_vector = embedding_index.get(word, None)\n        if embedding_vector is None:\n            embedding_vector = get_emb(embedding_index, word.lower(), word)\n            if embedding_vector is None:\n                embedding_vector = get_emb(embedding_index, word.upper(), word)\n                if embedding_vector is None:\n                    embedding_vector = get_emb(embedding_index, word.capitalize(), word)\n                    if embedding_vector is None:\n                        oov += 1\n                        # embedding_vector = (np.zeros((1, embed_size)))\n                        embedding_vector = np.random.normal(emb_mean, emb_std, size=(1, embed_size))\n                    else:\n                        oov_lower += 1\n                else:\n                    oov_upper += 1\n            else:\n                oov_cap += 1\n\n        embedding_matrix[i] = embedding_vector\n\n    print('oov %d/%d/%d/%d/%d' % (oov, oov_cap, oov_upper, oov_lower, len(word_index)))\n    return embedding_matrix\n\n\ndef load_embedding(vocab, max_vocab_size, embed_size):\n    # load embedding\n    embedding_matrix1 = embedding2numpy(embedding_glove, vocab['token2id'], max_vocab_size, embed_size,\n                                        emb_mean=-0.005838499, emb_std=0.48782197, report_stats=False)\n    # -0.005838499 0.48782197 0.37823704\n    # oov 9196\n    # embedding_matrix2 = embedding2numpy(embedding_fasttext, vocab.token2id, max_vocab_size, embed_size,\n    #                                    report_stats=False, emb_mean=-0.0033469985, emb_std=0.109855495, )\n    # -0.0033469985 0.109855495 0.07475414\n    # oov 12885\n    embedding_matrix2 = embedding2numpy(embedding_para, vocab['token2id'], max_vocab_size, embed_size,\n                                        emb_mean=-0.0053247833, emb_std=0.49346462, report_stats=False)\n    # -0.0053247833 0.49346462 0.3828983\n    # oov 9061\n    # embedding_w2v\n    # -0.003527845 0.13315111 0.09407869\n    # oov 18927\n    return [embedding_matrix1, embedding_matrix2]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cyclic learning rate\ndef set_lr(optimizer, lr):\n    for g in optimizer.param_groups:\n        g['lr'] = lr\n\n\nclass CyclicLR:\n    def __init__(self, optimizer, base_lr=0.001, max_lr=0.002, step_size=300., mode='triangular',\n                 gamma=0.99994, scale_fn=None, scale_mode='cycle'):\n        super(CyclicLR, self).__init__()\n        self.optimizer = optimizer\n        self.base_lr = base_lr\n        self.max_lr = max_lr\n        self.step_size = step_size\n        self.mode = mode\n        self.gamma = gamma\n        if scale_fn is None:\n            if self.mode == 'triangular':\n                self.scale_fn = lambda x: 1.\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = lambda x: 1 / (2. ** (x - 1))\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = lambda x: gamma ** x\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n        self.clr_iterations = 0.\n        self.trn_iterations = 0.\n        self.history = {}\n        self._reset()\n\n    def _reset(self, new_base_lr=None, new_max_lr=None,\n               new_step_size=None):\n        if new_base_lr is not None:\n            self.base_lr = new_base_lr\n        if new_max_lr is not None:\n            self.max_lr = new_max_lr\n        if new_step_size is not None:\n            self.step_size = new_step_size\n        self.clr_iterations = 0.\n\n    def clr(self):\n        cycle = np.floor(1 + self.clr_iterations / (2 * self.step_size))\n        x = np.abs(self.clr_iterations / self.step_size - 2 * cycle + 1)\n        if self.scale_mode == 'cycle':\n            return self.base_lr + (self.max_lr - self.base_lr) * np.maximum(0, (1 - x)) * self.scale_fn(cycle)\n        else:\n            return self.base_lr + (self.max_lr - self.base_lr) * np.maximum(0, (1 - x)) * self.scale_fn(\n                self.clr_iterations)\n\n    def on_train_begin(self):\n        if self.clr_iterations == 0:\n            set_lr(self.optimizer, self.base_lr)\n        else:\n            set_lr(self.optimizer, self.clr())\n\n    def on_batch_end(self):\n        self.trn_iterations += 1\n        self.clr_iterations += 1\n        set_lr(self.optimizer, self.clr())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model\n\nclass Capsule(nn.Module):\n    def __init__(self, input_dim_capsule=1024, num_capsule=5, dim_capsule=5, routings=4):\n        super(Capsule, self).__init__()\n        self.num_capsule = num_capsule\n        self.dim_capsule = dim_capsule\n        self.routings = routings\n        self.activation = self.squash\n        self.W = nn.Parameter(\n            nn.init.xavier_normal_(torch.empty(1, input_dim_capsule, self.num_capsule * self.dim_capsule)))\n\n    def forward(self, x):\n        u_hat_vecs = torch.matmul(x, self.W)\n        batch_size = x.size(0)\n        input_num_capsule = x.size(1)\n        u_hat_vecs = u_hat_vecs.view((batch_size, input_num_capsule,\n                                      self.num_capsule, self.dim_capsule))\n        u_hat_vecs = u_hat_vecs.permute(0, 2, 1,\n                                        3).contiguous()  # (batch_size,num_capsule,input_num_capsule,dim_capsule)\n        with torch.no_grad():\n            b = torch.zeros_like(u_hat_vecs[:, :, :, 0])\n        for i in range(self.routings):\n            c = torch.nn.functional.softmax(b, dim=1)  # (batch_size,num_capsule,input_num_capsule)\n            outputs = self.activation(torch.sum(c.unsqueeze(-1) * u_hat_vecs, dim=2))  # bij,bijk->bik\n            if i < self.routings - 1:\n                b = (torch.sum(outputs.unsqueeze(2) * u_hat_vecs, dim=-1))  # bik,bijk->bij\n        return outputs  # (batch_size, num_capsule, dim_capsule)\n\n    def squash(self, x, axis=-1):\n        s_squared_norm = (x ** 2).sum(axis, keepdim=True)\n        scale = torch.sqrt(s_squared_norm + 1e-7)\n        return x / scale\n\n\n#  model\nclass Attention(nn.Module):\n    def __init__(self, feature_dim, max_seq_len=70):\n        super().__init__()\n        self.attention_fc = nn.Linear(feature_dim, 1)\n        self.bias = nn.Parameter(torch.zeros(1, max_seq_len, 1, requires_grad=True))\n\n    def forward(self, rnn_output):\n        \"\"\"\n        forward attention scores and attended vectors\n        :param rnn_output: (#batch,#seq_len,#feature)\n        :return: attended_outputs (#batch,#feature)\n        \"\"\"\n        attention_weights = self.attention_fc(rnn_output)\n        seq_len = rnn_output.size(1)\n        attention_weights = self.bias[:, :seq_len, :] + attention_weights\n        attention_weights = torch.tanh(attention_weights)\n        attention_weights = torch.exp(attention_weights)\n        attention_weights_sum = torch.sum(attention_weights, dim=1, keepdim=True) + 1e-7\n        attention_weights = attention_weights / attention_weights_sum\n        attended = torch.sum(attention_weights * rnn_output, dim=1)\n        return attended\n\n\nclass InsincereModel(nn.Module):\n    def __init__(self, device, hidden_dim, hidden_dim_fc, embedding_matrixs, vocab_size=None, embedding_dim=None,\n                 dropout=0.1, num_capsule=5, dim_capsule=5, capsule_out_dim=1, alpha=0.8, beta=0.8,\n                 finetuning_vocab_size=120002,\n                 embedding_mode='mixup', max_seq_len=70):\n        super(InsincereModel, self).__init__()\n        self.beta = beta\n        self.embedding_mode = embedding_mode\n        self.finetuning_vocab_size = finetuning_vocab_size\n        self.alpha = alpha\n        vocab_size, embedding_dim = embedding_matrixs[0].shape\n        self.raw_embedding_weights = embedding_matrixs\n        self.embedding_0 = nn.Embedding(vocab_size, embedding_dim, padding_idx=0).from_pretrained(\n            torch.from_numpy(embedding_matrixs[0]))\n        self.embedding_1 = nn.Embedding(vocab_size, embedding_dim, padding_idx=0).from_pretrained(\n            torch.from_numpy(embedding_matrixs[1]))\n        self.embedding_mean = nn.Embedding(vocab_size, embedding_dim, padding_idx=0).from_pretrained(\n            torch.from_numpy((embedding_matrixs[0] + embedding_matrixs[1]) / 2))\n        self.learnable_embedding = nn.Embedding(finetuning_vocab_size, embedding_dim, padding_idx=0)\n        nn.init.constant_(self.learnable_embedding.weight, 0)\n        self.learn_embedding = False\n        self.spatial_dropout = nn.Dropout2d(p=0.2)\n        self.device = device\n        self.hidden_dim = hidden_dim\n        self.rnn0 = nn.LSTM(embedding_dim, int(hidden_dim / 2), num_layers=1, bidirectional=True, batch_first=True)\n        self.rnn1 = nn.GRU(hidden_dim, int(hidden_dim / 2), num_layers=1, bidirectional=True, batch_first=True)\n        self.capsule = Capsule(input_dim_capsule=self.hidden_dim, num_capsule=num_capsule, dim_capsule=dim_capsule)\n        self.dropout2 = nn.Dropout(0.3)\n        self.lincaps = nn.Linear(num_capsule * dim_capsule, capsule_out_dim)\n        self.attention1 = Attention(self.hidden_dim, max_seq_len=max_seq_len)\n        self.attention2 = Attention(self.hidden_dim, max_seq_len=max_seq_len)\n        self.fc = nn.Linear(hidden_dim * 4 + capsule_out_dim, hidden_dim_fc)\n        self.norm = torch.nn.LayerNorm(hidden_dim * 4 + capsule_out_dim)\n        self.dropout1 = nn.Dropout(0.2)\n        self.dropout_linear = nn.Dropout(p=dropout)\n        self.hidden2out = nn.Linear(hidden_dim_fc, 1)\n\n    def set_embedding_mode(self, embedding_mode):\n        self.embedding_mode = embedding_mode\n\n    def enable_learning_embedding(self):\n        self.learn_embedding = True\n\n    def init_weights(self):\n        ih = (param.data for name, param in self.named_parameters() if 'weight_ih' in name)\n        hh = (param.data for name, param in self.named_parameters() if 'weight_hh' in name)\n        b = (param.data for name, param in self.named_parameters() if 'bias' in name)\n        for k in ih:\n            nn.init.xavier_uniform_(k)\n        for k in hh:\n            nn.init.orthogonal_(k)\n        for k in b:\n            nn.init.constant_(k, 0)\n\n    def apply_spatial_dropout(self, emb):\n        emb = emb.permute(0, 2, 1).unsqueeze(-1)\n        emb = self.spatial_dropout(emb).squeeze(-1).permute(0, 2, 1)\n        return emb\n\n    def forward(self, seqs, lens, return_logits=True):\n        # forward embeddings\n        if self.embedding_mode == 'mixup':\n            emb0 = self.embedding_0(seqs)  # batch_size x seq_len x embedding_dim\n            emb1 = self.embedding_1(seqs)\n            prob = np.random.beta(self.alpha, self.beta, size=(seqs.size(0), 1, 1)).astype(np.float32)\n            prob = torch.from_numpy(prob).to(self.device)\n            emb = emb0 * prob + emb1 * (1 - prob)\n        elif self.embedding_mode == 'emb0':\n            emb = self.embedding_0(seqs)\n        elif self.embedding_mode == 'emb1':\n            emb = self.embedding_1(seqs)\n        elif self.embedding_mode == 'mean':\n            emb = self.embedding_mean(seqs)\n        else:\n            assert False\n        if self.learn_embedding:\n            seq_clamped = torch.clamp(seqs, 0, self.finetuning_vocab_size - 1)\n            emb_learned = self.learnable_embedding(seq_clamped)\n            emb = emb + emb_learned\n        emb = self.apply_spatial_dropout(emb)\n        # forward rnn encoder\n        lstm_output0, _ = self.rnn0(emb)\n        lstm_output1, _ = self.rnn1(lstm_output0)\n        # forward capsule\n        content3 = self.capsule(lstm_output1)\n        batch_size = content3.size(0)\n        content3 = content3.view(batch_size, -1)\n        content3 = self.dropout2(content3)\n        content3 = torch.relu(self.lincaps(content3))\n        # forward feature extractor\n        feature_att1 = self.attention1(lstm_output0)\n        feature_att2 = self.attention2(lstm_output1)\n        feature_avg2 = torch.mean(lstm_output1, dim=1)\n        feature_max2, _ = torch.max(lstm_output1, dim=1)\n        feature = torch.cat((feature_att1, feature_att2, feature_avg2, feature_max2, content3), dim=-1)\n        feature = self.norm(feature)\n        feature = self.dropout1(feature)\n        feature = torch.relu(feature)\n        # forward dense layer\n        out = self.fc(feature)\n        out = self.dropout_linear(out)\n        out = self.hidden2out(out)  # batch_size x 1\n        if not return_logits:\n            out = torch.sigmoid(out)\n        return out","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  util functions\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n\n    def __init__(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n\ndef count_parameters(model):\n    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n\n\ndef margin_score(targets, predictions):\n    return ((targets == 1) * (1 - predictions) + (targets == 0) * (predictions)).mean()\n\n\ndef report_perf(valid_dataset, predictions_va, threshold, idx, epoch_cur, desc='val set'):\n    val_f1 = f1_score(valid_dataset.targets, predictions_va > threshold)\n    val_auc = roc_auc_score(valid_dataset.targets, predictions_va)\n    val_margin = margin_score(valid_dataset.targets, predictions_va)\n    print('idx {} epoch {} {} f1 : {:.4f} auc : {:.4f} margin : {:.4f}'.format(\n        idx,\n        epoch_cur,\n        desc,\n        val_f1,\n        val_auc,\n        val_margin))\n\n\ndef get_gpu_memory_usage(device_id):\n    return round(torch.cuda.max_memory_allocated(device_id) / 1000 / 1000)\n\n\ndef avg(loss_list):\n    if len(loss_list) == 0:\n        return 0\n    else:\n        return sum(loss_list) / len(loss_list)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# evaluation\ndef eval_model(model, data_iter, device, order_index=None):\n    model.eval()\n    predictions = []\n    with torch.no_grad():\n        for batch_data in data_iter:\n            qid_batch, src_sents, src_seqs, src_lens, tgts = batch_data\n            src_seqs = src_seqs.to(device)\n            out = model(src_seqs, src_lens, return_logits=False)\n            predictions.append(out)\n    predictions = torch.cat(predictions, dim=0)\n    if order_index is not None:\n        predictions = predictions[order_index]\n    predictions = predictions.to('cpu').numpy().ravel()\n    return predictions","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cross validation\n\ndef cv(train_df, test_df, device=None, n_folds=10, shared_resources=None, share=True, **kwargs):\n    if device is None:\n        device = torch.device(\"cuda:{}\".format(0) if torch.cuda.is_available() else \"cpu\")\n    max_vocab_size = kwargs['max_vocab_size']\n    embed_size = kwargs['embed_size']\n    threshold = kwargs['threshold']\n    max_seq_len = kwargs['max_seq_len']\n    if shared_resources is None:\n        shared_resources = {}\n    if share:\n        if 'vocab' not in shared_resources:\n            # also include the test set\n\n            counter = build_counter(chain(train_df['question_text'], test_df['question_text']))\n            vocab = build_vocab(counter, max_vocab_size=max_vocab_size)\n            shared_resources['vocab'] = vocab\n            # tokenize sentences\n            seqs = []\n            for sent in tqdm(train_df['question_text'], desc='tokenize'):\n                seq = tokens2ids(sent.split()[:max_seq_len], vocab['token2id'])\n                seqs.append(seq)\n            train_df['src_seqs'] = seqs\n            seqs = []\n            for sent in tqdm(test_df['question_text'], desc='tokenize'):\n                seq = tokens2ids(sent.split()[:max_seq_len], vocab['token2id'])\n                seqs.append(seq)\n            test_df['src_seqs'] = seqs\n    if 'embedding_matrix' not in shared_resources:\n        embedding_matrix = load_embedding(shared_resources['vocab'], max_vocab_size, embed_size)\n        shared_resources['embedding_matrix'] = embedding_matrix\n    splits = list(\n        StratifiedKFold(n_splits=n_folds, shuffle=True).split(train_df['target'], train_df['target']))\n    scores = []\n    best_threshold = []\n    best_threshold_global = None\n    best_score = -1\n    predictions_train_reduced = []\n    targets_train = []\n    predictions_tes_reduced = np.zeros((len(test_df), n_folds))\n    predictions_te =  np.zeros((len(test_df),))\n    for idx, (train_idx, valid_idx) in enumerate(splits):\n        grow_df = train_df.iloc[train_idx].reset_index(drop=True)\n        dev_df = train_df.iloc[valid_idx].reset_index(drop=True)\n        predictions_te_i, predictions_va, targets_va, best_threshold_i = main(grow_df, dev_df, test_df, device,\n                                                                              **kwargs,\n                                                                              idx=idx,\n                                                                              shared_resources=shared_resources,\n                                                                              return_reduced=True)\n        # predictions_va_raw shape (#len_va,n_models)\n        predictions_tes_reduced[:, idx] = predictions_te_i\n        scores.append([f1_score(targets_va, predictions_va > threshold), roc_auc_score(targets_va, predictions_va)])\n        best_threshold.append(best_threshold_i)\n        predictions_te += predictions_te_i / n_folds\n        predictions_train_reduced.append(predictions_va)\n        targets_train.append(targets_va)\n    # calculate model coefficient\n    coeff = (np.corrcoef(predictions_tes_reduced, rowvar=False).sum() - n_folds) / n_folds / (n_folds - 1)\n    # create data set for stacking\n    predictions_train_reduced = np.concatenate(predictions_train_reduced)\n    targets_train = np.concatenate(targets_train)  # len_train\n    # train optimal combining weights\n\n    # simple average\n    for t in np.arange(0, 1, 0.01):\n        score = f1_score(targets_train, predictions_train_reduced > t)\n        if score > best_score:\n            best_score = score\n            best_threshold_global = t\n    print('avg of best threshold {} macro-f1 best threshold {} best score {}'.format(best_threshold,\n                                                                                     best_threshold_global, best_score))\n    return predictions_te, predictions_te, scores, best_threshold_global, coeff","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#main routine\ndef main(train_df, valid_df, test_df, device=None, epochs=3, fine_tuning_epochs=3, batch_size=512, learning_rate=0.001,\n         learning_rate_max_offset=0.001, dropout=0.1,\n         threshold=None,\n         max_vocab_size=95000, embed_size=300, max_seq_len=70, print_every_step=500, idx=0, shared_resources=None,\n         return_reduced=True):\n    if device is None:\n        device = torch.device(\"cuda:{}\".format(0) if torch.cuda.is_available() else \"cpu\")\n\n    if shared_resources is None:\n        shared_resources = {}\n    batch_time = AverageMeter()\n    data_time = AverageMeter()\n    mean_len = AverageMeter()\n    # build vocab of raw df\n\n    if 'vocab' not in shared_resources:\n        counter = build_counter(chain(train_df['question_text'], test_df['question_text']))\n        vocab = build_vocab(counter, max_vocab_size=max_vocab_size)\n    else:\n        vocab = shared_resources['vocab']\n    if 'embedding_matrix' not in shared_resources:\n        embedding_matrix = load_embedding(vocab, max_vocab_size, embed_size)\n    else:\n        embedding_matrix = shared_resources['embedding_matrix']\n    # create test dataset\n    test_dataset = TextDataset(test_df, vocab=vocab, max_seq_len=max_seq_len)\n    tb = BucketSampler(test_dataset, test_dataset.get_keys(), batch_size=batch_size,\n                       shuffle_data=False)\n    test_iter = DataLoader(dataset=test_dataset,\n                           batch_size=batch_size,\n                           sampler=tb,\n                           # shuffle=False,\n                           num_workers=0,\n                           collate_fn=collate_fn)\n\n    train_dataset = TextDataset(train_df, vocab=vocab, max_seq_len=max_seq_len)\n    # keys = train_dataset.get_keys()  # for bucket sorting\n    valid_dataset = TextDataset(valid_df, vocab=vocab, max_seq_len=max_seq_len)\n    vb = BucketSampler(valid_dataset, valid_dataset.get_keys(), batch_size=batch_size,\n                       shuffle_data=False)\n    valid_index_reverse = vb.get_reverse_indexes()\n    # init model and optimizers\n    model = InsincereModel(device, hidden_dim=256, hidden_dim_fc=16, dropout=dropout,\n                           embedding_matrixs=embedding_matrix,\n                           vocab_size=len(vocab['token2id']),\n                           embedding_dim=embed_size, max_seq_len=max_seq_len)\n    if idx == 0:\n        print(model)\n        print('total trainable {}'.format(count_parameters(model)))\n    model = model.to(device)\n    optimizer = optim.Adam([p for p in model.parameters() if p.requires_grad], lr=learning_rate)\n\n    # init iterator\n    train_iter = DataLoader(dataset=train_dataset,\n                            batch_size=batch_size,\n                            # shuffle=True,\n                            # sampler=NegativeSubSampler(train_dataset, train_dataset.targets),\n                            sampler=BucketSampler(train_dataset, train_dataset.get_keys(), bucket_size=batch_size * 20,\n                                                  batch_size=batch_size),\n                            num_workers=0,\n                            collate_fn=collate_fn)\n\n    valid_iter = DataLoader(dataset=valid_dataset,\n                            batch_size=batch_size,\n                            sampler=vb,\n                            # shuffle=False,\n                            collate_fn=collate_fn)\n\n    # train model\n\n    loss_list = []\n    global_steps = 0\n    total_steps = epochs * len(train_iter)\n    loss_fn = torch.nn.BCEWithLogitsLoss()\n    end = time.time()\n    predictions_tes = []\n    predictions_vas = []\n    n_fge = 0\n    clr = CyclicLR(optimizer, base_lr=learning_rate, max_lr=learning_rate + learning_rate_max_offset,\n                   step_size=300, mode='exp_range')\n    clr.on_train_begin()\n    fine_tuning_epochs = epochs - fine_tuning_epochs\n    predictions_te = None\n    for epoch in tqdm(range(epochs)):\n\n        fine_tuning = epoch >= fine_tuning_epochs\n        start_fine_tuning = fine_tuning_epochs == epoch\n        if start_fine_tuning:\n            model.enable_learning_embedding()\n            optimizer = optim.Adam([p for p in model.parameters() if p.requires_grad], lr=learning_rate)\n            # fine tuning embedding layer\n            global_steps = 0\n            total_steps = (epochs - fine_tuning_epochs) * len(train_iter)\n            clr = CyclicLR(optimizer, base_lr=learning_rate, max_lr=learning_rate + learning_rate_max_offset,\n                           step_size=int(len(train_iter) / 8))\n            clr.on_train_begin()\n            predictions_te = np.zeros((len(test_df),))\n            predictions_va = np.zeros((len(valid_dataset.targets),))\n        for batch_data in train_iter:\n            data_time.update(time.time() - end)\n            qids, src_sents, src_seqs, src_lens, tgts = batch_data\n            mean_len.update(sum(src_lens))\n            src_seqs = src_seqs.to(device)\n            tgts = tgts.to(device)\n            model.train()\n            optimizer.zero_grad()\n\n            out = model(src_seqs, src_lens, return_logits=True).view(-1)\n            loss = loss_fn(out, tgts)\n            loss.backward()\n            optimizer.step()\n\n            loss_list.append(loss.detach().to('cpu').item())\n\n            global_steps += 1\n            batch_time.update(time.time() - end)\n            end = time.time()\n            if global_steps % print_every_step == 0:\n                curr_gpu_memory_usage = get_gpu_memory_usage(device_id=torch.cuda.current_device())\n                print('Global step: {}/{} Total loss: {:.4f}  Current GPU memory '\n                      'usage: {} maxlen {} '.format(global_steps, total_steps, avg(loss_list), curr_gpu_memory_usage,\n                                                    mean_len.avg))\n                loss_list = []\n\n                # print(f'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\\t'\n                #      f'Data {data_time.val:.3f} ({data_time.avg:.3f})\\t')\n            if fine_tuning and global_steps % (2 * clr.step_size) == 0:\n                predictions_te_tmp2 = eval_model(model, test_iter, device)\n                predictions_va_tmp2 = eval_model(model, valid_iter, device, valid_index_reverse)\n                report_perf(valid_dataset, predictions_va_tmp2, threshold, idx, epoch,\n                            desc='val set mean')\n                predictions_te = predictions_te * n_fge + (\n                    predictions_te_tmp2)\n                predictions_va = predictions_va * n_fge + (\n                    predictions_va_tmp2)\n                predictions_te /= n_fge + 1\n                predictions_va /= n_fge + 1\n                report_perf(valid_dataset, predictions_va, threshold, idx, epoch\n                            , desc='val set (fge)')\n                predictions_tes.append(predictions_te_tmp2.reshape([-1, 1]))\n                predictions_vas.append(predictions_va_tmp2.reshape([-1, 1]))\n                n_fge += 1\n\n            clr.on_batch_end()\n        if not fine_tuning:\n            predictions_va = eval_model(model, valid_iter, device, valid_index_reverse)\n            report_perf(valid_dataset, predictions_va, threshold, idx, epoch)\n    # pprint(model.attention1.bias.data.to('cpu'))\n    # pprint(model.attention2.bias.data.to('cpu'))\n    # reorder index\n    if predictions_te is not None:\n        predictions_te = predictions_te[tb.get_reverse_indexes()]\n    else:\n        predictions_te = eval_model(model, test_iter, device, tb.get_reverse_indexes())\n    best_score = -1\n    best_threshold = None\n    for t in np.arange(0, 1, 0.01):\n        score = f1_score(valid_dataset.targets, predictions_va > t)\n        if score > best_score:\n            best_score = score\n            best_threshold = t\n    print('best threshold on validation set: {:.2f} score {:.4f}'.format(best_threshold, best_score))\n    if not return_reduced and len(predictions_vas) > 0:\n        predictions_te = np.concatenate(predictions_tes, axis=1)\n        predictions_te = predictions_te[tb.get_reverse_indexes(), :]\n        predictions_va = np.concatenate(predictions_vas, axis=1)\n    # # Save the model\n    # model_path = 'trained_model.pth'\n    # save_model(model, model_path)\n    # print(\"Trained model saved to:\", model_path)\n\n    # # Make predictions\n    # predictions_te, _, _, _ = main(train_df, test_df, test_df, **args)\n\n    # # Calculate accuracy\n    # accuracy = calculate_accuracy(test_df['target'].values, predictions_te)\n    # print(f\"Accuracy: {accuracy:.4f}\")\n    # make predictions\n    return predictions_te, predictions_va, valid_dataset.targets, best_threshold","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nprint(torch.__version__)\nprint(torch.version.cuda)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# seeding\nset_seed(233)\nepochs = 8\nbatch_size = 512\nlearning_rate = 0.001\nlearning_rate_max_offset = 0.002\nfine_tuning_epochs = 2\nthreshold = 0.31\nmax_vocab_size = 120000\nembed_size = 300\nprint_every_step = 500\nmax_seq_len = 70\nshare = True\ndropout = 0.1\nsub = pd.read_csv('../input/sample_submission.csv')\ntrain_df, test_df = load_data()\n# shuffling\ntrn_idx = np.random.permutation(len(train_df))\ntrain_df = train_df.iloc[trn_idx].reset_index(drop=True)\nn_folds = 5\nn_repeats = 1\nargs = {'epochs': epochs, 'batch_size': batch_size, 'learning_rate': learning_rate, 'threshold': threshold,\n        'max_vocab_size': max_vocab_size,\n        'embed_size': embed_size, 'print_every_step': print_every_step, 'dropout': dropout,\n        'learning_rate_max_offset': learning_rate_max_offset,\n        'fine_tuning_epochs': fine_tuning_epochs, 'max_seq_len': max_seq_len}\npredictions_te_all = np.zeros((len(test_df),))\nfor _ in range(n_repeats):\n    if n_folds > 1:\n        _, predictions_te, _, threshold, coeffs = cv(train_df, test_df, n_folds=n_folds, share=share, **args)\n        print('coeff between predictions {}'.format(coeffs))\n    else:\n        predictions_te, _, _, _ = main(train_df, test_df, test_df, **args)\n    predictions_te_all += predictions_te / n_repeats\nsub.prediction = predictions_te_all > threshold\nsub.to_csv(\"submission.csv\", index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}