{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import argparse\nimport json\nimport math\nimport os\nimport sys\nimport unidecode\nimport random\nimport re\nimport time\nimport yaml\nfrom abc import ABCMeta, abstractmethod\nfrom collections import defaultdict, Counter\nfrom copy import deepcopy\nfrom functools import partial\nfrom multiprocessing import Pool\nfrom pathlib import Path\n\nimport nltk\nimport gensim\nimport sklearn\nimport torch\nimport torch.nn.functional as F\nimport numpy as np\nimport pandas as pd\nfrom gensim.corpora import Dictionary\nfrom gensim.models import KeyedVectors\nfrom gensim.models import Word2Vec, Doc2Vec, FastText\nfrom sklearn import metrics\nfrom torch import nn\nfrom torch.utils.data import DataLoader, WeightedRandomSampler\nfrom tqdm import tqdm\n%load_ext Cython\nimport zipfile","metadata":{"_uuid":"265398a16adabc6afead04691b34432ba4535219","execution":{"iopub.status.busy":"2022-02-02T06:09:33.424842Z","iopub.execute_input":"2022-02-02T06:09:33.425425Z","iopub.status.idle":"2022-02-02T06:09:33.441069Z","shell.execute_reply.started":"2022-02-02T06:09:33.425203Z","shell.execute_reply":"2022-02-02T06:09:33.439153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"** رسم نمودار های مربوط به دقت و یاداوری را من به کد اضافه کرده ام \nدر این کد تغییرات کوچکی برای خواندن دیتا داده شده است **","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-02-02T06:09:37.44922Z","iopub.execute_input":"2022-02-02T06:09:37.449554Z","iopub.status.idle":"2022-02-02T06:09:37.454608Z","shell.execute_reply.started":"2022-02-02T06:09:37.449493Z","shell.execute_reply":"2022-02-02T06:09:37.453179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" #این بخش را بخاطر انکه بتوانم دیتای حجیم اموزشی را از خود کگل بخوانم، افزودم","metadata":{}},{"cell_type":"code","source":"# ../input/quora-insincere-questions-classification/embeddings.zip\n# %%time\n### unzipping all the pretrained embeddings\n# !unzip /kaggle/input/quora-insincere-questions-classification/embeddings.zip\n# ../input//train.csv\n# ! unzip -f /kaggle/input/quora-insincere-questions-classification/embeddings.zip\nwith zipfile.ZipFile(\"../input/quora-insincere-questions-classification/embeddings.zip\",\"r\") as z:\n    z.extractall(\".\")","metadata":{"execution":{"iopub.status.busy":"2022-02-02T06:09:39.230787Z","iopub.execute_input":"2022-02-02T06:09:39.231159Z","iopub.status.idle":"2022-02-02T06:09:40.001033Z","shell.execute_reply.started":"2022-02-02T06:09:39.231094Z","shell.execute_reply":"2022-02-02T06:09:39.999259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Experiment config","metadata":{"_uuid":"fe6c2e9b49e682fc479f90934d53dd55a0b14440"}},{"cell_type":"markdown","source":"## Config","metadata":{"_uuid":"ae3fa4d4bc3b125f0f67bd5e93b6a14a29e8d1f1"}},{"cell_type":"code","source":"modules = \"\"\"\nclass ExperimentConfigBuilder(ExperimentConfigBuilderBase):\n\n    default_config = dict(\n        test=False,\n        device=0,\n        maxlen=72,\n        vocab_mincount=5,\n        scale_batchsize=[],\n        validate_from=4,\n    )\n\n    @property\n    def modules(self):\n        return [\n            TextNormalizer,\n            TextTokenizer,\n            WordEmbeddingFeaturizer,\n            WordExtraFeaturizer,\n            SentenceExtraFeaturizer,\n            Embedding,\n            Encoder,\n            Aggregator,\n            MLP,\n        ]\n\n\ndef build_model(config, embedding_matrix, n_sentence_extra_features):\n    embedding = Embedding(config, embedding_matrix)\n    encoder = Encoder(config, embedding.out_size)\n    aggregator = Aggregator(config)\n    mlp = MLP(config, encoder.out_size + n_sentence_extra_features)\n    out = nn.Linear(config.mlp_n_hiddens[-1], 1)\n    lossfunc = nn.BCEWithLogitsLoss()\n\n    return BinaryClassifier(\n        embedding=embedding,\n        encoder=encoder,\n        aggregator=aggregator,\n        mlp=mlp,\n        out=out,\n        lossfunc=lossfunc,\n    )\n\n\n# =======  Preprocessing modules  =======\n\nclass TextNormalizer(TextNormalizerPresets):\n    pass\n\n\nclass TextTokenizer(TextTokenizerPresets):\n    pass\n\n\nclass WordEmbeddingFeaturizer(WordEmbeddingFeaturizerPresets):\n    pass\n\n\nclass WordExtraFeaturizer(WordExtraFeaturizerPresets):\n\n    default_config = dict(\n        word_extra_features=['idf', 'unk'],\n    )\n\n\nclass SentenceExtraFeaturizer(SentenceExtraFeaturizerPresets):\n\n    default_config = dict(\n        sentence_extra_features=['char', 'word'],\n    )\n\n\nclass Preprocessor(PreprocessorPresets):\n\n    embedding_sampling = 400\n\n    def build_word_features(self, word_embedding_featurizer,\n                            embedding_matrices, word_extra_features):\n        embedding = np.stack(list(embedding_matrices.values()))\n\n        # Concat embedding\n        embedding = np.concatenate(embedding, axis=1)\n        vocab = word_embedding_featurizer.vocab\n        embedding[vocab.lfq & vocab.unk] = 0\n\n        # Embedding random sampling\n        n_embed = embedding.shape[1]\n        n_select = self.embedding_sampling\n        idx = np.random.permutation(n_embed)[:n_select]\n        embedding = embedding[:, idx]\n\n        word_features = np.concatenate(\n            [embedding, word_extra_features], axis=1)\n        return word_features\n\n\n# =======  Training modules  =======\n\nclass Embedding(EmbeddingPresets):\n    pass\n\n\nclass Encoder(EncoderPresets):\n    pass\n\n\nclass Aggregator(AggregatorPresets):\n    pass\n\n\nclass MLP(MLPPresets):\n    pass\n\n\nclass Ensembler(EnsemblerPresets):\n    pass\n\n\"\"\"\n# /kaggle/input/quora-insincere-questions-classification\nos.environ['DATADIR'] = '.'\n# os.environ['DATADIR'] = '/kaggle/input/quora-insincere-questions-classification'","metadata":{"_uuid":"94d05af05886a1a9e61d892adf29dc184b64225d","execution":{"iopub.status.busy":"2022-02-02T06:25:38.553628Z","iopub.execute_input":"2022-02-02T06:25:38.554175Z","iopub.status.idle":"2022-02-02T06:25:38.599633Z","shell.execute_reply.started":"2022-02-02T06:25:38.5541Z","shell.execute_reply":"2022-02-02T06:25:38.59807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Library codes","metadata":{"_uuid":"ba7ba267545621661b8c3d138729c934c8f5c026"}},{"cell_type":"markdown","source":"## Config","metadata":{"_uuid":"8cf738031a390005e553735ecf3df680b62ffa1c"}},{"cell_type":"code","source":"class ExperimentConfigBuilderBase(metaclass=ABCMeta):\n\n    default_config = None\n\n    def add_args(self, parser):\n        parser.add_argument('--modelfile', '-m', type=Path)\n        parser.add_argument('--outdir-top', type=Path, default=Path('results'))\n        parser.add_argument('--outdir-bottom', type=str, default='default')\n        parser.add_argument('--device', '-g', type=int)\n        parser.add_argument('--test', action='store_true')\n        parser.add_argument('--logging', action='store_true')\n        parser.add_argument('--n-rows', type=int)\n\n        parser.add_argument('--seed', type=int, default=1029)\n        parser.add_argument('--optuna-trials', type=int)\n        parser.add_argument('--gridsearch', action='store_true')\n        parser.add_argument('--holdout', action='store_true')\n        parser.add_argument('--cv', type=int, default=5)\n        parser.add_argument('--cv-part', type=int)\n        parser.add_argument('--processes', type=int, default=2)\n\n        parser.add_argument('--lr', type=float, default=1e-3)\n        parser.add_argument('--batchsize', type=int, default=512)\n        parser.add_argument('--batchsize-valid', type=int, default=1024)\n        parser.add_argument('--scale-batchsize', type=int, nargs='+',\n                            default=[])\n        parser.add_argument('--epochs', type=int, default=5)\n        parser.add_argument('--validate-from', type=int)\n        parser.add_argument('--pos-weight', type=float, default=1.)\n        parser.add_argument('--maxlen', type=float, default=72)\n        parser.add_argument('--vocab-mincount', type=float, default=5)\n        parser.add_argument('--ensembler-n-snapshots', type=int, default=1)\n\n    @abstractmethod\n    def modules(self):\n        raise NotImplementedError()\n\n    def build(self, args=None):\n        assert self.default_config is not None\n        parser = argparse.ArgumentParser()\n        self.add_args(parser)\n        parser.set_defaults(**self.default_config)\n\n        for module in self.modules:\n            module.add_args(parser)\n        config, extra_config = parser.parse_known_args(args)\n\n        for module in self.modules:\n            if hasattr(module, 'add_extra_args'):\n                module.add_extra_args(parser, config)\n\n        if config.test:\n            parser.set_defaults(**dict(\n                n_rows=500,\n                batchsize=64,\n                validate_from=0,\n                epochs=3,\n                cv_part=2,\n                ensembler_test_size=1.,\n            ))\n\n        config = parser.parse_args(args)\n        if config.modelfile is not None:\n            config.outdir = config.outdir_top / config.modelfile.stem \\\n                / config.outdir_bottom\n        else:\n            config.outdir = Path('.')\n\n        return config","metadata":{"_uuid":"5d0230cf7aee7447da70420495de65f0832ef5c1","execution":{"iopub.status.busy":"2022-02-02T06:09:54.610613Z","iopub.execute_input":"2022-02-02T06:09:54.61105Z","iopub.status.idle":"2022-02-02T06:09:54.631607Z","shell.execute_reply.started":"2022-02-02T06:09:54.610987Z","shell.execute_reply":"2022-02-02T06:09:54.630313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Datasets","metadata":{"_uuid":"5b8cfaa77fbe7fa86287c63ef12c19b24dbafd92"}},{"cell_type":"code","source":"def load_qiqc(n_rows=None):\n    train_df = pd.read_csv(f'{os.environ[\"DATADIR\"]}/train.csv', nrows=n_rows)\n    submit_df = pd.read_csv(f'{os.environ[\"DATADIR\"]}/test.csv', nrows=n_rows)\n    n_labels = {\n        0: (train_df.target == 0).sum(),\n        1: (train_df.target == 1).sum(),\n    }\n    train_df['target'] = train_df.target.astype('f')\n    train_df['weights'] = train_df.target.apply(lambda t: 1 / n_labels[t])\n\n    return train_df, submit_df\n\n\ndef build_datasets(train_df, submit_df, holdout=False, seed=0):\n    submit_dataset = QIQCDataset(submit_df)\n    if holdout:\n        # Train : Test split for holdout training\n        splitter = sklearn.model_selection.StratifiedShuffleSplit(\n            n_splits=1, test_size=0.1, random_state=seed)\n        train_indices, test_indices = list(splitter.split(\n            train_df, train_df.target))[0]\n        train_indices.sort(), test_indices.sort()\n        train_dataset = QIQCDataset(\n            train_df.iloc[train_indices].reset_index(drop=True))\n        test_dataset = QIQCDataset(\n            train_df.iloc[test_indices].reset_index(drop=True))\n    else:\n        train_dataset = QIQCDataset(train_df)\n        test_dataset = QIQCDataset(train_df.head(0))\n\n    return train_dataset, test_dataset, submit_dataset\n\n\nclass QIQCDataset(object):\n\n    def __init__(self, df):\n        self.df = df\n\n    @property\n    def tokens(self):\n        return self.df.tokens.values\n\n    @tokens.setter\n    def tokens(self, tokens):\n        self.df['tokens'] = tokens\n\n    @property\n    def positives(self):\n        return self.df[self.df.target == 1]\n\n    @property\n    def negatives(self):\n        return self.df[self.df.target == 0]\n\n    def build(self, device):\n        self._X = self.tids\n        self.X = torch.Tensor(self._X).type(torch.long).to(device)\n        if 'target' in self.df:\n            self._t = self.df.target[:, None]\n            self._W = self.df.weights\n            self.t = torch.Tensor(self._t).type(torch.float).to(device)\n            self.W = torch.Tensor(self._W).type(torch.float).to(device)\n        if hasattr(self, '_X2'):\n            self.X2 = torch.Tensor(self._X2).type(torch.float).to(device)\n        else:\n            self._X2 = np.zeros((self._X.shape[0], 1), 'f')\n            self.X2 = torch.Tensor(self._X2).type(torch.float).to(device)\n\n    def build_labeled_dataset(self, indices):\n        return torch.utils.data.TensorDataset(\n            self.X[indices], self.X2[indices],\n            self.t[indices], self.W[indices])","metadata":{"_uuid":"8b341992cf2a89d34489ba87a9151d53c2382296","execution":{"iopub.status.busy":"2022-02-02T06:09:59.375828Z","iopub.execute_input":"2022-02-02T06:09:59.376204Z","iopub.status.idle":"2022-02-02T06:09:59.392542Z","shell.execute_reply.started":"2022-02-02T06:09:59.376148Z","shell.execute_reply":"2022-02-02T06:09:59.391592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Registries","metadata":{"_uuid":"74df4b02f3a1e9428d7df3f874fa1d4d829e546d"}},{"cell_type":"code","source":"# Registries for preprocessing\nNORMALIZER_REGISTRY = {}\nTOKENIZER_REGISTRY = {}\nWORD_EMBEDDING_FEATURIZER_REGISTRY = {}\nWORD_EXTRA_FEATURIZER_REGISTRY = {}\nSENTENCE_EXTRA_FEATURIZER_REGISTRY = {}\n\n# Registries for training\nENCODER_REGISTRY = {}\nAGGREGATOR_REGISTRY = {}\nATTENTION_REGISTRY = {}\n\n\ndef register_preprocessor(name):\n    def register_cls(cls):\n        NORMALIZER_REGISTRY[name] = cls\n        return cls\n    return register_cls\n\n\ndef register_tokenizer(name):\n    def register_cls(cls):\n        TOKENIZER_REGISTRY[name] = cls\n        return cls\n    return register_cls\n\n\ndef register_word_embedding_features(name):\n    def register_cls(cls):\n        WORD_EMBEDDING_FEATURIZER_REGISTRY[name] = cls\n        return cls\n    return register_cls\n\n\ndef register_word_extra_features(name):\n    def register_cls(cls):\n        WORD_EXTRA_FEATURIZER_REGISTRY[name] = cls\n        return cls\n    return register_cls\n\n\ndef register_sentence_extra_features(name):\n    def register_cls(cls):\n        SENTENCE_EXTRA_FEATURIZER_REGISTRY[name] = cls\n        return cls\n    return register_cls\n\n\ndef register_encoder(name):\n    def register_cls(cls):\n        ENCODER_REGISTRY[name] = cls\n        return cls\n    return register_cls\n\n\ndef register_aggregator(name):\n    def register_cls(cls):\n        AGGREGATOR_REGISTRY[name] = cls\n        return cls\n    return register_cls\n\n\ndef register_attention(name):\n    def register_cls(cls):\n        ATTENTION_REGISTRY[name] = cls\n        return cls\n    return register_cls\n","metadata":{"_uuid":"d020317f35e394340ecf22e2fd3eee33cf96212e","execution":{"iopub.status.busy":"2022-02-02T06:10:02.9714Z","iopub.execute_input":"2022-02-02T06:10:02.971783Z","iopub.status.idle":"2022-02-02T06:10:02.988327Z","shell.execute_reply.started":"2022-02-02T06:10:02.971703Z","shell.execute_reply":"2022-02-02T06:10:02.986907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocessing","metadata":{"_uuid":"58e2e5ef76e10157fe4c8c370e324a619cd81da0"}},{"cell_type":"markdown","source":"### Vocab","metadata":{"_uuid":"389e6c38dac0851c117d3b5469d3919e82626ff4"}},{"cell_type":"code","source":"class WordVocab(object):\n\n    def __init__(self, mincount=1):\n        self.counter = Counter()\n        self.n_documents = 0\n        self._counters = {}\n        self._n_documents = defaultdict(int)\n        self.mincount = mincount\n\n    def __len__(self):\n        return len(self.token2id)\n\n    def add_documents(self, documents, name):\n        self._counters[name] = Counter()\n        for document in documents:\n            bow = dict.fromkeys(document, 1)\n            self._counters[name].update(bow)\n            self.counter.update(bow)\n            self.n_documents += 1\n            self._n_documents[name] += 1\n\n    def build(self):\n        counter = dict(self.counter.most_common())\n        self.word_freq = {\n            **{'<PAD>': 0},\n            **counter,\n        }\n        self.token2id = {\n            **{'<PAD>': 0},\n            **{word: i + 1 for i, word in enumerate(counter)}\n        }\n        self.lfq = np.array(list(self.word_freq.values())) < self.mincount\n        self.hfq = ~self.lfq\n","metadata":{"_uuid":"2383c175f33dda038af10fff62718faaf66b03e2","execution":{"iopub.status.busy":"2022-02-02T06:10:06.992179Z","iopub.execute_input":"2022-02-02T06:10:06.992517Z","iopub.status.idle":"2022-02-02T06:10:07.003434Z","shell.execute_reply.started":"2022-02-02T06:10:06.992431Z","shell.execute_reply":"2022-02-02T06:10:07.00214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Normalizers","metadata":{"_uuid":"4a6921c6d0603ea19e563548a54798f2a4880d51"}},{"cell_type":"code","source":"%%cython\nimport re\n\nimport numpy as np\ncimport numpy as np\n\n\ncdef class StringReplacer:\n    cpdef public dict rule\n    cpdef list keys\n    cpdef list values\n    cpdef int n_rules\n\n    def __init__(self, dict rule):\n        self.rule = rule\n        self.keys = list(rule.keys())\n        self.values = list(rule.values())\n        self.n_rules = len(rule)\n\n    def __call__(self, str x):\n        cdef int i\n        for i in range(self.n_rules):\n            if self.keys[i] in x:\n                x = x.replace(self.keys[i], self.values[i])\n        return x\n\n    def __getstate__(self):\n        return (self.rule, self.keys, self.values, self.n_rules)\n\n    def __setstate__(self, state):\n        self.rule, self.keys, self.values, self.n_rules = state\n\n\ncdef class RegExpReplacer:\n    cdef dict rule\n    cdef list keys\n    cdef list values\n    cdef regexp\n    cdef int n_rules\n\n    def __init__(self, dict rule):\n        self.rule = rule\n        self.keys = list(rule.keys())\n        self.values = list(rule.values())\n        self.regexp = re.compile('(%s)' % '|'.join(self.keys))\n        self.n_rules = len(rule)\n\n    @property\n    def rule(self):\n        return self.rule\n\n    def __call__(self, str x):\n        def replace(match):\n            x = match.group(0)\n            if x in self.rule:\n                return self.rule[x]\n            else:\n                for i in range(self.n_rules):\n                    x = re.sub(self.keys[i], self.values[i], x)\n                return x\n        return self.regexp.sub(replace, x)\n\n\ncpdef str cylower(str x):\n    return x.lower()\n\n\nCache = {}\nis_alphabet = re.compile(r'[a-zA-Z]')\n\n\n\ncpdef str unidecode_weak(str string):\n    \"\"\"Transliterate an Unicode object into an ASCII string\n    >>> unidecode(u\"\\u5317\\u4EB0\")\n    \"Bei Jing \"\n    \"\"\"\n\n    cdef list retval = []\n    cdef int i = 0\n    cdef int n = len(string)\n    cdef str char\n\n    for i in range(n):\n        char = string[i]\n        codepoint = ord(char)\n\n        if codepoint < 0x80: # Basic ASCII\n            retval.append(char)\n            continue\n\n        if codepoint > 0xeffff:\n            continue  # Characters in Private Use Area and above are ignored\n\n        section = codepoint >> 8   # Chop off the last two hex digits\n        position = codepoint % 256 # Last two hex digits\n\n        try:\n            table = Cache[section]\n        except KeyError:\n            try:\n                mod = __import__('unidecode.x%03x'%(section), [], [], ['data'])\n            except ImportError:\n                Cache[section] = None\n                continue   # No match: ignore this character and carry on.\n\n            Cache[section] = table = mod.data\n\n        if table and len(table) > position:\n            if table[position] == '[?]' or is_alphabet.match(table[position]):\n                retval.append(' ' + char + ' ')\n            else:\n                retval.append(table[position])\n\n    return ''.join(retval)\n","metadata":{"_uuid":"31efade437cb270c0ea53e65bef25064feb51901","execution":{"iopub.status.busy":"2022-02-02T06:10:10.711757Z","iopub.execute_input":"2022-02-02T06:10:10.712124Z","iopub.status.idle":"2022-02-02T06:10:14.415304Z","shell.execute_reply.started":"2022-02-02T06:10:10.712068Z","shell.execute_reply":"2022-02-02T06:10:14.414015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PunctSpacer(StringReplacer):\n\n    def __init__(self, edge_only=False):\n        puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', '█', '½', '…', '“', '★', '”', '–', '●', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', '¯', '♦', '¤', '▲', '¸', '¾', '⋅', '‘', '∞', '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]  # NOQA\n        if edge_only:\n            rule = {\n                **dict([(f' {p}', f' {p} ') for p in puncts]),\n                **dict([(f'{p} ', f' {p} ') for p in puncts]),\n            }\n        else:\n            rule = dict([(p, f' {p} ') for p in puncts])\n        super().__init__(rule)\n\n\nclass NumberReplacer(RegExpReplacer):\n\n    def __init__(self, with_underscore=False):\n        prefix, suffix = '', ''\n        if with_underscore:\n            prefix += ' __'\n            suffix = '__ '\n        rule = {\n            '[0-9]{5,}': f'{prefix}#####{suffix}',\n            '[0-9]{4}': f'{prefix}####{suffix}',\n            '[0-9]{3}': f'{prefix}###{suffix}',\n            '[0-9]{2}': f'{prefix}##{suffix}',\n        }\n        super().__init__(rule)\n\n\nclass KerasFilterReplacer(StringReplacer):\n\n    def __init__(self):\n        filters = '!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n'\n        rule = dict([(f, ' ') for f in filters])\n        super().__init__(rule)\n\n\nclass MisspellReplacer(StringReplacer):\n\n    def __init__(self):\n        rule = {\n            \"ain't\": \"is not\",\n            \"aren't\": \"are not\",\n            \"can't\": \"cannot\",\n            \"'cause\": \"because\",\n            \"could've\": \"could have\",\n            \"couldn't\": \"could not\",\n            \"didn't\": \"did not\",\n            \"doesn't\": \"does not\",\n            \"don't\": \"do not\",\n            \"hadn't\": \"had not\",\n            \"hasn't\": \"has not\",\n            \"haven't\": \"have not\",\n            \"he'd\": \"he would\",\n            \"he'll\": \"he will\",\n            \"he's\": \"he is\",\n            \"how'd'y\": \"how do you\",\n            \"how'd\": \"how did\",\n            \"how'll\": \"how will\",\n            \"how's\": \"how is\",\n            \"i'd've\": \"i would have\",\n            \"i'd\": \"i would\",\n            \"i'll've\": \"i will have\",\n            \"i'll\": \"i will\",\n            \"i'm\": \"i am\",\n            \"i've\": \"i have\",\n            \"isn't\": \"is not\",\n            \"it'd've\": \"it would have\",\n            \"it'd\": \"it would\",\n            \"it'll've\": \"it will have\",\n            \"it'll\": \"it will\",\n            \"it's\": \"it is\",\n            \"let's\": \"let us\",\n            \"ma'am\": \"madam\",\n            \"mayn't\": \"may not\",\n            \"might've\": \"might have\",\n            \"mightn't've\": \"might not have\",\n            \"mightn't\": \"might not\",\n            \"must've\": \"must have\",\n            \"mustn't've\": \"must not have\",\n            \"mustn't\": \"must not\",\n            \"needn't've\": \"need not have\",\n            \"needn't\": \"need not\",\n            \"o'clock\": \"of the clock\",\n            \"oughtn't've\": \"ought not have\",\n            \"oughtn't\": \"ought not\",\n            \"shan't've\": \"shall not have\",\n            \"shan't\": \"shall not\",\n            \"sha'n't\": \"shall not\",\n            \"she'd've\": \"she would have\",\n            \"she'd\": \"she would\",\n            \"she'll've\": \"she will have\",\n            \"she'll\": \"she will\",\n            \"she's\": \"she is\",\n            \"should've\": \"should have\",\n            \"shouldn't've\": \"should not have\",\n            \"shouldn't\": \"should not\",\n            \"so've\": \"so have\",\n            \"so's\": \"so as\",\n            \"this's\": \"this is\",\n            \"that'd've\": \"that would have\",\n            \"that'd\": \"that would\",\n            \"that's\": \"that is\",\n            \"there'd've\": \"there would have\",\n            \"there'd\": \"there would\",\n            \"there's\": \"there is\",\n            \"here's\": \"here is\",\n            \"they'd've\": \"they would have\",\n            \"they'd\": \"they would\",\n            \"they'll've\": \"they will have\",\n            \"they'll\": \"they will\",\n            \"they're\": \"they are\",\n            \"they've\": \"they have\",\n            \"to've\": \"to have\",\n            \"wasn't\": \"was not\",\n            \"we'd've\": \"we would have\",\n            \"we'd\": \"we would\",\n            \"we'll've\": \"we will have\",\n            \"we'll\": \"we will\",\n            \"we're\": \"we are\",\n            \"we've\": \"we have\",\n            \"weren't\": \"were not\",\n            \"what'll've\": \"what will have\",\n            \"what'll\": \"what will\",\n            \"what're\": \"what are\",\n            \"what's\": \"what is\",\n            \"what've\": \"what have\",\n            \"when's\": \"when is\",\n            \"when've\": \"when have\",\n            \"where'd\": \"where did\",\n            \"where's\": \"where is\",\n            \"where've\": \"where have\",\n            \"who'll've\": \"who will have\",\n            \"who'll\": \"who will\",\n            \"who's\": \"who is\",\n            \"who've\": \"who have\",\n            \"why's\": \"why is\",\n            \"why've\": \"why have\",\n            \"will've\": \"will have\",\n            \"won't've\": \"will not have\",\n            \"won't\": \"will not\",\n            \"would've\": \"would have\",\n            \"wouldn't've\": \"would not have\",\n            \"wouldn't\": \"would not\",\n            \"y'all'd've\": \"you all would have\",\n            \"y'all'd\": \"you all would\",\n            \"y'all're\": \"you all are\",\n            \"y'all've\": \"you all have\",\n            \"y'all\": \"you all\",\n            \"you'd've\": \"you would have\",\n            \"you'd\": \"you would\",\n            \"you'll've\": \"you will have\",\n            \"you'll\": \"you will\",\n            \"you're\": \"you are\",\n            \"you've\": \"you have\",\n            \"colour\": \"color\",\n            \"centre\": \"center\",\n            \"favourite\": \"favorite\",\n            \"travelling\": \"traveling\",\n            \"counselling\": \"counseling\",\n            \"theatre\": \"theater\",\n            \"cancelled\": \"canceled\",\n            \"labour\": \"labor\",\n            \"organisation\": \"organization\",\n            \"wwii\": \"world war 2\",\n            \"citicise\": \"criticize\",\n            \"youtu \": \"youtube \",\n            \"qoura\": \"quora\",\n            \"sallary\": \"salary\",\n            \"whta\": \"what\",\n            \"narcisist\": \"narcissist\",\n            \"howdo\": \"how do\",\n            \"whatare\": \"what are\",\n            \"howcan\": \"how can\",\n            \"howmuch\": \"how much\",\n            \"howmany\": \"how many\",\n            \"whydo\": \"why do\",\n            \"doi\": \"do i\",\n            \"thebest\": \"the best\",\n            \"howdoes\": \"how does\",\n            \"mastrubation\": \"masturbation\",\n            \"mastrubate\": \"masturbate\",\n            \"mastrubating\": \"masturbating\",\n            \"pennis\": \"penis\",\n            \"etherium\": \"ethereum\",\n            \"narcissit\": \"narcissist\",\n            \"bigdata\": \"big data\",\n            \"2k17\": \"2017\",\n            \"2k18\": \"2018\",\n            \"qouta\": \"quota\",\n            \"exboyfriend\": \"ex boyfriend\",\n            \"airhostess\": \"air hostess\",\n            \"whst\": \"what\",\n            \"watsapp\": \"whatsapp\",\n            \"demonitisation\": \"demonetization\",\n            \"demonitization\": \"demonetization\",\n            \"demonetisation\": \"demonetization\",\n        }\n        super().__init__(rule)\n\n\nregister_preprocessor('lower')(cylower)\nregister_preprocessor('punct')(PunctSpacer())\nregister_preprocessor('unidecode')(unidecode)\nregister_preprocessor('unidecode_weak')(unidecode_weak)\nregister_preprocessor('number')(NumberReplacer())\nregister_preprocessor('number+underscore')(\n    NumberReplacer(with_underscore=True))\nregister_preprocessor('misspell')(MisspellReplacer())\nregister_preprocessor('keras')(KerasFilterReplacer())","metadata":{"_uuid":"b69a0a7f3deea6934905660cf75441b31ac0ac54","execution":{"iopub.status.busy":"2022-02-02T06:10:32.266775Z","iopub.execute_input":"2022-02-02T06:10:32.267107Z","iopub.status.idle":"2022-02-02T06:10:32.680542Z","shell.execute_reply.started":"2022-02-02T06:10:32.267048Z","shell.execute_reply":"2022-02-02T06:10:32.67911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Featurizers","metadata":{"_uuid":"b377e5831c2d83a883355019ab9f488e7b6e4f66"}},{"cell_type":"code","source":"def load_pretrained_vectors(names, token2id, test=False):\n    assert isinstance(names, list)\n    with Pool(processes=len(names)) as pool:\n        f = partial(load_pretrained_vector, token2id=token2id, test=test)\n        vectors = pool.map(f, names)\n    return dict([(n, v) for n, v in zip(names, vectors)])\n\n\ndef load_pretrained_vector(name, token2id, test=False):\n    loader = dict(\n        gnews=GNewsPretrainedVector,\n        wnews=WNewsPretrainedVector,\n        paragram=ParagramPretrainedVector,\n#         glove=GlovePretrainedVector,\n    )\n    return loader[name].load(token2id, test)\n\n\nclass BasePretrainedVector(object):\n\n    @classmethod\n    def load(cls, token2id, test=False, limit=None):\n        embed_shape = (len(token2id), 300)\n        freqs = np.zeros((len(token2id)), dtype='f')\n\n        if test:\n            np.random.seed(0)\n            vectors = np.random.normal(0, 1, embed_shape)\n            vectors[0] = 0\n            vectors[len(token2id) // 2:] = 0\n        else:\n            vectors = np.zeros(embed_shape, dtype='f')\n            path = f'{os.environ[\"DATADIR\"]}/{cls.path}'\n            for i, o in enumerate(\n                    open(path, encoding=\"utf8\", errors='ignore')):\n                token, *vector = o.split(' ')\n                token = str.lower(token)\n                if token not in token2id or len(o) <= 100:\n                    continue\n                if limit is not None and i > limit:\n                    break\n                freqs[token2id[token]] += 1\n                vectors[token2id[token]] += np.array(vector, 'f')\n\n        vectors[freqs != 0] /= freqs[freqs != 0][:, None]\n        vec = KeyedVectors(300)\n        vec.add(list(token2id.keys()), vectors, replace=True)\n\n        return vec\n\n\nclass GNewsPretrainedVector(object):\n\n    name = 'GoogleNews-vectors-negative300'\n    path = f'/{name}/{name}.bin'\n\n    @classmethod\n    def load(cls, tokens, limit=None):\n        raise NotImplementedError\n        path = f'{os.environ[\"DATADIR\"]}/{cls.path}'\n        return KeyedVectors.load_word2vec_format(\n            path, binary=True, limit=limit)\n\n\nclass WNewsPretrainedVector(BasePretrainedVector):\n\n    name = 'wiki-news-300d-1M'\n    path = f'/{name}/{name}.vec'\n\n\nclass ParagramPretrainedVector(BasePretrainedVector):\n\n    name = 'paragram_300_sl999'\n    path = f'/{name}/{name}.txt'\n\n\nclass GlovePretrainedVector(BasePretrainedVector):\n\n    name = 'glove.840B.300d'\n    path = f'/{name}/{name}.txt'","metadata":{"_uuid":"584c36f099b5d69b5291c962cb4c4196eaa75b7e","execution":{"iopub.status.busy":"2022-02-02T06:10:44.717235Z","iopub.execute_input":"2022-02-02T06:10:44.717553Z","iopub.status.idle":"2022-02-02T06:10:44.734147Z","shell.execute_reply.started":"2022-02-02T06:10:44.717484Z","shell.execute_reply":"2022-02-02T06:10:44.732144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@register_word_embedding_features('pretrained')\nclass PretrainedVectorFeaturizer(object):\n\n    def __init__(self, config, vocab):\n        self.config = config\n        self.vocab = vocab\n\n    @classmethod\n    def add_args(self, parser):\n        pass\n\n    def __call__(self, features, datasets):\n        # Nothing to do\n        return features\n\n\nclass Any2VecFeaturizer(object):\n\n    def __init__(self, config, vocab):\n        self.config = config\n        self.vocab = vocab\n\n    def build_fillvalue(self, mode, initialW):\n        n_embed = initialW.shape[1]\n        n_fill = initialW[self.vocab.unk].shape\n        assert mode in {'zeros', 'mean', 'noise'}\n        if mode == 'zeros':\n            return np.zeros(n_embed, 'f')\n        elif mode == 'mean':\n            return initialW.mean(axis=0)\n        elif mode == 'noise':\n            mean, std = initialW.mean(), initialW.std()\n            return np.random.normal(mean, std, (n_fill, n_embed))\n\n    def __call__(self, features, datasets):\n        tokens = np.concatenate([d.tokens for d in datasets])\n        model = self.build_model()\n        model.build_vocab_from_freq(self.vocab.word_freq)\n        initialW = features.copy()\n        initialW[self.vocab.unk] = self.build_fillvalue(\n            self.config.finetune_word2vec_init_unk, initialW)\n        idxmap = np.array(\n            [self.vocab.token2id[w] for w in model.wv.index2entity])\n        model = self.initialize(model, initialW, idxmap)\n        model.train(tokens, total_examples=len(tokens), epochs=model.epochs)\n        finetunedW = np.zeros((initialW.shape), 'f')\n        for i, word in enumerate(self.vocab.token2id):\n            if word in model.wv:\n                finetunedW[i] = model.wv.get_vector(word)\n        return finetunedW\n\n\n@register_word_embedding_features('word2vec')\nclass Word2VecFeaturizer(Any2VecFeaturizer):\n\n    @classmethod\n    def add_args(self, parser):\n        parser.add_argument('--finetune-word2vec-init-unk', type=str,\n                            choices=['zeros', 'mean', 'noise'])\n        parser.add_argument('--finetune-word2vec-mincount', type=int)\n        parser.add_argument('--finetune-word2vec-workers', type=int)\n        parser.add_argument('--finetune-word2vec-iter', type=int)\n        parser.add_argument('--finetune-word2vec-size', type=int)\n        parser.add_argument('--finetune-word2vec-window', type=int, default=5)\n        parser.add_argument('--finetune-word2vec-sorted-vocab', type=int,\n                            default=0)\n        parser.add_argument('--finetune-word2vec-sg', type=int, choices=[0, 1])\n\n    def build_model(self):\n        model = Word2Vec(\n            min_count=self.config.finetune_word2vec_mincount,\n            workers=self.config.finetune_word2vec_workers,\n            iter=self.config.finetune_word2vec_iter,\n            size=self.config.finetune_word2vec_size,\n            window=self.config.finetune_word2vec_window,\n            sg=self.config.finetune_word2vec_sg,\n        )\n        return model\n\n    def initialize(self, model, initialW, idxmap):\n        model.wv.vectors[:] = initialW[idxmap]\n        model.trainables.syn1neg[:] = initialW[idxmap]\n        return model\n\n\n@register_word_embedding_features('fasttext')\nclass FastTextFeaturizer(Any2VecFeaturizer):\n\n    @classmethod\n    def add_args(self, parser):\n        parser.add_argument('--finetune-fasttext-init-unk', type=str,\n                            choices=['zeros', 'mean', 'noise'])\n        parser.add_argument('--finetune-fasttext-mincount', type=int)\n        parser.add_argument('--finetune-fasttext-workers', type=int)\n        parser.add_argument('--finetune-fasttext-iter', type=int)\n        parser.add_argument('--finetune-fasttext-size', type=int)\n        parser.add_argument('--finetune-fasttext-sg', type=int, choices=[0, 1])\n        parser.add_argument('--finetune-fasttext-min_n', type=int)\n        parser.add_argument('--finetune-fasttext-max_n', type=int)\n\n    def build_model(self):\n        model = FastText(\n            min_count=self.config.finetune_fasttext_mincount,\n            workers=self.config.finetune_fasttext_workers,\n            iter=self.config.finetune_fasttext_iter,\n            size=self.config.finetune_fasttext_size,\n            sg=self.config.finetune_fasttext_sg,\n            min_n=self.config.finetune_fasttext_min_n,\n            max_n=self.config.finetune_fasttext_max_n,\n        )\n        return model\n\n    def initialize(self, model, initialW, idxmap):\n        model.wv.vectors[:] = initialW[idxmap]\n        model.wv.vectors_vocab[:] = initialW[idxmap]\n        model.trainables.syn1neg[:] = initialW[idxmap]\n        return model","metadata":{"_uuid":"5213478751cb87cfb53075e6241029f7e64bf132","execution":{"iopub.status.busy":"2022-02-02T06:10:50.497839Z","iopub.execute_input":"2022-02-02T06:10:50.498244Z","iopub.status.idle":"2022-02-02T06:10:50.541034Z","shell.execute_reply.started":"2022-02-02T06:10:50.498179Z","shell.execute_reply":"2022-02-02T06:10:50.539906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@register_word_extra_features('idf')\nclass IDFWordFeaturizer(object):\n\n    def __call__(self, vocab):\n        dfs = np.array(list(vocab.word_freq.values()))\n        dfs[0] = vocab.n_documents\n        features = np.log(vocab.n_documents / dfs)\n        features = features[:, None]\n        return features\n\n\n@register_word_extra_features('unk')\nclass UnkWordFeaturizer(object):\n\n    def __call__(self, vocab):\n        features = vocab.unk.astype('f')\n        features[0] = 0\n        features = features[:, None]\n        return features","metadata":{"_uuid":"08f65c6569b616b1968600bc25fb71fdce3d4b01","execution":{"iopub.status.busy":"2022-02-02T06:10:57.31462Z","iopub.execute_input":"2022-02-02T06:10:57.315233Z","iopub.status.idle":"2022-02-02T06:10:57.323081Z","shell.execute_reply.started":"2022-02-02T06:10:57.31494Z","shell.execute_reply":"2022-02-02T06:10:57.321527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@register_sentence_extra_features('char')\nclass CharacterStatisticsFeaturizer(object):\n\n    n_dims = 3\n\n    def __call__(self, sentence):\n        feature = {}\n        feature['n_chars'] = len(sentence)\n        feature['n_caps'] = sum(1 for char in sentence if char.isupper())\n        feature['caps_rate'] = feature['n_caps'] / feature['n_chars']\n        features = np.array(list(feature.values()))\n        return features\n\n\n@register_sentence_extra_features('word')\nclass WordStatisticsFeaturizer(object):\n\n    n_dims = 3\n\n    def __call__(self, sentence):\n        feature = {}\n        tokens = sentence.split()\n        feature['n_words'] = len(tokens)\n        feature['unique_words'] = len(set(tokens))\n        feature['unique_rate'] = feature['unique_words'] / feature['n_words']\n        features = np.array(list(feature.values()))\n        return features","metadata":{"_uuid":"46357e517227107e6279b9a1fab2343ee40a9f03","execution":{"iopub.status.busy":"2022-02-02T06:11:01.390281Z","iopub.execute_input":"2022-02-02T06:11:01.390668Z","iopub.status.idle":"2022-02-02T06:11:01.412024Z","shell.execute_reply.started":"2022-02-02T06:11:01.390612Z","shell.execute_reply":"2022-02-02T06:11:01.410688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Tokenizer","metadata":{"_uuid":"b27798fd45971455def10ff158d3fe11f9c8f3d7"}},{"cell_type":"code","source":"%%cython\ncpdef list cysplit(str x):\n    return x.split()","metadata":{"_uuid":"3475624974a1703aabe270e48cbbe6ac60dc3c31","execution":{"iopub.status.busy":"2022-02-02T06:11:07.502749Z","iopub.execute_input":"2022-02-02T06:11:07.503137Z","iopub.status.idle":"2022-02-02T06:11:07.821321Z","shell.execute_reply.started":"2022-02-02T06:11:07.503079Z","shell.execute_reply":"2022-02-02T06:11:07.819792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"register_tokenizer('space')(cysplit)\nregister_tokenizer('word_tokenize')(nltk.word_tokenize)","metadata":{"_uuid":"8714158f97c7a3374982254e96b4fb0be2c5d67b","execution":{"iopub.status.busy":"2022-02-02T06:11:10.313245Z","iopub.execute_input":"2022-02-02T06:11:10.31364Z","iopub.status.idle":"2022-02-02T06:11:10.323128Z","shell.execute_reply.started":"2022-02-02T06:11:10.313546Z","shell.execute_reply":"2022-02-02T06:11:10.321827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Wrappers","metadata":{"_uuid":"9667c26e4e72bf07c7f6ff42538574162ccbdadc"}},{"cell_type":"code","source":"class TextNormalizerWrapper(object):\n\n    registry = NORMALIZER_REGISTRY\n    default_config = None\n\n    def __init__(self, config):\n        self.normalizers = [self.registry[n] for n in config.normalizers]\n\n    @classmethod\n    def add_args(cls, parser):\n        assert isinstance(cls.default_config, dict)\n        parser.add_argument(\n            '--normalizers', nargs='+', choices=cls.registry)\n        parser.set_defaults(**cls.default_config)\n\n    def __call__(self, x):\n        for normalizer in self.normalizers:\n            x = normalizer(x)\n        return x\n\n    \nclass TextTokenizerWrapper(object):\n\n    registry = TOKENIZER_REGISTRY\n    default_config = None\n\n    def __init__(self, config):\n        self.tokenizer = self.registry[config.tokenizer]\n\n    @classmethod\n    def add_args(cls, parser):\n        assert isinstance(cls.default_config, dict)\n        parser.add_argument('--tokenizer', choices=cls.registry)\n        parser.set_defaults(**cls.default_config)\n\n    def __call__(self, x):\n        return self.tokenizer(x)\n\n    \nclass WordEmbeddingFeaturizerWrapper(object):\n\n    registry = WORD_EMBEDDING_FEATURIZER_REGISTRY\n    default_config = None\n    default_extra_config = None\n\n    def __init__(self, config, vocab):\n        self.config = config\n        self.vocab = vocab\n        self.featurizers = {\n            k: self.registry[k](config, vocab)\n            for k in config.word_embedding_features}\n\n    @classmethod\n    def add_args(cls, parser):\n        assert isinstance(cls.default_config, dict)\n        parser.add_argument(\n            '--use-pretrained-vectors', nargs='+',\n            choices=[ 'paragram', 'wnews', 'gnews'])\n        parser.add_argument(\n            '--word-embedding-features', nargs='+', choices=cls.registry)\n        parser.set_defaults(**cls.default_config)\n\n    @classmethod\n    def add_extra_args(cls, parser, config):\n        assert isinstance(cls.default_extra_config, dict)\n        for featurizer in config.word_embedding_features:\n            cls.registry[featurizer].add_args(parser)\n        parser.set_defaults(**cls.default_extra_config)\n\n    def __call__(self, features, datasets):\n        return {k: feat(features, datasets)\n                for k, feat in self.featurizers.items()}\n\n\nclass WordExtraFeaturizerWrapper(object):\n\n    registry = WORD_EXTRA_FEATURIZER_REGISTRY\n    default_config = None\n\n    def __init__(self, config, vocab):\n        self.config = config\n        self.vocab = vocab\n        self.featurizers = {\n            k: self.registry[k]() for k in config.word_extra_features}\n\n    @classmethod\n    def add_args(cls, parser):\n        parser.add_argument(\n            '--word-extra-features', nargs='+', choices=cls.registry)\n        parser.set_defaults(**cls.default_config)\n\n    def __call__(self, vocab):\n        empty = np.empty([len(vocab), 0])\n        return np.concatenate([empty, *[\n            f(vocab) for f in self.featurizers.values()]], axis=1)\n\n\nclass SentenceExtraFeaturizerWrapper(object):\n\n    registry = SENTENCE_EXTRA_FEATURIZER_REGISTRY\n    default_config = None\n\n    def __init__(self, config):\n        self.config = config\n        self.featurizers = {\n            k: self.registry[k]() for k in config.sentence_extra_features}\n        self.n_dims = sum(list(f.n_dims for f in self.featurizers.values()))\n\n    @classmethod\n    def add_args(cls, parser):\n        parser.add_argument(\n            '--sentence-extra-features', nargs='+', choices=cls.registry)\n        parser.set_defaults(**cls.default_config)\n\n    def __call__(self, sentence):\n        empty = np.empty((0,))\n        return np.concatenate([empty, *[\n            f(sentence) for f in self.featurizers.values()]], axis=0)\n\n    def fit_standardize(self, features):\n        assert features.ndim == 2\n        self.mean = features.mean(axis=0)\n        self.std = features.std(axis=0)\n        self.std = np.where(self.std != 0, self.std, 1)\n        return (features - self.mean) / self.std\n\n    def standardize(self, features):\n        assert hasattr(self, 'mean'), hasattr(self, 'std')\n        return (features - self.mean) / self.std\n","metadata":{"_uuid":"79f3c1b5119a8bdc16269671dcdbfd72ee44a21d","execution":{"iopub.status.busy":"2022-02-02T06:11:14.134115Z","iopub.execute_input":"2022-02-02T06:11:14.13453Z","iopub.status.idle":"2022-02-02T06:11:14.158945Z","shell.execute_reply.started":"2022-02-02T06:11:14.134458Z","shell.execute_reply":"2022-02-02T06:11:14.157458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Preprocessor","metadata":{"_uuid":"9aecfa261368ff7b834d4dfb5257cf26e3d3b877"}},{"cell_type":"code","source":"class WordbasedPreprocessor():\n\n    def tokenize(self, datasets, normalizer, tokenizer):\n        tokenize = Pipeline(normalizer, tokenizer)\n        apply_tokenize = ApplyNdArray(tokenize, processes=2, dtype=object)\n        tokens = [apply_tokenize(d.df.question_text.values) for d in datasets]\n        return tokens\n\n    def build_vocab(self, datasets, config):\n        train_dataset, test_dataset, submit_dataset = datasets\n        vocab = WordVocab(mincount=config.vocab_mincount)\n        vocab.add_documents(train_dataset.positives.tokens, 'train-pos')\n        vocab.add_documents(train_dataset.negatives.tokens, 'train-neg')\n        vocab.add_documents(test_dataset.positives.tokens, 'test-pos')\n        vocab.add_documents(test_dataset.negatives.tokens, 'test-neg')\n        vocab.add_documents(submit_dataset.df.tokens, 'submit')\n        vocab.build()\n        return vocab\n\n    def build_tokenids(self, datasets, vocab, config):\n        token2id = lambda xs: pad_sequence(  # NOQA\n            [vocab.token2id[x] for x in xs], config.maxlen)\n        apply_token2id = ApplyNdArray(\n            token2id, processes=1, dtype='i', dims=(config.maxlen,))\n        tokenids = [apply_token2id(d.df.tokens.values) for d in datasets]\n        return tokenids\n\n    def build_sentence_features(self, datasets, sentence_extra_featurizer):\n        train_dataset, test_dataset, submit_dataset = datasets\n        apply_featurize = ApplyNdArray(\n            sentence_extra_featurizer, processes=1, dtype='f',\n            dims=(sentence_extra_featurizer.n_dims,))\n        _X2 = [apply_featurize(d.df.question_text.values) for d in datasets]\n        _train_X2, _test_X2, _submit_X2 = _X2\n        train_X2 = sentence_extra_featurizer.fit_standardize(_train_X2)\n        test_X2 = sentence_extra_featurizer.standardize(_test_X2)\n        submit_X2 = sentence_extra_featurizer.standardize(_submit_X2)\n        return train_X2, test_X2, submit_X2\n\n    def build_embedding_matrices(self, datasets, word_embedding_featurizer,\n                                 vocab, pretrained_vectors):\n        pretrained_vectors_merged = np.stack(\n            [wv.vectors for wv in pretrained_vectors.values()]).mean(axis=0)\n        vocab.unk = (pretrained_vectors_merged == 0).all(axis=1)\n        vocab.known = ~vocab.unk\n        embedding_matrices = word_embedding_featurizer(\n            pretrained_vectors_merged, datasets)\n        return embedding_matrices\n\n    def build_word_features(self, word_embedding_featurizer,\n                            embedding_matrices, word_extra_features):\n        embedding = np.stack(list(embedding_matrices.values()))\n        embedding = embedding.mean(axis=0)\n        word_features = np.concatenate(\n            [embedding, word_extra_features], axis=1)\n        return word_features","metadata":{"_uuid":"5c90256ff3625e2f0e0c96eecf30e502ca9159b6","execution":{"iopub.status.busy":"2022-02-02T06:11:19.125263Z","iopub.execute_input":"2022-02-02T06:11:19.125615Z","iopub.status.idle":"2022-02-02T06:11:19.14307Z","shell.execute_reply.started":"2022-02-02T06:11:19.125555Z","shell.execute_reply":"2022-02-02T06:11:19.141425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## NN modules","metadata":{"_uuid":"6c3f4efda0306adace01c1f34cdf16e42345ca91"}},{"cell_type":"markdown","source":"### Encoder","metadata":{"_uuid":"0b710c056eb82dc780fda757bf5425b0e7dcdf89"}},{"cell_type":"code","source":"class RNNEncoderBase(nn.Module):\n\n    def __init__(self, config, modules, in_size):\n        super().__init__()\n        rnns = []\n        input_size = in_size\n        for module in modules:\n            rnn = module(\n                input_size=input_size,\n                hidden_size=config.encoder_n_hidden,\n                bidirectional=config.encoder_bidirectional,\n                batch_first=True,\n            )\n            n_direction = int(config.encoder_bidirectional) + 1\n            input_size = n_direction * config.encoder_n_hidden\n            rnns.append(rnn)\n        self.rnns = nn.ModuleList(rnns)\n        self.out_size = n_direction * config.encoder_n_hidden\n\n    @classmethod\n    def add_args(self, parser):\n        parser.add_argument('--encoder-bidirectional', type=bool, default=True)\n        parser.add_argument('--encoder-dropout', type=float, default=0.)\n        parser.add_argument('--encoder-n-hidden', type=int)\n        parser.add_argument('--encoder-n-layers', type=int)\n        parser.add_argument('--encoder-aggregator', type=str,\n                            choices=AGGREGATOR_REGISTRY)\n\n    def forward(self, input, mask):\n        h = input\n        for rnn in self.rnns:\n            h, _ = rnn(h)\n        return h\n\n\n@register_encoder('lstm')\nclass LSTMEncoder(RNNEncoderBase):\n\n    def __init__(self, config, in_size):\n        modules = [nn.LSTM] * config.encoder_n_layers\n        super().__init__(config, modules, in_size)\n\n\n@register_encoder('gru')\nclass GRUEncoder(RNNEncoderBase):\n\n    def __init__(self, config, in_size):\n        assert config.encoder_n_layers > 1\n        modules = [nn.GRU] * config.encoder_n_layers\n        super().__init__(config, modules, in_size)\n\n\n@register_encoder('lstmgru')\nclass LSTMGRUEncoder(RNNEncoderBase):\n\n    def __init__(self, config, in_size):\n        assert config.encoder_n_layers > 1\n        modules = [nn.LSTM] * (config.encoder_n_layers - 1) + [nn.GRU]\n        super().__init__(config, modules, in_size)\n\n\n@register_encoder('grulstm')\nclass GRULSTMEncoder(RNNEncoderBase):\n\n    def __init__(self, config, in_size):\n        assert config.encoder_n_layers > 1\n        modules = [nn.GRU] * (config.encoder_n_layers - 1) + [nn.LSTM]\n        super().__init__(config, modules, in_size)","metadata":{"_uuid":"299eb88ddb5d16621fb9a8bea4bb3bc30a2fe431","execution":{"iopub.status.busy":"2022-02-02T06:11:23.794495Z","iopub.execute_input":"2022-02-02T06:11:23.794858Z","iopub.status.idle":"2022-02-02T06:11:23.809288Z","shell.execute_reply.started":"2022-02-02T06:11:23.7948Z","shell.execute_reply":"2022-02-02T06:11:23.808141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Aggregator","metadata":{"_uuid":"c0d2ca402d44ddadf492f4acc1ecc6a271b4653b"}},{"cell_type":"code","source":"@register_aggregator('max')\nclass MaxPoolingAggregator(nn.Module):\n\n    def __call__(self, hs, mask):\n        if mask is not None:\n            hs = hs.masked_fill(~mask.unsqueeze(2), -np.inf)\n        h = hs.max(dim=1)[0]\n        return h\n\n\n@register_aggregator('sum')\nclass SumPoolingAggregator(nn.Module):\n\n    def __call__(self, hs, mask):\n        if mask is not None:\n            hs = hs.masked_fill(~mask.unsqueeze(2), 0)\n        h = hs.sum(dim=1)\n        return h\n\n\n@register_aggregator('avg')\nclass AvgPoolingAggregator(nn.Module):\n\n    def __call__(self, hs, mask):\n        if mask is not None:\n            hs = hs.masked_fill(~mask.unsqueeze(2), 0)\n        h = hs.sum(dim=1)\n        maxlen = mask.sum(dim=1)\n        h /= maxlen[:, None].type(torch.float)\n        return h\n","metadata":{"_uuid":"8e2807bdbc3838e10b549a6188f0096a5f5bcdf1","execution":{"iopub.status.busy":"2022-02-02T06:11:27.789383Z","iopub.execute_input":"2022-02-02T06:11:27.789766Z","iopub.status.idle":"2022-02-02T06:11:27.798972Z","shell.execute_reply.started":"2022-02-02T06:11:27.789685Z","shell.execute_reply":"2022-02-02T06:11:27.79734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Ensembler","metadata":{"_uuid":"499506a2b873dcdc3fb5f3066cf5e6b7783d6ced"}},{"cell_type":"code","source":"class BaseEnsembler(metaclass=ABCMeta):\n\n    def __init__(self, config, models, results):\n        super().__init__()\n        self.config = config\n        self.models = models\n        self.results = results\n\n    @abstractmethod\n    def fit(self, X, t, test_size=0.1):\n        pass\n\n    @abstractmethod\n    def predict_proba(self, X, X2):\n        pass\n\n    def predict(self, X, X2):\n        y = self.predict_proba(X, X2)\n        return (y > self.threshold).astype('i')\n\n    \nclass AverageEnsembler(BaseEnsembler):\n\n    def __init__(self, config, models, results):\n        self.config = config\n        self.models = models\n        self.results = results\n        self.device = config.device\n        self.batchsize_train = config.batchsize\n        self.batchsize_valid = config.batchsize_valid\n        self.threshold_cv = np.array(\n            [m.threshold for m in models]).mean()\n        self.threshold = self.threshold_cv\n\n    def fit(self, X, X2, t, test_size=0.1):\n        # Nothing to do\n        pass\n\n    def predict_proba(self, X, X2):\n        pred_X = X.to(self.device)\n        pred_X2 = X2.to(self.device)\n        dataset = torch.utils.data.TensorDataset(pred_X, pred_X2)\n        iterator = DataLoader(\n            dataset, batch_size=self.batchsize_valid, shuffle=False)\n        ys = defaultdict(list)\n        for batch in tqdm(iterator, desc='submit', leave=False):\n            for i, model in enumerate(self.models):\n                model.eval()\n                ys[i].append(model.predict_proba(*batch))\n        ys = np.concatenate(\n            [np.concatenate(_ys) for _ys in ys.values()], axis=1)\n        y = ys.mean(axis=1, keepdims=True)\n        return y\n","metadata":{"_uuid":"5d94c80036184d23bcc0b5f72c541299af7760cc","execution":{"iopub.status.busy":"2022-02-02T06:11:32.322205Z","iopub.execute_input":"2022-02-02T06:11:32.32259Z","iopub.status.idle":"2022-02-02T06:11:32.336356Z","shell.execute_reply.started":"2022-02-02T06:11:32.322515Z","shell.execute_reply":"2022-02-02T06:11:32.335118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Classifier","metadata":{"_uuid":"c99afd2aa368d75d5ce34fc4baac133ceae654e1"}},{"cell_type":"code","source":"class BinaryClassifier(nn.Module):\n\n    default_config = None\n\n    def __init__(self, embedding, encoder, aggregator, mlp, out, lossfunc):\n        super().__init__()\n        self.embedding = embedding\n        self.encoder = encoder\n        self.aggregator = aggregator\n        self.mlp = mlp\n        self.out = out\n        print(\"mlp\",out)\n        self.lossfunc = lossfunc\n\n    def calc_loss(self, X, X2, t, W=None):\n        y = self.forward(X, X2)\n        loss = self.lossfunc(y, t)\n        output = dict(\n            y=torch.sigmoid(y).cpu().detach().numpy(),\n            t=t.cpu().detach().numpy(),\n            loss=loss.cpu().detach().numpy(),\n        )\n        return loss, output\n\n    def to_device(self, device):\n        self.device = device\n        self.to(device)\n        return self\n\n    def forward(self, X, X2):\n        h = self.predict_features(X, X2)\n        out = self.out(h)\n        return out\n\n    def predict_proba(self, X, X2):\n        y = self.forward(X, X2)\n        proba = torch.sigmoid(y).cpu().detach().numpy()\n        return proba\n\n    def predict_features(self, X, X2):\n        mask = X != 0\n        maxlen = (mask == 1).any(dim=0).sum()\n        X = X[:, :maxlen]\n        mask = mask[:, :maxlen]\n\n        h = self.embedding(X)\n        h = self.encoder(h, mask)\n        h = self.aggregator(h, mask)\n        h = self.mlp(h, X2)\n        return h","metadata":{"_uuid":"7b424873f73da2526c875dd4155fa0ed579a6ef7","execution":{"iopub.status.busy":"2022-02-02T06:11:37.067849Z","iopub.execute_input":"2022-02-02T06:11:37.06822Z","iopub.status.idle":"2022-02-02T06:11:37.081786Z","shell.execute_reply.started":"2022-02-02T06:11:37.068155Z","shell.execute_reply":"2022-02-02T06:11:37.080693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Wrappers","metadata":{"_uuid":"2dcb12487daf529b254a65336b5b9a1d866857ac"}},{"cell_type":"code","source":"class NNModuleWrapperBase(nn.Module, metaclass=ABCMeta):\n\n    @abstractmethod\n    def add_args(cls, parser):\n        raise NotImplementedError()\n\n    @abstractmethod\n    def add_extra_args(cls, parser):\n        raise NotImplementedError()\n\n        \nclass EmbeddingWrapper(NNModuleWrapperBase):\n\n    default_config = None\n\n    def __init__(self, config, embedding_matrix):\n        super().__init__()\n        self.config = config\n        self.module = nn.Embedding.from_pretrained(\n            torch.Tensor(embedding_matrix), freeze=True)\n        if self.config.embedding_dropout1d > 0:\n            self.dropout1d = nn.Dropout(config.embedding_dropout1d)\n        if self.config.embedding_dropout2d > 0:\n            self.dropout2d = nn.Dropout2d(config.embedding_dropout2d)\n        if self.config.embedding_spatial_dropout > 0:\n            self.spatial_dropout = nn.Dropout2d(\n                config.embedding_spatial_dropout)\n        self.out_size = embedding_matrix.shape[1]\n\n    @classmethod\n    def add_args(cls, parser):\n        assert isinstance(cls.default_config, dict)\n        parser.add_argument('--embedding-dropout1d', type=float, default=0.)\n        parser.add_argument('--embedding-dropout2d', type=float, default=0.)\n        parser.add_argument('--embedding-spatial-dropout',\n                            type=float, default=0.)\n        parser.set_defaults(**cls.default_config)\n\n    @classmethod\n    def add_extra_args(cls, parser, config):\n        pass\n\n    def forward(self, X):\n        h = self.module(X)\n        if self.config.embedding_dropout1d > 0:\n            h = self.dropout1d(h)\n        if self.config.embedding_dropout2d > 0:\n            h = self.dropout2d(h)\n        if self.config.embedding_spatial_dropout > 0:\n            h = h.permute(0, 2, 1)\n            h = self.spatial_dropout(h)\n            h = h.permute(0, 2, 1)\n        return h\n\n    \nclass EncoderWrapper(nn.Module):\n\n    registry = ENCODER_REGISTRY\n\n    def __init__(self, config, in_size):\n        super().__init__()\n        self.config = config\n        self.module = self.registry[config.encoder](config, in_size)\n        self.out_size = self.module.out_size\n\n    @classmethod\n    def add_args(cls, parser):\n        assert isinstance(cls.default_config, dict)\n        parser.add_argument(\n            '--encoder', choices=cls.registry)\n        parser.set_defaults(**cls.default_config)\n\n    @classmethod\n    def add_extra_args(cls, parser, config):\n        assert isinstance(cls.default_extra_config, dict)\n        cls.registry[config.encoder].add_args(parser)\n        parser.set_defaults(**cls.default_extra_config)\n\n    def forward(self, X, mask):\n        h = self.module(X, mask)\n        return h\n\n    \nclass AggregatorWrapper(NNModuleWrapperBase):\n\n    default_config = None\n    registry = AGGREGATOR_REGISTRY\n\n    def __init__(self, config):\n        super().__init__()\n        self.config = config\n        self.module = self.registry[config.aggregator]()\n\n    @classmethod\n    def add_args(cls, parser):\n        assert isinstance(cls.default_config, dict)\n        parser.add_argument('--aggregator',\n                            choices=cls.registry)\n        parser.set_defaults(**cls.default_config)\n\n    @classmethod\n    def add_extra_args(cls, parser, config):\n        pass\n\n    def forward(self, X, mask):\n        h = self.module(X, mask)\n        return h\n\n    \nclass MLPWrapper(NNModuleWrapperBase):\n\n    default_config = None\n\n    def __init__(self, config, in_size):\n        super().__init__()\n        self.in_size = in_size\n        self.config = config\n        assert isinstance(config.mlp_n_hiddens, list)\n        layers = []\n        if config.mlp_bn0:\n            layers.append(nn.BatchNorm1d(in_size))\n        if config.mlp_dropout0 > 0:\n            layers.append(nn.Dropout(config.mlp_dropout0))\n        for n_hidden in config.mlp_n_hiddens:\n            layers.append(nn.Linear(in_size, n_hidden))\n            if config.mlp_actfun is not None:\n                layers.append(config.mlp_actfun)\n            if config.mlp_bn:\n                layers.append(nn.BatchNorm1d(n_hidden))\n            if config.mlp_dropout > 0:\n                layers.append(nn.Dropout(config.mlp_dropout))\n            in_size = n_hidden\n        self.layers = nn.Sequential(*layers)\n\n    @classmethod\n    def add_args(cls, parser):\n        assert isinstance(cls.default_config, dict)\n        parser.add_argument('--mlp-n-hiddens', type=list)\n        parser.add_argument('--mlp-bn', type=bool)\n        parser.add_argument('--mlp-bn0', type=bool)\n        parser.add_argument('--mlp-dropout', type=float, default=0.)\n        parser.add_argument('--mlp-dropout0', type=float, default=0.)\n        parser.add_argument('--mlp-actfun', default=0.)\n        parser.set_defaults(**cls.default_config)\n\n    @classmethod\n    def add_extra_args(cls, parser, config):\n        pass\n\n    def forward(self, X, X2):\n        h = X\n        if X.shape[1] + X2.shape[1] == self.in_size:\n            h = torch.cat([h, X2], dim=1)\n        h = self.layers(h)\n        return h\n","metadata":{"_uuid":"8ec05bb6aacf562834ac6ba0ec7b436f54b56550","execution":{"iopub.status.busy":"2022-02-02T06:11:43.548318Z","iopub.execute_input":"2022-02-02T06:11:43.548643Z","iopub.status.idle":"2022-02-02T06:11:43.57667Z","shell.execute_reply.started":"2022-02-02T06:11:43.548589Z","shell.execute_reply":"2022-02-02T06:11:43.575435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Presets","metadata":{"_uuid":"7570cb3f9c60ca8e79132669f14a4045276176dc"}},{"cell_type":"code","source":"# =======  Experiment configuration  =======\n\nclass ExperimentConfigBuilderPresets(ExperimentConfigBuilderBase):\n\n    default_config = dict(\n        maxlen=72,\n        vocab_mincount=5,\n        scale_batchsize=[],\n        validate_from=2,\n    )\n\n\n# =======  Preprocessing modules  =======\n\nclass TextNormalizerPresets(TextNormalizerWrapper):\n\n    default_config = dict(\n        normalizers=[\n            'lower',\n            'misspell',\n            'punct',\n            'number+underscore'\n        ]\n    )\n\n\nclass TextTokenizerPresets(TextTokenizerWrapper):\n\n    default_config = dict(\n        tokenizer='space'\n    )\n\n\nclass WordEmbeddingFeaturizerPresets(WordEmbeddingFeaturizerWrapper):\n\n    default_config = dict(\n        use_pretrained_vectors=[ 'paragram'],\n        word_embedding_features=['pretrained', 'word2vec'],\n    )\n    default_extra_config = dict(\n        finetune_word2vec_init_unk='zeros',\n        finetune_word2vec_mincount=1,\n        finetune_word2vec_workers=1,\n        finetune_word2vec_iter=5,\n        finetune_word2vec_size=300,\n        finetune_word2vec_sg=0,\n        finetune_word2vec_sorted_vocab=0,\n    )\n\n\nclass WordExtraFeaturizerPresets(WordExtraFeaturizerWrapper):\n\n    default_config = dict(\n        word_extra_features=[],\n    )\n\n\nclass SentenceExtraFeaturizerPresets(SentenceExtraFeaturizerWrapper):\n\n    default_config = dict(\n        sentence_extra_features=[],\n    )\n\n\nclass PreprocessorPresets(WordbasedPreprocessor):\n\n    def build_word_features(self, word_embedding_featurizer,\n                            embedding_matrices, word_extra_features):\n        embedding = np.stack(list(embedding_matrices.values()))\n\n        # Add noise\n        unk = (embedding[0] == 0).all(axis=1)\n        mean, std = embedding[0, ~unk].mean(), embedding[0, ~unk].std()\n        unk_and_hfq = unk & word_embedding_featurizer.vocab.hfq\n        noise = np.random.normal(\n            mean, std, (unk_and_hfq.sum(), embedding[0].shape[1]))\n        embedding[0, unk_and_hfq] = noise\n        embedding[0, 0] = 0\n\n        embedding = embedding.mean(axis=0)\n        word_features = np.concatenate(\n            [embedding, word_extra_features], axis=1)\n        return word_features\n\n\n# =======  Training modules  =======\n\nclass EmbeddingPresets(EmbeddingWrapper):\n\n    default_config = dict(\n        embedding_dropout1d=0.2,\n    )\n\n\nclass EncoderPresets(EncoderWrapper):\n\n    default_config = dict(\n        encoder='lstm',\n    )\n    default_extra_config = dict(\n        encoder_bidirectional=True,\n        encoder_dropout=0.,\n        encoder_n_layers=2,\n        encoder_n_hidden=128,\n    )\n\n\nclass AggregatorPresets(AggregatorWrapper):\n\n    default_config = dict(\n        aggregator='max',\n    )\n\n\nclass MLPPresets(MLPWrapper):\n\n    default_config = dict(\n        mlp_n_hiddens=[128, 128],\n        mlp_bn0=False,\n        mlp_dropout0=0.,\n        mlp_bn=True,\n        mlp_actfun=nn.ReLU(True),\n    )\n\n\nclass EnsemblerPresets(AverageEnsembler):\n    pass\n","metadata":{"_uuid":"9473fb9029e9a68c8dfa865a0674e0c0bfaf74e1","execution":{"iopub.status.busy":"2022-02-02T06:11:52.002265Z","iopub.execute_input":"2022-02-02T06:11:52.002578Z","iopub.status.idle":"2022-02-02T06:11:52.024888Z","shell.execute_reply.started":"2022-02-02T06:11:52.002518Z","shell.execute_reply":"2022-02-02T06:11:52.023792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training modules","metadata":{"_uuid":"a50590eb55f7652b81ca1548edc46e468cc52201"}},{"cell_type":"markdown","source":"### Model selection","metadata":{"_uuid":"d8d4d183555c69f4b60266cacc37721740b9e76b"}},{"cell_type":"code","source":"def classification_metrics(ys, ts):\n    scores = {}\n\n    if len(np.unique(ts)) > 1:\n        # Search optimal threshold\n        precs, recs, thresholds = metrics.precision_recall_curve(ts, ys)\n        thresholds = np.append(thresholds, 1.001)\n        idx = (precs != 0) * (recs != 0)\n        precs, recs, thresholds = precs[idx], recs[idx], thresholds[idx]\n        fbetas = 2 / (1 / precs + 1 / recs)\n        best_idx = np.argmax(fbetas)\n        threshold = thresholds[best_idx]\n        prec = precs[best_idx]\n        rec = recs[best_idx]\n        fbeta = fbetas[best_idx]\n\n        scores['ap'] = metrics.average_precision_score(ts, ys)\n        scores['rocauc'] = metrics.roc_auc_score(ts, ys)\n        scores['threshold'] = threshold\n        scores['prec'] = prec\n        scores['rec'] = rec\n        scores['fbeta'] = fbeta\n        \n        print(\"threshold\",threshold)\n\n    return scores\n\n\nclass ClassificationResult(object):\n\n    def __init__(self, name, outdir=None, postfix=None, main_metrics='fbeta'):\n        self.initialize()\n        self.name = name\n        self.postfix = postfix\n        self.outdir = outdir\n        self.summary = None\n        self.main_metrics = main_metrics\n        self.n_trained = 0\n\n    def initialize(self):\n        self.losses = []\n        self.ys = []\n        self.ts = []\n\n    def add_record(self, loss, y, t):\n        self.losses.append(loss)\n        self.ys.append(y)\n        self.ts.append(t)\n        self.n_trained += len(y)\n\n    def calc_score(self, epoch):\n        loss = np.array(self.losses).mean()\n        self.ys, self.ts = np.concatenate(self.ys), np.concatenate(self.ts)\n        score = classification_metrics(self.ys, self.ts)\n        summary = dict(name=self.name, loss=loss, **score)\n        if len(score) > 0:\n            if self.summary is None:\n                self.summary = pd.DataFrame([summary], index=[epoch])\n                self.summary.index.name = 'epoch'\n            else:\n                self.summary.loc[epoch] = summary\n        if self.best_epoch == epoch:\n            self.best_ys = self.ys\n            self.best_ts = self.ts\n        self.initialize()\n\n    def get_dict(self):\n        loss, fbeta, epoch = 0, 0, 0\n        if self.summary is not None:\n            row = self.summary.iloc[-1]\n            epoch = row.name\n            loss = row.loss\n            fbeta = row.fbeta\n        return {\n            'epoch': epoch,\n            'loss': loss,\n            'fbeta': fbeta,\n        }\n\n    @property\n    def fbeta(self):\n        if self.summary is None:\n            return 0\n        else:\n            return self.summary.fbeta[-1]\n\n    @property\n    def best_fbeta(self):\n        return self.summary[self.main_metrics].max()\n\n    @property\n    def best_epoch(self):\n        return self.summary[self.main_metrics].idxmax()\n\n    @property\n    def best_threshold(self):\n        idx = self.summary[self.main_metrics].idxmax()\n        return self.summary['threshold'][idx]","metadata":{"_uuid":"6402a4ca7280ac6174c390f94605ae8b106fc74d","execution":{"iopub.status.busy":"2022-02-02T06:11:58.843618Z","iopub.execute_input":"2022-02-02T06:11:58.843996Z","iopub.status.idle":"2022-02-02T06:11:58.865784Z","shell.execute_reply.started":"2022-02-02T06:11:58.84392Z","shell.execute_reply":"2022-02-02T06:11:58.864475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utils","metadata":{"_uuid":"e30da7b12dc882bcb1fb917bc7b1fe3bce43f315"}},{"cell_type":"code","source":"%%cython\nimport numpy as np\ncimport numpy as np\nfrom multiprocessing import Pool\n\n\ncdef class ApplyNdArray:\n    cdef func\n    cdef dtype\n    cdef dims\n    cdef int processes\n\n    def __init__(self, func, processes=1, dtype=object, dims=None):\n        self.func = func\n        self.processes = processes\n        self.dtype = dtype\n        self.dims = dims\n\n    def __call__(self, arr):\n        if self.processes == 1:\n            return self.apply(arr)\n        else:\n            return self.apply_parallel(arr)\n\n    cpdef apply(self, arr):\n        cdef int i\n        cdef int n = len(arr)\n        if self.dims is not None:\n            shape = (n, *self.dims)\n        else:\n            shape = n\n        cdef res = np.empty(shape, dtype=self.dtype)\n        for i in range(n):\n            res[i] = self.func(arr[i])\n        return res\n\n    cpdef apply_parallel(self, arr):\n        cdef list arrs = np.array_split(arr, self.processes)\n        with Pool(processes=self.processes) as pool:\n            outputs = pool.map(self.apply, arrs)\n        return np.concatenate(outputs, axis=0)\n","metadata":{"_uuid":"bceeedb79f91b2136fb6a0bfe077e9cbe05c166f","execution":{"iopub.status.busy":"2022-02-02T06:12:07.49438Z","iopub.execute_input":"2022-02-02T06:12:07.495109Z","iopub.status.idle":"2022-02-02T06:12:09.800544Z","shell.execute_reply.started":"2022-02-02T06:12:07.494764Z","shell.execute_reply":"2022-02-02T06:12:09.799337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_module(filename):\n    assert isinstance(filename, Path)\n    name = filename.stem\n    spec = importlib.util.spec_from_file_location(name, filename)\n    mod = importlib.util.module_from_spec(spec)\n    spec.loader.exec_module(mod)\n    sys.modules[mod.__name__] = mod\n    return mod\n\n\ndef rmtree_after_confirmation(path, force=False):\n    if Path(path).exists():\n        if not force and not prompter.yesno('Overwrite %s?' % path):\n            sys.exit(0)\n        else:\n            shutil.rmtree(path)\n\n\ndef pad_sequence(xs, length, padding_value=0):\n    assert isinstance(xs, list)\n    n_padding = length - len(xs)\n    return np.array(xs + [padding_value] * n_padding, 'i')[:length]\n\n\ndef set_seed(seed=0):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\n\nclass Pipeline(object):\n\n    def __init__(self, *modules):\n        self.modules = modules\n\n    def __call__(self, x):\n        for module in self.modules:\n            x = module(x)\n        return x","metadata":{"_uuid":"37299fdada70ec09b6275849662dc47c9bc2e5d2","execution":{"iopub.status.busy":"2022-02-02T06:12:18.575145Z","iopub.execute_input":"2022-02-02T06:12:18.575597Z","iopub.status.idle":"2022-02-02T06:12:18.599156Z","shell.execute_reply.started":"2022-02-02T06:12:18.575536Z","shell.execute_reply":"2022-02-02T06:12:18.597883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{"_uuid":"24fe4916af2f186063ad06d34493e6f6963d8c32"}},{"cell_type":"markdown","source":"## Setup & preprocess","metadata":{"_uuid":"6eae683c4269ba9340161b512c78a28c6b205057"}},{"cell_type":"code","source":"%%time\nexec(modules)\nconfig = ExperimentConfigBuilder().build(args=[])\nprint(config)\nstart = time.time()\nset_seed(config.seed)\n\ntrain_df, submit_df = load_qiqc(n_rows=config.n_rows)\ndatasets = build_datasets(train_df, submit_df, config.holdout, config.seed)\ntrain_dataset, test_dataset, submit_dataset = datasets","metadata":{"_uuid":"cdc1274a8e6aa310796cf37b98a6cabdddf23862","execution":{"iopub.status.busy":"2022-02-02T06:12:23.139624Z","iopub.execute_input":"2022-02-02T06:12:23.14026Z","iopub.status.idle":"2022-02-02T06:12:30.303058Z","shell.execute_reply.started":"2022-02-02T06:12:23.139954Z","shell.execute_reply":"2022-02-02T06:12:30.301828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nprint('Tokenize texts...')\npreprocessor = Preprocessor()\nnormalizer = TextNormalizer(config)\ntokenizer = TextTokenizer(config)\ntrain_dataset.tokens, test_dataset.tokens, submit_dataset.tokens = \\\n    preprocessor.tokenize(datasets, normalizer, tokenizer)","metadata":{"_uuid":"24b919658c124fa870172c427af2984ce2d3a4ba","execution":{"iopub.status.busy":"2022-02-02T06:12:36.249815Z","iopub.execute_input":"2022-02-02T06:12:36.253889Z","iopub.status.idle":"2022-02-02T06:13:49.224528Z","shell.execute_reply.started":"2022-02-02T06:12:36.253809Z","shell.execute_reply":"2022-02-02T06:13:49.222093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset.tokens","metadata":{"execution":{"iopub.status.busy":"2022-02-02T06:15:45.084108Z","iopub.execute_input":"2022-02-02T06:15:45.08446Z","iopub.status.idle":"2022-02-02T06:15:45.092767Z","shell.execute_reply.started":"2022-02-02T06:15:45.0844Z","shell.execute_reply":"2022-02-02T06:15:45.091716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nprint('Build vocabulary...')\nvocab = preprocessor.build_vocab(datasets, config)","metadata":{"_uuid":"4181f91134b4f874e676b9e1f5b2c4d30567a54b","execution":{"iopub.status.busy":"2022-02-02T06:15:47.346437Z","iopub.execute_input":"2022-02-02T06:15:47.346826Z","iopub.status.idle":"2022-02-02T06:16:32.679474Z","shell.execute_reply.started":"2022-02-02T06:15:47.346767Z","shell.execute_reply":"2022-02-02T06:16:32.678599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build vocabulary","metadata":{"_uuid":"c63de35321d1c8b00bf0d6115c21b0200d1b2c19"}},{"cell_type":"code","source":"%%time\nprint('Build token ids...')\ntrain_dataset.tids, test_dataset.tids, submit_dataset.tids = \\\n    preprocessor.build_tokenids(datasets, vocab, config)","metadata":{"_uuid":"255daafa472ee422e12d1bff536ad118f97910cd","execution":{"iopub.status.busy":"2022-02-02T06:19:42.791749Z","iopub.execute_input":"2022-02-02T06:19:42.792143Z","iopub.status.idle":"2022-02-02T06:20:19.956352Z","shell.execute_reply.started":"2022-02-02T06:19:42.792074Z","shell.execute_reply":"2022-02-02T06:20:19.954129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit_dataset.tids","metadata":{"execution":{"iopub.status.busy":"2022-02-02T06:21:43.735864Z","iopub.execute_input":"2022-02-02T06:21:43.736342Z","iopub.status.idle":"2022-02-02T06:21:43.747573Z","shell.execute_reply.started":"2022-02-02T06:21:43.736274Z","shell.execute_reply":"2022-02-02T06:21:43.746367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build featurizers","metadata":{"_uuid":"9ebeb2420fba705a8177f21b8b65e9a02e8ee0e5"}},{"cell_type":"code","source":"%%time\nprint('Build sentence extra features...')\nsentence_extra_featurizer = SentenceExtraFeaturizer(config)\ntrain_dataset._X2, test_dataset._X2, submit_dataset._X2 = \\\n    preprocessor.build_sentence_features(\n        datasets, sentence_extra_featurizer)\n[d.build(config.device) for d in datasets]","metadata":{"_uuid":"88b1726354e1487ccdc43cedb028f147ee5c74b8","execution":{"iopub.status.busy":"2022-02-02T06:21:49.209069Z","iopub.execute_input":"2022-02-02T06:21:49.209492Z","iopub.status.idle":"2022-02-02T06:22:35.084686Z","shell.execute_reply.started":"2022-02-02T06:21:49.209423Z","shell.execute_reply":"2022-02-02T06:22:35.083676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(sentence_extra_featurizer)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T06:24:27.773597Z","iopub.execute_input":"2022-02-02T06:24:27.773962Z","iopub.status.idle":"2022-02-02T06:24:27.781522Z","shell.execute_reply.started":"2022-02-02T06:24:27.773871Z","shell.execute_reply":"2022-02-02T06:24:27.780105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nprint('Load pretrained vectors...')\npretrained_vectors = load_pretrained_vectors(\n    config.use_pretrained_vectors, vocab.token2id, test=config.test)","metadata":{"_uuid":"6de736b25070158d393062b2113001176a8566d5","execution":{"iopub.status.busy":"2022-02-02T06:25:51.250355Z","iopub.execute_input":"2022-02-02T06:25:51.250672Z","iopub.status.idle":"2022-02-02T06:27:06.296199Z","shell.execute_reply.started":"2022-02-02T06:25:51.250613Z","shell.execute_reply":"2022-02-02T06:27:06.294823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nprint('Build word embedding matrix...')\nword_embedding_featurizer = WordEmbeddingFeaturizer(config, vocab)\nembedding_matrices = preprocessor.build_embedding_matrices(\n    datasets, word_embedding_featurizer, vocab, pretrained_vectors)","metadata":{"_uuid":"1071d85ca9f3b51de8e156f0a3163e8ee0c64b03","execution":{"iopub.status.busy":"2022-02-02T06:28:11.383519Z","iopub.execute_input":"2022-02-02T06:28:11.383867Z","iopub.status.idle":"2022-02-02T06:34:16.356369Z","shell.execute_reply.started":"2022-02-02T06:28:11.383806Z","shell.execute_reply":"2022-02-02T06:34:16.355173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nprint('Build word extra features...')\nword_extra_featurizer = WordExtraFeaturizer(config, vocab)\nword_extra_features = word_extra_featurizer(vocab)","metadata":{"_uuid":"f366043c9adce2f305bb1927a8282a56d5897c6c","execution":{"iopub.status.busy":"2022-02-02T06:38:10.493623Z","iopub.execute_input":"2022-02-02T06:38:10.493997Z","iopub.status.idle":"2022-02-02T06:38:10.552781Z","shell.execute_reply.started":"2022-02-02T06:38:10.49392Z","shell.execute_reply":"2022-02-02T06:38:10.551689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build models","metadata":{"_uuid":"edbe848c81ab4d19f3cc1cf17d71dc1192eb1dd6"}},{"cell_type":"code","source":"%%time\nprint('Build models...')\nword_features_cv = [\n    preprocessor.build_word_features(\n        word_embedding_featurizer, embedding_matrices, word_extra_features)\n    for i in range(config.cv)]\n\nmodels = [\n    build_model(\n        config, word_features, sentence_extra_featurizer.n_dims\n    ) for word_features in word_features_cv]","metadata":{"_uuid":"ba7f1f85e7fdc25a9157e1316f5cf3364dc6972f","execution":{"iopub.status.busy":"2022-02-02T06:38:16.803194Z","iopub.execute_input":"2022-02-02T06:38:16.803543Z","iopub.status.idle":"2022-02-02T06:38:41.83429Z","shell.execute_reply.started":"2022-02-02T06:38:16.803482Z","shell.execute_reply":"2022-02-02T06:38:41.830816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(word_features_cv)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T06:39:34.761298Z","iopub.execute_input":"2022-02-02T06:39:34.76162Z","iopub.status.idle":"2022-02-02T06:39:34.773862Z","shell.execute_reply.started":"2022-02-02T06:39:34.76156Z","shell.execute_reply":"2022-02-02T06:39:34.772599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(models[0])","metadata":{"_uuid":"fa4f5241149964fbf40ff06ba94354d4a27d2b9e","execution":{"iopub.status.busy":"2022-02-02T06:39:42.348455Z","iopub.execute_input":"2022-02-02T06:39:42.348785Z","iopub.status.idle":"2022-02-02T06:39:42.35703Z","shell.execute_reply.started":"2022-02-02T06:39:42.348723Z","shell.execute_reply":"2022-02-02T06:39:42.35578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{"_uuid":"069f2f284733d64b44c106854e6df0e1637418b1"}},{"cell_type":"code","source":"%%time\nprint('Start training...')\nsplitter = sklearn.model_selection.StratifiedKFold(\n    n_splits=config.cv, shuffle=True, random_state=config.seed)\ntrain_results, valid_results = [], []\nbest_models = []\n\nfor i_cv, (train_indices, valid_indices) in enumerate(\n        splitter.split(train_dataset.df, train_dataset.df.target)):\n    if config.cv_part is not None and i_cv >= config.cv_part:\n        break\n    train_tensor = train_dataset.build_labeled_dataset(train_indices)\n    valid_tensor = train_dataset.build_labeled_dataset(valid_indices)\n    valid_iter = DataLoader(\n        valid_tensor, batch_size=config.batchsize_valid)\n\n    model = models.pop(0)\n    model = model.to_device(config.device)\n    model_snapshots = []\n    optimizer = torch.optim.Adam(model.parameters(), config.lr)\n    train_result = ClassificationResult('train', config.outdir, str(i_cv))\n    valid_result = ClassificationResult('valid', config.outdir, str(i_cv))\n\n    batchsize = config.batchsize\n    for epoch in range(config.epochs):\n        if epoch in config.scale_batchsize:\n            batchsize *= 2\n            print(f'Batchsize: {batchsize}')\n        epoch_start = time.time()\n        sampler = None\n        train_iter = DataLoader(\n            train_tensor, sampler=sampler, drop_last=True,\n            batch_size=batchsize, shuffle=sampler is None)\n        _summary = []\n\n        # Training loop\n        for i, batch in enumerate(\n                tqdm(train_iter, desc='train', leave=False)):\n            model.train()\n            optimizer.zero_grad()\n            loss, output = model.calc_loss(*batch)\n            loss.backward()\n            optimizer.step()\n            train_result.add_record(**output)\n        train_result.calc_score(epoch)\n        _summary.append(train_result.summary.iloc[-1])\n\n        # Validation loop\n        if epoch >= config.validate_from:\n            for i, batch in enumerate(\n                    tqdm(valid_iter, desc='valid', leave=False)):\n                model.eval()\n                loss, output = model.calc_loss(*batch)\n                valid_result.add_record(**output)\n            valid_result.calc_score(epoch)\n            _summary.append(valid_result.summary.iloc[-1])\n\n            _model = deepcopy(model)\n            _model.threshold = valid_result.summary.threshold[epoch]\n            model_snapshots.append(_model)\n\n        summary = pd.DataFrame(_summary).set_index('name')\n        epoch_time = time.time() - epoch_start\n        pbar = '#' * (i_cv + 1) + '-' * (config.cv - 1 - i_cv)\n        tqdm.write(f'\\n{pbar} cv: {i_cv} / {config.cv}, epoch {epoch}, '\n                   f'time: {epoch_time}')\n        tqdm.write(str(summary))\n\n    train_results.append(train_result)\n    valid_results.append(valid_result)\n    best_indices = valid_result.summary.fbeta.argsort()[::-1]\n    best_models.extend([model_snapshots[i] for i in\n                        best_indices[:config.ensembler_n_snapshots]])","metadata":{"_uuid":"47e16610dab4b6adfa01e39cf5b179c4bdedea78","execution":{"iopub.status.busy":"2022-02-02T06:39:46.939735Z","iopub.execute_input":"2022-02-02T06:39:46.940092Z","iopub.status.idle":"2022-02-02T07:13:59.34588Z","shell.execute_reply.started":"2022-02-02T06:39:46.940028Z","shell.execute_reply":"2022-02-02T07:13:59.342971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build ensembler and make CV scores","metadata":{"_uuid":"fac52039ffe472b4259a4e38d14f73d43553e172"}},{"cell_type":"code","source":"# Build ensembler\ntrain_X, train_X2, train_t = \\\n    train_dataset.X, train_dataset.X2, train_dataset.t\nensembler = Ensembler(config, best_models, valid_results)\nensembler.fit(train_X, train_X2, train_t)\nscores = dict(\n    valid_fbeta=np.array([r.best_fbeta for r in valid_results]).mean(),\n    valid_epoch=np.array([r.best_epoch for r in valid_results]).mean(),\n    threshold_cv=ensembler.threshold_cv,\n    threshold=ensembler.threshold,\n    elapsed_time=time.time() - start,\n)","metadata":{"_uuid":"2a701bb825b115be4207cddf9317f166d6283265","execution":{"iopub.status.busy":"2022-02-02T07:17:37.356098Z","iopub.execute_input":"2022-02-02T07:17:37.35704Z","iopub.status.idle":"2022-02-02T07:17:37.381793Z","shell.execute_reply.started":"2022-02-02T07:17:37.356937Z","shell.execute_reply":"2022-02-02T07:17:37.380584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Holdout evaluation / Logging for error analysis","metadata":{"_uuid":"f941f2b851391622cb51f4efeb9b5a88dcbe9435"}},{"cell_type":"code","source":"if config.holdout:\n    test_X, test_X2, test_t = \\\n        test_dataset.X, test_dataset.X2, test_dataset._t\n    y, t = ensembler.predict_proba(test_X, test_X2), test_t\n    y_pred = y > ensembler.threshold\n    y_pred_cv = y > ensembler.threshold_cv\n    result = classification_metrics(y_pred, t)\n    result_cv = classification_metrics(y_pred_cv, t)\n    result_theoretical = classification_metrics(y, t)\n    scores.update(dict(\n        test_fbeta=result['fbeta'],\n        test_fbeta_cv=result_cv['fbeta'],\n        test_fbeta_theoretical=result_theoretical['fbeta'],\n        test_threshold_theoretical=result_theoretical['threshold'],\n    ))","metadata":{"_uuid":"484bd320bc5899dccef95a0e493db5500f9246aa","execution":{"iopub.status.busy":"2022-02-02T07:17:39.478635Z","iopub.execute_input":"2022-02-02T07:17:39.479186Z","iopub.status.idle":"2022-02-02T07:17:39.487164Z","shell.execute_reply.started":"2022-02-02T07:17:39.478939Z","shell.execute_reply":"2022-02-02T07:17:39.485906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(scores['threshold_cv'])","metadata":{"execution":{"iopub.status.busy":"2022-02-02T07:17:43.054261Z","iopub.execute_input":"2022-02-02T07:17:43.054599Z","iopub.status.idle":"2022-02-02T07:17:43.06077Z","shell.execute_reply.started":"2022-02-02T07:17:43.054538Z","shell.execute_reply":"2022-02-02T07:17:43.05942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predict submit datasets\nsubmit_y = ensembler.predict(submit_dataset.X, submit_dataset.X2)\nsubmit_df['prediction'] = submit_y\nsubmit_df = submit_df[['qid', 'prediction']]\nsubmit_df.to_csv(config.outdir / 'submission.csv', index=False)","metadata":{"_uuid":"51a956ee03328b1c2ab9dd589e9e0e4bad426d54","execution":{"iopub.status.busy":"2022-02-02T07:18:31.637471Z","iopub.execute_input":"2022-02-02T07:18:31.637812Z","iopub.status.idle":"2022-02-02T07:19:23.782923Z","shell.execute_reply.started":"2022-02-02T07:18:31.637756Z","shell.execute_reply":"2022-02-02T07:19:23.782041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, (train, valid) in enumerate(zip(train_results, valid_results)):\n    df = pd.concat([train.summary, valid.summary])\n    df = df.set_index(['name', df.index], drop=True)\n#     plt.plot(scores['threshold_cv'])\n    display(df)","metadata":{"_uuid":"d9843e601e8201f29a9394f794b83e9b7a9f0d5b","execution":{"iopub.status.busy":"2022-02-02T07:20:50.550841Z","iopub.execute_input":"2022-02-02T07:20:50.55122Z","iopub.status.idle":"2022-02-02T07:20:50.716046Z","shell.execute_reply.started":"2022-02-02T07:20:50.551159Z","shell.execute_reply":"2022-02-02T07:20:50.714771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fbeta = train.summary['fbeta']\nloss = train.summary['loss']\nprec = train.summary['prec']\nrec =train.summary['rec']","metadata":{"execution":{"iopub.status.busy":"2022-02-02T07:20:57.407589Z","iopub.execute_input":"2022-02-02T07:20:57.408019Z","iopub.status.idle":"2022-02-02T07:20:57.41353Z","shell.execute_reply.started":"2022-02-02T07:20:57.407919Z","shell.execute_reply":"2022-02-02T07:20:57.412225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = [0,1,2,3,4]\nfig, ax = plt.subplots(figsize=(5, 2.7))\n# ax.plot(x, x, label='linear')  # Plot some data on the axes.\nax.plot(x, rec, label='rec')  # Plot more data on the axes...\nax.plot(x, prec, label='prec')  # ... and some more.\nax.set_xlabel('epoch')  # Add an x-label to the axes.\nax.set_ylabel('value')  # Add a y-label to the axes.\nax.set_title(\"prec - rec\")  # Add a title to the axes.\nax.legend();  # Add a legend.","metadata":{"execution":{"iopub.status.busy":"2022-02-02T07:20:59.506509Z","iopub.execute_input":"2022-02-02T07:20:59.506844Z","iopub.status.idle":"2022-02-02T07:20:59.855589Z","shell.execute_reply.started":"2022-02-02T07:20:59.506783Z","shell.execute_reply":"2022-02-02T07:20:59.854369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = [0,1,2,3,4]\nfig, ax = plt.subplots(figsize=(5, 2.7))\n# ax.plot(x, x, label='linear')  # Plot some data on the axes.\nax.plot(x, loss, label='loss')  # Plot more data on the axes...\nax.set_xlabel('epoch')  # Add an x-label to the axes.\nax.set_ylabel('value')  # Add a y-label to the axes.\nax.set_title(\"loss\")  # Add a title to the axes.\nax.legend();  # Add a legend.","metadata":{"execution":{"iopub.status.busy":"2022-02-02T07:21:05.97239Z","iopub.execute_input":"2022-02-02T07:21:05.972735Z","iopub.status.idle":"2022-02-02T07:21:06.269158Z","shell.execute_reply.started":"2022-02-02T07:21:05.972661Z","shell.execute_reply":"2022-02-02T07:21:06.267953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame([scores])","metadata":{"_uuid":"9cffa7ca15338c9add8cf2ba5a2aa0f5891641ba","execution":{"iopub.status.busy":"2022-01-30T11:39:44.293722Z","iopub.status.idle":"2022-01-30T11:39:44.294673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}