{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n# Input data files are available in the read-only \"../input/\" directory\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-15T09:46:39.974169Z","iopub.execute_input":"2022-08-15T09:46:39.974830Z","iopub.status.idle":"2022-08-15T09:46:40.002496Z","shell.execute_reply.started":"2022-08-15T09:46:39.974739Z","shell.execute_reply":"2022-08-15T09:46:40.001302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# pytorch-lifestream (ptls)\n\nThis is library for sequential data: https://github.com/dllllb/pytorch-lifestream\n\nData preprocessed with https://www.kaggle.com/code/ivkireev/amex-ptls-data-preprocessing\n\nThis notebook realise the same pipeline as https://www.kaggle.com/code/ivkireev/amex-ptls-baseline-supervised-neural-network,\nbut seqence encoder replaced from GRU to Transformer. Final quality looks the same but transformer allows to use more complicated pretrain methods (not showed here).\n","metadata":{}},{"cell_type":"code","source":"!pip uninstall tensorflow-transform tfx-bsl -y\n!pip install pyspark\n!pip install git+https://github.com/dllllb/pytorch-lifestream.git@main\n!pip install pyarrow==5.0.0  # comment it for cpu kernel","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-15T09:46:40.007515Z","iopub.execute_input":"2022-08-15T09:46:40.007882Z","iopub.status.idle":"2022-08-15T09:49:16.585459Z","shell.execute_reply.started":"2022-08-15T09:46:40.007855Z","shell.execute_reply":"2022-08-15T09:49:16.583707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Imports","metadata":{}},{"cell_type":"code","source":"import pickle\nfrom copy import deepcopy\nfrom functools import partial\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport pytorch_lightning as pl\n\nimport torch\nimport torchmetrics\n\nfrom sklearn.model_selection import train_test_split\n\nimport ptls\nfrom ptls.data_load.datasets import ParquetDataset, parquet_file_scan\nfrom ptls.data_load.iterable_processing import IterableShuffle\nfrom ptls.data_load.padded_batch import PaddedBatch\nfrom ptls.data_load.utils import collate_feature_dict\nfrom ptls.frames import PtlsDataModule\nfrom ptls.frames.inference_module import InferenceModule\nfrom ptls.frames.supervised import SeqToTargetIterableDataset, SequenceToTarget\nfrom ptls.nn import TrxEncoder, RnnEncoder, TransformerEncoder, PBDropout, PBLinear, PBL2Norm, PBLayerNorm, PBReLU\n","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:49:44.695344Z","iopub.execute_input":"2022-08-15T09:49:44.695772Z","iopub.status.idle":"2022-08-15T09:49:51.333380Z","shell.execute_reply.started":"2022-08-15T09:49:44.695736Z","shell.execute_reply":"2022-08-15T09:49:51.331457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read the data\n\nData was prepared in `ptls` format. This is parquet files where each row represent one data sample - one client. Features are the sequential fields with arrays of length `sequence_length`\n\nDemo of data usage is in https://www.kaggle.com/code/ivkireev/ptls-data-usage","metadata":{}},{"cell_type":"code","source":"# Read preprocessor which keep column names and dictionary sizes for categorical fields\nwith open('../input/amex-ptls-data/preprocessor.pickle', 'rb') as f:\n    preprocessor = pickle.load(f)","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:51:10.722866Z","iopub.execute_input":"2022-08-15T09:51:10.723777Z","iopub.status.idle":"2022-08-15T09:51:10.804014Z","shell.execute_reply.started":"2022-08-15T09:51:10.723736Z","shell.execute_reply":"2022-08-15T09:51:10.802713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessor.get_category_dictionary_sizes()","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:51:16.231177Z","iopub.execute_input":"2022-08-15T09:51:16.231602Z","iopub.status.idle":"2022-08-15T09:51:16.244119Z","shell.execute_reply.started":"2022-08-15T09:51:16.231557Z","shell.execute_reply":"2022-08-15T09:51:16.242417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_train_files = parquet_file_scan('../input/amex-ptls-data/data_train.parquet')\ntest_files = parquet_file_scan('../input/amex-ptls-data/data_test.parquet')\n\nlen(all_train_files), len(test_files)","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:51:16.270151Z","iopub.execute_input":"2022-08-15T09:51:16.271388Z","iopub.status.idle":"2022-08-15T09:51:16.378934Z","shell.execute_reply.started":"2022-08-15T09:51:16.271316Z","shell.execute_reply":"2022-08-15T09:51:16.377626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files, valid_files = train_test_split(all_train_files, test_size=0.2, random_state=242)\nlen(train_files), len(valid_files)","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:51:16.382140Z","iopub.execute_input":"2022-08-15T09:51:16.383175Z","iopub.status.idle":"2022-08-15T09:51:16.396198Z","shell.execute_reply.started":"2022-08-15T09:51:16.383114Z","shell.execute_reply":"2022-08-15T09:51:16.394527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class NumConcat:\n    \"\"\"Concatenete numerical columns which a collected to single field\n    Work with one tensor is faster than iterating over 150+ numerical columns in amex dataset.\n    \"\"\"\n    def __call__(self, data):\n        self.data = data\n        return iter(self)\n        \n    def __iter__(self):\n        for rec in self.data:\n            rec['num_cols'] = np.stack(rec['num_cols'].tolist(), axis=1)\n            rec['num_cols'] = torch.from_numpy(rec['num_cols'])\n            yield rec\n","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:51:16.399321Z","iopub.execute_input":"2022-08-15T09:51:16.399879Z","iopub.status.idle":"2022-08-15T09:51:16.409789Z","shell.execute_reply.started":"2022-08-15T09:51:16.399808Z","shell.execute_reply":"2022-08-15T09:51:16.408413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We will use ParquetDataset in iterable mode\nall_train_dataset = ParquetDataset(all_train_files, shuffle_files=True, i_filters=[\n    NumConcat(),\n    IterableShuffle(buffer_size=1000),  # this shuffle rows with buffer \n])\ntrain_dataset = ParquetDataset(train_files, shuffle_files=True, i_filters=[\n    NumConcat(),\n    IterableShuffle(buffer_size=1000),\n])\nvalid_dataset = ParquetDataset(valid_files, i_filters=[NumConcat()])\ntest_dataset = ParquetDataset(test_files, i_filters=[NumConcat()])","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:51:16.413638Z","iopub.execute_input":"2022-08-15T09:51:16.415202Z","iopub.status.idle":"2022-08-15T09:51:16.424787Z","shell.execute_reply.started":"2022-08-15T09:51:16.415120Z","shell.execute_reply":"2022-08-15T09:51:16.423033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# augmentations for train data\nall_train_dataset = ptls.data_load.datasets.AugmentationDataset(\n    all_train_dataset,\n    f_augmentations=[\n        ptls.data_load.augmentations.DropoutTrx(0.15),\n    ],\n)\ntrain_dataset = ptls.data_load.datasets.AugmentationDataset(\n    train_dataset,\n    f_augmentations=[\n        ptls.data_load.augmentations.DropoutTrx(0.15),\n    ],\n)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Encoder network","metadata":{}},{"cell_type":"code","source":"class NaImputer(torch.nn.Module):\n    \"\"\"Replace NaN with 0 value\n    \"\"\"\n    def forward(self, x: PaddedBatch):\n        new_x = {k: self.impute(k, v) for k, v in x.payload.items()}\n        return PaddedBatch(new_x, x.seq_lens)\n    \n    def impute(self, k, v):\n        if not PaddedBatch.is_seq_feature(k, v):\n            return v\n        return v.masked_fill(torch.isnan(v), 0.0)","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:51:16.426970Z","iopub.execute_input":"2022-08-15T09:51:16.427448Z","iopub.status.idle":"2022-08-15T09:51:16.438652Z","shell.execute_reply.started":"2022-08-15T09:51:16.427409Z","shell.execute_reply":"2022-08-15T09:51:16.437039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def sign_log(x):\n    return torch.log1p(torch.abs(x)) * torch.sign(x)\n    \nclass TrxEncoderExt(torch.nn.Module):\n    \"\"\"Supports numerical features stacked with `NumConcat`.\n    Implements the same behavior as ptls.nn.TrxEncoder but faster.\n    \"\"\"\n    def __init__(self, trx_encoder, numeric_count=177):\n        super().__init__()\n        \n        self.trx_encoder = trx_encoder\n        self.numeric_count = numeric_count\n        self.bn = torch.nn.BatchNorm1d(numeric_count)\n        \n    def forward(self, x):\n        x_categorical = self.trx_encoder(x)\n        \n        num_cols = x.payload['num_cols']\n        B, T, H = num_cols.size()\n        \n        num_log = (self.bn(sign_log(num_cols).view(B * T, H)).view(B, T, H) *\n            x.seq_len_mask.float().unsqueeze(2)).clamp(-3, 3)\n        return PaddedBatch(\n            torch.cat([\n                x_categorical.payload, \n                num_log,\n            ], dim=2),\n            x.seq_lens,\n        )\n    \n    @property\n    def output_size(self):\n        return self.trx_encoder.output_size + self.numeric_count\n","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:51:16.440659Z","iopub.execute_input":"2022-08-15T09:51:16.442340Z","iopub.status.idle":"2022-08-15T09:51:16.454888Z","shell.execute_reply.started":"2022-08-15T09:51:16.442250Z","shell.execute_reply":"2022-08-15T09:51:16.453381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Metrics\n\nTaken from https://www.kaggle.com/code/inversion/amex-competition-metric-python\nand covered to `torchmetrics.Metric` for NN usage","metadata":{}},{"cell_type":"code","source":"def top_four_percent_captured(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n    df = (pd.concat([y_true, y_pred], axis='columns')\n          .sort_values('prediction', ascending=False))\n    df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n    four_pct_cutoff = int(0.04 * df['weight'].sum())\n    df['weight_cumsum'] = df['weight'].cumsum()\n    df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n    return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n\ndef weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n    df = (pd.concat([y_true, y_pred], axis='columns')\n          .sort_values('prediction', ascending=False))\n    df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n    df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n    total_pos = (df['target'] * df['weight']).sum()\n    df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n    df['lorentz'] = df['cum_pos_found'] / total_pos\n    df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n    return df['gini'].sum()\n\ndef normalized_weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n    y_true_pred = y_true.rename(columns={'target': 'prediction'})\n    return weighted_gini(y_true, y_pred) / weighted_gini(y_true, y_true_pred)\n\ndef amex_metric(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n    g = normalized_weighted_gini(y_true, y_pred)\n    d = top_four_percent_captured(y_true, y_pred)\n\n    return 0.5 * (g + d), g, d","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-15T09:51:16.458694Z","iopub.execute_input":"2022-08-15T09:51:16.459934Z","iopub.status.idle":"2022-08-15T09:51:16.480447Z","shell.execute_reply.started":"2022-08-15T09:51:16.459874Z","shell.execute_reply":"2022-08-15T09:51:16.478956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TopFourPercentCaptured(torchmetrics.Metric):\n    full_state_update = True\n    compute_on_cpu = True\n    \n    def __init__(self):\n        super().__init__()\n    \n        self.add_state('y_pred', []),\n        self.add_state('y_true', []),\n\n    def update(self, y_pred, y_true):\n        self.y_pred.append(y_pred.detach().cpu())\n        self.y_true.append(y_true.detach().cpu())\n        \n    def compute(self):\n        y_pred = pd.DataFrame({'prediction': torch.cat(self.y_pred).numpy()})\n        y_true = pd.DataFrame({'target': torch.cat(self.y_true).numpy()})\n        return top_four_percent_captured(y_true, y_pred)\n    \n    \nclass NormalizedWeightedGini(torchmetrics.Metric):\n    full_state_update = True\n    compute_on_cpu = True\n    \n    def __init__(self):\n        super().__init__()\n    \n        self.add_state('y_pred', []),\n        self.add_state('y_true', []),\n\n    def update(self, y_pred, y_true):\n        self.y_pred.append(y_pred.detach().cpu())\n        self.y_true.append(y_true.detach().cpu())\n        \n    def compute(self):\n        y_pred = pd.DataFrame({'prediction': torch.cat(self.y_pred).numpy()})\n        y_true = pd.DataFrame({'target': torch.cat(self.y_true).numpy()})\n        return normalized_weighted_gini(y_true, y_pred)\n    \n    \nclass AmexMetric(torchmetrics.Metric):\n    full_state_update = True\n    compute_on_cpu = True\n    \n    def __init__(self):\n        super().__init__()\n    \n        self.add_state('y_pred', []),\n        self.add_state('y_true', []),\n\n    def update(self, y_pred, y_true):\n        self.y_pred.append(y_pred.detach().cpu())\n        self.y_true.append(y_true.detach().cpu())\n        \n    def compute(self):\n        y_pred = pd.DataFrame({'prediction': torch.cat(self.y_pred).numpy()})\n        y_true = pd.DataFrame({'target': torch.cat(self.y_true).numpy()})\n        return amex_metric(y_true, y_pred)[0]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-15T09:51:16.484183Z","iopub.execute_input":"2022-08-15T09:51:16.485192Z","iopub.status.idle":"2022-08-15T09:51:16.504809Z","shell.execute_reply.started":"2022-08-15T09:51:16.485123Z","shell.execute_reply":"2022-08-15T09:51:16.503109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loss","metadata":{}},{"cell_type":"code","source":"class PairwiseMarginRankingLoss(torch.nn.Module):\n    \"\"\"Ranking loss for amex task\n    \"\"\"\n    def __init__(self, margin):\n        super().__init__()\n\n        self.margin = margin\n\n    def forward(self, pred, true):\n        pred_0 = pred[true == 0]\n        pred_1 = pred[true == 1]\n        loss = pred_0.view(1, -1) - pred_1.view(-1, 1) + self.margin\n        loss = torch.nn.functional.relu(loss)\n        return loss.mean()","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:51:16.507429Z","iopub.execute_input":"2022-08-15T09:51:16.508005Z","iopub.status.idle":"2022-08-15T09:51:16.522445Z","shell.execute_reply.started":"2022-08-15T09:51:16.507960Z","shell.execute_reply":"2022-08-15T09:51:16.520415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lightning Module\n\nhttps://pytorch-lightning.readthedocs.io/en/1.6.4/common/lightning_module.html","metadata":{}},{"cell_type":"code","source":"class SequenceToTargetExt(SequenceToTarget):\n    \"\"\"LightningModule for supervised task or for inference\n    \"\"\"\n    def configure_optimizers(self):\n        parameters = [\n            {\n                'params': [p for n, p in self.named_parameters() \n                           if n.startswith('seq_encoder.1.') or n.startswith('seq_encoder.3.')],\n                 'weight_decay': 1e-4,\n            },\n            {\n                'params': [p for n, p in self.named_parameters()\n                           if not (n.startswith('seq_encoder.1.') or n.startswith('seq_encoder.3.'))]\n            },\n        ]\n        optimizer = self.optimizer_partial(parameters)\n        scheduler = self.lr_scheduler_partial(optimizer)\n        scheduler = {'scheduler': scheduler, 'interval': 'step'}\n        return [optimizer], [scheduler]\n","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:51:16.526207Z","iopub.execute_input":"2022-08-15T09:51:16.526663Z","iopub.status.idle":"2022-08-15T09:51:16.539980Z","shell.execute_reply.started":"2022-08-15T09:51:16.526622Z","shell.execute_reply":"2022-08-15T09:51:16.538521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train\n\nPrepare `datamodule`, model as `pl.LightningModile` and `pl.Trainer`\n\nRun train with validation","metadata":{}},{"cell_type":"code","source":"def get_model():\n    trx_encoder = TrxEncoderExt(TrxEncoder(\n        embeddings={col: {'in': d, 'out': 4} for col, d in preprocessor.get_category_dictionary_sizes().items()},\n        numeric_values={},  # processed by TrxEncoderExt\n    #     numeric_values={col: 'identity' for col in num_fields},\n    ))\n\n    seq_encoder = torch.nn.Sequential(\n        NaImputer(),\n        trx_encoder,\n        PBDropout(0.1),\n        PBLinear(trx_encoder.output_size, 160),\n        PBLayerNorm(160),\n        TransformerEncoder(\n            input_size=160, \n            n_heads=32,\n            dim_hidden=1024,\n            dropout=0.3,\n            n_layers=2,\n            max_seq_len=15,\n            is_reduce_sequence=True,\n        ),\n    )\n    \n    model = SequenceToTargetExt(\n        seq_encoder=seq_encoder,\n        head=torch.nn.Sequential(\n            torch.nn.Linear(seq_encoder[-1].embedding_size, 1),\n            torch.nn.Sigmoid(),\n            torch.nn.Flatten(start_dim=0),\n        ),\n        loss=PairwiseMarginRankingLoss(0.1),\n        metric_list={\n    #         'auc': torchmetrics.AUROC(),\n    #         'top4': TopFourPercentCaptured(),\n    #         'nwg': NormalizedWeightedGini(),\n            'amex': AmexMetric(),\n        },\n        optimizer_partial=partial(\n            torch.optim.Adam,\n            lr=0.001,\n            weight_decay=0,\n        ),\n        lr_scheduler_partial=partial(\n            torch.optim.lr_scheduler.OneCycleLR,\n            max_lr=0.0006,\n            total_steps=10000,\n            pct_start=0.25,\n            anneal_strategy='cos',\n            cycle_momentum=False,\n            div_factor=16,\n            final_div_factor=100000,\n            three_phase=True,\n        ),\n    )\n    for n, p in model.named_parameters():\n        print(n, p.shape)\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:51:16.543317Z","iopub.execute_input":"2022-08-15T09:51:16.543754Z","iopub.status.idle":"2022-08-15T09:51:16.558115Z","shell.execute_reply.started":"2022-08-15T09:51:16.543713Z","shell.execute_reply":"2022-08-15T09:51:16.556532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dm = PtlsDataModule(\n    train_data=SeqToTargetIterableDataset(train_dataset, target_col_name='target'),\n    valid_data=SeqToTargetIterableDataset(valid_dataset, target_col_name='target'),\n    train_batch_size=512,\n    valid_batch_size=1024,\n    train_num_workers=4,\n    valid_num_workers=4,\n)\n\nmodel = get_model()\n\ntrainer = pl.Trainer(\n    gpus=1,\n    max_steps=10000,\n    enable_checkpointing=False,\n    enable_progress_bar=True,\n    callbacks=[\n        pl.callbacks.LearningRateMonitor(),\n    ],\n)","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:51:16.617686Z","iopub.execute_input":"2022-08-15T09:51:16.618017Z","iopub.status.idle":"2022-08-15T09:51:16.866392Z","shell.execute_reply.started":"2022-08-15T09:51:16.617991Z","shell.execute_reply":"2022-08-15T09:51:16.864920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('\\n'.join([\n    'Running...',\n    f'version = {trainer.logger.version}',\n]))\n\ntrainer.fit(model, dm)\n\nprint(trainer.logged_metrics)","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:51:16.869058Z","iopub.execute_input":"2022-08-15T09:51:16.870341Z","iopub.status.idle":"2022-08-15T09:56:17.070091Z","shell.execute_reply.started":"2022-08-15T09:51:16.870266Z","shell.execute_reply":"2022-08-15T09:56:17.067955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Results","metadata":{}},{"cell_type":"code","source":"import ptls.tb_interface as tb","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:56:17.077401Z","iopub.execute_input":"2022-08-15T09:56:17.084348Z","iopub.status.idle":"2022-08-15T09:56:17.221397Z","shell.execute_reply.started":"2022-08-15T09:56:17.084287Z","shell.execute_reply":"2022-08-15T09:56:17.219990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_tb = tb.get_scalars('lightning_logs/')","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:56:17.229630Z","iopub.execute_input":"2022-08-15T09:56:17.232731Z","iopub.status.idle":"2022-08-15T09:56:17.284024Z","shell.execute_reply.started":"2022-08-15T09:56:17.232685Z","shell.execute_reply":"2022-08-15T09:56:17.282360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_tb['tag'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:56:17.289612Z","iopub.execute_input":"2022-08-15T09:56:17.290127Z","iopub.status.idle":"2022-08-15T09:56:17.315661Z","shell.execute_reply.started":"2022-08-15T09:56:17.290079Z","shell.execute_reply":"2022-08-15T09:56:17.311660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for tag in ['val_amex', 'train_amex']:\n    df = df_tb[lambda x: x['tag'] == tag].set_index(['step', 'version'])['value'].unstack()\n    df.plot(figsize=(8, 4), title=tag)\n    plt.show()\n    print(df.max())","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:56:17.321270Z","iopub.execute_input":"2022-08-15T09:56:17.324175Z","iopub.status.idle":"2022-08-15T09:56:18.146207Z","shell.execute_reply.started":"2022-08-15T09:56:17.324105Z","shell.execute_reply":"2022-08-15T09:56:18.143023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Final Train\n\nTrain on full frain data","metadata":{}},{"cell_type":"code","source":"dm = PtlsDataModule(\n    train_data=SeqToTargetIterableDataset(all_train_dataset, target_col_name='target'),\n    train_batch_size=512,\n    train_num_workers=4,\n)\n\nmodel = get_model()\n\ntrainer = pl.Trainer(\n    gpus=1,\n    max_steps=10000,\n    enable_checkpointing=False,\n    enable_progress_bar=False,\n    callbacks=[\n        pl.callbacks.LearningRateMonitor(),\n    ],\n)\n\nprint('\\n'.join([\n    'Running train on full data...',\n    f'version = {trainer.logger.version}',\n]))\n\n\ntrainer.fit(model, dm)\n\nprint(trainer.logged_metrics)","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:56:18.147781Z","iopub.status.idle":"2022-08-15T09:56:18.149295Z","shell.execute_reply.started":"2022-08-15T09:56:18.148939Z","shell.execute_reply":"2022-08-15T09:56:18.148970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference on test data","metadata":{}},{"cell_type":"code","source":"test_dataloader = torch.utils.data.DataLoader(\n    test_dataset,\n    collate_fn=collate_feature_dict,\n    num_workers=4,\n    batch_size=1024,\n    \n)","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:56:18.151486Z","iopub.status.idle":"2022-08-15T09:56:18.152611Z","shell.execute_reply.started":"2022-08-15T09:56:18.152260Z","shell.execute_reply":"2022-08-15T09:56:18.152292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_predict = trainer.predict(InferenceModule(model, model_out_name='prediction'), test_dataloader)","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:56:18.154761Z","iopub.status.idle":"2022-08-15T09:56:18.155881Z","shell.execute_reply.started":"2022-08-15T09:56:18.155508Z","shell.execute_reply":"2022-08-15T09:56:18.155541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_predict[0].head()","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:56:18.157843Z","iopub.status.idle":"2022-08-15T09:56:18.158948Z","shell.execute_reply.started":"2022-08-15T09:56:18.158543Z","shell.execute_reply":"2022-08-15T09:56:18.158574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_predict = pd.concat(df_predict, axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:56:18.161024Z","iopub.status.idle":"2022-08-15T09:56:18.162203Z","shell.execute_reply.started":"2022-08-15T09:56:18.161799Z","shell.execute_reply":"2022-08-15T09:56:18.161833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_predict.to_csv('ptls_submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:56:18.164219Z","iopub.status.idle":"2022-08-15T09:56:18.165338Z","shell.execute_reply.started":"2022-08-15T09:56:18.164954Z","shell.execute_reply":"2022-08-15T09:56:18.164989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head ptls_submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:56:18.167389Z","iopub.status.idle":"2022-08-15T09:56:18.168457Z","shell.execute_reply.started":"2022-08-15T09:56:18.168093Z","shell.execute_reply":"2022-08-15T09:56:18.168124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head ../input/amex-default-prediction/sample_submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-08-15T09:56:18.170478Z","iopub.status.idle":"2022-08-15T09:56:18.171669Z","shell.execute_reply.started":"2022-08-15T09:56:18.171330Z","shell.execute_reply":"2022-08-15T09:56:18.171362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# More info about ptls\n\nLinks:\n- Repo: https://github.com/dllllb/pytorch-lifestream\n- Demos: https://github.com/dllllb/pytorch-lifestream/tree/main/demo\n- Docs: https://dllllb.github.io/pytorch-lifestream/","metadata":{}},{"cell_type":"code","source":"from IPython.display import HTML\n\nHTML('''\n<!-- Place this tag in your head or just before your close body tag. -->\n<script async defer src=\"https://buttons.github.io/buttons.js\"></script>\n\n<p>Star <code>pytorch-lifestream</code> on github if you like this notebook</p>\n<!-- Place this tag where you want the button to render. -->\n<a class=\"github-button\" href=\"https://github.com/dllllb/pytorch-lifestream\" data-icon=\"octicon-star\" data-size=\"large\" data-show-count=\"true\" aria-label=\"Star dllllb/pytorch-lifestream on GitHub\">Star</a>\n''')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-15T09:56:18.173758Z","iopub.status.idle":"2022-08-15T09:56:18.174908Z","shell.execute_reply.started":"2022-08-15T09:56:18.174505Z","shell.execute_reply":"2022-08-15T09:56:18.174542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}