{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install git+https://github.com/pabloppp/pytorch-tools -U","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom pathlib import Path\nimport torch.nn.functional as F\nimport pytorch_lightning as pl\nimport torch\nfrom torchtools.optim import RangerLars\nfrom torchtools.lr_scheduler import DelayerScheduler\nimport os","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class VolcanicDataset(torch.utils.data.IterableDataset):\n    \n    def __init__(self, base_path, df=None, from_directory = False, is_train=True, target_mean=None, target_std=None, downsample=None):\n        \"\"\"\n        :param Path base_path: Path where read the segments.\n        :param DataFrame df: Dataframe containing the segments.\n        :param bool from_directory: Indicates if you will read the segments directly from the directory and not using df\n        :param float target_mean: Mean of the target used for standarization\n        :param float target_std: Standard deviation of the target used for standarization\n        :param bool downsample: Downsample the dataframe with a specific period determined by the value specified.\n        \"\"\"\n        self.df = df\n        self.base_path = base_path\n        self.delta = 1e-16\n        self.is_train = is_train\n        self.from_directory = from_directory\n        self.target_mean, self.target_std = target_mean, target_std\n        self.downsample = downsample\n \n        \n    def segments_iterable(self):\n        \n        if self.df is None and self.from_directory:\n            segments = os.listdir(self.base_path)\n        else:\n            segments = self.df.iterrows()\n        \n        for idx, data in enumerate(segments):\n            \n            # Obtaining the segment name\n            if self.is_train:\n                segment = data[1].segment_id\n                y = data[1].time_to_eruption\n            elif self.from_directory:\n                segment = data.split('.')[0]\n            else:\n                segment = data\n            \n            # Segment read\n            segment_timeseries_path = self.base_path/f'{segment}.csv'\n            segment_timeseries = pd.read_csv(segment_timeseries_path)\n            segment_timeseries = segment_timeseries.fillna(0) # Fill nan values with zeros meaning no signal\n            \n            if self.downsample is not None: # Downsampling step\n                segment_timeseries = segment_timeseries.reset_index()\n                segment_timeseries['index'] = segment_timeseries['index']//self.downsample\n                segment_timeseries = segment_timeseries.groupby('index').mean()\n            \n            # Each segment is standarized with its mean and standar deviation independently (Not the best approach)\n            segment_timeseries = (segment_timeseries - segment_timeseries.mean())/(segment_timeseries.std() + self.delta) \n            X = torch.tensor(segment_timeseries.values)\n            \n            \n            if self.is_train and self.df is not None:\n                y = (y-self.target_mean)/self.target_std # The target is standarized too\n                yield X, y\n            else:\n                yield X\n        \n    def __iter__(self):\n        return self.segments_iterable()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class VolcanicDataModule(pl.LightningDataModule):\n\n    def __init__(self, batch_size=32, random_state=123, downsample=None):\n        super().__init__()\n        self.batch_size = batch_size\n        self.random_state = random_state\n        self.downsample = downsample\n\n    def prepare_data(self):\n        self.path = Path('/kaggle/input/predict-volcanic-eruptions-ingv-oe')\n        train_path = self.path/'train.csv'\n        self.train = pd.read_csv(train_path)\n\n    def setup(self, stage):\n        \n        if stage == 'fit':\n            \n            # split dataset\n            train_size = int(len(self.train)*0.8)\n            self.train = self.train.sample(frac=1, random_state=self.random_state).reset_index(drop=True)\n            train_df_target = self.train.loc[:train_size,'time_to_eruption']\n            \n            self.train_dataset = VolcanicDataset(self.path/'train', self.train.loc[:train_size,:], target_mean=train_df_target.mean(), target_std=train_df_target.std(), downsample=self.downsample)\n            self.valid_dataset = VolcanicDataset(self.path/'train', self.train.loc[train_size:,:], target_mean=train_df_target.mean(), target_std=train_df_target.std(), downsample=self.downsample)\n\n    # return the dataloader for each split\n    def train_dataloader(self):\n        train_dataset = torch.utils.data.DataLoader(self.train_dataset, batch_size=self.batch_size,num_workers=4)\n        return train_dataset\n\n    def val_dataloader(self):\n        valid_dataset = torch.utils.data.DataLoader(self.valid_dataset, batch_size=self.batch_size,num_workers=4)\n        return valid_dataset\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\n--------------------------- Wavenet model ---------------------------\n\"\"\"\n\n# from https://www.kaggle.com/hanjoonchoe/wavenet-lstm-pytorch-ignite-ver\nclass WaveBlock(torch.nn.Module):# 1.34e7\n\n    def __init__(self, in_channels, out_channels, dilation_rates, kernel_size):\n        super(WaveBlock, self).__init__()\n        self.num_rates = dilation_rates\n        self.convs = torch.nn.ModuleList()\n        self.filter_convs = torch.nn.ModuleList()\n        self.gate_convs = torch.nn.ModuleList()\n\n        self.convs.append(torch.nn.Conv1d(in_channels, out_channels, kernel_size=1))\n        dilation_rates = [2 ** i for i in range(dilation_rates)]\n        for dilation_rate in dilation_rates:\n            self.filter_convs.append(\n                torch.nn.Conv1d(out_channels, out_channels, kernel_size=kernel_size, padding=int((dilation_rate*(kernel_size-1))/2), dilation=dilation_rate))\n            self.gate_convs.append(\n                torch.nn.Conv1d(out_channels, out_channels, kernel_size=kernel_size, padding=int((dilation_rate*(kernel_size-1))/2), dilation=dilation_rate))\n            self.convs.append(torch.nn.Conv1d(out_channels, out_channels, kernel_size=1))\n\n    def forward(self, x):\n        x = self.convs[0](x)\n        res = x\n        for i in range(self.num_rates):\n            x = torch.tanh(self.filter_convs[i](x)) * torch.sigmoid(self.gate_convs[i](x))\n            x = self.convs[i + 1](x)\n            res = res + x\n        return res\n\n    \nclass Wavenet(torch.nn.Module):\n\n    def __init__(self):\n        super(Wavenet, self).__init__()\n        self.wave_block1 = WaveBlock(10, 16, 12, 3)\n        self.wave_block2 = WaveBlock(16, 32, 8, 3)\n        #self.wave_block3 = WaveBlock(32, 64, 4, 3)\n        #self.wave_block4 = WaveBlock(64, 128, 1, 3)\n        self.fc = torch.nn.Linear(1920032, 1)\n\n    def forward(self, x):\n        x = x.permute(0, 2, 1).float()\n\n        x = self.wave_block1(x)\n        x = self.wave_block2(x)\n        #x = self.wave_block3(x)\n        #x = self.wave_block4(x)\n        x = x.permute(0, 2, 1)\n        x = x.reshape(x.size(0), -1)\n        x = self.fc(x)\n        \n        return x\n\n\"\"\"\n--------------------------- 1D Convolutional model ---------------------------\n\"\"\"\n\nclass SimpleConvnet(torch.nn.Module): # 1.17e7\n\n    def __init__(self):\n        super(SimpleConvnet, self).__init__()\n        self.conv1 = torch.nn.utils.weight_norm(torch.nn.Conv1d(10, 16, 3))\n        self.conv2 = torch.nn.utils.weight_norm(torch.nn.Conv1d(16, 32, 3))\n\n        #self.linear1 = torch.nn.utils.weight_norm(torch.nn.Linear(128, 32))\n        self.linear2 = torch.nn.utils.weight_norm(torch.nn.Linear(479936, 1))\n\n    def forward(self, x):\n        x = x.permute(0, 2, 1).float()\n\n        x = F.relu(self.conv1(x))\n        x = torch.nn.AvgPool1d(kernel_size=2)(x)\n        #x = torch.nn.Dropout(p=0.3)(x)\n        x = F.relu(self.conv2(x))\n        x = torch.nn.AvgPool1d(kernel_size=2)(x)\n        #x = torch.nn.Dropout(p=0.3)(x)\n        \n        x = x.reshape(x.size(0), -1)\n        #print(x.size())\n        #x = self.linear1(x)\n        y_pred = self.linear2(x)\n        \n        return y_pred\n    \n\"\"\"\n--------------------------- 1D Convolutional + LSTM model ---------------------------\n\"\"\"\n\nclass ConvLSTM(torch.nn.Module): #2.31e7\n\n    def __init__(self):\n        super(ConvLSTM, self).__init__()\n        self.conv1 = torch.nn.utils.weight_norm(torch.nn.Conv1d(10, 16, 3))\n        self.conv2 = torch.nn.utils.weight_norm(torch.nn.Conv1d(16, 32, 3))\n        lstm1 = torch.nn.utils.weight_norm(torch.nn.utils.weight_norm(torch.nn.LSTM(32,64, bidirectional=True), 'weight_ih_l0'), 'weight_hh_l0')\n        self.lstm1 = torch.nn.utils.weight_norm(torch.nn.utils.weight_norm(lstm1, 'weight_ih_l0_reverse'), 'weight_hh_l0_reverse')\n        self.lstm1.flatten_parameters()\n\n        self.linear1 = torch.nn.utils.weight_norm(torch.nn.Linear(128, 32)) \n        self.linear2 = torch.nn.utils.weight_norm(torch.nn.Linear(32, 1))\n\n    def forward(self, x):\n        x = x.permute(0, 2, 1).float()\n\n        x = F.relu(self.conv1(x))\n        x = torch.nn.AvgPool1d(kernel_size=2)(x)\n        x = torch.nn.Dropout(p=0.3)(x)\n        x = F.relu(self.conv2(x))\n        x = torch.nn.AvgPool1d(kernel_size=2)(x)\n        x = torch.nn.Dropout(p=0.3)(x)\n        \n        x = x.permute(2, 0, 1)\n\n        x, _ = self.lstm1(x)\n        x = torch.nn.Dropout(p=0.3)(x)\n        \n        \n        \n        x = x.permute(1, 0, 2)\n        x = x[:,-1,:]\n        x = x.reshape(x.size(0), -1)\n\n        x = self.linear1(x)\n        y_pred = self.linear2(x)\n        \n        return y_pred\n\n\"\"\"\n--------------------------- Wrapper model ---------------------------\n\"\"\"\n\n\nclass ConvNet(pl.LightningModule): \n    \n    \n    def __init__(self, structure='baseline'):\n        super(ConvNet, self).__init__()\n        \n        if structure=='baseline':\n            self.model = SimpleConvnet()\n        elif structure=='wavenet':\n            self.model = Wavenet()\n        elif structure=='convlstm':\n            self.model = ConvLSTM()\n        \n    def forward(self, x):\n\n        return self.model(x)\n    \n    def configure_optimizers(self):\n        #optimizer = torch.optim.Adam(self.parameters(), lr=1e-3)\n        total_epochs = 5\n        delay_epochs = int(0.7*total_epochs)\n        \n        optimizer = RangerLars(self.parameters(), lr=1e-2, betas=(.95,.9), eps=1e-1, weight_decay=1e-2)\n        \n        base_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, delay_epochs) # delay the scheduler for 10 steps\n        delayed_scheduler = DelayerScheduler(optimizer, total_epochs - delay_epochs, base_scheduler)\n\n        return optimizer\n    \n    def training_step(self, batch, batch_idx):\n        x, y = batch\n        y_hat = self(x)\n        loss_func = torch.nn.L1Loss()\n        loss = loss_func(y_hat.squeeze(), y) #F.cross_entropy(y_hat, y)\n        result = pl.TrainResult(loss)\n\n        result.log('train_loss', loss, prog_bar=True)\n        return result\n    \n    def validation_step(self, batch, batch_idx):\n        x, y = batch\n        y_hat = self(x)\n        loss_func = torch.nn.L1Loss()\n        loss = loss_func(y_hat.squeeze(), y) #F.cross_entropy(y_hat, y)\n        # Checkpoint model based on validation loss\n        result = pl.EvalResult(checkpoint_on=loss)\n        result.val_loss = loss\n\n        return result\n    \n    def validation_epoch_end(self, validation_step_output_result):\n        all_validation_step_loss = validation_step_output_result.val_loss\n        all_validation_step_acc = validation_step_output_result.val_acc\n\n        \n        validation_step_output_result.log('val_loss', torch.mean(all_validation_step_loss), prog_bar=True)\n        \n        return validation_step_output_result\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Note that the sized of the full connected layers will change if you downsample the dataframe, take care of this**"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_module = VolcanicDataModule(batch_size=32)\nmodel = ConvNet().float()\n\nearly_stop_callback = pl.callbacks.EarlyStopping(\n   monitor='val_loss',\n   min_delta=0.00,\n   patience=3,\n   verbose=False,\n   mode='min'\n)\ntrainer = pl.Trainer(gpus=1, early_stop_callback=early_stop_callback, max_epochs=5, deterministic=True) \ntrainer.fit(model, data_module)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = # Load the model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = Path('/kaggle/input/predict-volcanic-eruptions-ingv-oe')\ntest_path = self.path/'test'\nsample_submission = pd.read_csv('sample_submission.csv')\ntest_dataset = VolcanicDataset(test_path, sample_submission, is_train=False)\n\npredictions = []\nfor x in test_dataset:\n    predictions.append(model(x))\n    \nsample_submission['time_to_eruption'] = predictions\nsample_submission.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"TODO:\n* [X] Downsampling timeseries.\n* [X] Add LSTM.\n* Autoencoder.\n* Use datatable for faster reading."},{"metadata":{},"cell_type":"markdown","source":"This does not seems to be the best option in this scenario. Maybe a more \"traditional\" way is better but this can be a starting point."},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}