{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Normalizing and split"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn import preprocessing","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def normalize(X_train, X_valid, X_test, normalize_opt, excluded_feat):\n    feats = [f for f in X_train.columns if f not in excluded_feat]\n    if normalize_opt is not None:\n        if normalize_opt == 'min_max':\n            scaler = preprocessing.MinMaxScaler()\n        scaler = scaler.fit(X_train[feats])\n        X_train[feats] = scaler.transform(X_train[feats])\n        X_valid[feats] = scaler.transform(X_valid[feats])\n        X_test[feats] = scaler.transform(X_test[feats])\n    return X_train, X_valid, X_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH_DATASET = '/kaggle/input/vulcanic-preprocessing/'\n\ntrain_sample = pd.read_csv(f'{PATH_DATASET}/train_sample.csv')\ntargets = pd.read_csv(f'{PATH_DATASET}/targets.csv')\ntest = pd.read_csv(f'{PATH_DATASET}/test.csv').iloc[:,1:]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# LSTM-based neural network"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install pywick","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\n\nfrom sklearn import preprocessing\nfrom torch.nn import functional as F\nfrom torch import nn\nfrom pytorch_lightning.core.lightning import LightningModule\nfrom pywick.optimizers.nadam import Nadam\nfrom torch.utils.data import TensorDataset, DataLoader\nfrom torchvision import transforms\nfrom pytorch_lightning import Trainer\nfrom pytorch_lightning.callbacks import ModelCheckpoint\nfrom sklearn.metrics import mean_squared_error as mse","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_MODELS = 20\nBATCH_SIZE = 9999\nNUM_EPOCHS = 1500\nPATH_MODEL = '/kaggle/working/models/'\nPATH_DATA = '/kaggle/input/predict-volcanic-eruptions-ingv-oe/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class VolcanicLSTM(LightningModule):\n    \n    def __init__(self, num_features):\n        super().__init__()\n        \n        self.bn = nn.BatchNorm1d(num_features=num_features)\n        self.lstm = nn.LSTM(input_size=num_features, hidden_size=128, num_layers=1)\n        \n        self.conv1 = nn.Conv1d(in_channels=128, out_channels=128, kernel_size=2, padding=1, stride=2)\n        self.conv2 = nn.Conv1d(in_channels=128, out_channels=84, kernel_size=2, padding=1, stride=2)\n        self.conv3 = nn.Conv1d(in_channels=84, out_channels=64, kernel_size=2, padding=1, stride=2)\n        \n        self.flat = nn.Flatten()\n        self.lin1 = nn.Linear(in_features=64, out_features=64)\n        self.lin2 = nn.Linear(in_features=64, out_features=32)\n        self.lin3 = nn.Linear(in_features=32, out_features=1)\n        \n        \n    def forward(self, x):\n        batch_size, _, _ = x.size()\n        x = x.view(batch_size, -1)\n        x = self.bn(x)\n        x = torch.unsqueeze(x, 1)\n        x, _ = self.lstm(x)\n        \n        x = x.permute(0, 2, 1)\n        x = self.conv1(x)\n        x = F.relu(x)\n        x = self.conv2(x)\n        x = F.relu(x)\n        x = self.conv3(x)\n        x = F.relu(x)\n\n        x = self.flat(x)\n        x = self.lin1(x)\n        x = F.relu(x)\n        x = self.lin2(x)\n        x = F.relu(x)\n        x = self.lin3(x)\n        x = F.relu(x)\n\n        return x\n    \n    def configure_optimizers(self):\n        return Nadam(self.parameters(), lr=0.005)\n    \n    def training_step(self, batch, batch_idx):\n        x, y = batch\n        preds = self(x)\n        loss = F.l1_loss(preds, y)\n        return loss\n    \n    def validation_step(self, batch, batch_idx):\n        x, y = batch\n        preds = self(x)\n        loss = F.l1_loss(preds, y)\n        self.log('val_loss', loss)\n        return loss\n    \n    def train_dataloader(self):\n        tensor_x = torch.Tensor(train_x)\n        tensor_y = torch.Tensor(train_y)\n        \n        dataset = TensorDataset(tensor_x, tensor_y)\n        \n        return DataLoader(dataset, batch_size=BATCH_SIZE)\n\n    def val_dataloader(self):\n        tensor_x = torch.Tensor(valid_x)\n        tensor_y = torch.Tensor(valid_y)\n        \n        dataset = TensorDataset(tensor_x, tensor_y)\n        \n        return DataLoader(dataset, batch_size=BATCH_SIZE)\n\n\n    def test_dataloader(self):\n        tensor_x = torch.Tensor(test_scaled)\n        \n        dataset = TensorDataset(tensor_x)\n        \n        return DataLoader(dataset, batch_size=BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv(f'{PATH_DATA}/sample_submission.csv')\n\nsub_final = np.zeros(len(submission))\navg_valid_mae = 0\nnum_used_models = 0\n\ni = 0\n\nwhile i < NUM_MODELS:\n    print('\\nRunning model ', i)\n    \n    train_x, valid_x, train_y, valid_y = train_test_split(train_sample, targets, test_size=0.2, random_state=0)\n    train_x, valid_x, test_scaled = normalize(train_x.copy(), valid_x.copy(), test.copy(), 'min_max', [])\n    train_x = train_x.values.reshape(train_x.shape[0], 1, train_x.shape[1])\n    valid_x = valid_x.values.reshape(valid_x.shape[0], 1, valid_x.shape[1])\n    train_y = train_y.to_numpy()\n    valid_y = valid_y.to_numpy()\n    test_scaled = test_scaled.values.reshape(test_scaled.shape[0], 1, test_scaled.shape[1])\n\n    if not os.path.exists(f'{PATH_MODEL}/{str(i)}'):\n        os.makedirs(f'{PATH_MODEL}/{str(i)}')\n        \n    model = VolcanicLSTM(num_features=train_x.shape[-1])\n    checkpoint_callback = ModelCheckpoint(\n        monitor='val_loss',\n        dirpath=f'{PATH_MODEL}/{str(i)}',\n        filename='best_model-{epoch}'\n    )\n    trainer = Trainer(gpus=1, callbacks=[checkpoint_callback], min_epochs=1, max_epochs=NUM_EPOCHS, progress_bar_refresh_rate=0)\n    \n    trainer.fit(model)\n    \n    print(checkpoint_callback.best_model_path)\n    test_model = VolcanicLSTM.load_from_checkpoint(checkpoint_callback.best_model_path, num_features=train_x.shape[-1])\n    \n    valid_preds = torch.squeeze(test_model(torch.Tensor(valid_x))).detach().numpy()\n    mae = mse(valid_y, valid_preds, squared=False)\n    print(f'{i} MAE: {mae:.0f}')\n    if mae < 10000000:\n        avg_valid_mae += mae \n        sub_final += torch.squeeze(test_model(torch.Tensor(test_scaled))).detach().numpy() \n        num_used_models += 1\n        \n    i += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"avg_valid_mae /= num_used_models\nsub_final /= num_used_models\n\nprint(f'\\nNumber of used models: {num_used_models}')\nprint(f'\\nAverage validation MAE for used models: {avg_valid_mae:.0f}')\n\nsubmission['time_to_eruption'] = sub_final\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}