{"cells":[{"metadata":{},"cell_type":"markdown","source":"This is replication of https://www.kaggle.com/wimwim/wavenet-lstm (converted to pytorch + ignite vers.)<br>\n\n\nI should say I don't have any knowledge of the dataset.<br>\nPreprocessing is entirely cited from original notebook.<br>\nThis is a practice of lstm model, so I mainly focus on building a model quickly getting grasp of concept(my purpose as usual). ;-)<br>\n\nIgnite code is partially cited from https://www.kaggle.com/yhn112/resnet18-baseline-pytorch-ignite\n\nWarning : Code is dirty needed to clean up :-)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom tqdm import tqdm_notebook\nfrom torch.utils.data import DataLoader, Dataset\nimport ignite\nfrom ignite.engine import Events, create_supervised_trainer, create_supervised_evaluator\nfrom ignite.metrics import Accuracy, Loss\nfrom ignite.contrib.handlers import TensorboardLogger, ProgressBar\nfrom torch.optim.lr_scheduler import ExponentialLR\nfrom ignite.engine.engine import Engine, State, Events\nfrom ignite.utils import convert_tensor\nimport gc","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"%%writefile preprocess.py\nimport numpy as np\nimport pandas as pd\nimport gc\nfrom tqdm import tqdm_notebook\ndef preprocess():\n    train = pd.read_csv('/kaggle/input/LANL-Earthquake-Prediction/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32})\n    etq_meta = [\n    {\"start\":0,         \"end\":5656574},\n    {\"start\":5656574,   \"end\":50085878},\n    {\"start\":50085878,  \"end\":104677356},\n    {\"start\":104677356, \"end\":138772453},\n    {\"start\":138772453, \"end\":187641820},\n    {\"start\":187641820, \"end\":218652630},\n    {\"start\":218652630, \"end\":245829585},\n    {\"start\":245829585, \"end\":307838917},\n    {\"start\":307838917, \"end\":338276287},\n    {\"start\":338276287, \"end\":375377848},\n    {\"start\":375377848, \"end\":419368880},\n    {\"start\":419368880, \"end\":461811623},\n    {\"start\":461811623, \"end\":495800225},\n    {\"start\":495800225, \"end\":528777115},\n    {\"start\":528777115, \"end\":585568144},\n    {\"start\":585568144, \"end\":621985673},\n    {\"start\":621985673, \"end\":629145480},\n    ]\n\n    df = []\n    for i in [2, 7, 0, 4, 11, 13, 9, 1, 14, 10]:\n        df.append(train[etq_meta[i]['start']:etq_meta[i]['start']+150000*((etq_meta[i]['end'] - etq_meta[i]['start'])//150000)])\n    \n    train = pd.concat(df)\n\n    num_seg = len(train)//150000\n    train_X = []\n    train_y = []\n    for i in tqdm_notebook(range(num_seg)):\n    #     train_X.append(fft_process(train['acoustic_data'].iloc[150000 * i:150000 * i + 150000]))\n        if 100000 * i + 150000 < len(train):\n            train_X.append(train['acoustic_data'].iloc[150000 * i:150000 * i + 150000])\n            train_y.append(train['time_to_failure'].iloc[150000 * i + 149999])\n    del train\n    gc.collect()\n    train_X = np.array(train_X,dtype = np.float32)\n    train_y = np.array(train_y,dtype = np.float32)\n\n    X_mean = train_X.mean(0)\n    X_std = train_X.std(0)\n    train_X -= X_mean\n    train_X /= X_std\n    y_mean = train_y.mean()\n    y_std = train_y.std()\n    train_y -= y_mean\n    train_y /= y_std\n\n    train_X = np.expand_dims(train_X,-1)\n    np.save('train_x.npy',train_X)\n    np.save('train_y.npy',train_y)\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from preprocess import preprocess\npreprocess()\ntrain_X = np.load('train_x.npy')\ntrain_y = np.load('train_y.npy')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Attention(nn.Module):\n    def __init__(self, feature_dim, step_dim, bias=True, **kwargs):\n        super(Attention, self).__init__(**kwargs)\n        \n        self.supports_masking = True\n\n        self.bias = bias\n        self.feature_dim = feature_dim\n        self.step_dim = step_dim\n        self.features_dim = 0\n        \n        weight = torch.zeros(feature_dim, 1)\n        nn.init.xavier_uniform_(weight)\n        self.weight = nn.Parameter(weight)\n        \n        if bias:\n            self.b = nn.Parameter(torch.zeros(step_dim))\n        \n    def forward(self, x, mask=None):\n        feature_dim = self.feature_dim\n        step_dim = self.step_dim\n\n        eij = torch.mm(\n            x.contiguous().view(-1, feature_dim), \n            self.weight\n        ).view(-1, step_dim)\n        \n        if self.bias:\n            eij = eij + self.b\n            \n        eij = torch.tanh(eij)\n        a = torch.exp(eij)\n        \n        if mask is not None:\n            a = a * mask\n\n        a = a / torch.sum(a, 1, keepdim=True) + 1e-10\n\n        weighted_input = x * torch.unsqueeze(a, -1)\n        return torch.sum(weighted_input, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Wave_Block(nn.Module):\n    \n    def __init__(self,in_channels,out_channels,dilation_rates):\n        super(Wave_Block,self).__init__()\n        self.num_rates = dilation_rates\n        self.convs = nn.ModuleList()\n        self.filter_convs = nn.ModuleList()\n        self.gate_convs = nn.ModuleList()\n        \n        self.convs.append(nn.Conv1d(in_channels,out_channels,kernel_size=1))\n        dilation_rates = [2**i for i in range(dilation_rates)]\n        for dilation_rate in dilation_rates:\n            self.filter_convs.append(nn.Conv1d(out_channels,out_channels,kernel_size=3,padding=dilation_rate,dilation=dilation_rate))\n            self.gate_convs.append(nn.Conv1d(out_channels,out_channels,kernel_size=3,padding=dilation_rate,dilation=dilation_rate))\n            self.convs.append(nn.Conv1d(out_channels,out_channels,kernel_size=1))\n            \n    def forward(self,x):\n        x = self.convs[0](x)\n        res = x\n        for i in range(self.num_rates):\n            x = F.tanh(self.filter_convs[i](x))*F.sigmoid(self.gate_convs[i](x))\n            x = self.convs[i+1](x)\n            x += res\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Wave_LSTM(nn.Module):\n    def __init__(self):\n        super().__init__()\n        in_channels = 150000\n\n        self.LSTM = nn.GRU(input_size=in_channels//10**3,hidden_size=in_channels//10**3,num_layers=64,batch_first=True,bidirectional=True)\n        self.conv1 = nn.Linear(300,128)\n        self.conv2 = nn.Linear(128,1)\n        self.attention = Attention(300,64)\n        self.avgpool1d = nn.AvgPool1d(10)\n        self.wave_block1 = Wave_Block(1,16,8)\n        self.wave_block2 = Wave_Block(16,32,5)\n        self.wave_block3 = Wave_Block(32,64,3)\n            \n    def forward(self,x):\n        x = self.wave_block1(x)\n        #shrinking\n        x = self.avgpool1d(x)\n        x = self.wave_block2(x)\n        #shrinking\n        x = self.avgpool1d(x)\n        x = self.wave_block3(x)\n        #shrinking\n        x = self.avgpool1d(x)\n        x,_ = self.LSTM(x)\n        x = self.attention(x)\n        x = F.dropout(x,0.2)\n        x = self.conv1(x)\n        x = self.conv2(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install torchsummary\nfrom torchsummary import summary\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel = Wave_LSTM().to(device)\nsummary(model.to(device), input_size=(1, 150000))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Dataset(Dataset):\n    def __init__(self,features,target):\n        super().__init__()\n        self.features = features\n        self.target = target\n        \n    def __len__(self):\n        return len(self.target)\n    \n    def __getitem__(self,idx):\n        feat = self.features[idx]\n        trg = self.target[idx]\n        \n        return feat,trg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(train_X, train_y, test_size=0.10, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_data_loaders(batch_size=32):\n    train_dataset = Dataset(X_train,y_train)\n    val_dataset = Dataset(X_test,y_test)\n    train_loader = torch.utils.data.DataLoader(train_dataset,batch_size=batch_size,shuffle=True)\n    val_loader = torch.utils.data.DataLoader(val_dataset,batch_size=batch_size,shuffle=True)\n    return train_loader,val_loader\ntrain_loader , val_loader = get_data_loaders()\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _prepare_batch(batch, device=None, non_blocking=False):\n    \"\"\"Prepare batch for training: pass to a device with options.\n\n    \"\"\"\n    x, y = batch\n    return (convert_tensor(x, device=device, non_blocking=non_blocking),\n            convert_tensor(y, device=device, non_blocking=non_blocking))\ndef create_supervised_trainer1(model, optimizer, loss_fn, metrics={}, device=None):\n\n    def _update(engine, batch):\n        model.train()\n        optimizer.zero_grad()\n        x, y = _prepare_batch(batch, device=device)\n        y_pred = model(x.permute(0,2,1))\n        loss = loss_fn(y_pred, y)\n        loss.backward()\n        optimizer.step()\n        return loss.item(), y_pred, y\n\n    def _metrics_transform(output):\n        return output[1], output[2]\n\n    engine = Engine(_update)\n\n    for name, metric in metrics.items():\n        metric._output_transform = _metrics_transform\n        metric.attach(engine, name)\n\n    return engine\n\ndef create_supervised_evaluator1(model, metrics=None,\n                                device=None, non_blocking=False,\n                                prepare_batch=_prepare_batch,\n                                output_transform=lambda x, y, y_pred: (y_pred, y,)):\n    \"\"\"\n    Factory function for creating an evaluator for supervised models.\n\n    Args:\n        model (`torch.nn.Module`): the model to train.\n        metrics (dict of str - :class:`~ignite.metrics.Metric`): a map of metric names to Metrics.\n        device (str, optional): device type specification (default: None).\n            Applies to both model and batches.\n        non_blocking (bool, optional): if True and this copy is between CPU and GPU, the copy may occur asynchronously\n            with respect to the host. For other cases, this argument has no effect.\n        prepare_batch (callable, optional): function that receives `batch`, `device`, `non_blocking` and outputs\n            tuple of tensors `(batch_x, batch_y)`.\n        output_transform (callable, optional): function that receives 'x', 'y', 'y_pred' and returns value\n            to be assigned to engine's state.output after each iteration. Default is returning `(y_pred, y,)` which fits\n            output expected by metrics. If you change it you should use `output_transform` in metrics.\n\n    Note: `engine.state.output` for this engine is defind by `output_transform` parameter and is\n        a tuple of `(batch_pred, batch_y)` by default.\n\n    Returns:\n        Engine: an evaluator engine with supervised inference function.\n    \"\"\"\n    metrics = metrics or {}\n\n    if device:\n        model\n\n    def _inference(engine, batch):\n        model.eval()\n        with torch.no_grad():\n            x, y = prepare_batch(batch, device=device, non_blocking=non_blocking)\n            y_pred = model(x.permute(0,2,1))\n            return output_transform(x, y, y_pred)\n\n    engine = Engine(_inference)\n\n    for name, metric in metrics.items():\n        metric.attach(engine, name)\n\n    return engine","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\ncriterion = nn.MSELoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=0.003)\nmetrics = {\n    'loss': Loss(criterion),\n}\nmodel = Wave_LSTM()\ntrainer = create_supervised_trainer1(model.to(device), optimizer, criterion, device=device)\nval_evaluator = create_supervised_evaluator1(model.to(device), metrics=metrics, device=device)\n@trainer.on(Events.EPOCH_COMPLETED)\ndef compute_and_display_val_metrics(engine):\n    epoch = engine.state.epoch\n    metrics = val_evaluator.run(val_loader).metrics\n    print(\"Validation Results - Epoch: {}  Average Loss: {:.4f}\"\n          .format(engine.state.epoch, \n                      metrics['loss']))\npbar = ProgressBar(bar_format='')\npbar.attach(trainer, output_transform=lambda x: {'loss': x})\n\nlr_scheduler = ExponentialLR(optimizer, gamma=0.95)\n@trainer.on(Events.EPOCH_COMPLETED)\ndef update_lr_scheduler(engine):\n    lr_scheduler.step()\n    lr = float(optimizer.param_groups[0]['lr'])\n    print(\"Learning rate: {}\".format(lr))\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainer.run(train_loader, max_epochs=10)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}