{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset,DataLoader\nfrom ignite.engine import Events, create_supervised_trainer, create_supervised_evaluator\nfrom ignite.metrics import Accuracy, Loss\nfrom ignite.engine.engine import Engine, State, Events\nfrom ignite.utils import convert_tensor\nfrom ignite.contrib.handlers import TensorboardLogger, ProgressBar\nfrom torch.optim import Adam\nimport gc\nimport warnings  \nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### I will not use whole training set ;-)"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%writefile preproc.py\nimport gc\nfrom tqdm import tqdm\nimport pandas as pd\nimport numpy as np\nimport pyarrow.parquet as pq\ndef get_data():\n\n    train = pq.read_table('/kaggle/input/vsb-power-line-fault-detection/train.parquet',columns = [str(i) for i in range(0,6000)])\n    train = train.to_pandas().T\n    target = pd.read_csv('/kaggle/input/vsb-power-line-fault-detection/metadata_train.csv')['target'].iloc[0:6000]\n    gc.collect()\n    return train.values,target.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from preproc import get_data\nx_train, y_train = get_data()\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PowerlineDataset(Dataset):\n    def __init__(self,features,labels):\n        super().__init__()\n        self.labels = labels\n        self.features = features\n        \n    def __len__(self):\n        return len(self.labels)\n    \n    def __getitem__(self,idx):\n        labels = self.labels[idx]\n        features = self.features[idx]\n        return features, labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Attention(nn.Module):\n    \"\"\" Applies attention mechanism on the `context` using the `query`.\n    **Thank you** to IBM for their initial implementation of :class:`Attention`. Here is\n    their `License\n    <https://github.com/IBM/pytorch-seq2seq/blob/master/LICENSE>`__.\n    Args:\n        dimensions (int): Dimensionality of the query and context.\n        attention_type (str, optional): How to compute the attention score:\n            * dot: :math:`score(H_j,q) = H_j^T q`\n            * general: :math:`score(H_j, q) = H_j^T W_a q`\n    Example:\n         >>> attention = Attention(256)\n         >>> query = torch.randn(5, 1, 256)\n         >>> context = torch.randn(5, 5, 256)\n         >>> output, weights = attention(query, context)\n         >>> output.size()\n         torch.Size([5, 1, 256])\n         >>> weights.size()\n         torch.Size([5, 1, 5])\n    \"\"\"\n\n    def __init__(self, dimensions, attention_type='general'):\n        super(Attention, self).__init__()\n\n        if attention_type not in ['dot', 'general']:\n            raise ValueError('Invalid attention type selected.')\n\n        self.attention_type = attention_type\n        if self.attention_type == 'general':\n            self.linear_in = nn.Linear(dimensions, dimensions, bias=False)\n\n        self.linear_out = nn.Linear(dimensions * 2, dimensions, bias=False)\n        self.softmax = nn.Softmax(dim=-1)\n        self.tanh = nn.Tanh()\n\n    def forward(self, query, context):\n        \"\"\"\n        Args:\n            query (:class:`torch.FloatTensor` [batch size, output length, dimensions]): Sequence of\n                queries to query the context.\n            context (:class:`torch.FloatTensor` [batch size, query length, dimensions]): Data\n                overwhich to apply the attention mechanism.\n        Returns:\n            :class:`tuple` with `output` and `weights`:\n            * **output** (:class:`torch.LongTensor` [batch size, output length, dimensions]):\n              Tensor containing the attended features.\n            * **weights** (:class:`torch.FloatTensor` [batch size, output length, query length]):\n              Tensor containing attention weights.\n        \"\"\"\n        batch_size, output_len, dimensions = query.size()\n        query_len = context.size(1)\n\n        if self.attention_type == \"general\":\n            query = query.reshape(batch_size * output_len, dimensions)\n            query = self.linear_in(query)\n            query = query.reshape(batch_size, output_len, dimensions)\n\n        # TODO: Include mask on PADDING_INDEX?\n\n        # (batch_size, output_len, dimensions) * (batch_size, query_len, dimensions) ->\n        # (batch_size, output_len, query_len)\n        attention_scores = torch.bmm(query, context.transpose(1, 2).contiguous())\n\n        # Compute weights across every context sequence\n        attention_scores = attention_scores.view(batch_size * output_len, query_len)\n        attention_weights = self.softmax(attention_scores)\n        attention_weights = attention_weights.view(batch_size, output_len, query_len)\n\n        # (batch_size, output_len, query_len) * (batch_size, query_len, dimensions) ->\n        # (batch_size, output_len, dimensions)\n        mix = torch.bmm(attention_weights, context)\n\n        # concat -> (batch_size * output_len, 2*dimensions)\n        combined = torch.cat((mix, query), dim=2)\n        combined = combined.view(batch_size * output_len, 2 * dimensions)\n\n        # Apply linear_out on every 2nd dimension of concat\n        # output -> (batch_size, output_len, dimensions)\n        output = self.linear_out(combined).view(batch_size, output_len, dimensions)\n        output = self.tanh(output)\n\n        return output, attention_weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Dilated_Dense_Layers(nn.Module):\n    def __init__(self,in_channels,out_channels,dilation_rates):\n        super().__init__()\n        self.dilation_rates = dilation_rates\n        self.dilated_conv = nn.ModuleList()\n        self.casual_conv1 = nn.Conv1d(in_channels,out_channels,kernel_size=1)\n        self.casual_conv2 = nn.Conv1d(out_channels,out_channels,kernel_size=1)\n        self.avgpool1d = nn.AvgPool1d(10)\n        dilation_rates = [2**dilation_rate for dilation_rate in range(dilation_rates)]\n        for dilation_rate in dilation_rates:\n            self.dilated_conv.append(nn.Conv1d(out_channels,out_channels,kernel_size=3,padding=dilation_rate,dilation=dilation_rate))\n        \n    def forward(self,x):\n        x = self.casual_conv1(x)\n        res_x = x\n        for i in range(self.dilation_rates):\n            x = F.tanh(self.dilated_conv[i](x))*F.sigmoid(self.dilated_conv[i](x))\n            x += res_x\n        x = self.casual_conv2(x)\n        x = self.avgpool1d(x)\n        return x\n    \nclass WaveLSTM(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.out_channels = 16\n        self.dilation_dense_layers1 = Dilated_Dense_Layers(1,self.out_channels,4)\n        self.dilation_dense_layers2 = Dilated_Dense_Layers(self.out_channels,self.out_channels*2,3)\n        self.dilation_dense_layers3 = Dilated_Dense_Layers(self.out_channels*2,self.out_channels*4,2)\n        self.dilation_dense_layers4 = Dilated_Dense_Layers(self.out_channels*4,self.out_channels*8,1)\n        self.LSTM = nn.GRU(800,800,bidirectional=True,batch_first=True)\n        self.fc = nn.Linear(800,1)\n        \n    def forward(self,x):\n        x = self.dilation_dense_layers1(x.unsqueeze(1).float())\n        x = self.dilation_dense_layers2(x)\n        x = self.dilation_dense_layers3(x)\n        a,b = self.LSTM(x)\n        x,_ = Attention(800)(b.permute(1,0,2)[:,0,:].unsqueeze(1),a[:,:,:800])\n        x = self.fc(x)\n        return F.sigmoid(x)\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = WaveLSTM()\nprint(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataloader():\n    from sklearn.model_selection import train_test_split\n    x_train1, x_test1, y_train1, y_test1 = train_test_split(x_train, y_train, test_size=0.33, random_state=42)\n    train_dataset = PowerlineDataset(x_train1,y_train1)\n    train_loader = torch.utils.data.DataLoader(train_dataset,batch_size=4,shuffle=True)\n    valid_dataset = PowerlineDataset(x_test1,y_test1)\n    valid_loader = torch.utils.data.DataLoader(valid_dataset,batch_size=4,shuffle=True)\n    gc.collect()\n    return train_loader,valid_loader\ntrain_loader,valid_loader = get_dataloader()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _prepare_batch(batch, device=None, non_blocking=False):\n    \"\"\"Prepare batch for training: pass to a device with options.\n\n    \"\"\"\n    x, y = batch\n    return (convert_tensor(x, device=device, non_blocking=non_blocking),\n            convert_tensor(y, device=device, non_blocking=non_blocking))\n    \ndef create_supervised_evaluator1(model, metrics=None,\n                                device=None, non_blocking=False,\n                                prepare_batch=_prepare_batch,\n                                output_transform=lambda x, y, y_pred: (y_pred, y,)):\n    \"\"\"\n    Factory function for creating an evaluator for supervised models.\n\n    Args:\n        model (`torch.nn.Module`): the model to train.\n        metrics (dict of str - :class:`~ignite.metrics.Metric`): a map of metric names to Metrics.\n        device (str, optional): device type specification (default: None).\n            Applies to both model and batches.\n        non_blocking (bool, optional): if True and this copy is between CPU and GPU, the copy may occur asynchronously\n            with respect to the host. For other cases, this argument has no effect.\n        prepare_batch (callable, optional): function that receives `batch`, `device`, `non_blocking` and outputs\n            tuple of tensors `(batch_x, batch_y)`.\n        output_transform (callable, optional): function that receives 'x', 'y', 'y_pred' and returns value\n            to be assigned to engine's state.output after each iteration. Default is returning `(y_pred, y,)` which fits\n            output expected by metrics. If you change it you should use `output_transform` in metrics.\n\n    Note: `engine.state.output` for this engine is defind by `output_transform` parameter and is\n        a tuple of `(batch_pred, batch_y)` by default.\n\n    Returns:\n        Engine: an evaluator engine with supervised inference function.\n    \"\"\"\n    metrics = metrics or {}\n\n    if device:\n        model\n\n    def _inference(engine, batch):\n        model.eval()\n        with torch.no_grad():\n            x, y = prepare_batch(batch, device=device, non_blocking=non_blocking)\n            y_pred = model(x)\n            return output_transform(x, y.float(), y_pred)\n\n    engine = Engine(_inference)\n\n    for name, metric in metrics.items():\n        metric.attach(engine, name)\n\n    return engine\n\ndef create_supervised_trainer1(model, optimizer, loss_fn, metrics={}, device=None):\n\n    def _update(engine, batch):\n        model.train()\n        optimizer.zero_grad()\n        x, y = _prepare_batch(batch, device=device)\n        y_pred = model(x)\n        loss = loss_fn(y_pred, y.float())\n        loss.backward()\n        optimizer.step()\n        return loss.item(), y_pred, y.float()\n\n    def _metrics_transform(output):\n        return output[0],output[1], output[2]\n\n    engine = Engine(_update)\n\n    for name, metric in metrics.items():\n        metric._output_transform = _metrics_transform\n        metric.attach(engine, name)\n\n    return engine\ndevice = 'cuda:0' if torch.cuda.is_available() else 'cpu'\noptimizer = Adam(model.parameters(),lr=0.0003)\ncriterion = nn.BCELoss()\nmetrics = {\n    'loss': Loss(criterion),\n}\ntrainer = create_supervised_trainer1(model.to(device), optimizer,criterion, device=device)\nval_evaluator = create_supervised_evaluator1(model.to(device), metrics=metrics, device=device)\npbar = ProgressBar(bar_format='')\npbar.attach(trainer, output_transform=lambda x: {'loss': x[0]})\n@trainer.on(Events.EPOCH_COMPLETED)\ndef compute_and_display_val_metrics(engine):\n    epoch = engine.state.epoch\n    metrics = val_evaluator.run(valid_loader).metrics\n    print(\"Validation Results - Epoch: {}  Average Loss: {:.4f}\"\n          .format(engine.state.epoch, \n                      metrics['loss']))\n@trainer.on(Events.EPOCH_COMPLETED)\ndef save_model(engine):\n    torch.save(model.state_dict(), 'saved_model.pth')\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if torch.cuda.is_available():\n    torch.set_default_tensor_type(torch.cuda.FloatTensor)\ntrainer.run(train_loader, max_epochs=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestPowerlineDataset(Dataset):\n    def __init__(self,features):\n        super().__init__()\n        self.features = features\n        \n    def __len__(self):\n        return len(self.features)\n    \n    def __getitem__(self,idx):\n\n        features = self.features[idx]\n        return features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def test_load(phase,signal_ids,batch_size):\n    x_test = pq.read_table('/kaggle/input/vsb-power-line-fault-detection/test.parquet',columns =[str(signal_ids[i]) for i in range(batch_size*phase,batch_size*(phase+1))])\n    x_test1 = x_test.to_pandas().T\n    return x_test1.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_loader\ndel valid_loader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\nimport pyarrow.parquet as pq\ndef saver():\n    if torch.cuda.is_available():\n        torch.set_default_tensor_type(torch.cuda.FloatTensor)\n    device = 'cuda:0' if torch.cuda.is_available() else 'cpu'\n    submission = pd.read_csv('/kaggle/input/vsb-power-line-fault-detection/sample_submission.csv')\n    signal_ids = pd.read_csv('/kaggle/input/vsb-power-line-fault-detection/metadata_test.csv')['signal_id'].values\n    model.load_state_dict(torch.load('saved_model.pth'))\n    model.eval()\n    predictions = []\n    iterations = 5\n    batch_size = len(signal_ids)//5\n    for i in range(iterations):\n        predictions = []\n        print('iteraion {}'.format(i+1))\n        x_test = test_load(phase=i,batch_size=batch_size,signal_ids=signal_ids)\n        test_dataset = TestPowerlineDataset(x_test)\n        test_loader = torch.utils.data.DataLoader(test_dataset,batch_size=20,shuffle=False)\n        with torch.no_grad():\n            for idx , inputs in tqdm(enumerate(test_loader),total=len(test_loader)):\n                preds = model(inputs.float()).to(device)\n                predictions.append(np.squeeze(preds.cpu().detach().numpy()))\n        \n            predictions = np.round(np.hstack(predictions)).astype(int)\n            submission[batch_size*i:batch_size*(i+1)].target = predictions\n            gc.collect()\n    submission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#saver()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}