{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Prediction using HQS(High-Quality Signal) and CNN model","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport os\nimport pyarrow.parquet as pq\nfrom tqdm import tqdm\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport torch\nimport torch.nn as nn\nfrom torchvision.models import resnet50, resnet18","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-06T00:15:41.345152Z","iopub.execute_input":"2023-03-06T00:15:41.345690Z","iopub.status.idle":"2023-03-06T00:15:41.351169Z","shell.execute_reply.started":"2023-03-06T00:15:41.345654Z","shell.execute_reply":"2023-03-06T00:15:41.349979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load dataset for visualization\n## Define parquet-loading function for each batchfile\n","metadata":{}},{"cell_type":"code","source":"def get_batch(batchfile):\n    batch1 = pq.ParquetFile(batchfile)\n    it = batch1.iter_batches()\n    batch1 = next(it).to_pandas()\n    batch1['Batch'] = batchfile.split('/')[-1].split('batch_')[1].split('.')[0]\n    return(batch1)\ndef get_pq(pqfile):\n    pq_df = pq.ParquetFile(pqfile)\n    it = pq_df.iter_batches()\n    pq_df = next(it).to_pandas()\n    return(pq_df)","metadata":{"execution":{"iopub.status.busy":"2023-03-06T00:15:42.061858Z","iopub.execute_input":"2023-03-06T00:15:42.062620Z","iopub.status.idle":"2023-03-06T00:15:42.069441Z","shell.execute_reply.started":"2023-03-06T00:15:42.062579Z","shell.execute_reply":"2023-03-06T00:15:42.068480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Appending parquet dataset","metadata":{}},{"cell_type":"code","source":"sensor = pd.read_csv('/kaggle/input/icecube-neutrinos-in-deep-ice/sensor_geometry.csv')\ntrain_meta = pq.ParquetFile('/kaggle/input/icecube-neutrinos-in-deep-ice/train_meta.parquet')\nit = train_meta.iter_batches()\ntrain_meta = next(it).to_pandas()","metadata":{"execution":{"iopub.status.busy":"2023-03-06T00:15:42.564822Z","iopub.execute_input":"2023-03-06T00:15:42.565149Z","iopub.status.idle":"2023-03-06T00:15:58.237292Z","shell.execute_reply.started":"2023-03-06T00:15:42.565120Z","shell.execute_reply":"2023-03-06T00:15:58.236164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_batch = '/kaggle/input/icecube-neutrinos-in-deep-ice/train/'\n# Just appending two batches, just to see if there's difference between two kinds of signals\nsensor_info = [get_batch(path_batch+'batch_'+str(i+1)+'.parquet') for i in tqdm(range(2))]\nsensor_info_df = pd.concat(sensor_info).reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-03-06T00:15:58.241035Z","iopub.execute_input":"2023-03-06T00:15:58.241318Z","iopub.status.idle":"2023-03-06T00:16:00.811686Z","shell.execute_reply.started":"2023-03-06T00:15:58.241291Z","shell.execute_reply":"2023-03-06T00:16:00.810586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check for data\nsensor_info_df.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-03-06T00:16:00.815945Z","iopub.execute_input":"2023-03-06T00:16:00.819853Z","iopub.status.idle":"2023-03-06T00:16:00.855105Z","shell.execute_reply.started":"2023-03-06T00:16:00.819807Z","shell.execute_reply":"2023-03-06T00:16:00.854197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## According to organizer's description... \n##### https://storage.googleapis.com/kaggle-forum-message-attachments/1958559/18618/kaggle_webinar_small.pdf\n#### \"Typically, the most important feature of the pulses for directional reconstruction is not their position, but their relative time!\"\n> We'll use time as our prime feature for our input generation.","metadata":{}},{"cell_type":"markdown","source":"## Plotting time-charge graph colored by Auxiliary annotation\n- For Batch 1 , first 16 events' data","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (20,10))\nsns.set(font_scale = 1,style = 'ticks')\nfor i in range(16):\n    plt.subplot(4,4,i+1)\n    df = sensor_info_df[(sensor_info_df.event_id==sensor_info_df.event_id.unique()[i])]\n    sns.lineplot(data = df,x = 'time',y = 'charge',hue = 'auxiliary',)\n    plt.title('Batch 1, Event ID'+str(sensor_info_df.event_id.unique()[i]))\n    plt.legend(loc = 'upper right')\n    sns.despine()\nplt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2023-03-06T00:16:00.857815Z","iopub.execute_input":"2023-03-06T00:16:00.858446Z","iopub.status.idle":"2023-03-06T00:16:03.777224Z","shell.execute_reply.started":"2023-03-06T00:16:00.858404Z","shell.execute_reply":"2023-03-06T00:16:03.776261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## This looks auxiliary False signals provide more reliable peaks - so named it as HQS ( High-quality Signal )","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:19:38.028365Z","iopub.execute_input":"2023-02-03T05:19:38.028816Z","iopub.status.idle":"2023-02-03T05:19:38.045916Z","shell.execute_reply.started":"2023-02-03T05:19:38.028780Z","shell.execute_reply":"2023-02-03T05:19:38.045081Z"}}},{"cell_type":"markdown","source":"# ResNet for detecting direction using HQS ( High-Quality Signal)","metadata":{}},{"cell_type":"markdown","source":"## Dataset Processing","metadata":{}},{"cell_type":"markdown","source":"#### Metadata table provides eventwise target value, so training data should be processed in a same manner\n#### But The number of signals varies across each event ids to we need to fix the size of it ( CNN / conventional ML approach )\n#### Or approach in a RNN manner : signal processing. \n\n> In this notebook, we'll be using pretrained CNN models!","metadata":{"execution":{"iopub.status.busy":"2023-02-03T05:31:57.141777Z","iopub.execute_input":"2023-02-03T05:31:57.142219Z","iopub.status.idle":"2023-02-03T05:31:57.147163Z","shell.execute_reply.started":"2023-02-03T05:31:57.142185Z","shell.execute_reply":"2023-02-03T05:31:57.146142Z"}}},{"cell_type":"markdown","source":"## Define preproc function for generating input tensors","metadata":{}},{"cell_type":"code","source":"def process_tabular(df_selected):\n    '''\n    Processing input tensor.\n    Input Tensor : 3D Tensor\n    With time, batch_id, charge.\n    '''\n    torch_tensor = torch.tensor(df_selected.astype(int).values)\n    torch_tensor_tf = torch_tensor.unsqueeze(-1)\n    torch_tensor_tf = torch_tensor_tf.type(torch.LongTensor)\n    torch_tensor_tf = torch_tensor_tf.permute(1,2,0)\n    torch_tensor_tf = torch_tensor_tf.float()\n    return(torch_tensor_tf)","metadata":{"execution":{"iopub.status.busy":"2023-03-06T00:16:03.778268Z","iopub.execute_input":"2023-03-06T00:16:03.778633Z","iopub.status.idle":"2023-03-06T00:16:03.786100Z","shell.execute_reply.started":"2023-03-06T00:16:03.778597Z","shell.execute_reply":"2023-03-06T00:16:03.785144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define torch Dataset","metadata":{}},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\n\nclass ICECUDE_Dataset(Dataset):\n    def __init__(self, parquetfile, batch_dir, batch_num,mode='Train'):\n        self.batch_num = batch_num\n        self.train_meta = pq.read_table(parquetfile,filters=[('batch_id','==',self.batch_num)]).to_pandas().reset_index(drop = True)\n        # Appending only HQS\n        self.sensor_info_df = pq.read_table(batch_dir+'batch_'+str(batch_num)+'.parquet',filters=[('auxiliary','==',False)]).to_pandas()\n        self.sensor_info_df = self.sensor_info_df.drop('auxiliary',axis=1).reset_index()\n        self.dataset_mode = mode\n    def __len__(self):\n        return self.sensor_info_df.event_id.nunique()\n\n    def __getitem__(self, idx):\n        '''\n        Tensors will be iterated using event_id, in DataLoader.\n        Since Parquet batch files are quite large, parquet files of interest loaded in __init__ function.\n        And then generate input tensors using event_id as an index.\n        '''\n        event_id = self.sensor_info_df.event_id.unique()\n        sensor_info_df_tmp = self.sensor_info_df\n        sensor_info_df_tmp = sensor_info_df_tmp[sensor_info_df_tmp.event_id==event_id[idx]].drop('event_id',axis=1)\n        train_meta_tmp = self.train_meta[self.train_meta.event_id==event_id[idx]]\n        input_tensor = process_tabular(sensor_info_df_tmp)\n        if self.dataset_mode=='Train':\n            label = torch.Tensor(train_meta_tmp[['azimuth','zenith']].values).squeeze()\n            sample = {'input_tensor': input_tensor,'label':label}\n        else : \n            sample = {'input_tensor':input_tensor}\n        return sample\n","metadata":{"execution":{"iopub.status.busy":"2023-03-06T00:16:03.787792Z","iopub.execute_input":"2023-03-06T00:16:03.788428Z","iopub.status.idle":"2023-03-06T00:16:03.804443Z","shell.execute_reply.started":"2023-03-06T00:16:03.788393Z","shell.execute_reply":"2023-03-06T00:16:03.803493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load model : ResNet50 for this case\n- ResNet module implementation from https://pseudo-lab.github.io/pytorch-guide/docs/ch03-1.html","metadata":{}},{"cell_type":"code","source":"# 기본 ResNet 34층\ndef resnet34(pretrained=False, progress=True, **kwargs):\n    return _resnet('resnet34', BasicBlock, [3, 4, 6, 3], pretrained, progress, **kwargs)\n\n# 기본 ResNet 50층\ndef resnet50(pretrained=False, progress=True, **kwargs):\n    return _resnet('resnet50', Bottleneck, [3, 4, 6, 3], pretrained, progress, **kwargs)\n\n# Wide ResNet\ndef wide_resnet50_2(pretrained=False, progress=True, **kwargs):\n    kwargs['width_per_group'] = 64 * 2\n    return _resnet('wide_resnet50_2', Bottleneck, [3, 4, 6, 3], pretrained, progress, **kwargs)\n\n# ResNext\ndef resnext50_32x4d(pretrained=False, progress=True, **kwargs):\n    kwargs['groups'] = 32 # input channel을 32개의 그룹으로 분할 (cardinality)\n    kwargs['width_per_group'] = 4 # 각 그룹당 4(=128/32)개의 채널으로 구성.\n    # 각 그룹당 channel 4의 output feautre map 생성, concatenate해서 128개로 다시 생성.\n    \n    return _resnet('resnext50_32x4d', Bottleneck, [3, 4, 6, 3], pretrained, progress, **kwargs)\ndef _resnet(arch, block, layers, pretrained, progress, **kwargs):\n    r\"\"\"\n    - pretrained: pretrained된 모델 가중치를 불러오기 (saved by caffe)\n    - arch: ResNet모델 이름\n    - block: 어떤 block 형태 사용할지 (\"Basic or Bottleneck\")\n    - layers: 해당 block이 몇번 사용되는지를 list형태로 넘겨주는 부분\n    \"\"\"\n    model = ResNet(block, layers, **kwargs)\n    if pretrained:\n        state_dict = load_state_dict_from_url(model_urls[arch], progress=progress)\n        model.load_state_dict(state_dict)\n    return model\n\ndef conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1):\n    r\"\"\"\n    3x3 convolution with padding\n    - in_planes: in_channels\n    - out_channels: out_channels\n    - bias=False: BatchNorm에 bias가 포함되어 있으므로, conv2d는 bias=False로 설정.\n    \"\"\"\n    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,\n                     padding=dilation, groups=groups, bias=False, dilation=dilation)\n\ndef conv1x1(in_planes, out_planes, stride=1):\n    \"\"\"1x1 convolution\"\"\"\n    return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)\n\nclass BasicBlock(nn.Module):\n    expansion = 1\n\n    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,\n                 base_width=64, dilation=1, norm_layer=None):\n        r\"\"\"\n         - inplanes: input channel size\n         - planes: output channel size\n         - groups, base_width: ResNext나 Wide ResNet의 경우 사용\n        \"\"\"\n        super(BasicBlock, self).__init__()\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        if groups != 1 or base_width != 64:\n            raise ValueError('BasicBlock only supports groups=1 and base_width=64')\n        if dilation > 1:\n            raise NotImplementedError(\"Dilation > 1 not supported in BasicBlock\")\n            \n        # Basic Block의 구조\n        self.conv1 = conv3x3(inplanes, planes, stride)  # conv1에서 downsample\n        self.bn1 = norm_layer(planes)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = conv3x3(planes, planes)\n        self.bn2 = norm_layer(planes)\n        self.downsample = downsample\n        self.stride = stride\n\n    def forward(self, x):\n        identity = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        \n        # short connection\n        if self.downsample is not None:\n            identity = self.downsample(x)\n            \n        # identity mapping시 identity mapping후 ReLU를 적용합니다.\n        # 그 이유는, ReLU를 통과하면 양의 값만 남기 때문에 Residual의 의미가 제대로 유지되지 않기 때문입니다.\n        out += identity\n        out = self.relu(out)\n\n        return out\n    \nclass Bottleneck(nn.Module):\n    # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2)\n    # while original implementation places the stride at the first 1x1 convolution(self.conv1)\n    # according to \"Deep residual learning for image recognition\"https://arxiv.org/abs/1512.03385.\n    # This variant is also known as ResNet V1.5 and improves accuracy according to\n    # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch.\n\n    expansion = 4 # 블록 내에서 차원을 증가시키는 3번째 conv layer에서의 확장계수\n    \n    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,\n                 base_width=64, dilation=1, norm_layer=None):\n        super(Bottleneck, self).__init__()\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        # ResNext나 WideResNet의 경우 사용\n        width = int(planes * (base_width / 64.)) * groups\n        \n        # Bottleneck Block의 구조\n        self.conv1 = conv1x1(inplanes, width)\n        self.bn1 = norm_layer(width)\n        self.conv2 = conv3x3(width, width, stride, groups, dilation) # conv2에서 downsample\n        self.bn2 = norm_layer(width)\n        self.conv3 = conv1x1(width, planes * self.expansion)\n        self.bn3 = norm_layer(planes * self.expansion)\n        self.relu = nn.ReLU(inplace=True)\n        self.downsample = downsample\n        self.stride = stride\n\n    def forward(self, x):\n        identity = x\n        # 1x1 convolution layer\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n        # 3x3 convolution layer\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n        # 1x1 convolution layer\n        out = self.conv3(out)\n        out = self.bn3(out)\n        # skip connection\n        if self.downsample is not None:\n            identity = self.downsample(x)\n\n        out += identity\n        out = self.relu(out)\n\n        return out\n    \nclass ResNet(nn.Module):\n\n    def __init__(self, block, layers, num_classes=1000, zero_init_residual=False,\n                 groups=1, width_per_group=64, replace_stride_with_dilation=None,\n                 norm_layer=None):\n        super(ResNet, self).__init__()\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        self._norm_layer = norm_layer\n        # default values\n        self.inplanes = 64 # input feature map\n        self.dilation = 1\n        # stride를 dilation으로 대체할지 선택\n        if replace_stride_with_dilation is None:\n            # each element in the tuple indicates if we should replace\n            # the 2x2 stride with a dilated convolution instead\n            replace_stride_with_dilation = [False, False, False]\n        if len(replace_stride_with_dilation) != 3:\n            raise ValueError(\"replace_stride_with_dilation should be None \"\n                             \"or a 3-element tuple, got {}\".format(replace_stride_with_dilation))\n        self.groups = groups\n        self.base_width = width_per_group\n        \n        r\"\"\"\n        - 처음 입력에 적용되는 self.conv1과 self.bn1, self.relu는 모든 ResNet에서 동일 \n        - 3: 입력으로 RGB 이미지를 사용하기 때문에 convolution layer에 들어오는 input의 channel 수는 3\n        \"\"\"\n        self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False)\n        self.bn1 = norm_layer(self.inplanes)\n        self.relu = nn.ReLU(inplace=True)\n        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n        \n        r\"\"\"\n        - 아래부터 block 형태와 갯수가 ResNet층마다 변화\n        - self.layer1 ~ 4: 필터의 개수는 각 block들을 거치면서 증가(64->128->256->512)\n        - self.avgpool: 모든 block을 거친 후에는 Adaptive AvgPool2d를 적용하여 (n, 512, 1, 1)의 텐서로\n        - self.fc: 이후 fc layer를 연결\n        \"\"\"\n        self.layer1 = self._make_layer(block, 64, layers[0])\n        self.layer2 = self._make_layer(block, 128, layers[1], stride=2, # 여기서부터 downsampling적용\n                                       dilate=replace_stride_with_dilation[0])\n        self.layer3 = self._make_layer(block, 256, layers[2], stride=2,\n                                       dilate=replace_stride_with_dilation[1])\n        self.layer4 = self._make_layer(block, 512, layers[3], stride=2,\n                                       dilate=replace_stride_with_dilation[2])\n        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n        self.fc = nn.Linear(512 * block.expansion, num_classes)\n\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):\n                nn.init.constant_(m.weight, 1)\n                nn.init.constant_(m.bias, 0)\n\n        # Zero-initialize the last BN in each residual branch,\n        # so that the residual branch starts with zeros, and each residual block behaves like an identity.\n        # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677\n        if zero_init_residual:\n            for m in self.modules():\n                if isinstance(m, Bottleneck):\n                    nn.init.constant_(m.bn3.weight, 0)\n                elif isinstance(m, BasicBlock):\n                    nn.init.constant_(m.bn2.weight, 0)\n\n    def _make_layer(self, block, planes, blocks, stride=1, dilate=False):\n        r\"\"\"\n        convolution layer 생성 함수\n        - block: block종류 지정\n        - planes: feature map size (input shape)\n        - blocks: layers[0]와 같이, 해당 블록이 몇개 생성돼야하는지, 블록의 갯수 (layer 반복해서 쌓는 개수)\n        - stride와 dilate은 고정\n        \"\"\"\n        norm_layer = self._norm_layer\n        downsample = None\n        previous_dilation = self.dilation\n        if dilate:\n            self.dilation *= stride\n            stride = 1\n        \n        # the number of filters is doubled: self.inplanes와 planes 사이즈를 맞춰주기 위한 projection shortcut\n        # the feature map size is halved: stride=2로 downsampling\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            downsample = nn.Sequential(\n                conv1x1(self.inplanes, planes * block.expansion, stride),\n                norm_layer(planes * block.expansion),\n            )\n\n        layers = []\n        # 블록 내 시작 layer, downsampling 필요\n        layers.append(block(self.inplanes, planes, stride, downsample, self.groups,\n                            self.base_width, previous_dilation, norm_layer))\n        self.inplanes = planes * block.expansion # inplanes 업데이트\n        # 동일 블록 반복\n        for _ in range(1, blocks):\n            layers.append(block(self.inplanes, planes, groups=self.groups,\n                                base_width=self.base_width, dilation=self.dilation,\n                                norm_layer=norm_layer))\n\n        return nn.Sequential(*layers)\n\n    def _forward_impl(self, x):\n        # See note [TorchScript super()]\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        x = self.maxpool(x)\n\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n\n        x = self.avgpool(x)\n        x = torch.flatten(x, 1)\n        x = self.fc(x)\n\n        return x\n\n    def forward(self, x):\n        return self._forward_impl(x)","metadata":{"execution":{"iopub.status.busy":"2023-03-06T00:16:03.805966Z","iopub.execute_input":"2023-03-06T00:16:03.806408Z","iopub.status.idle":"2023-03-06T00:16:03.846814Z","shell.execute_reply.started":"2023-03-06T00:16:03.806374Z","shell.execute_reply":"2023-03-06T00:16:03.846013Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Change the last layer to binary target","metadata":{}},{"cell_type":"code","source":"import torch.nn as nn\nmodel = resnet34(pretrained = False)\nmodel.fc = nn.Sequential(nn.ReLU(),nn.Linear(in_features=512, out_features=2)) # Changed FC layer for our task","metadata":{"execution":{"iopub.status.busy":"2023-03-06T01:00:08.421233Z","iopub.execute_input":"2023-03-06T01:00:08.421612Z","iopub.status.idle":"2023-03-06T01:00:08.751985Z","shell.execute_reply.started":"2023-03-06T01:00:08.421579Z","shell.execute_reply":"2023-03-06T01:00:08.751021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def collate_fn(batch):\n    '''\n    Resize 3D Tensors to the biggest tensor in a single batch.\n    This code would resize all tensors in each batch to the tensor which have largest size (w*h).\n    This collate function is applied for batch training, which requires same size of inputs to be feeded.\n    '''\n    \n    # Find the largest width and height in the batch\n    max_width = max(dat['input_tensor'].shape[2] for dat in batch)\n    max_height = max(dat['input_tensor'].shape[1] for dat in batch)\n    channels = max(dat['input_tensor'].shape[0] for dat in batch)\n    \n    resized_batch = []\n    for dat in batch:\n        tensor = dat['input_tensor']\n        resized_tensor = torch.zeros((tensor.shape[0], max_height, max_width), dtype=tensor.dtype)\n        resized_tensor[:, :tensor.shape[1], :tensor.shape[2]] = tensor\n        resized_batch.append(resized_tensor)\n        \n    labels = [tensor['label'] for tensor in batch]\n    \n    labels = torch.stack(labels).squeeze(-1)\n    resized_batch = torch.stack(resized_batch)\n    \n    samples = {'input_tensor' : resized_batch, 'label':labels}\n\n    return samples","metadata":{"execution":{"iopub.status.busy":"2023-03-06T01:00:09.266210Z","iopub.execute_input":"2023-03-06T01:00:09.266859Z","iopub.status.idle":"2023-03-06T01:00:09.274936Z","shell.execute_reply.started":"2023-03-06T01:00:09.266824Z","shell.execute_reply":"2023-03-06T01:00:09.273834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\n\ndef evaluation(dataloader):\n    predictions = torch.tensor([], dtype=torch.float).to(device) # Tensor for prediction value appending\n    actual = torch.tensor([], dtype=torch.float).to(device) # Tensor for answer value appending\n    with torch.no_grad():\n        model.eval()\n        for data in dataloader:\n            inputs, values = data['input_tensor'].float().to(device),data['label'].to(device)\n            outputs = model(inputs).to(device)\n            predictions = torch.cat((predictions, torch.stack([torch.argmax(o) for o in outputs])),0)\n            actual = torch.cat((actual, values), 0)\n    predictions = predictions.cpu().numpy()\n    actual = actual.cpu().numpy()\n    rmse = np.sqrt(mean_squared_error(predictions, actual))\n    return rmse","metadata":{"execution":{"iopub.status.busy":"2023-03-06T01:00:09.440285Z","iopub.execute_input":"2023-03-06T01:00:09.440647Z","iopub.status.idle":"2023-03-06T01:00:09.450191Z","shell.execute_reply.started":"2023-03-06T01:00:09.440614Z","shell.execute_reply":"2023-03-06T01:00:09.449114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Defining parameters for training","metadata":{"execution":{"iopub.status.busy":"2023-02-06T06:44:25.982793Z","iopub.execute_input":"2023-02-06T06:44:25.983177Z","iopub.status.idle":"2023-02-06T06:44:25.987858Z","shell.execute_reply.started":"2023-02-06T06:44:25.983141Z","shell.execute_reply":"2023-02-06T06:44:25.986747Z"}}},{"cell_type":"code","source":"from torch import optim\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n\npqfile = '/kaggle/input/icecube-neutrinos-in-deep-ice/train_meta.parquet'\npath_batch = '/kaggle/input/icecube-neutrinos-in-deep-ice/train/'\n\nbatch_num=2 # There are 660 batches total, and the batch number should be iterated in range(660).\nlr = 1e-06\nnum_epochs = 1\nbatch_size = 16\n\nmodel = model.to(device)\noptimizer = optim.Adam(model.parameters(), lr=lr)\nloss_function = nn.BCEWithLogitsLoss().to(device)","metadata":{"execution":{"iopub.status.busy":"2023-03-06T01:00:09.761188Z","iopub.execute_input":"2023-03-06T01:00:09.763650Z","iopub.status.idle":"2023-03-06T01:00:09.802645Z","shell.execute_reply.started":"2023-03-06T01:00:09.763612Z","shell.execute_reply":"2023-03-06T01:00:09.801588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train : Validation : Test split 75% : 10% : 15%","metadata":{}},{"cell_type":"code","source":"ice_dataset = ICECUDE_Dataset(pqfile,path_batch,batch_num)\nproportions = [.75, .10, .15]\nlengths = [int(p * len(ice_dataset)) for p in proportions]\nlengths[-1] = len(ice_dataset) - sum(lengths[:-1])\ntrain_dataset, val_dataset, test_dataset = torch.utils.data.random_split(ice_dataset, lengths)\n\ntrain_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True,collate_fn=collate_fn)\nval_dataloader = DataLoader(val_dataset, batch_size=batch_size, shuffle=True,collate_fn=collate_fn)\ntest_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=True,collate_fn=collate_fn)","metadata":{"execution":{"iopub.status.busy":"2023-03-06T01:00:10.088364Z","iopub.execute_input":"2023-03-06T01:00:10.089029Z","iopub.status.idle":"2023-03-06T01:00:18.897000Z","shell.execute_reply.started":"2023-03-06T01:00:10.088993Z","shell.execute_reply":"2023-03-06T01:00:18.895998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n    'num_epochs':num_epochs,\n    'optimizer':optimizer,\n    'loss_function':loss_function,\n    'train_dataloader':train_dataloader,\n    'val_dataloader': val_dataloader,\n    'test_dataloader': test_dataloader,\n    'device':device,\n    'num_epoch' : num_epochs\n}","metadata":{"execution":{"iopub.status.busy":"2023-03-06T01:00:18.900771Z","iopub.execute_input":"2023-03-06T01:00:18.901563Z","iopub.status.idle":"2023-03-06T01:00:18.906991Z","shell.execute_reply.started":"2023-03-06T01:00:18.901506Z","shell.execute_reply":"2023-03-06T01:00:18.905973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check for inputs","metadata":{"execution":{"iopub.status.busy":"2023-02-07T03:35:12.665444Z","iopub.execute_input":"2023-02-07T03:35:12.666060Z","iopub.status.idle":"2023-02-07T03:35:12.702107Z","shell.execute_reply.started":"2023-02-07T03:35:12.665931Z","shell.execute_reply":"2023-02-07T03:35:12.700858Z"}}},{"cell_type":"code","source":"train_dataset[0]['input_tensor'].shape ","metadata":{"execution":{"iopub.status.busy":"2023-03-06T01:00:18.911334Z","iopub.execute_input":"2023-03-06T01:00:18.911941Z","iopub.status.idle":"2023-03-06T01:00:19.463567Z","shell.execute_reply.started":"2023-03-06T01:00:18.911903Z","shell.execute_reply":"2023-03-06T01:00:19.462570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset[0]['label'] # This would be 'azimuth','zenith'","metadata":{"execution":{"iopub.status.busy":"2023-03-06T01:00:19.465818Z","iopub.execute_input":"2023-03-06T01:00:19.466439Z","iopub.status.idle":"2023-03-06T01:00:19.628396Z","shell.execute_reply.started":"2023-03-06T01:00:19.466400Z","shell.execute_reply":"2023-03-06T01:00:19.627406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(model, params):\n    model.train()\n    loss_function=params[\"loss_function\"]\n    train_dataloader=params[\"train_dataloader\"]\n    val_dataloader=params[\"val_dataloader\"]\n    test_dataloader=params[\"test_dataloader\"]\n        \n    device=params[\"device\"]\n    for epoch in range(0, num_epochs):\n        with tqdm(train_dataloader,unit = 'batch') as tepoch:\n            for dat in train_dataloader:\n                tepoch.set_description(f\"Epoch {epoch}\")\n                inputs, labels = dat['input_tensor'].to(device),dat['label'].to(device)\n                optimizer.zero_grad() \n                outputs = model(inputs).to(device)\n                train_loss = loss_function(outputs.float(),labels.float())\n                train_loss = train_loss.requires_grad_(True)\n                train_loss.backward()\n                optimizer.step()\n                tepoch.set_postfix(loss=train_loss.item())\n\n    model.eval()\n    train_rmse = evaluation(train_dataloader) \n    val_rmse = evaluation(val_dataloader)\n    \n    print(\" Train Loss: %.4f, Validation Loss: %.4f\" %(train_rmse, val_rmse)) \n    import gc\n    torch.cuda.empty_cache()\n    gc.collect()\n    return(0)\n\n\ntrain(model, params)","metadata":{"execution":{"iopub.status.busy":"2023-03-06T01:00:19.630555Z","iopub.execute_input":"2023-03-06T01:00:19.631619Z","iopub.status.idle":"2023-03-06T01:16:30.914936Z","shell.execute_reply.started":"2023-03-06T01:00:19.631577Z","shell.execute_reply":"2023-03-06T01:16:30.913393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # train_rmse = evaluation(trainloader) \n# val_dataloader=params[\"val_dataloader\"]\n\n# val_rmse = evaluation(val_dataloader)\n# print(\" Train Loss: %.4f, Validation Loss: %.4f\" %(train_rmse, val_rmse)) ","metadata":{"execution":{"iopub.status.busy":"2023-02-13T05:08:33.605357Z","iopub.execute_input":"2023-02-13T05:08:33.606050Z","iopub.status.idle":"2023-02-13T05:08:33.611593Z","shell.execute_reply.started":"2023-02-13T05:08:33.606015Z","shell.execute_reply":"2023-02-13T05:08:33.609653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference | Submission","metadata":{}},{"cell_type":"code","source":"pqfile = '/kaggle/input/icecube-neutrinos-in-deep-ice/test_meta.parquet'\npath_batch = '/kaggle/input/icecube-neutrinos-in-deep-ice/test/'\nbatch_num=661\ninference_dataset = ICECUDE_Dataset(pqfile,path_batch,batch_num,'test')\ninference_dataloader = DataLoader(inference_dataset, batch_size=1)","metadata":{"execution":{"iopub.status.busy":"2023-03-06T01:16:35.320224Z","iopub.execute_input":"2023-03-06T01:16:35.320695Z","iopub.status.idle":"2023-03-06T01:16:35.344727Z","shell.execute_reply.started":"2023-03-06T01:16:35.320654Z","shell.execute_reply":"2023-03-06T01:16:35.343844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval()\noutput_lst=[]\nfor dat in inference_dataloader:\n    inputs = dat['input_tensor'].to(device)\n    outputs = model(inputs).to(device)\n    outputs = outputs.cpu().detach().squeeze().numpy()\n    outputs = outputs.tolist()\n    output_lst.append(outputs)","metadata":{"execution":{"iopub.status.busy":"2023-03-06T01:16:38.264327Z","iopub.execute_input":"2023-03-06T01:16:38.264697Z","iopub.status.idle":"2023-03-06T01:16:38.350554Z","shell.execute_reply.started":"2023-03-06T01:16:38.264665Z","shell.execute_reply":"2023-03-06T01:16:38.349665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub = pq.read_table('/kaggle/input/icecube-neutrinos-in-deep-ice/sample_submission.parquet').to_pandas()\nbatch661 = pq.read_table('/kaggle/input/icecube-neutrinos-in-deep-ice/test/batch_661.parquet').to_pandas()","metadata":{"execution":{"iopub.status.busy":"2023-03-06T01:16:40.362295Z","iopub.execute_input":"2023-03-06T01:16:40.362686Z","iopub.status.idle":"2023-03-06T01:16:40.379854Z","shell.execute_reply.started":"2023-03-06T01:16:40.362654Z","shell.execute_reply":"2023-03-06T01:16:40.378920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub","metadata":{"execution":{"iopub.status.busy":"2023-03-06T01:16:40.945334Z","iopub.execute_input":"2023-03-06T01:16:40.945748Z","iopub.status.idle":"2023-03-06T01:16:40.956727Z","shell.execute_reply.started":"2023-03-06T01:16:40.945712Z","shell.execute_reply":"2023-03-06T01:16:40.955604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame(output_lst)\nsubmission.index = batch661.index.unique().tolist()\nsubmission.reset_index(inplace = True)\nsubmission.columns = ['event_id','azimuth','zenith']","metadata":{"execution":{"iopub.status.busy":"2023-03-06T01:16:41.831344Z","iopub.execute_input":"2023-03-06T01:16:41.831717Z","iopub.status.idle":"2023-03-06T01:16:41.838841Z","shell.execute_reply.started":"2023-03-06T01:16:41.831684Z","shell.execute_reply":"2023-03-06T01:16:41.837877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('sumbission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-03-06T01:16:43.922118Z","iopub.execute_input":"2023-03-06T01:16:43.923273Z","iopub.status.idle":"2023-03-06T01:16:43.932580Z","shell.execute_reply.started":"2023-03-06T01:16:43.923222Z","shell.execute_reply":"2023-03-06T01:16:43.931562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2023-03-06T01:16:45.877660Z","iopub.execute_input":"2023-03-06T01:16:45.878552Z","iopub.status.idle":"2023-03-06T01:16:45.890468Z","shell.execute_reply.started":"2023-03-06T01:16:45.878489Z","shell.execute_reply":"2023-03-06T01:16:45.889489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# If you like my notebook, please upvote for it! ","metadata":{}}]}