{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"dockerImageVersionId":30664,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Continuous Seizure Detection Based on Transformer and Long-Term iEEG\n\nYulin Sun, Weipeng Jin, Xiaopeng Si, Xingjian Zhang, Jiale Cao, Le Wang, Shaoya Yin, Dong Ming\n\nhttps://ieeexplore.ieee.org/document/9858598\n\nAbstract:\n\nAutomatic seizure detection algorithms are necessary for patients with refractory epilepsy. Many excellent algorithms have achieved good results in seizure detection. Still, most of them are based on discontinuous intracranial electroencephalogram (iEEG) and ignore the impact of different channels on detection. This study aimed to evaluate the proposed algorithm using continuous, long-term iEEG to show its applicability in clinical routine. In this study, we introduced the ability of the transformer network to calculate the attention between the channels of input signals into seizure detection. We proposed an end-to-end model that included convolution and transformer layers. The model did not need feature engineering or format transformation of the original multi-channel time series. Through evaluation on two datasets, we demonstrated experimentally that the transformer layer could improve the performance of the seizure detection algorithm. For the SWEC-ETHZ iEEG dataset, we achieved 97.5% event-based sensitivity, 0.06/h FDR, and 13.7 s latency. For the TJU-HH iEEG dataset, we achieved 98.1% event-based sensitivity, 0.22/h FDR, and 9.9 s latency. In addition, statistics showed that the model allocated more attention to the channels close to the seizure onset zone within 20 s after the seizure onset, which improved the explainability of the model. This paper provides a new method to improve the performance and explainability of automatic seizure detection.","metadata":{}},{"cell_type":"code","source":"# From the discussion\n# https://www.kaggle.com/competitions/hms-harmful-brain-activity-classification/discussion/481700#2681650\n# From the work of\n# https://www.kaggle.com/sunyuri\nimport numpy as np\nimport torch\nimport torch.nn as nn\n\nchannels = 19# The ones from eeg: 19 + 1 raw, 2 + 2 + 2 + 2 Chris Deotte, 4 + 4 + 4 + 4 + 2 + 1 example figures...\nBS = 32# Just batch size\nW = 2048# Width of the eeg\nin_channels = 1# Each eeg channel is preprocessed separately\nhidden_size = 128# The dimension of the Transformer transformations: Key, Query and OUT\nnum_heads = 4# Number of Attention Heads\nnum_blocks = 1# Number of Transformer blocks\npooling_type = 'avg'# The pooling applied before classification layer: 'avg','max' or 'cls'\ndropout_rate = .2# Drop rate\n\ndef positional_encoding(length, embed_dim):\n\n    dim = embed_dim // 2\n\n    position = np.arange(length)[:, np.newaxis]\n    dim = np.arange(dim)[np.newaxis, :] / dim\n\n    angle = 1 / (10000 ** dim)\n    angle = position * angle\n\n    pos_embed = np.concatenate([np.sin(angle), np.cos(angle)], axis=-1)\n    pos_embed = torch.from_numpy(pos_embed).float()\n\n    return pos_embed\n\n\nclass ConvBlock(nn.Module):\n\n    def __init__(self, in_channels, out_channels):\n\n        super(ConvBlock, self).__init__()\n\n        self.conv1 = nn.Conv2d(in_channels=in_channels, out_channels=32, kernel_size=(4, 1), stride=(2, 1), padding=(1, 0), padding_mode='zeros', bias=True)\n        self.bn1 = nn.BatchNorm2d(num_features=32)\n        self.conv2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=(4, 1), stride=(2, 1), padding=(1, 0), padding_mode='zeros', bias=True)\n        self.bn2 = nn.BatchNorm2d(num_features=64)\n        self.conv3 = nn.Conv2d(in_channels=64, out_channels=out_channels, kernel_size=(4, 1), stride=(2, 1), padding=(1, 0), padding_mode='zeros', bias=True)\n        self.activation = nn.LeakyReLU(inplace=False)\n\n    def forward(self, x):\n\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.activation(x)\n\n        x = self.conv2(x)\n        x = self.bn2(x)\n        x = self.activation(x)\n\n        x = self.conv3(x)\n\n        return x\n\n\nclass MLP(nn.Module):\n\n    def __init__(self, embed_dim, hidden_dim):\n\n        super(MLP, self).__init__()\n\n        self.mlp = nn.Sequential(\n            nn.Linear(in_features=embed_dim, out_features=hidden_dim),\n            nn.ReLU(inplace=True),\n            nn.Linear(in_features=hidden_dim, out_features=embed_dim),\n        )\n\n    def forward(self, x):\n        return self.mlp(x)\n\n\nclass TransformerBlock(nn.Module):\n\n    def __init__(self, embed_dim, num_heads, out_dim):\n        super(TransformerBlock, self).__init__()\n\n        self.attention = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=num_heads, batch_first=True)\n        self.mlp = MLP(embed_dim=embed_dim, hidden_dim=out_dim)\n        self.ln1 = nn.LayerNorm(embed_dim)\n        self.ln2 = nn.LayerNorm(out_dim)\n\n    def forward(self, x):\n\n        x = self.ln1(x)\n        x = self.attention(x, x, x)[0]\n        x = self.ln2(x)\n        x = self.mlp(x)\n\n        return x\n\n\nclass HybridTransformer(nn.Module):\n\n    def __init__(self, in_channels, hidden_size, num_heads, num_blocks, pooling_type, dropout_rate, *head_args):\n\n        super(HybridTransformer, self).__init__()\n\n        self.conv_block = ConvBlock(in_channels=1, out_channels=hidden_size)\n        self.encoder = nn.ModuleList([\n            TransformerBlock(\n                embed_dim=hidden_size,\n                num_heads=num_heads,\n                out_dim=hidden_size,\n            ) for _ in range(num_blocks)\n        ])\n\n        self.positional_embeddings = torch.nn.Parameter(positional_encoding(channels, hidden_size))\n        self.cls_token = nn.Parameter(torch.zeros((1, hidden_size)))\n\n        self.pooling_type = pooling_type\n        self.dropout = nn.Dropout(dropout_rate) if dropout_rate > 0 else nn.Identity()\n#--------------------------------------------------------------------------------------------------\n        self.head = nn.Linear(hidden_size,6)#ClassificationHead(input_dimensions=hidden_size, **head_args)\n#--------------------------------------------------------------------------------------------------\n    def forward(self, x):\n\n        # Add channel dimension and pass it to conv 2d block\n        x = x.unsqueeze(dim=1).permute(0, 1, 3, 2)\n        x = self.conv_block(x)\n\n        # Average features along time dimension\n        x = torch.mean(x, dim=2).permute(0, 2, 1)\n\n        # Add positional embeddings and concatenate cls token\n        x += self.positional_embeddings\n        x = torch.cat([\n            self.cls_token.unsqueeze(0).repeat(x.size(0), 1, 1),\n            x\n        ], 1)\n\n        # Pass it to transformer encoder\n        for block in self.encoder:\n            x = block(x)\n\n        if self.pooling_type == 'avg':\n            x = torch.mean(x, dim=1)\n        elif self.pooling_type == 'max':\n            x = torch.max(x, dim=1)[0]\n        elif self.pooling_type == 'cls':\n            x = x[:, 0, :]\n\n        x = self.dropout(x)\n        output = self.head(x)\n\n        return output","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HybridTransformer(in_channels,hidden_size,num_heads,num_blocks,pooling_type,dropout_rate)(torch.rand(BS,channels,W)).shape","metadata":{"execution":{"iopub.status.busy":"2024-03-04T20:23:16.897611Z","iopub.execute_input":"2024-03-04T20:23:16.898697Z","iopub.status.idle":"2024-03-04T20:23:17.618736Z","shell.execute_reply.started":"2024-03-04T20:23:16.898650Z","shell.execute_reply":"2024-03-04T20:23:17.617577Z"},"trusted":true},"execution_count":null,"outputs":[]}]}