{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":14461,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":11986},{"sourceId":16406,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":13670},{"sourceId":16461,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":13712},{"sourceId":16462,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":13713}],"dockerImageVersionId":30665,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nimport time\n\nfrom IPython.display import clear_output\nfrom tqdm import tqdm\nfrom tqdm.contrib import tzip\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport umap\n\nfrom sklearn.metrics.pairwise import euclidean_distances\nfrom sklearn.model_selection import train_test_split\n\nimport torch\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.transforms import Resize\nimport matplotlib.pyplot as plt\nimport torch.nn.functional as F\n\n\n# import albumentations as A\nimport albumentations as albu\nfrom albumentations.pytorch import ToTensorV2","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:44:03.552329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/train.csv\")\nTARGETS = train_df.columns[-6:]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/test.csv')\nprint('Test shape',test.shape)\ntest.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ ALL SPECTROGRAMS\nPATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/'\nfiles2 = os.listdir(PATH2)\nprint(f'There are {len(files2)} test spectrogram parquets')\n    \nspectrograms2 = {}\nfor i, f in enumerate(files2):\n    if i % 100 == 0:\n        print(i, ', ',end='')\n    tmp = pd.read_parquet(f'{PATH2}{f}')\n    name = int(f.split('.')[0])\n    spectrograms2[name] = tmp.iloc[:, 1:].values\n    \n# RENAME FOR DATALOADER\ntest = test.rename({'spectrogram_id': 'spec_id'}, axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pywt, librosa\n\nUSE_WAVELET = None \n\nNAMES = ['LL','LP','RP','RR']\n\nFEATS = [['Fp1','F7','T3','T5','O1'],\n         ['Fp1','F3','C3','P3','O1'],\n         ['Fp2','F8','T4','T6','O2'],\n         ['Fp2','F4','C4','P4','O2']]\n\ndirectory_path = 'EEG_Spectrograms/'\nif not os.path.exists(directory_path):\n    os.makedirs(directory_path)\n    \n# DENOISE FUNCTION\ndef maddest(d, axis=None):\n    return np.mean(np.absolute(d - np.mean(d, axis)), axis)\n\ndef denoise(x, wavelet='haar', level=1):    \n    coeff = pywt.wavedec(x, wavelet, mode=\"per\")\n    sigma = (1/0.6745) * maddest(coeff[-level])\n\n    uthresh = sigma * np.sqrt(2*np.log(len(x)))\n    coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])\n\n    ret=pywt.waverec(coeff, wavelet, mode='per')\n    \n    return ret\n\n\ndef spectrogram_from_eeg(parquet_path, display=False):\n    \n    # LOAD MIDDLE 50 SECONDS OF EEG SERIES\n    eeg = pd.read_parquet(parquet_path)\n    middle = (len(eeg)-10_000)//2\n    eeg = eeg.iloc[middle:middle+10_000]\n    \n    # VARIABLE TO HOLD SPECTROGRAM\n    img = np.zeros((128,256,4),dtype='float32')\n    \n    if display: plt.figure(figsize=(10,7))\n    signals = []\n    for k in range(4):\n        COLS = FEATS[k]\n        \n        for kk in range(4):\n        \n            # COMPUTE PAIR DIFFERENCES\n            x = eeg[COLS[kk]].values - eeg[COLS[kk+1]].values\n\n            # FILL NANS\n            m = np.nanmean(x)\n            if np.isnan(x).mean()<1: x = np.nan_to_num(x,nan=m)\n            else: x[:] = 0\n\n            # DENOISE\n            if USE_WAVELET:\n                x = denoise(x, wavelet=USE_WAVELET)\n            signals.append(x)\n\n            # RAW SPECTROGRAM\n            mel_spec = librosa.feature.melspectrogram(y=x, sr=200, hop_length=len(x)//256, \n                  n_fft=1024, n_mels=128, fmin=0, fmax=20, win_length=128)\n\n            # LOG TRANSFORM\n            width = (mel_spec.shape[1]//32)*32\n            mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max).astype(np.float32)[:,:width]\n\n            # STANDARDIZE TO -1 TO 1\n            mel_spec_db = (mel_spec_db+40)/40 \n            img[:,:,k] += mel_spec_db\n                \n        # AVERAGE THE 4 MONTAGE DIFFERENCES\n        img[:,:,k] /= 4.0\n        \n        if display:\n            plt.subplot(2,2,k+1)\n            plt.imshow(img[:,:,k],aspect='auto',origin='lower')\n            plt.title(f'EEG {eeg_id} - Spectrogram {NAMES[k]}')\n            \n    if display: \n        plt.show()\n        plt.figure(figsize=(10,5))\n        offset = 0\n        for k in range(4):\n            if k>0: offset -= signals[3-k].min()\n            plt.plot(range(10_000),signals[k]+offset,label=NAMES[3-k])\n            offset += signals[3-k].max()\n        plt.legend()\n        plt.title(f'EEG {eeg_id} Signals')\n        plt.show()\n        print(); print('#'*25); print()\n        \n    return img","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ ALL EEG SPECTROGRAMS\nPATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/'\nDISPLAY = 1\nEEG_IDS2 = test.eeg_id.unique()\nall_eegs2 = {}\n\nprint('Converting Test EEG to Spectrograms...'); print()\nfor i, eeg_id in enumerate(EEG_IDS2):\n        \n    # CREATE SPECTROGRAM FROM EEG PARQUET\n    img = spectrogram_from_eeg(f'{PATH2}{eeg_id}.parquet', i < DISPLAY)\n    all_eegs2[eeg_id] = img\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TARS = {'Seizure':0, 'LPD':1, 'GPD':2, 'LRDA':3, 'GRDA':4, 'Other':5}\nTARS2 = {x: y for y, x in TARS.items()}\n\n\nclass EEGDataset(Dataset):\n    \n#     def __init__(self, data, augment=False, mode='train', specs=spectrograms, eeg_specs=all_eegs): \n    def __init__(self, data, augment=False, mode='train', eeg_specs=all_eegs2): \n        self.data = data\n        self.augment = augment\n        self.mode = mode\n#         self.specs = specs\n        self.eeg_specs = eeg_specs\n        \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, index):\n        return self.__getitems__([index])\n    \n    def __getitems__(self, indices):\n        X, y = self._generate_data(indices)\n        if self.augment:\n            X = self.__augment(X) \n        if self.mode == 'train':\n            return list(zip(X, y))\n        else:\n            return X\n    \n#     def _generate_data(self, indexes):\n#         X = np.zeros((len(indexes), 128, 256, 4),dtype='float32')\n#         y = np.zeros((len(indexes), 6),dtype='float32')\n#         img = np.ones((128, 256),dtype='float32')\n        \n#         for j, i in enumerate(indexes):\n#             row = self.data.iloc[i]\n        \n#             # EEG SPECTROGRAMS\n#             img = self.eeg_specs[row.eeg_id]\n#             X[j, :, :, :] = img\n                \n#             if self.mode != 'test':\n#                 y[j,] = row[TARGETS]\n            \n#         return X, y\n    def _generate_data(self, indexes):\n        X = np.zeros((len(indexes), 4, 128, 256), dtype='float32')  # Adjusted the shape here\n        y = np.zeros((len(indexes), 6), dtype='float32')\n        \n        for j, i in enumerate(indexes):\n            row = self.data.iloc[i]\n        \n            # EEG SPECTROGRAMS\n            img = self.eeg_specs[row.eeg_id]\n            img = np.transpose(img, (2, 0, 1))  # Adjusted the order here to match (channels, height, width)\n            X[j] = img\n                \n            if self.mode != 'test':\n                # Assuming you have a correct way to set y[j,] based on your labels\n                y[j,] = row[TARGETS]  # Make sure TARGETS is correctly defined elsewhere\n            \n#         return torch.tensor(X), torch.tensor(y)\n        return X, y\n    \n    def _random_transform(self, img):\n        composition = albu.Compose([\n            albu.HorizontalFlip(p=0.5),\n            # albu.CoarseDropout(max_holes=8,max_height=32,max_width=32,fill_value=0,p=0.5),\n        ])\n        return composition(image=img)['image']\n            \n    def __augment(self, img_batch):\n        for i in range(img_batch.shape[0]):\n            img_batch[i,] = self._random_transform(img_batch[i,])\n        return img_batch","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomUpsample(nn.Module):\n    def __init__(self, scale_factor=(2, 2)):\n        super().__init__()\n        self.scale_factor = scale_factor\n\n    def forward(self, x):\n        return F.interpolate(x, scale_factor=self.scale_factor, mode='nearest')\n\n\nclass ResNetBlock(nn.Module):\n    def __init__(self, in_channels, kernel_size, modify=False, bn=True, scale_factor=(1, 1)):\n        super().__init__()\n        self.modify = modify\n        if modify=='downsample':\n            self.conv1 = nn.Conv2d(in_channels=in_channels, out_channels=in_channels*2, stride=2, kernel_size=kernel_size, padding=kernel_size//2, bias=False)\n            self.conv2 = nn.Conv2d(in_channels=in_channels*2, out_channels=in_channels*2, kernel_size=kernel_size, padding=kernel_size//2,bias=False)\n            if bn:\n                self.bn1 = nn.BatchNorm2d(in_channels*2)\n                self.bn2 = nn.BatchNorm2d(in_channels*2)\n            else:\n                self.bn1 = nn.Identity()\n                self.bn2 = nn.Identity()\n                \n        elif modify=='upsample':\n#             self.conv1 = CustomUpsample(scale_factor=scale_factor)\n            self.conv1 = nn.ConvTranspose2d(in_channels=in_channels, out_channels=in_channels//2, stride=2, kernel_size=kernel_size, output_padding=scale_factor, padding=kernel_size//2, bias=False)\n            self.conv2 = nn.Conv2d(in_channels=in_channels//2, out_channels=in_channels//2, kernel_size=kernel_size, padding=kernel_size//2, bias=False)\n            self.bn1 = nn.BatchNorm2d(in_channels//2)\n            self.bn2 = nn.BatchNorm2d(in_channels//2)\n        else:\n            self.conv1 = nn.Conv2d(in_channels=in_channels, out_channels=in_channels, kernel_size=kernel_size, padding=kernel_size//2)\n            self.conv2 = nn.Conv2d(in_channels=in_channels, out_channels=in_channels, kernel_size=kernel_size, padding=kernel_size//2)\n            self.bn1 = nn.BatchNorm2d(in_channels)\n            self.bn2 = nn.BatchNorm2d(in_channels)\n        self.act = nn.ReLU()\n        \n        if modify=='downsample':\n            self.proj = nn.Conv2d(in_channels=in_channels, out_channels=in_channels*2, stride=2, kernel_size=kernel_size, padding=kernel_size//2)\n        if modify=='upsample':\n            self.proj = nn.ConvTranspose2d(in_channels=in_channels, out_channels=in_channels//2, stride=2, kernel_size=kernel_size, output_padding=scale_factor, padding=kernel_size//2)\n\n\n    def forward(self, x):\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.act(out)\n        out = self.conv2(out)\n        out = self.bn2(out)\n        if self.modify:\n            x = self.proj(x)\n        out = x + out\n        out = self.act(out)\n        return out\n\n    \nclass Encoder(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.conv = nn.Conv2d(4, 16, 7, 1, 7//2)\n#         2x specs\n#         self.conv = nn.Conv2d(8, 16, 7, 1, 7//2)\n        self.rnb1 = ResNetBlock(16, 3, modify='downsample')\n        self.rnb2 = ResNetBlock(32, 3, modify='downsample')\n        self.rnb3 = ResNetBlock(64, 3, modify='downsample')\n        self.rnb4 = ResNetBlock(128, 3, modify='downsample')\n        self.rnb5 = ResNetBlock(256, 3, modify='downsample')\n        self.rnb6 = ResNetBlock(512, 3, modify='downsample')\n        self.rnb7 = ResNetBlock(1024, 3, modify='downsample')\n        self.rnb8 = ResNetBlock(2048, 3, modify='downsample')\n        \n    def forward(self, x):\n#         print(f\"Input to Encoder: {x.size()}\")\n        x = self.conv(x)\n#         print(f\"After conv: {x.size()}\")\n        x = self.rnb1(x)\n#         print(f\"After rnb1: {x.size()}\")\n        x = self.rnb2(x)\n#         print(f\"After rnb2: {x.size()}\")\n        x = self.rnb3(x)\n#         print(f\"After rnb3: {x.size()}\")\n        x = self.rnb4(x)\n#         print(f\"After rnb4: {x.size()}\")\n        x = self.rnb5(x)\n#         print(f\"After rnb5: {x.size()}\")\n        x = self.rnb6(x)\n#         print(f\"After rnb6: {x.size()}\")\n        x = self.rnb7(x)\n#         print(f\"After rnb7: {x.size()}\")\n        x = self.rnb8(x)\n#         print(f\"After rnb8: {x.size()}\")\n        return x\n    \nclass Decoder(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.rnb1 = ResNetBlock(4096, 3, modify='upsample', scale_factor=(0, 1))\n        self.rnb2 = ResNetBlock(2048, 3, modify='upsample')\n        self.rnb3 = ResNetBlock(1024, 3, modify='upsample')\n        self.rnb4 = ResNetBlock(512, 3, modify='upsample')\n        self.rnb5 = ResNetBlock(256, 3, modify='upsample')\n        self.rnb6 = ResNetBlock(128, 3, modify='upsample')\n        self.rnb7 = ResNetBlock(64, 3, modify='upsample')\n        self.rnb8 = ResNetBlock(32, 3, modify='upsample')\n#         self.rnb9 = ResNetBlock(16, 3, modify='upsample')\n        self.conv = nn.Conv2d(16, 4, 3, 1, 3//2)\n\n    def forward(self, x):\n#         print(f\"Input to Decoder: {x.size()}\")\n        x = self.rnb1(x)\n#         print(f\"After rnb1: {x.size()}\")\n        x = self.rnb2(x)\n#         print(f\"After rnb2: {x.size()}\")\n        x = self.rnb3(x)\n#         print(f\"After rnb3: {x.size()}\")\n        x = self.rnb4(x)\n#         print(f\"After rnb4: {x.size()}\")\n        x = self.rnb5(x)\n#         print(f\"After rnb5: {x.size()}\")\n        x = self.rnb6(x)\n#         print(f\"After rnb6: {x.size()}\")\n        x = self.rnb7(x)\n#         print(f\"After rnb7: {x.size()}\")\n        x = self.rnb8(x)\n#         print(f\"After rnb8: {x.size()}\")\n#         x = self.rnb9(x)\n        x = self.conv(x)\n#         print(f\"After final conv: {x.size()}\")\n        return x","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SimpleAE(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.net = nn.Sequential(\n            Encoder(),\n            Decoder()\n        )\n        \n    def forward(self, x):\n        return self.net(x)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SimpleNet2(nn.Module):\n    def __init__(self):\n        super(SimpleNet2, self).__init__()\n        self.fc1 = nn.Linear(4096, 1024)  # First hidden layer\n        self.dropout1 = nn.Dropout(0.5)\n        self.fc2 = nn.Linear(1024, 512)   # Second hidden layer\n#         self.dropout2 = nn.Dropout(0.5)\n        self.fc3 = nn.Linear(512, 128)    # Third hidden layer\n        self.output = nn.Linear(128, 6)   # Output layer\n        \n    def forward(self, x):\n        x = F.relu(self.fc1(x))\n        x = self.dropout1(x)\n        x = F.relu(self.fc2(x))\n#         x = self.dropout2(x)\n        x = F.relu(self.fc3(x))\n        x = self.output(x)\n        return x\n    \n\nclass SimpleNet1(nn.Module):\n    def __init__(self):\n        super(SimpleNet1, self).__init__()\n        self.fc1 = nn.Linear(4096, 1024)  # First hidden layer\n        self.dropout1 = nn.Dropout(0.5)\n        self.fc2 = nn.Linear(1024, 128)   # Second hidden layer\n#         self.dropout2 = nn.Dropout(0.5)\n#         self.fc3 = nn.Linear(512, 128)    # Third hidden layer\n        self.output = nn.Linear(128, 6)   # Output layer\n        \n    def forward(self, x):\n        x = F.relu(self.fc1(x))\n        x = self.dropout1(x)\n        x = F.relu(self.fc2(x))\n#         x = self.dropout2(x)\n#         x = F.relu(self.fc3(x))\n        x = self.output(x)\n        return x\n    \nclass ShitNet(nn.Module):\n    def __init__(self):\n        super(ShitNet, self).__init__()\n        self.fc1 = nn.Linear(4096, 1024)  # First hidden layer\n#         self.dropout1 = nn.Dropout(0.5)\n        self.fc2 = nn.Linear(1024, 512)   # Second hidden layer\n#         self.dropout2 = nn.Dropout(0.5)\n        self.fc3 = nn.Linear(512, 128)    # Third hidden layer\n        self.output = nn.Linear(128, 6)   # Output layer\n        \n    def forward(self, x):\n        x = F.relu(self.fc1(x))\n#         x = self.dropout1(x)\n        x = F.relu(self.fc2(x))\n#         x = self.dropout2(x)\n        x = F.relu(self.fc3(x))\n        x = self.output(x)\n        return x\n    \n\nclass TransformerModel(nn.Module):\n    def __init__(self, input_dim=4096, d_model=512, nhead=8, num_encoder_layers=3, dim_feedforward=2048):\n        super(TransformerModel, self).__init__()\n        \n        # Correctly use d_model from the argument\n        self.encoder = nn.Linear(input_dim, d_model)\n        encoder_layers = nn.TransformerEncoderLayer(d_model=d_model, nhead=nhead, dim_feedforward=dim_feedforward)\n        self.transformer_encoder = nn.TransformerEncoder(encoder_layer=encoder_layers, num_layers=num_encoder_layers)\n        \n#         self.output_layer = nn.Linear(d_model, 7)\n        self.output_layer = nn.Linear(d_model, 6)\n        \n    def forward(self, src):\n        # src is expected to be of shape [batch_size, input_dim]\n        # We first project src to the d_model dimension\n        src = self.encoder(src)\n        \n        # Adjust src to have shape [seq_len, batch_size, d_model] for the transformer\n        # For simplicity, treat the whole input as a sequence of length 1\n        src = src.unsqueeze(1)  # [batch_size, 1, d_model]\n        \n        # TransformerEncoder requires input of shape [seq_len, batch_size, d_model]\n        output = self.transformer_encoder(src)\n        \n        # Now, output is of shape [seq_len, batch_size, d_model], we remove the seq_len dimension\n        output = output.squeeze(1)  # [batch_size, d_model]\n        \n        output = self.output_layer(output)\n        return output","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# INFER EFFICIENTNET ON TEST\npreds = []\n# test_ds = EEGDataset(test, mode='test', specs=spectrograms2, eeg_specs=all_eegs2)\ntest_ds = EEGDataset(test, mode='test', eeg_specs=all_eegs2)\ntest_loader = DataLoader(test_ds, shuffle=False, batch_size=64, num_workers=3)\n\nfor i in range(5):\n    print('#'*25)\n    print(f'### Testing Fold {i+1}')\n\n#     ckpt_file = f'/kaggle/input/best_model.pth/pytorch/hms-autoencoder/1/best_model.pth'\n    \n    model_ae = SimpleAE()\n    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n    model_ae= nn.DataParallel(model_ae)\n    model_ae.load_state_dict(torch.load('/kaggle/input/best_model.pth/pytorch/hms-autoencoder/1/best_model.pth'))\n    model_ae.to(device).eval()\n    encoder = nn.DataParallel(model_ae.module.net[0])\n    encoder.eval()\n    \n#     ckpt_file = f'/kaggle/input/simplenet1/pytorch/1/1/model_simplenet_20_e4.pth'\n    model = ShitNet()\n    model.load_state_dict(torch.load('/kaggle/input/shitmodel/pytorch/1/1/model_shitnet1.pth'))\n    model.to(device).eval()\n    \n    \n    fold_preds = []\n\n    with torch.inference_mode():\n        for test_batch in test_loader:\n            test_batch = test_batch.to(device)\n            emb = encoder(test_batch)\n            embs = emb.squeeze(2, 3)\n            embs = embs.to(device)\n#             print(emb.shape)\n#             print(embs.shape)\n#             embs.append(emb.squeeze(2, 3))\n            pred = model(embs)\n#             print(pred)\n            # Apply softmax to the first 6 outputs to get probabilities\n            probabilities = F.softmax(pred, dim=1)\n#             continuous_values = pred\n#             print(probabilities)\n            \n#             fold_preds.append(pred.cpu().numpy())\n            fold_preds.append(probabilities.cpu().numpy())\n        fold_preds = np.concatenate(fold_preds)\n        \n    preds.append(fold_preds)\n\npred = np.mean(preds,axis=0)\nprint()\nprint('Test preds shape',pred.shape)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.DataFrame({'eeg_id': test.eeg_id.values})\nsub[TARGETS] = pred\nsub.to_csv('submission.csv',index=False)\nprint('Submissionn shape',sub.shape)\nsub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SANITY CHECK TO CONFIRM PREDICTIONS SUM TO ONE\nsub.iloc[:,-6:].sum(axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}