{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7392733,"sourceType":"datasetVersion","datasetId":4297749},{"sourceId":164058829,"sourceType":"kernelVersion"}],"dockerImageVersionId":30646,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# About this notebook\n\nThis is the inference of this model trained [here](https://www.kaggle.com/code/medali1992/hms-resnet50-spark-train).\n\n## Version 1\n\n### Hyperparams\n\n\n```\n scheduler='OneCycleLR' \n  print_freq=50\n  num_workers = 1\n  model_name = 'resnet50'\n  optimizer='Adam'\n  stage1_epochs = 10\n  stage1_epochs = 20\n  eps = 1e-6\n  lr = 1e-3\n  batch_size = 64\n  weight_decay = 1e-2\n  seed = 2024\n```\n\n## Version2\n\n* I changed the CV sheme, first stage train on all data second stage train on data with total_evaluators >= 10\n* Used data augmentation for training\n\n### Hyperparams\n\n```\n  scheduler='OneCycleLR' \n  print_freq=50\n  num_workers = 1\n  model_name = 'resnet50'\n  optimizer='Adam'\n  stage1_epochs = 8\n  stage1_epochs = 8\n  eps = 1e-6\n  lr = 1e-3\n  batch_size = 64\n  weight_decay = 1e-2\n  seed = 2024\n```","metadata":{}},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nfrom glob import glob\nimport sys\nimport math\nimport time\nimport random\nimport shutil\nfrom pathlib import Path\nfrom typing import Dict, List\nfrom scipy.stats import entropy\nfrom scipy.signal import butter, lfilter, freqz\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\nsys.path.append('/kaggle/input/kaggle-kl-div')\nfrom kaggle_kl_div import score\nimport numpy as np\nimport pandas as pd\nfrom sklearn import preprocessing\nfrom sklearn.metrics import accuracy_score, log_loss\nfrom tqdm.auto import tqdm\nfrom functools import partial\nimport cv2\nfrom PIL import Image\nimport torch\nimport torch.nn as nn\nimport pytorch_lightning as pl\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD, AdamW\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau, OneCycleLR, CosineAnnealingLR, CosineAnnealingWarmRestarts\nfrom sklearn.preprocessing import LabelEncoder\nfrom torchvision.transforms import v2\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.model_selection import train_test_split\nimport albumentations as A\nfrom albumentations import (Compose, Normalize, Resize, RandomResizedCrop, HorizontalFlip, VerticalFlip, ShiftScaleRotate, Transpose)\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\nimport timm\nimport warnings \nwarnings.filterwarnings('ignore')\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nfrom matplotlib import pyplot as plt\nimport joblib\nos.environ['CUDA_VISIBLE_DEVICES'] = \"0,1\"\nVERSION=2","metadata":{"execution":{"iopub.status.busy":"2024-02-29T08:45:45.237330Z","iopub.execute_input":"2024-02-29T08:45:45.237717Z","iopub.status.idle":"2024-02-29T08:46:01.037028Z","shell.execute_reply.started":"2024-02-29T08:45:45.237685Z","shell.execute_reply":"2024-02-29T08:46:01.035826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"class CFG:\n    PATH = '/kaggle/input/hms-harmful-brain-activity-classification/'\n    test_eeg = \"/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/\"\n    test_csv = \"/kaggle/input/hms-harmful-brain-activity-classification/test.csv\"\n    test_spectrograms = \"/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/\"\n    model_name = 'resnet50'\n    SparK = False\n    FREEZE = False\n    seed = 2024\n    in_channels = 8\n    target_size = 6\n    batch_size = 32\n    num_workers = 1\n\n    \nmodel_weights = [x for x in glob(\"/kaggle/input/hms-resnet50-spark-train/pop_2_weight_oof/*.pth\")]\nmodel_weights","metadata":{"execution":{"iopub.status.busy":"2024-02-29T08:46:01.039034Z","iopub.execute_input":"2024-02-29T08:46:01.039372Z","iopub.status.idle":"2024-02-29T08:46:01.054650Z","shell.execute_reply.started":"2024-02-29T08:46:01.039344Z","shell.execute_reply":"2024-02-29T08:46:01.053539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils ","metadata":{}},{"cell_type":"code","source":"import pywt, librosa\n\nUSE_WAVELET = None \n\nNAMES = ['LL','LP','RP','RR']\n\nFEATS = [['Fp1','F7','T3','T5','O1'],\n         ['Fp1','F3','C3','P3','O1'],\n         ['Fp2','F8','T4','T6','O2'],\n         ['Fp2','F4','C4','P4','O2']]\n\n# DENOISE FUNCTION\ndef maddest(d, axis=None):\n    return np.mean(np.absolute(d - np.mean(d, axis)), axis)\n\ndef denoise(x, wavelet='haar', level=1):    \n    coeff = pywt.wavedec(x, wavelet, mode=\"per\")\n    sigma = (1/0.6745) * maddest(coeff[-level])\n\n    uthresh = sigma * np.sqrt(2*np.log(len(x)))\n    coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])\n\n    ret=pywt.waverec(coeff, wavelet, mode='per')\n    \n    return ret\n\ndef spectrogram_from_eeg(parquet_path, display=False, offset=None):\n    \n    # LOAD MIDDLE 50 SECONDS OF EEG SERIES\n    eeg = pd.read_parquet(parquet_path)\n#     print(eeg.shape)\n    if offset is None:\n        middle = (len(eeg)-10_000)//2\n        eeg = eeg.iloc[middle:middle+10_000]\n    else:\n        eeg = eeg.iloc[offset:offset+10_000]\n    \n    # VARIABLE TO HOLD SPECTROGRAM\n    img = np.zeros((128,256,4),dtype='float32')\n    \n    if display: plt.figure(figsize=(10,7))\n    signals = []\n    for k in range(4):\n        COLS = FEATS[k]\n        \n        for kk in range(4):\n        \n            # COMPUTE PAIR DIFFERENCES\n            x = eeg[COLS[kk]].values - eeg[COLS[kk+1]].values\n\n            # FILL NANS\n            m = np.nanmean(x)\n            if np.isnan(x).mean() < 1: \n                x = np.nan_to_num(x,nan=m)\n            else: x[:] = 0\n\n            # DENOISE\n            if USE_WAVELET:\n                x = denoise(x, wavelet=USE_WAVELET)\n            signals.append(x)\n\n            # RAW SPECTROGRAM\n            mel_spec = librosa.feature.melspectrogram(y=x, sr=200, hop_length=len(x)//256, \n                  n_fft=1024, n_mels=128, fmin=0, fmax=20, win_length=128)\n\n            # LOG TRANSFORM\n            width = (mel_spec.shape[1]//32)*32\n            mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max).astype(np.float32)[:,:width]\n\n            # STANDARDIZE TO -1 TO 1\n            mel_spec_db = (mel_spec_db+40)/40 \n            img[:,:,k] += mel_spec_db\n                \n        # AVERAGE THE 4 MONTAGE DIFFERENCES\n        img[:,:,k] /= 4.0\n        \n        if display:\n            plt.subplot(2,2,k+1)\n            plt.imshow(img[:,:,k],aspect='auto',origin='lower')\n#             plt.title(f'EEG {eeg_id} - Spectrogram {NAMES[k]}')\n            \n    if display: \n        plt.show()\n        plt.figure(figsize=(10,5))\n        offset = 0\n        for k in range(4):\n            if k>0: offset -= signals[3-k].min()\n            plt.plot(range(10_000),signals[k]+offset,label=NAMES[3-k])\n            offset += signals[3-k].max()\n        plt.legend()\n#         plt.title(f'EEG {eeg_id} Signals')\n        plt.show()\n        print(); print('#'*25); print()\n        \n    return img","metadata":{"execution":{"iopub.status.busy":"2024-02-29T08:46:01.056227Z","iopub.execute_input":"2024-02-29T08:46:01.056678Z","iopub.status.idle":"2024-02-29T08:46:01.173993Z","shell.execute_reply.started":"2024-02-29T08:46:01.056649Z","shell.execute_reply":"2024-02-29T08:46:01.172927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Data","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv(CFG.test_csv)\nprint('Test shape',test.shape)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-29T08:46:01.175357Z","iopub.execute_input":"2024-02-29T08:46:01.176114Z","iopub.status.idle":"2024-02-29T08:46:01.202732Z","shell.execute_reply.started":"2024-02-29T08:46:01.176061Z","shell.execute_reply":"2024-02-29T08:46:01.201545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ ALL SPECTROGRAMS\nfiles2 = os.listdir(CFG.test_spectrograms)\nprint(f'There are {len(files2)} test spectrogram parquets')\n    \nall_spectrograms = {}\nfor i,f in enumerate(files2):\n    if i%100==0: print(i,', ',end='')\n    tmp = pd.read_parquet(f'{CFG.test_spectrograms}{f}')\n    name = int(f.split('.')[0])\n    all_spectrograms[name] = tmp.iloc[:,1:].values\n    \n# RENAME FOR DATALOADER\ntest = test.rename({'spectrogram_id':'spec_id'},axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-02-29T08:46:01.206045Z","iopub.execute_input":"2024-02-29T08:46:01.206421Z","iopub.status.idle":"2024-02-29T08:46:01.461257Z","shell.execute_reply.started":"2024-02-29T08:46:01.206391Z","shell.execute_reply":"2024-02-29T08:46:01.460177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ ALL EEG SPECTROGRAMS\nDISPLAY = 1\nEEG_IDS2 = test.eeg_id.unique()\nall_eegs = {}\n\nprint('Converting Test EEG to Spectrograms...'); print()\nfor i,eeg_id in enumerate(EEG_IDS2):\n        \n    # CREATE SPECTROGRAM FROM EEG PARQUET\n    img = spectrogram_from_eeg(f'{CFG.test_eeg}{eeg_id}.parquet', i<DISPLAY)\n    all_eegs[eeg_id] = img","metadata":{"execution":{"iopub.status.busy":"2024-02-29T08:46:01.463227Z","iopub.execute_input":"2024-02-29T08:46:01.463710Z","iopub.status.idle":"2024-02-29T08:46:17.815625Z","shell.execute_reply.started":"2024-02-29T08:46:01.463669Z","shell.execute_reply":"2024-02-29T08:46:17.814468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class CustomDataset(Dataset):\n    def __init__(\n        self, df: pd.DataFrame,\n        augment: bool = False, mode: str = 'train',\n        specs: Dict[int, np.ndarray] = all_spectrograms,\n        eeg_specs: Dict[int, np.ndarray] = all_eegs\n    ): \n        self.df = df\n        self.augment = augment\n        self.mode = mode\n        self.spectograms = specs\n        self.eeg_spectograms = eeg_specs\n        \n    def __len__(self):\n        \"\"\"\n        Denotes the number of batches per epoch.\n        \"\"\"\n        return len(self.df)\n        \n    def __getitem__(self, index):\n        \"\"\"\n        Generate one batch of data.\n        \"\"\"\n        X, y = self.__data_generation(index)\n        if self.augment:\n            X = self.__transform(X) \n        return {\"spectrogram\":torch.tensor(X, dtype=torch.float32), \"labels\":torch.tensor(y, dtype=torch.float32)}\n                        \n    def __data_generation(self, index):\n        \"\"\"\n        Generates data containing batch_size samples.\n        \"\"\"\n        X = np.zeros((128, 256, 8), dtype='float32')\n        y = np.zeros(6, dtype='float32')\n        img = np.ones((128,256), dtype='float32')\n        row = self.df.iloc[index]\n        if self.mode=='test': \n            r = 0\n        else: \n            r = int(row['spectrogram_label_offset_seconds'] // 2)\n            \n        for region in range(4):\n            img = self.spectograms[row.spec_id][r:r+300, region*100:(region+1)*100].T\n            \n            # Log transform spectogram\n            img = np.clip(img, np.exp(-4), np.exp(8))\n            img = np.log(img)\n\n            # Standarize per image\n            ep = 1e-6\n            mu = np.nanmean(img.flatten())\n            std = np.nanstd(img.flatten())\n            img = (img-mu)/(std+ep)\n            img = np.nan_to_num(img, nan=0.0)\n            X[14:-14, :, region] = img[:, 22:-22] / 2.0\n            img = self.eeg_spectograms[row.eeg_id]\n            X[:, :, 4:] = img\n                \n            if self.mode != 'test':\n                y = row[TARGETS].values.astype(np.float32)\n            \n        return X, y\n    \n    def __transform(self, img):\n        transforms = A.Compose([\n            A.HorizontalFlip(p=0.5),\n        ])\n        return transforms(image=img)['image']","metadata":{"execution":{"iopub.status.busy":"2024-02-29T08:46:17.818548Z","iopub.execute_input":"2024-02-29T08:46:17.819560Z","iopub.status.idle":"2024-02-29T08:46:18.316553Z","shell.execute_reply.started":"2024-02-29T08:46:17.819512Z","shell.execute_reply":"2024-02-29T08:46:18.314260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = CustomDataset(test, mode=\"test\")\ntest_loader = DataLoader(\n    test_dataset,\n    batch_size=CFG.batch_size,\n    shuffle=False,\n    num_workers=CFG.num_workers, pin_memory=True, drop_last=False\n)\nX = test_dataset[0]\nprint(f\"X shape: {X['spectrogram'].shape}\")","metadata":{"execution":{"iopub.status.busy":"2024-02-29T08:46:18.319479Z","iopub.execute_input":"2024-02-29T08:46:18.320835Z","iopub.status.idle":"2024-02-29T08:46:18.366311Z","shell.execute_reply.started":"2024-02-29T08:46:18.320753Z","shell.execute_reply":"2024-02-29T08:46:18.365100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# model","metadata":{}},{"cell_type":"code","source":"class CustomModel(nn.Module):\n    def __init__(self, config, num_classes: int = 6, pretrained: bool = True):\n        super(CustomModel, self).__init__()\n        self.USE_KAGGLE_SPECTROGRAMS = True\n        self.USE_EEG_SPECTROGRAMS = True\n        self.model = timm.create_model(\n            config.model_name,\n            pretrained=pretrained,\n        )\n        # Optionally load state from checkpoint\n        if config.SparK:\n            state = torch.load('/kaggle/input/resnet50d-spark/resnet50_1kpretrained_timm_style.pth', 'cpu')\n            self.model.load_state_dict(state.get('module', state), strict=False)\n        if config.FREEZE:\n            for i,(name, param) in enumerate(list(self.model.named_parameters())\\\n                                             [0:config.NUM_FROZEN_LAYERS]):\n                param.requires_grad = False\n\n        self.features = nn.Sequential(*list(self.model.children())[:-2])\n        self.custom_layers = nn.Sequential(\n            nn.AdaptiveAvgPool2d(1),\n            nn.Flatten(),\n            nn.Linear(self.model.num_features, num_classes)\n        )\n\n    def __reshape_input(self, x):\n        \"\"\"\n        Reshapes input (128, 256, 8) -> (512, 512, 3) monotone image.\n        \"\"\" \n        # === Get spectograms ===\n        spectograms = [x[:, :, :, i:i+1] for i in range(4)]\n        spectograms = torch.cat(spectograms, dim=1)\n        \n        # === Get EEG spectograms ===\n        eegs = [x[:, :, :, i:i+1] for i in range(4,8)]\n        eegs = torch.cat(eegs, dim=1)\n        \n        # === Reshape (512,512,3) ===\n        if self.USE_KAGGLE_SPECTROGRAMS & self.USE_EEG_SPECTROGRAMS:\n            x = torch.cat([spectograms, eegs], dim=2)\n        elif self.USE_EEG_SPECTROGRAMS:\n            x = eegs\n        else:\n            x = spectograms\n            \n        x = torch.cat([x,x,x], dim=3)\n        x = x.permute(0, 3, 1, 2)\n        return x\n    \n    def forward(self, x):\n        x = self.__reshape_input(x)\n        x = self.features(x)\n        x = self.custom_layers(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2024-02-29T08:46:18.372214Z","iopub.execute_input":"2024-02-29T08:46:18.375413Z","iopub.status.idle":"2024-02-29T08:46:18.399499Z","shell.execute_reply.started":"2024-02-29T08:46:18.375360Z","shell.execute_reply":"2024-02-29T08:46:18.397951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference Function","metadata":{}},{"cell_type":"code","source":"def inference_function(test_loader, model, device):\n    model.eval() # set model in evaluation mode\n    softmax = nn.Softmax(dim=1)\n    prediction_dict = {}\n    preds = []\n    with tqdm(test_loader, unit=\"test_batch\", desc='Inference') as tqdm_test_loader:\n        for step, batch in enumerate(tqdm_test_loader):\n            X = batch.pop(\"spectrogram\").to(device) # send inputs to `device`\n            batch_size = X.size(0)\n            with torch.no_grad():\n                y_preds = model(X) # forward propagation pass\n            y_preds = softmax(y_preds)\n            preds.append(y_preds.to('cpu').numpy()) # save predictions\n                \n    prediction_dict[\"predictions\"] = np.concatenate(preds) # np.array() of shape (fold_size, target_cols)\n    return prediction_dict","metadata":{"execution":{"iopub.status.busy":"2024-02-29T08:46:18.403411Z","iopub.execute_input":"2024-02-29T08:46:18.403885Z","iopub.status.idle":"2024-02-29T08:46:18.419623Z","shell.execute_reply.started":"2024-02-29T08:46:18.403843Z","shell.execute_reply":"2024-02-29T08:46:18.418286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"predictions = []\n\nfor model_weight in model_weights:\n    test_dataset = CustomDataset(test, mode=\"test\")\n    train_loader = DataLoader(\n        test_dataset,\n        batch_size=CFG.batch_size,\n        shuffle=False,\n        num_workers=CFG.num_workers,\n        pin_memory=True,\n        drop_last=False\n    )\n    model = CustomModel(CFG, pretrained=False)\n    checkpoint = torch.load(model_weight, map_location=device)\n    model.load_state_dict(checkpoint[\"model\"])\n    model.to(device)\n    prediction_dict = inference_function(test_loader, model, device)\n    predictions.append(prediction_dict[\"predictions\"])\n    torch.cuda.empty_cache()\n    gc.collect()\n    \npredictions = np.array(predictions)\npredictions = np.mean(predictions, axis=0)","metadata":{"execution":{"iopub.status.busy":"2024-02-29T08:46:18.423062Z","iopub.execute_input":"2024-02-29T08:46:18.424136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"TARGETS = ['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']\nsub = pd.DataFrame({'eeg_id': test.eeg_id.values})\nsub[TARGETS] = predictions\nsub.to_csv(f'submission.csv',index=False)\nprint(f'Submission shape: {sub.shape}')\nsub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}