{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7392733,"sourceType":"datasetVersion","datasetId":4297749},{"sourceId":7392775,"sourceType":"datasetVersion","datasetId":4297782},{"sourceId":7447509,"sourceType":"datasetVersion","datasetId":4334995},{"sourceId":7607885,"sourceType":"datasetVersion","datasetId":4429716}],"dockerImageVersionId":30648,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install neptune\n","metadata":{"execution":{"iopub.status.busy":"2024-02-14T18:14:25.523740Z","iopub.execute_input":"2024-02-14T18:14:25.525123Z","iopub.status.idle":"2024-02-14T18:14:55.918765Z","shell.execute_reply.started":"2024-02-14T18:14:25.525064Z","shell.execute_reply":"2024-02-14T18:14:55.917493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os, gc\nimport timm \nimport wandb\nimport pandas as pd, numpy as np\nimport matplotlib.pyplot as plt\nimport torch\nfrom torch.utils.data import Dataset , DataLoader\nfrom datasets import DatasetDict\nfrom pynvml import *\nimport torch.nn as nn\nUSE_KAGGLE_SPECTROGRAMS = True\nUSE_EEG_SPECTROGRAMS = True\n","metadata":{"execution":{"iopub.status.busy":"2024-02-14T18:14:55.921515Z","iopub.execute_input":"2024-02-14T18:14:55.921974Z","iopub.status.idle":"2024-02-14T18:14:59.855387Z","shell.execute_reply.started":"2024-02-14T18:14:55.921933Z","shell.execute_reply":"2024-02-14T18:14:59.854032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import neptune\nfrom kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\ntoken = user_secrets.get_secret(\"NEPTUNE_API_TOKEN\")\n","metadata":{"execution":{"iopub.status.busy":"2024-02-14T18:14:59.857184Z","iopub.execute_input":"2024-02-14T18:14:59.858385Z","iopub.status.idle":"2024-02-14T18:15:00.849899Z","shell.execute_reply.started":"2024-02-14T18:14:59.858328Z","shell.execute_reply":"2024-02-14T18:15:00.848402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"def print_gpu_utilization():\n    nvmlInit()\n    handle = nvmlDeviceGetHandleByIndex(0)\n    info = nvmlDeviceGetMemoryInfo(handle)\n    print(f\"GPU memory occupied: {info.used//1024**2} MB.\")\n","metadata":{"execution":{"iopub.status.busy":"2024-02-11T21:48:27.835861Z","iopub.execute_input":"2024-02-11T21:48:27.836176Z","iopub.status.idle":"2024-02-11T21:48:27.843036Z","shell.execute_reply.started":"2024-02-11T21:48:27.836147Z","shell.execute_reply":"2024-02-11T21:48:27.842057Z"}}},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\nTARGETS = df.columns[-6:]\nprint('Train shape:', df.shape )\nprint('Targets', list(TARGETS))\ndf.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-14T18:15:00.853836Z","iopub.execute_input":"2024-02-14T18:15:00.854837Z","iopub.status.idle":"2024-02-14T18:15:01.277572Z","shell.execute_reply.started":"2024-02-14T18:15:00.854780Z","shell.execute_reply":"2024-02-14T18:15:01.276135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TARGETS","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## only train on non overlapping eegs since test data doesnt contain overlapping eegs , --> group by eegs and then compye agg values","metadata":{}},{"cell_type":"code","source":"train = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds','eeg_label_offset_seconds']].agg(\n    {'spectrogram_id':'first','spectrogram_label_offset_seconds':'min'})\ntrain.columns = ['spec_id','min',]\n\ntmp = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n    {'spectrogram_label_offset_seconds':'max'})\ntrain['max'] = tmp\n\ntmp = df.groupby('eeg_id')[['patient_id']].agg('first')\ntrain['patient_id'] = tmp\n\ntmp = df.groupby('eeg_id')[TARGETS].agg('sum')\nfor t in TARGETS:\n    train[t] = tmp[t].values\n    \ny_data = train[TARGETS].values\ny_data = y_data / y_data.sum(axis=1,keepdims=True)\ntrain[TARGETS] = y_data\n\ntmp = df.groupby('eeg_id')[['expert_consensus']].agg('first')\ntrain['target'] = tmp\n\ntrain = train.reset_index()\nprint('Train non-overlapp eeg_id shape:', train.shape )\ntrain.head()\ndel df , tmp , y_data\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-14T18:15:01.279496Z","iopub.execute_input":"2024-02-14T18:15:01.279914Z","iopub.status.idle":"2024-02-14T18:15:01.885770Z","shell.execute_reply.started":"2024-02-14T18:15:01.279882Z","shell.execute_reply":"2024-02-14T18:15:01.884515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nitos = dict((k,v) for k,v in enumerate(TARGETS))\nstoi = dict((v,k) for k,v in enumerate(TARGETS))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Kaggle Spectrograms","metadata":{}},{"cell_type":"code","source":"%%time\nREAD_SPEC_FILES = False\n\n# READ ALL SKagglePECTROGRAMS\nPATH = '/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/'\nfiles = os.listdir(PATH)\nprint(f'There are {len(files)} spectrogram parquets')\n\nif READ_SPEC_FILES:    \n    spectrograms = {}\n    for i,f in enumerate(files):\n        if i%100==0: print(i,', ',end='')\n        tmp = pd.read_parquet(f'{PATH}{f}')\n        name = int(f.split('.')[0])\n        spectrograms[name] = tmp.iloc[:,1:].values\nelse:\n    spectrograms = np.load('/kaggle/input/brain-spectrograms/specs.npy',allow_pickle=True).item()","metadata":{"execution":{"iopub.status.busy":"2024-02-14T18:15:01.887980Z","iopub.execute_input":"2024-02-14T18:15:01.888880Z","iopub.status.idle":"2024-02-14T18:16:14.101952Z","shell.execute_reply.started":"2024-02-14T18:15:01.888828Z","shell.execute_reply":"2024-02-14T18:16:14.099996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Spectrograms from Brain EEGS","metadata":{}},{"cell_type":"code","source":"%%time\nREAD_EEG_SPEC_FILES = False\n\nif READ_EEG_SPEC_FILES:\n    all_eegs = {}\n    for i,e in enumerate(train.eeg_id.values):\n        if i%100==0: print(i,', ',end='')\n        x = np.load(f'/kaggle/input/brain-eeg-spectrograms/EEG_Spectrograms/{e}.npy')\n        all_eegs[e] = x\nelse:\n    all_eegs = np.load('/kaggle/input/brain-eeg-spectrograms/eeg_specs.npy',allow_pickle=True).item()\n","metadata":{"execution":{"iopub.status.busy":"2024-02-14T18:16:14.104551Z","iopub.execute_input":"2024-02-14T18:16:14.105206Z","iopub.status.idle":"2024-02-14T18:17:47.612842Z","shell.execute_reply.started":"2024-02-14T18:16:14.105149Z","shell.execute_reply":"2024-02-14T18:17:47.610112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(all_eegs[train.iloc[0][\"eeg_id\"]].shape)\nprint(spectrograms[train.iloc[0][\"spec_id\"]].shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c={}\nc[0]=5\nc","metadata":{"execution":{"iopub.status.busy":"2024-02-14T18:46:27.741440Z","iopub.execute_input":"2024-02-14T18:46:27.741983Z","iopub.status.idle":"2024-02-14T18:46:27.751242Z","shell.execute_reply.started":"2024-02-14T18:46:27.741948Z","shell.execute_reply":"2024-02-14T18:46:27.749757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import albumentations as A\nimport torchaudio.transforms as Ta\nimport torchvision.transforms as Tv\ndef apply_reshape(X,channels,n_returns):\n    r=None\n    if(n_returns==1):\n        r=(512,512)\n    elif(n_returns==2):\n        r=(256,512)\n    elif(n_returns==8):\n        r=(128,256)\n    \n    if(channels==3):\n        X = np.expand_dims(X, axis=3)\n        X = np.tile(X, (1, 1, 1, 3))\n        X = np.transpose(X, (0, 2, 1, 3))\n        X = X.reshape(*r, 3)\n    else :\n        X = X.reshape(*r, 1)\n    return X\n        \nclass SpecsDataset(Dataset):\n    def __init__(self, data,specs,eeg_specs,targets,mode,channels=3,n_returns=1,A_transform=None,T_transform=None):\n        self.data = data.reset_index(drop=True)\n        self.specs = specs\n        self.eeg_specs = eeg_specs\n        self.targets = targets\n        self.mode = mode\n        self.A_transform = A_transform\n        self.T_transform = T_transform\n        self.channels = channels\n        self.n_returns=n_returns\n        \n    def __len__(self):\n        \"\"\"Return the size of the dataset.\"\"\"\n        return len(self.data)\n\n    def __getitem__(self, index):\n        \"\"\"\n        Args:\n            index (int): Index of the dataset sample to retrieve.\n\n        Returns:\n            tuple: (sample, target) where target is the corresponding label.\n        \"\"\"\n        X = np.zeros((128,256,8),dtype='float32')\n        y = np.zeros((6),dtype='float32')\n        img = np.ones((128,256),dtype='float32')\n        \n        \n        row = self.data.iloc[index]\n        if self.mode=='test': \n            r = 0\n        else: \n            r = int( (row['min'] + row['max'])//4 )\n\n        for k in range(4):\n            # EXTRACT 300 ROWS OF SPECTROGRAM\n            img = self.specs[row.spec_id][r:r+300,k*100:(k+1)*100].T\n\n            # LOG TRANSFORM SPECTROGRAM\n            \n            img = np.clip(img,1,np.exp(100))\n            img = np.log(img)\n            \n            # STANDARDIZE PER IMAGE\n            ep = 1e-6\n            m = np.nanmean(img.flatten())\n            s = np.nanstd(img.flatten())\n            img = (img-m)/(s+ep)\n            img = np.nan_to_num(img, nan=0.0)\n\n            # CROP TO 256 TIME STEPS\n            X[14:-14,:,k] = img[:,22:-22] / 2.0\n        \n        X[:,:,4:]=self.eeg_specs[row.eeg_id]\n      \n        sample = {\"specs\":{}}  # Dictionary to store the results\n\n        \n        step = int(8 / self.n_returns)\n        cur=0\n        for i in range(self.n_returns) :\n              # Calculate the step for downsampling\n            sample[\"specs\"][i] = apply_reshape(X[:, :, cur:cur+step],self.channels,self.n_returns)\n            cur+=step    \n        \n            if self.A_transform:\n                sample[\"specs\"][i] = self.A_transform(image=sample[\"specs\"][i])[\"image\"]\n            sample[\"specs\"][i] = torch.from_numpy(sample[\"specs\"][i].transpose(2,0,1)).to(dtype=torch.float32)\n            sample[\"specs\"][i] = torch.nan_to_num(sample[\"specs\"][i], nan=0)\n            if self.T_transform:\n                sample[\"specs\"][i] = self.T_transform(sample[\"specs\"][i])\n\n        if self.mode!='test':\n            y = row[self.targets]\n            sample[\"label\"]=torch.tensor(y.values.tolist(), dtype=torch.float32)\n            return sample\n\n        return sample \n       \n\n\ntotal_indexes=list(range(len(train)))\nnp.random.shuffle(total_indexes)\nval_prop=0.05\nnum_val_indexes = int(len(total_indexes) * val_prop)\n\n# Extract train and validation indexes\ntrain_idx = total_indexes[num_val_indexes:]\nval_idx = total_indexes[:num_val_indexes]\n\n\n# resize 3x32x32 to 3x224x224 -> convert to Pytorch tensor -> normalize\n\n# Corrected code\nA_transf = A.Compose([\n     A.HorizontalFlip(p=0.5),\n     #A.Resize(height=224, width=224),\n     \n])\nT_transf = Tv.Compose([\n     Ta.TimeMasking(time_mask_param=60),\n     Ta.FrequencyMasking(freq_mask_param=60),\n])\n    \n\ntrainds = SpecsDataset(train.iloc[train_idx], spectrograms, all_eegs,TARGETS,mode='train',channels=1,n_returns=8,A_transform=A_transf,T_transform=T_transf)\nvalds = SpecsDataset(train.iloc[val_idx] , spectrograms, all_eegs,TARGETS,mode='train',A_transform=A_transf,T_transform=T_transf)\n\n\n\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-02-14T18:51:33.629426Z","iopub.execute_input":"2024-02-14T18:51:33.629987Z","iopub.status.idle":"2024-02-14T18:51:33.685400Z","shell.execute_reply.started":"2024-02-14T18:51:33.629949Z","shell.execute_reply":"2024-02-14T18:51:33.684311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainds[1000]","metadata":{"execution":{"iopub.status.busy":"2024-02-14T18:52:09.521628Z","iopub.execute_input":"2024-02-14T18:52:09.522244Z","iopub.status.idle":"2024-02-14T18:52:09.577319Z","shell.execute_reply.started":"2024-02-14T18:52:09.522193Z","shell.execute_reply":"2024-02-14T18:52:09.575745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\n# Assuming you have defined X and returns elsewhere\nX = np.zeros((128, 256, 8), dtype='float32')\nreturns = [1, 2, 8]  # Example list of returns\n\nres = {}  # Dictionary to store the results\n\nfor r in returns:\n    step = int(8 / r)\n    cur=0\n    for i in range(r) :\n          # Calculate the step for downsampling\n        res[f\"spec_{r}_{i}\"] = X[:, :, cur:cur+step]  # Downsample along the third axis and store in dictionary\n        cur+=step    \n# Print the shapes of the downsampled arrays\nfor key, value in res.items():\n    print(f\"Shape of {key}: {value.shape}\")","metadata":{"execution":{"iopub.status.busy":"2024-02-14T18:06:13.038029Z","iopub.execute_input":"2024-02-14T18:06:13.038477Z","iopub.status.idle":"2024-02-14T18:06:13.048540Z","shell.execute_reply.started":"2024-02-14T18:06:13.038444Z","shell.execute_reply":"2024-02-14T18:06:13.047147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X[:, :, ::2].shape","metadata":{"execution":{"iopub.status.busy":"2024-02-14T18:03:07.138211Z","iopub.execute_input":"2024-02-14T18:03:07.138757Z","iopub.status.idle":"2024-02-14T18:03:07.147631Z","shell.execute_reply.started":"2024-02-14T18:03:07.138714Z","shell.execute_reply":"2024-02-14T18:03:07.146166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import librosa\nspec = trainds[15][\"spec\"]\neeg_spec = trainds[15][\"eeg_spec\"]\ndef plot(spec, title=\"Original\"):\n    plt.title(title)\n    plt.imshow(librosa.power_to_db(spec), origin=\"lower\", aspect=\"auto\")\n    plt.colorbar(format='%+2.0f dB')\n    plt.xlabel('Time')\n    plt.ylabel('Frequency')\n    plt.tight_layout()\n\n\n\nplt.figure(figsize=(10, 5))\n\nplt.subplot(1, 2, 1)  # Subplot with 1 row, 2 columns, and index 1 (left subplot)\nplot(spec[0], title=\"Original\")\n\nplt.subplot(1, 2, 2)  # Subplot with 1 row, 2 columns, and index 2 (right subplot)\nplot(eeg_spec[0], title=\"Original\")\n\nplt.tight_layout()  # Adjust layout to prevent overlap\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-14T14:56:14.845425Z","iopub.execute_input":"2024-02-14T14:56:14.845849Z","iopub.status.idle":"2024-02-14T14:56:15.895295Z","shell.execute_reply.started":"2024-02-14T14:56:14.845817Z","shell.execute_reply":"2024-02-14T14:56:15.894302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/test.csv')\nprint('Test shape',test.shape)\ntest.head()\nPATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/'\nfiles2 = os.listdir(PATH2)\nprint(f'There are {len(files2)} test spectrogram parquets')\n    \nspectrograms2 = {}\nfor i,f in enumerate(files2):\n    if i%100==0: print(i,', ',end='')\n    tmp = pd.read_parquet(f'{PATH2}{f}')\n    name = int(f.split('.')[0])\n    spectrograms2[name] = tmp.iloc[:,1:].values\n    \n# RENAME FOR DATALOADER\ntest = test.rename({'spectrogram_id':'spec_id'},axis=1)\n\nimport pywt, librosa\n\nUSE_WAVELET = None \n\nNAMES = ['LL','LP','RP','RR']\n\nFEATS = [['Fp1','F7','T3','T5','O1'],\n         ['Fp1','F3','C3','P3','O1'],\n         ['Fp2','F8','T4','T6','O2'],\n         ['Fp2','F4','C4','P4','O2']]\n\n# DENOISE FUNCTION\ndef maddest(d, axis=None):\n    return np.mean(np.absolute(d - np.mean(d, axis)), axis)\n\ndef denoise(x, wavelet='haar', level=1):    \n    coeff = pywt.wavedec(x, wavelet, mode=\"per\")\n    sigma = (1/0.6745) * maddest(coeff[-level])\n\n    uthresh = sigma * np.sqrt(2*np.log(len(x)))\n    coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])\n\n    ret=pywt.waverec(coeff, wavelet, mode='per')\n    \n    return ret\n\ndef spectrogram_from_eeg(parquet_path, display=False):\n    \n    # LOAD MIDDLE 50 SECONDS OF EEG SERIES\n    eeg = pd.read_parquet(parquet_path)\n    middle = (len(eeg)-10_000)//2\n    eeg = eeg.iloc[middle:middle+10_000]\n    \n    # VARIABLE TO HOLD SPECTROGRAM\n    img = np.zeros((128,256,4),dtype='float32')\n    \n    if display: plt.figure(figsize=(10,7))\n    signals = []\n    for k in range(4):\n        COLS = FEATS[k]\n        \n        for kk in range(4):\n        \n            # COMPUTE PAIR DIFFERENCES\n            x = eeg[COLS[kk]].values - eeg[COLS[kk+1]].values\n\n            # FILL NANS\n            m = np.nanmean(x)\n            if np.isnan(x).mean()<1: x = np.nan_to_num(x,nan=m)\n            else: x[:] = 0\n\n            # DENOISE\n            if USE_WAVELET:\n                x = denoise(x, wavelet=USE_WAVELET)\n            signals.append(x)\n\n            # RAW SPECTROGRAM\n            mel_spec = librosa.feature.melspectrogram(y=x, sr=200, hop_length=len(x)//256, \n                  n_fft=1024, n_mels=128, fmin=0, fmax=20, win_length=128)\n\n            # LOG TRANSFORM\n            width = (mel_spec.shape[1]//32)*32\n            mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max).astype(np.float32)[:,:width]\n\n            # STANDARDIZE TO -1 TO 1\n            mel_spec_db = (mel_spec_db+40)/40 \n            img[:,:,k] += mel_spec_db\n                \n        # AVERAGE THE 4 MONTAGE DIFFERENCES\n        img[:,:,k] /= 4.0\n        \n        if display:\n            plt.subplot(2,2,k+1)\n            plt.imshow(img[:,:,k],aspect='auto',origin='lower')\n            plt.title(f'EEG {eeg_id} - Spectrogram {NAMES[k]}')\n            \n    if display: \n        plt.show()\n        plt.figure(figsize=(10,5))\n        offset = 0\n        for k in range(4):\n            if k>0: offset -= signals[3-k].min()\n            plt.plot(range(10_000),signals[k]+offset,label=NAMES[3-k])\n            offset += signals[3-k].max()\n        plt.legend()\n        plt.title(f'EEG {eeg_id} Signals')\n        plt.show()\n        print(); print('#'*25); print()\n        \n    return img\n\n# READ ALL EEG SPECTROGRAMS\nPATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/'\nDISPLAY = 1\nEEG_IDS2 = test.eeg_id.unique()\nall_eegs2 = {}\n\nprint('Converting Test EEG to Spectrograms...'); print()\nfor i,eeg_id in enumerate(EEG_IDS2):\n        \n    # CREATE SPECTROGRAM FROM EEG PARQUET\n    img = spectrogram_from_eeg(f'{PATH2}{eeg_id}.parquet', i<DISPLAY)\n    all_eegs2[eeg_id] = img\n\n    \n# INFER EFFICIENTNET ON TEST\n\ntestds = SpecsDataset(test, spectrograms2, all_eegs2,TARGETS,mode='test',A_transform=None,T_transform=None)\n","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-02-14T14:52:26.985472Z","iopub.execute_input":"2024-02-14T14:52:26.986492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Hyperparams","metadata":{}},{"cell_type":"code","source":"import time\n\nconfig = {\n    'model_name': 'efficientnet_b0',\n    'parallel':True,\n    'LOSS': 'KLdiv',\n    'train_bs': 28,\n    'val_bs': 32,\n    'num_workers': 8,\n    'epochs': 5,\n    'splits': 5,\n    'lr': 1e-3,\n    \"warmup_steps\":0,\n    \"weight_decay\":0,\n    'label_smoothing':0,\n    'image_size':512,\n    \"num_heads\": 2,\n    \"channels\":3,\n    \"n_returns\":8,\n    \"augmentations\":\"  hflip(0.5) + time_masking(60) + freq_masking(60) + no mixed precision\",\n    'competition': 'HMS - Harmful Brain Activity Classification',\n    'comment':\"None\"\n}","metadata":{"execution":{"iopub.status.busy":"2024-02-14T18:31:24.020209Z","iopub.execute_input":"2024-02-14T18:31:24.020719Z","iopub.status.idle":"2024-02-14T18:31:24.030031Z","shell.execute_reply.started":"2024-02-14T18:31:24.020678Z","shell.execute_reply":"2024-02-14T18:31:24.028397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Models Architectures","metadata":{}},{"cell_type":"code","source":"\nclass ModelforClassification(nn.Module):\n    def __init__(self, config, pretrained=True):\n        super(ModelforClassification, self).__init__()\n        self.models = timm.create_model(config['model_name'], pretrained=pretrained,in_chans=config[\"channels\"])\n        self.model.classifier =nn.Sequential(\n                nn.Linear(in_features=self.model.classifier.in_features, out_features=len(TARGETS)),\n                #nn.SiLU(),            # Swish activation function (also known as SiLU)\n                #nn.Dropout(p=0.3),\n                #nn.Linear(in_features=512, out_features=128),\n                #nn.SiLU(),            # Swish activation\n                #nn.Linear(in_features=128, out_features=len(TARGETS))\n            )\n        \n    def forward(self, x):\n        return self.model(x)\n    \nclass ParallelModelforClassification(nn.Module):\n    def __init__(self, config, pretrained=True):\n        super(ParallelModelforClassification, self).__init__()\n        self.models = ModuleList()\n        self.out_features=0\n        for _ in range(config[\"n_returns\"]):\n            \n            self.pretrained_model = timm.create_model(config['model_name'], pretrained=pretrained,in_chans=config[\"channels\"])\n            self.out_features+=self.pretrained_model.classifier.in_features\n            self.models.append(nn.Sequential(*list(self.pretrained_model.children())[:-1]))\n\n        self.classifier =nn.Sequential(\n                nn.Linear(in_features = self.out_features , out_features=6),\n                #nn.SiLU(),            # Swish activation function (also known as SiLU)\n                #nn.Dropout(p=0.3),\n                #nn.Linear(in_features=512, out_features=128),\n                #nn.SiLU(),            # Swish activation\n                #nn.Linear(in_features=128, out_features=len(TARGETS))\n            )\n        \n    def forward(self, x): \n        tensor_list = []\n        # Assuming x is a list of tensors\n        for i, model in enumerate(self.models):\n            # Assuming x[i] is the input tensor for the i-th model\n            output = model(x[i])\n            tensor_list.append(output)\n        # Concatenate the tensors along dim=1\n        z = torch.cat(tensor_list, dim=1)\n        return self.classifier(z)\n  \nclass ParallelModelWithAttentionforClassification(nn.Module):\n    def __init__(self, config, pretrained=True):\n        super(ParallelModelWithAttentionforClassification, self).__init__()\n        self.models = ModuleList()\n        self.out_features=0\n        for _ in range(config[\"n_returns\"]):\n            \n            self.pretrained_model = timm.create_model(config['model_name'], pretrained=pretrained,in_chans=config[\"channels\"])\n            self.out_features=self.pretrained_model.classifier.in_features\n            self.models.append(nn.Sequential(*list(self.pretrained_model.children())[:-1]))\n\n        self.classifier =nn.Sequential(\n                nn.Linear(in_features = self.out_features , out_features=6),\n                #nn.SiLU(),            # Swish activation function (also known as SiLU)\n                #nn.Dropout(p=0.3),\n                #nn.Linear(in_features=512, out_features=128),\n                #nn.SiLU(),            # Swish activation\n                #nn.Linear(in_features=128, out_features=len(TARGETS))\n            )\n        self.multihead_attn = nn.MultiheadAttention(self.out_features, num_heads=config[\"num_heads\"])\n        \n        self.classifier =nn.Sequential(\n                nn.Linear(in_features = self.out_features  , out_features=6),\n                #nn.SiLU(),            # Swish activation function (also known as SiLU)\n                #nn.Dropout(p=0.3),\n                #nn.Linear(in_features=512, out_features=128),\n                #nn.SiLU(),            # Swish activation\n                #nn.Linear(in_features=128, out_features=len(TARGETS))\n            )\n        \n    def forward(self, x):\n        z=[]\n        for i,model in enumerate(self.models) :\n            out=model(X[i])\n            z.append(out) \n        \n        z = self.multihead_attn(z[0], z[1], z[1],need_weights=False)[0]\n        return self.classifier(z)\n\n    \ndef init_model(accelerator, config, pretrained=True):\n    with accelerator.main_process_first():\n        \n        if(config[\"num_heads\"] is not None):\n            model = ParallelModelWithAttentionforClassification(config=config, pretrained=pretrained)\n        else(config[\"parallel\"]):\n            model = ParallelModelforClassification(config=config, pretrained=pretrained)\n        else :\n            model = ModelforClassification(config=config, pretrained=pretrained)\n\n    return model\n\n","metadata":{"execution":{"iopub.status.busy":"2024-02-14T15:49:28.217095Z","iopub.execute_input":"2024-02-14T15:49:28.217579Z","iopub.status.idle":"2024-02-14T15:49:28.241035Z","shell.execute_reply.started":"2024-02-14T15:49:28.217542Z","shell.execute_reply":"2024-02-14T15:49:28.239828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_step(model,X,y,optimizer,scheduler,criterion,fold,seed,accelerator,run):\n    \n    optimizer.zero_grad()\n    logits = model(X)\n    log_soft= torch.nn.functional.log_softmax(logits, dim=-1)\n    # Compute KL divergence loss\n    loss = criterion(log_soft,y)\n    accelerator.backward(loss)\n    optimizer.step()\n    scheduler.step()\n    if accelerator.is_main_process:\n        if fold is None :\n            run[f\"metrics/train_loss_seed_{seed}\"].append(loss.item())\n        else :\n            run[f\"metrics/fold_{fold}/train_loss\"].append(loss.item())\n        \n    return loss\ndef eval_step(model,X,y,criterion,fold,accelerator,run):\n    logits = model(X)\n    log_soft= torch.nn.functional.log_softmax(logits, dim=-1)\n    # Compute KL divergence loss\n    loss = criterion(log_soft,y)\n    \n    if accelerator.is_main_process:\n        run[f\"metrics/fold_{fold}/val_loss\"].append(loss.item())\n        \n    return loss\n\ndef train_epoch(model,train_dl,optimizer,scheduler,criterion,accelerator,epoch,fold,seed,train_log_each=10):\n    epoch_train_loss=0\n    model.train()\n    for i,batch in enumerate(train_dl):\n        X,y = batch[\"specs\"],batch[\"label\"]\n        loss = train_step(model,X,X1,y,optimizer,scheduler,criterion,fold,seed,accelerator,run)\n        epoch_train_loss+=loss.item()\n        if(i%train_log_each==0):\n            accelerator.print(\n                f'Batch {i}/{len(train_dl)} | Train Loss: {loss.item():.4f}'\n            )\n\n    epoch_train_loss/=len(train_dl)\n    return epoch_train_loss\ndef eval_epoch(model,val_dl,criterion,accelerator,epoch,fold,eval_log_each=10)    :\n    epoch_val_loss=0\n    model.eval()\n    with torch.no_grad():\n        for i,batch in enumerate(val_dl):\n            X,y = batch[\"specs\"]  ,batch[\"label\"]\n            loss = eval_step(model,X,y,criterion,fold,accelerator,run)\n            epoch_val_loss+=loss.item()\n            if(i%eval_log_each==0):\n                accelerator.print(\n                    f'Batch {i}/{len(val_dl)} | Val Loss: {loss.item():.4f}'\n                )\n          \n    epoch_val_loss/=len(val_dl)\n    \n    return epoch_val_loss\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2024-02-14T14:53:09.527471Z","iopub.execute_input":"2024-02-14T14:53:09.527944Z","iopub.status.idle":"2024-02-14T14:53:09.548246Z","shell.execute_reply.started":"2024-02-14T14:53:09.527897Z","shell.execute_reply":"2024-02-14T14:53:09.546375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/kaggle-kl-div')\nfrom kaggle_kl_div import score\ndef compute_cv(oof_preds,all_true):\n    oof = pd.DataFrame(oof_preds.copy())\n    oof['id'] = np.arange(len(oof))\n\n    true = pd.DataFrame(all_true.copy())\n    true['id'] = np.arange(len(true))\n\n    cv = score(solution=true, submission=oof, row_id_column_name='id')\n    return cv ","metadata":{"execution":{"iopub.status.busy":"2024-02-14T14:53:14.205348Z","iopub.execute_input":"2024-02-14T14:53:14.205905Z","iopub.status.idle":"2024-02-14T14:53:14.216495Z","shell.execute_reply.started":"2024-02-14T14:53:14.205862Z","shell.execute_reply":"2024-02-14T14:53:14.214979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedGroupKFold\nfrom transformers import  AdamW ,get_cosine_schedule_with_warmup\nfrom tqdm import tqdm\nfrom accelerate import Accelerator\nfrom accelerate.utils import LoggerType,set_seed\n\n\ndef training_folds(mixed_precision:str, seed:int,run):\n    set_seed(seed)\n    accelerator = Accelerator(mixed_precision=mixed_precision)\n    \n    # Configure train function\n    \n    accelerator.print(f'Using device: {accelerator.device}')\n    criterion=nn.KLDivLoss(reduction=\"batchmean\")\n    sgkf = StratifiedGroupKFold(n_splits=config[\"splits\"],random_state=seed,shuffle=True)\n    groups=train[\"patient_id\"]\n    X=train.drop(columns=[\"target\"])\n    y=train[\"target\"]\n    oof_preds = np.zeros((len(train), len(TARGETS)))\n    test_preds = np.zeros((len(testds),len(TARGETS)))\n    testdl = DataLoader(testds, batch_size=32, shuffle=False)\n    test_dl = accelerator.prepare(testdl)\n    \n    \n    \n    all_oof = []\n    all_true = []\n    \n    for fold, (train_idx, val_idx) in enumerate(sgkf.split(X, y, groups=groups)):\n        trainds_ = SpecsDataset(train.iloc[train_idx], spectrograms, all_eegs, TARGETS, mode='train',channels=config[\"channels\"],n_returns=config[\"n_returns\"], A_transform=A_transf,T_transform=T_transf)\n        valds_ = SpecsDataset(train.iloc[val_idx], spectrograms, all_eegs, TARGETS, mode='train',channels=config[\"channels\"],n_returns=config[\"n_returns\"], A_transform=None,T_transform=None)\n        train_dl = DataLoader(trainds_, batch_size=config[\"train_bs\"], shuffle=False)\n        val_dl = DataLoader(valds_, batch_size=config[\"val_bs\"], shuffle=False)\n        model = init_model(accelerator, config, pretrained=True)\n        optimizer = torch.optim.AdamW(params=model.parameters())\n        config[\"warmup_steps\"]=len(train_dl)\n        scheduler = get_cosine_schedule_with_warmup(optimizer=optimizer, num_warmup_steps=config[\"warmup_steps\"], num_training_steps=len(train_dl) * config[\"epochs\"])\n        model, optimizer,train_dl,val_dl,scheduler = accelerator.prepare(model, optimizer, train_dl,val_dl,scheduler)\n\n        fold_oof=[]\n        fold_true=[]\n        gc.collect()\n        \n        fold+=1\n        \n        \n        for epoch in range(config[\"epochs\"]):\n            start=time.time()\n            epoch_train_loss=train_epoch(model,train_dl,optimizer,scheduler,criterion,accelerator,epoch,fold,seed=None)\n            epoch_val_loss=eval_epoch(model,val_dl,criterion,accelerator,epoch,fold)\n            end=time.time()\n            accelerator.print(f'Fold {fold} | Epoch {epoch+1}/{config[\"epochs\"]} | Train Loss: {epoch_train_loss:.4f} | Val Loss: {epoch_val_loss:.4f} | Time: {end-start}')\n\n        #accelerator.end_training()\n        \n        accelerator.wait_for_everyone() \n        model = accelerator.unwrap_model(model)\n        checkpoint = {\n                'fold': fold,\n                'model_state_dict': model.state_dict(),\n                'optimizer_state_dict': optimizer.state_dict(),\n                'scheduler_state_dict': scheduler.state_dict(),\n            }\n        accelerator.save(checkpoint, f'checkpoint_fold{fold}.ckpt')\n        if accelerator.is_main_process:\n            run[f\"checkpoints/checkpoint_fold{fold}\"].upload(f\"checkpoint_fold{fold}.ckpt\")\n        # End wandb run\n        \n        gc.collect()\n        # Out-of-fold predictions\n        val_outputs = []\n        val_targets = []\n        model.eval()\n        with torch.no_grad():\n            for i,batch in enumerate(val_dl):\n                X,targets = batch[\"specs\"] , batch[\"label\"] \n                outputs = model(X).softmax(dim=1)\n                outputs, targets = accelerator.gather_for_metrics((\n                    outputs, targets\n                ))\n                val_outputs.append(outputs.cpu().detach().numpy())\n                val_targets.append(targets.cpu().detach().numpy())\n                \n\n        val_outputs = np.concatenate(val_outputs, axis=0)\n        val_targets = np.concatenate(val_targets, axis=0)\n\n        all_oof.extend(val_outputs)\n        all_true.extend(val_targets)\n        #val_preds = np.concatenate(val_preds, axis=0)\n        #oof_preds[val_idx] = val_preds\n\n        score=compute_cv(val_outputs,val_targets)\n        accelerator.print(f'Fold {fold} | val KL div loss : {score:.4f}')\n        if accelerator.is_main_process:\n            run[f\"metrics/CV/KLDivLoss\"].append(score)\n        \n        \n        del model \n        gc.collect()\n    # Store OOF predictions in the original DataFrame\n    \n    if accelerator.is_main_process:\n        run['configuration'] = config # This will create a ‘parameters directory containing the PARAMS dictionary\n    score=compute_cv(all_oof,all_true)\n    accelerator.print(f'N° splits: {config[\"splits\"]} | N° epochs: {config[\"epochs\"]} | CV KL DIV {score}')\n    \n   \n    if accelerator.is_main_process:\n        run[\"metrics/CV/KLDivLoss\"].append(score)\n        \n    if accelerator.is_main_process:\n            run[\"metrics/placeholder_1\"]=score\n    if accelerator.is_main_process:\n        run[\"metrics/placeholder_2\"]=score\n                     \n        \n# Train on full data using 3 different seeds\ndef training_full(mixed_precision:str,seeds:list ,run):\n    \n    accelerator = Accelerator(mixed_precision=mixed_precision)\n    accelerator.print(f'Using device: {accelerator.device}')\n    criterion=nn.KLDivLoss(reduction=\"batchmean\")\n   \n    for i,seed in enumerate(seeds) :\n        set_seed(seed)\n        trainds_ = SpecsDataset(train, spectrograms, all_eegs, TARGETS, mode='train',channels=config[\"channels\"],n_returns=config[\"n_returns\"], A_transform=A_transf,T_transform=T_transf)\n        train_dl = DataLoader(trainds_, batch_size=config[\"train_bs\"], shuffle=True)\n        model = init_model(accelerator, config, pretrained=True)\n        optimizer = torch.optim.AdamW(params=model.parameters())\n        scheduler = get_cosine_schedule_with_warmup(optimizer=optimizer, num_warmup_steps=config[\"warmup_steps\"], num_training_steps=len(train_dl) * config[\"epochs\"])\n        model, optimizer,train_dl,scheduler = accelerator.prepare(model, optimizer, train_dl,scheduler)\n        \n        for epoch in range(config[\"epochs\"]):\n            start=time.time()\n            epoch_train_loss=train_epoch(model,train_dl,optimizer,scheduler,criterion,accelerator,epoch,fold=None,seed=i+1)\n            end=time.time()\n            accelerator.print(f'Full Data Training | Seed {i+1} | Epoch {epoch+1}/{config[\"epochs\"]} | Train Loss: {epoch_train_loss:.4f} | Time: {end-start}')\n            \n        model = accelerator.unwrap_model(model)\n        checkpoint = {\n                'model_state_dict': model.state_dict(),\n                'optimizer_state_dict': optimizer.state_dict(),\n                'scheduler_state_dict': scheduler.state_dict(),\n            }\n        accelerator.save(checkpoint, f'checkpoint_full_seed{i+1}.ckpt')\n        if accelerator.is_main_process:\n            run[f\"checkpoints/checkpoint_full_seed{i+1}\"].upload(f\"checkpoint_full_seed{i+1}.ckpt\")\n        #val_preds = np.concatenate(val_preds, axis=0)\n        #oof_preds[val_idx] = val_preds\n\n        \n        del model \n        gc.collect()\n    # Store OOF predictions in the original DataFrame\n   \n    ","metadata":{"execution":{"iopub.status.busy":"2024-02-14T14:54:03.881613Z","iopub.execute_input":"2024-02-14T14:54:03.882096Z","iopub.status.idle":"2024-02-14T14:54:03.922849Z","shell.execute_reply.started":"2024-02-14T14:54:03.882063Z","shell.execute_reply":"2024-02-14T14:54:03.921267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from accelerate import notebook_launcher\nLAUNCH_FOLDS_TRAINING=False\nif(LAUNCH_FOLDS_TRAINING):\n    run = neptune.init_run(\n            project=\"techinix/HMS---Harmful-Brain-Activity-Classification\",\n            api_token=token,\n        )  # your credentials\n\n    args = (\"no\", 42,run)\n    #notebook_launcher(training_loop, args, num_processes=2)\n    training_folds(*args)\n    run.stop()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LAUNCH_FULL_TRAINING=True\nif(LAUNCH_FULL_TRAINING):\n    run = neptune.init_run(project=\"techinix/HMS---Harmful-Brain-Activity-Classification\",\n                    api_token=token,with_id=\"HMSHAR-60\")\n    config = run[f\"configuration\"].fetch()\n    print(config)\n    seeds = [42,85,10]\n    args = (\"no\",seeds,run)\n    training_full(*args)\n    run.stop()","metadata":{},"execution_count":null,"outputs":[]}]}