{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7392733,"sourceType":"datasetVersion","datasetId":4297749},{"sourceId":7392775,"sourceType":"datasetVersion","datasetId":4297782},{"sourceId":7551663,"sourceType":"datasetVersion","datasetId":4398343},{"sourceId":7598753,"sourceType":"datasetVersion","datasetId":4356455},{"sourceId":7748835,"sourceType":"datasetVersion","datasetId":4424231}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Let's look at the data we have here!!!","metadata":{}},{"cell_type":"code","source":"import pandas as pd\ntrain_df = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\ndisplay(train_df.tail())\n\nprint(f'\\n# of Unique EEG: {len(train_df.eeg_id.unique())}')\nprint(f'# of Unique Spectrogram: {len(train_df.spectrogram_id.unique())}')\nprint(f'# of Unique Patients: {len(train_df.patient_id.unique())}')","metadata":{"execution":{"iopub.status.busy":"2024-03-03T11:56:45.009742Z","iopub.execute_input":"2024-03-03T11:56:45.010080Z","iopub.status.idle":"2024-03-03T11:56:45.766567Z","shell.execute_reply.started":"2024-03-03T11:56:45.010052Z","shell.execute_reply":"2024-03-03T11:56:45.764600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Observations:**\n\n1. Multiple eeg_ids may have the same spectrogram_id.\n2. Same eeg_id may have the multiple targets.\n3. There can be different targets for the same spectrogram_id depending on the time intervals during which they were observed.\n\nFor each ***unique target in eeg_id***, Let's get corresponding *spectrogram_id* along with the time interval (min and max of spectrogram_label_offset_seconds) during which a specific brain activity was detected.\n\na.k.a ** Preprocessing**","metadata":{}},{"cell_type":"code","source":"# Group by 'eeg_id' and targets compute the minimum and maximum values of 'spectrogram_label_offset_seconds'\ngroup_by_cols = [\n 'eeg_id', 'seizure_vote', 'lpd_vote',\n 'gpd_vote', 'lrda_vote', 'grda_vote',\n 'other_vote'\n]\n\ntemp1 = train_df.groupby(group_by_cols).agg(\n min_offset_eeg=(\"eeg_label_offset_seconds\", \"min\"),\n max_offset_eeg=(\"eeg_label_offset_seconds\", \"max\"),\n min_offset_spec=(\"spectrogram_label_offset_seconds\", \"min\"),\n max_offset_spec=(\"spectrogram_label_offset_seconds\", \"max\")\n).reset_index()\n\n\n# Columns to Aggregate by first value\ncolumns_to_keep = [\n 'spectrogram_id', 'patient_id', 'expert_consensus', 'label_id'\n #'seizure_vote', 'lpd_vote','gpd_vote',\n #'lrda_vote', 'grda_vote', 'other_vote'\n]\n\ntemp2 = train_df.groupby(group_by_cols)[columns_to_keep].agg('first')\n\n# Merge based on the 'eeg_id' and targets\ntrain_df = pd.merge(temp1, temp2, on=group_by_cols, how='inner')\n\n# Convert targets into probabilities.\nTARGETS = [\n 'seizure_vote', 'lpd_vote', 'gpd_vote',\n 'lrda_vote', 'grda_vote', 'other_vote'\n]\nrow_sum = train_df[TARGETS].sum(axis=1)\ntrain_df[[s + '_prop' for s in TARGETS]] = train_df[TARGETS].div(row_sum, axis=0)\n\n# Rename the expert_consensus column to target.\ntrain_df.rename(columns={'expert_consensus': 'target'}, inplace=True)\n\n# Display\ndisplay(train_df.tail(5))\n\n# Perform garbage collection\nimport gc\n\ndel temp1, temp2, row_sum\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-03T11:56:47.260513Z","iopub.execute_input":"2024-03-03T11:56:47.260878Z","iopub.status.idle":"2024-03-03T11:56:47.463577Z","shell.execute_reply.started":"2024-03-03T11:56:47.260849Z","shell.execute_reply":"2024-03-03T11:56:47.462743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Analysis\n\n* Probability Distribution of Training Data for every class according to votes.","metadata":{}},{"cell_type":"code","source":"TRAIN_PROB_DISTRIBUTION = {}\n\n# Get the Max Vote Value\nMAX_VOTES = train_df[TARGETS].sum(axis=1).max()\n\n# Group by 'target' column and sum the specified columns for each group\nsum_of_columns = train_df.groupby('target')[TARGETS].sum()\n\n# Normalize the sums to get proportions\nsum_of_columns_normalized = sum_of_columns.div(sum_of_columns.sum(axis=1), axis=0)\n\nfor t, temp_df in sum_of_columns_normalized.iterrows():\n TRAIN_PROB_DISTRIBUTION[t] = temp_df.values\n\n# Rename Keys\nTRAIN_PROB_DISTRIBUTION = {key.lower() + '_vote': value for key, value in TRAIN_PROB_DISTRIBUTION.items()}\n\nTRAIN_PROB_DISTRIBUTION","metadata":{"execution":{"iopub.status.busy":"2024-03-03T11:56:48.759977Z","iopub.execute_input":"2024-03-03T11:56:48.760343Z","iopub.status.idle":"2024-03-03T11:56:48.784354Z","shell.execute_reply.started":"2024-03-03T11:56:48.760290Z","shell.execute_reply":"2024-03-03T11:56:48.783522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\ndef vote_normalization(labels, max_val = MAX_VOTES):\n '''\n Given an vote array, normalize it to the Max Votes.\n '''\n # Normalize the array\n normalized_labels = (labels / np.sum(labels)) * max_val\n labels = np.round(normalized_labels).astype(int)\n \n return labels\n\n# Test\nlabels = train_df.sample(1)[TARGETS].values.astype(np.float32)\nnormalized_labels = vote_normalization(labels)\n\nprint(f'Before Normalization: {labels}')\nprint(f'After Normalization: {normalized_labels}')","metadata":{"execution":{"iopub.status.busy":"2024-03-03T11:56:49.550079Z","iopub.execute_input":"2024-03-03T11:56:49.550452Z","iopub.status.idle":"2024-03-03T11:56:49.563883Z","shell.execute_reply.started":"2024-03-03T11:56:49.550421Z","shell.execute_reply":"2024-03-03T11:56:49.562822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Total unique Training data based on the ***eeg_id and targets*** are: **20183**","metadata":{}},{"cell_type":"markdown","source":"Since i am going to train the model on Spectrogram images (Following the footsteps of [CHRIS DEOTTE](https://www.kaggle.com/code/cdeotte/efficientnetb2-starter-lb-0-57/notebook) and [TAWARA](https://www.kaggle.com/code/ttahara/hms-hbac-resnet34d-baseline-training/)), Lets read a spectrogram file and convert it to image to see how it looks.\n\na.k.a ** Visualization**\n\nBefore this, Let me drop few quick knowledge bombs for better digestion of the things :p\n\n1. **The spectrogram can be seen as a three-way plot of time on the x axis, frequency on the y axis, and power as color. [[Link]](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC8901534/)**\n\n2. **Each spectrogram image has four panels: left lateral (LL), right lateral (RL), left parasagittal (LP), right parasagittal (RP).[[Link]](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7461156/) Each panel will correspond to 1 image so for a single spectrogram, we have a total of 4 images.**","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\nimport cv2\n\n# Randomly sample a row from df\nsample_row = train_df.sample(1)\nspec_id = sample_row.iloc[0]['spectrogram_id']\ntarget = sample_row.iloc[0]['target']\n\n# Read spectrogram file\nspec = pd.read_parquet(f'/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/{spec_id}.parquet')\nspec_arr = spec.fillna(0).values[:, 1:].T.astype(\"float32\") # Fill nan value with zero\n\n# Let's get the image for each panel.\nPANELS = ['LL', 'RL', 'LP', 'RP']\n\n# Calculate the number of rows and columns for the subplot\nnum_rows = 2\nnum_cols = 2\n\n# Adjust the figsize accordingly\nfig, ax = plt.subplots(num_rows, num_cols, figsize=(3, 3))\n\n# Flatten the axes array if needed for easy indexing\nax = ax.flatten()\n\n# Storing Channel wise Info\nX = np.zeros((4, 100, spec_arr.shape[1]), dtype=np.float32)\n\nfor (idx, panel) in enumerate(PANELS):\n img = spec_arr[100 * idx: 100 * idx + 100, :]\n\n # log transform (Taken from TAWARA Notebook)\n img = np.clip(img, np.exp(-4), np.exp(8))\n img = np.log(img)\n\n # normalize per image\n eps = 1e-6\n img_mean = img.mean(axis=(0, 1))\n img = img - img_mean\n img_std = img.std(axis=(0, 1))\n img = img / (img_std + eps)\n\n X[idx, :, :] = img\n\n # Resize for visualize purpose\n img_resize = cv2.resize(img, (128, 128))\n\n # Plot\n ax[idx].imshow(img_resize)\n ax[idx].set_title(f'{panel}')\n ax[idx].axis('off')\n\nfig.suptitle(f'Target: {target}', fontsize=12)\n\n# Adjust layout for better spacing\nplt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2024-03-03T11:57:04.875436Z","iopub.execute_input":"2024-03-03T11:57:04.875762Z","iopub.status.idle":"2024-03-03T11:57:05.371617Z","shell.execute_reply.started":"2024-03-03T11:57:04.875738Z","shell.execute_reply":"2024-03-03T11:57:05.370656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It's upto us that how do we like to process these images. i.e, Vertical/ Horizontal/ 2H-2V stack.","metadata":{}},{"cell_type":"markdown","source":"Let's prepare the data for training by splitting it into 5-folds for submission. We'll allocate the data to each fold based on the **patient_id**.\n\n**Code Reference:** [HMS-HBAC: ResNet34d Baseline [Training]](https://www.kaggle.com/code/ttahara/hms-hbac-resnet34d-baseline-training/)","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedGroupKFold\nN_FOLDS = 5\nRANDAM_SEED = 42\n\nsgkf = StratifiedGroupKFold(n_splits=N_FOLDS, shuffle=True, random_state=RANDAM_SEED)\n\ntrain_df[\"fold\"] = -1\n\nfor fold_id, (_, val_idx) in enumerate(\n sgkf.split(train_df, y=train_df[\"target\"], groups=train_df[\"patient_id\"])\n):\n train_df.loc[val_idx, \"fold\"] = fold_id\n \n# Display\ndisplay(train_df.tail(5))\n\nprint('\\nFoldwise Training Data: \\n')\nprint(train_df['fold'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2024-03-03T11:57:11.864886Z","iopub.execute_input":"2024-03-03T11:57:11.865232Z","iopub.status.idle":"2024-03-03T11:57:13.563052Z","shell.execute_reply.started":"2024-03-03T11:57:11.865205Z","shell.execute_reply":"2024-03-03T11:57:13.562114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We are going to use spectrogram for training, It is not efficient to read the spectrogram parquet_file every time during training so let's save all spectrograms in an array. But we don't even need to do that. Thanks to [CHRIS DEOTTE](https://www.kaggle.com/datasets/cdeotte/brain-spectrograms) for his amazing Dataset.","metadata":{}},{"cell_type":"code","source":"# READ ALL SPECTROGRAMS\nspectrograms = np.load('/kaggle/input/brain-spectrograms/specs.npy',allow_pickle=True).item() # Link - https://www.kaggle.com/datasets/cdeotte/brain-spectrograms\neeg_spectrograms = np.load('/kaggle/input/hbac-eeg-spectrograms/eeg_specs.npy',allow_pickle=True).item() # Link - https://www.kaggle.com/datasets/vikramsandu/hbac-eeg-spectrograms\nprint(f'# of Total Spectrograms: {len(spectrograms)}')\nprint(f'# of Total EEG Spectrograms: {len(eeg_spectrograms)}')","metadata":{"execution":{"iopub.status.busy":"2024-03-03T11:57:15.250184Z","iopub.execute_input":"2024-03-03T11:57:15.251151Z","iopub.status.idle":"2024-03-03T12:00:24.145295Z","shell.execute_reply.started":"2024-03-03T11:57:15.251112Z","shell.execute_reply":"2024-03-03T12:00:24.144359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's write the Pytorch Dataset Class.","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset\n\nclass HBACDataset(Dataset):\n\n def __init__(self,\n df,\n specs=spectrograms,\n eeg_specs=eeg_spectrograms,\n mode='train',\n transform=None, # Augmentations\n prob_shuffle = 0.0 # Probability of Reassigning labels\n ):\n self.df = df\n self.specs = specs\n self.eeg_specs = eeg_specs\n self.mode = mode\n self.transform = transform\n self.prob_shuffle = prob_shuffle\n\n def __len__(self):\n return len(self.df)\n\n def _normalize(self, img):\n # LOG TRANSFORM SPECTROGRAM\n img = np.clip(img, np.exp(-4), np.exp(8))\n img = np.log(img)\n # STANDARDIZE PER IMAGE\n ep = 1e-6\n m = np.nanmean(img.flatten())\n s = np.nanstd(img.flatten())\n img = (img - m) / (s + ep)\n img = np.nan_to_num(img, nan=0.0)\n return img\n \n '''\n Sample an one hot array based on given distribution.\n ''' \n @staticmethod\n def sample_by_dist(arr, p_dist):\n sampled_array = np.zeros_like(arr)\n sampled_array[np.random.choice(len(arr), p=p_dist)] = 1.\n return sampled_array\n \n '''\n Probabilistic Model for Vote Reassignment.\n '''\n def reassign_votes(self, label):\n new_label = np.zeros_like(label, dtype=np.float32)\n for (idx, num_idx) in enumerate(label.astype(np.int32)):\n p_dist = TRAIN_PROB_DISTRIBUTION[TARGETS[idx]]\n for _ in range(num_idx):\n temp_arr = np.eye(1, len(TARGETS), idx)[0]#.astype(np.float32)\n new_label += HBACDataset.sample_by_dist(temp_arr, p_dist)\n\n return new_label\n \n '''\n Given an vote array, normalize it to the Max Votes.\n '''\n def vote_normalization(self, labels, max_val = MAX_VOTES):\n # Normalize the array\n normalized_labels = (labels / np.sum(labels)) * max_val\n normalized_labels = np.round(normalized_labels).astype(int)\n return normalized_labels\n \n\n def __getitem__(self, idx):\n\n # Get the Spectrogram Interval of the Particular Activity.\n minimum_spec = self.df['min_offset_spec'].iloc[idx]\n maximum_spec = self.df['max_offset_spec'].iloc[idx]\n\n if self.mode == 'train':\n #r_spec = np.random.randint(minimum_spec, maximum_spec+1)//2\n r_spec = int((minimum_spec + maximum_spec) // 4)\n elif self.mode == 'valid':\n r_spec = int((minimum_spec + maximum_spec) // 4)\n else:\n r_spec = 0\n\n spec_id = self.df.iloc[idx]['spectrogram_id']\n\n minimum_eeg = self.df['min_offset_eeg'].iloc[idx]\n maximum_eeg = self.df['max_offset_eeg'].iloc[idx]\n eeg_id = self.df.iloc[idx]['eeg_id']\n key = f'{eeg_id}_{int(minimum_eeg)}_{int(maximum_eeg)}'\n\n input_img = []\n X = np.zeros((4, 100, 300), dtype=np.float32)\n X_es = np.zeros((4, 100, 300), dtype=np.float32)\n\n for k in range(4):\n\n # Kaggle Spectrogram Image\n img = self.specs[spec_id][r_spec:r_spec+300, k*100:(k+1)*100].T\n X[k, :, :] = img\n img = self._normalize(img)\n\n # EEG Spectrogram Image\n img_es = self.eeg_specs[key][:,:,k]\n img_es = cv2.resize(img_es, (300, 100))\n X_es[k, :, :] = img_es\n\n # Concat\n img = np.hstack([img, np.zeros((img.shape[0], 40), dtype=img.dtype), img_es])\n img = np.vstack([img, np.zeros((28, img.shape[1]), dtype=img.dtype)])\n input_img.append(img)\n\n # Adding Max Panel Image\n X = self._normalize(X)\n X_max = np.max(X, axis=0)\n\n X_max_es = np.max(X_es, axis=0)\n X_max = np.hstack([X_max, np.zeros((X_max.shape[0], 40), dtype=X_max.dtype), X_max_es])\n X_max = np.vstack([X_max, np.zeros((28, X_max.shape[1]), dtype=X_max.dtype)])\n input_img.append(X_max)\n\n input_img = np.vstack(input_img)\n input_img = input_img[..., None] # Hz x Time x Channel\n \n \n if self.transform is not None:\n input_img = self.transform(image=input_img)[\"image\"]\n \n #input_img = input_img[..., None] # Hz x Time x Channel\n \n labels = self.df.iloc[idx][TARGETS].values.astype(np.float32)\n\n # Reassign labels\n if self.mode == 'train':\n if np.random.uniform() < self.prob_shuffle:\n # Label Normalization\n labels = self.vote_normalization(labels)\n labels = self.reassign_votes(labels)\n \n \n # Normalize\n sum_labels = np.sum(labels)\n labels = labels / sum_labels\n \n return {\"data\": input_img, \"target\": torch.tensor(labels)}","metadata":{"execution":{"iopub.status.busy":"2024-03-03T12:23:24.217195Z","iopub.execute_input":"2024-03-03T12:23:24.217894Z","iopub.status.idle":"2024-03-03T12:23:24.244781Z","shell.execute_reply.started":"2024-03-03T12:23:24.217860Z","shell.execute_reply":"2024-03-03T12:23:24.243751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Custom Data Augmentations**","metadata":{}},{"cell_type":"markdown","source":"**CutMix**\n\n* This process involves, Selecting a horizontal piece of the two images of the same labels, (with confidence as 1.0) and replacing them with each other.","metadata":{}},{"cell_type":"code","source":"'''\n Applies on the Batch\n'''\nimport random\n\ndef hbac_cutmix(data, \n target,\n cutmix_thr = 0.75, # Threshold to consider samples\n max_cuts = 2,\n cut_eeg_spec = True # CutMix in EEG Specs\n ):\n \n # CutMix Data\n cutmix_data = data.clone()\n \n for label_idx in range(target.size(1)):\n # Indices with confidence score greater than cutmix_thr for particular target\n indices = torch.nonzero((target[:, label_idx] >= cutmix_thr), as_tuple=False)\n \n # Skip if less than 2 samples with confidence score 1.0\n if len(indices) < 2:\n continue\n \n for _ in range(max_cuts):\n # Original Data\n data_orig = data[indices]\n\n # Shuffle\n shuffled_indices = torch.randperm(len(indices))\n data_shuffled = data_orig[shuffled_indices]\n\n # CutMix augmentation logic\n #start = random.randint(0, margin) if random.choice([True, False]) else random.randint(300-max_size-margin, 300-max_size)\n #size = random.randint(min_size, max_size)\n start = random.randint(0, 5) * 128\n size = 100\n\n # CutMix in Specs\n cutmix_data[indices, start:start+size, :] = data_shuffled[:, :, start:start+size, :]\n\n # CutMix in EEG Specs\n #if cut_eeg_spec:\n # start = 300 + 40 + start # Size + Padding + Start\n # cutmix_data[indices, start:start+size, :] = data_shuffled[:, :, start:start+size, :]\n \n return cutmix_data, target","metadata":{"execution":{"iopub.status.busy":"2024-03-03T12:23:26.080751Z","iopub.execute_input":"2024-03-03T12:23:26.081137Z","iopub.status.idle":"2024-03-03T12:23:26.090065Z","shell.execute_reply.started":"2024-03-03T12:23:26.081108Z","shell.execute_reply":"2024-03-03T12:23:26.089018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Test if CutMix working.","metadata":{}},{"cell_type":"code","source":"# Initialize Dataset Class\nfrom torch.utils.data import DataLoader\n\ntest_df = train_df[train_df['seizure_vote_prop'] >= 0.75].reset_index(drop=True)\nds = HBACDataset(test_df, prob_shuffle=0)\ndl = DataLoader(ds, batch_size=32, shuffle=True)\n\nfor (batch_idx, batch) in enumerate(dl):\n inputs = batch['data']\n labels = batch['target']\n \n # CutMix\n cutmix_data, target = hbac_cutmix(inputs, labels)\n\n break\n\n\nf, ax = plt.subplots(2, 2, figsize=(8, 8))\nax[0,0].imshow(inputs[0])\nax[0, 0].set_title(f'Input')\nax[1,0].imshow(inputs[1])\nax[1, 0].set_title(f'Input')\nax[0,1].imshow(cutmix_data[0])\nax[0, 1].set_title(f'CutMix')\nax[1,1].imshow(cutmix_data[1])\nax[1, 1].set_title(f'CutMix')\n\n# Garbage Collection\ndel test_df, ds, dl, inputs, labels, cutmix_data, target\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-03T12:23:36.760743Z","iopub.execute_input":"2024-03-03T12:23:36.761823Z","iopub.status.idle":"2024-03-03T12:23:38.974451Z","shell.execute_reply.started":"2024-03-03T12:23:36.761789Z","shell.execute_reply":"2024-03-03T12:23:38.973472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**MixUP**\n\n* This Augmentation mixes up two images of the same category in some proportion.","metadata":{}},{"cell_type":"code","source":"'''\n Applies on the batch\n'''\nimport random\n\ndef hbac_mixup(data, \n target, \n mixup_thr=1.0\n ):\n \n # Mixup Data\n mixup_data = data.clone()\n \n for label_idx in range(target.size(1)):\n # Indices with confidence score greater than cutmix_thr for particular target\n indices = torch.nonzero((target[:, label_idx] >= mixup_thr), as_tuple=False)\n \n # Skip if less than 2 samples with confidence score 1.0\n if len(indices) < 2:\n continue\n \n # Original Data\n data_orig = data[indices]\n \n # Shuffle\n shuffled_indices = torch.randperm(len(indices))\n data_shuffled = data_orig[shuffled_indices]\n \n # MixUp\n mixup_weight = random.random() \n mixup_data[indices, :, :] = mixup_weight * mixup_data[indices, :, :] + (1-mixup_weight) * data_shuffled\n \n return mixup_data, target","metadata":{"execution":{"iopub.status.busy":"2024-03-03T12:16:43.060267Z","iopub.execute_input":"2024-03-03T12:16:43.061135Z","iopub.status.idle":"2024-03-03T12:16:43.068531Z","shell.execute_reply.started":"2024-03-03T12:16:43.061102Z","shell.execute_reply":"2024-03-03T12:16:43.067561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test\n# Initialize Dataset Class\nfrom torch.utils.data import DataLoader\n\ntest_df = train_df[train_df['seizure_vote_prop'] >= 1.0].reset_index(drop=True)\nds = HBACDataset(test_df)\ndl = DataLoader(ds, batch_size=32, shuffle=True)\n\nfor (batch_idx, batch) in enumerate(dl):\n inputs = batch['data']\n labels = batch['target']\n \n # CutMix\n mixup_data, target = hbac_mixup(inputs, labels)\n break\n\nf, ax = plt.subplots(2, 2, figsize=(8, 8))\nax[0,0].imshow(inputs[0])\nax[0, 0].set_title(f'Input')\nax[1,0].imshow(inputs[1])\nax[1, 0].set_title(f'Input')\nax[0,1].imshow(mixup_data[0])\nax[0, 1].set_title(f'MixUp')\nax[1,1].imshow(mixup_data[1])\nax[1, 1].set_title(f'MixUp')\n\n\n# Garbage Collection\ndel test_df, ds, dl, inputs, labels, mixup_data, target\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-03T12:16:44.460139Z","iopub.execute_input":"2024-03-03T12:16:44.460836Z","iopub.status.idle":"2024-03-03T12:16:46.625462Z","shell.execute_reply.started":"2024-03-03T12:16:44.460805Z","shell.execute_reply":"2024-03-03T12:16:46.624416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Other Augmentations**","metadata":{}},{"cell_type":"code","source":"from copy import deepcopy\nimport albumentations as A\n\nimport random\nimport numpy as np\n\ndef hbac_cutout(img, max_percent=0.20):\n '''\n Cutout Time window but not from the central.\n '''\n img_copy = np.copy(img)\n #h, w = img_copy.shape[:2]\n w = 300\n max_n = int(w * max_percent)\n\n # Cutout either from beginning, ending, or both\n n = random.randint(0, max_n)\n m = random.randint(0, max_n)\n\n prob = np.random.uniform()\n\n if prob <= 0.40:\n # Cutout from the start\n img_copy[:, :n] = 0\n elif 0.40 < prob <= 0.80:\n # Cutout from the end\n img_copy[:, w-m:w] = 0\n else:\n # Both\n img_copy[:, :n] = 0\n img_copy[:, w-m:w] = 0\n\n return img_copy\n\n\n''' Albumentation Wrapper for this function'''\n\nfrom albumentations.core.transforms_interface import ImageOnlyTransform\n\nclass HBACCutout(ImageOnlyTransform):\n def apply(self, img, **params):\n return hbac_cutout(img)\n \n#--------------------------------------------------------------------------\n\ndef hbac_translate_and_mix(img, \n max_translation = 50, # In Pixels\n min_mix_prob = 0.75\n ):\n \n '''\n Translate and MixUp of the same image.\n '''\n \n # Tanslate\n t = np.random.randint(-max_translation, max_translation) # Amount of Translation\n \n M = np.float32([[1, 0, t], [0, 1, 0]])\n translated_image = cv2.warpAffine(img, M, (img.shape[1], img.shape[0]))\n translated_image = translated_image[..., None] \n \n # Mix\n m = np.random.uniform(min_mix_prob, 1) # MixUp Amount\n img = m * img + (1-m) * translated_image\n \n # Reset the Middle Part to Zero.\n img[:, 300: 340] = 0\n \n return img\n\n\nclass HBACTransMix(ImageOnlyTransform):\n def apply(self, img, **params):\n return hbac_translate_and_mix(img)\n \n# ----------------------------------------------------------------------------\n\ndef hbac_random_crop_resizing(img, \n min_crop_size=100, # Minimum Cropping Size\n max_crop_size=300 # Maximum Cropping Size\n ):\n \n '''\n Randomly Crop and Resize it back to the original Size.\n '''\n img_copy = np.copy(img)\n \n # Algorithm\n for i in range(5):\n \n # Params\n middle = max_crop_size//2\n start = middle - random.randint(min_crop_size//2, max_crop_size//2)\n end = middle + random.randint(min_crop_size//2, max_crop_size//2)\n \n # Randomly Resize Specs\n temp1 = img_copy[128*i:128*i+100, start:end]\n temp1 = cv2.resize(temp1, (300, 100))\n img_copy[128*i:128*i+100, 0:300, 0] = temp1\n \n # Randomly Resize EEG-Specs\n temp2 = img_copy[i:i+100, start+40+300:end+40+300]\n temp2 = cv2.resize(temp2, (300, 100))\n img_copy[128*i:128*i+100, 300+40:600+40, 0] = temp2\n \n return img_copy\n\nclass HBACRandomResizedCrop(ImageOnlyTransform):\n def apply(self, img, **params):\n return hbac_random_crop_resizing(img)","metadata":{"execution":{"iopub.status.busy":"2024-03-03T12:16:54.403005Z","iopub.execute_input":"2024-03-03T12:16:54.403615Z","iopub.status.idle":"2024-03-03T12:16:56.099467Z","shell.execute_reply.started":"2024-03-03T12:16:54.403582Z","shell.execute_reply":"2024-03-03T12:16:56.098681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test\nimport random\nimport albumentations as A\n\ntransform = A.Compose([\n HBACTransMix(p = 1.0),\n])\n\n# Initialize Dataset Class\nds = HBACDataset(train_df, transform=None)\n\n# Get a sample\nsample = ds[random.randint(0, len(ds))]\n\n# Get Data and Label from sample\ndata = sample['data']\nlabel = TARGETS[sample['target'].argmax().item()]\n\n# Plot\nplt.figure(figsize=(8, 8))\nplt.imshow(data)\nplt.title(f'Target: {label}')\n\n# Garbage Collection\ndel ds, data, label\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-03T12:23:55.369799Z","iopub.execute_input":"2024-03-03T12:23:55.370150Z","iopub.status.idle":"2024-03-03T12:23:56.230851Z","shell.execute_reply.started":"2024-03-03T12:23:55.370123Z","shell.execute_reply":"2024-03-03T12:23:56.229198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's create a model that we are going to use for Training. \n\n* Among the shared code within the community for this competition, **ResNet34 and EfficientNet** emerge as the most prominent models with the highest public score notebooks.\n\n* We'll start our exploration with these models.\n\n**Code Reference:** [HMS-HBAC: ResNet34d Baseline [Training]](https://www.kaggle.com/code/ttahara/hms-hbac-resnet34d-baseline-training/)","metadata":{}},{"cell_type":"code","source":"import timm\nimport torch.nn as nn\n\nclass HBACSpecModel(nn.Module):\n\n def __init__(\n self,\n model_name: str,\n pretrained: bool,\n in_channels: int,\n num_classes: int,\n ):\n super().__init__()\n self.model = timm.create_model(\n model_name=model_name, pretrained=pretrained,\n num_classes=num_classes, in_chans=in_channels)\n\n def forward(self, x):\n x = self.model(x)\n return x","metadata":{"execution":{"iopub.status.busy":"2024-03-03T12:24:00.909769Z","iopub.execute_input":"2024-03-03T12:24:00.910148Z","iopub.status.idle":"2024-03-03T12:24:00.916702Z","shell.execute_reply.started":"2024-03-03T12:24:00.910117Z","shell.execute_reply":"2024-03-03T12:24:00.915496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since KLDivLoss align with the competition metric, we will use KLDiv Loss as Loss Function.","metadata":{}},{"cell_type":"code","source":"class KLDivLossWithLogits(nn.KLDivLoss):\n\n def __init__(self):\n super().__init__(reduction=\"batchmean\")\n\n def forward(self, y, t):\n y = nn.functional.log_softmax(y, dim=1)\n loss = super().forward(y, t)\n\n return loss","metadata":{"execution":{"iopub.status.busy":"2024-03-03T12:24:01.679713Z","iopub.execute_input":"2024-03-03T12:24:01.680071Z","iopub.status.idle":"2024-03-03T12:24:01.685553Z","shell.execute_reply.started":"2024-03-03T12:24:01.680042Z","shell.execute_reply":"2024-03-03T12:24:01.684650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"import albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom torch.utils.data import DataLoader\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.nn.parallel import DataParallel\nfrom torch import optim\nfrom torch.optim import lr_scheduler\n\n# Check if GPU is available.\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f'DEVICE: {device}\\n')\n\n# Transformations\ntrain_transform = A.Compose([\n #HBACRandomResizedCrop(p=0.4),\n ToTensorV2(p=1.0)\n])\n\nval_transform = A.Compose([\n ToTensorV2(p=1.0)\n])\n \n# Loop through all folds. (5-Fold)\nfor fold in range(5):\n print('----------------------------------------------------------')\n print(f'\\nFold: {fold}')\n \n # Train-Valid Split for the Fold\n train = train_df[train_df['fold'] != fold].reset_index(drop=True)\n valid = train_df[train_df['fold'] == fold].reset_index(drop=True)\n \n # Dataset with Dynamic Vote Asignment (Only For Training)\n train_ds = HBACDataset(train, transform=train_transform, mode='train')\n valid_ds = HBACDataset(valid, transform=val_transform, mode='valid')\n\n print(f'Train - {len(train_ds)}, Test - {len(valid_ds)}')\n \n # Dataloader\n batch_size = 32\n train_dataloader = DataLoader(train_ds, batch_size=batch_size, shuffle=True)\n valid_dataloader = DataLoader(valid_ds, batch_size=batch_size, shuffle=False)\n \n # Initialize the Model\n model = HBACSpecModel(model_name='efficientnet_b2', \n pretrained=True, \n in_channels=1, \n num_classes=6\n )\n \n # Used for Mixed Precision Training\n scaler = GradScaler()\n\n # Move your model to the GPU and wrap it with DataParallel (To utilize T4X2)\n model = model.to(device)\n model = DataParallel(model)\n \n # Loss Function\n loss_func = KLDivLossWithLogits()\n\n # Prepare optimizer\n optimizer = optim.AdamW(model.parameters(),\n lr=1.0e-03,\n weight_decay = 1.0e-02\n )\n NUM_EPOCHS = 5\n\n # 1cycle policy\n scheduler = lr_scheduler.OneCycleLR(\n optimizer=optimizer, epochs=NUM_EPOCHS,\n pct_start=0.0, steps_per_epoch=len(train_dataloader),\n max_lr=1.0e-03, div_factor=25, final_div_factor=4.0e-01\n )\n\n best_eval_loss = float('inf') # Initialize the best evaluation loss to infinity\n\n # Do Evaluation 2 Times in every Epochs\n doEval = False\n eval_step = len(train_dataloader)//2 - 1\n \n print('\\n--------------------------------------------')\n print(f'\\033[93mEpoch Step Train-Loss Valid-Loss\\033[0m')\n print('--------------------------------------------')\n \n for epoch in range(NUM_EPOCHS):\n \n total_loss = 0\n \n for (batch_idx, batch) in enumerate(train_dataloader):\n model.train() # Set the model to Training mode\n with autocast():\n inputs = batch['data'].to(device)\n labels = batch['target'].to(device)\n \n # MixUp Augmentation\n if np.random.uniform() <= 0.35:\n inputs, labels = hbac_cutmix(inputs, labels)\n\n # Forward loop\n optimizer.zero_grad() # Ensures Gradient doesn't accumulate.\n\n predictions = model(inputs)\n\n # Compute KLDIV loss\n loss = loss_func(predictions, labels)\n\n # BackProp\n scaler.scale(loss).backward() # Scale the loss value\n scaler.step(optimizer)\n scaler.update()\n\n # Accumulate the Loss\n total_loss += loss.item()\n\n if (batch_idx != 0) and (batch_idx%eval_step == 0):\n # Calculate epoch-level metrics\n epoch_loss = total_loss / len(train_dataloader)\n\n # Evaluation\n model.eval() # Set model to evaluation mode\n eval_loss = 0\n\n with torch.no_grad():\n for batch in valid_dataloader:\n with autocast():\n inputs = batch['data'].to(device)\n labels = batch['target'].to(device)\n\n predictions = model(inputs)\n loss = loss_func(predictions, labels)\n\n eval_loss += loss.item()\n\n # Calculate evaluation metrics\n eval_epoch_loss = eval_loss / len(valid_dataloader)\n\n print(f'{epoch+1:<10} {(batch_idx//eval_step):<8} {epoch_loss:<10.4f}', end = \" \")\n\n # Save the Best Model\n if eval_epoch_loss < best_eval_loss:\n best_eval_loss = eval_epoch_loss\n print(f'\\033[32m{best_eval_loss:<10.4f}\\033[0m')\n # Access the actual model from the DataParallel object\n actual_model = model.module\n # Save\n torch.save(actual_model, f'EffnetB2_eeg_specs_panelCutMix_Max_2_Prob_0_35_fold_{fold}.pth')\n\n else:\n print(f'\\033[31m{eval_epoch_loss:<10.4f}\\033[0m')\n \n \n # Memory Management\n del train_ds, valid_ds, train_dataloader, valid_dataloader, model, actual_model\n gc.collect()\n torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2024-03-03T12:24:21.749510Z","iopub.execute_input":"2024-03-03T12:24:21.749904Z","iopub.status.idle":"2024-03-03T17:04:02.906486Z","shell.execute_reply.started":"2024-03-03T12:24:21.749874Z","shell.execute_reply":"2024-03-03T17:04:02.905652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's Check the OOF Score.","metadata":{}},{"cell_type":"code","source":"from tqdm import tqdm\n\nlabel_arr = train_df[TARGETS].values\noof_pred_arr = np.zeros((len(train_df), 6))\n\nfor fold in range(5):\n \n # Get valid Dataloader\n valid = train_df[train_df['fold'] == fold].reset_index()\n valid_ds = HBACDataset(valid, transform=val_transform, mode='valid')\n valid_dataloader = DataLoader(valid_ds, batch_size=64, shuffle=False)\n \n # Get Model\n model = torch.load( f'/kaggle/working/EffnetB2_eeg_specs_panelCutMix_Max_2_Prob_0_35_fold_{fold}.pth')\n \n # Run Inference\n model.to(device)\n model.eval()\n pred_list = []\n with torch.no_grad():\n for batch in tqdm(valid_dataloader):\n x = batch[\"data\"].to(device)\n y = model(x)\n pred_list.append(y.softmax(dim=1).detach().cpu().numpy())\n \n pred_arr = np.concatenate(pred_list)\n oof_pred_arr[valid['index'].values] = pred_arr\n \n del model, valid, valid_ds, valid_dataloader\n torch.cuda.empty_cache()\n gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-03T17:18:44.788395Z","iopub.execute_input":"2024-03-03T17:18:44.788778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert Target to probs\nrow_sum = train_df[TARGETS].sum(axis=1)\ntrain_df[TARGETS] = train_df[TARGETS].div(row_sum, axis=0)\ntrain_df.tail()","metadata":{"execution":{"iopub.status.busy":"2024-02-25T11:47:24.970400Z","iopub.execute_input":"2024-02-25T11:47:24.971385Z","iopub.status.idle":"2024-02-25T11:47:25.011493Z","shell.execute_reply.started":"2024-02-25T11:47:24.971353Z","shell.execute_reply":"2024-02-25T11:47:25.010595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/kaggle-kl-div')\nfrom kaggle_kl_div import score\n\ntrue = train_df[[\"label_id\"] + TARGETS].copy()\ntrue_copy = true.copy()\noof = pd.DataFrame(oof_pred_arr, columns=TARGETS)\noof.insert(0, \"label_id\", train_df[\"label_id\"])\noof_copy = oof.copy()\n\ncv_score = score(solution=true, submission=oof, row_id_column_name='label_id')\nprint('CV Score KL-Div for EfficientNet-B0',cv_score)","metadata":{"execution":{"iopub.status.busy":"2024-02-25T11:47:25.808007Z","iopub.execute_input":"2024-02-25T11:47:25.808381Z","iopub.status.idle":"2024-02-25T11:47:25.869677Z","shell.execute_reply.started":"2024-02-25T11:47:25.808352Z","shell.execute_reply":"2024-02-25T11:47:25.868755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Classwise KLD Score Analysis","metadata":{}},{"cell_type":"code","source":"for t in train_df['target'].unique():\n t_labels = train_df[train_df['target']==t]['label_id'].values\n true_filtered_df = true_copy[true_copy['label_id'].isin(t_labels)].reset_index(drop=True)\n oof_filtered_df = oof_copy[oof_copy['label_id'].isin(t_labels)].reset_index(drop=True)\n cv_score = score(solution=true_filtered_df, submission=oof_filtered_df, row_id_column_name='label_id')\n print(f'Target: {t} CV-Score: {cv_score}')","metadata":{"execution":{"iopub.status.busy":"2024-02-25T11:47:38.871788Z","iopub.execute_input":"2024-02-25T11:47:38.872139Z","iopub.status.idle":"2024-02-25T11:47:39.088017Z","shell.execute_reply.started":"2024-02-25T11:47:38.872114Z","shell.execute_reply":"2024-02-25T11:47:39.087068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Class CutMix-CV MixUp-CV D/I AuxLoss-CV D/I\n#--------------------------------------------------------------------\n# Seizure 0.8386 0.7313 D 0.6749 D\n# LPD 0.6017 0.6805 I 0.7092 I\n# GPD 0.6080 0.5625 D 0.5502 D \n# LRDA 1.2434 1.1401 D 1.1756 D\n# GRDA 0.8362 0.8805 I 0.8796 I\n# Other 0.4649 0.4569 D 0.4585 D\n#--------------------------------------------------------------------\n# OOF-CV 0.6551 0.6377 D 0.6326 D\n# LB 0.39 0.41 I 0.42 I","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Performance Tracker","metadata":{}},{"cell_type":"markdown","source":"**1. Efficientnet-b2 + EEG Spectrograms + CutMix-Panel (Max-2) + Prob(p=0.35)**\n\n\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
FoldModel_nameCVLB
0 Efficientnet-b20.6227NA
1 Efficientnet-b20.6455NA
2 Efficientnet-b20.6584NA
3 Efficientnet-b20.7219NA
4 Efficientnet-b20.6441NA
OOF/5-Fold Efficientnet-b2NANA
","metadata":{}},{"cell_type":"markdown","source":"**1. Efficientnet-b2 + EEG Spectrograms + LabelNorm + ProbShffle(p=0.2)**\n\n\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
FoldModel_nameCVLB
0 Efficientnet-b20.6134NA
1 Efficientnet-b20.6256NA
2 Efficientnet-b20.6265NA
3 Efficientnet-b20.6821NA
4 Efficientnet-b20.6287NA
OOF/5-Fold Efficientnet-b20.63620.39
","metadata":{}},{"cell_type":"markdown","source":"**Ideas**\n\n1. Test Time Augmentation (Random Resizing Crop)","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}