{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":14030,"sourceType":"modelInstanceVersion","modelInstanceId":11617}],"dockerImageVersionId":30665,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Setup","metadata":{}},{"cell_type":"code","source":"import warnings\nimport timm\nfrom fastai.vision.all import *\nfrom fastcore.parallel import *\n\npath = Path('/kaggle/input/hms-harmful-brain-activity-classification')\n\npath.ls()","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-06T22:18:42.769565Z","iopub.execute_input":"2024-03-06T22:18:42.770204Z","iopub.status.idle":"2024-03-06T22:18:42.778376Z","shell.execute_reply.started":"2024-03-06T22:18:42.770172Z","shell.execute_reply":"2024-03-06T22:18:42.777463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Background\n\nIn this notebook, I will submit predictions from the resnet34 trained in [my previous notebook](https://www.kaggle.com/code/vishalbakshi/hms-hbac-fastai-stacked-images-train) after applying Test Time Augmentation (TTA) to them.\n\nI am referencing the following notebooks:\n\n- [HMS - HBAC - Fastai Starter](https://www.kaggle.com/code/sonujha090/hms-hbac-fastai-starter)\n- [HMS-HBAC: KerasCV Starter Notebook](https://www.kaggle.com/code/awsaf49/hms-hbac-kerascv-starter-notebook)\n- [Small models: Road to the Top, Part 2](https://www.kaggle.com/code/jhoward/small-models-road-to-the-top-part-2/#Test-time-augmentation) (and the rest of the series)","metadata":{}},{"cell_type":"markdown","source":"## Converting Training and Test Data to Images","metadata":{}},{"cell_type":"code","source":"# create temporary folders to hold spectrograms\nSPEC_DIR = \"/tmp/dataset/hms-hbac\"\nos.makedirs(SPEC_DIR+'/train_spectrograms', exist_ok=True)\nos.makedirs(SPEC_DIR+'/test_spectrograms', exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-06T22:18:51.555295Z","iopub.execute_input":"2024-03-06T22:18:51.555658Z","iopub.status.idle":"2024-03-06T22:18:51.560951Z","shell.execute_reply.started":"2024-03-06T22:18:51.555629Z","shell.execute_reply":"2024-03-06T22:18:51.560011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_spec(spec_id, split=\"train\"):\n    # read the data\n    data = pd.read_parquet(path/f'{split}_spectrograms'/f'{spec_id}.parquet')\n    \n    # replace NA with 0\n    data = data.fillna(0)\n    \n    # convert DataFrame to array\n    data = data.values[:, 1:]\n    \n    # transpose\n    data = data.T\n    data = data.astype(\"float32\")\n    \n    # convert array to PILImage\n    im = PILImage.create(Image.fromarray((data * 255).astype(np.uint8)))\n    im.save(f\"{SPEC_DIR}/{split}_spectrograms/{spec_id}.png\")","metadata":{"execution":{"iopub.status.busy":"2024-03-06T22:18:52.444237Z","iopub.execute_input":"2024-03-06T22:18:52.445049Z","iopub.status.idle":"2024-03-06T22:18:52.450887Z","shell.execute_reply.started":"2024-03-06T22:18:52.445015Z","shell.execute_reply":"2024-03-06T22:18:52.450002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I'll load the training data so I can get all of the `spectrogram_id` values:","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(path/'train.csv')\ndf.head(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-06T22:18:54.194391Z","iopub.execute_input":"2024-03-06T22:18:54.194876Z","iopub.status.idle":"2024-03-06T22:18:54.447563Z","shell.execute_reply.started":"2024-03-06T22:18:54.194837Z","shell.execute_reply":"2024-03-06T22:18:54.446621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spec_ids = df[\"spectrogram_id\"].unique()","metadata":{"execution":{"iopub.status.busy":"2024-03-06T22:18:54.507083Z","iopub.execute_input":"2024-03-06T22:18:54.507358Z","iopub.status.idle":"2024-03-06T22:18:54.516831Z","shell.execute_reply.started":"2024-03-06T22:18:54.507336Z","shell.execute_reply":"2024-03-06T22:18:54.515885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(spec_ids)","metadata":{"execution":{"iopub.status.busy":"2024-03-06T22:18:55.950416Z","iopub.execute_input":"2024-03-06T22:18:55.950776Z","iopub.status.idle":"2024-03-06T22:18:55.956592Z","shell.execute_reply.started":"2024-03-06T22:18:55.950746Z","shell.execute_reply":"2024-03-06T22:18:55.955669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"And then save them as image files using `fastcore.parallel` which takes about four and a half minutes.","metadata":{}},{"cell_type":"code","source":"warnings.filterwarnings(\"ignore\")\nparallel(process_spec, spec_ids, split='train', n_workers=4)\nwarnings.filterwarnings(\"default\")","metadata":{"execution":{"iopub.status.busy":"2024-03-06T22:18:56.831038Z","iopub.execute_input":"2024-03-06T22:18:56.831839Z","iopub.status.idle":"2024-03-06T22:23:05.102825Z","shell.execute_reply.started":"2024-03-06T22:18:56.831809Z","shell.execute_reply":"2024-03-06T22:23:05.101879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I'll load a training spectrogram image to make sure:","metadata":{}},{"cell_type":"code","source":"PILImage.create(Path('/tmp/dataset/hms-hbac/train_spectrograms').ls()[0])","metadata":{"execution":{"iopub.status.busy":"2024-03-06T22:23:12.151689Z","iopub.execute_input":"2024-03-06T22:23:12.152531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Looks good!","metadata":{}},{"cell_type":"markdown","source":"I'll then prep the test data:","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv(path/'test.csv')\ntest_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-06T22:23:25.748994Z","iopub.execute_input":"2024-03-06T22:23:25.749350Z","iopub.status.idle":"2024-03-06T22:23:25.765208Z","shell.execute_reply.started":"2024-03-06T22:23:25.749314Z","shell.execute_reply":"2024-03-06T22:23:25.764185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spec_ids = test_df['spectrogram_id'].unique()\nlen(spec_ids)","metadata":{"execution":{"iopub.status.busy":"2024-03-06T22:23:26.627140Z","iopub.execute_input":"2024-03-06T22:23:26.627490Z","iopub.status.idle":"2024-03-06T22:23:26.633802Z","shell.execute_reply.started":"2024-03-06T22:23:26.627462Z","shell.execute_reply":"2024-03-06T22:23:26.632941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warnings.filterwarnings(\"ignore\")\nparallel(process_spec, spec_ids, split='test', n_workers=4)\nwarnings.filterwarnings(\"default\")","metadata":{"execution":{"iopub.status.busy":"2024-03-06T22:23:28.726572Z","iopub.execute_input":"2024-03-06T22:23:28.726926Z","iopub.status.idle":"2024-03-06T22:23:29.003173Z","shell.execute_reply.started":"2024-03-06T22:23:28.726898Z","shell.execute_reply":"2024-03-06T22:23:29.002112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PILImage.create(Path('/tmp/dataset/hms-hbac/test_spectrograms').ls()[0])","metadata":{"execution":{"iopub.status.busy":"2024-03-06T22:24:28.207371Z","iopub.execute_input":"2024-03-06T22:24:28.207753Z","iopub.status.idle":"2024-03-06T22:24:28.263055Z","shell.execute_reply.started":"2024-03-06T22:24:28.207720Z","shell.execute_reply":"2024-03-06T22:24:28.262167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating the `DataLoaders` Object","metadata":{}},{"cell_type":"markdown","source":"I'll wrap all of my code to prep data for training into a single cell.","metadata":{}},{"cell_type":"code","source":"df['img_path'] = '/tmp/dataset/hms-hbac/train_spectrograms/' + df['spectrogram_id'].astype(str) + '.png'\n    \ncols = ['eeg_id', 'spectrogram_id', 'img_path', 'seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']\nagg_funcs = {c: 'sum' for c in cols if 'vote' in c}\n\nunique_df = df[cols].groupby(['eeg_id', 'spectrogram_id', 'img_path'], as_index=False).agg(agg_funcs)\nunique_df['target'] = unique_df[[c for c in cols if 'vote' in c]].idxmax(axis=1)\n    \ntrain_bool = [False for _ in range(int(0.8 * len(unique_df)))]\nvalid_bool = [True for _ in range(len(unique_df) - int(0.8 * len(unique_df)))]\nis_valid_bool = pd.Series(train_bool + valid_bool).sample(frac=1).reset_index(drop=True)\n\nunique_df[\"is_valid\"] = is_valid_bool\nunique_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-06T22:24:37.330520Z","iopub.execute_input":"2024-03-06T22:24:37.330904Z","iopub.status.idle":"2024-03-06T22:24:37.528926Z","shell.execute_reply.started":"2024-03-06T22:24:37.330874Z","shell.execute_reply":"2024-03-06T22:24:37.527763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now I can create my `DataBlock`. Since I now have paths to images ready to train on, I can use `ImageBlock` for my inputs. I'll also make sure to `Resize` my images so they are 224 x 224 squares.","metadata":{}},{"cell_type":"code","source":"dblock = DataBlock(\n            blocks=(ImageBlock, CategoryBlock),\n            splitter=ColSplitter(),\n            get_x=ColReader('img_path'),\n            get_y=ColReader('target'),\n            item_tfms=Resize(224, method='squish'))\n\ndls = dblock.dataloaders(unique_df)","metadata":{"execution":{"iopub.status.busy":"2024-03-06T22:24:39.715503Z","iopub.execute_input":"2024-03-06T22:24:39.715875Z","iopub.status.idle":"2024-03-06T22:24:41.189668Z","shell.execute_reply.started":"2024-03-06T22:24:39.715846Z","shell.execute_reply":"2024-03-06T22:24:41.188256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.show_batch(nrows=1, ncols=3)","metadata":{"execution":{"iopub.status.busy":"2024-03-06T22:24:42.893965Z","iopub.execute_input":"2024-03-06T22:24:42.894697Z","iopub.status.idle":"2024-03-06T22:24:44.159217Z","shell.execute_reply.started":"2024-03-06T22:24:42.894662Z","shell.execute_reply":"2024-03-06T22:24:44.158250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check dls vocab\ndls.vocab","metadata":{"execution":{"iopub.status.busy":"2024-03-06T22:24:46.563430Z","iopub.execute_input":"2024-03-06T22:24:46.564292Z","iopub.status.idle":"2024-03-06T22:24:46.569916Z","shell.execute_reply.started":"2024-03-06T22:24:46.564260Z","shell.execute_reply":"2024-03-06T22:24:46.568958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"fastai uses `timm` when you specify the architecture as a string:","metadata":{}},{"cell_type":"code","source":"learn = vision_learner(dls, 'resnet34' ,metrics=accuracy, pretrained=False)","metadata":{"execution":{"iopub.status.busy":"2024-03-06T22:24:47.782305Z","iopub.execute_input":"2024-03-06T22:24:47.783223Z","iopub.status.idle":"2024-03-06T22:24:48.238852Z","shell.execute_reply.started":"2024-03-06T22:24:47.783182Z","shell.execute_reply":"2024-03-06T22:24:48.237860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.load('/kaggle/input/hms-hbac-resnet34/pytorch/2/1/hms_hbac_resnet34_stacked')","metadata":{"execution":{"iopub.status.busy":"2024-03-06T22:24:50.015419Z","iopub.execute_input":"2024-03-06T22:24:50.015868Z","iopub.status.idle":"2024-03-06T22:24:50.946020Z","shell.execute_reply.started":"2024-03-06T22:24:50.015827Z","shell.execute_reply":"2024-03-06T22:24:50.944941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['img_path'] = '/tmp/dataset/hms-hbac/test_spectrograms/' + test_df['spectrogram_id'].astype(str) + '.png'\ntst_dl = learn.dls.test_dl(test_df)\ntst_dl.show_batch()","metadata":{"execution":{"iopub.status.busy":"2024-03-06T22:24:52.679973Z","iopub.execute_input":"2024-03-06T22:24:52.680769Z","iopub.status.idle":"2024-03-06T22:24:52.804729Z","shell.execute_reply.started":"2024-03-06T22:24:52.680735Z","shell.execute_reply":"2024-03-06T22:24:52.803734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Using Test Time Augmentation","metadata":{}},{"cell_type":"markdown","source":"Test Time Augmentation (TTA) applies data augmentations to the test data before making predictions on them. I'll illustrate the difference between validation set accuracy with and without TTA.","metadata":{}},{"cell_type":"code","source":"valid = learn.dls.valid\npreds,targs = learn.get_preds(dl=valid)","metadata":{"execution":{"iopub.status.busy":"2024-03-06T22:24:55.613099Z","iopub.execute_input":"2024-03-06T22:24:55.613462Z","iopub.status.idle":"2024-03-06T22:25:12.839235Z","shell.execute_reply.started":"2024-03-06T22:24:55.613429Z","shell.execute_reply":"2024-03-06T22:25:12.838055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# validation set accuracy\naccuracy(preds, targs)","metadata":{"execution":{"iopub.status.busy":"2024-03-06T22:25:55.900861Z","iopub.execute_input":"2024-03-06T22:25:55.901297Z","iopub.status.idle":"2024-03-06T22:25:55.931177Z","shell.execute_reply.started":"2024-03-06T22:25:55.901265Z","shell.execute_reply":"2024-03-06T22:25:55.930174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tta_preds,_ = learn.tta(dl=valid)","metadata":{"execution":{"iopub.status.busy":"2024-03-06T22:26:17.911178Z","iopub.execute_input":"2024-03-06T22:26:17.912165Z","iopub.status.idle":"2024-03-06T22:27:44.418638Z","shell.execute_reply.started":"2024-03-06T22:26:17.912122Z","shell.execute_reply":"2024-03-06T22:27:44.417404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TTA validation set accuracy\naccuracy(tta_preds, targs)","metadata":{"execution":{"iopub.status.busy":"2024-03-06T22:27:48.840666Z","iopub.execute_input":"2024-03-06T22:27:48.841054Z","iopub.status.idle":"2024-03-06T22:27:48.855131Z","shell.execute_reply.started":"2024-03-06T22:27:48.841023Z","shell.execute_reply":"2024-03-06T22:27:48.854039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Interesting---I'm getting exactly the same accuracy with and without TTA. I'll have to experiment further when I do the next step in this process.","metadata":{}},{"cell_type":"markdown","source":"I'll still try out TTA predictions for the test set and see how they score. If I get a score less than 2.02, I'll be inclined to use TTA moving forward.","metadata":{}},{"cell_type":"code","source":"probs,_= learn.tta(dl=tst_dl)\nprobs_df = pd.DataFrame(probs, columns=dls.vocab)\nprobs_df['eeg_id'] = test_df['eeg_id']\nprobs_df = probs_df[['eeg_id', 'seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']]\nprobs_df","metadata":{"execution":{"iopub.status.busy":"2024-03-06T22:29:16.468734Z","iopub.execute_input":"2024-03-06T22:29:16.469555Z","iopub.status.idle":"2024-03-06T22:29:17.463557Z","shell.execute_reply.started":"2024-03-06T22:29:16.469516Z","shell.execute_reply":"2024-03-06T22:29:17.462331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"probs_df[['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']].sum(axis=1) == 1","metadata":{"execution":{"iopub.status.busy":"2024-03-06T22:29:31.612675Z","iopub.execute_input":"2024-03-06T22:29:31.613049Z","iopub.status.idle":"2024-03-06T22:29:31.623215Z","shell.execute_reply.started":"2024-03-06T22:29:31.613019Z","shell.execute_reply":"2024-03-06T22:29:31.622172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"probs_df.to_csv('submission.csv', index=False)\n!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2024-03-06T22:29:34.015209Z","iopub.execute_input":"2024-03-06T22:29:34.015572Z","iopub.status.idle":"2024-03-06T22:29:35.000749Z","shell.execute_reply.started":"2024-03-06T22:29:34.015543Z","shell.execute_reply":"2024-03-06T22:29:34.999633Z"},"trusted":true},"execution_count":null,"outputs":[]}]}