{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":14030,"sourceType":"modelInstanceVersion","modelInstanceId":11617}],"dockerImageVersionId":30665,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Setup","metadata":{}},{"cell_type":"code","source":"import warnings\nimport timm\nfrom fastai.vision.all import *\nfrom fastcore.parallel import *\n\npath = Path('/kaggle/input/hms-harmful-brain-activity-classification')\n\npath.ls()","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-05T20:40:56.058138Z","iopub.execute_input":"2024-03-05T20:40:56.058757Z","iopub.status.idle":"2024-03-05T20:41:11.428135Z","shell.execute_reply.started":"2024-03-05T20:40:56.058722Z","shell.execute_reply":"2024-03-05T20:41:11.427149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Background\n\nIn this notebook, I will submit predictions from the resnet34 trained in [my previous notebook](https://www.kaggle.com/code/vishalbakshi/hms-hbac-fastai-stacked-images-train).","metadata":{}},{"cell_type":"markdown","source":"## Converting Training and Test Data to Images","metadata":{}},{"cell_type":"code","source":"# create temporary folders to hold spectrograms\nSPEC_DIR = \"/tmp/dataset/hms-hbac\"\nos.makedirs(SPEC_DIR+'/train_spectrograms', exist_ok=True)\nos.makedirs(SPEC_DIR+'/test_spectrograms', exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-05T20:41:39.268675Z","iopub.execute_input":"2024-03-05T20:41:39.269712Z","iopub.status.idle":"2024-03-05T20:41:39.274548Z","shell.execute_reply.started":"2024-03-05T20:41:39.269675Z","shell.execute_reply":"2024-03-05T20:41:39.273600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_spec(spec_id, split=\"train\"):\n    # read the data\n    data = pd.read_parquet(path/f'{split}_spectrograms'/f'{spec_id}.parquet')\n    \n    # replace NA with 0\n    data = data.fillna(0)\n    \n    # convert DataFrame to array\n    data = data.values[:, 1:]\n    \n    # transpose\n    data = data.T\n    data = data.astype(\"float32\")\n    \n    # convert array to PILImage\n    im = PILImage.create(Image.fromarray((data * 255).astype(np.uint8)))\n    im.save(f\"{SPEC_DIR}/{split}_spectrograms/{spec_id}.png\")","metadata":{"execution":{"iopub.status.busy":"2024-03-05T20:42:43.360205Z","iopub.execute_input":"2024-03-05T20:42:43.360566Z","iopub.status.idle":"2024-03-05T20:42:43.367041Z","shell.execute_reply.started":"2024-03-05T20:42:43.360539Z","shell.execute_reply":"2024-03-05T20:42:43.365961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I'll load the training data so I can get all of the `spectrogram_id` values:","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(path/'train.csv')\ndf.head(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-05T20:42:44.203984Z","iopub.execute_input":"2024-03-05T20:42:44.204947Z","iopub.status.idle":"2024-03-05T20:42:44.507745Z","shell.execute_reply.started":"2024-03-05T20:42:44.204905Z","shell.execute_reply":"2024-03-05T20:42:44.506814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spec_ids = df[\"spectrogram_id\"].unique()","metadata":{"execution":{"iopub.status.busy":"2024-03-05T20:42:47.459653Z","iopub.execute_input":"2024-03-05T20:42:47.460362Z","iopub.status.idle":"2024-03-05T20:42:47.473202Z","shell.execute_reply.started":"2024-03-05T20:42:47.460326Z","shell.execute_reply":"2024-03-05T20:42:47.472154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(spec_ids)","metadata":{"execution":{"iopub.status.busy":"2024-03-05T20:42:48.339553Z","iopub.execute_input":"2024-03-05T20:42:48.340572Z","iopub.status.idle":"2024-03-05T20:42:48.346569Z","shell.execute_reply.started":"2024-03-05T20:42:48.340534Z","shell.execute_reply":"2024-03-05T20:42:48.345423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"And then save them as image files using `fastcore.parallel` which takes about four and a half minutes.","metadata":{}},{"cell_type":"code","source":"warnings.filterwarnings(\"ignore\")\nparallel(process_spec, spec_ids, split='train', n_workers=4)\nwarnings.filterwarnings(\"default\")","metadata":{"execution":{"iopub.status.busy":"2024-03-05T20:43:00.652356Z","iopub.execute_input":"2024-03-05T20:43:00.652741Z","iopub.status.idle":"2024-03-05T20:48:23.816485Z","shell.execute_reply.started":"2024-03-05T20:43:00.652712Z","shell.execute_reply":"2024-03-05T20:48:23.815192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I'll load a training spectrogram image to make sure:","metadata":{}},{"cell_type":"code","source":"PILImage.create(Path('/tmp/dataset/hms-hbac/train_spectrograms').ls()[0])","metadata":{"execution":{"iopub.status.busy":"2024-03-05T20:49:38.382564Z","iopub.execute_input":"2024-03-05T20:49:38.382956Z","iopub.status.idle":"2024-03-05T20:49:38.467656Z","shell.execute_reply.started":"2024-03-05T20:49:38.382923Z","shell.execute_reply":"2024-03-05T20:49:38.466761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Looks good!","metadata":{}},{"cell_type":"markdown","source":"I'll then prep the test data:","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv(path/'test.csv')\ntest_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-05T20:49:42.670971Z","iopub.execute_input":"2024-03-05T20:49:42.671345Z","iopub.status.idle":"2024-03-05T20:49:42.690103Z","shell.execute_reply.started":"2024-03-05T20:49:42.671296Z","shell.execute_reply":"2024-03-05T20:49:42.689104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spec_ids = test_df['spectrogram_id'].unique()\nlen(spec_ids)","metadata":{"execution":{"iopub.status.busy":"2024-03-05T20:49:45.318064Z","iopub.execute_input":"2024-03-05T20:49:45.318449Z","iopub.status.idle":"2024-03-05T20:49:45.325245Z","shell.execute_reply.started":"2024-03-05T20:49:45.318418Z","shell.execute_reply":"2024-03-05T20:49:45.324262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warnings.filterwarnings(\"ignore\")\nparallel(process_spec, spec_ids, split='test', n_workers=4)\nwarnings.filterwarnings(\"default\")","metadata":{"execution":{"iopub.status.busy":"2024-03-05T20:49:46.346215Z","iopub.execute_input":"2024-03-05T20:49:46.346854Z","iopub.status.idle":"2024-03-05T20:49:46.610517Z","shell.execute_reply.started":"2024-03-05T20:49:46.346820Z","shell.execute_reply":"2024-03-05T20:49:46.609388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PILImage.create(Path('/tmp/dataset/hms-hbac/test_spectrograms').ls()[0])","metadata":{"execution":{"iopub.status.busy":"2024-03-05T20:49:49.334288Z","iopub.execute_input":"2024-03-05T20:49:49.335202Z","iopub.status.idle":"2024-03-05T20:49:49.392267Z","shell.execute_reply.started":"2024-03-05T20:49:49.335164Z","shell.execute_reply":"2024-03-05T20:49:49.391333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating the `DataLoaders` Object","metadata":{}},{"cell_type":"markdown","source":"I'll wrap all of my code to prep data for training into a single cell.","metadata":{}},{"cell_type":"code","source":"df['img_path'] = '/tmp/dataset/hms-hbac/train_spectrograms/' + df['spectrogram_id'].astype(str) + '.png'\n    \ncols = ['eeg_id', 'spectrogram_id', 'img_path', 'seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']\nagg_funcs = {c: 'sum' for c in cols if 'vote' in c}\n\nunique_df = df[cols].groupby(['eeg_id', 'spectrogram_id', 'img_path'], as_index=False).agg(agg_funcs)\nunique_df['target'] = unique_df[[c for c in cols if 'vote' in c]].idxmax(axis=1)\n    \ntrain_bool = [False for _ in range(int(0.8 * len(unique_df)))]\nvalid_bool = [True for _ in range(len(unique_df) - int(0.8 * len(unique_df)))]\nis_valid_bool = pd.Series(train_bool + valid_bool).sample(frac=1).reset_index(drop=True)\n\nunique_df[\"is_valid\"] = is_valid_bool\nunique_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-05T20:49:53.761926Z","iopub.execute_input":"2024-03-05T20:49:53.762285Z","iopub.status.idle":"2024-03-05T20:49:53.951584Z","shell.execute_reply.started":"2024-03-05T20:49:53.762256Z","shell.execute_reply":"2024-03-05T20:49:53.950635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now I can create my `DataBlock`. Since I now have paths to images ready to train on, I can use `ImageBlock` for my inputs. I'll also make sure to `Resize` my images so they are 224 x 224 squares.","metadata":{}},{"cell_type":"code","source":"dblock = DataBlock(\n            blocks=(ImageBlock, CategoryBlock),\n            splitter=ColSplitter(),\n            get_x=ColReader('img_path'),\n            get_y=ColReader('target'),\n            item_tfms=Resize(224, method='squish'))\n\ndls = dblock.dataloaders(unique_df)","metadata":{"execution":{"iopub.status.busy":"2024-03-05T20:49:56.946516Z","iopub.execute_input":"2024-03-05T20:49:56.946933Z","iopub.status.idle":"2024-03-05T20:49:58.600532Z","shell.execute_reply.started":"2024-03-05T20:49:56.946902Z","shell.execute_reply":"2024-03-05T20:49:58.599640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.show_batch(nrows=1, ncols=3)","metadata":{"execution":{"iopub.status.busy":"2024-03-05T20:50:01.242054Z","iopub.execute_input":"2024-03-05T20:50:01.242536Z","iopub.status.idle":"2024-03-05T20:50:02.387194Z","shell.execute_reply.started":"2024-03-05T20:50:01.242505Z","shell.execute_reply":"2024-03-05T20:50:02.386257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check dls vocab\ndls.vocab","metadata":{"execution":{"iopub.status.busy":"2024-03-05T20:50:05.459338Z","iopub.execute_input":"2024-03-05T20:50:05.459736Z","iopub.status.idle":"2024-03-05T20:50:05.467365Z","shell.execute_reply.started":"2024-03-05T20:50:05.459704Z","shell.execute_reply":"2024-03-05T20:50:05.466078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"fastai uses `timm` when you specify the architecture as a string:","metadata":{}},{"cell_type":"code","source":"learn = vision_learner(dls, 'resnet34' ,metrics=accuracy, pretrained=False)","metadata":{"execution":{"iopub.status.busy":"2024-03-05T20:50:09.275021Z","iopub.execute_input":"2024-03-05T20:50:09.275426Z","iopub.status.idle":"2024-03-05T20:50:09.740926Z","shell.execute_reply.started":"2024-03-05T20:50:09.275397Z","shell.execute_reply":"2024-03-05T20:50:09.739912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.load('/kaggle/input/hms-hbac-resnet34/pytorch/2/1/hms_hbac_resnet34_stacked')","metadata":{"execution":{"iopub.status.busy":"2024-03-05T20:50:20.887197Z","iopub.execute_input":"2024-03-05T20:50:20.887609Z","iopub.status.idle":"2024-03-05T20:50:22.010164Z","shell.execute_reply.started":"2024-03-05T20:50:20.887578Z","shell.execute_reply":"2024-03-05T20:50:22.008876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['img_path'] = '/tmp/dataset/hms-hbac/test_spectrograms/' + test_df['spectrogram_id'].astype(str) + '.png'\ntst_dl = learn.dls.test_dl(test_df)\ntst_dl.show_batch()","metadata":{"execution":{"iopub.status.busy":"2024-03-05T20:50:25.686384Z","iopub.execute_input":"2024-03-05T20:50:25.686771Z","iopub.status.idle":"2024-03-05T20:50:25.819631Z","shell.execute_reply.started":"2024-03-05T20:50:25.686744Z","shell.execute_reply":"2024-03-05T20:50:25.818613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"probs,_,idxs = learn.get_preds(dl=tst_dl, with_decoded=True)\nprobs_df = pd.DataFrame(probs, columns=dls.vocab)\nprobs_df['eeg_id'] = test_df['eeg_id']\nprobs_df = probs_df[['eeg_id', 'seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']]\nprobs_df","metadata":{"execution":{"iopub.status.busy":"2024-03-05T20:50:29.949850Z","iopub.execute_input":"2024-03-05T20:50:29.950247Z","iopub.status.idle":"2024-03-05T20:50:31.685982Z","shell.execute_reply.started":"2024-03-05T20:50:29.950216Z","shell.execute_reply":"2024-03-05T20:50:31.684721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"probs_df[['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']].sum(axis=1) == 1","metadata":{"execution":{"iopub.status.busy":"2024-03-05T20:50:34.781434Z","iopub.execute_input":"2024-03-05T20:50:34.782422Z","iopub.status.idle":"2024-03-05T20:50:34.792802Z","shell.execute_reply.started":"2024-03-05T20:50:34.782385Z","shell.execute_reply":"2024-03-05T20:50:34.791601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"probs_df.to_csv('submission.csv', index=False)\n!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2024-03-05T20:50:38.306793Z","iopub.execute_input":"2024-03-05T20:50:38.307147Z","iopub.status.idle":"2024-03-05T20:50:39.327990Z","shell.execute_reply.started":"2024-03-05T20:50:38.307119Z","shell.execute_reply":"2024-03-05T20:50:39.326777Z"},"trusted":true},"execution_count":null,"outputs":[]}]}