{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"dockerImageVersionId":30648,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from fastai.vision.all import *\nimport joblib\nfrom tqdm import tqdm\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-03T05:27:36.981436Z","iopub.execute_input":"2024-02-03T05:27:36.981789Z","iopub.status.idle":"2024-02-03T05:27:36.986937Z","shell.execute_reply.started":"2024-02-03T05:27:36.981760Z","shell.execute_reply":"2024-02-03T05:27:36.985909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = Path(\"/kaggle/input/hms-harmful-brain-activity-classification\")\nPath.BASE_PATH = Path\npath.ls()","metadata":{"execution":{"iopub.status.busy":"2024-02-03T05:28:31.181968Z","iopub.execute_input":"2024-02-03T05:28:31.182448Z","iopub.status.idle":"2024-02-03T05:28:31.190844Z","shell.execute_reply.started":"2024-02-03T05:28:31.182409Z","shell.execute_reply":"2024-02-03T05:28:31.189885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(path/'train.csv')\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-03T05:28:31.549869Z","iopub.execute_input":"2024-02-03T05:28:31.550797Z","iopub.status.idle":"2024-02-03T05:28:31.731420Z","shell.execute_reply.started":"2024-02-03T05:28:31.550771Z","shell.execute_reply":"2024-02-03T05:28:31.730536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/code/awsaf49/hms-hbac-kerascv-starter-notebook\ndf = pd.read_csv(f'{path}/train.csv')\ndf['eeg_path'] = f'{path}/train_eegs/'+df['eeg_id'].astype(str)+'.parquet'\ndf['spec_path'] = f'{path}/train_spectrograms/'+df['spectrogram_id'].astype(str)+'.parquet'\ndf['spec2_path'] = f'{path}/train_spectrograms/'+df['spectrogram_id'].astype(str)+'.npy'\ndf['class_name'] = df.expert_consensus.copy()\ndisplay(df.head(2))\n\n\ntest_df = pd.read_csv(f'{path}/test.csv')\ntest_df['eeg_path'] = f'{path}/test_eegs/'+test_df['eeg_id'].astype(str)+'.parquet'\ntest_df['spec_path'] = f'{path}/test_spectrograms/'+test_df['spectrogram_id'].astype(str)+'.parquet'\ntest_df['spec2_path'] = f'{path}/test_spectrograms/'+test_df['spectrogram_id'].astype(str)+'.npy'\ndisplay(test_df.head(2))","metadata":{"execution":{"iopub.status.busy":"2024-02-03T05:28:32.350767Z","iopub.execute_input":"2024-02-03T05:28:32.351278Z","iopub.status.idle":"2024-02-03T05:28:32.788417Z","shell.execute_reply.started":"2024-02-03T05:28:32.351242Z","shell.execute_reply":"2024-02-03T05:28:32.787513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SPEC_DIR = \"/tmp/dataset/hms-hbac\"\nos.makedirs(SPEC_DIR+'/train_spectrograms', exist_ok=True)\nos.makedirs(SPEC_DIR+'/test_spectrograms', exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2024-02-03T05:21:42.459656Z","iopub.execute_input":"2024-02-03T05:21:42.459909Z","iopub.status.idle":"2024-02-03T05:21:42.464961Z","shell.execute_reply.started":"2024-02-03T05:21:42.459881Z","shell.execute_reply":"2024-02-03T05:21:42.464115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/code/awsaf49/hms-hbac-kerascv-starter-notebook\n# Define a function to process a single eeg_id\ndef process_spec(spec_id, split=\"train\"):\n    spec_path = f\"{path}/{split}_spectrograms/{spec_id}.parquet\"\n    spec = pd.read_parquet(spec_path)\n    spec = spec.fillna(0).values[:, 1:].T # fill NaN values with 0, transpose for (Time, Freq) -> (Freq, Time)\n    spec = spec.astype(\"float32\")\n    np.save(f\"{SPEC_DIR}/{split}_spectrograms/{spec_id}.npy\", spec)\n    \n# Get unique spec_ids of train and valid data\nspec_ids = df[\"spectrogram_id\"].unique()\n\n# Parallelize the processing using joblib for training data\n_ = joblib.Parallel(n_jobs=-1, backend=\"loky\")(\n    joblib.delayed(process_spec)(spec_id, \"train\")\n    for spec_id in tqdm(spec_ids, total=len(spec_ids))\n)\n\n# Get unique spec_ids of test data\ntest_spec_ids = test_df[\"spectrogram_id\"].unique()\n\n# Parallelize the processing using joblib for test data\n_ = joblib.Parallel(n_jobs=-1, backend=\"loky\")(\n    joblib.delayed(process_spec)(spec_id, \"test\")\n    for spec_id in tqdm(test_spec_ids, total=len(test_spec_ids))\n)","metadata":{"execution":{"iopub.status.busy":"2024-02-03T05:28:37.700663Z","iopub.execute_input":"2024-02-03T05:28:37.701037Z","iopub.status.idle":"2024-02-03T05:28:37.706517Z","shell.execute_reply.started":"2024-02-03T05:28:37.700986Z","shell.execute_reply":"2024-02-03T05:28:37.705183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_path = Path('/tmp/dataset/hms-hbac/')","metadata":{"execution":{"iopub.status.busy":"2024-02-03T05:28:45.699028Z","iopub.execute_input":"2024-02-03T05:28:45.699825Z","iopub.status.idle":"2024-02-03T05:28:45.704961Z","shell.execute_reply.started":"2024-02-03T05:28:45.699788Z","shell.execute_reply":"2024-02-03T05:28:45.703941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['spec2_path'] = f'{image_path}/train_spectrograms/'+df['spectrogram_id'].astype(str)+'.npy'","metadata":{"execution":{"iopub.status.busy":"2024-02-03T05:28:46.028079Z","iopub.execute_input":"2024-02-03T05:28:46.028466Z","iopub.status.idle":"2024-02-03T05:28:46.121702Z","shell.execute_reply.started":"2024-02-03T05:28:46.028438Z","shell.execute_reply":"2024-02-03T05:28:46.120684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df = df.groupby(\"spectrogram_id\").head(1).reset_index(drop=True)\nfrom sklearn.model_selection import train_test_split\ntrain_df, valid_df = train_test_split(df, test_size=0.2)\nprint(f\"# Num Train: {len(train_df)} | Num Valid: {len(valid_df)}\")","metadata":{"execution":{"iopub.status.busy":"2024-02-03T05:28:46.465353Z","iopub.execute_input":"2024-02-03T05:28:46.466155Z","iopub.status.idle":"2024-02-03T05:28:46.530074Z","shell.execute_reply.started":"2024-02-03T05:28:46.466116Z","shell.execute_reply":"2024-02-03T05:28:46.529163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ArrayToPILImage(x):\n    x = np.load(x)\n    return PILImage.create(Image.fromarray((x * 255).astype(np.uint8)))\n\ndef ImageBlock(cls=PILImage):\n    return TransformBlock(type_tfms=ArrayToPILImage, batch_tfms=IntToFloatTensor)","metadata":{"execution":{"iopub.status.busy":"2024-02-03T05:28:47.528737Z","iopub.execute_input":"2024-02-03T05:28:47.529121Z","iopub.status.idle":"2024-02-03T05:28:47.534825Z","shell.execute_reply.started":"2024-02-03T05:28:47.529093Z","shell.execute_reply":"2024-02-03T05:28:47.533829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dblock = DataBlock(\n    blocks=(ImageBlock, CategoryBlock),\n    get_x=ColReader(17),  \n    get_y=ColReader(18),\n    splitter=RandomSplitter(),\n    item_tfms=Resize(224),\n    batch_tfms=aug_transforms(size=224, min_scale=0.75)\n)","metadata":{"execution":{"iopub.status.busy":"2024-02-03T05:28:48.066976Z","iopub.execute_input":"2024-02-03T05:28:48.067348Z","iopub.status.idle":"2024-02-03T05:28:48.074688Z","shell.execute_reply.started":"2024-02-03T05:28:48.067320Z","shell.execute_reply":"2024-02-03T05:28:48.073605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds = dblock.datasets(train_df)","metadata":{"execution":{"iopub.status.busy":"2024-02-03T05:28:48.637970Z","iopub.execute_input":"2024-02-03T05:28:48.638331Z","iopub.status.idle":"2024-02-03T05:28:55.904665Z","shell.execute_reply.started":"2024-02-03T05:28:48.638304Z","shell.execute_reply":"2024-02-03T05:28:55.903671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls = dblock.dataloaders(train_df)","metadata":{"execution":{"iopub.status.busy":"2024-02-03T05:28:55.907552Z","iopub.execute_input":"2024-02-03T05:28:55.908293Z","iopub.status.idle":"2024-02-03T05:28:56.497652Z","shell.execute_reply.started":"2024-02-03T05:28:55.908258Z","shell.execute_reply":"2024-02-03T05:28:56.496594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.show_batch()","metadata":{"execution":{"iopub.status.busy":"2024-02-03T05:28:56.498958Z","iopub.execute_input":"2024-02-03T05:28:56.499352Z","iopub.status.idle":"2024-02-03T05:28:57.772021Z","shell.execute_reply.started":"2024-02-03T05:28:56.499319Z","shell.execute_reply":"2024-02-03T05:28:57.770958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn = vision_learner(dls, resnet34, metrics=accuracy)\nlearn.fine_tune(3)","metadata":{"execution":{"iopub.status.busy":"2024-02-03T05:28:57.774371Z","iopub.execute_input":"2024-02-03T05:28:57.774776Z","iopub.status.idle":"2024-02-03T05:44:20.141252Z","shell.execute_reply.started":"2024-02-03T05:28:57.774742Z","shell.execute_reply":"2024-02-03T05:44:20.139935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Inference In test data","metadata":{"execution":{"iopub.status.busy":"2024-02-03T05:54:03.231052Z","iopub.execute_input":"2024-02-03T05:54:03.231975Z","iopub.status.idle":"2024-02-03T05:54:03.238179Z","shell.execute_reply.started":"2024-02-03T05:54:03.231942Z","shell.execute_reply":"2024-02-03T05:54:03.237093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-03T05:54:10.134065Z","iopub.execute_input":"2024-02-03T05:54:10.134884Z","iopub.status.idle":"2024-02-03T05:54:10.148508Z","shell.execute_reply.started":"2024-02-03T05:54:10.134837Z","shell.execute_reply":"2024-02-03T05:54:10.147441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['spec2_path'] = f'{image_path}/test_spectrograms/'+test_df['spectrogram_id'].astype(str)+'.npy'","metadata":{"execution":{"iopub.status.busy":"2024-02-03T05:59:54.284791Z","iopub.execute_input":"2024-02-03T05:59:54.285179Z","iopub.status.idle":"2024-02-03T05:59:54.290942Z","shell.execute_reply.started":"2024-02-03T05:59:54.285149Z","shell.execute_reply":"2024-02-03T05:59:54.289911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-03T05:59:58.913132Z","iopub.execute_input":"2024-02-03T05:59:58.913497Z","iopub.status.idle":"2024-02-03T05:59:58.925134Z","shell.execute_reply.started":"2024-02-03T05:59:58.913469Z","shell.execute_reply":"2024-02-03T05:59:58.924206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dl = learn.dls.test_dl(test_df)","metadata":{"execution":{"iopub.status.busy":"2024-02-03T06:00:12.016052Z","iopub.execute_input":"2024-02-03T06:00:12.016424Z","iopub.status.idle":"2024-02-03T06:00:12.037873Z","shell.execute_reply.started":"2024-02-03T06:00:12.016393Z","shell.execute_reply":"2024-02-03T06:00:12.036637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = []\n\nfor i in range(len(test_df)):\n    lbl, clas, probs = learn.predict(test_df.iloc[i]['spec2_path'])\n    predictions.append(probs.numpy())\n\npredictions_df = pd.DataFrame(predictions, columns=dls.vocab)\n\npredictions_df['eeg_id'] = test_df['eeg_id'] \n\npredictions_df.to_csv('submission_test.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-02-03T06:10:21.923441Z","iopub.execute_input":"2024-02-03T06:10:21.923809Z","iopub.status.idle":"2024-02-03T06:10:21.984967Z","shell.execute_reply.started":"2024-02-03T06:10:21.923779Z","shell.execute_reply":"2024-02-03T06:10:21.984123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions_df","metadata":{"execution":{"iopub.status.busy":"2024-02-03T06:10:58.859048Z","iopub.execute_input":"2024-02-03T06:10:58.859885Z","iopub.status.idle":"2024-02-03T06:10:58.873874Z","shell.execute_reply.started":"2024-02-03T06:10:58.859843Z","shell.execute_reply":"2024-02-03T06:10:58.872926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.export()","metadata":{},"execution_count":null,"outputs":[]}]}