{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":307163,"sourceType":"datasetVersion","datasetId":128088},{"sourceId":7952771,"sourceType":"datasetVersion","datasetId":4677094},{"sourceId":504,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":381},{"sourceId":992,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":846},{"sourceId":6086,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":4624}],"dockerImageVersionId":30673,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np\nimport pandas as pd\nimport os\nfrom fastai.vision.all import *\nfrom fastai.tabular.all import *\nfrom pathlib import Path\nfrom PIL import Image\nfrom fastcore.parallel import parallel\nfrom tqdm.notebook import tqdm\n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-30T02:26:29.445795Z","iopub.execute_input":"2024-03-30T02:26:29.446136Z","iopub.status.idle":"2024-03-30T02:26:42.439767Z","shell.execute_reply.started":"2024-03-30T02:26:29.446110Z","shell.execute_reply":"2024-03-30T02:26:42.438820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_path = Path('/kaggle/input/hms-harmful-brain-activity-classification')\nbase_path.ls()","metadata":{"execution":{"iopub.status.busy":"2024-03-30T02:26:42.441220Z","iopub.execute_input":"2024-03-30T02:26:42.441479Z","iopub.status.idle":"2024-03-30T02:26:42.451458Z","shell.execute_reply.started":"2024-03-30T02:26:42.441457Z","shell.execute_reply":"2024-03-30T02:26:42.450463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_eegs = pd.read_parquet(base_path/'test_eegs')\ndf_test_spectograms = pd.read_parquet(base_path/'test_spectrograms')\ndf_test_csv = pd.read_csv(base_path/'test.csv')  \ndf_train_csv = pd.read_csv(base_path/'train.csv')","metadata":{"execution":{"iopub.status.busy":"2024-03-30T02:26:42.452593Z","iopub.execute_input":"2024-03-30T02:26:42.452885Z","iopub.status.idle":"2024-03-30T02:26:42.982556Z","shell.execute_reply.started":"2024-03-30T02:26:42.452836Z","shell.execute_reply":"2024-03-30T02:26:42.981722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SPEC_DIR = \"/tmp/dataset/hms-hbac\"\nEEG_DIR = \"/tmp/dataset/hms-hbac\"\nos.makedirs(SPEC_DIR+'/train_spectrograms', exist_ok=True)\nos.makedirs(SPEC_DIR+'/test_spectrograms', exist_ok=True)\nos.makedirs(EEG_DIR+'/train_eegs', exist_ok=True)\nos.makedirs(EEG_DIR+'/test_eegs', exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T02:26:42.984655Z","iopub.execute_input":"2024-03-30T02:26:42.985002Z","iopub.status.idle":"2024-03-30T02:26:42.991237Z","shell.execute_reply.started":"2024-03-30T02:26:42.984974Z","shell.execute_reply":"2024-03-30T02:26:42.990339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_spec(spec_id, split=\"train\"):\n    # Read the data\n    data = pd.read_parquet(base_path / f'{split}_spectrograms' / f'{spec_id}.parquet')\n    \n    # Replace NA with 0\n    data = data.fillna(0)\n    \n    # Convert DataFrame to array\n    data = data.values[:, 1:]\n    \n    # Transpose\n    data = data.T\n    data = data.astype(\"float32\")\n    \n    # Convert array to PILImage\n    im = PILImage.create(Image.fromarray((data * 255).astype(np.uint8)))\n    im.save(os.path.join(SPEC_DIR, f'{split}_spectrograms', f'{spec_id}.png'))","metadata":{"execution":{"iopub.status.busy":"2024-03-30T02:26:42.992370Z","iopub.execute_input":"2024-03-30T02:26:42.992706Z","iopub.status.idle":"2024-03-30T02:26:43.001794Z","shell.execute_reply.started":"2024-03-30T02:26:42.992659Z","shell.execute_reply":"2024-03-30T02:26:43.000649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_csv = pd.read_csv(base_path/'train.csv')\nspec_ids = df_train_csv[\"spectrogram_id\"].unique()\nlen(spec_ids)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T02:26:43.002939Z","iopub.execute_input":"2024-03-30T02:26:43.003277Z","iopub.status.idle":"2024-03-30T02:26:43.182140Z","shell.execute_reply.started":"2024-03-30T02:26:43.003253Z","shell.execute_reply":"2024-03-30T02:26:43.181204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# parallel(process_spec, spec_ids, split='train', n_workers=4)\nparallel(process_spec, spec_ids, split='train', n_workers=4);\n","metadata":{"execution":{"iopub.status.busy":"2024-03-30T02:26:43.183354Z","iopub.execute_input":"2024-03-30T02:26:43.183604Z","iopub.status.idle":"2024-03-30T02:30:53.772975Z","shell.execute_reply.started":"2024-03-30T02:26:43.183582Z","shell.execute_reply":"2024-03-30T02:30:53.771772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nfrom PIL import Image\nfrom IPython.display import display\n\n# Specify the directory containing the images\nimages_dir = Path('/tmp/dataset/hms-hbac/train_spectrograms')\n\n# Get the first 10 image paths\nimage_paths = list(images_dir.glob('*'))[:10]\n\n# Open and display each image\nfor image_path in image_paths:\n    with Image.open(image_path) as img:\n        # Convert the image to RGB mode if it's not already in that mode\n#         img = img.convert('RGB')\n        display(img)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T02:30:53.774340Z","iopub.execute_input":"2024-03-30T02:30:53.774644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom pathlib import Path\nfrom PIL import Image\n\n# Specify the directory containing the images\nimages_dir = Path('/tmp/dataset/hms-hbac/train_spectrograms')\n\n# Get the first 10 image paths\nimage_paths = list(images_dir.glob('*'))[:10]\n\n# Open and display each image\nfor image_path in image_paths:\n    with Image.open(image_path) as img:\n        img_array = img.convert(\"RGB\")  # Convert to RGB array\n        plt.imshow(img_array, cmap='viridis')  # Display the image data with viridis colormap\n        plt.colorbar()  # Add color bar to show intensity scale\n        plt.title('Visual Representation of Numeric Data')\n        plt.show()\n","metadata":{"execution":{"iopub.status.idle":"2024-03-30T02:30:58.003784Z","shell.execute_reply.started":"2024-03-30T02:30:54.014250Z","shell.execute_reply":"2024-03-30T02:30:58.002888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_spec(spec_id, split=\"test\"):\n    # Read the data\n    data = pd.read_parquet(base_path / f'{split}_spectrograms' / f'{spec_id}.parquet')\n    \n    # Replace NA with 0\n    data = data.fillna(0)\n    \n    # Convert DataFrame to array\n    data = data.values[:, 1:]\n    \n    # Transpose\n    data = data.T\n    data = data.astype(\"float32\")\n    \n    # Convert array to PILImage\n    im = PILImage.create(Image.fromarray((data * 255).astype(np.uint8)))\n    im.save(os.path.join(SPEC_DIR, f'{split}_spectrograms', f'{spec_id}.png'))","metadata":{"execution":{"iopub.status.busy":"2024-03-30T02:30:58.008308Z","iopub.execute_input":"2024-03-30T02:30:58.008945Z","iopub.status.idle":"2024-03-30T02:30:58.016198Z","shell.execute_reply.started":"2024-03-30T02:30:58.008910Z","shell.execute_reply":"2024-03-30T02:30:58.015118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_csv = pd.read_csv(base_path/'test.csv')\nspec_ids = df_test_csv[\"spectrogram_id\"].unique()\nlen(spec_ids)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T02:30:58.017524Z","iopub.execute_input":"2024-03-30T02:30:58.017884Z","iopub.status.idle":"2024-03-30T02:30:58.101406Z","shell.execute_reply.started":"2024-03-30T02:30:58.017836Z","shell.execute_reply":"2024-03-30T02:30:58.100323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parallel(process_spec, spec_ids, split='test', n_workers=4)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T02:30:58.103041Z","iopub.execute_input":"2024-03-30T02:30:58.103423Z","iopub.status.idle":"2024-03-30T02:30:58.291888Z","shell.execute_reply.started":"2024-03-30T02:30:58.103388Z","shell.execute_reply":"2024-03-30T02:30:58.290862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nfrom PIL import Image\nfrom IPython.display import display\n\n# Specify the directory containing the images\nimages_dir_test_spec = Path('/tmp/dataset/hms-hbac/test_spectrograms')\n\n# Get the first 10 image paths\nimage_paths = list(images_dir_test_spec.glob('*'))[:10]\n\n# # Open and display each image\n# for image_path in image_paths:\n#     with Image.open(image_path) as img:\n#         # Convert the image to RGB mode if it's not already in that mode\n# #         img = img.convert('RGB')\n#         display(img)\n    \n# Open and display each image\nfor image_path in image_paths:\n    with Image.open(image_path) as img:\n        img_array = img.convert(\"RGB\")  # Convert to RGB array\n        plt.imshow(img_array, cmap='viridis')  # Display the image data with viridis colormap\n        plt.colorbar()  # Add color bar to show intensity scale\n        plt.title('Visual Representation of Numeric Data')\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-30T02:30:58.293585Z","iopub.execute_input":"2024-03-30T02:30:58.294454Z","iopub.status.idle":"2024-03-30T02:30:58.683025Z","shell.execute_reply.started":"2024-03-30T02:30:58.294414Z","shell.execute_reply":"2024-03-30T02:30:58.682156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_dir_test_spec.ls()","metadata":{"execution":{"iopub.status.busy":"2024-03-30T02:30:58.684294Z","iopub.execute_input":"2024-03-30T02:30:58.684891Z","iopub.status.idle":"2024-03-30T02:30:58.690907Z","shell.execute_reply.started":"2024-03-30T02:30:58.684858Z","shell.execute_reply":"2024-03-30T02:30:58.689997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the columns for image paths and targets\ndf_train_csv['img_path'] = '/tmp/dataset/hms-hbac/train_spectrograms/' + df_train_csv['spectrogram_id'].astype(str) + '.png'\neeg_id = df_train_csv['eeg_id']\nspectrogram_id = df_train_csv['spectrogram_id']\ntarget_cols = ['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']  # Adjust these to the actual column names containing the target labels\n\ndf_train_csv['target'] = df_train_csv[[c for c in target_cols if 'vote' in c]].idxmax(axis=1)\nbatch_tfms = [Brightness(max_lighting=0.5, p=1.0), Contrast(max_lighting=0.5, p=1.0)]\n\ndblock = DataBlock(\n    blocks=(ImageBlock, CategoryBlock),\n    splitter=RandomSplitter(),  # or specify your splitter here\n    get_x=ColReader('img_path'),\n    get_y=ColReader('target'),\n    item_tfms=Resize(224, method='squish'), batch_tfms = batch_tfms)\n    #item_tfms=[Resize(256)],\n    #batch_tfms = aug_transforms())\n\ndls = dblock.dataloaders(df_train_csv)\n","metadata":{"execution":{"iopub.status.busy":"2024-03-30T02:30:58.692265Z","iopub.execute_input":"2024-03-30T02:30:58.692702Z","iopub.status.idle":"2024-03-30T02:31:07.248472Z","shell.execute_reply.started":"2024-03-30T02:30:58.692670Z","shell.execute_reply":"2024-03-30T02:31:07.247569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.show_batch()","metadata":{"execution":{"iopub.status.busy":"2024-03-30T02:31:07.249668Z","iopub.execute_input":"2024-03-30T02:31:07.250042Z","iopub.status.idle":"2024-03-30T02:31:09.125330Z","shell.execute_reply.started":"2024-03-30T02:31:07.250008Z","shell.execute_reply":"2024-03-30T02:31:09.124416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Load the manually added model\n# model_path = \"/kaggle/input/convnext/tensorflow2/tiny-1k-224/1/saved_model.pb\"\n# model = torch.load(model_path)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T02:31:09.126751Z","iopub.execute_input":"2024-03-30T02:31:09.127431Z","iopub.status.idle":"2024-03-30T02:31:09.131415Z","shell.execute_reply.started":"2024-03-30T02:31:09.127395Z","shell.execute_reply":"2024-03-30T02:31:09.130411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# learn = vision_learner(dls, model, metrics=error_rate).to_fp16()  # Convert to mixed precision and move to GPU\n# # learn.model = learn.model.to('cuda')  # Move model to GPU","metadata":{"execution":{"iopub.status.busy":"2024-03-30T02:31:09.132748Z","iopub.execute_input":"2024-03-30T02:31:09.133081Z","iopub.status.idle":"2024-03-30T02:31:09.142766Z","shell.execute_reply.started":"2024-03-30T02:31:09.133053Z","shell.execute_reply":"2024-03-30T02:31:09.141732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.exists('/root/.cache/torch/hub/checkpoints/'):\n    os.makedirs('/root/.cache/torch/hub/checkpoints/')\n    \n    \n!cp '/kaggle/input/resnet34-27032024/resnet34-b627a593.pth' '/root/.cache/torch/hub/checkpoints/resnet34-b627a593.pth'\n# !cp '/kaggle/input/resnet-50/tensorflow2/classification/1/saved_model.pb' '/root/.cache/torch/hub/checkpoints/tensorflow2/classification/1/saved_model.pb'","metadata":{"execution":{"iopub.status.busy":"2024-03-30T02:31:09.144104Z","iopub.execute_input":"2024-03-30T02:31:09.144394Z","iopub.status.idle":"2024-03-30T02:31:11.439228Z","shell.execute_reply.started":"2024-03-30T02:31:09.144369Z","shell.execute_reply":"2024-03-30T02:31:11.438063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn = vision_learner(dls, resnet34, metrics=error_rate).to_fp16() \nlearn.model = learn.model.to('cuda')\n","metadata":{"execution":{"iopub.status.busy":"2024-03-30T02:31:11.441088Z","iopub.execute_input":"2024-03-30T02:31:11.441967Z","iopub.status.idle":"2024-03-30T02:31:12.148802Z","shell.execute_reply.started":"2024-03-30T02:31:11.441925Z","shell.execute_reply":"2024-03-30T02:31:12.147736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.fit(4)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T02:31:12.149990Z","iopub.execute_input":"2024-03-30T02:31:12.150273Z","iopub.status.idle":"2024-03-30T03:01:25.175005Z","shell.execute_reply.started":"2024-03-30T02:31:12.150250Z","shell.execute_reply":"2024-03-30T03:01:25.173832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_csv['img_path'] = '/tmp/dataset/hms-hbac/test_spectrograms/' + df_test_csv['spectrogram_id'].astype(str) + '.png'\ntst_dl = learn.dls.test_dl(df_test_csv)\ntst_dl.show_batch()","metadata":{"execution":{"iopub.status.busy":"2024-03-30T03:01:25.177106Z","iopub.execute_input":"2024-03-30T03:01:25.177982Z","iopub.status.idle":"2024-03-30T03:01:25.335124Z","shell.execute_reply.started":"2024-03-30T03:01:25.177942Z","shell.execute_reply":"2024-03-30T03:01:25.333820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the trained model\nlearn.save('hms_hbac_test_3a')\n","metadata":{"execution":{"iopub.status.busy":"2024-03-30T03:01:25.337086Z","iopub.execute_input":"2024-03-30T03:01:25.338194Z","iopub.status.idle":"2024-03-30T03:01:25.591385Z","shell.execute_reply.started":"2024-03-30T03:01:25.338139Z","shell.execute_reply":"2024-03-30T03:01:25.590306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.load('hms_hbac_test_3a')","metadata":{"execution":{"iopub.status.busy":"2024-03-30T03:01:25.592785Z","iopub.execute_input":"2024-03-30T03:01:25.593701Z","iopub.status.idle":"2024-03-30T03:01:25.713353Z","shell.execute_reply.started":"2024-03-30T03:01:25.593651Z","shell.execute_reply":"2024-03-30T03:01:25.712393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the trained model\nlearn = learn.load('hms_hbac_test_3a')\n\n# Get predictions and probabilities for the test set\ntest_preds, test_probs, _ = learn.get_preds(dl=tst_dl, with_decoded=True)\n\n# Debugging: Print the values of test_preds and test_probs to understand the issue\nprint(\"test_preds:\", test_preds)\nprint(\"test_probs:\", test_probs)\n\n# Append predictions and probabilities to the test dataframe\nif test_probs is not None:\n    df_test_csv['prediction'] = test_preds.argmax(dim=1)\n    df_test_csv['probabilities'] = test_probs.numpy()\nelse:\n    print(\"Error: test_probs is None.\")\n\n# Display the updated test dataframe\nprint(df_test_csv.head())\n","metadata":{"execution":{"iopub.status.busy":"2024-03-30T03:01:25.714770Z","iopub.execute_input":"2024-03-30T03:01:25.715145Z","iopub.status.idle":"2024-03-30T03:01:26.213044Z","shell.execute_reply.started":"2024-03-30T03:01:25.715113Z","shell.execute_reply":"2024-03-30T03:01:26.211897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"probs,_,idxs = learn.get_preds(dl=tst_dl, with_decoded=True)\nprobs_df = pd.DataFrame(probs, columns=dls.vocab)\nprobs_df['eeg_id'] = df_test_csv['eeg_id']\nprobs_df = probs_df[['eeg_id', 'seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']]\nprobs_df","metadata":{"execution":{"iopub.status.busy":"2024-03-30T03:01:26.215064Z","iopub.execute_input":"2024-03-30T03:01:26.215964Z","iopub.status.idle":"2024-03-30T03:01:26.415968Z","shell.execute_reply.started":"2024-03-30T03:01:26.215907Z","shell.execute_reply":"2024-03-30T03:01:26.414913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"probs_df.to_csv('submission.csv', index=False)\n!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2024-03-30T03:01:26.417542Z","iopub.execute_input":"2024-03-30T03:01:26.417927Z","iopub.status.idle":"2024-03-30T03:01:27.424055Z","shell.execute_reply.started":"2024-03-30T03:01:26.417875Z","shell.execute_reply":"2024-03-30T03:01:27.423094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}