{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":2132855,"sourceType":"datasetVersion","datasetId":900016},{"sourceId":9457443,"sourceType":"datasetVersion","datasetId":5749297}],"dockerImageVersionId":30747,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#importing libraries\nimport pandas as pd\nimport numpy as np\nimport glob   #to find the pathnames using wildcard\nimport matplotlib.pyplot as plt\nimport cv2\n\nimport torch\nimport torchvision.transforms as transforms\npd.set_option('display.max_columns',None)\nimport pydicom\nfrom tqdm.notebook import tqdm_notebook\nimport warnings\ntqdm_notebook.pandas()\nfrom tqdm import tqdm                                    # To print the Progress bar\nimport time","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-09-25T10:27:48.396872Z","iopub.execute_input":"2024-09-25T10:27:48.397288Z","iopub.status.idle":"2024-09-25T10:27:54.219129Z","shell.execute_reply.started":"2024-09-25T10:27:48.397255Z","shell.execute_reply":"2024-09-25T10:27:54.218301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Defining file paths and loading mandatory libraries","metadata":{}},{"cell_type":"code","source":"#get imagepaths\n\ntrain_image_paths = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/*/*/*'\n\n#Creating the dataloader for\nimage_df = pd.DataFrame(glob.glob(train_image_paths),columns=['Path'])","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:27:54.220889Z","iopub.execute_input":"2024-09-25T10:27:54.221309Z","iopub.status.idle":"2024-09-25T10:28:20.445123Z","shell.execute_reply.started":"2024-09-25T10:27:54.221282Z","shell.execute_reply":"2024-09-25T10:28:20.444250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train1 = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv')\n","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:20.446185Z","iopub.execute_input":"2024-09-25T10:28:20.446464Z","iopub.status.idle":"2024-09-25T10:28:20.477170Z","shell.execute_reply.started":"2024-09-25T10:28:20.446439Z","shell.execute_reply":"2024-09-25T10:28:20.476453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Checking for missing data and treating the dataset","metadata":{}},{"cell_type":"code","source":"train1.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:20.479132Z","iopub.execute_input":"2024-09-25T10:28:20.479409Z","iopub.status.idle":"2024-09-25T10:28:20.497802Z","shell.execute_reply.started":"2024-09-25T10:28:20.479385Z","shell.execute_reply":"2024-09-25T10:28:20.496982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#dropping missing value\ntrain1.dropna(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:20.498918Z","iopub.execute_input":"2024-09-25T10:28:20.499270Z","iopub.status.idle":"2024-09-25T10:28:20.513054Z","shell.execute_reply.started":"2024-09-25T10:28:20.499238Z","shell.execute_reply":"2024-09-25T10:28:20.512334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train1.columns","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:20.514047Z","iopub.execute_input":"2024-09-25T10:28:20.514358Z","iopub.status.idle":"2024-09-25T10:28:20.523877Z","shell.execute_reply.started":"2024-09-25T10:28:20.514325Z","shell.execute_reply":"2024-09-25T10:28:20.522871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Checking the class imbalance and performing undersampling of majority class","metadata":{}},{"cell_type":"code","source":"## adding the majority class column and getting the value count for each majority class, this is done for proper data sampling\ntrain1['Majority_class'] = train1[train1.columns[1:]].apply(lambda row: row.mode()[0],axis=1)\nvalue_count_dict = {}\nfor col in train1.columns[1:]:\n    value_count_dict[col] = train1[col].value_counts()\n\n\nvalue_count_df = pd.DataFrame(value_count_dict)\nvalue_count_df['Average'] = value_count_df.mean(axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:20.524961Z","iopub.execute_input":"2024-09-25T10:28:20.525278Z","iopub.status.idle":"2024-09-25T10:28:20.778249Z","shell.execute_reply.started":"2024-09-25T10:28:20.525254Z","shell.execute_reply":"2024-09-25T10:28:20.777160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train1.shape","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:20.779448Z","iopub.execute_input":"2024-09-25T10:28:20.779753Z","iopub.status.idle":"2024-09-25T10:28:20.785887Z","shell.execute_reply.started":"2024-09-25T10:28:20.779727Z","shell.execute_reply":"2024-09-25T10:28:20.784970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"value_count_df","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:20.787122Z","iopub.execute_input":"2024-09-25T10:28:20.787463Z","iopub.status.idle":"2024-09-25T10:28:20.813291Z","shell.execute_reply.started":"2024-09-25T10:28:20.787428Z","shell.execute_reply":"2024-09-25T10:28:20.812453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Minimum class samples are present for {value_count_df.Majority_class.idxmin()}\")\nprint(f\"Minimum class samples count is {value_count_df.Majority_class.min()}\")\nmincount = value_count_df.Majority_class.min()","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:20.817056Z","iopub.execute_input":"2024-09-25T10:28:20.817407Z","iopub.status.idle":"2024-09-25T10:28:20.823423Z","shell.execute_reply.started":"2024-09-25T10:28:20.817381Z","shell.execute_reply":"2024-09-25T10:28:20.822299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#resampleing the training_data - Undersampling\ntrain_final_class1 = train1[train1['Majority_class']=='Normal/Mild'].sample(mincount )\ntrain_final_class2 = train1[train1['Majority_class']=='Moderate'].sample(mincount )\ntrain_final_class3 = train1[train1['Majority_class']=='Severe'].sample(mincount)\ntrain_final_sample = pd.concat([train_final_class1,train_final_class2,train_final_class3],axis=0)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:20.824744Z","iopub.execute_input":"2024-09-25T10:28:20.825036Z","iopub.status.idle":"2024-09-25T10:28:20.837866Z","shell.execute_reply.started":"2024-09-25T10:28:20.825012Z","shell.execute_reply":"2024-09-25T10:28:20.837069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_sample.drop(columns=['Majority_class'],inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:20.839117Z","iopub.execute_input":"2024-09-25T10:28:20.839413Z","iopub.status.idle":"2024-09-25T10:28:20.845057Z","shell.execute_reply.started":"2024-09-25T10:28:20.839383Z","shell.execute_reply":"2024-09-25T10:28:20.844127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Doing some needed data transformation to prepare of model ingestion","metadata":{}},{"cell_type":"code","source":"\ntrain_transform1 = pd.melt(train_final_sample,id_vars=['study_id'],var_name='Condition',value_name='Severity')\n#split the string into Condition and lumber levels\ndef extract_before_segment(input_string, segment_prefix='l'):\n    # Split the string by underscores\n    parts = input_string.split('_')\n    \n    # Find the index of the first segment that starts with 'l' and contains a digit\n    segment_index = next(\n        (i for i, part in enumerate(parts) if part.startswith(segment_prefix) and any(char.isdigit() for char in part)),\n        None\n    )\n    \n    # Join parts before the identified segment index\n    if segment_index is not None:\n        before_segment = '_'.join(parts[:segment_index])\n    else:\n       print('Split not happened for ' + input_string)\n    \n    \n    return before_segment\n\n\ntrain_transform1['Condition_'] = train_transform1['Condition'].apply(extract_before_segment)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:20.845962Z","iopub.execute_input":"2024-09-25T10:28:20.846206Z","iopub.status.idle":"2024-09-25T10:28:20.868859Z","shell.execute_reply.started":"2024-09-25T10:28:20.846184Z","shell.execute_reply":"2024-09-25T10:28:20.867969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#split the string into Condition and lumber levels\ndef extract_after_segment(input_string, segment_prefix='l'):\n    # Split the string by underscores\n    parts = input_string.split('_')\n    \n    # Find the index of the first segment that starts with 'l' and contains a digit\n    segment_index = next(\n        (i for i, part in enumerate(parts) if part.startswith(segment_prefix) and any(char.isdigit() for char in part)),\n        None\n    )\n    \n    # Join parts before the identified segment index\n    if segment_index is not None:\n        after_segment = '_'.join(parts[segment_index:])\n    else:\n       print('Split not happened for ' + input_string)\n    \n    \n    return after_segment\ntrain_transform1['Lumber_level'] = train_transform1['Condition'].apply(extract_after_segment)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:20.869910Z","iopub.execute_input":"2024-09-25T10:28:20.870244Z","iopub.status.idle":"2024-09-25T10:28:20.884420Z","shell.execute_reply.started":"2024-09-25T10:28:20.870219Z","shell.execute_reply":"2024-09-25T10:28:20.883474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain_transform1.drop(columns=['Condition'],inplace=True)\ntrain_transform2 = train_transform1.pivot(index=['study_id','Condition_'],columns=['Lumber_level'],values='Severity')\ntrain_transform2 = train_transform2.reset_index()\ntrain_transform2 = train_transform2.rename_axis(None, axis=1)  \n","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:20.885402Z","iopub.execute_input":"2024-09-25T10:28:20.885692Z","iopub.status.idle":"2024-09-25T10:28:20.904385Z","shell.execute_reply.started":"2024-09-25T10:28:20.885669Z","shell.execute_reply":"2024-09-25T10:28:20.903413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transform2.Condition_.unique()","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:20.905403Z","iopub.execute_input":"2024-09-25T10:28:20.905710Z","iopub.status.idle":"2024-09-25T10:28:20.911939Z","shell.execute_reply.started":"2024-09-25T10:28:20.905684Z","shell.execute_reply":"2024-09-25T10:28:20.911046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading the other datasets","metadata":{}},{"cell_type":"code","source":"train_series_descriptions = pd.read_csv(\"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_series_descriptions.csv\")\ntrain_label_coordinates = pd.read_csv(\"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_label_coordinates.csv\")\ntest_series_descriptions = pd.read_csv(\"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_series_descriptions.csv\")\nsample_submission = pd.read_csv(\"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-09-25T11:21:22.570998Z","iopub.execute_input":"2024-09-25T11:21:22.571851Z","iopub.status.idle":"2024-09-25T11:21:22.660990Z","shell.execute_reply.started":"2024-09-25T11:21:22.571820Z","shell.execute_reply":"2024-09-25T11:21:22.660169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series_descriptions.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:21.049865Z","iopub.execute_input":"2024-09-25T10:28:21.050489Z","iopub.status.idle":"2024-09-25T10:28:21.060892Z","shell.execute_reply.started":"2024-09-25T10:28:21.050455Z","shell.execute_reply":"2024-09-25T10:28:21.059969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label_coordinates.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:21.062053Z","iopub.execute_input":"2024-09-25T10:28:21.062405Z","iopub.status.idle":"2024-09-25T10:28:21.080070Z","shell.execute_reply.started":"2024-09-25T10:28:21.062374Z","shell.execute_reply":"2024-09-25T10:28:21.078470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df['study_id'] = image_df.Path.str.split('/').str[5]\nimage_df['series_id'] = image_df.Path.str.split('/').str[6]\nimage_df['instance_number'] = image_df.Path.str.split('/').str[7].str.replace(\".dcm\",\"\")","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:21.083847Z","iopub.execute_input":"2024-09-25T10:28:21.084199Z","iopub.status.idle":"2024-09-25T10:28:22.653443Z","shell.execute_reply.started":"2024-09-25T10:28:21.084170Z","shell.execute_reply":"2024-09-25T10:28:22.652572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df['study_id'] = image_df['study_id'].astype(int)\nimage_df['series_id'] = image_df['series_id'].astype(int)\nimage_df['instance_number'] = image_df['instance_number'].astype(int)\ntrain_final = train_transform2.merge(image_df,on=['study_id'])","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:22.654622Z","iopub.execute_input":"2024-09-25T10:28:22.654946Z","iopub.status.idle":"2024-09-25T10:28:22.768597Z","shell.execute_reply.started":"2024-09-25T10:28:22.654915Z","shell.execute_reply":"2024-09-25T10:28:22.767805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final = train_final.reindex(columns=['study_id','series_id','Condition_','instance_number','Path','l1_l2','l2_l3','l3_l4','l4_l5','l5_s1'])","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:22.769555Z","iopub.execute_input":"2024-09-25T10:28:22.769854Z","iopub.status.idle":"2024-09-25T10:28:22.812615Z","shell.execute_reply.started":"2024-09-25T10:28:22.769821Z","shell.execute_reply":"2024-09-25T10:28:22.811644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final[['instance_number','l2_l3']].groupby('l2_l3').count()","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:22.813797Z","iopub.execute_input":"2024-09-25T10:28:22.814080Z","iopub.status.idle":"2024-09-25T10:28:22.831107Z","shell.execute_reply.started":"2024-09-25T10:28:22.814050Z","shell.execute_reply":"2024-09-25T10:28:22.830109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Two inputs\n# input1 - image data\n# input2 - condition\ndata = np.array(train_final.Path)\ndata_2 = np.array(train_final.Condition_)\n\nlabel_1 = np.array(train_final.l1_l2)\nlabel_2 = np.array(train_final.l2_l3)\nlabel_3 = np.array(train_final.l3_l4)\nlabel_4 = np.array(train_final.l4_l5)\nlabel_5 = np.array(train_final.l5_s1)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:22.832203Z","iopub.execute_input":"2024-09-25T10:28:22.832535Z","iopub.status.idle":"2024-09-25T10:28:22.840018Z","shell.execute_reply.started":"2024-09-25T10:28:22.832504Z","shell.execute_reply":"2024-09-25T10:28:22.839182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Encoding the categorical data","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import OneHotEncoder,LabelEncoder\nohe = OneHotEncoder(sparse_output=False)\ndata2 = ohe.fit_transform(data_2.reshape(-1,1))\n\nlabel1_encoder = LabelEncoder().fit(label_1)\nlabel2_encoder = LabelEncoder().fit(label_2)\nlabel3_encoder = LabelEncoder().fit(label_3)\nlabel4_encoder = LabelEncoder().fit(label_4)\nlabel5_encoder = LabelEncoder().fit(label_5)\n\nlabel1 = label1_encoder.transform(label_1)\nlabel2 = label2_encoder.transform(label_2)\nlabel3 = label3_encoder.transform(label_3)\nlabel4 = label4_encoder.transform(label_4)\nlabel5 = label5_encoder.transform(label_5)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:22.841103Z","iopub.execute_input":"2024-09-25T10:28:22.841425Z","iopub.status.idle":"2024-09-25T10:28:23.336166Z","shell.execute_reply.started":"2024-09-25T10:28:22.841395Z","shell.execute_reply":"2024-09-25T10:28:23.335387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.reshape(-1,1)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:23.337421Z","iopub.execute_input":"2024-09-25T10:28:23.338048Z","iopub.status.idle":"2024-09-25T10:28:23.342527Z","shell.execute_reply.started":"2024-09-25T10:28:23.338015Z","shell.execute_reply":"2024-09-25T10:28:23.341614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"combined_input = np.hstack((data2,data))","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:23.343893Z","iopub.execute_input":"2024-09-25T10:28:23.344218Z","iopub.status.idle":"2024-09-25T10:28:23.357207Z","shell.execute_reply.started":"2024-09-25T10:28:23.344195Z","shell.execute_reply":"2024-09-25T10:28:23.356279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_levels = np.stack((label1,label2,label3,label4,label5),axis=1)\ntarget_levels","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:23.366331Z","iopub.execute_input":"2024-09-25T10:28:23.366628Z","iopub.status.idle":"2024-09-25T10:28:23.373858Z","shell.execute_reply.started":"2024-09-25T10:28:23.366605Z","shell.execute_reply":"2024-09-25T10:28:23.372891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Spliting the training data into training and validation data","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nrandom_state = 42\nx_train, x_test, y_train, y_test = train_test_split(combined_input,target_levels,shuffle=True,random_state=random_state,\n                                                   stratify=target_levels)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:23.375276Z","iopub.execute_input":"2024-09-25T10:28:23.375674Z","iopub.status.idle":"2024-09-25T10:28:23.651464Z","shell.execute_reply.started":"2024-09-25T10:28:23.375612Z","shell.execute_reply":"2024-09-25T10:28:23.650686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Defining the custom dataset and dataloaders ","metadata":{}},{"cell_type":"code","source":"#Creating the dataloader\nfrom torch.utils.data import Dataset, DataLoader\nimport numpy as np\nfrom torchvision.transforms import v2\n\nfrom PIL import Image \n","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:23.652562Z","iopub.execute_input":"2024-09-25T10:28:23.652866Z","iopub.status.idle":"2024-09-25T10:28:23.694481Z","shell.execute_reply.started":"2024-09-25T10:28:23.652841Z","shell.execute_reply":"2024-09-25T10:28:23.693241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 64","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:23.695855Z","iopub.execute_input":"2024-09-25T10:28:23.696625Z","iopub.status.idle":"2024-09-25T10:28:23.700913Z","shell.execute_reply.started":"2024-09-25T10:28:23.696567Z","shell.execute_reply":"2024-09-25T10:28:23.699911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomDataset(Dataset):\n    def __init__(self,data,labels,transforms=None,augment_transform=None):\n        labels = labels\n        self.images = [(data[i][5:],data[i][:5],labels[i]) for i in range(len(data))]\n        self.transforms = transforms\n        self.augment_transform = augment_transform\n    def __len__(self):\n        return len(self.images)\n    def __getitem__(self,idx):\n        sample = np.array(pydicom.dcmread(self.images[idx][0].item()).pixel_array,dtype='uint8')\n        sample = Image.fromarray(sample)\n        condition = torch.tensor(self.images[idx][1].astype(np.float32))\n        label  = torch.tensor(self.images[idx][2].astype(np.float32))\n       \n        if self.transforms is not None:\n            if torch.all(label == 0) or torch.all(label == 2):\n                sample = self.augment_transform(sample)\n            else:\n                sample = self.transforms(sample)\n        return sample,condition,label\n\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:23.702270Z","iopub.execute_input":"2024-09-25T10:28:23.702696Z","iopub.status.idle":"2024-09-25T10:28:23.714107Z","shell.execute_reply.started":"2024-09-25T10:28:23.702643Z","shell.execute_reply":"2024-09-25T10:28:23.713141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Defining the transformation that will be applied to Majority and Minority classes","metadata":{}},{"cell_type":"code","source":"transform = v2.Compose([\n    v2.ToPILImage(),\n    v2.Resize(size=[224,224]),\n    v2.Grayscale(),\n    v2.ToTensor(),\n    v2.ToDtype(torch.float32,scale=True),\n    v2.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    \n])\n\naugment_transform = v2.Compose([\n    v2.ToPILImage(),\n    v2.Resize(size=[224,224]),\n    v2.RandomHorizontalFlip(),\n    v2.RandomRotation(20),\n    v2.RandomResizedCrop(224, scale=(0.8, 1.0)),\n    v2.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1),\n    \n    v2.Grayscale(),\n    v2.ToTensor(),\n    v2.ToDtype(torch.float32,scale=True),\n    v2.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])\n    ","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:23.715354Z","iopub.execute_input":"2024-09-25T10:28:23.715794Z","iopub.status.idle":"2024-09-25T10:28:23.786645Z","shell.execute_reply.started":"2024-09-25T10:28:23.715762Z","shell.execute_reply":"2024-09-25T10:28:23.785544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##Creating the train and test datasets\ntrain_dataset = CustomDataset(x_train,y_train,transforms=transform,augment_transform=augment_transform)\ntest_dataset = CustomDataset(x_test,y_test,transforms=transform,augment_transform=augment_transform)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:23.787878Z","iopub.execute_input":"2024-09-25T10:28:23.788211Z","iopub.status.idle":"2024-09-25T10:28:23.825487Z","shell.execute_reply.started":"2024-09-25T10:28:23.788186Z","shell.execute_reply":"2024-09-25T10:28:23.824589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Creating a custom data loaders to get all the labels inside the datasets\ncustom_train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size,shuffle=False, num_workers=4, pin_memory=True)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:23.826530Z","iopub.execute_input":"2024-09-25T10:28:23.826818Z","iopub.status.idle":"2024-09-25T10:28:23.833724Z","shell.execute_reply.started":"2024-09-25T10:28:23.826794Z","shell.execute_reply":"2024-09-25T10:28:23.832836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_ = []\nwith torch.no_grad():\n    for _,_,label in tqdm(custom_train_loader):\n        labels_.append(label.cpu().numpy())\nlabels_ = np.concatenate(labels_)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:28:23.834905Z","iopub.execute_input":"2024-09-25T10:28:23.835250Z","iopub.status.idle":"2024-09-25T10:29:46.093982Z","shell.execute_reply.started":"2024-09-25T10:28:23.835219Z","shell.execute_reply":"2024-09-25T10:29:46.092879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_.shape","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:29:46.095760Z","iopub.execute_input":"2024-09-25T10:29:46.096107Z","iopub.status.idle":"2024-09-25T10:29:46.103223Z","shell.execute_reply.started":"2024-09-25T10:29:46.096074Z","shell.execute_reply":"2024-09-25T10:29:46.102169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Calculating the class weights","metadata":{}},{"cell_type":"code","source":"#converting one hot encoded labels to class indices\n#     class_indices = np.unique(labels_)\ndef calculate_class_count(labels):\n    class_counts = np.bincount(labels.astype(int))\n    class_weights = 1./class_counts\n    return class_weights","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:29:46.104661Z","iopub.execute_input":"2024-09-25T10:29:46.104965Z","iopub.status.idle":"2024-09-25T10:29:46.115020Z","shell.execute_reply.started":"2024-09-25T10:29:46.104939Z","shell.execute_reply":"2024-09-25T10:29:46.113965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# t1 = np.array(labels_)\nt2 = np.hsplit(labels_,5)\nclass_weight1 = calculate_class_count(t2[0].flatten())\nclass_weight2 = calculate_class_count(t2[1].flatten())\nclass_weight3 = calculate_class_count(t2[2].flatten())\nclass_weight4 = calculate_class_count(t2[3].flatten())\nclass_weight5 = calculate_class_count(t2[4].flatten())","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:29:46.116221Z","iopub.execute_input":"2024-09-25T10:29:46.116538Z","iopub.status.idle":"2024-09-25T10:29:46.126271Z","shell.execute_reply.started":"2024-09-25T10:29:46.116512Z","shell.execute_reply":"2024-09-25T10:29:46.125534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_weights = (class_weight1[t2[0].astype(int).flatten()] +\n                  class_weight2[t2[1].astype(int).flatten()] +\n                  class_weight3[t2[2].astype(int).flatten()] +\n                  class_weight4[t2[3].astype(int).flatten()] +\n                  class_weight5[t2[4].astype(int).flatten()])\n                 ","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:29:46.127340Z","iopub.execute_input":"2024-09-25T10:29:46.127682Z","iopub.status.idle":"2024-09-25T10:29:46.135662Z","shell.execute_reply.started":"2024-09-25T10:29:46.127650Z","shell.execute_reply":"2024-09-25T10:29:46.134892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import WeightedRandomSampler\n# sample_weights = [class_weights[label] for label in labels_]\n    \nsampler = WeightedRandomSampler(weights=sample_weights, num_samples=len(sample_weights), replacement=True)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:29:46.136785Z","iopub.execute_input":"2024-09-25T10:29:46.137179Z","iopub.status.idle":"2024-09-25T10:29:46.143546Z","shell.execute_reply.started":"2024-09-25T10:29:46.137148Z","shell.execute_reply":"2024-09-25T10:29:46.142754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Creating the data loaders\n\ntrain_batch = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size,sampler=sampler,num_workers=4, pin_memory=True)\ntest_batch = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False,num_workers=4, pin_memory=True)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:29:46.144504Z","iopub.execute_input":"2024-09-25T10:29:46.144774Z","iopub.status.idle":"2024-09-25T10:29:46.152213Z","shell.execute_reply.started":"2024-09-25T10:29:46.144752Z","shell.execute_reply":"2024-09-25T10:29:46.151271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Defining the model - used transfer learning from efficient net model","metadata":{}},{"cell_type":"code","source":"## defining the model\nimport torch.nn.functional as F   #for torch functions like RELU\nimport torch.nn as nn\nimport torch.optim as optim\nfrom sklearn.metrics import accuracy_score","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:29:46.153519Z","iopub.execute_input":"2024-09-25T10:29:46.154089Z","iopub.status.idle":"2024-09-25T10:29:46.160346Z","shell.execute_reply.started":"2024-09-25T10:29:46.154065Z","shell.execute_reply":"2024-09-25T10:29:46.159642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n# device = torch.device(\"cpu\")","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:29:46.161405Z","iopub.execute_input":"2024-09-25T10:29:46.161756Z","iopub.status.idle":"2024-09-25T10:29:46.169066Z","shell.execute_reply.started":"2024-09-25T10:29:46.161731Z","shell.execute_reply":"2024-09-25T10:29:46.168136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learning_rate = 0.0001\nbatch_size = 64\n","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:29:46.170151Z","iopub.execute_input":"2024-09-25T10:29:46.170457Z","iopub.status.idle":"2024-09-25T10:29:46.180999Z","shell.execute_reply.started":"2024-09-25T10:29:46.170423Z","shell.execute_reply":"2024-09-25T10:29:46.180051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Trying a different model\nfrom torchvision import models\nimport sys\nsys.path.append('/kaggle/input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master')\nfrom efficientnet_pytorch import EfficientNet","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:29:46.182056Z","iopub.execute_input":"2024-09-25T10:29:46.182315Z","iopub.status.idle":"2024-09-25T10:29:46.220529Z","shell.execute_reply.started":"2024-09-25T10:29:46.182287Z","shell.execute_reply":"2024-09-25T10:29:46.219732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass Predictor(torch.nn.Module):\n    def __init__(self):\n        super(Predictor, self).__init__()\n        self.model = EfficientNet.from_name('efficientnet-b0')\n        self.state_dict = torch.load('/kaggle/input/efficientnet-pytorch/efficientnet-b0-08094119.pth')\n        self.model.load_state_dict(self.state_dict)\n        \n        self.in_features = self.model._fc.in_features\n        self.model._fc = torch.nn.Identity()\n        self.fc0 = torch.nn.Linear(self.in_features + 5 ,1000)\n        self.relu = torch.nn.ReLU()\n        self.fc1 = torch.nn.Linear(1000, 3)\n        self.fc2 = torch.nn.Linear(1000, 3)\n        self.fc3 = torch.nn.Linear(1000, 3)\n        self.fc4 = torch.nn.Linear(1000, 3)\n        self.fc5 = torch.nn.Linear(1000, 3)\n        \n\n        \n    def forward(self, x,z):\n        x = self.model(x)\n        combined = torch.cat((x,z),dim=1)\n        combined = self.fc0(combined)\n        combined = self.relu(combined)\n        output_1 = self.fc1(combined)\n        output_2 = self.fc2(combined)\n        output_3 = self.fc3(combined)\n        output_4 = self.fc4(combined)\n        output_5 = self.fc5(combined)\n        \n        return output_1,output_2,output_3,output_4,output_5\n","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:29:46.221675Z","iopub.execute_input":"2024-09-25T10:29:46.221987Z","iopub.status.idle":"2024-09-25T10:29:46.231944Z","shell.execute_reply.started":"2024-09-25T10:29:46.221957Z","shell.execute_reply":"2024-09-25T10:29:46.231068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training loop","metadata":{}},{"cell_type":"code","source":"#defining the training loop\n\n#### Train model\n\nstart_time = time.time() #(for showing time)\nimport torch.nn.functional as F\naccuracy = 0\ndef train_loop(t,epoch,dataloader,model,loss_fn1,loss_fn2,loss_fn3,loss_fn4,loss_fn5,optimizer,device):\n    size = dataloader.__len__()\n    model.train()\n    running_loss = 0.   #(set loss 0)\n    running_corrects = 0 \n    total_samples = 0\n    correct = 0\n    accuracy1 = 0\n    accuracy2 = 0\n    accuracy3 = 0\n    accuracy4 = 0\n    accuracy5 = 0\n    accuracy6 = 0\n    with tqdm(dataloader, total=len(dataloader),desc=f'Training, phase {t} :') as loader:\n        for X,z,y in loader:\n            inputs = X.to(device)\n            z = z.to(device)\n            y = y.to(device)\n            ## unpack the target variables\n            unpack1 = np.hsplit(y,5)\n            label1 = unpack1[0].long().flatten() \n            label2 = unpack1[1].long().flatten() \n            label3 = unpack1[2].long().flatten() \n            label4 = unpack1[3].long().flatten() \n            label5 = unpack1[4].long().flatten() \n            \n#             labels = y.float().to(device)\n            optimizer.zero_grad()\n            logits1,logits2,logits3,logits4,logits5 = model(inputs,z)\n            pred1  = F.softmax(logits1,dim=1)\n            pred2  = F.softmax(logits2,dim=1)\n            pred3  = F.softmax(logits3,dim=1)\n            pred4  = F.softmax(logits4,dim=1)\n            pred5  = F.softmax(logits5,dim=1)\n            pred_1    = torch.argmax(pred1,1).float()\n            pred_2    = torch.argmax(pred2,1).float()\n            pred_3    = torch.argmax(pred3,1).float()\n            pred_4    = torch.argmax(pred4,1).float()\n            pred_5    = torch.argmax(pred5,1).float()\n            loss1 = loss_fn1(logits1,label1)\n            loss2 = loss_fn2(logits2,label2)\n            loss3 = loss_fn3(logits3,label3)\n            loss4 = loss_fn4(logits4,label4)\n            loss5 = loss_fn5(logits5,label5)\n            \n            loss  = loss1 + loss2 + loss3 + loss4 + loss5 \n#             loss.requires_grad = True\n            loss.backward()\n            optimizer.step()\n            \n            \n            running_loss += loss.item()\n            \n            \n            \n\n            accuracy1 += accuracy_score(label1.cpu().detach().numpy(),pred_1.cpu().detach().numpy())\n            accuracy2 += accuracy_score(label2.cpu().detach().numpy(),pred_2.cpu().detach().numpy())\n            accuracy3 += accuracy_score(label3.cpu().detach().numpy(),pred_3.cpu().detach().numpy())\n            accuracy4 += accuracy_score(label4.cpu().detach().numpy(),pred_4.cpu().detach().numpy())\n            accuracy5 += accuracy_score(label5.cpu().detach().numpy(),pred_5.cpu().detach().numpy())\n            \n            loader.set_postfix(loss = running_loss / len(dataloader), \n                               accuracy_l1_l2 = accuracy1/len(dataloader), \n                               accuracy_l2_l3 = accuracy2/len(dataloader), \n                               accuracy_l3_l4 = accuracy3/len(dataloader), \n                               accuracy_l4_l5 = accuracy4/len(dataloader), \n                               accuracy_l5_s1 = accuracy5/len(dataloader)\n                               \n                              )\n    \n            gc.collect()\n\n            \n            torch.cuda.empty_cache()\n        ","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-09-25T10:29:46.233280Z","iopub.execute_input":"2024-09-25T10:29:46.233957Z","iopub.status.idle":"2024-09-25T10:29:46.253104Z","shell.execute_reply.started":"2024-09-25T10:29:46.233932Z","shell.execute_reply":"2024-09-25T10:29:46.252148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"use_previous_trained_weights = True\n\nif use_previous_trained_weights == False:\n    predictor = Predictor().to(device)\n    epochs = 20\n    print('Baseline model loaded')\nelse:\n    epochs = 3\n    predictor = torch.load('//kaggle/input/pretrained-saved-model/efficientnet_RSNA_trained_saved_weights_22092024 .pth')\n    print('Pretrained model loaded')\n# predictor = MultiOutputEffNet(len(l1_l2_encoder.classes_)).to(device)\n# Initialize the loss function\nloss_fn1 = nn.CrossEntropyLoss()\nloss_fn2 = nn.CrossEntropyLoss()\nloss_fn3 = nn.CrossEntropyLoss()\nloss_fn4 = nn.CrossEntropyLoss()\nloss_fn5 = nn.CrossEntropyLoss()\n# loss_fn6 = nn.CrossEntropyLoss()\n# optimizer = torch.optim.SGD(predictor.parameters(), lr=learning_rate,momentum=0.9)\noptimizer = torch.optim.Adam(predictor.parameters(), lr=learning_rate)\n# Set the random seeds\n# torch.manual_seed(42)\n# torch.cuda.manual_seed(42)\n# scripted_predictor = torch.jit.script(predictor).to(device)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:29:46.254091Z","iopub.execute_input":"2024-09-25T10:29:46.254354Z","iopub.status.idle":"2024-09-25T10:29:46.740550Z","shell.execute_reply.started":"2024-09-25T10:29:46.254332Z","shell.execute_reply":"2024-09-25T10:29:46.739616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_dataset.__getitem__(0)\nimport gc\ngc.collect()\ntorch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:29:46.741820Z","iopub.execute_input":"2024-09-25T10:29:46.742616Z","iopub.status.idle":"2024-09-25T10:29:46.923401Z","shell.execute_reply.started":"2024-09-25T10:29:46.742546Z","shell.execute_reply":"2024-09-25T10:29:46.922300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for t in range(epochs):\n    print(f\"Epoch {t+1}\\n-------------------------------\")\n    train_loop(t,epochs,train_batch,predictor, loss_fn1,loss_fn2,loss_fn3,loss_fn4,loss_fn5, optimizer,device)\nprint(\"Done!\")","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:29:46.924849Z","iopub.execute_input":"2024-09-25T10:29:46.925315Z","iopub.status.idle":"2024-09-25T10:37:19.229462Z","shell.execute_reply.started":"2024-09-25T10:29:46.925273Z","shell.execute_reply":"2024-09-25T10:37:19.228431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if use_previous_trained_weights == False:\n    torch.save(predictor,\"/kaggle/working/efficientnet_RSNA_trained_saved_weights_22092024.pth\")","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:19.230923Z","iopub.execute_input":"2024-09-25T10:37:19.231295Z","iopub.status.idle":"2024-09-25T10:37:19.235712Z","shell.execute_reply.started":"2024-09-25T10:37:19.231250Z","shell.execute_reply":"2024-09-25T10:37:19.234697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating classification report and confusion metrics","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import classification_report, confusion_matrix\ndef test_loop(epoch,num_epoch,dataloader,model,loss_fn1,loss_fn2,loss_fn3,loss_fn4,loss_fn5,optimizer,device):\n    size = dataloader.__len__()\n    model.eval()\n    val_loss = 0.   #(set loss 0)\n    val_correct1 = 0 \n    val_correct2 = 0\n    val_correct3 = 0\n    val_correct4 = 0\n    val_correct5 = 0\n    \n    \n    val_total = 0\n    all_labels1 = []\n    all_labels2 = []\n    all_labels3 = []\n    all_labels4 = []\n    all_labels5 = []\n    \n    all_preds1  = []\n    all_preds2  = []\n    all_preds3  = []\n    all_preds4  = []\n    all_preds5  = []\n    \n    \n    for batch, (X,z,y) in enumerate(tqdm(dataloader)):\n        inputs = X.to(device)\n        z = z.to(device)\n#         labels = y.float().to(device)\n        y  = y.to(device)\n        unpack1 = np.hsplit(y,5)\n        label1 = unpack1[0].long().flatten() \n        label2 = unpack1[1].long().flatten() \n        label3 = unpack1[2].long().flatten() \n        label4 = unpack1[3].long().flatten() \n        label5 = unpack1[4].long().flatten() \n        \n        optimizer.zero_grad()\n        with torch.no_grad():\n            logits1,logits2,logits3,logits4,logits5 = model(inputs,z)\n        loss1 = loss_fn1(logits1,label1)\n        loss2 = loss_fn2(logits2,label2)\n        loss3 = loss_fn3(logits3,label3)\n        loss4 = loss_fn4(logits4,label4)\n        loss5 = loss_fn5(logits5,label5)\n        \n        loss  = loss1 + loss2 + loss3 + loss4 + loss5        \n        \n        pred1  = F.softmax(logits1,dim=1)\n        pred2  = F.softmax(logits2,dim=1)\n        pred3  = F.softmax(logits3,dim=1)\n        pred4  = F.softmax(logits4,dim=1)\n        pred5  = F.softmax(logits5,dim=1)\n        \n        \n        val_loss += loss.item() * X.size(0)\n        \n        _,predicted1 = torch.max(pred1,1)\n        _,predicted2 = torch.max(pred2,1)\n        _,predicted3 = torch.max(pred3,1)\n        _,predicted4 = torch.max(pred4,1)\n        _,predicted5 = torch.max(pred5,1)\n       \n        val_correct1 +=(predicted1 == label1).sum().item()\n        val_correct2 +=(predicted2 == label2).sum().item()\n        val_correct3 +=(predicted3 == label3).sum().item()\n        val_correct4 +=(predicted4 == label4).sum().item()\n        val_correct5 +=(predicted5 == label5).sum().item()\n        \n        \n        val_total   +=label1.size(0)\n        \n        all_labels1.extend(label1.cpu().numpy())\n        all_preds1.extend(predicted1.cpu().numpy())\n\n        all_labels2.extend(label2.cpu().numpy())\n        all_preds2.extend(predicted2.cpu().numpy())\n        \n        all_labels3.extend(label3.cpu().numpy())\n        all_preds3.extend(predicted3.cpu().numpy())\n        \n        all_labels4.extend(label4.cpu().numpy())\n        all_preds4.extend(predicted4.cpu().numpy())\n        \n        all_labels5.extend(label5.cpu().numpy())\n        all_preds5.extend(predicted5.cpu().numpy())\n        gc.collect()\n        torch.cuda.empty_cache()\n        \n        \n        \n    val_loss = val_loss / len(dataloader.dataset)\n    val_acc1 =  100 * val_correct1/val_total\n    val_acc2 =  100 * val_correct2/val_total\n    val_acc3 =  100 * val_correct3/val_total\n    val_acc4 =  100 * val_correct4/val_total\n    val_acc5 =  100 * val_correct5/val_total\n    \n    \n    print(f'Validation Loss: {val_loss:.4f}')\n    print(f'Validation Accuracy1: {val_acc1:.2f}%')\n    print(f'Validation Accuracy2: {val_acc2:.2f}%')\n    print(f'Validation Accuracy3: {val_acc3:.2f}%')\n    print(f'Validation Accuracy3: {val_acc4:.2f}%')\n    print(f'Validation Accuracy5: {val_acc5:.2f}%')\n    \n    \n    # Generate classification report\n    print(\"Classification Report:\")\n    print(classification_report(all_labels1, all_preds1, target_names=['Mild', 'Normal', 'Severe']))\n    print(classification_report(all_labels2, all_preds2, target_names=['Mild', 'Normal', 'Severe']))\n    print(classification_report(all_labels3, all_preds3, target_names=['Mild', 'Normal', 'Severe']))\n    print(classification_report(all_labels4, all_preds4, target_names=['Mild', 'Normal', 'Severe']))\n    print(classification_report(all_labels5, all_preds5, target_names=['Mild', 'Normal', 'Severe']))\n    \n    # Optionally, you can also print the confusion matrix\n    print(\"Confusion Matrix:\")\n    print(confusion_matrix(all_labels1, all_preds1))\n    print(confusion_matrix(all_labels2, all_preds2))\n    print(confusion_matrix(all_labels3, all_preds3))\n    print(confusion_matrix(all_labels4, all_preds4))\n    print(confusion_matrix(all_labels5, all_preds5))\n    \n        \n        ","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:19.236969Z","iopub.execute_input":"2024-09-25T10:37:19.237253Z","iopub.status.idle":"2024-09-25T10:37:19.261974Z","shell.execute_reply.started":"2024-09-25T10:37:19.237229Z","shell.execute_reply":"2024-09-25T10:37:19.261087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_loop(1,epochs,test_batch,predictor, loss_fn1,loss_fn2,loss_fn3,loss_fn4,loss_fn5, optimizer,device)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:19.263100Z","iopub.execute_input":"2024-09-25T10:37:19.263367Z","iopub.status.idle":"2024-09-25T10:37:51.358160Z","shell.execute_reply.started":"2024-09-25T10:37:19.263343Z","shell.execute_reply":"2024-09-25T10:37:51.357189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Now we will run our model over the test data","metadata":{}},{"cell_type":"code","source":"test_series_descriptions.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:51.360165Z","iopub.execute_input":"2024-09-25T10:37:51.360643Z","iopub.status.idle":"2024-09-25T10:37:51.371715Z","shell.execute_reply.started":"2024-09-25T10:37:51.360575Z","shell.execute_reply":"2024-09-25T10:37:51.370753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_series_descriptions.series_description.unique()","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:51.372869Z","iopub.execute_input":"2024-09-25T10:37:51.373159Z","iopub.status.idle":"2024-09-25T10:37:51.382254Z","shell.execute_reply.started":"2024-09-25T10:37:51.373123Z","shell.execute_reply":"2024-09-25T10:37:51.381412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Creating a dictionary of series and condition mapping\nseries_condition ={'Sagittal T1':['left_neural_foraminal_narrowing','right_neural_foraminal_narrowing','spinal_canal_stenosis'],\n                  'Sagittal T2/STIR':['spinal_canal_stenosis'],\n                  'Axial plane':['left_subarticular_stenosis','right_subarticular_stenosis']}","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:51.383296Z","iopub.execute_input":"2024-09-25T10:37:51.383573Z","iopub.status.idle":"2024-09-25T10:37:51.391319Z","shell.execute_reply.started":"2024-09-25T10:37:51.383538Z","shell.execute_reply":"2024-09-25T10:37:51.390498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_condition_df  = pd.DataFrame([(key, condition) for key, conditions in series_condition.items() for condition in conditions], \n                  columns=['series_description', 'Condition_'])","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:51.392218Z","iopub.execute_input":"2024-09-25T10:37:51.392458Z","iopub.status.idle":"2024-09-25T10:37:51.405642Z","shell.execute_reply.started":"2024-09-25T10:37:51.392438Z","shell.execute_reply":"2024-09-25T10:37:51.404631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_condition_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:51.406870Z","iopub.execute_input":"2024-09-25T10:37:51.407345Z","iopub.status.idle":"2024-09-25T10:37:51.419365Z","shell.execute_reply.started":"2024-09-25T10:37:51.407312Z","shell.execute_reply":"2024-09-25T10:37:51.418436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_df_complete = test_series_descriptions.merge(series_condition_df,on='series_description')\nseries_df_complete.series_description.unique","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:51.420668Z","iopub.execute_input":"2024-09-25T10:37:51.421099Z","iopub.status.idle":"2024-09-25T10:37:51.431789Z","shell.execute_reply.started":"2024-09-25T10:37:51.421065Z","shell.execute_reply":"2024-09-25T10:37:51.430910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_image_paths = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_images/*/*/*'\n#Creating the dataloader for\ntest_image_df = pd.DataFrame(glob.glob(test_image_paths),columns=['Path'])\ntest_image_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:51.432898Z","iopub.execute_input":"2024-09-25T10:37:51.433297Z","iopub.status.idle":"2024-09-25T10:37:51.468899Z","shell.execute_reply.started":"2024-09-25T10:37:51.433265Z","shell.execute_reply":"2024-09-25T10:37:51.468018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_image_df['study_id']        = test_image_df.Path.str.split('/').str[5]\ntest_image_df['series_id']       = test_image_df.Path.str.split('/').str[6]\ntest_image_df['instance_number'] = test_image_df.Path.str.split('/').str[7].str.replace(\".dcm\",\"\")\ntest_image_df['study_id']        = test_image_df['study_id'].astype(int)\ntest_image_df['series_id']       = test_image_df['series_id'].astype(int)\ntest_image_df['instance_number'] = test_image_df['instance_number'].astype(int)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:51.470180Z","iopub.execute_input":"2024-09-25T10:37:51.470473Z","iopub.status.idle":"2024-09-25T10:37:51.480724Z","shell.execute_reply.started":"2024-09-25T10:37:51.470447Z","shell.execute_reply":"2024-09-25T10:37:51.479794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_image_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:51.482067Z","iopub.execute_input":"2024-09-25T10:37:51.482494Z","iopub.status.idle":"2024-09-25T10:37:51.495524Z","shell.execute_reply.started":"2024-09-25T10:37:51.482462Z","shell.execute_reply":"2024-09-25T10:37:51.494641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test_df = series_df_complete.merge(test_image_df,on=['study_id','series_id'])","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:51.496699Z","iopub.execute_input":"2024-09-25T10:37:51.497042Z","iopub.status.idle":"2024-09-25T10:37:51.506374Z","shell.execute_reply.started":"2024-09-25T10:37:51.497018Z","shell.execute_reply":"2024-09-25T10:37:51.505652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:51.507433Z","iopub.execute_input":"2024-09-25T10:37:51.507938Z","iopub.status.idle":"2024-09-25T10:37:51.520425Z","shell.execute_reply.started":"2024-09-25T10:37:51.507911Z","shell.execute_reply":"2024-09-25T10:37:51.519475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test_df.Condition_.unique()","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:51.521704Z","iopub.execute_input":"2024-09-25T10:37:51.522134Z","iopub.status.idle":"2024-09-25T10:37:51.530220Z","shell.execute_reply.started":"2024-09-25T10:37:51.522101Z","shell.execute_reply":"2024-09-25T10:37:51.529324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = np.array(final_test_df.Path)\ntest_data_2 = np.array(final_test_df.Condition_)\ntest_data_3 = np.array(final_test_df.study_id)\ntest_data_4 = np.array(final_test_df.series_id)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:51.531184Z","iopub.execute_input":"2024-09-25T10:37:51.531433Z","iopub.status.idle":"2024-09-25T10:37:51.538660Z","shell.execute_reply.started":"2024-09-25T10:37:51.531410Z","shell.execute_reply":"2024-09-25T10:37:51.537822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_2_encoded = ohe.transform(test_data_2.reshape(-1,1))\ntest_data = test_data.reshape(-1,1)\ncombined_test_input = np.hstack((test_data_2_encoded,test_data,test_data_3.reshape(-1,1),test_data_4.reshape(-1,1)))","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:51.540086Z","iopub.execute_input":"2024-09-25T10:37:51.540464Z","iopub.status.idle":"2024-09-25T10:37:51.549152Z","shell.execute_reply.started":"2024-09-25T10:37:51.540432Z","shell.execute_reply":"2024-09-25T10:37:51.548195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"combined_test_input.shape","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:51.550643Z","iopub.execute_input":"2024-09-25T10:37:51.551034Z","iopub.status.idle":"2024-09-25T10:37:51.559190Z","shell.execute_reply.started":"2024-09-25T10:37:51.551007Z","shell.execute_reply":"2024-09-25T10:37:51.558376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##Creating the test dataset\nclass CustomTestDataset(Dataset):\n    def __init__(self,data,transforms=None,augment_transform=None):\n#         print(data)\n        self.images = [(data[i][5],data[i][:5],data[i][6],data[i][7]) for i in range(len(data))]\n        self.transforms = transforms\n        self.augment_transform = augment_transform\n    def __len__(self):\n        return len(self.images)\n    def __getitem__(self,idx):\n#         print(self.images[idx])\n        sample = np.array(pydicom.dcmread(self.images[idx][0]).pixel_array,dtype='uint8')\n        sample = Image.fromarray(sample)\n        condition = torch.tensor(self.images[idx][1].astype(np.float32))\n        path      = self.images[idx][0]\n        study_id  = self.images[idx][2]\n        series_id = self.images[idx][3]\n        if self.transforms is not None:\n            sample = self.transforms(sample)\n        return sample,condition,path,study_id,series_id\n","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:51.560272Z","iopub.execute_input":"2024-09-25T10:37:51.560572Z","iopub.status.idle":"2024-09-25T10:37:51.572469Z","shell.execute_reply.started":"2024-09-25T10:37:51.560543Z","shell.execute_reply":"2024-09-25T10:37:51.571705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Final_test_dataset = CustomTestDataset(combined_test_input,transforms=transform)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:51.573486Z","iopub.execute_input":"2024-09-25T10:37:51.573826Z","iopub.status.idle":"2024-09-25T10:37:51.581834Z","shell.execute_reply.started":"2024-09-25T10:37:51.573803Z","shell.execute_reply":"2024-09-25T10:37:51.581034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Final_test_batch = torch.utils.data.DataLoader(Final_test_dataset, batch_size=batch_size, shuffle=False,num_workers=4, pin_memory=True)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:51.582854Z","iopub.execute_input":"2024-09-25T10:37:51.583113Z","iopub.status.idle":"2024-09-25T10:37:51.592003Z","shell.execute_reply.started":"2024-09-25T10:37:51.583090Z","shell.execute_reply":"2024-09-25T10:37:51.591077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test_loop(dataloader,model,device):\n    size = dataloader.__len__()\n    model.eval()\n    pred1 = []\n    pred2 = []\n    pred3 = []\n    pred4 = []\n    pred5 = []\n    image_path = []\n    condition = []\n    study_id_list = []\n    series_id_list = []\n    for batch, (X,z,path,study_id,series_id) in enumerate(tqdm(dataloader)):\n        inputs = X.to(device)\n        z = z.to(device)\n        with torch.no_grad():\n            logits1,logits2,logits3,logits4,logits5 = model(inputs,z)\n        \n        pred1.append(F.softmax(logits1,dim=1))\n        pred2.append(F.softmax(logits2,dim=1))\n        pred3.append(F.softmax(logits3,dim=1))\n        pred4.append(F.softmax(logits4,dim=1))\n        pred5.append(F.softmax(logits5,dim=1))\n        image_path.append(path)\n        condition.append(z)\n        study_id_list.append(study_id)\n        series_id_list.append(series_id)\n    return image_path,pred1,pred2,pred3,pred4,pred5,study_id_list,series_id_list,condition","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:51.593112Z","iopub.execute_input":"2024-09-25T10:37:51.593390Z","iopub.status.idle":"2024-09-25T10:37:51.603288Z","shell.execute_reply.started":"2024-09-25T10:37:51.593365Z","shell.execute_reply":"2024-09-25T10:37:51.602356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_path_arr,pred1,pred2,pred3,pred4,pred5,study_id,series_id,condition = test_loop(Final_test_batch,predictor,device)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:51.604149Z","iopub.execute_input":"2024-09-25T10:37:51.604394Z","iopub.status.idle":"2024-09-25T10:37:53.865215Z","shell.execute_reply.started":"2024-09-25T10:37:51.604368Z","shell.execute_reply":"2024-09-25T10:37:53.864099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"condition = torch.cat(condition,dim=0)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:53.866817Z","iopub.execute_input":"2024-09-25T10:37:53.867101Z","iopub.status.idle":"2024-09-25T10:37:53.872055Z","shell.execute_reply.started":"2024-09-25T10:37:53.867074Z","shell.execute_reply":"2024-09-25T10:37:53.871019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_path = []\nfor i in range(len(image_path_arr)):\n    image_path += image_path_arr[i]\nstudy_id   = torch.cat(study_id,dim=0).cpu().numpy().reshape(-1,1)\nseries_id  = torch.cat(series_id,dim=0).cpu().numpy().reshape(-1,1)\ncondition_decode = ohe.inverse_transform(condition.cpu().numpy())","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:53.873125Z","iopub.execute_input":"2024-09-25T10:37:53.873417Z","iopub.status.idle":"2024-09-25T10:37:53.882357Z","shell.execute_reply.started":"2024-09-25T10:37:53.873393Z","shell.execute_reply":"2024-09-25T10:37:53.881624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\npred1 = torch.cat(pred1,dim=0).cpu().numpy().reshape(-1,3)\npred2 = torch.cat(pred2,dim=0).cpu().numpy().reshape(-1,3)\npred3 = torch.cat(pred3,dim=0).cpu().numpy().reshape(-1,3)\npred4 = torch.cat(pred4,dim=0).cpu().numpy().reshape(-1,3)\npred5 = torch.cat(pred5,dim=0).cpu().numpy().reshape(-1,3)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:53.883505Z","iopub.execute_input":"2024-09-25T10:37:53.883959Z","iopub.status.idle":"2024-09-25T10:37:53.892345Z","shell.execute_reply.started":"2024-09-25T10:37:53.883928Z","shell.execute_reply":"2024-09-25T10:37:53.891258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Result_df = pd.DataFrame(image_path,columns=['Path'])\nResult_df['study_id'] = study_id.tolist()\nResult_df['series_id'] = series_id.tolist()","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:53.893265Z","iopub.execute_input":"2024-09-25T10:37:53.893695Z","iopub.status.idle":"2024-09-25T10:37:53.904284Z","shell.execute_reply.started":"2024-09-25T10:37:53.893658Z","shell.execute_reply":"2024-09-25T10:37:53.903378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Result_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:53.905384Z","iopub.execute_input":"2024-09-25T10:37:53.905795Z","iopub.status.idle":"2024-09-25T10:37:53.924012Z","shell.execute_reply.started":"2024-09-25T10:37:53.905770Z","shell.execute_reply":"2024-09-25T10:37:53.922974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Result_df['l1_l2'] = pred1.tolist()\nResult_df['l2_l3'] = pred2.tolist()\nResult_df['l3_l4'] = pred3.tolist()\nResult_df['l4_l5'] = pred4.tolist()\nResult_df['l5_s1'] = pred5.tolist()\n","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:53.925008Z","iopub.execute_input":"2024-09-25T10:37:53.925298Z","iopub.status.idle":"2024-09-25T10:37:53.933355Z","shell.execute_reply.started":"2024-09-25T10:37:53.925260Z","shell.execute_reply":"2024-09-25T10:37:53.932610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Result_df['Condition'] = condition_decode.tolist()","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:53.934335Z","iopub.execute_input":"2024-09-25T10:37:53.934612Z","iopub.status.idle":"2024-09-25T10:37:53.946488Z","shell.execute_reply.started":"2024-09-25T10:37:53.934568Z","shell.execute_reply":"2024-09-25T10:37:53.945633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Result_df1 = Result_df.drop('Path',axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:53.957669Z","iopub.execute_input":"2024-09-25T10:37:53.957932Z","iopub.status.idle":"2024-09-25T10:37:53.962650Z","shell.execute_reply.started":"2024-09-25T10:37:53.957908Z","shell.execute_reply":"2024-09-25T10:37:53.961809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Result_df_transform = pd.melt(Result_df1,id_vars=['study_id','series_id','Condition'],var_name='levels',value_name='Severity')","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:53.963837Z","iopub.execute_input":"2024-09-25T10:37:53.964149Z","iopub.status.idle":"2024-09-25T10:37:53.975515Z","shell.execute_reply.started":"2024-09-25T10:37:53.964119Z","shell.execute_reply":"2024-09-25T10:37:53.974555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Result_df_transform[['Moderate','Normal','Severe']] = pd.DataFrame(Result_df_transform['Severity'].tolist(),index = Result_df_transform.index)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:53.976723Z","iopub.execute_input":"2024-09-25T10:37:53.976977Z","iopub.status.idle":"2024-09-25T10:37:53.985931Z","shell.execute_reply.started":"2024-09-25T10:37:53.976955Z","shell.execute_reply":"2024-09-25T10:37:53.985127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Result_df_transform.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:53.986962Z","iopub.execute_input":"2024-09-25T10:37:53.987295Z","iopub.status.idle":"2024-09-25T10:37:54.008911Z","shell.execute_reply.started":"2024-09-25T10:37:53.987229Z","shell.execute_reply":"2024-09-25T10:37:54.007982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Result_df_transform.drop(columns=['series_id','Severity'],inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:54.010216Z","iopub.execute_input":"2024-09-25T10:37:54.010593Z","iopub.status.idle":"2024-09-25T10:37:54.016919Z","shell.execute_reply.started":"2024-09-25T10:37:54.010546Z","shell.execute_reply":"2024-09-25T10:37:54.016000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Result_df_transform['study_id'] = Result_df_transform['study_id'].apply(lambda x: x[0])\nResult_df_transform['Condition'] = Result_df_transform['Condition'].apply(lambda x: x[0])","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:54.017999Z","iopub.execute_input":"2024-09-25T10:37:54.018257Z","iopub.status.idle":"2024-09-25T10:37:54.028232Z","shell.execute_reply.started":"2024-09-25T10:37:54.018234Z","shell.execute_reply":"2024-09-25T10:37:54.027334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Finalizing the ouput predictions","metadata":{}},{"cell_type":"code","source":"aggregated_df = Result_df_transform.groupby(['study_id', 'Condition', 'levels']).agg({\n    'Normal': 'mean',\n    'Moderate': 'mean',\n    'Severe': 'mean'\n}).reset_index()","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:54.029273Z","iopub.execute_input":"2024-09-25T10:37:54.029537Z","iopub.status.idle":"2024-09-25T10:37:54.046385Z","shell.execute_reply.started":"2024-09-25T10:37:54.029513Z","shell.execute_reply":"2024-09-25T10:37:54.045449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aggregated_df","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:54.047487Z","iopub.execute_input":"2024-09-25T10:37:54.047818Z","iopub.status.idle":"2024-09-25T10:37:54.061750Z","shell.execute_reply.started":"2024-09-25T10:37:54.047792Z","shell.execute_reply":"2024-09-25T10:37:54.060739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aggregated_df['row_id'] = aggregated_df['study_id'].astype(str) + '_' + aggregated_df['Condition'].astype(str) + '_' + aggregated_df['levels'].astype(str)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:54.062905Z","iopub.execute_input":"2024-09-25T10:37:54.063201Z","iopub.status.idle":"2024-09-25T10:37:54.071391Z","shell.execute_reply.started":"2024-09-25T10:37:54.063176Z","shell.execute_reply":"2024-09-25T10:37:54.070634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aggregated_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:54.074602Z","iopub.execute_input":"2024-09-25T10:37:54.074892Z","iopub.status.idle":"2024-09-25T10:37:54.090238Z","shell.execute_reply.started":"2024-09-25T10:37:54.074868Z","shell.execute_reply":"2024-09-25T10:37:54.089182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aggregated_df.drop(columns=['study_id','Condition','levels'],inplace=True)\naggregated_df.rename(columns={'Normal':'normal_mild','Moderate':'moderate','Severe':'severe'},inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:54.091474Z","iopub.execute_input":"2024-09-25T10:37:54.091846Z","iopub.status.idle":"2024-09-25T10:37:54.100021Z","shell.execute_reply.started":"2024-09-25T10:37:54.091813Z","shell.execute_reply":"2024-09-25T10:37:54.098826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aggregated_df = aggregated_df[['row_id','normal_mild','moderate','severe']]","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:54.101216Z","iopub.execute_input":"2024-09-25T10:37:54.101544Z","iopub.status.idle":"2024-09-25T10:37:54.109519Z","shell.execute_reply.started":"2024-09-25T10:37:54.101519Z","shell.execute_reply":"2024-09-25T10:37:54.108553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aggregated_df.dtypes","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:37:54.110768Z","iopub.execute_input":"2024-09-25T10:37:54.111048Z","iopub.status.idle":"2024-09-25T10:37:54.122521Z","shell.execute_reply.started":"2024-09-25T10:37:54.111024Z","shell.execute_reply":"2024-09-25T10:37:54.121621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_normalize = ['normal_mild', 'moderate', 'severe']\n\n# # Normalizing each row\n# aggregated_df[columns_to_normalize] = aggregated_df[columns_to_normalize].div(aggregated_df[columns_to_normalize].sum(axis=1), axis=0)\n\n# Optionally round the values after normalizing to avoid precision issues\naggregated_df[columns_to_normalize] = aggregated_df[columns_to_normalize].round(6)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T11:20:48.645928Z","iopub.execute_input":"2024-09-25T11:20:48.646300Z","iopub.status.idle":"2024-09-25T11:20:48.655096Z","shell.execute_reply.started":"2024-09-25T11:20:48.646271Z","shell.execute_reply":"2024-09-25T11:20:48.654037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.drop(columns=columns_to_normalize,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T11:21:35.388975Z","iopub.execute_input":"2024-09-25T11:21:35.389596Z","iopub.status.idle":"2024-09-25T11:21:35.394628Z","shell.execute_reply.started":"2024-09-25T11:21:35.389556Z","shell.execute_reply":"2024-09-25T11:21:35.393635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = sample_submission.merge(aggregated_df,on='row_id',how='left')","metadata":{"execution":{"iopub.status.busy":"2024-09-25T11:25:56.757990Z","iopub.execute_input":"2024-09-25T11:25:56.758608Z","iopub.status.idle":"2024-09-25T11:25:56.766225Z","shell.execute_reply.started":"2024-09-25T11:25:56.758562Z","shell.execute_reply":"2024-09-25T11:25:56.765189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.loc[submission_df[columns_to_normalize].isna().any(axis=1),['normal_mild', 'moderate', 'severe']] = [0.333333, 0.333333, 0.333333]\nsubmission_df[columns_to_normalize] = submission_df[columns_to_normalize].round(6)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T11:25:58.639495Z","iopub.execute_input":"2024-09-25T11:25:58.639866Z","iopub.status.idle":"2024-09-25T11:25:58.650469Z","shell.execute_reply.started":"2024-09-25T11:25:58.639836Z","shell.execute_reply":"2024-09-25T11:25:58.649342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df","metadata":{"execution":{"iopub.status.busy":"2024-09-25T11:26:00.724776Z","iopub.execute_input":"2024-09-25T11:26:00.725141Z","iopub.status.idle":"2024-09-25T11:26:00.739547Z","shell.execute_reply.started":"2024-09-25T11:26:00.725111Z","shell.execute_reply":"2024-09-25T11:26:00.738641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# submit the notebook to Output folder\nsubmission_df.to_csv('/kaggle/working/submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2024-09-25T11:27:07.757729Z","iopub.execute_input":"2024-09-25T11:27:07.758409Z","iopub.status.idle":"2024-09-25T11:27:07.763997Z","shell.execute_reply.started":"2024-09-25T11:27:07.758379Z","shell.execute_reply":"2024-09-25T11:27:07.762978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}