{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"dockerImageVersionId":30823,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\ncounter = 0  # Sayaç başlat\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        counter += 1\n        if counter == 15:  # 5 dosya yazdırdıktan sonra dur\n            break\n    if counter == 15:  # İç döngü kırıldığında dış döngüyü de kır\n        break\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true,"execution":{"iopub.status.busy":"2025-01-14T14:14:18.123782Z","iopub.execute_input":"2025-01-14T14:14:18.124050Z","iopub.status.idle":"2025-01-14T14:14:22.845326Z","shell.execute_reply.started":"2025-01-14T14:14:18.124027Z","shell.execute_reply":"2025-01-14T14:14:22.844636Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport pydicom\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models, optimizers\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nimport cv2\n","metadata":{"execution":{"iopub.status.busy":"2025-01-14T14:14:22.848733Z","iopub.execute_input":"2025-01-14T14:14:22.849025Z","iopub.status.idle":"2025-01-14T14:14:32.058225Z","shell.execute_reply.started":"2025-01-14T14:14:22.848996Z","shell.execute_reply":"2025-01-14T14:14:32.057474Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# read data\ntrain_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/'\n\ntrain  = pd.read_csv(train_path + 'train.csv')\nlabel = pd.read_csv(train_path + 'train_label_coordinates.csv')\ntrain_desc  = pd.read_csv(train_path + 'train_series_descriptions.csv')\ntest_desc   = pd.read_csv(train_path + 'test_series_descriptions.csv')\nsub         = pd.read_csv(train_path + 'sample_submission.csv')\nlen(test_desc) #number of test_description.csv rows ","metadata":{"execution":{"iopub.status.busy":"2025-01-14T14:14:32.059274Z","iopub.execute_input":"2025-01-14T14:14:32.059734Z","iopub.status.idle":"2025-01-14T14:14:32.224579Z","shell.execute_reply.started":"2025-01-14T14:14:32.059710Z","shell.execute_reply":"2025-01-14T14:14:32.223771Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train_desc.columns)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T14:20:12.658587Z","iopub.execute_input":"2025-01-14T14:20:12.658975Z","iopub.status.idle":"2025-01-14T14:20:12.665438Z","shell.execute_reply.started":"2025-01-14T14:20:12.658946Z","shell.execute_reply":"2025-01-14T14:20:12.664468Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def generate_image_paths_with_naming(df, data_dir):\n    image_paths = []\n    for index, row in df.iterrows():\n        study_id = str(row['study_id'])\n        condition = row['series_description'].replace(' ', '_').lower()  # Series description bilgisi kullanılabilir\n        level = row['series_description'].split('/')[-1]  # Seviyeyi serinin adının son kısmından alabiliriz\n        instance_number = str(row['instance_number'])\n\n        # Klasör yapısını oluştur\n        study_dir = os.path.join(data_dir, study_id)\n        series_dir = os.path.join(study_dir, row['series_id'], instance_number + '.dcm')\n\n        if os.path.exists(series_dir):\n            image_paths.append(series_dir)\n\n    return image_paths\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T14:20:33.350109Z","iopub.execute_input":"2025-01-14T14:20:33.350415Z","iopub.status.idle":"2025-01-14T14:20:33.355777Z","shell.execute_reply.started":"2025-01-14T14:20:33.350391Z","shell.execute_reply":"2025-01-14T14:20:33.354821Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Veri ön işleme işlemleri\ndef load_dicom_images(image_paths, target_size=(128, 128)):\n    images = []\n    for path in image_paths:\n        ds = pydicom.dcmread(path)\n        img = ds.pixel_array\n        img = cv2.resize(img, target_size)  # Görselleri boyutlandır\n        img = img / np.max(img)  # Normalize et\n        images.append(img)\n    return np.array(images, dtype=np.float32).reshape(-1, target_size[0], target_size[1], 1)\n\n# Eğitim ve test görüntü yollarını al\ntrain_image_paths = generate_image_paths(train_desc, f'{train_path}/train_images')\ntest_image_paths = generate_image_paths(test_desc, f'{train_path}/test_images')\n\n# Eğitim ve test görüntülerini yükle\ntrain_images = load_dicom_images(train_image_paths)\ntest_images = load_dicom_images(test_image_paths)\n\n# Etiketleri encode et\nencoder = LabelEncoder()\nlabels = final_merged_df['severity'].values\nlabels_encoded = encoder.fit_transform(labels)\n\n# Eğitim ve doğrulama verilerini ayır\nX_train, X_val, y_train, y_val = train_test_split(train_images, labels_encoded, test_size=0.2, random_state=42)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T14:28:14.036256Z","iopub.execute_input":"2025-01-14T14:28:14.036604Z","iopub.status.idle":"2025-01-14T14:28:14.056218Z","shell.execute_reply.started":"2025-01-14T14:28:14.036577Z","shell.execute_reply":"2025-01-14T14:28:14.055101Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_desc.head(5)","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:35:31.389567Z","iopub.execute_input":"2025-01-08T21:35:31.389846Z","iopub.status.idle":"2025-01-08T21:35:31.402977Z","shell.execute_reply.started":"2025-01-08T21:35:31.389825Z","shell.execute_reply":"2025-01-08T21:35:31.402190Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_desc.head(5)","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:35:33.225382Z","iopub.execute_input":"2025-01-08T21:35:33.225698Z","iopub.status.idle":"2025-01-08T21:35:33.233355Z","shell.execute_reply.started":"2025-01-08T21:35:33.225674Z","shell.execute_reply":"2025-01-08T21:35:33.232399Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head(5)","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:35:35.387329Z","iopub.execute_input":"2025-01-08T21:35:35.387675Z","iopub.status.idle":"2025-01-08T21:35:35.407970Z","shell.execute_reply.started":"2025-01-08T21:35:35.387646Z","shell.execute_reply":"2025-01-08T21:35:35.407256Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function to generate image paths based on directory structure\ndef generate_image_paths(df, data_dir):\n    image_paths = []\n    for study_id, series_id in zip(df['study_id'], df['series_id']):\n        study_dir = os.path.join(data_dir, str(study_id))\n        series_dir = os.path.join(study_dir, str(series_id))\n        images = os.listdir(series_dir)\n        image_paths.extend([os.path.join(series_dir, img) for img in images])\n    return image_paths\n\n# Generate image paths for train and test data\ntrain_image_paths = generate_image_paths(train_desc, f'{train_path}/train_images')\ntest_image_paths = generate_image_paths(test_desc, f'{train_path}/test_images')","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:35:37.496002Z","iopub.execute_input":"2025-01-08T21:35:37.496283Z","iopub.status.idle":"2025-01-08T21:36:44.446442Z","shell.execute_reply.started":"2025-01-08T21:35:37.496261Z","shell.execute_reply":"2025-01-08T21:36:44.445690Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(train_desc)","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:36:44.447420Z","iopub.execute_input":"2025-01-08T21:36:44.447749Z","iopub.status.idle":"2025-01-08T21:36:44.452312Z","shell.execute_reply.started":"2025-01-08T21:36:44.447724Z","shell.execute_reply":"2025-01-08T21:36:44.451478Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(train_image_paths)","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:36:44.453512Z","iopub.execute_input":"2025-01-08T21:36:44.453810Z","iopub.status.idle":"2025-01-08T21:36:44.467837Z","shell.execute_reply.started":"2025-01-08T21:36:44.453775Z","shell.execute_reply":"2025-01-08T21:36:44.467150Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define function to reshape a single row of the DataFrame\ndef reshape_row(row):\n    data = {'study_id': [], 'condition': [], 'level': [], 'severity': []}\n    \n    for column, value in row.items():\n        if column not in ['study_id', 'series_id', 'instance_number', 'x', 'y', 'series_description']:\n            parts = column.split('_')\n            condition = ' '.join([word.capitalize() for word in parts[:-2]])\n            level = parts[-2].capitalize() + '/' + parts[-1].capitalize()\n            data['study_id'].append(row['study_id'])\n            data['condition'].append(condition)\n            data['level'].append(level)\n            data['severity'].append(value)\n    \n    return pd.DataFrame(data)\n\n# Reshape the DataFrame for all rows\nnew_train_df = pd.concat([reshape_row(row) for _, row in train.iterrows()], ignore_index=True)\n\n# Display the first few rows of the reshaped dataframe\nnew_train_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:36:47.506686Z","iopub.execute_input":"2025-01-08T21:36:47.506970Z","iopub.status.idle":"2025-01-08T21:36:48.548816Z","shell.execute_reply.started":"2025-01-08T21:36:47.506950Z","shell.execute_reply":"2025-01-08T21:36:48.547936Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Print columns in a neat way\nprint(\"\\nColumns in new_train_df:\")\nprint(\",\".join(new_train_df.columns))\n\nprint(\"\\nColumns in label:\")\nprint(\",\".join(label.columns))\n\nprint(\"\\nColumns in test_desc:\")\nprint(\",\".join(test_desc.columns))\n\nprint(\"\\nColumns in sub:\")\nprint(\",\".join(sub.columns))","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:36:50.592000Z","iopub.execute_input":"2025-01-08T21:36:50.592277Z","iopub.status.idle":"2025-01-08T21:36:50.598944Z","shell.execute_reply.started":"2025-01-08T21:36:50.592257Z","shell.execute_reply":"2025-01-08T21:36:50.598118Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Merge the dataframes on the common columns\nmerged_df = pd.merge(new_train_df, label, on=['study_id', 'condition', 'level'], how='inner')\n# Merge the dataframes on the common column 'series_id'\nfinal_merged_df = pd.merge(merged_df, train_desc, on='series_id', how='inner')","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:36:52.453883Z","iopub.execute_input":"2025-01-08T21:36:52.454165Z","iopub.status.idle":"2025-01-08T21:36:52.517501Z","shell.execute_reply.started":"2025-01-08T21:36:52.454143Z","shell.execute_reply":"2025-01-08T21:36:52.516804Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Merge the dataframes on the common column 'series_id'\nfinal_merged_df = pd.merge(merged_df, train_desc, on=['series_id','study_id'], how='inner')\n# Display the first few rows of the final merged dataframe\nfinal_merged_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:36:54.100050Z","iopub.execute_input":"2025-01-08T21:36:54.100382Z","iopub.status.idle":"2025-01-08T21:36:54.126895Z","shell.execute_reply.started":"2025-01-08T21:36:54.100310Z","shell.execute_reply":"2025-01-08T21:36:54.126073Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n# Create the row_id column\nfinal_merged_df['row_id'] = (\n    final_merged_df['study_id'].astype(str) + '_' +\n    final_merged_df['condition'].str.lower().str.replace(' ', '_') + '_' +\n    final_merged_df['level'].str.lower().str.replace('/', '_')\n)\n\n# Create the image_path column\nfinal_merged_df['image_path'] = (\n    f'{train_path}/train_images/' + \n    final_merged_df['study_id'].astype(str) + '/' +\n    final_merged_df['series_id'].astype(str) + '/' +\n    final_merged_df['instance_number'].astype(str) + '.dcm'\n)\n\n# Note: Check image path, since there's 1 instance id, for 1 image, but there's many more images other than the ones labelled in the instance ID. \n\n# Display the updated dataframe\nfinal_merged_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:36:55.963841Z","iopub.execute_input":"2025-01-08T21:36:55.964121Z","iopub.status.idle":"2025-01-08T21:36:56.143554Z","shell.execute_reply.started":"2025-01-08T21:36:55.964100Z","shell.execute_reply":"2025-01-08T21:36:56.142720Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_merged_df[final_merged_df[\"severity\"] == \"Normal/Mild\"].value_counts().sum()","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:36:59.741933Z","iopub.execute_input":"2025-01-08T21:36:59.742260Z","iopub.status.idle":"2025-01-08T21:36:59.844611Z","shell.execute_reply.started":"2025-01-08T21:36:59.742231Z","shell.execute_reply":"2025-01-08T21:36:59.843815Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_merged_df[final_merged_df[\"severity\"] == \"Moderate\"].value_counts().sum()","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:37:01.210079Z","iopub.execute_input":"2025-01-08T21:37:01.210417Z","iopub.status.idle":"2025-01-08T21:37:01.244882Z","shell.execute_reply.started":"2025-01-08T21:37:01.210384Z","shell.execute_reply":"2025-01-08T21:37:01.244248Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_merged_df[final_merged_df[\"severity\"] == \"Severe\"].value_counts().sum()","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:37:02.685194Z","iopub.execute_input":"2025-01-08T21:37:02.685545Z","iopub.status.idle":"2025-01-08T21:37:02.707964Z","shell.execute_reply.started":"2025-01-08T21:37:02.685514Z","shell.execute_reply":"2025-01-08T21:37:02.707267Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import pandas as pd\n\n# # En düşük sınıf sayısını belirleyelim\n# min_class_count = 3081\n\n# # Normal/Mild ve Moderate sınıflarını azaltalım\n# normal_mild_df = final_merged_df[final_merged_df[\"severity\"] == \"Normal/Mild\"].sample(n=min_class_count, random_state=42)\n# moderate_df = final_merged_df[final_merged_df[\"severity\"] == \"Moderate\"].sample(n=min_class_count, random_state=42)\n# severe_df = final_merged_df[final_merged_df[\"severity\"] == \"Severe\"]\n\n# # İndeksleri sıfırlayalım\n# normal_mild_df = normal_mild_df.reset_index(drop=True)\n# moderate_df = moderate_df.reset_index(drop=True)\n# severe_df = severe_df.reset_index(drop=True)\n\n# # Verileri birleştirelim ve final_merged_df'yi güncelleyelim\n# final_merged_df = pd.concat([normal_mild_df, moderate_df, severe_df])\n\n# # Sonuçları kontrol edelim\n# print(final_merged_df[\"severity\"].value_counts())\n","metadata":{"execution":{"iopub.execute_input":"2024-12-18T23:02:13.340662Z","iopub.status.busy":"2024-12-18T23:02:13.340328Z","iopub.status.idle":"2024-12-18T23:02:13.385922Z","shell.execute_reply":"2024-12-18T23:02:13.384961Z","shell.execute_reply.started":"2024-12-18T23:02:13.340635Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define the base path for test images\nbase_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_images/'\n\n# Function to get image paths for a series\ndef get_image_paths(row):\n    series_path = os.path.join(base_path, str(row['study_id']), str(row['series_id']))\n    if os.path.exists(series_path):\n        return [os.path.join(series_path, f) for f in os.listdir(series_path) if os.path.isfile(os.path.join(series_path, f))]\n    return []\n\n# Mapping of series_description to conditions\ncondition_mapping = {\n    'Sagittal T1': {'left': 'left_neural_foraminal_narrowing', 'right': 'right_neural_foraminal_narrowing'},\n    'Axial T2': {'left': 'left_subarticular_stenosis', 'right': 'right_subarticular_stenosis'},\n    'Sagittal T2/STIR': 'spinal_canal_stenosis'\n}\n\n# Create a list to store the expanded rows\nexpanded_rows = []\n\n# Expand the dataframe by adding new rows for each file path\nfor index, row in test_desc.iterrows():\n    image_paths = get_image_paths(row)\n    conditions = condition_mapping.get(row['series_description'], {})\n    if isinstance(conditions, str):  # Single condition\n        conditions = {'left': conditions, 'right': conditions}\n    for side, condition in conditions.items():\n        for image_path in image_paths:\n            expanded_rows.append({\n                'study_id': row['study_id'],\n                'series_id': row['series_id'],\n                'series_description': row['series_description'],\n                'image_path': image_path,\n                'condition': condition,\n                'row_id': f\"{row['study_id']}_{condition}\"\n            })\n\n# Create a new dataframe from the expanded rows\nexpanded_test_desc = pd.DataFrame(expanded_rows)\n\n# Display the resulting dataframe\nexpanded_test_desc.head(5)","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:37:10.013112Z","iopub.execute_input":"2025-01-08T21:37:10.013458Z","iopub.status.idle":"2025-01-08T21:37:10.178939Z","shell.execute_reply.started":"2025-01-08T21:37:10.013427Z","shell.execute_reply":"2025-01-08T21:37:10.178260Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# change severity column labels\n#Normal/Mild': 'normal_mild', 'Moderate': 'moderate', 'Severe': 'severe'}\nfinal_merged_df['severity'] = final_merged_df['severity'].map({'Normal/Mild': 'normal_mild', 'Moderate': 'moderate', 'Severe': 'severe'})","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:37:12.473090Z","iopub.execute_input":"2025-01-08T21:37:12.473417Z","iopub.status.idle":"2025-01-08T21:37:12.481053Z","shell.execute_reply.started":"2025-01-08T21:37:12.473389Z","shell.execute_reply":"2025-01-08T21:37:12.480163Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_data = expanded_test_desc\ntrain_data = final_merged_df","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:37:14.257785Z","iopub.execute_input":"2025-01-08T21:37:14.258089Z","iopub.status.idle":"2025-01-08T21:37:14.261784Z","shell.execute_reply.started":"2025-01-08T21:37:14.258062Z","shell.execute_reply":"2025-01-08T21:37:14.260829Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.head(5)","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:37:16.062169Z","iopub.execute_input":"2025-01-08T21:37:16.062501Z","iopub.status.idle":"2025-01-08T21:37:16.074213Z","shell.execute_reply.started":"2025-01-08T21:37:16.062472Z","shell.execute_reply":"2025-01-08T21:37:16.073560Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data['series_description'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:37:17.921606Z","iopub.execute_input":"2025-01-08T21:37:17.921907Z","iopub.status.idle":"2025-01-08T21:37:17.931669Z","shell.execute_reply.started":"2025-01-08T21:37:17.921882Z","shell.execute_reply":"2025-01-08T21:37:17.930810Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_dicom(path):\n    dicom = pydicom.dcmread(path)\n    data = dicom.pixel_array\n    data = data - np.min(data)\n    if np.max(data) != 0:\n        data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:37:19.635300Z","iopub.execute_input":"2025-01-08T21:37:19.635622Z","iopub.status.idle":"2025-01-08T21:37:19.639763Z","shell.execute_reply.started":"2025-01-08T21:37:19.635597Z","shell.execute_reply":"2025-01-08T21:37:19.638902Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import random\nimport matplotlib.pyplot as plt\n\n# Yeni sıfırlanmış indekslerle rastgele seçim yapalım\nfinal_merged_df_reset = final_merged_df.reset_index(drop=True)\n\n# Rastgele iki indeks seçelim\nselected_indices = random.sample(range(len(final_merged_df_reset)), 2)\n\nimages = []\nrow_ids = []\n\n# Seçilen indekslerle görselleri yükleyelim\nfor i in selected_indices:\n    image = load_dicom(final_merged_df_reset['image_path'][i])  # Yeni sıfırlanmış indeksi kullan\n    images.append(image)\n    row_ids.append(final_merged_df_reset['row_id'][i])  # Yeni sıfırlanmış indeksi kullan\n\n# Görselleri çizdirelim\nfig, ax = plt.subplots(1, 2, figsize=(8, 4))\nfor i in range(2):\n    ax[i].imshow(images[i], cmap='gray')\n    ax[i].set_title(f'Row ID: {row_ids[i]}', fontsize=8)\n    ax[i].axis('off')\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:37:21.761977Z","iopub.execute_input":"2025-01-08T21:37:21.762257Z","iopub.status.idle":"2025-01-08T21:37:22.185168Z","shell.execute_reply.started":"2025-01-08T21:37:21.762234Z","shell.execute_reply":"2025-01-08T21:37:22.184294Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data ","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:37:24.841853Z","iopub.execute_input":"2025-01-08T21:37:24.842140Z","iopub.status.idle":"2025-01-08T21:37:24.856019Z","shell.execute_reply.started":"2025-01-08T21:37:24.842116Z","shell.execute_reply":"2025-01-08T21:37:24.855261Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data = train_data.dropna()","metadata":{"execution":{"iopub.status.busy":"2025-01-08T23:07:12.284834Z","iopub.execute_input":"2025-01-08T23:07:12.285134Z","iopub.status.idle":"2025-01-08T23:07:12.314436Z","shell.execute_reply.started":"2025-01-08T23:07:12.285109Z","shell.execute_reply":"2025-01-08T23:07:12.313728Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as transforms\nimport torch\nimport torch.optim.lr_scheduler as lr_scheduler\nfrom tqdm import tqdm\nimport numpy as np\nfrom PIL import Image\nimport pydicom  # Assuming you're working with DICOM files\n\n# Define a function to load DICOM images (if not already done)\ndef load_dicom(image_path):\n    # Load the DICOM file (you can use pydicom or another library for this)\n    dicom = pydicom.dcmread(image_path)\n    \n    # Convert the pixel data to a numpy array, then to a PIL image\n    image = dicom.pixel_array.astype(np.uint8)  # Assuming the image is in pixel_array\n    pil_image = Image.fromarray(image)  # Convert numpy array to PIL image\n    return pil_image\n\n# Update the CustomDataset to ensure image is in PIL format and prepared for CapsNet\nclass CustomDataset(Dataset):\n    def __init__(self, dataframe, transform=None):\n        self.dataframe = dataframe\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.dataframe)\n\n    def __getitem__(self, index):\n        image_path = self.dataframe['image_path'][index]\n        image = load_dicom(image_path)  # Load image using load_dicom\n        \n        label = self.dataframe['severity'][index]  # Numeric label from the dataframe\n        \n        if self.transform:\n            image = self.transform(image)  # Apply transformations\n\n        return image, label\n\n# Function to create datasets and dataloaders for each series description\ndef create_datasets_and_loaders(df, series_description, transform, batch_size=8):\n    filtered_df = df[df['series_description'] == series_description]\n    \n    # %5'ini al frac değerini değiştirerek trainde verinin ne kadarını kullanacağınızı belirleyebilirsiniz\n    filtered_df = filtered_df.sample(frac=1.0, random_state=42)  \n    \n    train_df, val_df = train_test_split(filtered_df, test_size=0.2, random_state=42)\n    train_df = train_df.reset_index(drop=True)\n    val_df = val_df.reset_index(drop=True)\n\n    train_dataset = CustomDataset(train_df, transform)\n    val_dataset = CustomDataset(val_df, transform)\n\n    trainloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n    valloader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)\n    \n    return trainloader, valloader, len(train_df), len(val_df)\n\n\n# Define the transforms for data augmentation\ntransform = transforms.Compose([\n    transforms.Resize((224, 224)),  # CapsNet için uygun boyutta yeniden boyutlandır\n    transforms.Grayscale(num_output_channels=3),  # CapsNet RGB ile çalışır, tek kanal gri tonlama görüntüsünü 3 kanala dönüştür\n    transforms.ToTensor(),  # PIL -> Tensor dönüşümü, [0, 1] aralığı\n    transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])  # Normalizasyon (opsiyonel)\n])\n\n# Create dataloaders for each series description\ntrainloader_t1, valloader_t1, len_train_t1, len_val_t1 = create_datasets_and_loaders(train_data, 'Sagittal T1', transform)\ntrainloader_t2, valloader_t2, len_train_t2, len_val_t2 = create_datasets_and_loaders(train_data, 'Axial T2', transform)\ntrainloader_t2stir, valloader_t2stir, len_train_t2stir, len_val_t2stir = create_datasets_and_loaders(train_data, 'Sagittal T2/STIR', transform)\n\n# Store dataloaders and lengths in dictionaries\ndataloaders = {\n    'Sagittal T1': (trainloader_t1, valloader_t1),\n    'Axial T2': (trainloader_t2, valloader_t2),\n    'Sagittal T2/STIR': (trainloader_t2stir, valloader_t2stir)\n}\n\nlengths = {\n    'Sagittal T1': (len_train_t1, len_val_t1),\n    'Axial T2': (len_train_t2, len_val_t2),\n    'Sagittal T2/STIR': (len_train_t2stir, len_val_t2stir)\n}\n\n# Label mapping for severity\nlabel_map = {'normal_mild': 0, 'moderate': 1, 'severe': 2}\ntrain_data['severity'] = train_data['severity'].map(label_map)\n","metadata":{"execution":{"iopub.status.busy":"2025-01-09T00:17:47.612097Z","iopub.execute_input":"2025-01-09T00:17:47.612458Z","iopub.status.idle":"2025-01-09T00:17:47.681778Z","shell.execute_reply.started":"2025-01-09T00:17:47.612428Z","shell.execute_reply":"2025-01-09T00:17:47.681052Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def visualize_batch(dataloader):\n    images, labels = next(iter(dataloader))\n    \n    # Plot a grid of images (rows = batch size, columns = number of images)\n    fig, axes = plt.subplots(1, len(images), figsize=(20, 5))\n    \n    for i, (img, lbl) in enumerate(zip(images, labels)):\n        ax = axes[i]\n        \n        img = img.permute(1, 2, 0).cpu().numpy()  # Convert from (C, H, W) to (H, W, C)\n        \n        # Normalize back to [0, 1] for better visualization (assuming Normalize was used)\n        img = (img - img.min()) / (img.max() - img.min())\n        \n        ax.imshow(img)  # Show image in RGB\n        ax.set_title(f\"Label: {lbl.item()}\")  # Convert tensor label to a regular Python number\n        ax.axis('off')  # Hide axes\n\n    plt.show()\n\n# Visualize samples from each dataloader\nprint(\"Visualizing Sagittal T1 samples\")\nvisualize_batch(trainloader_t1)\nprint(\"Visualizing Axial T2 samples\")\nvisualize_batch(trainloader_t2)\nprint(\"Visualizing Sagittal T2/STIR samples\")\nvisualize_batch(trainloader_t2stir)\n","metadata":{"execution":{"iopub.status.busy":"2025-01-09T00:18:17.356404Z","iopub.execute_input":"2025-01-09T00:18:17.356702Z","iopub.status.idle":"2025-01-09T00:18:19.526564Z","shell.execute_reply.started":"2025-01-09T00:18:17.356679Z","shell.execute_reply":"2025-01-09T00:18:19.525619Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Bir batch'ten bir örnek alın\nimages, labels = next(iter(trainloader_t2))\nimage = images[0]  # İlk görüntüyü seç\nlabel = labels[0]  # İlk görüntünün etiketini seç\n\n# Görüntü boyutlarını HWC formatına dönüştür\nimage = image.permute(1, 2, 0)  # CHW'den HWC'ye dönüştür\n\n# Görüntüyü çiz\nplt.figure(figsize=(8, 4))\nplt.imshow(image, cmap='gray')  # Gri tonlamada gösterim\nplt.title(f\"Label: {label.item()}\")  # Etiketi başlık olarak ekle\nplt.axis('off')\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2025-01-09T00:18:23.565546Z","iopub.execute_input":"2025-01-09T00:18:23.565837Z","iopub.status.idle":"2025-01-09T00:18:23.912150Z","shell.execute_reply.started":"2025-01-09T00:18:23.565814Z","shell.execute_reply":"2025-01-09T00:18:23.911368Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport matplotlib.pyplot as plt\n\nclass CapsuleLayer(nn.Module):\n    def __init__(self, num_capsules, num_routes, in_channels, out_channels):\n        super(CapsuleLayer, self).__init__()\n        self.num_capsules = num_capsules\n        self.num_routes = num_routes\n        self.out_channels = out_channels\n\n        # Define route weights\n        self.route_weights = nn.Parameter(\n            torch.randn(num_capsules, num_routes, in_channels, out_channels)\n        )\n\n    def forward(self, x):\n        # x: [batch_size, num_routes, in_channels]\n        x = x.unsqueeze(1).unsqueeze(4)  # [batch_size, 1, num_routes, in_channels, 1]\n\n        # Fix: Match dimensions for matmul\n        # Use reshape instead of view to ensure correct memory layout\n        x = x.reshape(x.size(0), -1, x.size(3))  # [batch_size, num_routes, in_channels]\n        \n        priors = torch.matmul(self.route_weights, x)  # [num_capsules, num_routes, out_channels]\n        \n        logits = torch.zeros(*priors.size()).to(x.device)  # [num_capsules, num_routes, out_channels]\n        return self.routing(logits, priors)\n\n    def routing(self, logits, priors, iterations=3):\n        for i in range(iterations):\n            probs = F.softmax(logits, dim=2)\n            outputs = self.squash((probs * priors).sum(dim=2, keepdim=True))\n            if i < iterations - 1:\n                logits = logits + torch.matmul(priors.transpose(2, 3), outputs)\n        return outputs.squeeze(3)\n\n    def squash(self, inputs):\n        norm = (inputs ** 2).sum(dim=-1, keepdim=True)\n        scale = norm / (1 + norm)\n        return scale * inputs / torch.sqrt(norm + 1e-8)\n\n\n# CapsNet Ana Model\nclass CapsNet(nn.Module):\n    def __init__(self, input_dim, num_classes):\n        super(CapsNet, self).__init__()\n        self.conv1 = nn.Conv2d(input_dim, 256, kernel_size=9, stride=1)\n        self.primary_capsules = nn.Conv2d(256, 8*32, kernel_size=9, stride=2)\n        self.digit_capsules = CapsuleLayer(num_capsules=num_classes, num_routes=32*6*6, in_channels=8, out_channels=16)\n\n    def forward(self, x):\n        x = F.relu(self.conv1(x), inplace=True)  # İlk Conv katmanı\n        x = self.primary_capsules(x)  # [batch_size, 8*32, H, W]\n        x = x.view(x.size(0), 32, 8, -1).permute(0, 1, 3, 2)  # [batch_size, 32, H*W, 8]\n        x = self.digit_capsules(x)  # [batch_size, num_classes, out_channels]\n        return x.norm(dim=-1)  # Vektör normları\n\n# Visualization function for Capsule activations\ndef visualize_capsules(capsule_activations, num_capsules, title=\"Capsule Activations\"):\n    \"\"\"\n    Visualize the activations of capsule networks. \n    Assuming that the capsule activations are 2D or 1D arrays\n    \"\"\"\n\n    fig, axes = plt.subplots(1, num_capsules, figsize=(20, 5))\n    for i in range(num_capsules):\n        ax = axes[i]\n        ax.imshow(capsule_activations[i].cpu().detach().numpy(), cmap='viridis')  # Visualize activations\n        ax.set_title(f\"Capsule {i+1}\")\n        ax.axis('off')  # Hide axes\n    plt.suptitle(title, fontsize=16)\n    plt.show()\n\n# Initialize CapsNet Model\ninput_dim = 1  # Grayscale images\nnum_classes = 10  # Number of output capsules (digits, e.g., 0-9 for MNIST)\nmodel = CapsNet(input_dim, num_classes)\n\n# Example forward pass\n# Let's assume we're using a batch of images of shape (batch_size, 1, 28, 28)\ndummy_input = torch.randn(8, 1, 28, 28)  # Example batch size of 8\noutput = model(dummy_input)  # Forward pass through CapsNet\n\n# Visualize the capsule activations for the first sample in the batch\n# Output is a tensor of shape [batch_size, num_classes, out_channels]\nsample_activations = output[0]  # Take the first sample's output\n\n# Visualize activations for all capsules\nvisualize_capsules(sample_activations, num_classes, title=\"Capsule Activations for Sample 1\")\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-09T00:20:46.831127Z","iopub.execute_input":"2025-01-09T00:20:46.831505Z","iopub.status.idle":"2025-01-09T00:20:46.944936Z","shell.execute_reply.started":"2025-01-09T00:20:46.831469Z","shell.execute_reply":"2025-01-09T00:20:46.943782Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class MarginLoss(nn.Module):\n    def forward(self, labels, predictions):\n        left = F.relu(0.9 - predictions, inplace=True) ** 2\n        right = F.relu(predictions - 0.1, inplace=True) ** 2\n        loss = labels * left + 0.5 * (1.0 - labels) * right\n        return loss.sum(dim=1).mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-09T00:20:53.327310Z","iopub.execute_input":"2025-01-09T00:20:53.327687Z","iopub.status.idle":"2025-01-09T00:20:53.332220Z","shell.execute_reply.started":"2025-01-09T00:20:53.327657Z","shell.execute_reply":"2025-01-09T00:20:53.331276Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cihaz ayarı\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Model, kayıp ve optimizasyon\nmodel = CapsNet(input_dim=1, num_classes=3).to(device)\ncriterion = MarginLoss()\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\n# Eğitim döngüsü\nepochs = 10\nfor epoch in range(epochs):\n    model.train()\n    running_loss = 0.0\n\n    for batch in trainloader:\n        # Eğer batch bir liste (list) türünde ise\n        if isinstance(batch, list):\n            images, labels = batch\n        # Eğer batch bir tuple ise (images, labels) ayıralım\n        elif isinstance(batch, tuple):\n            images, labels = batch\n        # Eğer batch bir dictionary ise\n        elif isinstance(batch, dict):\n            images = batch['image']  # 'image' anahtarını kullanarak\n            labels = batch['label']  # 'label' anahtarını kullanarak\n        else:\n            raise ValueError(f\"Beklenmeyen veri formatı: {type(batch)}\")\n\n        # Verileri cihaza taşıyoruz\n        if isinstance(images, torch.Tensor):\n            images = images.to(device)\n        if isinstance(labels, torch.Tensor):\n            labels = labels.to(device)\n\n        # Model tahmini\n        outputs = model(images)\n\n        # Etiketleri one-hot encoding yapıyoruz\n        one_hot_labels = F.one_hot(labels, num_classes=3).float()\n\n        # Kayıp hesaplama\n        loss = criterion(one_hot_labels, outputs)\n\n        # Geri yayılım ve optimizasyon\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        running_loss += loss.item()\n\n    print(f\"Epoch {epoch+1}, Loss: {running_loss / len(trainloader)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-09T00:20:55.258798Z","iopub.execute_input":"2025-01-09T00:20:55.259082Z","iopub.status.idle":"2025-01-09T00:20:55.354011Z","shell.execute_reply.started":"2025-01-09T00:20:55.259060Z","shell.execute_reply":"2025-01-09T00:20:55.352768Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.eval()\ncorrect = 0\ntotal = 0\n\nwith torch.no_grad():\n    for images, labels in valloader:\n        images, labels = images.to(device), labels.to(device)\n        outputs = model(images)\n        _, predicted = outputs.max(1)\n        correct += (predicted == labels).sum().item()\n        total += labels.size(0)\n\nprint(f\"Validation Accuracy: {100 * correct / total:.2f}%\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T23:53:20.208487Z","iopub.execute_input":"2025-01-08T23:53:20.208786Z","iopub.status.idle":"2025-01-08T23:53:20.377724Z","shell.execute_reply.started":"2025-01-08T23:53:20.208763Z","shell.execute_reply":"2025-01-08T23:53:20.376624Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data['level'].unique()","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:50:54.109075Z","iopub.execute_input":"2025-01-08T21:50:54.109434Z","iopub.status.idle":"2025-01-08T21:50:54.117326Z","shell.execute_reply.started":"2025-01-08T21:50:54.109399Z","shell.execute_reply":"2025-01-08T21:50:54.116713Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"expanded_test_desc.head(5)","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:50:58.275224Z","iopub.execute_input":"2025-01-08T21:50:58.275576Z","iopub.status.idle":"2025-01-08T21:50:58.284919Z","shell.execute_reply.started":"2025-01-08T21:50:58.275546Z","shell.execute_reply":"2025-01-08T21:50:58.284067Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"levels = ['l1_l2', 'l2_l3', 'l3_l4', 'l4_l5', 'l5_s1']\n\n# Function to update row_id with levels\ndef update_row_id(row, levels):\n    level = levels[row.name % len(levels)]\n    return f\"{row['study_id']}_{row['condition']}_{level}\"\n\n# Update row_id in expanded_test_desc to include levels\nexpanded_test_desc['row_id'] = expanded_test_desc.apply(lambda row: update_row_id(row, levels), axis=1)","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:51:02.282757Z","iopub.execute_input":"2025-01-08T21:51:02.283074Z","iopub.status.idle":"2025-01-08T21:51:02.290097Z","shell.execute_reply.started":"2025-01-08T21:51:02.283044Z","shell.execute_reply":"2025-01-08T21:51:02.289253Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"expanded_test_desc","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:51:04.354717Z","iopub.execute_input":"2025-01-08T21:51:04.355030Z","iopub.status.idle":"2025-01-08T21:51:04.366657Z","shell.execute_reply.started":"2025-01-08T21:51:04.355004Z","shell.execute_reply":"2025-01-08T21:51:04.365931Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define a custom test dataset class\nclass TestDataset(Dataset):\n    def __init__(self, dataframe, transform=None):\n        self.dataframe = dataframe\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.dataframe)\n\n    def __getitem__(self, index):\n        image_path = self.dataframe['image_path'][index]\n        image = load_dicom(image_path)  # Define this function to load your DICOM images\n        if self.transform:\n            image = self.transform(image)\n        return image\n\n# Define the transforms\ntransform = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.Resize((224, 224)),\n    transforms.Grayscale(num_output_channels=3),\n    transforms.ToTensor(),\n])\n\n# Create a test dataset and dataloader\ntest_dataset = TestDataset(expanded_test_desc, transform)\ntestloader = DataLoader(test_dataset, batch_size=1, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:51:08.431154Z","iopub.execute_input":"2025-01-08T21:51:08.431495Z","iopub.status.idle":"2025-01-08T21:51:08.437509Z","shell.execute_reply.started":"2025-01-08T21:51:08.431468Z","shell.execute_reply":"2025-01-08T21:51:08.436590Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for image in testloader:\n    print(image.shape)\n    break","metadata":{"execution":{"iopub.status.busy":"2025-01-08T21:51:10.917130Z","iopub.execute_input":"2025-01-08T21:51:10.917447Z","iopub.status.idle":"2025-01-08T21:51:10.974444Z","shell.execute_reply.started":"2025-01-08T21:51:10.917418Z","shell.execute_reply":"2025-01-08T21:51:10.973251Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define models in a dictionary (use a different name to avoid conflicts)\nmodel_dict = {\n    'Sagittal T1': sagittal_t1_model,\n    'Axial T2': axial_t2_model,\n    'Sagittal T2/STIR': sagittal_t2stir_model,\n}\n\n# Function to get the model based on series_description\ndef get_model(series_description):\n    return model_dict.get(series_description, None)\n\n# Function to make predictions on the test data\ndef predict_test_data(testloader, expanded_test_desc):\n    predictions = []\n    normal_mild_probs = []\n    moderate_probs = []\n    severe_probs = []\n    \n    # Set each model to evaluation mode\n    for model in model_dict.values():\n        model.eval()\n\n    with torch.no_grad():  # Disable gradient calculation during inference\n        for idx, images in enumerate(tqdm(testloader)):  # Iterate through the test data\n            images = images.to(device)  # Move images to the device\n            series_description = expanded_test_desc.iloc[idx]['series_description']  # Get description from DataFrame\n            \n            # Get the model corresponding to the series description\n            model = get_model(series_description)\n            \n            if model:  # If a valid model is found\n                outputs = model(images)  # Forward pass through the model\n                probs = torch.softmax(outputs, dim=1).squeeze(0)  # Get the probabilities for each class\n                normal_mild_probs.append(probs[0].item())  # Probability for normal/mild class\n                moderate_probs.append(probs[1].item())  # Probability for moderate class\n                severe_probs.append(probs[2].item())  # Probability for severe class\n                predictions.append(probs)  # Append the full prediction\n            else:  # If no model is found for the description\n                normal_mild_probs.append(None)\n                moderate_probs.append(None)\n                severe_probs.append(None)\n                predictions.append(None)\n\n    return normal_mild_probs, moderate_probs, severe_probs, predictions\n\n# Make predictions on the test data\nnormal_mild_probs, moderate_probs, severe_probs, test_predictions = predict_test_data(testloader, expanded_test_desc)\n","metadata":{"execution":{"iopub.execute_input":"2024-12-18T22:51:46.416705Z","iopub.status.busy":"2024-12-18T22:51:46.416376Z","iopub.status.idle":"2024-12-18T22:51:51.562603Z","shell.execute_reply":"2024-12-18T22:51:51.561720Z","shell.execute_reply.started":"2024-12-18T22:51:46.416678Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_predictions[0]","metadata":{"execution":{"iopub.execute_input":"2024-12-18T22:51:55.942634Z","iopub.status.busy":"2024-12-18T22:51:55.942252Z","iopub.status.idle":"2024-12-18T22:51:55.949850Z","shell.execute_reply":"2024-12-18T22:51:55.948845Z","shell.execute_reply.started":"2024-12-18T22:51:55.942600Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Add predictions and probabilities to the test DataFrame\nexpanded_test_desc['normal_mild'] = normal_mild_probs\nexpanded_test_desc['moderate'] = moderate_probs\nexpanded_test_desc['severe'] = severe_probs","metadata":{"execution":{"iopub.execute_input":"2024-12-18T22:51:57.815409Z","iopub.status.busy":"2024-12-18T22:51:57.815058Z","iopub.status.idle":"2024-12-18T22:51:57.820541Z","shell.execute_reply":"2024-12-18T22:51:57.819452Z","shell.execute_reply.started":"2024-12-18T22:51:57.815382Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = expanded_test_desc[[\"row_id\",\"normal_mild\",\"moderate\",\"severe\"]]","metadata":{"execution":{"iopub.execute_input":"2024-12-18T22:51:59.650877Z","iopub.status.busy":"2024-12-18T22:51:59.650518Z","iopub.status.idle":"2024-12-18T22:51:59.656439Z","shell.execute_reply":"2024-12-18T22:51:59.655482Z","shell.execute_reply.started":"2024-12-18T22:51:59.650847Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission.head(10)","metadata":{"execution":{"iopub.execute_input":"2024-12-18T22:52:01.373342Z","iopub.status.busy":"2024-12-18T22:52:01.373022Z","iopub.status.idle":"2024-12-18T22:52:01.385398Z","shell.execute_reply":"2024-12-18T22:52:01.384424Z","shell.execute_reply.started":"2024-12-18T22:52:01.373319Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group by 'row_id' and sum the values\ngrouped_submission = submission.groupby('row_id').max().reset_index()\n\n# Normalize the columns\ngrouped_submission[['normal_mild', 'moderate', 'severe']] = grouped_submission[['normal_mild', 'moderate', 'severe']].div(grouped_submission[['normal_mild', 'moderate', 'severe']].sum(axis=1), axis=0)\n\n# Check the first 3 rows\ngrouped_submission","metadata":{"execution":{"iopub.execute_input":"2024-12-18T22:52:03.532310Z","iopub.status.busy":"2024-12-18T22:52:03.531984Z","iopub.status.idle":"2024-12-18T22:52:03.557995Z","shell.execute_reply":"2024-12-18T22:52:03.557050Z","shell.execute_reply.started":"2024-12-18T22:52:03.532286Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(grouped_submission)","metadata":{"execution":{"iopub.execute_input":"2024-12-18T22:52:06.222313Z","iopub.status.busy":"2024-12-18T22:52:06.221980Z","iopub.status.idle":"2024-12-18T22:52:06.228096Z","shell.execute_reply":"2024-12-18T22:52:06.227093Z","shell.execute_reply.started":"2024-12-18T22:52:06.222288Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub[['normal_mild', 'moderate', 'severe']] = grouped_submission[['normal_mild', 'moderate', 'severe']]","metadata":{"execution":{"iopub.execute_input":"2024-12-18T22:52:08.172276Z","iopub.status.busy":"2024-12-18T22:52:08.171954Z","iopub.status.idle":"2024-12-18T22:52:08.178482Z","shell.execute_reply":"2024-12-18T22:52:08.177356Z","shell.execute_reply.started":"2024-12-18T22:52:08.172254Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\n# Save the DataFrame to \"submission.csv\" in the desired directory\nsub.to_csv(\"/kaggle/working/submission.csv\", index=False)","metadata":{"execution":{"iopub.execute_input":"2024-12-18T22:52:09.752436Z","iopub.status.busy":"2024-12-18T22:52:09.752117Z","iopub.status.idle":"2024-12-18T22:52:09.760032Z","shell.execute_reply":"2024-12-18T22:52:09.758988Z","shell.execute_reply.started":"2024-12-18T22:52:09.752410Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub.head(5)","metadata":{"execution":{"iopub.execute_input":"2024-12-18T22:52:11.368688Z","iopub.status.busy":"2024-12-18T22:52:11.368348Z","iopub.status.idle":"2024-12-18T22:52:11.378923Z","shell.execute_reply":"2024-12-18T22:52:11.377937Z","shell.execute_reply.started":"2024-12-18T22:52:11.368661Z"}},"outputs":[],"execution_count":null}]}