{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":992,"sourceType":"modelInstanceVersion","modelInstanceId":846,"modelId":101}],"dockerImageVersionId":30823,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":4482.163829,"end_time":"2025-01-09T01:29:38.304294","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-01-09T00:14:56.140465","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"https://www.kaggle.com/code/borabingol/fork-of-fixed-train-with-severe-focused-augmentati/notebook with confusion matrix. \nUsing old .pt method. ","metadata":{"papermill":{"duration":0.009361,"end_time":"2025-01-09T00:14:58.709272","exception":false,"start_time":"2025-01-09T00:14:58.699911","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import seaborn as sns\nimport cv2 \n\nimport matplotlib.pyplot as plt\nimport os\nimport time\nimport numpy as np\nimport glob\nimport json\nimport collections\nimport torch\nimport torch.nn as nn\n\nimport pydicom as dicom\nimport matplotlib.patches as patches\n\nfrom matplotlib import animation, rc\nimport pandas as pd\n\nimport pydicom as dicom # dicom\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:14:53.303942Z","iopub.execute_input":"2025-01-15T16:14:53.304255Z","iopub.status.idle":"2025-01-15T16:14:58.723750Z","shell.execute_reply.started":"2025-01-15T16:14:53.304214Z","shell.execute_reply":"2025-01-15T16:14:58.723036Z"},"papermill":{"duration":6.508597,"end_time":"2025-01-09T00:15:05.226496","exception":false,"start_time":"2025-01-09T00:14:58.717899","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# read data\ntrain_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/'\n\ntrain  = pd.read_csv(train_path + 'train.csv')\nlabel = pd.read_csv(train_path + 'train_label_coordinates.csv')\ntrain_desc  = pd.read_csv(train_path + 'train_series_descriptions.csv')\ntest_desc   = pd.read_csv(train_path + 'test_series_descriptions.csv')\nsub         = pd.read_csv(train_path + 'sample_submission.csv')\nlen(test_desc) #number of test_description.csv rows ","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:14:58.942870Z","iopub.execute_input":"2025-01-15T16:14:58.943135Z","iopub.status.idle":"2025-01-15T16:14:59.036298Z","shell.execute_reply.started":"2025-01-15T16:14:58.943111Z","shell.execute_reply":"2025-01-15T16:14:59.035319Z"},"papermill":{"duration":0.187605,"end_time":"2025-01-09T00:15:05.422679","exception":false,"start_time":"2025-01-09T00:15:05.235074","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_desc.head(5)","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:15:01.853262Z","iopub.execute_input":"2025-01-15T16:15:01.853633Z","iopub.status.idle":"2025-01-15T16:15:01.867686Z","shell.execute_reply.started":"2025-01-15T16:15:01.853593Z","shell.execute_reply":"2025-01-15T16:15:01.866861Z"},"papermill":{"duration":0.026881,"end_time":"2025-01-09T00:15:05.458818","exception":false,"start_time":"2025-01-09T00:15:05.431937","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_desc.head(5)","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:15:04.312263Z","iopub.execute_input":"2025-01-15T16:15:04.312637Z","iopub.status.idle":"2025-01-15T16:15:04.320950Z","shell.execute_reply.started":"2025-01-15T16:15:04.312605Z","shell.execute_reply":"2025-01-15T16:15:04.320000Z"},"papermill":{"duration":0.020564,"end_time":"2025-01-09T00:15:05.488479","exception":false,"start_time":"2025-01-09T00:15:05.467915","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head(5)","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:15:05.965533Z","iopub.execute_input":"2025-01-15T16:15:05.965866Z","iopub.status.idle":"2025-01-15T16:15:05.986853Z","shell.execute_reply.started":"2025-01-15T16:15:05.965820Z","shell.execute_reply":"2025-01-15T16:15:05.985991Z"},"papermill":{"duration":0.036396,"end_time":"2025-01-09T00:15:05.533522","exception":false,"start_time":"2025-01-09T00:15:05.497126","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function to generate image paths based on directory structure\ndef generate_image_paths(df, data_dir):\n    image_paths = []\n    for study_id, series_id in zip(df['study_id'], df['series_id']):\n        study_dir = os.path.join(data_dir, str(study_id))\n        series_dir = os.path.join(study_dir, str(series_id))\n        images = os.listdir(series_dir)\n        image_paths.extend([os.path.join(series_dir, img) for img in images])\n    return image_paths\n\n# Generate image paths for train and test data\ntrain_image_paths = generate_image_paths(train_desc, f'{train_path}/train_images')\ntest_image_paths = generate_image_paths(test_desc, f'{train_path}/test_images')","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:15:08.642062Z","iopub.execute_input":"2025-01-15T16:15:08.642401Z","iopub.status.idle":"2025-01-15T16:16:55.880323Z","shell.execute_reply.started":"2025-01-15T16:15:08.642364Z","shell.execute_reply":"2025-01-15T16:16:55.879592Z"},"papermill":{"duration":29.880413,"end_time":"2025-01-09T00:15:35.423962","exception":false,"start_time":"2025-01-09T00:15:05.543549","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(train_desc)","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:16:55.881308Z","iopub.execute_input":"2025-01-15T16:16:55.881559Z","iopub.status.idle":"2025-01-15T16:16:55.886563Z","shell.execute_reply.started":"2025-01-15T16:16:55.881538Z","shell.execute_reply":"2025-01-15T16:16:55.885842Z"},"papermill":{"duration":0.017087,"end_time":"2025-01-09T00:15:35.450513","exception":false,"start_time":"2025-01-09T00:15:35.433426","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(train_image_paths)","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:16:55.887943Z","iopub.execute_input":"2025-01-15T16:16:55.888243Z","iopub.status.idle":"2025-01-15T16:16:55.902309Z","shell.execute_reply.started":"2025-01-15T16:16:55.888216Z","shell.execute_reply":"2025-01-15T16:16:55.901697Z"},"papermill":{"duration":0.016481,"end_time":"2025-01-09T00:15:35.476183","exception":false,"start_time":"2025-01-09T00:15:35.459702","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define function to reshape a single row of the DataFrame\ndef reshape_row(row):\n    data = {'study_id': [], 'condition': [], 'level': [], 'severity': []}\n    \n    for column, value in row.items():\n        if column not in ['study_id', 'series_id', 'instance_number', 'x', 'y', 'series_description']:\n            parts = column.split('_')\n            condition = ' '.join([word.capitalize() for word in parts[:-2]])\n            level = parts[-2].capitalize() + '/' + parts[-1].capitalize()\n            data['study_id'].append(row['study_id'])\n            data['condition'].append(condition)\n            data['level'].append(level)\n            data['severity'].append(value)\n    \n    return pd.DataFrame(data)\n\n# Reshape the DataFrame for all rows\nnew_train_df = pd.concat([reshape_row(row) for _, row in train.iterrows()], ignore_index=True)\n\n# Display the first few rows of the reshaped dataframe\nnew_train_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:17:21.770087Z","iopub.execute_input":"2025-01-15T16:17:21.770409Z","iopub.status.idle":"2025-01-15T16:17:22.829718Z","shell.execute_reply.started":"2025-01-15T16:17:21.770383Z","shell.execute_reply":"2025-01-15T16:17:22.828836Z"},"papermill":{"duration":1.237419,"end_time":"2025-01-09T00:15:36.722904","exception":false,"start_time":"2025-01-09T00:15:35.485485","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Print columns in a neat way\nprint(\"\\nColumns in new_train_df:\")\nprint(\",\".join(new_train_df.columns))\n\nprint(\"\\nColumns in label:\")\nprint(\",\".join(label.columns))\n\nprint(\"\\nColumns in test_desc:\")\nprint(\",\".join(test_desc.columns))\n\nprint(\"\\nColumns in sub:\")\nprint(\",\".join(sub.columns))","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:18:06.732762Z","iopub.execute_input":"2025-01-15T16:18:06.733056Z","iopub.status.idle":"2025-01-15T16:18:06.740172Z","shell.execute_reply.started":"2025-01-15T16:18:06.733031Z","shell.execute_reply":"2025-01-15T16:18:06.739206Z"},"papermill":{"duration":0.019543,"end_time":"2025-01-09T00:15:36.752177","exception":false,"start_time":"2025-01-09T00:15:36.732634","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Merge the dataframes on the common columns\nmerged_df = pd.merge(new_train_df, label, on=['study_id', 'condition', 'level'], how='inner')\n# Merge the dataframes on the common column 'series_id'\nfinal_merged_df = pd.merge(merged_df, train_desc, on='series_id', how='inner')","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:18:08.631559Z","iopub.execute_input":"2025-01-15T16:18:08.631884Z","iopub.status.idle":"2025-01-15T16:18:08.700337Z","shell.execute_reply.started":"2025-01-15T16:18:08.631854Z","shell.execute_reply":"2025-01-15T16:18:08.699396Z"},"papermill":{"duration":0.08866,"end_time":"2025-01-09T00:15:36.850138","exception":false,"start_time":"2025-01-09T00:15:36.761478","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Merge the dataframes on the common column 'series_id'\nfinal_merged_df = pd.merge(merged_df, train_desc, on=['series_id','study_id'], how='inner')\n# Display the first few rows of the final merged dataframe\nfinal_merged_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:18:11.045107Z","iopub.execute_input":"2025-01-15T16:18:11.045402Z","iopub.status.idle":"2025-01-15T16:18:11.075856Z","shell.execute_reply.started":"2025-01-15T16:18:11.045379Z","shell.execute_reply":"2025-01-15T16:18:11.075077Z"},"papermill":{"duration":0.044704,"end_time":"2025-01-09T00:15:36.905518","exception":false,"start_time":"2025-01-09T00:15:36.860814","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n# Create the row_id column\nfinal_merged_df['row_id'] = (\n    final_merged_df['study_id'].astype(str) + '_' +\n    final_merged_df['condition'].str.lower().str.replace(' ', '_') + '_' +\n    final_merged_df['level'].str.lower().str.replace('/', '_')\n)\n\n# Create the image_path column\nfinal_merged_df['image_path'] = (\n    f'{train_path}/train_images/' + \n    final_merged_df['study_id'].astype(str) + '/' +\n    final_merged_df['series_id'].astype(str) + '/' +\n    final_merged_df['instance_number'].astype(str) + '.dcm'\n)\n\n# Note: Check image path, since there's 1 instance id, for 1 image, but there's many more images other than the ones labelled in the instance ID. \n\n# Display the updated dataframe\nfinal_merged_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:18:14.398315Z","iopub.execute_input":"2025-01-15T16:18:14.398644Z","iopub.status.idle":"2025-01-15T16:18:14.591682Z","shell.execute_reply.started":"2025-01-15T16:18:14.398618Z","shell.execute_reply":"2025-01-15T16:18:14.590786Z"},"papermill":{"duration":0.236444,"end_time":"2025-01-09T00:15:37.151474","exception":false,"start_time":"2025-01-09T00:15:36.91503","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_merged_df[final_merged_df[\"severity\"] == \"Normal/Mild\"].value_counts().sum()","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:18:17.523892Z","iopub.execute_input":"2025-01-15T16:18:17.524195Z","iopub.status.idle":"2025-01-15T16:18:17.634865Z","shell.execute_reply.started":"2025-01-15T16:18:17.524171Z","shell.execute_reply":"2025-01-15T16:18:17.634004Z"},"papermill":{"duration":0.140083,"end_time":"2025-01-09T00:15:37.301542","exception":false,"start_time":"2025-01-09T00:15:37.161459","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_merged_df[final_merged_df[\"severity\"] == \"Moderate\"].value_counts().sum()","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:18:19.559492Z","iopub.execute_input":"2025-01-15T16:18:19.559829Z","iopub.status.idle":"2025-01-15T16:18:19.599420Z","shell.execute_reply.started":"2025-01-15T16:18:19.559799Z","shell.execute_reply":"2025-01-15T16:18:19.598612Z"},"papermill":{"duration":0.053051,"end_time":"2025-01-09T00:15:37.364328","exception":false,"start_time":"2025-01-09T00:15:37.311277","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_merged_df[final_merged_df[\"severity\"] == \"Severe\"].value_counts().sum()","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:18:21.857667Z","iopub.execute_input":"2025-01-15T16:18:21.858000Z","iopub.status.idle":"2025-01-15T16:18:21.880746Z","shell.execute_reply.started":"2025-01-15T16:18:21.857975Z","shell.execute_reply":"2025-01-15T16:18:21.880033Z"},"papermill":{"duration":0.036633,"end_time":"2025-01-09T00:15:37.410819","exception":false,"start_time":"2025-01-09T00:15:37.374186","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n#import pandas as pd\n\n# En düşük sınıf sayısını belirleyelim\n#min_class_count = 3081\n\n# Normal/Mild ve Moderate sınıflarını azaltalım\n#moderate_df = final_merged_df[final_merged_df[\"severity\"] == \"Moderate\"].sample(n=min_class_count, random_state=42)\n#severe_df = final_merged_df[final_merged_df[\"severity\"] == \"Severe\"]\n\n# İndeksleri sıfırlayalım\n#normal_mild_df = normal_mild_df.reset_index(drop=True)\n#moderate_df = moderate_df.reset_index(drop=True)\n#severe_df = severe_df.reset_index(drop=True)\n\n# Verileri birleştirelim ve final_merged_df'yi güncelleyelim\n#final_merged_df = pd.concat([normal_mild_df, moderate_df, severe_df])\n\n# Sonuçları kontrol edelim\n#print(final_merged_df[\"severity\"].value_counts())\n","metadata":{"execution":{"iopub.status.busy":"2025-01-12T23:42:03.445551Z","iopub.execute_input":"2025-01-12T23:42:03.445930Z","iopub.status.idle":"2025-01-12T23:42:03.481925Z","shell.execute_reply.started":"2025-01-12T23:42:03.445898Z","shell.execute_reply":"2025-01-12T23:42:03.481244Z"},"papermill":{"duration":0.017709,"end_time":"2025-01-09T00:15:37.438498","exception":false,"start_time":"2025-01-09T00:15:37.420789","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_merged_df[final_merged_df[\"severity\"] == \"Normal/Mild\"].value_counts().sum()","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:18:41.635902Z","iopub.execute_input":"2025-01-15T16:18:41.636181Z","iopub.status.idle":"2025-01-15T16:18:41.744251Z","shell.execute_reply.started":"2025-01-15T16:18:41.636160Z","shell.execute_reply":"2025-01-15T16:18:41.743527Z"},"papermill":{"duration":0.130419,"end_time":"2025-01-09T00:15:37.578689","exception":false,"start_time":"2025-01-09T00:15:37.44827","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_merged_df[final_merged_df[\"severity\"] == \"Moderate\"].value_counts().sum()","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:18:43.576885Z","iopub.execute_input":"2025-01-15T16:18:43.577196Z","iopub.status.idle":"2025-01-15T16:18:43.615208Z","shell.execute_reply.started":"2025-01-15T16:18:43.577168Z","shell.execute_reply":"2025-01-15T16:18:43.614474Z"},"papermill":{"duration":0.053386,"end_time":"2025-01-09T00:15:37.642174","exception":false,"start_time":"2025-01-09T00:15:37.588788","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_merged_df[final_merged_df[\"severity\"] == \"Severe\"].value_counts().sum()","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:18:57.335553Z","iopub.execute_input":"2025-01-15T16:18:57.335878Z","iopub.status.idle":"2025-01-15T16:18:57.359839Z","shell.execute_reply.started":"2025-01-15T16:18:57.335848Z","shell.execute_reply":"2025-01-15T16:18:57.359040Z"},"papermill":{"duration":0.037292,"end_time":"2025-01-09T00:15:37.689826","exception":false,"start_time":"2025-01-09T00:15:37.652534","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define the base path for test images\nbase_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_images/'\n\n# Function to get image paths for a series\ndef get_image_paths(row):\n    series_path = os.path.join(base_path, str(row['study_id']), str(row['series_id']))\n    if os.path.exists(series_path):\n        return [os.path.join(series_path, f) for f in os.listdir(series_path) if os.path.isfile(os.path.join(series_path, f))]\n    return []\n\n# Mapping of series_description to conditions\ncondition_mapping = {\n    'Sagittal T1': {'left': 'left_neural_foraminal_narrowing', 'right': 'right_neural_foraminal_narrowing'},\n    'Axial T2': {'left': 'left_subarticular_stenosis', 'right': 'right_subarticular_stenosis'},\n    'Sagittal T2/STIR': 'spinal_canal_stenosis'\n}\n\n# Create a list to store the expanded rows\nexpanded_rows = []\n\n# Expand the dataframe by adding new rows for each file path\nfor index, row in test_desc.iterrows():\n    image_paths = get_image_paths(row)\n    conditions = condition_mapping.get(row['series_description'], {})\n    if isinstance(conditions, str):  # Single condition\n        conditions = {'left': conditions, 'right': conditions}\n    for side, condition in conditions.items():\n        for image_path in image_paths:\n            expanded_rows.append({\n                'study_id': row['study_id'],\n                'series_id': row['series_id'],\n                'series_description': row['series_description'],\n                'image_path': image_path,\n                'condition': condition,\n                'row_id': f\"{row['study_id']}_{condition}\"\n            })\n\n# Create a new dataframe from the expanded rows\nexpanded_test_desc = pd.DataFrame(expanded_rows)\n\n# Display the resulting dataframe\nexpanded_test_desc.head(5)","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:19:01.042866Z","iopub.execute_input":"2025-01-15T16:19:01.043149Z","iopub.status.idle":"2025-01-15T16:19:01.222910Z","shell.execute_reply.started":"2025-01-15T16:19:01.043126Z","shell.execute_reply":"2025-01-15T16:19:01.222132Z"},"papermill":{"duration":0.075235,"end_time":"2025-01-09T00:15:37.775714","exception":false,"start_time":"2025-01-09T00:15:37.700479","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# change severity column labels\n#Normal/Mild': 'normal_mild', 'Moderate': 'moderate', 'Severe': 'severe'}\nfinal_merged_df['severity'] = final_merged_df['severity'].map({'Normal/Mild': 'normal_mild', 'Moderate': 'moderate', 'Severe': 'severe'})","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:19:03.541125Z","iopub.execute_input":"2025-01-15T16:19:03.541435Z","iopub.status.idle":"2025-01-15T16:19:03.549548Z","shell.execute_reply.started":"2025-01-15T16:19:03.541412Z","shell.execute_reply":"2025-01-15T16:19:03.548531Z"},"papermill":{"duration":0.021752,"end_time":"2025-01-09T00:15:37.808531","exception":false,"start_time":"2025-01-09T00:15:37.786779","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_data = expanded_test_desc\ntrain_data = final_merged_df","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:19:09.989769Z","iopub.execute_input":"2025-01-15T16:19:09.990113Z","iopub.status.idle":"2025-01-15T16:19:09.993953Z","shell.execute_reply.started":"2025-01-15T16:19:09.990082Z","shell.execute_reply":"2025-01-15T16:19:09.992915Z"},"papermill":{"duration":0.017475,"end_time":"2025-01-09T00:15:37.836652","exception":false,"start_time":"2025-01-09T00:15:37.819177","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.head(5)","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:19:12.026148Z","iopub.execute_input":"2025-01-15T16:19:12.026616Z","iopub.status.idle":"2025-01-15T16:19:12.042567Z","shell.execute_reply.started":"2025-01-15T16:19:12.026579Z","shell.execute_reply":"2025-01-15T16:19:12.041393Z"},"papermill":{"duration":0.026748,"end_time":"2025-01-09T00:15:37.873988","exception":false,"start_time":"2025-01-09T00:15:37.84724","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data['series_description'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:19:14.405280Z","iopub.execute_input":"2025-01-15T16:19:14.405663Z","iopub.status.idle":"2025-01-15T16:19:14.415463Z","shell.execute_reply.started":"2025-01-15T16:19:14.405631Z","shell.execute_reply":"2025-01-15T16:19:14.414543Z"},"papermill":{"duration":0.024078,"end_time":"2025-01-09T00:15:37.909124","exception":false,"start_time":"2025-01-09T00:15:37.885046","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_dicom(path):\n    dicom = pydicom.dcmread(path)\n    data = dicom.pixel_array\n    data = data - np.min(data)\n    if np.max(data) != 0:\n        data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:19:18.054220Z","iopub.execute_input":"2025-01-15T16:19:18.054615Z","iopub.status.idle":"2025-01-15T16:19:18.058849Z","shell.execute_reply.started":"2025-01-15T16:19:18.054581Z","shell.execute_reply":"2025-01-15T16:19:18.058001Z"},"papermill":{"duration":0.018,"end_time":"2025-01-09T00:15:37.938029","exception":false,"start_time":"2025-01-09T00:15:37.920029","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import random\nimport matplotlib.pyplot as plt\n\n# Yeni sıfırlanmış indekslerle rastgele seçim yapalım\nfinal_merged_df_reset = final_merged_df.reset_index(drop=True)\n\n# Rastgele iki indeks seçelim\nselected_indices = random.sample(range(len(final_merged_df_reset)), 2)\n\nimages = []\nrow_ids = []\n\n# Seçilen indekslerle görselleri yükleyelim\nfor i in selected_indices:\n    image = load_dicom(final_merged_df_reset['image_path'][i])  # Yeni sıfırlanmış indeksi kullan\n    images.append(image)\n    row_ids.append(final_merged_df_reset['row_id'][i])  # Yeni sıfırlanmış indeksi kullan\n\n# Görselleri çizdirelim\nfig, ax = plt.subplots(1, 2, figsize=(8, 4))\nfor i in range(2):\n    ax[i].imshow(images[i], cmap='gray')\n    ax[i].set_title(f'Row ID: {row_ids[i]}', fontsize=8)\n    ax[i].axis('off')\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:19:22.870101Z","iopub.execute_input":"2025-01-15T16:19:22.870415Z","iopub.status.idle":"2025-01-15T16:19:23.219380Z","shell.execute_reply.started":"2025-01-15T16:19:22.870389Z","shell.execute_reply":"2025-01-15T16:19:23.218331Z"},"papermill":{"duration":0.42299,"end_time":"2025-01-09T00:15:38.373968","exception":false,"start_time":"2025-01-09T00:15:37.950978","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data ","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:19:26.279011Z","iopub.execute_input":"2025-01-15T16:19:26.279332Z","iopub.status.idle":"2025-01-15T16:19:26.294533Z","shell.execute_reply.started":"2025-01-15T16:19:26.279310Z","shell.execute_reply":"2025-01-15T16:19:26.293633Z"},"papermill":{"duration":0.03771,"end_time":"2025-01-09T00:15:38.427833","exception":false,"start_time":"2025-01-09T00:15:38.390123","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data = train_data.dropna()","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:19:29.161206Z","iopub.execute_input":"2025-01-15T16:19:29.161645Z","iopub.status.idle":"2025-01-15T16:19:29.187302Z","shell.execute_reply.started":"2025-01-15T16:19:29.161603Z","shell.execute_reply":"2025-01-15T16:19:29.186415Z"},"papermill":{"duration":0.04814,"end_time":"2025-01-09T00:15:38.493096","exception":false,"start_time":"2025-01-09T00:15:38.444956","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom torch.utils.data import Dataset, DataLoader, WeightedRandomSampler\nimport torchvision.transforms as transforms  # Bu satır eklendi\nimport torch\nimport torch.optim.lr_scheduler as lr_scheduler\nfrom tqdm import tqdm\nimport numpy as np\n\n# CustomDataset sınıfı burada kalacak\nclass CustomDataset(Dataset):\n    def __init__(self, dataframe, transform=None, apply_augmentation_to_all=True):\n        self.dataframe = dataframe\n        self.transform = transform\n        self.apply_augmentation_to_all = apply_augmentation_to_all  # Tüm sınıflara augmentasyon uygula\n\n    def __len__(self):\n        return len(self.dataframe) * 2  # Her iki versiyonu kullanarak veri kümesini iki katına çıkarıyoruz\n\n    def __getitem__(self, index):\n        original_index = index // 2  # Her iki versiyon için aynı orijinal index'i al\n        is_augmented = index % 2  # Orijinal mi, augmentasyonlu mu olduğunu kontrol et\n\n        image_path = self.dataframe['image_path'][original_index]\n        image = load_dicom(image_path)  # DICOM dosyasını yükle\n        label = self.dataframe['severity'][original_index]  # Etiketi al\n\n        # Orijinal görüntü\n        original_image = np.stack([image] * 3, axis=-1)  # 1 kanal olan görüntüyü 3 kanala dönüştür\n\n        # Augmentasyonu uygula\n        if is_augmented and self.apply_augmentation_to_all:\n            image = self.apply_augmentation(image)  # Sadece augmentasyonlu versiyon için\n            augmented_image = np.stack([image] * 3, axis=-1)\n        else:\n            augmented_image = original_image  # Orijinal görüntü\n\n        # Transformasyonu uygula\n        if self.transform:\n            original_image = self.transform(original_image)\n            augmented_image = self.transform(augmented_image)\n\n        return augmented_image, label  # Augmentasyonlu görüntü ve etiket döndürülür\n\n    # Normalize DICOM görüntüsü\n    def normalize_dicom(self, image, window_min=-1000, window_max=400):\n        image = np.clip(image, window_min, window_max)\n        return (image - window_min) / (window_max - window_min)\n\n    # Augmentasyon uygulama\n    def apply_augmentation(self, image):\n        augmentations = [\n            lambda img: np.rot90(img, k=np.random.randint(1, 4)),  # Random rotation\n            lambda img: np.fliplr(img),  # Horizontal flip\n            lambda img: np.flipud(img),  # Vertical flip\n            lambda img: self.random_crop(img),  # Random crop\n            lambda img: self.add_gaussian_noise(img),  # Gaussian noise\n            lambda img: self.adjust_brightness(img),  # Brightness adjustment\n            lambda img: self.adjust_contrast(img),  # Contrast adjustment\n        ]\n        \n        # Bir augmentasyon seç ve uygula\n        augmentation = np.random.choice(augmentations)\n        return augmentation(image)\n\n    # Random crop\n    def random_crop(self, image, crop_size=(224, 224)):\n        h, w = image.shape\n        new_h, new_w = crop_size\n        \n        # Yükseklik ve genişlik farkının negatif olmamasını sağla\n        top = np.random.randint(0, max(h - new_h, 1))  # min 1\n        left = np.random.randint(0, max(w - new_w, 1))  # min 1\n        \n        return image[top:top + new_h, left:left + new_w]\n\n    # Gaussian noise ekleme\n    def add_gaussian_noise(self, image, mean=0, std=0.01):\n        noise = np.random.normal(mean, std, image.shape)\n        return np.clip(image + noise, 0, 255)  # 0-255 arasında tutmak\n\n    # Parlaklık ayarı\n    def adjust_brightness(self, image, factor_range=(0.95, 1.05)):\n        factor = np.random.uniform(*factor_range)\n        return np.clip(image * factor, 0, 255)  # 0-255 arasında tutmak\n\n    # Kontrast ayarı\n    def adjust_contrast(self, image, factor_range=(0.95, 1.05)):\n        mean = np.mean(image)\n        factor = np.random.uniform(*factor_range)\n        return np.clip((image - mean) * factor + mean, 0, 255)  # 0-255 arasında tutmak\n\n\n# WeightedRandomSampler ile dengeli DataLoader oluşturma\ndef create_balanced_dataloader(dataset, batch_size):\n    # 'dataframe' üzerinden sınıf sayısını hesaplayalım\n    class_counts = dataset.dataframe['severity'].value_counts()\n    class_weights = 1.0 / class_counts  # Her sınıfın ters frekansını alıyoruz\n    sample_weights = dataset.dataframe['severity'].map(class_weights).values  # Sınıf ağırlıklarını örneklere atıyoruz\n\n    # WeightedRandomSampler oluşturma\n    sampler = WeightedRandomSampler(sample_weights, len(sample_weights))\n\n    # DataLoader oluşturma\n    return DataLoader(dataset, batch_size=batch_size, sampler=sampler)\n\n# Function to create datasets and dataloaders for each series description\ndef create_datasets_and_loaders(df, series_description, transform, batch_size=8):\n    # Belirtilen seriye göre filtreleme\n    filtered_df = df[df['series_description'] == series_description]\n    \n    # Tüm verinin %100'ünü kullanıyoruz (isteğe bağlı olarak frac değiştirilebilir)\n    filtered_df = filtered_df.sample(frac=1.0, random_state=42)  \n    \n    # Eğitim ve doğrulama setlerine bölme\n    train_df, val_df = train_test_split(filtered_df, test_size=0.2, random_state=42)\n    train_df = train_df.reset_index(drop=True)\n    val_df = val_df.reset_index(drop=True)\n\n    # Dataset oluşturma (augmentasyon ve transform tüm veriye uygulanır)\n    train_dataset = CustomDataset(train_df, transform=transform, apply_augmentation_to_all=True)  # Eğitimde augmentasyon\n    val_dataset = CustomDataset(val_df, transform=transform, apply_augmentation_to_all=False)  # Validasyonda augmentasyon yok\n\n    # DataLoader oluşturma\n    trainloader = create_balanced_dataloader(train_dataset, batch_size=batch_size)  # Dengeleme eklendi\n    valloader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)  # Doğrulama seti normal şekilde\n\n    return trainloader, valloader, len(train_df), len(val_df)\n\n\n# Define the transforms\ntransform = transforms.Compose([\n    transforms.Lambda(lambda x: (x * 255).astype(np.uint8)),  # Convert back to uint8 for PIL\n    transforms.ToPILImage(),\n    transforms.Resize((224, 224)),\n    transforms.Grayscale(num_output_channels=3),\n    transforms.ToTensor(),\n])\n\n# Create dataloaders for each series description\ndataloaders = {}\nlengths = {}\n\n# DataLoader'ları oluşturma\ntrainloader_t1, valloader_t1, len_train_t1, len_val_t1 = create_datasets_and_loaders(\n    train_data, 'Sagittal T1', transform\n)\ntrainloader_t2, valloader_t2, len_train_t2, len_val_t2 = create_datasets_and_loaders(\n    train_data, 'Axial T2', transform\n)\ntrainloader_t2stir, valloader_t2stir, len_train_t2stir, len_val_t2stir = create_datasets_and_loaders(\n    train_data, 'Sagittal T2/STIR', transform\n)\n\ndataloaders['Sagittal T1'] = (trainloader_t1, valloader_t1)\ndataloaders['Axial T2'] = (trainloader_t2, valloader_t2)\ndataloaders['Sagittal T2/STIR'] = (trainloader_t2stir, valloader_t2stir)\n\nlengths['Sagittal T1'] = (len_train_t1, len_val_t1)\nlengths['Axial T2'] = (len_train_t2, len_val_t2)\nlengths['Sagittal T2/STIR'] = (len_train_t2stir, len_val_t2stir)\n\n\n# Dictionary mapping labels to indices\nlabel_map = {'Mild': 0, 'Moderate': 1, 'Severe': 2}\n\n# Eğitim setinin ve doğrulama setinin uzunluğunu yazdırma\nprint(f\"Özgün Eğitim seti uzunluğu: {len(trainloader_t1.dataset) / 2}\")  # Yarıya bölüyoruz çünkü her örnek iki kere sayılıyor\nprint(f\"Özgün Doğrulama seti uzunluğu: {len(valloader_t1.dataset) / 2}\")  # Aynı şekilde\nprint(f\"Eğitim setinin uzunluğu: {len(trainloader_t1.dataset)}\")  # Eğitim setinin uzunluğunu yazdır\nprint(f\"Doğrulama setinin uzunluğu: {len(valloader_t1.dataset)}\")  # Doğrulama setinin uzunluğunu yazdır\n","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:29:11.607821Z","iopub.execute_input":"2025-01-15T16:29:11.608146Z","iopub.status.idle":"2025-01-15T16:29:13.508424Z","shell.execute_reply.started":"2025-01-15T16:29:11.608120Z","shell.execute_reply":"2025-01-15T16:29:13.507587Z"},"papermill":{"duration":2.026555,"end_time":"2025-01-09T00:15:40.53659","exception":false,"start_time":"2025-01-09T00:15:38.510035","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Eğitim veri setindeki sınıf dağılımı\ntrain_class_distribution = train_data['severity'].value_counts()\n\nprint(\"Eğitim veri seti sınıf dağılımı:\")\nprint(train_class_distribution)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-15T16:29:35.246030Z","iopub.execute_input":"2025-01-15T16:29:35.246595Z","iopub.status.idle":"2025-01-15T16:29:35.255674Z","shell.execute_reply.started":"2025-01-15T16:29:35.246559Z","shell.execute_reply":"2025-01-15T16:29:35.254817Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Bir örnek alın\nsample_index = 0  # İlk örnek\naugmented_image, label = trainloader_t1.dataset[sample_index]\noriginal_image, label = trainloader_t1.dataset[sample_index * 2]  # Orijinal versiyonu alın\n\n# Görselleştirme\nimport matplotlib.pyplot as plt\n\nplt.figure(figsize=(12, 6))\nplt.subplot(1, 2, 1)\nplt.title(\"Augmented Image\")\nplt.imshow(augmented_image.permute(1, 2, 0).numpy())  # Tensor'u numpy olarak göster\n\nplt.subplot(1, 2, 2)\nplt.title(\"Original Image\")\nplt.imshow(original_image.permute(1, 2, 0).numpy())  # Orijinal tensor\n\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-15T16:29:44.558584Z","iopub.execute_input":"2025-01-15T16:29:44.558925Z","iopub.status.idle":"2025-01-15T16:29:45.236616Z","shell.execute_reply.started":"2025-01-15T16:29:44.558904Z","shell.execute_reply":"2025-01-15T16:29:45.235583Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Function to visualize a batch of images\ndef visualize_batch(dataloader):\n    images, labels = next(iter(dataloader))\n    fig, axes = plt.subplots(1, len(images), figsize=(20, 5))\n    for i, (img, lbl) in enumerate(zip(images, labels)):\n        ax = axes[i]\n        img = img.permute(1, 2, 0)  # Convert to HWC for visualization\n        ax.imshow(img)\n        ax.set_title(f\"Label: {lbl}\")\n        ax.axis('off')\n    plt.show()\n\n# Visualize samples from each dataloader\nprint(\"Visualizing Sagittal T1 samples\")\nvisualize_batch(trainloader_t1)\nprint(\"Visualizing Axial T2 samples\")\nvisualize_batch(trainloader_t2)\nprint(\"Visualizing Sagittal T2/STIR samples\")\nvisualize_batch(trainloader_t2stir)","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:29:50.153479Z","iopub.execute_input":"2025-01-15T16:29:50.153841Z","iopub.status.idle":"2025-01-15T16:29:52.800627Z","shell.execute_reply.started":"2025-01-15T16:29:50.153815Z","shell.execute_reply":"2025-01-15T16:29:52.799642Z"},"papermill":{"duration":2.834385,"end_time":"2025-01-09T00:15:43.390082","exception":false,"start_time":"2025-01-09T00:15:40.555697","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nimage, label = next(iter(trainloader_t2))\nsample = image[1].permute(1, 2, 0)  #sample\n\n# Görüntü ve etiketleri çiz\nplt.figure(figsize=(8, 4))\nplt.imshow(sample)  # Eğer 3 kanallı ise cmap kullanmaya gerek yok\nplt.title(label[1])  # Etiketi yazdır\nplt.axis('off')  # Ekseni kapat\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:29:55.239123Z","iopub.execute_input":"2025-01-15T16:29:55.239426Z","iopub.status.idle":"2025-01-15T16:29:55.565920Z","shell.execute_reply.started":"2025-01-15T16:29:55.239403Z","shell.execute_reply":"2025-01-15T16:29:55.565015Z"},"papermill":{"duration":0.476766,"end_time":"2025-01-09T00:15:43.904431","exception":false,"start_time":"2025-01-09T00:15:43.427665","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision.models as models\nfrom torchvision import transforms\nfrom torch.utils.data import DataLoader\nfrom sklearn.model_selection import train_test_split\nimport pandas as pd\nfrom tqdm import tqdm\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:29:57.958267Z","iopub.execute_input":"2025-01-15T16:29:57.958593Z","iopub.status.idle":"2025-01-15T16:29:58.016913Z","shell.execute_reply.started":"2025-01-15T16:29:57.958568Z","shell.execute_reply":"2025-01-15T16:29:58.015969Z"},"papermill":{"duration":0.108177,"end_time":"2025-01-09T00:15:44.053793","exception":false,"start_time":"2025-01-09T00:15:43.945616","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision.models as models\nfrom torchvision.models import ResNet50_Weights  # ResNet50 ağırlıkları için enum'u import et\n\nclass CustomResNet50(nn.Module):\n    def __init__(self, num_classes=3, pretrained_weights=None):\n        super(CustomResNet50, self).__init__()\n        \n        # pretrained=True yerine weights=ResNet50_Weights.IMAGENET1K_V1 kullanarak ağırlıkları yükleyin\n        self.model = models.resnet50(weights=ResNet50_Weights.IMAGENET1K_V1).to(device)\n        \n        # Eğer manuel ağırlık yolu verilmişse, bu ağırlıkları yükle\n        if pretrained_weights:\n            self.model.load_state_dict(torch.load(pretrained_weights))\n        \n        num_ftrs = self.model.fc.in_features  # Son katmanın özellik sayısını al\n        self.model.fc = nn.Linear(num_ftrs, num_classes)  # Son katmanı değiştirme\n\n    def forward(self, x):\n        return self.model(x)\n\n    def unfreeze_middle_layers(self):\n        \"\"\"Orta katmanları çöz.\"\"\"\n        for name, param in self.model.named_parameters():\n            if 'layer3' in name or 'layer4' in name:  \n                param.requires_grad = True\n            else:\n                param.requires_grad = False\n\n# Modeli başlat\nsagittal_t1_model = CustomResNet50(num_classes=3).to(device)\naxial_t2_model = CustomResNet50(num_classes=3).to(device)\nsagittal_t2stir_model = CustomResNet50(num_classes=3).to(device)\n\n# Orta katmanları çözmek için\nfor model in [sagittal_t1_model, axial_t2_model, sagittal_t2stir_model]:\n    model.unfreeze_middle_layers()  # Orta katmanları çöz\n\n# Eğitim parametreleri\nweights = torch.tensor([1.0, 2.0, 4.0])\ncriterion = nn.CrossEntropyLoss(weight=weights.to(device))\n\n# Optimizer ayarları\noptimizer_sagittal_t1 = torch.optim.Adam(sagittal_t1_model.parameters(), lr=0.001)\noptimizer_axial_t2 = torch.optim.Adam(axial_t2_model.parameters(), lr=0.001)\noptimizer_sagittal_t2stir = torch.optim.Adam(sagittal_t2stir_model.parameters(), lr=0.001)\n\n# Modelleri ve optimizörleri saklamak için dictionary\nmodels_dict = {\n    'Sagittal T1': sagittal_t1_model,\n    'Axial T2': axial_t2_model,\n    'Sagittal T2/STIR': sagittal_t2stir_model,\n}\n\noptimizers_dict = {\n    'Sagittal T1': optimizer_sagittal_t1,\n    'Axial T2': optimizer_axial_t2,\n    'Sagittal T2/STIR': optimizer_sagittal_t2stir,\n}\n\n# Eğitim yapılabilir parametrelerin sayısını yazdır\nfor model_name, model in models_dict.items():\n    trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n    print(f\"Trainable parameters for {model_name}: {trainable_params}\")\n","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:30:00.406669Z","iopub.execute_input":"2025-01-15T16:30:00.406980Z","iopub.status.idle":"2025-01-15T16:30:02.816904Z","shell.execute_reply.started":"2025-01-15T16:30:00.406957Z","shell.execute_reply":"2025-01-15T16:30:02.816061Z"},"papermill":{"duration":2.836557,"end_time":"2025-01-09T00:15:46.930047","exception":false,"start_time":"2025-01-09T00:15:44.09349","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"label_map = {'normal_mild': 0, 'moderate': 1, 'severe': 2}","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:30:05.623669Z","iopub.execute_input":"2025-01-15T16:30:05.624001Z","iopub.status.idle":"2025-01-15T16:30:05.628496Z","shell.execute_reply.started":"2025-01-15T16:30:05.623974Z","shell.execute_reply":"2025-01-15T16:30:05.627505Z"},"papermill":{"duration":0.048133,"end_time":"2025-01-09T00:15:47.018442","exception":false,"start_time":"2025-01-09T00:15:46.970309","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for images, labels in trainloader_t2:\n    labels = torch.tensor([label_map[label] for label in labels])\n    labels = labels.to(device)\n    print(labels)\n    break","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:30:14.851381Z","iopub.execute_input":"2025-01-15T16:30:14.851789Z","iopub.status.idle":"2025-01-15T16:30:15.147751Z","shell.execute_reply.started":"2025-01-15T16:30:14.851759Z","shell.execute_reply":"2025-01-15T16:30:15.146908Z"},"papermill":{"duration":0.2219,"end_time":"2025-01-09T00:15:47.279888","exception":false,"start_time":"2025-01-09T00:15:47.057988","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import precision_score, recall_score, roc_auc_score, cohen_kappa_score, matthews_corrcoef, balanced_accuracy_score\n","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:30:20.076860Z","iopub.execute_input":"2025-01-15T16:30:20.077148Z","iopub.status.idle":"2025-01-15T16:30:20.081203Z","shell.execute_reply.started":"2025-01-15T16:30:20.077124Z","shell.execute_reply":"2025-01-15T16:30:20.080223Z"},"papermill":{"duration":0.04643,"end_time":"2025-01-09T00:15:47.366002","exception":false,"start_time":"2025-01-09T00:15:47.319572","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch.optim.lr_scheduler as lr_scheduler\nfrom copy import deepcopy\nfrom sklearn.metrics import (\n    confusion_matrix,\n    f1_score,\n    precision_score,\n    recall_score,\n    cohen_kappa_score,\n    matthews_corrcoef,\n    balanced_accuracy_score,\n    roc_auc_score,\n    classification_report\n)\n\ndef train_model(model, trainloader, valloader, len_train, len_val, optimizer, num_epochs=10, patience=3):\n    # Learning rate scheduler\n    scheduler = lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.1)\n\n    best_val_acc = 0.0\n    best_model_wts = deepcopy(model.state_dict())\n    counter = 0\n\n    for epoch in range(num_epochs):\n        model.train()\n        train_loss = 0\n        correct_train = 0\n\n        with tqdm(trainloader, unit=\"batch\") as tepoch:\n            for images, labels in tepoch:\n                images, labels = images.to(device), torch.tensor([label_map[label] for label in labels]).to(device)\n                optimizer.zero_grad()\n                outputs = model(images)\n                loss = criterion(outputs, labels)\n                loss.backward()\n                optimizer.step()\n                train_loss += loss.item()\n\n                probabilities = torch.softmax(outputs, dim=1)\n                _, predicted = torch.max(probabilities, 1)\n                correct_train += (predicted == labels).sum().item()\n\n                tepoch.set_postfix(epoch=epoch+1)\n\n        scheduler.step()\n\n        train_loss /= len(trainloader)\n        train_acc = 100 * correct_train / len_train\n\n        model.eval()\n        val_loss, correct_val = 0, 0\n        all_labels = []\n        all_predictions = []\n\n        with torch.no_grad():\n            with tqdm(valloader, unit=\"batch\") as vepoch:\n                for images, labels in vepoch:\n                    images, labels = images.to(device), torch.tensor([label_map[label] for label in labels]).to(device)\n                    outputs = model(images)\n                    loss = criterion(outputs, labels)\n                    val_loss += loss.item()\n\n                    probabilities = torch.softmax(outputs, dim=1)\n\n                    if probabilities.dim() == 1:\n                        _, predicted = torch.max(probabilities, 0)\n                    else:\n                        _, predicted = torch.max(probabilities, 1)\n                    correct_val += (predicted == labels).sum().item()\n\n                    all_labels.extend(labels.cpu().numpy())\n                    all_predictions.extend(predicted.cpu().numpy())\n\n                    vepoch.set_postfix(epoch=epoch+1)\n\n        val_loss /= len(valloader)\n        val_acc = 100 * correct_val / len_val\n\n        # Confusion Matrix ve F1 Skoru Hesaplama\n        conf_matrix = confusion_matrix(all_labels, all_predictions)\n        f1 = f1_score(all_labels, all_predictions, average=\"weighted\")\n        class_report = classification_report(all_labels, all_predictions)\n\n        # Precision ve Recall\n        precision = precision_score(all_labels, all_predictions, average=\"weighted\")\n        recall = recall_score(all_labels, all_predictions, average=\"weighted\")\n\n        # Cohen's Kappa\n        kappa = cohen_kappa_score(all_labels, all_predictions)\n\n        # Matthews Correlation Coefficient (MCC)\n        mcc = matthews_corrcoef(all_labels, all_predictions)\n\n        # Balanced Accuracy\n        balanced_acc = balanced_accuracy_score(all_labels, all_predictions)\n\n        # ROC-AUC (Eğer çok sınıflıysa one-vs-rest yaklaşımı kullanılır)\n        try:\n            roc_auc = roc_auc_score(all_labels, torch.nn.functional.one_hot(torch.tensor(all_predictions), num_classes=len(set(all_labels))).numpy(), multi_class=\"ovr\")\n        except ValueError:\n            roc_auc = \"N/A (ROC-AUC çok sınıflı problemde uygun olmayabilir)\"\n\n        # Çıktılar\n        print(f\"Precision: {precision:.4f}\")\n        print(f\"Recall: {recall:.4f}\")\n        print(f\"Cohen's Kappa: {kappa:.4f}\")\n        print(f\"MCC: {mcc:.4f}\")\n        print(f\"Balanced Accuracy: {balanced_acc:.4f}\")\n        print(f\"ROC-AUC Score: {roc_auc}\")\n        print(f\"Epoch {epoch+1}, Train Loss: {train_loss:.4f}, Train Acc: {train_acc:.2f}%, Val Loss: {val_loss:.4f}, Val Acc: {val_acc:.2f}%\")\n        print(f\"Confusion Matrix:\\n{conf_matrix}\")\n        print(f\"F1 Score: {f1:.4f}\")\n        print(f\"Classification Report:\\n{class_report}\")\n\n        # Save the best model and check for early stopping\n        if val_acc > best_val_acc:\n            best_val_acc = val_acc\n            best_model_wts = deepcopy(model.state_dict())\n            counter = 0\n            torch.save(best_model_wts, f'best_model_{epoch+1}.pth')\n        else:\n            counter += 1\n\n        # Early stopping\n        if counter >= patience:\n            print(f\"Early stopping triggered after {epoch+1} epochs\")\n            break\n\n    # Load best model weights\n    model.load_state_dict(best_model_wts)\n    return model, best_val_acc","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:30:27.924639Z","iopub.execute_input":"2025-01-15T16:30:27.924939Z","iopub.status.idle":"2025-01-15T16:30:27.938854Z","shell.execute_reply.started":"2025-01-15T16:30:27.924916Z","shell.execute_reply":"2025-01-15T16:30:27.937827Z"},"papermill":{"duration":0.059272,"end_time":"2025-01-09T00:15:47.465438","exception":false,"start_time":"2025-01-09T00:15:47.406166","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#!rm -rf /kaggle/working/*","metadata":{"execution":{"iopub.status.busy":"2025-01-12T23:05:40.802562Z","iopub.status.idle":"2025-01-12T23:05:40.802963Z","shell.execute_reply":"2025-01-12T23:05:40.802778Z"},"papermill":{"duration":0.046443,"end_time":"2025-01-09T00:15:47.551645","exception":false,"start_time":"2025-01-09T00:15:47.505202","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Eğitim işlemi\nfor desc, model in models_dict.items():  # Burada 'models_dict' kullanılmalı\n    if desc == 'Sagittal T1':\n        trainloader, valloader, len_train, len_val = trainloader_t1, valloader_t1, len_train_t1, len_val_t1\n    elif desc == 'Axial T2':\n        trainloader, valloader, len_train, len_val = trainloader_t2, valloader_t2, len_train_t2, len_val_t2\n    elif desc == 'Sagittal T2/STIR':\n        trainloader, valloader, len_train, len_val = trainloader_t2stir, valloader_t2stir, len_train_t2stir, len_val_t2stir\n    \n    print(f\"Training model for {desc}\")\n    train_model(model, trainloader, valloader, len_train, len_val, optimizers_dict[desc])  # Burada 'optimizers_dict' kullanılmalı","metadata":{"execution":{"iopub.status.busy":"2025-01-15T16:30:31.180697Z","iopub.execute_input":"2025-01-15T16:30:31.181036Z"},"papermill":{"duration":4419.417047,"end_time":"2025-01-09T01:29:27.008864","exception":false,"start_time":"2025-01-09T00:15:47.591817","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data['level'].unique()","metadata":{"execution":{"iopub.status.busy":"2025-01-13T00:48:15.512461Z","iopub.execute_input":"2025-01-13T00:48:15.512758Z","iopub.status.idle":"2025-01-13T00:48:15.520535Z","shell.execute_reply.started":"2025-01-13T00:48:15.512735Z","shell.execute_reply":"2025-01-13T00:48:15.519680Z"},"papermill":{"duration":2.916602,"end_time":"2025-01-09T01:29:32.841403","exception":false,"start_time":"2025-01-09T01:29:29.924801","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null}]}