{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"dockerImageVersionId":30887,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# # For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# # Input data files are available in the read-only \"../input/\" directory\n# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-02-16T15:32:47.282642Z","iopub.execute_input":"2025-02-16T15:32:47.283021Z","iopub.status.idle":"2025-02-16T15:32:47.288993Z","shell.execute_reply.started":"2025-02-16T15:32:47.282984Z","shell.execute_reply":"2025-02-16T15:32:47.287742Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import os\n# import numpy as np\n# import pandas as pd\n# import pydicom\n# import tensorflow as tf\n# from tensorflow.keras import layers, models, optimizers, callbacks\n# from sklearn.model_selection import train_test_split\n# from sklearn.utils.class_weight import compute_class_weight\n# from tqdm import tqdm\n# import matplotlib.pyplot as plt\n# import seaborn as sns\n# from sklearn.metrics import confusion_matrix, classification_report\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-16T15:32:47.291301Z","iopub.execute_input":"2025-02-16T15:32:47.291613Z","iopub.status.idle":"2025-02-16T15:32:47.304942Z","shell.execute_reply.started":"2025-02-16T15:32:47.291583Z","shell.execute_reply":"2025-02-16T15:32:47.304024Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ------------------------------\n# # 1. بارگذاری فایل‌های CSV و تعریف مسیرها\n# # ------------------------------\n# train_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/'\n\n# train_df      = pd.read_csv(os.path.join(train_path, 'train.csv'))\n# label_df      = pd.read_csv(os.path.join(train_path, 'train_label_coordinates.csv'))\n# train_desc_df = pd.read_csv(os.path.join(train_path, 'train_series_descriptions.csv'))\n# test_desc_df  = pd.read_csv(os.path.join(train_path, 'test_series_descriptions.csv'))\n# sub           = pd.read_csv(os.path.join(train_path, 'sample_submission.csv'))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-16T15:32:47.306072Z","iopub.execute_input":"2025-02-16T15:32:47.306365Z","iopub.status.idle":"2025-02-16T15:32:47.328545Z","shell.execute_reply.started":"2025-02-16T15:32:47.306301Z","shell.execute_reply":"2025-02-16T15:32:47.327579Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ------------------------------\n# # 2. تابع برای تولید مسیرهای تصاویر\n# # ------------------------------\n# def generate_image_paths(df, data_dir):\n#     image_paths = []\n#     for study_id, series_id in zip(df['study_id'], df['series_id']):\n#         study_dir = os.path.join(data_dir, str(study_id))\n#         series_dir = os.path.join(study_dir, str(series_id))\n#         if os.path.exists(series_dir):\n#             images = os.listdir(series_dir)\n#             image_paths.extend([os.path.join(series_dir, img) for img in images])\n#     return image_paths\n\n# train_image_paths = generate_image_paths(train_desc_df, os.path.join(train_path, 'train_images'))\n# test_image_paths  = generate_image_paths(test_desc_df, os.path.join(train_path, 'test_images'))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-16T15:32:47.329478Z","iopub.execute_input":"2025-02-16T15:32:47.329788Z","iopub.status.idle":"2025-02-16T15:32:47.347620Z","shell.execute_reply.started":"2025-02-16T15:32:47.329758Z","shell.execute_reply":"2025-02-16T15:32:47.346596Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ------------------------------\n# # 3. تغییر شکل داده‌های train و ادغام دیتا فریم‌ها\n# # ------------------------------\n# def reshape_row(row):\n#     data = {'study_id': [], 'condition': [], 'level': [], 'severity': []}\n#     for column, value in row.items():\n#         if column not in ['study_id', 'series_id', 'instance_number', 'x', 'y', 'series_description']:\n#             parts = column.split('_')\n#             condition = ' '.join([word.capitalize() for word in parts[:-2]])\n#             level = parts[-2].capitalize() + '/' + parts[-1].capitalize()\n#             data['study_id'].append(row['study_id'])\n#             data['condition'].append(condition)\n#             data['level'].append(level)\n#             data['severity'].append(value)\n#     return pd.DataFrame(data)\n\n# new_train_df = pd.concat([reshape_row(row) for _, row in train_df.iterrows()], ignore_index=True)\n\n# merged_df = pd.merge(new_train_df, label_df, on=['study_id', 'condition', 'level'], how='inner')\n# final_merged_df = pd.merge(merged_df, train_desc_df, on=['series_id','study_id'], how='inner')\n\n# final_merged_df['row_id'] = (final_merged_df['study_id'].astype(str) + '_' +\n#                                final_merged_df['condition'].str.lower().str.replace(' ', '_') + '_' +\n#                                final_merged_df['level'].str.lower().str.replace('/', '_'))\n\n# final_merged_df['image_path'] = (os.path.join(train_path, 'train_images') + '/' +\n#                                  final_merged_df['study_id'].astype(str) + '/' +\n#                                  final_merged_df['series_id'].astype(str) + '/' +\n#                                  final_merged_df['instance_number'].astype(str) + '.dcm')\n\n# # تغییر برچسب severity به حروف کوچک\n# final_merged_df['severity'] = final_merged_df['severity'].map({\n#     'Normal/Mild': 'normal_mild',\n#     'Moderate': 'moderate',\n#     'Severe': 'severe'\n# })\n\n# # فیلتر کردن ردیف‌هایی که مسیر تصویر موجود است\n# def check_exists(path):\n#     return os.path.exists(path)\n# final_merged_df = final_merged_df[final_merged_df['image_path'].apply(check_exists)]\n\n# # نگاشت برچسب‌ها به اعداد صحیح\n# severity_map = {'normal_mild': 0, 'moderate': 1, 'severe': 2}\n# final_merged_df['severity'] = final_merged_df['severity'].map(severity_map)\n\n# # استفاده از final_merged_df به عنوان داده‌های آموزشی\n# train_data = final_merged_df.copy()\n# train_data = train_data.dropna()   # حذف ردیف‌های دارای NaN\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-16T15:32:47.348766Z","iopub.execute_input":"2025-02-16T15:32:47.349104Z","iopub.status.idle":"2025-02-16T15:32:47.364809Z","shell.execute_reply.started":"2025-02-16T15:32:47.349070Z","shell.execute_reply":"2025-02-16T15:32:47.363875Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ------------------------------\n# # 4. محاسبه وزن کلاس‌ها\n# # ------------------------------\n# classes = np.unique(train_data['severity'])\n# class_weights = compute_class_weight(class_weight='balanced', classes=classes, y=train_data['severity'])\n# class_weights_dict = {int(c): w for c, w in zip(classes, class_weights)}\n# print(\"وزن کلاس‌ها:\", class_weights_dict)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-16T15:32:47.365664Z","iopub.execute_input":"2025-02-16T15:32:47.365942Z","iopub.status.idle":"2025-02-16T15:32:47.385461Z","shell.execute_reply.started":"2025-02-16T15:32:47.365907Z","shell.execute_reply":"2025-02-16T15:32:47.384644Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ------------------------------\n# # 5. توابع بارگذاری و پیش‌پردازش تصاویر DICOM\n# # ------------------------------\n# def load_dicom_image(path):\n#     \"\"\"\n#     تابعی برای بارگذاری تصویر DICOM\n#     \"\"\"\n#     path = path.numpy().decode('utf-8')  # تبدیل EagerTensor به رشته\n#     ds = pydicom.dcmread(path)\n#     data = ds.pixel_array.astype(np.float32)\n#     data = data - np.min(data)\n#     if np.max(data) != 0:\n#         data = data / np.max(data)\n#     data = (data * 255).astype(np.uint8)\n#     return data\n\n# def load_and_preprocess(path, label=None):\n#     \"\"\"\n#     - بارگذاری تصویر DICOM با استفاده از tf.py_function\n#     - افزودن بعد کانال (برای تصاویر خاکستری)\n#     - تغییر اندازه به 224x224، تبدیل از grayscale به RGB و نرمال‌سازی به [0, 1]\n#     \"\"\"\n#     image = tf.py_function(func=lambda p: load_dicom_image(p), inp=[path], Tout=tf.uint8)\n#     image.set_shape([None, None])\n#     image = tf.expand_dims(image, axis=-1)  # تبدیل (ارتفاع, عرض) به (ارتفاع, عرض, 1)\n#     image = tf.image.resize(image, [224, 224])\n#     image = tf.image.grayscale_to_rgb(image)\n#     image = tf.cast(image, tf.float32) / 255.0\n#     if label is None:\n#         return image\n#     else:\n#         return image, label\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-16T15:32:47.386337Z","iopub.execute_input":"2025-02-16T15:32:47.386558Z","iopub.status.idle":"2025-02-16T15:32:47.402735Z","shell.execute_reply.started":"2025-02-16T15:32:47.386538Z","shell.execute_reply":"2025-02-16T15:32:47.401963Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ------------------------------\n# # 6. ایجاد دیتاست‌های TensorFlow برای داده‌های آموزشی\n# # ------------------------------\n# def create_datasets(df, batch_size=8):\n#     train_paths = df['image_path'].values\n#     train_labels = df['severity'].values\n    \n#     train_ds = tf.data.Dataset.from_tensor_slices((train_paths, train_labels))\n#     train_ds = train_ds.map(lambda p, l: load_and_preprocess(p, l),\n#                             num_parallel_calls=tf.data.AUTOTUNE)\n#     train_ds = train_ds.shuffle(buffer_size=len(df)).batch(batch_size).prefetch(tf.data.AUTOTUNE)\n    \n#     return train_ds, len(df)\n\n# # ایجاد دیتاست برای داده‌های آموزشی (از تمام داده‌ها استفاده می‌کنیم)\n# train_ds, len_train = create_datasets(train_data, batch_size=8)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-16T15:32:47.403764Z","iopub.execute_input":"2025-02-16T15:32:47.404063Z","iopub.status.idle":"2025-02-16T15:32:47.424165Z","shell.execute_reply.started":"2025-02-16T15:32:47.404034Z","shell.execute_reply":"2025-02-16T15:32:47.423200Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ------------------------------\n# # 7. تعریف مدل VGG16 با استفاده از Keras (TensorFlow)\n# # ------------------------------\n# def create_vgg16_model(num_classes=3):\n#     base_model = tf.keras.applications.VGG16(include_top=False,\n#                                              input_shape=(224, 224, 3),\n#                                              weights='imagenet')\n#     base_model.trainable = False\n#     x = layers.Flatten()(base_model.output)\n#     x = layers.Dense(4096, activation='relu')(x)\n#     x = layers.Dense(4096, activation='relu')(x)\n#     outputs = layers.Dense(num_classes, activation='softmax')(x)\n#     model = tf.keras.Model(inputs=base_model.input, outputs=outputs)\n#     return model\n\n# # ایجاد مدل VGG16\n# model = create_vgg16_model(num_classes=3)\n\n# model.compile(optimizer=optimizers.Adam(learning_rate=0.001),\n#               loss='sparse_categorical_crossentropy',\n#               metrics=['accuracy'])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-16T15:32:47.424915Z","iopub.execute_input":"2025-02-16T15:32:47.425227Z","iopub.status.idle":"2025-02-16T15:32:47.447873Z","shell.execute_reply.started":"2025-02-16T15:32:47.425198Z","shell.execute_reply":"2025-02-16T15:32:47.447089Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ------------------------------\n# # 8. آموزش مدل با وزن‌دهی کلاس‌ها\n# # ------------------------------\n# es_callback = callbacks.EarlyStopping(monitor='val_accuracy', patience=3, restore_best_weights=True)\n# ckpt_callback = callbacks.ModelCheckpoint('best_model.keras', monitor='val_accuracy', save_best_only=True)\n\n# # ایجاد دیتاست اعتبارسنجی (مثلاً 10% از داده‌های آموزشی)\n# val_split = 0.1\n# df_train, df_val = train_test_split(train_data, test_size=val_split, random_state=42)\n\n# # ایجاد دیتاست برای داده‌های آموزشی و اعتبارسنجی\n# train_ds, len_train = create_datasets(df_train, batch_size=8)\n# val_ds, len_val = create_datasets(df_val, batch_size=8)\n\n# # آموزش مدل\n# history = model.fit(train_ds,\n#                     epochs=1,\n#                     validation_data=val_ds,  # استفاده از دیتاست اعتبارسنجی\n#                     callbacks=[es_callback, ckpt_callback],\n#                     class_weight=class_weights_dict)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-16T15:32:47.448771Z","iopub.execute_input":"2025-02-16T15:32:47.449075Z","iopub.status.idle":"2025-02-16T15:32:47.466136Z","shell.execute_reply.started":"2025-02-16T15:32:47.449045Z","shell.execute_reply":"2025-02-16T15:32:47.465333Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ------------------------------\n# # 9. رسم نمودارهای تاریخچه آموزش و محاسبه Confusion Matrix\n# # ------------------------------\n# def plot_history_and_confusion(model, history, val_ds, model_name):\n#     # رسم نمودار loss و accuracy\n#     plt.figure(figsize=(12,5))\n    \n#     plt.subplot(1,2,1)\n#     plt.plot(history.history['loss'], label='Train Loss')\n#     plt.plot(history.history['val_loss'], label='Validation Loss')\n#     plt.title(f'{model_name} Loss')\n#     plt.xlabel('Epochs')\n#     plt.ylabel('Loss')\n#     plt.legend()\n    \n#     plt.subplot(1,2,2)\n#     plt.plot(history.history['accuracy'], label='Train Accuracy')\n#     plt.plot(history.history['val_accuracy'], label='Validation Accuracy')\n#     plt.title(f'{model_name} Accuracy')\n#     plt.xlabel('Epochs')\n#     plt.ylabel('Accuracy')\n#     plt.legend()\n#     plt.show()\n    \n#     # محاسبه پیش‌بینی‌ها روی داده‌های اعتبارسنجی\n#     y_true = []\n#     y_pred = []\n#     for images, labels in val_ds:\n#         preds = model.predict(images)\n#         y_pred.extend(np.argmax(preds, axis=1))\n#         y_true.extend(labels.numpy())\n    \n#     cm = confusion_matrix(y_true, y_pred)\n#     plt.figure(figsize=(8,6))\n#     sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')\n#     plt.title(f'Confusion Matrix for {model_name}')\n#     plt.xlabel('Predicted')\n#     plt.ylabel('True')\n#     plt.show()\n    \n#     print(f'Classification Report for {model_name}:\\n', classification_report(y_true, y_pred))\n\n# # رسم تاریخچه آموزش و محاسبه Confusion Matrix\n# plot_history_and_confusion(model, history, val_ds, \"VGG16 Model\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-16T15:32:47.467118Z","iopub.execute_input":"2025-02-16T15:32:47.467405Z","iopub.status.idle":"2025-02-16T15:32:47.484296Z","shell.execute_reply.started":"2025-02-16T15:32:47.467377Z","shell.execute_reply":"2025-02-16T15:32:47.483375Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ------------------------------\n# # 10. پیش‌بینی روی داده‌های Test و ایجاد سابمیشن\n# # ------------------------------\n# # ایجاد دیتاست برای داده‌های تست\n# expanded_rows = []\n# for index, row in test_desc_df.iterrows():\n#     study_id = row['study_id']\n#     series_id = row['series_id']\n#     series_description = row['series_description']\n#     series_path = os.path.join(train_path, 'test_images', str(study_id), str(series_id))\n#     if os.path.exists(series_path):\n#         image_files = [os.path.join(series_path, f) for f in os.listdir(series_path)\n#                        if os.path.isfile(os.path.join(series_path, f))]\n#         conditions = {'left': 'left_neural_foraminal_narrowing', 'right': 'right_neural_foraminal_narrowing'}  # sample mapping\n#         for side, condition in conditions.items():\n#             for image_path in image_files:\n#                 expanded_rows.append({\n#                     'study_id': study_id,\n#                     'series_id': series_id,\n#                     'series_description': series_description,\n#                     'image_path': image_path,\n#                     'condition': condition,\n#                     'row_id': f\"{study_id}_{condition}\"\n#                 })\n\n# expanded_test_desc = pd.DataFrame(expanded_rows)\n\n# # به‌روزرسانی row_id با اضافه کردن سطح (level)\n# levels = ['l1_l2', 'l2_l3', 'l3_l4', 'l4_l5', 'l5_s1']\n# def update_row_id(row, levels):\n#     level = levels[row.name % len(levels)]\n#     return f\"{row['study_id']}_{row['condition']}_{level}\"\n\n# expanded_test_desc['row_id'] = expanded_test_desc.apply(lambda row: update_row_id(row, levels), axis=1)\n\n# # ایجاد دیتاست تست (بدون برچسب)\n# test_paths = expanded_test_desc['image_path'].values\n# test_ds = tf.data.Dataset.from_tensor_slices(test_paths)\n# test_ds = test_ds.map(lambda p: load_and_preprocess(p),\n#                       num_parallel_calls=tf.data.AUTOTUNE)\n# test_ds = test_ds.batch(1)\n\n# # پیش‌بینی برای داده‌های تست\n# normal_mild_probs = []\n# moderate_probs = []\n# severe_probs = []\n# predictions_list = []\n\n# for i, batch in enumerate(tqdm(test_ds)):\n#     preds = model.predict(batch)\n#     preds = preds[0]\n#     normal_mild_probs.append(preds[0])\n#     moderate_probs.append(preds[1])\n#     severe_probs.append(preds[2])\n#     predictions_list.append(preds)\n\n# # اضافه کردن نتایج پیش‌بینی به دیتافریم تست\n# expanded_test_desc['normal_mild'] = normal_mild_probs\n# expanded_test_desc['moderate'] = moderate_probs\n# expanded_test_desc['severe'] = severe_probs\n\n# # ترکیب نتایج پیش‌بینی‌ها\n# submission_df = expanded_test_desc[[\"row_id\", \"normal_mild\", \"moderate\", \"severe\"]]\n# grouped_submission = submission_df.groupby('row_id').max().reset_index()\n\n# # ذخیره سابمیشن\n# sub[['normal_mild', 'moderate', 'severe']] = grouped_submission[['normal_mild', 'moderate', 'severe']]\n# sub.to_csv(\"/kaggle/working/submission.csv\", index=False)\n\n# print(\"نمونه سابمیشن:\")\n# print(sub.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-16T15:32:47.485200Z","iopub.execute_input":"2025-02-16T15:32:47.485477Z","iopub.status.idle":"2025-02-16T15:32:47.505606Z","shell.execute_reply.started":"2025-02-16T15:32:47.485449Z","shell.execute_reply":"2025-02-16T15:32:47.504602Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ------------------------------\n# # 11. ذخیره مدل\n# # ------------------------------\n# model.save(\"Vgg16_model.keras\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-16T15:32:47.506587Z","iopub.execute_input":"2025-02-16T15:32:47.506868Z","iopub.status.idle":"2025-02-16T15:32:47.523762Z","shell.execute_reply.started":"2025-02-16T15:32:47.506840Z","shell.execute_reply":"2025-02-16T15:32:47.522965Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport pydicom\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models, optimizers, callbacks\nfrom sklearn.model_selection import train_test_split\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import confusion_matrix, classification_report\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-16T15:32:47.524435Z","iopub.execute_input":"2025-02-16T15:32:47.524620Z","iopub.status.idle":"2025-02-16T15:32:47.541646Z","shell.execute_reply.started":"2025-02-16T15:32:47.524604Z","shell.execute_reply":"2025-02-16T15:32:47.540785Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ------------------------------\n# 1. بارگذاری فایل‌های CSV و تعریف مسیرها\n# ------------------------------\ntrain_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/'\n\ntrain_df      = pd.read_csv(os.path.join(train_path, 'train.csv'))\nlabel_df      = pd.read_csv(os.path.join(train_path, 'train_label_coordinates.csv'))\ntrain_desc_df = pd.read_csv(os.path.join(train_path, 'train_series_descriptions.csv'))\ntest_desc_df  = pd.read_csv(os.path.join(train_path, 'test_series_descriptions.csv'))\nsub           = pd.read_csv(os.path.join(train_path, 'sample_submission.csv'))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-16T15:32:47.542566Z","iopub.execute_input":"2025-02-16T15:32:47.542850Z","iopub.status.idle":"2025-02-16T15:32:47.655929Z","shell.execute_reply.started":"2025-02-16T15:32:47.542820Z","shell.execute_reply":"2025-02-16T15:32:47.654954Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ------------------------------\n# 2. تابع برای تولید مسیرهای تصاویر\n# ------------------------------\ndef generate_image_paths(df, data_dir):\n    image_paths = []\n    for study_id, series_id in zip(df['study_id'], df['series_id']):\n        study_dir = os.path.join(data_dir, str(study_id))\n        series_dir = os.path.join(study_dir, str(series_id))\n        if os.path.exists(series_dir):\n            images = os.listdir(series_dir)\n            image_paths.extend([os.path.join(series_dir, img) for img in images])\n    return image_paths\n\ntrain_image_paths = generate_image_paths(train_desc_df, os.path.join(train_path, 'train_images'))\ntest_image_paths  = generate_image_paths(test_desc_df, os.path.join(train_path, 'test_images'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-16T15:32:47.656988Z","iopub.execute_input":"2025-02-16T15:32:47.657294Z","iopub.status.idle":"2025-02-16T15:32:56.989482Z","shell.execute_reply.started":"2025-02-16T15:32:47.657266Z","shell.execute_reply":"2025-02-16T15:32:56.988566Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ------------------------------\n# 3. تغییر شکل داده‌های train و ادغام دیتا فریم‌ها\n# ------------------------------\ndef reshape_row(row):\n    data = {'study_id': [], 'condition': [], 'level': [], 'severity': []}\n    for column, value in row.items():\n        if column not in ['study_id', 'series_id', 'instance_number', 'x', 'y', 'series_description']:\n            parts = column.split('_')\n            condition = ' '.join([word.capitalize() for word in parts[:-2]])\n            level = parts[-2].capitalize() + '/' + parts[-1].capitalize()\n            data['study_id'].append(row['study_id'])\n            data['condition'].append(condition)\n            data['level'].append(level)\n            data['severity'].append(value)\n    return pd.DataFrame(data)\n\nnew_train_df = pd.concat([reshape_row(row) for _, row in train_df.iterrows()], ignore_index=True)\n\nmerged_df = pd.merge(new_train_df, label_df, on=['study_id', 'condition', 'level'], how='inner')\nfinal_merged_df = pd.merge(merged_df, train_desc_df, on=['series_id','study_id'], how='inner')\n\nfinal_merged_df['row_id'] = (final_merged_df['study_id'].astype(str) + '_' +\n                               final_merged_df['condition'].str.lower().str.replace(' ', '_') + '_' +\n                               final_merged_df['level'].str.lower().str.replace('/', '_'))\n\nfinal_merged_df['image_path'] = (os.path.join(train_path, 'train_images') + '/' +\n                                 final_merged_df['study_id'].astype(str) + '/' +\n                                 final_merged_df['series_id'].astype(str) + '/' +\n                                 final_merged_df['instance_number'].astype(str) + '.dcm')\n\n# تغییر برچسب severity به حروف کوچک\nfinal_merged_df['severity'] = final_merged_df['severity'].map({\n    'Normal/Mild': 'normal_mild',\n    'Moderate': 'moderate',\n    'Severe': 'severe'\n})\n\n# فیلتر کردن ردیف‌هایی که مسیر تصویر موجود است\ndef check_exists(path):\n    return os.path.exists(path)\nfinal_merged_df = final_merged_df[final_merged_df['image_path'].apply(check_exists)]\n\n# نگاشت برچسب‌ها به اعداد صحیح\nseverity_map = {'normal_mild': 0, 'moderate': 1, 'severe': 2}\nfinal_merged_df['severity'] = final_merged_df['severity'].map(severity_map)\n\n# استفاده از final_merged_df به عنوان داده‌های آموزشی\ntrain_data = final_merged_df.copy()\ntrain_data = train_data.dropna()   # حذف ردیف‌های دارای NaN","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-16T15:32:56.990438Z","iopub.execute_input":"2025-02-16T15:32:56.990767Z","iopub.status.idle":"2025-02-16T15:33:25.202533Z","shell.execute_reply.started":"2025-02-16T15:32:56.990730Z","shell.execute_reply":"2025-02-16T15:33:25.201591Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ------------------------------\n# 4. محاسبه وزن کلاس‌ها بر اساس تعداد تصاویر\n# ------------------------------\nclass_counts = train_data['severity'].value_counts()\ntotal_images = class_counts.sum()\n\n# محاسبه ضرایب\nweights = {label: total_images / count for label, count in class_counts.items()}\n\n# نرمال‌سازی وزن‌ها\ntotal_weight = sum(weights.values())\nnormalized_weights = {int(label): weight / total_weight for label, weight in weights.items()}\n\nprint(\"وزن کلاس‌ها:\", normalized_weights)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-16T15:33:25.203817Z","iopub.execute_input":"2025-02-16T15:33:25.204086Z","iopub.status.idle":"2025-02-16T15:33:25.211347Z","shell.execute_reply.started":"2025-02-16T15:33:25.204066Z","shell.execute_reply":"2025-02-16T15:33:25.210441Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ------------------------------\n# 5. توابع بارگذاری و پیش‌پردازش تصاویر DICOM\n# ------------------------------\ndef load_dicom_image(path):\n    \"\"\"\n    تابعی برای بارگذاری تصویر DICOM\n    \"\"\"\n    path = path.numpy().decode('utf-8')  # تبدیل EagerTensor به رشته\n    ds = pydicom.dcmread(path)\n    data = ds.pixel_array.astype(np.float32)\n    data = data - np.min(data)\n    if np.max(data) != 0:\n        data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data\n\ndef load_and_preprocess(path, label=None):\n    \"\"\"\n    - بارگذاری تصویر DICOM با استفاده از tf.py_function\n    - افزودن بعد کانال (برای تصاویر خاکستری)\n    - تغییر اندازه به 224x224، تبدیل از grayscale به RGB و نرمال‌سازی به [0, 1]\n    \"\"\"\n    image = tf.py_function(func=lambda p: load_dicom_image(p), inp=[path], Tout=tf.uint8)\n    image.set_shape([None, None])\n    image = tf.expand_dims(image, axis=-1)  # تبدیل (ارتفاع, عرض) به (ارتفاع, عرض, 1)\n    image = tf.image.resize(image, [224, 224])\n    image = tf.image.grayscale_to_rgb(image)\n    image = tf.cast(image, tf.float32) / 255.0\n    if label is None:\n        return image\n    else:\n        return image, label\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-16T15:33:25.212181Z","iopub.execute_input":"2025-02-16T15:33:25.212500Z","iopub.status.idle":"2025-02-16T15:33:25.233005Z","shell.execute_reply.started":"2025-02-16T15:33:25.212457Z","shell.execute_reply":"2025-02-16T15:33:25.232249Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ------------------------------\n# 6. ایجاد دیتاست‌های TensorFlow برای هر سری توضیحی\n# ------------------------------\ndef create_datasets(df, series_description, batch_size=8):\n    filtered_df = df[df['series_description'] == series_description]\n    if filtered_df.empty:\n        raise ValueError(f\"داده‌ای برای سری توضیحی: {series_description} پیدا نشد.\")\n    train_df_part, val_df_part = train_test_split(filtered_df, test_size=0.2, random_state=42)\n    \n    train_paths = train_df_part['image_path'].values\n    train_labels = train_df_part['severity'].values\n    val_paths = val_df_part['image_path'].values\n    val_labels = val_df_part['severity'].values\n    \n    train_ds = tf.data.Dataset.from_tensor_slices((train_paths, train_labels))\n    train_ds = train_ds.map(lambda p, l: load_and_preprocess(p, l),\n                            num_parallel_calls=tf.data.AUTOTUNE)\n    train_ds = train_ds.shuffle(buffer_size=len(train_df_part)).batch(batch_size).prefetch(tf.data.AUTOTUNE)\n    \n    val_ds = tf.data.Dataset.from_tensor_slices((val_paths, val_labels))\n    val_ds = val_ds.map(lambda p, l: load_and_preprocess(p, l),\n                        num_parallel_calls=tf.data.AUTOTUNE)\n    val_ds = val_ds.batch(batch_size).prefetch(tf.data.AUTOTUNE)\n    \n    return train_ds, val_ds, len(train_df_part), len(val_df_part)\n\n# ایجاد دیتاست‌ها برای سه سری توضیحی\ntrain_ds_t1, val_ds_t1, len_train_t1, len_val_t1 = create_datasets(train_data, 'Sagittal T1', batch_size=8)\ntrain_ds_t2, val_ds_t2, len_train_t2, len_val_t2 = create_datasets(train_data, 'Axial T2', batch_size=8)\ntrain_ds_t2stir, val_ds_t2stir, len_train_t2stir, len_val_t2stir = create_datasets(train_data, 'Sagittal T2/STIR', batch_size=8)\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-16T15:33:25.236151Z","iopub.execute_input":"2025-02-16T15:33:25.236349Z","iopub.status.idle":"2025-02-16T15:33:25.526196Z","shell.execute_reply.started":"2025-02-16T15:33:25.236332Z","shell.execute_reply":"2025-02-16T15:33:25.525450Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ------------------------------\n# 7. تعریف مدل VGG16 با استفاده از Keras (TensorFlow)\n# ------------------------------\ndef create_vgg16_model(num_classes=3):\n    base_model = tf.keras.applications.VGG16(include_top=False,\n                                             input_shape=(224, 224, 3),\n                                             weights='imagenet')\n    base_model.trainable = False\n    x = layers.Flatten()(base_model.output)\n    x = layers.Dense(4096, activation='relu')(x)\n    x = layers.Dense(4096, activation='relu')(x)\n    outputs = layers.Dense(num_classes, activation='softmax')(x)\n    model = tf.keras.Model(inputs=base_model.input, outputs=outputs)\n    return model\n\n# ایجاد سه مدل مجزا برای سری‌های مختلف\nmodel_t1 = create_vgg16_model(num_classes=3)\nmodel_t2 = create_vgg16_model(num_classes=3)\nmodel_t2stir = create_vgg16_model(num_classes=3)\n\nmodel_t1.compile(optimizer=optimizers.Adam(learning_rate=0.001),\n                 loss='sparse_categorical_crossentropy',\n                 metrics=['accuracy'])\nmodel_t2.compile(optimizer=optimizers.Adam(learning_rate=0.001),\n                 loss='sparse_categorical_crossentropy',\n                 metrics=['accuracy'])\nmodel_t2stir.compile(optimizer=optimizers.Adam(learning_rate=0.001),\n                     loss='sparse_categorical_crossentropy',\n                     metrics=['accuracy'])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-16T15:33:25.527511Z","iopub.execute_input":"2025-02-16T15:33:25.527764Z","iopub.status.idle":"2025-02-16T15:33:26.619062Z","shell.execute_reply.started":"2025-02-16T15:33:25.527743Z","shell.execute_reply":"2025-02-16T15:33:26.618391Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ------------------------------\n# 8. آموزش مدل‌ها با وزن‌دهی کلاس‌ها\n# ------------------------------\nes_callback_t1 = callbacks.EarlyStopping(monitor='val_accuracy', patience=3, restore_best_weights=True)\nckpt_callback_t1 = callbacks.ModelCheckpoint('best_model_t1.keras', monitor='val_accuracy', save_best_only=True)\n\nes_callback_t2 = callbacks.EarlyStopping(monitor='val_accuracy', patience=3, restore_best_weights=True)\nckpt_callback_t2 = callbacks.ModelCheckpoint('best_model_t2.keras', monitor='val_accuracy', save_best_only=True)\n\nes_callback_t2stir = callbacks.EarlyStopping(monitor='val_accuracy', patience=3, restore_best_weights=True)\nckpt_callback_t2stir = callbacks.ModelCheckpoint('best_model_t2stir.keras', monitor='val_accuracy', save_best_only=True)\n\n# آموزش مدل‌ها\nhistory_t1 = model_t1.fit(train_ds_t1, epochs=50, validation_data=val_ds_t1,\n                          callbacks=[es_callback_t1, ckpt_callback_t1],\n                          class_weight=normalized_weights)\n\nhistory_t2 = model_t2.fit(train_ds_t2, epochs=50, validation_data=val_ds_t2,\n                          callbacks=[es_callback_t2, ckpt_callback_t2],\n                          class_weight=normalized_weights)\n\nhistory_t2stir = model_t2stir.fit(train_ds_t2stir, epochs=50, validation_data=val_ds_t2stir,\n                                  callbacks=[es_callback_t2stir, ckpt_callback_t2stir],\n                                  class_weight=normalized_weights)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-16T15:33:26.620227Z","iopub.execute_input":"2025-02-16T15:33:26.620518Z","iopub.status.idle":"2025-02-16T15:45:45.410318Z","shell.execute_reply.started":"2025-02-16T15:33:26.620495Z","shell.execute_reply":"2025-02-16T15:45:45.409624Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ------------------------------\n# 9. رسم نمودارهای تاریخچه آموزش و محاسبه Confusion Matrix برای هر مدل\n# ------------------------------\ndef plot_history_and_confusion(model, history, val_ds, model_name):\n    # رسم نمودار loss و accuracy\n    plt.figure(figsize=(12,5))\n    \n    plt.subplot(1,2,1)\n    plt.plot(history.history['loss'], label='Train Loss')\n    plt.plot(history.history['val_loss'], label='Validation Loss')\n    plt.title(f'{model_name} Loss')\n    plt.xlabel('Epochs')\n    plt.ylabel('Loss')\n    plt.legend()\n    \n    plt.subplot(1,2,2)\n    plt.plot(history.history['accuracy'], label='Train Accuracy')\n    plt.plot(history.history['val_accuracy'], label='Validation Accuracy')\n    plt.title(f'{model_name} Accuracy')\n    plt.xlabel('Epochs')\n    plt.ylabel('Accuracy')\n    plt.legend()\n    plt.show()\n    \n    # محاسبه پیش‌بینی‌ها روی داده‌های اعتبارسنجی\n    y_true = []\n    y_pred = []\n    for images, labels in val_ds:\n        preds = model.predict(images)\n        y_pred.extend(np.argmax(preds, axis=1))\n        y_true.extend(labels.numpy())\n    \n    cm = confusion_matrix(y_true, y_pred)\n    plt.figure(figsize=(8,6))\n    sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')\n    plt.title(f'Confusion Matrix for {model_name}')\n    plt.xlabel('Predicted')\n    plt.ylabel('True')\n    plt.show()\n    \n    print(f'Classification Report for {model_name}:\\n', classification_report(y_true, y_pred))\n\n# رسم نمودارها و confusion matrix برای هر مدل\nplot_history_and_confusion(model_t1, history_t1, val_ds_t1, \"VGG16 - Sagittal T1\")\nplot_history_and_confusion(model_t2, history_t2, val_ds_t2, \"VGG16 - Axial T2\")\nplot_history_and_confusion(model_t2stir, history_t2stir, val_ds_t2stir, \"VGG16 - Sagittal T2/STIR\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-16T15:45:45.411944Z","iopub.execute_input":"2025-02-16T15:45:45.412273Z","iopub.status.idle":"2025-02-16T15:48:59.842376Z","shell.execute_reply.started":"2025-02-16T15:45:45.412236Z","shell.execute_reply":"2025-02-16T15:48:59.841605Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ------------------------------\n# 10. پیش‌بینی روی داده‌های Test و ایجاد سابمیشن\n# ------------------------------\n# ایجاد دیتاست برای داده‌های تست\nexpanded_rows = []\nfor index, row in test_desc_df.iterrows():\n    study_id = row['study_id']\n    series_id = row['series_id']\n    series_description = row['series_description']\n    series_path = os.path.join(train_path, 'test_images', str(study_id), str(series_id))\n    if os.path.exists(series_path):\n        image_files = [os.path.join(series_path, f) for f in os.listdir(series_path)\n                       if os.path.isfile(os.path.join(series_path, f))]\n        for image_path in image_files:\n            expanded_rows.append({\n                'study_id': study_id,\n                'series_id': series_id,\n                'series_description': series_description,\n                'image_path': image_path,\n                'row_id': f\"{study_id}_{series_id}\"\n            })\n\nexpanded_test_desc = pd.DataFrame(expanded_rows)\n\n# ایجاد دیتاست تست (بدون برچسب)\ntest_paths = expanded_test_desc['image_path'].values\ntest_ds = tf.data.Dataset.from_tensor_slices(test_paths)\ntest_ds = test_ds.map(lambda p: load_and_preprocess(p),\n                      num_parallel_calls=tf.data.AUTOTUNE)\ntest_ds = test_ds.batch(1)\n\n# پیش‌بینی برای داده‌های تست\nnormal_mild_probs = []\nmoderate_probs = []\nsevere_probs = []\npredictions_list = []\n\n# دیکشنری مدل‌ها بر اساس سری توضیحی\nmodels_dict = {\n    'Sagittal T1': model_t1,\n    'Axial T2': model_t2,\n    'Sagittal T2/STIR': model_t2stir\n}\n\nfor i, batch in enumerate(tqdm(test_ds)):\n    series_description = expanded_test_desc.iloc[i]['series_description']\n    model_used = models_dict.get(series_description, None)\n    if model_used is None:\n        normal_mild_probs.append(None)\n        moderate_probs.append(None)\n        severe_probs.append(None)\n        predictions_list.append(None)\n    else:\n        preds = model_used.predict(batch)\n        preds = preds[0]\n        normal_mild_probs.append(preds[0])\n        moderate_probs.append(preds[1])\n        severe_probs.append(preds[2])\n        predictions_list.append(preds)\n\n# اضافه کردن نتایج پیش‌بینی به دیتافریم تست\nexpanded_test_desc['normal_mild'] = normal_mild_probs\nexpanded_test_desc['moderate'] = moderate_probs\nexpanded_test_desc['severe'] = severe_probs\n\n# ترکیب نتایج پیش‌بینی‌ها\nsubmission_df = expanded_test_desc[[\"row_id\", \"normal_mild\", \"moderate\", \"severe\"]]\ngrouped_submission = submission_df.groupby('row_id').max().reset_index()\n\n# ذخیره سابمیشن\nsub[['normal_mild', 'moderate', 'severe']] = grouped_submission[['normal_mild', 'moderate', 'severe']]\nsub.to_csv(\"/kaggle/working/submission.csv\", index=False)\n\nprint(\"نمونه سابمیشن:\")\nprint(sub.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-16T15:48:59.843381Z","iopub.execute_input":"2025-02-16T15:48:59.843640Z","iopub.status.idle":"2025-02-16T15:49:07.236371Z","shell.execute_reply.started":"2025-02-16T15:48:59.843618Z","shell.execute_reply":"2025-02-16T15:49:07.235545Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ذخیره مدل‌ها\nmodel_t1.save(\"Vgg16_t1.keras\")\nmodel_t2.save(\"Vgg16_t2.keras\")\nmodel_t2stir.save(\"Vgg16_t2stir.keras\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-16T15:49:07.237152Z","iopub.execute_input":"2025-02-16T15:49:07.237401Z","iopub.status.idle":"2025-02-16T15:49:33.996745Z","shell.execute_reply.started":"2025-02-16T15:49:07.237369Z","shell.execute_reply":"2025-02-16T15:49:33.995641Z"}},"outputs":[],"execution_count":null}]}