{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"dockerImageVersionId":30887,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# # This Python 3 environment comes with many helpful analytics libraries installed\n# # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# # For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# # Input data files are available in the read-only \"../input/\" directory\n# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-02-16T23:40:56.208590Z","iopub.execute_input":"2025-02-16T23:40:56.208975Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport pydicom\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models, optimizers, callbacks\nfrom sklearn.model_selection import train_test_split\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import confusion_matrix, classification_report","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ------------------------------\n# 1. بارگذاری فایل‌های CSV و تعریف مسیرها\n# ------------------------------\ntrain_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/'\n\ntrain_df      = pd.read_csv(os.path.join(train_path, 'train.csv'))\nlabel_df      = pd.read_csv(os.path.join(train_path, 'train_label_coordinates.csv'))\ntrain_desc_df = pd.read_csv(os.path.join(train_path, 'train_series_descriptions.csv'))\ntest_desc_df  = pd.read_csv(os.path.join(train_path, 'test_series_descriptions.csv'))\nsub           = pd.read_csv(os.path.join(train_path, 'sample_submission.csv'))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ------------------------------\n# 2. تابع برای تولید مسیرهای تصاویر\n# ------------------------------\ndef generate_image_paths(df, data_dir):\n    image_paths = []\n    for study_id, series_id in zip(df['study_id'], df['series_id']):\n        study_dir = os.path.join(data_dir, str(study_id))\n        series_dir = os.path.join(study_dir, str(series_id))\n        if os.path.exists(series_dir):\n            images = os.listdir(series_dir)\n            image_paths.extend([os.path.join(series_dir, img) for img in images])\n    return image_paths\n\ntrain_image_paths = generate_image_paths(train_desc_df, os.path.join(train_path, 'train_images'))\ntest_image_paths  = generate_image_paths(test_desc_df, os.path.join(train_path, 'test_images'))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ------------------------------\n# 3. تغییر شکل داده‌های train و ادغام دیتا فریم‌ها\n# ------------------------------\ndef reshape_row(row):\n    data = {'study_id': [], 'condition': [], 'level': [], 'severity': []}\n    for column, value in row.items():\n        if column not in ['study_id', 'series_id', 'instance_number', 'x', 'y', 'series_description']:\n            parts = column.split('_')\n            condition = ' '.join([word.capitalize() for word in parts[:-2]])\n            level = parts[-2].capitalize() + '/' + parts[-1].capitalize()\n            data['study_id'].append(row['study_id'])\n            data['condition'].append(condition)\n            data['level'].append(level)\n            data['severity'].append(value)\n    return pd.DataFrame(data)\n\nnew_train_df = pd.concat([reshape_row(row) for _, row in train_df.iterrows()], ignore_index=True)\n\nmerged_df = pd.merge(new_train_df, label_df, on=['study_id', 'condition', 'level'], how='inner')\nfinal_merged_df = pd.merge(merged_df, train_desc_df, on=['series_id','study_id'], how='inner')\n\nfinal_merged_df['row_id'] = (final_merged_df['study_id'].astype(str) + '_' +\n                               final_merged_df['condition'].str.lower().str.replace(' ', '_') + '_' +\n                               final_merged_df['level'].str.lower().str.replace('/', '_'))\n\nfinal_merged_df['image_path'] = (os.path.join(train_path, 'train_images') + '/' +\n                                 final_merged_df['study_id'].astype(str) + '/' +\n                                 final_merged_df['series_id'].astype(str) + '/' +\n                                 final_merged_df['instance_number'].astype(str) + '.dcm')\n\n# تغییر برچسب severity به حروف کوچک\nfinal_merged_df['severity'] = final_merged_df['severity'].map({\n    'Normal/Mild': 'normal_mild',\n    'Moderate': 'moderate',\n    'Severe': 'severe'\n})\n\n# فیلتر کردن ردیف‌هایی که مسیر تصویر موجود است\ndef check_exists(path):\n    return os.path.exists(path)\nfinal_merged_df = final_merged_df[final_merged_df['image_path'].apply(check_exists)]\n\n# نگاشت برچسب‌ها به اعداد صحیح\nseverity_map = {'normal_mild': 0, 'moderate': 1, 'severe': 2}\nfinal_merged_df['severity'] = final_merged_df['severity'].map(severity_map)\n\n# استفاده از final_merged_df به عنوان داده‌های آموزشی\ntrain_data = final_merged_df.copy()\ntrain_data = train_data.dropna()   # حذف ردیف‌های دارای NaN\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ------------------------------\n# 4. تنظیم وزن کلاس‌ها به مقادیر دلخواه\n# ------------------------------\n# تنظیم وزن‌ها به صورت دستی\nnormalized_weights = {\n    0: 0.142,   # normal_mild\n    1: 0.2857,  # moderate\n    2: 0.571    # severe\n}\n\nprint(\"وزن کلاس‌ها:\", normalized_weights)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ------------------------------\n# 5. توابع بارگذاری و پیش‌پردازش تصاویر DICOM\n# ------------------------------\ndef load_dicom_image(path):\n    \"\"\"\n    تابعی برای بارگذاری تصویر DICOM\n    \"\"\"\n    path = path.numpy().decode('utf-8')  # تبدیل EagerTensor به رشته\n    ds = pydicom.dcmread(path)\n    data = ds.pixel_array.astype(np.float32)\n    data = data - np.min(data)\n    if np.max(data) != 0:\n        data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data\n\ndef load_and_preprocess(path, label=None):\n    \"\"\"\n    - بارگذاری تصویر DICOM با استفاده از tf.py_function\n    - افزودن بعد کانال (برای تصاویر خاکستری)\n    - تغییر اندازه به 224x224، تبدیل از grayscale به RGB و نرمال‌سازی به [0, 1]\n    \"\"\"\n    image = tf.py_function(func=lambda p: load_dicom_image(p), inp=[path], Tout=tf.uint8)\n    image.set_shape([None, None])\n    image = tf.expand_dims(image, axis=-1)  # تبدیل (ارتفاع, عرض) به (ارتفاع, عرض, 1)\n    image = tf.image.resize(image, [224, 224])\n    image = tf.image.grayscale_to_rgb(image)\n    image = tf.cast(image, tf.float32) / 255.0\n    if label is None:\n        return image\n    else:\n        return image, label\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ------------------------------\n# 6. ایجاد دیتاست‌های TensorFlow برای هر سری توضیحی\n# ------------------------------\ndef create_datasets(df, series_description, batch_size=8):\n    filtered_df = df[df['series_description'] == series_description]\n    if filtered_df.empty:\n        raise ValueError(f\"داده‌ای برای سری توضیحی: {series_description} پیدا نشد.\")\n    train_df_part, val_df_part = train_test_split(filtered_df, test_size=0.2, random_state=42)\n    \n    train_paths = train_df_part['image_path'].values\n    train_labels = train_df_part['severity'].values\n    val_paths = val_df_part['image_path'].values\n    val_labels = val_df_part['severity'].values\n    \n    train_ds = tf.data.Dataset.from_tensor_slices((train_paths, train_labels))\n    train_ds = train_ds.map(lambda p, l: load_and_preprocess(p, l),\n                            num_parallel_calls=tf.data.AUTOTUNE)\n    train_ds = train_ds.shuffle(buffer_size=len(train_df_part)).batch(batch_size).prefetch(tf.data.AUTOTUNE)\n    \n    val_ds = tf.data.Dataset.from_tensor_slices((val_paths, val_labels))\n    val_ds = val_ds.map(lambda p, l: load_and_preprocess(p, l),\n                        num_parallel_calls=tf.data.AUTOTUNE)\n    val_ds = val_ds.batch(batch_size).prefetch(tf.data.AUTOTUNE)\n    \n    return train_ds, val_ds, len(train_df_part), len(val_df_part)\n\n# ایجاد دیتاست‌ها برای سه سری توضیحی\ntrain_ds_t1, val_ds_t1, len_train_t1, len_val_t1 = create_datasets(train_data, 'Sagittal T1', batch_size=8)\ntrain_ds_t2, val_ds_t2, len_train_t2, len_val_t2 = create_datasets(train_data, 'Axial T2', batch_size=8)\ntrain_ds_t2stir, val_ds_t2stir, len_train_t2stir, len_val_t2stir = create_datasets(train_data, 'Sagittal T2/STIR', batch_size=8)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ------------------------------\n# 7. تعریف مدل VGG16 با استفاده از Keras (TensorFlow)\n# ------------------------------\ndef create_vgg16_model(num_classes=3):\n    base_model = tf.keras.applications.VGG16(include_top=False,\n                                             input_shape=(224, 224, 3),\n                                             weights='imagenet')\n    base_model.trainable = False\n    x = layers.Flatten()(base_model.output)\n    x = layers.Dense(4096, activation='relu')(x)\n    x = layers.Dense(4096, activation='relu')(x)\n    outputs = layers.Dense(num_classes, activation='softmax')(x)\n    model = tf.keras.Model(inputs=base_model.input, outputs=outputs)\n    return model\n\n# ایجاد سه مدل مجزا برای سری‌های مختلف\nmodel_t1 = create_vgg16_model(num_classes=3)\nmodel_t2 = create_vgg16_model(num_classes=3)\nmodel_t2stir = create_vgg16_model(num_classes=3)\n\nmodel_t1.compile(optimizer=optimizers.Adam(learning_rate=0.0001),\n                 loss='sparse_categorical_crossentropy',\n                 metrics=['accuracy'])\nmodel_t2.compile(optimizer=optimizers.Adam(learning_rate=0.0001),\n                 loss='sparse_categorical_crossentropy',\n                 metrics=['accuracy'])\nmodel_t2stir.compile(optimizer=optimizers.Adam(learning_rate=0.0001),\n                     loss='sparse_categorical_crossentropy',\n                     metrics=['accuracy'])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\n# ------------------------------\n# 8. آموزش مدل‌ها با وزن‌دهی کلاس‌ها\n# ------------------------------\nes_callback_t1 = callbacks.EarlyStopping(monitor='val_accuracy', patience=3, restore_best_weights=True)\nckpt_callback_t1 = callbacks.ModelCheckpoint('best_model_t1.keras', monitor='val_accuracy', save_best_only=True)\n\nes_callback_t2 = callbacks.EarlyStopping(monitor='val_accuracy', patience=3, restore_best_weights=True)\nckpt_callback_t2 = callbacks.ModelCheckpoint('best_model_t2.keras', monitor='val_accuracy', save_best_only=True)\n\nes_callback_t2stir = callbacks.EarlyStopping(monitor='val_accuracy', patience=3, restore_best_weights=True)\nckpt_callback_t2stir = callbacks.ModelCheckpoint('best_model_t2stir.keras', monitor='val_accuracy', save_best_only=True)\n\n# آموزش مدل‌ها\nhistory_t1 = model_t1.fit(train_ds_t1, epochs=50, validation_data=val_ds_t1,\n                          callbacks=[es_callback_t1, ckpt_callback_t1],\n                          class_weight=normalized_weights)\n\nhistory_t2 = model_t2.fit(train_ds_t2, epochs=50, validation_data=val_ds_t2,\n                          callbacks=[es_callback_t2, ckpt_callback_t2],\n                          class_weight=normalized_weights)\n\nhistory_t2stir = model_t2stir.fit(train_ds_t2stir, epochs=50, validation_data=val_ds_t2stir,\n                                  callbacks=[es_callback_t2stir, ckpt_callback_t2stir],\n                                  class_weight=normalized_weights)\n\n\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.idle":"2025-02-16T23:57:35.626411Z","shell.execute_reply.started":"2025-02-16T23:43:56.174648Z","shell.execute_reply":"2025-02-16T23:57:35.625351Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ------------------------------\n# 9. رسم نمودارهای تاریخچه آموزش و محاسبه Confusion Matrix برای هر مدل\n# ------------------------------\ndef plot_history_and_confusion(model, history, val_ds, model_name):\n    # رسم نمودار loss و accuracy\n    plt.figure(figsize=(12,5))\n    \n    plt.subplot(1,2,1)\n    plt.plot(history.history['loss'], label='Train Loss')\n    plt.plot(history.history['val_loss'], label='Validation Loss')\n    plt.title(f'{model_name} Loss')\n    plt.xlabel('Epochs')\n    plt.ylabel('Loss')\n    plt.legend()\n    \n    plt.subplot(1,2,2)\n    plt.plot(history.history['accuracy'], label='Train Accuracy')\n    plt.plot(history.history['val_accuracy'], label='Validation Accuracy')\n    plt.title(f'{model_name} Accuracy')\n    plt.xlabel('Epochs')\n    plt.ylabel('Accuracy')\n    plt.legend()\n    plt.show()\n    \n    # محاسبه پیش‌بینی‌ها روی داده‌های اعتبارسنجی\n    y_true = []\n    y_pred = []\n    for images, labels in val_ds:\n        preds = model.predict(images)\n        y_pred.extend(np.argmax(preds, axis=1))\n        y_true.extend(labels.numpy())\n    \n    cm = confusion_matrix(y_true, y_pred)\n    plt.figure(figsize=(8,6))\n    sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')\n    plt.title(f'Confusion Matrix for {model_name}')\n    plt.xlabel('Predicted')\n    plt.ylabel('True')\n    plt.show()\n    \n    print(f'Classification Report for {model_name}:\\n', classification_report(y_true, y_pred))\n\n# رسم نمودارها و confusion matrix برای هر مدل\nplot_history_and_confusion(model_t1, history_t1, val_ds_t1, \"VGG16 - Sagittal T1\")\nplot_history_and_confusion(model_t2, history_t2, val_ds_t2, \"VGG16 - Axial T2\")\nplot_history_and_confusion(model_t2stir, history_t2stir, val_ds_t2stir, \"VGG16 - Sagittal T2/STIR\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-16T23:57:35.628108Z","iopub.execute_input":"2025-02-16T23:57:35.628503Z","iopub.status.idle":"2025-02-17T00:01:11.131026Z","shell.execute_reply.started":"2025-02-16T23:57:35.628466Z","shell.execute_reply":"2025-02-17T00:01:11.130119Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ------------------------------\n# 10. پیش‌بینی روی داده‌های Test و ایجاد سابمیشن\n# ------------------------------\n# دیکشنری مدل‌ها بر اساس سری توضیحی\nmodels_dict = {\n    'Sagittal T1': model_t1,\n    'Axial T2': model_t2,\n    'Sagittal T2/STIR': model_t2stir\n}\n\n# پیش‌بینی برای داده‌های تست\nnormal_mild_probs = []\nmoderate_probs = []\nsevere_probs = []\npredictions_list = []\n\n# ایجاد دیتاست برای داده‌های تست\nexpanded_rows = []\nfor index, row in test_desc_df.iterrows():\n    study_id = row['study_id']\n    series_id = row['series_id']\n    series_description = row['series_description']\n    series_path = os.path.join(train_path, 'test_images', str(study_id), str(series_id))\n    if os.path.exists(series_path):\n        image_files = [os.path.join(series_path, f) for f in os.listdir(series_path)\n                       if os.path.isfile(os.path.join(series_path, f))]\n        for image_path in image_files:\n            expanded_rows.append({\n                'study_id': study_id,\n                'series_id': series_id,\n                'series_description': series_description,\n                'image_path': image_path,\n                'row_id': f\"{study_id}_{series_id}\"\n            })\n\nexpanded_test_desc = pd.DataFrame(expanded_rows)\n\n# ایجاد دیتاست تست (بدون برچسب)\ntest_paths = expanded_test_desc['image_path'].values\ntest_ds = tf.data.Dataset.from_tensor_slices(test_paths)\ntest_ds = test_ds.map(lambda p: load_and_preprocess(p),\n                      num_parallel_calls=tf.data.AUTOTUNE)\ntest_ds = test_ds.batch(1)\n\nfor i, batch in enumerate(tqdm(test_ds)):\n    series_description = expanded_test_desc.iloc[i]['series_description']\n    model_used = models_dict.get(series_description, None)\n    if model_used is None:\n        normal_mild_probs.append(None)\n        moderate_probs.append(None)\n        severe_probs.append(None)\n        predictions_list.append(None)\n    else:\n        preds = model_used.predict(batch)\n        preds = preds[0]\n        normal_mild_probs.append(preds[0])\n        moderate_probs.append(preds[1])\n        severe_probs.append(preds[2])\n        predictions_list.append(preds)\n\n# اضافه کردن نتایج پیش‌بینی به دیتافریم تست\nexpanded_test_desc['normal_mild'] = normal_mild_probs\nexpanded_test_desc['moderate'] = moderate_probs\nexpanded_test_desc['severe'] = severe_probs\n\n# ترکیب نتایج پیش‌بینی‌ها\nsubmission_df = expanded_test_desc[[\"row_id\", \"normal_mild\", \"moderate\", \"severe\"]]\ngrouped_submission = submission_df.groupby('row_id').max().reset_index()\n\n# ذخیره سابمیشن\nsub[['normal_mild', 'moderate', 'severe']] = grouped_submission[['normal_mild', 'moderate', 'severe']]\nsub.to_csv(\"/kaggle/working/submission.csv\", index=False)\n\nprint(\"نمونه سابمیشن:\")\nprint(sub.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-17T00:01:11.131993Z","iopub.execute_input":"2025-02-17T00:01:11.132352Z","iopub.status.idle":"2025-02-17T00:01:20.586946Z","shell.execute_reply.started":"2025-02-17T00:01:11.132308Z","shell.execute_reply":"2025-02-17T00:01:20.585746Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ذخیره مدل‌ها\nmodel_t1.save(\"Vgg16_t1.keras\")\nmodel_t2.save(\"Vgg16_t2.keras\")\nmodel_t2stir.save(\"Vgg16_t2stir.keras\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-17T00:01:20.588213Z","iopub.execute_input":"2025-02-17T00:01:20.588635Z","iopub.status.idle":"2025-02-17T00:01:43.355657Z","shell.execute_reply.started":"2025-02-17T00:01:20.588594Z","shell.execute_reply":"2025-02-17T00:01:43.354555Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}