{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# # This Python 3 environment comes with many helpful analytics libraries installed\n# # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# # For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# # Input data files are available in the read-only \"../input/\" directory\n# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install tensorflow==2.15.0 albumentations pydicom pandas numpy scikit-learn matplotlib seaborn","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport pydicom\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models, callbacks, optimizers\nfrom tensorflow.keras.applications import EfficientNetB1\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix, roc_curve, auc\nfrom sklearn.utils.class_weight import compute_class_weight\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom datetime import datetime\nimport albumentations as A\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\n# Verify pandas installation\ntry:\n    print(f\"Pandas version: {pd.__version__}\")\nexcept AttributeError as e:\n    raise ImportError(\"Pandas is not properly installed or imported. Please ensure pandas is installed using 'pip install pandas'.\") from e\n\n# Print session info\nprint(f\"Training started at: {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')} UTC\")\nprint(f\"User: masoudshahrian\")\n\n# Create output directories\nos.makedirs('model_results', exist_ok=True)\nos.makedirs('saved_models', exist_ok=True)\nos.makedirs('logs', exist_ok=True)\n\n# Set random seeds\nnp.random.seed(42)\ntf.random.set_seed(42)\n\n# Enable mixed precision\ntf.keras.mixed_precision.set_global_policy('mixed_bfloat16')\n\n# Define dataset path\ntrain_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/'\n\ndef load_and_process_data(train_path):\n    print(\"Loading and processing data...\")\n    \n    # Verify CSV files exist\n    for file in ['train.csv', 'train_label_coordinates.csv', 'train_series_descriptions.csv']:\n        file_path = os.path.join(train_path, file)\n        if not os.path.exists(file_path):\n            raise FileNotFoundError(f\"Required file not found: {file_path}\")\n    \n    train = pd.read_csv(os.path.join(train_path, 'train.csv'))\n    label = pd.read_csv(os.path.join(train_path, 'train_label_coordinates.csv'))\n    train_desc = pd.read_csv(os.path.join(train_path, 'train_series_descriptions.csv'))\n    \n    def reshape_row(row):\n        data = {'study_id': [], 'condition': [], 'level': [], 'severity': []}\n        for column, value in row.items():\n            if column != 'study_id':\n                parts = column.split('_')\n                condition = ' '.join([word.capitalize() for word in parts[:-2]])\n                level = parts[-2].capitalize() + '/' + parts[-1].capitalize()\n                data['study_id'].append(row['study_id'])\n                data['condition'].append(condition)\n                data['level'].append(level)\n                data['severity'].append(value)\n        return pd.DataFrame(data)\n    \n    new_train_df = pd.concat([reshape_row(row) for _, row in train.iterrows()], ignore_index=True)\n    merged_df = pd.merge(new_train_df, label, on=['study_id', 'condition', 'level'], how='inner')\n    final_merged_df = pd.merge(merged_df, train_desc, on=['series_id', 'study_id'], how='inner')\n    \n    # Select multiple images per series (up to 3)\n    grouped = final_merged_df.groupby(['study_id', 'series_id', 'series_description'])\n    processed_data = []\n    for name, group in grouped:\n        selected_instances = group.sort_values('instance_number').head(3)\n        for _, row in selected_instances.iterrows():\n            processed_data.append(row)\n    \n    final_merged_df = pd.DataFrame(processed_data)\n    \n    final_merged_df['image_path'] = (\n        train_path + 'train_images/' +\n        final_merged_df['study_id'].astype(str) + '/' +\n        final_merged_df['series_id'].astype(str) + '/' +\n        final_merged_df['instance_number'].astype(str) + '.dcm'\n    )\n    \n    final_merged_df['severity'] = final_merged_df['severity'].map({\n        'Normal/Mild': 'normal_mild',\n        'Moderate': 'moderate',\n        'Severe': 'severe'\n    })\n    \n    valid_data = final_merged_df[final_merged_df['severity'].isin(['normal_mild', 'moderate', 'severe'])]\n    print(f\"Total valid samples: {len(valid_data)}\")\n    \n    return valid_data\n\n@tf.function\ndef load_dicom_tf(path):\n    try:\n        dicom = pydicom.dcmread(path.numpy().decode('utf-8'))\n        data = dicom.pixel_array\n        window_center = dicom.get('WindowCenter', 1000)\n        window_width = dicom.get('WindowWidth', 2000)\n        data = np.clip(data, window_center - window_width // 2, window_center + window_width // 2)\n        data = (data - (window_center - window_width // 2)) / window_width\n        data = np.expand_dims(data, axis=-1)\n        return data.astype(np.float32)\n    except Exception as e:\n        print(f\"Error loading DICOM: {e}\")\n        return np.zeros((224, 224, 1), dtype=np.float32)\n\ndef albumentations_augmentation(image):\n    transform = A.Compose([\n        A.RandomBrightnessContrast(p=0.5),\n        A.Rotate(limit=30, p=0.5),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.GaussNoise(p=0.3),\n        A.RandomGamma(p=0.3),\n    ])\n    augmented = transform(image=image)\n    return augmented['image']\n\n@tf.function\ndef preprocess_image(image_path, label, is_training=False):\n    image = tf.py_function(load_dicom_tf, [image_path], tf.float32)\n    image.set_shape([None, None, 1])\n    image = tf.image.resize(image, [224, 224])\n    image = tf.image.grayscale_to_rgb(image)\n    \n    if is_training:\n        image = tf.py_function(lambda x: albumentations_augmentation(x.numpy()), [image], tf.float32)\n        image.set_shape([224, 224, 3])\n    \n    image = tf.keras.applications.efficientnet.preprocess_input(image)\n    \n    # Ensure label is 1D\n    label = tf.squeeze(label)\n    return image, label\n\ndef compute_class_weights(df):\n    labels = df['severity'].map({'normal_mild': 0, 'moderate': 1, 'severe': 2}).values\n    class_weights = compute_class_weight('balanced', classes=np.unique(labels), y=labels)\n    return dict(zip(np.unique(labels), class_weights))\n\ndef create_dataset(df, batch_size=32, is_training=False):\n    # Oversampling for training data\n    if is_training:\n        class_counts = df['severity'].value_counts()\n        max_count = class_counts.max()\n        oversampled_dfs = []\n        for severity in class_counts.index:\n            class_df = df[df['severity'] == severity]\n            repeat_factor = int(max_count // len(class_df)) + 1\n            oversampled_dfs.append(pd.concat([class_df] * repeat_factor)[:max_count])\n        df = pd.concat(oversampled_dfs).sample(frac=1).reset_index(drop=True)\n    \n    # Create labels after oversampling\n    labels = df['severity'].map({'normal_mild': 0, 'moderate': 1, 'severe': 2}).astype(np.int32)\n    \n    # Verify lengths match\n    if len(df['image_path']) != len(labels):\n        raise ValueError(f\"Mismatch in lengths: image_path ({len(df['image_path'])}) vs labels ({len(labels)})\")\n    \n    # Ensure labels are 1D\n    labels = np.squeeze(labels)\n    print(f\"Labels shape before dataset creation: {labels.shape}\")\n    \n    dataset = tf.data.Dataset.from_tensor_slices((df['image_path'], labels))\n    \n    dataset = dataset.map(\n        lambda x, y: preprocess_image(x, y, is_training),\n        num_parallel_calls=tf.data.AUTOTUNE\n    )\n    \n    if is_training:\n        dataset = dataset.shuffle(1000)\n        dataset = dataset.repeat()\n    \n    dataset = dataset.batch(batch_size)\n    dataset = dataset.prefetch(tf.data.AUTOTUNE)\n    \n    # Debug dataset shapes\n    for image, label in dataset.take(1):\n        print(f\"Batch image shape: {image.shape}\")\n        print(f\"Batch label shape: {label.shape}\")\n    \n    return dataset\n\ndef build_unet_encoder(input_shape=(224, 224, 3)):\n    inputs = layers.Input(shape=input_shape)\n    \n    # Encoder Block 1\n    c1 = layers.Conv2D(64, (3, 3), activation='relu', padding='same')(inputs)  # (224, 224, 64)\n    c1 = layers.Conv2D(64, (3, 3), activation='relu', padding='same')(c1)     # (224, 224, 64)\n    p1 = layers.MaxPooling2D((2, 2))(c1)                                      # (112, 112, 64)\n    \n    # Encoder Block 2\n    c2 = layers.Conv2D(128, (3, 3), activation='relu', padding='same')(p1)    # (112, 112, 128)\n    c2 = layers.Conv2D(128, (3, 3), activation='relu', padding='same')(c2)    # (112, 112, 128)\n    p2 = layers.MaxPooling2D((2, 2))(c2)                                      # (56, 56, 128)\n    \n    # Encoder Block 3\n    c3 = layers.Conv2D(256, (3, 3), activation='relu', padding='same')(p2)    # (56, 56, 256)\n    c3 = layers.Conv2D(256, (3, 3), activation='relu', padding='same')(c3)    # (56, 56, 256)\n    p3 = layers.MaxPooling2D((2, 2))(c3)                                      # (28, 28, 256)\n    \n    # Encoder Block 4\n    c4 = layers.Conv2D(512, (3, 3), activation='relu', padding='same')(p3)    # (28, 28, 512)\n    c4 = layers.Conv2D(512, (3, 3), activation='relu', padding='same')(c4)    # (28, 28, 512)\n    p4 = layers.MaxPooling2D((2, 2))(c4)                                      # (14, 14, 512)\n    \n    # Encoder Block 5\n    c5 = layers.Conv2D(1024, (3, 3), activation='relu', padding='same')(p4)   # (14, 14, 1024)\n    c5 = layers.Conv2D(1024, (3, 3), activation='relu', padding='same')(c5)   # (14, 14, 1024)\n    p5 = layers.MaxPooling2D((2, 2))(c5)                                      # (7, 7, 1024)\n    \n    return models.Model(inputs, p5, name='unet_encoder')\n\ndef build_hybrid_model(input_shape=(224, 224, 3), num_classes=3):\n    unet_encoder = build_unet_encoder(input_shape)\n    efficientnet = EfficientNetB1(\n        include_top=False,\n        weights='imagenet',\n        input_shape=input_shape\n    )\n    \n    for layer in efficientnet.layers:\n        if 'block6' in layer.name or 'block7' in layer.name:\n            layer.trainable = True\n        else:\n            layer.trainable = False\n    \n    inputs = layers.Input(shape=input_shape)  # (224, 224, 3)\n    unet_features = unet_encoder(inputs)     # (7, 7, 1024)\n    \n    adapted_features = layers.Conv2D(3, (1, 1), padding='same')(unet_features)  # (7, 7, 3)\n    adapted_features = layers.BatchNormalization()(adapted_features)            # (7, 7, 3)\n    adapted_features = layers.ReLU()(adapted_features)                          # (7, 7, 3)\n    adapted_features = layers.UpSampling2D(size=(4, 4))(adapted_features)       # (28, 28, 3)\n    adapted_features = layers.UpSampling2D(size=(4, 4))(adapted_features)       # (112, 112, 3)\n    adapted_features = layers.UpSampling2D(size=(2, 2))(adapted_features)       # (224, 224, 3)\n    \n    # Debug shapes\n    print(f\"Inputs shape: {inputs.shape}\")\n    print(f\"Adapted features shape: {adapted_features.shape}\")\n    \n    combined_input = layers.Add()([inputs, adapted_features])  # (224, 224, 3)\n    eff_features = efficientnet(combined_input)\n    \n    attention = layers.Conv2D(eff_features.shape[-1], (1, 1), activation='sigmoid')(eff_features)\n    eff_features = layers.Multiply()([eff_features, attention])\n    \n    x = layers.GlobalAveragePooling2D()(eff_features)\n    x = layers.Dense(512, activation='relu')(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Dropout(0.5)(x)\n    x = layers.Dense(256, activation='relu')(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Dropout(0.3)(x)\n    outputs = layers.Dense(num_classes)(x)\n    \n    return models.Model(inputs, outputs, name='hybrid_unet_efficientnet')\n\ndef focal_loss(gamma=2., alpha=4., class_weights=None):\n    def focal_loss_with_logits(y_true, y_pred):\n        # Ensure y_true is 1D\n        y_true = tf.squeeze(y_true)\n        y_true = tf.cast(y_true, tf.int32)\n        \n        ce = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y_true, logits=y_pred)\n        probs = tf.nn.softmax(y_pred)\n        probs = tf.gather(probs, y_true, batch_dims=1)\n        \n        if class_weights is not None:\n            # Use compute dtype from mixed precision policy\n            compute_dtype = tf.keras.mixed_precision.global_policy().compute_dtype\n            weights = tf.gather(\n                tf.constant(list(class_weights.values()), dtype=compute_dtype), y_true\n            )\n            ce = ce * weights\n        \n        return tf.reduce_mean(alpha * tf.pow(1. - probs, gamma) * ce)\n    return focal_loss_with_logits\n\ndef train_model(model, train_ds, val_ds, series_name, epochs=5, steps_per_epoch=100):\n    optimizer = optimizers.Adam(learning_rate=1e-4)\n    \n    global train_df\n    class_weights = compute_class_weights(train_df)\n    \n    model.compile(\n        optimizer=optimizer,\n        loss=focal_loss(class_weights=class_weights),\n        metrics=['accuracy']\n    )\n    \n    log_dir = f\"logs/{series_name}_{datetime.utcnow().strftime('%Y%m%d-%H%M%S')}\"\n    \n    lr_schedule = tf.keras.experimental.CosineDecay(\n        initial_learning_rate=1e-4,\n        decay_steps=epochs * steps_per_epoch,\n        alpha=1e-6\n    )\n    \n    callbacks_list = [\n        callbacks.EarlyStopping(monitor='val_loss', patience=15, restore_best_weights=True),\n        callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5, min_lr=1e-6),\n        callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1),\n        callbacks.LearningRateScheduler(lr_schedule)\n    ]\n    \n    history = model.fit(\n        train_ds,\n        validation_data=val_ds,\n        epochs=epochs,\n        steps_per_epoch=steps_per_epoch,\n        validation_steps=30,\n        callbacks=callbacks_list,\n        verbose=1\n    )\n    \n    return history\n\ndef evaluate_model(model, test_ds, class_names):\n    y_true = []\n    y_pred = []\n    y_pred_probs = []\n    \n    for images, labels in test_ds:\n        predictions = model.predict(images)\n        y_pred_probs.extend(tf.nn.softmax(predictions).numpy())\n        y_pred.extend(np.argmax(predictions, axis=1))\n        y_true.extend(labels.numpy())\n    \n    y_true = np.array(y_true)\n    y_pred = np.array(y_pred)\n    y_pred_probs = np.array(y_pred_probs)\n    \n    cm = confusion_matrix(y_true, y_pred)\n    \n    class_metrics = {}\n    for i, class_name in enumerate(class_names):\n        true_class = (y_true == i)\n        pred_class = (y_pred == i)\n        \n        tp = np.sum((true_class) & (pred_class))\n        fp = np.sum((~true_class) & (pred_class))\n        fn = np.sum((true_class) & (~pred_class))\n        tn = np.sum((~true_class) & (~pred_class))\n        \n        sensitivity = tp / (tp + fn) if (tp + fn) > 0 else 0\n        specificity = tn / (tn + fp) if (tn + fp) > 0 else 0\n        precision = tp / (tp + fp) if (tp + fp) > 0 else 0\n        f1 = 2 * (precision * sensitivity) / (precision + sensitivity) if (precision + sensitivity) > 0 else 0\n        \n        class_metrics[class_name] = {\n            'TP': int(tp),\n            'FP': int(fp),\n            'FN': int(fn),\n            'TN': int(tn),\n            'Sensitivity': sensitivity,\n            'Specificity': specificity,\n            'Precision': precision,\n            'F1-Score': f1\n        }\n    \n    overall_metrics = {\n        'accuracy': np.mean(y_pred == y_true),\n        'weighted_precision': precision_score(y_true, y_pred, average='weighted'),\n        'weighted_recall': recall_score(y_true, y_pred, average='weighted'),\n        'weighted_f1': f1_score(y_true, y_pred, average='weighted')\n    }\n    \n    errors = []\n    for i, (y_t, y_p) in enumerate(zip(y_true, y_pred)):\n        if y_t != y_p:\n            errors.append({\n                'True': class_names[y_t],\n                'Predicted': class_names[y_p],\n                'Probability': y_pred_probs[i].max()\n            })\n    \n    error_df = pd.DataFrame(errors)\n    \n    return {\n        'confusion_matrix': cm,\n        'class_metrics': class_metrics,\n        'overall_metrics': overall_metrics,\n        'predictions': y_pred_probs,\n        'y_true': y_true,\n        'error_analysis': error_df\n    }\n\ndef plot_results(history, results, class_names, series_name):\n    output_dir = 'model_results'\n    os.makedirs(output_dir, exist_ok=True)\n    \n    fig = plt.figure(figsize=(20, 15))\n    \n    # Training History - Loss\n    plt.subplot(2, 2, 1)\n    plt.plot(history.history['loss'], label='Train Loss')\n    plt.plot(history.history['val_loss'], label='Val Loss')\n    plt.title(f'{series_name} - Training Loss', pad=20)\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.legend()\n    plt.grid(True)\n    \n    # Training History - Accuracy\n    plt.subplot(2, 2, 2)\n    plt.plot(history.history['accuracy'], label='Train Acc')\n    plt.plot(history.history['val_accuracy'], label='Val Acc')\n    plt.title(f'{series_name} - Training Accuracy', pad=20)\n    plt.xlabel('Epoch')\n    plt.ylabel('Accuracy')\n    plt.legend()\n    plt.grid(True)\n    \n    # Confusion Matrix with percentages\n    plt.subplot(2, 2, 3)\n    cm = results['confusion_matrix']\n    cm_sum = np.sum(cm, axis=1, keepdims=True)\n    cm_perc = cm / cm_sum * 100\n    \n    sns.heatmap(cm, annot=np.array([[f'{int(x)}\\n({y:.1f}%)' for x, y in zip(row_true, row_perc)] \n                                   for row_true, row_perc in zip(cm, cm_perc)]),\n                fmt='', cmap='Blues', xticklabels=class_names, yticklabels=class_names)\n    plt.title(f'{series_name} - Confusion Matrix\\nwith Counts and Percentages', pad=20)\n    plt.xlabel('Predicted')\n    plt.ylabel('True')\n    \n    # ROC Curve\n    plt.subplot(2, 2, 4)\n    y_true_one_hot = tf.keras.utils.to_categorical(results['y_true'], num_classes=len(class_names))\n    y_pred_probs = results['predictions']\n    \n    for i, class_name in enumerate(class_names):\n        fpr, tpr, _ = roc_curve(y_true_one_hot[:, i], y_pred_probs[:, i])\n        roc_auc = auc(fpr, tpr)\n        plt.plot(fpr, tpr, label=f'{class_name} (AUC = {roc_auc:.2f})')\n    \n    plt.plot([0, 1], [0, 1], 'k--')\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title(f'{series_name} - ROC Curve')\n    plt.legend(loc='lower right')\n    plt.grid(True)\n    \n    plt.tight_layout(h_pad=1.0, w_pad=1.0)\n    safe_series_name = series_name.lower().replace(\"/\", \"_\").replace(\" \", \"_\")\n    output_file = os.path.join(output_dir, f'{safe_series_name}_detailed_results.png')\n    plt.savefig(output_file, dpi=300, bbox_inches='tight')\n    plt.close()\n    \n    # Error Analysis\n    print(\"\\nError Analysis:\")\n    print(results['error_analysis'].groupby(['True', 'Predicted']).size().unstack(fill_value=0))\n    \n    # Print detailed metrics\n    print(f\"\\nDetailed Results for {series_name}:\")\n    print(\"=\" * 50)\n    print(\"Overall Metrics:\")\n    for metric, value in results['overall_metrics'].items():\n        print(f\"{metric}: {value:.4f}\")\n    \n    print(\"\\nPer-Class Metrics:\")\n    for class_name in class_names:\n        print(f\"\\n{class_name}:\")\n        metrics = results['class_metrics'][class_name]\n        print(f\"TP: {metrics['TP']}, FP: {metrics['FP']}, FN: {metrics['FN']}, TN: {metrics['TN']}\")\n        print(f\"Sensitivity: {metrics['Sensitivity']:.4f}\")\n        print(f\"Specificity: {metrics['Specificity']:.4f}\")\n        print(f\"Precision: {metrics['Precision']:.4f}\")\n        print(f\"F1-Score: {metrics['F1-Score']:.4f}\")\n\ndef main():\n    global train_df\n    final_merged_df = load_and_process_data(train_path)\n    \n    series_types = ['Sagittal T1', 'Axial T2', 'Sagittal T2/STIR']\n    class_names = ['normal_mild', 'moderate', 'severe']\n    \n    for series_name in series_types:\n        print(f\"\\nProcessing {series_name}\")\n        \n        series_df = final_merged_df[final_merged_df['series_description'] == series_name].copy()\n        if series_df.empty:\n            print(f\"No data for {series_name}\")\n            continue\n        \n        train_df, temp_df = train_test_split(series_df, test_size=0.3, stratify=series_df['severity'])\n        val_df, test_df = train_test_split(temp_df, test_size=0.5, stratify=temp_df['severity'])\n        \n        print(f\"\\nDataset splits for {series_name}:\")\n        print(f\"Train samples: {len(train_df)}\")\n        print(f\"Validation samples: {len(val_df)}\")\n        print(f\"Test samples: {len(test_df)}\")\n        \n        train_ds = create_dataset(train_df, batch_size=32, is_training=True)\n        val_ds = create_dataset(val_df, batch_size=32)\n        test_ds = create_dataset(test_df, batch_size=32)\n        \n        model = build_hybrid_model()\n        history = train_model(model, train_ds, val_ds, series_name)\n        \n        results = evaluate_model(model, test_ds, class_names)\n        plot_results(history, results, class_names, series_name)\n        \n        safe_series_name = series_name.lower().replace(\"/\", \"_\").replace(\" \", \"_\")\n        model_file = os.path.join('saved_models', f'{safe_series_name}_model.h5')\n        model.save(model_file)\n        \n        print(f\"Completed {series_name} at: {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')} UTC\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import os\n# import numpy as np\n# import pandas as pd\n# import cv2\n# import pydicom\n# import tensorflow as tf\n# from tensorflow.keras import layers, models, callbacks\n# from tensorflow.keras.applications import EfficientNetB1\n# from sklearn.model_selection import train_test_split\n# from sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix, roc_curve, auc\n# from sklearn.utils.class_weight import compute_class_weight\n# import matplotlib.pyplot as plt\n# import seaborn as sns\n# from datetime import datetime\n# import albumentations as A\n# from tensorflow_addons.optimizers import Lookahead\n# import tensorflow.keras.optimizers as optimizers","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\n# # Print session info\n# print(f\"Training started at: 2025-06-05 16:52:20 UTC\")\n# print(f\"User: masoudshahrian\")\n\n# # Create output directories\n# os.makedirs('model_results', exist_ok=True)\n# os.makedirs('saved_models', exist_ok=True)\n# os.makedirs('logs', exist_ok=True)\n\n# # Set random seeds\n# np.random.seed(42)\n# tf.random.set_seed(42)\n\n# # Enable mixed precision\n# tf.keras.mixed_precision.set_global_policy('mixed_bfloat16')\n\n# # Define dataset path\n# train_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/'\n\n# def load_and_process_data(train_path):\n#     print(\"Loading and processing data...\")\n    \n#     train = pd.read_csv(os.path.join(train_path, 'train.csv'))\n#     label = pd.read_csv(os.path.join(train_path, 'train_label_coordinates.csv'))\n#     train_desc = pd.read_csv(os.path.join(train_path, 'train_series_descriptions.csv'))\n    \n#     def reshape_row(row):\n#         data = {'study_id': [], 'condition': [], 'level': [], 'severity': []}\n#         for column, value in row.items():\n#             if column != 'study_id':\n#                 parts = column.split('_')\n#                 condition = ' '.join([word.capitalize() for word in parts[:-2]])\n#                 level = parts[-2].capitalize() + '/' + parts[-1].capitalize()\n#                 data['study_id'].append(row['study_id'])\n#                 data['condition'].append(condition)\n#                 data['level'].append(level)\n#                 data['severity'].append(value)\n#         return pd.DataFrame(data)\n    \n#     new_train_df = pd.concat([reshape_row(row) for _, row in train.iterrows()], ignore_index=True)\n#     merged_df = pd.merge(new_train_df, label, on=['study_id', 'condition', 'level'], how='inner')\n#     final_merged_df = pd.merge(merged_df, train_desc, on=['series_id', 'study_id'], how='inner')\n    \n#     # Select multiple images per series\n#     grouped = final_merged_df.groupby(['study_id', 'series_id', 'series_description'])\n#     processed_data = []\n#     for name, group in grouped:\n#         selected_instances = group.sort_values('instance_number').head(3)  # Take up to 3 images\n#         for _, row in selected_instances.iterrows():\n#             processed_data.append(row)\n    \n#     final_merged_df = pd.DataFrame(processed_data)\n    \n#     final_merged_df['image_path'] = (\n#         train_path + 'train_images/' +\n#         final_merged_df['study_id'].astype(str) + '/' +\n#         final_merged_df['series_id'].astype(str) + '/' +\n#         final_merged_df['instance_number'].astype(str) + '.dcm'\n#     )\n    \n#     final_merged_df['severity'] = final_merged_df['severity'].map({\n#         'Normal/Mild': 'normal_mild',\n#         'Moderate': 'moderate',\n#         'Severe': 'severe'\n#     })\n    \n#     valid_data = final_merged_df[final_merged_df['severity'].isin(['normal_mild', 'moderate', 'severe'])]\n#     print(f\"Total valid samples: {len(valid_data)}\")\n    \n#     return valid_data\n\n# @tf.function\n# def load_dicom_tf(path):\n#     try:\n#         dicom = pydicom.dcmread(path.numpy().decode('utf-8'))\n#         data = dicom.pixel_array\n#         window_center = dicom.get('WindowCenter', 1000)\n#         window_width = dicom.get('WindowWidth', 2000)\n#         data = np.clip(data, window_center - window_width // 2, window_center + window_width // 2)\n#         data = (data - (window_center - window_width // 2)) / window_width\n#         data = np.expand_dims(data, axis=-1)\n#         return data.astype(np.float32)\n#     except Exception as e:\n#         print(f\"Error loading DICOM: {e}\")\n#         return np.zeros((224, 224, 1), dtype=np.float32)\n\n# def albumentations_augmentation(image):\n#     transform = A.Compose([\n#         A.RandomBrightnessContrast(p=0.5),\n#         A.Rotate(limit=30, p=0.5),\n#         A.HorizontalFlip(p=0.5),\n#         A.VerticalFlip(p=0.5),\n#         A.GaussNoise(p=0.3),\n#         A.RandomGamma(p=0.3),\n#     ])\n#     augmented = transform(image=image)\n#     return augmented['image']\n\n# @tf.function\n# def preprocess_image(image_path, label=None, is_training=False):\n#     image = tf.py_function(load_dicom_tf, [image_path], tf.float32)\n#     image.set_shape([None, None, 1])\n#     image = tf.image.resize(image, [224, 224])\n#     image = tf.image.grayscale_to_rgb(image)\n    \n#     if is_training:\n#         image = tf.py_function(lambda x: albumentations_augmentation(x.numpy()), [image], tf.float32)\n#         image.set_shape([224, 224, 3])\n    \n#     image = tf.keras.applications.efficientnet.preprocess_input(image)\n#     return (image, label) if label is not None else image\n\n# def compute_class_weights(df):\n#     labels = df['severity'].map({'normal_mild': 0, 'moderate': 1, 'severe': 2}).values\n#     class_weights = compute_class_weight('balanced', classes=np.unique(labels), y=labels)\n#     return dict(zip(np.unique(labels), class_weights))\n\n# def create_dataset(df, batch_size=32, is_training=False):\n#     labels = df['severity'].map({'normal_mild': 0, 'moderate': 1, 'severe': 2}).astype(np.int32)\n    \n#     if is_training:\n#         class_counts = df['severity'].value_counts()\n#         max_count = class_counts.max()\n#         oversampled_dfs = []\n#         for severity in class_counts.index:\n#             class_df = df[df['severity'] == severity]\n#             repeat_factor = int(max_count // len(class_df)) + 1\n#             oversampled_dfs.append(pd.concat([class_df] * repeat_factor)[:max_count])\n#         df = pd.concat(oversampled_dfs).sample(frac=1).reset_index(drop=True)\n    \n#     dataset = tf.data.Dataset.from_tensor_slices((df['image_path'], labels))\n    \n#     dataset = dataset.map(\n#         lambda x, y: preprocess_image(x, y, is_training),\n#         num_parallel_calls=tf.data.AUTOTUNE\n#     )\n    \n#     if is_training:\n#         dataset = dataset.shuffle(1000)\n#         dataset = dataset.repeat()\n    \n#     dataset = dataset.batch(batch_size)\n#     dataset = dataset.prefetch(tf.data.AUTOTUNE)\n    \n#     return dataset\n\n# def build_hybrid_model(input_shape=(224, 224, 3), num_classes=3):\n#     efficientnet = EfficientNetB1(\n#         include_top=False,\n#         weights='imagenet',\n#         input_shape=input_shape\n#     )\n    \n#     for layer in efficientnet.layers[:100]:\n#         layer.trainable = False\n    \n#     inputs = layers.Input(shape=input_shape)\n#     x = efficientnet(inputs)\n    \n#     attention = layers.Conv2D(x.shape[-1], 1, activation='sigmoid')(x)\n#     x = layers.Multiply()([x, attention])\n    \n#     x = layers.GlobalAveragePooling2D()(x)\n#     x = layers.Dense(512, activation='relu')(x)\n#     x = layers.BatchNormalization()(x)\n#     x = layers.Dropout(0.5)(x)\n#     x = layers.Dense(256, activation='relu')(x)\n#     x = layers.BatchNormalization()(x)\n#     x = layers.Dropout(0.3)(x)\n#     outputs = layers.Dense(num_classes)(x)\n    \n#     return models.Model(inputs, outputs)\n\n# def focal_loss(gamma=2., alpha=4., class_weights=None):\n#     def focal_loss_with_logits(y_true, y_pred):\n#         y_true = tf.cast(y_true, tf.int32)\n#         ce = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y_true, logits=y_pred)\n#         probs = tf.nn.softmax(y_pred)\n#         probs = tf.gather(probs, y_true, batch_dims=1)\n        \n#         if class_weights is not None:\n#             weights = tf.gather(tf.constant(list(class_weights.values()), dtype=tf.float32), y_true)\n#             ce = ce * weights\n        \n#         return tf.reduce_mean(alpha * tf.pow(1. - probs, gamma) * ce)\n#     return focal_loss_with_logits\n\n# def train_model(model, train_ds, val_ds, series_name, epochs=2, steps_per_epoch=100):\n#     base_optimizer = optimizers.Adam(learning_rate=1e-4)\n#     optimizer = Lookahead(base_optimizer)\n    \n#     # Compute class weights from the training data\n#     global train_df\n#     class_weights = compute_class_weights(train_df)\n    \n#     model.compile(\n#         optimizer=optimizer,\n#         loss=focal_loss(class_weights=class_weights),\n#         metrics=['accuracy']\n#     )\n    \n#     log_dir = f\"logs/{series_name}_{datetime.utcnow().strftime('%Y%m%d-%H%M%S')}\"\n    \n#     lr_schedule = tf.keras.experimental.CosineDecay(\n#         initial_learning_rate=1e-4,\n#         decay_steps=epochs * steps_per_epoch,\n#         alpha=1e-6\n#     )\n    \n#     callbacks_list = [\n#         callbacks.EarlyStopping(monitor='val_loss', patience=15, restore_best_weights=True),\n#         callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5, min_lr=1e-6),\n#         callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1),\n#         callbacks.LearningRateScheduler(lr_schedule)\n#     ]\n    \n#     history = model.fit(\n#         train_ds,\n#         validation_data=val_ds,\n#         epochs=epochs,\n#         steps_per_epoch=steps_per_epoch,\n#         validation_steps=30,\n#         callbacks=callbacks_list,\n#         verbose=1\n#     )\n    \n#     return history\n\n# def evaluate_model(model, test_ds, class_names):\n#     y_true = []\n#     y_pred = []\n#     y_pred_probs = []\n    \n#     for images, labels in test_ds:\n#         predictions = model.predict(images)\n#         y_pred_probs.extend(tf.nn.softmax(predictions).numpy())\n#         y_pred.extend(np.argmax(predictions, axis=1))\n#         y_true.extend(labels.numpy())\n    \n#     y_true = np.array(y_true)\n#     y_pred = np.array(y_pred)\n#     y_pred_probs = np.array(y_pred_probs)\n    \n#     cm = confusion_matrix(y_true, y_pred)\n    \n#     class_metrics = {}\n#     for i, class_name in enumerate(class_names):\n#         true_class = (y_true == i)\n#         pred_class = (y_pred == i)\n        \n#         tp = np.sum((true_class) & (pred_class))\n#         fp = np.sum((~true_class) & (pred_class))\n#         fn = np.sum((true_class) & (~pred_class))\n#         tn = np.sum((~true_class) & (~pred_class))\n        \n#         sensitivity = tp / (tp + fn) if (tp + fn) > 0 else 0\n#         specificity = tn / (tn + fp) if (tn + fp) > 0 else 0\n#         precision = tp / (tp + fp) if (tp + fp) > 0 else 0\n#         f1 = 2 * (precision * sensitivity) / (precision + sensitivity) if (precision + sensitivity) > 0 else 0\n        \n#         class_metrics[class_name] = {\n#             'TP': int(tp),\n#             'FP': int(fp),\n#             'FN': int(fn),\n#             'TN': int(tn),\n#             'Sensitivity': sensitivity,\n#             'Specificity': specificity,\n#             'Precision': precision,\n#             'F1-Score': f1\n#         }\n    \n#     overall_metrics = {\n#         'accuracy': np.mean(y_pred == y_true),\n#         'weighted_precision': precision_score(y_true, y_pred, average='weighted'),\n#         'weighted_recall': recall_score(y_true, y_pred, average='weighted'),\n#         'weighted_f1': f1_score(y_true, y_pred, average='weighted')\n#     }\n    \n#     return {\n#         'confusion_matrix': cm,\n#         'class_metrics': class_metrics,\n#         'overall_metrics': overall_metrics,\n#         'predictions': y_pred_probs,\n#         'y_true': y_true\n#     }\n\n# def plot_results(history, results, class_names, series_name):\n#     output_dir = 'model_results'\n#     os.makedirs(output_dir, exist_ok=True)\n    \n#     fig = plt.figure(figsize=(20, 15))\n    \n#     # Training History - Loss\n#     plt.subplot(2, 2, 1)\n#     plt.plot(history.history['loss'], label='Train Loss')\n#     plt.plot(history.history['val_loss'], label='Val Loss')\n#     plt.title(f'{series_name} - Training Loss', pad=20)\n#     plt.xlabel('Epoch')\n#     plt.ylabel('Loss')\n#     plt.legend()\n#     plt.grid(True)\n    \n#     # Training History - Accuracy\n#     plt.subplot(2, 2, 2)\n#     plt.plot(history.history['accuracy'], label='Train Acc')\n#     plt.plot(history.history['val_accuracy'], label='Val Acc')\n#     plt.title(f'{series_name} - Training Accuracy', pad=20)\n#     plt.xlabel('Epoch')\n#     plt.ylabel('Accuracy')\n#     plt.legend()\n#     plt.grid(True)\n    \n#     # Confusion Matrix with percentages\n#     plt.subplot(2, 2, 3)\n#     cm = results['confusion_matrix']\n#     cm_sum = np.sum(cm, axis=1, keepdims=True)\n#     cm_perc = cm / cm_sum * 100\n    \n#     sns.heatmap(cm, annot=np.array([[f'{int(x)}\\n({y:.1f}%)' for x, y in zip(row_true, row_perc)] \n#                                    for row_true, row_perc in zip(cm, cm_perc)]),\n#                 fmt='', cmap='Blues', xticklabels=class_names, yticklabels=class_names)\n#     plt.title(f'{series_name} - Confusion Matrix\\nwith Counts and Percentages', pad=20)\n#     plt.xlabel('Predicted')\n#     plt.ylabel('True')\n    \n#     # Metrics Table\n#     plt.subplot(2, 2, 4)\n#     plt.axis('off')\n    \n#     table_data = []\n#     table_colors = []\n#     metrics_display = ['Sensitivity', 'Specificity', 'Precision', 'F1-Score']\n    \n#     header = ['Metrics'] + class_names + ['Overall']\n#     table_data.append(header)\n#     table_colors.append(['lightgray'] * len(header))\n    \n#     for metric in metrics_display:\n#         row = [metric]\n#         for class_name in class_names:\n#             value = results['class_metrics'][class_name][metric]\n#             row.append(f'{value:.3f}')\n#         if metric == 'Precision':\n#             row.append(f'{results[\"overall_metrics\"][\"weighted_precision\"]:.3f}')\n#         elif metric == 'Sensitivity':\n#             row.append(f'{results[\"overall_metrics\"][\"weighted_recall\"]:.3f}')\n#         elif metric == 'F1-Score':\n#             row.append(f'{results[\"overall_metrics\"][\"weighted_f1\"]:.3f}')\n#         else:\n#             row.append('-')\n#         table_data.append(row)\n#         table_colors.append(['white'] * len(header))\n    \n#     acc_row = ['Accuracy'] + ['-'] * len(class_names) + [f'{results[\"overall_metrics\"][\"accuracy\"]:.3f}']\n#     table_data.append(acc_row)\n#     table_colors.append(['white'] * len(header))\n    \n#     table = plt.table(cellText=table_data,\n#                      cellColours=table_colors,\n#                      cellLoc='center',\n#                      loc='center',\n#                      bbox=[0.1, 0.1, 0.8, 0.8])\n    \n#     table.auto_set_font_size(False)\n#     table.set_fontsize(9)\n#     table.scale(1.2, 1.5)\n    \n#     plt.title(f'{series_name} - Performance Metrics', pad=20)\n    \n#     plt.tight_layout(h_pad=1.0, w_pad=1.0)\n#     safe_series_name = series_name.lower().replace(\"/\", \"_\").replace(\" \", \"_\")\n#     output_file = os.path.join(output_dir, f'{safe_series_name}_detailed_results.png')\n#     plt.savefig(output_file, dpi=300, bbox_inches='tight')\n#     plt.close()\n    \n#     # ROC Curve\n#     plt.figure(figsize=(10, 8))\n#     y_true_one_hot = tf.keras.utils.to_categorical(results['y_true'], num_classes=len(class_names))\n#     y_pred_probs = results['predictions']\n    \n#     for i, class_name in enumerate(class_names):\n#         fpr, tpr, _ = roc_curve(y_true_one_hot[:, i], y_pred_probs[:, i])\n#         roc_auc = auc(fpr, tpr)\n#         plt.plot(fpr, tpr, label=f'{class_name} (AUC = {roc_auc:.2f})')\n    \n#     plt.plot([0, 1], [0, 1], 'k--')\n#     plt.xlabel('False Positive Rate')\n#     plt.ylabel('True Positive Rate')\n#     plt.title(f'{series_name} - ROC Curve')\n#     plt.legend(loc='lower right')\n#     plt.grid(True)\n#     plt.savefig(os.path.join(output_dir, f'{safe_series_name}_roc_curve.png'), dpi=300)\n#     plt.close()\n    \n#     # Print detailed metrics\n#     print(f\"\\nDetailed Results for {series_name}:\")\n#     print(\"=\" * 50)\n#     print(\"Overall Metrics:\")\n#     for metric, value in results['overall_metrics'].items():\n#         print(f\"{metric}: {value:.4f}\")\n    \n#     print(\"\\nPer-Class Metrics:\")\n#     for class_name in class_names:\n#         print(f\"\\n{class_name}:\")\n#         metrics = results['class_metrics'][class_name]\n#         print(f\"TP: {metrics['TP']}, FP: {metrics['FP']}, FN: {metrics['FN']}, TN: {metrics['TN']}\")\n#         print(f\"Sensitivity: {metrics['Sensitivity']:.4f}\")\n#         print(f\"Specificity: {metrics['Specificity']:.4f}\")\n#         print(f\"Precision: {metrics['Precision']:.4f}\")\n#         print(f\"F1-Score: {metrics['F1-Score']:.4f}\")\n\n# def main():\n#     global train_df  # Make train_df accessible globally for class weight computation\n#     final_merged_df = load_and_process_data(train_path)\n    \n#     series_types = ['Sagittal T1', 'Axial T2', 'Sagittal T2/STIR']\n#     class_names = ['normal_mild', 'moderate', 'severe']\n    \n#     for series_name in series_types:\n#         print(f\"\\nProcessing {series_name}\")\n        \n#         series_df = final_merged_df[final_merged_df['series_description'] == series_name].copy()\n#         if series_df.empty:\n#             print(f\"No data for {series_name}\")\n#             continue\n        \n#         train_df, temp_df = train_test_split(series_df, test_size=0.3, stratify=series_df['severity'])\n#         val_df, test_df = train_test_split(temp_df, test_size=0.5, stratify=temp_df['severity'])\n        \n#         print(f\"\\nDataset splits for {series_name}:\")\n#         print(f\"Train samples: {len(train_df)}\")\n#         print(f\"Validation samples: {len(val_df)}\")\n#         print(f\"Test samples: {len(test_df)}\")\n        \n#         train_ds = create_dataset(train_df, batch_size=32, is_training=True)\n#         val_ds = create_dataset(val_df, batch_size=32)\n#         test_ds = create_dataset(test_df, batch_size=32)\n        \n#         model = build_hybrid_model()\n#         history = train_model(model, train_ds, val_ds, series_name)\n        \n#         results = evaluate_model(model, test_ds, class_names)\n#         plot_results(history, results, class_names, series_name)\n        \n#         safe_series_name = series_name.lower().replace(\"/\", \"_\").replace(\" \", \"_\")\n#         model_file = os.path.join('saved_models', f'{safe_series_name}_model.h5')\n#         model.save(model_file)\n        \n#         print(f\"Completed {series_name} at: {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')} UTC\")\n\n# if __name__ == \"__main__\":\n#     main()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import os\n# import numpy as np\n# import pandas as pd\n# import tensorflow as tf\n# from tensorflow.keras import layers, models, callbacks\n# from tensorflow.keras.applications import EfficientNetB1\n# from tensorflow_addons.optimizers import Lookahead\n# from sklearn.utils.class_weight import compute_class_weight\n# from sklearn.metrics import confusion_matrix, precision_score, recall_score, f1_score, roc_curve, auc\n# import albumentations as A\n# import pydicom\n# import matplotlib.pyplot as plt\n# from datetime import datetime\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# # تابع محاسبه وزن‌های کلاس\n# def compute_class_weights(df):\n#     labels = df['severity'].map({'normal_mild': 0, 'moderate': 1, 'severe': 2}).values\n#     class_weights = compute_class_weight('balanced', classes=np.unique(labels), y=labels)\n#     return dict(zip(np.unique(labels), class_weights))\n\n# # تابع Focal Loss با وزن‌های کلاس\n# def focal_loss(gamma=2., alpha=4., class_weights=None):\n#     def focal_loss_with_logits(y_true, y_pred):\n#         y_true = tf.cast(y_true, tf.int32)\n#         ce = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y_true, logits=y_pred)\n#         probs = tf.nn.softmax(y_pred)\n#         probs = tf.gather(probs, y_true, batch_dims=1)\n        \n#         if class_weights is not None:\n#             weights = tf.gather(tf.constant(list(class_weights.values()), dtype=tf.float32), y_true)\n#             ce = ce * weights\n        \n#         return tf.reduce_mean(alpha * tf.pow(1. - probs, gamma) * ce)\n#     return focal_loss_with_logits\n\n# # تابع Data Augmentation با Albumentations\n# def albumentations_augmentation(image):\n#     transform = A.Compose([\n#         A.RandomBrightnessContrast(p=0.5),\n#         A.Rotate(limit=30, p=0.5),\n#         A.HorizontalFlip(p=0.5),\n#         A.VerticalFlip(p=0.5),\n#         A.GaussNoise(p=0.3),\n#         A.RandomGamma(p=0.3),\n#     ])\n#     augmented = transform(image=image)\n#     return augmented['image']\n\n# # تابع بارگذاری و نرمال‌سازی تصاویر DICOM با Windowing\n# @tf.function\n# def load_dicom_tf(path):\n#     try:\n#         dicom = pydicom.dcmread(path.numpy().decode('utf-8'))\n#         data = dicom.pixel_array\n#         window_center = dicom.get('WindowCenter', 1000)\n#         window_width = dicom.get('WindowWidth', 2000)\n#         data = np.clip(data, window_center - window_width // 2, window_center + window_width // 2)\n#         data = (data - (window_center - window_width // 2)) / window_width\n#         data = np.expand_dims(data, axis=-1)\n#         return data.astype(np.float32)\n#     except Exception as e:\n#         print(f\"Error loading DICOM: {e}\")\n#         return np.zeros((224, 224, 1), dtype=np.float32)\n\n# # تابع پیش‌پردازش تصاویر با پشتیبانی از چندین تصویر\n# @tf.function\n# def preprocess_image(image_paths, label=None, is_training=False):\n#     images = [tf.py_function(load_dicom_tf, [path], tf.float32) for path in image_paths]\n#     images = [tf.image.resize(img, [224, 224]) for img in images]\n#     images = [tf.image.grayscale_to_rgb(img) for img in images]\n#     image = tf.reduce_mean(tf.stack(images), axis=0)\n    \n#     if is_training:\n#         image = tf.py_function(lambda x: albumentations_augmentation(x.numpy()), [image], tf.float32)\n#         image.set_shape([224, 224, 3])\n    \n#     image = tf.keras.applications.efficientnet.preprocess_input(image)\n#     return (image, label) if label is not None else image\n\n# # تابع ایجاد دیتاست با Oversampling\n# def create_dataset(df, batch_size=32, is_training=False):\n#     if is_training:\n#         class_counts = df['severity'].value_counts()\n#         max_count = class_counts.max()\n#         oversampled_dfs = []\n#         for severity in class_counts.index:\n#             class_df = df[df['severity'] == severity]\n#             repeat_factor = int(max_count // len(class_df)) + 1\n#             oversampled_dfs.append(pd.concat([class_df] * repeat_factor)[:max_count])\n#         df = pd.concat(oversampled_dfs).sample(frac=1).reset_index(drop=True)\n    \n#     dataset = tf.data.Dataset.from_tensor_slices((\n#         df['image_paths'],\n#         df['severity'].map({'normal_mild': 0, 'moderate': 1, 'severe': 2}).astype(np.int32)\n#     ))\n    \n#     dataset = dataset.map(\n#         lambda x, y: preprocess_image(x, y, is_training),\n#         num_parallel_calls=tf.data.AUTOTUNE\n#     )\n    \n#     if is_training:\n#         dataset = dataset.shuffle(1000)\n#         dataset = dataset.repeat()\n    \n#     dataset = dataset.batch(batch_size)\n#     dataset = dataset.prefetch(tf.data.AUTOTUNE)\n#     return dataset\n\n# # تابع ساخت مدل هیبریدی\n# def build_hybrid_model(input_shape=(224, 224, 3), num_classes=3):\n#     inputs = layers.Input(shape=input_shape)\n#     efficientnet = EfficientNetB1(include_top=False, weights='imagenet', input_shape=input_shape)\n    \n#     for layer in efficientnet.layers:\n#         if 'block6' in layer.name or 'block7' in layer.name:\n#             layer.trainable = True\n#         else:\n#             layer.trainable = False\n    \n#     eff_features = efficientnet(inputs)\n#     attention = layers.Conv2D(eff_features.shape[-1], (1, 1), activation='sigmoid')(eff_features)\n#     eff_features = layers.Multiply()([eff_features, attention])\n    \n#     x = layers.GlobalAveragePooling2D()(eff_features)\n#     x = layers.Dense(512, activation='relu')(x)\n#     x = layers.BatchNormalization()(x)\n#     x = layers.Dropout(0.5)(x)\n#     outputs = layers.Dense(num_classes)(x)\n    \n#     return models.Model(inputs, outputs, name='hybrid_efficientnet')\n\n# # تابع آموزش مدل\n# def train_model(model, train_ds, val_ds, series_name, epochs=100, steps_per_epoch=100):\n#     base_optimizer = tf.keras.optimizers.Adam(learning_rate=1e-4)\n#     optimizer = Lookahead(base_optimizer)\n    \n#     class_weights = compute_class_weights(train_df)\n#     model.compile(\n#         optimizer=optimizer,\n#         loss=focal_loss(class_weights=class_weights),\n#         metrics=['accuracy']\n#     )\n    \n#     lr_schedule = tf.keras.experimental.CosineDecay(\n#         initial_learning_rate=1e-4,\n#         decay_steps=epochs * steps_per_epoch,\n#         alpha=1e-6\n#     )\n    \n#     callbacks_list = [\n#         callbacks.EarlyStopping(monitor='val_loss', patience=15, restore_best_weights=True),\n#         callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5, min_lr=1e-6),\n#         callbacks.LearningRateScheduler(lr_schedule)\n#     ]\n    \n#     history = model.fit(\n#         train_ds,\n#         validation_data=val_ds,\n#         epochs=epochs,\n#         steps_per_epoch=steps_per_epoch,\n#         validation_steps=30,\n#         callbacks=callbacks_list,\n#         verbose=1\n#     )\n#     return history\n\n# # تابع ارزیابی مدل\n# def evaluate_model(model, test_ds, class_names):\n#     y_true, y_pred, y_pred_probs = [], [], []\n    \n#     for images, labels in test_ds:\n#         predictions = model.predict(images)\n#         y_pred_probs.extend(tf.nn.softmax(predictions).numpy())\n#         y_pred.extend(np.argmax(predictions, axis=1))\n#         y_true.extend(labels.numpy())\n    \n#     y_true, y_pred, y_pred_probs = np.array(y_true), np.array(y_pred), np.array(y_pred_probs)\n#     cm = confusion_matrix(y_true, y_pred)\n    \n#     overall_metrics = {\n#         'accuracy': np.mean(y_pred == y_true),\n#         'weighted_f1': f1_score(y_true, y_pred, average='weighted')\n#     }\n    \n#     return {'confusion_matrix': cm, 'overall_metrics': overall_metrics, 'y_true': y_true, 'y_pred_probs': y_pred_probs}\n\n# # تابع رسم نتایج\n# def plot_results(history, results, class_names, series_name):\n#     output_dir = 'model_results'\n#     os.makedirs(output_dir, exist_ok=True)\n    \n#     fig = plt.figure(figsize=(15, 10))\n    \n#     # رسم Loss\n#     plt.subplot(2, 2, 1)\n#     plt.plot(history.history['loss'], label='Train Loss')\n#     plt.plot(history.history['val_loss'], label='Val Loss')\n#     plt.title(f'{series_name} - Loss')\n#     plt.legend()\n    \n#     # رسم Accuracy\n#     plt.subplot(2, 2, 2)\n#     plt.plot(history.history['accuracy'], label='Train Acc')\n#     plt.plot(history.history['val_accuracy'], label='Val Acc')\n#     plt.title(f'{series_name} - Accuracy')\n#     plt.legend()\n    \n#     # رسم ماتریس Confusion\n#     plt.subplot(2, 2, 3)\n#     plt.imshow(results['confusion_matrix'], cmap='Blues')\n#     plt.title(f'{series_name} - Confusion Matrix')\n#     plt.colorbar()\n#     plt.xticks(range(len(class_names)), class_names, rotation=45)\n#     plt.yticks(range(len(class_names)), class_names)\n    \n#     # رسم منحنی ROC\n#     plt.subplot(2, 2, 4)\n#     y_true_one_hot = tf.keras.utils.to_categorical(results['y_true'], num_classes=len(class_names))\n#     for i, class_name in enumerate(class_names):\n#         fpr, tpr, _ = roc_curve(y_true_one_hot[:, i], results['y_pred_probs'][:, i])\n#         roc_auc = auc(fpr, tpr)\n#         plt.plot(fpr, tpr, label=f'{class_name} (AUC = {roc_auc:.2f})')\n#     plt.plot([0, 1], [0, 1], 'k--')\n#     plt.title(f'{series_name} - ROC Curve')\n#     plt.legend(loc='lower right')\n    \n#     plt.tight_layout()\n#     output_file = os.path.join(output_dir, f'{series_name}_results.png')\n#     plt.savefig(output_file, dpi=300)\n#     plt.close()\n\n# # فرض می‌کنیم train_df یک DataFrame با ستون‌های 'image_paths' (لیست مسیرها) و 'severity' است\n# train_df = pd.DataFrame()  # باید با داده‌های واقعی پر شود\n# val_df = train_df.sample(frac=0.2)\n# train_df = train_df.drop(val_df.index)\n# test_df = val_df.sample(frac=0.5)\n# val_df = val_df.drop(test_df.index)\n\n# # ایجاد دیتاست‌ها\n# train_ds = create_dataset(train_df, is_training=True)\n# val_ds = create_dataset(val_df)\n# test_ds = create_dataset(test_df)\n\n# # ساخت و آموزش مدل\n# model = build_hybrid_model()\n# history = train_model(model, train_ds, val_ds, \"example_series\")\n\n# # ارزیابی و رسم نتایج\n# class_names = ['normal_mild', 'moderate', 'severe']\n# results = evaluate_model(model, test_ds, class_names)\n# plot_results(history, results, class_names, \"example_series\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}