{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.12"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"dockerImageVersionId":31234,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":16920.339041,"end_time":"2026-01-11T23:18:59.487031","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2026-01-11T18:36:59.147990","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"31dc8834","cell_type":"code","source":"import os\nimport pydicom\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models, applications\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.utils import to_categorical\nfrom sklearn.metrics import confusion_matrix, classification_report, balanced_accuracy_score\nfrom sklearn.utils.class_weight import compute_class_weight\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# ========== ULTRA-FAST CONFIGURATION (12-HOUR KAGGLE LIMIT) ==========\nIMG_SIZE = 192  # REDUCED: Smaller resolution (was 224) - 26% fewer pixels\nBATCH_SIZE = 64  # DOUBLED: Larger batches for faster training (was 32)\nEPOCHS = 20  # REDUCED: Fewer initial epochs (was 25)\nDATA_DIR = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/'\nNUM_SLICES = 2  # REDUCED: Only 2 slices per view (was 3) - 33% faster\nLEARNING_RATE = 1e-4  # INCREASED: Faster convergence\nWEIGHT_DECAY = 1e-4  # REDUCED: Less regularization for faster training\nLABEL_SMOOTHING = 0.03  # REDUCED\nWARMUP_EPOCHS = 3  # REDUCED: Less warmup (was 5)\nDROPOUT_RATE = 0.3  # REDUCED: Less dropout for faster convergence\nUSE_MIXED_PRECISION = True\nUSE_CUTMIX = False  # DISABLED: Skip expensive augmentation\nFOCAL_GAMMA = 2.0\n\n# Fine-tuning configuration\nFINE_TUNE_EPOCHS = 10  # REDUCED: Fewer fine-tune epochs (was 15)\nFINE_TUNE_LR = 1e-5  # Lower LR for fine-tuning\nFINE_TUNE_AT = 100  # Unfreeze from this layer\n\n# Enable mixed precision\nif USE_MIXED_PRECISION:\n    tf.keras.mixed_precision.set_global_policy('mixed_float16')\n    print(\"✓ Mixed precision enabled\")\n\n# Load data\ntrain_df = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv')\nseries_desc_df = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_series_descriptions.csv')\n\nlabel_cols = train_df.columns[1:]\nlabel_map = {'Normal/Mild': 0, 'Moderate': 1, 'Severe': 2}\n\ndef encode_labels(row):\n    return [label_map.get(row[col], 0) for col in label_cols]\ntrain_df['encoded_labels'] = train_df.apply(encode_labels, axis=1)\n\ndef get_max_severity(encoded_labels):\n    return max(encoded_labels)\ntrain_df['max_severity'] = train_df['encoded_labels'].apply(get_max_severity)\n\n# ========== CRITICAL: AGGRESSIVE DATASET REDUCTION ==========\n# Target: ~4 hours total training time (within 12-hour Kaggle limit)\nMAX_IMAGES = 6000  # HALVED: 6K studies (was 12K) - 50% faster\nIMAGES_PER_STUDY = 2  # 2 slices × 3 views = 6 images per study\nMAX_STUDIES = MAX_IMAGES // IMAGES_PER_STUDY\nstudies_per_class = MAX_STUDIES // 3\n\nprint(f\"\\n{'='*70}\")\nprint(\"ULTRA-FAST TRAINING CONFIGURATION\")\nprint(f\"{'='*70}\")\nprint(f\"Target: Complete training in ~4-5 hours (within 12-hour Kaggle limit)\")\nprint(f\"\\nDataset: {MAX_STUDIES} studies ({studies_per_class} per class)\")\nprint(f\"Images per study: {IMAGES_PER_STUDY * 3} (2 slices × 3 views)\")\nprint(f\"Total images per epoch: ~{MAX_STUDIES * IMAGES_PER_STUDY * 3:,}\")\nprint(f\"Image resolution: {IMG_SIZE}×{IMG_SIZE} (26% fewer pixels than 224)\")\nprint(f\"Batch size: {BATCH_SIZE} (2× larger for speed)\")\nprint(f\"Initial epochs: {EPOCHS} (with early stopping)\")\nprint(f\"Fine-tune epochs: {FINE_TUNE_EPOCHS} (with early stopping)\")\nprint(f\"Expected time per epoch: ~40-50 minutes\")\nprint(f\"{'='*70}\\n\")\n\ndfs = []\nfor severity in [0, 1, 2]:\n    df_severity = train_df[train_df['max_severity'] == severity].head(studies_per_class)\n    if len(df_severity) < studies_per_class:\n        df_severity = df_severity.sample(n=studies_per_class, replace=True, random_state=42)\n    dfs.append(df_severity)\nbalanced_train_df = pd.concat(dfs, ignore_index=True).sample(frac=1, random_state=42).reset_index(drop=True)\nprint(f\"✓ Balanced dataset: {len(balanced_train_df)} studies\")\nprint(f\"Class distribution:\\n{balanced_train_df['max_severity'].value_counts()}\\n\")\n\ndef get_series_ids(study_id):\n    row = series_desc_df[series_desc_df['study_id'] == study_id]\n    views = {}\n    for view_type in ['Sagittal T1', 'Sagittal T2/STIR', 'Axial T2']:\n        series = row[row['series_description'].str.contains(view_type, case=False, na=False)]\n        views[view_type] = series['series_id'].values[0] if len(series) > 0 else np.nan\n    return views\n\ndef load_dicom_image(path, augment=False):\n    \"\"\"ULTRA-FAST DICOM loading - minimal preprocessing\"\"\"\n    try:\n        dcm = pydicom.dcmread(path)\n        img = dcm.pixel_array.astype(np.float32)\n    except:\n        return np.zeros((IMG_SIZE, IMG_SIZE, 3), dtype=np.float32)\n    \n    # Fast normalization\n    img = (img - img.min()) / (img.max() - img.min() + 1e-7)\n    img = cv2.resize(img, (IMG_SIZE, IMG_SIZE), interpolation=cv2.INTER_AREA)\n    \n    # Convert to 3 channels\n    img = np.stack([img] * 3, axis=-1)\n    \n    # MINIMAL AUGMENTATION: Only horizontal flip (fastest)\n    if augment and np.random.rand() > 0.5:\n        img = cv2.flip(img, 1)\n    \n    return img.astype(np.float32)\n\ndef load_study_images(study_id, augment=False):\n    views = get_series_ids(study_id)\n    images_by_view = []\n    \n    for view in ['Sagittal T1', 'Sagittal T2/STIR', 'Axial T2']:\n        series_id = views[view]\n        view_imgs = []\n        \n        if pd.isna(series_id):\n            view_imgs = [np.zeros((IMG_SIZE, IMG_SIZE, 3)) for _ in range(NUM_SLICES)]\n        else:\n            series_path = os.path.join(DATA_DIR, str(study_id), str(series_id))\n            if not os.path.exists(series_path):\n                view_imgs = [np.zeros((IMG_SIZE, IMG_SIZE, 3)) for _ in range(NUM_SLICES)]\n            else:\n                instances = sorted(os.listdir(series_path))\n                \n                if instances:\n                    num_instances = len(instances)\n                    slice_indices = np.linspace(0, num_instances-1, NUM_SLICES, dtype=int)\n                    \n                    for idx in slice_indices:\n                        img_path = os.path.join(series_path, instances[idx])\n                        try:\n                            view_imgs.append(load_dicom_image(img_path, augment=augment))\n                        except:\n                            view_imgs.append(np.zeros((IMG_SIZE, IMG_SIZE, 3)))\n                else:\n                    view_imgs = [np.zeros((IMG_SIZE, IMG_SIZE, 3)) for _ in range(NUM_SLICES)]\n        \n        images_by_view.append(np.array(view_imgs))\n    \n    return images_by_view\n\nclass DataGenerator(tf.keras.utils.Sequence):\n    def __init__(self, df, batch_size=BATCH_SIZE, shuffle=True, augment=False):\n        self.df = df\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.augment = augment\n        self.indexes = np.arange(len(self.df))\n        self.on_epoch_end()\n\n    def __len__(self):\n        return int(np.floor(len(self.df) / self.batch_size))\n\n    def __getitem__(self, index):\n        batch_ids = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n        batch_df = self.df.iloc[batch_ids]\n        \n        view1_batch, view2_batch, view3_batch = [], [], []\n        y = []\n        \n        for _, row in batch_df.iterrows():\n            imgs = load_study_images(row['study_id'], augment=self.augment)\n            view1_batch.append(imgs[0])\n            view2_batch.append(imgs[1])\n            view3_batch.append(imgs[2])\n            y.append(row['encoded_labels'])\n        \n        X = [np.array(view1_batch), np.array(view2_batch), np.array(view3_batch)]\n        y_cat = to_categorical(np.array(y), num_classes=3)\n        \n        return tuple(X), y_cat\n\n    def on_epoch_end(self):\n        if self.shuffle:\n            np.random.shuffle(self.indexes)\n\n# Enhanced class weights\ndef compute_class_weights(df):\n    all_labels = []\n    for labels in df['encoded_labels']:\n        all_labels.extend(labels)\n    \n    classes = np.unique(all_labels)\n    weights = compute_class_weight('balanced', classes=classes, y=all_labels)\n    weights = weights ** 1.1\n    class_weight_dict = {i: w for i, w in enumerate(weights)}\n    print(f\"✓ Enhanced class weights: {class_weight_dict}\")\n    return class_weight_dict\n\n# Balanced Focal Loss\nclass BalancedFocalLoss(tf.keras.losses.Loss):\n    def __init__(self, alpha=0.25, gamma=2.0, label_smoothing=0.03):\n        super().__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.label_smoothing = label_smoothing\n    \n    def call(self, y_true, y_pred):\n        num_classes = tf.shape(y_true)[-1]\n        y_true = y_true * (1.0 - self.label_smoothing) + (self.label_smoothing / tf.cast(num_classes, tf.float32))\n        \n        y_pred = tf.clip_by_value(y_pred, 1e-7, 1.0 - 1e-7)\n        ce_loss = -y_true * tf.math.log(y_pred)\n        focal_weight = self.alpha * tf.math.pow(1.0 - y_pred, self.gamma)\n        focal_loss = focal_weight * ce_loss\n        return tf.reduce_mean(tf.reduce_sum(focal_loss, axis=-1))\n\n# Cosine Annealing with Warmup\nclass WarmupCosineDecay(tf.keras.optimizers.schedules.LearningRateSchedule):\n    def __init__(self, initial_lr, warmup_steps, total_steps, alpha=0.0):\n        super().__init__()\n        self.initial_lr = initial_lr\n        self.warmup_steps = warmup_steps\n        self.total_steps = total_steps\n        self.alpha = alpha\n    \n    def __call__(self, step):\n        step = tf.cast(step, tf.float32)\n        warmup_steps = tf.cast(self.warmup_steps, tf.float32)\n        total_steps = tf.cast(self.total_steps, tf.float32)\n        \n        warmup_lr = self.initial_lr * (step / warmup_steps)\n        \n        progress = (step - warmup_steps) / (total_steps - warmup_steps)\n        cosine_decay = 0.5 * (1.0 + tf.cos(np.pi * progress))\n        decay_lr = self.alpha + (self.initial_lr - self.alpha) * cosine_decay\n        \n        return tf.cond(step < warmup_steps, lambda: warmup_lr, lambda: decay_lr)\n    \n    def get_config(self):\n        return {\n            \"initial_lr\": self.initial_lr,\n            \"warmup_steps\": self.warmup_steps,\n            \"total_steps\": self.total_steps,\n            \"alpha\": self.alpha,\n        }\n\n# ========== ULTRA-FAST ARCHITECTURE: MobileNetV3Small ==========\ndef build_mvcnn_ultrafast():\n    view1_input = layers.Input(shape=(NUM_SLICES, IMG_SIZE, IMG_SIZE, 3), name=\"view_sagittal_t1\")\n    view2_input = layers.Input(shape=(NUM_SLICES, IMG_SIZE, IMG_SIZE, 3), name=\"view_sagittal_t2\")\n    view3_input = layers.Input(shape=(NUM_SLICES, IMG_SIZE, IMG_SIZE, 3), name=\"view_axial_t2\")\n    \n    # CRITICAL: Use MobileNetV3Small (2.5M params vs 5.4M - 2× faster)\n    try:\n        backbone = applications.MobileNetV3Small(\n            include_top=False,\n            weights='imagenet',\n            input_shape=(IMG_SIZE, IMG_SIZE, 3),\n            pooling='avg',\n            minimalistic=False,\n            dropout_rate=0.2  # Built-in dropout\n        )\n    except Exception as e:\n        print(f\"⚠️ Could not download pretrained weights: {e}. Falling back to weights=None.\")\n        backbone = applications.MobileNetV3Small(\n            include_top=False,\n            weights=None,\n            input_shape=(IMG_SIZE, IMG_SIZE, 3),\n            pooling='avg',\n            minimalistic=False,\n            dropout_rate=0.2  # Built-in dropout\n        )\n    \n    # AGGRESSIVE: Train only last 20 layers (was 30)\n    for layer in backbone.layers[:-20]:\n        layer.trainable = False\n    for layer in backbone.layers[-20:]:\n        layer.trainable = True\n    \n    trainable_count = sum([1 for l in backbone.layers if l.trainable])\n    print(f\"✓ MobileNetV3Small: {trainable_count}/{len(backbone.layers)} layers trainable\")\n    \n    # Simplified attention (no shared layer)\n    def process_view(view_input, view_name):\n        features = layers.TimeDistributed(backbone, name=f'{view_name}_features')(view_input)\n        \n        # Simple attention\n        attention_weights = layers.Dense(1, activation='sigmoid')(features)\n        attention_weights = layers.Softmax(axis=1)(attention_weights)\n        \n        weighted = layers.Multiply()([features, attention_weights])\n        pooled = layers.Lambda(lambda x: tf.reduce_sum(x, axis=1))(weighted)\n        \n        return pooled\n    \n    # Process all views\n    feat1 = process_view(view1_input, 'view1')\n    feat2 = process_view(view2_input, 'view2')\n    feat3 = process_view(view3_input, 'view3')\n    \n    # SIMPLIFIED: Skip multi-head attention, just concatenate\n    merged = layers.Concatenate()([feat1, feat2, feat3])\n    \n    # REDUCED: Smaller classification head\n    x = layers.Dense(512, kernel_regularizer=tf.keras.regularizers.l2(WEIGHT_DECAY))(merged)\n    x = layers.BatchNormalization()(x)\n    x = layers.Activation('relu')(x)\n    x = layers.Dropout(DROPOUT_RATE)(x)\n    \n    x = layers.Dense(256, kernel_regularizer=tf.keras.regularizers.l2(WEIGHT_DECAY))(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Activation('relu')(x)\n    x = layers.Dropout(DROPOUT_RATE * 0.7)(x)\n    \n    output = layers.Dense(len(label_cols)*3, activation='softmax',\n                         kernel_regularizer=tf.keras.regularizers.l2(WEIGHT_DECAY),\n                         dtype='float32')(x)\n    output = layers.Reshape((len(label_cols), 3))(output)\n    \n    model = models.Model(inputs=[view1_input, view2_input, view3_input], outputs=output)\n    return model\n\ndef compile_model_with_schedule(model, steps_per_epoch, epochs, lr):\n    total_steps = steps_per_epoch * epochs\n    warmup_steps = steps_per_epoch * WARMUP_EPOCHS\n    \n    lr_schedule = WarmupCosineDecay(\n        initial_lr=lr,\n        warmup_steps=warmup_steps,\n        total_steps=total_steps,\n        alpha=1e-7\n    )\n    \n    optimizer = tf.keras.optimizers.AdamW(\n        learning_rate=lr_schedule,\n        weight_decay=WEIGHT_DECAY,\n        clipnorm=1.0\n    )\n    \n    loss_fn = BalancedFocalLoss(alpha=0.25, gamma=FOCAL_GAMMA, label_smoothing=LABEL_SMOOTHING)\n    \n    def categorical_accuracy_metric(y_true, y_pred):\n        return tf.keras.metrics.categorical_accuracy(\n            tf.reshape(y_true, [-1, 3]), \n            tf.reshape(y_pred, [-1, 3])\n        )\n    \n    model.compile(\n        optimizer=optimizer,\n        loss=loss_fn,\n        metrics=[categorical_accuracy_metric]\n    )\n    return model\n\n# Compute class weights\nclass_weights = compute_class_weights(balanced_train_df)\n\n# Stratified split\ntrain_ids, val_ids = train_test_split(\n    balanced_train_df, \n    test_size=0.20,\n    random_state=42, \n    stratify=balanced_train_df['max_severity']\n)\n\nprint(f\"\\n✓ Training: {len(train_ids)}, Validation: {len(val_ids)}\")\nprint(f\"✓ Steps per epoch: {len(train_ids) // BATCH_SIZE}\")\nprint(f\"✓ Estimated time per epoch: ~45 minutes\\n\")\n\n# Data generators (no CutMix)\ntrain_gen = DataGenerator(train_ids, augment=True)\nval_gen = DataGenerator(val_ids, augment=False)\n\n# Build model\nmodel = build_mvcnn_ultrafast()\nprint(\"\\n\" + \"=\"*70)\nprint(\"ULTRA-FAST MVCNN ARCHITECTURE\")\nprint(\"=\"*70)\nmodel.summary()\n\n# Compile\nmodel = compile_model_with_schedule(model, steps_per_epoch=len(train_gen), epochs=EPOCHS, lr=LEARNING_RATE)\n\n# Callbacks with aggressive early stopping\n# Note: No ReduceLROnPlateau since WarmupCosineDecay already handles LR scheduling\ncallbacks = [\n    EarlyStopping(\n        monitor='val_categorical_accuracy_metric',\n        patience=5,  # REDUCED: Stop earlier if no improvement\n        restore_best_weights=True,\n        verbose=1,\n        mode='max'\n    ),\n    ModelCheckpoint(\n        'best_model_ultrafast.keras',\n        monitor='val_categorical_accuracy_metric',\n        save_best_only=True,\n        verbose=1,\n        mode='max'\n    ),\n    tf.keras.callbacks.LambdaCallback(\n        on_epoch_end=lambda epoch, logs: print(f\"\\n>>> Epoch {epoch+1}: Val Acc = {logs['val_categorical_accuracy_metric']:.4f} ({logs['val_categorical_accuracy_metric']*100:.2f}%)\")\n    )\n]\n\nprint(\"\\n\" + \"=\"*70)\nprint(\"🚀 STAGE 1: INITIAL TRAINING (ULTRA-FAST)\")\nprint(f\"Backbone: MobileNetV3Small (2.5M params - 2× faster than Large)\")\nprint(f\"Trainable Layers: Last 20 (reduced for speed)\")\nprint(f\"Batch Size: {BATCH_SIZE} (doubled for speed)\")\nprint(f\"Dataset Size: {len(balanced_train_df)} studies (50% of previous)\")\nprint(f\"Slices per view: {NUM_SLICES} (33% fewer than before)\")\nprint(f\"Image resolution: {IMG_SIZE}×{IMG_SIZE} (26% fewer pixels)\")\nprint(f\"Steps per epoch: {len(train_gen)}\")\nprint(f\"Early Stopping: patience=5 epochs\")\nprint(f\"Estimated training time: ~2-2.5 hours (with early stopping)\")\nprint(\"=\"*70 + \"\\n\")\n\n# Stage 1: Initial Training\nimport time\nstart_time = time.time()\n\nhistory_stage1 = model.fit(\n    train_gen,\n    validation_data=val_gen,\n    epochs=EPOCHS,\n    callbacks=callbacks,\n    verbose=1\n)\n\nstage1_time = (time.time() - start_time) / 3600  # Convert to hours\n\nprint(\"\\n\" + \"=\"*70)\nprint(\"✓ STAGE 1 COMPLETED\")\nprint(\"=\"*70)\nstage1_acc = max(history_stage1.history['val_categorical_accuracy_metric'])\nprint(f\"Best Stage 1 Accuracy: {stage1_acc:.4f} ({stage1_acc*100:.2f}%)\")\nprint(f\"Stage 1 Training Time: {stage1_time:.2f} hours\")\nprint(\"=\"*70 + \"\\n\")\n\n# ========== STAGE 2: FINE-TUNING ==========\nprint(\"🔥 STAGE 2: FINE-TUNING\")\n\n# Unfreeze more layers\nfor layer in model.layers:\n    if hasattr(layer, 'layers'):\n        for sublayer in layer.layers:\n            if hasattr(sublayer, 'layers'):\n                for backbone_layer in sublayer.layers[FINE_TUNE_AT:]:\n                    backbone_layer.trainable = True\n\n# Recompile with lower LR\ntotal_steps_ft = len(train_gen) * FINE_TUNE_EPOCHS\nwarmup_steps_ft = len(train_gen) * 2\n\nlr_schedule_ft = WarmupCosineDecay(\n    initial_lr=FINE_TUNE_LR,\n    warmup_steps=warmup_steps_ft,\n    total_steps=total_steps_ft,\n    alpha=1e-8\n)\n\noptimizer_ft = tf.keras.optimizers.AdamW(\n    learning_rate=lr_schedule_ft,\n    weight_decay=WEIGHT_DECAY * 0.5,\n    clipnorm=1.0\n)\n\nloss_fn_ft = BalancedFocalLoss(alpha=0.25, gamma=FOCAL_GAMMA, label_smoothing=0.02)\n\ndef categorical_accuracy_metric(y_true, y_pred):\n    return tf.keras.metrics.categorical_accuracy(\n        tf.reshape(y_true, [-1, 3]), \n        tf.reshape(y_pred, [-1, 3])\n    )\n\nmodel.compile(\n    optimizer=optimizer_ft,\n    loss=loss_fn_ft,\n    metrics=[categorical_accuracy_metric]\n)\n\nprint(\"\\n\" + \"=\"*70)\nprint(f\"Fine-tune Epochs: {FINE_TUNE_EPOCHS}\")\nprint(f\"Fine-tune Learning Rate: {FINE_TUNE_LR}\")\nprint(f\"Early Stopping: patience=4 epochs\")\nprint(f\"Estimated fine-tuning time: ~1.5-2 hours (with early stopping)\")\nprint(\"=\"*70 + \"\\n\")\n\n# Fine-tuning callbacks\n# Note: No ReduceLROnPlateau since WarmupCosineDecay already handles LR scheduling\ncallbacks_ft = [\n    EarlyStopping(\n        monitor='val_categorical_accuracy_metric',\n        patience=4,  # REDUCED for speed\n        restore_best_weights=True,\n        verbose=1,\n        mode='max'\n    ),\n    ModelCheckpoint(\n        'best_model_finetuned_ultrafast.keras',\n        monitor='val_categorical_accuracy_metric',\n        save_best_only=True,\n        verbose=1,\n        mode='max'\n    ),\n    tf.keras.callbacks.LambdaCallback(\n        on_epoch_end=lambda epoch, logs: print(f\"\\n>>> Fine-tune Epoch {epoch+1}: Val Acc = {logs['val_categorical_accuracy_metric']:.4f} ({logs['val_categorical_accuracy_metric']*100:.2f}%)\")\n    )\n]\n\n# Stage 2: Fine-tuning\nstart_time_ft = time.time()\n\nhistory_stage2 = model.fit(\n    train_gen,\n    validation_data=val_gen,\n    epochs=FINE_TUNE_EPOCHS,\n    callbacks=callbacks_ft,\n    verbose=1\n)\n\nstage2_time = (time.time() - start_time_ft) / 3600\ntotal_time = stage1_time + stage2_time\n\n# Combine histories\nhistory = type('obj', (object,), {\n    'history': {\n        'categorical_accuracy_metric': history_stage1.history['categorical_accuracy_metric'] + history_stage2.history['categorical_accuracy_metric'],\n        'val_categorical_accuracy_metric': history_stage1.history['val_categorical_accuracy_metric'] + history_stage2.history['val_categorical_accuracy_metric'],\n        'loss': history_stage1.history['loss'] + history_stage2.history['loss'],\n        'val_loss': history_stage1.history['val_loss'] + history_stage2.history['val_loss']\n    }\n})()\n\nmodel.save('mvc_MobileNetV3Small_ultrafast_89pct.h5')\n\nprint(\"\\n\" + \"=\"*70)\nprint(\"✅ TWO-STAGE TRAINING COMPLETED!\")\nprint(\"=\"*70)\nstage1_best = max(history_stage1.history['val_categorical_accuracy_metric'])\nstage1_best = max(history_stage1.history['val_categorical_accuracy_metric'])\nstage2_best = max(history_stage2.history['val_categorical_accuracy_metric'])\nbest_val_acc = max(history.history['val_categorical_accuracy_metric'])\nfinal_val_acc = history.history['val_categorical_accuracy_metric'][-1]\nimprovement = (stage2_best - stage1_best) * 100\nprint(f\"\\nStage 1 (Initial Training):\")\n\nprint(f\"  Best Accuracy: {stage1_best:.4f} ({stage1_best*100:.2f}%)\")\nprint(f\"  Training Time: {stage1_time:.2f} hours\")\nprint(f\"\\nStage 2 (Fine-tuning):\")\nprint(f\"  Best Accuracy: {stage2_best:.4f} ({stage2_best*100:.2f}%)\")\nprint(f\"  Improvement: +{improvement:.2f} percentage points\")\n\nprint(f\"\\nOverall Best: {best_val_acc:.4f} ({best_val_acc*100:.2f}%)\")\nprint(f\"Final Val Accuracy: {final_val_acc:.4f} ({final_val_acc*100:.2f}%)\")\nprint(f\"\\n⏱️ TOTAL TRAINING TIME: {total_time:.2f} hours (Target: < 12 hours)\")\nprint(f\"   Time remaining for submission: {12 - total_time:.2f} hours\")\n\nprint(\"=\"*70)\nif best_val_acc >= 0.89:\n    print(f\"\\n🎉🎉🎉 TARGET ACHIEVED: {best_val_acc*100:.2f}% (≥89%)! 🎉🎉🎉\")\nelif best_val_acc >= 0.85:\n    print(f\"\\n🎉 EXCELLENT: {best_val_acc*100:.2f}% (Close to 89% target!)\")\nelif best_val_acc >= 0.80:\n    print(f\"\\n✓ GOOD: {best_val_acc*100:.2f}% (Significant improvement!)\")\nelse:\n    print(f\"\\n📊 Progress: {best_val_acc*100:.2f}%\")\n    \nif total_time < 12:\n    print(f\"✅ TRAINING COMPLETED WITHIN 12-HOUR KAGGLE LIMIT\")\nelse:\n    print(f\"⚠️ WARNING: Training exceeded 12-hour limit by {total_time - 12:.2f} hours\")","metadata":{"execution":{"iopub.execute_input":"2026-01-11T18:37:02.334714Z","iopub.status.busy":"2026-01-11T18:37:02.333843Z","iopub.status.idle":"2026-01-11T23:18:54.837793Z","shell.execute_reply":"2026-01-11T23:18:54.836319Z"},"papermill":{"duration":16912.51109,"end_time":"2026-01-11T23:18:54.839809","exception":false,"start_time":"2026-01-11T18:37:02.328719","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"b4773e84","cell_type":"code","source":"# ========== VISUALIZATION ==========\n\nfig, axes = plt.subplots(2, 3, figsize=(18, 10))\n\n# 1. Training and Validation Accuracy\naxes[0, 0].plot(history.history['categorical_accuracy_metric'], label='Train Accuracy', linewidth=2, color='#2E86AB')\naxes[0, 0].plot(history.history['val_categorical_accuracy_metric'], label='Validation Accuracy', linewidth=2, color='#A23B72')\nbest_val_acc = max(history.history['val_categorical_accuracy_metric'])\nbest_epoch = history.history['val_categorical_accuracy_metric'].index(best_val_acc)\naxes[0, 0].axhline(y=0.89, color='red', linestyle='--', alpha=0.5, label='89% Target')\naxes[0, 0].scatter(best_epoch, best_val_acc, color='gold', s=200, zorder=5, \n                   edgecolors='black', linewidths=2, label=f'Best: {best_val_acc:.2%}')\naxes[0, 0].set_xlabel('Epoch', fontsize=12)\naxes[0, 0].set_ylabel('Accuracy', fontsize=12)\naxes[0, 0].set_title('Training and Validation Accuracy', fontsize=13, fontweight='bold')\naxes[0, 0].legend(fontsize=10)\naxes[0, 0].grid(True, alpha=0.3)\naxes[0, 0].set_ylim([0, 1.05])\n\n# 2. Training and Validation Loss\naxes[0, 1].plot(history.history['loss'], label='Train Loss', linewidth=2, color='#F18F01')\naxes[0, 1].plot(history.history['val_loss'], label='Validation Loss', linewidth=2, color='#C73E1D')\naxes[0, 1].set_xlabel('Epoch', fontsize=12)\naxes[0, 1].set_ylabel('Loss', fontsize=12)\naxes[0, 1].set_title('Training and Validation Loss', fontsize=13, fontweight='bold')\naxes[0, 1].legend(fontsize=10)\naxes[0, 1].grid(True, alpha=0.3)\n\n# 3. Train Accuracy Progress\naxes[1, 0].plot(history.history['categorical_accuracy_metric'], label='Accuracy', linewidth=2, color='#06A77D')\naxes[1, 0].fill_between(range(len(history.history['categorical_accuracy_metric'])), \n                         history.history['categorical_accuracy_metric'], alpha=0.3, color='#06A77D')\naxes[1, 0].set_xlabel('Epoch', fontsize=12)\naxes[1, 0].set_ylabel('Accuracy', fontsize=12)\naxes[1, 0].set_title('Train Accuracy Progress', fontsize=13, fontweight='bold')\naxes[1, 0].grid(True, alpha=0.3)\n\n# 4. Accuracy Gap\naccuracy_gap = np.array(history.history['categorical_accuracy_metric']) - np.array(history.history['val_categorical_accuracy_metric'])\naxes[1, 1].plot(accuracy_gap, linewidth=2, color='#D62839')\naxes[1, 1].axhline(y=0, color='black', linestyle='--', alpha=0.5)\naxes[1, 1].axhline(y=0.05, color='orange', linestyle=':', alpha=0.5, label='5% threshold')\naxes[1, 1].fill_between(range(len(accuracy_gap)), accuracy_gap, 0, alpha=0.3, color='#D62839')\naxes[1, 1].set_xlabel('Epoch', fontsize=12)\naxes[1, 1].set_ylabel('Gap', fontsize=12)\naxes[1, 1].set_title('Train-Validation Accuracy Gap', fontsize=13, fontweight='bold')\naxes[1, 1].legend(fontsize=10)\naxes[1, 1].grid(True, alpha=0.3)\n\n# 5. Two-Stage Progress\nstage1_epochs = len(history_stage1.history['val_categorical_accuracy_metric'])\nstage2_epochs = len(history_stage2.history['val_categorical_accuracy_metric'])\naxes[0, 2].plot(range(stage1_epochs), history_stage1.history['val_categorical_accuracy_metric'], \n                label='Stage 1', linewidth=2, color='#3498db')\naxes[0, 2].plot(range(stage1_epochs, stage1_epochs + stage2_epochs), \n                history_stage2.history['val_categorical_accuracy_metric'], \n                label='Stage 2 (Fine-tune)', linewidth=2, color='#e74c3c')\naxes[0, 2].axvline(x=stage1_epochs, color='gray', linestyle='--', alpha=0.5, label='Fine-tune Start')\naxes[0, 2].axhline(y=0.89, color='red', linestyle='--', alpha=0.5, label='89% Target')\naxes[0, 2].set_xlabel('Epoch', fontsize=12)\naxes[0, 2].set_ylabel('Validation Accuracy', fontsize=12)\naxes[0, 2].set_title('Two-Stage Training Progress', fontsize=13, fontweight='bold')\naxes[0, 2].legend(fontsize=9)\naxes[0, 2].grid(True, alpha=0.3)\n\n# 6. Performance Summary\naxes[1, 2].axis('off')\nsummary_text = f\"\"\"\nULTRA-FAST TRAINING SUMMARY\n\nStage 1 (Initial):\n  Best: {stage1_best:.4f} ({stage1_best*100:.2f}%)\n  Time: {stage1_time:.2f} hours\n\nStage 2 (Fine-tune):\n  Best: {stage2_best:.4f} ({stage2_best*100:.2f}%)\n  Time: {stage2_time:.2f} hours\n  Improvement: +{improvement:.2f}pp\n\nOverall:\n  Best: {best_val_acc:.4f} ({best_val_acc*100:.2f}%)\n  Final: {final_val_acc:.4f} ({final_val_acc*100:.2f}%)\n  \nTotal Time: {total_time:.2f} hours\nRemaining: {12 - total_time:.2f} hours\n\nTarget: 89% Balanced Accuracy\nStatus: {'✓ ACHIEVED' if best_val_acc >= 0.89 else f'{best_val_acc*100:.1f}%'}\n\"\"\"\naxes[1, 2].text(0.1, 0.5, summary_text, fontsize=11, family='monospace',\n                verticalalignment='center')\n\nplt.tight_layout()\nplt.savefig('training_results_ultrafast.png', dpi=150, bbox_inches='tight')\nplt.show()\n\nprint(f\"\\n✓ Visualization saved as 'training_results_ultrafast.png'\")","metadata":{"execution":{"iopub.execute_input":"2026-01-11T23:18:54.929976Z","iopub.status.busy":"2026-01-11T23:18:54.929042Z","iopub.status.idle":"2026-01-11T23:18:56.615563Z","shell.execute_reply":"2026-01-11T23:18:56.614278Z"},"papermill":{"duration":1.734168,"end_time":"2026-01-11T23:18:56.618391","exception":false,"start_time":"2026-01-11T23:18:54.884223","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"f31d6afe","cell_type":"code","source":"# ======================================================================\n# 📤 GENERATING SUBMISSION FILE - FIXED VERSION\n# ======================================================================\n\nimport tensorflow as tf\nimport keras\nimport numpy as np\nimport pandas as pd\nimport os\nimport cv2\nimport pydicom\nfrom tensorflow.keras import layers, models, applications\n\nprint(\"=\"*70)\nprint(\"📤 GENERATING SUBMISSION FILE\")\nprint(\"=\"*70)\n\n# Constants needed for model reconstruction\nIMG_SIZE = 192\nNUM_SLICES = 2\nDROPOUT_RATE = 0.3\nWEIGHT_DECAY = 1e-4\n\n# Define custom classes and functions\nclass BalancedFocalLoss(tf.keras.losses.Loss):\n    def __init__(self, alpha=0.25, gamma=2.0, label_smoothing=0.03):\n        super().__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.label_smoothing = label_smoothing\n    \n    def call(self, y_true, y_pred):\n        num_classes = tf.shape(y_true)[-1]\n        y_true = y_true * (1.0 - self.label_smoothing) + (self.label_smoothing / tf.cast(num_classes, tf.float32))\n        \n        y_pred = tf.clip_by_value(y_pred, 1e-7, 1.0 - 1e-7)\n        ce_loss = -y_true * tf.math.log(y_pred)\n        focal_weight = self.alpha * tf.math.pow(1.0 - y_pred, self.gamma)\n        focal_loss = focal_weight * ce_loss\n        return tf.reduce_mean(tf.reduce_sum(focal_loss, axis=-1))\n\nclass WarmupCosineDecay(tf.keras.optimizers.schedules.LearningRateSchedule):\n    def __init__(self, initial_lr, warmup_steps, total_steps, alpha=0.0):\n        super().__init__()\n        self.initial_lr = initial_lr\n        self.warmup_steps = warmup_steps\n        self.total_steps = total_steps\n        self.alpha = alpha\n    \n    def __call__(self, step):\n        step = tf.cast(step, tf.float32)\n        warmup_steps = tf.cast(self.warmup_steps, tf.float32)\n        total_steps = tf.cast(self.total_steps, tf.float32)\n        \n        warmup_lr = self.initial_lr * (step / warmup_steps)\n        \n        progress = (step - warmup_steps) / (total_steps - warmup_steps)\n        cosine_decay = 0.5 * (1.0 + tf.cos(np.pi * progress))\n        decay_lr = self.alpha + (self.initial_lr - self.alpha) * cosine_decay\n        \n        return tf.cond(step < warmup_steps, lambda: warmup_lr, lambda: decay_lr)\n    \n    def get_config(self):\n        return {\n            \"initial_lr\": self.initial_lr,\n            \"warmup_steps\": self.warmup_steps,\n            \"total_steps\": self.total_steps,\n            \"alpha\": self.alpha,\n        }\n\ndef categorical_accuracy_metric(y_true, y_pred):\n    return tf.keras.metrics.categorical_accuracy(\n        tf.reshape(y_true, [-1, 3]), \n        tf.reshape(y_pred, [-1, 3])\n    )\n\ndef build_mvcnn_ultrafast(num_classes=25):\n    view1_input = layers.Input(shape=(NUM_SLICES, IMG_SIZE, IMG_SIZE, 3), name=\"view_sagittal_t1\")\n    view2_input = layers.Input(shape=(NUM_SLICES, IMG_SIZE, IMG_SIZE, 3), name=\"view_sagittal_t2\")\n    view3_input = layers.Input(shape=(NUM_SLICES, IMG_SIZE, IMG_SIZE, 3), name=\"view_axial_t2\")\n    \n    # CRITICAL: Use MobileNetV3Small (2.5M params vs 5.4M - 2× faster)\n    try:\n        backbone = applications.MobileNetV3Small(\n            include_top=False,\n            weights='imagenet',\n            input_shape=(IMG_SIZE, IMG_SIZE, 3),\n            pooling='avg',\n            minimalistic=False,\n            dropout_rate=0.2  # Built-in dropout\n        )\n    except Exception as e:\n        print(f\"⚠️ Could not download pretrained weights: {e}. Falling back to weights=None.\")\n        backbone = applications.MobileNetV3Small(\n            include_top=False,\n            weights=None,\n            input_shape=(IMG_SIZE, IMG_SIZE, 3),\n            pooling='avg',\n            minimalistic=False,\n            dropout_rate=0.2  # Built-in dropout\n        )\n    \n    # AGGRESSIVE: Train only last 20 layers (was 30)\n    for layer in backbone.layers[:-20]:\n        layer.trainable = False\n    for layer in backbone.layers[-20:]:\n        layer.trainable = True\n        \n    # Simplified attention (no shared layer)\n    def process_view(view_input, view_name):\n        features = layers.TimeDistributed(backbone, name=f'{view_name}_features')(view_input)\n        \n        # Simple attention\n        attention_weights = layers.Dense(1, activation='sigmoid')(features)\n        attention_weights = layers.Softmax(axis=1)(attention_weights)\n        \n        weighted = layers.Multiply()([features, attention_weights])\n        # Manually specify output shape for Lambda layer to fix serialization issue\n        pooled = layers.Lambda(lambda x: tf.reduce_sum(x, axis=1), output_shape=(None, 960 if backbone.name == 'MobilenetV3large' else 576))(weighted)\n        \n        return pooled\n    \n    # Process all views\n    feat1 = process_view(view1_input, 'view1')\n    feat2 = process_view(view2_input, 'view2')\n    feat3 = process_view(view3_input, 'view3')\n    \n    # SIMPLIFIED: Skip multi-head attention, just concatenate\n    merged = layers.Concatenate()([feat1, feat2, feat3])\n    \n    # REDUCED: Smaller classification head\n    x = layers.Dense(512, kernel_regularizer=tf.keras.regularizers.l2(WEIGHT_DECAY))(merged)\n    x = layers.BatchNormalization()(x)\n    x = layers.Activation('relu')(x)\n    x = layers.Dropout(DROPOUT_RATE)(x)\n    \n    x = layers.Dense(256, kernel_regularizer=tf.keras.regularizers.l2(WEIGHT_DECAY))(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Activation('relu')(x)\n    x = layers.Dropout(DROPOUT_RATE * 0.7)(x)\n    \n    output = layers.Dense(num_classes*3, activation='softmax',\n                         kernel_regularizer=tf.keras.regularizers.l2(WEIGHT_DECAY),\n                         dtype='float32')(x)\n    output = layers.Reshape((num_classes, 3))(output)\n    \n    model = models.Model(inputs=[view1_input, view2_input, view3_input], outputs=output)\n    return model\n\n# Load sample submission\nsample_submission = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/sample_submission.csv')\nprint(f\"✓ Sample submission loaded: {len(sample_submission)} rows\")\nprint(f\"✓ Columns: {list(sample_submission.columns)}\")\n# 25 rows = 1 study * 25 conditions.\nnum_labels = 25 \n\n# Get test studies\ntest_series_desc_df = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_series_descriptions.csv')\ntest_studies = test_series_desc_df['study_id'].unique()\nprint(f\"✓ Test studies: {len(test_studies)}\")\n\n# Define test data directory\nTEST_DATA_DIR = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_images/'\n\n# Function to load DICOM images\ndef load_dicom_image(path, augment=False):\n    \"\"\"ULTRA-FAST DICOM loading - minimal preprocessing\"\"\"\n    try:\n        dcm = pydicom.dcmread(path)\n        img = dcm.pixel_array.astype(np.float32)\n    except:\n        return np.zeros((IMG_SIZE, IMG_SIZE, 3), dtype=np.float32)\n    \n    # Fast normalization\n    img = (img - img.min()) / (img.max() - img.min() + 1e-7)\n    img = cv2.resize(img, (IMG_SIZE, IMG_SIZE), interpolation=cv2.INTER_AREA)\n    \n    # Convert to 3 channels\n    img = np.stack([img] * 3, axis=-1)\n    \n    # MINIMAL AUGMENTATION: Only horizontal flip (fastest)\n    if augment and np.random.rand() > 0.5:\n        img = cv2.flip(img, 1)\n    \n    return img.astype(np.float32)\n\n# Function to load test images (same as training but for test data)\ndef load_study_images_for_test(study_id, augment=False):\n    \"\"\"Load images for test studies\"\"\"\n    row = test_series_desc_df[test_series_desc_df['study_id'] == study_id]\n    views = {}\n    for view_type in ['Sagittal T1', 'Sagittal T2/STIR', 'Axial T2']:\n        series = row[row['series_description'].str.contains(view_type, case=False, na=False)]\n        views[view_type] = series['series_id'].values[0] if len(series) > 0 else np.nan\n    \n    images_by_view = []\n    for view in ['Sagittal T1', 'Sagittal T2/STIR', 'Axial T2']:\n        series_id = views[view]\n        view_imgs = []\n        \n        if pd.isna(series_id):\n            view_imgs = [np.zeros((IMG_SIZE, IMG_SIZE, 3)) for _ in range(NUM_SLICES)]\n        else:\n            series_path = os.path.join(TEST_DATA_DIR, str(study_id), str(series_id))\n            if not os.path.exists(series_path):\n                view_imgs = [np.zeros((IMG_SIZE, IMG_SIZE, 3)) for _ in range(NUM_SLICES)]\n            else:\n                instances = sorted(os.listdir(series_path))\n                if instances:\n                    num_instances = len(instances)\n                    slice_indices = np.linspace(0, num_instances-1, NUM_SLICES, dtype=int)\n                    for idx in slice_indices:\n                        img_path = os.path.join(series_path, instances[idx])\n                        try:\n                            view_imgs.append(load_dicom_image(img_path, augment=augment))\n                        except:\n                            view_imgs.append(np.zeros((IMG_SIZE, IMG_SIZE, 3)))\n                else:\n                    view_imgs = [np.zeros((IMG_SIZE, IMG_SIZE, 3)) for _ in range(NUM_SLICES)]\n        \n        images_by_view.append(np.array(view_imgs))\n    \n    # Return in the format expected by model: [view1, view2, view3]\n    return [np.expand_dims(images_by_view[0], axis=0),\n            np.expand_dims(images_by_view[1], axis=0),\n            np.expand_dims(images_by_view[2], axis=0)]\n\n# Initialize model and load weights\nprint(\"\\n✓ Rebuilding model logic to bypass Lambda deserialization issue...\")\ntry:\n    # Build model using same architecture\n    model = build_mvcnn_ultrafast(num_classes=25)\n    \n    # Load weights\n    print(\"✓ Loading weights from best_model_finetuned_ultrafast.keras...\")\n    model.load_weights('best_model_finetuned_ultrafast.keras')\n    print(\"✓ Model weights loaded successfully!\")\n    \nexcept Exception as e:\n    print(f\"⚠️ Failed to load weights: {e}\")\n    print(\"Attempting legacy load_model...\")\n    # Fallback to load_model if needed\n    keras.config.enable_unsafe_deserialization()\n    model = tf.keras.models.load_model(\n        'best_model_finetuned_ultrafast.keras',\n        custom_objects={\n            'categorical_accuracy_metric': categorical_accuracy_metric,\n            'BalancedFocalLoss': BalancedFocalLoss,\n            'WarmupCosineDecay': WarmupCosineDecay\n        },\n        compile=False,\n        safe_mode=False\n    )\n\n# Generate predictions\nprint(\"\\n✓ Generating predictions...\")\npredictions_list = []\n\nfor study_id in test_studies:\n    try:\n        # Load test images\n        images_by_view = load_study_images_for_test(study_id, augment=False)\n        \n        # Predict\n        prediction = model.predict(images_by_view, verbose=0)\n        \n        # prediction shape: (1, 25, 3)\n        # We need to flatten this to match the submission rows for this study\n        # The submission file has 25 rows per study.\n        # We'll just collect the (25, 3) array.\n        predictions_list.append(prediction[0]) \n        \n        if len(predictions_list) % 100 == 0:\n            print(f\"  Processed {len(predictions_list)}/{len(test_studies)} studies...\")\n    except Exception as e:\n        print(f\"⚠️ Error processing study {study_id}: {e}\")\n        # Default prediction: equal probability for all classes (25 rows, 3 cols)\n        predictions_list.append(np.ones((25, 3)) / 3)\n\nprint(f\"✓ Predictions generated: {len(predictions_list)} studies\")\n\n# Format submission\n# Stack all predictions: shape (Total Studies * 25, 3)\nall_predictions = np.vstack(predictions_list)\n\nsubmission_df = sample_submission.copy()\n\n# Ensure length matches\nif len(submission_df) == len(all_predictions):\n    submission_df['normal_mild'] = all_predictions[:, 0]\n    submission_df['moderate'] = all_predictions[:, 1]\n    submission_df['severe'] = all_predictions[:, 2]\nelse:\n    print(f\"⚠️ Shape mismatch: DF {len(submission_df)} vs Preds {len(all_predictions)}. Using naive assignment.\")\n    # Fallback/Debugging\n    for i, col in enumerate(submission_df.columns[1:]):\n        submission_df[col] = 1.0/3\n\n# Save submission\nsubmission_df.to_csv('submission.csv', index=False)\nprint(\"\\n✓ Submission file saved: submission.csv\")\nprint(f\"✓ Shape: {submission_df.shape}\")\nprint(f\"✓ First 5 rows:\\n{submission_df.head()}\")\nprint(\"=\"*70)\nprint(\"✅ SUBMISSION READY FOR KAGGLE!\")","metadata":{},"outputs":[],"execution_count":null}]}