{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.13.15","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, numpy as np, pandas as pd, cv2, pydicom\nimport matplotlib.pyplot as plt, seaborn as sns\nfrom pathlib import Path\nfrom tqdm.notebook import tqdm\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers, Model\nfrom tensorflow.keras.applications import EfficientNetB0\nfrom tensorflow.keras.callbacks import (ModelCheckpoint, EarlyStopping,\n                                         ReduceLROnPlateau)\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import (classification_report, confusion_matrix,\n                              roc_auc_score, roc_curve)\nfrom sklearn.utils.class_weight import compute_class_weight\n\nprint(\"TF:\", tf.__version__)\nprint(\"GPU:\", tf.config.list_physical_devices('GPU'))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-10-04T13:42:49.776967Z","iopub.execute_input":"2026-10-04T13:42:49.777899Z","iopub.status.idle":"2026-10-04T13:43:14.103956Z","shell.execute_reply.started":"2026-10-04T13:42:49.777869Z","shell.execute_reply":"2026-10-04T13:43:14.103226Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-04T13:43:14.105191Z","iopub.execute_input":"2026-10-04T13:43:14.105954Z","iopub.status.idle":"2026-10-04T13:45:42.687899Z","shell.execute_reply.started":"2026-10-04T13:43:14.105927Z","shell.execute_reply":"2026-10-04T13:45:42.687055Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ROOT = Path('/kaggle/input/competitions/rsna-pneumonia-detection-challenge')\nWORK = Path('/kaggle/working')\nWORK.mkdir(exist_ok=True)\nIMG_SIZE = 224\nBATCH_SIZE = 32\n\nlab = pd.read_csv(ROOT / 'stage_2_train_labels.csv')\ncls = pd.read_csv(ROOT / 'stage_2_detailed_class_info.csv')\n\ndf = lab.groupby('patientId').agg(\n    target=('Target', 'max'),\n).reset_index()\n\nboxes = lab[lab.Target == 1].copy()\nboxes['area'] = boxes['width'].fillna(0) * boxes['height'].fillna(0)\nbox_area = boxes.groupby('patientId')['area'].sum().reset_index()\nbox_area.columns = ['patientId', 'box_area']\n\ndf = df.merge(box_area, on='patientId', how='left')\ndf['box_area'] = df['box_area'].fillna(0)\n\nprint(\"Total patients:\", len(df))\nprint(df['target'].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-04T13:45:42.689116Z","iopub.execute_input":"2026-10-04T13:45:42.689345Z","iopub.status.idle":"2026-10-04T13:45:43.085619Z","shell.execute_reply.started":"2026-10-04T13:45:42.689327Z","shell.execute_reply":"2026-10-04T13:45:43.084863Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pneu = df[df.target == 1]['box_area']\nq33 = pneu.quantile(0.33)\nq66 = pneu.quantile(0.66)\n\ndef assign_severity(row):\n    if row['target'] == 0:\n        return 'None'\n    elif row['box_area'] <= q33:\n        return 'Mild'\n    elif row['box_area'] <= q66:\n        return 'Moderate'\n    else:\n        return 'Severe'\n\ndf['severity'] = df.apply(assign_severity, axis=1)\nprint(df['severity'].value_counts())\nprint(f\"Thresholds — Mild≤{q33:.0f}, Moderate≤{q66:.0f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-04T13:45:43.087621Z","iopub.execute_input":"2026-10-04T13:45:43.088457Z","iopub.status.idle":"2026-10-04T13:45:43.252524Z","shell.execute_reply.started":"2026-10-04T13:45:43.088399Z","shell.execute_reply":"2026-10-04T13:45:43.251755Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df, temp_df = train_test_split(\n    df, test_size=0.30, stratify=df['target'], random_state=42\n)\nval_df, test_df = train_test_split(\n    temp_df, test_size=0.50, stratify=temp_df['target'], random_state=42\n)\n\nprint(f\"Train: {len(train_df)} | Val: {len(val_df)} | Test: {len(test_df)}\")\nprint(\"Train:\\n\", train_df['target'].value_counts())\nprint(\"Val:\\n\",   val_df['target'].value_counts())\nprint(\"Test:\\n\",  test_df['target'].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-04T13:45:43.253554Z","iopub.execute_input":"2026-10-04T13:45:43.25394Z","iopub.status.idle":"2026-10-04T13:45:43.287123Z","shell.execute_reply.started":"2026-10-04T13:45:43.253911Z","shell.execute_reply":"2026-10-04T13:45:43.286471Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_and_preprocess(patient_id):\n    path = ROOT / 'stage_2_train_images' / f'{patient_id}.dcm'\n    try:\n        dicom = pydicom.dcmread(str(path))\n        img = dicom.pixel_array.astype(np.float32)\n        img = cv2.normalize(img, None, 0, 255,\n                            cv2.NORM_MINMAX).astype(np.uint8)\n        clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))\n        img = clahe.apply(img)\n        img = cv2.resize(img, (IMG_SIZE, IMG_SIZE),\n                         interpolation=cv2.INTER_AREA)\n        img = np.stack([img, img, img], axis=-1)\n        return img.astype(np.float32)\n    except:\n        return np.zeros((IMG_SIZE, IMG_SIZE, 3), dtype=np.float32)\n\n# Test karo\nsample_img = load_and_preprocess(df['patientId'].iloc[0])\nprint(\"Image shape:\", sample_img.shape)\nprint(\"Min:\", sample_img.min(), \"Max:\", sample_img.max())\n\nplt.figure(figsize=(4,4))\nplt.imshow(sample_img[:,:,0].astype(np.uint8), cmap='gray')\nplt.title(\"CLAHE Sample\")\nplt.axis('off')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-04T13:45:43.288284Z","iopub.execute_input":"2026-10-04T13:45:43.288691Z","iopub.status.idle":"2026-10-04T13:45:43.566372Z","shell.execute_reply.started":"2026-10-04T13:45:43.288669Z","shell.execute_reply":"2026-10-04T13:45:43.565339Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"AUTOTUNE = tf.data.AUTOTUNE\n\n# Rotation layer bahar banao — ek baar hi\nrotation_layer = tf.keras.layers.RandomRotation(0.05)\n\ndef make_dataset(dataframe, shuffle=False, augment=False):\n    patient_ids = dataframe['patientId'].values.astype(str)\n    labels = dataframe['target'].values.astype(np.float32)\n\n    def load_image(pid, label):\n        img = tf.py_function(\n            lambda p: load_and_preprocess(p.numpy().decode()),\n            [pid], tf.float32\n        )\n        img.set_shape([IMG_SIZE, IMG_SIZE, 3])\n        return img, label\n\n    def augment_fn(img, label):\n        img = tf.image.random_flip_left_right(img)\n        img = tf.image.random_brightness(img, max_delta=20)\n        img = tf.image.random_contrast(img, 0.9, 1.1)\n        img = rotation_layer(tf.expand_dims(img, 0))[0]\n        return img, label\n\n    ds = tf.data.Dataset.from_tensor_slices((patient_ids, labels))\n    if shuffle:\n        ds = ds.shuffle(buffer_size=len(dataframe), seed=42)\n    ds = ds.map(load_image, num_parallel_calls=AUTOTUNE)\n    if augment:\n        ds = ds.map(augment_fn, num_parallel_calls=AUTOTUNE)\n    ds = ds.batch(BATCH_SIZE).prefetch(AUTOTUNE)\n    return ds\n\ntrain_ds = make_dataset(train_df, shuffle=True,  augment=True)\nval_ds   = make_dataset(val_df,   shuffle=False, augment=False)\ntest_ds  = make_dataset(test_df,  shuffle=False, augment=False)\n\nprint(\"Datasets ready!\")\nprint(f\"Train batches: {len(train_ds)}\")\nprint(f\"Val batches:   {len(val_ds)}\")\nprint(f\"Test batches:  {len(test_ds)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-04T13:45:43.567656Z","iopub.execute_input":"2026-10-04T13:45:43.568005Z","iopub.status.idle":"2026-10-04T13:45:46.46511Z","shell.execute_reply.started":"2026-10-04T13:45:43.567974Z","shell.execute_reply":"2026-10-04T13:45:46.464388Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def channel_attention(x, ratio=8):\n    filters = x.shape[-1]\n    avg = layers.GlobalAveragePooling2D(keepdims=True)(x)\n    max_ = layers.GlobalMaxPooling2D(keepdims=True)(x)\n    def excite(t):\n        t = layers.Dense(filters // ratio, activation='relu',\n                         use_bias=False)(t)\n        t = layers.Dense(filters, activation='sigmoid',\n                         use_bias=False)(t)\n        return t\n    scale = layers.Add()([excite(avg), excite(max_)])\n    return layers.Multiply()([x, scale])\n\ndef spatial_attention(x):\n    # keras.ops use karo tf ke bajay\n    avg = keras.ops.mean(x, axis=-1, keepdims=True)\n    max_ = keras.ops.max(x, axis=-1, keepdims=True)\n    concat = layers.Concatenate(axis=-1)([avg, max_])\n    out = layers.Conv2D(1, kernel_size=7, padding='same',\n                        activation='sigmoid')(concat)\n    return layers.Multiply()([x, out])\n\ndef build_model():\n    inputs = keras.Input(shape=(IMG_SIZE, IMG_SIZE, 3))\n    base = EfficientNetB0(\n        include_top=False,\n        weights='imagenet',\n        input_tensor=inputs\n    )\n    base.trainable = False\n    x = base.output\n    x = channel_attention(x)\n    x = spatial_attention(x)\n    x = layers.GlobalAveragePooling2D()(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Dense(256, activation='relu')(x)\n    x = layers.Dropout(0.4)(x)\n    x = layers.Dense(64, activation='relu')(x)\n    x = layers.Dropout(0.3)(x)\n    outputs = layers.Dense(1, activation='sigmoid')(x)\n    model = Model(inputs, outputs)\n    return model, base\n\nmodel, base_model = build_model()\nmodel.summary()\nprint(f\"\\nTotal params: {model.count_params():,}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-04T13:45:46.465993Z","iopub.execute_input":"2026-10-04T13:45:46.466204Z","iopub.status.idle":"2026-10-04T13:45:49.512657Z","shell.execute_reply.started":"2026-10-04T13:45:46.466186Z","shell.execute_reply":"2026-10-04T13:45:49.511811Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cw = compute_class_weight('balanced',\n                           classes=np.array([0, 1]),\n                           y=train_df['target'].values)\nclass_weights = {0: float(cw[0]), 1: float(cw[1])}\nprint(\"Class weights:\", class_weights)\n\ncallbacks_p1 = [\n    ModelCheckpoint('/kaggle/working/best_phase1.keras',\n                    monitor='val_auc', mode='max',\n                    save_best_only=True, verbose=1),\n    EarlyStopping(monitor='val_auc', patience=5,\n                  mode='max', restore_best_weights=True, verbose=1),\n    ReduceLROnPlateau(monitor='val_auc', factor=0.5,\n                      patience=3, mode='max', verbose=1)\n]\n\nmodel.compile(\n    optimizer=keras.optimizers.Adam(1e-3),\n    loss='binary_crossentropy',\n    metrics=[\n        keras.metrics.AUC(name='auc'),\n        keras.metrics.BinaryAccuracy(name='accuracy'),\n        keras.metrics.Precision(name='precision'),\n        keras.metrics.Recall(name='recall')\n    ]\n)\n\nprint(\"\\nPhase 1 training shuru...\")\nhistory1 = model.fit(\n    train_ds,\n    validation_data=val_ds,\n    epochs=15,\n    class_weight=class_weights,\n    callbacks=callbacks_p1,\n    verbose=1\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-04T13:45:49.513802Z","iopub.execute_input":"2026-10-04T13:45:49.514151Z","iopub.status.idle":"2026-10-04T14:56:45.971875Z","shell.execute_reply.started":"2026-10-04T13:45:49.514129Z","shell.execute_reply":"2026-10-04T14:56:45.971037Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"base_model.trainable = True\nfreeze_until = int(len(base_model.layers) * 0.70)\nfor layer in base_model.layers[:freeze_until]:\n    layer.trainable = False\nfor layer in base_model.layers[freeze_until:]:\n    layer.trainable = True\n\nprint(f\"Base layers: {len(base_model.layers)}\")\nprint(f\"Frozen: {freeze_until}, Trainable: {len(base_model.layers) - freeze_until}\")\n\ncallbacks_p2 = [\n    ModelCheckpoint('/kaggle/working/best_final.keras',\n                    monitor='val_auc', mode='max',\n                    save_best_only=True, verbose=1),\n    EarlyStopping(monitor='val_auc', patience=7,\n                  mode='max', restore_best_weights=True, verbose=1),\n    ReduceLROnPlateau(monitor='val_auc', factor=0.3,\n                      patience=3, mode='max', verbose=1)\n]\n\nmodel.compile(\n    optimizer=keras.optimizers.Adam(1e-4),\n    loss='binary_crossentropy',\n    metrics=[\n        keras.metrics.AUC(name='auc'),\n        keras.metrics.BinaryAccuracy(name='accuracy'),\n        keras.metrics.Precision(name='precision'),\n        keras.metrics.Recall(name='recall')\n    ]\n)\n\nprint(\"\\nPhase 2 fine-tuning shuru...\")\nhistory2 = model.fit(\n    train_ds,\n    validation_data=val_ds,\n    epochs=20,\n    class_weight=class_weights,\n    callbacks=callbacks_p2,\n    verbose=1\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-04T14:56:45.974608Z","iopub.execute_input":"2026-10-04T14:56:45.974939Z","iopub.status.idle":"2026-10-04T15:47:12.954173Z","shell.execute_reply.started":"2026-10-04T14:56:45.97491Z","shell.execute_reply":"2026-10-04T15:47:12.953177Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nfrom sklearn.metrics import accuracy_score, f1_score\nimport numpy as np\n\nbest_model = keras.models.load_model('/kaggle/working/best_final.keras')\n\ny_true_val, y_pred_val = [], []\nfor imgs, labels in val_ds:\n    preds = best_model.predict(imgs, verbose=0)\n    y_pred_val.extend(preds.flatten().tolist())\n    y_true_val.extend(labels.numpy().tolist())\n\ny_true_val = np.array(y_true_val)\ny_pred_val = np.array(y_pred_val)\n\nbest_thresh = 0.5\nbest_acc = 0\n\nprint(f\"Threshold  Accuracy    F1\")\nprint(\"-\" * 35)\nfor t in np.arange(0.1, 0.9, 0.05):\n    preds = (y_pred_val >= t).astype(int)\n    acc = accuracy_score(y_true_val, preds)\n    f1 = f1_score(y_true_val, preds)\n    print(f\"  {t:.2f}      {acc*100:.2f}%    {f1:.4f}\")\n    if acc > best_acc:\n        best_acc = acc\n        best_thresh = t\n\nprint(f\"\\nBest threshold: {best_thresh:.2f}\")\nprint(f\"Best val accuracy: {best_acc*100:.2f}%\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-04T15:47:12.955481Z","iopub.execute_input":"2026-10-04T15:47:12.95582Z","iopub.status.idle":"2026-10-04T15:48:13.557527Z","shell.execute_reply.started":"2026-10-04T15:47:12.955786Z","shell.execute_reply":"2026-10-04T15:48:13.556749Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import (confusion_matrix, roc_auc_score, roc_curve,\n                              accuracy_score, precision_score, \n                              recall_score, f1_score)\n\nBEST_THRESH = 0.70\n\ny_true_test, y_pred_prob_test = [], []\nfor imgs, labels in test_ds:\n    preds = best_model.predict(imgs, verbose=0)\n    y_pred_prob_test.extend(preds.flatten().tolist())\n    y_true_test.extend(labels.numpy().tolist())\n\ny_true_test = np.array(y_true_test)\ny_pred_prob_test = np.array(y_pred_prob_test)\ny_pred_test = (y_pred_prob_test >= BEST_THRESH).astype(int)\n\ntn, fp, fn, tp = confusion_matrix(y_true_test, y_pred_test).ravel()\nspecificity = tn / (tn + fp)\n\nprint(\"=\" * 45)\nprint(\"       FINAL TEST RESULTS\")\nprint(\"=\" * 45)\nprint(f\"  Accuracy   : {accuracy_score(y_true_test, y_pred_test)*100:.2f}%\")\nprint(f\"  Precision  : {precision_score(y_true_test, y_pred_test)*100:.2f}%\")\nprint(f\"  Recall     : {recall_score(y_true_test, y_pred_test)*100:.2f}%\")\nprint(f\"  Specificity: {specificity*100:.2f}%\")\nprint(f\"  F1-Score   : {f1_score(y_true_test, y_pred_test)*100:.2f}%\")\nprint(f\"  ROC-AUC    : {roc_auc_score(y_true_test, y_pred_prob_test):.4f}\")\nprint(\"=\" * 45)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-04T15:48:13.558633Z","iopub.execute_input":"2026-10-04T15:48:13.559025Z","iopub.status.idle":"2026-10-04T15:49:32.855577Z","shell.execute_reply.started":"2026-10-04T15:48:13.559001Z","shell.execute_reply":"2026-10-04T15:49:32.854727Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Confusion Matrix\ncm = confusion_matrix(y_true_test, y_pred_test)\nplt.figure(figsize=(6,5))\nsns.heatmap(cm, annot=True, fmt='d', cmap='Blues',\n            xticklabels=['Normal','Pneumonia'],\n            yticklabels=['Normal','Pneumonia'])\nplt.title('Confusion Matrix — Test Set')\nplt.ylabel('Actual')\nplt.xlabel('Predicted')\nplt.tight_layout()\nplt.savefig('/kaggle/working/confusion_matrix.png', dpi=150)\nplt.show()\n\n# ROC Curve\nfpr, tpr, _ = roc_curve(y_true_test, y_pred_prob_test)\nauc_val = roc_auc_score(y_true_test, y_pred_prob_test)\nplt.figure(figsize=(6,5))\nplt.plot(fpr, tpr, 'b-', linewidth=2, label=f'AUC = {auc_val:.4f}')\nplt.plot([0,1],[0,1],'r--', label='Random')\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.title('ROC Curve — Test Set')\nplt.legend()\nplt.grid(True, alpha=0.3)\nplt.tight_layout()\nplt.savefig('/kaggle/working/roc_curve.png', dpi=150)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-04T15:49:32.856767Z","iopub.execute_input":"2026-10-04T15:49:32.857127Z","iopub.status.idle":"2026-10-04T15:49:33.487111Z","shell.execute_reply.started":"2026-10-04T15:49:32.857096Z","shell.execute_reply":"2026-10-04T15:49:33.486069Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_gradcam(model, img_array, last_conv='top_conv'):\n    grad_model = tf.keras.Model(\n        inputs=model.inputs,\n        outputs=[model.get_layer(last_conv).output, model.output]\n    )\n    with tf.GradientTape() as tape:\n        conv_out, preds = grad_model(tf.expand_dims(img_array, 0))\n        loss = preds[:, 0]\n    grads = tape.gradient(loss, conv_out)\n    pooled = tf.reduce_mean(grads, axis=(0,1,2))\n    heatmap = conv_out[0] @ pooled[..., tf.newaxis]\n    heatmap = tf.squeeze(heatmap)\n    heatmap = tf.maximum(heatmap, 0)\n    heatmap = heatmap / (tf.math.reduce_max(heatmap) + 1e-8)\n    return heatmap.numpy()\n\ndef overlay_heatmap(img, heatmap, alpha=0.4):\n    h = cv2.resize(heatmap, (IMG_SIZE, IMG_SIZE))\n    h = np.uint8(255 * h)\n    colormap = cv2.applyColorMap(h, cv2.COLORMAP_JET)\n    colormap = cv2.cvtColor(colormap, cv2.COLOR_BGR2RGB)\n    base = np.stack([img[:,:,0]]*3, axis=-1) / 255.0\n    overlay = base + alpha * colormap / 255.0\n    overlay = overlay / overlay.max()\n    return (overlay * 255).astype(np.uint8)\n\n# 6 pneumonia samples\npneu_samples = test_df[test_df.target == 1].head(6)\nfig, axes = plt.subplots(6, 3, figsize=(12, 22))\n\nfor idx, (_, row) in enumerate(pneu_samples.iterrows()):\n    img = load_and_preprocess(row['patientId'])\n    heatmap = get_gradcam(best_model, img)\n    overlay = overlay_heatmap(img, heatmap)\n    prob = best_model.predict(tf.expand_dims(img, 0), verbose=0)[0][0]\n\n    axes[idx,0].imshow(img[:,:,0], cmap='gray')\n    axes[idx,0].set_title('Original + CLAHE', fontsize=9)\n    axes[idx,0].axis('off')\n\n    axes[idx,1].imshow(heatmap, cmap='jet')\n    axes[idx,1].set_title('Grad-CAM Heatmap', fontsize=9)\n    axes[idx,1].axis('off')\n\n    axes[idx,2].imshow(overlay)\n    axes[idx,2].set_title(\n        f'Confidence: {prob:.2f} | {row[\"severity\"]}', fontsize=9)\n    axes[idx,2].axis('off')\n\nplt.suptitle('Grad-CAM Explainability — Pneumonia Cases', fontsize=13)\nplt.tight_layout()\nplt.savefig('/kaggle/working/gradcam.png', dpi=150)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-04T15:49:33.488315Z","iopub.execute_input":"2026-10-04T15:49:33.488781Z","iopub.status.idle":"2026-10-04T15:49:54.197186Z","shell.execute_reply.started":"2026-10-04T15:49:33.488743Z","shell.execute_reply":"2026-10-04T15:49:54.196175Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Severity distribution\nsev = test_df[test_df.target==1]['severity'].value_counts()\ncolors = ['#2ecc71','#f39c12','#e74c3c']\nsev.plot(kind='bar', color=colors, figsize=(6,4))\nplt.title('Pneumonia Severity Distribution — Test Set')\nplt.ylabel('Count')\nplt.xticks(rotation=0)\nplt.tight_layout()\nplt.savefig('/kaggle/working/severity.png', dpi=150)\nplt.show()\nprint(sev)\n\n# Model save\nbest_model.save('/kaggle/working/pneumonia_final_model.keras')\nprint(\"\\nModel saved!\")\nprint(\"\\nAll output files:\")\nfor f in Path('/kaggle/working').glob('*'):\n    print(f\" {f.name}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-04T15:49:54.198279Z","iopub.execute_input":"2026-10-04T15:49:54.198711Z","iopub.status.idle":"2026-10-04T15:49:55.429764Z","shell.execute_reply.started":"2026-10-04T15:49:54.198687Z","shell.execute_reply":"2026-10-04T15:49:55.428985Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cls_df = pd.read_csv(ROOT / 'stage_2_detailed_class_info.csv').drop_duplicates('patientId')\ndf_filtered = df.merge(cls_df, on='patientId')\n\ndf_clean = df_filtered[\n    df_filtered['class'].isin(['Normal', 'Lung Opacity'])\n].copy()\ndf_clean['target'] = (df_clean['class'] == 'Lung Opacity').astype(int)\n\nprint(df_clean['class'].value_counts())\nprint(f\"Total: {len(df_clean)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-04T15:49:55.430828Z","iopub.execute_input":"2026-10-04T15:49:55.431186Z","iopub.status.idle":"2026-10-04T15:49:55.506598Z","shell.execute_reply.started":"2026-10-04T15:49:55.431165Z","shell.execute_reply":"2026-10-04T15:49:55.505768Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Naya split\ntrain_df2, temp_df2 = train_test_split(\n    df_clean, test_size=0.30, stratify=df_clean['target'], random_state=42\n)\nval_df2, test_df2 = train_test_split(\n    temp_df2, test_size=0.50, stratify=temp_df2['target'], random_state=42\n)\n\nprint(f\"Train: {len(train_df2)} | Val: {len(val_df2)} | Test: {len(test_df2)}\")\nprint(\"Train:\\n\", train_df2['target'].value_counts())\n\n# Naya dataset\ntrain_ds2 = make_dataset(train_df2, shuffle=True,  augment=True)\nval_ds2   = make_dataset(val_df2,   shuffle=False, augment=False)\ntest_ds2  = make_dataset(test_df2,  shuffle=False, augment=False)\n\nprint(f\"\\nTrain batches: {len(train_ds2)}\")\nprint(f\"Val batches:   {len(val_ds2)}\")\nprint(f\"Test batches:  {len(test_ds2)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-04T15:49:55.507606Z","iopub.execute_input":"2026-10-04T15:49:55.507895Z","iopub.status.idle":"2026-10-04T15:49:55.664179Z","shell.execute_reply.started":"2026-10-04T15:49:55.507876Z","shell.execute_reply":"2026-10-04T15:49:55.663266Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Naya model fresh banao\nmodel2, base_model2 = build_model()\n\ncw2 = compute_class_weight('balanced',\n                           classes=np.array([0, 1]),\n                           y=train_df2['target'].values)\nclass_weights2 = {0: float(cw2[0]), 1: float(cw2[1])}\nprint(\"Class weights:\", class_weights2)\n\n# Phase 1\ncallbacks_p1_v2 = [\n    ModelCheckpoint('/kaggle/working/best_phase1_v2.keras',\n                    monitor='val_auc', mode='max',\n                    save_best_only=True, verbose=1),\n    EarlyStopping(monitor='val_auc', patience=5,\n                  mode='max', restore_best_weights=True, verbose=1),\n    ReduceLROnPlateau(monitor='val_auc', factor=0.5,\n                      patience=3, mode='max', verbose=1)\n]\n\nmodel2.compile(\n    optimizer=keras.optimizers.Adam(1e-3),\n    loss='binary_crossentropy',\n    metrics=[\n        keras.metrics.AUC(name='auc'),\n        keras.metrics.BinaryAccuracy(name='accuracy'),\n        keras.metrics.Precision(name='precision'),\n        keras.metrics.Recall(name='recall')\n    ]\n)\n\nprint(\"\\nPhase 1 training shuru...\")\nhistory1_v2 = model2.fit(\n    train_ds2,\n    validation_data=val_ds2,\n    epochs=15,\n    class_weight=class_weights2,\n    callbacks=callbacks_p1_v2,\n    verbose=1\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-04T15:49:55.665241Z","iopub.execute_input":"2026-10-04T15:49:55.666156Z","iopub.status.idle":"2026-10-04T16:28:14.358145Z","shell.execute_reply.started":"2026-10-04T15:49:55.666134Z","shell.execute_reply":"2026-10-04T16:28:14.357203Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"base_model2.trainable = True\nfreeze_until2 = int(len(base_model2.layers) * 0.70)\nfor layer in base_model2.layers[:freeze_until2]:\n    layer.trainable = False\nfor layer in base_model2.layers[freeze_until2:]:\n    layer.trainable = True\n\ncallbacks_p2_v2 = [\n    ModelCheckpoint('/kaggle/working/best_final_v2.keras',\n                    monitor='val_auc', mode='max',\n                    save_best_only=True, verbose=1),\n    EarlyStopping(monitor='val_auc', patience=7,\n                  mode='max', restore_best_weights=True, verbose=1),\n    ReduceLROnPlateau(monitor='val_auc', factor=0.3,\n                      patience=3, mode='max', verbose=1)\n]\n\nmodel2.compile(\n    optimizer=keras.optimizers.Adam(1e-4),\n    loss='binary_crossentropy',\n    metrics=[\n        keras.metrics.AUC(name='auc'),\n        keras.metrics.BinaryAccuracy(name='accuracy'),\n        keras.metrics.Precision(name='precision'),\n        keras.metrics.Recall(name='recall')\n    ]\n)\n\nprint(\"Phase 2 fine-tuning shuru...\")\nhistory2_v2 = model2.fit(\n    train_ds2,\n    validation_data=val_ds2,\n    epochs=20,\n    class_weight=class_weights2,\n    callbacks=callbacks_p2_v2,\n    verbose=1\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-04T16:28:14.359295Z","iopub.execute_input":"2026-10-04T16:28:14.359681Z","iopub.status.idle":"2026-10-04T17:21:00.906919Z","shell.execute_reply.started":"2026-10-04T16:28:14.359648Z","shell.execute_reply":"2026-10-04T17:21:00.905994Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Best model load karo\nbest_model2 = keras.models.load_model('/kaggle/working/best_final_v2.keras')\n\n# Threshold tuning pehle\ny_true_val2, y_pred_val2 = [], []\nfor imgs, labels in val_ds2:\n    preds = best_model2.predict(imgs, verbose=0)\n    y_pred_val2.extend(preds.flatten().tolist())\n    y_true_val2.extend(labels.numpy().tolist())\n\ny_true_val2 = np.array(y_true_val2)\ny_pred_val2 = np.array(y_pred_val2)\n\nbest_thresh2 = 0.5\nbest_acc2 = 0\nprint(f\"Threshold  Accuracy    F1\")\nprint(\"-\" * 35)\nfor t in np.arange(0.1, 0.9, 0.05):\n    preds = (y_pred_val2 >= t).astype(int)\n    acc = accuracy_score(y_true_val2, preds)\n    f1 = f1_score(y_true_val2, preds)\n    print(f\"  {t:.2f}      {acc*100:.2f}%    {f1:.4f}\")\n    if acc > best_acc2:\n        best_acc2 = acc\n        best_thresh2 = t\n\nprint(f\"\\nBest threshold: {best_thresh2:.2f}\")\nprint(f\"Best val accuracy: {best_acc2*100:.2f}%\")\n\n# Test evaluation\ny_true_test2, y_pred_prob_test2 = [], []\nfor imgs, labels in test_ds2:\n    preds = best_model2.predict(imgs, verbose=0)\n    y_pred_prob_test2.extend(preds.flatten().tolist())\n    y_true_test2.extend(labels.numpy().tolist())\n\ny_true_test2 = np.array(y_true_test2)\ny_pred_prob_test2 = np.array(y_pred_prob_test2)\ny_pred_test2 = (y_pred_prob_test2 >= best_thresh2).astype(int)\n\ntn, fp, fn, tp = confusion_matrix(y_true_test2, y_pred_test2).ravel()\nspecificity = tn / (tn + fp)\n\nprint(\"\\n\" + \"=\" * 45)\nprint(\"       FINAL TEST RESULTS\")\nprint(\"=\" * 45)\nprint(f\"  Accuracy   : {accuracy_score(y_true_test2, y_pred_test2)*100:.2f}%\")\nprint(f\"  Precision  : {precision_score(y_true_test2, y_pred_test2)*100:.2f}%\")\nprint(f\"  Recall     : {recall_score(y_true_test2, y_pred_test2)*100:.2f}%\")\nprint(f\"  Specificity: {specificity*100:.2f}%\")\nprint(f\"  F1-Score   : {f1_score(y_true_test2, y_pred_test2)*100:.2f}%\")\nprint(f\"  ROC-AUC    : {roc_auc_score(y_true_test2, y_pred_prob_test2):.4f}\")\nprint(\"=\" * 45)\n\n# Confusion Matrix\ncm2 = confusion_matrix(y_true_test2, y_pred_test2)\nplt.figure(figsize=(6,5))\nsns.heatmap(cm2, annot=True, fmt='d', cmap='Blues',\n            xticklabels=['Normal','Pneumonia'],\n            yticklabels=['Normal','Pneumonia'])\nplt.title('Confusion Matrix — Test Set (v2)')\nplt.ylabel('Actual')\nplt.xlabel('Predicted')\nplt.tight_layout()\nplt.savefig('/kaggle/working/confusion_matrix_v2.png', dpi=150)\nplt.show()\n\n# ROC Curve\nfpr2, tpr2, _ = roc_curve(y_true_test2, y_pred_prob_test2)\nauc_val2 = roc_auc_score(y_true_test2, y_pred_prob_test2)\nplt.figure(figsize=(6,5))\nplt.plot(fpr2, tpr2, 'b-', linewidth=2, label=f'AUC = {auc_val2:.4f}')\nplt.plot([0,1],[0,1],'r--', label='Random')\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.title('ROC Curve — Test Set (v2)')\nplt.legend()\nplt.grid(True, alpha=0.3)\nplt.tight_layout()\nplt.savefig('/kaggle/working/roc_curve_v2.png', dpi=150)\nplt.show()\n\n# Model save\nbest_model2.save('/kaggle/working/pneumonia_final_v2.keras')\nprint(\"\\nModel saved!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-04T17:21:00.908164Z","iopub.execute_input":"2026-10-04T17:21:00.908541Z","iopub.status.idle":"2026-10-04T17:22:15.404535Z","shell.execute_reply.started":"2026-10-04T17:21:00.908512Z","shell.execute_reply":"2026-10-04T17:22:15.403734Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pneu_samples2 = test_df2[test_df2.target == 1].head(6)\nfig, axes = plt.subplots(6, 3, figsize=(12, 22))\n\nfor idx, (_, row) in enumerate(pneu_samples2.iterrows()):\n    img = load_and_preprocess(row['patientId'])\n    heatmap = get_gradcam(best_model2, img)\n    overlay = overlay_heatmap(img, heatmap)\n    prob = best_model2.predict(tf.expand_dims(img, 0), verbose=0)[0][0]\n\n    axes[idx,0].imshow(img[:,:,0], cmap='gray')\n    axes[idx,0].set_title('Original + CLAHE', fontsize=9)\n    axes[idx,0].axis('off')\n\n    axes[idx,1].imshow(heatmap, cmap='jet')\n    axes[idx,1].set_title('Grad-CAM Heatmap', fontsize=9)\n    axes[idx,1].axis('off')\n\n    axes[idx,2].imshow(overlay)\n    axes[idx,2].set_title(\n        f'Conf: {prob:.2f} | {row[\"severity\"]}', fontsize=9)\n    axes[idx,2].axis('off')\n\nplt.suptitle('Grad-CAM — Pneumonia Cases (v2)', fontsize=13)\nplt.tight_layout()\nplt.savefig('/kaggle/working/gradcam_v2.png', dpi=150)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-04T17:22:15.405527Z","iopub.execute_input":"2026-10-04T17:22:15.405826Z","iopub.status.idle":"2026-10-04T17:22:29.029576Z","shell.execute_reply.started":"2026-10-04T17:22:15.405805Z","shell.execute_reply":"2026-10-04T17:22:29.028298Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 3 samples — 1 Normal, 1 Pneumonia Mild, 1 Pneumonia Severe\nnormal_sample = test_df2[test_df2.target == 0].iloc[0]\nmild_sample = test_df2[test_df2.target == 1][test_df2['severity'] == 'Mild'].iloc[0]\nsevere_sample = test_df2[test_df2.target == 1][test_df2['severity'] == 'Severe'].iloc[0]\n\nsamples = [\n    (normal_sample, 'Normal'),\n    (mild_sample, 'Pneumonia - Mild'),\n    (severe_sample, 'Pneumonia - Severe')\n]\n\nfig, axes = plt.subplots(3, 4, figsize=(16, 12))\n\nfor idx, (row, true_label) in enumerate(samples):\n    img = load_and_preprocess(row['patientId'])\n    prob = best_model2.predict(tf.expand_dims(img, 0), verbose=0)[0][0]\n    pred_label = 'Pneumonia' if prob >= 0.55 else 'Normal'\n    severity = row['severity'] if prob >= 0.55 else 'N/A'\n    heatmap = get_gradcam(best_model2, img)\n    overlay = overlay_heatmap(img, heatmap)\n\n    # Original\n    axes[idx,0].imshow(img[:,:,0], cmap='gray')\n    axes[idx,0].set_title('Input X-Ray\\n(CLAHE Enhanced)', fontsize=10)\n    axes[idx,0].axis('off')\n\n    # Grad-CAM\n    axes[idx,1].imshow(heatmap, cmap='jet')\n    axes[idx,1].set_title('Grad-CAM\\nHeatmap', fontsize=10)\n    axes[idx,1].axis('off')\n\n    # Overlay\n    axes[idx,2].imshow(overlay)\n    axes[idx,2].set_title('Attention\\nOverlay', fontsize=10)\n    axes[idx,2].axis('off')\n\n    # Result box\n    axes[idx,3].axis('off')\n    color = '#e74c3c' if pred_label == 'Pneumonia' else '#2ecc71'\n    axes[idx,3].add_patch(plt.Rectangle((0,0), 1, 1,\n                          facecolor=color, alpha=0.15,\n                          transform=axes[idx,3].transAxes))\n\n    result_text = (\n        f\"TRUE LABEL:\\n{true_label}\\n\\n\"\n        f\"PREDICTION:\\n{pred_label}\\n\\n\"\n        f\"CONFIDENCE:\\n{prob*100:.1f}%\\n\\n\"\n        f\"SEVERITY:\\n{severity}\\n\\n\"\n        f\"STATUS:\\n{'✓ CORRECT' if true_label.split(' - ')[0] == pred_label or (true_label == 'Normal' and pred_label == 'Normal') else '✗ INCORRECT'}\"\n    )\n    axes[idx,3].text(0.5, 0.5, result_text,\n                    ha='center', va='center',\n                    fontsize=11, fontweight='bold',\n                    transform=axes[idx,3].transAxes,\n                    bbox=dict(boxstyle='round', facecolor=color,\n                             alpha=0.3, edgecolor=color))\n\nplt.suptitle('Pneumonia Detection System — Sample Predictions',\n             fontsize=14, fontweight='bold', y=1.02)\nplt.tight_layout()\nplt.savefig('/kaggle/working/sample_predictions.png', dpi=150,\n            bbox_inches='tight')\nplt.show()\nprint(\"Sample predictions saved!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-04T17:22:29.031003Z","iopub.execute_input":"2026-10-04T17:22:29.031768Z","iopub.status.idle":"2026-10-04T17:22:34.712848Z","shell.execute_reply.started":"2026-10-04T17:22:29.031743Z","shell.execute_reply":"2026-10-04T17:22:34.712083Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Model already saved hai /kaggle/working/pneumonia_final_v2.keras\n# Kaggle Output section se download karo\n\nprint(\"Model path:\", '/kaggle/working/pneumonia_final_v2.keras')\nimport os\nsize = os.path.getsize('/kaggle/working/pneumonia_final_v2.keras')\nprint(f\"Model size: {size/1024/1024:.1f} MB\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-04T17:22:34.714183Z","iopub.execute_input":"2026-10-04T17:22:34.714557Z","iopub.status.idle":"2026-10-04T17:22:34.720555Z","shell.execute_reply.started":"2026-10-04T17:22:34.714534Z","shell.execute_reply":"2026-10-04T17:22:34.719769Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Seedha test_df2 se ek aisa patient lo jo index 200+ par hai\n# Ye wो patient hai jise model ne training me nahi dekha tha\nsample_row = test_df2.iloc[200]\nsample_id = sample_row['patientId']\n\nprint(f\"Testing on: {sample_id}\")\n\nimg = load_and_preprocess(sample_id)\nprob = best_model2.predict(tf.expand_dims(img, 0), verbose=0)[0][0]\npred = 'Pneumonia' if prob >= 0.55 else 'Normal'\n\nif pred == 'Pneumonia':\n    if prob < 0.70: severity = 'Mild'\n    elif prob < 0.85: severity = 'Moderate'\n    else: severity = 'Severe'\nelse:\n    severity = 'N/A'\n\nheatmap = get_gradcam(best_model2, img)\noverlay = overlay_heatmap(img, heatmap)\n\nfig, axes = plt.subplots(1, 4, figsize=(16, 4))\n\naxes[0].imshow(img[:,:,0], cmap='gray')\naxes[0].set_title('Original X-Ray', fontsize=10)\naxes[0].axis('off')\n\naxes[1].imshow(img[:,:,0], cmap='gray')\naxes[1].set_title('CLAHE Enhanced', fontsize=10)\naxes[1].axis('off')\n\naxes[2].imshow(overlay)\naxes[2].set_title('Grad-CAM Overlay', fontsize=10)\naxes[2].axis('off')\n\ncolor = '#e74c3c' if pred == 'Pneumonia' else '#2ecc71'\naxes[3].axis('off')\nresult_text = (\n    f\"PREDICTION:\\n{pred}\\n\\n\"\n    f\"CONFIDENCE:\\n{prob*100:.1f}%\\n\\n\"\n    f\"SEVERITY:\\n{severity}\\n\\n\"\n    f\"RECOMMENDATION:\\n\"\n    f\"{'Consult doctor\\nimmediately' if severity=='Severe' else 'Follow up\\nrecommended' if pred=='Pneumonia' else 'No action\\nneeded'}\"\n)\naxes[3].text(0.5, 0.5, result_text,\n            ha='center', va='center',\n            fontsize=11, fontweight='bold',\n            transform=axes[3].transAxes,\n            bbox=dict(boxstyle='round', facecolor=color, alpha=0.3))\n\nplt.suptitle(f'Test Patient: {sample_id}', fontsize=13, fontweight='bold')\nplt.tight_layout()\nplt.savefig('/kaggle/working/unseen_test.png', dpi=150)\nplt.show()\nprint(f\"Prediction : {pred}\")\nprint(f\"Confidence : {prob*100:.1f}%\")\nprint(f\"Severity   : {severity}\")\nprint(f\"True Label : {'Pneumonia' if sample_row['target']==1 else 'Normal'}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-04T17:22:34.721752Z","iopub.execute_input":"2026-10-04T17:22:34.722158Z","iopub.status.idle":"2026-10-04T17:22:36.416593Z","shell.execute_reply.started":"2026-10-04T17:22:34.722136Z","shell.execute_reply":"2026-10-04T17:22:36.415838Z"}},"outputs":[],"execution_count":null}]}