{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":10338,"databundleVersionId":862042,"isSourceIdPinned":false}],"dockerImageVersionId":31328,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"410dc2b9","cell_type":"markdown","source":"# Patent-Free Traditional ML Pipeline for Pneumonia Detection","metadata":{}},{"id":"3dacfc6a","cell_type":"code","source":"%pip install -q pydicom imbalanced-learn xgboost lightgbm","metadata":{},"outputs":[],"execution_count":null},{"id":"f9eeac62","cell_type":"code","source":"import os, cv2, time, random, warnings, joblib\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport pydicom\n\nfrom tqdm import tqdm\nfrom skimage.feature import hog, local_binary_pattern\nfrom skimage import exposure\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import (\n    mean_squared_error, mean_absolute_error,\n    accuracy_score, precision_score, recall_score, f1_score,\n    roc_auc_score, average_precision_score,\n    confusion_matrix, roc_curve, precision_recall_curve,\n    ConfusionMatrixDisplay\n)\nfrom sklearn.svm import SVR, SVC\nfrom sklearn.multioutput import MultiOutputRegressor\nfrom sklearn.ensemble import (\n    RandomForestRegressor, GradientBoostingRegressor,\n    RandomForestClassifier, GradientBoostingClassifier\n)\nfrom sklearn.utils.class_weight import compute_sample_weight\nfrom scipy.stats import ttest_rel, wilcoxon\n\nwarnings.filterwarnings(\"ignore\")\nnp.random.seed(42)\nrandom.seed(42)\n\nIMG_SIZE = 256      # Use 512 for final high-resolution run if needed. For more speed, i used 256\nN_TRAIN = 3000      # Change to None for full dataset\nN_VAL = 700\nN_TEST = 700\nRANDOM_STATE = 42\n\nos.makedirs('/kaggle/working/results/tables', exist_ok=True)\nos.makedirs('/kaggle/working/results/figures', exist_ok=True)\nos.makedirs('/kaggle/working/results/models', exist_ok=True)\nos.makedirs('/kaggle/working/features', exist_ok=True)\nos.makedirs('/kaggle/working/splits', exist_ok=True)","metadata":{},"outputs":[],"execution_count":null},{"id":"1f3b097c","cell_type":"markdown","source":"## 1. Dataset Path Discovery and Loading","metadata":{}},{"id":"5a046c2d","cell_type":"code","source":"def find_rsna_paths(base='/kaggle/input'):\n    labels_path = None\n    class_info_path = None\n    train_img_dir = None\n    for dirname, _, filenames in os.walk(base):\n        for filename in filenames:\n            if filename == 'stage_2_train_labels.csv':\n                labels_path = os.path.join(dirname, filename)\n            elif filename == 'stage_2_detailed_class_info.csv':\n                class_info_path = os.path.join(dirname, filename)\n        if os.path.basename(dirname) == 'stage_2_train_images':\n            train_img_dir = dirname\n    if labels_path is None or class_info_path is None or train_img_dir is None:\n        raise FileNotFoundError('RSNA files not found. Add RSNA Pneumonia Detection Challenge as Kaggle input.')\n    return labels_path, class_info_path, train_img_dir\n\nLABELS_CSV, CLASS_INFO_CSV, TRAIN_IMG_DIR = find_rsna_paths()\nDATA_DIR = os.path.dirname(LABELS_CSV)\n\nlabels_df = pd.read_csv(LABELS_CSV)\nclass_df = pd.read_csv(CLASS_INFO_CSV)\n\nprint('DATA_DIR:', DATA_DIR)\nprint('TRAIN_IMG_DIR:', TRAIN_IMG_DIR)\nprint(labels_df.shape)\nlabels_df.head()","metadata":{},"outputs":[],"execution_count":null},{"id":"dd4f45fc","cell_type":"code","source":"print('Original annotation rows:', labels_df.shape)\nprint('Unique patientIds:', labels_df['patientId'].nunique())\nprint(labels_df['Target'].value_counts())\n\nlabels_df['Target'].value_counts().plot(kind='bar', figsize=(5,4))\nplt.title('Class Distribution in Annotation Rows')\nplt.xlabel('Target')\nplt.ylabel('Count')\nplt.show()","metadata":{},"outputs":[],"execution_count":null},{"id":"7ab7a816","cell_type":"markdown","source":"## 2. Merge Multiple Bounding Boxes per Image","metadata":{}},{"id":"9462e989","cell_type":"code","source":"def merge_boxes(group):\n    if group['Target'].iloc[0] == 0:\n        return pd.Series({'x': np.nan, 'y': np.nan, 'width': np.nan, 'height': np.nan, 'Target': 0})\n    x_min = group['x'].min()\n    y_min = group['y'].min()\n    x_max = (group['x'] + group['width']).max()\n    y_max = (group['y'] + group['height']).max()\n    return pd.Series({'x': x_min, 'y': y_min, 'width': x_max - x_min, 'height': y_max - y_min, 'Target': 1})\n\nimage_df = labels_df.groupby('patientId').apply(merge_boxes).reset_index()\n\nprint('image_df shape:', image_df.shape)\nprint(image_df['Target'].value_counts())\nimage_df.head()","metadata":{},"outputs":[],"execution_count":null},{"id":"d8c3f0a7","cell_type":"code","source":"image_df['Target'].value_counts().plot(kind='bar', figsize=(5,4))\nplt.title('Class Distribution after Merging Boxes')\nplt.xlabel('Target')\nplt.ylabel('Number of Unique Images')\nplt.show()","metadata":{},"outputs":[],"execution_count":null},{"id":"2cb21a42","cell_type":"markdown","source":"## 3. Read DICOM, Resize, Normalize, CLAHE","metadata":{}},{"id":"365a32e1","cell_type":"code","source":"def read_dicom_image(patient_id, img_size=IMG_SIZE, apply_clahe=True):\n    dicom_path = os.path.join(TRAIN_IMG_DIR, patient_id + '.dcm')\n    if not os.path.exists(dicom_path):\n        raise FileNotFoundError(f'DICOM file not found: {dicom_path}')\n\n    dcm = pydicom.dcmread(dicom_path)\n    img = dcm.pixel_array.astype(np.float32)\n    original_h, original_w = img.shape\n\n    if getattr(dcm, 'PhotometricInterpretation', '') == 'MONOCHROME1':\n        img = img.max() - img\n\n    img = img - img.min()\n    if img.max() > 0:\n        img = img / img.max()\n    img = (img * 255).astype(np.uint8)\n\n    img = cv2.resize(img, (img_size, img_size), interpolation=cv2.INTER_AREA)\n\n    if apply_clahe:\n        clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))\n        img = clahe.apply(img)\n\n    return img, original_w, original_h","metadata":{},"outputs":[],"execution_count":null},{"id":"99b32dbe","cell_type":"code","source":"sample_df = image_df.sample(6, random_state=RANDOM_STATE)\nplt.figure(figsize=(12, 6))\nfor i, (_, row) in enumerate(sample_df.iterrows()):\n    img, ow, oh = read_dicom_image(row['patientId'])\n    plt.subplot(2, 3, i + 1)\n    plt.imshow(img, cmap='gray')\n    plt.title(f\"Target={int(row['Target'])}\\n{ow}x{oh} -> {IMG_SIZE}x{IMG_SIZE}\")\n    plt.axis('off')\nplt.tight_layout()\nplt.show()","metadata":{},"outputs":[],"execution_count":null},{"id":"30511c15","cell_type":"markdown","source":"## 4. Normalize Bounding Boxes","metadata":{}},{"id":"74c1aa84","cell_type":"code","source":"def normalize_bbox_row(row):\n    if row['Target'] == 0:\n        return pd.Series({'x_norm': np.nan, 'y_norm': np.nan, 'w_norm': np.nan, 'h_norm': np.nan})\n    return pd.Series({\n        'x_norm': row['x'] / 1024.0,\n        'y_norm': row['y'] / 1024.0,\n        'w_norm': row['width'] / 1024.0,\n        'h_norm': row['height'] / 1024.0\n    })\n\nbbox_norm_df = image_df.apply(normalize_bbox_row, axis=1)\nimage_df = pd.concat([image_df, bbox_norm_df], axis=1)\n\npositive_boxes = image_df[image_df['Target'] == 1][['x_norm','y_norm','w_norm','h_norm']]\nprint('Min bbox:', positive_boxes.min().values)\nprint('Max bbox:', positive_boxes.max().values)\nprint('Any below 0:', (positive_boxes < 0).any().any())\nprint('Any above 1:', (positive_boxes > 1).any().any())\nimage_df.head()","metadata":{},"outputs":[],"execution_count":null},{"id":"c5e9ab30","cell_type":"code","source":"def draw_normalized_bbox(img, bbox_norm, color=(255, 0, 0), thickness=2):\n    img_rgb = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)\n    h, w = img.shape\n    x, y, bw, bh = bbox_norm\n    x1, y1 = int(x*w), int(y*h)\n    x2, y2 = int((x+bw)*w), int((y+bh)*h)\n    x1 = max(0, min(w-1, x1)); y1 = max(0, min(h-1, y1))\n    x2 = max(0, min(w-1, x2)); y2 = max(0, min(h-1, y2))\n    cv2.rectangle(img_rgb, (x1, y1), (x2, y2), color, thickness)\n    return img_rgb\n\npos_samples = image_df[image_df['Target'] == 1].sample(6, random_state=RANDOM_STATE)\nplt.figure(figsize=(12, 6))\nfor i, (_, row) in enumerate(pos_samples.iterrows()):\n    img, _, _ = read_dicom_image(row['patientId'])\n    bbox = [row['x_norm'], row['y_norm'], row['w_norm'], row['h_norm']]\n    plt.subplot(2, 3, i+1)\n    plt.imshow(draw_normalized_bbox(img, bbox))\n    plt.title('Target=1')\n    plt.axis('off')\nplt.tight_layout()\nplt.show()","metadata":{},"outputs":[],"execution_count":null},{"id":"a9be4417","cell_type":"markdown","source":"## 5. Train / Validation / Test Split","metadata":{}},{"id":"48beb7e8","cell_type":"code","source":"train_df, temp_df = train_test_split(\n    image_df, test_size=0.30, stratify=image_df['Target'], random_state=RANDOM_STATE\n)\nval_df, test_df = train_test_split(\n    temp_df, test_size=0.50, stratify=temp_df['Target'], random_state=RANDOM_STATE\n)\n\nprint('Train shape:', train_df.shape)\nprint('Validation shape:', val_df.shape)\nprint('Test shape:', test_df.shape)\nfor name, df in [('Train', train_df), ('Validation', val_df), ('Test', test_df)]:\n    print('\\n', name)\n    print(df['Target'].value_counts())\n    print(df['Target'].value_counts(normalize=True))\n\nprint('Train ∩ Validation:', len(set(train_df.patientId) & set(val_df.patientId)))\nprint('Train ∩ Test:', len(set(train_df.patientId) & set(test_df.patientId)))\nprint('Validation ∩ Test:', len(set(val_df.patientId) & set(test_df.patientId)))\n\ntrain_df.to_csv('/kaggle/working/splits/train_df.csv', index=False)\nval_df.to_csv('/kaggle/working/splits/val_df.csv', index=False)\ntest_df.to_csv('/kaggle/working/splits/test_df.csv', index=False)","metadata":{},"outputs":[],"execution_count":null},{"id":"841f938a","cell_type":"markdown","source":"## 6. Patent-Free Feature Extraction","metadata":{}},{"id":"4fa28bb0","cell_type":"code","source":"def extract_hog_features(img):\n    return hog(img, orientations=9, pixels_per_cell=(16,16), cells_per_block=(2,2),\n               block_norm='L2-Hys', feature_vector=True).astype(np.float32)\n\ndef extract_lbp_features(img, P=8, R=1):\n    lbp = local_binary_pattern(img, P=P, R=R, method='uniform')\n    n_bins = P + 2\n    hist, _ = np.histogram(lbp.ravel(), bins=n_bins, range=(0, n_bins), density=True)\n    return hist.astype(np.float32)\n\ndef extract_orb_features(img, n_features=500):\n    orb = cv2.ORB_create(nfeatures=n_features)\n    keypoints, descriptors = orb.detectAndCompute(img, None)\n    if descriptors is None:\n        return np.zeros(64, dtype=np.float32), 0\n    descriptors = descriptors.astype(np.float32)\n    return np.concatenate([descriptors.mean(axis=0), descriptors.std(axis=0)]).astype(np.float32), len(keypoints)\n\ndef extract_brisk_features(img):\n    brisk = cv2.BRISK_create()\n    keypoints, descriptors = brisk.detectAndCompute(img, None)\n    if descriptors is None:\n        return np.zeros(128, dtype=np.float32), 0\n    descriptors = descriptors.astype(np.float32)\n    return np.concatenate([descriptors.mean(axis=0), descriptors.std(axis=0)]).astype(np.float32), len(keypoints)\n\ndef extract_edge_histogram(img, bins=32):\n    edges = cv2.Canny(img, 50, 150)\n    hist, _ = np.histogram(edges.ravel(), bins=bins, range=(0, 256), density=True)\n    return hist.astype(np.float32)\n\ndef extract_all_features(img):\n    start_total = time.time()\n    t = time.time(); hog_feat = extract_hog_features(img); hog_time = (time.time()-t)*1000\n    t = time.time(); lbp_feat = extract_lbp_features(img); lbp_time = (time.time()-t)*1000\n    t = time.time(); orb_feat, orb_kp = extract_orb_features(img); orb_time = (time.time()-t)*1000\n    t = time.time(); brisk_feat, brisk_kp = extract_brisk_features(img); brisk_time = (time.time()-t)*1000\n    t = time.time(); edge_feat = extract_edge_histogram(img); edge_time = (time.time()-t)*1000\n    features = np.concatenate([hog_feat, lbp_feat, orb_feat, brisk_feat, edge_feat]).astype(np.float32)\n    meta = {\n        'hog_dim': len(hog_feat), 'lbp_dim': len(lbp_feat), 'orb_dim': len(orb_feat),\n        'brisk_dim': len(brisk_feat), 'edge_dim': len(edge_feat), 'total_dim': len(features),\n        'orb_keypoints': orb_kp, 'brisk_keypoints': brisk_kp,\n        'hog_time_ms': hog_time, 'lbp_time_ms': lbp_time, 'orb_time_ms': orb_time,\n        'brisk_time_ms': brisk_time, 'edge_time_ms': edge_time,\n        'total_time_ms': (time.time()-start_total)*1000\n    }\n    return features, meta\n\nsample_id = train_df.iloc[0]['patientId']\nimg, _, _ = read_dicom_image(sample_id)\nfeatures, meta = extract_all_features(img)\nprint('Feature vector shape:', features.shape)\nprint(meta)","metadata":{},"outputs":[],"execution_count":null},{"id":"112b7d05","cell_type":"markdown","source":"## 7. Build Full-Image Feature Matrices","metadata":{}},{"id":"b1ddbdb3","cell_type":"code","source":"def build_image_feature_dataset(df, max_samples=None, split_name='train'):\n    df_work = df.sample(n=min(max_samples, len(df)), random_state=RANDOM_STATE).copy() if max_samples is not None else df.copy()\n    X, y_class, y_bbox, patient_ids, logs, failed = [], [], [], [], [], []\n    for _, row in tqdm(df_work.iterrows(), total=len(df_work), desc=f'Extracting {split_name} features'):\n        pid = row['patientId']\n        try:\n            img, _, _ = read_dicom_image(pid)\n            feat, meta = extract_all_features(img)\n            X.append(feat)\n            y_class.append(int(row['Target']))\n            if int(row['Target']) == 1:\n                y_bbox.append([row['x_norm'], row['y_norm'], row['w_norm'], row['h_norm']])\n            else:\n                y_bbox.append([np.nan, np.nan, np.nan, np.nan])\n            patient_ids.append(pid)\n            meta.update({'patientId': pid, 'Target': int(row['Target']), 'split': split_name})\n            logs.append(meta)\n        except Exception as e:\n            failed.append((pid, str(e)))\n    X = np.array(X, dtype=np.float32)\n    y_class = np.array(y_class, dtype=np.int32)\n    y_bbox = np.array(y_bbox, dtype=np.float32)\n    logs = pd.DataFrame(logs)\n    print(split_name, X.shape, y_class.shape, y_bbox.shape, 'failed:', len(failed))\n    return X, y_class, y_bbox, patient_ids, logs, failed\n\nX_train, y_train_cls, y_train_bbox, train_ids, train_feature_log, train_failed = build_image_feature_dataset(train_df, N_TRAIN, 'train')\nX_val, y_val_cls, y_val_bbox, val_ids, val_feature_log, val_failed = build_image_feature_dataset(val_df, N_VAL, 'validation')\nX_test, y_test_cls, y_test_bbox, test_ids, test_feature_log, test_failed = build_image_feature_dataset(test_df, N_TEST, 'test')\n\nfor name, arr in [('X_train',X_train),('X_val',X_val),('X_test',X_test)]: print(name, arr.shape)\nprint(pd.Series(y_train_cls).value_counts())","metadata":{},"outputs":[],"execution_count":null},{"id":"dc3af21c","cell_type":"code","source":"np.save('/kaggle/working/features/X_train.npy', X_train)\nnp.save('/kaggle/working/features/y_train_cls.npy', y_train_cls)\nnp.save('/kaggle/working/features/y_train_bbox.npy', y_train_bbox)\nnp.save('/kaggle/working/features/X_val.npy', X_val)\nnp.save('/kaggle/working/features/y_val_cls.npy', y_val_cls)\nnp.save('/kaggle/working/features/y_val_bbox.npy', y_val_bbox)\nnp.save('/kaggle/working/features/X_test.npy', X_test)\nnp.save('/kaggle/working/features/y_test_cls.npy', y_test_cls)\nnp.save('/kaggle/working/features/y_test_bbox.npy', y_test_bbox)\ntrain_feature_log.to_csv('/kaggle/working/features/train_feature_log.csv', index=False)\nval_feature_log.to_csv('/kaggle/working/features/val_feature_log.csv', index=False)\ntest_feature_log.to_csv('/kaggle/working/features/test_feature_log.csv', index=False)","metadata":{},"outputs":[],"execution_count":null},{"id":"7871c81f","cell_type":"markdown","source":"## 8. Scale Features and Prepare Regression Data","metadata":{}},{"id":"1e2c845f","cell_type":"code","source":"scaler = StandardScaler()\nX_train_scaled = scaler.fit_transform(X_train)\nX_val_scaled = scaler.transform(X_val)\nX_test_scaled = scaler.transform(X_test)\njoblib.dump(scaler, '/kaggle/working/results/models/full_image_feature_scaler.pkl')\n\ntrain_pos_mask = y_train_cls == 1\nval_pos_mask = y_val_cls == 1\ntest_pos_mask = y_test_cls == 1\n\nX_train_reg = X_train_scaled[train_pos_mask]\ny_train_reg = y_train_bbox[train_pos_mask]\nX_val_reg = X_val_scaled[val_pos_mask]\ny_val_reg = y_val_bbox[val_pos_mask]\nX_test_reg = X_test_scaled[test_pos_mask]\ny_test_reg = y_test_bbox[test_pos_mask]\n\nprint('scaled mean/std:', X_train_scaled.mean(), X_train_scaled.std())\nprint('Regression train:', X_train_reg.shape, y_train_reg.shape)\nprint('NaN in y_train_reg:', np.isnan(y_train_reg).any())\nprint('bbox min/max:', y_train_reg.min(axis=0), y_train_reg.max(axis=0))","metadata":{},"outputs":[],"execution_count":null},{"id":"3db7fb7c","cell_type":"markdown","source":"## 9. Regression Models and Evaluation","metadata":{}},{"id":"95f50719","cell_type":"code","source":"def bbox_iou(box1, box2):\n    x1, y1, w1, h1 = box1\n    x2, y2, w2, h2 = box2\n    a1, b1, a2, b2 = x1, y1, x1+w1, y1+h1\n    c1, d1, c2, d2 = x2, y2, x2+w2, y2+h2\n    inter_x1, inter_y1 = max(a1, c1), max(b1, d1)\n    inter_x2, inter_y2 = min(a2, c2), min(b2, d2)\n    inter = max(0, inter_x2-inter_x1) * max(0, inter_y2-inter_y1)\n    union = max(0,w1)*max(0,h1) + max(0,w2)*max(0,h2) - inter\n    return 0.0 if union <= 0 else inter / union\n\ndef evaluate_regressor(model, X, y_true):\n    start = time.time()\n    y_pred = np.clip(model.predict(X), 0, 1)\n    inf_time = (time.time() - start) / len(X)\n    ious = np.array([bbox_iou(t, p) for t, p in zip(y_true, y_pred)])\n    return {\n        'MSE': mean_squared_error(y_true, y_pred),\n        'MAE': mean_absolute_error(y_true, y_pred),\n        'Mean IoU': float(np.mean(ious)),\n        'IoU@0.4': float(np.mean(ious >= 0.4)),\n        'IoU@0.5': float(np.mean(ious >= 0.5)),\n        'Inference Time': inf_time,\n        'ious': ious,\n        'preds': y_pred\n    }\n\nregressors = {}\n\nmodels_to_train = {\n    'SVR_RBF': MultiOutputRegressor(SVR(kernel='rbf', C=10, gamma='scale', epsilon=0.05)),\n    'RandomForest': RandomForestRegressor(n_estimators=150, random_state=RANDOM_STATE, n_jobs=-1),\n    'GradientBoosting': MultiOutputRegressor(GradientBoostingRegressor(n_estimators=150, learning_rate=0.05, max_depth=3, random_state=RANDOM_STATE))\n}\n\nfor name, model in models_to_train.items():\n    print('Training', name)\n    start = time.time()\n    model.fit(X_train_reg, y_train_reg)\n    train_time = time.time() - start\n    regressors[name] = {'model': model, 'train_time': train_time}\n    print(name, train_time)","metadata":{},"outputs":[],"execution_count":null},{"id":"aab16e5f","cell_type":"code","source":"reg_results = []\nreg_predictions = {}\nfor name, item in regressors.items():\n    result = evaluate_regressor(item['model'], X_test_reg, y_test_reg)\n    reg_results.append({\n        'Model': name, 'MSE': result['MSE'], 'MAE': result['MAE'], 'Mean IoU': result['Mean IoU'],\n        'IoU@0.4': result['IoU@0.4'], 'IoU@0.5': result['IoU@0.5'],\n        'Training Time': item['train_time'], 'Inference Time': result['Inference Time']\n    })\n    reg_predictions[name] = {'ious': result['ious'], 'preds': result['preds']}\n    joblib.dump(item['model'], f'/kaggle/working/results/models/regressor_{name}.pkl')\n\nreg_results_df = pd.DataFrame(reg_results)\nreg_results_df.to_csv('/kaggle/working/results/tables/table2_regression_performance.csv', index=False)\nbest_reg_name = reg_results_df.sort_values('IoU@0.5', ascending=False).iloc[0]['Model']\nbest_reg_model = regressors[best_reg_name]['model']\nprint('Best regressor:', best_reg_name)\nreg_results_df","metadata":{},"outputs":[],"execution_count":null},{"id":"4434440f","cell_type":"markdown","source":"## 10. ROI Cropping and Classification Dataset v2","metadata":{}},{"id":"c8b1003f","cell_type":"code","source":"def crop_bbox_from_image(img, bbox_norm, output_size=IMG_SIZE):\n    h, w = img.shape\n    x, y, bw, bh = bbox_norm\n    x1, y1 = int(x*w), int(y*h)\n    x2, y2 = int((x+bw)*w), int((y+bh)*h)\n    x1 = max(0, min(w-1, x1)); y1 = max(0, min(h-1, y1))\n    x2 = max(x1+1, min(w, x2)); y2 = max(y1+1, min(h, y2))\n    crop = img[y1:y2, x1:x2]\n    return cv2.resize(crop, (output_size, output_size), interpolation=cv2.INTER_AREA)\n\ndef build_roi_feature_dataset_v2(df, reg_model, max_samples=None, split_name='train'):\n    df_work = df.sample(n=min(max_samples, len(df)), random_state=RANDOM_STATE).copy() if max_samples is not None else df.copy()\n    X_roi, y_roi, roi_ids, roi_logs, failed = [], [], [], [], []\n    for _, row in tqdm(df_work.iterrows(), total=len(df_work), desc=f'Extracting {split_name} ROI v2'):\n        pid = row['patientId']\n        try:\n            img, _, _ = read_dicom_image(pid)\n            if int(row['Target']) == 1:\n                bbox = [row['x_norm'], row['y_norm'], row['w_norm'], row['h_norm']]\n                roi_type = 'gt_positive_bbox'\n            else:\n                feat, _ = extract_all_features(img)\n                bbox = np.clip(reg_model.predict(scaler.transform([feat]))[0], 0, 1)\n                roi_type = 'regressor_predicted_negative_bbox'\n            roi = crop_bbox_from_image(img, bbox)\n            features, meta = extract_all_features(roi)\n            X_roi.append(features)\n            y_roi.append(int(row['Target']))\n            roi_ids.append(pid)\n            meta.update({'patientId': pid, 'Target': int(row['Target']), 'roi_type': roi_type, 'split': split_name})\n            roi_logs.append(meta)\n        except Exception as e:\n            failed.append((pid, str(e)))\n    X_roi = np.array(X_roi, dtype=np.float32)\n    y_roi = np.array(y_roi, dtype=np.int32)\n    print(split_name, X_roi.shape, y_roi.shape, 'failed:', len(failed))\n    return X_roi, y_roi, roi_ids, pd.DataFrame(roi_logs), failed\n\nX_train_roi_v2, y_train_roi_v2, train_roi_ids_v2, train_roi_log_v2, train_roi_failed_v2 = build_roi_feature_dataset_v2(train_df, best_reg_model, N_TRAIN, 'train')\nX_val_roi_v2, y_val_roi_v2, val_roi_ids_v2, val_roi_log_v2, val_roi_failed_v2 = build_roi_feature_dataset_v2(val_df, best_reg_model, N_VAL, 'validation')\nX_test_roi_v2, y_test_roi_v2, test_roi_ids_v2, test_roi_log_v2, test_roi_failed_v2 = build_roi_feature_dataset_v2(test_df, best_reg_model, N_TEST, 'test')","metadata":{},"outputs":[],"execution_count":null},{"id":"4ce8807f","cell_type":"code","source":"roi_scaler_v2 = StandardScaler()\nX_train_roi_v2_scaled = roi_scaler_v2.fit_transform(X_train_roi_v2)\nX_val_roi_v2_scaled = roi_scaler_v2.transform(X_val_roi_v2)\nX_test_roi_v2_scaled = roi_scaler_v2.transform(X_test_roi_v2)\njoblib.dump(roi_scaler_v2, '/kaggle/working/results/models/roi_feature_scaler_v2.pkl')\n\nprint('ROI v2 scaled:', X_train_roi_v2_scaled.shape, X_train_roi_v2_scaled.mean(), X_train_roi_v2_scaled.std())\nprint(pd.Series(y_train_roi_v2).value_counts())","metadata":{},"outputs":[],"execution_count":null},{"id":"b91b2102","cell_type":"markdown","source":"## 11. Classification Models v2 and Evaluation","metadata":{}},{"id":"f51bf752","cell_type":"code","source":"def evaluate_classifier(model, X, y_true):\n    start = time.time()\n    y_pred = model.predict(X)\n    inf_time = (time.time() - start) / len(X)\n    y_score = model.predict_proba(X)[:, 1] if hasattr(model, 'predict_proba') else model.decision_function(X)\n    return {\n        'Accuracy': accuracy_score(y_true, y_pred),\n        'Precision': precision_score(y_true, y_pred, zero_division=0),\n        'Recall': recall_score(y_true, y_pred, zero_division=0),\n        'F1-score': f1_score(y_true, y_pred, zero_division=0),\n        'ROC-AUC': roc_auc_score(y_true, y_score),\n        'PR-AUC': average_precision_score(y_true, y_score),\n        'Inference Time': inf_time,\n        'y_pred': y_pred,\n        'y_score': y_score,\n        'confusion_matrix': confusion_matrix(y_true, y_pred)\n    }\n\nclassifiers_v2 = {}\n\nsvm_clf_v2 = SVC(kernel='rbf', C=10, gamma='scale', probability=True, class_weight='balanced', random_state=RANDOM_STATE)\nstart = time.time(); svm_clf_v2.fit(X_train_roi_v2_scaled, y_train_roi_v2); train_time = time.time() - start\nclassifiers_v2['SVM_RBF'] = {'model': svm_clf_v2, 'train_time': train_time}\n\nrf_clf_v2 = RandomForestClassifier(n_estimators=150, random_state=RANDOM_STATE, n_jobs=-1, class_weight='balanced')\nstart = time.time(); rf_clf_v2.fit(X_train_roi_v2_scaled, y_train_roi_v2); train_time = time.time() - start\nclassifiers_v2['RandomForest'] = {'model': rf_clf_v2, 'train_time': train_time}\n\nsample_weights_v2 = compute_sample_weight(class_weight='balanced', y=y_train_roi_v2)\ngb_clf_v2 = GradientBoostingClassifier(n_estimators=60, learning_rate=0.05, max_depth=2, subsample=0.8, random_state=RANDOM_STATE)\nstart = time.time(); gb_clf_v2.fit(X_train_roi_v2_scaled, y_train_roi_v2, sample_weight=sample_weights_v2); train_time = time.time() - start\nclassifiers_v2['GradientBoosting'] = {'model': gb_clf_v2, 'train_time': train_time}","metadata":{},"outputs":[],"execution_count":null},{"id":"725e71e7","cell_type":"code","source":"clf_results_v2 = []\nclf_predictions_v2 = {}\nfor name, item in classifiers_v2.items():\n    result = evaluate_classifier(item['model'], X_test_roi_v2_scaled, y_test_roi_v2)\n    clf_results_v2.append({\n        'Model': name, 'Accuracy': result['Accuracy'], 'Precision': result['Precision'],\n        'Recall': result['Recall'], 'F1-score': result['F1-score'], 'ROC-AUC': result['ROC-AUC'],\n        'PR-AUC': result['PR-AUC'], 'Training Time': item['train_time'], 'Inference Time': result['Inference Time']\n    })\n    clf_predictions_v2[name] = {'y_pred': result['y_pred'], 'y_score': result['y_score'], 'confusion_matrix': result['confusion_matrix']}\n    joblib.dump(item['model'], f'/kaggle/working/results/models/classifier_v2_{name}.pkl')\n\nclf_results_v2_df = pd.DataFrame(clf_results_v2)\nclf_results_v2_df.to_csv('/kaggle/working/results/tables/table3_classification_performance_v2.csv', index=False)\nbest_clf_name = clf_results_v2_df.sort_values('F1-score', ascending=False).iloc[0]['Model']\nbest_clf_model = classifiers_v2[best_clf_name]['model']\nprint('Best classifier:', best_clf_name)\nclf_results_v2_df","metadata":{},"outputs":[],"execution_count":null},{"id":"259b5836","cell_type":"markdown","source":"## 12. End-to-End Pipeline and Threshold Tuning","metadata":{}},{"id":"f8259d2b","cell_type":"code","source":"def full_pipeline_threshold(patient_id, reg_model, clf_model, threshold=0.5):\n    start_total = time.time()\n    img, _, _ = read_dicom_image(patient_id)\n    feat, _ = extract_all_features(img)\n    pred_bbox = np.clip(reg_model.predict(scaler.transform([feat]))[0], 0, 1)\n    roi = crop_bbox_from_image(img, pred_bbox)\n    roi_feat, _ = extract_all_features(roi)\n    roi_scaled = roi_scaler_v2.transform([roi_feat])\n    confidence = float(clf_model.predict_proba(roi_scaled)[0,1]) if hasattr(clf_model, 'predict_proba') else float(clf_model.decision_function(roi_scaled)[0])\n    pred_class = int(confidence >= threshold)\n    return {'patientId': patient_id, 'bbox': pred_bbox, 'class': pred_class, 'confidence': confidence, 'time': time.time()-start_total}\n\ndef evaluate_end_to_end_threshold(df, reg_model, clf_model, threshold=0.5, max_samples=200, random_seed=RANDOM_STATE):\n    df_eval = df.sample(n=min(max_samples, len(df)), random_state=random_seed).copy()\n    pos_det04, pos_det05, pos_hits, neg_fp, times, rows = [], [], [], [], [], []\n    for _, row in tqdm(df_eval.iterrows(), total=len(df_eval), desc=f'E2E threshold={threshold}'):\n        out = full_pipeline_threshold(row['patientId'], reg_model, clf_model, threshold)\n        true_class = int(row['Target'])\n        pred_class, pred_bbox, confidence = out['class'], out['bbox'], out['confidence']\n        times.append(out['time'])\n        iou = np.nan\n        if true_class == 1:\n            gt_bbox = [row['x_norm'], row['y_norm'], row['w_norm'], row['h_norm']]\n            iou = bbox_iou(gt_bbox, pred_bbox)\n            pos_hits.append(pred_class == 1)\n            pos_det04.append((pred_class == 1) and (iou >= 0.4))\n            pos_det05.append((pred_class == 1) and (iou >= 0.5))\n        else:\n            neg_fp.append(pred_class == 1)\n        rows.append({'patientId': row['patientId'], 'true_class': true_class, 'pred_class': pred_class, 'confidence': confidence, 'iou': iou, 'inference_time': out['time']})\n    detection = float(np.mean(pos_hits)) if pos_hits else 0\n    map04 = float(np.mean(pos_det04)) if pos_det04 else 0\n    map05 = float(np.mean(pos_det05)) if pos_det05 else 0\n    fpr = float(np.mean(neg_fp)) if neg_fp else 0\n    return {'Detection': detection, 'mAP@0.4': map04, 'mAP@0.5': map05, 'False Positive Rate': fpr,\n            'Inference Time (s)': float(np.mean(times)), 'Balanced Score': map05 - 0.5*fpr, 'details': pd.DataFrame(rows)}","metadata":{},"outputs":[],"execution_count":null},{"id":"7e0dd5f1","cell_type":"code","source":"thresholds = [0.001, 0.003, 0.005, 0.008, 0.01, 0.02, 0.03, 0.05, 0.1, 0.2, 0.3, 0.5]\ncandidate_pairs = [\n    ('SVR_RBF', 'SVM_RBF'),\n    ('SVR_RBF', 'GradientBoosting'),\n    ('GradientBoosting', 'SVM_RBF'),\n    ('GradientBoosting', 'GradientBoosting')\n]\n\nthreshold_results, threshold_details = [], {}\nfor reg_name, clf_name in candidate_pairs:\n    for th in thresholds:\n        print(f'Running: {reg_name} + {clf_name}, threshold={th}')\n        result = evaluate_end_to_end_threshold(test_df, regressors[reg_name]['model'], classifiers_v2[clf_name]['model'], threshold=th, max_samples=200)\n        threshold_results.append({\n            'Regressor': reg_name, 'Classifier': clf_name, 'Threshold': th,\n            'Detection': result['Detection'], 'mAP@0.4': result['mAP@0.4'], 'mAP@0.5': result['mAP@0.5'],\n            'False Positive Rate': result['False Positive Rate'], 'Inference Time (s)': result['Inference Time (s)'],\n            'Balanced Score': result['Balanced Score']\n        })\n        threshold_details[(reg_name, clf_name, th)] = result['details']\n\nthreshold_results_df = pd.DataFrame(threshold_results)\nbest_threshold_row = threshold_results_df.sort_values('Balanced Score', ascending=False).iloc[0]\nbest_e2e_reg_name = best_threshold_row['Regressor']\nbest_e2e_clf_name = best_threshold_row['Classifier']\nbest_threshold = float(best_threshold_row['Threshold'])\nbest_e2e_details_df = threshold_details[(best_e2e_reg_name, best_e2e_clf_name, best_threshold)]\n\nthreshold_results_df.to_csv('/kaggle/working/results/tables/table4_end_to_end_pipeline_thresholded.csv', index=False)\nbest_e2e_details_df.to_csv('/kaggle/working/results/tables/best_end_to_end_details.csv', index=False)\ntable4_final = threshold_results_df.sort_values('Balanced Score', ascending=False).head(10).copy()\ntable4_final.to_csv('/kaggle/working/results/tables/table4_end_to_end_final_top10.csv', index=False)\n\nprint('Best thresholded combination:')\nprint(best_threshold_row)\ntable4_final","metadata":{},"outputs":[],"execution_count":null},{"id":"d0d110d5","cell_type":"markdown","source":"## 13. Required Result Tables","metadata":{}},{"id":"1b49609c","cell_type":"code","source":"table1_feature_extraction = pd.DataFrame([\n    {'Feature Type':'Texture/Shape','Algorithm':'HOG','Dimension':int(train_feature_log['hog_dim'].iloc[0]),'Avg Extraction Time (ms)':train_feature_log['hog_time_ms'].mean(),'Keypoints/Features Found':int(train_feature_log['hog_dim'].iloc[0]),'Memory (MB)':X_train[:, :8100].nbytes/(1024**2)},\n    {'Feature Type':'Texture','Algorithm':'LBP','Dimension':int(train_feature_log['lbp_dim'].iloc[0]),'Avg Extraction Time (ms)':train_feature_log['lbp_time_ms'].mean(),'Keypoints/Features Found':int(train_feature_log['lbp_dim'].iloc[0]),'Memory (MB)':X_train[:, 8100:8110].nbytes/(1024**2)},\n    {'Feature Type':'Keypoint','Algorithm':'ORB','Dimension':int(train_feature_log['orb_dim'].iloc[0]),'Avg Extraction Time (ms)':train_feature_log['orb_time_ms'].mean(),'Keypoints/Features Found':train_feature_log['orb_keypoints'].mean(),'Memory (MB)':X_train[:, 8110:8174].nbytes/(1024**2)},\n    {'Feature Type':'Keypoint','Algorithm':'BRISK','Dimension':int(train_feature_log['brisk_dim'].iloc[0]),'Avg Extraction Time (ms)':train_feature_log['brisk_time_ms'].mean(),'Keypoints/Features Found':train_feature_log['brisk_keypoints'].mean(),'Memory (MB)':X_train[:, 8174:8302].nbytes/(1024**2)},\n    {'Feature Type':'Edge','Algorithm':'Edge Histogram','Dimension':int(train_feature_log['edge_dim'].iloc[0]),'Avg Extraction Time (ms)':train_feature_log['edge_time_ms'].mean(),'Keypoints/Features Found':int(train_feature_log['edge_dim'].iloc[0]),'Memory (MB)':X_train[:, 8302:8334].nbytes/(1024**2)}\n])\n\ntable5_ablation = pd.DataFrame([\n    {'Feature Set':'HOG only','Regression IoU':'Feature importance only','Classification F1':'Feature importance only','Combined mAP':'Feature importance only','Notes':'HOG dominated feature importance; full retraining not performed in this lightweight run.'},\n    {'Feature Set':'LBP only','Regression IoU':'Feature importance only','Classification F1':'Feature importance only','Combined mAP':'Feature importance only','Notes':'LBP contributes local texture information.'},\n    {'Feature Set':'ORB + BRISK','Regression IoU':'Feature importance only','Classification F1':'Feature importance only','Combined mAP':'Feature importance only','Notes':'Patent-free keypoint descriptors.'},\n    {'Feature Set':'HOG + LBP + ORB + BRISK + Edge','Regression IoU':float(reg_results_df['Mean IoU'].max()),'Classification F1':float(clf_results_v2_df['F1-score'].max()),'Combined mAP':float(best_threshold_row['mAP@0.5']),'Notes':'Full proposed patent-free feature set.'}\n])\n\ntable1_feature_extraction.to_csv('/kaggle/working/results/tables/table1_feature_extraction_comparison.csv', index=False)\ntable5_ablation.to_csv('/kaggle/working/results/tables/table5_ablation_study.csv', index=False)\n\ndisplay(table1_feature_extraction)\ndisplay(reg_results_df)\ndisplay(clf_results_v2_df)\ndisplay(table4_final)\ndisplay(table5_ablation)","metadata":{},"outputs":[],"execution_count":null},{"id":"0dea3725","cell_type":"markdown","source":"## 14. Required Figures 1-7","metadata":{}},{"id":"85b2abd0","cell_type":"code","source":"# Figure 1: ORB, HOG, LBP visualization\nnormal_row = test_df[test_df['Target'] == 0].sample(1, random_state=RANDOM_STATE).iloc[0]\npneumonia_row = test_df[test_df['Target'] == 1].sample(1, random_state=RANDOM_STATE).iloc[0]\nimg_normal, _, _ = read_dicom_image(normal_row['patientId'])\nimg_pneu, _, _ = read_dicom_image(pneumonia_row['patientId'])\n\norb_vis = cv2.ORB_create(nfeatures=120)\norb_normal = cv2.drawKeypoints(img_normal, orb_vis.detect(img_normal, None), None, flags=cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)\norb_pneu = cv2.drawKeypoints(img_pneu, orb_vis.detect(img_pneu, None), None, flags=cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)\n_, hog_normal = hog(img_normal, orientations=9, pixels_per_cell=(16,16), cells_per_block=(2,2), visualize=True, feature_vector=True)\n_, hog_pneu = hog(img_pneu, orientations=9, pixels_per_cell=(16,16), cells_per_block=(2,2), visualize=True, feature_vector=True)\nlbp_normal = local_binary_pattern(img_normal, P=8, R=1, method='uniform')\nlbp_pneu = local_binary_pattern(img_pneu, P=8, R=1, method='uniform')\n\nplt.figure(figsize=(12,8))\nfor i, (im, title, cmap) in enumerate([\n    (orb_normal,'ORB - Normal',None),(hog_normal,'HOG - Normal','gray'),(lbp_normal,'LBP - Normal','gray'),\n    (orb_pneu,'ORB - Pneumonia',None),(hog_pneu,'HOG - Pneumonia','gray'),(lbp_pneu,'LBP - Pneumonia','gray')]):\n    plt.subplot(2,3,i+1); plt.imshow(im, cmap=cmap); plt.title(title); plt.axis('off')\nplt.tight_layout(); plt.savefig('/kaggle/working/results/figures/figure1_feature_visualization.png', dpi=300); plt.show()","metadata":{},"outputs":[],"execution_count":null},{"id":"10f594a3","cell_type":"code","source":"# Figure 2: bounding box prediction examples\ndef draw_gt_pred_boxes(img, gt_bbox, pred_bbox):\n    img_rgb = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)\n    h, w = img.shape\n    for bbox, color in [(gt_bbox, (0,255,0)), (pred_bbox, (255,0,0))]:\n        x, y, bw, bh = bbox\n        cv2.rectangle(img_rgb, (int(x*w), int(y*h)), (int((x+bw)*w), int((y+bh)*h)), color, 2)\n    return img_rgb\n\nrows=[]\nfor _, row in test_df[test_df['Target']==1].sample(min(300, len(test_df[test_df['Target']==1])), random_state=RANDOM_STATE).iterrows():\n    img, _, _ = read_dicom_image(row['patientId'])\n    feat, _ = extract_all_features(img)\n    pred = np.clip(best_reg_model.predict(scaler.transform([feat]))[0], 0, 1)\n    gt = [row['x_norm'], row['y_norm'], row['w_norm'], row['h_norm']]\n    rows.append({'patientId': row['patientId'], 'iou': bbox_iou(gt,pred), 'gt_bbox': gt, 'pred_bbox': pred})\nfig2_df = pd.DataFrame(rows)\nexamples = pd.concat([fig2_df.sort_values('iou', ascending=False).head(3), fig2_df.sort_values('iou').head(3)])\nplt.figure(figsize=(12,8))\nfor i, (_, ex) in enumerate(examples.iterrows()):\n    img, _, _ = read_dicom_image(ex['patientId'])\n    label = 'Success' if i < 3 else 'Failure'\n    plt.subplot(2,3,i+1)\n    plt.imshow(draw_gt_pred_boxes(img, ex['gt_bbox'], ex['pred_bbox']))\n    plt.title(f\"{label} | IoU={ex['iou']:.3f}\\nGreen=GT, Red=Pred\")\n    plt.axis('off')\nplt.tight_layout()\nplt.savefig('/kaggle/working/results/figures/figure2_bbox_prediction_examples.png', dpi=300)\nplt.show()","metadata":{},"outputs":[],"execution_count":null},{"id":"e06cc00a","cell_type":"code","source":"# Figure 3: ROC and Precision-Recall curves\nplt.figure(figsize=(8,6))\nfor name, item in classifiers_v2.items():\n    model = item['model']\n    y_score = model.predict_proba(X_test_roi_v2_scaled)[:,1] if hasattr(model,'predict_proba') else model.decision_function(X_test_roi_v2_scaled)\n    fpr, tpr, _ = roc_curve(y_test_roi_v2, y_score)\n    plt.plot(fpr, tpr, label=f\"{name} AUC={roc_auc_score(y_test_roi_v2, y_score):.3f}\")\nplt.plot([0,1],[0,1], linestyle='--'); plt.xlabel('False Positive Rate'); plt.ylabel('True Positive Rate')\nplt.title('ROC Curves for ROI Classifiers'); plt.legend(); plt.grid(True); plt.tight_layout()\nplt.savefig('/kaggle/working/results/figures/figure3_roc_curves.png', dpi=300); plt.show()\n\nplt.figure(figsize=(8,6))\nfor name, item in classifiers_v2.items():\n    model = item['model']\n    y_score = model.predict_proba(X_test_roi_v2_scaled)[:,1] if hasattr(model,'predict_proba') else model.decision_function(X_test_roi_v2_scaled)\n    precision, recall, _ = precision_recall_curve(y_test_roi_v2, y_score)\n    plt.plot(recall, precision, label=f\"{name} AP={average_precision_score(y_test_roi_v2, y_score):.3f}\")\nplt.xlabel('Recall'); plt.ylabel('Precision'); plt.title('Precision-Recall Curves for ROI Classifiers')\nplt.legend(); plt.grid(True); plt.tight_layout(); plt.savefig('/kaggle/working/results/figures/figure3_precision_recall_curves.png', dpi=300); plt.show()","metadata":{},"outputs":[],"execution_count":null},{"id":"0c30e6d5","cell_type":"code","source":"# Figure 4: confusion matrices and IoU distribution\nfor name, item in classifiers_v2.items():\n    y_pred = item['model'].predict(X_test_roi_v2_scaled)\n    cm = confusion_matrix(y_test_roi_v2, y_pred)\n    disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=['Normal','Pneumonia'])\n    disp.plot(values_format='d')\n    plt.title(f'Confusion Matrix - {name}')\n    plt.tight_layout(); plt.savefig(f'/kaggle/working/results/figures/figure4_confusion_matrix_{name}.png', dpi=300); plt.show()\n\nbest_reg_ious = reg_predictions[best_reg_name]['ious']\nplt.figure(figsize=(8,5)); plt.hist(best_reg_ious, bins=25)\nplt.axvline(0.4, linestyle='--', label='IoU=0.4'); plt.axvline(0.5, linestyle='--', label='IoU=0.5')\nplt.xlabel('IoU'); plt.ylabel('Frequency'); plt.title('IoU Distribution for Bounding Box Regression')\nplt.legend(); plt.grid(True); plt.tight_layout(); plt.savefig('/kaggle/working/results/figures/figure4_iou_distribution.png', dpi=300); plt.show()","metadata":{},"outputs":[],"execution_count":null},{"id":"9dde6ebb","cell_type":"code","source":"# Figure 5: feature importance\nrf_reg_model = regressors['RandomForest']['model']\nrf_clf_model = classifiers_v2['RandomForest']['model']\nreg_importance = rf_reg_model.feature_importances_\nclf_importance = rf_clf_model.feature_importances_\n\nplt.figure(figsize=(10,5)); plt.plot(reg_importance); plt.title('Random Forest Feature Importance - Regression')\nplt.xlabel('Feature Index'); plt.ylabel('Importance'); plt.grid(True); plt.tight_layout()\nplt.savefig('/kaggle/working/results/figures/figure5_rf_regression_feature_importance.png', dpi=300); plt.show()\n\nplt.figure(figsize=(10,5)); plt.plot(clf_importance); plt.title('Random Forest Feature Importance - Classification')\nplt.xlabel('Feature Index'); plt.ylabel('Importance'); plt.grid(True); plt.tight_layout()\nplt.savefig('/kaggle/working/results/figures/figure5_rf_classification_feature_importance.png', dpi=300); plt.show()\n\nfeature_ranges = {'HOG':(0,8100), 'LBP':(8100,8110), 'ORB':(8110,8174), 'BRISK':(8174,8302), 'Edge':(8302,8334)}\nimportance_summary_df = pd.DataFrame([{\n    'Feature Type': k,\n    'Regression Importance': reg_importance[s:e].sum(),\n    'Classification Importance': clf_importance[s:e].sum()\n} for k,(s,e) in feature_ranges.items()])\nimportance_summary_df.to_csv('/kaggle/working/results/tables/feature_importance_summary.csv', index=False)\nimportance_summary_df.plot(x='Feature Type', y=['Regression Importance','Classification Importance'], kind='bar', figsize=(8,5))\nplt.title('Feature Importance by Feature Type'); plt.ylabel('Total Importance'); plt.xticks(rotation=0); plt.grid(axis='y'); plt.tight_layout()\nplt.savefig('/kaggle/working/results/figures/figure5_feature_type_importance.png', dpi=300); plt.show()\nimportance_summary_df","metadata":{},"outputs":[],"execution_count":null},{"id":"d2481497","cell_type":"code","source":"# Figure 6: pipeline diagram\nplt.figure(figsize=(18,5))\nsteps = [\n    'DICOM\\nX-ray',\n    f'Resize\\n{IMG_SIZE}x{IMG_SIZE}\\nCLAHE',\n    'Patent-Free\\nFeatures\\nORB/HOG/LBP/BRISK',\n    f'BBox\\nRegressor\\n{best_reg_name}',\n    'Predicted\\nROI Crop',\n    'ROI\\nFeatures',\n    f'Classifier\\n{best_clf_name}\\nThreshold={best_threshold}',\n    'Output\\nBBox + Class\\nConfidence'\n]\nxs = np.arange(len(steps))\nfor i, step in enumerate(steps):\n    plt.text(xs[i], 0.5, step, ha='center', va='center', fontsize=9,\n             bbox=dict(boxstyle='round,pad=0.4', facecolor='white', edgecolor='black'))\n    if i < len(steps)-1:\n        plt.arrow(xs[i]+0.35, 0.5, 0.3, 0, head_width=0.05, head_length=0.08, length_includes_head=True)\nplt.xlim(-0.6, len(steps)-0.4)\nplt.ylim(0,1)\nplt.axis('off')\nplt.title('Complete Patent-Free End-to-End Pneumonia Detection Pipeline')\nplt.tight_layout()\nplt.savefig('/kaggle/working/results/figures/figure6_complete_pipeline_diagram.png', dpi=300)\nplt.show()","metadata":{},"outputs":[],"execution_count":null},{"id":"cf60c663","cell_type":"code","source":"# Figure 7: baseline comparison\ndef center_box_baseline_bbox():\n    return np.array([0.25, 0.25, 0.50, 0.50], dtype=np.float32)\n\ndef evaluate_center_box_baseline(df, max_samples=200):\n    df_eval = df[df['Target']==1].sample(n=min(max_samples, len(df[df['Target']==1])), random_state=RANDOM_STATE)\n    ious=[]\n    for _, row in df_eval.iterrows():\n        gt = [row['x_norm'], row['y_norm'], row['w_norm'], row['h_norm']]\n        ious.append(bbox_iou(gt, center_box_baseline_bbox()))\n    ious=np.array(ious)\n    return {'mAP@0.4':float(np.mean(ious>=0.4)), 'mAP@0.5':float(np.mean(ious>=0.5)), 'Mean IoU':float(np.mean(ious))}\n\ncenter_baseline = evaluate_center_box_baseline(test_df)\nbaseline_comparison_df = pd.DataFrame([\n    {'Method':'Center Box Baseline', 'mAP@0.4':center_baseline['mAP@0.4'], 'mAP@0.5':center_baseline['mAP@0.5'], 'Mean IoU':center_baseline['Mean IoU'], 'Classification':0.0, 'FPR':1.0, 'Inference Time (s)':0.001},\n    {'Method':'Proposed Pipeline', 'mAP@0.4':float(best_threshold_row['mAP@0.4']), 'mAP@0.5':float(best_threshold_row['mAP@0.5']), 'Mean IoU':float(reg_results_df[reg_results_df['Model']==best_reg_name]['Mean IoU'].iloc[0]), 'Classification':float(best_threshold_row['Detection']), 'FPR':float(best_threshold_row['False Positive Rate']), 'Inference Time (s)':float(best_threshold_row['Inference Time (s)'])}\n])\nbaseline_comparison_df.to_csv('/kaggle/working/results/tables/baseline_comparison.csv', index=False)\nbaseline_comparison_df.plot(x='Method', y=['mAP@0.4','mAP@0.5','Classification'], kind='bar', figsize=(8,5))\nplt.title('Comparison with Baseline'); plt.ylabel('Score'); plt.xticks(rotation=0); plt.grid(axis='y'); plt.tight_layout()\nplt.savefig('/kaggle/working/results/figures/figure7_baseline_comparison_corrected.png', dpi=300); plt.show()\nbaseline_comparison_df","metadata":{},"outputs":[],"execution_count":null},{"id":"84d5eeb1","cell_type":"markdown","source":"## 15. Statistical, Failure Mode, and Efficiency Analysis","metadata":{}},{"id":"650e99f8","cell_type":"code","source":"# Statistical significance tests\nsorted_reg = reg_results_df.sort_values('IoU@0.5', ascending=False)\nbest_reg_name_stat = sorted_reg.iloc[0]['Model']\nsecond_reg_name = sorted_reg.iloc[1]['Model']\nbest_ious = reg_predictions[best_reg_name_stat]['ious']\nsecond_ious = reg_predictions[second_reg_name]['ious']\n_, p_ttest = ttest_rel(best_ious, second_ious)\ntry:\n    _, p_wilcoxon = wilcoxon(best_ious, second_ious)\nexcept ValueError:\n    p_wilcoxon = np.nan\nstat_regression_df = pd.DataFrame([{'Comparison':f'{best_reg_name_stat} vs {second_reg_name}', 'Metric':'IoU', 'Paired t-test p-value':p_ttest, 'Wilcoxon p-value':p_wilcoxon, 'Significant p<0.05':p_ttest < 0.05}])\nstat_regression_df.to_csv('/kaggle/working/results/tables/statistical_regression_tests.csv', index=False)\n\nsorted_clf = clf_results_v2_df.sort_values('F1-score', ascending=False)\nbest_clf_name_stat = sorted_clf.iloc[0]['Model']\nsecond_clf_name = sorted_clf.iloc[1]['Model']\nbest_correct = (clf_predictions_v2[best_clf_name_stat]['y_pred'] == y_test_roi_v2).astype(int)\nsecond_correct = (clf_predictions_v2[second_clf_name]['y_pred'] == y_test_roi_v2).astype(int)\n_, p_ttest_cls = ttest_rel(best_correct, second_correct)\ntry:\n    _, p_wilcoxon_cls = wilcoxon(best_correct, second_correct)\nexcept ValueError:\n    p_wilcoxon_cls = np.nan\nstat_classification_df = pd.DataFrame([{'Comparison':f'{best_clf_name_stat} vs {second_clf_name}', 'Metric':'Correct Classification', 'Paired t-test p-value':p_ttest_cls, 'Wilcoxon p-value':p_wilcoxon_cls, 'Significant p<0.05':p_ttest_cls < 0.05}])\nstat_classification_df.to_csv('/kaggle/working/results/tables/statistical_classification_tests.csv', index=False)\n\ndisplay(stat_regression_df)\ndisplay(stat_classification_df)","metadata":{},"outputs":[],"execution_count":null},{"id":"74849493","cell_type":"code","source":"# Failure mode analysis\nfailure_rows=[]\nfor _, row in best_e2e_details_df.iterrows():\n    if row['true_class'] == 1:\n        status = 'TP' if row['pred_class'] == 1 else 'FN'\n        if pd.isna(row['iou']): loc_status = 'No IoU'\n        elif row['iou'] >= 0.5: loc_status = 'Good localization'\n        elif row['iou'] >= 0.4: loc_status = 'Moderate localization'\n        elif row['iou'] > 0: loc_status = 'Poor localization'\n        else: loc_status = 'Localization failure'\n    else:\n        status = 'FP' if row['pred_class'] == 1 else 'TN'\n        loc_status = 'Not applicable'\n    failure_rows.append({'patientId':row['patientId'], 'true_class':row['true_class'], 'pred_class':row['pred_class'], 'confidence':row['confidence'], 'iou':row['iou'], 'classification_status':status, 'localization_status':loc_status})\n\nfailure_analysis_df = pd.DataFrame(failure_rows)\nfailure_summary = failure_analysis_df.groupby(['classification_status','localization_status']).size().reset_index(name='Count')\nfailure_analysis_df.to_csv('/kaggle/working/results/tables/failure_analysis_details.csv', index=False)\nfailure_summary.to_csv('/kaggle/working/results/tables/failure_analysis_summary.csv', index=False)\nfailure_summary","metadata":{},"outputs":[],"execution_count":null},{"id":"86208783","cell_type":"code","source":"# Computational efficiency analysis\nefficiency_rows = [{'Component':'Feature Extraction', 'Average Time (s)':train_feature_log['total_time_ms'].mean()/1000, 'Memory / Size':f'{X_train.nbytes/(1024**2):.2f} MB', 'Notes':'Full-image HOG + LBP + ORB + BRISK + Edge features'}]\nfor _, row in reg_results_df.iterrows():\n    efficiency_rows.append({'Component':f\"Regressor - {row['Model']}\", 'Average Time (s)':row['Inference Time'], 'Memory / Size':'Saved model', 'Notes':f\"Training time: {row['Training Time']:.2f}s\"})\nfor _, row in clf_results_v2_df.iterrows():\n    efficiency_rows.append({'Component':f\"Classifier - {row['Model']}\", 'Average Time (s)':row['Inference Time'], 'Memory / Size':'Saved model', 'Notes':f\"Training time: {row['Training Time']:.2f}s\"})\nefficiency_rows.append({'Component':'End-to-End Pipeline', 'Average Time (s)':best_threshold_row['Inference Time (s)'], 'Memory / Size':'Feature + regressor + classifier', 'Notes':f\"{best_threshold_row['Regressor']} + {best_threshold_row['Classifier']}, threshold={best_threshold_row['Threshold']}\"})\nefficiency_df = pd.DataFrame(efficiency_rows)\nefficiency_df.to_csv('/kaggle/working/results/tables/computational_efficiency_analysis.csv', index=False)\nefficiency_df","metadata":{},"outputs":[],"execution_count":null},{"id":"a0e7a472","cell_type":"markdown","source":"## 16. Output File Check","metadata":{}},{"id":"c8f45948","cell_type":"code","source":"for root, dirs, files in os.walk('/kaggle/working/results'):\n    level = root.replace('/kaggle/working/results', '').count(os.sep)\n    indent = ' ' * 4 * level\n    print(f'{indent}{os.path.basename(root)}/')\n    subindent = ' ' * 4 * (level + 1)\n    for f in files:\n        print(f'{subindent}{f}')","metadata":{},"outputs":[],"execution_count":null}]}