{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":10338,"databundleVersionId":862042},{"sourceType":"kernelVersion","sourceId":309042418}],"dockerImageVersionId":31329,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#for kaggle\nimport os\nos.environ[\"RSNA_DATA_DIR\"] = \"/kaggle/input/competitions/rsna-pneumonia-detection-challenge\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-04-30T10:44:48.315426Z","iopub.execute_input":"2026-04-30T10:44:48.315659Z","iopub.status.idle":"2026-04-30T10:44:48.33324Z","shell.execute_reply.started":"2026-04-30T10:44:48.315632Z","shell.execute_reply":"2026-04-30T10:44:48.332595Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import shutil\nfrom pathlib import Path\n\nsrc = Path(\"/kaggle/input/notebooks/moemenelsayed/demo-grad/Code\")\ndst = Path(\"/kaggle/working/Code\")\n\nif not src.exists():\n    raise FileNotFoundError(f\"Source folder not found: {src}\")\n\nif dst.exists():\n    shutil.rmtree(dst)\n\nshutil.copytree(src, dst)\n\nprint(\"Copied successfully!\")\nprint(f\"From: {src}\")\nprint(f\"To:   {dst}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-30T10:44:52.385023Z","iopub.execute_input":"2026-04-30T10:44:52.385911Z","iopub.status.idle":"2026-04-30T10:52:17.35075Z","shell.execute_reply.started":"2026-04-30T10:44:52.385874Z","shell.execute_reply":"2026-04-30T10:52:17.350065Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nfor root, dirs, files in os.walk(\"/kaggle/working/Code\"):\n    level = root.replace(\"/kaggle/working/Code\", \"\").count(os.sep)\n    indent = \" \" * 4 * level\n    print(f\"{indent}{Path(root).name}/\")\n    for f in files[:20]:\n        print(f\"{indent}    {f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-30T10:59:44.530243Z","iopub.execute_input":"2026-04-30T10:59:44.530587Z","iopub.status.idle":"2026-04-30T10:59:44.568745Z","shell.execute_reply.started":"2026-04-30T10:59:44.53056Z","shell.execute_reply":"2026-04-30T10:59:44.568012Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#for kaggle\nimport os\nos.environ[\"RSNA_DATA_DIR\"] = \"/kaggle/input/competitions/rsna-pneumonia-detection-challenge\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-30T10:59:52.139945Z","iopub.execute_input":"2026-04-30T10:59:52.140462Z","iopub.status.idle":"2026-04-30T10:59:52.144006Z","shell.execute_reply.started":"2026-04-30T10:59:52.140433Z","shell.execute_reply":"2026-04-30T10:59:52.143301Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 0.1) Project path setup - Output/Working only\nimport os\nimport sys\nimport json\nfrom pathlib import Path\n\ndef _is_project_root(path: Path) -> bool:\n    \"\"\"\n    Check if this folder is the project root.\n    The project root must contain src/config.py\n    \"\"\"\n    return path.is_dir() and (path / \"src\" / \"config.py\").is_file()\n\n\ndef find_project_root() -> Path:\n    \"\"\"\n    Find project root only inside Kaggle working/output area.\n    It will NOT search or copy anything from /kaggle/input.\n    \"\"\"\n\n    direct_candidates = [\n        Path.cwd(),\n        Path(\"/kaggle/working/Code\"),\n        Path(\"/kaggle/working/kaggle-output/Code\"),\n        Path(\"/kaggle/working/extracted_code\"),\n    ]\n\n    # 1) Try direct known paths\n    for candidate in direct_candidates:\n        if _is_project_root(candidate):\n            return candidate.resolve()\n\n    # 2) Search only inside /kaggle/working\n    working_base = Path(\"/kaggle/working\")\n\n    if working_base.exists():\n        for child in working_base.iterdir():\n            if _is_project_root(child):\n                return child.resolve()\n\n            if child.is_dir():\n                for nested in child.iterdir():\n                    if _is_project_root(nested):\n                        return nested.resolve()\n\n    raise FileNotFoundError(\n        \"Project root not found in /kaggle/working only. \"\n        \"Expected src/config.py inside paths like:\\n\"\n        \"- /kaggle/working/Code\\n\"\n        \"- /kaggle/working/kaggle-output/Code\\n\"\n        \"- /kaggle/working/all_kernel_output/Code\"\n    )\n\n\nPROJECT_ROOT = find_project_root()\n\n# Change current working directory to project root\nos.chdir(PROJECT_ROOT)\n\n# Add project root to Python path\nproject_root_str = str(PROJECT_ROOT)\nif project_root_str not in sys.path:\n    sys.path.insert(0, project_root_str)\n\n# Store useful paths\nPATHS = {\n    \"PROJECT_ROOT\": PROJECT_ROOT,\n    \"SRC_DIR\": PROJECT_ROOT / \"src\",\n    \"ARTIFACTS_DIR\": PROJECT_ROOT / \"artifacts\",\n    \"PNG_IMAGES_DIR\": PROJECT_ROOT / \"png_images\",\n    \"WORKING_DIR\": Path(\"/kaggle/working\"),\n}\n\n# Create artifacts folder if it does not exist\nPATHS[\"ARTIFACTS_DIR\"].mkdir(parents=True, exist_ok=True)\n\nprint(\"PROJECT_ROOT:\", PATHS[\"PROJECT_ROOT\"])\nprint(\"SRC_DIR:\", PATHS[\"SRC_DIR\"])\nprint(\"ARTIFACTS_DIR:\", PATHS[\"ARTIFACTS_DIR\"])\nprint(\"PNG_IMAGES_DIR:\", PATHS[\"PNG_IMAGES_DIR\"])\nprint(\"Working dir:\", os.getcwd())\nprint(\"src/config.py exists:\", (PROJECT_ROOT / \"src\" / \"config.py\").is_file())\nprint(\"Searching from /kaggle/input:\", False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-30T11:00:03.068767Z","iopub.execute_input":"2026-04-30T11:00:03.069042Z","iopub.status.idle":"2026-04-30T11:00:03.079375Z","shell.execute_reply.started":"2026-04-30T11:00:03.069016Z","shell.execute_reply":"2026-04-30T11:00:03.078423Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# 0.2) Imports from your project (same codebase)\nfrom src.dataset import explore_dataset\nfrom src.preprocessing import convert_dicom_to_png\nfrom src.visualization import show_pneumonia_example\nfrom src.yolo_dataset import build_yolo_dataset\nfrom src.yolo_visualization import show_yolo_samples\n\nfrom src.detection.train_yolo import train_yolo\nfrom src.detection.train_fasterrcnn import train_fasterrcnn\nfrom src.detection.train_retinanet import train_retinanet\n\nfrom src.classification.train_resnet import train_resnet\nfrom src.classification.train_densenet import train_densenet\nfrom src.classification.train_efficientnet import train_efficientnet\n\nfrom src.phase4_optimization import run_phase4_optimization\nfrom src.phase5_retrain import run_phase5_retrain\nfrom src.phase6_explainability import run_phase6_gradcam\nfrom src.phase7_final_evaluation import run_phase7_final_evaluation\nfrom src.phase8_demo import run_phase8_demo\nfrom src.preflight import run_preflight_checks\nfrom src.config import YOLO_DATASET_DIR\n\n# Config flags\nRUN_PHASE1 = False\nRUN_PHASE2 = False\nRUN_PHASE3 = False\nRUN_PHASE4 = False\nRUN_PHASE5 = False\nRUN_PHASE6 = False\nRUN_PHASE7 = False\nRUN_PHASE8 = False\n\nQUICK_MODE = False\nif QUICK_MODE:\n    DET_BASELINE_EPOCHS = 1\n    CLS_BASELINE_EPOCHS = 1\n    OPT_QUICK_EPOCHS = 1\n    OPT_POPULATION = 3\n    OPT_ITERATIONS = 1\n    DET_RETRAIN_EPOCHS = 2\n    CLS_RETRAIN_EPOCHS = 2\nelse:\n    DET_BASELINE_EPOCHS = 2\n    CLS_BASELINE_EPOCHS = 3\n    OPT_QUICK_EPOCHS = 1\n    OPT_POPULATION = 4\n    OPT_ITERATIONS = 2\n    DET_RETRAIN_EPOCHS = 20\n    CLS_RETRAIN_EPOCHS = 8\n\nDEMO_IMAGE_SOURCE = os.path.join(YOLO_DATASET_DIR, \"val\", \"images\")\nresults = {}\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-30T11:00:16.559854Z","iopub.execute_input":"2026-04-30T11:00:16.56064Z","iopub.status.idle":"2026-04-30T11:00:34.567406Z","shell.execute_reply.started":"2026-04-30T11:00:16.560608Z","shell.execute_reply":"2026-04-30T11:00:34.566725Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = None\npreflight = run_preflight_checks()\nresults[\"preflight\"] = preflight\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-30T11:00:34.568784Z","iopub.execute_input":"2026-04-30T11:00:34.56934Z","iopub.status.idle":"2026-04-30T11:00:34.576479Z","shell.execute_reply.started":"2026-04-30T11:00:34.569272Z","shell.execute_reply":"2026-04-30T11:00:34.575726Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if RUN_PHASE4:\n    try:\n        phase4 = run_phase4_optimization(\n            quick_epochs=OPT_QUICK_EPOCHS,\n            population=OPT_POPULATION,\n            iterations=OPT_ITERATIONS,\n        )\n        results[\"phase4\"] = {\"status\": \"ok\", \"data\": phase4}\n    except Exception as e:\n        results[\"phase4\"] = {\"status\": \"failed\", \"error\": str(e)}\n        print(\"Phase 4 failed:\", e)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-27T15:45:42.382667Z","iopub.execute_input":"2026-04-27T15:45:42.383337Z","iopub.status.idle":"2026-04-27T21:30:33.134131Z","shell.execute_reply.started":"2026-04-27T15:45:42.383303Z","shell.execute_reply":"2026-04-27T21:30:33.133329Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================\n# Phase 4 - ONLY:\n# Faster R-CNN, RetinaNet, DenseNet121\n# Ready Kaggle Cell\n# =========================\n\nimport os\nimport sys\nimport json\nimport gc\nfrom pathlib import Path\nfrom typing import Callable, Dict, List, Tuple\n\nimport torch\n\n# -------------------------\n# 1) Find project root\n# -------------------------\ndef _is_project_root(path: Path) -> bool:\n    return path.is_dir() and (path / \"src\" / \"config.py\").is_file()\n\ndef find_project_root() -> Path:\n    candidates = [\n        Path.cwd(),\n        Path(\"/kaggle/working/Code\"),\n        Path(\"/kaggle/working\"),\n    ]\n\n    for candidate in candidates:\n        if _is_project_root(candidate):\n            return candidate.resolve()\n\n    for base in [Path(\"/kaggle/working\"), Path(\"/kaggle/input\")]:\n        if not base.exists():\n            continue\n\n        for p in base.rglob(\"src/config.py\"):\n            return p.parent.parent.resolve()\n\n    raise FileNotFoundError(\"Project root not found. Expected src/config.py inside project folder.\")\n\nPROJECT_ROOT = find_project_root()\nos.chdir(PROJECT_ROOT)\nsys.path.insert(0, str(PROJECT_ROOT))\n\nprint(f\"Project root: {PROJECT_ROOT}\")\n\n# -------------------------\n# 2) Imports from project\n# -------------------------\nfrom src.config import ARTIFACT_DIR\n\nfrom src.classification.train_densenet import train_densenet\nfrom src.detection.train_fasterrcnn import train_fasterrcnn\nfrom src.detection.train_retinanet import train_retinanet\n\nfrom src.optimization.algorithms import (\n    SearchDimension,\n    pso_optimize,\n    gwo_optimize,\n    sa_optimize,\n)\n\nBEST_PARAMS_PATH = os.path.join(\n    ARTIFACT_DIR,\n    \"phase4_best_hyperparameters_only_fasterrcnn_retinanet_densenet121.json\"\n)\n\n# -------------------------\n# 3) Helpers\n# -------------------------\ndef _cleanup():\n    gc.collect()\n    if torch.cuda.is_available():\n        torch.cuda.empty_cache()\n\ndef _save_json(path: str, data: Dict):\n    os.makedirs(os.path.dirname(path), exist_ok=True)\n    with open(path, \"w\", encoding=\"utf-8\") as f:\n        json.dump(data, f, indent=2)\n\n# -------------------------\n# 4) Evaluation functions\n# -------------------------\ndef _evaluate_detection_model(\n    model_name: str,\n    params: Dict[str, float],\n    quick_epochs: int\n) -> float:\n    try:\n        batch_size = int(params[\"batch_size\"])\n\n        if model_name == \"fasterrcnn\":\n            metrics = train_fasterrcnn(\n                epochs=quick_epochs,\n                lr=float(params[\"lr\"]),\n                batch_size=batch_size,\n                weight_decay=float(params[\"weight_decay\"]),\n            )\n            return float(metrics.get(\"recall\", 0.0))\n\n        if model_name == \"retinanet\":\n            metrics = train_retinanet(\n                epochs=quick_epochs,\n                lr=float(params[\"lr\"]),\n                batch_size=batch_size,\n                weight_decay=float(params[\"weight_decay\"]),\n            )\n            return float(metrics.get(\"recall\", 0.0))\n\n        raise ValueError(f\"Unknown detection model: {model_name}\")\n\n    except Exception as exc:\n        print(f\"[Phase4] Detection eval failed for {model_name} with {params}: {exc}\")\n        return 0.0\n\n    finally:\n        _cleanup()\n\ndef _evaluate_classification_model(\n    model_name: str,\n    params: Dict[str, float],\n    quick_epochs: int\n) -> float:\n    try:\n        if model_name == \"densenet121\":\n            metrics = train_densenet(\n                epochs=quick_epochs,\n                lr=float(params[\"lr\"]),\n                batch_size=int(params[\"batch_size\"]),\n                dropout=float(params[\"dropout\"]),\n                weight_decay=float(params[\"weight_decay\"]),\n            )\n            return float(metrics.get(\"auc\", 0.0))\n\n        raise ValueError(f\"Unknown classification model: {model_name}\")\n\n    except Exception as exc:\n        print(f\"[Phase4] Classification eval failed for {model_name} with {params}: {exc}\")\n        return 0.0\n\n    finally:\n        _cleanup()\n\n# -------------------------\n# 5) Run PSO, GWO, SA\n# -------------------------\ndef _run_all_algorithms(\n    objective: Callable[[Dict[str, float]], float],\n    dims: List[SearchDimension],\n    population: int,\n    iterations: int,\n) -> Tuple[Dict[str, float], float, Dict[str, Dict[str, float]]]:\n\n    algo_results = {}\n\n    best_params, best_score = pso_optimize(\n        objective,\n        dims,\n        population=population,\n        iterations=iterations,\n    )\n    algo_results[\"PSO\"] = {\n        \"best_params\": best_params,\n        \"best_score\": best_score,\n    }\n\n    params, score = gwo_optimize(\n        objective,\n        dims,\n        population=population,\n        iterations=iterations,\n    )\n    algo_results[\"GWO\"] = {\n        \"best_params\": params,\n        \"best_score\": score,\n    }\n\n    if score > best_score:\n        best_params, best_score = params, score\n\n    params, score = sa_optimize(\n        objective,\n        dims,\n        iterations=max(10, population * iterations),\n    )\n    algo_results[\"SA\"] = {\n        \"best_params\": params,\n        \"best_score\": score,\n    }\n\n    if score > best_score:\n        best_params, best_score = params, score\n\n    return best_params, best_score, algo_results\n\n# -------------------------\n# 6) Main Phase 4 function\n# -------------------------\ndef run_phase4_optimization(\n    quick_epochs: int = 1,\n    population: int = 3,\n    iterations: int = 2,\n):\n    print(\"Phase 4: Nature-Inspired Hyperparameter Optimization\")\n    print(\"Running ONLY: Faster R-CNN, RetinaNet, DenseNet121\")\n\n    all_results: Dict[str, Dict] = {\"_errors\": {}}\n\n    # Safer for Kaggle memory with detection models\n    detection_dims = [\n        SearchDimension(\"lr\", 1e-5, 5e-3, \"float\"),\n        SearchDimension(\"batch_size\", 1, 4, \"int\"),\n        SearchDimension(\"weight_decay\", 1e-6, 1e-2, \"float\"),\n    ]\n\n    classification_dims = [\n        SearchDimension(\"lr\", 1e-5, 1e-3, \"float\"),\n        SearchDimension(\"batch_size\", 4, 16, \"int\"),\n        SearchDimension(\"dropout\", 0.1, 0.6, \"float\"),\n        SearchDimension(\"weight_decay\", 1e-7, 1e-2, \"float\"),\n    ]\n\n    # -------------------------\n    # Detection models\n    # -------------------------\n    for det_model in [\"fasterrcnn\", \"retinanet\"]:\n        print(f\"\\n========== Optimizing {det_model} ==========\")\n\n        objective = lambda p, m=det_model: _evaluate_detection_model(\n            m,\n            p,\n            quick_epochs=quick_epochs,\n        )\n\n        try:\n            best_params, best_score, algo_results = _run_all_algorithms(\n                objective=objective,\n                dims=detection_dims,\n                population=population,\n                iterations=iterations,\n            )\n\n            all_results[det_model] = {\n                \"task\": \"detection\",\n                \"score_name\": \"recall@0.5\",\n                \"best_score\": float(best_score),\n                \"best_hyperparameters\": best_params,\n                \"algorithms\": algo_results,\n            }\n\n            print(f\"[Phase4] {det_model} best score={best_score:.4f}\")\n            print(f\"[Phase4] best params: {best_params}\")\n\n        except Exception as exc:\n            all_results[\"_errors\"][det_model] = str(exc)\n            print(f\"[Phase4] {det_model} optimization failed: {exc}\")\n\n        _save_json(BEST_PARAMS_PATH, all_results)\n\n    # -------------------------\n    # Classification model\n    # -------------------------\n    for cls_model in [\"densenet121\"]:\n        print(f\"\\n========== Optimizing {cls_model} ==========\")\n\n        objective = lambda p, m=cls_model: _evaluate_classification_model(\n            m,\n            p,\n            quick_epochs=quick_epochs,\n        )\n\n        try:\n            best_params, best_score, algo_results = _run_all_algorithms(\n                objective=objective,\n                dims=classification_dims,\n                population=population,\n                iterations=iterations,\n            )\n\n            all_results[cls_model] = {\n                \"task\": \"classification\",\n                \"score_name\": \"auc\",\n                \"best_score\": float(best_score),\n                \"best_hyperparameters\": best_params,\n                \"algorithms\": algo_results,\n            }\n\n            print(f\"[Phase4] {cls_model} best score={best_score:.4f}\")\n            print(f\"[Phase4] best params: {best_params}\")\n\n        except Exception as exc:\n            all_results[\"_errors\"][cls_model] = str(exc)\n            print(f\"[Phase4] {cls_model} optimization failed: {exc}\")\n\n        _save_json(BEST_PARAMS_PATH, all_results)\n\n    print(f\"\\nPhase 4 results saved to:\")\n    print(BEST_PARAMS_PATH)\n\n    return all_results\n\n# -------------------------\n# 7) Run now\n# -------------------------\nresults = run_phase4_optimization(\n    quick_epochs=1,\n    population=3,\n    iterations=2,\n)\n\nresults","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-30T11:00:50.817092Z","iopub.execute_input":"2026-04-30T11:00:50.817685Z","execution_failed":"2026-04-30T22:44:46.995Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!mkdir -p /kaggle/working/kaggle-output\n!kaggle kernels output moemenelsayed/notebook150d555b1f \\\n-p /kaggle/working/kaggle-output \\\n-o","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-30T22:54:52.857802Z","iopub.execute_input":"2026-04-30T22:54:52.858069Z","iopub.status.idle":"2026-04-30T22:56:18.445909Z","shell.execute_reply.started":"2026-04-30T22:54:52.858037Z","shell.execute_reply":"2026-04-30T22:56:18.445206Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!cd /kaggle/working/kaggle-output/Code && zip -r /kaggle/working/artifacts.zip artifacts","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-30T22:56:18.44765Z","iopub.execute_input":"2026-04-30T22:56:18.448711Z","iopub.status.idle":"2026-04-30T22:56:40.865395Z","shell.execute_reply.started":"2026-04-30T22:56:18.44868Z","shell.execute_reply":"2026-04-30T22:56:40.86453Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!ls -lh /kaggle/working/artifacts.zip","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-30T22:56:40.866916Z","iopub.execute_input":"2026-04-30T22:56:40.867221Z","iopub.status.idle":"2026-04-30T22:56:40.988187Z","shell.execute_reply.started":"2026-04-30T22:56:40.867192Z","shell.execute_reply":"2026-04-30T22:56:40.987241Z"}},"outputs":[],"execution_count":null}]}