{"cells": [{"cell_type": "markdown", "metadata": {}, "source": "# Perseptron v2 - 06 Explainability\n\nBu notebook proposal uyumlu explainability ciktilarini uretir. Tabular taraf icin SHAP varsa SHAP, yoksa permutation fallback kullanilir. CNN tarafi icin Grad-CAM ornekleri uretilir.\n\nCiktilar:\n\n- `reports/proposal_v2/proposal_v2_shap_summary.csv`\n- `reports/proposal_v2/gradcam_examples/*.png`"}, {"cell_type": "markdown", "metadata": {}, "source": "## Ortam ve output klasorleri\n\nBu hucre Kaggle icin yazilabilir output klasorlerini hazirlar ve onceki notebook outputlari `Add Data` ile eklendiyse bunlari geri yukler. Repo icindeki `.py` modulleri import edilmez."}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "from pathlib import Path\nimport json\nimport os\nimport random\nimport shutil\nimport warnings\n\nimport numpy as np\nimport pandas as pd\n\nwarnings.filterwarnings('ignore')\n\nIS_KAGGLE = Path('/kaggle').exists()\nWORK_DIR = Path('/kaggle/working') if IS_KAGGLE else Path.cwd()\nREPORTS_DIR = WORK_DIR / 'reports' / 'proposal_v2'\nMODELS_DIR = WORK_DIR / 'models' / 'proposal_v2'\nGRADCAM_DIR = REPORTS_DIR / 'gradcam_examples'\nfor path in [REPORTS_DIR, REPORTS_DIR / 'folds', MODELS_DIR, GRADCAM_DIR]:\n    path.mkdir(parents=True, exist_ok=True)\n\ndef candidate_output_roots(input_root):\n    roots = []\n    seen = set()\n\n    def add_root(path):\n        resolved = path.resolve()\n        if resolved not in seen:\n            roots.append(path)\n            seen.add(resolved)\n\n    notebooks_root = input_root / 'notebooks'\n    if notebooks_root.exists():\n        for path in notebooks_root.rglob('*'):\n            if path.is_dir() and ((path / 'reports' / 'proposal_v2').exists() or (path / 'models' / 'proposal_v2').exists()):\n                add_root(path)\n    datasets_root = input_root / 'datasets'\n    if datasets_root.exists():\n        for path in datasets_root.rglob('*'):\n            if not path.is_dir():\n                continue\n            name = path.name.lower()\n            if any(token in name for token in ['proposal', 'report', 'output', 'fold']):\n                add_root(path)\n    for path in input_root.glob('*'):\n        if path.is_dir() and path.name not in {'competitions', 'datasets', 'notebooks'}:\n            add_root(path)\n    return roots\n\ndef merge_or_copy_file(source, target):\n    target.parent.mkdir(parents=True, exist_ok=True)\n    if source.suffix == '.csv' and target.exists():\n        try:\n            current = pd.read_csv(target)\n            incoming = pd.read_csv(source)\n            merged = pd.concat([current, incoming], ignore_index=True)\n            if {'fold_id', 'model'}.issubset(merged.columns):\n                merged = merged.drop_duplicates(['fold_id', 'model'], keep='last')\n            elif 'customer_id' in merged.columns and 'model' in merged.columns:\n                keys = [column for column in ['fold_id', 'customer_id', 'model'] if column in merged.columns]\n                merged = merged.drop_duplicates(keys, keep='last')\n            else:\n                merged = merged.drop_duplicates(keep='last')\n            merged.to_csv(target, index=False)\n            print(f'Merged previous CSV: {source} -> {target}')\n            return\n        except Exception as exc:\n            print(f'CSV merge failed, falling back to copy for {source}: {exc}')\n    if not target.exists() or source.stat().st_size != target.stat().st_size:\n        shutil.copy2(source, target)\n        print(f'Restored previous output file: {source} -> {target}')\n\ndef restore_previous_outputs():\n    input_root = Path('/kaggle/input')\n    if not input_root.exists():\n        return\n    for root in candidate_output_roots(input_root):\n        for source in root.rglob('*'):\n            if not source.is_file():\n                continue\n            try:\n                relative = source.relative_to(root)\n            except ValueError:\n                relative = Path(source.name)\n            relative_text = relative.as_posix()\n            name = source.name\n            if relative_text.startswith('reports/proposal_v2/'):\n                target = WORK_DIR / relative\n            elif relative_text.startswith('models/proposal_v2/'):\n                target = WORK_DIR / relative\n            elif name.endswith('.pt'):\n                target = MODELS_DIR / name\n            elif name.endswith('_gradcam.png'):\n                target = GRADCAM_DIR / name\n            elif name.startswith('proposal_v2_') and name.endswith(('.csv', '.json', '.md')):\n                target = REPORTS_DIR / name\n            elif name.startswith('fold') and name.endswith('_context.json'):\n                target = REPORTS_DIR / 'folds' / name\n            else:\n                continue\n            merge_or_copy_file(source, target)\n\nrestore_previous_outputs()\nprint('WORK_DIR    =', WORK_DIR)\nprint('REPORTS_DIR =', REPORTS_DIR)\nprint('MODELS_DIR  =', MODELS_DIR)\n"}, {"cell_type": "markdown", "metadata": {}, "source": "## Ortak sabitler\n\nBu hucre pathleri, feature listelerini, model adlarini ve deney varsayilanlarini tanimlar."}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "from dataclasses import dataclass\n\nDATA_DIR = WORK_DIR / 'data'\nREPORTS_DIR = WORK_DIR / 'reports' / 'proposal_v2'\nMODELS_DIR = WORK_DIR / 'models' / 'proposal_v2'\n\nDEFAULT_SEED = 42\nDEFAULT_N_FOLDS = 5\nDEFAULT_VALIDATION_DAYS = 7\n\nCUSTOMER_NUMERIC_FEATURES = ['FN', 'Active', 'age']\nARTICLE_NUMERIC_FEATURES = [\n    'product_code',\n    'product_type_no',\n    'graphical_appearance_no',\n    'colour_group_code',\n    'perceived_colour_value_id',\n    'perceived_colour_master_id',\n    'department_no',\n    'index_group_no',\n    'section_no',\n    'garment_group_no',\n]\nVISUAL_NUMERIC_FEATURES = ['visual_similarity', 'visual_history_count']\nTABULAR_NUMERIC_FEATURES = CUSTOMER_NUMERIC_FEATURES + ARTICLE_NUMERIC_FEATURES\nFUSION_NUMERIC_FEATURES = TABULAR_NUMERIC_FEATURES + VISUAL_NUMERIC_FEATURES\n\nCUSTOMER_CATEGORICAL_FEATURES = ['club_member_status', 'fashion_news_frequency']\nARTICLE_CATEGORICAL_FEATURES = [\n    'product_type_name',\n    'product_group_name',\n    'graphical_appearance_name',\n    'colour_group_name',\n    'perceived_colour_value_name',\n    'perceived_colour_master_name',\n    'department_name',\n    'index_code',\n    'index_name',\n    'index_group_name',\n    'section_name',\n    'garment_group_name',\n]\nCATEGORICAL_FEATURES = CUSTOMER_CATEGORICAL_FEATURES + ARTICLE_CATEGORICAL_FEATURES\nMODEL_NAMES = ('tabular_only', 'image_history', 'late_fusion')\n\n@dataclass(frozen=True)\nclass V2Defaults:\n    n_folds: int = DEFAULT_N_FOLDS\n    validation_days: int = DEFAULT_VALIDATION_DAYS\n    seed: int = DEFAULT_SEED\n    top_k: int = 12\n    precision_k: int = 10\n    candidate_limit: int = 5000\n    visual_neighbors: int = 3000\n    co_purchase_per_item: int = 300\n    hybrid_weights: tuple[float, ...] = (0.25, 0.45, 0.65)\n    negatives_per_positive: int = 1\n    train_batch_size: int = 4096\n    epochs: int = 3\n    learning_rate: float = 1e-3\n\ndef ensure_v2_dirs():\n    for path in [REPORTS_DIR, REPORTS_DIR / 'folds', REPORTS_DIR / 'gradcam_examples', MODELS_DIR]:\n        path.mkdir(parents=True, exist_ok=True)"}, {"cell_type": "markdown", "metadata": {}, "source": "## Veri yukleme yardimcilari\n\nBu hucre H&M raw dosyalarini, image klasorunu ve EfficientNet embedding cache dosyalarini Kaggle inputlari veya local klasorlerden bulmak icin yardimci fonksiyonlari tanimlar."}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "\nimport os\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\n\n\n\ndef log(message: str) -> None:\n    print(message, flush=True)\n\n\ndef _candidate_raw_dirs() -> list[Path]:\n    candidates: list[Path] = []\n    env_dir = os.environ.get(\"HM_RAW_DIR\")\n    if env_dir:\n        candidates.append(Path(env_dir))\n    candidates.append(DATA_DIR / \"raw\")\n    kaggle_root = Path(\"/kaggle/input\")\n    if kaggle_root.exists():\n        for path in kaggle_root.rglob(\"transactions_train.csv\"):\n            candidates.append(path.parent)\n    return candidates\n\n\ndef resolve_raw_dir(raw_dir: str | Path | None = None) -> Path:\n    candidates = [Path(raw_dir)] if raw_dir else _candidate_raw_dirs()\n    for candidate in candidates:\n        if (candidate / \"transactions_train.csv\").exists():\n            return candidate\n    raise FileNotFoundError(\"Could not find H&M raw data directory. Set HM_RAW_DIR or pass --raw-dir.\")\n\n\ndef resolve_images_dir(raw_dir: Path, images_dir: str | Path | None = None) -> Path:\n    if images_dir:\n        return Path(images_dir)\n    for candidate in [raw_dir / \"images\", DATA_DIR / \"images\" / \"hm_images\"]:\n        if candidate.exists():\n            return candidate\n    return raw_dir / \"images\"\n\n\ndef _find_file_by_name(root: Path, name: str) -> Path | None:\n    if not root.exists():\n        return None\n    for path in root.rglob(name):\n        return path\n    return None\n\n\ndef resolve_embedding_paths(\n    embeddings_path: str | Path | None = None,\n    embedding_ids_path: str | Path | None = None,\n) -> tuple[Path, Path]:\n    if embeddings_path and embedding_ids_path:\n        return Path(embeddings_path), Path(embedding_ids_path)\n\n    env_embeddings = os.environ.get(\"HM_EMBEDDINGS_PATH\")\n    env_ids = os.environ.get(\"HM_EMBEDDING_IDS_PATH\")\n    if env_embeddings and env_ids:\n        return Path(env_embeddings), Path(env_ids)\n\n    local_embeddings = DATA_DIR / \"embeddings\" / \"final_kaggle\" / \"article_image_embeddings_popular.npy\"\n    local_ids = DATA_DIR / \"embeddings\" / \"final_kaggle\" / \"article_image_embedding_ids_popular.csv\"\n    if local_embeddings.exists() and local_ids.exists():\n        return local_embeddings, local_ids\n\n    kaggle_root = Path(\"/kaggle/input\")\n    embeddings = _find_file_by_name(kaggle_root, \"article_image_embeddings_popular.npy\")\n    ids = _find_file_by_name(kaggle_root, \"article_image_embedding_ids_popular.csv\")\n    if embeddings and ids:\n        return embeddings, ids\n\n    raise FileNotFoundError(\"Could not find EfficientNet embedding cache paths.\")\n\n\ndef read_transactions(raw_dir: Path) -> pd.DataFrame:\n    transactions = pd.read_csv(raw_dir / \"transactions_train.csv\", dtype={\"article_id\": str})\n    transactions[\"article_id\"] = transactions[\"article_id\"].astype(str).str.zfill(10)\n    transactions[\"t_dat\"] = pd.to_datetime(transactions[\"t_dat\"])\n    transactions[\"price\"] = transactions[\"price\"].astype(\"float32\")\n    transactions[\"sales_channel_id\"] = transactions[\"sales_channel_id\"].astype(\"int8\")\n    return transactions\n\n\ndef read_customers(raw_dir: Path) -> pd.DataFrame:\n    customers = pd.read_csv(raw_dir / \"customers.csv\")\n    customers[\"FN\"] = customers[\"FN\"].fillna(0).astype(\"float32\")\n    customers[\"Active\"] = customers[\"Active\"].fillna(0).astype(\"float32\")\n    customers[\"age\"] = customers[\"age\"].fillna(customers[\"age\"].median()).astype(\"float32\")\n    for column in [\"club_member_status\", \"fashion_news_frequency\"]:\n        customers[column] = customers[column].fillna(\"UNKNOWN\").astype(str)\n    return customers\n\n\ndef read_articles(raw_dir: Path) -> pd.DataFrame:\n    articles = pd.read_csv(raw_dir / \"articles.csv\", dtype={\"article_id\": str})\n    articles[\"article_id\"] = articles[\"article_id\"].astype(str).str.zfill(10)\n    for column in articles.columns:\n        if articles[column].dtype == \"object\":\n            articles[column] = articles[column].fillna(\"UNKNOWN\").astype(str)\n    return articles\n\n\ndef load_core_tables(raw_dir: str | Path | None = None) -> tuple[Path, pd.DataFrame, pd.DataFrame, pd.DataFrame]:\n    resolved = resolve_raw_dir(raw_dir)\n    log(f\"Using raw data: {resolved}\")\n    return resolved, read_transactions(resolved), read_customers(resolved), read_articles(resolved)\n\n\ndef load_embeddings(\n    embeddings_path: str | Path | None = None,\n    embedding_ids_path: str | Path | None = None,\n    mmap_mode: str | None = \"r\",\n) -> tuple[np.ndarray, list[str], dict[str, int]]:\n    emb_path, ids_path = resolve_embedding_paths(embeddings_path, embedding_ids_path)\n    log(f\"Using embeddings: {emb_path}\")\n    embeddings = np.load(emb_path, mmap_mode=mmap_mode)\n    ids_frame = pd.read_csv(ids_path, dtype={\"article_id\": str})\n    article_ids = ids_frame[\"article_id\"].astype(str).str.zfill(10).tolist()\n    article_to_index = {article_id: index for index, article_id in enumerate(article_ids)}\n    return embeddings, article_ids, article_to_index\n\n\ndef article_image_path(images_dir: Path, article_id: str) -> Path:\n    padded = str(article_id).zfill(10)\n    nested = images_dir / padded[:3] / f\"{padded}.jpg\"\n    if nested.exists():\n        return nested\n    return images_dir / f\"{padded}.jpg\""}, {"cell_type": "markdown", "metadata": {}, "source": "## Explainability yardimci kodu\n\nBu hucre model reconstruction, validation sample hazirligi, SHAP/permutation importance ve Grad-CAM generation kodunu tanimlar."}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "\nimport json\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\n\n\n\ndef make_cutoff(transactions: pd.DataFrame, validation_days: int) -> pd.Timestamp:\n    return transactions[\"t_dat\"].max() - pd.Timedelta(days=validation_days)\n\n\ndef sample_positive_pairs(\n    transactions: pd.DataFrame,\n    customers: set[str],\n    cutoff: pd.Timestamp,\n    max_positives: int | None,\n    seed: int,\n) -> pd.DataFrame:\n    positives = transactions[\n        transactions[\"customer_id\"].isin(customers) & (transactions[\"t_dat\"] <= cutoff)\n    ][[\"customer_id\", \"article_id\"]].drop_duplicates()\n    positives[\"label\"] = 1\n    if max_positives and len(positives) > max_positives:\n        positives = positives.sample(max_positives, random_state=seed)\n    return positives.reset_index(drop=True)\n\n\ndef make_validation_positive_pairs(\n    transactions: pd.DataFrame,\n    customers: set[str],\n    cutoff: pd.Timestamp,\n    max_positives: int | None,\n    seed: int,\n) -> pd.DataFrame:\n    positives = transactions[\n        transactions[\"customer_id\"].isin(customers) & (transactions[\"t_dat\"] > cutoff)\n    ][[\"customer_id\", \"article_id\"]].drop_duplicates()\n    positives[\"label\"] = 1\n    if max_positives and len(positives) > max_positives:\n        positives = positives.sample(max_positives, random_state=seed)\n    return positives.reset_index(drop=True)\n\n\ndef add_negative_pairs(\n    positives: pd.DataFrame,\n    article_pool: np.ndarray,\n    negatives_per_positive: int,\n    seed: int,\n) -> pd.DataFrame:\n    rng = np.random.default_rng(seed)\n    neg_customers = np.repeat(positives[\"customer_id\"].to_numpy(), negatives_per_positive)\n    neg_articles = rng.choice(article_pool, size=len(neg_customers), replace=True)\n    negatives = pd.DataFrame({\"customer_id\": neg_customers, \"article_id\": neg_articles, \"label\": 0})\n    data = pd.concat([positives, negatives], ignore_index=True)\n    data = data.drop_duplicates([\"customer_id\", \"article_id\", \"label\"])\n    return data.sample(frac=1.0, random_state=seed).reset_index(drop=True)\n\n\ndef build_history_frame(transactions: pd.DataFrame, customers: set[str], cutoff: pd.Timestamp) -> pd.DataFrame:\n    return transactions[\n        transactions[\"customer_id\"].isin(customers) & (transactions[\"t_dat\"] <= cutoff)\n    ][[\"customer_id\", \"article_id\"]]\n\n\ndef build_customer_profiles(\n    history: pd.DataFrame,\n    embeddings: np.ndarray,\n    article_to_index: dict[str, int],\n) -> tuple[dict[str, int], np.ndarray, np.ndarray, dict[tuple[str, str], int]]:\n    history = history[history[\"article_id\"].isin(article_to_index)].copy()\n    customer_ids = sorted(history[\"customer_id\"].unique())\n    customer_to_index = {customer_id: index for index, customer_id in enumerate(customer_ids)}\n    sums = np.zeros((len(customer_ids), embeddings.shape[1]), dtype=\"float32\")\n    counts = np.zeros(len(customer_ids), dtype=\"float32\")\n    pair_counts: dict[tuple[str, str], int] = {}\n    for row in history.itertuples(index=False):\n        customer_idx = customer_to_index[row.customer_id]\n        article_idx = article_to_index[row.article_id]\n        sums[customer_idx] += embeddings[article_idx]\n        counts[customer_idx] += 1.0\n        key = (row.customer_id, row.article_id)\n        pair_counts[key] = pair_counts.get(key, 0) + 1\n    return customer_to_index, sums, counts, pair_counts\n\n\ndef l2_normalize_rows(values: np.ndarray, eps: float = 1e-8) -> np.ndarray:\n    norms = np.linalg.norm(values, axis=1, keepdims=True)\n    return values / np.maximum(norms, eps)\n\n\ndef visual_arrays_for_pairs(\n    pairs: pd.DataFrame,\n    embeddings: np.ndarray,\n    article_to_index: dict[str, int],\n    customer_to_index: dict[str, int],\n    profile_sums: np.ndarray,\n    profile_counts: np.ndarray,\n    pair_counts: dict[tuple[str, str], int],\n) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:\n    image_dim = embeddings.shape[1]\n    article_emb = np.zeros((len(pairs), image_dim), dtype=\"float32\")\n    profile_emb = np.zeros((len(pairs), image_dim), dtype=\"float32\")\n    visual_similarity = np.zeros(len(pairs), dtype=\"float32\")\n    visual_history_count = np.zeros(len(pairs), dtype=\"float32\")\n\n    for index, row in enumerate(pairs[[\"customer_id\", \"article_id\"]].itertuples(index=False)):\n        article_idx = article_to_index.get(row.article_id)\n        customer_idx = customer_to_index.get(row.customer_id)\n        if article_idx is None or customer_idx is None:\n            continue\n        candidate = np.asarray(embeddings[article_idx], dtype=\"float32\")\n        count_to_remove = pair_counts.get((row.customer_id, row.article_id), 0)\n        usable_count = max(float(profile_counts[customer_idx] - count_to_remove), 0.0)\n        if usable_count > 0:\n            profile = (profile_sums[customer_idx] - count_to_remove * candidate) / usable_count\n        else:\n            profile = np.zeros(image_dim, dtype=\"float32\")\n        article_emb[index] = candidate\n        profile_emb[index] = profile\n        visual_history_count[index] = usable_count\n        denom = np.linalg.norm(candidate) * np.linalg.norm(profile)\n        visual_similarity[index] = float(np.dot(candidate, profile) / denom) if denom > 0 else 0.0\n    return article_emb, profile_emb, visual_similarity, visual_history_count\n\n\ndef merge_metadata(pairs: pd.DataFrame, customers: pd.DataFrame, articles: pd.DataFrame) -> pd.DataFrame:\n    frame = pairs.merge(customers, on=\"customer_id\", how=\"left\").merge(articles, on=\"article_id\", how=\"left\")\n    return frame\n\n\ndef fit_tabular_metadata(frame: pd.DataFrame, numeric_features: list[str]) -> dict:\n    metadata = {\n        \"numeric_features\": numeric_features,\n        \"categorical_features\": CATEGORICAL_FEATURES,\n        \"numeric_mean\": {},\n        \"numeric_std\": {},\n        \"category_maps\": {},\n    }\n    for column in numeric_features:\n        values = pd.to_numeric(frame[column], errors=\"coerce\").astype(\"float32\")\n        metadata[\"numeric_mean\"][column] = float(values.mean()) if len(values) else 0.0\n        std = float(values.std()) if len(values) else 1.0\n        metadata[\"numeric_std\"][column] = std if std > 1e-8 else 1.0\n    for column in CATEGORICAL_FEATURES:\n        values = frame[column].fillna(\"UNKNOWN\").astype(str)\n        categories = [\"__UNK__\"] + sorted(values.unique().tolist())\n        metadata[\"category_maps\"][column] = {value: index for index, value in enumerate(categories)}\n    return metadata\n\n\ndef encode_tabular(frame: pd.DataFrame, metadata: dict) -> tuple[np.ndarray, np.ndarray]:\n    numeric_columns = []\n    for column in metadata[\"numeric_features\"]:\n        values = pd.to_numeric(frame[column], errors=\"coerce\").fillna(metadata[\"numeric_mean\"][column]).astype(\"float32\")\n        values = (values - metadata[\"numeric_mean\"][column]) / metadata[\"numeric_std\"][column]\n        numeric_columns.append(values.to_numpy(dtype=\"float32\"))\n    numeric = np.stack(numeric_columns, axis=1).astype(\"float32\")\n\n    categorical_columns = []\n    for column in metadata[\"categorical_features\"]:\n        mapping = metadata[\"category_maps\"][column]\n        values = frame[column].fillna(\"UNKNOWN\").astype(str).map(mapping).fillna(0).astype(\"int64\")\n        categorical_columns.append(values.to_numpy(dtype=\"int64\"))\n    categorical = np.stack(categorical_columns, axis=1).astype(\"int64\")\n    return numeric, categorical\n\n\ndef category_sizes(metadata: dict) -> list[int]:\n    return [len(metadata[\"category_maps\"][column]) for column in metadata[\"categorical_features\"]]\n\n\ndef metadata_to_jsonable(metadata: dict) -> dict:\n    return json.loads(json.dumps(metadata))\n\n\ndef numeric_features_for_model(model_name: str) -> list[str]:\n    if model_name == \"late_fusion\":\n        return FUSION_NUMERIC_FEATURES\n    return TABULAR_NUMERIC_FEATURES\n\n\n\nimport torch\nfrom torch import nn\nfrom torchvision import models\n\n\ndef embedding_dim(size: int) -> int:\n    return min(50, max(4, int(size**0.25 * 8)))\n\n\nclass TabularOnlyMLP(nn.Module):\n    def __init__(self, numeric_dim: int, category_sizes: list[int]) -> None:\n        super().__init__()\n        self.embeddings = nn.ModuleList(\n            [nn.Embedding(size, embedding_dim(size)) for size in category_sizes]\n        )\n        cat_dim = sum(embedding.embedding_dim for embedding in self.embeddings)\n        self.net = nn.Sequential(\n            nn.Linear(numeric_dim + cat_dim, 256),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.Linear(256, 128),\n            nn.ReLU(),\n            nn.Dropout(0.1),\n            nn.Linear(128, 1),\n        )\n\n    def forward(self, numeric: torch.Tensor, categorical: torch.Tensor) -> torch.Tensor:\n        embedded = [emb(categorical[:, idx]) for idx, emb in enumerate(self.embeddings)]\n        x = torch.cat([numeric, *embedded], dim=1)\n        return self.net(x).squeeze(1)\n\n\nclass ImageHistoryMLP(nn.Module):\n    def __init__(self, image_dim: int) -> None:\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(image_dim * 2 + 2, 512),\n            nn.ReLU(),\n            nn.Dropout(0.25),\n            nn.Linear(512, 128),\n            nn.ReLU(),\n            nn.Dropout(0.1),\n            nn.Linear(128, 1),\n        )\n\n    def forward(\n        self,\n        article_embedding: torch.Tensor,\n        profile_embedding: torch.Tensor,\n        visual_similarity: torch.Tensor,\n        visual_history_count: torch.Tensor,\n    ) -> torch.Tensor:\n        visual_extra = torch.stack([visual_similarity, visual_history_count], dim=1)\n        x = torch.cat([article_embedding, profile_embedding, visual_extra], dim=1)\n        return self.net(x).squeeze(1)\n\n\nclass MultimodalLateFusion(nn.Module):\n    def __init__(self, numeric_dim: int, category_sizes: list[int], image_dim: int) -> None:\n        super().__init__()\n        self.embeddings = nn.ModuleList(\n            [nn.Embedding(size, embedding_dim(size)) for size in category_sizes]\n        )\n        cat_dim = sum(embedding.embedding_dim for embedding in self.embeddings)\n        self.tabular_branch = nn.Sequential(\n            nn.Linear(numeric_dim + cat_dim, 256),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.Linear(256, 128),\n            nn.ReLU(),\n        )\n        self.visual_branch = nn.Sequential(\n            nn.Linear(image_dim * 2 + 2, 512),\n            nn.ReLU(),\n            nn.Dropout(0.25),\n            nn.Linear(512, 128),\n            nn.ReLU(),\n        )\n        self.fusion_head = nn.Sequential(\n            nn.Linear(256, 128),\n            nn.ReLU(),\n            nn.Dropout(0.1),\n            nn.Linear(128, 1),\n        )\n\n    def forward(\n        self,\n        numeric: torch.Tensor,\n        categorical: torch.Tensor,\n        article_embedding: torch.Tensor,\n        profile_embedding: torch.Tensor,\n        visual_similarity: torch.Tensor,\n        visual_history_count: torch.Tensor,\n    ) -> torch.Tensor:\n        embedded = [emb(categorical[:, idx]) for idx, emb in enumerate(self.embeddings)]\n        tabular = self.tabular_branch(torch.cat([numeric, *embedded], dim=1))\n        visual_extra = torch.stack([visual_similarity, visual_history_count], dim=1)\n        visual = self.visual_branch(torch.cat([article_embedding, profile_embedding, visual_extra], dim=1))\n        return self.fusion_head(torch.cat([tabular, visual], dim=1)).squeeze(1)\n\n\nclass EfficientNetBinaryClassifier(nn.Module):\n    def __init__(self, train_backbone: bool = False) -> None:\n        super().__init__()\n        try:\n            weights = models.EfficientNet_B0_Weights.DEFAULT\n            self.weights = weights\n            self.backbone = models.efficientnet_b0(weights=weights)\n        except Exception as exc:\n            print('Pretrained EfficientNet weights unavailable, using random weights:', exc)\n            weights = None\n            self.weights = weights\n            self.backbone = models.efficientnet_b0(weights=None)\n        in_features = self.backbone.classifier[1].in_features\n        self.backbone.classifier = nn.Sequential(nn.Dropout(0.2), nn.Linear(in_features, 1))\n        if not train_backbone:\n            for parameter in self.backbone.features.parameters():\n                parameter.requires_grad = False\n\n    def forward(self, images: torch.Tensor) -> torch.Tensor:\n        return self.backbone(images).squeeze(1)\n\n\ndef build_model(model_name: str, metadata: dict, image_dim: int) -> nn.Module:\n    category_sizes = [len(metadata[\"category_maps\"][column]) for column in metadata[\"categorical_features\"]]\n    numeric_dim = len(metadata[\"numeric_features\"])\n    if model_name == \"tabular_only\":\n        return TabularOnlyMLP(numeric_dim, category_sizes)\n    if model_name == \"image_history\":\n        return ImageHistoryMLP(image_dim)\n    if model_name == \"late_fusion\":\n        return MultimodalLateFusion(numeric_dim, category_sizes, image_dim)\n    raise ValueError(f\"Unknown model: {model_name}\")\n\n\n\nimport argparse\nimport json\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom torch import nn\nfrom torch.utils.data import DataLoader, Dataset\nfrom tqdm.auto import tqdm\n\n\n\nclass PairDataset(Dataset):\n    def __init__(\n        self,\n        numeric: np.ndarray | None,\n        categorical: np.ndarray | None,\n        article_emb: np.ndarray | None,\n        profile_emb: np.ndarray | None,\n        visual_similarity: np.ndarray | None,\n        visual_history_count: np.ndarray | None,\n        labels: np.ndarray,\n    ) -> None:\n        self.numeric = torch.tensor(numeric, dtype=torch.float32) if numeric is not None else None\n        self.categorical = torch.tensor(categorical, dtype=torch.long) if categorical is not None else None\n        self.article_emb = torch.tensor(article_emb, dtype=torch.float32) if article_emb is not None else None\n        self.profile_emb = torch.tensor(profile_emb, dtype=torch.float32) if profile_emb is not None else None\n        self.visual_similarity = (\n            torch.tensor(visual_similarity, dtype=torch.float32) if visual_similarity is not None else None\n        )\n        self.visual_history_count = (\n            torch.tensor(visual_history_count, dtype=torch.float32) if visual_history_count is not None else None\n        )\n        self.labels = torch.tensor(labels, dtype=torch.float32)\n\n    def __len__(self) -> int:\n        return len(self.labels)\n\n    def __getitem__(self, index: int) -> dict[str, torch.Tensor]:\n        item = {\"label\": self.labels[index]}\n        if self.numeric is not None:\n            item[\"numeric\"] = self.numeric[index]\n            item[\"categorical\"] = self.categorical[index]\n        if self.article_emb is not None:\n            item[\"article_emb\"] = self.article_emb[index]\n            item[\"profile_emb\"] = self.profile_emb[index]\n            item[\"visual_similarity\"] = self.visual_similarity[index]\n            item[\"visual_history_count\"] = self.visual_history_count[index]\n        return item\n\n\ndef set_seed(seed: int) -> None:\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n\n\ndef forward_model(model_name: str, model: nn.Module, batch: dict[str, torch.Tensor], device: torch.device) -> torch.Tensor:\n    if model_name == \"tabular_only\":\n        return model(batch[\"numeric\"].to(device), batch[\"categorical\"].to(device))\n    if model_name == \"image_history\":\n        return model(\n            batch[\"article_emb\"].to(device),\n            batch[\"profile_emb\"].to(device),\n            batch[\"visual_similarity\"].to(device),\n            batch[\"visual_history_count\"].to(device),\n        )\n    if model_name == \"late_fusion\":\n        return model(\n            batch[\"numeric\"].to(device),\n            batch[\"categorical\"].to(device),\n            batch[\"article_emb\"].to(device),\n            batch[\"profile_emb\"].to(device),\n            batch[\"visual_similarity\"].to(device),\n            batch[\"visual_history_count\"].to(device),\n        )\n    raise ValueError(model_name)\n\n\ndef evaluate(model_name: str, model: nn.Module, loader: DataLoader, device: torch.device) -> dict:\n    model.eval()\n    y_true: list[float] = []\n    y_prob: list[float] = []\n    with torch.no_grad():\n        for batch in loader:\n            logits = forward_model(model_name, model, batch, device)\n            probabilities = torch.sigmoid(logits).detach().cpu().numpy()\n            y_prob.extend(probabilities.tolist())\n            y_true.extend(batch[\"label\"].numpy().tolist())\n    labels = np.array(y_true)\n    probabilities = np.array(y_prob)\n    predictions = (probabilities >= 0.5).astype(int)\n    return {\n        \"auc_roc\": float(roc_auc_score(labels, probabilities)) if len(np.unique(labels)) > 1 else float(\"nan\"),\n        \"accuracy\": float(accuracy_score(labels, predictions)),\n    }\n\n\ndef train_model(\n    model_name: str,\n    model: nn.Module,\n    train_loader: DataLoader,\n    val_loader: DataLoader,\n    device: torch.device,\n    epochs: int,\n    learning_rate: float,\n) -> dict:\n    model.to(device)\n    optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)\n    criterion = nn.BCEWithLogitsLoss()\n    history = []\n    for epoch in range(1, epochs + 1):\n        model.train()\n        losses = []\n        for batch in tqdm(train_loader, desc=f\"{model_name} epoch {epoch}/{epochs}\"):\n            optimizer.zero_grad(set_to_none=True)\n            logits = forward_model(model_name, model, batch, device)\n            loss = criterion(logits, batch[\"label\"].to(device))\n            loss.backward()\n            optimizer.step()\n            losses.append(float(loss.detach().cpu()))\n        metrics = evaluate(model_name, model, val_loader, device)\n        metrics[\"epoch\"] = epoch\n        metrics[\"loss\"] = float(np.mean(losses)) if losses else float(\"nan\")\n        history.append(metrics)\n        log(f\"{model_name} epoch {epoch}: loss={metrics['loss']:.4f} auc={metrics['auc_roc']:.4f} acc={metrics['accuracy']:.4f}\")\n    return {\"history\": history, \"final\": history[-1] if history else {}}\n\n\ndef prepare_fold_data(args: argparse.Namespace) -> tuple[dict, dict]:\n    raw_dir, transactions, customers, articles = load_core_tables(args.raw_dir)\n    embeddings, article_ids, article_to_index = load_embeddings(args.embeddings_path, args.embedding_ids_path, mmap_mode=None)\n\n    folds = pd.read_csv(args.folds_csv)\n    cutoff = make_cutoff(transactions, args.validation_days)\n    val_customers = set(folds.loc[folds[\"fold_id\"] == args.fold_id, \"customer_id\"])\n    train_customers = set(folds.loc[folds[\"fold_id\"] != args.fold_id, \"customer_id\"])\n    if train_customers & val_customers:\n        raise RuntimeError(\"Customer leakage detected between train and validation.\")\n\n    article_pool = np.array([article_id for article_id in article_ids if article_id in set(articles[\"article_id\"])])\n    train_positive = sample_positive_pairs(transactions, train_customers, cutoff, args.max_train_positives, args.seed)\n    val_positive = make_validation_positive_pairs(transactions, val_customers, cutoff, args.max_val_positives, args.seed)\n    train_pairs = add_negative_pairs(train_positive, article_pool, args.negatives_per_positive, args.seed)\n    val_pairs = add_negative_pairs(val_positive, article_pool, args.negatives_per_positive, args.seed + 1000)\n\n    train_pair_customers = set(train_pairs[\"customer_id\"])\n    val_pair_customers = set(val_pairs[\"customer_id\"])\n    train_history = build_history_frame(transactions, train_pair_customers, cutoff)\n    val_history = build_history_frame(transactions, val_pair_customers, cutoff)\n    train_customer_to_index, train_sums, train_counts, train_pair_counts = build_customer_profiles(\n        train_history, embeddings, article_to_index\n    )\n    val_customer_to_index, val_sums, val_counts, val_pair_counts = build_customer_profiles(\n        val_history, embeddings, article_to_index\n    )\n\n    train_pairs = train_pairs[\n        train_pairs[\"article_id\"].isin(article_to_index) & train_pairs[\"customer_id\"].isin(train_customer_to_index)\n    ].reset_index(drop=True)\n    val_pairs = val_pairs[\n        val_pairs[\"article_id\"].isin(article_to_index) & val_pairs[\"customer_id\"].isin(val_customer_to_index)\n    ].reset_index(drop=True)\n\n    train_article_emb, train_profile_emb, train_sim, train_hist = visual_arrays_for_pairs(\n        train_pairs, embeddings, article_to_index, train_customer_to_index, train_sums, train_counts, train_pair_counts\n    )\n    val_article_emb, val_profile_emb, val_sim, val_hist = visual_arrays_for_pairs(\n        val_pairs, embeddings, article_to_index, val_customer_to_index, val_sums, val_counts, val_pair_counts\n    )\n\n    train_frame = merge_metadata(train_pairs, customers, articles)\n    val_frame = merge_metadata(val_pairs, customers, articles)\n    for frame, sim, hist in [(train_frame, train_sim, train_hist), (val_frame, val_sim, val_hist)]:\n        frame[\"visual_similarity\"] = sim\n        frame[\"visual_history_count\"] = hist\n\n    context = {\n        \"raw_dir\": str(raw_dir),\n        \"fold_id\": args.fold_id,\n        \"cutoff\": str(cutoff.date()),\n        \"train_customers\": len(train_customers),\n        \"validation_customers\": len(val_customers),\n        \"train_rows\": len(train_pairs),\n        \"validation_rows\": len(val_pairs),\n        \"image_dim\": int(embeddings.shape[1]),\n        \"article_ids\": article_ids,\n    }\n    arrays = {\n        \"train_pairs\": train_pairs,\n        \"val_pairs\": val_pairs,\n        \"train_frame\": train_frame,\n        \"val_frame\": val_frame,\n        \"train_article_emb\": train_article_emb,\n        \"train_profile_emb\": train_profile_emb,\n        \"train_sim\": train_sim,\n        \"train_hist\": train_hist,\n        \"val_article_emb\": val_article_emb,\n        \"val_profile_emb\": val_profile_emb,\n        \"val_sim\": val_sim,\n        \"val_hist\": val_hist,\n    }\n    return context, arrays\n\n\ndef train_one_requested_model(model_name: str, args: argparse.Namespace, context: dict, arrays: dict) -> dict:\n    numeric_features = numeric_features_for_model(model_name)\n    if model_name == \"image_history\":\n        metadata = {\"numeric_features\": [], \"categorical_features\": [], \"category_maps\": {}}\n        train_numeric = train_categorical = val_numeric = val_categorical = None\n    else:\n        metadata = fit_tabular_metadata(arrays[\"train_frame\"], numeric_features)\n        train_numeric, train_categorical = encode_tabular(arrays[\"train_frame\"], metadata)\n        val_numeric, val_categorical = encode_tabular(arrays[\"val_frame\"], metadata)\n\n    train_dataset = PairDataset(\n        train_numeric,\n        train_categorical,\n        None if model_name == \"tabular_only\" else arrays[\"train_article_emb\"],\n        None if model_name == \"tabular_only\" else arrays[\"train_profile_emb\"],\n        None if model_name == \"tabular_only\" else arrays[\"train_sim\"],\n        None if model_name == \"tabular_only\" else arrays[\"train_hist\"],\n        arrays[\"train_pairs\"][\"label\"].to_numpy(dtype=\"float32\"),\n    )\n    val_dataset = PairDataset(\n        val_numeric,\n        val_categorical,\n        None if model_name == \"tabular_only\" else arrays[\"val_article_emb\"],\n        None if model_name == \"tabular_only\" else arrays[\"val_profile_emb\"],\n        None if model_name == \"tabular_only\" else arrays[\"val_sim\"],\n        None if model_name == \"tabular_only\" else arrays[\"val_hist\"],\n        arrays[\"val_pairs\"][\"label\"].to_numpy(dtype=\"float32\"),\n    )\n    train_loader = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=0)\n    val_loader = DataLoader(val_dataset, batch_size=args.batch_size, shuffle=False, num_workers=0)\n\n    model = build_model(model_name, metadata, context[\"image_dim\"])\n    device = torch.device(args.device or (\"cuda\" if torch.cuda.is_available() else \"cpu\"))\n    result = train_model(model_name, model, train_loader, val_loader, device, args.epochs, args.learning_rate)\n\n    checkpoint = {\n        \"model_name\": model_name,\n        \"fold_id\": args.fold_id,\n        \"state_dict\": model.state_dict(),\n        \"metadata\": metadata_to_jsonable(metadata),\n        \"image_dim\": context[\"image_dim\"],\n        \"context\": context,\n        \"metrics\": result,\n    }\n    output_path = args.model_dir / f\"{model_name}_fold{args.fold_id}.pt\"\n    args.model_dir.mkdir(parents=True, exist_ok=True)\n    torch.save(checkpoint, output_path)\n    log(f\"Saved checkpoint: {output_path}\")\n    row = {\n        \"fold_id\": args.fold_id,\n        \"model\": model_name,\n        \"auc_roc\": result[\"final\"].get(\"auc_roc\"),\n        \"accuracy\": result[\"final\"].get(\"accuracy\"),\n        \"train_rows\": context[\"train_rows\"],\n        \"validation_rows\": context[\"validation_rows\"],\n        \"checkpoint\": str(output_path),\n    }\n    return row\n\n\ndef parse_models(raw: str) -> list[str]:\n    values = [value.strip() for value in raw.split(\",\") if value.strip()]\n    unknown = set(values) - set(MODEL_NAMES)\n    if unknown:\n        raise ValueError(f\"Unknown models: {sorted(unknown)}\")\n    return values\n\n\ndef main() -> None:\n    defaults = V2Defaults()\n    parser = argparse.ArgumentParser(description=\"Train proposal v2 tabular, image-history, and late-fusion models.\")\n    parser.add_argument(\"--raw-dir\", type=Path, default=None)\n    parser.add_argument(\"--embeddings-path\", type=Path, default=None)\n    parser.add_argument(\"--embedding-ids-path\", type=Path, default=None)\n    parser.add_argument(\"--folds-csv\", type=Path, default=REPORTS_DIR / \"proposal_v2_fold_splits.csv\")\n    parser.add_argument(\"--fold-id\", type=int, default=0)\n    parser.add_argument(\"--models\", default=\"tabular_only,image_history,late_fusion\")\n    parser.add_argument(\"--validation-days\", type=int, default=defaults.validation_days)\n    parser.add_argument(\"--negatives-per-positive\", type=int, default=defaults.negatives_per_positive)\n    parser.add_argument(\"--max-train-positives\", type=int, default=200_000)\n    parser.add_argument(\"--max-val-positives\", type=int, default=50_000)\n    parser.add_argument(\"--epochs\", type=int, default=defaults.epochs)\n    parser.add_argument(\"--batch-size\", type=int, default=defaults.train_batch_size)\n    parser.add_argument(\"--learning-rate\", type=float, default=defaults.learning_rate)\n    parser.add_argument(\"--seed\", type=int, default=DEFAULT_SEED)\n    parser.add_argument(\"--device\", default=None)\n    parser.add_argument(\"--model-dir\", type=Path, default=MODELS_DIR)\n    parser.add_argument(\"--metrics-csv\", type=Path, default=REPORTS_DIR / \"proposal_v2_classification_metrics.csv\")\n    parser.add_argument(\"--context-json\", type=Path, default=None)\n    args = parser.parse_args()\n\n    ensure_v2_dirs()\n    set_seed(args.seed)\n    context, arrays = prepare_fold_data(args)\n    rows = []\n    for model_name in parse_models(args.models):\n        rows.append(train_one_requested_model(model_name, args, context, arrays))\n\n    metrics = pd.DataFrame(rows)\n    args.metrics_csv.parent.mkdir(parents=True, exist_ok=True)\n    if args.metrics_csv.exists():\n        previous = pd.read_csv(args.metrics_csv)\n        metrics = pd.concat([previous, metrics], ignore_index=True)\n        metrics = metrics.drop_duplicates([\"fold_id\", \"model\"], keep=\"last\")\n    metrics.to_csv(args.metrics_csv, index=False)\n    context_path = args.context_json or (REPORTS_DIR / \"folds\" / f\"fold{args.fold_id}_training_context.json\")\n    context_path.write_text(json.dumps(context, indent=2), encoding=\"utf-8\")\n    log(f\"Saved metrics: {args.metrics_csv}\")\n    log(f\"Saved context: {context_path}\")\n\n\n\nimport argparse\nfrom pathlib import Path\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom PIL import Image\nfrom sklearn.metrics import roc_auc_score\n\n\n\ndef feature_group(feature: str) -> str:\n    if feature in {\"FN\", \"Active\", \"age\", \"club_member_status\", \"fashion_news_frequency\"}:\n        return \"customer_metadata\"\n    if feature in {\"visual_similarity\", \"visual_history_count\"}:\n        return \"visual_branch\"\n    return \"article_metadata\"\n\n\ndef run_tabular_shap(args: argparse.Namespace) -> None:\n    checkpoint_path = args.tabular_checkpoint or MODELS_DIR / f\"tabular_only_fold{args.fold_id}.pt\"\n    if not checkpoint_path.exists():\n        pd.DataFrame(\n            [\n                {\n                    \"feature\": \"SHAP_NOT_RUN\",\n                    \"feature_group\": \"missing_checkpoint\",\n                    \"mean_abs_shap\": np.nan,\n                    \"status\": f\"checkpoint_not_found:{checkpoint_path}\",\n                }\n            ]\n        ).to_csv(args.shap_output, index=False)\n        log(f\"SHAP skipped; checkpoint not found: {checkpoint_path}\")\n        return\n\n    shap_module = None\n    try:\n        import shap as shap_module  # type: ignore\n    except Exception as exc:\n        log(f\"SHAP dependency not available; falling back to permutation importance: {exc}\")\n\n    checkpoint = torch.load(checkpoint_path, map_location=\"cpu\", weights_only=False)\n    metadata = checkpoint[\"metadata\"]\n    _, transactions, customers, articles = load_core_tables(args.raw_dir)\n    folds = pd.read_csv(args.folds_csv)\n    cutoff = make_cutoff(transactions, args.validation_days)\n    val_customers = set(folds.loc[folds[\"fold_id\"] == args.fold_id, \"customer_id\"])\n    positives = make_validation_positive_pairs(\n        transactions,\n        val_customers,\n        cutoff,\n        args.shap_explain_rows,\n        args.seed,\n    )\n    article_pool = articles[\"article_id\"].astype(str).to_numpy()\n    pairs = add_negative_pairs(positives, article_pool, 1, args.seed + 1000)\n    frame = merge_metadata(pairs, customers, articles).head(args.shap_explain_rows).copy()\n    numeric, categorical = encode_tabular(frame, metadata)\n    encoded = np.concatenate([numeric, categorical.astype(\"float32\")], axis=1)\n    feature_names = metadata[\"numeric_features\"] + metadata[\"categorical_features\"]\n    background = encoded[: min(args.shap_background_rows, len(encoded))]\n    explain = encoded[: min(args.shap_explain_rows, len(encoded))]\n\n    model = build_model(\"tabular_only\", metadata, checkpoint[\"image_dim\"])\n    model.load_state_dict(checkpoint[\"state_dict\"])\n    model.eval()\n\n    numeric_dim = len(metadata[\"numeric_features\"])\n\n    def predict_fn(values: np.ndarray) -> np.ndarray:\n        numeric_values = torch.tensor(values[:, :numeric_dim], dtype=torch.float32)\n        categorical_values = torch.tensor(np.rint(values[:, numeric_dim:]).clip(min=0), dtype=torch.long)\n        with torch.no_grad():\n            return torch.sigmoid(model(numeric_values, categorical_values)).numpy()\n\n    if shap_module is not None:\n        explainer = shap_module.KernelExplainer(predict_fn, background)\n        shap_values = explainer.shap_values(explain, nsamples=args.shap_nsamples)\n        shap_array = np.asarray(shap_values)\n        if shap_array.ndim == 3:\n            shap_array = shap_array[0]\n        mean_abs = np.abs(shap_array).mean(axis=0)\n        rows = [\n            {\n                \"feature\": feature,\n                \"feature_group\": feature_group(feature),\n                \"mean_abs_shap\": float(value),\n                \"status\": \"shap\",\n            }\n            for feature, value in zip(feature_names, mean_abs)\n        ]\n    else:\n        labels = pairs.head(len(explain))[\"label\"].to_numpy()\n        baseline = roc_auc_score(labels, predict_fn(explain)) if len(np.unique(labels)) > 1 else 0.5\n        rng = np.random.default_rng(args.seed)\n        rows = []\n        for index, feature in enumerate(feature_names):\n            perturbed = explain.copy()\n            perturbed[:, index] = rng.permutation(perturbed[:, index])\n            score = roc_auc_score(labels, predict_fn(perturbed)) if len(np.unique(labels)) > 1 else 0.5\n            rows.append(\n                {\n                    \"feature\": feature,\n                    \"feature_group\": feature_group(feature),\n                    \"mean_abs_shap\": float(max(baseline - score, 0.0)),\n                    \"status\": \"permutation_fallback\",\n                }\n            )\n    output = pd.DataFrame(rows).sort_values(\"mean_abs_shap\", ascending=False)\n    output.to_csv(args.shap_output, index=False)\n    log(f\"Saved SHAP summary: {args.shap_output}\")\n\n\ndef simple_gradcam_heatmap(model: EfficientNetBinaryClassifier, image_tensor: torch.Tensor, device: torch.device) -> np.ndarray:\n    activations = []\n\n    def forward_hook(_, __, output):\n        activations.append(output)\n\n    target_layer = model.backbone.features[-1]\n    handle_f = target_layer.register_forward_hook(forward_hook)\n    model.zero_grad(set_to_none=True)\n    batch = image_tensor.to(device).unsqueeze(0)\n    batch.requires_grad_(True)\n    score = model(batch)\n    if not activations:\n        handle_f.remove()\n        raise RuntimeError(\"Grad-CAM activations were not captured for this image.\")\n    acts = activations[0]\n    grads = torch.autograd.grad(score.sum(), acts, retain_graph=False, allow_unused=True)[0]\n    handle_f.remove()\n\n    if grads is None:\n        raise RuntimeError(\"Grad-CAM gradients were not captured for this image.\")\n    weights = grads.mean(dim=(2, 3), keepdim=True)\n    cam = torch.relu((weights * acts).sum(dim=1)).squeeze().detach().cpu().numpy()\n    cam = cam - cam.min()\n    cam = cam / max(cam.max(), 1e-8)\n    return cam\n\n\ndef run_gradcam_examples(\n    checkpoint_path: Path,\n    raw_dir: Path | None,\n    images_dir: Path | None,\n    output_dir: Path,\n    max_examples: int,\n    device_name: str | None,\n) -> None:\n    raw_dir, transactions, _, _ = load_core_tables(raw_dir)\n    resolved_images = resolve_images_dir(raw_dir, images_dir)\n    checkpoint = torch.load(checkpoint_path, map_location=\"cpu\", weights_only=False)\n    model = EfficientNetBinaryClassifier(train_backbone=checkpoint.get(\"train_backbone\", False))\n    model.load_state_dict(checkpoint[\"state_dict\"])\n    device = torch.device(device_name or (\"cuda\" if torch.cuda.is_available() else \"cpu\"))\n    model.to(device).eval()\n    transform = model.weights.transforms()\n    output_dir.mkdir(parents=True, exist_ok=True)\n\n    article_ids = transactions[\"article_id\"].astype(str).str.zfill(10).drop_duplicates().head(max_examples * 10)\n    saved = 0\n    for article_id in article_ids:\n        path = article_image_path(resolved_images, article_id)\n        if not path.exists():\n            continue\n        original = Image.open(path).convert(\"RGB\")\n        tensor = transform(original)\n        try:\n            heatmap = simple_gradcam_heatmap(model, tensor, device)\n        except RuntimeError as exc:\n            log(f\"Grad-CAM skipped for {article_id}: {exc}\")\n            continue\n        fig, axes = plt.subplots(1, 2, figsize=(8, 4))\n        axes[0].imshow(original)\n        axes[0].set_title(article_id)\n        axes[0].axis(\"off\")\n        axes[1].imshow(original)\n        axes[1].imshow(heatmap, cmap=\"jet\", alpha=0.45, extent=(0, original.width, original.height, 0))\n        axes[1].set_title(\"Grad-CAM\")\n        axes[1].axis(\"off\")\n        fig.tight_layout()\n        fig.savefig(output_dir / f\"{article_id}_gradcam.png\", dpi=150)\n        plt.close(fig)\n        saved += 1\n        if saved >= max_examples:\n            break\n    log(f\"Saved {saved} Grad-CAM examples to {output_dir}\")\n\n\ndef main() -> None:\n    parser = argparse.ArgumentParser(description=\"Run proposal v2 SHAP/permutation explainability and Grad-CAM examples.\")\n    parser.add_argument(\"--raw-dir\", type=Path, default=None)\n    parser.add_argument(\"--embeddings-path\", type=Path, default=None)\n    parser.add_argument(\"--embedding-ids-path\", type=Path, default=None)\n    parser.add_argument(\"--folds-csv\", type=Path, default=REPORTS_DIR / \"proposal_v2_fold_splits.csv\")\n    parser.add_argument(\"--fold-id\", type=int, default=0)\n    parser.add_argument(\"--validation-days\", type=int, default=7)\n    parser.add_argument(\"--images-dir\", type=Path, default=None)\n    parser.add_argument(\"--tabular-checkpoint\", type=Path, default=None)\n    parser.add_argument(\"--cnn-checkpoint\", type=Path, default=None)\n    parser.add_argument(\"--shap-output\", type=Path, default=REPORTS_DIR / \"proposal_v2_shap_summary.csv\")\n    parser.add_argument(\"--shap-background-rows\", type=int, default=80)\n    parser.add_argument(\"--shap-explain-rows\", type=int, default=40)\n    parser.add_argument(\"--shap-nsamples\", type=int, default=100)\n    parser.add_argument(\"--seed\", type=int, default=42)\n    parser.add_argument(\"--gradcam-dir\", type=Path, default=REPORTS_DIR / \"gradcam_examples\")\n    parser.add_argument(\"--max-gradcam-examples\", type=int, default=12)\n    parser.add_argument(\"--device\", default=None)\n    args = parser.parse_args()\n\n    ensure_v2_dirs()\n    run_tabular_shap(args)\n    checkpoint = args.cnn_checkpoint or MODELS_DIR / \"image_only_effnet_cnn_fold0.pt\"\n    if checkpoint.exists():\n        run_gradcam_examples(checkpoint, args.raw_dir, args.images_dir, args.gradcam_dir, args.max_gradcam_examples, args.device)\n    else:\n        log(f\"Grad-CAM skipped; CNN checkpoint not found: {checkpoint}\")\n"}, {"cell_type": "markdown", "metadata": {}, "source": "## Parametreler\n\nVarsayilan row sayilari kucuk tutulur. Checkpoint pathleri dogrulandiktan sonra gerekirse artirilabilir."}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "FOLD_ID = 0\nRAW_DIR = None\nEMBEDDINGS_PATH = None\nEMBEDDING_IDS_PATH = None\nFOLDS_CSV = REPORTS_DIR / 'proposal_v2_fold_splits.csv'\nIMAGES_DIR = None\nTABULAR_CHECKPOINT = None\nCNN_CHECKPOINT = None\nVALIDATION_DAYS = 7\nSHAP_BACKGROUND_ROWS = 10\nSHAP_EXPLAIN_ROWS = 5\nSHAP_NSAMPLES = 20\nRANDOM_SEED = 42\nMAX_GRADCAM_EXAMPLES = 4\nDEVICE = None"}, {"cell_type": "markdown", "metadata": {}, "source": "## Explanation ciktilarini uret\n\nBu hucre tabular importance CSV dosyasini ve gerekli checkpoint varsa Grad-CAM image dosyalarini yazar."}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "ensure_v2_dirs()\nargs = argparse.Namespace(\n    raw_dir=RAW_DIR,\n    embeddings_path=EMBEDDINGS_PATH,\n    embedding_ids_path=EMBEDDING_IDS_PATH,\n    folds_csv=FOLDS_CSV,\n    fold_id=FOLD_ID,\n    validation_days=VALIDATION_DAYS,\n    images_dir=IMAGES_DIR,\n    tabular_checkpoint=TABULAR_CHECKPOINT,\n    cnn_checkpoint=CNN_CHECKPOINT,\n    shap_output=REPORTS_DIR / 'proposal_v2_shap_summary.csv',\n    shap_background_rows=SHAP_BACKGROUND_ROWS,\n    shap_explain_rows=SHAP_EXPLAIN_ROWS,\n    shap_nsamples=SHAP_NSAMPLES,\n    seed=RANDOM_SEED,\n    gradcam_dir=REPORTS_DIR / 'gradcam_examples',\n    max_gradcam_examples=MAX_GRADCAM_EXAMPLES,\n    device=DEVICE,\n)\nrun_tabular_shap(args)\ncheckpoint = args.cnn_checkpoint or MODELS_DIR / 'image_only_effnet_cnn_fold0.pt'\nif checkpoint.exists():\n    run_gradcam_examples(checkpoint, args.raw_dir, args.images_dir, args.gradcam_dir, args.max_gradcam_examples, args.device)\nelse:\n    log(f'Grad-CAM skipped; CNN checkpoint not found: {checkpoint}')"}], "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"name": "python", "pygments_lexer": "ipython3"}}, "nbformat": 4, "nbformat_minor": 5}