{"cells": [{"cell_type": "markdown", "metadata": {}, "source": "# Perseptron v2 - 00 Customer-Level Fold Hazirligi\n\nBu notebook proposal uyumlu customer-level 5-fold split dosyasini uretir. Ayni musteri ayni fold icinde hem train hem validation tarafinda yer almaz.\n\nCiktilar:\n\n- `reports/proposal_v2/proposal_v2_fold_splits.csv`\n- `reports/proposal_v2/proposal_v2_fold_protocol_summary.json`"}, {"cell_type": "markdown", "metadata": {}, "source": "## Ortam ve output klasorleri\n\nBu hucre Kaggle icin yazilabilir output klasorlerini hazirlar ve onceki notebook outputlari `Add Data` ile eklendiyse bunlari geri yukler. Repo icindeki `.py` modulleri import edilmez."}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "from pathlib import Path\nimport json\nimport os\nimport random\nimport shutil\nimport warnings\n\nimport numpy as np\nimport pandas as pd\n\nwarnings.filterwarnings('ignore')\n\nIS_KAGGLE = Path('/kaggle').exists()\nWORK_DIR = Path('/kaggle/working') if IS_KAGGLE else Path.cwd()\nREPORTS_DIR = WORK_DIR / 'reports' / 'proposal_v2'\nMODELS_DIR = WORK_DIR / 'models' / 'proposal_v2'\nGRADCAM_DIR = REPORTS_DIR / 'gradcam_examples'\nfor path in [REPORTS_DIR, REPORTS_DIR / 'folds', MODELS_DIR, GRADCAM_DIR]:\n    path.mkdir(parents=True, exist_ok=True)\n\ndef candidate_output_roots(input_root):\n    roots = []\n    seen = set()\n\n    def add_root(path):\n        resolved = path.resolve()\n        if resolved not in seen:\n            roots.append(path)\n            seen.add(resolved)\n\n    notebooks_root = input_root / 'notebooks'\n    if notebooks_root.exists():\n        for path in notebooks_root.rglob('*'):\n            if path.is_dir() and ((path / 'reports' / 'proposal_v2').exists() or (path / 'models' / 'proposal_v2').exists()):\n                add_root(path)\n    datasets_root = input_root / 'datasets'\n    if datasets_root.exists():\n        for path in datasets_root.rglob('*'):\n            if not path.is_dir():\n                continue\n            name = path.name.lower()\n            if any(token in name for token in ['proposal', 'report', 'output', 'fold']):\n                add_root(path)\n    for path in input_root.glob('*'):\n        if path.is_dir() and path.name not in {'competitions', 'datasets', 'notebooks'}:\n            add_root(path)\n    return roots\n\ndef merge_or_copy_file(source, target):\n    target.parent.mkdir(parents=True, exist_ok=True)\n    if source.suffix == '.csv' and target.exists():\n        try:\n            current = pd.read_csv(target)\n            incoming = pd.read_csv(source)\n            merged = pd.concat([current, incoming], ignore_index=True)\n            if {'fold_id', 'model'}.issubset(merged.columns):\n                merged = merged.drop_duplicates(['fold_id', 'model'], keep='last')\n            elif 'customer_id' in merged.columns and 'model' in merged.columns:\n                keys = [column for column in ['fold_id', 'customer_id', 'model'] if column in merged.columns]\n                merged = merged.drop_duplicates(keys, keep='last')\n            else:\n                merged = merged.drop_duplicates(keep='last')\n            merged.to_csv(target, index=False)\n            print(f'Merged previous CSV: {source} -> {target}')\n            return\n        except Exception as exc:\n            print(f'CSV merge failed, falling back to copy for {source}: {exc}')\n    if not target.exists() or source.stat().st_size != target.stat().st_size:\n        shutil.copy2(source, target)\n        print(f'Restored previous output file: {source} -> {target}')\n\ndef restore_previous_outputs():\n    input_root = Path('/kaggle/input')\n    if not input_root.exists():\n        return\n    for root in candidate_output_roots(input_root):\n        for source in root.rglob('*'):\n            if not source.is_file():\n                continue\n            try:\n                relative = source.relative_to(root)\n            except ValueError:\n                relative = Path(source.name)\n            relative_text = relative.as_posix()\n            name = source.name\n            if relative_text.startswith('reports/proposal_v2/'):\n                target = WORK_DIR / relative\n            elif relative_text.startswith('models/proposal_v2/'):\n                target = WORK_DIR / relative\n            elif name.endswith('.pt'):\n                target = MODELS_DIR / name\n            elif name.endswith('_gradcam.png'):\n                target = GRADCAM_DIR / name\n            elif name.startswith('proposal_v2_') and name.endswith(('.csv', '.json', '.md')):\n                target = REPORTS_DIR / name\n            elif name.startswith('fold') and name.endswith('_context.json'):\n                target = REPORTS_DIR / 'folds' / name\n            else:\n                continue\n            merge_or_copy_file(source, target)\n\nrestore_previous_outputs()\nprint('WORK_DIR    =', WORK_DIR)\nprint('REPORTS_DIR =', REPORTS_DIR)\nprint('MODELS_DIR  =', MODELS_DIR)\n"}, {"cell_type": "markdown", "metadata": {}, "source": "## Ortak sabitler\n\nBu hucre pathleri, feature listelerini, model adlarini ve deney varsayilanlarini tanimlar."}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "from dataclasses import dataclass\n\nDATA_DIR = WORK_DIR / 'data'\nREPORTS_DIR = WORK_DIR / 'reports' / 'proposal_v2'\nMODELS_DIR = WORK_DIR / 'models' / 'proposal_v2'\n\nDEFAULT_SEED = 42\nDEFAULT_N_FOLDS = 5\nDEFAULT_VALIDATION_DAYS = 7\n\nCUSTOMER_NUMERIC_FEATURES = ['FN', 'Active', 'age']\nARTICLE_NUMERIC_FEATURES = [\n    'product_code',\n    'product_type_no',\n    'graphical_appearance_no',\n    'colour_group_code',\n    'perceived_colour_value_id',\n    'perceived_colour_master_id',\n    'department_no',\n    'index_group_no',\n    'section_no',\n    'garment_group_no',\n]\nVISUAL_NUMERIC_FEATURES = ['visual_similarity', 'visual_history_count']\nTABULAR_NUMERIC_FEATURES = CUSTOMER_NUMERIC_FEATURES + ARTICLE_NUMERIC_FEATURES\nFUSION_NUMERIC_FEATURES = TABULAR_NUMERIC_FEATURES + VISUAL_NUMERIC_FEATURES\n\nCUSTOMER_CATEGORICAL_FEATURES = ['club_member_status', 'fashion_news_frequency']\nARTICLE_CATEGORICAL_FEATURES = [\n    'product_type_name',\n    'product_group_name',\n    'graphical_appearance_name',\n    'colour_group_name',\n    'perceived_colour_value_name',\n    'perceived_colour_master_name',\n    'department_name',\n    'index_code',\n    'index_name',\n    'index_group_name',\n    'section_name',\n    'garment_group_name',\n]\nCATEGORICAL_FEATURES = CUSTOMER_CATEGORICAL_FEATURES + ARTICLE_CATEGORICAL_FEATURES\nMODEL_NAMES = ('tabular_only', 'image_history', 'late_fusion')\n\n@dataclass(frozen=True)\nclass V2Defaults:\n    n_folds: int = DEFAULT_N_FOLDS\n    validation_days: int = DEFAULT_VALIDATION_DAYS\n    seed: int = DEFAULT_SEED\n    top_k: int = 12\n    precision_k: int = 10\n    candidate_limit: int = 5000\n    visual_neighbors: int = 3000\n    co_purchase_per_item: int = 300\n    hybrid_weights: tuple[float, ...] = (0.25, 0.45, 0.65)\n    negatives_per_positive: int = 1\n    train_batch_size: int = 4096\n    epochs: int = 3\n    learning_rate: float = 1e-3\n\ndef ensure_v2_dirs():\n    for path in [REPORTS_DIR, REPORTS_DIR / 'folds', REPORTS_DIR / 'gradcam_examples', MODELS_DIR]:\n        path.mkdir(parents=True, exist_ok=True)"}, {"cell_type": "markdown", "metadata": {}, "source": "## Veri yukleme yardimcilari\n\nBu hucre H&M raw dosyalarini, image klasorunu ve gerekiyorsa embedding cache dosyalarini Kaggle inputlari veya local klasorlerden bulmak icin yardimci fonksiyonlari tanimlar."}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "\nimport os\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\n\n\n\ndef log(message: str) -> None:\n    print(message, flush=True)\n\n\ndef _candidate_raw_dirs() -> list[Path]:\n    candidates: list[Path] = []\n    env_dir = os.environ.get(\"HM_RAW_DIR\")\n    if env_dir:\n        candidates.append(Path(env_dir))\n    candidates.append(DATA_DIR / \"raw\")\n    kaggle_root = Path(\"/kaggle/input\")\n    if kaggle_root.exists():\n        for path in kaggle_root.rglob(\"transactions_train.csv\"):\n            candidates.append(path.parent)\n    return candidates\n\n\ndef resolve_raw_dir(raw_dir: str | Path | None = None) -> Path:\n    candidates = [Path(raw_dir)] if raw_dir else _candidate_raw_dirs()\n    for candidate in candidates:\n        if (candidate / \"transactions_train.csv\").exists():\n            return candidate\n    raise FileNotFoundError(\"Could not find H&M raw data directory. Set HM_RAW_DIR or pass --raw-dir.\")\n\n\ndef resolve_images_dir(raw_dir: Path, images_dir: str | Path | None = None) -> Path:\n    if images_dir:\n        return Path(images_dir)\n    for candidate in [raw_dir / \"images\", DATA_DIR / \"images\" / \"hm_images\"]:\n        if candidate.exists():\n            return candidate\n    return raw_dir / \"images\"\n\n\ndef _find_file_by_name(root: Path, name: str) -> Path | None:\n    if not root.exists():\n        return None\n    for path in root.rglob(name):\n        return path\n    return None\n\n\ndef resolve_embedding_paths(\n    embeddings_path: str | Path | None = None,\n    embedding_ids_path: str | Path | None = None,\n) -> tuple[Path, Path]:\n    if embeddings_path and embedding_ids_path:\n        return Path(embeddings_path), Path(embedding_ids_path)\n\n    env_embeddings = os.environ.get(\"HM_EMBEDDINGS_PATH\")\n    env_ids = os.environ.get(\"HM_EMBEDDING_IDS_PATH\")\n    if env_embeddings and env_ids:\n        return Path(env_embeddings), Path(env_ids)\n\n    local_embeddings = DATA_DIR / \"embeddings\" / \"final_kaggle\" / \"article_image_embeddings_popular.npy\"\n    local_ids = DATA_DIR / \"embeddings\" / \"final_kaggle\" / \"article_image_embedding_ids_popular.csv\"\n    if local_embeddings.exists() and local_ids.exists():\n        return local_embeddings, local_ids\n\n    kaggle_root = Path(\"/kaggle/input\")\n    embeddings = _find_file_by_name(kaggle_root, \"article_image_embeddings_popular.npy\")\n    ids = _find_file_by_name(kaggle_root, \"article_image_embedding_ids_popular.csv\")\n    if embeddings and ids:\n        return embeddings, ids\n\n    raise FileNotFoundError(\"Could not find EfficientNet embedding cache paths.\")\n\n\ndef read_transactions(raw_dir: Path) -> pd.DataFrame:\n    transactions = pd.read_csv(raw_dir / \"transactions_train.csv\", dtype={\"article_id\": str})\n    transactions[\"article_id\"] = transactions[\"article_id\"].astype(str).str.zfill(10)\n    transactions[\"t_dat\"] = pd.to_datetime(transactions[\"t_dat\"])\n    transactions[\"price\"] = transactions[\"price\"].astype(\"float32\")\n    transactions[\"sales_channel_id\"] = transactions[\"sales_channel_id\"].astype(\"int8\")\n    return transactions\n\n\ndef read_customers(raw_dir: Path) -> pd.DataFrame:\n    customers = pd.read_csv(raw_dir / \"customers.csv\")\n    customers[\"FN\"] = customers[\"FN\"].fillna(0).astype(\"float32\")\n    customers[\"Active\"] = customers[\"Active\"].fillna(0).astype(\"float32\")\n    customers[\"age\"] = customers[\"age\"].fillna(customers[\"age\"].median()).astype(\"float32\")\n    for column in [\"club_member_status\", \"fashion_news_frequency\"]:\n        customers[column] = customers[column].fillna(\"UNKNOWN\").astype(str)\n    return customers\n\n\ndef read_articles(raw_dir: Path) -> pd.DataFrame:\n    articles = pd.read_csv(raw_dir / \"articles.csv\", dtype={\"article_id\": str})\n    articles[\"article_id\"] = articles[\"article_id\"].astype(str).str.zfill(10)\n    for column in articles.columns:\n        if articles[column].dtype == \"object\":\n            articles[column] = articles[column].fillna(\"UNKNOWN\").astype(str)\n    return articles\n\n\ndef load_core_tables(raw_dir: str | Path | None = None) -> tuple[Path, pd.DataFrame, pd.DataFrame, pd.DataFrame]:\n    resolved = resolve_raw_dir(raw_dir)\n    log(f\"Using raw data: {resolved}\")\n    return resolved, read_transactions(resolved), read_customers(resolved), read_articles(resolved)\n\n\ndef load_embeddings(\n    embeddings_path: str | Path | None = None,\n    embedding_ids_path: str | Path | None = None,\n    mmap_mode: str | None = \"r\",\n) -> tuple[np.ndarray, list[str], dict[str, int]]:\n    emb_path, ids_path = resolve_embedding_paths(embeddings_path, embedding_ids_path)\n    log(f\"Using embeddings: {emb_path}\")\n    embeddings = np.load(emb_path, mmap_mode=mmap_mode)\n    ids_frame = pd.read_csv(ids_path, dtype={\"article_id\": str})\n    article_ids = ids_frame[\"article_id\"].astype(str).str.zfill(10).tolist()\n    article_to_index = {article_id: index for index, article_id in enumerate(article_ids)}\n    return embeddings, article_ids, article_to_index\n\n\ndef article_image_path(images_dir: Path, article_id: str) -> Path:\n    padded = str(article_id).zfill(10)\n    nested = images_dir / padded[:3] / f\"{padded}.jpg\"\n    if nested.exists():\n        return nested\n    return images_dir / f\"{padded}.jpg\""}, {"cell_type": "markdown", "metadata": {}, "source": "## Fold yardimci fonksiyonlari\n\nBu hucre customer-level split uretimi ve fold leakage kontrolu icin gereken fonksiyonlari tanimlar."}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "\nimport argparse\nimport json\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import KFold\n\n\n\ndef build_customer_folds(\n    transactions: pd.DataFrame,\n    n_folds: int = DEFAULT_N_FOLDS,\n    seed: int = DEFAULT_SEED,\n    min_history: int = 2,\n) -> pd.DataFrame:\n    customer_counts = transactions.groupby(\"customer_id\").size()\n    eligible_customers = customer_counts[customer_counts >= min_history].index.to_numpy()\n    eligible_customers = np.array(sorted(eligible_customers))\n\n    fold_rows: list[dict] = []\n    splitter = KFold(n_splits=n_folds, shuffle=True, random_state=seed)\n    for fold_id, (_, val_idx) in enumerate(splitter.split(eligible_customers)):\n        for customer_id in eligible_customers[val_idx]:\n            fold_rows.append({\"customer_id\": customer_id, \"fold_id\": fold_id})\n    return pd.DataFrame(fold_rows)\n\n\ndef validate_fold_protocol(\n    transactions: pd.DataFrame,\n    folds: pd.DataFrame,\n    validation_days: int = DEFAULT_VALIDATION_DAYS,\n) -> dict:\n    cutoff = transactions[\"t_dat\"].max() - pd.Timedelta(days=validation_days)\n    summary = {\n        \"cutoff\": str(cutoff.date()),\n        \"validation_days\": validation_days,\n        \"folds\": [],\n    }\n    for fold_id in sorted(folds[\"fold_id\"].unique()):\n        val_customers = set(folds.loc[folds[\"fold_id\"] == fold_id, \"customer_id\"])\n        train_customers = set(folds.loc[folds[\"fold_id\"] != fold_id, \"customer_id\"])\n        intersection = train_customers & val_customers\n        val_tx = transactions[transactions[\"customer_id\"].isin(val_customers)]\n        history = val_tx[val_tx[\"t_dat\"] <= cutoff]\n        truth = val_tx[val_tx[\"t_dat\"] > cutoff]\n        summary[\"folds\"].append(\n            {\n                \"fold_id\": int(fold_id),\n                \"train_customers\": len(train_customers),\n                \"validation_customers\": len(val_customers),\n                \"customer_intersection\": len(intersection),\n                \"validation_history_rows\": int(len(history)),\n                \"validation_truth_rows\": int(len(truth)),\n                \"validation_truth_customers\": int(truth[\"customer_id\"].nunique()),\n            }\n        )\n    return summary\n\n\ndef main() -> None:\n    parser = argparse.ArgumentParser(description=\"Create proposal v2 customer-level 5-fold splits.\")\n    parser.add_argument(\"--raw-dir\", type=Path, default=None)\n    parser.add_argument(\"--n-folds\", type=int, default=DEFAULT_N_FOLDS)\n    parser.add_argument(\"--seed\", type=int, default=DEFAULT_SEED)\n    parser.add_argument(\"--min-history\", type=int, default=2)\n    parser.add_argument(\"--validation-days\", type=int, default=DEFAULT_VALIDATION_DAYS)\n    parser.add_argument(\"--output\", type=Path, default=REPORTS_DIR / \"proposal_v2_fold_splits.csv\")\n    parser.add_argument(\"--summary\", type=Path, default=REPORTS_DIR / \"proposal_v2_fold_protocol_summary.json\")\n    args = parser.parse_args()\n\n    ensure_v2_dirs()\n    _, transactions, _, _ = load_core_tables(args.raw_dir)\n    folds = build_customer_folds(transactions, args.n_folds, args.seed, args.min_history)\n    summary = validate_fold_protocol(transactions, folds, args.validation_days)\n\n    args.output.parent.mkdir(parents=True, exist_ok=True)\n    folds.to_csv(args.output, index=False)\n    args.summary.write_text(json.dumps(summary, indent=2), encoding=\"utf-8\")\n    log(f\"Saved folds: {args.output}\")\n    log(f\"Saved protocol summary: {args.summary}\")\n    for row in summary[\"folds\"]:\n        if row[\"customer_intersection\"] != 0:\n            raise RuntimeError(f\"Fold {row['fold_id']} has customer leakage.\")"}, {"cell_type": "markdown", "metadata": {}, "source": "## Parametreler\n\n`N_FOLDS` proposal uyumu icin 5 olarak tutulur. `MIN_HISTORY`, yeterli gecmisi olmayan musterileri elemek icin kullanilir."}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "RAW_DIR = None\nN_FOLDS = 5\nRANDOM_SEED = 42\nMIN_HISTORY = 2\nVALIDATION_DAYS = 7"}, {"cell_type": "markdown", "metadata": {}, "source": "## Fold ciktilarini uret\n\nBu hucre transaction verisini yukler, fold CSV dosyasini yazar, protocol summary JSON dosyasini yazar ve fold kontrol tablosunu gosterir."}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "ensure_v2_dirs()\nraw_dir, transactions, customers, articles = load_core_tables(RAW_DIR)\nfolds = build_customer_folds(transactions, N_FOLDS, RANDOM_SEED, MIN_HISTORY)\nsummary = validate_fold_protocol(transactions, folds, VALIDATION_DAYS)\n\nfolds_path = REPORTS_DIR / 'proposal_v2_fold_splits.csv'\nsummary_path = REPORTS_DIR / 'proposal_v2_fold_protocol_summary.json'\nfolds.to_csv(folds_path, index=False)\nsummary_path.write_text(json.dumps(summary, indent=2), encoding='utf-8')\nfor row in summary['folds']:\n    if row['customer_intersection'] != 0:\n        raise RuntimeError(f\"Fold {row['fold_id']} has customer leakage.\")\nprint('Saved folds:', folds_path)\nprint('Saved summary:', summary_path)\ndisplay(pd.DataFrame(summary['folds']))"}], "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"name": "python", "pygments_lexer": "ipython3"}}, "nbformat": 4, "nbformat_minor": 5}