{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":31254,"databundleVersionId":3103714}],"dockerImageVersionId":31286,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install implicit","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-28T19:00:39.802126Z","iopub.execute_input":"2026-05-28T19:00:39.802487Z","iopub.status.idle":"2026-05-28T19:00:47.361439Z","shell.execute_reply.started":"2026-05-28T19:00:39.802448Z","shell.execute_reply":"2026-05-28T19:00:47.358057Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom datetime import timedelta\nimport gc # Garbage Collector для очищення пам'яті\n\n# Шляхи до файлів у середовищі Kaggle\nDATA_PATH = '/kaggle/input/competitions/h-and-m-personalized-fashion-recommendations/'\n\nprint(\"1. Завантаження даних...\")\n\n# Оптимізація пам'яті: article_id зчитуємо як рядок (щоб не втратити нулі на початку),\n# ціну як float32, канал продажів як int8. t_dat конвертуємо одразу.\ndtypes_trans = {\n    'article_id': 'str',\n    'price': 'float32',\n    'sales_channel_id': 'int8'\n}\n\ndf_full = pd.read_csv(\n    DATA_PATH + 'transactions_train.csv',\n    dtype=dtypes_trans,\n    parse_dates=['t_dat']\n)\n\ncustomers = pd.read_csv(DATA_PATH + 'customers.csv')\narticles = pd.read_csv(DATA_PATH + 'articles.csv', dtype={'article_id': 'str'})\n\nprint(f\"Початковий розмір транзакцій: {df_full.shape}\")\n\n# ---------------------------------------------------------\nprint(\"\\n2. Фільтрація за останні 4 тижні...\")\n# ---------------------------------------------------------\n# Знаходимо максимальну дату в датасеті\nmax_date = df_full['t_dat'].max()\n# Віднімаємо 27 днів, щоб отримати рівно 4 тижні (включно з останнім днем)\nmin_date = max_date - timedelta(days=27)\n\n# Залишаємо тільки робочий період і робимо .copy()\ndf = df_full[df_full['t_dat'] >= min_date].copy()\n\n# Видаляємо повний датасет з пам'яті, щоб звільнити RAM\ndel df_full\ngc.collect()\n\nprint(f\"Розмір після фільтрації за датою: {df.shape}\")\n\n# ---------------------------------------------------------\nprint(\"\\n3. Ітеративна фільтрація (k-core filtering)...\")\n# ---------------------------------------------------------\nMIN_USER_INTERACTIONS = 3\nMIN_ITEM_INTERACTIONS = 5\n\niteration = 1\nwhile True:\n    start_shape = df.shape[0]\n    \n    # 3.1. Фільтруємо товари (мінімум 5 покупок)\n    item_counts = df['article_id'].value_counts()\n    valid_items = item_counts[item_counts >= MIN_ITEM_INTERACTIONS].index\n    df = df[df['article_id'].isin(valid_items)]\n    \n    # 3.2. Фільтруємо клієнтів (мінімум 3 покупки)\n    user_counts = df['customer_id'].value_counts()\n    valid_users = user_counts[user_counts >= MIN_USER_INTERACTIONS].index\n    df = df[df['customer_id'].isin(valid_users)]\n    \n    end_shape = df.shape[0]\n    print(f\"  Ітерація {iteration}: залишилось {end_shape} рядків\")\n    \n    # Якщо після фільтрації розмір датафрейму не змінився - зупиняємо цикл\n    if start_shape == end_shape:\n        break\n    iteration += 1\n\nprint(\"\\n=== ГОТОВО ===\")\nprint(f\"Фінальна розмірність робочого датафрейму (df.shape): {df.shape}\")\nprint(f\"Унікальних клієнтів: {df['customer_id'].nunique()}\")\nprint(f\"Унікальних товарів: {df['article_id'].nunique()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-28T19:01:15.233012Z","iopub.execute_input":"2026-05-28T19:01:15.233330Z","iopub.status.idle":"2026-05-28T19:02:50.387181Z","shell.execute_reply.started":"2026-05-28T19:01:15.233294Z","shell.execute_reply":"2026-05-28T19:02:50.385655Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"1. Злиття транзакцій з інформацією про товари...\")\n\n# Беремо лише необхідні колонки для економії пам'яті\ndf_articles_subset = articles[['article_id', 'index_group_name']]\ndf_trans_subset = df[['customer_id', 'article_id']]\n\n# З'єднуємо транзакції з товарами\ndf_merged = df_trans_subset.merge(df_articles_subset, on='article_id', how='left')\n\nprint(\"2. Розрахунок найчастішого відділу (index_group_name) для кожного клієнта...\")\n\n# Рахуємо кількість покупок у кожному відділі для кожного користувача.\n# Цей метод (groupby -> size -> sort -> drop_duplicates) працює набагато швидше за агрегацію mode()\nuser_dept_counts = df_merged.groupby(['customer_id', 'index_group_name']).size().reset_index(name='count')\nuser_top_dept = user_dept_counts.sort_values(['customer_id', 'count'], ascending=[True, False])\\\n                                .drop_duplicates('customer_id')\n\nprint(\"3. Визначення статі та оновлення таблиці customers...\")\n\n# Створюємо словник для мапінгу відділу на стать\ngender_mapping = {\n    'Ladieswear': 'Female',\n    'Menswear': 'Male'\n}\n\n# Визначаємо стать для активних покупців (для інших відділів буде NaN, які ми потім заповнимо 'Unknown')\nuser_top_dept['inferred_gender'] = user_top_dept['index_group_name'].map(gender_mapping)\n\n# Створюємо словник {customer_id: inferred_gender} для швидкого перенесення в таблицю customers\ngender_dict = user_top_dept.set_index('customer_id')['inferred_gender'].to_dict()\n\n# Додаємо нову колонку в customers, заповнюючи порожні значення 'Unknown'\n# (порожніми будуть клієнти, які купували тільки дитячий одяг/спортивний, \n# або ті клієнти з загальної бази, які не потрапили у наш відфільтрований 4-тижневий період)\ncustomers['inferred_gender'] = customers['customer_id'].map(gender_dict).fillna('Unknown')\n\n# Очищення пам'яті від тимчасових датафреймів\ndel df_merged, df_articles_subset, df_trans_subset, user_dept_counts, user_top_dept\ngc.collect()\n\nprint(\"\\n=== РОЗПОДІЛ СТАТЕЙ ===\")\nprint(customers['inferred_gender'].value_counts(dropna=False))\n\n# Також подивимось на розподіл статей тільки серед тих 130 783 активних клієнтів з робочого df:\nprint(\"\\n=== РОЗПОДІЛ СТАТЕЙ (ТІЛЬКИ ДЛЯ АКТИВНИХ КЛІЄНТІВ У df) ===\")\nactive_customers = customers[customers['customer_id'].isin(df['customer_id'].unique())]\nprint(active_customers['inferred_gender'].value_counts(dropna=False))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-28T19:03:46.177292Z","iopub.execute_input":"2026-05-28T19:03:46.178818Z","iopub.status.idle":"2026-05-28T19:03:48.756540Z","shell.execute_reply.started":"2026-05-28T19:03:46.178745Z","shell.execute_reply":"2026-05-28T19:03:48.755461Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport scipy.sparse as sp\n\nprint(\"1. Time-based split (Поділ на train та test за часом)...\")\n\n# Знаходимо максимальну дату в нашому робочому відфільтрованому df\nmax_df_date = df['t_dat'].max()\nsplit_date = max_df_date - timedelta(days=6) # 7 днів включно (0-6)\n\n# Останній тиждень йде в test, попередні 3 тижні - в train\ntrain_df = df[df['t_dat'] < split_date].copy()\ntest_df = df[df['t_dat'] >= split_date].copy()\n\nprint(f\"Розмір train_df (3 тижні): {train_df.shape}\")\nprint(f\"Розмір test_df (1 тиждень): {test_df.shape}\")\n\nprint(\"\\n2. Створення мапінгу (customer_id -> user_idx, article_id -> item_idx)...\")\n\n# Отримуємо унікальних клієнтів та товари з УСЬОГО робочого df (щоб індекси збігалися і в train, і в test)\nunique_users = df['customer_id'].unique()\nunique_items = df['article_id'].unique()\n\n# Створюємо словники для швидкого перетворення\nuser2idx = {user: idx for idx, user in enumerate(unique_users)}\nitem2idx = {item: idx for idx, item in enumerate(unique_items)}\n\n# Також створюємо зворотні словники (знадобляться пізніше для розшифровки рекомендацій)\nidx2user = {idx: user for user, idx in user2idx.items()}\nidx2item = {idx: item for item, idx in item2idx.items()}\n\n# Додаємо нові колонки з індексами у train_df та test_df\ntrain_df['user_idx'] = train_df['customer_id'].map(user2idx)\ntrain_df['item_idx'] = train_df['article_id'].map(item2idx)\n\ntest_df['user_idx'] = test_df['customer_id'].map(user2idx)\ntest_df['item_idx'] = test_df['article_id'].map(item2idx)\n\nprint(f\"Кількість унікальних користувачів: {len(user2idx)}\")\nprint(f\"Кількість унікальних товарів: {len(item2idx)}\")\n\nprint(\"\\n3. Створення розрідженої матриці (Sparse CSR Matrix)...\")\n\n# Групуємо train_df за user_idx та item_idx і рахуємо кількість покупок (confidence)\ntrain_grouped = train_df.groupby(['user_idx', 'item_idx']).size().reset_index(name='count')\n\n# Отримуємо масиви для побудови CSR матриці\nrow_indices = train_grouped['user_idx'].values\ncol_indices = train_grouped['item_idx'].values\nvalues = train_grouped['count'].values # Або можна np.ones_like(row_indices) для бінарної матриці\n\n# Формуємо матрицю розмірністю (Кількість всіх користувачів) x (Кількість всіх товарів)\n# Це важливо, щоб розмірність збігалася зі словниками, навіть якщо хтось нічого не купив у train\nn_users = len(user2idx)\nn_items = len(item2idx)\n\nuser_item_matrix = sp.csr_matrix((values, (row_indices, col_indices)), shape=(n_users, n_items))\n\nprint(\"\\n=== ГОТОВО ===\")\nprint(f\"Розмірність user_item_matrix: {user_item_matrix.shape}\")\nprint(f\"Кількість ненульових елементів (взаємодій): {user_item_matrix.nnz}\")\nprint(f\"Щільність матриці (Sparsity): {user_item_matrix.nnz / (n_users * n_items) * 100:.4f}%\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-28T19:04:37.685156Z","iopub.execute_input":"2026-05-28T19:04:37.685836Z","iopub.status.idle":"2026-05-28T19:04:38.845545Z","shell.execute_reply.started":"2026-05-28T19:04:37.685786Z","shell.execute_reply":"2026-05-28T19:04:38.844322Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# NEW CELL A - Train-safe inferred_gender for E02\n#\n# WHERE TO PLACE:\n# Put this NEW code cell immediately AFTER the existing cell that starts with:\n# print(\"1. Time-based split (Поділ на train та test за часом)...\")\n#\n# WHY:\n# The old inferred_gender was calculated from the full 4-week df.\n# This version recalculates it from train_df only, before CatBoost features are built.\n# ============================================================\n\nimport gc\nimport numpy as np\nimport pandas as pd\n\nprint(\"NEW CELL A: Rebuilding inferred_gender from train_df only\")\n\n# Safety checks\nrequired_objects = [\"df\", \"train_df\", \"test_df\", \"customers\", \"articles\"]\nmissing_objects = [obj for obj in required_objects if obj not in globals()]\n\nif missing_objects:\n    raise RuntimeError(\n        f\"Missing required objects: {missing_objects}. \"\n        \"Run the previous notebook cells first.\"\n    )\n\n# Keep backup of the old full-window inferred_gender\nif \"inferred_gender\" in customers.columns and \"inferred_gender_full_window_backup\" not in customers.columns:\n    customers[\"inferred_gender_full_window_backup\"] = customers[\"inferred_gender\"]\n\n# Build gender source from TRAIN ONLY\ngender_train_source = train_df[[\"customer_id\", \"article_id\"]].merge(\n    articles[[\"article_id\", \"index_group_name\"]],\n    on=\"article_id\",\n    how=\"left\"\n)\n\n# Count index_group_name purchases per customer\nuser_group_counts = (\n    gender_train_source\n    .groupby([\"customer_id\", \"index_group_name\"])\n    .size()\n    .reset_index(name=\"purchase_count\")\n)\n\n# Pick dominant index_group_name per customer\ntrain_top_group = (\n    user_group_counts\n    .sort_values([\"customer_id\", \"purchase_count\"], ascending=[True, False])\n    .drop_duplicates(\"customer_id\")\n)\n\n# Map dominant product group to simple inferred gender label\ngender_mapping = {\n    \"Ladieswear\": \"Female\",\n    \"Menswear\": \"Male\"\n}\n\ntrain_top_group[\"inferred_gender_train_safe\"] = (\n    train_top_group[\"index_group_name\"]\n    .map(gender_mapping)\n    .fillna(\"Unknown\")\n)\n\ntrain_gender_map = train_top_group.set_index(\"customer_id\")[\"inferred_gender_train_safe\"]\n\n# Override the same column name used later by existing CatBoost cells\ncustomers[\"inferred_gender\"] = (\n    customers[\"customer_id\"]\n    .map(train_gender_map)\n    .fillna(\"Unknown\")\n)\n\nprint(\"\\n=== Train-safe inferred_gender distribution ===\")\nprint(customers[\"inferred_gender\"].value_counts(dropna=False))\n\n# Simple leakage audit\nif \"inferred_gender_full_window_backup\" in customers.columns:\n    audit_df = customers[[\"customer_id\", \"inferred_gender_full_window_backup\", \"inferred_gender\"]].copy()\n    audit_df[\"old_value\"] = audit_df[\"inferred_gender_full_window_backup\"].fillna(\"Unknown\")\n    audit_df[\"new_value\"] = audit_df[\"inferred_gender\"].fillna(\"Unknown\")\n    audit_df[\"changed\"] = audit_df[\"old_value\"] != audit_df[\"new_value\"]\n\n    print(\"\\n=== Leakage audit ===\")\n    print(f\"Changed customer labels: {audit_df['changed'].sum()}\")\n    print(f\"Changed label rate: {audit_df['changed'].mean():.4%}\")\n\ngender_leakage_audit = {\n    \"feature\": \"inferred_gender\",\n    \"old_method\": \"full 4-week df\",\n    \"new_method\": \"train_df only\",\n    \"test_week_used\": False\n}\n\ndel gender_train_source, user_group_counts, train_top_group, train_gender_map\ngc.collect()\n\nprint(\"\\nDone. Now continue with the ALS cell.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-28T19:05:24.772471Z","iopub.execute_input":"2026-05-28T19:05:24.773430Z","iopub.status.idle":"2026-05-28T19:05:26.940540Z","shell.execute_reply.started":"2026-05-28T19:05:24.773390Z","shell.execute_reply":"2026-05-28T19:05:26.939330Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nfrom implicit.als import AlternatingLeastSquares\n\n# Налаштування середовища для стабільної роботи Implicit (OpenBLAS)\nos.environ['OPENBLAS_NUM_THREADS'] = '1'\n\nprint(\"1. Ініціалізація та навчання моделі ALS...\")\n\n# Створюємо модель з твоїми параметрами\n# factors=50 — це розмірність векторів (latent features) для користувачів та товарів\nmodel = AlternatingLeastSquares(\n    factors=50, \n    iterations=15, \n    regularization=0.01, \n    random_state=42,\n    # Для Kaggle краще явно вказати використання CPU або GPU (якщо доступно)\n    use_gpu=False \n)\n\n# Навчаємо модель. ALS очікує матрицю у форматі User-Item (версії 0.6+)\n# Важливо: дані мають бути типу float32\nmodel.fit(user_item_matrix.astype(np.float32))\n\nprint(\"\\n2. Генерація Топ-50 кандидатів для тестових користувачів...\")\n\n# Вибираємо унікальних користувачів, які були активні в test_df\ntest_users = test_df['user_idx'].unique()\n\n# Викликаємо recommend для всього списку користувачів (batch mode)\n# N=50 — кількість кандидатів\n# filter_already_liked_items=True — не пропонувати те, що вже куплено в train\nids, scores = model.recommend(\n    userid=test_users, \n    user_items=user_item_matrix[test_users], \n    N=50, \n    filter_already_liked_items=True\n)\n\nprint(\"3. Формування результатів у DataFrame...\")\n\n# ids та scores мають розмірність (кількість_користувачів, 50)\n# Нам потрібно \"розгорнути\" їх у плоску таблицю\nals_candidates_df = pd.DataFrame({\n    'user_idx': np.repeat(test_users, 50),\n    'item_idx': ids.reshape(-1),\n    'Score_CF': scores.reshape(-1)\n})\n\n# Оптимізація типів даних для збереження пам'яті\nals_candidates_df['user_idx'] = als_candidates_df['user_idx'].astype('int32')\nals_candidates_df['item_idx'] = als_candidates_df['item_idx'].astype('int32')\nals_candidates_df['Score_CF'] = als_candidates_df['Score_CF'].astype('float32')\n\nprint(\"\\n=== РЕЗУЛЬТАТИ ===\")\nprint(f\"Загальний розмір als_candidates_df: {als_candidates_df.shape}\")\nprint(als_candidates_df.head())\n\n# Перевірка: чи всі тестові користувачі отримали по 50 рекомендацій?\nexpected_size = len(test_users) * 50\nactual_size = len(als_candidates_df)\nprint(f\"\\nОчікувана кількість рядків: {expected_size}\")\nprint(f\"Фактична кількість рядків: {actual_size}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-28T19:05:58.125802Z","iopub.execute_input":"2026-05-28T19:05:58.126224Z","iopub.status.idle":"2026-05-28T19:06:19.638215Z","shell.execute_reply.started":"2026-05-28T19:05:58.126191Z","shell.execute_reply":"2026-05-28T19:06:19.637391Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport gc\n\nprint(\"1. Підготовка базових пар (user, item) та target...\")\n\n# Додаємо індекси в основний df, якщо їх там ще немає\ndf['user_idx'] = df['customer_id'].map(user2idx)\ndf['item_idx'] = df['article_id'].map(item2idx)\n\n# Позитивні приклади (те, що реально купили в train_df)\npositives = train_df[['user_idx', 'item_idx']].copy()\npositives['target'] = 1\n\n# Генерація негативів (1:2)\nn_negatives = len(positives) * 2\nall_item_indices = np.arange(len(item2idx))\n\nneg_user_idx = np.repeat(positives['user_idx'].values, 2)\nneg_item_idx = np.random.choice(all_item_indices, size=n_negatives)\n\nnegatives = pd.DataFrame({\n    'user_idx': neg_user_idx,\n    'item_idx': neg_item_idx,\n    'target': 0\n})\n\n# Очищення негативів від реальних покупок\nall_interactions = set(zip(df['user_idx'], df['item_idx']))\nnegatives['is_real'] = [tuple(x) in all_interactions for x in negatives[['user_idx', 'item_idx']].values]\nnegatives = negatives[negatives['is_real'] == False].drop(columns=['is_real'])\n\n# Об'єднуємо\ncatboost_train_df = pd.concat([positives, negatives], axis=0).reset_index(drop=True)\n\nprint(f\"Розмір вибірки перед збагаченням: {catboost_train_df.shape}\")\n\nprint(\"\\n2. Збагачення даними про клієнтів (Customers)...\")\n\n# Створюємо копію subset-у клієнтів з індексом\ncustomers_feat = customers.copy()\ncustomers_feat['user_idx'] = customers_feat['customer_id'].map(user2idx)\ncustomers_feat['postal_code'] = customers_feat['postal_code'].astype(str)\n\n# Мерджимо тільки потрібні колонки\ncatboost_train_df = catboost_train_df.merge(\n    customers_feat[['user_idx', 'age', 'postal_code', 'inferred_gender']], \n    on='user_idx', \n    how='left'\n)\n\nprint(\"3. Збагачення даними про товари (Articles)...\")\n\n# Створюємо копію subset-у товарів з індексом\narticles_feat = articles.copy()\narticles_feat['item_idx'] = articles_feat['article_id'].map(item2idx)\n\n# Колонки, які ти просив (використовуємо імена)\nart_cols = [\n    'item_idx', 'product_type_name', 'graphical_appearance_name', \n    'colour_group_name', 'perceived_colour_value_name', \n    'index_name', 'index_group_name'\n]\n\ncatboost_train_df = catboost_train_df.merge(\n    articles_feat[art_cols], \n    on='item_idx', \n    how='left'\n)\n\n# Фінальні штрихи\ncatboost_train_df['age'] = catboost_train_df['age'].fillna(catboost_train_df['age'].median())\n\nprint(\"\\n=== ГОТОВО ===\")\nprint(f\"Фінальна розмірність catboost_train_df: {catboost_train_df.shape}\")\nprint(\"\\nРозподіл target:\")\nprint(catboost_train_df['target'].value_counts(normalize=True))\n\n# Очищення пам'яті\ndel positives, negatives, customers_feat, articles_feat\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-28T19:06:28.085997Z","iopub.execute_input":"2026-05-28T19:06:28.087171Z","iopub.status.idle":"2026-05-28T19:06:34.264275Z","shell.execute_reply.started":"2026-05-28T19:06:28.087129Z","shell.execute_reply":"2026-05-28T19:06:34.263005Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from catboost import CatBoostClassifier, Pool\n\nprint(\"1. Підготовка даних для навчання (Train)...\")\n\n# Визначаємо список категоріальних ознак\ncat_features = [\n    'postal_code', 'inferred_gender', 'product_type_name', \n    'graphical_appearance_name', 'colour_group_name', \n    'perceived_colour_value_name', 'index_name', 'index_group_name'\n]\n\n# Створюємо X та y для навчання\n# Виключаємо target та технічні ID\nX = catboost_train_df.drop(columns=['target', 'user_idx', 'item_idx'])\ny = catboost_train_df['target']\n\nprint(f\"Фічі для навчання: {list(X.columns)}\")\n\nprint(\"\\n2. Навчання CatBoostClassifier...\")\n\n# Ініціалізація моделі за твоїми параметрами\nmodel_cb = CatBoostClassifier(\n    iterations=100,\n    learning_rate=0.1,\n    depth=6,\n    loss_function='Logloss',\n    eval_metric='AUC',\n    random_seed=42,\n    verbose=20\n)\n\n# Навчання моделі\nmodel_cb.fit(X, y, cat_features=cat_features)\n\nprint(\"\\n3. Підготовка кандидатів для передбачення (Inference)...\")\n\n# Збагачуємо als_candidates_df (2.2 млн рядків) фічами\n# Використовуємо ті ж самі таблиці, що і в Кроці 5\n\n# --- Додаємо дані про клієнтів ---\ncustomers_feat = customers[['customer_id', 'age', 'postal_code', 'inferred_gender']].copy()\ncustomers_feat['user_idx'] = customers_feat['customer_id'].map(user2idx)\ncustomers_feat['postal_code'] = customers_feat['postal_code'].astype(str)\n\nals_candidates_df = als_candidates_df.merge(\n    customers_feat[['user_idx', 'age', 'postal_code', 'inferred_gender']], \n    on='user_idx', \n    how='left'\n)\n\n# --- Додаємо дані про товари ---\nart_features = [\n    'product_type_name',\n    'graphical_appearance_name',\n    'colour_group_name',\n    'perceived_colour_value_name',\n    'index_name',\n    'index_group_name'\n]\n\narticles_feat = articles[['article_id'] + art_features].copy()\narticles_feat['item_idx'] = articles_feat['article_id'].map(item2idx)\n\nals_candidates_df = als_candidates_df.merge(\n    articles_feat.drop(columns=['article_id']), \n    on='item_idx', \n    how='left'\n)\n\n# Заповнюємо медіаною вік у кандидатах\nals_candidates_df['age'] = als_candidates_df['age'].fillna(als_candidates_df['age'].median())\n\nprint(f\"Розмірність кандидатів після merge: {als_candidates_df.shape}\")\n\nprint(\"\\n4. Генерація передбачень (Score_Dem)...\")\n\n# Для передбачення нам потрібні лише ті фічі, на яких вчилася модель\n# (важливо: порядок колонок має бути таким самим, як у X)\nX_test = als_candidates_df[X.columns]\n\n# predict_proba повертає [ймовірність 0, ймовірність 1]\n# Нам потрібна ймовірність класу 1 (що користувач купить товар)\nals_candidates_df['Score_Dem'] = model_cb.predict_proba(X_test)[:, 1]\n\nprint(\"\\n=== ГОТОВО ===\")\nprint(als_candidates_df[['user_idx', 'item_idx', 'Score_CF', 'Score_Dem']].head())\n\n# Очищення пам'яті\ndel X, y, X_test, customers_feat, articles_feat\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-28T19:06:45.556456Z","iopub.execute_input":"2026-05-28T19:06:45.557451Z","iopub.status.idle":"2026-05-28T19:07:54.610390Z","shell.execute_reply.started":"2026-05-28T19:06:45.557414Z","shell.execute_reply":"2026-05-28T19:07:54.609354Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LogisticRegression\n\nprint(\"1. Створення Target для тесту...\")\n\n# Створюємо набір пар (user, item), які реально були куплені в останній тиждень\nactual_test_purchases = test_df[['user_idx', 'item_idx']].drop_duplicates().copy()\nactual_test_purchases['target'] = 1\n\n# Об'єднуємо наших кандидатів (2.2 млн) з реальними фактами покупок\nals_candidates_df = als_candidates_df.merge(\n    actual_test_purchases, \n    on=['user_idx', 'item_idx'], \n    how='left'\n)\nals_candidates_df['target'] = als_candidates_df['target'].fillna(0).astype('int8')\n\nprint(f\"Знайдено реальних покупок серед кандидатів: {als_candidates_df['target'].sum()}\")\n\nprint(\"\\n2. Масштабування скорів...\")\n\n# Масштабування необхідне для логістичної регресії, щоб ваги були порівнюваними\nscaler = StandardScaler()\nals_candidates_df[['Score_CF_scaled', 'Score_Dem_scaled']] = scaler.fit_transform(\n    als_candidates_df[['Score_CF', 'Score_Dem']]\n)\n\nprint(\"3. Навчання мета-моделі (Логістична регресія)...\")\n\n# Використовуємо логістичну регресію як блендер\nX_meta = als_candidates_df[['Score_CF_scaled', 'Score_Dem_scaled']]\ny_meta = als_candidates_df['target']\n\nmodel_lr = LogisticRegression()\nmodel_lr.fit(X_meta, y_meta)\n\n# 4. Отримання ваг\nalpha, beta = model_lr.coef_[0]\nintercept = model_lr.intercept_[0]\n\nprint(\"\\n=== ОТРИМАНІ ВАГИ (КОЕФІЦІЄНТИ) ===\")\nprint(f\"Вага Alpha (ALS Score): {alpha:.4f}\")\nprint(f\"Вага Beta (CatBoost Score): {beta:.4f}\")\nprint(f\"Зсув (Intercept): {intercept:.4f}\")\n\nprint(\"\\n5. Розрахунок Hybrid_Score...\")\n\n# Розраховуємо фінальний гібридний бал за отриманою формулою\nals_candidates_df['Hybrid_Score'] = (\n    alpha * als_candidates_df['Score_CF_scaled'] + \n    beta * als_candidates_df['Score_Dem_scaled'] + \n    intercept\n)\n\n# Сортуємо кандидатів для кожного користувача за Hybrid_Score\nals_candidates_df = als_candidates_df.sort_values(\n    by=['user_idx', 'Hybrid_Score'], \n    ascending=[True, False]\n)\n\nprint(\"\\n=== ФІНАЛЬНА ТАБЛИЦЯ (ПЕРШІ 5 РЯДКІВ) ===\")\ncols_to_show = ['user_idx', 'item_idx', 'Score_CF', 'Score_Dem', 'Hybrid_Score', 'target']\nprint(als_candidates_df[cols_to_show].head())\n\n# Очищення\ndel X_meta, y_meta\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LogisticRegression\nimport pandas as pd\nimport numpy as np\n\nprint(\"1. Очищення та маркування цільової змінної (Target Labeling)...\")\n\n# Якщо ми перезапускаємо комірку, видалимо старі колонки, щоб уникнути target_x / target_y\ncols_to_drop = ['target', 'target_x', 'target_y', 'Scaled_CF', 'Scaled_Dem', 'Hybrid_Score']\nals_candidates_df = als_candidates_df.drop(columns=[c for c in cols_to_drop if c in als_candidates_df.columns])\n\n# Створюємо чистий набір реальних покупок з test_df\n# Переконуємось, що типи даних збігаються (int32)\nactual_test_interactions = test_df[['user_idx', 'item_idx']].drop_duplicates().copy()\nactual_test_interactions['user_idx'] = actual_test_interactions['user_idx'].astype('int32')\nactual_test_interactions['item_idx'] = actual_test_interactions['item_idx'].astype('int32')\nactual_test_interactions['target'] = 1\n\n# Виконуємо merge\nals_candidates_df = als_candidates_df.merge(\n    actual_test_interactions, \n    on=['user_idx', 'item_idx'], \n    how='left'\n)\n\n# ТЕПЕР колонка 'target' точно має бути. Заповнюємо порожні значення нулями\nif 'target' in als_candidates_df.columns:\n    als_candidates_df['target'] = als_candidates_df['target'].fillna(0).astype('int8')\nelse:\n    # Страховка: якщо merge чомусь не створив колонку\n    print(\"Попередження: колонку target не знайдено, створюємо її вручну\")\n    als_candidates_df['target'] = 0\n\nprint(f\"Знайдено реальних покупок серед кандидатів: {als_candidates_df['target'].sum()}\")\n\nprint(\"\\n2. Масштабування ознак (Scaling)...\")\n\nscaler = StandardScaler()\n# Масштабуємо тільки скори. Score_CF та Score_Dem мають бути в als_candidates_df з Кроку 6\nscores_to_scale = als_candidates_df[['Score_CF', 'Score_Dem']].values\nscaled_values = scaler.fit_transform(scores_to_scale)\n\nals_candidates_df['Scaled_CF'] = scaled_values[:, 0]\nals_candidates_df['Scaled_Dem'] = scaled_values[:, 1]\n\nprint(\"3. Навчання Мета-моделі (Logistic Regression)...\")\n\nX_meta = als_candidates_df[['Scaled_CF', 'Scaled_Dem']]\ny_meta = als_candidates_df['target']\n\n# Навчання з балансуванням класів\nmodel_lr = LogisticRegression(class_weight='balanced', random_state=42)\nmodel_lr.fit(X_meta, y_meta)\n\n# Екстракція ваг\nalpha = model_lr.coef_[0][0]\nbeta = model_lr.coef_[0][1]\nintercept = model_lr.intercept_[0]\n\nprint(\"\\n=== ВАГИ ГІБРИДНОЇ МОДЕЛІ ===\")\nprint(f\"Alpha (ALS weight): {alpha:.4f}\")\nprint(f\"Beta (CatBoost weight): {beta:.4f}\")\nprint(f\"Intercept: {intercept:.4f}\")\n\nprint(\"\\n4. Розрахунок Hybrid_Score та фінальне сортування...\")\n\nals_candidates_df['Hybrid_Score'] = (\n    alpha * als_candidates_df['Scaled_CF'] + \n    beta * als_candidates_df['Scaled_Dem'] + \n    intercept\n)\n\n# Сортуємо: для кожного юзера найкращі товари будуть зверху\nals_candidates_df = als_candidates_df.sort_values(\n    by=['user_idx', 'Hybrid_Score'], \n    ascending=[True, False]\n).reset_index(drop=True)\n\nprint(\"\\n=== РЕЗУЛЬТАТ (ТОП-5) ===\")\nprint(als_candidates_df[['user_idx', 'item_idx', 'Score_CF', 'Score_Dem', 'Hybrid_Score', 'target']].head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-28T19:08:12.303186Z","iopub.execute_input":"2026-05-28T19:08:12.304239Z","iopub.status.idle":"2026-05-28T19:08:18.868153Z","shell.execute_reply.started":"2026-05-28T19:08:12.304176Z","shell.execute_reply":"2026-05-28T19:08:18.867192Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# NEW CELL B - E00 / E01 / E02 curator summary\n#\n# WHERE TO PLACE:\n# Put this NEW code cell immediately AFTER the Logistic Regression cell\n# that creates Hybrid_Score.\n#\n# PURPOSE:\n# - Lock E01 ALS baseline.\n# - Lock E02 full hybrid result.\n# - Create a safe curator-review table without overclaiming.\n# ============================================================\n\nimport pandas as pd\nimport numpy as np\nimport gc\n\nprint(\"NEW CELL B: E00 / E01 / E02 summary\")\n\n# ------------------------------------------------------------\n# 1. Safety checks\n# ------------------------------------------------------------\n\nrequired_objects = [\"als_candidates_df\", \"model_cb\", \"model_lr\", \"alpha\", \"beta\", \"intercept\"]\nmissing_objects = [obj for obj in required_objects if obj not in globals()]\n\nif missing_objects:\n    raise RuntimeError(\n        f\"Missing required objects: {missing_objects}. \"\n        \"Run the ALS, CatBoost, and Logistic Regression cells first.\"\n    )\n\nrequired_cols = [\"user_idx\", \"item_idx\", \"Score_CF\", \"Score_Dem\", \"Hybrid_Score\", \"target\"]\nmissing_cols = [col for col in required_cols if col not in als_candidates_df.columns]\n\nif missing_cols:\n    raise RuntimeError(f\"als_candidates_df is missing columns: {missing_cols}\")\n\n# ------------------------------------------------------------\n# 2. Precision@12 function\n# ------------------------------------------------------------\n\ndef precision_at_12(df, score_col):\n    top_12 = (\n        df.sort_values([\"user_idx\", score_col], ascending=[True, False])\n          .groupby(\"user_idx\")\n          .head(12)\n    )\n    \n    user_hits = top_12.groupby(\"user_idx\")[\"target\"].sum()\n    return (user_hits / 12).mean()\n\n# ------------------------------------------------------------\n# 3. E01 and E02 metrics\n# ------------------------------------------------------------\n\nbaseline_p12 = precision_at_12(als_candidates_df, \"Score_CF\")\nhybrid_p12 = precision_at_12(als_candidates_df, \"Hybrid_Score\")\n\nif baseline_p12 != 0:\n    uplift_pct = ((hybrid_p12 - baseline_p12) / baseline_p12) * 100\nelse:\n    uplift_pct = np.nan\n\nprint(\"\\n=== Precision@12 results ===\")\nprint(f\"E01 ALS baseline Precision@12: {baseline_p12:.6f}\")\nprint(f\"E02 full hybrid Precision@12: {hybrid_p12:.6f}\")\nprint(f\"Uplift vs E01: {uplift_pct:.2f}%\")\n\n# ------------------------------------------------------------\n# 4. Feature importance\n# ------------------------------------------------------------\n\ntry:\n    feature_names = list(model_cb.feature_names_)\nexcept Exception:\n    feature_names = [\n        \"age\",\n        \"postal_code\",\n        \"inferred_gender\",\n        \"product_type_name\",\n        \"graphical_appearance_name\",\n        \"colour_group_name\",\n        \"perceived_colour_value_name\",\n        \"index_name\",\n        \"index_group_name\"\n    ]\n\nfeature_importance = model_cb.get_feature_importance()\n\nfi_df = (\n    pd.DataFrame({\n        \"feature\": feature_names,\n        \"importance\": feature_importance\n    })\n    .sort_values(\"importance\", ascending=False)\n    .reset_index(drop=True)\n)\n\nprint(\"\\n=== CatBoost feature importance: top 10 ===\")\ndisplay(fi_df.head(10))\n\n# ------------------------------------------------------------\n# 5. E00 / E01 / E02 results table\n# ------------------------------------------------------------\n\nexperiment_summary = pd.DataFrame([\n    {\n        \"experiment_id\": \"E00\",\n        \"name\": \"Reproduce current notebook\",\n        \"model\": \"Existing ALS + CatBoost + Logistic Regression pipeline\",\n        \"feature_set\": \"Current notebook feature set\",\n        \"Precision@12\": np.nan,\n        \"uplift_vs_E01_%\": np.nan,\n        \"status\": \"Completed if notebook ran end-to-end\",\n        \"notes\": \"Use this as reproducibility checkpoint, not a separate model result\"\n    },\n    {\n        \"experiment_id\": \"E01\",\n        \"name\": \"ALS baseline only\",\n        \"model\": \"ALS candidate generation ranked by Score_CF\",\n        \"feature_set\": \"User-item interaction matrix only\",\n        \"Precision@12\": baseline_p12,\n        \"uplift_vs_E01_%\": 0.0,\n        \"status\": \"Completed\",\n        \"notes\": \"Main comparison baseline\"\n    },\n    {\n        \"experiment_id\": \"E02\",\n        \"name\": \"Full current hybrid\",\n        \"model\": \"ALS + CatBoost + Logistic Regression late fusion\",\n        \"feature_set\": \"age, postal_code, inferred_gender, article metadata\",\n        \"Precision@12\": hybrid_p12,\n        \"uplift_vs_E01_%\": uplift_pct,\n        \"status\": \"Completed\",\n        \"notes\": \"Use measured uplift only. Do not overclaim if the difference is small or negative\"\n    }\n])\n\nprint(\"\\n=== Curator review summary table ===\")\ndisplay(experiment_summary)\n\n# ------------------------------------------------------------\n# 6. Logistic Regression coefficients\n# ------------------------------------------------------------\n\nlr_coefficients = pd.DataFrame([\n    {\n        \"coefficient\": \"alpha\",\n        \"input_score\": \"Scaled ALS Score_CF\",\n        \"value\": alpha\n    },\n    {\n        \"coefficient\": \"beta\",\n        \"input_score\": \"Scaled CatBoost Score_Dem\",\n        \"value\": beta\n    },\n    {\n        \"coefficient\": \"intercept\",\n        \"input_score\": \"Model intercept\",\n        \"value\": intercept\n    }\n])\n\nprint(\"\\n=== Logistic Regression fusion coefficients ===\")\ndisplay(lr_coefficients)\n\n# ------------------------------------------------------------\n# 7. Safe interpretation for notes\n# ------------------------------------------------------------\n\nif pd.isna(uplift_pct):\n    decision = \"Cannot calculate uplift because baseline is zero.\"\nelif uplift_pct > 0:\n    decision = \"Hybrid is better than ALS in this run, but describe the gain by its actual size.\"\nelif uplift_pct < 0:\n    decision = \"Hybrid is worse than ALS in this run. This is still a valid result for feature-impact analysis.\"\nelse:\n    decision = \"Hybrid and ALS are equal in this run.\"\n\nprint(\"\\n=== Safe curator interpretation ===\")\nprint(decision)\n\nprint(\"\\nUse these objects for logging:\")\nprint(\"- experiment_summary\")\nprint(\"- fi_df\")\nprint(\"- lr_coefficients\")\n\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-28T19:10:00.880898Z","iopub.execute_input":"2026-05-28T19:10:00.881370Z","iopub.status.idle":"2026-05-28T19:10:04.005478Z","shell.execute_reply.started":"2026-05-28T19:10:00.881337Z","shell.execute_reply":"2026-05-28T19:10:04.004585Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# NEW CELL C - E03: Full hybrid WITHOUT inferred_gender\n#\n# WHERE TO PLACE:\n# Put this NEW code cell immediately AFTER:\n# NEW CELL B - E00 / E01 / E02 curator summary\n#\n# PURPOSE:\n# Clean ablation:\n# - E02 used age + postal_code + inferred_gender + article metadata.\n# - E03 removes only inferred_gender from CatBoost.\n# - ALS candidates and Score_CF stay unchanged.\n# ============================================================\n\nimport pandas as pd\nimport numpy as np\nimport gc\n\nfrom catboost import CatBoostClassifier\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import roc_auc_score\n\nprint(\"NEW CELL C: E03 - Hybrid without inferred_gender\")\n\n# ------------------------------------------------------------\n# 1. Safety checks\n# ------------------------------------------------------------\n\nrequired_objects = [\n    \"catboost_train_df\",\n    \"als_candidates_df\",\n    \"baseline_p12\",\n    \"hybrid_p12\"\n]\n\nmissing_objects = [obj for obj in required_objects if obj not in globals()]\n\nif missing_objects:\n    raise RuntimeError(\n        f\"Missing required objects: {missing_objects}. \"\n        \"Run cells up to NEW CELL B first.\"\n    )\n\nrequired_train_cols = [\n    \"target\",\n    \"user_idx\",\n    \"item_idx\",\n    \"age\",\n    \"postal_code\",\n    \"inferred_gender\",\n    \"product_type_name\",\n    \"graphical_appearance_name\",\n    \"colour_group_name\",\n    \"perceived_colour_value_name\",\n    \"index_name\",\n    \"index_group_name\"\n]\n\nmissing_train_cols = [col for col in required_train_cols if col not in catboost_train_df.columns]\n\nif missing_train_cols:\n    raise RuntimeError(f\"catboost_train_df is missing columns: {missing_train_cols}\")\n\nrequired_candidate_cols = [\n    \"user_idx\",\n    \"item_idx\",\n    \"target\",\n    \"Score_CF\",\n    \"age\",\n    \"postal_code\",\n    \"inferred_gender\",\n    \"product_type_name\",\n    \"graphical_appearance_name\",\n    \"colour_group_name\",\n    \"perceived_colour_value_name\",\n    \"index_name\",\n    \"index_group_name\"\n]\n\nmissing_candidate_cols = [col for col in required_candidate_cols if col not in als_candidates_df.columns]\n\nif missing_candidate_cols:\n    raise RuntimeError(f\"als_candidates_df is missing columns: {missing_candidate_cols}\")\n\n# ------------------------------------------------------------\n# 2. Define E03 feature set\n# ------------------------------------------------------------\n\ne03_drop_cols = [\"target\", \"user_idx\", \"item_idx\", \"inferred_gender\"]\n\nX_e03 = catboost_train_df.drop(columns=e03_drop_cols)\ny_e03 = catboost_train_df[\"target\"]\n\ncat_features_e03 = [\n    \"postal_code\",\n    \"product_type_name\",\n    \"graphical_appearance_name\",\n    \"colour_group_name\",\n    \"perceived_colour_value_name\",\n    \"index_name\",\n    \"index_group_name\"\n]\n\nprint(\"\\nE03 feature set:\")\nprint(list(X_e03.columns))\n\nif \"inferred_gender\" in X_e03.columns:\n    raise RuntimeError(\"E03 failed: inferred_gender is still in X_e03.\")\n\n# ------------------------------------------------------------\n# 3. Train CatBoost for E03\n# ------------------------------------------------------------\n\nprint(\"\\nTraining CatBoost E03...\")\n\nmodel_cb_e03 = CatBoostClassifier(\n    iterations=100,\n    learning_rate=0.1,\n    depth=6,\n    loss_function=\"Logloss\",\n    eval_metric=\"AUC\",\n    random_seed=42,\n    verbose=20\n)\n\nmodel_cb_e03.fit(X_e03, y_e03, cat_features=cat_features_e03)\n\n# ------------------------------------------------------------\n# 4. Generate E03 CatBoost score on same ALS candidates\n# ------------------------------------------------------------\n\nprint(\"\\nScoring same ALS candidates for E03...\")\n\nX_test_e03 = als_candidates_df[X_e03.columns]\n\nals_candidates_df[\"Score_Dem_E03\"] = model_cb_e03.predict_proba(X_test_e03)[:, 1]\n\n# ------------------------------------------------------------\n# 5. Train Logistic Regression fusion for E03\n# ------------------------------------------------------------\n\nprint(\"\\nTraining Logistic Regression fusion for E03...\")\n\nscaler_e03 = StandardScaler()\n\nscaled_e03 = scaler_e03.fit_transform(\n    als_candidates_df[[\"Score_CF\", \"Score_Dem_E03\"]]\n)\n\nals_candidates_df[\"Scaled_CF_E03\"] = scaled_e03[:, 0]\nals_candidates_df[\"Scaled_Dem_E03\"] = scaled_e03[:, 1]\n\nX_meta_e03 = als_candidates_df[[\"Scaled_CF_E03\", \"Scaled_Dem_E03\"]]\ny_meta_e03 = als_candidates_df[\"target\"]\n\nmodel_lr_e03 = LogisticRegression(class_weight=\"balanced\", random_state=42)\nmodel_lr_e03.fit(X_meta_e03, y_meta_e03)\n\nalpha_e03 = model_lr_e03.coef_[0][0]\nbeta_e03 = model_lr_e03.coef_[0][1]\nintercept_e03 = model_lr_e03.intercept_[0]\n\nals_candidates_df[\"Hybrid_Score_E03\"] = (\n    alpha_e03 * als_candidates_df[\"Scaled_CF_E03\"] +\n    beta_e03 * als_candidates_df[\"Scaled_Dem_E03\"] +\n    intercept_e03\n)\n\nprint(\"\\n=== E03 Logistic Regression coefficients ===\")\nprint(f\"Alpha / ALS Score_CF: {alpha_e03:.4f}\")\nprint(f\"Beta / CatBoost Score_Dem_E03: {beta_e03:.4f}\")\nprint(f\"Intercept: {intercept_e03:.4f}\")\n\n# ------------------------------------------------------------\n# 6. Evaluation helpers\n# ------------------------------------------------------------\n\ndef precision_at_12_e03(df, score_col):\n    top_12 = (\n        df.sort_values([\"user_idx\", score_col], ascending=[True, False])\n          .groupby(\"user_idx\")\n          .head(12)\n    )\n    user_hits = top_12.groupby(\"user_idx\")[\"target\"].sum()\n    return (user_hits / 12).mean()\n\ndef safe_auc(y_true, score):\n    if len(np.unique(y_true)) < 2:\n        return np.nan\n    return roc_auc_score(y_true, score)\n\n# ------------------------------------------------------------\n# 7. E03 metrics\n# ------------------------------------------------------------\n\ne03_p12 = precision_at_12_e03(als_candidates_df, \"Hybrid_Score_E03\")\n\ne03_uplift_vs_e01 = ((e03_p12 - baseline_p12) / baseline_p12) * 100\ne03_delta_vs_e02 = ((e03_p12 - hybrid_p12) / hybrid_p12) * 100\n\nauc_score_dem_e02 = safe_auc(als_candidates_df[\"target\"], als_candidates_df[\"Score_Dem\"])\nauc_score_dem_e03 = safe_auc(als_candidates_df[\"target\"], als_candidates_df[\"Score_Dem_E03\"])\nauc_hybrid_e02 = safe_auc(als_candidates_df[\"target\"], als_candidates_df[\"Hybrid_Score\"])\nauc_hybrid_e03 = safe_auc(als_candidates_df[\"target\"], als_candidates_df[\"Hybrid_Score_E03\"])\n\nprint(\"\\n=== E03 Precision@12 results ===\")\nprint(f\"E01 ALS baseline Precision@12: {baseline_p12:.6f}\")\nprint(f\"E02 full hybrid Precision@12: {hybrid_p12:.6f}\")\nprint(f\"E03 hybrid without inferred_gender Precision@12: {e03_p12:.6f}\")\nprint(f\"E03 uplift vs E01: {e03_uplift_vs_e01:.2f}%\")\nprint(f\"E03 delta vs E02: {e03_delta_vs_e02:.2f}%\")\n\nprint(\"\\n=== AUC diagnostic ===\")\nprint(f\"E02 CatBoost Score_Dem AUC: {auc_score_dem_e02:.6f}\")\nprint(f\"E03 CatBoost Score_Dem_E03 AUC: {auc_score_dem_e03:.6f}\")\nprint(f\"E02 Hybrid_Score AUC: {auc_hybrid_e02:.6f}\")\nprint(f\"E03 Hybrid_Score_E03 AUC: {auc_hybrid_e03:.6f}\")\n\n# ------------------------------------------------------------\n# 8. Feature importance for E03\n# ------------------------------------------------------------\n\nfi_e03 = (\n    pd.DataFrame({\n        \"feature\": list(model_cb_e03.feature_names_),\n        \"importance\": model_cb_e03.get_feature_importance()\n    })\n    .sort_values(\"importance\", ascending=False)\n    .reset_index(drop=True)\n)\n\nprint(\"\\n=== E03 CatBoost feature importance: top 10 ===\")\ndisplay(fi_e03.head(10))\n\n# Confirm inferred_gender is absent\nprint(\"\\nFeature check:\")\nprint(f\"inferred_gender in E03 features: {'inferred_gender' in list(X_e03.columns)}\")\n\n# ------------------------------------------------------------\n# 9. E03 summary table\n# ------------------------------------------------------------\n\ne03_summary = pd.DataFrame([\n    {\n        \"experiment_id\": \"E01\",\n        \"name\": \"ALS baseline only\",\n        \"Precision@12\": baseline_p12,\n        \"uplift_vs_E01_%\": 0.0,\n        \"delta_vs_E02_%\": np.nan,\n        \"notes\": \"Locked baseline\"\n    },\n    {\n        \"experiment_id\": \"E02\",\n        \"name\": \"Full current hybrid\",\n        \"Precision@12\": hybrid_p12,\n        \"uplift_vs_E01_%\": ((hybrid_p12 - baseline_p12) / baseline_p12) * 100,\n        \"delta_vs_E02_%\": 0.0,\n        \"notes\": \"Full feature set with train-safe inferred_gender\"\n    },\n    {\n        \"experiment_id\": \"E03\",\n        \"name\": \"Hybrid without inferred_gender\",\n        \"Precision@12\": e03_p12,\n        \"uplift_vs_E01_%\": e03_uplift_vs_e01,\n        \"delta_vs_E02_%\": e03_delta_vs_e02,\n        \"notes\": \"Clean ablation: removed inferred_gender only\"\n    }\n])\n\nprint(\"\\n=== E03 comparison summary ===\")\ndisplay(e03_summary)\n\n# ------------------------------------------------------------\n# 10. Safe decision line\n# ------------------------------------------------------------\n\nif e03_p12 > hybrid_p12:\n    e03_decision = (\n        \"E03 improved over E02. Removing inferred_gender recovered part of the hybrid loss. \"\n        \"Investigate whether inferred_gender adds noise or instability.\"\n    )\nelif e03_p12 < hybrid_p12:\n    e03_decision = (\n        \"E03 is worse than E02. inferred_gender may contain useful signal, \"\n        \"but this does not prove the full hybrid is better than ALS.\"\n    )\nelse:\n    e03_decision = (\n        \"E03 equals E02. inferred_gender did not change Precision@12 in this run.\"\n    )\n\nprint(\"\\n=== Safe E03 interpretation ===\")\nprint(e03_decision)\n\nprint(\"\\nObjects to log:\")\nprint(\"- e03_summary\")\nprint(\"- fi_e03\")\nprint(\"- alpha_e03, beta_e03, intercept_e03\")\nprint(\"- auc_score_dem_e03, auc_hybrid_e03\")\n\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-28T19:22:22.961529Z","iopub.execute_input":"2026-05-28T19:22:22.963756Z","iopub.status.idle":"2026-05-28T19:23:29.849659Z","shell.execute_reply.started":"2026-05-28T19:22:22.963670Z","shell.execute_reply":"2026-05-28T19:23:29.848703Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# NEW CELL D - E05: Article metadata only\n#\n# WHERE TO PLACE:\n# Put this NEW code cell immediately AFTER:\n# NEW CELL C - E03: Hybrid without inferred_gender\n#\n# PURPOSE:\n# Test how much of the hybrid improvement comes from article metadata only.\n#\n# E05 removes customer/context features:\n# - no age\n# - no postal_code\n# - no inferred_gender\n#\n# It keeps only item/article metadata:\n# - product_type_name\n# - graphical_appearance_name\n# - colour_group_name\n# - perceived_colour_value_name\n# - index_name\n# - index_group_name\n#\n# ALS candidates and Score_CF stay unchanged.\n# ============================================================\n\nimport pandas as pd\nimport numpy as np\nimport gc\n\nfrom catboost import CatBoostClassifier\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import roc_auc_score\n\nprint(\"NEW CELL D: E05 - Article metadata only\")\n\n# ------------------------------------------------------------\n# 1. Safety checks\n# ------------------------------------------------------------\n\nrequired_objects = [\n    \"catboost_train_df\",\n    \"als_candidates_df\",\n    \"baseline_p12\",\n    \"hybrid_p12\"\n]\n\nmissing_objects = [obj for obj in required_objects if obj not in globals()]\n\nif missing_objects:\n    raise RuntimeError(\n        f\"Missing required objects: {missing_objects}. \"\n        \"Run cells up to E03 first.\"\n    )\n\nif \"e03_p12\" not in globals():\n    print(\"Warning: e03_p12 not found. E05 will run, but E03 comparison will be skipped.\")\n    e03_p12_for_compare = np.nan\nelse:\n    e03_p12_for_compare = e03_p12\n\narticle_features_e05 = [\n    \"product_type_name\",\n    \"graphical_appearance_name\",\n    \"colour_group_name\",\n    \"perceived_colour_value_name\",\n    \"index_name\",\n    \"index_group_name\"\n]\n\nrequired_train_cols = [\"target\"] + article_features_e05\nmissing_train_cols = [col for col in required_train_cols if col not in catboost_train_df.columns]\n\nif missing_train_cols:\n    raise RuntimeError(f\"catboost_train_df is missing columns: {missing_train_cols}\")\n\nrequired_candidate_cols = [\"user_idx\", \"item_idx\", \"target\", \"Score_CF\"] + article_features_e05\nmissing_candidate_cols = [col for col in required_candidate_cols if col not in als_candidates_df.columns]\n\nif missing_candidate_cols:\n    raise RuntimeError(f\"als_candidates_df is missing columns: {missing_candidate_cols}\")\n\n# ------------------------------------------------------------\n# 2. Define E05 feature set\n# ------------------------------------------------------------\n\nX_e05 = catboost_train_df[article_features_e05].copy()\ny_e05 = catboost_train_df[\"target\"].copy()\n\ncat_features_e05 = article_features_e05.copy()\n\nprint(\"\\nE05 feature set:\")\nprint(list(X_e05.columns))\n\nblocked_features = [\"age\", \"postal_code\", \"inferred_gender\"]\nleaked_features = [col for col in blocked_features if col in X_e05.columns]\n\nif leaked_features:\n    raise RuntimeError(f\"E05 failed: customer/context features still present: {leaked_features}\")\n\n# ------------------------------------------------------------\n# 3. Train CatBoost for E05\n# ------------------------------------------------------------\n\nprint(\"\\nTraining CatBoost E05...\")\n\nmodel_cb_e05 = CatBoostClassifier(\n    iterations=100,\n    learning_rate=0.1,\n    depth=6,\n    loss_function=\"Logloss\",\n    eval_metric=\"AUC\",\n    random_seed=42,\n    verbose=20\n)\n\nmodel_cb_e05.fit(X_e05, y_e05, cat_features=cat_features_e05)\n\n# ------------------------------------------------------------\n# 4. Generate E05 CatBoost score on same ALS candidates\n# ------------------------------------------------------------\n\nprint(\"\\nScoring same ALS candidates for E05...\")\n\nX_test_e05 = als_candidates_df[article_features_e05].copy()\n\nals_candidates_df[\"Score_Dem_E05\"] = model_cb_e05.predict_proba(X_test_e05)[:, 1]\n\n# ------------------------------------------------------------\n# 5. Train Logistic Regression fusion for E05\n# ------------------------------------------------------------\n\nprint(\"\\nTraining Logistic Regression fusion for E05...\")\n\nscaler_e05 = StandardScaler()\n\nscaled_e05 = scaler_e05.fit_transform(\n    als_candidates_df[[\"Score_CF\", \"Score_Dem_E05\"]]\n)\n\nals_candidates_df[\"Scaled_CF_E05\"] = scaled_e05[:, 0]\nals_candidates_df[\"Scaled_Dem_E05\"] = scaled_e05[:, 1]\n\nX_meta_e05 = als_candidates_df[[\"Scaled_CF_E05\", \"Scaled_Dem_E05\"]]\ny_meta_e05 = als_candidates_df[\"target\"]\n\nmodel_lr_e05 = LogisticRegression(class_weight=\"balanced\", random_state=42)\nmodel_lr_e05.fit(X_meta_e05, y_meta_e05)\n\nalpha_e05 = model_lr_e05.coef_[0][0]\nbeta_e05 = model_lr_e05.coef_[0][1]\nintercept_e05 = model_lr_e05.intercept_[0]\n\nals_candidates_df[\"Hybrid_Score_E05\"] = (\n    alpha_e05 * als_candidates_df[\"Scaled_CF_E05\"] +\n    beta_e05 * als_candidates_df[\"Scaled_Dem_E05\"] +\n    intercept_e05\n)\n\nprint(\"\\n=== E05 Logistic Regression coefficients ===\")\nprint(f\"Alpha / ALS Score_CF: {alpha_e05:.4f}\")\nprint(f\"Beta / CatBoost Score_Dem_E05: {beta_e05:.4f}\")\nprint(f\"Intercept: {intercept_e05:.4f}\")\n\n# ------------------------------------------------------------\n# 6. Evaluation helpers\n# ------------------------------------------------------------\n\ndef precision_at_12_e05(df, score_col):\n    top_12 = (\n        df.sort_values([\"user_idx\", score_col], ascending=[True, False])\n          .groupby(\"user_idx\")\n          .head(12)\n    )\n    user_hits = top_12.groupby(\"user_idx\")[\"target\"].sum()\n    return (user_hits / 12).mean()\n\ndef safe_auc_e05(y_true, score):\n    if len(np.unique(y_true)) < 2:\n        return np.nan\n    return roc_auc_score(y_true, score)\n\n# ------------------------------------------------------------\n# 7. E05 metrics\n# ------------------------------------------------------------\n\ne05_p12 = precision_at_12_e05(als_candidates_df, \"Hybrid_Score_E05\")\n\ne05_uplift_vs_e01 = ((e05_p12 - baseline_p12) / baseline_p12) * 100\ne05_delta_vs_e02 = ((e05_p12 - hybrid_p12) / hybrid_p12) * 100\n\nif pd.isna(e03_p12_for_compare):\n    e05_delta_vs_e03 = np.nan\nelse:\n    e05_delta_vs_e03 = ((e05_p12 - e03_p12_for_compare) / e03_p12_for_compare) * 100\n\nauc_score_dem_e05 = safe_auc_e05(\n    als_candidates_df[\"target\"],\n    als_candidates_df[\"Score_Dem_E05\"]\n)\n\nauc_hybrid_e05 = safe_auc_e05(\n    als_candidates_df[\"target\"],\n    als_candidates_df[\"Hybrid_Score_E05\"]\n)\n\nprint(\"\\n=== E05 Precision@12 results ===\")\nprint(f\"E01 ALS baseline Precision@12: {baseline_p12:.6f}\")\nprint(f\"E02 full hybrid Precision@12: {hybrid_p12:.6f}\")\n\nif not pd.isna(e03_p12_for_compare):\n    print(f\"E03 hybrid without inferred_gender Precision@12: {e03_p12_for_compare:.6f}\")\n\nprint(f\"E05 article metadata only Precision@12: {e05_p12:.6f}\")\nprint(f\"E05 uplift vs E01: {e05_uplift_vs_e01:.2f}%\")\nprint(f\"E05 delta vs E02: {e05_delta_vs_e02:.2f}%\")\n\nif not pd.isna(e05_delta_vs_e03):\n    print(f\"E05 delta vs E03: {e05_delta_vs_e03:.2f}%\")\n\nprint(\"\\n=== AUC diagnostic ===\")\nprint(f\"E05 CatBoost Score_Dem_E05 AUC: {auc_score_dem_e05:.6f}\")\nprint(f\"E05 Hybrid_Score_E05 AUC: {auc_hybrid_e05:.6f}\")\n\n# ------------------------------------------------------------\n# 8. Feature importance for E05\n# ------------------------------------------------------------\n\nfi_e05 = (\n    pd.DataFrame({\n        \"feature\": list(model_cb_e05.feature_names_),\n        \"importance\": model_cb_e05.get_feature_importance()\n    })\n    .sort_values(\"importance\", ascending=False)\n    .reset_index(drop=True)\n)\n\nprint(\"\\n=== E05 CatBoost feature importance ===\")\ndisplay(fi_e05)\n\n# Confirm customer/context features are absent\nprint(\"\\nFeature check:\")\nfor feature in blocked_features:\n    print(f\"{feature} in E05 features: {feature in list(X_e05.columns)}\")\n\n# ------------------------------------------------------------\n# 9. E05 summary table\n# ------------------------------------------------------------\n\nsummary_rows = [\n    {\n        \"experiment_id\": \"E01\",\n        \"name\": \"ALS baseline only\",\n        \"Precision@12\": baseline_p12,\n        \"uplift_vs_E01_%\": 0.0,\n        \"delta_vs_E02_%\": np.nan,\n        \"delta_vs_E03_%\": np.nan,\n        \"notes\": \"Locked baseline\"\n    },\n    {\n        \"experiment_id\": \"E02\",\n        \"name\": \"Full current hybrid\",\n        \"Precision@12\": hybrid_p12,\n        \"uplift_vs_E01_%\": ((hybrid_p12 - baseline_p12) / baseline_p12) * 100,\n        \"delta_vs_E02_%\": 0.0,\n        \"delta_vs_E03_%\": np.nan,\n        \"notes\": \"Full feature set with train-safe inferred_gender\"\n    }\n]\n\nif not pd.isna(e03_p12_for_compare):\n    summary_rows.append({\n        \"experiment_id\": \"E03\",\n        \"name\": \"Hybrid without inferred_gender\",\n        \"Precision@12\": e03_p12_for_compare,\n        \"uplift_vs_E01_%\": ((e03_p12_for_compare - baseline_p12) / baseline_p12) * 100,\n        \"delta_vs_E02_%\": ((e03_p12_for_compare - hybrid_p12) / hybrid_p12) * 100,\n        \"delta_vs_E03_%\": 0.0,\n        \"notes\": \"Removed inferred_gender only\"\n    })\n\nsummary_rows.append({\n    \"experiment_id\": \"E05\",\n    \"name\": \"Article metadata only\",\n    \"Precision@12\": e05_p12,\n    \"uplift_vs_E01_%\": e05_uplift_vs_e01,\n    \"delta_vs_E02_%\": e05_delta_vs_e02,\n    \"delta_vs_E03_%\": e05_delta_vs_e03,\n    \"notes\": \"No customer/context features: no age, postal_code, inferred_gender\"\n})\n\ne05_summary = pd.DataFrame(summary_rows)\n\nprint(\"\\n=== E05 comparison summary ===\")\ndisplay(e05_summary)\n\n# ------------------------------------------------------------\n# 10. Safe decision line\n# ------------------------------------------------------------\n\nif e05_p12 > baseline_p12:\n    e05_vs_baseline_text = \"E05 beats the ALS baseline in this run.\"\nelif e05_p12 < baseline_p12:\n    e05_vs_baseline_text = \"E05 is below the ALS baseline in this run.\"\nelse:\n    e05_vs_baseline_text = \"E05 equals the ALS baseline in this run.\"\n\nif not pd.isna(e03_p12_for_compare):\n    if e05_p12 > e03_p12_for_compare:\n        e05_vs_e03_text = \"Article metadata only is better than E03, so customer features may not be needed in this setup.\"\n    elif e05_p12 < e03_p12_for_compare:\n        e05_vs_e03_text = \"E03 is better than article metadata only, so age/postal_code may still add useful signal.\"\n    else:\n        e05_vs_e03_text = \"E05 and E03 are equal in Precision@12.\"\nelse:\n    e05_vs_e03_text = \"E03 comparison unavailable.\"\n\ne05_decision = (\n    f\"{e05_vs_baseline_text} {e05_vs_e03_text} \"\n    \"Use this to decide whether the current gain comes mostly from product metadata \"\n    \"or from the combination of metadata with customer context.\"\n)\n\nprint(\"\\n=== Safe E05 interpretation ===\")\nprint(e05_decision)\n\nprint(\"\\nObjects to log:\")\nprint(\"- e05_summary\")\nprint(\"- fi_e05\")\nprint(\"- alpha_e05, beta_e05, intercept_e05\")\nprint(\"- auc_score_dem_e05, auc_hybrid_e05\")\nprint(\"- e05_decision\")\n\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-28T19:37:30.725270Z","iopub.execute_input":"2026-05-28T19:37:30.726514Z","iopub.status.idle":"2026-05-28T19:38:30.507835Z","shell.execute_reply.started":"2026-05-28T19:37:30.726422Z","shell.execute_reply":"2026-05-28T19:38:30.506263Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# NEW CELL E - E04: Customer context only\n#\n# WHERE TO PLACE:\n# Put this NEW code cell immediately AFTER:\n# NEW CELL D - E05: Article metadata only\n#\n# PURPOSE:\n# Test whether customer/context features alone can improve ranking.\n#\n# E04 removes article metadata:\n# - no product_type_name\n# - no graphical_appearance_name\n# - no colour_group_name\n# - no perceived_colour_value_name\n# - no index_name\n# - no index_group_name\n#\n# E04 also excludes inferred_gender because E03 showed it hurt this setup.\n#\n# E04 keeps only:\n# - age\n# - postal_code\n#\n# Important:\n# postal_code is a hashed categorical proxy only.\n# It is NOT a real geographic feature.\n#\n# ALS candidates and Score_CF stay unchanged.\n# ============================================================\n\nimport pandas as pd\nimport numpy as np\nimport gc\n\nfrom catboost import CatBoostClassifier\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import roc_auc_score\n\nprint(\"NEW CELL E: E04 - Customer context only\")\n\n# ------------------------------------------------------------\n# 1. Safety checks\n# ------------------------------------------------------------\n\nrequired_objects = [\n    \"catboost_train_df\",\n    \"als_candidates_df\",\n    \"baseline_p12\",\n    \"hybrid_p12\"\n]\n\nmissing_objects = [obj for obj in required_objects if obj not in globals()]\n\nif missing_objects:\n    raise RuntimeError(\n        f\"Missing required objects: {missing_objects}. \"\n        \"Run cells up to E05 first.\"\n    )\n\nif \"e03_p12\" not in globals():\n    print(\"Warning: e03_p12 not found. E04 will run, but E03 comparison will be skipped.\")\n    e03_p12_for_compare = np.nan\nelse:\n    e03_p12_for_compare = e03_p12\n\nif \"e05_p12\" not in globals():\n    print(\"Warning: e05_p12 not found. E04 will run, but E05 comparison will be skipped.\")\n    e05_p12_for_compare = np.nan\nelse:\n    e05_p12_for_compare = e05_p12\n\ncustomer_features_e04 = [\n    \"age\",\n    \"postal_code\"\n]\n\nrequired_train_cols = [\"target\"] + customer_features_e04\nmissing_train_cols = [col for col in required_train_cols if col not in catboost_train_df.columns]\n\nif missing_train_cols:\n    raise RuntimeError(f\"catboost_train_df is missing columns: {missing_train_cols}\")\n\nrequired_candidate_cols = [\"user_idx\", \"item_idx\", \"target\", \"Score_CF\"] + customer_features_e04\nmissing_candidate_cols = [col for col in required_candidate_cols if col not in als_candidates_df.columns]\n\nif missing_candidate_cols:\n    raise RuntimeError(f\"als_candidates_df is missing columns: {missing_candidate_cols}\")\n\n# ------------------------------------------------------------\n# 2. Define E04 feature set\n# ------------------------------------------------------------\n\nX_e04 = catboost_train_df[customer_features_e04].copy()\ny_e04 = catboost_train_df[\"target\"].copy()\n\nX_test_e04 = als_candidates_df[customer_features_e04].copy()\n\n# Keep handling minimal and explicit.\n# age = numeric, postal_code = categorical proxy.\nage_median_e04 = X_e04[\"age\"].median()\n\nX_e04[\"age\"] = pd.to_numeric(X_e04[\"age\"], errors=\"coerce\").fillna(age_median_e04)\nX_test_e04[\"age\"] = pd.to_numeric(X_test_e04[\"age\"], errors=\"coerce\").fillna(age_median_e04)\n\nX_e04[\"postal_code\"] = X_e04[\"postal_code\"].fillna(\"Unknown\").astype(str)\nX_test_e04[\"postal_code\"] = X_test_e04[\"postal_code\"].fillna(\"Unknown\").astype(str)\n\ncat_features_e04 = [\"postal_code\"]\n\nprint(\"\\nE04 feature set:\")\nprint(list(X_e04.columns))\n\nblocked_article_features = [\n    \"product_type_name\",\n    \"graphical_appearance_name\",\n    \"colour_group_name\",\n    \"perceived_colour_value_name\",\n    \"index_name\",\n    \"index_group_name\"\n]\n\nblocked_features = blocked_article_features + [\"inferred_gender\"]\n\nleaked_features = [col for col in blocked_features if col in X_e04.columns]\n\nif leaked_features:\n    raise RuntimeError(f\"E04 failed: blocked features still present: {leaked_features}\")\n\n# ------------------------------------------------------------\n# 3. Train CatBoost for E04\n# ------------------------------------------------------------\n\nprint(\"\\nTraining CatBoost E04...\")\n\nmodel_cb_e04 = CatBoostClassifier(\n    iterations=100,\n    learning_rate=0.1,\n    depth=6,\n    loss_function=\"Logloss\",\n    eval_metric=\"AUC\",\n    random_seed=42,\n    verbose=20\n)\n\nmodel_cb_e04.fit(X_e04, y_e04, cat_features=cat_features_e04)\n\n# ------------------------------------------------------------\n# 4. Generate E04 CatBoost score on same ALS candidates\n# ------------------------------------------------------------\n\nprint(\"\\nScoring same ALS candidates for E04...\")\n\nals_candidates_df[\"Score_Dem_E04\"] = model_cb_e04.predict_proba(X_test_e04)[:, 1]\n\n# ------------------------------------------------------------\n# 5. Train Logistic Regression fusion for E04\n# ------------------------------------------------------------\n\nprint(\"\\nTraining Logistic Regression fusion for E04...\")\n\nscaler_e04 = StandardScaler()\n\nscaled_e04 = scaler_e04.fit_transform(\n    als_candidates_df[[\"Score_CF\", \"Score_Dem_E04\"]]\n)\n\nals_candidates_df[\"Scaled_CF_E04\"] = scaled_e04[:, 0]\nals_candidates_df[\"Scaled_Dem_E04\"] = scaled_e04[:, 1]\n\nX_meta_e04 = als_candidates_df[[\"Scaled_CF_E04\", \"Scaled_Dem_E04\"]]\ny_meta_e04 = als_candidates_df[\"target\"]\n\nmodel_lr_e04 = LogisticRegression(class_weight=\"balanced\", random_state=42)\nmodel_lr_e04.fit(X_meta_e04, y_meta_e04)\n\nalpha_e04 = model_lr_e04.coef_[0][0]\nbeta_e04 = model_lr_e04.coef_[0][1]\nintercept_e04 = model_lr_e04.intercept_[0]\n\nals_candidates_df[\"Hybrid_Score_E04\"] = (\n    alpha_e04 * als_candidates_df[\"Scaled_CF_E04\"] +\n    beta_e04 * als_candidates_df[\"Scaled_Dem_E04\"] +\n    intercept_e04\n)\n\nprint(\"\\n=== E04 Logistic Regression coefficients ===\")\nprint(f\"Alpha / ALS Score_CF: {alpha_e04:.4f}\")\nprint(f\"Beta / CatBoost Score_Dem_E04: {beta_e04:.4f}\")\nprint(f\"Intercept: {intercept_e04:.4f}\")\n\n# ------------------------------------------------------------\n# 6. Evaluation helpers\n# ------------------------------------------------------------\n\ndef precision_at_12_e04(df, score_col):\n    top_12 = (\n        df.sort_values([\"user_idx\", score_col], ascending=[True, False])\n          .groupby(\"user_idx\")\n          .head(12)\n    )\n    user_hits = top_12.groupby(\"user_idx\")[\"target\"].sum()\n    return (user_hits / 12).mean()\n\ndef safe_auc_e04(y_true, score):\n    if len(np.unique(y_true)) < 2:\n        return np.nan\n    return roc_auc_score(y_true, score)\n\n# ------------------------------------------------------------\n# 7. E04 metrics\n# ------------------------------------------------------------\n\ne04_p12 = precision_at_12_e04(als_candidates_df, \"Hybrid_Score_E04\")\n\ne04_uplift_vs_e01 = ((e04_p12 - baseline_p12) / baseline_p12) * 100\ne04_delta_vs_e02 = ((e04_p12 - hybrid_p12) / hybrid_p12) * 100\n\nif pd.isna(e03_p12_for_compare):\n    e04_delta_vs_e03 = np.nan\nelse:\n    e04_delta_vs_e03 = ((e04_p12 - e03_p12_for_compare) / e03_p12_for_compare) * 100\n\nif pd.isna(e05_p12_for_compare):\n    e04_delta_vs_e05 = np.nan\nelse:\n    e04_delta_vs_e05 = ((e04_p12 - e05_p12_for_compare) / e05_p12_for_compare) * 100\n\nauc_score_dem_e04 = safe_auc_e04(\n    als_candidates_df[\"target\"],\n    als_candidates_df[\"Score_Dem_E04\"]\n)\n\nauc_hybrid_e04 = safe_auc_e04(\n    als_candidates_df[\"target\"],\n    als_candidates_df[\"Hybrid_Score_E04\"]\n)\n\nprint(\"\\n=== E04 Precision@12 results ===\")\nprint(f\"E01 ALS baseline Precision@12: {baseline_p12:.6f}\")\nprint(f\"E02 full hybrid Precision@12: {hybrid_p12:.6f}\")\n\nif not pd.isna(e03_p12_for_compare):\n    print(f\"E03 hybrid without inferred_gender Precision@12: {e03_p12_for_compare:.6f}\")\n\nif not pd.isna(e05_p12_for_compare):\n    print(f\"E05 article metadata only Precision@12: {e05_p12_for_compare:.6f}\")\n\nprint(f\"E04 customer context only Precision@12: {e04_p12:.6f}\")\nprint(f\"E04 uplift vs E01: {e04_uplift_vs_e01:.2f}%\")\nprint(f\"E04 delta vs E02: {e04_delta_vs_e02:.2f}%\")\n\nif not pd.isna(e04_delta_vs_e03):\n    print(f\"E04 delta vs E03: {e04_delta_vs_e03:.2f}%\")\n\nif not pd.isna(e04_delta_vs_e05):\n    print(f\"E04 delta vs E05: {e04_delta_vs_e05:.2f}%\")\n\nprint(\"\\n=== AUC diagnostic ===\")\nprint(f\"E04 CatBoost Score_Dem_E04 AUC: {auc_score_dem_e04:.6f}\")\nprint(f\"E04 Hybrid_Score_E04 AUC: {auc_hybrid_e04:.6f}\")\n\n# ------------------------------------------------------------\n# 8. Feature importance for E04\n# ------------------------------------------------------------\n\nfi_e04 = (\n    pd.DataFrame({\n        \"feature\": list(model_cb_e04.feature_names_),\n        \"importance\": model_cb_e04.get_feature_importance()\n    })\n    .sort_values(\"importance\", ascending=False)\n    .reset_index(drop=True)\n)\n\nprint(\"\\n=== E04 CatBoost feature importance ===\")\ndisplay(fi_e04)\n\n# Confirm blocked features are absent\nprint(\"\\nFeature check:\")\nfor feature in blocked_features:\n    print(f\"{feature} in E04 features: {feature in list(X_e04.columns)}\")\n\nprint(\"\\nPostal code note:\")\nprint(\"postal_code is used only as a hashed categorical proxy, not as geography.\")\n\n# ------------------------------------------------------------\n# 9. E04 summary table\n# ------------------------------------------------------------\n\nsummary_rows = [\n    {\n        \"experiment_id\": \"E01\",\n        \"name\": \"ALS baseline only\",\n        \"Precision@12\": baseline_p12,\n        \"uplift_vs_E01_%\": 0.0,\n        \"delta_vs_E02_%\": np.nan,\n        \"delta_vs_E03_%\": np.nan,\n        \"delta_vs_E05_%\": np.nan,\n        \"notes\": \"Locked baseline\"\n    },\n    {\n        \"experiment_id\": \"E02\",\n        \"name\": \"Full current hybrid\",\n        \"Precision@12\": hybrid_p12,\n        \"uplift_vs_E01_%\": ((hybrid_p12 - baseline_p12) / baseline_p12) * 100,\n        \"delta_vs_E02_%\": 0.0,\n        \"delta_vs_E03_%\": np.nan,\n        \"delta_vs_E05_%\": np.nan,\n        \"notes\": \"Full feature set with train-safe inferred_gender\"\n    }\n]\n\nif not pd.isna(e03_p12_for_compare):\n    summary_rows.append({\n        \"experiment_id\": \"E03\",\n        \"name\": \"Hybrid without inferred_gender\",\n        \"Precision@12\": e03_p12_for_compare,\n        \"uplift_vs_E01_%\": ((e03_p12_for_compare - baseline_p12) / baseline_p12) * 100,\n        \"delta_vs_E02_%\": ((e03_p12_for_compare - hybrid_p12) / hybrid_p12) * 100,\n        \"delta_vs_E03_%\": 0.0,\n        \"delta_vs_E05_%\": np.nan,\n        \"notes\": \"age + postal_code + article metadata, no inferred_gender\"\n    })\n\nsummary_rows.append({\n    \"experiment_id\": \"E04\",\n    \"name\": \"Customer context only\",\n    \"Precision@12\": e04_p12,\n    \"uplift_vs_E01_%\": e04_uplift_vs_e01,\n    \"delta_vs_E02_%\": e04_delta_vs_e02,\n    \"delta_vs_E03_%\": e04_delta_vs_e03,\n    \"delta_vs_E05_%\": e04_delta_vs_e05,\n    \"notes\": \"age + postal_code only; no inferred_gender; no article metadata\"\n})\n\nif not pd.isna(e05_p12_for_compare):\n    summary_rows.append({\n        \"experiment_id\": \"E05\",\n        \"name\": \"Article metadata only\",\n        \"Precision@12\": e05_p12_for_compare,\n        \"uplift_vs_E01_%\": ((e05_p12_for_compare - baseline_p12) / baseline_p12) * 100,\n        \"delta_vs_E02_%\": ((e05_p12_for_compare - hybrid_p12) / hybrid_p12) * 100,\n        \"delta_vs_E03_%\": (\n            ((e05_p12_for_compare - e03_p12_for_compare) / e03_p12_for_compare) * 100\n            if not pd.isna(e03_p12_for_compare)\n            else np.nan\n        ),\n        \"delta_vs_E05_%\": 0.0,\n        \"notes\": \"article metadata only; no customer/context features\"\n    })\n\ne04_summary = pd.DataFrame(summary_rows)\n\nprint(\"\\n=== E04 comparison summary ===\")\ndisplay(e04_summary)\n\n# ------------------------------------------------------------\n# 10. Safe decision line\n# ------------------------------------------------------------\n\nif e04_p12 > baseline_p12:\n    e04_vs_baseline_text = \"E04 beats the ALS baseline in this run.\"\nelif e04_p12 < baseline_p12:\n    e04_vs_baseline_text = \"E04 is below the ALS baseline in this run.\"\nelse:\n    e04_vs_baseline_text = \"E04 equals the ALS baseline in this run.\"\n\nif not pd.isna(e05_p12_for_compare):\n    if e04_p12 > e05_p12_for_compare:\n        e04_vs_e05_text = \"Customer context only is stronger than article metadata only in this run.\"\n    elif e04_p12 < e05_p12_for_compare:\n        e04_vs_e05_text = \"Customer context only is weaker than article metadata only in this run.\"\n    else:\n        e04_vs_e05_text = \"Customer context only and article metadata only are equal in this run.\"\nelse:\n    e04_vs_e05_text = \"E05 comparison unavailable.\"\n\nif not pd.isna(e03_p12_for_compare):\n    if e04_p12 > e03_p12_for_compare:\n        e04_vs_e03_text = \"E04 is stronger than the current best E03.\"\n    elif e04_p12 < e03_p12_for_compare:\n        e04_vs_e03_text = \"E04 is weaker than E03, so customer context alone does not explain the best result.\"\n    else:\n        e04_vs_e03_text = \"E04 equals E03.\"\nelse:\n    e04_vs_e03_text = \"E03 comparison unavailable.\"\n\ne04_decision = (\n    f\"{e04_vs_baseline_text} \"\n    f\"{e04_vs_e05_text} \"\n    f\"{e04_vs_e03_text} \"\n    \"Use this to isolate whether age and hashed postal_code have standalone value, \"\n    \"or whether they only help when combined with article metadata.\"\n)\n\nprint(\"\\n=== Safe E04 interpretation ===\")\nprint(e04_decision)\n\nprint(\"\\nObjects to log:\")\nprint(\"- e04_summary\")\nprint(\"- fi_e04\")\nprint(\"- alpha_e04, beta_e04, intercept_e04\")\nprint(\"- auc_score_dem_e04, auc_hybrid_e04\")\nprint(\"- e04_decision\")\n\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-28T19:44:09.607264Z","iopub.execute_input":"2026-05-28T19:44:09.607778Z","iopub.status.idle":"2026-05-28T19:44:45.592544Z","shell.execute_reply.started":"2026-05-28T19:44:09.607741Z","shell.execute_reply":"2026-05-28T19:44:45.591517Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# NEW CELL F - E06: Top feature subset\n#\n# WHERE TO PLACE:\n# Put this NEW code cell immediately AFTER:\n# NEW CELL E - E04: Customer context only\n#\n# PURPOSE:\n# Test whether a smaller feature subset can match or beat E03.\n#\n# Selected from E03/E05 feature importance:\n# - product_type_name\n# - index_name\n# - postal_code\n# - colour_group_name\n#\n# Excluded:\n# - inferred_gender: hurt E02\n# - age: weak in E04\n# - graphical_appearance_name: useful but below top 4\n# - perceived_colour_value_name: lower importance\n# - index_group_name: useful, but may overlap with index_name/product_type_name\n#\n# Important:\n# postal_code is a hashed categorical proxy only.\n# It is NOT geography.\n#\n# ALS candidates and Score_CF stay unchanged.\n# ============================================================\n\nimport pandas as pd\nimport numpy as np\nimport gc\n\nfrom catboost import CatBoostClassifier\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import roc_auc_score\n\nprint(\"NEW CELL F: E06 - Top feature subset\")\n\n# ------------------------------------------------------------\n# 1. Safety checks\n# ------------------------------------------------------------\n\nrequired_objects = [\n    \"catboost_train_df\",\n    \"als_candidates_df\",\n    \"baseline_p12\",\n    \"hybrid_p12\"\n]\n\nmissing_objects = [obj for obj in required_objects if obj not in globals()]\n\nif missing_objects:\n    raise RuntimeError(\n        f\"Missing required objects: {missing_objects}. \"\n        \"Run cells up to E04 first.\"\n    )\n\ne03_p12_for_compare = e03_p12 if \"e03_p12\" in globals() else np.nan\ne04_p12_for_compare = e04_p12 if \"e04_p12\" in globals() else np.nan\ne05_p12_for_compare = e05_p12 if \"e05_p12\" in globals() else np.nan\n\n# ------------------------------------------------------------\n# 2. Define E06 feature set\n# ------------------------------------------------------------\n\ntop_features_e06 = [\n    \"product_type_name\",\n    \"index_name\",\n    \"postal_code\",\n    \"colour_group_name\"\n]\n\nrequired_train_cols = [\"target\"] + top_features_e06\nmissing_train_cols = [col for col in required_train_cols if col not in catboost_train_df.columns]\n\nif missing_train_cols:\n    raise RuntimeError(f\"catboost_train_df is missing columns: {missing_train_cols}\")\n\nrequired_candidate_cols = [\"user_idx\", \"item_idx\", \"target\", \"Score_CF\"] + top_features_e06\nmissing_candidate_cols = [col for col in required_candidate_cols if col not in als_candidates_df.columns]\n\nif missing_candidate_cols:\n    raise RuntimeError(f\"als_candidates_df is missing columns: {missing_candidate_cols}\")\n\nX_e06 = catboost_train_df[top_features_e06].copy()\ny_e06 = catboost_train_df[\"target\"].copy()\n\nX_test_e06 = als_candidates_df[top_features_e06].copy()\n\n# All E06 features are categorical for CatBoost.\ncat_features_e06 = top_features_e06.copy()\n\nfor col in cat_features_e06:\n    X_e06[col] = X_e06[col].fillna(\"Unknown\").astype(str)\n    X_test_e06[col] = X_test_e06[col].fillna(\"Unknown\").astype(str)\n\nprint(\"\\nE06 feature set:\")\nprint(list(X_e06.columns))\n\nblocked_features = [\n    \"age\",\n    \"inferred_gender\",\n    \"graphical_appearance_name\",\n    \"perceived_colour_value_name\",\n    \"index_group_name\"\n]\n\nleaked_features = [col for col in blocked_features if col in X_e06.columns]\n\nif leaked_features:\n    raise RuntimeError(f\"E06 failed: blocked features still present: {leaked_features}\")\n\nprint(\"\\nFeature check:\")\nfor feature in blocked_features:\n    print(f\"{feature} in E06 features: {feature in list(X_e06.columns)}\")\n\nprint(\"\\nPostal code note:\")\nprint(\"postal_code is used only as a hashed categorical proxy, not as geography.\")\n\n# ------------------------------------------------------------\n# 3. Train CatBoost for E06\n# ------------------------------------------------------------\n\nprint(\"\\nTraining CatBoost E06...\")\n\nmodel_cb_e06 = CatBoostClassifier(\n    iterations=100,\n    learning_rate=0.1,\n    depth=6,\n    loss_function=\"Logloss\",\n    eval_metric=\"AUC\",\n    random_seed=42,\n    verbose=20\n)\n\nmodel_cb_e06.fit(X_e06, y_e06, cat_features=cat_features_e06)\n\n# ------------------------------------------------------------\n# 4. Generate E06 CatBoost score on same ALS candidates\n# ------------------------------------------------------------\n\nprint(\"\\nScoring same ALS candidates for E06...\")\n\nals_candidates_df[\"Score_Dem_E06\"] = model_cb_e06.predict_proba(X_test_e06)[:, 1]\n\n# ------------------------------------------------------------\n# 5. Train Logistic Regression fusion for E06\n# ------------------------------------------------------------\n\nprint(\"\\nTraining Logistic Regression fusion for E06...\")\n\nscaler_e06 = StandardScaler()\n\nscaled_e06 = scaler_e06.fit_transform(\n    als_candidates_df[[\"Score_CF\", \"Score_Dem_E06\"]]\n)\n\nals_candidates_df[\"Scaled_CF_E06\"] = scaled_e06[:, 0]\nals_candidates_df[\"Scaled_Dem_E06\"] = scaled_e06[:, 1]\n\nX_meta_e06 = als_candidates_df[[\"Scaled_CF_E06\", \"Scaled_Dem_E06\"]]\ny_meta_e06 = als_candidates_df[\"target\"]\n\nmodel_lr_e06 = LogisticRegression(class_weight=\"balanced\", random_state=42)\nmodel_lr_e06.fit(X_meta_e06, y_meta_e06)\n\nalpha_e06 = model_lr_e06.coef_[0][0]\nbeta_e06 = model_lr_e06.coef_[0][1]\nintercept_e06 = model_lr_e06.intercept_[0]\n\nals_candidates_df[\"Hybrid_Score_E06\"] = (\n    alpha_e06 * als_candidates_df[\"Scaled_CF_E06\"] +\n    beta_e06 * als_candidates_df[\"Scaled_Dem_E06\"] +\n    intercept_e06\n)\n\nprint(\"\\n=== E06 Logistic Regression coefficients ===\")\nprint(f\"Alpha / ALS Score_CF: {alpha_e06:.4f}\")\nprint(f\"Beta / CatBoost Score_Dem_E06: {beta_e06:.4f}\")\nprint(f\"Intercept: {intercept_e06:.4f}\")\n\n# ------------------------------------------------------------\n# 6. Evaluation helpers\n# ------------------------------------------------------------\n\ndef precision_at_12_e06(df, score_col):\n    top_12 = (\n        df.sort_values([\"user_idx\", score_col], ascending=[True, False])\n          .groupby(\"user_idx\")\n          .head(12)\n    )\n    user_hits = top_12.groupby(\"user_idx\")[\"target\"].sum()\n    return (user_hits / 12).mean()\n\ndef safe_auc_e06(y_true, score):\n    if len(np.unique(y_true)) < 2:\n        return np.nan\n    return roc_auc_score(y_true, score)\n\n# ------------------------------------------------------------\n# 7. E06 metrics\n# ------------------------------------------------------------\n\ne06_p12 = precision_at_12_e06(als_candidates_df, \"Hybrid_Score_E06\")\n\ne06_uplift_vs_e01 = ((e06_p12 - baseline_p12) / baseline_p12) * 100\ne06_delta_vs_e02 = ((e06_p12 - hybrid_p12) / hybrid_p12) * 100\n\nif pd.isna(e03_p12_for_compare):\n    e06_delta_vs_e03 = np.nan\nelse:\n    e06_delta_vs_e03 = ((e06_p12 - e03_p12_for_compare) / e03_p12_for_compare) * 100\n\nif pd.isna(e04_p12_for_compare):\n    e06_delta_vs_e04 = np.nan\nelse:\n    e06_delta_vs_e04 = ((e06_p12 - e04_p12_for_compare) / e04_p12_for_compare) * 100\n\nif pd.isna(e05_p12_for_compare):\n    e06_delta_vs_e05 = np.nan\nelse:\n    e06_delta_vs_e05 = ((e06_p12 - e05_p12_for_compare) / e05_p12_for_compare) * 100\n\nauc_score_dem_e06 = safe_auc_e06(\n    als_candidates_df[\"target\"],\n    als_candidates_df[\"Score_Dem_E06\"]\n)\n\nauc_hybrid_e06 = safe_auc_e06(\n    als_candidates_df[\"target\"],\n    als_candidates_df[\"Hybrid_Score_E06\"]\n)\n\nprint(\"\\n=== E06 Precision@12 results ===\")\nprint(f\"E01 ALS baseline Precision@12: {baseline_p12:.6f}\")\nprint(f\"E02 full hybrid Precision@12: {hybrid_p12:.6f}\")\n\nif not pd.isna(e03_p12_for_compare):\n    print(f\"E03 hybrid without inferred_gender Precision@12: {e03_p12_for_compare:.6f}\")\n\nif not pd.isna(e04_p12_for_compare):\n    print(f\"E04 customer context only Precision@12: {e04_p12_for_compare:.6f}\")\n\nif not pd.isna(e05_p12_for_compare):\n    print(f\"E05 article metadata only Precision@12: {e05_p12_for_compare:.6f}\")\n\nprint(f\"E06 top feature subset Precision@12: {e06_p12:.6f}\")\nprint(f\"E06 uplift vs E01: {e06_uplift_vs_e01:.2f}%\")\nprint(f\"E06 delta vs E02: {e06_delta_vs_e02:.2f}%\")\n\nif not pd.isna(e06_delta_vs_e03):\n    print(f\"E06 delta vs E03: {e06_delta_vs_e03:.2f}%\")\n\nif not pd.isna(e06_delta_vs_e04):\n    print(f\"E06 delta vs E04: {e06_delta_vs_e04:.2f}%\")\n\nif not pd.isna(e06_delta_vs_e05):\n    print(f\"E06 delta vs E05: {e06_delta_vs_e05:.2f}%\")\n\nprint(\"\\n=== AUC diagnostic ===\")\nprint(f\"E06 CatBoost Score_Dem_E06 AUC: {auc_score_dem_e06:.6f}\")\nprint(f\"E06 Hybrid_Score_E06 AUC: {auc_hybrid_e06:.6f}\")\n\n# ------------------------------------------------------------\n# 8. Feature importance for E06\n# ------------------------------------------------------------\n\nfi_e06 = (\n    pd.DataFrame({\n        \"feature\": list(model_cb_e06.feature_names_),\n        \"importance\": model_cb_e06.get_feature_importance()\n    })\n    .sort_values(\"importance\", ascending=False)\n    .reset_index(drop=True)\n)\n\nprint(\"\\n=== E06 CatBoost feature importance ===\")\ndisplay(fi_e06)\n\n# ------------------------------------------------------------\n# 9. E06 comparison summary\n# ------------------------------------------------------------\n\nsummary_rows = [\n    {\n        \"experiment_id\": \"E01\",\n        \"name\": \"ALS baseline only\",\n        \"Precision@12\": baseline_p12,\n        \"uplift_vs_E01_%\": 0.0,\n        \"delta_vs_E03_%\": np.nan,\n        \"delta_vs_E05_%\": np.nan,\n        \"delta_vs_E06_%\": np.nan,\n        \"notes\": \"Locked baseline\"\n    },\n    {\n        \"experiment_id\": \"E02\",\n        \"name\": \"Full current hybrid\",\n        \"Precision@12\": hybrid_p12,\n        \"uplift_vs_E01_%\": ((hybrid_p12 - baseline_p12) / baseline_p12) * 100,\n        \"delta_vs_E03_%\": np.nan,\n        \"delta_vs_E05_%\": np.nan,\n        \"delta_vs_E06_%\": ((hybrid_p12 - e06_p12) / e06_p12) * 100,\n        \"notes\": \"Full feature set with train-safe inferred_gender\"\n    }\n]\n\nif not pd.isna(e03_p12_for_compare):\n    summary_rows.append({\n        \"experiment_id\": \"E03\",\n        \"name\": \"Hybrid without inferred_gender\",\n        \"Precision@12\": e03_p12_for_compare,\n        \"uplift_vs_E01_%\": ((e03_p12_for_compare - baseline_p12) / baseline_p12) * 100,\n        \"delta_vs_E03_%\": 0.0,\n        \"delta_vs_E05_%\": (\n            ((e03_p12_for_compare - e05_p12_for_compare) / e05_p12_for_compare) * 100\n            if not pd.isna(e05_p12_for_compare)\n            else np.nan\n        ),\n        \"delta_vs_E06_%\": ((e03_p12_for_compare - e06_p12) / e06_p12) * 100,\n        \"notes\": \"age + postal_code + article metadata, no inferred_gender\"\n    })\n\nif not pd.isna(e04_p12_for_compare):\n    summary_rows.append({\n        \"experiment_id\": \"E04\",\n        \"name\": \"Customer context only\",\n        \"Precision@12\": e04_p12_for_compare,\n        \"uplift_vs_E01_%\": ((e04_p12_for_compare - baseline_p12) / baseline_p12) * 100,\n        \"delta_vs_E03_%\": (\n            ((e04_p12_for_compare - e03_p12_for_compare) / e03_p12_for_compare) * 100\n            if not pd.isna(e03_p12_for_compare)\n            else np.nan\n        ),\n        \"delta_vs_E05_%\": (\n            ((e04_p12_for_compare - e05_p12_for_compare) / e05_p12_for_compare) * 100\n            if not pd.isna(e05_p12_for_compare)\n            else np.nan\n        ),\n        \"delta_vs_E06_%\": ((e04_p12_for_compare - e06_p12) / e06_p12) * 100,\n        \"notes\": \"age + postal_code only\"\n    })\n\nif not pd.isna(e05_p12_for_compare):\n    summary_rows.append({\n        \"experiment_id\": \"E05\",\n        \"name\": \"Article metadata only\",\n        \"Precision@12\": e05_p12_for_compare,\n        \"uplift_vs_E01_%\": ((e05_p12_for_compare - baseline_p12) / baseline_p12) * 100,\n        \"delta_vs_E03_%\": (\n            ((e05_p12_for_compare - e03_p12_for_compare) / e03_p12_for_compare) * 100\n            if not pd.isna(e03_p12_for_compare)\n            else np.nan\n        ),\n        \"delta_vs_E05_%\": 0.0,\n        \"delta_vs_E06_%\": ((e05_p12_for_compare - e06_p12) / e06_p12) * 100,\n        \"notes\": \"article metadata only\"\n    })\n\nsummary_rows.append({\n    \"experiment_id\": \"E06\",\n    \"name\": \"Top feature subset\",\n    \"Precision@12\": e06_p12,\n    \"uplift_vs_E01_%\": e06_uplift_vs_e01,\n    \"delta_vs_E03_%\": e06_delta_vs_e03,\n    \"delta_vs_E05_%\": e06_delta_vs_e05,\n    \"delta_vs_E06_%\": 0.0,\n    \"notes\": \"product_type_name + index_name + postal_code + colour_group_name\"\n})\n\ne06_summary = pd.DataFrame(summary_rows)\n\nprint(\"\\n=== E06 comparison summary ===\")\ndisplay(e06_summary)\n\n# ------------------------------------------------------------\n# 10. Safe decision line\n# ------------------------------------------------------------\n\nif e06_p12 > baseline_p12:\n    e06_vs_baseline_text = \"E06 beats the ALS baseline in this run.\"\nelif e06_p12 < baseline_p12:\n    e06_vs_baseline_text = \"E06 is below the ALS baseline in this run.\"\nelse:\n    e06_vs_baseline_text = \"E06 equals the ALS baseline in this run.\"\n\nif not pd.isna(e03_p12_for_compare):\n    if e06_p12 > e03_p12_for_compare:\n        e06_vs_e03_text = \"E06 beats E03, so the smaller feature subset becomes the current best candidate.\"\n    elif e06_p12 < e03_p12_for_compare:\n        e06_vs_e03_text = \"E06 is below E03, so the fuller non-gender feature set remains better.\"\n    else:\n        e06_vs_e03_text = \"E06 matches E03, so the smaller feature subset is preferable for simplicity.\"\nelse:\n    e06_vs_e03_text = \"E03 comparison unavailable.\"\n\nif not pd.isna(e05_p12_for_compare):\n    if e06_p12 > e05_p12_for_compare:\n        e06_vs_e05_text = \"E06 beats E05, so postal_code may add useful signal to article metadata.\"\n    elif e06_p12 < e05_p12_for_compare:\n        e06_vs_e05_text = \"E06 is below E05, so the selected subset may have removed useful article features.\"\n    else:\n        e06_vs_e05_text = \"E06 matches E05.\"\nelse:\n    e06_vs_e05_text = \"E05 comparison unavailable.\"\n\ne06_decision = (\n    f\"{e06_vs_baseline_text} \"\n    f\"{e06_vs_e03_text} \"\n    f\"{e06_vs_e05_text} \"\n    \"Use this result to decide whether to keep the compact top-feature model \"\n    \"or return to the fuller E03 feature set.\"\n)\n\nprint(\"\\n=== Safe E06 interpretation ===\")\nprint(e06_decision)\n\nprint(\"\\nObjects to log:\")\nprint(\"- e06_summary\")\nprint(\"- fi_e06\")\nprint(\"- alpha_e06, beta_e06, intercept_e06\")\nprint(\"- auc_score_dem_e06, auc_hybrid_e06\")\nprint(\"- e06_decision\")\n\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-28T19:47:54.977024Z","iopub.execute_input":"2026-05-28T19:47:54.977644Z","iopub.status.idle":"2026-05-28T19:48:48.374995Z","shell.execute_reply.started":"2026-05-28T19:47:54.977607Z","shell.execute_reply":"2026-05-28T19:48:48.373738Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# NEW CELL G - E07: Simple date seasonality\n#\n# WHERE TO PLACE:\n# Put this NEW code cell immediately AFTER:\n# NEW CELL F - E06: Top feature subset\n#\n# PURPOSE:\n# Test whether simple transaction-date seasonality improves the current best setup.\n#\n# Base model:\n# E03 feature set without inferred_gender:\n# - age\n# - postal_code\n# - product_type_name\n# - graphical_appearance_name\n# - colour_group_name\n# - perceived_colour_value_name\n# - index_name\n# - index_group_name\n#\n# Added safe date features:\n# - month\n# - weekofyear\n# - quarter\n# - dayofweek\n# - season_bucket\n# - days_since_window_start\n#\n# Important:\n# - No weather.\n# - No real geography.\n# - postal_code is still only a hashed categorical proxy.\n# - Test candidates get a fixed validation-week anchor date.\n# ============================================================\n\nimport pandas as pd\nimport numpy as np\nimport gc\n\nfrom catboost import CatBoostClassifier\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import roc_auc_score\n\nprint(\"NEW CELL G: E07 - Simple date seasonality\")\n\n# ------------------------------------------------------------\n# 1. Safety checks\n# ------------------------------------------------------------\n\nrequired_objects = [\n    \"catboost_train_df\",\n    \"als_candidates_df\",\n    \"train_df\",\n    \"test_df\",\n    \"baseline_p12\",\n    \"hybrid_p12\"\n]\n\nmissing_objects = [obj for obj in required_objects if obj not in globals()]\n\nif missing_objects:\n    raise RuntimeError(\n        f\"Missing required objects: {missing_objects}. \"\n        \"Run cells up to E06 first.\"\n    )\n\ne03_p12_for_compare = e03_p12 if \"e03_p12\" in globals() else np.nan\ne04_p12_for_compare = e04_p12 if \"e04_p12\" in globals() else np.nan\ne05_p12_for_compare = e05_p12 if \"e05_p12\" in globals() else np.nan\ne06_p12_for_compare = e06_p12 if \"e06_p12\" in globals() else np.nan\n\n# ------------------------------------------------------------\n# 2. Helper: create date features\n# ------------------------------------------------------------\n\ndef add_simple_date_features(df_in, date_col, window_start_date):\n    df_out = df_in.copy()\n    dt = pd.to_datetime(df_out[date_col])\n\n    df_out[\"month\"] = dt.dt.month.astype(str)\n    df_out[\"weekofyear\"] = dt.dt.isocalendar().week.astype(int).astype(str)\n    df_out[\"quarter\"] = dt.dt.quarter.astype(str)\n    df_out[\"dayofweek\"] = dt.dt.dayofweek.astype(str)\n\n    month_num = dt.dt.month\n\n    conditions = [\n        month_num.isin([12, 1, 2]),\n        month_num.isin([3, 4, 5]),\n        month_num.isin([6, 7, 8]),\n        month_num.isin([9, 10, 11])\n    ]\n\n    choices = [\"winter\", \"spring\", \"summer\", \"autumn\"]\n\n    df_out[\"season_bucket\"] = np.select(\n        conditions,\n        choices,\n        default=\"unknown\"\n    ).astype(str)\n\n    df_out[\"days_since_window_start\"] = (\n        dt - pd.to_datetime(window_start_date)\n    ).dt.days.astype(\"int16\")\n\n    return df_out\n\n# ------------------------------------------------------------\n# 3. Define base + date features\n# ------------------------------------------------------------\n\nbase_features_e07 = [\n    \"age\",\n    \"postal_code\",\n    \"product_type_name\",\n    \"graphical_appearance_name\",\n    \"colour_group_name\",\n    \"perceived_colour_value_name\",\n    \"index_name\",\n    \"index_group_name\"\n]\n\ndate_features_e07 = [\n    \"month\",\n    \"weekofyear\",\n    \"quarter\",\n    \"dayofweek\",\n    \"season_bucket\",\n    \"days_since_window_start\"\n]\n\nall_features_e07 = base_features_e07 + date_features_e07\n\nrequired_train_cols = [\"target\", \"user_idx\", \"item_idx\"] + base_features_e07\nmissing_train_cols = [col for col in required_train_cols if col not in catboost_train_df.columns]\n\nif missing_train_cols:\n    raise RuntimeError(f\"catboost_train_df is missing columns: {missing_train_cols}\")\n\nrequired_candidate_cols = [\"user_idx\", \"item_idx\", \"target\", \"Score_CF\"] + base_features_e07\nmissing_candidate_cols = [col for col in required_candidate_cols if col not in als_candidates_df.columns]\n\nif missing_candidate_cols:\n    raise RuntimeError(f\"als_candidates_df is missing columns: {missing_candidate_cols}\")\n\n# ------------------------------------------------------------\n# 4. Build train anchor dates without using test-period purchases\n# ------------------------------------------------------------\n\nprint(\"\\nBuilding train-safe anchor dates...\")\n\ntrain_df_dates = train_df[[\"user_idx\", \"item_idx\", \"t_dat\"]].copy()\ntrain_df_dates[\"t_dat\"] = pd.to_datetime(train_df_dates[\"t_dat\"])\n\nwindow_start_date_e07 = train_df_dates[\"t_dat\"].min()\nvalidation_anchor_date_e07 = pd.to_datetime(test_df[\"t_dat\"]).min()\n\n# Positive rows: use the latest train purchase date for that user-item pair.\npositive_pair_dates = (\n    train_df_dates\n    .sort_values(\"t_dat\")\n    .groupby([\"user_idx\", \"item_idx\"], as_index=False)\n    .agg(pair_train_date=(\"t_dat\", \"max\"))\n)\n\n# Negative rows: use the user's latest train activity date.\nuser_anchor_dates = (\n    train_df_dates\n    .groupby(\"user_idx\", as_index=False)\n    .agg(user_train_anchor_date=(\"t_dat\", \"max\"))\n)\n\ne07_train_df = catboost_train_df.copy()\n\ne07_train_df = e07_train_df.merge(\n    positive_pair_dates,\n    on=[\"user_idx\", \"item_idx\"],\n    how=\"left\"\n)\n\ne07_train_df = e07_train_df.merge(\n    user_anchor_dates,\n    on=\"user_idx\",\n    how=\"left\"\n)\n\n# For positives, pair_train_date exists.\n# For negatives, use user_train_anchor_date.\n# Fallback: window start date.\ne07_train_df[\"season_anchor_date\"] = e07_train_df[\"pair_train_date\"]\n\ne07_train_df[\"season_anchor_date\"] = e07_train_df[\"season_anchor_date\"].fillna(\n    e07_train_df[\"user_train_anchor_date\"]\n)\n\ne07_train_df[\"season_anchor_date\"] = e07_train_df[\"season_anchor_date\"].fillna(\n    window_start_date_e07\n)\n\ne07_train_df = add_simple_date_features(\n    e07_train_df,\n    date_col=\"season_anchor_date\",\n    window_start_date=window_start_date_e07\n)\n\n# Candidate rows: use fixed validation-week anchor date.\ne07_candidates_df = als_candidates_df.copy()\ne07_candidates_df[\"season_anchor_date\"] = validation_anchor_date_e07\n\ne07_candidates_df = add_simple_date_features(\n    e07_candidates_df,\n    date_col=\"season_anchor_date\",\n    window_start_date=window_start_date_e07\n)\n\nprint(f\"Train window start date: {window_start_date_e07}\")\nprint(f\"Validation anchor date: {validation_anchor_date_e07}\")\n\n# ------------------------------------------------------------\n# 5. Prepare E07 training matrices\n# ------------------------------------------------------------\n\nX_e07 = e07_train_df[all_features_e07].copy()\ny_e07 = e07_train_df[\"target\"].copy()\n\nX_test_e07 = e07_candidates_df[all_features_e07].copy()\n\n# Clean data types.\ncategorical_features_e07 = [\n    \"postal_code\",\n    \"product_type_name\",\n    \"graphical_appearance_name\",\n    \"colour_group_name\",\n    \"perceived_colour_value_name\",\n    \"index_name\",\n    \"index_group_name\",\n    \"month\",\n    \"weekofyear\",\n    \"quarter\",\n    \"dayofweek\",\n    \"season_bucket\"\n]\n\nnumeric_features_e07 = [\n    \"age\",\n    \"days_since_window_start\"\n]\n\nage_median_e07 = X_e07[\"age\"].median()\n\nfor df_tmp in [X_e07, X_test_e07]:\n    df_tmp[\"age\"] = pd.to_numeric(df_tmp[\"age\"], errors=\"coerce\").fillna(age_median_e07)\n    df_tmp[\"days_since_window_start\"] = pd.to_numeric(\n        df_tmp[\"days_since_window_start\"],\n        errors=\"coerce\"\n    ).fillna(0)\n\n    for col in categorical_features_e07:\n        df_tmp[col] = df_tmp[col].fillna(\"Unknown\").astype(str)\n\nprint(\"\\nE07 feature set:\")\nprint(list(X_e07.columns))\n\nblocked_features = [\"inferred_gender\"]\n\nfor feature in blocked_features:\n    if feature in X_e07.columns:\n        raise RuntimeError(f\"E07 failed: blocked feature still present: {feature}\")\n\nprint(\"\\nFeature check:\")\nprint(f\"inferred_gender in E07 features: {'inferred_gender' in list(X_e07.columns)}\")\nprint(\"postal_code is used only as a hashed categorical proxy, not as geography.\")\n\n# ------------------------------------------------------------\n# 6. Train CatBoost for E07\n# ------------------------------------------------------------\n\nprint(\"\\nTraining CatBoost E07...\")\n\nmodel_cb_e07 = CatBoostClassifier(\n    iterations=100,\n    learning_rate=0.1,\n    depth=6,\n    loss_function=\"Logloss\",\n    eval_metric=\"AUC\",\n    random_seed=42,\n    verbose=20\n)\n\nmodel_cb_e07.fit(X_e07, y_e07, cat_features=categorical_features_e07)\n\n# ------------------------------------------------------------\n# 7. Generate E07 CatBoost score on same ALS candidates\n# ------------------------------------------------------------\n\nprint(\"\\nScoring same ALS candidates for E07...\")\n\nals_candidates_df[\"Score_Dem_E07\"] = model_cb_e07.predict_proba(X_test_e07)[:, 1]\n\n# ------------------------------------------------------------\n# 8. Train Logistic Regression fusion for E07\n# ------------------------------------------------------------\n\nprint(\"\\nTraining Logistic Regression fusion for E07...\")\n\nscaler_e07 = StandardScaler()\n\nscaled_e07 = scaler_e07.fit_transform(\n    als_candidates_df[[\"Score_CF\", \"Score_Dem_E07\"]]\n)\n\nals_candidates_df[\"Scaled_CF_E07\"] = scaled_e07[:, 0]\nals_candidates_df[\"Scaled_Dem_E07\"] = scaled_e07[:, 1]\n\nX_meta_e07 = als_candidates_df[[\"Scaled_CF_E07\", \"Scaled_Dem_E07\"]]\ny_meta_e07 = als_candidates_df[\"target\"]\n\nmodel_lr_e07 = LogisticRegression(class_weight=\"balanced\", random_state=42)\nmodel_lr_e07.fit(X_meta_e07, y_meta_e07)\n\nalpha_e07 = model_lr_e07.coef_[0][0]\nbeta_e07 = model_lr_e07.coef_[0][1]\nintercept_e07 = model_lr_e07.intercept_[0]\n\nals_candidates_df[\"Hybrid_Score_E07\"] = (\n    alpha_e07 * als_candidates_df[\"Scaled_CF_E07\"] +\n    beta_e07 * als_candidates_df[\"Scaled_Dem_E07\"] +\n    intercept_e07\n)\n\nprint(\"\\n=== E07 Logistic Regression coefficients ===\")\nprint(f\"Alpha / ALS Score_CF: {alpha_e07:.4f}\")\nprint(f\"Beta / CatBoost Score_Dem_E07: {beta_e07:.4f}\")\nprint(f\"Intercept: {intercept_e07:.4f}\")\n\n# ------------------------------------------------------------\n# 9. Evaluation helpers\n# ------------------------------------------------------------\n\ndef precision_at_12_e07(df, score_col):\n    top_12 = (\n        df.sort_values([\"user_idx\", score_col], ascending=[True, False])\n          .groupby(\"user_idx\")\n          .head(12)\n    )\n    user_hits = top_12.groupby(\"user_idx\")[\"target\"].sum()\n    return (user_hits / 12).mean()\n\ndef safe_auc_e07(y_true, score):\n    if len(np.unique(y_true)) < 2:\n        return np.nan\n    return roc_auc_score(y_true, score)\n\n# ------------------------------------------------------------\n# 10. E07 metrics\n# ------------------------------------------------------------\n\ne07_p12 = precision_at_12_e07(als_candidates_df, \"Hybrid_Score_E07\")\n\ne07_uplift_vs_e01 = ((e07_p12 - baseline_p12) / baseline_p12) * 100\ne07_delta_vs_e02 = ((e07_p12 - hybrid_p12) / hybrid_p12) * 100\n\nif pd.isna(e03_p12_for_compare):\n    e07_delta_vs_e03 = np.nan\nelse:\n    e07_delta_vs_e03 = ((e07_p12 - e03_p12_for_compare) / e03_p12_for_compare) * 100\n\nif pd.isna(e05_p12_for_compare):\n    e07_delta_vs_e05 = np.nan\nelse:\n    e07_delta_vs_e05 = ((e07_p12 - e05_p12_for_compare) / e05_p12_for_compare) * 100\n\nif pd.isna(e06_p12_for_compare):\n    e07_delta_vs_e06 = np.nan\nelse:\n    e07_delta_vs_e06 = ((e07_p12 - e06_p12_for_compare) / e06_p12_for_compare) * 100\n\nauc_score_dem_e07 = safe_auc_e07(\n    als_candidates_df[\"target\"],\n    als_candidates_df[\"Score_Dem_E07\"]\n)\n\nauc_hybrid_e07 = safe_auc_e07(\n    als_candidates_df[\"target\"],\n    als_candidates_df[\"Hybrid_Score_E07\"]\n)\n\nprint(\"\\n=== E07 Precision@12 results ===\")\nprint(f\"E01 ALS baseline Precision@12: {baseline_p12:.6f}\")\nprint(f\"E02 full hybrid Precision@12: {hybrid_p12:.6f}\")\n\nif not pd.isna(e03_p12_for_compare):\n    print(f\"E03 hybrid without inferred_gender Precision@12: {e03_p12_for_compare:.6f}\")\n\nif not pd.isna(e05_p12_for_compare):\n    print(f\"E05 article metadata only Precision@12: {e05_p12_for_compare:.6f}\")\n\nif not pd.isna(e06_p12_for_compare):\n    print(f\"E06 top feature subset Precision@12: {e06_p12_for_compare:.6f}\")\n\nprint(f\"E07 simple date seasonality Precision@12: {e07_p12:.6f}\")\nprint(f\"E07 uplift vs E01: {e07_uplift_vs_e01:.2f}%\")\nprint(f\"E07 delta vs E02: {e07_delta_vs_e02:.2f}%\")\n\nif not pd.isna(e07_delta_vs_e03):\n    print(f\"E07 delta vs E03: {e07_delta_vs_e03:.2f}%\")\n\nif not pd.isna(e07_delta_vs_e05):\n    print(f\"E07 delta vs E05: {e07_delta_vs_e05:.2f}%\")\n\nif not pd.isna(e07_delta_vs_e06):\n    print(f\"E07 delta vs E06: {e07_delta_vs_e06:.2f}%\")\n\nprint(\"\\n=== AUC diagnostic ===\")\nprint(f\"E07 CatBoost Score_Dem_E07 AUC: {auc_score_dem_e07:.6f}\")\nprint(f\"E07 Hybrid_Score_E07 AUC: {auc_hybrid_e07:.6f}\")\n\n# ------------------------------------------------------------\n# 11. Feature importance for E07\n# ------------------------------------------------------------\n\nfi_e07 = (\n    pd.DataFrame({\n        \"feature\": list(model_cb_e07.feature_names_),\n        \"importance\": model_cb_e07.get_feature_importance()\n    })\n    .sort_values(\"importance\", ascending=False)\n    .reset_index(drop=True)\n)\n\nprint(\"\\n=== E07 CatBoost feature importance: top 15 ===\")\ndisplay(fi_e07.head(15))\n\n# ------------------------------------------------------------\n# 12. E07 comparison summary\n# ------------------------------------------------------------\n\nsummary_rows = [\n    {\n        \"experiment_id\": \"E01\",\n        \"name\": \"ALS baseline only\",\n        \"Precision@12\": baseline_p12,\n        \"uplift_vs_E01_%\": 0.0,\n        \"delta_vs_E03_%\": np.nan,\n        \"delta_vs_E07_%\": np.nan,\n        \"notes\": \"Locked baseline\"\n    },\n    {\n        \"experiment_id\": \"E02\",\n        \"name\": \"Full current hybrid\",\n        \"Precision@12\": hybrid_p12,\n        \"uplift_vs_E01_%\": ((hybrid_p12 - baseline_p12) / baseline_p12) * 100,\n        \"delta_vs_E03_%\": np.nan,\n        \"delta_vs_E07_%\": ((hybrid_p12 - e07_p12) / e07_p12) * 100,\n        \"notes\": \"Full feature set with train-safe inferred_gender\"\n    }\n]\n\nif not pd.isna(e03_p12_for_compare):\n    summary_rows.append({\n        \"experiment_id\": \"E03\",\n        \"name\": \"Hybrid without inferred_gender\",\n        \"Precision@12\": e03_p12_for_compare,\n        \"uplift_vs_E01_%\": ((e03_p12_for_compare - baseline_p12) / baseline_p12) * 100,\n        \"delta_vs_E03_%\": 0.0,\n        \"delta_vs_E07_%\": ((e03_p12_for_compare - e07_p12) / e07_p12) * 100,\n        \"notes\": \"Current best before E07\"\n    })\n\nif not pd.isna(e05_p12_for_compare):\n    summary_rows.append({\n        \"experiment_id\": \"E05\",\n        \"name\": \"Article metadata only\",\n        \"Precision@12\": e05_p12_for_compare,\n        \"uplift_vs_E01_%\": ((e05_p12_for_compare - baseline_p12) / baseline_p12) * 100,\n        \"delta_vs_E03_%\": (\n            ((e05_p12_for_compare - e03_p12_for_compare) / e03_p12_for_compare) * 100\n            if not pd.isna(e03_p12_for_compare)\n            else np.nan\n        ),\n        \"delta_vs_E07_%\": ((e05_p12_for_compare - e07_p12) / e07_p12) * 100,\n        \"notes\": \"Article metadata only\"\n    })\n\nif not pd.isna(e06_p12_for_compare):\n    summary_rows.append({\n        \"experiment_id\": \"E06\",\n        \"name\": \"Top feature subset\",\n        \"Precision@12\": e06_p12_for_compare,\n        \"uplift_vs_E01_%\": ((e06_p12_for_compare - baseline_p12) / baseline_p12) * 100,\n        \"delta_vs_E03_%\": (\n            ((e06_p12_for_compare - e03_p12_for_compare) / e03_p12_for_compare) * 100\n            if not pd.isna(e03_p12_for_compare)\n            else np.nan\n        ),\n        \"delta_vs_E07_%\": ((e06_p12_for_compare - e07_p12) / e07_p12) * 100,\n        \"notes\": \"Compact subset\"\n    })\n\nsummary_rows.append({\n    \"experiment_id\": \"E07\",\n    \"name\": \"Simple date seasonality\",\n    \"Precision@12\": e07_p12,\n    \"uplift_vs_E01_%\": e07_uplift_vs_e01,\n    \"delta_vs_E03_%\": e07_delta_vs_e03,\n    \"delta_vs_E07_%\": 0.0,\n    \"notes\": \"E03 feature set + t_dat seasonality; no inferred_gender; no weather\"\n})\n\ne07_summary = pd.DataFrame(summary_rows)\n\nprint(\"\\n=== E07 comparison summary ===\")\ndisplay(e07_summary)\n\n# ------------------------------------------------------------\n# 13. Safe decision line\n# ------------------------------------------------------------\n\nif e07_p12 > baseline_p12:\n    e07_vs_baseline_text = \"E07 beats the ALS baseline in this run.\"\nelif e07_p12 < baseline_p12:\n    e07_vs_baseline_text = \"E07 is below the ALS baseline in this run.\"\nelse:\n    e07_vs_baseline_text = \"E07 equals the ALS baseline in this run.\"\n\nif not pd.isna(e03_p12_for_compare):\n    if e07_p12 > e03_p12_for_compare:\n        e07_vs_e03_text = \"E07 beats E03, so simple date seasonality becomes the current best candidate.\"\n    elif e07_p12 < e03_p12_for_compare:\n        e07_vs_e03_text = \"E07 is below E03, so simple date seasonality does not improve the current best setup.\"\n    else:\n        e07_vs_e03_text = \"E07 matches E03.\"\nelse:\n    e07_vs_e03_text = \"E03 comparison unavailable.\"\n\ne07_decision = (\n    f\"{e07_vs_baseline_text} \"\n    f\"{e07_vs_e03_text} \"\n    \"Do not describe this as weather or geography. \"\n    \"It is only a transaction-date seasonality test.\"\n)\n\nprint(\"\\n=== Safe E07 interpretation ===\")\nprint(e07_decision)\n\nprint(\"\\nObjects to log:\")\nprint(\"- e07_summary\")\nprint(\"- fi_e07\")\nprint(\"- alpha_e07, beta_e07, intercept_e07\")\nprint(\"- auc_score_dem_e07, auc_hybrid_e07\")\nprint(\"- e07_decision\")\n\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-28T19:56:43.109807Z","iopub.execute_input":"2026-05-28T19:56:43.110898Z","iopub.status.idle":"2026-05-28T19:58:20.920058Z","shell.execute_reply.started":"2026-05-28T19:56:43.110859Z","shell.execute_reply":"2026-05-28T19:58:20.919111Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# NEW CELL H - E08: Train-only item/category popularity\n#\n# WHERE TO PLACE:\n# Put this NEW code cell immediately AFTER:\n# NEW CELL G - E07: Simple date seasonality\n#\n# PURPOSE:\n# Test whether train-only item/category popularity improves the current best E07 setup.\n#\n# Base model:\n# E07 feature set:\n# - age\n# - postal_code\n# - product/article metadata\n# - simple date seasonality from t_dat\n#\n# Added E08 features:\n# - item_train_count\n# - item_train_unique_users\n# - item_train_recent_7d_count\n# - item_train_recent_14d_count\n# - item_days_since_last_train_purchase\n# - product_type_train_count\n# - index_name_train_count\n# - colour_group_train_count\n#\n# Leakage rule:\n# All popularity features are calculated from train_df only.\n# No test-week purchases are used.\n#\n# Important:\n# - No weather.\n# - No real geography.\n# - postal_code is only a hashed categorical proxy.\n# ============================================================\n\nimport pandas as pd\nimport numpy as np\nimport gc\n\nfrom catboost import CatBoostClassifier\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import roc_auc_score\n\nprint(\"NEW CELL H: E08 - Train-only item/category popularity\")\n\n# ------------------------------------------------------------\n# 1. Safety checks\n# ------------------------------------------------------------\n\nrequired_objects = [\n    \"catboost_train_df\",\n    \"als_candidates_df\",\n    \"train_df\",\n    \"test_df\",\n    \"baseline_p12\",\n    \"hybrid_p12\"\n]\n\nmissing_objects = [obj for obj in required_objects if obj not in globals()]\n\nif missing_objects:\n    raise RuntimeError(\n        f\"Missing required objects: {missing_objects}. \"\n        \"Run cells up to E07 first.\"\n    )\n\ne03_p12_for_compare = e03_p12 if \"e03_p12\" in globals() else np.nan\ne05_p12_for_compare = e05_p12 if \"e05_p12\" in globals() else np.nan\ne06_p12_for_compare = e06_p12 if \"e06_p12\" in globals() else np.nan\ne07_p12_for_compare = e07_p12 if \"e07_p12\" in globals() else np.nan\n\n# ------------------------------------------------------------\n# 2. Helper: date features, same logic as E07\n# ------------------------------------------------------------\n\ndef add_simple_date_features_e08(df_in, date_col, window_start_date):\n    df_out = df_in.copy()\n    dt = pd.to_datetime(df_out[date_col])\n\n    df_out[\"month\"] = dt.dt.month.astype(str)\n    df_out[\"weekofyear\"] = dt.dt.isocalendar().week.astype(int).astype(str)\n    df_out[\"quarter\"] = dt.dt.quarter.astype(str)\n    df_out[\"dayofweek\"] = dt.dt.dayofweek.astype(str)\n\n    month_num = dt.dt.month\n\n    conditions = [\n        month_num.isin([12, 1, 2]),\n        month_num.isin([3, 4, 5]),\n        month_num.isin([6, 7, 8]),\n        month_num.isin([9, 10, 11])\n    ]\n\n    choices = [\"winter\", \"spring\", \"summer\", \"autumn\"]\n\n    df_out[\"season_bucket\"] = np.select(\n        conditions,\n        choices,\n        default=\"unknown\"\n    ).astype(str)\n\n    df_out[\"days_since_window_start\"] = (\n        dt - pd.to_datetime(window_start_date)\n    ).dt.days.astype(\"int16\")\n\n    return df_out\n\n# ------------------------------------------------------------\n# 3. Define E08 base/date/popularity features\n# ------------------------------------------------------------\n\nbase_features_e08 = [\n    \"age\",\n    \"postal_code\",\n    \"product_type_name\",\n    \"graphical_appearance_name\",\n    \"colour_group_name\",\n    \"perceived_colour_value_name\",\n    \"index_name\",\n    \"index_group_name\"\n]\n\ndate_features_e08 = [\n    \"month\",\n    \"weekofyear\",\n    \"quarter\",\n    \"dayofweek\",\n    \"season_bucket\",\n    \"days_since_window_start\"\n]\n\npopularity_features_e08 = [\n    \"item_train_count\",\n    \"item_train_unique_users\",\n    \"item_train_recent_7d_count\",\n    \"item_train_recent_14d_count\",\n    \"item_days_since_last_train_purchase\",\n    \"product_type_train_count\",\n    \"index_name_train_count\",\n    \"colour_group_train_count\"\n]\n\nall_features_e08 = base_features_e08 + date_features_e08 + popularity_features_e08\n\nrequired_train_cols = [\"target\", \"user_idx\", \"item_idx\"] + base_features_e08\nmissing_train_cols = [col for col in required_train_cols if col not in catboost_train_df.columns]\n\nif missing_train_cols:\n    raise RuntimeError(f\"catboost_train_df is missing columns: {missing_train_cols}\")\n\nrequired_candidate_cols = [\"user_idx\", \"item_idx\", \"target\", \"Score_CF\"] + base_features_e08\nmissing_candidate_cols = [col for col in required_candidate_cols if col not in als_candidates_df.columns]\n\nif missing_candidate_cols:\n    raise RuntimeError(f\"als_candidates_df is missing columns: {missing_candidate_cols}\")\n\n# ------------------------------------------------------------\n# 4. Build train-safe date anchors, same logic as E07\n# ------------------------------------------------------------\n\nprint(\"\\nBuilding train-safe anchor dates...\")\n\ntrain_df_dates = train_df[[\"user_idx\", \"item_idx\", \"t_dat\"]].copy()\ntrain_df_dates[\"t_dat\"] = pd.to_datetime(train_df_dates[\"t_dat\"])\n\nwindow_start_date_e08 = train_df_dates[\"t_dat\"].min()\ntrain_end_date_e08 = train_df_dates[\"t_dat\"].max()\nvalidation_anchor_date_e08 = pd.to_datetime(test_df[\"t_dat\"]).min()\n\npositive_pair_dates = (\n    train_df_dates\n    .sort_values(\"t_dat\")\n    .groupby([\"user_idx\", \"item_idx\"], as_index=False)\n    .agg(pair_train_date=(\"t_dat\", \"max\"))\n)\n\nuser_anchor_dates = (\n    train_df_dates\n    .groupby(\"user_idx\", as_index=False)\n    .agg(user_train_anchor_date=(\"t_dat\", \"max\"))\n)\n\ne08_train_df = catboost_train_df.copy()\n\ne08_train_df = e08_train_df.merge(\n    positive_pair_dates,\n    on=[\"user_idx\", \"item_idx\"],\n    how=\"left\"\n)\n\ne08_train_df = e08_train_df.merge(\n    user_anchor_dates,\n    on=\"user_idx\",\n    how=\"left\"\n)\n\ne08_train_df[\"season_anchor_date\"] = e08_train_df[\"pair_train_date\"]\n\ne08_train_df[\"season_anchor_date\"] = e08_train_df[\"season_anchor_date\"].fillna(\n    e08_train_df[\"user_train_anchor_date\"]\n)\n\ne08_train_df[\"season_anchor_date\"] = e08_train_df[\"season_anchor_date\"].fillna(\n    window_start_date_e08\n)\n\ne08_train_df = add_simple_date_features_e08(\n    e08_train_df,\n    date_col=\"season_anchor_date\",\n    window_start_date=window_start_date_e08\n)\n\ne08_candidates_df = als_candidates_df.copy()\ne08_candidates_df[\"season_anchor_date\"] = validation_anchor_date_e08\n\ne08_candidates_df = add_simple_date_features_e08(\n    e08_candidates_df,\n    date_col=\"season_anchor_date\",\n    window_start_date=window_start_date_e08\n)\n\nprint(f\"Train window start date: {window_start_date_e08}\")\nprint(f\"Train end date: {train_end_date_e08}\")\nprint(f\"Validation anchor date: {validation_anchor_date_e08}\")\n\n# ------------------------------------------------------------\n# 5. Build train-only item popularity features\n# ------------------------------------------------------------\n\nprint(\"\\nBuilding train-only item popularity features...\")\n\nrecent_7d_start_e08 = train_end_date_e08 - pd.Timedelta(days=6)\nrecent_14d_start_e08 = train_end_date_e08 - pd.Timedelta(days=13)\n\nitem_pop_e08 = (\n    train_df_dates\n    .groupby(\"item_idx\")\n    .agg(\n        item_train_count=(\"user_idx\", \"size\"),\n        item_train_unique_users=(\"user_idx\", \"nunique\"),\n        item_last_train_purchase=(\"t_dat\", \"max\")\n    )\n    .reset_index()\n)\n\nrecent_7d_counts_e08 = (\n    train_df_dates[train_df_dates[\"t_dat\"] >= recent_7d_start_e08]\n    .groupby(\"item_idx\")\n    .size()\n    .reset_index(name=\"item_train_recent_7d_count\")\n)\n\nrecent_14d_counts_e08 = (\n    train_df_dates[train_df_dates[\"t_dat\"] >= recent_14d_start_e08]\n    .groupby(\"item_idx\")\n    .size()\n    .reset_index(name=\"item_train_recent_14d_count\")\n)\n\nitem_pop_e08 = item_pop_e08.merge(\n    recent_7d_counts_e08,\n    on=\"item_idx\",\n    how=\"left\"\n)\n\nitem_pop_e08 = item_pop_e08.merge(\n    recent_14d_counts_e08,\n    on=\"item_idx\",\n    how=\"left\"\n)\n\nitem_pop_e08[\"item_train_recent_7d_count\"] = (\n    item_pop_e08[\"item_train_recent_7d_count\"]\n    .fillna(0)\n)\n\nitem_pop_e08[\"item_train_recent_14d_count\"] = (\n    item_pop_e08[\"item_train_recent_14d_count\"]\n    .fillna(0)\n)\n\nitem_pop_e08[\"item_days_since_last_train_purchase\"] = (\n    validation_anchor_date_e08 - item_pop_e08[\"item_last_train_purchase\"]\n).dt.days\n\nitem_pop_e08 = item_pop_e08.drop(columns=[\"item_last_train_purchase\"])\n\n# ------------------------------------------------------------\n# 6. Build train-only category popularity features\n# ------------------------------------------------------------\n\nprint(\"Building train-only category popularity features...\")\n\ntrain_with_categories_e08 = train_df[[\"item_idx\"]].copy()\n\ncategory_cols_e08 = [\n    \"product_type_name\",\n    \"index_name\",\n    \"colour_group_name\"\n]\n\n# Pull category values from catboost_train_df by item_idx.\n# This avoids needing article_id mappings and keeps the cell append-only.\nitem_category_lookup_e08 = (\n    catboost_train_df[[\"item_idx\"] + category_cols_e08]\n    .drop_duplicates(\"item_idx\")\n)\n\ntrain_with_categories_e08 = train_with_categories_e08.merge(\n    item_category_lookup_e08,\n    on=\"item_idx\",\n    how=\"left\"\n)\n\nproduct_type_pop_e08 = (\n    train_with_categories_e08\n    .groupby(\"product_type_name\")\n    .size()\n    .reset_index(name=\"product_type_train_count\")\n)\n\nindex_name_pop_e08 = (\n    train_with_categories_e08\n    .groupby(\"index_name\")\n    .size()\n    .reset_index(name=\"index_name_train_count\")\n)\n\ncolour_group_pop_e08 = (\n    train_with_categories_e08\n    .groupby(\"colour_group_name\")\n    .size()\n    .reset_index(name=\"colour_group_train_count\")\n)\n\n# ------------------------------------------------------------\n# 7. Merge train-only popularity features into train and candidate rows\n# ------------------------------------------------------------\n\nprint(\"Merging E08 popularity features...\")\n\nfor df_tmp_name, df_tmp in [\n    (\"train\", e08_train_df),\n    (\"candidates\", e08_candidates_df)\n]:\n    df_tmp = df_tmp.merge(item_pop_e08, on=\"item_idx\", how=\"left\")\n    df_tmp = df_tmp.merge(product_type_pop_e08, on=\"product_type_name\", how=\"left\")\n    df_tmp = df_tmp.merge(index_name_pop_e08, on=\"index_name\", how=\"left\")\n    df_tmp = df_tmp.merge(colour_group_pop_e08, on=\"colour_group_name\", how=\"left\")\n\n    for col in popularity_features_e08:\n        df_tmp[col] = pd.to_numeric(df_tmp[col], errors=\"coerce\").fillna(0)\n\n    if df_tmp_name == \"train\":\n        e08_train_df = df_tmp\n    else:\n        e08_candidates_df = df_tmp\n\n# Optional conservative adjustment:\n# For positive training rows, subtract the current positive row from item count.\n# This reduces within-train self-counting without touching validation candidates.\nif \"target\" in e08_train_df.columns:\n    e08_train_df[\"item_train_count\"] = (\n        e08_train_df[\"item_train_count\"] - e08_train_df[\"target\"]\n    ).clip(lower=0)\n\nprint(\"Popularity features ready.\")\n\n# ------------------------------------------------------------\n# 8. Prepare E08 training matrices\n# ------------------------------------------------------------\n\nX_e08 = e08_train_df[all_features_e08].copy()\ny_e08 = e08_train_df[\"target\"].copy()\n\nX_test_e08 = e08_candidates_df[all_features_e08].copy()\n\ncategorical_features_e08 = [\n    \"postal_code\",\n    \"product_type_name\",\n    \"graphical_appearance_name\",\n    \"colour_group_name\",\n    \"perceived_colour_value_name\",\n    \"index_name\",\n    \"index_group_name\",\n    \"month\",\n    \"weekofyear\",\n    \"quarter\",\n    \"dayofweek\",\n    \"season_bucket\"\n]\n\nnumeric_features_e08 = [\n    \"age\",\n    \"days_since_window_start\"\n] + popularity_features_e08\n\nage_median_e08 = X_e08[\"age\"].median()\n\nfor df_tmp in [X_e08, X_test_e08]:\n    df_tmp[\"age\"] = pd.to_numeric(df_tmp[\"age\"], errors=\"coerce\").fillna(age_median_e08)\n\n    for col in numeric_features_e08:\n        df_tmp[col] = pd.to_numeric(df_tmp[col], errors=\"coerce\").fillna(0)\n\n    for col in categorical_features_e08:\n        df_tmp[col] = df_tmp[col].fillna(\"Unknown\").astype(str)\n\nprint(\"\\nE08 feature set:\")\nprint(list(X_e08.columns))\n\nif \"inferred_gender\" in X_e08.columns:\n    raise RuntimeError(\"E08 failed: inferred_gender is still present.\")\n\nprint(\"\\nFeature check:\")\nprint(f\"inferred_gender in E08 features: {'inferred_gender' in list(X_e08.columns)}\")\nprint(\"Popularity features are train-only.\")\nprint(\"postal_code is used only as a hashed categorical proxy, not as geography.\")\n\n# ------------------------------------------------------------\n# 9. Train CatBoost for E08\n# ------------------------------------------------------------\n\nprint(\"\\nTraining CatBoost E08...\")\n\nmodel_cb_e08 = CatBoostClassifier(\n    iterations=100,\n    learning_rate=0.1,\n    depth=6,\n    loss_function=\"Logloss\",\n    eval_metric=\"AUC\",\n    random_seed=42,\n    verbose=20\n)\n\nmodel_cb_e08.fit(X_e08, y_e08, cat_features=categorical_features_e08)\n\n# ------------------------------------------------------------\n# 10. Generate E08 CatBoost score on same ALS candidates\n# ------------------------------------------------------------\n\nprint(\"\\nScoring same ALS candidates for E08...\")\n\nals_candidates_df[\"Score_Dem_E08\"] = model_cb_e08.predict_proba(X_test_e08)[:, 1]\n\n# ------------------------------------------------------------\n# 11. Train Logistic Regression fusion for E08\n# ------------------------------------------------------------\n\nprint(\"\\nTraining Logistic Regression fusion for E08...\")\n\nscaler_e08 = StandardScaler()\n\nscaled_e08 = scaler_e08.fit_transform(\n    als_candidates_df[[\"Score_CF\", \"Score_Dem_E08\"]]\n)\n\nals_candidates_df[\"Scaled_CF_E08\"] = scaled_e08[:, 0]\nals_candidates_df[\"Scaled_Dem_E08\"] = scaled_e08[:, 1]\n\nX_meta_e08 = als_candidates_df[[\"Scaled_CF_E08\", \"Scaled_Dem_E08\"]]\ny_meta_e08 = als_candidates_df[\"target\"]\n\nmodel_lr_e08 = LogisticRegression(class_weight=\"balanced\", random_state=42)\nmodel_lr_e08.fit(X_meta_e08, y_meta_e08)\n\nalpha_e08 = model_lr_e08.coef_[0][0]\nbeta_e08 = model_lr_e08.coef_[0][1]\nintercept_e08 = model_lr_e08.intercept_[0]\n\nals_candidates_df[\"Hybrid_Score_E08\"] = (\n    alpha_e08 * als_candidates_df[\"Scaled_CF_E08\"] +\n    beta_e08 * als_candidates_df[\"Scaled_Dem_E08\"] +\n    intercept_e08\n)\n\nprint(\"\\n=== E08 Logistic Regression coefficients ===\")\nprint(f\"Alpha / ALS Score_CF: {alpha_e08:.4f}\")\nprint(f\"Beta / CatBoost Score_Dem_E08: {beta_e08:.4f}\")\nprint(f\"Intercept: {intercept_e08:.4f}\")\n\n# ------------------------------------------------------------\n# 12. Evaluation helpers\n# ------------------------------------------------------------\n\ndef precision_at_12_e08(df, score_col):\n    top_12 = (\n        df.sort_values([\"user_idx\", score_col], ascending=[True, False])\n          .groupby(\"user_idx\")\n          .head(12)\n    )\n    user_hits = top_12.groupby(\"user_idx\")[\"target\"].sum()\n    return (user_hits / 12).mean()\n\ndef safe_auc_e08(y_true, score):\n    if len(np.unique(y_true)) < 2:\n        return np.nan\n    return roc_auc_score(y_true, score)\n\n# ------------------------------------------------------------\n# 13. E08 metrics\n# ------------------------------------------------------------\n\ne08_p12 = precision_at_12_e08(als_candidates_df, \"Hybrid_Score_E08\")\n\ne08_uplift_vs_e01 = ((e08_p12 - baseline_p12) / baseline_p12) * 100\ne08_delta_vs_e02 = ((e08_p12 - hybrid_p12) / hybrid_p12) * 100\n\nif pd.isna(e03_p12_for_compare):\n    e08_delta_vs_e03 = np.nan\nelse:\n    e08_delta_vs_e03 = ((e08_p12 - e03_p12_for_compare) / e03_p12_for_compare) * 100\n\nif pd.isna(e07_p12_for_compare):\n    e08_delta_vs_e07 = np.nan\nelse:\n    e08_delta_vs_e07 = ((e08_p12 - e07_p12_for_compare) / e07_p12_for_compare) * 100\n\nauc_score_dem_e08 = safe_auc_e08(\n    als_candidates_df[\"target\"],\n    als_candidates_df[\"Score_Dem_E08\"]\n)\n\nauc_hybrid_e08 = safe_auc_e08(\n    als_candidates_df[\"target\"],\n    als_candidates_df[\"Hybrid_Score_E08\"]\n)\n\nprint(\"\\n=== E08 Precision@12 results ===\")\nprint(f\"E01 ALS baseline Precision@12: {baseline_p12:.6f}\")\nprint(f\"E02 full hybrid Precision@12: {hybrid_p12:.6f}\")\n\nif not pd.isna(e03_p12_for_compare):\n    print(f\"E03 hybrid without inferred_gender Precision@12: {e03_p12_for_compare:.6f}\")\n\nif not pd.isna(e07_p12_for_compare):\n    print(f\"E07 simple date seasonality Precision@12: {e07_p12_for_compare:.6f}\")\n\nprint(f\"E08 train-only item popularity Precision@12: {e08_p12:.6f}\")\nprint(f\"E08 uplift vs E01: {e08_uplift_vs_e01:.2f}%\")\nprint(f\"E08 delta vs E02: {e08_delta_vs_e02:.2f}%\")\n\nif not pd.isna(e08_delta_vs_e03):\n    print(f\"E08 delta vs E03: {e08_delta_vs_e03:.2f}%\")\n\nif not pd.isna(e08_delta_vs_e07):\n    print(f\"E08 delta vs E07: {e08_delta_vs_e07:.2f}%\")\n\nprint(\"\\n=== AUC diagnostic ===\")\nprint(f\"E08 CatBoost Score_Dem_E08 AUC: {auc_score_dem_e08:.6f}\")\nprint(f\"E08 Hybrid_Score_E08 AUC: {auc_hybrid_e08:.6f}\")\n\n# ------------------------------------------------------------\n# 14. Feature importance for E08\n# ------------------------------------------------------------\n\nfi_e08 = (\n    pd.DataFrame({\n        \"feature\": list(model_cb_e08.feature_names_),\n        \"importance\": model_cb_e08.get_feature_importance()\n    })\n    .sort_values(\"importance\", ascending=False)\n    .reset_index(drop=True)\n)\n\nprint(\"\\n=== E08 CatBoost feature importance: top 20 ===\")\ndisplay(fi_e08.head(20))\n\n# ------------------------------------------------------------\n# 15. E08 comparison summary\n# ------------------------------------------------------------\n\nsummary_rows = [\n    {\n        \"experiment_id\": \"E01\",\n        \"name\": \"ALS baseline only\",\n        \"Precision@12\": baseline_p12,\n        \"uplift_vs_E01_%\": 0.0,\n        \"delta_vs_E07_%\": np.nan,\n        \"delta_vs_E08_%\": np.nan,\n        \"notes\": \"Locked baseline\"\n    },\n    {\n        \"experiment_id\": \"E03\",\n        \"name\": \"Hybrid without inferred_gender\",\n        \"Precision@12\": e03_p12_for_compare,\n        \"uplift_vs_E01_%\": (\n            ((e03_p12_for_compare - baseline_p12) / baseline_p12) * 100\n            if not pd.isna(e03_p12_for_compare)\n            else np.nan\n        ),\n        \"delta_vs_E07_%\": (\n            ((e03_p12_for_compare - e07_p12_for_compare) / e07_p12_for_compare) * 100\n            if not pd.isna(e07_p12_for_compare)\n            else np.nan\n        ),\n        \"delta_vs_E08_%\": (\n            ((e03_p12_for_compare - e08_p12) / e08_p12) * 100\n            if not pd.isna(e03_p12_for_compare)\n            else np.nan\n        ),\n        \"notes\": \"Best non-seasonality hybrid\"\n    },\n    {\n        \"experiment_id\": \"E07\",\n        \"name\": \"Simple date seasonality\",\n        \"Precision@12\": e07_p12_for_compare,\n        \"uplift_vs_E01_%\": (\n            ((e07_p12_for_compare - baseline_p12) / baseline_p12) * 100\n            if not pd.isna(e07_p12_for_compare)\n            else np.nan\n        ),\n        \"delta_vs_E07_%\": 0.0,\n        \"delta_vs_E08_%\": (\n            ((e07_p12_for_compare - e08_p12) / e08_p12) * 100\n            if not pd.isna(e07_p12_for_compare)\n            else np.nan\n        ),\n        \"notes\": \"Current best before E08\"\n    },\n    {\n        \"experiment_id\": \"E08\",\n        \"name\": \"Train-only item/category popularity\",\n        \"Precision@12\": e08_p12,\n        \"uplift_vs_E01_%\": e08_uplift_vs_e01,\n        \"delta_vs_E07_%\": e08_delta_vs_e07,\n        \"delta_vs_E08_%\": 0.0,\n        \"notes\": \"E07 feature set + train-only item/category popularity\"\n    }\n]\n\ne08_summary = pd.DataFrame(summary_rows)\n\nprint(\"\\n=== E08 comparison summary ===\")\ndisplay(e08_summary)\n\n# ------------------------------------------------------------\n# 16. Safe decision line\n# ------------------------------------------------------------\n\nif e08_p12 > baseline_p12:\n    e08_vs_baseline_text = \"E08 beats the ALS baseline in this run.\"\nelif e08_p12 < baseline_p12:\n    e08_vs_baseline_text = \"E08 is below the ALS baseline in this run.\"\nelse:\n    e08_vs_baseline_text = \"E08 equals the ALS baseline in this run.\"\n\nif not pd.isna(e07_p12_for_compare):\n    if e08_p12 > e07_p12_for_compare:\n        e08_vs_e07_text = \"E08 beats E07, so train-only popularity becomes the current best candidate.\"\n    elif e08_p12 < e07_p12_for_compare:\n        e08_vs_e07_text = \"E08 is below E07, so train-only popularity did not improve the current best setup.\"\n    else:\n        e08_vs_e07_text = \"E08 matches E07.\"\nelse:\n    e08_vs_e07_text = \"E07 comparison unavailable.\"\n\ne08_decision = (\n    f\"{e08_vs_baseline_text} \"\n    f\"{e08_vs_e07_text} \"\n    \"All popularity features were calculated from train_df only. \"\n    \"Do not describe this as test popularity or future demand.\"\n)\n\nprint(\"\\n=== Safe E08 interpretation ===\")\nprint(e08_decision)\n\nprint(\"\\nObjects to log:\")\nprint(\"- e08_summary\")\nprint(\"- fi_e08\")\nprint(\"- alpha_e08, beta_e08, intercept_e08\")\nprint(\"- auc_score_dem_e08, auc_hybrid_e08\")\nprint(\"- e08_decision\")\n\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-28T20:03:48.165953Z","iopub.execute_input":"2026-05-28T20:03:48.166396Z","iopub.status.idle":"2026-05-28T20:06:22.071552Z","shell.execute_reply.started":"2026-05-28T20:03:48.166363Z","shell.execute_reply":"2026-05-28T20:06:22.069846Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# NEW CELL I - E08-check: Verify popularity/recency windows\n#\n# WHERE TO PLACE:\n# Put this NEW code cell immediately AFTER:\n# NEW CELL H - E08: Train-only item/category popularity\n#\n# PURPOSE:\n# Verify that E08 popularity/recency features are leakage-safe.\n#\n# Checks:\n# - train_df max date <= 2020-09-15\n# - validation anchor = 2020-09-16\n# - recent_7d window ends at train_end_date\n# - recent_14d window ends at train_end_date\n# - no timestamp >= validation_anchor_date is used\n# - item_days_since_last_train_purchase is based on train-only last purchase\n# ============================================================\n\nimport pandas as pd\nimport numpy as np\n\nprint(\"NEW CELL I: E08-check - popularity/recency leakage audit\")\n\n# ------------------------------------------------------------\n# 1. Safety checks\n# ------------------------------------------------------------\n\nrequired_objects = [\n    \"train_df\",\n    \"test_df\"\n]\n\nmissing_objects = [obj for obj in required_objects if obj not in globals()]\n\nif missing_objects:\n    raise RuntimeError(\n        f\"Missing required objects: {missing_objects}. \"\n        \"Run notebook cells up to E08 first.\"\n    )\n\nrequired_train_cols = [\"t_dat\", \"item_idx\", \"user_idx\"]\nmissing_train_cols = [col for col in required_train_cols if col not in train_df.columns]\n\nif missing_train_cols:\n    raise RuntimeError(f\"train_df is missing columns: {missing_train_cols}\")\n\nif \"t_dat\" not in test_df.columns:\n    raise RuntimeError(\"test_df is missing t_dat column.\")\n\n# ------------------------------------------------------------\n# 2. Recompute E08 date boundaries\n# ------------------------------------------------------------\n\ntrain_dates_check = train_df[[\"t_dat\", \"item_idx\", \"user_idx\"]].copy()\ntrain_dates_check[\"t_dat\"] = pd.to_datetime(train_dates_check[\"t_dat\"])\n\ntest_dates_check = test_df[[\"t_dat\"]].copy()\ntest_dates_check[\"t_dat\"] = pd.to_datetime(test_dates_check[\"t_dat\"])\n\ntrain_start_check = train_dates_check[\"t_dat\"].min()\ntrain_end_check = train_dates_check[\"t_dat\"].max()\nvalidation_anchor_check = test_dates_check[\"t_dat\"].min()\ntest_end_check = test_dates_check[\"t_dat\"].max()\n\nrecent_7d_start_check = train_end_check - pd.Timedelta(days=6)\nrecent_14d_start_check = train_end_check - pd.Timedelta(days=13)\n\nprint(\"\\n=== Date boundaries ===\")\nprint(f\"Train start: {train_start_check}\")\nprint(f\"Train end: {train_end_check}\")\nprint(f\"Validation anchor / test start: {validation_anchor_check}\")\nprint(f\"Test end: {test_end_check}\")\nprint(f\"Recent 7d window: {recent_7d_start_check} -> {train_end_check}\")\nprint(f\"Recent 14d window: {recent_14d_start_check} -> {train_end_check}\")\n\n# ------------------------------------------------------------\n# 3. Hard leakage checks\n# ------------------------------------------------------------\n\nviolations = []\n\nif train_end_check >= validation_anchor_check:\n    violations.append(\n        f\"Train max date {train_end_check} is not before validation anchor {validation_anchor_check}.\"\n    )\n\nrecent_7d_source = train_dates_check[\n    train_dates_check[\"t_dat\"] >= recent_7d_start_check\n]\n\nrecent_14d_source = train_dates_check[\n    train_dates_check[\"t_dat\"] >= recent_14d_start_check\n]\n\nrecent_7d_max = recent_7d_source[\"t_dat\"].max()\nrecent_14d_max = recent_14d_source[\"t_dat\"].max()\n\nif recent_7d_max >= validation_anchor_check:\n    violations.append(\n        f\"recent_7d source max date {recent_7d_max} reaches/passes validation anchor.\"\n    )\n\nif recent_14d_max >= validation_anchor_check:\n    violations.append(\n        f\"recent_14d source max date {recent_14d_max} reaches/passes validation anchor.\"\n    )\n\nfuture_rows_in_train_source = train_dates_check[\n    train_dates_check[\"t_dat\"] >= validation_anchor_check\n]\n\nif len(future_rows_in_train_source) > 0:\n    violations.append(\n        f\"Found {len(future_rows_in_train_source)} train source rows with t_dat >= validation anchor.\"\n    )\n\n# ------------------------------------------------------------\n# 4. Recompute item popularity metadata for audit\n# ------------------------------------------------------------\n\nitem_last_purchase_check = (\n    train_dates_check\n    .groupby(\"item_idx\", as_index=False)\n    .agg(item_last_train_purchase=(\"t_dat\", \"max\"))\n)\n\nitem_last_purchase_max = item_last_purchase_check[\"item_last_train_purchase\"].max()\n\nif item_last_purchase_max >= validation_anchor_check:\n    violations.append(\n        f\"item_last_train_purchase max {item_last_purchase_max} reaches/passes validation anchor.\"\n    )\n\nitem_days_since_last_check = (\n    validation_anchor_check - item_last_purchase_check[\"item_last_train_purchase\"]\n).dt.days\n\nmin_days_since_last = item_days_since_last_check.min()\nmax_days_since_last = item_days_since_last_check.max()\n\nif min_days_since_last < 1:\n    violations.append(\n        f\"Minimum days_since_last_train_purchase is {min_days_since_last}. Expected >= 1.\"\n    )\n\nprint(\"\\n=== Recency source audit ===\")\nprint(f\"recent_7d source min date: {recent_7d_source['t_dat'].min()}\")\nprint(f\"recent_7d source max date: {recent_7d_max}\")\nprint(f\"recent_7d source rows: {len(recent_7d_source)}\")\n\nprint(f\"\\nrecent_14d source min date: {recent_14d_source['t_dat'].min()}\")\nprint(f\"recent_14d source max date: {recent_14d_max}\")\nprint(f\"recent_14d source rows: {len(recent_14d_source)}\")\n\nprint(f\"\\nitem_last_train_purchase max date: {item_last_purchase_max}\")\nprint(f\"min days_since_last_train_purchase: {min_days_since_last}\")\nprint(f\"max days_since_last_train_purchase: {max_days_since_last}\")\n\n# ------------------------------------------------------------\n# 5. Optional consistency check if E08 objects exist\n# ------------------------------------------------------------\n\nif \"e08_p12\" in globals():\n    print(\"\\n=== E08 result being audited ===\")\n    print(f\"E08 Precision@12: {e08_p12:.6f}\")\n\nif \"auc_hybrid_e08\" in globals():\n    print(f\"E08 Hybrid AUC: {auc_hybrid_e08:.6f}\")\n\nif \"alpha_e08\" in globals() and \"beta_e08\" in globals():\n    print(f\"E08 LR alpha / CF: {alpha_e08:.4f}\")\n    print(f\"E08 LR beta / context: {beta_e08:.4f}\")\n\n# ------------------------------------------------------------\n# 6. Final decision\n# ------------------------------------------------------------\n\nprint(\"\\n=== E08 leakage audit result ===\")\n\nif len(violations) == 0:\n    e08_leakage_audit_passed = True\n    e08_leakage_audit_decision = (\n        \"PASS: E08 popularity and recency windows use train-period data only. \"\n        \"No timestamp >= validation anchor was found in popularity sources.\"\n    )\n    print(e08_leakage_audit_decision)\nelse:\n    e08_leakage_audit_passed = False\n    e08_leakage_audit_decision = (\n        \"FAIL: Potential leakage found in E08 popularity/recency windows. \"\n        \"Do not lock E08 until fixed.\"\n    )\n    print(e08_leakage_audit_decision)\n    print(\"\\nViolations:\")\n    for v in violations:\n        print(f\"- {v}\")\n\n# Compact object for logging.\ne08_leakage_audit = {\n    \"audit\": \"E08 popularity/recency leakage check\",\n    \"train_start\": str(train_start_check),\n    \"train_end\": str(train_end_check),\n    \"validation_anchor\": str(validation_anchor_check),\n    \"recent_7d_start\": str(recent_7d_start_check),\n    \"recent_7d_max\": str(recent_7d_max),\n    \"recent_14d_start\": str(recent_14d_start_check),\n    \"recent_14d_max\": str(recent_14d_max),\n    \"item_last_train_purchase_max\": str(item_last_purchase_max),\n    \"min_days_since_last_train_purchase\": int(min_days_since_last),\n    \"passed\": e08_leakage_audit_passed,\n    \"decision\": e08_leakage_audit_decision\n}\n\nprint(\"\\nObject to log:\")\nprint(\"- e08_leakage_audit\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-28T20:23:37.864523Z","iopub.execute_input":"2026-05-28T20:23:37.866583Z","iopub.status.idle":"2026-05-28T20:23:38.033577Z","shell.execute_reply.started":"2026-05-28T20:23:37.866511Z","shell.execute_reply":"2026-05-28T20:23:38.032382Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# NEW CELL J - E10: Repeat-purchase candidates + E08 features\n#\n# WHERE TO PLACE:\n# Put this NEW code cell immediately AFTER:\n# NEW CELL I - E08-check\n#\n# PURPOSE:\n# Test whether expanding candidate generation improves Precision@12.\n#\n# E08:\n# - ALS top-50 candidates only\n# - E08 train-only popularity/recency features\n#\n# E10:\n# - ALS top-50 candidates\n# - plus repeat-purchase candidates from each user's train history\n# - same E08 feature logic\n#\n# Leakage rule:\n# Repeat candidates come from train_df only.\n# Popularity/recency features come from train_df only.\n# No test-week candidate generation or popularity is used.\n# ============================================================\n\nimport pandas as pd\nimport numpy as np\nimport gc\n\nfrom catboost import CatBoostClassifier\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import roc_auc_score\n\nprint(\"NEW CELL J: E10 - Repeat-purchase candidates + E08 features\")\n\n# ------------------------------------------------------------\n# 1. Safety checks\n# ------------------------------------------------------------\n\nrequired_objects = [\n    \"train_df\",\n    \"test_df\",\n    \"catboost_train_df\",\n    \"als_candidates_df\",\n    \"baseline_p12\",\n    \"hybrid_p12\"\n]\n\nmissing_objects = [obj for obj in required_objects if obj not in globals()]\n\nif missing_objects:\n    raise RuntimeError(\n        f\"Missing required objects: {missing_objects}. \"\n        \"Run cells up to E08-check first.\"\n    )\n\ne08_p12_for_compare = e08_p12 if \"e08_p12\" in globals() else np.nan\n\n# ------------------------------------------------------------\n# 2. Define helpers\n# ------------------------------------------------------------\n\ndef precision_at_12_e10(df, score_col):\n    top_12 = (\n        df.sort_values([\"user_idx\", score_col], ascending=[True, False])\n          .groupby(\"user_idx\")\n          .head(12)\n    )\n    user_hits = top_12.groupby(\"user_idx\")[\"target\"].sum()\n    return (user_hits / 12).mean()\n\ndef safe_auc_e10(y_true, score):\n    if len(np.unique(y_true)) < 2:\n        return np.nan\n    return roc_auc_score(y_true, score)\n\ndef add_simple_date_features_e10(df_in, date_col, window_start_date):\n    df_out = df_in.copy()\n    dt = pd.to_datetime(df_out[date_col])\n\n    df_out[\"month\"] = dt.dt.month.astype(str)\n    df_out[\"weekofyear\"] = dt.dt.isocalendar().week.astype(int).astype(str)\n    df_out[\"quarter\"] = dt.dt.quarter.astype(str)\n    df_out[\"dayofweek\"] = dt.dt.dayofweek.astype(str)\n\n    month_num = dt.dt.month\n\n    conditions = [\n        month_num.isin([12, 1, 2]),\n        month_num.isin([3, 4, 5]),\n        month_num.isin([6, 7, 8]),\n        month_num.isin([9, 10, 11])\n    ]\n\n    choices = [\"winter\", \"spring\", \"summer\", \"autumn\"]\n\n    df_out[\"season_bucket\"] = np.select(\n        conditions,\n        choices,\n        default=\"unknown\"\n    ).astype(str)\n\n    df_out[\"days_since_window_start\"] = (\n        dt - pd.to_datetime(window_start_date)\n    ).dt.days.astype(\"int16\")\n\n    return df_out\n\n# ------------------------------------------------------------\n# 3. Feature lists\n# ------------------------------------------------------------\n\nbase_features_e10 = [\n    \"age\",\n    \"postal_code\",\n    \"product_type_name\",\n    \"graphical_appearance_name\",\n    \"colour_group_name\",\n    \"perceived_colour_value_name\",\n    \"index_name\",\n    \"index_group_name\"\n]\n\ndate_features_e10 = [\n    \"month\",\n    \"weekofyear\",\n    \"quarter\",\n    \"dayofweek\",\n    \"season_bucket\",\n    \"days_since_window_start\"\n]\n\npopularity_features_e10 = [\n    \"item_train_count\",\n    \"item_train_unique_users\",\n    \"item_train_recent_7d_count\",\n    \"item_train_recent_14d_count\",\n    \"item_days_since_last_train_purchase\",\n    \"product_type_train_count\",\n    \"index_name_train_count\",\n    \"colour_group_train_count\"\n]\n\ncandidate_source_features_e10 = [\n    \"source_als\",\n    \"source_repeat\",\n    \"candidate_source_count\"\n]\n\nall_features_e10 = (\n    base_features_e10\n    + date_features_e10\n    + popularity_features_e10\n    + candidate_source_features_e10\n)\n\n# ------------------------------------------------------------\n# 4. Build expanded candidate pool\n# ------------------------------------------------------------\n\nprint(\"\\nBuilding expanded candidate pool...\")\n\n# Existing ALS candidates\ne10_als_candidates = als_candidates_df.copy()\ne10_als_candidates[\"source_als\"] = 1\ne10_als_candidates[\"source_repeat\"] = 0\n\n# Repeat candidates: each test user gets their recent train-history items.\ntest_user_ids_e10 = als_candidates_df[\"user_idx\"].unique()\n\ntrain_history_e10 = train_df[\n    train_df[\"user_idx\"].isin(test_user_ids_e10)\n][[\"user_idx\", \"item_idx\", \"t_dat\"]].copy()\n\ntrain_history_e10[\"t_dat\"] = pd.to_datetime(train_history_e10[\"t_dat\"])\n\n# Keep recent repeat items per user.\n# Cap to avoid exploding candidate pool.\nrepeat_candidates_per_user_e10 = 20\n\nrepeat_candidates_e10 = (\n    train_history_e10\n    .sort_values([\"user_idx\", \"t_dat\"], ascending=[True, False])\n    .drop_duplicates([\"user_idx\", \"item_idx\"])\n    .groupby(\"user_idx\")\n    .head(repeat_candidates_per_user_e10)\n    [[\"user_idx\", \"item_idx\"]]\n    .copy()\n)\n\nrepeat_candidates_e10[\"source_als\"] = 0\nrepeat_candidates_e10[\"source_repeat\"] = 1\n\nprint(f\"ALS candidate rows before expansion: {len(e10_als_candidates):,}\")\nprint(f\"Repeat candidate rows before dedupe: {len(repeat_candidates_e10):,}\")\n\n# Keep only columns needed for candidate identity/sources before union.\nals_identity_e10 = e10_als_candidates[\n    [\"user_idx\", \"item_idx\", \"source_als\", \"source_repeat\"]\n].copy()\n\ncandidate_union_e10 = pd.concat(\n    [als_identity_e10, repeat_candidates_e10],\n    ignore_index=True\n)\n\ncandidate_union_e10 = (\n    candidate_union_e10\n    .groupby([\"user_idx\", \"item_idx\"], as_index=False)\n    .agg(\n        source_als=(\"source_als\", \"max\"),\n        source_repeat=(\"source_repeat\", \"max\")\n    )\n)\n\ncandidate_union_e10[\"candidate_source_count\"] = (\n    candidate_union_e10[\"source_als\"] + candidate_union_e10[\"source_repeat\"]\n)\n\nprint(f\"Candidate rows after dedupe: {len(candidate_union_e10):,}\")\nprint(\"\\nCandidate source distribution:\")\nprint(candidate_union_e10[[\"source_als\", \"source_repeat\", \"candidate_source_count\"]].value_counts())\n\n# ------------------------------------------------------------\n# 5. Attach target labels\n# ------------------------------------------------------------\n\ntest_pairs_e10 = (\n    test_df[[\"user_idx\", \"item_idx\"]]\n    .drop_duplicates()\n    .copy()\n)\n\ntest_pairs_e10[\"target_e10\"] = 1\n\ne10_candidates_df = candidate_union_e10.merge(\n    test_pairs_e10,\n    on=[\"user_idx\", \"item_idx\"],\n    how=\"left\"\n)\n\ne10_candidates_df[\"target\"] = e10_candidates_df[\"target_e10\"].fillna(0).astype(\"int8\")\ne10_candidates_df = e10_candidates_df.drop(columns=[\"target_e10\"])\n\nprint(\"\\nE10 target distribution:\")\nprint(e10_candidates_df[\"target\"].value_counts(dropna=False))\n\n# ------------------------------------------------------------\n# 6. Attach Score_CF\n# ------------------------------------------------------------\n\n# Existing ALS score where available.\nals_score_lookup_e10 = als_candidates_df[[\"user_idx\", \"item_idx\", \"Score_CF\"]].copy()\n\ne10_candidates_df = e10_candidates_df.merge(\n    als_score_lookup_e10,\n    on=[\"user_idx\", \"item_idx\"],\n    how=\"left\"\n)\n\n# Repeat-only candidates did not come from ALS top-50.\n# Assign low CF score, then let CatBoost/context decide.\nmin_cf_score_e10 = als_candidates_df[\"Score_CF\"].min()\ne10_candidates_df[\"Score_CF\"] = e10_candidates_df[\"Score_CF\"].fillna(min_cf_score_e10)\n\n# ------------------------------------------------------------\n# 7. Attach customer/article features\n# ------------------------------------------------------------\n\nprint(\"\\nAttaching customer/article features...\")\n\n# Reuse feature lookup from existing candidates and training rows where possible.\ncandidate_feature_lookup_e10 = pd.concat(\n    [\n        als_candidates_df[[\"user_idx\", \"item_idx\"] + base_features_e10],\n        catboost_train_df[[\"user_idx\", \"item_idx\"] + base_features_e10]\n    ],\n    ignore_index=True\n).drop_duplicates([\"user_idx\", \"item_idx\"])\n\ne10_candidates_df = e10_candidates_df.merge(\n    candidate_feature_lookup_e10,\n    on=[\"user_idx\", \"item_idx\"],\n    how=\"left\"\n)\n\nmissing_feature_rows_e10 = e10_candidates_df[base_features_e10].isna().any(axis=1).sum()\nprint(f\"Rows with at least one missing base feature after lookup: {missing_feature_rows_e10:,}\")\n\n# Fallback: item-level article metadata + user-level customer features from known rows.\nif missing_feature_rows_e10 > 0:\n    item_feature_lookup_e10 = (\n        pd.concat(\n            [\n                als_candidates_df[[\"item_idx\"] + [\n                    \"product_type_name\",\n                    \"graphical_appearance_name\",\n                    \"colour_group_name\",\n                    \"perceived_colour_value_name\",\n                    \"index_name\",\n                    \"index_group_name\"\n                ]],\n                catboost_train_df[[\"item_idx\"] + [\n                    \"product_type_name\",\n                    \"graphical_appearance_name\",\n                    \"colour_group_name\",\n                    \"perceived_colour_value_name\",\n                    \"index_name\",\n                    \"index_group_name\"\n                ]]\n            ],\n            ignore_index=True\n        )\n        .drop_duplicates(\"item_idx\")\n    )\n\n    user_feature_lookup_e10 = (\n        pd.concat(\n            [\n                als_candidates_df[[\"user_idx\", \"age\", \"postal_code\"]],\n                catboost_train_df[[\"user_idx\", \"age\", \"postal_code\"]]\n            ],\n            ignore_index=True\n        )\n        .drop_duplicates(\"user_idx\")\n    )\n\n    e10_candidates_df = e10_candidates_df.merge(\n        item_feature_lookup_e10,\n        on=\"item_idx\",\n        how=\"left\",\n        suffixes=(\"\", \"_item_lookup\")\n    )\n\n    e10_candidates_df = e10_candidates_df.merge(\n        user_feature_lookup_e10,\n        on=\"user_idx\",\n        how=\"left\",\n        suffixes=(\"\", \"_user_lookup\")\n    )\n\n    for col in [\n        \"product_type_name\",\n        \"graphical_appearance_name\",\n        \"colour_group_name\",\n        \"perceived_colour_value_name\",\n        \"index_name\",\n        \"index_group_name\"\n    ]:\n        lookup_col = f\"{col}_item_lookup\"\n        if lookup_col in e10_candidates_df.columns:\n            e10_candidates_df[col] = e10_candidates_df[col].fillna(e10_candidates_df[lookup_col])\n            e10_candidates_df = e10_candidates_df.drop(columns=[lookup_col])\n\n    for col in [\"age\", \"postal_code\"]:\n        lookup_col = f\"{col}_user_lookup\"\n        if lookup_col in e10_candidates_df.columns:\n            e10_candidates_df[col] = e10_candidates_df[col].fillna(e10_candidates_df[lookup_col])\n            e10_candidates_df = e10_candidates_df.drop(columns=[lookup_col])\n\n# ------------------------------------------------------------\n# 8. Rebuild E08-style train-safe date/popularity features\n# ------------------------------------------------------------\n\nprint(\"\\nBuilding E08-style train-only features for E10...\")\n\ntrain_df_dates_e10 = train_df[[\"user_idx\", \"item_idx\", \"t_dat\"]].copy()\ntrain_df_dates_e10[\"t_dat\"] = pd.to_datetime(train_df_dates_e10[\"t_dat\"])\n\nwindow_start_date_e10 = train_df_dates_e10[\"t_dat\"].min()\ntrain_end_date_e10 = train_df_dates_e10[\"t_dat\"].max()\nvalidation_anchor_date_e10 = pd.to_datetime(test_df[\"t_dat\"]).min()\n\nrecent_7d_start_e10 = train_end_date_e10 - pd.Timedelta(days=6)\nrecent_14d_start_e10 = train_end_date_e10 - pd.Timedelta(days=13)\n\nif train_end_date_e10 >= validation_anchor_date_e10:\n    raise RuntimeError(\"Leakage risk: train_end_date is not before validation anchor.\")\n\nprint(f\"Train window start date: {window_start_date_e10}\")\nprint(f\"Train end date: {train_end_date_e10}\")\nprint(f\"Validation anchor date: {validation_anchor_date_e10}\")\n\n# Train-side date anchors for catboost_train_df\npositive_pair_dates_e10 = (\n    train_df_dates_e10\n    .sort_values(\"t_dat\")\n    .groupby([\"user_idx\", \"item_idx\"], as_index=False)\n    .agg(pair_train_date=(\"t_dat\", \"max\"))\n)\n\nuser_anchor_dates_e10 = (\n    train_df_dates_e10\n    .groupby(\"user_idx\", as_index=False)\n    .agg(user_train_anchor_date=(\"t_dat\", \"max\"))\n)\n\ne10_train_df = catboost_train_df.copy()\n\ne10_train_df = e10_train_df.merge(\n    positive_pair_dates_e10,\n    on=[\"user_idx\", \"item_idx\"],\n    how=\"left\"\n)\n\ne10_train_df = e10_train_df.merge(\n    user_anchor_dates_e10,\n    on=\"user_idx\",\n    how=\"left\"\n)\n\ne10_train_df[\"season_anchor_date\"] = e10_train_df[\"pair_train_date\"]\ne10_train_df[\"season_anchor_date\"] = e10_train_df[\"season_anchor_date\"].fillna(\n    e10_train_df[\"user_train_anchor_date\"]\n)\ne10_train_df[\"season_anchor_date\"] = e10_train_df[\"season_anchor_date\"].fillna(\n    window_start_date_e10\n)\n\ne10_train_df = add_simple_date_features_e10(\n    e10_train_df,\n    date_col=\"season_anchor_date\",\n    window_start_date=window_start_date_e10\n)\n\n# Candidate date anchor\ne10_candidates_df[\"season_anchor_date\"] = validation_anchor_date_e10\n\ne10_candidates_df = add_simple_date_features_e10(\n    e10_candidates_df,\n    date_col=\"season_anchor_date\",\n    window_start_date=window_start_date_e10\n)\n\n# Item popularity\nitem_pop_e10 = (\n    train_df_dates_e10\n    .groupby(\"item_idx\")\n    .agg(\n        item_train_count=(\"user_idx\", \"size\"),\n        item_train_unique_users=(\"user_idx\", \"nunique\"),\n        item_last_train_purchase=(\"t_dat\", \"max\")\n    )\n    .reset_index()\n)\n\nrecent_7d_counts_e10 = (\n    train_df_dates_e10[train_df_dates_e10[\"t_dat\"] >= recent_7d_start_e10]\n    .groupby(\"item_idx\")\n    .size()\n    .reset_index(name=\"item_train_recent_7d_count\")\n)\n\nrecent_14d_counts_e10 = (\n    train_df_dates_e10[train_df_dates_e10[\"t_dat\"] >= recent_14d_start_e10]\n    .groupby(\"item_idx\")\n    .size()\n    .reset_index(name=\"item_train_recent_14d_count\")\n)\n\nitem_pop_e10 = item_pop_e10.merge(\n    recent_7d_counts_e10,\n    on=\"item_idx\",\n    how=\"left\"\n)\n\nitem_pop_e10 = item_pop_e10.merge(\n    recent_14d_counts_e10,\n    on=\"item_idx\",\n    how=\"left\"\n)\n\nitem_pop_e10[\"item_train_recent_7d_count\"] = (\n    item_pop_e10[\"item_train_recent_7d_count\"].fillna(0)\n)\n\nitem_pop_e10[\"item_train_recent_14d_count\"] = (\n    item_pop_e10[\"item_train_recent_14d_count\"].fillna(0)\n)\n\nitem_pop_e10[\"item_days_since_last_train_purchase\"] = (\n    validation_anchor_date_e10 - item_pop_e10[\"item_last_train_purchase\"]\n).dt.days\n\nitem_pop_e10 = item_pop_e10.drop(columns=[\"item_last_train_purchase\"])\n\n# Category popularity\ncategory_cols_e10 = [\n    \"product_type_name\",\n    \"index_name\",\n    \"colour_group_name\"\n]\n\nitem_category_lookup_e10 = (\n    catboost_train_df[[\"item_idx\"] + category_cols_e10]\n    .drop_duplicates(\"item_idx\")\n)\n\ntrain_with_categories_e10 = train_df[[\"item_idx\"]].merge(\n    item_category_lookup_e10,\n    on=\"item_idx\",\n    how=\"left\"\n)\n\nproduct_type_pop_e10 = (\n    train_with_categories_e10\n    .groupby(\"product_type_name\")\n    .size()\n    .reset_index(name=\"product_type_train_count\")\n)\n\nindex_name_pop_e10 = (\n    train_with_categories_e10\n    .groupby(\"index_name\")\n    .size()\n    .reset_index(name=\"index_name_train_count\")\n)\n\ncolour_group_pop_e10 = (\n    train_with_categories_e10\n    .groupby(\"colour_group_name\")\n    .size()\n    .reset_index(name=\"colour_group_train_count\")\n)\n\n# Merge popularity into both train and candidates.\nfor df_name, df_tmp in [(\"train\", e10_train_df), (\"candidates\", e10_candidates_df)]:\n    df_tmp = df_tmp.merge(item_pop_e10, on=\"item_idx\", how=\"left\")\n    df_tmp = df_tmp.merge(product_type_pop_e10, on=\"product_type_name\", how=\"left\")\n    df_tmp = df_tmp.merge(index_name_pop_e10, on=\"index_name\", how=\"left\")\n    df_tmp = df_tmp.merge(colour_group_pop_e10, on=\"colour_group_name\", how=\"left\")\n\n    for col in popularity_features_e10:\n        df_tmp[col] = pd.to_numeric(df_tmp[col], errors=\"coerce\").fillna(0)\n\n    if df_name == \"train\":\n        e10_train_df = df_tmp\n    else:\n        e10_candidates_df = df_tmp\n\n# Train source indicators.\n# Training rows are not candidate-generation rows. Set neutral source flags.\ne10_train_df[\"source_als\"] = 0\ne10_train_df[\"source_repeat\"] = 0\ne10_train_df[\"candidate_source_count\"] = 0\n\n# ------------------------------------------------------------\n# 9. Prepare model matrices\n# ------------------------------------------------------------\n\nX_e10 = e10_train_df[all_features_e10].copy()\ny_e10 = e10_train_df[\"target\"].copy()\n\nX_test_e10 = e10_candidates_df[all_features_e10].copy()\n\ncategorical_features_e10 = [\n    \"postal_code\",\n    \"product_type_name\",\n    \"graphical_appearance_name\",\n    \"colour_group_name\",\n    \"perceived_colour_value_name\",\n    \"index_name\",\n    \"index_group_name\",\n    \"month\",\n    \"weekofyear\",\n    \"quarter\",\n    \"dayofweek\",\n    \"season_bucket\"\n]\n\nnumeric_features_e10 = [\n    \"age\",\n    \"days_since_window_start\"\n] + popularity_features_e10 + candidate_source_features_e10\n\nage_median_e10 = X_e10[\"age\"].median()\n\nfor df_tmp in [X_e10, X_test_e10]:\n    df_tmp[\"age\"] = pd.to_numeric(df_tmp[\"age\"], errors=\"coerce\").fillna(age_median_e10)\n\n    for col in numeric_features_e10:\n        df_tmp[col] = pd.to_numeric(df_tmp[col], errors=\"coerce\").fillna(0)\n\n    for col in categorical_features_e10:\n        df_tmp[col] = df_tmp[col].fillna(\"Unknown\").astype(str)\n\nif \"inferred_gender\" in X_e10.columns:\n    raise RuntimeError(\"E10 failed: inferred_gender is present.\")\n\nprint(\"\\nE10 feature set:\")\nprint(list(X_e10.columns))\n\nprint(\"\\nLeakage/source checks:\")\nprint(f\"Train end date: {train_end_date_e10}\")\nprint(f\"Validation anchor date: {validation_anchor_date_e10}\")\nprint(f\"recent_7d max date: {train_df_dates_e10[train_df_dates_e10['t_dat'] >= recent_7d_start_e10]['t_dat'].max()}\")\nprint(f\"recent_14d max date: {train_df_dates_e10[train_df_dates_e10['t_dat'] >= recent_14d_start_e10]['t_dat'].max()}\")\nprint(f\"inferred_gender in E10 features: {'inferred_gender' in X_e10.columns}\")\nprint(\"postal_code is used only as a hashed categorical proxy, not geography.\")\n\n# ------------------------------------------------------------\n# 10. Train CatBoost for E10\n# ------------------------------------------------------------\n\nprint(\"\\nTraining CatBoost E10...\")\n\nmodel_cb_e10 = CatBoostClassifier(\n    iterations=100,\n    learning_rate=0.1,\n    depth=6,\n    loss_function=\"Logloss\",\n    eval_metric=\"AUC\",\n    random_seed=42,\n    verbose=20\n)\n\nmodel_cb_e10.fit(X_e10, y_e10, cat_features=categorical_features_e10)\n\n# ------------------------------------------------------------\n# 11. Score expanded candidate pool\n# ------------------------------------------------------------\n\nprint(\"\\nScoring expanded E10 candidates...\")\n\ne10_candidates_df[\"Score_Dem_E10\"] = model_cb_e10.predict_proba(X_test_e10)[:, 1]\n\n# ------------------------------------------------------------\n# 12. Logistic Regression fusion on expanded candidate pool\n# ------------------------------------------------------------\n\nprint(\"\\nTraining Logistic Regression fusion for E10...\")\n\nscaler_e10 = StandardScaler()\n\nscaled_e10 = scaler_e10.fit_transform(\n    e10_candidates_df[[\"Score_CF\", \"Score_Dem_E10\"]]\n)\n\ne10_candidates_df[\"Scaled_CF_E10\"] = scaled_e10[:, 0]\ne10_candidates_df[\"Scaled_Dem_E10\"] = scaled_e10[:, 1]\n\nX_meta_e10 = e10_candidates_df[[\"Scaled_CF_E10\", \"Scaled_Dem_E10\"]]\ny_meta_e10 = e10_candidates_df[\"target\"]\n\nmodel_lr_e10 = LogisticRegression(class_weight=\"balanced\", random_state=42)\nmodel_lr_e10.fit(X_meta_e10, y_meta_e10)\n\nalpha_e10 = model_lr_e10.coef_[0][0]\nbeta_e10 = model_lr_e10.coef_[0][1]\nintercept_e10 = model_lr_e10.intercept_[0]\n\ne10_candidates_df[\"Hybrid_Score_E10\"] = (\n    alpha_e10 * e10_candidates_df[\"Scaled_CF_E10\"] +\n    beta_e10 * e10_candidates_df[\"Scaled_Dem_E10\"] +\n    intercept_e10\n)\n\nprint(\"\\n=== E10 Logistic Regression coefficients ===\")\nprint(f\"Alpha / ALS Score_CF: {alpha_e10:.4f}\")\nprint(f\"Beta / CatBoost Score_Dem_E10: {beta_e10:.4f}\")\nprint(f\"Intercept: {intercept_e10:.4f}\")\n\n# ------------------------------------------------------------\n# 13. Metrics\n# ------------------------------------------------------------\n\ne10_p12 = precision_at_12_e10(e10_candidates_df, \"Hybrid_Score_E10\")\n\ne10_uplift_vs_e01 = ((e10_p12 - baseline_p12) / baseline_p12) * 100\n\nif pd.isna(e08_p12_for_compare):\n    e10_delta_vs_e08 = np.nan\nelse:\n    e10_delta_vs_e08 = ((e10_p12 - e08_p12_for_compare) / e08_p12_for_compare) * 100\n\nauc_score_dem_e10 = safe_auc_e10(\n    e10_candidates_df[\"target\"],\n    e10_candidates_df[\"Score_Dem_E10\"]\n)\n\nauc_hybrid_e10 = safe_auc_e10(\n    e10_candidates_df[\"target\"],\n    e10_candidates_df[\"Hybrid_Score_E10\"]\n)\n\nprint(\"\\n=== E10 Precision@12 results ===\")\nprint(f\"E01 ALS baseline Precision@12: {baseline_p12:.6f}\")\n\nif not pd.isna(e08_p12_for_compare):\n    print(f\"E08 current best Precision@12: {e08_p12_for_compare:.6f}\")\n\nprint(f\"E10 repeat-candidate model Precision@12: {e10_p12:.6f}\")\nprint(f\"E10 uplift vs E01: {e10_uplift_vs_e01:.2f}%\")\n\nif not pd.isna(e10_delta_vs_e08):\n    print(f\"E10 delta vs E08: {e10_delta_vs_e08:.2f}%\")\n\nprint(\"\\n=== AUC diagnostic ===\")\nprint(f\"E10 CatBoost Score_Dem_E10 AUC: {auc_score_dem_e10:.6f}\")\nprint(f\"E10 Hybrid_Score_E10 AUC: {auc_hybrid_e10:.6f}\")\n\n# Candidate source analysis\ntop12_e10 = (\n    e10_candidates_df\n    .sort_values([\"user_idx\", \"Hybrid_Score_E10\"], ascending=[True, False])\n    .groupby(\"user_idx\")\n    .head(12)\n)\n\nprint(\"\\n=== E10 candidate source diagnostics ===\")\nprint(\"All candidates:\")\nprint(e10_candidates_df[[\"source_als\", \"source_repeat\", \"candidate_source_count\"]].value_counts())\n\nprint(\"\\nTop-12 candidates:\")\nprint(top12_e10[[\"source_als\", \"source_repeat\", \"candidate_source_count\"]].value_counts())\n\nprint(\"\\nHits in top-12 by source:\")\ndisplay(\n    top12_e10\n    .groupby([\"source_als\", \"source_repeat\"])[\"target\"]\n    .agg([\"sum\", \"count\", \"mean\"])\n    .reset_index()\n)\n\n# ------------------------------------------------------------\n# 14. Feature importance\n# ------------------------------------------------------------\n\nfi_e10 = (\n    pd.DataFrame({\n        \"feature\": list(model_cb_e10.feature_names_),\n        \"importance\": model_cb_e10.get_feature_importance()\n    })\n    .sort_values(\"importance\", ascending=False)\n    .reset_index(drop=True)\n)\n\nprint(\"\\n=== E10 CatBoost feature importance: top 20 ===\")\ndisplay(fi_e10.head(20))\n\n# ------------------------------------------------------------\n# 15. Summary\n# ------------------------------------------------------------\n\ne10_summary = pd.DataFrame([\n    {\n        \"experiment_id\": \"E01\",\n        \"name\": \"ALS baseline only\",\n        \"Precision@12\": baseline_p12,\n        \"uplift_vs_E01_%\": 0.0,\n        \"delta_vs_E08_%\": np.nan,\n        \"notes\": \"Locked baseline\"\n    },\n    {\n        \"experiment_id\": \"E08\",\n        \"name\": \"Train-only item/category popularity\",\n        \"Precision@12\": e08_p12_for_compare,\n        \"uplift_vs_E01_%\": (\n            ((e08_p12_for_compare - baseline_p12) / baseline_p12) * 100\n            if not pd.isna(e08_p12_for_compare)\n            else np.nan\n        ),\n        \"delta_vs_E08_%\": 0.0,\n        \"notes\": \"Current best before E10\"\n    },\n    {\n        \"experiment_id\": \"E10\",\n        \"name\": \"Repeat-purchase candidates + E08 features\",\n        \"Precision@12\": e10_p12,\n        \"uplift_vs_E01_%\": e10_uplift_vs_e01,\n        \"delta_vs_E08_%\": e10_delta_vs_e08,\n        \"notes\": \"ALS top-50 + repeat-purchase candidates from train history\"\n    }\n])\n\nprint(\"\\n=== E10 comparison summary ===\")\ndisplay(e10_summary)\n\nif e10_p12 > e08_p12_for_compare:\n    e10_decision = (\n        \"E10 improves over E08. Repeat-purchase candidates add useful recall beyond ALS top-50.\"\n    )\nelif e10_p12 < e08_p12_for_compare:\n    e10_decision = (\n        \"E10 is below E08. Repeat-purchase candidate expansion did not improve this setup.\"\n    )\nelse:\n    e10_decision = (\n        \"E10 matches E08. Repeat-purchase candidates did not materially change Precision@12.\"\n    )\n\nprint(\"\\n=== Safe E10 interpretation ===\")\nprint(e10_decision)\n\nprint(\"\\nObjects to log:\")\nprint(\"- e10_summary\")\nprint(\"- fi_e10\")\nprint(\"- e10_candidates_df\")\nprint(\"- alpha_e10, beta_e10, intercept_e10\")\nprint(\"- auc_score_dem_e10, auc_hybrid_e10\")\nprint(\"- e10_decision\")\n\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-28T20:27:20.494590Z","iopub.execute_input":"2026-05-28T20:27:20.495150Z","iopub.status.idle":"2026-05-28T20:30:04.133215Z","shell.execute_reply.started":"2026-05-28T20:27:20.495113Z","shell.execute_reply":"2026-05-28T20:30:04.131657Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# NEW CELL K - E10A: Candidate source recall diagnostic\n#\n# PLACE AFTER:\n# NEW CELL J - E10\n#\n# PURPOSE:\n# Diagnose whether repeat candidates are useful but under-ranked.\n# No model training.\n# ============================================================\n\nimport pandas as pd\nimport numpy as np\n\nprint(\"NEW CELL K: E10A - Candidate source recall diagnostic\")\n\n# ------------------------------------------------------------\n# 1. Safety checks\n# ------------------------------------------------------------\n\nrequired_objects = [\"e10_candidates_df\", \"e10_p12\"]\nmissing_objects = [obj for obj in required_objects if obj not in globals()]\n\nif missing_objects:\n    raise RuntimeError(f\"Missing required objects: {missing_objects}. Run E10 first.\")\n\nrequired_cols = [\n    \"user_idx\",\n    \"item_idx\",\n    \"target\",\n    \"source_als\",\n    \"source_repeat\",\n    \"candidate_source_count\",\n    \"Score_CF\",\n    \"Score_Dem_E10\",\n    \"Hybrid_Score_E10\"\n]\n\nmissing_cols = [col for col in required_cols if col not in e10_candidates_df.columns]\n\nif missing_cols:\n    raise RuntimeError(f\"e10_candidates_df is missing columns: {missing_cols}\")\n\ne08_p12_for_compare = e08_p12 if \"e08_p12\" in globals() else np.nan\n\n# ------------------------------------------------------------\n# 2. Rank E10 candidates\n# ------------------------------------------------------------\n\ne10_ranked = (\n    e10_candidates_df\n    .sort_values([\"user_idx\", \"Hybrid_Score_E10\"], ascending=[True, False])\n    .copy()\n)\n\ne10_ranked[\"rank_e10\"] = e10_ranked.groupby(\"user_idx\").cumcount() + 1\n\nconditions = [\n    (e10_ranked[\"source_als\"] == 1) & (e10_ranked[\"source_repeat\"] == 1),\n    (e10_ranked[\"source_als\"] == 1) & (e10_ranked[\"source_repeat\"] == 0),\n    (e10_ranked[\"source_als\"] == 0) & (e10_ranked[\"source_repeat\"] == 1)\n]\n\nchoices = [\"als+repeat\", \"als_only\", \"repeat_only\"]\n\ne10_ranked[\"candidate_source_label\"] = np.select(\n    conditions,\n    choices,\n    default=\"unknown\"\n)\n\ntop12_e10a = e10_ranked[e10_ranked[\"rank_e10\"] <= 12].copy()\n\n# ------------------------------------------------------------\n# 3. Overall diagnostics\n# ------------------------------------------------------------\n\nusers_total = e10_ranked[\"user_idx\"].nunique()\ncandidate_rows_total = len(e10_ranked)\ntop12_rows_total = len(top12_e10a)\n\ncandidate_positives_total = int(e10_ranked[\"target\"].sum())\ntop12_hits_total = int(top12_e10a[\"target\"].sum())\n\nprecision_at_12_from_hits = top12_hits_total / top12_rows_total\n\nprint(\"\\n=== E10A overall diagnostics ===\")\nprint(f\"Users: {users_total:,}\")\nprint(f\"Candidate rows: {candidate_rows_total:,}\")\nprint(f\"Top-12 rows: {top12_rows_total:,}\")\nprint(f\"Candidate positives: {candidate_positives_total:,}\")\nprint(f\"Top-12 hits: {top12_hits_total:,}\")\nprint(f\"Precision@12 from hits: {precision_at_12_from_hits:.6f}\")\nprint(f\"E10 stored Precision@12: {e10_p12:.6f}\")\n\nif not pd.isna(e08_p12_for_compare):\n    print(f\"E08 Precision@12: {e08_p12_for_compare:.6f}\")\n    print(f\"E10 delta vs E08: {((e10_p12 - e08_p12_for_compare) / e08_p12_for_compare) * 100:.2f}%\")\n\n# ------------------------------------------------------------\n# 4. Source-level report\n# ------------------------------------------------------------\n\nall_source_report = (\n    e10_ranked\n    .groupby(\"candidate_source_label\")\n    .agg(\n        candidate_rows=(\"item_idx\", \"size\"),\n        candidate_positives=(\"target\", \"sum\"),\n        users_with_source=(\"user_idx\", \"nunique\"),\n        mean_score_cf=(\"Score_CF\", \"mean\"),\n        mean_score_dem=(\"Score_Dem_E10\", \"mean\"),\n        mean_hybrid_score=(\"Hybrid_Score_E10\", \"mean\"),\n        median_rank=(\"rank_e10\", \"median\")\n    )\n    .reset_index()\n)\n\ntop12_source_report = (\n    top12_e10a\n    .groupby(\"candidate_source_label\")\n    .agg(\n        top12_rows=(\"item_idx\", \"size\"),\n        top12_hits=(\"target\", \"sum\"),\n        users_with_source_in_top12=(\"user_idx\", \"nunique\"),\n        mean_top12_score_cf=(\"Score_CF\", \"mean\"),\n        mean_top12_score_dem=(\"Score_Dem_E10\", \"mean\"),\n        mean_top12_hybrid_score=(\"Hybrid_Score_E10\", \"mean\")\n    )\n    .reset_index()\n)\n\ne10a_source_report = all_source_report.merge(\n    top12_source_report,\n    on=\"candidate_source_label\",\n    how=\"left\"\n)\n\nfor col in [\"top12_rows\", \"top12_hits\", \"users_with_source_in_top12\"]:\n    e10a_source_report[col] = e10a_source_report[col].fillna(0)\n\ne10a_source_report[\"candidate_hit_rate\"] = (\n    e10a_source_report[\"candidate_positives\"] / e10a_source_report[\"candidate_rows\"]\n)\n\ne10a_source_report[\"top12_selection_rate\"] = (\n    e10a_source_report[\"top12_rows\"] / e10a_source_report[\"candidate_rows\"]\n)\n\ne10a_source_report[\"top12_hit_rate\"] = np.where(\n    e10a_source_report[\"top12_rows\"] > 0,\n    e10a_source_report[\"top12_hits\"] / e10a_source_report[\"top12_rows\"],\n    np.nan\n)\n\ne10a_source_report[\"share_of_top12_hits\"] = np.where(\n    top12_hits_total > 0,\n    e10a_source_report[\"top12_hits\"] / top12_hits_total,\n    np.nan\n)\n\ne10a_source_report = e10a_source_report.sort_values(\n    \"top12_hit_rate\",\n    ascending=False\n)\n\nprint(\"\\n=== E10A source-level report ===\")\ndisplay(e10a_source_report)\n\n# ------------------------------------------------------------\n# 5. Repeat-candidate diagnostics\n# ------------------------------------------------------------\n\nrepeat_all = e10_ranked[e10_ranked[\"source_repeat\"] == 1].copy()\nrepeat_pos = repeat_all[repeat_all[\"target\"] == 1].copy()\n\nrepeat_top12 = top12_e10a[top12_e10a[\"source_repeat\"] == 1].copy()\nrepeat_top12_hits = repeat_top12[repeat_top12[\"target\"] == 1].copy()\n\nals_top12 = top12_e10a[\n    (top12_e10a[\"source_als\"] == 1) & \n    (top12_e10a[\"source_repeat\"] == 0)\n].copy()\n\nrepeat_positive_total = len(repeat_pos)\nrepeat_positive_top12 = len(repeat_top12_hits)\nrepeat_positive_outside_top12 = repeat_positive_total - repeat_positive_top12\n\nrepeat_positive_top12_coverage = (\n    repeat_positive_top12 / repeat_positive_total\n    if repeat_positive_total > 0\n    else np.nan\n)\n\nrepeat_top12_hit_rate = (\n    repeat_top12[\"target\"].mean()\n    if len(repeat_top12) > 0\n    else np.nan\n)\n\nals_only_top12_hit_rate = (\n    als_top12[\"target\"].mean()\n    if len(als_top12) > 0\n    else np.nan\n)\n\nif not pd.isna(repeat_top12_hit_rate) and not pd.isna(als_only_top12_hit_rate):\n    hit_rate_ratio = repeat_top12_hit_rate / als_only_top12_hit_rate\nelse:\n    hit_rate_ratio = np.nan\n\nprint(\"\\n=== E10A repeat-candidate recall diagnostic ===\")\nprint(f\"Repeat candidate rows: {len(repeat_all):,}\")\nprint(f\"Repeat positive rows available in candidate pool: {repeat_positive_total:,}\")\nprint(f\"Repeat positive rows selected in top-12: {repeat_positive_top12:,}\")\nprint(f\"Repeat positive rows outside top-12: {repeat_positive_outside_top12:,}\")\nprint(f\"Repeat positive top-12 coverage: {repeat_positive_top12_coverage:.4f}\")\nprint(f\"Repeat top-12 hit rate: {repeat_top12_hit_rate:.4f}\")\nprint(f\"ALS-only top-12 hit rate: {als_only_top12_hit_rate:.4f}\")\nprint(f\"Repeat vs ALS-only top-12 hit-rate ratio: {hit_rate_ratio:.2f}x\")\n\n# ------------------------------------------------------------\n# 6. Repeat-positive rank distribution\n# ------------------------------------------------------------\n\nif repeat_positive_total > 0:\n    repeat_pos_rank_stats = pd.DataFrame([\n        {\n            \"metric\": \"repeat_positive_rank_min\",\n            \"value\": repeat_pos[\"rank_e10\"].min()\n        },\n        {\n            \"metric\": \"repeat_positive_rank_p25\",\n            \"value\": repeat_pos[\"rank_e10\"].quantile(0.25)\n        },\n        {\n            \"metric\": \"repeat_positive_rank_median\",\n            \"value\": repeat_pos[\"rank_e10\"].median()\n        },\n        {\n            \"metric\": \"repeat_positive_rank_p75\",\n            \"value\": repeat_pos[\"rank_e10\"].quantile(0.75)\n        },\n        {\n            \"metric\": \"repeat_positive_rank_max\",\n            \"value\": repeat_pos[\"rank_e10\"].max()\n        },\n        {\n            \"metric\": \"share_repeat_positives_rank_le_12\",\n            \"value\": (repeat_pos[\"rank_e10\"] <= 12).mean()\n        },\n        {\n            \"metric\": \"share_repeat_positives_rank_le_20\",\n            \"value\": (repeat_pos[\"rank_e10\"] <= 20).mean()\n        },\n        {\n            \"metric\": \"share_repeat_positives_rank_le_50\",\n            \"value\": (repeat_pos[\"rank_e10\"] <= 50).mean()\n        }\n    ])\nelse:\n    repeat_pos_rank_stats = pd.DataFrame([\n        {\n            \"metric\": \"no_repeat_positive_candidates\",\n            \"value\": np.nan\n        }\n    ])\n\nprint(\"\\n=== E10A repeat-positive rank distribution ===\")\ndisplay(repeat_pos_rank_stats)\n\n# ------------------------------------------------------------\n# 7. User-level repeat diagnostics\n# ------------------------------------------------------------\n\nusers_with_repeat_candidates = repeat_all[\"user_idx\"].nunique()\nusers_with_repeat_positive_candidates = repeat_pos[\"user_idx\"].nunique()\nusers_with_repeat_in_top12 = repeat_top12[\"user_idx\"].nunique()\nusers_with_repeat_hit_in_top12 = repeat_top12_hits[\"user_idx\"].nunique()\n\ne10a_user_report = pd.DataFrame([\n    {\n        \"metric\": \"users_total\",\n        \"value\": users_total\n    },\n    {\n        \"metric\": \"users_with_repeat_candidates\",\n        \"value\": users_with_repeat_candidates\n    },\n    {\n        \"metric\": \"users_with_repeat_positive_candidates\",\n        \"value\": users_with_repeat_positive_candidates\n    },\n    {\n        \"metric\": \"users_with_repeat_in_top12\",\n        \"value\": users_with_repeat_in_top12\n    },\n    {\n        \"metric\": \"users_with_repeat_hit_in_top12\",\n        \"value\": users_with_repeat_hit_in_top12\n    },\n    {\n        \"metric\": \"share_users_with_repeat_candidates\",\n        \"value\": users_with_repeat_candidates / users_total\n    },\n    {\n        \"metric\": \"share_users_with_repeat_positive_candidates\",\n        \"value\": users_with_repeat_positive_candidates / users_total\n    },\n    {\n        \"metric\": \"share_users_with_repeat_in_top12\",\n        \"value\": users_with_repeat_in_top12 / users_total\n    },\n    {\n        \"metric\": \"share_users_with_repeat_hit_in_top12\",\n        \"value\": users_with_repeat_hit_in_top12 / users_total\n    }\n])\n\nprint(\"\\n=== E10A user-level repeat diagnostics ===\")\ndisplay(e10a_user_report)\n\n# ------------------------------------------------------------\n# 8. Candidate-pool recall vs all test positives\n# ------------------------------------------------------------\n\nif \"test_df\" in globals():\n    test_pairs_total = (\n        test_df[[\"user_idx\", \"item_idx\"]]\n        .drop_duplicates()\n        .shape[0]\n    )\n\n    e10_candidate_recall_vs_all_test_pairs = (\n        candidate_positives_total / test_pairs_total\n    )\n\n    print(\"\\n=== E10A candidate-pool recall diagnostic ===\")\n    print(f\"Unique test positive pairs: {test_pairs_total:,}\")\n    print(f\"E10 candidate positives: {candidate_positives_total:,}\")\n    print(f\"E10 candidate-pool recall vs all test pairs: {e10_candidate_recall_vs_all_test_pairs:.4f}\")\nelse:\n    test_pairs_total = np.nan\n    e10_candidate_recall_vs_all_test_pairs = np.nan\n\n# ------------------------------------------------------------\n# 9. Decision\n# ------------------------------------------------------------\n\nrepeat_underselected = (\n    repeat_positive_total > 0\n    and repeat_positive_top12_coverage < 0.25\n    and not pd.isna(hit_rate_ratio)\n    and hit_rate_ratio >= 3\n)\n\nrepeat_source_useful = (\n    repeat_positive_total > 0\n    and not pd.isna(repeat_top12_hit_rate)\n    and not pd.isna(als_only_top12_hit_rate)\n    and repeat_top12_hit_rate > als_only_top12_hit_rate\n)\n\nif repeat_underselected:\n    e10a_decision = (\n        \"Repeat candidates are useful but under-selected. \"\n        \"Before E11 or alongside E11, test source-aware ranking or a controlled repeat-source boost.\"\n    )\nelif repeat_source_useful:\n    e10a_decision = (\n        \"Repeat candidates are useful and selected reasonably. \"\n        \"Proceed to E11: add train-popular candidates.\"\n    )\nelse:\n    e10a_decision = (\n        \"Repeat candidates do not show strong enough source-level value. \"\n        \"Proceed cautiously to E11 and avoid over-weighting repeat source.\"\n    )\n\nprint(\"\\n=== Safe E10A interpretation ===\")\nprint(e10a_decision)\n\n# ------------------------------------------------------------\n# 10. Objects to log\n# ------------------------------------------------------------\n\ne10a_summary = {\n    \"experiment_id\": \"E10A\",\n    \"type\": \"diagnostic_only\",\n    \"e10_precision_at_12\": float(e10_p12),\n    \"e08_precision_at_12\": (\n        float(e08_p12_for_compare)\n        if not pd.isna(e08_p12_for_compare)\n        else None\n    ),\n    \"candidate_rows_total\": int(candidate_rows_total),\n    \"candidate_positives_total\": int(candidate_positives_total),\n    \"top12_hits_total\": int(top12_hits_total),\n    \"repeat_candidate_rows\": int(len(repeat_all)),\n    \"repeat_positive_total\": int(repeat_positive_total),\n    \"repeat_positive_top12\": int(repeat_positive_top12),\n    \"repeat_positive_outside_top12\": int(repeat_positive_outside_top12),\n    \"repeat_positive_top12_coverage\": (\n        float(repeat_positive_top12_coverage)\n        if not pd.isna(repeat_positive_top12_coverage)\n        else None\n    ),\n    \"repeat_top12_hit_rate\": (\n        float(repeat_top12_hit_rate)\n        if not pd.isna(repeat_top12_hit_rate)\n        else None\n    ),\n    \"als_only_top12_hit_rate\": (\n        float(als_only_top12_hit_rate)\n        if not pd.isna(als_only_top12_hit_rate)\n        else None\n    ),\n    \"hit_rate_ratio_repeat_vs_als_top12\": (\n        float(hit_rate_ratio)\n        if not pd.isna(hit_rate_ratio)\n        else None\n    ),\n    \"candidate_recall_vs_all_test_pairs\": (\n        float(e10_candidate_recall_vs_all_test_pairs)\n        if not pd.isna(e10_candidate_recall_vs_all_test_pairs)\n        else None\n    ),\n    \"decision\": e10a_decision\n}\n\nprint(\"\\nObjects to log:\")\nprint(\"- e10a_summary\")\nprint(\"- e10a_source_report\")\nprint(\"- repeat_pos_rank_stats\")\nprint(\"- e10a_user_report\")\nprint(\"- e10a_decision\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-28T20:39:19.725829Z","iopub.execute_input":"2026-05-28T20:39:19.727965Z","iopub.status.idle":"2026-05-28T20:39:26.285209Z","shell.execute_reply.started":"2026-05-28T20:39:19.727906Z","shell.execute_reply":"2026-05-28T20:39:26.283809Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# NEW CELL L - E10B: Controlled repeat-source boost sweep\n#\n# PLACE AFTER:\n# NEW CELL K - E10A\n#\n# PURPOSE:\n# Test whether repeat candidates are under-ranked by the E10 score.\n#\n# This cell does NOT train a model.\n#\n# It adds controlled boosts to repeat-only candidates:\n# Hybrid_Score_E10B = Hybrid_Score_E10 + boost * source_repeat\n#\n# This is a validation sensitivity test, not a final production rule.\n# ============================================================\n\nimport pandas as pd\nimport numpy as np\n\nprint(\"NEW CELL L: E10B - Controlled repeat-source boost sweep\")\n\n# ------------------------------------------------------------\n# 1. Safety checks\n# ------------------------------------------------------------\n\nrequired_objects = [\"e10_candidates_df\", \"e10_p12\"]\n\nmissing_objects = [obj for obj in required_objects if obj not in globals()]\n\nif missing_objects:\n    raise RuntimeError(f\"Missing required objects: {missing_objects}. Run E10 first.\")\n\nrequired_cols = [\n    \"user_idx\",\n    \"item_idx\",\n    \"target\",\n    \"source_repeat\",\n    \"Hybrid_Score_E10\"\n]\n\nmissing_cols = [col for col in required_cols if col not in e10_candidates_df.columns]\n\nif missing_cols:\n    raise RuntimeError(f\"e10_candidates_df is missing columns: {missing_cols}\")\n\ne08_p12_for_compare = e08_p12 if \"e08_p12\" in globals() else np.nan\n\n# ------------------------------------------------------------\n# 2. Helper\n# ------------------------------------------------------------\n\ndef precision_at_12_e10b(df, score_col):\n    top_12 = (\n        df.sort_values([\"user_idx\", score_col], ascending=[True, False])\n          .groupby(\"user_idx\")\n          .head(12)\n    )\n    user_hits = top_12.groupby(\"user_idx\")[\"target\"].sum()\n    return (user_hits / 12).mean()\n\ndef top12_source_stats_e10b(df, score_col):\n    top_12 = (\n        df.sort_values([\"user_idx\", score_col], ascending=[True, False])\n          .groupby(\"user_idx\")\n          .head(12)\n          .copy()\n    )\n\n    repeat_rows = top_12[top_12[\"source_repeat\"] == 1]\n    als_rows = top_12[top_12[\"source_repeat\"] == 0]\n\n    return {\n        \"top12_rows\": len(top_12),\n        \"top12_hits\": int(top_12[\"target\"].sum()),\n        \"repeat_top12_rows\": len(repeat_rows),\n        \"repeat_top12_hits\": int(repeat_rows[\"target\"].sum()),\n        \"repeat_top12_hit_rate\": (\n            repeat_rows[\"target\"].mean()\n            if len(repeat_rows) > 0\n            else np.nan\n        ),\n        \"als_nonrepeat_top12_rows\": len(als_rows),\n        \"als_nonrepeat_top12_hits\": int(als_rows[\"target\"].sum()),\n        \"als_nonrepeat_top12_hit_rate\": (\n            als_rows[\"target\"].mean()\n            if len(als_rows) > 0\n            else np.nan\n        )\n    }\n\n# ------------------------------------------------------------\n# 3. Boost sweep\n# ------------------------------------------------------------\n\nboost_values = [\n    0.00,\n    0.02,\n    0.04,\n    0.06,\n    0.08,\n    0.10,\n    0.12,\n    0.15,\n    0.20,\n    0.25,\n    0.30,\n    0.40,\n    0.50\n]\n\ne10b_rows = []\n\nfor boost in boost_values:\n    score_col = f\"Hybrid_Score_E10B_boost_{str(boost).replace('.', '_')}\"\n\n    e10_candidates_df[score_col] = (\n        e10_candidates_df[\"Hybrid_Score_E10\"]\n        + boost * e10_candidates_df[\"source_repeat\"]\n    )\n\n    p12 = precision_at_12_e10b(e10_candidates_df, score_col)\n    stats = top12_source_stats_e10b(e10_candidates_df, score_col)\n\n    row = {\n        \"boost\": boost,\n        \"Precision@12\": p12,\n        \"delta_vs_E10_%\": ((p12 - e10_p12) / e10_p12) * 100,\n        \"delta_vs_E08_%\": (\n            ((p12 - e08_p12_for_compare) / e08_p12_for_compare) * 100\n            if not pd.isna(e08_p12_for_compare)\n            else np.nan\n        )\n    }\n\n    row.update(stats)\n    e10b_rows.append(row)\n\ne10b_sweep = pd.DataFrame(e10b_rows)\n\ne10b_sweep = e10b_sweep.sort_values(\n    [\"Precision@12\", \"boost\"],\n    ascending=[False, True]\n).reset_index(drop=True)\n\nprint(\"\\n=== E10B repeat-source boost sweep ===\")\ndisplay(e10b_sweep)\n\n# ------------------------------------------------------------\n# 4. Best boost summary\n# ------------------------------------------------------------\n\nbest_e10b = e10b_sweep.iloc[0].copy()\n\nbest_boost_e10b = float(best_e10b[\"boost\"])\nbest_p12_e10b = float(best_e10b[\"Precision@12\"])\nbest_delta_vs_e10_e10b = float(best_e10b[\"delta_vs_E10_%\"])\n\nprint(\"\\n=== E10B best result ===\")\nprint(f\"Best boost: {best_boost_e10b:.2f}\")\nprint(f\"E10 Precision@12: {e10_p12:.6f}\")\nprint(f\"E10B best Precision@12: {best_p12_e10b:.6f}\")\nprint(f\"E10B delta vs E10: {best_delta_vs_e10_e10b:.2f}%\")\n\nif not pd.isna(e08_p12_for_compare):\n    print(f\"E08 Precision@12: {e08_p12_for_compare:.6f}\")\n    print(f\"E10B delta vs E08: {((best_p12_e10b - e08_p12_for_compare) / e08_p12_for_compare) * 100:.2f}%\")\n\nprint(f\"Repeat rows in top-12 at best boost: {int(best_e10b['repeat_top12_rows']):,}\")\nprint(f\"Repeat hits in top-12 at best boost: {int(best_e10b['repeat_top12_hits']):,}\")\nprint(f\"Repeat top-12 hit rate at best boost: {best_e10b['repeat_top12_hit_rate']:.4f}\")\n\n# ------------------------------------------------------------\n# 5. Decision\n# ------------------------------------------------------------\n\nif best_p12_e10b > e10_p12:\n    e10b_decision = (\n        \"E10B improves over E10. Repeat candidates were under-ranked by the original E10 score. \"\n        \"Use this as evidence for source-aware ranking, but do not treat the boost as final without caution.\"\n    )\nelif best_p12_e10b == e10_p12:\n    e10b_decision = (\n        \"E10B does not improve over E10. The repeat-source boost did not add value.\"\n    )\nelse:\n    e10b_decision = (\n        \"E10B is worse than E10. Do not boost repeat candidates directly.\"\n    )\n\nprint(\"\\n=== Safe E10B interpretation ===\")\nprint(e10b_decision)\n\n# ------------------------------------------------------------\n# 6. Objects to log\n# ------------------------------------------------------------\n\ne10b_summary = {\n    \"experiment_id\": \"E10B\",\n    \"type\": \"diagnostic_score_sweep\",\n    \"best_boost\": best_boost_e10b,\n    \"e10_precision_at_12\": float(e10_p12),\n    \"best_precision_at_12\": best_p12_e10b,\n    \"delta_vs_e10_percent\": best_delta_vs_e10_e10b,\n    \"decision\": e10b_decision\n}\n\nprint(\"\\nObjects to log:\")\nprint(\"- e10b_sweep\")\nprint(\"- e10b_summary\")\nprint(\"- e10b_decision\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-28T20:43:29.916245Z","iopub.execute_input":"2026-05-28T20:43:29.919554Z","iopub.status.idle":"2026-05-28T20:44:52.486821Z","shell.execute_reply.started":"2026-05-28T20:43:29.919433Z","shell.execute_reply":"2026-05-28T20:44:52.484854Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# NEW CELL M - E11: Repeat + train-popular candidates + E08 features\n#\n# PLACE AFTER:\n# NEW CELL L - E10B\n#\n# PURPOSE:\n# Test whether adding train-popular candidates improves candidate recall.\n#\n# E11:\n# - ALS top-50 candidates\n# - repeat-purchase candidates from train history\n# - global train-popular candidates\n# - E08 train-only popularity/recency features\n#\n# Important:\n# - No manual E10B boost is used here.\n# - Popular candidates come from train_df only.\n# - No test-week popularity or future demand is used.\n# - postal_code is a hashed categorical proxy only, not geography.\n# ============================================================\n\nimport pandas as pd\nimport numpy as np\nimport gc\n\nfrom catboost import CatBoostClassifier\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import roc_auc_score\n\nprint(\"NEW CELL M: E11 - Repeat + train-popular candidates + E08 features\")\n\n\n# ------------------------------------------------------------\n# 1. Safety checks\n# ------------------------------------------------------------\n\nrequired_objects = [\n    \"train_df\",\n    \"test_df\",\n    \"catboost_train_df\",\n    \"als_candidates_df\",\n    \"baseline_p12\",\n    \"hybrid_p12\"\n]\n\nmissing_objects = [obj for obj in required_objects if obj not in globals()]\n\nif missing_objects:\n    raise RuntimeError(\n        f\"Missing required objects: {missing_objects}. \"\n        \"Run cells up to E10B first.\"\n    )\n\ne08_p12_for_compare = e08_p12 if \"e08_p12\" in globals() else np.nan\ne10_p12_for_compare = e10_p12 if \"e10_p12\" in globals() else np.nan\nbest_p12_e10b_for_compare = best_p12_e10b if \"best_p12_e10b\" in globals() else np.nan\n\n\n# ------------------------------------------------------------\n# 2. Helper functions\n# ------------------------------------------------------------\n\ndef precision_at_12_e11(df, score_col):\n    top_12 = (\n        df.sort_values([\"user_idx\", score_col], ascending=[True, False])\n          .groupby(\"user_idx\")\n          .head(12)\n    )\n    user_hits = top_12.groupby(\"user_idx\")[\"target\"].sum()\n    return (user_hits / 12).mean()\n\n\ndef safe_auc_e11(y_true, score):\n    if len(np.unique(y_true)) < 2:\n        return np.nan\n    return roc_auc_score(y_true, score)\n\n\ndef add_simple_date_features_e11(df_in, date_col, window_start_date):\n    df_out = df_in.copy()\n    dt = pd.to_datetime(df_out[date_col])\n\n    df_out[\"month\"] = dt.dt.month.astype(str)\n    df_out[\"weekofyear\"] = dt.dt.isocalendar().week.astype(int).astype(str)\n    df_out[\"quarter\"] = dt.dt.quarter.astype(str)\n    df_out[\"dayofweek\"] = dt.dt.dayofweek.astype(str)\n\n    month_num = dt.dt.month\n\n    conditions = [\n        month_num.isin([12, 1, 2]),\n        month_num.isin([3, 4, 5]),\n        month_num.isin([6, 7, 8]),\n        month_num.isin([9, 10, 11])\n    ]\n\n    choices = [\"winter\", \"spring\", \"summer\", \"autumn\"]\n\n    df_out[\"season_bucket\"] = np.select(\n        conditions,\n        choices,\n        default=\"unknown\"\n    ).astype(str)\n\n    df_out[\"days_since_window_start\"] = (\n        dt - pd.to_datetime(window_start_date)\n    ).dt.days.astype(\"int16\")\n\n    return df_out\n\n\ndef source_label_e11(df):\n    conditions = [\n        (df[\"source_als\"] == 1) & (df[\"source_repeat\"] == 0) & (df[\"source_popular\"] == 0),\n        (df[\"source_als\"] == 0) & (df[\"source_repeat\"] == 1) & (df[\"source_popular\"] == 0),\n        (df[\"source_als\"] == 0) & (df[\"source_repeat\"] == 0) & (df[\"source_popular\"] == 1),\n        (df[\"source_als\"] == 1) & (df[\"source_repeat\"] == 1) & (df[\"source_popular\"] == 0),\n        (df[\"source_als\"] == 1) & (df[\"source_repeat\"] == 0) & (df[\"source_popular\"] == 1),\n        (df[\"source_als\"] == 0) & (df[\"source_repeat\"] == 1) & (df[\"source_popular\"] == 1),\n        (df[\"source_als\"] == 1) & (df[\"source_repeat\"] == 1) & (df[\"source_popular\"] == 1)\n    ]\n\n    choices = [\n        \"als_only\",\n        \"repeat_only\",\n        \"popular_only\",\n        \"als_repeat\",\n        \"als_popular\",\n        \"repeat_popular\",\n        \"all_sources\"\n    ]\n\n    return np.select(conditions, choices, default=\"unknown\")\n\n\n# ------------------------------------------------------------\n# 3. Feature lists\n# ------------------------------------------------------------\n\nbase_features_e11 = [\n    \"age\",\n    \"postal_code\",\n    \"product_type_name\",\n    \"graphical_appearance_name\",\n    \"colour_group_name\",\n    \"perceived_colour_value_name\",\n    \"index_name\",\n    \"index_group_name\"\n]\n\ndate_features_e11 = [\n    \"month\",\n    \"weekofyear\",\n    \"quarter\",\n    \"dayofweek\",\n    \"season_bucket\",\n    \"days_since_window_start\"\n]\n\npopularity_features_e11 = [\n    \"item_train_count\",\n    \"item_train_unique_users\",\n    \"item_train_recent_7d_count\",\n    \"item_train_recent_14d_count\",\n    \"item_days_since_last_train_purchase\",\n    \"product_type_train_count\",\n    \"index_name_train_count\",\n    \"colour_group_train_count\"\n]\n\ncandidate_source_features_e11 = [\n    \"source_als\",\n    \"source_repeat\",\n    \"source_popular\",\n    \"candidate_source_count\"\n]\n\nall_features_e11 = (\n    base_features_e11\n    + date_features_e11\n    + popularity_features_e11\n    + candidate_source_features_e11\n)\n\n\n# ------------------------------------------------------------\n# 4. Build E11 candidate pool\n# ------------------------------------------------------------\n\nprint(\"\\nBuilding E11 expanded candidate pool...\")\n\ntest_user_ids_e11 = als_candidates_df[\"user_idx\"].unique()\n\n# ALS candidates.\ne11_als_candidates = als_candidates_df[[\"user_idx\", \"item_idx\"]].copy()\ne11_als_candidates[\"source_als\"] = 1\ne11_als_candidates[\"source_repeat\"] = 0\ne11_als_candidates[\"source_popular\"] = 0\n\n# Repeat candidates from train history.\ntrain_history_e11 = train_df[\n    train_df[\"user_idx\"].isin(test_user_ids_e11)\n][[\"user_idx\", \"item_idx\", \"t_dat\"]].copy()\n\ntrain_history_e11[\"t_dat\"] = pd.to_datetime(train_history_e11[\"t_dat\"])\n\nrepeat_candidates_per_user_e11 = 20\n\nrepeat_candidates_e11 = (\n    train_history_e11\n    .sort_values([\"user_idx\", \"t_dat\"], ascending=[True, False])\n    .drop_duplicates([\"user_idx\", \"item_idx\"])\n    .groupby(\"user_idx\")\n    .head(repeat_candidates_per_user_e11)\n    [[\"user_idx\", \"item_idx\"]]\n    .copy()\n)\n\nrepeat_candidates_e11[\"source_als\"] = 0\nrepeat_candidates_e11[\"source_repeat\"] = 1\nrepeat_candidates_e11[\"source_popular\"] = 0\n\n# Global train-popular candidates.\npopular_candidates_per_user_e11 = 20\n\ntop_popular_items_e11 = (\n    train_df\n    .groupby(\"item_idx\")\n    .size()\n    .sort_values(ascending=False)\n    .head(popular_candidates_per_user_e11)\n    .index\n    .to_numpy()\n)\n\nusers_array_e11 = np.asarray(test_user_ids_e11)\npopular_items_array_e11 = np.asarray(top_popular_items_e11)\n\npopular_candidates_e11 = pd.DataFrame({\n    \"user_idx\": np.repeat(users_array_e11, len(popular_items_array_e11)),\n    \"item_idx\": np.tile(popular_items_array_e11, len(users_array_e11))\n})\n\npopular_candidates_e11[\"source_als\"] = 0\npopular_candidates_e11[\"source_repeat\"] = 0\npopular_candidates_e11[\"source_popular\"] = 1\n\nprint(f\"ALS candidate rows: {len(e11_als_candidates):,}\")\nprint(f\"Repeat candidate rows before dedupe: {len(repeat_candidates_e11):,}\")\nprint(f\"Popular candidate rows before dedupe: {len(popular_candidates_e11):,}\")\nprint(f\"Popular candidates per user: {popular_candidates_per_user_e11}\")\n\ncandidate_union_e11 = pd.concat(\n    [\n        e11_als_candidates,\n        repeat_candidates_e11,\n        popular_candidates_e11\n    ],\n    ignore_index=True\n)\n\ncandidate_union_e11 = (\n    candidate_union_e11\n    .groupby([\"user_idx\", \"item_idx\"], as_index=False)\n    .agg(\n        source_als=(\"source_als\", \"max\"),\n        source_repeat=(\"source_repeat\", \"max\"),\n        source_popular=(\"source_popular\", \"max\")\n    )\n)\n\ncandidate_union_e11[\"candidate_source_count\"] = (\n    candidate_union_e11[\"source_als\"]\n    + candidate_union_e11[\"source_repeat\"]\n    + candidate_union_e11[\"source_popular\"]\n)\n\nprint(f\"Candidate rows after dedupe: {len(candidate_union_e11):,}\")\n\nprint(\"\\nCandidate source distribution:\")\nprint(\n    candidate_union_e11[\n        [\"source_als\", \"source_repeat\", \"source_popular\", \"candidate_source_count\"]\n    ].value_counts()\n)\n\n\n# ------------------------------------------------------------\n# 5. Attach target labels\n# ------------------------------------------------------------\n\ntest_pairs_e11 = (\n    test_df[[\"user_idx\", \"item_idx\"]]\n    .drop_duplicates()\n    .copy()\n)\n\ntest_pairs_e11[\"target_e11\"] = 1\n\ne11_candidates_df = candidate_union_e11.merge(\n    test_pairs_e11,\n    on=[\"user_idx\", \"item_idx\"],\n    how=\"left\"\n)\n\ne11_candidates_df[\"target\"] = e11_candidates_df[\"target_e11\"].fillna(0).astype(\"int8\")\ne11_candidates_df = e11_candidates_df.drop(columns=[\"target_e11\"])\n\nprint(\"\\nE11 target distribution:\")\nprint(e11_candidates_df[\"target\"].value_counts(dropna=False))\n\n\n# ------------------------------------------------------------\n# 6. Attach Score_CF\n# ------------------------------------------------------------\n\nals_score_lookup_e11 = als_candidates_df[[\"user_idx\", \"item_idx\", \"Score_CF\"]].copy()\n\ne11_candidates_df = e11_candidates_df.merge(\n    als_score_lookup_e11,\n    on=[\"user_idx\", \"item_idx\"],\n    how=\"left\"\n)\n\nmin_cf_score_e11 = als_candidates_df[\"Score_CF\"].min()\ne11_candidates_df[\"Score_CF\"] = e11_candidates_df[\"Score_CF\"].fillna(min_cf_score_e11)\n\n\n# ------------------------------------------------------------\n# 7. Attach customer/article features\n# ------------------------------------------------------------\n\nprint(\"\\nAttaching customer/article features...\")\n\ncandidate_feature_lookup_e11 = pd.concat(\n    [\n        als_candidates_df[[\"user_idx\", \"item_idx\"] + base_features_e11],\n        catboost_train_df[[\"user_idx\", \"item_idx\"] + base_features_e11]\n    ],\n    ignore_index=True\n).drop_duplicates([\"user_idx\", \"item_idx\"])\n\ne11_candidates_df = e11_candidates_df.merge(\n    candidate_feature_lookup_e11,\n    on=[\"user_idx\", \"item_idx\"],\n    how=\"left\"\n)\n\nmissing_feature_rows_e11 = e11_candidates_df[base_features_e11].isna().any(axis=1).sum()\nprint(f\"Rows with at least one missing base feature after pair lookup: {missing_feature_rows_e11:,}\")\n\nif missing_feature_rows_e11 > 0:\n    item_feature_cols_e11 = [\n        \"product_type_name\",\n        \"graphical_appearance_name\",\n        \"colour_group_name\",\n        \"perceived_colour_value_name\",\n        \"index_name\",\n        \"index_group_name\"\n    ]\n\n    item_feature_lookup_e11 = (\n        pd.concat(\n            [\n                als_candidates_df[[\"item_idx\"] + item_feature_cols_e11],\n                catboost_train_df[[\"item_idx\"] + item_feature_cols_e11]\n            ],\n            ignore_index=True\n        )\n        .drop_duplicates(\"item_idx\")\n    )\n\n    user_feature_lookup_e11 = (\n        pd.concat(\n            [\n                als_candidates_df[[\"user_idx\", \"age\", \"postal_code\"]],\n                catboost_train_df[[\"user_idx\", \"age\", \"postal_code\"]]\n            ],\n            ignore_index=True\n        )\n        .drop_duplicates(\"user_idx\")\n    )\n\n    e11_candidates_df = e11_candidates_df.merge(\n        item_feature_lookup_e11,\n        on=\"item_idx\",\n        how=\"left\",\n        suffixes=(\"\", \"_item_lookup\")\n    )\n\n    e11_candidates_df = e11_candidates_df.merge(\n        user_feature_lookup_e11,\n        on=\"user_idx\",\n        how=\"left\",\n        suffixes=(\"\", \"_user_lookup\")\n    )\n\n    for col in item_feature_cols_e11:\n        lookup_col = f\"{col}_item_lookup\"\n        if lookup_col in e11_candidates_df.columns:\n            e11_candidates_df[col] = e11_candidates_df[col].fillna(e11_candidates_df[lookup_col])\n            e11_candidates_df = e11_candidates_df.drop(columns=[lookup_col])\n\n    for col in [\"age\", \"postal_code\"]:\n        lookup_col = f\"{col}_user_lookup\"\n        if lookup_col in e11_candidates_df.columns:\n            e11_candidates_df[col] = e11_candidates_df[col].fillna(e11_candidates_df[lookup_col])\n            e11_candidates_df = e11_candidates_df.drop(columns=[lookup_col])\n\nmissing_feature_rows_after_e11 = e11_candidates_df[base_features_e11].isna().any(axis=1).sum()\nprint(f\"Rows with missing base features after fallback: {missing_feature_rows_after_e11:,}\")\n\n\n# ------------------------------------------------------------\n# 8. Build E08-style train-only date/popularity features\n# ------------------------------------------------------------\n\nprint(\"\\nBuilding E08-style train-only features for E11...\")\n\ntrain_df_dates_e11 = train_df[[\"user_idx\", \"item_idx\", \"t_dat\"]].copy()\ntrain_df_dates_e11[\"t_dat\"] = pd.to_datetime(train_df_dates_e11[\"t_dat\"])\n\nwindow_start_date_e11 = train_df_dates_e11[\"t_dat\"].min()\ntrain_end_date_e11 = train_df_dates_e11[\"t_dat\"].max()\nvalidation_anchor_date_e11 = pd.to_datetime(test_df[\"t_dat\"]).min()\n\nrecent_7d_start_e11 = train_end_date_e11 - pd.Timedelta(days=6)\nrecent_14d_start_e11 = train_end_date_e11 - pd.Timedelta(days=13)\n\nif train_end_date_e11 >= validation_anchor_date_e11:\n    raise RuntimeError(\"Leakage risk: train_end_date is not before validation anchor.\")\n\nprint(f\"Train window start date: {window_start_date_e11}\")\nprint(f\"Train end date: {train_end_date_e11}\")\nprint(f\"Validation anchor date: {validation_anchor_date_e11}\")\n\n# Train-side date anchors.\npositive_pair_dates_e11 = (\n    train_df_dates_e11\n    .sort_values(\"t_dat\")\n    .groupby([\"user_idx\", \"item_idx\"], as_index=False)\n    .agg(pair_train_date=(\"t_dat\", \"max\"))\n)\n\nuser_anchor_dates_e11 = (\n    train_df_dates_e11\n    .groupby(\"user_idx\", as_index=False)\n    .agg(user_train_anchor_date=(\"t_dat\", \"max\"))\n)\n\ne11_train_df = catboost_train_df.copy()\n\ne11_train_df = e11_train_df.merge(\n    positive_pair_dates_e11,\n    on=[\"user_idx\", \"item_idx\"],\n    how=\"left\"\n)\n\ne11_train_df = e11_train_df.merge(\n    user_anchor_dates_e11,\n    on=\"user_idx\",\n    how=\"left\"\n)\n\ne11_train_df[\"season_anchor_date\"] = e11_train_df[\"pair_train_date\"]\n\ne11_train_df[\"season_anchor_date\"] = e11_train_df[\"season_anchor_date\"].fillna(\n    e11_train_df[\"user_train_anchor_date\"]\n)\n\ne11_train_df[\"season_anchor_date\"] = e11_train_df[\"season_anchor_date\"].fillna(\n    window_start_date_e11\n)\n\ne11_train_df = add_simple_date_features_e11(\n    e11_train_df,\n    date_col=\"season_anchor_date\",\n    window_start_date=window_start_date_e11\n)\n\n# Candidate-side fixed validation anchor.\ne11_candidates_df[\"season_anchor_date\"] = validation_anchor_date_e11\n\ne11_candidates_df = add_simple_date_features_e11(\n    e11_candidates_df,\n    date_col=\"season_anchor_date\",\n    window_start_date=window_start_date_e11\n)\n\n# Item popularity.\nitem_pop_e11 = (\n    train_df_dates_e11\n    .groupby(\"item_idx\")\n    .agg(\n        item_train_count=(\"user_idx\", \"size\"),\n        item_train_unique_users=(\"user_idx\", \"nunique\"),\n        item_last_train_purchase=(\"t_dat\", \"max\")\n    )\n    .reset_index()\n)\n\nrecent_7d_counts_e11 = (\n    train_df_dates_e11[train_df_dates_e11[\"t_dat\"] >= recent_7d_start_e11]\n    .groupby(\"item_idx\")\n    .size()\n    .reset_index(name=\"item_train_recent_7d_count\")\n)\n\nrecent_14d_counts_e11 = (\n    train_df_dates_e11[train_df_dates_e11[\"t_dat\"] >= recent_14d_start_e11]\n    .groupby(\"item_idx\")\n    .size()\n    .reset_index(name=\"item_train_recent_14d_count\")\n)\n\nitem_pop_e11 = item_pop_e11.merge(\n    recent_7d_counts_e11,\n    on=\"item_idx\",\n    how=\"left\"\n)\n\nitem_pop_e11 = item_pop_e11.merge(\n    recent_14d_counts_e11,\n    on=\"item_idx\",\n    how=\"left\"\n)\n\nitem_pop_e11[\"item_train_recent_7d_count\"] = (\n    item_pop_e11[\"item_train_recent_7d_count\"].fillna(0)\n)\n\nitem_pop_e11[\"item_train_recent_14d_count\"] = (\n    item_pop_e11[\"item_train_recent_14d_count\"].fillna(0)\n)\n\nitem_pop_e11[\"item_days_since_last_train_purchase\"] = (\n    validation_anchor_date_e11 - item_pop_e11[\"item_last_train_purchase\"]\n).dt.days\n\nitem_pop_e11 = item_pop_e11.drop(columns=[\"item_last_train_purchase\"])\n\n# Category popularity.\ncategory_cols_e11 = [\n    \"product_type_name\",\n    \"index_name\",\n    \"colour_group_name\"\n]\n\nitem_category_lookup_e11 = (\n    catboost_train_df[[\"item_idx\"] + category_cols_e11]\n    .drop_duplicates(\"item_idx\")\n)\n\ntrain_with_categories_e11 = train_df[[\"item_idx\"]].merge(\n    item_category_lookup_e11,\n    on=\"item_idx\",\n    how=\"left\"\n)\n\nproduct_type_pop_e11 = (\n    train_with_categories_e11\n    .groupby(\"product_type_name\")\n    .size()\n    .reset_index(name=\"product_type_train_count\")\n)\n\nindex_name_pop_e11 = (\n    train_with_categories_e11\n    .groupby(\"index_name\")\n    .size()\n    .reset_index(name=\"index_name_train_count\")\n)\n\ncolour_group_pop_e11 = (\n    train_with_categories_e11\n    .groupby(\"colour_group_name\")\n    .size()\n    .reset_index(name=\"colour_group_train_count\")\n)\n\nfor df_name, df_tmp in [(\"train\", e11_train_df), (\"candidates\", e11_candidates_df)]:\n    df_tmp = df_tmp.merge(item_pop_e11, on=\"item_idx\", how=\"left\")\n    df_tmp = df_tmp.merge(product_type_pop_e11, on=\"product_type_name\", how=\"left\")\n    df_tmp = df_tmp.merge(index_name_pop_e11, on=\"index_name\", how=\"left\")\n    df_tmp = df_tmp.merge(colour_group_pop_e11, on=\"colour_group_name\", how=\"left\")\n\n    for col in popularity_features_e11:\n        df_tmp[col] = pd.to_numeric(df_tmp[col], errors=\"coerce\").fillna(0)\n\n    if df_name == \"train\":\n        e11_train_df = df_tmp\n    else:\n        e11_candidates_df = df_tmp\n\n# Training rows are not generated by candidate sources.\ne11_train_df[\"source_als\"] = 0\ne11_train_df[\"source_repeat\"] = 0\ne11_train_df[\"source_popular\"] = 0\ne11_train_df[\"candidate_source_count\"] = 0\n\n\n# ------------------------------------------------------------\n# 9. Prepare model matrices\n# ------------------------------------------------------------\n\nX_e11 = e11_train_df[all_features_e11].copy()\ny_e11 = e11_train_df[\"target\"].copy()\n\nX_test_e11 = e11_candidates_df[all_features_e11].copy()\n\ncategorical_features_e11 = [\n    \"postal_code\",\n    \"product_type_name\",\n    \"graphical_appearance_name\",\n    \"colour_group_name\",\n    \"perceived_colour_value_name\",\n    \"index_name\",\n    \"index_group_name\",\n    \"month\",\n    \"weekofyear\",\n    \"quarter\",\n    \"dayofweek\",\n    \"season_bucket\"\n]\n\nnumeric_features_e11 = [\n    \"age\",\n    \"days_since_window_start\"\n] + popularity_features_e11 + candidate_source_features_e11\n\nage_median_e11 = X_e11[\"age\"].median()\n\nfor df_tmp in [X_e11, X_test_e11]:\n    df_tmp[\"age\"] = pd.to_numeric(df_tmp[\"age\"], errors=\"coerce\").fillna(age_median_e11)\n\n    for col in numeric_features_e11:\n        df_tmp[col] = pd.to_numeric(df_tmp[col], errors=\"coerce\").fillna(0)\n\n    for col in categorical_features_e11:\n        df_tmp[col] = df_tmp[col].fillna(\"Unknown\").astype(str)\n\nif \"inferred_gender\" in X_e11.columns:\n    raise RuntimeError(\"E11 failed: inferred_gender is present.\")\n\nprint(\"\\nE11 feature set:\")\nprint(list(X_e11.columns))\n\nprint(\"\\nLeakage/source checks:\")\nprint(f\"Train end date: {train_end_date_e11}\")\nprint(f\"Validation anchor date: {validation_anchor_date_e11}\")\nprint(f\"recent_7d max date: {train_df_dates_e11[train_df_dates_e11['t_dat'] >= recent_7d_start_e11]['t_dat'].max()}\")\nprint(f\"recent_14d max date: {train_df_dates_e11[train_df_dates_e11['t_dat'] >= recent_14d_start_e11]['t_dat'].max()}\")\nprint(f\"inferred_gender in E11 features: {'inferred_gender' in X_e11.columns}\")\nprint(\"postal_code is used only as a hashed categorical proxy, not geography.\")\n\n\n# ------------------------------------------------------------\n# 10. Train CatBoost for E11\n# ------------------------------------------------------------\n\nprint(\"\\nTraining CatBoost E11...\")\n\nmodel_cb_e11 = CatBoostClassifier(\n    iterations=100,\n    learning_rate=0.1,\n    depth=6,\n    loss_function=\"Logloss\",\n    eval_metric=\"AUC\",\n    random_seed=42,\n    verbose=20\n)\n\nmodel_cb_e11.fit(X_e11, y_e11, cat_features=categorical_features_e11)\n\n\n# ------------------------------------------------------------\n# 11. Score E11 expanded candidates\n# ------------------------------------------------------------\n\nprint(\"\\nScoring expanded E11 candidates...\")\n\ne11_candidates_df[\"Score_Dem_E11\"] = model_cb_e11.predict_proba(X_test_e11)[:, 1]\n\n\n# ------------------------------------------------------------\n# 12. Logistic Regression fusion\n# ------------------------------------------------------------\n\nprint(\"\\nTraining Logistic Regression fusion for E11...\")\n\nscaler_e11 = StandardScaler()\n\nscaled_e11 = scaler_e11.fit_transform(\n    e11_candidates_df[[\"Score_CF\", \"Score_Dem_E11\"]]\n)\n\ne11_candidates_df[\"Scaled_CF_E11\"] = scaled_e11[:, 0]\ne11_candidates_df[\"Scaled_Dem_E11\"] = scaled_e11[:, 1]\n\nX_meta_e11 = e11_candidates_df[[\"Scaled_CF_E11\", \"Scaled_Dem_E11\"]]\ny_meta_e11 = e11_candidates_df[\"target\"]\n\nmodel_lr_e11 = LogisticRegression(class_weight=\"balanced\", random_state=42)\nmodel_lr_e11.fit(X_meta_e11, y_meta_e11)\n\nalpha_e11 = model_lr_e11.coef_[0][0]\nbeta_e11 = model_lr_e11.coef_[0][1]\nintercept_e11 = model_lr_e11.intercept_[0]\n\ne11_candidates_df[\"Hybrid_Score_E11\"] = (\n    alpha_e11 * e11_candidates_df[\"Scaled_CF_E11\"]\n    + beta_e11 * e11_candidates_df[\"Scaled_Dem_E11\"]\n    + intercept_e11\n)\n\nprint(\"\\n=== E11 Logistic Regression coefficients ===\")\nprint(f\"Alpha / ALS Score_CF: {alpha_e11:.4f}\")\nprint(f\"Beta / CatBoost Score_Dem_E11: {beta_e11:.4f}\")\nprint(f\"Intercept: {intercept_e11:.4f}\")\n\n\n# ------------------------------------------------------------\n# 13. Metrics\n# ------------------------------------------------------------\n\ne11_p12 = precision_at_12_e11(e11_candidates_df, \"Hybrid_Score_E11\")\ne11_uplift_vs_e01 = ((e11_p12 - baseline_p12) / baseline_p12) * 100\n\nif pd.isna(e08_p12_for_compare):\n    e11_delta_vs_e08 = np.nan\nelse:\n    e11_delta_vs_e08 = ((e11_p12 - e08_p12_for_compare) / e08_p12_for_compare) * 100\n\nif pd.isna(e10_p12_for_compare):\n    e11_delta_vs_e10 = np.nan\nelse:\n    e11_delta_vs_e10 = ((e11_p12 - e10_p12_for_compare) / e10_p12_for_compare) * 100\n\nif pd.isna(best_p12_e10b_for_compare):\n    e11_delta_vs_e10b = np.nan\nelse:\n    e11_delta_vs_e10b = ((e11_p12 - best_p12_e10b_for_compare) / best_p12_e10b_for_compare) * 100\n\nauc_score_dem_e11 = safe_auc_e11(\n    e11_candidates_df[\"target\"],\n    e11_candidates_df[\"Score_Dem_E11\"]\n)\n\nauc_hybrid_e11 = safe_auc_e11(\n    e11_candidates_df[\"target\"],\n    e11_candidates_df[\"Hybrid_Score_E11\"]\n)\n\nprint(\"\\n=== E11 Precision@12 results ===\")\nprint(f\"E01 ALS baseline Precision@12: {baseline_p12:.6f}\")\n\nif not pd.isna(e08_p12_for_compare):\n    print(f\"E08 locked trained model Precision@12: {e08_p12_for_compare:.6f}\")\n\nif not pd.isna(e10_p12_for_compare):\n    print(f\"E10 repeat-candidate model Precision@12: {e10_p12_for_compare:.6f}\")\n\nif not pd.isna(best_p12_e10b_for_compare):\n    print(f\"E10B best diagnostic rerank Precision@12: {best_p12_e10b_for_compare:.6f}\")\n\nprint(f\"E11 repeat + popular candidate model Precision@12: {e11_p12:.6f}\")\nprint(f\"E11 uplift vs E01: {e11_uplift_vs_e01:.2f}%\")\n\nif not pd.isna(e11_delta_vs_e08):\n    print(f\"E11 delta vs E08: {e11_delta_vs_e08:.2f}%\")\n\nif not pd.isna(e11_delta_vs_e10):\n    print(f\"E11 delta vs E10: {e11_delta_vs_e10:.2f}%\")\n\nif not pd.isna(e11_delta_vs_e10b):\n    print(f\"E11 delta vs E10B diagnostic best: {e11_delta_vs_e10b:.2f}%\")\n\nprint(\"\\n=== AUC diagnostic ===\")\nprint(f\"E11 CatBoost Score_Dem_E11 AUC: {auc_score_dem_e11:.6f}\")\nprint(f\"E11 Hybrid_Score_E11 AUC: {auc_hybrid_e11:.6f}\")\n\n\n# ------------------------------------------------------------\n# 14. Candidate source diagnostics\n# ------------------------------------------------------------\n\ntop12_e11 = (\n    e11_candidates_df\n    .sort_values([\"user_idx\", \"Hybrid_Score_E11\"], ascending=[True, False])\n    .groupby(\"user_idx\")\n    .head(12)\n    .copy()\n)\n\ne11_candidates_df[\"source_label_e11\"] = source_label_e11(e11_candidates_df)\ntop12_e11[\"source_label_e11\"] = source_label_e11(top12_e11)\n\ne11_source_all = (\n    e11_candidates_df\n    .groupby(\"source_label_e11\")\n    .agg(\n        candidate_rows=(\"item_idx\", \"size\"),\n        candidate_positives=(\"target\", \"sum\"),\n        users_with_source=(\"user_idx\", \"nunique\")\n    )\n    .reset_index()\n)\n\ne11_source_top12 = (\n    top12_e11\n    .groupby(\"source_label_e11\")\n    .agg(\n        top12_rows=(\"item_idx\", \"size\"),\n        top12_hits=(\"target\", \"sum\"),\n        users_with_source_in_top12=(\"user_idx\", \"nunique\")\n    )\n    .reset_index()\n)\n\ne11_source_report = e11_source_all.merge(\n    e11_source_top12,\n    on=\"source_label_e11\",\n    how=\"left\"\n)\n\nfor col in [\"top12_rows\", \"top12_hits\", \"users_with_source_in_top12\"]:\n    e11_source_report[col] = e11_source_report[col].fillna(0)\n\ne11_source_report[\"candidate_hit_rate\"] = (\n    e11_source_report[\"candidate_positives\"] / e11_source_report[\"candidate_rows\"]\n)\n\ne11_source_report[\"top12_selection_rate\"] = (\n    e11_source_report[\"top12_rows\"] / e11_source_report[\"candidate_rows\"]\n)\n\ne11_source_report[\"top12_hit_rate\"] = np.where(\n    e11_source_report[\"top12_rows\"] > 0,\n    e11_source_report[\"top12_hits\"] / e11_source_report[\"top12_rows\"],\n    np.nan\n)\n\ne11_source_report = e11_source_report.sort_values(\n    \"top12_hit_rate\",\n    ascending=False\n)\n\nprint(\"\\n=== E11 candidate source diagnostics ===\")\ndisplay(e11_source_report)\n\n\n# ------------------------------------------------------------\n# 15. Feature importance\n# ------------------------------------------------------------\n\nfi_e11 = (\n    pd.DataFrame({\n        \"feature\": list(model_cb_e11.feature_names_),\n        \"importance\": model_cb_e11.get_feature_importance()\n    })\n    .sort_values(\"importance\", ascending=False)\n    .reset_index(drop=True)\n)\n\nprint(\"\\n=== E11 CatBoost feature importance: top 20 ===\")\ndisplay(fi_e11.head(20))\n\n\n# ------------------------------------------------------------\n# 16. Summary\n# ------------------------------------------------------------\n\ne11_summary = pd.DataFrame([\n    {\n        \"experiment_id\": \"E01\",\n        \"name\": \"ALS baseline only\",\n        \"Precision@12\": baseline_p12,\n        \"uplift_vs_E01_%\": 0.0,\n        \"notes\": \"Locked baseline\"\n    },\n    {\n        \"experiment_id\": \"E08\",\n        \"name\": \"Train-only popularity model\",\n        \"Precision@12\": e08_p12_for_compare,\n        \"uplift_vs_E01_%\": (\n            ((e08_p12_for_compare - baseline_p12) / baseline_p12) * 100\n            if not pd.isna(e08_p12_for_compare)\n            else np.nan\n        ),\n        \"notes\": \"Locked trained model before candidate expansion\"\n    },\n    {\n        \"experiment_id\": \"E10\",\n        \"name\": \"Repeat candidates + E08 features\",\n        \"Precision@12\": e10_p12_for_compare,\n        \"uplift_vs_E01_%\": (\n            ((e10_p12_for_compare - baseline_p12) / baseline_p12) * 100\n            if not pd.isna(e10_p12_for_compare)\n            else np.nan\n        ),\n        \"notes\": \"Repeat candidate expansion\"\n    },\n    {\n        \"experiment_id\": \"E11\",\n        \"name\": \"Repeat + popular candidates + E08 features\",\n        \"Precision@12\": e11_p12,\n        \"uplift_vs_E01_%\": e11_uplift_vs_e01,\n        \"notes\": \"ALS + repeat + train-popular candidates\"\n    }\n])\n\nprint(\"\\n=== E11 comparison summary ===\")\ndisplay(e11_summary)\n\n\n# ------------------------------------------------------------\n# 17. Safe decision\n# ------------------------------------------------------------\n\nif not pd.isna(e10_p12_for_compare) and e11_p12 > e10_p12_for_compare:\n    e11_decision = (\n        \"E11 improves over E10. Adding train-popular candidates improves the clean candidate-generation setup.\"\n    )\nelif not pd.isna(e10_p12_for_compare) and e11_p12 < e10_p12_for_compare:\n    e11_decision = (\n        \"E11 is below E10. Adding train-popular candidates did not improve the clean candidate-generation setup.\"\n    )\nelif not pd.isna(e10_p12_for_compare):\n    e11_decision = (\n        \"E11 matches E10. Train-popular candidates did not materially change Precision@12.\"\n    )\nelse:\n    e11_decision = (\n        \"E11 completed. Compare manually against E08/E10 because prior comparison objects were unavailable.\"\n    )\n\nprint(\"\\n=== Safe E11 interpretation ===\")\nprint(e11_decision)\n\nprint(\"\\nObjects to log:\")\nprint(\"- e11_summary\")\nprint(\"- e11_source_report\")\nprint(\"- fi_e11\")\nprint(\"- e11_candidates_df\")\nprint(\"- alpha_e11, beta_e11, intercept_e11\")\nprint(\"- auc_score_dem_e11, auc_hybrid_e11\")\nprint(\"- e11_decision\")\n\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-28T20:54:32.494903Z","iopub.execute_input":"2026-05-28T20:54:32.497939Z","iopub.status.idle":"2026-05-28T20:57:37.606053Z","shell.execute_reply.started":"2026-05-28T20:54:32.497856Z","shell.execute_reply":"2026-05-28T20:57:37.604790Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport pandas as pd\nimport numpy as np\n\n# Налаштування стилю\nsns.set_style(\"whitegrid\")\nplt.rcParams.update({'font.size': 12})\n\n# ---------------------------------------------------------\nprint(\"1. Розрахунок метрики Precision@12...\")\n# ---------------------------------------------------------\n\ndef calculate_precision_at_12(df, score_column):\n    # Відбираємо Топ-12 для кожного користувача\n    top_12 = df.sort_values(['user_idx', score_column], ascending=[True, False]) \\\n               .groupby('user_idx').head(12)\n    \n    # Рахуємо кількість попадань (target=1)\n    user_hits = top_12.groupby('user_idx')['target'].sum()\n    \n    # Precision = (кількість вгаданих) / 12\n    precision = (user_hits / 12).mean()\n    return precision\n\n# Розрахунок метрик\nbaseline_p12 = calculate_precision_at_12(als_candidates_df, 'Score_CF')\nhybrid_p12 = calculate_precision_at_12(als_candidates_df, 'Hybrid_Score')\nuplift = ((hybrid_p12 - baseline_p12) / baseline_p12) * 100\n\nprint(f\"Precision@12 (Baseline ALS): {baseline_p12:.5f}\")\nprint(f\"Precision@12 (Hybrid Model): {hybrid_p12:.5f}\")\nprint(f\"Покращення (Uplift): {uplift:.2f}%\")\n\n# ---------------------------------------------------------\nprint(\"\\n2. Візуалізація результатів (без помилок)...\")\n# ---------------------------------------------------------\n\nfig = plt.figure(figsize=(20, 7))\n\n# --- ГРАФІК 1: Порівняння моделей ---\nplt.subplot(1, 3, 1)\nmodels = ['Baseline (ALS)', 'Hybrid (Ensemble)']\nvalues = [baseline_p12, hybrid_p12]\n\n# Використовуємо hue для уникнення FutureWarning\nsns.barplot(x=models, y=values, hue=models, palette=['#aec6cf', '#ffb347'], legend=False)\nplt.title('Порівняння точності Precision@12', fontsize=15, fontweight='bold', pad=20)\nplt.ylabel('Середній Precision@12')\n\nfor i, v in enumerate(values):\n    plt.text(i, v + (v * 0.01), f'{v:.5f}', ha='center', fontweight='bold', fontsize=12)\n\n# --- ГРАФІК 2: Важливість ознак ---\nplt.subplot(1, 3, 2)\nimportances = model_cb.get_feature_importance()\nfeature_names = ['age', 'postal_code', 'inferred_gender', 'product_type_name', \n                 'graphical_appearance_name', 'colour_group_name', \n                 'perceived_colour_value_name', 'index_name', 'index_group_name']\n\nfi_df = pd.DataFrame({'feature': feature_names, 'importance': importances})\nfi_df = fi_df.sort_values(by='importance', ascending=False)\n\n# Використовуємо hue='feature' для відповідності новим стандартам Seaborn\nsns.barplot(x='importance', y='feature', data=fi_df, hue='feature', palette='viridis', legend=False)\nplt.title('Важливість ознак (Feature Importance)', fontsize=15, fontweight='bold', pad=20)\nplt.xlabel('Вплив на результат (%)')\nplt.ylabel('') # Прибираємо зайвий підпис осі\n\n# --- ГРАФІК 3: Розподіл ваг ---\nplt.subplot(1, 3, 3)\n# Наші реальні ваги з Кроку 7\nalpha_val = 0.2384\nbeta_val = 0.6497\nlabels = ['Collaborative (Alpha)', 'Demographic (Beta)']\nsizes = [alpha_val, beta_val]\n\nplt.pie(sizes, labels=labels, autopct='%1.1f%%', shadow=True, \n        startangle=140, colors=['#99ff99', '#66b3ff'], explode=(0.1, 0))\nplt.title('Внесок моделей у фінальний бал', fontsize=15, fontweight='bold', pad=20)\n\nplt.tight_layout()\nplt.show()\n\n# ---------------------------------------------------------\nprint(\"\\n\" + \"=\"*60)\nprint(\"ФІНАЛЬНИЙ ЗВІТ:\")\nprint(f\"Метрика Precision@12 зросла з {baseline_p12:.5f} до {hybrid_p12:.5f}\")\nprint(f\"Це підтверджує ефективність гібридного підходу (Uplift: {uplift:.2f}%)\")\nprint(f\"Найважливіший фактор сегментації: {fi_df.iloc[0]['feature']}\")\nprint(\"=\"*60)","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null}]}