{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.17","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":105399,"databundleVersionId":12733338,"sourceType":"competition"}],"dockerImageVersionId":31042,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# AeroClub RecSys 2025 - CatBoost Ranking Baseline\n\nThis notebook implements a ranking approach using CatBoost for the AeroClub recommendation challenge. The task is to predict which flight option a user will select from a list of available flights.\n\n## Key Features:\n- Feature engineering for flight data (duration, price, segments, etc.)\n- CatBoost Ranker with YetiRank loss function\n- Proper group-based train/validation split\n- Evaluation metrics including LogLoss and Top-1 Accuracy","metadata":{}},{"cell_type":"code","source":"# Import necessary libraries\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import GroupShuffleSplit\nfrom sklearn.metrics import log_loss\nimport matplotlib.pyplot as plt\n# from kaggle.api.kaggle_api_extended import KaggleApi\n\n# # Set display options for better readability\n# pd.set_option('display.max_columns', 50)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T13:37:31.711383Z","iopub.execute_input":"2025-06-23T13:37:31.711650Z","iopub.status.idle":"2025-06-23T13:37:37.679822Z","shell.execute_reply.started":"2025-06-23T13:37:31.711620Z","shell.execute_reply":"2025-06-23T13:37:37.674865Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1. Configuration","metadata":{}},{"cell_type":"code","source":"# Global parameters\nTRAIN_SAMPLE_FRAC = 0.20  # Sample 30% of data for faster iteration\nRANDOM_STATE = 42\nnp.random.seed(RANDOM_STATE)\n\n# Initialize Kaggle API\n# api = KaggleApi()\n# api.authenticate()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T13:37:37.680639Z","iopub.execute_input":"2025-06-23T13:37:37.680990Z","iopub.status.idle":"2025-06-23T13:37:37.689627Z","shell.execute_reply.started":"2025-06-23T13:37:37.680965Z","shell.execute_reply":"2025-06-23T13:37:37.686241Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Load Data","metadata":{}},{"cell_type":"code","source":"# Load parquet files\ntrain = pd.read_parquet('/kaggle/input/aeroclub-recsys-2025/train.parquet')\ntest = pd.read_parquet('/kaggle/input/aeroclub-recsys-2025/test.parquet')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T13:37:37.909187Z","iopub.execute_input":"2025-06-23T13:37:37.909461Z","iopub.status.idle":"2025-06-23T13:38:10.537304Z","shell.execute_reply.started":"2025-06-23T13:37:37.909438Z","shell.execute_reply":"2025-06-23T13:38:10.531617Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Train shape: {train.shape}, Test shape: {test.shape}\")\nprint(f\"Unique ranker_ids in train: {train['ranker_id'].nunique():,}\")\nprint(f\"Selected rate: {train['selected'].mean():.3f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T13:38:13.224243Z","iopub.execute_input":"2025-06-23T13:38:13.224574Z","iopub.status.idle":"2025-06-23T13:38:14.613195Z","shell.execute_reply.started":"2025-06-23T13:38:13.224503Z","shell.execute_reply":"2025-06-23T13:38:14.608605Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. Data Sampling & Preprocessing","metadata":{}},{"cell_type":"code","source":"# Sample by ranker_id to keep groups intact\nif TRAIN_SAMPLE_FRAC < 1.0:\n    unique_rankers = train['ranker_id'].unique()\n    n_sample = int(len(unique_rankers) * TRAIN_SAMPLE_FRAC)\n    sampled_rankers = np.random.RandomState(RANDOM_STATE).choice(\n        unique_rankers, size=n_sample, replace=False\n    )\n    train = train[train['ranker_id'].isin(sampled_rankers)]\n    print(f\"Sampled train to {len(train):,} rows ({train['ranker_id'].nunique():,} groups)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T13:38:16.373721Z","iopub.execute_input":"2025-06-23T13:38:16.374014Z","iopub.status.idle":"2025-06-23T13:38:24.870590Z","shell.execute_reply.started":"2025-06-23T13:38:16.373988Z","shell.execute_reply":"2025-06-23T13:38:24.864804Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Convert ranker_id to string for CatBoost\ntrain['ranker_id'] = train['ranker_id'].astype(str)\ntest['ranker_id'] = test['ranker_id'].astype(str)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T13:38:25.984899Z","iopub.execute_input":"2025-06-23T13:38:25.985179Z","iopub.status.idle":"2025-06-23T13:38:26.183714Z","shell.execute_reply.started":"2025-06-23T13:38:25.985154Z","shell.execute_reply":"2025-06-23T13:38:26.178319Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. Feature Engineering","metadata":{}},{"cell_type":"code","source":"cat_features = [\n    'nationality', 'searchRoute', 'corporateTariffCode',\n    # Leg 0 segments 0-1\n    'legs0_segments0_aircraft_code', 'legs0_segments0_arrivalTo_airport_city_iata',\n    'legs0_segments0_arrivalTo_airport_iata', 'legs0_segments0_departureFrom_airport_iata',\n    'legs0_segments0_marketingCarrier_code', 'legs0_segments0_operatingCarrier_code',\n    'legs0_segments0_flightNumber',\n    'legs0_segments1_aircraft_code', 'legs0_segments1_arrivalTo_airport_city_iata',\n    'legs0_segments1_arrivalTo_airport_iata', 'legs0_segments1_departureFrom_airport_iata',\n    'legs0_segments1_marketingCarrier_code', 'legs0_segments1_operatingCarrier_code',\n    'legs0_segments1_flightNumber',\n    # Leg 1 segments 0-1\n    'legs1_segments0_aircraft_code', 'legs1_segments0_arrivalTo_airport_city_iata',\n    'legs1_segments0_arrivalTo_airport_iata', 'legs1_segments0_departureFrom_airport_iata',\n    'legs1_segments0_marketingCarrier_code', 'legs1_segments0_operatingCarrier_code',\n    'legs1_segments0_flightNumber',\n    'legs1_segments1_aircraft_code', 'legs1_segments1_arrivalTo_airport_city_iata',\n    'legs1_segments1_arrivalTo_airport_iata', 'legs1_segments1_departureFrom_airport_iata',\n    'legs1_segments1_marketingCarrier_code', 'legs1_segments1_operatingCarrier_code',\n    'legs1_segments1_flightNumber'\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T13:38:27.180631Z","iopub.execute_input":"2025-06-23T13:38:27.180970Z","iopub.status.idle":"2025-06-23T13:38:27.193229Z","shell.execute_reply.started":"2025-06-23T13:38:27.180943Z","shell.execute_reply":"2025-06-23T13:38:27.187343Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_features(df):\n    \"\"\"\n    Return a copy of df enriched with engineered features for flight-ranking models.\n    \"\"\"\n    df = df.copy()\n\n    def hms_to_minutes(s: pd.Series) -> np.ndarray:\n        \"\"\"Vectorised 'HH:MM:SS' → minutes (seconds ignored).\"\"\"\n        mask = s.notna()\n        out = np.zeros(len(s), dtype=float)\n        if mask.any():\n            parts = s[mask].astype(str).str.split(':', expand=True)\n            out[mask] = (\n                pd.to_numeric(parts[0], errors=\"coerce\").fillna(0) * 60\n                + pd.to_numeric(parts[1], errors=\"coerce\").fillna(0)\n            )\n        return out\n\n    # Duration columns\n    dur_cols = (\n        [\"legs0_duration\", \"legs1_duration\"]\n        + [f\"legs{l}_segments{s}_duration\" for l in (0, 1) for s in (0, 1)]\n    )\n    for col in dur_cols:\n        if col in df.columns:\n            df[col] = hms_to_minutes(df[col])\n\n    # Feature container\n    feat = {}\n\n    # Price\n    feat[\"price_per_tax\"] = df[\"totalPrice\"] / (df[\"taxes\"] + 1)\n    feat[\"tax_rate\"] = df[\"taxes\"] / (df[\"totalPrice\"] + 1)\n    feat[\"log_price\"] = np.log1p(df[\"totalPrice\"])\n\n    # Durations\n    df[\"total_duration\"] = df[\"legs0_duration\"].fillna(0) + df[\"legs1_duration\"].fillna(0)\n    feat[\"duration_ratio\"] = np.where(\n        df[\"legs1_duration\"].fillna(0) > 0,\n        df[\"legs0_duration\"] / (df[\"legs1_duration\"] + 1),\n        1.0,\n    )\n\n    # Segment counts\n    for leg in (0, 1):\n        seg_cols = [f\"legs{leg}_segments{i}_duration\" for i in (0, 1)]\n        feat[f\"n_segments_leg{leg}\"] = df[seg_cols].notna().sum(axis=1)\n    feat[\"total_segments\"] = feat[\"n_segments_leg0\"] + feat[\"n_segments_leg1\"]\n\n    # Trip type\n    feat[\"is_one_way\"] = df[\"legs1_duration\"].isna().astype(int)\n\n    # Rank features\n    grp = df.groupby(\"ranker_id\")\n    feat[\"price_rank\"] = grp[\"totalPrice\"].rank()\n    feat[\"price_pct_rank\"] = grp[\"totalPrice\"].rank(pct=True)\n    feat[\"duration_rank\"] = grp[\"total_duration\"].rank()\n    feat[\"is_cheapest\"] = (grp[\"totalPrice\"].transform(\"min\") == df[\"totalPrice\"]).astype(int)\n    feat[\"is_most_expensive\"] = (grp[\"totalPrice\"].transform(\"max\") == df[\"totalPrice\"]).astype(int)\n    feat[\"price_from_median\"] = grp[\"totalPrice\"].transform(\n        lambda x: (x - x.median()) / (x.std() + 1)\n    )\n\n    # Frequent-flyer\n    ff = df[\"frequentFlyer\"].fillna(\"\").astype(str)\n    feat[\"n_ff_programs\"] = ff.str.count(\"/\") + (ff != \"\")\n    airlines = [\"SU\", \"S7\", \"U6\", \"TK\", \"DP\", \"UT\", \"EK\", \"N4\", \"5N\", \"LH\"]\n    for al in airlines:\n        feat[f\"ff_{al}\"] = ff.str.contains(rf\"\\b{al}\\b\").astype(int)\n    feat[\"ff_matches_carrier\"] = np.select(\n        [\n            (feat[f\"ff_{al}\"] == 1)\n            & (df[\"legs0_segments0_marketingCarrier_code\"] == al)\n            for al in [\"SU\", \"S7\", \"U6\", \"TK\"]\n        ],\n        [1, 1, 1, 1],\n        default=0,\n    )\n\n    # Binary flags\n    feat.update(\n        dict(\n            is_vip_freq=((df[\"isVip\"] == 1) | (feat[\"n_ff_programs\"] > 0)).astype(int),\n            has_return=(~df[\"legs1_duration\"].isna()).astype(int),\n            has_corporate_tariff=(~df[\"corporateTariffCode\"].isna()).astype(int),\n        )\n    )\n\n    # Baggage and fees\n    feat[\"baggage_total\"] = (\n        df[\"legs0_segments0_baggageAllowance_quantity\"].fillna(0)\n        + df[\"legs1_segments0_baggageAllowance_quantity\"].fillna(0)\n    )\n    feat[\"has_baggage\"] = (feat[\"baggage_total\"] > 0).astype(int)\n    feat[\"total_fees\"] = (\n        df[\"miniRules0_monetaryAmount\"].fillna(0) + df[\"miniRules1_monetaryAmount\"].fillna(0)\n    )\n    feat[\"has_fees\"] = (feat[\"total_fees\"] > 0).astype(int)\n    feat[\"fee_rate\"] = feat[\"total_fees\"] / (df[\"totalPrice\"] + 1)\n\n    # Time-of-day\n    for col in (\"legs0_departureAt\", \"legs0_arrivalAt\", \"legs1_departureAt\", \"legs1_arrivalAt\"):\n        if col in df.columns:\n            dt = pd.to_datetime(df[col], errors=\"coerce\")\n            feat[f\"{col}_hour\"] = dt.dt.hour.fillna(12)\n            feat[f\"{col}_weekday\"] = dt.dt.weekday.fillna(0)\n            h = dt.dt.hour.fillna(12)\n            feat[f\"{col}_business_time\"] = (((6 <= h) & (h <= 9)) | ((17 <= h) & (h <= 20))).astype(int)\n\n    # Direct-flight flags\n    feat[\"is_direct_leg0\"] = (feat[\"n_segments_leg0\"] == 1).astype(int)\n    feat[\"is_direct_leg1\"] = (feat[\"n_segments_leg1\"] == 1).astype(int)\n    feat[\"both_direct\"] = feat[\"is_direct_leg0\"] & feat[\"is_direct_leg1\"]\n\n    # Cheapest direct\n    df[\"_direct\"] = feat[\"n_segments_leg0\"] == 1\n    direct_min_price = df.loc[df[\"_direct\"]].groupby(\"ranker_id\")[\"totalPrice\"].min()\n    feat[\"is_direct_cheapest\"] = (\n        df[\"_direct\"] & (df[\"totalPrice\"] == df[\"ranker_id\"].map(direct_min_price))\n    ).astype(int)\n    df.drop(columns=\"_direct\", inplace=True)\n\n    # Misc flags\n    feat[\"has_access_tp\"] = (df[\"pricingInfo_isAccessTP\"] == 1).astype(int)\n    feat[\"group_size\"] = df.groupby(\"ranker_id\")[\"Id\"].transform(\"count\")\n    feat[\"group_size_log\"] = np.log1p(feat[\"group_size\"])\n    feat[\"is_major_carrier\"] = df[\"legs0_segments0_marketingCarrier_code\"].isin([\"SU\", \"S7\", \"U6\"]).astype(int)\n    popular_routes = {\"MOWLED/LEDMOW\", \"LEDMOW/MOWLED\", \"MOWLED\", \"LEDMOW\", \"MOWAER/AERMOW\"}\n    feat[\"is_popular_route\"] = df[\"searchRoute\"].isin(popular_routes).astype(int)\n    feat[\"avg_cabin_class\"] = df[[\"legs0_segments0_cabinClass\", \"legs1_segments0_cabinClass\"]].mean(axis=1)\n    feat[\"cabin_class_diff\"] = (\n        df[\"legs0_segments0_cabinClass\"].fillna(0) - df[\"legs1_segments0_cabinClass\"].fillna(0)\n    )\n\n    # Merge new features\n    df = pd.concat([df, pd.DataFrame(feat, index=df.index)], axis=1)\n\n    # Final NaN handling (loop avoids duplicate-column error)\n    for col in df.select_dtypes(include=\"number\").columns:\n        df[col] = df[col].fillna(0)\n    for col in df.select_dtypes(include=\"object\").columns:\n        df[col] = df[col].fillna(\"missing\")\n\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T13:38:27.836866Z","iopub.execute_input":"2025-06-23T13:38:27.837154Z","iopub.status.idle":"2025-06-23T13:38:27.858722Z","shell.execute_reply.started":"2025-06-23T13:38:27.837129Z","shell.execute_reply":"2025-06-23T13:38:27.854432Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Apply feature engineering\ntrain = create_features(train)\ntest = create_features(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T13:38:29.554663Z","iopub.execute_input":"2025-06-23T13:38:29.554963Z","iopub.status.idle":"2025-06-23T13:40:42.761947Z","shell.execute_reply.started":"2025-06-23T13:38:29.554936Z","shell.execute_reply":"2025-06-23T13:40:42.757144Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. Feature Selection","metadata":{}},{"cell_type":"code","source":"# Exclude columns\nexclude_cols = ['Id', 'ranker_id', 'selected', 'profileId', 'requestDate',\n                'legs0_departureAt', 'legs0_arrivalAt', 'legs1_departureAt', 'legs1_arrivalAt',\n                'miniRules0_percentage', 'miniRules1_percentage',  # >90% missing\n                'frequentFlyer']  # Already processed\n\n# Exclude segment 2-3 columns (>98% missing)\nfor leg in [0, 1]:\n    for seg in [2, 3]:\n        for suffix in ['aircraft_code', 'arrivalTo_airport_city_iata', 'arrivalTo_airport_iata',\n                      'baggageAllowance_quantity', 'baggageAllowance_weightMeasurementType',\n                      'cabinClass', 'departureFrom_airport_iata', 'duration', 'flightNumber',\n                      'marketingCarrier_code', 'operatingCarrier_code', 'seatsAvailable']:\n            exclude_cols.append(f'legs{leg}_segments{seg}_{suffix}')\n\nfeature_cols = [col for col in train.columns if col not in exclude_cols]\ncat_features_final = [col for col in cat_features if col in feature_cols]\n\nprint(f\"Using {len(feature_cols)} features ({len(cat_features_final)} categorical)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T13:41:00.824764Z","iopub.execute_input":"2025-06-23T13:41:00.825065Z","iopub.status.idle":"2025-06-23T13:41:00.838170Z","shell.execute_reply.started":"2025-06-23T13:41:00.825040Z","shell.execute_reply":"2025-06-23T13:41:00.832597Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6. Train/Validation Split","metadata":{}},{"cell_type":"code","source":"# Prepare data\nX_train = train[feature_cols]\ny_train = train['selected']\ngroups_train = train['ranker_id']\n\nX_test = test[feature_cols]\ngroups_test = test['ranker_id']\n\n# Group-based split\ngss = GroupShuffleSplit(n_splits=1, test_size=0.2, random_state=RANDOM_STATE)\ntrain_idx, val_idx = next(gss.split(X_train, y_train, groups_train))\n\nX_tr, X_val = X_train.iloc[train_idx], X_train.iloc[val_idx]\ny_tr, y_val = y_train.iloc[train_idx], y_train.iloc[val_idx]\ngroups_tr, groups_val = groups_train.iloc[train_idx], groups_train.iloc[val_idx]\n\nprint(f\"Train: {len(X_tr):,} rows, Val: {len(X_val):,} rows, Test: {len(X_test):,} rows\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T13:41:03.617318Z","iopub.execute_input":"2025-06-23T13:41:03.617811Z","iopub.status.idle":"2025-06-23T13:41:11.862457Z","shell.execute_reply.started":"2025-06-23T13:41:03.617752Z","shell.execute_reply":"2025-06-23T13:41:11.857824Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Quick data exploration\n# fig, axes = plt.subplots(1, 2, figsize=(12, 4))\n\n# # Selection rate by price rank\n# train_sample = train.sample(min(10000, len(train)))\n# price_rank_selection = train_sample.groupby('price_rank')['selected'].mean()\n# axes[0].plot(price_rank_selection.index[:20], price_rank_selection.values[:20], marker='o')\n# axes[0].set_xlabel('Price Rank within Group')\n# axes[0].set_ylabel('Selection Rate')\n# axes[0].set_title('Selection Rate by Price Rank')\n\n# # Direct vs connecting flights\n# direct_selection = train.groupby('total_segments')['selected'].mean()\n# axes[1].bar(direct_selection.index, direct_selection.values)\n# axes[1].set_xlabel('Total Segments')\n# axes[1].set_ylabel('Selection Rate')\n# axes[1].set_title('Selection Rate by Number of Segments')\n\n# plt.tight_layout()\n# plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T13:41:51.358397Z","iopub.execute_input":"2025-06-23T13:41:51.358815Z","iopub.status.idle":"2025-06-23T13:41:51.910795Z","shell.execute_reply.started":"2025-06-23T13:41:51.358776Z","shell.execute_reply":"2025-06-23T13:41:51.906716Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7. Model Training","metadata":{}},{"cell_type":"code","source":"%%capture\n!pip install -U catboost","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T00:29:59.524414Z","iopub.execute_input":"2025-06-24T00:29:59.524785Z","iopub.status.idle":"2025-06-24T00:30:23.871421Z","shell.execute_reply.started":"2025-06-24T00:29:59.524758Z","shell.execute_reply":"2025-06-24T00:30:23.865878Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from catboost import CatBoostRanker, Pool","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create CatBoost pools\ntrain_pool = Pool(X_tr, y_tr, group_id=groups_tr, cat_features=cat_features_final)\nval_pool = Pool(X_val, y_val, group_id=groups_val, cat_features=cat_features_final)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T13:42:36.546498Z","iopub.execute_input":"2025-06-23T13:42:36.546884Z","iopub.status.idle":"2025-06-23T13:42:46.662032Z","shell.execute_reply.started":"2025-06-23T13:42:36.546854Z","shell.execute_reply":"2025-06-23T13:42:46.657387Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Initialize CatBoost Ranker\nmodel = CatBoostRanker(\n    loss_function='YetiRank',\n    iterations=1000,\n    learning_rate=0.02,\n    depth=8,\n    l2_leaf_reg=0.2,\n    random_seed=RANDOM_STATE,\n    eval_metric='PrecisionAt:top=3',\n    early_stopping_rounds=100,\n    verbose=20,\n    task_type='CPU',  # Change to 'GPU' if available\n    cat_features=cat_features_final,\n#     grow_policy='Lossguide'\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T13:42:46.663127Z","iopub.execute_input":"2025-06-23T13:42:46.663353Z","iopub.status.idle":"2025-06-23T13:42:46.674307Z","shell.execute_reply.started":"2025-06-23T13:42:46.663330Z","shell.execute_reply":"2025-06-23T13:42:46.669101Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train model\nmodel.fit(train_pool, eval_set=val_pool, use_best_model=True);","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T13:42:46.675454Z","iopub.execute_input":"2025-06-23T13:42:46.675698Z","iopub.status.idle":"2025-06-23T13:46:01.941587Z","shell.execute_reply.started":"2025-06-23T13:42:46.675674Z","shell.execute_reply":"2025-06-23T13:46:01.936039Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 8. Model Evaluation","metadata":{}},{"cell_type":"code","source":"# Convert scores to probabilities using sigmoid\ndef sigmoid(x):\n    return 1 / (1 + np.exp(-x / 10))  # Scale factor for CatBoost scores\n\n# HitRate@3 calculation\ndef calculate_hitrate_at_k(df, k=3):\n    \"\"\"Calculate HitRate@k for groups with >10 options\"\"\"\n    hits = []\n    for ranker_id, group in df.groupby('ranker_id'):\n        # Only consider groups with >10 options\n        if len(group) > 10:\n            # Get top-k predictions\n            top_k = group.nlargest(k, 'pred')\n            # Check if selected item is in top-k\n            hit = (top_k['selected'] == 1).any()\n            hits.append(hit)\n    return np.mean(hits) if hits else 0.0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T01:10:22.863802Z","iopub.execute_input":"2025-06-24T01:10:22.864172Z","iopub.status.idle":"2025-06-24T01:10:22.878514Z","shell.execute_reply.started":"2025-06-24T01:10:22.864143Z","shell.execute_reply":"2025-06-24T01:10:22.872095Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Evaluate on validation\nval_preds = model.predict(X_val)\nval_df = pd.DataFrame({\n    'ranker_id': groups_val,\n    'pred': val_preds,\n    'selected': y_val\n})\n\n# Get top prediction per group\ntop_preds = val_df.loc[val_df.groupby('ranker_id')['pred'].idxmax()]\ntop_preds['prob'] = sigmoid(top_preds['pred'])\nval_logloss = log_loss(top_preds['selected'], top_preds['prob'])\n\nhitrate_at_3 = calculate_hitrate_at_k(val_df, k=3)\n\n# Additional metrics\nval_accuracy = (top_preds['selected'] == 1).mean()\ngroup_sizes = val_df.groupby('ranker_id').size()\navg_group_size = group_sizes.mean()\n\nprint(f\"HitRate@3 (groups >10):  {hitrate_at_3:.4f}\")\nprint(f\"\\nLogLoss:                 {val_logloss:.4f}\")\nprint(f\"Top-1 Accuracy:          {val_accuracy:.4f}\")\nprint(f\"Groups with >10 options: {(group_sizes > 10).sum()} / {len(group_sizes)} ({(group_sizes > 10).mean():.1%})\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T13:46:39.658673Z","iopub.execute_input":"2025-06-23T13:46:39.658972Z","iopub.status.idle":"2025-06-23T13:46:41.720345Z","shell.execute_reply.started":"2025-06-23T13:46:39.658945Z","shell.execute_reply":"2025-06-23T13:46:41.715385Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Feature importance\n# feature_importance = pd.DataFrame({\n#     'feature': feature_cols,\n#     'importance': model.get_feature_importance(data=val_pool, type='LossFunctionChange')\n# }).sort_values('importance', ascending=False)\n\n# plt.figure(figsize=(10, 8))\n# top_features = feature_importance.head(20)\n# plt.barh(range(len(top_features)), top_features['importance'])\n# plt.yticks(range(len(top_features)), top_features['feature'])\n# plt.xlabel('Feature Importance')\n# plt.title('Top 20 Most Important Features')\n# plt.gca().invert_yaxis()\n# plt.tight_layout()\n# plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T13:46:52.031393Z","iopub.execute_input":"2025-06-23T13:46:52.031746Z","iopub.status.idle":"2025-06-23T13:49:08.638066Z","shell.execute_reply.started":"2025-06-23T13:46:52.031717Z","shell.execute_reply":"2025-06-23T13:49:08.631862Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 9. Generate Predictions","metadata":{}},{"cell_type":"code","source":"# Create test pool and predict\ntest_pool = Pool(X_test, group_id=groups_test, cat_features=cat_features_final)\ntest_preds = model.predict(test_pool)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T13:49:49.665083Z","iopub.execute_input":"2025-06-23T13:49:49.665466Z","iopub.status.idle":"2025-06-23T13:50:31.225319Z","shell.execute_reply.started":"2025-06-23T13:49:49.665432Z","shell.execute_reply":"2025-06-23T13:50:31.218123Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create submission\nsubmission = test[['Id', 'ranker_id']].copy()\nsubmission['pred_score'] = test_preds\n\n# Assign ranks (1 = best option)\nsubmission['selected'] = submission.groupby('ranker_id')['pred_score'].rank(\n    ascending=False, method='first'\n).astype(int)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T13:50:31.226139Z","iopub.execute_input":"2025-06-23T13:50:31.226355Z","iopub.status.idle":"2025-06-23T13:50:33.605990Z","shell.execute_reply.started":"2025-06-23T13:50:31.226331Z","shell.execute_reply":"2025-06-23T13:50:33.600768Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Verify ranking integrity\n# assert submission.groupby('ranker_id')['selected'].apply(\n#     lambda x: sorted(x.tolist()) == list(range(1, len(x)+1))\n# ).all(), \"Invalid ranking!\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T13:50:33.608491Z","iopub.execute_input":"2025-06-23T13:50:33.608959Z","iopub.status.idle":"2025-06-23T13:50:36.039127Z","shell.execute_reply.started":"2025-06-23T13:50:33.608932Z","shell.execute_reply":"2025-06-23T13:50:36.033637Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save submission\n# submission[['Id', 'ranker_id', 'selected']].to_parquet('submission.parquet', index=False)\nsubmission[['Id', 'ranker_id', 'selected']].to_csv('submission.csv', index=False)\nprint(f\"Submission saved. Shape: {submission.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T13:54:24.573555Z","iopub.execute_input":"2025-06-23T13:54:24.573900Z","iopub.status.idle":"2025-06-23T13:54:36.794753Z","shell.execute_reply.started":"2025-06-23T13:54:24.573873Z","shell.execute_reply":"2025-06-23T13:54:36.788238Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Submit to Competition with API","metadata":{}},{"cell_type":"code","source":"# # Submit to competition\n# api.competition_submit(\n#     file_name=\"submission.parquet\", \n#     competition=\"aeroclub-recsys-2025\", \n#     message=\"CatBoost Ranking Baseline\"\n# )","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Potential improvements:\n- Add more sophisticated time-based features\n- Engineer features based on user preferences (profile analysis)\n- Experiment with different ranking loss functions\n- Ensemble with other models (LightGBM, XGBoost)\n- Hyperparameter tuning","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}