{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.18"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":105399,"databundleVersionId":12733338,"sourceType":"competition"}],"dockerImageVersionId":31091,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":6343.409895,"end_time":"2025-08-11T19:27:05.865848","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-08-11T17:41:22.455953","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"18a4068f","cell_type":"code","source":"!pip install polars\n!pip install xgboost","metadata":{"execution":{"iopub.execute_input":"2025-08-11T17:41:25.603430Z","iopub.status.busy":"2025-08-11T17:41:25.603227Z","iopub.status.idle":"2025-08-11T17:41:42.639026Z","shell.execute_reply":"2025-08-11T17:41:42.634437Z"},"papermill":{"duration":17.04618,"end_time":"2025-08-11T17:41:42.642100","exception":false,"start_time":"2025-08-11T17:41:25.595920","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"07bf81e9","cell_type":"code","source":"import os\nimport math\nimport gc\nfrom typing import List, Dict, Tuple\n\nimport numpy as np\nimport polars as pl\nimport xgboost as xgb\n\nfrom sklearn.model_selection import GroupKFold\n","metadata":{"execution":{"iopub.execute_input":"2025-08-11T17:41:42.662138Z","iopub.status.busy":"2025-08-11T17:41:42.661896Z","iopub.status.idle":"2025-08-11T17:41:49.271581Z","shell.execute_reply":"2025-08-11T17:41:49.267113Z"},"papermill":{"duration":6.624193,"end_time":"2025-08-11T17:41:49.274809","exception":false,"start_time":"2025-08-11T17:41:42.650616","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"7d8434e4","cell_type":"code","source":"# -----------------------------\n# Config\n# -----------------------------\nDATA_DIR = \"/kaggle/input/aeroclub-recsys-2025\"\nTRAIN_PATH = os.path.join(DATA_DIR, \"train.parquet\")\nTEST_PATH  = os.path.join(DATA_DIR, \"test.parquet\")\n\nRANDOM_STATE = 42\nnp.random.seed(RANDOM_STATE)\n\n# Outer data split\nHOLDOUT_FRAC = 0.05   # validator_ share\nN_FOLDS       = 10     # folds on the remaining 95%\nUSE_FOLD      = 0     # train on fold==0 to start\n\n# Target encoding (m-estimate smoothing)\nTE_PRIOR_M = 50.0\n\n\n","metadata":{"execution":{"iopub.execute_input":"2025-08-11T17:41:49.295997Z","iopub.status.busy":"2025-08-11T17:41:49.295654Z","iopub.status.idle":"2025-08-11T17:41:49.305470Z","shell.execute_reply":"2025-08-11T17:41:49.301487Z"},"papermill":{"duration":0.023299,"end_time":"2025-08-11T17:41:49.307942","exception":false,"start_time":"2025-08-11T17:41:49.284643","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"c76a90ba","cell_type":"code","source":"# -----------------------------\n# Columns / helpers\n# -----------------------------\nBASIC_CAT_COLS = [\n    \"nationality\", \"searchRoute\", \"corporateTariffCode\", \"bySelf\", \"sex\", \"companyID\",\n    # leg 0 segments 0-1\n    \"legs0_segments0_aircraft_code\", \"legs0_segments0_arrivalTo_airport_city_iata\",\n    \"legs0_segments0_arrivalTo_airport_iata\", \"legs0_segments0_departureFrom_airport_iata\",\n    \"legs0_segments0_marketingCarrier_code\", \"legs0_segments0_operatingCarrier_code\",\n    \"legs0_segments0_flightNumber\",\n    \"legs0_segments1_aircraft_code\", \"legs0_segments1_arrivalTo_airport_city_iata\",\n    \"legs0_segments1_arrivalTo_airport_iata\", \"legs0_segments1_departureFrom_airport_iata\",\n    \"legs0_segments1_marketingCarrier_code\", \"legs0_segments1_operatingCarrier_code\",\n    \"legs0_segments1_flightNumber\",\n    # leg 1 segments 0-1\n    \"legs1_segments0_aircraft_code\", \"legs1_segments0_arrivalTo_airport_city_iata\",\n    \"legs1_segments0_arrivalTo_airport_iata\", \"legs1_segments0_departureFrom_airport_iata\",\n    \"legs1_segments0_marketingCarrier_code\", \"legs1_segments0_operatingCarrier_code\",\n    \"legs1_segments0_flightNumber\",\n    \"legs1_segments1_aircraft_code\", \"legs1_segments1_arrivalTo_airport_city_iata\",\n    \"legs1_segments1_arrivalTo_airport_iata\", \"legs1_segments1_departureFrom_airport_iata\",\n    \"legs1_segments1_marketingCarrier_code\", \"legs1_segments1_operatingCarrier_code\",\n    \"legs1_segments1_flightNumber\",\n]\nCARRIER_COLS = [\n    \"legs0_segments0_marketingCarrier_code\",\n    \"legs1_segments0_marketingCarrier_code\",\n]\nEXCLUDE_COLS = set([\n    \"Id\", \"ranker_id\", \"selected\", \"profileId\", \"requestDate\",\n    \"legs0_departureAt\", \"legs0_arrivalAt\", \"legs1_departureAt\", \"legs1_arrivalAt\",\n    \"miniRules0_percentage\", \"miniRules1_percentage\",\n    \"pricingInfo_passengerCount\",\n])","metadata":{"execution":{"iopub.execute_input":"2025-08-11T17:41:49.326498Z","iopub.status.busy":"2025-08-11T17:41:49.326288Z","iopub.status.idle":"2025-08-11T17:41:49.342737Z","shell.execute_reply":"2025-08-11T17:41:49.337590Z"},"papermill":{"duration":0.029657,"end_time":"2025-08-11T17:41:49.345145","exception":false,"start_time":"2025-08-11T17:41:49.315488","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"ddef6b2c","cell_type":"code","source":"# -----------------------------\n# IO\n# -----------------------------\ndef read_data() -> Tuple[pl.DataFrame, pl.DataFrame]:\n    train = pl.read_parquet(TRAIN_PATH)\n    test  = pl.read_parquet(TEST_PATH)\n    for c in [\"Id\", \"ranker_id\"]:\n        if c not in train.columns or c not in test.columns:\n            raise ValueError(f\"Missing required column `{c}` in train or test.\")\n    if \"selected\" not in train.columns:\n        raise ValueError(\"Train must contain `selected`.\")\n    return train, test\n\n# -----------------------------\n# datetime handling for requestDate\n# -----------------------------\ndef _ensure_datetime_column(df: pl.DataFrame, col: str, out_col: str) -> pl.DataFrame:\n    if col not in df.columns:\n        return df.with_columns(pl.lit(None).alias(out_col))\n    dtype = df.schema[col]\n    dtype_str = str(dtype)\n    if dtype_str.startswith(\"Utf8\"):\n        return df.with_columns(pl.col(col).str.to_datetime(strict=False).alias(out_col))\n    elif \"Datetime\" in dtype_str:\n        return df.with_columns(pl.col(col).cast(pl.Datetime).alias(out_col))\n    elif \"Date\" in dtype_str:\n        return df.with_columns(pl.col(col).cast(pl.Datetime).alias(out_col))\n    else:\n        # last-resort cast\n        return df.with_columns(pl.col(col).cast(pl.Datetime, strict=False).alias(out_col))\n\n\n# -----------------------------\n# Feature engineering (label-free)\n# -----------------------------\ndef dur_to_min(expr: pl.Expr) -> pl.Expr:\n    \"\"\"\n    Robustly convert duration-like fields to minutes.\n    - If the column is already numeric => return as-is.\n    - Else parse strings like \"D.HH:MM:SS\" or \"HH:MM:SS\".\n    \"\"\"\n    # If numeric, keep it (cast ensures we don't crash on non-numeric)\n    num = expr.cast(pl.Float64, strict=False)\n\n    # Parse string forms\n    s = expr.cast(pl.Utf8, strict=False)\n    # Remove leading \"D.\" (days prefix) when present, but also keep 'days' separately\n    t = pl.when(s.str.contains(r\"^\\d+\\.\", literal=True)).then(s.str.replace(r\"^\\d+\\.\", \"\")).otherwise(s)\n    days    = s.str.extract(r\"^(\\d+)\\.\", 1).cast(pl.Int64).fill_null(0) * 1440\n    hours   = t.str.extract(r\"^(\\d+):\", 1).cast(pl.Int64).fill_null(0) * 60\n    minutes = t.str.extract(r\":(\\d+):\", 1).cast(pl.Int64).fill_null(0)\n\n    parsed = (days + hours + minutes).cast(pl.Float64)\n\n    return pl.when(num.is_not_null()).then(num).otherwise(parsed).fill_null(0)\n\n\ndef _ensure_datetime_column(df: pl.DataFrame, col: str, out_col: str) -> pl.DataFrame:\n    if col not in df.columns:\n        return df.with_columns(pl.lit(None).alias(out_col))\n    # robustly cast string/Date/Datetime -> Datetime without schema errors\n    return df.with_columns(pl.col(col).cast(pl.Datetime, strict=False).alias(out_col))\n\n\ndef session_time_split(train: pl.DataFrame, holdout_frac=0.05, n_folds=5, seed=42):\n    if \"ranker_id\" not in train.columns:\n        raise ValueError(\"ranker_id is required.\")\n    if \"requestDate\" in train.columns:\n        df = _ensure_datetime_column(train.select([\"ranker_id\", \"requestDate\"]), \"requestDate\", \"requestDate_dt\")\n        sess_time = (\n            df.group_by(\"ranker_id\")\n              .agg(pl.col(\"requestDate_dt\").min().alias(\"session_time\"))\n              .sort(\"session_time\")\n        )\n    else:\n        sess_time = (\n            train.select(\"ranker_id\")\n                 .unique()\n                 .with_columns(pl.lit(None).alias(\"session_time\"))\n                 .sort(\"ranker_id\")\n        )\n\n    n_sessions = sess_time.height\n    n_holdout = max(1, int(math.ceil(n_sessions * holdout_frac)))\n\n    holdout_ids = set(sess_time.tail(n_holdout)[\"ranker_id\"].to_list())\n    train_ids   = set(sess_time.head(n_sessions - n_holdout)[\"ranker_id\"].to_list())\n\n    # GroupKFold over remaining 95% sessions (deterministic shuffle)\n    sessions = list(train_ids)\n    rng = np.random.default_rng(seed)\n    rng.shuffle(sessions)\n\n    X_dummy = np.zeros(len(sessions))\n    y_dummy = np.zeros(len(sessions))\n    groups  = np.array(sessions)\n\n    skf = GroupKFold(n_splits=n_folds)\n    fold_ids: List[set] = []\n    for _, val_idx in skf.split(X_dummy, y_dummy, groups=groups):\n        fold_ids.append(set(np.array(sessions)[val_idx].tolist()))\n    return train_ids, holdout_ids, fold_ids\n\n\n\ndef build_features(df: pl.DataFrame) -> pl.DataFrame:\n    out = df.clone()\n\n    # --- Durations → minutes (robust to string/numeric) ---\n    dur_cols = [\"legs0_duration\", \"legs1_duration\"] + \\\n               [f\"legs{l}_segments{s}_duration\" for l in (0, 1) for s in (0, 1)]\n    dur_exprs = [dur_to_min(pl.col(c)).alias(c) for c in dur_cols if c in out.columns]\n    if dur_exprs:\n        out = out.with_columns(dur_exprs)\n\n    # --- Simple label-free features ---\n    mc_cols = [f\"legs{l}_segments{s}_marketingCarrier_code\" for l in (0,1) for s in range(4)]\n    mc_exists = [c for c in mc_cols if c in out.columns]\n\n    out = out.with_columns([\n        (pl.col(\"taxes\") / (pl.col(\"totalPrice\") + 1)).alias(\"tax_rate\"),\n        pl.col(\"totalPrice\").log1p().alias(\"log_price\"),\n        (pl.col(\"legs0_duration\").fill_null(0) + pl.col(\"legs1_duration\").fill_null(0)).alias(\"total_duration\"),\n        pl.when(pl.col(\"legs1_duration\").fill_null(0) > 0)\n          .then(pl.col(\"legs0_duration\") / (pl.col(\"legs1_duration\") + 1))\n          .otherwise(1.0).alias(\"duration_ratio\"),\n        (pl.sum_horizontal(pl.col(c).is_not_null().cast(pl.UInt8) for c in mc_exists) if mc_exists else pl.lit(0)).alias(\"n_marketing_codes_present\"),\n        (pl.col(\"frequentFlyer\").fill_null(\"\").str.count_matches(\"/\") + (pl.col(\"frequentFlyer\").fill_null(\"\") != \"\").cast(pl.Int32)).alias(\"n_ff_programs\"),\n        pl.col(\"corporateTariffCode\").is_not_null().cast(pl.Int32).alias(\"has_corporate_tariff\"),\n        (pl.col(\"pricingInfo_isAccessTP\") == 1).cast(pl.Int32).alias(\"has_access_tp\"),\n    ])\n\n    # --- Segment counts (per leg) ---\n    seg_exprs = []\n    for leg in (0, 1):\n        seg_cols = [f\"legs{leg}_segments{s}_duration\" for s in range(4) if f\"legs{leg}_segments{s}_duration\" in out.columns]\n        if seg_cols:\n            seg_exprs.append(pl.sum_horizontal(pl.col(c).is_not_null() for c in seg_cols).cast(pl.Int32).alias(f\"n_segments_leg{leg}\"))\n        else:\n            seg_exprs.append(pl.lit(0).cast(pl.Int32).alias(f\"n_segments_leg{leg}\"))\n    out = out.with_columns(seg_exprs).with_columns([\n        (pl.col(\"n_segments_leg0\") + pl.col(\"n_segments_leg1\")).alias(\"total_segments\"),\n        (pl.col(\"n_segments_leg0\") == 1).cast(pl.Int32).alias(\"is_direct_leg0\"),\n    ])\n\n    # --- One-way flag (needs to exist before is_direct_leg1) ---\n    out = out.with_columns([\n        pl.when(\n            pl.col(\"legs1_duration\").is_null() |\n            (pl.col(\"legs1_duration\") == 0) |\n            pl.col(\"legs1_segments0_departureFrom_airport_iata\").is_null()\n        ).then(1).otherwise(0).cast(pl.Int32).alias(\"is_one_way\"),\n    ])\n\n    # --- Direct leg1 depends on is_one_way (create in separate step) ---\n    out = out.with_columns([\n        pl.when(pl.col(\"is_one_way\") == 1)\n          .then(0)\n          .otherwise((pl.col(\"n_segments_leg1\") == 1).cast(pl.Int32))\n          .alias(\"is_direct_leg1\"),\n    ])\n\n    # --- Use direct flags; create group_size first, then group_size_log in a separate step ---\n    out = out.with_columns([\n        (pl.col(\"is_direct_leg0\") & pl.col(\"is_direct_leg1\")).cast(pl.Int32).alias(\"both_direct\"),\n        ((pl.col(\"isVip\") == 1) | (pl.col(\"n_ff_programs\") > 0)).cast(pl.Int32).alias(\"is_vip_freq\"),\n        pl.col(\"Id\").count().over(\"ranker_id\").alias(\"group_size\"),\n    ])\n    \n    # IMPORTANT: compute group_size_log in its own call so the column exists\n    out = out.with_columns([\n        pl.col(\"group_size\").log1p().alias(\"group_size_log\"),\n    ])\n\n    # --- Price/duration ranks within session (label-free) ---\n    out = out.with_columns([\n        pl.col(\"totalPrice\").rank().over(\"ranker_id\").alias(\"price_rank\"),\n        (pl.col(\"totalPrice\").rank(\"average\").over(\"ranker_id\") / pl.col(\"totalPrice\").count().over(\"ranker_id\")).alias(\"price_pct_rank\"),\n        (pl.col(\"totalPrice\") == pl.col(\"totalPrice\").min().over(\"ranker_id\")).cast(pl.Int32).alias(\"is_cheapest\"),\n        ((pl.col(\"totalPrice\") - pl.col(\"totalPrice\").median().over(\"ranker_id\")) / (pl.col(\"totalPrice\").std().over(\"ranker_id\") + 1)).alias(\"price_from_median\"),\n        pl.col(\"total_duration\").rank().over(\"ranker_id\").alias(\"duration_rank\"),\n    ])\n\n    # --- Time-of-day/week if timestamps exist (robust cast) ---\n    time_feats = []\n    for col in (\"legs0_departureAt\",\"legs0_arrivalAt\",\"legs1_departureAt\",\"legs1_arrivalAt\"):\n        if col in out.columns:\n            dt = pl.col(col).cast(pl.Datetime, strict=False)\n            h  = dt.dt.hour().fill_null(12)\n            time_feats += [\n                h.alias(f\"{col}_hour\"),\n                dt.dt.weekday().fill_null(0).alias(f\"{col}_weekday\"),\n                (((h >= 6) & (h <= 9)) | ((h >= 17) & (h <= 20))).cast(pl.Int32).alias(f\"{col}_business_time\"),\n            ]\n    if time_feats:\n        out = out.with_columns(time_feats)\n\n    # --- Fill nulls (numeric→0, strings→\"missing\") ---\n    num_cols = out.select(pl.selectors.numeric()).columns\n    str_cols = out.select(pl.selectors.string()).columns\n    out = out.with_columns([pl.col(c).fill_null(0) for c in num_cols] + [pl.col(c).fill_null(\"missing\") for c in str_cols])\n\n    return out\n\n# -----------------------------\n# Label encoders fit on 95% only (unsupervised) → map everywhere\n# -----------------------------\ndef fit_label_maps(train_df: pl.DataFrame, cat_cols: List[str]) -> Dict[str, pl.DataFrame]:\n    maps = {}\n    for c in cat_cols:\n        if c not in train_df.columns:\n            continue\n        mapping = (train_df.select(c).unique().drop_nulls().with_row_index(name=f\"{c}__code\").rename({c: f\"{c}\"}))\n        maps[c] = mapping\n    return maps\n\ndef apply_label_maps(df: pl.DataFrame, maps: Dict[str, pl.DataFrame]) -> pl.DataFrame:\n    out = df\n    for c, mapping in maps.items():\n        if c not in out.columns:\n            continue\n        out = (out.join(mapping, on=c, how=\"left\")\n                  .with_columns(pl.col(f\"{c}__code\").fill_null(-1).cast(pl.Int32).alias(f\"{c}__le\"))\n                  .drop(f\"{c}__code\"))\n    return out\n\n# -----------------------------\n# Target encoding: fit mappings on TRAIN ONLY (per-fold), apply to val/test\n# -----------------------------\ndef fit_te_mappings(df_train: pl.DataFrame, cols: List[str], target_col: str, m: float) -> Tuple[Dict[str, pl.DataFrame], float]:\n    prior = df_train.select(pl.col(target_col).mean()).item()\n    maps = {}\n    for c in cols:\n        if c not in df_train.columns:\n            continue\n        mp = (\n            df_train.group_by(c)\n                    .agg([pl.len().alias(\"cnt\"), pl.col(target_col).sum().alias(\"sum_y\")])\n                    .with_columns(((pl.col(\"sum_y\") + m * prior) / (pl.col(\"cnt\") + m)).alias(f\"{c}_te\"))\n                    .select([c, f\"{c}_te\"])\n        )\n        maps[c] = mp\n    return maps, prior\n\ndef apply_te_mappings(df: pl.DataFrame, maps: Dict[str, pl.DataFrame], prior: float) -> pl.DataFrame:\n    out = df\n    for c, mp in maps.items():\n        if c not in out.columns:\n            continue\n        te_col = f\"{c}_te\"\n        out = out.join(mp, on=c, how=\"left\").with_columns(pl.col(te_col).fill_null(prior).alias(te_col))\n    return out\n\n# -----------------------------\n# Build model matrices (grouped)\n# -----------------------------\ndef select_feature_cols(df: pl.DataFrame) -> List[str]:\n    numeric = [c for c in df.select(pl.selectors.numeric()).columns if c not in EXCLUDE_COLS]\n    le_cols = [c for c in df.columns if c.endswith(\"__le\")]\n    te_cols = [c for c in df.columns if c.endswith(\"_te\")]\n    feat = sorted(set(numeric + le_cols + te_cols) - EXCLUDE_COLS)\n    return feat\n\ndef to_dmatrix(df: pl.DataFrame, feat_cols: List[str]) -> Tuple[xgb.DMatrix, np.ndarray, np.ndarray]:\n    ordered = df.sort([\"ranker_id\"])\n    group_sizes = ordered.group_by(\"ranker_id\", maintain_order=True).agg(pl.len())[\"len\"].to_numpy()\n    X = ordered.select(feat_cols).to_numpy()\n    y = ordered[\"selected\"].to_numpy() if \"selected\" in ordered.columns else None\n    dmat = xgb.DMatrix(X, label=y, group=group_sizes, feature_names=feat_cols)\n    return dmat, group_sizes, ordered[\"Id\"].to_numpy()\n\ndef hitrate_at_3(y_true: np.ndarray, y_pred: np.ndarray, group_sizes: np.ndarray) -> float:\n    start = 0\n    hits = 0\n    considered = 0\n    for g in group_sizes:\n        end = start + g\n        if g > 10:\n            considered += 1\n            order = np.argsort(-y_pred[start:end])\n            top_true = y_true[start:end][order][:3]\n            hits += int(top_true.max() == 1)\n        start = end\n    return hits / considered if considered > 0 else 0.0\n\n    \n","metadata":{"execution":{"iopub.execute_input":"2025-08-11T17:41:49.364131Z","iopub.status.busy":"2025-08-11T17:41:49.363922Z","iopub.status.idle":"2025-08-11T17:41:49.413552Z","shell.execute_reply":"2025-08-11T17:41:49.408577Z"},"papermill":{"duration":0.062987,"end_time":"2025-08-11T17:41:49.415912","exception":false,"start_time":"2025-08-11T17:41:49.352925","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"027396da","cell_type":"code","source":"# 1) Load\ntrain_raw, test_raw = read_data()\n\n# 2) Split sessions FIRST (fixes your error by handling datetime robustly)\ntrain_ids, holdout_ids, fold_ids = session_time_split(\n    train_raw, holdout_frac=HOLDOUT_FRAC, n_folds=N_FOLDS, seed=RANDOM_STATE\n)\n\ntrain_95 = train_raw.filter(pl.col(\"ranker_id\").is_in(list(train_ids)))\nvalidator_ = train_raw.filter(pl.col(\"ranker_id\").is_in(list(holdout_ids)))\n","metadata":{"execution":{"iopub.execute_input":"2025-08-11T17:41:49.434654Z","iopub.status.busy":"2025-08-11T17:41:49.434447Z","iopub.status.idle":"2025-08-11T17:41:54.447632Z","shell.execute_reply":"2025-08-11T17:41:54.440147Z"},"papermill":{"duration":5.026656,"end_time":"2025-08-11T17:41:54.450327","exception":false,"start_time":"2025-08-11T17:41:49.423671","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"bde27968","cell_type":"code","source":"train_95.head()","metadata":{"execution":{"iopub.execute_input":"2025-08-11T17:41:54.470220Z","iopub.status.busy":"2025-08-11T17:41:54.469971Z","iopub.status.idle":"2025-08-11T17:41:54.504117Z","shell.execute_reply":"2025-08-11T17:41:54.499877Z"},"papermill":{"duration":0.047179,"end_time":"2025-08-11T17:41:54.506134","exception":false,"start_time":"2025-08-11T17:41:54.458955","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"991820ee","cell_type":"code","source":"\n# 3) Build base features (label-free) for 95%, validator_, and test\ntrain_95_f = build_features(train_95)\nvalidator_f = build_features(validator_)\ntest_f = build_features(test_raw.with_columns(pl.lit(0, dtype=pl.Int8).alias(\"selected\")))  # dummy target\n","metadata":{"execution":{"iopub.execute_input":"2025-08-11T17:41:54.525969Z","iopub.status.busy":"2025-08-11T17:41:54.525707Z","iopub.status.idle":"2025-08-11T17:42:49.887631Z","shell.execute_reply":"2025-08-11T17:42:49.882698Z"},"papermill":{"duration":55.37744,"end_time":"2025-08-11T17:42:49.891699","exception":false,"start_time":"2025-08-11T17:41:54.514259","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"974e9637","cell_type":"code","source":"# 4) Fit label encoders on 95% only, apply to all\ncat_cols = [c for c in BASIC_CAT_COLS if c in train_95_f.columns]\nle_maps = fit_label_maps(train_95_f, cat_cols)\ntrain_95_f = apply_label_maps(train_95_f, le_maps)\nvalidator_f = apply_label_maps(validator_f, le_maps)\ntest_f = apply_label_maps(test_f, le_maps)\n","metadata":{"execution":{"iopub.execute_input":"2025-08-11T17:42:49.912433Z","iopub.status.busy":"2025-08-11T17:42:49.912171Z","iopub.status.idle":"2025-08-11T17:43:07.059744Z","shell.execute_reply":"2025-08-11T17:43:07.055130Z"},"papermill":{"duration":17.162259,"end_time":"2025-08-11T17:43:07.062611","exception":false,"start_time":"2025-08-11T17:42:49.900352","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"7a8ab3a4","cell_type":"markdown","source":"## 95% Data Train","metadata":{"papermill":{"duration":0.008071,"end_time":"2025-08-11T17:43:07.079227","exception":false,"start_time":"2025-08-11T17:43:07.071156","status":"completed"},"tags":[]}},{"id":"32b5f1c5","cell_type":"code","source":"\n\nNUM_BOOST_ROUND = 2_000\nEARLY_STOPPING_ROUNDS = 200\nVERBOSE_EVAL = 50\n\n\nIdentifier = 'SeriesA-R8_Shallow'\n\nXGB_PARAMS = {\n    'objective': 'rank:pairwise',\n    'eval_metric': 'ndcg@3',\n    \"learning_rate\": 0.0215,\n    \"max_depth\": 12,\n    \"min_child_weight\": 8,\n    \"subsample\": 0.92,\n    \"colsample_bytree\": 0.42,\n    \"gamma\": 3.3084297630544888,\n    \"lambda\": 5.952586917313028,\n    \"alpha\": 0.6395254133055179,\n    \"seed\": RANDOM_STATE,\n    \"n_jobs\": -1,\n    # \"tree_method\": \"gpu_hist\",  # enable if GPU is available\n}\n","metadata":{"execution":{"iopub.execute_input":"2025-08-11T17:43:07.098229Z","iopub.status.busy":"2025-08-11T17:43:07.097981Z","iopub.status.idle":"2025-08-11T17:43:07.107812Z","shell.execute_reply":"2025-08-11T17:43:07.103822Z"},"papermill":{"duration":0.022997,"end_time":"2025-08-11T17:43:07.110123","exception":false,"start_time":"2025-08-11T17:43:07.087126","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"8351c3c5","cell_type":"code","source":"# 6) FULL OOF on 95%: train K models, collect OOF preds and test preds\n# Pre-allocate OOF container\ntrain_95_f = train_95_f.with_row_index(name=\"__row__\")\noof_pred = np.zeros(train_95_f.height, dtype=np.float32)\nbest_iters = []\ntest_fold_preds = []\n\nfor k, val_ids in enumerate(fold_ids):\n    if k < 3:\n        print(f\"\\n[OOF] Fold {k}/{N_FOLDS-1}\")\n        tr_ids_k = set(train_ids) - val_ids\n        tr_k = train_95_f.filter(pl.col(\"ranker_id\").is_in(list(tr_ids_k))).drop(\"__row__\")\n        va_k = train_95_f.filter(pl.col(\"ranker_id\").is_in(list(val_ids))).drop(\"__row__\")\n    \n        te_cols = [c for c in CARRIER_COLS if c in tr_k.columns]\n    \n    \n        # Per-fold TE\n        te_maps_k, te_prior_k = fit_te_mappings(tr_k.select([\"selected\"] + te_cols), te_cols, \"selected\", TE_PRIOR_M)\n        tr_k = apply_te_mappings(tr_k, te_maps_k, te_prior_k)\n        va_k = apply_te_mappings(va_k, te_maps_k, te_prior_k)\n    \n        feat_cols_k = select_feature_cols(tr_k)\n        dtr_k, gtr_k, _ = to_dmatrix(tr_k, feat_cols_k)\n        dva_k, gva_k, ids_va_order = to_dmatrix(va_k, feat_cols_k)\n    \n        mdl_k = xgb.train(\n            XGB_PARAMS, dtr_k,\n            num_boost_round=NUM_BOOST_ROUND,\n            evals=[(dtr_k, \"train\"), (dva_k, \"val\")],\n            early_stopping_rounds=EARLY_STOPPING_ROUNDS,\n            verbose_eval=VERBOSE_EVAL\n        )\n        best_iters.append(mdl_k.best_iteration)\n    \n        # OOF preds on validation fold\n        va_pred_k = mdl_k.predict(dva_k, iteration_range=(0, mdl_k.best_iteration + 1))\n    \n        # Map back to original rows by Id (keeps safety)\n        fold_oof = pl.DataFrame({\"Id\": ids_va_order, \"oof_score\": va_pred_k})\n        # Align to train_95_f rows\n        idx_map = train_95_f.select([\"__row__\", \"Id\"]).join(fold_oof, on=\"Id\", how=\"left\")\n        mask = idx_map[\"oof_score\"].to_numpy()\n        valid_mask = ~np.isnan(mask)\n        oof_pred[idx_map[\"__row__\"].to_numpy()[valid_mask]] = mask[valid_mask].astype(np.float32)\n    \n        # Test preds for this fold (using fold's TE mapping)\n        test_k = apply_te_mappings(test_f, te_maps_k, te_prior_k)\n        dtest_k, _, _ = to_dmatrix(test_k, feat_cols_k)\n        test_fold_preds.append(mdl_k.predict(dtest_k, iteration_range=(0, mdl_k.best_iteration + 1)))\n    \n        # Clean\n        del dtr_k, dva_k, dtest_k\n        # del  mdl_k\n        gc.collect()\n\n\nmean_best_iter = int(np.mean(best_iters))\nprint(f\"\\n[OOF] Done. Mean best_iteration across folds: {mean_best_iter}\")","metadata":{"execution":{"iopub.execute_input":"2025-08-11T17:43:07.129016Z","iopub.status.busy":"2025-08-11T17:43:07.128784Z","iopub.status.idle":"2025-08-11T19:26:52.470453Z","shell.execute_reply":"2025-08-11T19:26:52.466348Z"},"papermill":{"duration":6225.372864,"end_time":"2025-08-11T19:26:52.490860","exception":false,"start_time":"2025-08-11T17:43:07.117996","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"8f7725b5","cell_type":"code","source":"# Save OOF for 95%\n\n#oof_df = train_95_f.select([\"Id\",\"ranker_id\"]).with_columns(pl.Series(\"xgb_ranker_oof\", oof_pred))\n#oof_path = f\"oof_95_xgb_ranker_{Identifier}.csv\"\n#oof_df.write_csv(oof_path)\n#print(f\"Saved OOF (95%) to {oof_path}\")\n","metadata":{"execution":{"iopub.execute_input":"2025-08-11T19:26:52.535986Z","iopub.status.busy":"2025-08-11T19:26:52.535728Z","iopub.status.idle":"2025-08-11T19:26:52.544267Z","shell.execute_reply":"2025-08-11T19:26:52.539997Z"},"papermill":{"duration":0.034446,"end_time":"2025-08-11T19:26:52.546364","exception":false,"start_time":"2025-08-11T19:26:52.511918","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"c5993e25","cell_type":"markdown","source":"# We Usually train on full data & validate on holdout but not imporant here\n\n- Replace model_full with the last trained model","metadata":{"papermill":{"duration":0.020181,"end_time":"2025-08-11T19:26:52.586901","exception":false,"start_time":"2025-08-11T19:26:52.566720","status":"completed"},"tags":[]}},{"id":"6001ccfc","cell_type":"code","source":"RUN_holdout_evaluation = False","metadata":{"execution":{"iopub.execute_input":"2025-08-11T19:26:52.630438Z","iopub.status.busy":"2025-08-11T19:26:52.630231Z","iopub.status.idle":"2025-08-11T19:26:52.638820Z","shell.execute_reply":"2025-08-11T19:26:52.634585Z"},"papermill":{"duration":0.03428,"end_time":"2025-08-11T19:26:52.641366","exception":false,"start_time":"2025-08-11T19:26:52.607086","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"2219e444","cell_type":"code","source":"if RUN_holdout_evaluation:\n    model_full = mdl_k.copy()\n    \n    te_maps_full, te_prior_full = fit_te_mappings(train_95_f.drop(\"__row__\").select([\"selected\"] + te_cols), te_cols, \"selected\", TE_PRIOR_M)\n    tr_full = apply_te_mappings(train_95_f.drop(\"__row__\"), te_maps_full, te_prior_full)\n    val5 = apply_te_mappings(validator_f, te_maps_full, te_prior_full)\n    \n    feat_cols_full = select_feature_cols(tr_full)\n    dtr_full, gtr_full, _ = to_dmatrix(tr_full, feat_cols_full)\n    dval5,  gval5, id_val5_order = to_dmatrix(val5, feat_cols_full)","metadata":{"execution":{"iopub.execute_input":"2025-08-11T19:26:52.685902Z","iopub.status.busy":"2025-08-11T19:26:52.685684Z","iopub.status.idle":"2025-08-11T19:26:52.696688Z","shell.execute_reply":"2025-08-11T19:26:52.691232Z"},"papermill":{"duration":0.036841,"end_time":"2025-08-11T19:26:52.698807","exception":false,"start_time":"2025-08-11T19:26:52.661966","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"76c66b37","cell_type":"code","source":"if RUN_holdout_evaluation:\n    \n    val5_pred = model_full.predict(dval5, iteration_range=(0, mean_best_iter))\n    \n    #val5_out = pl.DataFrame({\"Id\": id_val5_order, \"ranker_id\": val5.sort([\"ranker_id\"])[\"ranker_id\"], \"xgb_ranker_pred\": val5_pred})\n    #val5_path = f\"pred_5pct_from_95_xgb_ranker_{Identifier}.csv\"\n    #val5_out.write_csv(val5_path)\n    #print(f\"Saved 5% validator_ preds to {val5_path}\")\n    \n    \n    y_val5 = val5.sort([\"ranker_id\"])[\"selected\"].to_numpy()\n    hr3_holdout = hitrate_at_3(y_val5, val5_pred, gval5)\n    print(f\"[HOLDOUT 5%] HitRate@3: {hr3_holdout:.4f} (groups>10 only)\")\n","metadata":{"execution":{"iopub.execute_input":"2025-08-11T19:26:52.743052Z","iopub.status.busy":"2025-08-11T19:26:52.742821Z","iopub.status.idle":"2025-08-11T19:26:52.753297Z","shell.execute_reply":"2025-08-11T19:26:52.748382Z"},"papermill":{"duration":0.037026,"end_time":"2025-08-11T19:26:52.756136","exception":false,"start_time":"2025-08-11T19:26:52.719110","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"4f1ae8f9","cell_type":"code","source":"# Submission\ntest_avg = np.mean(np.vstack(test_fold_preds), axis=0).astype(np.float32)\n\ntest_sorted = (\n    test_f.sort([\"ranker_id\"])\n          .select([\"Id\", \"ranker_id\"])\n          .with_columns(pl.Series(\"selected\", test_avg))\n)\n\nsubmission_sorted = (\n    test_sorted\n    .with_columns(\n        pl.col(\"selected\")\n          .rank(method=\"ordinal\", descending=True)\n          .over(\"ranker_id\")\n          .cast(pl.Int32)\n          .alias(\"selected\")\n    )\n    .select([\"Id\", \"ranker_id\", \"selected\"])\n)\n\ntest_order = test_f.select(\"Id\").with_row_index(\"ord\")\nsubmission = (\n    submission_sorted.join(test_order, on=\"Id\", how=\"inner\")\n                     .sort(\"ord\")\n                     .select([\"Id\", \"ranker_id\", \"selected\"])\n)\n\nscores = (\n    test_sorted.join(test_order, on=\"Id\", how=\"inner\")\n               .sort(\"ord\")\n               .select([\"Id\", \"ranker_id\", \"selected\"])\n)\n\n# (f) write files\nscores_path = f\"pred_test_from_95cv_xgb_ranker_{Identifier}.csv\"\nsub_path    = f\"submission.csv\"\nscores.write_csv(scores_path)\nsubmission.write_csv(sub_path)\nprint(f\"Saved test scores to {scores_path}\")\nprint(f\"Saved submission to {sub_path}\")\n","metadata":{"execution":{"iopub.execute_input":"2025-08-11T19:26:52.800309Z","iopub.status.busy":"2025-08-11T19:26:52.800105Z","iopub.status.idle":"2025-08-11T19:26:55.698403Z","shell.execute_reply":"2025-08-11T19:26:55.692538Z"},"papermill":{"duration":2.924579,"end_time":"2025-08-11T19:26:55.701213","exception":false,"start_time":"2025-08-11T19:26:52.776634","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"12bb5b65","cell_type":"code","source":"","metadata":{"papermill":{"duration":0.021147,"end_time":"2025-08-11T19:26:55.745504","exception":false,"start_time":"2025-08-11T19:26:55.724357","status":"completed"},"tags":[]},"outputs":[],"execution_count":null}]}