{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":105399,"databundleVersionId":12733338,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"0c16d8c9","cell_type":"markdown","source":"# Requiments","metadata":{}},{"id":"6f14a3d9","cell_type":"code","source":"!pip -q install xgboost==2.1.1","metadata":{},"outputs":[],"execution_count":null},{"id":"d58c746a","cell_type":"code","source":"import gc\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport pyarrow.parquet as pq\nimport xgboost as xgb\nimport matplotlib.pyplot as plt\nfrom pathlib import Path\n\nwarnings.filterwarnings(\"ignore\")","metadata":{},"outputs":[],"execution_count":null},{"id":"4c1bb750","cell_type":"markdown","source":"# Knobs","metadata":{}},{"id":"e30cdd18","cell_type":"code","source":"RANDOM_STATE = 42\nnp.random.seed(RANDOM_STATE)\nINPUT_PATH = Path(\"/kaggle/input/aeroclub-recsys-2025\")\nETA = 0.06\nEARLY_STOP_ROUNDS = 120\nTEST_BATCH_ROWS = 1_000_000 \nTE_USE_FULL_TRAIN = False     # Полный TE (требует больше RAM)\nTRAIN_GROUP_FRACTION = 0.55   # 0.7–0.8, если GPU/RAM позволяют\nMAX_DEPTH = 8\nMIN_CHILD_WEIGHT = 25\nMAX_BIN = 96                  # синхронизировать во всех QuantileMatrix\nNUM_BOOST_ROUND = 1200        # 600k/300k при OOM\nUSE_POSTPROCESS = True\nPOSTPROCESS_EPS = 0.02","metadata":{},"outputs":[],"execution_count":null},{"id":"3cee79d0","cell_type":"markdown","source":"# Utils","metadata":{}},{"id":"aa9a0c94","cell_type":"code","source":"def parquet_columns(path):\n    try:\n        return set(pq.ParquetFile(str(path)).schema.names)\n    except Exception:\n        return set()\n\ndef dur_to_minutes(s: pd.Series) -> pd.Series:\n    s = s.astype(\"string\")\n    m = s.str.extract(r\"^(?:(\\d+)\\.)?(\\d+):(\\d+):\\d+$\")\n    days  = pd.to_numeric(m[0], errors=\"coerce\").fillna(0).astype(np.int64)\n    hours = pd.to_numeric(m[1], errors=\"coerce\").fillna(0).astype(np.int64)\n    mins  = pd.to_numeric(m[2], errors=\"coerce\").fillna(0).astype(np.int64)\n\n    return (days * 1440 + hours * 60 + mins).astype(np.int32)\n\ndef hash_encode(series, mod=2_147_483_647):\n    return (pd.util.hash_pandas_object(series.astype(\"string\"), index=False).values.astype(\"uint64\") % mod).astype(\"int32\")\n\ndef hitrate_at_3(df_pred):\n    cnt = df_pred.groupby(\"ranker_id\")[\"pred\"].transform(\"count\")\n    df2 = df_pred[cnt > 10].copy()\n\n    if df2.empty: \n        return np.nan\n    \n    df2[\"rk\"] = df2.groupby(\"ranker_id\")[\"pred\"].rank(method=\"first\", ascending=False)\n\n    return df2.groupby(\"ranker_id\").apply(lambda g: int((g.sort_values(\"rk\").head(3)[\"selected\"].max()) == 1)).mean()\n\ndef kfold_oof_te(df_train, df_test, col, target=\"selected\", group_col=\"ranker_id\", nfold=5, m_smooth=50.0, add_count=True):\n    gids = df_train[group_col].drop_duplicates()\n    folds = np.array_split(gids.sample(frac=1.0, random_state=RANDOM_STATE), nfold)\n    global_mean = df_train[target].mean()\n    tr_te = pd.Series(np.nan, index=df_train.index, dtype=\"float32\")\n    tr_ct = pd.Series(np.nan, index=df_train.index, dtype=\"float32\")\n\n    for i in range(nfold):\n        valid_groups = set(folds[i].values)\n        is_valid = df_train[group_col].isin(valid_groups)\n        tr_idx = df_train.index[~is_valid]\n        va_idx = df_train.index[is_valid]\n        stats = (df_train.loc[tr_idx, [col, target]].groupby(col)[target].agg([\"sum\", \"count\"]))\n        stats[\"mean\"] = stats[\"sum\"] / stats[\"count\"].clip(lower=1)\n        stats[\"post\"] = (stats[\"mean\"] * stats[\"count\"] + global_mean * m_smooth) / (stats[\"count\"]+m_smooth)\n        mp_post = stats[\"post\"].astype(\"float32\").to_dict()\n        mp_cnt  = stats[\"count\"].astype(\"float32\").to_dict()\n        tr_te.loc[va_idx] = df_train.loc[va_idx, col].map(mp_post).fillna(global_mean).astype(\"float32\")\n        tr_ct.loc[va_idx] = df_train.loc[va_idx, col].map(mp_cnt ).fillna(0).astype(\"float32\")\n        del stats, mp_post, mp_cnt\n        gc.collect()\n\n    full_stats = df_train.groupby(col)[target].agg([\"sum\", \"count\"])\n    full_stats[\"mean\"] = full_stats[\"sum\"] / full_stats[\"count\"].clip(lower=1)\n    full_stats[\"post\"] = (full_stats[\"mean\"] * full_stats[\"count\"] + global_mean * m_smooth) / (full_stats[\"count\"] + m_smooth)\n    mp_post_full = full_stats[\"post\"].astype(\"float32\").to_dict()\n    mp_cnt_full  = full_stats[\"count\"].astype(\"float32\").to_dict()\n    te_te = df_test[col].map(mp_post_full).fillna(global_mean).astype(\"float32\")\n    te_ct = df_test[col].map(mp_cnt_full ).fillna(0).astype(\"float32\")\n    \n    return (tr_te.values, te_te.values, tr_ct.values, te_ct.values) if add_count else (tr_te.values, te_te.values, None, None)","metadata":{},"outputs":[],"execution_count":null},{"id":"33c4336c","cell_type":"markdown","source":"# Data","metadata":{}},{"id":"51a68d1a","cell_type":"code","source":"train_pq = INPUT_PATH / \"train.parquet\"\ntest_pq  = INPUT_PATH / \"test.parquet\"\nassert train_pq and test_pq, \"Не удалось найти train/test parquet-файлы\"\n\navail = parquet_columns(train_pq) | parquet_columns(test_pq)\n\nbase_num = [\n    \"totalPrice\", \"taxes\", \"miniRules0_monetaryAmount\", \"miniRules1_monetaryAmount\", \"legs0_duration\", \"legs1_duration\",\n    \"legs0_segments0_baggageAllowance_quantity\", \"legs1_segments0_baggageAllowance_quantity\", \"pricingInfo_isAccessTP\",\n    \"legs0_segments1_duration\", \"legs1_segments1_duration\",\n]\nbase_cat = [\n    \"nationality\", \"searchRoute\", \"corporateTariffCode\", \"bySelf\", \"sex\", \"legs0_segments0_departureFrom_airport_iata\",\n    \"legs0_segments0_arrivalTo_airport_iata\", \"legs0_segments0_marketingCarrier_code\", \"legs0_segments0_operatingCarrier_code\",\n    \"legs0_segments0_aircraft_code\",\n]\ntime_cols = [\"legs0_departureAt\", \"legs0_arrivalAt\", \"requestDate\"]\nkeys_cols = [\"Id\", \"ranker_id\", \"selected\"]\n\nuse_cols_train = [c for c in (set(base_num)|set(base_cat)|set(time_cols)|set(keys_cols)) if c in avail]\nuse_cols_test  = [c for c in use_cols_train if c != \"selected\"]\n\ntrain = pd.read_parquet(train_pq, columns=use_cols_train)\ntest  = pd.read_parquet(test_pq,  columns=use_cols_test)\nif \"selected\" not in test.columns:\n    test[\"selected\"] = 0\n\n# durations -> minutes\nfor col in [\"legs0_duration\", \"legs1_duration\", \"legs0_segments1_duration\", \"legs1_segments1_duration\"]:\n    if col in train.columns:\n        train[col] = dur_to_minutes(train[col])\n        test[col]  = dur_to_minutes(test[col])","metadata":{},"outputs":[],"execution_count":null},{"id":"32f79ea2","cell_type":"markdown","source":"# Обработка признаков","metadata":{}},{"id":"cee41b41","cell_type":"markdown","source":"## Feature engineering","metadata":{}},{"id":"0e53a3b8","cell_type":"code","source":"def build_feats(df: pd.DataFrame) -> pd.DataFrame:\n    out = df.copy()\n\n    if {\"totalPrice\", \"taxes\"} <= set(out.columns):\n        out[\"price_per_tax\"] = out[\"totalPrice\"]/(out[\"taxes\"] + 1.0)\n        out[\"tax_rate\"] = out[\"taxes\"] / (out[\"totalPrice\"] + 1.0)\n        out[\"log_price\"] = np.log1p(out[\"totalPrice\"].astype(np.float64)).astype(\"float32\")\n    else:\n        out[\"price_per_tax\"] = 0.0\n        out[\"tax_rate\"] = 0.0\n        out[\"log_price\"] = 0.0\n\n    out[\"total_duration\"] = out.get(\"legs0_duration\", 0).astype(\"int64\") + out.get(\"legs1_duration\", 0).astype(\"int64\")\n    out[\"total_duration\"] = out[\"total_duration\"].astype(\"float32\")\n    out[\"has_access_tp\"] = (out.get(\"pricingInfo_isAccessTP\", 0)==1).astype(\"int8\")\n    out[\"has_baggage\"] = (\n        out.get(\"legs0_segments0_baggageAllowance_quantity\", 0).fillna(0).astype(\"float32\") +\n        out.get(\"legs1_segments0_baggageAllowance_quantity\", 0).fillna(0).astype(\"float32\")\n    > 0).astype(\"int8\")\n\n    out[\"has_fees\"] = (\n        out.get(\"miniRules0_monetaryAmount\", 0).fillna(0).astype(\"float32\") +\n        out.get(\"miniRules1_monetaryAmount\", 0).fillna(0).astype(\"float32\")\n    > 0).astype(\"int8\")\n\n    out[\"group_size\"] = out.groupby(\"ranker_id\")[\"Id\"].transform(\"count\").astype(\"int32\")\n    out[\"group_size_log\"] = np.log1p(out[\"group_size\"].astype(\"float32\")).astype(\"float32\")\n\n    for tcol in [\"legs0_departureAt\", \"legs0_arrivalAt\", \"requestDate\"]:\n        if tcol in out.columns:\n            dt = pd.to_datetime(out[tcol], errors=\"coerce\")\n            out[f\"{tcol}_hour\"] = dt.dt.hour.fillna(12).astype(\"int16\")\n            out[f\"{tcol}_weekday\"] = dt.dt.weekday.fillna(0).astype(\"int16\")\n        else:\n            out[f\"{tcol}_hour\"] = 12\n            out[f\"{tcol}_weekday\"] = 0\n    \n    dep = pd.to_datetime(out.get(\"legs0_departureAt\"), errors=\"coerce\")\n    req = pd.to_datetime(out.get(\"requestDate\"), errors=\"coerce\")\n    dtd = (dep-req).dt.total_seconds() / 3600.0\n    out[\"days_to_departure\"] = np.clip(dtd.fillna(0).astype(\"float32\") / 24.0, -30.0, 365.0).astype(\"float32\")\n    dep_hr = out[\"legs0_departureAt_hour\"].astype(\"int16\")\n    out[\"dep_morning\"] = ((dep_hr>=6)&(dep_hr<=9)).astype(\"int8\")\n    out[\"dep_evening\"] = ((dep_hr>=17)&(dep_hr<=20)).astype(\"int8\")\n    out[\"dep_redeye\"] = ((dep_hr<=5)|(dep_hr>=22)).astype(\"int8\")\n\n    if \"totalPrice\" in out.columns:\n        grp = out.groupby(\"ranker_id\")[\"totalPrice\"]\n        out[\"price_rank\"] = grp.rank(method=\"average\").astype(\"float32\")\n        cnt = grp.transform(\"count\").astype(\"float32\")\n        out[\"price_pct_rank\"] = (out[\"price_rank\"] / cnt).astype(\"float32\")\n        med = grp.transform(\"median\").astype(\"float32\")\n        std = grp.transform(\"std\").fillna(0).astype(\"float32\")\n        out[\"price_z\"] = ((out[\"totalPrice\"].astype(\"float32\") - med) / (std + 1e-3)).astype(\"float32\")\n        mn = grp.transform(\"min\").astype(\"float32\")\n        mx = grp.transform(\"max\").astype(\"float32\")\n        out[\"price_min_norm\"] = ((out[\"totalPrice\"].astype(\"float32\") - mn) / (mx - mn + 1e-3)).astype(\"float32\")\n        out[\"is_cheapest\"] = (out[\"totalPrice\"] == mn).astype(\"int8\")\n    else:\n        out[\"price_pct_rank\"] = 0.5\n        out[\"price_z\"] = 0.0\n        out[\"price_min_norm\"] = 0.5\n        out[\"is_cheapest\"] = 0\n\n    grp_d = out.groupby(\"ranker_id\")[\"total_duration\"]\n    out[\"dur_rank\"] = grp_d.rank(method=\"average\").astype(\"float32\")\n    cntd = grp_d.transform(\"count\").astype(\"float32\")\n    out[\"dur_pct_rank\"] = (out[\"dur_rank\"] / cntd).astype(\"float32\")\n    out[\"has_conn_leg0\"] = (out.get(\"legs0_segments1_duration\", pd.Series(0)).fillna(0).astype(\"int32\") > 0).astype(\"int8\")\n    out[\"has_conn_leg1\"] = (out.get(\"legs1_segments1_duration\", pd.Series(0)).fillna(0).astype(\"int32\") > 0).astype(\"int8\")\n    out[\"n_stops\"] = out[\"has_conn_leg0\"].astype(\"int16\") + out[\"has_conn_leg1\"].astype(\"int16\")\n    out[\"is_direct_l0\"] = (out[\"has_conn_leg0\"] == 0).astype(\"int8\")\n    \n    return out\n\ntrain = build_feats(train)\ntest  = build_feats(test)","metadata":{},"outputs":[],"execution_count":null},{"id":"6bff366f","cell_type":"markdown","source":"## TE (без утечек)","metadata":{}},{"id":"7eaac143","cell_type":"code","source":"te_cols = [c for c in [\n    \"searchRoute\",\n    \"legs0_segments0_marketingCarrier_code\",\n    \"legs0_segments0_departureFrom_airport_iata\",\n    \"legs0_segments0_arrivalTo_airport_iata\",\n    \"corporateTariffCode\",\n    \"nationality\",\n] if c in train.columns]\n\nif {\"searchRoute\", \"legs0_segments0_marketingCarrier_code\"} <= set(train.columns):\n    train[\"routeXcarrier\"] = (train[\"searchRoute\"].astype(\"string\") + \"__\" + train[\"legs0_segments0_marketingCarrier_code\"].astype(\"string\"))\n    test [\"routeXcarrier\"] = (test [\"searchRoute\"].astype(\"string\") + \"__\" + test[\"legs0_segments0_marketingCarrier_code\"].astype(\"string\"))\n    te_cols.append(\"routeXcarrier\")\n\n# Подвыборка групп для обучения\nrng = np.random.default_rng(RANDOM_STATE)\nuniq_groups = train[\"ranker_id\"].unique()\nsample_n = int(len(uniq_groups) * TRAIN_GROUP_FRACTION)\nsample_groups = rng.choice(uniq_groups, size=max(sample_n,1), replace=False)\nmask_sub = train[\"ranker_id\"].isin(sample_groups)\ntrain_sub = train.loc[mask_sub].copy().sort_values([\"ranker_id\", \"Id\"]).reset_index(drop=True)\n\nte_train_source = train if TE_USE_FULL_TRAIN else train_sub\n\nfor c in te_cols:\n    te_train_source[c] = te_train_source[c].astype(\"string\").fillna(\"missing\")\n    test[c] = test[c].astype(\"string\").fillna(\"missing\")\n    tr_te, te_te, tr_ct, te_ct = kfold_oof_te(te_train_source, test, col=c, target=\"selected\", group_col=\"ranker_id\", nfold=5, m_smooth=50.0, add_count=True)\n    \n    if TE_USE_FULL_TRAIN:\n        train[f\"{c}_te\"] = tr_te\n        train[f\"{c}_cnt\"] = tr_ct\n    else:\n        train_sub[f\"{c}_te\"] = tr_te\n        train_sub[f\"{c}_cnt\"] = tr_ct\n\n    test[f\"{c}_te\"] = te_te\n    test[f\"{c}_cnt\"] = te_ct\n    gc.collect()","metadata":{},"outputs":[],"execution_count":null},{"id":"a04452d4","cell_type":"markdown","source":"## Хэш-кодирование категориальных признаков","metadata":{}},{"id":"c3125ab5","cell_type":"code","source":"train_used = train.loc[mask_sub].copy() if TE_USE_FULL_TRAIN else train_sub\n\nhash_cats = [c for c in [\"bySelf\", \"sex\", \"legs0_segments0_operatingCarrier_code\", \"legs0_segments0_aircraft_code\"] if c in train_used.columns]\n\nfor c in hash_cats:\n    train_used[c] = hash_encode(train_used[c])\n    if c in test.columns:\n        test[c] = hash_encode(test[c])","metadata":{},"outputs":[],"execution_count":null},{"id":"f02c7413","cell_type":"markdown","source":"## Групповое разбиение train/validation 90/10","metadata":{}},{"id":"b9ee1aa1","cell_type":"code","source":"gids = train_used[\"ranker_id\"].drop_duplicates().sample(frac=1.0, random_state=RANDOM_STATE)\nsplit_g = int(len(gids) * 0.9)\ng_tr = set(gids.iloc[:split_g])\ng_va = set(gids.iloc[split_g:])\n\ntr_part = train_used[train_used[\"ranker_id\"].isin(g_tr)].copy().sort_values([\"ranker_id\", \"Id\"])\nva_part = train_used[train_used[\"ranker_id\"].isin(g_va)].copy().sort_values([\"ranker_id\", \"Id\"])","metadata":{},"outputs":[],"execution_count":null},{"id":"589d8565","cell_type":"markdown","source":"# Выбор признаков","metadata":{}},{"id":"49bdfcde","cell_type":"code","source":"num_feats = [c for c in [\n    \"price_per_tax\", \"tax_rate\", \"log_price\", \"total_duration\", \"has_access_tp\", \"has_baggage\", \"has_fees\", \"group_size\", \"group_size_log\",\n    \"price_pct_rank\", \"price_z\", \"price_min_norm\", \"is_cheapest\", \"dur_pct_rank\", \"legs0_departureAt_hour\", \"legs0_departureAt_weekday\",\n    \"days_to_departure\", \"dep_morning\", \"dep_evening\", \"dep_redeye\", \"has_conn_leg0\", \"has_conn_leg1\", \"n_stops\", \"is_direct_l0\",\n] + [f\"{c}_te\" for c in te_cols] + [f\"{c}_cnt\" for c in te_cols] if c in train_used.columns]\n\ncat_feats = [c for c in hash_cats if c in train_used.columns]\n\nfor c in num_feats:\n    tr_part[c] = pd.to_numeric(tr_part[c], errors=\"coerce\").fillna(0).astype(\"float32\")\n    va_part[c] = pd.to_numeric(va_part[c], errors=\"coerce\").fillna(0).astype(\"float32\")\nfor c in cat_feats:\n    tr_part[c] = pd.to_numeric(tr_part[c], errors=\"coerce\").fillna(0).astype(\"int32\")\n    va_part[c] = pd.to_numeric(va_part[c], errors=\"coerce\").fillna(0).astype(\"int32\")\n\nfeature_cols = num_feats + cat_feats\n\nX_tr = tr_part[feature_cols].astype(\"float32\").values\ny_tr = tr_part[\"selected\"].astype(\"int32\").values\ngrp_tr = tr_part.groupby(\"ranker_id\", sort=False).size().to_numpy()\n\nX_va = va_part[feature_cols].astype(\"float32\").values\ny_va = va_part[\"selected\"].astype(\"int32\").values\ngrp_va = va_part.groupby(\"ranker_id\", sort=False).size().to_numpy()\n\n# Веса групп (каждой группе — одно число)\ngrp_order_tr = tr_part.groupby(\"ranker_id\", sort=False)[\"group_size\"].first()\ngw_tr = np.where(grp_order_tr.to_numpy() > 10, 1.2, 0.8).astype(\"float32\")\ngrp_order_va = va_part.groupby(\"ranker_id\", sort=False)[\"group_size\"].first()\ngw_va = np.where(grp_order_va.to_numpy() > 10, 1.2, 0.8).astype(\"float32\")","metadata":{},"outputs":[],"execution_count":null},{"id":"8d3769cd","cell_type":"markdown","source":"# Обучение модели","metadata":{}},{"id":"fbe225cb","cell_type":"markdown","source":"## Train (GPU)","metadata":{}},{"id":"08d655b8","cell_type":"code","source":"qtrain = xgb.QuantileDMatrix(X_tr, label=y_tr, group=grp_tr, weight=gw_tr, max_bin=MAX_BIN)\nqval = xgb.QuantileDMatrix(X_va, label=y_va, group=grp_va, weight=gw_va, max_bin=MAX_BIN)\n\nparams = {\n    \"objective\": \"rank:ndcg\",\n    \"eval_metric\": \"ndcg@3\",\n    \"device\": \"cuda\",\n    \"tree_method\": \"gpu_hist\",\n    \"single_precision_histogram\": True,\n    \"max_bin\": MAX_BIN,\n    \"max_depth\": MAX_DEPTH,\n    \"min_child_weight\": MIN_CHILD_WEIGHT,\n    \"subsample\": 0.85,\n    \"colsample_bytree\": 0.75,\n    \"lambda\": 12.0,\n    \"learning_rate\": ETA,\n    \"seed\": RANDOM_STATE,\n    \"lambdarank_num_pair_per_sample\": 16,\n}\n\nprint(\"Training (GPU):\")\n\nbooster = xgb.train(\n    params, qtrain, num_boost_round=NUM_BOOST_ROUND,\n    evals=[(qtrain,\"train\"),(qval,\"val\")],\n    early_stopping_rounds=EARLY_STOP_ROUNDS, verbose_eval=100\n)\n\n# Val HR@3 (> 10)\nva_scores = booster.predict(qval, iteration_range=(0, booster.best_iteration+1))\nva_eval = va_part[[\"ranker_id\", \"selected\"]].copy()\nva_eval[\"pred\"] = va_scores\nhr3 = hitrate_at_3(va_eval)\n\nprint(f\"Val HR@3 (> 10): {hr3:.3f}\")","metadata":{},"outputs":[],"execution_count":null},{"id":"cf07588e","cell_type":"markdown","source":"## Предсказание для TEST (пакетами) + постобработка лёгкая","metadata":{}},{"id":"93b60b1a","cell_type":"code","source":"for c in num_feats:\n    test[c] = pd.to_numeric(test[c], errors=\"coerce\").fillna(0).astype(\"float32\")\nfor c in cat_feats:\n    if c in test.columns:\n        test[c] = pd.to_numeric(test[c], errors=\"coerce\").fillna(0).astype(\"int32\")\n\nscores = np.empty(len(test), dtype=np.float32)\nfor start in range(0, len(test), TEST_BATCH_ROWS):\n    end = min(len(test), start + TEST_BATCH_ROWS)\n    X_chunk = test.iloc[start:end][feature_cols].astype(\"float32\").values\n    qtest = xgb.QuantileDMatrix(X_chunk, max_bin=MAX_BIN)\n    scores[start:end] = booster.predict(qtest, iteration_range=(0, booster.best_iteration+1))\n    del X_chunk, qtest\n    gc.collect()\n\nif USE_POSTPROCESS:\n    cond = (\n        (test.get(\"is_direct_l0\", pd.Series(0)).values.astype(np.int8) == 1) &\n        (test.get(\"price_pct_rank\", pd.Series(1.0)).values.astype(np.float32) < 0.25) &\n        (test.get(\"has_access_tp\", pd.Series(0)).values.astype(np.int8) == 1)\n    )\n    scores = scores + POSTPROCESS_EPS * cond.astype(np.float32)","metadata":{},"outputs":[],"execution_count":null},{"id":"722337e2","cell_type":"markdown","source":"# Анализ ошибок","metadata":{}},{"id":"c05dc02d","cell_type":"code","source":"va_eval = va_part[[\"ranker_id\", \"selected\"]].copy()\nva_eval[\"pred\"] = va_scores\nva_eval = va_eval.copy()\n\ndef _true_mask(g: pd.DataFrame) -> pd.Series:\n    if (g[\"selected\"] == 1).any():\n        return g[\"selected\"] == 1\n    \n    return g[\"selected\"] == g[\"selected\"].max()\n\ndef per_group_stats(df: pd.DataFrame) -> pd.DataFrame:\n    rows = []\n    for rid, g in df.groupby(\"ranker_id\", sort=False):\n        g_sorted = g.sort_values(\"pred\", ascending=False).reset_index(drop=True)\n        true_idx = np.where(_true_mask(g_sorted).to_numpy())[0]\n        true_rank = int(true_idx[0] + 1) if len(true_idx) else None  # 1-based\n        hit3 = 1 if (true_rank is not None and true_rank <= 3) else 0\n        rows.append({\"ranker_id\": rid, \"group_size\": len(g_sorted), \"true_rank\": true_rank, \"hit@3\": hit3})\n\n    return pd.DataFrame(rows)\n\ngstat = per_group_stats(va_eval)\n\nhr3 = gstat[\"hit@3\"].mean()\nprint(f\"Val HR@3 (по группам): {hr3:.4f}\")\nprint(\"Распределение true_rank (первые значения):\")\nprint(gstat[\"true_rank\"].describe(percentiles=[0.5, 0.75, 0.9, 0.95, 0.99]))\n\nplt.figure()\nplt.hist(va_eval[\"pred\"].astype(float), bins=60)\nplt.title(\"Распределение предсказаний (val)\")\nplt.xlabel(\"pred\")\nplt.ylabel(\"count\")\nplt.show()\n\nplt.figure()\nplt.hist(gstat[\"true_rank\"].dropna().astype(int), bins=50)\nplt.title(\"Позиция истинного объекта в ранжировании (val)\")\nplt.xlabel(\"true_rank (1 = лучше всего)\")\nplt.ylabel(\"число групп\")\nplt.show()\n\nplt.figure()\ntmp = gstat.groupby(\"group_size\")[\"hit@3\"].mean().reset_index()\nplt.plot(tmp[\"group_size\"], tmp[\"hit@3\"])\nplt.title(\"Средний hit@3 vs размер группы\")\nplt.xlabel(\"group_size\")\nplt.ylabel(\"mean hit@3\")\nplt.show()\n\nworst = gstat.sort_values([\"true_rank\", \"group_size\"], ascending=[False, False]).head(10)\nprint(\"\\nТоп-10 худших групп по позиции истинного объекта:\")\ndisplay(worst)\n\ndef show_group_details(ranker_id, topn=8):\n    g = va_eval[va_eval[\"ranker_id\"] == ranker_id].copy()\n    g = g.sort_values(\"pred\", ascending=False).reset_index(drop=True)\n    g[\"pred_rank\"] = np.arange(1, len(g)+1)\n    g[\"is_true\"] = _true_mask(g)\n    cols = [\"pred_rank\", \"pred\", \"selected\", \"is_true\"]\n\n    return g[cols].head(topn)\n\nif len(worst) > 0:\n    rid0 = worst.iloc[0][\"ranker_id\"]\n    print(f\"\\nДетали худшей группы ranker_id={rid0}:\")\n    display(show_group_details(rid0, topn=12))\n\nmiss = (gstat[\"hit@3\"] == 0).mean()\nprint(f\"\\nДоля групп, где истинный объект НЕ в топ-3: {miss:.4f}\")","metadata":{},"outputs":[],"execution_count":null},{"id":"6480f3cf","cell_type":"markdown","source":"# Submission","metadata":{}},{"id":"d301a5bb","cell_type":"code","source":"sub = test[[\"Id\", \"ranker_id\"]].copy()\nsub[\"score\"] = scores\nsub[\"selected\"] = sub.groupby(\"ranker_id\")[\"score\"].rank(method=\"first\", ascending=False).astype(int)\nsub = sub[[\"Id\", \"ranker_id\", \"selected\"]]\n\n# validate permutation 1..N\nok = sub.groupby(\"ranker_id\")[\"selected\"].apply(lambda s: np.array_equal(np.sort(s.values), np.arange(1, len(s)+1))).all()\nassert ok, \"Перестановка 1..N для каждого ranker_id должна быть корректной\"\n\nsub.to_csv(\"submission.csv\", index=False)\n\nprint(\"Saved submission.csv to\", sub.shape)","metadata":{},"outputs":[],"execution_count":null}]}