{"metadata":{"kernelspec":{"display_name":"gml-base","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.11"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":105399,"databundleVersionId":12733338,"sourceType":"competition"}],"dockerImageVersionId":31042,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":3289.853226,"end_time":"2025-08-12T21:54:55.521816","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-08-12T21:00:05.66859","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# AeroClub RecSys 2025 - XGBoost Ranking Baseline\n\nThis notebook implements an improved ranking approach using XGBoost and Polars for the AeroClub recommendation challenge.","metadata":{"papermill":{"duration":0.008331,"end_time":"2025-08-12T21:00:08.897571","exception":false,"start_time":"2025-08-12T21:00:08.88924","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Выполняется ли локально\nIS_LOCAL = False\nIS_LOAD_MAP = False\nnum_boost_round = 3000","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%capture\n\nif not IS_LOCAL:\n    !pip install -U xgboost\n    !pip install -U polars\n    !pip install -U lightgbm","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nfrom collections import defaultdict\n\nimport matplotlib.pyplot as plt\nimport time\nimport xgboost as xgb\n\nimport gc\nimport random\nimport pickle\n\nfrom functools import reduce\nimport operator\n\nfrom scipy.special import expit  # это σ(x), сигмоида\n\nfrom datetime import datetime\nfrom tqdm import tqdm\n\nfrom copy import deepcopy\n\nfrom scipy import sparse\n\nimport lightgbm as lgb\n\nRANDOM_STATE = 42\nnp.random.seed(RANDOM_STATE)\n\npl.Config.set_fmt_str_lengths(200)\npl.Config.set_tbl_rows(50)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Helpers","metadata":{}},{"cell_type":"code","source":"def hitrate_at_3(y_true, y_pred, groups):\n    df = pl.DataFrame({\n        'group': groups,\n        'pred': y_pred,\n        'true': y_true\n    })\n    \n    return (\n        df.filter(pl.col(\"group\").count().over(\"group\") > 10)\n        .sort([\"group\", \"pred\"], descending=[False, True])\n        .group_by(\"group\", maintain_order=True)\n        .head(3)\n        .group_by(\"group\")\n        .agg(pl.col(\"true\").max())\n        .select(pl.col(\"true\").mean())\n        .item()\n    )","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"markdown","source":"### Обработка категориальных переменных","metadata":{}},{"cell_type":"code","source":"# Категориальные признаки\ncat_features_final = [\n    'nationality',\n    'companyID',\n    'corporateTariffCode',\n    # 'bySelf',\n    # 'sex',\n    'legs0_segments0_aircraft_code',\n    'legs0_segments0_arrivalTo_airport_city_iata',\n    'legs0_segments0_arrivalTo_airport_iata',\n    'legs0_segments0_departureFrom_airport_iata',\n    'legs0_segments0_marketingCarrier_code',\n    'legs0_segments0_operatingCarrier_code',\n    'legs0_segments1_aircraft_code',\n    'legs0_segments1_arrivalTo_airport_city_iata',\n    'legs0_segments1_arrivalTo_airport_iata',\n    'legs0_segments1_departureFrom_airport_iata',\n    'legs0_segments1_marketingCarrier_code',\n    'legs0_segments1_operatingCarrier_code',\n    'legs0_segments2_aircraft_code',\n    'legs0_segments2_arrivalTo_airport_city_iata',\n    'legs0_segments2_arrivalTo_airport_iata',\n    'legs0_segments2_departureFrom_airport_iata',\n    'legs0_segments2_marketingCarrier_code',\n    'legs0_segments2_operatingCarrier_code',\n    'legs1_segments0_aircraft_code',\n    'legs1_segments0_arrivalTo_airport_city_iata',\n    'legs1_segments0_arrivalTo_airport_iata',\n    'legs1_segments0_departureFrom_airport_iata',\n    'legs1_segments0_marketingCarrier_code',\n    'legs1_segments0_operatingCarrier_code',\n    'legs1_segments1_aircraft_code',\n    'legs1_segments1_arrivalTo_airport_city_iata',\n    'legs1_segments1_arrivalTo_airport_iata',\n    'legs1_segments1_departureFrom_airport_iata',\n    'legs1_segments1_marketingCarrier_code',\n    'legs1_segments1_operatingCarrier_code',\n    'legs1_segments2_aircraft_code',\n    'legs1_segments2_arrivalTo_airport_city_iata',\n    'legs1_segments2_arrivalTo_airport_iata',\n    'legs1_segments2_departureFrom_airport_iata',\n    'legs1_segments2_marketingCarrier_code',\n    'legs1_segments2_operatingCarrier_code',\n    'legs0_segments0_flightNumber',\n    'legs0_segments1_flightNumber',\n    'legs0_segments2_flightNumber',\n    'legs1_segments0_flightNumber',\n    'legs1_segments1_flightNumber',\n    'legs1_segments2_flightNumber',\n]","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def encode_categories(df: pl.DataFrame, cat_cols: list[str]) -> tuple[pl.DataFrame, dict[str, dict[str, int]]]:\n    '''\n    Создаёт маппинг для категориальных колонок и кодирует их в Int16,\n    заменяя неизвестные на -1. Значение \"missing\" всегда кодируется как -1.\n    '''\n    cat_map = {}\n    for col in cat_cols:\n        unique_vals = df[col].drop_nulls().unique().to_list()\n        \n        # Определяем тип колонки\n        col_dtype = df.schema[col]\n\n        # Создаем маппинг в зависимости от типа данных\n        if col_dtype == pl.Utf8:\n            # Для строковых колонок: \"missing\" → -1\n            unique_vals = [v for v in unique_vals if v != \"missing\"]\n            mapping = {\"missing\": -1}\n        else:\n            # Для числовых колонок: -1 → -1\n            unique_vals = [v for v in unique_vals if v != -1]\n            mapping = {-1: -1}\n\n        mapping.update({v: i for i, v in enumerate(unique_vals)})\n\n        cat_map[col] = mapping\n        max_index = -1  # используется как default для неизвестных\n\n        df = df.with_columns([\n            pl.col(col)\n            .replace_strict(mapping, default=max_index)\n            .cast(pl.Int16)\n            .alias(col)\n        ])\n\n    return df, cat_map\n\n\ndef apply_category_map(df: pl.DataFrame, cat_map: dict[str, dict[str, int]]) -> pl.DataFrame:\n    '''\n    Применяет ранее созданный маппинг к другому DataFrame (например, test), подставляя -1 для unseen значений.\n    '''\n    for col, mapping in cat_map.items():\n        #max_index = max(mapping.values()) + 1\n        max_index = -1\n        df = df.with_columns([\n            pl.col(col)\n            .replace_strict(mapping, default=max_index)\n            .cast(pl.Int16)\n            .alias(col)\n        ])\n\n    return df\n\n    \ndef get_cat_map():\n    global cat_features_final\n    '''\n    Возвращает мапинг категориальных фич по всему датасету трейн плюс тест\n    '''\n    train = pl.read_parquet('/kaggle/input/aeroclub-recsys-2025/train.parquet').drop('__index_level_0__')\n    test = pl.read_parquet('/kaggle/input/aeroclub-recsys-2025/test.parquet').drop('__index_level_0__').with_columns(pl.lit(0, dtype=pl.Int64).alias(\"selected\"))\n    data_raw = pl.concat((train, test))\n    del train, test\n    \n    data_raw, cat_map = encode_categories(data_raw, cat_features_final)\n\n    return cat_map","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Создание исторических признаков по клиентам","metadata":{}},{"cell_type":"code","source":"# More efficient duration to minutes converter\ndef dur_to_min(col):\n    # Extract days and time parts in one pass\n    days = col.str.extract(r\"^(\\d+)\\.\", 1).cast(pl.Int64).fill_null(0) * 1440\n    time_str = pl.when(col.str.contains(r\"^\\d+\\.\")).then(col.str.replace(r\"^\\d+\\.\", \"\")).otherwise(col)\n    hours = time_str.str.extract(r\"^(\\d+):\", 1).cast(pl.Int64).fill_null(0) * 60\n    minutes = time_str.str.extract(r\":(\\d+):\", 1).cast(pl.Int64).fill_null(0)\n    return (days + hours + minutes).fill_null(0)\n\n\n# def make_history_avg(df, source_col, target_col, group_col):\ndef make_history_avg(df, source_cols, group_col, suffix):\n    '''\n    Добавляет историческую информацию по выборам билетов для пользователя или компании\n    для числовых колонок\n    '''\n    # global df_means\n    \n    # Создаем датафрейм с выбранными записями (selected=1)\n    selected_df = df.filter(pl.col(\"selected\") == 1).select([\"ranker_id\", \"requestDate\", group_col] + source_cols)\n    \n    # Создаем словари для быстрого доступа\n    ranker_to_profile = dict(zip(\n        selected_df[\"ranker_id\"].to_list(),\n        selected_df[group_col].to_list()\n    ))\n\n    # Словарь ranker_id -> requestDate\n    ranker_to_timestamp = dict(zip(\n        selected_df[\"ranker_id\"].to_list(),\n        selected_df[\"requestDate\"].to_list()\n    ))\n    \n    # Создаем датафрейм с историческими данными (исключая текущие группы)\n    history_df = selected_df.select([\"ranker_id\", \"requestDate\", group_col] + source_cols)\n    \n    all_stats_dict = {}  # ranker_id -> {col_mean: val, col_std: val}\n    \n    # Получаем уникальные значения ranker_id\n    unique_ranker_ids = df[\"ranker_id\"].unique().to_list()\n        \n    # Проходим по каждой группе\n    for current_ranker_id in tqdm(unique_ranker_ids, desc=\"Обработка ranker_id\", mininterval=10.0):\n        # Получаем profile_id из словаря\n        current_profile_id = ranker_to_profile.get(current_ranker_id)\n\n        # timestamp текущей группы\n        current_timestamp = ranker_to_timestamp.get(current_ranker_id)\n        \n        # Фильтруем исторические данные для текущего профиля\n        profile_history = history_df.filter(\n            (pl.col(group_col) == current_profile_id) &\n            (pl.col(\"ranker_id\") != current_ranker_id)\n            & (pl.col(\"requestDate\") < current_timestamp)\n        )\n\n        \n        # Вычисляем статистики для всех колонок сразу\n        agg_result = profile_history.select([\n            *[pl.col(col).mean().alias(f\"{col}{suffix}_mean\") for col in source_cols],\n            *[pl.col(col).std().alias(f\"{col}{suffix}_std\") for col in source_cols],\n            *[pl.col(col).count().cast(pl.Int16).alias(f\"{col}{suffix}_count\") for col in source_cols],  # Добавляем счётчики\n            *[pl.col(col).median().alias(f\"{col}{suffix}_median\") for col in source_cols], \n            *[pl.col(col).quantile(0.25).alias(f\"{col}{suffix}_q25\") for col in source_cols],\n            *[pl.col(col).quantile(0.75).alias(f\"{col}{suffix}_q75\") for col in source_cols]\n        ])\n\n        if agg_result.height > 0:\n            row = agg_result.row(0)\n            n_cols = len(source_cols)\n            all_stats_dict[current_ranker_id] = {\n                **{\n                    f\"{col}{suffix}_mean\": row[i] if row[i] is not None else -1\n                    for i, col in enumerate(source_cols)\n                },\n                **{\n                    f\"{col}{suffix}_std\": row[i+n_cols] if row[i+n_cols] is not None else -1\n                    for i, col in enumerate(source_cols)\n                },\n                **{\n                    f\"{col}{suffix}_count\": row[i+2*n_cols] if row[i+2*n_cols] is not None else -1  # Добавляем счётчики\n                    for i, col in enumerate(source_cols)\n                },\n                **{\n                    f\"{col}{suffix}_median\": row[i+3*n_cols] if row[i+3*n_cols] is not None else -1  # Добавляем счётчики\n                    for i, col in enumerate(source_cols)\n                },\n                **{\n                    f\"{col}{suffix}_q25\": row[i+4*n_cols] if row[i+4*n_cols] is not None else -1  # Добавляем счётчики\n                    for i, col in enumerate(source_cols)\n                },\n                **{\n                    f\"{col}{suffix}_q75\": row[i+5*n_cols] if row[i+5*n_cols] is not None else -1  # Добавляем счётчики\n                    for i, col in enumerate(source_cols)\n                },\n            }\n        else:\n            all_stats_dict[current_ranker_id] = {\n                **{f\"{col}{suffix}_mean\": -1 for col in source_cols},\n                **{f\"{col}{suffix}_std\": -1 for col in source_cols},\n                **{f\"{col}{suffix}_count\": -1 for col in source_cols}  # По умолчанию 0 записей\n                **{f\"{col}{suffix}_median\": -1 for col in source_cols}  # По умолчанию 0 записей\n                **{f\"{col}{suffix}_q25\": -1 for col in source_cols}  # По умолчанию 0 записей\n                **{f\"{col}{suffix}_q75\": -1 for col in source_cols}  # По умолчанию 0 записей\n            }\n    \n    # Создаем датафрейм для обновления\n    update_data = []\n    for ranker_id, stats in all_stats_dict.items():\n        row = {\"ranker_id\": ranker_id, **stats}\n        update_data.append(row)\n    \n    update_df = pl.DataFrame(update_data)\n    \n    # Обновляем основной датафрейм\n    df = df.join(update_df, on=\"ranker_id\", how=\"left\")\n    \n    # Заполняем пропуски и приводим типы\n    for col in source_cols:\n        df = df.with_columns([\n            pl.col(f\"{col}{suffix}_mean\").fill_null(-1),\n            pl.col(f\"{col}{suffix}_std\").fill_null(-1),\n            pl.col(f\"{col}{suffix}_count\").cast(pl.Int16).fill_null(-1),\n            pl.col(f\"{col}{suffix}_median\").fill_null(-1),\n            pl.col(f\"{col}{suffix}_q25\").fill_null(-1),\n            pl.col(f\"{col}{suffix}_q75\").fill_null(-1),\n        ])\n\n    # Создаем список агрегаций\n    agg_exprs = []\n\n    for col in source_cols:\n        # Среднее значение\n        agg_exprs.append(pl.col(col).mean().alias(f\"{col}{suffix}_mean\"))\n    for col in source_cols:\n        # Стандартное отклонение\n        agg_exprs.append(pl.col(col).std().alias(f\"{col}{suffix}_std\"))\n    for col in source_cols:\n        # Количество записей (не null)\n        agg_exprs.append(pl.col(col).count().alias(f\"{col}{suffix}_count\"))\n    for col in source_cols:\n        # Количество записей (не null)\n        agg_exprs.append(pl.col(col).median().alias(f\"{col}{suffix}_median\"))\n    for col in source_cols:\n        # Количество записей (не null)\n        agg_exprs.append(pl.col(col).quantile(0.25).alias(f\"{col}{suffix}_q25\"))\n    for col in source_cols:\n        # Количество записей (не null)\n        agg_exprs.append(pl.col(col).quantile(0.75).alias(f\"{col}{suffix}_q75\"))\n\n    # Применяем агрегации\n    df_stats = (\n        df.filter(pl.col(\"selected\") == 1)\n        .group_by(group_col)\n        .agg(agg_exprs)\n    )\n    # df_stats = to_32(df_stats)\n\n    return df, df_stats","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Генерация новых признаков","metadata":{}},{"cell_type":"code","source":"def make_fetures(df, selected_df, split_groups=False, is_train=False):\n    global df_stats_pr, df_stats_co, profile_counts\n    \n    # Time features - batch process\n    time_exprs = []\n    for col in (\"legs0_departureAt\", \"legs0_arrivalAt\", \"legs1_departureAt\", \"legs1_arrivalAt\"):\n        if col in df.columns:\n            dt = pl.col(col).str.to_datetime(strict=False)\n            h = dt.dt.hour().fill_null(12)\n            time_exprs.extend([\n                h.alias(f\"{col}_hour\"),\n                dt.dt.weekday().fill_null(0).alias(f\"{col}_weekday\"),\n                (((h >= 6) & (h <= 9)) | ((h >= 17) & (h <= 20))).cast(pl.Int32).alias(f\"{col}_business_time\")\n            ])\n    if time_exprs:\n        df = df.with_columns(time_exprs)\n\n    # Process duration columns\n    dur_cols = [\"legs0_duration\", \"legs1_duration\"] + [f\"legs{l}_segments{s}_duration\" for l in (0, 1) for s in (0, 1, 2)]\n    dur_exprs = [dur_to_min(pl.col(c)).alias(c) for c in dur_cols if c in df.columns]\n\n    # Apply duration transformations first\n    if dur_exprs:\n        df = df.with_columns(dur_exprs)\n\n    # Precompute marketing carrier columns check\n    mc_cols = [f'legs{l}_segments{s}_marketingCarrier_code' for l in (0, 1) for s in range(4)]\n    mc_exists = [col for col in mc_cols if col in df.columns]\n\n    # Combine all initial transformations\n    df = df.with_columns([\n            # Price features\n            (pl.col(\"totalPrice\") / (pl.col(\"taxes\") + 1)).alias(\"price_per_tax\"),\n            (pl.col(\"taxes\") / (pl.col(\"totalPrice\") + 1)).alias(\"tax_rate\"),\n            pl.col(\"totalPrice\").log1p().alias(\"log_price\"),\n            \n            # Duration features\n            (pl.col(\"legs0_duration\").fill_null(0) + pl.col(\"legs1_duration\").fill_null(0)).alias(\"total_duration\"),\n            pl.when(pl.col(\"legs1_duration\").fill_null(0) > 0)\n                .then(pl.col(\"legs0_duration\") / (pl.col(\"legs1_duration\") + 1))\n                .otherwise(1.0).alias(\"duration_ratio\"),\n            \n            # Trip type\n            (pl.col(\"legs1_duration\").is_null() | \n            (pl.col(\"legs1_duration\") == 0) | \n            pl.col(\"legs1_segments0_departureFrom_airport_iata\").is_null()).cast(pl.Int32).alias(\"is_one_way\"),\n            \n            # Total segments count\n            (pl.sum_horizontal(pl.col(col).is_not_null().cast(pl.UInt8) for col in mc_exists) \n            if mc_exists else pl.lit(0)).alias(\"l0_seg\"),\n            \n            # FF features\n            (pl.col(\"frequentFlyer\").fill_null(\"\").str.count_matches(\"/\") + \n            (pl.col(\"frequentFlyer\").fill_null(\"\") != \"\").cast(pl.Int32)).alias(\"n_ff_programs\"),\n            \n            # Binary features\n            pl.col(\"corporateTariffCode\").is_not_null().cast(pl.Int32).alias(\"has_corporate_tariff\"),\n            (pl.col(\"pricingInfo_isAccessTP\") == 1).cast(pl.Int32).alias(\"has_access_tp\"),\n            \n            # Baggage & fees\n            (pl.col(\"legs0_segments0_baggageAllowance_quantity\").fill_null(0) + \n            pl.col(\"legs1_segments0_baggageAllowance_quantity\").fill_null(0)).alias(\"baggage_total\"),\n\n            (\n                (pl.col(\"miniRules0_monetaryAmount\") == 0)\n                & (pl.col(\"miniRules0_statusInfos\") == 1)\n            )\n            .cast(pl.Int8)\n            .alias(\"free_cancel\"),\n            (\n                (pl.col(\"miniRules1_monetaryAmount\") == 0)\n                & (pl.col(\"miniRules1_statusInfos\") == 1)\n            )\n            .cast(pl.Int8)\n            .alias(\"free_exchange\"),\n            \n            # Routes & carriers\n            pl.col(\"searchRoute\").is_in([\"MOWLED/LEDMOW\", \"LEDMOW/MOWLED\", \"MOWLED\", \"LEDMOW\", \"MOWAER/AERMOW\"])\n                .cast(pl.Int32).alias(\"is_popular_route\"),\n            \n            # Cabin\n            pl.mean_horizontal([\"legs0_segments0_cabinClass\", \"legs1_segments0_cabinClass\"]).alias(\"avg_cabin_class\"),\n            (pl.col(\"legs0_segments0_cabinClass\").fill_null(0) - \n            pl.col(\"legs1_segments0_cabinClass\").fill_null(0)).alias(\"cabin_class_diff\"),\n    ])\n\n    # Segment counts - more efficient\n    seg_exprs = []\n    for leg in (0, 1):\n        seg_cols = [f\"legs{leg}_segments{s}_duration\" for s in range(4) if f\"legs{leg}_segments{s}_duration\" in df.columns]\n        if seg_cols:\n            seg_exprs.append(\n                pl.sum_horizontal(pl.col(c).is_not_null() for c in seg_cols)\n                    .cast(pl.Int32).alias(f\"n_segments_leg{leg}\")\n            )\n        else:\n            seg_exprs.append(pl.lit(0).cast(pl.Int32).alias(f\"n_segments_leg{leg}\"))\n\n    # Add segment-based features\n    # First create segment counts\n    df = df.with_columns(seg_exprs)\n\n    # Then use them for derived features\n    df = df.with_columns([\n        (pl.col(\"n_segments_leg0\") + pl.col(\"n_segments_leg1\")).alias(\"total_segments\"),\n        (pl.col(\"n_segments_leg0\") == 1).cast(pl.Int32).alias(\"is_direct_leg0\"),\n        pl.when(pl.col(\"is_one_way\") == 1).then(0)\n            .otherwise((pl.col(\"n_segments_leg1\") == 1).cast(pl.Int32)).alias(\"is_direct_leg1\"),\n    ])\n\n    # More derived features\n    df = df.with_columns([\n        (pl.col(\"is_direct_leg0\") & pl.col(\"is_direct_leg1\")).cast(pl.Int32).alias(\"both_direct\"),\n        ((pl.col(\"isVip\") == 1) | (pl.col(\"n_ff_programs\") > 0)).cast(pl.Int32).alias(\"is_vip_freq\"),\n        (pl.col(\"baggage_total\") > 0).cast(pl.Int32).alias(\"has_baggage\"),\n        pl.col(\"Id\").count().over(\"ranker_id\").alias(\"group_size\"),\n    ])\n\n    # Add major carrier flag if column exists\n    if \"legs0_segments0_marketingCarrier_code\" in df.columns:\n        df = df.with_columns(\n            pl.col(\"legs0_segments0_marketingCarrier_code\").is_in([\"SU\", \"S7\", \"U6\"])\n                .cast(pl.Int32).alias(\"is_major_carrier\")\n        )\n    else:\n        df = df.with_columns(pl.lit(0).alias(\"is_major_carrier\"))\n\n    df = df.with_columns(pl.col(\"group_size\").log1p().alias(\"group_size_log\"))\n\n    # Batch rank computations - more efficient with single pass\n    # First apply the columns that will be used for ranking\n    df = df.with_columns([\n        pl.col(\"group_size\").log1p().alias(\"group_size_log\"),\n    ])\n\n    # Price and duration basic ranks\n    rank_exprs = []\n    for col, alias in [(\"totalPrice\", \"price\"), (\"total_duration\", \"duration\")]:\n        rank_exprs.append(pl.col(col).rank().over(\"ranker_id\").alias(f\"{alias}_rank\"))\n\n    # Price-specific features\n    price_exprs = [\n        (pl.col(\"totalPrice\").rank(\"average\").over(\"ranker_id\") / \n        pl.col(\"totalPrice\").count().over(\"ranker_id\")).alias(\"price_pct_rank\"),\n        (pl.col(\"totalPrice\") == pl.col(\"totalPrice\").min().over(\"ranker_id\")).cast(pl.Int32).alias(\"is_cheapest\"),\n        ((pl.col(\"totalPrice\") - pl.col(\"totalPrice\").median().over(\"ranker_id\")) / \n        (pl.col(\"totalPrice\").std().over(\"ranker_id\") + 1)).alias(\"price_from_median\"),\n\n        ((pl.col(\"total_duration\") - pl.col(\"total_duration\").median().over(\"ranker_id\")) / \n        (pl.col(\"total_duration\").std().over(\"ranker_id\") + 1)).alias(\"duration_from_median\"),\n\n        ((pl.col(\"legs0_duration\") - pl.col(\"legs0_duration\").median().over(\"ranker_id\")) / \n        (pl.col(\"legs0_duration\").std().over(\"ranker_id\") + 1)).alias(\"legs0_duration_from_median\"),\n\n        ((pl.col(\"legs1_duration\") - pl.col(\"legs1_duration\").median().over(\"ranker_id\")) / \n        (pl.col(\"legs1_duration\").std().over(\"ranker_id\") + 1)).alias(\"legs1_duration_from_median\"),\n\n        (pl.col(\"l0_seg\") == pl.col(\"l0_seg\").min().over(\"ranker_id\")).cast(pl.Int32).alias(\"is_min_segments\"),\n    ]\n\n    # Apply initial ranks\n    df = df.with_columns(rank_exprs + price_exprs)\n\n    # Cheapest direct - more efficient\n    direct_cheapest = (\n        df.filter(pl.col(\"is_direct_leg0\") == 1)\n        .group_by(\"ranker_id\")\n        .agg(pl.col(\"totalPrice\").min().alias(\"min_direct\"))\n    )\n\n    df = df.join(direct_cheapest, on=\"ranker_id\", how=\"left\").with_columns(\n        ((pl.col(\"is_direct_leg0\") == 1) & \n        (pl.col(\"totalPrice\") == pl.col(\"min_direct\"))).cast(pl.Int32).fill_null(0).alias(\"is_direct_cheapest\")\n    ).drop(\"min_direct\")\n\n    # Popularity features - efficient join\n    df = (\n        df.join(\n            df.group_by('legs0_segments0_marketingCarrier_code').agg(pl.mean('selected').alias('carrier0_pop')),\n            on='legs0_segments0_marketingCarrier_code', \n            how='left'\n        )\n        .join(\n            df.group_by('legs1_segments0_marketingCarrier_code').agg(pl.mean('selected').alias('carrier1_pop')),\n            on='legs1_segments0_marketingCarrier_code', \n            how='left'\n        )\n        .with_columns([\n            pl.col('carrier0_pop').fill_null(0.0),\n            pl.col('carrier1_pop').fill_null(0.0),\n        ])\n    )\n\n    # Final features including popularity\n    df = df.with_columns([\n        (pl.col('carrier0_pop') * pl.col('carrier1_pop')).alias('carrier_pop_product'),\n    ])\n\n    source_cols=['legs0_departureAt_hour','legs1_departureAt_hour','legs0_arrivalAt_hour','legs1_arrivalAt_hour',\n                 'price_rank', 'price_from_median', 'duration_rank', 'avg_cabin_class',\n                 'baggage_total', 'l0_seg',\n                 # 'legs0_segments0_seatsAvailable', 'legs0_segments0_baggageAllowance_quantity', 'legs0_segments0_cabinClass',\n                 'miniRules1_statusInfos', 'miniRules0_statusInfos',\n                 'duration_from_median', 'legs0_duration_from_median', 'legs1_duration_from_median',\n                 ]\n\n    if is_train:\n        print(\"avg на начало:\", datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\"))\n\n        df, df_stats_pr = make_history_avg(df, source_cols=source_cols, group_col=\"profileId\", suffix='_pr')\n        df, df_stats_co = make_history_avg(df, source_cols=source_cols, group_col=\"companyID\", suffix='_co')\n        print(\"avg на завершение:\", datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\"))\n        selected_df = df.filter(pl.col(\"selected\") == 1)\n\n        # Считаем количество уникальных ranker_id для каждого profileId\n        # и сохраняем в profile_counts для использования в валидационном или тестовом датасете\n        profile_counts = (\n            df\n            .group_by(\"profileId\")\n            .agg(pl.col(\"ranker_id\").n_unique().alias(\"ranker_count\"))\n        )\n\n        # Делаем счетчики с нуля не используя\n        df = df.with_columns(\n            pl.col(\"ranker_id\")\n            .n_unique()\n            .over(\"profileId\")\n            .cast(pl.Int16)\n            .alias(\"ranker_count\")\n        )\n\n        # # Присоединяем обратно к исходному df\n        # df = df.join(profile_counts, on=\"profileId\")\n\n    else:\n        # Присоединяем статистики фичей по отдельным клиентам\n        df = df.join(\n            df_stats_pr,\n            on=\"profileId\",\n            how=\"left\"\n        )\n\n        # Присоединяем статистики фичей по компаниям клиентов\n        df = df.join(\n            df_stats_co,\n            on=\"companyID\",\n            how=\"left\"\n        )\n\n        # Присоединяем profile_counts к new_df\n        df = df.join(profile_counts, on=\"profileId\", how=\"left\")\n\n        # Заполняем пропуски 1\n        df = df.with_columns(\n            pl.col(\"ranker_count\").fill_null(1)\n        )\n\n    print ('Обрабатываем пропуски')\n    \n    for col in df.select(pl.selectors.numeric()).columns:\n        df = df.with_columns(pl.col(col).fill_null(-1).alias(col))\n\n    for col in df.select(pl.selectors.string()).columns:\n        df = df.with_columns(pl.col(col).fill_null(\"missing\").alias(col))\n\n    return df, selected_df","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Feature Selection","metadata":{}},{"cell_type":"code","source":"def select_fetures(data):\n    # Возвращает data еще со всеми столбцами\n    # В feature_cols указаны те столбцы что нужно потом оставить для обучения\n\n    # Columns to exclude (uninformative or problematic)\n    exclude_cols = [\n        'Id', 'ranker_id', 'selected', 'profileId',     'requestDate',\n        'legs0_departureAt', 'legs0_arrivalAt', 'legs1_departureAt', 'legs1_arrivalAt',\n        'miniRules0_percentage', 'miniRules1_percentage',  # >90% missing\n        'frequentFlyer',  # Already processed\n        'group_size',\n        'searchRoute',\n        # Exclude constant columns\n        'pricingInfo_passengerCount', \n    ]\n\n    # Exclude segment 3 columns (>98% missing)\n    for leg in [0, 1]:\n        for seg in [3]:\n            for suffix in ['aircraft_code', 'arrivalTo_airport_city_iata', 'arrivalTo_airport_iata',\n                        'baggageAllowance_quantity', 'baggageAllowance_weightMeasurementType',\n                        'cabinClass', 'departureFrom_airport_iata', 'duration', 'flightNumber',\n                        'marketingCarrier_code', 'operatingCarrier_code', 'seatsAvailable']:\n                exclude_cols.append(f'legs{leg}_segments{seg}_{suffix}')\n\n    # # Исключаем слишком уникальные фичи\n    # for leg in [0, 1]:\n    #     for seg in [0, 1, 2]:\n    #         for suffix in ['flightNumber',\n    #                        #'arrivalTo_airport_city_iata', 'arrivalTo_airport_iata', 'departureFrom_airport_iata'\n    #                        ]:\n    #             exclude_cols.append(f'legs{leg}_segments{seg}_{suffix}')\n\n    feature_cols = [col for col in data.columns if col not in exclude_cols]\n    \n    print(f\"Using {len(feature_cols)} features ({len(cat_features_final)} categorical)\")\n    groups = data.select('ranker_id')\n\n    # return data, groups, feature_cols\n    return data, groups, feature_cols#, cat_features_final\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Подготовка к тренировке моделей","metadata":{}},{"cell_type":"markdown","source":"### Prepair Data","metadata":{}},{"cell_type":"code","source":"# Load data\ntrain = pl.read_parquet('/kaggle/input/aeroclub-recsys-2025/train.parquet').drop('__index_level_0__')\n\nn2 = train.height\nn1 = 14501077 if IS_LOCAL else train.height\n\nif IS_LOCAL:\n    validate=train[n1:n2]\n    train = train[:n1]","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"requestDate_tr = train.select('requestDate')\nprofileId_tr =  train.select('profileId')\n# Создаем новые признаки\ntrain, train_selected_df = make_fetures(train,\n                                        selected_df = None,\n                                        is_train=True)\n\n# Выбираем необходимые для дальнейшего обучения признаки\ntrain, groups_tr, feature_cols = select_fetures(train)\n\nprint('feature_cols:', feature_cols)\n\ny_tr = train.select('selected')\ntrain = train.select(feature_cols)\n# Кодируем train\ntrain, cat_mapping = encode_categories(train, cat_features_final)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_dmatrix(df, split_groups=False):\n    '''\n    Генерирует DMatrix для XGBoost\n    '''\n    df, _ = make_fetures(df, selected_df = train_selected_df, split_groups=split_groups)\n    df, groups_df, feature_cols = select_fetures(df)\n\n    y_df = df.select('selected')\n    df = df.select(feature_cols)\n    # Применяем маппинг к df\n    df = apply_category_map(df, cat_mapping)\n    group_sizes_df = groups_df.group_by('ranker_id').agg(pl.len()).sort('ranker_id')['len'].to_numpy()\n    feature_names=list(df.columns)\n    \n    d_df = xgb.DMatrix(df, label=y_df, missing=-1, group=group_sizes_df, feature_names=feature_names)\n\n    return d_df, y_df, groups_df, df","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if IS_LOCAL:\n    validate = validate.filter(\n        pl.len().over(\"ranker_id\") > 10\n    )\n    dval, y_va, groups_va, validate = get_dmatrix(validate, split_groups=True)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Optuna","metadata":{"papermill":{"duration":0.757851,"end_time":"2025-08-12T21:28:56.082843","exception":false,"start_time":"2025-08-12T21:28:55.324992","status":"completed"},"tags":[]}},{"cell_type":"code","source":"IS_OPTUNA_XGB = False\n\nif IS_OPTUNA_XGB:\n    import optuna\n    \n    train_source = train.select(feature_cols).with_columns(\n        groups_tr[\"ranker_id\"],\n        y_tr[\"selected\"],\n        requestDate_tr[\"requestDate\"]\n    )\n\n\n    MAX_PER_GROUP = 50\n\n    def objective(trial):\n        params = {\n            #'objective': 'rank:ndcg',\n            'objective': 'rank:pairwise',\n            'eval_metric': 'ndcg@3',\n            'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.05),\n            'max_depth': trial.suggest_int('max_depth', 3, 50),\n            'min_child_weight': trial.suggest_int('min_child_weight', 1, 60),\n            'subsample': trial.suggest_float('subsample', 0.9, 1.0),\n            'colsample_bytree': trial.suggest_float('colsample_bytree', 0.1, 0.6),\n            'lambda': trial.suggest_float('lambda', 0.0, 15.0),\n            'alpha': trial.suggest_float('alpha', 0.0, 5.0),\n            'gamma': trial.suggest_float('gamma', 0, 7),\n            'n_jobs': -1,\n            'verbosity': 0,\n            'seed': 42\n        }\n\n        train = train_source.clone()\n\n        rng = np.random.default_rng(RANDOM_STATE)\n\n        # Получаем уникальные ranker_id\n        unique_ids = train.select(\"ranker_id\").unique()\n        shifts = rng.integers(-2, 11, size=unique_ids.height)  # от -2 до 30 включительно\n\n        # Создаем таблицу с ranker_id и соответствующим сдвигом\n        shift_table = unique_ids.with_columns([\n            pl.Series(\"group_shift\", shifts)\n        ])\n\n        # Добавляем в train индекс строки и объединяем с shift_table по ranker_id\n        rand_series = pl.Series(\"rand\", rng.random(len(train)))\n        train = (\n            train\n            .with_row_index(\"row_idx\")\n            .with_columns(rand_series)\n            .join(shift_table, on=\"ranker_id\", how=\"left\")  # добавляем group_shift\n            .with_columns([\n                pl.len().over(\"ranker_id\").alias(\"grp_size\"),\n                pl.col(\"rand\").rank(method=\"dense\").over(\"ranker_id\").alias(\"rand_rank\"),\n                (MAX_PER_GROUP + pl.col(\"group_shift\")).alias(\"adjusted_max\")\n            ])\n            .filter(\n                (pl.col(\"grp_size\") <= MAX_PER_GROUP) |             # маленькие группы — целиком\n                (pl.col(\"selected\") == 1) |                         # выбранные строки всегда\n                (pl.col(\"rand_rank\") <= pl.col(\"adjusted_max\"))     # большие группы — отбираем с учетом смещения\n            )\n            .sort(\"row_idx\")\n            .drop([\"grp_size\", \"rand\", \"rand_rank\", \"row_idx\", \"adjusted_max\", \"group_shift\", \"requestDate\"])\n        )\n        \n        y_tr = train.select('selected')\n        groups_tr = train.select('ranker_id')\n        train = train.drop([\"selected\", \"ranker_id\"])\n\n        group_sizes_tr = groups_tr.group_by('ranker_id').agg(pl.len()).sort('ranker_id')['len'].to_numpy()\n        dtrain = xgb.DMatrix(train, label=y_tr, group=group_sizes_tr, feature_names=list(train.columns))\n\n        model = xgb.train(\n            params,\n            dtrain,\n            num_boost_round=800,\n            #num_boost_round=100,\n            evals=[(dval, 'val')],\n            #early_stopping_rounds=200,\n            verbose_eval=False\n        )\n        \n        xgb_va_preds = model.predict(dval)\n        xgb_hr3 = hitrate_at_3(y_va, xgb_va_preds, groups_va)\n        return -xgb_hr3\n\n    # Создание и запуск Optuna\n    study = optuna.create_study(direction='minimize')\n    study.optimize(objective, n_trials=300)\n\n    print(\"Best trial:\")\n    print(study.best_trial.params)","metadata":{"execution":{"iopub.execute_input":"2025-08-12T21:28:57.690728Z","iopub.status.busy":"2025-08-12T21:28:57.690325Z","iopub.status.idle":"2025-08-12T21:28:57.711199Z","shell.execute_reply":"2025-08-12T21:28:57.70557Z"},"papermill":{"duration":0.830312,"end_time":"2025-08-12T21:28:57.713931","exception":false,"start_time":"2025-08-12T21:28:56.883619","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"IS_OPTUNA_LGB = False\n\nif IS_OPTUNA_LGB:\n    import optuna\n    \n    train_source = train.select(feature_cols).with_columns(\n        groups_tr[\"ranker_id\"],\n        y_tr[\"selected\"],\n        requestDate_tr[\"requestDate\"]\n    )\n\n    MAX_PER_GROUP = 50\n\n    def objective(trial):\n        params = {\n            'objective': 'lambdarank',\n            'metric': 'ndcg',\n            'ndcg_eval_at': [3],\n            \n            # Дерево\n            'num_leaves': trial.suggest_int('num_leaves', 64, 2048),  # ~ depth 6-10\n            'min_child_samples': trial.suggest_int('min_child_samples', 2, 50),\n            \n            # Сэмплирование\n            'subsample': trial.suggest_float('subsample', 0.7, 1.0),\n            'colsample_bytree': trial.suggest_float('colsample_bytree', 0.2, 0.8),\n            \n            # Регуляризация\n            'lambda_l2': trial.suggest_float('lambda_l2', 1.0, 50.0, log=True),\n            'lambda_l1': trial.suggest_float('lambda_l1', 0.0, 1.0),\n            \n            # Обучение\n            'learning_rate': trial.suggest_float('learning_rate', 0.002, 0.05),\n            'bagging_freq': 1,\n            'force_row_wise': trial.suggest_categorical('force_row_wise', [True, False]),\n            'verbosity': -1,\n            'seed': RANDOM_STATE,\n            'n_jobs': -1,\n        }\n\n        train = train_source.clone()\n\n        rng = np.random.default_rng(RANDOM_STATE)\n\n        # Получаем уникальные ranker_id\n        unique_ids = train.select(\"ranker_id\").unique()\n        shifts = rng.integers(-2, 11, size=unique_ids.height)  # от -2 до 30 включительно\n\n        # Создаем таблицу с ranker_id и соответствующим сдвигом\n        shift_table = unique_ids.with_columns([\n            pl.Series(\"group_shift\", shifts)\n        ])\n\n        # Добавляем в train индекс строки и объединяем с shift_table по ranker_id\n        rand_series = pl.Series(\"rand\", rng.random(len(train)))\n        train = (\n            train\n            .with_row_index(\"row_idx\")\n            .with_columns(rand_series)\n            .join(shift_table, on=\"ranker_id\", how=\"left\")  # добавляем group_shift\n            .with_columns([\n                pl.len().over(\"ranker_id\").alias(\"grp_size\"),\n                pl.col(\"rand\").rank(method=\"dense\").over(\"ranker_id\").alias(\"rand_rank\"),\n                (MAX_PER_GROUP + pl.col(\"group_shift\")).alias(\"adjusted_max\")\n            ])\n            .filter(\n                (pl.col(\"grp_size\") <= MAX_PER_GROUP) |             # маленькие группы — целиком\n                (pl.col(\"selected\") == 1) |                         # выбранные строки всегда\n                (pl.col(\"rand_rank\") <= pl.col(\"adjusted_max\"))     # большие группы — отбираем с учетом смещения\n            )\n            .sort(\"row_idx\")\n            .drop([\"grp_size\", \"rand\", \"rand_rank\", \"row_idx\", \"adjusted_max\", \"group_shift\", \"requestDate\"])\n        )\n        \n        y_tr = train.select('selected').to_numpy().flatten()\n        groups_tr = train.select('ranker_id')\n        train = train.drop([\"selected\", \"ranker_id\"])\n        # -----------------------------------\n\n        # Подготовка данных для LightGBM\n        train_data = lgb.Dataset(\n            train,\n            label=y_tr,\n            group=groups_tr.group_by('ranker_id').agg(pl.len()).sort('ranker_id')['len'].to_list(),\n            feature_name=list(train.columns),\n            # categorical_feature=cat_features_final,\n        )\n\n        if IS_LOCAL:\n            val_data = lgb.Dataset(\n                validate,\n                label=y_va.to_numpy().flatten(),\n                group=groups_va.group_by('ranker_id').agg(pl.len()).sort('ranker_id')['len'].to_list(),\n                reference=train_data,\n                feature_name=list(validate.columns),\n                #categorical_feature=cat_features_final,\n            )\n        else:\n            val_data = None\n        \n        # Обучаем модель\n        model = lgb.train(\n            params,\n            train_data,\n            num_boost_round=800,\n            # valid_sets=val_data,\n            # callbacks=[lgb.log_evaluation(50)] if IS_LOCAL else None\n            # callbacks=[biased_bagging_callback(neg_fraction=0.8)]\n        )\n\n        lgb_va_preds_1 = model.predict(validate)\n        # # Evaluate XGBoost\n        lgb_hr3 = hitrate_at_3(y_va, lgb_va_preds_1, groups_va)\n        return -lgb_hr3\n\n    # Создание и запуск Optuna\n    study = optuna.create_study(direction='minimize')\n    study.optimize(objective, n_trials=300)\n\n    print(\"Best trial:\")\n    print(study.best_trial.params)","metadata":{"execution":{"iopub.execute_input":"2025-08-12T21:28:59.326608Z","iopub.status.busy":"2025-08-12T21:28:59.325737Z","iopub.status.idle":"2025-08-12T21:28:59.348508Z","shell.execute_reply":"2025-08-12T21:28:59.343597Z"},"papermill":{"duration":0.86407,"end_time":"2025-08-12T21:28:59.350965","exception":false,"start_time":"2025-08-12T21:28:58.486895","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Тренировка моделей","metadata":{"papermill":{"duration":0.799965,"end_time":"2025-08-12T21:29:00.992121","exception":false,"start_time":"2025-08-12T21:29:00.192156","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Исходные данные до удаления столбцов и строк для тренировки\ntrain_source = train.select(feature_cols).with_columns(\n    groups_tr[\"ranker_id\"],\n    y_tr[\"selected\"],\n    requestDate_tr[\"requestDate\"],\n    profileId_tr[\"profileId\"]\n)\n\nif IS_LOCAL:\n    # Исходные данные до удаления столбцов и строк для валидации\n    val_source = validate.with_columns(\n        groups_va[\"ranker_id\"],\n        y_va[\"selected\"],\n    )","metadata":{"execution":{"iopub.execute_input":"2025-08-12T21:29:02.614074Z","iopub.status.busy":"2025-08-12T21:29:02.613731Z","iopub.status.idle":"2025-08-12T21:29:02.628968Z","shell.execute_reply":"2025-08-12T21:29:02.624629Z"},"papermill":{"duration":0.826171,"end_time":"2025-08-12T21:29:02.631828","exception":false,"start_time":"2025-08-12T21:29:01.805657","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def drop_random_groups(df: pl.DataFrame, frac: float, group_col: str = \"ranker_id\") -> pl.DataFrame:\n    \"\"\"\n    Удаляет случайную долю групп из датафрейма.\n    \n    Parameters\n    ----------\n    df : pl.DataFrame\n        Исходный DataFrame.\n    frac : float\n        Доля групп, которую нужно удалить (0 < frac <= 1).\n    group_col : str\n        Название колонки, по которой определяются группы.\n    \n    Returns\n    -------\n    pl.DataFrame\n        DataFrame без удалённых групп.\n    \"\"\"\n    if not (0 < frac <= 1):\n        raise ValueError(\"frac должно быть в диапазоне (0, 1].\")\n    \n    # Уникальные группы\n    unique_groups = df.select(group_col).unique()[group_col].to_list()\n    \n    # Кол-во групп для удаления (минимум 1)\n    n_remove = max(1, int(len(unique_groups) * frac))\n    \n    # Случайный выбор групп\n    groups_to_remove = random.sample(unique_groups, n_remove)\n    \n    # Фильтрация\n    return df.filter(~pl.col(group_col).is_in(groups_to_remove))","metadata":{"execution":{"iopub.execute_input":"2025-08-12T21:29:04.20458Z","iopub.status.busy":"2025-08-12T21:29:04.204236Z","iopub.status.idle":"2025-08-12T21:29:04.217531Z","shell.execute_reply":"2025-08-12T21:29:04.213227Z"},"papermill":{"duration":0.820636,"end_time":"2025-08-12T21:29:04.220249","exception":false,"start_time":"2025-08-12T21:29:03.399613","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def make_short_groups(max_per_group=40):\n    '''Делает короткие группы из тех что превышают заданный размер'''\n\n    global train_source\n\n    train = train_source.clone()\n\n    rng = np.random.default_rng(RANDOM_STATE)\n\n    # Получаем уникальные ranker_id\n    unique_ids = train.select(\"ranker_id\").unique()\n    shifts = rng.integers(-2, max_per_group//5, size=unique_ids.height)  # от -2 до 30 включительно\n\n    # Создаем таблицу с ranker_id и соответствующим сдвигом\n    shift_table = unique_ids.with_columns([\n        pl.Series(\"group_shift\", shifts)\n    ])\n\n    # Добавляем в train индекс строки и объединяем с shift_table по ranker_id\n    rand_series = pl.Series(\"rand\", rng.random(len(train)))\n    print(train.height)\n    train = (\n        train\n        .with_row_index(\"row_idx\")\n        .with_columns(rand_series)\n        .join(shift_table, on=\"ranker_id\", how=\"left\")  # добавляем group_shift\n        .with_columns([\n            pl.len().over(\"ranker_id\").alias(\"grp_size\"),\n            pl.col(\"rand\").rank(method=\"dense\").over(\"ranker_id\").alias(\"rand_rank\"),\n            (max_per_group + pl.col(\"group_shift\")).alias(\"adjusted_max\")\n        ])\n        .filter(\n            (pl.col(\"grp_size\") <= max_per_group) |             # маленькие группы — целиком\n            (pl.col(\"selected\") == 1) |                         # выбранные строки всегда\n            (pl.col(\"rand_rank\") <= pl.col(\"adjusted_max\"))     # большие группы — отбираем с учетом смещения\n        )\n        .sort(\"row_idx\")\n        .drop([\"grp_size\", \"rand\", \"rand_rank\", \"row_idx\", \"adjusted_max\", \"group_shift\"])\n    )\n    print(train.height)\n\n    return train","metadata":{"execution":{"iopub.execute_input":"2025-08-12T21:29:05.878061Z","iopub.status.busy":"2025-08-12T21:29:05.877737Z","iopub.status.idle":"2025-08-12T21:29:05.89296Z","shell.execute_reply":"2025-08-12T21:29:05.88725Z"},"papermill":{"duration":0.866106,"end_time":"2025-08-12T21:29:05.895242","exception":false,"start_time":"2025-08-12T21:29:05.029136","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_names=list(train.columns)","metadata":{"execution":{"iopub.execute_input":"2025-08-12T21:29:07.518675Z","iopub.status.busy":"2025-08-12T21:29:07.518331Z","iopub.status.idle":"2025-08-12T21:29:07.530295Z","shell.execute_reply":"2025-08-12T21:29:07.524925Z"},"papermill":{"duration":0.857782,"end_time":"2025-08-12T21:29:07.532667","exception":false,"start_time":"2025-08-12T21:29:06.674885","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_fin_df(df, model_idx):\n    '''\n    В зависимости от индекса model_ids модели удаляем колонки в датафрейме df\n    '''\n    if model_idx % 2 == 0:\n        # Эту модель учим без истории пользователя\n        # Формируем список колонок, которые надо удалить\n        suffixes = (\"pr_std\", \"pr_mean\", \"pr_count\", \"pr_median\")\n\n        cols_to_drop = [col for col in df.columns if col.endswith(suffixes)]\n\n        # Удаляем их\n        df = df.drop(cols_to_drop)\n    \n    return df\n\ndef mask_predictions(df: pl.DataFrame, preds: np.ndarray, model_idx: int) -> np.ndarray:\n    \"\"\"\n    Маскирует предсказания в зависимости от model_idx.\n    \n    Параметры:\n        df (pl.DataFrame): Датафрейм с колонкой 'ranker_count'.\n        preds (np.ndarray): Массив предсказаний для всех строк.\n        model_idx (int): Индекс модели (итерации).\n    \n    Возвращает:\n        np.ndarray: Массив предсказаний с примененной маской.\n    \"\"\"\n    preds = preds.copy()  # чтобы не портить исходный массив\n\n    ranker_count = df[\"ranker_count\"].to_numpy()\n\n    if model_idx % 2 == 0:\n        # четный индекс → обнуляем, где ranker_count > 1\n        preds[ranker_count > 1] = 0\n    else:\n        # нечетный индекс → обнуляем, где ranker_count == 1\n        preds[ranker_count == 1] = 0\n\n    return preds","metadata":{"execution":{"iopub.execute_input":"2025-08-12T21:29:09.126824Z","iopub.status.busy":"2025-08-12T21:29:09.126453Z","iopub.status.idle":"2025-08-12T21:29:09.141072Z","shell.execute_reply":"2025-08-12T21:29:09.135515Z"},"papermill":{"duration":0.783829,"end_time":"2025-08-12T21:29:09.143678","exception":false,"start_time":"2025-08-12T21:29:08.359849","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def ensure_1pct_minus1(df: pl.DataFrame) -> pl.DataFrame:\n    '''Добавляет 1% пропусков случайно. Пропуски это -1.'''\n    \n    df_new = df.clone()\n\n    n_rows = df.height\n    target_count = int(np.floor(n_rows * 0.005))  # 1% от строк\n\n    for col in df.columns:\n        series = df_new[col]\n        \n        # Пропускаем булевые колонки\n        if series.dtype == pl.Boolean:\n            continue\n\n        current_minus1_count = (series == -1).sum()\n\n        if current_minus1_count >= target_count:\n            # Уже есть ≥1% -1, пропускаем\n            continue\n\n        # Сколько ещё нужно добавить -1\n        need_to_replace = target_count - current_minus1_count\n        if need_to_replace <= 0:\n            continue\n\n        # Индексы, которые не равны -1\n        available_indices = np.where(series.to_numpy() != -1)[0]\n\n        # Случайные индексы для замены\n        replace_indices = np.random.choice(\n            available_indices, size=need_to_replace, replace=False\n        )\n\n        # Создаём копию колонки с заменой\n        col_values = series.to_numpy().copy()\n        col_values[replace_indices] = -1\n\n        # Обновляем колонку\n        df_new = df_new.with_columns(pl.Series(name=col, values=col_values))\n\n    return df_new","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### XGBoost","metadata":{"papermill":{"duration":0.768079,"end_time":"2025-08-12T21:29:13.977434","exception":false,"start_time":"2025-08-12T21:29:13.209355","status":"completed"},"tags":[]}},{"cell_type":"code","source":"params = {\n    'objective': 'rank:pairwise',\n    #'objective': 'rank:ndcg',\n    'eval_metric': 'ndcg@3',\n    'learning_rate': 0.03430629350470104,\n    'max_depth': 20,\n    'min_child_weight': 24,\n    'subsample': 0.9680833434687813,\n    'colsample_bytree': 0.1859969541434444,\n    'lambda': 13.73979976725866,\n    'alpha': 4.315374828140574,\n    'gamma': 0.145064313893779,\n    # 'n_jobs': 48,\n    'n_jobs': -1,\n}\n\nMAX_PER_GROUP = 50\nKEEP = 0.8  # оставить 80% невыбранных (т.е. удалить 20%)\n\n# Обучение каждой модели\nprint(f\"\\nTraining model with params: {params}\")\n\ntrain = make_short_groups(max_per_group=MAX_PER_GROUP)\n\ny_tr = train.select('selected')\ngroups_tr = train.select('ranker_id')\ntrain = train.drop([\"selected\", \"ranker_id\", \"requestDate\", \"profileId\"])\n\ntrain = ensure_1pct_minus1(train)\n\ngroup_sizes_tr = groups_tr.group_by('ranker_id').agg(pl.len()).sort('ranker_id')['len'].to_numpy()\ndtrain = xgb.DMatrix(train, label=y_tr, missing=-1, group=group_sizes_tr, feature_names=list(train.columns))\n\nevals = [(dtrain, 'train'), (dval, 'val')] if IS_LOCAL else None\n\n\n# Обучаем модель\nmodel = xgb.train(\n    params,\n    dtrain,\n    num_boost_round=num_boost_round,\n    evals=evals if evals else [],\n    verbose_eval=200\n)\n\nif IS_LOCAL:\n    xgb_va_preds_1 = model.predict(dval)\n    # # Evaluate XGBoost\n    xgb_hr3 = hitrate_at_3(y_va, xgb_va_preds_1, groups_va)\n    print(f\"HitRate@3: {xgb_hr3:.8f}\")","metadata":{"execution":{"iopub.execute_input":"2025-08-12T21:29:15.599588Z","iopub.status.busy":"2025-08-12T21:29:15.599247Z","iopub.status.idle":"2025-08-12T21:52:57.403584Z","shell.execute_reply":"2025-08-12T21:52:57.397993Z"},"papermill":{"duration":1423.409826,"end_time":"2025-08-12T21:52:58.19553","exception":false,"start_time":"2025-08-12T21:29:14.785704","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Check Metric","metadata":{"papermill":{"duration":0.79777,"end_time":"2025-08-12T21:52:59.796931","exception":false,"start_time":"2025-08-12T21:52:58.999161","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def apply_hour_penalty(df: pl.DataFrame, hour_col: str, hour_round: int = 1, penalty_factor: float = 0.2) -> pl.DataFrame:\n    rounded_col = f\"{hour_col}_rounded\"\n\n    # 1. Округляем часы и заменяем 24 → 0\n    df = df.with_columns(\n        pl.when(\n            ((pl.col(hour_col) / hour_round).round() * hour_round).cast(pl.Int8) == 24\n        )\n        .then(0)\n        .otherwise((pl.col(hour_col) / hour_round).round() * hour_round)\n        .cast(pl.Int8)\n        .alias(rounded_col)\n    )\n\n    # 2. Находим максимум pred_score в группе ranker_id + округлённый час\n    max_col = f\"max_score_same_{rounded_col}\"\n    df = df.with_columns(\n        pl.max(\"pred_score\")\n        .over([\"ranker_id\", rounded_col])\n        .alias(max_col)\n    )\n\n    # 3. Обновляем pred_score с учётом штрафа\n    df = df.with_columns(\n        (\n            pl.col(\"pred_score\")\n            - penalty_factor * (pl.col(max_col) - pl.col(\"pred_score\"))\n        ).alias(\"pred_score\")\n    )\n\n    return df","metadata":{"execution":{"iopub.execute_input":"2025-08-12T21:53:01.353525Z","iopub.status.busy":"2025-08-12T21:53:01.35325Z","iopub.status.idle":"2025-08-12T21:53:01.365957Z","shell.execute_reply":"2025-08-12T21:53:01.360755Z"},"papermill":{"duration":0.816643,"end_time":"2025-08-12T21:53:01.368592","exception":false,"start_time":"2025-08-12T21:53:00.551949","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Скор после пост обработки","metadata":{"papermill":{"duration":0.829097,"end_time":"2025-08-12T21:53:04.660422","exception":false,"start_time":"2025-08-12T21:53:03.831325","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if IS_LOCAL:\n    penalty_factor = 0.05\n    hour_round = 2\n\n    val_score = val_source.with_columns(pl.Series('pred_score', xgb_va_preds_1))\n\n    val_score = apply_hour_penalty(val_score, \"legs0_departureAt_hour\", hour_round=hour_round, penalty_factor=penalty_factor)\n    val_score = apply_hour_penalty(val_score, \"legs0_arrivalAt_hour\", hour_round=hour_round, penalty_factor=penalty_factor)\n\n    reorder_score = val_score[\"pred_score\"].to_numpy()\n    xgb_hr3 = hitrate_at_3(y_va, reorder_score, groups_va)\n    print(f\"HitRate@3 All: {xgb_hr3:.8f}\")","metadata":{"execution":{"iopub.execute_input":"2025-08-12T21:53:06.23235Z","iopub.status.busy":"2025-08-12T21:53:06.232066Z","iopub.status.idle":"2025-08-12T21:53:06.242501Z","shell.execute_reply":"2025-08-12T21:53:06.238267Z"},"papermill":{"duration":0.818754,"end_time":"2025-08-12T21:53:06.244635","exception":false,"start_time":"2025-08-12T21:53:05.425881","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Важность признаков","metadata":{"papermill":{"duration":0.796999,"end_time":"2025-08-12T21:53:07.838322","exception":false,"start_time":"2025-08-12T21:53:07.041323","status":"completed"},"tags":[]}},{"cell_type":"code","source":"xgb_importance = model.get_score(importance_type='gain')\nxgb_importance_df = pl.DataFrame(\n    [{'feature': k, 'importance': v} for k, v in xgb_importance.items()]\n).sort('importance', descending=bool(1))\nprint(xgb_importance_df.head(1000).to_pandas().to_string())","metadata":{"execution":{"iopub.execute_input":"2025-08-12T21:53:09.429047Z","iopub.status.busy":"2025-08-12T21:53:09.428785Z","iopub.status.idle":"2025-08-12T21:53:09.439298Z","shell.execute_reply":"2025-08-12T21:53:09.434439Z"},"papermill":{"duration":0.836742,"end_time":"2025-08-12T21:53:09.441794","exception":false,"start_time":"2025-08-12T21:53:08.605052","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Error analysis and visualization","metadata":{"papermill":{"duration":0.805775,"end_time":"2025-08-12T21:53:12.691601","exception":false,"start_time":"2025-08-12T21:53:11.885826","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if IS_LOCAL:\n    # Color palette\n    red = (0.86, 0.08, 0.24)\n    blue = (0.12, 0.56, 1.0)\n\n    # Prepare data for analysis\n    va_df = pl.DataFrame({\n        'ranker_id': groups_va.to_numpy().flatten(),\n        'pred_score': xgb_va_preds_1,\n        'selected': y_va.to_numpy().flatten()\n    })\n\n    # Add group size and filter\n    va_df = va_df.join(\n        va_df.group_by('ranker_id').agg(pl.len().alias('group_size')), \n        on='ranker_id'\n    ).filter(pl.col('group_size') > 10)\n\n    # Calculate group size quantiles\n    size_quantiles = va_df.select('ranker_id', 'group_size').unique().select(\n        pl.col('group_size').quantile(0.25).alias('q25'),\n        pl.col('group_size').quantile(0.50).alias('q50'),\n        pl.col('group_size').quantile(0.75).alias('q75')\n    ).to_dicts()[0]\n\n    # Function to calculate hitrate curve efficiently\n    def calculate_hitrate_curve(df, k_values):\n        # Sort once and calculate all k values\n        sorted_df = df.sort([\"ranker_id\", \"pred_score\"], descending=[False, True])\n        return [\n            sorted_df.group_by(\"ranker_id\", maintain_order=True)\n            .head(k)\n            .group_by(\"ranker_id\")\n            .agg(pl.col(\"selected\").max().alias(\"hit\"))\n            .select(pl.col(\"hit\").mean())\n            .item()\n            for k in k_values\n        ]\n\n    # Calculate curves\n    k_values = list(range(1, 21))\n    curves = {\n        'All groups (>10)': calculate_hitrate_curve(va_df, k_values),\n        f'Small (11-{int(size_quantiles[\"q25\"])})': calculate_hitrate_curve(\n            va_df.filter(pl.col('group_size') <= size_quantiles['q25']), k_values\n        ),\n        f'Medium ({int(size_quantiles[\"q25\"]+1)}-{int(size_quantiles[\"q75\"])})': calculate_hitrate_curve(\n            va_df.filter((pl.col('group_size') > size_quantiles['q25']) & \n                        (pl.col('group_size') <= size_quantiles['q75'])), k_values\n        ),\n        f'Large (>{int(size_quantiles[\"q75\"])})': calculate_hitrate_curve(\n            va_df.filter(pl.col('group_size') > size_quantiles['q75']), k_values\n        )\n    }\n\n    # Calculate hitrate@3 by group size using log-scale bins\n    # Create log-scale bins\n    min_size = va_df['group_size'].min()\n    max_size = va_df['group_size'].max()\n    bins = np.logspace(np.log10(min_size), np.log10(max_size), 51)  # 51 edges = 50 bins\n\n    # Calculate hitrate@3 for each ranker_id\n    ranker_hr3 = (\n        va_df.sort([\"ranker_id\", \"pred_score\"], descending=[False, True])\n        .group_by(\"ranker_id\", maintain_order=True)\n        .agg([\n            pl.col(\"selected\").head(3).max().alias(\"hit_top3\"),\n            pl.col(\"group_size\").first()\n        ])\n    )\n\n    # Assign bins and calculate hitrate per bin\n    bin_centers = (bins[:-1] + bins[1:]) / 2  # Geometric mean would be more accurate for log scale\n    bin_indices = np.digitize(ranker_hr3['group_size'].to_numpy(), bins) - 1\n\n    size_analysis = pl.DataFrame({\n        'bin_idx': bin_indices,\n        'bin_center': bin_centers[np.clip(bin_indices, 0, len(bin_centers)-1)],\n        'hit_top3': ranker_hr3['hit_top3']\n    }).group_by(['bin_idx', 'bin_center']).agg([\n        pl.col('hit_top3').mean().alias('hitrate3'),\n        pl.len().alias('n_groups')\n    ]).filter(pl.col('n_groups') >= 3).sort('bin_center')  # At least 3 groups per bin\n\n    # Create combined figure\n    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(8, 4), dpi=400)\n\n    # Left plot: HitRate@k curves\n    # Create color gradient from blue to red for size groups\n    colors = ['black']  # All groups is black\n    for i in range(3):  # 3 size groups\n        t = i / 2  # 0, 0.5, 1\n        color = tuple(blue[j] * (1 - t) + red[j] * t for j in range(3))\n        colors.append(color)\n\n    for (label, hitrates), color in zip(curves.items(), colors):\n        ax1.plot(k_values, hitrates, marker='o', label=label, color=color, markersize=3)\n    ax1.set_xlabel('k (top-k predictions)')\n    ax1.set_ylabel('HitRate@k')\n    ax1.set_title('HitRate@k by Group Size')\n    ax1.legend(fontsize=8)\n    ax1.grid(True, alpha=0.3)\n    ax1.set_xlim(0, 21)\n    ax1.set_ylim(-0.025, 1.025)\n\n    # Right plot: HitRate@3 vs Group Size (log scale)\n    ax2.scatter(size_analysis['bin_center'], size_analysis['hitrate3'], s=30, alpha=0.6, color=blue)\n    ax2.set_xlabel('Group Size')\n    ax2.set_ylabel('HitRate@3')\n    ax2.set_title('HitRate@3 vs Group Size')\n    ax2.set_xscale('log')\n    ax2.grid(True, alpha=0.3)\n\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.execute_input":"2025-08-12T21:53:14.244886Z","iopub.status.busy":"2025-08-12T21:53:14.2446Z","iopub.status.idle":"2025-08-12T21:53:14.26521Z","shell.execute_reply":"2025-08-12T21:53:14.262195Z"},"papermill":{"duration":0.823971,"end_time":"2025-08-12T21:53:14.268766","exception":false,"start_time":"2025-08-12T21:53:13.444795","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if IS_LOCAL:\n    # Summary\n    print(f\"HitRate@1: {curves['All groups (>10)'][0]:.3f}\")\n    print(f\"HitRate@3: {curves['All groups (>10)'][2]:.3f}\")\n    print(f\"HitRate@5: {curves['All groups (>10)'][4]:.3f}\")\n    print(f\"HitRate@10: {curves['All groups (>10)'][9]:.3f}\")","metadata":{"execution":{"iopub.execute_input":"2025-08-12T21:53:15.855513Z","iopub.status.busy":"2025-08-12T21:53:15.855245Z","iopub.status.idle":"2025-08-12T21:53:15.864906Z","shell.execute_reply":"2025-08-12T21:53:15.860656Z"},"papermill":{"duration":0.78983,"end_time":"2025-08-12T21:53:15.867162","exception":false,"start_time":"2025-08-12T21:53:15.077332","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Submission","metadata":{"papermill":{"duration":0.801615,"end_time":"2025-08-12T21:53:19.127969","exception":false,"start_time":"2025-08-12T21:53:18.326354","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### Предсказания","metadata":{"papermill":{"duration":0.75359,"end_time":"2025-08-12T21:53:20.675971","exception":false,"start_time":"2025-08-12T21:53:19.922381","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if not IS_LOCAL:\n    # Load data\n    del train\n    del train_source\n\n    test = pl.read_parquet('/kaggle/input/aeroclub-recsys-2025/test.parquet').drop('__index_level_0__').with_columns(pl.lit(0, dtype=pl.Int64).alias(\"selected\"))\n\n    test, _ = make_fetures(test, selected_df=train_selected_df)\n    test, groups_te, feature_cols = select_fetures(test)\n\n    y_te = test.select('selected')\n    test = test.select(feature_cols)\n    # Применяем маппинг к test\n    test = apply_category_map(test, cat_mapping)\n\n    group_sizes_te = groups_te.group_by('ranker_id').agg(pl.len()).sort('ranker_id')['len'].to_numpy()\n    dtest  = xgb.DMatrix(test, label=y_te, missing=-1, group=group_sizes_te, feature_names=test.columns)\n\n    preds = model.predict(dtest)","metadata":{"execution":{"iopub.execute_input":"2025-08-12T21:53:22.298094Z","iopub.status.busy":"2025-08-12T21:53:22.297578Z","iopub.status.idle":"2025-08-12T21:54:35.502679Z","shell.execute_reply":"2025-08-12T21:54:35.497644Z"},"papermill":{"duration":74.013535,"end_time":"2025-08-12T21:54:35.505397","exception":false,"start_time":"2025-08-12T21:53:21.491862","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if not IS_LOCAL:\n    IS_RERANK = True\n\n    if IS_RERANK:\n        test_score = test.with_columns(\n            groups_te[\"ranker_id\"],\n            y_te[\"selected\"],\n        )\n        penalty_factor = 0.1\n        hour_round = 1\n\n        test_score = test_score.with_columns(pl.Series('pred_score', preds))\n\n        test_score = apply_hour_penalty(test_score, \"legs0_departureAt_hour\", hour_round=hour_round, penalty_factor=penalty_factor)\n        test_score = apply_hour_penalty(test_score, \"legs0_arrivalAt_hour\", hour_round=hour_round, penalty_factor=penalty_factor)\n\n        preds = test_score[\"pred_score\"].to_numpy()","metadata":{"execution":{"iopub.execute_input":"2025-08-12T21:54:37.051404Z","iopub.status.busy":"2025-08-12T21:54:37.050984Z","iopub.status.idle":"2025-08-12T21:54:37.496948Z","shell.execute_reply":"2025-08-12T21:54:37.492552Z"},"papermill":{"duration":1.240534,"end_time":"2025-08-12T21:54:37.499568","exception":false,"start_time":"2025-08-12T21:54:36.259034","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Постобработка предсказания на тесте","metadata":{"papermill":{"duration":0.746362,"end_time":"2025-08-12T21:54:39.046768","exception":false,"start_time":"2025-08-12T21:54:38.300406","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if not IS_LOCAL:\n    gc.collect()\n\n    test = pl.read_parquet('/kaggle/input/aeroclub-recsys-2025/test.parquet').drop('__index_level_0__').with_columns(pl.lit(0, dtype=pl.Int64).alias(\"selected\"))\n\n    # Без постобработки\n    submission_xgb = (\n        test.select(['Id', 'ranker_id'])\n        #.with_columns(pl.Series('pred_score', xgb_model_1.predict(dtest)+xgb_model_2.predict(dtest)))\n        .with_columns(pl.Series('pred_score', preds))\n        .with_columns(\n            pl.col('pred_score')\n            .rank(method='ordinal', descending=True)\n            .over('ranker_id')\n            .cast(pl.Int32)\n            .alias('selected')\n        )\n        .select(['Id', 'ranker_id', 'selected'])\n    )\n\n    submission_xgb.write_csv('submission.csv')","metadata":{"execution":{"iopub.execute_input":"2025-08-12T21:54:40.676401Z","iopub.status.busy":"2025-08-12T21:54:40.676107Z","iopub.status.idle":"2025-08-12T21:54:42.312228Z","shell.execute_reply":"2025-08-12T21:54:42.306772Z"},"papermill":{"duration":2.480816,"end_time":"2025-08-12T21:54:42.315055","exception":false,"start_time":"2025-08-12T21:54:39.834239","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.749071,"end_time":"2025-08-12T21:54:43.896247","exception":false,"start_time":"2025-08-12T21:54:43.147176","status":"completed"},"tags":[]},"outputs":[],"execution_count":null}]}