{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":105399,"databundleVersionId":12733338,"sourceType":"competition"},{"sourceId":14537789,"sourceType":"datasetVersion","datasetId":9285244}],"dockerImageVersionId":31259,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"pip install duckdb pandas pyarrow","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T15:46:14.549493Z","iopub.execute_input":"2026-01-18T15:46:14.549944Z","iopub.status.idle":"2026-01-18T15:46:18.969511Z","shell.execute_reply.started":"2026-01-18T15:46:14.549868Z","shell.execute_reply":"2026-01-18T15:46:18.967809Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Загрузка данных","metadata":{}},{"cell_type":"markdown","source":"**Так как из-за размера датасета не хватает мощности, необходимо порезать исходный датасет. Для этого буду использовать DuckDB, чтобы взять 2 миллиона строк из train и 1 миллион строк из test**","metadata":{}},{"cell_type":"code","source":"def quick_reduce_dataset(input_file, output_file, method='sample', **kwargs):\n    \"\"\"\n    Универсальная функция для уменьшения датасета\n    \n    Параметры:\n        method: 'sample' - случайная выборка\n                'first' - первые N строк\n                'time' - по времени\n                'users' - по пользователям\n    \"\"\"\n    conn = duckdb.connect()\n    \n    if method == 'sample':\n        if 'sample_size' in kwargs:\n            query = f\"\"\"\n            COPY (\n                SELECT * \n                FROM read_parquet('{input_file}')\n                USING SAMPLE {kwargs['sample_size']}\n            ) TO '{output_file}' (FORMAT PARQUET)\n            \"\"\"\n        else:\n            query = f\"\"\"\n            COPY (\n                SELECT * \n                FROM read_parquet('{input_file}')\n                TABLESAMPLE (1 PERCENT)\n            ) TO '{output_file}' (FORMAT PARQUET)\n            \"\"\"\n    \n\n    \n    \n    print(f\"Выполнение запроса...\")\n    conn.execute(query)\n    \n    original_count = conn.execute(f\"SELECT COUNT(*) FROM '{input_file}'\").fetchone()[0]\n    sample_count = conn.execute(f\"SELECT COUNT(*) FROM '{output_file}'\").fetchone()[0]\n    \n    print(f\"Оригинальный файл: {original_count:,} строк\")\n    print(f\"Выборка: {sample_count:,} строк\")\n    print(f\"Уменьшение: {sample_count/original_count*100:.2f}% от оригинала\")\n    \n    conn.close()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T15:46:20.542342Z","iopub.execute_input":"2026-01-18T15:46:20.542788Z","iopub.status.idle":"2026-01-18T15:46:20.552015Z","shell.execute_reply.started":"2026-01-18T15:46:20.542746Z","shell.execute_reply":"2026-01-18T15:46:20.550822Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import duckdb","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T15:46:35.873123Z","iopub.execute_input":"2026-01-18T15:46:35.873514Z","iopub.status.idle":"2026-01-18T15:46:36.041884Z","shell.execute_reply.started":"2026-01-18T15:46:35.873485Z","shell.execute_reply":"2026-01-18T15:46:36.040323Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"quick_reduce_dataset(\n    '/kaggle/input/aeroclub-recsys-2025/train.parquet', \n    'train_small.parquet',\n    method='sample',\n    sample_size=2000000\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T15:46:37.211026Z","iopub.execute_input":"2026-01-18T15:46:37.212165Z","iopub.status.idle":"2026-01-18T15:47:59.200570Z","shell.execute_reply.started":"2026-01-18T15:46:37.212121Z","shell.execute_reply":"2026-01-18T15:47:59.199295Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\ntrain=pd.read_parquet('/kaggle/working/train_small.parquet')\ntrain.head","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T15:48:44.955644Z","iopub.execute_input":"2026-01-18T15:48:44.956188Z","iopub.status.idle":"2026-01-18T15:48:53.020005Z","shell.execute_reply.started":"2026-01-18T15:48:44.956148Z","shell.execute_reply":"2026-01-18T15:48:53.018662Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"quick_reduce_dataset(\n    '/kaggle/input/aeroclub-recsys-2025/test.parquet', \n    'test_small.parquet',\n    method='sample',\n    sample_size=1000000\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T15:48:58.371779Z","iopub.execute_input":"2026-01-18T15:48:58.372484Z","iopub.status.idle":"2026-01-18T15:49:20.497631Z","shell.execute_reply.started":"2026-01-18T15:48:58.372424Z","shell.execute_reply":"2026-01-18T15:49:20.496286Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test=pd.read_parquet('/kaggle/working/test_small.parquet')\ntest.head","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T15:49:23.190012Z","iopub.execute_input":"2026-01-18T15:49:23.191598Z","iopub.status.idle":"2026-01-18T15:49:29.180385Z","shell.execute_reply.started":"2026-01-18T15:49:23.191457Z","shell.execute_reply":"2026-01-18T15:49:29.179197Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Обработка и выбор признаков","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T17:39:15.196818Z","iopub.execute_input":"2026-01-18T17:39:15.197215Z","iopub.status.idle":"2026-01-18T17:39:15.204191Z","shell.execute_reply.started":"2026-01-18T17:39:15.197186Z","shell.execute_reply":"2026-01-18T17:39:15.202063Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T15:49:34.503097Z","iopub.execute_input":"2026-01-18T15:49:34.503476Z","iopub.status.idle":"2026-01-18T15:49:34.531222Z","shell.execute_reply.started":"2026-01-18T15:49:34.503448Z","shell.execute_reply":"2026-01-18T15:49:34.529902Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T15:49:36.092852Z","iopub.execute_input":"2026-01-18T15:49:36.093240Z","iopub.status.idle":"2026-01-18T15:49:36.107404Z","shell.execute_reply.started":"2026-01-18T15:49:36.093209Z","shell.execute_reply":"2026-01-18T15:49:36.106164Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"group_sizes = train.groupby('ranker_id').size()\ntest_group_sizes = test.groupby('ranker_id').size()\n\nprint(f\"Общее количество групп (ranker_id) в train: {len(group_sizes)}\")\nprint(f\"Общее количество групп (ranker_id) в test: {len(test_group_sizes)}\")\n\nprint(f\"\\nСтатистика размера групп в train:\")\nprint(f\"Мин: {group_sizes.min()}\")\nprint(f\"Макс: {group_sizes.max()}\")\nprint(f\"Медиана: {group_sizes.median()}\")\nprint(f\"Среднее: {group_sizes.mean():.2f}\")\nprint(f\"Стандартное отклонение: {group_sizes.std():.2f}\")\n\nfig, axes = plt.subplots(1, 3, figsize=(15, 5))\n\naxes[0].hist(group_sizes, bins=50, edgecolor='black', alpha=0.7)\naxes[0].axvline(x=10, color='red', linestyle='--', label='Граница для метрики (10)')\naxes[0].set_xlabel('Размер группы')\naxes[0].set_ylabel('Частота')\naxes[0].set_title('Распределение размеров групп (train)')\naxes[0].legend()\naxes[0].grid(True, alpha=0.3)\n\naxes[1].boxplot(group_sizes.values, vert=False)\naxes[1].set_xlabel('Размер группы')\naxes[1].set_title('Боксплот размеров групп (train)')\naxes[1].grid(True, alpha=0.3)\n\naxes[2].hist(np.log1p(group_sizes), bins=50, edgecolor='black', alpha=0.7)\naxes[2].set_xlabel('log(Размер группы + 1)')\naxes[2].set_ylabel('Частота')\naxes[2].set_title('Распределение размеров групп (лог. шкала)')\naxes[2].grid(True, alpha=0.3)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T17:39:17.040320Z","iopub.execute_input":"2026-01-18T17:39:17.040751Z","iopub.status.idle":"2026-01-18T17:39:18.492926Z","shell.execute_reply.started":"2026-01-18T17:39:17.040711Z","shell.execute_reply":"2026-01-18T17:39:18.491182Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Функция обработки признаков датасета**","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T15:49:43.303536Z","iopub.execute_input":"2026-01-18T15:49:43.304524Z","iopub.status.idle":"2026-01-18T15:49:45.373357Z","shell.execute_reply.started":"2026-01-18T15:49:43.304478Z","shell.execute_reply":"2026-01-18T15:49:45.372212Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ndef feature_eng(df):\n    \n    # Изменяем формат временных колонок\n    \n    datetime_cols = ['requestDate', 'legs0_departureAt', 'legs0_arrivalAt', 'legs1_departureAt', 'legs1_arrivalAt']\n    for col in datetime_cols:\n        if col in df.columns:\n            df[col] = pd.to_datetime(df[col], errors='coerce')\n    \n    # Извлекаем день, месяц и час из departureAt\n    \n    if 'legs0_departureAt' in df.columns:\n        df['legs0_departure_hour'] = df['legs0_departureAt'].dt.hour\n        df['legs0_departure_dayofweek'] = df['legs0_departureAt'].dt.dayofweek\n        df['legs0_departure_month'] = df['legs0_departureAt'].dt.month\n    if 'legs1_departureAt' in df.columns:\n        df['legs1_departure_hour'] = df['legs1_departureAt'].dt.hour\n        df['legs1_departure_dayofweek'] = df['legs1_departureAt'].dt.dayofweek\n        df['legs1_departure_month'] = df['legs1_departureAt'].dt.month\n\n\n    # Конвертируем duration в минуты\n    \n    if 'legs0_duration' in df.columns:\n        df['legs0_duration'] = pd.to_numeric(df['legs0_duration'], errors='coerce')\n        df['legs0_duration_min'] = df['legs0_duration'] / 60\n    if 'legs1_duration' in df.columns:\n        df['legs1_duration'] = pd.to_numeric(df['legs1_duration'], errors='coerce')\n        df['legs1_duration_min'] = df['legs1_duration'] / 60\n    \n    \n    # Признаки из сегментов legs \n    \n    segment_cols = [col for col in df.columns if 'segments' in col]\n    \n    # Количество сегментов в legs0 и legs1\n    \n    legs0_segment_cols = [col for col in segment_cols if 'legs0' in col and 'departureFrom' in col]\n    df['legs0_num_segments'] = len(legs0_segment_cols)\n    legs1_segment_cols = [col for col in segment_cols if 'legs1' in col and 'departureFrom' in col]\n    df['legs1_num_segments'] = len(legs1_segment_cols)\n\n    \n    # Заполним признаки относительно мини правил\n    \n    df['miniRules0_monetaryAmount'] = pd.to_numeric(df['miniRules0_monetaryAmount'], errors='coerce')\n    df['miniRules1_monetaryAmount'] = pd.to_numeric(df['miniRules1_monetaryAmount'], errors='coerce')\n    df['has_cancellation_penalty'] = (df['miniRules0_monetaryAmount'] > 0).astype(int)\n    df['has_exchange_penalty'] = (df['miniRules1_monetaryAmount'] > 0).astype(int)\n    df['cancellation_penalty_amount'] = df['miniRules0_monetaryAmount'].fillna(0)\n    df['exchange_penalty_amount'] = df['miniRules1_monetaryAmount'].fillna(0)\n    \n  \n    \n    # Кодирум категориальные переменные\n    \n    cat_cols = ['sex', 'nationality', 'frequentFlyer', 'corporateTariffCode', 'searchRoute']\n    for col in cat_cols:\n        if col in df.columns:\n            le = LabelEncoder()\n            df[col + '_encoded'] = le.fit_transform(df[col].astype(str))\n    \n    # Удалим колонки, которые нам не нужны\n    \n    drop_cols = ['Id', 'profileId', 'companyID', 'requestDate', 'legs0_departureAt', 'legs0_arrivalAt', \n                 'legs1_departureAt', 'legs1_arrivalAt', 'sex', 'nationality', 'frequentFlyer', 'corporateTariffCode', 'searchRoute']\n    drop_cols += segment_cols\n    df = df.drop(columns=[col for col in drop_cols if col in df.columns], errors='ignore')\n    \n    # Заполним пустые значения\n    \n    df = df.fillna(-1)\n    \n    return df\n\ntrain_feature_eng = feature_eng(train.copy())\ntest_feature_eng = feature_eng(test.copy())\n\nprint(\"Размер обработанного train:\", train_feature_eng.shape)\nprint(\"Размер обработанного test:\", test_feature_eng.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T17:11:11.162227Z","iopub.execute_input":"2026-01-18T17:11:11.162804Z","iopub.status.idle":"2026-01-18T17:11:37.046883Z","shell.execute_reply.started":"2026-01-18T17:11:11.162766Z","shell.execute_reply":"2026-01-18T17:11:37.045775Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Обучение модели","metadata":{}},{"cell_type":"markdown","source":"Для выполнения задачи была выбрана модель CatBoost","metadata":{}},{"cell_type":"code","source":"import catboost as cb\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\n\n# Подготовим данные\n\ntrain_df = train_feature_eng.copy()\ntrain_df = train_df.sort_values('ranker_id').reset_index(drop=True)\ntrain_df['group_id'] = train_df.groupby('ranker_id').ngroup()\n\n# Разделение на train/val по группам\ngroup_ids = train_df['group_id'].unique()\ntrain_group_ids, val_group_ids = train_test_split(group_ids, test_size=0.2, random_state=42)\n\ntrain_data = train_df[train_df['group_id'].isin(train_group_ids)]\nval_data = train_df[train_df['group_id'].isin(val_group_ids)]\n\nfeatures = [col for col in train_data.columns if col not in ['ranker_id', 'selected', 'group_id']]\n\n# Обучаем модель\nmodel = cb.CatBoost({\n    'loss_function': 'PairLogit',\n    'iterations': 300,\n    'learning_rate': 0.05,\n    'depth': 6,\n    'verbose': 100,\n    'random_seed': 42,\n})\n\nmodel.fit(\n    train_data[features],\n    train_data['selected'],\n    group_id=train_data['group_id'],\n    verbose=100\n)\n\n# Предсказание\nval_pred = model.predict(val_data[features])\n\n# Расчет HitScore@3\ndef hitscore_at_3(y_true, y_pred, group_col):\n    hits = 0\n    total_groups = len(group_col.unique())\n    \n    for group_id in group_col.unique():\n        group_mask = (group_col == group_id)\n        group_pred = y_pred[group_mask]\n        group_true = y_true[group_mask]\n        \n        # Топ-3 предсказания\n        top_3_idx = np.argsort(group_pred)[-3:][::-1]\n        \n        if any(group_true[idx] == 1 for idx in top_3_idx):\n            hits += 1\n    \n    return hits / total_groups\n\n# Результат\nscore = hitscore_at_3(val_data['selected'].values, val_pred, val_data['group_id'])\nprint(f\"\\nHitScore@3: {score:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T17:13:22.904501Z","iopub.execute_input":"2026-01-18T17:13:22.905748Z","iopub.status.idle":"2026-01-18T17:19:23.089171Z","shell.execute_reply.started":"2026-01-18T17:13:22.905647Z","shell.execute_reply":"2026-01-18T17:19:23.087514Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test = test_feature_eng.drop(columns=['selected', 'ranker_id'] if 'selected' in test_feature_eng.columns else ['ranker_id'], errors='ignore')\ntest_pred = model.predict(X_test)\n\nsubmission = test[['Id', 'ranker_id']].copy()\nsubmission['prediction'] = test_pred\n\nsubmission['selected'] = submission.groupby('ranker_id')['prediction'].rank(\n    ascending=False, method='first'\n).astype(int)\n\nsubmission['rank'] = submission.groupby('ranker_id')['prediction'].rank(ascending=False, method='first').astype(int)\nsubmission = submission[['Id', 'ranker_id', 'rank']].rename(columns={'rank': 'selected'})\n\nsubmission[['Id', 'ranker_id', 'selected']].to_csv('submission.csv', index=False)\nprint(f\"Submission saved. Shape: {submission.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T17:56:36.012705Z","iopub.execute_input":"2026-01-18T17:56:36.014165Z","iopub.status.idle":"2026-01-18T17:56:39.944467Z","shell.execute_reply.started":"2026-01-18T17:56:36.014117Z","shell.execute_reply":"2026-01-18T17:56:39.942840Z"}},"outputs":[],"execution_count":null}]}