{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":105399,"databundleVersionId":12733338,"sourceType":"competition"}],"dockerImageVersionId":31260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%pip install polars","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:17:43.843201Z","iopub.execute_input":"2026-01-18T18:17:43.843523Z","iopub.status.idle":"2026-01-18T18:17:47.746721Z","shell.execute_reply.started":"2026-01-18T18:17:43.843497Z","shell.execute_reply":"2026-01-18T18:17:47.745887Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Загружаем данные в Polars (для Pandas их оказалось слишком много)","metadata":{}},{"cell_type":"markdown","source":"Допустимо использовать Pandas или Polars для загрузки и обработки данных.","metadata":{}},{"cell_type":"markdown","source":"Загружаю не все данные, так как больше не позволяет объем RAM (всего в train ~18 млн строк, беру 10 млн - для сравнения в test ~7 млн)","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\nimport polars as pl\nimport pandas as pd\nimport numpy as np\n\ntrain = pl.read_parquet(r'/kaggle/input/aeroclub-recsys-2025/train.parquet', n_rows=10_000_000)\ntest = pl.read_parquet(r'/kaggle/input/aeroclub-recsys-2025/test.parquet')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:17:47.748449Z","iopub.execute_input":"2026-01-18T18:17:47.748704Z","iopub.status.idle":"2026-01-18T18:17:57.911701Z","shell.execute_reply.started":"2026-01-18T18:17:47.748676Z","shell.execute_reply":"2026-01-18T18:17:57.911102Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:17:57.912620Z","iopub.execute_input":"2026-01-18T18:17:57.913083Z","iopub.status.idle":"2026-01-18T18:17:57.935386Z","shell.execute_reply.started":"2026-01-18T18:17:57.913048Z","shell.execute_reply":"2026-01-18T18:17:57.934861Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:17:57.936826Z","iopub.execute_input":"2026-01-18T18:17:57.937110Z","iopub.status.idle":"2026-01-18T18:17:57.944525Z","shell.execute_reply.started":"2026-01-18T18:17:57.937092Z","shell.execute_reply":"2026-01-18T18:17:57.943838Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Размер train: {train.shape}\")\nprint(f\"Размер test: {test.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:17:57.945518Z","iopub.execute_input":"2026-01-18T18:17:57.945883Z","iopub.status.idle":"2026-01-18T18:17:57.957093Z","shell.execute_reply.started":"2026-01-18T18:17:57.945858Z","shell.execute_reply":"2026-01-18T18:17:57.956450Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Разница в одну колонку - это целевой selected","metadata":{}},{"cell_type":"markdown","source":"# Обработка признаков (4 баллов) и Выбор признаков (4 баллов)","metadata":{}},{"cell_type":"markdown","source":"Решил эти пункты делать вместе, так как данных много и какие-то колонки сразу анализирую и удаляю, а какие-то анализирую и на их основе вывожу другие признаки. Какие-то просто анализирую и оставляю","metadata":{}},{"cell_type":"markdown","source":"Необходимо выполните предварительный анализ и обработку признаков. Создайте новые или удалите \"не нужные\" признаки по вашему мнению.  ","metadata":{}},{"cell_type":"markdown","source":"### Анализ целевой переменной (колонка selected)","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nfrom scipy import stats\nfrom datetime import datetime\n\n\nplt.figure(figsize=(10, 6))\ntarget_pd = train.select('selected').to_pandas()\nax = sns.countplot(data=target_pd, x='selected')\nplt.title('Распределение целевой переменной \"selected\"')\nplt.xlabel('Выбран ли билет (0 - нет, 1 - да)')\nplt.ylabel('Количество')\nplt.tight_layout()\nplt.show()\n\nprint(target_pd.isna().sum())\nprint(target_pd.value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:17:57.957915Z","iopub.execute_input":"2026-01-18T18:17:57.958141Z","iopub.status.idle":"2026-01-18T18:18:13.825096Z","shell.execute_reply.started":"2026-01-18T18:17:57.958119Z","shell.execute_reply":"2026-01-18T18:18:13.824303Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Есть сильный дисбаланс классов, но это объяснимо данными - в рамках одной сессии пользователь выбирает 1 полет из всего перечня.","metadata":{}},{"cell_type":"markdown","source":"### Анализ пропущенных значений","metadata":{}},{"cell_type":"code","source":"def analyze_missing_values(df: pl.DataFrame, df_name: str = \"train\"):\n    print(f\"Анализ пропусков в {df_name}\")\n    \n    missing_info = []\n\n    # для разных типов данных по-разному собираются пропуски, поэтому делаю сумму по всем null / nan / empty для понимания\n    for col in df.columns:\n        null_count = df.select(pl.col(col).is_null().sum()).item()\n        \n        nan_count = 0\n        if df[col].dtype in [pl.Float32, pl.Float64]:\n            nan_count = df.select(pl.col(col).is_nan().sum()).item()\n        \n        empty_str_count = 0\n        if df[col].dtype == pl.Utf8:\n            empty_str_count = df.select(\n                (pl.col(col).str.strip_chars().str.len_chars() == 0).sum()\n            ).item()\n        \n        total_missing = null_count + nan_count + empty_str_count\n        missing_percent = (total_missing / len(df)) * 100 if len(df) > 0 else 0\n        \n        missing_info.append({\n            'feature': col,\n            'dtype': str(df[col].dtype),\n            'null_count': null_count,\n            'nan_count': nan_count,\n            'empty_str_count': empty_str_count,\n            'total_missing': total_missing,\n            'missing_percent': missing_percent\n        })\n    \n    summary_df = pl.DataFrame(missing_info)\n    summary_df = summary_df.sort('missing_percent', descending=True)\n    \n    print(\"\\nОБЩАЯ СТАТИСТИКА ПРОПУСКОВ:\")\n    total_cols = len(summary_df)\n    no_missing = summary_df.filter(pl.col('total_missing') == 0).shape[0]\n    some_missing = summary_df.filter(pl.col('total_missing') > 0).shape[0]\n    high_missing = summary_df.filter(pl.col('missing_percent') > 60).shape[0]\n    full_missing = summary_df.filter(pl.col('missing_percent') > 99).shape[0]\n    \n    print(f\"Всего колонок: {total_cols}\")\n    print(f\"Без пропусков: {no_missing}\")\n    print(f\"С пропусками: {some_missing}\")\n    print(f\"С >70% пропусков: {high_missing}\")\n    print(f\"С >99% пропусков: {full_missing}\")\n    \n    avg_missing = summary_df.filter(pl.col('total_missing') > 0)['missing_percent'].mean()\n    max_missing = summary_df['missing_percent'].max()\n    print(f\"Средний % пропусков (среди колонок с пропусками): {avg_missing:.1f}%\")\n    print()\n    \n    # считаю признак как признак для удаления, если у него >60% пропусков\n    high_missing_cols = summary_df.filter(pl.col('missing_percent') > 60)['feature'].to_list()\n    return summary_df, high_missing_cols\n\ntrain_missing_summary, train_high_missing = analyze_missing_values(train, \"train\")\ntest_missing_summary, test_high_missing = analyze_missing_values(test, \"test\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:18:13.826240Z","iopub.execute_input":"2026-01-18T18:18:13.826639Z","iopub.status.idle":"2026-01-18T18:18:41.790755Z","shell.execute_reply.started":"2026-01-18T18:18:13.826616Z","shell.execute_reply":"2026-01-18T18:18:41.790134Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Как мы видим, что в train, что в test, есть колонки с полностью пропущенными значениями. Также есть колонки с высоким процентом пропусков среди всех значений, причем количественная статистика среди таких колонок по train и test сильно похожа, поэтому ниже еще сравню совпадения колонок с пропусками между train и test. Также очевидно, что большинство таких признаков подлежит удалению в дальнейшем, но сначала визуализируем их","metadata":{}},{"cell_type":"code","source":"train_features = set(train_missing_summary['feature'].to_list())\ntest_features = set(test_missing_summary['feature'].to_list())\n\ncommon_features = sorted(list(train_features.intersection(test_features)))\nprint(f\"Общих признаков: {len(common_features)}\")\n\nif common_features:\n    comparison_data = []\n    \n    for feature in common_features:\n        train_percent = train_missing_summary.filter(pl.col('feature') == feature)['missing_percent'].item()\n        test_percent = test_missing_summary.filter(pl.col('feature') == feature)['missing_percent'].item()\n        percent_diff = abs(train_percent - test_percent)\n        \n        comparison_data.append({\n            'feature': feature,\n            'train_missing_%': round(train_percent, 2),\n            'test_missing_%': round(test_percent, 2),\n            'diff_%': round(percent_diff, 2),\n            'train_dtype': train_missing_summary.filter(pl.col('feature') == feature)['dtype'].item(),\n            'test_dtype': test_missing_summary.filter(pl.col('feature') == feature)['dtype'].item()\n        })\n    \n    comparison_df = pl.DataFrame(comparison_data).sort('diff_%', descending=True)\n    \n    print(f\"\\nСравнение {len(common_features)} общих признаков:\")\n    print(f\"Средняя разница в % пропусков: {comparison_df['diff_%'].mean():.6f}%\")\n    print(f\"Максимальная разница: {comparison_df['diff_%'].max():.6f}%\")\n    print(f\"Минимальная разница: {comparison_df['diff_%'].min():.6f}%\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:18:41.791721Z","iopub.execute_input":"2026-01-18T18:18:41.792031Z","iopub.status.idle":"2026-01-18T18:18:41.950875Z","shell.execute_reply.started":"2026-01-18T18:18:41.792010Z","shell.execute_reply":"2026-01-18T18:18:41.950116Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(14, 8))\ntop_100 = train_missing_summary.head(100).to_pandas()\nbars = plt.barh(top_100['feature'][::-1], top_100['missing_percent'][::-1], \n                color=['red' if p > 60 else 'blue' for p in top_100['missing_percent'][::-1]])\nplt.xlabel('Процент пропусков (%)', fontsize=12)\nplt.title('Топ-100 признаков с наибольшим процентом пропусков', fontsize=14, pad=20)\nplt.axvline(x=60, color='red', linestyle='--', alpha=0.7, label='60% порог удаления')\nplt.legend()\nplt.grid(True, alpha=0.3, axis='x')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:18:41.951917Z","iopub.execute_input":"2026-01-18T18:18:41.952850Z","iopub.status.idle":"2026-01-18T18:18:43.149936Z","shell.execute_reply.started":"2026-01-18T18:18:41.952817Z","shell.execute_reply":"2026-01-18T18:18:43.149074Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(14, 8))\ntop_100 = test_missing_summary.head(100).to_pandas()\nbars = plt.barh(top_100['feature'][::-1], top_100['missing_percent'][::-1], \n                color=['red' if p > 60 else 'blue' for p in top_100['missing_percent'][::-1]])\nplt.xlabel('Процент пропусков (%)', fontsize=12)\nplt.title('Топ-100 признаков с наибольшим процентом пропусков', fontsize=14, pad=20)\nplt.axvline(x=60, color='red', linestyle='--', alpha=0.7, label='60% порог удаления')\nplt.legend()\nplt.grid(True, alpha=0.3, axis='x')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:18:43.152234Z","iopub.execute_input":"2026-01-18T18:18:43.152532Z","iopub.status.idle":"2026-01-18T18:18:44.060787Z","shell.execute_reply.started":"2026-01-18T18:18:43.152510Z","shell.execute_reply":"2026-01-18T18:18:44.060031Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Вывел для примера уникальные значения по одной колонке","metadata":{}},{"cell_type":"code","source":"train['legs1_segments3_aircraft_code'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:18:44.061942Z","iopub.execute_input":"2026-01-18T18:18:44.062380Z","iopub.status.idle":"2026-01-18T18:18:44.536582Z","shell.execute_reply.started":"2026-01-18T18:18:44.062355Z","shell.execute_reply":"2026-01-18T18:18:44.535995Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"И конечная круговая диаграмма с распределением признаков. По ней видно, что удалению подлежит большая часть признаков - чтож, придется удалять.","metadata":{}},{"cell_type":"code","source":"categories = {\n    '0%': train_missing_summary.filter(pl.col('missing_percent') == 0).shape[0],\n    '0-10%': train_missing_summary.filter((pl.col('missing_percent') > 0) & (pl.col('missing_percent') <= 10)).shape[0],\n    '10-60%': train_missing_summary.filter((pl.col('missing_percent') > 10) & (pl.col('missing_percent') <= 60)).shape[0],\n    '>60%': train_missing_summary.filter(pl.col('missing_percent') > 60).shape[0]\n}\n\nplt.figure(figsize=(10, 6))\ncolors = ['green', 'lightblue', 'orange', 'red']\nplt.pie(categories.values(), labels=categories.keys(), autopct='%1.1f%%', \n        colors=colors, startangle=90, explode=(0.05, 0.05, 0.05, 0.1),\n        textprops={'fontsize': 11})\nplt.title('Распределение признаков по проценту пропусков', fontsize=14, pad=20)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:18:44.537495Z","iopub.execute_input":"2026-01-18T18:18:44.537736Z","iopub.status.idle":"2026-01-18T18:18:44.624264Z","shell.execute_reply.started":"2026-01-18T18:18:44.537715Z","shell.execute_reply":"2026-01-18T18:18:44.623543Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Удаление признаков в тестовых данных правильнее было бы производить на основе train данных, так как test модель не должна видеть, поэтому так и делаю.","metadata":{}},{"cell_type":"code","source":"print(f\"Найдено признаков для удаления: {len(train_high_missing)}\")\ntrain = train.drop(train_high_missing)\ntest = test.drop(train_high_missing)\nprint(f\"Удалено {len(train_high_missing)} признаков из тренировочных и тестовых данных\")\nprint(f\"Новый размер train: {train.shape}\")\nprint(f\"Новый размер test: {test.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:18:44.625337Z","iopub.execute_input":"2026-01-18T18:18:44.625879Z","iopub.status.idle":"2026-01-18T18:18:45.408495Z","shell.execute_reply.started":"2026-01-18T18:18:44.625850Z","shell.execute_reply":"2026-01-18T18:18:45.407851Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Теперь еще раз посмотрим статистику по пропускам","metadata":{}},{"cell_type":"code","source":"train_missing_summary, train_high_missing = analyze_missing_values(train, \"train\")\n\nplt.figure(figsize=(14, 8))\ntop_missing_for_plot = train_missing_summary.to_pandas()\nbars = plt.barh(top_missing_for_plot['feature'][::-1], top_missing_for_plot['missing_percent'][::-1], \n                color='blue')\nplt.xlabel('Процент пропусков (%)', fontsize=12)\nplt.title('Топ признаков с наибольшим процентом пропусков', fontsize=14, pad=20)\nplt.legend()\nplt.grid(True, alpha=0.3, axis='x')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:18:45.409292Z","iopub.execute_input":"2026-01-18T18:18:45.409505Z","iopub.status.idle":"2026-01-18T18:18:54.091349Z","shell.execute_reply.started":"2026-01-18T18:18:45.409486Z","shell.execute_reply":"2026-01-18T18:18:54.090585Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Так гораздо лучше, но посмотрим на уникальные значения по corporateTariffCode","metadata":{}},{"cell_type":"code","source":"train['corporateTariffCode'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:18:54.092248Z","iopub.execute_input":"2026-01-18T18:18:54.092478Z","iopub.status.idle":"2026-01-18T18:18:54.325122Z","shell.execute_reply.started":"2026-01-18T18:18:54.092458Z","shell.execute_reply":"2026-01-18T18:18:54.324376Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"В описании данных написано \"corporateTariffCode - Corporate tariff code for business travel policies\", так что можно предположить, что такое количество пропусков обусловлено предметной областью.","metadata":{}},{"cell_type":"code","source":"train_missing_summary.filter(pl.col('missing_percent') > 0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:18:54.326072Z","iopub.execute_input":"2026-01-18T18:18:54.326359Z","iopub.status.idle":"2026-01-18T18:18:54.333186Z","shell.execute_reply.started":"2026-01-18T18:18:54.326337Z","shell.execute_reply":"2026-01-18T18:18:54.332442Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Перейдем к временным признакам","metadata":{}},{"cell_type":"code","source":"existing_temporal_cols = [col for col in train.columns \n                 if 'departure' in col.lower() or \n                 'arrival' in col.lower() or\n                 'duration' in col.lower() or \n                 'date' in col.lower() or\n                 'time' in col.lower()]\n\nprint(f\"Найдено {len(existing_temporal_cols)} временных признаков:\")\nprint(existing_temporal_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:18:54.334073Z","iopub.execute_input":"2026-01-18T18:18:54.334290Z","iopub.status.idle":"2026-01-18T18:18:54.346194Z","shell.execute_reply.started":"2026-01-18T18:18:54.334272Z","shell.execute_reply":"2026-01-18T18:18:54.345501Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Из описания данных в самом соревновании можно вывести важные колонки, которые перечислены ниже.","metadata":{}},{"cell_type":"code","source":"important_date_columns = ['legs0_departureAt', 'legs0_arrivalAt', 'legs0_duration', \n                 'legs1_departureAt', 'legs1_arrivalAt', 'legs1_duration',\n                 'requestDate']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:18:54.347035Z","iopub.execute_input":"2026-01-18T18:18:54.347719Z","iopub.status.idle":"2026-01-18T18:18:54.359600Z","shell.execute_reply.started":"2026-01-18T18:18:54.347681Z","shell.execute_reply":"2026-01-18T18:18:54.358995Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"На их основе создам другие признаки, а остальные колонки уберу (в том числе эти после создание новых фич). Важно учесть, что длительность полета трогать не нужно, так как в этой колонке сложно учесть разницу часовых поясов - вылет и прилет пишется по часовому поясу аэропорта соответственно.","metadata":{}},{"cell_type":"code","source":"def parse_time_string(time_str):    \n    time_str = str(time_str).strip()\n    \n    if '.' in time_str:\n        try:\n            day_part, time_part = time_str.split('.', 1)\n            days = int(day_part)\n            \n            time_parts = time_part.split(':')\n            if len(time_parts) >= 2:\n                hours = int(time_parts[0]) if time_parts[0] else 0\n                minutes = int(time_parts[1]) if time_parts[1] else 0\n                seconds = int(time_parts[2]) if len(time_parts) > 2 and time_parts[2] else 0\n                \n                total_hours = days * 24 + hours + minutes/60 + seconds/3600\n                return int(total_hours)\n        except:\n            pass\n    \n    try:\n        time_parts = time_str.split(':')\n        if len(time_parts) >= 2:\n            hours = int(time_parts[0]) if time_parts[0] else 0\n            minutes = int(time_parts[1]) if time_parts[1] else 0\n            seconds = int(time_parts[2]) if len(time_parts) > 2 and time_parts[2] else 0\n            \n            total_hours = hours + minutes/60 + seconds/3600\n            return int(total_hours)\n    except:\n        pass\n    \n    return 0\n\ndef safe_strptime(col_expr, format_str):\n    try:\n        return col_expr.str.strptime(pl.Datetime, format=format_str)\n    except:\n        return col_expr\n\ndef process_datetime_columns(df):\n    df = df.clone()\n    \n    if 'legs0_departureAt' in df.columns:\n        if df['legs0_departureAt'].dtype == pl.Utf8:\n            df = df.with_columns([\n                safe_strptime(pl.col('legs0_departureAt'), \"%Y-%m-%dT%H:%M:%S\").alias('departure_dt_0'),\n                safe_strptime(pl.col('legs0_arrivalAt'), \"%Y-%m-%dT%H:%M:%S\").alias('arrival_dt_0'),\n            ])\n        else:\n            df = df.with_columns([\n                pl.col('legs0_departureAt').alias('departure_dt_0'),\n                pl.col('legs0_arrivalAt').alias('arrival_dt_0'),\n            ])\n    \n    if 'legs1_departureAt' in df.columns:\n        if df['legs1_departureAt'].dtype == pl.Utf8:\n            df = df.with_columns([\n                safe_strptime(pl.col('legs1_departureAt'), \"%Y-%m-%dT%H:%M:%S\").alias('departure_dt_1'),\n                safe_strptime(pl.col('legs1_arrivalAt'), \"%Y-%m-%dT%H:%M:%S\").alias('arrival_dt_1'),\n            ])\n        else:\n            df = df.with_columns([\n                pl.col('legs1_departureAt').alias('departure_dt_1'),\n                pl.col('legs1_arrivalAt').alias('arrival_dt_1'),\n            ])\n    \n    for leg in ['0', '1']:\n        col_name = f'legs{leg}_duration'\n        if col_name in df.columns:\n            if df[col_name].dtype == pl.Utf8:\n                    df = df.with_columns(\n                        pl.col(col_name).map_elements(\n                            lambda x: parse_time_string(x),\n                            return_dtype=pl.Int64\n                        )\n                        .alias(f'duration_hours_{leg}')\n                    )\n    \n    if 'requestDate' in df.columns:\n        if df['requestDate'].dtype == pl.Utf8:\n            df = df.with_columns(\n                safe_strptime(pl.col('requestDate'), \"%Y-%m-%d %H:%M:%S\").alias('request_dt')\n            )\n        else:\n            df = df.with_columns(pl.col('requestDate').alias('request_dt'))\n    \n    if 'departure_dt_0' in df.columns and 'arrival_dt_0' in df.columns:\n        df = df.with_columns([\n            pl.col('departure_dt_0').dt.hour().alias('departure_hour_0'),\n            pl.col('departure_dt_0').dt.weekday().alias('departure_weekday_0'),\n            pl.col('departure_dt_0').dt.month().alias('departure_month_0'),\n            pl.col('departure_dt_0').dt.day().alias('departure_day_0'),\n        ])\n    \n    if 'departure_dt_1' in df.columns and 'arrival_dt_1' in df.columns:\n        df = df.with_columns([\n            pl.col('departure_dt_1').dt.hour().alias('departure_hour_1'),\n            pl.col('departure_dt_1').dt.weekday().alias('departure_weekday_1'),\n            pl.col('departure_dt_1').dt.month().alias('departure_month_1'),\n            pl.col('departure_dt_1').dt.day().alias('departure_day_1'),\n        ])\n    \n    if 'request_dt' in df.columns:\n        df = df.with_columns([\n            pl.col('request_dt').dt.hour().alias('request_hour'),\n            pl.col('request_dt').dt.weekday().alias('request_weekday'),\n            pl.col('request_dt').dt.month().alias('request_month'),\n            pl.col('request_dt').dt.day().alias('request_day'),\n        ])\n    \n    return df\n\n\nprint(\"\\nОбработка временных признаков...\")\ntrain = process_datetime_columns(train)\ntest = process_datetime_columns(test)\n\nprint(f\"\\nТипы данных после обработки:\")\nnew_temporal_features = [col for col in train.columns \n                         if any(x in col for x in ['request_', 'departure_', 'duration_hours_', 'arrival_']) and all(x not in col for x in ['departure_dt_0', 'departure_dt_1', 'arrival_dt_0', 'arrival_dt_1', 'request_dt'])]\n\nfor col in new_temporal_features:\n    print(f\"{col}: {train[col].dtype}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:18:54.360388Z","iopub.execute_input":"2026-01-18T18:18:54.360951Z","iopub.status.idle":"2026-01-18T18:19:26.137522Z","shell.execute_reply.started":"2026-01-18T18:18:54.360930Z","shell.execute_reply":"2026-01-18T18:19:26.136873Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Размер train: {train.shape}\")\nprint(f\"Размер test: {test.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:19:26.138672Z","iopub.execute_input":"2026-01-18T18:19:26.139038Z","iopub.status.idle":"2026-01-18T18:19:26.143102Z","shell.execute_reply.started":"2026-01-18T18:19:26.138994Z","shell.execute_reply":"2026-01-18T18:19:26.142330Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.drop(existing_temporal_cols)\ntrain = train.drop(['departure_dt_0', 'departure_dt_1', 'arrival_dt_0', 'arrival_dt_1', 'request_dt'])\ntest = test.drop(existing_temporal_cols)\ntest = test.drop(['departure_dt_0', 'departure_dt_1', 'arrival_dt_0', 'arrival_dt_1', 'request_dt'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:19:26.143897Z","iopub.execute_input":"2026-01-18T18:19:26.144087Z","iopub.status.idle":"2026-01-18T18:19:26.399229Z","shell.execute_reply.started":"2026-01-18T18:19:26.144070Z","shell.execute_reply":"2026-01-18T18:19:26.398545Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Размер train: {train.shape}\")\nprint(f\"Размер test: {test.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:19:26.400111Z","iopub.execute_input":"2026-01-18T18:19:26.400395Z","iopub.status.idle":"2026-01-18T18:19:26.404511Z","shell.execute_reply.started":"2026-01-18T18:19:26.400366Z","shell.execute_reply":"2026-01-18T18:19:26.403961Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Рассматривал создание разных признаков типа какое время суток (утро, день, вечер) или запоминание конкретной даты, но пришел к выводы об удалении таких признаков и достаточности тех, что оставил.","metadata":{}},{"cell_type":"markdown","source":"Также на этом моменте заметил признак pricingInfo_passengerCount, у которого все значения в train и test были абсолютно одинаковыми, поэтому решил проверить все признаки на это.","metadata":{}},{"cell_type":"code","source":"print('Уникальные значения в train')\nresult = train.select(pl.all().n_unique())\nfor col in result.columns:\n    if result[col][0] > 1:\n        continue\n    print(f\"{col}: {result[col][0]}\")\n\nprint('\\nУникальные значения в test')\nresult = test.select(pl.all().n_unique())\nfor col in result.columns:\n    if result[col][0] > 1:\n        continue\n    print(f\"{col}: {result[col][0]}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:19:26.405344Z","iopub.execute_input":"2026-01-18T18:19:26.405736Z","iopub.status.idle":"2026-01-18T18:19:30.863214Z","shell.execute_reply.started":"2026-01-18T18:19:26.405715Z","shell.execute_reply":"2026-01-18T18:19:30.862364Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Как можно увидеть, как раз признак pricingInfo_passengerCount имеет единственное уникальное значение. Также в train признак bySelf имеет единственное значение. Сам признак имеет описание \"bySelf - Whether user books flights independently\", и в test есть записи с разными значениями для него. Считаю, что учитывать это в обучении некорректно, так как оно будет мешать при инференсе на тестовых данных, поэтому удалю обе колонки в train и test","metadata":{}},{"cell_type":"code","source":"train = train.drop(['bySelf', 'pricingInfo_passengerCount'])\ntest = test.drop(['bySelf', 'pricingInfo_passengerCount'])\n\nprint(f\"Размер train: {train.shape}\")\nprint(f\"Размер test: {test.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:19:30.864109Z","iopub.execute_input":"2026-01-18T18:19:30.864341Z","iopub.status.idle":"2026-01-18T18:19:30.876358Z","shell.execute_reply.started":"2026-01-18T18:19:30.864321Z","shell.execute_reply":"2026-01-18T18:19:30.875739Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Далее провожу корреляционный анализ, чтобы еще уменьшить размерность признаков","metadata":{}},{"cell_type":"code","source":"corr_cols = [col for col in train.columns \n                     if train[col].dtype in [pl.Float32, pl.Float64, pl.Int32, pl.Int64]\n                     and col not in ['Id', 'profileId']]\nlen(corr_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:19:30.877151Z","iopub.execute_input":"2026-01-18T18:19:30.877343Z","iopub.status.idle":"2026-01-18T18:19:30.890481Z","shell.execute_reply.started":"2026-01-18T18:19:30.877325Z","shell.execute_reply":"2026-01-18T18:19:30.889763Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"corr_matrix = train.select(corr_cols).to_pandas().corr()\n\nplt.figure(figsize=(14, 12))\nmask = np.triu(np.ones_like(corr_matrix, dtype=bool))\nsns.heatmap(corr_matrix, mask=mask, annot=True, fmt='.2f', cmap='coolwarm',\n            center=0, square=True, linewidths=.5, cbar_kws={\"shrink\": .8})\nplt.title('Корреляционная матрица числовых признаков', fontsize=16)\nplt.tight_layout()\nplt.show()\n\nprint(\"\\nТоп-10 признаков по абсолютной корреляции с 'selected':\")\ntarget_corr = corr_matrix['selected'].drop('selected').abs().sort_values(ascending=False)\nprint(target_corr.head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:19:30.891991Z","iopub.execute_input":"2026-01-18T18:19:30.892256Z","iopub.status.idle":"2026-01-18T18:19:44.419612Z","shell.execute_reply.started":"2026-01-18T18:19:30.892233Z","shell.execute_reply":"2026-01-18T18:19:44.418890Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Корреляция с целевым признаком не высокая, но это можно было предположить. Но были обнаружены признаки, коррелирующие между собой - следовательно, можно рассмотреть их попарно и удалить тот, что коррелирует меньше с целевым признаком.","metadata":{}},{"cell_type":"code","source":"high_corr_pairs = []\nfor i in range(len(corr_matrix.columns)):\n    for j in range(i+1, len(corr_matrix.columns)):\n        if abs(corr_matrix.iloc[i, j]) > 0.8:\n            high_corr_pairs.append((\n                corr_matrix.columns[i], \n                corr_matrix.columns[j], \n                corr_matrix.iloc[i, j]\n            ))\n\nif high_corr_pairs:\n    print(f\"\\nНайдено высококоррелированных пар (>0.8): {len(high_corr_pairs)}. Выполняется удаление одного признака из них\")\n    \n    all_high_corr_features = set()\n    for pair in high_corr_pairs:\n        all_high_corr_features.add(pair[0])\n        all_high_corr_features.add(pair[1])\n\n    features_to_remove = set()\n    for feature1, feature2, corr_value in high_corr_pairs:\n        if feature1 in corr_matrix.columns and feature2 in corr_matrix.columns:\n            corr_with_target_1 = abs(corr_matrix.loc[feature1, 'selected']) if 'selected' in corr_matrix.columns else 0\n            corr_with_target_2 = abs(corr_matrix.loc[feature2, 'selected']) if 'selected' in corr_matrix.columns else 0\n            \n            # удаляется признак с меньшей корреляцией с целевой переменной\n            if corr_with_target_1 >= corr_with_target_2:\n                features_to_remove.add(feature2)\n            else:\n                features_to_remove.add(feature1)\n\n    if 'selected' in features_to_remove:\n        features_to_remove.remove('selected')\n    \n    print(f\"Признаков для удаления: {len(features_to_remove)}\")\n    if features_to_remove:\n        print(\"Признаки, которые будут удалены:\")\n        print(features_to_remove)\n        \n        train = train.drop(list(features_to_remove))\n        features_to_remove_test = [f for f in features_to_remove if f in test.columns]\n        if features_to_remove_test:\n            test = test.drop(features_to_remove_test)\n            print(f\"Удалено {len(features_to_remove_test)} признаков из test\")\n        \n        print(f\"После удаления высококоррелированных признаков:\")\n        print(f\"  Размер train: {train.shape}\")\n        print(f\"  Размер test: {test.shape}\")\n    else:\n        print(\"Нет признаков для удаления\")\nelse:\n    print(\"\\nВысококоррелированных признаков для удаления не найдено\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:19:44.420555Z","iopub.execute_input":"2026-01-18T18:19:44.420775Z","iopub.status.idle":"2026-01-18T18:19:44.461925Z","shell.execute_reply.started":"2026-01-18T18:19:44.420755Z","shell.execute_reply":"2026-01-18T18:19:44.461339Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Обучение модели (8 баллов)","metadata":{}},{"cell_type":"markdown","source":"На этом этапе допускается выполнить выбор модели, гиперпараметров, после чего необходимо обучить модель и выполнить предсказания на тестовых данных (необязательно из файла test). \n","metadata":{}},{"cell_type":"code","source":"%pip install xgboost","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:19:44.464549Z","iopub.execute_input":"2026-01-18T18:19:44.465149Z","iopub.status.idle":"2026-01-18T18:19:47.787724Z","shell.execute_reply.started":"2026-01-18T18:19:44.465123Z","shell.execute_reply":"2026-01-18T18:19:47.786951Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Для согласованности заполняю train/test вместе кодом ниже","metadata":{}},{"cell_type":"code","source":"import xgboost as xgb\nfrom typing import List, Dict\n\ndef fill_missing_values(df: pl.DataFrame) -> pl.DataFrame:\n    numer_cols = [col for col in df.columns if df[col].dtype in [pl.Float32, pl.Float64, pl.Int32, pl.Int64]]\n    cat_cols = [col for col in df.columns if df[col].dtype == pl.Utf8]\n    \n    fill_exprs = []\n    \n    for col in numer_cols:\n        fill_exprs.append(pl.col(col).fill_null(-1))\n    \n    for col in cat_cols:\n        fill_exprs.append(pl.col(col).fill_null(\"MISSING\"))\n    \n    return df.with_columns(fill_exprs)\n\ndef prepare_xgb_data(train: pl.DataFrame, test: pl.DataFrame, target_col: str = 'selected'):\n    train_filled = fill_missing_values(train)\n    test_filled = fill_missing_values(test)\n    \n    ignore_cols = ['Id', 'ranker_id', target_col, '__index_level_0__']\n    feature_cols = [col for col in train_filled.columns if col not in ignore_cols]\n    \n    numeric_cols = []\n    categorical_cols = []\n    \n    for col in feature_cols:\n        if col in train_filled.columns:\n            dtype = train_filled[col].dtype\n            if dtype in [pl.Utf8]:\n                categorical_cols.append(col)\n            else:\n                numeric_cols.append(col)\n    \n    all_dfs = []\n    combined = pl.concat([train_filled.select(feature_cols), test_filled.select(feature_cols)], how='vertical')\n    \n    for col in categorical_cols:\n        if col in combined.columns:\n            unique_vals = combined[col].unique().to_list()\n            val_to_idx = {val: i for i, val in enumerate(unique_vals)}\n            \n            train_filled = train_filled.with_columns(\n                pl.col(col).replace(val_to_idx, default=-1).alias(col)\n            )\n            test_filled = test_filled.with_columns(\n                pl.col(col).replace(val_to_idx, default=-1).alias(col)\n            )\n    \n    X_train = train_filled.select(feature_cols).to_numpy()\n    y_train = train_filled[target_col].to_numpy()\n    \n    group_counts = train_filled.group_by('ranker_id').agg(pl.len().alias('count')).sort('ranker_id')\n    groups = group_counts['count'].to_numpy()\n    \n    X_test = test_filled.select(feature_cols).to_numpy()\n    \n    return X_train, y_train, groups, X_test, test_filled['Id'].to_numpy(), test_filled['ranker_id'].to_numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:19:47.789177Z","iopub.execute_input":"2026-01-18T18:19:47.789442Z","iopub.status.idle":"2026-01-18T18:19:48.291693Z","shell.execute_reply.started":"2026-01-18T18:19:47.789413Z","shell.execute_reply":"2026-01-18T18:19:48.291096Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"В качестве обучения беру XGBoost с разными функциями потерь и одинаковыми гиперпараметрами, а также сохраняю модели для ансамбля\n","metadata":{}},{"cell_type":"code","source":"['rank:ndcg', 'rank:map', 'rank:pairwise', 'binary:logistic']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:19:48.292565Z","iopub.execute_input":"2026-01-18T18:19:48.293231Z","iopub.status.idle":"2026-01-18T18:19:48.298188Z","shell.execute_reply.started":"2026-01-18T18:19:48.293206Z","shell.execute_reply":"2026-01-18T18:19:48.297619Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train, y_train, groups, X_test, test_ids, test_ranker_ids = prepare_xgb_data(train, test)\n\nmodels = []\nobjectives = ['rank:ndcg', 'rank:map', 'rank:pairwise', 'binary:logistic']\n\nparams = {\n    'objective': 'rank:ndcg',\n    'eval_metric': 'ndcg@3',\n    'max_depth': 12,\n    'min_child_weight': 10,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'lambda': 100,\n    'learning_rate': 0.05,\n    'n_estimators': 1000,\n    'seed': 42,\n    'tree_method': 'hist',\n    'device': 'cuda',\n}\n\nfor objective in objectives:\n    print(f'\\nИспользуется {objective}')\n    params['objective'] = objective\n    dtrain = xgb.DMatrix(X_train, y_train, group=groups)\n    \n    print(f'Начинается обучение')\n    model = xgb.train(\n        params=params,\n        dtrain=dtrain,\n        num_boost_round=params['n_estimators'],\n        verbose_eval=100\n    )\n    print(f'Закончено обучение')\n    models.append(model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T18:19:48.299084Z","iopub.execute_input":"2026-01-18T18:19:48.299348Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Анализ ошибок (2 баллов)","metadata":{}},{"cell_type":"markdown","source":"Используйте графики для вывода значений полученной метрики (hitrate@3)","metadata":{}},{"cell_type":"markdown","source":"# Оформление работы (2 баллов)","metadata":{}},{"cell_type":"markdown","source":"Дополнительные баллы можно получить за наличие комментириев в работе, разделения кода на разделы, использование осмысленных названий в переменных, указания типов входных и выходных данных в кастомных функциях, графиков содержащие основные элементы (название графика, подписи осей), выводов. ","metadata":{}},{"cell_type":"markdown","source":"В анализе и обработке признаков постарался указать какие-то промежуточные выводы, которые делал","metadata":{}},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"all_preds = []\ndtest = xgb.DMatrix(X_test)\n\nfor model in models:\n    pred = model.predict(dtest)\n    if pred.min() < 0 or pred.max() > 1:\n        pred = 1 / (1 + np.exp(-pred))\n    all_preds.append(pred)\n\nensemble_pred = np.mean(all_preds, axis=0)\n\nresult_df = pl.DataFrame({\n    'Id': test_ids,\n    'ranker_id': test_ranker_ids,\n    'pred_score': ensemble_pred\n})\n\nresult_df = result_df.with_columns(\n    pl.col('pred_score').rank('ordinal', descending=True)\n    .over('ranker_id')\n    .alias('selected')\n)\n\nsubmission = result_df.select(['Id', 'ranker_id', 'selected'])\nsubmission.write_csv(f'submission.csv')\nprint(f\"Submission saved\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}