{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":105399,"databundleVersionId":12733338,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.preprocessing import LabelEncoder\nimport gc # Garbage Collector\nimport warnings\n\nwarnings.filterwarnings('ignore')\npd.set_option('display.max_columns', None)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2\n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print(f'Mem. usage decreased to {end_mem:5.2f} Mb ({100 * (start_mem - end_mem) / start_mem:.1f}% reduction)')\n    return df\n\ndef calculate_hit_rate_at_3(df_preds_with_true_and_rank):\n    \"\"\"\n    Calculates HitRate@3.\n    df_preds_with_true_and_rank must have:\n        - 'ranker_id'\n        - 'selected' (true binary target, 1 for chosen)\n        - 'predicted_rank' (rank assigned by the model, 1 is best)\n    \"\"\"\n    hits = 0\n    valid_queries_count = 0\n    \n    for ranker_id, group in df_preds_with_true_and_rank.groupby('ranker_id'):\n        if len(group) <= 10:\n            continue  # Skip groups with 10 or fewer options as per competition rules\n        \n        valid_queries_count += 1\n        \n        true_selected_item = group[group['selected'] == 1]\n        \n        if not true_selected_item.empty:\n            # Get the rank of the true selected item\n            rank_of_true_item = true_selected_item.iloc[0]['predicted_rank']\n            if rank_of_true_item <= 3:\n                hits += 1\n        # else:\n            # This shouldn't happen in validation if data is prepared correctly from train\n            # print(f\"Warning: No selected item found for ranker_id {ranker_id} in HitRate calculation.\")\n            \n    if valid_queries_count == 0:\n        return 0.0\n    return hits / valid_queries_count","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 3: Load Data\nimport pandas as pd\nimport numpy as np\nimport gc\n\n# DEFINE CORE COLUMNS TO LOAD INITIALLY (TỐI GIẢN HÓA + BỔ SUNG)\ninitial_core_columns = [\n    'Id', 'ranker_id', 'selected', 'profileId', 'companyID',\n    'requestDate', 'totalPrice', 'taxes', # Cần cho totalPrice_rank_in_group và price_per_tax, tax_ratio\n    'legs0_departureAt', 'legs0_arrivalAt', 'legs0_duration',\n    'legs1_departureAt', 'legs1_arrivalAt', 'legs1_duration', \n    # Removed legs1_segments0_departureFrom_airport_iata, legs0_segments_X... as num_segments is derived differently\n    'legs0_segments0_departureFrom_airport_iata', # Base for segment count logic\n    'searchRoute', # Cho is_round_trip\n    'pricingInfo_isAccessTP', # Cho is_compliant\n    'sex', 'nationality', 'isVip', # User info cơ bản\n    \n    # Bổ sung từ CatBoost ideas\n    'legs0_segments0_cabinClass', 'legs1_segments0_cabinClass', # Cabin class features\n    'miniRules0_monetaryAmount', 'miniRules1_monetaryAmount', # Fee features\n    'miniRules0_percentage', 'miniRules1_percentage', # Original free_cancel/exchange features\n    'legs0_segments0_baggageAllowance_quantity', # Baggage total\n    'legs1_segments0_baggageAllowance_quantity',\n    'corporateTariffCode', # has_corporate_tariff\n    'frequentFlyer', # frequentFlyer_binary\n]\n# Add all segment related columns up to segment 3 for more robust num_segments_legX calculation\nfor leg_idx in [0, 1]:\n    for seg_idx in range(4): # Catboost went up to 3, let's keep 4 for now\n        initial_core_columns.append(f'legs{leg_idx}_segments{seg_idx}_departureFrom_airport_iata')\n\ninitial_core_columns = list(set(initial_core_columns)) # Remove duplicates if any\n\ninitial_core_columns_test = [col for col in initial_core_columns if col != 'selected']\n\n\nprint(\"Loading a subset of columns for train_df...\")\ntrain_df = pd.read_parquet('/kaggle/input/aeroclub-recsys-2025/train.parquet', columns=initial_core_columns)\nprint(\"Loading a subset of columns for test_df...\")\ntest_df = pd.read_parquet('/kaggle/input/aeroclub-recsys-2025/test.parquet', columns=initial_core_columns_test)\nsample_submission_df = pd.read_parquet('/kaggle/input/aeroclub-recsys-2025/sample_submission.parquet')\n\nprint(\"\\nTrain DataFrame (after loading subset - BEFORE reduce_mem_usage and any FE):\")\ntrain_df.info(memory_usage='deep')\nprint(f\"\\nShape: {train_df.shape}\")\nprint(\"\\nTest DataFrame (after loading subset - BEFORE reduce_mem_usage and any FE):\")\ntest_df.info(memory_usage='deep')\nprint(f\"\\nShape: {test_df.shape}\")\n\nif 'Id' in test_df.columns and 'ranker_id' in test_df.columns:\n    test_ids_df = test_df[['Id', 'ranker_id']].copy()\nelse:\n    print(\"Warning: 'Id' or 'ranker_id' not found in loaded test_df columns.\")\n    test_ids_df = pd.DataFrame()\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 4: Feature Engineering \n\ndef create_initial_datetime_features(df):\n    loaded_cols = df.columns\n    potential_dt_cols = ['requestDate', 'legs0_departureAt', 'legs0_arrivalAt', 'legs1_departureAt', 'legs1_arrivalAt']\n    for col in potential_dt_cols:\n        if col in loaded_cols:\n            if not pd.api.types.is_datetime64_any_dtype(df[col]):\n                df[col] = pd.to_datetime(df[col].astype(str), errors='coerce')\n    return df\n\ndef create_remaining_features(df, is_train=True):\n    print(f\"Starting FE. Initial df memory: {df.memory_usage(deep=True).sum() / 1024**2:.2f} MB\")\n    \n    # --- Basic time, booking, route, segment features ---\n    potential_dt_cols_for_components = ['legs0_departureAt', 'legs0_arrivalAt', 'legs1_departureAt', 'legs1_arrivalAt']\n    for col in potential_dt_cols_for_components:\n        if col in df.columns and pd.api.types.is_datetime64_any_dtype(df[col]):\n             # Extract components, fill NaNs resulting from NaT, then cast\n             df[col + '_hour'] = df[col].dt.hour.fillna(-1).astype(np.int8) # Fill NaN with -1\n             df[col + '_dow'] = df[col].dt.dayofweek.fillna(-1).astype(np.int8) # Fill NaN with -1\n    \n    if 'legs0_departureAt' in df.columns and 'requestDate' in df.columns and \\\n       pd.api.types.is_datetime64_any_dtype(df['legs0_departureAt']) and \\\n       pd.api.types.is_datetime64_any_dtype(df['requestDate']):\n        # Calculate booking_lead_days only if both columns are valid datetimes\n        # NaT in legs0_departureAt or requestDate will result in NaT for the difference,\n        # then .dt.total_seconds() will produce NaN.\n        time_diff = (df['legs0_departureAt'] - df['requestDate'])\n        df['booking_lead_days'] = time_diff.dt.total_seconds() / (24 * 60 * 60)\n        df['booking_lead_days'] = df['booking_lead_days'].fillna(-1).astype(np.float32)\n    else: \n        df['booking_lead_days'] = np.float32(-1.0)\n    \n    if 'searchRoute' in df.columns: \n        df['is_round_trip'] = df['searchRoute'].astype(str).str.contains('/').astype(np.int8)\n    else: \n        df['is_round_trip'] = np.int8(-1)\n    \n    if 'legs1_departureAt' in df.columns:\n        # Ensure it's datetime before checking .notna() for num_legs calculation\n        if not pd.api.types.is_datetime64_any_dtype(df['legs1_departureAt']):\n            df['legs1_departureAt_dt'] = pd.to_datetime(df['legs1_departureAt'].astype(str), errors='coerce')\n            df['num_legs'] = (1 + df['legs1_departureAt_dt'].notna()).astype(np.int8)\n            df.drop(columns=['legs1_departureAt_dt'], inplace=True) # Drop temporary column\n        else:\n            df['num_legs'] = (1 + df['legs1_departureAt'].notna()).astype(np.int8)\n    else: \n        df['num_legs'] = np.int8(1) # Assumed 1 if leg1 departure not present\n        \n    # Ensure num_segments are initialized as int8 from the start\n    df['num_segments_leg0'] = np.int8(0)\n    df['num_segments_leg1'] = np.int8(0)\n    for i in range(4):\n        col_l0_seg = f'legs0_segments{i}_departureFrom_airport_iata'\n        if col_l0_seg in df.columns: df['num_segments_leg0'] += df[col_l0_seg].notna().astype(np.int8)\n        col_l1_seg = f'legs1_segments{i}_departureFrom_airport_iata'\n        if col_l1_seg in df.columns: df['num_segments_leg1'] += df[col_l1_seg].notna().astype(np.int8)\n    df['total_segments'] = (df['num_segments_leg0'] + df['num_segments_leg1']).astype(np.int8)\n    \n    df['is_direct_leg0'] = (df['num_segments_leg0'] == 1).astype(np.int8)\n    df['is_direct_leg1'] = (df['num_segments_leg1'] == 1).astype(np.int8)\n    df['both_direct'] = (df['is_direct_leg0'] & df['is_direct_leg1']).astype(np.int8)\n\n    # --- Duration and Price based features ---\n    for dur_col in ['legs0_duration', 'legs1_duration']:\n        if dur_col in df.columns:\n            if not pd.api.types.is_numeric_dtype(df[dur_col]):\n                df[dur_col] = pd.to_numeric(df[dur_col].astype(str), errors='coerce').fillna(0)\n            else: \n                df[dur_col] = df[dur_col].fillna(0)\n        else: \n            df[dur_col] = 0 \n    df['legs0_duration'] = df['legs0_duration'].astype(np.float32)\n    df['legs1_duration'] = df['legs1_duration'].astype(np.float32)\n    df['total_flight_duration'] = (df['legs0_duration'] + df['legs1_duration']).astype(np.float32)\n    \n    if 'totalPrice' in df.columns and 'total_flight_duration' in df.columns:\n        df['price_per_minute'] = (df['totalPrice'] / (df['total_flight_duration'] + 1e-6)).fillna(0).astype(np.float32)\n    else:\n        df['price_per_minute'] = np.float32(0.0)\n            \n    if 'totalPrice' in df.columns and 'taxes' in df.columns:\n        # Ensure taxes is numeric and fillna before division\n        df_taxes_numeric = pd.to_numeric(df['taxes'], errors='coerce').fillna(0)\n        df_totalPrice_numeric = pd.to_numeric(df['totalPrice'], errors='coerce').fillna(0)\n        \n        df['tax_ratio'] = (df_taxes_numeric / (df_totalPrice_numeric + 1e-6)).fillna(0).astype(np.float32)\n        df['price_per_tax'] = (df_totalPrice_numeric / (df_taxes_numeric + 1e-6)).fillna(0).astype(np.float32)\n        del df_taxes_numeric, df_totalPrice_numeric\n    else:\n        df['tax_ratio'] = np.float32(0.0)\n        df['price_per_tax'] = np.float32(0.0)\n        \n    if 'pricingInfo_isAccessTP' in df.columns: \n        df['is_compliant'] = pd.to_numeric(df['pricingInfo_isAccessTP'], errors='coerce').fillna(0).astype(np.int8)\n    else: \n        df['is_compliant'] = np.int8(-1)\n    \n    # --- Baggage features ---\n    df['baggage_leg0_qty'] = np.int8(0)\n    df['baggage_leg0_included'] = np.int8(-1)\n    if 'legs0_segments0_baggageAllowance_quantity' in df.columns: \n        df['baggage_leg0_qty'] = pd.to_numeric(df['legs0_segments0_baggageAllowance_quantity'], errors='coerce').fillna(0).astype(np.int8)\n        df['baggage_leg0_included'] = (df['baggage_leg0_qty'] > 0).astype(np.int8)\n        \n    df['baggage_leg1_qty'] = np.int8(0)\n    df['baggage_leg1_included'] = np.int8(0) \n    df['baggage_both_legs_included'] = np.int8(-1)\n\n    if 'legs1_segments0_baggageAllowance_quantity' in df.columns:\n        df['baggage_leg1_qty'] = pd.to_numeric(df['legs1_segments0_baggageAllowance_quantity'], errors='coerce').fillna(0).astype(np.int8)\n        df['baggage_leg1_included'] = (df['baggage_leg1_qty'] > 0).astype(np.int8)\n        # Check if baggage_leg0_included Series exists and is not entirely -1\n        if 'baggage_leg0_included' in df.columns and not (df['baggage_leg0_included'] == -1).all():\n             df['baggage_both_legs_included'] = (df['baggage_leg0_included'] & df['baggage_leg1_included']).astype(np.int8)\n    elif 'baggage_leg0_included' in df.columns and not (df['baggage_leg0_included'] == -1).all():\n        df['baggage_both_legs_included'] = df['baggage_leg0_included'].astype(np.int8)\n    df['baggage_total_qty'] = (df['baggage_leg0_qty'] + df['baggage_leg1_qty']).astype(np.int8)\n\n    # --- Rules based features (cancel, exchange, fees) ---\n    df['free_cancel'] = np.int8(-1); df['free_exchange'] = np.int8(-1)\n    if 'miniRules0_monetaryAmount' in df.columns and 'miniRules0_percentage' in df.columns:\n        monetary0 = pd.to_numeric(df['miniRules0_monetaryAmount'], errors='coerce').fillna(1)\n        percent0 = pd.to_numeric(df['miniRules0_percentage'], errors='coerce').fillna(1)\n        df['free_cancel'] = ((monetary0 == 0) & (percent0 == 0)).astype(np.int8)\n        del monetary0, percent0\n    if 'miniRules1_monetaryAmount' in df.columns and 'miniRules1_percentage' in df.columns:\n        monetary1 = pd.to_numeric(df['miniRules1_monetaryAmount'], errors='coerce').fillna(1)\n        percent1 = pd.to_numeric(df['miniRules1_percentage'], errors='coerce').fillna(1)\n        df['free_exchange'] = ((monetary1 == 0) & (percent1 == 0)).astype(np.int8)\n        del monetary1, percent1\n\n    df['total_fees'] = np.float32(0.0)\n    if 'miniRules0_monetaryAmount' in df.columns:\n        df['total_fees'] += pd.to_numeric(df['miniRules0_monetaryAmount'], errors='coerce').fillna(0)\n    if 'miniRules1_monetaryAmount' in df.columns:\n        df['total_fees'] += pd.to_numeric(df['miniRules1_monetaryAmount'], errors='coerce').fillna(0)\n    df['total_fees'] = df['total_fees'].astype(np.float32) # ensure it's float32 before division\n    df['has_fees'] = (df['total_fees'] > 0).astype(np.int8)\n\n    if 'totalPrice' in df.columns:\n        df_totalPrice_numeric = pd.to_numeric(df['totalPrice'], errors='coerce').fillna(0)\n        df['fee_rate'] = (df['total_fees'] / (df_totalPrice_numeric + 1e-6)).fillna(0).astype(np.float32)\n        del df_totalPrice_numeric\n    else: \n        df['fee_rate'] = np.float32(0.0)\n        \n    # --- Cabin Class features ---\n    df['legs0_segments0_cabinClass_num'] = np.nan\n    df['legs1_segments0_cabinClass_num'] = np.nan\n    if 'legs0_segments0_cabinClass' in df.columns:\n        df['legs0_segments0_cabinClass_num'] = pd.to_numeric(df['legs0_segments0_cabinClass'], errors='coerce')\n    if 'legs1_segments0_cabinClass' in df.columns:\n        df['legs1_segments0_cabinClass_num'] = pd.to_numeric(df['legs1_segments0_cabinClass'], errors='coerce')\n\n    # Fill NaN with a value like -1 before mean if both are NaN for a row, or use .mean() default behavior\n    avg_cabin_class_temp = df[['legs0_segments0_cabinClass_num', 'legs1_segments0_cabinClass_num']].mean(axis=1)\n    df['avg_cabin_class'] = avg_cabin_class_temp.fillna(-1).astype(np.float32)\n    \n    # When calculating diff, fill NaNs with the average if available, otherwise 0 or -1 if avg is also -1\n    l0_cabin_filled = df['legs0_segments0_cabinClass_num'].fillna(avg_cabin_class_temp).fillna(-1)\n    l1_cabin_filled = df['legs1_segments0_cabinClass_num'].fillna(avg_cabin_class_temp).fillna(-1)\n    df['cabin_class_diff'] = (l0_cabin_filled - l1_cabin_filled).astype(np.float32)\n    \n    df.drop(columns=['legs0_segments0_cabinClass_num', 'legs1_segments0_cabinClass_num'], inplace=True, errors='ignore')\n    del avg_cabin_class_temp, l0_cabin_filled, l1_cabin_filled\n\n    # --- Binary user/trip related features ---\n    df['frequentFlyer_binary'] = np.int8(0)\n    df['is_vip_freq'] = np.int8(0) \n    if 'isVip' in df.columns:\n        is_vip_col_temp = df['isVip'].fillna(0)\n        if pd.api.types.is_bool_dtype(is_vip_col_temp): is_vip_col_temp = is_vip_col_temp.astype(int)\n        else: is_vip_col_temp = pd.to_numeric(is_vip_col_temp, errors='coerce').fillna(0).astype(int)\n        df['is_vip_freq'] = (is_vip_col_temp == 1).astype(np.int8)\n        del is_vip_col_temp\n        \n    if 'frequentFlyer' in df.columns:\n        df['frequentFlyer_binary'] = pd.to_numeric(df['frequentFlyer'], errors='coerce').fillna(0).astype(np.int8)\n        df['is_vip_freq'] = (df['is_vip_freq'] | (df['frequentFlyer_binary'] == 1)).astype(np.int8)\n    \n    if 'corporateTariffCode' in df.columns:\n        df['has_corporate_tariff'] = (~df['corporateTariffCode'].astype(str).isna() & \\\n                                     (df['corporateTariffCode'].astype(str) != '') & \\\n                                     (df['corporateTariffCode'].astype(str).str.upper() != 'NAN') & \\\n                                     (df['corporateTariffCode'].astype(str).str.upper() != 'MISSING')).astype(np.int8)\n    else:\n        df['has_corporate_tariff'] = np.int8(-1)\n    \n    gc.collect()\n    print(f\"After basic FE. df memory: {df.memory_usage(deep=True).sum() / 1024**2:.2f} MB\")\n\n    # --- Group-wise features ---\n    group_key = 'ranker_id'\n    if group_key not in df.columns: return df\n\n    key_numeric_features = []\n    for col_candidate in ['totalPrice', 'total_flight_duration', 'booking_lead_days', 'fee_rate', 'total_fees']:\n        if col_candidate in df.columns and pd.api.types.is_numeric_dtype(df[col_candidate]):\n            if col_candidate == 'booking_lead_days' and (df[col_candidate] == -1.0).all():\n                continue\n            key_numeric_features.append(col_candidate)\n    \n    print(f\"Processing group-wise features for {'train' if is_train else 'test'} on columns: {key_numeric_features}\")\n    for col in key_numeric_features:\n        print(f\"  Calculating group features for {col}...\")\n        # Ensure source column is float32 for transform operations to maintain precision then downcast\n        source_col_float32 = df[col].astype(np.float32)\n\n        df[f'{col}_rank_in_group'] = df.groupby(group_key)[col].rank(method='dense', ascending=True).astype(np.float16)\n        df[f'{col}_pct_rank_in_group'] = df.groupby(group_key)[col].rank(method='dense', ascending=True, pct=True).astype(np.float16)\n        \n        group_min = df.groupby(group_key)[col].transform('min').astype(np.float32)\n        df[f'{col}_vs_group_min'] = (source_col_float32 - group_min).astype(np.float16)\n        df[f'is_min_{col}_in_group'] = (source_col_float32 == group_min).astype(np.int8) \n        del group_min; gc.collect()\n        \n        group_mean = df.groupby(group_key)[col].transform('mean').astype(np.float32)\n        df[f'{col}_vs_group_mean'] = (source_col_float32 - group_mean).astype(np.float16)\n        \n        group_std = df.groupby(group_key)[col].transform('std').astype(np.float32).fillna(np.float32(1e-6))\n        df[f'{col}_zscore_in_group'] = ((source_col_float32 - group_mean) / group_std).astype(np.float16)\n        del group_mean, group_std, source_col_float32; gc.collect()\n\n    if 'totalPrice' in df.columns and 'is_compliant' in df.columns:\n        df['price_compliant_temp'] = df['totalPrice'].astype(np.float32) # Ensure float for NaN\n        # Ensure is_compliant is numeric for loc\n        is_compliant_numeric = pd.to_numeric(df['is_compliant'], errors='coerce').fillna(0)\n        df.loc[is_compliant_numeric == 0, 'price_compliant_temp'] = np.nan\n        del is_compliant_numeric\n        \n        min_compliant_price_in_group = df.groupby(group_key)['price_compliant_temp'].transform('min').astype(np.float32)\n        df['price_vs_min_compliant_price'] = (df['totalPrice'].astype(np.float32) - min_compliant_price_in_group).astype(np.float16)\n        df['price_vs_min_compliant_price'] = df['price_vs_min_compliant_price'].fillna(np.float16(0.0)) \n        df.drop(columns=['price_compliant_temp'], inplace=True)\n        del min_compliant_price_in_group; gc.collect()\n    \n    print(f\"After group FE. df memory: {df.memory_usage(deep=True).sum() / 1024**2:.2f} MB\")\n\n    # --- User/Company Categorical ---\n    user_company_cats_loaded = [\n        'sex', 'nationality', 'isVip', \n        'corporateTariffCode', 'frequentFlyer',\n        'legs0_segments0_cabinClass', 'legs1_segments0_cabinClass'\n    ]\n    for col in user_company_cats_loaded:\n        if col in df.columns:\n            # Convert to object first if it's a nullable integer type or bool\n            if pd.api.types.is_bool_dtype(df[col]) or \\\n               (pd.api.types.is_integer_dtype(df[col]) and not pd.api.types.is_categorical_dtype(df[col])):\n                df[col] = df[col].astype('object')\n            # Or if it's float but represents categories (like cabinClass after to_numeric if it was loaded as float)\n            elif pd.api.types.is_float_dtype(df[col]) and col in ['legs0_segments0_cabinClass', 'legs1_segments0_cabinClass']:\n                 df[col] = df[col].astype('object')\n\n            df[col] = df[col].fillna('MISSING').astype('category')\n    \n    binary_cols_loaded = [c for c in ['bySelf', 'isAccess3D'] if c in df.columns]\n    for col in binary_cols_loaded: \n        df[col] = pd.to_numeric(df[col], errors='coerce').fillna(0).astype(np.int8)\n    \n    print(f\"End of FE. Final df memory: {df.memory_usage(deep=True).sum() / 1024**2:.2f} MB\")\n    df = reduce_mem_usage(df, verbose=False) # Reduce memory one last time within the function\n    print(f\"End of FE after final reduce_mem_usage: {df.memory_usage(deep=True).sum() / 1024**2:.2f} MB\")\n    return df\n\n# --- Execution part of Cell 4 (SEPARATE PROCESSING) ---\nprint(\"--- Processing TRAIN_DF ---\")\nprint(\"Initial datetime conversion for train_df...\")\ntrain_df_processed = create_initial_datetime_features(train_df.copy())\ndel train_df; gc.collect()\n\nprint(\"Applying reduce_mem_usage to train_df_processed before main FE...\")\ntrain_df_processed = reduce_mem_usage(train_df_processed) \ngc.collect()\n\nprint(\"Creating remaining features for train_df_processed...\")\ntrain_df_processed = create_remaining_features(train_df_processed, is_train=True)\ngc.collect()\n\n# No need for another reduce_mem_usage here if it's done at the end of create_remaining_features\n\n\ntrain_labels = train_df_processed['selected']\ntrain_ids = train_df_processed['Id']\ntrain_ranker_ids = train_df_processed['ranker_id']\n\nraw_datetime_col_names = ['requestDate', 'legs0_departureAt', 'legs0_arrivalAt', 'legs1_departureAt', 'legs1_arrivalAt']\noriginal_categorical_like_cols = [\n    'sex', 'nationality', 'isVip', \n    'corporateTariffCode', 'frequentFlyer',\n    'legs0_segments0_cabinClass', 'legs1_segments0_cabinClass' \n]\nid_cols_and_target = ['Id', 'ranker_id', 'selected', 'profileId', 'companyID', 'searchRoute'] \n\nexcluded_for_X_train = id_cols_and_target + raw_datetime_col_names + \\\n                       [c for c in original_categorical_like_cols if c in train_df_processed.columns]\ntrain_feature_cols = [col for col in train_df_processed.columns if col not in excluded_for_X_train]\n\n\nX = train_df_processed[train_feature_cols].copy()\ny = train_labels.copy()\nprint(f\"Shape of X_train: {X.shape}\"); print(f\"X_train memory: {X.memory_usage(deep=True).sum() / 1024**2:.2f} MB\")\ndel train_df_processed; gc.collect()\n\n\nprint(\"\\n--- Processing TEST_DF ---\")\nprint(\"Initial datetime conversion for test_df...\")\ntest_df_processed = create_initial_datetime_features(test_df.copy())\ndel test_df; gc.collect()\n\nprint(\"Applying reduce_mem_usage to test_df_processed before main FE...\")\ntest_df_processed = reduce_mem_usage(test_df_processed)\ngc.collect()\n\nprint(\"Creating remaining features for test_df_processed...\")\ntest_df_processed = create_remaining_features(test_df_processed, is_train=False)\ngc.collect()\n\n# No need for another reduce_mem_usage here\n\nX_test = pd.DataFrame(columns=train_feature_cols, index=test_df_processed.index)\nfor col in train_feature_cols:\n    if col in test_df_processed.columns:\n        X_test[col] = test_df_processed[col]\n    else:\n        # Check dtype of the column in X (train) to decide fill value for X_test\n        if X[col].dtype.name.startswith('float') or X[col].dtype.name.startswith('int'):\n             X_test[col] = 0 \n        else: # Should be category after LabelEncoding, or object if LE hasn't happened yet\n             X_test[col] = \"MISSING_IN_TEST\"\n\ndel test_df_processed; gc.collect()\n\nprint(f\"Shape of X_test: {X_test.shape}\"); print(f\"X_test memory: {X_test.memory_usage(deep=True).sum() / 1024**2:.2f} MB\")\nprint(f\"\\nFinal shapes before LabelEncoding: X_train: {X.shape}, X_test: {X_test.shape}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 5: Label Encoding\n# Define potential categorical features based on name patterns or known types\n# This list includes originals that were set to .astype('category') and new ones\npotential_cat_feature_names = [\n    'sex', 'nationality', 'isVip', 'corporateTariffCode', 'frequentFlyer', # These were original .astype('category')\n    # Add other features that were created as categories or should be treated as such\n    # e.g., airport codes if not already numerical and meant to be categorical\n    # 'legs0_segments0_departureFrom_airport_iata' # If it wasn't used to create num_segments and you want it as category\n]\n\ncategorical_features_for_encoding = []\nprint(\"\\nIdentifying categorical features for Label Encoding from X.columns...\")\n\nfor col in X.columns:\n    # Heuristic: if dtype is object or category, or if name is in our potential list\n    if X[col].dtype.name == 'object' or X[col].dtype.name == 'category' or col in potential_cat_feature_names:\n        # Additional check: if it's in potential_cat_feature_names but somehow became numeric due to FE, we might not want to LE it\n        # However, if it was explicitly made category in FE, it's fine.\n        # The original code converts 'sex', 'nationality', etc., to 'category' in FE.\n        \n        # Check if column exists in X_test before trying to access its dtype\n        if col in X_test.columns and (X_test[col].dtype.name == 'object' or X_test[col].dtype.name == 'category' or col in potential_cat_feature_names):\n            pass # It's also object/category in test or a known cat feature\n        elif col in X_test.columns and X_test[col].dtype.name not in ['object', 'category'] and col not in potential_cat_feature_names:\n            print(f\"Skipping LE for {col} as it's numeric in X_test and not in potential_cat_feature_names\")\n            continue # Skip if it's numeric in test and not explicitly listed as cat\n\n        print(f\"Column '{col}' (dtype: {X[col].dtype}) identified as categorical for encoding.\")\n        categorical_features_for_encoding.append(col)\n        le = LabelEncoder()\n        \n        # Handle missing columns in X_test more robustly during Label Encoding\n        if col in X_test.columns:\n            # Combine unique values from both train and test for fitting the encoder\n            # Ensure consistent handling of NaN/missing values by converting to string\n            X_col_str = X[col].astype(str).fillna('MISSING_CAT_VALUE')\n            X_test_col_str = X_test[col].astype(str).fillna('MISSING_CAT_VALUE')\n            \n            combined_col_data = pd.concat([X_col_str, X_test_col_str], axis=0).unique()\n            le.fit(combined_col_data)\n            \n            X[col] = le.transform(X_col_str)\n            X_test[col] = le.transform(X_test_col_str)\n        else:\n            # If column is not in X_test at all, only fit_transform on X\n            X_col_str = X[col].astype(str).fillna('MISSING_CAT_VALUE')\n            X[col] = le.fit_transform(X_col_str)\n            # X_test will not have this column if it wasn't created.\n            # If it *should* have been created but was missed, that's an earlier issue.\n            # For now, we assume X_test alignment handles this.\n\nprint(f\"\\nCategorical features processed with LabelEncoder: {categorical_features_for_encoding}\")\n\n# Ensure all columns are numeric after Label Encoding\nprint(\"\\nChecking for non-numeric columns after LabelEncoding...\")\nfor col in X.columns:\n    if not pd.api.types.is_numeric_dtype(X[col]):\n        print(f\"Warning: Non-numeric column post-LE in X: {col}, dtype: {X[col].dtype}. Forcing numeric, filling NaNs with -1.\")\n        X[col] = pd.to_numeric(X[col], errors='coerce').fillna(-1).astype(np.int32) # Or appropriate type\n    if col in X_test.columns and not pd.api.types.is_numeric_dtype(X_test[col]):\n        print(f\"Warning: Non-numeric column post-LE in X_test: {col}, dtype: {X_test[col].dtype}. Forcing numeric, filling NaNs with -1.\")\n        X_test[col] = pd.to_numeric(X_test[col], errors='coerce').fillna(-1).astype(np.int32)\n\nfinal_features_list = list(X.columns)\nprint(f\"\\nFinal features for model ({len(final_features_list)}): {final_features_list}\")\nprint(\"\\nX dtypes after all processing:\"); print(X.dtypes.value_counts())\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 6: Model Training\n\nimport lightgbm as lgb\nimport matplotlib.pyplot as plt\nimport gc\nimport numpy as np # Ensure numpy is imported\n\n# --- GLOBAL SUBSAMPLING FOR FINAL TRAINING ---\nGLOBAL_TRAIN_SAMPLE_FRAC = 0.3 \nNFOLDS = 5 # Keep as originally intended, subsampling will reduce fold data size\n\nif GLOBAL_TRAIN_SAMPLE_FRAC < 1.0:\n    print(f\"Applying GLOBAL subsample of {GLOBAL_TRAIN_SAMPLE_FRAC*100}% for K-Fold training.\")\n    \n    # Create a temporary DataFrame for sampling\n    # Ensure train_ranker_ids is from the full X, y before any previous HPO subsampling\n    # Assuming X, y, train_ranker_ids at this point are the full datasets after FE & LE\n    \n    unique_rankers_full = train_ranker_ids.unique()\n    n_sample_groups = int(len(unique_rankers_full) * GLOBAL_TRAIN_SAMPLE_FRAC)\n    \n    if n_sample_groups < NFOLDS and len(unique_rankers_full) >= NFOLDS:\n        print(f\"Warning: Sampled groups ({n_sample_groups}) less than NFOLDS ({NFOLDS}). Adjusting sample size to NFOLDS.\")\n        n_sample_groups = NFOLDS\n    elif n_sample_groups == 0 and len(unique_rankers_full) > 0:\n        n_sample_groups = 1 # Should be at least NFOLDS if possible\n        if n_sample_groups < NFOLDS:\n             print(f\"CRITICAL: Not enough groups to sample for {NFOLDS} folds. Using all available {len(unique_rankers_full)} groups.\")\n             n_sample_groups = len(unique_rankers_full)\n\n\n    if n_sample_groups > 0 and n_sample_groups >= NFOLDS :\n        np.random.seed(42)\n        sampled_ranker_ids_global = np.random.choice(unique_rankers_full, size=n_sample_groups, replace=False)\n        \n        # Get indices from the original full train_ranker_ids Series\n        sampled_indices_global = train_ranker_ids[train_ranker_ids.isin(sampled_ranker_ids_global)].index\n        \n        X_run = X.loc[sampled_indices_global].reset_index(drop=True)\n        y_run = y.loc[sampled_indices_global].reset_index(drop=True)\n        train_ranker_ids_run = train_ranker_ids.loc[sampled_indices_global].reset_index(drop=True)\n        \n        del sampled_indices_global, unique_rankers_full, sampled_ranker_ids_global\n        gc.collect()\n        print(f\"  X_run shape after global subsampling: {X_run.shape}\")\n    else:\n        print(\"  Global subsampling resulted in too few groups or zero groups. Using full data (might cause OOM).\")\n        X_run = X.copy() # \n        y_run = y.copy()\n        train_ranker_ids_run = train_ranker_ids.copy()\nelse:\n    print(\"Using full data for K-Fold training (GLOBAL_TRAIN_SAMPLE_FRAC = 1.0).\")\n    X_run = X.copy()\n    y_run = y.copy()\n    train_ranker_ids_run = train_ranker_ids.copy()\n\n\nparams = {'n_estimators': 1100, \n          'learning_rate': 0.07777978129553888,\n          'num_leaves': 55, 'max_depth': 10,\n          'min_child_samples': 80,\n          'subsample': 0.8,\n          'colsample_bytree': 0.6,\n          'max_bin': 255,\n          'reg_alpha': 3.2096452039244645,\n          'reg_lambda': 0.06801980497003189,\n          'min_split_gain': 0.14,\n          'objective': 'lambdarank',\n          'metric': 'ndcg',\n          'eval_at': [3],\n          'boosting_type': 'gbdt',\n          'random_state': 42,\n          'n_jobs': -1,\n          'verbose': -1,\n          'seed': 42}\n\ngroup_kfold = GroupKFold(n_splits=NFOLDS)\n\noof_preds_scores = np.zeros(len(X_run)) # Adjusted to X_run size\ntest_preds_scores = np.zeros(len(X_test))\nmodels = []\nfold_hit_rates = []\n\ncat_features_for_lgbm_indices_final = [\n    X_run.columns.get_loc(col_name) # Use X_run for locating columns\n    for col_name in categorical_features_for_encoding if col_name in X_run.columns\n]\n\nif cat_features_for_lgbm_indices_final:\n    print(f\"Using categorical feature indices for LightGBM: {cat_features_for_lgbm_indices_final}\")\n    print(f\"Corresponding feature names: {[X_run.columns[i] for i in cat_features_for_lgbm_indices_final]}\\n\")\nelse:\n    print(\"No categorical features identified for LightGBM native handling.\\n\")\n\n\nfor fold_, (train_idx, val_idx) in enumerate(group_kfold.split(X_run, y_run, groups=train_ranker_ids_run)):\n    print(f\"====== Fold {fold_ + 1}/{NFOLDS} ======\")\n    \n    if fold_ > 0:\n        gc.collect()\n\n    X_train_fold = X_run.iloc[train_idx]\n    y_train_fold = y_run.iloc[train_idx]\n    X_val_fold = X_run.iloc[val_idx]\n    y_val_fold = y_run.iloc[val_idx]\n\n    print(f\"  Train fold shape: {X_train_fold.shape}, Val fold shape: {X_val_fold.shape}\")\n\n    current_train_fold_ranker_ids = train_ranker_ids_run.iloc[train_idx]\n    current_val_fold_ranker_ids = train_ranker_ids_run.iloc[val_idx]\n\n    train_fold_groups = X_train_fold.groupby(current_train_fold_ranker_ids.values).size().to_list()\n    val_fold_groups = X_val_fold.groupby(current_val_fold_ranker_ids.values).size().to_list()\n\n    if not train_fold_groups or 0 in train_fold_groups or not val_fold_groups or 0 in val_fold_groups:\n        print(f\"Skipping fold {fold_ + 1} due to empty or zero-sized groups.\")\n        continue\n\n    ranker = lgb.LGBMRanker(**params)\n    try:\n        print(f\"  Starting LightGBM fit for fold {fold_ + 1} with params: {params}\")\n        ranker.fit(\n            X_train_fold, y_train_fold,\n            group=train_fold_groups,\n            eval_set=[(X_val_fold, y_val_fold)],\n            eval_group=[val_fold_groups],\n            eval_metric='ndcg',\n            callbacks=[lgb.early_stopping(100, verbose=100)],\n            categorical_feature=cat_features_for_lgbm_indices_final if cat_features_for_lgbm_indices_final else 'auto'\n        )\n        print(f\"  LightGBM fit completed for fold {fold_ + 1}.\")\n    except Exception as e:\n        print(f\"Error during LightGBM fit in fold {fold_ + 1}: {e}\")\n        # print(f\"  X_train_fold mem: {X_train_fold.memory_usage(deep=True).sum() / 1024**2:.2f} MB\")\n        # print(f\"  X_val_fold mem: {X_val_fold.memory_usage(deep=True).sum() / 1024**2:.2f} MB\")\n        # Fallback or break if needed\n        del X_train_fold, y_train_fold, X_val_fold, y_val_fold\n        del current_train_fold_ranker_ids, current_val_fold_ranker_ids\n        del train_fold_groups, val_fold_groups\n        gc.collect()\n        continue \n\n    models.append(ranker)\n    val_fold_scores = ranker.predict(X_val_fold)\n    oof_preds_scores[val_idx] = val_fold_scores # Store in subsampled OOF array\n\n    if not X_test.empty:\n        print(f\"  Predicting on X_test (shape: {X_test.shape}) for fold {fold_ + 1}...\")\n        current_test_preds = ranker.predict(X_test)\n        test_preds_scores += current_test_preds / NFOLDS\n        del current_test_preds\n        gc.collect()\n\n    val_df_for_metric = pd.DataFrame({\n        'ranker_id': current_val_fold_ranker_ids,\n        'selected': y_val_fold,\n        'score': val_fold_scores\n    })\n    val_df_for_metric['predicted_rank'] = val_df_for_metric.groupby('ranker_id')['score']\\\n        .rank(method='first', ascending=False).astype(int)\n\n    fold_hr3 = calculate_hit_rate_at_3(val_df_for_metric)\n    fold_hit_rates.append(fold_hr3)\n    print(f\"Fold {fold_ + 1} HitRate@3: {fold_hr3:.4f}\")\n\n    del X_train_fold, y_train_fold, X_val_fold, y_val_fold\n    del current_train_fold_ranker_ids, current_val_fold_ranker_ids\n    del train_fold_groups, val_fold_groups, ranker, val_fold_scores, val_df_for_metric\n    gc.collect()\n\n# Final overall evaluation\nif models and len(models) > 0: # Check if at least one model was trained\n    # If global subsampling was used, OOF score is on that subsample\n    print_oof_source = \"subsampled\" if GLOBAL_TRAIN_SAMPLE_FRAC < 1.0 else \"full\"\n    \n    oof_df_for_metric = pd.DataFrame({\n        'ranker_id': train_ranker_ids_run, \n        'selected': y_run,                 \n        'score': oof_preds_scores\n    })\n\n    oof_df_for_metric['predicted_rank'] = oof_df_for_metric.groupby('ranker_id')['score']\\\n        .rank(method='first', ascending=False).astype(int)\n\n    overall_oof_hr3 = calculate_hit_rate_at_3(oof_df_for_metric)\n    print(f\"\\nOverall OOF HitRate@3 on {print_oof_source} data (based on {len(models)} trained models): {overall_oof_hr3:.4f}\")\n    \n    if fold_hit_rates: print(f\"Mean Fold HitRate@3: {np.mean(fold_hit_rates):.4f}\")\n    \n    print(f\"\\nParameters used for final folds: {params}\")\n    print(\"\\nFeature Importances (from last successful model):\")\n    try:\n        lgb.plot_importance(models[-1], figsize=(12, max(18, int(len(X_run.columns)/1.5) if X_run.columns.size > 0 else 18)), \n                            max_num_features=len(X_run.columns) if X_run.columns.size > 0 else 20, \n                            importance_type='gain')\n        plt.show()\n    except Exception as e:\n        print(f\"Could not plot feature importance: {e}\")\nelse:\n    print(\"No models were trained successfully.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Use the test_ids_df we saved earlier which has original Id and ranker_id\nsubmission_df = test_ids_df.copy()\nsubmission_df['score'] = test_preds_scores \n\nsubmission_df['selected'] = submission_df.groupby('ranker_id')['score'].rank(method='first', ascending=False).astype(int)\n\n# Select only required columns and ensure correct order\nsubmission_df = submission_df[['Id', 'ranker_id', 'selected']]\n\n# Check submission format against sample\nprint(\"\\nSample Submission:\")\nprint(sample_submission_df.head())\nprint(\"\\nOur Submission:\")\nprint(submission_df.head())\n\n# Save submission\nsubmission_df.to_parquet('submission.parquet', index=False)\nsubmission_df.to_csv('submission.csv', index=False)\nprint(\"\\nSubmission file 'submission.parquet' created successfully.\")\nprint(f\"Submission shape: {submission_df.shape}\")\n\n# Basic validation of submission\n# 1. All Ids from test set are present\nassert len(submission_df) == len(test_ids_df), \"Number of rows doesn't match test set\"\nassert submission_df['Id'].nunique() == len(test_ids_df['Id'].unique()), \"Mismatch in unique Ids\"\n\n# 2. Ranks are integers and start from 1\nassert submission_df['selected'].min() >= 1, \"Ranks should be >= 1\"\nassert submission_df['selected'].dtype == 'int', \"Ranks should be integers\"\n\n# 3. Ranks are a valid permutation within each group\ndef check_rank_permutation(group):\n    N = len(group)\n    sorted_ranks = sorted(list(group['selected']))\n    expected_ranks = list(range(1, N + 1))\n    if sorted_ranks != expected_ranks:\n        print(f\"Invalid rank permutation for ranker_id: {group['ranker_id'].iloc[0]}\")\n        print(f\"Expected: {expected_ranks}, Got: {sorted_ranks}\")\n        return False\n    return True\n\nprint(\"Basic submission validation checks passed (row count, Id uniqueness, rank min value, rank dtype).\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}