{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.9.6"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":105399,"databundleVersionId":12733338,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"0448ac2f","cell_type":"code","source":"!pip install lightgbm -q","metadata":{},"outputs":[],"execution_count":null},{"id":"174f9a17","cell_type":"code","source":"import polars as pl\nimport pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nimport time\n\n\npl.Config.set_tbl_rows(-1)      # show all rows\npl.Config.set_tbl_cols(-1)      # show all columns\npl.Config.set_tbl_width_chars(200)   # optionally widen display if needed","metadata":{},"outputs":[],"execution_count":null},{"id":"9fc518ba","cell_type":"markdown","source":"### Constants","metadata":{}},{"id":"8c44303f","cell_type":"code","source":" # Set to True if you are creating a Kaggle submission\nKAGGLE_SUBMISSION = True","metadata":{},"outputs":[],"execution_count":null},{"id":"25b32523","cell_type":"markdown","source":"### Filepaths","metadata":{}},{"id":"9298eeeb","cell_type":"code","source":"if KAGGLE_SUBMISSION:\n    # Kaggle file path\n    train_file_path = \"/kaggle/input/aeroclub-recsys-2025/train.parquet\"\n    test_file_path = \"/kaggle/input/aeroclub-recsys-2025/test.parquet\"\n    sample_submission_file_path = \"/kaggle/input/aeroclub-recsys-2025/sample_submission.parquet\"\nelse:\n    # Local file paths\n    train_file_path = \"data/0_raw/train.parquet\"\n    test_file_path = \"data/0_raw/test.parquet\"\n    sample_submission_file_path = \"data/0_raw/sample_submission.parquet\"","metadata":{},"outputs":[],"execution_count":null},{"id":"cd6f549f","cell_type":"markdown","source":"### Load Raw Dataset","metadata":{}},{"id":"96185543","cell_type":"code","source":"# Load the dataset\ntrain = pl.read_parquet(train_file_path).drop('__index_level_0__')\ntest = pl.read_parquet(test_file_path).drop('__index_level_0__')","metadata":{},"outputs":[],"execution_count":null},{"id":"d865a8f1","cell_type":"code","source":"# Append empty target column to test set so it matches train set structure\ntest = test.with_columns(pl.lit(1.0).alias('selected'))","metadata":{},"outputs":[],"execution_count":null},{"id":"9319333a","cell_type":"code","source":"# Save the test Id for submission file\ntest_ids = test.select('Id').to_series().to_list()","metadata":{},"outputs":[],"execution_count":null},{"id":"49da8abb","cell_type":"markdown","source":"# Data Cleaning","metadata":{}},{"id":"247041b5","cell_type":"markdown","source":"### Reorder Columns","metadata":{}},{"id":"b53d460d","cell_type":"markdown","source":"#### Function","metadata":{}},{"id":"b65ce520","cell_type":"code","source":"# Define each logical group of columns according to the provided descriptions\n\n# 1. Identifiers and Metadata\nidentifiers = [\n    \"Id\", \"ranker_id\", \"profileId\", \"companyID\"\n]\n\n# 2. User Information\nuser_info = [\n    \"sex\", \"nationality\", \"frequentFlyer\", \"isVip\", \"bySelf\", \"isAccess3D\"\n]\n\n# 3. Company Information\ncompany_info = [\n    \"corporateTariffCode\"\n]\n\n# 4. Search and Route Information\nsearch_info = [\n    \"searchRoute\", \"requestDate\"\n]\n\n# 5. Pricing Information\npricing_info = [\n    \"totalPrice\", \"taxes\"\n]\n\n# 6. Flight Timing and Duration\ntiming_info = [\n    \"legs0_departureAt\", \"legs0_arrivalAt\", \"legs0_duration\",\n    \"legs1_departureAt\", \"legs1_arrivalAt\", \"legs1_duration\"\n]\n\n# 7. Flight Segments (dynamic)\ngeography_route = []\nairline_flight_details = []\nservice_characteristics = []\n\nfor leg in [0, 1]:\n    for seg in range(4):\n        prefix = f\"legs{leg}_segments{seg}_\"\n        geography_route.extend([\n            f\"{prefix}departureFrom_airport_iata\",\n            f\"{prefix}arrivalTo_airport_iata\",\n            f\"{prefix}arrivalTo_airport_city_iata\"\n        ])\n        airline_flight_details.extend([\n            f\"{prefix}marketingCarrier_code\",\n            f\"{prefix}operatingCarrier_code\",\n            f\"{prefix}aircraft_code\",\n            f\"{prefix}flightNumber\",\n            f\"{prefix}duration\"\n        ])\n        service_characteristics.extend([\n            f\"{prefix}baggageAllowance_quantity\",\n            f\"{prefix}baggageAllowance_weightMeasurementType\",\n            f\"{prefix}cabinClass\",\n            f\"{prefix}seatsAvailable\"\n        ])\n\n# 8. Cancellation and Exchange Rules\ncancellation_rules = [\n    \"miniRules0_monetaryAmount\", \"miniRules0_percentage\", \"miniRules0_statusInfos\",\n    \"miniRules1_monetaryAmount\", \"miniRules1_percentage\", \"miniRules1_statusInfos\"\n]\n\n# 9. Pricing Policy Information\npolicy_info = [\n    \"pricingInfo_isAccessTP\", \"pricingInfo_passengerCount\"\n]\n\n# 10. Target\ntarget = [\n    \"selected\"\n]\n\n# Final column ordering\nfinal_col_order = (\n    identifiers +\n    user_info +\n    company_info +\n    search_info +\n    pricing_info +\n    timing_info +\n    geography_route +\n    airline_flight_details +\n    service_characteristics +\n    cancellation_rules +\n    policy_info +\n    target\n)\n\n","metadata":{},"outputs":[],"execution_count":null},{"id":"3b4866e5","cell_type":"markdown","source":"### Run","metadata":{}},{"id":"8dcc9bec","cell_type":"code","source":"# Display the length to verify\nlen(final_col_order)\n\n# Reorder the DataFrame\ntrain_reordered = train[final_col_order]\ntest_reordered = test[final_col_order]\n\n# Assert the final column order matches the DataFrame columns\nassert len(final_col_order) == len(train.columns), \"Column count mismatch\"\nassert len(final_col_order) == len(test.columns), \"Column count mismatch in test set\"","metadata":{},"outputs":[],"execution_count":null},{"id":"c67e3117","cell_type":"code","source":"train_reordered.head()","metadata":{},"outputs":[],"execution_count":null},{"id":"0da3239e","cell_type":"markdown","source":"### Rename Columns","metadata":{}},{"id":"bd0468c9","cell_type":"markdown","source":"#### Function","metadata":{}},{"id":"c6ece59e","cell_type":"code","source":"# Build a rename map for static columns\nrename_map = {\n    \"Id\": \"flight_option_id\",\n    \"ranker_id\": \"search_session_id\",\n    \"profileId\": \"user_id\",\n    \"companyID\": \"company_id\",\n    \"sex\": \"user_gender\",\n    \"nationality\": \"user_nationality\",\n    \"frequentFlyer\": \"frequent_flyer_status\",\n    \"isVip\": \"vip_status\",\n    \"bySelf\": \"booked_by_self\",\n    \"isAccess3D\": \"internal_flag_3d\",\n    \"corporateTariffCode\": \"corporate_tariff_code\",\n    \"searchRoute\": \"route_type\",\n    \"requestDate\": \"search_time\",\n    \"totalPrice\": \"price_total\",\n    \"taxes\": \"price_taxes\",\n    \"legs0_departureAt\": \"outbound_departure_time\",\n    \"legs0_arrivalAt\": \"outbound_arrival_time\",\n    \"legs0_duration\": \"outbound_duration\",\n    \"legs1_departureAt\": \"return_departure_time\",\n    \"legs1_arrivalAt\": \"return_arrival_time\",\n    \"legs1_duration\": \"return_duration\",\n    \"miniRules0_monetaryAmount\": \"cancellation_fee_amount\",\n    \"miniRules0_percentage\": \"cancellation_fee_percent\",\n    \"miniRules0_statusInfos\": \"cancellation_rule_status\",\n    \"miniRules1_monetaryAmount\": \"exchange_fee_amount\",\n    \"miniRules1_percentage\": \"exchange_fee_percent\",\n    \"miniRules1_statusInfos\": \"exchange_rule_status\",\n    \"pricingInfo_isAccessTP\": \"is_compliant_with_corporate_travel_policy\",\n    \"pricingInfo_passengerCount\": \"num_of_passengers\",\n    \"selected\": \"flight_is_selected\"\n}\n\n# Add dynamic segment-based renaming\nleg_map = {0: \"outbound\", 1: \"return\"}\n\nfor leg in [0, 1]:\n    for seg in range(4):\n        leg_str = leg_map[leg]\n        prefix = f\"legs{leg}_segments{seg}_\"\n        rename_map.update({\n            f\"{prefix}departureFrom_airport_iata\": f\"{leg_str}_seg{seg}_departure_airport\",\n            f\"{prefix}arrivalTo_airport_iata\": f\"{leg_str}_seg{seg}_arrival_airport\",\n            f\"{prefix}arrivalTo_airport_city_iata\": f\"{leg_str}_seg{seg}_arrival_city\",\n            f\"{prefix}marketingCarrier_code\": f\"{leg_str}_seg{seg}_marketing_airline\",\n            f\"{prefix}operatingCarrier_code\": f\"{leg_str}_seg{seg}_operating_airline\",\n            f\"{prefix}aircraft_code\": f\"{leg_str}_seg{seg}_aircraft_type\",\n            f\"{prefix}flightNumber\": f\"{leg_str}_seg{seg}_flight_number\",\n            f\"{prefix}duration\": f\"{leg_str}_seg{seg}_segment_duration\",\n            f\"{prefix}baggageAllowance_quantity\": f\"{leg_str}_seg{seg}_baggage_quantity\",\n            f\"{prefix}baggageAllowance_weightMeasurementType\": f\"{leg_str}_seg{seg}_baggage_unit\",\n            f\"{prefix}cabinClass\": f\"{leg_str}_seg{seg}_cabin_class\",\n            f\"{prefix}seatsAvailable\": f\"{leg_str}_seg{seg}_seats_available\"\n        })\n\n# Show part of the rename_map as preview\npreview = list(rename_map.items())[:10]\npreview","metadata":{},"outputs":[],"execution_count":null},{"id":"621bfbd3","cell_type":"markdown","source":"#### Run","metadata":{}},{"id":"392b848e","cell_type":"code","source":"# Rename the DataFrame columns\ntrain_renamed = train_reordered.rename(rename_map)\ntest_renamed = test_reordered.rename(rename_map)\n\nassert train_renamed.shape[1] == 126, \"Column count mismatch after reordering.\"\nassert test_renamed.shape[1] == 126, \"Column count mismatch in test set after reordering.\"","metadata":{},"outputs":[],"execution_count":null},{"id":"5b723c9c","cell_type":"code","source":"train_renamed.head()","metadata":{},"outputs":[],"execution_count":null},{"id":"f2ad5eb2","cell_type":"markdown","source":"### Reduce Memory Usage","metadata":{}},{"id":"3388f968","cell_type":"markdown","source":"#### Function","metadata":{}},{"id":"a4dad444","cell_type":"code","source":"def reduce_mem_usage_polars(df: pl.DataFrame, verbose=True) -> pl.DataFrame:\n    start_mem = df.estimated_size() / 1024 ** 2\n    print(f\"Memory usage of dataframe is {start_mem:.2f} MB\")\n        \n    for col, dtype in df.schema.items():\n        if dtype in [pl.Int64, pl.Int32, pl.Int16, pl.Float64, pl.Float32]:\n            \n            c_min = df[col].min()\n            c_max = df[col].max()\n\n            # Skip columns where all values are null\n            if c_min is None or c_max is None:\n                if verbose:\n                    print(f\"Skipping column {col} because all values are null.\")\n                continue\n\n            # Integers\n            if dtype.is_integer():\n                if np.iinfo(np.int8).min < c_min and c_max < np.iinfo(np.int8).max:\n                    df = df.with_columns(pl.col(col).cast(pl.Int8))\n                    if verbose:\n                        print(f\"{col} downcasted to Int8\")\n                elif np.iinfo(np.int16).min < c_min and c_max < np.iinfo(np.int16).max:\n                    df = df.with_columns(pl.col(col).cast(pl.Int16))\n                    if verbose:\n                        print(f\"{col} downcasted to Int16\")\n                elif np.iinfo(np.int32).min < c_min and c_max < np.iinfo(np.int32).max:\n                    df = df.with_columns(pl.col(col).cast(pl.Int32))\n                    if verbose:\n                        print(f\"{col} downcasted to Int32\")\n                else:\n                    df = df.with_columns(pl.col(col).cast(pl.Int64))\n                    if verbose:\n                        print(f\"{col} left as Int64\")\n            \n            # Floats\n            elif dtype.is_float():\n                if np.finfo(np.float32).min < c_min and c_max < np.finfo(np.float32).max:\n                    df = df.with_columns(pl.col(col).cast(pl.Float32))\n                    if verbose:\n                        print(f\"{col} downcasted to Float32\")\n                else:\n                    df = df.with_columns(pl.col(col).cast(pl.Float64))\n                    if verbose:\n                        print(f\"{col} left as Float64\")\n\n    end_mem = df.estimated_size() / 1024 ** 2\n    print(f\"Memory usage after optimization is: {end_mem:.2f} MB\")\n    print(f\"Reduced by {100 * (start_mem - end_mem) / start_mem:.1f}%\")\n\n    return df","metadata":{},"outputs":[],"execution_count":null},{"id":"0d4d1420","cell_type":"markdown","source":"#### Run","metadata":{}},{"id":"e453274e","cell_type":"code","source":"df_reduced = reduce_mem_usage_polars(train_renamed, verbose=False)\ndf_reduced_test = reduce_mem_usage_polars(test_renamed, verbose=False)","metadata":{},"outputs":[],"execution_count":null},{"id":"2a1b985e","cell_type":"code","source":"df_reduced.head()","metadata":{},"outputs":[],"execution_count":null},{"id":"df62c9d6","cell_type":"markdown","source":"### Fix Missing Values","metadata":{}},{"id":"f8c0e2db","cell_type":"markdown","source":"#### Function","metadata":{}},{"id":"2a1df391","cell_type":"code","source":"columns_to_drop = [\n    # seg3 columns (100% missing)\n    \"outbound_seg3_departure_airport\",      # 100% missing values\n    \"outbound_seg3_arrival_airport\",        # 100% missing values\n    \"outbound_seg3_arrival_city\",           # 100% missing values\n    \"return_seg3_departure_airport\",        # 100% missing values\n    \"return_seg3_arrival_airport\",          # 100% missing values\n    \"return_seg3_arrival_city\",             # 100% missing values\n    \"outbound_seg3_marketing_airline\",      # 100% missing values\n    \"outbound_seg3_operating_airline\",      # 100% missing values\n    \"outbound_seg3_aircraft_type\",          # 100% missing values\n    \"outbound_seg3_flight_number\",          # 100% missing values\n    \"outbound_seg3_segment_duration\",       # 100% missing values\n    \"return_seg3_marketing_airline\",        # 100% missing values\n    \"return_seg3_operating_airline\",        # 100% missing values\n    \"return_seg3_aircraft_type\",            # 100% missing values\n    \"return_seg3_flight_number\",            # 100% missing values\n    \"return_seg3_segment_duration\",         # 100% missing values\n    \"outbound_seg3_baggage_quantity\",       # 100% missing values\n    \"outbound_seg3_baggage_unit\",           # 100% missing values\n    \"outbound_seg3_cabin_class\",            # 100% missing values\n    \"outbound_seg3_seats_available\",        # 100% missing values\n    \"return_seg3_baggage_quantity\",         # 100% missing values\n    \"return_seg3_baggage_unit\",             # 100% missing values\n    \"return_seg3_cabin_class\",              # 100% missing values\n    \"return_seg3_seats_available\",          # 100% missing values\n]    ","metadata":{},"outputs":[],"execution_count":null},{"id":"f775d8ee","cell_type":"markdown","source":"#### Run","metadata":{}},{"id":"fdf99782","cell_type":"code","source":"# Drop Columns\ndf_cleaned_missing_values = df_reduced.drop(columns_to_drop)\ndf_cleaned_missing_values_test = df_reduced_test.drop(columns_to_drop)","metadata":{},"outputs":[],"execution_count":null},{"id":"9c7e936e","cell_type":"markdown","source":"### Fix Incorrect DataTypes","metadata":{}},{"id":"26d95583","cell_type":"markdown","source":"#### Function","metadata":{}},{"id":"2ffbab89","cell_type":"code","source":"# dtypes = df_cleaned.dtypes\n# for col, dtype in zip(df_cleaned.columns, dtypes):\n#     print(f\"{col}: {dtype}\")","metadata":{},"outputs":[],"execution_count":null},{"id":"e9678050","cell_type":"code","source":"# ----------------------------\n# Function: Convert Booleans\n# ----------------------------\n\ndef convert_booleans_to_int8(df: pl.DataFrame, bool_cols: list[str]) -> pl.DataFrame:\n    \"\"\"\n    Convert boolean columns to Int8 (0/1) for compatibility with LightGBM.\n    \"\"\"\n    return df.with_columns([\n        pl.col(col).cast(pl.Int8) for col in bool_cols\n    ])\n\n# ----------------------------\n# Function: Extract Datetime Features\n# ----------------------------\n\ndef extract_datetime_features(df: pl.DataFrame, datetime_col: str) -> pl.DataFrame:\n    \"\"\"\n    Extract useful numeric features from datetime column,\n    then drop the original datetime column.\n    \"\"\"\n    df = df.with_columns([\n        pl.col(datetime_col).dt.hour().alias(f\"{datetime_col}_hour\"),\n        pl.col(datetime_col).dt.weekday().alias(f\"{datetime_col}_dayofweek\"),\n        pl.col(datetime_col).dt.month().alias(f\"{datetime_col}_month\"),\n    ])\n    df = df.drop(datetime_col)\n    return df\n\n# ----------------------------\n# Function: Cast Categoricals\n# ----------------------------\n\ndef cast_categoricals(df: pl.DataFrame, categorical_cols: list[str]) -> pl.DataFrame:\n    \"\"\"\n    Ensure categorical columns are stored as Utf8 strings.\n    \"\"\"\n    return df.with_columns([\n        pl.col(col).cast(pl.Utf8) for col in categorical_cols\n    ])\n\n# ----------------------------\n# Function: Convert Flag Columns\n# ----------------------------\n\ndef convert_flags_to_int8(df: pl.DataFrame, flag_cols: list[str]) -> pl.DataFrame:\n    \"\"\"\n    Convert flag columns (often stored as floats) to Int8.\n    \"\"\"\n    return df.with_columns([\n        pl.col(col).cast(pl.Int8) for col in flag_cols\n    ])\n\n# ----------------------------\n# Function: Process Durations\n# ----------------------------\n\ndef convert_duration_columns(df: pl.DataFrame, duration_cols: list[str]) -> pl.DataFrame:\n    \"\"\"\n    Convert mixed duration strings to seconds.\n    Handles:\n      - HH:MM:SS strings (e.g. \"02:40:00\")\n      - decimal hour strings (e.g. \"1.01\")\n      - nulls\n    \"\"\"\n    for col in duration_cols:\n        new_col = f\"{col}_seconds\"\n\n        # Create mask\n        mask_has_colon = df[col].str.contains(\":\").fill_null(False)\n\n        # Process rows WITH colons\n        df_with_colon = df.filter(mask_has_colon)\n        if df_with_colon.height > 0:\n            split_time = df_with_colon[col].str.split(\":\").list.eval(\n                pl.element().cast(pl.Float64)\n            )\n\n            hours = split_time.list.get(0).fill_null(0.0)\n            minutes = split_time.list.get(1).fill_null(0.0)\n            seconds = split_time.list.get(2).fill_null(0.0)\n\n            total_seconds_colon = (\n                hours * 3600.0\n                + minutes * 60.0\n                + seconds\n            ).alias(new_col)\n\n            df_with_colon = df_with_colon.with_columns(total_seconds_colon)\n        else:\n            # ensure empty frame has new column\n            df_with_colon = df.head(0).with_columns(pl.lit(None).alias(new_col))\n\n        # Process rows WITHOUT colons\n        df_no_colon = df.filter(~mask_has_colon)\n        if df_no_colon.height > 0:\n            total_seconds_decimal = (\n                df_no_colon[col]\n                .cast(pl.Float64)\n                .fill_null(0.0)\n                * 3600.0\n            ).alias(new_col)\n\n            df_no_colon = df_no_colon.with_columns(total_seconds_decimal)\n        else:\n            # ensure empty frame has new column\n            df_no_colon = df.head(0).with_columns(pl.lit(None).alias(new_col))\n\n        # Combine both halves\n        df_combined = pl.concat(\n            [df_with_colon, df_no_colon],\n            how=\"vertical\",\n            rechunk=True,\n        )\n\n        # Drop original string column\n        df_combined = df_combined.drop(col)\n\n        # replace df for next loop\n        df = df_combined\n\n    return df","metadata":{},"outputs":[],"execution_count":null},{"id":"ec23972e","cell_type":"markdown","source":"#### Run","metadata":{}},{"id":"2317302b","cell_type":"code","source":"bool_cols = [\n    \"user_gender\",\n    \"vip_status\",\n    \"booked_by_self\",\n    \"internal_flag_3d\",\n]\n\n# 1. Convert booleans\ndf_cleaned_dtype = convert_booleans_to_int8(df_cleaned_missing_values, bool_cols)\ndf_cleaned_dtype_test = convert_booleans_to_int8(df_cleaned_missing_values_test, bool_cols)","metadata":{},"outputs":[],"execution_count":null},{"id":"a38903f4","cell_type":"code","source":"datetime_col = \"search_time\"\n\n# 2. Extract datetime features\ndf_cleaned_dtype = extract_datetime_features(df_cleaned_dtype, datetime_col)\ndf_cleaned_dtype_test = extract_datetime_features(df_cleaned_dtype_test, datetime_col)","metadata":{},"outputs":[],"execution_count":null},{"id":"f0a9f370","cell_type":"code","source":"categorical_cols = [\n    \"search_session_id\",\n    \"frequent_flyer_status\",\n    \"route_type\",\n    \"outbound_departure_time\",\n    \"outbound_arrival_time\",\n    \"outbound_duration\",\n    \"return_departure_time\",\n    \"return_arrival_time\",\n    \"return_duration\",\n    \"outbound_seg0_departure_airport\",\n    \"outbound_seg0_arrival_airport\",\n    \"outbound_seg0_arrival_city\",\n    \"outbound_seg1_departure_airport\",\n    \"outbound_seg1_arrival_airport\",\n    \"outbound_seg1_arrival_city\",\n    \"outbound_seg2_departure_airport\",\n    \"outbound_seg2_arrival_airport\",\n    \"outbound_seg2_arrival_city\",\n    \"return_seg0_departure_airport\",\n    \"return_seg0_arrival_airport\",\n    \"return_seg0_arrival_city\",\n    \"return_seg1_departure_airport\",\n    \"return_seg1_arrival_airport\",\n    \"return_seg1_arrival_city\",\n    \"return_seg2_departure_airport\",\n    \"return_seg2_arrival_airport\",\n    \"return_seg2_arrival_city\",\n    \"outbound_seg0_marketing_airline\",\n    \"outbound_seg0_operating_airline\",\n    \"outbound_seg0_aircraft_type\",\n    \"outbound_seg0_flight_number\",\n    \"outbound_seg0_segment_duration\",\n    \"outbound_seg1_marketing_airline\",\n    \"outbound_seg1_operating_airline\",\n    \"outbound_seg1_aircraft_type\",\n    \"outbound_seg1_flight_number\",\n    \"outbound_seg1_segment_duration\",\n    \"outbound_seg2_marketing_airline\",\n    \"outbound_seg2_operating_airline\",\n    \"outbound_seg2_aircraft_type\",\n    \"outbound_seg2_flight_number\",\n    \"outbound_seg2_segment_duration\",\n    \"return_seg0_marketing_airline\",\n    \"return_seg0_operating_airline\",\n    \"return_seg0_aircraft_type\",\n    \"return_seg0_flight_number\",\n    \"return_seg0_segment_duration\",\n    \"return_seg1_marketing_airline\",\n    \"return_seg1_operating_airline\",\n    \"return_seg1_aircraft_type\",\n    \"return_seg1_flight_number\",\n    \"return_seg1_segment_duration\",\n    \"return_seg2_marketing_airline\",\n    \"return_seg2_operating_airline\",\n    \"return_seg2_aircraft_type\",\n    \"return_seg2_flight_number\",\n    \"return_seg2_segment_duration\"\n]\n\n# 3. Cast categoricals to Utf8\ndf_cleaned_dtype = cast_categoricals(df_cleaned_dtype, categorical_cols)\ndf_cleaned_dtype_test = cast_categoricals(df_cleaned_dtype_test, categorical_cols)","metadata":{},"outputs":[],"execution_count":null},{"id":"fae379e6","cell_type":"code","source":"flag_cols = [\n    \"cancellation_rule_status\",\n    \"exchange_rule_status\",\n    \"is_compliant_with_corporate_travel_policy\"\n]\n\n# 4. Convert flags to Int8\ndf_cleaned_dtype = convert_flags_to_int8(df_cleaned_dtype, flag_cols)\ndf_cleaned_dtype_test = convert_flags_to_int8(df_cleaned_dtype_test, flag_cols)","metadata":{},"outputs":[],"execution_count":null},{"id":"b2d72585","cell_type":"code","source":"# This takes awhile, we skip it for now\n\nduration_cols = [\n    \"outbound_duration\",\n    \"return_duration\",\n    \"outbound_seg0_segment_duration\",\n    \"outbound_seg1_segment_duration\",\n    \"outbound_seg2_segment_duration\",\n    \"return_seg0_segment_duration\",\n    \"return_seg1_segment_duration\",\n    \"return_seg2_segment_duration\",\n]\n\n# df_cleaned_dtype = convert_duration_columns(df_cleaned_dtype, duration_cols)\n\n\n# Drop the duration columsn for now\ndf_cleaned_dtype = df_cleaned_dtype.drop(duration_cols)\ndf_cleaned_dtype_test = df_cleaned_dtype_test.drop(duration_cols)","metadata":{},"outputs":[],"execution_count":null},{"id":"f16b6178","cell_type":"code","source":"print(\"Final DataFrame schema:\")\nfor item in df_cleaned_dtype.schema.items():\n    print(f\"{item[0]}: {item[1]}\")","metadata":{},"outputs":[],"execution_count":null},{"id":"9846ffcc","cell_type":"code","source":"df_cleaned_dtype.head()","metadata":{},"outputs":[],"execution_count":null},{"id":"779d92c0","cell_type":"markdown","source":"### Model Ready Check","metadata":{}},{"id":"92d18459","cell_type":"markdown","source":"#### Function","metadata":{}},{"id":"db8f86ea","cell_type":"code","source":"def check_ml_readiness(\n    df: pl.DataFrame,\n    target_col: str = None,\n    allow_nulls: bool = False,\n    verbose: bool = True\n) -> bool:\n    \"\"\"\n    Checks whether a Polars DataFrame is ready for ML modeling.\n\n    Parameters\n    ----------\n    df : pl.DataFrame\n        The DataFrame to check.\n    target_col : str, optional\n        Name of the target column to check for presence and type.\n    allow_nulls : bool, default False\n        Whether to allow missing values.\n    verbose : bool, default True\n        Whether to print the findings.\n\n    Returns\n    -------\n    is_ready : bool\n        True if the DataFrame looks ML-ready, else False.\n    \"\"\"\n\n    is_ready = True\n\n    # 1. Check for nulls\n    null_counts = df.null_count()\n    null_cols = null_counts.transpose().filter(pl.col(\"column_0\") > 0)\n\n    if not allow_nulls and null_cols.height > 0:\n        is_ready = False\n        if verbose:\n            print(\"⚠️ Columns with missing values:\")\n            print(null_cols)\n\n    # 2. Check dtypes\n    bad_cols = []\n    for col, dtype in df.schema.items():\n        if not (\n            dtype.is_numeric() or\n            dtype == pl.Utf8\n        ):\n            bad_cols.append((col, dtype))\n            is_ready = False\n\n    if bad_cols and verbose:\n        print(\"⚠️ Columns with unsupported types for ML:\")\n        for name, dtype in bad_cols:\n            print(f\"  {name}: {dtype}\")\n\n    # 3. Check duplicate columns\n    col_names = df.columns\n    duplicate_cols = set([x for x in col_names if col_names.count(x) > 1])\n    if duplicate_cols:\n        is_ready = False\n        if verbose:\n            print(\"⚠️ Duplicate column names found:\", duplicate_cols)\n\n    # 4. Check target column\n    if target_col is not None:\n        if target_col not in df.columns:\n            is_ready = False\n            if verbose:\n                print(f\"⚠️ Target column '{target_col}' not found.\")\n        else:\n            dtype = df.schema[target_col]\n            if not (dtype.is_numeric() or dtype == pl.Utf8):\n                is_ready = False\n                if verbose:\n                    print(\n                        f\"⚠️ Target column '{target_col}' has unsupported dtype: {dtype}\"\n                    )\n\n    if is_ready and verbose:\n        print(\"✅ DataFrame looks ML-ready!\")\n\n    return is_ready","metadata":{},"outputs":[],"execution_count":null},{"id":"ee5ffa69","cell_type":"markdown","source":"#### Run","metadata":{}},{"id":"91fbe3c5","cell_type":"code","source":"is_ready = check_ml_readiness(\n    df_cleaned_dtype,\n    target_col=\"flight_is_selected\",\n    allow_nulls=True,\n)\n\nis_ready_test = check_ml_readiness(\n    df_cleaned_dtype_test,\n    target_col=\"flight_is_selected\",\n    allow_nulls=True,\n)","metadata":{},"outputs":[],"execution_count":null},{"id":"8256f8b0","cell_type":"markdown","source":"# Split Data","metadata":{}},{"id":"6c439c17","cell_type":"code","source":"# skip train val for now","metadata":{},"outputs":[],"execution_count":null},{"id":"54e58a43","cell_type":"code","source":"# Define columns to exclude from features\ncols_to_exclude = [\n    \"flight_option_id\",\n    \"search_session_id\",\n    \"user_id\",\n    \"company_id\",\n    \"flight_is_selected\"\n]\n\n# Select feature columns\nfeature_cols = [\n    col for col in df_cleaned_dtype.columns\n    if col not in cols_to_exclude\n]\n\n\n# Number of training features\nprint(f\"Number of training features: {len(feature_cols)}\")\n\n# Number of test features\nprint(f\"Number of test features: {len(feature_cols)}\")\n\n# Polars → feature and target DataFrames\nX_pl = df_cleaned_dtype.select(feature_cols)\ny_pl = df_cleaned_dtype.select(\"flight_is_selected\")\ngroup_pl = df_cleaned_dtype.select(\"search_session_id\")","metadata":{},"outputs":[],"execution_count":null},{"id":"dc2877ad","cell_type":"code","source":"# ----------------------------\n# Measure time to convert Polars → pandas\n# ----------------------------\n\nstart = time.time()\n\n# Convert train set to pandas\nX = X_pl.to_pandas()\ny = y_pl.to_pandas().values.flatten()\ngroup_ids = group_pl.to_pandas().values.flatten()\n\nend = time.time()\n\nprint(f\"✅ Conversion from Polars to pandas took {end - start:.2f} seconds.\")\n\n# Optional: shape checks\nprint(\"X shape:\", X.shape)\nprint(\"y shape:\", y.shape)\nprint(\"group_ids shape:\", group_ids.shape)\n\n# ----------------------------\n# Convert test set to pandas\n# ----------------------------  \nX_test_pl = df_cleaned_dtype_test.select(feature_cols)\ngroup_test_pl = df_cleaned_dtype_test.select(\"search_session_id\")\n\nstart = time.time()\n\n# Convert test set to pandas\nX_test = X_test_pl.to_pandas()\ngroup_test_ids = group_test_pl.to_pandas().values.flatten()\nprint(\"X_test shape:\", X_test.shape)\nprint(\"group_test_ids shape:\", group_test_ids.shape)\n\nend = time.time()\n\nprint(f\"✅ Conversion from Polars to pandas took {end - start:.2f} seconds.\")","metadata":{},"outputs":[],"execution_count":null},{"id":"b77ef04e","cell_type":"code","source":"# ----------------------------\n# Compute group sizes\n# LightGBM requires group sizes, which are the counts of each unique group ID\n# This can be done using pandas for simplicity\n# ----------------------------\ngroup_sizes = (\n    pd.Series(group_ids)\n    .value_counts()\n    .sort_index()\n    .values\n)\nprint(f\"Total training rows: {len(y)}\")\nprint(f\"Number of groups: {len(group_sizes)}\")\n\n# Group sizes for test set\ngroup_test_sizes = (\n    pd.Series(group_test_ids)\n    .value_counts()\n    .sort_index()\n    .values\n)\nprint(f\"Total test rows: {len(X_test)}\")\nprint(f\"Number of test groups: {len(group_test_sizes)}\")","metadata":{},"outputs":[],"execution_count":null},{"id":"64434ebe","cell_type":"code","source":"# ----------------------------\n# Identify categorical columns\n# ----------------------------\n\ncategorical_cols = [\n    col for col in X.columns\n    if X[col].dtype == \"object\"\n]\n\nprint(f\"Found {len(categorical_cols)} categorical columns.\")\n\n# ----------------------------\n# Convert object columns → category dtype\n# ----------------------------\n\nfor col in categorical_cols:\n    X[col] = X[col].astype(\"category\")\n\nfor col in X_test.columns:\n    if col in categorical_cols:\n        X_test[col] = X_test[col].astype(\"category\")\n\nprint(\"✅ Converted object columns to category dtype.\")\n","metadata":{},"outputs":[],"execution_count":null},{"id":"f9955d51","cell_type":"code","source":"# ----------------------------\n# Create LightGBM Dataset\n# ----------------------------\ntrain_data = lgb.Dataset(\n    X,\n    label=y,\n    group=group_sizes,\n    feature_name=feature_cols,\n    categorical_feature = categorical_cols,\n)","metadata":{},"outputs":[],"execution_count":null},{"id":"a0b28adf","cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"id":"1fb43a24","cell_type":"code","source":"# skip for now ","metadata":{},"outputs":[],"execution_count":null},{"id":"ad721c0f","cell_type":"markdown","source":"# Model Training","metadata":{}},{"id":"c554e057","cell_type":"code","source":"# LightGBM model parameters\nparams = {\n    \"objective\": \"lambdarank\",\n    \"metric\": \"ndcg\",\n    \"boosting_type\": \"gbdt\",\n    \"learning_rate\": 0.05,\n    \"num_leaves\": 31,\n    \"verbosity\": -1\n}","metadata":{},"outputs":[],"execution_count":null},{"id":"baf6042a","cell_type":"code","source":"# Train LightGBM model\nmodel = lgb.train(\n    params,\n    train_data,\n    num_boost_round=100\n)","metadata":{},"outputs":[],"execution_count":null},{"id":"adfc7940","cell_type":"markdown","source":"# Feature Importance","metadata":{}},{"id":"7e453815","cell_type":"code","source":"# ----------------------------\n# Check feature importance\n# ----------------------------\n\nif not KAGGLE_SUBMISSION:\n    feature_imp = pd.DataFrame({\n        \"feature\": model.feature_name(),\n        \"importance\": model.feature_importance()\n    }).sort_values(\"importance\", ascending=False)\n\n    feature_imp.head()","metadata":{},"outputs":[],"execution_count":null},{"id":"72c65801","cell_type":"code","source":"import matplotlib.pyplot as plt\n\ntop_n = 20\n\nif not KAGGLE_SUBMISSION:\n    plt.figure(figsize=(10, 6))\n    plt.barh(\n        feature_imp[\"feature\"].head(top_n)[::-1],\n        feature_imp[\"importance\"].head(top_n)[::-1],\n        color=\"skyblue\"\n    )\n    plt.xlabel(\"Feature Importance\")\n    plt.ylabel(\"Feature\")\n    plt.title(f\"Top {top_n} Important Features (LightGBM)\")\n    plt.tight_layout()\n    plt.show()","metadata":{},"outputs":[],"execution_count":null},{"id":"cbf7262f","cell_type":"markdown","source":"# Model Evaluation on Val Set","metadata":{}},{"id":"7ce70632","cell_type":"code","source":"# pass","metadata":{},"outputs":[],"execution_count":null},{"id":"8806d757","cell_type":"markdown","source":"# Predict on Test Set","metadata":{}},{"id":"619f10b8","cell_type":"code","source":"# Predict on the test set\ntest_data = lgb.Dataset(\n    X_test,\n    group=group_test_ids,\n    feature_name=feature_cols,\n    categorical_feature=categorical_cols,\n)\npredictions = model.predict(X_test, num_iteration=model.best_iteration)","metadata":{},"outputs":[],"execution_count":null},{"id":"6eabc0e8","cell_type":"markdown","source":"# Prepare Submission File","metadata":{}},{"id":"9edaa912","cell_type":"markdown","source":"### Create DataFrame in the submission file format and append predictions","metadata":{}},{"id":"0c187809","cell_type":"code","source":"# Load sample submission file\nsample_submission = pl.read_parquet(sample_submission_file_path).drop('__index_level_0__')","metadata":{},"outputs":[],"execution_count":null},{"id":"e3ed11a5","cell_type":"code","source":"sample_submission.head()","metadata":{},"outputs":[],"execution_count":null},{"id":"6d314396","cell_type":"code","source":"# Create DataFrame from test IDs and predictions\ndf_preds = pd.DataFrame({\n    \"Id\": test_ids,    # ensure you saved this earlier\n    \"ranker_id\": group_test_ids,\n    \"selected\": predictions\n})","metadata":{},"outputs":[],"execution_count":null},{"id":"435765f0","cell_type":"code","source":"df_preds.head()","metadata":{},"outputs":[],"execution_count":null},{"id":"c1e5ebea","cell_type":"markdown","source":"### Rank predictions within each `ranker_id`","metadata":{}},{"id":"2bc55467","cell_type":"code","source":"# Assign ranks based on predictions\ndf_preds[\"selected\"] = (\n    df_preds\n    .groupby(\"ranker_id\")[\"selected\"]\n    .rank(method=\"first\", ascending=False)\n    .astype(int)\n)","metadata":{},"outputs":[],"execution_count":null},{"id":"5e4c1de6","cell_type":"code","source":"df_preds.head()","metadata":{},"outputs":[],"execution_count":null},{"id":"45bd510d","cell_type":"markdown","source":"### Validate Submission Format","metadata":{}},{"id":"8b010e2c","cell_type":"code","source":"submission_df = df_preds","metadata":{},"outputs":[],"execution_count":null},{"id":"639eeae4","cell_type":"code","source":"# 1. All Ids from test set are present\nassert len(submission_df) == len(df_preds), \"Mismatch in row count\"\n\n# 2. Ranks start from 1\nassert submission_df[\"selected\"].min() >= 1, \"Ranks should start from 1\"\n\n# 3. Ranks are integers\nassert submission_df[\"selected\"].dtype == int, \"Ranks should be integers\"\n\n# 4. Ranks are proper permutation in each group\ndef check_rank_permutation(group):\n    N = len(group)\n    expected_ranks = list(range(1, N + 1))\n    actual_ranks = sorted(group[\"selected\"].tolist())\n    if actual_ranks != expected_ranks:\n        print(f\"Invalid ranks for ranker_id: {group['ranker_id'].iloc[0]}\")\n        print(f\"Expected: {expected_ranks}, Got: {actual_ranks}\")\n        return False\n    return True\n\nvalid_ranks = submission_df.groupby(\"ranker_id\", group_keys=False).apply(check_rank_permutation)\nassert valid_ranks.all(), \"Some groups do not have valid rank permutations\"\n\nprint(\"✅ Submission passed all checks!\")","metadata":{},"outputs":[],"execution_count":null},{"id":"09cafdc0","cell_type":"code","source":"assert submission_df.shape[0] == 6897776, \"Submission DataFrame should have 6897776 rows\"","metadata":{},"outputs":[],"execution_count":null},{"id":"58d4ac30","cell_type":"markdown","source":"### Save as parquet","metadata":{}},{"id":"c7065854","cell_type":"code","source":"# Save file\nsubmission_df.to_parquet(\"data/submission/submission.parquet\", index=False)\n# submission_df.to_csv(\"submission.csv\", index=False)\n\nprint(f\"✅ Submission file saved. Shape: {submission_df.shape}\")\nprint(submission_df.head())","metadata":{},"outputs":[],"execution_count":null}]}