{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":105399,"databundleVersionId":12733338,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install lightgbm pyarrow fastparquet polars","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-08-16T17:42:48.438321Z","iopub.execute_input":"2025-08-16T17:42:48.439092Z","iopub.status.idle":"2025-08-16T17:42:51.872001Z","shell.execute_reply.started":"2025-08-16T17:42:48.439063Z","shell.execute_reply":"2025-08-16T17:42:51.870805Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.preprocessing import LabelEncoder\nimport gc\nimport polars as pl # Import Polars\n\n# Define the paths to your data\nTRAIN_PATH = '/kaggle/input/aeroclub-recsys-2025/train.parquet'\nTEST_PATH = '/kaggle/input/aeroclub-recsys-2025/test.parquet'\nSUBMISSION_PATH = '/kaggle/input/aeroclub-recsys-2025/sample_submission.parquet'\n\n# Define a strict list of columns to load with Polars to save memory upfront\npolars_load_cols = [\n    'Id', 'ranker_id', 'selected',\n    'totalPrice', 'taxes',\n    'requestDate',\n    'legs0_departureAt', 'legs0_arrivalAt', 'legs0_duration',\n    'legs1_departureAt', 'legs1_arrivalAt', 'legs1_duration',\n    'searchRoute',\n    # Add other simple, directly usable columns you need from the original dataset here.\n]\n\n# Define the schema for problematic non-datetime columns explicitly.\n# CRITICAL: DO NOT include requestDate or other datetime columns here. Let Polars infer them.\ncustom_schema = {\n    \"Id\": pl.Int64, # Assuming Id is a large integer (e.g., 100, 101)\n    \"ranker_id\": pl.Utf8, # ranker_id is typically a string identifier (e.g., 'abc123')\n    \"selected\": pl.Int64, # Corrected based on previous error: must be Int64 (0 or 1)\n    \n    \"totalPrice\": pl.Float64, # Ensure these are numeric\n    \"taxes\": pl.Float64,      # Ensure these are numeric\n    \n    \"legs0_duration\": pl.Float64, # Durations are numerical, ensure float\n    \"legs1_duration\": pl.Float64,\n    \n    \"searchRoute\": pl.Utf8, # Routes are strings\n}\n\n\ntry:\n    print(\"Loading train.parquet with Polars (selected columns and explicit schema for non-datetimes)...\")\n    train_pl = pl.read_parquet(TRAIN_PATH, columns=polars_load_cols, schema=custom_schema, rechunk=False)\n    print(f\"Train data loaded into Polars DataFrame. Shape: {train_pl.shape}\")\n\n    print(\"\\nLoading test.parquet with Polars (selected columns and explicit schema for non-datetimes)...\")\n    test_load_cols = [col for col in polars_load_cols if col != 'selected']\n    test_custom_schema = {k: v for k, v in custom_schema.items() if k != 'selected'}\n    \n    test_pl = pl.read_parquet(TEST_PATH, columns=test_load_cols, schema=test_custom_schema, rechunk=False)\n    print(f\"Test data loaded into Polars DataFrame. Shape: {test_pl.shape}\")\n\n    sample_submission_df = pd.read_parquet(SUBMISSION_PATH)\n\nexcept Exception as e:\n    print(f\"Error loading data with Polars: {e}\")\n    print(\"Please ensure the dataset path is correct and accessible.\")\n    print(\"Also verify that the `custom_schema` matches the actual data types in the parquet files.\")\n\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T17:42:51.874067Z","iopub.execute_input":"2025-08-16T17:42:51.874396Z","iopub.status.idle":"2025-08-16T17:42:52.058733Z","shell.execute_reply.started":"2025-08-16T17:42:51.874366Z","shell.execute_reply":"2025-08-16T17:42:52.057892Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def feature_engineer_polars(df_pl: pl.DataFrame) -> pl.DataFrame:\n    \"\"\"\n    Creates new features using Polars expressions, assuming core numeric columns are already numeric.\n    \"\"\"\n    \n    expressions = []\n\n    # 1. Datetime Conversions (make robust to missing columns and simplified format)\n    def create_dt_expr(col_name):\n        return (\n            pl.col(col_name)\n            .str.strptime(pl.Datetime, format=\"%Y-%m-%d %H:%M:%S\", strict=False)\n            .alias(f\"{col_name}_dt\")\n        ) if col_name in df_pl.columns else pl.lit(None, dtype=pl.Datetime).alias(f\"{col_name}_dt\")\n\n    expressions.append(create_dt_expr(\"requestDate\"))\n    expressions.append(create_dt_expr(\"legs0_departureAt\"))\n    expressions.append(create_dt_expr(\"legs0_arrivalAt\"))\n    expressions.append(create_dt_expr(\"legs1_departureAt\"))\n    expressions.append(create_dt_expr(\"legs1_arrivalAt\"))\n\n    # 2. Time-based features\n    if \"legs0_departureAt_dt\" in df_pl.columns and \"requestDate_dt\" in df_pl.columns:\n        expressions.append(\n            (pl.col(\"legs0_departureAt_dt\") - pl.col(\"requestDate_dt\"))\n            .dt.total_seconds()\n            .cast(pl.Float32)\n            .fill_null(0)\n            .truediv(24 * 3600)\n            .alias(\"booking_lead_time_days\")\n        )\n    else:\n        expressions.append(pl.lit(np.nan, dtype=pl.Float32).alias(\"booking_lead_time_days\"))\n    \n    if \"legs0_departureAt_dt\" in df_pl.columns:\n        expressions.append(\n            pl.col(\"legs0_departureAt_dt\").dt.hour().cast(pl.UInt8).alias(\"departure_hour\")\n        )\n        expressions.append(\n            pl.col(\"legs0_departureAt_dt\").dt.weekday().cast(pl.UInt8).alias(\"departure_day_of_week\")\n        )\n    else:\n        expressions.append(pl.lit(np.nan, dtype=pl.UInt8).alias(\"departure_hour\"))\n        expressions.append(pl.lit(np.nan, dtype=pl.UInt8).alias(\"departure_day_of_week\"))\n\n\n    # 3. Price-based features (totalPrice and taxes are now guaranteed numeric by schema)\n    if \"totalPrice\" in df_pl.columns and \"taxes\" in df_pl.columns:\n        expressions.append(\n            (pl.col(\"totalPrice\") - pl.col(\"taxes\")).alias(\"price_before_tax\")\n        )\n    else:\n        expressions.append(pl.lit(np.nan, dtype=pl.Float32).alias(\"price_before_tax\"))\n    \n    # Total duration (legs0_duration and legs1_duration are now guaranteed numeric by schema)\n    if \"legs0_duration\" in df_pl.columns and \"legs1_duration\" in df_pl.columns:\n        total_duration_expr = pl.col(\"legs0_duration\").fill_null(0) + pl.col(\"legs1_duration\").fill_null(0)\n        expressions.append(\n            pl.col(\"totalPrice\").truediv(total_duration_expr.replace(0, pl.lit(np.nan))).alias(\"price_per_minute\")\n        )\n    else:\n        expressions.append(pl.lit(np.nan, dtype=pl.Float32).alias(\"price_per_minute\"))\n\n\n    # 4. Route-based features\n    if \"searchRoute\" in df_pl.columns:\n        expressions.append(\n            pl.col(\"searchRoute\").str.contains(\"/\").cast(pl.UInt8).alias(\"is_round_trip\")\n        )\n        expressions.append(\n            pl.col(\"searchRoute\").str.split(\"/\").arr.get(0).alias(\"origin_airport\")\n        )\n        expressions.append(\n            pl.when(pl.col(\"searchRoute\").str.contains(\"/\")).then(pl.col(\"searchRoute\").str.split(\"/\").arr.get(1))\n            .otherwise(pl.col(\"searchRoute\").str.split(\"/\").arr.get(0)).alias(\"destination_airport\")\n        )\n    else:\n        expressions.append(pl.lit(np.nan, dtype=pl.UInt8).alias(\"is_round_trip\"))\n        expressions.append(pl.lit(\"UNKNOWN\", dtype=pl.Utf8).alias(\"origin_airport\"))\n        expressions.append(pl.lit(\"UNKNOWN\", dtype=pl.Utf8).alias(\"destination_airport\"))\n\n\n    # Apply all transformations\n    df_pl = df_pl.with_columns(expressions)\n    \n    # Fill remaining NaNs for numeric columns (after all computations)\n    # Note: totalPrice, taxes, legsX_duration are already filled with 0 from schema or initial FE.\n    numeric_cols_to_fill = [\n        \"booking_lead_time_days\", \"departure_hour\", \"departure_day_of_week\",\n        \"price_before_tax\", \"price_per_minute\"\n    ]\n    \n    for c in numeric_cols_to_fill:\n        if c in df_pl.columns:\n            df_pl = df_pl.with_columns(pl.col(c).fill_null(pl.col(c).median()).alias(c))\n\n    # Convert specific string/object columns to Categorical in Polars for memory efficiency\n    categorical_cols_pl_final = [\"origin_airport\", \"destination_airport\", \"companyID\", \"profileId\",\n                               \"sex\", \"nationality\", \"corporateTariffCode\"]\n    # Add other categorical columns if they exist from the initial load\n    for c in [\"legs0_segments0_marketingCarrier_code\", \"legs0_segments0_operatingCarrier_code\",\n              \"legs0_segments0_departureFrom_airport_iata\", \"legs0_segments0_arrivalTo_airport_iata\",\n              \"legs0_segments0_cabinClass\", \"legs0_segments0_aircraft_code\"]:\n        if c in df_pl.columns:\n            categorical_cols_pl_final.append(c)\n\n    for c in categorical_cols_pl_final:\n        if c in df_pl.columns:\n            df_pl = df_pl.with_columns(pl.col(c).fill_null(\"NULL_CATEGORY\").cast(pl.Categorical).alias(c))\n    \n    return df_pl\n\nprint(\"Performing feature engineering on Polars training data...\")\ntrain_pl = feature_engineer_polars(train_pl)\nprint(f\"Train data shape after feature engineering: {train_pl.shape}\")\n\nprint(\"\\nPerforming feature engineering on Polars test data...\")\ntest_pl = feature_engineer_polars(test_pl)\nprint(f\"Test data shape after feature engineering: {test_pl.shape}\")\n\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T17:42:52.059746Z","iopub.execute_input":"2025-08-16T17:42:52.059991Z","iopub.status.idle":"2025-08-16T17:42:52.089693Z","shell.execute_reply.started":"2025-08-16T17:42:52.059962Z","shell.execute_reply":"2025-08-16T17:42:52.088266Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Converting Polars DataFrames to Pandas DataFrames...\")\n\n# Convert to Pandas\ntrain_df_pd = train_pl.to_pandas()\ntest_df_pd = test_pl.to_pandas()\n\n# Drop original datetime columns as they are now replaced by engineered features\n# Note: if any of these columns were not loaded (e.g., in minimal_cols setting), they won't exist\n# We will explicitly drop the original datetime columns that feature_engineer_polars created new versions for.\n# And drop other potentially problematic original columns if not numeric/categorical.\ncols_to_drop_after_fe = [\n    'requestDate', 'legs0_departureAt', 'legs0_arrivalAt', 'legs1_departureAt', 'legs1_arrivalAt',\n    'searchRoute' # Original searchRoute column\n]\n\ntrain_df_pd = train_df_pd.drop(columns=[col for col in cols_to_drop_after_fe if col in train_df_pd.columns])\ntest_df_pd = test_df_pd.drop(columns=[col for col in cols_to_drop_after_fe if col in test_df_pd.columns])\n\n\n# Identify categorical features for encoding in Pandas\n# Ensure we don't include Id, ranker_id, selected, or original datetimes\ncategorical_cols_pd = train_df_pd.select_dtypes(include=['object', 'category']).columns.tolist()\n# Filter out identifiers that shouldn't be encoded\ncategorical_cols_pd = [col for col in categorical_cols_pd if col not in ['Id', 'ranker_id']]\n\n\nprint(f\"Categorical columns for Label Encoding: {categorical_cols_pd}\")\n\n# Combine train and test for consistent encoding\ncombined_df_pd = pd.concat([train_df_pd.drop('selected', axis=1), test_df_pd], ignore_index=True)\n\nfor col in categorical_cols_pd:\n    if col in combined_df_pd.columns:\n        le = LabelEncoder()\n        # Convert to string first to handle any mixed types that slipped through or NaNs\n        combined_df_pd.loc[:, col] = combined_df_pd[col].astype(str)\n        combined_df_pd.loc[:, col] = le.fit_transform(combined_df_pd[col])\n\n# Separate back into training and testing sets\ntrain_df_encoded = combined_df_pd.iloc[:len(train_df_pd)].copy()\ntest_df_encoded = combined_df_pd.iloc[len(train_df_pd):].copy()\n\n# Add the target variable back to the training set\ntrain_df_encoded.loc[:, 'selected'] = train_df_pd['selected']\n\ndel train_pl, test_pl, train_df_pd, test_df_pd, combined_df_pd\ngc.collect()\n\nprint(\"Data prepared for LightGBM.\")\n\n# Define the feature set\n# Exclude original identifiers and target\nfeatures = [col for col in train_df_encoded.columns if col not in [\n    'Id', 'ranker_id', 'selected'\n]]\n\n# Remove any features that might have been generated but are all NaNs (e.g., if their source column wasn't loaded)\nfeatures = [f for f in features if not train_df_encoded[f].isnull().all()]\n# Fill any remaining NaNs in numeric features with 0 or median (should be minimal after Polars FE)\nfor col in features:\n    if pd.api.types.is_numeric_dtype(train_df_encoded[col]):\n        train_df_encoded.loc[:, col].fillna(train_df_encoded[col].median(), inplace=True)\n        test_df_encoded.loc[:, col].fillna(test_df_encoded[col].median(), inplace=True)\n\nprint(f\"Number of features selected for training: {len(features)}\")\nprint(f\"Selected features: {features}\")\n\n# Prepare data for the ranking model\nX_train = train_df_encoded[features]\ny_train = train_df_encoded['selected']\nX_test = test_df_encoded[features]\n\n# Group by ranker_id for the ranking task\ntrain_groups = train_df_encoded.groupby('ranker_id').size().to_numpy()\n\nprint(f\"Training data shape (X_train): {X_train.shape}, (y_train): {y_train.shape}\")\nprint(f\"Test data shape (X_test): {X_test.shape}\")\n\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T17:42:52.090317Z","iopub.status.idle":"2025-08-16T17:42:52.090565Z","shell.execute_reply.started":"2025-08-16T17:42:52.090446Z","shell.execute_reply":"2025-08-16T17:42:52.090457Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Identify categorical features for encoding\n# Exclude original date/time columns as they've been used for features or will be dropped\n# Exclude already processed identifiers\ncategorical_cols = [col for col in train_df.select_dtypes(include=['object', 'category']).columns.tolist() if col not in [\n    'Id', 'ranker_id', 'searchRoute', # These are identifiers or transformed\n    # Original datetime columns are now properly typed and handled, no need to exclude from this list\n    # 'requestDate', 'legs0_departureAt', 'legs0_arrivalAt', 'legs1_departureAt', 'legs1_arrivalAt'\n]]\n\nprint(f\"Categorical columns to encode: {categorical_cols}\")\n\n# Combine train and test for consistent encoding. Drop 'selected' from train_df temporarily.\n# Use .copy() to prevent SettingWithCopyWarning\ncombined_df = pd.concat([train_df.drop('selected', axis=1), test_df], ignore_index=True)\n\nfor col in categorical_cols:\n    if col in combined_df.columns:\n        le = LabelEncoder()\n        # Ensure column is string type before encoding to avoid errors with mixed types/NaN\n        combined_df.loc[:, col] = combined_df[col].astype(str)\n        combined_df.loc[:, col] = le.fit_transform(combined_df[col])\n\n# Separate back into training and testing sets\ntrain_df_encoded = combined_df.iloc[:len(train_df)].copy() # Use .copy() to prevent SettingWithCopyWarning\ntest_df_encoded = combined_df.iloc[len(train_df):].copy() # Use .copy()\n\n# Add the target variable back to the training set\ntrain_df_encoded.loc[:, 'selected'] = train_df['selected']\n\ndel combined_df\ngc.collect()\n\nprint(\"Categorical features have been encoded.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T17:42:52.091641Z","iopub.status.idle":"2025-08-16T17:42:52.091986Z","shell.execute_reply.started":"2025-08-16T17:42:52.091779Z","shell.execute_reply":"2025-08-16T17:42:52.091797Z"}},"outputs":[],"execution_count":null}]}