{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":105399,"databundleVersionId":12733338,"sourceType":"competition"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-08-16T06:40:11.411413Z","iopub.execute_input":"2025-08-16T06:40:11.411714Z","iopub.status.idle":"2025-08-16T06:40:11.768638Z","shell.execute_reply.started":"2025-08-16T06:40:11.411692Z","shell.execute_reply":"2025-08-16T06:40:11.767533Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport lightgbm as lgb\nfrom sklearn.model_selection import GroupKFold\nimport numpy as np\nimport optuna\nimport datetime\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T06:40:17.316921Z","iopub.execute_input":"2025-08-16T06:40:17.317378Z","iopub.status.idle":"2025-08-16T06:40:24.377495Z","shell.execute_reply.started":"2025-08-16T06:40:17.317350Z","shell.execute_reply":"2025-08-16T06:40:24.376631Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 1. Load Data ---\nprint(datetime.datetime.now(), \"1. Loading data...\")\ntry:\n    train_df = pd.read_parquet('/kaggle/input/aeroclub-recsys-2025/train.parquet')\n    test_df = pd.read_parquet('/kaggle/input/aeroclub-recsys-2025/test.parquet')\nexcept FileNotFoundError:\n    print(\"Error: train.parquet or test.parquet not found. Please download them from Kaggle.\")\n    exit()\n\nprint(f\"{datetime.datetime.now()} >Training data shape: {train_df.shape}\")\nprint(f\"{datetime.datetime.now()} >Test data shape: {test_df.shape}\")\n\n# --- Speed Optimization: Downcast data types to save memory and speed up operations ---\ndef downcast_dtypes(df):\n    \"\"\"\n    Downcasts numerical columns to smaller types (e.g., float64 to float32).\n    This can significantly reduce memory usage and speed up computations.\n    \"\"\"\n    for col in df.columns:\n        # Downcast floats\n        if df[col].dtype == 'float64':\n            df[col] = df[col].astype('float32')\n        # Downcast integers\n        elif df[col].dtype == 'int64':\n            # Only downcast if it fits without overflow\n            if df[col].max() < np.iinfo(np.int32).max and df[col].min() > np.iinfo(np.int32).min:\n                df[col] = df[col].astype('int32')\n            elif df[col].max() < np.iinfo(np.int16).max and df[col].min() > np.iinfo(np.int16).min:\n                df[col] = df[col].astype('int16')\n    return df\n\nprint(f\"{datetime.datetime.now()} >Optimizing data types for faster processing...\")\ntrain_df = downcast_dtypes(train_df)\ntest_df = downcast_dtypes(test_df)\nprint(f\"{datetime.datetime.now()} >Downcast data types completed...\")\n\n# The 'ranker_id' is the crucial grouping variable for this competition.\ngroup_column = 'ranker_id'\n\n# We store the target variable and the groups from the original train_df\n# before dropping the 'selected' column for feature engineering.\ny_train = train_df['selected']\ngroups = train_df[group_column]\ntrain_len = len(train_df)\n\n# Drop the target variable from the training DataFrame to prepare for concatenation\ntrain_df = train_df.drop('selected', axis=1)\nprint(f\"{datetime.datetime.now()} >Dropped target variable from training dataframe...\")\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T06:40:25.577176Z","iopub.execute_input":"2025-08-16T06:40:25.577983Z","execution_failed":"2025-08-16T06:41:15.555Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 2. Feature Engineering & Preprocessing ---\nprint(f\"\\n{datetime.datetime.now()} 2. Starting feature engineering...\")\n\n# Combine data for consistent feature engineering across train and test sets\nall_data = pd.concat([train_df, test_df], axis=0)\nprint(f\"{datetime.datetime.now()} >Combined train and test dataframes...\")\n\n# Fill missing numerical values with a placeholder.\nnumerical_features = [\n    'price', 'duration_in_minutes', 'legs0_duration_in_minutes',\n    'legs1_duration_in_minutes', 'legs0_segments0_duration',\n    'legs0_segments1_duration', 'legs1_segments0_duration',\n    'legs1_segments1_duration', 'miniRules0_monetaryAmount',\n    'miniRules0_percentage', 'miniRules1_monetaryAmount',\n    'miniRules1_percentage'\n]\n\n# Filter to only existing columns\nnumerical_features = [col for col in numerical_features if col in all_data.columns]\n\n# Convert columns to numeric before filling missing values\nfor col in numerical_features:\n    # Coerce errors will turn non-numeric values into NaN\n    all_data[col] = pd.to_numeric(all_data[col], errors='coerce')\n\nfor col in numerical_features:\n    if col in all_data.columns:\n        all_data[col] = all_data[col].fillna(-1)\n\nprint(f\"{datetime.datetime.now()} >Converted columns to numeric before filling missing values...\")\n\n# --- NEW FEATURES (Optimized for speed) ---\n# Create a temporary DataFrame to hold all new features before merging\nnew_features_df = all_data[[group_column]].copy()\nprint(f\"{datetime.datetime.now()} >Created temp copy...\")\n\n# Dynamically select columns for group stats\ngroupby_cols = [col for col in ['price', 'duration_in_minutes'] if col in all_data.columns]\nif groupby_cols:\n    group_stats = all_data.groupby(group_column)[groupby_cols].agg(['min', 'max', 'mean', 'std']).reset_index()\n    group_stats.columns = [f'{col[0]}_{col[1]}_by_group' if col[1] != '' else col[0] for col in group_stats.columns]\n    new_features_df = new_features_df.merge(group_stats, on=group_column, how='left')\n\n    # Create relative features only if the base columns exist\n    if 'price' in all_data.columns and 'price_mean_by_group' in new_features_df.columns:\n        new_features_df['price_relative_to_mean'] = all_data['price'] - new_features_df['price_mean_by_group']\n    if 'price' in all_data.columns and 'price_max_by_group' in new_features_df.columns:\n        new_features_df['price_normalized_by_max'] = all_data['price'] / new_features_df['price_max_by_group']\n    if 'duration_in_minutes' in all_data.columns and 'duration_in_minutes_mean_by_group' in new_features_df.columns:\n        new_features_df['duration_relative_to_mean'] = all_data['duration_in_minutes'] - new_features_df['duration_in_minutes_mean_by_group']\n    if 'duration_in_minutes' in all_data.columns and 'duration_in_minutes_max_by_group' in new_features_df.columns:\n        new_features_df['duration_normalized_by_max'] = all_data['duration_in_minutes'] / new_features_df['duration_in_minutes_max_by_group']\n\n# Dynamically create rank features\nif 'price' in all_data.columns:\n    new_features_df['price_rank_in_group'] = all_data.groupby(group_column)['price'].rank(ascending=True, method='dense')\nif 'duration_in_minutes' in all_data.columns:\n    new_features_df['duration_rank_in_group'] = all_data.groupby(group_column)['duration_in_minutes'].rank(ascending=True, method='dense')\n\n# Features from nested data (number of segments/legs)\nif 'legs1_duration_in_minutes' in all_data.columns:\n    new_features_df['num_legs'] = all_data['legs1_duration_in_minutes'].apply(lambda x: 2 if x > -1 else 1)\nif 'legs0_segments1_duration' in all_data.columns:\n    new_features_df['num_segments_leg0'] = all_data['legs0_segments1_duration'].apply(lambda x: 2 if x > -1 else 1)\nif 'legs1_segments1_duration' in all_data.columns:\n    new_features_df['num_segments_leg1'] = all_data['legs1_segments1_duration'].apply(lambda x: 2 if x > -1 else 1)\n\n# Feature: Number of options in the group\ngroup_size = all_data.groupby(group_column).size().reset_index(name='group_size')\nnew_features_df = new_features_df.merge(group_size, on=group_column, how='left')\n\n# Feature: Time-based features from 'requestDate' (assuming it exists)\nif 'requestDate' in all_data.columns:\n    all_data['requestDate'] = pd.to_datetime(all_data['requestDate'])\n    new_features_df['request_day_of_week'] = all_data['requestDate'].dt.dayofweek\n    new_features_df['request_hour'] = all_data['requestDate'].dt.hour\n    all_data.drop('requestDate', axis=1, inplace=True)\n\n# Merge all new features into the main DataFrame at once\nall_data = pd.concat([all_data, new_features_df.drop(group_column, axis=1)], axis=1)\nprint(f\"{datetime.datetime.now()} >Merged all features into main dataframe...\")\n\n# Identify all categorical columns to convert to the 'category' dtype\ncategorical_features = [\n    'origin_city_iata', 'destination_city_iata', 'frequentFlyer',\n    'profileId', 'companyID',\n    'legs0_segments0_aircraft_code', 'legs0_segments0_arrivalTo_airport_city_iata',\n    'legs0_segments0_arrivalTo_airport_iata', 'legs0_segments0_departureFrom_airport_iata',\n    'legs0_segments0_flightNumber', 'legs0_segments0_marketingCarrier_code',\n    'legs0_segments0_operatingCarrier_code', 'legs0_segments1_aircraft_code',\n    'legs0_segments1_arrivalTo_airport_city_iata', 'legs0_segments1_arrivalTo_airport_iata',\n    'legs0_segments1_departureFrom_airport_iata', 'legs0_segments1_flightNumber',\n    'legs0_segments1_marketingCarrier_code', 'legs0_segments1_operatingCarrier_code',\n    'legs0_segments2_aircraft_code', 'legs0_segments2_arrivalTo_airport_city_iata',\n    'legs0_segments2_arrivalTo_airport_iata', 'legs0_segments2_departureFrom_airport_iata',\n    'legs0_segments2_flightNumber', 'legs0_segments2_marketingCarrier_code',\n    'legs0_segments2_operatingCarrier_code', 'legs0_segments3_aircraft_code',\n    'legs0_segments3_arrivalTo_airport_city_iata', 'legs0_segments3_arrivalTo_airport_iata',\n    'legs0_segments3_departureFrom_airport_iata', 'legs0_segments3_flightNumber',\n    'legs0_segments3_marketingCarrier_code', 'legs0_segments3_operatingCarrier_code',\n    'legs1_segments0_aircraft_code', 'legs1_segments0_arrivalTo_airport_city_iata',\n    'legs1_segments0_arrivalTo_airport_iata', 'legs1_segments0_departureFrom_airport_iata',\n    'legs1_segments0_flightNumber', 'legs1_segments0_marketingCarrier_code',\n    'legs1_segments0_operatingCarrier_code', 'legs1_segments1_aircraft_code',\n    'legs1_segments1_arrivalTo_airport_city_iata', 'legs1_segments1_arrivalTo_airport_iata',\n    'legs1_segments1_departureFrom_airport_iata', 'legs1_segments1_flightNumber',\n    'legs1_segments1_marketingCarrier_code', 'legs1_segments1_operatingCarrier_code',\n    'legs1_segments2_aircraft_code', 'legs1_segments2_arrivalTo_airport_city_iata',\n    'legs1_segments2_arrivalTo_airport_iata', 'legs1_segments2_departureFrom_airport_iata',\n    'legs1_segments2_flightNumber', 'legs1_segments2_marketingCarrier_code',\n    'legs1_segments2_operatingCarrier_code', 'legs1_segments3_aircraft_code',\n    'legs1_segments3_arrivalTo_airport_city_iata', 'legs1_segments3_arrivalTo_airport_iata',\n    'legs1_segments3_departureFrom_airport_iata', 'legs1_segments3_flightNumber',\n    'legs1_segments3_marketingCarrier_code', 'legs1_segments3_operatingCarrier_code',\n    'searchRoute'\n]\n\n# Fill missing values in categorical columns with a placeholder string.\nprint(f\"{datetime.datetime.now()} >Filling missing values in categorical features...\")\nfor col in categorical_features:\n    if col in all_data.columns:\n        all_data[col] = all_data[col].fillna('missing')\nprint(f\"{datetime.datetime.now()} >Missing values in categorical features filled.\")\n\n# Get a list of columns to drop, including non-numeric columns that should not be used as features, and 'Id' and the group column itself.\n# We keep only the numerical features and the columns we've explicitly defined as categorical.\n# This ensures no stray 'object' columns are passed to the model.\ncolumns_to_keep = set(numerical_features + list(new_features_df.columns) + categorical_features) - set(['ranker_id'])\ncolumns_to_drop = [col for col in all_data.columns if col not in columns_to_keep]\n\n# Convert the categorical columns to the 'category' dtype\nfor col in categorical_features:\n    if col in all_data.columns:\n        all_data[col] = all_data[col].astype('category')\nprint(f\"{datetime.datetime.now()} >Converted cat columns to 'category' dtype...\")\n\n# Drop the non-feature columns\nall_data.drop(columns_to_drop, axis=1, inplace=True, errors='ignore')\n\n# List of all features to use\nfeatures = [col for col in all_data.columns if col not in ['Id', 'ranker_id']]\n\n# Separate data back into train and test\nX_train = all_data[:train_len][features].copy()\nX_test = all_data[train_len:][features].copy()\n\n# Filter categorical_features list to only include columns present in X_train\ncategorical_features_final = [col for col in categorical_features if col in X_train.columns]\n\nprint(f\"{datetime.datetime.now()} >Preprocessing complete. Starting hyperparameter tuning with Optuna...\\n\")\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 3. Define the Optuna Objective Function and Metric ---\ndef hit_rate_at_3(y_true, y_pred, groups):\n    \"\"\"\n    Calculates the HitRate@3 metric.\n    \"\"\"\n    df = pd.DataFrame({'true': y_true, 'pred': y_pred, 'group': groups})\n\n    # Filter for groups with more than 10 options, as per competition rules\n    group_counts = df.groupby('group')['group'].transform('count')\n    valid_df = df[group_counts > 10].copy()\n\n    # Get the rank of the true selection within each group\n    # We only need the rank for the true selections, not for all predictions.\n    true_selections = valid_df[valid_df['true'] == 1].copy()\n    \n    # Calculate rank for all predictions in valid groups\n    valid_df['rank'] = valid_df.groupby('group')['pred'].rank(method='first', ascending=False)\n    \n    # Check if the true selections' rank is in the top 3\n    hits = valid_df[(valid_df['true'] == 1) & (valid_df['rank'] <= 3)]\n    \n    return len(hits) / len(true_selections) if len(true_selections) > 0 else 0\n\ndef objective(trial):\n    \"\"\"Objective function for Optuna to optimize.\"\"\"\n    lgb_params = {\n        'objective': 'lambdarank',\n        'metric': 'ndcg',\n        'n_estimators': trial.suggest_int('n_estimators', 50, 100),\n        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.5, log=True),\n        'num_leaves': trial.suggest_int('num_leaves', 8, 128),\n        'max_depth': trial.suggest_int('max_depth', 3, 10),\n        'feature_fraction': trial.suggest_float('feature_fraction', 0.5, 1.0),\n        'bagging_fraction': trial.suggest_float('bagging_fraction', 0.5, 1.0),\n        'bagging_freq': trial.suggest_int('bagging_freq', 1, 10),\n        'verbose': -1,\n        'n_jobs': -1,\n        'seed': 42\n    }\n\n    N_FOLDS = 5\n    gkf = GroupKFold(n_splits=N_FOLDS)\n    oof_preds = np.zeros(len(X_train))\n\n    for fold, (train_idx, val_idx) in enumerate(gkf.split(X_train, y_train, groups=groups)):\n        X_train_fold, y_train_fold = X_train.iloc[train_idx], y_train.iloc[train_idx]\n        groups_train_fold = groups.iloc[train_idx]\n        \n        X_val_fold, y_val_fold = X_train.iloc[val_idx], y_train.iloc[val_idx]\n        groups_val_fold = groups.iloc[val_idx]\n\n        group_counts_train = groups_train_fold.value_counts(sort=False).values\n        \n        model = lgb.LGBMRanker(**lgb_params)\n        \n        model.fit(X_train_fold, y_train_fold, group=group_counts_train,\n                  eval_set=[(X_val_fold, y_val_fold)],\n                  eval_group=[groups_val_fold.value_counts(sort=False).values],\n                  eval_metric='ndcg',\n                  callbacks=[lgb.early_stopping(stopping_rounds=20, verbose=False)],\n                  categorical_feature=categorical_features_final)\n        \n        val_preds = model.predict(X_val_fold)\n        oof_preds[val_idx] = val_preds\n\n    # Calculate overall HitRate@3 on the full out-of-fold predictions\n    overall_hr3 = hit_rate_at_3(y_train, oof_preds, groups)\n    return overall_hr3\n\n# --- 4. Run the Optuna Study ---\n# Optimize for 50 trials. Can increase this for a more thorough search.\nN_TRIALS = 100\nprint(f\"{datetime.datetime.now()} 3.& 4. Starting Optuna search for {N_TRIALS} trials...\")\nstudy = optuna.create_study(direction='maximize')\nstudy.optimize(objective, n_trials=N_TRIALS)\n\nprint(f\"\\n{datetime.datetime.now()}--- Optuna Tuning Complete ---\")\nprint(f\"Best Trial Score (Overall HitRate@3): {study.best_value:.4f}\")\nprint(\"Best Hyperparameters:\")\nfor key, value in study.best_params.items():\n    print(f\"  {key}: {value}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 5. Train the Final Model with Best Parameters ---\nprint(f\"\\n{datetime.datetime.now()} 5. Training final model with the best hyperparameters found...\")\nbest_params = study.best_params\nbest_params['objective'] = 'lambdarank'\nbest_params['metric'] = 'NDCG'\nbest_params['verbose'] = -1\nbest_params['n_jobs'] = -1\nbest_params['seed'] = 42\n\n# Identify categorical features again for the final model training\ncategorical_features_final_model = [col for col in categorical_features if col in X_train.columns]\n\nfinal_model = lgb.LGBMRanker(**best_params)\n\n# Train on the full training set\nfinal_model.fit(X_train, y_train, group=groups.value_counts(sort=False).values, categorical_feature=categorical_features_final_model)\nprint(f\"\\n{datetime.datetime.now()} >Training completed...\")\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 6. Generate Predictions and Submission File ---\nprint(f\"\\n{datetime.datetime.now()} 6. Generating final submission file...\")\ntest_df['score'] = final_model.predict(X_test)\n\n# Convert scores to ranks within each 'ranker_id' group\ntest_df['rank'] = test_df.groupby(group_column)['score'].rank(method='first', ascending=False).astype(int)\n\n# Create the submission file\nsubmission = test_df[['Id', 'ranker_id', 'rank']].copy()\nsubmission = submission[['Id', 'rank']]\nsubmission.rename(columns={'rank': 'selected'}, inplace=True)\n\n# Save the submission file\nsubmission.to_csv('/kaggle/working/aeroclub-recsys-2025/submission.csv', index=False)\nprint(f\"{datetime.datetime.now()} Submission file 'submission.csv' created successfully!\")\nprint(\"\\nFirst 5 rows of the submission file:\")\nprint(submission.head())\n# print(\"\\nFirst 5 rows of the test data:\")\n# print(/kaggle/input/aeroclub-recsys-2025/test_df.head())\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}