{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":105399,"databundleVersionId":12733338,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## IMPORTS","metadata":{}},{"cell_type":"code","source":"# ================== IMPORTS ==================\nimport pandas as pd\nimport numpy as np\nimport pyarrow.parquet as pq\nimport lightgbm as lgb\nfrom catboost import CatBoostRanker\nfrom sklearn.model_selection import GroupKFold\nimport gc\nimport re\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-22T09:29:26.533936Z","iopub.execute_input":"2025-06-22T09:29:26.534346Z","iopub.status.idle":"2025-06-22T09:29:29.020891Z","shell.execute_reply.started":"2025-06-22T09:29:26.534310Z","shell.execute_reply":"2025-06-22T09:29:29.020240Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## PARAMETERS","metadata":{}},{"cell_type":"code","source":"# ================== PARAMETERS ==================\nSAMPLE_GROUP_FRAC = 0.30  # Use 0.30 for RAM safety, increase if possible!\nTRAIN_PATH = '/kaggle/input/aeroclub-recsys-2025/train.parquet'\nTEST_PATH = '/kaggle/input/aeroclub-recsys-2025/test.parquet'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-22T09:29:34.007532Z","iopub.execute_input":"2025-06-22T09:29:34.008221Z","iopub.status.idle":"2025-06-22T09:29:34.012165Z","shell.execute_reply.started":"2025-06-22T09:29:34.008193Z","shell.execute_reply":"2025-06-22T09:29:34.011388Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## UTILITY: CONVERT DURATION","metadata":{}},{"cell_type":"code","source":"# ============= UTILITY: CONVERT DURATION =============\ndef duration_to_minutes(s):\n    \"\"\"Convert 'hh:mm:ss' or 'mm:ss' to minutes as float, else NaN.\"\"\"\n    if pd.isnull(s):\n        return np.nan\n    if isinstance(s, str):\n        parts = list(map(float, re.findall(r'\\d+', s)))\n        if len(parts) == 3:\n            return parts[0]*60 + parts[1] + parts[2]/60\n        elif len(parts) == 2:\n            return parts[0] + parts[1]/60\n        elif len(parts) == 1:\n            return parts[0]\n    return np.nan\n\ndef convert_duration_cols(df):\n    duration_cols = [col for col in df.columns if 'duration' in col]\n    for col in duration_cols:\n        df[col] = df[col].apply(duration_to_minutes)\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-22T09:29:38.385542Z","iopub.execute_input":"2025-06-22T09:29:38.385859Z","iopub.status.idle":"2025-06-22T09:29:38.392340Z","shell.execute_reply.started":"2025-06-22T09:29:38.385834Z","shell.execute_reply":"2025-06-22T09:29:38.391430Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## FEATURE ENGINEERING","metadata":{}},{"cell_type":"code","source":"# ============= FEATURE ENGINEERING =============\ndef add_features(df):\n    df['reqDate'] = pd.to_datetime(df['requestDate'], errors='coerce')\n    df['depDate'] = pd.to_datetime(df['legs0_departureAt'], errors='coerce')\n    df['hours_to_departure'] = (df['depDate'] - df['reqDate']).dt.total_seconds() / 3600\n    df['dep_hour'] = df['depDate'].dt.hour\n    df['dep_dow'] = df['depDate'].dt.dayofweek\n    df['is_cheapest'] = (df['totalPrice'] == df.groupby('ranker_id')['totalPrice'].transform('min')).astype(int)\n    df['price_rank'] = df.groupby('ranker_id')['totalPrice'].rank(method='min')\n    df['price_rel'] = df['totalPrice'] / df.groupby('ranker_id')['totalPrice'].transform('mean')\n    seg_cols = [f'legs0_segments{i}_flightNumber' for i in range(4)]\n    df['n_stops'] = df[seg_cols].notna().sum(axis=1)\n    df['is_direct'] = (df['n_stops'] == 1).astype(int)\n    df['loyalty_match'] = (df['frequentFlyer'] == df['legs0_segments0_marketingCarrier_code']).astype(int)\n    for col in ['totalPrice', 'taxes']:\n        if col in df.columns:\n            df[col + '_min'] = df.groupby('ranker_id')[col].transform('min')\n            df[col + '_max'] = df.groupby('ranker_id')[col].transform('max')\n            df[col + '_gap'] = df[col] - df[col + '_min']\n            df[col + '_pct'] = df[col] / (df[col + '_max'] + 1e-6)\n    df['group_size'] = df.groupby('ranker_id')['Id'].transform('count')\n    if 'legs0_departureAt' in df.columns:\n        df['is_earliest_dep'] = (df['legs0_departureAt'] == df.groupby('ranker_id')['legs0_departureAt'].transform('min')).astype(int)\n    if 'legs0_arrivalAt' in df.columns:\n        df['is_latest_arr'] = (df['legs0_arrivalAt'] == df.groupby('ranker_id')['legs0_arrivalAt'].transform('max')).astype(int)\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-22T09:29:42.335110Z","iopub.execute_input":"2025-06-22T09:29:42.335413Z","iopub.status.idle":"2025-06-22T09:29:42.345166Z","shell.execute_reply.started":"2025-06-22T09:29:42.335391Z","shell.execute_reply":"2025-06-22T09:29:42.344187Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## PREPROCESSING","metadata":{}},{"cell_type":"code","source":"# ============= PREPROCESSING =============\ncat_cols_cb = [\n    'frequentFlyer', 'nationality', 'legs0_segments0_aircraft_code',\n    'legs0_segments0_arrivalTo_airport_city_iata', 'legs0_segments0_arrivalTo_airport_iata',\n    'legs0_segments0_cabinClass', 'legs0_segments0_departureFrom_airport_iata',\n    'legs0_segments0_flightNumber', 'legs0_segments0_marketingCarrier_code', 'legs0_segments0_operatingCarrier_code',\n    'legs0_segments1_aircraft_code', 'legs0_segments1_arrivalTo_airport_city_iata', 'legs0_segments1_arrivalTo_airport_iata',\n    'legs0_segments1_departureFrom_airport_iata', 'legs0_segments1_flightNumber', 'legs0_segments1_marketingCarrier_code',\n    'legs0_segments1_operatingCarrier_code', 'legs0_segments2_aircraft_code', 'legs0_segments2_arrivalTo_airport_city_iata',\n    'legs0_segments2_arrivalTo_airport_iata', 'legs0_segments2_departureFrom_airport_iata', 'legs0_segments2_flightNumber',\n    'legs0_segments2_marketingCarrier_code', 'legs0_segments2_operatingCarrier_code', 'legs0_segments3_aircraft_code',\n    'legs0_segments3_arrivalTo_airport_city_iata', 'legs0_segments3_arrivalTo_airport_iata', 'legs0_segments3_departureFrom_airport_iata',\n    'legs0_segments3_flightNumber', 'legs0_segments3_marketingCarrier_code', 'legs0_segments3_operatingCarrier_code',\n    'legs1_arrivalAt', 'legs1_departureAt', 'legs1_segments0_aircraft_code', 'legs1_segments0_arrivalTo_airport_city_iata',\n    'legs1_segments0_arrivalTo_airport_iata', 'legs1_segments0_departureFrom_airport_iata', 'legs1_segments0_flightNumber',\n    'legs1_segments0_marketingCarrier_code', 'legs1_segments0_operatingCarrier_code', 'legs1_segments1_aircraft_code',\n    'legs1_segments1_arrivalTo_airport_city_iata', 'legs1_segments1_arrivalTo_airport_iata', 'legs1_segments1_departureFrom_airport_iata',\n    'legs1_segments1_flightNumber', 'legs1_segments1_marketingCarrier_code', 'legs1_segments1_operatingCarrier_code',\n    'legs1_segments2_aircraft_code', 'legs1_segments2_arrivalTo_airport_city_iata', 'legs1_segments2_arrivalTo_airport_iata',\n    'legs1_segments2_departureFrom_airport_iata', 'legs1_segments2_flightNumber', 'legs1_segments2_marketingCarrier_code',\n    'legs1_segments2_operatingCarrier_code', 'legs1_segments3_aircraft_code', 'legs1_segments3_arrivalTo_airport_city_iata',\n    'legs1_segments3_arrivalTo_airport_iata', 'legs1_segments3_departureFrom_airport_iata', 'legs1_segments3_flightNumber',\n    'legs1_segments3_marketingCarrier_code', 'legs1_segments3_operatingCarrier_code', 'searchRoute', 'sex'\n]\n\ndef preprocess(df, is_train=True):\n    # Convert durations FIRST!\n    df = convert_duration_cols(df)\n    # Datetime breakdown\n    for col in ['legs0_departureAt', 'legs0_arrivalAt', 'requestDate']:\n        if col in df.columns:\n            df[col] = pd.to_datetime(df[col], errors='coerce')\n            df[col + \"_hour\"] = df[col].dt.hour\n            df[col + \"_dow\"] = df[col].dt.dayofweek\n\n    # Categorical encoding for LightGBM (label encoding)\n    lgb_cat_cols = [\n        'frequentFlyer', 'legs0_segments0_marketingCarrier_code', 'legs0_segments0_cabinClass',\n        'legs0_segments0_arrivalTo_airport_city_iata', 'legs0_segments0_departureFrom_airport_iata',\n        'sex', 'nationality'\n    ]\n    if not hasattr(preprocess, \"label_maps\"):\n        preprocess.label_maps = {}\n    for col in lgb_cat_cols:\n        if col in df.columns:\n            df[col] = df[col].fillna(\"missing\").astype(str)\n            if is_train:\n                le_map = {v: i for i, v in enumerate(df[col].unique())}\n                preprocess.label_maps[col] = le_map\n                df[col + \"_le\"] = df[col].map(le_map).astype(int)\n            else:\n                le_map = preprocess.label_maps.get(col, {})\n                df[col + \"_le\"] = df[col].map(le_map).fillna(-1).astype(int)\n    # Fillna for numerics\n    for col in df.select_dtypes(include='number').columns:\n        df[col] = df[col].fillna(df[col].median())\n    df = add_features(df)\n    # Ensure all CatBoost categoricals are correct string/cat dtype and no NaN!\n    for col in cat_cols_cb:\n        if col in df.columns:\n            df[col] = df[col].astype(str).fillna('missing').astype('category')\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-22T09:29:49.194186Z","iopub.execute_input":"2025-06-22T09:29:49.194484Z","iopub.status.idle":"2025-06-22T09:29:49.204735Z","shell.execute_reply.started":"2025-06-22T09:29:49.194460Z","shell.execute_reply":"2025-06-22T09:29:49.203870Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## LOAD TRAIN DATA (BATCHED)","metadata":{}},{"cell_type":"code","source":"# ============= LOAD TRAIN DATA (BATCHED) =============\ntrain_file = pq.ParquetFile(TRAIN_PATH)\nsampled_chunks = []\nfor i in range(train_file.num_row_groups):\n    print(f\"Reading row group {i+1}/{train_file.num_row_groups}...\")\n    chunk = train_file.read_row_groups([i]).to_pandas()\n    group_sample = chunk['ranker_id'].drop_duplicates().sample(frac=SAMPLE_GROUP_FRAC, random_state=42)\n    sampled_chunk = chunk[chunk['ranker_id'].isin(group_sample)]\n    sampled_chunks.append(sampled_chunk)\ntrain = pd.concat(sampled_chunks, ignore_index=True)\ndel sampled_chunks, chunk, sampled_chunk; gc.collect()\nprint(\"Sampled train shape:\", train.shape)\nprint(\"Unique ranker_id:\", train['ranker_id'].nunique())\n\ntrain = preprocess(train, is_train=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-22T09:29:58.794637Z","iopub.execute_input":"2025-06-22T09:29:58.795503Z","iopub.status.idle":"2025-06-22T09:35:19.189399Z","shell.execute_reply.started":"2025-06-22T09:29:58.795473Z","shell.execute_reply":"2025-06-22T09:35:19.188458Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## SPLIT DATA","metadata":{}},{"cell_type":"code","source":"# ============= SPLIT DATA =============\ndrop_cols = [\n    'selected', 'Id', 'ranker_id', 'profileId', 'requestDate', 'legs0_departureAt', 'legs0_arrivalAt', 'depDate', 'reqDate'\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-22T09:35:28.652383Z","iopub.execute_input":"2025-06-22T09:35:28.652719Z","iopub.status.idle":"2025-06-22T09:35:28.657584Z","shell.execute_reply.started":"2025-06-22T09:35:28.652679Z","shell.execute_reply":"2025-06-22T09:35:28.656822Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# LightGBM\nX_lgb = train.drop(columns=[c for c in drop_cols if c in train.columns], errors='ignore').select_dtypes(include=[\"int\", \"float\", \"bool\"])\ny = train['selected'].astype(int)\ngroups = train.groupby('ranker_id').size().values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-22T09:35:33.826335Z","iopub.execute_input":"2025-06-22T09:35:33.826671Z","iopub.status.idle":"2025-06-22T09:35:41.063867Z","shell.execute_reply.started":"2025-06-22T09:35:33.826645Z","shell.execute_reply":"2025-06-22T09:35:41.062994Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# CatBoost: use all but drop only drop_cols\nX_cb = train.drop(columns=[c for c in drop_cols if c in train.columns], errors='ignore')\ny_cb = y.copy()\ngroups_cb = groups.copy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-22T09:35:46.744176Z","iopub.execute_input":"2025-06-22T09:35:46.744886Z","iopub.status.idle":"2025-06-22T09:35:48.983352Z","shell.execute_reply.started":"2025-06-22T09:35:46.744829Z","shell.execute_reply":"2025-06-22T09:35:48.982506Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## LIGHTGBM MODEL","metadata":{}},{"cell_type":"code","source":"# ============= LIGHTGBM MODEL =============\nparams = {\n    'objective': 'lambdarank',\n    'metric': 'ndcg',\n    'boosting_type': 'gbdt',\n    'learning_rate': 0.07,\n    'num_leaves': 51,\n    'random_state': 42,\n    'verbosity': -1,\n}\nfinal_train_data = lgb.Dataset(X_lgb, y, group=groups)\nfinal_model = lgb.train(\n    params,\n    final_train_data,\n    num_boost_round=120\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-22T09:35:55.601572Z","iopub.execute_input":"2025-06-22T09:35:55.602243Z","iopub.status.idle":"2025-06-22T09:37:44.256501Z","shell.execute_reply.started":"2025-06-22T09:35:55.602218Z","shell.execute_reply":"2025-06-22T09:37:44.255835Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## CATBOOST MODEL","metadata":{}},{"cell_type":"code","source":"# ============= CATBOOST MODEL =============\ncb_model = CatBoostRanker(\n    iterations=120,\n    learning_rate=0.07,\n    depth=8,\n    loss_function='YetiRank',\n    cat_features=cat_cols_cb,\n    random_seed=42,\n    verbose=20\n)\ncb_model.fit(X_cb, y_cb, group_id=train['ranker_id'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-22T09:38:01.094397Z","iopub.execute_input":"2025-06-22T09:38:01.094942Z","iopub.status.idle":"2025-06-22T11:29:14.881524Z","shell.execute_reply.started":"2025-06-22T09:38:01.094917Z","shell.execute_reply":"2025-06-22T11:29:14.879499Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## PREDICT ON TEST","metadata":{}},{"cell_type":"code","source":"# ============= PREDICT ON TEST =============\ntest_file = pq.ParquetFile(TEST_PATH)\ntest_chunks_lgb = []\ntest_chunks_cb = []\n\nfor i in range(test_file.num_row_groups):\n    print(f\"Predicting on test row group {i+1}/{test_file.num_row_groups}...\")\n    test_chunk = test_file.read_row_groups([i]).to_pandas()\n    test_chunk = preprocess(test_chunk, is_train=False)\n    # LightGBM\n    X_test_lgb = test_chunk.drop(columns=[c for c in drop_cols if c in test_chunk.columns], errors='ignore')\n    X_test_lgb = X_test_lgb.select_dtypes(include=[\"int\", \"float\", \"bool\"])\n    test_chunk['lgb_score'] = final_model.predict(X_test_lgb)\n    # CatBoost: All categorical string/cat, no NaN\n    X_test_cb = test_chunk.drop(columns=[c for c in drop_cols if c in test_chunk.columns], errors='ignore')\n    for col in cat_cols_cb:\n        if col in X_test_cb.columns:\n            X_test_cb[col] = X_test_cb[col].astype(str).fillna('missing').astype('category')\n    test_chunk['cb_score'] = cb_model.predict(X_test_cb)\n    test_chunks_lgb.append(test_chunk[['Id', 'ranker_id', 'lgb_score']])\n    test_chunks_cb.append(test_chunk[['Id', 'ranker_id', 'cb_score']])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-22T11:29:27.051245Z","iopub.execute_input":"2025-06-22T11:29:27.051875Z","iopub.status.idle":"2025-06-22T11:36:05.409550Z","shell.execute_reply.started":"2025-06-22T11:29:27.051850Z","shell.execute_reply":"2025-06-22T11:36:05.408857Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## ENSEMBLE AND RANK","metadata":{}},{"cell_type":"code","source":"# ============= ENSEMBLE AND RANK =============\ntest_full_lgb = pd.concat(test_chunks_lgb, ignore_index=True)\ntest_full_cb = pd.concat(test_chunks_cb, ignore_index=True)\ntest_full = test_full_lgb.merge(test_full_cb, on=['Id', 'ranker_id'])\ntest_full['ensemble_score'] = 0.5 * test_full['lgb_score'] + 0.5 * test_full['cb_score']\n\ntest_full = test_full.sort_values(['ranker_id', 'ensemble_score', 'Id'], ascending=[True, False, True])\ntest_full['selected'] = test_full.groupby('ranker_id')['ensemble_score'].rank(method='first', ascending=False).astype(int)\nsubmission = test_full[['Id', 'ranker_id', 'selected']].sort_values('Id')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-22T11:36:15.180920Z","iopub.execute_input":"2025-06-22T11:36:15.181234Z","iopub.status.idle":"2025-06-22T11:36:27.866330Z","shell.execute_reply.started":"2025-06-22T11:36:15.181211Z","shell.execute_reply":"2025-06-22T11:36:27.865617Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## DUPLICATE CHECK","metadata":{}},{"cell_type":"code","source":"# ============= DUPLICATE CHECK =============\nprint(\"Checking for duplicate or missing ranks in groups...\")\nerror_count = 0\nfor rid, group in submission.groupby('ranker_id'):\n    expected = set(range(1, len(group) + 1))\n    actual = set(group['selected'])\n    if expected != actual:\n        print(f\"❌ Error in group {rid}: expected ranks {expected}, got {actual}\")\n        error_count += 1\nif error_count == 0:\n    print(\"✅ All groups have a valid permutation of ranks (no duplicates, no gaps).\")\nelse:\n    print(f\"❌ {error_count} groups have an invalid ranking.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-22T11:36:32.686153Z","iopub.execute_input":"2025-06-22T11:36:32.686467Z","iopub.status.idle":"2025-06-22T11:36:36.231567Z","shell.execute_reply.started":"2025-06-22T11:36:32.686442Z","shell.execute_reply":"2025-06-22T11:36:36.230728Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## SUBMISSION","metadata":{}},{"cell_type":"code","source":"# ============= SAVE SUBMISSION =============\nsubmission.to_parquet('/kaggle/working/submission_5_1.parquet', index=False)\nsubmission.to_csv('/kaggle/working/submission_5_1.csv', index=False)\nprint(submission.head())\nprint(\"Submission shape:\", submission.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-22T11:36:54.442864Z","iopub.execute_input":"2025-06-22T11:36:54.443478Z","iopub.status.idle":"2025-06-22T11:37:09.897758Z","shell.execute_reply.started":"2025-06-22T11:36:54.443453Z","shell.execute_reply":"2025-06-22T11:37:09.896936Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}