{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":105399,"databundleVersionId":12733338,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-08-16T20:00:33.683978Z","iopub.execute_input":"2025-08-16T20:00:33.684429Z","iopub.status.idle":"2025-08-16T20:00:34.896818Z","shell.execute_reply.started":"2025-08-16T20:00:33.684396Z","shell.execute_reply":"2025-08-16T20:00:34.895488Z"}},"outputs":[{"name":"stdout","text":"/kaggle/input/aeroclub-recsys-2025/jsons_raw.tar.kaggle\n/kaggle/input/aeroclub-recsys-2025/train.parquet\n/kaggle/input/aeroclub-recsys-2025/sample_submission.parquet\n/kaggle/input/aeroclub-recsys-2025/jsons_structure.md\n/kaggle/input/aeroclub-recsys-2025/test.parquet\n","output_type":"stream"}],"execution_count":1},{"cell_type":"code","source":"# import pandas as pd\n# import numpy as np\n# import polars as pl\n# # Set pandas to display all columns\n# pd.set_option('display.max_columns', None)\n\n# # Define the file path for the training data\n# file_path = '/kaggle/input/aeroclub-recsys-2025/train.parquet'\n\n# # --- 1. Load the Data ---\n# # This is the fundamental command to read a Parquet file into a pandas DataFrame.\n# print(f\"Loading data from: {file_path}\")\n# try:\n#     df = pd.read_parquet(file_path)\n#     print(\"Data loaded successfully!\")\n\n#     # --- 2. Display DataFrame Head ---\n#     # Show the first few rows to confirm that the data has been loaded correctly.\n#     print(\"\\nFirst 5 rows of the DataFrame:\")\n#     print(df.head())\n\n# except Exception as e:\n#     print(f\"An error occurred while loading the data: {e}\")\n#     # Create an empty DataFrame if loading fails to prevent further errors\n#     df = pd.DataFrame()\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T20:00:34.898587Z","iopub.execute_input":"2025-08-16T20:00:34.899014Z","iopub.status.idle":"2025-08-16T20:00:34.904321Z","shell.execute_reply.started":"2025-08-16T20:00:34.898989Z","shell.execute_reply":"2025-08-16T20:00:34.903271Z"}},"outputs":[],"execution_count":2},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport polars as pl\ndf=pl.read_parquet('/kaggle/input/aeroclub-recsys-2025/train.parquet')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T20:00:34.905252Z","iopub.execute_input":"2025-08-16T20:00:34.905528Z","iopub.status.idle":"2025-08-16T20:00:49.59979Z","shell.execute_reply.started":"2025-08-16T20:00:34.905506Z","shell.execute_reply":"2025-08-16T20:00:49.598396Z"}},"outputs":[],"execution_count":3},{"cell_type":"code","source":"# import polars as pl\n\n# df = pl.scan_parquet(\"/kaggle/input/aeroclub-recsys-2025/train.parquet\")\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T20:00:49.601182Z","iopub.execute_input":"2025-08-16T20:00:49.601642Z","iopub.status.idle":"2025-08-16T20:00:49.606625Z","shell.execute_reply.started":"2025-08-16T20:00:49.60161Z","shell.execute_reply":"2025-08-16T20:00:49.605271Z"}},"outputs":[],"execution_count":4},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T20:00:49.60901Z","iopub.execute_input":"2025-08-16T20:00:49.609428Z","iopub.status.idle":"2025-08-16T20:00:49.651395Z","shell.execute_reply.started":"2025-08-16T20:00:49.609393Z","shell.execute_reply":"2025-08-16T20:00:49.650278Z"}},"outputs":[{"execution_count":5,"output_type":"execute_result","data":{"text/plain":"shape: (5, 127)\n┌─────┬────────┬───────────┬─────────────────┬───┬────────┬────────────┬──────────┬────────────────┐\n│ Id  ┆ bySelf ┆ companyID ┆ corporateTariff ┆ … ┆ taxes  ┆ totalPrice ┆ selected ┆ __index_level_ │\n│ --- ┆ ---    ┆ ---       ┆ Code            ┆   ┆ ---    ┆ ---        ┆ ---      ┆ 0__            │\n│ i64 ┆ bool   ┆ i64       ┆ ---             ┆   ┆ f64    ┆ f64        ┆ i64      ┆ ---            │\n│     ┆        ┆           ┆ i64             ┆   ┆        ┆            ┆          ┆ i64            │\n╞═════╪════════╪═══════════╪═════════════════╪═══╪════════╪════════════╪══════════╪════════════════╡\n│ 0   ┆ true   ┆ 57323     ┆ null            ┆ … ┆ 370.0  ┆ 16884.0    ┆ 1        ┆ 0              │\n│ 1   ┆ true   ┆ 57323     ┆ 123             ┆ … ┆ 2240.0 ┆ 51125.0    ┆ 0        ┆ 1              │\n│ 2   ┆ true   ┆ 57323     ┆ null            ┆ … ┆ 2240.0 ┆ 53695.0    ┆ 0        ┆ 2              │\n│ 3   ┆ true   ┆ 57323     ┆ 123             ┆ … ┆ 2240.0 ┆ 81880.0    ┆ 0        ┆ 3              │\n│ 4   ┆ true   ┆ 57323     ┆ null            ┆ … ┆ 2240.0 ┆ 86070.0    ┆ 0        ┆ 4              │\n└─────┴────────┴───────────┴─────────────────┴───┴────────┴────────────┴──────────┴────────────────┘","text/html":"<div><style>\n.dataframe > thead > tr,\n.dataframe > tbody > tr {\n  text-align: right;\n  white-space: pre-wrap;\n}\n</style>\n<small>shape: (5, 127)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>Id</th><th>bySelf</th><th>companyID</th><th>corporateTariffCode</th><th>frequentFlyer</th><th>nationality</th><th>isAccess3D</th><th>isVip</th><th>legs0_arrivalAt</th><th>legs0_departureAt</th><th>legs0_duration</th><th>legs0_segments0_aircraft_code</th><th>legs0_segments0_arrivalTo_airport_city_iata</th><th>legs0_segments0_arrivalTo_airport_iata</th><th>legs0_segments0_baggageAllowance_quantity</th><th>legs0_segments0_baggageAllowance_weightMeasurementType</th><th>legs0_segments0_cabinClass</th><th>legs0_segments0_departureFrom_airport_iata</th><th>legs0_segments0_duration</th><th>legs0_segments0_flightNumber</th><th>legs0_segments0_marketingCarrier_code</th><th>legs0_segments0_operatingCarrier_code</th><th>legs0_segments0_seatsAvailable</th><th>legs0_segments1_aircraft_code</th><th>legs0_segments1_arrivalTo_airport_city_iata</th><th>legs0_segments1_arrivalTo_airport_iata</th><th>legs0_segments1_baggageAllowance_quantity</th><th>legs0_segments1_baggageAllowance_weightMeasurementType</th><th>legs0_segments1_cabinClass</th><th>legs0_segments1_departureFrom_airport_iata</th><th>legs0_segments1_duration</th><th>legs0_segments1_flightNumber</th><th>legs0_segments1_marketingCarrier_code</th><th>legs0_segments1_operatingCarrier_code</th><th>legs0_segments1_seatsAvailable</th><th>legs0_segments2_aircraft_code</th><th>legs0_segments2_arrivalTo_airport_city_iata</th><th>&hellip;</th><th>legs1_segments2_baggageAllowance_weightMeasurementType</th><th>legs1_segments2_cabinClass</th><th>legs1_segments2_departureFrom_airport_iata</th><th>legs1_segments2_duration</th><th>legs1_segments2_flightNumber</th><th>legs1_segments2_marketingCarrier_code</th><th>legs1_segments2_operatingCarrier_code</th><th>legs1_segments2_seatsAvailable</th><th>legs1_segments3_aircraft_code</th><th>legs1_segments3_arrivalTo_airport_city_iata</th><th>legs1_segments3_arrivalTo_airport_iata</th><th>legs1_segments3_baggageAllowance_quantity</th><th>legs1_segments3_baggageAllowance_weightMeasurementType</th><th>legs1_segments3_cabinClass</th><th>legs1_segments3_departureFrom_airport_iata</th><th>legs1_segments3_duration</th><th>legs1_segments3_flightNumber</th><th>legs1_segments3_marketingCarrier_code</th><th>legs1_segments3_operatingCarrier_code</th><th>legs1_segments3_seatsAvailable</th><th>miniRules0_monetaryAmount</th><th>miniRules0_percentage</th><th>miniRules0_statusInfos</th><th>miniRules1_monetaryAmount</th><th>miniRules1_percentage</th><th>miniRules1_statusInfos</th><th>pricingInfo_isAccessTP</th><th>pricingInfo_passengerCount</th><th>profileId</th><th>ranker_id</th><th>requestDate</th><th>searchRoute</th><th>sex</th><th>taxes</th><th>totalPrice</th><th>selected</th><th>__index_level_0__</th></tr><tr><td>i64</td><td>bool</td><td>i64</td><td>i64</td><td>str</td><td>i64</td><td>bool</td><td>bool</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>f64</td><td>f64</td><td>f64</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>f64</td><td>str</td><td>str</td><td>str</td><td>f64</td><td>f64</td><td>f64</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>f64</td><td>str</td><td>str</td><td>&hellip;</td><td>f64</td><td>f64</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>f64</td><td>str</td><td>str</td><td>str</td><td>f64</td><td>f64</td><td>f64</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>i64</td><td>i64</td><td>str</td><td>datetime[ns]</td><td>str</td><td>bool</td><td>f64</td><td>f64</td><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>0</td><td>true</td><td>57323</td><td>null</td><td>&quot;S7/SU/UT&quot;</td><td>36</td><td>false</td><td>false</td><td>&quot;2024-06-15T16:20:00&quot;</td><td>&quot;2024-06-15T15:40:00&quot;</td><td>&quot;02:40:00&quot;</td><td>&quot;YK2&quot;</td><td>&quot;KJA&quot;</td><td>&quot;KJA&quot;</td><td>1.0</td><td>0.0</td><td>1.0</td><td>&quot;TLK&quot;</td><td>&quot;02:40:00&quot;</td><td>&quot;216&quot;</td><td>&quot;KV&quot;</td><td>&quot;KV&quot;</td><td>9.0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>&hellip;</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>1.0</td><td>1</td><td>2087645</td><td>&quot;98ce0dabf6964640b63079fbafd42c…</td><td>2024-05-17 03:03:08</td><td>&quot;TLKKJA/KJATLK&quot;</td><td>true</td><td>370.0</td><td>16884.0</td><td>1</td><td>0</td></tr><tr><td>1</td><td>true</td><td>57323</td><td>123</td><td>&quot;S7/SU/UT&quot;</td><td>36</td><td>true</td><td>false</td><td>&quot;2024-06-15T14:50:00&quot;</td><td>&quot;2024-06-15T09:25:00&quot;</td><td>&quot;07:25:00&quot;</td><td>&quot;E70&quot;</td><td>&quot;OVB&quot;</td><td>&quot;OVB&quot;</td><td>1.0</td><td>0.0</td><td>1.0</td><td>&quot;TLK&quot;</td><td>&quot;02:50:00&quot;</td><td>&quot;5358&quot;</td><td>&quot;S7&quot;</td><td>&quot;S7&quot;</td><td>4.0</td><td>&quot;E70&quot;</td><td>&quot;KJA&quot;</td><td>&quot;KJA&quot;</td><td>1.0</td><td>0.0</td><td>1.0</td><td>&quot;OVB&quot;</td><td>&quot;01:20:00&quot;</td><td>&quot;5311&quot;</td><td>&quot;S7&quot;</td><td>&quot;S7&quot;</td><td>4.0</td><td>null</td><td>null</td><td>&hellip;</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>2300.0</td><td>null</td><td>1.0</td><td>3500.0</td><td>null</td><td>1.0</td><td>1.0</td><td>1</td><td>2087645</td><td>&quot;98ce0dabf6964640b63079fbafd42c…</td><td>2024-05-17 03:03:08</td><td>&quot;TLKKJA/KJATLK&quot;</td><td>true</td><td>2240.0</td><td>51125.0</td><td>0</td><td>1</td></tr><tr><td>2</td><td>true</td><td>57323</td><td>null</td><td>&quot;S7/SU/UT&quot;</td><td>36</td><td>false</td><td>false</td><td>&quot;2024-06-15T14:50:00&quot;</td><td>&quot;2024-06-15T09:25:00&quot;</td><td>&quot;07:25:00&quot;</td><td>&quot;E70&quot;</td><td>&quot;OVB&quot;</td><td>&quot;OVB&quot;</td><td>1.0</td><td>0.0</td><td>1.0</td><td>&quot;TLK&quot;</td><td>&quot;02:50:00&quot;</td><td>&quot;5358&quot;</td><td>&quot;S7&quot;</td><td>&quot;S7&quot;</td><td>4.0</td><td>&quot;E70&quot;</td><td>&quot;KJA&quot;</td><td>&quot;KJA&quot;</td><td>1.0</td><td>0.0</td><td>1.0</td><td>&quot;OVB&quot;</td><td>&quot;01:20:00&quot;</td><td>&quot;5311&quot;</td><td>&quot;S7&quot;</td><td>&quot;S7&quot;</td><td>4.0</td><td>null</td><td>null</td><td>&hellip;</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>2300.0</td><td>null</td><td>1.0</td><td>3500.0</td><td>null</td><td>1.0</td><td>1.0</td><td>1</td><td>2087645</td><td>&quot;98ce0dabf6964640b63079fbafd42c…</td><td>2024-05-17 03:03:08</td><td>&quot;TLKKJA/KJATLK&quot;</td><td>true</td><td>2240.0</td><td>53695.0</td><td>0</td><td>2</td></tr><tr><td>3</td><td>true</td><td>57323</td><td>123</td><td>&quot;S7/SU/UT&quot;</td><td>36</td><td>true</td><td>false</td><td>&quot;2024-06-15T14:50:00&quot;</td><td>&quot;2024-06-15T09:25:00&quot;</td><td>&quot;07:25:00&quot;</td><td>&quot;E70&quot;</td><td>&quot;OVB&quot;</td><td>&quot;OVB&quot;</td><td>1.0</td><td>0.0</td><td>1.0</td><td>&quot;TLK&quot;</td><td>&quot;02:50:00&quot;</td><td>&quot;5358&quot;</td><td>&quot;S7&quot;</td><td>&quot;S7&quot;</td><td>4.0</td><td>&quot;E70&quot;</td><td>&quot;KJA&quot;</td><td>&quot;KJA&quot;</td><td>1.0</td><td>0.0</td><td>1.0</td><td>&quot;OVB&quot;</td><td>&quot;01:20:00&quot;</td><td>&quot;5311&quot;</td><td>&quot;S7&quot;</td><td>&quot;S7&quot;</td><td>4.0</td><td>null</td><td>null</td><td>&hellip;</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0.0</td><td>null</td><td>1.0</td><td>0.0</td><td>null</td><td>1.0</td><td>1.0</td><td>1</td><td>2087645</td><td>&quot;98ce0dabf6964640b63079fbafd42c…</td><td>2024-05-17 03:03:08</td><td>&quot;TLKKJA/KJATLK&quot;</td><td>true</td><td>2240.0</td><td>81880.0</td><td>0</td><td>3</td></tr><tr><td>4</td><td>true</td><td>57323</td><td>null</td><td>&quot;S7/SU/UT&quot;</td><td>36</td><td>false</td><td>false</td><td>&quot;2024-06-15T14:50:00&quot;</td><td>&quot;2024-06-15T09:25:00&quot;</td><td>&quot;07:25:00&quot;</td><td>&quot;E70&quot;</td><td>&quot;OVB&quot;</td><td>&quot;OVB&quot;</td><td>1.0</td><td>0.0</td><td>1.0</td><td>&quot;TLK&quot;</td><td>&quot;02:50:00&quot;</td><td>&quot;5358&quot;</td><td>&quot;S7&quot;</td><td>&quot;S7&quot;</td><td>4.0</td><td>&quot;E70&quot;</td><td>&quot;KJA&quot;</td><td>&quot;KJA&quot;</td><td>1.0</td><td>0.0</td><td>1.0</td><td>&quot;OVB&quot;</td><td>&quot;01:20:00&quot;</td><td>&quot;5311&quot;</td><td>&quot;S7&quot;</td><td>&quot;S7&quot;</td><td>4.0</td><td>null</td><td>null</td><td>&hellip;</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0.0</td><td>null</td><td>1.0</td><td>0.0</td><td>null</td><td>1.0</td><td>1.0</td><td>1</td><td>2087645</td><td>&quot;98ce0dabf6964640b63079fbafd42c…</td><td>2024-05-17 03:03:08</td><td>&quot;TLKKJA/KJATLK&quot;</td><td>true</td><td>2240.0</td><td>86070.0</td><td>0</td><td>4</td></tr></tbody></table></div>"},"metadata":{}}],"execution_count":5},{"cell_type":"code","source":"# --- 1. Identify Column Types ---\nprint(\"\\n--- Column Type Analysis ---\")\n\n# Create lists of column names based on their data type\nnumerical_cols = [col for col in df.columns if df[col].dtype in [pl.Int8, pl.Int16, pl.Int32, pl.Int64, pl.Float32, pl.Float64]]\ncategorical_cols = [col for col in df.columns if df[col].dtype == pl.String]\nother_cols = [col for col in df.columns if col not in numerical_cols and col not in categorical_cols]\n\n# Print the counts\nprint(f\"Total number of columns: {df.width}\")\nprint(f\"Number of numerical columns: {len(numerical_cols)}\")\nprint(f\"Number of categorical columns: {len(categorical_cols)}\")\nprint(f\"Number of other columns (e.g., boolean, temporal): {len(other_cols)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T20:00:49.652438Z","iopub.execute_input":"2025-08-16T20:00:49.652725Z","iopub.status.idle":"2025-08-16T20:00:49.6616Z","shell.execute_reply.started":"2025-08-16T20:00:49.6527Z","shell.execute_reply":"2025-08-16T20:00:49.660377Z"}},"outputs":[{"name":"stdout","text":"\n--- Column Type Analysis ---\nTotal number of columns: 127\nNumber of numerical columns: 49\nNumber of categorical columns: 73\nNumber of other columns (e.g., boolean, temporal): 5\n","output_type":"stream"}],"execution_count":6},{"cell_type":"code","source":"# --- 1. Identify and Store Column Names by Type ---\n\n# Create lists of column names based on their data type\nnumerical_cols = [col for col in df.columns if df[col].dtype in [pl.Int8, pl.Int16, pl.Int32, pl.Int64, pl.Float32, pl.Float64]]\ncategorical_cols = [col for col in df.columns if df[col].dtype == pl.String]\nother_cols = [col for col in df.columns if col not in numerical_cols and col not in categorical_cols]\n\n# --- 2. Print the Column Names ---\n\nprint(\"--- Numerical Column Names ---\")\nprint(numerical_cols)\n\nprint(\"\\n--- Categorical Column Names ---\")\nprint(categorical_cols)\n\nprint(\"\\n--- Other Column Names ---\")\nprint(other_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T20:00:49.662639Z","iopub.execute_input":"2025-08-16T20:00:49.662954Z","iopub.status.idle":"2025-08-16T20:00:49.684848Z","shell.execute_reply.started":"2025-08-16T20:00:49.66293Z","shell.execute_reply":"2025-08-16T20:00:49.683127Z"}},"outputs":[{"name":"stdout","text":"--- Numerical Column Names ---\n['Id', 'companyID', 'corporateTariffCode', 'nationality', 'legs0_segments0_baggageAllowance_quantity', 'legs0_segments0_baggageAllowance_weightMeasurementType', 'legs0_segments0_cabinClass', 'legs0_segments0_seatsAvailable', 'legs0_segments1_baggageAllowance_quantity', 'legs0_segments1_baggageAllowance_weightMeasurementType', 'legs0_segments1_cabinClass', 'legs0_segments1_seatsAvailable', 'legs0_segments2_baggageAllowance_quantity', 'legs0_segments2_baggageAllowance_weightMeasurementType', 'legs0_segments2_cabinClass', 'legs0_segments2_seatsAvailable', 'legs0_segments3_baggageAllowance_quantity', 'legs0_segments3_baggageAllowance_weightMeasurementType', 'legs0_segments3_cabinClass', 'legs0_segments3_seatsAvailable', 'legs1_segments0_baggageAllowance_quantity', 'legs1_segments0_baggageAllowance_weightMeasurementType', 'legs1_segments0_cabinClass', 'legs1_segments0_seatsAvailable', 'legs1_segments1_baggageAllowance_quantity', 'legs1_segments1_baggageAllowance_weightMeasurementType', 'legs1_segments1_cabinClass', 'legs1_segments1_seatsAvailable', 'legs1_segments2_baggageAllowance_quantity', 'legs1_segments2_baggageAllowance_weightMeasurementType', 'legs1_segments2_cabinClass', 'legs1_segments2_seatsAvailable', 'legs1_segments3_baggageAllowance_quantity', 'legs1_segments3_baggageAllowance_weightMeasurementType', 'legs1_segments3_cabinClass', 'legs1_segments3_seatsAvailable', 'miniRules0_monetaryAmount', 'miniRules0_percentage', 'miniRules0_statusInfos', 'miniRules1_monetaryAmount', 'miniRules1_percentage', 'miniRules1_statusInfos', 'pricingInfo_isAccessTP', 'pricingInfo_passengerCount', 'profileId', 'taxes', 'totalPrice', 'selected', '__index_level_0__']\n\n--- Categorical Column Names ---\n['frequentFlyer', 'legs0_arrivalAt', 'legs0_departureAt', 'legs0_duration', 'legs0_segments0_aircraft_code', 'legs0_segments0_arrivalTo_airport_city_iata', 'legs0_segments0_arrivalTo_airport_iata', 'legs0_segments0_departureFrom_airport_iata', 'legs0_segments0_duration', 'legs0_segments0_flightNumber', 'legs0_segments0_marketingCarrier_code', 'legs0_segments0_operatingCarrier_code', 'legs0_segments1_aircraft_code', 'legs0_segments1_arrivalTo_airport_city_iata', 'legs0_segments1_arrivalTo_airport_iata', 'legs0_segments1_departureFrom_airport_iata', 'legs0_segments1_duration', 'legs0_segments1_flightNumber', 'legs0_segments1_marketingCarrier_code', 'legs0_segments1_operatingCarrier_code', 'legs0_segments2_aircraft_code', 'legs0_segments2_arrivalTo_airport_city_iata', 'legs0_segments2_arrivalTo_airport_iata', 'legs0_segments2_departureFrom_airport_iata', 'legs0_segments2_duration', 'legs0_segments2_flightNumber', 'legs0_segments2_marketingCarrier_code', 'legs0_segments2_operatingCarrier_code', 'legs0_segments3_aircraft_code', 'legs0_segments3_arrivalTo_airport_city_iata', 'legs0_segments3_arrivalTo_airport_iata', 'legs0_segments3_departureFrom_airport_iata', 'legs0_segments3_duration', 'legs0_segments3_flightNumber', 'legs0_segments3_marketingCarrier_code', 'legs0_segments3_operatingCarrier_code', 'legs1_arrivalAt', 'legs1_departureAt', 'legs1_duration', 'legs1_segments0_aircraft_code', 'legs1_segments0_arrivalTo_airport_city_iata', 'legs1_segments0_arrivalTo_airport_iata', 'legs1_segments0_departureFrom_airport_iata', 'legs1_segments0_duration', 'legs1_segments0_flightNumber', 'legs1_segments0_marketingCarrier_code', 'legs1_segments0_operatingCarrier_code', 'legs1_segments1_aircraft_code', 'legs1_segments1_arrivalTo_airport_city_iata', 'legs1_segments1_arrivalTo_airport_iata', 'legs1_segments1_departureFrom_airport_iata', 'legs1_segments1_duration', 'legs1_segments1_flightNumber', 'legs1_segments1_marketingCarrier_code', 'legs1_segments1_operatingCarrier_code', 'legs1_segments2_aircraft_code', 'legs1_segments2_arrivalTo_airport_city_iata', 'legs1_segments2_arrivalTo_airport_iata', 'legs1_segments2_departureFrom_airport_iata', 'legs1_segments2_duration', 'legs1_segments2_flightNumber', 'legs1_segments2_marketingCarrier_code', 'legs1_segments2_operatingCarrier_code', 'legs1_segments3_aircraft_code', 'legs1_segments3_arrivalTo_airport_city_iata', 'legs1_segments3_arrivalTo_airport_iata', 'legs1_segments3_departureFrom_airport_iata', 'legs1_segments3_duration', 'legs1_segments3_flightNumber', 'legs1_segments3_marketingCarrier_code', 'legs1_segments3_operatingCarrier_code', 'ranker_id', 'searchRoute']\n\n--- Other Column Names ---\n['bySelf', 'isAccess3D', 'isVip', 'requestDate', 'sex']\n","output_type":"stream"}],"execution_count":7},{"cell_type":"code","source":"df.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T20:00:49.686177Z","iopub.execute_input":"2025-08-16T20:00:49.686717Z","iopub.status.idle":"2025-08-16T20:00:49.710541Z","shell.execute_reply.started":"2025-08-16T20:00:49.68668Z","shell.execute_reply":"2025-08-16T20:00:49.709694Z"}},"outputs":[{"execution_count":8,"output_type":"execute_result","data":{"text/plain":"(18145372, 127)"},"metadata":{}}],"execution_count":8},{"cell_type":"code","source":"# --- Missing Value Analysis (Corrected) ---\nprint(\"\\n--- Missing Value Analysis ---\")\n\n# Calculate the number of nulls in every column\nmissing_values_df = df.null_count()\n\n# Melt the DataFrame to turn it from wide to long format\n# This creates a 'variable' column (with original column names) \n# and a 'value' column (with the null counts)\nmelted_missing = missing_values_df.melt()\n\n# Rename columns for clarity\nmelted_missing = melted_missing.rename({\"variable\": \"column\", \"value\": \"null_count\"})\n\n# Now, you can filter on the 'null_count' column because it actually exists\ncolumns_with_missing = melted_missing.filter(pl.col(\"null_count\") > 0)\n\n# Get the total number of columns that have missing data\nnum_cols_with_missing = columns_with_missing.height\n\nif num_cols_with_missing > 0:\n    print(f\"Found {num_cols_with_missing} columns with missing values:\")\n    \n    # Sort the results to show the columns with the most missing values first\n    sorted_missing = columns_with_missing.sort(\"null_count\", descending=True)\n    \n    print(sorted_missing)\nelse:\n    print(\"No missing values found in the dataset.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T20:00:49.711875Z","iopub.execute_input":"2025-08-16T20:00:49.712143Z","iopub.status.idle":"2025-08-16T20:00:49.764037Z","shell.execute_reply.started":"2025-08-16T20:00:49.712112Z","shell.execute_reply":"2025-08-16T20:00:49.76305Z"}},"outputs":[{"name":"stdout","text":"\n--- Missing Value Analysis ---\nFound 103 columns with missing values:\nshape: (103, 2)\n┌─────────────────────────────────┬────────────┐\n│ column                          ┆ null_count │\n│ ---                             ┆ ---        │\n│ str                             ┆ u32        │\n╞═════════════════════════════════╪════════════╡\n│ legs1_segments3_aircraft_code   ┆ 18145366   │\n│ legs1_segments3_arrivalTo_airp… ┆ 18145366   │\n│ legs1_segments3_arrivalTo_airp… ┆ 18145366   │\n│ legs1_segments3_baggageAllowan… ┆ 18145366   │\n│ legs1_segments3_baggageAllowan… ┆ 18145366   │\n│ …                               ┆ …          │\n│ legs0_segments0_baggageAllowan… ┆ 1064       │\n│ legs0_segments0_arrivalTo_airp… ┆ 113        │\n│ legs0_segments0_aircraft_code   ┆ 14         │\n│ legs0_segments0_arrivalTo_airp… ┆ 6          │\n│ legs0_segments0_departureFrom_… ┆ 2          │\n└─────────────────────────────────┴────────────┘\n","output_type":"stream"},{"name":"stderr","text":"/tmp/ipykernel_210/1011474964.py:10: DeprecationWarning: `DataFrame.melt` is deprecated. Use `unpivot` instead, with `index` instead of `id_vars` and `on` instead of `value_vars`\n  melted_missing = missing_values_df.melt()\n","output_type":"stream"}],"execution_count":9},{"cell_type":"code","source":"# --- Assume df (Polars DataFrame) is already loaded ---\n\n# 1. Identify all string (categorical) columns\ncategorical_cols = [col for col in df.columns if df[col].dtype == pl.String]\n\n# 2. Define the missing value threshold (80%)\nthreshold = len(df) * 0.80\n\n# 3. Find and list the columns exceeding the threshold in one chain\ncols_to_remove = (\n    df.select(categorical_cols)\n    .null_count()\n    .melt()\n    .filter(pl.col(\"value\") > threshold)\n    .get_column(\"variable\")\n    .to_list()\n)\n\nprint(f\"Categorical columns with >80% missing values: {cols_to_remove}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T20:00:49.765117Z","iopub.execute_input":"2025-08-16T20:00:49.765392Z","iopub.status.idle":"2025-08-16T20:00:49.781805Z","shell.execute_reply.started":"2025-08-16T20:00:49.765371Z","shell.execute_reply":"2025-08-16T20:00:49.780706Z"}},"outputs":[{"name":"stdout","text":"Categorical columns with >80% missing values: ['legs0_segments2_aircraft_code', 'legs0_segments2_arrivalTo_airport_city_iata', 'legs0_segments2_arrivalTo_airport_iata', 'legs0_segments2_departureFrom_airport_iata', 'legs0_segments2_duration', 'legs0_segments2_flightNumber', 'legs0_segments2_marketingCarrier_code', 'legs0_segments2_operatingCarrier_code', 'legs0_segments3_aircraft_code', 'legs0_segments3_arrivalTo_airport_city_iata', 'legs0_segments3_arrivalTo_airport_iata', 'legs0_segments3_departureFrom_airport_iata', 'legs0_segments3_duration', 'legs0_segments3_flightNumber', 'legs0_segments3_marketingCarrier_code', 'legs0_segments3_operatingCarrier_code', 'legs1_segments1_aircraft_code', 'legs1_segments1_arrivalTo_airport_city_iata', 'legs1_segments1_arrivalTo_airport_iata', 'legs1_segments1_departureFrom_airport_iata', 'legs1_segments1_duration', 'legs1_segments1_flightNumber', 'legs1_segments1_marketingCarrier_code', 'legs1_segments1_operatingCarrier_code', 'legs1_segments2_aircraft_code', 'legs1_segments2_arrivalTo_airport_city_iata', 'legs1_segments2_arrivalTo_airport_iata', 'legs1_segments2_departureFrom_airport_iata', 'legs1_segments2_duration', 'legs1_segments2_flightNumber', 'legs1_segments2_marketingCarrier_code', 'legs1_segments2_operatingCarrier_code', 'legs1_segments3_aircraft_code', 'legs1_segments3_arrivalTo_airport_city_iata', 'legs1_segments3_arrivalTo_airport_iata', 'legs1_segments3_departureFrom_airport_iata', 'legs1_segments3_duration', 'legs1_segments3_flightNumber', 'legs1_segments3_marketingCarrier_code', 'legs1_segments3_operatingCarrier_code']\n","output_type":"stream"},{"name":"stderr","text":"/tmp/ipykernel_210/1503780565.py:13: DeprecationWarning: `DataFrame.melt` is deprecated. Use `unpivot` instead, with `index` instead of `id_vars` and `on` instead of `value_vars`\n  .melt()\n","output_type":"stream"}],"execution_count":10},{"cell_type":"code","source":"# --- Assume lf (Polars LazyFrame) is already defined ---\nfile_path = '/kaggle/input/aeroclub-recsys-2025/train.parquet'\nlf = pl.scan_parquet(file_path)\n# 1. Identify all numerical columns from the LazyFrame's schema\n# --- Assume lf (Polars LazyFrame) is already defined ---\n\n# 1. Identify all numerical columns from the LazyFrame's schema\nnumerical_cols = [\n    col for col, dtype in lf.schema.items()\n    if dtype in [pl.Int8, pl.Int16, pl.Int32, pl.Int64, pl.Float32, pl.Float64]\n]\n\n# 2. Efficiently calculate the total number of rows\n# 3. Define the missing value threshold (95%)\nthreshold =len(df) * 0.95\n\n# 4. Find and list the numerical columns exceeding the threshold\n# We use .unpivot() which is the new name for .melt()\nnumerical_cols_to_remove = (\n     df.select(numerical_cols)\n    .null_count()\n    .melt()\n    .filter(pl.col(\"value\") > threshold)\n    .get_column(\"variable\")\n    .to_list()\n)\n\nprint(f\"Numerical columns with >95% missing values: {numerical_cols_to_remove}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T20:00:49.782799Z","iopub.execute_input":"2025-08-16T20:00:49.783118Z","iopub.status.idle":"2025-08-16T20:00:49.808Z","shell.execute_reply.started":"2025-08-16T20:00:49.783077Z","shell.execute_reply":"2025-08-16T20:00:49.806934Z"}},"outputs":[{"name":"stdout","text":"Numerical columns with >95% missing values: ['legs0_segments2_baggageAllowance_quantity', 'legs0_segments2_baggageAllowance_weightMeasurementType', 'legs0_segments2_cabinClass', 'legs0_segments2_seatsAvailable', 'legs0_segments3_baggageAllowance_quantity', 'legs0_segments3_baggageAllowance_weightMeasurementType', 'legs0_segments3_cabinClass', 'legs0_segments3_seatsAvailable', 'legs1_segments2_baggageAllowance_quantity', 'legs1_segments2_baggageAllowance_weightMeasurementType', 'legs1_segments2_cabinClass', 'legs1_segments2_seatsAvailable', 'legs1_segments3_baggageAllowance_quantity', 'legs1_segments3_baggageAllowance_weightMeasurementType', 'legs1_segments3_cabinClass', 'legs1_segments3_seatsAvailable', 'miniRules0_percentage', 'miniRules1_percentage']\n","output_type":"stream"},{"name":"stderr","text":"/tmp/ipykernel_210/761606392.py:9: PerformanceWarning: Resolving the schema of a LazyFrame is a potentially expensive operation. Use `LazyFrame.collect_schema()` to get the schema without this warning.\n  col for col, dtype in lf.schema.items()\n/tmp/ipykernel_210/761606392.py:22: DeprecationWarning: `DataFrame.melt` is deprecated. Use `unpivot` instead, with `index` instead of `id_vars` and `on` instead of `value_vars`\n  .melt()\n","output_type":"stream"}],"execution_count":11},{"cell_type":"code","source":"# import polars as pl\n# import polars.selectors as cs\n# import xgboost as xgb\n# from sklearn.model_selection import GroupKFold\n# import numpy as np\n# from pathlib import Path\n# import json\n# import gc\n\n# # --- 0. Setup: Define Paths and Thresholds ---\n# TRAIN_PATH = \"/kaggle/input/aeroclub-recsys-2025/train.parquet\"\n# TEST_PATH = \"/kaggle/input/aeroclub-recsys-2025/test.parquet\"\n# SUBMISSION_PATH = \"/kaggle/working/submission.csv\"\n# WORK_DIR = Path(\"/kaggle/working\")\n\n# # Missing value thresholds from your EDA\n# CAT_MISSING_FRAC = 0.80\n# NUM_MISSING_FRAC = 0.95\n# BATCH_SIZE = 64 # For memory-safe null counting\n\n# # --- 1. Data Cleaning and Preprocessing Function ---\n# # This function encapsulates all cleaning, imputation, and encoding steps.\n# def preprocess(lf: pl.LazyFrame, is_train=True) -> pl.LazyFrame:\n#     \"\"\"Applies a full cleaning and encoding pipeline to a LazyFrame.\"\"\"\n#     print(\"Starting preprocessing...\")\n#     schema = lf.collect_schema()\n    \n#     categorical_cols = [c for c, t in schema.items() if t == pl.String]\n#     numerical_cols = [c for c, t in schema.items() if t in {pl.Int8, pl.Int16, pl.Int32, pl.Int64, pl.Float32, pl.Float64}]\n    \n#     if is_train:\n#         total_rows = lf.select(pl.len()).collect().item()\n#         thr_cat = int(total_rows * CAT_MISSING_FRAC)\n#         thr_num = int(total_rows * NUM_MISSING_FRAC)\n        \n#         # Memory-safe null counting in batches\n#         inspect_cols = categorical_cols + numerical_cols\n#         null_counts = {}\n#         for i in range(0, len(inspect_cols), BATCH_SIZE):\n#             subset = inspect_cols[i:i + BATCH_SIZE]\n#             res = lf.select([pl.col(c).null_count().alias(c) for c in subset]).collect(streaming=True)\n#             null_counts.update(res.to_dicts()[0])\n\n#         drop_cols = []\n#         for col_name, null_count in null_counts.items():\n#             if col_name in categorical_cols and null_count > thr_cat:\n#                 drop_cols.append(col_name)\n#             elif col_name in numerical_cols and null_count > thr_num:\n#                 drop_cols.append(col_name)\n        \n#         with open(WORK_DIR / \"drop_cols.json\", \"w\") as f:\n#             json.dump(drop_cols, f)\n#         print(f\"Identified {len(drop_cols)} columns to drop based on missing values.\")\n#     else:\n#         with open(WORK_DIR / \"drop_cols.json\", \"r\") as f:\n#             drop_cols = json.load(f)\n\n#     remaining_cols = [c for c in schema.keys() if c not in drop_cols]\n#     string_cols_to_encode = [c for c in remaining_cols if schema[c] == pl.String]\n#     numeric_cols_to_impute = [c for c in remaining_cols if schema[c] in {pl.Int8, pl.Int16, pl.Int32, pl.Int64, pl.Float32, pl.Float64}]\n    \n#     processed_lf = (\n#         lf.select(remaining_cols)\n#         .with_columns(\n#             *[pl.col(c).fill_null(pl.col(c).median()) for c in numeric_cols_to_impute],\n#             *[pl.col(c).fill_null(\"__MISSING__\") for c in string_cols_to_encode]\n#         )\n#         .with_columns(\n#             *[pl.col(c).cast(pl.Categorical).to_physical() for c in string_cols_to_encode]\n#         )\n#     )\n#     print(\"Preprocessing plan built.\")\n#     return processed_lf\n\n# # --- 2. Feature Engineering Function ---\n# def create_features(df: pl.LazyFrame) -> pl.LazyFrame:\n#     print(\"Building feature engineering plan...\")\n#     # Convert duration strings to minutes for calculations\n#     duration_cols = [c for c in df.columns if \"duration\" in c and df.schema[c] == pl.String]\n    \n#     df = df.with_columns([\n#         pl.col(c).str.to_duration().dt.total_minutes().alias(c) for c in duration_cols\n#     ])\n    \n#     return (\n#         df.with_columns(\n#             (pl.col(\"legs0_duration\").fill_null(0) + pl.col(\"legs1_duration\").fill_null(0)).alias(\"total_duration\"),\n#         ).with_columns(\n#             pl.col(\"totalPrice\").rank(method=\"ordinal\").over(\"ranker_id\").alias(\"price_rank_in_group\"),\n#             (pl.col(\"totalPrice\") - pl.col(\"totalPrice\").min().over(\"ranker_id\")).alias(\"price_diff_from_min\"),\n#             (pl.col(\"totalPrice\") == pl.col(\"totalPrice\").min().over(\"ranker_id\")).cast(pl.Int8).alias(\"is_cheapest_in_group\"),\n#             pl.col(\"total_duration\").rank(method=\"ordinal\").over(\"ranker_id\").alias(\"duration_rank_in_group\"),\n#             (pl.col(\"total_duration\") - pl.col(\"total_duration\").min().over(\"ranker_id\")).alias(\"duration_diff_from_min\"),\n#             (pl.col(\"total_duration\") == pl.col(\"total_duration\").min().over(\"ranker_id\")).cast(pl.Int8).alias(\"is_fastest_in_group\"),\n#         )\n#     )\n\n# # --- 3. Run Pipeline on Training Data Lazily ---\n# train_lf = pl.scan_parquet(TRAIN_PATH)\n# train_processed_lf = preprocess(train_lf, is_train=True)\n# train_featured_lf = create_features(train_processed_lf)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T20:00:49.809169Z","iopub.execute_input":"2025-08-16T20:00:49.809551Z","iopub.status.idle":"2025-08-16T20:00:49.81724Z","shell.execute_reply.started":"2025-08-16T20:00:49.809519Z","shell.execute_reply":"2025-08-16T20:00:49.816242Z"}},"outputs":[],"execution_count":12},{"cell_type":"code","source":"# --- 4. Create Validation Split ---\nprint(\"\\nCreating train/validation split...\")\ngroups_for_split = train_lf.select(\"ranker_id\").collect(streaming=True)\ngkf = GroupKFold(n_splits=5)\ntrain_idx, val_idx = next(gkf.split(X=groups_for_split, groups=groups_for_split[\"ranker_id\"]))\ndel groups_for_split; gc.collect()\n\nfeature_names = [col for col in train_featured_lf.columns if col not in [\"Id\", \"ranker_id\", \"selected\"]]\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T20:00:49.818167Z","iopub.execute_input":"2025-08-16T20:00:49.818503Z","iopub.status.idle":"2025-08-16T20:00:49.945261Z","shell.execute_reply.started":"2025-08-16T20:00:49.818474Z","shell.execute_reply":"2025-08-16T20:00:49.94356Z"}},"outputs":[{"name":"stdout","text":"\nCreating train/validation split...\n","output_type":"stream"},{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)","\u001b[0;32m/tmp/ipykernel_210/4152538858.py\u001b[0m in \u001b[0;36m<cell line: 0>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# --- 4. Create Validation Split ---\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"\\nCreating train/validation split...\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mgroups_for_split\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_lf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mselect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"ranker_id\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstreaming\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mgkf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mGroupKFold\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_splits\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mtrain_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_idx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgkf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgroups_for_split\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgroups\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgroups_for_split\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"ranker_id\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mNameError\u001b[0m: name 'train_lf' is not defined"],"ename":"NameError","evalue":"name 'train_lf' is not defined","output_type":"error"}],"execution_count":13},{"cell_type":"code","source":"# --- 5. Prepare DMatrix for XGBoost (Just-in-Time) ---\nprint(\"\\nPreparing DMatrix for XGBoost (Train Split)...\")\nX_train = train_featured_lf.filter(pl.col('__index_level_0__').is_in(train_idx)).select(feature_names).collect(streaming=True)\ny_train = train_featured_lf.filter(pl.col('__index_level_0__').is_in(train_idx)).select(\"selected\").collect(streaming=True)\ngroups_train = train_featured_lf.filter(pl.col('__index_level_0__').is_in(train_idx)).select(\"ranker_id\").collect(streaming=True)\ntrain_group_sizes = groups_train.group_by(\"ranker_id\", maintain_order=True).len()[\"len\"].to_numpy()\ndtrain = xgb.DMatrix(X_train, label=y_train, group=train_group_sizes, feature_names=feature_names)\ndel X_train, y_train, groups_train; gc.collect()\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T20:00:49.945815Z","iopub.status.idle":"2025-08-16T20:00:49.946096Z","shell.execute_reply.started":"2025-08-16T20:00:49.945967Z","shell.execute_reply":"2025-08-16T20:00:49.945979Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Preparing DMatrix for XGBoost (Validation Split)...\")\nX_val = train_featured_lf.filter(pl.col('__index_level_0__').is_in(val_idx)).select(feature_names).collect(streaming=True)\ny_val = train_featured_lf.filter(pl.col('__index_level_0__').is_in(val_idx)).select(\"selected\").collect(streaming=True)\ngroups_val = train_featured_lf.filter(pl.col('__index_level_0__').is_in(val_idx)).select(\"ranker_id\").collect(streaming=True)\nval_group_sizes = groups_val.group_by(\"ranker_id\", maintain_order=True).len()[\"len\"].to_numpy()\ndval = xgb.DMatrix(X_val, label=y_val, group=val_group_sizes, feature_names=feature_names)\ndel X_val, y_val, groups_val; gc.collect()\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T20:00:49.947118Z","iopub.status.idle":"2025-08-16T20:00:49.947463Z","shell.execute_reply.started":"2025-08-16T20:00:49.947273Z","shell.execute_reply":"2025-08-16T20:00:49.947287Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 6. Train the XGBoost Model ---\nprint(\"\\nTraining XGBoost model...\")\nxgb_params = {'objective': 'rank:pairwise',\n              'eval_metric': 'ndcg@3',\n              'eta': 0.05,\n              'max_depth': 7,\n              'seed': 42,\n              'tree_method': 'hist'}\nmodel = xgb.train(params=xgb_params, dtrain=dtrain, num_boost_round=1000, evals=[(dtrain, 'train'), (dval, 'eval')], early_stopping_rounds=50, verbose_eval=50)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T20:00:49.948688Z","iopub.status.idle":"2025-08-16T20:00:49.948982Z","shell.execute_reply.started":"2025-08-16T20:00:49.948843Z","shell.execute_reply":"2025-08-16T20:00:49.948855Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 7. Run Pipeline on Test Data ---\nprint(\"\\nProcessing test data for prediction...\")\ntest_lf = pl.scan_parquet(TEST_PATH)\ntest_processed_lf = preprocess(test_lf, is_train=False)\ntest_featured_lf = create_features(test_processed_lf)\n\nX_test = test_featured_lf.select(feature_names).collect(streaming=True)\ntest_groups = test_featured_lf.select(\"ranker_id\").collect(streaming=True)\ntest_group_sizes = test_groups.group_by(\"ranker_id\", maintain_order=True).len()[\"len\"].to_numpy()\ndtest = xgb.DMatrix(X_test, group=test_group_sizes, feature_names=feature_names)\ndel X_test, test_groups; gc.collect()\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T20:00:49.950801Z","iopub.status.idle":"2025-08-16T20:00:49.951111Z","shell.execute_reply.started":"2025-08-16T20:00:49.950948Z","shell.execute_reply":"2025-08-16T20:00:49.95096Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 8. Generate Predictions and Submission File ---\nprint(\"\\nGenerating predictions and creating submission file...\")\npredictions = model.predict(dtest)\n\ntest_original_df = pl.read_parquet(TEST_PATH)\n# Collect only the necessary columns from the lazy frame\nsubmission_df = test_featured_lf.select([\"Id\", \"ranker_id\"]).collect(streaming=True).with_columns(pl.Series(name=\"prediction_score\", values=predictions))\nsubmission_df = submission_df.with_columns(pl.col(\"prediction_score\").rank(method=\"ordinal\", descending=True).over(\"ranker_id\").cast(pl.Int32).alias(\"selected\"))\n\nfinal_submission = test_original_df.select(\"Id\").join(submission_df.select([\"Id\", \"ranker_id\", \"selected\"]), on=\"Id\", how=\"left\")\n\nfinal_submission.write_csv(SUBMISSION_PATH)\nprint(f\"Submission file saved successfully to: {SUBMISSION_PATH}\")\nprint(\"\\nFirst 5 rows of submission file:\")\nprint(final_submission.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T20:00:49.952546Z","iopub.status.idle":"2025-08-16T20:00:49.952832Z","shell.execute_reply.started":"2025-08-16T20:00:49.9527Z","shell.execute_reply":"2025-08-16T20:00:49.952714Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 1: Setup and Environment\n# ===========================\nprint(\"Step 1: Setting up environment...\")\n!pip install -U polars xgboost pyarrow\n\nimport polars as pl\nimport xgboost as xgb\nimport numpy as np\nimport gc\nimport os\nimport shutil\n\n# Define file paths and constants\nTRAIN_PATH = '/kaggle/input/aeroclub-recsys-2025/train.parquet'\nTEST_PATH = '/kaggle/input/aeroclub-recsys-2025/test.parquet'\nRANDOM_STATE = 42\n# BATCH_SIZE determines how many unique search sessions are processed at a time.\nRANKER_ID_BATCH_SIZE = 40000\n# Directory to store intermediate processed files\nPROCESSED_DIR = \"/kaggle/working/processed_batches/\"\n\n# XGBoost parameters from the original notebook\nXGB_PARAMS = {\n    'objective': 'rank:pairwise',\n    'eval_metric': 'ndcg@3',\n    'learning_rate': 0.022641389657079056,\n    'max_depth': 14,\n    'min_child_weight': 2,\n    'subsample': 0.8842234913702768,\n    'colsample_bytree': 0.45840689146263086,\n    'gamma': 3.3084297630544888,\n    'lambda': 6.952586917313028,\n    'alpha': 0.6395254133055179,\n    'seed': RANDOM_STATE,\n    'n_jobs': -1,\n}\n\n# Step 2: Expanded Column Selection & Aggregate Features\n# =======================================================\nREQUIRED_COLS = [\n    'Id', 'ranker_id', 'selected', 'totalPrice', 'taxes', 'legs0_duration', 'legs1_duration',\n    'legs0_segments0_duration', 'legs0_segments1_duration', 'legs0_segments2_duration',\n    'legs1_segments0_duration', 'legs1_segments1_duration', 'legs1_segments2_duration',\n    'legs0_departureAt', 'legs0_arrivalAt', 'legs1_departureAt', 'legs1_arrivalAt',\n    'frequentFlyer', 'corporateTariffCode',\n    'legs0_segments0_marketingCarrier_code', 'legs1_segments0_marketingCarrier_code',\n    'legs0_segments0_departureFrom_airport_iata', 'legs0_segments0_arrivalTo_airport_iata',\n    'legs1_segments0_departureFrom_airport_iata', 'legs1_segments0_arrivalTo_airport_iata',\n    'isVip', 'bySelf', 'legs0_segments0_cabinClass', 'legs1_segments0_cabinClass',\n    'miniRules0_monetaryAmount', 'miniRules1_monetaryAmount',\n    'legs0_segments0_baggageAllowance_quantity', 'legs1_segments0_baggageAllowance_quantity'\n]\n\ndef create_aggregate_features(train_path: str):\n    print(\"Step 2: Pre-computing richer aggregate features...\")\n    train_schema = pl.read_parquet_schema(train_path)\n    cols_to_scan = [c for c in REQUIRED_COLS if c in train_schema and c != 'Id']\n    scan_df = pl.scan_parquet(train_path).select(cols_to_scan)\n\n    carrier0_pop = scan_df.group_by(\"legs0_segments0_marketingCarrier_code\").agg(pl.mean('selected').alias('carrier0_pop')).collect(streaming=True)\n    carrier1_pop = scan_df.group_by(\"legs1_segments0_marketingCarrier_code\").agg(pl.mean('selected').alias('carrier1_pop')).collect(streaming=True)\n    round_trip_freq = (\n        scan_df\n        .with_columns(\n            (pl.col(\"legs0_segments0_departureFrom_airport_iata\") + \"_\" +\n             pl.col(\"legs0_segments0_arrivalTo_airport_iata\") + \"__\" +\n             pl.col(\"legs1_segments0_departureFrom_airport_iata\") + \"_\" +\n             pl.col(\"legs1_segments0_arrivalTo_airport_iata\")).alias(\"round_trip_route\")\n        )\n        .group_by(\"round_trip_route\")\n        .agg(pl.len().alias(\"round_trip_freq\"))\n        .collect(streaming=True)\n    )\n    \n    print(\"Aggregate features created.\")\n    return carrier0_pop, carrier1_pop, round_trip_freq\n\n\n# Step 3: Expanded Feature Engineering Pipeline\n# ===============================================\ndef feature_engineer(df: pl.DataFrame, carrier0_aggs: pl.DataFrame, carrier1_aggs: pl.DataFrame, route_aggs: pl.DataFrame) -> pl.DataFrame:\n    def duration_to_minutes(col_name):\n        return (\n            pl.col(col_name).str.extract(r\"(\\d+)\\.\", 1).cast(pl.UInt16).fill_null(0) * 1440 +\n            pl.col(col_name).str.extract(r\"(\\d+):\", 1).cast(pl.UInt16).fill_null(0) * 60 +\n            pl.col(col_name).str.extract(r\":(\\d+):\", 1).cast(pl.UInt16).fill_null(0)\n        ).cast(pl.Float32)\n\n    lazy_df = (\n        df.lazy()\n        # Block 1: Initial conversions\n        .with_columns([\n            pl.col(\"isVip\").cast(pl.Boolean),\n            pl.col(\"bySelf\").cast(pl.Boolean),\n            duration_to_minutes(\"legs0_duration\").alias(\"legs0_duration_min\"),\n            duration_to_minutes(\"legs1_duration\").alias(\"legs1_duration_min\"),\n        ])\n        # Block 2: Features dependent on Block 1\n        .with_columns([\n            (pl.col(\"legs0_duration_min\") + pl.col(\"legs1_duration_min\")).alias(\"total_duration\"),\n            (pl.when(pl.col(\"legs1_duration_min\") > 0).then(pl.col(\"legs0_duration_min\") / pl.col(\"legs1_duration_min\")).otherwise(1.0)).alias(\"duration_ratio\"),\n            (pl.sum_horizontal(pl.col(c).is_not_null() for c in df.columns if 'segments' in c and 'duration' in c)).alias(\"total_segments\"),\n            (pl.col(\"legs0_segments0_departureFrom_airport_iata\") + \"_\" +\n             pl.col(\"legs0_segments0_arrivalTo_airport_iata\") + \"__\" +\n             pl.col(\"legs1_segments0_departureFrom_airport_iata\") + \"_\" +\n             pl.col(\"legs1_segments0_arrivalTo_airport_iata\")).alias(\"round_trip_route\"),\n        ])\n        # Block 3: All other features, including those dependent on Block 2\n        .with_columns([\n            (pl.col(\"legs1_duration\").is_null()).alias(\"is_one_way\").cast(pl.UInt8),\n            pl.col(\"totalPrice\").log1p().alias(\"log_price\").cast(pl.Float32),\n            (pl.col(\"frequentFlyer\").fill_null(\"\").str.count_matches(\"/\") + 1).alias(\"n_ff_programs\").cast(pl.UInt8),\n            (pl.col(\"corporateTariffCode\").is_not_null()).alias(\"has_corporate_tariff\").cast(pl.UInt8),\n            pl.col(\"legs0_departureAt\").str.to_datetime(strict=False).dt.hour().alias(\"legs0_dep_hour\"),\n            pl.col(\"legs0_departureAt\").str.to_datetime(strict=False).dt.weekday().alias(\"legs0_dep_weekday\"),\n            pl.col(\"legs0_arrivalAt\").str.to_datetime(strict=False).dt.hour().alias(\"legs0_arr_hour\"),\n            pl.col(\"legs1_departureAt\").str.to_datetime(strict=False).dt.hour().alias(\"legs1_dep_hour\"),\n            pl.col(\"legs1_arrivalAt\").str.to_datetime(strict=False).dt.hour().alias(\"legs1_arr_hour\"),\n            pl.mean_horizontal([\"legs0_segments0_cabinClass\", \"legs1_segments0_cabinClass\"]).alias(\"avg_cabin_class\"),\n            (pl.col(\"legs0_segments0_cabinClass\").fill_null(0) - pl.col(\"legs1_segments0_cabinClass\").fill_null(0)).alias(\"cabin_class_diff\"),\n            (pl.col(\"miniRules0_monetaryAmount\").fill_null(0) + pl.col(\"miniRules1_monetaryAmount\").fill_null(0)).alias(\"total_fees\"),\n            ((pl.col(\"legs0_segments0_baggageAllowance_quantity\").fill_null(0) + pl.col(\"legs1_segments0_baggageAllowance_quantity\").fill_null(0)) > 0).alias(\"has_baggage\").cast(pl.UInt8),\n            (pl.col(\"totalPrice\") == pl.col(\"totalPrice\").min().over(\"ranker_id\")).alias(\"is_cheapest\").cast(pl.UInt8),\n            (pl.col(\"total_duration\").rank(\"average\").over(\"ranker_id\") / pl.col(\"Id\").count().over(\"ranker_id\")).alias(\"duration_pct_rank\"),\n            ((pl.col(\"totalPrice\") - pl.col(\"totalPrice\").median().over(\"ranker_id\")) / (pl.col(\"totalPrice\").std().over(\"ranker_id\") + 1e-6)).alias(\"price_from_median\"),\n            (pl.col(\"total_segments\") == pl.col(\"total_segments\").min().over(\"ranker_id\")).alias(\"is_min_segments\").cast(pl.UInt8),\n        ])\n        # Block 4: Joins\n        .join(carrier0_aggs.lazy(), on='legs0_segments0_marketingCarrier_code', how='left')\n        .join(carrier1_aggs.lazy(), on='legs1_segments0_marketingCarrier_code', how='left')\n        .join(route_aggs.lazy(), on='round_trip_route', how='left')\n        .with_columns([\n            pl.col(\"carrier0_pop\").fill_null(0.0),\n            pl.col(\"carrier1_pop\").fill_null(0.0),\n            pl.col(\"round_trip_freq\").fill_null(0),\n        ])\n    )\n    processed_df = lazy_df.collect(streaming=True).fill_null(0)\n    return processed_df\n\n# Step 4: Final Batched Workflow\n# ==============================\ndef run_pipeline_final():\n    # Pre-compute aggregates once\n    carrier0_aggs, carrier1_aggs, route_aggs = create_aggregate_features(TRAIN_PATH)\n    \n    # Create directory for cached files\n    os.makedirs(PROCESSED_DIR, exist_ok=True)\n    \n    print(\"\\n--- Processing Training Data in Batches and Caching to Disk---\")\n    train_schema = pl.read_parquet_schema(TRAIN_PATH)\n    train_cols_to_load = [c for c in REQUIRED_COLS if c in train_schema]\n    ranker_ids = pl.read_parquet(TRAIN_PATH, columns=['ranker_id']).unique().to_series()\n    \n    feature_cols_for_model = []\n    \n    for i in range(0, len(ranker_ids), RANKER_ID_BATCH_SIZE):\n        batch_ids = ranker_ids[i : i + RANKER_ID_BATCH_SIZE]\n        print(f\"Processing and caching batch {i // RANKER_ID_BATCH_SIZE + 1} of {len(ranker_ids) // RANKER_ID_BATCH_SIZE + 1}...\")\n        \n        raw_chunk = pl.scan_parquet(TRAIN_PATH).filter(pl.col('ranker_id').is_in(batch_ids)).collect()\n        featured_chunk = feature_engineer(raw_chunk, carrier0_aggs, carrier1_aggs, route_aggs)\n        \n        # Determine feature columns on the first batch\n        if not feature_cols_for_model:\n            model_ready_cols = featured_chunk.select(pl.selectors.numeric(), pl.selectors.boolean()).columns\n            id_and_target_cols = ['ranker_id', 'selected', 'Id']\n            feature_cols_for_model = [c for c in model_ready_cols if c not in id_and_target_cols]\n        \n        final_cols_for_chunk = ['ranker_id', 'selected'] + feature_cols_for_model\n        \n        featured_chunk.select(final_cols_for_chunk).write_parquet(f\"{PROCESSED_DIR}batch_{i}.parquet\")\n        \n        del raw_chunk, featured_chunk\n        gc.collect()\n\n    del carrier0_aggs, carrier1_aggs, route_aggs\n    gc.collect()\n    \n    print(\"\\nCreating LazyFrame from cached batches...\")\n    lazy_processed_data = pl.scan_parquet(f\"{PROCESSED_DIR}*.parquet\")\n    \n    total_rows = lazy_processed_data.select(pl.len()).collect().item()\n    n_val_split_point = int(total_rows * 0.9)\n\n    print(\"\\nMaterializing training and validation sets...\")\n    train_df = lazy_processed_data.slice(0, n_val_split_point).collect(streaming=True)\n    val_df = lazy_processed_data.slice(n_val_split_point, None).collect(streaming=True)\n    \n    print(f\"\\nTraining with {len(feature_cols_for_model)} model-ready features.\")\n\n    print(\"\\n--- Preparing XGBoost Matrices ---\")\n    group_sizes_tr = train_df.group_by('ranker_id', maintain_order=True).len()['len'].to_numpy()\n    dtrain = xgb.DMatrix(data=train_df[feature_cols_for_model].to_pandas(), label=train_df['selected'].to_numpy(), group=group_sizes_tr)\n    \n    group_sizes_va = val_df.group_by('ranker_id', maintain_order=True).len()['len'].to_numpy()\n    dval = xgb.DMatrix(data=val_df[feature_cols_for_model].to_pandas(), label=val_df['selected'].to_numpy(), group=group_sizes_va)\n\n    del train_df, val_df\n    gc.collect()\n\n    print(\"\\n--- Training XGBoost Model ---\")\n    model = xgb.train(XGB_PARAMS, dtrain, num_boost_round=1000, evals=[(dtrain, 'train'), (dval, 'val')], early_stopping_rounds=50, verbose_eval=100)\n    \n    del dtrain, dval\n    gc.collect()\n\n    print(\"\\n--- Processing Test Data ---\")\n    # For a large test set, this part should also be batched. Processing in one go for simplicity.\n    carrier0_aggs, carrier1_aggs, route_aggs = create_aggregate_features(TRAIN_PATH)\n    test_raw = pl.read_parquet(TEST_PATH, columns=[c for c in REQUIRED_COLS if c in pl.read_parquet_schema(TEST_PATH)])\n    test_ids = test_raw.select(['Id', 'ranker_id'])\n    test_featured = feature_engineer(test_raw, carrier0_aggs, carrier1_aggs, route_aggs)\n    \n    del test_raw, carrier0_aggs, carrier1_aggs, route_aggs\n    gc.collect()\n\n    dtest = xgb.DMatrix(data=test_featured[feature_cols_for_model].to_pandas())\n    del test_featured\n    gc.collect()\n\n    print(\"\\n--- Generating Submission ---\")\n    predictions = model.predict(dtest)\n    submission_df = test_ids.with_columns(pl.Series(name=\"score\", values=predictions))\n    submission_df = submission_df.with_columns(\n        pl.col('score').rank(method='ordinal', descending=True).over('ranker_id').cast(pl.UInt16).alias('selected')\n    ).select(['Id', 'ranker_id', 'selected'])\n    \n    submission_df.write_csv('submission.csv')\n    print(\"\\nSubmission file 'submission.csv' created successfully.\")\n    \n    shutil.rmtree(PROCESSED_DIR)\n\n# Execute the final pipeline\nrun_pipeline_final()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T22:14:38.648485Z","iopub.execute_input":"2025-08-16T22:14:38.64883Z","iopub.status.idle":"2025-08-16T23:24:56.522606Z","shell.execute_reply.started":"2025-08-16T22:14:38.648806Z","shell.execute_reply":"2025-08-16T23:24:56.512562Z"}},"outputs":[{"name":"stdout","text":"Step 1: Setting up environment...\nRequirement already satisfied: polars in /usr/local/lib/python3.11/dist-packages (1.21.0)\nCollecting polars\n  Downloading polars-1.32.3-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (15 kB)\nRequirement already satisfied: xgboost in /usr/local/lib/python3.11/dist-packages (2.0.3)\nCollecting xgboost\n  Downloading xgboost-3.0.4-py3-none-manylinux_2_28_x86_64.whl.metadata (2.1 kB)\nRequirement already satisfied: pyarrow in /usr/local/lib/python3.11/dist-packages (19.0.1)\nCollecting pyarrow\n  Downloading pyarrow-21.0.0-cp311-cp311-manylinux_2_28_x86_64.whl.metadata (3.3 kB)\nRequirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (from xgboost) (1.26.4)\nRequirement already satisfied: nvidia-nccl-cu12 in /usr/local/lib/python3.11/dist-packages (from xgboost) (2.21.5)\nRequirement already satisfied: scipy in /usr/local/lib/python3.11/dist-packages (from xgboost) (1.15.3)\nRequirement already satisfied: mkl_fft in /usr/local/lib/python3.11/dist-packages (from numpy->xgboost) (1.3.8)\nRequirement already satisfied: mkl_random in /usr/local/lib/python3.11/dist-packages (from numpy->xgboost) (1.2.4)\nRequirement already satisfied: mkl_umath in /usr/local/lib/python3.11/dist-packages (from numpy->xgboost) (0.1.1)\nRequirement already satisfied: mkl in /usr/local/lib/python3.11/dist-packages (from numpy->xgboost) (2025.2.0)\nRequirement already satisfied: tbb4py in /usr/local/lib/python3.11/dist-packages (from numpy->xgboost) (2022.2.0)\nRequirement already satisfied: mkl-service in /usr/local/lib/python3.11/dist-packages (from numpy->xgboost) (2.4.1)\nRequirement already satisfied: intel-openmp<2026,>=2024 in /usr/local/lib/python3.11/dist-packages (from mkl->numpy->xgboost) (2024.2.0)\nRequirement already satisfied: tbb==2022.* in /usr/local/lib/python3.11/dist-packages (from mkl->numpy->xgboost) (2022.2.0)\nRequirement already satisfied: tcmlib==1.* in /usr/local/lib/python3.11/dist-packages (from tbb==2022.*->mkl->numpy->xgboost) (1.4.0)\nRequirement already satisfied: intel-cmplr-lib-rt in /usr/local/lib/python3.11/dist-packages (from mkl_umath->numpy->xgboost) (2024.2.0)\nRequirement already satisfied: intel-cmplr-lib-ur==2024.2.0 in /usr/local/lib/python3.11/dist-packages (from intel-openmp<2026,>=2024->mkl->numpy->xgboost) (2024.2.0)\nDownloading polars-1.32.3-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (38.4 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m38.4/38.4 MB\u001b[0m \u001b[31m41.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hDownloading xgboost-3.0.4-py3-none-manylinux_2_28_x86_64.whl (94.9 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m94.9/94.9 MB\u001b[0m \u001b[31m17.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hDownloading pyarrow-21.0.0-cp311-cp311-manylinux_2_28_x86_64.whl (42.8 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.8/42.8 MB\u001b[0m \u001b[31m37.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hInstalling collected packages: pyarrow, polars, xgboost\n  Attempting uninstall: pyarrow\n    Found existing installation: pyarrow 19.0.1\n    Uninstalling pyarrow-19.0.1:\n      Successfully uninstalled pyarrow-19.0.1\n  Attempting uninstall: polars\n    Found existing installation: polars 1.21.0\n    Uninstalling polars-1.21.0:\n      Successfully uninstalled polars-1.21.0\n  Attempting uninstall: xgboost\n    Found existing installation: xgboost 2.0.3\n    Uninstalling xgboost-2.0.3:\n      Successfully uninstalled xgboost-2.0.3\n\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\nbigframes 2.8.0 requires google-cloud-bigquery-storage<3.0.0,>=2.30.0, which is not installed.\npylibcudf-cu12 25.2.2 requires pyarrow<20.0.0a0,>=14.0.0; platform_machine == \"x86_64\", but you have pyarrow 21.0.0 which is incompatible.\ncudf-cu12 25.2.2 requires pyarrow<20.0.0a0,>=14.0.0; platform_machine == \"x86_64\", but you have pyarrow 21.0.0 which is incompatible.\ndatasets 3.6.0 requires fsspec[http]<=2025.3.0,>=2023.1.0, but you have fsspec 2025.5.1 which is incompatible.\npandas-gbq 0.29.1 requires google-api-core<3.0.0,>=2.10.2, but you have google-api-core 1.34.1 which is incompatible.\ncudf-polars-cu12 25.2.2 requires polars<1.22,>=1.20, but you have polars 1.32.3 which is incompatible.\nbigframes 2.8.0 requires google-cloud-bigquery[bqstorage,pandas]>=3.31.0, but you have google-cloud-bigquery 3.25.0 which is incompatible.\nbigframes 2.8.0 requires rich<14,>=12.4.4, but you have rich 14.0.0 which is incompatible.\u001b[0m\u001b[31m\n\u001b[0mSuccessfully installed polars-1.32.3 pyarrow-21.0.0 xgboost-3.0.4\nStep 2: Pre-computing richer aggregate features...\n","output_type":"stream"},{"name":"stderr","text":"/tmp/ipykernel_36/3104658706.py:60: DeprecationWarning: the `streaming` parameter was deprecated in 1.25.0; use `engine` instead.\n  carrier0_pop = scan_df.group_by(\"legs0_segments0_marketingCarrier_code\").agg(pl.mean('selected').alias('carrier0_pop')).collect(streaming=True)\n/tmp/ipykernel_36/3104658706.py:61: DeprecationWarning: the `streaming` parameter was deprecated in 1.25.0; use `engine` instead.\n  carrier1_pop = scan_df.group_by(\"legs1_segments0_marketingCarrier_code\").agg(pl.mean('selected').alias('carrier1_pop')).collect(streaming=True)\n/tmp/ipykernel_36/3104658706.py:72: DeprecationWarning: the `streaming` parameter was deprecated in 1.25.0; use `engine` instead.\n  .collect(streaming=True)\n","output_type":"stream"},{"name":"stdout","text":"Aggregate features created.\n\n--- Processing Training Data in Batches and Caching to Disk---\nProcessing and caching batch 1 of 3...\n","output_type":"stream"},{"name":"stderr","text":"/tmp/ipykernel_36/3104658706.py:161: DeprecationWarning: `is_in` with a collection of the same datatype is ambiguous and deprecated.\nPlease use `implode` to return to previous behavior.\n\nSee https://github.com/pola-rs/polars/issues/22149 for more information.\n  raw_chunk = pl.scan_parquet(TRAIN_PATH).filter(pl.col('ranker_id').is_in(batch_ids)).collect()\n/tmp/ipykernel_36/3104658706.py:138: DeprecationWarning: the `streaming` parameter was deprecated in 1.25.0; use `engine` instead.\n  processed_df = lazy_df.collect(streaming=True).fill_null(0)\n","output_type":"stream"},{"name":"stdout","text":"Processing and caching batch 2 of 3...\nProcessing and caching batch 3 of 3...\n\nCreating LazyFrame from cached batches...\n\nMaterializing training and validation sets...\n","output_type":"stream"},{"name":"stderr","text":"/tmp/ipykernel_36/3104658706.py:187: DeprecationWarning: the `streaming` parameter was deprecated in 1.25.0; use `engine` instead.\n  train_df = lazy_processed_data.slice(0, n_val_split_point).collect(streaming=True)\n/tmp/ipykernel_36/3104658706.py:188: DeprecationWarning: the `streaming` parameter was deprecated in 1.25.0; use `engine` instead.\n  val_df = lazy_processed_data.slice(n_val_split_point, None).collect(streaming=True)\n","output_type":"stream"},{"name":"stdout","text":"\nTraining with 76 model-ready features.\n\n--- Preparing XGBoost Matrices ---\n\n--- Training XGBoost Model ---\n[0]\ttrain-ndcg@3:0.73227\tval-ndcg@3:0.76641\n[100]\ttrain-ndcg@3:0.85531\tval-ndcg@3:0.83162\n[200]\ttrain-ndcg@3:0.87257\tval-ndcg@3:0.83657\n[300]\ttrain-ndcg@3:0.88415\tval-ndcg@3:0.83983\n[400]\ttrain-ndcg@3:0.89209\tval-ndcg@3:0.84118\n[500]\ttrain-ndcg@3:0.89761\tval-ndcg@3:0.84206\n[600]\ttrain-ndcg@3:0.90164\tval-ndcg@3:0.84295\n[700]\ttrain-ndcg@3:0.90370\tval-ndcg@3:0.84365\n[774]\ttrain-ndcg@3:0.90463\tval-ndcg@3:0.84395\n\n--- Processing Test Data ---\nStep 2: Pre-computing richer aggregate features...\n","output_type":"stream"},{"name":"stderr","text":"/tmp/ipykernel_36/3104658706.py:60: DeprecationWarning: the `streaming` parameter was deprecated in 1.25.0; use `engine` instead.\n  carrier0_pop = scan_df.group_by(\"legs0_segments0_marketingCarrier_code\").agg(pl.mean('selected').alias('carrier0_pop')).collect(streaming=True)\n/tmp/ipykernel_36/3104658706.py:61: DeprecationWarning: the `streaming` parameter was deprecated in 1.25.0; use `engine` instead.\n  carrier1_pop = scan_df.group_by(\"legs1_segments0_marketingCarrier_code\").agg(pl.mean('selected').alias('carrier1_pop')).collect(streaming=True)\n/tmp/ipykernel_36/3104658706.py:72: DeprecationWarning: the `streaming` parameter was deprecated in 1.25.0; use `engine` instead.\n  .collect(streaming=True)\n","output_type":"stream"},{"name":"stdout","text":"Aggregate features created.\n","output_type":"stream"},{"name":"stderr","text":"/tmp/ipykernel_36/3104658706.py:138: DeprecationWarning: the `streaming` parameter was deprecated in 1.25.0; use `engine` instead.\n  processed_df = lazy_df.collect(streaming=True).fill_null(0)\n","output_type":"stream"},{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)","\u001b[0;32m/usr/local/lib/python3.11/dist-packages/polars/_utils/construction/series.py\u001b[0m in \u001b[0;36m_construct_series_with_fallbacks\u001b[0;34m(constructor, name, values, dtype, strict)\u001b[0m\n\u001b[1;32m    327\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 328\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mconstructor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstrict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    329\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mTypeError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mOverflowError\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mTypeError\u001b[0m: 'str' object cannot be interpreted as an integer","\nDuring handling of the above exception, another exception occurred:\n","\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)","\u001b[0;32m/usr/local/lib/python3.11/dist-packages/polars/_utils/getitem.py\u001b[0m in \u001b[0;36mget_df_item_by_key\u001b[0;34m(df, key)\u001b[0m\n\u001b[1;32m    166\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 167\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_select_rows\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[arg-type]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    168\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.11/dist-packages/polars/_utils/getitem.py\u001b[0m in \u001b[0;36m_select_rows\u001b[0;34m(df, key)\u001b[0m\n\u001b[1;32m    313\u001b[0m             \u001b[0m_raise_on_boolean_mask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 314\u001b[0;31m         \u001b[0ms\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSeries\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mInt64\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    315\u001b[0m         \u001b[0mindices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_convert_series_to_indices\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mheight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.11/dist-packages/polars/series/series.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, name, values, dtype, strict, nan_to_null)\u001b[0m\n\u001b[1;32m    303\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSequence\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 304\u001b[0;31m             self._s = sequence_to_pyseries(\n\u001b[0m\u001b[1;32m    305\u001b[0m                 \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.11/dist-packages/polars/_utils/construction/series.py\u001b[0m in \u001b[0;36msequence_to_pyseries\u001b[0;34m(name, values, dtype, strict, nan_to_null)\u001b[0m\n\u001b[1;32m    155\u001b[0m         \u001b[0mconstructor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpolars_type_to_constructor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 156\u001b[0;31m         pyseries = _construct_series_with_fallbacks(\n\u001b[0m\u001b[1;32m    157\u001b[0m             \u001b[0mconstructor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstrict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstrict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.11/dist-packages/polars/_utils/construction/series.py\u001b[0m in \u001b[0;36m_construct_series_with_fallbacks\u001b[0;34m(constructor, name, values, dtype, strict)\u001b[0m\n\u001b[1;32m    342\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 343\u001b[0;31m             return PySeries.new_from_any_values_and_dtype(\n\u001b[0m\u001b[1;32m    344\u001b[0m                 \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstrict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstrict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mTypeError\u001b[0m: unexpected value while building Series of type Int64; found value of type String: \"companyID\"\n\nHint: Try setting `strict=False` to allow passing data with mixed types.","\nDuring handling of the above exception, another exception occurred:\n","\u001b[0;31mColumnNotFoundError\u001b[0m                       Traceback (most recent call last)","\u001b[0;32m/tmp/ipykernel_36/3104658706.py\u001b[0m in \u001b[0;36m<cell line: 0>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    233\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    234\u001b[0m \u001b[0;31m# Execute the final pipeline\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 235\u001b[0;31m \u001b[0mrun_pipeline_final\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;32m/tmp/ipykernel_36/3104658706.py\u001b[0m in \u001b[0;36mrun_pipeline_final\u001b[0;34m()\u001b[0m\n\u001b[1;32m    216\u001b[0m     \u001b[0mgc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    217\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 218\u001b[0;31m     \u001b[0mdtest\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mxgb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDMatrix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtest_featured\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfeature_cols_for_model\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_pandas\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    219\u001b[0m     \u001b[0;32mdel\u001b[0m \u001b[0mtest_featured\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    220\u001b[0m     \u001b[0mgc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.11/dist-packages/polars/dataframe/frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   1393\u001b[0m         \u001b[0;31m└\u001b[0m\u001b[0;31m─\u001b[0m\u001b[0;31m─\u001b[0m\u001b[0;31m─\u001b[0m\u001b[0;31m─\u001b[0m\u001b[0;31m─\u001b[0m\u001b[0;31m┴\u001b[0m\u001b[0;31m─\u001b[0m\u001b[0;31m─\u001b[0m\u001b[0;31m─\u001b[0m\u001b[0;31m─\u001b[0m\u001b[0;31m─\u001b[0m\u001b[0;31m┴\u001b[0m\u001b[0;31m─\u001b[0m\u001b[0;31m─\u001b[0m\u001b[0;31m─\u001b[0m\u001b[0;31m─\u001b[0m\u001b[0;31m─\u001b[0m\u001b[0;31m┘\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1394\u001b[0m         \"\"\"\n\u001b[0;32m-> 1395\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mget_df_item_by_key\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1396\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1397\u001b[0m     def __setitem__(\n","\u001b[0;32m/usr/local/lib/python3.11/dist-packages/polars/_utils/getitem.py\u001b[0m in \u001b[0;36mget_df_item_by_key\u001b[0;34m(df, key)\u001b[0m\n\u001b[1;32m    167\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0m_select_rows\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[arg-type]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    168\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 169\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_select_columns\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    170\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    171\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.11/dist-packages/polars/_utils/getitem.py\u001b[0m in \u001b[0;36m_select_columns\u001b[0;34m(df, key)\u001b[0m\n\u001b[1;32m    215\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0m_select_columns_by_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[arg-type]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    216\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfirst\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 217\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0m_select_columns_by_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[arg-type]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    218\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    219\u001b[0m             \u001b[0mmsg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34mf\"cannot select columns using Sequence with elements of type {qualified_type_name(first)!r}\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.11/dist-packages/polars/_utils/getitem.py\u001b[0m in \u001b[0;36m_select_columns_by_name\u001b[0;34m(df, key)\u001b[0m\n\u001b[1;32m    267\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    268\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_select_columns_by_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mDataFrame\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mIterable\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mDataFrame\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 269\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_from_pydf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mselect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    270\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    271\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mColumnNotFoundError\u001b[0m: \"companyID\" not found"],"ename":"ColumnNotFoundError","evalue":"\"companyID\" not found","output_type":"error"}],"execution_count":1},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}