{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as pl\nimport polars as pl\nimport pandas as pd\nimport lightgbm as lgb\nimport kaggle_evaluation.jane_street_inference_server","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T02:18:07.560862Z","iopub.execute_input":"2024-12-27T02:18:07.561199Z","iopub.status.idle":"2024-12-27T02:18:07.566672Z","shell.execute_reply.started":"2024-12-27T02:18:07.561175Z","shell.execute_reply":"2024-12-27T02:18:07.565010Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nimport dask.dataframe as dd\ntrain_data = dd.read_parquet('/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet')\ntrain_data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T02:18:07.568630Z","iopub.execute_input":"2024-12-27T02:18:07.569021Z","iopub.status.idle":"2024-12-27T02:18:08.914254Z","shell.execute_reply.started":"2024-12-27T02:18:07.568989Z","shell.execute_reply":"2024-12-27T02:18:08.913121Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\n\ndef load_data(date_id_range=None, time_id_range=None, columns=None, return_type='pl'):\n    data_dir = '../input/jane-street-real-time-market-data-forecasting'\n    \n    # Load data using Polars lazy loading (scan_parquet)\n    data = pl.scan_parquet(f'{data_dir}/train.parquet')\n    \n    # Apply date_id filter if specified\n    if date_id_range is not None:\n        start_date, end_date = date_id_range\n        data = data.filter(\n            (pl.col('date_id') >= start_date) & (pl.col('date_id') <= end_date)\n        )\n    \n    # Apply time_id filter if specified\n    if time_id_range is not None:\n        start_time, end_time = time_id_range\n        data = data.filter(\n            (pl.col('time_id') >= start_time) & (pl.col('time_id') <= end_time)\n        )\n    \n    # Select specific columns if specified\n    if columns is not None:\n        data = data.select(columns)\n    \n    # Collect the data to execute the lazy operations\n    if return_type == 'pd':  # Convert to pandas DataFrame\n        return data.collect().to_pandas()\n    else:  # Return Polars DataFrame\n        return data.collect()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T02:18:08.915962Z","iopub.execute_input":"2024-12-27T02:18:08.916254Z","iopub.status.idle":"2024-12-27T02:18:08.923156Z","shell.execute_reply.started":"2024-12-27T02:18:08.916229Z","shell.execute_reply":"2024-12-27T02:18:08.921685Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target = 'responder_6'\nfeat_cols = [f\"feature_{i:02d}\" for i in range(79)]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T02:18:08.924340Z","iopub.execute_input":"2024-12-27T02:18:08.924761Z","iopub.status.idle":"2024-12-27T02:18:08.949086Z","shell.execute_reply.started":"2024-12-27T02:18:08.924724Z","shell.execute_reply":"2024-12-27T02:18:08.947869Z"}},"outputs":[],"execution_count":null}]}