{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":9638675,"sourceType":"datasetVersion","datasetId":5885447},{"sourceId":200929473,"sourceType":"kernelVersion"},{"sourceId":201774501,"sourceType":"kernelVersion"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-18T00:30:47.546695Z","iopub.execute_input":"2024-10-18T00:30:47.548236Z","iopub.status.idle":"2024-10-18T00:30:47.581902Z","shell.execute_reply.started":"2024-10-18T00:30:47.548179Z","shell.execute_reply":"2024-10-18T00:30:47.580763Z"},"trusted":true},"execution_count":24,"outputs":[{"name":"stdout","text":"/kaggle/input/jane-street-real-time-market-data-forecasting/responders.csv\n/kaggle/input/jane-street-real-time-market-data-forecasting/sample_submission.csv\n/kaggle/input/jane-street-real-time-market-data-forecasting/features.csv\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=4/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=5/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=6/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=3/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=1/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=8/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=2/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=0/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=7/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=9/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet/date_id=0/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet/date_id=0/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/jane_street_gateway.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/jane_street_inference_server.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/__init__.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/templates.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/base_gateway.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/relay.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/kaggle_evaluation.proto\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/__init__.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/generated/kaggle_evaluation_pb2.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/generated/kaggle_evaluation_pb2_grpc.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/generated/__init__.py\n","output_type":"stream"}]},{"cell_type":"code","source":"!pip install polars[gpu]","metadata":{"execution":{"iopub.status.busy":"2024-10-17T23:56:28.769392Z","iopub.execute_input":"2024-10-17T23:56:28.770336Z","iopub.status.idle":"2024-10-17T23:58:14.055573Z","shell.execute_reply.started":"2024-10-17T23:56:28.770277Z","shell.execute_reply":"2024-10-17T23:58:14.054162Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stdout","text":"Requirement already satisfied: polars[gpu] in /opt/conda/lib/python3.10/site-packages (1.9.0)\nCollecting cudf-polars-cu12 (from polars[gpu])\n  Downloading cudf_polars_cu12-24.10.1.tar.gz (2.2 kB)\n  Installing build dependencies ... \u001b[?25ldone\n\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n\u001b[?25hINFO: pip is looking at multiple versions of cudf-polars-cu12 to determine which version is compatible with other requirements. This could take a while.\n  Downloading cudf_polars_cu12-24.8.3.tar.gz (2.2 kB)\n  Installing build dependencies ... \u001b[?25ldone\n\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n\u001b[?25hCollecting cudf-cu12==24.8.* (from cudf-polars-cu12->polars[gpu])\n  Downloading cudf_cu12-24.8.3.tar.gz (2.6 kB)\n  Installing build dependencies ... \u001b[?25ldone\n\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n\u001b[?25hRequirement already satisfied: cachetools in /opt/conda/lib/python3.10/site-packages (from cudf-cu12==24.8.*->cudf-polars-cu12->polars[gpu]) (4.2.4)\nCollecting cuda-python<13.0a0,>=12.0 (from cudf-cu12==24.8.*->cudf-polars-cu12->polars[gpu])\n  Downloading cuda_python-12.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\nCollecting cupy-cuda12x>=12.0.0 (from cudf-cu12==24.8.*->cudf-polars-cu12->polars[gpu])\n  Downloading cupy_cuda12x-13.3.0-cp310-cp310-manylinux2014_x86_64.whl.metadata (2.7 kB)\nRequirement already satisfied: fsspec>=0.6.0 in /opt/conda/lib/python3.10/site-packages (from cudf-cu12==24.8.*->cudf-polars-cu12->polars[gpu]) (2024.6.1)\nRequirement already satisfied: numba>=0.57 in /opt/conda/lib/python3.10/site-packages (from cudf-cu12==24.8.*->cudf-polars-cu12->polars[gpu]) (0.60.0)\nRequirement already satisfied: numpy<2.0a0,>=1.23 in /opt/conda/lib/python3.10/site-packages (from cudf-cu12==24.8.*->cudf-polars-cu12->polars[gpu]) (1.26.4)\nCollecting nvtx>=0.2.1 (from cudf-cu12==24.8.*->cudf-polars-cu12->polars[gpu])\n  Downloading nvtx-0.2.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (740 bytes)\nRequirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from cudf-cu12==24.8.*->cudf-polars-cu12->polars[gpu]) (21.3)\nCollecting pandas<2.2.3dev0,>=2.0 (from cudf-cu12==24.8.*->cudf-polars-cu12->polars[gpu])\n  Downloading pandas-2.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (19 kB)\nCollecting pyarrow<16.2.0a0,>=16.1.0 (from cudf-cu12==24.8.*->cudf-polars-cu12->polars[gpu])\n  Downloading pyarrow-16.1.0-cp310-cp310-manylinux_2_28_x86_64.whl.metadata (3.0 kB)\nCollecting pynvjitlink-cu12 (from cudf-cu12==24.8.*->cudf-polars-cu12->polars[gpu])\n  Downloading pynvjitlink_cu12-0.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl.metadata (1.5 kB)\nRequirement already satisfied: rich in /opt/conda/lib/python3.10/site-packages (from cudf-cu12==24.8.*->cudf-polars-cu12->polars[gpu]) (13.7.1)\nCollecting rmm-cu12==24.8.* (from cudf-cu12==24.8.*->cudf-polars-cu12->polars[gpu])\n  Downloading rmm_cu12-24.8.2.tar.gz (13 kB)\n  Installing build dependencies ... \u001b[?25ldone\n\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n\u001b[?25hRequirement already satisfied: typing_extensions>=4.0.0 in /opt/conda/lib/python3.10/site-packages (from cudf-cu12==24.8.*->cudf-polars-cu12->polars[gpu]) (4.12.2)\nCollecting fastrlock>=0.5 (from cupy-cuda12x>=12.0.0->cudf-cu12==24.8.*->cudf-polars-cu12->polars[gpu])\n  Downloading fastrlock-0.8.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl.metadata (9.3 kB)\nRequirement already satisfied: llvmlite<0.44,>=0.43.0dev0 in /opt/conda/lib/python3.10/site-packages (from numba>=0.57->cudf-cu12==24.8.*->cudf-polars-cu12->polars[gpu]) (0.43.0)\nRequirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.10/site-packages (from pandas<2.2.3dev0,>=2.0->cudf-cu12==24.8.*->cudf-polars-cu12->polars[gpu]) (2.9.0.post0)\nRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas<2.2.3dev0,>=2.0->cudf-cu12==24.8.*->cudf-polars-cu12->polars[gpu]) (2024.1)\nRequirement already satisfied: tzdata>=2022.7 in /opt/conda/lib/python3.10/site-packages (from pandas<2.2.3dev0,>=2.0->cudf-cu12==24.8.*->cudf-polars-cu12->polars[gpu]) (2024.1)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging->cudf-cu12==24.8.*->cudf-polars-cu12->polars[gpu]) (3.1.2)\nRequirement already satisfied: markdown-it-py>=2.2.0 in /opt/conda/lib/python3.10/site-packages (from rich->cudf-cu12==24.8.*->cudf-polars-cu12->polars[gpu]) (3.0.0)\nRequirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/conda/lib/python3.10/site-packages (from rich->cudf-cu12==24.8.*->cudf-polars-cu12->polars[gpu]) (2.18.0)\nRequirement already satisfied: mdurl~=0.1 in /opt/conda/lib/python3.10/site-packages (from markdown-it-py>=2.2.0->rich->cudf-cu12==24.8.*->cudf-polars-cu12->polars[gpu]) (0.1.2)\nRequirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas<2.2.3dev0,>=2.0->cudf-cu12==24.8.*->cudf-polars-cu12->polars[gpu]) (1.16.0)\nDownloading cuda_python-12.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (24.2 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m24.2/24.2 MB\u001b[0m \u001b[31m52.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hDownloading cupy_cuda12x-13.3.0-cp310-cp310-manylinux2014_x86_64.whl (90.6 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m90.6/90.6 MB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m:00:01\u001b[0m\n\u001b[?25hDownloading nvtx-0.2.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (582 kB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m582.6/582.6 kB\u001b[0m \u001b[31m24.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hDownloading pandas-2.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (13.0 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.0/13.0 MB\u001b[0m \u001b[31m77.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hDownloading pyarrow-16.1.0-cp310-cp310-manylinux_2_28_x86_64.whl (40.8 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.8/40.8 MB\u001b[0m \u001b[31m34.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hDownloading pynvjitlink_cu12-0.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (24.2 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m24.2/24.2 MB\u001b[0m \u001b[31m55.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hDownloading fastrlock-0.8.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl (51 kB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m51.3/51.3 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hBuilding wheels for collected packages: cudf-polars-cu12, cudf-cu12, rmm-cu12\n  Building wheel for cudf-polars-cu12 (pyproject.toml) ... \u001b[?25ldone\n\u001b[?25h  Created wheel for cudf-polars-cu12: filename=cudf_polars_cu12-24.8.3-py3-none-any.whl size=53761 sha256=0964fd3809c31e8cf04952b18b5cbbde437428de5ed31154742da9c83b80917a\n  Stored in directory: /root/.cache/pip/wheels/f7/d1/b4/3297b3e349740af3c1bfe60761881dda36358e39dff3670729\n  Building wheel for cudf-cu12 (pyproject.toml) ... \u001b[?25ldone\n\u001b[?25h  Created wheel for cudf-cu12: filename=cudf_cu12-24.8.3-cp310-cp310-manylinux_2_28_x86_64.whl size=517792381 sha256=2388501cb61c5334370523596fee1dcb17936606fe3d1140f72f8d7716f2d5f5\n  Stored in directory: /root/.cache/pip/wheels/e0/93/07/7abc732f4849ee032553ce466567b5e4bf3b5916aba8db2907\n  Building wheel for rmm-cu12 (pyproject.toml) ... \u001b[?25ldone\n\u001b[?25h  Created wheel for rmm-cu12: filename=rmm_cu12-24.8.2-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl size=1743566 sha256=6c6dc399dd24e4cc418280f1faca1d74b1a852e25e7798bf6bccf5e1d64f0c01\n  Stored in directory: /root/.cache/pip/wheels/52/d3/4a/625ea3bd859c1353a9ca69712a3d58f8e83d8ac26ca7311983\nSuccessfully built cudf-polars-cu12 cudf-cu12 rmm-cu12\nInstalling collected packages: nvtx, fastrlock, cuda-python, pynvjitlink-cu12, pyarrow, cupy-cuda12x, rmm-cu12, pandas, cudf-cu12, cudf-polars-cu12\n  Attempting uninstall: pyarrow\n    Found existing installation: pyarrow 17.0.0\n    Uninstalling pyarrow-17.0.0:\n      Successfully uninstalled pyarrow-17.0.0\n  Attempting uninstall: pandas\n    Found existing installation: pandas 2.2.3\n    Uninstalling pandas-2.2.3:\n      Successfully uninstalled pandas-2.2.3\n\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\napache-beam 2.46.0 requires cloudpickle~=2.2.1, but you have cloudpickle 3.0.0 which is incompatible.\napache-beam 2.46.0 requires dill<0.3.2,>=0.3.1.1, but you have dill 0.3.8 which is incompatible.\napache-beam 2.46.0 requires numpy<1.25.0,>=1.14.3, but you have numpy 1.26.4 which is incompatible.\napache-beam 2.46.0 requires pyarrow<10.0.0,>=3.0.0, but you have pyarrow 16.1.0 which is incompatible.\nbeatrix-jupyterlab 2024.66.154055 requires jupyterlab~=3.6.0, but you have jupyterlab 4.2.5 which is incompatible.\nbigframes 0.22.0 requires google-cloud-bigquery[bqstorage,pandas]>=3.10.0, but you have google-cloud-bigquery 2.34.4 which is incompatible.\nbigframes 0.22.0 requires google-cloud-storage>=2.0.0, but you have google-cloud-storage 1.44.0 which is incompatible.\nbigframes 0.22.0 requires pandas<2.1.4,>=1.5.0, but you have pandas 2.2.2 which is incompatible.\ncesium 0.12.3 requires numpy<3.0,>=2.0, but you have numpy 1.26.4 which is incompatible.\ndataproc-jupyter-plugin 0.1.79 requires pydantic~=1.10.0, but you have pydantic 2.9.2 which is incompatible.\nibis-framework 7.1.0 requires pyarrow<15,>=2, but you have pyarrow 16.1.0 which is incompatible.\nlibpysal 4.9.2 requires packaging>=22, but you have packaging 21.3 which is incompatible.\nlibpysal 4.9.2 requires shapely>=2.0.1, but you have shapely 1.8.5.post1 which is incompatible.\nxarray 2024.9.0 requires packaging>=23.1, but you have packaging 21.3 which is incompatible.\nydata-profiling 4.10.0 requires scipy<1.14,>=1.4.1, but you have scipy 1.14.1 which is incompatible.\u001b[0m\u001b[31m\n\u001b[0mSuccessfully installed cuda-python-12.6.0 cudf-cu12-24.8.3 cudf-polars-cu12-24.8.3 cupy-cuda12x-13.3.0 fastrlock-0.8.2 nvtx-0.2.10 pandas-2.2.2 pyarrow-16.1.0 pynvjitlink-cu12-0.3.0 rmm-cu12-24.8.2\n","output_type":"stream"}]},{"cell_type":"code","source":"import polars as pl\nimport os","metadata":{"execution":{"iopub.status.busy":"2024-10-18T00:00:29.358131Z","iopub.execute_input":"2024-10-18T00:00:29.3586Z","iopub.status.idle":"2024-10-18T00:00:29.364218Z","shell.execute_reply.started":"2024-10-18T00:00:29.358546Z","shell.execute_reply":"2024-10-18T00:00:29.36311Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"code","source":"op_path = \"/kaggle/working\"\nip_path = \"/kaggle/input/timeseries-dataset\"\ntarget = \"responder_6\"\nversion_nb = \"V1_1\"\nstate = 42\n\noffline_strt_dt = 500\nfitted_models = {}","metadata":{"execution":{"iopub.status.busy":"2024-10-18T00:44:38.877779Z","iopub.execute_input":"2024-10-18T00:44:38.878281Z","iopub.status.idle":"2024-10-18T00:44:38.884783Z","shell.execute_reply.started":"2024-10-18T00:44:38.878238Z","shell.execute_reply":"2024-10-18T00:44:38.883736Z"},"trusted":true},"execution_count":28,"outputs":[]},{"cell_type":"code","source":"train = pl.scan_parquet(os.path.join(ip_path, \"XYtrain\"))\nsel_cols = train.collect_schema().names()\nsel_cols = [c for c in sel_cols \n            if c not in [\"id\", \"date_id\", \"time_id\", \"partition_id\"]]\ndrop_cols = [c for c in sel_cols if \"responder\" in c] + [\"weight\"]\nXtrain = train.filter(pl.col(\"date_id\").gt(offline_strt_dt)).\\\n        select(pl.col(sel_cols)).\\\n        collect(engine=\"gpu\").\\\n        to_pandas()\n\nXtrain.index = range(len(Xtrain))\nytrain = Xtrain[target]\nsw_tr = Xtrain[\"weight\"].values.flatten()\nXtrain = Xtrain.drop(drop_cols, axis=1, errors=\"ignore\")\n\ndev = pl.scan_parquet(os.path,join(ip_path, \"XYdev.parquet\"))\nXdev = dev.select(pl.col(sel_cols)).collect(engine=\"gpu\").to_pandas()\nXdev_index = range(len(Xdev))\nydev = Xdev[\"target\"]\nsw_dev = Xdev[\"weight\"].values.flatten()\nXdev = Xdev.drop(drop_cols, axis=1, errors=\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2024-10-18T00:44:40.244925Z","iopub.execute_input":"2024-10-18T00:44:40.24535Z","iopub.status.idle":"2024-10-18T00:44:40.291279Z","shell.execute_reply.started":"2024-10-18T00:44:40.245301Z","shell.execute_reply":"2024-10-18T00:44:40.28977Z"},"trusted":true},"execution_count":29,"outputs":[{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)","Cell \u001b[0;32mIn[29], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m train \u001b[38;5;241m=\u001b[39m pl\u001b[38;5;241m.\u001b[39mscan_parquet(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(ip_path, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mXYtrain\u001b[39m\u001b[38;5;124m\"\u001b[39m))\n\u001b[0;32m----> 2\u001b[0m sel_cols \u001b[38;5;241m=\u001b[39m \u001b[43mtrain\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcollect_schema\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mnames()\n\u001b[1;32m      3\u001b[0m sel_cols \u001b[38;5;241m=\u001b[39m [c \u001b[38;5;28;01mfor\u001b[39;00m c \u001b[38;5;129;01min\u001b[39;00m sel_cols \n\u001b[1;32m      4\u001b[0m             \u001b[38;5;28;01mif\u001b[39;00m c \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m [\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdate_id\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtime_id\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpartition_id\u001b[39m\u001b[38;5;124m\"\u001b[39m]]\n\u001b[1;32m      5\u001b[0m drop_cols \u001b[38;5;241m=\u001b[39m [c \u001b[38;5;28;01mfor\u001b[39;00m c \u001b[38;5;129;01min\u001b[39;00m sel_cols \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mresponder\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m c] \u001b[38;5;241m+\u001b[39m [\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mweight\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/polars/lazyframe/frame.py:2261\u001b[0m, in \u001b[0;36mLazyFrame.collect_schema\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   2231\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcollect_schema\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Schema:\n\u001b[1;32m   2232\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m   2233\u001b[0m \u001b[38;5;124;03m    Resolve the schema of this LazyFrame.\u001b[39;00m\n\u001b[1;32m   2234\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   2259\u001b[0m \u001b[38;5;124;03m    3\u001b[39;00m\n\u001b[1;32m   2260\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m-> 2261\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m Schema(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_ldf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcollect_schema\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m)\n","\u001b[0;31mFileNotFoundError\u001b[0m: No such file or directory (os error 2): /kaggle/input/timeseries-dataset/XYtrain"],"ename":"FileNotFoundError","evalue":"No such file or directory (os error 2): /kaggle/input/timeseries-dataset/XYtrain","output_type":"error"}]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}