{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-11-23T11:55:47.74248Z","iopub.execute_input":"2024-11-23T11:55:47.74272Z","iopub.status.idle":"2024-11-23T11:55:48.729895Z","shell.execute_reply.started":"2024-11-23T11:55:47.742693Z","shell.execute_reply":"2024-11-23T11:55:48.729022Z"}},"outputs":[],"execution_count":1},{"cell_type":"code","source":"import pandas as pd\nimport gc\n# Initialize a list to hold samples from each file\nsamples = []\n# Load a sample from each file\n#for i in range(10):\nfor i in [9]:\n    file_path = f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id={i}/part-0.parquet\"\n    chunk = pd.read_parquet(file_path)\n    \n    # Take a sample of the data (adjust sample size as needed)\n    #sample_chunk = chunk.sample(n=500000, random_state=42)  # For example, 100 rows\n    sample_chunk = chunk\n    samples.append(sample_chunk)\n# Concatenate all samples into one DataFrame if needed\ndel chunk\ngc.collect()  # Forces garbage collection\nsample_df = pd.concat(samples, ignore_index=True)\ndel samples\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T11:55:51.532541Z","iopub.execute_input":"2024-11-23T11:55:51.533256Z","iopub.status.idle":"2024-11-23T11:56:00.889285Z","shell.execute_reply.started":"2024-11-23T11:55:51.533222Z","shell.execute_reply":"2024-11-23T11:56:00.888315Z"}},"outputs":[{"execution_count":2,"output_type":"execute_result","data":{"text/plain":"0"},"metadata":{}}],"execution_count":2},{"cell_type":"code","source":"sample_df.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T11:56:05.972056Z","iopub.execute_input":"2024-11-23T11:56:05.97238Z","iopub.status.idle":"2024-11-23T11:56:05.998708Z","shell.execute_reply.started":"2024-11-23T11:56:05.972349Z","shell.execute_reply":"2024-11-23T11:56:05.99787Z"}},"outputs":[{"execution_count":3,"output_type":"execute_result","data":{"text/plain":"   date_id  time_id  symbol_id    weight  feature_00  feature_01  feature_02  \\\n0     1530        0          0  3.084694    1.153571    1.563784    0.697396   \n1     1530        0          1  2.232906    0.553354    1.730064    0.990195   \n2     1530        0          2  2.404948    1.532503    2.095852    0.919688   \n3     1530        0          3  1.986533    0.647099    1.687460    0.569406   \n4     1530        0          4  2.742601    1.096778    1.551411    0.632113   \n\n   feature_03  feature_04  feature_05  ...  feature_78  responder_0  \\\n0    0.756759    2.580965    0.171311  ...    0.999516     0.417462   \n1    0.611490    2.023031    0.319015  ...    0.160609    -0.318671   \n2    0.583715    2.330047    0.337096  ...   -0.065761     0.200878   \n3    1.061679    2.444131    0.150487  ...    0.526284    -0.349773   \n4    0.368218    2.181873    0.214604  ...   -0.965623    -0.373938   \n\n   responder_1  responder_2  responder_3  responder_4  responder_5  \\\n0     0.323897     0.601499     2.074103     0.746552     0.552013   \n1    -0.399384    -0.635306     2.092151     0.342582     0.757289   \n2    -0.006571     0.518870    -0.344441     0.641694    -0.646040   \n3    -0.235901    -0.428956    -1.903627    -1.214619    -0.469500   \n4    -0.209282    -0.095182    -1.598217     0.968505    -0.705594   \n\n   responder_6  responder_7  responder_8  \n0     3.071231     0.914794     0.997124  \n1     1.979042     0.967537     1.219739  \n2    -0.506260     0.739797    -2.041514  \n3    -2.590589    -0.946317    -0.390001  \n4    -1.579623     0.954296    -1.805623  \n\n[5 rows x 92 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>date_id</th>\n      <th>time_id</th>\n      <th>symbol_id</th>\n      <th>weight</th>\n      <th>feature_00</th>\n      <th>feature_01</th>\n      <th>feature_02</th>\n      <th>feature_03</th>\n      <th>feature_04</th>\n      <th>feature_05</th>\n      <th>...</th>\n      <th>feature_78</th>\n      <th>responder_0</th>\n      <th>responder_1</th>\n      <th>responder_2</th>\n      <th>responder_3</th>\n      <th>responder_4</th>\n      <th>responder_5</th>\n      <th>responder_6</th>\n      <th>responder_7</th>\n      <th>responder_8</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1530</td>\n      <td>0</td>\n      <td>0</td>\n      <td>3.084694</td>\n      <td>1.153571</td>\n      <td>1.563784</td>\n      <td>0.697396</td>\n      <td>0.756759</td>\n      <td>2.580965</td>\n      <td>0.171311</td>\n      <td>...</td>\n      <td>0.999516</td>\n      <td>0.417462</td>\n      <td>0.323897</td>\n      <td>0.601499</td>\n      <td>2.074103</td>\n      <td>0.746552</td>\n      <td>0.552013</td>\n      <td>3.071231</td>\n      <td>0.914794</td>\n      <td>0.997124</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1530</td>\n      <td>0</td>\n      <td>1</td>\n      <td>2.232906</td>\n      <td>0.553354</td>\n      <td>1.730064</td>\n      <td>0.990195</td>\n      <td>0.611490</td>\n      <td>2.023031</td>\n      <td>0.319015</td>\n      <td>...</td>\n      <td>0.160609</td>\n      <td>-0.318671</td>\n      <td>-0.399384</td>\n      <td>-0.635306</td>\n      <td>2.092151</td>\n      <td>0.342582</td>\n      <td>0.757289</td>\n      <td>1.979042</td>\n      <td>0.967537</td>\n      <td>1.219739</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1530</td>\n      <td>0</td>\n      <td>2</td>\n      <td>2.404948</td>\n      <td>1.532503</td>\n      <td>2.095852</td>\n      <td>0.919688</td>\n      <td>0.583715</td>\n      <td>2.330047</td>\n      <td>0.337096</td>\n      <td>...</td>\n      <td>-0.065761</td>\n      <td>0.200878</td>\n      <td>-0.006571</td>\n      <td>0.518870</td>\n      <td>-0.344441</td>\n      <td>0.641694</td>\n      <td>-0.646040</td>\n      <td>-0.506260</td>\n      <td>0.739797</td>\n      <td>-2.041514</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1530</td>\n      <td>0</td>\n      <td>3</td>\n      <td>1.986533</td>\n      <td>0.647099</td>\n      <td>1.687460</td>\n      <td>0.569406</td>\n      <td>1.061679</td>\n      <td>2.444131</td>\n      <td>0.150487</td>\n      <td>...</td>\n      <td>0.526284</td>\n      <td>-0.349773</td>\n      <td>-0.235901</td>\n      <td>-0.428956</td>\n      <td>-1.903627</td>\n      <td>-1.214619</td>\n      <td>-0.469500</td>\n      <td>-2.590589</td>\n      <td>-0.946317</td>\n      <td>-0.390001</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1530</td>\n      <td>0</td>\n      <td>4</td>\n      <td>2.742601</td>\n      <td>1.096778</td>\n      <td>1.551411</td>\n      <td>0.632113</td>\n      <td>0.368218</td>\n      <td>2.181873</td>\n      <td>0.214604</td>\n      <td>...</td>\n      <td>-0.965623</td>\n      <td>-0.373938</td>\n      <td>-0.209282</td>\n      <td>-0.095182</td>\n      <td>-1.598217</td>\n      <td>0.968505</td>\n      <td>-0.705594</td>\n      <td>-1.579623</td>\n      <td>0.954296</td>\n      <td>-1.805623</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 92 columns</p>\n</div>"},"metadata":{}}],"execution_count":3},{"cell_type":"code","source":"sample_df.tail()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T12:32:06.882383Z","iopub.execute_input":"2024-11-23T12:32:06.882741Z","iopub.status.idle":"2024-11-23T12:32:06.903486Z","shell.execute_reply.started":"2024-11-23T12:32:06.882709Z","shell.execute_reply":"2024-11-23T12:32:06.902652Z"}},"outputs":[{"execution_count":13,"output_type":"execute_result","data":{"text/plain":"         date_id  time_id  symbol_id    weight  feature_00  feature_01  \\\n6274571     1698      967         34  3.242493    2.525160   -0.721981   \n6274572     1698      967         35  1.079139    1.857906   -0.790646   \n6274573     1698      967         36  1.033172    2.515527   -0.672298   \n6274574     1698      967         37  1.243116    2.663298   -0.889112   \n6274575     1698      967         38  3.193685    2.728506   -0.745238   \n\n         feature_02  feature_03  feature_04  feature_05  ...  feature_78  \\\n6274571    2.544025    2.477615    0.417557    0.785812  ...    0.016936   \n6274572    2.745439    2.339877    0.845065    0.651370  ...    0.050860   \n6274573    2.289250    2.521592    0.255077    0.919892  ...    0.152333   \n6274574    2.313155    3.101428    0.324454    0.618944  ...   -0.029483   \n6274575    2.788789    2.343393    0.454731    0.862839  ...   -0.247774   \n\n         responder_0  responder_1  responder_2  responder_3  responder_4  \\\n6274571     0.243475     0.166927     0.384940    -0.174297    -0.066046   \n6274572     0.850152     0.909382     1.015314     0.235962     0.122539   \n6274573     0.395684    -0.292574    -3.215846    -0.535129    -0.178484   \n6274574     1.925987     0.479394     3.621867    -0.107114    -0.063599   \n6274575     1.228778     0.512562    -0.050865     0.160883     0.080756   \n\n         responder_5  responder_6  responder_7  responder_8  \n6274571    -0.038767    -0.132337    -0.022426    -0.252461  \n6274572     0.099559    -0.249584    -0.123571    -0.460630  \n6274573    -1.808150    -0.065355    -0.000367    -0.125170  \n6274574     1.204755    -0.148711    -0.026583    -0.256395  \n6274575    -0.078237    -0.138548    -0.038771    -0.211940  \n\n[5 rows x 92 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>date_id</th>\n      <th>time_id</th>\n      <th>symbol_id</th>\n      <th>weight</th>\n      <th>feature_00</th>\n      <th>feature_01</th>\n      <th>feature_02</th>\n      <th>feature_03</th>\n      <th>feature_04</th>\n      <th>feature_05</th>\n      <th>...</th>\n      <th>feature_78</th>\n      <th>responder_0</th>\n      <th>responder_1</th>\n      <th>responder_2</th>\n      <th>responder_3</th>\n      <th>responder_4</th>\n      <th>responder_5</th>\n      <th>responder_6</th>\n      <th>responder_7</th>\n      <th>responder_8</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>6274571</th>\n      <td>1698</td>\n      <td>967</td>\n      <td>34</td>\n      <td>3.242493</td>\n      <td>2.525160</td>\n      <td>-0.721981</td>\n      <td>2.544025</td>\n      <td>2.477615</td>\n      <td>0.417557</td>\n      <td>0.785812</td>\n      <td>...</td>\n      <td>0.016936</td>\n      <td>0.243475</td>\n      <td>0.166927</td>\n      <td>0.384940</td>\n      <td>-0.174297</td>\n      <td>-0.066046</td>\n      <td>-0.038767</td>\n      <td>-0.132337</td>\n      <td>-0.022426</td>\n      <td>-0.252461</td>\n    </tr>\n    <tr>\n      <th>6274572</th>\n      <td>1698</td>\n      <td>967</td>\n      <td>35</td>\n      <td>1.079139</td>\n      <td>1.857906</td>\n      <td>-0.790646</td>\n      <td>2.745439</td>\n      <td>2.339877</td>\n      <td>0.845065</td>\n      <td>0.651370</td>\n      <td>...</td>\n      <td>0.050860</td>\n      <td>0.850152</td>\n      <td>0.909382</td>\n      <td>1.015314</td>\n      <td>0.235962</td>\n      <td>0.122539</td>\n      <td>0.099559</td>\n      <td>-0.249584</td>\n      <td>-0.123571</td>\n      <td>-0.460630</td>\n    </tr>\n    <tr>\n      <th>6274573</th>\n      <td>1698</td>\n      <td>967</td>\n      <td>36</td>\n      <td>1.033172</td>\n      <td>2.515527</td>\n      <td>-0.672298</td>\n      <td>2.289250</td>\n      <td>2.521592</td>\n      <td>0.255077</td>\n      <td>0.919892</td>\n      <td>...</td>\n      <td>0.152333</td>\n      <td>0.395684</td>\n      <td>-0.292574</td>\n      <td>-3.215846</td>\n      <td>-0.535129</td>\n      <td>-0.178484</td>\n      <td>-1.808150</td>\n      <td>-0.065355</td>\n      <td>-0.000367</td>\n      <td>-0.125170</td>\n    </tr>\n    <tr>\n      <th>6274574</th>\n      <td>1698</td>\n      <td>967</td>\n      <td>37</td>\n      <td>1.243116</td>\n      <td>2.663298</td>\n      <td>-0.889112</td>\n      <td>2.313155</td>\n      <td>3.101428</td>\n      <td>0.324454</td>\n      <td>0.618944</td>\n      <td>...</td>\n      <td>-0.029483</td>\n      <td>1.925987</td>\n      <td>0.479394</td>\n      <td>3.621867</td>\n      <td>-0.107114</td>\n      <td>-0.063599</td>\n      <td>1.204755</td>\n      <td>-0.148711</td>\n      <td>-0.026583</td>\n      <td>-0.256395</td>\n    </tr>\n    <tr>\n      <th>6274575</th>\n      <td>1698</td>\n      <td>967</td>\n      <td>38</td>\n      <td>3.193685</td>\n      <td>2.728506</td>\n      <td>-0.745238</td>\n      <td>2.788789</td>\n      <td>2.343393</td>\n      <td>0.454731</td>\n      <td>0.862839</td>\n      <td>...</td>\n      <td>-0.247774</td>\n      <td>1.228778</td>\n      <td>0.512562</td>\n      <td>-0.050865</td>\n      <td>0.160883</td>\n      <td>0.080756</td>\n      <td>-0.078237</td>\n      <td>-0.138548</td>\n      <td>-0.038771</td>\n      <td>-0.211940</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 92 columns</p>\n</div>"},"metadata":{}}],"execution_count":13},{"cell_type":"code","source":"sample_df.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T11:56:21.300519Z","iopub.execute_input":"2024-11-23T11:56:21.300869Z","iopub.status.idle":"2024-11-23T11:56:21.306561Z","shell.execute_reply.started":"2024-11-23T11:56:21.300838Z","shell.execute_reply":"2024-11-23T11:56:21.305519Z"}},"outputs":[{"execution_count":5,"output_type":"execute_result","data":{"text/plain":"(6274576, 92)"},"metadata":{}}],"execution_count":5},{"cell_type":"code","source":"sample_df.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T12:27:48.155845Z","iopub.execute_input":"2024-11-23T12:27:48.156208Z","iopub.status.idle":"2024-11-23T12:28:06.240054Z","shell.execute_reply.started":"2024-11-23T12:27:48.156177Z","shell.execute_reply":"2024-11-23T12:28:06.239169Z"}},"outputs":[{"execution_count":12,"output_type":"execute_result","data":{"text/plain":"            date_id       time_id     symbol_id        weight    feature_00  \\\ncount  6.274576e+06  6.274576e+06  6.274576e+06  6.274576e+06  6.274576e+06   \nmean   1.614142e+03  4.835000e+02  1.901404e+01  2.361384e+00  1.873208e+00   \nstd    4.882171e+01  2.794374e+02  1.124457e+01  1.098897e+00  1.456878e+00   \nmin    1.530000e+03  0.000000e+00  0.000000e+00  5.324591e-01 -2.561013e+00   \n25%    1.572000e+03  2.417500e+02  9.000000e+00  1.523484e+00  7.104638e-01   \n50%    1.614000e+03  4.835000e+02  1.900000e+01  2.162010e+00  2.011877e+00   \n75%    1.656000e+03  7.252500e+02  2.900000e+01  2.966673e+00  3.047259e+00   \nmax    1.698000e+03  9.670000e+02  3.800000e+01  7.902302e+00  6.477002e+00   \n\n         feature_01    feature_02    feature_03    feature_04    feature_05  \\\ncount  6.274576e+06  6.274576e+06  6.274576e+06  6.274576e+06  6.274576e+06   \nmean   2.602108e-02  1.864772e+00  1.864176e+00 -1.212088e-02 -4.847535e-02   \nstd    1.139380e+00  1.448963e+00  1.449271e+00  1.000943e+00  9.488913e-01   \nmin   -4.274224e+00 -2.605949e+00 -2.540449e+00 -4.105798e+00 -1.239336e+01   \n25%   -7.398547e-01  7.112520e-01  7.093264e-01 -6.788018e-01 -5.004615e-01   \n50%   -2.175932e-02  2.012542e+00  2.011695e+00 -1.520159e-02 -4.794242e-02   \n75%    7.497100e-01  3.046757e+00  3.044584e+00  6.349058e-01  4.242871e-01   \nmax    4.709325e+00  6.490265e+00  6.695623e+00  4.405029e+00  1.384570e+01   \n\n       ...    feature_78   responder_0   responder_1   responder_2  \\\ncount  ...  6.272631e+06  6.274576e+06  6.274576e+06  6.274576e+06   \nmean   ...  1.325256e-02 -1.347268e-03 -8.555991e-04  3.291748e-04   \nstd    ...  9.861734e-01  4.165042e-01  4.201646e-01  4.270543e-01   \nmin    ... -4.876283e+00 -5.000000e+00 -5.000000e+00 -5.000000e+00   \n25%    ... -2.731894e-01 -1.645727e-01 -1.354915e-01 -2.052442e-01   \n50%    ... -1.445980e-01 -3.149339e-03 -2.037835e-02  2.297103e-05   \n75%    ...  8.683352e-02  1.594334e-01  1.022870e-01  2.057978e-01   \nmax    ...  1.725921e+02  5.000000e+00  5.000000e+00  5.000000e+00   \n\n        responder_3   responder_4   responder_5   responder_6   responder_7  \\\ncount  6.274576e+06  6.274576e+06  6.274576e+06  6.274576e+06  6.274576e+06   \nmean  -3.937317e-02 -3.898935e-02 -3.454752e-02 -3.780264e-03 -6.048032e-03   \nstd    1.014420e+00  9.403012e-01  9.721982e-01  8.141425e-01  8.285853e-01   \nmin   -5.000000e+00 -5.000000e+00 -5.000000e+00 -5.000000e+00 -5.000000e+00   \n25%   -3.006397e-01 -3.786564e-01 -2.284731e-01 -3.751850e-01 -4.153281e-01   \n50%   -3.339716e-02 -5.760912e-02 -1.520078e-02 -3.443441e-02 -5.519713e-02   \n75%    2.375821e-01  2.806393e-01  1.951360e-01  3.136375e-01  3.253073e-01   \nmax    5.000000e+00  5.000000e+00  5.000000e+00  5.000000e+00  5.000000e+00   \n\n        responder_8  \ncount  6.274576e+06  \nmean  -2.954880e-04  \nstd    7.954518e-01  \nmin   -5.000000e+00  \n25%   -3.402549e-01  \n50%   -1.534246e-02  \n75%    3.090042e-01  \nmax    5.000000e+00  \n\n[8 rows x 92 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>date_id</th>\n      <th>time_id</th>\n      <th>symbol_id</th>\n      <th>weight</th>\n      <th>feature_00</th>\n      <th>feature_01</th>\n      <th>feature_02</th>\n      <th>feature_03</th>\n      <th>feature_04</th>\n      <th>feature_05</th>\n      <th>...</th>\n      <th>feature_78</th>\n      <th>responder_0</th>\n      <th>responder_1</th>\n      <th>responder_2</th>\n      <th>responder_3</th>\n      <th>responder_4</th>\n      <th>responder_5</th>\n      <th>responder_6</th>\n      <th>responder_7</th>\n      <th>responder_8</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>count</th>\n      <td>6.274576e+06</td>\n      <td>6.274576e+06</td>\n      <td>6.274576e+06</td>\n      <td>6.274576e+06</td>\n      <td>6.274576e+06</td>\n      <td>6.274576e+06</td>\n      <td>6.274576e+06</td>\n      <td>6.274576e+06</td>\n      <td>6.274576e+06</td>\n      <td>6.274576e+06</td>\n      <td>...</td>\n      <td>6.272631e+06</td>\n      <td>6.274576e+06</td>\n      <td>6.274576e+06</td>\n      <td>6.274576e+06</td>\n      <td>6.274576e+06</td>\n      <td>6.274576e+06</td>\n      <td>6.274576e+06</td>\n      <td>6.274576e+06</td>\n      <td>6.274576e+06</td>\n      <td>6.274576e+06</td>\n    </tr>\n    <tr>\n      <th>mean</th>\n      <td>1.614142e+03</td>\n      <td>4.835000e+02</td>\n      <td>1.901404e+01</td>\n      <td>2.361384e+00</td>\n      <td>1.873208e+00</td>\n      <td>2.602108e-02</td>\n      <td>1.864772e+00</td>\n      <td>1.864176e+00</td>\n      <td>-1.212088e-02</td>\n      <td>-4.847535e-02</td>\n      <td>...</td>\n      <td>1.325256e-02</td>\n      <td>-1.347268e-03</td>\n      <td>-8.555991e-04</td>\n      <td>3.291748e-04</td>\n      <td>-3.937317e-02</td>\n      <td>-3.898935e-02</td>\n      <td>-3.454752e-02</td>\n      <td>-3.780264e-03</td>\n      <td>-6.048032e-03</td>\n      <td>-2.954880e-04</td>\n    </tr>\n    <tr>\n      <th>std</th>\n      <td>4.882171e+01</td>\n      <td>2.794374e+02</td>\n      <td>1.124457e+01</td>\n      <td>1.098897e+00</td>\n      <td>1.456878e+00</td>\n      <td>1.139380e+00</td>\n      <td>1.448963e+00</td>\n      <td>1.449271e+00</td>\n      <td>1.000943e+00</td>\n      <td>9.488913e-01</td>\n      <td>...</td>\n      <td>9.861734e-01</td>\n      <td>4.165042e-01</td>\n      <td>4.201646e-01</td>\n      <td>4.270543e-01</td>\n      <td>1.014420e+00</td>\n      <td>9.403012e-01</td>\n      <td>9.721982e-01</td>\n      <td>8.141425e-01</td>\n      <td>8.285853e-01</td>\n      <td>7.954518e-01</td>\n    </tr>\n    <tr>\n      <th>min</th>\n      <td>1.530000e+03</td>\n      <td>0.000000e+00</td>\n      <td>0.000000e+00</td>\n      <td>5.324591e-01</td>\n      <td>-2.561013e+00</td>\n      <td>-4.274224e+00</td>\n      <td>-2.605949e+00</td>\n      <td>-2.540449e+00</td>\n      <td>-4.105798e+00</td>\n      <td>-1.239336e+01</td>\n      <td>...</td>\n      <td>-4.876283e+00</td>\n      <td>-5.000000e+00</td>\n      <td>-5.000000e+00</td>\n      <td>-5.000000e+00</td>\n      <td>-5.000000e+00</td>\n      <td>-5.000000e+00</td>\n      <td>-5.000000e+00</td>\n      <td>-5.000000e+00</td>\n      <td>-5.000000e+00</td>\n      <td>-5.000000e+00</td>\n    </tr>\n    <tr>\n      <th>25%</th>\n      <td>1.572000e+03</td>\n      <td>2.417500e+02</td>\n      <td>9.000000e+00</td>\n      <td>1.523484e+00</td>\n      <td>7.104638e-01</td>\n      <td>-7.398547e-01</td>\n      <td>7.112520e-01</td>\n      <td>7.093264e-01</td>\n      <td>-6.788018e-01</td>\n      <td>-5.004615e-01</td>\n      <td>...</td>\n      <td>-2.731894e-01</td>\n      <td>-1.645727e-01</td>\n      <td>-1.354915e-01</td>\n      <td>-2.052442e-01</td>\n      <td>-3.006397e-01</td>\n      <td>-3.786564e-01</td>\n      <td>-2.284731e-01</td>\n      <td>-3.751850e-01</td>\n      <td>-4.153281e-01</td>\n      <td>-3.402549e-01</td>\n    </tr>\n    <tr>\n      <th>50%</th>\n      <td>1.614000e+03</td>\n      <td>4.835000e+02</td>\n      <td>1.900000e+01</td>\n      <td>2.162010e+00</td>\n      <td>2.011877e+00</td>\n      <td>-2.175932e-02</td>\n      <td>2.012542e+00</td>\n      <td>2.011695e+00</td>\n      <td>-1.520159e-02</td>\n      <td>-4.794242e-02</td>\n      <td>...</td>\n      <td>-1.445980e-01</td>\n      <td>-3.149339e-03</td>\n      <td>-2.037835e-02</td>\n      <td>2.297103e-05</td>\n      <td>-3.339716e-02</td>\n      <td>-5.760912e-02</td>\n      <td>-1.520078e-02</td>\n      <td>-3.443441e-02</td>\n      <td>-5.519713e-02</td>\n      <td>-1.534246e-02</td>\n    </tr>\n    <tr>\n      <th>75%</th>\n      <td>1.656000e+03</td>\n      <td>7.252500e+02</td>\n      <td>2.900000e+01</td>\n      <td>2.966673e+00</td>\n      <td>3.047259e+00</td>\n      <td>7.497100e-01</td>\n      <td>3.046757e+00</td>\n      <td>3.044584e+00</td>\n      <td>6.349058e-01</td>\n      <td>4.242871e-01</td>\n      <td>...</td>\n      <td>8.683352e-02</td>\n      <td>1.594334e-01</td>\n      <td>1.022870e-01</td>\n      <td>2.057978e-01</td>\n      <td>2.375821e-01</td>\n      <td>2.806393e-01</td>\n      <td>1.951360e-01</td>\n      <td>3.136375e-01</td>\n      <td>3.253073e-01</td>\n      <td>3.090042e-01</td>\n    </tr>\n    <tr>\n      <th>max</th>\n      <td>1.698000e+03</td>\n      <td>9.670000e+02</td>\n      <td>3.800000e+01</td>\n      <td>7.902302e+00</td>\n      <td>6.477002e+00</td>\n      <td>4.709325e+00</td>\n      <td>6.490265e+00</td>\n      <td>6.695623e+00</td>\n      <td>4.405029e+00</td>\n      <td>1.384570e+01</td>\n      <td>...</td>\n      <td>1.725921e+02</td>\n      <td>5.000000e+00</td>\n      <td>5.000000e+00</td>\n      <td>5.000000e+00</td>\n      <td>5.000000e+00</td>\n      <td>5.000000e+00</td>\n      <td>5.000000e+00</td>\n      <td>5.000000e+00</td>\n      <td>5.000000e+00</td>\n      <td>5.000000e+00</td>\n    </tr>\n  </tbody>\n</table>\n<p>8 rows × 92 columns</p>\n</div>"},"metadata":{}}],"execution_count":12},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models\n# Separate features and responders\nfeatures = sample_df.filter(regex='^feature_')\nresponders = sample_df.filter(regex='^responder_')\nweights = sample_df['weight']\n# Convert to numpy arrays for TensorFlow\nX = features.values  # Features for input\n#y = responders.values  # Responders for output\n# Assuming you have a DataFrame `y_train` with all responders\ny = responders[['responder_6']].values  # Keep only responder_6\nX = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)\ny = np.nan_to_num(y, nan=0.0, posinf=0.0, neginf=0.0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T12:03:27.999117Z","iopub.execute_input":"2024-11-23T12:03:27.999467Z","iopub.status.idle":"2024-11-23T12:03:43.061474Z","shell.execute_reply.started":"2024-11-23T12:03:27.999438Z","shell.execute_reply":"2024-11-23T12:03:43.060461Z"}},"outputs":[],"execution_count":6},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_val, y_train, y_val, weights_train, weights_val = train_test_split(\n    X, y, weights, test_size=0.2, random_state=42\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T12:03:43.063409Z","iopub.execute_input":"2024-11-23T12:03:43.063926Z","iopub.status.idle":"2024-11-23T12:03:49.424839Z","shell.execute_reply.started":"2024-11-23T12:03:43.063896Z","shell.execute_reply":"2024-11-23T12:03:49.423862Z"}},"outputs":[],"execution_count":7},{"cell_type":"code","source":"from tensorflow.keras import layers, models\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\n# Define the Autoencoder model\ninput_dim = X_train.shape[1]  # Number of features\nlatent_dim = 32  # Dimension of the bottleneck layer\nencoder_input = layers.Input(shape=(input_dim,))\nx = layers.Dense(128, activation='relu')(encoder_input)\nx = layers.Dense(64, activation='relu')(x)\nbottleneck = layers.Dense(latent_dim, activation='linear', name='bottleneck')(x)  # Encoder output\n# Decoder\nx = layers.Dense(64, activation='relu')(bottleneck)\nx = layers.Dense(128, activation='relu')(x)\ndecoder_output = layers.Dense(input_dim, activation='linear')(x)\nautoencoder = models.Model(encoder_input, decoder_output, name=\"Autoencoder\")\n# Compile the Autoencoder\nautoencoder.compile(optimizer=\"adam\", loss=\"mse\")\nautoencoder.summary()\n# Define callbacks\nearly_stopping = EarlyStopping(monitor=\"val_loss\", patience=10, restore_best_weights=True, min_delta = 0.00001)\nreduce_lr = ReduceLROnPlateau(monitor=\"val_loss\", factor=0.5, patience=3, min_lr=1e-6)\n# Train the Autoencoder\nhistory = autoencoder.fit(\n    X_train, X_train,\n    validation_data=(X_val, X_val),\n    epochs=1,\n    batch_size=32,\n    callbacks=[early_stopping, reduce_lr]\n)\n# Extract the encoder\nencoder = models.Model(encoder_input, bottleneck, name=\"Encoder\")\nencoder.save(\"/kaggle/working/pretrained_encoder.keras\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T12:11:50.92874Z","iopub.execute_input":"2024-11-23T12:11:50.929972Z","iopub.status.idle":"2024-11-23T12:16:16.054079Z","shell.execute_reply.started":"2024-11-23T12:11:50.929933Z","shell.execute_reply":"2024-11-23T12:16:16.053357Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"\u001b[1mModel: \"Autoencoder\"\u001b[0m\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"Autoencoder\"</span>\n</pre>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                   \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape          \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      Param #\u001b[0m\u001b[1m \u001b[0m┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n│ input_layer (\u001b[38;5;33mInputLayer\u001b[0m)        │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m79\u001b[0m)             │             \u001b[38;5;34m0\u001b[0m │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ dense (\u001b[38;5;33mDense\u001b[0m)                   │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m128\u001b[0m)            │        \u001b[38;5;34m10,240\u001b[0m │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ dense_1 (\u001b[38;5;33mDense\u001b[0m)                 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m64\u001b[0m)             │         \u001b[38;5;34m8,256\u001b[0m │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ bottleneck (\u001b[38;5;33mDense\u001b[0m)              │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m32\u001b[0m)             │         \u001b[38;5;34m2,080\u001b[0m │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ dense_2 (\u001b[38;5;33mDense\u001b[0m)                 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m64\u001b[0m)             │         \u001b[38;5;34m2,112\u001b[0m │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ dense_3 (\u001b[38;5;33mDense\u001b[0m)                 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m128\u001b[0m)            │         \u001b[38;5;34m8,320\u001b[0m │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ dense_4 (\u001b[38;5;33mDense\u001b[0m)                 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m79\u001b[0m)             │        \u001b[38;5;34m10,191\u001b[0m │\n└─────────────────────────────────┴────────────────────────┴───────────────┘\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n┃<span style=\"font-weight: bold\"> Layer (type)                    </span>┃<span style=\"font-weight: bold\"> Output Shape           </span>┃<span style=\"font-weight: bold\">       Param # </span>┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n│ input_layer (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)        │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">79</span>)             │             <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ dense (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                   │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)            │        <span style=\"color: #00af00; text-decoration-color: #00af00\">10,240</span> │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ dense_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)             │         <span style=\"color: #00af00; text-decoration-color: #00af00\">8,256</span> │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ bottleneck (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)              │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>)             │         <span style=\"color: #00af00; text-decoration-color: #00af00\">2,080</span> │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ dense_2 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)             │         <span style=\"color: #00af00; text-decoration-color: #00af00\">2,112</span> │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ dense_3 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)            │         <span style=\"color: #00af00; text-decoration-color: #00af00\">8,320</span> │\n├─────────────────────────────────┼────────────────────────┼───────────────┤\n│ dense_4 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">79</span>)             │        <span style=\"color: #00af00; text-decoration-color: #00af00\">10,191</span> │\n└─────────────────────────────────┴────────────────────────┴───────────────┘\n</pre>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"\u001b[1m Total params: \u001b[0m\u001b[38;5;34m41,199\u001b[0m (160.93 KB)\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">41,199</span> (160.93 KB)\n</pre>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m41,199\u001b[0m (160.93 KB)\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">41,199</span> (160.93 KB)\n</pre>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n</pre>\n"},"metadata":{}},{"name":"stderr","text":"WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\nI0000 00:00:1732363921.060147     144 service.cc:145] XLA service 0x7acb28006820 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\nI0000 00:00:1732363921.060205     144 service.cc:153]   StreamExecutor device (0): Tesla P100-PCIE-16GB, Compute Capability 6.0\n","output_type":"stream"},{"name":"stdout","text":"\u001b[1m   110/156865\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m3:38\u001b[0m 1ms/step - loss: 208.8549  ","output_type":"stream"},{"name":"stderr","text":"I0000 00:00:1732363922.420645     144 device_compiler.h:188] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.\n","output_type":"stream"},{"name":"stdout","text":"\u001b[1m156865/156865\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m256s\u001b[0m 2ms/step - loss: 0.6948 - val_loss: 0.0673 - learning_rate: 0.0010\n","output_type":"stream"}],"execution_count":8},{"cell_type":"code","source":"def learning_rate_scheduler_xgb(epoch):\n    initial_rate = 0.3\n    decay_rate = 0.999\n    return initial_rate * (decay_rate ** (np.log(epoch)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T12:17:57.549088Z","iopub.execute_input":"2024-11-23T12:17:57.549919Z","iopub.status.idle":"2024-11-23T12:17:57.554033Z","shell.execute_reply.started":"2024-11-23T12:17:57.549885Z","shell.execute_reply":"2024-11-23T12:17:57.55307Z"}},"outputs":[],"execution_count":9},{"cell_type":"code","source":"from xgboost import XGBRegressor\n# Create an XGBoost model\nmodel_xgb = XGBRegressor(\n    n_estimators=5000,\n    learning_rate=learning_rate_scheduler_xgb,\n    tree_method='hist',\n    max_depth=6,\n    random_state=42\n)\n# Fit the model with sample weights and validation dataset\nmodel_xgb.fit(\n    X_train,\n    y_train,\n #   sample_weight=weights_train,\n    eval_set=[(X_val, y_val)],\n#    sample_weight_eval_set=[weights_train, weights_val],\n    eval_metric='rmse',\n    early_stopping_rounds=10,\n    verbose=False\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T12:26:28.138857Z","iopub.execute_input":"2024-11-23T12:26:28.139151Z","iopub.status.idle":"2024-11-23T12:26:44.564041Z","shell.execute_reply.started":"2024-11-23T12:26:28.139122Z","shell.execute_reply":"2024-11-23T12:26:44.562635Z"}},"outputs":[{"name":"stderr","text":"Exception ignored on calling ctypes callback function: <bound method DataIter._next_wrapper of <xgboost.data.SingleBatchInternalIter object at 0x7acecda17940>>\nTraceback (most recent call last):\n  File \"/opt/conda/lib/python3.10/site-packages/xgboost/core.py\", line 589, in _next_wrapper\n    def _next_wrapper(self, this: None) -> int:  # pylint: disable=unused-argument\nKeyboardInterrupt: \n","output_type":"stream"},{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mXGBoostError\u001b[0m                              Traceback (most recent call last)","Cell \u001b[0;32mIn[11], line 11\u001b[0m\n\u001b[1;32m      3\u001b[0m model_xgb \u001b[38;5;241m=\u001b[39m XGBRegressor(\n\u001b[1;32m      4\u001b[0m     n_estimators\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m5000\u001b[39m,\n\u001b[1;32m      5\u001b[0m     learning_rate\u001b[38;5;241m=\u001b[39mlearning_rate_scheduler_xgb,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m      8\u001b[0m     random_state\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m42\u001b[39m\n\u001b[1;32m      9\u001b[0m )\n\u001b[1;32m     10\u001b[0m \u001b[38;5;66;03m# Fit the model with sample weights and validation dataset\u001b[39;00m\n\u001b[0;32m---> 11\u001b[0m \u001b[43mmodel_xgb\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     12\u001b[0m \u001b[43m    \u001b[49m\u001b[43mX_train\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     13\u001b[0m \u001b[43m    \u001b[49m\u001b[43my_train\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     14\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;66;43;03m#   sample_weight=weights_train,\u001b[39;49;00m\n\u001b[1;32m     15\u001b[0m \u001b[43m    \u001b[49m\u001b[43meval_set\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX_val\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_val\u001b[49m\u001b[43m)\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     16\u001b[0m \u001b[38;5;66;43;03m#    sample_weight_eval_set=[weights_train, weights_val],\u001b[39;49;00m\n\u001b[1;32m     17\u001b[0m \u001b[43m    \u001b[49m\u001b[43meval_metric\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mrmse\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m     18\u001b[0m \u001b[43m    \u001b[49m\u001b[43mearly_stopping_rounds\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m10\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m     19\u001b[0m \u001b[43m    \u001b[49m\u001b[43mverbose\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\n\u001b[1;32m     20\u001b[0m \u001b[43m)\u001b[49m\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/xgboost/core.py:730\u001b[0m, in \u001b[0;36mrequire_keyword_args.<locals>.throw_if.<locals>.inner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    728\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m k, arg \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(sig\u001b[38;5;241m.\u001b[39mparameters, args):\n\u001b[1;32m    729\u001b[0m     kwargs[k] \u001b[38;5;241m=\u001b[39m arg\n\u001b[0;32m--> 730\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/xgboost/sklearn.py:1055\u001b[0m, in \u001b[0;36mXGBModel.fit\u001b[0;34m(self, X, y, sample_weight, base_margin, eval_set, eval_metric, early_stopping_rounds, verbose, xgb_model, sample_weight_eval_set, base_margin_eval_set, feature_weights, callbacks)\u001b[0m\n\u001b[1;32m   1053\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m config_context(verbosity\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mverbosity):\n\u001b[1;32m   1054\u001b[0m     evals_result: TrainingCallback\u001b[38;5;241m.\u001b[39mEvalsLog \u001b[38;5;241m=\u001b[39m {}\n\u001b[0;32m-> 1055\u001b[0m     train_dmatrix, evals \u001b[38;5;241m=\u001b[39m \u001b[43m_wrap_evaluation_matrices\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1056\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmissing\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmissing\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1057\u001b[0m \u001b[43m        \u001b[49m\u001b[43mX\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1058\u001b[0m \u001b[43m        \u001b[49m\u001b[43my\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43my\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1059\u001b[0m \u001b[43m        \u001b[49m\u001b[43mgroup\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m   1060\u001b[0m \u001b[43m        \u001b[49m\u001b[43mqid\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m   1061\u001b[0m \u001b[43m        \u001b[49m\u001b[43msample_weight\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msample_weight\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1062\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbase_margin\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbase_margin\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1063\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfeature_weights\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfeature_weights\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1064\u001b[0m \u001b[43m        \u001b[49m\u001b[43meval_set\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43meval_set\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1065\u001b[0m \u001b[43m        \u001b[49m\u001b[43msample_weight_eval_set\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msample_weight_eval_set\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1066\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbase_margin_eval_set\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbase_margin_eval_set\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1067\u001b[0m \u001b[43m        \u001b[49m\u001b[43meval_group\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m   1068\u001b[0m \u001b[43m        \u001b[49m\u001b[43meval_qid\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m   1069\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcreate_dmatrix\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_create_dmatrix\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1070\u001b[0m \u001b[43m        \u001b[49m\u001b[43menable_categorical\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menable_categorical\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1071\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfeature_types\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfeature_types\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1072\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1073\u001b[0m     params \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_xgb_params()\n\u001b[1;32m   1075\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mcallable\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mobjective):\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/xgboost/sklearn.py:521\u001b[0m, in \u001b[0;36m_wrap_evaluation_matrices\u001b[0;34m(missing, X, y, group, qid, sample_weight, base_margin, feature_weights, eval_set, sample_weight_eval_set, base_margin_eval_set, eval_group, eval_qid, create_dmatrix, enable_categorical, feature_types)\u001b[0m\n\u001b[1;32m    501\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_wrap_evaluation_matrices\u001b[39m(\n\u001b[1;32m    502\u001b[0m     missing: \u001b[38;5;28mfloat\u001b[39m,\n\u001b[1;32m    503\u001b[0m     X: Any,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    517\u001b[0m     feature_types: Optional[FeatureTypes],\n\u001b[1;32m    518\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[Any, List[Tuple[Any, \u001b[38;5;28mstr\u001b[39m]]]:\n\u001b[1;32m    519\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Convert array_like evaluation matrices into DMatrix.  Perform validation on the\u001b[39;00m\n\u001b[1;32m    520\u001b[0m \u001b[38;5;124;03m    way.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 521\u001b[0m     train_dmatrix \u001b[38;5;241m=\u001b[39m \u001b[43mcreate_dmatrix\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    522\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    523\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlabel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43my\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    524\u001b[0m \u001b[43m        \u001b[49m\u001b[43mgroup\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    525\u001b[0m \u001b[43m        \u001b[49m\u001b[43mqid\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mqid\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    526\u001b[0m \u001b[43m        \u001b[49m\u001b[43mweight\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msample_weight\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    527\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbase_margin\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbase_margin\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    528\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfeature_weights\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfeature_weights\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    529\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmissing\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmissing\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    530\u001b[0m \u001b[43m        \u001b[49m\u001b[43menable_categorical\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43menable_categorical\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    531\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfeature_types\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfeature_types\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    532\u001b[0m \u001b[43m        \u001b[49m\u001b[43mref\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    533\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    535\u001b[0m     n_validation \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m eval_set \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(eval_set)\n\u001b[1;32m    537\u001b[0m     \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mvalidate_or_none\u001b[39m(meta: Optional[Sequence], name: \u001b[38;5;28mstr\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Sequence:\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/xgboost/sklearn.py:958\u001b[0m, in \u001b[0;36mXGBModel._create_dmatrix\u001b[0;34m(self, ref, **kwargs)\u001b[0m\n\u001b[1;32m    956\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m _can_use_qdm(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtree_method) \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbooster \u001b[38;5;241m!=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgblinear\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m    957\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 958\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mQuantileDMatrix\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    959\u001b[0m \u001b[43m            \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mref\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mref\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnthread\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mn_jobs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmax_bin\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmax_bin\u001b[49m\n\u001b[1;32m    960\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    961\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m:  \u001b[38;5;66;03m# `QuantileDMatrix` supports lesser types than DMatrix\u001b[39;00m\n\u001b[1;32m    962\u001b[0m         \u001b[38;5;28;01mpass\u001b[39;00m\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/xgboost/core.py:730\u001b[0m, in \u001b[0;36mrequire_keyword_args.<locals>.throw_if.<locals>.inner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    728\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m k, arg \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(sig\u001b[38;5;241m.\u001b[39mparameters, args):\n\u001b[1;32m    729\u001b[0m     kwargs[k] \u001b[38;5;241m=\u001b[39m arg\n\u001b[0;32m--> 730\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/xgboost/core.py:1529\u001b[0m, in \u001b[0;36mQuantileDMatrix.__init__\u001b[0;34m(self, data, label, weight, base_margin, missing, silent, feature_names, feature_types, nthread, max_bin, ref, group, qid, label_lower_bound, label_upper_bound, feature_weights, enable_categorical, data_split_mode)\u001b[0m\n\u001b[1;32m   1509\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28many\u001b[39m(\n\u001b[1;32m   1510\u001b[0m         info \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m   1511\u001b[0m         \u001b[38;5;28;01mfor\u001b[39;00m info \u001b[38;5;129;01min\u001b[39;00m (\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1522\u001b[0m         )\n\u001b[1;32m   1523\u001b[0m     ):\n\u001b[1;32m   1524\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m   1525\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIf data iterator is used as input, data like label should be \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   1526\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mspecified as batch argument.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   1527\u001b[0m         )\n\u001b[0;32m-> 1529\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_init\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1530\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1531\u001b[0m \u001b[43m    \u001b[49m\u001b[43mref\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mref\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1532\u001b[0m \u001b[43m    \u001b[49m\u001b[43mlabel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlabel\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1533\u001b[0m \u001b[43m    \u001b[49m\u001b[43mweight\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1534\u001b[0m \u001b[43m    \u001b[49m\u001b[43mbase_margin\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbase_margin\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1535\u001b[0m \u001b[43m    \u001b[49m\u001b[43mgroup\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1536\u001b[0m \u001b[43m    \u001b[49m\u001b[43mqid\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mqid\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1537\u001b[0m \u001b[43m    \u001b[49m\u001b[43mlabel_lower_bound\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlabel_lower_bound\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1538\u001b[0m \u001b[43m    \u001b[49m\u001b[43mlabel_upper_bound\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlabel_upper_bound\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1539\u001b[0m \u001b[43m    \u001b[49m\u001b[43mfeature_weights\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfeature_weights\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1540\u001b[0m \u001b[43m    \u001b[49m\u001b[43mfeature_names\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfeature_names\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1541\u001b[0m \u001b[43m    \u001b[49m\u001b[43mfeature_types\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfeature_types\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1542\u001b[0m \u001b[43m    \u001b[49m\u001b[43menable_categorical\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43menable_categorical\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1543\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/xgboost/core.py:1590\u001b[0m, in \u001b[0;36mQuantileDMatrix._init\u001b[0;34m(self, data, ref, enable_categorical, **meta)\u001b[0m\n\u001b[1;32m   1588\u001b[0m it\u001b[38;5;241m.\u001b[39mreraise()\n\u001b[1;32m   1589\u001b[0m \u001b[38;5;66;03m# delay check_call to throw intermediate exception first\u001b[39;00m\n\u001b[0;32m-> 1590\u001b[0m \u001b[43m_check_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43mret\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1591\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhandle \u001b[38;5;241m=\u001b[39m handle\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/xgboost/core.py:282\u001b[0m, in \u001b[0;36m_check_call\u001b[0;34m(ret)\u001b[0m\n\u001b[1;32m    271\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Check the return value of C API call\u001b[39;00m\n\u001b[1;32m    272\u001b[0m \n\u001b[1;32m    273\u001b[0m \u001b[38;5;124;03mThis function will raise exception when error occurs.\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    279\u001b[0m \u001b[38;5;124;03m    return value from API calls\u001b[39;00m\n\u001b[1;32m    280\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    281\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m ret \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m--> 282\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m XGBoostError(py_str(_LIB\u001b[38;5;241m.\u001b[39mXGBGetLastError()))\n","\u001b[0;31mXGBoostError\u001b[0m: [12:26:44] /workspace/src/common/quantile.h:770: Check failed: count <= total_entries (5019660 vs. 0) : \nStack trace:\n  [bt] (0) /opt/conda/lib/python3.10/site-packages/xgboost/lib/libxgboost.so(+0x1fd7ba) [0x7ac6fb21f7ba]\n  [bt] (1) /opt/conda/lib/python3.10/site-packages/xgboost/lib/libxgboost.so(+0x21e6e7) [0x7ac6fb2406e7]\n  [bt] (2) /opt/conda/lib/python3.10/site-packages/xgboost/lib/libxgboost.so(+0x21edcf) [0x7ac6fb240dcf]\n  [bt] (3) /opt/conda/lib/python3.10/site-packages/xgboost/lib/libxgboost.so(+0x3f65f1) [0x7ac6fb4185f1]\n  [bt] (4) /opt/conda/lib/python3.10/site-packages/xgboost/lib/libxgboost.so(+0x3f8858) [0x7ac6fb41a858]\n  [bt] (5) /opt/conda/lib/python3.10/site-packages/xgboost/lib/libxgboost.so(+0x3a2a07) [0x7ac6fb3c4a07]\n  [bt] (6) /opt/conda/lib/python3.10/site-packages/xgboost/lib/libxgboost.so(XGQuantileDMatrixCreateFromCallback+0x2b0) [0x7ac6fb187c40]\n  [bt] (7) /opt/conda/lib/python3.10/lib-dynload/../../libffi.so.8(+0x6a4a) [0x7acff3a0fa4a]\n  [bt] (8) /opt/conda/lib/python3.10/lib-dynload/../../libffi.so.8(+0x5fea) [0x7acff3a0efea]\n\n"],"ename":"XGBoostError","evalue":"[12:26:44] /workspace/src/common/quantile.h:770: Check failed: count <= total_entries (5019660 vs. 0) : \nStack trace:\n  [bt] (0) /opt/conda/lib/python3.10/site-packages/xgboost/lib/libxgboost.so(+0x1fd7ba) [0x7ac6fb21f7ba]\n  [bt] (1) /opt/conda/lib/python3.10/site-packages/xgboost/lib/libxgboost.so(+0x21e6e7) [0x7ac6fb2406e7]\n  [bt] (2) /opt/conda/lib/python3.10/site-packages/xgboost/lib/libxgboost.so(+0x21edcf) [0x7ac6fb240dcf]\n  [bt] (3) /opt/conda/lib/python3.10/site-packages/xgboost/lib/libxgboost.so(+0x3f65f1) [0x7ac6fb4185f1]\n  [bt] (4) /opt/conda/lib/python3.10/site-packages/xgboost/lib/libxgboost.so(+0x3f8858) [0x7ac6fb41a858]\n  [bt] (5) /opt/conda/lib/python3.10/site-packages/xgboost/lib/libxgboost.so(+0x3a2a07) [0x7ac6fb3c4a07]\n  [bt] (6) /opt/conda/lib/python3.10/site-packages/xgboost/lib/libxgboost.so(XGQuantileDMatrixCreateFromCallback+0x2b0) [0x7ac6fb187c40]\n  [bt] (7) /opt/conda/lib/python3.10/lib-dynload/../../libffi.so.8(+0x6a4a) [0x7acff3a0fa4a]\n  [bt] (8) /opt/conda/lib/python3.10/lib-dynload/../../libffi.so.8(+0x5fea) [0x7acff3a0efea]\n\n","output_type":"error"}],"execution_count":11},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}