{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.11"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":5456.26661,"end_time":"2025-05-24T10:59:14.597955","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-05-24T09:28:18.331345","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction","metadata":{"papermill":{"duration":0.004584,"end_time":"2025-05-24T09:28:33.615072","exception":false,"start_time":"2025-05-24T09:28:33.610488","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"- For the validation dataset, the data was split into three parts: [x|y|z]. Parts x and z were used for training, while y served as the validation set. This approach is beneficial because it leverages both the older and newer portions of the data and doesn't suffer much from future leak.\n- The competition dataset initially contained a very large number of columns. The first step was to reduce dimensionality, which was done in two stages:\n  - use sklearn.feature_selection.SelectKBest with keeping ~10% of all columns\n  - remove all columns that have negative correlation on validation for stability\n- For training I used ARD and XGB with ratio 0.3 (tuned on validation set)\n  - XGB was configured with high anti-overfitting params like reg_lambda and reg_alpha\n  - ARD was used as an additional safeguard against overfitting.\n\nDespite the simplicity of the solution, it demonstrated strong stability in the private leaderboard.","metadata":{}},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import ARDRegression\nfrom sklearn.feature_selection import SelectKBest\nfrom sklearn.feature_selection import f_regression\nfrom sklearn.linear_model import LinearRegression\nfrom scipy.stats import pearsonr\nfrom xgboost import XGBRegressor\nfrom pathlib import Path\n\nimport pandas as pd\nimport numpy as np\nimport warnings\nimport gc\n\nwarnings.filterwarnings(\"ignore\")","metadata":{"_kg_hide-output":true,"papermill":{"duration":10.023958,"end_time":"2025-05-24T09:28:43.643567","exception":false,"start_time":"2025-05-24T09:28:33.619609","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T14:50:47.727852Z","iopub.execute_input":"2025-07-31T14:50:47.728068Z","iopub.status.idle":"2025-07-31T14:50:50.521579Z","shell.execute_reply.started":"2025-07-31T14:50:47.728048Z","shell.execute_reply":"2025-07-31T14:50:50.520681Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"root_path = Path(\"/kaggle/input/drw-crypto-market-prediction\")\n\ntrain_path = root_path / \"train.parquet\"\ntest_path = root_path / \"test.parquet\"\nsample_submission_path = root_path / \"sample_submission.csv\"\n\ntarget = \"label\"\nseed = 42\n\nxgb_params = {\n    \"learning_rate\": 0.02,\n    \"max_depth\": 20,\n    \"max_leaves\": 12,\n    \"min_child_weight\": 16,\n    \"n_estimators\": 1500,\n    \"reg_alpha\": 10,\n    \"reg_lambda\": 100,\n    \"subsample\": 0.05,\n\n    \"n_jobs\": -1,\n    \"random_state\": seed,\n    \"verbosity\": 0\n}\n\nhyperparameters = {\n    'features': 70, # ~10%\n}","metadata":{"papermill":{"duration":0.012751,"end_time":"2025-05-24T09:28:43.661332","exception":false,"start_time":"2025-05-24T09:28:43.648581","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T14:50:50.522872Z","iopub.execute_input":"2025-07-31T14:50:50.523352Z","iopub.status.idle":"2025-07-31T14:50:50.529303Z","shell.execute_reply.started":"2025-07-31T14:50:50.523323Z","shell.execute_reply":"2025-07-31T14:50:50.528417Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data loading","metadata":{}},{"cell_type":"code","source":"train = pd.read_parquet(train_path).astype(np.float16) # to prevent OOM in kaggle\ntest = pd.read_parquet(test_path).astype(np.float16) # to prevent OOM in kaggle","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T14:50:50.531552Z","iopub.execute_input":"2025-07-31T14:50:50.532046Z","iopub.status.idle":"2025-07-31T14:51:42.991449Z","shell.execute_reply.started":"2025-07-31T14:50:50.532014Z","shell.execute_reply":"2025-07-31T14:51:42.990299Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"parts = np.array_split(np.arange(len(train)), 3)\ntrain_df = train.iloc[np.hstack([parts[0], parts[2]])].reset_index(drop=True)\nval_df = train.iloc[parts[1]].reset_index(drop=True)\n\nlabel = train_df[target]\nval_label = val_df[target]\n\ntrain_df.drop(columns=[target],inplace=True)\nval_df.drop(columns=[target],inplace=True)\n\ntest_df = test[train_df.columns]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T14:51:42.992331Z","iopub.execute_input":"2025-07-31T14:51:42.992558Z","iopub.status.idle":"2025-07-31T14:51:48.216926Z","shell.execute_reply.started":"2025-07-31T14:51:42.992540Z","shell.execute_reply":"2025-07-31T14:51:48.215963Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Column selection","metadata":{"papermill":{"duration":0.0045,"end_time":"2025-05-24T09:28:43.670566","exception":false,"start_time":"2025-05-24T09:28:43.666066","status":"completed"},"tags":[]}},{"cell_type":"code","source":"features = hyperparameters['features']\nselector = SelectKBest(f_regression, k=features)\nX_selected = selector.fit_transform(train_df, label)\nX_test_selected = selector.transform(val_df)\nselected_indices = selector.get_support()\nselected_columns = train_df.columns[selected_indices]\n\nfiltered_columns = [\"ask_qty\", \"bid_qty\", \"sell_qty\", \"volume\"]\n\nfor i, feature in enumerate(selected_columns):\n    X_train = (train_df[feature].to_numpy()).reshape(-1,1)\n    X_val = (val_df[feature].to_numpy()).reshape(-1,1)\n    lr = LinearRegression()\n    lr.fit(X_train, label)\n    val_preds = lr.predict(X_val)\n    score = pearsonr(val_label, val_preds)[0]\n    if score <= 0:\n        continue\n    filtered_columns.append(feature)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T14:51:48.217824Z","iopub.execute_input":"2025-07-31T14:51:48.218090Z","iopub.status.idle":"2025-07-31T14:52:06.815238Z","shell.execute_reply.started":"2025-07-31T14:51:48.218071Z","shell.execute_reply":"2025-07-31T14:52:06.814492Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"ard_regressor = ARDRegression()\nard_regressor.fit(train_df[filtered_columns],label)\nval_preds_ard = ard_regressor.predict(val_df[filtered_columns])\ntest_preds_ard = ard_regressor.predict(test_df[filtered_columns])\n\nxgb_regressor = XGBRegressor(**xgb_params)\nxgb_regressor.fit(train_df[filtered_columns],label)\nval_preds_xgb = xgb_regressor.predict(val_df[filtered_columns])\ntest_preds_xgb = xgb_regressor.predict(test_df[filtered_columns])\n\nprint(f\"Blending\")\nbest_score = 0\nbest_ratio = 0\nfor ratio  in np.linspace(0, 1, 11):\n    val_preds = val_preds_xgb * ratio + val_preds_ard * (1 - ratio) \n    score = pearsonr(val_label, val_preds)[0]\n    print(f\"ratio: {ratio:.1f} * xgb + {1 - ratio:.1f} * ard, score: {score:.4f}\")\n    if score > best_score:\n        best_score = score\n        best_ratio = ratio\n\ntest_preds = test_preds_xgb * best_ratio + test_preds_ard * (1 - best_ratio) \n    \nsubmission = pd.read_csv(sample_submission_path)\nsubmission[\"prediction\"] = test_preds\nsubmission.to_csv(f\"submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T14:54:16.975393Z","iopub.execute_input":"2025-07-31T14:54:16.975728Z","iopub.status.idle":"2025-07-31T14:54:57.174568Z","shell.execute_reply.started":"2025-07-31T14:54:16.975684Z","shell.execute_reply":"2025-07-31T14:54:57.173737Z"}},"outputs":[],"execution_count":null}]}