{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":249869065,"sourceType":"kernelVersion"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In this notebook, the precomputed results from the notebook \"The order of the test rows 2\" are used to reconstruct an approximate order in the new test DataFrame.","metadata":{}},{"cell_type":"code","source":"import tqdm\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport xgboost as xgb\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import Ridge","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-15T12:23:59.269970Z","iopub.execute_input":"2025-07-15T12:23:59.270312Z","iopub.status.idle":"2025-07-15T12:24:03.773193Z","shell.execute_reply.started":"2025-07-15T12:23:59.270283Z","shell.execute_reply":"2025-07-15T12:24:03.772229Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess_data(raw_df):\n    assert len(raw_df.shape) == 2\n\n    y = raw_df['label'].to_numpy()\n    assert y.shape == (raw_df.shape[0],)\n\n    cols = [\n        'X239', 'X743', 'X215', 'X623', 'X240', 'X71', 'X119', 'X157',\n        'X173', 'X198', 'X113', 'X324', 'X366', 'X462',\n    ]\n\n    df = raw_df[cols]\n    assert df.isna().sum().sum() == 0\n\n    df = pd.concat([\n        df.shift(-lag).add_suffix(f'_lead_{lag}')\n        for lag in [30, 40, 50, 60, 70, 80]\n    ], axis=1)\n\n    df = df.fillna(0.0)\n    assert 'label' not in df.columns\n    assert raw_df.shape[0] == df.shape[0] and (raw_df.index == df.index).all()\n    assert df.isna().sum().sum() == 0\n    assert df.shape[0] == y.shape[0]\n    return df, y","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-15T12:24:03.774076Z","iopub.execute_input":"2025-07-15T12:24:03.774479Z","iopub.status.idle":"2025-07-15T12:24:03.782827Z","shell.execute_reply.started":"2025-07-15T12:24:03.774455Z","shell.execute_reply":"2025-07-15T12:24:03.781937Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/train.parquet')\nX_train, y_train = preprocess_data(train_df)\ntrain_df = None\nprint(X_train.shape, y_train.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-15T12:24:03.785065Z","iopub.execute_input":"2025-07-15T12:24:03.785493Z","iopub.status.idle":"2025-07-15T12:24:29.009521Z","shell.execute_reply.started":"2025-07-15T12:24:03.785463Z","shell.execute_reply":"2025-07-15T12:24:29.008764Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scaler = StandardScaler()\nX_train.iloc[:, :] = scaler.fit_transform(X_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-15T12:24:29.010675Z","iopub.execute_input":"2025-07-15T12:24:29.010946Z","iopub.status.idle":"2025-07-15T12:24:30.166855Z","shell.execute_reply.started":"2025-07-15T12:24:29.010924Z","shell.execute_reply":"2025-07-15T12:24:30.166094Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\n# model = Ridge()\nmodel = xgb.XGBRegressor()\n\nmodel.fit(X_train, y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-15T12:24:30.167922Z","iopub.execute_input":"2025-07-15T12:24:30.168273Z","iopub.status.idle":"2025-07-15T12:24:41.262793Z","shell.execute_reply.started":"2025-07-15T12:24:30.168242Z","shell.execute_reply":"2025-07-15T12:24:41.261922Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# pd.Series(model.coef_, index=model.feature_names_in_).sort_values().plot(\n#     kind='barh', figsize=(16, 2 + len(model.coef_) // 4)\n# )\n\npd.Series(model.feature_importances_, index=X_train.columns).sort_values().plot(\n    kind='barh', figsize=(16, 2 + X_train.shape[1] // 4)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-15T12:25:33.019832Z","iopub.execute_input":"2025-07-15T12:25:33.020654Z","iopub.status.idle":"2025-07-15T12:25:34.131688Z","shell.execute_reply.started":"2025-07-15T12:25:33.020627Z","shell.execute_reply":"2025-07-15T12:25:34.130759Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train = None\ny_train = None","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-15T12:28:02.838445Z","iopub.execute_input":"2025-07-15T12:28:02.838777Z","iopub.status.idle":"2025-07-15T12:28:03.087580Z","shell.execute_reply.started":"2025-07-15T12:28:02.838753Z","shell.execute_reply":"2025-07-15T12:28:03.086669Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/test.parquet')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-15T12:28:03.327299Z","iopub.execute_input":"2025-07-15T12:28:03.327873Z","iopub.status.idle":"2025-07-15T12:28:26.912001Z","shell.execute_reply.started":"2025-07-15T12:28:03.327847Z","shell.execute_reply":"2025-07-15T12:28:26.911169Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Let's explore a bit the precomputed array:","metadata":{}},{"cell_type":"code","source":"t = pd.Series(pd.read_csv(\n    '/kaggle/input/the-order-of-the-test-rows-2/closest_rows.csv'\n)['0'].to_numpy())\nassert t.shape == (test_df.shape[0],)\nprint('Reconstructed timestamps share:', len(t[t >= 0]) / len(t))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-15T12:28:26.913502Z","iopub.execute_input":"2025-07-15T12:28:26.913911Z","iopub.status.idle":"2025-07-15T12:28:27.079324Z","shell.execute_reply.started":"2025-07-15T12:28:26.913880Z","shell.execute_reply":"2025-07-15T12:28:27.078539Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(16, 4))\nplt.plot(t.sort_values().to_numpy())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-15T12:28:27.080217Z","iopub.execute_input":"2025-07-15T12:28:27.080447Z","iopub.status.idle":"2025-07-15T12:28:27.401107Z","shell.execute_reply.started":"2025-07-15T12:28:27.080429Z","shell.execute_reply":"2025-07-15T12:28:27.400265Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(16, 4))\nplt.plot(t[t >= 0].sort_values().iloc[:1000].to_numpy())\nplt.axhline(10080, color='r', linestyle='--')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-15T12:28:27.402951Z","iopub.execute_input":"2025-07-15T12:28:27.403212Z","iopub.status.idle":"2025-07-15T12:28:27.666158Z","shell.execute_reply.started":"2025-07-15T12:28:27.403194Z","shell.execute_reply":"2025-07-15T12:28:27.665149Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"I think the rows that do not have a close match in the old dataset are likely the most recent ones.","metadata":{}},{"cell_type":"code","source":"t -= 10080\nt[t < 0] = 538149  # the most recent rows now have the biggest value\n\nt = t.sort_values()\nt[t <= len(t)] = np.arange(t[t <= len(t)].shape[0])\nt = t.sort_index()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-15T12:28:27.667162Z","iopub.execute_input":"2025-07-15T12:28:27.667473Z","iopub.status.idle":"2025-07-15T12:28:27.852585Z","shell.execute_reply.started":"2025-07-15T12:28:27.667450Z","shell.execute_reply":"2025-07-15T12:28:27.851679Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"t = pd.Series(np.arange(538150), index=t.to_numpy()).sort_index()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-15T12:28:27.853551Z","iopub.execute_input":"2025-07-15T12:28:27.853877Z","iopub.status.idle":"2025-07-15T12:28:27.938003Z","shell.execute_reply.started":"2025-07-15T12:28:27.853848Z","shell.execute_reply":"2025-07-15T12:28:27.937030Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Finally, sort the test dataset by time:","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(16, 4))\nplt.plot(test_df['X656'].to_numpy())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-15T12:28:27.938832Z","iopub.execute_input":"2025-07-15T12:28:27.939108Z","iopub.status.idle":"2025-07-15T12:28:29.806802Z","shell.execute_reply.started":"2025-07-15T12:28:27.939088Z","shell.execute_reply":"2025-07-15T12:28:29.805935Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = test_df.iloc[t.to_numpy()]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-15T12:28:29.807882Z","iopub.execute_input":"2025-07-15T12:28:29.808192Z","iopub.status.idle":"2025-07-15T12:28:34.321913Z","shell.execute_reply.started":"2025-07-15T12:28:29.808163Z","shell.execute_reply":"2025-07-15T12:28:34.320953Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(16, 4))\nplt.plot(test_df['X656'].to_numpy())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-15T12:28:34.322887Z","iopub.execute_input":"2025-07-15T12:28:34.323191Z","iopub.status.idle":"2025-07-15T12:28:34.828401Z","shell.execute_reply.started":"2025-07-15T12:28:34.323163Z","shell.execute_reply":"2025-07-15T12:28:34.827381Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Preprocess the test dataset and perform model inference:","metadata":{}},{"cell_type":"code","source":"X_test, _ = preprocess_data(test_df)\ntest_df = None\nprint(X_test.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-15T12:28:34.830687Z","iopub.execute_input":"2025-07-15T12:28:34.831013Z","iopub.status.idle":"2025-07-15T12:28:35.768813Z","shell.execute_reply.started":"2025-07-15T12:28:34.830963Z","shell.execute_reply":"2025-07-15T12:28:35.767941Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test.iloc[:, :] = scaler.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-15T12:28:35.769682Z","iopub.execute_input":"2025-07-15T12:28:35.769917Z","iopub.status.idle":"2025-07-15T12:28:36.270872Z","shell.execute_reply.started":"2025-07-15T12:28:35.769899Z","shell.execute_reply":"2025-07-15T12:28:36.270010Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = model.predict(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-15T12:28:36.271803Z","iopub.execute_input":"2025-07-15T12:28:36.272162Z","iopub.status.idle":"2025-07-15T12:28:37.187246Z","shell.execute_reply.started":"2025-07-15T12:28:36.272132Z","shell.execute_reply":"2025-07-15T12:28:37.186491Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.Series(y_pred).describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-15T12:28:37.187856Z","iopub.execute_input":"2025-07-15T12:28:37.188119Z","iopub.status.idle":"2025-07-15T12:28:37.217237Z","shell.execute_reply.started":"2025-07-15T12:28:37.188096Z","shell.execute_reply":"2025-07-15T12:28:37.216438Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(16, 4))\nplt.plot(np.cumsum(y_pred))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-15T12:28:37.218298Z","iopub.execute_input":"2025-07-15T12:28:37.218539Z","iopub.status.idle":"2025-07-15T12:28:37.485793Z","shell.execute_reply.started":"2025-07-15T12:28:37.218519Z","shell.execute_reply":"2025-07-15T12:28:37.484935Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Save the predictions to the submission file:","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/drw-crypto-market-prediction/sample_submission.csv')\n\nsubmission = submission.iloc[t.to_numpy()]\nsubmission['prediction'] = y_pred\nsubmission = submission.sort_index()\n\nsubmission.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-15T12:28:37.486740Z","iopub.execute_input":"2025-07-15T12:28:37.487022Z","iopub.status.idle":"2025-07-15T12:28:38.784008Z","shell.execute_reply.started":"2025-07-15T12:28:37.486996Z","shell.execute_reply":"2025-07-15T12:28:38.783291Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-15T12:28:38.784862Z","iopub.execute_input":"2025-07-15T12:28:38.785122Z","iopub.status.idle":"2025-07-15T12:28:38.805400Z","shell.execute_reply.started":"2025-07-15T12:28:38.785102Z","shell.execute_reply":"2025-07-15T12:28:38.804512Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}