{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"},{"sourceId":249229901,"sourceType":"kernelVersion"}],"dockerImageVersionId":31089,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In this notebook, for each row in the new test DataFrame, I try to find the most similar row in the old, sorted DataFrame (which is sorted by time).","metadata":{}},{"cell_type":"code","source":"import tqdm\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics.pairwise import pairwise_distances","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-10T22:49:38.229692Z","iopub.execute_input":"2025-07-10T22:49:38.230067Z","iopub.status.idle":"2025-07-10T22:49:42.141072Z","shell.execute_reply.started":"2025-07-10T22:49:38.230033Z","shell.execute_reply":"2025-07-10T22:49:42.140116Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Read the test data (only the first 250 features are used):","metadata":{}},{"cell_type":"code","source":"new_shuffled_test = pd.read_parquet(\n    '/kaggle/input/drw-crypto-market-prediction/test.parquet'\n).iloc[:, :250]  # new test data (shuffled)\nold_sorted_test = pd.read_parquet(\n    '/kaggle/input/the-order-of-the-test-rows/sorted_test.parquet'\n).iloc[:, :250]  # old test data (sorted by time)\nassert new_shuffled_test.shape == old_sorted_test.shape\nassert new_shuffled_test.index.is_monotonic_increasing\nassert new_shuffled_test.shape[0] == 538150\nassert 10 <= new_shuffled_test.shape[1] < 1000","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-10T22:49:42.142454Z","iopub.execute_input":"2025-07-10T22:49:42.142840Z","iopub.status.idle":"2025-07-10T22:50:33.059023Z","shell.execute_reply.started":"2025-07-10T22:49:42.142819Z","shell.execute_reply":"2025-07-10T22:50:33.058213Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"I think a linear transformation was applied to each feature.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(16, 4))\nplt.plot(old_sorted_test['X1'].sort_values().to_numpy())\nplt.plot(new_shuffled_test['X1'].sort_values().to_numpy())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-10T22:50:33.059824Z","iopub.execute_input":"2025-07-10T22:50:33.060144Z","iopub.status.idle":"2025-07-10T22:50:33.701541Z","shell.execute_reply.started":"2025-07-10T22:50:33.060111Z","shell.execute_reply":"2025-07-10T22:50:33.700512Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# for col in tqdm.tqdm(new_shuffled_test.columns):\n#     a, b = np.polyfit(\n#         x=old_sorted_test[col].sort_values().to_numpy(),\n#         y=new_shuffled_test[col].sort_values().to_numpy(),\n#         deg=1\n#     )\n#     old_sorted_test[col] = a * old_sorted_test[col].to_numpy() + b","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-10T22:50:33.702658Z","iopub.execute_input":"2025-07-10T22:50:33.702940Z","iopub.status.idle":"2025-07-10T22:50:33.707455Z","shell.execute_reply.started":"2025-07-10T22:50:33.702918Z","shell.execute_reply":"2025-07-10T22:50:33.706367Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Scale both arrays (this makes the linear transformation above optional):","metadata":{}},{"cell_type":"code","source":"new_shuffled_test = StandardScaler().fit_transform(new_shuffled_test)\nold_sorted_test = StandardScaler().fit_transform(old_sorted_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-10T22:50:33.709963Z","iopub.execute_input":"2025-07-10T22:50:33.710240Z","iopub.status.idle":"2025-07-10T22:50:38.706977Z","shell.execute_reply.started":"2025-07-10T22:50:33.710199Z","shell.execute_reply":"2025-07-10T22:50:38.705976Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Now we can try to find the most similar row from the old test data for each row in the new data.","metadata":{}},{"cell_type":"code","source":"def find_closest_row(distances, debug=False):\n    assert len(distances.shape) == 1\n    assert distances.shape[0] == 538150\n    assert np.all(distances >= 0.0)\n    assert debug in [False, True]\n\n    sorted_distances = np.sort(distances)\n    diff = np.diff(sorted_distances)\n    assert np.all(diff >= 0.0)\n    threshold = 1000.0 * np.median(diff[1000:10000])\n\n    if debug:\n        fig, axes = plt.subplots(2, 1, figsize=(16, 6))\n        axes[0].plot(sorted_distances[:50], marker='o', color='tab:blue')\n        axes[0].set_ylabel('distance')\n        axes[1].plot(diff[:49], marker='o', color='tab:orange')\n        axes[1].axhline(threshold, linestyle='--', color='r')\n        axes[1].set_ylabel('distance diff')\n\n    if diff[0] > threshold:\n        return np.argmin(distances)\n\n    return -1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-10T22:50:38.707882Z","iopub.execute_input":"2025-07-10T22:50:38.708215Z","iopub.status.idle":"2025-07-10T22:50:38.715999Z","shell.execute_reply.started":"2025-07-10T22:50:38.708185Z","shell.execute_reply":"2025-07-10T22:50:38.715006Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# For example, let's take a look at a random row (with index 1755):\nfind_closest_row(\n    pairwise_distances(new_shuffled_test[1755:1756], old_sorted_test)[0],\n    debug=True\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-10T22:50:38.716920Z","iopub.execute_input":"2025-07-10T22:50:38.717174Z","iopub.status.idle":"2025-07-10T22:50:39.425776Z","shell.execute_reply.started":"2025-07-10T22:50:38.717154Z","shell.execute_reply":"2025-07-10T22:50:39.424945Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# But not every row has a corresponding match in the old sorted dataset\nfind_closest_row(\n    pairwise_distances(new_shuffled_test[1800:1801], old_sorted_test)[0],\n    debug=True\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-10T22:50:39.426788Z","iopub.execute_input":"2025-07-10T22:50:39.427129Z","iopub.status.idle":"2025-07-10T22:50:40.198917Z","shell.execute_reply.started":"2025-07-10T22:50:39.427095Z","shell.execute_reply":"2025-07-10T22:50:40.198045Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"batch_size = 2500  # to avoid running out of memory\nclosest_rows = []  # the index of the closest row in the old test dataset\n# all_distances = []  # the distance matrix\n\nfor i in tqdm.tqdm(range(0, new_shuffled_test.shape[0], batch_size)):\n    window_begin = i\n    window_end = min(i + batch_size, new_shuffled_test.shape[0])\n    batch_distances = pairwise_distances(\n        new_shuffled_test[window_begin:window_end],\n        old_sorted_test\n    )\n    assert len(batch_distances.shape) == 2\n    assert batch_distances.shape[0] == window_end - window_begin\n    assert batch_distances.shape[1] == old_sorted_test.shape[0]\n    for row_distances in batch_distances:\n        closest_rows.append(find_closest_row(row_distances))\n        # all_distances.append(row_distances)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-10T22:50:40.200165Z","iopub.execute_input":"2025-07-10T22:50:40.200503Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_shuffled_test = None\nold_sorted_test = None","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Save the results:","metadata":{}},{"cell_type":"code","source":"closest_rows = pd.Series(closest_rows)\n# all_distances = pd.DataFrame(all_distances)\n# assert all_distances.shape == (len(closest_rows), len(closest_rows))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"closest_rows.to_csv('closest_rows.csv')\n# all_distances.to_csv('all_distances.csv')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"closest_rows","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}