{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"I created this notebook primarily to explore the visualizations (at the end of the notebook). My goal was to examine the data to gain a better understanding. Since the dataset contains many features, there are also numerous plots. While some might consider this excessive, I found it helpful for getting a comprehensive overview.","metadata":{}},{"cell_type":"markdown","source":"## Imports","metadata":{}},{"cell_type":"code","source":"import glob\nimport re\nimport gc\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\n\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T20:36:25.227137Z","iopub.execute_input":"2024-12-27T20:36:25.227455Z","iopub.status.idle":"2024-12-27T20:36:25.920791Z","shell.execute_reply.started":"2024-12-27T20:36:25.227430Z","shell.execute_reply":"2024-12-27T20:36:25.919758Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load Data","metadata":{}},{"cell_type":"code","source":"def extract_partition_id(file_path):\n    match = re.search(r'partition_id=(\\d+)', file_path)\n    if match: return int(match.group(1))\n    return None\n\n\nfile_paths = glob.glob('/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=*/part-*.parquet')\nfile_paths.sort()\nlazy_dfs = []\n\n\nfor idx, file_path in enumerate(tqdm(file_paths)):\n    lazy_df = pl.read_parquet(file_path).lazy().with_columns(\n        pl.lit(idx).alias(\"partition_id\")\n    )\n    \n    lazy_dfs.append(lazy_df)\n\ndel lazy_df\ngc.collect()\ntrain = pl.concat(lazy_dfs)\n\ndel lazy_dfs\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T20:36:27.306386Z","iopub.execute_input":"2024-12-27T20:36:27.306885Z","iopub.status.idle":"2024-12-27T20:37:30.875796Z","shell.execute_reply.started":"2024-12-27T20:36:27.306855Z","shell.execute_reply":"2024-12-27T20:37:30.874429Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head(5).collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T20:37:30.877434Z","iopub.execute_input":"2024-12-27T20:37:30.877881Z","iopub.status.idle":"2024-12-27T20:37:30.945799Z","shell.execute_reply.started":"2024-12-27T20:37:30.877836Z","shell.execute_reply":"2024-12-27T20:37:30.944328Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## EDA","metadata":{}},{"cell_type":"code","source":"cols = list(train.collect_schema().names())\nprint(cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T20:37:30.947967Z","iopub.execute_input":"2024-12-27T20:37:30.948352Z","iopub.status.idle":"2024-12-27T20:37:30.958304Z","shell.execute_reply.started":"2024-12-27T20:37:30.948309Z","shell.execute_reply":"2024-12-27T20:37:30.956774Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Getting the number of unique time measurements per day","metadata":{}},{"cell_type":"code","source":"res = train.group_by(\"date_id\").agg(\n    pl.col(\"time_id\").n_unique().alias(\"unique_times\")\n)\n\nres = res.collect().sort(by=['date_id'])\nres['unique_times'].unique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T20:37:30.960051Z","iopub.execute_input":"2024-12-27T20:37:30.960440Z","iopub.status.idle":"2024-12-27T20:37:32.253502Z","shell.execute_reply.started":"2024-12-27T20:37:30.960407Z","shell.execute_reply":"2024-12-27T20:37:32.252024Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"switches = res.with_columns(\n    (pl.col(\"unique_times\") != pl.col(\"unique_times\").shift(1)).alias(\"is_switch\")\n).filter(pl.col(\"is_switch\")).select([\"date_id\", \"unique_times\"])\n\nswitches","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T20:37:32.254979Z","iopub.execute_input":"2024-12-27T20:37:32.255579Z","iopub.status.idle":"2024-12-27T20:37:32.278072Z","shell.execute_reply.started":"2024-12-27T20:37:32.255537Z","shell.execute_reply":"2024-12-27T20:37:32.276769Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"It looks like at some place the sampling rate changes. I decided to keep all the data after that switch, so every day has the same sampling rate. For the plotting I actually decided to only use the last few partitions.","metadata":{}},{"cell_type":"code","source":"print(f'Shape before:\\t{train.collect().shape}')\ntrain = train.filter(pl.col(\"partition_id\") >= 8)\nprint(f'Shape after:\\t{train.collect().shape}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T20:38:25.622977Z","iopub.execute_input":"2024-12-27T20:38:25.623347Z","iopub.status.idle":"2024-12-27T20:38:25.743498Z","shell.execute_reply.started":"2024-12-27T20:38:25.623321Z","shell.execute_reply":"2024-12-27T20:38:25.742097Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Start date_id with 0","metadata":{}},{"cell_type":"code","source":"train = train.with_columns(\n    (pl.col(\"date_id\") - train.select('date_id').collect()[0].item()).alias(\"date_id\")\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T20:38:48.458812Z","iopub.execute_input":"2024-12-27T20:38:48.459300Z","iopub.status.idle":"2024-12-27T20:38:48.487211Z","shell.execute_reply.started":"2024-12-27T20:38:48.459259Z","shell.execute_reply":"2024-12-27T20:38:48.486060Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Create a new column so I can identify each unique timestamp with only one column","metadata":{}},{"cell_type":"code","source":"train = train.with_columns(\n    pl.col(\"date_id\").cast(pl.UInt32).alias(\"date_id\")\n)\n\n# Now, perform the calculation on the 'date' column\ntrain = train.with_columns(\n    (pl.col(\"date_id\") * 968 + pl.col(\"time_id\")).cast(pl.UInt32).alias(\"date\")\n)\n\ntrain = train.with_columns(\n    pl.col(\"date_id\").cast(pl.Int16).alias(\"date_id\")\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T20:39:02.023080Z","iopub.execute_input":"2024-12-27T20:39:02.023401Z","iopub.status.idle":"2024-12-27T20:39:02.029858Z","shell.execute_reply.started":"2024-12-27T20:39:02.023377Z","shell.execute_reply":"2024-12-27T20:39:02.028412Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head(5).collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T20:39:03.306244Z","iopub.execute_input":"2024-12-27T20:39:03.306700Z","iopub.status.idle":"2024-12-27T20:39:03.389660Z","shell.execute_reply.started":"2024-12-27T20:39:03.306665Z","shell.execute_reply":"2024-12-27T20:39:03.388375Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Check for which feature the values are the same for one day (time_id has no influence)","metadata":{}},{"cell_type":"code","source":"agg_df = train.group_by([\"date_id\", \"symbol_id\"]).agg([\n    pl.all().std().name.suffix('_std'),\n])\n\nresult = (\n    agg_df.select([\n        (pl.col(col).sum() == 0).alias(col)\n        for col in agg_df.collect_schema().keys()\n    ])\n    .collect()\n)\n\n# Extract the column names with sum 0\ncolumns_with_sum_zero = [col for col in result.columns if result[col][0]]\n\ndel agg_df\ndel result\ngc.collect()\n\nprint(columns_with_sum_zero)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T20:39:14.061237Z","iopub.execute_input":"2024-12-27T20:39:14.061710Z","iopub.status.idle":"2024-12-27T20:39:34.648538Z","shell.execute_reply.started":"2024-12-27T20:39:14.061668Z","shell.execute_reply":"2024-12-27T20:39:34.647298Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group by 'date_id' and calculate the std deviation for each column\nagg_df = train.group_by(\"date_id\").agg([\n    pl.all().std().name.suffix('_std'),\n])\n\nresult = (\n    agg_df.select([\n        (pl.col(col).sum() == 0).alias(col)\n        for col in agg_df.collect_schema().keys()\n    ])\n    .collect()\n)\n\n# Extract the column names with sum 0\ncolumns_with_sum_zero = [col for col in result.columns if result[col][0]]\n\ndel agg_df\ndel result\ngc.collect()\n\nprint(columns_with_sum_zero)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T20:40:14.008515Z","iopub.execute_input":"2024-12-27T20:40:14.009011Z","iopub.status.idle":"2024-12-27T20:40:22.714281Z","shell.execute_reply.started":"2024-12-27T20:40:14.008977Z","shell.execute_reply":"2024-12-27T20:40:22.713103Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"feature_61 same for even all symbols in one day","metadata":{}},{"cell_type":"markdown","source":"## Plotting\n\nFor the plotting I decided to use the cumsum. That is also why at the beginning the lines all start nearly at the same position.","metadata":{}},{"cell_type":"code","source":"# Function to filter and return data for a specified feature and symbol_id\ndef get_feature_data(df: pl.DataFrame, feature_name: str, symbol_id: int, cumsum=False):\n    # Filter rows based on the symbol_id\n    filtered_df = df.filter(pl.col(\"symbol_id\") == symbol_id)\n    \n    # Select the feature column, time_id, and date_id\n    selected_columns = filtered_df.select([pl.col(\"date\"), pl.col(feature_name)])\n\n    if cumsum:\n        selected_columns = selected_columns.with_columns(\n            pl.col(feature_name).cum_sum().alias(feature_name)\n        )\n    \n    # Convert to pandas for easier plotting\n    return selected_columns.collect().to_pandas()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T20:41:49.698165Z","iopub.execute_input":"2024-12-27T20:41:49.698585Z","iopub.status.idle":"2024-12-27T20:41:49.705125Z","shell.execute_reply.started":"2024-12-27T20:41:49.698557Z","shell.execute_reply":"2024-12-27T20:41:49.703614Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Looking at the reponders for each symbol, one can see that responders 3, 4 & 5 have jumps at the same time as well as 6, (7,) 8.\n\nYou can also see that the trends are quite similar. The reponders have similar trends between symbols. 3, 4 and 5 often have negative trends and the others are pretty much in the middle with 1 often being more positive.","metadata":{}},{"cell_type":"code","source":"cmap = plt.get_cmap('rainbow')\ncolors = cmap(np.linspace(0, 1, 9))\nreorder = lambda l, nc: sum((l[i::nc] for i in range(nc)), [])\nncols = 5\n\nfor symbol_id in range(39):\n    fig, ax = plt.subplots(figsize=(10, 5))\n    \n    for responder_id in range(9):\n        feature_name = f'responder_{responder_id}'\n        data = get_feature_data(train, feature_name, symbol_id, True)\n        ax.plot(data['date'], data[feature_name], color=colors[responder_id], linestyle='-', label=f'{feature_name}')\n\n    # Set plot titles and labels\n    ax.set_title(f'Line Chart for responders for symbol_id={symbol_id}')\n    ax.set_xlabel('Time')\n    ax.set_ylabel('Responder Value')\n    ax.grid(True)\n\n    h, l = ax.get_legend_handles_labels()\n    \n    plt.legend(reorder(h, ncols), reorder(l, ncols), loc='center', bbox_to_anchor=(0.5, -0.18), ncol=ncols, frameon=False)\n    plt.show()\n\ndel data\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T20:41:51.941250Z","iopub.execute_input":"2024-12-27T20:41:51.941693Z","iopub.status.idle":"2024-12-27T20:42:37.389525Z","shell.execute_reply.started":"2024-12-27T20:41:51.941661Z","shell.execute_reply":"2024-12-27T20:42:37.388271Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"I wanted to see if the jumps are the same for all symbols for one responder. Thats why I flipped the plotting:","metadata":{}},{"cell_type":"code","source":"cmap = plt.get_cmap('rainbow')\ncolors = cmap(np.linspace(0, 1, 39))\nreorder = lambda l, nc: sum((l[i::nc] for i in range(nc)), [])\nncols = 10\n\nfor responder_id in range(9):\n    fig, ax = plt.subplots(figsize=(10, 5))\n    \n    for symbol_id in range(39):\n        feature_name = f'responder_{responder_id}'\n        data = get_feature_data(train, feature_name, symbol_id, True)\n        ax.plot(data['date'], data[feature_name], color=colors[symbol_id], linestyle='-', label=f'{symbol_id}')\n\n    # Set plot titles and labels\n    ax.set_title(f'Line Chart for responders for responder={responder_id}')\n    ax.set_xlabel('Time')\n    ax.set_ylabel('Responder Value')\n    ax.grid(True)\n\n    h, l = ax.get_legend_handles_labels()\n    \n    plt.legend(reorder(h, ncols), reorder(l, ncols), loc='center', bbox_to_anchor=(0.5, -0.18), ncol=ncols, frameon=False)\n    plt.show()\n    \ndel data\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T20:43:05.559464Z","iopub.execute_input":"2024-12-27T20:43:05.559935Z","iopub.status.idle":"2024-12-27T20:43:41.659071Z","shell.execute_reply.started":"2024-12-27T20:43:05.559902Z","shell.execute_reply":"2024-12-27T20:43:41.657645Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Lastly I wanted to see the features vs symbols. As it looks sometimes the symbols differ just by the y-axis. Same behaviour. Sometimes completly different.\n\nWith a few symbols it looks like there are a few outliers. Maybe treatment needed?","metadata":{}},{"cell_type":"code","source":"cmap = plt.get_cmap('rainbow')\ncolors = cmap(np.linspace(0, 1, 39))\nreorder = lambda l, nc: sum((l[i::nc] for i in range(nc)), [])\nncols = 10\n\nfor feature_id in range(79):\n    fig, ax = plt.subplots(figsize=(10, 5))\n    \n    for symbol_id in range(39):\n        feature_name = f'feature_{str(feature_id).zfill(2)}'\n        data = get_feature_data(train, feature_name, symbol_id, True)\n        ax.plot(data['date'], data[feature_name], color=colors[symbol_id], linestyle='-', label=f'{symbol_id}')\n\n    # Set plot titles and labels\n    ax.set_title(f'Line Chart for responders for feature={feature_id}')\n    ax.set_xlabel('Time')\n    ax.set_ylabel('Feature Value')\n    ax.grid(True)\n\n    h, l = ax.get_legend_handles_labels()\n    \n    plt.legend(reorder(h, ncols), reorder(l, ncols), loc='center', bbox_to_anchor=(0.5, -0.18), ncol=ncols, frameon=False)\n    plt.show()\n\ndel data\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T20:44:20.584027Z","iopub.execute_input":"2024-12-27T20:44:20.584377Z","iopub.status.idle":"2024-12-27T20:49:13.286449Z","shell.execute_reply.started":"2024-12-27T20:44:20.584350Z","shell.execute_reply":"2024-12-27T20:49:13.284587Z"}},"outputs":[],"execution_count":null}]}