{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Distributional timeseries for **responders.** \n(I'll look at the features later...)\n\n### Takeaways. 🍔\n1. Variability in some of the responders has a sawtooth pattern with ~ 20 day periodicity. These may be monthly aggregate features, since this is approximately the number of trading days in a given month. Possibly we can calendar-date each `date_id` from this info alone.\n\n2. Variability seems to be reflected in all of the responders for major market events.\n\n3. Some responders deviate significantly from normality on particular dates.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport plotly.express as px\nfrom pathlib import Path\nfrom tqdm.notebook import tqdm\n\ninput_root = Path('/kaggle/input')\ndata_dir = input_root / 'jane-street-real-time-market-data-forecasting'\n\nfeature_data = pl.read_csv(data_dir / 'features.csv')\nresponder_data = pl.read_csv(data_dir / 'responders.csv')\n\nid_cols = ['date_id', 'time_id']\nfeatures = [f'feature_{i:0>2}' for i in range(79)]\nresponders = [f'responder_{i}' for i in range(9)]\n\n\ndef get_daily_stats(df: pl.DataFrame):\n    daily_stats = (\n        df\n        .unpivot(index='date_id', on=responders, variable_name='feature')\n        .group_by(['date_id', 'feature'])\n        .agg(\n            pl.col('value').null_count().alias('null_count'),\n            pl.col('value').count().alias('count'),\n            pl.col('value').mean().alias('mean'),\n            pl.col('value').std().alias('std'),\n            pl.col('value').min().alias('min'),\n            pl.col('value').quantile(0.25).alias('25%'),\n            pl.col('value').quantile(0.50).alias('50%'),\n            pl.col('value').quantile(0.75).alias('75%'),\n            pl.col('value').max().alias('max')\n        )    \n    )\n    return daily_stats\n\n\nall_daily_stats = []\nfor part in tqdm([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]):\n    df = pl.read_parquet(\n        data_dir / f'train.parquet/partition_id={part}',\n        columns=id_cols + responders\n    )\n    daily_stats = get_daily_stats(df)\n    all_daily_stats.append(daily_stats)\n    del df\n    \ndaily_stats = pl.concat(all_daily_stats).sort(by=['date_id', 'feature'])\ndel all_daily_stats\n\ndaily_stats.write_parquet('daily_stats.parquet')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-20T21:31:17.069194Z","iopub.execute_input":"2024-10-20T21:31:17.069725Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig = px.scatter(\n    daily_stats.filter(pl.col('feature').is_in(responders)),\n    x='date_id',\n    y=['25%', '50%', 'mean', '75%'],\n    facet_row='feature',\n    height=2000,\n    template='plotly_dark'\n)\n\nfig.update_yaxes(matches=None)\nfig.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}