{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 📋 Table of Contents\n* [Load Data](#load)\n* [Weights](#weights)\n* [Features](#features)\n* [Targets](#targets)\n* [Distribution fits on target](#dist_fit)\n* [Correlation Targets vs Features](#corr_target_features)\n* [Correlation of Targets](#corr_target)\n* [Deep dive for an example symbol](#symbol)","metadata":{}},{"cell_type":"code","source":"# install package for distribution fitting\n!pip install fitter","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-11-14T19:42:02.867417Z","iopub.execute_input":"2024-11-14T19:42:02.868042Z","iopub.status.idle":"2024-11-14T19:42:20.304069Z","shell.execute_reply.started":"2024-11-14T19:42:02.867990Z","shell.execute_reply":"2024-11-14T19:42:20.302739Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# packages\n\n# standard\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport time\n\n# garbage collection\nimport gc\n\n# plots\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# stats\nimport random\nimport scipy.stats\nfrom fitter import Fitter, get_common_distributions, get_distributions\n\n# other stuff\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-11-14T19:42:20.306477Z","iopub.execute_input":"2024-11-14T19:42:20.307313Z","iopub.status.idle":"2024-11-14T19:42:22.116151Z","shell.execute_reply.started":"2024-11-14T19:42:20.307266Z","shell.execute_reply":"2024-11-14T19:42:22.114684Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# show files\n!ls -l '../input/jane-street-real-time-market-data-forecasting'","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:42:22.117930Z","iopub.execute_input":"2024-11-14T19:42:22.118515Z","iopub.status.idle":"2024-11-14T19:42:23.281119Z","shell.execute_reply.started":"2024-11-14T19:42:22.118468Z","shell.execute_reply":"2024-11-14T19:42:23.279637Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# details - training data\n!ls -lR '../input/jane-street-real-time-market-data-forecasting/train.parquet'","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:42:23.285028Z","iopub.execute_input":"2024-11-14T19:42:23.285611Z","iopub.status.idle":"2024-11-14T19:42:24.465471Z","shell.execute_reply.started":"2024-11-14T19:42:23.285527Z","shell.execute_reply":"2024-11-14T19:42:24.463994Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# configs\npd.set_option('display.max_columns', None) # we want to display all columns in this notebook\npd.set_option('display.max_rows', 100) # increase number of displayed rows\n\n# random seed\nmy_random_seed = 123\nrandom.seed(128)\n\n# aesthetics\ndefault_color_1 = 'darkblue'\ndefault_color_2 = 'darkgreen'\ndefault_color_3 = 'darkred'","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:42:24.467131Z","iopub.execute_input":"2024-11-14T19:42:24.467543Z","iopub.status.idle":"2024-11-14T19:42:24.474415Z","shell.execute_reply.started":"2024-11-14T19:42:24.467498Z","shell.execute_reply":"2024-11-14T19:42:24.473358Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a id='load'></a>\n# Load Data","metadata":{}},{"cell_type":"code","source":"# load data\nt1 = time.time()\ndf_0 = pl.read_parquet('../input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=0/part-0.parquet')\ndf_1 = pl.read_parquet('../input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=1/part-0.parquet')\ndf_2 = pl.read_parquet('../input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=2/part-0.parquet')\ndf_3 = pl.read_parquet('../input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=3/part-0.parquet')\ndf_4 = pl.read_parquet('../input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=4/part-0.parquet')\ndf_5 = pl.read_parquet('../input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=5/part-0.parquet')\ndf_6 = pl.read_parquet('../input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=6/part-0.parquet')\ndf_7 = pl.read_parquet('../input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=7/part-0.parquet')\ndf_8 = pl.read_parquet('../input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=8/part-0.parquet')\ndf_9 = pl.read_parquet('../input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=9/part-0.parquet')\nt2 = time.time()\nprint('Elapsed time [s]:', np.round(t2-t1,4))","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:42:24.476224Z","iopub.execute_input":"2024-11-14T19:42:24.477032Z","iopub.status.idle":"2024-11-14T19:43:23.209878Z","shell.execute_reply.started":"2024-11-14T19:42:24.476974Z","shell.execute_reply":"2024-11-14T19:43:23.207965Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# combine in one data frame\ndf = pl.concat([df_0,df_1,df_2,df_3,df_4,df_5,df_6,df_7,df_8,df_9])","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:43:23.212418Z","iopub.execute_input":"2024-11-14T19:43:23.213729Z","iopub.status.idle":"2024-11-14T19:43:23.252614Z","shell.execute_reply.started":"2024-11-14T19:43:23.213651Z","shell.execute_reply":"2024-11-14T19:43:23.251414Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# clean up\ndel df_0,df_1,df_2,df_3,df_4,df_5,df_6,df_7,df_8,df_9\ngc.collect();","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:43:23.254447Z","iopub.execute_input":"2024-11-14T19:43:23.255295Z","iopub.status.idle":"2024-11-14T19:43:23.441069Z","shell.execute_reply.started":"2024-11-14T19:43:23.255240Z","shell.execute_reply":"2024-11-14T19:43:23.439377Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# preview\ndf.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:43:23.443113Z","iopub.execute_input":"2024-11-14T19:43:23.443654Z","iopub.status.idle":"2024-11-14T19:43:23.488902Z","shell.execute_reply.started":"2024-11-14T19:43:23.443596Z","shell.execute_reply":"2024-11-14T19:43:23.487366Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# data frame size\ndf.shape","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:43:23.495432Z","iopub.execute_input":"2024-11-14T19:43:23.496445Z","iopub.status.idle":"2024-11-14T19:43:23.508053Z","shell.execute_reply.started":"2024-11-14T19:43:23.496393Z","shell.execute_reply":"2024-11-14T19:43:23.506624Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a id='weights'></a>\n# Weights","metadata":{}},{"cell_type":"code","source":"# basic stats\ndf['weight'].describe()","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:43:23.509728Z","iopub.execute_input":"2024-11-14T19:43:23.510236Z","iopub.status.idle":"2024-11-14T19:43:27.056151Z","shell.execute_reply.started":"2024-11-14T19:43:23.510191Z","shell.execute_reply":"2024-11-14T19:43:27.054751Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# plot histogram\nplt.figure(figsize=(8,3))\nplt.hist(df['weight'], bins=100, color=default_color_1)\nplt.title('Distribution of weights')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:43:27.058011Z","iopub.execute_input":"2024-11-14T19:43:27.058423Z","iopub.status.idle":"2024-11-14T19:43:28.634744Z","shell.execute_reply.started":"2024-11-14T19:43:27.058383Z","shell.execute_reply":"2024-11-14T19:43:28.633570Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# average weight by symbol\nweight_by_symbol = df.group_by('symbol_id').agg(pl.col('weight').mean())\nweight_by_symbol = weight_by_symbol.to_pandas().sort_values(by='symbol_id').reset_index(drop=True)\nweight_by_symbol","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:43:28.636301Z","iopub.execute_input":"2024-11-14T19:43:28.636705Z","iopub.status.idle":"2024-11-14T19:43:29.511487Z","shell.execute_reply.started":"2024-11-14T19:43:28.636665Z","shell.execute_reply":"2024-11-14T19:43:29.510322Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a id='features'></a>\n# Features","metadata":{}},{"cell_type":"code","source":"features = ['feature_' + str(x).zfill(2) for x in range(78+1)]\nprint(features)","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:43:29.513054Z","iopub.execute_input":"2024-11-14T19:43:29.513434Z","iopub.status.idle":"2024-11-14T19:43:29.520295Z","shell.execute_reply.started":"2024-11-14T19:43:29.513394Z","shell.execute_reply":"2024-11-14T19:43:29.518940Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# plot feature distributions\nfor f in features:\n    plt.figure(figsize=(8,3))\n    plt.hist(df[f], bins=100, color=default_color_1)\n    plt.title(f)\n    plt.grid()\n    plt.show()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-11-14T19:43:29.522233Z","iopub.execute_input":"2024-11-14T19:43:29.522748Z","iopub.status.idle":"2024-11-14T19:47:29.743176Z","shell.execute_reply.started":"2024-11-14T19:43:29.522668Z","shell.execute_reply":"2024-11-14T19:47:29.741987Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a id='targets'></a>\n# Targets","metadata":{}},{"cell_type":"code","source":"# target names\ntargets = ['responder_' + str(x) for x in range(8+1)]\nprint(targets)","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:47:29.744944Z","iopub.execute_input":"2024-11-14T19:47:29.745353Z","iopub.status.idle":"2024-11-14T19:47:29.751706Z","shell.execute_reply.started":"2024-11-14T19:47:29.745313Z","shell.execute_reply":"2024-11-14T19:47:29.750635Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# basic stats\ndf[targets].describe()","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:47:29.753202Z","iopub.execute_input":"2024-11-14T19:47:29.753593Z","iopub.status.idle":"2024-11-14T19:47:57.197878Z","shell.execute_reply.started":"2024-11-14T19:47:29.753530Z","shell.execute_reply":"2024-11-14T19:47:57.196572Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 💡 Looks like an artificial cap at -/+5","metadata":{}},{"cell_type":"code","source":"# plot target distributions\nfor t in targets:\n    plt.figure(figsize=(8,3))\n    plt.hist(df[t], bins=100, color=default_color_3)\n    plt.title(t)\n    plt.grid()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:47:57.199969Z","iopub.execute_input":"2024-11-14T19:47:57.200406Z","iopub.status.idle":"2024-11-14T19:48:09.497608Z","shell.execute_reply.started":"2024-11-14T19:47:57.200363Z","shell.execute_reply":"2024-11-14T19:48:09.496411Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# this is our actual target\ntarget = 'responder_6'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T19:48:09.499385Z","iopub.execute_input":"2024-11-14T19:48:09.500430Z","iopub.status.idle":"2024-11-14T19:48:09.505636Z","shell.execute_reply.started":"2024-11-14T19:48:09.500371Z","shell.execute_reply":"2024-11-14T19:48:09.504283Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# calc mean by symbol\nmean_by_symbol = df.group_by(by='symbol_id').agg(pl.col(target).mean()).to_pandas()\nmean_by_symbol.sort_values(by='by', inplace=True)\nmean_by_symbol.reset_index(drop=True, inplace=True)\nmean_by_symbol.rename(columns={'by' : 'symbol_id',\n                               'responder_6' : 'mean_target'}, inplace=True)\nmean_by_symbol","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T20:06:53.947661Z","iopub.execute_input":"2024-11-14T20:06:53.948120Z","iopub.status.idle":"2024-11-14T20:06:54.631262Z","shell.execute_reply.started":"2024-11-14T20:06:53.948078Z","shell.execute_reply":"2024-11-14T20:06:54.630056Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# plot mean values\nplt.scatter(mean_by_symbol.symbol_id, mean_by_symbol.mean_target,\n            color=default_color_3)\nplt.title('Mean target value by symbol_id')\nplt.xlabel('symbol_id')\nplt.ylabel('mean(target)')\nplt.grid()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T20:08:39.123454Z","iopub.execute_input":"2024-11-14T20:08:39.123971Z","iopub.status.idle":"2024-11-14T20:08:39.411794Z","shell.execute_reply.started":"2024-11-14T20:08:39.123924Z","shell.execute_reply":"2024-11-14T20:08:39.410330Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a id='dist_fit'></a>\n# Distribution fits on target","metadata":{}},{"cell_type":"code","source":"# available distributions in fitter packages\nprint(get_distributions())","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:48:10.164244Z","iopub.execute_input":"2024-11-14T19:48:10.164619Z","iopub.status.idle":"2024-11-14T19:48:10.186702Z","shell.execute_reply.started":"2024-11-14T19:48:10.164567Z","shell.execute_reply":"2024-11-14T19:48:10.185478Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# try to fit a few distribution types to target 'responder_6'\n# we use a subset to achieve a reasonable run time\ndata = df[target].sample(50000, seed=my_random_seed)\ndist_fitter = Fitter(data,\n                     distributions=['lognorm', 't', 'cauchy',\n                                    'genhyperbolic', 'norminvgauss', 'tukeylambda', \n                                    'gennorm', 'dgamma', 'johnsonsu'], \n                     timeout=300)\ndist_fitter.fit()\nplt.figure(figsize=(12,5))\ndist_fitter.summary(9)","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:48:10.188280Z","iopub.execute_input":"2024-11-14T19:48:10.188761Z","iopub.status.idle":"2024-11-14T19:50:01.625302Z","shell.execute_reply.started":"2024-11-14T19:48:10.188716Z","shell.execute_reply":"2024-11-14T19:50:01.623984Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# just plotting the fitted PDFs\ndist_fitter.plot_pdf(Nbest=9, lw=1)","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:50:01.627102Z","iopub.execute_input":"2024-11-14T19:50:01.627465Z","iopub.status.idle":"2024-11-14T19:50:01.984183Z","shell.execute_reply.started":"2024-11-14T19:50:01.627422Z","shell.execute_reply":"2024-11-14T19:50:01.982732Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# best fit\ndist_fitter.get_best()","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:50:01.986091Z","iopub.execute_input":"2024-11-14T19:50:01.986626Z","iopub.status.idle":"2024-11-14T19:50:01.995962Z","shell.execute_reply.started":"2024-11-14T19:50:01.986567Z","shell.execute_reply":"2024-11-14T19:50:01.994598Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a id='corr_target_features'></a>\n# Correlation Targets vs Features","metadata":{}},{"cell_type":"code","source":"# init correlation matrix\ncorr_matrix = np.ones((79,9), dtype='float64')\n\n# calc correlations pairwise\nj = 0 # target index\nfor t in targets:\n    print('Calculating for target', t)\n    i = 0 # feature index\n    for f in features:\n        corr_ft = df.select(pl.corr(f, t, method='pearson'))[0,0]\n        corr_matrix[i][j] = corr_ft\n        i = i + 1\n    j = j + 1","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:50:01.997784Z","iopub.execute_input":"2024-11-14T19:50:01.998635Z","iopub.status.idle":"2024-11-14T19:55:27.362946Z","shell.execute_reply.started":"2024-11-14T19:50:01.998576Z","shell.execute_reply":"2024-11-14T19:55:27.361405Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# convert to data frame\ncorr_targets_features = pd.DataFrame(corr_matrix, columns=targets)\ncorr_targets_features.index = features","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:55:27.366232Z","iopub.execute_input":"2024-11-14T19:55:27.366860Z","iopub.status.idle":"2024-11-14T19:55:27.374625Z","shell.execute_reply.started":"2024-11-14T19:55:27.366800Z","shell.execute_reply":"2024-11-14T19:55:27.373361Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# show correlations between features and targets\ncorr_targets_features","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:55:27.376414Z","iopub.execute_input":"2024-11-14T19:55:27.376955Z","iopub.status.idle":"2024-11-14T19:55:27.434189Z","shell.execute_reply.started":"2024-11-14T19:55:27.376903Z","shell.execute_reply":"2024-11-14T19:55:27.432860Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# visualize correlations\nplt.figure(figsize=(6,16))\nsns.heatmap(corr_targets_features, cmap='RdYlGn',\n            vmin=-.25, vmax=.25,\n            linewidths=0.5, linecolor='black')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:56:31.518956Z","iopub.execute_input":"2024-11-14T19:56:31.519395Z","iopub.status.idle":"2024-11-14T19:56:32.545099Z","shell.execute_reply.started":"2024-11-14T19:56:31.519355Z","shell.execute_reply":"2024-11-14T19:56:32.543707Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# export result\ncorr_targets_features.to_csv('corr_targets_features.csv')","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:55:28.467474Z","iopub.execute_input":"2024-11-14T19:55:28.467953Z","iopub.status.idle":"2024-11-14T19:55:28.505520Z","shell.execute_reply.started":"2024-11-14T19:55:28.467903Z","shell.execute_reply":"2024-11-14T19:55:28.504368Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# visualize an example pair\nf = 'feature_36'\nt = 'responder_0'\nsns.jointplot(data=df, x=f, y=t,\n              kind='hist')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:55:28.507333Z","iopub.execute_input":"2024-11-14T19:55:28.507842Z","iopub.status.idle":"2024-11-14T19:55:47.439260Z","shell.execute_reply.started":"2024-11-14T19:55:28.507788Z","shell.execute_reply":"2024-11-14T19:55:47.437408Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a id='corr_target'></a>\n# Correlation of Targets","metadata":{}},{"cell_type":"code","source":"# init correlation matrix\ncorr_matrix = np.ones((9,9), dtype='float64')\n\n# calc correlations pairwise\nj = 0 # target index\nfor t in targets:\n    print('Calculating for target', t)\n    i = 0 # feature index\n    for f in targets:\n        corr_ft = df.select(pl.corr(f, t, method='pearson'))[0,0]\n        corr_matrix[i][j] = corr_ft\n        i = i + 1\n    j = j + 1","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:57:58.994286Z","iopub.execute_input":"2024-11-14T19:57:58.994783Z","iopub.status.idle":"2024-11-14T19:58:22.104874Z","shell.execute_reply.started":"2024-11-14T19:57:58.994740Z","shell.execute_reply":"2024-11-14T19:58:22.103565Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# convert to data frame\ncorr_targets = pd.DataFrame(corr_matrix, columns=targets)\ncorr_targets.index = targets","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:58:38.709895Z","iopub.execute_input":"2024-11-14T19:58:38.710422Z","iopub.status.idle":"2024-11-14T19:58:38.716859Z","shell.execute_reply.started":"2024-11-14T19:58:38.710380Z","shell.execute_reply":"2024-11-14T19:58:38.715428Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# visualize correlations\nplt.figure(figsize=(7,5))\nsns.heatmap(corr_targets, cmap='RdYlGn',\n            annot=True, vmin=-1, vmax=1,\n            linewidths=0.5, linecolor='black')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:58:41.782881Z","iopub.execute_input":"2024-11-14T19:58:41.783276Z","iopub.status.idle":"2024-11-14T19:58:42.383771Z","shell.execute_reply.started":"2024-11-14T19:58:41.783235Z","shell.execute_reply":"2024-11-14T19:58:42.382476Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# visualize an example pair\nt1 = 'responder_3'\nt2 = 'responder_6'\nsns.jointplot(data=df, x=t1, y=t2,\n              kind='hist')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:59:22.860495Z","iopub.execute_input":"2024-11-14T19:59:22.861620Z","iopub.status.idle":"2024-11-14T20:01:04.776636Z","shell.execute_reply.started":"2024-11-14T19:59:22.861525Z","shell.execute_reply":"2024-11-14T20:01:04.774610Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a id='symbol'></a>\n# Deep dive for an example symbol","metadata":{}},{"cell_type":"code","source":"# pick a specific symbol\nselected_symbol = 1\ndf_ex = df.filter(pl.col('symbol_id') == selected_symbol)\ndf_ex.shape","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:55:47.450056Z","iopub.status.idle":"2024-11-14T19:55:47.450509Z","shell.execute_reply.started":"2024-11-14T19:55:47.450289Z","shell.execute_reply":"2024-11-14T19:55:47.450312Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# basic stats\ndf_ex.describe()","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:55:47.453498Z","iopub.status.idle":"2024-11-14T19:55:47.454258Z","shell.execute_reply.started":"2024-11-14T19:55:47.453955Z","shell.execute_reply":"2024-11-14T19:55:47.453983Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# plot time series\nplt.figure(figsize=(12,4))\nplt.plot(df_ex[target], color=default_color_3,\n         alpha=1)\nplt.title('Symbol ' + str(selected_symbol))\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:55:47.456888Z","iopub.status.idle":"2024-11-14T19:55:47.457441Z","shell.execute_reply.started":"2024-11-14T19:55:47.457201Z","shell.execute_reply":"2024-11-14T19:55:47.457226Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### More than 1.5 million points => To dense for a meaningful plot, let's further drill down to a specific date:","metadata":{}},{"cell_type":"code","source":"# pick a specific date\nselected_date = 1\ndf_ex_date = df_ex.filter(pl.col('date_id') == selected_date)","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:55:47.459844Z","iopub.status.idle":"2024-11-14T19:55:47.460303Z","shell.execute_reply.started":"2024-11-14T19:55:47.460089Z","shell.execute_reply":"2024-11-14T19:55:47.460112Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# basic stats\ndf_ex_date.describe()","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:55:47.462286Z","iopub.status.idle":"2024-11-14T19:55:47.462816Z","shell.execute_reply.started":"2024-11-14T19:55:47.462541Z","shell.execute_reply":"2024-11-14T19:55:47.462594Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# plot time series\nplt.figure(figsize=(12,4))\nplt.plot(df_ex_date['time_id'], df_ex_date[target], color=default_color_3)\nplt.title('Symbol ' + str(selected_symbol) + ' / Date ' + str(selected_date))\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-14T19:55:47.464716Z","iopub.status.idle":"2024-11-14T19:55:47.465171Z","shell.execute_reply.started":"2024-11-14T19:55:47.464954Z","shell.execute_reply":"2024-11-14T19:55:47.464977Z"},"trusted":true},"outputs":[],"execution_count":null}]}