{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\nimport statsmodels.api as sm\nimport numpy as np # linear algebra\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 150)\npd.option_context('mode.use_inf_as_na', False)\n\"\"\"\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\"\"\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-03T19:20:10.507575Z","iopub.execute_input":"2024-12-03T19:20:10.509035Z","iopub.status.idle":"2024-12-03T19:20:13.162446Z","shell.execute_reply.started":"2024-12-03T19:20:10.508931Z","shell.execute_reply":"2024-12-03T19:20:13.160845Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#import training data\npath = \"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=\"\nend_partition = 5\nfor i in range(0,end_partition):\n    if i==0:\n        df_train = pd.read_parquet(path+str(i))\n    else:\n        df_train=pd.concat([df_train,pd.read_parquet(path+str(i))])\n\n#load training data\n\ndf_train.sort_values(by=['date_id','time_id'],inplace=True)\n#print(df_train.head(20))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T19:20:13.164255Z","iopub.execute_input":"2024-12-03T19:20:13.164750Z","iopub.status.idle":"2024-12-03T19:20:33.702108Z","shell.execute_reply.started":"2024-12-03T19:20:13.164714Z","shell.execute_reply":"2024-12-03T19:20:33.701016Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Make some sequential folds by date_id. Want to keep order of dates.\nfold_splits = 2\ndf_train['date_fold'] = pd.cut(df_train['date_id'],fold_splits,labels=False)\n#adjust dates for graphing lines\ndf_train['date_adj'] = df_train['date_id'] - df_train.join(df_train.groupby(['date_fold'])['date_id'].min(),on='date_fold',rsuffix='_min')['date_id_min']\ndf_train = df_train.replace([np.inf, -np.inf], np.nan)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T19:20:33.703268Z","iopub.execute_input":"2024-12-03T19:20:33.703575Z","iopub.status.idle":"2024-12-03T19:21:09.253371Z","shell.execute_reply.started":"2024-12-03T19:20:33.703545Z","shell.execute_reply":"2024-12-03T19:21:09.252266Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Examine line plots of target variable (responder_6). Stacking folds on top of each other for easier viewer.\n%matplotlib inline\nfig,ax = plt.subplots(fold_splits,1)\nfor x in range(0,fold_splits):\n    sns.lineplot(x = \"date_adj\", y = \"responder_6\", data = df_train[df_train['date_fold']==x],ax=ax[x])\n    if x<fold_splits-1:\n        ax[x].set_xlabel(None)\n        ax[x].set_xticks([]) \n    ax[x].set_ylim([-.5, .5])\n    ax[x].set_ylabel('fold '+str(x))\n    ax[x].set_yticks([])   \nplt.suptitle('Resp 6 - Line Plots by Day')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T14:36:17.361427Z","iopub.execute_input":"2024-12-03T14:36:17.361838Z","iopub.status.idle":"2024-12-03T14:38:46.409419Z","shell.execute_reply.started":"2024-12-03T14:36:17.361804Z","shell.execute_reply":"2024-12-03T14:38:46.408097Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Histograms & qq plots (vs normal distribution) of responder_6","metadata":{}},{"cell_type":"code","source":"fig,ax = plt.subplots(fold_splits,2,squeeze=False,figsize=(9,2*fold_splits),tight_layout=True)\nfor x in range(0,fold_splits):\n    sns.histplot(data = df_train[df_train['date_fold']==x]['responder_6'],ax=ax[x,0]) \n    desc = df_train[df_train['date_fold']==x]['responder_6'].describe()\n    ax[x,0].text(1,1,desc.loc[~desc.index.isin(['min','max'])].to_string(),horizontalalignment='right',verticalalignment='top'\n               ,fontstretch='condensed',fontsize='xx-small',transform=ax[x,0].transAxes)\n    pplot = sm.ProbPlot(df_train['responder_6'].sample(frac=.05), fit=True)\n    fig2 = pplot.qqplot(line=\"45\",ax=ax[x,1])\n    if x<fold_splits-1:\n        ax[x,0].set_xlabel(None)\n        ax[x,0].set_xticks([]) \n        ax[x,1].set_xlabel(None)\n        ax[x,1].set_xticks([])         \n    ax[x,0].set_ylabel('fold '+str(x))\n    ax[x,0].set_yticks([])\n    ax[x,1].set_ylabel(None)\n    ax[x,1].set_yticks([])\n\nax[0,0].set_title('Resp 6 - Histograms')\nax[0,1].set_title('Resp 6 - QQ Plots')\nplt.show() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T15:26:51.929317Z","iopub.execute_input":"2024-12-03T15:26:51.929722Z","iopub.status.idle":"2024-12-03T15:27:20.388535Z","shell.execute_reply.started":"2024-12-03T15:26:51.929693Z","shell.execute_reply":"2024-12-03T15:27:20.387230Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Want to check how many 5s/extreme values there are\nfor x in range(0,5):\n    if x+1==1:\n        df_train_xt_freqs = pd.DataFrame(df_train[abs(df_train['responder_6'])>=x+1].groupby('date_fold')['responder_6'].count()/(df_train.groupby('date_fold')['responder_6'].count()))\n        df_train_xt_freqs.rename(columns={'responder_6':'GT_1'},inplace=True)\n    else:\n        df_train_xt_freqs['GT_' + str(x+1)] = df_train[abs(df_train['responder_6'])>=x+1].groupby('date_fold')['responder_6'].count()/(df_train.groupby('date_fold')['responder_6'].count())\n\nprint(df_train_xt_freqs)\n\n\"\"\"\"\"\ndf_train['resp_6_gt1']=[True if abs(x) >= 1 else False for x in df_train['responder_6']]\ndf_train['resp_6_gt2']=[True if abs(x) >= 2 else False for x in df_train['responder_6']]\ndf_train['resp_6_gt3']=[True if abs(x) >= 3 else False for x in df_train['responder_6']]\ndf_train['resp_6_gt4']=[True if abs(x) >= 4 else False for x in df_train['responder_6']]\ndf_train['resp_6_gt5']=[True if abs(x) >= 5 else False for x in df_train['responder_6']]\n\"\"\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T15:35:16.614490Z","iopub.execute_input":"2024-12-03T15:35:16.614924Z","iopub.status.idle":"2024-12-03T15:35:20.301520Z","shell.execute_reply.started":"2024-12-03T15:35:16.614889Z","shell.execute_reply":"2024-12-03T15:35:20.300469Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Describe responder_6 by symbol_id for one fold\ndesc=df_train[df_train['date_fold']==0].groupby('symbol_id')['responder_6'].describe()\ndesc['99%']=df_train[df_train['date_fold']==0].groupby('symbol_id')['responder_6'].quantile(.99)\ndesc['1%']=df_train[df_train['date_fold']==0].groupby('symbol_id')['responder_6'].quantile(.01)\n#desc.sort_values(by='count',ascending=False,inplace=True)\nprint(desc[['count','mean','std','1%','25%','50%','75%','99%']])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T15:41:25.814044Z","iopub.execute_input":"2024-12-03T15:41:25.815418Z","iopub.status.idle":"2024-12-03T15:41:32.839979Z","shell.execute_reply.started":"2024-12-03T15:41:25.815367Z","shell.execute_reply":"2024-12-03T15:41:32.838816Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Check if daily means & std are generally stationary. \n#Can check correlation between daily mean & std with date_id\n\ndf_train_means = df_train.groupby('date_id',as_index=False)[[x for x in list(df_train) if 'responder' in x ]].mean()\ndf_train_std = df_train.groupby('date_id',as_index=False)[[x for x in list(df_train) if 'responder' in x ]].std()\nfig,ax = plt.subplots(2,1)\nsns.lineplot(x = 'date_id', y = \"responder_6\", data = df_train_means,ax=ax[0])\nax[0].set(xlabel=None,ylabel='Resp 6 Mean')\nsns.lineplot(x = 'date_id', y = \"responder_6\", data = df_train_std,ax=ax[1])\nax[1].set(ylabel='Resp 6 StDev')\nplt.show()\n\nprint(\"Resp 6 Mean Correlation with Date Id: \",df_train_means.corr()['responder_6'][['date_id']].iloc[0])\nprint(\"Resp 6 StDev Correlation with Date Id: \",df_train_std.corr()['responder_6'][['date_id']].iloc[0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T15:49:38.482451Z","iopub.execute_input":"2024-12-03T15:49:38.483399Z","iopub.status.idle":"2024-12-03T15:49:41.663555Z","shell.execute_reply.started":"2024-12-03T15:49:38.483344Z","shell.execute_reply":"2024-12-03T15:49:41.662293Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Testing if there's any autocorrelation. \nsm.graphics.tsa.plot_pacf(df_train_means['responder_6'].squeeze(), lags=30)\n#ax[0].title=\"Autocorrelation - Intraday Mean (Resp 6)\"\nplt.title(\"Autocorrelation - Daily Mean (Resp 6)\")\nsm.graphics.tsa.plot_pacf(df_train_std['responder_6'].squeeze(), lags=30)\nplt.title(\"Autocorrelation - Intraday StDev (Resp 6)\")\nplt.show()\n\n#Checking correlation of responder 6 vs responders & lagged responders\nfor x in list(df_train_means):\n    df_train_means[x + '_lag_1'] = df_train_means[x].shift(1)\nprint(df_train_means.corr()['responder_6'].sort_values(ascending=False,key=abs))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T15:56:43.406058Z","iopub.execute_input":"2024-12-03T15:56:43.407057Z","iopub.status.idle":"2024-12-03T15:56:43.941807Z","shell.execute_reply.started":"2024-12-03T15:56:43.407012Z","shell.execute_reply":"2024-12-03T15:56:43.940748Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Start examining features...\nfeature_cols = [col for col in list(df_train) if 'feature' in col]\n\n#Looking for missing values. Segmenting by fold\ntotals = pd.DataFrame(df_train.groupby('date_fold').size(),columns=['totals'])\nmissing_cols = {}\nfor feat in [col for col in list(df_train) if 'feature' in col]:\n    missing_cols[feat] = lambda x: x.isnull().sum()\nmissings=df_train.groupby('date_fold').agg(missing_cols)\nprint(missings.div(totals['totals'],axis=0))\n\n#Flag missing values\nfor feat in [col for col in list(df_train) if 'feature' in col]:\n    df_train[feat+'_miss'] = df_train[feat].isnull()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T16:01:50.030432Z","iopub.execute_input":"2024-12-03T16:01:50.031301Z","iopub.status.idle":"2024-12-03T16:02:09.444625Z","shell.execute_reply.started":"2024-12-03T16:01:50.031243Z","shell.execute_reply":"2024-12-03T16:02:09.443264Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#This is just a sample of feature histograms\nfeature_plots_list = feature_cols[0:10]\n#feature_plots = len(feature_plots_list)\nfig,ax = plt.subplots(nrows=len(feature_plots_list),ncols=2,squeeze=False,figsize=(9,2*len(feature_plots_list)),tight_layout=True)\nfor idx,feat in enumerate(feature_plots_list):\n    sns.histplot(data = df_train[feat],ax=ax[idx,0])        \n    ax[idx,0].set_ylabel(str(feat))\n    ax[idx,0].set_yticks([])   \n    ax[idx,0].set_xlabel(None)\n    desc = df_train[feat].describe()\n    ax[idx,0].text(1,1,desc.to_string(),horizontalalignment='right',verticalalignment='top'\n               ,fontstretch='condensed',fontsize='xx-small',transform=ax[idx,0].transAxes)\n    pplot = sm.ProbPlot(df_train[~df_train[feat].isnull()][feat].sample(frac=.05), fit=True)\n    fig2 = pplot.qqplot(line=\"45\",ax=ax[idx,1])\n    ax[idx,1].set_ylabel(None)\n    ax[idx,1].set_yticks([])\nplt.show() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T16:40:51.572351Z","iopub.execute_input":"2024-12-03T16:40:51.572834Z","iopub.status.idle":"2024-12-03T16:46:00.699035Z","shell.execute_reply.started":"2024-12-03T16:40:51.572798Z","shell.execute_reply":"2024-12-03T16:46:00.697863Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Build lagged response aggregations by symbol. Starting with fold 0.\n\naggregations = ['mean', 'std','median','min','max']\nresp_means_by_sym = df_train[df_train['date_fold']==0].groupby(['date_id','symbol_id'])[[x for x in list(df_train) if 'responder' in x ]].agg(aggregations)\n\nfor x in resp_means_by_sym.columns:\n    resp_means_by_sym[x[0]+'_'+x[1]]=resp_means_by_sym[x]\n    resp_means_by_sym[x[0]+'_'+x[1]+'_lag1']=resp_means_by_sym.groupby(['symbol_id'])[x[0]+'_'+x[1]].shift(1)\n    resp_means_by_sym.drop(columns=[x[0]+'_'+x[1],x],inplace=True)\nresp_means_by_sym.columns=resp_means_by_sym.columns.droplevel(1)\ndf_train_fold0=df_train['date_fold']==0].join(resp_means_by_sym,on=['date_id','symbol_id'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T19:21:49.329706Z","iopub.execute_input":"2024-12-03T19:21:49.330127Z","iopub.status.idle":"2024-12-03T19:21:59.647433Z","shell.execute_reply.started":"2024-12-03T19:21:49.330075Z","shell.execute_reply":"2024-12-03T19:21:59.646297Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(df_train_fold0[~df_train_fold0['responder_6_mean_lag1'].isnull()].corr()['responder_6'].sort_values(ascending=False,key=abs))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T19:47:30.760933Z","iopub.execute_input":"2024-12-03T19:47:30.762846Z","iopub.status.idle":"2024-12-03T19:52:33.435590Z","shell.execute_reply.started":"2024-12-03T19:47:30.762776Z","shell.execute_reply":"2024-12-03T19:52:33.434043Z"}},"outputs":[],"execution_count":null}]}