{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# EDA of Aggregate Features\n\nThere are many great EDA notebooks in this competition. Most of these notebooks provide analysis of the raw features of the train dataset. But when using gradient boosting methods, which are working really well in this competition, we train our models using the aggregate features. \n\nSo this notebook provides an EDA of these aggregate features. We can get to know about many interesting things by looking at this EDA like the distributiion of these features, which features are highly correlated with each other, which features are helpful in predicting the target, the outliners, etc. The notebook only uses Level Group 0-4 data, i.e., Questions 1-3. You can extend this to the remaining level groups also.\n\nI built this EDA taking help from [this][1] amazing EDA by @alijs1 and used the aggregate features of my notebook which you can find [here][2].\n\nI hope this will helps you!!\n\n[1]: https://www.kaggle.com/code/alijs1/ieee-transaction-columns-reference/notebook\n[2]: https://www.kaggle.com/code/shashwatraman/gpu-xgb-baseline-using-rapids-cudf-train","metadata":{}},{"cell_type":"code","source":"import cudf as pd\nimport numpy as np\nfrom sklearn.model_selection import KFold, GroupKFold\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import f1_score\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm\nfrom collections import defaultdict\nimport warnings\nfrom itertools import combinations\nimport gc\nimport pickle\nimport pandas\n\ntrain = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv')\n\ntargets = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\ntargets['session'] = pd.to_numeric( targets.session_id.str.split('_').list.get(0) )\ntargets['q'] = pd.to_numeric( targets.session_id.str.split('_q').list.get(1) )","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-03-09T14:12:14.879305Z","iopub.execute_input":"2023-03-09T14:12:14.879729Z","iopub.status.idle":"2023-03-09T14:12:42.256331Z","shell.execute_reply.started":"2023-03-09T14:12:14.879649Z","shell.execute_reply":"2023-03-09T14:12:42.255324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineer","metadata":{}},{"cell_type":"code","source":"#Calculate Elapsed Time Difference Column\ntrainp = train.to_pandas()\ntrainp['time_diff'] = (trainp['elapsed_time'] - trainp.groupby(['session_id','level_group'])['elapsed_time'].shift(1)).clip(0,1e7)\ntrain = pd.DataFrame(trainp)\ntrain.loc[train['time_diff']<0] = 0\n\ntrain1 = train[train[\"level_group\"]=='0-4']\ntrain2 = train[train[\"level_group\"]=='5-12']\ntrain3 = train[train[\"level_group\"]=='13-22']\n\nCATS = ['event_name','name','fqid','room_fqid','text_fqid']\n\nNUMS = ['room_coor_x','room_coor_y','screen_coor_x','screen_coor_y','hover_duration','time_diff']\n\nEVENTS = ['cutscene_click', 'person_click', 'navigate_click',\n       'observation_click', 'notification_click', 'object_click',\n       'object_hover', 'map_hover', 'map_click', 'checkpoint',\n       'notebook_click']\n\nNAMES = ['basic', 'undefined', 'close', 'open', 'prev', 'next']","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-03-09T14:12:42.258084Z","iopub.execute_input":"2023-03-09T14:12:42.258419Z","iopub.status.idle":"2023-03-09T14:12:55.766881Z","shell.execute_reply.started":"2023-03-09T14:12:42.258392Z","shell.execute_reply":"2023-03-09T14:12:55.765912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineer(x, grp):\n    \n    x['elapsed_time'] = x['elapsed_time'] / 1000\n    x['time_diff'] = x['time_diff'] / 1000\n    \n    #session duration\n    df_final = x.groupby('session_id')['index'].agg('count')\n    df_final.name = 'num_events'\n    df_final = df_final.reset_index()\n    df_final = df_final.set_index('session_id')\n    \n    #Bingo Features\n    if grp == '5-12':\n        \n        df_final['logbingo-logbook'] = x[(x['fqid']=='logbook.page.bingo')&(x['event_name']=='object_click')].groupby('session_id')['index'].agg('first') - x[x['fqid']=='logbook'].groupby('session_id')['index'].agg('first')\n        df_final['readerbingo-reader'] = x[(x['fqid']=='reader.paper2.bingo')&(x['event_name']=='object_click')].groupby('session_id')['index'].agg('first') - x[x['fqid']=='reader'].groupby('session_id')['index'].agg('first')\n        df_final['jourbingo-journalspic'] = x[(x['fqid']=='journals.pic_2.bingo')&(x['event_name']=='object_click')].groupby('session_id')['index'].agg('first') - x[x['fqid']=='journals.pic_0.next'].groupby('session_id')['index'].agg('first')\n        \n        df_final['logbingo-logbook_time'] = x[(x['fqid']=='logbook.page.bingo')&(x['event_name']=='object_click')].groupby('session_id')['elapsed_time'].agg('first') - x[x['fqid']=='logbook'].groupby('session_id')['elapsed_time'].agg('first')\n        df_final['readerbingo-reader_time'] = x[(x['fqid']=='reader.paper2.bingo')&(x['event_name']=='object_click')].groupby('session_id')['elapsed_time'].agg('first') - x[x['fqid']=='reader'].groupby('session_id')['elapsed_time'].agg('first')\n        df_final['jourbingo-journalspic_time'] = x[(x['fqid']=='journals.pic_2.bingo')&(x['event_name']=='object_click')].groupby('session_id')['elapsed_time'].agg('first') - x[x['fqid']=='journals.pic_0.next'].groupby('session_id')['elapsed_time'].agg('first')\n        \n    if grp=='13-22':\n        \n        df_final['readerbingo-reader_flag'] = x[(x['fqid']=='reader_flag.paper2.bingo')&(x['event_name']=='object_click')].groupby('session_id')['index'].agg('first') - x[x['fqid']=='reader_flag'].groupby('session_id')['index'].agg('first')\n        df_final['journalbingo-journals_flag'] = x[(x['fqid']=='journals_flag.pic_0.bingo')&(x['event_name']=='object_click')].groupby('session_id')['index'].agg('first') - x[x['fqid']=='journals_flag'].groupby('session_id')['index'].agg('first')\n        \n        df_final['readerbingo-reader_flag_time'] = x[(x['fqid']=='reader_flag.paper2.bingo')&(x['event_name']=='object_click')].groupby('session_id')['elapsed_time'].agg('first') - x[x['fqid']=='reader_flag'].groupby('session_id')['elapsed_time'].agg('first')\n        df_final['journalbingo-journals_flag_time'] = x[(x['fqid']=='journals_flag.pic_0.bingo')&(x['event_name']=='object_click')].groupby('session_id')['elapsed_time'].agg('first') - x[x['fqid']=='journals_flag'].groupby('session_id')['elapsed_time'].agg('first')\n        \n    df_final['first_elapsed_time'] = x.groupby('session_id')['elapsed_time'].agg('first')\n    df_final['elapsed_time'] = x.groupby('session_id')['elapsed_time'].agg('last') - df_final['first_elapsed_time']\n    \n    for c in CATS:\n        df_final[f'{c}_nuniques'] = x.groupby('session_id')[c].agg('nunique')\n    \n    for c in NUMS:\n        df_final[f'{c}_mean'] = x.groupby('session_id')[c].agg('mean')\n        df_final[f'{c}_min'] = x.groupby('session_id')[c].agg('min')\n        df_final[f'{c}_max'] = x.groupby('session_id')[c].agg('max')\n        \n    for c in EVENTS:\n        x[c] = (x.event_name == c).astype('int8')\n    for c in EVENTS:\n        df_final[f'{c}_sum'] = x.groupby('session_id')[c].agg('sum')\n    x.drop(EVENTS, axis=1, inplace=True)\n    \n    for c in EVENTS:\n        df_final[f'{c}_time_mean'] = x[x['event_name']==c].groupby('session_id')['time_diff'].mean()\n        df_final[f'{c}_time_min'] = x[x['event_name']==c].groupby('session_id')['time_diff'].min()\n        df_final[f'{c}_time_max'] = x[x['event_name']==c].groupby('session_id')['time_diff'].max()\n    \n    for c in NAMES:\n        x[c] = (x.name == c).astype('int8')\n    for c in NAMES:\n        df_final[f'{c}_sum'] = x.groupby('session_id')[c].agg('sum')\n    x.drop(NAMES, axis=1, inplace=True)\n    \n    for c in NAMES:\n        df_final[f'{c}_time_mean'] = x[x['name']==c].groupby('session_id')['time_diff'].mean()\n        df_final[f'{c}_time_min'] = x[x['name']==c].groupby('session_id')['time_diff'].min()\n        df_final[f'{c}_time_max'] = x[x['name']==c].groupby('session_id')['time_diff'].max()\n\n    return df_final","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-03-09T14:12:55.768432Z","iopub.execute_input":"2023-03-09T14:12:55.768813Z","iopub.status.idle":"2023-03-09T14:12:55.792797Z","shell.execute_reply.started":"2023-03-09T14:12:55.768778Z","shell.execute_reply":"2023-03-09T14:12:55.791576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf1 = feature_engineer(train1.copy(), grp='0-4')\nprint('df1 done')","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-03-09T14:12:55.795604Z","iopub.execute_input":"2023-03-09T14:12:55.796477Z","iopub.status.idle":"2023-03-09T14:12:58.548468Z","shell.execute_reply.started":"2023-03-09T14:12:55.796450Z","shell.execute_reply":"2023-03-09T14:12:58.547304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null1 = df1.isnull().sum().sort_values(ascending=False) / len(df1)\n\ndrop1 = list(null1[null1>0.9].index.to_pandas())\nprint(len(drop1))\n\nfor col in tqdm(df1.columns):\n    if df1[col].nunique()==1:\n        print(col)\n        drop1.append(col)\nprint(\"*********df1 DONE*********\")","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-03-09T14:12:58.550020Z","iopub.execute_input":"2023-03-09T14:12:58.550586Z","iopub.status.idle":"2023-03-09T14:12:58.738985Z","shell.execute_reply.started":"2023-03-09T14:12:58.550548Z","shell.execute_reply":"2023-03-09T14:12:58.738009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FEATURES1 = [c for c in df1.columns if c not in drop1+['first_elapsed_time']]\ndf = df1[FEATURES1].copy()\n\ndf = df.to_pandas()\n\ntargets = targets.to_pandas()\ntargets = targets[['session','q','correct']]\ntargets = targets.set_index('session')\n\ndf['q1'] = targets[targets['q']==1]['correct']\ndf['q2'] = targets[targets['q']==2]['correct']\ndf['q3'] = targets[targets['q']==3]['correct']\n\ndf = df.reset_index()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-03-09T14:12:58.740721Z","iopub.execute_input":"2023-03-09T14:12:58.741369Z","iopub.status.idle":"2023-03-09T14:12:58.960904Z","shell.execute_reply.started":"2023-03-09T14:12:58.741333Z","shell.execute_reply":"2023-03-09T14:12:58.959563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"class color:\n    BOLD = '\\033[1m'\n    UNDERLINE = '\\033[4m'\n    END = '\\033[0m'\n    \ndef hist(col):\n    plt.figure(figsize=(15, 5))\n    plt.hist(df[col], bins=200);\n    plt.title(f'{col} distribution: ');\n    plt.show()\n    \ndef bar(col):\n    plt.figure(figsize=(15, 5))\n    ax = plt.subplot()\n    ax.bar(df[col].value_counts().sort_index().index, df[col].value_counts().sort_index().values);\n    plt.xticks(df[col].value_counts().sort_index().index)\n    plt.setp(ax.get_xticklabels(), rotation=30, ha='right')\n    plt.title(f'{col} barplot: ');\n    plt.show()\n\ndef _desc(data, col, label):\n    d0 = data.describe().reset_index()\n    d0.columns = [col, label]\n    return d0.append({col:'unique values', label:data.unique().shape[0]}, ignore_index=True) \\\n             .append({col:'NaNs', label:data.isnull().sum()}, ignore_index=True) \\\n             .append({col:'NaNs share', label:np.round(data.isnull().sum() / data.shape[0], 4)}, ignore_index=True)\n\ndef desc(col):\n    d0 = _desc(df[col], col, 'Train')\n    d1 = _desc(df.loc[df['q1'] == 1, col], col, 'Correct Q1')\n    d2 = _desc(df.loc[df['q1'] == 0, col], col, 'Incorrect Q1')\n    d3 = _desc(df.loc[df['q2'] == 1, col], col, 'Correct Q2')\n    d4 = _desc(df.loc[df['q2'] == 0, col], col, 'Incorrect Q2')\n    d5 = _desc(df.loc[df['q3'] == 1, col], col, 'Correct Q3')\n    d6 = _desc(df.loc[df['q3'] == 0, col], col, 'Incorrect Q3')\n\n    dd = d0.merge(d1).merge(d2).merge(d3).merge(d4).merge(d5).merge(d6)\n    display(dd)\n    \n    if col not in ['session_id']:\n        print('Most popular values (NaN = -999):')\n        N = 10\n        f1 = df[['q1',col]].fillna(-999).groupby(col)['q1'].agg(['size','mean','sum']).reset_index().sort_values('size', ascending=False).reset_index(drop=True)\n        ff1 = f1.head(N)\n        ff1 = ff1.rename({'size':'Count in train (desc)','mean':'Mean Q1','sum':'Sum Q1'}, axis=1)\n        f2 = df[['q2',col]].fillna(-999).groupby(col)['q2'].agg(['size','mean','sum']).reset_index().sort_values('size', ascending=False).reset_index(drop=True)\n        ff2 = f2.head(N)\n        ff2 = ff2.rename({'size':'Count in train (desc)','mean':'Mean Q2','sum':'Sum Q2'}, axis=1)\n        f3 = df[['q3',col]].fillna(-999).groupby(col)['q3'].agg(['size','mean','sum']).reset_index().sort_values('size', ascending=False).reset_index(drop=True)\n        ff3 = f3.head(N)\n        ff3 = ff3.rename({'size':'Count in train (desc)','mean':'Mean Q3','sum':'Sum Q3'}, axis=1)\n        f = ff1.merge(ff2).merge(ff3)\n        display(f)\n\ndef corr(col):\n    N = None\n    dfx = df.head(N) if N is not None else df.copy()\n    corrs = dfx.corrwith(dfx[col]).reset_index().sort_values(0, ascending=False).reset_index(drop=True).rename({'index':'Column',0:'Correlation with ' + col}, axis=1)\n    corrs = corrs.tail(-1)\n    print('Most correlated values with ' + col + ':')\n    dfx = pandas.concat([corrs.head(6), corrs.dropna().tail(5)])\n    display(dfx)\n    \ndef categorical(col):\n    bar(col)\n    desc(col)\n    \ndef numeric(col):\n    hist(col)\n    desc(col)\n    corr(col)\n    \ndef start(col):\n    if col not in ['session_id','q1','q2','q3']:\n        print()\n        print(color.BOLD + col + color.END)\n        categorical(col) if df[col].nunique() <= 100 else numeric(col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-03-09T14:39:56.549137Z","iopub.execute_input":"2023-03-09T14:39:56.549501Z","iopub.status.idle":"2023-03-09T14:39:56.574088Z","shell.execute_reply.started":"2023-03-09T14:39:56.549470Z","shell.execute_reply":"2023-03-09T14:39:56.573012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-09T14:12:59.031841Z","iopub.execute_input":"2023-03-09T14:12:59.033263Z","iopub.status.idle":"2023-03-09T14:12:59.070549Z","shell.execute_reply.started":"2023-03-09T14:12:59.033218Z","shell.execute_reply":"2023-03-09T14:12:59.069570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Aggregate Features Dataset Shape:',df.shape)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T14:18:47.054275Z","iopub.execute_input":"2023-03-09T14:18:47.054659Z","iopub.status.idle":"2023-03-09T14:18:47.061002Z","shell.execute_reply.started":"2023-03-09T14:18:47.054626Z","shell.execute_reply":"2023-03-09T14:18:47.059700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Q1","metadata":{}},{"cell_type":"code","source":"COL = 'q1'\nprint()\nprint(f'{COL} Target Value Counts:')\ndfx = df[COL].value_counts().reset_index().rename({'index':'Value','q1':'Count'}, axis=1)\ndfx['Share'] = np.round(dfx['Count'] / dfx['Count'].sum(), 6)\ndisplay(dfx)\ncorr(COL)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-03-09T14:20:06.702814Z","iopub.execute_input":"2023-03-09T14:20:06.703170Z","iopub.status.idle":"2023-03-09T14:20:06.768669Z","shell.execute_reply.started":"2023-03-09T14:20:06.703141Z","shell.execute_reply":"2023-03-09T14:20:06.767723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Q2","metadata":{}},{"cell_type":"code","source":"COL = 'q2'\nprint()\nprint(f'{COL} Target Value Counts:')\nprint()\ndfx = df[COL].value_counts().reset_index().rename({'index':'Value','q2':'Count'}, axis=1)\ndfx['Share'] = np.round(dfx['Count'] / dfx['Count'].sum(), 6)\ndisplay(dfx)\nprint()\ncorr(COL)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-03-09T14:12:59.173359Z","iopub.execute_input":"2023-03-09T14:12:59.174102Z","iopub.status.idle":"2023-03-09T14:12:59.269471Z","shell.execute_reply.started":"2023-03-09T14:12:59.174065Z","shell.execute_reply":"2023-03-09T14:12:59.268468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Q3","metadata":{}},{"cell_type":"code","source":"COL = 'q3'\nprint()\nprint(f'{COL} Target Value Counts:')\nprint()\ndfx = df[COL].value_counts().reset_index().rename({'index':'Value','q3':'Count'}, axis=1)\ndfx['Share'] = np.round(dfx['Count'] / dfx['Count'].sum(), 6)\ndisplay(dfx)\nprint()\ncorr(COL)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-03-09T14:12:59.271106Z","iopub.execute_input":"2023-03-09T14:12:59.271771Z","iopub.status.idle":"2023-03-09T14:12:59.367840Z","shell.execute_reply.started":"2023-03-09T14:12:59.271734Z","shell.execute_reply":"2023-03-09T14:12:59.366624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Aggregate Features","metadata":{}},{"cell_type":"code","source":"for col in list(df.columns.values):\n    start(col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-03-09T14:40:00.272960Z","iopub.execute_input":"2023-03-09T14:40:00.273319Z","iopub.status.idle":"2023-03-09T14:40:53.999630Z","shell.execute_reply.started":"2023-03-09T14:40:00.273289Z","shell.execute_reply":"2023-03-09T14:40:53.998327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}