{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport polars as pl\nfrom matplotlib import pyplot as plt\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"id":"IanlX-Eqn2O5","execution":{"iopub.status.busy":"2023-05-05T16:59:06.382477Z","iopub.execute_input":"2023-05-05T16:59:06.382955Z","iopub.status.idle":"2023-05-05T16:59:06.636498Z","shell.execute_reply.started":"2023-05-05T16:59:06.382916Z","shell.execute_reply":"2023-05-05T16:59:06.635311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtyping = {\n    'session_id' : np.uint64,\n    'index' : np.uint8,\n    'elapsed_time' : np.uint8,\n    'event_name' : 'category',\n    'name' : 'category',\n    'level' : np.uint8,\n    'page' : np.float32,\n    'room_coor_x' : np.float32,\n    'room_coor_y' : np.float32,\n    'screen_coor_x' : np.float32,\n    'screen_coor_y' : np.float32,\n    'hover_duration' : np.float32,\n    'text' : 'category',\n    'fqid' : 'category',\n    'room_fqid' : 'category',\n    'text_fqid' : 'category',\n    'fullscreen' : np.bool8,\n    'hq' : np.bool8,\n    'music' : np.bool8,\n    'level_group' : 'category'\n}\ndataset_df = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv', dtype=dtyping)\ndataset_df.head(50)","metadata":{"id":"gLpK2yAen2O7","execution":{"iopub.status.busy":"2023-05-05T16:59:09.018226Z","iopub.execute_input":"2023-05-05T16:59:09.018733Z","iopub.status.idle":"2023-05-05T17:01:40.196226Z","shell.execute_reply.started":"2023-05-05T16:59:09.018694Z","shell.execute_reply":"2023-05-05T17:01:40.195146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df['session_id'].hist(bins = 20)","metadata":{"id":"-RTRVRiWn2O8","execution":{"iopub.status.busy":"2023-05-05T17:01:51.419617Z","iopub.execute_input":"2023-05-05T17:01:51.421696Z","iopub.status.idle":"2023-05-05T17:01:52.584716Z","shell.execute_reply.started":"2023-05-05T17:01:51.421605Z","shell.execute_reply":"2023-05-05T17:01:52.583408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df['event_name'].hist(bins = 30)","metadata":{"execution":{"iopub.status.busy":"2023-05-05T17:01:58.489484Z","iopub.execute_input":"2023-05-05T17:01:58.490008Z","iopub.status.idle":"2023-05-05T17:02:14.860346Z","shell.execute_reply.started":"2023-05-05T17:01:58.489968Z","shell.execute_reply":"2023-05-05T17:02:14.859231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df['name'].hist(bins = 30)","metadata":{"execution":{"iopub.status.busy":"2023-05-05T17:02:17.557100Z","iopub.execute_input":"2023-05-05T17:02:17.557559Z","iopub.status.idle":"2023-05-05T17:02:26.740652Z","shell.execute_reply.started":"2023-05-05T17:02:17.557509Z","shell.execute_reply":"2023-05-05T17:02:26.739163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df['page'].hist(bins = 30)","metadata":{"execution":{"iopub.status.busy":"2023-05-05T17:02:29.616739Z","iopub.execute_input":"2023-05-05T17:02:29.617239Z","iopub.status.idle":"2023-05-05T17:02:29.982895Z","shell.execute_reply.started":"2023-05-05T17:02:29.617192Z","shell.execute_reply":"2023-05-05T17:02:29.981167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df['level_group'].hist(bins = 30)","metadata":{"execution":{"iopub.status.busy":"2023-05-05T17:02:32.600084Z","iopub.execute_input":"2023-05-05T17:02:32.600529Z","iopub.status.idle":"2023-05-05T17:02:41.996655Z","shell.execute_reply.started":"2023-05-05T17:02:32.600477Z","shell.execute_reply":"2023-05-05T17:02:41.995386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\nlabels['session'] = labels['session_id'].str.split('_', expand = True)[0].astype(np.uint64)\nlabels['q'] = labels['session_id'].str.split('_q', expand = True)[1].astype(int)  \nlabels","metadata":{"execution":{"iopub.status.busy":"2023-05-05T17:02:45.353892Z","iopub.execute_input":"2023-05-05T17:02:45.354363Z","iopub.status.idle":"2023-05-05T17:02:49.728249Z","shell.execute_reply.started":"2023-05-05T17:02:45.354326Z","shell.execute_reply":"2023-05-05T17:02:49.726811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels['q'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-05-05T17:03:01.110603Z","iopub.execute_input":"2023-05-05T17:03:01.111094Z","iopub.status.idle":"2023-05-05T17:03:01.133224Z","shell.execute_reply.started":"2023-05-05T17:03:01.111056Z","shell.execute_reply":"2023-05-05T17:03:01.131749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_ids = list(np.unique(labels[\"session_id\"]))\nlen(session_ids)","metadata":{"execution":{"iopub.status.busy":"2023-05-05T17:03:03.519083Z","iopub.execute_input":"2023-05-05T17:03:03.519600Z","iopub.status.idle":"2023-05-05T17:03:04.030184Z","shell.execute_reply.started":"2023-05-05T17:03:03.519523Z","shell.execute_reply":"2023-05-05T17:03:04.029155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')","metadata":{"id":"KD4uayl2n2O9","execution":{"iopub.status.busy":"2023-05-05T17:03:07.733879Z","iopub.execute_input":"2023-05-05T17:03:07.734350Z","iopub.status.idle":"2023-05-05T17:03:08.121924Z","shell.execute_reply.started":"2023-05-05T17:03:07.734312Z","shell.execute_reply":"2023-05-05T17:03:08.120222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels['session'] = labels.session_id.apply(lambda x: int(x.split('_')[0]) )\nlabels['q'] = labels.session_id.apply(lambda x: int(x.split('_')[-1][1:]) )","metadata":{"id":"Kva8_Dbqn2O9","execution":{"iopub.status.busy":"2023-05-05T17:03:11.830136Z","iopub.execute_input":"2023-05-05T17:03:11.830957Z","iopub.status.idle":"2023-05-05T17:03:12.579556Z","shell.execute_reply.started":"2023-05-05T17:03:11.830913Z","shell.execute_reply":"2023-05-05T17:03:12.577982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display the first 5 examples\nlabels.head(5)","metadata":{"id":"0eD-KZMvn2O-","execution":{"iopub.status.busy":"2023-05-05T17:03:17.268285Z","iopub.execute_input":"2023-05-05T17:03:17.268719Z","iopub.status.idle":"2023-05-05T17:03:17.281530Z","shell.execute_reply.started":"2023-05-05T17:03:17.268682Z","shell.execute_reply":"2023-05-05T17:03:17.280139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(3, 3))\nplot_df = labels.correct.value_counts()\nplot_df.plot(kind=\"bar\", color=['b', 'c'])","metadata":{"id":"-l9wBCTYn2O_","execution":{"iopub.status.busy":"2023-05-05T17:03:20.000173Z","iopub.execute_input":"2023-05-05T17:03:20.000683Z","iopub.status.idle":"2023-05-05T17:03:20.183868Z","shell.execute_reply.started":"2023-05-05T17:03:20.000643Z","shell.execute_reply":"2023-05-05T17:03:20.182123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 20))\nplt.subplots_adjust(hspace=0.5, wspace=0.5)\nplt.suptitle(\"\\\"Correct\\\" column values for each question\", fontsize=14, y=0.94)\nfor n in range(1,19):\n    #print(n, str(n))\n    ax = plt.subplot(6, 3, n)\n\n    # filter df and plot ticker on the new subplot axis\n    plot_df = labels.loc[labels.q == n]\n    plot_df = plot_df.correct.value_counts()\n    plot_df.plot(ax=ax, kind=\"bar\", color=['b', 'c'])\n    \n    # chart formatting\n    ax.set_title(\"Question \" + str(n))\n    ax.set_xlabel(\"\")\n","metadata":{"id":"X238--97n2O_","execution":{"iopub.status.busy":"2023-05-05T17:03:25.000432Z","iopub.execute_input":"2023-05-05T17:03:25.001474Z","iopub.status.idle":"2023-05-05T17:03:26.975679Z","shell.execute_reply.started":"2023-05-05T17:03:25.001415Z","shell.execute_reply":"2023-05-05T17:03:26.974430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CATEGORICAL = ['event_name', 'name','fqid', 'room_fqid', 'text_fqid']\nNUMERICAL = ['elapsed_time','level','page','room_coor_x', 'room_coor_y', \n        'screen_coor_x', 'screen_coor_y', 'hover_duration']","metadata":{"id":"cCZWGiL_n2PA","execution":{"iopub.status.busy":"2023-05-05T17:03:37.553541Z","iopub.execute_input":"2023-05-05T17:03:37.555199Z","iopub.status.idle":"2023-05-05T17:03:37.561064Z","shell.execute_reply.started":"2023-05-05T17:03:37.555094Z","shell.execute_reply":"2023-05-05T17:03:37.559705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reference: https://www.kaggle.com/code/cdeotte/random-forest-baseline-0-664/notebook\n\ndef feature_engineer(dataset_df):\n    dfs = []\n    for c in CATEGORICAL:\n        tmp = dataset_df.groupby(['session_id','level_group'])[c].agg('nunique')\n        tmp.name = tmp.name + '_nunique'\n        dfs.append(tmp)\n    for c in NUMERICAL:\n        tmp = dataset_df.groupby(['session_id','level_group'])[c].agg('mean')\n        dfs.append(tmp)\n    for c in NUMERICAL:\n        tmp = dataset_df.groupby(['session_id','level_group'])[c].agg('std')\n        tmp.name = tmp.name + '_std'\n        dfs.append(tmp)\n    dataset_df = pd.concat(dfs,axis=1)\n    dataset_df = dataset_df.fillna(-1)\n    dataset_df = dataset_df.reset_index()\n    dataset_df = dataset_df.set_index('session_id')\n    return dataset_df","metadata":{"id":"nHWhAOtTn2PA","execution":{"iopub.status.busy":"2023-05-05T17:03:39.820756Z","iopub.execute_input":"2023-05-05T17:03:39.822334Z","iopub.status.idle":"2023-05-05T17:03:39.832221Z","shell.execute_reply.started":"2023-05-05T17:03:39.822262Z","shell.execute_reply":"2023-05-05T17:03:39.830635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df = feature_engineer(dataset_df)\nprint(\"Full prepared dataset shape is {}\".format(dataset_df.shape))","metadata":{"id":"JKcoPoemn2PA","execution":{"iopub.status.busy":"2023-05-05T17:03:43.075122Z","iopub.execute_input":"2023-05-05T17:03:43.075577Z","iopub.status.idle":"2023-05-05T17:04:34.307289Z","shell.execute_reply.started":"2023-05-05T17:03:43.075525Z","shell.execute_reply":"2023-05-05T17:04:34.306022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display the first 5 examples\ndataset_df.head(5)","metadata":{"id":"mvQEsdV1n2PB","execution":{"iopub.status.busy":"2023-05-05T17:04:51.197100Z","iopub.execute_input":"2023-05-05T17:04:51.197578Z","iopub.status.idle":"2023-05-05T17:04:51.233676Z","shell.execute_reply.started":"2023-05-05T17:04:51.197523Z","shell.execute_reply":"2023-05-05T17:04:51.231994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df.describe()","metadata":{"id":"DRusg-N1n2PB","execution":{"iopub.status.busy":"2023-05-05T17:04:54.302976Z","iopub.execute_input":"2023-05-05T17:04:54.303497Z","iopub.status.idle":"2023-05-05T17:04:54.471055Z","shell.execute_reply.started":"2023-05-05T17:04:54.303451Z","shell.execute_reply":"2023-05-05T17:04:54.469603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"figure, axis = plt.subplots(3, 2, figsize=(10, 10))\n\nfor name, data in dataset_df.groupby('level_group'):\n    axis[0, 0].plot(range(1, len(data['room_coor_x_std'])+1), data['room_coor_x_std'], label=name)\n    axis[0, 1].plot(range(1, len(data['room_coor_y_std'])+1), data['room_coor_y_std'], label=name)\n    axis[1, 0].plot(range(1, len(data['screen_coor_x_std'])+1), data['screen_coor_x_std'], label=name)\n    axis[1, 1].plot(range(1, len(data['screen_coor_y_std'])+1), data['screen_coor_y_std'], label=name)\n    axis[2, 0].plot(range(1, len(data['hover_duration'])+1), data['hover_duration_std'], label=name)\n    axis[2, 1].plot(range(1, len(data['elapsed_time_std'])+1), data['elapsed_time_std'], label=name)\n    \n\naxis[0, 0].set_title('room_coor_x')\naxis[0, 1].set_title('room_coor_y')\naxis[1, 0].set_title('screen_coor_x')\naxis[1, 1].set_title('screen_coor_y')\naxis[2, 0].set_title('hover_duration')\naxis[2, 1].set_title('elapsed_time_std')\n\nfor i in range(3):\n    axis[i, 0].legend()\n    axis[i, 1].legend()\n\nplt.show()","metadata":{"id":"mXIiaq_bn2PC","execution":{"iopub.status.busy":"2023-05-05T17:05:00.737542Z","iopub.execute_input":"2023-05-05T17:05:00.737996Z","iopub.status.idle":"2023-05-05T17:05:03.866863Z","shell.execute_reply.started":"2023-05-05T17:05:00.737960Z","shell.execute_reply":"2023-05-05T17:05:03.865642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now let us split the dataset into training and testing datasets:","metadata":{"id":"W0ex_Jkln2PC"}},{"cell_type":"code","source":"def split_dataset(dataset, test_ratio=0.20):\n    USER_LIST = dataset_df.index.unique()\n    split = int(len(USER_LIST) * (1 - 0.20))\n    return dataset.loc[USER_LIST[:split]], dataset.loc[USER_LIST[split:]]\n\ntrain_x, valid_x = split_dataset(dataset_df)\nprint(\"{} examples in training, {} examples in testing.\".format(\n    len(train_x), len(valid_x)))","metadata":{"id":"OZfTcCJfn2PC","execution":{"iopub.status.busy":"2023-05-05T17:05:07.227870Z","iopub.execute_input":"2023-05-05T17:05:07.228357Z","iopub.status.idle":"2023-05-05T17:05:07.307190Z","shell.execute_reply.started":"2023-05-05T17:05:07.228318Z","shell.execute_reply":"2023-05-05T17:05:07.305778Z"},"trusted":true},"execution_count":null,"outputs":[]}]}