{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-24T11:06:41.254986Z","iopub.execute_input":"2023-03-24T11:06:41.256554Z","iopub.status.idle":"2023-03-24T11:06:42.566476Z","shell.execute_reply.started":"2023-03-24T11:06:41.256496Z","shell.execute_reply":"2023-03-24T11:06:42.564826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events_df = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train.csv\",\n                       usecols=['fqid', 'text_fqid', 'room_fqid', 'text', 'level_group'])\nevents_df = events_df.drop_duplicates()","metadata":{"execution":{"iopub.status.busy":"2023-03-24T11:06:42.568917Z","iopub.execute_input":"2023-03-24T11:06:42.569340Z","iopub.status.idle":"2023-03-24T11:08:23.881334Z","shell.execute_reply.started":"2023-03-24T11:06:42.569305Z","shell.execute_reply":"2023-03-24T11:08:23.879860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(events_df.shape)\nevents_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-24T11:08:23.884477Z","iopub.execute_input":"2023-03-24T11:08:23.884874Z","iopub.status.idle":"2023-03-24T11:08:23.922137Z","shell.execute_reply.started":"2023-03-24T11:08:23.884835Z","shell.execute_reply":"2023-03-24T11:08:23.921284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train.csv\",\n                       usecols=['session_id', 'elapsed_time',\n                                'fqid', 'text_fqid', 'room_fqid',\n                                'level_group'],\n                       dtype={\n                           'elapsed_time': np.float32})\n\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-24T11:08:23.924255Z","iopub.execute_input":"2023-03-24T11:08:23.925030Z","iopub.status.idle":"2023-03-24T11:09:04.252252Z","shell.execute_reply.started":"2023-03-24T11:08:23.924981Z","shell.execute_reply":"2023-03-24T11:09:04.251020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = train_df.groupby('level_group')[['fqid', 'room_fqid', 'text_fqid']].nunique().reset_index().rename(columns={\n    'fqid': \"total_fqid\",\n    'room_fqid':'total_room_fqid',\n    'text_fqid': 'total_text_fqid'\n})\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-24T11:09:04.255039Z","iopub.execute_input":"2023-03-24T11:09:04.255476Z","iopub.status.idle":"2023-03-24T11:09:19.695127Z","shell.execute_reply.started":"2023-03-24T11:09:04.255429Z","shell.execute_reply":"2023-03-24T11:09:19.694340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax=plt.subplots(1, 3, figsize=(12, 5))\n\nfig.suptitle(\"Number of Unique categories for each level group\")\nsns.barplot(data=df, x='level_group', y='total_fqid', ax=ax[0], order=['0-4','5-12', '13-22'])\nsns.barplot(data=df, x='level_group', y='total_room_fqid', ax=ax[1], order=['0-4','5-12', '13-22'])\nsns.barplot(data=df, x='level_group', y='total_text_fqid', ax=ax[2], order=['0-4','5-12', '13-22'])\n\nax[0].set_title(\"fqid\")\nax[1].set_title(\"room fqid\")\nax[2].set_title(\"text fqid\")\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-24T11:09:19.698501Z","iopub.execute_input":"2023-03-24T11:09:19.700403Z","iopub.status.idle":"2023-03-24T11:09:20.057711Z","shell.execute_reply.started":"2023-03-24T11:09:19.700369Z","shell.execute_reply":"2023-03-24T11:09:20.056778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"session_df1 = train_df.groupby([\"session_id\", 'level_group'])[['fqid', 'room_fqid', 'text_fqid']].nunique().reset_index()\nsession_df1.head() ","metadata":{"execution":{"iopub.status.busy":"2023-03-24T11:09:20.059117Z","iopub.execute_input":"2023-03-24T11:09:20.059602Z","iopub.status.idle":"2023-03-24T11:09:35.284792Z","shell.execute_reply.started":"2023-03-24T11:09:20.059570Z","shell.execute_reply":"2023-03-24T11:09:35.284061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, l in enumerate(['0-4', '5-12', '13-22']):\n    fig,ax=plt.subplots(1, 3, figsize=(14, 4))\n    if i==0:\n        fig.suptitle(\"distribution of events per session\")\n    for k,colname in enumerate(['fqid', 'room_fqid', 'text_fqid']):\n        r = df[(df.level_group==l)][\"total_\"+colname].values[0]\n        tmp_df = session_df1[session_df1.level_group == l]\n        s = tmp_df[colname]/r\n        ax[k].hist(s, bins=100)\n        if i==0:\n            ax[k].set_title(colname)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-24T11:09:35.285978Z","iopub.execute_input":"2023-03-24T11:09:35.286425Z","iopub.status.idle":"2023-03-24T11:09:37.685328Z","shell.execute_reply.started":"2023-03-24T11:09:35.286396Z","shell.execute_reply":"2023-03-24T11:09:37.684413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_df2 = session_df1.groupby(\"level_group\")[['fqid', 'room_fqid', 'text_fqid']].quantile(0.9).reset_index()\nfig, ax=plt.subplots(1, 3, sharey=True, figsize=(12, 5))\n\nfig.suptitle(\"Number of Events missed on an average per level group\")\nfor k, colname in enumerate(['fqid', 'room_fqid', 'text_fqid']):\n    tmp_df = session_df2.merge(df)\n    tmp_df[colname] = (tmp_df[\"total_\"+colname]-tmp_df[colname])\n    ax[k].set_title(colname)\n    sns.barplot(data=tmp_df, x='level_group', y=colname, ax=ax[k], order=['0-4', '5-12', '13-22'])\n    \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-24T11:09:37.686423Z","iopub.execute_input":"2023-03-24T11:09:37.686854Z","iopub.status.idle":"2023-03-24T11:09:38.004930Z","shell.execute_reply.started":"2023-03-24T11:09:37.686822Z","shell.execute_reply":"2023-03-24T11:09:38.004070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1. not all the events are observed by the user.\n2. 90% of the users played max around 60% of (text_fqid, fqids)\n3. ","metadata":{}},{"cell_type":"code","source":"fqid_df = train_df.groupby(['level_group','fqid'])[['session_id']].agg(['count', 'nunique'])\ntext_fqid_df = train_df.groupby(['level_group','text_fqid'])[['session_id']].agg(['count', 'nunique'])\n\nfqid_df.columns=['noccurences', 'nsessions']\ntext_fqid_df.columns=['noccurences', 'nsessions']\n\nfqid_df = fqid_df.reset_index()\ntext_fqid_df = text_fqid_df.reset_index()\n\ntext_fqid_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-24T11:09:38.007408Z","iopub.execute_input":"2023-03-24T11:09:38.008072Z","iopub.status.idle":"2023-03-24T11:09:49.281111Z","shell.execute_reply.started":"2023-03-24T11:09:38.008038Z","shell.execute_reply":"2023-03-24T11:09:49.279118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nsessions = train_df.session_id.nunique()\nfig, ax = plt.subplots(2, 3, figsize=(12, 7))\n\nfig.suptitle(\"Events (vs) Proportion of sessions covered\")\nfor i, l in enumerate(['0-4', '5-12', '13-22']):\n    tmp_df = text_fqid_df[text_fqid_df.level_group==l]\n    tmp_df2 = fqid_df[fqid_df.level_group==l]\n    \n    nusers_arr = tmp_df.nsessions.sort_values().values/nsessions\n    nusers_arr2 = tmp_df2.nsessions.sort_values().values/nsessions\n    \n    ax[0, i].plot(nusers_arr)\n    ax[1, i].plot(nusers_arr2)\n    \n    ax[0, i].set_title(l+\":text fqid\")\n    ax[1, i].set_title(l+\":fqid\")\n    \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-24T11:09:49.283445Z","iopub.execute_input":"2023-03-24T11:09:49.283829Z","iopub.status.idle":"2023-03-24T11:09:49.936265Z","shell.execute_reply.started":"2023-03-24T11:09:49.283793Z","shell.execute_reply":"2023-03-24T11:09:49.935442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1. Not all the events occurred during every session\n2. few sessions occured very frequently than the others -> these could be the introductory once.\n3. few sessions are very rare.\n\n\n**Is there any event that is disjoint to any other event for each level?**","metadata":{}},{"cell_type":"code","source":"events_df[events_df.text_fqid == 'tunic.historicalsociety.basement.janitor']","metadata":{"execution":{"iopub.status.busy":"2023-03-24T11:09:49.937200Z","iopub.execute_input":"2023-03-24T11:09:49.937474Z","iopub.status.idle":"2023-03-24T11:09:49.956267Z","shell.execute_reply.started":"2023-03-24T11:09:49.937446Z","shell.execute_reply":"2023-03-24T11:09:49.955472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, l in enumerate(['0-4', '5-12', '13-22']):\n    tmp_df = text_fqid_df[text_fqid_df.level_group==l]\n    tmp_df = tmp_df.merge(events_df[['level_group', 'text_fqid', 'text']]).sort_values(\"nsessions\")\n    \n    top_10 = tmp_df.text.values[-10:]\n    last_10 = tmp_df.text.values[:10]\n    \n    print(\"============== level:{} ====================\".format(l))\n    print(\"********** frequent 10 text **************\")\n    print()\n    for t in top_10:\n        print(t)\n        print()\n    \n    print()\n    print(\"********** rare 10 text **************\")\n    for t in last_10:\n        print(t)\n        print()\n    print()","metadata":{"execution":{"iopub.status.busy":"2023-03-24T11:09:49.958204Z","iopub.execute_input":"2023-03-24T11:09:49.959310Z","iopub.status.idle":"2023-03-24T11:09:49.981859Z","shell.execute_reply.started":"2023-03-24T11:09:49.959264Z","shell.execute_reply":"2023-03-24T11:09:49.981143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}