{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-13T14:39:49.075505Z","iopub.execute_input":"2023-04-13T14:39:49.076342Z","iopub.status.idle":"2023-04-13T14:39:49.103170Z","shell.execute_reply.started":"2023-04-13T14:39:49.076292Z","shell.execute_reply":"2023-04-13T14:39:49.102044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport scipy.stats as stats\n","metadata":{"execution":{"iopub.status.busy":"2023-04-16T14:38:31.181061Z","iopub.execute_input":"2023-04-16T14:38:31.181416Z","iopub.status.idle":"2023-04-16T14:38:31.185978Z","shell.execute_reply.started":"2023-04-16T14:38:31.181359Z","shell.execute_reply":"2023-04-16T14:38:31.185264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 首先处理一下train_labels.csv\nlabels = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\n# 分割\nlabels[['session_id', 'question']] = labels['session_id'].str.split('_', expand=True)\n\n# 需要正确排序\nlabels['question'] = labels['question'].str.slice(1)\nlabels['question'] = labels['question'].astype(str)\n\nlabels.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-16T14:38:33.381567Z","iopub.execute_input":"2023-04-16T14:38:33.382162Z","iopub.status.idle":"2023-04-16T14:38:35.066723Z","shell.execute_reply.started":"2023-04-16T14:38:33.382127Z","shell.execute_reply":"2023-04-16T14:38:35.065498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 下面处理train.csv\n# 首先参考其他笔记本读入，后续可以参考其他方法优化读取数据\ndtypes={\n    'elapsed_time':np.int32,\n    'event_name':'category',\n    'name':'category',\n    'level':np.uint8,\n    'room_coor_x':np.float32,\n    'room_coor_y':np.float32,\n    'screen_coor_x':np.float32,\n    'screen_coor_y':np.float32,\n    'hover_duration':np.float32,\n    'text':'category',\n    'fqid':'category',\n    'room_fqid':'category',\n    'text_fqid':'category',\n    'fullscreen':'category',\n    'hq':'category',\n    'music':'category',\n    'level_group':'category'}\n\ndataset_df = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv', dtype=dtypes)\nprint(\"Full train dataset shape is {}\".format(dataset_df.shape))","metadata":{"execution":{"iopub.status.busy":"2023-04-16T14:42:14.428858Z","iopub.execute_input":"2023-04-16T14:42:14.429258Z","iopub.status.idle":"2023-04-16T14:43:18.843870Z","shell.execute_reply.started":"2023-04-16T14:42:14.429222Z","shell.execute_reply":"2023-04-16T14:43:18.842935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 查看train.csv的前几行\n# 此处我理解的train.csv如下：\n# 同一个session_id有多个取值，就是按照index去记录读取该名同学在屏幕上的操作,\n# 也即对某一行而言，依次是：id，第几次记录，从发现到开始记录所需要的时间，本次记录的事件类型的名字，本次记录事件的名字，这个事件的级别，页码，。。\n# 那么对于某一个id，我们就需要进行分类聚合，要根据level_group进行一个划分\n# 同时我们也发现，当level_group改变的时候，index也进行了突变\ndataset_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-16T14:43:37.014492Z","iopub.execute_input":"2023-04-16T14:43:37.015451Z","iopub.status.idle":"2023-04-16T14:43:37.045781Z","shell.execute_reply.started":"2023-04-16T14:43:37.015413Z","shell.execute_reply":"2023-04-16T14:43:37.044464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"","metadata":{"execution":{"iopub.status.busy":"2023-04-16T15:02:36.844964Z","iopub.execute_input":"2023-04-16T15:02:36.845364Z","iopub.status.idle":"2023-04-16T15:02:36.922142Z","shell.execute_reply.started":"2023-04-16T15:02:36.845326Z","shell.execute_reply":"2023-04-16T15:02:36.921289Z"}}},{"cell_type":"code","source":"dataset_df['text_fqid'].unique()","metadata":{"execution":{"iopub.status.busy":"2023-04-16T15:09:47.614727Z","iopub.execute_input":"2023-04-16T15:09:47.615462Z","iopub.status.idle":"2023-04-16T15:09:47.707043Z","shell.execute_reply.started":"2023-04-16T15:09:47.615422Z","shell.execute_reply":"2023-04-16T15:09:47.705909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# 所以我想到的是这样子：后期预测一下，看看再筛选掉一部分\n# 对于同一个id，同一个level_group,\n# index可以直接舍弃，\n# elapsed我觉得可以计算保留中位数，最大最小值，众数，均值，方差，\n# event_name只有11种，可以直接赋值，也可以考虑独热\n# name只有6种，可以直接赋值，也可以独热\n# level暂时存疑，因为我们选择的是level_group，此处暂定\n# page整体占比很少，但是数据量在十万级别，如何处理需要讨论\n# room_coor_x和room_coor_y是游戏内被点击到的坐标，\n# screen_coor_x和screen_coor_y是屏幕上被点击的坐标，这两个如何处理需要讨论\n# hover_duration是悬停事件，整体占比也很少，但是数据量仍在万级别，如何处理需要讨论\n# text有597种，或许可以直接舍弃，\n# fqid有129种\n# room_fqid有19种\n# text_fqid有127种\n# fullscreen、hq、music似乎是一致变化的，可以不用处理\n# 对于这些类别型变量，我们编码后。聚合成一个数字该怎么办？","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-04-13T14:39:51.423325Z","iopub.execute_input":"2023-04-13T14:39:51.423740Z","iopub.status.idle":"2023-04-13T14:39:51.810125Z","shell.execute_reply.started":"2023-04-13T14:39:51.423703Z","shell.execute_reply":"2023-04-13T14:39:51.808778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-04-13T14:42:42.744074Z","iopub.execute_input":"2023-04-13T14:42:42.745146Z","iopub.status.idle":"2023-04-13T14:42:43.072900Z","shell.execute_reply.started":"2023-04-13T14:42:42.745080Z","shell.execute_reply":"2023-04-13T14:42:43.071304Z"},"trusted":true},"execution_count":null,"outputs":[]}]}