{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 参考：\n### [代码参考资料一](https://www.kaggle.com/code/vadimkamaev/catboost)\n### [代码参考资料二](https://www.kaggle.com/code/zhangyue325/eda-xgboost-and-lgbm-baseline-ongoing)\n### [代码参考资料三](https://www.kaggle.com/code/cdeotte/xgboost-baseline-0-680#Compute-CV-Score)\n### [代码参考资料四](https://www.kaggle.com/code/leehomhuang/lb0-691-catboostbaseline-train)\n## [pandas及Groupby的内容教学](https://github.com/datawhalechina/joyful-pandas)","metadata":{"_uuid":"af67ebe1-7f6f-4ebb-a6ce-09d34357f5c6","_cell_guid":"5b6d7470-9551-4e24-a442-91995d3cd53e","trusted":true}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport warnings\nimport matplotlib.pyplot as plt\nimport matplotlib\nimport seaborn as sns\nimport missingno as msno # 缺失值\nfrom plotly.subplots import make_subplots\nimport plotly.graph_objects as go\nfrom tqdm import tqdm\n\ntheme_colors = ['#901272', '#94A832', '#6C7CD3', '#F16262', '#4c5760']\n%matplotlib inline\n\nwarnings.filterwarnings('ignore')\nplt.rcParams['font.sans-serif'] = ['SimHei'] # 设置字体\nplt.rcParams['axes.unicode_minus'] = False # 该语句解决图像中的'-'的乱码问题\nsns.set_style('whitegrid', {'font.sans-serif':['simhei', 'FangSong']})","metadata":{"_uuid":"066aaaac-6953-44c1-8e1d-5b3caa417749","_cell_guid":"6ab0dd81-73b6-4a1e-91a6-8845a30bde71","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:12:57.184697Z","iopub.execute_input":"2023-04-29T16:12:57.185125Z","iopub.status.idle":"2023-04-29T16:12:58.205098Z","shell.execute_reply.started":"2023-04-29T16:12:57.185078Z","shell.execute_reply":"2023-04-29T16:12:58.203908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 读取数据集\n### [category数据类型介绍](https://www.cnblogs.com/wqbin/p/12786217.html)","metadata":{"_uuid":"5f19d2ac-0348-44b9-b4e7-01751b91b315","_cell_guid":"006b43c4-0142-4ee3-89d2-41a7544a1676","trusted":true}},{"cell_type":"code","source":"def read_dataset():\n    dtypes = {\"session_id\": 'int64',\n              \"index\": np.int16,\n              \"elapsed_time\": np.int32,\n              \"event_name\": 'category',\n              \"name\": 'category',\n              \"level\": np.int8,\n              \"page\": np.float16,\n              \"room_coor_x\": np.float16,\n              \"room_coor_y\": np.float16,\n              \"screen_coor_x\": np.float16,\n              \"screen_coor_y\": np.float16,\n              \"hover_duration\": np.float32,\n              \"text\": 'category',\n              \"fqid\": 'category',\n              \"room_fqid\": 'category',\n              \"text_fqid\": 'category',\n              \"fullscreen\": np.int8,\n              \"hq\": np.int8,\n              \"music\": np.int8,\n              \"level_group\": 'category'\n              }\n    use_col = ['session_id', 'index', 'elapsed_time', 'event_name', 'name', 'level', 'page',\n               'room_coor_x', 'room_coor_y', 'screen_coor_x', 'screen_coor_y','hover_duration', 'text', 'fqid', 'room_fqid',\n               'text_fqid', 'fullscreen', 'hq', 'music', 'level_group']\n\n    train = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train.csv\", dtype=dtypes, usecols=use_col)\n    # 4G的数据如果合理的分配数据类型可以节省很多memory,这里只占用了2.4G\n    display(train.head(25))\n    \n    train_label = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train_labels.csv\")\n    display(train_label)\n    \n    test = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/test.csv', dtype=dtypes, usecols=use_col)\n    display(test.head(3))\n    \n    return train, train_label,test\n\n\ntrain_df, train_label_df, test_df = read_dataset()","metadata":{"_uuid":"1d6f0480-e788-464b-ace2-83a579dbf955","_cell_guid":"66a49afe-e8ae-4d20-b992-72ba0760242b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:12:58.207086Z","iopub.execute_input":"2023-04-29T16:12:58.207696Z","iopub.status.idle":"2023-04-29T16:15:11.671363Z","shell.execute_reply.started":"2023-04-29T16:12:58.207658Z","shell.execute_reply":"2023-04-29T16:15:11.669551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{"_uuid":"c17feae1-8034-4365-a317-4ba703e3bfae","_cell_guid":"a24cc37c-4f56-498b-9e55-44e0826de1cc","trusted":true}},{"cell_type":"markdown","source":"# 查看数据的缺失值和相关性","metadata":{"_uuid":"4e1e5c63-b150-4558-9fb4-7e4877777ce5","_cell_guid":"06f98e73-07ac-4fd2-bfff-23bbf6782fcc","trusted":true}},{"cell_type":"code","source":"train_missing = train_df.isna().sum() / len(train_df) * 100 # 缺失值占比\ntrain_missing_index = train_missing.index\ntrain_missing_values = train_missing.values\n\nfig,ax = plt.subplots(figsize=(18,9))\nbarchart = sns.barplot(x = train_missing_index, y = train_missing_values, ax=ax)\nbarchart.axes.set_title(\"训练数据的缺失值\", fontsize=24, loc='center')\nbarchart.bar_label(barchart.containers[0], fmt='%.1f%%', fontsize=18)\nbarchart.yaxis.set_tick_params(labelsize = 18) # 设置坐标轴的刻度和网格样式\nbarchart.xaxis.set_tick_params(rotation=40, labelsize = 16) # 也可以设置x轴索引的倾斜度\nplt.show()\n\ndel train_missing, train_missing_index, train_missing_values, fig, ax, barchart","metadata":{"_uuid":"83c8f3e1-db0f-4dc7-a9b6-228afd37c970","_cell_guid":"25ca5073-72ca-46b6-b7b5-91cdc8f2017a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:15:11.675858Z","iopub.execute_input":"2023-04-29T16:15:11.676221Z","iopub.status.idle":"2023-04-29T16:15:13.359691Z","shell.execute_reply.started":"2023-04-29T16:15:11.676185Z","shell.execute_reply":"2023-04-29T16:15:13.358212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(18,9))\ng = sns.heatmap(train_df.corr(), annot=True, square=True, cmap='coolwarm', annot_kws={'size':15}, fmt='.2f')\ng.tick_params(axis='x', labelsize=15)\ng.tick_params(axis='y', labelsize=15)\ng.set_title('训练集中的相关系数', size=20, pad=15)\nplt.show()","metadata":{"_uuid":"e3e9f30f-75e1-42a7-bad4-cc75480d109c","_cell_guid":"1a3eb0cd-8cd9-4fca-9955-b9251bff047d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:15:13.362228Z","iopub.execute_input":"2023-04-29T16:15:13.362595Z","iopub.status.idle":"2023-04-29T16:15:27.932729Z","shell.execute_reply.started":"2023-04-29T16:15:13.362551Z","shell.execute_reply":"2023-04-29T16:15:27.931549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 检查Session_id","metadata":{"_uuid":"298adb20-2f27-4bb3-b56e-b1743c274ad9","_cell_guid":"c430e346-8f97-4349-8493-04cf94fe100c","trusted":true}},{"cell_type":"code","source":"train_events_counts = train_df['session_id'].value_counts()\ntest_events_counts = test_df['session_id'].value_counts()\n\nprint(f'\\n 训练集中独特session_id数量: {len(train_events_counts)}')\nprint(f'\\n 测试集中独特session_id数量: {len(test_events_counts)}')\n\ndata = {\n    \"INDEX\": [\"独特session_id数量\", \"最少事件数目\", \"最大事件数目\"],\n    \"TRAIN\": [str(train_df['session_id'].nunique()),\n             str(train_events_counts.min()),\n             str(train_events_counts.max())],\n    \"TEST\": [str(test_df['session_id'].nunique()),\n            str(test_events_counts.min()),\n            str(test_events_counts.max())]\n}\n\ndel train_events_counts, test_events_counts\n\ndf = pd.DataFrame(data).set_index('INDEX')\ndf","metadata":{"_uuid":"6b07c796-6be2-4475-aa77-9754b0491855","_cell_guid":"ff78dc9e-f5ed-4dea-bc9f-ac9cdc136460","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:15:27.933962Z","iopub.execute_input":"2023-04-29T16:15:27.935102Z","iopub.status.idle":"2023-04-29T16:15:28.451140Z","shell.execute_reply.started":"2023-04-29T16:15:27.935005Z","shell.execute_reply":"2023-04-29T16:15:28.450203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* 可以看出一个session导致的事件最少都有几百次，最大的有上万次且现有的test数据只是最终测试的一半\n* 1个session可能蕴含着多个问题级别的事件event，每个问题级别的事件event的数量可能不一样，有可能在一个问题中的行动次数可能与正确或错误的答案相关，因此后续会对这个数据进一步检查\n# 检查index\n- index是每个session的event索引\n- 训练数据集和测试数据集的指数不是很干净\n- 1.出现重复的（session，index）  \n  2.一些session没有索引=0，例如session_id == '200508425704692'","metadata":{"_uuid":"ab620125-89d9-474c-ab50-6a5cf93c21f7","_cell_guid":"e3d10328-1005-468b-91a7-13c918b0c4fc","trusted":true}},{"cell_type":"code","source":"pk_tmp = [\"session_id\", \"index\"]\n# session_id和index两列的唯一值的session_id中的唯一值转化成列表list\nsess_dup = train_df.loc[train_df.duplicated(subset=pk_tmp, keep=False)]['session_id'].unique().tolist()\ntrain_dup = train_df.loc[train_df['session_id'].isin(sess_dup)].reset_index(drop=True) # 默认整数索引\nprint(f\"重复的('session', 'index') pairs: {len(sess_dup)}\")\n\ndel pk_tmp, sess_dup","metadata":{"_uuid":"75ce8045-db26-4009-a560-ae24b9cd8548","_cell_guid":"5d5dbeef-bdf6-4786-8e15-1ba48eca5b37","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:15:28.452280Z","iopub.execute_input":"2023-04-29T16:15:28.453378Z","iopub.status.idle":"2023-04-29T16:15:38.029636Z","shell.execute_reply.started":"2023-04-29T16:15:28.453334Z","shell.execute_reply":"2023-04-29T16:15:38.028486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**如果训练集train_df使用筛选index后的数据那么最后对于某一个level_group中的数据就会很少，而分到每一个session_id的数量就更少了，最主要的是这个index的筛选能力太强了，只要有重复的二元组的session_id就删掉**","metadata":{"_uuid":"703a7dfc-61ac-4611-ad5a-3e9f7f69bffa","_cell_guid":"4e7f21c5-53ee-4e3a-9d82-5ce6090add64","trusted":true}},{"cell_type":"code","source":"# train_df = train_dup\ndel train_dup","metadata":{"_uuid":"cf9acb22-7369-42d6-95d8-daf0a7737538","_cell_guid":"34ccbc4b-01e4-475e-b765-72c88693fcfd","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:15:38.030856Z","iopub.execute_input":"2023-04-29T16:15:38.031679Z","iopub.status.idle":"2023-04-29T16:15:38.036381Z","shell.execute_reply.started":"2023-04-29T16:15:38.031640Z","shell.execute_reply":"2023-04-29T16:15:38.035208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 检查elapsed_time\nelapsed_time是指从会话开始到事件被记录之间已经过去了多少时间（以毫秒计）。","metadata":{"_uuid":"5e94cabb-a917-40c8-b9ff-ae82800dcfbe","_cell_guid":"51ed0303-ebbd-4c5f-9e68-e03f68fc35d0","trusted":true}},{"cell_type":"code","source":"elapsed_time_train = np.round((train_df['elapsed_time']).astype(np.float64)/60000.0, 1)\nelapsed_time_test = np.round(test_df['elapsed_time'].astype(np.float64)/60000.0, 1)\n\nstat = pd.DataFrame([elapsed_time_train.describe().index,\n                    np.round(elapsed_time_train.describe(), 2).values,\n                    np.round(elapsed_time_test.describe(), 2).values]).T\n\nstat.columns = [' ', 'train', 'test']\nstat[['train', 'test']] = stat[['train', 'test']].astype(np.float64)\nprint('elapsed_time列的统计数据: ')\ndisplay(stat[1:].style.hide_index().format(precision=1).background_gradient() )\n\n# 隐藏行索引\ndel elapsed_time_train, elapsed_time_test, stat","metadata":{"_uuid":"2114e70e-0442-436d-a60b-cb2d0fed5f53","_cell_guid":"59498075-fdd0-4e2b-89c1-3bc6954405eb","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:15:38.037767Z","iopub.execute_input":"2023-04-29T16:15:38.038169Z","iopub.status.idle":"2023-04-29T16:15:40.230487Z","shell.execute_reply.started":"2023-04-29T16:15:38.038134Z","shell.execute_reply":"2023-04-29T16:15:40.229511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- elapsed_time应该与index相关联，随着时间的推移，预计会有更多的事件发生","metadata":{"_uuid":"fed2efa9-e6d3-4781-b402-3caffb7ba089","_cell_guid":"5c6e7d0b-6e9a-456c-94bd-ff1294e6bb6f","trusted":true}},{"cell_type":"code","source":"avg_elapsed_time = train_df.groupby('index')['elapsed_time'].mean() / 1000\n\nfig, ax = plt.subplots(figsize=(10,4))\nax.plot(avg_elapsed_time)\nax.axvline(2825, color='red', ls='--')\nplt.xlabel('Event index', fontsize=12)\nplt.ylabel('平均 elapsed time(s)', fontsize=12)\nplt.xlim([0,5000])\nplt.legend(['平均 elapsed time', '一般性规律处'])\n\nplt.show()\n\ndel avg_elapsed_time","metadata":{"_uuid":"44c131e2-8b46-412a-a922-707d5dea9cc6","_cell_guid":"dafe37e6-6454-40af-b53e-647bb79541b8","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:15:40.231877Z","iopub.execute_input":"2023-04-29T16:15:40.232282Z","iopub.status.idle":"2023-04-29T16:15:41.451181Z","shell.execute_reply.started":"2023-04-29T16:15:40.232250Z","shell.execute_reply":"2023-04-29T16:15:41.449949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"正如预期的那样，平均elapsed time随着index的增加而增加，但这种趋势只保持到指数2800左右，在那之后，行为变得不稳定，有一个高峰，然后下降，直到很晚才保持不变。  \n\n这种现象可以解释为随着事件数量的增加，有价值的例子越来越少，平均数开始代表独特的案例，这些案例很可能是离群值，这种离群值可能包括在较短的时间内进行垃圾点击以获得高数量的事件的个人，或者在锋值得情况下，不活跃的个人在很长得一段时间内，剔除那些个人长时间不活动的时段可能是有益的","metadata":{"_uuid":"3b457008-038f-4e00-85e4-64986e3d68ac","_cell_guid":"80652e90-6b12-4a67-8422-ecb2a2359b15","trusted":true}},{"cell_type":"markdown","source":"# 检查event_name和name列\n\nevent_name是事件的名称，他有11个唯一得值（包括cuscene_click, person_click, navigate_click, notification_click, object_click, object_hover, map_hover, map_click, checkpoint and notebook_click）\n\nname是事件类型的名称，他有6个值（包括basic，underfined，close，open，prev，and next）","metadata":{"_uuid":"bed8d33f-da41-4c44-9fef-acd3a9f7bb75","_cell_guid":"ccf09bda-2a8a-45a8-9c29-3a3ec3f0eb5e","trusted":true}},{"cell_type":"code","source":"fig, ax = plt.subplots(1,2,figsize=(16,6))\nevent_name = train_df['event_name'].value_counts()\nax[0].bar(event_name.index, event_name.values, edgecolor='black', color='#6495ED')\nax[0].tick_params(rotation=35)\nax[0].set_title('event_name 频率', fontsize=20)\nax[0].set_ylabel('count of event', fontsize=16)\n\nname = train_df['name'].value_counts()\nax[1].bar(name.index, name.values, edgecolor='black', color='#9FE2BF')\nax[1].tick_params(rotation=35)\nax[1].set_title('name 的频率', fontsize=20)\nax[1].set_ylabel('count of name', fontsize=16)\n\nplt.show()\n\ndel event_name, name","metadata":{"_uuid":"65876542-c66f-4f2f-b547-e8fffafe1913","_cell_guid":"b7003b03-026f-4b61-bd2c-36553d0f646e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:15:41.455378Z","iopub.execute_input":"2023-04-29T16:15:41.455767Z","iopub.status.idle":"2023-04-29T16:15:42.261775Z","shell.execute_reply.started":"2023-04-29T16:15:41.455731Z","shell.execute_reply":"2023-04-29T16:15:42.260779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pivot = train_df.pivot_table(index='name', columns='event_name', aggfunc='size')\npivot = (pivot.fillna(0) / 1000).round(decimals = 1)\nplt.figure(figsize=(18, 9))\nannotations = pivot.astype(str)\nannotations[ pivot == 0 ] = '0'\ng = sns.heatmap(pivot, annot=annotations, fmt='', cmap='GnBu')\nplt.title(\"event_name and name happend count\", fontsize=22)\nplt.xlabel('name', fontsize=15)\nplt.ylabel('event_name', fontsize=18)\nplt.show()\n\ndel pivot, g","metadata":{"_uuid":"e0ea4298-d053-4df6-98fe-0c52da2ac97f","_cell_guid":"79a1e495-b5fb-49b5-9a8c-c16e0bb7bf4e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:15:42.262936Z","iopub.execute_input":"2023-04-29T16:15:42.263864Z","iopub.status.idle":"2023-04-29T16:15:43.860115Z","shell.execute_reply.started":"2023-04-29T16:15:42.263828Z","shell.execute_reply":"2023-04-29T16:15:43.859227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 检查了测试集的分布，它与训练集的分布相当\n- 某些name只与特定的event_name一起出现，不是所有的组合都是可能的","metadata":{"_uuid":"a1d70dcd-a5bd-4bfe-bfbb-bc83def26c38","_cell_guid":"85b39587-706c-41d4-ba48-aba6fa24bae5","trusted":true}},{"cell_type":"markdown","source":"# 检查level列","metadata":{"_uuid":"f3f4d08a-b87d-4417-9d10-a73d71858a54","_cell_guid":"81abd835-3c4e-402c-9a2f-a921420178ef","trusted":true}},{"cell_type":"code","source":"# 问题级别level和相应会话（session）中的事件数（index_count）之间的关系\ngrouped_df = train_df.groupby(['session_id', 'level'])['index'].count().reset_index()\ngrouped_df.columns = ['session_id', 'level', 'index_count']\nmean_counts = grouped_df.groupby('level')['index_count'].mean()\ndisplay(round(mean_counts))\n\nxrange = range(0,23)\nplt.figure(figsize=(10,4))\nplt.plot(mean_counts.values)\nplt.scatter(xrange, mean_counts, color='black')\nplt.xticks(ticks=xrange)\nplt.title('level and mean event counts', fontsize=16)\nplt.xlabel('Level', fontsize=12)\nplt.ylabel('mean event  counts', fontsize=12)\nplt.show()\n\ndel grouped_df, mean_counts, xrange","metadata":{"_uuid":"117a4c60-3262-4bd9-9dc5-bec6066fe19b","_cell_guid":"799e94d9-b6c4-45a7-8284-ce20b81f506a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:15:43.861308Z","iopub.execute_input":"2023-04-29T16:15:43.861901Z","iopub.status.idle":"2023-04-29T16:15:45.745511Z","shell.execute_reply.started":"2023-04-29T16:15:43.861863Z","shell.execute_reply":"2023-04-29T16:15:45.744478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"很明显18级的事件活动水平明显提高了","metadata":{"_uuid":"da88d556-ce01-498d-8b88-3a1c89d4c1ce","_cell_guid":"dd2f2335-7a14-489a-b7b8-6337e10ba34f","trusted":true}},{"cell_type":"markdown","source":"# 检查page列","metadata":{"_uuid":"388a704d-fd7a-4a32-8190-836aacd9cb62","_cell_guid":"c7c51708-6a5f-4087-862b-d3666d3842f9","trusted":true}},{"cell_type":"code","source":"page = train_df[ ~train_df['page'].isna() ]\n\nprint(f\"\\nonly {len(page)} of `page` 在训练集中不是 NaN, 占训练集的 {len(page)/len(train_df)*100:.1f}% \")\n\npage_piv = page.groupby('page')['session_id'].count()\n\nplt.figure(figsize=(10,4))\nplt.bar(page_piv.index, page_piv.values, edgecolor='black', color='#9FE2BF')\nplt.title('count of page in train ', fontsize=22)\nplt.xlabel('page', fontsize=18)\nplt.ylabel('event num', fontsize=18)\nplt.show()\n\ndel page, page_piv","metadata":{"_uuid":"7d75473d-010b-402d-8b45-48c07406b810","_cell_guid":"b16c5f67-e843-4092-ab0c-9c3d8fdb0882","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:15:45.746746Z","iopub.execute_input":"2023-04-29T16:15:45.747702Z","iopub.status.idle":"2023-04-29T16:15:46.197586Z","shell.execute_reply.started":"2023-04-29T16:15:45.747656Z","shell.execute_reply":"2023-04-29T16:15:46.196462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 在这一列中有近乎98%的缺失值，因为只有当事件于笔记本有关时，才会显示一个页面","metadata":{"_uuid":"5eccec4b-a7ee-4544-aaa9-8e836c3fbe4a","_cell_guid":"ce7b9715-960b-491b-a47f-47f37e344581","trusted":true}},{"cell_type":"code","source":"\nnew_train = train_df[~train_df['page'].isna()]\nnew_train = new_train[~train_df['level'].isna()]\n\nlp_train = new_train.groupby('level')['page'].count()\n\ndisplay(lp_train.index)\n\nplt.figure(figsize=(10,4))\nplt.bar(lp_train.index, lp_train.values, tick_label = lp_train.index, edgecolor='black', color = '#9FE2BF')\nplt.title('count of page event in different level', fontsize=22)\nplt.xlabel('level', fontsize=18)\nplt.ylabel('count of page', fontsize=18)\nplt.show()\n\ndel new_train","metadata":{"_uuid":"e109a750-bda8-4f1b-85fb-cf4b14c33020","_cell_guid":"a11b4d2c-ec0f-4979-b0a0-792eddc4cd3c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:15:46.198910Z","iopub.execute_input":"2023-04-29T16:15:46.199739Z","iopub.status.idle":"2023-04-29T16:15:46.819643Z","shell.execute_reply.started":"2023-04-29T16:15:46.199695Z","shell.execute_reply":"2023-04-29T16:15:46.818402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#  检查Geo-Location Properties\n\n- room_coor_x, room_coor_y是点击的坐标，参与游戏中的房间（仅适用于点击事件）\n- screen_coor_x, screen_coor_y是点击的坐标，参考玩家的屏幕","metadata":{"_uuid":"0d383dc9-c3b3-4b15-baeb-a7bdb1e08912","_cell_guid":"d594db9a-1866-4c94-99dc-0c4dbf0b4f96","trusted":true}},{"cell_type":"code","source":"r_coor = train_df[ ~train_df['room_coor_x'].isna() ]\nprint(f\"训练数据中的位置数据有{len(r_coor)}个不是空的，占总训练集的{len(r_coor)/len(train_df)*100:.1f}%。\")\n\ndel r_coor","metadata":{"_uuid":"5f80c2a9-6217-4058-a958-fac9c438589a","_cell_guid":"66474ce0-4a9d-40df-8431-27004d514d0e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:15:46.821073Z","iopub.execute_input":"2023-04-29T16:15:46.821501Z","iopub.status.idle":"2023-04-29T16:15:49.090277Z","shell.execute_reply.started":"2023-04-29T16:15:46.821466Z","shell.execute_reply":"2023-04-29T16:15:49.089203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 检查hover_duration列\n\n- hover_duration是悬停发生的时间\n\n- hover_duration适用于事件名称为object_hover或map_hover时","metadata":{"_uuid":"c2f99279-c4d0-46b7-a47f-d3584cbb70d1","_cell_guid":"7917f874-dd14-4b34-8c3b-5b5dd72a04f2","trusted":true}},{"cell_type":"code","source":"hover = train_df[~train_df['hover_duration'].isna()]\nprint(f\"训练数据中的hover数据有{len(hover)}个不是NAN，占总训练集的{len(hover)/len(hover)*100:.1f}%。\")\nobject_hover = hover[ hover['event_name'] == 'object_hover' ]\nmap_hover = hover[ hover['event_name'] == 'map_hover']\n\nprint(f\"训练数据中的object hover有{len(object_hover) / len(hover) * 100:.1f}，剩下的{100 - len(object_hover) / len(hover)*100:.1f}%的悬停事件是map_hover。\")\n\ndel hover","metadata":{"_uuid":"246f6f6d-39d7-482d-90c0-375deb6c484c","_cell_guid":"cc0d4d78-6096-484f-80bd-0bc3f82a67f2","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:15:49.091665Z","iopub.execute_input":"2023-04-29T16:15:49.092083Z","iopub.status.idle":"2023-04-29T16:15:49.526967Z","shell.execute_reply.started":"2023-04-29T16:15:49.092050Z","shell.execute_reply":"2023-04-29T16:15:49.525637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"object_hover = np.round((object_hover['hover_duration']).astype(np.float64)/1000.0, 1)\nmap_hover = np.round((map_hover['hover_duration']).astype(np.float64)/1000.0, 1)\n\nstat = pd.DataFrame([object_hover.describe().index,\n                    np.round(object_hover.describe(), 2).values,\n                    np.round(map_hover.describe(), 2).values]).T\n\n\nstat.columns = [' ', 'object hover', 'map hover']\nstat[['object hover', 'map hover']] = stat[['object hover', 'map hover']].astype(np.float64)\nprint(\"object_hover和'map_hover'的统计数据（以秒计）：\")\ndisplay(stat[1:].style.hide_index().format(precision=1).background_gradient())\n\ndel object_hover, map_hover, stat","metadata":{"_uuid":"4bb79fc2-3886-4538-83b9-6167b9b69e34","_cell_guid":"8cf866b5-09b2-4ebe-b97e-5da7f276c3c0","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:15:49.528666Z","iopub.execute_input":"2023-04-29T16:15:49.529219Z","iopub.status.idle":"2023-04-29T16:15:49.694417Z","shell.execute_reply.started":"2023-04-29T16:15:49.529181Z","shell.execute_reply":"2023-04-29T16:15:49.693526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"很明显，平均值远高于中位数说明存在离群值，即用户停留在悬停上的时间很长，这样的例子应该被清除\n\n# 检查text和text_fqid列\n\n- 游戏中有许多的短语比其他的更加常见\n- 有些文本时ASCII unicode hex格式，对这些文本处理提取额外的特征可能是有用的","metadata":{"_uuid":"2c4b7995-d8c5-4149-aaae-92dc03381e41","_cell_guid":"92a45cb3-424a-449b-b755-275d3775b899","trusted":true}},{"cell_type":"code","source":"print(f\"\\nIn train daset, {(~train_df['text'].isna()).sum()} `text` of events are not NaN.它占整个训练数据集的{100 - train_df['text'].isna().sum() / len(train_df) * 100:.1f}% \")\nprint(f\"In train daset, {(~train_df['text_fqid'].isna()).sum()} `text_fqid` of events are not NaN. 它占整个训练数据集的{100 - train_df['text_fqid'].isna().sum() / len(train_df) * 100:.1f}% of the whole train dataset.\")","metadata":{"_uuid":"7f849b68-4b46-4aa4-81fd-1b46717d840d","_cell_guid":"b27cd362-723d-422c-9aa7-da069188ce9e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:15:49.696085Z","iopub.execute_input":"2023-04-29T16:15:49.696450Z","iopub.status.idle":"2023-04-29T16:15:49.876466Z","shell.execute_reply.started":"2023-04-29T16:15:49.696406Z","shell.execute_reply":"2023-04-29T16:15:49.875274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 分组EDA","metadata":{"_uuid":"b8a0f3b3-b995-4939-8f85-48be4da45542","_cell_guid":"a7acf567-0294-4edd-87b5-622ebe337193","trusted":true}},{"cell_type":"code","source":"df_0_4 = train_df[train_df['level_group'] == '0-4']\ndf_5_12 = train_df[train_df['level_group'] == '5-12']\ndf_13_22 = train_df[train_df['level_group'] == '13-22']\ndel train_df # 分开处理每一个阶段的数据,删除原来的data防止内存占用","metadata":{"_uuid":"88cfffef-e45e-4dad-96fd-9d74e9ebb754","_cell_guid":"afa59f2b-9c3b-48d0-830d-5ed4638e6124","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:15:49.878162Z","iopub.execute_input":"2023-04-29T16:15:49.878532Z","iopub.status.idle":"2023-04-29T16:15:52.352727Z","shell.execute_reply.started":"2023-04-29T16:15:49.878496Z","shell.execute_reply":"2023-04-29T16:15:52.351506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef my_plot(train, i):\n    train_missing = train.isnull().sum() / len(train) * 100\n    train_missing_index = train_missing.index\n    train_missing_values = train_missing.values\n    \n    barchart = sns.barplot(x = train_missing_index, y = train_missing_values)\n    barchart.axes.set_title(f\"Share of missing values in train data{i}\", fontsize=18, loc = 'center')\n    barchart.bar_label(barchart.containers[0], fmt=\"%.1f%%\", fontsize=9)\n    barchart.yaxis.set_tick_params(labelsize = 9)\n    barchart.xaxis.set_tick_params(rotation=40, labelsize = 6)\n    plt.show()\n    del train_missing, train_missing_index, train_missing_values","metadata":{"_uuid":"7abfec2d-c96a-453e-ab25-0c2c2c3029a7","_cell_guid":"e122fd2a-b8b4-4724-9a52-b2e05c1b19c8","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:15:52.354321Z","iopub.execute_input":"2023-04-29T16:15:52.354673Z","iopub.status.idle":"2023-04-29T16:15:52.361662Z","shell.execute_reply.started":"2023-04-29T16:15:52.354640Z","shell.execute_reply":"2023-04-29T16:15:52.360500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# fig, ax = plt.subplots(1, figsize = (18,10), sharex=True, sharey=True)\nplt.figure( num=1, figsize=(10,4))\nmy_plot(df_0_4, 1)","metadata":{"_uuid":"33dc6330-8fce-49e2-9372-1b9af8232f9c","_cell_guid":"fe774d3f-30f1-4cab-bbc9-4a654fb25ff9","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:15:52.363119Z","iopub.execute_input":"2023-04-29T16:15:52.363462Z","iopub.status.idle":"2023-04-29T16:15:52.947034Z","shell.execute_reply.started":"2023-04-29T16:15:52.363416Z","shell.execute_reply":"2023-04-29T16:15:52.945304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(num=2, figsize=(10,4))\nmy_plot(df_5_12, 2)","metadata":{"_uuid":"6a348a56-4f19-4d44-bb89-e9d2ff144895","_cell_guid":"2053bfd1-18b4-4570-8659-e019b90760b4","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:15:52.948346Z","iopub.execute_input":"2023-04-29T16:15:52.948888Z","iopub.status.idle":"2023-04-29T16:15:53.739659Z","shell.execute_reply.started":"2023-04-29T16:15:52.948852Z","shell.execute_reply":"2023-04-29T16:15:53.738673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(num=3, figsize=(10,4))\nmy_plot(df_13_22, 3)","metadata":{"_uuid":"18505f96-feaf-4f3d-85bd-aa8c5b0d9d27","_cell_guid":"67720f8b-8559-47a2-bf3c-eb779f28e467","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:15:53.740961Z","iopub.execute_input":"2023-04-29T16:15:53.741533Z","iopub.status.idle":"2023-04-29T16:15:54.721715Z","shell.execute_reply.started":"2023-04-29T16:15:53.741498Z","shell.execute_reply":"2023-04-29T16:15:54.720532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def my_heatmap(train, i):\n    g = sns.heatmap(train.corr(), annot=True, square=True, cmap='coolwarm', annot_kws={'size':15}, fmt='.2f')\n    g.tick_params(axis='x', labelsize=15)\n    g.tick_params(axis='y', labelsize=15)\n    g.set_title(f'Correlations in train dataset{i}', size=20, pad=15)\n    plt.show()","metadata":{"_uuid":"0cd739db-fd13-48dd-99fe-0f7ec8bd0429","_cell_guid":"2e216bbc-3b43-47e7-9c87-c8cadb9fed19","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:15:54.722937Z","iopub.execute_input":"2023-04-29T16:15:54.724103Z","iopub.status.idle":"2023-04-29T16:15:54.730435Z","shell.execute_reply.started":"2023-04-29T16:15:54.724056Z","shell.execute_reply":"2023-04-29T16:15:54.729283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(num=4, figsize=(18,9))\nmy_heatmap(df_0_4, 1)","metadata":{"_uuid":"9eaa7083-8fbf-4aff-b9bc-bdbd85cdd778","_cell_guid":"c7d10cbf-8dfa-4e9d-bece-db14d65ab032","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:15:54.731958Z","iopub.execute_input":"2023-04-29T16:15:54.732558Z","iopub.status.idle":"2023-04-29T16:15:57.906479Z","shell.execute_reply.started":"2023-04-29T16:15:54.732520Z","shell.execute_reply":"2023-04-29T16:15:57.905317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(18,9))\nmy_heatmap(df_5_12, 2)","metadata":{"_uuid":"8d1ceb9c-4629-4696-b932-717c70021f8a","_cell_guid":"6b45baab-b783-4e6e-b175-7fcc17aa2316","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:15:57.907925Z","iopub.execute_input":"2023-04-29T16:15:57.908377Z","iopub.status.idle":"2023-04-29T16:16:03.460673Z","shell.execute_reply.started":"2023-04-29T16:15:57.908341Z","shell.execute_reply":"2023-04-29T16:16:03.459387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(18,9))\nmy_heatmap(df_13_22, 3)","metadata":{"_uuid":"f0f958d8-eb61-405b-8cff-6ad1ac04d75b","_cell_guid":"be167f2d-7f2c-45fd-bd92-de52c4a6e489","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:16:03.462300Z","iopub.execute_input":"2023-04-29T16:16:03.462676Z","iopub.status.idle":"2023-04-29T16:16:11.177850Z","shell.execute_reply.started":"2023-04-29T16:16:03.462640Z","shell.execute_reply":"2023-04-29T16:16:11.176567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 检查train_label","metadata":{"_uuid":"7b46df80-907e-47cf-a065-43bb111cb5ab","_cell_guid":"e45f2da1-0418-48ef-ad99-2ff0a22a1601","trusted":true}},{"cell_type":"code","source":"\ntrain_label_df['session'] = train_label_df.session_id.apply(lambda s: int(s.split('_')[0]))\ntrain_label_df['q'] = train_label_df.session_id.apply(lambda s: int(s.split('_')[1][1:]))\nprint('labels shape:',train_label_df.shape)\n\nfor t in range(1,19):\n    print(f\"quiz{t} labels shape is: {train_label_df[ train_label_df.q == t ].shape}\")\n\ntrain_label_df.head()","metadata":{"_uuid":"d34143b6-4646-4582-83b9-453826125977","_cell_guid":"8ef48ef2-61f5-4957-8a05-15940bf65b66","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:16:11.184547Z","iopub.execute_input":"2023-04-29T16:16:11.184946Z","iopub.status.idle":"2023-04-29T16:16:11.978227Z","shell.execute_reply.started":"2023-04-29T16:16:11.184909Z","shell.execute_reply":"2023-04-29T16:16:11.977315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_df = train_label_df['correct'].value_counts()\nplt.figure()\nplot_df.plot(kind='bar', color = ['g','c'])\nplt.show()\ndel plot_df","metadata":{"_uuid":"96b7dabc-bc1b-4fce-8f72-fa98b510c3e3","_cell_guid":"a10022a5-0bed-440c-8bda-f2db3341e545","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:16:11.979703Z","iopub.execute_input":"2023-04-29T16:16:11.980297Z","iopub.status.idle":"2023-04-29T16:16:12.186637Z","shell.execute_reply.started":"2023-04-29T16:16:11.980260Z","shell.execute_reply":"2023-04-29T16:16:12.185759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,20))\nplt.subplots_adjust(hspace=0.5, wspace=0.5)\nplt.suptitle(\"\\\"Correct\\\" coluimn values for each question\", fontsize=14, y=0.94)\n\nfor n in range(1,19):\n    ax = plt.subplot(6,3,n)\n    plot_df = train_label_df.loc[train_label_df.q == n]\n    plot_df = plot_df.correct.value_counts()\n    plot_df.plot(ax=ax, kind='bar', color=['b', 'c'])\n    \n    ax.set_title(\"Question \" + str(n))\n    ax.set_xlabel(\"\")","metadata":{"_uuid":"6041def9-f448-4e44-8fdc-b8c5829e0eae","_cell_guid":"2560afcf-c6cd-41aa-ae31-ca1d13664af1","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:16:12.187912Z","iopub.execute_input":"2023-04-29T16:16:12.188424Z","iopub.status.idle":"2023-04-29T16:16:14.700935Z","shell.execute_reply.started":"2023-04-29T16:16:12.188390Z","shell.execute_reply":"2023-04-29T16:16:14.700033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineer","metadata":{"_uuid":"534d6bd4-64f7-483d-aea3-4a6b270125be","_cell_guid":"135f0501-27a2-4633-aacd-03c40784296a","trusted":true}},{"cell_type":"markdown","source":"### [DataFrame.diff](https://blog.csdn.net/qq_39910261/article/details/106208165)\n### [DataFrame.clip](https://blog.csdn.net/Kwoky/article/details/117752236)","metadata":{"_uuid":"c244f14e-fa60-48c6-9df9-d745a96026d2","_cell_guid":"56d1c18e-77a1-4d7f-9a0d-f6717e68e1cf","trusted":true}},{"cell_type":"markdown","source":"## Data preprocessing","metadata":{"_uuid":"883be0cc-4d23-43a4-ba07-2cdcbc589f6c","_cell_guid":"1ed9e127-5f66-4238-80a9-794cc18c948b","trusted":true}},{"cell_type":"code","source":"# 每个会话事件的唯一ID\nfqid_lists = ['worker', 'archivist', 'gramps', 'wells', 'toentry', 'confrontation', 'crane_ranger', \n              'groupconvo', 'flag_girl', 'tomap', 'tostacks', 'tobasement', 'archivist_glasses', \n              'boss', 'journals', 'seescratches', 'groupconvo_flag', 'cs', 'teddy', 'expert', \n              'businesscards', 'ch3start', 'tunic.historicalsociety', 'tofrontdesk', 'savedteddy', \n              'plaque', 'glasses', 'tunic.drycleaner', 'reader_flag', 'tunic.library', 'tracks', \n              'tunic.capitol_2', 'trigger_scarf', 'reader', 'directory', 'tunic.capitol_1', \n              'journals.pic_0.next', 'unlockdoor', 'tunic', 'what_happened', 'tunic.kohlcenter', \n              'tunic.humanecology', 'colorbook', 'logbook', 'businesscards.card_0.next', \n              'journals.hub.topics', 'logbook.page.bingo', 'journals.pic_1.next', 'journals_flag', \n              'reader.paper0.next', 'tracks.hub.deer', 'reader_flag.paper0.next', 'trigger_coffee', \n              'wellsbadge', 'journals.pic_2.next', 'tomicrofiche', 'journals_flag.pic_0.bingo', \n              'plaque.face.date', 'notebook', 'tocloset_dirty', 'businesscards.card_bingo.bingo', \n              'businesscards.card_1.next', 'tunic.wildlife', 'tunic.hub.slip', 'tocage', 'journals.pic_2.bingo', \n              'tocollectionflag', 'tocollection', 'chap4_finale_c', 'chap2_finale_c', 'lockeddoor', \n              'journals_flag.hub.topics', 'tunic.capitol_0', 'reader_flag.paper2.bingo', 'photo', \n              'tunic.flaghouse', 'reader.paper1.next', 'directory.closeup.archivist', 'intro', \n              'businesscards.card_bingo.next', 'reader.paper2.bingo', 'retirement_letter', \n              'remove_cup', 'journals_flag.pic_0.next', 'magnify', 'coffee', 'key', 'togrampa', \n              'reader_flag.paper1.next', 'janitor', 'tohallway', 'chap1_finale', 'report', \n              'outtolunch', 'journals_flag.hub.topics_old', 'journals_flag.pic_1.next', \n              'reader.paper2.next', 'chap1_finale_c', 'reader_flag.paper2.next', 'door_block_talk', \n              'journals_flag.pic_1.bingo', 'journals_flag.pic_2.next', 'journals_flag.pic_2.bingo', \n              'block_magnify', 'reader.paper0.prev', 'block', 'reader_flag.paper0.prev', 'block_0', \n              'door_block_clean', 'reader.paper2.prev', 'reader.paper1.prev', 'doorblock', 'tocloset', \n              'reader_flag.paper2.prev', 'reader_flag.paper1.prev', 'block_tomap2', 'journals_flag.pic_0_old.next', \n              'journals_flag.pic_1_old.next', 'block_tocollection', 'block_nelson', 'journals_flag.pic_2_old.next', \n              'block_tomap1', 'block_badge', 'need_glasses', 'block_badge_2', 'fox', 'block_1']\n\n# 每个文本事件的唯一ID\ntext_fqid_lists = ['tunic.historicalsociety.cage.confrontation', 'tunic.wildlife.center.crane_ranger.crane', \n              'tunic.historicalsociety.frontdesk.archivist.newspaper', 'tunic.historicalsociety.entry.groupconvo', \n              'tunic.wildlife.center.wells.nodeer', 'tunic.historicalsociety.frontdesk.archivist.have_glass', \n              'tunic.drycleaner.frontdesk.worker.hub', 'tunic.historicalsociety.closet_dirty.gramps.news', \n              'tunic.humanecology.frontdesk.worker.intro', 'tunic.historicalsociety.frontdesk.archivist_glasses.confrontation', 'tunic.historicalsociety.basement.seescratches', 'tunic.historicalsociety.collection.cs', \n              'tunic.flaghouse.entry.flag_girl.hello', 'tunic.historicalsociety.collection.gramps.found', 'tunic.historicalsociety.basement.ch3start', 'tunic.historicalsociety.entry.groupconvo_flag', \n              'tunic.library.frontdesk.worker.hello', 'tunic.library.frontdesk.worker.wells', \n              'tunic.historicalsociety.collection_flag.gramps.flag', 'tunic.historicalsociety.basement.savedteddy', 'tunic.library.frontdesk.worker.nelson', 'tunic.wildlife.center.expert.removed_cup', 'tunic.library.frontdesk.worker.flag', \n              'tunic.historicalsociety.frontdesk.archivist.hello', 'tunic.historicalsociety.closet.gramps.intro_0_cs_0', 'tunic.historicalsociety.entry.boss.flag', 'tunic.flaghouse.entry.flag_girl.symbol', 'tunic.historicalsociety.closet_dirty.trigger_scarf', \n              'tunic.drycleaner.frontdesk.worker.done', 'tunic.historicalsociety.closet_dirty.what_happened', 'tunic.wildlife.center.wells.animals', 'tunic.historicalsociety.closet.teddy.intro_0_cs_0', \n              'tunic.historicalsociety.cage.glasses.afterteddy', 'tunic.historicalsociety.cage.teddy.trapped', 'tunic.historicalsociety.cage.unlockdoor', 'tunic.historicalsociety.stacks.journals.pic_2.bingo', 'tunic.historicalsociety.entry.wells.flag', 'tunic.humanecology.frontdesk.worker.badger', 'tunic.historicalsociety.stacks.journals_flag.pic_0.bingo', 'tunic.historicalsociety.closet.intro', 'tunic.historicalsociety.closet.retirement_letter.hub', \n              'tunic.historicalsociety.entry.directory.closeup.archivist', 'tunic.historicalsociety.collection.tunic.slip', 'tunic.kohlcenter.halloffame.plaque.face.date', 'tunic.historicalsociety.closet_dirty.trigger_coffee', \n              'tunic.drycleaner.frontdesk.logbook.page.bingo', 'tunic.library.microfiche.reader.paper2.bingo', \n              'tunic.kohlcenter.halloffame.togrampa', 'tunic.capitol_2.hall.boss.haveyougotit', 'tunic.wildlife.center.wells.nodeer_recap', 'tunic.historicalsociety.cage.glasses.beforeteddy', 'tunic.historicalsociety.closet_dirty.gramps.helpclean', 'tunic.wildlife.center.expert.recap', 'tunic.historicalsociety.frontdesk.archivist.have_glass_recap', \n              'tunic.historicalsociety.stacks.journals_flag.pic_1.bingo', 'tunic.historicalsociety.cage.lockeddoor', 'tunic.historicalsociety.stacks.journals_flag.pic_2.bingo', 'tunic.historicalsociety.collection.gramps.lost', 'tunic.historicalsociety.closet.notebook', 'tunic.historicalsociety.frontdesk.magnify', 'tunic.humanecology.frontdesk.businesscards.card_bingo.bingo', 'tunic.wildlife.center.remove_cup', 'tunic.library.frontdesk.wellsbadge.hub', 'tunic.wildlife.center.tracks.hub.deer', \n              'tunic.historicalsociety.frontdesk.key', 'tunic.library.microfiche.reader_flag.paper2.bingo', 'tunic.flaghouse.entry.colorbook', 'tunic.wildlife.center.coffee', 'tunic.capitol_1.hall.boss.haveyougotit', 'tunic.historicalsociety.basement.janitor', 'tunic.historicalsociety.collection_flag.gramps.recap', 'tunic.wildlife.center.wells.animals2', 'tunic.flaghouse.entry.flag_girl.symbol_recap', 'tunic.historicalsociety.closet_dirty.photo', \n              'tunic.historicalsociety.stacks.outtolunch', 'tunic.library.frontdesk.worker.wells_recap', 'tunic.historicalsociety.frontdesk.archivist_glasses.confrontation_recap', 'tunic.capitol_0.hall.boss.talktogramps', 'tunic.historicalsociety.closet.photo', 'tunic.historicalsociety.collection.tunic', \n              'tunic.historicalsociety.closet.teddy.intro_0_cs_5', 'tunic.historicalsociety.closet_dirty.gramps.archivist', \n              'tunic.historicalsociety.closet_dirty.door_block_talk', 'tunic.historicalsociety.entry.boss.flag_recap', 'tunic.historicalsociety.frontdesk.archivist.need_glass_0', \n              'tunic.historicalsociety.entry.wells.talktogramps', 'tunic.historicalsociety.frontdesk.block_magnify', 'tunic.historicalsociety.frontdesk.archivist.foundtheodora', \n              'tunic.historicalsociety.closet_dirty.gramps.nothing', 'tunic.historicalsociety.closet_dirty.door_block_clean', 'tunic.capitol_1.hall.boss.writeitup', 'tunic.library.frontdesk.worker.nelson_recap', 'tunic.library.frontdesk.worker.hello_short', \n              'tunic.historicalsociety.stacks.block', 'tunic.historicalsociety.frontdesk.archivist.need_glass_1', 'tunic.historicalsociety.entry.boss.talktogramps', \n              'tunic.historicalsociety.frontdesk.archivist.newspaper_recap', 'tunic.historicalsociety.entry.wells.flag_recap', 'tunic.drycleaner.frontdesk.worker.done2', 'tunic.library.frontdesk.worker.flag_recap', \n              'tunic.humanecology.frontdesk.block_0', 'tunic.library.frontdesk.worker.preflag', 'tunic.historicalsociety.basement.gramps.seeyalater', 'tunic.flaghouse.entry.flag_girl.hello_recap', 'tunic.historicalsociety.closet.doorblock', \n              'tunic.drycleaner.frontdesk.worker.takealook', 'tunic.historicalsociety.basement.gramps.whatdo', 'tunic.library.frontdesk.worker.droppedbadge', \n              'tunic.historicalsociety.entry.block_tomap2', 'tunic.library.frontdesk.block_nelson', 'tunic.library.microfiche.block_0', \n              'tunic.historicalsociety.entry.block_tocollection', 'tunic.historicalsociety.entry.block_tomap1', 'tunic.historicalsociety.collection.gramps.look_0', \n              'tunic.library.frontdesk.block_badge', 'tunic.historicalsociety.cage.need_glasses', 'tunic.library.frontdesk.block_badge_2', \n              'tunic.kohlcenter.halloffame.block_0', 'tunic.capitol_0.hall.chap1_finale_c', 'tunic.capitol_1.hall.chap2_finale_c', \n              'tunic.capitol_2.hall.chap4_finale_c', 'tunic.wildlife.center.fox.concern', 'tunic.drycleaner.frontdesk.block_0', \n              'tunic.historicalsociety.entry.gramps.hub', 'tunic.humanecology.frontdesk.block_1', 'tunic.drycleaner.frontdesk.block_1']\n\n# 每个房间的唯一ID\nroom_fqid_lists = ['tunic.historicalsociety.entry', 'tunic.wildlife.center', 'tunic.historicalsociety.cage', \n              'tunic.library.frontdesk', 'tunic.historicalsociety.frontdesk', 'tunic.historicalsociety.stacks', \n              'tunic.historicalsociety.closet_dirty', 'tunic.humanecology.frontdesk', 'tunic.historicalsociety.basement', \n              'tunic.kohlcenter.halloffame', 'tunic.library.microfiche', 'tunic.drycleaner.frontdesk', 'tunic.historicalsociety.collection', \n              'tunic.historicalsociety.closet', 'tunic.flaghouse.entry', 'tunic.historicalsociety.collection_flag', 'tunic.capitol_1.hall', \n              'tunic.capitol_0.hall', 'tunic.capitol_2.hall']\n\n# 文本中某些词出现次数的统计\nDIALOGS = ['that', 'this', 'it', 'you','find','found','Found','notebook','Wells','wells','help',\n           'need', 'Oh','Ooh','Jo', 'flag', 'can','and','is','the','to']","metadata":{"_uuid":"bf4d1344-bb0f-4896-b90d-71ea3863875c","_cell_guid":"adb3638c-bd19-4865-8c9c-a501159f17ce","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:16:14.702342Z","iopub.execute_input":"2023-04-29T16:16:14.702923Z","iopub.status.idle":"2023-04-29T16:16:14.725346Z","shell.execute_reply.started":"2023-04-29T16:16:14.702884Z","shell.execute_reply":"2023-04-29T16:16:14.724230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### [Groupby简单技巧](https://zhuanlan.zhihu.com/p/101284491)\n\n### [Groupby.sum/Groupby.count](https://blog.csdn.net/qq_39290990/article/details/121435796)\n统计每组数据求和和数据的种类数\n### [Groupby.last](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.core.groupby.SeriesGroupBy.last.html)\n选择每组中的最后一行的数据\n### [DataFrame.join](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.join.html)\nDataFrame的列的拼接\n### [关于为什么生成的列都是NAN的问题](https://blog.csdn.net/S_o_l_o_n/article/details/90383176)","metadata":{"_uuid":"c2198a9f-c21a-4d7f-8431-93b17c057adf","_cell_guid":"c8e1064b-f9d6-40aa-9f51-c332e7707737","trusted":true}},{"cell_type":"code","source":"def delt_time_def(train_df):\n    train_df.sort_values(by=['session_id', 'elapsed_time'], inplace=True)\n    train_df['delt_time'] = train_df['elapsed_time'].diff(1) # 对于同一个玩家的触发两个相邻的事件时的时间差\n    train_df['delt_time'].fillna(0, inplace=True)\n    train_df['delt_time'].clip(0, 28000, inplace=True) # 为什么是103000 s = 1716 min = 28 h\n                                    # 将长时间不玩的人的elapsed_time替换掉\n    return train_df","metadata":{"_uuid":"2d12ac1c-583f-4b2e-97eb-d51c181f5129","_cell_guid":"017b7b9d-0880-4fd2-801f-e2f7acc45583","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:16:14.726804Z","iopub.execute_input":"2023-04-29T16:16:14.727526Z","iopub.status.idle":"2023-04-29T16:16:14.747931Z","shell.execute_reply.started":"2023-04-29T16:16:14.727484Z","shell.execute_reply":"2023-04-29T16:16:14.746882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def time_feature(train):\n    train[\"year\"] = pd.Series(train.index, index=train.index).apply(lambda x: int(str(x)[:2])).astype(np.uint8)\n    train[\"month\"] = pd.Series(train.index, index=train.index).apply(lambda x: int(str(x)[2:4])+1).astype(np.uint8)\n    train[\"day\"] = pd.Series(train.index, index=train.index).apply(lambda x: int(str(x)[4:6])).astype(np.uint8)\n    train[\"hour\"] = pd.Series(train.index, index=train.index).apply(lambda x: int(str(x)[6:8])).astype(np.uint8)\n    train[\"minute\"] = pd.Series(train.index, index=train.index).apply(lambda x: int(str(x)[8:10])).astype(np.uint8)\n    train[\"second\"] = pd.Series(train.index, index=train.index).apply(lambda x: int(str(x)[10:12])).astype(np.uint8)\n    \n    return train","metadata":{"_uuid":"d74fae2a-8cd9-45ff-897a-ea505bb8a4ea","_cell_guid":"145b180a-52e1-417d-a5ab-32f5f7c49f50","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:16:14.749220Z","iopub.execute_input":"2023-04-29T16:16:14.750111Z","iopub.status.idle":"2023-04-29T16:16:14.766538Z","shell.execute_reply.started":"2023-04-29T16:16:14.750037Z","shell.execute_reply":"2023-04-29T16:16:14.765142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineer(train):\n    # 固定限定ID数据, category分类型数据,可以统计这些类别下不同session_id事件出现的频次等\n    FIXED_EVENT_ID = ['event_name', 'name', 'fqid', 'room_fqid', 'text', 'text_fqid'] # page需要进行填充\n    # 数值型数据,可以统计其最大值,最小值,均值,标准差,分位数\n    NUMS = ['delt_time', 'hover_duration', 'room_coor_x', 'room_coor_y', 'screen_coor_x', \n           'screen_coor_y']\n    # 事件类型\n    EVENT = ['basic', 'undefined']\n    # 点击事件\n    EV_NAME = ['cutscene_click', 'person_click', 'navigate_click',\n       'observation_click', 'notification_click', 'object_click',\n       'object_hover', 'map_hover', 'map_click', 'checkpoint',\n       'notebook_click']\n    \n    # 一个空的DataFeame,最开始只有用户的session_id(去重后的用户会话)作为行索引,因此每一行表示这个用户的session_id\n    new_train = pd.DataFrame(index=train['session_id'].unique(), columns=[])\n    new_train = time_feature(new_train)\n    \n    # 不同点击事件类型的总的间隔时间的数据统计\n    for c in EV_NAME: # 统计每一个事件的点击次数，间隔时间的和与均值\n        # 统计每一个用户会话某一个点击事件的开始点击次数\n        new_train['count_ev_name_' + c] = train[ train['event_name'] == c ].groupby(['session_id'])['index'].count()\n        # 统计每一个用户会话某一个点击事件的相邻点击间隔时间的和\n        new_train['delt_time_ev_name_' + c] = train[ train['event_name'] == c ].groupby(['session_id'])['delt_time'].sum()\n        new_train['mean_delt_ev_name_' + c] = train[ train['event_name'] == c ].groupby(['session_id'])['delt_time'].mean()\n        new_train['std_delt_ev_name_' + c ] = train[ train['event_name'] == c ].groupby(['session_id'])['delt_time'].std()\n    \n    # 统计每一个用户最后一个发生的用户会话的时间（从开始到最后一个事件）\n    new_train['finish'] = train.groupby(['session_id'])['elapsed_time'].last(1)\n    # 统计每个用户在游戏时点击会话事件的总次数\n    new_train['len'] = train.groupby(['session_id'])['index'].count()\n    \n    # 在限定ID数据上的统计，统计每一个用户在限定ID C上的种类数量,后面会专门写一个函数来统计每个ID的时间上的统计信息\n    for c in FIXED_EVENT_ID:\n        tmp = train.groupby(['session_id'])[c].agg('nunique')\n        tmp.name = tmp.name + '_nunique' \n        new_train = new_train.join(tmp)  # 对DataFrame进行拼接新的一列横着拼接新的列，可以一次性拼接多个\n    \n    # 对不同悬浮事件类型的时间统计\n    new_train['object_hover_mean'] = train[~ train['hover_duration'].isna() ][ train['event_name'] == 'object_hover'].groupby(['session_id'])['hover_duration'].mean()\n    new_train['object_hover_std'] = train[~ train['hover_duration'].isna() ][ train['event_name'] == 'object_hover'].groupby(['session_id'])['hover_duration'].std()\n    \n    new_train['map_hover_mean'] = train[~ train['hover_duration'].isna() ][ train['event_name'] == 'map_hover'].groupby(['session_id'])['hover_duration'].mean()\n    new_train['map_hover_std'] = train[~ train['hover_duration'].isna() ][ train['event_name'] == 'map_hover'].groupby(['session_id'])['hover_duration'].std()\n    \n    # 统计每个session_id的数值类型的均值和标准差\n    \n    for c in NUMS: \n        tmp = train.groupby(['session_id'])[c].agg('mean')\n        new_train = new_train.join(tmp)\n        \n    \n    # 统计数值类型的标准差\n    for c in NUMS:\n        tmp = train.groupby(['session_id'])[c].agg('std')\n        tmp.name = tmp.name + '_std'\n        new_train = new_train.join(tmp)\n    \n    # 对text和ID的每一个具体内容都详细统计\n    \n    for c in DIALOGS:\n        new_train['word_' + c] = train[ train['text'].str.contains(c, na=False) ].groupby(['session_id'])['index'].count()\n        new_train['word_std_' + c] = train[ train['text'].str.contains(c, na=False) ].groupby(['session_id'])['delt_time'].std()\n        new_train['word_max_' + c] = train[ train['text'].str.contains(c, na=False) ].groupby(['session_id'])['delt_time'].max()\n        new_train['word_sum_' + c] = train[ train['text'].str.contains(c, na=False) ].groupby(['session_id'])['delt_time'].sum()\n        new_train['word_median_' + c] = train[ train['text'].str.contains(c, na=False) ].groupby(['session_id'])['delt_time'].median()\n        \n    for c in fqid_lists:\n        new_train[f'{c}_fqid_counts'] = train[ train['fqid'] == c ].groupby(['session_id'])['fqid'].count()\n        new_train[f'{c}_fqid_delt_time_std'] = train[ train['fqid'] == c ].groupby(['session_id'])['delt_time'].std()\n        new_train[f'{c}_fqid_delt_time_mean'] = train[ train['fqid'] == c ].groupby(['session_id'])['delt_time'].mean()\n        new_train[f'{c}_fqid_delt_time_sum'] = train[ train['fqid'] == c ].groupby(['session_id'])['delt_time'].sum()\n        new_train[f'{c}_fqid_delt_time_median'] = train[ train['fqid'] == c ].groupby(['session_id'])['delt_time'].median()\n        new_train[f'{c}_fqid_delt_time_max'] = train[ train['fqid'] == c ].groupby(['session_id'])['delt_time'].max()\n    \n    for c in text_fqid_lists:\n        new_train[f'{c}_text_fqid_counts'] = train[ train['text_fqid'] == c ].groupby('session_id')['text_fqid'].count()\n        new_train[f'{c}_text_fqid_delt_time_std'] = train[ train['text_fqid'] == c ].groupby('session_id')['delt_time'].std()\n        new_train[f'{c}_text_fqid_delt_time_mean'] = train[ train['text_fqid'] == c ].groupby('session_id')['delt_time'].mean()\n        new_train[f'{c}_text_fqid_delt_time_sum'] = train[ train['text_fqid'] == c ].groupby('session_id')['delt_time'].sum()\n        new_train[f'{c}_text_fqid_delt_time_median'] = train[ train['text_fqid'] == c ].groupby('session_id')['delt_time'].median()\n        new_train[f'{c}_text_fqid_delt_time_max'] = train[ train['text_fqid'] == c ].groupby('session_id')['delt_time'].max()\n        \n    for c in room_fqid_lists:\n        new_train[f'{c}_room_fqid_counts'] = train[ train['room_fqid'] == c ].groupby('session_id')['room_fqid'].count()\n        new_train[f'{c}_room_fqid_delt_time_std'] = train[ train['room_fqid'] == c ].groupby('session_id')['delt_time'].std()\n        new_train[f'{c}_room_fqid_delt_time_mean'] = train[ train['room_fqid'] == c ].groupby('session_id')['delt_time'].mean()\n        new_train[f'{c}_room_fqid_delt_time_sum'] = train[ train['room_fqid'] == c ].groupby('session_id')['delt_time'].sum()\n        new_train[f'{c}_room_fqid_delt_time_median'] = train[ train['room_fqid'] == c ].groupby('session_id')['delt_time'].median()\n        new_train[f'{c}_room_fqid_delt_time_max'] = train[ train['room_fqid'] == c ].groupby('session_id')['delt_time'].max()\n    \n    \n    \n#     new_train = new_train.fillna(-1) # 填充缺失值-1，根据缺失值的比例删除的特征\n    return new_train","metadata":{"_uuid":"348264af-1e37-4a8b-a389-b6063e0b0859","_cell_guid":"9dac9ad1-87ac-471d-b777-0402448fb13b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:16:14.768353Z","iopub.execute_input":"2023-04-29T16:16:14.768989Z","iopub.status.idle":"2023-04-29T16:16:14.801545Z","shell.execute_reply.started":"2023-04-29T16:16:14.768941Z","shell.execute_reply":"2023-04-29T16:16:14.800329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### [DataFrame.isin](https://blog.csdn.net/lzw2016/article/details/80472649)\nisin:接收一个列表，判断该列中的元素是否在列表中","metadata":{"_uuid":"421f8339-49b1-4202-8602-1e697d2ccb9b","_cell_guid":"720e0efd-e6df-4c7a-8ac8-e315713dad57","trusted":true}},{"cell_type":"code","source":"def drop_feature(train):\n    null_s = train.isna().sum().sort_values(ascending=False)/len(train)\n\n    drop_list = list(null_s[ null_s > 0.9 ].index) # 缺失值占比大于90%的列索引\n\n    print(\"筛选缺失值占比后需要删除的特征个数：\", len(drop_list))\n\n    for c in tqdm(train.columns): # 如果某一列的统计量只有一种值，那么将其删除\n        if train[c].nunique() == 1:\n            print(c)\n            drop_list.append(c)\n    \n    print(\"需要去除的特征总数：\", len(drop_list))\n    \n    return drop_list","metadata":{"_uuid":"82dfa051-ee52-4b63-a61e-e865f73cc1cd","_cell_guid":"90973f1a-ea0a-4243-a9f6-b11125d1d0d0","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:16:14.802858Z","iopub.execute_input":"2023-04-29T16:16:14.804127Z","iopub.status.idle":"2023-04-29T16:16:14.817538Z","shell.execute_reply.started":"2023-04-29T16:16:14.804083Z","shell.execute_reply":"2023-04-29T16:16:14.816520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nprint(\"****************************** train1 START ********************************\")\nkol_lvl = (df_0_4.groupby(['session_id'])['level'].agg('nunique') < 5) \n# 统计每组的level中的值的种类数量，将种类数小于5的组标记为True，也就是只要level数小于5说明这一阶段的都没有通关\n# print(kol_lvl.shape)\n# print(kol_lvl)\nlist_session = kol_lvl[kol_lvl].index  \n# 又因为kol_lvl的行索引就是session_id，因此直接将筛选的布尔矩阵的放进kol_lvl就可以，选择正确的id\n# print(kol_lvl[kol_lvl])\ndf_0_4 = df_0_4[~ df_0_4['session_id'].isin(list_session) ] \n# session_id的值不在这个list_session就为False，否则为True，然后取反：获得符合要求的id\nprint(df_0_4['session_id'].nunique())\n\ndf_0_4 = delt_time_def(df_0_4)\n# display(df_0_4.head())\ntrain1 = feature_engineer(df_0_4)\nprint(f\"train1 shape : {train1.shape}\")\n\ndrop_list1 = drop_feature(train1)\n\nprint(\"****************************** train1 DONE ********************************\")\n\ndel df_0_4","metadata":{"_uuid":"c0ce45a0-010e-4964-a8e3-360236afdb77","_cell_guid":"e8352d3c-7672-4af4-8a80-3365057d5d26","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:16:14.818904Z","iopub.execute_input":"2023-04-29T16:16:14.819834Z","iopub.status.idle":"2023-04-29T16:16:44.921621Z","shell.execute_reply.started":"2023-04-29T16:16:14.819796Z","shell.execute_reply":"2023-04-29T16:16:44.920513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nprint(\"****************************** train2 START ********************************\")\nkol_lvl = (df_5_12.groupby(['session_id'])['level'].agg('nunique') < 8) # 小于8个的id没有通关，不要\nlist_session = kol_lvl[kol_lvl].index  # index返回的是不符合要求的session_id，因为行索引就是session_id\n# 8是一个上界，也就是每一个id不同level最多只有八个即5，6，7，8，9，10，11，12\n# 返回的kol_lvl是列表，每一个session_id都对应一个True/False，\n# 如果为True说明这个session_id符合要求，否则说明这个id的level的数量大于8个\n# 然后利用kol_lvl[kol_lvl]来筛选不对应的session_id的下标index\n# print(list_session)\ndf_5_12 = df_5_12[~df_5_12['session_id'].isin(list_session)] # 如果某一列是不正确的id，则取反变成False不选择这一行\n\nprint(df_5_12['session_id'].nunique())\ndf_5_12 = delt_time_def(df_5_12)\ntrain2 = feature_engineer(df_5_12)\nprint(f\"train2 shape : {train2.shape}\")\n\ndrop_list2 = drop_feature(train2)\n\nprint(\"****************************** train2 DONE ********************************\")\n\ndel df_5_12","metadata":{"_uuid":"908f64f2-5285-402a-b80e-536b1468a3c0","_cell_guid":"83ccbe5a-12a1-41b5-b5d1-44a761b88e6b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:16:44.922926Z","iopub.execute_input":"2023-04-29T16:16:44.923339Z","iopub.status.idle":"2023-04-29T16:17:54.129621Z","shell.execute_reply.started":"2023-04-29T16:16:44.923296Z","shell.execute_reply":"2023-04-29T16:17:54.128552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nprint(\"****************************** train3 START ********************************\")\nkol_lvl = (df_13_22.groupby(['session_id'])['level'].agg('nunique') < 10) # 小于10个的id没有通关，不要\nlist_session = kol_lvl[kol_lvl].index\ndf_13_22 = df_13_22[~ df_13_22['session_id'].isin(list_session) ]\n\nprint(df_13_22['session_id'].nunique())\ndf_13_22 = delt_time_def(df_13_22)\ntrain3 = feature_engineer(df_13_22)\nprint(f\"train3 shape : {train3.shape}\")\n\ndrop_list3 = drop_feature(train3)\n\nprint(\"****************************** train3 DONE ********************************\")\n\ndel df_13_22, test_df","metadata":{"_uuid":"19038416-8979-446e-9ec6-c927263802d6","_cell_guid":"642acba7-59f0-48c2-b199-4df92e5a07b5","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:17:54.131222Z","iopub.execute_input":"2023-04-29T16:17:54.132586Z","iopub.status.idle":"2023-04-29T16:19:39.687647Z","shell.execute_reply.started":"2023-04-29T16:17:54.132526Z","shell.execute_reply":"2023-04-29T16:19:39.686566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FEATURES1 = [c for c in train1.columns if c not in drop_list1 + ['fullscreen', 'hq', 'music', 'level_group']] # 由于这里只有NUM类型的数据因此每个level的\nFEATURES2 = [c for c in train2.columns if c not in drop_list2 + ['fullscreen', 'hq', 'music', 'level_group']]\nFEATURES3 = [c for c in train3.columns if c not in drop_list3 + ['fullscreen', 'hq', 'music', 'level_group']]\nprint('We will train with', len(FEATURES1), len(FEATURES2), len(FEATURES3) ,'features')\nALL_USERS = train1.index.unique()\nprint('We will train with', len(ALL_USERS) ,'users info')","metadata":{"_uuid":"bab3093a-1f81-4da7-84e7-1b19eaca6a72","_cell_guid":"97b07a39-7f6d-412d-b88b-2f6d6521642d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:19:39.688991Z","iopub.execute_input":"2023-04-29T16:19:39.689847Z","iopub.status.idle":"2023-04-29T16:19:39.808516Z","shell.execute_reply.started":"2023-04-29T16:19:39.689804Z","shell.execute_reply":"2023-04-29T16:19:39.807131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### [不同的交叉验证类型1](https://blog.csdn.net/qq_16761099/article/details/106091354)\n### [不同的交叉验证类型2](https://zhuanlan.zhihu.com/p/52515873)\n### [交叉验证详解(评论比文章还好看)](https://zhuanlan.zhihu.com/p/24825503)","metadata":{"_uuid":"3816b17c-adad-4fb5-9bb2-26d66bb7cc7f","_cell_guid":"bb6adce6-ebb6-48fc-be5c-a10b232148f9","trusted":true}},{"cell_type":"markdown","source":"# Train XGBoost Model\n\n### [XGBoost的参数设置](https://blog.csdn.net/VariableX/article/details/107238137)\n### [xgb.feature_importances_](https://blog.csdn.net/jin_tmac/article/details/106099116)\n获得当前特征的重要程度，返回np.array，里面的数值就是每个特征的重要程度，数值的顺序就是特征的顺序\n### [关于维度的问题](https://blog.csdn.net/qq_42670023/article/details/123702493)\n输入维度出现错误","metadata":{"_uuid":"a792efde-9cf3-4a57-b941-bfa014039e62","_cell_guid":"cbaa9b74-00a4-4710-9853-7d98b01ae3fd","trusted":true}},{"cell_type":"code","source":"%%time\nfrom xgboost import XGBClassifier\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score\nfrom sklearn.model_selection import GroupKFold\n\nxgb_params = {\n    'booster' : 'gbtree',\n    'objective' : 'binary:logistic',\n    'eval_metric':'logloss',\n    'learning_rate': 0.02,\n    'alpha':8,\n    'max_depth': 4,\n    'n_estimators': 1000,\n    'early_stopping_rounds': 50,\n    'tree_method':'hist',\n    'subsample':0.8,\n    'colsample_bytree': 0.5,\n    'use_label_encoder' : False,\n    'seed' : 42,\n}\n\nfolds = 5\n\nmodels = {} #把不同问题训练出来的分类器保存下来\ngkf = GroupKFold(n_splits = folds) # 划分5组\nresults = [ [[], []]  for _ in range(18) ] # 18个问题，每个问题都是一个二分类问题results[0]存储每一组的验证标签results[1]存储每一组的预测验证标签\n# off_cat = pd.DataFrame(data=np.zeros((len(ALL_USERS), 18)), index=ALL_USERS, columns=[f'meta_{i}' for i in range(1,19)])\n\nprint(\"**************\"*2+ \" Model TRAIN \" + \"****************\"*2)\n\nfor q in range(1,19):\n    # 分别训练不同问题的分类器\n    if q <= 3:\n        grp = '0-4'\n        df = train1\n        FEATURES = FEATURES1\n        \n    elif q <= 13:\n        grp = '5-12'\n        df = train2\n        FEATURES = FEATURES2\n    \n    elif q <= 22:\n        grp = '13-22'\n        df = train3\n        FEATURES = FEATURES3\n        \n    print('#'*27)\n    print(f'Question{q}, with {len(FEATURES)} features:')\n    print('#'*27)\n    \n    feature_importance_df = pd.DataFrame()\n    \n    # 将session_id划分为5组进行K折交叉验证,每一组都会返回训练数据和验证数据的下标\n    # 这里的下标是传给split方法groups参数的索引列表中的下标\n    # 因此如果具体的训练数据还需要得到groups的输入索引列表对应位置上的元素才是真正数据的下标\n    for fold, (train_idx, valid_idx) in enumerate(gkf.split(X=df, groups=df.index)):\n    ################################ TRAIN DATA ####################################\n        print(f\"Fold{fold+1}:\")\n        # TRAIN DATA\n        train_x = df.iloc[train_idx]\n        train_users = train_x.index.values # 返回train数据集中存在的session_id的列表\n        train_y = train_label_df.loc[ train_label_df.q == q ].set_index('session').loc[train_users]\n        \n        # VALID DATA\n        valid_x = df.iloc[ valid_idx ]\n        valid_users = valid_x.index.values\n        valid_y = train_label_df.loc[ train_label_df.q == q ].set_index('session').loc[valid_users]\n        \n        \n        xgb_model = XGBClassifier(**xgb_params)\n        \n        xgb_model.fit(train_x[FEATURES].astype('float32'), train_y['correct'], \n                      eval_set=[(train_x[FEATURES].astype('float32'), train_y['correct']),\n                                (valid_x[FEATURES].astype('float32'), valid_y['correct'])],\n                      verbose=0)\n    \n        print(f\"train data  F1 score is \", str(f1_score(xgb_model.predict(train_x[FEATURES].astype('float32')), train_y['correct'])), end=', ')\n        print(f'test data F1 score is ', str(f1_score(xgb_model.predict(valid_x[FEATURES].astype('float32')), valid_y['correct'])), end='。\\n\\n')\n        \n        # SAVE MODEL, PREDICT VALID OOF\n        y = valid_y['correct']\n        y_hat = xgb_model.predict_proba(valid_x[FEATURES].astype('float32'))[:,1] # 这个索引？\n        models[(q, fold)] = xgb_model\n        \n        fold_importance_df = pd.DataFrame()\n        fold_importance_df[\"feature\"] = FEATURES\n        fold_importance_df['importance'] = xgb_model.feature_importances_\n        fold_importance_df['fold'] = fold + 1\n        feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0) # 上下堆叠\n        \n        results[q-1][0].append(y) # 第一层放的是验证集的真实标签y\n        results[q-1][1].append(y_hat) # 第二层放的是验证集的预测标签y_hat\n        \n    feature_importance_df = feature_importance_df.groupby(['feature'])['importance'].agg(['mean']).sort_values(by='mean', ascending=False)\n    display(feature_importance_df.head(10)) # 展示重要程度为前十的特征\n\n    \nprint(\"**************\"*2+ \" TRAIN DONE \" + \"****************\"*2)\n#############################################################################","metadata":{"_uuid":"31eda93d-ea0d-4acb-b466-2a492deb64e6","_cell_guid":"7cf10dfe-91c7-4514-aa5f-cb20540ebb26","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-29T16:19:39.810242Z","iopub.execute_input":"2023-04-29T16:19:39.810696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 模型评估和模型保存","metadata":{"_uuid":"cedeac79-3f36-4008-b0b1-19859db896fc","_cell_guid":"7e793a80-857a-4818-b84c-f732eb8a3ce3","trusted":true}},{"cell_type":"code","source":"def plot_threshold_f1_score(results, start, end):\n    true = pd.DataFrame(np.stack([_[0] for _ in results[start-1: end]]).T) # 验证集标签true\n    oof = pd.DataFrame(np.stack([_[1] for _ in results[start-1: end]]).T) # 验证集的预测标签oof\n\n    scores = []\n    best_score = 0\n    thresholds = []\n    best_threshold = 0\n    \n    \"\"\"\n    为什么选择输出每个预测值概率[0~1]而不是直接输出预测标签[0,1]：这是由于大于0.5不一定能够能够分类正确。\n    因此为了增加容错率，我们希望能够选一个能够使F1 score更大的最合适的阈值对结果进行判断。\n    这就是为什么要groupbyCV（分组交叉验证），这样就可以将多个组进行随机划分的数据集（也就是多个train_test_split）的数据集（包含训练数据和验证数据）\n    进行训练，得到多个学习器，保存每一个学习器对其对应的验证集进行预测的预测概率分布和真实的标签。\n    这样对于一个数据集来说我们相当于对他进行了5（groupby的数量）次交叉验证，并且在这个函数里综合计算五次的预测结果与真实值的F1 score\n    这样的F1 score更加健壮。\n    而阈值的选择则是自定义阈值的范围，然后用某个阈值进行筛选：\n    二分类问题，比阈值大的认为是1，比阈值小的认为是0（多分类问题呢？如果只做一次预测, 返回的应该是一个n维的概率分布，一般来说选择概率最大的哪一类作为预测结果）\n    多分类的分类函数为softmax：https://blog.csdn.net/qq_58152527/article/details/124073097\n    \"\"\"\n    \n    for threshold in np.arange(0.5, 0.8, 0.01): # threshold是判别阈值， 大于阈值的认为是1，小于阈值的认为是0买这个阈值的范围为[0.5, 0.7]\n        preds = (oof.values.reshape(-1) > threshold).astype('int') # 大于0.5+i的预测值，i会不断以0.01的步长增加, 直到0.5+i = 0.7\n        m = f1_score(true.values.reshape(-1), preds, average='macro')\n        scores.append(m)\n        thresholds.append(threshold)\n        if m > best_score:\n            best_score = m\n            best_threshold = threshold\n\n    plt.figure(figsize=(20,5))\n    plt.plot(thresholds, scores, '-o', color = 'blue')\n    plt.scatter([best_threshold], [best_score], color = 'blue')\n    plt.xlabel(\"Threshold\", size=14)\n    plt.ylabel(\"Validation F1 Score\", size=14)\n    plt.title(f\"Threshold vs. F1_Score with Best F1_Score = {best_score:.3f} at Best Threshold = {best_threshold:.3}\", size = 18)\n    plt.show()\n\n\n    print(f'When using optimal threshold = {best_threshold:.2f}...')\n    for k in range(0, end-start+1): # 每一部分都使用这一部分最优的阈值对预测概率进行判别\n        m = f1_score(true[k].values, (oof[k].values > best_threshold).astype('int'), average = 'macro')\n        print(f'Q{start+k}: F1 =', m)\n    m = f1_score(true.values.reshape(-1), (oof.values > best_threshold).reshape(-1).astype('int'), average = 'macro')\n    print('==> Overall F1 =', m)","metadata":{"_uuid":"98fd14b4-ba91-45a7-9d59-419a4bd4312a","_cell_guid":"66e4b796-6924-4b1d-a8a8-59c9f7268abb","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = [[np.concatenate(_) for _ in _] for _ in results] # 将5次预测结果或标签连接在一个list里\n\nfor idx,_ in enumerate(results): # 每个level_group的session_id个数不一样，因此需要绘制不同的阈值图\n    print(f\"q{idx+1}的标签长度:\",len(_[0]))\n    print(f\"q{idx+1}的预测标签长度:\", len(_[1]))\n    print()\n    \n\nplot_threshold_f1_score(results, 1, 3)\nplot_threshold_f1_score(results, 4, 13)\nplot_threshold_f1_score(results, 14, 18)\nprint()\ntrue_list = []\nfor result in results:\n    true_list.extend(result[0]) # extend可以直接取array里面的值放进列表里\n# print(np.array(true_list).shape)\n\npred_list = []\nfor result in results:\n    pred_list.extend(result[1])\n# print(np.array(pred_list).shape)\n\nm = f1_score(pd.Series(true_list), (pd.Series(pred_list) > 0.653).astype('int'), average='macro')\nprint(f\"==> Total Question F1 = {m}\")","metadata":{"_uuid":"1fa6141d-f8db-422d-bb88-24ac0875c3f1","_cell_guid":"02200b2d-98b3-453c-b178-a0e0ca2bf12e","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"*\"*30, \" Model SAVE \", \"*\"*30)\nfor (q, fold), xgb_model in tqdm(models.items()): # 保存模型\n    xgb_model.save_model(f'q{q}_fold{fold}.xgb')\nprint(f\"Model has successful saved\")\nprint(\"*\"*30, \" SAVE DONE \", \"*\"*30)","metadata":{"_uuid":"af3ea0ff-3670-4048-be8b-e9f0f1408b6e","_cell_guid":"e5f201e0-058d-493e-b4ab-81ad570cddc6","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{"_uuid":"5956af05-43c0-4f09-8119-bbba98aec000","_cell_guid":"c57b2b63-4bfa-4c97-b831-77e912f67a6c","trusted":true}},{"cell_type":"code","source":"import jo_wilder\nenv = jo_wilder.make_env()\niter_test = env.iter_test()\n\n# Clear Memory\nimport gc\ndel results, df, train1, train2, train3, true_list, pred_list, m\n_ = gc.collect()","metadata":{"_uuid":"a87ca6ba-bf74-4f7b-8319-fd3d119755a4","_cell_guid":"e0d32bb6-cff2-450a-a2b9-c3b0dcae4c8a","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n\nlimits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\nthreshold_dict = {}\nfor q in range(1,19):\n    if q >= 1 and q <= 3:\n        threshold=0.72 \n    elif q >= 4 and q <= 13:\n        threshold = 0.60 \n    elif q >= 14 and q <= 18:\n        threshold = 0.64\n    threshold_dict[q] = threshold\n\nfor (test, sample_submission) in iter_test:\n#     print(test)\n#     print(sample_submission)\n    grp = test.level_group.values[0]\n    session_id = test.session_id.values[0]\n    a,b = limits[grp]\n    \n    if a == 1:\n        FEATURES = FEATURES1\n    elif a == 4:\n        FEATURES = FEATURES2\n    elif a == 14:\n        FEATURES = FEATURES3\n    \n    test = test.drop(['fullscreen', 'hq', 'music'], axis=1)\n    test = delt_time_def(test)\n    test = feature_engineer(test)\n    test = test[FEATURES]\n    preds = []\n    \n    for q in range(a,b): # 当前session_id的问题q\n        print(q)\n#         mask = sample_submission.session_id.str.contains(f'q{q}') # 找到提交的submissio中包含q的行号\n        y_hat_list = []\n        \n        for fold in range(folds):\n            xgb_model = XGBClassifier()\n            xgb_model.load_model(f'/kaggle/working/q{q}_fold{fold}.xgb')\n            y_hat = xgb_model.predict_proba(test.astype('float32'))[0, 1]\n            y_hat_list.append(y_hat)\n            \n        y_mean = np.mean(y_hat_list)\n        preds.append(int( y_mean > threshold_dict[q] ))\n#         sample_submission.loc[mask, 'correct'] = int(y_mean > threshold_dict[q]) \n    sample_submission['correct'] = preds\n    env.predict(sample_submission)","metadata":{"_uuid":"22a7ef6b-9492-4bfb-8d4b-8578e588b51b","_cell_guid":"7b6accde-1c40-4cc3-9809-f4d4e714f158","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('submission.csv')\nprint(sub.shape, sub.correct.mean())\nsub.head(10)","metadata":{},"execution_count":null,"outputs":[]}]}