{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd, numpy as np, gc\nfrom sklearn.model_selection import KFold, GroupKFold\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import f1_score\nfrom sklearn.model_selection import GridSearchCV\nimport plotly.express as px\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-11T12:29:37.996269Z","iopub.execute_input":"2023-04-11T12:29:37.996678Z","iopub.status.idle":"2023-04-11T12:29:38.002274Z","shell.execute_reply.started":"2023-04-11T12:29:37.996644Z","shell.execute_reply":"2023-04-11T12:29:38.001435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train_labels.csv\")\ntmp = target.session_id.str.split(\"_\", expand =True)\ntarget[\"user_id\"] = tmp[0].astype(\"int\")\ntarget[\"q\"] = tmp[1].str.slice(1).astype(\"int\")\ntarget.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-11T12:29:38.004238Z","iopub.execute_input":"2023-04-11T12:29:38.005166Z","iopub.status.idle":"2023-04-11T12:29:39.831798Z","shell.execute_reply.started":"2023-04-11T12:29:38.005129Z","shell.execute_reply":"2023-04-11T12:29:39.830934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 將USER十等份，之後批次執行特徵工程","metadata":{}},{"cell_type":"code","source":"tmp = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train.csv\", usecols = ['session_id'])\nnum_userdata = tmp.groupby(\"session_id\").session_id.agg(\"count\") #count是每個user事件的筆數 / sum會將session的值都相加，不是我們要的\n\nDIVIDE= 10\nEQUAL_USER = np.ceil(len(num_userdata)/DIVIDE).astype(\"int\") #將user人數10等份，PART儲存每一等份該有的user數 \n\nread_row = []\nskip_row =[0]\nfor i in range(DIVIDE):\n    a = i * EQUAL_USER\n    b = (i+1) * EQUAL_USER\n    if b > len(tmp): b = len(tmp)\n    r = num_userdata.iloc[a:b].sum() #從第一位user到經過10等份的最後一位user，加總所有事件筆數\n    read_row.append(r)\n    skip_row.append(skip_row[-1]+r)\n    \nprint(tmp.shape)    \nprint(f\"Read_Row: {read_row}\")\nprint(f\"Skip_Row: {skip_row}\")\n# y_train = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train_labels.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-04-11T12:29:39.833258Z","iopub.execute_input":"2023-04-11T12:29:39.833805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 先取第一等份玩家資料","metadata":{}},{"cell_type":"code","source":"X_train =  pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train.csv\", \n                       usecols=['session_id', #user_id\n                              'index', \n                              'elapsed_time', \n                              'event_name', #event type\n                              'name', #the event name\n                              'level',\n#                               'room_coor_x', \n#                               'room_coor_y', \n#                               'screen_coor_x',\n#                               'screen_coor_y',\n                              'room_fqid',\n                              'level_group'],\n                      nrows = read_row[0])\n\nprint(f\"X_train Shape: {X_train.shape}\")\nX_train.elapsed_time = X_train.elapsed_time/60000 #convert millseconds to min\nX_train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 特徵工程","metadata":{}},{"cell_type":"code","source":"CATS =[\"index\",\"event_name\",\"name\",\"room_fqid\"]\nNUM = [\"elapsed_time\",\"level\"]\nEVENT = ['cutscene_click', 'person_click', 'navigate_click',\n       'observation_click', 'notification_click', 'object_click',\n       'object_hover', 'map_hover', 'map_click', 'checkpoint',\n       'notebook_click']\n\ndef feature_engineer(data):\n    tmp_df = []\n    for col_name in CATS:\n        tmp = data.groupby([\"session_id\", \"level_group\"])[col_name].agg(\"nunique\") #DataFrame\n        tmp.name = col_name + \"_nunique\"\n        tmp_df.append(tmp)\n\n    for col_name in NUM:\n        tmp = data.groupby([\"session_id\", \"level_group\"])[col_name].agg(\"mean\")\n        tmp.name = col_name + \"_mean\"\n        tmp_df.append(tmp)\n    #將各event name取出成為獨立一欄\n    for e in EVENT:\n        data[e] = (data.event_name == e).astype(\"int\")\n    for col_name in EVENT:\n        tmp = data.groupby([\"session_id\", \"level_group\"])[col_name].agg(\"sum\")\n        tmp.name = col_name + \"_sum\"\n        tmp_df.append(tmp)\n    \n    data.drop(EVENT, axis = 1, inplace =True) # 將上面做的獨立出來的Event欄位刪除\n    \n    # \"elapsed_time\" 單獨計算每個level_group所花的時間   \n    tmp = data.groupby([\"session_id\", \"level_group\"])[\"elapsed_time\"].apply(lambda x: x.max() - x.min())\n    tmp.name = \"playtime\" #此關卡所用的時間\n    tmp_df.append(tmp)\n        \n    df = pd.concat(tmp_df, axis = 1)\n    df = df.fillna(-1) ##?????????????????????\n    df = df.reset_index() #將sesion_id、level_group 從index拉回df column\n    df = df.set_index(\"session_id\")\n    df.head()\n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 分次讀取玩家資料做特徵工程\n# %%time\n\nall_divide = [] #最後要將all_divide所有df合併\nfor i in range(DIVIDE):\n    SKIP = 0 #當i=0時\n    if i> 0: SKIP = range(1, skip_row[i]+1) #參數skiprows的值需要從1開始算需要跳過的筆數\n    train = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train.csv\",\n                nrows = read_row[i],\n                skiprows = SKIP,\n                usecols=['session_id','index', 'elapsed_time','event_name','name','level','room_fqid','level_group'])\n    df = feature_engineer(train)\n    all_divide.append(df)\n    \ndf = pd.concat(all_divide, axis = 0)\nprint(df.shape)\ndf.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train XGBoost Model","metadata":{}},{"cell_type":"code","source":"ALL_USER = df.index.unique()\nFEATURE = df.columns[1:]\nprint(ALL_USER.shape, FEATURE.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# GroupKFold\ngkf =GroupKFold(n_splits = 5) #Each group will appear exactly once in the test set across all folds\noof = pd.DataFrame(np.zeros((len(ALL_USER),18)), index= ALL_USER,columns=range(1,19)) #row: user, col: q1~q18\nmodel ={}\n\nxgb_params = {\n    'objective' : 'binary:logistic',\n    'eval_metric':'aucpr',\n    'learning_rate': 0.05,\n    'max_depth': 4,\n    'n_estimators': 200,\n    'early_stopping_rounds': 10,\n    'tree_method':'hist',\n    'subsample':0.6,\n    'colsample_bytree': 0.6,\n    'use_label_encoder' : False\n}\n\nfor i,  (train_index, test_index)  in enumerate(gkf.split(X = df,groups=df.index)): #以session_id作為分組\n    print('#'*25)\n    print('Fold',i+1)\n    print('#'*25)\n    for Q in range(1,19):\n        if Q<=3: grp = '0-4'\n        elif Q<=13: grp = '5-12'\n        elif Q<=22: grp = '13-22'\n            \n        train_x = df.iloc[train_index]\n        train_x = train_x[train_x.level_group == grp][FEATURE].astype(\"float32\")\n        train_user = train_x.index.values\n        train_y = target[target.q == Q].set_index(\"user_id\").loc[train_user].correct\n        \n        val_x = df.iloc[test_index]\n        val_x = val_x[val_x.level_group == grp][FEATURE].astype(\"float32\")\n        val_user = val_x.index.values\n        val_y = target[target.q == Q].set_index(\"user_id\").loc[val_user].correct\n\n        \n\n\n        clf = XGBClassifier(**xgb_params)\n        clf.fit(train_x,train_y,\n                eval_set=[(train_x,train_y),(val_x,val_y)],\n                verbose = 0)\n        model[f\"{grp}_{Q}\"] = clf\n        # 表現得最好是在幾顆數時\n        print(f'{Q} ({clf.best_ntree_limit}), ',end='') \n        #拿表現最好的樹來預測\n        best_iteration = clf.get_booster().best_ntree_limit \n        oof.loc[val_user, Q] = clf.predict_proba(val_x, iteration_range=(0,best_iteration + 1))[:,1]\n        \n    print()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf.evals_result().keys()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv_metric = pd.DataFrame({\"train\":clf.evals_result()[\"validation_0\"][\"aucpr\"],\n                        \"validation\": clf.evals_result()[\"validation_1\"][\"aucpr\"],\n                        \"tree\": np.arange(1,len(clf.evals_result()[\"validation_0\"][\"aucpr\"])+1)})\nfig = px.line(cv_metric, x = \"tree\", y=[\"train\",\"validation\"])\nfig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 获取特征重要性得分\nfeature_importances = clf.feature_importances_\nfeature_importances_score = []\n# 输出各个特征的重要性得分\nfor i, score in enumerate(feature_importances):\n    feature_importances_score.append(score) \nFEA_IMP =pd.DataFrame({\"feature\": FEATURE, \"score\":feature_importances_score})\nFEA_IMP = FEA_IMP.sort_values(by=\"score\", ascending=False)\n\nprint(FEA_IMP.loc[FEA_IMP.score > 0])\npx.bar(FEA_IMP.loc[FEA_IMP.score > 0],x=\"feature\", y=\"score\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Find the best F1 score threshold","metadata":{}},{"cell_type":"code","source":"oof.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PUT TRUE LABELS INTO DATAFRAME WITH 18 COLUMNS\ntrue = oof.copy()\nfor Q in range(1,19):\n    # GET TRUE LABELS\n    tmp = target.loc[target.q == Q].set_index('user_id').loc[ALL_USER]\n    true[Q] = tmp.correct.values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"true.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores = []\nthresholds = []\nbest_score = 0; best_threshold = 0\n\nfor threshold in np.arange(0.4,0.81,0.01):\n    preds = (oof.values.reshape((-1))>threshold).astype('int')\n    m = f1_score(true.values.reshape((-1)), preds,average='macro') \n    scores.append(m.astype(\"float32\"))\n    thresholds.append(threshold.astype(\"float32\"))\n    \nbest_score = max(scores)\nbest_threshold = thresholds[scores.index(max(scores))]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize=(20,5))\nplt.plot(thresholds,scores,'-o',color='blue')\nplt.scatter(best_threshold, best_score, color='blue', s=300, alpha=1)\nplt.xlabel('Threshold',size=14)\nplt.ylabel('Validation F1 Score',size=14)\nplt.title(f'Threshold vs. F1_Score with Best F1_Score = {best_score:.3f} at Best Threshold = {best_threshold:.3}',size=18)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for Q in range(1,19):\n    # COMPUTE F1 SCORE PER QUESTION\n    m = f1_score(true[Q].values, (oof[Q].values>best_threshold).astype('int'),average='macro') \n    print(f'Q{Q}: F1 =',m)\n    \n# COMPUTE F1 SCORE OVERALL\nm = f1_score(true.values.reshape((-1)), (oof.values.reshape((-1))>best_threshold).astype('int'),average='macro')\nprint('Overall F1 =',m)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Infer test data","metadata":{}},{"cell_type":"code","source":"# IMPORT KAGGLE API\nimport jo_wilder\nenv = jo_wilder.make_env()\niter_test = env.iter_test()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"limits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\nfor i, (test, sample_submission) in enumerate(iter_test):\n    \n    # FEATURE ENGINEER TEST DATA\n    df = feature_engineer(test)\n    \n    # INFER TEST DATA\n#     print(i)\n    grp = test.level_group.values[0]\n    a,b = limits[grp]\n    for t in range(a,b):\n        clf = model[f'{grp}_{t}']\n        p = clf.predict_proba(df[FEATURE].astype('float32'))[0,1]\n        mask = sample_submission.session_id.str.contains(f'q{t}')\n        sample_submission.loc[mask,'correct'] = int( p > best_threshold )\n    \n    env.predict(sample_submission)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/test.csv\")\n# test.head(3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df = feature_engineer(test)\n# test_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample_submission = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/sample_submission.csv\")\n# tmp = sample_submission.session_id.str.split(\"_\", expand =True)\n# sample_submission[\"user_id\"] = tmp[0].astype(\"int\")\n# sample_submission[\"q\"] = tmp[1].str.slice(1).astype(\"int\")\n# sample_submission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # sub_df = pd.DataFrame(sample_submission.session_id)\n# ANS = []\n# for i in range(len(sample_submission)):\n#     level = sample_submission.iloc[i].session_level\n#     user = sample_submission.iloc[i].user_id\n#     q = sample_submission.iloc[i].q\n#     if level==0: grp = '0-4'\n#     elif level==1: grp = '5-12'\n#     elif level==2: grp = '13-22'\n    \n#     clf = model[f\"{grp}_{q}\"]\n#     proba = clf.predict_proba(test_df[(test_df.index == user) & (test_df.level_group == grp)][FEATURE])[0,1]\n#     ans = (proba > best_threshold).astype('int')\n#     ANS.append(ans)\n# #     print(ans)\n# #     sub_df.loc[sub_df.user_id == user, \"correct\"] = ans\n# sub_df = pd.DataFrame({\"session_id\": sample_submission.session_id, \"correct\":ANS})\n# sub_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sub_df.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}