{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":45533,"databundleVersionId":5748852,"sourceType":"competition"}],"dockerImageVersionId":30458,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🎉Purpose","metadata":{}},{"cell_type":"markdown","source":"Making **xgboost** model for student performance with **grid search**.  \n\nGrid search is useful for triyng many patterns of hyperparameter.","metadata":{}},{"cell_type":"markdown","source":"I also explain about Gridsearch in detail [here](https://www.kaggle.com/code/yutodennou/make-gridsearchcv-faster)  \n","metadata":{}},{"cell_type":"markdown","source":"# 🗃️Import Library","metadata":{}},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2023-11-07T15:29:33.570808Z","iopub.execute_input":"2023-11-07T15:29:33.571085Z","iopub.status.idle":"2023-11-07T15:29:33.619704Z","shell.execute_reply.started":"2023-11-07T15:29:33.571057Z","shell.execute_reply":"2023-11-07T15:29:33.618724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import OrdinalEncoder\nfrom sklearn.model_selection import cross_val_score, train_test_split, KFold, GroupKFold, StratifiedKFold, GridSearchCV\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import f1_score","metadata":{"papermill":{"duration":0.022197,"end_time":"2023-05-07T06:40:09.476077","exception":false,"start_time":"2023-05-07T06:40:09.453880","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-07T15:29:33.621164Z","iopub.execute_input":"2023-11-07T15:29:33.621442Z","iopub.status.idle":"2023-11-07T15:29:34.755137Z","shell.execute_reply.started":"2023-11-07T15:29:33.621414Z","shell.execute_reply":"2023-11-07T15:29:34.753838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🔍Import data","metadata":{}},{"cell_type":"code","source":"COLS = ['session_id', 'index', 'elapsed_time', 'event_name', 'name', 'level',\n       'page', 'room_coor_x', 'room_coor_y', 'screen_coor_x', 'screen_coor_y',\n       'hover_duration', 'text', 'fqid', 'room_fqid', 'text_fqid',\n       'level_group']","metadata":{"_kg_hide-output":true,"papermill":{"duration":0.015305,"end_time":"2023-05-07T06:40:09.497651","exception":false,"start_time":"2023-05-07T06:40:09.482346","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-07T15:29:34.757997Z","iopub.execute_input":"2023-11-07T15:29:34.759235Z","iopub.status.idle":"2023-11-07T15:29:34.765300Z","shell.execute_reply.started":"2023-11-07T15:29:34.759191Z","shell.execute_reply":"2023-11-07T15:29:34.763739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train.csv\", usecols=COLS, \n                dtype = {'index':np.int16, 'level':np.int8, 'page':np.float32, 'room_coor_x':np.float32, \n                        'room_coor_y':np.float32, 'screen_coor_x':np.float16, 'screen_coor_y':np.float16, \n                        'hover_duration':np.float32, 'event_name':'category', 'name':'category', \n                        'text':'category', 'fqid':'category', 'room_fqid':'category', \n                        'level_group':'category'})","metadata":{"papermill":{"duration":125.974098,"end_time":"2023-05-07T06:42:15.477805","exception":false,"start_time":"2023-05-07T06:40:09.503707","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-07T15:29:34.767057Z","iopub.execute_input":"2023-11-07T15:29:34.767792Z","iopub.status.idle":"2023-11-07T15:31:19.119662Z","shell.execute_reply.started":"2023-11-07T15:29:34.767742Z","shell.execute_reply":"2023-11-07T15:31:19.118518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train_labels.csv\")\ntrain_label['session'] = train_label.session_id.apply(lambda x: int(x.split('_')[0]))\ntrain_label['q'] = train_label.session_id.apply(lambda x: int(x.split('_')[-1][1:]))","metadata":{"papermill":{"duration":1.166126,"end_time":"2023-05-07T06:42:16.650406","exception":false,"start_time":"2023-05-07T06:42:15.484280","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-07T15:31:19.122506Z","iopub.execute_input":"2023-11-07T15:31:19.122880Z","iopub.status.idle":"2023-11-07T15:31:20.434737Z","shell.execute_reply.started":"2023-11-07T15:31:19.122846Z","shell.execute_reply":"2023-11-07T15:31:20.433859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categorical_features = train.select_dtypes(include = [\"category\",\"object\",\"bool\"]).columns\ncategorical_features = categorical_features.drop(\"level_group\")\nnumerical_features = train.select_dtypes(include = [\"int8\",\"int16\",\"int64\",\"float16\",\"float32\",\"float64\"]).columns","metadata":{"papermill":{"duration":1.245888,"end_time":"2023-05-07T06:42:17.902649","exception":false,"start_time":"2023-05-07T06:42:16.656761","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-07T15:31:20.436129Z","iopub.execute_input":"2023-11-07T15:31:20.436445Z","iopub.status.idle":"2023-11-07T15:31:21.581119Z","shell.execute_reply.started":"2023-11-07T15:31:20.436415Z","shell.execute_reply":"2023-11-07T15:31:21.580220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📏Preprocessing","metadata":{}},{"cell_type":"code","source":"def processing_na(train):\n    print(\"Na of numerical features: \" + str(train[numerical_features].isnull().values.sum()))\n    train[numerical_features] = train[numerical_features].fillna(train[numerical_features].median())\n    print(\"-> \" + str(train[numerical_features].isnull().values.sum()))\n    encoder = OrdinalEncoder()\n    train[categorical_features] = encoder.fit_transform(train[categorical_features])\n    print(\"Na of categorical features: \" + str(train[categorical_features].isnull().values.sum()))\n    train[categorical_features] = train[categorical_features].fillna(method=\"ffill\")\n    print(\"-> \" + str(train[categorical_features].isnull().values.sum()))\n    return train ","metadata":{"papermill":{"duration":1.092844,"end_time":"2023-05-07T06:42:19.002311","exception":false,"start_time":"2023-05-07T06:42:17.909467","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-07T15:31:21.582370Z","iopub.execute_input":"2023-11-07T15:31:21.582704Z","iopub.status.idle":"2023-11-07T15:31:21.591056Z","shell.execute_reply.started":"2023-11-07T15:31:21.582663Z","shell.execute_reply":"2023-11-07T15:31:21.590014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def processing_xy(train):\n    train['room_coor_d'] = np.sqrt(train['room_coor_x']**2+train['room_coor_x']**2)\n    train['screen_coor_d'] = np.sqrt(train['screen_coor_x']**2+train['screen_coor_x']**2)\n    return train","metadata":{"papermill":{"duration":5.434328,"end_time":"2023-05-07T06:43:34.652109","exception":false,"start_time":"2023-05-07T06:43:29.217781","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-07T15:31:21.592471Z","iopub.execute_input":"2023-11-07T15:31:21.592849Z","iopub.status.idle":"2023-11-07T15:31:21.603827Z","shell.execute_reply.started":"2023-11-07T15:31:21.592812Z","shell.execute_reply":"2023-11-07T15:31:21.602953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineer(train):\n    dfs = []\n    for c in categorical_features:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('nunique')\n        tmp.name = tmp.name + '_nunique'\n        dfs.append(tmp)\n    for c in numerical_features:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('mean')\n        tmp.name = tmp.name + '_mean'\n        dfs.append(tmp)\n    for c in numerical_features:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('std')\n        tmp.name = tmp.name + '_std'\n        dfs.append(tmp)\n    for c in numerical_features:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('max')-train.groupby(['session_id','level_group'])[c].agg('min')\n        tmp.name = tmp.name + '_delta'\n        dfs.append(tmp)\n    for c in categorical_features:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('count')\n        tmp.name = tmp.name + '_count'\n        dfs.append(tmp)\n    for c in categorical_features:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('sum')\n        tmp.name = tmp.name + '_sum'\n        dfs.append(tmp)\n    \n        \n    df_engineered = pd.concat(dfs,axis=1)\n    df_engineered = df_engineered.fillna(-1)\n    df_engineered = df_engineered.reset_index()\n    df_engineered = df_engineered.set_index('session_id')\n    return df_engineered","metadata":{"papermill":{"duration":0.022502,"end_time":"2023-05-07T06:43:34.682680","exception":false,"start_time":"2023-05-07T06:43:34.660178","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-07T15:31:21.604999Z","iopub.execute_input":"2023-11-07T15:31:21.605289Z","iopub.status.idle":"2023-11-07T15:31:21.618316Z","shell.execute_reply.started":"2023-11-07T15:31:21.605261Z","shell.execute_reply":"2023-11-07T15:31:21.617143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = processing_na(train)\ntrain = processing_xy(train)\ndf_tr = feature_engineer(train)","metadata":{"papermill":{"duration":88.430547,"end_time":"2023-05-07T06:45:03.335213","exception":false,"start_time":"2023-05-07T06:43:34.904666","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-07T15:31:21.619886Z","iopub.execute_input":"2023-11-07T15:31:21.620452Z","iopub.status.idle":"2023-11-07T15:34:07.831615Z","shell.execute_reply.started":"2023-11-07T15:31:21.620416Z","shell.execute_reply":"2023-11-07T15:34:07.830726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FEATURES = [c for c in df_tr.columns if c != 'level_group']\nALL_USERS = df_tr.index.unique()\nprint(len(FEATURES) ,'features, ', len(ALL_USERS) ,'users')","metadata":{"papermill":{"duration":0.02207,"end_time":"2023-05-07T06:45:03.363810","exception":false,"start_time":"2023-05-07T06:45:03.341740","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-07T15:34:07.834775Z","iopub.execute_input":"2023-11-07T15:34:07.835088Z","iopub.status.idle":"2023-11-07T15:34:07.843837Z","shell.execute_reply.started":"2023-11-07T15:34:07.835059Z","shell.execute_reply":"2023-11-07T15:34:07.842734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 💪Train Data","metadata":{}},{"cell_type":"markdown","source":"## Gridsearch","metadata":{}},{"cell_type":"markdown","source":"Gridsearch is a computationally heavy process. It often takes a tremendous amount of processing time depending on the data and the algorithm. We also know it's because exploring all options like below.\n```\nparams = {'eta': [0.01, 0.1, 1.0], 'gamma': [0.1, 0.5, 0.8], \n                  'n_estimators': [10, 100, 500], 'max_depth':[2, 4, 6], \n                  'min_child_weight': [1, 2], 'nthread': [2] }\n```","metadata":{}},{"cell_type":"code","source":"gkf = GroupKFold(n_splits=5)\nskf = StratifiedKFold(n_splits=2, shuffle=True, random_state=1)\noof = pd.DataFrame(data=np.zeros((len(ALL_USERS),18)), index=ALL_USERS)\nmodels = {}\nxgb_params = {\n    'objective' : 'binary:logistic',\n    'eval_metric':'logloss',\n}\n\n# Any Params You Want to Scan like below\n# 'eta':[0.01, 0.1, 1.0]\n# 'gammma':[0.1, 0.4]\n# 'learning_rate':[0.04, 0.06]\n# 'n_estimator': [1500, 2000, 2500]\n# 'max_depth': [5,6]\nparams = {'eta': [0.01, 0.1], 'gamma': [0.1, 0.4], 'learning_rate': [0.06],\n                  'n_estimators': [1000, 2000, 3000], 'max_depth':[5, 6, 7], \n                  'min_child_weight': [1, 2], 'nthread': [2] }\nfor i, (train_index, test_index) in enumerate(gkf.split(X=df_tr, groups=df_tr.index)):\n    print('-'*25)\n    print(i)\n    # 1->18\n    for t in range(1,19):\n        print(t,', ',end='')\n        \n        if t<=3: grp = '0-4'\n        elif t<=13: grp = '5-12'\n        elif t<=22: grp = '13-22'\n            \n        # Train\n        train_x = df_tr.iloc[train_index]\n        train_x = train_x.loc[train_x.level_group == grp]\n        train_users = train_x.index.values\n        train_y = train_label.loc[train_label.q==t].set_index('session').loc[train_users]\n        \n        # Valid\n        valid_x = df_tr.iloc[test_index]\n        valid_x = valid_x.loc[valid_x.level_group == grp]\n        valid_users = valid_x.index.values\n        valid_y = train_label.loc[train_label.q==t].set_index('session').loc[valid_users]\n        \n        # Model\n        model =  XGBClassifier(**xgb_params)\n        clf = GridSearchCV(estimator=model, param_grid=params, \n                    cv=skf, scoring=\"accuracy\", n_jobs=1, verbose=2, return_train_score=False)\n        clf.fit(train_x[FEATURES].astype('float32'), train_y['correct'],\n                eval_set=[ (valid_x[FEATURES].astype('float32'), valid_y['correct']) ],\n                verbose=0)\n                \n        models[f'{grp}_{t}'] = clf\n        oof.loc[valid_users, t-1] = clf.predict_proba(valid_x[FEATURES].astype('float32'))[:,1]\n","metadata":{"_kg_hide-output":true,"papermill":{"duration":0.013249,"end_time":"2023-05-07T06:53:03.704322","exception":false,"start_time":"2023-05-07T06:53:03.691073","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-07T15:34:07.845168Z","iopub.execute_input":"2023-11-07T15:34:07.845451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"true = oof.copy()\nfor k in range(18):\n    tmp = train_label.loc[train_label.q == k+1].set_index('session').loc[ALL_USERS]\n    true[k] = tmp.correct.values","metadata":{"papermill":{"duration":0.154275,"end_time":"2023-05-07T06:53:03.871715","exception":false,"start_time":"2023-05-07T06:53:03.717440","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores = []; thresholds = []\nbest_score = 0; best_threshold = 0\n\nfor threshold in np.arange(0.4,0.81,0.01):\n    print(f'{threshold:.02f}, ',end='')\n    preds = (oof.values.reshape((-1))>threshold).astype('int')\n    m = f1_score(true.values.reshape((-1)), preds, average='macro')   \n    scores.append(m)\n    thresholds.append(threshold)\n    if m>best_score:\n        best_score = m\n        best_threshold = threshold","metadata":{"papermill":{"duration":6.645241,"end_time":"2023-05-07T06:53:10.530466","exception":false,"start_time":"2023-05-07T06:53:03.885225","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k in range(18):\n    m = f1_score(true[k].values, (oof[k].values>best_threshold).astype('int'), average='macro')\n    print(f'Q{k}: F1 =',m)\n    \nm = f1_score(true.values.reshape((-1)), (oof.values.reshape((-1))>best_threshold).astype('int'), average='macro')\nprint('-> All F1 =',m)","metadata":{"papermill":{"duration":0.339855,"end_time":"2023-05-07T06:53:10.884239","exception":false,"start_time":"2023-05-07T06:53:10.544384","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🥅Submit","metadata":{}},{"cell_type":"code","source":"import jo_wilder\ntry:\n    jo_wilder.make_env.__called__ = False\n    env.__called__ = False\n    type(env)._state = type(type(env)._state).__dict__['INIT']\nexcept:\n    pass\n\nenv = jo_wilder.make_env()\niter_test = env.iter_test() ","metadata":{"papermill":{"duration":0.041955,"end_time":"2023-05-07T06:53:10.940385","exception":false,"start_time":"2023-05-07T06:53:10.898430","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"limits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\nfor (test, sample_submission) in iter_test:\n    test = processing_na(test)\n    test = processing_xy(test)\n    df = feature_engineer(test)\n    grp = test.level_group.values[0]\n    a,b = limits[grp]\n    for t in range(a,b):\n        clf = models[f'{grp}_{t}']\n        p = clf.predict_proba(df[FEATURES].astype('float32'))[:,1]\n        pint = [int(x>best_threshold) for x in p ]\n        mask = sample_submission.session_id.str.endswith(f'q{t}')\n        sample_submission.loc[mask,'correct'] = pint\n    \n    env.predict(sample_submission)","metadata":{"papermill":{"duration":1.336132,"end_time":"2023-05-07T06:53:12.290941","exception":false,"start_time":"2023-05-07T06:53:10.954809","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit = pd.read_csv('submission.csv')\nsubmit.head(30)","metadata":{"papermill":{"duration":0.047046,"end_time":"2023-05-07T06:53:12.354143","exception":false,"start_time":"2023-05-07T06:53:12.307097","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}