{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Prepare Data for Processing","metadata":{"papermill":{"duration":0.007424,"end_time":"2023-03-18T02:15:30.456866","exception":false,"start_time":"2023-03-18T02:15:30.449442","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# libraries\nimport numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.020911,"end_time":"2023-03-18T02:15:30.484224","exception":false,"start_time":"2023-03-18T02:15:30.463313","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-23T16:47:45.109877Z","iopub.execute_input":"2023-04-23T16:47:45.110143Z","iopub.status.idle":"2023-04-23T16:47:45.137055Z","shell.execute_reply.started":"2023-04-23T16:47:45.110116Z","shell.execute_reply":"2023-04-23T16:47:45.136187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<b>Exploratory Data Analysis</b>\n\n- session_id - the ID of the session the event took place in\n- index - the index of the event for the session\n- elapsed_time - how much time has passed (in milliseconds) between the start of the session and when the event was recorded\n- event_name - the name of the event type\n- name - the event name (e.g. identifies whether a notebook_click is is opening or closing the notebook)\n- level - what level of the game the event occurred in (0 to 22)\n- page - the page number of the event (only for notebook-related events)\n- room_coor_x - the coordinates of the click in reference to the in-game room (only for click events)\n- room_coor_y - the coordinates of the click in reference to the in-game room (only for click events)\n- screen_coor_x - the coordinates of the click in reference to the player’s screen (only for click events)\n- screen_coor_y - the coordinates of the click in reference to the player’s screen (only for click events)\n- hover_duration - how long (in milliseconds) the hover happened for (only for hover events)\n- text - the text the player sees during this event\n- fqid - the fully qualified ID of the event\n- room_fqid - the fully qualified ID of the room the event took place in\n- text_fqid - the fully qualified ID of the\n- fullscreen - whether the player is in fullscreen mode\n- hq - whether the game is in high-quality\n- music - whether the game music is on or off\n- level_group - which group of levels - and group of questions - this row belongs to (0-4, 5-12, 13-22)","metadata":{}},{"cell_type":"markdown","source":"We have 7 object type ,3 int type and 9 float64 type\nThere are only 3 unique session_id on test dataset. Therefore, we need to predict 3 x 18(questions) = 54 rows. Each row is an action in each question of session_id.","metadata":{}},{"cell_type":"code","source":"colnames = ['session_id', 'index', 'elapsed_time', 'event_name', 'name', 'level',\n       'page', 'room_coor_x', 'room_coor_y', 'screen_coor_x', 'screen_coor_y',\n       'hover_duration', 'text', 'fqid', 'room_fqid', 'text_fqid',\n       'level_group']\ndtype = {'index':np.int16, 'level':np.int8, 'page':np.float32, \n         'room_coor_x':np.float32, 'room_coor_y':np.float32, \n         'screen_coor_x':np.float16, 'screen_coor_y':np.float16, \n         'hover_duration':np.float32, 'event_name':'category', \n         'name':'category', 'text':'category', 'fqid':'category', \n         'room_fqid':'category', 'level_group':'category'}\n\ntrain = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train.csv\",\n                    usecols=colnames, dtype=dtype)\nprint(\"The training data is: \", train.shape)\ntrain.head()\n","metadata":{"papermill":{"duration":70.183738,"end_time":"2023-03-18T02:16:40.674373","exception":false,"start_time":"2023-03-18T02:15:30.490635","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-23T16:47:45.140352Z","iopub.execute_input":"2023-04-23T16:47:45.140700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Load data with specific type to reduce the data need to load to RAM","metadata":{}},{"cell_type":"code","source":"# Train Labels are in a separate file\ntrain_label = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train_labels.csv\")\n# Separate labels into session_id and question number\ntrain_label['session'] = train_label.session_id.apply(lambda x: int(x.split('_')[0]) )\n# returns the last split item ([-1]) and remove the first character of that item ([1:]) which is q\ntrain_label['q'] = train_label.session_id.apply(lambda x: int(x.split('_')[-1][1:]) ) \n\nprint(\"Train_labels shape: \", train_label.shape )\ntrain_label.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def summary(df):\n    print(f'data shape: {df.shape}')\n    summ = pd.DataFrame(df.dtypes, columns=['data type'])\n    summ['#missing'] = df.isnull().sum().values * 100\n    summ['%missing'] = df.isnull().sum().values / len(df)\n    summ['#unique'] = df.nunique().values\n    desc = pd.DataFrame(df.describe(include='all').transpose())\n    summ['min'] = desc['min'].values\n    summ['max'] = desc['max'].values\n    summ['first value'] = df.loc[0].values\n    summ['second value'] = df.loc[1].values\n    summ['third value'] = df.loc[2].values\n    \n    return summ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary_table = summary(train)\nsummary_table","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data preprocessing\n","metadata":{"papermill":{"duration":0.00622,"end_time":"2023-03-18T02:16:41.986169","exception":false,"start_time":"2023-03-18T02:16:41.979949","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Separate numerical features (minus the target) and categorical features for processing\ncategorical_features = train.select_dtypes(include = [\"category\",\"object\",\"bool\"]).columns\nnumerical_features = train.select_dtypes(include = [\"int8\",\"int16\",\"int64\",\"float16\",\"float32\",\"float64\"]).columns\ncategorical_features = categorical_features.drop(\"level_group\")\nprint(\"Numerical features : \" + str(len(numerical_features)))\nprint(\"Categorical features : \" + str(len(categorical_features)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import OrdinalEncoder\ndef handle_missing(train):\n    # replace missing values for numerical by median\n    print(\"Missing values for numerical features: \" + str(train[numerical_features].isnull().values.sum()))\n    train[numerical_features] = train[numerical_features].fillna(train[numerical_features].median())\n    print(\"Remaining Missing values for numerical features: \" + str(train[numerical_features].isnull().values.sum()))\n    # encode categorical \n    encoder = OrdinalEncoder()\n    train[categorical_features] = encoder.fit_transform(train[categorical_features])\n    # replace missing values for categorical by ffill\n    print(\"NAs for categorical features: \" + str(train[categorical_features].isnull().values.sum()))\n    train[categorical_features] = train[categorical_features].fillna(method=\"ffill\")\n    print(\"Remaining NAs for categorical features: \" + str(train[categorical_features].isnull().values.sum()))\n    train[categorical_features] = train[categorical_features].fillna(0)\n    print(\"Remaining NAs for categorical features: \" + str(train[categorical_features].isnull().values.sum()))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"`ffill` is the method replaces the NULL values with the value from the previous row, it use in case of missing value is categorical","metadata":{}},{"cell_type":"code","source":"handle_missing(train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Feature Engineering\n# reference: https://www.kaggle.com/code/cdeotte/random-forest-baseline-0-664/notebook\ndef feature_engineer(train):\n    \n    dfs = []\n    for c in categorical_features:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('nunique')\n        tmp.name = tmp.name + '_nunique'\n        dfs.append(tmp)\n    for c in numerical_features:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('mean')\n        tmp.name = tmp.name + '_mean'\n        dfs.append(tmp)\n    for c in numerical_features:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('std')\n        tmp.name = tmp.name + '_std'\n        dfs.append(tmp)\n    for c in numerical_features:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('max')-train.groupby(['session_id','level_group'])[c].agg('min')\n        tmp.name = tmp.name + '_delta'\n        dfs.append(tmp)\n    for c in categorical_features:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('count')\n        tmp.name = tmp.name + '_count'\n        dfs.append(tmp)\n    for c in categorical_features:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('sum')\n        tmp.name = tmp.name + '_sum'\n        dfs.append(tmp)\n    \n        \n    df_engineered = pd.concat(dfs,axis=1)\n    df_engineered = df_engineered.fillna(-1)\n    df_engineered = df_engineered.reset_index()\n    df_engineered = df_engineered.set_index('session_id')\n    print(\"The dataset after after apply feature engineering: \", df_engineered.shape)\n    return df_engineered","metadata":{"papermill":{"duration":0.022874,"end_time":"2023-03-18T02:16:42.44758","exception":false,"start_time":"2023-03-18T02:16:42.424706","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Feature Engineer Train \ndf_tr = feature_engineer(train)","metadata":{"papermill":{"duration":19.676353,"end_time":"2023-03-18T02:17:02.130894","exception":false,"start_time":"2023-03-18T02:16:42.454541","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FEATURES = [c for c in df_tr.columns if c != 'level_group']\nprint('We will train with', len(FEATURES) ,'features')\nALL_USERS = df_tr.index.unique()\nprint('We will train with', len(ALL_USERS) ,'users info')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling","metadata":{"papermill":{"duration":0.006826,"end_time":"2023-03-18T02:17:02.191031","exception":false,"start_time":"2023-03-18T02:17:02.184205","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Choose best threhold based on F1 Score","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import cross_val_score, train_test_split, KFold, GroupKFold\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import f1_score\nfrom lightgbm import LGBMClassifier","metadata":{"papermill":{"duration":1.546432,"end_time":"2023-03-18T02:17:03.744374","exception":false,"start_time":"2023-03-18T02:17:02.197942","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define features \nFEATURES = [c for c in df_tr.columns if c != 'level_group']\nALL_USERS = df_tr.index.unique()\nprint('We will train with', len(FEATURES) ,'features and ', len(ALL_USERS) ,'users info')","metadata":{"papermill":{"duration":0.021756,"end_time":"2023-03-18T02:17:03.773157","exception":false,"start_time":"2023-03-18T02:17:03.751401","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model\ngkf = GroupKFold(n_splits=5)\noof = pd.DataFrame(data=np.zeros((len(ALL_USERS),18)), index=ALL_USERS)\nmodels = {}\n\n# COMPUTE CV SCORE WITH 5 GROUP K FOLD\nfor i, (train_index, test_index) in enumerate(gkf.split(X=df_tr, groups=df_tr.index)):\n    print('#'*25)\n    print('### Fold',i+1)\n    print('#'*25)\n    \n    lgb_params = {\n        'objective' : 'binary',\n        'metric' : 'auc',\n        'learning_rate': 0.002,\n        'max_depth': 6,\n        'num_iterations': 1000\n    }\n    # ITERATE THRU QUESTIONS 1 THRU 18\n    for t in range(1,19):\n        print(t,', ',end='')\n        \n        # USE THIS TRAIN DATA WITH THESE QUESTIONS\n        if t<=3: grp = '0-4'\n        elif t<=13: grp = '5-12'\n        elif t<=22: grp = '13-22'\n            \n        # TRAIN DATA\n        train_x = df_tr.iloc[train_index]\n        train_x = train_x.loc[train_x.level_group == grp]\n        train_users = train_x.index.values\n        train_y = train_label.loc[train_label.q==t].set_index('session').loc[train_users]\n        \n        # VALID DATA\n        valid_x = df_tr.iloc[test_index]\n        valid_x = valid_x.loc[valid_x.level_group == grp]\n        valid_users = valid_x.index.values\n        valid_y = train_label.loc[train_label.q==t].set_index('session').loc[valid_users]\n        \n        # TRAIN MODEL\n        clf = LGBMClassifier(**lgb_params)\n        clf.fit(train_x[FEATURES].astype('float32'), train_y['correct'],\n                eval_set=[ (valid_x[FEATURES].astype('float32'), valid_y['correct']) ],\n                verbose=0)\n                \n        # SAVE MODEL, PREDICT VALID OOF\n        models[f'{grp}_{t}'] = clf\n        oof.loc[valid_users, t-1] = clf.predict_proba(valid_x[FEATURES].astype('float32'))[:,1]\n        \n    print()","metadata":{"papermill":{"duration":79.747021,"end_time":"2023-03-18T02:18:23.527417","exception":false,"start_time":"2023-03-18T02:17:03.780396","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PUT TRUE LABELS INTO DATAFRAME WITH 18 COLUMNS\ntrue = oof.copy()\nfor k in range(18):\n    # GET TRUE LABELS\n    tmp = train_label.loc[train_label.q == k+1].set_index('session').loc[ALL_USERS]\n    true[k] = tmp.correct.values","metadata":{"papermill":{"duration":0.108117,"end_time":"2023-03-18T02:18:23.64842","exception":false,"start_time":"2023-03-18T02:18:23.540303","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# FIND BEST THRESHOLD TO CONVERT PROBS INTO 1s AND 0s\nscores = []; thresholds = []\nbest_score = 0; best_threshold = 0\n\nfor threshold in np.arange(0.4,0.81,0.01):\n    print(f'{threshold:.02f}, ',end='')\n    preds = (oof.values.reshape((-1))>threshold).astype('int')\n    m = f1_score(true.values.reshape((-1)), preds, average='macro')   \n    scores.append(m)\n    thresholds.append(threshold)\n    if m>best_score:\n        best_score = m\n        best_threshold = threshold","metadata":{"papermill":{"duration":3.424247,"end_time":"2023-03-18T02:18:27.085158","exception":false,"start_time":"2023-03-18T02:18:23.660911","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PLOT THRESHOLD VS. F1_SCORE\nplt.figure(figsize=(20,5))\nplt.plot(thresholds,scores,'-o',color='blue')\nplt.scatter([best_threshold], [best_score], color='blue', s=300, alpha=1)\nplt.xlabel('Threshold',size=14)\nplt.ylabel('Validation F1 Score',size=14)\nplt.title(f'Threshold vs. F1_Score with Best F1_Score = {best_score:.3f} at Best Threshold = {best_threshold:.3}',size=18)\nplt.show()\n","metadata":{"papermill":{"duration":0.323364,"end_time":"2023-03-18T02:18:27.423339","exception":false,"start_time":"2023-03-18T02:18:27.099975","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print('When using optimal threshold...')\n\nfor k in range(18):\n        \n    # COMPUTE F1 SCORE PER QUESTION\n    m = f1_score(true[k].values, (oof[k].values>best_threshold).astype('int'), average='macro')\n    print(f'Q{k}: F1 =',m)\n    \n# COMPUTE F1 SCORE OVERALL\nm = f1_score(true.values.reshape((-1)), (oof.values.reshape((-1))>best_threshold).astype('int'), average='macro')\nprint('==> Overall F1 =',m)\n","metadata":{"papermill":{"duration":0.196045,"end_time":"2023-03-18T02:18:27.636692","exception":false,"start_time":"2023-03-18T02:18:27.440647","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{"papermill":{"duration":0.014939,"end_time":"2023-03-18T02:18:27.667699","exception":false,"start_time":"2023-03-18T02:18:27.65276","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Import Competition Specific Kaggle API\nimport jo_wilder\n#jo_wilder.make_env.__called__ = False  # use during development\nenv = jo_wilder.make_env() # You can only call make_env() once unless Notebook is restarted\niter_test = env.iter_test() # You can only iterate through a result from `env.iter_test()` once","metadata":{"papermill":{"duration":0.046784,"end_time":"2023-03-18T02:18:27.729718","exception":false,"start_time":"2023-03-18T02:18:27.682934","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nlimits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\nfor (test, sample_submission) in iter_test:\n    \n    handle_missing(test)    \n    df = feature_engineer(test)\n    grp = test.level_group.values[0]\n    a,b = limits[grp]\n    for t in range(a,b):\n        clf = models[f'{grp}_{t}']\n        p = clf.predict_proba(df[FEATURES].astype('float32'))[:,1]\n        pint = [int(x>best_threshold) for x in p ]\n        mask = sample_submission.session_id.str.endswith(f'q{t}')\n        sample_submission.loc[mask,'correct'] = pint\n    \n    env.predict(sample_submission)\n\nprint(\"Your submission was successfully saved!\")","metadata":{"papermill":{"duration":0.729821,"end_time":"2023-03-18T02:18:28.474724","exception":false,"start_time":"2023-03-18T02:18:27.744903","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit = pd.read_csv('submission.csv')\nsubmit ","metadata":{"papermill":{"duration":0.036251,"end_time":"2023-03-18T02:18:28.527567","exception":false,"start_time":"2023-03-18T02:18:28.491316","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit.correct.mean()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reference\nhttps://www.kaggle.com/code/cdeotte/xgboost-baseline-0-676#XGBoost-Baseline---LB-0.678 <br>\nhttps://www.kaggle.com/code/kimtaehun/lightgbm-baseline-with-aggregated-log-data","metadata":{}}]}