{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# LGBM + Early Stopping\nThis is a modified Version of Chris's notebook:\nhttps://www.kaggle.com/code/cdeotte/random-forest-baseline-0-664/notebook\n\nModifications Are:\n\n1- Use LGBM with Early Stopping\n\nSubmission takes about 40 minutes.\n\nPlease upvote if you found it useful!","metadata":{"papermill":{"duration":0.005932,"end_time":"2023-02-07T00:59:58.147501","exception":false,"start_time":"2023-02-07T00:59:58.141569","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import pandas as pd, numpy as np\nfrom sklearn.model_selection import KFold, GroupKFold\nimport lightgbm as lgb\nfrom sklearn.metrics import f1_score\nfrom tqdm import tqdm","metadata":{"papermill":{"duration":1.027875,"end_time":"2023-02-07T00:59:59.180261","exception":false,"start_time":"2023-02-07T00:59:58.152386","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-21T05:32:05.677931Z","iopub.execute_input":"2023-05-21T05:32:05.678386Z","iopub.status.idle":"2023-05-21T05:32:09.316897Z","shell.execute_reply.started":"2023-05-21T05:32:05.678296Z","shell.execute_reply":"2023-05-21T05:32:09.315695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Reduce Memory Usage\ndef reduce_memory_usage(df, categ = True):\n    \n    for col in df.columns:\n        col_type = df[col].dtype.name\n        if ((col_type != 'datetime64[ns]') & (col_type != 'category')):\n            if (col_type != 'object'):\n                c_min = df[col].min()\n                c_max = df[col].max()\n\n                if str(col_type)[:3] == 'int':\n                    if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                        df[col] = df[col].astype(np.int8)\n                    elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                        df[col] = df[col].astype(np.int16)\n                    elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                        df[col] = df[col].astype(np.int32)\n                    elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                        df[col] = df[col].astype(np.int64)\n\n                else:\n                    if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                        df[col] = df[col].astype(np.float16)\n                    elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                        df[col] = df[col].astype(np.float32)\n                    else:\n                        pass\n            else:\n                if categ:\n                    df[col] = df[col].astype('category')\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2023-05-21T05:32:09.319528Z","iopub.execute_input":"2023-05-21T05:32:09.320264Z","iopub.status.idle":"2023-05-21T05:32:09.341436Z","shell.execute_reply.started":"2023-05-21T05:32:09.320221Z","shell.execute_reply":"2023-05-21T05:32:09.340295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Train Data and Labels","metadata":{"papermill":{"duration":0.004542,"end_time":"2023-02-07T00:59:59.189777","exception":false,"start_time":"2023-02-07T00:59:59.185235","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv')\ntrain = reduce_memory_usage(train, categ = False)\nprint( train.shape )\ntrain.head()","metadata":{"papermill":{"duration":59.284316,"end_time":"2023-02-07T01:00:58.478743","exception":false,"start_time":"2023-02-07T00:59:59.194427","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-21T05:32:09.343576Z","iopub.execute_input":"2023-05-21T05:32:09.344058Z","iopub.status.idle":"2023-05-21T05:34:01.318806Z","shell.execute_reply.started":"2023-05-21T05:32:09.344020Z","shell.execute_reply":"2023-05-21T05:34:01.317494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\ntargets['session'] = targets.session_id.apply(lambda x: int(x.split('_')[0]) )\ntargets['q'] = targets.session_id.apply(lambda x: int(x.split('_')[-1][1:]) )\nprint( targets.shape )\ntargets.head()","metadata":{"papermill":{"duration":0.598155,"end_time":"2023-02-07T01:00:59.082015","exception":false,"start_time":"2023-02-07T01:00:58.48386","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-21T05:34:01.321779Z","iopub.execute_input":"2023-05-21T05:34:01.322676Z","iopub.status.idle":"2023-05-21T05:34:02.724879Z","shell.execute_reply.started":"2023-05-21T05:34:01.322630Z","shell.execute_reply":"2023-05-21T05:34:02.723839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineer\nWe create basic aggregate features. Try creating more features to boost CV and LB!","metadata":{"papermill":{"duration":0.005196,"end_time":"2023-02-07T01:00:59.092865","exception":false,"start_time":"2023-02-07T01:00:59.087669","status":"completed"},"tags":[]}},{"cell_type":"code","source":"CATS = ['event_name','fqid', 'room_fqid', 'text_fqid']\nNUMS = ['elapsed_time','level','page','room_coor_x', 'room_coor_y', \n        'screen_coor_x', 'screen_coor_y', 'hover_duration']","metadata":{"papermill":{"duration":0.014685,"end_time":"2023-02-07T01:00:59.112856","exception":false,"start_time":"2023-02-07T01:00:59.098171","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-21T05:34:02.726500Z","iopub.execute_input":"2023-05-21T05:34:02.727224Z","iopub.status.idle":"2023-05-21T05:34:02.735332Z","shell.execute_reply.started":"2023-05-21T05:34:02.727177Z","shell.execute_reply":"2023-05-21T05:34:02.733719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineer(_dataset):\n    \n    _dataset['ID'] = _dataset['session_id'].astype(str) + _dataset['level_group'].astype(str)    \n    dataset = pd.DataFrame(index=_dataset.groupby('ID').last().index)\n\n    feat_1 = 'ID'\n    \n    for Feature in tqdm(NUMS):\n        dataset[f'{Feature}_Agg_{feat_1}_mean'] = _dataset.groupby(feat_1)[Feature].mean()\n        dataset[f'{Feature}_Agg_{feat_1}_std'] = _dataset.groupby(feat_1)[Feature].std()\n        dataset[f'{Feature}_Agg_{feat_1}_min'] = _dataset.groupby(feat_1)[Feature].min()\n        dataset[f'{Feature}_Agg_{feat_1}_max'] = _dataset.groupby(feat_1)[Feature].max()\n        dataset = reduce_memory_usage(dataset, categ = True)\n        _dataset.drop(Feature,inplace=True,axis=1)\n        \n        \n    for Feature in tqdm(CATS):\n        dataset[f'{Feature}_Agg_{feat_1}_nunique'] = _dataset.groupby(feat_1)[Feature].nunique()\n        dataset[f'{Feature}_Agg_{feat_1}_count'] = _dataset.groupby(feat_1)[Feature].count()\n        dataset = reduce_memory_usage(dataset, categ = True)\n        _dataset.drop(Feature,inplace=True,axis=1)\n        \n    dataset['session_id'] = _dataset.groupby('ID')['session_id'].last()\n    dataset['level_group'] = _dataset.groupby('ID')['level_group'].last()\n    del _dataset    \n    dataset = dataset.fillna(-1)\n    dataset = dataset.reset_index()\n    dataset = dataset.set_index('session_id')\n\n\n    return dataset\n            ","metadata":{"execution":{"iopub.status.busy":"2023-05-21T05:34:02.737621Z","iopub.execute_input":"2023-05-21T05:34:02.740339Z","iopub.status.idle":"2023-05-21T05:34:02.755105Z","shell.execute_reply.started":"2023-05-21T05:34:02.740307Z","shell.execute_reply":"2023-05-21T05:34:02.754014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def feature_engineer(train):\n#     dfs = []\n#     for c in CATS:\n#         tmp = train.groupby(['session_id','level_group'])[c].agg('nunique')\n#         tmp.name = tmp.name + '_nunique'\n#         dfs.append(tmp)\n#     for c in NUMS:\n#         tmp = train.groupby(['session_id','level_group'])[c].agg('mean')\n#         dfs.append(tmp)\n#     for c in NUMS:\n#         tmp = train.groupby(['session_id','level_group'])[c].agg('std')\n#         tmp.name = tmp.name + '_std'\n#         dfs.append(tmp)\n#     df = pd.concat(dfs,axis=1)\n#     df = df.fillna(-1)\n#     df = df.reset_index()\n#     df = df.set_index('session_id')\n#     return df","metadata":{"papermill":{"duration":0.017716,"end_time":"2023-02-07T01:00:59.136021","exception":false,"start_time":"2023-02-07T01:00:59.118305","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-21T05:34:02.758800Z","iopub.execute_input":"2023-05-21T05:34:02.759096Z","iopub.status.idle":"2023-05-21T05:34:02.766258Z","shell.execute_reply.started":"2023-05-21T05:34:02.759069Z","shell.execute_reply":"2023-05-21T05:34:02.765282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf = feature_engineer(train)\ndf.replace([np.inf, -np.inf], np.nan, inplace=True)\ndf.fillna(-1,inplace=True)\nprint( df.shape )\ndf.head()","metadata":{"papermill":{"duration":34.516494,"end_time":"2023-02-07T01:01:33.658043","exception":false,"start_time":"2023-02-07T01:00:59.141549","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-21T05:34:02.768639Z","iopub.execute_input":"2023-05-21T05:34:02.769164Z","iopub.status.idle":"2023-05-21T05:38:17.203478Z","shell.execute_reply.started":"2023-05-21T05:34:02.769127Z","shell.execute_reply":"2023-05-21T05:38:17.202441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train LGBM Model\nWe train one model for each of 18 questions. Furthermore, we use data from `level_groups = '0-4'` to train model for questions 1-3, and `level groups '5-12'` to train questions 4 thru 13 and `level groups '13-22'` to train questions 14 thru 18. Because this is the data we get (to predict corresponding questions) from Kaggle's inference API during test inference. We can improve our model by saving a user's previous data from earlier `level_groups` and using that to predict future `level_groups`.","metadata":{"papermill":{"duration":0.00565,"end_time":"2023-02-07T01:01:33.669525","exception":false,"start_time":"2023-02-07T01:01:33.663875","status":"completed"},"tags":[]}},{"cell_type":"code","source":"FEATURES = [c for c in df.columns if c not in ['level_group','ID']]\nprint('We will train with', len(FEATURES) ,'features')\nALL_USERS = df.index.unique()\nprint('We will train with', len(ALL_USERS) ,'users info')","metadata":{"papermill":{"duration":0.014699,"end_time":"2023-02-07T01:01:33.689953","exception":false,"start_time":"2023-02-07T01:01:33.675254","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-21T05:38:17.205013Z","iopub.execute_input":"2023-05-21T05:38:17.206879Z","iopub.status.idle":"2023-05-21T05:38:17.220132Z","shell.execute_reply.started":"2023-05-21T05:38:17.206827Z","shell.execute_reply":"2023-05-21T05:38:17.219186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gkf = GroupKFold(n_splits=5)\noof = pd.DataFrame(data=np.zeros((len(ALL_USERS),18)), index=ALL_USERS)\nmodels = {}\n\n# COMPUTE CV SCORE WITH 5 GROUP K FOLD\nfor i, (train_index, test_index) in enumerate(gkf.split(X=df, groups=df.index)):\n    print('#'*25)\n    print('### Fold',i+1)\n    print('#'*25)\n    \n    # ITERATE THRU QUESTIONS 1 THRU 18\n    for t in range(1,19):\n        print(t,', ',end='')\n        \n        # USE THIS TRAIN DATA WITH THESE QUESTIONS\n        if t<=3: grp = '0-4'\n        elif t<=13: grp = '5-12'\n        elif t<=22: grp = '13-22'\n            \n        # TRAIN DATA\n        train_x = df.iloc[train_index]\n        train_x = train_x.loc[train_x.level_group == grp]\n        train_users = train_x.index.values\n        train_y = targets.loc[targets.q==t].set_index('session').loc[train_users]\n        \n        # VALID DATA\n        valid_x = df.iloc[test_index]\n        valid_x = valid_x.loc[valid_x.level_group == grp]\n        valid_users = valid_x.index.values\n        valid_y = targets.loc[targets.q==t].set_index('session').loc[valid_users]\n        \n        # TRAIN MODEL\n        params = {'objective':'binary', 'metric':'binary_logloss',\n                  'learning_rate': 0.002,'max_depth':4, 'subsample':0.8, 'feature_fraction':0.4,\n                   'random_state':42,\n                  'verbose':-1}\n        callbacks = [lgb.early_stopping(200, verbose=False), lgb.log_evaluation(period=0)]\n        \n        lgb_train = lgb.Dataset(train_x[FEATURES].astype('float32'), train_y['correct'].values)\n        lgb_eval = lgb.Dataset(valid_x[FEATURES].astype('float32'), valid_y['correct'].values)\n        clf = lgb.train(params, train_set=lgb_train, valid_sets=[lgb_eval],\n                              callbacks=callbacks,\n                              num_boost_round=3000\n                              )         \n        # SAVE MODEL, PREDICT VALID OOF\n        models[f'{grp}_{t}'] = clf\n        oof.loc[valid_users, t-1] = clf.predict(valid_x[FEATURES].astype('float32'))\n        \n    print()","metadata":{"papermill":{"duration":69.877213,"end_time":"2023-02-07T01:02:43.57299","exception":false,"start_time":"2023-02-07T01:01:33.695777","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-21T05:38:17.224578Z","iopub.execute_input":"2023-05-21T05:38:17.224867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"5","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Compute CV Score\nWe need to convert prediction probabilities into `1s` and `0s`. The competition metric is F1 Score which is the harmonic mean of precision and recall. Let's find the optimal threshold for `p > threshold` when to predict `1` and when to predict `0` to maximize F1 Score.","metadata":{"papermill":{"duration":0.011241,"end_time":"2023-02-07T01:02:43.59638","exception":false,"start_time":"2023-02-07T01:02:43.585139","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# PUT TRUE LABELS INTO DATAFRAME WITH 18 COLUMNS\ntrue = oof.copy()\nfor k in range(18):\n    # GET TRUE LABELS\n    tmp = targets.loc[targets.q == k+1].set_index('session').loc[ALL_USERS]\n    true[k] = tmp.correct.values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# FIND BEST THRESHOLD TO CONVERT PROBS INTO 1s AND 0s\nscores = []; thresholds = []\nbest_score = 0; best_threshold = 0\n\nfor threshold in np.arange(0.4,0.81,0.01):\n    print(f'{threshold:.02f}, ',end='')\n    preds = (oof.values.reshape((-1))>threshold).astype('int')\n    m = f1_score(true.values.reshape((-1)), preds, average='macro')   \n    scores.append(m)\n    thresholds.append(threshold)\n    if m>best_score:\n        best_score = m\n        best_threshold = threshold","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# PLOT THRESHOLD VS. F1_SCORE\nplt.figure(figsize=(20,5))\nplt.plot(thresholds,scores,'-o',color='blue')\nplt.scatter([best_threshold], [best_score], color='blue', s=300, alpha=1)\nplt.xlabel('Threshold',size=14)\nplt.ylabel('Validation F1 Score',size=14)\nplt.title(f'Threshold vs. F1_Score with Best F1_Score = {best_score:.3f} at Best Threshold = {best_threshold:.3}',size=18)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('When using optimal threshold...')\nfor k in range(18):\n        \n    # COMPUTE F1 SCORE PER QUESTION\n    m = f1_score(true[k].values, (oof[k].values>best_threshold).astype('int'), average='macro')\n    print(f'Q{k}: F1 =',m)\n    \n# COMPUTE F1 SCORE OVERALL\nm = f1_score(true.values.reshape((-1)), (oof.values.reshape((-1))>best_threshold).astype('int'), average='macro')\nprint('==> Overall F1 =',m)","metadata":{"papermill":{"duration":0.771134,"end_time":"2023-02-07T01:02:44.378465","exception":false,"start_time":"2023-02-07T01:02:43.607331","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Infer Test Data","metadata":{"papermill":{"duration":0.011075,"end_time":"2023-02-07T01:02:44.400918","exception":false,"start_time":"2023-02-07T01:02:44.389843","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import jo_wilder\nenv = jo_wilder.make_env() \niter_test = env.iter_test()","metadata":{"papermill":{"duration":0.052132,"end_time":"2023-02-07T01:02:44.464739","exception":false,"start_time":"2023-02-07T01:02:44.412607","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-24T22:16:56.418800Z","iopub.execute_input":"2023-05-24T22:16:56.419741Z","iopub.status.idle":"2023-05-24T22:16:56.444559Z","shell.execute_reply.started":"2023-05-24T22:16:56.419677Z","shell.execute_reply":"2023-05-24T22:16:56.443601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"limits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\nfor (test, sample_submission) in iter_test:\n    test = test.sort_values(by = 'index')\n    \n    df = feature_engineer(test)\n    grp = test.level_group.values[0]\n    a,b = limits[grp]\n    for t in range(a,b):\n        clf = models[f'{grp}_{t}']\n        p = clf.predict(df[FEATURES].astype('float32'))\n        mask = sample_submission.session_id.str.contains(f'q{t}')\n        sample_submission.loc[mask,'correct'] = int(p.item()>best_threshold)\n    \n    env.predict(sample_submission)","metadata":{"papermill":{"duration":1.002014,"end_time":"2023-02-07T01:02:45.47927","exception":false,"start_time":"2023-02-07T01:02:44.477256","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA submission.csv","metadata":{"papermill":{"duration":0.011427,"end_time":"2023-02-07T01:02:45.502331","exception":false,"start_time":"2023-02-07T01:02:45.490904","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# df = pd.read_csv('submission.csv')\n# print( df.shape )\n# df.head()","metadata":{"papermill":{"duration":0.027432,"end_time":"2023-02-07T01:02:45.541022","exception":false,"start_time":"2023-02-07T01:02:45.51359","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(df.correct.mean())","metadata":{"papermill":{"duration":0.020233,"end_time":"2023-02-07T01:02:45.57314","exception":false,"start_time":"2023-02-07T01:02:45.552907","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}