{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd, numpy as np, gc\nfrom sklearn.model_selection import KFold, GroupKFold\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import f1_score","metadata":{"papermill":{"duration":1.027875,"end_time":"2023-02-07T00:59:59.180261","exception":false,"start_time":"2023-02-07T00:59:58.152386","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-05T09:54:46.964479Z","iopub.execute_input":"2023-05-05T09:54:46.965005Z","iopub.status.idle":"2023-05-05T09:54:47.777183Z","shell.execute_reply.started":"2023-05-05T09:54:46.964905Z","shell.execute_reply":"2023-05-05T09:54:47.775984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Muat Data train dan Label \nPada 20 Maret 2023, Kaggle menggandakan ukuran data kereta api (diskusi [di sini][1]). Data kereta sekarang 4,7 GB! Untuk menghindari kesalahan memori, kami akan membaca data train sebanyak 10 bagian dan merancang setiap bagian sebelum membaca bagian berikutnya. Ini berfungsi karena rekayasa fitur menyusutkan ukuran setiap bagian.\n\n[1]: https://www.kaggle.com/competitions/predict-student-performance-from-game-play/discussion/396202","metadata":{"papermill":{"duration":0.004542,"end_time":"2023-02-07T00:59:59.189777","exception":false,"start_time":"2023-02-07T00:59:59.185235","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# HANYA BACA USER ID \ntmp = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train.csv\",usecols=[0])\ntmp = tmp.groupby('session_id').session_id.agg('count')\n\n# MENGHITUNG READS AND SKIPS\nPIECES = 10\nCHUNK = int( np.ceil(len(tmp)/PIECES) )\n\nreads = []\nskips = [0]\nfor k in range(PIECES):\n    a = k*CHUNK\n    b = (k+1)*CHUNK\n    if b>len(tmp): b=len(tmp)\n    r = tmp.iloc[a:b].sum()\n    reads.append(r)\n    skips.append(skips[-1]+r)\n    \nprint(f'Untuk menghindari kesalahan memori, kami akan membaca train dalam ukuran {PIECES} bagian :')\nprint(reads)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-05T09:55:08.552388Z","iopub.execute_input":"2023-05-05T09:55:08.552892Z","iopub.status.idle":"2023-05-05T09:56:32.651830Z","shell.execute_reply.started":"2023-05-05T09:55:08.552829Z","shell.execute_reply":"2023-05-05T09:56:32.650636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv', nrows=reads[0])\nprint('Train size of first piece:', train.shape )\ntrain.head()","metadata":{"papermill":{"duration":59.284316,"end_time":"2023-02-07T01:00:58.478743","exception":false,"start_time":"2023-02-07T00:59:59.194427","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-05T09:57:23.739083Z","iopub.execute_input":"2023-05-05T09:57:23.739472Z","iopub.status.idle":"2023-05-05T09:57:31.350983Z","shell.execute_reply.started":"2023-05-05T09:57:23.739440Z","shell.execute_reply":"2023-05-05T09:57:31.348999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\ntargets['session'] = targets.session_id.apply(lambda x: int(x.split('_')[0]) )\ntargets['q'] = targets.session_id.apply(lambda x: int(x.split('_')[-1][1:]) )\nprint( targets.shape )\ntargets.head()","metadata":{"papermill":{"duration":0.598155,"end_time":"2023-02-07T01:00:59.082015","exception":false,"start_time":"2023-02-07T01:00:58.48386","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-05T09:59:55.712299Z","iopub.execute_input":"2023-05-05T09:59:55.712809Z","iopub.status.idle":"2023-05-05T09:59:56.959597Z","shell.execute_reply.started":"2023-05-05T09:59:55.712771Z","shell.execute_reply":"2023-05-05T09:59:56.958349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineer","metadata":{"papermill":{"duration":0.005196,"end_time":"2023-02-07T01:00:59.092865","exception":false,"start_time":"2023-02-07T01:00:59.087669","status":"completed"},"tags":[]}},{"cell_type":"code","source":"CATS = ['event_name', 'fqid', 'room_fqid', 'text']\nNUMS = ['elapsed_time','level','page','room_coor_x', 'room_coor_y', \n        'screen_coor_x', 'screen_coor_y', 'hover_duration']\n\n# https://www.kaggle.com/code/kimtaehun/lightgbm-baseline-with-aggregated-log-data\nEVENTS = ['navigate_click','person_click','cutscene_click','object_click',\n          'map_hover','notification_click','map_click','observation_click',\n          'checkpoint']","metadata":{"papermill":{"duration":0.014685,"end_time":"2023-02-07T01:00:59.112856","exception":false,"start_time":"2023-02-07T01:00:59.098171","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-05T10:00:06.844030Z","iopub.execute_input":"2023-05-05T10:00:06.845013Z","iopub.status.idle":"2023-05-05T10:00:06.856809Z","shell.execute_reply.started":"2023-05-05T10:00:06.844958Z","shell.execute_reply":"2023-05-05T10:00:06.854642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineer(train):\n    \n    dfs = []\n    for c in CATS:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('nunique')\n        tmp.name = tmp.name + '_nunique'\n        dfs.append(tmp)\n    for c in NUMS:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('mean')\n        tmp.name = tmp.name + '_mean'\n        dfs.append(tmp)\n    for c in NUMS:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('std')\n        tmp.name = tmp.name + '_std'\n        dfs.append(tmp)\n    for c in EVENTS: \n        train[c] = (train.event_name == c).astype('int8')\n    for c in EVENTS + ['elapsed_time']:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('sum')\n        tmp.name = tmp.name + '_sum'\n        dfs.append(tmp)\n    train = train.drop(EVENTS,axis=1)\n        \n    df = pd.concat(dfs,axis=1)\n    df = df.fillna(-1)\n    df = df.reset_index()\n    df = df.set_index('session_id')\n    return df","metadata":{"papermill":{"duration":0.017716,"end_time":"2023-02-07T01:00:59.136021","exception":false,"start_time":"2023-02-07T01:00:59.118305","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-05T10:00:09.407713Z","iopub.execute_input":"2023-05-05T10:00:09.408146Z","iopub.status.idle":"2023-05-05T10:00:09.421508Z","shell.execute_reply.started":"2023-05-05T10:00:09.408113Z","shell.execute_reply":"2023-05-05T10:00:09.419647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# MEMPROSES DATA TRAIN SEBAGIAN-SEBAGIAN\nall_pieces = []\nprint(f'Processing train as {PIECES} pieces to avoid memory error... ')\nfor k in range(PIECES):\n    print(k,', ',end='')\n    SKIPS = 0\n    if k>0: SKIPS = range(1,skips[k]+1)\n    train = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv',\n                        nrows=reads[k], skiprows=SKIPS)\n    df = feature_engineer(train)\n    all_pieces.append(df)\n    \n# HUBUNGI SEMUA BAGIAN\nprint('\\n')\ndel train; gc.collect()\ndf = pd.concat(all_pieces, axis=0)\nprint('Shape of all train data after feature engineering:', df.shape )\ndf.head()","metadata":{"papermill":{"duration":34.516494,"end_time":"2023-02-07T01:01:33.658043","exception":false,"start_time":"2023-02-07T01:00:59.141549","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-05T10:00:12.777034Z","iopub.execute_input":"2023-05-05T10:00:12.777483Z","iopub.status.idle":"2023-05-05T10:06:23.031166Z","shell.execute_reply.started":"2023-05-05T10:00:12.777449Z","shell.execute_reply":"2023-05-05T10:06:23.029980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train XGBoost Model\nKami melatih satu model untuk setiap 18 pertanyaan. Selanjutnya, kami menggunakan data dari `level_groups = '0-4'` untuk melatih model untuk pertanyaan 1-3, dan `level groups '5-12'` untuk melatih soal 4 sampai 13 dan `level groups '13-22'` untuk melatih pertanyaan 14 hingga 18. Karena ini adalah data yang kami dapatkan (untuk memprediksi pertanyaan terkait) dari API inferensi Kaggle selama inferensi pengujian. Kami dapat meningkatkan model kami dengan menyimpan data pengguna sebelumnya dari sebelumnya `level_groups` dan menggunakannya untuk memprediksi masa depan `level_groups`.","metadata":{"papermill":{"duration":0.00565,"end_time":"2023-02-07T01:01:33.669525","exception":false,"start_time":"2023-02-07T01:01:33.663875","status":"completed"},"tags":[]}},{"cell_type":"code","source":"FEATURES = [c for c in df.columns if c != 'level_group']\nprint('We will train with', len(FEATURES) ,'features')\nALL_USERS = df.index.unique()\nprint('We will train with', len(ALL_USERS) ,'users info')","metadata":{"papermill":{"duration":0.014699,"end_time":"2023-02-07T01:01:33.689953","exception":false,"start_time":"2023-02-07T01:01:33.675254","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-05T10:08:06.952464Z","iopub.execute_input":"2023-05-05T10:08:06.953005Z","iopub.status.idle":"2023-05-05T10:08:06.965774Z","shell.execute_reply.started":"2023-05-05T10:08:06.952965Z","shell.execute_reply":"2023-05-05T10:08:06.964428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gkf = GroupKFold(n_splits=5)\noof = pd.DataFrame(data=np.zeros((len(ALL_USERS),18)), index=ALL_USERS)\nmodels = {}\n\n# HITUNG SKOR CV DENGAN KELOMPOK 5 GROUP K FOLD\nfor i, (train_index, test_index) in enumerate(gkf.split(X=df, groups=df.index)):\n    print('#'*25)\n    print('### Fold',i+1)\n    print('#'*25)\n    \n    xgb_params = {\n    'objective' : 'binary:logistic',\n    'eval_metric':'logloss',\n    'learning_rate': 0.05,\n    'max_depth': 4,\n    'n_estimators': 1000,\n    'early_stopping_rounds': 50,\n    'tree_method':'hist',\n    'subsample':0.8,\n    'colsample_bytree': 0.4,\n    'use_label_encoder' : False}\n    \n    # ITERASI MELALUI PERTANYAAN 1 SAMPAI 18\n    for t in range(1,19):\n        \n        # GUNAKAN DATA TRAIN INI DENGAN PERTANYAAN INI\n        if t<=3: grp = '0-4'\n        elif t<=13: grp = '5-12'\n        elif t<=22: grp = '13-22'\n            \n        # TRAIN DATA\n        train_x = df.iloc[train_index]\n        train_x = train_x.loc[train_x.level_group == grp]\n        train_users = train_x.index.values\n        train_y = targets.loc[targets.q==t].set_index('session').loc[train_users]\n        \n        # VALID DATA\n        valid_x = df.iloc[test_index]\n        valid_x = valid_x.loc[valid_x.level_group == grp]\n        valid_users = valid_x.index.values\n        valid_y = targets.loc[targets.q==t].set_index('session').loc[valid_users]\n        \n        # TRAIN MODEL        \n        clf =  XGBClassifier(**xgb_params)\n        clf.fit(train_x[FEATURES].astype('float32'), train_y['correct'],\n                eval_set=[ (valid_x[FEATURES].astype('float32'), valid_y['correct']) ],\n                verbose=0)\n        print(f'{t}({clf.best_ntree_limit}), ',end='')\n        \n        # SIMPAN MODEL, PREDIKSI VALID OOF\n        models[f'{grp}_{t}'] = clf\n        oof.loc[valid_users, t-1] = clf.predict_proba(valid_x[FEATURES].astype('float32'))[:,1]\n        \n    print()","metadata":{"papermill":{"duration":69.877213,"end_time":"2023-02-07T01:02:43.57299","exception":false,"start_time":"2023-02-07T01:01:33.695777","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-05T10:08:11.948403Z","iopub.execute_input":"2023-05-05T10:08:11.948925Z","iopub.status.idle":"2023-05-05T10:11:05.336416Z","shell.execute_reply.started":"2023-05-05T10:08:11.948862Z","shell.execute_reply":"2023-05-05T10:11:05.335221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Hitung Skor CV\n\nKita perlu mengonversi probabilitas prediksi menjadi `1s` dan `0s`. Metrik kompetisi adalah Skor F1 yang merupakan rata-rata harmonis dari presisi dan daya ingat. Mari kita temukan ambang batas optimal untuk `p > threshold` saat memprediksi `1` dan saat memprediksi `0` untuk memaksimalkan Skor F1.","metadata":{"papermill":{"duration":0.011241,"end_time":"2023-02-07T01:02:43.59638","exception":false,"start_time":"2023-02-07T01:02:43.585139","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# TEMPATKAN LABEL YANG TRUE KE DALAM DATAFRAME DENGAN 18 KOLOM\ntrue = oof.copy()\nfor k in range(18):\n    # DAPATKAN TRUE LABEL\n    tmp = targets.loc[targets.q == k+1].set_index('session').loc[ALL_USERS]\n    true[k] = tmp.correct.values","metadata":{"execution":{"iopub.status.busy":"2023-05-05T10:11:45.707665Z","iopub.execute_input":"2023-05-05T10:11:45.708221Z","iopub.status.idle":"2023-05-05T10:11:45.893675Z","shell.execute_reply.started":"2023-05-05T10:11:45.708182Z","shell.execute_reply":"2023-05-05T10:11:45.891568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# MENCARI THRESHOLD TERBAIK UNTUK MENG-CONVERT PROB MENJADI 1s DAN 0s\nscores = []; thresholds = []\nbest_score = 0; best_threshold = 0\n\nfor threshold in np.arange(0.4,0.81,0.01):\n    print(f'{threshold:.02f}, ',end='')\n    preds = (oof.values.reshape((-1))>threshold).astype('int')\n    m = f1_score(true.values.reshape((-1)), preds, average='macro')   \n    scores.append(m)\n    thresholds.append(threshold)\n    if m>best_score:\n        best_score = m\n        best_threshold = threshold","metadata":{"execution":{"iopub.status.busy":"2023-05-05T10:11:48.442318Z","iopub.execute_input":"2023-05-05T10:11:48.442789Z","iopub.status.idle":"2023-05-05T10:11:56.668663Z","shell.execute_reply.started":"2023-05-05T10:11:48.442750Z","shell.execute_reply":"2023-05-05T10:11:56.666817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# PLOT THRESHOLD VS. F1_SCORE\nplt.figure(figsize=(20,5))\nplt.plot(thresholds,scores,'-o',color='blue')\nplt.scatter([best_threshold], [best_score], color='blue', s=300, alpha=1)\nplt.xlabel('Threshold',size=14)\nplt.ylabel('Validation F1 Score',size=14)\nplt.title(f'Threshold vs. F1_Score dengan skor F1_Score Terbaik = {best_score:.3f} di Threshold Terbaik = {best_threshold:.3}',size=18)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-05T10:14:35.813111Z","iopub.execute_input":"2023-05-05T10:14:35.813540Z","iopub.status.idle":"2023-05-05T10:14:36.098753Z","shell.execute_reply.started":"2023-05-05T10:14:35.813508Z","shell.execute_reply":"2023-05-05T10:14:36.096899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Saat menggunakan threshold yang optimal...')\nfor k in range(18):\n        \n    # COMPUTE F1 SCORE PER QUESTION\n    m = f1_score(true[k].values, (oof[k].values>best_threshold).astype('int'), average='macro')\n    print(f'Q{k}: F1 =',m)\n    \n# COMPUTE F1 SCORE OVERALL\nm = f1_score(true.values.reshape((-1)), (oof.values.reshape((-1))>best_threshold).astype('int'), average='macro')\nprint('==> Overall F1 =',m)","metadata":{"papermill":{"duration":0.771134,"end_time":"2023-02-07T01:02:44.378465","exception":false,"start_time":"2023-02-07T01:02:43.607331","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-05T10:14:54.202015Z","iopub.execute_input":"2023-05-05T10:14:54.202425Z","iopub.status.idle":"2023-05-05T10:14:54.598461Z","shell.execute_reply.started":"2023-05-05T10:14:54.202393Z","shell.execute_reply":"2023-05-05T10:14:54.597092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Infer Test Data","metadata":{"papermill":{"duration":0.011075,"end_time":"2023-02-07T01:02:44.400918","exception":false,"start_time":"2023-02-07T01:02:44.389843","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# IMPORT KAGGLE API\nimport jo_wilder\nenv = jo_wilder.make_env()\niter_test = env.iter_test()\n\n# CLEAR MEMORY\nimport gc\ndel targets, df, oof, true\n_ = gc.collect()","metadata":{"papermill":{"duration":0.052132,"end_time":"2023-02-07T01:02:44.464739","exception":false,"start_time":"2023-02-07T01:02:44.412607","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-05T10:14:59.307204Z","iopub.execute_input":"2023-05-05T10:14:59.307612Z","iopub.status.idle":"2023-05-05T10:14:59.447097Z","shell.execute_reply.started":"2023-05-05T10:14:59.307578Z","shell.execute_reply":"2023-05-05T10:14:59.445717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"limits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\nfor (test, sample_submission) in iter_test:\n    \n    # FEATURE ENGINEER TEST DATA\n    df = feature_engineer(test)\n    \n    # INFER TEST DATA\n    grp = test.level_group.values[0]\n    a,b = limits[grp]\n    for t in range(a,b):\n        clf = models[f'{grp}_{t}']\n        p = clf.predict_proba(df[FEATURES].astype('float32'))[0,1]\n        mask = sample_submission.session_id.str.contains(f'q{t}')\n        sample_submission.loc[mask,'correct'] = int( p > best_threshold )\n    \n    env.predict(sample_submission)","metadata":{"papermill":{"duration":1.002014,"end_time":"2023-02-07T01:02:45.47927","exception":false,"start_time":"2023-02-07T01:02:44.477256","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-05T10:15:02.177501Z","iopub.execute_input":"2023-05-05T10:15:02.177937Z","iopub.status.idle":"2023-05-05T10:15:03.082187Z","shell.execute_reply.started":"2023-05-05T10:15:02.177901Z","shell.execute_reply":"2023-05-05T10:15:03.081053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA/Exploratory Data Analysis submission.csv","metadata":{"papermill":{"duration":0.011427,"end_time":"2023-02-07T01:02:45.502331","exception":false,"start_time":"2023-02-07T01:02:45.490904","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df = pd.read_csv('submission.csv')\nprint( df.shape )\ndf.head()","metadata":{"papermill":{"duration":0.027432,"end_time":"2023-02-07T01:02:45.541022","exception":false,"start_time":"2023-02-07T01:02:45.51359","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-05T10:15:06.111927Z","iopub.execute_input":"2023-05-05T10:15:06.112353Z","iopub.status.idle":"2023-05-05T10:15:06.133780Z","shell.execute_reply.started":"2023-05-05T10:15:06.112318Z","shell.execute_reply":"2023-05-05T10:15:06.132342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df.correct.mean())","metadata":{"papermill":{"duration":0.020233,"end_time":"2023-02-07T01:02:45.57314","exception":false,"start_time":"2023-02-07T01:02:45.552907","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-05T10:15:08.681511Z","iopub.execute_input":"2023-05-05T10:15:08.681940Z","iopub.status.idle":"2023-05-05T10:15:08.688237Z","shell.execute_reply.started":"2023-05-05T10:15:08.681901Z","shell.execute_reply":"2023-05-05T10:15:08.687147Z"},"trusted":true},"execution_count":null,"outputs":[]}]}