{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":45533,"databundleVersionId":5748852,"sourceType":"competition"},{"sourceId":10124145,"sourceType":"datasetVersion","datasetId":6247439}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd, numpy as np, gc\nfrom sklearn.model_selection import KFold, GroupKFold\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import f1_score\nfrom sklearn.model_selection import GridSearchCV\nimport plotly.express as px\nimport matplotlib.pyplot as plt\nfrom catboost import CatBoostClassifier, Pool\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T13:12:06.364289Z","iopub.execute_input":"2024-12-08T13:12:06.364679Z","iopub.status.idle":"2024-12-08T13:12:09.869958Z","shell.execute_reply.started":"2024-12-08T13:12:06.364642Z","shell.execute_reply":"2024-12-08T13:12:09.868905Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"targets = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train_labels.csv\")\ntmp = targets.session_id.str.split(\"_\", expand =True)\ntargets[\"user_id\"] = tmp[0].astype(\"int\")\ntargets[\"q\"] = tmp[1].str.slice(1).astype(\"int\")\ntargets.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T13:12:09.873307Z","iopub.execute_input":"2024-12-08T13:12:09.873892Z","iopub.status.idle":"2024-12-08T13:12:11.477015Z","shell.execute_reply.started":"2024-12-08T13:12:09.873846Z","shell.execute_reply":"2024-12-08T13:12:11.476164Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/processed-data/engineered_dataset.csv\",index_col = 0)\ndf.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T13:12:11.478004Z","iopub.execute_input":"2024-12-08T13:12:11.478277Z","iopub.status.idle":"2024-12-08T13:12:12.097786Z","shell.execute_reply.started":"2024-12-08T13:12:11.478249Z","shell.execute_reply":"2024-12-08T13:12:12.096829Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ALL_USERS = df.index.unique()\nFEATURES = df.columns[1:]\nprint(ALL_USERS.shape, FEATURES.shape)\n\nfeature_importance_df = pd.DataFrame()\nmodels = {}\nresults = [[[], []] for _ in range(18)]\ngkf = GroupKFold(n_splits=5)\noof_cat = pd.DataFrame(data=np.zeros((len(ALL_USERS),18)), index=ALL_USERS, columns=[f'meta_{i}' for i in range(1, 19)])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T13:12:12.099691Z","iopub.execute_input":"2024-12-08T13:12:12.100003Z","iopub.status.idle":"2024-12-08T13:12:12.110811Z","shell.execute_reply.started":"2024-12-08T13:12:12.099973Z","shell.execute_reply":"2024-12-08T13:12:12.110091Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df1 = df[df.level_group == '0-4']\ndf2 = df[df.level_group == '5-12']\ndf3 = df[df.level_group == '13-22']\ndel df\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T13:12:12.111621Z","iopub.execute_input":"2024-12-08T13:12:12.111897Z","iopub.status.idle":"2024-12-08T13:12:12.242184Z","shell.execute_reply.started":"2024-12-08T13:12:12.111870Z","shell.execute_reply":"2024-12-08T13:12:12.241252Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for q in tqdm(range(1, 19)):\n    if q <= 3:\n        grp = '0-4'\n        df = df1\n    elif q <= 13:\n        grp = '5-12'\n        df = df2\n    elif q <= 22:\n        grp = '13-22'\n        df = df3\n        \n    print(f'question {q}, with {len(FEATURES)} features')\n    print('-'*25)\n\n    cat_params = {\n        'iterations': 1000,\n        'early_stopping_rounds': 90,\n        'depth': 5,\n        'learning_rate': 0.02,\n        'loss_function': \"Logloss\",\n        'random_seed': 222222,\n        'metric_period': 1,\n        'subsample': 0.8,\n        'colsample_bylevel': 0.4,\n        'verbose': 0,\n        'l2_leaf_reg': 20,\n    }\n\n    for fold, (train_idx, valid_idx) in enumerate(gkf.split(X= df, groups= df.index)):\n        # TRAIN DATA\n        train_x = df.iloc[train_idx]\n        train_users = train_x.index.values\n        \n        train_y = targets.loc[targets.q == q].set_index('user_id').loc[train_users]\n\n        # VALID DATA\n        valid_x = df.iloc[valid_idx]\n        valid_users = valid_x.index.values\n        valid_y = targets.loc[targets.q == q].set_index('user_id').loc[valid_users]\n\n\n        train_pool = Pool(train_x[FEATURES].astype('float32'), train_y['correct'])\n        valid_pool = Pool(valid_x[FEATURES].astype('float32'), valid_y['correct'])\n\n\n        model = CatBoostClassifier(**cat_params)\n        model = model.fit(train_pool, eval_set=valid_pool)\n\n        y = valid_pool.get_label()\n        y_hat = model.predict_proba(valid_pool)[:,1]\n        models[(fold, q)] = model\n        \n        fold_importance_df = pd.DataFrame()\n        fold_importance_df[\"feature\"] = FEATURES\n        fold_importance_df[\"importance\"] = model.feature_importances_\n        fold_importance_df[\"fold\"] = fold + 1\n        feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n        \n        results[q - 1][0].append(y)\n        results[q - 1][1].append(y_hat)\n    feature_importance_df = feature_importance_df.groupby(['feature'])['importance'].agg(['mean']).sort_values(by='mean', ascending=False)\n    display(feature_importance_df.head(10))\nresults = [[np.concatenate(_) for _ in _] for _ in results]\n\n\nfor (fold,q), model in models.items():\n    model.save_model(f'fold{fold}_q{q}.cbm')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T13:12:12.243290Z","iopub.execute_input":"2024-12-08T13:12:12.243590Z","iopub.status.idle":"2024-12-08T13:18:23.244481Z","shell.execute_reply.started":"2024-12-08T13:12:12.243562Z","shell.execute_reply":"2024-12-08T13:18:23.243668Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"true = pd.DataFrame(np.stack([_[0] for _ in results]).T)\noof = pd.DataFrame(np.stack([_[1] for _ in results]).T)\n\nscores = []; thresholds = []\nbest_socre = 0; best_threshold = 0\n\nfor threshold in np.arange(0.5, 0.7, 0.01):\n    preds = (oof.values.reshape(-1) > threshold).astype('int')\n    m = f1_score(true.values.reshape(-1), preds, average='macro')\n    scores.append(m)\n    thresholds.append(threshold)\n    if m > best_socre:\n        best_socre = m\n        best_threshold = threshold\n\nplt.figure(figsize=(20, 5))\nplt.plot(thresholds, scores, '-o', color='blue')\nplt.scatter([best_threshold], [best_socre], color='blue')\nplt.xlabel(\"Threshold\", size=14)\nplt.ylabel(\"Validation F1 Score\",size=14)\nplt.title(f'Threshold vs. F1_Score with Best F1_Score={best_socre:.3f} at Best Threshold = {best_threshold:.3}', size=18)\nplt.show()\n\nprint(f'When using optimal threshold = {best_threshold:.2f}...')\nfor k in range(18):\n    m = f1_score(true[k].values, (oof[k].values > best_threshold).astype('int'), average = 'macro')\n    print(f'Q{k}: F1 =',m)\nm = f1_score(true.values.reshape(-1), (oof.values > best_threshold).reshape(-1).astype('int'), average = 'macro')\nprint('==> Overall F1 =', m)\nprint('s')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T13:18:23.245314Z","iopub.execute_input":"2024-12-08T13:18:23.245656Z","iopub.status.idle":"2024-12-08T13:18:27.693104Z","shell.execute_reply.started":"2024-12-08T13:18:23.245622Z","shell.execute_reply":"2024-12-08T13:18:27.692108Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pickle\nimportance_dict = {}\nfor t in range(1, 19):\n    if t<=3: \n        importance_dict[str(t)] = FEATURES\n    elif t<=13: \n        importance_dict[str(t)] = FEATURES\n    elif t<=22:\n        importance_dict[str(t)] = FEATURES\n\nf_save = open('importance_dict.pkl', 'wb')\npickle.dump(importance_dict, f_save)\nf_save.close()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T13:18:27.694340Z","iopub.execute_input":"2024-12-08T13:18:27.695284Z","iopub.status.idle":"2024-12-08T13:18:27.701285Z","shell.execute_reply.started":"2024-12-08T13:18:27.695245Z","shell.execute_reply":"2024-12-08T13:18:27.700339Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!zip -r model.zip /kaggle/working","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T13:18:27.702489Z","iopub.execute_input":"2024-12-08T13:18:27.702805Z","iopub.status.idle":"2024-12-08T13:18:31.636769Z","shell.execute_reply.started":"2024-12-08T13:18:27.702771Z","shell.execute_reply":"2024-12-08T13:18:31.635616Z"}},"outputs":[],"execution_count":null}]}