{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport pickle \n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.multioutput import MultiOutputClassifier\n\nfrom sklearn.metrics import f1_score, precision_score, recall_score, auc,roc_auc_score","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-27T19:03:12.273410Z","iopub.execute_input":"2023-06-27T19:03:12.273831Z","iopub.status.idle":"2023-06-27T19:03:12.280649Z","shell.execute_reply.started":"2023-06-27T19:03:12.273791Z","shell.execute_reply":"2023-06-27T19:03:12.279595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_pickle_obj(path):\n    with open(path, 'rb') as file:\n        obj = pickle.load(file)\n    return obj","metadata":{"execution":{"iopub.status.busy":"2023-06-27T19:03:12.282589Z","iopub.execute_input":"2023-06-27T19:03:12.283836Z","iopub.status.idle":"2023-06-27T19:03:12.296164Z","shell.execute_reply.started":"2023-06-27T19:03:12.283792Z","shell.execute_reply":"2023-06-27T19:03:12.294972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# train label","metadata":{}},{"cell_type":"code","source":"def get_group_level(q):\n    qno = int(q[1:])\n    if qno < 4:\n        return '0-4'\n    elif qno < 14:\n        return '5-12'\n    return '13-22'","metadata":{"execution":{"iopub.status.busy":"2023-06-27T19:03:12.298423Z","iopub.execute_input":"2023-06-27T19:03:12.299220Z","iopub.status.idle":"2023-06-27T19:03:12.309784Z","shell.execute_reply.started":"2023-06-27T19:03:12.299176Z","shell.execute_reply":"2023-06-27T19:03:12.308575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train_labels.csv\")\ntrain_label['q'] = train_label['session_id'].apply(lambda s: s.split(\"_\")[-1])\ntrain_label['qindex'] = train_label['q'].apply(lambda q: int(q[1:]))\ntrain_label['session_id'] = train_label['session_id'].apply(lambda s: int(s.split(\"_\")[0]))\ntrain_label['level_group'] = train_label.q.apply(get_group_level)\n\ntrain_label.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-27T19:03:12.311403Z","iopub.execute_input":"2023-06-27T19:03:12.312062Z","iopub.status.idle":"2023-06-27T19:03:14.386496Z","shell.execute_reply.started":"2023-06-27T19:03:12.312004Z","shell.execute_reply":"2023-06-27T19:03:14.385413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# generating features","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import KFold\nmodel_map = {\n    '0-4': {},\n    '5-12':{},\n    '13-22': {}\n}","metadata":{"execution":{"iopub.status.busy":"2023-06-27T19:03:14.389263Z","iopub.execute_input":"2023-06-27T19:03:14.390036Z","iopub.status.idle":"2023-06-27T19:03:14.394937Z","shell.execute_reply.started":"2023-06-27T19:03:14.389994Z","shell.execute_reply":"2023-06-27T19:03:14.393922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_df=[]\n\ndef train_model(lg, q, foldnum, feat_columns, fold_train_df, fold_val_df, val_session_ids):\n    print(\"Training model : foldnum:{}\".format(foldnum))\n    print(\"level group:{} | question:{}\".format(lg, q))\n    print(fold_train_df.shape, fold_val_df.shape)\n    \n    xtrain = fold_train_df[feat_columns].values\n    ytrain = fold_train_df['correct'].values\n    \n    xval = fold_val_df[feat_columns].values\n    yval = fold_val_df['correct'].values\n    \n    model = LogisticRegression(max_iter=10000).fit(xtrain, ytrain)\n    yval_hat = model.predict_proba(xval)\n    yval_hat = yval_hat[:, 1]\n    fscore = f1_score(yval, (yval_hat>=0.5).astype(int), average='macro')\n    print(\"foldnum:{} | q:{} | fscore:{:.4f}\".format(foldnum, q, fscore))\n    \n    if model_map[lg].get(q, None) is None:\n        model_map[lg][q] = []\n    model_map[lg][q].append(model)\n    \n    eval_df = pd.DataFrame.from_dict({\n        'session_id': fold_val_df['session_id'],\n        'level_group': lg,\n        'q': q,\n        'y': yval,\n        'yhat': yval_hat\n    })\n    \n    oof_df.append(eval_df)\n    print()","metadata":{"execution":{"iopub.status.busy":"2023-06-27T19:03:14.396720Z","iopub.execute_input":"2023-06-27T19:03:14.397492Z","iopub.status.idle":"2023-06-27T19:03:14.413456Z","shell.execute_reply.started":"2023-06-27T19:03:14.397430Z","shell.execute_reply":"2023-06-27T19:03:14.411986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-06-27T19:52:14.710240Z","iopub.execute_input":"2023-06-27T19:52:14.710872Z","iopub.status.idle":"2023-06-27T19:52:14.722334Z","shell.execute_reply.started":"2023-06-27T19:52:14.710819Z","shell.execute_reply":"2023-06-27T19:52:14.720988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_FOLDS=10\n\nfold_df = pd.read_csv(\"/kaggle/input/pspg-10folds/train_fold1.csv\")\nfeat_df = pd.read_parquet(\"/kaggle/input/psgp-10fold-nn-embedd/nn_5fold_embedd.parquet\").rename(columns={\n    'sess_id': 'session_id'\n})\nfeat_columns = [\"feat_{}\".format(i) for i in range(512)]\n\nfeat_df.fillna(0.0, inplace=True)\nfeat_df.replace(-np.inf, 0.0, inplace=True)\nfeat_df.replace(np.inf, 0.0, inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfor lg in ['0-4', '5-12', '13-22']:\n    lg_label_df = train_label[train_label.level_group == lg]\n    for q in lg_label_df['q'].unique():\n        for foldnum in range(NUM_FOLDS):\n            train_session_ids = fold_df[fold_df.foldnum != foldnum]['session_id'].unique()\n            val_session_ids = fold_df[fold_df.foldnum == foldnum]['session_id'].unique()\n            \n            \n            \n            fold_label = lg_label_df[lg_label_df.q == q]\n            \n            fold_train_df = feat_df[(feat_df.level_group == lg) & \n                                    (feat_df.foldnum == foldnum) & \n                                    (feat_df.session_id.isin(train_session_ids))].merge(fold_label[['session_id', \n                                                                                                  'level_group',\n                                                                                                  'correct']], \n                                                                                      how='left',\n                                                                                      on=['session_id', 'level_group'])\n            \n            fold_val_df = feat_df[(feat_df.level_group == lg) & \n                                  (feat_df.foldnum == foldnum) & \n                                  (feat_df.session_id.isin(val_session_ids))].merge(fold_label[['session_id', \n                                                                                              'level_group',\n                                                                                              'correct']], \n                                                                                  how='left',\n                                                                                  on=['session_id', 'level_group'])\n            \n            \n            train_model(lg, q, foldnum, feat_columns, fold_train_df, fold_val_df, val_session_ids)","metadata":{"execution":{"iopub.status.busy":"2023-06-27T19:39:06.857228Z","iopub.execute_input":"2023-06-27T19:39:06.857684Z","iopub.status.idle":"2023-06-27T19:39:55.443566Z","shell.execute_reply.started":"2023-06-27T19:39:06.857646Z","shell.execute_reply":"2023-06-27T19:39:55.442114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"linear_model_map.pkl\",'wb') as file:\n    pickle.dump(model_map, file)","metadata":{"execution":{"iopub.status.busy":"2023-06-27T19:03:14.537461Z","iopub.status.idle":"2023-06-27T19:03:14.538706Z","shell.execute_reply.started":"2023-06-27T19:03:14.538439Z","shell.execute_reply":"2023-06-27T19:03:14.538470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_df = pd.concat(oof_df)\noof_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-27T19:03:14.540168Z","iopub.status.idle":"2023-06-27T19:03:14.541210Z","shell.execute_reply.started":"2023-06-27T19:03:14.540967Z","shell.execute_reply":"2023-06-27T19:03:14.540995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_df.to_csv(\"linear_oof.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-27T19:03:14.542712Z","iopub.status.idle":"2023-06-27T19:03:14.543188Z","shell.execute_reply.started":"2023-06-27T19:03:14.542970Z","shell.execute_reply":"2023-06-27T19:03:14.542994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nth_lst=[]\nmacro_lst=[]\n\nbest_th = 0.0\nbest_macro = 0.0\n\nytrue = oof_df['y'].values\nypred = oof_df['yhat'].values\n\nfor th in np.arange(0.5, 0.8, 0.01):\n    m=f1_score(ytrue, (ypred>=th).astype(int), average='macro')\n    if m > best_macro:\n        best_macro = m\n        best_th = th\n    \n    th_lst.append(th)\n    macro_lst.append(m)","metadata":{"execution":{"iopub.status.busy":"2023-06-27T19:03:14.544570Z","iopub.status.idle":"2023-06-27T19:03:14.545423Z","shell.execute_reply.started":"2023-06-27T19:03:14.545173Z","shell.execute_reply":"2023-06-27T19:03:14.545198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Best Threshhold:{:.4f}\".format(best_th))\nprint(\"Best Macro:{:.5f}\".format(best_macro))","metadata":{"execution":{"iopub.status.busy":"2023-06-27T19:03:14.546732Z","iopub.status.idle":"2023-06-27T19:03:14.547753Z","shell.execute_reply.started":"2023-06-27T19:03:14.547518Z","shell.execute_reply":"2023-06-27T19:03:14.547545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.plot(th_lst, macro_lst)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-27T19:03:14.549331Z","iopub.status.idle":"2023-06-27T19:03:14.549760Z","shell.execute_reply.started":"2023-06-27T19:03:14.549550Z","shell.execute_reply":"2023-06-27T19:03:14.549572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}