{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-18T06:08:13.249958Z","iopub.execute_input":"2023-02-18T06:08:13.250476Z","iopub.status.idle":"2023-02-18T06:08:13.295687Z","shell.execute_reply.started":"2023-02-18T06:08:13.250371Z","shell.execute_reply":"2023-02-18T06:08:13.294437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reduce Memory Usage\n# reference : https://www.kaggle.com/code/arjanso/reducing-dataframe-memory-size-by-65 @ARJANGROEN\n\ndef reduce_memory_usage(df):\n    \n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype.name\n        if ((col_type != 'datetime64[ns]') & (col_type != 'category')):\n            if (col_type != 'object'):\n                c_min = df[col].min()\n                c_max = df[col].max()\n\n                if str(col_type)[:3] == 'int':\n                    if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                        df[col] = df[col].astype(np.int8)\n                    elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                        df[col] = df[col].astype(np.int16)\n                    elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                        df[col] = df[col].astype(np.int32)\n                    elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                        df[col] = df[col].astype(np.int64)\n\n                else:\n                    if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                        df[col] = df[col].astype(np.float16)\n                    elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                        df[col] = df[col].astype(np.float32)\n                    else:\n                        pass\n            else:\n                df[col] = df[col].astype('category')\n    mem_usg = df.memory_usage().sum() / 1024**2 \n    print(\"Memory usage became: \",mem_usg,\" MB\")\n    \n    return df\ntrain_df = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv')\ntrain_df = reduce_memory_usage(train_df)","metadata":{"execution":{"iopub.status.busy":"2023-02-18T06:08:13.665535Z","iopub.execute_input":"2023-02-18T06:08:13.666524Z","iopub.status.idle":"2023-02-18T06:09:47.479520Z","shell.execute_reply.started":"2023-02-18T06:08:13.666474Z","shell.execute_reply":"2023-02-18T06:09:47.478243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train_labels.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-02-18T06:09:47.481603Z","iopub.execute_input":"2023-02-18T06:09:47.482294Z","iopub.status.idle":"2023-02-18T06:09:47.762850Z","shell.execute_reply.started":"2023-02-18T06:09:47.482254Z","shell.execute_reply":"2023-02-18T06:09:47.761830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-02-18T06:09:47.764530Z","iopub.execute_input":"2023-02-18T06:09:47.765197Z","iopub.status.idle":"2023-02-18T06:09:47.917307Z","shell.execute_reply.started":"2023-02-18T06:09:47.765155Z","shell.execute_reply":"2023-02-18T06:09:47.916215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels[[\"session_id\",\"question\"]] =train_labels.session_id.str.split(\"_q\",expand=True)\ntrain_labels[\"question\"] = train_labels.question.astype(\"int\")","metadata":{"execution":{"iopub.status.busy":"2023-02-18T06:09:47.919682Z","iopub.execute_input":"2023-02-18T06:09:47.920330Z","iopub.status.idle":"2023-02-18T06:09:48.750355Z","shell.execute_reply.started":"2023-02-18T06:09:47.920290Z","shell.execute_reply":"2023-02-18T06:09:48.748498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Some EDA","metadata":{}},{"cell_type":"code","source":"import plotly.express as px","metadata":{"execution":{"iopub.status.busy":"2023-02-18T06:09:48.752512Z","iopub.execute_input":"2023-02-18T06:09:48.753012Z","iopub.status.idle":"2023-02-18T06:09:50.162450Z","shell.execute_reply.started":"2023-02-18T06:09:48.752969Z","shell.execute_reply":"2023-02-18T06:09:50.160764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.bar(train_labels.correct.value_counts()/len(train_labels))","metadata":{"execution":{"iopub.status.busy":"2023-02-18T06:09:50.164113Z","iopub.execute_input":"2023-02-18T06:09:50.165090Z","iopub.status.idle":"2023-02-18T06:09:51.537200Z","shell.execute_reply.started":"2023-02-18T06:09:50.165031Z","shell.execute_reply":"2023-02-18T06:09:51.535641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unqiue_ids = train_df.session_id.unique()","metadata":{"execution":{"iopub.status.busy":"2023-02-18T00:31:23.834305Z","iopub.execute_input":"2023-02-18T00:31:23.834860Z","iopub.status.idle":"2023-02-18T00:31:23.919888Z","shell.execute_reply.started":"2023-02-18T00:31:23.834827Z","shell.execute_reply":"2023-02-18T00:31:23.918447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(unqiue_ids)","metadata":{"execution":{"iopub.status.busy":"2023-02-18T00:31:23.921727Z","iopub.execute_input":"2023-02-18T00:31:23.922372Z","iopub.status.idle":"2023-02-18T00:31:23.928146Z","shell.execute_reply.started":"2023-02-18T00:31:23.922334Z","shell.execute_reply":"2023-02-18T00:31:23.927062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels.session_id.str.split(\"_\") ","metadata":{"execution":{"iopub.status.busy":"2023-02-18T00:31:23.929628Z","iopub.execute_input":"2023-02-18T00:31:23.930509Z","iopub.status.idle":"2023-02-18T00:31:24.402895Z","shell.execute_reply.started":"2023-02-18T00:31:23.930430Z","shell.execute_reply":"2023-02-18T00:31:24.401687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.line(train_labels[[\"correct\",\"question\"]].groupby(\"question\").mean(),title=\"Percentage of people who get the questions right the first time\")","metadata":{"execution":{"iopub.status.busy":"2023-02-18T00:31:25.143959Z","iopub.execute_input":"2023-02-18T00:31:25.144492Z","iopub.status.idle":"2023-02-18T00:31:26.451719Z","shell.execute_reply.started":"2023-02-18T00:31:25.144460Z","shell.execute_reply":"2023-02-18T00:31:26.450835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sessions = train_labels[(train_labels.question==1) & (train_labels.correct==1)].session_id.to_list()","metadata":{"execution":{"iopub.status.busy":"2023-02-18T00:31:26.452979Z","iopub.execute_input":"2023-02-18T00:31:26.453512Z","iopub.status.idle":"2023-02-18T00:31:26.461569Z","shell.execute_reply.started":"2023-02-18T00:31:26.453481Z","shell.execute_reply":"2023-02-18T00:31:26.460548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#most of the following code is generated using chatGPT.\n\nquestion_user=train_labels[[\"session_id\",\"question\",\"correct\"]].groupby([\"session_id\",\"question\"]).max().unstack(level=1)\nquestion_user.columns = [i for i in range(1,19)]\nprobs = []\nfor i in range(1,19):\n    probs.append(question_user[question_user[i]==1].mean())\n\nimport plotly.graph_objs as go\n\n# Create sample data\nx = [i for i in range(1,19)]\n\n# Create line traces for each line using a for loop\ntraces = []\nfor i in range(1,19):\n    traces.append(go.Scatter(x=x[i:], y=question_user[question_user[i]==1].mean()[i:], name=f'Prob given q{i} correct'))\n\n# Create a figure with the line traces\nfig = go.Figure(data=traces)\n\n\n# Show the figure\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-18T00:31:26.462923Z","iopub.execute_input":"2023-02-18T00:31:26.463562Z","iopub.status.idle":"2023-02-18T00:31:26.734044Z","shell.execute_reply.started":"2023-02-18T00:31:26.463527Z","shell.execute_reply":"2023-02-18T00:31:26.732804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.bar(train_df.level.value_counts(),title=\"event name distributions\").show()","metadata":{"execution":{"iopub.status.busy":"2023-02-18T00:31:26.735577Z","iopub.execute_input":"2023-02-18T00:31:26.735936Z","iopub.status.idle":"2023-02-18T00:31:26.903750Z","shell.execute_reply.started":"2023-02-18T00:31:26.735904Z","shell.execute_reply":"2023-02-18T00:31:26.902769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It seems that question are independent of one another, and the difficulty is independent of the level of the game. Question 13 seems to be the one that most users get wrong.","metadata":{}},{"cell_type":"code","source":"px.bar(train_df.isnull().mean(),title=\"percentage of null values\")","metadata":{"execution":{"iopub.status.busy":"2023-02-18T00:31:26.904969Z","iopub.execute_input":"2023-02-18T00:31:26.905850Z","iopub.status.idle":"2023-02-18T00:31:27.634411Z","shell.execute_reply.started":"2023-02-18T00:31:26.905802Z","shell.execute_reply":"2023-02-18T00:31:27.633392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_duration = train_df[[\"session_id\",\"elapsed_time\"]].groupby(\"session_id\").max()\ntime_duration = time_duration.clip(upper=np.percentile(time_duration,95))\npx.histogram(time_duration,title=\"distribution of total time for sessions\",nbins=100)","metadata":{"execution":{"iopub.status.busy":"2023-02-18T00:31:27.635677Z","iopub.execute_input":"2023-02-18T00:31:27.636021Z","iopub.status.idle":"2023-02-18T00:31:28.290007Z","shell.execute_reply.started":"2023-02-18T00:31:27.635991Z","shell.execute_reply":"2023-02-18T00:31:28.288989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.bar(train_df.event_name.value_counts(),title=\"event type distributions\").show()\npx.bar(train_df.name.value_counts(),title=\"event name distributions\").show()","metadata":{"execution":{"iopub.status.busy":"2023-02-18T00:31:28.291020Z","iopub.execute_input":"2023-02-18T00:31:28.291426Z","iopub.status.idle":"2023-02-18T00:31:28.572185Z","shell.execute_reply.started":"2023-02-18T00:31:28.291392Z","shell.execute_reply":"2023-02-18T00:31:28.571282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"z = train_df[[\"session_id\",\"elapsed_time\",\"level\"]].groupby([\"level\",\"session_id\"]).agg(lambda x: max(x)-min(x))\npx.line(z.reset_index()[[\"level\",\"elapsed_time\"]].groupby(\"level\").mean(),title=\"Average time spent in each level\")","metadata":{"execution":{"iopub.status.busy":"2023-02-18T00:31:28.573574Z","iopub.execute_input":"2023-02-18T00:31:28.574170Z","iopub.status.idle":"2023-02-18T00:31:35.190592Z","shell.execute_reply.started":"2023-02-18T00:31:28.574124Z","shell.execute_reply":"2023-02-18T00:31:35.188743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"room_coords = train_df[[\"room_coor_y\",\"room_coor_x\"]]//2\nroom_coords = room_coords-room_coords.min()\nheatmap = np.zeros([int(x)+1 for x in room_coords.max().tolist()])\nfor j,c in room_coords.value_counts().iteritems():\n    x,y = int(j[0]),int(j[1])\n    heatmap[x,y] = c\npx.imshow(heatmap,title=\"click click density around the screen\")","metadata":{"execution":{"iopub.status.busy":"2023-02-18T00:52:33.593204Z","iopub.execute_input":"2023-02-18T00:52:33.595557Z","iopub.status.idle":"2023-02-18T00:52:40.864539Z","shell.execute_reply.started":"2023-02-18T00:52:33.595447Z","shell.execute_reply":"2023-02-18T00:52:40.863552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"code","source":"train_df = train_df[[\"session_id\",\"index\",\"elapsed_time\",\"event_name\",\"name\",\"level\",\"level_group\",\"room_coor_x\",\"room_coor_y\"]]\ngc.collect()\ntrain_df = train_df.join(pd.get_dummies(train_df['event_name']))\ntrain_df = train_df.join(pd.get_dummies(train_df['name']))","metadata":{"execution":{"iopub.status.busy":"2023-02-18T06:09:51.539250Z","iopub.execute_input":"2023-02-18T06:09:51.539992Z","iopub.status.idle":"2023-02-18T06:09:53.494087Z","shell.execute_reply.started":"2023-02-18T06:09:51.539941Z","shell.execute_reply":"2023-02-18T06:09:53.492429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_cols = [\"elapsed_time\"]\ncount_cols = [\"index\"]\nstd_cols = [\"room_coor_x\",\"room_coor_y\"]\nsum_cols = ['navigate_click', 'person_click', 'cutscene_click',\n                  'object_click', 'object_hover', 'map_hover',\n                  'notification_click', 'notebook_click', 'map_click',\n                  'observation_click', 'checkpoint','undefined', 'basic', 'close', 'open', 'prev', 'next']\n","metadata":{"execution":{"iopub.status.busy":"2023-02-18T06:09:53.496233Z","iopub.execute_input":"2023-02-18T06:09:53.496673Z","iopub.status.idle":"2023-02-18T06:09:53.503101Z","shell.execute_reply.started":"2023-02-18T06:09:53.496640Z","shell.execute_reply":"2023-02-18T06:09:53.501775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GRPBY=[\"session_id\",\"level_group\"]","metadata":{"execution":{"iopub.status.busy":"2023-02-18T06:09:53.504767Z","iopub.execute_input":"2023-02-18T06:09:53.505385Z","iopub.status.idle":"2023-02-18T06:09:53.519093Z","shell.execute_reply.started":"2023-02-18T06:09:53.505347Z","shell.execute_reply":"2023-02-18T06:09:53.517366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feat_max = train_df[GRPBY+max_cols].groupby(GRPBY).max()\nfeat_count = train_df[GRPBY+count_cols].groupby(GRPBY).count().astype(int)\nfeat_count.columns = [\"total_events\"]\ncount_cols = [\"total_events\"]\nfeat_std = train_df[GRPBY+std_cols].groupby(GRPBY).std()\nfeat_sum = train_df[GRPBY+sum_cols].groupby(GRPBY).sum().astype(int)","metadata":{"execution":{"iopub.status.busy":"2023-02-18T06:09:53.523477Z","iopub.execute_input":"2023-02-18T06:09:53.523898Z","iopub.status.idle":"2023-02-18T06:09:58.963921Z","shell.execute_reply.started":"2023-02-18T06:09:53.523843Z","shell.execute_reply":"2023-02-18T06:09:58.962026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = feat_max.join(feat_count).join(feat_std).join(feat_sum).reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-02-18T06:09:58.965593Z","iopub.execute_input":"2023-02-18T06:09:58.966020Z","iopub.status.idle":"2023-02-18T06:09:59.304267Z","shell.execute_reply.started":"2023-02-18T06:09:58.965984Z","shell.execute_reply":"2023-02-18T06:09:59.302814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import roc_auc_score,f1_score\nimport lightgbm as lgb","metadata":{"execution":{"iopub.status.busy":"2023-02-18T06:09:59.305574Z","iopub.execute_input":"2023-02-18T06:09:59.305947Z","iopub.status.idle":"2023-02-18T06:10:00.374839Z","shell.execute_reply.started":"2023-02-18T06:09:59.305914Z","shell.execute_reply":"2023-02-18T06:10:00.373089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getgroup(x):\n    if x<=3:  return '0-4'\n    elif x<=13: return '5-12'\n    elif x<=22: return '13-22'\n    raise ValueError(\"Should be in range 0-22\")","metadata":{"execution":{"iopub.status.busy":"2023-02-18T06:10:00.377106Z","iopub.execute_input":"2023-02-18T06:10:00.377602Z","iopub.status.idle":"2023-02-18T06:10:00.384226Z","shell.execute_reply.started":"2023-02-18T06:10:00.377568Z","shell.execute_reply":"2023-02-18T06:10:00.383126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels[\"level_group\"] =train_labels[\"question\"].apply(getgroup)\ntrain_labels[\"session_id\"] = train_labels[\"session_id\"].astype(int) ","metadata":{"execution":{"iopub.status.busy":"2023-02-18T06:10:00.385742Z","iopub.execute_input":"2023-02-18T06:10:00.386574Z","iopub.status.idle":"2023-02-18T06:10:00.491140Z","shell.execute_reply.started":"2023-02-18T06:10:00.386485Z","shell.execute_reply":"2023-02-18T06:10:00.489157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2023-02-18T06:10:00.492848Z","iopub.execute_input":"2023-02-18T06:10:00.493341Z","iopub.status.idle":"2023-02-18T06:10:00.499108Z","shell.execute_reply.started":"2023-02-18T06:10:00.493306Z","shell.execute_reply":"2023-02-18T06:10:00.497657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Generated using ChatGPT\n\ndef get_optimal_threshold(y_true, y_pred):\n    \"\"\"\n    Given true labels and predicted probabilities, this function returns the threshold\n    which gives the maximum F1 score.\n    \"\"\"\n    thresholds = sorted(list(set(y_pred)))\n    best_f1 = 0\n    best_threshold = 0\n    for threshold in thresholds:\n        y_pred_binary = [int(p >= threshold) for p in y_pred]\n        f1 = f1_score(y_true, y_pred_binary)\n        if f1 > best_f1:\n            best_f1 = f1\n            best_threshold = threshold\n    return best_threshold","metadata":{"execution":{"iopub.status.busy":"2023-02-18T06:10:00.501053Z","iopub.execute_input":"2023-02-18T06:10:00.501823Z","iopub.status.idle":"2023-02-18T06:10:00.515367Z","shell.execute_reply.started":"2023-02-18T06:10:00.501765Z","shell.execute_reply":"2023-02-18T06:10:00.513745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Generated with help from ChatGPT\n\n## Group-level aggregate idea referenced from: \n# https://www.kaggle.com/code/kimtaehun/lightgbm-baseline-with-aggregated-log-data\n\n# specify the features and target variable\nfeatures = max_cols+count_cols+std_cols+sum_cols\n\n# extract the group column\ngroups = df['session_id']\ntarget_var = \"correct\"\n# specify the parameters for LightGBM\nparams = {\n    'objective': 'binary',\n    'metric': 'binary_logloss',\n    'boosting_type': 'gbdt',\n    'num_leaves': 31,\n    'learning_rate': 0.05,\n    'feature_fraction': 0.9,\n    'bagging_fraction': 0.8,\n    'bagging_freq': 5,\n    'verbose': 0\n}\n\n# initialize a list to store the validation scores\n\nthresholds = []\nmetrics = []\n# initialize a GroupKFold object with 5 folds\ngkf = GroupKFold(n_splits=5)\n\n# loop over the folds\nfor fold, (train_idx, valid_idx) in enumerate(gkf.split(df[features],np.zeros(len(df[features])), groups)):\n    print(f'------------------------Fold {fold}---------------------')\n    \n    model_dir = f'models/fold_{fold}'\n    os.makedirs(model_dir,exist_ok=True)\n    overall_true=[]\n    overall_preds = []\n    overall_preds_prob = []\n\n    for q in range(1,19):\n        print(f'------------------------Question {q}---------------------')\n        train_data = df.iloc[train_idx]\n        train_target = train_labels[train_labels.question==q][[\"session_id\",\"question\",\"correct\"]]\n        train_data = train_data[train_data.level_group==getgroup(q)].merge(train_target,how=\"left\",on=\"session_id\")\n        \n        valid_data = df.iloc[valid_idx]\n        valid_data = valid_data[valid_data.level_group==getgroup(q)].merge(train_target,how=\"left\",on=\"session_id\")\n        # create a LightGBM dataset for training and validation\n        train_dataset = lgb.Dataset(train_data[features], label=train_data[target_var])\n\n        # train the LightGBM model\n        model = lgb.train(params, train_dataset, num_boost_round=100,categorical_feature=[])\n\n        # make predictions on the validation set\n        preds = model.predict(valid_data[features])\n        train_preds = model.predict(train_data[features])\n\n        # evaluate the model's performance on the validation set\n        roc_auc = roc_auc_score(valid_data[target_var], preds)\n        \n        \n        THRES = get_optimal_threshold(train_data[target_var], train_preds)\n        \n        f1sc = f1_score(valid_data[target_var], preds>=THRES)\n        overall_true.extend(valid_data[target_var])\n        overall_preds.extend(preds>=THRES)\n        metrics.append([fold,q,roc_auc,f1sc,THRES])\n        print(f\"Validation ROC AUC score: {roc_auc} , F1 score: {f1sc}, THRES: {THRES}\")\n        model.save_model(f'{model_dir}/model_q{q}.txt')\n        print(\"========================================================================\")\n    overall_f1 = f1_score(overall_true,overall_preds)\n    overall_rocauc = f1_score(overall_true,overall_preds)\n\n    print(f\"Fold {fold}, F1 score: {overall_f1}\")\n    \npdf = pd.DataFrame(np.array(metrics)).round(4)\npdf.columns = [\"fold\",\"question\",\"ROC_AUC\",\"F1\",\"THRES\"]\npdf.to_csv(\"metrics.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2023-02-18T06:35:14.076512Z","iopub.execute_input":"2023-02-18T06:35:14.077001Z","iopub.status.idle":"2023-02-18T06:38:23.696071Z","shell.execute_reply.started":"2023-02-18T06:35:14.076965Z","shell.execute_reply":"2023-02-18T06:38:23.694293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pdf","metadata":{},"execution_count":null,"outputs":[]}]}