{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd, numpy as np, gc\nfrom sklearn.model_selection import KFold, GroupKFold\nfrom xgboost import XGBClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import f1_score\nimport itertools","metadata":{"papermill":{"duration":1.399925,"end_time":"2023-05-26T22:56:54.696745","exception":false,"start_time":"2023-05-26T22:56:53.296820","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-18T16:09:07.149605Z","iopub.execute_input":"2023-06-18T16:09:07.150229Z","iopub.status.idle":"2023-06-18T16:09:07.729901Z","shell.execute_reply.started":"2023-06-18T16:09:07.149867Z","shell.execute_reply":"2023-06-18T16:09:07.728919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/iterstrat')\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold","metadata":{"execution":{"iopub.status.busy":"2023-06-18T16:09:07.733770Z","iopub.execute_input":"2023-06-18T16:09:07.734249Z","iopub.status.idle":"2023-06-18T16:09:07.775452Z","shell.execute_reply.started":"2023-06-18T16:09:07.734215Z","shell.execute_reply":"2023-06-18T16:09:07.774748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mskf = MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-06-18T16:09:07.776167Z","iopub.execute_input":"2023-06-18T16:09:07.776403Z","iopub.status.idle":"2023-06-18T16:09:07.784126Z","shell.execute_reply.started":"2023-06-18T16:09:07.776380Z","shell.execute_reply":"2023-06-18T16:09:07.783481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ USER ID ONLY\ntmp = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train.csv\",usecols=[0])\ntmp = tmp.groupby('session_id').session_id.agg('count')\n\n# COMPUTE READS AND SKIPS\nPIECES = 10\nCHUNK = int( np.ceil(len(tmp)/PIECES) )\n\nreads = []\nskips = [0]\nfor k in range(PIECES):\n    a = k*CHUNK\n    b = (k+1)*CHUNK\n    if b>len(tmp): b=len(tmp)\n    r = tmp.iloc[a:b].sum()\n    reads.append(r)\n    skips.append(skips[-1]+r)\n    \nprint(f'To avoid memory error, we will read train in {PIECES} pieces of sizes:')\nprint(reads)","metadata":{"_kg_hide-input":true,"papermill":{"duration":63.099011,"end_time":"2023-05-26T22:57:57.803009","exception":false,"start_time":"2023-05-26T22:56:54.703998","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-18T16:09:07.785327Z","iopub.execute_input":"2023-06-18T16:09:07.785566Z","iopub.status.idle":"2023-06-18T16:10:03.234699Z","shell.execute_reply.started":"2023-06-18T16:09:07.785537Z","shell.execute_reply":"2023-06-18T16:10:03.234097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv', nrows=reads[0])\nprint('Train size of first piece:', train.shape )\ntrain.head()","metadata":{"papermill":{"duration":4.320314,"end_time":"2023-05-26T22:58:02.130181","exception":false,"start_time":"2023-05-26T22:57:57.809867","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-18T16:10:03.236603Z","iopub.execute_input":"2023-06-18T16:10:03.236950Z","iopub.status.idle":"2023-06-18T16:10:07.231294Z","shell.execute_reply.started":"2023-06-18T16:10:03.236928Z","shell.execute_reply":"2023-06-18T16:10:07.230335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\ntargets['session'] = targets.session_id.apply(lambda x: int(x.split('_')[0]) )\ntargets['q'] = targets.session_id.apply(lambda x: int(x.split('_')[-1][1:]) )\nprint( targets.shape )\ntargets.head()","metadata":{"papermill":{"duration":0.975469,"end_time":"2023-05-26T22:58:03.112901","exception":false,"start_time":"2023-05-26T22:58:02.137432","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-18T16:10:07.232403Z","iopub.execute_input":"2023-06-18T16:10:07.232791Z","iopub.status.idle":"2023-06-18T16:10:08.106709Z","shell.execute_reply.started":"2023-06-18T16:10:07.232767Z","shell.execute_reply":"2023-06-18T16:10:08.105500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mulilabe_targets = pd.pivot_table(targets, columns=['q'], values='correct', index='session').reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-06-18T16:10:08.108118Z","iopub.execute_input":"2023-06-18T16:10:08.108407Z","iopub.status.idle":"2023-06-18T16:10:08.277871Z","shell.execute_reply.started":"2023-06-18T16:10:08.108383Z","shell.execute_reply":"2023-06-18T16:10:08.276948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineer\nWe create basic aggregate features. Try creating more features to boost CV and LB! The idea for EVENTS feature is from [here][1]\n\n[1]: https://www.kaggle.com/code/kimtaehun/lightgbm-baseline-with-aggregated-log-data","metadata":{"papermill":{"duration":0.007437,"end_time":"2023-05-26T22:58:03.128124","exception":false,"start_time":"2023-05-26T22:58:03.120687","status":"completed"},"tags":[]}},{"cell_type":"code","source":"USE_TIME_QUESTIONS = True","metadata":{"execution":{"iopub.status.busy":"2023-06-18T16:10:08.279091Z","iopub.execute_input":"2023-06-18T16:10:08.279337Z","iopub.status.idle":"2023-06-18T16:10:08.283318Z","shell.execute_reply.started":"2023-06-18T16:10:08.279315Z","shell.execute_reply":"2023-06-18T16:10:08.282238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EVENTS = ['cutscene_click', 'person_click', 'navigate_click',\n       'observation_click', 'notification_click', 'object_click', 'map_click', \n       'notebook_click']","metadata":{"papermill":{"duration":0.017694,"end_time":"2023-05-26T22:58:03.153130","exception":false,"start_time":"2023-05-26T22:58:03.135436","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-18T16:10:08.284910Z","iopub.execute_input":"2023-06-18T16:10:08.285301Z","iopub.status.idle":"2023-06-18T16:10:08.294896Z","shell.execute_reply.started":"2023-06-18T16:10:08.285262Z","shell.execute_reply":"2023-06-18T16:10:08.293861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"RIGHT_FQID_DICT = {\n    'tunic': ((200, 50), (350, 200)), \n    'plaque.face.date': ((590, 500), (-80, -150)),\n    'plaque': ((590, 500), (-80, -150)),\n    'businesscards': ((150, 50), (-100, -150)),\n    'businesscards.card_bingo.bingo':  ((150, 50), (-100, -150)),\n    'logbook': ((500, -500), (40, -10)),\n    'logbook.page.bingo': ((500, -500), (40, -10)),\n    'reader': ((-150, -300), (-90, -120)),\n    'reader.paper2.bingo': ((-150, -300), (-90, -120)),\n    'tracks': ((1100, 950), (-320, -500)),\n    #'colorbook': True,\n    'reader_flag': ((0, -290), (110, -90))\n    #'journals_flag': True\n}","metadata":{"papermill":{"duration":0.018075,"end_time":"2023-05-26T22:58:03.178583","exception":false,"start_time":"2023-05-26T22:58:03.160508","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-18T16:11:00.888823Z","iopub.execute_input":"2023-06-18T16:11:00.889145Z","iopub.status.idle":"2023-06-18T16:11:00.896309Z","shell.execute_reply.started":"2023-06-18T16:11:00.889119Z","shell.execute_reply":"2023-06-18T16:11:00.895401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def right_click_object(corr_x, coor_y, fqid, coords):\n    \n    if (corr_x < coords[fqid][0][0]) & (corr_x > coords[fqid][0][1]) & (coor_y < coords[fqid][1][0]) & (coor_y < coords[fqid][1][1]):\n\n        return 1\n    else:\n        return 0","metadata":{"papermill":{"duration":0.01631,"end_time":"2023-05-26T22:58:03.202304","exception":false,"start_time":"2023-05-26T22:58:03.185994","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-18T16:11:05.459826Z","iopub.execute_input":"2023-06-18T16:11:05.460155Z","iopub.status.idle":"2023-06-18T16:11:05.465452Z","shell.execute_reply.started":"2023-06-18T16:11:05.460127Z","shell.execute_reply":"2023-06-18T16:11:05.464645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"COLUMNS_DROP = ['music', 'hq', 'fullscreen',\n               'screen_coor_y', 'screen_coor_x', 'page']","metadata":{"papermill":{"duration":0.015888,"end_time":"2023-05-26T22:58:03.225748","exception":false,"start_time":"2023-05-26T22:58:03.209860","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-18T16:11:06.927550Z","iopub.execute_input":"2023-06-18T16:11:06.927880Z","iopub.status.idle":"2023-06-18T16:11:06.931781Z","shell.execute_reply.started":"2023-06-18T16:11:06.927850Z","shell.execute_reply":"2023-06-18T16:11:06.930792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FQIDS_0_4 = ['tunic', 'plaque.face.date', 'plaque']\nFQIDS_5_12 = ['businesscards', 'businesscards.card_bingo.bingo',  'reader', 'reader.paper2.bingo', 'logbook', 'logbook.page.bingo']\nFQIDS_13_22 = ['tracks','reader_flag']","metadata":{"papermill":{"duration":0.016174,"end_time":"2023-05-26T22:58:03.249590","exception":false,"start_time":"2023-05-26T22:58:03.233416","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-18T16:11:07.748478Z","iopub.execute_input":"2023-06-18T16:11:07.750453Z","iopub.status.idle":"2023-06-18T16:11:07.754501Z","shell.execute_reply.started":"2023-06-18T16:11:07.750415Z","shell.execute_reply":"2023-06-18T16:11:07.753523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# FQIDS_0_4 = ['tunic.hub.slip', 'plaque.face.date']\n# FQIDS_5_12 = ['businesscards.card_bingo.bingo', 'reader']\n# FQIDS_13_22 = ['tracks','reader_flag']","metadata":{"execution":{"iopub.status.busy":"2023-06-18T16:10:08.314983Z","iopub.status.idle":"2023-06-18T16:10:08.315531Z","shell.execute_reply.started":"2023-06-18T16:10:08.315338Z","shell.execute_reply":"2023-06-18T16:10:08.315355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valids_5_12 = ['tunic.humanecology',\n 'tunic.drycleaner',\n 'tunic.capitol_2',\n 'tunic.flaghouse',\n 'tunic.capitol_1',\n 'tunic.historicalsociety',\n 'tunic.capitol_0',\n 'tunic.wildlife',\n 'tunic.library',\n 'tunic.kohlcenter']","metadata":{"papermill":{"duration":0.014748,"end_time":"2023-05-26T22:58:03.271913","exception":false,"start_time":"2023-05-26T22:58:03.257165","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-18T16:11:09.851123Z","iopub.execute_input":"2023-06-18T16:11:09.852462Z","iopub.status.idle":"2023-06-18T16:11:09.855960Z","shell.execute_reply.started":"2023-06-18T16:11:09.852418Z","shell.execute_reply":"2023-06-18T16:11:09.855402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg_dict = {\n    'diff_event': {'std', 'max', 'median'},\n}","metadata":{"execution":{"iopub.status.busy":"2023-06-18T16:11:10.893082Z","iopub.execute_input":"2023-06-18T16:11:10.893604Z","iopub.status.idle":"2023-06-18T16:11:10.896761Z","shell.execute_reply.started":"2023-06-18T16:11:10.893575Z","shell.execute_reply":"2023-06-18T16:11:10.896215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generic_features(train):\n    \n    train['diff_event'] = train['elapsed_time'].diff(1).clip(0, 103000).replace(np.nan, 0)\n    #train['diff_event_delt'] = train['diff_event'].diff(1)\n    \n    map_click_events = train.loc[train['event_name'] == 'map_click'].reset_index(drop=True)\n    \n    map_click_events = map_click_events.groupby(['session_id', 'fqid'])['count'].agg('sum').reset_index()\n    \n    map_click_events = pd.pivot_table(map_click_events, values='count', index='session_id', columns=['fqid'], fill_value=0)\n    map_click_events.columns = ['_'.join([col, 'map_event']).strip() for col in map_click_events.columns.values]\n    \n    map_click_events = map_click_events.reset_index(drop=True)\n    \n    median_diff_time_events_level = train.groupby(['session_id', 'event_name'])['diff_event'].agg('median').reset_index()\n    #median_diff_time_events_level.columns = ['_'.join([col[1], 'event_names_agg']) for col in median_diff_time_events_level.columns.values]\n    median_diff_time_events_level = pd.pivot_table(median_diff_time_events_level, values='diff_event', \n                                                   index='session_id', columns=['event_name'], fill_value=0)\n    \n    median_diff_time_events_level.columns = ['_'.join([col, 'diff_event_session']) for col in median_diff_time_events_level.columns.values.astype(str)]\n    \n    #median_diff_time_events_level.columns = ['_'.join([f\"{index}\", 'diff_event_session']) for index in range(len(median_diff_time_events_level.columns))]\n\n    median_diff_time_events_level = median_diff_time_events_level.reset_index(drop=True)\n\n    median_diff_time_room = train.groupby(['session_id', 'room_fqid'])['diff_event'].agg('median').reset_index()\n    median_diff_time_room = pd.pivot_table(median_diff_time_room, values='diff_event', \n                                                   index='session_id', columns=['room_fqid'], fill_value=0)\n    median_diff_time_room.columns = ['_'.join([col, 'diff_room_session']) for col in median_diff_time_room.columns.values.astype(str)]\n\n    median_diff_time_room = median_diff_time_room.reset_index(drop=True)\n    \n    median_event_diff_time_room = train.groupby(['session_id', 'room_fqid', 'event_name'])['diff_event'].agg('median').reset_index()\n    median_event_diff_time_room.rename(columns={'diff_event':'df', 'event_name': 'ev_na', 'room_fqid': 'room_ID'}, inplace=True)\n    \n    median_event_diff_time_room = pd.pivot_table(median_event_diff_time_room, values='df', index='session_id', columns=['room_ID', 'ev_na'], fill_value=0)\n    #median_event_diff_time_room.columns = ['_'.join([col, 'text_nunique_obs']) for col in median_event_diff_time_room.columns.values.astype(str)]\n    \n    median_event_diff_time_room = median_event_diff_time_room.reset_index(drop=True)\n    \n    count_room = train.groupby(['session_id', 'room_fqid'])['count'].agg('sum').reset_index()\n    count_room = pd.pivot_table(count_room, values='count', \n                                                   index='session_id', columns=['room_fqid'], fill_value=0)\n    count_room.columns = ['_'.join([col, 'count_room_session']) for col in count_room.columns.values.astype(str)]\n\n    count_room = count_room.reset_index(drop=True)\n    \n    median_diff_time_name = train.groupby(['session_id', 'name'])['diff_event'].agg('median').reset_index()\n    median_diff_time_name = pd.pivot_table(median_diff_time_name, values='diff_event', \n                                                   index='session_id', columns=['name'], fill_value=0)\n    median_diff_time_name.columns = ['_'.join([col, 'diff_name_session']) for col in median_diff_time_name.columns.values.astype(str)]\n\n    median_diff_time_name = median_diff_time_name.reset_index(drop=True)\n    \n    count_name = train.groupby(['session_id', 'name'])['count'].agg('sum').reset_index()\n    count_name = pd.pivot_table(count_name, values='count', \n                                                   index='session_id', columns=['name'], fill_value=0)\n    count_name.columns = ['_'.join([col, 'count_name_session']) for col in count_name.columns.values.astype(str)]\n\n    count_name = count_name.reset_index(drop=True)\n    \n    median_diff_time_text_fq = train.groupby(['session_id', 'text_fqid'])['diff_event'].agg('median').reset_index()\n    median_diff_time_text_fq = pd.pivot_table(median_diff_time_text_fq, values='diff_event', \n                                                   index='session_id', columns=['text_fqid'], fill_value=0)\n    median_diff_time_text_fq.columns = ['_'.join([col, 'diff_text_fq_session']) for col in median_diff_time_text_fq.columns.values.astype(str)]\n\n    median_diff_time_text_fq = median_diff_time_text_fq.reset_index(drop=True)\n    \n    count_text_fq = train.groupby(['session_id', 'text_fqid'])['count'].agg('sum').reset_index()\n    count_text_fq = pd.pivot_table(count_text_fq, values='count', \n                                                   index='session_id', columns=['text_fqid'], fill_value=0)\n    count_text_fq.columns = ['_'.join([col, 'count_text_fq_session']) for col in count_text_fq.columns.values.astype(str)]\n\n    count_text_fq = count_text_fq.reset_index(drop=True)\n    \n    median_diff_time_level_g = train.groupby(['session_id', 'level_group'])['diff_event'].agg('median').reset_index()\n    median_diff_time_level_g = pd.pivot_table(median_diff_time_level_g, values='diff_event', \n                                                   index='session_id', columns=['level_group'], fill_value=0)\n    median_diff_time_level_g.columns = ['_'.join([col, 'diff_level_g_session']) for col in median_diff_time_level_g.columns.values.astype(str)]\n\n    median_diff_time_level_g = median_diff_time_level_g.reset_index(drop=True)\n    \n    object_click_unique = train.loc[train['event_name'] == 'object_click'].groupby(['session_id', 'text_fqid'])['count'].agg('sum').reset_index()\n    object_click_unique = pd.pivot_table(object_click_unique, values='count', index='session_id', columns=['text_fqid'], fill_value=0)\n    object_click_unique.columns = ['_'.join([col, 'object_click_con']) for col in object_click_unique.columns.values.astype(str)]\n    \n    object_click_unique = object_click_unique.reset_index(drop=True)\n    \n#     person_click_unique = train.loc[train['event_name'] == 'person_click'].groupby(['session_id', 'fqid'])['count'].agg('sum').reset_index()\n#     person_click_unique = pd.pivot_table(person_click_unique, values='count', index='session_id', columns=['fqid'], fill_value=0)\n#     person_click_unique.columns = ['_'.join([col, 'person_click_con']) for col in person_click_unique.columns.values.astype(str)]\n    \n#     person_click_unique = person_click_unique.reset_index(drop=True)\n    \n    observation_unique_texts = train.loc[train['event_name'] == 'observation_click'].groupby(['session_id'])['text'].agg('nunique').reset_index().drop(columns=['session_id'])\n    observation_unique_texts.rename(columns={'text':'text_nunique_obs'}, inplace=True)\n    \n#     observation_unique_texts = pd.pivot_table(observation_unique_texts, values='text_nunique_obs', index='session_id', columns=['level'], fill_value=0)\n#     observation_unique_texts.columns = ['_'.join([col, 'text_nunique_obs']) for col in observation_unique_texts.columns.values.astype(str)]\n    \n#     observation_unique_texts = observation_unique_texts.reset_index(drop=True)\n    \n#     notif_unique_texts = train.loc[train['event_name'] == 'notification_click'].groupby(['session_id'])['text'].agg('nunique').reset_index().drop(columns=['session_id'])\n#     notif_unique_texts.rename(columns={'text':'notifi_nunique_obs'}, inplace=True)\n    \n#     notif_unique_texts = pd.pivot_table(notif_unique_texts, values='notifi_nunique_obs', index='session_id', columns=['level'], fill_value=0)\n#     notif_unique_texts.columns = ['_'.join([col, 'notifi_nunique_obs']) for col in notif_unique_texts.columns.values.astype(str)]\n    \n#     notif_unique_texts = notif_unique_texts.reset_index(drop=True)\n    \n    level_time = train.groupby(['session_id', 'level'])['diff_event'].agg('median').reset_index()\n    level_time = pd.pivot_table(level_time, values=['diff_event'], index='session_id', columns=['level'], fill_value=0)\n    #level_time.columns = ['_'.join([col, 'level_diff_event']) for col in level_time.columns.values.astype(str)]\n\n    level_time = level_time.reset_index(drop=True)#.drop(columns=['session_id'])\n    \n    level_count = train.groupby(['session_id', 'level'])['count'].agg('sum').reset_index()\n    level_count = pd.pivot_table(level_count, values='count', index='session_id', columns=['level'], fill_value=0)\n    level_count.columns = ['_'.join([col, 'level_count']) for col in level_count.columns.values.astype(str)]\n    \n    level_count = level_count.reset_index(drop=True)\n    \n    for c in EVENTS: \n        train[c] = (train.event_name == c).astype('int8')\n    \n    dfs = []\n    \n    dfs.append(train.groupby(['session_id'])['count'].agg('sum').reset_index())\n    \n    for c in EVENTS:\n\n        temp = train.groupby(['session_id'])[c].agg('sum').reset_index().drop(columns=['session_id'])\n\n        dfs.append(temp)\n        \n        del temp\n        \n    train = pd.concat(dfs,axis=1)\n    \n    train = train.reset_index(drop=True)\n    \n    df = pd.concat([train, level_time, median_diff_time_events_level, map_click_events,\n                   observation_unique_texts, object_click_unique, median_diff_time_room, median_diff_time_text_fq,\n                   median_diff_time_level_g, count_text_fq, median_diff_time_name, count_name, count_room,\n                   level_count, median_event_diff_time_room],axis=1)\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2023-06-18T16:11:12.852546Z","iopub.execute_input":"2023-06-18T16:11:12.853110Z","iopub.status.idle":"2023-06-18T16:11:12.882067Z","shell.execute_reply.started":"2023-06-18T16:11:12.853084Z","shell.execute_reply":"2023-06-18T16:11:12.881477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineer_0_4(train, columns_recreate=[], submit=False):\n    \n    train.sort_values(by=['session_id', 'elapsed_time'], inplace=True)\n    \n    max_time = train.groupby(['session_id'])['elapsed_time'].agg('max')#.reset_index(drop=True).drop(columns=['session_id'])\n    max_time.name = 'maximum time'\n    max_time = max_time.reset_index(drop=True).drop(columns=['session_id'])\n    \n    mim_time = train.groupby(['session_id'])['elapsed_time'].agg('min')#.reset_index(drop=True).drop(columns=['session_id'])\n    mim_time.name = 'minimum time'\n    mim_time = mim_time.reset_index(drop=True).drop(columns=['session_id'])\n    \n    time_plaque = train.loc[((train['event_name'] == 'navigate_click') & (train['fqid'] == 'plaque') |\n              (train['event_name'] == 'notification_click') & (train['text_fqid'] == 'tunic.kohlcenter.halloffame.plaque.face.date'))].reset_index()\n    \n    time_plaque['duration'] = time_plaque['elapsed_time'].diff(1).clip(0, 103000).replace(np.nan, 0)\n    \n    \n    time_plaque = time_plaque.groupby(['session_id'])['duration'].agg('max').reset_index(drop=True).drop(columns=['session_id'])\n    \n    train['count'] = 1\n\n#     text_1_time = train.loc[train['text'] == \"The slip is from 1916 but the team didn't start until 1974!\"].groupby(['session_id'])['elapsed_time'].agg('min').reset_index()\n#     text_1_time.rename(columns={'elapsed_time':'text_1_time'}, inplace=True)\n#     text_1_index = train.loc[train['text'] == \"The slip is from 1916 but the team didn't start until 1974!\"].groupby(['session_id'])['index'].agg('min').reset_index()\n#     text_1_index.rename(columns={'index':'text_1_index'}, inplace=True)\n#     text_2_time = train.loc[train['text'] == \"This looks like a clue!\"].groupby(['session_id'])['elapsed_time'].agg('min').reset_index()\n#     text_2_time.rename(columns={'elapsed_time':'text_2_time'}, inplace=True)\n#     text_2_index = train.loc[train['text'] == \"This looks like a clue!\"].groupby(['session_id'])['index'].agg('min').reset_index()\n#     text_2_index.rename(columns={'index':'text_2_index'}, inplace=True)\n#     text_3_time = train.loc[train['text'] == \"Youmans was a suffragist!!\"].groupby(['session_id'])['elapsed_time'].agg('min').reset_index()\n#     text_2_time.rename(columns={'elapsed_time':'text_3_time'}, inplace=True)\n#     text_3_index = train.loc[train['text'] == \"Youmans was a suffragist!!\"].groupby(['session_id'])['index'].agg('min').reset_index()\n#     text_3_index.rename(columns={'index':'text_3_index'}, inplace=True)\n\n    train.loc[((train['fqid'].isin(FQIDS_0_4)) & (train['event_name'] == 'object_click') \n                   & (train['name'] != 'close')), 'correct_clicks'] = train.loc[((train['fqid'].isin(FQIDS_0_4)) & (train['event_name'] == 'object_click')\n                                                                    & (train['name'] != 'close'))].apply(lambda row: right_click_object(row['room_coor_x'], \n                                                                                                                        row['room_coor_y'], \n                                                                                                                        row['fqid'], \n                                                                                                                        RIGHT_FQID_DICT), 1)\n\n    session_ids = train['session_id'].unique()\n    \n    session_ids_df = pd.DataFrame(session_ids, columns=['session_id'])\n#     text_1_time = pd.merge(session_ids_df, text_1_time, on=['session_id'], how='left').drop(columns=['session_id'])\n#     text_1_index = pd.merge(session_ids_df, text_1_index, on=['session_id'], how='left').drop(columns=['session_id'])\n#     text_2_time = pd.merge(session_ids_df, text_2_time, on=['session_id'], how='left').drop(columns=['session_id'])\n#     text_2_index = pd.merge(session_ids_df, text_2_index, on=['session_id'], how='left').drop(columns=['session_id'])\n#     text_3_time = pd.merge(session_ids_df, text_3_time, on=['session_id'], how='left').drop(columns=['session_id'])\n#     text_3_index = pd.merge(session_ids_df, text_3_index, on=['session_id'], how='left').drop(columns=['session_id'])\n    \n    correct_clicks = [0, 1]\n    combinations = list(itertools.product(session_ids, FQIDS_0_4, correct_clicks))\n\n    # Create a new dataframe with all possible combinations\n    new_df = pd.DataFrame(combinations, columns=['session_id', 'fqid', 'correct_clicks'])\n    \n    merged_df = pd.merge(new_df, train.groupby(['session_id', 'fqid', 'correct_clicks'])['count'].agg('sum').reset_index(),\n                         on=['session_id', 'fqid', 'correct_clicks'], \n                         how='left')\n    \n    merged_df['count'] = merged_df['count'].replace(np.nan, 0)\n    \n    table = pd.pivot_table(merged_df, values='count', index='session_id', columns=['fqid', 'correct_clicks'], fill_value=0)\n\n    # Flatten the column MultiIndex\n    #table.columns = ['_'.join(map(str, col)).strip() for col in table.columns.values]\n\n    # Reset the index to turn the session_id into a column\n    table = table.reset_index(drop=True)\n    \n    df = generic_features(train)\n    \n    df = pd.concat(\n        [\n            df,table, time_plaque, max_time, mim_time\n        ]\n        ,axis=1\n    )\n        \n    if submit:\n        df = df.reindex(columns=columns_recreate, fill_value=0)\n \n    return df\n\ndef feature_engineer_5_12(train, columns_recreate=[], submit=False):\n    \n    train.sort_values(by=['session_id', 'elapsed_time'], inplace=True)\n    \n    max_time = train.groupby(['session_id'])['elapsed_time'].agg('max')#.reset_index(drop=True).drop(columns=['session_id'])\n    max_time.name = 'maximum time'\n    max_time = max_time.reset_index(drop=True).drop(columns=['session_id'])\n    \n    mim_time = train.groupby(['session_id'])['elapsed_time'].agg('min')#.reset_index(drop=True).drop(columns=['session_id'])\n    mim_time.name = 'minimum time'\n    mim_time = mim_time.reset_index(drop=True).drop(columns=['session_id'])\n    \n    time_bingo = train.loc[((train['event_name'] == 'navigate_click') & (train['fqid'] == 'businesscards') |\n              (train['event_name'] == 'notification_click') & (train['text_fqid'] == 'tunic.humanecology.frontdesk.businesscards.card_bingo.bingo'))].reset_index()\n    \n    time_bingo['duration'] = time_bingo['elapsed_time'].diff(1).clip(0, 103000).replace(np.nan, 0)\n    \n    \n    time_bingo = time_bingo.groupby(['session_id'])['duration'].agg('max').reset_index(drop=True).drop(columns=['session_id'])\n    \n    time_loog_b = train.loc[((train['event_name'] == 'navigate_click') & (train['fqid'] == 'logbook') |\n              (train['text'] == \"It's a match!\") & (train['text_fqid'] == 'tunic.drycleaner.frontdesk.logbook.page.bingo'))].reset_index()\n    \n    time_loog_b['duration_2'] = time_loog_b['elapsed_time'].diff(1).clip(0, 103000).replace(np.nan, 0)\n    \n    \n    time_loog_b = time_loog_b.groupby(['session_id'])['duration_2'].agg('max').reset_index(drop=True).drop(columns=['session_id'])\n    \n    time_crofiche = train.loc[((train['event_name'] == 'navigate_click') & (train['fqid'] == 'tomicrofiche') |\n              (train['text'] == \"Youmans was a suffragist!\") & (train['text_fqid'] == 'tunic.library.microfiche.reader.paper2.bingo'))].reset_index()\n    \n    time_crofiche['duration_3'] = time_crofiche['elapsed_time'].diff(1).clip(0, 103000).replace(np.nan, 0)\n    \n    \n    time_crofiche = time_crofiche.groupby(['session_id'])['duration_3'].agg('max').reset_index(drop=True).drop(columns=['session_id'])\n    \n    #print(time_bingo)\n    \n    train['count'] = 1\n    \n#     text_1 = train.loc[train['text'] == \"Where did you get that coffee?\"].groupby(['session_id'])['elapsed_time'].agg('min').reset_index()\n#     text_1.rename(columns={'elapsed_time':'text_1_time'}, inplace=True)\n#     text_2 = train.loc[train['text'] == \"It's a match!\"].groupby(['session_id'])['elapsed_time'].agg('min').reset_index()\n#     text_2.rename(columns={'elapsed_time':'text_2_time'}, inplace=True)\n    \n    \n#     click_loog_book = train.loc[((train['text'] == \"Here's the log book.\")\n#                       | (train['text'] == \"It's a match!\"))].groupby(['session_id'])['index'].diff(1).replace(np.nan, 0).reset_index().drop(columns=['session_id'])\n\n    train.loc[((train['fqid'].isin(FQIDS_5_12)) & (train['event_name'] == 'object_click') \n                   & (train['name'] != 'close')), 'correct_clicks'] = train.loc[((train['fqid'].isin(FQIDS_5_12)) & (train['event_name'] == 'object_click')\n                                                                    & (train['name'] != 'close'))].apply(lambda row: right_click_object(row['room_coor_x'], \n                                                                                                                        row['room_coor_y'], \n                                                                                                                        row['fqid'], \n                                                                                                                        RIGHT_FQID_DICT), 1)\n    session_ids = train['session_id'].unique()\n    \n    session_ids_df = pd.DataFrame(session_ids, columns=['session_id'])\n#     text_1 = pd.merge(session_ids_df, text_1, on=['session_id'], how='left').drop(columns=['session_id'])\n#     text_2 = pd.merge(session_ids_df, text_2, on=['session_id'], how='left').drop(columns=['session_id'])\n    correct_clicks = [0, 1]\n    combinations = list(itertools.product(session_ids, FQIDS_5_12, correct_clicks))\n\n    # Create a new dataframe with all possible combinations\n    new_df = pd.DataFrame(combinations, columns=['session_id', 'fqid', 'correct_clicks'])\n    \n    merged_df = pd.merge(new_df, train.groupby(['session_id', 'fqid', 'correct_clicks'])['count'].agg('sum').reset_index(), \n                         on=['session_id', 'fqid', 'correct_clicks'], how='left')\n    \n    merged_df['count'] = merged_df['count'].replace(np.nan, 0)\n    \n    table = pd.pivot_table(merged_df, values='count', index='session_id', columns=['fqid', 'correct_clicks'], fill_value=0)\n\n    # Flatten the column MultiIndex\n    #table.columns = ['_'.join(map(str, col)).strip() for col in table.columns.values]\n\n    # Reset the index to turn the session_id into a column\n    table = table.reset_index(drop=True)\n    \n    df = generic_features(train)\n    \n    df = pd.concat(\n        [\n            df,table, time_bingo, time_loog_b, max_time, mim_time\n        ]\n        ,axis=1\n    )\n\n    if submit:\n        df = df.reindex(columns=columns_recreate, fill_value=0)\n    \n    return df\n\ndef feature_engineer_13_22(train, columns_recreate=[], submit=False):\n    \n    train.sort_values(by=['session_id', 'elapsed_time'], inplace=True)\n    \n    max_time = train.groupby(['session_id'])['elapsed_time'].agg('max')#.reset_index(drop=True).drop(columns=['session_id'])\n    max_time.name = 'maximum time'\n    max_time = max_time.reset_index(drop=True).drop(columns=['session_id'])\n    \n    mim_time = train.groupby(['session_id'])['elapsed_time'].agg('min')#.reset_index(drop=True).drop(columns=['session_id'])\n    mim_time.name = 'minimum time'\n    mim_time = mim_time.reset_index(drop=True).drop(columns=['session_id'])\n    \n    time_hoofprint = train.loc[((train['event_name'] == 'person_click') & (train['text'] == \"There's a diagram of animal tracks over there.\") |\n              (train['event_name'] == 'notification_click') & (train['text_fqid'] == 'tunic.wildlife.center.tracks.hub.deer'))].reset_index()\n    \n    time_hoofprint['duration'] = time_hoofprint['elapsed_time'].diff(1).clip(0, 103000).replace(np.nan, 0)\n    \n    \n    time_hoofprint = time_hoofprint.groupby(['session_id'])['duration'].agg('max').reset_index(drop=True).drop(columns=['session_id'])\n    \n    time_national_moviment = train.loc[((train['event_name'] == 'person_click') & (train['text'] == \"Your flag must have been part of a national movement!\") |\n              (train['event_name'] == 'notification_click') & (train['text_fqid'] == 'tunic.library.microfiche.reader_flag.paper2.bingo'))].reset_index()\n    \n    time_national_moviment['duration_2'] = time_national_moviment['elapsed_time'].diff(1).clip(0, 103000).replace(np.nan, 0)\n    \n    \n    time_national_moviment = time_national_moviment.groupby(['session_id'])['duration_2'].agg('max').reset_index(drop=True).drop(columns=['session_id'])\n    \n    \n    \n    train['count'] = 1\n    \n    train.loc[((train['fqid'].isin(FQIDS_13_22)) & (train['event_name'] == 'object_click') \n                   & (train['name'] != 'close')), 'correct_clicks'] = train.loc[((train['fqid'].isin(FQIDS_13_22)) & (train['event_name'] == 'object_click')\n                                                                    & (train['name'] != 'close'))].apply(lambda row: right_click_object(row['room_coor_x'], \n                                                                                                                        row['room_coor_y'], \n                                                                                                                        row['fqid'], \n                                                                                                                        RIGHT_FQID_DICT), 1)\n    session_ids = train['session_id'].unique()\n    correct_clicks = [0, 1]\n    combinations = list(itertools.product(session_ids, FQIDS_13_22, correct_clicks))\n\n    new_df = pd.DataFrame(combinations, columns=['session_id', 'fqid', 'correct_clicks'])\n    \n    merged_df = pd.merge(new_df, train.groupby(['session_id','fqid', 'correct_clicks'])['count'].agg('sum').reset_index(), \n                         on=['session_id', 'fqid', 'correct_clicks'], how='left')\n    \n    merged_df = merged_df.groupby(['session_id', 'fqid', 'correct_clicks'])['count'].agg('sum').reset_index()\n    table = pd.pivot_table(merged_df, values='count', index='session_id', columns=['fqid', 'correct_clicks'], fill_value=0)\n\n    table = table.reset_index(drop=True)\n    \n    df = generic_features(train)\n    \n    df = pd.concat(\n        [\n            df,table, max_time, mim_time#, time_hoofprint, time_national_moviment\n        ]\n        ,axis=1\n    )\n    \n    if submit:\n        df = df.reindex(columns=columns_recreate, fill_value=0)\n        \n    return df\n    ","metadata":{"execution":{"iopub.status.busy":"2023-06-18T16:11:13.837864Z","iopub.execute_input":"2023-06-18T16:11:13.838443Z","iopub.status.idle":"2023-06-18T16:11:13.877224Z","shell.execute_reply.started":"2023-06-18T16:11:13.838414Z","shell.execute_reply":"2023-06-18T16:11:13.876614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# PROCESS TRAIN DATA IN PIECES\nall_pieces_0_4 = []\nall_pieces_5_12 = []\nall_pieces_13_22 = []\nprint(f'Processing train as {PIECES} pieces to avoid memory error... ')\nfor k in range(PIECES):\n    print(k,', ',end='')\n    SKIPS = 0\n    if k>0: SKIPS = range(1,skips[k]+1)\n    train = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv',\n                        nrows=reads[k], skiprows=SKIPS).drop(columns=COLUMNS_DROP)\n    \n    df_0_4 = feature_engineer_0_4(train.loc[train['level_group'] == '0-4'].reset_index(drop=True))\n    df_5_12 = feature_engineer_5_12(train.loc[train['level_group'] == '5-12'].reset_index(drop=True))\n    df_13_22 = feature_engineer_13_22(train.loc[train['level_group'] == '13-22'].reset_index(drop=True))\n    all_pieces_0_4.append(df_0_4)\n    all_pieces_5_12.append(df_5_12)\n    all_pieces_13_22.append(df_13_22)\n    \n# CONCATENATE ALL PIECES\nprint('\\n')\ndel train; gc.collect()\ndf_0_4 = pd.concat(all_pieces_0_4, axis=0).reset_index(drop=True)\ndf_5_12 = pd.concat(all_pieces_5_12, axis=0).reset_index(drop=True)\ndf_13_22 = pd.concat(all_pieces_13_22, axis=0).reset_index(drop=True)\nprint('Shape of all train data 0-4 after feature engineering:', df_0_4.shape )\nprint('Shape of all train data 5-12 after feature engineering:', df_5_12.shape )\nprint('Shape of all train data 13-22 after feature engineering:', df_13_22.shape )","metadata":{"papermill":{"duration":238.357436,"end_time":"2023-05-26T23:02:01.752520","exception":false,"start_time":"2023-05-26T22:58:03.395084","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-18T16:11:15.197155Z","iopub.execute_input":"2023-06-18T16:11:15.197703Z","iopub.status.idle":"2023-06-18T16:15:29.006206Z","shell.execute_reply.started":"2023-06-18T16:11:15.197678Z","shell.execute_reply":"2023-06-18T16:15:29.005322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_0_4_columns = list(df_0_4)\ndf_5_12_columns = list(df_5_12)\ndf_13_22_columns = list(df_13_22)","metadata":{"papermill":{"duration":0.016818,"end_time":"2023-05-26T23:02:01.777293","exception":false,"start_time":"2023-05-26T23:02:01.760475","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-18T16:15:29.009302Z","iopub.execute_input":"2023-06-18T16:15:29.009847Z","iopub.status.idle":"2023-06-18T16:15:29.014501Z","shell.execute_reply.started":"2023-06-18T16:15:29.009816Z","shell.execute_reply":"2023-06-18T16:15:29.013835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train XGBoost Model\nWe train one model for each of 18 questions. Furthermore, we use data from `level_groups = '0-4'` to train model for questions 1-3, and `level groups '5-12'` to train questions 4 thru 13 and `level groups '13-22'` to train questions 14 thru 18. Because this is the data we get (to predict corresponding questions) from Kaggle's inference API during test inference. We can improve our model by saving a user's previous data from earlier `level_groups` and using that to predict future `level_groups`.","metadata":{"papermill":{"duration":0.007656,"end_time":"2023-05-26T23:02:01.792992","exception":false,"start_time":"2023-05-26T23:02:01.785336","status":"completed"},"tags":[]}},{"cell_type":"code","source":"mulilabe_targets","metadata":{"execution":{"iopub.status.busy":"2023-06-18T16:15:29.015605Z","iopub.execute_input":"2023-06-18T16:15:29.016051Z","iopub.status.idle":"2023-06-18T16:15:29.040883Z","shell.execute_reply.started":"2023-06-18T16:15:29.016024Z","shell.execute_reply":"2023-06-18T16:15:29.039836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_TIME_QUESTIONS:\n    df_13_22['time_questions'] = df_13_22['minimum time'] - df_5_12['maximum time']\n    df_5_12['time_questions'] = df_5_12['minimum time'] - df_0_4['maximum time']\n    df_13_22['time_questions_5_12'] = df_5_12['time_questions']\n    \n    df_13_22.drop(columns=['minimum time', 'maximum time'], inplace=True)\n    df_5_12.drop(columns=['minimum time', 'maximum time'], inplace=True)\n    df_0_4.drop(columns=['maximum time', 'minimum time'], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-18T16:15:29.042583Z","iopub.execute_input":"2023-06-18T16:15:29.042832Z","iopub.status.idle":"2023-06-18T16:15:29.069137Z","shell.execute_reply.started":"2023-06-18T16:15:29.042809Z","shell.execute_reply":"2023-06-18T16:15:29.068432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gkf = GroupKFold(n_splits=5)\noof = pd.DataFrame(data=np.zeros((len(mulilabe_targets),18)), index=mulilabe_targets['session'].values.reshape(-1))\nmodels = {}\n\ntresholds = []\n\n# COMPUTE CV SCORE WITH 5 GROUP K FOLD\nfor i, (train_index, test_index) in enumerate(mskf.split(X=mulilabe_targets['session'].values, y=mulilabe_targets.iloc[:, 2:])):\n#for i, (train_index, test_index) in enumerate(gkf.split(X=mulilabe_targets['session'].values, groups=df_0_4.index)):\n    print('#'*25)\n    print('### Fold',i+1)\n    print('#'*25)\n    \n    xgb_params = {\n    'objective' : 'binary:logistic',\n    'eval_metric':'logloss',\n    'learning_rate': 0.05,\n    'max_depth': 4,\n    'n_estimators': 1000,\n    'early_stopping_rounds': 250,\n    'tree_method':'hist',\n    'subsample':0.8,\n    'colsample_bytree': 0.4,\n    'use_label_encoder' : False}\n    \n    # ITERATE THRU QUESTIONS 1 THRU 18\n    folds_tresholds = []\n    for t in range(1,19):\n        \n        # USE THIS TRAIN DATA WITH THESE QUESTIONS\n        if t<=3: \n            train_x = df_0_4.iloc[train_index]\n            valid_x = df_0_4.iloc[test_index]\n        elif t<=13: \n            train_x = df_5_12.iloc[train_index]\n            valid_x = df_5_12.iloc[test_index]\n        elif t<=22: \n            train_x = df_13_22.iloc[train_index]\n            valid_x = df_13_22.iloc[test_index]\n            \n        # TRAIN DATA\n        train_users = train_x['session_id'].values\n        train_y = targets.loc[targets.q==t].set_index('session').loc[train_users]\n        \n        # VALID DATA\n        valid_users = valid_x['session_id'].values\n        valid_y = targets.loc[targets.q==t].set_index('session').loc[valid_users]\n        \n        # TRAIN MODEL        \n        clf =  XGBClassifier(**xgb_params)\n        clf.fit(train_x.iloc[:, 1:].astype('float32'), train_y['correct'],\n                eval_set=[ (valid_x.iloc[:, 1:].astype('float32'), valid_y['correct']) ],\n                verbose=0)\n        #clf = LogisticRegression(random_state=0)\n        #clf.fit(train_x.astype('float32'), train_y['correct'])\n        \n        prob = clf.predict_proba(valid_x.iloc[:, 1:].astype('float32'))[:,1]\n        \n        scores = []             \n        thresholds = []\n        best_score = 0; best_threshold = 0\n\n        for threshold in np.arange(0.4,0.81,0.01):\n    #         print(f'{threshold:.02f}, ',end='')\n            preds = (prob>threshold).astype('int')\n            m = f1_score(valid_y['correct'], preds, average='macro')   \n            scores.append(m)\n            thresholds.append(threshold)\n            if m>best_score:\n                best_score = m\n                best_threshold = threshold\n        \n        folds_tresholds.append(best_threshold)\n        \n        print(f'{t}({clf.best_ntree_limit})({best_score:.2f}/{best_threshold:.2f}), ',end='')\n        #print(f'{t}(({best_score:.2f}/{best_threshold:.2f}), ',end='')\n        \n        # SAVE MODEL, PREDICT VALID OOF\n        models[f'{t}'] = clf\n        oof.loc[valid_users, t-1] = clf.predict_proba(valid_x.iloc[:, 1:].astype('float32'))[:,1]\n        \n    tresholds.append(folds_tresholds)\n        \n    print()","metadata":{"papermill":{"duration":460.922053,"end_time":"2023-05-26T23:09:42.722843","exception":false,"start_time":"2023-05-26T23:02:01.800790","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-18T16:15:29.070088Z","iopub.execute_input":"2023-06-18T16:15:29.071035Z","iopub.status.idle":"2023-06-18T16:32:44.737253Z","shell.execute_reply.started":"2023-06-18T16:15:29.071006Z","shell.execute_reply":"2023-06-18T16:32:44.735616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Compute CV Score\nWe need to convert prediction probabilities into `1s` and `0s`. The competition metric is F1 Score which is the harmonic mean of precision and recall. Let's find the optimal threshold for `p > threshold` when to predict `1` and when to predict `0` to maximize F1 Score.","metadata":{"papermill":{"duration":0.012378,"end_time":"2023-05-26T23:09:42.748051","exception":false,"start_time":"2023-05-26T23:09:42.735673","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# PUT TRUE LABELS INTO DATAFRAME WITH 18 COLUMNS\ntrue = oof.copy()\nfor k in range(18):\n    # GET TRUE LABELS\n    tmp = targets.loc[targets.q == k+1].set_index('session').loc[df_0_4['session_id'].values.reshape(-1)]\n    true[k] = tmp.correct.values","metadata":{"papermill":{"duration":0.111224,"end_time":"2023-05-26T23:09:42.871755","exception":false,"start_time":"2023-05-26T23:09:42.760531","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-18T16:32:47.523902Z","iopub.execute_input":"2023-06-18T16:32:47.524232Z","iopub.status.idle":"2023-06-18T16:32:47.613919Z","shell.execute_reply.started":"2023-06-18T16:32:47.524205Z","shell.execute_reply":"2023-06-18T16:32:47.613250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# FIND BEST THRESHOLD TO CONVERT PROBS INTO 1s AND 0s\nscores = []; thresholds = []\nbest_score = 0; best_threshold = 0\n\nfor threshold in np.arange(0.4,0.81,0.01):\n    print(f'{threshold:.02f}, ',end='')\n    preds = (oof.values.reshape((-1))>threshold).astype('int')\n    m = f1_score(true.values.reshape((-1)), preds, average='macro')   \n    scores.append(m)\n    thresholds.append(threshold)\n    if m>best_score:\n        best_score = m\n        best_threshold = threshold","metadata":{"papermill":{"duration":4.436458,"end_time":"2023-05-26T23:09:47.321542","exception":false,"start_time":"2023-05-26T23:09:42.885084","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-18T16:32:48.315562Z","iopub.execute_input":"2023-06-18T16:32:48.315882Z","iopub.status.idle":"2023-06-18T16:32:52.562780Z","shell.execute_reply.started":"2023-06-18T16:32:48.315856Z","shell.execute_reply":"2023-06-18T16:32:52.561565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# PLOT THRESHOLD VS. F1_SCORE\nplt.figure(figsize=(20,5))\nplt.plot(thresholds,scores,'-o',color='blue')\nplt.scatter([best_threshold], [best_score], color='blue', s=300, alpha=1)\nplt.xlabel('Threshold',size=14)\nplt.ylabel('Validation F1 Score',size=14)\nplt.title(f'Threshold vs. F1_Score with Best F1_Score = {best_score:.3f} at Best Threshold = {best_threshold:.3}',size=18)\nplt.show()","metadata":{"papermill":{"duration":0.233147,"end_time":"2023-05-26T23:09:47.568358","exception":false,"start_time":"2023-05-26T23:09:47.335211","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-18T16:32:52.564101Z","iopub.execute_input":"2023-06-18T16:32:52.564365Z","iopub.status.idle":"2023-06-18T16:32:52.816890Z","shell.execute_reply.started":"2023-06-18T16:32:52.564342Z","shell.execute_reply":"2023-06-18T16:32:52.814359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('When using optimal threshold...')\nfor k in range(18):\n        \n    # COMPUTE F1 SCORE PER QUESTION\n    m = f1_score(true[k].values, (oof[k].values>best_threshold).astype('int'), average='macro')\n    print(f'Q{k + 1}: F1 =',m)\n    \n# COMPUTE F1 SCORE OVERALL\nm = f1_score(true.values.reshape((-1)), (oof.values.reshape((-1))>best_threshold).astype('int'), average='macro')\nprint('==> Overall F1 =',m)","metadata":{"papermill":{"duration":0.244531,"end_time":"2023-05-26T23:09:47.827535","exception":false,"start_time":"2023-05-26T23:09:47.583004","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-18T16:32:52.818308Z","iopub.execute_input":"2023-06-18T16:32:52.818583Z","iopub.status.idle":"2023-06-18T16:32:53.029936Z","shell.execute_reply.started":"2023-06-18T16:32:52.818558Z","shell.execute_reply":"2023-06-18T16:32:53.029339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Infer Test Data","metadata":{"papermill":{"duration":0.014381,"end_time":"2023-05-26T23:09:47.856394","exception":false,"start_time":"2023-05-26T23:09:47.842013","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#jo_wilder.make_env.__called__ = False","metadata":{"papermill":{"duration":0.022542,"end_time":"2023-05-26T23:09:47.893402","exception":false,"start_time":"2023-05-26T23:09:47.870860","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-18T16:10:08.334958Z","iopub.status.idle":"2023-06-18T16:10:08.335251Z","shell.execute_reply.started":"2023-06-18T16:10:08.335091Z","shell.execute_reply":"2023-06-18T16:10:08.335111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# IMPORT KAGGLE API\nimport jo_wilder\nenv = jo_wilder.make_env()\n#type(env)._state = type(type(env)._state).__dict__['INIT']\niter_test = env.iter_test()\n\n#CLEAR MEMORY\nimport gc\ndel targets, df_0_4, df_5_12, df_13_22, oof, true, all_pieces_0_4, all_pieces_5_12, all_pieces_13_22\n_ = gc.collect()","metadata":{"papermill":{"duration":0.170792,"end_time":"2023-05-26T23:09:48.078828","exception":false,"start_time":"2023-05-26T23:09:47.908036","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-18T16:10:08.335885Z","iopub.status.idle":"2023-06-18T16:10:08.336155Z","shell.execute_reply.started":"2023-06-18T16:10:08.336016Z","shell.execute_reply":"2023-06-18T16:10:08.336029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"limits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\ndict_data = {\n    '0-4': {},\n    '5-12': {},\n    '13-22': {}\n}\n\nfor (test, sample_submission) in iter_test:\n    \n    # FEATURE ENGINEER TEST DATA\n    # INFER TEST DATA\n    grp = test.level_group.values[0]\n    a,b = limits[grp]\n    \n    if grp == '0-4':\n        df = feature_engineer_0_4(test.drop(columns=COLUMNS_DROP), df_0_4_columns, True)\n    elif grp == '5-12':\n        df = feature_engineer_5_12(test.drop(columns=COLUMNS_DROP), df_5_12_columns, True)\n    elif grp == '13-22':\n        df = feature_engineer_13_22(test.drop(columns=COLUMNS_DROP), df_13_22_columns, True)\n        \n    if USE_TIME_QUESTIONS:\n        if grp == '0-4':\n            dict_data[grp]['level_data'] = df['maximum time'].values\n            \n            df.drop(columns=[ 'maximum time', 'minimum time'], inplace=True)\n        elif grp == '5-12':\n            dict_data[grp]['level_data'] = df['maximum time'].values\n\n            df['time_questions'] = df['minimum time'] - dict_data['0-4']['level_data']\n            dict_data[grp]['time_question_data'] = df['time_questions'].values\n            \n            df.drop(columns=['minimum time', 'maximum time'], inplace=True)\n        else:\n            df['time_questions'] = df['minimum time'] - dict_data['5-12']['level_data']\n            df['time_questions_5_12'] = dict_data['5-12']['time_question_data']\n            \n            df.drop(columns=['minimum time', 'maximum time'], inplace=True)\n    \n\n    for t in range(a,b):\n        clf = models[f'{t}']\n        p = clf.predict_proba(df.iloc[:, 1:].values.astype('float32'))[0,1]\n        mask = sample_submission.session_id.str.contains(f'q{t}')\n        sample_submission.loc[mask,'correct'] = int( p > best_threshold )\n\n    env.predict(sample_submission)","metadata":{"papermill":{"duration":1.766316,"end_time":"2023-05-26T23:09:49.859536","exception":false,"start_time":"2023-05-26T23:09:48.093220","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-18T16:10:08.337247Z","iopub.status.idle":"2023-06-18T16:10:08.337519Z","shell.execute_reply.started":"2023-06-18T16:10:08.337382Z","shell.execute_reply":"2023-06-18T16:10:08.337396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sdasdasd","metadata":{"papermill":{"duration":0.022954,"end_time":"2023-05-26T23:09:49.898529","exception":false,"start_time":"2023-05-26T23:09:49.875575","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-18T16:10:08.338314Z","iopub.status.idle":"2023-06-18T16:10:08.338581Z","shell.execute_reply.started":"2023-06-18T16:10:08.338446Z","shell.execute_reply":"2023-06-18T16:10:08.338459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA submission.csv","metadata":{"papermill":{"duration":0.01405,"end_time":"2023-05-26T23:09:49.927254","exception":false,"start_time":"2023-05-26T23:09:49.913204","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df = pd.read_csv('submission.csv')\nprint( df.shape )\ndf.head()","metadata":{"papermill":{"duration":0.029008,"end_time":"2023-05-26T23:09:49.970425","exception":false,"start_time":"2023-05-26T23:09:49.941417","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-18T16:10:08.339607Z","iopub.status.idle":"2023-06-18T16:10:08.339886Z","shell.execute_reply.started":"2023-06-18T16:10:08.339747Z","shell.execute_reply":"2023-06-18T16:10:08.339761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df.correct.mean())","metadata":{"papermill":{"duration":0.023178,"end_time":"2023-05-26T23:09:50.008374","exception":false,"start_time":"2023-05-26T23:09:49.985196","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-18T16:10:08.340837Z","iopub.status.idle":"2023-06-18T16:10:08.341113Z","shell.execute_reply.started":"2023-06-18T16:10:08.340971Z","shell.execute_reply":"2023-06-18T16:10:08.340984Z"},"trusted":true},"execution_count":null,"outputs":[]}]}