{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd, numpy as np, gc\nfrom sklearn.model_selection import KFold, GroupKFold\nfrom xgboost import XGBClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import f1_score\nimport itertools","metadata":{"papermill":{"duration":1.399925,"end_time":"2023-05-26T22:56:54.696745","exception":false,"start_time":"2023-05-26T22:56:53.29682","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-05T06:57:29.585387Z","iopub.execute_input":"2023-07-05T06:57:29.586533Z","iopub.status.idle":"2023-07-05T06:57:29.592694Z","shell.execute_reply.started":"2023-07-05T06:57:29.586489Z","shell.execute_reply":"2023-07-05T06:57:29.591655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/iterstrat')\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold","metadata":{"execution":{"iopub.status.busy":"2023-07-05T06:57:32.379442Z","iopub.execute_input":"2023-07-05T06:57:32.380253Z","iopub.status.idle":"2023-07-05T06:57:32.385060Z","shell.execute_reply.started":"2023-07-05T06:57:32.380214Z","shell.execute_reply":"2023-07-05T06:57:32.384057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mskf = MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T06:57:36.983888Z","iopub.execute_input":"2023-07-05T06:57:36.984302Z","iopub.status.idle":"2023-07-05T06:57:36.990021Z","shell.execute_reply.started":"2023-07-05T06:57:36.984267Z","shell.execute_reply":"2023-07-05T06:57:36.988996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ USER ID ONLY\ntmp = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train.csv\",usecols=[0])\ntmp = tmp.groupby('session_id').session_id.agg('count')\n\n# COMPUTE READS AND SKIPS\nPIECES = 10\nCHUNK = int( np.ceil(len(tmp)/PIECES) )\n\nreads = []\nskips = [0]\nfor k in range(PIECES):\n    a = k*CHUNK\n    b = (k+1)*CHUNK\n    if b>len(tmp): b=len(tmp)\n    r = tmp.iloc[a:b].sum()\n    reads.append(r)\n    skips.append(skips[-1]+r)\n    \nprint(f'To avoid memory error, we will read train in {PIECES} pieces of sizes:')\nprint(reads)","metadata":{"_kg_hide-input":true,"papermill":{"duration":63.099011,"end_time":"2023-05-26T22:57:57.803009","exception":false,"start_time":"2023-05-26T22:56:54.703998","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-05T06:57:42.108053Z","iopub.execute_input":"2023-07-05T06:57:42.108474Z","iopub.status.idle":"2023-07-05T06:58:52.822694Z","shell.execute_reply.started":"2023-07-05T06:57:42.108440Z","shell.execute_reply":"2023-07-05T06:58:52.819062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv', nrows=reads[0])\nprint('Train size of first piece:', train.shape )\ntrain.head()","metadata":{"papermill":{"duration":4.320314,"end_time":"2023-05-26T22:58:02.130181","exception":false,"start_time":"2023-05-26T22:57:57.809867","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-05T06:58:56.010617Z","iopub.execute_input":"2023-07-05T06:58:56.011850Z","iopub.status.idle":"2023-07-05T06:59:04.941912Z","shell.execute_reply.started":"2023-07-05T06:58:56.011805Z","shell.execute_reply":"2023-07-05T06:59:04.940978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\ntargets['session'] = targets.session_id.apply(lambda x: int(x.split('_')[0]) )\ntargets['q'] = targets.session_id.apply(lambda x: int(x.split('_')[-1][1:]) )\nprint( targets.shape )\ntargets.head()","metadata":{"papermill":{"duration":0.975469,"end_time":"2023-05-26T22:58:03.112901","exception":false,"start_time":"2023-05-26T22:58:02.137432","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-05T06:59:08.600734Z","iopub.execute_input":"2023-07-05T06:59:08.602081Z","iopub.status.idle":"2023-07-05T06:59:10.470401Z","shell.execute_reply.started":"2023-07-05T06:59:08.602032Z","shell.execute_reply":"2023-07-05T06:59:10.469296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mulilabe_targets = pd.pivot_table(targets, columns=['q'], values='correct', index='session').reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-07-05T06:59:14.818208Z","iopub.execute_input":"2023-07-05T06:59:14.818645Z","iopub.status.idle":"2023-07-05T06:59:15.058977Z","shell.execute_reply.started":"2023-07-05T06:59:14.818610Z","shell.execute_reply":"2023-07-05T06:59:15.057927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mulilabe_targets","metadata":{"execution":{"iopub.status.busy":"2023-07-05T06:59:20.833816Z","iopub.execute_input":"2023-07-05T06:59:20.834231Z","iopub.status.idle":"2023-07-05T06:59:20.856329Z","shell.execute_reply.started":"2023-07-05T06:59:20.834198Z","shell.execute_reply":"2023-07-05T06:59:20.855509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineer\nWe create basic aggregate features. Try creating more features to boost CV and LB! The idea for EVENTS feature is from [here][1]\n\n[1]: https://www.kaggle.com/code/kimtaehun/lightgbm-baseline-with-aggregated-log-data","metadata":{"papermill":{"duration":0.007437,"end_time":"2023-05-26T22:58:03.128124","exception":false,"start_time":"2023-05-26T22:58:03.120687","status":"completed"},"tags":[]}},{"cell_type":"code","source":"USE_TIME_QUESTIONS = True","metadata":{"execution":{"iopub.status.busy":"2023-07-05T06:59:49.783193Z","iopub.execute_input":"2023-07-05T06:59:49.783921Z","iopub.status.idle":"2023-07-05T06:59:49.788348Z","shell.execute_reply.started":"2023-07-05T06:59:49.783884Z","shell.execute_reply":"2023-07-05T06:59:49.787319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EVENTS = ['cutscene_click', 'person_click', 'navigate_click',\n       'observation_click', 'notification_click', 'object_click', 'map_click', \n       'notebook_click']","metadata":{"papermill":{"duration":0.017694,"end_time":"2023-05-26T22:58:03.15313","exception":false,"start_time":"2023-05-26T22:58:03.135436","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-05T06:59:53.202191Z","iopub.execute_input":"2023-07-05T06:59:53.203106Z","iopub.status.idle":"2023-07-05T06:59:53.207733Z","shell.execute_reply.started":"2023-07-05T06:59:53.203063Z","shell.execute_reply":"2023-07-05T06:59:53.206700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"RIGHT_FQID_DICT = {\n    'tunic': ((200, 50), (350, 200)), \n    'plaque.face.date': ((590, 500), (-80, -150)),\n    'plaque': ((590, 500), (-80, -150)),\n    'businesscards': ((150, 50), (-100, -150)),\n    'businesscards.card_bingo.bingo':  ((150, 50), (-100, -150)),\n    'logbook': ((500, -500), (40, -10)),\n    'logbook.page.bingo': ((500, -500), (40, -10)),\n    'reader': ((-150, -300), (-90, -120)),\n    'reader.paper2.bingo': ((-150, -300), (-90, -120)),\n    'tracks': ((1100, 950), (-320, -500)),\n    #'colorbook': True,\n    'reader_flag': ((0, -290), (110, -90))\n    #'journals_flag': True\n}","metadata":{"papermill":{"duration":0.018075,"end_time":"2023-05-26T22:58:03.178583","exception":false,"start_time":"2023-05-26T22:58:03.160508","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-05T06:59:55.561343Z","iopub.execute_input":"2023-07-05T06:59:55.562176Z","iopub.status.idle":"2023-07-05T06:59:55.572341Z","shell.execute_reply.started":"2023-07-05T06:59:55.562133Z","shell.execute_reply":"2023-07-05T06:59:55.571126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def right_click_object(corr_x, coor_y, fqid, coords):\n    \n    if (corr_x < coords[fqid][0][0]) & (corr_x > coords[fqid][0][1]) & (coor_y < coords[fqid][1][0]) & (coor_y < coords[fqid][1][1]):\n\n        return 1\n    else:\n        return 0","metadata":{"papermill":{"duration":0.01631,"end_time":"2023-05-26T22:58:03.202304","exception":false,"start_time":"2023-05-26T22:58:03.185994","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-05T06:59:59.232093Z","iopub.execute_input":"2023-07-05T06:59:59.232514Z","iopub.status.idle":"2023-07-05T06:59:59.239550Z","shell.execute_reply.started":"2023-07-05T06:59:59.232481Z","shell.execute_reply":"2023-07-05T06:59:59.238586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"COLUMNS_DROP = ['music', 'hq', 'fullscreen',\n               'screen_coor_y', 'screen_coor_x', 'page']","metadata":{"papermill":{"duration":0.015888,"end_time":"2023-05-26T22:58:03.225748","exception":false,"start_time":"2023-05-26T22:58:03.20986","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-05T07:00:02.368865Z","iopub.execute_input":"2023-07-05T07:00:02.369903Z","iopub.status.idle":"2023-07-05T07:00:02.374932Z","shell.execute_reply.started":"2023-07-05T07:00:02.369858Z","shell.execute_reply":"2023-07-05T07:00:02.373989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FQIDS_0_4 = ['tunic', 'plaque.face.date', 'plaque']\nFQIDS_5_12 = ['businesscards', 'businesscards.card_bingo.bingo',  'reader', 'reader.paper2.bingo', 'logbook', 'logbook.page.bingo']\nFQIDS_13_22 = ['tracks','reader_flag']","metadata":{"papermill":{"duration":0.016174,"end_time":"2023-05-26T22:58:03.24959","exception":false,"start_time":"2023-05-26T22:58:03.233416","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-05T07:00:04.424815Z","iopub.execute_input":"2023-07-05T07:00:04.425914Z","iopub.status.idle":"2023-07-05T07:00:04.431114Z","shell.execute_reply.started":"2023-07-05T07:00:04.425873Z","shell.execute_reply":"2023-07-05T07:00:04.430260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# FQIDS_0_4 = ['tunic.hub.slip', 'plaque.face.date']\n# FQIDS_5_12 = ['businesscards.card_bingo.bingo', 'reader']\n# FQIDS_13_22 = ['tracks','reader_flag']","metadata":{"execution":{"iopub.status.busy":"2023-07-05T06:18:57.011932Z","iopub.execute_input":"2023-07-05T06:18:57.012569Z","iopub.status.idle":"2023-07-05T06:18:57.023148Z","shell.execute_reply.started":"2023-07-05T06:18:57.012518Z","shell.execute_reply":"2023-07-05T06:18:57.022219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valids_5_12 = ['tunic.humanecology',\n 'tunic.drycleaner',\n 'tunic.capitol_2',\n 'tunic.flaghouse',\n 'tunic.capitol_1',\n 'tunic.historicalsociety',\n 'tunic.capitol_0',\n 'tunic.wildlife',\n 'tunic.library',\n 'tunic.kohlcenter']","metadata":{"papermill":{"duration":0.014748,"end_time":"2023-05-26T22:58:03.271913","exception":false,"start_time":"2023-05-26T22:58:03.257165","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-05T07:00:09.141457Z","iopub.execute_input":"2023-07-05T07:00:09.141886Z","iopub.status.idle":"2023-07-05T07:00:09.147789Z","shell.execute_reply.started":"2023-07-05T07:00:09.141852Z","shell.execute_reply":"2023-07-05T07:00:09.146683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg_dict = {\n    'diff_event': {'std', 'max', 'median'},\n}","metadata":{"execution":{"iopub.status.busy":"2023-07-05T07:00:11.675193Z","iopub.execute_input":"2023-07-05T07:00:11.675613Z","iopub.status.idle":"2023-07-05T07:00:11.680771Z","shell.execute_reply.started":"2023-07-05T07:00:11.675581Z","shell.execute_reply":"2023-07-05T07:00:11.679810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generic_features(train):\n    \n    train['diff_event'] = train['elapsed_time'].diff(1).clip(0, 103000).replace(np.nan, 0)\n    #train['diff_event_delt'] = train['diff_event'].diff(1)\n    \n    map_click_events = train.loc[train['event_name'] == 'map_click'].reset_index(drop=True)\n    \n    map_click_events = map_click_events.groupby(['session_id', 'fqid'])['count'].agg('sum').reset_index()\n    \n    map_click_events = pd.pivot_table(map_click_events, values='count', index='session_id', columns=['fqid'], fill_value=0)\n    map_click_events.columns = ['_'.join([col, 'map_event']).strip() for col in map_click_events.columns.values]\n    \n    map_click_events = map_click_events.reset_index(drop=True)\n    \n    median_diff_time_events_level = train.groupby(['session_id', 'event_name'])['diff_event'].agg('median').reset_index()\n    #median_diff_time_events_level.columns = ['_'.join([col[1], 'event_names_agg']) for col in median_diff_time_events_level.columns.values]\n    median_diff_time_events_level = pd.pivot_table(median_diff_time_events_level, values='diff_event', \n                                                   index='session_id', columns=['event_name'], fill_value=0)\n    \n    median_diff_time_events_level.columns = ['_'.join([col, 'diff_event_session']) for col in median_diff_time_events_level.columns.values.astype(str)]\n    \n    #median_diff_time_events_level.columns = ['_'.join([f\"{index}\", 'diff_event_session']) for index in range(len(median_diff_time_events_level.columns))]\n\n    median_diff_time_events_level = median_diff_time_events_level.reset_index(drop=True)\n\n    median_diff_time_room = train.groupby(['session_id', 'room_fqid'])['diff_event'].agg('median').reset_index()\n    median_diff_time_room = pd.pivot_table(median_diff_time_room, values='diff_event', \n                                                   index='session_id', columns=['room_fqid'], fill_value=0)\n    median_diff_time_room.columns = ['_'.join([col, 'diff_room_session']) for col in median_diff_time_room.columns.values.astype(str)]\n\n    median_diff_time_room = median_diff_time_room.reset_index(drop=True)\n    \n    median_event_diff_time_room = train.groupby(['session_id', 'room_fqid', 'event_name'])['diff_event'].agg('median').reset_index()\n    median_event_diff_time_room.rename(columns={'diff_event':'df', 'event_name': 'ev_na', 'room_fqid': 'room_ID'}, inplace=True)\n    \n    median_event_diff_time_room = pd.pivot_table(median_event_diff_time_room, values='df', index='session_id', columns=['room_ID', 'ev_na'], fill_value=0)\n    #median_event_diff_time_room.columns = ['_'.join([col, 'text_nunique_obs']) for col in median_event_diff_time_room.columns.values.astype(str)]\n    \n    median_event_diff_time_room = median_event_diff_time_room.reset_index(drop=True)\n    \n    count_room = train.groupby(['session_id', 'room_fqid'])['count'].agg('sum').reset_index()\n    count_room = pd.pivot_table(count_room, values='count', \n                                                   index='session_id', columns=['room_fqid'], fill_value=0)\n    count_room.columns = ['_'.join([col, 'count_room_session']) for col in count_room.columns.values.astype(str)]\n\n    count_room = count_room.reset_index(drop=True)\n    \n    median_diff_time_name = train.groupby(['session_id', 'name'])['diff_event'].agg('median').reset_index()\n    median_diff_time_name = pd.pivot_table(median_diff_time_name, values='diff_event', \n                                                   index='session_id', columns=['name'], fill_value=0)\n    median_diff_time_name.columns = ['_'.join([col, 'diff_name_session']) for col in median_diff_time_name.columns.values.astype(str)]\n\n    median_diff_time_name = median_diff_time_name.reset_index(drop=True)\n    \n    count_name = train.groupby(['session_id', 'name'])['count'].agg('sum').reset_index()\n    count_name = pd.pivot_table(count_name, values='count', \n                                                   index='session_id', columns=['name'], fill_value=0)\n    count_name.columns = ['_'.join([col, 'count_name_session']) for col in count_name.columns.values.astype(str)]\n\n    count_name = count_name.reset_index(drop=True)\n    \n    median_diff_time_text_fq = train.groupby(['session_id', 'text_fqid'])['diff_event'].agg('median').reset_index()\n    median_diff_time_text_fq = pd.pivot_table(median_diff_time_text_fq, values='diff_event', \n                                                   index='session_id', columns=['text_fqid'], fill_value=0)\n    median_diff_time_text_fq.columns = ['_'.join([col, 'diff_text_fq_session']) for col in median_diff_time_text_fq.columns.values.astype(str)]\n\n    median_diff_time_text_fq = median_diff_time_text_fq.reset_index(drop=True)\n    \n    count_text_fq = train.groupby(['session_id', 'text_fqid'])['count'].agg('sum').reset_index()\n    count_text_fq = pd.pivot_table(count_text_fq, values='count', \n                                                   index='session_id', columns=['text_fqid'], fill_value=0)\n    count_text_fq.columns = ['_'.join([col, 'count_text_fq_session']) for col in count_text_fq.columns.values.astype(str)]\n\n    count_text_fq = count_text_fq.reset_index(drop=True)\n    \n    median_diff_time_level_g = train.groupby(['session_id', 'level_group'])['diff_event'].agg('median').reset_index()\n    median_diff_time_level_g = pd.pivot_table(median_diff_time_level_g, values='diff_event', \n                                                   index='session_id', columns=['level_group'], fill_value=0)\n    median_diff_time_level_g.columns = ['_'.join([col, 'diff_level_g_session']) for col in median_diff_time_level_g.columns.values.astype(str)]\n\n    median_diff_time_level_g = median_diff_time_level_g.reset_index(drop=True)\n    \n    object_click_unique = train.loc[train['event_name'] == 'object_click'].groupby(['session_id', 'text_fqid'])['count'].agg('sum').reset_index()\n    object_click_unique = pd.pivot_table(object_click_unique, values='count', index='session_id', columns=['text_fqid'], fill_value=0)\n    object_click_unique.columns = ['_'.join([col, 'object_click_con']) for col in object_click_unique.columns.values.astype(str)]\n    \n    object_click_unique = object_click_unique.reset_index(drop=True)\n    \n#     person_click_unique = train.loc[train['event_name'] == 'person_click'].groupby(['session_id', 'fqid'])['count'].agg('sum').reset_index()\n#     person_click_unique = pd.pivot_table(person_click_unique, values='count', index='session_id', columns=['fqid'], fill_value=0)\n#     person_click_unique.columns = ['_'.join([col, 'person_click_con']) for col in person_click_unique.columns.values.astype(str)]\n    \n#     person_click_unique = person_click_unique.reset_index(drop=True)\n    \n    observation_unique_texts = train.loc[train['event_name'] == 'observation_click'].groupby(['session_id'])['text'].agg('nunique').reset_index().drop(columns=['session_id'])\n    observation_unique_texts.rename(columns={'text':'text_nunique_obs'}, inplace=True)\n    \n#     observation_unique_texts = pd.pivot_table(observation_unique_texts, values='text_nunique_obs', index='session_id', columns=['level'], fill_value=0)\n#     observation_unique_texts.columns = ['_'.join([col, 'text_nunique_obs']) for col in observation_unique_texts.columns.values.astype(str)]\n    \n#     observation_unique_texts = observation_unique_texts.reset_index(drop=True)\n    \n#     notif_unique_texts = train.loc[train['event_name'] == 'notification_click'].groupby(['session_id'])['text'].agg('nunique').reset_index().drop(columns=['session_id'])\n#     notif_unique_texts.rename(columns={'text':'notifi_nunique_obs'}, inplace=True)\n    \n#     notif_unique_texts = pd.pivot_table(notif_unique_texts, values='notifi_nunique_obs', index='session_id', columns=['level'], fill_value=0)\n#     notif_unique_texts.columns = ['_'.join([col, 'notifi_nunique_obs']) for col in notif_unique_texts.columns.values.astype(str)]\n    \n#     notif_unique_texts = notif_unique_texts.reset_index(drop=True)\n    \n    level_time = train.groupby(['session_id', 'level'])['diff_event'].agg('median').reset_index()\n    level_time = pd.pivot_table(level_time, values=['diff_event'], index='session_id', columns=['level'], fill_value=0)\n    #level_time.columns = ['_'.join([col, 'level_diff_event']) for col in level_time.columns.values.astype(str)]\n\n    level_time = level_time.reset_index(drop=True)#.drop(columns=['session_id'])\n    \n    level_count = train.groupby(['session_id', 'level'])['count'].agg('sum').reset_index()\n    level_count = pd.pivot_table(level_count, values='count', index='session_id', columns=['level'], fill_value=0)\n    level_count.columns = ['_'.join([col, 'level_count']) for col in level_count.columns.values.astype(str)]\n    \n    level_count = level_count.reset_index(drop=True)\n    \n    for c in EVENTS: \n        train[c] = (train.event_name == c).astype('int8')\n    \n    dfs = []\n    \n    dfs.append(train.groupby(['session_id'])['count'].agg('sum').reset_index())\n    \n    for c in EVENTS:\n\n        temp = train.groupby(['session_id'])[c].agg('sum').reset_index().drop(columns=['session_id'])\n\n        dfs.append(temp)\n        \n        del temp\n        \n    train = pd.concat(dfs,axis=1)\n    \n    train = train.reset_index(drop=True)\n    \n    df = pd.concat([train, level_time, median_diff_time_events_level, map_click_events,\n                   observation_unique_texts, object_click_unique, median_diff_time_room, median_diff_time_text_fq,\n                   median_diff_time_level_g, count_text_fq, median_diff_time_name, count_name, count_room,\n                   level_count, median_event_diff_time_room],axis=1)\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2023-07-05T07:00:15.027801Z","iopub.execute_input":"2023-07-05T07:00:15.028653Z","iopub.status.idle":"2023-07-05T07:00:15.081965Z","shell.execute_reply.started":"2023-07-05T07:00:15.028612Z","shell.execute_reply":"2023-07-05T07:00:15.080546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineer_0_4(train, columns_recreate=[], submit=False):\n    \n    train.sort_values(by=['session_id', 'elapsed_time'], inplace=True)\n    \n    max_time = train.groupby(['session_id'])['elapsed_time'].agg('max')#.reset_index(drop=True).drop(columns=['session_id'])\n    max_time.name = 'maximum time'\n    max_time = max_time.reset_index(drop=True).drop(columns=['session_id'])\n    \n    mim_time = train.groupby(['session_id'])['elapsed_time'].agg('min')#.reset_index(drop=True).drop(columns=['session_id'])\n    mim_time.name = 'minimum time'\n    mim_time = mim_time.reset_index(drop=True).drop(columns=['session_id'])\n    \n    time_plaque = train.loc[((train['event_name'] == 'navigate_click') & (train['fqid'] == 'plaque') |\n              (train['event_name'] == 'notification_click') & (train['text_fqid'] == 'tunic.kohlcenter.halloffame.plaque.face.date'))].reset_index()\n    \n    time_plaque['duration'] = time_plaque['elapsed_time'].diff(1).clip(0, 103000).replace(np.nan, 0)\n    \n    \n    time_plaque = time_plaque.groupby(['session_id'])['duration'].agg('max').reset_index(drop=True).drop(columns=['session_id'])\n    \n    train['count'] = 1\n\n#     text_1_time = train.loc[train['text'] == \"The slip is from 1916 but the team didn't start until 1974!\"].groupby(['session_id'])['elapsed_time'].agg('min').reset_index()\n#     text_1_time.rename(columns={'elapsed_time':'text_1_time'}, inplace=True)\n#     text_1_index = train.loc[train['text'] == \"The slip is from 1916 but the team didn't start until 1974!\"].groupby(['session_id'])['index'].agg('min').reset_index()\n#     text_1_index.rename(columns={'index':'text_1_index'}, inplace=True)\n#     text_2_time = train.loc[train['text'] == \"This looks like a clue!\"].groupby(['session_id'])['elapsed_time'].agg('min').reset_index()\n#     text_2_time.rename(columns={'elapsed_time':'text_2_time'}, inplace=True)\n#     text_2_index = train.loc[train['text'] == \"This looks like a clue!\"].groupby(['session_id'])['index'].agg('min').reset_index()\n#     text_2_index.rename(columns={'index':'text_2_index'}, inplace=True)\n#     text_3_time = train.loc[train['text'] == \"Youmans was a suffragist!!\"].groupby(['session_id'])['elapsed_time'].agg('min').reset_index()\n#     text_2_time.rename(columns={'elapsed_time':'text_3_time'}, inplace=True)\n#     text_3_index = train.loc[train['text'] == \"Youmans was a suffragist!!\"].groupby(['session_id'])['index'].agg('min').reset_index()\n#     text_3_index.rename(columns={'index':'text_3_index'}, inplace=True)\n\n    train.loc[((train['fqid'].isin(FQIDS_0_4)) & (train['event_name'] == 'object_click') \n                   & (train['name'] != 'close')), 'correct_clicks'] = train.loc[((train['fqid'].isin(FQIDS_0_4)) & (train['event_name'] == 'object_click')\n                                                                    & (train['name'] != 'close'))].apply(lambda row: right_click_object(row['room_coor_x'], \n                                                                                                                        row['room_coor_y'], \n                                                                                                                        row['fqid'], \n                                                                                                                        RIGHT_FQID_DICT), 1)\n\n    session_ids = train['session_id'].unique()\n    \n    session_ids_df = pd.DataFrame(session_ids, columns=['session_id'])\n#     text_1_time = pd.merge(session_ids_df, text_1_time, on=['session_id'], how='left').drop(columns=['session_id'])\n#     text_1_index = pd.merge(session_ids_df, text_1_index, on=['session_id'], how='left').drop(columns=['session_id'])\n#     text_2_time = pd.merge(session_ids_df, text_2_time, on=['session_id'], how='left').drop(columns=['session_id'])\n#     text_2_index = pd.merge(session_ids_df, text_2_index, on=['session_id'], how='left').drop(columns=['session_id'])\n#     text_3_time = pd.merge(session_ids_df, text_3_time, on=['session_id'], how='left').drop(columns=['session_id'])\n#     text_3_index = pd.merge(session_ids_df, text_3_index, on=['session_id'], how='left').drop(columns=['session_id'])\n    \n    correct_clicks = [0, 1]\n    combinations = list(itertools.product(session_ids, FQIDS_0_4, correct_clicks))\n\n    # Create a new dataframe with all possible combinations\n    new_df = pd.DataFrame(combinations, columns=['session_id', 'fqid', 'correct_clicks'])\n    \n    merged_df = pd.merge(new_df, train.groupby(['session_id', 'fqid', 'correct_clicks'])['count'].agg('sum').reset_index(),\n                         on=['session_id', 'fqid', 'correct_clicks'], \n                         how='left')\n    \n    merged_df['count'] = merged_df['count'].replace(np.nan, 0)\n    \n    table = pd.pivot_table(merged_df, values='count', index='session_id', columns=['fqid', 'correct_clicks'], fill_value=0)\n\n    # Flatten the column MultiIndex\n    #table.columns = ['_'.join(map(str, col)).strip() for col in table.columns.values]\n\n    # Reset the index to turn the session_id into a column\n    table = table.reset_index(drop=True)\n    \n    df = generic_features(train)\n    \n    df = pd.concat(\n        [\n            df,table, time_plaque, max_time, mim_time\n        ]\n        ,axis=1\n    )\n        \n    if submit:\n        df = df.reindex(columns=columns_recreate, fill_value=0)\n \n    return df\n\ndef feature_engineer_5_12(train, columns_recreate=[], submit=False):\n    \n    train.sort_values(by=['session_id', 'elapsed_time'], inplace=True)\n    \n    max_time = train.groupby(['session_id'])['elapsed_time'].agg('max')#.reset_index(drop=True).drop(columns=['session_id'])\n    max_time.name = 'maximum time'\n    max_time = max_time.reset_index(drop=True).drop(columns=['session_id'])\n    \n    mim_time = train.groupby(['session_id'])['elapsed_time'].agg('min')#.reset_index(drop=True).drop(columns=['session_id'])\n    mim_time.name = 'minimum time'\n    mim_time = mim_time.reset_index(drop=True).drop(columns=['session_id'])\n    \n    time_bingo = train.loc[((train['event_name'] == 'navigate_click') & (train['fqid'] == 'businesscards') |\n              (train['event_name'] == 'notification_click') & (train['text_fqid'] == 'tunic.humanecology.frontdesk.businesscards.card_bingo.bingo'))].reset_index()\n    \n    time_bingo['duration'] = time_bingo['elapsed_time'].diff(1).clip(0, 103000).replace(np.nan, 0)\n    \n    \n    time_bingo = time_bingo.groupby(['session_id'])['duration'].agg('max').reset_index(drop=True).drop(columns=['session_id'])\n    \n    time_loog_b = train.loc[((train['event_name'] == 'navigate_click') & (train['fqid'] == 'logbook') |\n              (train['text'] == \"It's a match!\") & (train['text_fqid'] == 'tunic.drycleaner.frontdesk.logbook.page.bingo'))].reset_index()\n    \n    time_loog_b['duration_2'] = time_loog_b['elapsed_time'].diff(1).clip(0, 103000).replace(np.nan, 0)\n    \n    \n    time_loog_b = time_loog_b.groupby(['session_id'])['duration_2'].agg('max').reset_index(drop=True).drop(columns=['session_id'])\n    \n    time_crofiche = train.loc[((train['event_name'] == 'navigate_click') & (train['fqid'] == 'tomicrofiche') |\n              (train['text'] == \"Youmans was a suffragist!\") & (train['text_fqid'] == 'tunic.library.microfiche.reader.paper2.bingo'))].reset_index()\n    \n    time_crofiche['duration_3'] = time_crofiche['elapsed_time'].diff(1).clip(0, 103000).replace(np.nan, 0)\n    \n    \n    time_crofiche = time_crofiche.groupby(['session_id'])['duration_3'].agg('max').reset_index(drop=True).drop(columns=['session_id'])\n    \n    #print(time_bingo)\n    \n    train['count'] = 1\n    \n#     text_1 = train.loc[train['text'] == \"Where did you get that coffee?\"].groupby(['session_id'])['elapsed_time'].agg('min').reset_index()\n#     text_1.rename(columns={'elapsed_time':'text_1_time'}, inplace=True)\n#     text_2 = train.loc[train['text'] == \"It's a match!\"].groupby(['session_id'])['elapsed_time'].agg('min').reset_index()\n#     text_2.rename(columns={'elapsed_time':'text_2_time'}, inplace=True)\n    \n    \n#     click_loog_book = train.loc[((train['text'] == \"Here's the log book.\")\n#                       | (train['text'] == \"It's a match!\"))].groupby(['session_id'])['index'].diff(1).replace(np.nan, 0).reset_index().drop(columns=['session_id'])\n\n    train.loc[((train['fqid'].isin(FQIDS_5_12)) & (train['event_name'] == 'object_click') \n                   & (train['name'] != 'close')), 'correct_clicks'] = train.loc[((train['fqid'].isin(FQIDS_5_12)) & (train['event_name'] == 'object_click')\n                                                                    & (train['name'] != 'close'))].apply(lambda row: right_click_object(row['room_coor_x'], \n                                                                                                                        row['room_coor_y'], \n                                                                                                                        row['fqid'], \n                                                                                                                        RIGHT_FQID_DICT), 1)\n    session_ids = train['session_id'].unique()\n    \n    session_ids_df = pd.DataFrame(session_ids, columns=['session_id'])\n#     text_1 = pd.merge(session_ids_df, text_1, on=['session_id'], how='left').drop(columns=['session_id'])\n#     text_2 = pd.merge(session_ids_df, text_2, on=['session_id'], how='left').drop(columns=['session_id'])\n    correct_clicks = [0, 1]\n    combinations = list(itertools.product(session_ids, FQIDS_5_12, correct_clicks))\n\n    # Create a new dataframe with all possible combinations\n    new_df = pd.DataFrame(combinations, columns=['session_id', 'fqid', 'correct_clicks'])\n    \n    merged_df = pd.merge(new_df, train.groupby(['session_id', 'fqid', 'correct_clicks'])['count'].agg('sum').reset_index(), \n                         on=['session_id', 'fqid', 'correct_clicks'], how='left')\n    \n    merged_df['count'] = merged_df['count'].replace(np.nan, 0)\n    \n    table = pd.pivot_table(merged_df, values='count', index='session_id', columns=['fqid', 'correct_clicks'], fill_value=0)\n\n    # Flatten the column MultiIndex\n    #table.columns = ['_'.join(map(str, col)).strip() for col in table.columns.values]\n\n    # Reset the index to turn the session_id into a column\n    table = table.reset_index(drop=True)\n    \n    df = generic_features(train)\n    \n    df = pd.concat(\n        [\n            df,table, time_bingo, time_loog_b, max_time, mim_time\n        ]\n        ,axis=1\n    )\n\n    if submit:\n        df = df.reindex(columns=columns_recreate, fill_value=0)\n    \n    return df\n\ndef feature_engineer_13_22(train, columns_recreate=[], submit=False):\n    \n    train.sort_values(by=['session_id', 'elapsed_time'], inplace=True)\n    \n    max_time = train.groupby(['session_id'])['elapsed_time'].agg('max')#.reset_index(drop=True).drop(columns=['session_id'])\n    max_time.name = 'maximum time'\n    max_time = max_time.reset_index(drop=True).drop(columns=['session_id'])\n    \n    mim_time = train.groupby(['session_id'])['elapsed_time'].agg('min')#.reset_index(drop=True).drop(columns=['session_id'])\n    mim_time.name = 'minimum time'\n    mim_time = mim_time.reset_index(drop=True).drop(columns=['session_id'])\n    \n    time_hoofprint = train.loc[((train['event_name'] == 'person_click') & (train['text'] == \"There's a diagram of animal tracks over there.\") |\n              (train['event_name'] == 'notification_click') & (train['text_fqid'] == 'tunic.wildlife.center.tracks.hub.deer'))].reset_index()\n    \n    time_hoofprint['duration'] = time_hoofprint['elapsed_time'].diff(1).clip(0, 103000).replace(np.nan, 0)\n    \n    \n    time_hoofprint = time_hoofprint.groupby(['session_id'])['duration'].agg('max').reset_index(drop=True).drop(columns=['session_id'])\n    \n    time_national_moviment = train.loc[((train['event_name'] == 'person_click') & (train['text'] == \"Your flag must have been part of a national movement!\") |\n              (train['event_name'] == 'notification_click') & (train['text_fqid'] == 'tunic.library.microfiche.reader_flag.paper2.bingo'))].reset_index()\n    \n    time_national_moviment['duration_2'] = time_national_moviment['elapsed_time'].diff(1).clip(0, 103000).replace(np.nan, 0)\n    \n    \n    time_national_moviment = time_national_moviment.groupby(['session_id'])['duration_2'].agg('max').reset_index(drop=True).drop(columns=['session_id'])\n    \n    \n    \n    train['count'] = 1\n    \n    train.loc[((train['fqid'].isin(FQIDS_13_22)) & (train['event_name'] == 'object_click') \n                   & (train['name'] != 'close')), 'correct_clicks'] = train.loc[((train['fqid'].isin(FQIDS_13_22)) & (train['event_name'] == 'object_click')\n                                                                    & (train['name'] != 'close'))].apply(lambda row: right_click_object(row['room_coor_x'], \n                                                                                                                        row['room_coor_y'], \n                                                                                                                        row['fqid'], \n                                                                                                                        RIGHT_FQID_DICT), 1)\n    session_ids = train['session_id'].unique()\n    correct_clicks = [0, 1]\n    combinations = list(itertools.product(session_ids, FQIDS_13_22, correct_clicks))\n\n    new_df = pd.DataFrame(combinations, columns=['session_id', 'fqid', 'correct_clicks'])\n    \n    merged_df = pd.merge(new_df, train.groupby(['session_id','fqid', 'correct_clicks'])['count'].agg('sum').reset_index(), \n                         on=['session_id', 'fqid', 'correct_clicks'], how='left')\n    \n    merged_df = merged_df.groupby(['session_id', 'fqid', 'correct_clicks'])['count'].agg('sum').reset_index()\n    table = pd.pivot_table(merged_df, values='count', index='session_id', columns=['fqid', 'correct_clicks'], fill_value=0)\n\n    table = table.reset_index(drop=True)\n    \n    df = generic_features(train)\n    \n    df = pd.concat(\n        [\n            df,table, max_time, mim_time#, time_hoofprint, time_national_moviment\n        ]\n        ,axis=1\n    )\n    \n    if submit:\n        df = df.reindex(columns=columns_recreate, fill_value=0)\n        \n    return df\n    ","metadata":{"execution":{"iopub.status.busy":"2023-07-05T07:00:20.647352Z","iopub.execute_input":"2023-07-05T07:00:20.647772Z","iopub.status.idle":"2023-07-05T07:00:20.721987Z","shell.execute_reply.started":"2023-07-05T07:00:20.647733Z","shell.execute_reply":"2023-07-05T07:00:20.720246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# PROCESS TRAIN DATA IN PIECES\nall_pieces_0_4 = []\nall_pieces_5_12 = []\nall_pieces_13_22 = []\nprint(f'Processing train as {PIECES} pieces to avoid memory error... ')\nfor k in range(PIECES):\n    print(k,', ',end='')\n    SKIPS = 0\n    if k>0: SKIPS = range(1,skips[k]+1)\n    train = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv',\n                        nrows=reads[k], skiprows=SKIPS).drop(columns=COLUMNS_DROP)\n    \n    df_0_4 = feature_engineer_0_4(train.loc[train['level_group'] == '0-4'].reset_index(drop=True))\n    df_5_12 = feature_engineer_5_12(train.loc[train['level_group'] == '5-12'].reset_index(drop=True))\n    df_13_22 = feature_engineer_13_22(train.loc[train['level_group'] == '13-22'].reset_index(drop=True))\n    all_pieces_0_4.append(df_0_4)\n    all_pieces_5_12.append(df_5_12)\n    all_pieces_13_22.append(df_13_22)\n    \n# CONCATENATE ALL PIECES\nprint('\\n')\ndel train; gc.collect()\ndf_0_4 = pd.concat(all_pieces_0_4, axis=0).reset_index(drop=True)\ndf_5_12 = pd.concat(all_pieces_5_12, axis=0).reset_index(drop=True)\ndf_13_22 = pd.concat(all_pieces_13_22, axis=0).reset_index(drop=True)\nprint('Shape of all train data 0-4 after feature engineering:', df_0_4.shape )\nprint('Shape of all train data 5-12 after feature engineering:', df_5_12.shape )\nprint('Shape of all train data 13-22 after feature engineering:', df_13_22.shape )","metadata":{"papermill":{"duration":238.357436,"end_time":"2023-05-26T23:02:01.75252","exception":false,"start_time":"2023-05-26T22:58:03.395084","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-05T07:00:28.834230Z","iopub.execute_input":"2023-07-05T07:00:28.834675Z","iopub.status.idle":"2023-07-05T07:09:43.421816Z","shell.execute_reply.started":"2023-07-05T07:00:28.834639Z","shell.execute_reply":"2023-07-05T07:09:43.420598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_0_4_columns = list(df_0_4)\ndf_5_12_columns = list(df_5_12)\ndf_13_22_columns = list(df_13_22)","metadata":{"papermill":{"duration":0.016818,"end_time":"2023-05-26T23:02:01.777293","exception":false,"start_time":"2023-05-26T23:02:01.760475","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-05T07:13:22.410883Z","iopub.execute_input":"2023-07-05T07:13:22.411393Z","iopub.status.idle":"2023-07-05T07:13:22.417227Z","shell.execute_reply.started":"2023-07-05T07:13:22.411338Z","shell.execute_reply":"2023-07-05T07:13:22.416386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train XGBoost Model\nWe train one model for each of 18 questions. Furthermore, we use data from `level_groups = '0-4'` to train model for questions 1-3, and `level groups '5-12'` to train questions 4 thru 13 and `level groups '13-22'` to train questions 14 thru 18. Because this is the data we get (to predict corresponding questions) from Kaggle's inference API during test inference. We can improve our model by saving a user's previous data from earlier `level_groups` and using that to predict future `level_groups`.","metadata":{"papermill":{"duration":0.007656,"end_time":"2023-05-26T23:02:01.792992","exception":false,"start_time":"2023-05-26T23:02:01.785336","status":"completed"},"tags":[]}},{"cell_type":"code","source":"mulilabe_targets","metadata":{"execution":{"iopub.status.busy":"2023-07-05T07:13:25.496909Z","iopub.execute_input":"2023-07-05T07:13:25.497334Z","iopub.status.idle":"2023-07-05T07:13:25.521222Z","shell.execute_reply.started":"2023-07-05T07:13:25.497284Z","shell.execute_reply":"2023-07-05T07:13:25.520375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_TIME_QUESTIONS:\n    df_13_22['time_questions'] = df_13_22['minimum time'] - df_5_12['maximum time']\n    df_5_12['time_questions'] = df_5_12['minimum time'] - df_0_4['maximum time']\n    df_13_22['time_questions_5_12'] = df_5_12['time_questions']\n    \n    df_13_22.drop(columns=['minimum time', 'maximum time'], inplace=True)\n    df_5_12.drop(columns=['minimum time', 'maximum time'], inplace=True)\n    df_0_4.drop(columns=['maximum time', 'minimum time'], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T07:13:29.615135Z","iopub.execute_input":"2023-07-05T07:13:29.615588Z","iopub.status.idle":"2023-07-05T07:13:29.690032Z","shell.execute_reply.started":"2023-07-05T07:13:29.615548Z","shell.execute_reply":"2023-07-05T07:13:29.689005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gkf = GroupKFold(n_splits=5)\noof = pd.DataFrame(data=np.zeros((len(mulilabe_targets),18)), index=mulilabe_targets['session'].values.reshape(-1))\nmodels = {}\n\ntresholds = []\n\n# COMPUTE CV SCORE WITH 5 GROUP K FOLD\nfor i, (train_index, test_index) in enumerate(mskf.split(X=mulilabe_targets['session'].values, y=mulilabe_targets.iloc[:, 2:])):\n#for i, (train_index, test_index) in enumerate(gkf.split(X=mulilabe_targets['session'].values, groups=df_0_4.index)):\n    print('#'*25)\n    print('### Fold',i+1)\n    print('#'*25)\n    \n    xgb_params = {\n    'objective' : 'binary:logistic',\n    'eval_metric':'logloss',\n    'learning_rate': 0.05,\n    'max_depth': 4,\n    'n_estimators': 1000,\n    'early_stopping_rounds': 250,\n    'tree_method':'hist',\n    'subsample':0.8,\n    'colsample_bytree': 0.4,\n    'use_label_encoder' : False}\n    \n    # ITERATE THRU QUESTIONS 1 THRU 18\n    folds_tresholds = []\n    for t in range(1,19):\n        \n        # USE THIS TRAIN DATA WITH THESE QUESTIONS\n        if t<=3: \n            train_x = df_0_4.iloc[train_index]\n            valid_x = df_0_4.iloc[test_index]\n        elif t<=13: \n            train_x = df_5_12.iloc[train_index]\n            valid_x = df_5_12.iloc[test_index]\n        elif t<=22: \n            train_x = df_13_22.iloc[train_index]\n            valid_x = df_13_22.iloc[test_index]\n            \n        # TRAIN DATA\n        train_users = train_x['session_id'].values\n        train_y = targets.loc[targets.q==t].set_index('session').loc[train_users]\n        \n        # VALID DATA\n        valid_users = valid_x['session_id'].values\n        valid_y = targets.loc[targets.q==t].set_index('session').loc[valid_users]\n        \n        # TRAIN MODEL        \n        clf =  XGBClassifier(**xgb_params)\n        clf.fit(train_x.iloc[:, 1:].astype('float32'), train_y['correct'],\n                eval_set=[ (valid_x.iloc[:, 1:].astype('float32'), valid_y['correct']) ],\n                verbose=0)\n        #clf = LogisticRegression(random_state=0)\n        #clf.fit(train_x.astype('float32'), train_y['correct'])\n        \n        prob = clf.predict_proba(valid_x.iloc[:, 1:].astype('float32'))[:,1]\n        \n        scores = []             \n        thresholds = []\n        best_score = 0; best_threshold = 0\n\n        for threshold in np.arange(0.4,0.81,0.01):\n    #         print(f'{threshold:.02f}, ',end='')\n            preds = (prob>threshold).astype('int')\n            m = f1_score(valid_y['correct'], preds, average='macro')   \n            scores.append(m)\n            thresholds.append(threshold)\n            if m>best_score:\n                best_score = m\n                best_threshold = threshold\n        \n        folds_tresholds.append(best_threshold)\n        \n        print(f'{t}({clf.best_ntree_limit})({best_score:.2f}/{best_threshold:.2f}), ',end='')\n        #print(f'{t}(({best_score:.2f}/{best_threshold:.2f}), ',end='')\n        \n        # SAVE MODEL, PREDICT VALID OOF\n        models[f'{t}'] = clf\n        oof.loc[valid_users, t-1] = clf.predict_proba(valid_x.iloc[:, 1:].astype('float32'))[:,1]\n        \n    tresholds.append(folds_tresholds)\n        \n    print()","metadata":{"papermill":{"duration":460.922053,"end_time":"2023-05-26T23:09:42.722843","exception":false,"start_time":"2023-05-26T23:02:01.80079","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-05T07:13:44.125512Z","iopub.execute_input":"2023-07-05T07:13:44.126181Z","iopub.status.idle":"2023-07-05T07:34:35.065426Z","shell.execute_reply.started":"2023-07-05T07:13:44.126142Z","shell.execute_reply":"2023-07-05T07:34:35.064378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof","metadata":{"execution":{"iopub.status.busy":"2023-07-05T07:34:35.119051Z","iopub.execute_input":"2023-07-05T07:34:35.119756Z","iopub.status.idle":"2023-07-05T07:34:35.154224Z","shell.execute_reply.started":"2023-07-05T07:34:35.119718Z","shell.execute_reply":"2023-07-05T07:34:35.152936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-05T07:34:35.675605Z","iopub.execute_input":"2023-07-05T07:34:35.676011Z","iopub.status.idle":"2023-07-05T07:34:35.702146Z","shell.execute_reply.started":"2023-07-05T07:34:35.675978Z","shell.execute_reply":"2023-07-05T07:34:35.700866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Compute CV Score\nWe need to convert prediction probabilities into `1s` and `0s`. The competition metric is F1 Score which is the harmonic mean of precision and recall. Let's find the optimal threshold for `p > threshold` when to predict `1` and when to predict `0` to maximize F1 Score.","metadata":{"papermill":{"duration":0.012378,"end_time":"2023-05-26T23:09:42.748051","exception":false,"start_time":"2023-05-26T23:09:42.735673","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# PUT TRUE LABELS INTO DATAFRAME WITH 18 COLUMNS\ntrue = oof.copy()\nfor k in range(18):\n    # GET TRUE LABELS\n    tmp = targets.loc[targets.q == k+1].set_index('session').loc[df_0_4['session_id'].values.reshape(-1)]\n    true[k] = tmp.correct.values","metadata":{"papermill":{"duration":0.111224,"end_time":"2023-05-26T23:09:42.871755","exception":false,"start_time":"2023-05-26T23:09:42.760531","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-05T07:36:43.729023Z","iopub.execute_input":"2023-07-05T07:36:43.729539Z","iopub.status.idle":"2023-07-05T07:36:43.863821Z","shell.execute_reply.started":"2023-07-05T07:36:43.729501Z","shell.execute_reply":"2023-07-05T07:36:43.862690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"true","metadata":{"execution":{"iopub.status.busy":"2023-07-05T07:36:50.637919Z","iopub.execute_input":"2023-07-05T07:36:50.638436Z","iopub.status.idle":"2023-07-05T07:36:50.664812Z","shell.execute_reply.started":"2023-07-05T07:36:50.638396Z","shell.execute_reply":"2023-07-05T07:36:50.663401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"true.to_csv('label.csv')","metadata":{"execution":{"iopub.status.busy":"2023-07-05T08:16:42.399174Z","iopub.execute_input":"2023-07-05T08:16:42.399659Z","iopub.status.idle":"2023-07-05T08:16:42.560863Z","shell.execute_reply.started":"2023-07-05T08:16:42.399617Z","shell.execute_reply":"2023-07-05T08:16:42.559697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# FIND BEST THRESHOLD TO CONVERT PROBS INTO 1s AND 0s\nscores = []; thresholds = []\nbest_score = 0; best_threshold = 0\n\nfor threshold in np.arange(0.4,0.81,0.01):\n    print(f'{threshold:.02f}, ',end='')\n    preds = (oof.values.reshape((-1))>threshold).astype('int')\n    m = f1_score(true.values.reshape((-1)), preds, average='macro')   \n    scores.append(m)\n    thresholds.append(threshold)\n    if m>best_score:\n        best_score = m\n        best_threshold = threshold","metadata":{"papermill":{"duration":4.436458,"end_time":"2023-05-26T23:09:47.321542","exception":false,"start_time":"2023-05-26T23:09:42.885084","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-05T06:49:13.026164Z","iopub.execute_input":"2023-07-05T06:49:13.026528Z","iopub.status.idle":"2023-07-05T06:49:21.272679Z","shell.execute_reply.started":"2023-07-05T06:49:13.026497Z","shell.execute_reply":"2023-07-05T06:49:21.271485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# PLOT THRESHOLD VS. F1_SCORE\nplt.figure(figsize=(20,5))\nplt.plot(thresholds,scores,'-o',color='blue')\nplt.scatter([best_threshold], [best_score], color='blue', s=300, alpha=1)\nplt.xlabel('Threshold',size=14)\nplt.ylabel('Validation F1 Score',size=14)\nplt.title(f'Threshold vs. F1_Score with Best F1_Score = {best_score:.3f} at Best Threshold = {best_threshold:.3}',size=18)\nplt.show()","metadata":{"papermill":{"duration":0.233147,"end_time":"2023-05-26T23:09:47.568358","exception":false,"start_time":"2023-05-26T23:09:47.335211","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-05T06:49:21.274353Z","iopub.execute_input":"2023-07-05T06:49:21.274689Z","iopub.status.idle":"2023-07-05T06:49:21.586283Z","shell.execute_reply.started":"2023-07-05T06:49:21.274657Z","shell.execute_reply":"2023-07-05T06:49:21.585387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('When using optimal threshold...')\nfor k in range(18):\n        \n    # COMPUTE F1 SCORE PER QUESTION\n    m = f1_score(true[k].values, (oof[k].values>best_threshold).astype('int'), average='macro')\n    print(f'Q{k + 1}: F1 =',m)\n    \n# COMPUTE F1 SCORE OVERALL\nm = f1_score(true.values.reshape((-1)), (oof.values.reshape((-1))>best_threshold).astype('int'), average='macro')\nprint('==> Overall F1 =',m)","metadata":{"papermill":{"duration":0.244531,"end_time":"2023-05-26T23:09:47.827535","exception":false,"start_time":"2023-05-26T23:09:47.583004","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-05T06:49:21.587749Z","iopub.execute_input":"2023-07-05T06:49:21.588336Z","iopub.status.idle":"2023-07-05T06:49:21.985428Z","shell.execute_reply.started":"2023-07-05T06:49:21.588291Z","shell.execute_reply":"2023-07-05T06:49:21.984401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Infer Test Data","metadata":{"papermill":{"duration":0.014381,"end_time":"2023-05-26T23:09:47.856394","exception":false,"start_time":"2023-05-26T23:09:47.842013","status":"completed"},"tags":[]}},{"cell_type":"code","source":"aaa = true.values.reshape((-1))","metadata":{"execution":{"iopub.status.busy":"2023-07-05T07:38:36.022671Z","iopub.execute_input":"2023-07-05T07:38:36.023088Z","iopub.status.idle":"2023-07-05T07:38:36.028093Z","shell.execute_reply.started":"2023-07-05T07:38:36.023054Z","shell.execute_reply":"2023-07-05T07:38:36.027169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof","metadata":{"execution":{"iopub.status.busy":"2023-07-05T08:15:43.164108Z","iopub.execute_input":"2023-07-05T08:15:43.164655Z","iopub.status.idle":"2023-07-05T08:15:43.198638Z","shell.execute_reply.started":"2023-07-05T08:15:43.164617Z","shell.execute_reply":"2023-07-05T08:15:43.197476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof.to_csv('output.csv')","metadata":{"execution":{"iopub.status.busy":"2023-07-05T08:17:07.035426Z","iopub.execute_input":"2023-07-05T08:17:07.035835Z","iopub.status.idle":"2023-07-05T08:17:07.944955Z","shell.execute_reply.started":"2023-07-05T08:17:07.035802Z","shell.execute_reply":"2023-07-05T08:17:07.943667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bbb = (oof.values.reshape((-1))>best_threshold).astype('int')","metadata":{"execution":{"iopub.status.busy":"2023-07-05T07:38:43.719592Z","iopub.execute_input":"2023-07-05T07:38:43.720026Z","iopub.status.idle":"2023-07-05T07:38:43.726774Z","shell.execute_reply.started":"2023-07-05T07:38:43.719991Z","shell.execute_reply":"2023-07-05T07:38:43.725651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aaa.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-05T07:39:06.566136Z","iopub.execute_input":"2023-07-05T07:39:06.566594Z","iopub.status.idle":"2023-07-05T07:39:06.575171Z","shell.execute_reply.started":"2023-07-05T07:39:06.566554Z","shell.execute_reply":"2023-07-05T07:39:06.574372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bbb.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-05T07:39:14.295347Z","iopub.execute_input":"2023-07-05T07:39:14.296494Z","iopub.status.idle":"2023-07-05T07:39:14.302786Z","shell.execute_reply.started":"2023-07-05T07:39:14.296451Z","shell.execute_reply":"2023-07-05T07:39:14.301848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score \naccuracy_score(aaa,bbb)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T07:44:45.641965Z","iopub.execute_input":"2023-07-05T07:44:45.642501Z","iopub.status.idle":"2023-07-05T07:44:45.696416Z","shell.execute_reply.started":"2023-07-05T07:44:45.642459Z","shell.execute_reply":"2023-07-05T07:44:45.695172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1_score(aaa,bbb, average='macro')","metadata":{"execution":{"iopub.status.busy":"2023-07-05T07:45:50.777565Z","iopub.execute_input":"2023-07-05T07:45:50.778347Z","iopub.status.idle":"2023-07-05T07:45:50.982481Z","shell.execute_reply.started":"2023-07-05T07:45:50.778296Z","shell.execute_reply":"2023-07-05T07:45:50.981558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\nfrom sklearn.metrics import precision_score, recall_score\nfrom sklearn.metrics import f1_score\nprint('*'*50)\nprint(accuracy_score(aaa,bbb))\nprint('*'*50)\n\nprecision = precision_score(aaa, bbb,average='micro')\nrecall =  recall_score(aaa,bbb)\nf1 = f1_score(aaa,bbb,average='macro')\nprint(precision)\nprint(recall)\nprint(f1)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-05T08:01:55.155603Z","iopub.execute_input":"2023-07-05T08:01:55.156771Z","iopub.status.idle":"2023-07-05T08:01:55.822849Z","shell.execute_reply.started":"2023-07-05T08:01:55.156729Z","shell.execute_reply":"2023-07-05T08:01:55.821714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#jo_wilder.make_env.__called__ = False","metadata":{"papermill":{"duration":0.022542,"end_time":"2023-05-26T23:09:47.893402","exception":false,"start_time":"2023-05-26T23:09:47.87086","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-05T06:49:21.991697Z","iopub.execute_input":"2023-07-05T06:49:21.992464Z","iopub.status.idle":"2023-07-05T06:49:21.996427Z","shell.execute_reply.started":"2023-07-05T06:49:21.992424Z","shell.execute_reply":"2023-07-05T06:49:21.995585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# IMPORT KAGGLE API\nimport jo_wilder\nenv = jo_wilder.make_env()\n#type(env)._state = type(type(env)._state).__dict__['INIT']\niter_test = env.iter_test()\n\n#CLEAR MEMORY\nimport gc\ndel targets, df_0_4, df_5_12, df_13_22, oof, true, all_pieces_0_4, all_pieces_5_12, all_pieces_13_22\n_ = gc.collect()","metadata":{"papermill":{"duration":0.170792,"end_time":"2023-05-26T23:09:48.078828","exception":false,"start_time":"2023-05-26T23:09:47.908036","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-05T06:49:21.998053Z","iopub.execute_input":"2023-07-05T06:49:21.998715Z","iopub.status.idle":"2023-07-05T06:49:22.157517Z","shell.execute_reply.started":"2023-07-05T06:49:21.998680Z","shell.execute_reply":"2023-07-05T06:49:22.156522Z"}}},{"cell_type":"markdown","source":"limits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\ndict_data = {\n    '0-4': {},\n    '5-12': {},\n    '13-22': {}\n}\n\nfor (test, sample_submission) in iter_test:\n    \n    # FEATURE ENGINEER TEST DATA\n    # INFER TEST DATA\n    grp = test.level_group.values[0]\n    a,b = limits[grp]\n    \n    if grp == '0-4':\n        df = feature_engineer_0_4(test.drop(columns=COLUMNS_DROP), df_0_4_columns, True)\n    elif grp == '5-12':\n        df = feature_engineer_5_12(test.drop(columns=COLUMNS_DROP), df_5_12_columns, True)\n    elif grp == '13-22':\n        df = feature_engineer_13_22(test.drop(columns=COLUMNS_DROP), df_13_22_columns, True)\n        \n    if USE_TIME_QUESTIONS:\n        if grp == '0-4':\n            dict_data[grp]['level_data'] = df['maximum time'].values\n            \n            df.drop(columns=[ 'maximum time', 'minimum time'], inplace=True)\n        elif grp == '5-12':\n            dict_data[grp]['level_data'] = df['maximum time'].values\n\n            df['time_questions'] = df['minimum time'] - dict_data['0-4']['level_data']\n            dict_data[grp]['time_question_data'] = df['time_questions'].values\n            \n            df.drop(columns=['minimum time', 'maximum time'], inplace=True)\n        else:\n            df['time_questions'] = df['minimum time'] - dict_data['5-12']['level_data']\n            df['time_questions_5_12'] = dict_data['5-12']['time_question_data']\n            \n            df.drop(columns=['minimum time', 'maximum time'], inplace=True)\n    \n\n    for t in range(a,b):\n        clf = models[f'{t}']\n        p = clf.predict_proba(df.iloc[:, 1:].values.astype('float32'))[0,1]\n        mask = sample_submission.session_id.str.contains(f'q{t}')\n        sample_submission.loc[mask,'correct'] = int( p > best_threshold )\n\n    env.predict(sample_submission)","metadata":{"papermill":{"duration":1.766316,"end_time":"2023-05-26T23:09:49.859536","exception":false,"start_time":"2023-05-26T23:09:48.09322","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-05T06:49:22.159459Z","iopub.execute_input":"2023-07-05T06:49:22.160160Z","iopub.status.idle":"2023-07-05T06:49:24.714078Z","shell.execute_reply.started":"2023-07-05T06:49:22.160123Z","shell.execute_reply":"2023-07-05T06:49:24.713089Z"}}},{"cell_type":"code","source":"#sdasdasd","metadata":{"papermill":{"duration":0.022954,"end_time":"2023-05-26T23:09:49.898529","exception":false,"start_time":"2023-05-26T23:09:49.875575","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-05T06:49:24.715286Z","iopub.execute_input":"2023-07-05T06:49:24.715810Z","iopub.status.idle":"2023-07-05T06:49:24.722470Z","shell.execute_reply.started":"2023-07-05T06:49:24.715778Z","shell.execute_reply":"2023-07-05T06:49:24.721199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA submission.csv","metadata":{"papermill":{"duration":0.01405,"end_time":"2023-05-26T23:09:49.927254","exception":false,"start_time":"2023-05-26T23:09:49.913204","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"df = pd.read_csv('submission.csv')\nprint( df.shape )\ndf.head()","metadata":{"papermill":{"duration":0.029008,"end_time":"2023-05-26T23:09:49.970425","exception":false,"start_time":"2023-05-26T23:09:49.941417","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-05T06:49:24.723903Z","iopub.execute_input":"2023-07-05T06:49:24.724377Z","iopub.status.idle":"2023-07-05T06:49:24.741853Z","shell.execute_reply.started":"2023-07-05T06:49:24.724344Z","shell.execute_reply":"2023-07-05T06:49:24.740655Z"}}},{"cell_type":"markdown","source":"print(df.correct.mean())","metadata":{"papermill":{"duration":0.023178,"end_time":"2023-05-26T23:09:50.008374","exception":false,"start_time":"2023-05-26T23:09:49.985196","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-05T06:49:24.743591Z","iopub.execute_input":"2023-07-05T06:49:24.743931Z","iopub.status.idle":"2023-07-05T06:49:24.750111Z","shell.execute_reply.started":"2023-07-05T06:49:24.743899Z","shell.execute_reply":"2023-07-05T06:49:24.748882Z"}}}]}