{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport polars as pl\n\nimport matplotlib.pyplot as plt\n\nimport gc\nimport pickle","metadata":{"execution":{"iopub.status.busy":"2023-05-25T15:46:06.969895Z","iopub.execute_input":"2023-05-25T15:46:06.970347Z","iopub.status.idle":"2023-05-25T15:46:07.215745Z","shell.execute_reply.started":"2023-05-25T15:46:06.970305Z","shell.execute_reply":"2023-05-25T15:46:07.214613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Constants\n\nDATA_PATH = '/kaggle/input/predict-student-performance-from-game-play'\nMODEL_PATH = '/kaggle/input/xgboost-model'\n# DATA_PATH = './'","metadata":{"execution":{"iopub.status.busy":"2023-05-25T15:46:07.218128Z","iopub.execute_input":"2023-05-25T15:46:07.218583Z","iopub.status.idle":"2023-05-25T15:46:07.223187Z","shell.execute_reply.started":"2023-05-25T15:46:07.218533Z","shell.execute_reply":"2023-05-25T15:46:07.222330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# copy from catboost mix\n\nCATS = ['event_name', 'name', 'fqid', 'room_fqid', 'text_fqid']\nNUMS = ['page', 'room_coor_x', 'room_coor_y', 'screen_coor_x', 'screen_coor_y',\n        'hover_duration', 'elapsed_time_diff']\nDIALOGS = ['that', 'this', 'it', 'you','find','found','Found','notebook','Wells','wells','help','need', 'Oh','Ooh','Jo', 'flag', 'can','and','is','the','to']\n\nname_feature = ['basic', 'undefined', 'close', 'open', 'prev', 'next']\nevent_name_feature = ['cutscene_click', 'person_click', 'navigate_click',\n       'observation_click', 'notification_click', 'object_click',\n       'object_hover', 'map_hover', 'map_click', 'checkpoint',\n       'notebook_click']\n\n\nsub_fqid_lists = {'0-4': ['gramps',\n 'wells',\n 'toentry',\n 'groupconvo',\n 'tomap',\n 'tostacks',\n 'tobasement',\n 'boss',\n 'cs',\n 'teddy',\n 'tunic.historicalsociety',\n 'plaque',\n 'directory',\n 'tunic',\n 'tunic.kohlcenter',\n 'plaque.face.date',\n 'notebook',\n 'tunic.hub.slip',\n 'tocollection',\n 'tunic.capitol_0',\n 'photo',\n 'intro',\n 'retirement_letter',\n 'togrampa',\n 'janitor',\n 'chap1_finale',\n 'report',\n 'outtolunch',\n 'chap1_finale_c',\n 'block_0',\n 'doorblock',\n 'tocloset',\n 'block_tomap2',\n 'block_tocollection',\n 'block_tomap1'],\n                  '5-12': ['worker',\n 'archivist',\n 'gramps',\n 'toentry',\n 'tomap',\n 'tostacks',\n 'tobasement',\n 'boss',\n 'journals',\n 'businesscards',\n 'tunic.historicalsociety',\n 'tofrontdesk',\n 'plaque',\n 'tunic.drycleaner',\n 'tunic.library',\n 'trigger_scarf',\n 'reader',\n 'directory',\n 'tunic.capitol_1',\n 'journals.pic_0.next',\n 'tunic',\n 'what_happened',\n 'tunic.kohlcenter',\n 'tunic.humanecology',\n 'logbook',\n 'businesscards.card_0.next',\n 'journals.hub.topics',\n 'logbook.page.bingo',\n 'journals.pic_1.next',\n 'reader.paper0.next',\n 'trigger_coffee',\n 'wellsbadge',\n 'journals.pic_2.next',\n 'tomicrofiche',\n 'tocloset_dirty',\n 'businesscards.card_bingo.bingo',\n 'businesscards.card_1.next',\n 'tunic.hub.slip',\n 'journals.pic_2.bingo',\n 'tocollection',\n 'chap2_finale_c',\n 'tunic.capitol_0',\n 'photo',\n 'reader.paper1.next',\n 'businesscards.card_bingo.next',\n 'reader.paper2.bingo',\n 'magnify',\n 'janitor',\n 'tohallway',\n 'outtolunch',\n 'reader.paper2.next',\n 'door_block_talk',\n 'block_magnify',\n 'reader.paper0.prev',\n 'block',\n 'block_0',\n 'door_block_clean',\n 'reader.paper2.prev',\n 'reader.paper1.prev',\n 'block_badge',\n 'block_badge_2',\n 'block_1'],\n                  '13-22': ['worker',\n 'gramps',\n 'wells',\n 'toentry',\n 'confrontation',\n 'crane_ranger',\n 'flag_girl',\n 'tomap',\n 'tostacks',\n 'tobasement',\n 'archivist_glasses',\n 'boss',\n 'journals',\n 'seescratches',\n 'groupconvo_flag',\n 'teddy',\n 'expert',\n 'businesscards',\n 'ch3start',\n 'tunic.historicalsociety',\n 'tofrontdesk',\n 'savedteddy',\n 'plaque',\n 'glasses',\n 'tunic.drycleaner',\n 'reader_flag',\n 'tunic.library',\n 'tracks',\n 'tunic.capitol_2',\n 'reader',\n 'directory',\n 'tunic.capitol_1',\n 'journals.pic_0.next',\n 'unlockdoor',\n 'tunic',\n 'tunic.kohlcenter',\n 'tunic.humanecology',\n 'colorbook',\n 'logbook',\n 'businesscards.card_0.next',\n 'journals.hub.topics',\n 'journals.pic_1.next',\n 'journals_flag',\n 'reader.paper0.next',\n 'tracks.hub.deer',\n 'reader_flag.paper0.next',\n 'journals.pic_2.next',\n 'tomicrofiche',\n 'journals_flag.pic_0.bingo',\n 'tocloset_dirty',\n 'businesscards.card_1.next',\n 'tunic.wildlife',\n 'tunic.hub.slip',\n 'tocage',\n 'journals.pic_2.bingo',\n 'tocollectionflag',\n 'tocollection',\n 'chap4_finale_c',\n 'lockeddoor',\n 'journals_flag.hub.topics',\n 'reader_flag.paper2.bingo',\n 'photo',\n 'tunic.flaghouse',\n 'reader.paper1.next',\n 'directory.closeup.archivist',\n 'businesscards.card_bingo.next',\n 'remove_cup',\n 'journals_flag.pic_0.next',\n 'coffee',\n 'key',\n 'reader_flag.paper1.next',\n 'tohallway',\n 'outtolunch',\n 'journals_flag.hub.topics_old',\n 'journals_flag.pic_1.next',\n 'reader.paper2.next',\n 'reader_flag.paper2.next',\n 'journals_flag.pic_1.bingo',\n 'journals_flag.pic_2.next',\n 'journals_flag.pic_2.bingo',\n 'reader.paper0.prev',\n 'reader_flag.paper0.prev',\n 'reader.paper2.prev',\n 'reader.paper1.prev',\n 'reader_flag.paper2.prev',\n 'reader_flag.paper1.prev',\n 'journals_flag.pic_0_old.next',\n 'journals_flag.pic_1_old.next',\n 'block_nelson',\n 'journals_flag.pic_2_old.next',\n 'need_glasses',\n 'fox'],\n                 }\n\nsub_room_lists = {'0-4': ['tunic.historicalsociety.entry',\n 'tunic.historicalsociety.stacks',\n 'tunic.historicalsociety.basement',\n 'tunic.kohlcenter.halloffame',\n 'tunic.historicalsociety.collection',\n 'tunic.historicalsociety.closet',\n 'tunic.capitol_0.hall'],\n                  '5-12': ['tunic.historicalsociety.entry',\n 'tunic.library.frontdesk',\n 'tunic.historicalsociety.frontdesk',\n 'tunic.historicalsociety.stacks',\n 'tunic.historicalsociety.closet_dirty',\n 'tunic.humanecology.frontdesk',\n 'tunic.historicalsociety.basement',\n 'tunic.kohlcenter.halloffame',\n 'tunic.library.microfiche',\n 'tunic.drycleaner.frontdesk',\n 'tunic.historicalsociety.collection',\n 'tunic.capitol_1.hall',\n 'tunic.capitol_0.hall'],\n                  '13-22': ['tunic.historicalsociety.entry',\n 'tunic.wildlife.center',\n 'tunic.historicalsociety.cage',\n 'tunic.library.frontdesk',\n 'tunic.historicalsociety.frontdesk',\n 'tunic.historicalsociety.stacks',\n 'tunic.historicalsociety.closet_dirty',\n 'tunic.humanecology.frontdesk',\n 'tunic.historicalsociety.basement',\n 'tunic.kohlcenter.halloffame',\n 'tunic.library.microfiche',\n 'tunic.drycleaner.frontdesk',\n 'tunic.historicalsociety.collection',\n 'tunic.flaghouse.entry',\n 'tunic.historicalsociety.collection_flag',\n 'tunic.capitol_1.hall',\n 'tunic.capitol_2.hall'],\n                 }\n\n\nsub_text_lists = {'0-4': ['tunic.historicalsociety.entry.groupconvo',\n 'tunic.historicalsociety.collection.cs',\n 'tunic.historicalsociety.collection.gramps.found',\n 'tunic.historicalsociety.closet.gramps.intro_0_cs_0',\n 'tunic.historicalsociety.closet.teddy.intro_0_cs_0',\n 'tunic.historicalsociety.closet.intro',\n 'tunic.historicalsociety.closet.retirement_letter.hub',\n 'tunic.historicalsociety.collection.tunic.slip',\n 'tunic.kohlcenter.halloffame.plaque.face.date',\n 'tunic.kohlcenter.halloffame.togrampa',\n 'tunic.historicalsociety.collection.gramps.lost',\n 'tunic.historicalsociety.closet.notebook',\n 'tunic.historicalsociety.basement.janitor',\n 'tunic.historicalsociety.stacks.outtolunch',\n 'tunic.historicalsociety.closet.photo',\n 'tunic.historicalsociety.collection.tunic',\n 'tunic.historicalsociety.closet.teddy.intro_0_cs_5',\n 'tunic.historicalsociety.entry.wells.talktogramps',\n 'tunic.historicalsociety.entry.boss.talktogramps',\n 'tunic.historicalsociety.closet.doorblock',\n 'tunic.historicalsociety.entry.block_tomap2',\n 'tunic.historicalsociety.entry.block_tocollection',\n 'tunic.historicalsociety.entry.block_tomap1',\n 'tunic.historicalsociety.collection.gramps.look_0',\n 'tunic.kohlcenter.halloffame.block_0',\n 'tunic.capitol_0.hall.chap1_finale_c',\n 'tunic.historicalsociety.entry.gramps.hub'],\n               '5-12': ['tunic.historicalsociety.frontdesk.archivist.newspaper',\n 'tunic.historicalsociety.frontdesk.archivist.have_glass',\n 'tunic.drycleaner.frontdesk.worker.hub',\n 'tunic.historicalsociety.closet_dirty.gramps.news',\n 'tunic.humanecology.frontdesk.worker.intro',\n 'tunic.library.frontdesk.worker.hello',\n 'tunic.library.frontdesk.worker.wells',\n 'tunic.historicalsociety.frontdesk.archivist.hello',\n 'tunic.historicalsociety.closet_dirty.trigger_scarf',\n 'tunic.drycleaner.frontdesk.worker.done',\n 'tunic.historicalsociety.closet_dirty.what_happened',\n 'tunic.historicalsociety.stacks.journals.pic_2.bingo',\n 'tunic.humanecology.frontdesk.worker.badger',\n 'tunic.historicalsociety.closet_dirty.trigger_coffee',\n 'tunic.drycleaner.frontdesk.logbook.page.bingo',\n 'tunic.library.microfiche.reader.paper2.bingo',\n 'tunic.historicalsociety.closet_dirty.gramps.helpclean',\n 'tunic.historicalsociety.frontdesk.archivist.have_glass_recap',\n 'tunic.historicalsociety.frontdesk.magnify',\n 'tunic.humanecology.frontdesk.businesscards.card_bingo.bingo',\n 'tunic.library.frontdesk.wellsbadge.hub',\n 'tunic.capitol_1.hall.boss.haveyougotit',\n 'tunic.historicalsociety.basement.janitor',\n 'tunic.historicalsociety.closet_dirty.photo',\n 'tunic.historicalsociety.stacks.outtolunch',\n 'tunic.library.frontdesk.worker.wells_recap',\n 'tunic.capitol_0.hall.boss.talktogramps',\n 'tunic.historicalsociety.closet_dirty.gramps.archivist',\n 'tunic.historicalsociety.closet_dirty.door_block_talk',\n 'tunic.historicalsociety.frontdesk.archivist.need_glass_0',\n 'tunic.historicalsociety.frontdesk.block_magnify',\n 'tunic.historicalsociety.frontdesk.archivist.foundtheodora',\n 'tunic.historicalsociety.closet_dirty.gramps.nothing',\n 'tunic.historicalsociety.closet_dirty.door_block_clean',\n 'tunic.library.frontdesk.worker.hello_short',\n 'tunic.historicalsociety.stacks.block',\n 'tunic.historicalsociety.frontdesk.archivist.need_glass_1',\n 'tunic.historicalsociety.frontdesk.archivist.newspaper_recap',\n 'tunic.drycleaner.frontdesk.worker.done2',\n 'tunic.humanecology.frontdesk.block_0',\n 'tunic.library.frontdesk.worker.preflag',\n 'tunic.drycleaner.frontdesk.worker.takealook',\n 'tunic.library.frontdesk.worker.droppedbadge',\n 'tunic.library.microfiche.block_0',\n 'tunic.library.frontdesk.block_badge',\n 'tunic.library.frontdesk.block_badge_2',\n 'tunic.capitol_1.hall.chap2_finale_c',\n 'tunic.drycleaner.frontdesk.block_0',\n 'tunic.humanecology.frontdesk.block_1',\n 'tunic.drycleaner.frontdesk.block_1'],\n               '13-22': ['tunic.historicalsociety.cage.confrontation',\n 'tunic.wildlife.center.crane_ranger.crane',\n 'tunic.wildlife.center.wells.nodeer',\n 'tunic.historicalsociety.frontdesk.archivist_glasses.confrontation',\n 'tunic.historicalsociety.basement.seescratches',\n 'tunic.flaghouse.entry.flag_girl.hello',\n 'tunic.historicalsociety.basement.ch3start',\n 'tunic.historicalsociety.entry.groupconvo_flag',\n 'tunic.historicalsociety.collection_flag.gramps.flag',\n 'tunic.historicalsociety.basement.savedteddy',\n 'tunic.library.frontdesk.worker.nelson',\n 'tunic.wildlife.center.expert.removed_cup',\n 'tunic.library.frontdesk.worker.flag',\n 'tunic.historicalsociety.entry.boss.flag',\n 'tunic.flaghouse.entry.flag_girl.symbol',\n 'tunic.wildlife.center.wells.animals',\n 'tunic.historicalsociety.cage.glasses.afterteddy',\n 'tunic.historicalsociety.cage.teddy.trapped',\n 'tunic.historicalsociety.cage.unlockdoor',\n 'tunic.historicalsociety.stacks.journals.pic_2.bingo',\n 'tunic.historicalsociety.entry.wells.flag',\n 'tunic.humanecology.frontdesk.worker.badger',\n 'tunic.historicalsociety.stacks.journals_flag.pic_0.bingo',\n 'tunic.historicalsociety.entry.directory.closeup.archivist',\n 'tunic.capitol_2.hall.boss.haveyougotit',\n 'tunic.wildlife.center.wells.nodeer_recap',\n 'tunic.historicalsociety.cage.glasses.beforeteddy',\n 'tunic.wildlife.center.expert.recap',\n 'tunic.historicalsociety.stacks.journals_flag.pic_1.bingo',\n 'tunic.historicalsociety.cage.lockeddoor',\n 'tunic.historicalsociety.stacks.journals_flag.pic_2.bingo',\n 'tunic.wildlife.center.remove_cup',\n 'tunic.wildlife.center.tracks.hub.deer',\n 'tunic.historicalsociety.frontdesk.key',\n 'tunic.library.microfiche.reader_flag.paper2.bingo',\n 'tunic.flaghouse.entry.colorbook',\n 'tunic.wildlife.center.coffee',\n 'tunic.historicalsociety.collection_flag.gramps.recap',\n 'tunic.wildlife.center.wells.animals2',\n 'tunic.flaghouse.entry.flag_girl.symbol_recap',\n 'tunic.historicalsociety.closet_dirty.photo',\n 'tunic.historicalsociety.stacks.outtolunch',\n 'tunic.historicalsociety.frontdesk.archivist_glasses.confrontation_recap',\n 'tunic.historicalsociety.entry.boss.flag_recap',\n 'tunic.capitol_1.hall.boss.writeitup',\n 'tunic.library.frontdesk.worker.nelson_recap',\n 'tunic.historicalsociety.entry.wells.flag_recap',\n 'tunic.drycleaner.frontdesk.worker.done2',\n 'tunic.library.frontdesk.worker.flag_recap',\n 'tunic.library.frontdesk.worker.preflag',\n 'tunic.historicalsociety.basement.gramps.seeyalater',\n 'tunic.flaghouse.entry.flag_girl.hello_recap',\n 'tunic.historicalsociety.basement.gramps.whatdo',\n 'tunic.library.frontdesk.block_nelson',\n 'tunic.historicalsociety.cage.need_glasses',\n 'tunic.capitol_2.hall.chap4_finale_c',\n 'tunic.wildlife.center.fox.concern']\n              }\n\n\nSUB_LEVELS = {'0-4': [1, 2, 3, 4],\n              '5-12': [5, 6, 7, 8, 9, 10, 11, 12],\n              '13-22': [13, 14, 15, 16, 17, 18, 19, 20, 21, 22]}\nlevel_groups = [\"0-4\", \"5-12\", \"13-22\"]\n\ncolumns = [\n    pl.col(\"page\").cast(pl.Float32),\n    (\n        (pl.col(\"elapsed_time\") - pl.col(\"elapsed_time\").shift(1))\n        .fill_null(0)\n        .clip(0, 1e9)\n        .over([\"session_id\", \"level\"])\n        .alias(\"elapsed_time_diff\")\n    ),\n    (\n        (pl.col(\"screen_coor_x\") - pl.col(\"screen_coor_x\").shift(1))\n        .abs()\n        .over([\"session_id\", \"level\"])\n    ),\n    (\n        (pl.col(\"screen_coor_y\") - pl.col(\"screen_coor_y\").shift(1))\n        .abs()\n        .over([\"session_id\", \"level\"])\n    ),\n#     pl.col(\"fqid\").fill_null(\"fqid_None\"),\n#     pl.col(\"text_fqid\").fill_null(\"text_fqid_None\")\n]","metadata":{"execution":{"iopub.status.busy":"2023-05-25T15:46:07.224731Z","iopub.execute_input":"2023-05-25T15:46:07.225274Z","iopub.status.idle":"2023-05-25T15:46:07.392562Z","shell.execute_reply.started":"2023-05-25T15:46:07.225238Z","shell.execute_reply":"2023-05-25T15:46:07.391416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg_over = ['session_id', 'level_group']\n\ncount_list = ['event_name', 'fqid', 'room_fqid', 'text_fqid']\nstatistics_list = ['room_coor_x', 'room_coor_y', 'screen_coor_x', 'screen_coor_y', 'elapsed_time', 'page', 'hover_duration']\n# elapsed_time_list = ['event_name', 'name', 'text', 'fqid', 'room_fqid', 'text_fqid', 'level']\n\n# My original fe function\ndef feature_engineering(df: pl.DataFrame):\n    features = df.groupby(agg_over).agg([\n        # Count number of different event\n        *[pl.col(c).unique().count().alias(f'{c}_count') for c in count_list],\n        # Calculate statistics\n        *[pl.col(c).mean().alias(f'{c}_mean') for c in statistics_list],\n        *[pl.col(c).std().alias(f'{c}_std') for c in statistics_list],\n        *[pl.col(c).min().alias(f'{c}_min') for c in statistics_list],\n        *[pl.col(c).max().alias(f'{c}_max') for c in statistics_list],\n        *[pl.col(c).sum().alias(f'{c}_sum2') for c in statistics_list],\n\n        # Calculate elapsed time among groupped value in each column\n        # mean, std, min, max, sum\n        # \n        # For example, in `event_name` column, say we have `event_a` and `event_b`\n        # and we calculate the value of `elapsed_time` for each event:\n        # event_a_elapsed_time_mean,\n        # event_a_elapsed_time_std,\n        # ...\n        # event_b_elapsed_time_mean,\n        # event_b_elapsed_time_std,\n        # ...\n        # *[pl.col('elapsed_time').mean().over(agg_over + [c]).alias(f'{c}_elapsed_time_mean') for c in elapsed_time_list],\n        # *[pl.col('elapsed_time').std().over(agg_over + [c]).alias(f'{c}_elapsed_time_std') for c in elapsed_time_list],\n        # *[pl.col('elapsed_time').min().over(agg_over + [c]).alias(f'{c}_elapsed_time_min') for c in elapsed_time_list],\n        # *[pl.col('elapsed_time').max().over(agg_over + [c]).alias(f'{c}_elapsed_time_max') for c in elapsed_time_list],\n        # *[pl.col('elapsed_time').sum().over(agg_over + [c]).alias(f'{c}_elapsed_time_sum') for c in elapsed_time_list],\n    ]).sort(by=['session_id'])\n\n    return features.to_pandas().set_index('session_id')","metadata":{"execution":{"iopub.status.busy":"2023-05-25T15:46:07.394450Z","iopub.execute_input":"2023-05-25T15:46:07.394931Z","iopub.status.idle":"2023-05-25T15:46:07.410093Z","shell.execute_reply.started":"2023-05-25T15:46:07.394890Z","shell.execute_reply":"2023-05-25T15:46:07.408899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineer(x, grp, use_extra, feature_suffix):\n    x = x.with_columns(columns)\n    LEVELS = SUB_LEVELS[grp]\n    text_lists = sub_text_lists[grp]\n    room_lists = sub_room_lists[grp]\n    fqid_lists = sub_fqid_lists[grp]\n    aggs = [\n        pl.col(\"index\").count().alias(f\"session_number_{feature_suffix}\"),\n\n        *[pl.col('index').filter(pl.col('text').str.contains(c)).count().alias(f'word_{c}') for c in DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text').str.contains(c))).mean().alias(f'word_mean_{c}') for c in\n          DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text').str.contains(c))).std().alias(f'word_std_{c}') for c in\n          DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text').str.contains(c))).max().alias(f'word_max_{c}') for c in\n          DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text').str.contains(c))).sum().alias(f'word_sum_{c}') for c in\n          DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text').str.contains(c))).median().alias(f'word_median_{c}') for c\n          in DIALOGS],\n\n        *[pl.col(c).drop_nulls().n_unique().alias(f\"{c}_unique_{feature_suffix}\") for c in CATS],\n\n        *[pl.col(c).mean().alias(f\"{c}_mean_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).std().alias(f\"{c}_std_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).min().alias(f\"{c}_min_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).max().alias(f\"{c}_max_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).median().alias(f\"{c}_median_{feature_suffix}\") for c in NUMS],\n\n        *[pl.col(\"fqid\").filter(pl.col(\"fqid\") == c).count().alias(f\"{c}_fqid_counts{feature_suffix}\")\n          for c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for\n          c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).median().alias(f\"{c}_ET_median_{feature_suffix}\") for\n          c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for\n          c in fqid_lists],\n\n        *[pl.col(\"text_fqid\").filter(pl.col(\"text_fqid\") == c).count().alias(f\"{c}_text_fqid_counts{feature_suffix}\")\n          for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).median().alias(f\"{c}_ET_median_{feature_suffix}\")\n          for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for\n          c in text_lists],\n\n        *[pl.col(\"room_fqid\").filter(pl.col(\"room_fqid\") == c).count().alias(f\"{c}_room_fqid_counts{feature_suffix}\")\n          for c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for\n          c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).median().alias(f\"{c}_ET_median_{feature_suffix}\")\n          for\n          c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for\n          c in room_lists],\n\n        *[pl.col(\"event_name\").filter(pl.col(\"event_name\") == c).count().alias(f\"{c}_event_name_counts{feature_suffix}\")\n          for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\")\n          for\n          c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).median().alias(\n            f\"{c}_ET_median_{feature_suffix}\") for\n          c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for\n          c in event_name_feature],\n\n        *[pl.col(\"name\").filter(pl.col(\"name\") == c).count().alias(f\"{c}_name_counts{feature_suffix}\") for c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).median().alias(f\"{c}_ET_median_{feature_suffix}\") for\n          c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for c in\n          name_feature],\n\n        *[pl.col(\"level\").filter(pl.col(\"level\") == c).count().alias(f\"{c}_LEVEL_count{feature_suffix}\") for c in\n          LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for c in\n          LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for c\n          in\n          LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for c in\n          LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).median().alias(f\"{c}_ET_median_{feature_suffix}\") for\n          c in\n          LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for c in\n          LEVELS],\n\n        *[pl.col(\"level_group\").filter(pl.col(\"level_group\") == c).count().alias(\n            f\"{c}_LEVEL_group_count{feature_suffix}\") for c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\")\n          for c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).median().alias(\n            f\"{c}_ET_median_{feature_suffix}\") for c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for\n          c in\n          level_groups],\n\n    ]\n\n    df = x.groupby(['session_id'], maintain_order=True).agg(aggs).sort(\"session_id\")\n\n    if use_extra:\n        if grp == '5-12':\n            aggs = [\n                pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"Here's the log book.\")\n                                              | (pl.col(\"fqid\") == 'logbook.page.bingo'))\n                    .apply(lambda s: s.max() - s.min()).alias(\"logbook_bingo_duration\"),\n                pl.col(\"index\").filter(\n                    (pl.col(\"text\") == \"Here's the log book.\") | (pl.col(\"fqid\") == 'logbook.page.bingo')).apply(\n                    lambda s: s.max() - s.min()).alias(\"logbook_bingo_indexCount\"),\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader')) | (\n                            pl.col(\"fqid\") == \"reader.paper2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\n                    \"reader_bingo_duration\"),\n                pl.col(\"index\").filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader')) | (\n                        pl.col(\"fqid\") == \"reader.paper2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\n                    \"reader_bingo_indexCount\"),\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals')) | (\n                            pl.col(\"fqid\") == \"journals.pic_2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\n                    \"journals_bingo_duration\"),\n                pl.col(\"index\").filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals')) | (\n                        pl.col(\"fqid\") == \"journals.pic_2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\n                    \"journals_bingo_indexCount\"),\n            ]\n            tmp = x.groupby([\"session_id\"], maintain_order=True).agg(aggs).sort(\"session_id\")\n            df = df.join(tmp, on=\"session_id\", how='left')\n\n        if grp == '13-22':\n            aggs = [\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader_flag')) | (\n                            pl.col(\"fqid\") == \"tunic.library.microfiche.reader_flag.paper2.bingo\")).apply(\n                    lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"reader_flag_duration\"),\n                pl.col(\"index\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader_flag')) | (\n                            pl.col(\"fqid\") == \"tunic.library.microfiche.reader_flag.paper2.bingo\")).apply(\n                    lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"reader_flag_indexCount\"),\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals_flag')) | (\n                            pl.col(\"fqid\") == \"journals_flag.pic_0.bingo\")).apply(\n                    lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"journalsFlag_bingo_duration\"),\n                pl.col(\"index\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals_flag')) | (\n                            pl.col(\"fqid\") == \"journals_flag.pic_0.bingo\")).apply(\n                    lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"journalsFlag_bingo_indexCount\")\n            ]\n            tmp = x.groupby([\"session_id\"], maintain_order=True).agg(aggs).sort(\"session_id\")\n            df = df.join(tmp, on=\"session_id\", how='left')\n\n    return df.to_pandas()\n\ndef time_feature(train):\n    train[\"year\"] = train[\"session_id\"].apply(lambda x: int(str(x)[:2])).astype(np.uint8)\n    train[\"month\"] = train[\"session_id\"].apply(lambda x: int(str(x)[2:4])+1).astype(np.uint8)\n    train[\"day\"] = train[\"session_id\"].apply(lambda x: int(str(x)[4:6])).astype(np.uint8)\n    train[\"hour\"] = train[\"session_id\"].apply(lambda x: int(str(x)[6:8])).astype(np.uint8)\n    train[\"minute\"] = train[\"session_id\"].apply(lambda x: int(str(x)[8:10])).astype(np.uint8)\n    train[\"second\"] = train[\"session_id\"].apply(lambda x: int(str(x)[10:12])).astype(np.uint8)\n    return train","metadata":{"execution":{"iopub.status.busy":"2023-05-25T15:46:07.413729Z","iopub.execute_input":"2023-05-25T15:46:07.414107Z","iopub.status.idle":"2023-05-25T15:46:07.505338Z","shell.execute_reply.started":"2023-05-25T15:46:07.414071Z","shell.execute_reply":"2023-05-25T15:46:07.503892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load model\nmodels = {}\nparams = {}\nwith open(MODEL_PATH + '/models.pkl', 'rb') as model_file:\n    models = pickle.load(model_file)\nwith open(MODEL_PATH + '/params.pkl', 'rb') as params_file:\n    params = pickle.load(params_file)\n    \nbest_threshold = params['best_threshold']","metadata":{"execution":{"iopub.status.busy":"2023-05-25T15:46:07.506853Z","iopub.execute_input":"2023-05-25T15:46:07.507915Z","iopub.status.idle":"2023-05-25T15:46:08.427674Z","shell.execute_reply.started":"2023-05-25T15:46:07.507871Z","shell.execute_reply":"2023-05-25T15:46:08.426529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# IMPORT KAGGLE API\nimport jo_wilder\nenv = jo_wilder.make_env()\niter_test = env.iter_test()\n\n# del train_labels, oof, true\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-05-25T15:46:08.429929Z","iopub.execute_input":"2023-05-25T15:46:08.430305Z","iopub.status.idle":"2023-05-25T15:46:08.586512Z","shell.execute_reply.started":"2023-05-25T15:46:08.430269Z","shell.execute_reply":"2023-05-25T15:46:08.585287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(models.keys())","metadata":{"execution":{"iopub.status.busy":"2023-05-25T15:46:08.588125Z","iopub.execute_input":"2023-05-25T15:46:08.588507Z","iopub.status.idle":"2023-05-25T15:46:08.594285Z","shell.execute_reply.started":"2023-05-25T15:46:08.588470Z","shell.execute_reply":"2023-05-25T15:46:08.593242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"limits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\nfor (test, sample_submission) in iter_test:\n    \n    # FEATURE ENGINEER TEST DATA original way\n    # df = feature_engineering(pl.from_pandas(test))\n    \n    # INFER TEST DATA\n    grp = test.level_group.values[0]\n    a,b = limits[grp]\n    \n    # learn from catboost mix (without time feature)\n    # test = pl.from_pandas(test).drop(['fullscreen', 'hq', 'music'])\n    # df = feature_engineer(test, grp, True, '').set_index('session_id')\n    \n    # learn from catboost mix (with time feature)\n    test = pl.from_pandas(test).drop(['fullscreen', 'hq', 'music'])\n    df = time_feature(feature_engineer(test, grp, True, '')).set_index('session_id')\n    FEATURES = [c for c in df.columns if c != 'level_group']\n    for t in range(a,b):\n        clf = models[f'{grp}_{t}']\n        p = clf.predict_proba(df[FEATURES].astype('float32'))[0,1]\n        mask = sample_submission.session_id.str.contains(f'q{t}')\n        sample_submission.loc[mask,'correct'] = int( p > best_threshold )\n    \n    env.predict(sample_submission)","metadata":{"execution":{"iopub.status.busy":"2023-05-25T15:46:08.595792Z","iopub.execute_input":"2023-05-25T15:46:08.596830Z","iopub.status.idle":"2023-05-25T15:46:11.750150Z","shell.execute_reply.started":"2023-05-25T15:46:08.596787Z","shell.execute_reply":"2023-05-25T15:46:11.749055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pl.read_csv('submission.csv')\nprint(test.shape)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-25T15:46:11.751410Z","iopub.execute_input":"2023-05-25T15:46:11.751762Z","iopub.status.idle":"2023-05-25T15:46:12.972212Z","shell.execute_reply.started":"2023-05-25T15:46:11.751727Z","shell.execute_reply":"2023-05-25T15:46:12.971259Z"},"trusted":true},"execution_count":null,"outputs":[]}]}