{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nimport sys\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport pickle\nfrom sklearn.linear_model import LogisticRegression\nimport lightgbm as lgb\nfrom catboost import CatBoostClassifier, Pool\n\nCATS = ['event_name', 'name', 'fqid', 'room_fqid', 'text_fqid']\nNUMS = ['page', 'room_coor_x', 'room_coor_y', 'screen_coor_x', 'screen_coor_y',\n        'hover_duration', 'elapsed_time_diff', 'fullscreen', 'hq', 'music']\nfqid_lists = ['worker', 'archivist', 'gramps', 'wells', 'toentry', 'confrontation', 'crane_ranger', 'groupconvo', 'flag_girl', 'tomap', 'tostacks', 'tobasement', 'archivist_glasses', 'boss', 'journals', 'seescratches', 'groupconvo_flag', 'cs', 'teddy', 'expert', 'businesscards', 'ch3start', 'tunic.historicalsociety', 'tofrontdesk', 'savedteddy', 'plaque', 'glasses', 'tunic.drycleaner', 'reader_flag', 'tunic.library', 'tracks', 'tunic.capitol_2', 'trigger_scarf', 'reader', 'directory', 'tunic.capitol_1', 'journals.pic_0.next', 'unlockdoor', 'tunic', 'what_happened', 'tunic.kohlcenter', 'tunic.humanecology', 'colorbook', 'logbook', 'businesscards.card_0.next', 'journals.hub.topics', 'logbook.page.bingo', 'journals.pic_1.next', 'journals_flag', 'reader.paper0.next', 'tracks.hub.deer', 'reader_flag.paper0.next', 'trigger_coffee', 'wellsbadge', 'journals.pic_2.next', 'tomicrofiche', 'journals_flag.pic_0.bingo', 'plaque.face.date', 'notebook', 'tocloset_dirty', 'businesscards.card_bingo.bingo', 'businesscards.card_1.next', 'tunic.wildlife', 'tunic.hub.slip', 'tocage', 'journals.pic_2.bingo', 'tocollectionflag', 'tocollection', 'chap4_finale_c', 'chap2_finale_c', 'lockeddoor', 'journals_flag.hub.topics', 'tunic.capitol_0', 'reader_flag.paper2.bingo', 'photo', 'tunic.flaghouse', 'reader.paper1.next', 'directory.closeup.archivist', 'intro', 'businesscards.card_bingo.next', 'reader.paper2.bingo', 'retirement_letter', 'remove_cup', 'journals_flag.pic_0.next', 'magnify', 'coffee', 'key', 'togrampa', 'reader_flag.paper1.next', 'janitor', 'tohallway', 'chap1_finale', 'report', 'outtolunch', 'journals_flag.hub.topics_old', 'journals_flag.pic_1.next', 'reader.paper2.next', 'chap1_finale_c', 'reader_flag.paper2.next', 'door_block_talk', 'journals_flag.pic_1.bingo', 'journals_flag.pic_2.next', 'journals_flag.pic_2.bingo', 'block_magnify', 'reader.paper0.prev', 'block', 'reader_flag.paper0.prev', 'block_0', 'door_block_clean', 'reader.paper2.prev', 'reader.paper1.prev', 'doorblock', 'tocloset', 'reader_flag.paper2.prev', 'reader_flag.paper1.prev', 'block_tomap2', 'journals_flag.pic_0_old.next', 'journals_flag.pic_1_old.next', 'block_tocollection', 'block_nelson', 'journals_flag.pic_2_old.next', 'block_tomap1', 'block_badge', 'need_glasses', 'block_badge_2', 'fox', 'block_1']\nDIALOGS = ['that', 'this', 'it', 'you','find','found','Found','notebook','Wells','wells','help','need', 'Oh','Ooh','Jo', 'flag', 'can','and','is','the','to']\n\n","metadata":{"papermill":{"duration":1.899897,"end_time":"2023-06-06T01:55:50.567425","exception":false,"start_time":"2023-06-06T01:55:48.667528","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-27T04:49:09.968064Z","iopub.execute_input":"2023-06-27T04:49:09.971297Z","iopub.status.idle":"2023-06-27T04:49:12.624094Z","shell.execute_reply.started":"2023-06-27T04:49:09.971172Z","shell.execute_reply":"2023-06-27T04:49:12.622930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"name_feature = ['basic', 'undefined', 'close', 'open', 'prev', 'next']\n","metadata":{"execution":{"iopub.status.busy":"2023-06-27T04:49:12.625970Z","iopub.execute_input":"2023-06-27T04:49:12.626280Z","iopub.status.idle":"2023-06-27T04:49:12.630577Z","shell.execute_reply.started":"2023-06-27T04:49:12.626252Z","shell.execute_reply":"2023-06-27T04:49:12.629742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text_lists = ['tunic.historicalsociety.cage.confrontation', 'tunic.wildlife.center.crane_ranger.crane', 'tunic.historicalsociety.frontdesk.archivist.newspaper', 'tunic.historicalsociety.entry.groupconvo', 'tunic.wildlife.center.wells.nodeer', 'tunic.historicalsociety.frontdesk.archivist.have_glass', 'tunic.drycleaner.frontdesk.worker.hub', 'tunic.historicalsociety.closet_dirty.gramps.news', 'tunic.humanecology.frontdesk.worker.intro', 'tunic.historicalsociety.frontdesk.archivist_glasses.confrontation', 'tunic.historicalsociety.basement.seescratches', 'tunic.historicalsociety.collection.cs', 'tunic.flaghouse.entry.flag_girl.hello', 'tunic.historicalsociety.collection.gramps.found', 'tunic.historicalsociety.basement.ch3start', 'tunic.historicalsociety.entry.groupconvo_flag', 'tunic.library.frontdesk.worker.hello', 'tunic.library.frontdesk.worker.wells', 'tunic.historicalsociety.collection_flag.gramps.flag', 'tunic.historicalsociety.basement.savedteddy', 'tunic.library.frontdesk.worker.nelson', 'tunic.wildlife.center.expert.removed_cup', 'tunic.library.frontdesk.worker.flag', 'tunic.historicalsociety.frontdesk.archivist.hello', 'tunic.historicalsociety.closet.gramps.intro_0_cs_0', 'tunic.historicalsociety.entry.boss.flag', 'tunic.flaghouse.entry.flag_girl.symbol', 'tunic.historicalsociety.closet_dirty.trigger_scarf', 'tunic.drycleaner.frontdesk.worker.done', 'tunic.historicalsociety.closet_dirty.what_happened', 'tunic.wildlife.center.wells.animals', 'tunic.historicalsociety.closet.teddy.intro_0_cs_0', 'tunic.historicalsociety.cage.glasses.afterteddy', 'tunic.historicalsociety.cage.teddy.trapped', 'tunic.historicalsociety.cage.unlockdoor', 'tunic.historicalsociety.stacks.journals.pic_2.bingo', 'tunic.historicalsociety.entry.wells.flag', 'tunic.humanecology.frontdesk.worker.badger', 'tunic.historicalsociety.stacks.journals_flag.pic_0.bingo', 'tunic.historicalsociety.closet.intro', 'tunic.historicalsociety.closet.retirement_letter.hub', 'tunic.historicalsociety.entry.directory.closeup.archivist', 'tunic.historicalsociety.collection.tunic.slip', 'tunic.kohlcenter.halloffame.plaque.face.date', 'tunic.historicalsociety.closet_dirty.trigger_coffee', 'tunic.drycleaner.frontdesk.logbook.page.bingo', 'tunic.library.microfiche.reader.paper2.bingo', 'tunic.kohlcenter.halloffame.togrampa', 'tunic.capitol_2.hall.boss.haveyougotit', 'tunic.wildlife.center.wells.nodeer_recap', 'tunic.historicalsociety.cage.glasses.beforeteddy', 'tunic.historicalsociety.closet_dirty.gramps.helpclean', 'tunic.wildlife.center.expert.recap', 'tunic.historicalsociety.frontdesk.archivist.have_glass_recap', 'tunic.historicalsociety.stacks.journals_flag.pic_1.bingo', 'tunic.historicalsociety.cage.lockeddoor', 'tunic.historicalsociety.stacks.journals_flag.pic_2.bingo', 'tunic.historicalsociety.collection.gramps.lost', 'tunic.historicalsociety.closet.notebook', 'tunic.historicalsociety.frontdesk.magnify', 'tunic.humanecology.frontdesk.businesscards.card_bingo.bingo', 'tunic.wildlife.center.remove_cup', 'tunic.library.frontdesk.wellsbadge.hub', 'tunic.wildlife.center.tracks.hub.deer', 'tunic.historicalsociety.frontdesk.key', 'tunic.library.microfiche.reader_flag.paper2.bingo', 'tunic.flaghouse.entry.colorbook', 'tunic.wildlife.center.coffee', 'tunic.capitol_1.hall.boss.haveyougotit', 'tunic.historicalsociety.basement.janitor', 'tunic.historicalsociety.collection_flag.gramps.recap', 'tunic.wildlife.center.wells.animals2', 'tunic.flaghouse.entry.flag_girl.symbol_recap', 'tunic.historicalsociety.closet_dirty.photo', 'tunic.historicalsociety.stacks.outtolunch', 'tunic.library.frontdesk.worker.wells_recap', 'tunic.historicalsociety.frontdesk.archivist_glasses.confrontation_recap', 'tunic.capitol_0.hall.boss.talktogramps', 'tunic.historicalsociety.closet.photo', 'tunic.historicalsociety.collection.tunic', 'tunic.historicalsociety.closet.teddy.intro_0_cs_5', 'tunic.historicalsociety.closet_dirty.gramps.archivist', 'tunic.historicalsociety.closet_dirty.door_block_talk', 'tunic.historicalsociety.entry.boss.flag_recap', 'tunic.historicalsociety.frontdesk.archivist.need_glass_0', 'tunic.historicalsociety.entry.wells.talktogramps', 'tunic.historicalsociety.frontdesk.block_magnify', 'tunic.historicalsociety.frontdesk.archivist.foundtheodora', 'tunic.historicalsociety.closet_dirty.gramps.nothing', 'tunic.historicalsociety.closet_dirty.door_block_clean', 'tunic.capitol_1.hall.boss.writeitup', 'tunic.library.frontdesk.worker.nelson_recap', 'tunic.library.frontdesk.worker.hello_short', 'tunic.historicalsociety.stacks.block', 'tunic.historicalsociety.frontdesk.archivist.need_glass_1', 'tunic.historicalsociety.entry.boss.talktogramps', 'tunic.historicalsociety.frontdesk.archivist.newspaper_recap', 'tunic.historicalsociety.entry.wells.flag_recap', 'tunic.drycleaner.frontdesk.worker.done2', 'tunic.library.frontdesk.worker.flag_recap', 'tunic.humanecology.frontdesk.block_0', 'tunic.library.frontdesk.worker.preflag', 'tunic.historicalsociety.basement.gramps.seeyalater', 'tunic.flaghouse.entry.flag_girl.hello_recap', 'tunic.historicalsociety.closet.doorblock', 'tunic.drycleaner.frontdesk.worker.takealook', 'tunic.historicalsociety.basement.gramps.whatdo', 'tunic.library.frontdesk.worker.droppedbadge', 'tunic.historicalsociety.entry.block_tomap2', 'tunic.library.frontdesk.block_nelson', 'tunic.library.microfiche.block_0', 'tunic.historicalsociety.entry.block_tocollection', 'tunic.historicalsociety.entry.block_tomap1', 'tunic.historicalsociety.collection.gramps.look_0', 'tunic.library.frontdesk.block_badge', 'tunic.historicalsociety.cage.need_glasses', 'tunic.library.frontdesk.block_badge_2', 'tunic.kohlcenter.halloffame.block_0', 'tunic.capitol_0.hall.chap1_finale_c', 'tunic.capitol_1.hall.chap2_finale_c', 'tunic.capitol_2.hall.chap4_finale_c', 'tunic.wildlife.center.fox.concern', 'tunic.drycleaner.frontdesk.block_0', 'tunic.historicalsociety.entry.gramps.hub', 'tunic.humanecology.frontdesk.block_1', 'tunic.drycleaner.frontdesk.block_1']\nroom_lists = ['tunic.historicalsociety.entry', 'tunic.wildlife.center', 'tunic.historicalsociety.cage', 'tunic.library.frontdesk', 'tunic.historicalsociety.frontdesk', 'tunic.historicalsociety.stacks', 'tunic.historicalsociety.closet_dirty', 'tunic.humanecology.frontdesk', 'tunic.historicalsociety.basement', 'tunic.kohlcenter.halloffame', 'tunic.library.microfiche', 'tunic.drycleaner.frontdesk', 'tunic.historicalsociety.collection', 'tunic.historicalsociety.closet', 'tunic.flaghouse.entry', 'tunic.historicalsociety.collection_flag', 'tunic.capitol_1.hall', 'tunic.capitol_0.hall', 'tunic.capitol_2.hall']\nevent_name_feature = ['cutscene_click', 'person_click', 'navigate_click',\n       'observation_click', 'notification_click', 'object_click',\n       'object_hover', 'map_hover', 'map_click', 'checkpoint',\n       'notebook_click']\n\nLEVELS = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22]\nlevel_groups = [\"0-4\", \"5-12\", \"13-22\"]","metadata":{"execution":{"iopub.status.busy":"2023-06-27T04:49:12.632145Z","iopub.execute_input":"2023-06-27T04:49:12.632734Z","iopub.status.idle":"2023-06-27T04:49:12.651317Z","shell.execute_reply.started":"2023-06-27T04:49:12.632701Z","shell.execute_reply":"2023-06-27T04:49:12.650282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ncolumns = [\n    pl.col(\"page\").cast(pl.Float32),\n    (\n        (pl.col(\"elapsed_time\") - pl.col(\"elapsed_time\").shift(1))\n        .fill_null(0)\n        .clip(0, 1e9)\n        .over([\"session_id\", \"level\"])\n        .alias(\"elapsed_time_diff\")\n    ),\n    (\n        (pl.col(\"screen_coor_x\") - pl.col(\"screen_coor_x\").shift(1))\n        .abs()\n        .over([\"session_id\", \"level\"])\n    ),\n    (\n        (pl.col(\"screen_coor_y\") - pl.col(\"screen_coor_y\").shift(1))\n        .abs()\n        .over([\"session_id\", \"level\"])\n    ),\n    pl.col(\"fqid\").fill_null(\"fqid_None\"),\n    pl.col(\"text_fqid\").fill_null(\"text_fqid_None\"),\n    pl.col(\"text\").fill_null(\"text_None\")\n\n]\n\ndef time_feature(train):\n    train[\"year\"] = train[\"session_id\"].apply(lambda x: int(str(x)[:2])).astype(np.uint8)\n    train[\"month\"] = train[\"session_id\"].apply(lambda x: int(str(x)[2:4])+1).astype(np.uint8)\n    train[\"day\"] = train[\"session_id\"].apply(lambda x: int(str(x)[4:6])).astype(np.uint8)\n    train[\"hour\"] = train[\"session_id\"].apply(lambda x: int(str(x)[6:8])).astype(np.uint8)\n    train[\"minute\"] = train[\"session_id\"].apply(lambda x: int(str(x)[8:10])).astype(np.uint8)\n    train[\"second\"] = train[\"session_id\"].apply(lambda x: int(str(x)[10:12])).astype(np.uint8)\n    return train\n","metadata":{"papermill":{"duration":0.029687,"end_time":"2023-06-06T01:55:50.604197","exception":false,"start_time":"2023-06-06T01:55:50.574510","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-27T04:49:12.653979Z","iopub.execute_input":"2023-06-27T04:49:12.654617Z","iopub.status.idle":"2023-06-27T04:49:12.681423Z","shell.execute_reply.started":"2023-06-27T04:49:12.654582Z","shell.execute_reply":"2023-06-27T04:49:12.680359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ndef feature_engineer(x, grp, use_extra, feature_suffix):\n    aggs = [\n        pl.col(\"index\").count().alias(f\"session_number_{feature_suffix}\"),\n\n        *[pl.col('index').filter(pl.col('text').str.contains(c)).count().alias(f'word_{c}') for c in DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text').str.contains(c))).mean().alias(f'word_mean_{c}') for c in\n          DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text').str.contains(c))).std().alias(f'word_std_{c}') for c in\n          DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text').str.contains(c))).max().alias(f'word_max_{c}') for c in\n          DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text').str.contains(c))).sum().alias(f'word_sum_{c}') for c in\n          DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text').str.contains(c))).median().alias(f'word_median_{c}') for c\n          in DIALOGS],\n\n\n        \n        *[pl.col(c).drop_nulls().n_unique().alias(f\"{c}_unique_{feature_suffix}\") for c in CATS],\n\n        *[pl.col(c).mean().alias(f\"{c}_mean_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).std().alias(f\"{c}_std_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).min().alias(f\"{c}_min_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).max().alias(f\"{c}_max_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).median().alias(f\"{c}_median_{feature_suffix}\") for c in NUMS],\n\n        *[pl.col(\"fqid\").filter(pl.col(\"fqid\") == c).count().alias(f\"{c}_fqid_counts{feature_suffix}\")\n          for c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for\n          c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).median().alias(f\"{c}_ET_median_{feature_suffix}\") for\n          c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for\n          c in fqid_lists],\n        \n        *[pl.col('index').filter(pl.col('text_code') == c).count().alias(f'{c}_text_code_counts{feature_suffix}') for c in text_list],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text_code') == c)).mean().alias(f'{c}_text_mean_{feature_suffix}') for c in\n          text_list],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text_code') == c)).std().alias(f'{c}_text_std_{feature_suffix}') for c in\n          text_list],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text_code') == c)).max().alias(f'{c}_text_max_{feature_suffix}') for c in\n          text_list],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text_code') == c)).sum().alias(f'{c}_text_sum_{feature_suffix}') for c in\n          text_list],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text_code') == c)).median().alias(f'{c}_text_median_{feature_suffix}') for c\n          in text_list],\n        \n        *[pl.col(\"text_fqid\").filter(pl.col(\"text_fqid\") == c).count().alias(f\"{c}_text_fqid_counts{feature_suffix}\")\n          for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).median().alias(f\"{c}_ET_median_{feature_suffix}\")\n          for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for\n          c in text_lists],\n\n        *[pl.col(\"room_fqid\").filter(pl.col(\"room_fqid\") == c).count().alias(f\"{c}_room_fqid_counts{feature_suffix}\")\n          for c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for\n          c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).median().alias(f\"{c}_ET_median_{feature_suffix}\")\n          for\n          c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for\n          c in room_lists],\n\n        *[pl.col(\"event_name\").filter(pl.col(\"event_name\") == c).count().alias(f\"{c}_event_name_counts{feature_suffix}\")\n          for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\")\n          for\n          c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).median().alias(\n            f\"{c}_ET_median_{feature_suffix}\") for\n          c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for\n          c in event_name_feature],\n\n        *[pl.col(\"name\").filter(pl.col(\"name\") == c).count().alias(f\"{c}_name_counts{feature_suffix}\") for c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).median().alias(f\"{c}_ET_median_{feature_suffix}\") for\n          c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for c in\n          name_feature],\n\n        *[pl.col(\"level\").filter(pl.col(\"level\") == c).count().alias(f\"{c}_LEVEL_count{feature_suffix}\") for c in\n          LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for c in\n          LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for c\n          in\n          LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for c in\n          LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).median().alias(f\"{c}_ET_median_{feature_suffix}\") for\n          c in\n          LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for c in\n          LEVELS],\n\n        *[pl.col(\"level_group\").filter(pl.col(\"level_group\") == c).count().alias(\n            f\"{c}_LEVEL_group_count{feature_suffix}\") for c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\")\n          for c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).median().alias(\n            f\"{c}_ET_median_{feature_suffix}\") for c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for\n          c in\n          level_groups],\n\n    ]\n\n    df = x.groupby(['session_id'], maintain_order=True).agg(aggs).sort(\"session_id\")\n\n    if use_extra:\n        if grp == '5-12':\n            aggs = [\n                pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"Here's the log book.\")\n                                              | (pl.col(\"fqid\") == 'logbook.page.bingo'))\n                    .apply(lambda s: s.max() - s.min()).alias(\"logbook_bingo_duration\"),\n                pl.col(\"index\").filter(\n                    (pl.col(\"text\") == \"Here's the log book.\") | (pl.col(\"fqid\") == 'logbook.page.bingo')).apply(\n                    lambda s: s.max() - s.min()).alias(\"logbook_bingo_indexCount\"),\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader')) | (\n                            pl.col(\"fqid\") == \"reader.paper2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\n                    \"reader_bingo_duration\"),\n                pl.col(\"index\").filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader')) | (\n                        pl.col(\"fqid\") == \"reader.paper2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\n                    \"reader_bingo_indexCount\"),\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals')) | (\n                            pl.col(\"fqid\") == \"journals.pic_2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\n                    \"journals_bingo_duration\"),\n                pl.col(\"index\").filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals')) | (\n                        pl.col(\"fqid\") == \"journals.pic_2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\n                    \"journals_bingo_indexCount\"),\n            ]\n            tmp = x.groupby([\"session_id\"], maintain_order=True).agg(aggs).sort(\"session_id\")\n            df = df.join(tmp, on=\"session_id\", how='left')\n\n        if grp == '13-22':\n            aggs = [\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader_flag')) | (\n                            pl.col(\"fqid\") == \"tunic.library.microfiche.reader_flag.paper2.bingo\")).apply(\n                    lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"reader_flag_duration\"),\n                pl.col(\"index\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader_flag')) | (\n                            pl.col(\"fqid\") == \"tunic.library.microfiche.reader_flag.paper2.bingo\")).apply(\n                    lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"reader_flag_indexCount\"),\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals_flag')) | (\n                            pl.col(\"fqid\") == \"journals_flag.pic_0.bingo\")).apply(\n                    lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"journalsFlag_bingo_duration\"),\n                pl.col(\"index\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals_flag')) | (\n                            pl.col(\"fqid\") == \"journals_flag.pic_0.bingo\")).apply(\n                    lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"journalsFlag_bingo_indexCount\")\n            ]\n            tmp = x.groupby([\"session_id\"], maintain_order=True).agg(aggs).sort(\"session_id\")\n            df = df.join(tmp, on=\"session_id\", how='left')\n\n    return df.to_pandas()\n\n\n","metadata":{"papermill":{"duration":0.12369,"end_time":"2023-06-06T01:55:50.734971","exception":false,"start_time":"2023-06-06T01:55:50.611281","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-27T04:49:12.683405Z","iopub.execute_input":"2023-06-27T04:49:12.684925Z","iopub.status.idle":"2023-06-27T04:49:12.773102Z","shell.execute_reply.started":"2023-06-27T04:49:12.684886Z","shell.execute_reply":"2023-06-27T04:49:12.771699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(f'/kaggle/input/parameters2/dataall83.pkl', 'rb') as fr:\n    [text_list,text_dict,importance_dict,models_list,models_stack] = pickle.load(fr)\n\n    \nwith open(f'/kaggle/input/parameters3/dataall84.pkl', 'rb') as fr:\n    [text_list,text_dict,importance_dict_,models_list_,models_stack_] = pickle.load(fr)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-06-27T04:49:12.779292Z","iopub.execute_input":"2023-06-27T04:49:12.782497Z","iopub.status.idle":"2023-06-27T04:49:15.683622Z","shell.execute_reply.started":"2023-06-27T04:49:12.782439Z","shell.execute_reply":"2023-06-27T04:49:15.682479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Run Inference","metadata":{"papermill":{"duration":0.006506,"end_time":"2023-06-06T01:55:50.778552","exception":false,"start_time":"2023-06-06T01:55:50.772046","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import jo_wilder\nenv = jo_wilder.make_env()\niter_test = env.iter_test()","metadata":{"papermill":{"duration":0.038869,"end_time":"2023-06-06T01:55:50.824272","exception":false,"start_time":"2023-06-06T01:55:50.785403","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-27T04:49:15.687704Z","iopub.execute_input":"2023-06-27T04:49:15.688063Z","iopub.status.idle":"2023-06-27T04:49:15.714508Z","shell.execute_reply.started":"2023-06-27T04:49:15.688032Z","shell.execute_reply":"2023-06-27T04:49:15.713681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"limits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\noofdict = {}\noofdict[0] = pd.DataFrame([])\noofdict[1] = pd.DataFrame([])\noofdict[2]= pd.DataFrame([])\noofdict[3] = pd.DataFrame([])\noofdict[4] = pd.DataFrame([])\n\n\noofdict_ = {}\noofdict_[0] = pd.DataFrame([])\noofdict_[1] = pd.DataFrame([])\noofdict_[2]= pd.DataFrame([])\noofdict_[3] = pd.DataFrame([])\noofdict_[4] = pd.DataFrame([])\n\nfor (test, sample_submission) in iter_test:\n    test = test.sort_values(by = 'index').reset_index(drop=True)\n    sample_submission['question'] = [int(label.split('_')[1].replace('q','')) for label in sample_submission['session_id']]    \n    sample_submission = sample_submission.sort_values(by = 'question').reset_index(drop=True)\n    sample_submission['correct'] = 1\n    sample_submission.loc[sample_submission.question.isin([5, 8, 10, 13, 15]), 'correct'] = 0  \n    \n    try:\n        grp = test.level_group.values[0]\n        session_id = test.session_id.values[0]\n        sample_submission_tmp = sample_submission.copy()\n        sample_submission_tmp['lgbname'] = ['lgb_'+xx.split('_')[1].replace('q','') for xx in sample_submission_tmp.session_id]\n        sample_submission_tmp['session_id'] = [int(xx.split('_')[0]) for xx in sample_submission_tmp.session_id]\n\n        df = pl.from_pandas(test).with_columns(columns)\n        ###map with dict \n        df = df.with_columns(\n            pl.col(\"text\").map_dict(text_dict).alias(\"text_code\")\n            )\n        df = feature_engineer(df, grp, use_extra=True, feature_suffix='')\n        df = time_feature(df)\n        preds_stacklist = []\n        for fold in range(5):\n            preds_prob = []\n            preds = []\n            a,b = limits[grp]\n            for q in range(a, b):\n                FEATURES = importance_dict[str(q)]\n                model = models_list[q-1][fold]\n                pred = model.predict(df[FEATURES].astype(np.float32))\n                preds_prob.append(pred)\n#                 preds_prob.append(pred)\n\n            sample_submission_tmp[\"prob\"] = np.concatenate(preds_prob)\n            \n            ###########\n#             oofdict[fold] = pd.concat([oofdict[fold],sample_submission_tmp]).reset_index(drop=True)\n#             oofresult = oofdict[fold][oofdict[fold].session_id.isin(sample_submission_tmp.session_id)]\n#             oofresult = pd.pivot(oofresult,'session_id','lgbname','prob')\n#             oofresult = oofresult.reindex(df.session_id)\n\n            oofdict[fold] = pd.concat([oofdict[fold],sample_submission_tmp]).reset_index(drop=True)\n            oofresult = oofdict[fold][oofdict[fold].session_id.isin(sample_submission_tmp.session_id)]\n            oofresult = pd.pivot(oofresult,'session_id','lgbname','prob')\n            oofresult = oofresult.reindex(df.session_id)\n\n\n            predsfinal = []\n            for q in range(a, b):\n                if q <= 3:\n                    FEATURES = [f'lgb_{xx}' for xx in [1,2,3]]\n                elif q <= 13:\n                    FEATURES = [f'lgb_{xx}' for xx in [1,2,3,4,5,6,7,8,9,10,11,12,13]]\n                elif q <= 22:\n                    FEATURES = [f'lgb_{xx}' for xx in [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18]]\n                model_stack = models_stack[q-1][fold]\n                pred = model_stack.predict_proba(oofresult[FEATURES].astype(np.float32))[:,1]\n\n                predsfinal.append(pred)\n            preds_stacklist.append(np.concatenate(predsfinal))\n            \n            \n        preds_stacklist2 = []\n        for fold in range(5):\n            preds_prob = []\n            preds = []\n            a,b = limits[grp]\n            for q in range(a, b):\n                FEATURES = importance_dict_[str(q)]\n                model = models_list_[q-1][fold]\n                pred = model.predict(df[FEATURES].astype(np.float32))\n                preds_prob.append(pred)\n#                 preds_prob.append(pred)\n\n            sample_submission_tmp[\"prob\"] = np.concatenate(preds_prob)\n            \n            ###########\n#             oofdict[fold] = pd.concat([oofdict[fold],sample_submission_tmp]).reset_index(drop=True)\n#             oofresult = oofdict[fold][oofdict[fold].session_id.isin(sample_submission_tmp.session_id)]\n#             oofresult = pd.pivot(oofresult,'session_id','lgbname','prob')\n#             oofresult = oofresult.reindex(df.session_id)\n\n            oofdict_[fold] = pd.concat([oofdict_[fold],sample_submission_tmp]).reset_index(drop=True)\n            oofresult = oofdict_[fold][oofdict_[fold].session_id.isin(sample_submission_tmp.session_id)]\n            oofresult = pd.pivot(oofresult,'session_id','lgbname','prob')\n            oofresult = oofresult.reindex(df.session_id)\n\n\n            predsfinal = []\n            for q in range(a, b):\n                if q <= 3:\n                    FEATURES = [f'lgb_{xx}' for xx in [1,2,3]]\n                elif q <= 13:\n                    FEATURES = [f'lgb_{xx}' for xx in [1,2,3,4,5,6,7,8,9,10,11,12,13]]\n                elif q <= 22:\n                    FEATURES = [f'lgb_{xx}' for xx in [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18]]\n                model_stack = models_stack_[q-1][fold]\n                pred = model_stack.predict_proba(oofresult[FEATURES].astype(np.float32))[:,1]\n\n                predsfinal.append(pred)\n            preds_stacklist2.append(np.concatenate(predsfinal))\n            \n            \n            \n        sample_submission[\"correct\"] = (np.mean(preds_stacklist,axis=0)/2+np.mean(preds_stacklist2,axis=0)/2 > 0.625)*1\n    except:\n        1\n\n    env.predict(sample_submission[['session_id','correct']])","metadata":{"execution":{"iopub.status.busy":"2023-06-27T04:50:04.573226Z","iopub.execute_input":"2023-06-27T04:50:04.573648Z","iopub.status.idle":"2023-06-27T04:50:04.608570Z","shell.execute_reply.started":"2023-06-27T04:50:04.573611Z","shell.execute_reply":"2023-06-27T04:50:04.607695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('submission.csv')\nprint(sub.shape, sub.correct.mean())\nsub","metadata":{"papermill":{"duration":0.040548,"end_time":"2023-06-06T01:55:58.378489","exception":false,"start_time":"2023-06-06T01:55:58.337941","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-27T04:50:06.393242Z","iopub.execute_input":"2023-06-27T04:50:06.394126Z","iopub.status.idle":"2023-06-27T04:50:06.411608Z","shell.execute_reply.started":"2023-06-27T04:50:06.394085Z","shell.execute_reply":"2023-06-27T04:50:06.410706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.008135,"end_time":"2023-06-06T01:55:58.395123","exception":false,"start_time":"2023-06-06T01:55:58.386988","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.008181,"end_time":"2023-06-06T01:55:58.412100","exception":false,"start_time":"2023-06-06T01:55:58.403919","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.008057,"end_time":"2023-06-06T01:55:58.428612","exception":false,"start_time":"2023-06-06T01:55:58.420555","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.008208,"end_time":"2023-06-06T01:55:58.445220","exception":false,"start_time":"2023-06-06T01:55:58.437012","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.008435,"end_time":"2023-06-06T01:55:58.462158","exception":false,"start_time":"2023-06-06T01:55:58.453723","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.008565,"end_time":"2023-06-06T01:55:58.479492","exception":false,"start_time":"2023-06-06T01:55:58.470927","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.008515,"end_time":"2023-06-06T01:55:58.496925","exception":false,"start_time":"2023-06-06T01:55:58.488410","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}