{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n","metadata":{}},{"cell_type":"markdown","source":"The notebook uses the preprocessing from From : https://www.kaggle.com/code/takanashihumbert/magic-bingo-train-part-lb-0-687\nand from https://www.kaggle.com/code/leehomhuang/catboost-baseline-with-lots-features-inference\nwith some new features.\n\nAnd ideas from https://www.kaggle.com/code/cdeotte/xgboost-baseline-0-676 as well.","metadata":{}},{"cell_type":"code","source":"# !pip install /kaggle/input/polars-for-student/polars-0.16.9-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n# !pip install /kaggle/input/polars-for-student/typing_extensions-4.5.0-py3-none-any.whl","metadata":{"execution":{"iopub.execute_input":"2023-03-09T08:46:18.317469Z","iopub.status.busy":"2023-03-09T08:46:18.317008Z","iopub.status.idle":"2023-03-09T08:47:41.600618Z","shell.execute_reply":"2023-03-09T08:47:41.598863Z","shell.execute_reply.started":"2023-03-09T08:46:18.317368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport os\n\nimport pandas as pd\nimport numpy as np\nimport warnings\nimport pickle\nimport polars as pl\n\nfrom collections import defaultdict\nfrom itertools import combinations\nimport pyarrow as pa\n\nfrom xgboost import XGBClassifier\nfrom sklearn.model_selection import KFold, GroupKFold\n\nfrom lightgbm import LGBMClassifier\nfrom lightgbm import early_stopping\nfrom lightgbm import log_evaluation\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score, f1_score\n\n\nimport matplotlib.pyplot as plt\nfrom colorama import Fore, Back, Style","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2023-03-09T08:48:29.967761Z","iopub.status.busy":"2023-03-09T08:48:29.967339Z","iopub.status.idle":"2023-03-09T08:48:29.975108Z","shell.execute_reply":"2023-03-09T08:48:29.974132Z","shell.execute_reply.started":"2023-03-09T08:48:29.967726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data preprocessing","metadata":{}},{"cell_type":"code","source":"CATS = ['event_name', 'name', 'fqid', 'room_fqid', 'text_fqid']\nNUMS = ['page', 'room_coor_x', 'room_coor_y', 'screen_coor_x', 'screen_coor_y',\n        'hover_duration', 'elapsed_time_diff']\n\nname_feature = ['basic', 'undefined', 'close', 'open', 'prev', 'next']\nevent_name_feature = ['cutscene_click', 'person_click', 'navigate_click',\n       'observation_click', 'notification_click', 'object_click',\n       'object_hover', 'map_hover', 'map_click', 'checkpoint',\n       'notebook_click']\n\n# from https://www.kaggle.com/code/leehomhuang/catboost-baseline-with-lots-features-inference :\nfqid_lists = ['worker', 'archivist', 'gramps', 'wells', 'toentry', 'confrontation', 'crane_ranger', 'groupconvo', 'flag_girl', 'tomap', 'tostacks', 'tobasement', 'archivist_glasses', 'boss', 'journals', 'seescratches', 'groupconvo_flag', 'cs', 'teddy', 'expert', 'businesscards', 'ch3start', 'tunic.historicalsociety', 'tofrontdesk', 'savedteddy', 'plaque', 'glasses', 'tunic.drycleaner', 'reader_flag', 'tunic.library', 'tracks', 'tunic.capitol_2', 'trigger_scarf', 'reader', 'directory', 'tunic.capitol_1', 'journals.pic_0.next', 'unlockdoor', 'tunic', 'what_happened', 'tunic.kohlcenter', 'tunic.humanecology', 'colorbook', 'logbook', 'businesscards.card_0.next', 'journals.hub.topics', 'logbook.page.bingo', 'journals.pic_1.next', 'journals_flag', 'reader.paper0.next', 'tracks.hub.deer', 'reader_flag.paper0.next', 'trigger_coffee', 'wellsbadge', 'journals.pic_2.next', 'tomicrofiche', 'journals_flag.pic_0.bingo', 'plaque.face.date', 'notebook', 'tocloset_dirty', 'businesscards.card_bingo.bingo', 'businesscards.card_1.next', 'tunic.wildlife', 'tunic.hub.slip', 'tocage', 'journals.pic_2.bingo', 'tocollectionflag', 'tocollection', 'chap4_finale_c', 'chap2_finale_c', 'lockeddoor', 'journals_flag.hub.topics', 'tunic.capitol_0', 'reader_flag.paper2.bingo', 'photo', 'tunic.flaghouse', 'reader.paper1.next', 'directory.closeup.archivist', 'intro', 'businesscards.card_bingo.next', 'reader.paper2.bingo', 'retirement_letter', 'remove_cup', 'journals_flag.pic_0.next', 'magnify', 'coffee', 'key', 'togrampa', 'reader_flag.paper1.next', 'janitor', 'tohallway', 'chap1_finale', 'report', 'outtolunch', 'journals_flag.hub.topics_old', 'journals_flag.pic_1.next', 'reader.paper2.next', 'chap1_finale_c', 'reader_flag.paper2.next', 'door_block_talk', 'journals_flag.pic_1.bingo', 'journals_flag.pic_2.next', 'journals_flag.pic_2.bingo', 'block_magnify', 'reader.paper0.prev', 'block', 'reader_flag.paper0.prev', 'block_0', 'door_block_clean', 'reader.paper2.prev', 'reader.paper1.prev', 'doorblock', 'tocloset', 'reader_flag.paper2.prev', 'reader_flag.paper1.prev', 'block_tomap2', 'journals_flag.pic_0_old.next', 'journals_flag.pic_1_old.next', 'block_tocollection', 'block_nelson', 'journals_flag.pic_2_old.next', 'block_tomap1', 'block_badge', 'need_glasses', 'block_badge_2', 'fox', 'block_1']\ntext_lists = ['tunic.historicalsociety.cage.confrontation', 'tunic.wildlife.center.crane_ranger.crane', 'tunic.historicalsociety.frontdesk.archivist.newspaper', 'tunic.historicalsociety.entry.groupconvo', 'tunic.wildlife.center.wells.nodeer', 'tunic.historicalsociety.frontdesk.archivist.have_glass', 'tunic.drycleaner.frontdesk.worker.hub', 'tunic.historicalsociety.closet_dirty.gramps.news', 'tunic.humanecology.frontdesk.worker.intro', 'tunic.historicalsociety.frontdesk.archivist_glasses.confrontation', 'tunic.historicalsociety.basement.seescratches', 'tunic.historicalsociety.collection.cs', 'tunic.flaghouse.entry.flag_girl.hello', 'tunic.historicalsociety.collection.gramps.found', 'tunic.historicalsociety.basement.ch3start', 'tunic.historicalsociety.entry.groupconvo_flag', 'tunic.library.frontdesk.worker.hello', 'tunic.library.frontdesk.worker.wells', 'tunic.historicalsociety.collection_flag.gramps.flag', 'tunic.historicalsociety.basement.savedteddy', 'tunic.library.frontdesk.worker.nelson', 'tunic.wildlife.center.expert.removed_cup', 'tunic.library.frontdesk.worker.flag', 'tunic.historicalsociety.frontdesk.archivist.hello', 'tunic.historicalsociety.closet.gramps.intro_0_cs_0', 'tunic.historicalsociety.entry.boss.flag', 'tunic.flaghouse.entry.flag_girl.symbol', 'tunic.historicalsociety.closet_dirty.trigger_scarf', 'tunic.drycleaner.frontdesk.worker.done', 'tunic.historicalsociety.closet_dirty.what_happened', 'tunic.wildlife.center.wells.animals', 'tunic.historicalsociety.closet.teddy.intro_0_cs_0', 'tunic.historicalsociety.cage.glasses.afterteddy', 'tunic.historicalsociety.cage.teddy.trapped', 'tunic.historicalsociety.cage.unlockdoor', 'tunic.historicalsociety.stacks.journals.pic_2.bingo', 'tunic.historicalsociety.entry.wells.flag', 'tunic.humanecology.frontdesk.worker.badger', 'tunic.historicalsociety.stacks.journals_flag.pic_0.bingo', 'tunic.historicalsociety.closet.intro', 'tunic.historicalsociety.closet.retirement_letter.hub', 'tunic.historicalsociety.entry.directory.closeup.archivist', 'tunic.historicalsociety.collection.tunic.slip', 'tunic.kohlcenter.halloffame.plaque.face.date', 'tunic.historicalsociety.closet_dirty.trigger_coffee', 'tunic.drycleaner.frontdesk.logbook.page.bingo', 'tunic.library.microfiche.reader.paper2.bingo', 'tunic.kohlcenter.halloffame.togrampa', 'tunic.capitol_2.hall.boss.haveyougotit', 'tunic.wildlife.center.wells.nodeer_recap', 'tunic.historicalsociety.cage.glasses.beforeteddy', 'tunic.historicalsociety.closet_dirty.gramps.helpclean', 'tunic.wildlife.center.expert.recap', 'tunic.historicalsociety.frontdesk.archivist.have_glass_recap', 'tunic.historicalsociety.stacks.journals_flag.pic_1.bingo', 'tunic.historicalsociety.cage.lockeddoor', 'tunic.historicalsociety.stacks.journals_flag.pic_2.bingo', 'tunic.historicalsociety.collection.gramps.lost', 'tunic.historicalsociety.closet.notebook', 'tunic.historicalsociety.frontdesk.magnify', 'tunic.humanecology.frontdesk.businesscards.card_bingo.bingo', 'tunic.wildlife.center.remove_cup', 'tunic.library.frontdesk.wellsbadge.hub', 'tunic.wildlife.center.tracks.hub.deer', 'tunic.historicalsociety.frontdesk.key', 'tunic.library.microfiche.reader_flag.paper2.bingo', 'tunic.flaghouse.entry.colorbook', 'tunic.wildlife.center.coffee', 'tunic.capitol_1.hall.boss.haveyougotit', 'tunic.historicalsociety.basement.janitor', 'tunic.historicalsociety.collection_flag.gramps.recap', 'tunic.wildlife.center.wells.animals2', 'tunic.flaghouse.entry.flag_girl.symbol_recap', 'tunic.historicalsociety.closet_dirty.photo', 'tunic.historicalsociety.stacks.outtolunch', 'tunic.library.frontdesk.worker.wells_recap', 'tunic.historicalsociety.frontdesk.archivist_glasses.confrontation_recap', 'tunic.capitol_0.hall.boss.talktogramps', 'tunic.historicalsociety.closet.photo', 'tunic.historicalsociety.collection.tunic', 'tunic.historicalsociety.closet.teddy.intro_0_cs_5', 'tunic.historicalsociety.closet_dirty.gramps.archivist', 'tunic.historicalsociety.closet_dirty.door_block_talk', 'tunic.historicalsociety.entry.boss.flag_recap', 'tunic.historicalsociety.frontdesk.archivist.need_glass_0', 'tunic.historicalsociety.entry.wells.talktogramps', 'tunic.historicalsociety.frontdesk.block_magnify', 'tunic.historicalsociety.frontdesk.archivist.foundtheodora', 'tunic.historicalsociety.closet_dirty.gramps.nothing', 'tunic.historicalsociety.closet_dirty.door_block_clean', 'tunic.capitol_1.hall.boss.writeitup', 'tunic.library.frontdesk.worker.nelson_recap', 'tunic.library.frontdesk.worker.hello_short', 'tunic.historicalsociety.stacks.block', 'tunic.historicalsociety.frontdesk.archivist.need_glass_1', 'tunic.historicalsociety.entry.boss.talktogramps', 'tunic.historicalsociety.frontdesk.archivist.newspaper_recap', 'tunic.historicalsociety.entry.wells.flag_recap', 'tunic.drycleaner.frontdesk.worker.done2', 'tunic.library.frontdesk.worker.flag_recap', 'tunic.humanecology.frontdesk.block_0', 'tunic.library.frontdesk.worker.preflag', 'tunic.historicalsociety.basement.gramps.seeyalater', 'tunic.flaghouse.entry.flag_girl.hello_recap', 'tunic.historicalsociety.closet.doorblock', 'tunic.drycleaner.frontdesk.worker.takealook', 'tunic.historicalsociety.basement.gramps.whatdo', 'tunic.library.frontdesk.worker.droppedbadge', 'tunic.historicalsociety.entry.block_tomap2', 'tunic.library.frontdesk.block_nelson', 'tunic.library.microfiche.block_0', 'tunic.historicalsociety.entry.block_tocollection', 'tunic.historicalsociety.entry.block_tomap1', 'tunic.historicalsociety.collection.gramps.look_0', 'tunic.library.frontdesk.block_badge', 'tunic.historicalsociety.cage.need_glasses', 'tunic.library.frontdesk.block_badge_2', 'tunic.kohlcenter.halloffame.block_0', 'tunic.capitol_0.hall.chap1_finale_c', 'tunic.capitol_1.hall.chap2_finale_c', 'tunic.capitol_2.hall.chap4_finale_c', 'tunic.wildlife.center.fox.concern', 'tunic.drycleaner.frontdesk.block_0', 'tunic.historicalsociety.entry.gramps.hub', 'tunic.humanecology.frontdesk.block_1', 'tunic.drycleaner.frontdesk.block_1']\nroom_lists = ['tunic.historicalsociety.entry', 'tunic.wildlife.center', 'tunic.historicalsociety.cage', 'tunic.library.frontdesk', 'tunic.historicalsociety.frontdesk', 'tunic.historicalsociety.stacks', 'tunic.historicalsociety.closet_dirty', 'tunic.humanecology.frontdesk', 'tunic.historicalsociety.basement', 'tunic.kohlcenter.halloffame', 'tunic.library.microfiche', 'tunic.drycleaner.frontdesk', 'tunic.historicalsociety.collection', 'tunic.historicalsociety.closet', 'tunic.flaghouse.entry', 'tunic.historicalsociety.collection_flag', 'tunic.capitol_1.hall', 'tunic.capitol_0.hall', 'tunic.capitol_2.hall']","metadata":{"execution":{"iopub.execute_input":"2023-03-09T08:48:33.992879Z","iopub.status.busy":"2023-03-09T08:48:33.992460Z","iopub.status.idle":"2023-03-09T08:48:34.013655Z","shell.execute_reply":"2023-03-09T08:48:34.012380Z","shell.execute_reply.started":"2023-03-09T08:48:33.992846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Some few updates of https://www.kaggle.com/code/takanashihumbert/magic-bingo-train-part-lb-0-687\n\ncolumns = [\n\n    pl.col(\"page\").cast(pl.Float32),\n    (\n        (pl.col(\"elapsed_time\") - pl.col(\"elapsed_time\").shift(1)) \n         .fill_null(0)\n         .clip(0, 1e9)\n         .over([\"session_id\", \"level_group\"])\n         .alias(\"elapsed_time_diff\")\n    ),\n    (\n        (pl.col(\"screen_coor_x\") - pl.col(\"screen_coor_x\").shift(1)) \n         .abs()\n         .over([\"session_id\", \"level_group\"])\n        .alias(\"location_x_diff\") \n    ),\n    (\n        (pl.col(\"screen_coor_y\") - pl.col(\"screen_coor_y\").shift(1)) \n         .abs()\n         .over([\"session_id\", \"level_group\"])\n        .alias(\"location_y_diff\") \n    ),\n    pl.col(\"fqid\").fill_null(\"fqid_None\"),\n    pl.col(\"text_fqid\").fill_null(\"text_fqid_None\")\n]","metadata":{"execution":{"iopub.execute_input":"2023-03-09T08:48:34.861482Z","iopub.status.busy":"2023-03-09T08:48:34.861086Z","iopub.status.idle":"2023-03-09T08:48:34.870410Z","shell.execute_reply":"2023-03-09T08:48:34.868958Z","shell.execute_reply.started":"2023-03-09T08:48:34.861449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extract_q10(df):\n    text_from = 'What happened here?!'\n    text_to = \"He's our expert record keeper.\"\n\n    a_positions = df.with_columns(pl.col(\"index\").alias(\"index_from\")).filter(\n        pl.col(\"text\") == text_from)[['session_id', 'index_from']].unique(subset=[\"session_id\"])\n    b_positions = df.with_columns(pl.col(\"index\").alias(\"index_to\")).filter(\n        pl.col(\"text\") == text_to)[['session_id', 'index_to']].unique(subset=[\"session_id\"])\n    print(a_positions.shape, b_positions.shape)\n    print(a_positions.unique(subset=[\"session_id\"]).shape, b_positions.unique(\n        subset=[\"session_id\"]).shape)\n    text_from_to = a_positions.join(b_positions, on='session_id', how='inner')\n    print(text_from_to.shape)\n    log_in_range = df.join(text_from_to, on='session_id', how='inner').filter(\n        (pl.col('index') >= pl.col('index_from')) & (\n            pl.col('index') <= pl.col('index_to'))\n    )\n\n    return log_in_range.groupby('session_id').agg([\n        pl.col('elapsed_time').apply(\n            lambda s: s.max() - s.min()).alias('q10_duration'),\n        pl.col('event_name').filter(pl.col('event_name') ==\n                                    'notebook_click').count().alias('q10_notebook_count'),\n    ]).sort('session_id')\n\n\ndef extract_q15(df: pl.DataFrame):\n    # colorbookを開いた回数\n    open_book = df.filter(\n        (pl.col('event_name') == 'object_click') &\n        (pl.col('name') == 'close') &\n        (pl.col('fqid') == 'colorbook')\n    ).groupby('session_id').count()\n\n    # colorbookを開いた累計時間\n    tmp_1 = df.filter(\n        (\n            (pl.col('name') == 'close') |\n            (pl.col('event_name') == 'navigate_click')\n        ) &\n        (pl.col('fqid') == 'colorbook')\n    ).with_columns(\n        (pl.col(\"elapsed_time\") - pl.col(\"elapsed_time\").shift(1))\n        .fill_null(0)\n        .clip(0, 1e9)\n        .over([\"session_id\", \"level_group\"])\n        .alias(\"q15_elapsed_time_diff\")\n    ).groupby('session_id').agg(\n        [\n            pl.col(\"q15_elapsed_time_diff\").sum().alias(\n                \"q15_elapsed_time_sum\")]).join(open_book, on='session_id', how='left').with_columns(\n        pl.col('count').alias('q15_open_count')).drop(['count'])\n    return tmp_1\n\ndef get_a(x:pl.DataFrame):\n    lst = orig_lst\n    nt = x.filter(\n        (pl.col(\"level_group\") == \"5-12\") &\n        (pl.col(\"session_id\") != 21020618143279870) &\n        (pl.col(\"session_id\") != 22030509455473956)\n    )\n    st_teddy_kidnap = nt.filter(\n        (pl.col(\"text\") == \"Oh no!\") | (pl.col(\"text\") == \"What the-\") &\n        (pl.col(\"fqid\") == \"what_happened\")).unique(subset=[\"event_name\", \"fqid\", \"text\", \"session_id\"]).with_columns(\n        [pl.lit(False).alias(l) for l in lst]).with_columns(pl.lit(True).alias(\"st_teddy_kidnap\"))\n    end_teddy_kidnap = nt.filter(\n        (pl.col(\"text\") == \"He's our expert record keeper.\") &\n        (pl.col(\"fqid\") == \"gramps\")).unique(subset=[\"event_name\", \"fqid\", \"text\", \"session_id\"]).with_columns(\n        [pl.lit(False).alias(l) for l in lst]).with_columns(pl.lit(True).alias(\"end_teddy_kidnap\"))\n    st_dialogue_arch = nt.filter(\n        (pl.col(\"text\") == \"I need your help!\") &\n        (pl.col(\"fqid\") == \"archivist\")).unique(subset=[\"event_name\", \"fqid\", \"text\", \"session_id\"]).with_columns(\n        [pl.lit(False).alias(l) for l in lst]).with_columns(pl.lit(True).alias(\"st_dialogue_arch\"))\n    end_dialogue_arch = nt.filter(\n        (pl.col(\"text\") == \"Great! Thanks for the help!\") &\n        (pl.col(\"fqid\") == \"archivist\")).unique(subset=[\"event_name\", \"fqid\", \"text\", \"session_id\"]).with_columns(\n        [pl.lit(False).alias(l) for l in lst]).with_columns(pl.lit(True).alias(\"end_dialogue_arch\"))\n    st_dialogue_cloth = nt.filter(\n        (pl.col(\"text\") == \"Hello there!\") &\n        (pl.col(\"fqid\") == \"worker\")).unique(subset=[\"event_name\", \"fqid\", \"text\", \"session_id\"]).with_columns(\n        [pl.lit(False).alias(l) for l in lst]).with_columns(pl.lit(True).alias(\"st_dialogue_cloth\"))\n    # (\n    #     (pl.col(\"text\") == \"Why don't you take a look?\") &\n    #     (pl.col(\"fqid\") == \"worker\")) |\n    st_tag_bingo_cloth = nt.filter(\n        (pl.col(\"event_name\") == \"navigate_click\") &\n        (pl.col(\"fqid\") == \"businesscards\")).unique(subset=[\"event_name\", \"fqid\", \"text\", \"session_id\"]).with_columns(\n        [pl.lit(False).alias(l) for l in lst]).with_columns(pl.lit(True).alias(\"st_tag_bingo_cloth\"))\n    find_tag_bingo_cloth = nt.filter(\n        (pl.col(\"text\") == \"This place was around in 1916! I can start there!\")).unique(subset=[\"event_name\", \"fqid\", \"text\", \"session_id\"]).with_columns(\n        [pl.lit(False).alias(l) for l in lst]).with_columns(pl.lit(True).alias(\"find_tag_bingo_cloth\"))\n    end_tag_bingo_cloth = nt.filter(\n        (pl.col(\"name\") == \"close\") &\n        (pl.col(\"fqid\") == \"businesscards\")).unique(subset=[\"event_name\", \"fqid\", \"text\", \"session_id\"]).with_columns(\n        [pl.lit(False).alias(l) for l in lst]).with_columns(pl.lit(True).alias(\"end_tag_bingo_cloth\"))\n    # (\n    #     ((pl.col(\"text\") == \"Okay. Thanks anyway.\") | (pl.col(\"text\") == \"Yeah. Thanks anyway.\")) &\n    #     (pl.col(\"fqid\") == \"worker\")) |\n    st_dialogue_clean = nt.filter(\n        ((pl.col(\"text\") == \"Hi! How can I help you?\") | (pl.col(\"text\") == \"Hi! *cough*\")) &\n        (pl.col(\"fqid\") == \"worker\")).unique(subset=[\"event_name\", \"fqid\", \"text\", \"session_id\"]).with_columns(\n        [pl.lit(False).alias(l) for l in lst]).with_columns(pl.lit(True).alias(\"st_dialogue_clean\"))\n    st_tag_bingo_clean = nt.filter(\n        (pl.col(\"text\") == \"Here's the log book.\") &\n        (pl.col(\"fqid\") == \"worker\")).unique(subset=[\"event_name\", \"fqid\", \"text\", \"session_id\"]).with_columns(\n        [pl.lit(False).alias(l) for l in lst]).with_columns(pl.lit(True).alias(\"st_tag_bingo_clean\"))\n    find_tag_bingo_clean = nt.filter(\n        (pl.col(\"text\") == \"It's a match!\")).unique(subset=[\"event_name\", \"fqid\", \"text\", \"session_id\"]).with_columns(\n        [pl.lit(False).alias(l) for l in lst]).with_columns(pl.lit(True).alias(\"find_tag_bingo_clean\"))\n    end_tag_bingo_clean = nt.filter(\n        (pl.col(\"text\") == \"Thanks for the help!\") &\n        (pl.col(\"fqid\") == \"worker\")).unique(subset=[\"event_name\", \"fqid\", \"text\", \"session_id\"]).with_columns(\n        [pl.lit(False).alias(l) for l in lst]).with_columns(pl.lit(True).alias(\"end_tag_bingo_clean\"))\n    # (\n    #     (pl.col(\"text\") == \"Oh, hello there!\") &\n    #     (pl.col(\"fqid\") == \"worker\")) |\n    st_paper_bingo = nt.filter(\n        (pl.col(\"event_name\") == \"navigate_click\") &\n        (pl.col(\"fqid\") == \"reader\")).unique(subset=[\"event_name\", \"fqid\", \"text\", \"session_id\"]).with_columns(\n        [pl.lit(False).alias(l) for l in lst]).with_columns(pl.lit(True).alias(\"st_paper_bingo\"))\n    find_paper_bingo = nt.filter(\n        (pl.col(\"event_name\") == \"object_click\") &\n        (pl.col(\"fqid\") == \"reader.paper2.bingo\")).unique(subset=[\"event_name\", \"fqid\", \"text\", \"session_id\"]).with_columns(\n        [pl.lit(False).alias(l) for l in lst]).with_columns(pl.lit(True).alias(\"find_paper_bingo\"))\n    end_paper_bingo = nt.filter(\n        (pl.col(\"name\") == \"close\") &\n        (pl.col(\"fqid\") == \"reader\")).unique(subset=[\"event_name\", \"fqid\", \"text\", \"session_id\"]).with_columns(\n        [pl.lit(False).alias(l) for l in lst]).with_columns(pl.lit(True).alias(\"end_paper_bingo\"))\n    find_wells_photo = nt.filter(\n        (pl.col(\"text\") == \"Wells! What was he doing here? I should ask the librarian.\")).unique(subset=[\"event_name\", \"fqid\", \"text\", \"session_id\"]).with_columns(\n        [pl.lit(False).alias(l) for l in lst]).with_columns(pl.lit(True).alias(\"find_wells_photo\"))\n    end_library = nt.filter(\n        (pl.col(\"text\") == \"You could ask the archivist. He knows everybody!\") &\n        (pl.col(\"fqid\") == \"worker\")).unique(subset=[\"event_name\", \"fqid\", \"text\", \"session_id\"]).with_columns(\n        [pl.lit(False).alias(l) for l in lst]).with_columns(pl.lit(True).alias(\"end_library\"))\n    st_dialogue_arch_2 = nt.filter(\n        ((pl.col(\"text\") == \"Can you help me? I need to find Wells!\") | (pl.col(\"text\") == \"I need to find Wells!!!\")) &\n        (pl.col(\"fqid\") == \"archivist\")).unique(subset=[\"event_name\", \"fqid\", \"text\", \"session_id\"]).with_columns(\n        [pl.lit(False).alias(l) for l in lst]).with_columns(pl.lit(True).alias(\"st_dialogue_arch_2\"))\n    st_jornals_bingo = nt.filter(\n        (pl.col(\"event_name\") == \"object_click\") &\n        (pl.col(\"fqid\") == \"journals\")).unique(subset=[\"event_name\", \"fqid\", \"text\", \"session_id\"]).with_columns(\n        [pl.lit(False).alias(l) for l in lst]).with_columns(pl.lit(True).alias(\"st_jornals_bingo\"))\n    find_jornals_bingo = nt.filter(\n        (pl.col(\"text\") == \"Hey, this is Youmans!\")).unique(subset=[\"event_name\", \"fqid\", \"text\", \"session_id\"]).with_columns(\n        [pl.lit(False).alias(l) for l in lst]).with_columns(pl.lit(True).alias(\"find_jornals_bingo\"))\n    k = pl.concat([\n        st_teddy_kidnap,\n        end_teddy_kidnap,\n        st_dialogue_arch,\n        end_dialogue_arch,\n        st_dialogue_cloth,\n        st_tag_bingo_cloth,\n        find_tag_bingo_cloth,\n        end_tag_bingo_cloth,\n        st_dialogue_clean,\n        st_tag_bingo_clean,\n        find_tag_bingo_clean,\n        end_tag_bingo_clean,\n        st_paper_bingo,\n        find_paper_bingo,\n        end_paper_bingo,\n        find_wells_photo,\n        end_library,\n        st_dialogue_arch_2,\n        st_jornals_bingo,\n        find_jornals_bingo\n    ]).sort([\"session_id\", \"index\"])  # .filter(pl.col(\"session_id\") == 20110215014150144)\n\n\n    # .unique(subset=[\"event_name\", \"fqid\", \"text\", \"session_id\"]).groupby(\"session_id\")\n    k = k.with_columns(\n        [\n            (pl.col(\"elapsed_time\") - pl.col(\"elapsed_time\").shift(1))\n            .fill_null(0)\n            .over([\"session_id\", \"level_group\"])\n            .alias(\"elapsed_time_diff\"),\n            (pl.col(\"index\") - pl.col(\"index\").shift(1))\n            .fill_null(0)\n            .over([\"session_id\", \"level_group\"])\n            .alias(\"index_diff\"),\n        ]\n    ).with_columns(\n        [\n            *[pl.when(pl.col(l)).then(pl.col(\"elapsed_time_diff\")\n                                    ).otherwise(pl.lit(None)).alias(f\"elapsed_diff_{l}\") for l in lst],\n            *[pl.when(pl.col(l)).then(pl.col(\"index_diff\")\n                                    ).otherwise(pl.lit(None)).alias(f\"index_diff_{l}\") for l in lst],\n        ]\n    )[[\"session_id\"]+[f\"elapsed_diff_{l}\" for l in lst] + [f\"index_diff_{l}\" for l in lst]]\n\n\n    a = None\n    for l in lst[1:]:\n        if a is None:\n            a = k.filter(pl.col(f\"elapsed_diff_{l}\").is_not_null())[\n                [\"session_id\", f\"elapsed_diff_{l}\", f\"index_diff_{l}\"]]\n        else:\n            a = a.join(k.filter(pl.col(f\"elapsed_diff_{l}\").is_not_null())[\n                    [\"session_id\", f\"elapsed_diff_{l}\", f\"index_diff_{l}\"]], on=\"session_id\")\n    return a\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineer_pl(x, grp, use_extra, feature_suffix):\n        \n    aggs = [\n        pl.col(\"index\").count().alias(f\"session_number_{feature_suffix}\"),\n      \n        *[pl.col(c).drop_nulls().n_unique().alias(f\"{c}_unique_{feature_suffix}\") for c in CATS],\n        *[pl.col(c).quantile(0.1, \"nearest\").alias(f\"{c}_quantile1_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).quantile(0.2, \"nearest\").alias(f\"{c}_quantile2_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).quantile(0.4, \"nearest\").alias(f\"{c}_quantile4_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).quantile(0.6, \"nearest\").alias(f\"{c}_quantile6_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).quantile(0.8, \"nearest\").alias(f\"{c}_quantile8_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).quantile(0.9, \"nearest\").alias(f\"{c}_quantile9_{feature_suffix}\") for c in NUMS],\n        \n        *[pl.col(c).mean().alias(f\"{c}_mean_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).std().alias(f\"{c}_std_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).min().alias(f\"{c}_min_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).max().alias(f\"{c}_max_{feature_suffix}\") for c in NUMS],\n        \n        *[pl.col(\"event_name\").filter(pl.col(\"event_name\") == c).count().alias(f\"{c}_event_name_counts{feature_suffix}\")for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\")==c).quantile(0.1, \"nearest\").alias(f\"{c}_ET_quantile1_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\")==c).quantile(0.2, \"nearest\").alias(f\"{c}_ET_quantile2_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\")==c).quantile(0.4, \"nearest\").alias(f\"{c}_ET_quantile4_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\")==c).quantile(0.6, \"nearest\").alias(f\"{c}_ET_quantile6_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\")==c).quantile(0.8, \"nearest\").alias(f\"{c}_ET_quantile8_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\")==c).quantile(0.9, \"nearest\").alias(f\"{c}_ET_quantile9_{feature_suffix}\") for c in event_name_feature],      \n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\")==c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\")==c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\")==c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\")==c).min().alias(f\"{c}_ET_min_{feature_suffix}\") for c in event_name_feature],\n     \n        *[pl.col(\"name\").filter(pl.col(\"name\") == c).count().alias(f\"{c}_name_counts{feature_suffix}\")for c in name_feature],   \n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\")==c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for c in name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\")==c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for c in name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\")==c).min().alias(f\"{c}_ET_min_{feature_suffix}\") for c in name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\")==c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for c in name_feature],  \n        \n        *[pl.col(\"room_fqid\").filter(pl.col(\"room_fqid\") == c).count().alias(f\"{c}_room_fqid_counts{feature_suffix}\")for c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).min().alias(f\"{c}_ET_min_{feature_suffix}\") for c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for c in room_lists],\n                \n        *[pl.col(\"fqid\").filter(pl.col(\"fqid\") == c).count().alias(f\"{c}_fqid_counts{feature_suffix}\")for c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).min().alias(f\"{c}_ET_min_{feature_suffix}\") for c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for c in fqid_lists],\n       \n        *[pl.col(\"text_fqid\").filter(pl.col(\"text_fqid\") == c).count().alias(f\"{c}_text_fqid_counts{feature_suffix}\") for c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).min().alias(f\"{c}_ET_min_{feature_suffix}\") for c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for c in text_lists],\n         \n        *[pl.col(\"location_x_diff\").filter(pl.col(\"event_name\")==c).mean().alias(f\"{c}_ET_mean_x{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"location_x_diff\").filter(pl.col(\"event_name\")==c).std().alias(f\"{c}_ET_std_x{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"location_x_diff\").filter(pl.col(\"event_name\")==c).max().alias(f\"{c}_ET_max_x{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"location_x_diff\").filter(pl.col(\"event_name\")==c).min().alias(f\"{c}_ET_min_x{feature_suffix}\") for c in event_name_feature],\n        ]\n    \n    df = x.groupby([\"session_id\"], maintain_order=True).agg(aggs).sort(\"session_id\")\n  \n    if use_extra:\n        if grp=='5-12':\n            aggs = [\n                pl.col(\"elapsed_time\").filter((pl.col(\"text\")==\"Here's the log book.\")|(pl.col(\"fqid\")=='logbook.page.bingo')).apply(lambda s: s.max()-s.min()).alias(\"logbook_bingo_duration\"),\n                pl.col(\"index\").filter((pl.col(\"text\")==\"Here's the log book.\")|(pl.col(\"fqid\")=='logbook.page.bingo')).apply(lambda s: s.max()-s.min()).alias(\"logbook_bingo_indexCount\"),\n                pl.col(\"elapsed_time\").filter(((pl.col(\"event_name\")=='navigate_click')&(pl.col(\"fqid\")=='reader'))|(pl.col(\"fqid\")==\"reader.paper2.bingo\")).apply(lambda s: s.max()-s.min()).alias(\"reader_bingo_duration\"),\n                pl.col(\"index\").filter(((pl.col(\"event_name\")=='navigate_click')&(pl.col(\"fqid\")=='reader'))|(pl.col(\"fqid\")==\"reader.paper2.bingo\")).apply(lambda s: s.max()-s.min()).alias(\"reader_bingo_indexCount\"),\n                pl.col(\"elapsed_time\").filter(((pl.col(\"event_name\")=='navigate_click')&(pl.col(\"fqid\")=='journals'))|(pl.col(\"fqid\")==\"journals.pic_2.bingo\")).apply(lambda s: s.max()-s.min()).alias(\"journals_bingo_duration\"),\n                pl.col(\"index\").filter(((pl.col(\"event_name\")=='navigate_click')&(pl.col(\"fqid\")=='journals'))|(pl.col(\"fqid\")==\"journals.pic_2.bingo\")).apply(lambda s: s.max()-s.min()).alias(\"journals_bingo_indexCount\"),\n            ]\n            tmp = x.groupby([\"session_id\"], maintain_order=True).agg(aggs).sort(\"session_id\")\n            df = df.join(tmp, on=\"session_id\", how='left')\n            \n            tmp_ = extract_q10(x)\n            df = df.join(tmp_, on=\"session_id\", how='left')\n\n\n        if grp=='13-22':\n            aggs = [\n                pl.col(\"elapsed_time\").filter(((pl.col(\"event_name\")=='navigate_click')&(pl.col(\"fqid\")=='reader_flag'))|(pl.col(\"fqid\")==\"tunic.library.microfiche.reader_flag.paper2.bingo\")).apply(lambda s: s.max()-s.min() if s.len()>0 else 0).alias(\"reader_flag_duration\"),\n                pl.col(\"index\").filter(((pl.col(\"event_name\")=='navigate_click')&(pl.col(\"fqid\")=='reader_flag'))|(pl.col(\"fqid\")==\"tunic.library.microfiche.reader_flag.paper2.bingo\")).apply(lambda s: s.max()-s.min() if s.len()>0 else 0).alias(\"reader_flag_indexCount\"),\n                pl.col(\"elapsed_time\").filter(((pl.col(\"event_name\")=='navigate_click')&(pl.col(\"fqid\")=='journals_flag'))|(pl.col(\"fqid\")==\"journals_flag.pic_0.bingo\")).apply(lambda s: s.max()-s.min() if s.len()>0 else 0).alias(\"journalsFlag_bingo_duration\"),\n                pl.col(\"index\").filter(((pl.col(\"event_name\")=='navigate_click')&(pl.col(\"fqid\")=='journals_flag'))|(pl.col(\"fqid\")==\"journals_flag.pic_0.bingo\")).apply(lambda s: s.max()-s.min() if s.len()>0 else 0).alias(\"journalsFlag_bingo_indexCount\"),\n            ]\n            tmp = x.groupby([\"session_id\"], maintain_order=True).agg(aggs).sort(\"session_id\")\n            df = df.join(tmp, on=\"session_id\", how='left')\n\n            tmp_ = extract_q15(x)\n            df = df.join(tmp_, on=\"session_id\", how='left')\n        \n    return df.to_pandas()","metadata":{"execution":{"iopub.execute_input":"2023-03-09T08:48:36.595526Z","iopub.status.busy":"2023-03-09T08:48:36.595154Z","iopub.status.idle":"2023-03-09T08:48:36.648986Z","shell.execute_reply":"2023-03-09T08:48:36.647730Z","shell.execute_reply.started":"2023-03-09T08:48:36.595496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# we prepare the dataset for the training by level :\ndf = (pl.read_csv(\"./train.csv\")\n      .drop([\"fullscreen\", \"hq\", \"music\"])\n      .with_columns(columns))\n\ndf1 = df.filter(pl.col(\"level_group\")=='0-4')\ndf2 = df.filter(pl.col(\"level_group\")=='5-12')\ndf3 = df.filter(pl.col(\"level_group\")=='13-22')\ndf1.shape,df2.shape,df3.shape","metadata":{"execution":{"iopub.execute_input":"2023-03-09T08:48:38.296702Z","iopub.status.busy":"2023-03-09T08:48:38.296301Z","iopub.status.idle":"2023-03-09T08:49:11.450039Z","shell.execute_reply":"2023-03-09T08:49:11.449038Z","shell.execute_reply.started":"2023-03-09T08:48:38.296668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df\ngc.collect()","metadata":{"execution":{"iopub.execute_input":"2023-03-09T08:49:20.576708Z","iopub.status.busy":"2023-03-09T08:49:20.576275Z","iopub.status.idle":"2023-03-09T08:49:20.982553Z","shell.execute_reply":"2023-03-09T08:49:20.981334Z","shell.execute_reply.started":"2023-03-09T08:49:20.576675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf1 = feature_engineer_pl(df1, grp='0-4', use_extra=True, feature_suffix='')\nprint('df1 done',df1.shape)\ndf2 = feature_engineer_pl(df2, grp='5-12', use_extra=True, feature_suffix='')\nprint('df2 done',df2.shape)\ndf3 = feature_engineer_pl(df3, grp='13-22', use_extra=True, feature_suffix='')\nprint('df3 done',df3.shape)","metadata":{"execution":{"iopub.execute_input":"2023-03-09T08:49:23.040132Z","iopub.status.busy":"2023-03-09T08:49:23.039736Z","iopub.status.idle":"2023-03-09T08:50:36.940493Z","shell.execute_reply":"2023-03-09T08:50:36.939162Z","shell.execute_reply.started":"2023-03-09T08:49:23.040100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# some cleaning...\nnull1 = df1.isnull().sum().sort_values(ascending=False) / len(df1)\nnull2 = df2.isnull().sum().sort_values(ascending=False) / len(df1)\nnull3 = df3.isnull().sum().sort_values(ascending=False) / len(df1)\n\ndrop1 = list(null1[null1>0.9].index)\ndrop2 = list(null2[null2>0.9].index)\ndrop3 = list(null3[null3>0.9].index)\nprint(len(drop1), len(drop2), len(drop3))\n\nfor col in df1.columns:\n    if df1[col].nunique()==1:\n        print(col)\n        drop1.append(col)\nprint(\"*********df1 DONE*********\")\nfor col in df2.columns:\n    if df2[col].nunique()==1:\n        print(col)\n        drop2.append(col)\nprint(\"*********df2 DONE*********\")\nfor col in df3.columns:\n    if df3[col].nunique()==1:\n        print(col)\n        drop3.append(col)\nprint(\"*********df3 DONE*********\")","metadata":{"execution":{"iopub.execute_input":"2023-03-09T08:50:40.903764Z","iopub.status.busy":"2023-03-09T08:50:40.901998Z","iopub.status.idle":"2023-03-09T08:50:42.863768Z","shell.execute_reply":"2023-03-09T08:50:42.862612Z","shell.execute_reply.started":"2023-03-09T08:50:40.903620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 = df1.set_index('session_id')\ndf2 = df2.set_index('session_id')\ndf3 = df3.set_index('session_id')\n\nFEATURES1 = [c for c in df1.columns if c not in drop1+['level_group']]\nFEATURES2_10 = [c for c in df2.columns if c not in drop2+['level_group']]\nFEATURES2 = [c for c in df2.columns if c not in drop2 +\n             ['level_group', 'q10_duration', 'q10_notebook_count']]\nFEATURES3_15 = [c for c in df3.columns if c not in drop3+['level_group']]\nFEATURES3 = [c for c in df3.columns if c not in drop3 +\n             ['level_group', 'q15_elapsed_time_sum', 'q15_open_count']]\nprint('We will train with', len(FEATURES1), len(FEATURES2), len(FEATURES3) ,'features')\nALL_USERS = df1.index.unique()\nprint('We will train with', len(ALL_USERS) ,'users info')","metadata":{"execution":{"iopub.execute_input":"2023-03-09T08:50:59.881143Z","iopub.status.busy":"2023-03-09T08:50:59.880754Z","iopub.status.idle":"2023-03-09T08:51:00.222072Z","shell.execute_reply":"2023-03-09T08:51:00.221064Z","shell.execute_reply.started":"2023-03-09T08:50:59.881110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FEATURES3_15\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_list = [df1, df2, df3]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# With previous training notebook (Kfold with 20 folds as performed in others notebooks) :\nestimators_xgb = [498, 448, 378, 364, 405, 495, 456, 249, 384, 405, 356, 262, 484, 381, 392, 248 ,248, 345]","metadata":{"execution":{"iopub.execute_input":"2023-03-09T08:51:17.083216Z","iopub.status.busy":"2023-03-09T08:51:17.082837Z","iopub.status.idle":"2023-03-09T08:51:17.088280Z","shell.execute_reply":"2023-03-09T08:51:17.087435Z","shell.execute_reply.started":"2023-03-09T08:51:17.083187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xgb_params = {\n        'booster': 'gbtree',\n        'tree_method': 'gpu_hist',\n        'objective': 'binary:logistic',\n        'eval_metric':'logloss',\n        'learning_rate': 0.02,\n        'alpha': 8,\n        'max_depth': 4,\n        'subsample':0.8,\n        'colsample_bytree': 0.5,\n        'seed': 42\n        }","metadata":{"execution":{"iopub.execute_input":"2023-03-09T08:51:17.566919Z","iopub.status.busy":"2023-03-09T08:51:17.566517Z","iopub.status.idle":"2023-03-09T08:51:17.573686Z","shell.execute_reply":"2023-03-09T08:51:17.572743Z","shell.execute_reply.started":"2023-03-09T08:51:17.566886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# gkf = GroupKFold(n_splits=20)\noof = pd.DataFrame(data=np.zeros((len(ALL_USERS),18)), index=ALL_USERS)\nmodels = {}\nlimits = {'0-4': (1, 4), '5-12': (4, 14), '13-22': (14, 19)}\n\ntargets = pd.read_csv('./train_labels.csv')\ntargets['session'] = targets.session_id.apply(lambda x: int(x.split('_')[0]))\ntargets['q'] = targets.session_id.apply(lambda x: int(x.split('_')[-1][1:]))\n\n# COMPUTE CV SCORE WITH 20 GROUP K FOLD\n\n# for 0-4 group training\nfor g_num, (a,b) in enumerate(limits.values()):\n    print(f\"for group {a}-{b}\")\n    df = df_list[g_num]\n    gkf = GroupKFold(n_splits=20)\n    for i, (train_index, test_index) in enumerate(gkf.split(X=df, groups=df.index)):\n        print('#'*25)\n        print('### Fold', i+1)\n        print('#'*25)\n\n\n        # ITERATE THRU QUESTIONS 1 THRU 18\n        for t in range(a, b):\n\n            # USE THIS TRAIN DATA WITH THESE QUESTIONS\n            if t <= 3:\n                grp = '0-4'\n                FEATURES = FEATURES1\n            elif t <= 13:\n                grp = '5-12'\n                if t == 10:\n                    FEATURES = FEATURES2_10\n                else:\n                    FEATURES = FEATURES2\n            elif t <= 22:\n                grp = '13-22'\n                if t==15:\n                    FEATURES = FEATURES3_15\n                else:\n                    FEATURES = FEATURES3\n\n            # TRAIN DATA\n            train_x = df.iloc[train_index]\n            # train_x = train_x.loc[train_x.level_group == grp]\n            train_users = train_x.index.values\n            train_y = targets.loc[targets.q == t].set_index(\n                'session').loc[train_users]\n\n            # VALID DATA\n            valid_x = df.iloc[test_index]\n            # valid_x = valid_x.loc[valid_x.level_group == grp]\n            valid_users = valid_x.index.values\n            valid_y = targets.loc[targets.q == t].set_index(\n                'session').loc[valid_users]\n\n            # TRAIN MODEL\n            clf = XGBClassifier(**xgb_params)\n            clf.fit(train_x[FEATURES].astype('float32'), train_y['correct'],\n                    eval_set=[(valid_x[FEATURES].astype(\n                        'float32'), valid_y['correct'])],\n                    verbose=0)\n            print(f'{t}({clf.best_ntree_limit}), ', end='')\n\n            # SAVE MODEL, PREDICT VALID OOF\n            models[f'{grp}_{t}'] = clf\n            oof.loc[valid_users, t -\n                    1] = clf.predict_proba(valid_x[FEATURES].astype('float32'))[:, 1]\n\n        print()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Compute CV Score","metadata":{}},{"cell_type":"code","source":"# PUT TRUE LABELS INTO DATAFRAME WITH 18 COLUMNS\ntrue = oof.copy()\nfor k in range(18):\n    # GET TRUE LABELS\n    tmp = targets.loc[targets.q == k+1].set_index('session').loc[ALL_USERS]\n    true[k] = tmp.correct.values\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# FIND BEST THRESHOLD TO CONVERT PROBS INTO 1s AND 0s\nscores = []\nthresholds = []\nbest_score = 0\nbest_threshold = 0\n\nfor threshold in np.arange(0.4, 0.81, 0.01):\n    print(f'{threshold:.02f}, ', end='')\n    preds = (oof.values.reshape((-1)) > threshold).astype('int')\n    m = f1_score(true.values.reshape((-1)), preds, average='macro')\n    scores.append(m)\n    thresholds.append(threshold)\n    if m > best_score:\n        best_score = m\n        best_threshold = threshold\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# PLOT THRESHOLD VS. F1_SCORE\nplt.figure(figsize=(20, 5))\nplt.plot(thresholds, scores, '-o', color='blue')\nplt.scatter([best_threshold], [best_score], color='blue', s=300, alpha=1)\nplt.xlabel('Threshold', size=14)\nplt.ylabel('Validation F1 Score', size=14)\nplt.title(\n    f'Threshold vs. F1_Score with Best F1_Score = {best_score:.3f} at Best Threshold = {best_threshold:.3}', size=18)\nplt.show()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('When using optimal threshold...')\nfor k in range(18):\n        \n    # COMPUTE F1 SCORE PER QUESTION\n    if k in [1,2,11,17]:\n        m = f1_score(true[k].values, np.ones_like(\n            oof[k].values), average='macro')\n    elif k in []:\n        m = f1_score(true[k].values, np.zeros_like(\n            oof[k].values), average='macro')\n    else:\n        m = f1_score(true[k].values, (oof[k].values>best_threshold).astype('int'), average='macro')\n    print(f'Q{k+1}: F1 =',m)\n    \n# # COMPUTE F1 SCORE OVERALL\n# tmp = (oof.values.reshape((-1)) > best_threshold).astype('int')\n# tmp\nm = f1_score(true.values.reshape((-1)), (oof.values.reshape((-1))>best_threshold).astype('int'), average='macro')\nprint('==> Overall F1 =',m)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## We fit and store the models for predictions","metadata":{}},{"cell_type":"code","source":"warnings.filterwarnings(\"ignore\")\ntargets = pd.read_csv('./train_labels.csv')\ntargets['session'] = targets.session_id.apply(lambda x: int(x.split('_')[0]))\ntargets['q'] = targets.session_id.apply(lambda x: int(x.split('_')[-1][1:]))\npred_xgb = np.zeros((df1.shape[0],18))     \n\nfor t in range(1,19):\n    # USE THIS TRAIN DATA WITH THESE QUESTIONS\n    if t<=3: \n        grp = '0-4'\n        df = df1\n        FEATURES = FEATURES1\n\n    elif t<=13: \n        grp = '5-12'\n        df = df2\n        FEATURES = FEATURES2\n\n    elif t<=22: \n        grp = '13-22'\n        df = df3\n        FEATURES = FEATURES3\n        \n    xgb_params['n_estimators'] = estimators_xgb[t-1]\n     \n    # TRAIN DATA\n    train_users = df.index.values\n    train_y = targets.loc[targets.q==t].set_index('session').loc[train_users] \n\n    clf =  XGBClassifier(**xgb_params)\n    clf.fit(df[FEATURES].astype('float32'), train_y['correct'], verbose = 0)\n    clf.save_model(f'XGB_question{t}.xgb')\n    \n    print(f'model XGB saved for question {t} with iterations = {estimators_xgb[t-1]}')","metadata":{"execution":{"iopub.execute_input":"2023-03-09T08:52:59.140013Z","iopub.status.busy":"2023-03-09T08:52:59.139585Z","iopub.status.idle":"2023-03-09T08:53:25.873674Z","shell.execute_reply":"2023-03-09T08:53:25.872331Z","shell.execute_reply.started":"2023-03-09T08:52:59.139980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"import jo_wilder\nenv = jo_wilder.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.execute_input":"2023-03-08T09:43:43.712196Z","iopub.status.busy":"2023-03-08T09:43:43.711704Z","iopub.status.idle":"2023-03-08T09:43:43.724635Z","shell.execute_reply":"2023-03-08T09:43:43.723490Z","shell.execute_reply.started":"2023-03-08T09:43:43.712149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"limits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\ncount = 0\n\nfor (sample_submission, test) in iter_test:\n        \n        session_id = test.session_id.values[0]\n        grp = test.level_group.values[0]\n        a,b = limits[grp]\n  \n\n        # ------------------- level 0-4 ---------------------------------\n        if a == 1:\n            FEATURES = FEATURES1\n            test = (pl.from_pandas(test)\n                  .drop([\"fullscreen\", \"hq\", \"music\"])\n                  .with_columns(columns))\n            test = feature_engineer_pl(test, grp, use_extra=True, feature_suffix='')\n            test = test[FEATURES]\n            \n\n        # ------------------- level 5-12 ---------------------------------\n        elif a == 4:\n            FEATURES = FEATURES2\n            test = (pl.from_pandas(test)\n                  .drop([\"fullscreen\", \"hq\", \"music\"])\n                  .with_columns(columns))\n            test = feature_engineer_pl(test, grp, use_extra=True, feature_suffix='')\n            test = test[FEATURES]\n\n        # ------------------- level 13-22 ---------------------------------    \n        elif a == 14:\n            FEATURES = FEATURES3\n            test = (pl.from_pandas(test)\n                  .drop([\"fullscreen\", \"hq\", \"music\"])\n                  .with_columns(columns))\n            test = feature_engineer_pl(test, grp, use_extra=True, feature_suffix='')\n            test = test[FEATURES]\n    \n        for t in range(a,b):\n\n            clf = XGBClassifier()\n            clf.load_model(f'/kaggle/working/XGB_question{t}.xgb')\n\n            mask = sample_submission.session_id.str.contains(f'q{t}')\n            p = clf.predict_proba(test.astype('float32'))[:,1]\n            sample_submission.loc[mask,'correct'] = int((p.item())>0.625)  \n                \n        env.predict(sample_submission)","metadata":{"execution":{"iopub.execute_input":"2023-03-08T09:44:07.620889Z","iopub.status.busy":"2023-03-08T09:44:07.620496Z","iopub.status.idle":"2023-03-08T09:44:09.685449Z","shell.execute_reply":"2023-03-08T09:44:09.684528Z","shell.execute_reply.started":"2023-03-08T09:44:07.620857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('submission.csv').head(10)","metadata":{"execution":{"iopub.execute_input":"2023-03-08T09:44:09.692186Z","iopub.status.busy":"2023-03-08T09:44:09.690039Z","iopub.status.idle":"2023-03-08T09:44:09.704336Z","shell.execute_reply":"2023-03-08T09:44:09.703172Z","shell.execute_reply.started":"2023-03-08T09:44:09.692149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DEBUG","metadata":{}},{"cell_type":"markdown","source":"### Usefull when you make updates","metadata":{}},{"cell_type":"code","source":"\"\"\"\ntest = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/test.csv')\ntest = test.drop('session_level', axis=1)\ntest_level1 = test[test['level_group'] == '0-4']\ntest_level2 = test[test['level_group'] == '5-12']\ntest_level3 = test[test['level_group'] == '13-22']\nsample_submission = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/sample_submission.csv')\n\nlimits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\n\nfor test_level in [test_level1,test_level2,test_level3]:\n    l = np.unique(test_level['level_group']).tolist()\n    print(f'\\n***************** level = {l}*******************\\n')\n    for session_id in [20090109393214576,20090312143683264,20090312331414616]:\n        print(f'--------- {session_id} ---------')\n        \n        test_level_session = test_level[test_level['session_id']==session_id]          \n        #display(test_level_session.head(2))\n        #------------------------------------\n        #grp = test.level_group.values[0]\n        grp = l[0] \n        a,b = limits[grp]\n        #------------------------------------           \n        columns = [\n\n            pl.col(\"page\").cast(pl.Float32),\n            (\n                (pl.col(\"elapsed_time\") - pl.col(\"elapsed_time\").shift(1)) \n                 .fill_null(0)\n                 .clip(0, 1e9)\n                 .over([\"session_id\", \"level_group\"])\n                 .alias(\"elapsed_time_diff\")\n            ),\n            (\n                (pl.col(\"screen_coor_x\") - pl.col(\"screen_coor_x\").shift(1)) \n                 .abs()\n                 .over([\"session_id\", \"level_group\"])\n                .alias(\"location_x_diff\") \n            ),\n            (\n                (pl.col(\"screen_coor_y\") - pl.col(\"screen_coor_y\").shift(1)) \n                 .abs()\n                 .over([\"session_id\", \"level_group\"])\n                .alias(\"location_y_diff\") \n            ),\n            pl.col(\"fqid\").fill_null(\"fqid_None\"),\n            pl.col(\"text_fqid\").fill_null(\"text_fqid_None\")\n        ]\n\n        # ------------------- level 0-4 ---------------------------------\n        if a == 1:\n            print('GRP =',grp)\n            FEATURES = FEATURES1\n            \n            test = (pl.from_pandas(test_level.drop([\"fullscreen\", \"hq\", \"music\"],axis=1))\n                  .with_columns(columns))\n            test = feature_engineer_pl(test, grp, use_extra=True, feature_suffix='')\n            test = test[FEATURES]\n            test = test.fillna(-1)\n            level = 3\n            w = 0\n            print('test shape',test_level.shape)\n            print('level =',level)\n          \n            scaler = scaler_0\n\n        # ------------------- level 5-12 ---------------------------------\n        elif a == 4:\n            print('GRP =',grp)\n            FEATURES = FEATURES2\n            test = (pl.from_pandas(test_level.drop([\"fullscreen\", \"hq\", \"music\"],axis=1))\n                  .with_columns(columns))\n            test = feature_engineer_pl(test, grp, use_extra=True, feature_suffix='')\n            test = test[FEATURES]\n            test = test.fillna(-1)\n            print('test shape',test.shape) # **********************\n            level = 10 \n            print('level =',level)\n            scaler = scaler_1\n\n        # ------------------- level 13-22 ---------------------------------    \n        elif a == 14:\n            print('GRP =',grp)\n            FEATURES = FEATURES3\n            test = (pl.from_pandas(test_level.drop([\"fullscreen\", \"hq\", \"music\"],axis=1))\n                  .with_columns(columns))\n            test = feature_engineer_pl(test, grp, use_extra=True, feature_suffix='')\n            test = test[FEATURES]\n            test = test.fillna(-1)\n            print('test shape',test.shape) # **********************\n            \n            level = 5\n            print('level =',level)\n            scaler = scaler_2\n         \n\n        # ---------- Predictions for the session_id and the level_group -------------    \n\n        X_test = scaler.transform(test)\n        #X_test = scaler.transform(test)\n        \n        X_test = torch.from_numpy(X_test.astype(np.float32))\n        pred_test = np.zeros((X_test.shape[0],level))\n\n        for i in range(N_SPLITS) :\n            with torch.no_grad():\n                model = dict_level[level][i]\n                model.eval()\n                pred = model(X_test.float())\n                pred_test += pred.numpy()/N_SPLITS\n               \n      \n        pred_test = pred_test.tolist()[0]\n        print(pred_test)           \n            \n        for t in range(a,b):\n            w = t-a\n            print(f'---------- t = {t} ----  t-a = {w} -------------')\n            mask = sample_submission.session_id.str.contains(f'q{t}')\n            sample_submission.loc[mask,'correct'] = int(pred_test[w]>0.61) \n            print('prediction = ',int(pred_test[w]>0.61))                  \n           \n        #env.predict(sample_submission)\n           \n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-03-04T09:31:31.812379Z","iopub.status.idle":"2023-03-04T09:31:31.813088Z","shell.execute_reply":"2023-03-04T09:31:31.812900Z","shell.execute_reply.started":"2023-03-04T09:31:31.812877Z"},"trusted":true},"execution_count":null,"outputs":[]}]}