{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Catboost_embed_model_train\n\nHi there!\n\n### This is the auxulary notebook to 'CatBoost with embeddings on coordinates [0.698 on validation]' notebook. \nFor detailed description please see main motebook here https://www.kaggle.com/code/ivanisaev/embeds-catboost-w-embed-features/ \n\nNote: It takes near 24 howrs to train and size of 18 CatBoost models. But I attached models for the first fold in dataset and use only one fold to speed up computation. If you will use 5 folds you will get 0.698 instead of 0.697\n\n","metadata":{}},{"cell_type":"code","source":"!pip install catboost","metadata":{"id":"Sz6uD8L8ozfH"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nfrom tqdm import tqdm\nfrom sklearn.metrics import f1_score\nfrom sklearn.model_selection import GroupKFold, KFold\nfrom catboost import CatBoostClassifier, Pool\nimport matplotlib.pyplot as plt\nimport warnings\nfrom itertools import combinations\nimport math\nwarnings.filterwarnings('ignore')\npd.set_option(\"display.max_columns\", None)\npd.set_option(\"display.max_rows\", 200)\n\ntargets = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\ntargets['session'] = targets.session_id.apply(lambda x: int(x.split('_')[0]))\ntargets['q'] = targets.session_id.apply(lambda x: int(x.split('_')[-1][1:]))\n\n\ncolumns = [\n    pl.col(\"page\").cast(pl.Float32),\n    (\n        (pl.col(\"elapsed_time\") - pl.col(\"elapsed_time\").shift(1))\n        .fill_null(0)\n        .clip(0, 1e9)\n        .over([\"session_id\", \"level\"])\n        .alias(\"elapsed_time_diff\")\n    ),\n    (\n        (pl.col(\"screen_coor_x\") - pl.col(\"screen_coor_x\").shift(1))\n        .abs()\n        .over([\"session_id\", \"level\"])\n    ),\n    (\n        (pl.col(\"screen_coor_y\") - pl.col(\"screen_coor_y\").shift(1))\n        .abs()\n        .over([\"session_id\", \"level\"])\n    ),\n    pl.col(\"fqid\").fill_null(\"fqid_None\"),\n    pl.col(\"text_fqid\").fill_null(\"text_fqid_None\")\n\n]\ndf = (pl.read_parquet('/kaggle/input/train-parquet/train.parquet')\n\n      .with_columns(columns))\n\ndf1 = df.filter(pl.col(\"level_group\") == '0-4')\ndf2 = df.filter(pl.col(\"level_group\") == '5-12')\ndf3 = df.filter(pl.col(\"level_group\") == '13-22')\n\nCATS = ['event_name', 'name', 'fqid', 'room_fqid', 'text_fqid']\nNUMS = ['page', 'room_coor_x', 'room_coor_y', 'screen_coor_x', 'screen_coor_y',\n        'hover_duration', 'elapsed_time_diff']\nfqid_lists = ['worker', 'archivist', 'gramps', 'wells', 'toentry', 'confrontation', 'crane_ranger', 'groupconvo', 'flag_girl', 'tomap', 'tostacks', 'tobasement', 'archivist_glasses', 'boss', 'journals', 'seescratches', 'groupconvo_flag', 'cs', 'teddy', 'expert', 'businesscards', 'ch3start', 'tunic.historicalsociety', 'tofrontdesk', 'savedteddy', 'plaque', 'glasses', 'tunic.drycleaner', 'reader_flag', 'tunic.library', 'tracks', 'tunic.capitol_2', 'trigger_scarf', 'reader', 'directory', 'tunic.capitol_1', 'journals.pic_0.next', 'unlockdoor', 'tunic', 'what_happened', 'tunic.kohlcenter', 'tunic.humanecology', 'colorbook', 'logbook', 'businesscards.card_0.next', 'journals.hub.topics', 'logbook.page.bingo', 'journals.pic_1.next', 'journals_flag', 'reader.paper0.next', 'tracks.hub.deer', 'reader_flag.paper0.next', 'trigger_coffee', 'wellsbadge', 'journals.pic_2.next', 'tomicrofiche', 'journals_flag.pic_0.bingo', 'plaque.face.date', 'notebook', 'tocloset_dirty', 'businesscards.card_bingo.bingo', 'businesscards.card_1.next', 'tunic.wildlife', 'tunic.hub.slip', 'tocage', 'journals.pic_2.bingo', 'tocollectionflag', 'tocollection', 'chap4_finale_c', 'chap2_finale_c', 'lockeddoor', 'journals_flag.hub.topics', 'tunic.capitol_0', 'reader_flag.paper2.bingo', 'photo', 'tunic.flaghouse', 'reader.paper1.next', 'directory.closeup.archivist', 'intro', 'businesscards.card_bingo.next', 'reader.paper2.bingo', 'retirement_letter', 'remove_cup', 'journals_flag.pic_0.next', 'magnify', 'coffee', 'key', 'togrampa', 'reader_flag.paper1.next', 'janitor', 'tohallway', 'chap1_finale', 'report', 'outtolunch', 'journals_flag.hub.topics_old', 'journals_flag.pic_1.next', 'reader.paper2.next', 'chap1_finale_c', 'reader_flag.paper2.next', 'door_block_talk', 'journals_flag.pic_1.bingo', 'journals_flag.pic_2.next', 'journals_flag.pic_2.bingo', 'block_magnify', 'reader.paper0.prev', 'block', 'reader_flag.paper0.prev', 'block_0', 'door_block_clean', 'reader.paper2.prev', 'reader.paper1.prev', 'doorblock', 'tocloset', 'reader_flag.paper2.prev', 'reader_flag.paper1.prev', 'block_tomap2', 'journals_flag.pic_0_old.next', 'journals_flag.pic_1_old.next', 'block_tocollection', 'block_nelson', 'journals_flag.pic_2_old.next', 'block_tomap1', 'block_badge', 'need_glasses', 'block_badge_2', 'fox', 'block_1']\nDIALOGS = ['that', 'this', 'it', 'you', 'flag', 'can','and','is','the','to']\nname_feature = ['basic', 'undefined', 'close', 'open', 'prev', 'next']\nevent_name_feature = ['cutscene_click', 'person_click', 'navigate_click',\n       'observation_click', 'notification_click', 'object_click',\n       'object_hover', 'map_hover', 'map_click', 'checkpoint',\n       'notebook_click']\ntext_lists = ['tunic.historicalsociety.cage.confrontation', 'tunic.wildlife.center.crane_ranger.crane', 'tunic.historicalsociety.frontdesk.archivist.newspaper', 'tunic.historicalsociety.entry.groupconvo', 'tunic.wildlife.center.wells.nodeer', 'tunic.historicalsociety.frontdesk.archivist.have_glass', 'tunic.drycleaner.frontdesk.worker.hub', 'tunic.historicalsociety.closet_dirty.gramps.news', 'tunic.humanecology.frontdesk.worker.intro', 'tunic.historicalsociety.frontdesk.archivist_glasses.confrontation', 'tunic.historicalsociety.basement.seescratches', 'tunic.historicalsociety.collection.cs', 'tunic.flaghouse.entry.flag_girl.hello', 'tunic.historicalsociety.collection.gramps.found', 'tunic.historicalsociety.basement.ch3start', 'tunic.historicalsociety.entry.groupconvo_flag', 'tunic.library.frontdesk.worker.hello', 'tunic.library.frontdesk.worker.wells', 'tunic.historicalsociety.collection_flag.gramps.flag', 'tunic.historicalsociety.basement.savedteddy', 'tunic.library.frontdesk.worker.nelson', 'tunic.wildlife.center.expert.removed_cup', 'tunic.library.frontdesk.worker.flag', 'tunic.historicalsociety.frontdesk.archivist.hello', 'tunic.historicalsociety.closet.gramps.intro_0_cs_0', 'tunic.historicalsociety.entry.boss.flag', 'tunic.flaghouse.entry.flag_girl.symbol', 'tunic.historicalsociety.closet_dirty.trigger_scarf', 'tunic.drycleaner.frontdesk.worker.done', 'tunic.historicalsociety.closet_dirty.what_happened', 'tunic.wildlife.center.wells.animals', 'tunic.historicalsociety.closet.teddy.intro_0_cs_0', 'tunic.historicalsociety.cage.glasses.afterteddy', 'tunic.historicalsociety.cage.teddy.trapped', 'tunic.historicalsociety.cage.unlockdoor', 'tunic.historicalsociety.stacks.journals.pic_2.bingo', 'tunic.historicalsociety.entry.wells.flag', 'tunic.humanecology.frontdesk.worker.badger', 'tunic.historicalsociety.stacks.journals_flag.pic_0.bingo', 'tunic.historicalsociety.closet.intro', 'tunic.historicalsociety.closet.retirement_letter.hub', 'tunic.historicalsociety.entry.directory.closeup.archivist', 'tunic.historicalsociety.collection.tunic.slip', 'tunic.kohlcenter.halloffame.plaque.face.date', 'tunic.historicalsociety.closet_dirty.trigger_coffee', 'tunic.drycleaner.frontdesk.logbook.page.bingo', 'tunic.library.microfiche.reader.paper2.bingo', 'tunic.kohlcenter.halloffame.togrampa', 'tunic.capitol_2.hall.boss.haveyougotit', 'tunic.wildlife.center.wells.nodeer_recap', 'tunic.historicalsociety.cage.glasses.beforeteddy', 'tunic.historicalsociety.closet_dirty.gramps.helpclean', 'tunic.wildlife.center.expert.recap', 'tunic.historicalsociety.frontdesk.archivist.have_glass_recap', 'tunic.historicalsociety.stacks.journals_flag.pic_1.bingo', 'tunic.historicalsociety.cage.lockeddoor', 'tunic.historicalsociety.stacks.journals_flag.pic_2.bingo', 'tunic.historicalsociety.collection.gramps.lost', 'tunic.historicalsociety.closet.notebook', 'tunic.historicalsociety.frontdesk.magnify', 'tunic.humanecology.frontdesk.businesscards.card_bingo.bingo', 'tunic.wildlife.center.remove_cup', 'tunic.library.frontdesk.wellsbadge.hub', 'tunic.wildlife.center.tracks.hub.deer', 'tunic.historicalsociety.frontdesk.key', 'tunic.library.microfiche.reader_flag.paper2.bingo', 'tunic.flaghouse.entry.colorbook', 'tunic.wildlife.center.coffee', 'tunic.capitol_1.hall.boss.haveyougotit', 'tunic.historicalsociety.basement.janitor', 'tunic.historicalsociety.collection_flag.gramps.recap', 'tunic.wildlife.center.wells.animals2', 'tunic.flaghouse.entry.flag_girl.symbol_recap', 'tunic.historicalsociety.closet_dirty.photo', 'tunic.historicalsociety.stacks.outtolunch', 'tunic.library.frontdesk.worker.wells_recap', 'tunic.historicalsociety.frontdesk.archivist_glasses.confrontation_recap', 'tunic.capitol_0.hall.boss.talktogramps', 'tunic.historicalsociety.closet.photo', 'tunic.historicalsociety.collection.tunic', 'tunic.historicalsociety.closet.teddy.intro_0_cs_5', 'tunic.historicalsociety.closet_dirty.gramps.archivist', 'tunic.historicalsociety.closet_dirty.door_block_talk', 'tunic.historicalsociety.entry.boss.flag_recap', 'tunic.historicalsociety.frontdesk.archivist.need_glass_0', 'tunic.historicalsociety.entry.wells.talktogramps', 'tunic.historicalsociety.frontdesk.block_magnify', 'tunic.historicalsociety.frontdesk.archivist.foundtheodora', 'tunic.historicalsociety.closet_dirty.gramps.nothing', 'tunic.historicalsociety.closet_dirty.door_block_clean', 'tunic.capitol_1.hall.boss.writeitup', 'tunic.library.frontdesk.worker.nelson_recap', 'tunic.library.frontdesk.worker.hello_short', 'tunic.historicalsociety.stacks.block', 'tunic.historicalsociety.frontdesk.archivist.need_glass_1', 'tunic.historicalsociety.entry.boss.talktogramps', 'tunic.historicalsociety.frontdesk.archivist.newspaper_recap', 'tunic.historicalsociety.entry.wells.flag_recap', 'tunic.drycleaner.frontdesk.worker.done2', 'tunic.library.frontdesk.worker.flag_recap', 'tunic.humanecology.frontdesk.block_0', 'tunic.library.frontdesk.worker.preflag', 'tunic.historicalsociety.basement.gramps.seeyalater', 'tunic.flaghouse.entry.flag_girl.hello_recap', 'tunic.historicalsociety.closet.doorblock', 'tunic.drycleaner.frontdesk.worker.takealook', 'tunic.historicalsociety.basement.gramps.whatdo', 'tunic.library.frontdesk.worker.droppedbadge', 'tunic.historicalsociety.entry.block_tomap2', 'tunic.library.frontdesk.block_nelson', 'tunic.library.microfiche.block_0', 'tunic.historicalsociety.entry.block_tocollection', 'tunic.historicalsociety.entry.block_tomap1', 'tunic.historicalsociety.collection.gramps.look_0', 'tunic.library.frontdesk.block_badge', 'tunic.historicalsociety.cage.need_glasses', 'tunic.library.frontdesk.block_badge_2', 'tunic.kohlcenter.halloffame.block_0', 'tunic.capitol_0.hall.chap1_finale_c', 'tunic.capitol_1.hall.chap2_finale_c', 'tunic.capitol_2.hall.chap4_finale_c', 'tunic.wildlife.center.fox.concern', 'tunic.drycleaner.frontdesk.block_0', 'tunic.historicalsociety.entry.gramps.hub', 'tunic.humanecology.frontdesk.block_1', 'tunic.drycleaner.frontdesk.block_1']\nroom_lists = ['tunic.historicalsociety.entry', 'tunic.wildlife.center', 'tunic.historicalsociety.cage', 'tunic.library.frontdesk', 'tunic.historicalsociety.frontdesk', 'tunic.historicalsociety.stacks', 'tunic.historicalsociety.closet_dirty', 'tunic.humanecology.frontdesk', 'tunic.historicalsociety.basement', 'tunic.kohlcenter.halloffame', 'tunic.library.microfiche', 'tunic.drycleaner.frontdesk', 'tunic.historicalsociety.collection', 'tunic.historicalsociety.closet', 'tunic.flaghouse.entry', 'tunic.historicalsociety.collection_flag', 'tunic.capitol_1.hall', 'tunic.capitol_0.hall', 'tunic.capitol_2.hall']\n\n\nLEVELS = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22]\nlevel_groups = [\"0-4\", \"5-12\", \"13-22\"]\ndef feature_engineer(x, grp, use_extra, feature_suffix):\n    aggs = [\n        pl.col(\"index\").count().alias(f\"session_number_{feature_suffix}\"),\n\n        *[pl.col('index').filter(pl.col('text').str.contains(c)).count().alias(f'word_{c}') for c in DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text').str.contains(c))).mean().alias(f'word_mean_{c}') for c in DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text').str.contains(c))).std().alias(f'word_std_{c}') for c in DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col('text').str.contains(c)).max().alias(f'word_max_{c}') for c in DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col('text').str.contains(c)).sum().alias(f'word_sum_{c}') for c in DIALOGS],\n\n        *[pl.col(c).drop_nulls().n_unique().alias(f\"{c}_unique_{feature_suffix}\") for c in CATS],\n\n        *[pl.col(c).quantile(0.1, \"nearest\").alias(f\"{c}_quantile1_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).quantile(0.2, \"nearest\").alias(f\"{c}_quantile2_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).quantile(0.4, \"nearest\").alias(f\"{c}_quantile4_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).quantile(0.6, \"nearest\").alias(f\"{c}_quantile6_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).quantile(0.8, \"nearest\").alias(f\"{c}_quantile8_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).quantile(0.9, \"nearest\").alias(f\"{c}_quantile9_{feature_suffix}\") for c in NUMS],\n\n        *[pl.col(c).mean().alias(f\"{c}_mean_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).std().alias(f\"{c}_std_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).min().alias(f\"{c}_min_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).max().alias(f\"{c}_max_{feature_suffix}\") for c in NUMS],\n\n        *[pl.col(\"fqid\").filter(pl.col(\"fqid\") == c).count().alias(f\"{c}_fqid_counts{feature_suffix}\")\n          for c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for\n          c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in fqid_lists],\n\n        *[pl.col(\"text_fqid\").filter(pl.col(\"text_fqid\") == c).count().alias(f\"{c}_text_fqid_counts{feature_suffix}\")\n          for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in text_lists],\n\n        *[pl.col(\"room_fqid\").filter(pl.col(\"room_fqid\") == c).count().alias(f\"{c}_room_fqid_counts{feature_suffix}\")\n          for c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for\n          c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in room_lists],\n\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).quantile(0.1, \"nearest\").alias(\n            f\"{c}_ET_quantile1_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).quantile(0.2, \"nearest\").alias(\n            f\"{c}_ET_quantile2_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).quantile(0.4, \"nearest\").alias(\n            f\"{c}_ET_quantile4_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).quantile(0.6, \"nearest\").alias(\n            f\"{c}_ET_quantile6_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).quantile(0.8, \"nearest\").alias(\n            f\"{c}_ET_quantile8_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).quantile(0.9, \"nearest\").alias(\n            f\"{c}_ET_quantile9_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"event_name\").filter(pl.col(\"event_name\") == c).count().alias(f\"{c}_event_name_counts{feature_suffix}\")\n          for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\")\n          for\n          c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in event_name_feature],\n\n        *[pl.col(\"name\").filter(pl.col(\"name\") == c).count().alias(f\"{c}_name_counts{feature_suffix}\") for c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for c in\n          name_feature],\n\n        *[pl.col(\"level\").filter(pl.col(\"level\") == c).count().alias(f\"{c}_LEVEL_count{feature_suffix}\") for c in\n          LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for c in\n          LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for c\n          in\n          LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for c in\n          LEVELS],\n\n        *[pl.col(\"level_group\").filter(pl.col(\"level_group\") == c).count().alias(\n            f\"{c}_LEVEL_group_count{feature_suffix}\") for c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\")\n          for c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in\n          level_groups],\n\n        *[pl.col(\"index\").filter((pl.col(\"level\") == c) & (pl.col('room_fqid') == d)).count().alias(\n            f\"{c}{d}_level_room_count{feature_suffix}\") for c in LEVELS for d in room_lists],\n\n    ]\n\n    df = x.groupby(['session_id'], maintain_order=True).agg(aggs).sort(\"session_id\")\n\n    if use_extra:\n        if grp == '5-12':\n            aggs = [\n                pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"Here's the log book.\")\n                                              | (pl.col(\"fqid\") == 'logbook.page.bingo'))\n                    .apply(lambda s: s.max() - s.min()).alias(\"logbook_bingo_duration\"),\n                pl.col(\"index\").filter(\n                    (pl.col(\"text\") == \"Here's the log book.\") | (pl.col(\"fqid\") == 'logbook.page.bingo')).apply(\n                    lambda s: s.max() - s.min()).alias(\"logbook_bingo_indexCount\"),\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader')) | (\n                            pl.col(\"fqid\") == \"reader.paper2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\n                    \"reader_bingo_duration\"),\n                pl.col(\"index\").filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader')) | (\n                        pl.col(\"fqid\") == \"reader.paper2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\n                    \"reader_bingo_indexCount\"),\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals')) | (\n                            pl.col(\"fqid\") == \"journals.pic_2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\n                    \"journals_bingo_duration\"),\n                pl.col(\"index\").filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals')) | (\n                        pl.col(\"fqid\") == \"journals.pic_2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\n                    \"journals_bingo_indexCount\"),\n            ]\n            tmp = x.groupby([\"session_id\"], maintain_order=True).agg(aggs).sort(\"session_id\")\n            df = df.join(tmp, on=\"session_id\", how='left')\n\n        if grp == '13-22':\n            aggs = [\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader_flag')) | (\n                            pl.col(\"fqid\") == \"tunic.library.microfiche.reader_flag.paper2.bingo\")).apply(\n                    lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"reader_flag_duration\"),\n                pl.col(\"index\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader_flag')) | (\n                            pl.col(\"fqid\") == \"tunic.library.microfiche.reader_flag.paper2.bingo\")).apply(\n                    lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"reader_flag_indexCount\"),\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals_flag')) | (\n                            pl.col(\"fqid\") == \"journals_flag.pic_0.bingo\")).apply(\n                    lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"journalsFlag_bingo_duration\"),\n                pl.col(\"index\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals_flag')) | (\n                            pl.col(\"fqid\") == \"journals_flag.pic_0.bingo\")).apply(\n                    lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"journalsFlag_bingo_indexCount\")\n            ]\n            tmp = x.groupby([\"session_id\"], maintain_order=True).agg(aggs).sort(\"session_id\")\n            df = df.join(tmp, on=\"session_id\", how='left')\n\n    return df.to_pandas()\n\ndf1 = feature_engineer(df1, grp='0-4', use_extra=True, feature_suffix='')\ndf2 = feature_engineer(df2, grp='5-12', use_extra=True, feature_suffix='')\ndf3 = feature_engineer(df3, grp='13-22', use_extra=True, feature_suffix='')\n\ndef time_feature(train):\n    train[\"year\"] = train[\"session_id\"].apply(lambda x: int(str(x)[:2])).astype(np.uint8)\n    train[\"month\"] = train[\"session_id\"].apply(lambda x: int(str(x)[2:4])+1).astype(np.uint8)\n    train[\"day\"] = train[\"session_id\"].apply(lambda x: int(str(x)[4:6])).astype(np.uint8)\n    train[\"hour\"] = train[\"session_id\"].apply(lambda x: int(str(x)[6:8])).astype(np.uint8)\n    train[\"minute\"] = train[\"session_id\"].apply(lambda x: int(str(x)[8:10])).astype(np.uint8)\n    train[\"second\"] = train[\"session_id\"].apply(lambda x: int(str(x)[10:12])).astype(np.uint8)\n\n\n    return train\n\ndf1 = time_feature(df1)\ndf2 = time_feature(df2)\ndf3 = time_feature(df3)\n\nnull1 = df1.isnull().sum().sort_values(ascending=False)/len(df1)\nnull2 = df2.isnull().sum().sort_values(ascending=False)/len(df2)\nnull3 = df3.isnull().sum().sort_values(ascending=False)/len(df3)\n\ndrop1 = list(null1[null1 > 0.9].index)\ndrop2 = list(null2[null2 > 0.9].index)\ndrop3 = list(null3[null3 > 0.9].index)\n\nprint(len(drop1), len(drop2), len(drop3))\n\nfor col in tqdm(df1.columns):\n    if df1[col].nunique() == 1:\n        print(col)\n        drop1.append(col)\nfor col in tqdm(df2.columns):\n    if df2[col].nunique() == 1:\n        print(col)\n        drop2.append(col)\nfor col in tqdm(df3.columns):\n    if df3[col].nunique() == 1:\n        print(col)\n        drop3.append(col)","metadata":{"id":"9lFGQSS8oxv7"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nemeb_df = pd.read_csv('/kaggle/input/embeddings-train-df/embeddings_train_df/embeddings_train_df.csv')\n# You can find embeddings_train_df in the /kaggle/working/embeddings_train_df","metadata":{"id":"A_7R0XwTqjJ3"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"emeb_df.shape","metadata":{"id":"oQ3oSzIhYkpH","outputId":"3d12a4d4-a57d-45e9-a63e-2de93c34a7fa"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(emeb_df)","metadata":{"id":"K6gpvjMoZFXX","outputId":"5dc954a2-d9d1-4f5e-d582-01dd5275536c"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"emeb_df['session_id'], emeb_df['level'] = emeb_df['sid_level'].str.split(' ', 1).str\nemeb_df = emeb_df.drop('sid_level', axis = 1)\n\nemeb_df_index = emeb_df[['session_id', 'level']].copy()\nemeb_df = emeb_df.drop(['session_id', 'level'], axis = 1)\n\nX_train_emb = pd.DataFrame(np.arange(len(emeb_df)))\nX_train_emb['embeddings'] = emeb_df.values.tolist()\nX_train_emb = X_train_emb.drop(0, axis = 1)\n\nemeb_df = pd.merge(emeb_df_index, X_train_emb, left_index=True, right_index=True)","metadata":{"id":"SSW_bF7tArDU"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"emeb_df = pd.merge(emeb_df_index, X_train_emb, left_index=True, right_index=True)","metadata":{"id":"N0jz3BvBBjHK"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"emeb_df['level'] = emeb_df['level'].astype(int)","metadata":{"id":"dUpoNFqDT41V"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"emeb_df = emeb_df.sort_values(by=['session_id', 'level'])","metadata":{"id":"PW3i5YtbTPcr"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"emeb_df","metadata":{"id":"uzj1LFQnVkuf","outputId":"a2bac142-1e18-41e5-98a4-f740b13c77c6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res = emeb_df.pivot(index='session_id', columns='level', values='embeddings').reset_index()\nres = res.set_index('session_id').reset_index().rename_axis(None, axis=1)\ncols = [f'level{x}_emb' for x in range(23)]\ncols.insert(0, 'session_id')\nres.columns = cols","metadata":{"id":"l-9X-zzfZ9zb"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res.head(3)","metadata":{"id":"pkx4LgSRd9N3","outputId":"a07fffb1-d174-4942-922f-3d137cdd08b2"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"emb_features = list(res.columns)","metadata":{"id":"JcTUHaO0hVLp"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"emb_features.remove('session_id')","metadata":{"id":"R8VQCqtTh6bm"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"emb_features","metadata":{"id":"DtO3c7r6hoCt","outputId":"f590daba-dfb1-4341-bf4b-d176c1276bd5"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res['session_id'] = res['session_id'].astype(int)","metadata":{"id":"3sWrkmwOg2L4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res['level7_emb'] = res['level7_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\nres['level15_emb'] = res['level15_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\nres['level20_emb'] = res['level20_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\nres['level21_emb'] = res['level21_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)","metadata":{"id":"ZK9WRr0PYVUz"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res.isnull().any()","metadata":{"id":"B1Cj7AMVYtS0","outputId":"bd6af224-5d0f-48c8-fa15-d8e09242b48f"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 = df1.merge(res, on = 'session_id')\ndf2 = df2.merge(res, on = 'session_id')\ndf3 = df3.merge(res, on = 'session_id')\n\ndf1 = df1.set_index('session_id')\ndf2 = df2.set_index('session_id')\ndf3 = df3.set_index('session_id')\n\n\nFEATURES1 = [c for c in df1.columns if c not in drop1+['level_group']]\nFEATURES2 = [c for c in df2.columns if c not in drop2+['level_group']]\nFEATURES3 = [c for c in df3.columns if c not in drop3+['level_group']]\nprint('We will train with', len(FEATURES1), len(FEATURES2), len(FEATURES3), 'features')\nALL_USERS = df1.index.unique()\nprint('We will train with', len(ALL_USERS), 'users info')","metadata":{"id":"SZ9rCsTkdZgx","outputId":"97ac0046-d2ab-4e95-d519-f17b3fc64581"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from catboost import CatBoostClassifier, metrics","metadata":{"id":"5plFqmhT9il3"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"emb_features","metadata":{"id":"_2edCzFnZyQd","outputId":"c680e48e-104c-49a0-b7eb-da2d424537a0"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc","metadata":{"id":"7WtSVqGzJNFs"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del(res)\ndel(df)\ndel(emeb_df)\ndel(emeb_df_index)\ngc.collect()","metadata":{"id":"O339_l_9IwLv","outputId":"689c64f7-caa9-4845-c305-b29731675078"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_importance_df = pd.DataFrame()\nmodels = {}\nresults = [[[], []] for _ in range(18)]\ngkf = GroupKFold(n_splits=5)\noof_cat = pd.DataFrame(data=np.zeros((len(ALL_USERS),18)), index=ALL_USERS, columns=[f'meta_{i}' for i in range(1, 19)])\nfor q in tqdm(range(1, 19)):\n    if q <= 3:\n        grp = '0-4'\n        df = df1\n        FEATURES = FEATURES1\n    elif q <= 13:\n        grp = '5-12'\n        df = df2\n        FEATURES = FEATURES2\n    elif q <= 22:\n        grp = '13-22'\n        df = df3\n        FEATURES = FEATURES3\n\n    print(\"#\"*25)\n    print(f'question{q}, with{len(FEATURES)}features')\n    print('#'*25)\n\n    cat_params = {\n        'iterations': 1000,\n        'early_stopping_rounds': 90,\n        'depth': 5,\n        'learning_rate': 0.02,\n        'loss_function': \"Logloss\",\n        'random_seed': 22222,\n        'embedding_features': emb_features,\n        'metric_period': 1,\n        'subsample': 0.8,\n        'colsample_bylevel': 0.4,\n        'verbose': 0,\n        'l2_leaf_reg': 20,\n    }\n    feature_importance_df = pd.DataFrame()\n\n    for fold, (train_idx, valid_idx) in enumerate(gkf.split(X=df, groups=df.index)):\n        if fold == 0:\n            print('#'*25)\n            print('### Fold',fold+1)\n            print('#'*25)\n            # TRAIN DATA\n            train_x = df.iloc[train_idx]\n            train_users = train_x.index.values\n            train_y = targets.loc[targets.q == q].set_index('session').loc[train_users]\n            # train_y = pd.DataFrame(train_y['correct']).values\n\n            # VALID DATA\n            valid_x = df.iloc[valid_idx]\n            valid_users = valid_x.index.values\n            valid_y = targets.loc[targets.q == q].set_index('session').loc[valid_users]\n\n\n            # train_pool = Pool(train_x[FEATURES].astype('float32'), train_y['correct'])\n            # valid_pool = Pool(valid_x[FEATURES].astype('float32'), valid_y['correct'])\n\n            train_pool = Pool(\n                    data=train_x[FEATURES],\n                    label=train_y['correct'],\n                    embedding_features=emb_features)\n            valid_pool = Pool(\n                    data=valid_x[FEATURES],\n                    label=valid_y['correct'],\n                    embedding_features=emb_features)\n\n            # model = CatBoostClassifier(**cat_params)\n            # model = model.fit(train_pool, eval_set=valid_pool)\n            model = CatBoostClassifier().load_model(f\"/kaggle/input/emb454545/catboost-emb-clean/fold0_q{q}.cbm\")\n            y = valid_pool.get_label()\n            y_hat = model.predict_proba(valid_pool)[:,1]\n\n            models[(fold, q)] = model\n\n            fold_importance_df = pd.DataFrame()\n            fold_importance_df[\"feature\"] = FEATURES\n            fold_importance_df[\"importance\"] = model.feature_importances_\n            fold_importance_df[\"fold\"] = fold + 1\n            feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n\n            results[q - 1][0].append(y)\n            results[q - 1][1].append(y_hat)\n        elif fold in [1, 2, 3, 4]:\n            pass \n    feature_importance_df = feature_importance_df.groupby(['feature'])['importance'].agg(['mean']).sort_values(by='mean', ascending=False)\n    display(feature_importance_df.head(10))","metadata":{"id":"xvw3htvdoxwB","outputId":"4048e568-81b9-4362-8ecf-40564a1ccf61"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = [[np.concatenate(_) for _ in _] for _ in results]","metadata":{"id":"59tGspYAtn7T"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for (fold,q), model in models.items():\n#     model.save_model(f'fold{fold}_q{q}.cbm')\n\ntrue = pd.DataFrame(np.stack([_[0] for _ in results]).T)\noof = pd.DataFrame(np.stack([_[1] for _ in results]).T)\n\nscores = []; thresholds = []\nbest_socre = 0; best_threshold = 0\n\nfor threshold in np.arange(0.5, 0.7, 0.01):\n    preds = (oof.values.reshape(-1) > threshold).astype('int')\n    m = f1_score(true.values.reshape(-1), preds, average='macro')\n    scores.append(m)\n    thresholds.append(threshold)\n    if m > best_socre:\n        best_socre = m\n        best_threshold = threshold\n\nplt.figure(figsize=(20, 5))\nplt.plot(thresholds, scores, '-o', color='blue')\nplt.scatter([best_threshold], [best_socre], color='blue')\nplt.xlabel(\"Threshold\", size=14)\nplt.ylabel(\"Validation F1 Score\",size=14)\nplt.title(f'Threshold vs. F1_Score with Best F1_Score={best_socre:.3f} at Best Threshold = {best_threshold:.3}', size=18)\nplt.show()\n\nprint(f'When using optimal threshold = {best_threshold:.2f}...')\nfor k in range(18):\n    m = f1_score(true[k].values, (oof[k].values > best_threshold).astype('int'), average = 'macro')\n    print(f'Q{k}: F1 =',m)\nm = f1_score(true.values.reshape(-1), (oof.values > best_threshold).reshape(-1).astype('int'), average = 'macro')\nprint('==> Overall F1 =', m)\nprint('s')","metadata":{"id":"ZAYQV1tptn4Q","outputId":"47053437-7bcc-4703-d674-c6e4fdf85c98"},"execution_count":null,"outputs":[]}]}