{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Catboost_embed_model with Coordinates Features (Pathes). Inference part","metadata":{"id":"wel9fZYkzgE7"}},{"cell_type":"markdown","source":"### Data Loading","metadata":{"id":"vWmwuvuBzgFA"}},{"cell_type":"markdown","source":"Hi there!\n\n### This is the auxulary notebook to 'CatBoost with embeddings on coordinates [0.698 on validation]' notebook.\n\nFor detailed description please see main motebook here https://www.kaggle.com/ivanisaev/embeds-catboost-w-embed-features\n\nNote: This inference doesn't fit 8Gb RAM constraints. This is just teplate of how to use it for 10 questions. \n\n### How you can use this stuff\n- For example you can make embeddings not per each level but per level group or add modells with embeddings only for particular question.\n- You can also use embedding models only for a part (for example half of questions for which it outperforms other open models). In the main notebook's description you can find scores of notebook with embeddings comapring to Catboost Mix scores ","metadata":{}},{"cell_type":"code","source":"!pip install catboost","metadata":{"execution":{"iopub.status.busy":"2023-05-28T23:34:22.651140Z","iopub.execute_input":"2023-05-28T23:34:22.652038Z","iopub.status.idle":"2023-05-28T23:34:37.774867Z","shell.execute_reply.started":"2023-05-28T23:34:22.651935Z","shell.execute_reply":"2023-05-28T23:34:37.773519Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from catboost import CatBoostClassifier, Pool\nmodels_list = [CatBoostClassifier().load_model(f\"/kaggle/input/emb454545/catboost-emb-clean/fold0_q{q}.cbm\") for q in range(1, 10)]","metadata":{"execution":{"iopub.status.busy":"2023-05-28T23:34:40.241820Z","iopub.execute_input":"2023-05-28T23:34:40.242273Z","iopub.status.idle":"2023-05-28T23:34:45.045957Z","shell.execute_reply.started":"2023-05-28T23:34:40.242229Z","shell.execute_reply":"2023-05-28T23:34:45.044847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\nf_read = open('/kaggle/input/emb454545/catboost-emb-clean/importance_dict.pkl', 'rb')\nimportance_dict = pickle.load(f_read)\nf_read.close()","metadata":{"execution":{"iopub.status.busy":"2023-05-28T23:37:06.706693Z","iopub.execute_input":"2023-05-28T23:37:06.707518Z","iopub.status.idle":"2023-05-28T23:37:06.720029Z","shell.execute_reply.started":"2023-05-28T23:37:06.707473Z","shell.execute_reply":"2023-05-28T23:37:06.718493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pytorch-lifestream\n!pip install \"torch<2\"\n!pip install -U \"pytorch-lightning<2\"\n!pip install -U \"torchvision<0.15.1\"","metadata":{"id":"fATsUqNsuFiS","execution":{"iopub.status.busy":"2023-05-28T23:37:10.330472Z","iopub.execute_input":"2023-05-28T23:37:10.334501Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip freeze | grep torch","metadata":{"id":"8zU-x0KWuILz","outputId":"f7b9a7fa-b081-48d3-a8a2-31dc01627455","collapsed":true,"jupyter":{"outputs_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nfrom tqdm import tqdm\nfrom sklearn.metrics import f1_score\nfrom sklearn.model_selection import GroupKFold, KFold\nfrom catboost import CatBoostClassifier, Pool\nimport matplotlib.pyplot as plt\nimport warnings\nfrom itertools import combinations\nimport math\nwarnings.filterwarnings('ignore')\npd.set_option(\"display.max_columns\", None)\npd.set_option(\"display.max_rows\", 200)\n\nfrom ptls.preprocessing import PandasDataPreprocessor\nimport torch\nfrom ptls.frames.supervised import SequenceToTarget\nfrom ptls.nn import TrxEncoder, RnnSeqEncoder\nimport pytorch_lightning as pl\nfrom ptls.data_load.datasets import inference_data_loader\n\nmax_height = 1261.7737454550663 \nmin_height = -1992.3545688360275 \nmax_width = 543.6164243795992 \nmin_width = -918.1623490877204\nnum_px = 8\nnum_py = 6\ngps = []\n\ndef get_patch_index(x, y, min_image_height, max_image_height, min_image_width, max_image_width, num_patches_height = 8, num_patches_width = 6):\n    if np.isnan(x):\n        return num_px * num_py + 1\n    patch_height = (max_image_height - min_image_height + 1) / num_patches_height\n    patch_width = (max_image_width - min_image_width + 1) / num_patches_width\n    patch_x = (x - min_image_height) // patch_height\n    patch_y = (y - min_image_width) // patch_width\n    patch_index = patch_x * num_patches_width + patch_y + 1\n    return patch_index","metadata":{"id":"0-Pmsu-0nzOp"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_emb_df(test):\n    \n    test = test[['session_id', 'level_group', 'level', 'event_name', 'elapsed_time', 'room_coor_x', 'room_coor_y']]\n\n    for _, session in test.groupby('session_id'):\n        for _, gp in session.groupby('level_group'):\n            gp['patch'] = gp.apply(lambda row : get_patch_index(row['room_coor_x'], row['room_coor_y'], min_height, max_height, min_width, max_width, num_px, num_py), axis = 1)\n            gps.append(gp)\n\n    df_patshed = pd.concat(gps)\n    df_patshed['time_diff'] = df_patshed.groupby('session_id')['elapsed_time'].diff()\n    df_patshed['time_diff'] = df_patshed['time_diff'].fillna(0)\n    df_patshed = df_patshed[df_patshed.time_diff >= 0]\n    df_patshed['event_name_factorized'] =  pd.factorize(df_patshed['event_name'])[0]\n    df_patshed['elapsed_time_to_datetime'] = pd.to_datetime(df_patshed['elapsed_time'])\n\n    df_patshed[\"sid_level\"] = df_patshed[\"session_id\"].astype(str) + ' ' + df_patshed[\"level\"].astype(str)\n    df_patshed = df_patshed.drop([\"session_id\", \"level\", \"level_group\", \"event_name\", 'elapsed_time_to_datetime'], axis = 1)\n\n    preprocessor = PandasDataPreprocessor(\n      col_id= 'sid_level',\n      col_event_time='elapsed_time',\n      event_time_transformation='none',\n      cols_category=['patch', 'event_name_factorized'],\n      cols_numerical=['time_diff'],\n      return_records=True,\n    )\n\n    df_patshed = preprocessor.fit_transform(df_patshed)\n    df_patshed = sorted(df_patshed, key=lambda x: x['sid_level'])\n\n    trx_encoder_params = dict(\n      embeddings_noise=0.003,\n      numeric_values={'time_diff': 'identity'},\n      embeddings={\n          'patch': {'in': 55, 'out': 55},\n          'event_name_factorized': {'in': 11, 'out': 11},\n      },\n    )\n\n    seq_encoder = RnnSeqEncoder(\n      trx_encoder=TrxEncoder(**trx_encoder_params),\n      hidden_size=256,\n      type='gru',\n    )\n\n    seq_encoder.load_state_dict(torch.load('/kaggle/input/jw-emb/jw-emb.pt'))\n    model = SequenceToTarget(seq_encoder)\n    model.eval();\n\n    trainer = pl.Trainer(gpus=1 if torch.cuda.is_available() else 0)\n    test_dl = inference_data_loader(df_patshed, num_workers=0, batch_size=256)\n    test_embeds = torch.vstack(trainer.predict(model, test_dl))\n\n    test_df_patched = pd.DataFrame(data=test_embeds, columns=[f'embed_{i}' for i in range(test_embeds.shape[1])])\n    test_df_patched['sid_level'] = [x['sid_level'] for x in df_patshed]\n\n    emeb_df = pd.DataFrame(np.arange(len(test_df_patched)))\n    emeb_df['embeddings'] = test_df_patched.iloc[:, :-1].values.tolist()\n    emeb_df = emeb_df.drop([0], axis=1)\n    emeb_df['sid_level'] = test_df_patched['sid_level']\n\n    emeb_df['session_id'], emeb_df['level'] = emeb_df['sid_level'].str.split(' ', 1).str\n    emeb_df = emeb_df.drop('sid_level', axis = 1)\n    emeb_df['level'] = emeb_df['level'].astype(int)\n    emeb_df = emeb_df.sort_values(by=['session_id', 'level'])\n\n    emeb_df['level'] = emeb_df['level'].astype(int)\n    emeb_df = emeb_df.sort_values(by=['session_id', 'level'])\n\n    res_emb = emeb_df.pivot(index='session_id', columns='level', values='embeddings').reset_index()\n    res_emb = res_emb.set_index('session_id').reset_index().rename_axis(None, axis=1)\n    cols = [f'level{x}_emb' for x in range(23)]\n    cols.insert(0, 'session_id')\n    res_emb.columns = cols\n    emb_features = list(res_emb.columns)\n    emb_features.remove('session_id')\n\n    res_emb['session_id'] = res_emb['session_id'].astype(int)\n    res_emb['level7_emb'] = res_emb['level7_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\n    res_emb['level15_emb'] = res_emb['level15_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\n    res_emb['level20_emb'] = res_emb['level20_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\n    res_emb['level21_emb'] = res_emb['level21_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\n\n    res_emb['level0_emb'] = res_emb['level0_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\n    res_emb['level1_emb'] = res_emb['level1_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\n    res_emb['level2_emb'] = res_emb['level2_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\n    res_emb['level3_emb'] = res_emb['level3_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\n    res_emb['level4_emb'] = res_emb['level4_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\n    res_emb['level5_emb'] = res_emb['level5_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\n    res_emb['level6_emb'] = res_emb['level6_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\n    res_emb['level8_emb'] = res_emb['level8_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\n    res_emb['level9_emb'] = res_emb['level9_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\n    res_emb['level10_emb'] = res_emb['level10_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\n    res_emb['level11_emb'] = res_emb['level11_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\n    res_emb['level12_emb'] = res_emb['level12_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\n    res_emb['level13_emb'] = res_emb['level13_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\n    res_emb['level14_emb'] = res_emb['level14_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\n    res_emb['level16_emb'] = res_emb['level16_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\n    res_emb['level17_emb'] = res_emb['level17_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\n    res_emb['level18_emb'] = res_emb['level18_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\n    res_emb['level19_emb'] = res_emb['level19_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\n    res_emb['level22_emb'] = res_emb['level22_emb'].apply(lambda d: d if isinstance(d, list) else [float(0)]*256)\n\n    return res_emb","metadata":{"id":"bVwOzwNsvqPR"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CATS = ['event_name', 'name', 'fqid', 'room_fqid', 'text_fqid']\nNUMS = ['page', 'room_coor_x', 'room_coor_y', 'screen_coor_x', 'screen_coor_y',\n        'hover_duration', 'elapsed_time_diff']\nfqid_lists = ['worker', 'archivist', 'gramps', 'wells', 'toentry', 'confrontation', 'crane_ranger', 'groupconvo', 'flag_girl', 'tomap', 'tostacks', 'tobasement', 'archivist_glasses', 'boss', 'journals', 'seescratches', 'groupconvo_flag', 'cs', 'teddy', 'expert', 'businesscards', 'ch3start', 'tunic.historicalsociety', 'tofrontdesk', 'savedteddy', 'plaque', 'glasses', 'tunic.drycleaner', 'reader_flag', 'tunic.library', 'tracks', 'tunic.capitol_2', 'trigger_scarf', 'reader', 'directory', 'tunic.capitol_1', 'journals.pic_0.next', 'unlockdoor', 'tunic', 'what_happened', 'tunic.kohlcenter', 'tunic.humanecology', 'colorbook', 'logbook', 'businesscards.card_0.next', 'journals.hub.topics', 'logbook.page.bingo', 'journals.pic_1.next', 'journals_flag', 'reader.paper0.next', 'tracks.hub.deer', 'reader_flag.paper0.next', 'trigger_coffee', 'wellsbadge', 'journals.pic_2.next', 'tomicrofiche', 'journals_flag.pic_0.bingo', 'plaque.face.date', 'notebook', 'tocloset_dirty', 'businesscards.card_bingo.bingo', 'businesscards.card_1.next', 'tunic.wildlife', 'tunic.hub.slip', 'tocage', 'journals.pic_2.bingo', 'tocollectionflag', 'tocollection', 'chap4_finale_c', 'chap2_finale_c', 'lockeddoor', 'journals_flag.hub.topics', 'tunic.capitol_0', 'reader_flag.paper2.bingo', 'photo', 'tunic.flaghouse', 'reader.paper1.next', 'directory.closeup.archivist', 'intro', 'businesscards.card_bingo.next', 'reader.paper2.bingo', 'retirement_letter', 'remove_cup', 'journals_flag.pic_0.next', 'magnify', 'coffee', 'key', 'togrampa', 'reader_flag.paper1.next', 'janitor', 'tohallway', 'chap1_finale', 'report', 'outtolunch', 'journals_flag.hub.topics_old', 'journals_flag.pic_1.next', 'reader.paper2.next', 'chap1_finale_c', 'reader_flag.paper2.next', 'door_block_talk', 'journals_flag.pic_1.bingo', 'journals_flag.pic_2.next', 'journals_flag.pic_2.bingo', 'block_magnify', 'reader.paper0.prev', 'block', 'reader_flag.paper0.prev', 'block_0', 'door_block_clean', 'reader.paper2.prev', 'reader.paper1.prev', 'doorblock', 'tocloset', 'reader_flag.paper2.prev', 'reader_flag.paper1.prev', 'block_tomap2', 'journals_flag.pic_0_old.next', 'journals_flag.pic_1_old.next', 'block_tocollection', 'block_nelson', 'journals_flag.pic_2_old.next', 'block_tomap1', 'block_badge', 'need_glasses', 'block_badge_2', 'fox', 'block_1']\nDIALOGS = ['that', 'this', 'it', 'you', 'flag', 'can','and','is','the','to']\nname_feature = ['basic', 'undefined', 'close', 'open', 'prev', 'next']\nevent_name_feature = ['cutscene_click', 'person_click', 'navigate_click',\n       'observation_click', 'notification_click', 'object_click',\n       'object_hover', 'map_hover', 'map_click', 'checkpoint',\n       'notebook_click']\ntext_lists = ['tunic.historicalsociety.cage.confrontation', 'tunic.wildlife.center.crane_ranger.crane', 'tunic.historicalsociety.frontdesk.archivist.newspaper', 'tunic.historicalsociety.entry.groupconvo', 'tunic.wildlife.center.wells.nodeer', 'tunic.historicalsociety.frontdesk.archivist.have_glass', 'tunic.drycleaner.frontdesk.worker.hub', 'tunic.historicalsociety.closet_dirty.gramps.news', 'tunic.humanecology.frontdesk.worker.intro', 'tunic.historicalsociety.frontdesk.archivist_glasses.confrontation', 'tunic.historicalsociety.basement.seescratches', 'tunic.historicalsociety.collection.cs', 'tunic.flaghouse.entry.flag_girl.hello', 'tunic.historicalsociety.collection.gramps.found', 'tunic.historicalsociety.basement.ch3start', 'tunic.historicalsociety.entry.groupconvo_flag', 'tunic.library.frontdesk.worker.hello', 'tunic.library.frontdesk.worker.wells', 'tunic.historicalsociety.collection_flag.gramps.flag', 'tunic.historicalsociety.basement.savedteddy', 'tunic.library.frontdesk.worker.nelson', 'tunic.wildlife.center.expert.removed_cup', 'tunic.library.frontdesk.worker.flag', 'tunic.historicalsociety.frontdesk.archivist.hello', 'tunic.historicalsociety.closet.gramps.intro_0_cs_0', 'tunic.historicalsociety.entry.boss.flag', 'tunic.flaghouse.entry.flag_girl.symbol', 'tunic.historicalsociety.closet_dirty.trigger_scarf', 'tunic.drycleaner.frontdesk.worker.done', 'tunic.historicalsociety.closet_dirty.what_happened', 'tunic.wildlife.center.wells.animals', 'tunic.historicalsociety.closet.teddy.intro_0_cs_0', 'tunic.historicalsociety.cage.glasses.afterteddy', 'tunic.historicalsociety.cage.teddy.trapped', 'tunic.historicalsociety.cage.unlockdoor', 'tunic.historicalsociety.stacks.journals.pic_2.bingo', 'tunic.historicalsociety.entry.wells.flag', 'tunic.humanecology.frontdesk.worker.badger', 'tunic.historicalsociety.stacks.journals_flag.pic_0.bingo', 'tunic.historicalsociety.closet.intro', 'tunic.historicalsociety.closet.retirement_letter.hub', 'tunic.historicalsociety.entry.directory.closeup.archivist', 'tunic.historicalsociety.collection.tunic.slip', 'tunic.kohlcenter.halloffame.plaque.face.date', 'tunic.historicalsociety.closet_dirty.trigger_coffee', 'tunic.drycleaner.frontdesk.logbook.page.bingo', 'tunic.library.microfiche.reader.paper2.bingo', 'tunic.kohlcenter.halloffame.togrampa', 'tunic.capitol_2.hall.boss.haveyougotit', 'tunic.wildlife.center.wells.nodeer_recap', 'tunic.historicalsociety.cage.glasses.beforeteddy', 'tunic.historicalsociety.closet_dirty.gramps.helpclean', 'tunic.wildlife.center.expert.recap', 'tunic.historicalsociety.frontdesk.archivist.have_glass_recap', 'tunic.historicalsociety.stacks.journals_flag.pic_1.bingo', 'tunic.historicalsociety.cage.lockeddoor', 'tunic.historicalsociety.stacks.journals_flag.pic_2.bingo', 'tunic.historicalsociety.collection.gramps.lost', 'tunic.historicalsociety.closet.notebook', 'tunic.historicalsociety.frontdesk.magnify', 'tunic.humanecology.frontdesk.businesscards.card_bingo.bingo', 'tunic.wildlife.center.remove_cup', 'tunic.library.frontdesk.wellsbadge.hub', 'tunic.wildlife.center.tracks.hub.deer', 'tunic.historicalsociety.frontdesk.key', 'tunic.library.microfiche.reader_flag.paper2.bingo', 'tunic.flaghouse.entry.colorbook', 'tunic.wildlife.center.coffee', 'tunic.capitol_1.hall.boss.haveyougotit', 'tunic.historicalsociety.basement.janitor', 'tunic.historicalsociety.collection_flag.gramps.recap', 'tunic.wildlife.center.wells.animals2', 'tunic.flaghouse.entry.flag_girl.symbol_recap', 'tunic.historicalsociety.closet_dirty.photo', 'tunic.historicalsociety.stacks.outtolunch', 'tunic.library.frontdesk.worker.wells_recap', 'tunic.historicalsociety.frontdesk.archivist_glasses.confrontation_recap', 'tunic.capitol_0.hall.boss.talktogramps', 'tunic.historicalsociety.closet.photo', 'tunic.historicalsociety.collection.tunic', 'tunic.historicalsociety.closet.teddy.intro_0_cs_5', 'tunic.historicalsociety.closet_dirty.gramps.archivist', 'tunic.historicalsociety.closet_dirty.door_block_talk', 'tunic.historicalsociety.entry.boss.flag_recap', 'tunic.historicalsociety.frontdesk.archivist.need_glass_0', 'tunic.historicalsociety.entry.wells.talktogramps', 'tunic.historicalsociety.frontdesk.block_magnify', 'tunic.historicalsociety.frontdesk.archivist.foundtheodora', 'tunic.historicalsociety.closet_dirty.gramps.nothing', 'tunic.historicalsociety.closet_dirty.door_block_clean', 'tunic.capitol_1.hall.boss.writeitup', 'tunic.library.frontdesk.worker.nelson_recap', 'tunic.library.frontdesk.worker.hello_short', 'tunic.historicalsociety.stacks.block', 'tunic.historicalsociety.frontdesk.archivist.need_glass_1', 'tunic.historicalsociety.entry.boss.talktogramps', 'tunic.historicalsociety.frontdesk.archivist.newspaper_recap', 'tunic.historicalsociety.entry.wells.flag_recap', 'tunic.drycleaner.frontdesk.worker.done2', 'tunic.library.frontdesk.worker.flag_recap', 'tunic.humanecology.frontdesk.block_0', 'tunic.library.frontdesk.worker.preflag', 'tunic.historicalsociety.basement.gramps.seeyalater', 'tunic.flaghouse.entry.flag_girl.hello_recap', 'tunic.historicalsociety.closet.doorblock', 'tunic.drycleaner.frontdesk.worker.takealook', 'tunic.historicalsociety.basement.gramps.whatdo', 'tunic.library.frontdesk.worker.droppedbadge', 'tunic.historicalsociety.entry.block_tomap2', 'tunic.library.frontdesk.block_nelson', 'tunic.library.microfiche.block_0', 'tunic.historicalsociety.entry.block_tocollection', 'tunic.historicalsociety.entry.block_tomap1', 'tunic.historicalsociety.collection.gramps.look_0', 'tunic.library.frontdesk.block_badge', 'tunic.historicalsociety.cage.need_glasses', 'tunic.library.frontdesk.block_badge_2', 'tunic.kohlcenter.halloffame.block_0', 'tunic.capitol_0.hall.chap1_finale_c', 'tunic.capitol_1.hall.chap2_finale_c', 'tunic.capitol_2.hall.chap4_finale_c', 'tunic.wildlife.center.fox.concern', 'tunic.drycleaner.frontdesk.block_0', 'tunic.historicalsociety.entry.gramps.hub', 'tunic.humanecology.frontdesk.block_1', 'tunic.drycleaner.frontdesk.block_1']\nroom_lists = ['tunic.historicalsociety.entry', 'tunic.wildlife.center', 'tunic.historicalsociety.cage', 'tunic.library.frontdesk', 'tunic.historicalsociety.frontdesk', 'tunic.historicalsociety.stacks', 'tunic.historicalsociety.closet_dirty', 'tunic.humanecology.frontdesk', 'tunic.historicalsociety.basement', 'tunic.kohlcenter.halloffame', 'tunic.library.microfiche', 'tunic.drycleaner.frontdesk', 'tunic.historicalsociety.collection', 'tunic.historicalsociety.closet', 'tunic.flaghouse.entry', 'tunic.historicalsociety.collection_flag', 'tunic.capitol_1.hall', 'tunic.capitol_0.hall', 'tunic.capitol_2.hall']\n\n\nLEVELS = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22]\nlevel_groups = [\"0-4\", \"5-12\", \"13-22\"]\ndef feature_engineer(x, grp, use_extra, feature_suffix):\n    aggs = [\n        pl.col(\"index\").count().alias(f\"session_number_{feature_suffix}\"),\n\n        *[pl.col('index').filter(pl.col('text').str.contains(c)).count().alias(f'word_{c}') for c in DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text').str.contains(c))).mean().alias(f'word_mean_{c}') for c in DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text').str.contains(c))).std().alias(f'word_std_{c}') for c in DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col('text').str.contains(c)).max().alias(f'word_max_{c}') for c in DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col('text').str.contains(c)).sum().alias(f'word_sum_{c}') for c in DIALOGS],\n\n        *[pl.col(c).drop_nulls().n_unique().alias(f\"{c}_unique_{feature_suffix}\") for c in CATS],\n\n        *[pl.col(c).quantile(0.1, \"nearest\").alias(f\"{c}_quantile1_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).quantile(0.2, \"nearest\").alias(f\"{c}_quantile2_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).quantile(0.4, \"nearest\").alias(f\"{c}_quantile4_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).quantile(0.6, \"nearest\").alias(f\"{c}_quantile6_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).quantile(0.8, \"nearest\").alias(f\"{c}_quantile8_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).quantile(0.9, \"nearest\").alias(f\"{c}_quantile9_{feature_suffix}\") for c in NUMS],\n\n        *[pl.col(c).mean().alias(f\"{c}_mean_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).std().alias(f\"{c}_std_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).min().alias(f\"{c}_min_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).max().alias(f\"{c}_max_{feature_suffix}\") for c in NUMS],\n\n        *[pl.col(\"fqid\").filter(pl.col(\"fqid\") == c).count().alias(f\"{c}_fqid_counts{feature_suffix}\")\n          for c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for\n          c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in fqid_lists],\n\n        *[pl.col(\"text_fqid\").filter(pl.col(\"text_fqid\") == c).count().alias(f\"{c}_text_fqid_counts{feature_suffix}\")\n          for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in text_lists],\n\n        *[pl.col(\"room_fqid\").filter(pl.col(\"room_fqid\") == c).count().alias(f\"{c}_room_fqid_counts{feature_suffix}\")\n          for c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for\n          c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in room_lists],\n\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).quantile(0.1, \"nearest\").alias(\n            f\"{c}_ET_quantile1_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).quantile(0.2, \"nearest\").alias(\n            f\"{c}_ET_quantile2_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).quantile(0.4, \"nearest\").alias(\n            f\"{c}_ET_quantile4_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).quantile(0.6, \"nearest\").alias(\n            f\"{c}_ET_quantile6_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).quantile(0.8, \"nearest\").alias(\n            f\"{c}_ET_quantile8_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).quantile(0.9, \"nearest\").alias(\n            f\"{c}_ET_quantile9_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"event_name\").filter(pl.col(\"event_name\") == c).count().alias(f\"{c}_event_name_counts{feature_suffix}\")\n          for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\")\n          for\n          c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in event_name_feature],\n\n        *[pl.col(\"name\").filter(pl.col(\"name\") == c).count().alias(f\"{c}_name_counts{feature_suffix}\") for c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for c in\n          name_feature],\n\n        *[pl.col(\"level\").filter(pl.col(\"level\") == c).count().alias(f\"{c}_LEVEL_count{feature_suffix}\") for c in\n          LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for c in\n          LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for c\n          in\n          LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for c in\n          LEVELS],\n\n        *[pl.col(\"level_group\").filter(pl.col(\"level_group\") == c).count().alias(\n            f\"{c}_LEVEL_group_count{feature_suffix}\") for c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\")\n          for c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in\n          level_groups],\n\n        *[pl.col(\"index\").filter((pl.col(\"level\") == c) & (pl.col('room_fqid') == d)).count().alias(\n            f\"{c}{d}_level_room_count{feature_suffix}\") for c in LEVELS for d in room_lists],\n\n    ]\n\n    df = x.groupby(['session_id'], maintain_order=True).agg(aggs).sort(\"session_id\")\n\n    if use_extra:\n        if grp == '5-12':\n            aggs = [\n                pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"Here's the log book.\")\n                                              | (pl.col(\"fqid\") == 'logbook.page.bingo'))\n                    .apply(lambda s: s.max() - s.min()).alias(\"logbook_bingo_duration\"),\n                pl.col(\"index\").filter(\n                    (pl.col(\"text\") == \"Here's the log book.\") | (pl.col(\"fqid\") == 'logbook.page.bingo')).apply(\n                    lambda s: s.max() - s.min()).alias(\"logbook_bingo_indexCount\"),\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader')) | (\n                            pl.col(\"fqid\") == \"reader.paper2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\n                    \"reader_bingo_duration\"),\n                pl.col(\"index\").filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader')) | (\n                        pl.col(\"fqid\") == \"reader.paper2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\n                    \"reader_bingo_indexCount\"),\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals')) | (\n                            pl.col(\"fqid\") == \"journals.pic_2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\n                    \"journals_bingo_duration\"),\n                pl.col(\"index\").filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals')) | (\n                        pl.col(\"fqid\") == \"journals.pic_2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\n                    \"journals_bingo_indexCount\"),\n            ]\n            tmp = x.groupby([\"session_id\"], maintain_order=True).agg(aggs).sort(\"session_id\")\n            df = df.join(tmp, on=\"session_id\", how='left')\n\n        if grp == '13-22':\n            aggs = [\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader_flag')) | (\n                            pl.col(\"fqid\") == \"tunic.library.microfiche.reader_flag.paper2.bingo\")).apply(\n                    lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"reader_flag_duration\"),\n                pl.col(\"index\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader_flag')) | (\n                            pl.col(\"fqid\") == \"tunic.library.microfiche.reader_flag.paper2.bingo\")).apply(\n                    lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"reader_flag_indexCount\"),\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals_flag')) | (\n                            pl.col(\"fqid\") == \"journals_flag.pic_0.bingo\")).apply(\n                    lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"journalsFlag_bingo_duration\"),\n                pl.col(\"index\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals_flag')) | (\n                            pl.col(\"fqid\") == \"journals_flag.pic_0.bingo\")).apply(\n                    lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"journalsFlag_bingo_indexCount\")\n            ]\n            tmp = x.groupby([\"session_id\"], maintain_order=True).agg(aggs).sort(\"session_id\")\n            df = df.join(tmp, on=\"session_id\", how='left')\n\n    return df.to_pandas()\n\ndef time_feature(train):\n    train[\"year\"] = train[\"session_id\"].apply(lambda x: int(str(x)[:2])).astype(np.uint8)\n    train[\"month\"] = train[\"session_id\"].apply(lambda x: int(str(x)[2:4])+1).astype(np.uint8)\n    train[\"day\"] = train[\"session_id\"].apply(lambda x: int(str(x)[4:6])).astype(np.uint8)\n    train[\"hour\"] = train[\"session_id\"].apply(lambda x: int(str(x)[6:8])).astype(np.uint8)\n    train[\"minute\"] = train[\"session_id\"].apply(lambda x: int(str(x)[8:10])).astype(np.uint8)\n    train[\"second\"] = train[\"session_id\"].apply(lambda x: int(str(x)[10:12])).astype(np.uint8)\n\n\n    return train","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import jo_wilder\ntry:\n    jo_wilder.make_env.__called__ = False\n    env.__called__ = False\n    type(env)._state = type(type(env)._state).__dict__['INIT']\nexcept:\n    pass\n\nenv = jo_wilder.make_env()\niter_test = env.iter_test() ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"limits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\nfor (sample_submission, test) in iter_test:\n    grp = test.level_group.values[0]\n    session_id = test.session_id.values[0]\n\n    \n    df = (pl.from_pandas(test)\n          .drop([\"fullscreen\", \"hq\", \"music\"])\n          .with_columns(columns))\n    df = feature_engineer(df, grp, use_extra=True, feature_suffix='')\n    df = time_feature(df)\n    \n    df = create_emb_df(df)\n    \n    fold = 0\n    preds = []\n    a,b = limits[grp]\n    for q in range(a, b):\n        print(q)\n            FEATURES = importance_dict[str(q)]\n            model = models_list[q-1][fold]\n            pred = model.predict_proba(df[FEATURES].astype(np.float32))[0,1]\n            \n        preds.append(int(pred > 0.625))\n\n    sample_submission[\"correct\"] = preds\n\n    env.predict(sample_submission)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('submission.csv')\nprint(sub.shape, sub.correct.mean())\nsub.head()","metadata":{},"execution_count":null,"outputs":[]}]}