{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd, numpy as np\nfrom catboost import CatBoostClassifier\nimport pickle\nimport sys\n\nimport numpy as np\nimport pandas as pd\nimport os\nimport pyarrow.parquet as pq\nimport pyarrow.dataset as ds\nfrom torch.utils.data import DataLoader,SubsetRandomSampler,Dataset\nfrom torch.utils.tensorboard import SummaryWriter\nimport torch\nimport torch.nn as nn\nfrom torch import optim\npd.options.mode.chained_assignment = None\nfrom IPython.display import clear_output\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nimport copy\nimport json\nimport imp\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import f1_score\nimport pandas as pd\nimport pandasql as ps\nimport gc\n\nimport warnings\nimport pickle\nimport polars as pl\n\nfrom collections import defaultdict\nfrom itertools import combinations\nimport pyarrow as pa\n\nimport lightgbm as lgb\nfrom lightgbm import LGBMClassifier\nfrom lightgbm import Booster\nfrom lightgbm import early_stopping\nfrom lightgbm import log_evaluation\n\nfrom sklearn.model_selection import GroupKFold, KFold, train_test_split\nfrom sklearn.metrics import roc_auc_score, f1_score\n\nimport matplotlib.pyplot as plt\nfrom colorama import Fore, Back, Style\n\nfrom tqdm.auto import tqdm\nimport sys\nimport datetime\n\nfrom catboost import CatBoostClassifier, Pool\nfrom sklearn.model_selection import KFold, GroupKFold\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import f1_score\n","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:22:59.704675Z","iopub.execute_input":"2023-06-03T07:22:59.705091Z","iopub.status.idle":"2023-06-03T07:22:59.717219Z","shell.execute_reply.started":"2023-06-03T07:22:59.70505Z","shell.execute_reply":"2023-06-03T07:22:59.71603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dic","metadata":{}},{"cell_type":"code","source":"text_replace_dic = pd.read_csv( '/kaggle/input/competition-game-data-dic/text_replace_dic.csv',delimiter=';')\nscript_version_dic = pd.read_csv( '/kaggle/input/competition-game-data-dic/script_version_dic.csv',delimiter=';') ","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:22:59.726627Z","iopub.execute_input":"2023-06-03T07:22:59.727759Z","iopub.status.idle":"2023-06-03T07:22:59.745469Z","shell.execute_reply.started":"2023-06-03T07:22:59.72769Z","shell.execute_reply":"2023-06-03T07:22:59.74413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# fichi","metadata":{}},{"cell_type":"code","source":"# First model (Catboost)\n\n#CATS = ['event_name', 'name', 'fqid', 'room_fqid', 'text_fqid']\n#NUMS = ['page', 'room_coor_x', 'room_coor_y', 'screen_coor_x', 'screen_coor_y',\n#        'hover_duration', 'elapsed_time_diff']\nCATS = ['event_name', 'name', 'fqid', 'room_fqid', 'text_fqid', 'script_version'  ]\nNUMS = ['page', 'room_coor_x', 'room_coor_y', 'screen_coor_x', 'screen_coor_y',\n        'hover_duration', 'elapsed_time_diff', 'delta_index_0_4', 'delta_index_5_12']\n\nDIALOGS = ['that', 'this', 'it', 'you','find','found','Found','notebook','Wells','wells','help','need', 'Oh','Ooh','Jo', 'flag', 'can','and','is','the','to']\n\nname_feature = ['basic', 'undefined', 'close', 'open', 'prev', 'next']\nevent_name_feature = ['cutscene_click', 'person_click', 'navigate_click',\n       'observation_click', 'notification_click', 'object_click',\n       'object_hover', 'map_hover', 'map_click', 'checkpoint',\n       'notebook_click']\n\n\nsub_fqid_lists = {'0-4': ['gramps',\n 'wells',\n 'toentry',\n 'groupconvo',\n 'tomap',\n 'tostacks',\n 'tobasement',\n 'boss',\n 'cs',\n 'teddy',\n 'tunic.historicalsociety',\n 'plaque',\n 'directory',\n 'tunic',\n 'tunic.kohlcenter',\n 'plaque.face.date',\n 'notebook',\n 'tunic.hub.slip',\n 'tocollection',\n 'tunic.capitol_0',\n 'photo',\n 'intro',\n 'retirement_letter',\n 'togrampa',\n 'janitor',\n 'chap1_finale',\n 'report',\n 'outtolunch',\n 'chap1_finale_c',\n 'block_0',\n 'doorblock',\n 'tocloset',\n 'block_tomap2',\n 'block_tocollection',\n 'block_tomap1'],\n                  '5-12': ['worker',\n 'archivist',\n 'gramps',\n 'toentry',\n 'tomap',\n 'tostacks',\n 'tobasement',\n 'boss',\n 'journals',\n 'businesscards',\n 'tunic.historicalsociety',\n 'tofrontdesk',\n 'plaque',\n 'tunic.drycleaner',\n 'tunic.library',\n 'trigger_scarf',\n 'reader',\n 'directory',\n 'tunic.capitol_1',\n 'journals.pic_0.next',\n 'tunic',\n 'what_happened',\n 'tunic.kohlcenter',\n 'tunic.humanecology',\n 'logbook',\n 'businesscards.card_0.next',\n 'journals.hub.topics',\n 'logbook.page.bingo',\n 'journals.pic_1.next',\n 'reader.paper0.next',\n 'trigger_coffee',\n 'wellsbadge',\n 'journals.pic_2.next',\n 'tomicrofiche',\n 'tocloset_dirty',\n 'businesscards.card_bingo.bingo',\n 'businesscards.card_1.next',\n 'tunic.hub.slip',\n 'journals.pic_2.bingo',\n 'tocollection',\n 'chap2_finale_c',\n 'tunic.capitol_0',\n 'photo',\n 'reader.paper1.next',\n 'businesscards.card_bingo.next',\n 'reader.paper2.bingo',\n 'magnify',\n 'janitor',\n 'tohallway',\n 'outtolunch',\n 'reader.paper2.next',\n 'door_block_talk',\n 'block_magnify',\n 'reader.paper0.prev',\n 'block',\n 'block_0',\n 'door_block_clean',\n 'reader.paper2.prev',\n 'reader.paper1.prev',\n 'block_badge',\n 'block_badge_2',\n 'block_1'],\n                  '13-22': ['worker',\n 'gramps',\n 'wells',\n 'toentry',\n 'confrontation',\n 'crane_ranger',\n 'flag_girl',\n 'tomap',\n 'tostacks',\n 'tobasement',\n 'archivist_glasses',\n 'boss',\n 'journals',\n 'seescratches',\n 'groupconvo_flag',\n 'teddy',\n 'expert',\n 'businesscards',\n 'ch3start',\n 'tunic.historicalsociety',\n 'tofrontdesk',\n 'savedteddy',\n 'plaque',\n 'glasses',\n 'tunic.drycleaner',\n 'reader_flag',\n 'tunic.library',\n 'tracks',\n 'tunic.capitol_2',\n 'reader',\n 'directory',\n 'tunic.capitol_1',\n 'journals.pic_0.next',\n 'unlockdoor',\n 'tunic',\n 'tunic.kohlcenter',\n 'tunic.humanecology',\n 'colorbook',\n 'logbook',\n 'businesscards.card_0.next',\n 'journals.hub.topics',\n 'journals.pic_1.next',\n 'journals_flag',\n 'reader.paper0.next',\n 'tracks.hub.deer',\n 'reader_flag.paper0.next',\n 'journals.pic_2.next',\n 'tomicrofiche',\n 'journals_flag.pic_0.bingo',\n 'tocloset_dirty',\n 'businesscards.card_1.next',\n 'tunic.wildlife',\n 'tunic.hub.slip',\n 'tocage',\n 'journals.pic_2.bingo',\n 'tocollectionflag',\n 'tocollection',\n 'chap4_finale_c',\n 'lockeddoor',\n 'journals_flag.hub.topics',\n 'reader_flag.paper2.bingo',\n 'photo',\n 'tunic.flaghouse',\n 'reader.paper1.next',\n 'directory.closeup.archivist',\n 'businesscards.card_bingo.next',\n 'remove_cup',\n 'journals_flag.pic_0.next',\n 'coffee',\n 'key',\n 'reader_flag.paper1.next',\n 'tohallway',\n 'outtolunch',\n 'journals_flag.hub.topics_old',\n 'journals_flag.pic_1.next',\n 'reader.paper2.next',\n 'reader_flag.paper2.next',\n 'journals_flag.pic_1.bingo',\n 'journals_flag.pic_2.next',\n 'journals_flag.pic_2.bingo',\n 'reader.paper0.prev',\n 'reader_flag.paper0.prev',\n 'reader.paper2.prev',\n 'reader.paper1.prev',\n 'reader_flag.paper2.prev',\n 'reader_flag.paper1.prev',\n 'journals_flag.pic_0_old.next',\n 'journals_flag.pic_1_old.next',\n 'block_nelson',\n 'journals_flag.pic_2_old.next',\n 'need_glasses',\n 'fox'],\n                 }\n\nsub_room_lists = {'0-4': ['tunic.historicalsociety.entry',\n 'tunic.historicalsociety.stacks',\n 'tunic.historicalsociety.basement',\n 'tunic.kohlcenter.halloffame',\n 'tunic.historicalsociety.collection',\n 'tunic.historicalsociety.closet',\n 'tunic.capitol_0.hall'],\n                  '5-12': ['tunic.historicalsociety.entry',\n 'tunic.library.frontdesk',\n 'tunic.historicalsociety.frontdesk',\n 'tunic.historicalsociety.stacks',\n 'tunic.historicalsociety.closet_dirty',\n 'tunic.humanecology.frontdesk',\n 'tunic.historicalsociety.basement',\n 'tunic.kohlcenter.halloffame',\n 'tunic.library.microfiche',\n 'tunic.drycleaner.frontdesk',\n 'tunic.historicalsociety.collection',\n 'tunic.capitol_1.hall',\n 'tunic.capitol_0.hall'],\n                  '13-22': ['tunic.historicalsociety.entry',\n 'tunic.wildlife.center',\n 'tunic.historicalsociety.cage',\n 'tunic.library.frontdesk',\n 'tunic.historicalsociety.frontdesk',\n 'tunic.historicalsociety.stacks',\n 'tunic.historicalsociety.closet_dirty',\n 'tunic.humanecology.frontdesk',\n 'tunic.historicalsociety.basement',\n 'tunic.kohlcenter.halloffame',\n 'tunic.library.microfiche',\n 'tunic.drycleaner.frontdesk',\n 'tunic.historicalsociety.collection',\n 'tunic.flaghouse.entry',\n 'tunic.historicalsociety.collection_flag',\n 'tunic.capitol_1.hall',\n 'tunic.capitol_2.hall'],\n                 }\n\n\nsub_text_lists = {'0-4': ['tunic.historicalsociety.entry.groupconvo',\n 'tunic.historicalsociety.collection.cs',\n 'tunic.historicalsociety.collection.gramps.found',\n 'tunic.historicalsociety.closet.gramps.intro_0_cs_0',\n 'tunic.historicalsociety.closet.teddy.intro_0_cs_0',\n 'tunic.historicalsociety.closet.intro',\n 'tunic.historicalsociety.closet.retirement_letter.hub',\n 'tunic.historicalsociety.collection.tunic.slip',\n 'tunic.kohlcenter.halloffame.plaque.face.date',\n 'tunic.kohlcenter.halloffame.togrampa',\n 'tunic.historicalsociety.collection.gramps.lost',\n 'tunic.historicalsociety.closet.notebook',\n 'tunic.historicalsociety.basement.janitor',\n 'tunic.historicalsociety.stacks.outtolunch',\n 'tunic.historicalsociety.closet.photo',\n 'tunic.historicalsociety.collection.tunic',\n 'tunic.historicalsociety.closet.teddy.intro_0_cs_5',\n 'tunic.historicalsociety.entry.wells.talktogramps',\n 'tunic.historicalsociety.entry.boss.talktogramps',\n 'tunic.historicalsociety.closet.doorblock',\n 'tunic.historicalsociety.entry.block_tomap2',\n 'tunic.historicalsociety.entry.block_tocollection',\n 'tunic.historicalsociety.entry.block_tomap1',\n 'tunic.historicalsociety.collection.gramps.look_0',\n 'tunic.kohlcenter.halloffame.block_0',\n 'tunic.capitol_0.hall.chap1_finale_c',\n 'tunic.historicalsociety.entry.gramps.hub'],\n               '5-12': ['tunic.historicalsociety.frontdesk.archivist.newspaper',\n 'tunic.historicalsociety.frontdesk.archivist.have_glass',\n 'tunic.drycleaner.frontdesk.worker.hub',\n 'tunic.historicalsociety.closet_dirty.gramps.news',\n 'tunic.humanecology.frontdesk.worker.intro',\n 'tunic.library.frontdesk.worker.hello',\n 'tunic.library.frontdesk.worker.wells',\n 'tunic.historicalsociety.frontdesk.archivist.hello',\n 'tunic.historicalsociety.closet_dirty.trigger_scarf',\n 'tunic.drycleaner.frontdesk.worker.done',\n 'tunic.historicalsociety.closet_dirty.what_happened',\n 'tunic.historicalsociety.stacks.journals.pic_2.bingo',\n 'tunic.humanecology.frontdesk.worker.badger',\n 'tunic.historicalsociety.closet_dirty.trigger_coffee',\n 'tunic.drycleaner.frontdesk.logbook.page.bingo',\n 'tunic.library.microfiche.reader.paper2.bingo',\n 'tunic.historicalsociety.closet_dirty.gramps.helpclean',\n 'tunic.historicalsociety.frontdesk.archivist.have_glass_recap',\n 'tunic.historicalsociety.frontdesk.magnify',\n 'tunic.humanecology.frontdesk.businesscards.card_bingo.bingo',\n 'tunic.library.frontdesk.wellsbadge.hub',\n 'tunic.capitol_1.hall.boss.haveyougotit',\n 'tunic.historicalsociety.basement.janitor',\n 'tunic.historicalsociety.closet_dirty.photo',\n 'tunic.historicalsociety.stacks.outtolunch',\n 'tunic.library.frontdesk.worker.wells_recap',\n 'tunic.capitol_0.hall.boss.talktogramps',\n 'tunic.historicalsociety.closet_dirty.gramps.archivist',\n 'tunic.historicalsociety.closet_dirty.door_block_talk',\n 'tunic.historicalsociety.frontdesk.archivist.need_glass_0',\n 'tunic.historicalsociety.frontdesk.block_magnify',\n 'tunic.historicalsociety.frontdesk.archivist.foundtheodora',\n 'tunic.historicalsociety.closet_dirty.gramps.nothing',\n 'tunic.historicalsociety.closet_dirty.door_block_clean',\n 'tunic.library.frontdesk.worker.hello_short',\n 'tunic.historicalsociety.stacks.block',\n 'tunic.historicalsociety.frontdesk.archivist.need_glass_1',\n 'tunic.historicalsociety.frontdesk.archivist.newspaper_recap',\n 'tunic.drycleaner.frontdesk.worker.done2',\n 'tunic.humanecology.frontdesk.block_0',\n 'tunic.library.frontdesk.worker.preflag',\n 'tunic.drycleaner.frontdesk.worker.takealook',\n 'tunic.library.frontdesk.worker.droppedbadge',\n 'tunic.library.microfiche.block_0',\n 'tunic.library.frontdesk.block_badge',\n 'tunic.library.frontdesk.block_badge_2',\n 'tunic.capitol_1.hall.chap2_finale_c',\n 'tunic.drycleaner.frontdesk.block_0',\n 'tunic.humanecology.frontdesk.block_1',\n 'tunic.drycleaner.frontdesk.block_1'],\n               '13-22': ['tunic.historicalsociety.cage.confrontation',\n 'tunic.wildlife.center.crane_ranger.crane',\n 'tunic.wildlife.center.wells.nodeer',\n 'tunic.historicalsociety.frontdesk.archivist_glasses.confrontation',\n 'tunic.historicalsociety.basement.seescratches',\n 'tunic.flaghouse.entry.flag_girl.hello',\n 'tunic.historicalsociety.basement.ch3start',\n 'tunic.historicalsociety.entry.groupconvo_flag',\n 'tunic.historicalsociety.collection_flag.gramps.flag',\n 'tunic.historicalsociety.basement.savedteddy',\n 'tunic.library.frontdesk.worker.nelson',\n 'tunic.wildlife.center.expert.removed_cup',\n 'tunic.library.frontdesk.worker.flag',\n 'tunic.historicalsociety.entry.boss.flag',\n 'tunic.flaghouse.entry.flag_girl.symbol',\n 'tunic.wildlife.center.wells.animals',\n 'tunic.historicalsociety.cage.glasses.afterteddy',\n 'tunic.historicalsociety.cage.teddy.trapped',\n 'tunic.historicalsociety.cage.unlockdoor',\n 'tunic.historicalsociety.stacks.journals.pic_2.bingo',\n 'tunic.historicalsociety.entry.wells.flag',\n 'tunic.humanecology.frontdesk.worker.badger',\n 'tunic.historicalsociety.stacks.journals_flag.pic_0.bingo',\n 'tunic.historicalsociety.entry.directory.closeup.archivist',\n 'tunic.capitol_2.hall.boss.haveyougotit',\n 'tunic.wildlife.center.wells.nodeer_recap',\n 'tunic.historicalsociety.cage.glasses.beforeteddy',\n 'tunic.wildlife.center.expert.recap',\n 'tunic.historicalsociety.stacks.journals_flag.pic_1.bingo',\n 'tunic.historicalsociety.cage.lockeddoor',\n 'tunic.historicalsociety.stacks.journals_flag.pic_2.bingo',\n 'tunic.wildlife.center.remove_cup',\n 'tunic.wildlife.center.tracks.hub.deer',\n 'tunic.historicalsociety.frontdesk.key',\n 'tunic.library.microfiche.reader_flag.paper2.bingo',\n 'tunic.flaghouse.entry.colorbook',\n 'tunic.wildlife.center.coffee',\n 'tunic.historicalsociety.collection_flag.gramps.recap',\n 'tunic.wildlife.center.wells.animals2',\n 'tunic.flaghouse.entry.flag_girl.symbol_recap',\n 'tunic.historicalsociety.closet_dirty.photo',\n 'tunic.historicalsociety.stacks.outtolunch',\n 'tunic.historicalsociety.frontdesk.archivist_glasses.confrontation_recap',\n 'tunic.historicalsociety.entry.boss.flag_recap',\n 'tunic.capitol_1.hall.boss.writeitup',\n 'tunic.library.frontdesk.worker.nelson_recap',\n 'tunic.historicalsociety.entry.wells.flag_recap',\n 'tunic.drycleaner.frontdesk.worker.done2',\n 'tunic.library.frontdesk.worker.flag_recap',\n 'tunic.library.frontdesk.worker.preflag',\n 'tunic.historicalsociety.basement.gramps.seeyalater',\n 'tunic.flaghouse.entry.flag_girl.hello_recap',\n 'tunic.historicalsociety.basement.gramps.whatdo',\n 'tunic.library.frontdesk.block_nelson',\n 'tunic.historicalsociety.cage.need_glasses',\n 'tunic.capitol_2.hall.chap4_finale_c',\n 'tunic.wildlife.center.fox.concern']\n              }\n\n\nSUB_LEVELS = {'0-4': [1, 2, 3, 4],\n              '5-12': [5, 6, 7, 8, 9, 10, 11, 12],\n              '13-22': [13, 14, 15, 16, 17, 18, 19, 20, 21, 22]}\nlevel_groups = [\"0-4\", \"5-12\", \"13-22\"]","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:22:59.773891Z","iopub.execute_input":"2023-06-03T07:22:59.77429Z","iopub.status.idle":"2023-06-03T07:22:59.805619Z","shell.execute_reply.started":"2023-06-03T07:22:59.774255Z","shell.execute_reply":"2023-06-03T07:22:59.804349Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fichi_1_2_3_4_python(train_row): \n\n    #print(\"#1 'script_version'\")\n    #1 'script_version'\n    train_row_temp = train_row[['session_id', 'index', 'text' , 'level_group'  ]].copy()\n    train_row_temp = train_row_temp.merge(script_version_dic, on='text')\n    \n    train_row_temp = train_row_temp[train_row_temp[['script_version']].notnull().all(1)]\n    train_row_temp = train_row_temp.groupby(['session_id','script_version'])['session_id'].count().reset_index(name=\"cnt\")\n    \n    train_row_temp['num_row'] = (train_row_temp\n      .sort_values(by=['cnt'],\n                   ascending=False) \n      .groupby('session_id', sort=False).cumcount().add(1)\n    )\n    train_row_temp = train_row_temp.loc[train_row_temp.num_row == 1] \n    train_row_temp = train_row_temp[['session_id','script_version']]\n    train_row = train_row.merge(train_row_temp, on='session_id')\n    del train_row_temp\n    gc.collect()\n    \n    #print(\"#2 3 'delta_index_0_4'   'delta_index_5_12'\")\n    #2 3 'delta_index_0_4'   'delta_index_5_12'\n    tmp = train_row[['session_id','index','level_group']]\n    f = lambda x: x.diff()\n       \n    tmp['delta_index'] = tmp.sort_values(['session_id','index']).groupby(['session_id'])['index'].transform(f)\n    tmp = tmp[tmp['delta_index'] >1]\n\n   \n    tmp5_12  = tmp[tmp['level_group'] == '5-12' ].groupby( ['session_id'   ]).agg({'delta_index':'max'}) .copy() \n    tmp5_12  = tmp5_12.rename(columns={\"delta_index\": \"delta_index_0_4\" })\n           \n    tmp5_12_1=tmp5_12.copy()\n    tmp5_12_2=tmp5_12.copy()    \n    tmp5_12_1[\"level_group\"]='5-12'\n    tmp5_12_2[\"level_group\"]='13-22'    \n    tmp5_12 = tmp5_12_1.append(tmp5_12_2)      \n    train_row = train_row.merge( tmp5_12, how = 'left', left_on=[ 'session_id', \"level_group\"] , right_on=[ 'session_id',\"level_group\"] ).copy()    \n    \n    del  tmp5_12, tmp5_12_1, tmp5_12_2 \n    gc.collect()    \n          \n    tmp13_22  = tmp[tmp['level_group'] == '13-22' ].groupby( ['session_id'   ]).agg({'delta_index':'max'}) .copy()      \n    tmp13_22 = tmp13_22.rename(columns={\"delta_index\": \"delta_index_5_12\" })\n    \n      \n    tmp13_22[\"level_group\"]='13-22'   \n    train_row = train_row.merge( tmp13_22, how = 'left', left_on=[ 'session_id',\"level_group\"] , right_on=['session_id', \"level_group\"] ).copy() \n\n      \n    del tmp, tmp13_22\n    gc.collect()\n    \n    #print(\"# 4 'text_replace'\")\n    # 4 'text_replace'\n    train_row = pd.merge(train_row, text_replace_dic, how = 'left', left_on=[\"text\"] , right_on=[\"text\"] ).copy()\n    train_row[\"text_replace\"] = train_row[\"text_replace\"].combine_first( train_row[\"text\"])    \n    \n    \n    return train_row","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:22:59.808011Z","iopub.execute_input":"2023-06-03T07:22:59.808368Z","iopub.status.idle":"2023-06-03T07:22:59.824303Z","shell.execute_reply.started":"2023-06-03T07:22:59.808335Z","shell.execute_reply":"2023-06-03T07:22:59.823158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fichi_1_2_3_4(train_row): \n    print('fichi_1_2_3_4..')     \n    #тестим фичи гринплама: расчет в питоне не запускаем\n    train_row =  fichi_1_2_3_4_python(train_row)\n    \n    # fill na\n    train_row[[ 'delta_index_0_4', 'delta_index_5_12']] = train_row[[ 'delta_index_0_4', 'delta_index_5_12']].fillna(0)\n    train_row[[ 'script_version']] = train_row[[ 'script_version']].fillna('')    \n    return train_row","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:22:59.825878Z","iopub.execute_input":"2023-06-03T07:22:59.826229Z","iopub.status.idle":"2023-06-03T07:22:59.844276Z","shell.execute_reply.started":"2023-06-03T07:22:59.826196Z","shell.execute_reply":"2023-06-03T07:22:59.84287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def time_feature(train):\n    train[\"year\"] = train[\"session_id\"].apply(lambda x: int(str(x)[:2])).astype(np.uint8)\n    train[\"month\"] = train[\"session_id\"].apply(lambda x: int(str(x)[2:4])+1).astype(np.uint8)\n    train[\"day\"] = train[\"session_id\"].apply(lambda x: int(str(x)[4:6])).astype(np.uint8)\n    train[\"hour\"] = train[\"session_id\"].apply(lambda x: int(str(x)[6:8])).astype(np.uint8)\n    train[\"minute\"] = train[\"session_id\"].apply(lambda x: int(str(x)[8:10])).astype(np.uint8)\n    train[\"second\"] = train[\"session_id\"].apply(lambda x: int(str(x)[10:12])).astype(np.uint8)\n    return train","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:22:59.845716Z","iopub.execute_input":"2023-06-03T07:22:59.846766Z","iopub.status.idle":"2023-06-03T07:22:59.859582Z","shell.execute_reply.started":"2023-06-03T07:22:59.846724Z","shell.execute_reply":"2023-06-03T07:22:59.85811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#global x\n#def feature_engineer(x, grp, use_extra, feature_suffix):\ndef feature_engineer_add_game_version(x, grp, use_extra, feature_suffix):\n    #global aggs\n    #globals()['x'] = x\n    LEVELS = SUB_LEVELS[grp]\n    text_lists = sub_text_lists[grp]\n    room_lists = sub_room_lists[grp]\n    fqid_lists = sub_fqid_lists[grp]\n    aggs = [\n        pl.col(\"index\").count().alias(f\"session_number_{feature_suffix}\"),\n\n        *[pl.col('index').filter(pl.col('text').str.contains(c)).count().alias(f'word_{c}') for c in DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text').str.contains(c))).mean().alias(f'word_mean_{c}') for c in\n          DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text').str.contains(c))).std().alias(f'word_std_{c}') for c in\n          DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text').str.contains(c))).max().alias(f'word_max_{c}') for c in\n          DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text').str.contains(c))).sum().alias(f'word_sum_{c}') for c in\n          DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text').str.contains(c))).median().alias(f'word_median_{c}') for c\n          in DIALOGS],\n        \n        \n        *[pl.col('index').filter(pl.col('text_replace').str.contains(c)).count().alias(f'word_replace_{c}') for c in DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text_replace').str.contains(c))).mean().alias(f'word_replace_mean_{c}') for c in\n          DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text_replace').str.contains(c))).std().alias(f'word_replace_std_{c}') for c in\n          DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text_replace').str.contains(c))).max().alias(f'word_replace_max_{c}') for c in\n          DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text_replace').str.contains(c))).sum().alias(f'word_replace_sum_{c}') for c in\n          DIALOGS],\n        *[pl.col(\"elapsed_time_diff\").filter((pl.col('text_replace').str.contains(c))).median().alias(f'word_replace_median_{c}') for c\n          in DIALOGS],\n        \n\n        *[pl.col(c).drop_nulls().n_unique().alias(f\"{c}_unique_{feature_suffix}\") for c in CATS],\n\n        *[pl.col(c).mean().alias(f\"{c}_mean_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).std().alias(f\"{c}_std_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).min().alias(f\"{c}_min_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).max().alias(f\"{c}_max_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).median().alias(f\"{c}_median_{feature_suffix}\") for c in NUMS],\n\n        *[pl.col(\"fqid\").filter(pl.col(\"fqid\") == c).count().alias(f\"{c}_fqid_counts{feature_suffix}\")\n          for c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for\n          c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).median().alias(f\"{c}_ET_median_{feature_suffix}\") for\n          c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for\n          c in fqid_lists],\n\n        *[pl.col(\"text_fqid\").filter(pl.col(\"text_fqid\") == c).count().alias(f\"{c}_text_fqid_counts{feature_suffix}\")\n          for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).median().alias(f\"{c}_ET_median_{feature_suffix}\")\n          for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for\n          c in text_lists],\n\n        *[pl.col(\"room_fqid\").filter(pl.col(\"room_fqid\") == c).count().alias(f\"{c}_room_fqid_counts{feature_suffix}\")\n          for c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for\n          c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).median().alias(f\"{c}_ET_median_{feature_suffix}\")\n          for\n          c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for\n          c in room_lists],\n\n        *[pl.col(\"event_name\").filter(pl.col(\"event_name\") == c).count().alias(f\"{c}_event_name_counts{feature_suffix}\")\n          for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\")\n          for\n          c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).median().alias(\n            f\"{c}_ET_median_{feature_suffix}\") for\n          c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for\n          c in event_name_feature],\n\n        *[pl.col(\"name\").filter(pl.col(\"name\") == c).count().alias(f\"{c}_name_counts{feature_suffix}\") for c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).median().alias(f\"{c}_ET_median_{feature_suffix}\") for\n          c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for c in\n          name_feature],\n\n        *[pl.col(\"level\").filter(pl.col(\"level\") == c).count().alias(f\"{c}_LEVEL_count{feature_suffix}\") for c in\n          LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for c in\n          LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for c\n          in\n          LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for c in\n          LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).median().alias(f\"{c}_ET_median_{feature_suffix}\") for\n          c in\n          LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for c in\n          LEVELS],\n\n        *[pl.col(\"level_group\").filter(pl.col(\"level_group\") == c).count().alias(\n            f\"{c}_LEVEL_group_count{feature_suffix}\") for c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\")\n          for c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).median().alias(\n            f\"{c}_ET_median_{feature_suffix}\") for c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for\n          c in\n          level_groups],\n\n    ]\n\n    print(\"df\")\n    df = x.groupby(['session_id'], maintain_order=True).agg(aggs).sort(\"session_id\")\n\n    print(\"use_extra\")  \n    if use_extra:\n        if grp == '5-12':\n            print(\"if grp == '5-12'\")              \n            aggs = [\n                pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"Here's the log book.\")\n                                              | (pl.col(\"fqid\") == 'logbook.page.bingo'))\n                    .apply(lambda s: ( s.max() if s.max() else 0 ) - (s.min() if s.min() else 0 )).alias(\"logbook_bingo_duration\"),\n                pl.col(\"index\").filter(\n                    (pl.col(\"text\") == \"Here's the log book.\") | (pl.col(\"fqid\") == 'logbook.page.bingo')).apply(\n                    lambda s: ( s.max() if s.max() else 0 ) - (s.min() if s.min() else 0 )).alias(\"logbook_bingo_indexCount\"),\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader')) | (\n                            pl.col(\"fqid\") == \"reader.paper2.bingo\")).apply(lambda s: ( s.max() if s.max() else 0 ) - (s.min() if s.min() else 0 )).alias(\n                    \"reader_bingo_duration\")\n                ,\n                pl.col(\"index\").filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader')) | (\n                        pl.col(\"fqid\") == \"reader.paper2.bingo\")).apply(lambda s: ( s.max() if s.max() else 0 ) - (s.min() if s.min() else 0 )).alias(\n                    \"reader_bingo_indexCount\"),\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals')) | (\n                            pl.col(\"fqid\") == \"journals.pic_2.bingo\")).apply(lambda s:  ( s.max() if s.max() else 0 ) - (s.min() if s.min() else 0 )).alias(\n                    \"journals_bingo_duration\"),\n                pl.col(\"index\").filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals')) | (\n                        pl.col(\"fqid\") == \"journals.pic_2.bingo\")).apply(lambda s:  ( s.max() if s.max() else 0 ) - (s.min() if s.min() else 0 )).alias(\n                    \"journals_bingo_indexCount\"),\n            ]\n            tmp = x.groupby([\"session_id\"], maintain_order=True).agg(aggs).sort(\"session_id\")\n            df = df.join(tmp, on=\"session_id\", how='left')\n\n        if grp == '13-22':\n            aggs = [\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader_flag')) | (\n                            pl.col(\"fqid\") == \"tunic.library.microfiche.reader_flag.paper2.bingo\")).apply(\n                    lambda s:  ( s.max() if s.max() else 0 ) - (s.min() if s.min() else 0 ) if s.len() > 0 else 0).alias(\"reader_flag_duration\"),\n                pl.col(\"index\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader_flag')) | (\n                            pl.col(\"fqid\") == \"tunic.library.microfiche.reader_flag.paper2.bingo\")).apply(\n                    lambda s: ( s.max() if s.max() else 0 ) - (s.min() if s.min() else 0 ) if s.len() > 0 else 0).alias(\"reader_flag_indexCount\"),\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals_flag')) | (\n                            pl.col(\"fqid\") == \"journals_flag.pic_0.bingo\")).apply(\n                    lambda s:  ( s.max() if s.max() else 0 ) - (s.min() if s.min() else 0 ) if s.len() > 0 else 0).alias(\"journalsFlag_bingo_duration\"),\n                pl.col(\"index\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals_flag')) | (\n                            pl.col(\"fqid\") == \"journals_flag.pic_0.bingo\")).apply(\n                    lambda s:  ( s.max() if s.max() else 0 ) - (s.min() if s.min() else 0 ) if s.len() > 0 else 0).alias(\"journalsFlag_bingo_indexCount\")\n            ]\n            tmp = x.groupby([\"session_id\"], maintain_order=True).agg(aggs).sort(\"session_id\")\n            df = df.join(tmp, on=\"session_id\", how='left')\n\n    return df.to_pandas()\ncolumns = [\n    pl.col(\"page\").cast(pl.Float32),\n    (\n        (pl.col(\"elapsed_time\") - pl.col(\"elapsed_time\").shift(1))\n        .fill_null(0)\n        .clip(0, 1e9)\n        .over([\"session_id\", \"level\"])\n        .alias(\"elapsed_time_diff\")\n    ),\n    (\n        (pl.col(\"screen_coor_x\") - pl.col(\"screen_coor_x\").shift(1))\n        .abs()\n        .over([\"session_id\", \"level\"])\n    ),\n    (\n        (pl.col(\"screen_coor_y\") - pl.col(\"screen_coor_y\").shift(1))\n        .abs()\n        .over([\"session_id\", \"level\"])\n    ),\n    pl.col(\"fqid\").fill_null(\"fqid_None\"),\n    pl.col(\"text_fqid\").fill_null(\"text_fqid_None\")\n\n]","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:22:59.863049Z","iopub.execute_input":"2023-06-03T07:22:59.863579Z","iopub.status.idle":"2023-06-03T07:22:59.93416Z","shell.execute_reply.started":"2023-06-03T07:22:59.863539Z","shell.execute_reply":"2023-06-03T07:22:59.933035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_q = {'0-4':[1,2,3], '5-12':[4,5,6,7,8,9,10,11,12,13], '13-22':[14,15,16,17,18]}\ngrp_M=['0-4','5-12','13-22' ]","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:22:59.935411Z","iopub.execute_input":"2023-06-03T07:22:59.935786Z","iopub.status.idle":"2023-06-03T07:22:59.952087Z","shell.execute_reply.started":"2023-06-03T07:22:59.93575Z","shell.execute_reply":"2023-06-03T07:22:59.951139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_feature1(grp, train): \n \n    #20:41   - 20:29\n    #grp = grp_M[0]\n    # 1 catboost model\n    \n    grpSee=[]\n    if grp == '0-4': \n         grpSee = ['0-4']\n    elif grp == '5-12': \n         grpSee = ['0-4', '5-12'] \n    elif grp == '13-22': \n         grpSee = ['0-4', '5-12', '13-22']              \n    \n    train = fichi_1_2_3_4(train) #my 4 fitures\n    #train.to_csv( f\"/home/datalab/nfs/competitionGame/kaggle/train_4fichi_GP_{grp}.csv\")    \n    #train.to_csv( f\"/home/datalab/nfs/competitionGame/kaggle/train_4fichi_python_{grp}.csv\")\n    #trainG = train.copy()\n    \n    df = (pl.from_pandas(   train[train['level_group'].isin( grpSee  )] )\n      .drop([\"fullscreen\", \"hq\", \"music\"])\n      .with_columns(columns))\n    \n    df = feature_engineer_add_game_version(df, grp, use_extra=True, feature_suffix='')    \n \n    #df = feature_engineer(df, grp, use_extra=True, feature_suffix='')\n    df = time_feature(df) \n    return df","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:22:59.953439Z","iopub.execute_input":"2023-06-03T07:22:59.954016Z","iopub.status.idle":"2023-06-03T07:22:59.967608Z","shell.execute_reply.started":"2023-06-03T07:22:59.953978Z","shell.execute_reply":"2023-06-03T07:22:59.9665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Train Data and Labels","metadata":{"papermill":{"duration":0.004542,"end_time":"2023-02-07T00:59:59.189777","exception":false,"start_time":"2023-02-07T00:59:59.185235","status":"completed"},"tags":[]}},{"cell_type":"code","source":"dtypes = {\"session_id\": 'int64',\n          \"index\": np.int16,\n          \"elapsed_time\": np.int32,\n          \"event_name\": 'category',\n          \"name\": 'category',\n          \"level\": np.int8,\n          \"page\": np.float16,\n          \"room_coor_x\": np.float16,\n          \"room_coor_y\": np.float16,\n          \"screen_coor_x\": np.float16,\n          \"screen_coor_y\": np.float16,\n          \"hover_duration\": np.float32,\n          \"text\": 'category',\n          \"fqid\": 'category',\n          \"room_fqid\": 'category',\n          \"text_fqid\": 'category',\n          \"fullscreen\": np.int8,\n          \"hq\": np.int8,\n          \"music\": np.int8,\n          \"level_group\": 'category'\n          }\nuse_col = ['session_id', 'index', 'elapsed_time', 'event_name', 'name', 'level', 'page',\n           'room_coor_x', 'room_coor_y', 'hover_duration', 'text', 'fqid', 'room_fqid', 'text_fqid', 'level_group']","metadata":{"papermill":{"duration":59.284316,"end_time":"2023-02-07T01:00:58.478743","exception":false,"start_time":"2023-02-07T00:59:59.194427","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-03T07:22:59.968938Z","iopub.execute_input":"2023-06-03T07:22:59.969819Z","iopub.status.idle":"2023-06-03T07:22:59.982762Z","shell.execute_reply.started":"2023-06-03T07:22:59.969763Z","shell.execute_reply":"2023-06-03T07:22:59.981889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#targets = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\n#targets['session'] = targets.session_id.apply(lambda x: int(x.split('_')[0]) )\n#targets['q'] = targets.session_id.apply(lambda x: int(x.split('_')[-1][1:]) )\n# print( targets.shape )\n# targets.head()","metadata":{"papermill":{"duration":0.598155,"end_time":"2023-02-07T01:00:59.082015","exception":false,"start_time":"2023-02-07T01:00:58.48386","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-03T07:22:59.98418Z","iopub.execute_input":"2023-06-03T07:22:59.984811Z","iopub.status.idle":"2023-06-03T07:22:59.99495Z","shell.execute_reply.started":"2023-06-03T07:22:59.984763Z","shell.execute_reply":"2023-06-03T07:22:59.994069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_df = pd.read_csv('/kaggle/input/featur/feature_sort.csv')","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:22:59.996295Z","iopub.execute_input":"2023-06-03T07:22:59.996886Z","iopub.status.idle":"2023-06-03T07:23:00.095039Z","shell.execute_reply.started":"2023-06-03T07:22:59.996851Z","shell.execute_reply":"2023-06-03T07:23:00.094078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineer","metadata":{"papermill":{"duration":0.005196,"end_time":"2023-02-07T01:00:59.092865","exception":false,"start_time":"2023-02-07T01:00:59.087669","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def delt_time_def(df):\n    df.sort_values(by=['session_id', 'elapsed_time'], inplace=True)\n    df['d_time'] = df['elapsed_time'].diff(1)\n    df['d_time'].fillna(0, inplace=True)\n    df['delt_time'] = df['d_time'].clip(0, 103000)\n    df['delt_time_next'] = df['delt_time'].shift(-1)\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:23:00.096492Z","iopub.execute_input":"2023-06-03T07:23:00.097126Z","iopub.status.idle":"2023-06-03T07:23:00.103096Z","shell.execute_reply.started":"2023-06-03T07:23:00.097088Z","shell.execute_reply":"2023-06-03T07:23:00.101898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineer(train, kol_f):\n    global kol_col, kol_col_max\n    kol_col = 9\n    kol_col_max = 11+kol_f*2\n    col = [i for i in range(0,kol_col_max)]\n    new_train = pd.DataFrame(index=train['session_id'].unique(), columns=col, dtype=np.float16)  \n    new_train[10] = new_train.index # \"session_id\"    \n\n    new_train[0] = train.groupby(['session_id'])['d_time'].quantile(q=0.3)\n    new_train[1] = train.groupby(['session_id'])['d_time'].quantile(q=0.8)\n    new_train[2] = train.groupby(['session_id'])['d_time'].quantile(q=0.5)\n    new_train[3] = train.groupby(['session_id'])['d_time'].quantile(q=0.65)\n    new_train[4] = train.groupby(['session_id'])['hover_duration'].agg('mean')\n    new_train[5] = train.groupby(['session_id'])['hover_duration'].agg('std')    \n    new_train[6] = new_train[10].apply(lambda x: int(str(x)[:2])).astype(np.uint8) # \"year\"\n    new_train[7] = new_train[10].apply(lambda x: int(str(x)[2:4])+1).astype(np.uint8) # \"month\"\n    new_train[8] = new_train[10].apply(lambda x: int(str(x)[4:6])).astype(np.uint8) # \"day\"\n    new_train[9] = new_train[10].apply(lambda x: int(str(x)[6:8])).astype(np.uint8) + new_train[10].apply(lambda x: int(str(x)[8:10])).astype(np.uint8)/60\n    new_train[10] = 0\n    new_train = new_train.fillna(-1)\n    \n    return new_train","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:23:00.104656Z","iopub.execute_input":"2023-06-03T07:23:00.105267Z","iopub.status.idle":"2023-06-03T07:23:00.118748Z","shell.execute_reply.started":"2023-06-03T07:23:00.105228Z","shell.execute_reply":"2023-06-03T07:23:00.117751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_next_t(row_f, new_train, train, gran_1, gran_2, i):\n    global kol_col\n    kol_col +=1\n    col1 = row_f['col1']\n    val1 = row_f['val1']\n    maska = (train[col1] == val1)\n    if row_f['kol_col'] == 1:       \n        new_train[kol_col] = train[maska].groupby(['session_id'])['delt_time_next'].sum()\n        if gran_1:\n            kol_col +=1\n            new_train[kol_col] = train[maska].groupby(['session_id'])['delt_time'].mean()\n        if gran_2:\n            kol_col +=1\n            new_train[kol_col] = train[maska].groupby(['session_id'])['index'].count()          \n    elif row_f['kol_col'] == 2: \n        col2 = row_f['col2']\n        val2 = row_f['val2']\n        maska = maska & (train[col2] == val2)        \n        new_train[kol_col] = train[maska].groupby(['session_id'])['delt_time_next'].sum()\n        if gran_1:\n            kol_col +=1\n            new_train[kol_col] = train[maska].groupby(['session_id'])['delt_time'].mean()\n        if gran_2:\n            kol_col +=1\n            new_train[kol_col] = train[maska].groupby(['session_id'])['index'].count()\n    return new_train","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:23:00.123633Z","iopub.execute_input":"2023-06-03T07:23:00.123997Z","iopub.status.idle":"2023-06-03T07:23:00.13485Z","shell.execute_reply.started":"2023-06-03T07:23:00.123963Z","shell.execute_reply":"2023-06-03T07:23:00.133537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_next_t_otvet(row_f, new_train, train, gran_1, gran_2, i):\n    global kol_col\n    kol_col +=1\n    col1 = row_f['col1']\n    val1 = row_f['val1']\n    maska = (train[col1] == val1)\n    if row_f['kol_col'] == 1:      \n        new_train[kol_col] = train[maska]['delt_time_next'].sum()\n        if gran_1:\n            kol_col +=1\n            new_train[kol_col] = train[maska]['delt_time'].mean()\n        if gran_2:\n            kol_col +=1\n            new_train[kol_col] = train[maska]['index'].count()          \n    elif row_f['kol_col'] == 2: \n        col2 = row_f['col2']\n        val2 = row_f['val2']\n        maska = maska & (train[col2] == val2)        \n        new_train[kol_col] = train[maska]['delt_time_next'].sum()\n        if gran_1:\n            kol_col +=1\n            new_train[kol_col] = train[maska]['delt_time'].mean()\n        if gran_2:\n            kol_col +=1\n            new_train[kol_col] = train[maska]['index'].count()\n    return new_train","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:23:00.136479Z","iopub.execute_input":"2023-06-03T07:23:00.136835Z","iopub.status.idle":"2023-06-03T07:23:00.151271Z","shell.execute_reply.started":"2023-06-03T07:23:00.136803Z","shell.execute_reply":"2023-06-03T07:23:00.150292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def experiment_feature_next_t_otvet(row_f, new_train, train, gran_1, gran_2, i):\n    global kol_col\n    kol_col +=1\n    if row_f['kol_col'] == 1: \n        maska = train[row_f['col1']] == row_f['val1']\n        new_train[kol_col] = train[maska]['delt_time_next'].sum()\n        if gran_1:\n            kol_col +=1\n            new_train[kol_col] = train[maska]['delt_time'].mean()\n        if gran_2:\n            kol_col +=1\n            new_train[kol_col] = train[maska]['index'].count()          \n    elif row_f['kol_col'] == 2: \n        col2 = row_f['col2']\n        val2 = row_f['val2']\n        maska = (train[col1] == val1) & (train[col2] == val2)        \n        new_train[kol_col] = train[maska]['delt_time_next'].sum()\n        if gran_1:\n            kol_col +=1\n            new_train[kol_col] = train[maska]['delt_time'].mean()\n        if gran_2:\n            kol_col +=1\n            new_train[kol_col] = train[maska]['index'].count()\n    return new_train","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:23:00.152713Z","iopub.execute_input":"2023-06-03T07:23:00.15334Z","iopub.status.idle":"2023-06-03T07:23:00.170749Z","shell.execute_reply.started":"2023-06-03T07:23:00.153302Z","shell.execute_reply":"2023-06-03T07:23:00.169723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_quest_otvet(new_train, train, quest, kol_f):\n    global kol_col\n    kol_col = 9\n    g1 = 0.7 \n    g2 = 0.3 \n\n    feature_q = feature_df[feature_df['quest'] == quest].copy()\n    feature_q.reset_index(drop=True, inplace=True)\n    \n    gran1 = round(kol_f * g1)\n    gran2 = round(kol_f * g2)    \n    for i in range(0, kol_f):         \n        row_f = feature_q.loc[i]\n        new_train = feature_next_t_otvet(row_f, new_train, train, i < gran1, i <  gran2, i) \n    col = [i for i in range(0,kol_col+1)]\n    return new_train[col]","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:23:00.172206Z","iopub.execute_input":"2023-06-03T07:23:00.172861Z","iopub.status.idle":"2023-06-03T07:23:00.185488Z","shell.execute_reply.started":"2023-06-03T07:23:00.172821Z","shell.execute_reply":"2023-06-03T07:23:00.184487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineer_new(new_train, train, feature_q, kol_f):\n    g1 = 0.7 \n    g2 = 0.3 \n    gran1 = round(kol_f * g1)\n    gran2 = round(kol_f * g2)    \n    for i in range(0, kol_f): \n        row_f = feature_q.loc[i]       \n        new_train = feature_next_t(row_f, new_train, train, i < gran1, i <  gran2, i)         \n    return new_train","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:23:00.187007Z","iopub.execute_input":"2023-06-03T07:23:00.187674Z","iopub.status.idle":"2023-06-03T07:23:00.201502Z","shell.execute_reply.started":"2023-06-03T07:23:00.187636Z","shell.execute_reply":"2023-06-03T07:23:00.200464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_quest(new_train, train, quest, kol_f):\n    global kol_col\n    kol_col = 9\n    feature_q = feature_df[feature_df['quest'] == quest].copy()\n    feature_q.reset_index(drop=True, inplace=True)\n    new_train = feature_engineer_new(new_train, train, feature_q, kol_f)\n    col = [i for i in range(0,kol_col+1)]\n    return new_train[col]","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:23:00.202989Z","iopub.execute_input":"2023-06-03T07:23:00.20362Z","iopub.status.idle":"2023-06-03T07:23:00.213834Z","shell.execute_reply.started":"2023-06-03T07:23:00.203565Z","shell.execute_reply":"2023-06-03T07:23:00.21281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"def create_model(old_train, quests, models, list_kol_f):\n    \n    kol_quest = len(quests)\n    # ITERATE THRU QUESTIONS\n    for q in quests:\n        print('### quest ', q, end='')\n        new_train = feature_engineer(old_train, list_kol_f[q])\n        train_x = feature_quest(new_train, old_train, q, list_kol_f[q])\n        print (' ---- ', 'train_q.shape = ', train_x.shape)\n           \n        # TRAIN DATA\n        train_users = train_x.index.values\n        train_y = targets.loc[targets.q==q].set_index('session').loc[train_users]\n\n        # TRAIN MODEL \n\n        model = CatBoostClassifier(\n            n_estimators = 300,\n            learning_rate= 0.045,\n            depth = 6\n        )\n        \n        model.fit(train_x.astype('float32'), train_y['correct'], verbose=False)\n\n        # SAVE MODEL, PREDICT VALID OOF\n        models[f'{q}'] = model\n    print('***')\n    \n    return models","metadata":{"execution":{"iopub.status.busy":"2023-06-02T18:53:33.925216Z","iopub.execute_input":"2023-06-02T18:53:33.925841Z","iopub.status.idle":"2023-06-02T18:53:33.938154Z","shell.execute_reply.started":"2023-06-02T18:53:33.925804Z","shell.execute_reply":"2023-06-02T18:53:33.937123Z"}}},{"cell_type":"code","source":"models = {}\nbest_threshold = 0.63","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:23:00.215297Z","iopub.execute_input":"2023-06-03T07:23:00.215994Z","iopub.status.idle":"2023-06-03T07:23:00.226929Z","shell.execute_reply.started":"2023-06-03T07:23:00.215955Z","shell.execute_reply":"2023-06-03T07:23:00.225988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_kol_f = {\n    1:140,3:110,\n    4:110, 5:220, 6:120, 7:110, 8:110, 9:100, 10:120, 11:120,\n    14: 110, 15:160, 16:105, 17:140             \n             }","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:23:00.228343Z","iopub.execute_input":"2023-06-03T07:23:00.228974Z","iopub.status.idle":"2023-06-03T07:23:00.238765Z","shell.execute_reply.started":"2023-06-03T07:23:00.228938Z","shell.execute_reply":"2023-06-03T07:23:00.237719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df0_4 = pd.read_csv('/kaggle/input/featur/train_0_4t.csv', dtype=dtypes) \n#kol_lvl = (df0_4 .groupby(['session_id'])['level'].agg('nunique') < 5)\n#list_session = kol_lvl[kol_lvl].index\n#df0_4  = df0_4 [~df0_4 ['session_id'].isin(list_session)]\n#df0_4 = delt_time_def(df0_4)\n\nquests_0_4 = [1, 3] \n# list_kol_f = {1:140,3:110}\n\n#models = create_model(df0_4, quests_0_4, models, list_kol_f)\n#del df0_4","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:23:00.2402Z","iopub.execute_input":"2023-06-03T07:23:00.24085Z","iopub.status.idle":"2023-06-03T07:23:00.252241Z","shell.execute_reply.started":"2023-06-03T07:23:00.240811Z","shell.execute_reply":"2023-06-03T07:23:00.250853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df5_12 = pd.read_csv('/kaggle/input/featur/train_5_12t.csv', dtype=dtypes)\n#kol_lvl = (df5_12.groupby(['session_id'])['level'].agg('nunique') < 8)\n#list_session = kol_lvl[kol_lvl].index\n#df5_12 = df5_12[~df5_12['session_id'].isin(list_session)]\n#df5_12 = delt_time_def(df5_12)\nquests_5_12 = [4, 5, 6, 7, 8, 9, 10, 11  ] \n\n# list_kol_f = {4:110, 5:220, 6:120, 7:110, 8:110, 9:100, 10:120, 11:120}\n\n#models = create_model(df5_12, quests_5_12, models, list_kol_f)\n#del df5_12","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:23:00.254133Z","iopub.execute_input":"2023-06-03T07:23:00.25449Z","iopub.status.idle":"2023-06-03T07:23:00.263566Z","shell.execute_reply.started":"2023-06-03T07:23:00.254458Z","shell.execute_reply":"2023-06-03T07:23:00.26254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df13_22 = pd.read_csv('/kaggle/input/featur/train_13_22t.csv', dtype=dtypes) \n#kol_lvl = (df13_22 .groupby(['session_id'])['level'].agg('nunique') < 10)\n#list_session = kol_lvl[kol_lvl].index\n#df13_22  = df13_22 [~df13_22 ['session_id'].isin(list_session)]\n#df13_22 = delt_time_def(df13_22)\n\nquests_13_22 = [14, 15, 16, 17, 18 ] \n# list_kol_f = {14: 110, 15:160, 16:105, 17:140}\n\n#models = create_model(df13_22, quests_13_22, models, list_kol_f)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:23:00.265139Z","iopub.execute_input":"2023-06-03T07:23:00.26581Z","iopub.status.idle":"2023-06-03T07:23:00.281029Z","shell.execute_reply.started":"2023-06-03T07:23:00.265773Z","shell.execute_reply":"2023-06-03T07:23:00.279921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" #Saving a Model\n #for q in quests_0_4 + quests_5_12 + quests_13_22:\n #    models[f'{q}'].save_model(f'cat_model_{q}.bin')","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:23:00.282676Z","iopub.execute_input":"2023-06-03T07:23:00.283329Z","iopub.status.idle":"2023-06-03T07:23:00.293966Z","shell.execute_reply.started":"2023-06-03T07:23:00.28329Z","shell.execute_reply":"2023-06-03T07:23:00.292536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Model Reading\ndir = '/kaggle/input/competition-game-model-catb5/' \nfor q in quests_0_4 + quests_5_12 + quests_13_22:\n   models[f'{q}'] = CatBoostClassifier().load_model(dir+f'cat_model_{q}.bin')","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:23:00.295678Z","iopub.execute_input":"2023-06-03T07:23:00.296033Z","iopub.status.idle":"2023-06-03T07:23:00.347133Z","shell.execute_reply.started":"2023-06-03T07:23:00.296001Z","shell.execute_reply":"2023-06-03T07:23:00.345719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Infer Test Data**","metadata":{}},{"cell_type":"code","source":"import jo_wilder\n\ntry:\n    jo_wilder.make_env.__called__ = False\n    env.__called__ = False\n    type(env)._state = type(type(env)._state).__dict__['INIT']\nexcept:\n    pass\n\nenv = jo_wilder.make_env()\niter_test = env.iter_test()    ","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:23:00.349003Z","iopub.execute_input":"2023-06-03T07:23:00.349462Z","iopub.status.idle":"2023-06-03T07:23:00.356467Z","shell.execute_reply.started":"2023-06-03T07:23:00.349426Z","shell.execute_reply":"2023-06-03T07:23:00.355476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:23:00.357921Z","iopub.execute_input":"2023-06-03T07:23:00.358577Z","iopub.status.idle":"2023-06-03T07:23:00.369525Z","shell.execute_reply.started":"2023-06-03T07:23:00.35854Z","shell.execute_reply":"2023-06-03T07:23:00.368313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\npred = pd.DataFrame()\npreds = pd.DataFrame()\ntest_0_4 = pd.DataFrame()\ntest_5_12 = pd.DataFrame()\ntest_13_22 = pd.DataFrame()\nglobal feature1\n\ng_end4 = 0\ng_end5 = 0\n\n\nlist_q = {'0-4':quests_0_4, '5-12':quests_5_12, '13-22':quests_13_22}\nfor (test, sam_sub) in iter_test:\n    sam_sub['question'] = [int(label.split('_')[1][1:]) for label in sam_sub['session_id']]    \n    grp = test.level_group.values[0]   \n    sam_sub['correct'] = 1\n    sam_sub.loc[sam_sub.question.isin([5, 8, 10, 13, 15]), 'correct'] = 0  \n    old_train = delt_time_def(test[test.level_group == grp])\n    \n    \n    #my\n     \n    sample_submission = sam_sub\n    global feature1\n   \n    quests = list_q[grp]\n    \n\n    #Датасет\n    if   grp== '0-4':\n           test_0_4   = test.copy() \n            \n           train = test_0_4.copy()  \n    elif grp== '5-12':\n           test_5_12   = test.copy()   \n            \n           train = test_0_4.copy() \n           train = train.append(test_5_12).copy() \n    elif grp== '13-22':\n           test_13_22   = test.copy()    \n           \n           train = test_0_4.copy() \n           train = train.append(test_5_12).copy() \n           train = train.append(test_13_22).copy()    \n    \n   \n    \n    #фичи  \n    df = create_feature1(grp, train)\n    del train   \n    gc.collect()     \n    if   grp== '0-4':        \n           feature_0_4   = df \n           feature1 = feature_0_4 \n    elif grp== '5-12':       \n           feature_5_12   = df.merge(feature_0_4,  on =\"session_id\",  suffixes = (\"_a\",\"_b\"))           \n           del feature_0_4   \n           gc.collect()  \n           feature1 = feature_5_12      \n    elif grp== '13-22':         \n           feature_13_22   =df.merge(feature_5_12,  on =\"session_id\",  suffixes = (\"_c\",\"_e\"))   \n           del feature_5_12   \n           gc.collect()             \n           feature1 = feature_13_22 \n    feature2 = feature1.copy()  \n    feature1 = feature2.set_index('session_id')\n    \n    #применение модель\n    preds = pd.DataFrame()\n    thresh = 0.63\n    k = 0\n    preds = pd.DataFrame() \n    #end my   \n    \n    for q in list_q[grp]:\n        \n        start4 = time.time()\n        \n        if q not in [2,12,13,18]:\n            new_train = feature_engineer(old_train, list_kol_f[q])\n            new_train = feature_quest_otvet(new_train, old_train, q, list_kol_f[q])\n            #  new_train = feature_quest(new_train, old_train, q, kol_f)\n        \n        \n        end4 = time.time() - start4\n        g_end4 += end4\n        \n        start5 = time.time()        \n        \n        clf = models[f'{q}']\n        \n        if q in [18]:\n            p = clf.predict_proba(feature1)[:,1]                \n        else:     \n            p = clf.predict_proba(new_train.astype('float32'))[:,1]        \n        \n        end5 = time.time() - start5\n        g_end5 += end5\n             \n        \n        mask = sam_sub.question == q \n        x = int(p[0]>best_threshold)\n        sam_sub.loc[mask,'correct'] = x      \n        \n        \n    sam_sub = sam_sub[['session_id', 'correct']]      \n    env.predict(sam_sub)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T07:23:00.371019Z","iopub.execute_input":"2023-06-03T07:23:00.371558Z","iopub.status.idle":"2023-06-03T07:23:19.448374Z","shell.execute_reply.started":"2023-06-03T07:23:00.371522Z","shell.execute_reply":"2023-06-03T07:23:19.447444Z"},"trusted":true},"execution_count":null,"outputs":[]}]}