{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Catboost Baseline","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-20T01:05:38.150107Z","iopub.execute_input":"2023-06-20T01:05:38.150525Z","iopub.status.idle":"2023-06-20T01:05:38.171711Z","shell.execute_reply.started":"2023-06-20T01:05:38.150492Z","shell.execute_reply":"2023-06-20T01:05:38.170237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import polars as pl\nimport gc\nfrom tqdm.auto import tqdm\nfrom sklearn.metrics import f1_score\nfrom sklearn.model_selection import GroupKFold, KFold\nfrom catboost import CatBoostClassifier, Pool\nimport matplotlib.pyplot as plt\nimport warnings\nfrom itertools import combinations\nimport math\nfrom time import sleep\nimport pickle\nimport optuna\nfrom optuna.visualization import plot_optimization_history\nfrom optuna.visualization import plot_param_importances\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import roc_auc_score\n\nwarnings.filterwarnings('ignore')\npd.set_option(\"display.max_columns\", None)\npd.set_option(\"display.max_rows\", 200)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:05:38.173596Z","iopub.execute_input":"2023-06-20T01:05:38.175123Z","iopub.status.idle":"2023-06-20T01:05:39.833543Z","shell.execute_reply.started":"2023-06-20T01:05:38.175080Z","shell.execute_reply":"2023-06-20T01:05:39.832547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading Files","metadata":{}},{"cell_type":"code","source":"targets = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\ntargets['session'] = targets.session_id.apply(lambda x: int(x.split('_')[0]))\ntargets['q'] = targets.session_id.apply(lambda x: int(x.split('_')[-1][1:]))","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:05:39.835095Z","iopub.execute_input":"2023-06-20T01:05:39.835733Z","iopub.status.idle":"2023-06-20T01:05:41.507544Z","shell.execute_reply.started":"2023-06-20T01:05:39.835696Z","shell.execute_reply":"2023-06-20T01:05:41.506501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = [\n    pl.col(\"page\").cast(pl.Float32),\n    (\n        (pl.col(\"elapsed_time\") - pl.col(\"elapsed_time\").shift(1))\n        .fill_null(0)\n        .clip(0, 1e9)\n        .over([\"session_id\", \"level\"])\n        .alias(\"elapsed_time_diff\")\n    ),\n    (\n        (pl.col(\"screen_coor_x\") - pl.col(\"screen_coor_x\").shift(1))\n        .abs()\n        .over([\"session_id\", \"level\"])\n    ),\n    (\n        (pl.col(\"screen_coor_y\") - pl.col(\"screen_coor_y\").shift(1))\n        .abs()\n        .over([\"session_id\", \"level\"])\n    ) ,\n    pl.col(\"fqid\").fill_null(\"fqid_None\"),\n    pl.col(\"text_fqid\").fill_null(\"text_fqid_None\")\n\n]","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:05:41.512874Z","iopub.execute_input":"2023-06-20T01:05:41.515592Z","iopub.status.idle":"2023-06-20T01:05:41.525700Z","shell.execute_reply.started":"2023-06-20T01:05:41.515542Z","shell.execute_reply":"2023-06-20T01:05:41.524737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtypes = {\"session_id\": pl.Int64,\n          \"elapsed_time\": pl.Int64,\n          \"event_name\": pl.Categorical,\n          \"name\": pl.Categorical,\n          \"level\": pl.Int8,\n          \"page\": pl.Float32,\n          \"room_coor_x\": pl.Float32,\n          \"room_coor_y\": pl.Float32,\n          \"screen_coor_x\": pl.Float32,\n          \"screen_coor_y\": pl.Float32,\n          \"hover_duration\": pl.Float32,\n          \"fqid\": pl.Categorical,\n          \"room_fqid\": pl.Categorical,\n          \"fullscreen\": pl.Int8,\n          \"hq\": pl.Int8,\n          \"music\": pl.Int8,\n          \"level_group\": pl.Categorical\n          }","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:05:41.530930Z","iopub.execute_input":"2023-06-20T01:05:41.533832Z","iopub.status.idle":"2023-06-20T01:05:41.544549Z","shell.execute_reply.started":"2023-06-20T01:05:41.533781Z","shell.execute_reply":"2023-06-20T01:05:41.543453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pl.toggle_string_cache(True)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:05:41.546016Z","iopub.execute_input":"2023-06-20T01:05:41.546472Z","iopub.status.idle":"2023-06-20T01:05:41.558198Z","shell.execute_reply.started":"2023-06-20T01:05:41.546434Z","shell.execute_reply":"2023-06-20T01:05:41.556769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntrain = (pl.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv',dtypes=dtypes)\n                .drop([\"fullscreen\", \"hq\", \"music\"])\n                .with_columns(columns)\n        )","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:05:41.562885Z","iopub.execute_input":"2023-06-20T01:05:41.563400Z","iopub.status.idle":"2023-06-20T01:06:42.397251Z","shell.execute_reply.started":"2023-06-20T01:05:41.563363Z","shell.execute_reply":"2023-06-20T01:06:42.395854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reducing training file","metadata":{}},{"cell_type":"code","source":"def reduce_memory_usage_pl(df, name):\n    \"\"\" Reduce memory usage by polars dataframe {df} with name {name} by changing its data types.\n        Original pandas version of this function: https://www.kaggle.com/code/arjanso/reducing-dataframe-memory-size-by-65 \"\"\"\n    print(f\"Memory usage of dataframe {name} is {round(df.estimated_size('mb'), 2)} MB\")\n    Numeric_Int_types = [pl.Int8,pl.Int16,pl.Int32,pl.Int64]\n    Numeric_Float_types = [pl.Float32,pl.Float64]    \n    for col in df.columns:\n        col_type = df[col].dtype\n        c_min = df[col].min()\n        c_max = df[col].max()\n        if col_type in Numeric_Int_types:\n            if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                df = df.with_columns(df[col].cast(pl.Int8))\n            elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                df = df.with_columns(df[col].cast(pl.Int16))\n            elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                df = df.with_columns(df[col].cast(pl.Int32))\n            elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                df = df.with_columns(df[col].cast(pl.Int64))\n        elif col_type in Numeric_Float_types:\n            if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                df = df.with_columns(df[col].cast(pl.Float32))\n            else:\n                pass\n        elif col_type == pl.Utf8:\n            df = df.with_columns(df[col].cast(pl.Categorical))\n        else:\n            pass\n    \n    print(f\"Memory usage of dataframe {name} became {round(df.estimated_size('mb'), 2)} MB\")\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:06:42.405449Z","iopub.execute_input":"2023-06-20T01:06:42.405869Z","iopub.status.idle":"2023-06-20T01:06:42.425908Z","shell.execute_reply.started":"2023-06-20T01:06:42.405834Z","shell.execute_reply":"2023-06-20T01:06:42.424419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reducing polar\ntrain = reduce_memory_usage_pl(train, \"train_subset\")","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:06:42.433427Z","iopub.execute_input":"2023-06-20T01:06:42.435873Z","iopub.status.idle":"2023-06-20T01:06:46.500047Z","shell.execute_reply.started":"2023-06-20T01:06:42.435824Z","shell.execute_reply":"2023-06-20T01:06:46.498572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 = train.filter(pl.col(\"level_group\")=='0-4')\ndf2 = train.filter(pl.col(\"level_group\")=='5-12')\ndf3 = train.filter(pl.col(\"level_group\")=='13-22')\ndf1.shape,df2.shape,df3.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:06:46.501459Z","iopub.execute_input":"2023-06-20T01:06:46.501994Z","iopub.status.idle":"2023-06-20T01:06:47.717907Z","shell.execute_reply.started":"2023-06-20T01:06:46.501959Z","shell.execute_reply":"2023-06-20T01:06:47.717076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df3.head(8)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:06:47.719206Z","iopub.execute_input":"2023-06-20T01:06:47.719694Z","iopub.status.idle":"2023-06-20T01:06:47.740444Z","shell.execute_reply.started":"2023-06-20T01:06:47.719662Z","shell.execute_reply":"2023-06-20T01:06:47.739426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Delete train to liberate memory\ndel train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:06:47.744409Z","iopub.execute_input":"2023-06-20T01:06:47.747636Z","iopub.status.idle":"2023-06-20T01:06:48.175972Z","shell.execute_reply.started":"2023-06-20T01:06:47.747587Z","shell.execute_reply":"2023-06-20T01:06:48.174825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"code","source":"CATS = ['event_name', 'name', 'fqid', 'room_fqid', 'text_fqid']\nNUMS = ['page', 'room_coor_x', 'room_coor_y', 'screen_coor_x', 'screen_coor_y',\n        'hover_duration', 'elapsed_time_diff']\n\nfqid_lists = ['worker', 'archivist', 'gramps', 'wells', 'toentry', 'confrontation', 'crane_ranger', 'groupconvo', 'flag_girl', 'tomap', 'tostacks', 'tobasement', 'archivist_glasses', 'boss', 'journals', 'seescratches', 'groupconvo_flag', 'cs', 'teddy', 'expert', 'businesscards', 'ch3start', 'tunic.historicalsociety', 'tofrontdesk', 'savedteddy', 'plaque', 'glasses', 'tunic.drycleaner', 'reader_flag', 'tunic.library', 'tracks', 'tunic.capitol_2', 'trigger_scarf', 'reader', 'directory', 'tunic.capitol_1', 'journals.pic_0.next', 'unlockdoor', 'tunic', 'what_happened', 'tunic.kohlcenter', 'tunic.humanecology', 'colorbook', 'logbook', 'businesscards.card_0.next', 'journals.hub.topics', 'logbook.page.bingo', 'journals.pic_1.next', 'journals_flag', 'reader.paper0.next', 'tracks.hub.deer', 'reader_flag.paper0.next', 'trigger_coffee', 'wellsbadge', 'journals.pic_2.next', 'tomicrofiche', 'journals_flag.pic_0.bingo', 'plaque.face.date', 'notebook', 'tocloset_dirty', 'businesscards.card_bingo.bingo', 'businesscards.card_1.next', 'tunic.wildlife', 'tunic.hub.slip', 'tocage', 'journals.pic_2.bingo', 'tocollectionflag', 'tocollection', 'chap4_finale_c', 'chap2_finale_c', 'lockeddoor', 'journals_flag.hub.topics', 'tunic.capitol_0', 'reader_flag.paper2.bingo', 'photo', 'tunic.flaghouse', 'reader.paper1.next', 'directory.closeup.archivist', 'intro', 'businesscards.card_bingo.next', 'reader.paper2.bingo', 'retirement_letter', 'remove_cup', 'journals_flag.pic_0.next', 'magnify', 'coffee', 'key', 'togrampa', 'reader_flag.paper1.next', 'janitor', 'tohallway', 'chap1_finale', 'report', 'outtolunch', 'journals_flag.hub.topics_old', 'journals_flag.pic_1.next', 'reader.paper2.next', 'chap1_finale_c', 'reader_flag.paper2.next', 'door_block_talk', 'journals_flag.pic_1.bingo', 'journals_flag.pic_2.next', 'journals_flag.pic_2.bingo', 'block_magnify', 'reader.paper0.prev', 'block', 'reader_flag.paper0.prev', 'block_0', 'door_block_clean', 'reader.paper2.prev', 'reader.paper1.prev', 'doorblock', 'tocloset', 'reader_flag.paper2.prev', 'reader_flag.paper1.prev', 'block_tomap2', 'journals_flag.pic_0_old.next', 'journals_flag.pic_1_old.next', 'block_tocollection', 'block_nelson', 'journals_flag.pic_2_old.next', 'block_tomap1', 'block_badge', 'need_glasses', 'block_badge_2', 'fox', 'block_1']\nDIALOGS = ['that', 'this', 'it', 'you','find','found','Found','notebook','Wells','wells','help','need', 'Oh','Ooh','Jo', 'flag', 'can','and','is','the','to']\n\nname_feature = ['basic', 'undefined', 'close', 'open', 'prev', 'next']\nevent_name_feature = ['cutscene_click', 'person_click', 'navigate_click',\n       'observation_click', 'notification_click', 'object_click',\n       'object_hover', 'map_hover', 'map_click', 'checkpoint',\n       'notebook_click']\n\ntext_lists = ['tunic.historicalsociety.cage.confrontation', 'tunic.wildlife.center.crane_ranger.crane', 'tunic.historicalsociety.frontdesk.archivist.newspaper', 'tunic.historicalsociety.entry.groupconvo', 'tunic.wildlife.center.wells.nodeer', 'tunic.historicalsociety.frontdesk.archivist.have_glass', 'tunic.drycleaner.frontdesk.worker.hub', 'tunic.historicalsociety.closet_dirty.gramps.news', 'tunic.humanecology.frontdesk.worker.intro', 'tunic.historicalsociety.frontdesk.archivist_glasses.confrontation', 'tunic.historicalsociety.basement.seescratches', 'tunic.historicalsociety.collection.cs', 'tunic.flaghouse.entry.flag_girl.hello', 'tunic.historicalsociety.collection.gramps.found', 'tunic.historicalsociety.basement.ch3start', 'tunic.historicalsociety.entry.groupconvo_flag', 'tunic.library.frontdesk.worker.hello', 'tunic.library.frontdesk.worker.wells', 'tunic.historicalsociety.collection_flag.gramps.flag', 'tunic.historicalsociety.basement.savedteddy', 'tunic.library.frontdesk.worker.nelson', 'tunic.wildlife.center.expert.removed_cup', 'tunic.library.frontdesk.worker.flag', 'tunic.historicalsociety.frontdesk.archivist.hello', 'tunic.historicalsociety.closet.gramps.intro_0_cs_0', 'tunic.historicalsociety.entry.boss.flag', 'tunic.flaghouse.entry.flag_girl.symbol', 'tunic.historicalsociety.closet_dirty.trigger_scarf', 'tunic.drycleaner.frontdesk.worker.done', 'tunic.historicalsociety.closet_dirty.what_happened', 'tunic.wildlife.center.wells.animals', 'tunic.historicalsociety.closet.teddy.intro_0_cs_0', 'tunic.historicalsociety.cage.glasses.afterteddy', 'tunic.historicalsociety.cage.teddy.trapped', 'tunic.historicalsociety.cage.unlockdoor', 'tunic.historicalsociety.stacks.journals.pic_2.bingo', 'tunic.historicalsociety.entry.wells.flag', 'tunic.humanecology.frontdesk.worker.badger', 'tunic.historicalsociety.stacks.journals_flag.pic_0.bingo', 'tunic.historicalsociety.closet.intro', 'tunic.historicalsociety.closet.retirement_letter.hub', 'tunic.historicalsociety.entry.directory.closeup.archivist', 'tunic.historicalsociety.collection.tunic.slip', 'tunic.kohlcenter.halloffame.plaque.face.date', 'tunic.historicalsociety.closet_dirty.trigger_coffee', 'tunic.drycleaner.frontdesk.logbook.page.bingo', 'tunic.library.microfiche.reader.paper2.bingo', 'tunic.kohlcenter.halloffame.togrampa', 'tunic.capitol_2.hall.boss.haveyougotit', 'tunic.wildlife.center.wells.nodeer_recap', 'tunic.historicalsociety.cage.glasses.beforeteddy', 'tunic.historicalsociety.closet_dirty.gramps.helpclean', 'tunic.wildlife.center.expert.recap', 'tunic.historicalsociety.frontdesk.archivist.have_glass_recap', 'tunic.historicalsociety.stacks.journals_flag.pic_1.bingo', 'tunic.historicalsociety.cage.lockeddoor', 'tunic.historicalsociety.stacks.journals_flag.pic_2.bingo', 'tunic.historicalsociety.collection.gramps.lost', 'tunic.historicalsociety.closet.notebook', 'tunic.historicalsociety.frontdesk.magnify', 'tunic.humanecology.frontdesk.businesscards.card_bingo.bingo', 'tunic.wildlife.center.remove_cup', 'tunic.library.frontdesk.wellsbadge.hub', 'tunic.wildlife.center.tracks.hub.deer', 'tunic.historicalsociety.frontdesk.key', 'tunic.library.microfiche.reader_flag.paper2.bingo', 'tunic.flaghouse.entry.colorbook', 'tunic.wildlife.center.coffee', 'tunic.capitol_1.hall.boss.haveyougotit', 'tunic.historicalsociety.basement.janitor', 'tunic.historicalsociety.collection_flag.gramps.recap', 'tunic.wildlife.center.wells.animals2', 'tunic.flaghouse.entry.flag_girl.symbol_recap', 'tunic.historicalsociety.closet_dirty.photo', 'tunic.historicalsociety.stacks.outtolunch', 'tunic.library.frontdesk.worker.wells_recap', 'tunic.historicalsociety.frontdesk.archivist_glasses.confrontation_recap', 'tunic.capitol_0.hall.boss.talktogramps', 'tunic.historicalsociety.closet.photo', 'tunic.historicalsociety.collection.tunic', 'tunic.historicalsociety.closet.teddy.intro_0_cs_5', 'tunic.historicalsociety.closet_dirty.gramps.archivist', 'tunic.historicalsociety.closet_dirty.door_block_talk', 'tunic.historicalsociety.entry.boss.flag_recap', 'tunic.historicalsociety.frontdesk.archivist.need_glass_0', 'tunic.historicalsociety.entry.wells.talktogramps', 'tunic.historicalsociety.frontdesk.block_magnify', 'tunic.historicalsociety.frontdesk.archivist.foundtheodora', 'tunic.historicalsociety.closet_dirty.gramps.nothing', 'tunic.historicalsociety.closet_dirty.door_block_clean', 'tunic.capitol_1.hall.boss.writeitup', 'tunic.library.frontdesk.worker.nelson_recap', 'tunic.library.frontdesk.worker.hello_short', 'tunic.historicalsociety.stacks.block', 'tunic.historicalsociety.frontdesk.archivist.need_glass_1', 'tunic.historicalsociety.entry.boss.talktogramps', 'tunic.historicalsociety.frontdesk.archivist.newspaper_recap', 'tunic.historicalsociety.entry.wells.flag_recap', 'tunic.drycleaner.frontdesk.worker.done2', 'tunic.library.frontdesk.worker.flag_recap', 'tunic.humanecology.frontdesk.block_0', 'tunic.library.frontdesk.worker.preflag', 'tunic.historicalsociety.basement.gramps.seeyalater', 'tunic.flaghouse.entry.flag_girl.hello_recap', 'tunic.historicalsociety.closet.doorblock', 'tunic.drycleaner.frontdesk.worker.takealook', 'tunic.historicalsociety.basement.gramps.whatdo', 'tunic.library.frontdesk.worker.droppedbadge', 'tunic.historicalsociety.entry.block_tomap2', 'tunic.library.frontdesk.block_nelson', 'tunic.library.microfiche.block_0', 'tunic.historicalsociety.entry.block_tocollection', 'tunic.historicalsociety.entry.block_tomap1', 'tunic.historicalsociety.collection.gramps.look_0', 'tunic.library.frontdesk.block_badge', 'tunic.historicalsociety.cage.need_glasses', 'tunic.library.frontdesk.block_badge_2', 'tunic.kohlcenter.halloffame.block_0', 'tunic.capitol_0.hall.chap1_finale_c', 'tunic.capitol_1.hall.chap2_finale_c', 'tunic.capitol_2.hall.chap4_finale_c', 'tunic.wildlife.center.fox.concern', 'tunic.drycleaner.frontdesk.block_0', 'tunic.historicalsociety.entry.gramps.hub', 'tunic.humanecology.frontdesk.block_1', 'tunic.drycleaner.frontdesk.block_1']\nroom_lists = ['tunic.historicalsociety.entry', 'tunic.wildlife.center', 'tunic.historicalsociety.cage', 'tunic.library.frontdesk', 'tunic.historicalsociety.frontdesk', 'tunic.historicalsociety.stacks', 'tunic.historicalsociety.closet_dirty', 'tunic.humanecology.frontdesk', 'tunic.historicalsociety.basement', 'tunic.kohlcenter.halloffame', 'tunic.library.microfiche', 'tunic.drycleaner.frontdesk', 'tunic.historicalsociety.collection', 'tunic.historicalsociety.closet', 'tunic.flaghouse.entry', 'tunic.historicalsociety.collection_flag', 'tunic.capitol_1.hall', 'tunic.capitol_0.hall', 'tunic.capitol_2.hall']\n\nLEVELS = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22]\nlevel_groups = [\"0-4\", \"5-12\", \"13-22\"]","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:06:48.177304Z","iopub.execute_input":"2023-06-20T01:06:48.177626Z","iopub.status.idle":"2023-06-20T01:06:48.202480Z","shell.execute_reply.started":"2023-06-20T01:06:48.177595Z","shell.execute_reply":"2023-06-20T01:06:48.201361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineer(x, grp, use_extra, feature_suffix):\n    aggs = [\n        pl.col(\"index\").count().alias(f\"session_number_{feature_suffix}\"),\n\n        *[pl.col(c).drop_nulls().n_unique().alias(f\"{c}_unique_{feature_suffix}\") for c in CATS],\n\n        *[pl.col(c).std().alias(f\"{c}_std_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).mean().alias(f\"{c}_mean_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).min().alias(f\"{c}_min_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).max().alias(f\"{c}_max_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).sum().alias(f\"{c}_sum_{feature_suffix}\") for c in NUMS],\n\n        *[pl.col(\"fqid\").filter(pl.col(\"fqid\") == c).count().alias(f\"{c}_fqid_counts{feature_suffix}\")\n          for c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for\n          c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for\n          c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).min().alias(f\"{c}_ET_min_{feature_suffix}\") for\n          c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in fqid_lists],\n\n\n        *[pl.col(\"text_fqid\").filter(pl.col(\"text_fqid\") == c).count().alias(f\"{c}_text_fqid_counts{feature_suffix}\") for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).min().alias(f\"{c}_ET_min_{feature_suffix}\") for\n          c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in text_lists],\n\n        *[pl.col(\"room_fqid\").filter(pl.col(\"room_fqid\") == c).count().alias(f\"{c}_room_fqid_counts{feature_suffix}\")\n          for c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for\n          c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for\n          c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for\n          c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).min().alias(f\"{c}_ET_min_{feature_suffix}\") for\n          c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in room_lists],\n\n        *[pl.col(\"event_name\").filter(pl.col(\"event_name\") == c).count().alias(f\"{c}_event_name_counts{feature_suffix}\")\n          for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\")for\n          c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for\n          c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for\n          c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).min().alias(f\"{c}_ET_min_{feature_suffix}\") for\n          c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for\n          c in event_name_feature],\n\n        *[pl.col(\"name\").filter(pl.col(\"name\") == c).count().alias(f\"{c}_name_counts{feature_suffix}\") for c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).min().alias(f\"{c}_ET_min_{feature_suffix}\") for c in\n          name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for c in\n          name_feature],\n\n        *[pl.col(\"level\").filter(pl.col(\"level\") == c).count().alias(f\"{c}_LEVEL_count{feature_suffix}\") for c in LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for c in\n          LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for c in\n          LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for c in\n          LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).min().alias(f\"{c}_ET_min_{feature_suffix}\") for c in\n          LEVELS],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for c in\n          LEVELS],\n\n        *[pl.col(\"level_group\").filter(pl.col(\"level_group\") == c).count().alias(f\"{c}_LEVEL_group_count{feature_suffix}\") for c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).min().alias(f\"{c}_ET_min_{feature_suffix}\") for c in\n          level_groups],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"level_group\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for c in\n          level_groups],\n\n        *[pl.col(\"index\").filter((pl.col(\"level\") == c) & (pl.col('room_fqid') == d)).count().alias(f\"{c}{d}_level_room_count{feature_suffix}\") for c in LEVELS for d in room_lists],\n\n\n    ]\n\n    df = x.groupby(['session_id'], maintain_order=True).agg(aggs).sort(\"session_id\")\n\n    if use_extra:\n        if grp == '5-12':\n            aggs = [\n                pl.col(\"elapsed_time\").filter((pl.col(\"text\")==\"Here's the log book.\")\n                                              |(pl.col(\"fqid\")=='logbook.page.bingo'))\n                    .apply(lambda s: s.max()-s.min()).alias(\"logbook_bingo_duration\"),\n                pl.col(\"index\").filter(\n                    (pl.col(\"text\") == \"Here's the log book.\") | (pl.col(\"fqid\") == 'logbook.page.bingo')).apply(\n                    lambda s: s.max() - s.min()).alias(\"logbook_bingo_indexCount\"),\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader')) | (\n                                pl.col(\"fqid\") == \"reader.paper2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\n                    \"reader_bingo_duration\"),\n                pl.col(\"index\").filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader')) | (\n                            pl.col(\"fqid\") == \"reader.paper2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\n                    \"reader_bingo_indexCount\"),\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals')) | (\n                                pl.col(\"fqid\") == \"journals.pic_2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\n                    \"journals_bingo_duration\"),\n                pl.col(\"index\").filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals')) | (\n                            pl.col(\"fqid\") == \"journals.pic_2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\n                    \"journals_bingo_indexCount\"),\n            ]\n            tmp = x.groupby([\"session_id\"], maintain_order=True).agg(aggs).sort(\"session_id\")\n            df = df.join(tmp, on=\"session_id\", how='left')\n\n        if grp == '13-22':\n            aggs = [\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader_flag')) | (\n                                pl.col(\"fqid\") == \"tunic.library.microfiche.reader_flag.paper2.bingo\")).apply(\n                    lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"reader_flag_duration\"),\n                pl.col(\"index\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader_flag')) | (\n                                pl.col(\"fqid\") == \"tunic.library.microfiche.reader_flag.paper2.bingo\")).apply(\n                    lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"reader_flag_indexCount\"),\n                pl.col(\"elapsed_time\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals_flag')) | (\n                                pl.col(\"fqid\") == \"journals_flag.pic_0.bingo\")).apply(\n                    lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"journalsFlag_bingo_duration\"),\n                pl.col(\"index\").filter(\n                    ((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals_flag')) | (\n                                pl.col(\"fqid\") == \"journals_flag.pic_0.bingo\")).apply(\n                    lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"journalsFlag_bingo_indexCount\")\n            ]\n            tmp = x.groupby([\"session_id\"], maintain_order=True).agg(aggs).sort(\"session_id\")\n            df = df.join(tmp, on=\"session_id\", how='left')\n\n    return df.to_pandas()","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:06:48.204060Z","iopub.execute_input":"2023-06-20T01:06:48.204603Z","iopub.status.idle":"2023-06-20T01:06:48.264450Z","shell.execute_reply.started":"2023-06-20T01:06:48.204569Z","shell.execute_reply":"2023-06-20T01:06:48.263541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf1 = feature_engineer(df1, grp='0-4', use_extra=True, feature_suffix='')","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:06:48.265589Z","iopub.execute_input":"2023-06-20T01:06:48.266395Z","iopub.status.idle":"2023-06-20T01:06:55.409768Z","shell.execute_reply.started":"2023-06-20T01:06:48.266359Z","shell.execute_reply":"2023-06-20T01:06:55.408784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf2 = feature_engineer(df2, grp='5-12', use_extra=True, feature_suffix='')","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:06:55.411101Z","iopub.execute_input":"2023-06-20T01:06:55.411610Z","iopub.status.idle":"2023-06-20T01:07:13.350094Z","shell.execute_reply.started":"2023-06-20T01:06:55.411579Z","shell.execute_reply":"2023-06-20T01:07:13.348920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf3 = feature_engineer(df3, grp='13-22', use_extra=True, feature_suffix='')","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:07:13.354115Z","iopub.execute_input":"2023-06-20T01:07:13.354506Z","iopub.status.idle":"2023-06-20T01:07:41.493489Z","shell.execute_reply.started":"2023-06-20T01:07:13.354474Z","shell.execute_reply":"2023-06-20T01:07:41.492277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.head(8)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:07:41.497710Z","iopub.execute_input":"2023-06-20T01:07:41.498119Z","iopub.status.idle":"2023-06-20T01:07:43.339402Z","shell.execute_reply.started":"2023-06-20T01:07:41.498084Z","shell.execute_reply":"2023-06-20T01:07:43.338264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:07:43.340418Z","iopub.execute_input":"2023-06-20T01:07:43.340753Z","iopub.status.idle":"2023-06-20T01:07:43.347386Z","shell.execute_reply.started":"2023-06-20T01:07:43.340724Z","shell.execute_reply":"2023-06-20T01:07:43.346431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def time_feature(train):\n    train[\"year\"] = train[\"session_id\"].apply(lambda x: int(str(x)[:2])).astype(np.uint8)\n    train[\"month\"] = train[\"session_id\"].apply(lambda x: int(str(x)[2:4])+1).astype(np.uint8)\n    train[\"day\"] = train[\"session_id\"].apply(lambda x: int(str(x)[4:6])).astype(np.uint8)\n    train[\"hour\"] = train[\"session_id\"].apply(lambda x: int(str(x)[6:8])).astype(np.uint8)\n    train[\"minute\"] = train[\"session_id\"].apply(lambda x: int(str(x)[8:10])).astype(np.uint8)\n    train[\"second\"] = train[\"session_id\"].apply(lambda x: int(str(x)[10:12])).astype(np.uint8)\n    \n    return train","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:07:43.349267Z","iopub.execute_input":"2023-06-20T01:07:43.350183Z","iopub.status.idle":"2023-06-20T01:07:43.367248Z","shell.execute_reply.started":"2023-06-20T01:07:43.350142Z","shell.execute_reply":"2023-06-20T01:07:43.365993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 = time_feature(df1)\ndf2 = time_feature(df2)\ndf3 = time_feature(df3)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:07:43.369064Z","iopub.execute_input":"2023-06-20T01:07:43.369816Z","iopub.status.idle":"2023-06-20T01:07:43.941833Z","shell.execute_reply.started":"2023-06-20T01:07:43.369775Z","shell.execute_reply":"2023-06-20T01:07:43.940496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.head(8)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:07:43.943329Z","iopub.execute_input":"2023-06-20T01:07:43.943655Z","iopub.status.idle":"2023-06-20T01:07:45.730248Z","shell.execute_reply.started":"2023-06-20T01:07:43.943626Z","shell.execute_reply":"2023-06-20T01:07:45.729284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:07:45.731387Z","iopub.execute_input":"2023-06-20T01:07:45.732309Z","iopub.status.idle":"2023-06-20T01:07:45.738095Z","shell.execute_reply.started":"2023-06-20T01:07:45.732273Z","shell.execute_reply":"2023-06-20T01:07:45.737148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null1 = df1.isnull().sum().sort_values(ascending=False)/len(df1)\nnull2 = df2.isnull().sum().sort_values(ascending=False)/len(df2)\nnull3 = df3.isnull().sum().sort_values(ascending=False)/len(df3)\n\ndrop1 = list(null1[null1 > 0.7].index)\ndrop2 = list(null2[null2 > 0.7].index)\ndrop3 = list(null3[null3 > 0.7].index)\n\nprint(len(drop1), len(drop2), len(drop3))\n\nfor col in tqdm(df1.columns):\n    if df1[col].nunique() == 1:\n        #print(col)\n        drop1.append(col)\n        #sleep(0.1)\nfor col in tqdm(df2.columns):\n    if df2[col].nunique() == 1:\n        #print(col)\n        drop2.append(col)\n        #sleep(0.1)\nfor col in tqdm(df3.columns):\n    if df3[col].nunique() == 1:\n        #print(col)\n        drop3.append(col)\n        #sleep(0.1)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:07:45.744544Z","iopub.execute_input":"2023-06-20T01:07:45.745615Z","iopub.status.idle":"2023-06-20T01:07:49.022947Z","shell.execute_reply.started":"2023-06-20T01:07:45.745560Z","shell.execute_reply":"2023-06-20T01:07:49.022070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 = df1.set_index('session_id')\ndf2 = df2.set_index('session_id')\ndf3 = df3.set_index('session_id')","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:07:49.024139Z","iopub.execute_input":"2023-06-20T01:07:49.025001Z","iopub.status.idle":"2023-06-20T01:07:49.463362Z","shell.execute_reply.started":"2023-06-20T01:07:49.024963Z","shell.execute_reply":"2023-06-20T01:07:49.462194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FEATURES1 = [c for c in df1.columns if c not in drop1+['level_group']]\nFEATURES2 = [c for c in df2.columns if c not in drop2+['level_group']]\nFEATURES3 = [c for c in df3.columns if c not in drop3+['level_group']]\n\nprint('We will train with', len(FEATURES1), len(FEATURES2), len(FEATURES3), 'features')\nALL_USERS = df1.index.unique()\nprint('We will train with', len(ALL_USERS), 'users info')","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:07:49.465112Z","iopub.execute_input":"2023-06-20T01:07:49.466114Z","iopub.status.idle":"2023-06-20T01:07:49.688309Z","shell.execute_reply.started":"2023-06-20T01:07:49.466037Z","shell.execute_reply":"2023-06-20T01:07:49.686937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Optuna Parameters optimization","metadata":{}},{"cell_type":"code","source":"def objective(trial,X,y):\n    \n    train_x, valid_x, train_y, valid_y = train_test_split(X,y, test_size=0.3)\n    \n    #cat_params = {\n    #    \"depth\": trial.suggest_int(\"depth\", 1, 9),\n    #    \"bootstrap_type\": trial.suggest_categorical(\n    #        \"bootstrap_type\", [\"Bayesian\", \"Bernoulli\", \"MVS\"]\n    #    ),\n    #    'n_estimators': trial.suggest_int('n_estimators', 150, 750),\n    #    'learning_rate': trial.suggest_loguniform('learning_rate', 1e-4, 1),\n    #    \"objective\": trial.suggest_categorical(\"objective\", [\"Logloss\", \"CrossEntropy\"]),\n    #    'random_seed': 42,\n    #    \"colsample_bylevel\": trial.suggest_float(\"colsample_bylevel\", 0.05, 1.0),\n    #    \"min_data_in_leaf\": trial.suggest_int(\"min_data_in_leaf\", 1, 100),\n    #    \"used_ram_limit\": \"6gb\",\n    #    'eval_metric': 'AUC'\n    #}\n    \n    cat_params = {\n        \"depth\": trial.suggest_int(\"depth\", 2, 9),\n        \"bootstrap_type\": trial.suggest_categorical(\n            \"bootstrap_type\", [\"Bayesian\", \"Bernoulli\", \"MVS\"]\n        ),\n        'n_estimators': trial.suggest_int('n_estimators', 100, 750),\n        'learning_rate': 0.05,\n        \"objective\": \"Logloss\",\n        'random_seed': 42,\n        \"used_ram_limit\": \"6gb\",\n        'eval_metric': 'AUC'\n    }\n\n    if cat_params[\"bootstrap_type\"] == \"Bayesian\":\n        cat_params[\"bagging_temperature\"] = trial.suggest_float(\"bagging_temperature\", 0, 10)\n    elif cat_params[\"bootstrap_type\"] == \"Bernoulli\":\n        cat_params[\"subsample\"] = trial.suggest_float(\"subsample\", 0.1, 1)\n        \n    gbm = CatBoostClassifier(**cat_params,verbose=False)\n\n    gbm.fit(train_x, train_y, eval_set=[(valid_x, valid_y)], early_stopping_rounds=60)\n\n    preds = gbm.predict(valid_x)\n    pred_labels = (preds > 0.62).astype('int')\n\n    accuracy = accuracy_score(valid_y, pred_labels)\n    \n    return accuracy","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:07:49.689931Z","iopub.execute_input":"2023-06-20T01:07:49.690332Z","iopub.status.idle":"2023-06-20T01:07:49.700822Z","shell.execute_reply.started":"2023-06-20T01:07:49.690297Z","shell.execute_reply":"2023-06-20T01:07:49.699763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n\nrf_best_params = []\nstudies = []\n\n# ITERATE THRU QUESTIONS 1 THRU 18\nfor t in tqdm(range(1,19)): #1-19\n        \n    if t <= 3:\n        grp = '0-4'\n        df = df1\n        FEATURES = FEATURES1\n    elif t <= 13:\n        grp = '5-12'\n        df = df2\n        FEATURES = FEATURES2\n    elif t <= 22:\n        grp = '13-22'\n        df = df3\n        FEATURES = FEATURES3\n\n    print(\"\")    \n    print(\"*\"*25)\n    print(f'question{t}, with{len(FEATURES)}features')\n    print('*'*25)\n            \n    # TRAIN DATA\n    train_x = df[FEATURES]\n    train_users = train_x.index.values\n    train_y = targets.loc[targets.q==t].set_index('session').loc[train_users]\n    \n    # Wrap the objective inside a lambda and call objective inside it\n    func = lambda trial: objective(trial, train_x, train_y['correct'])\n    \n    #Optuna \n    study = optuna.create_study(study_name=f\"catboost_{t}\",direction=\"maximize\")\n    study.optimize(func, n_trials=10, timeout=500)\n    \n    print(f'--- Model: {grp}_{t}')\n    print('Number of finished trials:', len(study.trials))\n    print('Best trial:', study.best_trial.params)\n    print('Best value:', study.best_value)\n    \n    rf_best_params.append(study.best_trial.params)\n    studies.append(study)\n        \nprint(\"-\"*80)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-20T01:07:49.702532Z","iopub.execute_input":"2023-06-20T01:07:49.703196Z","iopub.status.idle":"2023-06-20T03:38:26.114430Z","shell.execute_reply.started":"2023-06-20T01:07:49.703147Z","shell.execute_reply":"2023-06-20T03:38:26.112961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Historic\nplot_optimization_history(studies[15])","metadata":{"execution":{"iopub.status.busy":"2023-06-20T03:38:26.117129Z","iopub.execute_input":"2023-06-20T03:38:26.117509Z","iopub.status.idle":"2023-06-20T03:38:26.472660Z","shell.execute_reply.started":"2023-06-20T03:38:26.117476Z","shell.execute_reply":"2023-06-20T03:38:26.471425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Importance\noptuna.visualization.plot_param_importances(studies[15])","metadata":{"execution":{"iopub.status.busy":"2023-06-20T03:38:26.474343Z","iopub.execute_input":"2023-06-20T03:38:26.474690Z","iopub.status.idle":"2023-06-20T03:38:26.779099Z","shell.execute_reply.started":"2023-06-20T03:38:26.474659Z","shell.execute_reply":"2023-06-20T03:38:26.777959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#optuna.visualization.plot_slice(studies[15], params=['n_estimators','learning_rate', 'min_data_in_leaf'])","metadata":{"execution":{"iopub.status.busy":"2023-06-20T03:38:26.780943Z","iopub.execute_input":"2023-06-20T03:38:26.781356Z","iopub.status.idle":"2023-06-20T03:38:27.137804Z","shell.execute_reply.started":"2023-06-20T03:38:26.781320Z","shell.execute_reply":"2023-06-20T03:38:27.136385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#optuna.visualization.plot_slice(studies[15], params=['depth','colsample_bylevel', 'bootstrap_type'])","metadata":{"execution":{"iopub.status.busy":"2023-06-20T03:38:27.139203Z","iopub.status.idle":"2023-06-20T03:38:27.139902Z","shell.execute_reply.started":"2023-06-20T03:38:27.139674Z","shell.execute_reply":"2023-06-20T03:38:27.139697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Printing best parameters for the 18 models","metadata":{}},{"cell_type":"code","source":"#Print all params\nfor i in range(len(rf_best_params)):    \n    print(\"Model: \",i+1)\n    print(rf_best_params[i])","metadata":{"execution":{"iopub.status.busy":"2023-06-20T03:55:30.992252Z","iopub.execute_input":"2023-06-20T03:55:30.992786Z","iopub.status.idle":"2023-06-20T03:55:30.999568Z","shell.execute_reply.started":"2023-06-20T03:55:30.992746Z","shell.execute_reply":"2023-06-20T03:55:30.998533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Save best params using Picke\n\nf_save = open('best_params.pkl', 'wb')\npickle.dump(rf_best_params, f_save)\nf_save.close()","metadata":{"execution":{"iopub.status.busy":"2023-06-20T03:55:39.071808Z","iopub.execute_input":"2023-06-20T03:55:39.072261Z","iopub.status.idle":"2023-06-20T03:55:39.078600Z","shell.execute_reply.started":"2023-06-20T03:55:39.072224Z","shell.execute_reply":"2023-06-20T03:55:39.077588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"importance_dict = {}\nfor t in range(1, 19):\n    if t<=3: \n        importance_dict[str(t)] = FEATURES1\n    elif t<=13: \n        importance_dict[str(t)] = FEATURES2\n    elif t<=22:\n        importance_dict[str(t)] = FEATURES3\n\nf_save = open('importance_dict.pkl', 'wb')\npickle.dump(importance_dict, f_save)\nf_save.close()","metadata":{"execution":{"iopub.status.busy":"2023-06-20T03:55:42.458549Z","iopub.execute_input":"2023-06-20T03:55:42.458966Z","iopub.status.idle":"2023-06-20T03:55:42.466724Z","shell.execute_reply.started":"2023-06-20T03:55:42.458932Z","shell.execute_reply":"2023-06-20T03:55:42.465529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Delete from memory unused variables\n","metadata":{"execution":{"iopub.status.busy":"2023-06-20T03:38:27.147728Z","iopub.status.idle":"2023-06-20T03:38:27.148429Z","shell.execute_reply.started":"2023-06-20T03:38:27.148215Z","shell.execute_reply":"2023-06-20T03:38:27.148238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train CatBoost models","metadata":{}},{"cell_type":"code","source":"feature_importance_df = pd.DataFrame()\nmodels = {}\nresults = [[[], []] for _ in range(18)]\ngkf = GroupKFold(n_splits=2)\noof_cat = pd.DataFrame(data=np.zeros((len(ALL_USERS),18)), index=ALL_USERS, columns=[f'meta_{i}' for i in range(1, 19)])","metadata":{"execution":{"iopub.status.busy":"2023-06-20T03:55:47.033290Z","iopub.execute_input":"2023-06-20T03:55:47.033739Z","iopub.status.idle":"2023-06-20T03:55:47.042601Z","shell.execute_reply.started":"2023-06-20T03:55:47.033700Z","shell.execute_reply":"2023-06-20T03:55:47.041683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_params = pickle.load(open(\"best_params.pkl\", \"rb\"))","metadata":{"execution":{"iopub.status.busy":"2023-06-20T03:55:51.219041Z","iopub.execute_input":"2023-06-20T03:55:51.219495Z","iopub.status.idle":"2023-06-20T03:55:51.224689Z","shell.execute_reply.started":"2023-06-20T03:55:51.219457Z","shell.execute_reply":"2023-06-20T03:55:51.223752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for q in tqdm(range(1, 19)):\n    if q <= 3:\n        grp = '0-4'\n        df = df1\n        FEATURES = FEATURES1\n    elif q <= 13:\n        grp = '5-12'\n        df = df2\n        FEATURES = FEATURES2\n    elif q <= 22:\n        grp = '13-22'\n        df = df3\n        FEATURES = FEATURES3\n\n    print(\"-\"*25)\n    print(f'question{q}, with{len(FEATURES)}features')\n    print('-'*25)\n    \n    feature_importance_df = pd.DataFrame()\n    \n    for fold, (train_idx, valid_idx) in enumerate(gkf.split(X=df, groups=df.index)):\n        # TRAIN DATA\n        train_x = df.iloc[train_idx]\n        train_users = train_x.index.values\n        train_y = targets.loc[targets.q == q].set_index('session').loc[train_users]\n\n        # VALID DATA\n        valid_x = df.iloc[valid_idx]\n        valid_users = valid_x.index.values\n        valid_y = targets.loc[targets.q == q].set_index('session').loc[valid_users]\n\n        train_pool = Pool(train_x[FEATURES].astype('float32'), train_y['correct'])\n        valid_pool = Pool(valid_x[FEATURES].astype('float32'), valid_y['correct'])\n\n        model = CatBoostClassifier(**best_params[q-1],verbose=False)\n        model = model.fit(train_pool, eval_set=valid_pool)\n\n        y = valid_pool.get_label()\n        y_hat = model.predict_proba(valid_pool)[:,1]\n        models[(fold, q)] = model\n        \n        fold_importance_df = pd.DataFrame()\n        fold_importance_df[\"feature\"] = FEATURES\n        fold_importance_df[\"importance\"] = model.feature_importances_\n        fold_importance_df[\"fold\"] = fold + 1\n        feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n        \n        results[q - 1][0].append(y)\n        results[q - 1][1].append(y_hat)\n        \n    feature_importance_df = feature_importance_df.groupby(['feature'])['importance'].agg(['mean']).sort_values(by='mean', ascending=False)\n    display(feature_importance_df.head(10))\nresults = [[np.concatenate(_) for _ in _] for _ in results]","metadata":{"execution":{"iopub.status.busy":"2023-06-20T03:55:55.399982Z","iopub.execute_input":"2023-06-20T03:55:55.400436Z","iopub.status.idle":"2023-06-20T04:42:14.179446Z","shell.execute_reply.started":"2023-06-20T03:55:55.400402Z","shell.execute_reply":"2023-06-20T04:42:14.177757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for (fold,q), model in models.items():\n    model.save_model(f'fold{fold}_q{q}.cbm')","metadata":{"execution":{"iopub.status.busy":"2023-06-20T04:42:14.182517Z","iopub.execute_input":"2023-06-20T04:42:14.183319Z","iopub.status.idle":"2023-06-20T04:42:14.503795Z","shell.execute_reply.started":"2023-06-20T04:42:14.183276Z","shell.execute_reply":"2023-06-20T04:42:14.502567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"true = pd.DataFrame(np.stack([_[0] for _ in results]).T)\noof = pd.DataFrame(np.stack([_[1] for _ in results]).T)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T04:42:14.508306Z","iopub.execute_input":"2023-06-20T04:42:14.508734Z","iopub.status.idle":"2023-06-20T04:42:14.521151Z","shell.execute_reply.started":"2023-06-20T04:42:14.508699Z","shell.execute_reply":"2023-06-20T04:42:14.519961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores = []; thresholds = []\nbest_socre = 0; best_threshold = 0","metadata":{"execution":{"iopub.status.busy":"2023-06-20T04:42:14.524449Z","iopub.execute_input":"2023-06-20T04:42:14.525698Z","iopub.status.idle":"2023-06-20T04:42:14.532057Z","shell.execute_reply.started":"2023-06-20T04:42:14.525636Z","shell.execute_reply":"2023-06-20T04:42:14.531086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for threshold in np.arange(0.5, 0.7, 0.01):\n    preds = (oof.values.reshape(-1) > threshold).astype('int')\n    m = f1_score(true.values.reshape(-1), preds, average='macro')\n    scores.append(m)\n    thresholds.append(threshold)\n    if m > best_socre:\n        best_socre = m\n        best_threshold = threshold\n\nplt.figure(figsize=(15, 5))\nplt.plot(thresholds, scores, '-o', color='green')\nplt.scatter([best_threshold], [best_socre], color='green')\nplt.xlabel(\"Threshold\", size=14)\nplt.ylabel(\"Validation F1 Score\",size=14)\nplt.title(f'Threshold vs. F1_Score with Best F1_Score={best_socre:.3f} at Best Threshold = {best_threshold:.3}', size=18)\nplt.show()\n\nprint(f'When using optimal threshold = {best_threshold:.2f}...')\nfor k in range(18):\n    m = f1_score(true[k].values, (oof[k].values > best_threshold).astype('int'), average = 'macro')\n    print(f'Q{k+1}: F1 =',m)\nm = f1_score(true.values.reshape(-1), (oof.values > best_threshold).reshape(-1).astype('int'), average = 'macro')\nprint('==> Overall F1 =', m)\nprint('s')","metadata":{"execution":{"iopub.status.busy":"2023-06-20T04:42:14.533623Z","iopub.execute_input":"2023-06-20T04:42:14.533982Z","iopub.status.idle":"2023-06-20T04:42:18.475049Z","shell.execute_reply.started":"2023-06-20T04:42:14.533951Z","shell.execute_reply":"2023-06-20T04:42:18.474077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission Jo Wilder","metadata":{}},{"cell_type":"code","source":"import jo_wilder_310 as jo_wilder\nenv = jo_wilder.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2023-06-20T04:42:18.476926Z","iopub.execute_input":"2023-06-20T04:42:18.477657Z","iopub.status.idle":"2023-06-20T04:42:18.504822Z","shell.execute_reply.started":"2023-06-20T04:42:18.477617Z","shell.execute_reply":"2023-06-20T04:42:18.503879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models_list = [[CatBoostClassifier().load_model(\n    f\"fold{fold}_q{q}.cbm\"\n) for fold in range(2)] for q in range(1, 19)]","metadata":{"execution":{"iopub.status.busy":"2023-06-20T04:42:18.506420Z","iopub.execute_input":"2023-06-20T04:42:18.507105Z","iopub.status.idle":"2023-06-20T04:42:18.557445Z","shell.execute_reply.started":"2023-06-20T04:42:18.507065Z","shell.execute_reply":"2023-06-20T04:42:18.556404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"limits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\nfor test, sample_submission in iter_test:\n    test = test.sort_values(by = 'index')\n    sample_submission['q'] = sample_submission.session_id.apply(lambda x: x.split(\"_q\")[1]).astype(int)\n    sample_submission = sample_submission.sort_values(by = 'q')\n    sample_submission.drop('q',axis=1,inplace=True)\n    \n    sample_submission['question'] = [int(label.split('_')[1][1:]) for label in sample_submission['session_id']]\n    grp = test.level_group.values[0]\n    session_id = test.session_id.values[0]\n    a,b = limits[grp]    \n        \n    df = (pl.from_pandas(test)\n          .drop([\"fullscreen\", \"hq\", \"music\"])\n          .with_columns(columns))\n    df = feature_engineer(df, grp, use_extra=True, feature_suffix='')\n    df = time_feature(df)\n    \n    fold = 1\n    preds = []\n    a,b = limits[grp]\n    \n    for q in range(a, b):\n        print(\"Question: \",q)\n        FEATURES = importance_dict[str(q)]\n        model = models_list[q-1][fold]\n        \n        pred = model.predict_proba(df[FEATURES].astype(np.float32))[:,1]\n        mask = sample_submission.question == q \n        sample_submission.loc[mask, 'correct'] = (pred > best_threshold).astype('int') \n\n    env.predict(sample_submission[['session_id', 'correct']])","metadata":{"execution":{"iopub.status.busy":"2023-06-20T04:42:18.559059Z","iopub.execute_input":"2023-06-20T04:42:18.559719Z","iopub.status.idle":"2023-06-20T04:42:24.949753Z","shell.execute_reply.started":"2023-06-20T04:42:18.559680Z","shell.execute_reply":"2023-06-20T04:42:24.948363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('submission.csv')\nprint(\"Shape: \",sub.shape)\nsub1 = sub[sub[\"session_id\"].str.contains(\"session_id\") == False]\nprint(\"Shape: \",sub1.shape)\nsub1[\"correct\"] = pd.to_numeric(sub1[\"correct\"], downcast=\"float\")\nprint(\"==> Test MEAN: \",np.mean(sub1[\"correct\"]))\nsub1.head(18)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T04:42:24.951405Z","iopub.execute_input":"2023-06-20T04:42:24.951793Z","iopub.status.idle":"2023-06-20T04:42:24.976472Z","shell.execute_reply.started":"2023-06-20T04:42:24.951755Z","shell.execute_reply":"2023-06-20T04:42:24.975419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}