{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"# !pip install polars","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:23:11.435912Z","iopub.execute_input":"2023-04-17T22:23:11.436276Z","iopub.status.idle":"2023-04-17T22:23:11.457658Z","shell.execute_reply.started":"2023-04-17T22:23:11.436244Z","shell.execute_reply":"2023-04-17T22:23:11.456863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading training file","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport warnings\nimport polars as pl\nimport gc\nimport os\nfrom sklearn.model_selection import KFold, GroupKFold\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import f1_score\nfrom vowpalwabbit.sklearn_vw import VWClassifier\nfrom sklearn.metrics import classification_report\nfrom sklearn.model_selection import GridSearchCV,RandomizedSearchCV\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n# settings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-17T22:23:11.458976Z","iopub.execute_input":"2023-04-17T22:23:11.459259Z","iopub.status.idle":"2023-04-17T22:23:12.538274Z","shell.execute_reply.started":"2023-04-17T22:23:11.459231Z","shell.execute_reply":"2023-04-17T22:23:12.537458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtypes = {\"session_id\": pl.Int64,\n          \"elapsed_time\": pl.Int64,\n          \"event_name\": pl.Categorical,\n          \"name\": pl.Categorical,\n          \"level\": pl.Int8,\n          \"page\": pl.Float32,\n          \"room_coor_x\": pl.Float32,\n          \"room_coor_y\": pl.Float32,\n          \"screen_coor_x\": pl.Float32,\n          \"screen_coor_y\": pl.Float32,\n          \"hover_duration\": pl.Float32,\n          \"text\": pl.Categorical,\n          \"fqid\": pl.Categorical,\n          \"room_fqid\": pl.Categorical,\n          \"text_fqid\": pl.Categorical,\n          \"fullscreen\": pl.Int8,\n          \"hq\": pl.Int8,\n          \"music\": pl.Int8,\n          \"level_group\": pl.Categorical\n          }","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:23:12.542102Z","iopub.execute_input":"2023-04-17T22:23:12.544077Z","iopub.status.idle":"2023-04-17T22:23:12.551896Z","shell.execute_reply.started":"2023-04-17T22:23:12.544041Z","shell.execute_reply":"2023-04-17T22:23:12.550994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = [\n\n    pl.col(\"page\").cast(pl.Float32),\n    (\n        (pl.col(\"elapsed_time\") - pl.col(\"elapsed_time\").shift(1)) \n         .fill_null(0)\n         .clip(0, 1e9)\n         .over([\"session_id\", \"level_group\"])\n         .alias(\"elapsed_time_diff\")\n    ),\n    (\n        (pl.col(\"screen_coor_x\") - pl.col(\"screen_coor_x\").shift(1)) \n         .fill_null(0)\n         .abs()\n         .over([\"session_id\", \"level_group\"])\n        .alias(\"location_x_diff\") \n    ),\n    (\n        (pl.col(\"screen_coor_y\") - pl.col(\"screen_coor_y\").shift(1)) \n         .fill_null(0)\n         .abs()\n         .over([\"session_id\", \"level_group\"])\n        .alias(\"location_y_diff\") \n    )\n]","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:23:12.556778Z","iopub.execute_input":"2023-04-17T22:23:12.558911Z","iopub.status.idle":"2023-04-17T22:23:12.581726Z","shell.execute_reply.started":"2023-04-17T22:23:12.558878Z","shell.execute_reply":"2023-04-17T22:23:12.580591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pl.StringCache()\npl.toggle_string_cache(True)\n#pub fn toggle_string_cache(toggle: bool)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:23:12.585655Z","iopub.execute_input":"2023-04-17T22:23:12.587710Z","iopub.status.idle":"2023-04-17T22:23:12.593358Z","shell.execute_reply.started":"2023-04-17T22:23:12.587676Z","shell.execute_reply":"2023-04-17T22:23:12.592372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reading from Kaggle Environment","metadata":{}},{"cell_type":"code","source":"%%time\nfiles_status = True\n\ntry:    \n    #train = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv', dtype=dtypes)\n    train = (pl.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv',dtypes=dtypes)\n                .drop([\"fullscreen\", \"hq\", \"music\"])\n                .with_columns(columns)\n              )\n    targets = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\n    #test = pd.read_csv(\"'/kaggle/input/predict-student-performance-from-game-play/test.csv'\")\nexcept OSError as e:\n    print(\"files not found in Kaggle environment: \",e.errno)\n    files_status = False","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:23:12.597724Z","iopub.execute_input":"2023-04-17T22:23:12.600101Z","iopub.status.idle":"2023-04-17T22:24:24.365522Z","shell.execute_reply.started":"2023-04-17T22:23:12.600066Z","shell.execute_reply":"2023-04-17T22:24:24.364149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reading from local environment (files folder)","metadata":{}},{"cell_type":"code","source":"%%time\n#If environment is not kaggle notebook, read them from a folder \n\ntry:\n    if files_status == False:\n        train = (pl.read_csv(\"predict-student-performance-from-game-play/train.csv\", dtypes=dtypes)\n                    .drop([\"fullscreen\", \"hq\", \"music\"])\n                    .with_columns(columns)\n                )\n        targets = pd.read_csv('predict-student-performance-from-game-play/train_labels.csv')\nexcept OSError as e:\n    print(\"Files not found\")","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:24:24.367041Z","iopub.execute_input":"2023-04-17T22:24:24.367435Z","iopub.status.idle":"2023-04-17T22:24:24.378128Z","shell.execute_reply.started":"2023-04-17T22:24:24.367403Z","shell.execute_reply":"2023-04-17T22:24:24.377134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Memory usage of dataframe is {round(train.estimated_size('mb'), 2)} MB\")","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:24:24.379207Z","iopub.execute_input":"2023-04-17T22:24:24.379485Z","iopub.status.idle":"2023-04-17T22:24:24.386023Z","shell.execute_reply.started":"2023-04-17T22:24:24.379459Z","shell.execute_reply":"2023-04-17T22:24:24.385196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reducing training file","metadata":{}},{"cell_type":"code","source":"def reduce_memory_usage_pl(df, name):\n    \"\"\" Reduce memory usage by polars dataframe {df} with name {name} by changing its data types.\n        Original pandas version of this function: https://www.kaggle.com/code/arjanso/reducing-dataframe-memory-size-by-65 \"\"\"\n    print(f\"Memory usage of dataframe {name} is {round(df.estimated_size('mb'), 2)} MB\")\n    Numeric_Int_types = [pl.Int8,pl.Int16,pl.Int32,pl.Int64]\n    Numeric_Float_types = [pl.Float32,pl.Float64]    \n    for col in df.columns:\n        col_type = df[col].dtype\n        c_min = df[col].min()\n        c_max = df[col].max()\n        if col_type in Numeric_Int_types:\n            if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                df = df.with_columns(df[col].cast(pl.Int8))\n            elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                df = df.with_columns(df[col].cast(pl.Int16))\n            elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                df = df.with_columns(df[col].cast(pl.Int32))\n            elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                df = df.with_columns(df[col].cast(pl.Int64))\n        elif col_type in Numeric_Float_types:\n            if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                df = df.with_columns(df[col].cast(pl.Float32))\n            else:\n                pass\n        elif col_type == pl.Utf8:\n            df = df.with_columns(df[col].cast(pl.Categorical))\n        else:\n            pass\n    \n    print(f\"Memory usage of dataframe {name} became {round(df.estimated_size('mb'), 2)} MB\")\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:24:24.387093Z","iopub.execute_input":"2023-04-17T22:24:24.387711Z","iopub.status.idle":"2023-04-17T22:24:24.400370Z","shell.execute_reply.started":"2023-04-17T22:24:24.387678Z","shell.execute_reply":"2023-04-17T22:24:24.399310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reducing polar\ntrain = reduce_memory_usage_pl(train, \"train_subset\")","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:24:24.404297Z","iopub.execute_input":"2023-04-17T22:24:24.404646Z","iopub.status.idle":"2023-04-17T22:24:26.221817Z","shell.execute_reply.started":"2023-04-17T22:24:24.404616Z","shell.execute_reply":"2023-04-17T22:24:26.220970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train.columns)\nprint(train.shape)\nprint(len(train))\nprint(type(train))","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:24:26.222822Z","iopub.execute_input":"2023-04-17T22:24:26.223639Z","iopub.status.idle":"2023-04-17T22:24:26.228563Z","shell.execute_reply.started":"2023-04-17T22:24:26.223606Z","shell.execute_reply":"2023-04-17T22:24:26.227522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.tail(4)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:24:26.229807Z","iopub.execute_input":"2023-04-17T22:24:26.230478Z","iopub.status.idle":"2023-04-17T22:24:26.253712Z","shell.execute_reply.started":"2023-04-17T22:24:26.230426Z","shell.execute_reply":"2023-04-17T22:24:26.252797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering for training file","metadata":{}},{"cell_type":"code","source":"CATS = ['event_name', 'name', 'fqid', 'room_fqid', 'text_fqid']\nNUMS = ['page', 'room_coor_x', 'room_coor_y', 'screen_coor_x', 'screen_coor_y',\n        'hover_duration', 'elapsed_time_diff']\n\nname_feature = ['basic', 'undefined', 'close', 'open', 'prev', 'next']\nevent_name_feature = ['cutscene_click', 'person_click', 'navigate_click',\n       'observation_click', 'notification_click', 'object_click',\n       'object_hover', 'map_hover', 'map_click', 'checkpoint',\n       'notebook_click']\n\n# from https://www.kaggle.com/code/leehomhuang/catboost-baseline-with-lots-features-inference :\nfqid_lists = ['worker', 'archivist', 'gramps', 'wells', 'toentry', 'confrontation', 'crane_ranger', 'groupconvo', 'flag_girl', 'tomap', 'tostacks', 'tobasement', 'archivist_glasses', 'boss', 'journals', 'seescratches', 'groupconvo_flag', 'cs', 'teddy', 'expert', 'businesscards', 'ch3start', 'tunic.historicalsociety', 'tofrontdesk', 'savedteddy', 'plaque', 'glasses', 'tunic.drycleaner', 'reader_flag', 'tunic.library', 'tracks', 'tunic.capitol_2', 'trigger_scarf', 'reader', 'directory', 'tunic.capitol_1', 'journals.pic_0.next', 'unlockdoor', 'tunic', 'what_happened', 'tunic.kohlcenter', 'tunic.humanecology', 'colorbook', 'logbook', 'businesscards.card_0.next', 'journals.hub.topics', 'logbook.page.bingo', 'journals.pic_1.next', 'journals_flag', 'reader.paper0.next', 'tracks.hub.deer', 'reader_flag.paper0.next', 'trigger_coffee', 'wellsbadge', 'journals.pic_2.next', 'tomicrofiche', 'journals_flag.pic_0.bingo', 'plaque.face.date', 'notebook', 'tocloset_dirty', 'businesscards.card_bingo.bingo', 'businesscards.card_1.next', 'tunic.wildlife', 'tunic.hub.slip', 'tocage', 'journals.pic_2.bingo', 'tocollectionflag', 'tocollection', 'chap4_finale_c', 'chap2_finale_c', 'lockeddoor', 'journals_flag.hub.topics', 'tunic.capitol_0', 'reader_flag.paper2.bingo', 'photo', 'tunic.flaghouse', 'reader.paper1.next', 'directory.closeup.archivist', 'intro', 'businesscards.card_bingo.next', 'reader.paper2.bingo', 'retirement_letter', 'remove_cup', 'journals_flag.pic_0.next', 'magnify', 'coffee', 'key', 'togrampa', 'reader_flag.paper1.next', 'janitor', 'tohallway', 'chap1_finale', 'report', 'outtolunch', 'journals_flag.hub.topics_old', 'journals_flag.pic_1.next', 'reader.paper2.next', 'chap1_finale_c', 'reader_flag.paper2.next', 'door_block_talk', 'journals_flag.pic_1.bingo', 'journals_flag.pic_2.next', 'journals_flag.pic_2.bingo', 'block_magnify', 'reader.paper0.prev', 'block', 'reader_flag.paper0.prev', 'block_0', 'door_block_clean', 'reader.paper2.prev', 'reader.paper1.prev', 'doorblock', 'tocloset', 'reader_flag.paper2.prev', 'reader_flag.paper1.prev', 'block_tomap2', 'journals_flag.pic_0_old.next', 'journals_flag.pic_1_old.next', 'block_tocollection', 'block_nelson', 'journals_flag.pic_2_old.next', 'block_tomap1', 'block_badge', 'need_glasses', 'block_badge_2', 'fox', 'block_1']\ntext_lists = ['tunic.historicalsociety.cage.confrontation', 'tunic.wildlife.center.crane_ranger.crane', 'tunic.historicalsociety.frontdesk.archivist.newspaper', 'tunic.historicalsociety.entry.groupconvo', 'tunic.wildlife.center.wells.nodeer', 'tunic.historicalsociety.frontdesk.archivist.have_glass', 'tunic.drycleaner.frontdesk.worker.hub', 'tunic.historicalsociety.closet_dirty.gramps.news', 'tunic.humanecology.frontdesk.worker.intro', 'tunic.historicalsociety.frontdesk.archivist_glasses.confrontation', 'tunic.historicalsociety.basement.seescratches', 'tunic.historicalsociety.collection.cs', 'tunic.flaghouse.entry.flag_girl.hello', 'tunic.historicalsociety.collection.gramps.found', 'tunic.historicalsociety.basement.ch3start', 'tunic.historicalsociety.entry.groupconvo_flag', 'tunic.library.frontdesk.worker.hello', 'tunic.library.frontdesk.worker.wells', 'tunic.historicalsociety.collection_flag.gramps.flag', 'tunic.historicalsociety.basement.savedteddy', 'tunic.library.frontdesk.worker.nelson', 'tunic.wildlife.center.expert.removed_cup', 'tunic.library.frontdesk.worker.flag', 'tunic.historicalsociety.frontdesk.archivist.hello', 'tunic.historicalsociety.closet.gramps.intro_0_cs_0', 'tunic.historicalsociety.entry.boss.flag', 'tunic.flaghouse.entry.flag_girl.symbol', 'tunic.historicalsociety.closet_dirty.trigger_scarf', 'tunic.drycleaner.frontdesk.worker.done', 'tunic.historicalsociety.closet_dirty.what_happened', 'tunic.wildlife.center.wells.animals', 'tunic.historicalsociety.closet.teddy.intro_0_cs_0', 'tunic.historicalsociety.cage.glasses.afterteddy', 'tunic.historicalsociety.cage.teddy.trapped', 'tunic.historicalsociety.cage.unlockdoor', 'tunic.historicalsociety.stacks.journals.pic_2.bingo', 'tunic.historicalsociety.entry.wells.flag', 'tunic.humanecology.frontdesk.worker.badger', 'tunic.historicalsociety.stacks.journals_flag.pic_0.bingo', 'tunic.historicalsociety.closet.intro', 'tunic.historicalsociety.closet.retirement_letter.hub', 'tunic.historicalsociety.entry.directory.closeup.archivist', 'tunic.historicalsociety.collection.tunic.slip', 'tunic.kohlcenter.halloffame.plaque.face.date', 'tunic.historicalsociety.closet_dirty.trigger_coffee', 'tunic.drycleaner.frontdesk.logbook.page.bingo', 'tunic.library.microfiche.reader.paper2.bingo', 'tunic.kohlcenter.halloffame.togrampa', 'tunic.capitol_2.hall.boss.haveyougotit', 'tunic.wildlife.center.wells.nodeer_recap', 'tunic.historicalsociety.cage.glasses.beforeteddy', 'tunic.historicalsociety.closet_dirty.gramps.helpclean', 'tunic.wildlife.center.expert.recap', 'tunic.historicalsociety.frontdesk.archivist.have_glass_recap', 'tunic.historicalsociety.stacks.journals_flag.pic_1.bingo', 'tunic.historicalsociety.cage.lockeddoor', 'tunic.historicalsociety.stacks.journals_flag.pic_2.bingo', 'tunic.historicalsociety.collection.gramps.lost', 'tunic.historicalsociety.closet.notebook', 'tunic.historicalsociety.frontdesk.magnify', 'tunic.humanecology.frontdesk.businesscards.card_bingo.bingo', 'tunic.wildlife.center.remove_cup', 'tunic.library.frontdesk.wellsbadge.hub', 'tunic.wildlife.center.tracks.hub.deer', 'tunic.historicalsociety.frontdesk.key', 'tunic.library.microfiche.reader_flag.paper2.bingo', 'tunic.flaghouse.entry.colorbook', 'tunic.wildlife.center.coffee', 'tunic.capitol_1.hall.boss.haveyougotit', 'tunic.historicalsociety.basement.janitor', 'tunic.historicalsociety.collection_flag.gramps.recap', 'tunic.wildlife.center.wells.animals2', 'tunic.flaghouse.entry.flag_girl.symbol_recap', 'tunic.historicalsociety.closet_dirty.photo', 'tunic.historicalsociety.stacks.outtolunch', 'tunic.library.frontdesk.worker.wells_recap', 'tunic.historicalsociety.frontdesk.archivist_glasses.confrontation_recap', 'tunic.capitol_0.hall.boss.talktogramps', 'tunic.historicalsociety.closet.photo', 'tunic.historicalsociety.collection.tunic', 'tunic.historicalsociety.closet.teddy.intro_0_cs_5', 'tunic.historicalsociety.closet_dirty.gramps.archivist', 'tunic.historicalsociety.closet_dirty.door_block_talk', 'tunic.historicalsociety.entry.boss.flag_recap', 'tunic.historicalsociety.frontdesk.archivist.need_glass_0', 'tunic.historicalsociety.entry.wells.talktogramps', 'tunic.historicalsociety.frontdesk.block_magnify', 'tunic.historicalsociety.frontdesk.archivist.foundtheodora', 'tunic.historicalsociety.closet_dirty.gramps.nothing', 'tunic.historicalsociety.closet_dirty.door_block_clean', 'tunic.capitol_1.hall.boss.writeitup', 'tunic.library.frontdesk.worker.nelson_recap', 'tunic.library.frontdesk.worker.hello_short', 'tunic.historicalsociety.stacks.block', 'tunic.historicalsociety.frontdesk.archivist.need_glass_1', 'tunic.historicalsociety.entry.boss.talktogramps', 'tunic.historicalsociety.frontdesk.archivist.newspaper_recap', 'tunic.historicalsociety.entry.wells.flag_recap', 'tunic.drycleaner.frontdesk.worker.done2', 'tunic.library.frontdesk.worker.flag_recap', 'tunic.humanecology.frontdesk.block_0', 'tunic.library.frontdesk.worker.preflag', 'tunic.historicalsociety.basement.gramps.seeyalater', 'tunic.flaghouse.entry.flag_girl.hello_recap', 'tunic.historicalsociety.closet.doorblock', 'tunic.drycleaner.frontdesk.worker.takealook', 'tunic.historicalsociety.basement.gramps.whatdo', 'tunic.library.frontdesk.worker.droppedbadge', 'tunic.historicalsociety.entry.block_tomap2', 'tunic.library.frontdesk.block_nelson', 'tunic.library.microfiche.block_0', 'tunic.historicalsociety.entry.block_tocollection', 'tunic.historicalsociety.entry.block_tomap1', 'tunic.historicalsociety.collection.gramps.look_0', 'tunic.library.frontdesk.block_badge', 'tunic.historicalsociety.cage.need_glasses', 'tunic.library.frontdesk.block_badge_2', 'tunic.kohlcenter.halloffame.block_0', 'tunic.capitol_0.hall.chap1_finale_c', 'tunic.capitol_1.hall.chap2_finale_c', 'tunic.capitol_2.hall.chap4_finale_c', 'tunic.wildlife.center.fox.concern', 'tunic.drycleaner.frontdesk.block_0', 'tunic.historicalsociety.entry.gramps.hub', 'tunic.humanecology.frontdesk.block_1', 'tunic.drycleaner.frontdesk.block_1']\nroom_lists = ['tunic.historicalsociety.entry', 'tunic.wildlife.center', 'tunic.historicalsociety.cage', 'tunic.library.frontdesk', 'tunic.historicalsociety.frontdesk', 'tunic.historicalsociety.stacks', 'tunic.historicalsociety.closet_dirty', 'tunic.humanecology.frontdesk', 'tunic.historicalsociety.basement', 'tunic.kohlcenter.halloffame', 'tunic.library.microfiche', 'tunic.drycleaner.frontdesk', 'tunic.historicalsociety.collection', 'tunic.historicalsociety.closet', 'tunic.flaghouse.entry', 'tunic.historicalsociety.collection_flag', 'tunic.capitol_1.hall', 'tunic.capitol_0.hall', 'tunic.capitol_2.hall']","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:24:26.255050Z","iopub.execute_input":"2023-04-17T22:24:26.255499Z","iopub.status.idle":"2023-04-17T22:24:26.274413Z","shell.execute_reply.started":"2023-04-17T22:24:26.255467Z","shell.execute_reply":"2023-04-17T22:24:26.273625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineer_pl(x, grp, use_extra, feature_suffix):\n        \n    aggs = [\n        pl.col(\"index\").count().alias(f\"session_number_{feature_suffix}\"),\n      \n        *[pl.col(c).drop_nulls().n_unique().alias(f\"{c}_unique_{feature_suffix}\") for c in CATS],\n        [pl.col(c).quantile(0.1, \"nearest\").alias(f\"{c}_quantile1_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).quantile(0.2, \"nearest\").alias(f\"{c}_quantile2_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).quantile(0.4, \"nearest\").alias(f\"{c}_quantile4_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).quantile(0.6, \"nearest\").alias(f\"{c}_quantile6_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).quantile(0.8, \"nearest\").alias(f\"{c}_quantile8_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).quantile(0.9, \"nearest\").alias(f\"{c}_quantile9_{feature_suffix}\") for c in NUMS],\n        \n        *[pl.col(c).mean().alias(f\"{c}_mean_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).std().alias(f\"{c}_std_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).min().alias(f\"{c}_min_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).max().alias(f\"{c}_max_{feature_suffix}\") for c in NUMS],\n        \n        *[pl.col(\"event_name\").filter(pl.col(\"event_name\") == c).count().alias(f\"{c}_event_name_counts{feature_suffix}\")for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\")==c).quantile(0.1, \"nearest\").alias(f\"{c}_ET_quantile1_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\")==c).quantile(0.2, \"nearest\").alias(f\"{c}_ET_quantile2_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\")==c).quantile(0.4, \"nearest\").alias(f\"{c}_ET_quantile4_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\")==c).quantile(0.6, \"nearest\").alias(f\"{c}_ET_quantile6_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\")==c).quantile(0.8, \"nearest\").alias(f\"{c}_ET_quantile8_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\")==c).quantile(0.9, \"nearest\").alias(f\"{c}_ET_quantile9_{feature_suffix}\") for c in event_name_feature],      \n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\")==c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\")==c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\")==c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"event_name\")==c).min().alias(f\"{c}_ET_min_{feature_suffix}\") for c in event_name_feature],\n     \n        *[pl.col(\"name\").filter(pl.col(\"name\") == c).count().alias(f\"{c}_name_counts{feature_suffix}\")for c in name_feature],   \n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\")==c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for c in name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\")==c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for c in name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\")==c).min().alias(f\"{c}_ET_min_{feature_suffix}\") for c in name_feature],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"name\")==c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for c in name_feature],  \n        \n        *[pl.col(\"room_fqid\").filter(pl.col(\"room_fqid\") == c).count().alias(f\"{c}_room_fqid_counts{feature_suffix}\")for c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).min().alias(f\"{c}_ET_min_{feature_suffix}\") for c in room_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"room_fqid\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for c in room_lists],\n                \n        *[pl.col(\"fqid\").filter(pl.col(\"fqid\") == c).count().alias(f\"{c}_fqid_counts{feature_suffix}\")for c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).min().alias(f\"{c}_ET_min_{feature_suffix}\") for c in fqid_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"fqid\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for c in fqid_lists],\n       \n        *[pl.col(\"text_fqid\").filter(pl.col(\"text_fqid\") == c).count().alias(f\"{c}_text_fqid_counts{feature_suffix}\") for c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).std().alias(f\"{c}_ET_std_{feature_suffix}\") for c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).mean().alias(f\"{c}_ET_mean_{feature_suffix}\") for c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).max().alias(f\"{c}_ET_max_{feature_suffix}\") for c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).min().alias(f\"{c}_ET_min_{feature_suffix}\") for c in text_lists],\n        *[pl.col(\"elapsed_time_diff\").filter(pl.col(\"text_fqid\") == c).sum().alias(f\"{c}_ET_sum_{feature_suffix}\") for c in text_lists],\n         \n        *[pl.col(\"location_x_diff\").filter(pl.col(\"event_name\")==c).mean().alias(f\"{c}_ET_mean_x{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"location_x_diff\").filter(pl.col(\"event_name\")==c).std().alias(f\"{c}_ET_std_x{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"location_x_diff\").filter(pl.col(\"event_name\")==c).max().alias(f\"{c}_ET_max_x{feature_suffix}\") for c in event_name_feature],\n        *[pl.col(\"location_x_diff\").filter(pl.col(\"event_name\")==c).min().alias(f\"{c}_ET_min_x{feature_suffix}\") for c in event_name_feature],\n        ]\n    \n    df = x.groupby([\"session_id\"], maintain_order=True).agg(aggs).sort(\"session_id\")\n  \n    if use_extra:\n        if grp=='5-12':\n            aggs = [\n                pl.col(\"elapsed_time\").filter((pl.col(\"text\")==\"Here's the log book.\")|(pl.col(\"fqid\")=='logbook.page.bingo')).apply(lambda s: s.max()-s.min()).alias(\"logbook_bingo_duration\"),\n                pl.col(\"index\").filter((pl.col(\"text\")==\"Here's the log book.\")|(pl.col(\"fqid\")=='logbook.page.bingo')).apply(lambda s: s.max()-s.min()).alias(\"logbook_bingo_indexCount\"),\n                pl.col(\"elapsed_time\").filter(((pl.col(\"event_name\")=='navigate_click')&(pl.col(\"fqid\")=='reader'))|(pl.col(\"fqid\")==\"reader.paper2.bingo\")).apply(lambda s: s.max()-s.min()).alias(\"reader_bingo_duration\"),\n                pl.col(\"index\").filter(((pl.col(\"event_name\")=='navigate_click')&(pl.col(\"fqid\")=='reader'))|(pl.col(\"fqid\")==\"reader.paper2.bingo\")).apply(lambda s: s.max()-s.min()).alias(\"reader_bingo_indexCount\"),\n                pl.col(\"elapsed_time\").filter(((pl.col(\"event_name\")=='navigate_click')&(pl.col(\"fqid\")=='journals'))|(pl.col(\"fqid\")==\"journals.pic_2.bingo\")).apply(lambda s: s.max()-s.min()).alias(\"journals_bingo_duration\"),\n                pl.col(\"index\").filter(((pl.col(\"event_name\")=='navigate_click')&(pl.col(\"fqid\")=='journals'))|(pl.col(\"fqid\")==\"journals.pic_2.bingo\")).apply(lambda s: s.max()-s.min()).alias(\"journals_bingo_indexCount\"),\n            ]\n            tmp = x.groupby([\"session_id\"], maintain_order=True).agg(aggs).sort(\"session_id\")\n            df = df.join(tmp, on=\"session_id\", how='left')\n\n        if grp=='13-22':\n            aggs = [\n                pl.col(\"elapsed_time\").filter(((pl.col(\"event_name\")=='navigate_click')&(pl.col(\"fqid\")=='reader_flag'))|(pl.col(\"fqid\")==\"tunic.library.microfiche.reader_flag.paper2.bingo\")).apply(lambda s: s.max()-s.min() if s.len()>0 else 0).alias(\"reader_flag_duration\"),\n                pl.col(\"index\").filter(((pl.col(\"event_name\")=='navigate_click')&(pl.col(\"fqid\")=='reader_flag'))|(pl.col(\"fqid\")==\"tunic.library.microfiche.reader_flag.paper2.bingo\")).apply(lambda s: s.max()-s.min() if s.len()>0 else 0).alias(\"reader_flag_indexCount\"),\n                pl.col(\"elapsed_time\").filter(((pl.col(\"event_name\")=='navigate_click')&(pl.col(\"fqid\")=='journals_flag'))|(pl.col(\"fqid\")==\"journals_flag.pic_0.bingo\")).apply(lambda s: s.max()-s.min() if s.len()>0 else 0).alias(\"journalsFlag_bingo_duration\"),\n                pl.col(\"index\").filter(((pl.col(\"event_name\")=='navigate_click')&(pl.col(\"fqid\")=='journals_flag'))|(pl.col(\"fqid\")==\"journals_flag.pic_0.bingo\")).apply(lambda s: s.max()-s.min() if s.len()>0 else 0).alias(\"journalsFlag_bingo_indexCount\"),\n            ]\n            tmp = x.groupby([\"session_id\"], maintain_order=True).agg(aggs).sort(\"session_id\")\n            df = df.join(tmp, on=\"session_id\", how='left')\n        \n    return df.to_pandas()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:24:26.275850Z","iopub.execute_input":"2023-04-17T22:24:26.276247Z","iopub.status.idle":"2023-04-17T22:24:26.327275Z","shell.execute_reply.started":"2023-04-17T22:24:26.276216Z","shell.execute_reply":"2023-04-17T22:24:26.326107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Filling Null Values","metadata":{}},{"cell_type":"code","source":"#Quantity of nulls per column\ndef nulls_per_column(df_polar):\n    len_df_polar = len(df_polar)\n    for col in df_polar.get_columns():\n        per_nulls = round((col.is_null().sum() / len_df_polar) * 100,2)\n        print(f'{col.name} - {per_nulls}%')","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:24:26.328769Z","iopub.execute_input":"2023-04-17T22:24:26.329233Z","iopub.status.idle":"2023-04-17T22:24:26.342964Z","shell.execute_reply.started":"2023-04-17T22:24:26.329199Z","shell.execute_reply":"2023-04-17T22:24:26.341952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_missing_values(df_polar):\n    \n    \n    filled_polar = df_polar.select(\n        pl.col('session_id'),\n        pl.col('index'),\n        pl.col('elapsed_time'),\n        pl.col('event_name'),\n        pl.col('name'),\n        pl.col('level'),\n        pl.col('page').fill_null(0),\n        pl.col('room_coor_x').fill_null(strategy=\"backward\"),\n        pl.col('room_coor_x').fill_null(strategy='backward'),\n        pl.col('room_coor_y').fill_null(strategy='backward'),\n        pl.col('screen_coor_x').fill_null(strategy='backward'),\n        pl.col('screen_coor_y').fill_null(strategy='backward'),\n        pl.col('hover_duration').fill_null(0),\n        pl.col('text').fill_null('No Text'),\n        pl.col('fqid').fill_null('No fqid'),\n        pl.col('room_fqid'),\n        pl.col('text_fqid').fill_null('No text_fqid'),\n        pl.col('level_group'),\n        pl.col('elapsed_time_diff'),\n        pl.col('location_x_diff'),\n        pl.col('location_y_diff')\n    )\n    \n    filled_polar.tail(5)\n    \n    return filled_polar","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:24:26.344211Z","iopub.execute_input":"2023-04-17T22:24:26.344568Z","iopub.status.idle":"2023-04-17T22:24:26.354376Z","shell.execute_reply.started":"2023-04-17T22:24:26.344519Z","shell.execute_reply":"2023-04-17T22:24:26.353300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Nulls of train dataset\nnulls_per_column(train)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:24:26.355843Z","iopub.execute_input":"2023-04-17T22:24:26.356262Z","iopub.status.idle":"2023-04-17T22:24:26.391777Z","shell.execute_reply.started":"2023-04-17T22:24:26.356232Z","shell.execute_reply":"2023-04-17T22:24:26.390745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf1 = train.filter(pl.col(\"level_group\")=='0-4')\ndf2 = train.filter(pl.col(\"level_group\")=='5-12')\ndf3 = train.filter(pl.col(\"level_group\")=='13-22')\nprint(\"df1: \",df1.shape,\" df2: \",df2.shape,\" df3: \",df3.shape)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:24:26.392920Z","iopub.execute_input":"2023-04-17T22:24:26.393399Z","iopub.status.idle":"2023-04-17T22:24:28.829525Z","shell.execute_reply.started":"2023-04-17T22:24:26.393369Z","shell.execute_reply":"2023-04-17T22:24:28.828707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Delete train to liberate memory\ndel train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:24:28.830403Z","iopub.execute_input":"2023-04-17T22:24:28.830700Z","iopub.status.idle":"2023-04-17T22:24:28.963756Z","shell.execute_reply.started":"2023-04-17T22:24:28.830672Z","shell.execute_reply":"2023-04-17T22:24:28.962815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 = df1.fill_null(0)\ndf1 = df1.fill_nan(0)\ndf2 = df2.fill_null(0)\ndf2 = df2.fill_nan(0)\ndf3 = df3.fill_null(0)\ndf3 = df3.fill_nan(0)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:24:28.965207Z","iopub.execute_input":"2023-04-17T22:24:28.965532Z","iopub.status.idle":"2023-04-17T22:24:31.207813Z","shell.execute_reply.started":"2023-04-17T22:24:28.965502Z","shell.execute_reply":"2023-04-17T22:24:31.206939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nulls_per_column(df1)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:24:31.212149Z","iopub.execute_input":"2023-04-17T22:24:31.214331Z","iopub.status.idle":"2023-04-17T22:24:31.226743Z","shell.execute_reply.started":"2023-04-17T22:24:31.214278Z","shell.execute_reply":"2023-04-17T22:24:31.225718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.tail(5)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:24:31.227943Z","iopub.execute_input":"2023-04-17T22:24:31.228444Z","iopub.status.idle":"2023-04-17T22:24:31.237048Z","shell.execute_reply.started":"2023-04-17T22:24:31.228412Z","shell.execute_reply":"2023-04-17T22:24:31.235910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf1 = feature_engineer_pl(df1, grp='0-4', use_extra=True, feature_suffix='')\nprint('df1 done, shape: ',df1.shape)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:24:31.238111Z","iopub.execute_input":"2023-04-17T22:24:31.238438Z","iopub.status.idle":"2023-04-17T22:24:39.047601Z","shell.execute_reply.started":"2023-04-17T22:24:31.238395Z","shell.execute_reply":"2023-04-17T22:24:39.046713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf2 = feature_engineer_pl(df2, grp='5-12', use_extra=True, feature_suffix='')\nprint('df2 done, shape: ',df2.shape)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:24:39.051514Z","iopub.execute_input":"2023-04-17T22:24:39.052793Z","iopub.status.idle":"2023-04-17T22:24:58.146302Z","shell.execute_reply.started":"2023-04-17T22:24:39.052758Z","shell.execute_reply":"2023-04-17T22:24:58.144311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf3 = feature_engineer_pl(df3, grp='13-22', use_extra=True, feature_suffix='')\nprint('df3 done, shape: ',df3.shape)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:24:58.147837Z","iopub.execute_input":"2023-04-17T22:24:58.148142Z","iopub.status.idle":"2023-04-17T22:25:27.184282Z","shell.execute_reply.started":"2023-04-17T22:24:58.148112Z","shell.execute_reply":"2023-04-17T22:25:27.183339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.isna().sum().sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:25:27.188133Z","iopub.execute_input":"2023-04-17T22:25:27.188791Z","iopub.status.idle":"2023-04-17T22:25:27.325485Z","shell.execute_reply.started":"2023-04-17T22:25:27.188755Z","shell.execute_reply":"2023-04-17T22:25:27.324632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# some cleaning...\nnull1 = df1.isnull().sum().sort_values(ascending=False) / len(df1)\nnull2 = df2.isnull().sum().sort_values(ascending=False) / len(df1)\nnull3 = df3.isnull().sum().sort_values(ascending=False) / len(df1)\n\ndrop1 = list(null1[null1>0.9].index)\ndrop2 = list(null2[null2>0.9].index)\ndrop3 = list(null3[null3>0.9].index)\nprint(\"Drop 1: \",len(drop1), len(drop2), len(drop3))\n\nfor col in df1.columns:\n    if df1[col].nunique()==1:\n        drop1.append(col)\nprint(\"*********df1 DONE*********\")\nfor col in df2.columns:\n    if df2[col].nunique()==1:\n        drop2.append(col)\nprint(\"*********df2 DONE*********\")\nfor col in df3.columns:\n    if df3[col].nunique()==1:\n        drop3.append(col)\nprint(\"*********df3 DONE*********\")","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:25:27.326891Z","iopub.execute_input":"2023-04-17T22:25:27.327461Z","iopub.status.idle":"2023-04-17T22:25:29.806222Z","shell.execute_reply.started":"2023-04-17T22:25:27.327418Z","shell.execute_reply":"2023-04-17T22:25:29.805224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 = df1.set_index('session_id')\ndf2 = df2.set_index('session_id')\ndf3 = df3.set_index('session_id')\n\nFEATURES1 = [c for c in df1.columns if c not in drop1+['level_group']]\nFEATURES2 = [c for c in df2.columns if c not in drop2+['level_group']]\nFEATURES3 = [c for c in df3.columns if c not in drop3+['level_group']]\nprint('We will train with', len(FEATURES1), len(FEATURES2), len(FEATURES3) ,'features')\nALL_USERS = df1.index.unique()\nprint('We will train with', len(ALL_USERS) ,'users info')","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:25:29.813170Z","iopub.execute_input":"2023-04-17T22:25:29.813503Z","iopub.status.idle":"2023-04-17T22:25:30.969807Z","shell.execute_reply.started":"2023-04-17T22:25:29.813472Z","shell.execute_reply":"2023-04-17T22:25:30.967629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1[FEATURES1] = df1[FEATURES1].fillna(0)\ndf2[FEATURES2] = df2[FEATURES2].fillna(0)\ndf3[FEATURES3] = df3[FEATURES3].fillna(0)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:25:30.971170Z","iopub.execute_input":"2023-04-17T22:25:30.971487Z","iopub.status.idle":"2023-04-17T22:25:32.816276Z","shell.execute_reply.started":"2023-04-17T22:25:30.971457Z","shell.execute_reply":"2023-04-17T22:25:32.815382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#listing columns with NaN values\ndf1[FEATURES1].columns[df1[FEATURES1].isna().any()].tolist()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:25:32.817558Z","iopub.execute_input":"2023-04-17T22:25:32.817828Z","iopub.status.idle":"2023-04-17T22:25:32.898586Z","shell.execute_reply.started":"2023-04-17T22:25:32.817801Z","shell.execute_reply":"2023-04-17T22:25:32.897594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2[FEATURES2].columns[df2[FEATURES2].isna().any()].tolist()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:25:32.899907Z","iopub.execute_input":"2023-04-17T22:25:32.900198Z","iopub.status.idle":"2023-04-17T22:25:33.019038Z","shell.execute_reply.started":"2023-04-17T22:25:32.900170Z","shell.execute_reply":"2023-04-17T22:25:33.017928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df3[FEATURES3].columns[df3[FEATURES3].isna().any()].tolist()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:25:33.020370Z","iopub.execute_input":"2023-04-17T22:25:33.020696Z","iopub.status.idle":"2023-04-17T22:25:33.154637Z","shell.execute_reply.started":"2023-04-17T22:25:33.020669Z","shell.execute_reply":"2023-04-17T22:25:33.153599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:25:33.155893Z","iopub.execute_input":"2023-04-17T22:25:33.156164Z","iopub.status.idle":"2023-04-17T22:25:33.200463Z","shell.execute_reply.started":"2023-04-17T22:25:33.156138Z","shell.execute_reply":"2023-04-17T22:25:33.199518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:25:33.201611Z","iopub.execute_input":"2023-04-17T22:25:33.201869Z","iopub.status.idle":"2023-04-17T22:25:33.206586Z","shell.execute_reply.started":"2023-04-17T22:25:33.201843Z","shell.execute_reply":"2023-04-17T22:25:33.205830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# splitting dataframe in a particular size\ndf1_split = df1.sample(frac=0.15,random_state=200)\ndf1.reset_index()\ndf1_split.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:25:33.207824Z","iopub.execute_input":"2023-04-17T22:25:33.208335Z","iopub.status.idle":"2023-04-17T22:25:33.394464Z","shell.execute_reply.started":"2023-04-17T22:25:33.208305Z","shell.execute_reply":"2023-04-17T22:25:33.393618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1_split.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:25:33.395718Z","iopub.execute_input":"2023-04-17T22:25:33.396229Z","iopub.status.idle":"2023-04-17T22:25:33.423096Z","shell.execute_reply.started":"2023-04-17T22:25:33.396197Z","shell.execute_reply":"2023-04-17T22:25:33.422122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# splitting dataframe in a particular size\ndf2_split = df2.sample(frac=0.15,random_state=200)\ndf2.reset_index()\ndf2_split.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:25:33.424580Z","iopub.execute_input":"2023-04-17T22:25:33.424960Z","iopub.status.idle":"2023-04-17T22:25:33.601314Z","shell.execute_reply.started":"2023-04-17T22:25:33.424924Z","shell.execute_reply":"2023-04-17T22:25:33.600501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# splitting dataframe in a particular size\ndf3_split = df3.sample(frac=0.15,random_state=200)\ndf3.reset_index()\ndf3_split.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:25:33.602491Z","iopub.execute_input":"2023-04-17T22:25:33.602981Z","iopub.status.idle":"2023-04-17T22:25:33.774941Z","shell.execute_reply.started":"2023-04-17T22:25:33.602950Z","shell.execute_reply":"2023-04-17T22:25:33.774102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Cleaning Variables to free RAM Memory","metadata":{}},{"cell_type":"code","source":"all_variables = dir()\n  \n# Iterate over the whole list where dir( )\n# is stored.\nfor name in all_variables:\n    \n    # Print the item if it doesn't start with '__'\n    if not name.startswith('__'):\n        myvalue = eval(name)\n        #print(name, \"is\", type(myvalue), \"and is equal to \", myvalue)\n        print(name, \"is\", type(myvalue))","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:25:33.776074Z","iopub.execute_input":"2023-04-17T22:25:33.776564Z","iopub.status.idle":"2023-04-17T22:25:33.784573Z","shell.execute_reply.started":"2023-04-17T22:25:33.776517Z","shell.execute_reply":"2023-04-17T22:25:33.783619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#del clean_missing_values,col\n#del null1,null2,null3,drop1,drop2,drop3,dtypes,event_name_feature,room_lists,text_lists\n#del fqid_lists,name_feature,reduce_memory_usage_pl,\n","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:25:33.785928Z","iopub.execute_input":"2023-04-17T22:25:33.786503Z","iopub.status.idle":"2023-04-17T22:25:33.794515Z","shell.execute_reply.started":"2023-04-17T22:25:33.786470Z","shell.execute_reply":"2023-04-17T22:25:33.793485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:25:33.795676Z","iopub.execute_input":"2023-04-17T22:25:33.796160Z","iopub.status.idle":"2023-04-17T22:25:33.917949Z","shell.execute_reply.started":"2023-04-17T22:25:33.796131Z","shell.execute_reply":"2023-04-17T22:25:33.916969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Group KFold","metadata":{}},{"cell_type":"code","source":"gkf = GroupKFold(n_splits=2)\noof = pd.DataFrame(data=np.zeros((len(ALL_USERS),18)), index=ALL_USERS)\nmodels = {}\nprint(oof.head(3))\nprint(\"Shape oof: \",oof.shape)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:25:33.919380Z","iopub.execute_input":"2023-04-17T22:25:33.920092Z","iopub.status.idle":"2023-04-17T22:25:33.938472Z","shell.execute_reply.started":"2023-04-17T22:25:33.920056Z","shell.execute_reply":"2023-04-17T22:25:33.937465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Randomized Search CV","metadata":{}},{"cell_type":"code","source":"param_grid = {\n    'n_estimators': [25,40,60,80,90,100],\n    'max_features': ['sqrt', 'log2', None],\n    'max_depth': [3,6,9],\n    'max_leaf_nodes': [3,6,9],\n    'min_samples_split' : [2, 5],\n    'bootstrap' : [True,False]\n}","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:25:33.939807Z","iopub.execute_input":"2023-04-17T22:25:33.940079Z","iopub.status.idle":"2023-04-17T22:25:33.944965Z","shell.execute_reply.started":"2023-04-17T22:25:33.940052Z","shell.execute_reply":"2023-04-17T22:25:33.943915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Randomized Search CV\ngrid_search = RandomizedSearchCV(estimator = RandomForestClassifier(), param_distributions = param_grid, cv = 2, verbose=2, n_jobs = 3) ","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:25:33.946398Z","iopub.execute_input":"2023-04-17T22:25:33.946824Z","iopub.status.idle":"2023-04-17T22:25:33.954794Z","shell.execute_reply.started":"2023-04-17T22:25:33.946783Z","shell.execute_reply":"2023-04-17T22:25:33.953893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets['session'] = targets.session_id.apply(lambda x: int(x.split('_')[0]))\ntargets['q'] = targets.session_id.apply(lambda x: int(x.split('_')[-1][1:]))","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:25:33.956066Z","iopub.execute_input":"2023-04-17T22:25:33.956326Z","iopub.status.idle":"2023-04-17T22:25:34.669212Z","shell.execute_reply.started":"2023-04-17T22:25:33.956300Z","shell.execute_reply":"2023-04-17T22:25:34.668340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nrf_best_params = []\n\n# ITERATE THRU QUESTIONS 1 THRU 18\nfor t in range(1,19):\n#for t in range(1,2):           \n    if t<=3: \n        grp = '0-4'\n        df = df1_split\n        FEATURES = FEATURES1\n\n    elif t<=13: \n        grp = '5-12'\n        df = df2_split\n        FEATURES = FEATURES2\n\n    elif t<=22: \n        grp = '13-22'\n        df = df3_split\n        FEATURES = FEATURES3\n            \n    # TRAIN DATA\n    train_x = df\n    #train_x = train_x.loc[train_x.level_group == grp]\n    train_users = train_x.index.values\n    train_y = targets.loc[targets.q==t].set_index('session').loc[train_users]\n             \n    # TRAIN MODEL\n    grid_search.fit(train_x[FEATURES].astype('float32'), train_y['correct'])\n    print(f'*** Grid Search Model: {grp}_{t}')\n    print()\n    #print(grid_search.best_estimator_)\n    rf_best_params.append(grid_search.best_params_)\n        \nprint()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:25:34.670579Z","iopub.execute_input":"2023-04-17T22:25:34.670851Z","iopub.status.idle":"2023-04-17T22:51:00.353341Z","shell.execute_reply.started":"2023-04-17T22:25:34.670825Z","shell.execute_reply":"2023-04-17T22:51:00.351909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(rf_best_params)):    \n    print(\"Model: \",i+1,\" \",rf_best_params[i])","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:51:00.355795Z","iopub.execute_input":"2023-04-17T22:51:00.356162Z","iopub.status.idle":"2023-04-17T22:51:00.363368Z","shell.execute_reply.started":"2023-04-17T22:51:00.356115Z","shell.execute_reply":"2023-04-17T22:51:00.362201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df1_split\ndel df2_split\ndel df3_split\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:51:00.365023Z","iopub.execute_input":"2023-04-17T22:51:00.365345Z","iopub.status.idle":"2023-04-17T22:51:00.597486Z","shell.execute_reply.started":"2023-04-17T22:51:00.365315Z","shell.execute_reply":"2023-04-17T22:51:00.596644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Random Forest","metadata":{}},{"cell_type":"markdown","source":"## Questions 1 to 3","metadata":{}},{"cell_type":"code","source":"%%time\nwarnings.filterwarnings(\"ignore\")\n\n# COMPUTE CV SCORE WITH 2 GROUP K FOLD\nfor i, (train_index, test_index) in enumerate(gkf.split(X=df1[FEATURES1], groups=df1[FEATURES1].index)):\n    \n    print('-'*25)\n    print('--- Fold',i+1)\n    print('-'*25)\n    print('Questions: ')\n\n    for t in range(1,4):\n    #for t in range(1,2):\n\n        print(t,' - ',end='')\n        bp = rf_best_params[t-1]\n\n        # USE THIS TRAIN DATA WITH THESE QUESTIONS\n        if t<=3: \n            grp = '0-4'\n            df = df1\n            FEATURES = FEATURES1\n\n        elif t<=13: \n            grp = '5-12'\n            df = df2\n            FEATURES = FEATURES2\n\n        elif t<=22: \n            grp = '13-22'\n            df = df3\n            FEATURES = FEATURES3\n\n        bp = rf_best_params[t-1]\n\n        # TRAIN DATA\n        train_x = df[FEATURES].iloc[train_index]\n        train_users = train_x.index.values\n        train_y = targets.loc[targets.q==t].set_index('session').loc[train_users]\n\n        # VALID DATA\n        valid_x = df[FEATURES].iloc[test_index]\n        valid_users = valid_x.index.values\n        valid_y = targets.loc[targets.q==t].set_index('session').loc[valid_users]\n\n        # TRAIN MODEL\n        clf = RandomForestClassifier(n_estimators=bp['n_estimators'],max_depth=bp['max_depth'],min_samples_split=bp['min_samples_split'],\n                                         max_features=bp['max_features'],max_leaf_nodes=bp['max_leaf_nodes'],bootstrap=bp['bootstrap']) \n        clf.fit(train_x[FEATURES].astype('float32'), train_y['correct'])\n\n        # SAVE MODEL, PREDICT VALID OOF\n        models[f'{grp}_{t}'] = clf\n        oof.loc[valid_users, t-1] = clf.predict_proba(valid_x[FEATURES].astype('float32'))[:,1]\n        print(f'Model Random Forest saved for question {t}')\n\n","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:51:00.598673Z","iopub.execute_input":"2023-04-17T22:51:00.599643Z","iopub.status.idle":"2023-04-17T22:54:04.755146Z","shell.execute_reply.started":"2023-04-17T22:51:00.599606Z","shell.execute_reply":"2023-04-17T22:54:04.753946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Questions 4 to 13","metadata":{}},{"cell_type":"code","source":"%%time\nwarnings.filterwarnings(\"ignore\")\n\n# COMPUTE CV SCORE WITH 2 GROUP K FOLD\nfor i, (train_index, test_index) in enumerate(gkf.split(X=df2[FEATURES2], groups=df2[FEATURES2].index)):\n    \n    print('-'*25)\n    print('--- Fold',i+1)\n    print('-'*25)\n    print('Questions: ')\n\n    for t in range(4,14):\n    #for t in range(1,19):\n\n        print(t,', ',end='')\n        bp = rf_best_params[t-1]\n\n        # USE THIS TRAIN DATA WITH THESE QUESTIONS\n        if t<=3: \n            grp = '0-4'\n            df = df1\n            FEATURES = FEATURES1\n\n        elif t<=13: \n            grp = '5-12'\n            df = df2\n            FEATURES = FEATURES2\n\n        elif t<=22: \n            grp = '13-22'\n            df = df3\n            FEATURES = FEATURES3\n\n        bp = rf_best_params[t-1]\n\n        # TRAIN DATA\n        train_x = df[FEATURES].iloc[train_index]\n        train_users = train_x.index.values\n        train_y = targets.loc[targets.q==t].set_index('session').loc[train_users]\n\n        # VALID DATA\n        valid_x = df[FEATURES].iloc[test_index]\n        valid_users = valid_x.index.values\n        valid_y = targets.loc[targets.q==t].set_index('session').loc[valid_users]\n\n        # TRAIN MODEL\n        clf = RandomForestClassifier(n_estimators=bp['n_estimators'],max_depth=bp['max_depth'],min_samples_split=bp['min_samples_split'],\n                                         max_features=bp['max_features'],max_leaf_nodes=bp['max_leaf_nodes'],bootstrap=bp['bootstrap']) \n        clf.fit(train_x[FEATURES].astype('float32'), train_y['correct'])\n\n        # SAVE MODEL, PREDICT VALID OOF\n        models[f'{grp}_{t}'] = clf\n        oof.loc[valid_users, t-1] = clf.predict_proba(valid_x[FEATURES].astype('float32'))[:,1]\n        print(f'Model Random Forest saved for question {t}')\n","metadata":{"execution":{"iopub.status.busy":"2023-04-17T22:54:04.756422Z","iopub.execute_input":"2023-04-17T22:54:04.756745Z","iopub.status.idle":"2023-04-17T23:13:06.173502Z","shell.execute_reply.started":"2023-04-17T22:54:04.756715Z","shell.execute_reply":"2023-04-17T23:13:06.172293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models","metadata":{"execution":{"iopub.status.busy":"2023-04-17T23:13:06.174821Z","iopub.execute_input":"2023-04-17T23:13:06.175233Z","iopub.status.idle":"2023-04-17T23:13:06.196029Z","shell.execute_reply.started":"2023-04-17T23:13:06.175202Z","shell.execute_reply":"2023-04-17T23:13:06.195040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Questions 14 to 18","metadata":{}},{"cell_type":"code","source":"%%time\nwarnings.filterwarnings(\"ignore\")\n\n# COMPUTE CV SCORE WITH 2 GROUP K FOLD\nfor i, (train_index, test_index) in enumerate(gkf.split(X=df3[FEATURES3], groups=df3[FEATURES3].index)):\n    \n    print('-'*25)\n    print('--- Fold',i+1)\n    print('-'*25)\n    print('Questions: ')\n\n    for t in range(14,19):\n    #for t in range(1,19):\n\n        print(t,', ',end='')\n        bp = rf_best_params[t-1]\n\n        # USE THIS TRAIN DATA WITH THESE QUESTIONS\n        if t<=3: \n            grp = '0-4'\n            df = df1\n            FEATURES = FEATURES1\n\n        elif t<=13: \n            grp = '5-12'\n            df = df2\n            FEATURES = FEATURES2\n\n        elif t<=22: \n            grp = '13-22'\n            df = df3\n            FEATURES = FEATURES3\n\n        bp = rf_best_params[t-1]\n\n        # TRAIN DATA\n        train_x = df[FEATURES].iloc[train_index]\n        train_users = train_x.index.values\n        train_y = targets.loc[targets.q==t].set_index('session').loc[train_users]\n\n        # VALID DATA\n        valid_x = df[FEATURES].iloc[test_index]\n        valid_users = valid_x.index.values\n        valid_y = targets.loc[targets.q==t].set_index('session').loc[valid_users]\n\n        # TRAIN MODEL\n        clf = RandomForestClassifier(n_estimators=bp['n_estimators'],max_depth=bp['max_depth'],min_samples_split=bp['min_samples_split'],\n                                         max_features=bp['max_features'],max_leaf_nodes=bp['max_leaf_nodes'],bootstrap=bp['bootstrap']) \n        clf.fit(train_x[FEATURES].astype('float32'), train_y['correct'])\n\n        # SAVE MODEL, PREDICT VALID OOF\n        models[f'{grp}_{t}'] = clf\n        oof.loc[valid_users, t-1] = clf.predict_proba(valid_x[FEATURES].astype('float32'))[:,1]\n        print(f'Model Random Forest saved for question {t}')\n","metadata":{"execution":{"iopub.status.busy":"2023-04-17T23:13:06.197417Z","iopub.execute_input":"2023-04-17T23:13:06.197831Z","iopub.status.idle":"2023-04-17T23:23:25.613529Z","shell.execute_reply.started":"2023-04-17T23:13:06.197799Z","shell.execute_reply":"2023-04-17T23:23:25.612604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df, df1, df2, df3\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T23:23:25.614853Z","iopub.execute_input":"2023-04-17T23:23:25.615379Z","iopub.status.idle":"2023-04-17T23:23:25.749132Z","shell.execute_reply.started":"2023-04-17T23:23:25.615345Z","shell.execute_reply":"2023-04-17T23:23:25.748301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(oof.head(5))\nprint(\"Shape oof: \",oof.shape)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T23:23:25.750303Z","iopub.execute_input":"2023-04-17T23:23:25.750794Z","iopub.status.idle":"2023-04-17T23:23:25.766391Z","shell.execute_reply.started":"2023-04-17T23:23:25.750763Z","shell.execute_reply":"2023-04-17T23:23:25.765621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntrue = oof.copy()\nfor k in range(18):\n    tmp = targets.loc[targets.q == k+1].set_index('session').loc[ALL_USERS]\n    true[k] = tmp.correct.values","metadata":{"execution":{"iopub.status.busy":"2023-04-17T23:23:25.767737Z","iopub.execute_input":"2023-04-17T23:23:25.768231Z","iopub.status.idle":"2023-04-17T23:23:25.882861Z","shell.execute_reply.started":"2023-04-17T23:23:25.768201Z","shell.execute_reply":"2023-04-17T23:23:25.882005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(true.head(5))\nprint(\"Shape true: \",true.shape)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T23:23:25.884130Z","iopub.execute_input":"2023-04-17T23:23:25.884684Z","iopub.status.idle":"2023-04-17T23:23:25.893352Z","shell.execute_reply.started":"2023-04-17T23:23:25.884650Z","shell.execute_reply":"2023-04-17T23:23:25.892609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Best Threshold","metadata":{}},{"cell_type":"code","source":"scores = []; thresholds = []\nbest_score = 0; best_threshold = 0\n\nfor threshold in np.arange(0.4,0.81,0.01):\n    print(f'{threshold:.02f}, ',end='')\n    preds = (oof.values.reshape((-1))>threshold).astype('int')\n    m = f1_score(true.values.reshape((-1)), preds, average='macro')   \n    scores.append(m)\n    thresholds.append(threshold)\n    if m>best_score:\n        best_score = m\n        best_threshold = threshold","metadata":{"execution":{"iopub.status.busy":"2023-04-17T23:23:25.894652Z","iopub.execute_input":"2023-04-17T23:23:25.895142Z","iopub.status.idle":"2023-04-17T23:23:32.065015Z","shell.execute_reply.started":"2023-04-17T23:23:25.895113Z","shell.execute_reply":"2023-04-17T23:23:32.063832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Best score: \",best_score)\nprint(\"Best threshold: \",best_threshold)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T23:23:32.066768Z","iopub.execute_input":"2023-04-17T23:23:32.067168Z","iopub.status.idle":"2023-04-17T23:23:32.072352Z","shell.execute_reply.started":"2023-04-17T23:23:32.067135Z","shell.execute_reply":"2023-04-17T23:23:32.071307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\ngcolor = \"#3d9979\"\nplt.figure(figsize=(20,5))\nplt.plot(thresholds,scores,'-o',color=gcolor)\nplt.scatter([best_threshold], [best_score], color='green', s=300, alpha=1)\nplt.xlabel('Threshold',size=14)\nplt.ylabel('Validation F1 Score',size=14)\nplt.title(f'Threshold vs. F1_Score with Best F1_Score = {best_score:.3f} at Best Threshold = {best_threshold:.3}',size=18)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T23:23:32.073627Z","iopub.execute_input":"2023-04-17T23:23:32.074590Z","iopub.status.idle":"2023-04-17T23:23:32.354195Z","shell.execute_reply.started":"2023-04-17T23:23:32.074513Z","shell.execute_reply":"2023-04-17T23:23:32.353247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nprint('When using optimal threshold...')\nfor k in range(18):\n        \n    m = f1_score(true[k].values, (oof[k].values>best_threshold).astype('int'), average='macro')\n    print(f'Q{k+1}: F1 =',m)\n    \nm = f1_score(true.values.reshape((-1)), (oof.values.reshape((-1))>best_threshold).astype('int'), average='macro')\nprint()\nprint('==> Overall F1 =',m)\nprint()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T23:23:32.355252Z","iopub.execute_input":"2023-04-17T23:23:32.355638Z","iopub.status.idle":"2023-04-17T23:23:32.663079Z","shell.execute_reply.started":"2023-04-17T23:23:32.355603Z","shell.execute_reply":"2023-04-17T23:23:32.661980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# performance evaluatio metrics\nprint()\nprint(\"==> Overall Matrix = \")\nprint(classification_report(true.values.reshape((-1)), (oof.values.reshape((-1))>best_threshold).astype('int')))\nprint()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T23:23:32.664415Z","iopub.execute_input":"2023-04-17T23:23:32.664845Z","iopub.status.idle":"2023-04-17T23:23:33.338735Z","shell.execute_reply.started":"2023-04-17T23:23:32.664813Z","shell.execute_reply":"2023-04-17T23:23:33.337650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission Jo Wilder","metadata":{}},{"cell_type":"code","source":"import jo_wilder\nenv = jo_wilder.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T23:23:33.339946Z","iopub.execute_input":"2023-04-17T23:23:33.340341Z","iopub.status.idle":"2023-04-17T23:23:33.365186Z","shell.execute_reply.started":"2023-04-17T23:23:33.340310Z","shell.execute_reply":"2023-04-17T23:23:33.364140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"limits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\nfor test, sample_submission in iter_test:\n    sample_submission['question'] = [int(label.split('_')[1][1:]) for label in sample_submission['session_id']]\n    grp = test.level_group.values[0]\n    a,b = limits[grp]\n    \n    # ------------------- level 0-4 ---------------------------------\n    if a == 1:\n        FEATURES = FEATURES1\n        test = (pl.from_pandas(test)\n                  .drop([\"fullscreen\", \"hq\", \"music\"])\n                  .with_columns(columns))\n        test = feature_engineer_pl(test, grp, use_extra=True, feature_suffix='')\n        test = test[FEATURES]\n        test = test.fillna(0)\n            \n\n    # ------------------- level 5-12 ---------------------------------\n    elif a == 4:\n        FEATURES = FEATURES2\n        test = (pl.from_pandas(test)\n                  .drop([\"fullscreen\", \"hq\", \"music\"])\n                  .with_columns(columns))\n        test = feature_engineer_pl(test, grp, use_extra=True, feature_suffix='')\n        test = test[FEATURES]\n        test = test.fillna(0)\n\n    # ------------------- level 13-22 ---------------------------------    \n    elif a == 14:\n        FEATURES = FEATURES3\n        test = (pl.from_pandas(test)\n                  .drop([\"fullscreen\", \"hq\", \"music\"])\n                  .with_columns(columns))\n        test = feature_engineer_pl(test, grp, use_extra=True, feature_suffix='')\n        test = test[FEATURES]\n        test = test.fillna(0)\n    \n\n    # INFER TEST DATA\n    \n    for t in range(a,b):\n    #for t in range(1,2):\n        clf = models[f'{grp}_{t}']\n        \n        p = clf.predict_proba(test.astype('float32'))[:,1]\n        mask = sample_submission.question == t    \n        sample_submission.loc[mask, 'correct'] = (p > best_threshold).astype('int') \n    env.predict(sample_submission[['session_id', 'correct']])","metadata":{"execution":{"iopub.status.busy":"2023-04-17T23:23:33.366782Z","iopub.execute_input":"2023-04-17T23:23:33.367121Z","iopub.status.idle":"2023-04-17T23:23:34.098211Z","shell.execute_reply.started":"2023-04-17T23:23:33.367089Z","shell.execute_reply":"2023-04-17T23:23:34.097300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('submission.csv')\nprint( df.shape )\ndf.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T23:23:34.099494Z","iopub.execute_input":"2023-04-17T23:23:34.100045Z","iopub.status.idle":"2023-04-17T23:23:34.113176Z","shell.execute_reply.started":"2023-04-17T23:23:34.100010Z","shell.execute_reply":"2023-04-17T23:23:34.112277Z"},"trusted":true},"execution_count":null,"outputs":[]}]}