{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"The code below is an improved version of the datatable `LinearModel` baseline posted [here](https://www.kaggle.com/code/kononenko/datatable-linearmodel-0-676lb-in-6-seconds). \n\nIt scores `0.687` on the both public and private test data, and is ranked 7th of 2051 on [the efficiency LB](https://www.kaggle.com/code/philculliton/student-performance-efficiency-leaderboard/).\n\n## Reading the training data","metadata":{}},{"cell_type":"code","source":"%%time\nimport sys\nsys.path.append('/kaggle/input/pip-install-datatable/packages/')\n\nfrom datatable import dt, f, g\nfrom datatable.models import LinearModel\n\nfrom sklearn.metrics import f1_score\nimport numpy as np\nimport pandas as pd\n\nDEBUG = True\ndt.options.nthreads = 2\n\n# Read the train data\nDT_train_full = dt.fread(\"/kaggle/input/predict-student-performance-from-game-play/train.csv\")\n\n# Read labels, parse session ids and question numbers\nDT_labels = dt.Frame(\"/kaggle/input/predict-student-performance-from-game-play/train_labels.csv\")\nDT_labels = DT_labels[:, {\n                          \"session_id\" : f.session_id[:17].as_type(dt.int64),\n                          \"q\" : f.session_id[19:].as_type(dt.int8),\n                          \"correct\" : f.correct\n                     }]","metadata":{"execution":{"iopub.status.busy":"2023-06-30T04:41:41.797113Z","iopub.execute_input":"2023-06-30T04:41:41.797877Z","iopub.status.idle":"2023-06-30T04:43:02.813202Z","shell.execute_reply.started":"2023-06-30T04:41:41.797841Z","shell.execute_reply":"2023-06-30T04:43:02.812236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Misc functions and categorical data","metadata":{}},{"cell_type":"code","source":"level_groups= {\"0-4\"   : (1, 4),   # questions 1-3\n               \"5-12\"  : (4, 14),  # questions 4-13\n               \"13-22\" : (14, 19)} # questions 14-18\n\n# Get the level group for the q-th question\ndef get_level_group(q):\n    for level_group, (q_from, q_to) in level_groups.items():\n        if q >= q_from and q < q_to:\n            return level_group\n\n# Prepare training data for q-th question\ndef get_train_data(q):\n    # Preparing training data for q-th question\n    level_group_q = get_level_group(q)\n    DT_train_q = DTs_train[level_group_q]\n\n    # Preparing labels\n    DT_labels_q = DT_labels[f.q == q, :]\n    DT_labels_q.key = [\"session_id\"]\n    DT_labels_q = DT_train_q[:, g.correct, dt.join(DT_labels_q)]\n\n    # Remove `session_id` from training data\n    DT_train_q = DT_train_q[:, 1:] \n    return DT_train_q, DT_labels_q\n\n\n# For each of the categorical columns/level groups, \n# unique values were extracted and sorted by their importance \n# in the descending order.\nCAT_VALS = {\n    \"name\" : ('basic', 'close', 'open', 'prev', 'next'),\n    \n    \"event_name\" : {\n        \"0-4\"   : ('object_click', 'notebook_click', 'cutscene_click', 'person_click', 'notification_click', 'observation_click', 'navigate_click', 'object_hover', 'checkpoint', 'map_click', 'map_hover'),\n        \"5-12\"  : ('object_click', 'navigate_click', 'map_click', 'cutscene_click', 'person_click', 'checkpoint', 'map_hover', 'notification_click', 'notebook_click', 'object_hover', 'observation_click'),\n        \"13-22\" : ('navigate_click', 'person_click', 'map_click', 'notebook_click', 'cutscene_click', 'notification_click', 'object_hover', 'object_click', 'map_hover', 'observation_click', 'checkpoint')\n    },\n\n    \"room_fqid\" : {\n        \"0-4\"   : ('tunic.historicalsociety.collection', 'tunic.kohlcenter.halloffame', 'tunic.historicalsociety.stacks', 'tunic.historicalsociety.closet', 'tunic.capitol_0.hall', 'tunic.historicalsociety.entry', 'tunic.historicalsociety.basement'),\n        \"5-12\"  : ('tunic.historicalsociety.stacks', 'tunic.historicalsociety.frontdesk', 'tunic.library.microfiche', 'tunic.kohlcenter.halloffame', 'tunic.capitol_1.hall', 'tunic.library.frontdesk', 'tunic.historicalsociety.collection', 'tunic.capitol_0.hall', 'tunic.humanecology.frontdesk', 'tunic.drycleaner.frontdesk', 'tunic.historicalsociety.basement', 'tunic.historicalsociety.closet_dirty', 'tunic.historicalsociety.entry'),\n        \"13-22\" : ('tunic.flaghouse.entry', 'tunic.historicalsociety.closet_dirty', 'tunic.wildlife.center', 'tunic.library.frontdesk', 'tunic.historicalsociety.frontdesk', 'tunic.capitol_2.hall', 'tunic.historicalsociety.stacks', 'tunic.historicalsociety.collection_flag', 'tunic.historicalsociety.entry', 'tunic.drycleaner.frontdesk', 'tunic.humanecology.frontdesk', 'tunic.historicalsociety.cage', 'tunic.library.microfiche', 'tunic.capitol_1.hall', 'tunic.historicalsociety.basement', 'tunic.kohlcenter.halloffame', 'tunic.historicalsociety.collection')\n    }, \n    \n    \"fqid\" : {\n        \"0-4\"   : ('plaque', 'outtolunch', 'gramps', 'report', 'doorblock', 'directory', 'togrampa', 'tunic', 'plaque.face.date', 'retirement_letter', 'chap1_finale', 'boss', 'tomap', 'block_0', 'groupconvo', 'block_tomap2', 'teddy', 'tocloset', 'notebook', 'intro', 'tocollection', 'chap1_finale_c', 'janitor', 'tostacks', 'tunic.historicalsociety', 'cs', 'tunic.hub.slip', 'wells', 'block_tocollection', 'tunic.kohlcenter', 'tobasement', 'photo', 'tunic.capitol_0', 'toentry', 'block_tomap1'),\n        \"5-12\"  : ('businesscards.card_1.next', 'journals.pic_1.next', 'archivist', 'businesscards.card_bingo.next', 'reader', 'logbook', 'reader.paper1.next', 'businesscards.card_bingo.bingo', 'tobasement', 'block', 'tunic.kohlcenter', 'door_block_talk', 'businesscards.card_0.next', 'journals.pic_2.next', 'gramps', 'businesscards', 'reader.paper2.prev', 'what_happened', 'tostacks', 'tunic.humanecology', 'trigger_scarf', 'tunic.historicalsociety', 'tocollection', 'logbook.page.bingo', 'tomap', 'journals', 'journals.pic_2.bingo', 'tunic.capitol_0', 'tunic.drycleaner', 'journals.pic_0.next', 'block_badge', 'tunic.capitol_1', 'photo', 'reader.paper0.next', 'worker', 'tunic', 'block_badge_2', 'toentry', 'tofrontdesk', 'tunic.hub.slip', 'tohallway', 'block_magnify', 'tocloset_dirty', 'reader.paper0.prev', 'journals.hub.topics', 'directory', 'chap2_finale_c', 'reader.paper2.bingo', 'boss', 'door_block_clean', 'trigger_coffee', 'reader.paper1.prev', 'tomicrofiche', 'block_0', 'outtolunch', 'reader.paper2.next', 'tunic.library', 'wellsbadge', 'janitor', 'plaque', 'block_1', 'magnify'),   \n        \"13-22\" : ('archivist_glasses', 'flag_girl', 'directory.closeup.archivist', 'tracks', 'confrontation', 'journals_flag.pic_0.next', 'tocloset_dirty', 'reader_flag.paper1.next', 'journals_flag.pic_1.next', 'expert', 'ch3start', 'tunic.flaghouse', 'toentry', 'gramps', 'savedteddy', 'tunic.historicalsociety', 'tunic.wildlife', 'key', 'boss', 'journals_flag', 'tostacks', 'journals', 'journals_flag.pic_0.bingo', 'reader_flag.paper0.next', 'tunic.humanecology', 'journals_flag.pic_2.bingo', 'journals.hub.topics', 'glasses', 'colorbook', 'journals.pic_2.bingo', 'tohallway', 'groupconvo_flag', 'businesscards.card_0.next', 'photo', 'reader.paper1.next', 'journals_flag.pic_0_old.next', 'directory', 'tofrontdesk', 'reader.paper2.next', 'journals_flag.hub.topics', 'tocollection', 'coffee', 'teddy', 'tobasement', 'tunic.library', 'journals_flag.pic_2.next', 'reader.paper0.next', 'reader_flag.paper1.prev', 'journals_flag.pic_2_old.next', 'tunic.kohlcenter', 'tocage', 'businesscards.card_1.next', 'tunic.capitol_2', 'reader_flag.paper2.next', 'seescratches', 'businesscards', 'remove_cup', 'worker', 'tunic', 'tocollectionflag', 'journals_flag.pic_1_old.next', 'unlockdoor', 'journals_flag.pic_1.bingo', 'reader_flag.paper0.prev', 'businesscards.card_bingo.next', 'tomicrofiche', 'need_glasses', 'fox', 'reader_flag.paper2.prev', 'chap4_finale_c', 'journals_flag.hub.topics_old', 'tunic.capitol_1', 'journals.pic_1.next', 'tunic.drycleaner', 'outtolunch', 'logbook', 'reader', 'block_nelson', 'lockeddoor', 'plaque', 'tracks.hub.deer', 'tunic.hub.slip', 'reader_flag.paper2.bingo', 'reader.paper1.prev', 'journals.pic_0.next', 'reader_flag', 'tomap', 'wells', 'crane_ranger', 'journals.pic_2.next', 'reader.paper0.prev', 'reader.paper2.prev')\n    }, \n\n    \"text_fqid\" : {\n        \"0-4\"   : ('tunic.historicalsociety.stacks.outtolunch', 'tunic.historicalsociety.collection.gramps.lost', 'tunic.historicalsociety.closet.doorblock', 'tunic.historicalsociety.closet.retirement_letter.hub', 'tunic.historicalsociety.entry.boss.talktogramps', 'tunic.historicalsociety.closet.intro', 'tunic.historicalsociety.collection.cs', 'tunic.kohlcenter.halloffame.block_0', 'tunic.historicalsociety.entry.groupconvo', 'tunic.capitol_0.hall.chap1_finale_c', 'tunic.historicalsociety.closet.teddy.intro_0_cs_5', 'tunic.historicalsociety.collection.gramps.look_0', 'tunic.historicalsociety.basement.janitor', 'tunic.historicalsociety.closet.gramps.intro_0_cs_0', 'tunic.historicalsociety.entry.block_tomap2', 'tunic.historicalsociety.entry.gramps.hub', 'tunic.historicalsociety.collection.gramps.found', 'tunic.kohlcenter.halloffame.togrampa', 'tunic.historicalsociety.closet.teddy.intro_0_cs_0', 'tunic.historicalsociety.closet.photo', 'tunic.kohlcenter.halloffame.plaque.face.date', 'tunic.historicalsociety.entry.block_tomap1', 'tunic.historicalsociety.collection.tunic', 'tunic.historicalsociety.collection.tunic.slip', 'tunic.historicalsociety.entry.wells.talktogramps', 'tunic.historicalsociety.closet.notebook', 'tunic.historicalsociety.entry.block_tocollection'),\n        \"5-12\"  : ('tunic.historicalsociety.frontdesk.archivist.foundtheodora', 'tunic.historicalsociety.stacks.block', 'tunic.historicalsociety.closet_dirty.door_block_talk', 'tunic.historicalsociety.closet_dirty.gramps.archivist', 'tunic.drycleaner.frontdesk.worker.takealook', 'tunic.historicalsociety.closet_dirty.gramps.helpclean', 'tunic.historicalsociety.closet_dirty.gramps.nothing', 'tunic.historicalsociety.closet_dirty.what_happened', 'tunic.historicalsociety.stacks.journals.pic_2.bingo', 'tunic.capitol_0.hall.boss.talktogramps', 'tunic.historicalsociety.frontdesk.archivist.newspaper_recap', 'tunic.library.frontdesk.worker.hello_short', 'tunic.library.frontdesk.worker.droppedbadge', 'tunic.drycleaner.frontdesk.worker.done', 'tunic.drycleaner.frontdesk.logbook.page.bingo', 'tunic.library.frontdesk.block_badge', 'tunic.humanecology.frontdesk.worker.intro', 'tunic.historicalsociety.frontdesk.archivist.have_glass_recap', 'tunic.historicalsociety.frontdesk.archivist.need_glass_0', 'tunic.drycleaner.frontdesk.block_0', 'tunic.capitol_1.hall.boss.haveyougotit', 'tunic.drycleaner.frontdesk.worker.hub', 'tunic.historicalsociety.frontdesk.magnify', 'tunic.library.frontdesk.worker.wells', 'tunic.historicalsociety.closet_dirty.photo', 'tunic.library.frontdesk.worker.hello', 'tunic.historicalsociety.closet_dirty.gramps.news', 'tunic.historicalsociety.frontdesk.archivist.have_glass', 'tunic.historicalsociety.stacks.outtolunch', 'tunic.drycleaner.frontdesk.block_1', 'tunic.historicalsociety.closet_dirty.trigger_scarf', 'tunic.historicalsociety.closet_dirty.trigger_coffee', 'tunic.historicalsociety.frontdesk.archivist.newspaper', 'tunic.historicalsociety.closet_dirty.door_block_clean', 'tunic.library.frontdesk.block_badge_2', 'tunic.humanecology.frontdesk.block_0', 'tunic.drycleaner.frontdesk.worker.done2', 'tunic.historicalsociety.frontdesk.block_magnify', 'tunic.humanecology.frontdesk.businesscards.card_bingo.bingo', 'tunic.historicalsociety.frontdesk.archivist.hello', 'tunic.library.microfiche.reader.paper2.bingo', 'tunic.library.frontdesk.wellsbadge.hub', 'tunic.humanecology.frontdesk.block_1', 'tunic.humanecology.frontdesk.worker.badger', 'tunic.historicalsociety.frontdesk.archivist.need_glass_1', 'tunic.library.frontdesk.worker.wells_recap', 'tunic.library.frontdesk.worker.preflag', 'tunic.library.microfiche.block_0', 'tunic.historicalsociety.basement.janitor', 'tunic.capitol_1.hall.chap2_finale_c'),\n        \"13-22\" : ('tunic.historicalsociety.frontdesk.archivist_glasses.confrontation_recap', 'tunic.historicalsociety.entry.directory.closeup.archivist', 'tunic.flaghouse.entry.flag_girl.symbol_recap', 'tunic.wildlife.center.expert.recap', 'tunic.library.frontdesk.worker.flag_recap', 'tunic.historicalsociety.stacks.journals_flag.pic_1.bingo', 'tunic.capitol_2.hall.boss.haveyougotit', 'tunic.historicalsociety.basement.savedteddy', 'tunic.historicalsociety.basement.ch3start', 'tunic.historicalsociety.collection_flag.gramps.recap', 'tunic.historicalsociety.closet_dirty.photo', 'tunic.wildlife.center.expert.removed_cup', 'tunic.historicalsociety.entry.boss.flag_recap', 'tunic.historicalsociety.cage.glasses.beforeteddy', 'tunic.historicalsociety.stacks.journals_flag.pic_0.bingo', 'tunic.flaghouse.entry.flag_girl.hello_recap', 'tunic.historicalsociety.stacks.journals_flag.pic_2.bingo', 'tunic.capitol_1.hall.boss.writeitup', 'tunic.historicalsociety.basement.seescratches', 'tunic.library.frontdesk.worker.nelson_recap', 'tunic.historicalsociety.frontdesk.archivist_glasses.confrontation', 'tunic.historicalsociety.cage.teddy.trapped', 'tunic.wildlife.center.wells.nodeer', 'tunic.historicalsociety.entry.groupconvo_flag', 'tunic.humanecology.frontdesk.worker.badger', 'tunic.historicalsociety.entry.boss.flag', 'tunic.historicalsociety.cage.glasses.afterteddy', 'tunic.wildlife.center.fox.concern', 'tunic.historicalsociety.entry.wells.flag_recap', 'tunic.historicalsociety.entry.wells.flag', 'tunic.drycleaner.frontdesk.worker.done2', 'tunic.historicalsociety.basement.gramps.whatdo', 'tunic.historicalsociety.collection_flag.gramps.flag', 'tunic.wildlife.center.wells.animals2', 'tunic.library.frontdesk.worker.flag', 'tunic.wildlife.center.wells.nodeer_recap', 'tunic.wildlife.center.tracks.hub.deer', 'tunic.historicalsociety.cage.need_glasses', 'tunic.library.frontdesk.worker.preflag', 'tunic.wildlife.center.remove_cup', 'tunic.wildlife.center.crane_ranger.crane', 'tunic.library.microfiche.reader_flag.paper2.bingo', 'tunic.library.frontdesk.worker.nelson', 'tunic.historicalsociety.stacks.journals.pic_2.bingo', 'tunic.library.frontdesk.block_nelson', 'tunic.historicalsociety.stacks.outtolunch', 'tunic.wildlife.center.wells.animals', 'tunic.flaghouse.entry.flag_girl.hello', 'tunic.wildlife.center.coffee', 'tunic.historicalsociety.basement.gramps.seeyalater', 'tunic.historicalsociety.cage.confrontation', 'tunic.flaghouse.entry.flag_girl.symbol', 'tunic.historicalsociety.frontdesk.key', 'tunic.historicalsociety.cage.lockeddoor', 'tunic.flaghouse.entry.colorbook', 'tunic.capitol_2.hall.chap4_finale_c', 'tunic.historicalsociety.cage.unlockdoor')\n    },\n\n    \"text\" : {\n        \"0-4\"  : ('Better check back later.', \"Hey Jo, let's take a look at the shirt!\", \"I feel like I'm forgetting something.\", '\\\\u00f0\\\\u0178\\\\u02dc\\\\u00b4', 'Your grampa is waiting for you in the collection room.', 'Um... what did you want me to do again?', 'Gramps said to look for clues. Better look around.', 'Head over to the Basketball Center.', \"Hmm. Button's still not working.\", 'I should go talk to Gramps!', 'Just talking to Teddy.', 'Gramps is a great historian!', 'Gramps is the best historian ever!', 'Will do, Boss.', 'Now where did I put my notebook?', \"This can't be right!\", 'Have a look at the artifact!', 'Gramps is in trouble for losing papers?', 'Hopefully you can rustle up some clues!', 'Whatcha doing over there, Jo?', \"I'm not so sure that this is a basketball jersey.\", 'Hopefully you can find some clues!', \"I'll record this in my notebook.\", 'Come on, Jo!', \"Meet me back in my office and we'll get started!\", 'Found it!', \"That's it!\", \"I'll be at the Capitol. Let me know if you find anything!\", \"It's true, they do keep going missing lately.\", 'I love these photos of me and Teddy!', \"Why don't you head to the Basketball Center and rustle up some clues?\", \"Let's get started. The Wisconsin Wonders exhibit opens tomorrow!\", 'We need to talk about that missing paperwork.', \"Why don't you go catch up with your grampa?\", 'Hey!', 'Wait, you mean Wells is wrong?!', 'This button never works!', 'I knew it!', 'Grab your notebook and come upstairs!', 'I gotta run to my meeting!', 'Who wants to investigate the shirt artifact?', 'I should see what Grampa is up to!', 'Ooh, I like clues!', \"The slip is from 1916 but the team didn't start until 1974!\", \"I get to go to Gramps's meeting!\", 'Could be. But we need evidence!', 'Find anything?', 'Ugh. Meetings are so boring.', 'Besides, I already figured out the shirt.', \"Why don't you go play with your grampa?\", 'Meetings are BORING!', 'That settles it.', 'Go ahead, take a peek at the shirt!', 'I should see what Gramps is up to!', \"It's a women's basketball jersey!\", 'Our shirt is too old to be a basketball jersey!', 'I need to get to the Capitol and tell Gramps!', 'Sure thing, Jo. Grab your notebook and come upstairs!', 'Sure!', \"Leopold, why don't you help me set up in the Capitol?\", 'Hot Dog! I knew it!', 'Yes! This cool old slip from 1916.', 'This looks like a clue!', 'Do I have to?', 'Yes! This old slip from 1916.', 'Can I take a closer look?', \"It's already all done!\", 'Just this old slip from 1916.', \"Wow, that's so cool, Gramps!\", 'Wells, finish up your report.', \"Look at that! It's the bee's knees!\", 'Can I come, Gramps?', 'What a fascinating artifact!', 'So? History is boring!', \"Not Leopold here. He's been losing papers lately.\", \"I'll hurry back and then we can go exploring!\", 'Well, Leopold here is always losing papers...', 'Did you do all of them?', 'No... because history is boring!', 'See?', 'A boring old shirt.', 'Plus, my teacher said I could help you out for extra credit!', \"Hmmm. Shouldn't you be doing your homework?\", 'Teddy and I were gonna go climb that huge tree out back!', 'Then do it for me!', \"Well, I did SOME of those. I just couldn't find them!\", 'Hooray, a boring old shirt.', \"No way, Gramps. You're the best!\", 'Can we hurry up, Gramps?', 'I suppose historians are boring, too?', 'Hang tight, Teddy.', 'Ha. Told you so!', 'Your teacher said you could help me for extra credit.', 'Your teacher said you missed 7 assignments in a row!', 'See you later, Teddy.', \"Hmmm. Don't forget about your homework.\", \"Well, that's good enough for me.\"),\n        \"5-12\" : ('The archivist said I should look in the stacks.', 'Better check back later.', 'So much cleaning to do...', \"Poor Gramps! I should make sure he's okay.\", 'Take a look!', \"Can't believe I lost my reading glasses.\", 'Nice seeing you, Jolie!', 'What should I do first?', 'What are you waiting for? The Stacks are right outside the door.', 'Youmans was a suffragist!', 'Head upstairs and talk to the archivist. He might be able to help!', 'This button never works!', 'You better get to the capitol!', \"It's a match!\", 'Theodora wearing the shirt!', \"Did you drop something, Dear? There's a card on the floor.\", 'Can you help me tidy up?', \"I'm sure you'll find Theodora in there somewhere!\", \"I think he's in trouble!\", \"Jolie! I was hoping you'd stop by. Any news on the shirt artifact?\", 'I found it!', 'But he never goes anywhere without his scarf!', \"Guess it couldn't hurt to let you take a look.\", 'Did you drop something, Dear?', \"Hmm. Button's still not working.\", \"I haven't quite figured it out just yet...\", \"Ah, that's better!\", 'I should check that logbook to see who owned this slip...', 'I knew I could count on you, Jo!', 'Maybe he just got scared and ran off.', \"Oh, that's from Bean Town.\", 'I bet the archivist could use this!', \"And he messed up Gramps's office, too!\", 'Easy, Jo.', 'He needs our help!', 'I figured out that you kidnapped him!', 'I hope you find your badger, kid.', 'I knew you could do it, Jo!', \"It'll be okay, Jo. We'll find Teddy!\", 'Jo, meet me back at my office.', 'Leopold, can you run back to the museum?', 'Mrs. M, I think Wells kidnapped Teddy.', 'Nice work on the shirt, Jolie!', 'Now can I tell you what happened to Teddy?', \"Sorry I'm late.\", 'Sounds good, Boss.', \"Wells! Where's Teddy? Is he okay?\", \"Why don't you prove your case?\", \"Well, get on it. I'm counting on you and your gramps to figure this out!\", 'I should find out if she can help me!', 'Theodora Youmans must be the owner!', \"Weren't you going to check out our microfiche?\", 'Maybe I can help!', 'Hey, this is Youmans!', 'I should ask the librarian why Wells was here.', 'I used to have a magnifying glass around here\\\\u00e2\\\\u20ac\\\\u00a6', \"What's a textile expert?\", 'What happened here?!', 'I should stay and look for clues!', '\\\\Taxidermy: the art of preparing, stuffing, and mounting the skins of animals.\\\\', 'Wow!', 'Yep.', \"Here's the log book.\", \"And look! She's wearing the shirt!\", 'The libarian said I could find some information on Youmans in here...', 'Welcome back, Jolie. Did you figure out the shirt?', \"Nope, that's from Bean Town. I only drink Holdgers!\", 'I should go to the Capitol and tell everyone!', \"I'm afraid my papers have gone missing in this mess.\", 'Where should I go again?', 'You can talk to a textile expert there.', 'Head over to the university.', 'Ha! Good one.', 'Your gramps is awesome! Always full of stories.', 'This place was around in 1916! I can start there!', \"I've got a stack of business cards from my favorite cleaners.\", 'Yes! I was wondering-', 'They study clothes and fabric.', \"Here's a call number to find more info in the Stacks.\", 'Do you know anything about this slip?', \"Sorry, can't help you.\", \"I don't need that right now.\", 'Wow! What is all this stuff?', \"Sorry, I'm in a hurry.\", 'Do you know who Theodora Youmans is?', \"But I hear the museum's got one on the loose!\", \"Well, I can't show our log books to just anybody.\", 'You could try the archivist. Maybe he can help you find Wells!', \"Why don't you take a look?\", 'Great! Thanks for the help!', \"And I'll figure out the shirt, too.\", \"You'll have to get started without me.\", \"You haven't seen any badgers around here, have you?\", 'Please?', 'Wells? I knew it!', 'Right outside the door.', \"Teddy's scarf! Somebody must've taken him!\", \"I wonder if there's a clue in those business cards...\", 'Not sure. Here, let me look it up.', 'Nope. But Youmans and other suffragists worked hard to change that.', 'She helped get votes for women!', \"Why didn't you say so?\", 'Badgers? No.', 'Wells! What was he doing here? I should ask the librarian.', 'Theodora Youmans? Of course!', 'Theodora Youmans? Is that who owned the shirt?', \"It's for Grampa Leo. He's a historian!\", \"I'm also looking for Theodora Youmans. Have you heard of her?\", 'He was looking for a taxidermist.', 'Do you have any info on Theodora Youmans?', 'Did you have a question?', \"Wait a sec. Women couldn't vote?!\", 'I love these photos of me and Teddy.', 'Looks like a dry cleaning receipt.', 'She led marches and helped women get the right to vote!', 'Wait a minute!', 'What was Wells doing here?', 'Do you know what this slip is?', \"I'm afraid not.\", \"Check out our microfiche. It's right through that door.\", 'How can I help you?', 'Have you seen a badger around here?', 'An old shirt? Try the university.', 'Hi, Mrs. M.', \"Why don't you go upstairs and see the archivist?\", \"It's our Norwegian Craft exhibit!\", 'Where did you get that coffee?', 'Thanks to them, Wisconsin was the first state to approve votes for women!', 'I found it on an old shirt.', 'Please let me know if you do.', \"Unless you're too busy horsing around.\", 'AND I know who took Teddy!', \"And where's your grampa?\", 'Did you figure out the shirt?', 'I had some cleaning up to do in my office.', 'One step at a time, Jo.', 'Sorry for the delay, Boss.', 'Who is Teddy?', \"It's such a nice fall day.\", 'I need to find Wells right away! Do you know where he is?', 'Is this your coffee, Gramps?', 'Where are the Stacks?', \"What's a taxidermist?\", \"Who could've done this?\", 'Well? What are you still doing here?', \"He's our expert record keeper.\", 'Hello there!', 'Can I give you the tour?', 'I need your help!', 'What are you still doing here,  Jolie?', 'I should help Gramps clean.', \"Maybe there's a clue in this mess!\", 'Go find your grampa and get to work!', 'I ran into Wells there this morning.', 'Yeah. Thanks anyway.', \"Hmmm... not sure. Why don't you try the library?\", 'Youmans was a suffragist here in Wisconsin.', 'You could ask the archivist. He knows everybody!', 'Oh, hello there!', \"I'm Leopold's grandkid!\", 'Oh no... Teddy!', 'Slow down, Jo.', 'Okay. Thanks anyway.', 'Run along to the university.', 'Please? This is really important.', 'And you are?', \"Sorry, I'm too busy for kids right now.\", 'Are you okay?', \"Ha! You're funny.\", 'Try not to panic, Jo.', 'Thanks. Did you figure out the shirt?', \"You look like you're on a mission.\", \"I don't know!\", \"Okay. I'll find Teddy!\", 'Oh my!', 'A little horse!', 'Two missions, actually!', 'Nice decorations.', 'I need to find Wells right away!! Do you know where he is?', 'Ooh, thanks!', 'Ugh. Fine.', 'Knew what?', '*cough cough*', '*COUGH COUGH COUGH*', 'Can you help me? I need to find Wells!', \"He's always trying to get you in trouble, and he doesn't like animals!\", \"Don't worry, Gramps. I'll find Teddy!\", 'Hold your horses, Jo.', 'Did you have a question or not?', \"You're still here? I'm trying to work!\", '*grumble grumble*', \"You're right, Gramps. Let's investigate!\", \"Yup, that's him!\", 'Can you help me-', 'Leo... you mean Leopold?', \"Calm down, kid. I haven't seen him.\", 'Thanks.', 'But what if Wells kidnapped Teddy?', \"Please let me know if you do. It's important!\", 'Then we need evidence.', 'Can you help-', \"It must've been Wells.\", 'Thanks for the help!', 'Hi! *cough*', 'Yikes... this could take a while.', 'Yes!', 'Ha! What do you call a pony with a sore throat?', \"Oh, I'm fine! Just a little hoarse.\", 'Hi! How can I help you?', \"He's wrong about old shirts and his name rhymes with \\\\smells\\\\...\", \"I don't have time for kids.\", 'Ooh, nice decorations!', 'Can you help me find Wells?', 'I need to find the owner of this slip.', 'Can you help me? I need to find the owner of this slip.', 'I have an idea.', 'Could be. But we need evidence.', 'Now I Just need to find all the cleaners from way back in 1916.', 'Now if only I could read this thing.', \"I can't calm down. This is important!\", 'BUT WELLS STOLE TEDDY!', 'Who are you?', 'Now I just need to find all the cleaners from wayyyy back in 1916.', 'AND he stole Teddy!', 'Here I am!', 'Wells sabotaged Gramps!', 'Um, are you okay?', \"I haven't seen him.\", \"Fine. Let's investigate!\", 'Now if only I could read this thing. Blasted tiny letters...', 'Huh?', 'I need to find Wells!!!', 'I got that one from my Gramps!', 'Oh no!', 'I got here and the whole place was ransacked!', 'What the-', 'I got here and the whole place was a mess!'),\n        \"13-22\": ('The Stacks are right outside the door. Go find some photos!', \"If I were you, I'd go to the library and do some digging.\", 'Thanks for your help, kid!', 'There are some old newspapers loaded up in the microfiche.', 'I love these photos of me and Teddy.', \"Check out the archives. They've got tons of old photos!\", \"The archivist must've taken Teddy!\", 'Those are the same glasses!', 'There should be some info about that symbol in my book.', 'Gramps must be up in the collection room.', 'I wonder whose glasses these are.', 'Teddy, here I come!', 'Ugh... I think that lynx is looking at me funny.', 'Head back to the museum. Your gramps is waiting for you.', 'And you, Frank-', 'Ugh! Those cups are all over the place.', 'You did it! Thanks, kid.', \"I need to get her free. She won't hold still!\", 'Oh yeah, cranes eat insects!', \"I'm telling you, Boss. Taxidermy is the way to go!\", \"We'll find him, Jo.\", \"Let's go find him!\", \"Wait! Can't I do it?\", \"It's locked!\", \"I'm investigating this flag.\", 'Check out the next artifact!', \"Come on, let's get out of here!\", 'Actually, I went to school with somebody who LOVES old flags.', 'Your flag must have been part of a national movement!', 'Alright, Jolie. Back to work.', 'What is it?', \"Go check the microfiche. Maybe you'll find something!\", 'Nice seeing you, Jolie!', 'Sure! Give it a try.', 'Something to do with ecology and Wisconsin.', \"Why don't you go talk to the boss?\", 'Hey!', 'Wells got in trouble for littering at the Wildlife Center.', \"He's been eating my lunch every day this week!\", '\\\\u00f0\\\\u0178\\\\u00a6\\\\u2014', \"I haven't quite figured it out just yet...\", \"YOU'RE the new history detective everybody's talking about?\", \"Why isn't the button working?\", \"I'm not sure.\", 'Want to look for more clues?', 'What are you doing down here?', 'You could try the archives.', \"It's OK, girl! Look, I found you a cricket!\", \"That hoofprint doesn't match the flag!\", 'I found the key!', \"She's right outside.\", \"I've got Wells's ID!\", 'Oh no! What happened to that crane?', \"Well, get on it. I'm counting on you to figure this out!\", \"There's a diagram of animal tracks over there.\", 'Thanks for coming, Boss.', \"Jolie! I was hoping you'd stop by. Any news on the flag artifact?\", 'I found the flag! Governor Nelson used it on the first Earth Day!', 'Whoever lost these glasses probably took Teddy!', 'Better check back later.', 'I should check out that pair of glasses.', 'Hey, look at those scratches!', \"I'll go look at everyone's pictures!\", 'Nonsense. I want live animals at the exhibit, not stuffed ones.', \"We'll need a key card.\", 'It could be an early design for the Wisconsin state flag!', 'I need to learn more about this flag!', 'The archivist had him locked up!', \"Ah, Jolie! I'm glad you're here.\", 'Luckily there are tons of insects around here...', \"Maybe she'll let me take off the cup!\", \"Here's your scarf back!\", 'We need to calm her down, Teddy.', 'The symbol on the flag looks sort of like a deer hoof.', \"Come on, kid. You're slowing me down.\", 'Huh?', \"Are you going home now? Tomorrow's the big day!\", 'He got a park named after him? Cool!', 'Thanks!', \"But I hear the museum's got one on the loose!\", \"I'm putting you in charge of the flag case.\", 'Hmm. You could try the Aldo Leopold Wildlife Center.', \"It's lucky we found her.\", \"Yep. I'm a vexillophile!\", 'The exhibit opens tomorrow.', \"What's going on here?\", \"Looks like it's not a deer hoof.\", 'Wells, meet Teddy.', 'What now, kid?', 'Look at all those activists!', \"Hey! That's Governor Nelson in front of our flag!\", \"It's an ecology flag!\", 'I should ask the librarian where to go next.', 'Does it look like a deer hoof?', 'I captured a badger in our museum!', 'This is perfect for the exhibit.', \"Hang on. I'll get you out of there!\", \"I'm sure they'll be able to help.\", 'Badgers? No.', 'Make sure to get some old photos for the exhibit, like last time!', \"Jolie- keep your badger under control, or he'll have to go.\", 'Jo!', \"Come on, kid. Let's go.\", '\\\\u00f0\\\\u0178\\\\u02dc\\\\u0160', \"You've got a million flags here!\", 'Head over to the Wildlife Center!', 'Jolie! Where have you been?', 'Aha! Good catch, Jo.', 'Wait! What?! Really?', 'Wow! You figured it out!', 'Teddy!!!', 'Good idea. Thanks!', 'Her beak is stuck in a coffee cup.', 'She should be able to help you out.', \"You're becoming quite the detective, Jo.\", 'Your grampa is waiting for you in the collection room.', 'I told you!', 'Go take a look!', 'Any ideas?', 'Hey, nice dog! What breed is he?', \"I'm investigating this symbol.\", 'Well... it looks hand-stitched.', 'Though the archivist might be too busy to help...', '\\\\u00f0\\\\u0178\\\\u00a7\\\\u02dc', \"You haven't seen any badgers around here, have you?\", 'Ugh...', 'My friend is a flag expert.', '\\\\u00f0\\\\u0178\\\\u008d\\\\u00a9', 'Wells, you already have a job to do.', 'What kind of photos do you need?', 'Ugh. Fine.', \"I'll ride with you!\", 'Now I just need some old photos, like last time.', 'Yeah. Thanks anyway.', \"Here's a call number for the Stacks. Go find some photos.\", \"Actually, he's a badger.\", \"But cranes can't do yoga, Teddy!\", 'Do you know where I can find a deer expert?', 'The kidnapper probably took Teddy on the elevator!', \"Oh, trust me. He'll make time.\", 'Wait a minute...', 'Oh! There was a staff directory in the entryway!', \"Yes! It's the key for Teddy's cage!\", 'Not sure. Do I look like a deer expert to you?', 'Yoga does sound nice.', 'Good catch!', 'Poor foxes!', 'Not sure.', 'I think I can help with your animal problem.', 'People sure drink a lot of coffee around here.', 'Hmm... those stripes remind me of the American flag.', \"Teddy's helping too.\", 'How can I find out whose glasses these are?', \"Oh, cool! I've never seen a badger in real life.\", 'I had one, but Teddy chewed it up.', 'Can Teddy and I help?', 'I think I might be able to help you.', \"It's for the flag display!\", \"We're just looking for photos for the flag display.\", 'Can you take a look?', '\\\\u00f0\\\\u0178\\\\u02dc\\\\u00ad', \"So? What'd you find out?\", \"He says he'd be willing to help out.\", \"It's such a nice fall day.\", 'We just have to keep our eyes open!', \"Teddy! I'm glad to see you.\", 'Guess so!', 'Can I ride with you?', 'Okay. Thanks anyway.', 'It just means flag expert. How can I help?', \"We'll find Teddy.\", 'Notice any clues about this flag?', 'The boss is gonna love it!!!', \"Hey, I've seen that symbol before! Check it out!\", 'I should go to the Capitol and tell Mrs. M!', \"Why don't you go talk to her? I'll let her know you're coming.\", \"Actually, we're just here for some photos.\", \"Cranes don't eat donuts!\", 'And this place is dirty, and itchy, and-', 'What are you doing here?', \"I can't believe this.\", 'Ugh. I have to head over there and check out the animals.', 'Welcome back, Dear! How can I help you?', 'YOU?!', 'Oh no... they got sick from polluted water?', 'What is it, Teddy?', 'Go on, tell the boss what you found!', 'Do you know what this flag was used for?', 'Can I help you with anything?', 'Okay. Thanks!', 'Gadzooks! Poor critter.', 'I love it!', \"And look! She's wearing the shirt!\", 'Hey, this is Youmans!', \"Ha! I don't need your help.\", 'Do you really think that symbol is a deer hoof?', 'Hey, Wells...', 'Careful. That beak is sharp!', 'It has something to do with ecology.', 'Yes!!!', 'I should go to the Capitol and tell everyone!', 'FINE. That possum better not scratch my leather seats...', \"I'm a historian, not a zookeeper!\", 'Besides, you just ate my last snack.', 'And we still need to figure out that flag!', \"Teddy! I'm sure glad to see you.\", 'The boss is gonna love it!', \"Fine. Then I guess you don't want a real, live badger for the exhibit.\", 'Poor badger.', \"He's. A. Badger.\", \"He's a badger!\", \"What's a vexillophile?\", \"Don't worry, he won't! (And he's a badger, by the way.)\", \"We're looking for some photos.\", \"Hmm. Let's see...\", \"I'll be in the collection room. Come find me when you're ready to check out the artifact.\", 'Good luck!', 'Got one!', \"Gah. I can't believe this.\", \"Don't worry, Teddy won't eat your lunch anymore!\", '\\\\u00f0\\\\u0178\\\\u2122\\\\u201e', 'We still need to figure out that flag. Do you know anyone who could help?', 'I have to head over there and check out the animals.', \"Actually, badgers aren't rodents-\", 'And how did that badger get free?', 'Whatever.', \"You again! Don't let him hurt me!\", 'Great. Just great. Could this day get any worse?!', \"Oh no. If I don't impress the boss soon,  I'm gonna get fired!\", \"Fine, fine. Let's see...\", \"I think it's a flag! Pretty interesting, huh?\", \"Just please, don't let your badger eat them!\", \"No he hasn't!\", 'A vexy-wha?', 'Ooh... \\\\Ecology flag, by Ron Cobb.\\\\', '\\\\Ecology flag, by Ron Cobb.\\\\', \"Let's go help Gramps!\", \"You can't just steal Jolie's pet. Don't you know badgers are protected animals?\", 'Are you sure? I know where you can find a real, live badger for the exhibit!', 'GRRRRRRR', 'A real, live ferret!', \"It's okay, Gramps. I'll go by myself.\", 'Come on, Teddy.', 'What?!', \"Okay. I'll try.\", \"I need to take the artifact upstairs. Why don't you investigate those scratch marks?\", \"Yes, he has. I've seen him eating homework and important papers, too.\", \"Of course you do. You've got a rodent following you around.\", 'He has??', \"I've seen him eating homework and important papers, too.\", \"I can't go with you. I need to take the artifact upstairs.\", \"You can't just steal Jolie's pet.\", 'And my homework?!?!', \"I don't have time for this.\", 'See?!', \"Come on, Teddy. Let's go help Gramps!\", 'GAH! And what is THAT doing out of its cage?!', \"I don't have time for this, Gramps.\", 'Wait- me?', \"Did you steal Gramps's paperwork too?!\", 'You stole Teddy! How could you?!', \"Let's go find Gramps!\", 'What should we do next?', \"I'm here to rescue my friend!\", \"Jo! I can't go with you. I need to take the artifact upstairs.\", \"No thanks. I don't need help from kids.\", \"Yes!!! I'm saved!\", \"Let's follow those scratch marks!\", \"That thing's a monster!\", \"You're right, Jo!\", 'Wha?!', \"Great Scott, you're right!\", 'Besides, he looks friendly to me.', \"He's still missing!\", \"It's really cool, Gramps. But I'm worried about Teddy.\", '\\\\u00e2\\\\u009d\\\\u00a4\\\\u00ef\\\\u00b8\\\\u008f', '\\\\u00f0\\\\u0178\\\\u02dc\\\\u009d', 'Teddy! Did you really eat his lunch?', '\\\\u00f0\\\\u0178\\\\u02dc\\\\u0090', 'Teddy is still missing!', 'YEAH!', \"I think it's a flag! Pretty spiffy, eh?\")\n    },\n}\n\nfor cat, vals in CAT_VALS.items():\n    print(f\"{cat}\")\n    if type(vals) is dict:\n        for level_group, val in vals.items():\n            print(f\"  {level_group:5}: {len(val)}\")\n    else:\n        print(f\"  {len(vals)}\")","metadata":{"execution":{"iopub.status.busy":"2023-06-30T04:43:42.758944Z","iopub.execute_input":"2023-06-30T04:43:42.759353Z","iopub.status.idle":"2023-06-30T04:43:42.832747Z","shell.execute_reply.started":"2023-06-30T04:43:42.759320Z","shell.execute_reply":"2023-06-30T04:43:42.831905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"%%time\n\n# Main feature engineering function\ndef get_features(DT, level_group): \n    features = {\n        \"count_log\"               : dt.math.log1p(dt.count()),\n        \"elapsed_time_log_max\"    : dt.math.log1p(f.elapsed_time.max()),\n        \"level_log_mean\"          : dt.math.log1p(dt.mean(f.level)),\n        \"screen_coor_x_range_log\" : dt.math.log1p(f.screen_coor_x.max() - f.screen_coor_x.min()),\n        \"screen_coor_y_range_log\" : dt.math.log1p(f.screen_coor_y.max() - f.screen_coor_y.min()),\n    }        \n    \n    CAT_FEATURES = {\n        \"fqid\"       : 30,\n        \"event_name\" : 10,\n        \"room_fqid\"  : 10,\n        \"name\"       : 0,\n        \"text_fqid\"  : 0,\n        \"text\"       : 0,\n    }\n    \n    for col, nvals in CAT_FEATURES.items():\n        features[col + \"_nunique\"] = f[col].nunique()\n\n        if nvals > 0:\n            vals = CAT_VALS[col]\n            if type(vals) is dict:\n                vals = vals[level_group]\n\n            for val in vals[:nvals]:\n                feature_name = col + \"_\" + val + \"_log_count\"\n                features[feature_name] = dt.math.log1p(dt.sum(f[col] == val))\n            \n    return DT[:, features, dt.by(\"session_id\")]\n\n\n# Prepare training data for each level_group\nDTs_train = {}\nfor level_group in level_groups:\n    DTs_train[level_group] = get_features(\n        DT_train_full[f.level_group==level_group, :-1], \n        level_group\n    )\n    \n    if DEBUG:\n        print(f\"{level_group:6}: {DTs_train[level_group].ncols - 1} features\")\n        print(*DTs_train[level_group].names[1:], sep=\", \")\n        print()\n        assert(DTs_train[level_group].nrows == DT_labels.nrows//18)\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-30T04:44:02.939728Z","iopub.execute_input":"2023-06-30T04:44:02.943029Z","iopub.status.idle":"2023-06-30T04:45:42.252659Z","shell.execute_reply.started":"2023-06-30T04:44:02.942965Z","shell.execute_reply":"2023-06-30T04:45:42.251675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training one model per question","metadata":{}},{"cell_type":"code","source":"%%time\n\n# Model parameters\neta0 = 0.0005 # Learning rate\nnepochs = 18  # Number of training epochs \nmodels = []\nstats = []\n\n# Fit one LinearModel per question.\nif DEBUG:\n    print(\"Fitting...\")\n    DT_y_true = dt.Frame()\n    DT_y_pred = dt.Frame()\n\nfor q in range(1, 19):        \n    DT_train_q, DT_labels_q = get_train_data(q)\n\n    if DEBUG:\n        print(f\"  Question {q:2}: {DT_train_q.nrows} sessions\")\n   \n    stats.append({\"mean\" : DT_train_q.mean(), \"sd\" : DT_train_q.sd()})\n    stats[q - 1][\"sd\"].replace(0.0, 1.0)    \n    f_std = (f[:] - stats[q - 1][\"mean\"]) / stats[q - 1][\"sd\"]\n    DT_train_q = DT_train_q[:, f_std]\n\n    model = LinearModel(eta0=eta0, nepochs=nepochs)\n    model.fit(DT_train_q, DT_labels_q)\n    models.append(model)\n    \n    if DEBUG:\n        DT_y_pred_q = model.predict(DT_train_q[:, f[:].fillna(value=0)])\n        DT_y_pred.rbind(DT_y_pred_q)    \n        DT_y_true.rbind(DT_labels_q)\n        \nif DEBUG:\n    print(\"Done\")\n    \n    \n# Determine the best threshold\nif DEBUG:\n    f1_max = 0\n    threshold_max = None\n    y_true = DT_y_true.to_list()[0]    \n    threshold_range = np.arange(0.61, 0.68, 0.01)\n\n    print()\n    for threshold in threshold_range:\n        y_pred = DT_y_pred[:, f[\"True\"] > threshold].to_list()[0]\n        f1 = f1_score(y_true, y_pred, average=\"macro\")\n        print(f\"Threshold: {threshold:4} -> F1: {f1}\")\n        if f1 > f1_max:\n            f1_max = f1\n            threshold_max = threshold\n\n    if len(threshold_range) > 1:\n        print(f\"Best threshold: {threshold_max} -> F1: {f1_max}\")\n        print()","metadata":{"execution":{"iopub.status.busy":"2023-06-30T04:53:51.593727Z","iopub.execute_input":"2023-06-30T04:53:51.594118Z","iopub.status.idle":"2023-06-30T04:54:56.030870Z","shell.execute_reply.started":"2023-06-30T04:53:51.594088Z","shell.execute_reply":"2023-06-30T04:54:56.029376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference","metadata":{}},{"cell_type":"code","source":"%%time\nimport jo_wilder_310\n\njo_wilder_310.make_env.__called__ = False\nenv = jo_wilder_310.make_env()\niter_test = env.iter_test()\n\nfor (PD_test, PD_submission) in iter_test:\n    # Create datatable frame from pandas\n    DT_test = dt.Frame(PD_test)\n    \n    # Get the level group information. We are supposed to predict\n    # for one level group at a time.\n    level_group = DT_test[0, \"level_group\"]\n    (q_from, q_to) = level_groups[level_group]\n    \n    # Do feature engineering for the test data\n    DT_test = get_features(DT_test, level_group)\n    \n    # Remove `session_id` from the test data\n    DT_test = DT_test[:, 1:]\n\n    for q in range(q_from, q_to):\n        # Predict for the standardized test data\n        f_std = (f[:] - stats[q - 1][\"mean\"]) / stats[q - 1][\"sd\"]\n        DT_test_std = DT_test[:, f_std][:, f[:].fillna(value=0)]\n        DT_p = models[q - 1].predict(DT_test_std)\n                \n        # Convert probabilities to labels by using F1-optimal threshold\n        DT_p = DT_p[:, dt.int8(f[\"True\"] > threshold_max)]\n        \n        # Update predictions for the q-th question\n        mask_q = PD_submission.session_id.str.contains(f'q{q}')\n        PD_submission.loc[mask_q, 'correct'] = DT_p.to_list()\n    \n    if DEBUG:\n        print(f\"Questions {q_from}-{q_to - 1}\")\n        print(PD_submission)\n        print()\n        \n    env.predict(PD_submission)","metadata":{"execution":{"iopub.status.busy":"2023-06-30T04:55:18.138130Z","iopub.execute_input":"2023-06-30T04:55:18.138532Z","iopub.status.idle":"2023-06-30T04:55:18.439810Z","shell.execute_reply.started":"2023-06-30T04:55:18.138501Z","shell.execute_reply":"2023-06-30T04:55:18.436256Z"},"trusted":true},"execution_count":null,"outputs":[]}]}