{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Using Polars\n\nI learnt `polars` from last OTTO competition, which helped our team to do fast feature engineering when we faced `cudf`'s GPU memory errors.\n\nIn this competition, although the data size is not as large as OTTO's, it is still convenient to use `polars` with cpu for easy train-inference adaption. Moreover, from my own experiment, the `polars` is faster than `pandas` in Kaggle's notebook even there is only 2-core CPU avalaible, which restricts the full strength of `polars` parallelism.\n\nAnd last, the current Kaggle environment (2023-02-17) has `polars` supported! You can save about 40s from installing it.","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport sys\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport pickle\nimport xgboost as xgb\nfrom catboost import CatBoostClassifier, Pool\nimport lightgbm as lgb","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-08T16:53:15.504447Z","iopub.execute_input":"2023-06-08T16:53:15.505115Z","iopub.status.idle":"2023-06-08T16:53:16.801358Z","shell.execute_reply.started":"2023-06-08T16:53:15.505023Z","shell.execute_reply":"2023-06-08T16:53:16.800253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"catboost_base = \"/kaggle/input/psp-catboost-sort-by-time-only\"\nxgbboost_base = \"/kaggle/input/xgboost-sort-by-time\"\nlightgbm_base = \"/kaggle/input/psp-lgb-20230608\"","metadata":{"execution":{"iopub.status.busy":"2023-06-08T16:53:16.802648Z","iopub.execute_input":"2023-06-08T16:53:16.803019Z","iopub.status.idle":"2023-06-08T16:53:16.808342Z","shell.execute_reply.started":"2023-06-08T16:53:16.802986Z","shell.execute_reply":"2023-06-08T16:53:16.807187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f_read = open('/kaggle/input/psp-catboost-sort-by-time-only/unused_data_values.pkl', 'rb')\nunused_data_values = pickle.load(f_read)\nf_read.close()","metadata":{"execution":{"iopub.status.busy":"2023-06-08T16:53:16.809743Z","iopub.execute_input":"2023-06-08T16:53:16.810208Z","iopub.status.idle":"2023-06-08T16:53:16.823210Z","shell.execute_reply.started":"2023-06-08T16:53:16.810174Z","shell.execute_reply":"2023-06-08T16:53:16.822254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trn_qs = [\n    (\"0-4\", 1, 4),\n    (\"5-12\", 4, 14),\n    (\"13-22\", 14, 19),\n]\n\nmodel_dict1 = {}\nmodel_dict2 = {}\nmodel_dict3 = {}\n\nfeature_dict1 = {}\nfeature_dict2 = {}\nfeature_dict3 = {}\n\nfor grp_idx, (grp, a, b) in enumerate(trn_qs):\n    model_dict1[grp] = []\n    model_dict2[grp] = []\n    model_dict3[grp] = []\n    for fold in range(5):\n        model_dict1[grp].append(\n            CatBoostClassifier().load_model(\n                f\"{catboost_base}/fold{fold}_grp_{grp}_final.cbm\"\n            )\n        )\n        model_dict2[grp].append(\n            xgb.Booster(\n                model_file=f\"{xgbboost_base}/fold{fold}_grp_{grp}_final.bin\"\n            )\n        )\n        model_dict3[grp].append(\n            lgb.Booster(\n                model_file=f\"{lightgbm_base}/fold{fold}_grp_{grp}_final.bin\"\n            )\n        )\n    _imp1 = pd.read_csv(f\"{catboost_base}/importance_0_grp_{grp}_final.csv\")\n    _imp2 = pd.read_csv(f\"{xgbboost_base}/importance_0_grp_{grp}_final.csv\")\n    _imp3 = pd.read_csv(f\"{lightgbm_base}/importance_0_grp_{grp}_final.csv\")\n\n    print(grp, len(_imp1))\n    feature_dict1[grp] = list(_imp1.feature.values)\n    feature_dict2[grp] = list(_imp2.feature.values)\n    feature_dict3[grp] = list(_imp3.feature.values)","metadata":{"execution":{"iopub.status.busy":"2023-06-08T16:53:16.825764Z","iopub.execute_input":"2023-06-08T16:53:16.826400Z","iopub.status.idle":"2023-06-08T16:53:17.984170Z","shell.execute_reply.started":"2023-06-08T16:53:16.826365Z","shell.execute_reply":"2023-06-08T16:53:17.982514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CATS = ['event_name', 'name','fqid', 'room_fqid', 'text_fqid']\n\nevent_name_feature = ['checkpoint',\n 'cutscene_click',\n 'map_click',\n 'map_hover',\n 'navigate_click',\n 'notebook_click',\n 'notification_click',\n 'object_click',\n 'object_hover',\n 'observation_click',\n 'person_click']\n\nfqid_feature = ['archivist',\n 'archivist_glasses',\n 'block',\n 'block_0',\n 'block_1',\n 'block_badge',\n 'block_badge_2',\n 'block_magnify',\n 'block_nelson',\n 'block_tocollection',\n 'block_tomap1',\n 'block_tomap2',\n 'boss',\n 'businesscards',\n 'businesscards.card_0.next',\n 'businesscards.card_1.next',\n 'businesscards.card_bingo.bingo',\n 'businesscards.card_bingo.next',\n 'ch3start',\n 'chap1_finale',\n 'chap1_finale_c',\n 'chap2_finale',\n 'chap2_finale_c',\n 'chap4_finale_c',\n 'coffee',\n 'colorbook',\n 'confrontation',\n 'crane_ranger',\n 'cs',\n 'directory',\n 'directory.closeup.archivist',\n 'door_block_clean',\n 'door_block_talk',\n 'doorblock',\n 'expert',\n 'flag_girl',\n 'fox',\n 'glasses',\n 'gramps',\n 'groupconvo',\n 'groupconvo_flag',\n 'intro',\n 'janitor',\n 'journals',\n 'journals.hub.topics',\n 'journals.pic_0.next',\n 'journals.pic_1.next',\n 'journals.pic_2.bingo',\n 'journals.pic_2.next',\n 'journals_flag',\n 'journals_flag.hub.topics',\n 'journals_flag.hub.topics_old',\n 'journals_flag.pic_0.bingo',\n 'journals_flag.pic_0.next',\n 'journals_flag.pic_0_old.next',\n 'journals_flag.pic_1.bingo',\n 'journals_flag.pic_1.next',\n 'journals_flag.pic_1_old.next',\n 'journals_flag.pic_2.bingo',\n 'journals_flag.pic_2.next',\n 'journals_flag.pic_2_old.next',\n 'key',\n 'lockeddoor',\n 'logbook',\n 'logbook.page.bingo',\n 'magnify',\n 'need_glasses',\n 'notebook',\n 'outtolunch',\n 'photo',\n 'plaque',\n 'plaque.face.date',\n 'reader',\n 'reader.paper0.next',\n 'reader.paper0.prev',\n 'reader.paper1.next',\n 'reader.paper1.prev',\n 'reader.paper2.bingo',\n 'reader.paper2.next',\n 'reader.paper2.prev',\n 'reader_flag',\n 'reader_flag.paper0.next',\n 'reader_flag.paper0.prev',\n 'reader_flag.paper1.next',\n 'reader_flag.paper1.prev',\n 'reader_flag.paper2.bingo',\n 'reader_flag.paper2.next',\n 'reader_flag.paper2.prev',\n 'remove_cup',\n 'report',\n 'retirement_letter',\n 'savedteddy',\n 'seescratches',\n 'teddy',\n 'tobasement',\n 'tocage',\n 'tocloset',\n 'tocloset_dirty',\n 'tocollection',\n 'tocollectionflag',\n 'toentry',\n 'tofrontdesk',\n 'togrampa',\n 'tohallway',\n 'tomap',\n 'tomicrofiche',\n 'tostacks',\n 'tracks',\n 'tracks.hub.deer',\n 'trigger_coffee',\n 'trigger_scarf',\n 'tunic',\n 'tunic.capitol_0',\n 'tunic.capitol_1',\n 'tunic.capitol_2',\n 'tunic.drycleaner',\n 'tunic.flaghouse',\n 'tunic.historicalsociety',\n 'tunic.hub.slip',\n 'tunic.humanecology',\n 'tunic.kohlcenter',\n 'tunic.library',\n 'tunic.wildlife',\n 'unlockdoor',\n 'wells',\n 'wellsbadge',\n 'what_happened',\n 'worker',\n 'fqid_None']\n\nroom_fqid_feature = ['tunic.capitol_0.hall',\n 'tunic.capitol_1.hall',\n 'tunic.capitol_2.hall',\n 'tunic.drycleaner.frontdesk',\n 'tunic.flaghouse.entry',\n 'tunic.historicalsociety.basement',\n 'tunic.historicalsociety.cage',\n 'tunic.historicalsociety.closet',\n 'tunic.historicalsociety.closet_dirty',\n 'tunic.historicalsociety.collection',\n 'tunic.historicalsociety.collection_flag',\n 'tunic.historicalsociety.entry',\n 'tunic.historicalsociety.frontdesk',\n 'tunic.historicalsociety.stacks',\n 'tunic.humanecology.frontdesk',\n 'tunic.kohlcenter.halloffame',\n 'tunic.library.frontdesk',\n 'tunic.library.microfiche',\n 'tunic.wildlife.center']\n\ntext_feature = ['*COUGH COUGH COUGH*',\n '*cough cough*',\n '*grumble grumble*',\n 'A boring old shirt.',\n 'A little horse!',\n 'A real, live ferret!',\n 'A vexy-wha?',\n 'AND I know who took Teddy!',\n 'AND he stole Teddy!',\n 'Actually, I went to school with somebody who LOVES old flags.',\n \"Actually, badgers aren't rodents-\",\n \"Actually, he's a badger.\",\n \"Actually, we're just here for some photos.\",\n \"Ah, Jolie! I'm glad you're here.\",\n \"Ah, that's better!\",\n 'Aha! Good catch, Jo.',\n 'Alright, Jolie. Back to work.',\n 'An old shirt? Try the university.',\n \"And I'll figure out the shirt, too.\",\n \"And he messed up Gramps's office, too!\",\n 'And how did that badger get free?',\n \"And look! She's wearing the shirt!\",\n 'And my homework?!?!',\n 'And this place is dirty, and itchy, and-',\n 'And we still need to figure out that flag!',\n \"And where's your grampa?\",\n 'And you are?',\n 'And you, Frank-',\n 'Any ideas?',\n \"Are you going home now? Tomorrow's the big day!\",\n 'Are you okay?',\n 'Are you sure? I know where you can find a real, live badger for the exhibit!',\n 'BUT WELLS STOLE TEDDY!',\n 'Badgers? No.',\n 'Besides, I already figured out the shirt.',\n 'Besides, he looks friendly to me.',\n 'Besides, you just ate my last snack.',\n 'Better check back later.',\n \"But I hear the museum's got one on the loose!\",\n \"But cranes can't do yoga, Teddy!\",\n 'But he never goes anywhere without his scarf!',\n 'But what if Wells kidnapped Teddy?',\n \"Calm down, kid. I haven't seen him.\",\n 'Can I come, Gramps?',\n 'Can I give you the tour?',\n 'Can I help you with anything?',\n 'Can I ride with you?',\n 'Can I take a closer look?',\n 'Can Teddy and I help?',\n 'Can we hurry up, Gramps?',\n 'Can you help me find Wells?',\n 'Can you help me tidy up?',\n 'Can you help me-',\n 'Can you help me? I need to find Wells!',\n 'Can you help me? I need to find the owner of this slip.',\n 'Can you help-',\n 'Can you take a look?',\n \"Can't believe I lost my reading glasses.\",\n 'Careful. That beak is sharp!',\n \"Check out our microfiche. It's right through that door.\",\n \"Check out the archives. They've got tons of old photos!\",\n 'Check out the next artifact!',\n 'Come on, Jo!',\n 'Come on, Teddy.',\n \"Come on, Teddy. Let's go help Gramps!\",\n \"Come on, kid. Let's go.\",\n \"Come on, kid. You're slowing me down.\",\n \"Come on, let's get out of here!\",\n 'Could be. But we need evidence!',\n 'Could be. But we need evidence.',\n \"Cranes don't eat donuts!\",\n 'Did you do all of them?',\n 'Did you drop something, Dear?',\n \"Did you drop something, Dear? There's a card on the floor.\",\n 'Did you figure out the shirt?',\n 'Did you have a question or not?',\n 'Did you have a question?',\n \"Did you steal Gramps's paperwork too?!\",\n 'Do I have to?',\n 'Do you have any info on Theodora Youmans?',\n 'Do you know anything about this slip?',\n 'Do you know what this flag was used for?',\n 'Do you know what this slip is?',\n 'Do you know where I can find a deer expert?',\n 'Do you know who Theodora Youmans is?',\n 'Do you really think that symbol is a deer hoof?',\n 'Does it look like a deer hoof?',\n \"Don't worry, Gramps. I'll find Teddy!\",\n \"Don't worry, Teddy won't eat your lunch anymore!\",\n \"Don't worry, he won't! (And he's a badger, by the way.)\",\n 'Easy, Jo.',\n 'FINE. That possum better not scratch my leather seats...',\n 'Find anything?',\n \"Fine, fine. Let's see...\",\n \"Fine. Let's investigate!\",\n \"Fine. Then I guess you don't want a real, live badger for the exhibit.\",\n 'Found it!',\n 'GAH! And what is THAT doing out of its cage?!',\n 'GRRRRRRR',\n 'Gadzooks! Poor critter.',\n \"Gah. I can't believe this.\",\n 'Go ahead, take a peek at the shirt!',\n \"Go check the microfiche. Maybe you'll find something!\",\n 'Go find your grampa and get to work!',\n 'Go on, tell the boss what you found!',\n 'Go take a look!',\n 'Good catch!',\n 'Good idea. Thanks!',\n 'Good luck!',\n 'Got one!',\n 'Grab your notebook and come upstairs!',\n 'Gramps is a great historian!',\n 'Gramps is in trouble for losing papers?',\n 'Gramps is the best historian ever!',\n 'Gramps must be up in the collection room.',\n 'Gramps said to look for clues. Better look around.',\n \"Great Scott, you're right!\",\n 'Great! Thanks for the help!',\n 'Great. Just great. Could this day get any worse?!',\n \"Guess it couldn't hurt to let you take a look.\",\n 'Guess so!',\n 'Ha! Good one.',\n \"Ha! I don't need your help.\",\n 'Ha! What do you call a pony with a sore throat?',\n \"Ha! You're funny.\",\n 'Ha. Told you so!',\n \"Hang on. I'll get you out of there!\",\n 'Hang tight, Teddy.',\n 'Have a look at the artifact!',\n 'Have you seen a badger around here?',\n 'He got a park named after him? Cool!',\n 'He has??',\n 'He needs our help!',\n \"He says he'd be willing to help out.\",\n 'He was looking for a taxidermist.',\n \"He's a badger!\",\n \"He's always trying to get you in trouble, and he doesn't like animals!\",\n \"He's been eating my lunch every day this week!\",\n \"He's our expert record keeper.\",\n \"He's still missing!\",\n \"He's wrong about old shirts and his name rhymes with \\\\smells\\\\...\",\n \"He's. A. Badger.\",\n 'Head back to the museum. Your gramps is waiting for you.',\n 'Head over to the Basketball Center.',\n 'Head over to the Wildlife Center!',\n 'Head over to the university.',\n 'Head upstairs and talk to the archivist. He might be able to help!',\n 'Hello there!',\n 'Her beak is stuck in a coffee cup.',\n 'Here I am!',\n \"Here's a call number for the Stacks. Go find some photos.\",\n \"Here's a call number to find more info in the Stacks.\",\n \"Here's the log book.\",\n \"Here's your scarf back!\",\n \"Hey Jo, let's take a look at the shirt!\",\n 'Hey!',\n \"Hey! That's Governor Nelson in front of our flag!\",\n \"Hey, I've seen that symbol before! Check it out!\",\n 'Hey, Wells...',\n 'Hey, look at those scratches!',\n 'Hey, nice dog! What breed is he?',\n 'Hey, this is Youmans!',\n 'Hi! *cough*',\n 'Hi! How can I help you?',\n 'Hi, Mrs. M.',\n \"Hmm. Button's still not working.\",\n \"Hmm. Let's see...\",\n 'Hmm. You could try the Aldo Leopold Wildlife Center.',\n 'Hmm... those stripes remind me of the American flag.',\n \"Hmmm. Don't forget about your homework.\",\n \"Hmmm. Shouldn't you be doing your homework?\",\n \"Hmmm... not sure. Why don't you try the library?\",\n 'Hold your horses, Jo.',\n 'Hooray, a boring old shirt.',\n 'Hopefully you can find some clues!',\n 'Hopefully you can rustle up some clues!',\n 'Hot Dog! I knew it!',\n 'How can I find out whose glasses these are?',\n 'How can I help you?',\n 'Huh?',\n 'I bet the archivist could use this!',\n \"I can't believe this.\",\n \"I can't calm down. This is important!\",\n \"I can't go with you. I need to take the artifact upstairs.\",\n 'I captured a badger in our museum!',\n \"I don't have time for kids.\",\n \"I don't have time for this, Gramps.\",\n \"I don't have time for this.\",\n \"I don't know!\",\n \"I don't need that right now.\",\n \"I feel like I'm forgetting something.\",\n 'I figured out that you kidnapped him!',\n 'I found it on an old shirt.',\n 'I found it!',\n 'I found the flag! Governor Nelson used it on the first Earth Day!',\n 'I found the key!',\n \"I get to go to Gramps's meeting!\",\n 'I got here and the whole place was a mess!',\n 'I got here and the whole place was ransacked!',\n 'I got that one from my Gramps!',\n 'I gotta run to my meeting!',\n 'I had one, but Teddy chewed it up.',\n 'I had some cleaning up to do in my office.',\n 'I have an idea.',\n 'I have to head over there and check out the animals.',\n \"I haven't quite figured it out just yet...\",\n \"I haven't seen him.\",\n 'I hope you find your badger, kid.',\n 'I knew I could count on you, Jo!',\n 'I knew it!',\n 'I knew you could do it, Jo!',\n 'I love it!',\n 'I love these photos of me and Teddy!',\n 'I love these photos of me and Teddy.',\n 'I need to find Wells right away! Do you know where he is?',\n 'I need to find Wells right away!! Do you know where he is?',\n 'I need to find Wells!!!',\n 'I need to find the owner of this slip.',\n \"I need to get her free. She won't hold still!\",\n 'I need to get to the Capitol and tell Gramps!',\n 'I need to learn more about this flag!',\n \"I need to take the artifact upstairs. Why don't you investigate those scratch marks?\",\n 'I need your help!',\n 'I ran into Wells there this morning.',\n 'I should ask the librarian where to go next.',\n 'I should ask the librarian why Wells was here.',\n 'I should check out that pair of glasses.',\n 'I should check that logbook to see who owned this slip...',\n 'I should find out if she can help me!',\n 'I should go talk to Gramps!',\n 'I should go to the Capitol and tell Mrs. M!',\n 'I should go to the Capitol and tell everyone!',\n 'I should help Gramps clean.',\n 'I should see what Grampa is up to!',\n 'I should see what Gramps is up to!',\n 'I should stay and look for clues!',\n 'I suppose historians are boring, too?',\n 'I think I can help with your animal problem.',\n 'I think I might be able to help you.',\n \"I think he's in trouble!\",\n \"I think it's a flag! Pretty interesting, huh?\",\n \"I think it's a flag! Pretty spiffy, eh?\",\n 'I told you!',\n 'I used to have a magnifying glass around here\\\\u00e2\\\\u20ac\\\\u00a6',\n \"I wonder if there's a clue in those business cards...\",\n 'I wonder whose glasses these are.',\n \"I'll be at the Capitol. Let me know if you find anything!\",\n \"I'll be in the collection room. Come find me when you're ready to check out the artifact.\",\n \"I'll go look at everyone's pictures!\",\n \"I'll hurry back and then we can go exploring!\",\n \"I'll record this in my notebook.\",\n \"I'll ride with you!\",\n \"I'm Leopold's grandkid!\",\n \"I'm a historian, not a zookeeper!\",\n \"I'm afraid my papers have gone missing in this mess.\",\n \"I'm afraid not.\",\n \"I'm also looking for Theodora Youmans. Have you heard of her?\",\n \"I'm here to rescue my friend!\",\n \"I'm investigating this flag.\",\n \"I'm investigating this symbol.\",\n \"I'm not so sure that this is a basketball jersey.\",\n \"I'm not sure.\",\n \"I'm putting you in charge of the flag case.\",\n \"I'm sure they'll be able to help.\",\n \"I'm sure you'll find Theodora in there somewhere!\",\n \"I'm telling you, Boss. Taxidermy is the way to go!\",\n \"I've got Wells's ID!\",\n \"I've got a stack of business cards from my favorite cleaners.\",\n \"I've seen him eating homework and important papers, too.\",\n \"If I were you, I'd go to the library and do some digging.\",\n 'Is this your coffee, Gramps?',\n 'It could be an early design for the Wisconsin state flag!',\n 'It has something to do with ecology.',\n 'It just means flag expert. How can I help?',\n \"It must've been Wells.\",\n \"It'll be okay, Jo. We'll find Teddy!\",\n \"It's OK, girl! Look, I found you a cricket!\",\n \"It's a match!\",\n \"It's a women's basketball jersey!\",\n \"It's already all done!\",\n \"It's an ecology flag!\",\n \"It's for Grampa Leo. He's a historian!\",\n \"It's for the flag display!\",\n \"It's locked!\",\n \"It's lucky we found her.\",\n \"It's okay, Gramps. I'll go by myself.\",\n \"It's our Norwegian Craft exhibit!\",\n \"It's really cool, Gramps. But I'm worried about Teddy.\",\n \"It's such a nice fall day.\",\n \"It's true, they do keep going missing lately.\",\n 'Jo!',\n \"Jo! I can't go with you. I need to take the artifact upstairs.\",\n 'Jo, meet me back at my office.',\n \"Jolie! I was hoping you'd stop by. Any news on the flag artifact?\",\n \"Jolie! I was hoping you'd stop by. Any news on the shirt artifact?\",\n 'Jolie! Where have you been?',\n \"Jolie- keep your badger under control, or he'll have to go.\",\n \"Just please, don't let your badger eat them!\",\n 'Just talking to Teddy.',\n 'Just this old slip from 1916.',\n 'Knew what?',\n 'Leo... you mean Leopold?',\n 'Leopold, can you run back to the museum?',\n \"Leopold, why don't you help me set up in the Capitol?\",\n \"Let's follow those scratch marks!\",\n \"Let's get started. The Wisconsin Wonders exhibit opens tomorrow!\",\n \"Let's go find Gramps!\",\n \"Let's go find him!\",\n \"Let's go help Gramps!\",\n 'Look at all those activists!',\n \"Look at that! It's the bee's knees!\",\n 'Looks like a dry cleaning receipt.',\n \"Looks like it's not a deer hoof.\",\n 'Luckily there are tons of insects around here...',\n 'Make sure to get some old photos for the exhibit, like last time!',\n 'Maybe I can help!',\n 'Maybe he just got scared and ran off.',\n \"Maybe she'll let me take off the cup!\",\n \"Maybe there's a clue in this mess!\",\n \"Meet me back in my office and we'll get started!\",\n 'Meetings are BORING!',\n 'Mrs. M, I think Wells kidnapped Teddy.',\n 'My friend is a flag expert.',\n 'Nice decorations.',\n 'Nice seeing you, Jolie!',\n 'Nice work on the shirt, Jolie!',\n \"No he hasn't!\",\n \"No thanks. I don't need help from kids.\",\n \"No way, Gramps. You're the best!\",\n 'No... because history is boring!',\n 'Nonsense. I want live animals at the exhibit, not stuffed ones.',\n \"Nope, that's from Bean Town. I only drink Holdgers!\",\n 'Nope. But Youmans and other suffragists worked hard to change that.',\n \"Not Leopold here. He's been losing papers lately.\",\n 'Not sure.',\n 'Not sure. Do I look like a deer expert to you?',\n 'Not sure. Here, let me look it up.',\n 'Notice any clues about this flag?',\n 'Now I Just need to find all the cleaners from way back in 1916.',\n 'Now I just need some old photos, like last time.',\n 'Now I just need to find all the cleaners from wayyyy back in 1916.',\n 'Now can I tell you what happened to Teddy?',\n 'Now if only I could read this thing.',\n 'Now if only I could read this thing. Blasted tiny letters...',\n 'Now where did I put my notebook?',\n \"Of course you do. You've got a rodent following you around.\",\n 'Oh my!',\n 'Oh no!',\n 'Oh no! What happened to that crane?',\n \"Oh no. If I don't impress the boss soon,  I'm gonna get fired!\",\n 'Oh no... Teddy!',\n 'Oh no... they got sick from polluted water?',\n 'Oh yeah, cranes eat insects!',\n 'Oh! There was a staff directory in the entryway!',\n \"Oh, I'm fine! Just a little hoarse.\",\n \"Oh, cool! I've never seen a badger in real life.\",\n 'Oh, hello there!',\n \"Oh, that's from Bean Town.\",\n \"Oh, trust me. He'll make time.\",\n \"Okay. I'll find Teddy!\",\n \"Okay. I'll try.\",\n 'Okay. Thanks anyway.',\n 'Okay. Thanks!',\n 'One step at a time, Jo.',\n 'Ooh, I like clues!',\n 'Ooh, nice decorations!',\n 'Ooh, thanks!',\n 'Ooh... \\\\Ecology flag, by Ron Cobb.\\\\',\n 'Our shirt is too old to be a basketball jersey!',\n 'People sure drink a lot of coffee around here.',\n 'Please let me know if you do.',\n \"Please let me know if you do. It's important!\",\n 'Please?',\n 'Please? This is really important.',\n 'Plus, my teacher said I could help you out for extra credit!',\n \"Poor Gramps! I should make sure he's okay.\",\n 'Poor badger.',\n 'Poor foxes!',\n 'Right outside the door.',\n 'Run along to the university.',\n 'See you later, Teddy.',\n 'See?',\n 'See?!',\n 'She helped get votes for women!',\n 'She led marches and helped women get the right to vote!',\n 'She should be able to help you out.',\n \"She's right outside.\",\n 'Slow down, Jo.',\n 'So much cleaning to do...',\n 'So? History is boring!',\n \"So? What'd you find out?\",\n 'Something to do with ecology and Wisconsin.',\n \"Sorry I'm late.\",\n 'Sorry for the delay, Boss.',\n \"Sorry, I'm in a hurry.\",\n \"Sorry, I'm too busy for kids right now.\",\n \"Sorry, can't help you.\",\n 'Sounds good, Boss.',\n 'Sure thing, Jo. Grab your notebook and come upstairs!',\n 'Sure!',\n 'Sure! Give it a try.',\n 'Take a look!',\n 'Teddy and I were gonna go climb that huge tree out back!',\n 'Teddy is still missing!',\n 'Teddy! Did you really eat his lunch?',\n \"Teddy! I'm glad to see you.\",\n \"Teddy! I'm sure glad to see you.\",\n 'Teddy!!!',\n \"Teddy's helping too.\",\n \"Teddy's scarf! Somebody must've taken him!\",\n 'Teddy, here I come!',\n 'Thanks for coming, Boss.',\n 'Thanks for the help!',\n 'Thanks for your help, kid!',\n 'Thanks to them, Wisconsin was the first state to approve votes for women!',\n 'Thanks!',\n 'Thanks.',\n 'Thanks. Did you figure out the shirt?',\n \"That hoofprint doesn't match the flag!\",\n 'That settles it.',\n \"That thing's a monster!\",\n \"That's it!\",\n 'The Stacks are right outside the door. Go find some photos!',\n 'The archivist had him locked up!',\n \"The archivist must've taken Teddy!\",\n 'The archivist said I should look in the stacks.',\n 'The boss is gonna love it!',\n 'The boss is gonna love it!!!',\n 'The exhibit opens tomorrow.',\n 'The kidnapper probably took Teddy on the elevator!',\n 'The libarian said I could find some information on Youmans in here...',\n \"The slip is from 1916 but the team didn't start until 1974!\",\n 'The symbol on the flag looks sort of like a deer hoof.',\n 'Then do it for me!',\n 'Then we need evidence.',\n 'Theodora Youmans must be the owner!',\n 'Theodora Youmans? Is that who owned the shirt?',\n 'Theodora Youmans? Of course!',\n 'Theodora wearing the shirt!',\n 'There are some old newspapers loaded up in the microfiche.',\n 'There should be some info about that symbol in my book.',\n \"There's a diagram of animal tracks over there.\",\n 'They study clothes and fabric.',\n 'This button never works!',\n \"This can't be right!\",\n 'This is perfect for the exhibit.',\n 'This looks like a clue!',\n 'This place was around in 1916! I can start there!',\n 'Those are the same glasses!',\n 'Though the archivist might be too busy to help...',\n 'Try not to panic, Jo.',\n 'Two missions, actually!',\n 'Ugh! Those cups are all over the place.',\n 'Ugh. Fine.',\n 'Ugh. I have to head over there and check out the animals.',\n 'Ugh. Meetings are so boring.',\n 'Ugh...',\n 'Ugh... I think that lynx is looking at me funny.',\n 'Um, are you okay?',\n 'Um... what did you want me to do again?',\n \"Unless you're too busy horsing around.\",\n 'Wait a minute!',\n 'Wait a minute...',\n \"Wait a sec. Women couldn't vote?!\",\n \"Wait! Can't I do it?\",\n 'Wait! What?! Really?',\n 'Wait, you mean Wells is wrong?!',\n 'Wait- me?',\n 'Want to look for more clues?',\n 'We just have to keep our eyes open!',\n 'We need to calm her down, Teddy.',\n 'We need to talk about that missing paperwork.',\n 'We still need to figure out that flag. Do you know anyone who could help?',\n \"We'll find Teddy.\",\n \"We'll find him, Jo.\",\n \"We'll need a key card.\",\n \"We're just looking for photos for the flag display.\",\n \"We're looking for some photos.\",\n 'Welcome back, Dear! How can I help you?',\n 'Welcome back, Jolie. Did you figure out the shirt?',\n \"Well, I can't show our log books to just anybody.\",\n \"Well, I did SOME of those. I just couldn't find them!\",\n 'Well, Leopold here is always losing papers...',\n \"Well, get on it. I'm counting on you and your gramps to figure this out!\",\n \"Well, get on it. I'm counting on you to figure this out!\",\n \"Well, that's good enough for me.\",\n 'Well... it looks hand-stitched.',\n 'Well? What are you still doing here?',\n 'Wells got in trouble for littering at the Wildlife Center.',\n 'Wells sabotaged Gramps!',\n 'Wells! What was he doing here? I should ask the librarian.',\n \"Wells! Where's Teddy? Is he okay?\",\n 'Wells, finish up your report.',\n 'Wells, meet Teddy.',\n 'Wells, you already have a job to do.',\n 'Wells? I knew it!',\n \"Weren't you going to check out our microfiche?\",\n 'Wha?!',\n 'What a fascinating artifact!',\n 'What are you doing down here?',\n 'What are you doing here?',\n 'What are you still doing here,  Jolie?',\n 'What are you waiting for? The Stacks are right outside the door.',\n 'What happened here?!',\n 'What is it, Teddy?',\n 'What is it?',\n 'What kind of photos do you need?',\n 'What now, kid?',\n 'What should I do first?',\n 'What should we do next?',\n 'What the-',\n 'What was Wells doing here?',\n \"What's a taxidermist?\",\n \"What's a textile expert?\",\n \"What's a vexillophile? \",\n \"What's going on here?\",\n 'What?!',\n 'Whatcha doing over there, Jo?',\n 'Whatever.',\n 'Where are the Stacks?',\n 'Where did you get that coffee?',\n 'Where should I go again?',\n 'Who are you?',\n \"Who could've done this?\",\n 'Who is Teddy?',\n 'Who wants to investigate the shirt artifact?',\n 'Whoever lost these glasses probably took Teddy!',\n \"Why didn't you say so?\",\n \"Why don't you go catch up with your grampa?\",\n \"Why don't you go play with your grampa?\",\n \"Why don't you go talk to her? I'll let her know you're coming.\",\n \"Why don't you go talk to the boss?\",\n \"Why don't you go upstairs and see the archivist?\",\n \"Why don't you head to the Basketball Center and rustle up some clues?\",\n \"Why don't you prove your case?\",\n \"Why don't you take a look?\",\n \"Why isn't the button working?\",\n 'Will do, Boss.',\n 'Wow!',\n 'Wow! What is all this stuff?',\n 'Wow! You figured it out!',\n \"Wow, that's so cool, Gramps!\",\n 'YEAH!',\n \"YOU'RE the new history detective everybody's talking about?\",\n 'YOU?!',\n 'Yeah. Thanks anyway.',\n 'Yep.',\n \"Yep. I'm a vexillophile!\",\n 'Yes!',\n 'Yes! I was wondering-',\n \"Yes! It's the key for Teddy's cage!\",\n 'Yes! This cool old slip from 1916.',\n 'Yes! This old slip from 1916.',\n 'Yes!!!',\n \"Yes!!! I'm saved!\",\n \"Yes, he has. I've seen him eating homework and important papers, too.\",\n 'Yikes... this could take a while.',\n 'Yoga does sound nice.',\n \"You again! Don't let him hurt me!\",\n 'You better get to the capitol!',\n 'You can talk to a textile expert there.',\n \"You can't just steal Jolie's pet.\",\n \"You can't just steal Jolie's pet. Don't you know badgers are protected animals?\",\n 'You could ask the archivist. He knows everybody!',\n 'You could try the archives.',\n 'You could try the archivist. Maybe he can help you find Wells!',\n 'You did it! Thanks, kid.',\n \"You haven't seen any badgers around here, have you?\",\n \"You look like you're on a mission.\",\n 'You stole Teddy! How could you?!',\n \"You'll have to get started without me.\",\n \"You're becoming quite the detective, Jo.\",\n \"You're right, Gramps. Let's investigate!\",\n \"You're right, Jo!\",\n \"You're still here? I'm trying to work!\",\n \"You've got a million flags here!\",\n 'Youmans was a suffragist here in Wisconsin.',\n 'Youmans was a suffragist!',\n 'Your flag must have been part of a national movement!',\n 'Your grampa is waiting for you in the collection room.',\n 'Your gramps is awesome! Always full of stories.',\n 'Your teacher said you could help me for extra credit.',\n 'Your teacher said you missed 7 assignments in a row!',\n \"Yup, that's him!\",\n '\\\\Ecology flag, by Ron Cobb.\\\\',\n '\\\\Taxidermy: the art of preparing, stuffing, and mounting the skins of animals.\\\\',\n '\\\\u00e2\\\\u009d\\\\u00a4\\\\u00ef\\\\u00b8\\\\u008f',\n '\\\\u00f0\\\\u0178\\\\u008d\\\\u00a9',\n '\\\\u00f0\\\\u0178\\\\u00a6\\\\u2014',\n '\\\\u00f0\\\\u0178\\\\u00a7\\\\u02dc',\n '\\\\u00f0\\\\u0178\\\\u02dc\\\\u0090',\n '\\\\u00f0\\\\u0178\\\\u02dc\\\\u009d',\n '\\\\u00f0\\\\u0178\\\\u02dc\\\\u00ad',\n '\\\\u00f0\\\\u0178\\\\u02dc\\\\u00b4',\n '\\\\u00f0\\\\u0178\\\\u02dc\\\\u0160',\n '\\\\u00f0\\\\u0178\\\\u2122\\\\u201e',\n 'undefined',\n 'text_None',\n 'None',\n]\n\n\n\ntext_fqid_feature = ['tunic.capitol_0.hall.boss.talktogramps',\n 'tunic.capitol_0.hall.chap1_finale_c',\n 'tunic.capitol_1.hall.boss.haveyougotit',\n 'tunic.capitol_1.hall.boss.writeitup',\n 'tunic.capitol_1.hall.chap2_finale_c',\n 'tunic.capitol_2.hall.boss.haveyougotit',\n 'tunic.capitol_2.hall.chap4_finale_c',\n 'tunic.drycleaner.frontdesk.block_0',\n 'tunic.drycleaner.frontdesk.block_1',\n 'tunic.drycleaner.frontdesk.logbook.page.bingo',\n 'tunic.drycleaner.frontdesk.worker.done',\n 'tunic.drycleaner.frontdesk.worker.done2',\n 'tunic.drycleaner.frontdesk.worker.hub',\n 'tunic.drycleaner.frontdesk.worker.takealook',\n 'tunic.flaghouse.entry.colorbook',\n 'tunic.flaghouse.entry.flag_girl.hello',\n 'tunic.flaghouse.entry.flag_girl.hello_recap',\n 'tunic.flaghouse.entry.flag_girl.symbol',\n 'tunic.flaghouse.entry.flag_girl.symbol_recap',\n 'tunic.historicalsociety.basement.ch3start',\n 'tunic.historicalsociety.basement.gramps.seeyalater',\n 'tunic.historicalsociety.basement.gramps.whatdo',\n 'tunic.historicalsociety.basement.janitor',\n 'tunic.historicalsociety.basement.savedteddy',\n 'tunic.historicalsociety.basement.seescratches',\n 'tunic.historicalsociety.cage.confrontation',\n 'tunic.historicalsociety.cage.glasses.afterteddy',\n 'tunic.historicalsociety.cage.glasses.beforeteddy',\n 'tunic.historicalsociety.cage.lockeddoor',\n 'tunic.historicalsociety.cage.need_glasses',\n 'tunic.historicalsociety.cage.teddy.trapped',\n 'tunic.historicalsociety.cage.unlockdoor',\n 'tunic.historicalsociety.closet.doorblock',\n 'tunic.historicalsociety.closet.gramps.intro_0_cs_0',\n 'tunic.historicalsociety.closet.intro',\n 'tunic.historicalsociety.closet.notebook',\n 'tunic.historicalsociety.closet.photo',\n 'tunic.historicalsociety.closet.retirement_letter.hub',\n 'tunic.historicalsociety.closet.teddy.intro_0_cs_0',\n 'tunic.historicalsociety.closet.teddy.intro_0_cs_5',\n 'tunic.historicalsociety.closet_dirty.door_block_clean',\n 'tunic.historicalsociety.closet_dirty.door_block_talk',\n 'tunic.historicalsociety.closet_dirty.gramps.archivist',\n 'tunic.historicalsociety.closet_dirty.gramps.helpclean',\n 'tunic.historicalsociety.closet_dirty.gramps.news',\n 'tunic.historicalsociety.closet_dirty.gramps.nothing',\n 'tunic.historicalsociety.closet_dirty.photo',\n 'tunic.historicalsociety.closet_dirty.trigger_coffee',\n 'tunic.historicalsociety.closet_dirty.trigger_scarf',\n 'tunic.historicalsociety.closet_dirty.what_happened',\n 'tunic.historicalsociety.collection.cs',\n 'tunic.historicalsociety.collection.gramps.found',\n 'tunic.historicalsociety.collection.gramps.look_0',\n 'tunic.historicalsociety.collection.gramps.lost',\n 'tunic.historicalsociety.collection.tunic',\n 'tunic.historicalsociety.collection.tunic.slip',\n 'tunic.historicalsociety.collection_flag.gramps.flag',\n 'tunic.historicalsociety.collection_flag.gramps.recap',\n 'tunic.historicalsociety.entry.block_tocollection',\n 'tunic.historicalsociety.entry.block_tomap1',\n 'tunic.historicalsociety.entry.block_tomap2',\n 'tunic.historicalsociety.entry.boss.flag',\n 'tunic.historicalsociety.entry.boss.flag_recap',\n 'tunic.historicalsociety.entry.boss.talktogramps',\n 'tunic.historicalsociety.entry.directory.closeup.archivist',\n 'tunic.historicalsociety.entry.gramps.hub',\n 'tunic.historicalsociety.entry.groupconvo',\n 'tunic.historicalsociety.entry.groupconvo_flag',\n 'tunic.historicalsociety.entry.wells.flag',\n 'tunic.historicalsociety.entry.wells.flag_recap',\n 'tunic.historicalsociety.entry.wells.talktogramps',\n 'tunic.historicalsociety.frontdesk.archivist.foundtheodora',\n 'tunic.historicalsociety.frontdesk.archivist.have_glass',\n 'tunic.historicalsociety.frontdesk.archivist.have_glass_recap',\n 'tunic.historicalsociety.frontdesk.archivist.hello',\n 'tunic.historicalsociety.frontdesk.archivist.need_glass_0',\n 'tunic.historicalsociety.frontdesk.archivist.need_glass_1',\n 'tunic.historicalsociety.frontdesk.archivist.newspaper',\n 'tunic.historicalsociety.frontdesk.archivist.newspaper_recap',\n 'tunic.historicalsociety.frontdesk.archivist_glasses.confrontation',\n 'tunic.historicalsociety.frontdesk.archivist_glasses.confrontation_recap',\n 'tunic.historicalsociety.frontdesk.block_magnify',\n 'tunic.historicalsociety.frontdesk.key',\n 'tunic.historicalsociety.frontdesk.magnify',\n 'tunic.historicalsociety.stacks.block',\n 'tunic.historicalsociety.stacks.journals.pic_2.bingo',\n 'tunic.historicalsociety.stacks.journals_flag.pic_0.bingo',\n 'tunic.historicalsociety.stacks.journals_flag.pic_1.bingo',\n 'tunic.historicalsociety.stacks.journals_flag.pic_2.bingo',\n 'tunic.historicalsociety.stacks.outtolunch',\n 'tunic.humanecology.frontdesk.block_0',\n 'tunic.humanecology.frontdesk.block_1',\n 'tunic.humanecology.frontdesk.businesscards.card_bingo.bingo',\n 'tunic.humanecology.frontdesk.worker.badger',\n 'tunic.humanecology.frontdesk.worker.intro',\n 'tunic.kohlcenter.halloffame.block_0',\n 'tunic.kohlcenter.halloffame.plaque.face.date',\n 'tunic.kohlcenter.halloffame.togrampa',\n 'tunic.library.frontdesk.block_badge',\n 'tunic.library.frontdesk.block_badge_2',\n 'tunic.library.frontdesk.block_nelson',\n 'tunic.library.frontdesk.wellsbadge.hub',\n 'tunic.library.frontdesk.worker.droppedbadge',\n 'tunic.library.frontdesk.worker.flag',\n 'tunic.library.frontdesk.worker.flag_recap',\n 'tunic.library.frontdesk.worker.hello',\n 'tunic.library.frontdesk.worker.hello_short',\n 'tunic.library.frontdesk.worker.nelson',\n 'tunic.library.frontdesk.worker.nelson_recap',\n 'tunic.library.frontdesk.worker.preflag',\n 'tunic.library.frontdesk.worker.wells',\n 'tunic.library.frontdesk.worker.wells_recap',\n 'tunic.library.microfiche.block_0',\n 'tunic.library.microfiche.reader.paper2.bingo',\n 'tunic.library.microfiche.reader_flag.paper2.bingo',\n 'tunic.wildlife.center.coffee',\n 'tunic.wildlife.center.crane_ranger.crane',\n 'tunic.wildlife.center.expert.recap',\n 'tunic.wildlife.center.expert.removed_cup',\n 'tunic.wildlife.center.fox.concern',\n 'tunic.wildlife.center.remove_cup',\n 'tunic.wildlife.center.tracks.hub.deer',\n 'tunic.wildlife.center.wells.animals',\n 'tunic.wildlife.center.wells.animals2',\n 'tunic.wildlife.center.wells.nodeer',\n 'tunic.wildlife.center.wells.nodeer_recap',\n 'text_fqid_None']\n\n\nname_feature = ['basic', 'undefined', 'close', 'open', 'prev', 'next']\n\npage_feature = [0,1,2,3,4,5,6]\n\nNUMS = [ \n    'room_dis',\n    'screen_dis',\n    'hover_duration', \n    'elapsed_time_diff',\n    'elapsed_time_diff_forward'\n]\n","metadata":{"execution":{"iopub.status.busy":"2023-06-08T16:53:17.986102Z","iopub.execute_input":"2023-06-08T16:53:17.986513Z","iopub.status.idle":"2023-06-08T16:53:18.078089Z","shell.execute_reply.started":"2023-06-08T16:53:17.986476Z","shell.execute_reply":"2023-06-08T16:53:18.076757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def time_feature(train):\n    train[\"year\"] = train[\"session_id\"].apply(lambda x: int(str(x)[:2])).astype(np.uint8)\n    train[\"month\"] = train[\"session_id\"].apply(lambda x: int(str(x)[2:4])+1).astype(np.uint8)\n    train[\"day\"] = train[\"session_id\"].apply(lambda x: int(str(x)[4:6])).astype(np.uint8)\n    train[\"hour\"] = train[\"session_id\"].apply(lambda x: int(str(x)[6:8])).astype(np.uint8)\n    train[\"minute\"] = train[\"session_id\"].apply(lambda x: int(str(x)[8:10])).astype(np.uint8)\n    train[\"second\"] = train[\"session_id\"].apply(lambda x: int(str(x)[10:12])).astype(np.uint8)\n    return train\n\ncolumns = [\n    pl.col(\"page\").cast(pl.Float32),\n    (\n        (pl.col(\"elapsed_time\") - pl.col(\"elapsed_time\").shift(1)) # time used for each action\n         .fill_null(0)\n         .clip(0, 1e9)\n         .over([\"session_id\", \"level_group\"])\n         .alias(\"elapsed_time_diff\")\n    ),\n    (\n        (pl.col(\"elapsed_time\").shift(-1) - pl.col(\"elapsed_time\")) # time used for each action\n         .fill_null(0)\n         .clip(0, 1e9)\n         .over([\"session_id\", \"level_group\"])\n         .alias(\"elapsed_time_diff_forward\")\n    ),\n    (\n        (pl.col(\"screen_coor_x\") - pl.col(\"screen_coor_x\").shift(1)) # location x changed for click \n         .abs()\n         .over([\"session_id\", \"level_group\"])\n    ),\n    (\n        (pl.col(\"screen_coor_y\") - pl.col(\"screen_coor_y\").shift(1)) # location y changed for click \n         .abs()\n         .over([\"session_id\", \"level_group\"])\n    ),\n    pl.col(\"fqid\").fill_null(\"fqid_None\"),\n    pl.col(\"text_fqid\").fill_null(\"text_fqid_None\"),\n    pl.col(\"text\").fill_null(\"text_None\"),\n    pl.col(\"room_coor_x\").forward_fill().alias(\"room_coor_x\"),\n    pl.col(\"room_coor_y\").forward_fill().alias(\"room_coor_y\"),\n    pl.col(\"screen_coor_x\").forward_fill().alias(\"screen_coor_x\"),\n    pl.col(\"screen_coor_y\").forward_fill().alias(\"screen_coor_y\"),\n]\n\ncolumns_1 = [\n    (pl.col(\"room_coor_x\") - pl.col(\"room_coor_x\").shift(1)).over([\"session_id\"]).pow(2).alias(\"room_coor_x_dis\"),\n    (pl.col(\"room_coor_y\") - pl.col(\"room_coor_y\").shift(1)).over([\"session_id\"]).pow(2).alias(\"room_coor_y_dis\"),    \n    (pl.col(\"screen_coor_x\") - pl.col(\"screen_coor_x\").shift(1)).over([\"session_id\"]).pow(2).alias(\"screen_coor_x_dis\"),\n    (pl.col(\"screen_coor_y\") - pl.col(\"screen_coor_y\").shift(1)).over([\"session_id\"]).pow(2).alias(\"screen_coor_y_dis\"),    \n    pl.col(\"fqid\").cast(pl.Utf8),\n    pl.col(\"room_fqid\").cast(pl.Utf8),\n    pl.col(\"text_fqid\").cast(pl.Utf8),\n]\n\ncolumns_2 = [\n    (pl.col(\"room_coor_x_dis\") + pl.col(\"room_coor_y_dis\")).pow(1/2).alias(\"room_dis\"),\n    (pl.col(\"screen_coor_x_dis\") + pl.col(\"screen_coor_y_dis\")).pow(1/2).alias(\"screen_dis\"),\n]","metadata":{"execution":{"iopub.status.busy":"2023-06-08T16:53:18.079595Z","iopub.execute_input":"2023-06-08T16:53:18.080008Z","iopub.status.idle":"2023-06-08T16:53:18.107893Z","shell.execute_reply.started":"2023-06-08T16:53:18.079937Z","shell.execute_reply":"2023-06-08T16:53:18.106364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineer(x, grp, use_extra, feature_suffix):\n    pl.toggle_string_cache(True)\n    if (grp=='0-4'):\n        level_feature = range(5)\n    elif (grp=='5-12'):\n#         level_feature = [0,1,2,3,4,5,6,7,8,9,10,11,12]\n        level_feature = range(5, 13)\n    else:\n#         level_feature = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22]\n        level_feature = range(13, 23)\n        \n    aggs = [\n        pl.col(\"index\").count().alias(f\"session_number_{feature_suffix}\"),\n        pl.col(\"elapsed_time\").apply(lambda s: s.max()-s.min()).alias(f\"session_time_{feature_suffix}\"),\n        \n        *[pl.col(c).drop_nulls().n_unique().alias(f\"{c}_unique_{feature_suffix}\") for c in CATS],\n        \n        *[pl.col(c).sum().alias(f\"{c}_sum_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).mean().alias(f\"{c}_mean_{feature_suffix}\") for c in NUMS],\n#         *[pl.col(c).min().alias(f\"{c}_min_{feature_suffix}\") for c in NUMS],\n        *[pl.col(c).max().alias(f\"{c}_max_{feature_suffix}\") for c in NUMS],\n#         *[pl.col(c).std().alias(f\"{c}_std_{feature_suffix}\") for c in NUMS],\n#         *[pl.col(c).count().alias(f\"{c}_count_{feature_suffix}\") for c in NUMS],\n        \n    ]\n    \n    for cat in ['fqid', 'room_fqid', 'name', 'text', 'text_fqid', 'level', 'page']:\n        for col in ['elapsed_time_diff']:\n            aggs.extend([\n                *[pl.col(col).filter(pl.col(cat)==c).sum().alias(f\"{cat}_{c}_{col}_sum_{feature_suffix}\") for c in eval(f'{cat}_feature')],\n#                 *[pl.col(col).filter(pl.col(cat)==c).mean().alias(f\"{cat}_{c}_{col}_mean_{feature_suffix}\") for c in eval(f'{cat}_feature')],\n                *[pl.col(col).filter(pl.col(cat)==c).max().alias(f\"{cat}_{c}_{col}_max_{feature_suffix}\") for c in eval(f'{cat}_feature')],\n#                 *[pl.col(col).filter(pl.col(cat)==c).min().alias(f\"{cat}_{c}_{col}_min_{feature_suffix}\") for c in eval(f'{cat}_feature')],\n#                 *[pl.col(col).filter(pl.col(cat)==c).std().alias(f\"{cat}_{c}_{col}_std_{feature_suffix}\") for c in eval(f'{cat}_feature')],\n                *[pl.col(col).filter((pl.col(cat)==c) & (~pl.col(col).is_null())).count().alias(f\"{cat}_{c}_{col}_count_{feature_suffix}\") for c in eval(f'{cat}_feature')],\n            ])\n        for col in ['elapsed_time_diff_forward', 'hover_duration', 'room_dis', 'screen_dis']:\n            aggs.extend([\n                *[pl.col(col).filter(pl.col(cat)==c).sum().alias(f\"{cat}_{c}_{col}_sum_{feature_suffix}\") for c in eval(f'{cat}_feature')],\n#                 *[pl.col(col).filter(pl.col(cat)==c).mean().alias(f\"{cat}_{c}_{col}_mean_{feature_suffix}\") for c in eval(f'{cat}_feature')],\n                *[pl.col(col).filter(pl.col(cat)==c).max().alias(f\"{cat}_{c}_{col}_max_{feature_suffix}\") for c in eval(f'{cat}_feature')],\n#                 *[pl.col(col).filter(pl.col(cat)==c).min().alias(f\"{cat}_{c}_{col}_min_{feature_suffix}\") for c in eval(f'{cat}_feature')],\n#                 *[pl.col(col).filter(pl.col(cat)==c).std().alias(f\"{cat}_{c}_{col}_std_{feature_suffix}\") for c in eval(f'{cat}_feature')],\n            ])\n        \n        aggs.extend([\n                *[pl.col('elapsed_time').filter(pl.col(cat)==c).diff().fill_null(0).clip(0, 1e9).sum().alias(f\"{cat}_{c}_ET2_sum_{feature_suffix}\") for c in eval(f'{cat}_feature')],\n#                 *[pl.col('elapsed_time').filter(pl.col(cat)==c).diff().fill_null(0).clip(0, 1e9).mean().alias(f\"{cat}_{c}_ET2_mean_{feature_suffix}\") for c in eval(f'{cat}_feature')],\n                *[pl.col('elapsed_time').filter(pl.col(cat)==c).diff().fill_null(0).clip(0, 1e9).max().alias(f\"{cat}_{c}_ET2_max_{feature_suffix}\") for c in eval(f'{cat}_feature')],\n#                 *[pl.col('elapsed_time').filter(pl.col(cat)==c).diff().fill_null(0).clip(0, 1e9).min().alias(f\"{cat}_{c}_ET2_min_{feature_suffix}\") for c in eval(f'{cat}_feature')],\n#                 *[pl.col('elapsed_time').filter(pl.col(cat)==c).diff().fill_null(0).clip(0, 1e9).std().alias(f\"{cat}_{c}_ET2_std_{feature_suffix}\") for c in eval(f'{cat}_feature')],\n            ])\n    \n#     aggs.extend([*[pl.col('elapsed_time').filter(pl.col('level')==c).\\\n#                    apply(lambda s: s.max()-s.min())\\\n#                    .alias(f\"level_{c}_duration_{feature_suffix}\") for c in level_feature]])\n\n    for col in ['elapsed_time_diff']:\n\n        aggs.extend([\n             *[pl.col(col).filter((pl.col(\"level\") == level) & (pl.col(\"text\").is_in(unused_data_values[level][\"text\"]))).count().alias(f\"level_{level}_unused_text_{col}_counts\") for level in level_feature],\n             *[pl.col(col).filter((pl.col(\"level\") == level) & (pl.col(\"fqid\").is_in(unused_data_values[level][\"fqid\"]))).count().alias(f\"level_{level}_unused_fqid_{col}_counts\") for level in level_feature],\n             *[pl.col(col).filter((pl.col(\"level\") == level) & (pl.col(\"room_fqid\").is_in(unused_data_values[level][\"room_fqid\"]))).count().alias(f\"level_{level}_unused_room_fqid_{col}_counts\") for level in level_feature],\n             *[pl.col(col).filter((pl.col(\"level\") == level) & (pl.col(\"text_fqid\").is_in(unused_data_values[level][\"text_fqid\"]))).count().alias(f\"level_{level}_unused_text_fqid_{col}_counts\") for level in level_feature],\n             *[pl.col(col).filter((pl.col(\"level\") == level) & (pl.col(\"text\").is_in(unused_data_values[level][\"text\"]))).sum().alias(f\"level_{level}_unused_text_{col}_sum\") for level in level_feature],\n             *[pl.col(col).filter((pl.col(\"level\") == level) & (pl.col(\"fqid\").is_in(unused_data_values[level][\"fqid\"]))).sum().alias(f\"level_{level}_unused_fqid_{col}_sum\") for level in level_feature],\n             *[pl.col(col).filter((pl.col(\"level\") == level) & (pl.col(\"room_fqid\").is_in(unused_data_values[level][\"room_fqid\"]))).sum().alias(f\"level_{level}_unused_room_fqid_{col}_sum\") for level in level_feature],\n             *[pl.col(col).filter((pl.col(\"level\") == level) & (pl.col(\"text_fqid\").is_in(unused_data_values[level][\"text_fqid\"]))).sum().alias(f\"level_{level}_unused_text_fqid_{col}_sum\") for level in level_feature],\n#              *[pl.col(col).filter((pl.col(\"level\") == level) & (pl.col(\"text\").is_in(unused_data_values[level][\"text\"]))).max().alias(f\"level_{level}_unused_text_{col}_max\") for level in level_feature],\n#              *[pl.col(col).filter((pl.col(\"level\") == level) & (pl.col(\"fqid\").is_in(unused_data_values[level][\"fqid\"]))).max().alias(f\"level_{level}_unused_fqid_{col}_max\") for level in level_feature],\n#              *[pl.col(col).filter((pl.col(\"level\") == level) & (pl.col(\"room_fqid\").is_in(unused_data_values[level][\"room_fqid\"]))).max().alias(f\"level_{level}_unused_room_fqid_{col}_max\") for level in level_feature],\n#              *[pl.col(col).filter((pl.col(\"level\") == level) & (pl.col(\"text_fqid\").is_in(unused_data_values[level][\"text_fqid\"]))).max().alias(f\"level_{level}_unused_text_fqid_{col}_max\") for level in level_feature],\n        ])\n    \n#     for col in ['elapsed_time_diff_forward', 'hover_duration', 'room_dis', 'screen_dis']:\n        \n#         aggs.extend([\n#              *[pl.col(col).filter((pl.col(\"level\") == level) & (pl.col(\"text\").is_in(unused_data_values[level][\"text\"]))).sum().alias(f\"level_{level}_unused_text_{col}_sum\") for level in level_feature],\n#              *[pl.col(col).filter((pl.col(\"level\") == level) & (pl.col(\"fqid\").is_in(unused_data_values[level][\"fqid\"]))).sum().alias(f\"level_{level}_unused_fqid_{col}_sum\") for level in level_feature],\n#              *[pl.col(col).filter((pl.col(\"level\") == level) & (pl.col(\"room_fqid\").is_in(unused_data_values[level][\"room_fqid\"]))).sum().alias(f\"level_{level}_unused_room_fqid_{col}_sum\") for level in level_feature],\n#              *[pl.col(col).filter((pl.col(\"level\") == level) & (pl.col(\"text_fqid\").is_in(unused_data_values[level][\"text_fqid\"]))).sum().alias(f\"level_{level}_unused_text_fqid_{col}_sum\") for level in level_feature],\n#              *[pl.col(col).filter((pl.col(\"level\") == level) & (pl.col(\"text\").is_in(unused_data_values[level][\"text\"]))).max().alias(f\"level_{level}_unused_text_{col}_max\") for level in level_feature],\n#              *[pl.col(col).filter((pl.col(\"level\") == level) & (pl.col(\"fqid\").is_in(unused_data_values[level][\"fqid\"]))).max().alias(f\"level_{level}_unused_fqid_{col}_max\") for level in level_feature],\n#              *[pl.col(col).filter((pl.col(\"level\") == level) & (pl.col(\"room_fqid\").is_in(unused_data_values[level][\"room_fqid\"]))).max().alias(f\"level_{level}_unused_room_fqid_{col}_max\") for level in level_feature],\n#              *[pl.col(col).filter((pl.col(\"level\") == level) & (pl.col(\"text_fqid\").is_in(unused_data_values[level][\"text_fqid\"]))).max().alias(f\"level_{level}_unused_text_fqid_{col}_max\") for level in level_feature],\n#         ])\n    \n    \n    df = x.groupby([\"session_id\"], maintain_order=True).agg(aggs).sort(\"session_id\")\n    \n    if use_extra:\n#         if grp == '0-4':\n#             aggs = [\n#                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"Now where did I put my notebook?\") | (pl.col(\"text\") == \"Found it!\")).apply(lambda s: s.max() - s.min()).alias(\"find_notebook_duration\"),\n#                 pl.col(\"index\").filter((pl.col(\"text\") == \"Now where did I put my notebook?\") | (pl.col(\"text\") == \"Found it!\")).apply(lambda s: s.max() - s.min()).alias(\"find_notebook_indexCount\"),\n#                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"Found it!\") | (pl.col(\"text\") == \"Let's get started. The Wisconsin Wonders exhibit opens tomorrow!\")).apply(lambda s: s.max() - s.min()).alias(\"go_upstairs_duration\"),\n#                 pl.col(\"index\").filter((pl.col(\"text\") == \"Found it!\") | (pl.col(\"text\") == \"Let's get started. The Wisconsin Wonders exhibit opens tomorrow!\")).apply(lambda s: s.max() - s.min()).alias(\"go_upstairs_events\"),\n#                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"Hey Jo, let's take a look at the shirt!\") | (pl.col(\"text\") == \"This looks like a clue!\")).apply(lambda s: s.max() - s.min()).alias(\"find_jersey_clue_duration\"),\n#                 pl.col(\"index\").filter((pl.col(\"text\") == \"Hey Jo, let's take a look at the shirt!\") | (pl.col(\"text\") == \"This looks like a clue!\")).apply(lambda s: s.max() - s.min()).alias(\"find_jersey_clue_indexCount\"),\n#                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"I'll be at the Capitol. Let me know if you find anything!\") | (pl.col(\"text\") == \"Our shirt is too old to be a basketball jersey!\")).apply(lambda s: s.max() - s.min()).alias(\"find_jersey_old_label_duration\"),\n#                 pl.col(\"index\").filter((pl.col(\"text\") == \"I'll be at the Capitol. Let me know if you find anything!\") | (pl.col(\"text\") == \"Our shirt is too old to be a basketball jersey!\")).apply(lambda s: s.max() - s.min()).alias(\"find_jersey_old_label_indexCount\"),\n#                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"I need to get to the Capitol and tell Gramps!\") | (pl.col(\"fqid\") ==\"chap1_finale\")).apply(lambda s: s.max() - s.min()).alias(\"to_chap1_final_duration\"),\n#                 pl.col(\"index\").filter((pl.col(\"text\") == \"I need to get to the Capitol and tell Gramps!\") | (pl.col(\"fqid\") == \"chap1_finale\")).apply(lambda s: s.max() - s.min()).alias(\"to_chap1_final_indexCount\"),\n#                 pl.col(\"elapsed_time\").filter((pl.col(\"fqid\") ==\"chap1_finale\") | (pl.col(\"fqid\") ==\"chap1_finale_c\")).apply(lambda s: s.max() - s.min()).alias(\"chap1_answer_duration\"),\n#                 pl.col(\"index\").filter((pl.col(\"fqid\") ==\"chap1_finale\") | (pl.col(\"fqid\") == \"chap1_finale_c\")).apply(lambda s: s.max() - s.min()).alias(\"chap1_answer_indexCount\"),\n#             ]\n#             tmp = x.groupby([\"session_id\"], maintain_order=True).agg(aggs).sort(\"session_id\") \n#             df = df.join(tmp, on=\"session_id\", how='left')\n        if grp=='5-12':\n            aggs = [\n                pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"Here's the log book.\") | (pl.col(\"fqid\") == 'logbook.page.bingo')).apply(lambda s: s.max() - s.min()).alias(\"logbook_bingo_duration\"),\n                pl.col(\"index\").filter((pl.col(\"text\") == \"Here's the log book.\") | (pl.col(\"fqid\") == 'logbook.page.bingo')).apply(lambda s: s.max() - s.min()).alias(\"logbook_bingo_indexCount\"),\n                pl.col(\"elapsed_time\").filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader')) | (pl.col(\"fqid\") == \"reader.paper2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\"reader_bingo_duration\"),\n                pl.col(\"index\").filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader')) | (pl.col(\"fqid\") == \"reader.paper2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\"reader_bingo_indexCount\"),\n                pl.col(\"elapsed_time\").filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals')) | (pl.col(\"fqid\") == \"journals.pic_2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\"journals_bingo_duration\"),\n                pl.col(\"index\").filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals')) | (pl.col(\"fqid\") == \"journals.pic_2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\"journals_bingo_indexCount\"),\n#                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"I got here and the whole place was a mess!\") | (pl.col(\"text\") == \"He's our expert record keeper.\")).apply(lambda s: s.max() - s.min()).alias(\"tidy_up_duration\"),\n#                 pl.col(\"index\").filter((pl.col(\"text\") == \"I got here and the whole place was a mess!\") | (pl.col(\"text\") == \"He's our expert record keeper.\")).apply(lambda s: s.max() - s.min()).alias(\"tidy_up_indexCount\"),\n#                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"He's our expert record keeper.\") | (pl.col(\"text\") == \"I need your help!\")).apply(lambda s: s.max() - s.min()).alias(\"find_archivist_duration\"),\n#                 pl.col(\"index\").filter((pl.col(\"text\") == \"He's our expert record keeper.\") | (pl.col(\"text\") == \"I need your help!\")).apply(lambda s: s.max() - s.min()).alias(\"find_archivist_indexCount\"),\n#                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"Now if only I could read this thing.\") | (pl.col(\"text\") == \"I bet the archivist could use this!\")).apply(lambda s: s.max() - s.min()).alias(\"find_glasses_duration\"),\n#                 pl.col(\"index\").filter((pl.col(\"text\") == \"Now if only I could read this thing.\") | (pl.col(\"text\") == \"I bet the archivist could use this!\")).apply(lambda s: s.max() - s.min()).alias(\"find_glasses_indexCount\"),\n# #                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"Great! Thanks for the help!\") | (pl.col(\"text\") == \"Hello there!\")).apply(lambda s: s.max() - s.min()).alias(\"find_textile_expert_duration\"),\n# #                 pl.col(\"index\").filter((pl.col(\"text\") == \"Great! Thanks for the help!\") | (pl.col(\"text\") == \"Hello there!\")).apply(lambda s: s.max() - s.min()).alias(\"find_textile_expert_indexCount\"),\n# #                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"This place was around in 1916! I can start there!\") | (pl.col(\"text\") == \"Hi! How can I help you?\")).apply(lambda s: s.max() - s.min()).alias(\"find_worker_duration\"),\n# #                 pl.col(\"index\").filter((pl.col(\"text\") == \"This place was around in 1916! I can start there!\") | (pl.col(\"text\") == \"Hi! How can I help you?\")).apply(lambda s: s.max() - s.min()).alias(\"find_worker_indexCount\"),\n#                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"He re's the log book.\") | (pl.col(\"text\") == \"It's a match!\")).apply(lambda s: s.max() - s.min()).alias(\"search_logbook_duration\"),\n#                 pl.col(\"index\").filter((pl.col(\"text\") == \"He re's the log book.\") | (pl.col(\"text\") == \"It's a match!\")).apply(lambda s: s.max() - s.min()).alias(\"search_logbook_indexCount\"),\n#                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"Thanks for the help!\") | (pl.col(\"text\") == \"Oh, hello there!\")).apply(lambda s: s.max() - s.min()).alias(\"go_to_library_duration\"),\n#                 pl.col(\"index\").filter((pl.col(\"text\") == \"Thanks for the help!\") | (pl.col(\"text\") == \"Oh, hello there!\")).apply(lambda s: s.max() - s.min()).alias(\"go_to_library_indexCount\"),\n#                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"Check out our microfiche. It's right through that door.\") | (pl.col(\"text\") == \"Youmans was a suffragist!\")).apply(lambda s: s.max() - s.min()).alias(\"check_microfiche_duration\"),\n#                 pl.col(\"index\").filter((pl.col(\"text\") == \"Check out our microfiche. It's right through that door.\") | (pl.col(\"text\") == \"Youmans was a suffragist!\")).apply(lambda s: s.max() - s.min()).alias(\"check_microfiche_indexCount\"),\n#                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"She helped get votes for women!\") | (pl.col(\"text\") == \"What was Wells doing here?\")).apply(lambda s: s.max() - s.min()).alias(\"pick_up_wells_card_duration\"),\n#                 pl.col(\"index\").filter((pl.col(\"text\") == \"She helped get votes for women!\") | (pl.col(\"text\") == \"What was Wells doing here?\")).apply(lambda s: s.max() - s.min()).alias(\"pick_up_wells_card_indexCount\"),\n#                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"You could ask the archivist. He knows everybody!\") | (pl.col(\"text\") == \"Can you help me? I need to find Wells!\")).apply(lambda s: s.max() - s.min()).alias(\"find_archivist_2_duration\"),\n#                 pl.col(\"index\").filter((pl.col(\"text\") == \"You could ask the archivist. He knows everybody!\") | (pl.col(\"text\") == \"Can you help me? I need to find Wells!\")).apply(lambda s: s.max() - s.min()).alias(\"find_archivist_2_indexCount\"),\n#                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"Right outside the door.\") | (pl.col(\"fqid\") == \"journals.pic_2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\"find_journals_pic_duration\"),\n#                 pl.col(\"index\").filter((pl.col(\"text\") == \"Right outside the door.\") | (pl.col(\"fqid\") == \"journals.pic_2.bingo\")).apply(lambda s: s.max() - s.min()).alias(\"find_journals_pic_indexCount\"),\n#                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"I should go to the Capitol and tell everyone!\") | (pl.col(\"fqid\") == \"chap2_finale_c\")).apply(lambda s: s.max() - s.min()).alias(\"go_to_capital_duration\"),\n#                 pl.col(\"index\").filter((pl.col(\"text\") == \"I should go to the Capitol and tell everyone!\") | (pl.col(\"fqid\") == \"chap2_finale_c\")).apply(lambda s: s.max() - s.min()).alias(\"go_to_capital_indexCount\"),\n                pl.col(\"elapsed_time\").filter((pl.col(\"fqid\") == \"chap2_finale_c\") | (pl.col(\"event_name\") == \"checkpoint\")).apply(lambda s: s.max() - s.min()).alias(\"chap2_answer_duration\"),\n                pl.col(\"index\").filter((pl.col(\"fqid\") == \"chap2_finale_c\") | (pl.col(\"event_name\") == \"checkpoint\")).apply(lambda s: s.max() - s.min()).alias(\"chap2_answer_indexCount\"),\n                (pl.col(\"elapsed_time\").filter(pl.col(\"level_group\") == \"5-12\").min() - pl.col(\"elapsed_time\").filter(pl.col(\"level_group\") == \"0-4\").max()).alias(\"chap1_answer_time\")\n            ]\n            tmp = x.groupby([\"session_id\"], maintain_order=True).agg(aggs).sort(\"session_id\")\n            df = df.join(tmp, on=\"session_id\", how='left')\n\n        if grp=='13-22':\n            aggs = [\n                pl.col(\"elapsed_time\").filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader_flag')) | (pl.col(\"fqid\") == \"tunic.library.microfiche.reader_flag.paper2.bingo\")).apply(lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"reader_flag_duration\"),\n                pl.col(\"index\").filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader_flag')) | (pl.col(\"fqid\") == \"tunic.library.microfiche.reader_flag.paper2.bingo\")).apply(lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"reader_flag_indexCount\"),\n                pl.col(\"elapsed_time\").filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals_flag')) | (pl.col(\"fqid\") == \"journals_flag.pic_0.bingo\")).apply(lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"journalsFlag_bingo_duration\"),\n                pl.col(\"index\").filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals_flag')) | (pl.col(\"fqid\") == \"journals_flag.pic_0.bingo\")).apply(lambda s: s.max() - s.min() if s.len() > 0 else 0).alias(\"journalsFlag_bingo_indexCount\"),\n# #                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"I'll go look at everyone's pictures!\") | (pl.col(\"text\") == \"Those are the same glasses!\")).apply(lambda s: s.max() - s.min()).alias(\"find_glasses_duration\"),\n# #                 pl.col(\"index\").filter((pl.col(\"text\") == \"I'll go look at everyone's pictures!\") | (pl.col(\"text\") == \"Those are the same glasses!\")).apply(lambda s: s.max() - s.min()).alias(\"find_glasses_indexCount\"),\n#                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"The archivist must've taken Teddy!\") | (pl.col(\"text\") == \"Yes! It's the key for Teddy's cage!\")).apply(lambda s: s.max() - s.min()).alias(\"find_key_duration\"),\n#                 pl.col(\"index\").filter((pl.col(\"text\") == \"The archivist must've taken Teddy!\") | (pl.col(\"text\") == \"Yes! It's the key for Teddy's cage!\")).apply(lambda s: s.max() - s.min()).alias(\"find_key_indexCount\"),\n#                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"Yes! It's the key for Teddy's cage!\") | (pl.col(\"text\") == \"I found the key!\")).apply(lambda s: s.max() - s.min()).alias(\"unlock_teddy_duration\"),\n#                 pl.col(\"index\").filter((pl.col(\"text\") == \"Yes! It's the key for Teddy's cage!\") | (pl.col(\"text\") == \"I found the key!\")).apply(lambda s: s.max() - s.min()).alias(\"unlock_teddy_indexCount\"),\n# #                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"Come on, Teddy. Let's go help Gramps!\") | (pl.col(\"text\") == \"Teddy! I'm glad to see you.\")).apply(lambda s: s.max() - s.min()).alias(\"go_to_collection_room_duration\"),\n# #                 pl.col(\"index\").filter((pl.col(\"text\") == \"Come on, Teddy. Let's go help Gramps!\") | (pl.col(\"text\") == \"Teddy! I'm glad to see you.\")).apply(lambda s: s.max() - s.min()).alias(\"go_to_collection_room_indexCount\"),\n#                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"I'll ride with you!\") | (pl.col(\"text\") == \"Oh no! What happened to that crane?\")).apply(lambda s: s.max() - s.min()).alias(\"go_to_wildlife_center_duration\"),\n#                 pl.col(\"index\").filter((pl.col(\"text\") == \"I'll ride with you!\") | (pl.col(\"text\") == \"Oh no! What happened to that crane?\")).apply(lambda s: s.max() - s.min()).alias(\"go_to_wildlife_center_indexCount\"),\n#                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"That hoofprint doesn't match the flag!\") | (pl.col(\"text\") == \"Hey, nice dog! What breed is he?\")).apply(lambda s: s.max() - s.min()).alias(\"meet_flag_expert_duration\"),\n#                 pl.col(\"index\").filter((pl.col(\"text\") == \"That hoofprint doesn't match the flag!\") | (pl.col(\"text\") == \"Hey, nice dog! What breed is he?\")).apply(lambda s: s.max() - s.min()).alias(\"meet_flag_expert_indexCount\"),\n#                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"Hey, nice dog! What breed is he?\") | (pl.col(\"text\") == \"Welcome back, Dear! How can I help you?\")).apply(lambda s: s.max() - s.min()).alias(\"go_to_library_2_duration\"),\n#                 pl.col(\"index\").filter((pl.col(\"text\") == \"Hey, nice dog! What breed is he?\") | (pl.col(\"text\") == \"Welcome back, Dear! How can I help you?\")).apply(lambda s: s.max() - s.min()).alias(\"go_to_library_2_indexCount\"),\n#                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"Go check the microfiche. Maybe you'll find something!\") | (pl.col(\"text\") == \"Hey! That's Governor Nelson in front of our flag!\")).apply(lambda s: s.max() - s.min()).alias(\"check_microfiche_2_duration\"),\n#                 pl.col(\"index\").filter((pl.col(\"text\") == \"Go check the microfiche. Maybe you'll find something!\") | (pl.col(\"text\") == \"Hey! That's Governor Nelson in front of our flag!\")).apply(lambda s: s.max() - s.min()).alias(\"check_microfiche_2_indexCount\"),\n#                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"Okay. Thanks!\") | (pl.col(\"text\") == \"It's for the flag display!\")).apply(lambda s: s.max() - s.min()).alias(\"find_archivist_3_duration\"),\n#                 pl.col(\"index\").filter((pl.col(\"text\") == \"Okay. Thanks!\") | (pl.col(\"text\") == \"It's for the flag display!\")).apply(lambda s: s.max() - s.min()).alias(\"find_archivist_3_indexCount\"),\n#                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"Here's a call number for the Stacks. Go find some photos.\") | (pl.col(\"text\") == \"Look at all those activists!\")).apply(lambda s: s.max() - s.min()).alias(\"find_photos_duration\"),\n#                 pl.col(\"index\").filter((pl.col(\"text\") == \"Here's a call number for the Stacks. Go find some photos.\") | (pl.col(\"text\") == \"Look at all those activists!\")).apply(lambda s: s.max() - s.min()).alias(\"find_photos_indexCount\"),\n#                 pl.col(\"elapsed_time\").filter((pl.col(\"text\") == \"I should go to the Capitol and tell Mrs. M!\") | (pl.col(\"fqid\") == \"chap4_finale_c\")).apply(lambda s: s.max() - s.min()).alias(\"go_to_capital_final_duration\"),\n#                 pl.col(\"index\").filter((pl.col(\"text\") == \"I should go to the Capitol and tell Mrs. M!\") | (pl.col(\"fqid\") == \"chap4_finale_c\")).apply(lambda s: s.max() - s.min()).alias(\"go_to_capital_final_indexCount\"),\n                pl.col(\"elapsed_time\").filter((pl.col(\"fqid\") == \"chap4_finale_c\") | (pl.col(\"event_name\") == \"checkpoint\")).apply(lambda s: s.max() - s.min()).alias(\"chap4_answer_duration\"),\n                pl.col(\"index\").filter((pl.col(\"fqid\") == \"chap4_finale_c\") | (pl.col(\"event_name\") == \"checkpoint\")).apply(lambda s: s.max() - s.min()).alias(\"chap4_answer_indexCount\"),\n                (pl.col(\"elapsed_time\").filter(pl.col(\"level_group\") == \"13-22\").min() - pl.col(\"elapsed_time\").filter(pl.col(\"level_group\") == \"5-12\").max()).alias(\"chap2_answer_time\")\n            ]\n            tmp = x.groupby([\"session_id\"], maintain_order=True).agg(aggs).sort(\"session_id\")\n            df = df.join(tmp, on=\"session_id\", how='left')\n        \n    return df.to_pandas()","metadata":{"execution":{"iopub.status.busy":"2023-06-08T16:53:18.109907Z","iopub.execute_input":"2023-06-08T16:53:18.110940Z","iopub.status.idle":"2023-06-08T16:53:18.194538Z","shell.execute_reply.started":"2023-06-08T16:53:18.110894Z","shell.execute_reply":"2023-06-08T16:53:18.193083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import jo_wilder\nenv = jo_wilder.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2023-06-08T16:53:18.195874Z","iopub.execute_input":"2023-06-08T16:53:18.196233Z","iopub.status.idle":"2023-06-08T16:53:18.215297Z","shell.execute_reply.started":"2023-06-08T16:53:18.196200Z","shell.execute_reply":"2023-06-08T16:53:18.213808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cached_dfs = {}","metadata":{"execution":{"iopub.status.busy":"2023-06-08T16:53:18.217071Z","iopub.execute_input":"2023-06-08T16:53:18.218086Z","iopub.status.idle":"2023-06-08T16:53:18.224230Z","shell.execute_reply.started":"2023-06-08T16:53:18.218038Z","shell.execute_reply":"2023-06-08T16:53:18.222761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\n    \nfilename = \"/kaggle/input/psp-model-ensemble-0608/blend_full.bin\"\nloaded_model = pickle.load(open(filename, 'rb'))\n\nlimits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\nfor (test, sample_submission) in iter_test:\n    \n    sub_cols = sample_submission.columns\n    sample_submission['q'] = sample_submission['session_id'].str.split('_').str[1].str[1:].astype(int)\n    sample_submission = sample_submission.sort_values('q')\n    sample_submission = sample_submission[sub_cols]\n    \n    grp = test.level_group.values[0]\n    session_id = test.session_id.values[0]\n    \n    cached_dfs[grp] = test.copy()\n    \n    if grp == '0-4':\n        pd_df = cached_dfs['0-4']\n        assert '5-12' not in cached_dfs\n        assert '13-22' not in cached_dfs\n    elif grp == '5-12':\n        assert '13-22' not in cached_dfs\n        pd_df = pd.concat([cached_dfs['0-4'], cached_dfs['5-12']])\n    else:\n        pd_df = pd.concat([cached_dfs['0-4'], cached_dfs['5-12'], cached_dfs['13-22']])\n        # Reset\n        cached_dfs = {}\n        \n    df = (pl.from_pandas(pd_df)\n          .drop([\"fullscreen\", \"hq\", \"music\"])\n          .sort(\"elapsed_time\")\n          .with_columns(columns))\n    \n    df = df.with_columns(columns_1)\n    df = df.with_columns(columns_2)\n        \n    # Make sure only 1 session in the df\n    assert df.select(pl.col(\"session_id\").n_unique()).item() == 1\n    \n    df = feature_engineer(df, grp, use_extra=True, feature_suffix='')\n    df = time_feature(df)\n    \n    # Make sure only 1 session in the df\n    assert df.shape[0] == 1\n        \n    features1 = feature_dict1[grp]\n    features2 = feature_dict2[grp]\n    features3 = feature_dict3[grp]\n\n    a, b = limits[grp]\n    # Repeat the row b-a times\n    df = df.loc[df.index.repeat(b-a)]\n    df[\"q\"] = list(range(a, b))\n\n    df[\"preds1\"] = 0\n    df[\"preds2\"] = 0\n    df[\"preds3\"] = 0\n\n    for fold in range(5):\n        model = model_dict1[grp][fold]    \n        _preds = model.predict_proba(df[features1])[:, 1]\n        df[\"preds1\"] += _preds / 5.0\n        \n        model = model_dict2[grp][fold]    \n        _preds = model.predict(xgb.DMatrix(df[features2]))\n        df[\"preds2\"] += _preds / 5.0\n    \n        model = model_dict3[grp][fold]    \n        _preds = model.predict(df[features2])\n        df[\"preds3\"] += _preds / 5.0\n\n    preds = loaded_model.predict_proba(df[[\"preds1\", \"preds2\", \"preds3\"]].values)[:,1]\n    \n    preds = (preds > 0.6597518113109433).astype(int)\n    assert len(preds) == sample_submission.shape[0] \n    sample_submission[\"correct\"] = preds\n    env.predict(sample_submission)","metadata":{"execution":{"iopub.status.busy":"2023-06-08T16:53:18.225800Z","iopub.execute_input":"2023-06-08T16:53:18.226659Z","iopub.status.idle":"2023-06-08T16:53:32.254083Z","shell.execute_reply.started":"2023-06-08T16:53:18.226617Z","shell.execute_reply":"2023-06-08T16:53:32.253139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('submission.csv')\nprint(sub.shape, sub.correct.mean())\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-08T16:53:32.255270Z","iopub.execute_input":"2023-06-08T16:53:32.256351Z","iopub.status.idle":"2023-06-08T16:53:32.272501Z","shell.execute_reply.started":"2023-06-08T16:53:32.256311Z","shell.execute_reply":"2023-06-08T16:53:32.271306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}