{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# [GamePlay] Further Improvement by Tf-Idf words (Eng/日本語)\n\nThis is continued from my [previous noteboook](https://www.kaggle.com/code/junjitakeshima/gameplay-slightly-improved-by-dialogue-jpn-eng), where I tried to improve Public score utilizing the dialogue information (column 'text' in Training data) based on the idea that this game is to answer the questions after accumulating various knowledge through the conversation with people shown on the screen.  \n\nI was able to slightly improve the score **from 0.676(baseline) to 0.677** using the counts of Demonstrative pronoun 'that, this, it'.  \n\nThis notebook is to show we can improve further to 0.678 using the specific words extracted by Tf-Idf approach.  \n\nThe score itself(0.678) is not so high yet, but I'd be happy if you think this attempt is interesting and refer to this notebook just as a recipe to improve your own model even a little bit.  \n\n\nこのNotebookは、私の[前回Notebook](https://www.kaggle.com/code/junjitakeshima/gameplay-slightly-improved-by-dialogue-jpn-eng)の続きです。  \n前回は、このゲームが画面上に現れる登場人物との会話を通じて知識を蓄積し質問に答えていくものであろうとの推測のもと、訓練データの中の「会話（text列）」の情報を用いてスコアを向上を目指せないかというアプローチをとりました。  \nそのうえで、「that、this、it」という指示代名詞の登場回数を特徴量とすることで、僅かながらスコアを**0.676(Baseline)から0.677へと向上**させることができました。  \n今回は、前回の「特定語を特徴量とすることによるスコア向上」をさらに深堀りして、Tf-Idfを用いて特定語を抽出し更にスコアを**0.678へと向上**させることができたというものです。  \n引き続き、スコア自体はそれほど高いものではありませんが、みなさんのモデルによるスコアを少しでも改善できるレシピとしてご覧いただければ幸いです。\n\n![image.png](attachment:19c22560-b4ce-4f40-9587-c08108925d38.png)\n","metadata":{},"attachments":{"19c22560-b4ce-4f40-9587-c08108925d38.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAnwAAAC7CAYAAAD/n0NzAAAgAElEQVR4nO3du4vjVv8/8Ld+PP/GBgLSFMZstZX0hVQp5GmcLQxbBMMGpNKGZZpgWAJmmyEglTY8C8NTBFzsurFVpApEqlIF42IkCGT/kPMr5IsuR7Ik23Pxvl8wkMweHR3p6PLRuY0ihBAgIiIioov1/x67AERERER0Xgz4iIiIiC4cAz4iIiKiC8eAj4iIiOjCMeAjIiIiunD/KftHRVEeqhxEREREdCalAR8A/CX+eohyEBER0Ym8Ul7x/U07r5RX7NIlIiIiunQM+IiIiIguHAM+IiIiogvHgI+IiIjowjHgIyIiIrpwDPiIiIiILhwDPnomfHxQ3uK36LHLQcc5VI9f8JvxCh+8xG/ct3jFuid6YD4+KK/wKvfzFm/t3+A3vR+j3/BW+QC/QbrTPwu+wLff4m3i+N7av+HLqbJ/YhjwEdGR8kHaSV19j9fW9/im9oZnLhfRxXuJd+Ff+Eskf37BT/gdA61C0HZqjZ8FMl/wm/EDBvgJv+yO7TN+wu/4oUpA+gwx4COiJ+2F+QY/T97AUB+7JEQEvIAx+QXv9E/4r/uwbWEnfRZ4/8OveIfPEwMv9nvYHdvAvryQjwEfERER1fAC/9d7ib/X/z52QRr7ch8B7W8Swd7WC7y5+4zPE+MRSnVeDPjo+Yp+w1vjN3zZjMPYji/5sPnq/OJ92I3NkI7LSGz/W3J770tBOuCL+2H333vJ7SX7KhqzIv198lhe4a0hGStz7HE32Y+bzNPf5+l9wCvlB/waAJ86m3Ewp/4yLjh/XxJlytXbwXIdqLPU8WfqvVZ9En0lvA94lXs2bsbdSZ8JB567MoX3WIX7OePFlQpM/5Dfr+oLSSBYZR9HPFcSeRx8NjfEgI+euX/wP+M9/uj+gr/EX/i8VBEN3+M39wPez7/bjM34jO9Xv+K9tPvhH/zP/h++ufkYj+EIv0fU+UE67utf9y3er7/FT3dvEg+DeBzI762f8Hk7BqT1O35I3sTq/+F7/RP+yOT5ZfE7/ra+g5HJ67+7vP7CL6N/CsbKHHPcdfYD/Gm/xx9XP+Kj+At/hS7U1WCfp/kz/hKf8U4HXi8342Ae4svY+4AfZt/ip+34omy9lZarQp0l5Oq9cn0SXaov+HP2N162mo6mi/Dfis/dKmWpcz/vmN/hNT5hUCmgqrKPI58riTyqPpvrYsBHz1vwCRh9xM9mfMu8MH/E9/rf+HX9HT7uxma8wJvRa3n3QxDh25uf92NC1Df4uHyNT+PsTfo7/lj/hI/Z8SObcSC/DPb7MgYf4baTgVbc/fFpnr5l/13/jdddI51X28XHwX5MyQvzZ7iWZKzMMcddaz+/4p/ufj9QDfx4dFfO3/hVk83+e7VrlTvEn3/Cy97/pertF+clovsKrQSV6mxLVu+y+oxfgKn6JLpE0Rf8Zr/Hr3iNnwb5drCqvq/03K1Aej//gnf4HX+WBnIGfhaf8a79OwbappWxaPZxlX0c/VxBvWdzAwz46HnT3+FHM/9r6Yt39a/kYaLim+wAYPM7vA4yD4sA+PYmn+c28Mg+9r5ppYOiF1dqZv8+/pi+xneJsvvzT9IvZqMrCdqOOO5a+0G6jKchm/m3nyX3Tj+cwzetl/h79mfquF4MPuJjhRdQ1ToDUFjvL66/x8tkd1D0J34PznGuiB6T5ONMe4/f8RM++z8f0Zpd8blbgfx+foE3/ke8OTi54wXeTD5uekm+x7fYBH+ZVrkq+zjFc6Xes7m+/xydA9HXQP8e/5d7eHzBvytA7UqG/V6pwOxffMHmS838Dq87/8Wf0Zv4AeH9gU/6t/gxk9ff0x/waijb/7f7vI7yUPs5rxeDX/Bu/R4/KL8C+ku87v2EH68NvDj4gK9RZ0BBvQNQ3+An6xX+8H6GYQII/2F3Ll2gl3gXVgmcHkvx/VzXC/MN3phv8GYSd6v+YACf/Td4UWkfp3iunP/ZzICP6EEY+M4a4L+LL3gzeIEv9xFe9n7M3byvl3/h5wdoJXqo/ZxP/GX+ZvIFX7x/8ef8v3g/HADO50qtfKdgdF9jMPfxs2nAn3/C6+7PD7JfIjqnF3hz9w6/a//gX+DBP37P+Wxmly5RVvQvqvUqvMA3bUjHjcmm/O+b5b/gzxnw/XXyX4vzOq2H2s9DeYEXpoE3k4/4GL4Dhv87MLi5Xp2VMr/D6+kf8CXd80RUU+XnblLz59kX77cDf7Ejwr9R1X2c4rly/mczAz76ym1v6r0vi9/xd1FXXobRfZ0bS1Y4g20bIER/4nfk85fnFS9r8NY93WIfD7Wf84mXLWj6FzRq1Vl5TvjO+oQ/7D/wid259LXTvsXLIG4V24vvq7zjnrtJsvG88WzXt/it5Bnx7/xX/Hpb8LwL/0mVpco+TvFcOfezmQEfffV+v/2wm5n1xfsN74d/4/XoTbWWHvNHvMOveL9bm+4LfPf9ZrZWNgcD31kR/l38A0gG9+bz2pYH+P76hOHEyfcTf5l+mvu5B9V5vMA3LeBT50NiRt0X+Le/ZsbRFZSrVp2VM7qv8Wn6ibNzidRvoOIT/tiupxf5+GC8xz/tl/m0uop/jnnuJrwY/BLfz27ieea+x6+Bim9KWt2NiYvX00F6XVFs1jHtfEqVpdI+TvFcOfM7gGP46Cun4qfJd/jDeIVBAAAv8Xr5ucYYihd4438G7Pf4QRkAAF5a7zaDffO+aQGDYYR3oexf47y+SeQF/fUZBk2ffj/GjYvX/QF+UADoxcd/Ki8GH/EZH/Bee4XNEeCl5eZWx5eXq16dldK+xcuzzGQmem4M/Lx8jbedH/AKAPTXcO8+4pvFW3xaZ5K2v8PPE+BD4+dukuR+1t/BDd8caHWPl2Xx7fd4rwywbYd8qb/G98vPeJMqS5V9nOK5ct53gCKEEIX/qCj4S/x1/F6InqLoN7zV/sFP4pjlBehr9sV9ix/WPz3MYtNENbxSXvH9TTuvlFds4SMiaiSKu1rehQz2iOjp4xg+IqI6vA+bBWh/x/dPeo0yIqI9dukSERFdGHbpUtIr5RVb+IiIiIguHQM+IiIiogvHgI+IiIjowjHgIyIiIrpwBydtEBEREdHzdnAdvpJ4kIiIiJ4gRVH4/qYdRVHYpUtERER06RjwEREREV04BnxEREREF44BHxEREdGFY8BHREREdOEY8BERERFduAYBnwdbUaBIfgzDhhedvpCn4cFWDLgVyhe5BpSKaYmIiIieuoYtfDqcUECI5E+IUW+FjnYBgdJVD5bVw9Vjl4OIiIiekAiubewbumwXdUKeyLNh7BrKDNipgCmCa8gb1BRFgWJ7NfNLO2GXrgpzcAdHDzBbPO+ITzUHmEwGMNXHLgkRERE9DRFcQ8OsNUK4behqzaAZFYM+z4bWAUbbBrNwBMz6cHdxnIqBn21Mi/fj6IDVNWvml3bwL23Uo+KqfdociYiIiB6dd4shHIQDE3F70Kaha6bh1htgYpZuDLszhbUU+8Yk1cTEL90ovd9U0vr5nXjSRoT7FdC+yjSNRR5cY98EqhSM9YvcTNOkdEBgBC/ZnGq4px83GLkwFBte9neGiwgRvEQ5DduTRPYPUEYiIiJ6MFEc4CAd4cQNXdN5QbPaljfHVHdwUyG+y+wV7ngKazRI77dBfqcL+CIPtqFh1l5molwPttbBrLdtAhUIR0BHywRUng1t1ko0Tfaw6mhId1nHzanj1j6vu9E6n9cZLew+5lc38IWACJdorzro5/rgH7eMREREdFrhOoDe0nK/11o6sLov7daVB4sVbFr3soFdk/waBnwBhlpmMGF/jtYohJ9t0/TmWOkO7nZNoIBq3sDRV7iPksmm0HvXiabJAe4cHatUolsM20v4qbwmWFpTjB9ipkgwxLrrY7ItpGripqcjWIdPp4xERER0YgU9mADUCmPZdsFi5MI2DvVk7nlzSetew/xON0v3rgvM+1CygxfNCXw/X1ggQDJO0lo6gtkita068OEP9lt686k0uja7VjroOhsL2TGTWY9fRiIiInpy1rewb4GbzcSMcNnDqtMvXtkkcjGelsQdNfM7XZeuamIw8bFsD6Flpw5Hm3FvhgHDUKAoGoZBZvPBHZz2DJqiQDEM2K6HKFXoOLoOhlp+qnJnerA59WE8hzISERHRQwtWLdxMBonevwHuHGB4Kx/wFS1mCPQW8k1IzfI7+V/aMLsWMJ3vx6tFLgxNw3jdwujuDnf+fopxmorBxIcQIcLRCK31GH1NgZEJVa2lbMqygJC2Ij6O51BGIiIiqiqenJEaZrYR3a8q5aD3rnMxgBpnKm0MCteBdJum+Z39T6t5t0PACeFPBjBVtULAo0I149ZCP3SA4e0meCw+2U/HcygjERER1aW1dOnQrEOBWdm2xTzMp/Ixg83yO0PAF92vAKuLuMu5aJBjiHWqSzdexkSyiHSK2bVy4/yA+E+hGUUrDT6w51BGIiIiqkfeelY8mSO17XUPerL3c8OLo7p8sBjdYwUdkikBzfLDSQO+CJ5rQBsCzm7+8GZ9mvF+IkfkubCNe7SsVNGhtYBpJ7k+XwTvdohgFzwCMG/gYIh+Yu27yHPRHwK969qL25zHcygjERER1bN9v7vb93sEz+7nlk2JXAOKkvkzs+oAI2uKTio2sNGZ6omYKSFcI0AbhXFk3fxwymVZFA3jWRvL0EdiYi3MSbifjKEo6M+BG3+AK6S7PtWBj9ABxloiPywRppZ5UTHwQ4ww3uWnjdfoZfZZr9zbxaDr/T28YseWkYiIiJ6e+P3eW2/f7xrG6CHMjc9vQ9fzS7WYkxDL1hj9XWyAXMy0Fd2vgJIJG3XzAwBFCCGKMlMUBSX/TERERE8Q39+UpCjK+SdtEBEREdHjYsBHREREdOEY8BERERFdOAZ8RERERBeOAR8RERHRhWPAR0RERHThGPARERERXbj/HEqgKMpDlIOIiIhOiO9vSjoY8HHhRiIioueFCy9TEhdeJiIiIvoKMOAjIiIiunAM+IiIiIguHAM+IiIiogvHgI+IiIjowjHgIyIiIrpwRwZ8HmxFgWK4iGpuGbkGFNs7STrPVvZpPBuKYsPblc+AKylc5BpQFAXSrD270TE1UfU8XKLiepOL60xen4/Hg20oUBreB0RERA/huIDPm2NqWbCCNcITFei0NLT0AOtc4SIsZoBl6Vjd51/R3nwKvXcN9UHK+NxFcI2CwPnUrnqwrB6uGm18nnJ6dgfT9hKhEBD+4EzXzAOeYyIiKhHBteMGI0VRYNj1PvQjz4ax2VZRDNgFLRjV0qXLohh2aYPIUQGfN5/C6k7QtaaYP+LLSGvp0Fva9n+g6y3E/6fiqg1JUBdiHbTR7bYR5KLBCPcroH3FcO/ciutNTjUHmEwGMJ9Y1egtjR8HREQXL4JraJi1RvFHvggxas2gVe3d8WxoHWAUCgghIMIRMOvDzcZPldJtyoJtWQTCETDUinvBjgj4PMynFromYHYtTB8z4iuhtXQEs0W6MqJ7rPQWNK0FfTrPdCOGWAc6WmWRBxEREX1dvFsM4eBuYG4+8lWYgzs4GOL2YAjkwe5MYS0n+0YL1cTE9zEwG6TblmViYp9sgqUVYLaQR3zNAz5vjqnVhQkAZhdWLnBKH4BrGPumSa8oFq6aLm/XIqdeod2+2p+A6x70TJdztJghaF9BVa/R01dINQBG91jpPVznmmwqNONGLoxNpB+59u6/T3F81ZqRK6QpGJ+YG0u4O5YInrtvWjZsb7+tZ0NRNAwDYNrZNCkn8ojcTJO05HiL6k1+ClwY2XF+JyhnoyZ6z4aiKOhMgWCoxdulPqsieMk8DRfS6o6S10TcJJ9KV1b22nVZdF1WK2uV+iQiulRR3P2XeU/FPYkHG728Oaa6gxuzPFnVdN79qnDoWb7nMtY44Iu7c7cl0tDSi7p1I7hGB7PettnxDt37W9yum6bLUwc+JruTY2IySZwp9QptpIO6cB1syq7iqp0e47cLBnNlq96MG7oG+usWRnfJMV3Nj6/a/o9sai6wsPuYX93AFwIiXKK96qC/DWzMCYQI4eiAtdw0PW/PvWdDm7USTdI9rDpaahxaab09VDmbnjdzAiEElhagOyGEEPAHairPcWvf1H43WqOjZSeleLC15DURN8mn0pWWvZ7i67JCWSvUJxHRJQvXwX4YUoLW0oHVfek7Qx4sNk9nDvzEO2e3Ne5XSMRmGaJE8T8vhQVdOOH+N6GjC1hLSVJLQHdEmPs10umrpqstFI4Osc8iFI6+L3u23EsLQncypSgpm54+CULXdWEdcx5kpNumj6NyGQvS5eovdISO5HkrSJc7vwX73WybO7d1hI7QYYll7nfNy1n5vBUovF4kdSqrC/1QvTYou7Quy67LCmU9S30S0dkceL1TbQXPYSEKn8XpJJtnaOgIS4cAIABdWMuwUTppCR29sBwARLMWPkm3p3rdk4yHK57xqrX0RunqU3HdS8zGjRaYYV929boHfReZR7hf6ehl+nPLypZrOg2AlqQt9pjjk2+rYuD72Ab4tcpYWTxGswnZ2ElV+kVyCs3LeY7z5s2n0q9As2ul8zQn8KUze2Uzy49Udl1WKOvD1icR0QVa38K+BW78TY/OsodVp5+fZFE1XULkGtCGbSxLVotoFPBFixmCYAhtN55HgaINEWS6TqvPeD3vzFj1qr1/WYXrdJeteoX2boxfPHs3XYzisqnxFOB0M27B+L/mx1dl25plfADq4A5OexZfI4YB2/UQPbkhX+c4b3Ge23F9qZ/ONJ9ntBl7aBgwDGU3Xu/kSq7LKmV9HvVJRPR0BasWbiaDxCSLAe4cYJiZ8VE13VbkGtBmbSzDCcraPhoEfPEads52LE/iJ3RQODvkUWmt3cSN6H6VadHQ0NpO3PDmmB5YGoSqUjGY+BAiRDgaobUeo69lJzZcrt14u+xP8usrcmFoGsbrFkZ3d7jzxW683pMr61den0T0tSta5m0z7q4CWW+SrHGhajpgG+z1EPqTg0uW1Q/4ogVmuVawfYHS3T7FJyizZcV0DSVm44ZrZLps44kbs0UUB4OSrtPSSq4wuPK446uy7SnKeC4qVNPEYOLDDx1geFv61zQe1jnOW/W69m6HgBPCnwxgquoj1FGT6/Ip1ycR0fkUDfUJ18HBP9ZQdZhQreFEnn2wGzepdsAXLWYItsuxZJldWMEMyUY+s2vl18FDfILSm1ZL18x2Nm6E+1U+WN2OVwrXgbR7T162CIuZfMaOzDHHJ11LEBFcw9gtxFi5jInWzmy604mX+XgOMzhPUbfV8oy/xIzdyplF3ckhKl/yJ6jLqmV9LvVJRHQu8la2akO2yuY5ILuUXIV08Xp9KzgHunFTymZ85P+5ZJZKdoZJZhtYy83MkVAsHUvounwW5eF0zYSOLnRLNitSbGYxWsLKzY5Ml01PlU0yGyZ05Pkn8mh2fJv9O8td3qGjC6RmrFYso1gKC9jP+AmXwtJ1YVmyWbqZGbFCNvt1P9M4zKaDJfYTi8LjZ1wXztJtXs7q501OPps3m6cQ4dIRemZm+9JCaj/h0hGW7gjHks96zpe9Rl0euC4PlfUs9UlEZ3Pg9U6NZN/FoVha+fdF/LzMxxPZ53i4tBqnq7tKAgBRL+AreLmmk8helqFwLD01vVi+jEvVdA1slu+Qn6D4xVn+kk+WDUK3igLHqnnUPb7M/nVH5GdpVyijEPFyIJs00OOXuHxZlmqBVBxobPNLBDBOYj9IBxWNHBnwFZWz8nmTKF6+ZfMgSJznfDL5fqV5FpW9al0euC4Pl/UM9UlEZ8OA71wOvy9CxxJ6wTN/6ei5Z7ZsH+XpNg1IKPiRxBQAhLL5DylFUVDyz0RERPQE8f1NSYqiHPO3dImIiIjoOWDAR0RERHThGPARERERXTgGfEREREQXjgEfERER0YVjwEdERER04RjwEREREV24/xxKoCjKQ5SDiIiITojvb0o6GPBx4UYiIqLnhQsvUxIXXiYiIiL6CjDgIyIiIrpwDPiIiIiILhwDPiIiIqILx4CPiIiI6MIx4CMiIiK6cEcGfB5sRYFiuIhqbhm5BhTbO1k6kvFgGwqUhnWUy0sx4EoyiVwDiqJAWk2e3XDfxfvb7BWuUbBPIiIiSjku4PPmmFoWrGCN8EQFujyPF5h4dgfT9hKhEBD+AGpRwsiFoWwCw9RPMuDS0NIDrHMVHWExAyxLx+o+H5158yn03rVk3/F5OT4QJSIieigRXNvYvScNu947LPLsxPvWgJ1q1di8F6XvY0Xa+FWeX9pRAZ83n8LqTtC1ppizpeVJ0ltacaCXSujEgWHqx8dgt7GKqzYkQV2IddBGt9tGkIsGI9yvgPaVmvqd59owlD7QsxoeFRER0UOL4BoaZq3R5n0ZYtSaQavacOHZ0DrAKNy8Y8MRMOvD3cVPKgZ+9j0c78fRAatr1swv7YiAz8N8aqFrAmbXwpQR38XTWjqC2SJ9YUf3WOktaFoL+nSO9FUQYh3oaGnp383XLYxCH4OrByg0ERHRKXi3GMLB3cDcNKSoMAd3cDDE7cEQyIPdmcJaTmBu20BUExPfx8As3XC335tUuvr5NQ/4vDmmVhcmAJhdWLmXfbpgrmHsmxy9oli4ajqZKs2sFdIUjDnLjSWMXBiGi2jXYrXN09tv69lQFA3DAJh28k2ykZtpiq18vAeOw7OhKAo6UyAYanGakmZehGsE7auDLYHqdQ96pvs+WszibdVr9PQVUg2A0T1Weg/XqYxNTCaD/QVKRET0DERxl1XmXRn3fh1s9PLmmOrZoK3SXuGOp7BGmWFZDfJrHPDF3bnbPWlo6UXduhFco4NZb9sEeofu/S1u103TFe3jUDPrkU2xBRZ2H/OrG/hCQIRLtFcd9LfBlTnZN8UuN02uk80582xos1aiKbaHVUerMNavwnGYEwghsLQA3QkhhIA/OEGEpV6hjXRQF66DzXWg4qqdHuO3CwaP3zMREdGjCtcB9HSXFYC49wur+9JYQh4sViBt3WuWX8OAz8N8muyqU3Hd0+URrrQJdIIugmbppMUpamadYRGVpfGxbA/3AVpdwRDrro/JtrlKNXHT0yVj2SRF3kxm2DfFDnDnyCc+pDc8w3EAAO5TrYaKYSPf4JiduBHhfrW/DrRW+hooujmqCzDUigawxi2nRERE5ycbkx5Tr9oHt969DyN3v3pGhZ49by5p3WuYX7OAT9JVp173JGO4imdpai29UToZ+bYqBv5+0kFZ/lUCNLl4DGMTsvFw6sA/2BJ3nuMAMF3j6sbfDRBd9lboaHamPuPAfheURgvMsL8O1Ose9N1XThwM9q6Pad/T4YSyAaz7QaxERETPwvoW9i1ws5mYES57WHX6xcuPRS7G05I4o2Z+jQK+aDFDEAyhJVtctCGCTHdfWUScybFiuqbbHojMDzTFnoM6uIPTnsXn0DBgux6ig4U403GYEwiRGPi5bSHVpxhnrhz1qr0PVLNj/9QrtHdj/OLZu42qlIiI6MIEqxZuJvvWOtUc4M4BhgUzPqLFDIHeQlE/Wd38GgR88bprspaX0AFmi4cOnZ4rFYOJDyFChKMRWusx+tqByRUPKm7Ny7Uaaq3dxI3ofpXpstXQ2k7c8OaYllyoREREz0fR0mSb8XQVyHrnyhprwnVQsI5ts/zqB3zRArOClptU60/8m8ITlNmyYrqm2x6oqEedWKBCNU0MJj780AGGtyWznZ/AcSRm44ZrZLps44kbs0UUB4MlFyoREdFzUjR06lBgVrZtMQ/zaXHvZZNhXLUDvmgxQ7BdjiXL7MIKEhMlEK/Rl1u7DfEJSm9aLZ2MdH04RHANY7cAoTz/CItZZmJBogUrm+50Ini20eivb1Q+jho815C0LMZ55hZ63M3GjXC/ygf+ZtdCsA4RroOGXfRERERPj7z1rNqQtLJ5DtLGmugeK2TXsT0iP9QO+IqCgC0TXStId+uaN3AwhLZbny5et268yoy4r5pOQt0sfNh392vgRW4fw6CNq21RN/n3U/n3NzNek7NPrtDGFPPtTJfIg230sW43nSGwX6Mn2fKptYBpJzkTNoJ3OywOpreqHkcN5lUbwbCfmN0TwbP70qngQBxgr+a3mEHSZau1oK/mmK+KL1QiIqJnZ/v+dRPvX8m7Mv778pm/Ba8OMLKm6CTW6o08G52pDkf2og3XCFAyDr5ufqgb8EULzILyman51jYVAz+Eg/Fmkke8bt1dL1f6iulk4m176/FuIkl/1sMynCSCp02aXf4axusewtzfmDUxWVpYdeIFi5X+HN07HzetKuWQM2+WsFadzQSNeL08deAjdICxtl9mZIwlwsmhab9Vj6NOAScQYQ8Ya4myFOepXveA6RSQNWGrV2gHU0yRXXB5K/G3AuOVoTfHYRTPVCIiInp02Vij6F3Zhq7nl2oxJyGWrTH6mzhFGwPLMPknTPei+xVwYBx8nfwAQBFCiKLMFEVByT8TERHRE8T3NyUpinLM39IlIiIioueAAR8RERHRhWPAR0RERHThGPARERERXTgGfEREREQXjgEfERER0YVjwEdERER04f5zKIGiKA9RDiIiIjohvr8p6WDAx4UbiYiInhcuvExJXHiZiIiI6CvAgI+IiIjowjHgIyIiIrpwDPiIiIiILhwDPiKiDM+24UWPXYqvVOTBtr3HLgXRxWHAR0SUELkGxq0bmOpjl+QrpZq4ac3BmI/otI4M+DzYigLFcFH3YzhyDSgV7uiq6UjGg20oUA7W0aYecz8GDNtDxJaOs4tcA4piwOW5flyRi/6sh7tBhWgvcmEoNp7K0+lBr6GKx960TOrgBq3x0zm3RJfguIDPm2NqWbCCNcITFejyRHAN5VG+Vj27g2l7iVAICH+A8leYDicUECLxE95hhDE0zfjKvrbPWWcFeV/1YFk9XJ1jl1SZdztDe5S9Vx7qHj5yP9Jr6Niyn6NMVT7KzG4AABXgSURBVKgYjIDO1/Xgoecg8mAbxr5hxDBgVx7/EcFzbRiJhhXDTjbGFDW+7H/iW6JqurSDCy+X8eZTWF2BLhTMvQlM85jc6Bz0lnYg0CuhqjAnPsKuDa1joBX6qNLwQfWp5gAT3j+PK3IxXvVwN3nsgjTzFK+ho8pkdmF15vAmJp7YYdFXy4OtdTAFAOjQ9QBBEGDa0YClOHite7aGTrwxdF1HEAQIpkNoKyDcNsroOvTclgGCIPOrqukSjgj4PMynFroTwISFztzDhBHfRVLNCZaWgs6th8FTe6MQnUi0mAG9u+YfSHRiJrpWh40J9GRE7jgO9nRnF6BFrgFtGGA6dnFjlvSkefYm2LOwFJP4IyZyYfRnaPe2beAmJr7kYvdsKJ0pYC03QWXVdBmiROk/Ly0Ba7n9H2HBEsvixMLRdQFAALqwlqEIHT2xfd10MqFwrO22ELrliLBJmqUloOd/nytH6Ahdd0QoQrF0LKHv8lzut11au33tfhJ5hInttsdbzYHjkOxXd8ryXgoLuihPIjsvoVgmy6E7IncIyfNkJep2s7NwmTx3sjqrcLzZ/RTVxy5pyXkvq7PdPjZ5ZM9HmLx+IaBb6fNxKO/kPZT9/9LfV6gH6WndnzPH0ouvw0PHfeL6efD7dJclRP6RVK3OwkPHdMy1UVX22jg2z1Mce8F1XPnZl3rPUB0HXu9UWygcPb5m05fkUlib67jsHbq0qryLpVtWyv9QOgCi8Ri+uDt3G0JqaOlTzKXDLSK4Rgez3igeSybu0L2/xe26abqifWiYtbbbhhi1ZtBSExWqpKlvYfcxv7qBLwREuER71UF/O0LZnECIEI4OWMvNuLht2O3Z0GYtjLbj5sIeVh2twliZCsdhTiCEwNICdCeEEAL+sX2xWgt6aqxmXI7xrhwCd6M1OppsoPUat0Yf8+4dhBAIl22shn24ro3+vIu7zXH0VsP9uatzvAml9QEcPu9ldbYRugb66xZGd8mvubipf3/9CoQjpM9Hhbx31Gv0JPdUtJghsLqJLq469SCzxq19i6sb/+B1KD/uE9fPY9ynm/3er3S0tMyGFepsbhu4Lc37xNdGVcfmeZJjl6j77FvdH1X3RKcRYh0AgOQ5AQAIsC6czBDhfhX/V/uq3rt416pojUqHVFVKVxYvFv9zvkWosCWu4Et8aWW+NqumkxZH3vrk6IkyluSfirjrtBzkIn3ZeYi/CrLpcvvdbHsw+q96HAW/K8j0cAtf6AgdmfMpqZfcPqXnafOllD8p8t9VOd6K9VHtvMvrLG4t0oVVcJ1LW7707HktyTvTEiK7p3KtUFXrQSZbp8k8k8dy4LhPWT+Pe58W3QNldVbtmI66NqoqaP09Ks9jj11SplrPvkTrMtVz4PVOdW2u+XwL2r7lr/iZu299sxxHWHqi962wZyu93TGte0Ic08IX3WOl93CdiCLV6x706TzXquDNp9B717l+ba2lN0onI99WxcDfTzIoyz8oDssPsNBt+BGutXQEs0Xqy1Ud+Adb4s5zHPV58yl0yWeO2bXy5dAd3EjOkyU7eZmv+XrHe7g+mp73nQBoyQ7GnMCXzoQu++orp161M+djM242sfta9SDVRu6D0+zCCmZYJE9SwXGfun6e2n16krzPcG08Dc3O69H3INGzFWA6HGIa6ND1OLYJpsPCHoz9mMF0vNU0XaOAL1rMEARDaMlpwNoQAVa4T5U6bsY83IRZNV3TbYvT5F+qD0Md3MFpz+JzaBiw3Srr3T32cWyDg7gcwVDLTwfvTE9YjtMfb7PznlB2Q0WbKfeGAcNQoCgahiUzpg7KBl7eHFO9hX1491D1gILjPnX9PPb1fUanvjaesVr3YLjGV3qa6FJZS4TCh+/7EGIJCwCCIW5zQxoiLGbx1S/7CK6frlHAF2ExQ37NNiEQOsBs8WwfyQ9MxWDiQ4gQ4WiE1nqMvqbAeKIr70aLGYJUsJEY15P9Objm32M603mPXBiahvG6hdHdHe58sRv/1JyJrhXs7qnofiW9oZ9nPTw1Kq7aZ2pxO8u18ZzVvAfbV7yO6fGpV2hL/2E7tq9Ko5UO58ZMXM8atp2Yq/vM9R8tMNuMGeyVNu9VTIcmAV+0wCyQdAMh/gpPN9WriD/MD71Mq6Zrum1xmihuUnjEB4oK1TQxmPjwQwcY3pYMtn+s44i/IPbBxjH1Vcc5j7fOeT/Mux0CTgh/MoCpqierh33XbPyhlb6hz1QP0T1WlRKeun4e9z7VWjqm8plnRznXtfH8Hb4H05MDiR7TNjjLfBjunpdFkzlKti0RLWab1m15vFU3HdAg4MvPEkyQjP0xu1ZuvAYAhOsgs2m1dDKyMSHxbD8DrleW/yaQSdZSbjbqPt3pRPDsZn+9ovJxnFDk9jEMLIwSY2yK6ityDRju6V6apz3e5uf9UL7yrsj9l19jZhfWdA4vWmCGfLfq8fWQHYaxbc0tHwtSvv/m1+Nj3qfqdQ/6ybuNz3htPEt17sH8mFWixxN/kALAdLwfc7cLuBLPzMjz4HkevN1f4FBx3Yub8qZzb/+MKWmd28U+mZ61rKrpgNoBX/xALf7iSndBxb+6gYMhNHt7kPFYlvEq059RNZ2EOriDgyH67v5ExkFKG1fbom7y76fy72MIJ/13M9UrtDHFfFtRkQfb6GPdbtr/El8kqUqGCq0FTDs29n+RJYJ3OywOpreqHscJRJEH1zagzdpYhpN0uXLlACLPRX8I9K5P+IQ+6fFWPe+yOivP96qdeQh4LmzjHi2rIG3lvE10rRXuF2tANj7jBPUwu92fj3jbAFbuz4sVFe/E1+Oj3acA1AFG7ZlkLE3dOpNse8S1cdzfyD02z2OOXZ5f5WefN8fq0POQ6AGZk/2YO02Jx+Nqm8G4+2dmhMW4g06ng854//GqDkawdADTzmZbYzP3AZKlVPbLuJT3bFRNF6sX8EULzILyL658a5uKgR/CwXgzySNeD+uul92yajqZeNveerybSNKf9TJByibNLn8N43Vv/+dMdkxMlhZWnc1A+P4c3TsfN60q5ZAzb5awVp3NIOX4wa8OfIQOMNa2A+01jLFEeHCdrKrHUVeAoZYe+K/1x1i3Rgj9Ccxc5nE5Rtifc228Ru/kf37ttMdb9bzL6qyMOQn3A9EVBf05cOMPcIV8F2XdvLUWMByuCsZnHFsPbYwmXcz7m207M7SXYY0/h3Xq6/Hx7lMAMG96WI3zdVK3zlLbHn1tyEcOVT+m4/I85thlqt2DEdwxMOJf9qEnxcQkXMLSdez+jJluwan0zDQx8UM41nbbAIAOy5G99/c9AOU9JVXTxZTN+izyf1QUlPwzET1nkQtDW2MkJmxFSYhcA33cPaFlQiK4xgLXJ52Ec448TydyDdxe+U/ubwM/J3x/U5KiKM2WZSEiulTqwMdofZvocnxk3i3WB5ZbeBJ5nkrk4nY9YrBHdGJs4SP6WrGFj+hi8f1NSYqiMOAjIiK6NHx/UxK7dImIiIi+Agz4iIiIiC4cAz4iIiKiC8eAj4iIiOjC/edQAkVRHqIcREREdEJ8f1PSwYCPs3yIiIieF87SpSTO0iUiIiL6CjDgIyIiIrpwDPiIiIiILhwDPiIiIqILx4CPiIiI6MIx4CMiIiK6cEcGfB5sRYFiuIhqbhm5BhTbO1k6kvFgGwqUhnVEGZELQ7Fx6GqMXAOKYsBtcsIr7uMsIheGsrleSn6M1IE1ucY82BXPj/z+f77XNZ9nRPRYjgv4vDmmlgUrWCM8UYEuTwTXUPAYz3jP7mDaXiIUAsIfQH34InydrnqwrB6uHrscdakD+EJA7H6WsKDDCZO/E/AH+yvpMa4xXtdE9GgiD7Zh7D+CDQO2V/WzM4Ln2qkPa8NOfrRuGtFKfpKxROS5+49fRYGiGLBdr/Aj+ODCy2W8+RRWV6ALBXNvAtM8Jjc6B72l8YX4wFRzgMlXdC88xjXG65qIHp4HW+tgCgDQoesBgiDAtKMBS3Hwue/ZGjrxxtB1HUEQIJgOoa2AcPvxquvQc1sGCIJcZtB2mVmw2itMpwGmww6m6yWEpDBHtPB5mE8tdE3A7FqYztlNQU9BBM82oDxWt2gtz6msRERft8gdx8Ge7iAUPnxfIHTi8Gw6PjC8xLM3wZ6FpRDwfR8idKDrOqzetj/IxMT34Wd/Rlb8z9ZyE1RGcMfTza8EhD/BZOJvemUATMfSITPNAz5vjqnVhQkAZhfWdF7y0vLg7ppAy5o/q6aTieDaRkEzaY00ni0dF5QbexO5MAwXUaaJ1rATzameDUXRMAyAaWfT5JrII0o17dZrFi49Ds+GoijoTIFgqEnGXcmyTJ57BYphI1WcorFl0t9vA5lN+QwXuUPbnb/Necie80PlSeSTbNI2jD7mrTsIMUH6+6ZCmWqIiup8e2wF56paWSvso+oxHTrPTTW5xg5nWn7/19lnwXXpGrJxf7IxhRWeFQfPbbXnWfPnABE9nAiLWdzMZo32Q0nUwSgOsoIZFiW3rjfftAs6N/vnvTqA7/uYDMySHgsPdmcKQIdzs90yxDoAAB0tLZlWQ0sHgABryTi7xgFf3J273bmGlj6FvJEvgmt0MOuN4jE34g7d+1vcrpumK9qHhllru22IUWsGLfUArpKmvoXdx/zqJh77FC7RXnXQ3745zAmECOHomyhciH0zq2dDm7Uw2o6PCntYdbQKY/0qHIc5gRACSwvQnTA37iovbqben3uBcAR0tMQLU71GT1LH0WKGYBv4J8o3bu3zuhut03klhK6B/rqF0V1yLFaF8gDxC1eboTUKN+PLQozaAaa5K71emQ6Z2wZui+q8SOWyVt3HKc7zEWpfY4dUuP/r7FN2vUYLzGDBQmbMcXSPld7D9S6res8K+bmt+Dxr/BwgoodVFGRtyYOsWIT7Vfxf7at6z8ldq6I1wv5xZ6IbR5kYatuPxAie28cwAIC49zVHlCj+56WwoAsn3P8mdHQBaylJagnojghzv0Y6fdV00uLItg2FoyfKWJK/njyQgnS54wsdoQMiW7T8eQiFo+fT5fa72Tb7u5yqx1Hwu6I89UPnT8jreGlljm1pSesrV5bQEbquC6vgmmlanvjatETqt1XLdEjVOg8doWfKULmsVfdxivN8UP4+P7i/pvnWuP+r7lN2b+tOmLtmpee2yj124BqucjyNnwNEBxx4vVNdm2czcs/E+B0PlD2XlsLabGs5jrA26QEI3co/J2Tb5bNepvLZ/eiWWEoyBCCatfDlvogB9boHXdKt682n0HvXuVYFraU3Sicj31bFwPd3EXFZ/kFxWH5AQRRdgdbSEcwWqRYDdeAfbCU5y3GYE/jS2Y7pLxb1qg2s7lMzirbjOFPlk3z+mF0rX74AaN1ITmDF8lRVq0wHNa/zU+7jJOf5CTnm/i+SPRfR/QrtKxVaS8fqfn8Vh+sg0VtR8x4rOLdVj6fpc4CInqMA0+EQ00CHrsfPgmA6LOw92I8ZTMdbMROTuyWszCNSb7egFTw+GgV80WKGIBhCS04X1oYIsMJ9ZhzM/apKE2bVdE23LU6TD2Iehjq4g9OexefQ2EylPliIMx5HtBmLaBgwDGU39jDF7MJKjlPw5pjqLezDjrh82/FVqZ/ONF8+6UVcvTzxR8YY7m7MUxQv2ZHpYq5apnj9vOIp8MeoVtaqTniez6D+eTzm/i+htRIfoREWM6ClxXWBXZCV/WipeY9Jz23142n2HCCiZ8tabiZ8JCZZBEPc5gd978YMyj4e4dkwtMQSVeESjqXHAWTBRMAGAV/84MyuzSWEQOgAs7JRi5SgYjDxIUSIcDRCaz1GXzvFwPcGIheGpmG8bmF0d4c7X+zGHqaZ6FrBro6j+5X0QtyNV8z+VF0zrWp51AFGVoDZeBv4aBhjiVAyHb1KmdSBn/v3ky2vUqOsVR19ns/krOexVkGu0dO3H6Eh1tgEZ+oV2tu1Q6N7rFIfLQ9eyKfzHCCiYuoV2tJ/2I7tq/KRF0+82KfaTrJAqtcBQDzmeDNmsJf9qoxcGJ0pAlhYTjb5qSYGEx/LeJouxpJnSP2AL1pgFrQhOy71qp3pnlARfxQfenhVTdd02+I0Ufwp/ogvSBWqGVeUHzrA8LZkIsF5jsO7HQJOCH8ygKmqpXnsu8niwD99IR5Tj/XLE7kGxq0Qvp9YFHiSne10mjIdq1pZq3oax3Q65zoeFXFjXhS3Ru/uDxNdK57QEcUXcaIeTnGPNTmeOs8BInp4BTNgo3vE8zGKJnOUbFsiWswQx5GSeCtcx/8m+VjdDhuRDVeqHfDlZ2UmZLv8sAkQMmNU4vIGmU2rpZORjYOJZ8kZcL2y/OMm09RYKK0FPfeXQ/ZNq6cRL6fRpLuw8nHUKIu8+2n/1ZIpQLwET7TADPnurKJ6jFwDhlvlgKuXJ1wHCGYLeF6EKIoKu7OPL9Pxqpa1qqdwTKd0zP1fRr1qI1iHiO5XqXF623F84Tr70XKae6za8TR/DhDRQ4s/5ID0mnu7wCwxvCPyPHieB283hEfFdW+zXt88scRWSSve7lkh64HQWvHizMEs0xWc6AaWPKtqBnxxZlbhiPJ0l1/8qxs4GELbrSMWj80arzL9c1XTSaiDOzgYop/4kyKR28cwaONqW9RN/v1U/n0M4eAuOUBavUIbU8y3FRV5sI0+1u2mg8fjiyRVyVChtYBpJ7m2XATvdlgcTG9VPY665UtewJ4L27hHy5IWAF1rhfvFOtMyUlS+OL/+EOhdV+nXq14ecxLCac/QGffR7/fRT6zZl2rNPrpMx6tc1soZPv4xndQR9395vl1Yqzlu10h9favXPWB9i/lK8vV8inus0vEc8RwgogdnTvZj7jQlHl+uDbNr80VYjDvodDrojPcffepgFE+wmHY22xqbuQ/ILLkS57FdxkXaq6AOMNosyzLtxO8R2zb2Y911CyPZs6pwNrCQTOuWLDeRFTq6fIkUS99MG9aFtQwLlqmomk6658S2ELruSKYmZ9IUTYdeWpvp1/spzlWW3tgdf25NjcT06cS5CZ3EfgChW8uS6dn1j6P6khny/Iq2Dx29YJr4Pr9lIj/oVj5t6EiWXqlXnqWl55YvEUKIcGkJ5OqmQpkOqVrnknSVy1rnujr6PB/ygMuyCCGq3v9197m0IHkmbZY7KHy2VLjHDp7basfT/DlAVOzA652aCpfC0jPP3VSwsV+mpTwW2i7TIrvft8uxlD/rwqUl9NTSLHrh8wOAUDb/IaUoCkr+mejxRC4MbY2R9K9UeLCVMVqhjyexusVzKisRXQS+vylJURQw4KNnavPXENpL3CUnP0QR3Ns+hhhJ/3j043hOZSWiS8D3NyUpinLE39IlelQqBn6IHsb78XCKAqXfx7o1Omq5k9N7TmUlIqJLxBY+IiKiC8P3NyWxhY+IiIjoK8CAj4iIiOjCMeAjIiIiunAM+IiIiIgu3H8OJVAU5SHKQURERCfE9zcl/X+eymfVHLv5zAAAAABJRU5ErkJggg=="}}},{"cell_type":"code","source":"import os\nimport gc\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.metrics import f1_score\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import KFold, GroupKFold\nimport lightgbm as lgbm\nfrom lightgbm.sklearn import LGBMRegressor\npd.set_option(\"display.max_columns\", None)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-06T23:16:32.243104Z","iopub.execute_input":"2023-03-06T23:16:32.243930Z","iopub.status.idle":"2023-03-06T23:16:35.439759Z","shell.execute_reply.started":"2023-03-06T23:16:32.243888Z","shell.execute_reply":"2023-03-06T23:16:35.438394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv')\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-06T23:16:35.442679Z","iopub.execute_input":"2023-03-06T23:16:35.443053Z","iopub.status.idle":"2023-03-06T23:17:37.048864Z","shell.execute_reply.started":"2023-03-06T23:16:35.443008Z","shell.execute_reply":"2023-03-06T23:17:37.047822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_df = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\ntarget_df['session'] = target_df.session_id.apply(lambda x: int(x.split('_')[0]) )\ntarget_df['q'] = target_df.session_id.apply(lambda x: int(x.split('_')[-1][1:]) )","metadata":{"execution":{"iopub.status.busy":"2023-03-06T23:17:37.050062Z","iopub.execute_input":"2023-03-06T23:17:37.050474Z","iopub.status.idle":"2023-03-06T23:17:37.647254Z","shell.execute_reply.started":"2023-03-06T23:17:37.050422Z","shell.execute_reply":"2023-03-06T23:17:37.646275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Check Tf-Idf of 'text' in previous notebook\n\nFirst, let's check Tf-Idf of the dialogue (column 'text') words grouped by 'session_id' and 'level' in my previous notebook.\n\nまずは、前回Notebookで「セッション」「レベル」毎に集約した会話（text列）のTf-Idfを調べてみます。","metadata":{}},{"cell_type":"code","source":"CATS   = ['event_name', 'fqid', 'room_fqid', 'text']\nNUMS   = ['elapsed_time','level','page','room_coor_x', 'room_coor_y', \n        'screen_coor_x', 'screen_coor_y', 'hover_duration']\nEVENTS = ['navigate_click','person_click','cutscene_click','object_click',\n          'map_hover','notification_click','map_click','observation_click', 'checkpoint']\nDIALOGS = ['that', 'this', 'it']","metadata":{"execution":{"iopub.status.busy":"2023-03-06T23:17:37.649639Z","iopub.execute_input":"2023-03-06T23:17:37.650161Z","iopub.status.idle":"2023-03-06T23:17:37.655058Z","shell.execute_reply.started":"2023-03-06T23:17:37.650127Z","shell.execute_reply":"2023-03-06T23:17:37.654297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineer_before(train):\n    dfs = []\n           \n    for c in CATS:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('nunique')\n        tmp.name = tmp.name + '_nunique'\n        dfs.append(tmp)\n    for c in NUMS:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('mean')\n        tmp.name = tmp.name + '_mean'\n        dfs.append(tmp)    \n    for c in EVENTS: \n        train[c] = (train.event_name == c).astype('int8')\n    for c in EVENTS + ['elapsed_time']:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('sum')\n        tmp.name = tmp.name + '_sum'\n        dfs.append(tmp)\n    \n    for c in DIALOGS:\n        tmp = train.groupby(['session_id','level_group'])['text'].apply(lambda x: (x.str.count(c)).sum())\n        tmp.name = c\n        dfs.append(tmp)\n    \n    train['text'] = train['text'].astype(str).apply(lambda x : '' if x == 'nan' else x+\" \")\n    tmp = train.groupby(['session_id','level_group'])['text'].agg('sum')\n    tmp.name = \"words\"\n    dfs.append(tmp)\n        \n    df = pd.concat(dfs,axis=1)\n    df = df.fillna(-1)    \n    df = df.reset_index()\n    df = df.set_index('session_id')\n        \n    return df","metadata":{"execution":{"iopub.status.busy":"2023-03-06T23:17:37.656224Z","iopub.execute_input":"2023-03-06T23:17:37.656743Z","iopub.status.idle":"2023-03-06T23:17:37.668224Z","shell.execute_reply.started":"2023-03-06T23:17:37.656710Z","shell.execute_reply":"2023-03-06T23:17:37.667423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf = feature_engineer_before(train_df)\ndf","metadata":{"execution":{"iopub.status.busy":"2023-03-06T23:17:37.669421Z","iopub.execute_input":"2023-03-06T23:17:37.669949Z","iopub.status.idle":"2023-03-06T23:19:41.229921Z","shell.execute_reply.started":"2023-03-06T23:17:37.669917Z","shell.execute_reply":"2023-03-06T23:19:41.228795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_index = df.index\ndf_index\ntfvec = TfidfVectorizer()\ntfv   = tfvec.fit_transform(df['words']).toarray()\nfeature_names = np.array(tfvec.get_feature_names())\ndf_tfidf = pd.DataFrame(tfv, columns=feature_names)\ndf_tfidf.index = df_index\ndf_tfidf","metadata":{"execution":{"iopub.status.busy":"2023-03-06T23:19:41.231203Z","iopub.execute_input":"2023-03-06T23:19:41.231636Z","iopub.status.idle":"2023-03-06T23:20:00.229802Z","shell.execute_reply.started":"2023-03-06T23:19:41.231602Z","shell.execute_reply":"2023-03-06T23:20:00.228812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As above, let's check the Rarity(Tf-Idf) of each word in each dialogue grouped by Session and Level.  \nDemonstrative pronouns 'that, this, it' show approxmately 0.10-0.15, which looks relatively higher than other words.\n\n上記の通り、セッション・レベルで集約された会話毎に、各単語のレア度（Tf-Idf）を表示してみます。\n前回特定語としてピックアップしたthat,this,itなどは平均（mean）してTf-Idfが0.10～0.15程度と、比較的高めのように見えます。","metadata":{}},{"cell_type":"markdown","source":"# 2. First attempt\n\nFirst, try to count all words whose Tf-Idf is higher than 0.15 as a feature.  \nThis attempt didn't go well unfortunately, and the score went down from 0.677 to 0.676.  \n\nまずは、各セッション・レベル集約グループの会話の中に含まれているTf-idfが0.15以上の単語を全てカウントし、各グループの特徴量としてみます。\n結果としては空振りで、スコアは0.677から0.676へと少し悪化してしまいました。","metadata":{}},{"cell_type":"code","source":"tmp_bool = (df_tfidf > 0.15)\ntmp = pd.DataFrame(tmp_bool.sum(axis=1))\ntmp.columns = ['tfidf_015cnt']\ndf = pd.concat([df, tmp], axis=1)\ndf","metadata":{"execution":{"iopub.status.busy":"2023-03-06T23:20:00.230721Z","iopub.execute_input":"2023-03-06T23:20:00.231000Z","iopub.status.idle":"2023-03-06T23:20:00.327488Z","shell.execute_reply.started":"2023-03-06T23:20:00.230972Z","shell.execute_reply":"2023-03-06T23:20:00.326534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Second attempt\n\nNext attempt is to extract the specific words whose Tf-Idf is averagely higher than 0.15 in all dialogues.  \n\n上記２がうまくいかなかったので、次に、全会話において平均してTf-Idfが0.15以上となっている単語を抽出してみます。","metadata":{}},{"cell_type":"code","source":"tfidf_desc = pd.DataFrame(df_tfidf.describe().transpose())\ntfidf_desc","metadata":{"execution":{"iopub.status.busy":"2023-03-06T23:20:00.329005Z","iopub.execute_input":"2023-03-06T23:20:00.329774Z","iopub.status.idle":"2023-03-06T23:20:05.619199Z","shell.execute_reply.started":"2023-03-06T23:20:00.329695Z","shell.execute_reply":"2023-03-06T23:20:05.618372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfidf_desc[tfidf_desc['mean']>0.15]","metadata":{"execution":{"iopub.status.busy":"2023-03-06T23:20:05.622861Z","iopub.execute_input":"2023-03-06T23:20:05.623487Z","iopub.status.idle":"2023-03-06T23:20:05.638654Z","shell.execute_reply.started":"2023-03-06T23:20:05.623437Z","shell.execute_reply":"2023-03-06T23:20:05.637708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As above, under the condition Tf-Idf>0.15, only 4 specific words were extracted, which look a bit few (also, the words 'that''this' used as feature in my previous notebook are not extracted).  \n\nLet's change the criteria from >0.15 to >0.10.  \nThen, the extracted words increased to 10, and the words'that''this' are now included. Let's use the total count of these words as features.  \n\nThe result is, the Public score was slightly improved from **0.677 to 0.678**.  \n\nTo be honest, I'm not exactly sure why the score is improved by these words since the words such as 'the''is' don't seem specific words, but I think this result itself is quite interesting (^o^)/.  \n\n\nTf-Idf>0.15では特徴語が４つしか抽出されず、ちょっと少ないようです（前回Notebookで特定語として注目したthat,thisという語も入ってきません）ので、基準を0.10以上にしてみます。  \nこうすると、特徴語が10個まで増え、that,this,itも入ってきました。これらの語句の登場回数も特徴量として加えてみます。  \n結果としては、スコアが**0.677→0.678**へと少し改善しました。  \n正直、theやisなど特徴語といえるか微妙なものも含まれているのでスコアが改善する理由は正確には分からないのですが、面白い結果だと思います。","metadata":{}},{"cell_type":"code","source":"tfidf_desc[tfidf_desc['mean']>0.10]","metadata":{"execution":{"iopub.status.busy":"2023-03-06T23:20:05.640090Z","iopub.execute_input":"2023-03-06T23:20:05.640648Z","iopub.status.idle":"2023-03-06T23:20:05.666156Z","shell.execute_reply.started":"2023-03-06T23:20:05.640614Z","shell.execute_reply":"2023-03-06T23:20:05.665278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df, tfidf_desc, tmp, tmp_bool, tfvec, tfv\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-03-06T23:20:05.667733Z","iopub.execute_input":"2023-03-06T23:20:05.668180Z","iopub.status.idle":"2023-03-06T23:20:05.806383Z","shell.execute_reply.started":"2023-03-06T23:20:05.668086Z","shell.execute_reply":"2023-03-06T23:20:05.805187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DIALOGS = ['that', 'this', 'it', 'you', 'flag', 'can','and','is','the','to']","metadata":{"execution":{"iopub.status.busy":"2023-03-06T23:20:05.807959Z","iopub.execute_input":"2023-03-06T23:20:05.808281Z","iopub.status.idle":"2023-03-06T23:20:05.817122Z","shell.execute_reply.started":"2023-03-06T23:20:05.808252Z","shell.execute_reply":"2023-03-06T23:20:05.816141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineer(train):\n    dfs = []\n    for c in CATS:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('nunique')\n        tmp.name = tmp.name + '_nunique'\n        dfs.append(tmp)\n    for c in NUMS:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('mean')\n        tmp.name = tmp.name + '_mean'\n        dfs.append(tmp)    \n    for c in EVENTS: \n        train[c] = (train.event_name == c).astype('int8')\n    for c in EVENTS + ['elapsed_time']:\n        tmp = train.groupby(['session_id','level_group'])[c].agg('sum')\n        tmp.name = tmp.name + '_sum'\n        dfs.append(tmp)\n    \n    for c in DIALOGS:\n        tmp = train.groupby(['session_id','level_group'])['text'].apply(lambda x: (x.str.count(c)).sum())\n        tmp.name = c\n        dfs.append(tmp)\n          \n    df = pd.concat(dfs,axis=1)\n    df = df.fillna(-1)    \n    df = df.reset_index()\n    df = df.set_index('session_id')\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2023-03-06T23:20:05.818639Z","iopub.execute_input":"2023-03-06T23:20:05.818944Z","iopub.status.idle":"2023-03-06T23:20:05.832091Z","shell.execute_reply.started":"2023-03-06T23:20:05.818915Z","shell.execute_reply":"2023-03-06T23:20:05.830913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf = feature_engineer(train_df)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-06T23:20:05.833488Z","iopub.execute_input":"2023-03-06T23:20:05.834511Z","iopub.status.idle":"2023-03-06T23:25:14.527477Z","shell.execute_reply.started":"2023-03-06T23:20:05.834470Z","shell.execute_reply":"2023-03-06T23:25:14.526005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_df\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-03-06T23:25:14.529104Z","iopub.execute_input":"2023-03-06T23:25:14.529511Z","iopub.status.idle":"2023-03-06T23:25:14.796676Z","shell.execute_reply.started":"2023-03-06T23:25:14.529438Z","shell.execute_reply":"2023-03-06T23:25:14.795669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FEATURES = [c for c in df.columns if c != 'level_group']\nALL_USERS = df.index.unique()","metadata":{"execution":{"iopub.status.busy":"2023-03-06T23:25:14.797923Z","iopub.execute_input":"2023-03-06T23:25:14.798430Z","iopub.status.idle":"2023-03-06T23:25:14.811894Z","shell.execute_reply.started":"2023-03-06T23:25:14.798397Z","shell.execute_reply":"2023-03-06T23:25:14.810908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_FOLDS = 3\n\ngkf = GroupKFold(n_splits=N_FOLDS)\noof = pd.DataFrame(data=np.zeros((len(ALL_USERS),18)), index=ALL_USERS)\nmodels = {}\n\nfor i, (train_index, test_index) in enumerate(gkf.split(X=df, groups=df.index)):\n    print('#'*25)\n    print('### Fold',i+1)\n    print('#'*25)\n    \n    for t in range(1,19):\n                \n        if t<=3: grp = '0-4'\n        elif t<=13: grp = '5-12'\n        elif t<=22: grp = '13-22'\n            \n        # TRAIN DATA\n        train_x = df.iloc[train_index]\n        train_x = train_x.loc[train_x.level_group == grp]\n        train_users = train_x.index.values\n        train_y = target_df.loc[target_df.q==t].set_index('session').loc[train_users]\n        \n        # VALID DATA\n        valid_x = df.iloc[test_index]\n        valid_x = valid_x.loc[valid_x.level_group == grp]\n        valid_users = valid_x.index.values\n        valid_y = target_df.loc[target_df.q==t].set_index('session').loc[valid_users]\n        \n        # TRAIN MODEL\n        model = LGBMRegressor(learning_rate=0.027, \\\n                    num_leaves=15, \\\n                    n_estimators=200, \\\n                    min_child_samples=20, \\\n                    boosting_type='gbdt',\n                    subsample_for_bin=1000,\n                    max_depth=-1,\n                    colsample_bytree=0.8)\n        model.fit(train_x[FEATURES].astype('float32'), train_y['correct'])\n                \n        models[f'{i}_{grp}_{t}'] = model\n        oof.loc[valid_users, t-1] = model.predict(valid_x[FEATURES])\n        \n    print()","metadata":{"execution":{"iopub.status.busy":"2023-03-06T23:25:14.813178Z","iopub.execute_input":"2023-03-06T23:25:14.813739Z","iopub.status.idle":"2023-03-06T23:25:46.625247Z","shell.execute_reply.started":"2023-03-06T23:25:14.813703Z","shell.execute_reply":"2023-03-06T23:25:46.624374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"true = oof.copy()\nfor k in range(18):\n    # GET TRUE LABELS\n    tmp = target_df.loc[target_df.q == k+1].set_index('session').loc[ALL_USERS]\n    true[k] = tmp.correct.values","metadata":{"execution":{"iopub.status.busy":"2023-03-06T23:25:46.626515Z","iopub.execute_input":"2023-03-06T23:25:46.627013Z","iopub.status.idle":"2023-03-06T23:25:46.696411Z","shell.execute_reply.started":"2023-03-06T23:25:46.626980Z","shell.execute_reply":"2023-03-06T23:25:46.695525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores = []; thresholds = []\nbest_score = 0; best_threshold = 0\n\nfor threshold in np.arange(0.4,0.81,0.01):\n    print(f'{threshold:.02f}, ',end='')\n    preds = (oof.values.reshape((-1))>threshold).astype('int')\n    m = f1_score(true.values.reshape((-1)), preds, average='macro')   \n    scores.append(m)\n    thresholds.append(threshold)\n    if m>best_score:\n        best_score = m\n        best_threshold = threshold","metadata":{"execution":{"iopub.status.busy":"2023-03-06T23:25:46.697674Z","iopub.execute_input":"2023-03-06T23:25:46.698191Z","iopub.status.idle":"2023-03-06T23:25:49.860891Z","shell.execute_reply.started":"2023-03-06T23:25:46.698157Z","shell.execute_reply":"2023-03-06T23:25:49.859962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del target_df, df, oof, true\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-03-06T23:25:49.862169Z","iopub.execute_input":"2023-03-06T23:25:49.862713Z","iopub.status.idle":"2023-03-06T23:25:49.999918Z","shell.execute_reply.started":"2023-03-06T23:25:49.862678Z","shell.execute_reply":"2023-03-06T23:25:49.998917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import jo_wilder\nenv = jo_wilder.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2023-03-06T23:25:50.001473Z","iopub.execute_input":"2023-03-06T23:25:50.001789Z","iopub.status.idle":"2023-03-06T23:25:50.031683Z","shell.execute_reply.started":"2023-03-06T23:25:50.001759Z","shell.execute_reply":"2023-03-06T23:25:50.030757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(qid, grp, test):\n    val = 0\n    for fold in range(N_FOLDS):\n        val += models[f'{fold}_{grp}_{qid}'].predict(test[FEATURES])[0]\n    return val > best_threshold*N_FOLDS","metadata":{"execution":{"iopub.status.busy":"2023-03-06T23:25:50.032997Z","iopub.execute_input":"2023-03-06T23:25:50.033297Z","iopub.status.idle":"2023-03-06T23:25:50.037930Z","shell.execute_reply.started":"2023-03-06T23:25:50.033268Z","shell.execute_reply":"2023-03-06T23:25:50.037048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for (sample_submission, test) in iter_test:\n       \n    df = feature_engineer(test)\n        \n    grp = test.level_group.values[0]\n    sample_submission['qid'] = sample_submission['session_id'].apply(lambda x: x.split(\"_\")[1][1:]).astype(int)\n    sample_submission['correct'] = sample_submission['qid'].apply(lambda x: predict(x, grp, df)).astype(int)\n    del sample_submission['qid']\n    \n    env.predict(sample_submission)","metadata":{"execution":{"iopub.status.busy":"2023-03-06T23:25:50.039058Z","iopub.execute_input":"2023-03-06T23:25:50.039540Z","iopub.status.idle":"2023-03-06T23:25:51.104920Z","shell.execute_reply.started":"2023-03-06T23:25:50.039508Z","shell.execute_reply":"2023-03-06T23:25:51.103717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv(\"submission.csv\")\nsub.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-03-06T23:25:51.106162Z","iopub.execute_input":"2023-03-06T23:25:51.106509Z","iopub.status.idle":"2023-03-06T23:25:51.117571Z","shell.execute_reply.started":"2023-03-06T23:25:51.106446Z","shell.execute_reply":"2023-03-06T23:25:51.116599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}