{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## About this notebook\n\n+ train on 2021 regular season data(use update data\n+ cv on may,2021(test player)1.2833 but this score is leakage\n  + publicLB 1.1133　Why doesn't it match the emulation?\n+ cv on july,2021(include allplayer) 0.7153 this score is not leakage     maybe....","metadata":{}},{"cell_type":"markdown","source":"#### about stats\n![無題.png](attachment:de7b72c3-b2bc-4aa6-98b8-77711ca1cc3f.png)  \n1月と2月と3月のターゲットの値の記述統計量を4月に特徴として使う  \nUse descriptive statistics of target values for January, February, and March as features in April","metadata":{},"attachments":{"de7b72c3-b2bc-4aa6-98b8-77711ca1cc3f.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAmUAAACYCAIAAAAqfgmeAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAAChcSURBVHhe7ZzNixtrluZVf0GtGm2aEbXKxcA1FLRFL+pqFlPOXjSZm7mdi8bkpkwueoxWJuGuZoqkRINBM2BICkxnL5LOhqTQwgWCyYVgcqGmDSPqLioH42lBQiIwNCpcXATe3HnejziKL51QxLFehUvnt7ClfOKJJ+JVxDnxJTV+sMxmM/ciyyeLf5NBogLNJXivRAWaS/BeiQo0l+C9EhVoLsF7JSrQXMJ5tV+m4b0SFWguwXslKtBcgvdKVKC5BO+VqEBzCd4rUUFhrvbLNLxXogLNJXivRAWaS/BeiQo0l+C9EhVoLsF7JSoozNV+mYb3SlSguQTvlahAcwneK1GB5hK8V6ICzSV4r0QFhbnaL9PwXokKNJfgvRIVaC7BeyUq0FyC90pUoLkE75WooDC3gSnA+/fv3YssDxb/JoNEBbuW21AURVHqiq/UGVxt1/PLNLxXogJ8Hv5VWDQ3DJobBs0Ng+YSrrZ7WfslwXslKtDtLwyaGwbNDYPmhoHJdbXdy9ovCd4rUYFuf2HQ3DBobhg0NwxMrqvtXtZ+SfBeiQp0+wuD5oZBc8OguWFgcl1t97L2S4L3SlSg218YNDcMmhsGzQ0Dk+tqu5e1XxK8V6IC3f7CoLlh0NwwaG4YmFxX272s/ZLgvRIV6PYXBs0Ng+aGQXPDwOS62u5l7ZcE75WoQLe/MGhuGDQ3DJobBibX1faG++/h4cG9yPK9xb/JIFHBruXq9hcGzQ2D5oZBc8OAXF+pM7jarv0yDe+VqEC3+zBobhg0NwyaGwbk+kqdwdV2v1h6PZbgvRIV6HYfBs0Ng+aGQXPDwOS62u5l7ZcE75WoQLe/MGhuGDQ3DJobBibX1XYvF/aP2ajfG+RM41T/JgOvgnX6Vm40qbnwKqjQL91ieHXcw8j2xl4ivLoawXawGPfaOZHrUSF3fnfVPWy34Gw09/a7V3dzL5ShQu7ibtA72muKYmX726Tfzvtw16F87mxwBFOS8tkw+VfrYwb62H3ArfZxf7Ryj2Aolzu7OjRpWQ6vSobD41+tzQLb877drqqvbpXc+fj8xOUitjfY/Pa8olDMx333aWO3OrmYrLcYJdd3enXUzP0syxYv+fpWK19MrqvtXi7qH7foD7kDYdWN9i3TmrLRkZoPr4Ly/dIvhldD98v5pL9vdrnyldRRNtdug43WYe9qOBoNr05NeLs3Xnh5bUqvr9njGs39U+QOr3qH2N6rxFYe52jNw/VLsyU9OjrtxxlOvbg2pXMTH/Dg/BhvmifD0rW8XO58fOHXMKJ33MEs2r1JyY+49PraY6Bm5/h8YFe3g43sKO/4v4CyubPBMTbhRO5mt+cVhWI+PMEa28Wg3WqdMS+zvtPBidlzsrW6QvESrm/l8sXkutruZe2XxArv1vrlAie2ZgO3VKvipXOnF/tmbWNl2/5l/6JsIS+Zi90a3SO2YdvYCoWtZC6xGJ8+arbbj6qOdOncqTnlOq34qS4pm2vGtXkU+4Dng+MAn28as/aJxViXsrmTHtpFd0TblV3dR/2Jf7s2JXMnfWxJ8f3IFvIN5TKFwixHfPXtVrfOp405+Vcs88lVF4cClkStrlq8MLV/tZrV865evjAn/yqDq+1e5vrH/eWBWyZDNBrmKoM/3cX5bue4P6ZjU9dZxoPjPWyjewevvzN/XF5la5mT4zvzgWEdo1xz9uwuWpirBedmZmbp7i9jF3ASH4RdeNe3Ds3suvtmWZp7R3ZBPrx99fSx+YMNix9WzEbR1REK8kSzunz+JLp6Eq1U4jrSweV91C9H02G0Um5iv1SrwaT+1brYpGanOxz7IatEudz5sLvXTGxvP/xwd44jttLxJdd3ct5pd+LVxDbQ7NFSIeXH2WBObh+djqONswKlc8enldYvTdlxRv3slG6OOVQbZ0+2rK1N2VyzFyVW2O3ApT/jcrm2bqTO2qcXnUazV7ZhrpHLFArbLhN/WXe/Wmt9XXlsHZ5PhliG+FwrFy/R+grKF5PraruXuX758f7tzSscjnVOB6PRZIbusxh1o7P7m5s3ly+fmvPwdlTnzIo0m832ycXw5vr1m+/QP9xVtuXlADqEt7n2ooW/ZkEXS67eYek+3k9G54noCLf0NutRu93ys7Yn3ocnJ49bBy+vb6LrTKfRgZVdjEYb0/7mN//kLkssj26jWX399NX1zY1bDH9IspjRYtzcvL3/6Ptls+lyR8MLe3S1f2GW+TP3y0n/yN1rsVtliU0uQfncFBUPyCW5Cxy1nrTDXDczmK3DXrQRjHTZXHuC1RuM+/awq/oN23K5fgXpGDZUbgK7EFQ0SlI2dzE6RYk5unArOZ9coBCUvwpcaZxjp3UG0y/LHyGtkbu6UNhb5KlIuxjF1zXWWt/ZoOs3H1MW40GVi5dofXNYt3wxuba0V7oei/7dbvl3Vv3kjxSdbntJ5/yOVLuw8cvHrm+ZdUQuNuVmY99OHmGvFr24+Qhv9jNwuDm7LJwSRLO2m0Hj8NI2LrxHZ0c5cIdzbszctG59sSJozb6dRrMyqdZrPwdK9othxahfxq9w2BGg3FXA5V+Vpuwml0CQa3D3A8pfrquca8fXsN8vX9QAnP7VupgbXL56C0a6ZK7dPM2heXQb0Rx2VbnBVS7XXJ1stDudpjl0pMO9zfePGHavj+9A5aiQOx/3sYtGYLva/PGBW8mT+PGerRM59ayAMrmZzdf+IdUbk8VtJeXWNyqS/l2MsruUaH0zrF++mFxb+Nf9fZ/RmbsQmcGq33+6hf71a3d6ZV43zm5JfXjzbCl6bl+4aZBrXu49f30T59VTyL8cmTnb2WWj7ZxdbuOFzbLYa8cHl793Knhn/mAX5kN8MaL1ffvyq0bjue3M0ays1XpjK0KL4dV0rp/YLvKGfq+g7CaXQJD7w+LOHI1Xus1UNXdxNzL946K79o36FCVzzQ61XEHBSJfMnV4d77USA3t3jnq66fN4dzQSO8x0hwsVDohKri/hDqErnlyCsrn4gHFIkDguaR1fTTd+fODODGK5raOjtRpVijK5mc3X9ujU9mwn2ol+Wap8IddX6gy28K/7+z55/fLjh3dvb4b//Ou//7u/tfcKo9YSazPW+3vTshK9xfc12y8n8bujCf7LP/+b6T2+UXljhJ2z71tRSwO+X8Jo1OgPdgL7yrXG5fr66c3co1lZ61r9MpbrJ/7T65dzW2jMlaxKZwKC9XXYfX3D/cPcXojvT4KRFq+vCy99b7Fcru2XqUG1j4CUvvRdcX3tp1rhcgVRMtfewEtUTLu67cRVrXWosL6zkf8eh/sWy9xcUgjbL+0fdvP8smz5Qq6v1Bls4V/3932Sz8cuJuf+waTm3tdfHzx7cdHD4WK0zHZvdK+t913OpfJoHWez36364Kx3reuxsbHym4HrrvQHO4F9FV0AitbXT2/eRLOyVutNzNwvhlfTuX5i10H9X/LANP5VaZZrUoFquTj9wcfcPin//YYIwfp6bGHL2z5YyuTagc1no7krME8Alf6cy+Xm3UOL7QklqLa+7uZF2aw4FdY3de232pNk8s83b1mKKZObKRSbvn+5xBdJ/y5G2eIlWt+ICuWLybWFv9L9S/eU9Lm9AWDVT+4+jF/mWC+x4gezZaaOmMfmDgqmmc2m1pqzAVnvZ+yXdgeJFiNaX7sqbrOJZmWt2i8X9otNzYr3DyPK5c6GJ3vN40HixpLdsTd63mOe5kowOEXk8TleJR4xW4eS62s+0tSesfn1tSfU5q5+YlvafB0n8up3Scrl5q1bkH45Oe+0kufxvkCVXfcyudlCsfL52MJGUnJ9fZH072KULV6y9QUVyxeTawt/lX7pP3I31lb9YJ6boWWO9RKrZp/3MRfA3DTItTfFk5+dOadoPXn9HbyrPgM350zf8suW1y9XPu/jH/iOZmWtu94v3QPN613z5yiZaw9f4tfpqj5oJBhn0UiXzLUnWok9Y3yKEUgdMqxByVz3IErs4w2U6zBn0OUvsSeosl1lSszG7xO7A5NYitucyw9zqdyczdc2zNgBg1174fMvefgi6d/FKLtLCde3cvlicm3hX7dfvrWFrDewR9z+tzK6F8PR6Ob61bMnrWar5c4XDbFeYr3oH9OLQyy/+/LFoH/YajaX/dKvXTTDwUXXXOtt927tncaohvpows64XL+Mgpjvk5gprTXbL/1iXN8sv08Sy/UT/4n0Sxx9Ypz2Drv+V1iI0hdmy66vKyiJ5zIqdW3BOItGutr6Rpu++/pTkPWNP4jino8NNs62WFf4LaE4ZXPno67drroX5nng6j8cVXp9ba1sHfZN4au+OZfKzdt8cUrvK1+5tS+5vqYIbr9fCsoXk2vbwrr98tN06H8RwH43g36LsNF6fPD89e1sZs7v/QFLrJc4r3m1/PVG+xMC5outZpood/kji41W+9B+m4e8yWiPV9N9y47dyn4JEr9X0I3/jGI0K2vN9svlYpy9/RPvl/ZiTS6l4+Hxr9Ym9dMVwX7nc4lgpOXrG/sFjRJUWd/kD5tWu0ddJTe775SnQu5ynO36Vvsh16q5rlZW/tnaUrkrNl967shsZhv6/Vjz0W69X0rKFybyrzLYtrB2v/RvMlRU3c+aTIPnRmwrl/k8NormhkFzw6C5YdBcwtV2L2+0f0z67b3Er5zZuyX2BrT2yzBobhg0NwyaGwbNJVxt9/Jm+4e/iG9vSiUv42u/DIPmhkFzw6C5YdBcwtX2NX+vYOV38NdUP3x3+fzA/apBc+/Js1c37g7jxnNXsa1c3f7CoLlh0NwwaG4YkOsrdQZX2wP1y1VovwyD5oZBc8OguWHYwVxfqTO42u4Xa9eui+r12DBobhg0NwyaG4Ya5rra7mXtlwTvlahAt78waG4YNDcMmhsGJtfVdi9rvyR4r0QFuv2FQXPDoLlh0NwwMLmutntZ+yXBeyUq0O0vDJobBs0Ng+aGgcl1td3L2i8J3itRgW5/YdDcMGhuGDQ3DEyuq+1e1n5J8F6JCnT7C4PmhkFzw6C5YWByXW33svZLgvdKVKDbXxg0NwyaGwbNDQOT62q7l7VfErxXogLd/sKguWHQ3DBobhiYXFfbvaz9kuC9EhXo9hcGzQ2D5oZBc8PA5Lra3kDnAO/fv3cvsjxY/JsMEhXsWi4+D0VRFKWe+EqdwdV2307x3r3I4vqqf5NBogLNJXivRAWaS/BeiQq+xFzUCP8mA+8FX2JuZRVoLsF7JSqoba72yzS8V6ICzSV4r0QFmkvwXkjaLwleBZpL8F6JCmqbq/0yDe+VqEBzCd4rUYHmErwXkvZLgleB5hK8V6KC2uZqv0zDeyUq0FyC90pUoLkE74Wk/ZLgVaC5BO+VqKC2udov0/BeiQo0l+C9EhVoLsF7IWm/JHgVaC7BeyUqqG2u9ss0vFeiAs0leK9EBZpL8F5I2i8JXgWaS/BeiQpqm6v9Mg3vlahAcwneK1GB5hK8F5L2S4JXgeYSvFeigtrmar9Mw3slKtBcgvdKVKC5BO+FpP2S4FWguQTvlaigtrkN99/Dw4N7keV7i3+TQaICzSV4r0QFmkvwXokKvsRc9C3/JgPvBV9ibmUVaC7BeyUqqG2u9ss0vFeiAs0leK9EBZpL8F5I2i8JXgWaS/BeiQpqm6vXY9PwXokKNJfgvRIVaC7BeyFt67rotnIrq0BzCd4rUUFtc7VfpuG9EhVoLsF7JSrQXIL3QtJ+SfAq0FyC90pUUNtc7ZdpeK9EBZpL8F6JCj5D7rjXaBxeJWezVFfA597fvOwN8ifIznk26tPEWTWFZH21XxK8CjSX4L0SFdQ2V/tlGt4rUYHmErxXooLPkPv5++XtWXaOEZk5J+IzahrJ+mq/JHgVaC7BeyUqqG2u9ss0vFeiAs0leK9EBZ8hV/vln2huZRVoLsF7JSqoba72yzS8V6ICzSV4r0QFydz5+Pxkf6+JAt3c23/26vYDeY1y2G5BMVrnuG80p6b7pZn0STSTk/Px3P89AXLn4/5xx0+HGY7cHGZXBzbEEs02mf7105d+prOrQ/s3i5kYi/Th9mXObCMk44xZ+jcZeC/4EnMrq0BzCd4rUUFtc7VfpuG9EhVoLsF7JSqI5U6vjtBlWoe9q+FoeNU7RHt6fPZ2AWUx6jYbzc7x+WA0gtQ/bqOEP3751s050S/9TA7OLt/cvHEzafcmZiZJpr/99lGjud+9MLMcnJs5Nk9HmG4xe3vz6mmj0TmFMpmZvyTSL18+fYz0dn+CuSxmk9H5cWzij7dnX+XONkIyztovCV4FmkvwXokKapur/TIN75WoQHMJ3itRAeUuRqfoSt1Yb3n3+knr8Qv8YT7stlvxM8gfphf7jcbB5b2dc6xf0kwoF1O22omO5fjtt8Y09e9++OHuvNNq98fmZep6bCods8WCxfREuzbOg8vc2Tok46z9kuBVoLkE75WooLa5+nsFaXivRAWaS/BeiQoo9+0ZOt3ZrXtj4bymL/3s17+3qutR9+YlzYTzWv6l/59wivrs9e27j/4vxPffj35Jc8xg5mv0r1+/c3+IxX/69Ptf/+dG4y9yZ+uQjDP6ln+TgfeCLzG3sgo0l+C9EhXUNlf7ZRreK1GB5hK8V6KCKPf++pt0j0p7P3549/bm5vr1y+dPH9tbib8cWXXZsJYzSXszPPy///XqwN2QbLQeHzx7df02ys7vl1H63//d3/7FfzCuqLcn+uX3/z7+n39t5cxsHZJx/tnPfubmrGyUH//4x37QM/CfEdC6QfBeiQoKc/V6bBreK1GB5hK8V6KCKNc+ObO86mlYeheTc3MjEjT3Op3Dk9OL3jHeoGMZdXlBdDmTpXcFLncxmwwuuv7xnEa7NzbXbV0LXC5LMv3g2YvXZ0/xpuevsiaux7rc3Nk6dLsieK9EBZJcfGb+TQbeC3ScCd4rUUFhrvbLNLxXogLNJXivRAWUO+k1G82oB1k+vT1r7R1c3P0w6T9CSzqfLB90XYy6KGqZfrmcyTJ30mvtHWImKdLrOx+dIuRkiIxUv0ylY7Yfb57z/dK9NsRm69DtiuC9EhVIcrVfErwKapur/TIN75WoQHMJ3itRAeVmnvdZ3L74qtE8G0dnjcuHaMwjOObMLdsvM8/7LMboV8k2bJme/81PW8eDZQc2T+bk9st0+qdPH948N+l5/XJ6+fRx6+l17mwdul0RvFeiAkmu9kuCV0Ftc7VfpuG9EhVoLsF7JSqI5bqvgrTt9zaGF90OOt831/cQJn3ztYxO92Jov6Rxst9qtlqYNNsvaSZPX13f3LxxMznK+2W73/3DIaTOcf8KM/VfUfFfPPn06e3LrxqN/d7AfkUklf7siU+P+qU9/4wmvr/+ZtVsHbpdEbxXogJJrvZLgldBbXO1X6bhvRIVaC7BeyUqSOYuf6+g1T5+aX+SwP8dncfeQ2y1D7sX49ls+KzRePL6nRET/RIsf68AM+nn/1yByZ0Oe9Etxubefvfqzk+J0Hdvnu+buGbPfMsykf789e39/ZsTtMgLf845HXZpYus9y52tQ7crgvdKVCDJxSfn32TgvUDHmeC9EhUU5mq/TMN7JSrQXIL3SlSguQTvlahAcwneC0n7JcGroLa52i/T8F6JCjSX4L0SFWguwXslKtBcgvdC0n5J8Cqoba72yzS8V6ICzSV4r0QFmkvwXokKNJfgvZC0XxK8Cmqbq79XkIb3SlSguQTvlahAcwneK1GB5hK8FxL6pX+TgfcCHWeC90pUUJir/TIN75WoQHMJ3itRgeYSvFeiAs0leC8k7ZcEr4La5ur12DS8V6ICzSV4r0QFmkvwXokKNJfgvZD0eizBq6C2udov0/BeiQo0l+C9EhVoLsF7JSrQXCLX2+v10CZ/8YtfQHL9Eq/x4le/+pWbwJHrjaPjTPBeiQoKc7VfpuG9EhVoLsF7JSrQXIL3SlSgucQqL7oj+NGPfkT/Aq9FrPISOs4E75WooDBX+2Ua3itRgeYSvFeiAs0leK9EBZpLrPLu7++7Hkk8efLEaxGrvISOM8F7JSoozNV+mYb3SlSguQTvlahAcwneK1GB5hKM1/fJCP/XGIzXoeNM8F6JCgpztV+m4b0SFWguwXslKtBcgvdKVKC5BOONn2JmTy4B43XoOBO8V6KCwlztl2l4r0QFmkvwXokKNJfgvRIVaC7Be323XPGILO8FOs4E75WooDBX+2Ua3itRgeYSvFeiAs0leK9EBZpL8N6f//znaJa5J5eA9wIdZ4L3SlRQmKv9Mg3vlahAcwneK1GB5hK8V6ICzSV47x/+8Ief/OQnf/zjH/37JLwX6DgTvFeigsLcBqYA79+/dy+yPFj8mwwSFWguwXslKti1XHftS1EUIX6PysDvgxIV1LZe6fllGt4rUYHmErxXogLs5/5VWDQ3DNvK3TWYceb3QYkKaluv/HBoHSd4r0QFmkvwXokKtH+EYddydw1mnPl9UKKC2tYrPxxaxwneK1GB5hK8V6IC7R9h2LXcXYMZZ34flKigtvXKD4fWcYL3SlSguQTvlahA+0cYdi1312DGmd8HJSqobb3yw6F1nOC9EhVoLsF7JSrQ/hGGXcvdNZhx5vdBiQpqW6/8cGgdJ3ivRAWaS/BeiQq0f4Rh13J3DWac+X1QooLa1is/HFrHCd4rUYHmErxXogLtH2HYtdxdgxlnfh+UqKC29coPh9ZxgvdKVKC5BO+VqED7Rxh2LXfXYMaZ3wclKqhtvWq4/x4eHtyLLN9b/JsMEhVoLsF7JSrYtVztH2HYtdxdA+Ps96gM/D4oUUFt65X2yzS8V6ICzSV4r0QF2j/CsGu5uwbG2e9RGfh9UKKC2tYrv9npdUKC90pUoLkE75WoQPtHGHYtd9dgxpnfByUqqG298sOhdZzgvRIVaC7BeyUq0P4Rhl3L3TWYceb3QYkKaluv/HAEWb7psLu/18RH0Di8mro/Vcyd310+f/W26lKB2n4e/k0GiQp2LTe5ny/GvXajN/bvVjC/u+oetltm82zu7Xev7uZeyGNxN+gd2W05OW0ytySTfrtRuJj5rJc7H540se+t/EjApGd30DicAbJ/laXkrObj8xNXHoqHXzbOG2PNVRj3zFjkUO2z3yRYKP8qA78PSlRQ23rlhyPE8o1PsR3t9waj0WgyW7i/VcudXR02Gme3VZcK1Pbz8G8ySFSwa7mx/Xw+6e+bEsYWI9tRG63D3tVwNBpenRpHuzf2W2ma6dVRE5vyKSYeXvUO0WOjaZn6UoBbgk32S7vUfM9ye9b+ST/OxXh162JyS83KjWjn+BzVIRr+/sSLOayzvoFZfxWmQz8cRPdwD7XxaMB8MtuBGWd+H5SooLb1yg9HgOWz+096Z62Wq/0yBa+CXct1+/liNuqbbmbhGtH0Yj920cNg/7J/EfsLgbO0RuNRrJnaaV2xQ477W0kW49NHzXb70eb65RT7TLuNlsz2S3NMmxiHApjcErNajE6bycOTSR9DcTry77JUHeeNUX4VlpgPnzk62ybMOPP7oEQFta1Xfjg2vHy2wS2JasJs9N//5qe2nrXax73BXXyDWXx3ffb0a3f5FvJh98rLiasZZk525okyM7s8MP3UvbbqcNzfR1CrfTqyc5n97jf9Y3fxLR0dLXM+EhXsWt/aVi4+V7+lNDvd4TizhSSZD7t7zVRtvztfdXF0ct5pd+JnDraBujZkc0tjTk0enY7vChaToSDXnrxiBc2IcP3SNNVSS7A6t8Ss7Ph13Y65JtXGeXNUWIWIxajbbODTr+LdOMw48/ugRAW1rVd+ODa8fIvZZDQ47TQanVNzOdZc2F9M+p1m48//6lt3BSxxVcsfsrcOzi4hjgYXXXN1o3kyNFdz5nduVk9f3dzYOa3RL5vNlrkQPLy6MheEEP2Xf7a8+JaKjpY5H4kKdq1vbSvX7ueT/lF/ZPKzW0gh88ExTiK5S4KOxXxyddKma2lV6rjplnbzq7CYEWyu6ZbNI3M4UNQvzTnh0cXoqmsOLs2RpBu/1azOLTErt1RTe/dPnLsdKqyCA5/No6iy1RBmnPl9UKKC2tYrPxwBls9WA9pZTUNsdof/d5lrj7P8FTAcwu89Prv9SHO2lzuolthZ0fXYbJnJ9MvGyXB5+Gai/+zZP8W20Hj0Z1vfXHatb20rN7mfl25E7mZi/uXYJfb81bDfn/jNC2/ci7UxD/n4+1ylF3MJk2se8okKckG/nF7gOHR5D84eSLpGu4qVuSVmNRsc4dCk02m5O8Kjgb2Gzt/NKz/OG6XKKliYy/61gBlnfh+UqKC29Src7xXcmyZ2cHlv37yzb16+ub6+vom4fvH1coL0nK37+Q06aPTmlyOv2nemexL3//jXRnev47EGG73/3y59qiUencxNI1HBrv1uwLZyk/t5uUa0uLswT26wfcKwuBuZ0ugufvjrEyXr+PLMz7CJfkknr5aCfjnut1vLicF8iAPJxvFg5dnPytwSs3JHtOb0zP8Bfxqc4BPw907yKDnOm6bKKhjMPc76nlwCrJTfozLw+6BEBbWtV1vql7dnZvvKoXn21k5gvP/+8G//5/bmzeWrF8+e2BuZkVvUL4ui48ucRaKCXetb28rFx+n3eEOJRjQf9zqmWV4kbqYXYM8S7MXbZG4B5rJGvC1//n5pFizetQr6ZQ5j86WQ1Tfmyqzvqlm5ZpPqGvZh+t7Ky+GlxnnzVFkF8/mfFrfU7YK18ntUBn4flKigtvXKb3YBzn/tNhXtrPZSFkrDqtzpsPu16ZDmTkCnc9w973c7S7edVZnrsfEaYaO//e3G1zeX2l5n8G8ySFSwrVx8xP6VIbuF5DO9Om7hVPFkWHBmmcE+3GI2smQuj12qfMr1NACPfxXHbur5rNuWac3ywZz8q2JWzQqHDdkFigrECsrkBqDKKvh2ue4HsR2Yceb3QYkKaluv/HAEWL5E47K3NzoX0/xc+7DFV8/f3H+kOVtD5LazSvbLxLHd5Ay9dlW/tHP6y//xr/5thvgyZ5GooLbbgX+TQaKCbeUm93O7DRQUpoX9mmZzeStyFbPhyV4zdV3RblPmblWZOm4egktgH2M7Pscr+n7yuuTnzu/cjIlz7FfLR+7S2AKfPAG0jaDCeV6pWSX2bYedGPXBv81QZpxDUGEV7O851LxdcuPM74MSFdS2XvnhCLB8ycblLmGd/nZKufYraHv26oTd217YK6xevDvH5Cv6pfs+3PJRRszoK0y8ql+66P/4X2P3VmLRn299c9m1vrWt3OR+Xtwv3ZfNC29ZWuwX6+LPaMSeDmLqSzHrtPUVrJlrdqxkVY/jTnjig2BPCbmnhFfllpuV3SPj140r526N8qtgJ6jzvUsDM878PihRQW3rlR+OAMuXalyuyDT+/K/69gG6/rG9beR2MPNIX6PROnh5fWMfrjvaa7ZaraXbdsivnr1+4w6T3eRt+yje4Py40zr6hrkea6N/iujWYU7051vfXHatb20rN7mf5zWi+NUytwHtHXb9z60Q0YXZ+MTRpuu/j2Q2uOXmk8wtyRb6ZWrvsIcNtGMU/MqRYXUuP6vcYuCfpnUPl1bN3Rb8KuRUIXN7k+untYAZZ34flKigtvXKD0eA5ctuMou7wbf+9wqae53EF5YWd1fP3TM+VhlO5/ZRsuje+GLy6hur+ie2Mbn7aVr3u40L80zP6n4Jpv/7H3r+9wrS0fFlziJRwa71rW3l4nP1rwx2G1jdL+3hVy6RJ9kvgfmxWfql0JNz2nzwB/+qAnmLuSZr5hb1SzAbrfohjzzYXGZWucXA75FmRC8m/HkXm7stmFXIGefMX+oIM878PihRQW3rlR8OreME75WoQHMJ3itRwTr1dHp1uH5zWnPibdVxQe74VFC1v8DcbSFa323BjDO/D0pUUNt65YdD6zjBeyUq0FyC90pUsEY9vTvfX/+L4utOvK06Xjl3Puy2BN9o+OJyt4VwfbcFM878PihRQW3rlR8OreME75WoQHMJ3itRQXE9nfSPCx+FJdaeeFt1vGrufNAt/+2ZGF9a7raQru+2YMaZ3wclKqhtvQr3ewW5aC7BeyUq2LXcL61vSdFcZRNgnP0elYHfByUqqG290n6ZhvdKVKC5BO+VqED7Rxh2LXfXwDj7PSoDvw9KVFDbeuU3O71OSPBeiQo0l+C9EhVo/wjDruXuGsw48/ugRAW1rVd+OLSOE7xXogLNJXivRAXaP8Kwa7m7BjPO/D4oUUFt65UfDq3jBO+VqEBzCd4rUYH2jzDsWu6uwYwzvw9KVFDbeuWHQ+s4wXslKtBcgvdKVKD9Iwy7lrtrMOPM74MSFdS2Xvnh0DpO8F6JCjSX4L0SFWj/CMOu5e4azDjz+6BEBbWtV344tI4TvFeiAs0leK9EBdo/wrBrubsGM878PihRQW3rlR8OreME75WoQHMJ3itRgfaPMOxa7q7BjDO/D0pUUNt65YdD6zjBeyUq0FyC90pUoP0jDLuWu2sw48zvgxIV1LZe+eHQOk7wXokKNJfgvRIVaP8Iw67l7hrMOPP7oEQFta1XDUwB3r9/715kebD4NxkkKtBcgvdKVLBrudjPFUWR4/eoDPw+KFFBbeuVP3zAe/cii+ur/k0GiQo0l+C9EhVoLsF7JSrQXIL3SlSguQTvlahAcwnn1X6ZhvdKVKC5BO+VqEBzCd4rUYHmErxXogLNJXivRAWFudov0/BeiQo0l+C9EhVoLsF7JSrQXIL3SlSguQTvlaigMFf7ZRreK1GB5hK8V6ICzSV4r0QFmkvwXokKNJfgvRIVFOZqv0zDeyUq0FyC90pUoLkE75WoQHMJ3itRgeYSvFeigsJc7ZdpeK9EBZpL8F6JCjSX4L0SFWguwXslKtBcgvdKVFCYq/0yDe+VqEBzCd4rUYHmErxXogLNJXivRAWaS/BeiQoKc7VfpuG9EhVoLsF7JSrQXIL3SlSguQTvlahAcwneK1FBYW7D/ffw8OBeZPne4t9kkKhAcwneK1GB5hK8V6ICzSV4r0QFmkvwXokKNJdwXu2XaXivRAWaS/BeiQo0l+C9EhVoLsF7JSrQXIL3SlRQmKvXY9PwXokKNJfgvRIVaC7BeyUq0FyC90pUoLkE75WooDBX+2Ua3itRgeYSvFeiAs0leK9EBZpL8F6JCjSX4L0SFRTmar9Mw3slKtBcgvdKVKC5BO+VqEBzCd4rUYHmErxXooLCXO2XaXivRAWaS/BeiQo0l+C9EhVoLsF7JSrQXIL3SlRQmKv9Mg3vlahAcwneK1GB5hK8V6ICzSV4r0QFmkvwXokKCnO1X6bhvRIVaC7BeyUq0FyC90pUoLkE75WoQHMJ3itRQWGu9ss0vFeiAs0leK9EBZpL8F6JCjSX4L0SFWguwXslKijM1X6ZhvdKVKC5BO+VqEBzCd4rUYHmErxXogLNJXivRAWFufp7BWl4r0QFmkvwXokKNJfgvRIVaC7BeyUq0FyC90pUUJir/TIN75WoQHMJ3itRgeYSvFeiAs0leK9EBZpL8F6JCgpz9XpsGt4rUYHmErxXogLNJXivRAWaS/BeiQo0l+C9EhUU5mq/TMN7JSrQXIL3SlSguQTvlahAcwneK1GB5hK8V6KCwlztl2l4r0QFmkvwXokKNJfgvRIVaC7BeyUq0FyC90pUUJir/TIN75WoQHMJ3itRgeYSvFeiAs0leK9EBZpL8F6JCgpztV+m4b0SFWguwXslKtBcgvdKVKC5BO+VqEBzCd4rUUFhrvbLNLxXogLNJXivRAWaS/BeiQo0l+C9EhVoLsF7JSooyP306f8DhaRf/lhXOkwAAAAASUVORK5CYII="}}},{"cell_type":"markdown","source":"## Reference\n\nThank you for publishing a great notebook and dataset!\n\n+ @columbia2131 [MLB lightGBM Starter Dataset&Code[en, ja]](https://www.kaggle.com/columbia2131/mlb-lightgbm-starter-dataset-code-en-ja)\n+ @naotaka1128 [Creating Unnested Dataset](https://www.kaggle.com/naotaka1128/creating-unnested-dataset)\n+ @mlconsult [create player descriptive stats dataset](https://www.kaggle.com/mlconsult/create-player-descriptive-stats-dataset)\n+ @kaito510 [Player Salary + MLB lightGBM Starter](https://www.kaggle.com/kaito510/player-salary-mlb-lightgbm-starter)\n+ @kohashi0000 [1.36 simple_LightGBM](https://www.kaggle.com/kohashi0000/1-36-simple-lightgbm)\n+ @somayyehgholami, @mehrankazeminia [[Fork of] LightGBM + CatBoost + ANN 2505f2](https://www.kaggle.com/somayyehgholami/fork-of-lightgbm-catboost-ann-2505f2)\n+ @nyanpn [API Emulator for debugging your code locally](https://www.kaggle.com/nyanpn/api-emulator-for-debugging-your-code-locally)","metadata":{}},{"cell_type":"markdown","source":"## Get env","metadata":{}},{"cell_type":"code","source":"# 環境によって処理を変えるためのもの\nimport sys\nIN_COLAB = 'google.colab' in sys.modules\nIN_KAGGLE = 'kaggle_web_client' in sys.modules\nLOCAL = not (IN_KAGGLE or IN_COLAB)\nprint(f'IN_COLAB:{IN_COLAB}, IN_KAGGLE:{IN_KAGGLE}, LOCAL:{LOCAL}')","metadata":{"execution":{"iopub.status.busy":"2021-08-02T23:46:32.162083Z","iopub.execute_input":"2021-08-02T23:46:32.162868Z","iopub.status.idle":"2021-08-02T23:46:32.170964Z","shell.execute_reply.started":"2021-08-02T23:46:32.162822Z","shell.execute_reply":"2021-08-02T23:46:32.170163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unnest updatedfile","metadata":{}},{"cell_type":"code","source":"%%python\nimport numpy as np\nimport pandas as pd\nimport os\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2\n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int64)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float32)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float64)\n                else:\n                    df[col] = df[col].astype(np.float64)\n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df\n\nif not os.path.isfile('./train_updated_playerBoxScores.pickle'):\n    # drop playerTwitterFollowers, teamTwitterFollowers from example_test\n    df = pd.read_csv(f\"../input/mlb-player-digital-engagement-forecasting/train_updated.csv\").dropna(axis=1,how='all')\n    df = df.query(\"date >= 20210501\")\n    daily_data_nested_df_names = df.drop('date', axis = 1).columns.values.tolist()\n    \n    for df_name in daily_data_nested_df_names:\n        date_nested_table = df[['date', df_name]]\n    \n        date_nested_table = (date_nested_table[\n          ~pd.isna(date_nested_table[df_name])\n          ].\n          reset_index(drop = True)\n          )\n    \n        daily_dfs_collection = []\n    \n        for date_index, date_row in date_nested_table.iterrows():\n            daily_df = pd.read_json(date_row[df_name])\n    \n            daily_df['dailyDataDate'] = date_row['date']\n    \n            daily_dfs_collection = daily_dfs_collection + [daily_df]\n    \n        # Concatenate all daily dfs into single df for each row\n        unnested_table = (pd.concat(daily_dfs_collection,\n          ignore_index = True).\n          # Set and reset index to move 'dailyDataDate' to front of df\n          set_index('dailyDataDate').\n          reset_index()\n          )\n        #print(f\"{file}_{df_name}.pickle\")\n        #display(unnested_table.head(3))\n        reduce_mem_usage(unnested_table).to_pickle(f\"train_updated_{df_name}.pickle\")\n        #print('\\n'*2)\n    \n        # Clean up tables and collection of daily data frames for this df\n        del(date_nested_table, daily_dfs_collection, unnested_table)","metadata":{"execution":{"iopub.status.busy":"2021-08-02T23:46:37.050043Z","iopub.execute_input":"2021-08-02T23:46:37.050720Z","iopub.status.idle":"2021-08-02T23:48:45.247731Z","shell.execute_reply.started":"2021-08-02T23:46:37.050665Z","shell.execute_reply":"2021-08-02T23:48:45.246733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\nimport numpy as np\nimport pandas as pd\nfrom numpy import mean,std\nfrom scipy.stats import norm\nimport statistics as st\nimport warnings\n\nwarnings.simplefilter('ignore')\n\ndef calc_probs(year, pid, df, temp, patern):\n    to_append=[year, pid,'','','','','','','','','','','','','','','','','','','','','','','','']\n    targets=['target1','target2','target3','target4']\n    z=2\n    for target in targets:\n        target_prob = temp[target].tolist()\n        mean = np.mean(target_prob) # 平均値\n        std = np.std(target_prob) # 標準偏差\n        median = st.median(target_prob) # 中央値\n        distribution = norm(mean, std) # ノルム\n        min_weight = min(target_prob) # 最小値\n        max_weight = max(target_prob) # 最大値\n        values = list(np.linspace(min_weight, max_weight)) # デフォルト50\n        probabilities = [distribution.pdf(v) for v in values]\n        max_value = max(probabilities)\n        max_index = probabilities.index(max_value) # 確率密度関数の頂点はlinspaceの何個目か\n        to_append[z]=mean # 平均\n        to_append[z+1]=median # 中央地\n        to_append[z+2]=std # 標準偏差\n        to_append[z+3]=min_weight # 最小値\n        to_append[z+4]=max_weight # 最大値\n        # よくわからないので複数パターン用意\n        # ============================\n        if patern == 1:\n            to_append[z+5]=target_prob[max_index] # 確率密度変数の頂点\n        elif patern == 2:\n            to_append[z+5]=sorted(target_prob)[max_index] # 実測値\n        elif patern == 3:\n            to_append[z+5]=values[max_index] # 正規分布の中心\n        z=z+6\n    df_length = len(df)\n    df.loc[df_length] = to_append\n    return df\n\n# 2021年8月以降用のスタッツを作る\ntargets = pd.read_pickle('./train_updated_nextDayPlayerEngagement.pickle')\ntargets = targets.query('20210601 <= dailyDataDate')\n\n# CREATE DATAFRAME to store probabilities\ncolumn_names = [\"year\", \"playerId\", \"target1_mean\",\"target1_median\",\"target1_std\",\"target1_min\",\"target1_max\",\"target1_prob\", \n                \"target2_mean\",\"target2_median\",\"target2_std\",\"target2_min\",\"target2_max\",\"target2_prob\", \n                \"target3_mean\",\"target3_median\",\"target3_std\",\"target3_min\",\"target3_max\",\"target3_prob\", \n                \"target4_mean\",\"target4_median\",\"target4_std\",\"target4_min\",\"target4_max\",\"target4_prob\"]\nplayer_target_probs = pd.DataFrame(columns = column_names)\nyear_by_probs = pd.DataFrame(columns = column_names)\n\nyears = [\"2021\"]\ndfs = [targets]\n\nfor year, df in zip(years, dfs):\n    playerId_list = df.playerId.unique().tolist()\n    for pid in playerId_list:\n        temp = df[df['playerId'] == pid]\n        player_target_stats=calc_probs(year, pid, player_target_probs, temp, patern=3)\n\ndf = pd.read_csv('../input/mlb-features/statsdata.csv')\ndf8 = player_target_stats.copy()\ndf9 = player_target_stats.copy()\ndf10 = player_target_stats.copy()\ndf8['month'] = 8\ndf9['month'] = 9\ndf10['month'] = 10\nplayer_target_stats = pd.concat([df, df8, df9, df10],axis=0).reset_index(drop=True)\nprint(player_target_stats.groupby(['year','month']).size())\nplayer_target_stats.to_csv('player_target_stats.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2021-08-02T23:52:03.559716Z","iopub.execute_input":"2021-08-02T23:52:03.560089Z","iopub.status.idle":"2021-08-02T23:53:37.025076Z","shell.execute_reply.started":"2021-08-02T23:52:03.560056Z","shell.execute_reply":"2021-08-02T23:53:37.024107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Libraries","metadata":{}},{"cell_type":"code","source":"# Standard library\nimport os, sys, gc, time, warnings, shutil, random\nfrom pathlib import Path\nfrom contextlib import contextmanager\n\n# third party\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\n\nfrom sklearn import preprocessing\nfrom sklearn.metrics import mean_absolute_error\nimport lightgbm as lgb\n\n# \nimport mlb\n\npd.set_option('display.max_rows', 500)\nprint(lgb.__version__)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-02T23:54:55.336206Z","iopub.execute_input":"2021-08-02T23:54:55.336652Z","iopub.status.idle":"2021-08-02T23:54:57.348427Z","shell.execute_reply.started":"2021-08-02T23:54:55.336614Z","shell.execute_reply":"2021-08-02T23:54:57.347304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Config","metadata":{}},{"cell_type":"code","source":"class CFG:\n    seed = 29","metadata":{"execution":{"iopub.status.busy":"2021-08-02T23:54:57.349978Z","iopub.execute_input":"2021-08-02T23:54:57.350283Z","iopub.status.idle":"2021-08-02T23:54:57.354671Z","shell.execute_reply.started":"2021-08-02T23:54:57.350246Z","shell.execute_reply":"2021-08-02T23:54:57.353384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" # Utills","metadata":{}},{"cell_type":"code","source":"# Seed\ndef set_seed(seed: int = 29):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    \nset_seed(CFG.seed)","metadata":{"execution":{"iopub.status.busy":"2021-08-02T23:54:57.356533Z","iopub.execute_input":"2021-08-02T23:54:57.356847Z","iopub.status.idle":"2021-08-02T23:54:57.368348Z","shell.execute_reply.started":"2021-08-02T23:54:57.356818Z","shell.execute_reply":"2021-08-02T23:54:57.367133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Road","metadata":{}},{"cell_type":"code","source":"INPUT_DIR = Path('../input')\nUNNESTED_DIR = INPUT_DIR / 'mlb-unnested'\n\n# non update files\n# ======================================================================================\n#df_players = pd.read_pickle(UNNESTED_DIR / 'players.pickle')\ndf_players = pd.read_csv(INPUT_DIR / 'playerscsv/NEWplayers.csv') # salarydata\ndf_teams = pd.read_pickle(UNNESTED_DIR / 'teams.pickle').rename(columns = {'id':'teamId'})\n\n# update files\n# ======================================================================================\ndf_targets = pd.concat([pd.read_pickle(UNNESTED_DIR / 'train_nextDayPlayerEngagement.pickle'),\n                        pd.read_pickle('./train_updated_nextDayPlayerEngagement.pickle')],axis=0).reset_index(drop=True)\ndf_games = pd.concat([pd.read_pickle(UNNESTED_DIR / 'train_games.pickle'),\n                      pd.read_pickle('./train_updated_games.pickle')],axis=0).reset_index(drop=True)\ndf_rosters = pd.concat([pd.read_pickle(UNNESTED_DIR / 'train_rosters.pickle'),\n                        pd.read_pickle('./train_updated_rosters.pickle')],axis=0).reset_index(drop=True)\ndf_scores = pd.concat([pd.read_pickle(UNNESTED_DIR / 'train_playerBoxScores.pickle'),\n                       pd.read_pickle('./train_updated_playerBoxScores.pickle')],axis=0).reset_index(drop=True)\ndf_team_scores = pd.concat([pd.read_pickle(UNNESTED_DIR / 'train_teamBoxScores.pickle'),\n                            pd.read_pickle('./train_updated_teamBoxScores.pickle')],axis=0).reset_index(drop=True)\ndf_transactions = pd.concat([pd.read_pickle(UNNESTED_DIR / 'train_transactions.pickle'),\n                             pd.read_pickle('./train_updated_transactions.pickle')],axis=0).reset_index(drop=True)\ndf_standings = pd.concat([pd.read_pickle(UNNESTED_DIR / 'train_standings.pickle'),\n                          pd.read_pickle('./train_updated_standings.pickle')],axis=0).reset_index(drop=True)\ndf_awards = pd.concat([pd.read_pickle(UNNESTED_DIR / 'train_awards.pickle'),\n                       pd.read_pickle('./train_updated_awards.pickle')],axis=0).reset_index(drop=True)\ntwitter_players = pd.concat([pd.read_pickle(UNNESTED_DIR / 'train_playerTwitterFollowers.pickle'),\n                             pd.read_pickle('./train_updated_playerTwitterFollowers.pickle')],axis=0).reset_index(drop=True)\ntwitter_team = pd.concat([pd.read_pickle(UNNESTED_DIR / 'train_teamTwitterFollowers.pickle'),\n                          pd.read_pickle('./train_updated_teamTwitterFollowers.pickle')],axis=0).reset_index(drop=True)\n\n# ネストを外すときnanの場合があり、そのときは同じ形のdataframeを作る必要がある\ncolumns_dict = {'games': df_games.columns,\n                'rosters': df_rosters.columns,\n                'playerBoxScores': df_scores.columns,\n                'teamBoxScores': df_team_scores.columns,\n                'transactions': df_transactions.columns,\n                'standings': df_standings.columns,\n                'awards': df_awards.columns,\n                'playerTwitterFollowers':twitter_players.columns,\n                'teamTwitterFollowers':twitter_team.columns}","metadata":{"execution":{"iopub.status.busy":"2021-08-02T23:54:58.889495Z","iopub.execute_input":"2021-08-02T23:54:58.889885Z","iopub.status.idle":"2021-08-02T23:55:05.826814Z","shell.execute_reply.started":"2021-08-02T23:54:58.889855Z","shell.execute_reply":"2021-08-02T23:55:05.825996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Setting COL list\n# ======================================================================================\n# COL_PLAYERS = ['playerId', 'primaryPositionCode', 'american']\nCOL_PLAYERS = ['playerId', 'primaryPositionName', 'american', 'salary']\nCOL_TEAMS = ['teamId', 'leagueId', 'divisionId']\n\nCOL_ROSTERS = ['dailyDataDate', 'playerId', 'teamId', 'statusCode']\nCOL_STANDINGS = ['dailyDataDate', 'teamId', 'wins', 'losses', 'lastTenWins','lastTenLosses']\nCOL_SCORES = [i for i in df_scores.columns.to_list() if i not in ['gamePk','gameDate','gameTimeUTC','teamId','teamName','playerName','positionName','positionType','jerseyNum', 'battingOrder']]\n\nCOL_STANDINGS = ['wins', 'losses', 'lastTenWins','lastTenLosses']\n\n\ntmp_feature_set = set(COL_PLAYERS + COL_ROSTERS + COL_SCORES + COL_TEAMS + COL_STANDINGS)\ntmp_feature_set.discard('dailyDataDate')\ntmp_feature_set.discard('gameDate')\ntmp_feature_set.discard('playerId')\nCOL_FEATURES = list(tmp_feature_set)\nCOL_TARGETS = ['target1', 'target2', 'target3', 'target4']","metadata":{"execution":{"iopub.status.busy":"2021-08-02T23:55:05.828156Z","iopub.execute_input":"2021-08-02T23:55:05.828672Z","iopub.status.idle":"2021-08-02T23:55:05.835784Z","shell.execute_reply.started":"2021-08-02T23:55:05.828637Z","shell.execute_reply":"2021-08-02T23:55:05.835027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocess & Feature engineering","metadata":{}},{"cell_type":"code","source":"def FE_team_score(df_team_scores):\n    \"\"\"\n    その日チームの勝敗、どういう勝ち方をしたかどうかの特徴量を作成する\n    \"\"\"\n    df_team_scores = df_team_scores.rename(columns={'runsScored':'team_runsScored', 'runsPitching':'team_runsPitching'})\n    # 勝ち負け\n    df_team_scores.loc[df_team_scores['team_runsScored'] > df_team_scores['team_runsPitching'], ['team_win']] = 1\n    # 得失点差\n    df_team_scores['team_runsdiff'] = df_team_scores['team_runsScored'] - df_team_scores['team_runsPitching']\n    # 完封勝ち\n    df_team_scores.loc[(df_team_scores['team_runsScored'] > 0) & (df_team_scores['team_runsPitching'] == 0),['team_shutout_win']] = 1\n    df_team_scores.loc[(df_team_scores['team_runsScored'] == 0) & (df_team_scores['team_runsPitching'] > 0),['team_shutout_lose']] = 1\n    # fillna\n    df_team_scores[['team_win','team_shutout_win','team_shutout_lose']] = df_team_scores[['team_win','team_shutout_win','team_shutout_lose']].fillna(0)\n    # double header\n    df_team_scores = df_team_scores.groupby(['dailyDataDate','teamId']).sum().reset_index()\n    \n    df_team_scores = df_team_scores[['dailyDataDate','teamId'] + COL_TEAMSCORE]\n    return df_team_scores\n","metadata":{"execution":{"iopub.status.busy":"2021-08-02T23:55:05.837273Z","iopub.execute_input":"2021-08-02T23:55:05.837711Z","iopub.status.idle":"2021-08-02T23:55:05.853069Z","shell.execute_reply.started":"2021-08-02T23:55:05.837681Z","shell.execute_reply":"2021-08-02T23:55:05.852193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Players\n# =========================================================================================\ndf_players['american'] = df_players['birthCountry'].apply(lambda x:1 if x == 'USA' else 0)\n\n# playerBoxScores \n# =========================================================================================\nprint(f'scores shape {df_scores.shape}')\ndf_scores = df_scores.groupby(['playerId','dailyDataDate']).sum().reset_index()\nprint(f'marged shape {df_scores.shape}')\n\n# teamBoxScores \n# =========================================================================================\nCOL_TEAMSCORE = ['team_win','team_runsScored','team_runsPitching', 'team_runsdiff','team_shutout_win','team_shutout_lose']\nCOL_FEATURES = COL_FEATURES + COL_TEAMSCORE\nprint(f'team scores shape {df_team_scores.shape}')\ndf_team_scores = FE_team_score(df_team_scores)\nprint(f'team scores shape {df_team_scores.shape}')\n\n# award \n# =========================================================================================\nCOL_AWARDS = ['dailyDataDate', 'playerId','num_of_award']\nCOL_FEATURES = COL_FEATURES + ['num_of_award']\n\ndf_awards = df_awards.groupby([\"dailyDataDate\",\"playerId\"]).size().reset_index()\ndf_awards = df_awards.rename(columns={0: 'num_of_award'})\n\n# transaction\n# =========================================================================================\nCOL_TRANSACTION = ['trade']\nCOL_FEATURES = COL_FEATURES + COL_TRANSACTION\n# 0行でも一応動く\ndf_transactions = df_transactions.query('typeDesc == \"Trade\"').dropna(subset=['playerId']).reset_index(drop=True)\ndf_transactions = df_transactions[['dailyDataDate','playerId']]\ndf_transactions = df_transactions.drop_duplicates().reset_index(drop=True)\ndf_transactions['trade'] = 1\n\n# twitter\n# =========================================================================================\n# twitter_players = pd.read_pickle(UNNESTED_DIR / 'train_playerTwitterFollowers.pickle')\n# twitter_team = pd.read_pickle(UNNESTED_DIR / 'train_teamTwitterFollowers.pickle')\n# \n# df_train['yearmonth'] = df_train['dailyDataDate'].astype(str).str[:6].astype(np.int64)\n# twitter_players['yearmonth'] = twitter_players['dailyDataDate'].astype(str).str[:6].astype(np.int64)\n# twitter_team['yearmonth'] = twitter_team['dailyDataDate'].astype(str).str[:6].astype(np.int64)\n# \n# twitter_players = twitter_players.rename(columns={'numberOfFollowers': 'numberOfFollowers_player'})\n# twitter_team = twitter_team.rename(columns={'numberOfFollowers': 'numberOfFollowers_team'})\n# \n# df_train = df_train.merge(twitter_players[['yearmonth', 'playerId','numberOfFollowers_player']], on=['yearmonth', 'playerId'], how='left')\n# df_train = df_train.merge(twitter_team[['yearmonth', 'teamId','numberOfFollowers_team']], on=['yearmonth', 'teamId'], how='left')\n# df_train[['numberOfFollowers_player','numberOfFollowers_team']] = df_train[['numberOfFollowers_player','numberOfFollowers_team']].fillna(-1)\n# df_train = df_train.drop(columns='yearmonth')\n# COL_TWITTER = ['numberOfFollowers_player', 'numberOfFollowers_team']\n# COL_FEATURES = COL_FEATURES + COL_TWITTER\n","metadata":{"execution":{"iopub.status.busy":"2021-08-02T23:55:05.854300Z","iopub.execute_input":"2021-08-02T23:55:05.854751Z","iopub.status.idle":"2021-08-02T23:55:06.967710Z","shell.execute_reply.started":"2021-08-02T23:55:05.854712Z","shell.execute_reply":"2021-08-02T23:55:06.966670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nprint(df_targets.shape)\n# Focus on regular season data\ndf_targets = df_targets.query('20180329 <= dailyDataDate <= 20181001 | \\\n                               20190328 <= dailyDataDate <= 20190929 | \\\n                               20200723 <= dailyDataDate <= 20200927 | \\\n                               20210401 <= dailyDataDate').reset_index(drop=True)\n\nprint(f\"filtered{df_targets.shape}\")\n# Create train dataframe\ndf_train = df_targets.merge(df_players[COL_PLAYERS], on=['playerId'],how='left')\ngc.collect()\nprint(df_train.shape, 'after_players')\nprint('--------------------------------------')\n\ndf_train = df_train.merge(df_rosters[COL_ROSTERS], on=['playerId','dailyDataDate'], how='left')\ngc.collect()\nprint(df_train.shape, 'after_rosters')\nprint('--------------------------------------')\n\ndf_train = df_train.merge(df_scores[COL_SCORES], on=['playerId','dailyDataDate'], how='left')\ngc.collect()\nprint(df_train.shape, 'after_scores')\nprint('--------------------------------------')\n\ndf_train = df_train.merge(df_team_scores[['dailyDataDate','teamId'] + COL_TEAMSCORE], on=['dailyDataDate','teamId'], how= 'left')\ngc.collect()\nprint(df_train.shape, 'after_team_scores')\nprint('--------------------------------------')\n\ndf_train = df_train.merge(df_teams[COL_TEAMS], on=['teamId'], how='left')\ngc.collect()\nprint(df_train.shape, 'after_teams')\nprint('--------------------------------------')\n\ndf_train = df_train.merge(df_standings[['dailyDataDate', 'teamId'] + COL_STANDINGS], on=['dailyDataDate','teamId'], how= 'left')\ngc.collect()\nprint(df_train.shape, 'after_standings')\nprint('--------------------------------------')\n\ndf_train = df_train.merge(df_awards[COL_AWARDS], on=['dailyDataDate', 'playerId'], how='left')\ngc.collect()\nprint(df_train.shape, 'after_awards')\nprint('--------------------------------------')\nprint(df_train.shape)\n\ndf_train = df_train.merge(df_transactions[['dailyDataDate','playerId'] + COL_TRANSACTION], on=['dailyDataDate','playerId'], how='left')\ngc.collect()\nprint(df_train.shape, 'after_transactions')\nprint('--------------------------------------')\nprint(df_train.shape)\n# print object columns\ndf_train.select_dtypes(include=['object']).columns","metadata":{"execution":{"iopub.status.busy":"2021-08-02T23:55:06.968886Z","iopub.execute_input":"2021-08-02T23:55:06.969168Z","iopub.status.idle":"2021-08-02T23:55:14.140214Z","shell.execute_reply.started":"2021-08-02T23:55:06.969141Z","shell.execute_reply":"2021-08-02T23:55:14.138905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 打点がチームの得点の何割か /What percentage of the team's score is RBI?","metadata":{}},{"cell_type":"code","source":"df_train['rbi_teamruns'] = df_train['rbi'] / df_train['team_runsScored']\nCOL_FEATURES = COL_FEATURES + ['rbi_teamruns']","metadata":{"execution":{"iopub.status.busy":"2021-08-02T23:55:14.141705Z","iopub.execute_input":"2021-08-02T23:55:14.142102Z","iopub.status.idle":"2021-08-02T23:55:14.152827Z","shell.execute_reply.started":"2021-08-02T23:55:14.142067Z","shell.execute_reply":"2021-08-02T23:55:14.151669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 記述統計量の追加 / add stats","metadata":{"execution":{"iopub.status.busy":"2021-08-03T00:04:15.300779Z","iopub.execute_input":"2021-08-03T00:04:15.301540Z","iopub.status.idle":"2021-08-03T00:04:15.307769Z","shell.execute_reply.started":"2021-08-03T00:04:15.301482Z","shell.execute_reply":"2021-08-03T00:04:15.306837Z"}}},{"cell_type":"code","source":"if os.path.isfile('./player_target_stats.csv'):\n    df_stats = pd.read_csv('./player_target_stats.csv')\n    df_train['year'] = df_train['dailyDataDate'].astype(str).str[:4].astype(np.int64)\n    df_train['month'] = df_train['dailyDataDate'].astype(str).str[4:6].astype(np.int64)\n    df_train = df_train.merge(df_stats, on=['year','month', 'playerId'], how='left')\n    df_train = df_train.drop(columns=['year', 'month'])\n    df_stats = df_stats.drop(columns=['month'])\nelse:\n    df_stats = pd.read_csv('../input/mlb-features/player_target_stats_pattern3.csv')\n    df_train['year'] = df_train['dailyDataDate'].astype(str).str[:4].astype(np.int64)\n    df_train = df_train.merge(df_stats, on=['year', 'playerId'], how='left')\n    df_train = df_train.drop(columns='year')\n\nstas_feat = df_stats.columns.to_list()[2:]\nCOL_FEATURES = COL_FEATURES + stas_feat","metadata":{"execution":{"iopub.status.busy":"2021-08-02T23:55:14.154837Z","iopub.execute_input":"2021-08-02T23:55:14.155172Z","iopub.status.idle":"2021-08-02T23:55:21.125786Z","shell.execute_reply.started":"2021-08-02T23:55:14.155141Z","shell.execute_reply":"2021-08-02T23:55:21.124705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 人気選手がホームランを打ったか特徴量 / Whether a popular player hit a home run","metadata":{}},{"cell_type":"code","source":"HR_dict = {545361:'HR_Trout', 592450:'HR_Judge', 592885:'HR_Yelich', 660271:'HR_Ohtani', 660670:'HR_Acuna'}\ndef get_HR(df):\n    COL_HR = []\n    HR_list = [pd.DataFrame({'dailyDataDate' : [0,0,0,0,0],\n              'playerId': [545361, 592450, 592885, 660271, 660670],\n              'homeRuns':[0,0,0,0,0]})]\n    for key in HR_dict:\n        df_tmp = df.query(f\"playerId == {key} & homeRuns > 0\")[['dailyDataDate','playerId', 'homeRuns']]\n        HR_list.append(df_tmp)\n        COL_HR.append(HR_dict[key])\n    df_HR = pd.concat(HR_list, axis=0)\n    df_HR = df_HR.groupby(['dailyDataDate','playerId']).sum().reset_index()\n    df_HR = df_HR.pivot(index='dailyDataDate', columns='playerId', values='homeRuns')\n    df_HR = df_HR.rename(columns=HR_dict)\n    return df_HR, COL_HR\ndf_HR, COL_HR = get_HR(df_train.copy())\ndf_train = df_train.merge(df_HR, on=['dailyDataDate'], how='left')\ndf_train[COL_HR] = df_train[COL_HR].fillna(0)\n# 特徴量配列に追記\nCOL_FEATURES = COL_FEATURES + COL_HR","metadata":{"execution":{"iopub.status.busy":"2021-08-02T23:55:21.127257Z","iopub.execute_input":"2021-08-02T23:55:21.127584Z","iopub.status.idle":"2021-08-02T23:55:23.805937Z","shell.execute_reply.started":"2021-08-02T23:55:21.127553Z","shell.execute_reply":"2021-08-02T23:55:23.804779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# label encoding\nplayer2num = {c: i for i, c in enumerate(df_train['playerId'].unique())}\nposition2num = {c: i for i, c in enumerate(df_train['primaryPositionName'].unique())} # salaryデータ\nteamid2num = {c: i for i, c in enumerate(df_train['teamId'].unique())}\nstatus2num = {c: i for i, c in enumerate(df_train['statusCode'].unique())}\nleagueId2num = {c: i for i, c in enumerate(df_train['leagueId'].unique())}\ndivisionId2num = {c: i for i, c in enumerate(df_train['divisionId'].unique())}\n\ndf_train['label_playerId'] = df_train['playerId'].map(player2num)\ndf_train['primaryPositionName'] = df_train['primaryPositionName'].map(position2num)\ndf_train['teamId'] = df_train['teamId'].map(teamid2num)\ndf_train['statusCode'] = df_train['statusCode'].map(status2num)\ndf_train['leagueId'] = df_train['leagueId'].map(leagueId2num)\ndf_train['divisionId'] = df_train['divisionId'].map(divisionId2num)\nCOL_FEATURES = COL_FEATURES + ['label_playerId']","metadata":{"execution":{"iopub.status.busy":"2021-08-02T23:55:23.809535Z","iopub.execute_input":"2021-08-02T23:55:23.809847Z","iopub.status.idle":"2021-08-02T23:55:24.929808Z","shell.execute_reply.started":"2021-08-02T23:55:23.809818Z","shell.execute_reply":"2021-08-02T23:55:24.928649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### aggregate　NaN","metadata":{}},{"cell_type":"code","source":"df_train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-08-02T23:55:24.932187Z","iopub.execute_input":"2021-08-02T23:55:24.932655Z","iopub.status.idle":"2021-08-02T23:55:25.321384Z","shell.execute_reply.started":"2021-08-02T23:55:24.932610Z","shell.execute_reply":"2021-08-02T23:55:25.320295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"set(df_train.columns).difference(set(COL_FEATURES))","metadata":{"execution":{"iopub.status.busy":"2021-08-02T23:55:25.418955Z","iopub.execute_input":"2021-08-02T23:55:25.419321Z","iopub.status.idle":"2021-08-02T23:55:25.426397Z","shell.execute_reply.started":"2021-08-02T23:55:25.419287Z","shell.execute_reply":"2021-08-02T23:55:25.425161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CV Split","metadata":{}},{"cell_type":"code","source":"# save\nimport pickle\ndf_train.to_pickle('df_train.pickle')\nwith open('COL_FEATURES.pickle', mode='wb') as f:\n        pickle.dump(COL_FEATURES,f)","metadata":{"execution":{"iopub.status.busy":"2021-08-02T23:55:38.406011Z","iopub.execute_input":"2021-08-02T23:55:38.406482Z","iopub.status.idle":"2021-08-02T23:55:41.676593Z","shell.execute_reply.started":"2021-08-02T23:55:38.406441Z","shell.execute_reply":"2021-08-02T23:55:41.675663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X = df_train.query(\"dailyDataDate > 20210101\")[COL_FEATURES]\ntrain_y = df_train.query(\"dailyDataDate > 20210101\")[COL_TARGETS]\n#train_X = df_train[COL_FEATURES]\n#train_y = df_train[COL_TARGETS]\n\n_index = (df_train['dailyDataDate'] < 20210601)\nX_train = train_X.loc[_index].reset_index(drop=True)\ny_train = train_y.loc[_index].reset_index(drop=True)\nX_valid = train_X.loc[~_index].reset_index(drop=True)\ny_valid = train_y.loc[~_index].reset_index(drop=True)\nprint(X_train.shape, X_valid.shape)","metadata":{"execution":{"iopub.status.busy":"2021-08-02T23:55:49.604713Z","iopub.execute_input":"2021-08-02T23:55:49.605500Z","iopub.status.idle":"2021-08-02T23:55:50.212047Z","shell.execute_reply.started":"2021-08-02T23:55:49.605459Z","shell.execute_reply":"2021-08-02T23:55:50.211171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## model","metadata":{}},{"cell_type":"code","source":"def fit_lgbm(X_train, y_train, X_valid, y_valid,  params: dict=None, seed=42,verbose=100):\n    oof_pred = np.zeros(len(y_valid), dtype=np.float32)\n    lgb_train = lgb.Dataset(X_train, y_train)\n    lgb_valid = lgb.Dataset(X_valid, y_valid)\n    params[\"seed\"] = seed\n    model = lgb.train(params,\n                      lgb_train,\n                      #categorical_feature=['statusCode', 'primaryPositionCode'],\n                      valid_sets=[lgb_train, lgb_valid],\n                      verbose_eval=100,\n                      num_boost_round=10000,\n                      early_stopping_rounds=100\n                     )\n    oof_pred = model.predict(X_valid)\n    score = mean_absolute_error(oof_pred, y_valid)\n    print('mae:', score)\n    _ = lgb.plot_importance(model,max_num_features=20,figsize=(10,10))\n    return oof_pred, model, score","metadata":{"execution":{"iopub.status.busy":"2021-08-02T23:56:07.120386Z","iopub.execute_input":"2021-08-02T23:56:07.121110Z","iopub.status.idle":"2021-08-02T23:56:07.129813Z","shell.execute_reply.started":"2021-08-02T23:56:07.121068Z","shell.execute_reply":"2021-08-02T23:56:07.128193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# training lightgbm\nparams1 = {'objective':'mae',\n           'reg_alpha': 0.14947461820098767, \n           'reg_lambda': 0.10185644384043743, \n           'n_estimators': 3633, \n           'learning_rate': 0.08046301304430488, \n           'num_leaves': 674, \n           'feature_fraction': 0.9101240539122566, \n           'bagging_fraction': 0.9884451442950513, \n           'bagging_freq': 8, \n           'min_child_samples': 51}\n\nparams2 = {'objective':'mae',\n           'reg_alpha': 0.1,\n           'reg_lambda': 0.1, \n           'n_estimators': 80,\n           'learning_rate': 0.1,\n           'random_state': 42,\n           \"num_leaves\": 22}\n\nparams3 = {'objective':'mae',\n           'reg_alpha': 0.1,\n           'reg_lambda': 0.1, \n           'n_estimators': 10000,\n           'learning_rate': 0.1,\n           'random_state': 42,\n           \"num_leaves\": 100\n           }\n\nparams4 = {'objective':'mae',\n           'reg_alpha': 0.016468100279441976, \n           'reg_lambda': 0.09128335764019105, \n           'n_estimators': 9868, \n           'learning_rate': 0.10528150510326864, \n           'num_leaves': 157, \n           'feature_fraction': 0.5419185713426886, \n           'bagging_fraction': 0.2637405128936662, \n           'bagging_freq': 19, \n           'min_child_samples': 71}\n\n\n\noof1_1, model1_1, score1_1 = fit_lgbm(\n    X_train, y_train['target1'],\n    X_valid, y_valid['target1'],\n    params1,29\n)\noof1_2, model1_2, score1_2 = fit_lgbm(\n    X_train, y_train['target1'],\n    X_valid, y_valid['target1'],\n    params1,42\n)\noof2_1, model2_1, score2_1 = fit_lgbm(\n    X_train, y_train['target2'],\n    X_valid, y_valid['target2'],\n    params2,29\n)\noof2_2, model2_2, score2_2 = fit_lgbm(\n    X_train, y_train['target2'],\n    X_valid, y_valid['target2'],\n    params2,42\n)\noof3_1, model3_1, score3_1 = fit_lgbm(\n    X_train, y_train['target3'],\n    X_valid, y_valid['target3'],\n    params3,29\n)\noof3_2, model3_2, score3_2 = fit_lgbm(\n    X_train, y_train['target3'],\n    X_valid, y_valid['target3'],\n    params3,42\n)\noof4_1, model4_1, score4_1 = fit_lgbm(\n    X_train, y_train['target4'],\n    X_valid, y_valid['target4'],\n    params4,29\n)\noof4_2, model4_2, score4_2 = fit_lgbm(\n    X_train, y_train['target4'],\n    X_valid, y_valid['target4'],\n    params4,42\n)\nscore1 = (score1_1+score2_1+score3_1+score4_1) / 4\nscore2 = (score1_2+score2_2+score3_2+score4_2) / 4\nprint(f'score1: {score1}')\nprint(f'score2: {score2}')","metadata":{"execution":{"iopub.status.busy":"2021-08-02T23:57:01.369534Z","iopub.execute_input":"2021-08-02T23:57:01.370409Z","iopub.status.idle":"2021-08-02T23:58:54.493719Z","shell.execute_reply.started":"2021-08-02T23:57:01.370346Z","shell.execute_reply":"2021-08-02T23:58:54.492607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score1 = mean_absolute_error((oof1_1+oof1_2)/2, y_valid['target1'])\nscore2 = mean_absolute_error((oof2_1+oof2_2)/2, y_valid['target2'])\nscore3 = mean_absolute_error((oof3_1+oof3_2)/2, y_valid['target3'])\nscore4 = mean_absolute_error((oof4_1+oof4_2)/2, y_valid['target4'])\nscore = (score1+score2+score3+score4) / 4\nprint(f'score: {score}')","metadata":{"execution":{"iopub.status.busy":"2021-08-03T00:18:09.235372Z","iopub.execute_input":"2021-08-03T00:18:09.235846Z","iopub.status.idle":"2021-08-03T00:18:09.254373Z","shell.execute_reply.started":"2021-08-03T00:18:09.235812Z","shell.execute_reply":"2021-08-03T00:18:09.253172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## train on alldata","metadata":{}},{"cell_type":"code","source":"def fit_lgbm_all(X_train, y_train, params: dict=None, seed=42, verbose=100):\n    lgb_train = lgb.Dataset(X_train, y_train)\n    params[\"seed\"] = seed\n    model = lgb.train(params,lgb_train)\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-08-03T00:19:17.032116Z","iopub.execute_input":"2021-08-03T00:19:17.032942Z","iopub.status.idle":"2021-08-03T00:19:17.039246Z","shell.execute_reply.started":"2021-08-03T00:19:17.032882Z","shell.execute_reply":"2021-08-03T00:19:17.038219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X = df_train.query(\"dailyDataDate > 20210101\")[COL_FEATURES]\ntrain_y = df_train.query(\"dailyDataDate > 20210101\")[COL_TARGETS]\nprint(train_X.shape, train_y.shape)\nbest_param1 = dict(params1)\nbest_param1['num_iterations'] = int(model1_1.best_iteration*1.1)\nmodel1_1 = fit_lgbm_all(train_X, train_y['target1'], best_param1, 29)\nbest_param1['num_iterations'] = int(model1_2.best_iteration*1.1)\nmodel1_2 = fit_lgbm_all(train_X, train_y['target1'], best_param1, 42)\n\nbest_param2 = dict(params2)\nbest_param2['num_iterations'] = int(model2_1.best_iteration*1.1)\nmodel2_1 = fit_lgbm_all(train_X, train_y['target2'], best_param2, 29)\nbest_param2['num_iterations'] = int(model2_2.best_iteration*1.1)\nmodel2_2 = fit_lgbm_all(train_X, train_y['target2'], best_param2, 42)\n\nbest_param3 = dict(params3)\nbest_param3['num_iterations'] = int(model3_1.best_iteration*1.1)\nmodel3_1 = fit_lgbm_all(train_X, train_y['target3'], best_param3, 29)\nbest_param3['num_iterations'] = int(model3_2.best_iteration*1.1)\nmodel3_2 = fit_lgbm_all(train_X, train_y['target3'], best_param3, 42)\n\nbest_param4 = dict(params4)\nbest_param4['num_iterations'] = int(model4_1.best_iteration*1.1)\nmodel4_1 = fit_lgbm_all(train_X, train_y['target4'], best_param4, 29)\nbest_param4['num_iterations'] = int(model4_2.best_iteration*1.1)\nmodel4_2 = fit_lgbm_all(train_X, train_y['target4'], best_param4, 42)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T00:19:22.899113Z","iopub.execute_input":"2021-08-03T00:19:22.899820Z","iopub.status.idle":"2021-08-03T01:03:52.386848Z","shell.execute_reply.started":"2021-08-03T00:19:22.899757Z","shell.execute_reply":"2021-08-03T01:03:52.383325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del(train_X, train_y, df_games, df_targets)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-08-03T01:03:52.389419Z","iopub.status.idle":"2021-08-03T01:03:52.389883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference & emulator","metadata":{}},{"cell_type":"code","source":"import os\nimport warnings\nfrom typing import Optional, Tuple\n\n\nclass Environment:\n    def __init__(self,\n                 data_dir: str,\n                 eval_start_day: int,\n                 eval_end_day: Optional[int],\n                 use_updated: bool,\n                 multiple_days_per_iter: bool):\n        warnings.warn('this is mock module for mlb')\n\n        postfix = '_updated' if use_updated else ''\n        \n        # recommend to replace this with pickle, feather etc to speedup preparing data\n        df_train = pd.read_pickle(os.path.join(data_dir, f'train{postfix}.pkl'))\n\n        players = pd.read_pickle('../input/mlb-unnested/players.pickle')\n\n        self.players = players[players['playerForTestSetAndFuturePreds'] == True]['playerId'].astype(str)\n        if eval_end_day is not None:\n            self.df_train = df_train.set_index('date').loc[eval_start_day:eval_end_day]\n        else:\n            self.df_train = df_train.set_index('date').loc[eval_start_day:]\n        self.date = self.df_train.index.values\n        self.n_rows = len(self.df_train)\n        self.multiple_days_per_iter = multiple_days_per_iter\n\n        assert self.n_rows > 0, 'no data to emulate'\n\n    def predict(self, df: pd.DataFrame) -> None:\n        # if you want to emulate public LB, store your prediction here and calculate MAE\n        pass\n\n    def iter_test(self) -> Tuple[pd.DataFrame, pd.DataFrame]:\n        if self.multiple_days_per_iter:\n            for i in range(self.n_rows // 2):\n                date1 = self.date[2 * i]\n                date2 = self.date[2 * i + 1]\n                sample_sub1 = self._make_sample_sub(date1)\n                sample_sub2 = self._make_sample_sub(date2)\n                sample_sub = pd.concat([sample_sub1, sample_sub2]).reset_index(drop=True)\n                df = self.df_train.loc[date1:date2]\n\n                yield df, sample_sub.set_index('date')\n        else:\n            for i in range(self.n_rows):\n                date = self.date[i]\n                sample_sub = self._make_sample_sub(date)\n                df = self.df_train.loc[date:date]\n\n                yield df, sample_sub.set_index('date')\n\n    def _make_sample_sub(self, date: int) -> pd.DataFrame:\n        next_day = (pd.to_datetime(date, format='%Y%m%d') + pd.to_timedelta(1, 'd')).strftime('%Y%m%d')\n        sample_sub = pd.DataFrame()\n        sample_sub['date_playerId'] = next_day + '_' + self.players\n        sample_sub['target1'] = 0\n        sample_sub['target2'] = 0\n        sample_sub['target3'] = 0\n        sample_sub['target4'] = 0\n        sample_sub['date'] = date\n        return sample_sub\n    \nclass MLBEmulator:\n    def __init__(self,\n                 data_dir: str = '../input/mlb-features',\n                 eval_start_day: int = 20210401,\n                 eval_end_day: Optional[int] = 20210430,\n                 use_updated: bool = True,\n                 multiple_days_per_iter: bool = False):\n        self.data_dir = data_dir\n        self.eval_start_day = eval_start_day\n        self.eval_end_day = eval_end_day\n        self.use_updated = use_updated\n        self.multiple_days_per_iter = multiple_days_per_iter\n\n    def make_env(self) -> Environment:\n        return Environment(self.data_dir,\n                           self.eval_start_day,\n                           self.eval_end_day,\n                           self.use_updated,\n                           self.multiple_days_per_iter)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T01:03:52.390844Z","iopub.status.idle":"2021-08-03T01:03:52.391276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df, verbose=False):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2\n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int64)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float32)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float64)\n                else:\n                    df[col] = df[col].astype(np.float64)\n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df\n\ndef get_unnested_data(df: pd.DataFrame, sample_prediction_df: pd.DataFrame):\n    # ['games', 'rosters', 'playerBoxScores', 'teamBoxScores', 'transactions', 'standings', 'awards', 'events']\n    #daily_data_nested_df_names = df.drop('date', axis = 1).columns.values.tolist()\n    daily_data_nested_df_names =['games', 'rosters', 'playerBoxScores', 'teamBoxScores','awards','transactions','standings']\n    dfs_dict = {}\n    for df_name in daily_data_nested_df_names:\n        #print(df_name)\n        date_nested_table = df[['date', df_name]]\n        \n        date_nested_table = (date_nested_table[\n            ~pd.isna(date_nested_table[df_name])\n            ].reset_index(drop = True))\n        #Dealing with missing values\n        #print(len(date_nested_table))\n        daily_dfs_collection = []\n        \n        if len(date_nested_table) == 0:\n            daily_df = pd.DataFrame({'dailyDataDate':sample_prediction_df['dailyDataDate'],\n                                     'playerId': sample_prediction_df['playerId']})\n            for col in columns_dict[df_name]:\n                if col in ['dailyDataDate', 'playerId']: continue\n                daily_df[col] = np.nan\n            daily_dfs_collection = daily_dfs_collection + [daily_df]\n        else:\n            for date_index, date_row in date_nested_table.iterrows():\n                daily_df = pd.read_json(date_row[df_name])\n                daily_df['dailyDataDate'] = date_row['date']\n                daily_dfs_collection = daily_dfs_collection + [daily_df]\n            \n        unnested_table = (pd.concat(daily_dfs_collection,\n            ignore_index = True).\n            # Set and reset index to move 'dailyDataDate' to front of df\n            set_index('dailyDataDate').\n            reset_index()\n            )\n        reduce_mem_usage(unnested_table).to_pickle(f\"test_{df_name}.pickle\")\n        dfs_dict[df_name] = reduce_mem_usage(unnested_table)\n        del(date_nested_table, daily_dfs_collection, unnested_table)\n    return dfs_dict","metadata":{"execution":{"iopub.status.busy":"2021-08-03T01:03:52.392153Z","iopub.status.idle":"2021-08-03T01:03:52.392564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def inference(test_df, sample_prediction_df):\n    dfs_dict = get_unnested_data(test_df, sample_prediction_df)\n    df_test_rosters = dfs_dict['rosters']\n    df_test_games = dfs_dict['games']\n    df_test_scores = dfs_dict['playerBoxScores']\n    df_test_team_scores = dfs_dict['teamBoxScores']\n    df_test_awards = dfs_dict['awards']\n    df_test_transactions = dfs_dict['transactions']\n    df_test_standings = dfs_dict['standings']\n\n    # FE\n    # ==========================================\n    df_test_team_scores = FE_team_score(df_test_team_scores)\n\n    df_test_scores = df_test_scores.groupby(['playerId','dailyDataDate']).sum().reset_index()\n    #df_test_scores = df_test_scores.drop_duplicates(subset=['playerId','dailyDataDate']).reset_index()\n    df_test = sample_prediction_df[['playerId','dailyDataDate']].copy()\n    df_test = df_test.merge(df_players[COL_PLAYERS], on=['playerId'],how='left')\n    df_test = df_test.merge(df_test_rosters[COL_ROSTERS], on=['playerId','dailyDataDate'], how='left')\n    df_test = df_test.merge(df_test_scores[COL_SCORES], on=['playerId','dailyDataDate'], how='left')\n    df_test = df_test.merge(df_test_team_scores[['dailyDataDate','teamId'] + COL_TEAMSCORE], on=['dailyDataDate','teamId'], how= 'left')\n    df_test = df_test.merge(df_teams[COL_TEAMS],on=['teamId'], how='left')\n    \n    #standings\n    if test_df['standings'].iloc[0] == test_df['standings'].iloc[0]: # nanだとelseに行く\n        df_test = df_test.merge(df_test_standings[['dailyDataDate', 'teamId'] + COL_STANDINGS], on=['dailyDataDate','teamId'], how= 'left')\n    else:\n        df_test[COL_STANDINGS] = np.nan\n    # awards\n    df_test_awards = df_test_awards.dropna(how='any')\n    if len(df_test_awards) > 0:\n        df_test_awards = df_test_awards.groupby([\"dailyDataDate\",\"playerId\"]).size().reset_index()\n        df_test_awards = df_test_awards.rename(columns={0: 'num_of_award'})\n        df_test = df_test.merge(df_test_awards[COL_AWARDS], on=['dailyDataDate', 'playerId'], how='left')\n    else:\n        df_test['num_of_award'] = np.nan\n        \n    # transaction\n    df_test_transactions = df_test_transactions.query('typeDesc == \"Trade\"').dropna(subset=['playerId']).reset_index(drop=True)\n    if len(df_test_transactions) > 0:\n        df_test_transactions = df_test_transactions[[\"dailyDataDate\",\"playerId\"]]\n        df_test_transactions = df_test_transactions.drop_duplicates().reset_index(drop=True)\n        df_test_transactions['trade'] = 1\n        df_test = df_test.merge(df_test_transactions[[\"dailyDataDate\",\"playerId\"] + COL_TRANSACTION], on=[\"dailyDataDate\",\"playerId\"], how='left')\n    else:\n        df_test['trade'] = 0\n    # rbiの割合\n    df_test['rbi_teamruns'] = df_test['rbi'] / df_train['team_runsScored']\n    \n    # 記述統計\n    if os.path.isfile('./player_target_stats.csv'):\n        df_stats = pd.read_csv('./player_target_stats.csv')\n        df_test['year'] = df_test['dailyDataDate'].astype(str).str[:4].astype(np.int64)\n        df_test['month'] = df_test['dailyDataDate'].astype(str).str[4:6].astype(np.int64)\n        df_test = df_test.merge(df_stats, on=['year','month', 'playerId'], how='left')\n        df_test = df_test.drop(columns=['year', 'month'])\n    else:\n        df_stats = pd.read_csv('../input/mlb-features/player_target_stats_pattern3.csv')\n        df_test['year'] = df_test['dailyDataDate'].astype(str).str[:4].astype(np.int64)\n        df_test = df_test.merge(df_stats, on=['year', 'playerId'], how='left')\n        df_test = df_test.drop(columns='year')\n    \n    # HR\n    df_HR, _ = get_HR(df_test.copy())\n    if len(df_HR) > 0:\n        df_test = df_test.merge(df_HR, on=['dailyDataDate'], how='left')\n        df_test[COL_HR] = df_test[COL_HR].fillna(0)\n    else:\n        df_test[COL_HR] = 0\n    \n    # Label Encoding\n    df_test['label_playerId'] = df_test['playerId'].map(player2num)\n    df_test['primaryPositionName'] = df_test['primaryPositionName'].map(position2num)\n    df_test['teamId'] = df_test['teamId'].map(teamid2num)\n    df_test['statusCode'] = df_test['statusCode'].map(status2num)\n    df_test['leagueId'] = df_test['leagueId'].map(leagueId2num)\n    df_test['divisionId'] = df_test['divisionId'].map(divisionId2num)\n    \n    test_X = df_test[COL_FEATURES]\n    # predict\n    pred1_1 = model1_1.predict(test_X)\n    pred2_1 = model2_1.predict(test_X)\n    pred3_1 = model3_1.predict(test_X)\n    pred4_1 = model4_1.predict(test_X)\n    pred1_2 = model1_2.predict(test_X)\n    pred2_2 = model2_2.predict(test_X)\n    pred3_2 = model3_2.predict(test_X)\n    pred4_2 = model4_2.predict(test_X)\n    \n    # merge submission\n    sample_prediction_df['target1'] = np.clip((pred1_1 + pred1_2)/2, 0, 100)\n    sample_prediction_df['target2'] = np.clip((pred2_1 + pred2_2)/2, 0, 100)\n    sample_prediction_df['target3'] = np.clip((pred3_1 + pred3_2)/2, 0, 100)\n    sample_prediction_df['target4'] = np.clip((pred4_1 + pred4_2)/2, 0, 100)\n    \n    #　大谷\n    if df_test[\"HR_Ohtani\"][0] > 0:\n        sample_prediction_df.loc[sample_prediction_df['playerId'] ==660271, ['target1', 'target2', 'target3', 'target4']] = 100   \n    \n    sample_prediction_df = sample_prediction_df.fillna(0.)\n    del sample_prediction_df['playerId'], sample_prediction_df['dailyDataDate']\n    \n    return sample_prediction_df\n    #env.predict(sample_prediction_df)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T01:03:52.393432Z","iopub.status.idle":"2021-08-03T01:03:52.393829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"emulation_mode = False\n\nif emulation_mode:\n    mlb = MLBEmulator(eval_start_day=20210501, eval_end_day=20210531)\nelse:\n    import mlb\nenv = mlb.make_env()# initialize the environment\n\niter_test = env.iter_test() # iterator which loops over each date in test set\n\nfor (test_df, sample_prediction_df) in iter_test: # make predictions here\n    #sample_prediction_df = sample_prediction_df.reset_index(drop=True)\n    sample_prediction_df = sample_prediction_df.reset_index().rename(columns={'date':'dailyDataDate'})\n    sample_prediction_df['playerId'] = sample_prediction_df['date_playerId']\\\n                                        .map(lambda x: int(x.split('_')[1]))\n    \n    # ==========================================\n    test_df = test_df.reset_index().rename(columns={'index':'date'})\n    \n    sample_prediction_df = inference(test_df, sample_prediction_df)\n    \n    env.predict(sample_prediction_df)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T01:03:52.394673Z","iopub.status.idle":"2021-08-03T01:03:52.395081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The emulator","metadata":{}},{"cell_type":"markdown","source":"score1.2487188781157268","metadata":{}},{"cell_type":"code","source":"mlb = MLBEmulator(eval_start_day=20210501, eval_end_day=20210531)\nenv = mlb.make_env()# initialize the environment\niter_test = env.iter_test() # iterator which loops over each date in test set\ncol = ['target1_', 'target2_', 'target3_', 'target4_']\noof_preds = []\nscores=0\nfor n, (test_df, sample_prediction_df) in enumerate(iter_test):\n    #sample_prediction_df = sample_prediction_df.reset_index(drop=True)\n    sample_prediction_df = sample_prediction_df.reset_index().rename(columns={'date':'dailyDataDate'})\n    sample_prediction_df['playerId'] = sample_prediction_df['date_playerId']\\\n                                        .map(lambda x: int(x.split('_')[1]))\n    \n    # ==========================================\n    test_df = test_df.reset_index().rename(columns={'index':'date'})\n    sample_prediction_df = inference(test_df, sample_prediction_df)\n    \n    #env.predict(sample_prediction_df)\n    \n    targets = pd.read_json(test_df['nextDayPlayerEngagement'][0])\n    targets.columns = ['engagementMetricsDate', 'playerId'] + col\n    sample_prediction_df['playerId'] = sample_prediction_df['date_playerId']\\\n                                        .map(lambda x: int(x.split('_')[1]))\n    oof_pred = sample_prediction_df.merge(targets[['playerId'] + col], on='playerId', how='left')\n    # mae by day\n    score = mean_absolute_error(oof_pred[['target1', 'target2', 'target3', 'target4']].values, oof_pred[col].values)\n    print(f\"{score}\")\n    scores += score\n    oof_preds.append(oof_pred)\n\noof_df = pd.concat(oof_preds,axis=0).reset_index(drop=True)\nprint('=*'*30)\nprint(f\"score{scores/len(oof_preds)}\")","metadata":{"execution":{"iopub.status.busy":"2021-08-03T01:03:52.396109Z","iopub.status.idle":"2021-08-03T01:03:52.396537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### nan test","metadata":{}},{"cell_type":"code","source":"# _test_df = [pd.read_csv('../input/mlb-player-digital-engagement-forecasting/example_test.csv').query(\"date == 20210426\"),\n#             pd.read_csv('../input/mlb-player-digital-engagement-forecasting/example_test.csv').query(\"date == 20210427\"),\n#             pd.read_csv('../input/mlb-player-digital-engagement-forecasting/example_test.csv').query(\"date == 20210428\"),\n#             pd.read_csv('../input/mlb-player-digital-engagement-forecasting/example_test.csv').query(\"date == 20210429\"),\n#             pd.read_csv('../input/mlb-player-digital-engagement-forecasting/example_test.csv').query(\"date == 20210430\")]\n# sample = [pd.read_csv('../input/mlb-player-digital-engagement-forecasting/example_sample_submission.csv').query('date == 20210426').set_index('date'),\n#           pd.read_csv('../input/mlb-player-digital-engagement-forecasting/example_sample_submission.csv').query('date == 20210427').set_index('date'),\n#           pd.read_csv('../input/mlb-player-digital-engagement-forecasting/example_sample_submission.csv').query('date == 20210428').set_index('date'),\n#           pd.read_csv('../input/mlb-player-digital-engagement-forecasting/example_sample_submission.csv').query('date == 20210429').set_index('date'),\n#           pd.read_csv('../input/mlb-player-digital-engagement-forecasting/example_sample_submission.csv').query('date == 20210430').set_index('date')]\n# # nanのテスト\n# _test_df[0].iloc[:,1:] = np.nan\n# pred = []\n# for i in range(5):\n#     test_df = _test_df[i]\n#     sample_prediction_df = sample[i]\n#     #sample_prediction_df = sample_prediction_df.reset_index(drop=True)\n#     sample_prediction_df = sample_prediction_df.reset_index().rename(columns={'date':'dailyDataDate'})\n#     sample_prediction_df['playerId'] = sample_prediction_df['date_playerId']\\\n#                                         .map(lambda x: int(x.split('_')[1]))\n\n#     sample_prediction_df = inference(test_df, sample_prediction_df)\n#     pred.append(sample_prediction_df)\n    \n#     #dfs_dict = get_unnested_data(test_df, sample_prediction_df)\n# sub = pd.concat(pred, axis=0)","metadata":{"execution":{"iopub.status.busy":"2021-07-31T09:37:07.185708Z","iopub.execute_input":"2021-07-31T09:37:07.186172Z","iopub.status.idle":"2021-07-31T09:37:07.196347Z","shell.execute_reply.started":"2021-07-31T09:37:07.186141Z","shell.execute_reply":"2021-07-31T09:37:07.195644Z"},"trusted":true},"execution_count":null,"outputs":[]}]}