{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom sklearn.metrics import mean_absolute_error\nfrom datetime import timedelta\nfrom functools import reduce\nfrom tqdm import tqdm\nimport lightgbm as lgbm\nimport mlb","metadata":{"execution":{"iopub.status.busy":"2021-06-28T12:33:31.9119Z","iopub.execute_input":"2021-06-28T12:33:31.912317Z","iopub.status.idle":"2021-06-28T12:33:34.130089Z","shell.execute_reply.started":"2021-06-28T12:33:31.912234Z","shell.execute_reply":"2021-06-28T12:33:34.129208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_DIR = Path('../input/mlb-player-digital-engagement-forecasting')\nTRAIN_DIR = Path('../input/mlb-pdef-train-dataset')","metadata":{"execution":{"iopub.status.busy":"2021-06-28T12:33:34.131377Z","iopub.execute_input":"2021-06-28T12:33:34.131633Z","iopub.status.idle":"2021-06-28T12:33:34.135585Z","shell.execute_reply.started":"2021-06-28T12:33:34.131607Z","shell.execute_reply":"2021-06-28T12:33:34.134636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"players = pd.read_csv(BASE_DIR / 'players.csv')\n\nrosters = pd.read_pickle(TRAIN_DIR / 'rosters_train.pkl')\ntargets = pd.read_pickle(TRAIN_DIR / 'nextDayPlayerEngagement_train.pkl')\nfollowers = pd.read_pickle(TRAIN_DIR / 'playerTwitterFollowers_train.pkl')\nteam_followers = pd.read_pickle(TRAIN_DIR / 'teamTwitterFollowers_train.pkl')\nteam_followers = team_followers.rename(columns={'numberOfFollowers': 'teamFollowers'})\nscores = pd.read_pickle(TRAIN_DIR / 'playerBoxScores_train.pkl')\nscores = scores.groupby(['playerId', 'date']).sum().reset_index()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T12:33:34.137471Z","iopub.execute_input":"2021-06-28T12:33:34.13813Z","iopub.status.idle":"2021-06-28T12:33:39.150415Z","shell.execute_reply.started":"2021-06-28T12:33:34.138056Z","shell.execute_reply":"2021-06-28T12:33:39.149374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os","metadata":{"execution":{"iopub.status.busy":"2021-06-28T15:39:01.841606Z","iopub.execute_input":"2021-06-28T15:39:01.842045Z","iopub.status.idle":"2021-06-28T15:39:01.846731Z","shell.execute_reply.started":"2021-06-28T15:39:01.842008Z","shell.execute_reply":"2021-06-28T15:39:01.845437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir(TRAIN_DIR)","metadata":{"execution":{"iopub.status.busy":"2021-06-28T15:39:15.886114Z","iopub.execute_input":"2021-06-28T15:39:15.886869Z","iopub.status.idle":"2021-06-28T15:39:15.90422Z","shell.execute_reply.started":"2021-06-28T15:39:15.886827Z","shell.execute_reply":"2021-06-28T15:39:15.903312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"awards = pd.read_csv(TRAIN_DIR / 'awards_train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-28T15:40:37.162695Z","iopub.execute_input":"2021-06-28T15:40:37.163056Z","iopub.status.idle":"2021-06-28T15:40:37.215934Z","shell.execute_reply.started":"2021-06-28T15:40:37.163025Z","shell.execute_reply":"2021-06-28T15:40:37.21486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"awards.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T15:40:44.968213Z","iopub.execute_input":"2021-06-28T15:40:44.968719Z","iopub.status.idle":"2021-06-28T15:40:44.998222Z","shell.execute_reply.started":"2021-06-28T15:40:44.968681Z","shell.execute_reply":"2021-06-28T15:40:44.997286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Target Visualization**","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\nsns.set_style('whitegrid')\nsns.set(font_scale = 1.5)\nfig, axs = plt.subplots(2,2, figsize = (20, 10))\nsns.kdeplot(ax=axs[0,0], data=targets['target1'])\nsns.kdeplot(ax=axs[0,1], data=targets['target2'])\nsns.kdeplot(ax=axs[1,0], data=targets['target3'])\nsns.kdeplot(ax=axs[1,1], data=targets['target4'])\nbbox = axs[0,0].get_position()\nbbox2 = axs[0,1].get_position()\n\ncenter=(bbox2.x1) * 0.4 + (bbox.x1) * 0.25\nplt.suptitle('Distribution of targets', x = center)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-28T13:03:38.275928Z","iopub.execute_input":"2021-06-28T13:03:38.276687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def draw_kde_plot(col = 'target1'):\n    sns.set_style('whitegrid')\n    sns.set(font_scale = 1.5)\n    fig, axs = plt.subplots(2,2, figsize = (15, 10))\n    g = sns.kdeplot(ax=axs[0,0], data=targets[col])\n    g.set_xlabel('original')\n    g = sns.kdeplot(ax=axs[0,1], data=targets[col]**2)\n    g.set_xlabel('squared')\n    g = sns.kdeplot(ax=axs[1,0], data=targets[col]**4)\n    g.set_xlabel('power 4')\n    g = sns.kdeplot(ax=axs[1,1], data = np.log(targets[col]+1))\n    g.set_xlabel('log')\n\n\n\n    bbox = axs[0,0].get_position()\n    bbox2 = axs[0,1].get_position()\n    center=(bbox2.x1) * 0.4 + (bbox.x1) * 0.25\n    plt.suptitle(f'Transformation of {col}', x = center)\n    plt.tight_layout()\n","metadata":{"execution":{"iopub.status.busy":"2021-06-28T14:13:32.962889Z","iopub.execute_input":"2021-06-28T14:13:32.963264Z","iopub.status.idle":"2021-06-28T14:13:32.972498Z","shell.execute_reply.started":"2021-06-28T14:13:32.963229Z","shell.execute_reply":"2021-06-28T14:13:32.971688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# **Target2 has highest Skewness**\n","metadata":{}},{"cell_type":"code","source":"for col in ['target1', 'target2', 'target3', 'target4']:\n    draw_kde_plot(col)","metadata":{"execution":{"iopub.status.busy":"2021-06-28T14:13:37.320911Z","iopub.execute_input":"2021-06-28T14:13:37.321533Z","iopub.status.idle":"2021-06-28T14:16:34.447214Z","shell.execute_reply.started":"2021-06-28T14:13:37.321496Z","shell.execute_reply":"2021-06-28T14:16:34.446308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_style('whitegrid')\nsns.set(font_scale = 1.5)\n\n\nfig, axs = plt.subplots(1,1, figsize = (20,8))\nsns.lineplot(ax=axs, x = np.arange(1,10001),\n             y = targets.sample(10000, random_state=500)['target1'],\n             legend='full', label = 'target1')\nsns.lineplot(ax=axs, x = np.arange(1,10001),\n             y = targets.sample(10000, random_state=500)['target2'],\n             legend='full', label = 'target2')\nsns.lineplot(ax=axs, x = np.arange(1,10001), \n             y = targets.sample(10000, random_state=500)['target3'], \n             legend='full', label = 'target3')\nsns.lineplot(ax=axs,x = np.arange(1,10001), \n             y = targets.sample(10000, random_state=500)['target4'], \n             legend='full', label = 'target4')\n\nbbox = axs.get_position()\ncenter=0.5*(bbox.x1)\nplt.suptitle('Comparision of targets', x = center)\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-28T13:06:40.993253Z","iopub.execute_input":"2021-06-28T13:06:40.993857Z","iopub.status.idle":"2021-06-28T13:06:44.079169Z","shell.execute_reply.started":"2021-06-28T13:06:40.993819Z","shell.execute_reply":"2021-06-28T13:06:44.077914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_style('ticks')\nsns.set(font_scale = 1.5)\n\n\nfig, axs = plt.subplots(2,2, figsize = (20,8))\nsns.lineplot(ax=axs[0,0], x = np.arange(1,10001),\n             y = targets.sample(10000, random_state=500)['target1'],\n             legend='full', label = 'target1')\nsns.lineplot(ax=axs[0,1], x = np.arange(1,10001),\n             y = targets.sample(10000, random_state=500)['target2'],\n             legend='full', label = 'target2')\nsns.lineplot(ax=axs[1,0], x = np.arange(1,10001), \n             y = targets.sample(10000, random_state=500)['target3'], \n             legend='full', label = 'target3')\nsns.lineplot(ax=axs[1,1], x = np.arange(1,10001), \n             y = targets.sample(10000, random_state=500)['target4'], \n             legend='full', label = 'target4')\n\nplt.title('Comparision of targets, side by side view')\n","metadata":{"execution":{"iopub.status.busy":"2021-06-28T12:58:25.729372Z","iopub.execute_input":"2021-06-28T12:58:25.729802Z","iopub.status.idle":"2021-06-28T12:58:29.242705Z","shell.execute_reply.started":"2021-06-28T12:58:25.729764Z","shell.execute_reply":"2021-06-28T12:58:29.241573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets['year'] = pd.to_datetime(targets['date'], format = '%Y%m%d').dt.year","metadata":{"execution":{"iopub.status.busy":"2021-06-28T14:21:23.328604Z","iopub.execute_input":"2021-06-28T14:21:23.329134Z","iopub.status.idle":"2021-06-28T14:21:23.608228Z","shell.execute_reply.started":"2021-06-28T14:21:23.329095Z","shell.execute_reply":"2021-06-28T14:21:23.607304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# We have less data for year 4, since we need to predict for the future","metadata":{}},{"cell_type":"markdown","source":"May be we should have different validation strategy","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"targets['year'].value_counts().plot(kind = 'bar')","metadata":{"execution":{"iopub.status.busy":"2021-06-28T14:23:40.241046Z","iopub.execute_input":"2021-06-28T14:23:40.241487Z","iopub.status.idle":"2021-06-28T14:23:40.397989Z","shell.execute_reply.started":"2021-06-28T14:23:40.241412Z","shell.execute_reply":"2021-06-28T14:23:40.397183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\nsns.set(style=\"whitegrid\")\nsns.set(font_scale = 2)\nsns.color_palette(\"Set2\")\n\n\n\nfig, axs = plt.subplots(4,1, figsize = (20,20))\nsns.lineplot(ax=axs[0], \n             x = np.arange(1,10001),\n             data = targets.sample(10000, random_state=100),\n             y = 'target1',\n             hue = 'year',\n             palette='tab10',\n             linewidth=2.5)\n\nsns.lineplot(ax=axs[1], \n             x = np.arange(1,10001),\n             data = targets.sample(10000, random_state=100),\n             y = 'target2',\n             hue = 'year',\n             palette='tab10',\n             linewidth=2.5)\n\nsns.lineplot(ax=axs[2], \n             x = np.arange(1,10001),\n             data = targets.sample(10000, random_state=100),\n             y = 'target3',\n             hue = 'year',\n             palette='tab10',\n             linewidth=2.5)\n\nsns.lineplot(ax=axs[3], \n             x = np.arange(1,10001),\n             data = targets.sample(10000, random_state=100),\n             y = 'target4',\n             hue = 'year',\n             palette='tab10',\n             linewidth=2.5)\n\nbbox = axs[0].get_position()\ncenter=0.5*(bbox.x1)\nplt.suptitle('targets over years', x = center)","metadata":{"execution":{"iopub.status.busy":"2021-06-28T14:26:44.597433Z","iopub.execute_input":"2021-06-28T14:26:44.597808Z","iopub.status.idle":"2021-06-28T14:26:49.000243Z","shell.execute_reply.started":"2021-06-28T14:26:44.597777Z","shell.execute_reply":"2021-06-28T14:26:48.99921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef plot_target_for_player(col = 'target1', playerid = 683734):\n    \n    sns.set(style=\"whitegrid\")\n    sns.set(font_scale = 2)\n    sns.color_palette(\"Set2\")\n\n    fig, axs = plt.subplots(1,1, figsize = (20,8))\n\n    sns.lineplot(ax=axs, x = np.arange(365),\n                 data = targets[((targets.year==2018) & (targets.playerId==playerid))],\n                 y = col,\n                 label = '2018',\n                 linewidth=2.5)\n\n    sns.lineplot(ax=axs, \n                 x =  np.arange(365),\n                 data = targets[((targets.year==2019) & (targets.playerId==playerid))],\n                 y = col,\n                 label = '2019',\n                 linewidth=2.5)\n\n    sns.lineplot(ax=axs, \n                 x =  np.arange(366),\n                 data = targets[((targets.year==2020) & (targets.playerId==playerid))],\n                 y = col,\n                 label = '2020',\n                 linewidth=2.5)\n\n    sns.lineplot(ax=axs, \n                 x =  np.arange(120),\n                 data = targets[((targets.year==2021) & (targets.playerId==playerid))],\n                 y = col,\n                 label = '2021',\n                 linewidth=2.5)\n    \n    bbox = axs.get_position()\n    center=0.5*(bbox.x1)\n    plt.suptitle(f'player Id {playerid}', x = center)","metadata":{"execution":{"iopub.status.busy":"2021-06-28T14:51:47.536741Z","iopub.execute_input":"2021-06-28T14:51:47.537343Z","iopub.status.idle":"2021-06-28T14:51:47.549167Z","shell.execute_reply.started":"2021-06-28T14:51:47.537289Z","shell.execute_reply":"2021-06-28T14:51:47.548374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# There is definitely seasonality for targets, seems like we can remove year 2018 from modelling","metadata":{}},{"cell_type":"code","source":"plot_target_for_player()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T14:51:49.567504Z","iopub.execute_input":"2021-06-28T14:51:49.568058Z","iopub.status.idle":"2021-06-28T14:51:49.995945Z","shell.execute_reply.started":"2021-06-28T14:51:49.568007Z","shell.execute_reply":"2021-06-28T14:51:49.995158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_target_for_player('target2')","metadata":{"execution":{"iopub.status.busy":"2021-06-28T14:42:09.270584Z","iopub.execute_input":"2021-06-28T14:42:09.270988Z","iopub.status.idle":"2021-06-28T14:42:09.699225Z","shell.execute_reply.started":"2021-06-28T14:42:09.270949Z","shell.execute_reply":"2021-06-28T14:42:09.698253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_target_for_player('target3')","metadata":{"execution":{"iopub.status.busy":"2021-06-28T14:42:54.27339Z","iopub.execute_input":"2021-06-28T14:42:54.273713Z","iopub.status.idle":"2021-06-28T14:42:54.69479Z","shell.execute_reply.started":"2021-06-28T14:42:54.273684Z","shell.execute_reply":"2021-06-28T14:42:54.693902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_target_for_player('target4')","metadata":{"execution":{"iopub.status.busy":"2021-06-28T14:43:30.886325Z","iopub.execute_input":"2021-06-28T14:43:30.886718Z","iopub.status.idle":"2021-06-28T14:43:31.322236Z","shell.execute_reply.started":"2021-06-28T14:43:30.886682Z","shell.execute_reply":"2021-06-28T14:43:31.321089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_target_for_player('target1',477132)","metadata":{"execution":{"iopub.status.busy":"2021-06-28T15:02:50.215447Z","iopub.execute_input":"2021-06-28T15:02:50.215826Z","iopub.status.idle":"2021-06-28T15:02:50.718452Z","shell.execute_reply.started":"2021-06-28T15:02:50.215782Z","shell.execute_reply":"2021-06-28T15:02:50.717276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_target_for_player('target2',477132)","metadata":{"execution":{"iopub.status.busy":"2021-06-28T15:03:05.604138Z","iopub.execute_input":"2021-06-28T15:03:05.604506Z","iopub.status.idle":"2021-06-28T15:03:06.100582Z","shell.execute_reply.started":"2021-06-28T15:03:05.604476Z","shell.execute_reply":"2021-06-28T15:03:06.099481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_target_for_player('target3',477132)","metadata":{"execution":{"iopub.status.busy":"2021-06-28T15:03:25.882587Z","iopub.execute_input":"2021-06-28T15:03:25.882991Z","iopub.status.idle":"2021-06-28T15:03:26.363596Z","shell.execute_reply.started":"2021-06-28T15:03:25.882954Z","shell.execute_reply":"2021-06-28T15:03:26.362695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_target_for_player('target4',477132)","metadata":{"execution":{"iopub.status.busy":"2021-06-28T15:03:39.659743Z","iopub.execute_input":"2021-06-28T15:03:39.660107Z","iopub.status.idle":"2021-06-28T15:03:40.122454Z","shell.execute_reply.started":"2021-06-28T15:03:39.660076Z","shell.execute_reply":"2021-06-28T15:03:40.12144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"awards['awardDateMonth'] = pd.to_datetime(awards['awardDate'], format = '%Y-%m-%d').dt.month\nawards['awardDateYear'] = pd.to_datetime(awards['awardDate'], format = '%Y-%m-%d').dt.year","metadata":{"execution":{"iopub.status.busy":"2021-06-28T16:08:25.57981Z","iopub.execute_input":"2021-06-28T16:08:25.580203Z","iopub.status.idle":"2021-06-28T16:08:25.594003Z","shell.execute_reply.started":"2021-06-28T16:08:25.580168Z","shell.execute_reply":"2021-06-28T16:08:25.59283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_awards(playerid=477132):\n    if len(awards[awards.playerId==playerid]['awardDateYear'].value_counts()) > 0:\n        awards[awards.playerId==playerid]['awardDateYear'].value_counts().plot(kind = 'bar')\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T16:08:27.744956Z","iopub.execute_input":"2021-06-28T16:08:27.745365Z","iopub.status.idle":"2021-06-28T16:08:27.750451Z","shell.execute_reply.started":"2021-06-28T16:08:27.745335Z","shell.execute_reply":"2021-06-28T16:08:27.749616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_awards(477132)","metadata":{"execution":{"iopub.status.busy":"2021-06-28T16:08:30.16092Z","iopub.execute_input":"2021-06-28T16:08:30.16134Z","iopub.status.idle":"2021-06-28T16:08:30.289762Z","shell.execute_reply.started":"2021-06-28T16:08:30.161305Z","shell.execute_reply":"2021-06-28T16:08:30.288492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# There seems to be relationship between number of awards and targets, higher awards the player is popular","metadata":{}},{"cell_type":"code","source":"from scipy.stats import boxcox\nxt, _ = boxcox(targets['target1'].values + 1)\nsns.distplot(xt)","metadata":{"execution":{"iopub.status.busy":"2021-06-28T16:24:18.276679Z","iopub.execute_input":"2021-06-28T16:24:18.277103Z","iopub.status.idle":"2021-06-28T16:24:36.254768Z","shell.execute_reply.started":"2021-06-28T16:24:18.277051Z","shell.execute_reply":"2021-06-28T16:24:36.254073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xt, _ = boxcox(targets['target4'].values + 1)\nsns.distplot(xt)","metadata":{"execution":{"iopub.status.busy":"2021-06-28T16:31:50.061621Z","iopub.execute_input":"2021-06-28T16:31:50.062361Z","iopub.status.idle":"2021-06-28T16:32:06.890243Z","shell.execute_reply.started":"2021-06-28T16:31:50.0623Z","shell.execute_reply":"2021-06-28T16:32:06.889251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xt, _ = boxcox(targets['target3'].values + 1)\nsns.distplot(xt)","metadata":{"execution":{"iopub.status.busy":"2021-06-28T16:34:20.411902Z","iopub.execute_input":"2021-06-28T16:34:20.412356Z","iopub.status.idle":"2021-06-28T16:34:39.63988Z","shell.execute_reply.started":"2021-06-28T16:34:20.412314Z","shell.execute_reply":"2021-06-28T16:34:39.638877Z"},"trusted":true},"execution_count":null,"outputs":[]}]}