{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from PIL import Image\nimg=Image.open('../input/mlb-img/mlb.jpg')\nimg","metadata":{"_kg_hide-input":true,"_kg_hide-output":false,"execution":{"iopub.status.busy":"2021-06-27T06:09:22.844256Z","iopub.execute_input":"2021-06-27T06:09:22.844674Z","iopub.status.idle":"2021-06-27T06:09:24.124494Z","shell.execute_reply.started":"2021-06-27T06:09:22.844638Z","shell.execute_reply":"2021-06-27T06:09:24.123162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Major League Baseball (MLB) is an American professional baseball organization and the oldest of the major professional sports leagues in the United States and Canada. A **total of 30 teams** play in Major League Baseball: **15 teams in the National League (NL)** and **15 in the American League (AL)**. The NL and AL were formed in 1876 and 1901, respectively. Beginning in 1903, the two leagues cooperated but remained legally separate entities until 2000 when they merged into a single organization led by the Commissioner of Baseball. The league is headquartered in Midtown Manhattan. \n**Most recent champion(s) :\tLos Angeles Dodgers(7th title)\nMost titles :\tNew York Yankees(27 titles)**","metadata":{}},{"cell_type":"code","source":"colorscales=['aggrnyl', 'agsunset', 'algae', 'amp', 'armyrose', 'balance','blackbody', 'bluered', 'blues', 'blugrn', 'bluyl', 'brbg',\n             'brwnyl', 'bugn', 'bupu', 'burg', 'burgyl', 'cividis', 'curl',\n             'darkmint', 'deep', 'delta', 'dense', 'earth', 'edge', 'electric',\n             'emrld', 'fall', 'geyser', 'gnbu', 'gray', 'greens', 'greys',\n             'haline', 'hot', 'hsv', 'ice', 'icefire', 'inferno', 'jet',\n             'magenta', 'magma', 'matter', 'mint', 'mrybm', 'mygbm', 'oranges',\n             'orrd', 'oryel', 'oxy', 'peach', 'phase', 'picnic', 'pinkyl',\n             'piyg', 'plasma', 'plotly3', 'portland', 'prgn', 'pubu', 'pubugn',\n             'puor', 'purd', 'purp', 'purples', 'purpor', 'rainbow', 'rdbu',\n             'rdgy', 'rdpu', 'rdylbu', 'rdylgn', 'redor', 'reds', 'solar',\n             'spectral', 'speed', 'sunset', 'sunsetdark', 'teal', 'tealgrn',\n             'tealrose', 'tempo', 'temps', 'thermal', 'tropic', 'turbid',\n             'turbo', 'twilight', 'viridis', 'ylgn', 'ylgnbu', 'ylorbr',\n             'ylorrd']\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-27T14:45:49.463584Z","iopub.execute_input":"2021-06-27T14:45:49.464252Z","iopub.status.idle":"2021-06-27T14:45:49.478020Z","shell.execute_reply.started":"2021-06-27T14:45:49.464124Z","shell.execute_reply":"2021-06-27T14:45:49.477214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings \nwarnings.filterwarnings('ignore')\nimport numpy as np\nimport pandas as pd\nfrom tqdm.notebook import tqdm\nimport json\nimport gc\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\nimport plotly.express as px\nfrom plotly import tools\nfrom plotly.offline import iplot,init_notebook_mode\ninit_notebook_mode()\nplt.style.use('seaborn-notebook')\n#%matplotlib inline\nimport random","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-27T14:45:51.025587Z","iopub.execute_input":"2021-06-27T14:45:51.026108Z","iopub.status.idle":"2021-06-27T14:45:53.373341Z","shell.execute_reply.started":"2021-06-27T14:45:51.026066Z","shell.execute_reply":"2021-06-27T14:45:53.372087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrainn=pd.read_csv('../input/mlb-player-digital-engagement-forecasting/train.csv')\nexample_test=pd.read_csv('../input/mlb-player-digital-engagement-forecasting/example_test.csv')\nawards=pd.read_csv('../input/mlb-player-digital-engagement-forecasting/awards.csv')\nseasons=pd.read_csv('../input/mlb-player-digital-engagement-forecasting/seasons.csv')\nteams=pd.read_csv('../input/mlb-player-digital-engagement-forecasting/teams.csv')\nsub=pd.read_csv('../input/mlb-player-digital-engagement-forecasting/example_sample_submission.csv')\nplayers=pd.read_csv('../input/mlb-player-digital-engagement-forecasting/players.csv')\ntrain=trainn.copy(deep=True)\n\n##process date\nall_date_train=train.date.values\ntrain['year']=[int(str(val)[:4]) for val in all_date_train]\ntrain['month']=[int(str(val)[4:6]) for val in all_date_train]\ntrain['day']=[int(str(val)[6:]) for val in all_date_train]\ntarget_cols=['target1', 'target2', 'target3', 'target4']\n\ntrain.head()","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2021-06-27T14:45:53.376595Z","iopub.execute_input":"2021-06-27T14:45:53.377056Z","iopub.status.idle":"2021-06-27T14:47:09.457422Z","shell.execute_reply.started":"2021-06-27T14:45:53.377013Z","shell.execute_reply":"2021-06-27T14:47:09.456392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Process train.csv","metadata":{}},{"cell_type":"markdown","source":"### Engagement \n","metadata":{}},{"cell_type":"code","source":"############################### ENGAGEMENT COLUMNS PARSING ################################\n\nall_nextday_engage=train.nextDayPlayerEngagement.values\nengagement_cols=['engagementMetricsDate','date_tr','playerId','target1','target2','target3','target4']\nnext_engagement_processed=dict(zip(engagement_cols,[[] for val in engagement_cols]))\n\n### convert JSON string to dict - format\nmax_len_json=0;temp=[]\nfor val in tqdm(all_nextday_engage):\n    temp.append(json.loads(val))\ntrain['nextDayPlayerEngagement']=temp\ntemp=[]\n\n### compute length of all engagements(turns out every player has 2061 engagements )\nfor val in tqdm(all_nextday_engage):\n    if len(val)>max_len_json:\n        max_len_json=len(val)\n    temp.append(len(val))\ntrain['eng_len']=temp\n\n### parse json data\nfor id1,eng in tqdm(zip(train.date.values,all_nextday_engage)):\n    for val in eng:\n        for key in val.keys():\n            next_engagement_processed[key].append(val[key])\n        next_engagement_processed['date_tr'].append(id1)\n\nengagement_df=pd.DataFrame(next_engagement_processed)\nengagement_df[engagement_df.columns[0]]=pd.to_datetime(engagement_df[engagement_df.columns[0]])\nengagement_df['day']=engagement_df['engagementMetricsDate'].dt.day\nengagement_df['month']=engagement_df['engagementMetricsDate'].dt.month\nengagement_df['year']=engagement_df['engagementMetricsDate'].dt.year\nengagement_df.sample(5)","metadata":{"execution":{"iopub.status.busy":"2021-06-26T07:35:01.626153Z","iopub.execute_input":"2021-06-26T07:35:01.626592Z","iopub.status.idle":"2021-06-26T07:35:24.359185Z","shell.execute_reply.started":"2021-06-26T07:35:01.62655Z","shell.execute_reply":"2021-06-26T07:35:24.357777Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for cols in target_cols:\n    print(f\"Skewness of {cols} : \",engagement_df[cols].skew())\nengagement_df[target_cols].describe()\n","metadata":{"execution":{"iopub.status.busy":"2021-06-26T07:35:24.361146Z","iopub.execute_input":"2021-06-26T07:35:24.361631Z","iopub.status.idle":"2021-06-26T07:35:25.179262Z","shell.execute_reply.started":"2021-06-26T07:35:24.361586Z","shell.execute_reply":"2021-06-26T07:35:25.177677Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp=engagement_df[target_cols].sample(frac=0.1)\ntemp['year']=engagement_df['year'].astype(str)\ng=sns.pairplot(temp, hue='year', height=2.5)\ng.add_legend()\nplt.suptitle(\" Target variable distribution across all Seasons\")","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:49:24.09758Z","iopub.execute_input":"2021-06-24T16:49:24.097978Z","iopub.status.idle":"2021-06-24T16:54:18.652024Z","shell.execute_reply.started":"2021-06-24T16:49:24.097946Z","shell.execute_reply":"2021-06-24T16:54:18.651249Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Games representation","metadata":{}},{"cell_type":"code","source":"############################## GAMES COLUMNS PARSINBG########################\n\nall_games=train[~train.games.isna()].games.values\ngame_keys=list(json.loads(all_games[0])[0].keys())\ngames_processed=dict(zip(game_keys,[[] for val in game_keys]))\ngames_processed['date_tr']=[]\n\n\n### parse and store data\n### select only NON-NAN entries\ntemp=train[~train.games.isna()]\n\nfor date,game in tqdm(zip(temp.date.values,all_games)):\n    game=json.loads(game)\n    for val in game:\n        for key in val.keys():\n            games_processed[key].append(val[key])\n        games_processed['date_tr'].append(date)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-27T14:53:14.732804Z","iopub.execute_input":"2021-06-27T14:53:14.733393Z","iopub.status.idle":"2021-06-27T14:53:14.953970Z","shell.execute_reply.started":"2021-06-27T14:53:14.733356Z","shell.execute_reply":"2021-06-27T14:53:14.953111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"games_df=pd.DataFrame(games_processed)\ngames_df['gameDate']=pd.to_datetime(games_df['gameDate'])\ngames_df['day']=games_df['gameDate'].dt.day\ngames_df['month']=games_df['gameDate'].dt.month\ngames_df['year']=games_df['gameDate'].dt.year\ngames_df.sample(5)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-27T14:53:16.740763Z","iopub.execute_input":"2021-06-27T14:53:16.741083Z","iopub.status.idle":"2021-06-27T14:53:16.860642Z","shell.execute_reply.started":"2021-06-27T14:53:16.741056Z","shell.execute_reply":"2021-06-27T14:53:16.859952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gametype_desc=[{\"id\":\"S\",\"description\":\"Spring Training\"},{\"id\":\"R\",\"description\":\"Regular Season\"},{\"id\":\"F\",\"description\":\"Wild Card Game\"},{\"id\":\"D\",\"description\":\"Division Series\"},{\"id\":\"L\",\"description\":\"League Championship Series\"},{\"id\":\"W\",\"description\":\"World Series\"},{\"id\":\"C\",\"description\":\"Championship\"},{\"id\":\"N\",\"description\":\"Nineteenth Century Series\"},{\"id\":\"P\",\"description\":\"Playoffs\"},{\"id\":\"A\",\"description\":\"All-Star Game\"},{\"id\":\"I\",\"description\":\"Intrasquad\"},{\"id\":\"E\",\"description\":\"Exhibition\"}]\ngametype_map=dict()\nfor val in gametype_desc:\n    gametype_map[val['id']]=val['description']\ngames_df['gameType']=games_df['gameType'].map(gametype_map)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-27T07:53:48.369384Z","iopub.execute_input":"2021-06-27T07:53:48.369764Z","iopub.status.idle":"2021-06-27T07:53:48.379451Z","shell.execute_reply.started":"2021-06-27T07:53:48.369721Z","shell.execute_reply":"2021-06-27T07:53:48.378599Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gametype_dict=games_df['gameType'].value_counts().to_dict()\ngametype_keys=gametype_dict.keys()\n\ncolors = [plt.cm.Spectral(i/float(len(gametype_keys))) for i in range(len(gametype_keys))]\n\nfig = plt.figure(figsize = (20, 8))\nax = fig.add_subplot(1,2,1)\n\nax.bar(gametype_keys, gametype_dict.values(), color = colors)\n\nfor k, v in gametype_dict.items():\n    ax.text(k, v+1, v, fontsize = 14, horizontalalignment='center', verticalalignment='center')\n\nax.tick_params(axis='x', labelrotation = 45, labelsize = 12)\nax.set_ylim(0, 7000)\nax.set_title(\"Weightage of gaming sessions\", fontsize = 14);\n#################################  season   ##########################################\nseason_dict=games_df['season'].value_counts().to_dict()\nseason_keys=list(season_dict.keys())\nfor keys in season_keys:\n    season_dict[f'Season_{keys}']=season_dict.pop(keys)\nseason_keys=season_dict.keys()\n\n\ncolors = [plt.cm.RdGy_r(i/float(len(season_keys))) for i in range(len(season_keys))]\n\n\nax1 = fig.add_subplot(1,2,2)\nax1.bar(season_keys, season_dict.values(), color = colors)\n\nfor k, v in season_dict.items():\n    ax1.text(k, v+1, v, fontsize = 14, horizontalalignment='center', verticalalignment='center')\n\nax1.tick_params(axis='x', labelrotation = 45, labelsize = 12)\nax1.set_title(\"Number of games per season\", fontsize = 14);\n\n\n#######################\ngames_df['resumedFrom']=games_df['resumedFrom'].astype(str)\ngametype_dict=games_df['resumedFrom'].value_counts().to_dict()\ngametype_keys=gametype_dict.keys()\n\ncolors = [plt.cm.Spectral(i/float(len(gametype_keys))) for i in range(len(gametype_keys))]\n\nfig = plt.figure(figsize = (20, 8))\nax = fig.add_subplot(1,2,1)\n\nax.bar(gametype_keys, gametype_dict.values(), color = colors)\n\nfor k, v in gametype_dict.items():\n    ax.text(k, v+1, v, fontsize = 14, horizontalalignment='center', verticalalignment='center')\nplt.xlabel(\"Resumed date\")\nplt.xlabel(\"Resumed date count\")\nax.tick_params(axis='x', labelrotation = 45, labelsize = 12)\nax.set_ylim(0, 9000)\nax.set_title(\"Weightage of gaming sessions\", fontsize = 14)\n\n########################################\ngames_df['resumeDate']=games_df['resumeDate'].astype(str)\nseason_dict=games_df['resumeDate'].value_counts().to_dict()\nseason_keys=list(season_dict.keys())\n\n\n\ncolors = [plt.cm.jet(i/float(len(season_keys))) for i in range(len(season_keys))]\n\n\nax1 = fig.add_subplot(1,2,2)\nax1.bar(season_keys, season_dict.values(), color = colors)\n\nfor k, v in season_dict.items():\n    ax1.text(k, v+1, v, fontsize = 14, horizontalalignment='center', verticalalignment='center')\n\nax1.tick_params(axis='x', labelrotation = 45, labelsize = 12)\nax1.set_title(\"Number of games per season\", fontsize = 14);\n","metadata":{"execution":{"iopub.status.busy":"2021-06-27T07:53:54.401142Z","iopub.execute_input":"2021-06-27T07:53:54.40151Z","iopub.status.idle":"2021-06-27T07:53:55.322844Z","shell.execute_reply.started":"2021-06-27T07:53:54.401478Z","shell.execute_reply":"2021-06-27T07:53:55.321744Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"###### Inference #####\n\n* Most number of games are played as Regular Season, Training Games comprises of roughly 1/6th of total number of games. There are very few Exhibition matches and rare ALL-Star matches.\n* There is full season data for 2018-19 and  2019-20 series,  but for season 2020-21 there is roughly 1/3rd games and the deadly Corona tookover.For 2021-22 its live now and around 1/4th games is played. \n* Total of 2 games in 2018,2020,2021 and 4 games in 2018 wasn't completed on scheduled time, so they played the following match days\n","metadata":{}},{"cell_type":"markdown","source":"* resumeDate - Time game was resumed (if abandoned, otherwise null).\n* resumedFrom - Time game was originally abandoned (if abandoned, otherwise null).\n* codedGameState - Game status code, various types can be found here.\n* detailedGameState - Game status, various types can be found here.","metadata":{}},{"cell_type":"code","source":"games_df['codedGameState']=games_df['codedGameState'].astype(str)\ngametype_dict=games_df['codedGameState'].value_counts().to_dict()\ngametype_keys=gametype_dict.keys()\n\ncolors = [plt.cm.Spectral(i/float(len(gametype_keys))) for i in range(len(gametype_keys))]\n\nfig = plt.figure(figsize = (20, 8))\nax = fig.add_subplot(1,2,1)\n\nax.bar(gametype_keys, gametype_dict.values(), color = colors)\n\nfor k, v in gametype_dict.items():\n    ax.text(k, v+1, v, fontsize = 14, horizontalalignment='center', verticalalignment='center')\nplt.xlabel(\"codedGameState\")\nplt.ylabel(\"Resumed date count\")\nax.tick_params(axis='x', labelrotation = 90, labelsize = 12)\nax.set_ylim(0, 9000)\nax.set_title(\"Weightage of gaming sessions\", fontsize = 14)\n\n########################################\ngames_df['detailedGameState']=games_df['detailedGameState'].astype(str)\nseason_dict=games_df['detailedGameState'].value_counts().to_dict()\nseason_keys=list(season_dict.keys())\n\n\naa=plt.cm.jet\ncolors = [aa(i/float(len(season_keys))) for i in range(len(season_keys))]\n\n\nax1 = fig.add_subplot(1,2,2)\nax1.bar(season_keys, season_dict.values(), color = colors)\n\nfor k, v in season_dict.items():\n    ax1.text(k, v+1, v, fontsize = 14, horizontalalignment='center', verticalalignment='center')\nplt.xlabel(\"detailedGameState\")\nplt.ylabel(\"detailedGameState count\")\nax1.tick_params(axis='x', labelrotation = 90, labelsize = 12)\nax1.set_title(\"Number of games per season\", fontsize = 14);\n","metadata":{"execution":{"iopub.status.busy":"2021-06-27T07:55:53.031264Z","iopub.execute_input":"2021-06-27T07:55:53.03164Z","iopub.status.idle":"2021-06-27T07:55:53.457587Z","shell.execute_reply.started":"2021-06-27T07:55:53.031607Z","shell.execute_reply":"2021-06-27T07:55:53.456473Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* homeWins - Number of current wins on season for home team.\n* homeScore - Runs scored by home team.\n* awayWins - Number of current wins on season for away team.\n* awayLosses - Number of current losses on season for away team.\n","metadata":{}},{"cell_type":"code","source":"all_titles=('Total home runs scored throughout all seasons','Total home game win throughout all seasons',\n'Total away runs scored throughout all seasons','Total away game win throughout all season ')\n\n\nfig = make_subplots(rows=2, cols=2,subplot_titles=all_titles,\n    specs=[[ {\"type\": \"sunburst\"},{\"type\": \"sunburst\"}],[{\"type\": \"sunburst\"},{\"type\": \"sunburst\"}]])   \n\nrow_=[1,1,2,2];col_=[1,2,1,2]\n\nfor i,cols in enumerate(['homeScore',\"homeWinner\",'awayScore','awayWinner']):\n    fig_ = px.sunburst(games_df, path=[ \"season\"],values=cols)\n    trace =go.Sunburst(\n                labels=fig_['data'][0]['labels'].tolist(),\n                    parents=fig_['data'][0]['parents'].tolist(),\n                values=fig_['data'][0]['values'].tolist(),\n                        ids=fig_['data'][0]['ids'].tolist(),\n                marker = { 'colorscale':random.choice(colorscales)},)\n    \n    fig.add_trace(trace,row=row_[i],col=col_[i])\n    \n\nfig.update_layout(title_text='Games Stats with respect to player poitions', title_x=0.5, height=800,width=900,\n                  margin=dict(r=10, t=50, b=40, l=60))\n#iplot(fig)","metadata":{"execution":{"iopub.status.busy":"2021-06-27T07:57:56.459514Z","iopub.execute_input":"2021-06-27T07:57:56.459893Z","iopub.status.idle":"2021-06-27T07:57:57.269779Z","shell.execute_reply.started":"2021-06-27T07:57:56.459858Z","shell.execute_reply":"2021-06-27T07:57:57.268751Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Inference #####\n\n* Net home and away runs scores over all seasons in almost same ,however the total number homegame win outweighs net away game wins by roughly 10%-15% for all seasons.","metadata":{}},{"cell_type":"markdown","source":"### Roster Data","metadata":{}},{"cell_type":"code","source":"  \n############################## ROSTER COLUMN PARSING########################\n\n\nall_roster=train[~train.rosters.isna()].rosters.values\nroster_keys=list(json.loads(all_roster[0])[0].keys())\nroster_processed=dict(zip(roster_keys,[[] for val in roster_keys]))\nroster_processed['date_tr']=[]\n\n\n### parse and store data\n### select only NON-NAN entries\ntemp=train[~train.rosters.isna()]\n\nfor date,game in tqdm(zip(temp.date.values,all_roster)):\n    game=json.loads(game)\n    for val in game:\n        for key in val.keys():\n            roster_processed[key].append(val[key])\n        roster_processed['date_tr'].append(date)\nroster_df=pd.DataFrame(roster_processed)\nroster_df.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:22:39.427638Z","iopub.execute_input":"2021-06-24T16:22:39.428171Z","iopub.status.idle":"2021-06-24T16:22:48.009777Z","shell.execute_reply.started":"2021-06-24T16:22:39.428128Z","shell.execute_reply":"2021-06-24T16:22:48.008629Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize = (20, 4))\n\nfor i,cols in enumerate(['statusCode','status']):\n    ax = fig.add_subplot(1,2,i+1)\n    \n    roster_df[cols]=roster_df[cols].astype('str')\n    temp_dict=roster_df[cols].value_counts().to_dict()\n    temp_keys=list(temp_dict.keys())\n    colors = [plt.cm.jet(i/float(len(temp_keys))) for i in range(len(temp_keys))]\n    \n    ax.bar(temp_keys, temp_dict.values(), color = colors)\n\n    for k, v in temp_dict.items():\n        ax.text(k, v+1, v, fontsize = 14, horizontalalignment='center', verticalalignment='center')\n    plt.xlabel(cols)\n    plt.ylabel(f\"{cols} count\")\n    ax.tick_params(axis='x', labelrotation = 45, labelsize = 12)\n    ax.set_title(f\"{cols} Value Count plot\", fontsize = 14)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:24:19.774291Z","iopub.execute_input":"2021-06-24T16:24:19.774704Z","iopub.status.idle":"2021-06-24T16:24:21.281191Z","shell.execute_reply.started":"2021-06-24T16:24:19.774668Z","shell.execute_reply":"2021-06-24T16:24:21.280129Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Player Box Scores","metadata":{}},{"cell_type":"code","source":"############################## PLAYERBOXSCORES COLUMN PARSING########################\n\nall_playerBoxScores=train[~train.playerBoxScores.isna()].playerBoxScores.values\nplayerBoxScores_keys=list(json.loads(all_playerBoxScores[0])[0].keys())\nplayerBoxScores_processed=dict(zip(playerBoxScores_keys,[[] for val in playerBoxScores_keys]))\nplayerBoxScores_processed['date_tr']=[]\n\n\n### parse and store data\n### select only NON-NAN entries\ntemp=train[~train.playerBoxScores.isna()]\n\nfor date,box_scores in tqdm(zip(temp.date.values,all_playerBoxScores)):\n    box_scores=json.loads(box_scores)\n    for val in box_scores:\n        for key in val.keys():\n            playerBoxScores_processed[key].append(val[key])\n        playerBoxScores_processed['date_tr'].append(date)\nboxscores_df=pd.DataFrame(playerBoxScores_processed)\nboxscores_df.gameDate=pd.to_datetime(boxscores_df.gameDate)\nboxscores_df['year']=boxscores_df.gameDate.dt.year\nboxscores_df['month']=boxscores_df.gameDate.dt.month\nboxscores_df['day']=boxscores_df.gameDate.dt.day\n\nboxscores_df.head(2)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-27T14:53:28.619322Z","iopub.execute_input":"2021-06-27T14:53:28.619848Z","iopub.status.idle":"2021-06-27T14:53:42.896268Z","shell.execute_reply.started":"2021-06-27T14:53:28.619812Z","shell.execute_reply":"2021-06-27T14:53:42.895099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"true=True;false=False\nposition_code_map=dict()\nposition_code=[{\"shortName\":\"Pitcher\",\"fullName\":\"Pitcher\",\"abbrev\":\"P\",\"code\":\"1\",\"type\":\"Pitcher\",\"formalName\":\"Pitcher\",\"gamePosition\":true,\"pitcher\":true,\"fielder\":false,\"outfield\":false,\"displayName\":\"Pitcher\"},{\"shortName\":\"Catcher\",\"fullName\":\"Catcher\",\"abbrev\":\"C\",\"code\":\"2\",\"type\":\"Catcher\",\"formalName\":\"Catcher\",\"gamePosition\":true,\"pitcher\":false,\"fielder\":true,\"outfield\":false,\"displayName\":\"Catcher\"},{\"shortName\":\"1st Base\",\"fullName\":\"First Base\",\"abbrev\":\"1B\",\"code\":\"3\",\"type\":\"Infielder\",\"formalName\":\"First Baseman\",\"gamePosition\":true,\"pitcher\":false,\"fielder\":true,\"outfield\":false,\"displayName\":\"First Base\"},{\"shortName\":\"2nd Base\",\"fullName\":\"Second Base\",\"abbrev\":\"2B\",\"code\":\"4\",\"type\":\"Infielder\",\"formalName\":\"Second Baseman\",\"gamePosition\":true,\"pitcher\":false,\"fielder\":true,\"outfield\":false,\"displayName\":\"Second Base\"},{\"shortName\":\"3rd Base\",\"fullName\":\"Third Base\",\"abbrev\":\"3B\",\"code\":\"5\",\"type\":\"Infielder\",\"formalName\":\"Third Baseman\",\"gamePosition\":true,\"pitcher\":false,\"fielder\":true,\"outfield\":false,\"displayName\":\"Third Base\"},{\"shortName\":\"Shortstop\",\"fullName\":\"Shortstop\",\"abbrev\":\"SS\",\"code\":\"6\",\"type\":\"Infielder\",\"formalName\":\"Shortstop\",\"gamePosition\":true,\"pitcher\":false,\"fielder\":true,\"outfield\":false,\"displayName\":\"Shortstop\"},{\"shortName\":\"Left Field\",\"fullName\":\"Outfielder\",\"abbrev\":\"LF\",\"code\":\"7\",\"type\":\"Outfielder\",\"formalName\":\"Left Fielder\",\"gamePosition\":true,\"pitcher\":false,\"fielder\":true,\"outfield\":true,\"displayName\":\"Outfielder\"},{\"shortName\":\"Center Field\",\"fullName\":\"Outfielder\",\"abbrev\":\"CF\",\"code\":\"8\",\"type\":\"Outfielder\",\"formalName\":\"Center Fielder\",\"gamePosition\":true,\"pitcher\":false,\"fielder\":true,\"outfield\":true,\"displayName\":\"Outfielder\"},{\"shortName\":\"Right Field\",\"fullName\":\"Outfielder\",\"abbrev\":\"RF\",\"code\":\"9\",\"type\":\"Outfielder\",\"formalName\":\"Right Fielder\",\"gamePosition\":true,\"pitcher\":false,\"fielder\":true,\"outfield\":true,\"displayName\":\"Outfielder\"},{\"shortName\":\"Designated Hitter\",\"fullName\":\"Designated Hitter\",\"abbrev\":\"DH\",\"code\":\"10\",\"type\":\"Hitter\",\"formalName\":\"Designated Hitter\",\"gamePosition\":true,\"pitcher\":false,\"fielder\":false,\"outfield\":false,\"displayName\":\"Designated Hitter\"},{\"shortName\":\"Pinch Hitter\",\"fullName\":\"Pinch Hitter\",\"abbrev\":\"PH\",\"code\":\"11\",\"type\":\"Hitter\",\"formalName\":\"Pinch Hitter\",\"gamePosition\":true,\"pitcher\":false,\"fielder\":false,\"outfield\":false,\"displayName\":\"Pinch Hitter\"},{\"shortName\":\"Pinch Runner\",\"fullName\":\"Pinch Runner\",\"abbrev\":\"PR\",\"code\":\"12\",\"type\":\"Runner\",\"formalName\":\"Pinch Runner\",\"gamePosition\":true,\"pitcher\":false,\"fielder\":false,\"outfield\":false,\"displayName\":\"Pinch Runner\"},{\"shortName\":\"Extra Hitter\",\"fullName\":\"Extra Hitter\",\"abbrev\":\"EH\",\"code\":\"13\",\"type\":\"Hitter\",\"formalName\":\"Extra Hitter\",\"gamePosition\":true,\"pitcher\":false,\"fielder\":false,\"outfield\":false,\"displayName\":\"Extra Hitter\"},{\"shortName\":\"Base Runner\",\"fullName\":\"Base Runner\",\"abbrev\":\"BR\",\"code\":\"BR\",\"type\":\"Runner\",\"formalName\":\"Base Runner\",\"gamePosition\":false,\"pitcher\":false,\"fielder\":false,\"outfield\":false,\"displayName\":\"Base Runner\"},{\"shortName\":\"Outfield\",\"fullName\":\"Outfield\",\"abbrev\":\"OF\",\"code\":\"O\",\"type\":\"Outfielder\",\"formalName\":\"Outfield\",\"gamePosition\":false,\"pitcher\":false,\"fielder\":true,\"outfield\":true,\"displayName\":\"Outfield\"},{\"shortName\":\"Infield\",\"fullName\":\"Infield\",\"abbrev\":\"IF\",\"code\":\"I\",\"type\":\"Infielder\",\"formalName\":\"Infield\",\"gamePosition\":false,\"pitcher\":false,\"fielder\":true,\"outfield\":false,\"displayName\":\"Infield\"},{\"shortName\":\"Starting Pitcher\",\"fullName\":\"Starting Pitcher\",\"abbrev\":\"SP\",\"code\":\"S\",\"type\":\"Pitcher\",\"formalName\":\"Starting Pitcher\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":false,\"outfield\":false,\"displayName\":\"Starting Pitcher\"},{\"shortName\":\"Relief Pitcher\",\"fullName\":\"Relief Pitcher\",\"abbrev\":\"RP\",\"code\":\"E\",\"type\":\"Pitcher\",\"formalName\":\"Relief Pitcher\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":false,\"outfield\":false,\"displayName\":\"Relief Pitcher\"},{\"shortName\":\"Closer\",\"fullName\":\"Closer\",\"abbrev\":\"CP\",\"code\":\"C\",\"type\":\"Pitcher\",\"formalName\":\"Closer\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":false,\"outfield\":false,\"displayName\":\"Closer\"},{\"shortName\":\"Utility\",\"fullName\":\"Utility\",\"abbrev\":\"UT\",\"code\":\"U\",\"type\":\"Infielder\",\"formalName\":\"Utility\",\"gamePosition\":false,\"pitcher\":false,\"fielder\":true,\"outfield\":false,\"displayName\":\"Utility\"},{\"shortName\":\"Utility Infielder\",\"fullName\":\"Utility Infielder\",\"abbrev\":\"UI\",\"code\":\"V\",\"type\":\"Infielder\",\"formalName\":\"Utility Infielder\",\"gamePosition\":false,\"pitcher\":false,\"fielder\":true,\"outfield\":false,\"displayName\":\"Utility Infielder\"},{\"shortName\":\"Utility Outfielder\",\"fullName\":\"Utility Outfielder\",\"abbrev\":\"UO\",\"code\":\"W\",\"type\":\"Outfielder\",\"formalName\":\"Utility Outfielder\",\"gamePosition\":false,\"pitcher\":false,\"fielder\":true,\"outfield\":true,\"displayName\":\"Utility Outfielder\"},{\"shortName\":\"Right-Handed Pitcher\",\"fullName\":\"Right-Handed Pitcher\",\"abbrev\":\"RHP\",\"code\":\"K\",\"type\":\"Pitcher\",\"formalName\":\"Right-Handed Pitcher\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":false,\"outfield\":false,\"displayName\":\"Right-Handed Pitcher\"},{\"shortName\":\"Left-Handed Pitcher\",\"fullName\":\"Left-Handed Pitcher\",\"abbrev\":\"LHP\",\"code\":\"L\",\"type\":\"Pitcher\",\"formalName\":\"Left-Handed Pitcher\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":false,\"outfield\":false,\"displayName\":\"Left-Handed Pitcher\"},{\"shortName\":\"Right-Handed Starter\",\"fullName\":\"Right-Handed Starter\",\"abbrev\":\"RHS\",\"code\":\"M\",\"type\":\"Pitcher\",\"formalName\":\"Right-Handed Starter\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":false,\"outfield\":false,\"displayName\":\"Right-Handed Starter\"},{\"shortName\":\"Left-Handed Starter\",\"fullName\":\"Left-Handed Starter\",\"abbrev\":\"LHS\",\"code\":\"N\",\"type\":\"Pitcher\",\"formalName\":\"Left-Handed Starter\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":false,\"outfield\":false,\"displayName\":\"Left-Handed Starter\"},{\"shortName\":\"Left-Handed Reliever\",\"fullName\":\"Left-Handed Reliever\",\"abbrev\":\"LHR\",\"code\":\"G\",\"type\":\"Pitcher\",\"formalName\":\"Left-Handed Reliever\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":false,\"outfield\":false,\"displayName\":\"Left-Handed Reliever\"},{\"shortName\":\"Right-Handed Reliever\",\"fullName\":\"Right-Handed Reliever\",\"abbrev\":\"RHR\",\"code\":\"F\",\"type\":\"Pitcher\",\"formalName\":\"Right-Handed Reliever\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":false,\"outfield\":false,\"displayName\":\"Right-Handed Reliever\"},{\"shortName\":\"Pitcher - Infielder\",\"fullName\":\"Pitcher - Infielder\",\"abbrev\":\"P-IF\",\"code\":\"A\",\"type\":\"Two-Way Player\",\"formalName\":\"Pitcher - Infielder\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":true,\"outfield\":false,\"displayName\":\"Pitcher - Infielder\"},{\"shortName\":\"Pitcher - Outfielder\",\"fullName\":\"Pitcher - Outfielder\",\"abbrev\":\"P-OF\",\"code\":\"J\",\"type\":\"Two-Way Player\",\"formalName\":\"Pitcher - Outfielder\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":true,\"outfield\":true,\"displayName\":\"Pitcher - Outfielder\"},{\"shortName\":\"Pitcher - Utility\",\"fullName\":\"Pitcher - Utility\",\"abbrev\":\"P-UT\",\"code\":\"Z\",\"type\":\"Two-Way Player\",\"formalName\":\"Pitcher - Utility\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":true,\"outfield\":false,\"displayName\":\"Pitcher - Utility\"},{\"shortName\":\"Two-Way Player\",\"fullName\":\"Two-Way Player\",\"abbrev\":\"TWP\",\"code\":\"Y\",\"type\":\"Two-Way Player\",\"formalName\":\"Two-Way Player\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":false,\"outfield\":false,\"displayName\":\"Two-Way Player\"},{\"shortName\":\"Batter\",\"fullName\":\"Batter\",\"abbrev\":\"B\",\"code\":\"10\",\"type\":\"Batter\",\"formalName\":\"Batter\",\"gamePosition\":false,\"pitcher\":false,\"fielder\":false,\"outfield\":false,\"displayName\":\"Batter\"},{\"shortName\":\"Unknown\",\"fullName\":\"Unknown\",\"abbrev\":\"X\",\"code\":\"X\",\"type\":\"Unknown\",\"formalName\":\"Unknown\",\"gamePosition\":false,\"pitcher\":false,\"fielder\":false,\"outfield\":false,\"displayName\":\"Unknown\"},{\"shortName\":\"Runner on First\",\"fullName\":\"Runner on First\",\"abbrev\":\"R1\",\"code\":\"R1\",\"type\":\"Runner\",\"formalName\":\"Runner on First\",\"gamePosition\":false,\"pitcher\":false,\"fielder\":false,\"outfield\":false,\"displayName\":\"Runner on First\"},{\"shortName\":\"Runner on Second\",\"fullName\":\"Runner on Second\",\"abbrev\":\"R2\",\"code\":\"R2\",\"type\":\"Runner\",\"formalName\":\"Runner on Second\",\"gamePosition\":false,\"pitcher\":false,\"fielder\":false,\"outfield\":false,\"displayName\":\"Runner on Second\"},{\"shortName\":\"Runner on Third\",\"fullName\":\"Runner on Third\",\"abbrev\":\"R3\",\"code\":\"R3\",\"type\":\"Runner\",\"formalName\":\"Runner on Third\",\"gamePosition\":false,\"pitcher\":false,\"fielder\":false,\"outfield\":false,\"displayName\":\"Runner on Third\"}]\nfor val in position_code:\n    position_code_map[val['code']]=val['formalName']\nboxscores_df['positionCode']=boxscores_df['positionCode'].map(position_code_map)\n\nposition_type_map=dict()\nposition_type=[{\"shortName\":\"Pitcher\",\"fullName\":\"Pitcher\",\"abbrev\":\"P\",\"code\":\"1\",\"type\":\"Pitcher\",\"formalName\":\"Pitcher\",\"gamePosition\":true,\"pitcher\":true,\"fielder\":false,\"outfield\":false,\"displayName\":\"Pitcher\"},{\"shortName\":\"Catcher\",\"fullName\":\"Catcher\",\"abbrev\":\"C\",\"code\":\"2\",\"type\":\"Catcher\",\"formalName\":\"Catcher\",\"gamePosition\":true,\"pitcher\":false,\"fielder\":true,\"outfield\":false,\"displayName\":\"Catcher\"},{\"shortName\":\"1st Base\",\"fullName\":\"First Base\",\"abbrev\":\"1B\",\"code\":\"3\",\"type\":\"Infielder\",\"formalName\":\"First Baseman\",\"gamePosition\":true,\"pitcher\":false,\"fielder\":true,\"outfield\":false,\"displayName\":\"First Base\"},{\"shortName\":\"2nd Base\",\"fullName\":\"Second Base\",\"abbrev\":\"2B\",\"code\":\"4\",\"type\":\"Infielder\",\"formalName\":\"Second Baseman\",\"gamePosition\":true,\"pitcher\":false,\"fielder\":true,\"outfield\":false,\"displayName\":\"Second Base\"},{\"shortName\":\"3rd Base\",\"fullName\":\"Third Base\",\"abbrev\":\"3B\",\"code\":\"5\",\"type\":\"Infielder\",\"formalName\":\"Third Baseman\",\"gamePosition\":true,\"pitcher\":false,\"fielder\":true,\"outfield\":false,\"displayName\":\"Third Base\"},{\"shortName\":\"Shortstop\",\"fullName\":\"Shortstop\",\"abbrev\":\"SS\",\"code\":\"6\",\"type\":\"Infielder\",\"formalName\":\"Shortstop\",\"gamePosition\":true,\"pitcher\":false,\"fielder\":true,\"outfield\":false,\"displayName\":\"Shortstop\"},{\"shortName\":\"Left Field\",\"fullName\":\"Outfielder\",\"abbrev\":\"LF\",\"code\":\"7\",\"type\":\"Outfielder\",\"formalName\":\"Left Fielder\",\"gamePosition\":true,\"pitcher\":false,\"fielder\":true,\"outfield\":true,\"displayName\":\"Outfielder\"},{\"shortName\":\"Center Field\",\"fullName\":\"Outfielder\",\"abbrev\":\"CF\",\"code\":\"8\",\"type\":\"Outfielder\",\"formalName\":\"Center Fielder\",\"gamePosition\":true,\"pitcher\":false,\"fielder\":true,\"outfield\":true,\"displayName\":\"Outfielder\"},{\"shortName\":\"Right Field\",\"fullName\":\"Outfielder\",\"abbrev\":\"RF\",\"code\":\"9\",\"type\":\"Outfielder\",\"formalName\":\"Right Fielder\",\"gamePosition\":true,\"pitcher\":false,\"fielder\":true,\"outfield\":true,\"displayName\":\"Outfielder\"},{\"shortName\":\"Designated Hitter\",\"fullName\":\"Designated Hitter\",\"abbrev\":\"DH\",\"code\":\"10\",\"type\":\"Hitter\",\"formalName\":\"Designated Hitter\",\"gamePosition\":true,\"pitcher\":false,\"fielder\":false,\"outfield\":false,\"displayName\":\"Designated Hitter\"},{\"shortName\":\"Pinch Hitter\",\"fullName\":\"Pinch Hitter\",\"abbrev\":\"PH\",\"code\":\"11\",\"type\":\"Hitter\",\"formalName\":\"Pinch Hitter\",\"gamePosition\":true,\"pitcher\":false,\"fielder\":false,\"outfield\":false,\"displayName\":\"Pinch Hitter\"},{\"shortName\":\"Pinch Runner\",\"fullName\":\"Pinch Runner\",\"abbrev\":\"PR\",\"code\":\"12\",\"type\":\"Runner\",\"formalName\":\"Pinch Runner\",\"gamePosition\":true,\"pitcher\":false,\"fielder\":false,\"outfield\":false,\"displayName\":\"Pinch Runner\"},{\"shortName\":\"Extra Hitter\",\"fullName\":\"Extra Hitter\",\"abbrev\":\"EH\",\"code\":\"13\",\"type\":\"Hitter\",\"formalName\":\"Extra Hitter\",\"gamePosition\":true,\"pitcher\":false,\"fielder\":false,\"outfield\":false,\"displayName\":\"Extra Hitter\"},{\"shortName\":\"Base Runner\",\"fullName\":\"Base Runner\",\"abbrev\":\"BR\",\"code\":\"BR\",\"type\":\"Runner\",\"formalName\":\"Base Runner\",\"gamePosition\":false,\"pitcher\":false,\"fielder\":false,\"outfield\":false,\"displayName\":\"Base Runner\"},{\"shortName\":\"Outfield\",\"fullName\":\"Outfield\",\"abbrev\":\"OF\",\"code\":\"O\",\"type\":\"Outfielder\",\"formalName\":\"Outfield\",\"gamePosition\":false,\"pitcher\":false,\"fielder\":true,\"outfield\":true,\"displayName\":\"Outfield\"},{\"shortName\":\"Infield\",\"fullName\":\"Infield\",\"abbrev\":\"IF\",\"code\":\"I\",\"type\":\"Infielder\",\"formalName\":\"Infield\",\"gamePosition\":false,\"pitcher\":false,\"fielder\":true,\"outfield\":false,\"displayName\":\"Infield\"},{\"shortName\":\"Starting Pitcher\",\"fullName\":\"Starting Pitcher\",\"abbrev\":\"SP\",\"code\":\"S\",\"type\":\"Pitcher\",\"formalName\":\"Starting Pitcher\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":false,\"outfield\":false,\"displayName\":\"Starting Pitcher\"},{\"shortName\":\"Relief Pitcher\",\"fullName\":\"Relief Pitcher\",\"abbrev\":\"RP\",\"code\":\"E\",\"type\":\"Pitcher\",\"formalName\":\"Relief Pitcher\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":false,\"outfield\":false,\"displayName\":\"Relief Pitcher\"},{\"shortName\":\"Closer\",\"fullName\":\"Closer\",\"abbrev\":\"CP\",\"code\":\"C\",\"type\":\"Pitcher\",\"formalName\":\"Closer\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":false,\"outfield\":false,\"displayName\":\"Closer\"},{\"shortName\":\"Utility\",\"fullName\":\"Utility\",\"abbrev\":\"UT\",\"code\":\"U\",\"type\":\"Infielder\",\"formalName\":\"Utility\",\"gamePosition\":false,\"pitcher\":false,\"fielder\":true,\"outfield\":false,\"displayName\":\"Utility\"},{\"shortName\":\"Utility Infielder\",\"fullName\":\"Utility Infielder\",\"abbrev\":\"UI\",\"code\":\"V\",\"type\":\"Infielder\",\"formalName\":\"Utility Infielder\",\"gamePosition\":false,\"pitcher\":false,\"fielder\":true,\"outfield\":false,\"displayName\":\"Utility Infielder\"},{\"shortName\":\"Utility Outfielder\",\"fullName\":\"Utility Outfielder\",\"abbrev\":\"UO\",\"code\":\"W\",\"type\":\"Outfielder\",\"formalName\":\"Utility Outfielder\",\"gamePosition\":false,\"pitcher\":false,\"fielder\":true,\"outfield\":true,\"displayName\":\"Utility Outfielder\"},{\"shortName\":\"Right-Handed Pitcher\",\"fullName\":\"Right-Handed Pitcher\",\"abbrev\":\"RHP\",\"code\":\"K\",\"type\":\"Pitcher\",\"formalName\":\"Right-Handed Pitcher\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":false,\"outfield\":false,\"displayName\":\"Right-Handed Pitcher\"},{\"shortName\":\"Left-Handed Pitcher\",\"fullName\":\"Left-Handed Pitcher\",\"abbrev\":\"LHP\",\"code\":\"L\",\"type\":\"Pitcher\",\"formalName\":\"Left-Handed Pitcher\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":false,\"outfield\":false,\"displayName\":\"Left-Handed Pitcher\"},{\"shortName\":\"Right-Handed Starter\",\"fullName\":\"Right-Handed Starter\",\"abbrev\":\"RHS\",\"code\":\"M\",\"type\":\"Pitcher\",\"formalName\":\"Right-Handed Starter\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":false,\"outfield\":false,\"displayName\":\"Right-Handed Starter\"},{\"shortName\":\"Left-Handed Starter\",\"fullName\":\"Left-Handed Starter\",\"abbrev\":\"LHS\",\"code\":\"N\",\"type\":\"Pitcher\",\"formalName\":\"Left-Handed Starter\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":false,\"outfield\":false,\"displayName\":\"Left-Handed Starter\"},{\"shortName\":\"Left-Handed Reliever\",\"fullName\":\"Left-Handed Reliever\",\"abbrev\":\"LHR\",\"code\":\"G\",\"type\":\"Pitcher\",\"formalName\":\"Left-Handed Reliever\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":false,\"outfield\":false,\"displayName\":\"Left-Handed Reliever\"},{\"shortName\":\"Right-Handed Reliever\",\"fullName\":\"Right-Handed Reliever\",\"abbrev\":\"RHR\",\"code\":\"F\",\"type\":\"Pitcher\",\"formalName\":\"Right-Handed Reliever\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":false,\"outfield\":false,\"displayName\":\"Right-Handed Reliever\"},{\"shortName\":\"Pitcher - Infielder\",\"fullName\":\"Pitcher - Infielder\",\"abbrev\":\"P-IF\",\"code\":\"A\",\"type\":\"Two-Way Player\",\"formalName\":\"Pitcher - Infielder\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":true,\"outfield\":false,\"displayName\":\"Pitcher - Infielder\"},{\"shortName\":\"Pitcher - Outfielder\",\"fullName\":\"Pitcher - Outfielder\",\"abbrev\":\"P-OF\",\"code\":\"J\",\"type\":\"Two-Way Player\",\"formalName\":\"Pitcher - Outfielder\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":true,\"outfield\":true,\"displayName\":\"Pitcher - Outfielder\"},{\"shortName\":\"Pitcher - Utility\",\"fullName\":\"Pitcher - Utility\",\"abbrev\":\"P-UT\",\"code\":\"Z\",\"type\":\"Two-Way Player\",\"formalName\":\"Pitcher - Utility\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":true,\"outfield\":false,\"displayName\":\"Pitcher - Utility\"},{\"shortName\":\"Two-Way Player\",\"fullName\":\"Two-Way Player\",\"abbrev\":\"TWP\",\"code\":\"Y\",\"type\":\"Two-Way Player\",\"formalName\":\"Two-Way Player\",\"gamePosition\":false,\"pitcher\":true,\"fielder\":false,\"outfield\":false,\"displayName\":\"Two-Way Player\"},{\"shortName\":\"Batter\",\"fullName\":\"Batter\",\"abbrev\":\"B\",\"code\":\"10\",\"type\":\"Batter\",\"formalName\":\"Batter\",\"gamePosition\":false,\"pitcher\":false,\"fielder\":false,\"outfield\":false,\"displayName\":\"Batter\"},{\"shortName\":\"Unknown\",\"fullName\":\"Unknown\",\"abbrev\":\"X\",\"code\":\"X\",\"type\":\"Unknown\",\"formalName\":\"Unknown\",\"gamePosition\":false,\"pitcher\":false,\"fielder\":false,\"outfield\":false,\"displayName\":\"Unknown\"},{\"shortName\":\"Runner on First\",\"fullName\":\"Runner on First\",\"abbrev\":\"R1\",\"code\":\"R1\",\"type\":\"Runner\",\"formalName\":\"Runner on First\",\"gamePosition\":false,\"pitcher\":false,\"fielder\":false,\"outfield\":false,\"displayName\":\"Runner on First\"},{\"shortName\":\"Runner on Second\",\"fullName\":\"Runner on Second\",\"abbrev\":\"R2\",\"code\":\"R2\",\"type\":\"Runner\",\"formalName\":\"Runner on Second\",\"gamePosition\":false,\"pitcher\":false,\"fielder\":false,\"outfield\":false,\"displayName\":\"Runner on Second\"},{\"shortName\":\"Runner on Third\",\"fullName\":\"Runner on Third\",\"abbrev\":\"R3\",\"code\":\"R3\",\"type\":\"Runner\",\"formalName\":\"Runner on Third\",\"gamePosition\":false,\"pitcher\":false,\"fielder\":false,\"outfield\":false,\"displayName\":\"Runner on Third\"}]\nfor val in position_type:\n    position_type_map[val['code']]=val['type']\n \nboxscores_df['positionType']=boxscores_df['positionType'].astype(str)\nboxscores_df['positionType']=boxscores_df['positionType'].map(position_type_map)\n### still cant figure out why outputing wrong dtype\nboxscores_df['positionType'].dtypes","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-27T14:53:42.898310Z","iopub.execute_input":"2021-06-27T14:53:42.898612Z","iopub.status.idle":"2021-06-27T14:53:43.106823Z","shell.execute_reply.started":"2021-06-27T14:53:42.898585Z","shell.execute_reply":"2021-06-27T14:53:43.106111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import missingno as msno\n\nmsno.matrix(boxscores_df)","metadata":{"execution":{"iopub.status.busy":"2021-06-27T14:53:43.108109Z","iopub.execute_input":"2021-06-27T14:53:43.108502Z","iopub.status.idle":"2021-06-27T14:53:48.195023Z","shell.execute_reply.started":"2021-06-27T14:53:43.108463Z","shell.execute_reply":"2021-06-27T14:53:48.194059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Inference #####\n\nThere are some columns towards the end of dataframe and some columns at beginning have complete missing data, so they are removed from df.","metadata":{}},{"cell_type":"code","source":"\ntemp=[]\nfor cols in boxscores_df.columns:\n    if boxscores_df[cols].nunique()==1 or boxscores_df[cols].isna().sum()>0.9*len(boxscores_df):\n        temp.append(cols)\nboxscores_df.drop(columns=temp,inplace=True)\n\n####### Fillnna values ########\n\nfor cols in boxscores_df.columns[13:]:\n    boxscores_df[cols]=boxscores_df[cols].fillna(boxscores_df[cols].mean())\n    boxscores_df[cols] = boxscores_df[cols].round(decimals=0)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-27T14:53:48.196560Z","iopub.execute_input":"2021-06-27T14:53:48.197143Z","iopub.status.idle":"2021-06-27T14:53:49.145707Z","shell.execute_reply.started":"2021-06-27T14:53:48.197100Z","shell.execute_reply":"2021-06-27T14:53:49.144785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* runsScored - Game total runs scored.\n* doubles - Game total doubles.\n* triples - Game total triples.\n* homeRuns - Game total home runs.","metadata":{}},{"cell_type":"code","source":"all_titles=('Total runs scored throughout all seasons for each teams','Total doubles hit throughout all season for each team',\n'Total triples hit throughout all season for each team','Total home_runs scored throughout all season for each team')\n\n\nfig = make_subplots(rows=2, cols=2,subplot_titles=all_titles,\n    specs=[[ {\"type\": \"sunburst\"},{\"type\": \"sunburst\"}],[{\"type\": \"sunburst\"},{\"type\": \"sunburst\"}]])   \n\nrow_=[1,1,2,2];col_=[1,2,1,2]\n\nfor i,cols in enumerate(['runsScored',\"doubles\",'triples','homeRuns']):\n    fig_ = px.sunburst(boxscores_df, path=[ \"year\",\"teamName\"],values=cols)\n    trace =go.Sunburst(\n                labels=fig_['data'][0]['labels'].tolist(),\n                    parents=fig_['data'][0]['parents'].tolist(),\n                values=fig_['data'][0]['values'].tolist(),\n                        ids=fig_['data'][0]['ids'].tolist(),\n                marker = { 'colorscale':random.choice(colorscales)},)\n    \n    fig.add_trace(trace,row=row_[i],col=col_[i])\n    \n\nfig.update_layout(title_text='Games Stats with respect to player poitions', title_x=0.5, height=800,width=900,\n                  margin=dict(r=10, t=50, b=40, l=60))\n#iplot(fig)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-27T14:53:49.146994Z","iopub.execute_input":"2021-06-27T14:53:49.147281Z","iopub.status.idle":"2021-06-27T14:54:04.365625Z","shell.execute_reply.started":"2021-06-27T14:53:49.147252Z","shell.execute_reply":"2021-06-27T14:54:04.364658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Inference #####\n\n* In Year 2018 \n\n* 900+ runs scores : - 1 team (Boston Red Socks -960)\n* 250+ runs scored in home : - 2 teams , (Highest- New York Yankees)\n* 35+ triples scored : - 7 teams,(Highest- Arizona DiamondBacks)\n* 300+ doubles scored : - 6, (Highest- Boston Red Socks - 380)\n\n\n* Year 2019 \n* 900+ runs scores : - 6 teams,(Highest- Houston Astros - 991)\n* 250+ runs scored in home : - 3 teams , (Highest- New York Yankees)\n* 35+ triples scored : - teams, (Highest- Colorado Rockies)\n* 300+ doubles scored : - 9, (Highest- Houston Astros - 348)","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"num_games_season=pd.DataFrame(games_df.groupby('season').gamePk.count()).reset_index()\nsns.barplot(x=num_games_season.season,y=num_games_season.gamePk)\nplt.title(\"Number of games played in respective seasons\")","metadata":{"execution":{"iopub.status.busy":"2021-06-27T14:54:04.367716Z","iopub.execute_input":"2021-06-27T14:54:04.368149Z","iopub.status.idle":"2021-06-27T14:54:04.522823Z","shell.execute_reply.started":"2021-06-27T14:54:04.368107Z","shell.execute_reply":"2021-06-27T14:54:04.521720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Note #####\n\n* From the above plot it's clear that number of games played in season 2020 and 2021 are much less than what it should be in actual. So if we compute **mean stats(flyOuts,runScored,etc..) of all team across all seasons** , then it will be **unfair due to unequal number of games** played. Here comes the idea of adjusted_mean where **stats per game** is computed for respective seasons and **multiplied with number of games played in 2018 season(Reference season)**.\n* The adjusted mean is a rough value and used just for intuitive purpose.","metadata":{}},{"cell_type":"code","source":"#Generate a set of 100 random colors\ndef generate_color(num_colors=50):\n    x = lambda: random.randint(random.choice([50,100,0,150]),255)\n    y = lambda: random.randint(0,random.choice([200,255]))\n    z = lambda: random.randint(random.choice([150,100,0]),255)\n    random_colors=[('#%02X%02X%02X' % (x(),y(),z())) for i in range(num_colors)]\n    return random_colors\n\n\nnum_games_season=dict(games_df.groupby('season').gamePk.count())\n\ndef getplayer_box_scores(cols2_plot,fig_size,orient):\n    fig = plt.figure(figsize=fig_size)\n    plt.subplots_adjust(wspace=0.5)\n    random_colors = generate_color(num_colors=50)\n    for i,cols in enumerate(cols2_plot):\n        if orient == 'single_row':\n            ax = fig.add_subplot(len(cols2_plot),1,i+1)\n        if orient == 'grid':\n            ax = fig.add_subplot(len(cols2_plot)/2,2,i+1)\n            \n\n        temp=boxscores_df.groupby(['year','teamName'])[cols].sum().reset_index()\n\n        a=temp.teamName.unique()\n        temp_=np.zeros(len(a))\n        temp_df=pd.DataFrame()\n        temp_df['teamName']=temp['teamName'].unique()\n        for year in [2018,2019,2020,2021]:\n            tmp_dict=temp[temp.year==year][['teamName',cols]].to_dict(orient='list')\n            tmp_dict=dict(zip(tmp_dict['teamName'],tmp_dict[cols]))\n\n            temp_df[year]=temp_df['teamName'].map(tmp_dict).fillna(0)\n\n        x = temp_df.teamName.values\n        y1 = np.array(temp_df[2018])\n        y2 = np.array(temp_df[2019])\n        y3 = np.array(temp_df[2020])\n        y4 = np.array(temp_df[2021])\n\n        # plot bars in stack manner\n        line1=plt.barh(x, y1, color=random.choice(random_colors))\n        line1_mean=plt.axvline(np.mean(y1),color=random.choice(random_colors), linewidth = 3, linestyle='--')\n        \n        line2=plt.barh(x, y2, left=y1, color=random.choice(random_colors))\n        line2_mean=plt.axvline(np.mean(y1)+np.mean(y2),color=random.choice(random_colors), linewidth = 3, linestyle='--')\n        line2_mean_adj=plt.axvline((np.mean(y2)/num_games_season[2019])*num_games_season[2018],color=random.choice(random_colors), linewidth = 3, linestyle='--')\n\n        line3=plt.barh(x, y3, left=y1+y2, color=random.choice(random_colors))\n        line3_mean_adj=plt.axvline((np.mean(y3)/num_games_season[2020])*num_games_season[2018],color=random.choice(random_colors), linewidth = 3, linestyle='--')\n        line3_mean=plt.axvline(np.mean(y1)+np.mean(y2)+np.mean(y3),color=random.choice(random_colors), linewidth = 3, linestyle='--')\n\n        line4=plt.barh(x, y4, left=y1+y2+y3, color=random.choice(random_colors))\n        line4_mean_adj=plt.axvline((np.mean(y4)/num_games_season[2021])*num_games_season[2018],color=random.choice(random_colors), linewidth = 3, linestyle='--')\n        line4_mean=plt.axvline(np.mean(y1)+np.mean(y2)+np.mean(y3)+np.mean(y4),color=random.choice(random_colors), linewidth = 3,linestyle='--')\n\n        ax.legend([line1, line2, line3, line4,line1_mean,line2_mean,line2_mean_adj,line3_mean_adj,line3_mean,line4_mean_adj,line4_mean], ['season_2018', 'season_2019', 'season_2020','season_2021',f'{cols}_mean_2018',f'{cols}_mean_2019',f'{cols}_adjusted_mean_2019',f'{cols}_adjusted_mean_2020',f'{cols}_mean_2020',f'{cols}_adjusted_mean_2021',f'{cols}_mean_2021'])        \n        \n        plt.ylabel(\"Teams\")\n        plt.xlabel(f\"{cols} Sum\")\n        ax.tick_params(axis='x', labelrotation = 90, labelsize = 12)\n        ax.set_title(f\"Total {cols} for all team across all seasons(2018-21)\", fontsize = 14)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-27T14:55:22.864821Z","iopub.execute_input":"2021-06-27T14:55:22.865246Z","iopub.status.idle":"2021-06-27T14:55:22.890920Z","shell.execute_reply.started":"2021-06-27T14:55:22.865208Z","shell.execute_reply":"2021-06-27T14:55:22.890008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* flyOuts - Game total fly outs.\n* groundOuts - Game total ground outs.\n* strikeOuts - Game total strike outs.\n","metadata":{}},{"cell_type":"code","source":"cols2_plot=['groundOuts','strikeOuts','flyOuts']\ngetplayer_box_scores(cols2_plot,fig_size=(18,27),orient='single_row')","metadata":{"execution":{"iopub.status.busy":"2021-06-27T14:55:24.362660Z","iopub.execute_input":"2021-06-27T14:55:24.363084Z","iopub.status.idle":"2021-06-27T14:55:27.561411Z","shell.execute_reply.started":"2021-06-27T14:55:24.363049Z","shell.execute_reply":"2021-06-27T14:55:27.560307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* plateAppearances - Game total plate appearances.\n* totalBases - Game total bases.\n* rbi - Game total runs batted in.\n* leftOnBase - Game total runners left on base.","metadata":{}},{"cell_type":"code","source":"cols2_plot=['plateAppearances','totalBases','rbi','leftOnBase']\ngetplayer_box_scores(cols2_plot,fig_size=(18,36),orient='single_row')","metadata":{"execution":{"iopub.status.busy":"2021-06-27T14:55:27.562788Z","iopub.execute_input":"2021-06-27T14:55:27.563046Z","iopub.status.idle":"2021-06-27T14:55:32.024675Z","shell.execute_reply.started":"2021-06-27T14:55:27.563020Z","shell.execute_reply":"2021-06-27T14:55:32.023701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* baseOnBalls - Game total walks.\n* intentionalWalks - Game total intentional walks.\n* hits - Game total hits.\n* hitByPitch - Game total hit by pitches.","metadata":{}},{"cell_type":"code","source":"cols2_plot=['baseOnBalls','intentionalWalks','hits','hitByPitch']\ngetplayer_box_scores(cols2_plot,fig_size=(18,36),orient='single_row')","metadata":{"execution":{"iopub.status.busy":"2021-06-27T14:55:32.026330Z","iopub.execute_input":"2021-06-27T14:55:32.026617Z","iopub.status.idle":"2021-06-27T14:55:36.597380Z","shell.execute_reply.started":"2021-06-27T14:55:32.026589Z","shell.execute_reply":"2021-06-27T14:55:36.596353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* sacBunts - Game total sacrifice bunts.\n* sacFlies - Game total sacrifice flies.\n* catchersInterference - Game total catchers interference reached on.\n* pickoffs - Game total number of times picked off base.","metadata":{}},{"cell_type":"code","source":"cols2_plot=['sacBunts','sacFlies','catchersInterference','pickoffs']\ngetplayer_box_scores(cols2_plot,fig_size=(20,20),orient='grid')","metadata":{"execution":{"iopub.status.busy":"2021-06-27T14:55:36.598757Z","iopub.execute_input":"2021-06-27T14:55:36.599071Z","iopub.status.idle":"2021-06-27T14:55:40.745188Z","shell.execute_reply.started":"2021-06-27T14:55:36.599045Z","shell.execute_reply":"2021-06-27T14:55:40.744194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* assists - Game total number of assists.\n* putOuts - Game total number of putouts.\n* errors - Game total number of errors.\n* chances - Game total fielding chances.","metadata":{}},{"cell_type":"code","source":"cols2_plot=['assists', 'putOuts', 'errors', 'chances']\ngetplayer_box_scores(cols2_plot,fig_size=(18,36),orient='single_row')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-27T14:55:57.020286Z","iopub.execute_input":"2021-06-27T14:55:57.020871Z","iopub.status.idle":"2021-06-27T14:56:01.722878Z","shell.execute_reply.started":"2021-06-27T14:55:57.020823Z","shell.execute_reply":"2021-06-27T14:56:01.721795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}