{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom sklearn.metrics import mean_absolute_error\nfrom datetime import timedelta\nfrom functools import reduce\nfrom tqdm import tqdm\nimport lightgbm as lgbm\nimport mlb\nimport matplotlib.pyplot as plt\n\nimport plotly.express as px\n\nfeatures that we have now","metadata":{"execution":{"iopub.status.busy":"2021-07-11T09:24:52.712065Z","iopub.execute_input":"2021-07-11T09:24:52.712571Z","iopub.status.idle":"2021-07-11T09:24:55.602842Z","shell.execute_reply.started":"2021-07-11T09:24:52.712469Z","shell.execute_reply":"2021-07-11T09:24:55.601801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# feature_cols = ['label_playerId', 'label_primaryPositionName', 'label_teamId',\n#        'label_status', 'battingOrder', 'gamesPlayedBatting', 'flyOuts',\n#        'groundOuts', 'runsScored', 'doubles', 'triples', 'homeRuns',\n#        'strikeOuts', 'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch',\n#        'atBats', 'caughtStealing', 'stolenBases', 'groundIntoDoublePlay',\n#        'groundIntoTriplePlay', 'plateAppearances', 'totalBases', 'rbi',\n#        'leftOnBase', 'sacBunts', 'sacFlies', 'catchersInterference',\n#        'pickoffs', 'gamesPlayedPitching', 'gamesStartedPitching',\n#        'completeGamesPitching', 'shutoutsPitching', 'winsPitching',\n#        'lossesPitching', 'flyOutsPitching', 'airOutsPitching',\n#        'groundOutsPitching', 'runsPitching', 'doublesPitching',\n#        'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching',\n#        'baseOnBallsPitching', 'intentionalWalksPitching', 'hitsPitching',\n#        'hitByPitchPitching', 'atBatsPitching', 'caughtStealingPitching',\n#        'stolenBasesPitching', 'inningsPitched', 'saveOpportunities',\n#        'earnedRuns', 'battersFaced', 'outsPitching', 'pitchesThrown', 'balls',\n#        'strikes', 'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching',\n#        'rbiPitching', 'gamesFinishedPitching', 'inheritedRunners',\n#        'inheritedRunnersScored', 'catchersInterferencePitching',\n#        'sacBuntsPitching', 'sacFliesPitching', 'saves', 'holds', 'blownSaves',\n#        'assists', 'putOuts', 'errors', 'chances','target1_mean',\n#  'target1_median',\n#  'target1_std',\n#  'target1_min',\n#  'target1_max',\n#  'target1_prob',\n#  'target2_mean',\n#  'target2_median',\n#  'target2_std',\n#  'target2_min',\n#  'target2_max',\n#  'target2_prob',\n#  'target3_mean',\n#  'target3_median',\n#  'target3_std',\n#  'target3_min',\n#  'target3_max',\n#  'target3_prob',\n#  'target4_mean',\n#  'target4_median',\n#  'target4_std',\n#  'target4_min',\n#  'target4_max',\n#  'target4_prob']\n# feature_cols2 = ['label_playerId', 'label_primaryPositionName', 'label_teamId',\n#        'label_status', 'battingOrder', 'gamesPlayedBatting', 'flyOuts',\n#        'groundOuts', 'runsScored', 'doubles', 'triples', 'homeRuns',\n#        'strikeOuts', 'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch',\n#        'atBats', 'caughtStealing', 'stolenBases', 'groundIntoDoublePlay',\n#        'groundIntoTriplePlay', 'plateAppearances', 'totalBases', 'rbi',\n#        'leftOnBase', 'sacBunts', 'sacFlies', 'catchersInterference',\n#        'pickoffs', 'gamesPlayedPitching', 'gamesStartedPitching',\n#        'completeGamesPitching', 'shutoutsPitching', 'winsPitching',\n#        'lossesPitching', 'flyOutsPitching', 'airOutsPitching',\n#        'groundOutsPitching', 'runsPitching', 'doublesPitching',\n#        'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching',\n#        'baseOnBallsPitching', 'intentionalWalksPitching', 'hitsPitching',\n#        'hitByPitchPitching', 'atBatsPitching', 'caughtStealingPitching',\n#        'stolenBasesPitching', 'inningsPitched', 'saveOpportunities',\n#        'earnedRuns', 'battersFaced', 'outsPitching', 'pitchesThrown', 'balls',\n#        'strikes', 'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching',\n#        'rbiPitching', 'gamesFinishedPitching', 'inheritedRunners',\n#        'inheritedRunnersScored', 'catchersInterferencePitching',\n#        'sacBuntsPitching', 'sacFliesPitching', 'saves', 'holds', 'blownSaves',\n#        'assists', 'putOuts', 'errors', 'chances','target1_mean',\n#  'target1_median',\n#  'target1_std',\n#  'target1_min',\n#  'target1_max',\n#  'target1_prob',\n#  'target2_mean',\n#  'target2_median',\n#  'target2_std',\n#  'target2_min',\n#  'target2_max',\n#  'target2_prob',\n#  'target3_mean',\n#  'target3_median',\n#  'target3_std',\n#  'target3_min',\n#  'target3_max',\n#  'target3_prob',\n#  'target4_mean',\n#  'target4_median',\n#  'target4_std',\n#  'target4_min',\n#  'target4_max',\n#  'target4_prob',\n#     'target1']","metadata":{"execution":{"iopub.status.busy":"2021-07-11T09:24:55.604704Z","iopub.execute_input":"2021-07-11T09:24:55.605354Z","iopub.status.idle":"2021-07-11T09:24:55.614079Z","shell.execute_reply.started":"2021-07-11T09:24:55.605303Z","shell.execute_reply":"2021-07-11T09:24:55.612687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_DIR = Path('../input/mlb-player-digital-engagement-forecasting')\nTRAIN_DIR = Path('../input/mlb-pdef-train-dataset')","metadata":{"execution":{"iopub.status.busy":"2021-07-11T09:24:55.616613Z","iopub.execute_input":"2021-07-11T09:24:55.617126Z","iopub.status.idle":"2021-07-11T09:24:55.634944Z","shell.execute_reply.started":"2021-07-11T09:24:55.617075Z","shell.execute_reply":"2021-07-11T09:24:55.633406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### In the current data, we use only \n> rosters_train.pkl <br>\n> nextDayPlayerEngagement_train.pkl <br>\n> playerBoxScores_train.pkl <br>\n> player_target_stats.csv","metadata":{}},{"cell_type":"code","source":"players = pd.read_csv(BASE_DIR / 'players.csv')\n\nrosters = pd.read_pickle(TRAIN_DIR / 'rosters_train.pkl')\ntargets = pd.read_pickle(TRAIN_DIR / 'nextDayPlayerEngagement_train.pkl')\nfollowers = pd.read_pickle(TRAIN_DIR / 'playerTwitterFollowers_train.pkl')\nteam_followers = pd.read_pickle(TRAIN_DIR / 'teamTwitterFollowers_train.pkl')\nteam_followers = team_followers.rename(columns={'numberOfFollowers': 'teamFollowers'})\nscores = pd.read_pickle(TRAIN_DIR / 'playerBoxScores_train.pkl')\nscores = scores.groupby(['playerId', 'date']).sum().reset_index()","metadata":{"execution":{"iopub.status.busy":"2021-07-11T09:28:13.251661Z","iopub.execute_input":"2021-07-11T09:28:13.252062Z","iopub.status.idle":"2021-07-11T09:28:14.914594Z","shell.execute_reply.started":"2021-07-11T09:28:13.252029Z","shell.execute_reply":"2021-07-11T09:28:14.913272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets","metadata":{"execution":{"iopub.status.busy":"2021-07-11T09:28:14.916282Z","iopub.execute_input":"2021-07-11T09:28:14.916591Z","iopub.status.idle":"2021-07-11T09:28:14.942149Z","shell.execute_reply.started":"2021-07-11T09:28:14.916561Z","shell.execute_reply":"2021-07-11T09:28:14.941332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_sub = pd.read_csv('../input/endtoendsolution/submission.csv')\nbest_sub","metadata":{"execution":{"iopub.status.busy":"2021-07-11T09:28:16.096975Z","iopub.execute_input":"2021-07-11T09:28:16.097581Z","iopub.status.idle":"2021-07-11T09:28:16.127356Z","shell.execute_reply.started":"2021-07-11T09:28:16.097532Z","shell.execute_reply":"2021-07-11T09:28:16.126503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2021-07-11T09:28:16.295258Z","iopub.execute_input":"2021-07-11T09:28:16.295940Z","iopub.status.idle":"2021-07-11T09:28:16.301047Z","shell.execute_reply.started":"2021-07-11T09:28:16.295839Z","shell.execute_reply":"2021-07-11T09:28:16.299856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets_cols = ['target1', 'target2', 'target3', 'target4']\n\nfor target in targets_cols:\n    sns.distplot(targets[target])\n    plt.title(target)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-11T09:28:16.844250Z","iopub.execute_input":"2021-07-11T09:28:16.844712Z","iopub.status.idle":"2021-07-11T09:28:57.817363Z","shell.execute_reply.started":"2021-07-11T09:28:16.844674Z","shell.execute_reply":"2021-07-11T09:28:57.816082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets_cols = ['target1', 'target2', 'target3', 'target4']\n\nfor target in targets_cols:\n    sns.distplot(best_sub[target])\n    plt.title(target)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-11T09:29:06.195521Z","iopub.execute_input":"2021-07-11T09:29:06.195932Z","iopub.status.idle":"2021-07-11T09:29:07.488527Z","shell.execute_reply.started":"2021-07-11T09:29:06.195897Z","shell.execute_reply":"2021-07-11T09:29:07.487163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Rosters EDA","metadata":{}},{"cell_type":"markdown","source":"- playerId - Unique identifier for a player.\n- gameDate - dat of the game\n- teamId - teamId that player is on that date.\n- statusCode - Roster status abbreviation.\n- status - Descriptive roster status.","metadata":{}},{"cell_type":"code","source":"rosters.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:33.269665Z","iopub.execute_input":"2021-07-08T06:52:33.270076Z","iopub.status.idle":"2021-07-08T06:52:33.289075Z","shell.execute_reply.started":"2021-07-08T06:52:33.270032Z","shell.execute_reply":"2021-07-08T06:52:33.288082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rosters.status.unique()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:33.290369Z","iopub.execute_input":"2021-07-08T06:52:33.290796Z","iopub.status.idle":"2021-07-08T06:52:33.410639Z","shell.execute_reply.started":"2021-07-08T06:52:33.290754Z","shell.execute_reply":"2021-07-08T06:52:33.409486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"statusCode = rosters.statusCode.value_counts()\n\nfig = px.bar(x=statusCode.index, y=np.log(statusCode.values), text=statusCode.values,\n            labels={'x': 'roster status code', 'y': 'values'},\n            title='Number of players under each roster status(log)', \n            color=statusCode.index)\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:33.412079Z","iopub.execute_input":"2021-07-08T06:52:33.41248Z","iopub.status.idle":"2021-07-08T06:52:34.860361Z","shell.execute_reply.started":"2021-07-08T06:52:33.412439Z","shell.execute_reply":"2021-07-08T06:52:34.859327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rosters['year'] = pd.to_datetime(rosters['date'], format = '%Y%m%d').dt.year","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:34.862424Z","iopub.execute_input":"2021-07-08T06:52:34.862749Z","iopub.status.idle":"2021-07-08T06:52:35.029669Z","shell.execute_reply.started":"2021-07-08T06:52:34.86272Z","shell.execute_reply":"2021-07-08T06:52:35.028692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"status_change_per_year = pd.DataFrame(rosters.groupby('year')['statusCode'].value_counts())\nstatus_change_per_year.columns = ['count']\nstatus_change_per_year = status_change_per_year.reset_index()\n\nfig = px.bar(x=status_change_per_year['statusCode'], y=np.log(status_change_per_year['count']), \n             text=status_change_per_year['count'],\n             animation_frame=status_change_per_year['year'], animation_group=status_change_per_year['count'],\n             labels={'x': 'roster status code', 'y': 'values'},\n             title='Number of players under each roster status per year(log)', \n             color=status_change_per_year['statusCode'])\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:35.031912Z","iopub.execute_input":"2021-07-08T06:52:35.032682Z","iopub.status.idle":"2021-07-08T06:52:35.568379Z","shell.execute_reply.started":"2021-07-08T06:52:35.032634Z","shell.execute_reply":"2021-07-08T06:52:35.567379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Player twitter followers EDA","metadata":{}},{"cell_type":"markdown","source":"Twitter following data was collected by MLB from Twitter APIs for Major League players, on the first of each month, dating back to 1/1/2018. This dataset is not exhaustive of all players over all months, as not every player has/had a Twitter account, players may create/delete/reinstate accounts at random, or other scenarios preventing follower data from being collected on a given date. <br>\n\n- date - Date of follower count.\n- playerId - Unique identifier for a player.\n- playerName\n- accountName - Name on player twitter account.\n- twitterHandle - Player twitter handle.\n- numberOfFollowers - Number of followers","metadata":{}},{"cell_type":"code","source":"followers.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:35.569594Z","iopub.execute_input":"2021-07-08T06:52:35.569892Z","iopub.status.idle":"2021-07-08T06:52:35.583074Z","shell.execute_reply.started":"2021-07-08T06:52:35.569866Z","shell.execute_reply":"2021-07-08T06:52:35.581765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"followers.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:35.58466Z","iopub.execute_input":"2021-07-08T06:52:35.585077Z","iopub.status.idle":"2021-07-08T06:52:35.595832Z","shell.execute_reply.started":"2021-07-08T06:52:35.585036Z","shell.execute_reply":"2021-07-08T06:52:35.595024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"followers.playerName.unique().shape","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:35.597011Z","iopub.execute_input":"2021-07-08T06:52:35.597579Z","iopub.status.idle":"2021-07-08T06:52:35.613611Z","shell.execute_reply.started":"2021-07-08T06:52:35.597537Z","shell.execute_reply":"2021-07-08T06:52:35.612634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"most_famous_player = followers.groupby('playerName')['numberOfFollowers'].max().sort_values(ascending=False)[:20]\n\nfig = px.bar(x=most_famous_player.index, y=most_famous_player.values, text=most_famous_player.values,\n            labels={'x': 'Name', 'y': 'followers'},\n            title='Top 20 the most famous players by tweeter followers', \n            color=most_famous_player.index)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:35.61485Z","iopub.execute_input":"2021-07-08T06:52:35.615391Z","iopub.status.idle":"2021-07-08T06:52:35.765388Z","shell.execute_reply.started":"2021-07-08T06:52:35.615349Z","shell.execute_reply":"2021-07-08T06:52:35.764381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"followers['date'] = pd.to_datetime(followers['date'], format = '%Y%m%d')","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:35.766748Z","iopub.execute_input":"2021-07-08T06:52:35.767032Z","iopub.status.idle":"2021-07-08T06:52:35.774023Z","shell.execute_reply.started":"2021-07-08T06:52:35.767003Z","shell.execute_reply":"2021-07-08T06:52:35.773108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"followers_growth = followers.groupby(['playerId', 'date']).agg({'numberOfFollowers':'min'}).reset_index()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:11.771939Z","iopub.status.idle":"2021-07-08T06:52:11.772402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_followers = \\\n    pd.DataFrame(followers_growth.groupby('playerId').agg({'numberOfFollowers':'min'})['numberOfFollowers']).reset_index()\nmin_followers.columns = ['playerId', 'min_f']\n\nmax_followers = \\\n    pd.DataFrame(followers_growth.groupby('playerId').agg({'numberOfFollowers':'max'})['numberOfFollowers']).reset_index()\nmax_followers.columns = ['playerId', 'max_f']","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:11.780085Z","iopub.status.idle":"2021-07-08T06:52:11.780597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_growth(min_max_dataset):\n    A = min_max_dataset['min_f']\n    B = min_max_dataset['max_f']\n    return (B-A)/A*100","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:11.782077Z","iopub.status.idle":"2021-07-08T06:52:11.782535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_max_followers = min_followers.merge(max_followers, on='playerId', how='left')\nmin_max_followers['growth%'] = get_growth(min_max_followers)\nmin_max_followers['growth_num'] = min_max_followers['max_f']-min_max_followers['min_f']\nmin_max_followers = min_max_followers.sort_values(by=['growth_num', 'growth%'], ascending=False)[:20]","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:11.794545Z","iopub.status.idle":"2021-07-08T06:52:11.795155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_max_followers","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:11.801165Z","iopub.status.idle":"2021-07-08T06:52:11.801746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.bar(x=min_max_followers['playerId'], y=min_max_followers['growth_num'], \n             text=round(min_max_followers['growth%']),\n             labels={'x': 'playerId', 'y': 'followers'},\n             title='Top 20 fastest growing players by tweeter followers', \n             color=min_max_followers['playerId'])\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:11.813086Z","iopub.status.idle":"2021-07-08T06:52:11.813702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Team followers EDA","metadata":{}},{"cell_type":"markdown","source":"Twitter following data was collected by MLB from Twitter APIs for all 30 Major League teams, on the first of each month, dating back to 1/1/2018. <br>\n\n- date - Date of follower count.\n- teamId - Unique identifier for a team.\n- teamName\n- accountName - Name on team twitter account.\n- twitterHandle - Team twitter handle.\n- numberOfFollowers - Number of followers.","metadata":{}},{"cell_type":"code","source":"team_followers.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:11.840966Z","iopub.status.idle":"2021-07-08T06:52:11.841591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"team_followers.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:11.85074Z","iopub.status.idle":"2021-07-08T06:52:11.851501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"most_famous_teams = team_followers.groupby('teamName')['teamFollowers'].max().sort_values(ascending=False)\n\nfig = px.bar(x=most_famous_teams.index, y=most_famous_teams.values, text=most_famous_teams.values,\n            labels={'x': 'Team name', 'y': 'followers'},\n            title='The most famous teams by tweeter followers', \n            color=most_famous_teams.index)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:11.858906Z","iopub.status.idle":"2021-07-08T06:52:11.859584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Player Box Scores EDA","metadata":{}},{"cell_type":"markdown","source":"- home - Binary, 1 if home team, 0 if away.\n- gamePk - unique identifier for game.\n- gameDate\n- gameTimeUTC - First pitch time in UTC.\n- teamId - Unique identifier for team.\n- teamName\n- playerId - Unique identifier for player.\n- playerName\n- jerseyNum\n- positionCode - Number position code, details are here.\n- positionName - Text position display, details are here.\n- positionType - Position group, details are here.\n- battingOrder - Format: “###”, where the first digit indicates the batting order spot, and the second two digits indicate the sequence in which that player occupied that batting order spot. Examples: “300”, which indicates the starter in the third spot in the batting order; “903,” which indicates the fourth player (after 900, 901 and 902) occupy the ninth spot in the batting order. Only populates if appeared in game.\n- gamesPlayedBatting - 1 if player entered the game as a batter, runner or fielder.\n- flyOuts - Game total fly outs.\n- groundOuts - Game total ground outs.\n- runsScored - Game total runs scored.\n- doubles - Game total doubles.\n- triples - Game total triples.\n- homeRuns - Game total home runs.\n- strikeOuts - Game total strike outs.\n- baseOnBalls - Game total walks.\n- intentionalWalks - Game total intentional walks.\n- hits - Game total hits.\n- hitByPitch - Game total hit by pitches.\n- atBats - Game total at-bats.\n- caughtStealing - Game total caught stealing.\n- stolenBases - Game total stolen bases.\n- groundIntoDoublePlay - Game total double plays grounded into.\n- groundIntoTriplePlay - Game total triple plays grounded into.\n- plateAppearances - Game total plate appearances.\n- totalBases - Game total bases.\n- rbi - Game total runs batted in.\n- leftOnBase - Game total runners left on base.\n- sacBunts - Game total sacrifice bunts.\n- sacFlies - Game total sacrifice flies.\n- catchersInterference - Game total catchers interference reached on.\n- pickoffs - Game total number of times picked off base.\n- gamesPlayedPitching- Binary, 1 if player entered the game as a pitcher.\n- gamesStartedPitching- Binary, 1 if player was game’s starting pitcher.\n- completeGamesPitching - Binary, 1 if credited with complete game.\n- shutoutsPitching - Binary, 1 if credited with shutout.\n- winsPitching - Binary, 1 if credited with win.\n- lossesPitching - Binary, 1 if credited with loss.\n- flyOutsPitching - Game total of flyouts allowed.\n- airOutsPitching - Game total of air outs (flyouts + popouts) allowed.\n- groundOutsPitching - Game total ground outs allowed.\n- runsPitching - Game total runs allowed.\n- doublesPitching - Game total doubles allowed.\n- triplesPitching - Game total triples allowed.\n- homeRunsPitching - Game total home runs allowed.\n- strikeOutsPitching - Game total strike outs allowed.\n- baseOnBallsPitching - Game total walks allowed.\n- intentionalWalksPitching - Game total intentional walks allowed.\n- hitsPitching - Game total hits allowed.\n- hitByPitchPitching - Game total hit by pitches allowed.\n- atBatsPitching - Game total at-bats against.\n- caughtStealingPitching - Game total caught stealing against.\n- stolenBasesPitching - Game total stolen bases allowed.\n- inningsPitched - Game total innings pitched.\n- saveOpportunities - Binary, 1 if credited with save opportunity.\n- earnedRuns - Game total earned runs allowed.\n- battersFaced - Game total batters faced.\n- outsPitching - Game total outs recorded.\n- pitchesThrown - Game total number of pitches thrown.\n- balls - Game total balls thrown.\n- strikes - Game total strikes thrown.\n- hitBatsmen - Game total hit batters.\n- balks - Game total balks.\n- wildPitches - Game total number of wild pitches thrown.\n- pickoffsPitching - Game total number of pickoffs.\n- rbiPitching - Game total number of runs batted in allowed.\n- inheritedRunners - Game total of inherited runners assumed.\n- inheritedRunnersScored- Game total of inherited runners that scored.\n- catchersInterferencePitching - Game total of catcher’s interference occurred by battery.\n- sacBuntsPitching - Game total sacrifice bunts allowed.\n- sacFliesPitching - Game total sacrifice flies allowed.\n- saves - Binary, 1 if credited with save.\n- holds - Binary, 1 if credited with hold.\n- blownSaves - Binary, 1 if credited with blown save.\n- assists - Game total number of assists.\n- putOuts - Game total number of putouts.\n- errors - Game total number of errors.\n- chances - Game total fielding chances.","metadata":{}},{"cell_type":"code","source":"scores.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:11.882862Z","iopub.execute_input":"2021-07-08T06:52:11.883496Z","iopub.status.idle":"2021-07-08T06:52:11.901692Z","shell.execute_reply.started":"2021-07-08T06:52:11.883444Z","shell.execute_reply":"2021-07-08T06:52:11.899947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"st = scores.columns[2:45]\nnd = scores.columns[45:]","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:11.902911Z","iopub.status.idle":"2021-07-08T06:52:11.903469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores[st].describe()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:11.905333Z","iopub.status.idle":"2021-07-08T06:52:11.906087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores[nd].describe()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:11.915905Z","iopub.status.idle":"2021-07-08T06:52:11.91648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Awards EDA","metadata":{}},{"cell_type":"code","source":"awards = pd.read_csv(TRAIN_DIR / 'awards_train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:11.933325Z","iopub.status.idle":"2021-07-08T06:52:11.933937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"awards['awardDateMonth'] = pd.to_datetime(awards['awardDate'], format = '%Y-%m-%d').dt.month\nawards['awardDateYear'] = pd.to_datetime(awards['awardDate'], format = '%Y-%m-%d').dt.year","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:11.942018Z","iopub.status.idle":"2021-07-08T06:52:11.942569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"awards.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:11.953787Z","iopub.status.idle":"2021-07-08T06:52:11.954569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"most_awarded_player_per_year = pd.DataFrame(awards.groupby('awardDateYear')['playerName'].value_counts())\nmost_awarded_player_per_year.columns = ['num_of_awards']\nmost_awarded_player_per_year = most_awarded_player_per_year.reset_index()\nyear2018 = most_awarded_player_per_year[most_awarded_player_per_year['awardDateYear'] == 2018][:20]\nyear2019 = most_awarded_player_per_year[most_awarded_player_per_year['awardDateYear'] == 2019][:20]\nyear2020 = most_awarded_player_per_year[most_awarded_player_per_year['awardDateYear'] == 2020][:20]\nyear2021 = most_awarded_player_per_year[most_awarded_player_per_year['awardDateYear'] == 2021][:20]\nmost_awarded_player_per_year = year2018.append([year2019, year2020, year2021])\n\nfig = px.bar(x=most_awarded_player_per_year['playerName'], y=most_awarded_player_per_year['num_of_awards'], \n             text=most_awarded_player_per_year['num_of_awards'],\n             animation_frame=most_awarded_player_per_year['awardDateYear'],\n             animation_group=most_awarded_player_per_year['num_of_awards'],\n             labels={'y': 'number of awards', 'x': ' '},\n             title='Top the most awarded players per year', \n             color=most_awarded_player_per_year['playerName'])\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:11.961506Z","iopub.status.idle":"2021-07-08T06:52:11.962283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"most_frequent_award = awards.awardName.value_counts()[:25]\n\nfig = px.pie(most_frequent_award, values=most_frequent_award.values, names=most_frequent_award.index,\n             title='Most frequent awards')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:11.972741Z","iopub.status.idle":"2021-07-08T06:52:11.973483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_awards(playerid=477132):\n    if len(awards[awards.playerId==playerid]['awardDateYear'].value_counts()) > 0:\n        awards[awards.playerId==playerid]['awardDateYear'].value_counts().plot(kind = 'bar')\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:11.982432Z","iopub.status.idle":"2021-07-08T06:52:11.983158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_awards(477132)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:11.992694Z","iopub.status.idle":"2021-07-08T06:52:11.993408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Conclusion","metadata":{}},{"cell_type":"markdown","source":"- about rosters: most players play under the A status and I don't think that this can help in training the model.\n- about players followers: it is logical that with high popularity, the interest of fans will be greater than with a low number of readers. I also showed the growth of followers on the graph, this can serve as a feature, as it seems to me, but the correctness still needs to be improved.\n- about teams followers: the players in the most popular teams were followed on average more than in the unpopular ones, and this can also become a feature.\n- about awards: here, too, it is interesting to do. Players receive awards and fan engagement increases, if this award is some kind of prestigious, then interest rises even more.\n\nI also want to make an EDA based on other data such as standings, teamBoxScores, transactions, games.","metadata":{}},{"cell_type":"markdown","source":"# Events EDA","metadata":{}},{"cell_type":"markdown","source":"* gamePk - unique identifier for game.\n* gameDate\n* gameTimeUTC - First pitch time in UTC.\n* season\n* gameType - Game type, various types can be found here.\n* playId - Statcast play guid.\n* eventId\n* inning - Inning AB occurred.\n* halfInning- “top” or “bottom” inning indicator.\n* homeScore - Home score at start of the event.\n* awayScore - Away score at start of the event.\n* menOnBase - Split used if runner on base – ie (RISP, Empty).\n* atBatIndex - At-Bat index number, begins with 0.\n* atBatDesc - Play-by-play description of AB.\n* atBatEvent - Event type result of the atBat, various types can be found here.\n* hasOut - Binary, 1 if runner is out on play.\n* pitcherTeamId - Unique identifier for pitching team.\n* isPitcherHome - Binary, 1 if pitcher is home team.\n* pitcherTeam - team name of pitching team.\n* hitterTeamId - Unique identifier for hitting team.\n* hitterTeam - Team name of hitting team.\n* pitcherId\n* pitcherName\n* isStarter - Binary, 1 if player was game’s starting pitcher.\n* pitcherHand - Player throw hand: “L”,”R”.\n* hitterId\n* hitterName\n* batSide - Player bat side: “L”,”R”.\n* pitchNumber - Pitch sequence number of AB.\n* balls - Ball count after the event.\n* strikes - Strike count after the event.\n* isGB - Binary, 1 if at bat is a ground ball.\n* isLD - Binary, 1 if at bat is a line drive.\n* isFB - Binary, 1 if at bat is a fly ball.\n* isPU - Binary, 1 if at bat is a pop up.\n* launchSpeed - Measured speed of the hit ball.\n* launchAngle - Vertical angle relative to the horizon at which the hit was launched.\n* totalDistance - Total distance ball traveled.\n* event - Events that may occur during an at-bat: strikeout, groundout, substitutions, stolen base, wild pitch, game advisory, etc.\n* description - Text description of event.\n* rbi - Number of runs batted in during AB.\n* pitchType - Pitch type classification code, various types can be found here.\n* call - Pitch or throw result classification code, various types can be found here.\n* outs - Current/Final outs of AB.\n* inPlay - True/False if ball was hit into play.\n* isPaOver - Binary, 1 if this event is the end of the plate appearance.\n* startSpeed - Speed in MPH of the ball at 50 feet in front of home plate.\n* endSpeed - Speed in MPH of the ball as it crosses the front edge of home plate (0,0 in the x axis).\n* nastyFactor - Evaluates several properties of each pitch, and rates the \"nastiness\" of the pitch on a scale from 0-100.\n* breakAngle - Degrees clockwise (batter’s view) that the plane of the pitch deviates from the vertical.\n* breakLength - Max distance that the pitch separates from the straight line between pitch start and pitch end.\n* breakY - Distance from home plate where the break is greatest.\n* spinRate - Rate of spin on the ball after it was released by pitcher in RPMs.\n* spinDirection - The axis of rotation for the ball at release given as an angle that reflects how the spin will influence the ball trajectory. Pure back-spin is 180 degrees, pure side-spin that puts the ball to the 1b side is 90 degrees, pure-side spin that pulls the ball to the 3b side is 270 degrees, and pure top-spin is 0 or 360 degrees.\n* pX - Horizontal position in feet of the ball as it crosses the front axis of home plate.\n* pZ - Vertical position in feet above home plate of the ball as it crosses the front axis of home plate.\n* aX - Ball acceleration on the z axis.\n* aY - Ball acceleration on the y axis.\n* aZ - Ball acceleration on the z axis.\n* pfxX - Horizontal movement of the ball in inches.\n* pfxZ - Vertical movement of the ball in inches.\n* vX0 - Velocity of the ball from the x axis.\n* vY0 - Velocity of the ball from the y axis; this is negative because 0,0,0 is behind the batter and the ball travels from pitcher mound towards 0,0,0.\n* vZ0 - Velocity of the ball from the z axis.\n* x - X coordinate where pitch crossed front of home plate.\n* y - Y coordinate where pitch crossed front of home plate.\n* x0 - Coordinate location of the ball at the point it was released from the pitchers hand on the x axis (time = 0).\n* y0 - Coordinate location of the ball at the point it was released from the pitchers hand on the y axis (time = 0).\n* z0 - Coordinate location of the ball at the point it was released from the pitchers hand on the z axis (time = 0).\n* type - Type of event, either ‘pitch’ or ‘action’\n* zone - Zone location number - see image below.","metadata":{}},{"cell_type":"code","source":"events = pd.read_csv('../input/mlb-pdef-train-dataset/events_train.csv')\nevents.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:12.035458Z","iopub.execute_input":"2021-07-08T06:52:12.036131Z","iopub.status.idle":"2021-07-08T06:52:12.057543Z","shell.execute_reply.started":"2021-07-08T06:52:12.036091Z","shell.execute_reply":"2021-07-08T06:52:12.055486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events.event.unique()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:12.05885Z","iopub.status.idle":"2021-07-08T06:52:12.059356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events['year'] = pd.to_datetime(events.date, format = '%Y%m%d').dt.year\nevents['month'] = pd.to_datetime(events.date, format = '%Y%m%d').dt.month","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:12.064804Z","iopub.status.idle":"2021-07-08T06:52:12.065436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events2020 = events[events['year'] == 2020]\nevents2020.month.unique()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:12.081684Z","iopub.status.idle":"2021-07-08T06:52:12.082443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"most_frequent_events = events.event.value_counts()[:25]\n\nfig = px.pie(most_frequent_events, values=most_frequent_events.values, names=most_frequent_events.index,\n             title='Most frequent events')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:12.093073Z","iopub.status.idle":"2021-07-08T06:52:12.093878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:12.875675Z","iopub.execute_input":"2021-07-08T06:52:12.876266Z","iopub.status.idle":"2021-07-08T06:52:12.894069Z","shell.execute_reply.started":"2021-07-08T06:52:12.876232Z","shell.execute_reply":"2021-07-08T06:52:12.892848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events.gameType.unique()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:52:13.39704Z","iopub.execute_input":"2021-07-08T06:52:13.397506Z","iopub.status.idle":"2021-07-08T06:52:13.417808Z","shell.execute_reply.started":"2021-07-08T06:52:13.397469Z","shell.execute_reply":"2021-07-08T06:52:13.416084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* R : Regular Season\n* F : Wild Card Game\n* D : Division Series\n* L : League Championship Series\n* W : World Series","metadata":{}},{"cell_type":"code","source":"gameType = events.gameType.value_counts()\ntext = {'R' : 'Regular Season',\n               'F' : 'Wild Card Game',\n               'D' : 'Division Series',\n               'L' : 'League Championship Series',\n               'W' : 'World Series'}\n\nfig = px.bar(gameType, x=gameType.index, y=np.log(gameType.values),\n             color=gameType.index, text=text,\n             title='log number of game types',\n             labels={'index':'type', 'y':'value'})\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-07T09:01:05.757528Z","iopub.execute_input":"2021-07-07T09:01:05.757805Z","iopub.status.idle":"2021-07-07T09:01:06.284803Z","shell.execute_reply.started":"2021-07-07T09:01:05.757779Z","shell.execute_reply":"2021-07-07T09:01:06.283615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inning = events.inning\n\nfig = px.histogram(inning, nbins=20,\n                   title='distribution of innings')\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-07T09:01:06.287103Z","iopub.execute_input":"2021-07-07T09:01:06.287729Z","iopub.status.idle":"2021-07-07T09:01:17.482246Z","shell.execute_reply.started":"2021-07-07T09:01:06.28768Z","shell.execute_reply":"2021-07-07T09:01:17.480747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#","metadata":{}},{"cell_type":"markdown","source":"# Target EDA","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\nsns.set_style('whitegrid')\nsns.set(font_scale = 1.5)\nfig, axs = plt.subplots(2,2, figsize = (20, 10))\nsns.kdeplot(ax=axs[0,0], data=targets['target1'])\nsns.kdeplot(ax=axs[0,1], data=targets['target2'])\nsns.kdeplot(ax=axs[1,0], data=targets['target3'])\nsns.kdeplot(ax=axs[1,1], data=targets['target4'])\nbbox = axs[0,0].get_position()\nbbox2 = axs[0,1].get_position()\n\ncenter=(bbox2.x1) * 0.4 + (bbox.x1) * 0.25\nplt.suptitle('Distribution of targets', x = center)\n","metadata":{"execution":{"iopub.status.busy":"2021-07-01T09:23:58.848216Z","iopub.execute_input":"2021-07-01T09:23:58.848535Z","iopub.status.idle":"2021-07-01T09:24:47.963864Z","shell.execute_reply.started":"2021-07-01T09:23:58.848505Z","shell.execute_reply":"2021-07-01T09:24:47.962944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def draw_kde_plot(col = 'target1'):\n    sns.set_style('whitegrid')\n    sns.set(font_scale = 1.5)\n    fig, axs = plt.subplots(2,2, figsize = (15, 10))\n    g = sns.kdeplot(ax=axs[0,0], data=targets[col])\n    g.set_xlabel('original')\n    g = sns.kdeplot(ax=axs[0,1], data=targets[col]**2)\n    g.set_xlabel('squared')\n    g = sns.kdeplot(ax=axs[1,0], data=targets[col]**4)\n    g.set_xlabel('power 4')\n    g = sns.kdeplot(ax=axs[1,1], data = np.log(targets[col]+1))\n    g.set_xlabel('log')\n\n\n\n    bbox = axs[0,0].get_position()\n    bbox2 = axs[0,1].get_position()\n    center=(bbox2.x1) * 0.4 + (bbox.x1) * 0.25\n    plt.suptitle(f'Transformation of {col}', x = center)\n    plt.tight_layout()\n","metadata":{"execution":{"iopub.status.busy":"2021-07-01T09:24:47.96617Z","iopub.execute_input":"2021-07-01T09:24:47.966625Z","iopub.status.idle":"2021-07-01T09:24:47.977536Z","shell.execute_reply.started":"2021-07-01T09:24:47.966573Z","shell.execute_reply":"2021-07-01T09:24:47.97655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# **Target2 has highest Skewness**\n","metadata":{}},{"cell_type":"code","source":"for col in ['target1', 'target2', 'target3', 'target4']:\n    draw_kde_plot(col)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T09:24:47.979022Z","iopub.execute_input":"2021-07-01T09:24:47.97933Z","iopub.status.idle":"2021-07-01T09:27:57.035147Z","shell.execute_reply.started":"2021-07-01T09:24:47.979299Z","shell.execute_reply":"2021-07-01T09:27:57.034047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_style('whitegrid')\nsns.set(font_scale = 1.5)\n\n\nfig, axs = plt.subplots(1,1, figsize = (20,8))\nsns.lineplot(ax=axs, x = np.arange(1,10001),\n             y = targets.sample(10000, random_state=500)['target1'],\n             legend='full', label = 'target1')\nsns.lineplot(ax=axs, x = np.arange(1,10001),\n             y = targets.sample(10000, random_state=500)['target2'],\n             legend='full', label = 'target2')\nsns.lineplot(ax=axs, x = np.arange(1,10001), \n             y = targets.sample(10000, random_state=500)['target3'], \n             legend='full', label = 'target3')\nsns.lineplot(ax=axs,x = np.arange(1,10001), \n             y = targets.sample(10000, random_state=500)['target4'], \n             legend='full', label = 'target4')\n\nbbox = axs.get_position()\ncenter=0.5*(bbox.x1)\nplt.suptitle('Comparision of targets', x = center)\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-01T09:27:57.036614Z","iopub.execute_input":"2021-07-01T09:27:57.036965Z","iopub.status.idle":"2021-07-01T09:28:00.178885Z","shell.execute_reply.started":"2021-07-01T09:27:57.03693Z","shell.execute_reply":"2021-07-01T09:28:00.177764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_style('ticks')\nsns.set(font_scale = 1.5)\n\n\nfig, axs = plt.subplots(2,2, figsize = (20,8))\nsns.lineplot(ax=axs[0,0], x = np.arange(1,10001),\n             y = targets.sample(10000, random_state=500)['target1'],\n             legend='full', label = 'target1')\nsns.lineplot(ax=axs[0,1], x = np.arange(1,10001),\n             y = targets.sample(10000, random_state=500)['target2'],\n             legend='full', label = 'target2')\nsns.lineplot(ax=axs[1,0], x = np.arange(1,10001), \n             y = targets.sample(10000, random_state=500)['target3'], \n             legend='full', label = 'target3')\nsns.lineplot(ax=axs[1,1], x = np.arange(1,10001), \n             y = targets.sample(10000, random_state=500)['target4'], \n             legend='full', label = 'target4')\n\nplt.title('Comparision of targets, side by side view')\n","metadata":{"execution":{"iopub.status.busy":"2021-07-01T09:28:00.180346Z","iopub.execute_input":"2021-07-01T09:28:00.180693Z","iopub.status.idle":"2021-07-01T09:28:03.779776Z","shell.execute_reply.started":"2021-07-01T09:28:00.180624Z","shell.execute_reply":"2021-07-01T09:28:03.778443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets['year'] = pd.to_datetime(targets['date'], format = '%Y%m%d').dt.year","metadata":{"execution":{"iopub.status.busy":"2021-07-01T09:28:03.781369Z","iopub.execute_input":"2021-07-01T09:28:03.781727Z","iopub.status.idle":"2021-07-01T09:28:04.056558Z","shell.execute_reply.started":"2021-07-01T09:28:03.78169Z","shell.execute_reply":"2021-07-01T09:28:04.055393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# We have less data for year 4, since we need to predict for the future","metadata":{}},{"cell_type":"markdown","source":"May be we should have different validation strategy","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"targets['year'].value_counts().plot(kind = 'bar')","metadata":{"execution":{"iopub.status.busy":"2021-07-01T09:28:04.058171Z","iopub.execute_input":"2021-07-01T09:28:04.058549Z","iopub.status.idle":"2021-07-01T09:28:04.223492Z","shell.execute_reply.started":"2021-07-01T09:28:04.058514Z","shell.execute_reply":"2021-07-01T09:28:04.222322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\nsns.set(style=\"whitegrid\")\nsns.set(font_scale = 2)\nsns.color_palette(\"Set2\")\n\n\n\nfig, axs = plt.subplots(4,1, figsize = (20,20))\nsns.lineplot(ax=axs[0], \n             x = np.arange(1,10001),\n             data = targets.sample(10000, random_state=100),\n             y = 'target1',\n             hue = 'year',\n             palette='tab10',\n             linewidth=2.5)\n\nsns.lineplot(ax=axs[1], \n             x = np.arange(1,10001),\n             data = targets.sample(10000, random_state=100),\n             y = 'target2',\n             hue = 'year',\n             palette='tab10',\n             linewidth=2.5)\n\nsns.lineplot(ax=axs[2], \n             x = np.arange(1,10001),\n             data = targets.sample(10000, random_state=100),\n             y = 'target3',\n             hue = 'year',\n             palette='tab10',\n             linewidth=2.5)\n\nsns.lineplot(ax=axs[3], \n             x = np.arange(1,10001),\n             data = targets.sample(10000, random_state=100),\n             y = 'target4',\n             hue = 'year',\n             palette='tab10',\n             linewidth=2.5)\n\nbbox = axs[0].get_position()\ncenter=0.5*(bbox.x1)\nplt.suptitle('targets over years', x = center)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T09:28:04.224738Z","iopub.execute_input":"2021-07-01T09:28:04.225034Z","iopub.status.idle":"2021-07-01T09:28:08.262092Z","shell.execute_reply.started":"2021-07-01T09:28:04.225005Z","shell.execute_reply":"2021-07-01T09:28:08.260884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef plot_target_for_player(col = 'target1', playerid = 683734):\n    \n    sns.set(style=\"whitegrid\")\n    sns.set(font_scale = 2)\n    sns.color_palette(\"Set2\")\n\n    fig, axs = plt.subplots(1,1, figsize = (20,8))\n\n    sns.lineplot(ax=axs, x = np.arange(365),\n                 data = targets[((targets.year==2018) & (targets.playerId==playerid))],\n                 y = col,\n                 label = '2018',\n                 linewidth=2.5)\n\n    sns.lineplot(ax=axs, \n                 x =  np.arange(365),\n                 data = targets[((targets.year==2019) & (targets.playerId==playerid))],\n                 y = col,\n                 label = '2019',\n                 linewidth=2.5)\n\n    sns.lineplot(ax=axs, \n                 x =  np.arange(366),\n                 data = targets[((targets.year==2020) & (targets.playerId==playerid))],\n                 y = col,\n                 label = '2020',\n                 linewidth=2.5)\n\n    sns.lineplot(ax=axs, \n                 x =  np.arange(120),\n                 data = targets[((targets.year==2021) & (targets.playerId==playerid))],\n                 y = col,\n                 label = '2021',\n                 linewidth=2.5)\n    \n    bbox = axs.get_position()\n    center=0.5*(bbox.x1)\n    plt.suptitle(f'player Id {playerid}', x = center)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T09:28:08.263713Z","iopub.execute_input":"2021-07-01T09:28:08.264156Z","iopub.status.idle":"2021-07-01T09:28:08.27989Z","shell.execute_reply.started":"2021-07-01T09:28:08.264108Z","shell.execute_reply":"2021-07-01T09:28:08.278518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# There is definitely seasonality for targets, seems like we can remove year 2018 from modelling","metadata":{}},{"cell_type":"code","source":"plot_target_for_player()","metadata":{"execution":{"iopub.status.busy":"2021-07-01T09:28:08.28145Z","iopub.execute_input":"2021-07-01T09:28:08.281857Z","iopub.status.idle":"2021-07-01T09:28:08.77893Z","shell.execute_reply.started":"2021-07-01T09:28:08.281796Z","shell.execute_reply":"2021-07-01T09:28:08.777715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_target_for_player('target2')","metadata":{"execution":{"iopub.status.busy":"2021-07-01T09:28:08.780455Z","iopub.execute_input":"2021-07-01T09:28:08.780815Z","iopub.status.idle":"2021-07-01T09:28:09.259985Z","shell.execute_reply.started":"2021-07-01T09:28:08.780773Z","shell.execute_reply":"2021-07-01T09:28:09.258804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_target_for_player('target3')","metadata":{"execution":{"iopub.status.busy":"2021-07-01T09:28:09.261716Z","iopub.execute_input":"2021-07-01T09:28:09.262161Z","iopub.status.idle":"2021-07-01T09:28:09.740791Z","shell.execute_reply.started":"2021-07-01T09:28:09.262111Z","shell.execute_reply":"2021-07-01T09:28:09.739488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_target_for_player('target4')","metadata":{"execution":{"iopub.status.busy":"2021-07-01T09:28:09.742294Z","iopub.execute_input":"2021-07-01T09:28:09.742672Z","iopub.status.idle":"2021-07-01T09:28:10.234195Z","shell.execute_reply.started":"2021-07-01T09:28:09.742604Z","shell.execute_reply":"2021-07-01T09:28:10.232919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_target_for_player('target1',477132)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T09:28:10.235831Z","iopub.execute_input":"2021-07-01T09:28:10.236161Z","iopub.status.idle":"2021-07-01T09:28:10.764833Z","shell.execute_reply.started":"2021-07-01T09:28:10.236129Z","shell.execute_reply":"2021-07-01T09:28:10.763745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_target_for_player('target2',477132)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T09:28:10.766234Z","iopub.execute_input":"2021-07-01T09:28:10.766574Z","iopub.status.idle":"2021-07-01T09:28:11.343394Z","shell.execute_reply.started":"2021-07-01T09:28:10.766538Z","shell.execute_reply":"2021-07-01T09:28:11.342369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_target_for_player('target3',477132)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T09:28:11.344899Z","iopub.execute_input":"2021-07-01T09:28:11.345527Z","iopub.status.idle":"2021-07-01T09:28:11.868718Z","shell.execute_reply.started":"2021-07-01T09:28:11.345479Z","shell.execute_reply":"2021-07-01T09:28:11.867896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_target_for_player('target4',477132)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T09:28:11.869773Z","iopub.execute_input":"2021-07-01T09:28:11.870198Z","iopub.status.idle":"2021-07-01T09:28:12.370567Z","shell.execute_reply.started":"2021-07-01T09:28:11.870167Z","shell.execute_reply":"2021-07-01T09:28:12.369696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# There seems to be relationship between number of awards and targets, higher awards the player is popular","metadata":{}},{"cell_type":"code","source":"from scipy.stats import boxcox\nxt, _ = boxcox(targets['target1'].values + 1)\nsns.distplot(xt)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T09:28:12.371676Z","iopub.execute_input":"2021-07-01T09:28:12.372113Z","iopub.status.idle":"2021-07-01T09:28:28.858118Z","shell.execute_reply.started":"2021-07-01T09:28:12.37208Z","shell.execute_reply":"2021-07-01T09:28:28.856983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xt, _ = boxcox(targets['target4'].values + 1)\nsns.distplot(xt)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T09:28:28.859544Z","iopub.execute_input":"2021-07-01T09:28:28.859937Z","iopub.status.idle":"2021-07-01T09:28:45.880542Z","shell.execute_reply.started":"2021-07-01T09:28:28.859901Z","shell.execute_reply":"2021-07-01T09:28:45.879433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xt, _ = boxcox(targets['target3'].values + 1)\nsns.distplot(xt)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T09:28:45.882179Z","iopub.execute_input":"2021-07-01T09:28:45.882523Z","iopub.status.idle":"2021-07-01T09:29:01.745909Z","shell.execute_reply.started":"2021-07-01T09:28:45.882486Z","shell.execute_reply":"2021-07-01T09:29:01.744768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Target4 has highest correlation with Twitter follower count**","metadata":{}},{"cell_type":"markdown","source":"we use now ->\n","metadata":{}},{"cell_type":"markdown","source":"['label_playerId', 'label_primaryPositionName', 'label_teamId',\n'label_status', 'battingOrder', 'gamesPlayedBatting', 'flyOuts',\n'groundOuts', 'runsScored', 'doubles', 'triples', 'homeRuns',\n'strikeOuts', 'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch',\n'atBats', 'caughtStealing', 'stolenBases', 'groundIntoDoublePlay',\n'groundIntoTriplePlay', 'plateAppearances', 'totalBases', 'rbi',\n'leftOnBase', 'sacBunts', 'sacFlies', 'catchersInterference',\n'pickoffs', 'gamesPlayedPitching', 'gamesStartedPitching',\n'completeGamesPitching', 'shutoutsPitching', 'winsPitching',\n'lossesPitching', 'flyOutsPitching', 'airOutsPitching',\n'groundOutsPitching', 'runsPitching', 'doublesPitching',\n'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching',\n'baseOnBallsPitching', 'intentionalWalksPitching', 'hitsPitching',\n'hitByPitchPitching', 'atBatsPitching', 'caughtStealingPitching',\n'stolenBasesPitching', 'inningsPitched', 'saveOpportunities',\n'earnedRuns', 'battersFaced', 'outsPitching', 'pitchesThrown', 'balls',\n'strikes', 'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching',\n'rbiPitching', 'gamesFinishedPitching', 'inheritedRunners',\n'inheritedRunnersScored', 'catchersInterferencePitching',\n'sacBuntsPitching', 'sacFliesPitching', 'saves', 'holds', 'blownSaves',\n'assists', 'putOuts', 'errors', 'chances','target1_mean',\n'target1_median',\n'target1_std',\n'target1_min',\n'target1_max',\n'target1_prob','target2_mean','target2_median','target2_std','target2_min',\n'target2_max','target2_prob','target3_mean','target3_median',target3_std','target3_min','target3_max','target3_prob','target4_mean','target4_median',\n'target4_std','target4_min','target4_max','target4_prob']","metadata":{}},{"cell_type":"markdown","source":"correlated(<0.5) ->","metadata":{}},{"cell_type":"markdown","source":"groundIntoTriplePlay, caughtStealing, stolenBases, triples, sacBuntsPitching, catchersInterference, pickoffs, gamesPlayedPitching, gamesStartedPitching, completeGamesPitching, shutoutsPitching, winsPitching, lossesPitching, flyOutsPitching, airOutsPitching, groundOutsPitching, runsPitching,\nTotal doublesPitching, Total triplesPitching, Totla homeRunsPitching, strikeOutsPitching, Total baseOnBallsPitching, \nTotal internationalWalksPitching, Total hitsPitching, Total hitByPitchPitching, Total atBatsPitching, TotalcaughtSteallingPitching,\nTotal stolenBasesPitching, Total inningsPitched, saveOpportunities, earnedRuns, battersFaced, outsPitching, \nTotal pitchesThrown, Total balls, Total strikes, Total hitBatsmen, Total balks, Total wildPitches, Total pickoffsPitching, rbiPitching\nTotal gamesFinishedPitching, Total inheritedRunners,  Total inheritedRunnersScored, Total catchersInferencePitching, sacBuntsPitching, \nTotal sacFliesPitching, Total saves, Total holds, Total blownSaves, Total assists, Total putOuts, \n","metadata":{}},{"cell_type":"code","source":"import seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2021-07-08T07:02:54.228987Z","iopub.execute_input":"2021-07-08T07:02:54.229337Z","iopub.status.idle":"2021-07-08T07:02:54.307799Z","shell.execute_reply.started":"2021-07-08T07:02:54.229305Z","shell.execute_reply":"2021-07-08T07:02:54.306196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"player_target_stats = pd.read_csv(\"../input/player-target-stats/player_target_stats.csv\")\ntargets_agg =targets.groupby('playerId')[['target1', 'target2', 'target3', 'target4']].agg('median').reset_index()\nplayer_target_stats_for_corr = pd.merge(player_target_stats, targets_agg, on = ['playerId'], how = 'left')\nplt.figure(figsize=(20, 20))\nplt.xticks(rotation=25)\ncorr = player_target_stats_for_corr.drop(columns =['playerId']).corr()\n\n\nsns.heatmap(corr,\n            vmin=0,\n            vmax=1,\n            center= 0,\n            cmap=\"RdYlGn\"\n       )\nplt.title('targets corr')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-07T10:42:50.304181Z","iopub.execute_input":"2021-07-07T10:42:50.304784Z","iopub.status.idle":"2021-07-07T10:42:51.879355Z","shell.execute_reply.started":"2021-07-07T10:42:50.304745Z","shell.execute_reply":"2021-07-07T10:42:51.878364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"followers_agg =followers.groupby('playerId')['numberOfFollowers'].agg('median').reset_index()\ntargets_agg =targets.groupby('playerId')[['target1', 'target2', 'target3', 'target4']].agg('median').reset_index()\nfollowers_agg.columns = ['playerId', '#Followers']\nfollowers_agg = pd.merge(followers_agg, targets_agg, on = ['playerId'], how = 'left')\nplt.figure(figsize=(10, 2))\nplt.xticks(rotation=45)\nplt.suptitle(\"Median Target vs Median Twitter Followers\", fontsize =15)\n\ncorr = followers_agg.drop(columns =['playerId']).corr()\n#mask = np.triu(np.ones_like(corr, dtype=bool))\n\nx_axis_labels = ['#Followers', 'target1','target2', 'target3', 'target4'] \nsns.heatmap(np.array(corr['#Followers']).reshape((1,5)),\n            annot = True,\n            xticklabels=x_axis_labels,\n            vmin=0,\n            vmax=1,\n            center= 0,\n            cmap=\"RdYlGn\"\n       )","metadata":{"execution":{"iopub.status.busy":"2021-07-07T10:40:15.464725Z","iopub.execute_input":"2021-07-07T10:40:15.465116Z","iopub.status.idle":"2021-07-07T10:40:16.076734Z","shell.execute_reply.started":"2021-07-07T10:40:15.465078Z","shell.execute_reply":"2021-07-07T10:40:16.075824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tbs = pd.read_csv('../input/mlb-pdef-train-dataset/transactions_train.csv')\ntbs.info()","metadata":{"execution":{"iopub.status.busy":"2021-07-07T10:51:08.702528Z","iopub.execute_input":"2021-07-07T10:51:08.702908Z","iopub.status.idle":"2021-07-07T10:51:08.855388Z","shell.execute_reply.started":"2021-07-07T10:51:08.702872Z","shell.execute_reply":"2021-07-07T10:51:08.854184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tbs_cols = tbs.columns.tolist()\n\ntbs_cols = [col for col in tbs_cols if col not in ['playerId', 'playerName', 'fromTeamName',\n                                                   'toTeamName', 'effectiveDate', 'resolutionDate',\n                                                   'typeCode', 'typeDesc', 'description', 'index']]\nfor col in tbs_cols:\n            tbs_agg = tbs.groupby('playerId')[col].agg('sum').reset_index()\n            tbs_agg.columns = ['playerId', \"#\"+col]\n            targets_agg =targets.groupby('playerId')[['target1', 'target2', 'target3', 'target4']].agg('median').reset_index()\n            tbs_agg = pd.merge(tbs_agg, targets_agg, on = ['playerId'], how = 'left')\n            \n            plt.figure(figsize=(10, 2))\n            plt.suptitle(f\"Total {col} vs Median targets\", fontsize =15)\n            corr = tbs_agg.drop(columns =['playerId']).corr()\n            mask = np.triu(np.ones_like(corr, dtype=bool))\n            sns.set(font_scale=1.4)\n            plt.xticks(rotation=45)\n\n            \n            x_axis_labels = [\"#\"+col, 'target1','target2', 'target3', 'target4'] \n\n            sns.heatmap(np.array(corr[\"#\"+col]).reshape((1,5)),\n            annot = True,\n            xticklabels=x_axis_labels,\n            vmin=0,\n            vmax=1,\n            center= 0,\n            cmap=\"RdYlGn\")           \n            \n            plt.show()\n            plt.close()","metadata":{"execution":{"iopub.status.busy":"2021-07-07T10:53:28.020976Z","iopub.execute_input":"2021-07-07T10:53:28.021334Z","iopub.status.idle":"2021-07-07T10:53:30.650337Z","shell.execute_reply.started":"2021-07-07T10:53:28.021301Z","shell.execute_reply":"2021-07-07T10:53:30.649277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rost_repl = {'A':0, 'D60':1, 'RM':2, 'D10':3, 'D7':4, 'PL':4, 'SU':5, 'FME':6, 'BRV':7, 'RES':8,\n       'DEC':9}\nrosters.statusCode.replace(rost_repl, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-07T12:22:34.98053Z","iopub.execute_input":"2021-07-07T12:22:34.980863Z","iopub.status.idle":"2021-07-07T12:22:35.787105Z","shell.execute_reply.started":"2021-07-07T12:22:34.980835Z","shell.execute_reply":"2021-07-07T12:22:35.785991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rosters.info()","metadata":{"execution":{"iopub.status.busy":"2021-07-07T12:22:40.090578Z","iopub.execute_input":"2021-07-07T12:22:40.090903Z","iopub.status.idle":"2021-07-07T12:22:40.230399Z","shell.execute_reply.started":"2021-07-07T12:22:40.090873Z","shell.execute_reply":"2021-07-07T12:22:40.229554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rosters_cols = rosters.columns.tolist()\n\nrosters_cols = [col for col in rosters if col not in ['playerId', 'gameDate', 'status']]\nfor col in rosters_cols:\n            rosters_agg = rosters.groupby('playerId')[col].agg('sum').reset_index()\n            rosters_agg.columns = ['playerId', \"#\"+col]\n            targets_agg =targets.groupby('playerId')[['target1', 'target2', 'target3', 'target4']].agg('median').reset_index()\n            rosters_agg = pd.merge(rosters_agg, targets_agg, on = ['playerId'], how = 'left')\n            \n            plt.figure(figsize=(10, 2))\n            plt.suptitle(f\"Total {col} vs Median targets\", fontsize =15)\n            corr = rosters_agg.drop(columns =['playerId']).corr()\n            mask = np.triu(np.ones_like(corr, dtype=bool))\n            sns.set(font_scale=1.4)\n            plt.xticks(rotation=45)\n\n            \n            x_axis_labels = [\"#\"+col, 'target1','target2', 'target3', 'target4'] \n\n            sns.heatmap(np.array(corr[\"#\"+col]).reshape((1,5)),\n            annot = True,\n            xticklabels=x_axis_labels,\n            vmin=0,\n            vmax=1,\n            center= 0,\n            cmap=\"RdYlGn\")           \n            \n            plt.show()\n            plt.close()","metadata":{"execution":{"iopub.status.busy":"2021-07-07T12:24:32.886506Z","iopub.execute_input":"2021-07-07T12:24:32.887055Z","iopub.status.idle":"2021-07-07T12:24:35.513432Z","shell.execute_reply.started":"2021-07-07T12:24:32.887007Z","shell.execute_reply":"2021-07-07T12:24:35.512491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"qcut = pd.qcut(targets['target2'], q=10, duplicates='drop', labels=[1,2,3,4,5,6,7, 8, 9, 10])\ntop_deciles_index = qcut[qcut == 10].index\ntarget2_years = targets.iloc[top_deciles_index]","metadata":{"execution":{"iopub.status.busy":"2021-07-07T13:30:45.867947Z","iopub.execute_input":"2021-07-07T13:30:45.868339Z","iopub.status.idle":"2021-07-07T13:30:46.270282Z","shell.execute_reply.started":"2021-07-07T13:30:45.868306Z","shell.execute_reply":"2021-07-07T13:30:46.269098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"qcut","metadata":{"execution":{"iopub.status.busy":"2021-07-07T13:30:50.505496Z","iopub.execute_input":"2021-07-07T13:30:50.505818Z","iopub.status.idle":"2021-07-07T13:30:50.513101Z","shell.execute_reply.started":"2021-07-07T13:30:50.505789Z","shell.execute_reply":"2021-07-07T13:30:50.512353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target2_years['year'] = pd.to_datetime(targets.loc[top_deciles_index].date, format = '%Y%m%d').dt.year\ntarget2_years['month'] = pd.to_datetime(targets.loc[top_deciles_index].date, format = '%Y%m%d').dt.month\ntarget2_years.groupby('year')['month'].value_counts().sort_index()","metadata":{"execution":{"iopub.status.busy":"2021-07-07T14:03:50.419636Z","iopub.execute_input":"2021-07-07T14:03:50.419997Z","iopub.status.idle":"2021-07-07T14:03:50.55859Z","shell.execute_reply.started":"2021-07-07T14:03:50.419961Z","shell.execute_reply":"2021-07-07T14:03:50.557682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events","metadata":{"execution":{"iopub.status.busy":"2021-07-07T13:54:15.091376Z","iopub.execute_input":"2021-07-07T13:54:15.091754Z","iopub.status.idle":"2021-07-07T13:54:15.106844Z","shell.execute_reply.started":"2021-07-07T13:54:15.091724Z","shell.execute_reply":"2021-07-07T13:54:15.105917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets","metadata":{"execution":{"iopub.status.busy":"2021-07-07T13:24:56.68223Z","iopub.execute_input":"2021-07-07T13:24:56.682563Z","iopub.status.idle":"2021-07-07T13:24:56.705337Z","shell.execute_reply.started":"2021-07-07T13:24:56.682533Z","shell.execute_reply":"2021-07-07T13:24:56.704121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"qcut1 = pd.qcut(targets['target1'], q=10, duplicates='drop', labels=[1,2,3,4,5,6,7])\ntop_deciles_index1 = qcut1[qcut1 == 7].index\ntarget1_years = targets.iloc[top_deciles_index1]\ntarget1_years['year'] = pd.to_datetime(targets.iloc[top_deciles_index1].date, format = '%Y%m%d').dt.year\ntarget1_years.year.value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-07-07T13:34:24.10249Z","iopub.execute_input":"2021-07-07T13:34:24.102864Z","iopub.status.idle":"2021-07-07T13:34:24.486212Z","shell.execute_reply.started":"2021-07-07T13:34:24.102833Z","shell.execute_reply":"2021-07-07T13:34:24.485209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"qcut3 = pd.qcut(targets['target3'], q=10, duplicates='drop', labels=[1,2,3,4,5,6,7])\ntop_deciles_index3 = qcut3[qcut3 == 7].index\ntarget3_years = targets.iloc[top_deciles_index3]\ntarget3_years['year'] = pd.to_datetime(targets.iloc[top_deciles_index3].date, format = '%Y%m%d').dt.year\ntarget3_years.year.value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-07-07T13:35:15.942773Z","iopub.execute_input":"2021-07-07T13:35:15.94313Z","iopub.status.idle":"2021-07-07T13:35:16.295312Z","shell.execute_reply.started":"2021-07-07T13:35:15.943097Z","shell.execute_reply":"2021-07-07T13:35:16.294206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"qcut4 = pd.qcut(targets['target4'], q=10, duplicates='drop', labels=[1,2,3,4,5,6,7,8,9])\ntop_deciles_index4 = qcut4[qcut4 == 9].index\ntarget4_years = targets.iloc[top_deciles_index4]\ntarget4_years['year'] = pd.to_datetime(targets.iloc[top_deciles_index4].date, format = '%Y%m%d').dt.year\ntarget4_years.year.value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-07-07T13:36:05.785264Z","iopub.execute_input":"2021-07-07T13:36:05.785603Z","iopub.status.idle":"2021-07-07T13:36:06.223719Z","shell.execute_reply.started":"2021-07-07T13:36:05.785571Z","shell.execute_reply":"2021-07-07T13:36:06.222853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"awards = pd.read_csv(TRAIN_DIR / 'awards_train.csv')\n\nawards_agg = awards.groupby('playerId')['awardId'].agg('count').reset_index()\nawards_agg.columns = ['playerId', '#Awards']\ntargets_agg =targets.groupby('playerId')[['target1', 'target2', 'target3', 'target4']].agg('median').reset_index()\nfollowers_agg = pd.merge(awards_agg, targets_agg, on = ['playerId'], how = 'left')\nplt.figure(figsize=(10, 2))\nplt.xticks(rotation=45)\nplt.suptitle(\"Total number of awards vs Median targets\", fontsize =15)\ncorr = followers_agg.drop(columns =['playerId']).corr()\nmask = np.triu(np.ones_like(corr, dtype=bool))\n\nx_axis_labels = ['#Awards', 'target1','target2', 'target3', 'target4'] \nsns.heatmap(np.array(corr['#Awards']).reshape((1,5)),\n            annot = True,\n            xticklabels=x_axis_labels,\n            vmin=0,\n            vmax=1,\n            center= 0,\n            cmap=\"RdYlGn\"\n       )","metadata":{"execution":{"iopub.status.busy":"2021-07-08T07:02:58.382801Z","iopub.execute_input":"2021-07-08T07:02:58.38329Z","iopub.status.idle":"2021-07-08T07:02:59.055669Z","shell.execute_reply.started":"2021-07-08T07:02:58.383251Z","shell.execute_reply":"2021-07-08T07:02:59.054851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores_cols = scores.columns.tolist()\n\nscores_cols = [col for col in scores_cols if col not in ['playerId', 'date', 'home', 'gamePk', 'teamId', 'battingOrder']]\nfor col in scores_cols:\n            scores_agg = scores.groupby('playerId')[col].agg('sum').reset_index()\n            scores_agg.columns = ['playerId', \"#\"+col]\n            targets_agg =targets.groupby('playerId')[['target1', 'target2', 'target3', 'target4']].agg('median').reset_index()\n            scores_agg = pd.merge(scores_agg, targets_agg, on = ['playerId'], how = 'left')\n            plt.figure(figsize=(10, 2))\n            plt.suptitle(f\"Total {col} vs Median targets\", fontsize =15)\n            corr = scores_agg.drop(columns =['playerId']).corr()\n            mask = np.triu(np.ones_like(corr, dtype=bool))\n            sns.set(font_scale=1.4)\n            plt.xticks(rotation=45)\n            \n            x_axis_labels = [\"#\"+col, 'target1','target2', 'target3', 'target4'] \n\n            sns.heatmap(np.array(corr[\"#\"+col]).reshape((1,5)),\n            annot = True,\n            xticklabels=x_axis_labels,\n            vmin=0,\n            vmax=1,\n            center= 0,\n            cmap=\"RdYlGn\")           \n            \n            plt.show()\n            plt.close()","metadata":{"execution":{"iopub.status.busy":"2021-07-01T09:29:07.767356Z","iopub.execute_input":"2021-07-01T09:29:07.767833Z","iopub.status.idle":"2021-07-01T09:29:57.988308Z","shell.execute_reply.started":"2021-07-01T09:29:07.767784Z","shell.execute_reply":"2021-07-01T09:29:57.987121Z"},"trusted":true},"execution_count":null,"outputs":[]}]}