{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Events vs Target\nIn this notebook, I will use the event level meta-model to analyze how engagement is related to the play-by-play information recorded in events.\n\nThe event-level meta model was not only used for the EDA, but also for the actual features of the [3rd place solution](https://www.kaggle.com/c/mlb-player-digital-engagement-forecasting/discussion/256620), which improved the score by about 0.001.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-19T12:26:24.35948Z","iopub.execute_input":"2021-09-19T12:26:24.359838Z","iopub.status.idle":"2021-09-19T12:26:34.701355Z","shell.execute_reply.started":"2021-09-19T12:26:24.35973Z","shell.execute_reply":"2021-09-19T12:26:34.700453Z"}}},{"cell_type":"code","source":"import gc\nimport json\n\nimport pandas as pd\nimport numpy as np\nimport shap\nimport lightgbm as lgb\nimport matplotlib.pyplot as plt\n\nfrom tqdm import tqdm\n%matplotlib inline","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!pip install nyaggle","metadata":{"execution":{"iopub.status.busy":"2021-09-19T12:26:34.703105Z","iopub.execute_input":"2021-09-19T12:26:34.703368Z","iopub.status.idle":"2021-09-19T12:26:45.404012Z","shell.execute_reply.started":"2021-09-19T12:26:34.703338Z","shell.execute_reply":"2021-09-19T12:26:45.402874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nyaggle.validation import TimeSeriesSplit","metadata":{"execution":{"iopub.status.busy":"2021-09-19T12:26:45.405848Z","iopub.execute_input":"2021-09-19T12:26:45.406198Z","iopub.status.idle":"2021-09-19T12:26:45.91787Z","shell.execute_reply.started":"2021-09-19T12:26:45.406153Z","shell.execute_reply":"2021-09-19T12:26:45.916907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/mlb-player-digital-engagement-forecasting/train.csv', \n                 usecols=['date', 'nextDayPlayerEngagement', 'events'])","metadata":{"execution":{"iopub.status.busy":"2021-09-19T12:26:45.919867Z","iopub.execute_input":"2021-09-19T12:26:45.92013Z","iopub.status.idle":"2021-09-19T12:28:28.124163Z","shell.execute_reply.started":"2021-09-19T12:26:45.920102Z","shell.execute_reply":"2021-09-19T12:28:28.123288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_subdata(df, col):\n    eng = []\n    for i, row in tqdm(df.iterrows()):\n        try:\n            loaded = json.loads(row[col])\n            for l in loaded:\n                l['dailyDataDate'] = row.date\n            eng.extend(loaded)\n        except:\n            pass\n\n    return pd.DataFrame(eng)\n\nengagement = load_subdata(df, 'nextDayPlayerEngagement')\nevents = load_subdata(df, 'events')\n\nfor c in events.columns:\n    if events[c].dtype.name == 'float64':\n        events[c] = events[c].astype(np.float32)\n\ndel df\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-09-19T12:28:28.125234Z","iopub.execute_input":"2021-09-19T12:28:28.125478Z","iopub.status.idle":"2021-09-19T12:31:45.350511Z","shell.execute_reply.started":"2021-09-19T12:28:28.125427Z","shell.execute_reply":"2021-09-19T12:31:45.349555Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prep_events(events: pd.DataFrame, sort_by_date: bool = True):\n    events_p = events.copy()\n    events_p['asPitcher'] = 1\n    events_p['playerId'] = events_p['pitcherId']\n    events_p['teamId'] = events_p['pitcherTeamId']\n\n    events_h = events.copy()\n    events_h['asPitcher'] = 0\n    events_h['playerId'] = events_p['hitterId']\n    events_h['teamId'] = events_p['hitterTeamId']\n\n    if sort_by_date:\n        events_stacked = pd.concat([events_p, events_h]).sort_values(\n            by=['dailyDataDate', 'gamePk', 'inning']).reset_index(\n            drop=True)\n        events_stacked['dailyDataDate'] = pd.to_datetime(events_stacked['dailyDataDate'], format='%Y%m%d')\n    else:\n        events_stacked = pd.concat([events_p, events_h]).reset_index(drop=True)\n\n    drop_cols = [\n        'gameDate', 'season', 'playId', 'pitcherTeam', 'hitterTeam', 'pitcherName', 'pitcherHand', 'hitterName',\n        'batSide', 'atBatDesc', 'gameTimeUTC',\n        'description', 'halfInning', 'hitterTeamId', 'pitcherTeamId', 'gamePk', 'pitcherId', 'hitterId'\n    ]\n\n    events_stacked.drop(drop_cols, axis=1, inplace=True)\n\n    events_stacked['scoreDiff'] = events_stacked['homeScore'] - events_stacked['awayScore']\n\n    cats = {\n        'menOnBase': {\n            'Empty': 1,\n            'Men_On': 2,\n            'RISP': 3,\n            'Loaded': 4\n        },\n        'gameType': {\n            'R': 1,\n            'D': 2,\n            'L': 3,\n            'F': 4,\n            'W': 5\n        },\n        'atBatEvent': {\n            'Strikeout': 1,\n            'Groundout': 2,\n            'Single': 3,\n            'Walk': 4,\n            'Flyout': 5,\n            'Lineout': 6,\n            'Pop Out': 7,\n            'Double': 8,\n            'Home Run': 9,\n            'Forceout': 10,\n            'Grounded Into DP': 11,\n            'Hit By Pitch': 12,\n            'Field Error': 13,\n            'Sac Fly': 14,\n            'Intent Walk': 15,\n            'Triple': 16,\n            'Double Play': 17,\n            'Sac Bunt': 18,\n            'Fielders Choice Out': 19,\n            'Fielders Choice': 20,\n            'Strikeout Double Play': 21,\n            'Caught Stealing 2B': 22,\n            'Bunt Groundout': 23,\n            'Catcher Interference': 24,\n            'Bunt Pop Out': 25,\n            'Batter Interference': 26,\n            'Runner Out': 27,\n            'Pickoff Caught Stealing 2B': 28,\n            'Fan Interference': 29,\n            'Pickoff 1B': 30,\n            'Caught Stealing 3B': 31,\n            'Caught Stealing Home': 32,\n            'Pickoff 2B': 33,\n            'Sac Fly Double Play': 34,\n            'Bunt Lineout': 35,\n            'Wild Pitch': 36,\n            'Pickoff Caught Stealing Home': 37,\n            'Triple Play': 38,\n            'Pickoff Caught Stealing 3B': 39,\n            'Pickoff 3B': 40,\n            'Game Advisory': 41,\n            'Stolen Base 2B': 42,\n            'Sac Bunt Double Play': 43,\n            'Runner Double Play': 44,\n            'Passed Ball': 45,\n            'Pickoff Error 1B': 46,\n            'Balk': 47\n        },\n        'event': {\n            'Strikeout': 1,\n            'Groundout': 2,\n            'Single': 3,\n            'Game Advisory': 4,\n            'Flyout': 5,\n            'Pitching Substitution': 6,\n            'Walk': 7,\n            'Lineout': 8,\n            'Pop Out': 9,\n            'Double': 10,\n            'Home Run': 11,\n            'Offensive Substitution': 12,\n            'Defensive Switch': 13,\n            'Forceout': 14,\n            'Grounded Into DP': 15,\n            'Defensive Sub': 16,\n            'Hit By Pitch': 17,\n            'Stolen Base 2B': 18,\n            'Wild Pitch': 19,\n            'Field Error': 20,\n            'Sac Fly': 21,\n            'Intent Walk': 22,\n            'Triple': 23,\n            'Sac Bunt': 24,\n            'Caught Stealing 2B': 25,\n            'Double Play': 26,\n            'Passed Ball': 27,\n            'Injury': 28,\n            'Fielders Choice Out': 29,\n            'Fielders Choice': 30,\n            'Stolen Base 3B': 31,\n            'Defensive Indiff': 32,\n            'Bunt Groundout': 33,\n            'Ejection': 34,\n            'Balk': 35,\n            'Strikeout Double Play': 36,\n            'Runner Placed On Base': 37,\n            'Pickoff Error 1B': 38,\n            'Bunt Pop Out': 39,\n            'Pitch Challenge': 40,\n            'Runner Out': 41,\n            'Pickoff 1B': 42,\n            'Pickoff Caught Stealing 2B': 43,\n            'Caught Stealing 3B': 44,\n            'Catcher Interference': 45,\n            'Error': 46,\n            'Umpire Substitution': 47,\n            'Batter Interference': 48,\n            'Pickoff Error 2B': 49,\n            'Pitcher Switch': 50,\n            'Fan Interference': 51,\n            'Pickoff 2B': 52,\n            'Caught Stealing Home': 53,\n            'Stolen Base Home': 54,\n            'Sac Fly Double Play': 55,\n            'Bunt Lineout': 56,\n            'Pickoff Caught Stealing 3B': 57,\n            'Pickoff 3B': 58,\n            'Pickoff Caught Stealing Home': 59,\n            'Other Advance': 60,\n            'Pickoff Error 3B': 61,\n            'Triple Play': 62,\n            'Sac Bunt Double Play': 63},\n        'pitchType': {'FF': 1,\n                      'SL': 2,\n                      'CH': 3,\n                      'SI': 4,\n                      'CU': 5,\n                      'FT': 6,\n                      'FC': 7,\n                      'KC': 8,\n                      'FS': 9,\n                      'KN': 10,\n                      'EP': 11,\n                      'CS': 12,\n                      'FO': 13,\n                      'PO': 14,\n                      'SC': 15,\n                      'FA': 16,\n                      'AB': 17},\n        'call': {'B': 1,\n                 'F': 2,\n                 'C': 3,\n                 'X': 4,\n                 'S': 5,\n                 'D': 6,\n                 '*B': 7,\n                 'E': 8,\n                 'T': 9,\n                 'W': 10,\n                 'V': 11,\n                 'H': 12,\n                 'L': 13,\n                 'M': 14,\n                 'P': 15,\n                 'O': 16,\n                 'Q': 17,\n                 'R': 18},\n        'type': {\n            'pitch': 1,\n            'action': 2\n        }\n    }\n\n    for k in cats:\n        events_stacked[k] = events_stacked[k].map(cats[k])\n\n    return events_stacked\n","metadata":{"execution":{"iopub.status.busy":"2021-09-19T12:31:45.352291Z","iopub.execute_input":"2021-09-19T12:31:45.352584Z","iopub.status.idle":"2021-09-19T12:31:45.381541Z","shell.execute_reply.started":"2021-09-19T12:31:45.352549Z","shell.execute_reply":"2021-09-19T12:31:45.380841Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"engagement['dailyDataDate'] = pd.to_datetime(engagement['dailyDataDate'], format='%Y%m%d')\n\nevent_df = prep_events(events)\n\nevent_df = pd.merge(event_df, engagement, how='left', on=['dailyDataDate', 'playerId'])\nevent_df = event_df[~event_df['target1'].isnull()]\n\ndel engagement\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-09-19T12:31:45.470386Z","iopub.execute_input":"2021-09-19T12:31:45.470675Z","iopub.status.idle":"2021-09-19T12:32:25.078916Z","shell.execute_reply.started":"2021-09-19T12:31:45.47064Z","shell.execute_reply":"2021-09-19T12:32:25.078255Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hitter Events vs Target","metadata":{}},{"cell_type":"code","source":"params = {\n    'objective': 'mae',\n    'metrics': 'mae',\n    'num_leaves': 256,\n    'max_depth': 16,\n    'n_estimators': 100\n}\n\nhitter_df = event_df[event_df.asPitcher==0]\n\nX = hitter_df.drop(['target1', 'target2', 'target3', 'target4', 'dailyDataDate', 'playerId', 'teamId', 'engagementMetricsDate'], axis=1).astype(np.float32)\ny = hitter_df['target1']\n\nmodel = lgb.LGBMRegressor(**params)\n\nmodel.fit(X, y, categorical_feature=['atBatEvent', 'event', 'menOnBase', 'gameType', 'pitchType', 'call'])\n\ndel hitter_df","metadata":{"execution":{"iopub.status.busy":"2021-09-19T12:44:17.463625Z","iopub.execute_input":"2021-09-19T12:44:17.46433Z","iopub.status.idle":"2021-09-19T12:44:57.68019Z","shell.execute_reply.started":"2021-09-19T12:44:17.464287Z","shell.execute_reply":"2021-09-19T12:44:57.67956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"explainer = shap.Explainer(model)\nshap_values_hitter = explainer(X.sample(50000))","metadata":{"execution":{"iopub.status.busy":"2021-09-19T12:45:32.167156Z","iopub.execute_input":"2021-09-19T12:45:32.167532Z","iopub.status.idle":"2021-09-19T12:54:49.05912Z","shell.execute_reply.started":"2021-09-19T12:45:32.167491Z","shell.execute_reply":"2021-09-19T12:54:49.058369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shap.plots.beeswarm(shap_values_hitter)","metadata":{"execution":{"iopub.status.busy":"2021-09-19T12:54:49.060962Z","iopub.execute_input":"2021-09-19T12:54:49.061914Z","iopub.status.idle":"2021-09-19T12:54:54.361437Z","shell.execute_reply.started":"2021-09-19T12:54:49.061868Z","shell.execute_reply":"2021-09-19T12:54:54.360475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shap.plots.scatter(shap_values_hitter[:,\"inning\"])","metadata":{"execution":{"iopub.status.busy":"2021-09-19T12:57:04.335117Z","iopub.execute_input":"2021-09-19T12:57:04.335482Z","iopub.status.idle":"2021-09-19T12:57:07.399724Z","shell.execute_reply.started":"2021-09-19T12:57:04.335435Z","shell.execute_reply":"2021-09-19T12:57:07.399092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Hitters who appear in extra innings tend to get higher engagement.","metadata":{}},{"cell_type":"code","source":"shap.plots.scatter(shap_values_hitter[:,\"scoreDiff\"])","metadata":{"execution":{"iopub.status.busy":"2021-09-19T12:57:42.519097Z","iopub.execute_input":"2021-09-19T12:57:42.519417Z","iopub.status.idle":"2021-09-19T12:57:45.752008Z","shell.execute_reply.started":"2021-09-19T12:57:42.519386Z","shell.execute_reply":"2021-09-19T12:57:45.751245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shap.plots.scatter(shap_values_hitter[:,\"rbi\"])","metadata":{"execution":{"iopub.status.busy":"2021-09-19T12:57:57.198097Z","iopub.execute_input":"2021-09-19T12:57:57.198572Z","iopub.status.idle":"2021-09-19T12:58:00.511612Z","shell.execute_reply.started":"2021-09-19T12:57:57.19854Z","shell.execute_reply":"2021-09-19T12:58:00.510643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shap.plots.scatter(shap_values_hitter[:,\"gameType\"], show=False)\nplt.xlim(0, 6)\nplt.xticks(np.arange(0, 6), ['', 'Regular Season', 'Division Series', 'League Championship Series', 'Wild Card Game', 'World Series'], rotation='vertical')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-19T13:20:09.224293Z","iopub.execute_input":"2021-09-19T13:20:09.225092Z","iopub.status.idle":"2021-09-19T13:20:09.787813Z","shell.execute_reply.started":"2021-09-19T13:20:09.225044Z","shell.execute_reply":"2021-09-19T13:20:09.786839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bat_events = ['','Strikeout',\n            'Groundout',\n            'Single',\n            'Walk',\n            'Flyout',\n            'Lineout',\n            'Pop Out',\n            'Double',\n            'Home Run',\n            'Forceout',\n            'Grounded Into DP',\n            'Hit By Pitch',\n            'Field Error',\n            'Sac Fly',\n            'Intent Walk',\n            'Triple',\n            'Double Play',\n            'Sac Bunt',\n            'Fielders Choice Out',\n             'Strikeout Double Play']\n\nshap.plots.scatter(shap_values_hitter[:,\"atBatEvent\"], color=shap_values_hitter, show=False)\nplt.xlim(0, 20)\nplt.xticks(np.arange(21), bat_events, rotation='vertical')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-19T13:14:40.943536Z","iopub.execute_input":"2021-09-19T13:14:40.943868Z","iopub.status.idle":"2021-09-19T13:14:44.499744Z","shell.execute_reply.started":"2021-09-19T13:14:40.943835Z","shell.execute_reply":"2021-09-19T13:14:44.498542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Obviously, home runs are related to hitters' engagement. Interestingly, triples also seem to boost engagement.","metadata":{}},{"cell_type":"markdown","source":"## Pitcher Events vs Target","metadata":{}},{"cell_type":"code","source":"pitcher_df = event_df[event_df.asPitcher==1]\n\nX = pitcher_df.drop(['target1', 'target2', 'target3', 'target4', 'dailyDataDate', 'playerId', 'teamId', 'engagementMetricsDate'], axis=1).astype(np.float32)\ny = pitcher_df['target1']\n\nmodel = lgb.LGBMRegressor(**params)\n\nmodel.fit(X, y, categorical_feature=['atBatEvent', 'event', 'menOnBase', 'gameType', 'pitchType', 'call'])\n\ndel pitcher_df","metadata":{"execution":{"iopub.status.busy":"2021-09-19T13:20:33.255813Z","iopub.execute_input":"2021-09-19T13:20:33.256095Z","iopub.status.idle":"2021-09-19T13:21:21.71321Z","shell.execute_reply.started":"2021-09-19T13:20:33.256068Z","shell.execute_reply":"2021-09-19T13:21:21.712237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"explainer = shap.Explainer(model)\nshap_values_pitcher = explainer(X.sample(50000))","metadata":{"execution":{"iopub.status.busy":"2021-09-19T13:21:21.715335Z","iopub.execute_input":"2021-09-19T13:21:21.715676Z","iopub.status.idle":"2021-09-19T13:29:59.699694Z","shell.execute_reply.started":"2021-09-19T13:21:21.715632Z","shell.execute_reply":"2021-09-19T13:29:59.6987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shap.plots.beeswarm(shap_values_pitcher)","metadata":{"execution":{"iopub.status.busy":"2021-09-19T13:29:59.701103Z","iopub.execute_input":"2021-09-19T13:29:59.701651Z","iopub.status.idle":"2021-09-19T13:30:04.942218Z","shell.execute_reply.started":"2021-09-19T13:29:59.701604Z","shell.execute_reply":"2021-09-19T13:30:04.941196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shap.plots.scatter(shap_values_pitcher[:,\"nastyFactor\"])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The relationship between nastyfactor and engagement seems to be weak.","metadata":{}},{"cell_type":"code","source":"shap.plots.scatter(shap_values_pitcher[:,\"scoreDiff\"])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The relationship between score difference and target shows that hitters who appear when the score difference is zero tend to get higher engagement, while the opposite is true for pitchers.","metadata":{}},{"cell_type":"code","source":"shap.plots.scatter(shap_values_pitcher[:,\"gameType\"], show=False)\nplt.xlim(0, 6)\nplt.xticks(np.arange(0, 6), ['', 'Regular Season', 'Division Series', 'League Championship Series', 'Wild Card Game', 'World Series'], rotation='vertical')\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bat_events = ['','Strikeout',\n            'Groundout',\n            'Single',\n            'Walk',\n            'Flyout',\n            'Lineout',\n            'Pop Out',\n            'Double',\n            'Home Run',\n            'Forceout',\n            'Grounded Into DP',\n            'Hit By Pitch',\n            'Field Error',\n            'Sac Fly',\n            'Intent Walk',\n            'Triple',\n            'Double Play',\n            'Sac Bunt',\n            'Fielders Choice Out',\n             'Strikeout Double Play']\n\nshap.plots.scatter(shap_values_pitcher[:,\"atBatEvent\"], color=shap_values_pitcher, show=False)\nplt.xlim(0, 20)\nplt.xticks(np.arange(21), bat_events, rotation='vertical')\nplt.show()","metadata":{"jupyter":{"source_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pitch_type = ['', 'FF', 'SL', 'CH', 'SI', 'CU', 'FT', 'FC', 'KC', 'FS', 'KN', 'EP', 'CS', 'FO', 'PO', 'SC', 'FA', 'AB']\n\nshap.plots.scatter(shap_values_pitcher[:,\"pitchType\"], color=shap_values_pitcher, show=False)\nplt.xlim(0, 19)\nplt.xticks(np.arange(18), pitch_type, rotation='vertical')\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shap.plots.scatter(shap_values_pitcher[:,\"inning\"])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shap.plots.scatter(shap_values_pitcher[:,\"x0\"])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shap.plots.scatter(shap_values_pitcher[:,\"zone\"])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shap.plots.scatter(shap_values_pitcher[:,\"nastyFactor\"])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shap.plots.scatter(shap_values_pitcher[:,\"startSpeed\"])","metadata":{},"execution_count":null,"outputs":[]}]}