{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook focuses on extracting features from event data for each player according to gamedate as per 'PITCH' or 'HIT' event. \n\nMy other notebook contributions:\n\nplease upvote if you like it!\n\n* https://www.kaggle.com/debojit23/mlb-sabermetrics-batting-and-pitching-stats\n* https://www.kaggle.com/debojit23/baseball-field-structure-matplotlib","metadata":{}},{"cell_type":"code","source":"import numpy as np\n\n#basic declaration\nnull=np.nan;true=True;false=False\nball_weight=dict();ball_rad=dict()\nair_density=dict()\nball_weight['lb']=0.32;ball_weight['oz']=5.125;\nball_rad['ft']=0.12;ball_rad['inch']=1.45;\nair_density['mass']=0.0023\nair_density['weight']=0.075\npi=3.14;drag_coeff=0.3\ngravity=32.1740 \n\n#https://www.grc.nasa.gov/www/k-12/airplane/balldrag.html\n\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df\n","metadata":{"execution":{"iopub.status.busy":"2021-07-29T13:25:48.751778Z","iopub.execute_input":"2021-07-29T13:25:48.752225Z","iopub.status.idle":"2021-07-29T13:25:48.769397Z","shell.execute_reply.started":"2021-07-29T13:25:48.752188Z","shell.execute_reply":"2021-07-29T13:25:48.768107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings \nwarnings.filterwarnings('ignore')\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom sklearn.metrics import mean_absolute_error\nfrom datetime import timedelta\nfrom functools import reduce\nfrom tqdm import tqdm\nimport lightgbm as lgbm\nimport mlb\nimport gc\nimport tensorflow as tf\nfrom tensorflow.keras.callbacks import Callback, ReduceLROnPlateau, ModelCheckpoint, EarlyStopping\npd.set_option('display.max_columns', 500)","metadata":{"execution":{"iopub.status.busy":"2021-07-29T13:24:10.617184Z","iopub.execute_input":"2021-07-29T13:24:10.617603Z","iopub.status.idle":"2021-07-29T13:24:18.445440Z","shell.execute_reply.started":"2021-07-29T13:24:10.617566Z","shell.execute_reply":"2021-07-29T13:24:18.443723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"players=pd.read_pickle('../input/mlb-pdef-train-dataset/playerTwitterFollowers_train.pkl')\nzz=players[['playerId','playerName']].to_dict(orient='l')\nplayer_map=dict(zip(zz['playerId'],zz['playerName']))\nplayers.head(2)\n","metadata":{"execution":{"iopub.status.busy":"2021-07-29T13:24:18.448018Z","iopub.execute_input":"2021-07-29T13:24:18.448562Z","iopub.status.idle":"2021-07-29T13:24:18.592822Z","shell.execute_reply.started":"2021-07-29T13:24:18.448487Z","shell.execute_reply":"2021-07-29T13:24:18.591934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=pd.read_csv('../input/mlb-player-digital-engagement-forecasting/train.csv')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-29T13:24:18.594466Z","iopub.execute_input":"2021-07-29T13:24:18.595064Z","iopub.status.idle":"2021-07-29T13:25:48.749723Z","shell.execute_reply.started":"2021-07-29T13:24:18.595024Z","shell.execute_reply":"2021-07-29T13:25:48.748771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pitching events\nsum_cols_pitch=['is_bad_pitch','is_pitchplayed_beforehomeplate']\nmean_std_cols_pitch=['pfxX', 'pfxZ','net_pitch_acceleration','net_pitchlaunch_velocity', 'approxhorz_pitch_distance',\n               'is_pitchplayed_beforehomeplate', 'drag_force', 'magnus_force','startSpeed', 'endSpeed','nastyFactor',\\\n               'breakAngle','spinRate', 'spinDirection', 'breakY',]\n\n\ndef get_eventpitching(temp_event) -> pd.DataFrame:\n\n    all_events_pitch=pd.DataFrame()\n    \n    temp_pitcher=pd.DataFrame(columns=['playerId','date'])\n    temp_pitcher['playerId']=temp_event['pitcherId'].unique()\n    temp_pitcher['date']=temp_event['gameDate'].iloc[0]\n\n    pitchevents=temp_event[temp_event.aX.notnull()]\n    pitchevents=pitchevents.fillna(0)\n    pitchevents=get_pitchkinematics(pitchevents)\n\n    zz=get_meanstd_stats(pitchevents,mean_std_cols=mean_std_cols_pitch)\n    temp_pitcher=temp_pitcher.merge(zz,left_on='playerId',right_on='pitcherId',how='left')\n    temp_pitcher.drop(columns=['pitcherId'],inplace=True)\n    zz=get_sum_stats(pitchevents)\n    temp_pitcher=temp_pitcher.merge(zz,left_on='playerId',right_on='pitcherId',how='left')\n    temp_pitcher.drop(columns=['pitcherId'],inplace=True)\n    all_events_pitch=all_events_pitch.append(temp_pitcher)\n    \n    return all_events_pitch\n\ndef get_singleevent(event) -> pd.DataFrame:\n    return pd.DataFrame(eval(event))\n\ndef get_meanstd_stats(pitchevents,mean_std_cols) -> pd.DataFrame:\n    all_stats=pd.DataFrame(columns=['pitcherId'])\n    all_stats['pitcherId']=pitchevents.pitcherId.unique()\n    for col in mean_std_cols:\n        temp = pitchevents.groupby('pitcherId')[col].agg({'mean','std','min','max'}).reset_index().fillna(0)\n        temp.rename(columns={'mean':f'{col}_mean','std':f'{col}_std','min':f'{col}_min','max':f'{col}_max'},inplace=True)\n        all_stats=all_stats.merge(temp,on='pitcherId')\n    return all_stats\ndef get_sum_stats(pitchevents) -> pd.DataFrame:\n    all_stats=pd.DataFrame(columns=['pitcherId'])\n    all_stats['pitcherId']=pitchevents.pitcherId.unique()\n    for col in sum_cols_pitch:\n        temp = pitchevents.groupby('pitcherId')[col].agg({'sum'}).reset_index().fillna(0)\n        temp.rename(columns={'sum':f'{col}_sum'},inplace=True)\n        all_stats=all_stats.merge(temp,on='pitcherId')\n    return all_stats\n\ndef get_pitchkinematics(pitchevents) -> pd.DataFrame:\n    pitchevents['net_pitch_acceleration']=np.sqrt((pitchevents.aX.values)**2+(pitchevents.aY.values)**2+(pitchevents.aZ.values)**2)\n    pitchevents['net_pitchlaunch_velocity']=np.sqrt((pitchevents.vX0.values)**2+(pitchevents.vY0.values)**2+(pitchevents.vZ0.values)**2)\n    pitchevents['approxhorz_pitch_distance']=np.sqrt((pitchevents.x.values-pitchevents.x0.values)**2+(pitchevents.y.values-pitchevents.y0.values)**2)\n    pitchevents['is_bad_pitch']=[1 if val<0 else 0 for val in pitchevents.pZ.values]\n    pitchevents['is_pitchplayed_beforehomeplate']=[1 if val<0 else 0 for val in pitchevents.pX.values]\n    pitchevents['drag_force']=0.5*drag_coeff*pi*air_density['mass']*(ball_rad['ft']**2)*(pitchevents['net_pitchlaunch_velocity'].values**2)\n    pitchevents['magnus_force']=0.5*pi*air_density['mass']*(ball_rad['ft']**3)*(pitchevents['spinRate'].values)*(pitchevents['net_pitchlaunch_velocity'].values)\n    return pitchevents\nfor i in tqdm(range(len(train))):\n    if str(train.iloc[i]['events'])!='nan':\n        temp_event = get_singleevent(train.iloc[i]['events'])\n        all_events_pitch = get_eventpitching(temp_event)\n        \n    else:\n        continue\n        \nall_events_pitch.sample(5)","metadata":{"execution":{"iopub.status.busy":"2021-07-29T13:26:42.067820Z","iopub.execute_input":"2021-07-29T13:26:42.068268Z","iopub.status.idle":"2021-07-29T13:26:44.116898Z","shell.execute_reply.started":"2021-07-29T13:26:42.068233Z","shell.execute_reply":"2021-07-29T13:26:44.115147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#*0.0174533->degree convert to rad\n# *1.467 -> MPH to ft/s\n#g=32.1740 ft/s2\n\nmean_std_cols_hit=['launchSpeed','launchAngle','max_heightreached','flighttime']\n\ndef get_eventhit(temp_event) -> pd.DataFrame:\n    \n    all_events_hit=pd.DataFrame()\n\n    temp_pitcher=pd.DataFrame(columns=['playerId','date'])\n    temp_pitcher['playerId']=temp_event['pitcherId'].unique()\n    temp_pitcher['date']=temp_event['gameDate'].iloc[0]\n\n    hitevents=temp_event[temp_event.launchSpeed.notnull()].fillna(0)\n    hitevents=get_hitkinematics(hitevents)\n\n    zz=get_meanstd_stats(hitevents,mean_std_cols=mean_std_cols_hit)\n    temp_pitcher=temp_pitcher.merge(zz,left_on='playerId',right_on='hitterId',how='left')\n    temp_pitcher.drop(columns=['hitterId'],inplace=True)\n    temp_pitcher=temp_pitcher.fillna(0)\n    all_events_hit=all_events_hit.append(temp_pitcher)\n\n    return all_events_hit\n\ndef get_meanstd_stats(hitevents,mean_std_cols) -> pd.DataFrame:\n    all_stats=pd.DataFrame(columns=['hitterId'])\n    all_stats['hitterId']=hitevents.hitterId.unique()\n    for col in mean_std_cols:\n        temp = hitevents.groupby('hitterId')[col].agg({'mean','std','min','max'}).reset_index().fillna(0)\n        temp.rename(columns={'mean':f'{col}_mean','std':f'{col}_std','min':f'{col}_min','max':f'{col}_max'},inplace=True)\n        all_stats=all_stats.merge(temp,on='hitterId')\n    return all_stats\n\ndef get_hitkinematics(hitevents) -> pd.DataFrame:\n    hitevents['max_heightreached']=((np.sin(hitevents['launchAngle'].values*0.0174533)*(hitevents['launchSpeed'].values*1.467))**2)/(2*gravity)\n    hitevents['flighttime']=2*(np.sin(hitevents['launchAngle'].values*0.0174533)*(hitevents['launchSpeed'].values*1.467))/gravity\n    return hitevents\n\n\nfor i in tqdm(range(len(train))):\n    if str(train.iloc[i]['events'])!='nan':\n        temp_event = get_singleevent(train.iloc[i]['events'])\n        all_events_hit=get_eventhit(temp_event)\n        \n    else:\n        continue\n\n        \n        \nall_events_hit.sample(5)","metadata":{"execution":{"iopub.status.busy":"2021-07-29T13:28:47.040729Z","iopub.execute_input":"2021-07-29T13:28:47.041262Z","iopub.status.idle":"2021-07-29T13:28:48.736262Z","shell.execute_reply.started":"2021-07-29T13:28:47.041221Z","shell.execute_reply":"2021-07-29T13:28:48.735281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_eventsum(temp_event) -> pd.DataFrame:\n\n    all_events_sum=pd.DataFrame()\n    temp_pitcher=pd.DataFrame(columns=['playerId','date'])\n    temp_pitcher['playerId']=temp_event['pitcherId'].unique()\n    temp_pitcher['date']=temp_event['gameDate'].iloc[0]      \n\n    zz=get_sum_stats(temp_event)\n\n    temp_pitcher=temp_pitcher.merge(zz,left_on='playerId',right_on='hitterId',how='left')\n    temp_pitcher.drop(columns=['hitterId'],inplace=True)\n    temp_pitcher=temp_pitcher.fillna(0)\n    all_events_sum=all_events_sum.append(temp_pitcher)\n    return all_events_sum\n\nsum_cols=['isGB', 'isLD', 'isFB', 'isPU','isPaOver']\n\ndef get_sum_stats(temp_event) -> pd.DataFrame:\n    all_stats=pd.DataFrame(columns=['hitterId'])\n    all_stats['hitterId']=temp_event.hitterId.unique()\n    for col in sum_cols:\n        temp = temp_event.groupby('hitterId')[col].agg({'sum'}).reset_index().fillna(0)\n        temp.rename(columns={'sum':f'{col}_sum'},inplace=True)\n        all_stats=all_stats.merge(temp,on='hitterId')\n    return all_stats\n\nfor i in tqdm(range(len(train))):\n    if str(train.iloc[i]['events'])!='nan':\n        temp_event = get_singleevent(train.iloc[i]['events'])\n        temp_event=temp_event.fillna(0)\n        all_events_sum=get_eventsum(temp_event)\n        \n        \nall_events_sum.sample(5)","metadata":{"execution":{"iopub.status.busy":"2021-07-29T13:31:01.768723Z","iopub.execute_input":"2021-07-29T13:31:01.769269Z","iopub.status.idle":"2021-07-29T13:31:03.619416Z","shell.execute_reply.started":"2021-07-29T13:31:01.769237Z","shell.execute_reply":"2021-07-29T13:31:03.618494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pivot_cols_pitch=['breakLength','pitchNumber','menOnBase','call']\ndef get_eventcrosstab(temp_event) -> pd.DataFrame:\n\n    all_events_crosstab=pd.DataFrame()\n\n    temp_pitcher=pd.DataFrame(columns=['playerId','date'])\n    temp_pitcher['playerId']=temp_event['pitcherId'].unique()\n    temp_pitcher['date']=temp_event['gameDate'].iloc[0]      \n\n    zz=get_crosstab_stats(temp_event)\n\n    temp_pitcher=temp_pitcher.merge(zz,left_on='playerId',right_on='pitcherId',how='left')\n    temp_pitcher.drop(columns=['pitcherId'],inplace=True)\n    temp_pitcher=temp_pitcher.fillna(0)\n    all_events_crosstab=all_events_crosstab.append(temp_pitcher)\n    return all_events_crosstab\ndef get_crosstab_stats(temp_event) -> pd.DataFrame:\n    all_stats=pd.DataFrame(columns=['pitcherId'])\n    all_stats['pitcherId']=temp_event.pitcherId.unique()\n    for col in pivot_cols_pitch:\n        zz=pd.crosstab(temp_event['pitcherId'],temp_event[col])\n        zz.columns=[f'{col}_'+str(val) for val in zz.columns]\n        all_stats=all_stats.merge(zz,on='pitcherId')\n    return all_stats\nfor i in tqdm(range(len(train))):\n    if str(train.iloc[i]['events'])!='nan':\n        temp_event = get_singleevent(train.iloc[i]['events'])\n        temp_event=temp_event.fillna(0)\n        all_events_crosstab=get_eventcrosstab(temp_event)\n        \nall_events_crosstab.sample(5)","metadata":{"execution":{"iopub.status.busy":"2021-07-29T13:35:23.808523Z","iopub.execute_input":"2021-07-29T13:35:23.809153Z","iopub.status.idle":"2021-07-29T13:35:27.948511Z","shell.execute_reply.started":"2021-07-29T13:35:23.809105Z","shell.execute_reply":"2021-07-29T13:35:27.947511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"standings=pd.read_pickle('../input/mlb-pdef-train-dataset/standings_train.pkl')\n\nfor cols in ['divisionChamp','divisionLeader','wildCardLeader']:\n    standings[cols]=standings[cols].fillna(False)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_teamids=standings.teamId.unique()\nzz=standings[standings.teamId==all_teamids[10]]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crosstab_cols=['streakCode','divisionRank', 'leagueRank', 'wildCardRank','divisionChamp', 'divisionLeader','wildCardLeader']\nnormal_cols=[ 'leagueGamesBack','sportGamesBack', 'divisionGamesBack', 'wins', 'losses', 'pct',\\\n               'runsAllowed', 'runsScored']+list(standings.columns[20:-2])\ndef get_single_team(zz,crosstab_cols) -> pd.DataFrame:\n    cols2_drop=[f'wildCardRank_{val}' for val in np.arange(6,12)]+[f'leagueRank_{val}' for val in np.arange(6,16)]    \n    temp_feat=pd.DataFrame()\n    temp_feat['teamId']=zz['teamId']\n    temp_feat['gameDate']=zz['gameDate']\n    for cols in crosstab_cols:\n        temp=pd.crosstab(zz.gameDate,zz[cols])\n        temp.columns=[f'{cols}_{val}' for val in temp.columns]   \n        try:\n\n            if cols == 'wildCardRank':\n                temp=temp[[f'wildCardRank_{val}' for val in np.arange(1,6)]]\n            if cols == 'leagueRank':\n                temp=temp[[f'leagueRank_{val}' for val in np.arange(1,6)]]\n        except:\n            continue\n        temp_feat=temp_feat.merge(temp,on='gameDate',how='left')\n    return temp_feat","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zz=standings[standings.teamId==all_teamids[0]].reset_index(drop=True)\ntemp_feat=get_single_team(zz,crosstab_cols)\nfeat_cols=temp_feat.columns\n\ndef get_standing_feats(standings,crosstab_cols,normal_cols) -> pd.DataFrame:\n    standing_data=pd.DataFrame(columns=feat_cols)\n    \n    all_teamids=standings.teamId.unique()\n    for ids in tqdm(all_teamids):\n        zz=standings[standings.teamId==ids].reset_index(drop=True)\n        temp_feat=get_single_team(zz,crosstab_cols)\n        #print(temp_feat.shape,zz.shape)\n        for cols in normal_cols:\n            temp_feat[cols]=zz[cols]\n        standing_data=standing_data.append(temp_feat)\n        \n    return standing_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nstanding_data=get_standing_feats(standings,crosstab_cols=crosstab_cols,normal_cols=normal_cols)\nstanding_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"standing_data=standing_data.fillna(0)\nstanding_data.to_csv('standing_data.csv',index=None)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"standing_data.isna().sum()[:49]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}