{"cells":[{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"collapsed":true,"trusted":false},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom datetime import datetime\nimport datetime\nfrom sklearn.preprocessing import StandardScaler\nfrom datetime import timedelta\nimport re\nfrom scipy import sparse\nfrom sklearn.linear_model import Ridge\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import GridSearchCV\nimport math\n\nimport pandas as pd\nimport numpy as np\nfrom sklearn import linear_model\nfrom sklearn import model_selection\nfrom sklearn import metrics\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import classification_report\nimport matplotlib.pyplot as plt\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import learning_curve\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.linear_model import Ridge\nfrom sklearn import preprocessing\nfrom datetime import datetime, timedelta\nimport datetime\nfrom xgboost.sklearn import XGBRegressor\nfrom xgboost.sklearn import XGBClassifier\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import cross_val_predict\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import r2_score\nimport seaborn as sns\n\nimport statsmodels.api as sm\n\nfrom sklearn.linear_model import Ridge\nfrom sklearn.linear_model import RidgeCV\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.linear_model import LogisticRegression\nfrom scipy.stats.mstats import winsorize\n\nfrom sklearn.cluster import MiniBatchKMeans \nfrom scipy.spatial.distance import cdist\nimport matplotlib.style as style\n\nplay_list = pd.read_csv('../input/nfl-playing-surface-analytics/PlayList.csv', delimiter=',')\n# play_list.info()\n# play_list.head().T\n\ninjuries = pd.read_csv('../input/nfl-playing-surface-analytics/InjuryRecord.csv', delimiter=',')\n# injuries.info()\n# injuries.head()\n\n# Pick first body part for duplicate injuries\nplay_list['BodyPart'] = play_list[['PlayKey']].merge(injuries.drop_duplicates(subset=['PlayKey']), how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['BodyPart'].values\nplay_list['Surface'] = play_list[['PlayKey']].merge(injuries.drop_duplicates(subset=['PlayKey']), how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['Surface'].values\nplay_list['has_injury'] = np.where(play_list['BodyPart'].isnull(), 0, 1)\n\nplay_list['PlayType'] = play_list.PlayType.str.replace(' Not Returned| Returned','')\n\nplay_list['workload_num_plays'] = play_list.PlayKey.str.replace('.*-','').astype(int)\n\nprior_game = play_list.copy().sort_values(['PlayerKey','PlayerDay'])[['PlayerKey','PlayerDay','FieldType']].drop_duplicates()\nprior_game['prior_PlayerDay'] = prior_game['PlayerDay'].shift()\nprior_game['prior_PlayerKey'] = prior_game['PlayerKey'].shift()\nprior_game['prior_FieldType'] = prior_game['FieldType'].shift()\n\nprior_game['days_rest'] = np.where(prior_game['prior_PlayerKey']==prior_game['PlayerKey'],\n                                  prior_game['PlayerDay']-prior_game['prior_PlayerDay'], np.nan)\nprior_game['prior_field_type'] = np.where(prior_game['prior_PlayerKey']==prior_game['PlayerKey'], prior_game['prior_FieldType'], 'Unknown')\n\nplay_list['workload_days_rest'] = play_list[['PlayerKey','PlayerDay']].merge(prior_game, how='left',left_on=['PlayerKey','PlayerDay'],right_on=['PlayerKey','PlayerDay'],validate='many_to_one')['days_rest'].values\n# 4, 6, 7, 8+\nplay_list['workload_days_rest'] = np.where(play_list['workload_days_rest']<=7, play_list['workload_days_rest'].fillna('-1').astype(int).astype(str), '8+')\n\nplay_list['stadium_type'] = np.where(play_list['StadiumType'].str.contains('Out|Ourdoor|Oudoor|Open|Cloudy|open|Heinz Field|Bowl'),\n                                     'Open air', 'Indoor')\n\nplay_list['weather'] = np.where(play_list['stadium_type']==\"Indoor\", 'Indoor',\n                            np.where(play_list['Weather'].str.contains('Rain|Showers|rain'), 'Rain', \n                            np.where(play_list['Weather'].str.contains('Snow|snow|Cold'), 'Cold/Snow', \n                            np.where(play_list['Weather'].str.contains('Cloudy|cloudy|Clouidy|Coudy|Overcast|Hazy'), 'Cloudy', \n                            np.where(play_list['Weather'].str.contains('Clear|clear|Fair'), 'Clear/Fair', \n                            np.where(play_list['Weather'].str.contains('Sunny|sunny|Sun|Heat'), 'Sunny', 'Unknown'))))))\n\nplay_list['injury'] = play_list.BodyPart.map({'Ankle':'Ankle/Foot','Foot':'Ankle/Foot','Knee':'Knee'}).fillna('No Injury')\n\nplay_list['role'] = play_list['PlayType'] + '_' + play_list['PositionGroup']\n\nplay_list['field_type_previous_game'] = play_list[['PlayerKey','PlayerDay']].merge(prior_game, how='left',left_on=['PlayerKey','PlayerDay'],right_on=['PlayerKey','PlayerDay'],validate='many_to_one')['prior_field_type'].values\nplay_list['field_type_previous_game'] = play_list['field_type_previous_game'] + ' -> ' + play_list['FieldType']\nplay_list['field_type_previous_game_Natural_to_Synthetic'] = np.where(play_list['field_type_previous_game'] == 'Natural -> Synthetic', 1, 0)\nplay_list['field_type_previous_game_Synthetic_to_Natural'] = np.where(play_list['field_type_previous_game'] == 'Synthetic -> Natural', 1, 0)\nplay_list['field_type_previous_game_Same'] = np.where(play_list['field_type_previous_game'] == play_list['FieldType'], 1, 0)\n\nmost_freq_field_type = play_list.groupby(['PlayerKey','FieldType'])['PlayKey'].count().reset_index().sort_values('PlayKey',ascending=False).drop_duplicates(subset=['PlayerKey'])[['PlayerKey','FieldType']]\nplay_list['field_type_most_frequent'] = play_list[['PlayerKey']].merge(most_freq_field_type, how='left',left_on=['PlayerKey'],right_on=['PlayerKey'],validate='many_to_one')['FieldType'].values\nplay_list['field_type_most_frequent'] = play_list['field_type_most_frequent'] + ' -> ' + play_list['FieldType']\nplay_list['field_type_most_frequent_Natural_to_Synthetic'] = np.where(play_list['field_type_most_frequent'] == 'Natural -> Synthetic', 1, 0)\nplay_list['field_type_most_frequent_Synthetic_to_Natural'] = np.where(play_list['field_type_most_frequent'] == 'Synthetic -> Natural', 1, 0)\nplay_list['field_type_most_frequent_Same'] = np.where(play_list['field_type_most_frequent'] == play_list['FieldType'], 1, 0)\n\ntracking_data = pd.read_csv('../input/nfl-playing-surface-analytics/PlayerTrackData.csv', delimiter=',',\n           dtype={'x': np.float32,'y': np.float32,'dir': np.float32,'dis': np.float32,'time': np.float32},\n           usecols=['PlayKey','time','event','x','y','dir','dis']).sort_values(['PlayKey','time'])\n# tracking_data.info()\n# tracking_data.head()\n\ntracking_data['time_prev'] = np.where(tracking_data.PlayKey != tracking_data.PlayKey.shift(), np.nan, tracking_data.time.shift()).astype(np.float32)\ntracking_data['speed'] = round(tracking_data['dis']/(tracking_data['time']-tracking_data['time_prev']),2)\ntracking_data['speed_prev'] = np.where(tracking_data.PlayKey != tracking_data.PlayKey.shift(), np.nan, tracking_data.speed.shift()).astype(np.float32)\ntracking_data['dir_prev'] = np.where(tracking_data.PlayKey != tracking_data.PlayKey.shift(), np.nan, tracking_data.dir.shift()).astype(np.float32)\n\nplay_start = tracking_data[tracking_data.event.isin(['ball_snap','kickoff','snap_direct','onside_kick'])].drop_duplicates(subset=['PlayKey'])\n# print(len(tracking_data.drop_duplicates(subset=['PlayKey'])))\n# print(len(play_start))\n\nplay_end = tracking_data[tracking_data.event.isin(['pass_outcome_incomplete','tackle',\n        'out_of_bounds','qb_sack','touchback','touchdown','field_goal','extra_point','fair_catch','pass_outcome_touchdown',\n        'qb_kneel','punt_downed','field_goal_missed','qb_spike','extra_point_missed','safety','two_point_conversion',\n        'kick_recovered','field_goal_blocked','extra_point_blocked','qb_strip_sack'])].drop_duplicates(subset=['PlayKey'])\n# print(len(tracking_data.drop_duplicates(subset=['PlayKey'])))\n# print(len(play_end))\n\n# print(len(tracking_data))\n\ntracking_data['time_start'] = tracking_data[['PlayKey']].merge(play_start, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['time'].values\ntracking_data = tracking_data[tracking_data['time'] >= tracking_data['time_start']]\n# print(len(tracking_data))\n\ntracking_data['time_end'] = tracking_data[['PlayKey']].merge(play_end, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['time'].values\ntracking_data = tracking_data[tracking_data['time'] <= tracking_data['time_end']]\n# print(len(tracking_data))\n\ntracking_data['x_start'] = tracking_data[['PlayKey']].merge(play_start, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['x'].values\ntracking_data['y_start'] = tracking_data[['PlayKey']].merge(play_start, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['y'].values\ntracking_data['dir_start'] = tracking_data[['PlayKey']].merge(play_start, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['dir'].values\n\ntracking_data['x_end'] = tracking_data[['PlayKey']].merge(play_end, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['x'].values\ntracking_data['y_end'] = tracking_data[['PlayKey']].merge(play_end, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['y'].values\n\ntracking_data['x_prev'] = np.where(tracking_data.PlayKey != tracking_data.PlayKey.shift(), np.nan, tracking_data.x.shift()).astype(np.float32)\ntracking_data['y_prev'] = np.where(tracking_data.PlayKey != tracking_data.PlayKey.shift(), np.nan, tracking_data.y.shift()).astype(np.float32)\n\ntracking_data['time_norm'] = round(tracking_data['time'] - tracking_data['time_start'],1)\n\ntracking_data['acceleration'] = (tracking_data['speed'] - tracking_data['speed_prev'])/(tracking_data['time']-tracking_data['time_prev']).astype(np.float32)\n\n# This normalizes to 0 -> 90 where 0 is fully horizontal (i.e. facing the sideline) and 90 is fully vertical (i.e. facing an end zone)\ntracking_data['dir_norm'] = np.where(tracking_data['dir']<=90, tracking_data['dir'],\n                                np.where(tracking_data['dir']<=180, 180-tracking_data['dir'],\n                                    np.where(tracking_data['dir']<=270, np.abs(180-tracking_data['dir']), 360-tracking_data['dir'])))\n# normalize to 100\ntracking_data['dir_norm'] = tracking_data['dir_norm'] / 90 * 100\n\nmax_speed_time = tracking_data.sort_values(['PlayKey','speed'], ascending=False).drop_duplicates(\n    subset=['PlayKey'])[['PlayKey','time_norm','x','y','dir']]\n\ntracking_data['max_speed_time'] = tracking_data[['PlayKey']].merge(max_speed_time, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['time_norm'].values\ntracking_data['max_speed_x'] = tracking_data[['PlayKey']].merge(max_speed_time, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['x'].values\ntracking_data['max_speed_y'] = tracking_data[['PlayKey']].merge(max_speed_time, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['y'].values\ntracking_data['max_speed_dir'] = tracking_data[['PlayKey']].merge(max_speed_time, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['dir'].values\n\nmax_speed_times = tracking_data.query('time_norm >= max_speed_time-1 and time_norm <= max_speed_time+1').copy()\nmax_speed_times['time_norm_max_speed'] = round((max_speed_times['time_norm'] - max_speed_times['max_speed_time'])*10).astype(int)\n\ndef rotate(ox, oy, px, py, degrees):\n    radians = np.deg2rad(degrees)\n    x, y = px-ox, py-oy\n    xx = x * np.cos(radians) + y * -np.sin(radians)\n    yy = x * np.sin(radians) + y * np.cos(radians)\n    return xx, yy\n    \nmax_speed_times['x_norm_max_speed'], max_speed_times['y_norm_max_speed'] = rotate(\n                                        max_speed_times['max_speed_x'], max_speed_times['max_speed_y'],\n                                        max_speed_times['x'], max_speed_times['y'],\n                                        max_speed_times['max_speed_dir'])\n\ntracking_data['x_norm_prev'], tracking_data['y_norm_prev'] = rotate(\n                                        tracking_data['x_prev'], tracking_data['y_prev'],\n                                        tracking_data['x'], tracking_data['y'],\n                                        tracking_data['dir_prev'])\n\n# max horizontal change in direction/movement (relateive to prior movement direction)\n# max vertical change in direction/movement\ntracking_data['lateral_speed'] = (tracking_data['x_norm_prev']/(tracking_data['time']-tracking_data['time_prev'])).abs()\ntracking_data['vertical_speed'] = tracking_data['y_norm_prev']/(tracking_data['time']-tracking_data['time_prev'])\n\n# Speed (yards/sec)\nspeed_max = tracking_data.groupby(['PlayKey'])['speed'].max().reset_index().rename(columns={'speed':'speed_max'})\nspeed_avg = tracking_data.groupby(['PlayKey'])['speed'].mean().reset_index().rename(columns={'speed':'speed_avg'})\n\n# Directional changes\n\n#   Horizontal vs. Vertical distance ratio (based on start/end position)\n#   0 = all distance traveled horizontal i.e. toward sideline\n#   100 = all distance traveled vertical i.e. toward end zone\nhv_distance = tracking_data.drop_duplicates(['PlayKey']).copy()\nhv_distance['hv_distance'] = np.abs(hv_distance['x_end'] - hv_distance['x_start']) + np.abs(hv_distance['y_end'] - hv_distance['y_start'])\nhv_distance['hv_distance'] = np.where(hv_distance['hv_distance']>0, \n                                      (np.abs(hv_distance['y_end'] - hv_distance['y_start']) / hv_distance['hv_distance'])*100, 50)\nhv_distance = hv_distance[['PlayKey','hv_distance']]\n\n#   Horizontal vs. Vertical direction\n#   0 = all movement horizontal i.e. toward sideline\n#   100 = all movement vertical i.e. toward end zone\nhv_direction = tracking_data.groupby(['PlayKey'])['dir_norm'].mean().reset_index().rename(columns={'dir_norm':'hv_direction'})\n\n# total dir change\ndir_change = tracking_data.groupby(['PlayKey'])['dir'].agg({'max','min'}).reset_index()\ndir_change['total_dir_change'] = dir_change['max']-dir_change['min']\ndir_change = dir_change[['PlayKey','total_dir_change']]\n\nmax_lateral_speed = tracking_data.groupby(['PlayKey'])['lateral_speed'].max().reset_index().rename(columns={'lateral_speed':'max_lateral_speed'})\nmax_vertical_speed = tracking_data.groupby(['PlayKey'])['vertical_speed'].max().reset_index().rename(columns={'vertical_speed':'max_vertical_speed'})\nmin_vertical_speed = tracking_data.groupby(['PlayKey'])['vertical_speed'].min().reset_index().rename(columns={'vertical_speed':'min_vertical_speed'})\n\n# Acceleration\nacceleration_max = tracking_data.groupby(['PlayKey'])['acceleration'].max().reset_index().rename(columns={'acceleration':'acceleration_max'})\nseconds_until_max_speed = tracking_data.sort_values('speed', ascending=False).drop_duplicates('PlayKey')[['PlayKey','time_norm']].rename(columns={'time_norm':'seconds_until_max_speed'})\n\n# Deceleration\ndeceleration_max = tracking_data.groupby(['PlayKey'])['acceleration'].min().reset_index().rename(columns={'acceleration':'deceleration_max'})\n\n# Distance\ntotal_distance = tracking_data.groupby(['PlayKey'])['dis'].sum().reset_index().rename(columns={'dis':'total_distance'})\n\n# Play length\nplay_length = tracking_data.groupby(['PlayKey'])['time_norm'].max().reset_index().rename(columns={'time_norm':'play_length'})\n\ncluster_features = pd.concat([max_speed_times.pivot('PlayKey','time_norm_max_speed','x_norm_max_speed').add_prefix('x_'),\n                 max_speed_times.pivot('PlayKey','time_norm_max_speed','y_norm_max_speed').add_prefix('y_')], axis=1, sort=False)\nfor prefix in ['x','y']:\n    for mult in [1,-1]:\n        for n in range(1,11):\n            prev = mult*(n-1)\n            cur = round(mult*n,1)\n            col_prev = '%s_%s'%(prefix, prev)\n            col = '%s_%s'%(prefix, cur)\n            cluster_features[col] = winsorize(cluster_features[col].fillna(cluster_features[col_prev]), limits=[0.01, 0.01])\n            \n# transpose\nfor n in [10,9,8,7,6,5,4,3,2,1]:\n    cluster_features['x_%s'%n] = np.where(cluster_features['x_-1'] < 0, -cluster_features['x_%s'%n], cluster_features['x_%s'%n])\n    cluster_features['x_-%s'%n] = np.where(cluster_features['x_-1'] < 0, -cluster_features['x_-%s'%n], cluster_features['x_-%s'%n])\n    \nX = StandardScaler().fit_transform(cluster_features[[\n        'x_-10', 'x_-9', 'x_-8', 'x_-7', 'x_-6', 'x_-5', 'x_-4', 'x_-3', 'x_-2', 'x_-1',\n    'y_-10', 'y_-9', 'y_-8', 'y_-7', 'y_-6', 'y_-5', 'y_-4', 'y_-3', 'y_-2', 'y_-1']])\n\nkmeans = MiniBatchKMeans(n_clusters=100, random_state=0).fit(X)\ncluster_prev_max_speed = cluster_features.copy()\ncluster_prev_max_speed['cluster_num'] = kmeans.labels_\ncluster_prev_max_speed['cluster'] = 'cluster_'+cluster_prev_max_speed['cluster_num'].astype(str)\ncluster_prev_max_speed = cluster_prev_max_speed.reset_index()[['PlayKey','cluster_num','cluster']]\ncluster_prev_max_speed.groupby('cluster')['cluster'].count().describe()\n\nX = StandardScaler().fit_transform(cluster_features[[\n    'x_1', 'x_2', 'x_3','x_4', 'x_5', 'x_6', 'x_7', 'x_8', 'x_9', 'x_10',\n    'y_1', 'y_2', 'y_3', 'y_4','y_5', 'y_6', 'y_7', 'y_8', 'y_9', 'y_10']])\n\nkmeans = MiniBatchKMeans(n_clusters=100, random_state=0).fit(X)\ncluster_post_max_speed = cluster_features.copy()\ncluster_post_max_speed['cluster_num'] = kmeans.labels_\ncluster_post_max_speed['cluster'] = 'cluster_'+cluster_post_max_speed['cluster_num'].astype(str)\ncluster_post_max_speed = cluster_post_max_speed.reset_index()[['PlayKey','cluster_num','cluster']]\ncluster_post_max_speed.groupby('cluster')['cluster'].count().describe()\n\nplay_list['player_speed_max'] = play_list[['PlayKey']].merge(speed_max, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['speed_max'].values\nplay_list['player_speed_avg'] = play_list[['PlayKey']].merge(speed_avg, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['speed_avg'].values\n\nplay_list['player_hv_distance'] = play_list[['PlayKey']].merge(hv_distance, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['hv_distance'].values\nplay_list['player_hv_direction'] = play_list[['PlayKey']].merge(hv_direction, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['hv_direction'].values\n\nplay_list['player_total_dir_change'] = play_list[['PlayKey']].merge(dir_change, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['total_dir_change'].values\n\nplay_list['player_max_lateral_speed'] = play_list[['PlayKey']].merge(max_lateral_speed, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['max_lateral_speed'].values\nplay_list['player_max_vertical_speed'] = play_list[['PlayKey']].merge(max_vertical_speed, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['max_vertical_speed'].values\nplay_list['player_min_vertical_speed'] = play_list[['PlayKey']].merge(min_vertical_speed, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['min_vertical_speed'].values\n\nplay_list['player_acceleration_max'] = play_list[['PlayKey']].merge(acceleration_max, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['acceleration_max'].values\nplay_list['player_seconds_until_max_speed'] = play_list[['PlayKey']].merge(seconds_until_max_speed, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['seconds_until_max_speed'].values\nplay_list['player_deceleration_max'] = -1*play_list[['PlayKey']].merge(deceleration_max, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['deceleration_max'].values\n\nplay_list['player_total_distance'] = play_list[['PlayKey']].merge(total_distance, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['total_distance'].values\n\nplay_list['play_length'] = play_list[['PlayKey']].merge(play_length, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['play_length'].values\n\nplay_list['player_total_distance'] = play_list[['PlayKey']].merge(total_distance, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['total_distance'].values\nplay_list['fingerprint_post_max_speed'] = play_list[['PlayKey']].merge(cluster_post_max_speed, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['cluster'].values\nplay_list['fingerprint_prev_max_speed'] = play_list[['PlayKey']].merge(cluster_prev_max_speed, how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')['cluster'].values\n\nfor c in ['player_speed_max','player_speed_avg','player_acceleration_max','player_deceleration_max','player_seconds_until_max_speed',\n          'play_length','player_total_distance','player_total_dir_change','player_max_lateral_speed',\n         'player_max_vertical_speed','player_min_vertical_speed']:\n    play_list[c] = np.where(play_list[c].isnull(), np.nan, winsorize(play_list[c], limits=[0.01, 0.01]))\n    \nplay_list['weather_field_type'] = play_list['FieldType'] + '*' + play_list['weather']\n\nstyle.use('fivethirtyeight')\nsns.set_context('paper')\n\ndef plot_plays(plays, groupby='PlayKey', col_wrap=4):\n    grid = sns.FacetGrid(plays, col=groupby, col_order=sorted(set(plays[groupby])),\n                  col_wrap=col_wrap, height=3, aspect=1.0, sharex=False, sharey=False,despine=False)\n\n    grid.map(plt.axvline, x=0, ls=\":\", c=\".5\")\n    grid.map(plt.axvline, x=53.3, c=\".5\")\n    grid.map(plt.axvline, x=53.33-.5, ls=\":\", c=\".5\")\n    grid.map(plt.axvline, x=23, ls=\":\", c=\".5\")\n    grid.map(plt.axvline, x=53.33-23, ls=\":\", c=\".5\")\n\n    grid.map(plt.axhline, y=0, c=\".5\")\n    grid.map(plt.axhline, y=100, c=\".5\")\n    grid.map(plt.axhline, y=50, c=\".5\")\n    grid.map(plt.axhline, y=-10, c=\".5\")\n    grid.map(plt.axhline, y=110, c=\".5\")\n\n    grid.map(sns.scatterplot,'y','x', 'speed', size_norm=(1,10), size=plays['time_norm'], palette='Reds')\n    grid.set(\n            yticks=range(-10,120,10), xticks=[],\n        yticklabels=['','G','10','20','30','40','50','40','30','20','10','G',''],\n            ylabel='', xlabel='',\n             ylim=(-11,111), xlim=(-0.5,53.3))\n\n    grid.fig.tight_layout(w_pad=1)\n    plt.show()\n    \ndef plot_whisker(metric):\n    sns.boxplot(x='injury', y=metric, hue='FieldType', data=plays)\n    plt.legend(loc='center left',bbox_to_anchor=(1,0.5))\n    plt.show()\n\ndef plot_heatmap(metric):\n    play_list.groupby(['FieldType','injury'])[metric].mean().to_frame().round(3)\n    ax = sns.heatmap(play_list.groupby(['FieldType','injury'])[metric].mean().to_frame().reset_index().pivot('injury','FieldType',metric),\n         annot=True, fmt=\".2f\", cmap=\"Greens\", cbar=True)\n    ax.xaxis.tick_top()\n    ax.set_title(metric)\n    plt.show()\n    play_list.hist(metric, by='injury', sharex=True)\n    plt.show()\n    \ndef plot_cluster_examples(cluster_plays, col):\n    df = tracking_data[tracking_data.PlayKey.isin(cluster_plays.groupby('cluster').max()['PlayKey'])].copy()\n    df = df.merge(plays[['PlayKey',col]], how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')\n    df['cluster'] = df[col] + ' | ' + df['PlayKey'].astype(str)\n    plot_plays(df, 'cluster', 5)\n    \ndef plot_metric_examples(metric, unit=''):\n    max_min = plays.drop_duplicates(subset=['PlayKey']).sort_values(metric)['PlayKey'].values\n    samples = [max_min[int(len(max_min)*p)] for p in [.01,.25,.5,.75,.99]]\n    df = tracking_data[tracking_data.PlayKey.isin(samples)].copy()\n    df = df.merge(plays[['PlayKey',metric]], how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one').sort_values(metric)\n    df[metric] = df[metric].rank(method='dense').astype(int).map({1:'p1',2:'p25',3:'p50',4:'p75',5:'p99'}) + \" (\" + df[metric].apply(lambda m: \"%.1f\" % (m)) + ' %s)'%unit\n    plot_plays(df, metric, 5)\n    \ndef modify_cluster_df(df):\n    df['cluster '] = df['cluster'] + '\\nmax_speed = ~' + round(df['avg_speed'],1).astype(str) + ' yards/sec'\n    # filter to 1000 plays (12 rows per play)\n    return df.head(12*1000)\n\ndef plot_max_speed(df, color='Purples', ylim=None, title=''):\n    grid = sns.FacetGrid(df, col='cluster ', col_order=sorted(set(df['cluster '])),\n                  col_wrap=5, height=3, aspect=1.0, sharey=True, sharex=True,despine=True)\n\n    grid.map(plt.axvline, x=0, c=\".5\")\n    grid.map(plt.axhline, y=0, c=\".5\")\n\n    grid.map(sns.lineplot,'x_norm_max_speed','y_norm_max_speed', 'PlayKey', \n             palette=color, sort=False, ci=None)\n\n    grid.set(#xticks=[-1,0,1], yticks=[-3,-2,-1,0],\n             xlim=(-2.5,2.5), \n        ylim=ylim\n    )\n    grid.fig.tight_layout()\n    plt.suptitle(title)\n    plt.subplots_adjust(top=.75)\n    plt.show()\n    \ndef basic_bar(category, sort=False):\n    df = play_list.groupby([category,'injury'])['PlayKey'].count().to_frame().reset_index()\n    df['% injuries'] = 100*(np.where(df['injury']=='No Injury', 0, df['PlayKey'] / len(play_list.query('has_injury==1'))))\n    df['% plays'] = 100*(df['PlayKey'] / len(play_list))\n    df = df.groupby(category)[['% injuries','% plays']].sum()\n    if sort:\n        df = df.sort_values('% plays', ascending=False)\n    ax = df.plot.barh(color=['tab:red','gray'])\n    ax.invert_yaxis()\n    plt.legend(loc='center left',bbox_to_anchor=(1,0.5))\n    plt.xlabel('%')\n    plt.show()\n    \ndef plot_ci(cur, title):\n    g = sns.boxplot(x=list(cur[['0.975]','[0.025']].values), y=list(cur.index.values), hue=list(cur['Model'].values),\n                   linewidth=.5)\n\n    fig = plt.gcf()\n    plt.title('%s (Confidence Interval)'%title)\n#     plt.xlim(-xlim,xlim)\n    plt.show()\n    return cur.sort_values(['variable','Model'])\n\nprint(len(play_list), len(play_list.drop_duplicates(subset=['PlayerKey'])))\nplays = play_list[~play_list.PlayType.isin(['Extra Point','Field Goal','0'])]\nprint(len(plays), len(plays.drop_duplicates(subset=['PlayerKey'])))\nplays = plays.dropna(subset=['weather','PlayType'])\nprint(len(plays), len(plays.drop_duplicates(subset=['PlayerKey'])))\nplays = plays.dropna(subset=['weather','PlayType','player_speed_max'])\nprint(len(plays), len(plays.drop_duplicates(subset=['PlayerKey'])))\n\nplays.info()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":false},"cell_type":"code","source":"plays['fixed_effects'] = plays.PlayerKey.astype(str)# + '/' + plays.PlayType\n# plays['fixed_effects'] = plays.PositionGroup.astype(str) + '/' + plays.PlayType\n\nX_cols = [\n    # movement\n#     'fingerprint_prev_max_speed','fingerprint_post_max_speed',\n    'player_speed_max', #'player_speed_avg',\n    'player_hv_distance', 'player_hv_direction', 'player_acceleration_max',\n    'player_total_dir_change','player_max_lateral_speed', 'player_min_vertical_speed', #'player_max_vertical_speed',\n    'player_seconds_until_max_speed', 'player_deceleration_max', 'player_total_distance',\n    # context\n    'play_length', 'workload_days_rest', 'workload_num_plays',    \n#     'PositionGroup',\n    'PlayType',\n    # stadium/weather\n    'weather',\n    # playing surface\n    'FieldType',\n    'weather_field_type',\n#     'field_type_previous_game','field_type_most_frequent',\n    'field_type_previous_game_Natural_to_Synthetic','field_type_previous_game_Synthetic_to_Natural',\n    'field_type_most_frequent_Natural_to_Synthetic','field_type_most_frequent_Synthetic_to_Natural',\n    # FE\n    'fixed_effects'\n]\nX = pd.get_dummies(plays[X_cols], drop_first=False)\nX = X.drop(['workload_days_rest_7',#'fingerprint_prev_max_speed_cluster_0','fingerprint_post_max_speed_cluster_0',\n            'PlayType_Rush','weather_Indoor','FieldType_Natural','weather_field_type_Natural*Indoor','fixed_effects_26624'],axis=1)\n# X = pd.DataFrame(StandardScaler().fit_transform(X), columns=X.columns).astype(np.float32)\nfor c in ['player_speed_max', #'player_speed_avg',\n    'player_hv_distance', 'player_hv_direction', 'player_acceleration_max',\n    'player_total_dir_change','player_max_lateral_speed', 'player_min_vertical_speed', #'player_max_vertical_speed',\n    'player_seconds_until_max_speed', 'player_deceleration_max', 'player_total_distance',\n    'play_length', \n       'workload_num_plays']:\n    X[c] = StandardScaler().fit_transform(X[[c]]).reshape(-1, 1)\n\n# model = LinearRegression().fit(X,y)\n# results = pd.DataFrame(model.coef_.T, X.columns, columns=['coef']).sort_values('coef',ascending=False)\n\ny = plays['has_injury'].values\n%time ols = sm.OLS(y, X).fit()\nstats_injury = ols.summary().tables[0]\nols = ols.summary().tables[1].data[1:]\nresults_injury = pd.DataFrame(ols, columns=['variable','coef','std err','t','P>|t|','[0.025','0.975]'], dtype='float32')\nresults_injury['Model'] = 'Injury'\nresults_injury = results_injury.set_index('variable',drop=True).sort_values('coef',ascending=False)\n\ny = np.where(plays['injury']=='Ankle/Foot', 1, 0)\n%time ols = sm.OLS(y, X).fit()\nstats_ankle = ols.summary().tables[0]\nols = ols.summary().tables[1].data[1:]\nresults_ankle = pd.DataFrame(ols, columns=['variable','coef','std err','t','P>|t|','[0.025','0.975]'], dtype='float32')\nresults_ankle['Model'] = 'Ankle'\nresults_ankle = results_ankle.set_index('variable',drop=True).sort_values('coef',ascending=False)\n\ny = np.where(plays['injury']=='Knee', 1, 0)\n%time ols = sm.OLS(y, X).fit()\nstats_knee = ols.summary().tables[0]\nols = ols.summary().tables[1].data[1:]\nresults_knee = pd.DataFrame(ols, columns=['variable','coef','std err','t','P>|t|','[0.025','0.975]'], dtype='float32')\nresults_knee['Model'] = 'Knee'\nresults_knee = results_knee.set_index('variable',drop=True).sort_values('coef',ascending=False)\n\nall_results = pd.concat([results_injury, results_ankle, results_knee])\n\nX_cols = [\n        # movement\n        'fingerprint_prev_max_speed','fingerprint_post_max_speed',\n        'player_speed_max', #'player_speed_avg',\n        'player_hv_distance', 'player_hv_direction', 'player_acceleration_max',\n        'player_total_dir_change','player_max_lateral_speed', 'player_min_vertical_speed', #'player_max_vertical_speed',\n        'player_seconds_until_max_speed', 'player_deceleration_max', 'player_total_distance',\n        # context\n        'play_length', 'workload_days_rest', 'workload_num_plays',    \n    #     'PositionGroup',\n        'PlayType',\n        # stadium/weather\n        'weather',\n        # playing surface\n        'FieldType',\n        'weather_field_type',\n    #     'field_type_previous_game','field_type_most_frequent',\n        'field_type_previous_game_Natural_to_Synthetic','field_type_previous_game_Synthetic_to_Natural',\n        'field_type_most_frequent_Natural_to_Synthetic','field_type_most_frequent_Synthetic_to_Natural',\n        # FE\n        'fixed_effects'\n]\nX = pd.get_dummies(plays[X_cols], drop_first=False)\nX = X.drop(['workload_days_rest_7','fingerprint_prev_max_speed_cluster_0','fingerprint_post_max_speed_cluster_0',\n            'PlayType_Rush','weather_Indoor','FieldType_Natural','weather_field_type_Natural*Indoor','fixed_effects_26624'],axis=1)\n# X = pd.DataFrame(StandardScaler().fit_transform(X), columns=X.columns).astype(np.float32)\nfor c in ['player_speed_max', #'player_speed_avg',\n    'player_hv_distance', 'player_hv_direction', 'player_acceleration_max',\n    'player_total_dir_change','player_max_lateral_speed', 'player_min_vertical_speed', #'player_max_vertical_speed',\n    'player_seconds_until_max_speed', 'player_deceleration_max', 'player_total_distance',\n    'play_length', \n       'workload_num_plays']:\n    X[c] = StandardScaler().fit_transform(X[[c]]).reshape(-1, 1)\n\n# model = LinearRegression().fit(X,y)\n# results = pd.DataFrame(model.coef_.T, X.columns, columns=['coef']).sort_values('coef',ascending=False)\n\ny = plays['has_injury'].values\n%time ols = sm.OLS(y, X).fit()\nstats_injury = ols.summary().tables[0]\nols = ols.summary().tables[1].data[1:]\nresults_injury_fp = pd.DataFrame(ols, columns=['variable','coef','std err','t','P>|t|','[0.025','0.975]'], dtype='float32')\nresults_injury_fp['Model'] = 'Injury'\nresults_injury_fp = results_injury_fp.set_index('variable',drop=True).sort_values('coef',ascending=False)\n\ny = np.where(plays['injury']=='Ankle/Foot', 1, 0)\n%time ols = sm.OLS(y, X).fit()\nstats_ankle = ols.summary().tables[0]\nols = ols.summary().tables[1].data[1:]\nresults_ankle_fp = pd.DataFrame(ols, columns=['variable','coef','std err','t','P>|t|','[0.025','0.975]'], dtype='float32')\nresults_ankle_fp['Model'] = 'Ankle'\nresults_ankle_fp = results_ankle_fp.set_index('variable',drop=True).sort_values('coef',ascending=False)\n\ny = np.where(plays['injury']=='Knee', 1, 0)\n%time ols = sm.OLS(y, X).fit()\nstats_knee = ols.summary().tables[0]\nols = ols.summary().tables[1].data[1:]\nresults_knee_fp = pd.DataFrame(ols, columns=['variable','coef','std err','t','P>|t|','[0.025','0.975]'], dtype='float32')\nresults_knee_fp['Model'] = 'Knee'\nresults_knee_fp = results_knee_fp.set_index('variable',drop=True).sort_values('coef',ascending=False)\n\nall_results_fp = pd.concat([results_injury_fp, results_ankle_fp, results_knee_fp])","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":false},"cell_type":"code","source":"X_cols = [\n    # movement\n#     'fingerprint_prev_max_speed','fingerprint_post_max_speed',\n#     'player_speed_max', #'player_speed_avg',\n#     'player_hv_distance', 'player_hv_direction', 'player_acceleration_max',\n#     'player_total_dir_change','player_max_lateral_speed', 'player_min_vertical_speed', #'player_max_vertical_speed',\n#     'player_seconds_until_max_speed', 'player_deceleration_max', 'player_total_distance',\n    # context\n    'play_length', 'workload_days_rest', 'workload_num_plays',    \n#     'PositionGroup',\n    'PlayType',\n    # stadium/weather\n    'weather',\n    # playing surface\n    'FieldType',\n    'weather_field_type',\n#     'field_type_previous_game','field_type_most_frequent',\n    'field_type_previous_game_Natural_to_Synthetic','field_type_previous_game_Synthetic_to_Natural',\n    'field_type_most_frequent_Natural_to_Synthetic','field_type_most_frequent_Synthetic_to_Natural',\n    # FE\n    'fixed_effects'\n]\nX = pd.get_dummies(plays[X_cols], drop_first=False)\nX = X.drop(['workload_days_rest_7',#'fingerprint_prev_max_speed_cluster_0','fingerprint_post_max_speed_cluster_0',\n            'PlayType_Rush','weather_Indoor','FieldType_Natural','weather_field_type_Natural*Indoor','fixed_effects_26624'],axis=1)\n\ndef run_ols(y, type):\n    ols = sm.OLS(y, X).fit()\n    ols = ols.summary().tables[1].data[1:]\n    results_cur = pd.DataFrame(ols, columns=['variable','coef','std err','t','P>|t|','[0.025','0.975]'], dtype='float32')\n    results_cur['Model'] = type\n    results_cur = results_cur.set_index('variable',drop=True).sort_values('coef',ascending=False)\n    results_cur = results_cur[~results_cur.index.str.startswith('fixed_effects')]\n    return results_cur\n\nresults_speed = run_ols(plays['player_speed_max'], 'Speed Max')\nresults_lateral = run_ols(plays['player_max_lateral_speed'], 'Lateral Speed Max')\nresults_acc = run_ols(plays['player_acceleration_max'], 'Acceleration Max')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# NFL 1st and Future - Analytics\n*Can you investigate the relationship between the playing surface and the injury and performance of NFL athletes?*"},{"metadata":{"_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"tracking_injured = tracking_data[tracking_data.PlayKey.isin(injuries.PlayKey)].copy()\ntracking_injured = tracking_injured.merge(play_list[['PlayKey','BodyPart','PlayType','PositionGroup']], how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one')\ntracking_injured['PlayType'] = tracking_injured.PlayType.str.replace(' Not Returned| Returned','')\ntracking_injured['BodyPart'] = tracking_injured.BodyPart.map({'Ankle':'Ankle/Foot','Foot':'Ankle/Foot','Knee':'Knee'})\n\ntracking_injured['Injury'] = tracking_injured['PlayType'] + ' | ' + tracking_injured['BodyPart']\nplot_plays(tracking_injured, 'Injury')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Key Takeaways / Application\n1. After controlling for all other factors, ***there are aspects of player movement that appear to have a statistically significant impact on injury risk***.\n * Higher Risk:\n     * Max Lateral Speed\n     * Max Speed\n     * Natural field type in the Cold/Snow\n * Lower Risk:\n     * Larger time until reaching max speed\n     * Higher Acceleration Max (after controlling for other factors)\n     * Higher HV Distance (i.e. vertical distance)\n1. Synthetic turf ***does have*** a statistically significant impact on many of the factors that impact injury risk - it increases:\n  * Max speed\n  * Lateral max speed\n  * Acceleration max\n1. ***Switching field types might matter*** - While not statistically significant, there were observed trends that switching field types increases injury risk (relative to previous game and relative to the players most common field type). There is an argument to be made that the NFL should standardize Natural/Synthentic even if it for everyone on Sythentc. Alternatively, the NFL could try to account for this through scheduling (limit switching field types in back to back weeks) or even encouraging within division teams to try to have same field type, etc. It might be interesting to extend this analysis to teams practice facilities and see how much that matters.","attachments":{}},{"metadata":{},"cell_type":"markdown","source":"# Intro\n1. Define movement metrics\n1. Identify specific variables that present an elevated risk of injury\n 1. Data Exploration\n 1. Movement metric trends\n 1. Methodology: Fixed Effects Model\n1. Evaluation of differences in player movement between playing surfaces\n 1. Are there specific movement patterns that correlate with the acute onset of injury (in general or by specific injury location)?\n 1. Are there summary metrics of player movement which influence risk of injury?\n 1. How do playing surface, game scenario, player movement, and weather interact to influence the risk of injury?\n1. Evaluation of differences in player movement between playing surfaces\n 1. For any metrics of player movement shown to influence risk of injury, do differences in these metrics exist across playing surfaces for the non-injured player population?\n 1. Do any of the environmental or game scenario variables influence player movement in the non-injured player population?"},{"metadata":{},"cell_type":"markdown","source":"# 1. Define movement metrics\n## Metric Definitions\n\nTo help understand what types of player movement contribute towards injuries, we must first define metrics that characterize player movement on the field. To help understand what types of movement these metrics are picking up on I have included sample play diagrams at different percentiles of each metric.\n\n## Speed\n\n* ***Speed Max:*** Maximum speed of player on a play (yards/sec)\n* ***Speed Avg:*** Average speed of player on a play (yards/sec)\n* ***Lateral Speed Max:*** Max horizontal change in direction (i.e. sharp cut) - normalized to prior NGS data point direction of motion (yards/sec)\n* ***Max Vertical Speed:*** Max horizontal change in direction (i.e. straight line running) - normalized to prior NGS data point direction of motion (yards/sec)\n* ***Min Vertical Speed:*** Min vertical change in direction (i.e. backwards) - normalized to prior NGS data point direction of motion (yards/sec)"},{"metadata":{"_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"style.use('fivethirtyeight')\nsns.set_context('paper')\n\nplot_metric_examples('player_speed_max', 'yd/s')\nplot_metric_examples('player_speed_avg', 'yd/s')\nplot_metric_examples('player_max_lateral_speed', 'yd/s')\nplot_metric_examples('player_max_vertical_speed', 'yd/s')\nplot_metric_examples('player_min_vertical_speed', 'yd/s')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Directional changes\n* ***HV Distance:*** Ratio of horizontal vs. vertical distance (based on start/end position)\n * 0 = all distance traveled vertical i.e. toward end zone\n * 100 = all distance traveled horizontal i.e. toward sideline\n* ***HV Direction:*** Ratio of horizontal vs. vertical movement direction\n * 0 = all movement horizontal i.e. toward sideline\n * 100 = all movement vertical i.e. toward end zone\n* ***Total Direction Change:*** Difference between max/min direction (dir) on a play (degrees)"},{"metadata":{"_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"plot_metric_examples('player_hv_distance', '')\nplot_metric_examples('player_hv_direction', '')\nplot_metric_examples('player_total_dir_change', '')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Acceleration/Deceleration\n* ***Acceleration Max:*** Maximum acceleration of player (yd/s²)\n* ***Deceleration Max:*** Maximum deceleration of player (yd/s²)"},{"metadata":{"_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"plot_metric_examples('player_acceleration_max', 'yd/s²')\nplot_metric_examples('player_deceleration_max', 'yd/s²')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Distance\n* ***Total Distance:*** Distance traveled on the play from snap to end of play (yards)"},{"metadata":{"_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"plot_metric_examples('player_total_distance', 'yards')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Time\n* ***Seconds until max speed:*** Seconds after snap until player reached max speed (seconds)\n* ***Play Length:*** Time from snap to end of play (seconds)"},{"metadata":{"_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"plot_metric_examples('player_seconds_until_max_speed', '')\nplot_metric_examples('play_length', 'secs')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Normalized direction\nTo calculate a handful of the metrics above, we had to normalize a players movement relative to the location data point. The reason for doing this is to better understand how players are shifting direction across each data point.\n\nLater on I will be relying on another variation of this where I normalize a players movement relative to the NGS data point where the player had his max speed on the play. When doing so I will be drilling into the second just prior/after the time stamp when he reaches his max speed. This will allow us to better understand what kind of movement the player is doing during the riskiest points of the play (i.e. when traveling his fastest).\n\nTo make this more clear I will show an example below. As you can see the players max speed is transformed to the origin (0,0). The vertical axis represents the movement direction of the player at the time of his \"max speed\". Note that I used the direction instead of orientation based on instructions saying to avoid using orientation."},{"metadata":{"_kg_hide-input":true,"collapsed":true,"trusted":false},"cell_type":"code","source":"plot_plays(tracking_data[tracking_data.PlayKey.isin(['30068-24-22'])], 'PlayKey')\n\nsns.scatterplot(data=max_speed_times[max_speed_times.PlayKey.isin(['30068-24-22'])], x = 'y', y = 'x', hue='PlayKey')\nplt.ylim(0,120)\nplt.xlim(0,53.3)\nplt.title('Actual XY second before/after max speed')\nplt.show()\n\nsns.scatterplot(data=max_speed_times[max_speed_times.PlayKey.isin(['30068-24-22'])], x = 'x_norm_max_speed', y = 'y_norm_max_speed', hue='PlayKey')\nplt.xlim(-2,2)\nplt.title('Normalized XY second before/after max speed')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. Identify variables that present an elevated risk of injury\n## Data Exploration\n\nLooking at descriptive stats we can see the high level trends. The correlations that might be promising include:\n1. Bad weather (rain, cold, snow) appears to increase injury risk\n1. Surprisingly, days rest might have an inverse relationship with injury risk - muscles not in game shape?\n1. Play type / position matter i.e. WR/LB and Kickoff/Punt appear to be higher risk\n1. Changing field types might increase risk - this appears to be true for the following:\n 1. A player playing on a different field type than their last game\n 1. Players who more commonly play on Natural but then play on Synthetic - this could also be due to home vs. road differences"},{"metadata":{"trusted":false,"_kg_hide-input":true},"cell_type":"code","source":"basic_bar('weather', sort=True)\nbasic_bar('workload_days_rest')\nbasic_bar('FieldType')\nbasic_bar('PositionGroup', sort=True)\nbasic_bar('PlayType', sort=True)\nbasic_bar('field_type_previous_game')\nbasic_bar('field_type_most_frequent')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Player Movement\n\nMovement metric trends theories that look interesting to me include:\n1. Higher speed/acceleration = higher injury risk\n1. Knee injuries = long plays with straight running\n 1. HV distance is lower (i.e. more vertical running)\n 1. Play length, seconds until max speed, total distance all higher\n 1. Potentially punt/kick coverage based on above\n1. Ankle injuries = higher lateral max speeds"},{"metadata":{"trusted":false,"_kg_hide-input":true},"cell_type":"code","source":"plot_whisker('player_speed_max')\nplot_whisker('player_speed_avg')\nplot_whisker('player_hv_distance')\nplot_whisker('player_hv_direction')\nplot_whisker('player_acceleration_max')\nplot_whisker('player_seconds_until_max_speed')\nplot_whisker('player_deceleration_max')\nplot_whisker('player_total_distance')\nplot_whisker('play_length')\nplot_whisker('player_total_dir_change')\nplot_whisker('player_max_lateral_speed')\nplot_whisker('player_max_vertical_speed')\nplot_whisker('player_min_vertical_speed')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Methodology\nWith many factors all working together to contribute to injuries, we need to ensure that we are controling for as many factors as possible. Therefore it is important to extend beyond simple correlations to a model which can help us tease out where any larger signals might be coming from.\n\n## Fixed Effects Model\n\nhttps://en.wikipedia.org/wiki/Fixed_effects_model\n\n\nI implemented a player fixed effects model to identify the causal relationship between risk factors such as player movements and injury risk. The player fixed effect controls for any characteristics of a player that are constant in the data set (e.g. prior injury history, height, weight, movement abilities, role in play), eliminating the link between them and the risk factors of interest. The model estimates are then identified offf of within-player changes in the risk factors of interest. The average number of samples for each player-fixed-effects group is ~1021.\n\nAside from the fixed effects our model will include:\n1. player movement metrics (speed, accceleration, lateral, distance, etc.)\n1. play context (length, type)\n1. workload (rest, num plays in game)\n1. weather (including interactions with field type)\n1. field type (current, change from last game, change from most common)\n\nWe will then use this general approach for a few different tasks:\n1. Estimate how player movement and other factors influence injury risk (overall and specific to body part) \n1. Estimate how other factors influence high risk player movement metrics across the entire population (predominately non-injured)\n1. Understand what types of max speed movement fingerprints (see below) are higher/lower risk"},{"metadata":{},"cell_type":"markdown","source":"### Data\n\nThe model is built off of 255275 samples which is >95% of the original data set provided. I filtered out Extra Point and Field Goals as there wasn't any injuries during those plays (or much movement to learn from) and dropped a small number of samples with null values."},{"metadata":{"_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"X = plays[[\n    # movement\n#         'fingerprint_prev_max_speed','fingerprint_post_max_speed',\n        'player_speed_max', #'player_speed_avg',\n        'player_hv_distance', 'player_hv_direction', 'player_acceleration_max',\n        'player_total_dir_change','player_max_lateral_speed', 'player_min_vertical_speed', #'player_max_vertical_speed',\n        'player_seconds_until_max_speed', 'player_deceleration_max', 'player_total_distance',\n        # context\n        'play_length', 'workload_days_rest', 'workload_num_plays',    \n    #     'PositionGroup',\n    #     'PlayType',\n        # stadium/weather\n        'weather',\n        # playing surface\n        'FieldType',\n        'weather_field_type',\n    #     'field_type_previous_game','field_type_most_frequent',\n        'field_type_previous_game_Natural_to_Synthetic','field_type_previous_game_Synthetic_to_Natural',\n        'field_type_most_frequent_Natural_to_Synthetic','field_type_most_frequent_Synthetic_to_Natural',\n        # FE\n#         'fixed_effects'\n]].info()\n\nX = plays[['player_speed_max', #'player_speed_avg',\n        'player_hv_distance', 'player_hv_direction', 'player_acceleration_max',\n        'player_total_dir_change','player_max_lateral_speed', 'player_min_vertical_speed', #'player_max_vertical_speed',\n        'player_seconds_until_max_speed', 'player_deceleration_max', 'player_total_distance',]]\nsns.heatmap(X.corr(), annot=True, fmt=\".2f\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Results\nBelow are the statistically significant results. See the Appedix A.3 for coefficients and CI's for all variables."},{"metadata":{"_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"all_filtered = all_results[~((all_results.index.str.startswith('fixed_effects'))|(\n    all_results.index.str.startswith('fingerprint')))].sort_values('0.975]',ascending=False).round(6)\n\nplot_ci(all_filtered[all_filtered['P>|t|']<=.05], 'Statistically Significant Variables')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. Evaluation of differences in player movement between playing surfaces\n\n*For any metrics of player movement shown to influence risk of injury, do differences in these metrics exist across playing surfaces for the non-injured player population?*\n\nWith max speed, max lateral speed, and max acceleration all appearing to impact injury risk. Here we can see that Field Type does have a statistically significant impact on these metrics.\n\n*Do any of the environmental or game scenario variables influence player movement in the non-injured player population?*\n\nYes we can see there are additional factors which have a statistically significant impact on player movement injury risk factors."},{"metadata":{"_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"results_non_injured = pd.concat([results_speed, results_lateral, results_acc])\nresults_non_injured = results_non_injured[results_non_injured.index.str.startswith('Field')]\nplot_ci(results_non_injured[results_non_injured['P>|t|']<=.05], 'Statistically Significant Variables')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4. Additional Insights - Movement Fingerprint Clustering\n\nWith max speed being such an important aspect of player risk, I decided to dig into the player movement at the time when a player is traveling his max speed on a play. My thinking was that if a player is doing certain types of movement at that time then they may be a higher risk (i.e. sharp cut at high speed). To do so I normalized movement into the second just before/after max speed (touched on above). Then I used k-mean clustering to find \"fingerprints\" for these segments of player movement. \n\nNotes:\n* Each play has two \"fingerprints\" - one for the second just before max speed and one just after\n* I used a k-value of 100 for both sets of fingerprints - this was chosen based on the \"elbow method\"\n* We need to be careful to not overly inferr from the post max speed fingerprint as the injury may be showing up at that time (i.e. player abruptly stops)\n\nThen I used these fingerprints as one-hot encoded features in a similar fixed effects regression to see which ones were most/least predictive of an injury (i.e. high risk vs. low risk).\n\nWe can see that some of the movement clusters are actually statistically significant. Further research can be done to understand how these results change by injury type or for other segments of the play beyond max speed."},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":false,"trusted":false},"cell_type":"code","source":"prev_max_speed_clusters = max_speed_times[max_speed_times.PlayKey.isin(max_speed_times.query('time_norm_max_speed==-1 and x_norm_max_speed >= 0').PlayKey)].merge(cluster_prev_max_speed, \n                                                how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one').copy()\npost_max_speed_clusters = max_speed_times[max_speed_times.PlayKey.isin(max_speed_times.query('time_norm_max_speed==-1 and x_norm_max_speed >= 0').PlayKey)].merge(cluster_post_max_speed, \n                                                how='left',left_on=['PlayKey'],right_on=['PlayKey'],validate='many_to_one').copy()\n\navg_speed_cluster_prev = plays.groupby('fingerprint_prev_max_speed')['player_speed_max'].mean().reset_index()[['fingerprint_prev_max_speed','player_speed_max']]\navg_speed_cluster_post = plays.groupby('fingerprint_post_max_speed')['player_speed_max'].mean().reset_index()[['fingerprint_post_max_speed','player_speed_max']]\n\nprev_max_speed_clusters['avg_speed'] = prev_max_speed_clusters[['cluster']].merge(avg_speed_cluster_prev, how='left',left_on=['cluster'],right_on=['fingerprint_prev_max_speed'],validate='many_to_one')['player_speed_max'].values\npost_max_speed_clusters['avg_speed'] = post_max_speed_clusters[['cluster']].merge(avg_speed_cluster_post, how='left',left_on=['cluster'],right_on=['fingerprint_post_max_speed'],validate='many_to_one')['player_speed_max'].values\n\ntop_n = 5\nhigh_risk_pre = prev_max_speed_clusters[prev_max_speed_clusters['cluster'].isin(\n    results_injury_fp[results_injury_fp.index.str.startswith('fingerprint_prev')].head(top_n).index.str.replace('.*cluster','cluster'))].query('time_norm_max_speed<=1').copy()\nhigh_risk_post = post_max_speed_clusters[post_max_speed_clusters['cluster'].isin(\n    results_injury_fp[results_injury_fp.index.str.startswith('fingerprint_post')].head(top_n).index.str.replace('.*cluster','cluster'))].query('time_norm_max_speed>=-1').copy()\n\nlow_risk_pre = prev_max_speed_clusters[prev_max_speed_clusters['cluster'].isin(\n    results_injury_fp[results_injury_fp.index.str.startswith('fingerprint_prev')].tail(top_n).index.str.replace('.*cluster','cluster'))].query('time_norm_max_speed<=1').copy()\nlow_risk_post = post_max_speed_clusters[post_max_speed_clusters['cluster'].isin(\n    results_injury_fp[results_injury_fp.index.str.startswith('fingerprint_post')].tail(top_n).index.str.replace('.*cluster','cluster'))].query('time_norm_max_speed>=-1').copy()\n\nhigh_risk_pre = modify_cluster_df(high_risk_pre)\nhigh_risk_post = modify_cluster_df(high_risk_post)\nlow_risk_pre = modify_cluster_df(low_risk_pre)\nlow_risk_post = modify_cluster_df(low_risk_post)\n\nplot_ci(results_injury_fp[(~results_injury_fp.index.str.startswith('fixed'))&(results_injury_fp['P>|t|']<=.05)], 'Statistically Significant Variables')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"plot_max_speed(high_risk_pre, color='Oranges', ylim=(-10,1), title='High Risk Movement (pre-max speed)')\nplot_cluster_examples(high_risk_pre, 'fingerprint_prev_max_speed')\n\nplot_max_speed(low_risk_pre, color='Greens', ylim=(-10,1), title='Low Risk Movement (pre-max speed)')\nplot_cluster_examples(low_risk_pre, 'fingerprint_prev_max_speed')\n\nplot_max_speed(high_risk_post, color='Oranges', ylim=(-1, 10), title='High Risk Movement (post-max speed)')\nplot_cluster_examples(high_risk_post, 'fingerprint_post_max_speed')\n\nplot_max_speed(low_risk_post, color='Greens', ylim=(-1, 10), title='Low Risk Movement (post-max speed)')\nplot_cluster_examples(low_risk_post, 'fingerprint_post_max_speed')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Appendix"},{"metadata":{},"cell_type":"markdown","source":"## A.1 Injuries by categoricals"},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":false,"trusted":false},"cell_type":"code","source":"basic_bar('weather', sort=True)\nbasic_bar('workload_days_rest')\nbasic_bar('FieldType')\nbasic_bar('PositionGroup', sort=True)\nbasic_bar('PlayType', sort=True)\nbasic_bar('field_type_previous_game')\nbasic_bar('field_type_most_frequent')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## A.2 Injury movement by PlayType"},{"metadata":{"trusted":false,"_kg_hide-input":true},"cell_type":"code","source":"tracking_injured['Injury'] = tracking_injured['PositionGroup'] + ' | ' + tracking_injured['BodyPart']\nplot_plays(tracking_injured, 'Injury')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## A.3 Complete Model Results"},{"metadata":{},"cell_type":"markdown","source":"### Model Summaries"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"print(stats_injury)\nprint(stats_knee)\nprint(stats_ankle)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Player Movement"},{"metadata":{"trusted":false,"_kg_hide-input":true},"cell_type":"code","source":"plot_ci(all_filtered[all_filtered.index.str.startswith('player_')], 'Player Movement')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Playing Surface"},{"metadata":{"trusted":false,"_kg_hide-input":true},"cell_type":"code","source":"plot_ci(all_filtered[all_filtered.index.str.startswith('FieldType')], 'Playing Surface')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Field Type Changes"},{"metadata":{"trusted":false,"_kg_hide-input":true},"cell_type":"code","source":"plot_ci(all_filtered[all_filtered.index.str.startswith('field_type')], 'Field Type')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Workload"},{"metadata":{"trusted":false,"_kg_hide-input":true},"cell_type":"code","source":"plot_ci(all_filtered[all_filtered.index.str.startswith('workload')], 'Workload')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Weather"},{"metadata":{"trusted":false,"_kg_hide-input":true},"cell_type":"code","source":"plot_ci(all_filtered[all_filtered.index.str.startswith('weather')], 'Weather')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Play Context"},{"metadata":{"trusted":false,"_kg_hide-input":true},"cell_type":"code","source":"plot_ci(all_filtered[(all_filtered.index.str.startswith('play_'))|(all_filtered.index.str.startswith('PlayType'))], 'Play')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.5"}},"nbformat":4,"nbformat_minor":1}