{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nimport lightgbm as lgb\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn import metrics\nfrom sklearn.metrics import roc_auc_score\nfrom scipy import stats\nimport shap\nimport os\nimport gc\nimport sys\n\ngc.enable()\n\nsns.set(rc={'figure.figsize':(8,6)}, style = \"white\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"pt_raw = pd.read_csv(\"../input/nfl-playing-surface-analytics/PlayerTrackData.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Preprocessing dataset**\n* obtain summary data from player track dataset\n* check for nan, abnormal data\n* merge three dataset into one"},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocessing(pt_df):\n    \n    print (\"Start preprocessing\")\n    #pt1 = pt_df.drop([\"event\"], axis = 1)\n    #pt2 = pt_df.drop([\"time\", \"event\"], axis = 1)\n    \n    o_diff_arr = np.zeros(len(pt_df['o']), np.float32)\n    o_diff_arr[1:] = np.abs(pt_df['o'][:-1].to_numpy() - pt_df['o'][1:].to_numpy())    \n    pt_df['o_diff'] = o_diff_arr\n    del o_diff_arr\n\n    dir_diff_arr = np.zeros(len(pt_df['dir']), np.float32)\n    dir_diff_arr[1:] = np.abs(pt_df['dir'][:-1].to_numpy() - pt_df['dir'][1:].to_numpy())\n    pt_df['dir_diff'] = dir_diff_arr\n    del dir_diff_arr\n    gc.collect()\n    \n    print (\"Start Player Track Summary\")\n    pt_df[['PlayKey', 'dir', 'o', 's', 'o_diff', 'dir_diff']].groupby(\"PlayKey\").mean().to_csv(\"pt_mean.csv\")\n    pt_df.drop([\"time\", \"event\", \"dis\", \"x\", \"y\"], axis = 1).groupby(\"PlayKey\").min().to_csv(\"pt_min.csv\")\n    pt_df.drop([\"event\", \"dis\", \"x\", \"y\"], axis = 1).groupby(\"PlayKey\").max().to_csv(\"pt_max.csv\")\n    pt_df.drop([\"time\", \"event\", 'x', 'y', 'dis'], axis = 1).groupby(\"PlayKey\").std().to_csv(\"pt_std.csv\")\n    pt_df[[\"PlayKey\", \"dis\"]].groupby(\"PlayKey\").sum().to_csv(\"pt_sum.csv\")\n\n    del pt_df\n    gc.collect()\n    \n    final_df = pd.read_csv(\"pt_mean.csv\").merge(pd.read_csv(\"pt_max.csv\"), how = \"left\", \n                          on = \"PlayKey\", suffixes=('', '_playmax'))\n    final_df = final_df.merge(pd.read_csv(\"pt_min.csv\"), how = \"left\", \n                          on = \"PlayKey\", suffixes=('', '_playmin'))\n    final_df = final_df.merge(pd.read_csv(\"pt_std.csv\"), how = \"left\", \n                          on = \"PlayKey\", suffixes=('', '_playstd'))\n    final_df = final_df.merge(pd.read_csv(\"pt_sum.csv\"), how = \"left\", \n                          on = \"PlayKey\", suffixes=('', '_playsum'))\n    \n    print (\"Start Play List Summary\")\n    #pl_df = pd.read_csv(\"PlayList.csv\")\n    pl_df = pd.read_csv(\"../input/nfl-playing-surface-analytics/PlayList.csv\")\n    pl_new = pl_df.drop([\"PlayerKey\", \"GameID\", \"PositionGroup\", \"Weather\", \"Position\", \"PlayerDay\"], axis = 1)\n    del pl_df\n    gc.collect()\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Outdoors', \"StadiumType\"] = \"Outdoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Indoors', \"StadiumType\"] = \"Indoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Oudoor', \"StadiumType\"] = \"Outdoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Open', \"StadiumType\"] = \"Outdoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Closed Dome', \"StadiumType\"] = \"Indoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Domed, closed', \"StadiumType\"] = \"Indoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Domed', \"StadiumType\"] = \"Indoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Dome', \"StadiumType\"] = \"Indoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Retr. Roof-Closed', \"StadiumType\"] = \"Indoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Outdoor Retr Roof-Open', \"StadiumType\"] = \"Outdoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Retractable Roof', \"StadiumType\"] = \"Indoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Ourdoor', \"StadiumType\"] = \"Outdoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Indoor, Roof Closed', \"StadiumType\"] = \"Indoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Retr. Roof - Closed', \"StadiumType\"] = \"Indoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Bowl', \"StadiumType\"] = \"Outdoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Outddors', \"StadiumType\"] = \"Outdoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Retr. Roof-Open', \"StadiumType\"] = \"Outdoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Dome, closed', \"StadiumType\"] = \"Indoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Indoor, Open Roof', \"StadiumType\"] = \"Indoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Domed, Open', \"StadiumType\"] = \"Outdoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Domed, open', \"StadiumType\"] = \"Outdoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Heinz Field', \"StadiumType\"] = \"Outdoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Cloudy', \"StadiumType\"] = \"Outdoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Retr. Roof - Open', \"StadiumType\"] = \"Outdoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Retr. Roof Closed', \"StadiumType\"] = \"Indoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Outdor', \"StadiumType\"] = \"Outdoor\"\n    pl_new.loc[pl_new[\"StadiumType\"] == 'Outside', \"StadiumType\"] = \"Outdoor\"\n    #pl_new[\"Surface\"] = 1\n    #pl_new.loc[pl_new[\"FieldType\"] == \"Synthetic\", \"Surface\"] = 0\n    #pl_new = pl_new.drop([\"FieldType\"], axis = 1)\n    temp_mean = int(pl_new.loc[pl_new[\"Temperature\"]!= -999][\"Temperature\"].mean())\n    pl_new.loc[pl_new[\"Temperature\"]== -999, \"Temperature\"] = temp_mean\n    \n    pl_new = pl_new[pl_new[\"PlayType\"] != '0']\n    \n    print (\"Start Injury Data Summary\")\n    #in_df = pd.read_csv(\"InjuryRecord.csv\")\n    in_df = pd.read_csv(\"../input/nfl-playing-surface-analytics/InjuryRecord.csv\")\n    in_pk = in_df[[\"PlayKey\"]].copy()\n    in_pk.loc[:, \"Injury\"] = 1\n    \n    print (\"Merging\")\n    final_df = pl_new.merge(final_df, how = \"left\", on = \"PlayKey\").dropna()\n\n    final_df = final_df.merge(in_pk, how = \"left\", on = \"PlayKey\")\n    #final_df['Injury'].astypes('int32')\n    final_df.set_index('PlayKey', inplace = True)\n    final_df.Injury = final_df.Injury.fillna(0)\n\n    final_df = final_df.astype({'Injury': 'int32'})\n    print (\"Finish preprocessing\")\n    \n    return (final_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"injury_data = preprocessing(pt_raw)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Nature of injury**\n* EDA"},{"metadata":{"trusted":true},"cell_type":"code","source":"injury_raw = pd.read_csv(\"../input/nfl-playing-surface-analytics/InjuryRecord.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"body_injury_df = injury_raw.groupby(\"BodyPart\").count()[[\"PlayerKey\"]].reset_index().rename(columns = {\"PlayerKey\": \"Count\"})\nbodypart_plt = sns.catplot(x = \"Count\", y = \"BodyPart\", kind = \"bar\", data = body_injury_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"injury_raw[\"InjuryDuration\"] = \"1\"\ninjury_raw.loc[injury_raw[\"DM_M1\"]==1, \"InjuryDuration\"]=\"Less than 7 days\"\ninjury_raw.loc[injury_raw[\"DM_M7\"]==1, \"InjuryDuration\"]=\"7 to 27 days\"\ninjury_raw.loc[injury_raw[\"DM_M28\"]==1, \"InjuryDuration\"]=\"28 to 41 days\"\ninjury_raw.loc[injury_raw[\"DM_M42\"]==1, \"InjuryDuration\"]=\"Equal or more than 42 days\"\nduration_injury_df = injury_raw.groupby(\"InjuryDuration\").count()[[\"PlayerKey\"]].rename(columns = {\"PlayerKey\": \"Count\"}).reset_index()\nduration_plt = sns.catplot(x = \"Count\", y = \"InjuryDuration\", kind = \"bar\", order = [\"Less than 7 days\", \"7 to 27 days\", \"28 to 41 days\", \"Equal or more than 42 days\"], data = duration_injury_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"duration_body_injury_df = injury_raw.groupby([\"BodyPart\", \"InjuryDuration\"]).count()[[\"PlayerKey\"]].reset_index().rename(columns = {\"PlayerKey\": \"Count\"})\nbody_dur_plt = sns.catplot(x = \"Count\", y = \"BodyPart\", hue = \"InjuryDuration\",kind = \"bar\", hue_order = [\"Less than 7 days\", \"7 to 27 days\", \"28 to 41 days\", \"Equal or more than 42 days\"], data = duration_body_injury_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"surface_injury_df = injury_raw.groupby(\"Surface\").count()[[\"PlayerKey\"]].reset_index().rename(columns = {\"PlayerKey\": \"Count\"})\nsurface_plt = sns.catplot(x = \"Count\", y = \"Surface\", kind = \"bar\", data = surface_injury_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"surface_body_injury_df = injury_raw.groupby([\"Surface\", \"BodyPart\"]).count()[[\"PlayerKey\"]].reset_index().rename(columns = {\"PlayerKey\": \"Count\"})\nbody_sur_plt = sns.catplot(x = \"Count\", y = \"BodyPart\", hue = \"Surface\",kind = \"bar\", data = surface_body_injury_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Link injury to play and player record"},{"metadata":{"trusted":true},"cell_type":"code","source":"trueInjury_data = injury_data[injury_data['Injury'] == 1]\ntrueInjury_data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trueInjury_data.columns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Roster position and injury"},{"metadata":{"trusted":true},"cell_type":"code","source":"trueinjury_pos_df = trueInjury_data.groupby(\"RosterPosition\").count()[[\"Injury\"]].reset_index().rename(columns = {\"Injury\": \"Count\"})\nall_pos_df = injury_data.groupby(\"RosterPosition\").count()[[\"Injury\"]].reset_index().rename(columns = {\"Injury\": \"Count\"})\npos_injury_df = all_pos_df.merge(trueinjury_pos_df, on = \"RosterPosition\", how = \"left\", suffixes = ('_all', '_injury')).fillna(0)\npos_injury_df[\"injury_rate\"] = pos_injury_df[\"Count_injury\"]/pos_injury_df[\"Count_all\"]\npos_injury_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trueinjury_type_df = trueInjury_data.groupby(\"PlayType\").count()[[\"Injury\"]].reset_index().rename(columns = {\"Injury\": \"Count\"})\nall_type_df = injury_data.groupby(\"PlayType\").count()[[\"Injury\"]].reset_index().rename(columns = {\"Injury\": \"Count\"})\ntype_injury_df = all_type_df.merge(trueinjury_type_df, on = \"PlayType\", how = \"left\", suffixes = ('_all', '_injury')).fillna(0)\ntype_injury_df[\"injury_rate\"] = type_injury_df[\"Count_injury\"]/type_injury_df[\"Count_all\"]\ntype_injury_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pl_raw = pd.read_csv(\"../input/nfl-playing-surface-analytics/PlayList.csv\")\npos_injury_df = injury_raw.merge(pl_raw, how = 'left', on = 'PlayKey')[['BodyPart', 'RosterPosition', 'PlayKey']].groupby(\n    'RosterPosition').count().reset_index().rename(columns = {'PlayKey': 'Count'})\nsns.catplot(x = \"Count\", y = 'RosterPosition', kind = \"bar\", data = pos_injury_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pos_body_injury_df = injury_raw.merge(pl_raw, how = 'left', on = 'PlayKey')[['BodyPart', 'RosterPosition', 'PlayKey']].groupby(\n    ['BodyPart', 'RosterPosition']).count().reset_index().rename(columns = {'PlayKey': 'Count'})\nsns.catplot(x = \"Count\", y = 'RosterPosition', hue = 'BodyPart', kind = \"bar\", data = pos_body_injury_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"type_body_injury_df = injury_raw.merge(pl_raw, how = 'left', on = 'PlayKey')[['BodyPart', 'PlayType', 'PlayKey']].groupby(\n    ['BodyPart', 'PlayType']).count().reset_index().rename(columns = {'PlayKey': 'Count'})\nsns.catplot(x = \"Count\", y = 'PlayType', hue = 'BodyPart', kind = \"bar\", data = type_body_injury_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Cause of injury**\n* Prepare data for lgbm\n* Train and validate model\n* Interpretation"},{"metadata":{},"cell_type":"markdown","source":"**Prepare dataset ready for lgb model**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def lgb_ready(df_bf):\n    \n    obj_col = []\n    \n    n_col = df_bf.shape[1]\n    for i in range(n_col):\n\n        if (df_bf.iloc[:,i].dtypes == 'object'):\n            n_uni = df_bf.iloc[:,i].nunique()\n            l_uni = df_bf.iloc[:,i].unique()\n            print (l_uni)\n            uni_int = [k for k in range(n_uni)]\n    \n            df_bf.iloc[:,i].replace(l_uni, uni_int, inplace=True)\n            obj_col.append(i)\n        \n    return (obj_col)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"injury_obj = lgb_ready(injury_data)\ninjury_obj","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Prepare model for training**\n* Predictor, response\n* Traing set, validation set"},{"metadata":{"trusted":true},"cell_type":"code","source":"injury_X = injury_data.drop([\"Injury\"], axis = 1)\ninjury_y = injury_data[\"Injury\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_valid, y_train, y_valid = train_test_split(\n    injury_X, injury_y, test_size=0.25, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = lgb.Dataset(X_train, label = y_train, categorical_feature = injury_obj)\nvalid_dataset = lgb.Dataset(X_valid, label = y_valid, categorical_feature = injury_obj)\n\nparams = {\n    'task': 'train',\n    'boosting_type': 'gbdt',\n    'objective': 'binary',\n    'metric': 'auc',\n    'is_unbalance':True,\n    'max_bin': 255,\n    'learning_rate': 0.01,\n    'num_leaves': 31,\n    'max_depth': 12,\n    #'feature_fraction': 0.9,\n    #'bagging_fraction': 0.9,\n    #'bagging_freq': 10,\n    #'bagging_seed': 0,\n    'min_data_in_leaf': 7,\n    'num_threads': 1,\n    'random_state': 3\n}\n\nbst = lgb.train(params, train_dataset, num_boost_round = 145,\n                valid_sets=[train_dataset,valid_dataset], verbose_eval=5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Model performace**\n* Prediction accuracy\n* Confusion matrix"},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = bst.predict(X_valid).astype(int)\n#print (y_pred)\n#sum(y_pred>0.5)\nprint (sum(y_pred==y_valid)/len(y_valid))\nconfusion_matrix(y_valid, y_pred)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Model interpretation**\n* SHAP values\n* Dependency plot"},{"metadata":{"trusted":true},"cell_type":"code","source":"shap_values = shap.TreeExplainer(bst).shap_values(X_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shap.summary_plot(shap_values[1], X_valid, plot_type = \"bar\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shap.summary_plot(shap_values[1], X_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shap.dependence_plot('PlayerGame', shap_values[1], X_valid, show = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shap.dependence_plot('dir_diff', shap_values[1], X_valid, show = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shap.dependence_plot('PlayerGamePlay', shap_values[1], X_valid, show = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shap.dependence_plot('s', shap_values[1], X_valid, show = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shap.dependence_plot('dir', shap_values[1], X_valid, show = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shap.dependence_plot('Temperature', shap_values[1], X_valid, show = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shap.dependence_plot('o_diff', shap_values[1], X_valid, show = False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Movement across surfaces**\n* compare key movement metrics of healthy players across surfaces\n* conduct hypothesis test on the mean of two distributions"},{"metadata":{"trusted":true},"cell_type":"code","source":"healthy_data = injury_data[injury_data['Injury']==0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def surface_move(feature):\n    \n    natural_df = healthy_data[healthy_data['FieldType']==1][feature].to_numpy()\n    synthetic_df = healthy_data[healthy_data['FieldType']==0][feature].to_numpy()\n    \n    n_m = np.mean(natural_df)\n    s_m = np.mean(synthetic_df)\n    n_std = np.std(natural_df)\n    s_std = np.std(synthetic_df)\n    \n    plot1 = sns.kdeplot(natural_df, shade = True, label = \"Natural\", c = \"blue\")\n    plt.axvline(np.mean(natural_df), c = \"blue\")\n    sns.kdeplot(synthetic_df, shade = True, label = \"Synthetic\", c = \"darkred\")\n    plt.axvline(np.mean(synthetic_df), c = \"darkred\");\n    \n    plt.figure()\n    \n    plot2 = sns.kdeplot(natural_df, shade = True, label = \"Natural\", c = \"blue\").set(xlim=(n_m - s_m/2, n_m + s_m/2))\n    plt.axvline(np.mean(natural_df), c = \"blue\")\n    sns.kdeplot(synthetic_df, shade = True, label = \"Synthetic\", c = \"darkred\")\n    plt.axvline(np.mean(synthetic_df), c = \"darkred\");\n    \n    print (\"Natural mean:\", n_m)\n    print (\"Synthetic mean:\", s_m)\n    print (\"Natural std:\", n_std)\n    print (\"Synthetic std:\", s_std)\n    print (stats.ttest_ind(natural_df, synthetic_df, equal_var = False))\n    \n    return (plot1, plot2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"surface_move('dir_diff')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Mean direction change is significantly higher on natural surface"},{"metadata":{"trusted":true},"cell_type":"code","source":"surface_move('s')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Mean speed is significantly higher on synthetic surface"},{"metadata":{"trusted":true},"cell_type":"code","source":"surface_move('o_diff')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Mean orientation change is significantly higher on natural surface"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}