{"cells":[{"metadata":{"trusted":true,"_uuid":"5b2adf666b1b4bb95c6f28c0567d1cff7c89e2dd","_kg_hide-input":true},"cell_type":"code","source":"from IPython.display import HTML\nstyle = \"\"\"\n<style>\n    .header1 { font-family:'Arial';font-size:30px; color:Black; font-weight:800;}\n    .header2 { \n        font-family:'Arial';\n        font-size:18px; \n        color:Black; \n        font-weight:600;\n        border-bottom: 1px solid; \n        margin-bottom: 8px;\n        margin-top: 8px;\n        width: 100%;\n        \n    }\n    .header3 { font-family:'Arial';font-size:16px; color:Black; font-weight:600;}\n    .para { font-family:'Arial';font-size:14px; color:Black;}\n    .flex-columns {\n        display: flex;\n        flex-direction: row;\n        flex-wrap: wrap;\n    }\n    .flex-container {\n         padding: 20px;\n    }\n    \n    .flex-container-large {\n         padding: 20px;\n         max-width: 40%;\n    }\n    \n    .flex-container-small {\n         padding: 20px;\n         max-width: 17.5%;\n    }\n    \n    .list-items {\n        margin: 10px;\n    }\n    \n    .list-items li {\n        color: #3692CC;\n        font-weight: 500;\n    }\n</style>\n\"\"\"\nHTML(style)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f014d41efd8d43c203a619de45127b5e733175a"},"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport math\nimport numpy as np\n%matplotlib inline\n\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bd594ebb893c326afe5b18c11b6f26089d3c29bf"},"cell_type":"markdown","source":"<div class=\"header1\"> Building The Model </div>\n<div class=\"header2\">Goal: Create a model which predicts concussions based on key features</div>\n<div class=\"para\">\n    This sections builds on Part 1 which can be found in my submission. This notebook has been split due to memory restrictions. Some code blocks have been hidden for readability, click the (...) to expand them.\n </div>\n\n<div class=\"header3\">Why build an ML Model?</div>\n<div class=\"para\">\n    Building a model which accurately predict based off features researched above, would verify that the slecetd features are indeed good indicators of concussions. If we are able to succesfully classify concussions based off a set of lead / lag indicators then we can attempt to visualise the behaviour of the model and perform a sensitivity analysis to understand what particular values of each features make it most likely for concussions to occur. We can then use this criteria to select video replays which represent a general / typical concussion event.\n</div>\n\n<div class=\"header3\">Feature selection</div>\n<div class=\"para\">\nIn the above research we have gone through almost every useful feature provided in the dataset. Some additional research into data on turf type / weather conditions has been left out as it did not prove a significant factros.\n\nBased on the above feature suitability study the following features have been chosen:\n<div class=\"list-items\">\n    <li>Yard Line Distance</li>\n    <li>Score Difference</li>\n    <li>Temperature</li>\n    <li>Player Role</li>\n    <li>Velocity</li>\n    <li>Play Duration</li>\n    <li><b>Label:</b><i> Concussed</i></li>\n</div>\n\nWe will train the model to predict whether a particular player in a give play / down will be concussed.\n</div>"},{"metadata":{"trusted":true,"_uuid":"310ffa5ebb75e7eac14f70f893fb1abcb99b9b87"},"cell_type":"code","source":"from sklearn.decomposition import PCA\nfrom sklearn.svm import SVC\nfrom imblearn.over_sampling import SMOTE\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix, make_scorer, f1_score\nfrom sklearn.model_selection import GridSearchCV\nimport itertools\nimport datetime\n\ndef round_time(dt=None, round_to=60):\n    if dt == None: \n        dt = datetime.datetime.now()\n    seconds = (dt - dt.min).seconds\n    rounding = (seconds+round_to/2) // round_to * round_to\n    return dt + datetime.timedelta(0,rounding-seconds,-dt.microsecond)\n\n## Helper function from sklearn docs\ndef plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n\n    print(cm)\n\n    ax0.imshow(cm, interpolation='nearest', cmap=cmap)\n    ax0.set_title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    fmt = '.2f' if normalize else 'd'\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, format(cm[i, j], fmt),\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.tight_layout()\n    \ndef plot_2d_space(X, y,ax_):   \n    colors = ['#1F77B4', '#FF7F0E']\n    markers = ['o', 's']\n    for l, c, m in zip(np.unique(y), colors, markers):\n        ax_.scatter(\n            X[y==l, 0],\n            X[y==l, 1],\n            c=c, label=l, marker=m)\n    ax_.legend(loc='upper right', labels=['No Concussion', 'Concussion'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3708ee08d40e17556c464a4bb21b524dd119c442"},"cell_type":"code","source":"data_selected_raw = pd.read_csv('../input/nfl-selected-features/data_selected.csv') \ndata_selected = data_selected_raw.drop(columns=['Season_Year','GameKey','PlayID','GSISID'])\nfeatures = data_selected[['YardLineDist','ScoreDifference','Role','Temperature','Velocity','Play_Duration','Concussed']]\n\n# One hot encode categorical role\nrole_enc = pd.get_dummies(features['Role'])\nfeatures_w_enc_role = features.merge(right=role_enc, how='inner', left_index=True, right_index=True)\nfeatures_w_enc_role = features_w_enc_role.drop(columns=['Role'])\nfeatures_w_enc_role = features_w_enc_role.dropna()\n\n# Smote Oversampling\nsmote = SMOTE(sampling_strategy='minority')\nX_sm, y_sm = smote.fit_sample(features_w_enc_role.drop(columns=['Concussed']),features_w_enc_role['Concussed'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"6599368e58ec99f073977963b3b9f9ed83b33192","_kg_hide-input":true},"cell_type":"code","source":"plt.figure(figsize=(12,5))\nax0 = plt.subplot2grid((1,2),(0,0))\nax1 = plt.subplot2grid((1,2),(0,1))\nax0.set_title('Imbalanced dataset projection (Fig 1)')\nax1.set_title('SMOTE rebalanced dataset projection (Fig 2)')\n\n# Imbalanced original data set\nX_pca, y_pca = features_w_enc_role.drop(columns=['Concussed']),features_w_enc_role['Concussed']\npca = PCA(n_components=2)\nX_pca_fit = pca.fit_transform(X_pca)\nplot_2d_space(X_pca_fit, y_pca, ax0)\n\n# Rebalance with SMOTE\nX_pca, y_pca = X_sm, y_sm\npca = PCA(n_components=2)\nX_sm_pca = pca.fit_transform(X_sm)\nplot_2d_space(X_sm_pca, y_pca,ax1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"afb191f42d968e420763036d2b80bc718fe06665"},"cell_type":"markdown","source":"<div class=\"header2\">Class Imablance and Oversampling</div>\n<div class=\"para\">\n        There are 6 features used in this model, so its difficult to visualise the feature space! I've used a technique call principle component analysis which allows us to view a projection of the data onto 2d space for illustration purposes. In the figure 1. it is immediately obvious There is an enormous imbalance between non concussions (blue dots) and concussions (orange dots). This is a problem, as any machine learing algorithm which tries to predict on this data set can achieve excellent accuracy by simply predicting every example to be a non concussion and be right almost 100% of the time. This is an extremely common issue in many applications including medical scans and fraud detection, an generally accepted solution is to use an oversampling technique called SMOTE (Synthetic Minority Oversampling Technique). \n        </div>\n<div class=\"header2\">Fixing the imablance</div>\n<div class=\"flex-columns\">\n    <div class=\"flex-container\">\n        <div class=\"header3\">What is SMOTE</div>\n        <div class=\"para\">\n        Synthetic Minority Oversampling Technique is a complicated way of saying: for each concussion example select the 5 nearset other concussion examples, form a line in the feature space between them and randomly interpolate along the line to create new synthetic concussions. This works on the assumption that ineterpolating between similar features will yield new examples of the same class.\n        </div>\n    </div>\n    <div class=\"flex-container\">\n        <div class=\"header3\">Effect of SMOTE on Dataset</div>\n        <div class=\"para\">\n        We can visualise what SMOTE has done to our data in figure 2, by synthetically generating more examples we have now balanced the dataset. This will help to better produce better generalisation of decision boundaries found using the machine learning algorithm.\n        </div>\n    </div>\n</div>\n\n<div class=\"header2\">Linear Seperability of the Data and Support Vector Machines</div>\n<div class=\"flex-columns\">\n    <div class=\"flex-container\">\n        <div class=\"header3\">Why Support Vector Machines</div>\n        <div class=\"para\">\n                In figure 1 and 2 it becomes obvious that there is no linear decision boundary, a line we could draw in order to perfectly seperate the two classes (Non concussions and Concussions). This rules out most linear classifier except for support vector machines, which can use the kernel trick to project our finite dimensional feature vector into an infinite dimensional space in which they can be easily separated.\n         </div>\n    </div>\n    <div class=\"flex-container\">\n        <div class=\"header3\">What does this actually mean?</div>\n        <div class=\"para\"> In english this means the support vector machine will be able to linearly separate the 2 classes by engineering an inifite number of features through a clever mathematical trick called a kernel, these infinite features will be special combinations of our original 6 features and will make our data points linearly seperable in this infinite space.\n        </div>\n    </div>\n</div>\n\n<div class=\"header2\">Lets build the SVM model</div>"},{"metadata":{"trusted":false,"_uuid":"3c487d2255a227673f381eddb894a3f436f677a0"},"cell_type":"code","source":"X_train, X_ct, y_train, y_ct = train_test_split(X_sm, y_sm, test_size=0.6, random_state=12)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"5bbd53d7690bfa6afe5ff750da78dde0af9e95c2"},"cell_type":"code","source":"# Cross validation set for model selection\nX_cv, X_test, y_cv, y_test = train_test_split(X_ct, y_ct, test_size=0.75, random_state=12)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"8574f532af947cd580dafbeec2ed4c72a3d48332"},"cell_type":"code","source":"f1_scores = {}\nconfusion_matrices = {}\n\n# This may take some time on your kernel be patient :)\nfor c_try in [0.001,0.01,0.1,1,10]:\n    print(\"Using: %.4f\" % c_try)\n    clf_opt = SVC(probability=True, C=c_try)\n    clf_opt.fit(X_train, y_train)\n    \n    y_predicted = clf_opt.predict(X_cv)\n\n    model_f1_score = f1_score(y_cv, y_predicted)\n    conf_matrix = confusion_matrix(y_cv, y_predicted)\n\n    f1_scores[c_try] = [model_f1_score]\n    confusion_matrices[c_try] = [conf_matrix]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"fbb80d298143893e980ec7fd4b0ecde25101267e","_kg_hide-input":true},"cell_type":"code","source":"print(\"F1 Score C=1: %.4f\" % f1_scores[1][0])\nprint(\"F1 Score C=10: %.4f\" % f1_scores[10][0])\n\nplt.figure(figsize=(15,4))\nax1 = plt.subplot2grid((1,3), (0,0))\nax2 = plt.subplot2grid((1,3), (0,1))\nax3 = plt.subplot2grid((1,3), (0,2))\n\nc_performance = pd.DataFrame(f1_scores).transpose()\nc_performance_log = c_performance.copy()\nc_performance_log.index = np.log10(c_performance_log.index)\n\nc_performance_log.plot(ax=ax1)\nax1.set_xticks([-3,-2,-1,0,1])\nax1.set_xlabel('C Value (Log Base 10)')\nax1.set_ylabel('F1 Score')\nax1.set_title('F1 score curve for varying C')\n\ncf1 = np.array(confusion_matrices[1][0])\ncf10 = np.array(confusion_matrices[10][0])\n\nnorm_cm = cf1.astype('float') / cf1.sum(axis=1)[:, np.newaxis]\nsns.heatmap(norm_cm, ax=ax2, annot=True, fmt=\".3f\", cmap='Blues', yticklabels=['Normal', 'Concussion'], xticklabels=['Normal', 'Concussion'])\nax2.set_title('Confusion Matrix (C=1)')\n\nnorm_cm = cf10 / cf10.sum(axis=1)[:, np.newaxis]\nsns.heatmap(norm_cm, ax=ax3, annot=True, fmt=\".3f\", cmap='Blues', yticklabels=['Normal', 'Concussion'], xticklabels=['Normal', 'Concussion'])\nax3.set_title('Confusion Matrix (C=10)')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6b91bf8c378b3533900c0f412855b855086fe7ff"},"cell_type":"markdown","source":"<div class=\"header2\">Model Selection Metrics</div>\n<div class=\"para\">\n        Above we have trained 5 SVM models on a training set of 40% of our data and an additional 15% of our data as a validation set.\n        We can see from the f1 score graph which has a log scale for the C Value (x = 1 corresponds to 10 ^ 1 = 10) that a C value of 10 leads to the best model score and the curve is more or less flattening out, thus this is what we will utilise in the final model. We will talk more about the scoring mechanism and train / test set proportions below\n        </div>"},{"metadata":{"trusted":false,"_uuid":"cfd5704b9a6678d7ba2e1c3d3b8e906e605a0d90"},"cell_type":"code","source":"# Undersample all data 185674 data points is too many for under 60 features, SVC is O(n^3) and is slow on large datasets\n# Using SMOTE oversampled dataset C=1 (Optimal)\n# Note random_state has been fixed to ensure when you build my code you get the same SVM as I do.\n\nclf_init = SVC(probability=True, C=10)\nclf_init.fit(X_train, y_train)\ny_predicted = clf_init.predict(X_test)\n\nmodel_f1_score = f1_score(y_test, y_predicted)\nconf_matrix = confusion_matrix(y_test, y_predicted)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"10cc0cd812fb8e4d3c6865bedc6641a82bcac7ce","_kg_hide-input":true},"cell_type":"code","source":"X_test_distances = pd.DataFrame(clf_init.decision_function(X_test))\nX_train_distances = pd.DataFrame(clf_init.decision_function(X_train))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d9bbf42529762307a2e5882a0563c33f52daf758","_kg_hide-input":true},"cell_type":"code","source":"from sklearn.externals import joblib\njoblib.dump(clf_init, 'nfl_concussions_model_w_probs_v2.pkl', compress=9)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e49f7dc793c62acd5f2c70e144698fb32107ce9b","_kg_hide-input":true},"cell_type":"code","source":"print(\"F1 Score: %.4f\" % model_f1_score)\n\nplt.figure(figsize=(15,4))\n\nax1 = plt.subplot2grid((1,3), (0,0))\nax2 = plt.subplot2grid((1,3), (0,1))\nax3 = plt.subplot2grid((1,3), (0,2))\n\nsns.distplot(X_train_distances[y_train == 1], color='r', kde=False, ax = ax1, norm_hist=True)\nsns.distplot(X_train_distances[y_train == 0], color='b', kde=False, ax = ax1, norm_hist=True)\n\nax1.legend(['Concussion','No Concussion'])\nax1.set_xlim((-2,2))\nax1.axvline(x=1, linestyle='dashed', linewidth=0.8, c='black')\nax1.axvline(x=-1, linestyle='dashed', linewidth=0.8, c='black')\nax1.axvline(x=0, linestyle='dashed', linewidth=1, c='black')\nax1.set_title('Histogram of Projections \\n Training Separation From Decision Boundary')\n\nsns.distplot(X_test_distances[y_test == 1], color='r', kde=False, ax = ax2, norm_hist=True)\nsns.distplot(X_test_distances[y_test == 0], color='b', kde=False, ax = ax2, norm_hist=True)\n\nax2.legend(['Concussion','No Concussion'])\nax2.set_xlim((-2,2))\nax2.axvline(x=1, linestyle='dashed', linewidth=0.8, c='black')\nax2.axvline(x=-1, linestyle='dashed', linewidth=0.8, c='black')\nax2.axvline(x=0, linestyle='dashed', linewidth=1, c='black')\nax2.set_title('Histogram of Projections \\n Test Data Separation From Decision Boundary')\n\nnorm_cm = conf_matrix.astype('float') / conf_matrix.sum(axis=1)[:, np.newaxis]\nsns.heatmap(norm_cm, ax=ax3, annot=True, fmt=\".3f\", cmap='Blues', yticklabels=['Normal', 'Concussion'], xticklabels=['Normal', 'Concussion'])\n\nax1.set_ylim((0,1))\nax2.set_ylim((0,1))\nax3.set_title('Confusion Matrix')\nax3.set_xlabel('Predicted Label')\nax3.set_ylabel('Real Label')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eaeb3d4976d7eb630ace5b92d29ea7407d5929f0"},"cell_type":"markdown","source":"<div class=\"header2\">Performance Metrics</div>\n<div class=\"para\">\n        We trained the SVM on 40% of our data and used 45% for testing and held the remaining 15% for cross validation in the model selection stage, using a C parameter of 10 (This was discovered to be the optimum in testing above). The SKlearn SVC model has Big O Performance greater than quadratic, meaning that using a dataset with 180,000 data points would be computationally expensive and thus we only used 40% of the data to train the model. \n        \n        In order to evaluate the performance of our SVM we utilise histograms of projections, we interpret the centre vertical line as the decision boundary and the two vertical lines to the left and right as the margin of seperation between the two classes on which the support vectors lie (Concussed / Non Concussed). The confusion matrix quadrants show explicity the proportions of the data correctly / incorrectly labelled. The F1 Score (Top Left) is a score with scale 0 to 1 where one represents both high recall (Correctly Classifed concussions and few concussions classified as non concussion) and high precision (Correctly classified concussions and few non concussions classified as concussions).\n        </div>\n<div class=\"header2\">Details</div>\n<div class=\"flex-columns\">\n    <div class=\"flex-container\">\n        <div class=\"header3\">Historgrams of Projections</div>\n        <div class=\"para\">\n        From the histogram of projections we can see that in both training and testing the red data points (concussions) are all on the far side of the right hand margin, this is desriable as it indicates perfect classfication of concussions. We can see however that there is some leakge across the decision boundary for non concussions. By looking at the shape of the distribution we can see that preformance is consistent across test and training data, indicating that the model is generalising well to the remaining 60% of our dataset.\n        </div>\n    </div>\n    <div class=\"flex-container\">\n        <div class=\"header3\">Confusion Matrix & F1 Score</div>\n        <div class=\"para\">\n        The F1 Score is already very close to one (0.9986), we are unlikely to get much better than this. We can see from the confusion matrix that no missclasfications of concussions occured whilst only 0.3% of non concussions were incorrectly classfied.\n        </div>\n    </div>\n</div>\n<div class=\"header2\">Summary</div>\n<div class=\"para\">\n       The model appears to be working efficiently and effectively. Next step is a sensitivity analysis to our original features.\n</div>"},{"metadata":{"trusted":false,"_uuid":"5753daf4a211bc214588dadc6460273ea037c4df","_kg_hide-input":true},"cell_type":"code","source":"# Helper Function\ndef platt_generate_graphs(cols,cols_non_binary,resolution=500, show_key=False, disable_plot=False):\n    synthetic_probabilities = {}\n    \n    for col in cols:\n        if col in cols_non_binary:\n            x_min_col = x_min[col][0]\n            x_max_col = x_max[col][0]\n            x_range_col = x_max_col - x_min_col\n\n            synthetic_range = np.arange(start=x_min_col,stop=x_max_col,step=x_range_col/500)\n            X_synthetic = pd.concat([x_mean]*resolution, ignore_index=True)\n        else:\n            x_min_col = 0\n            x_max_col = 1\n            x_range_col = 1\n            synthetic_range = np.arange(start=x_min_col,stop=x_max_col+1,step=x_range_col)\n            X_synthetic = pd.concat([x_mean]*2, ignore_index=True)\n\n        X_synthetic[col] = synthetic_range\n        class_probabilities = clf_init.predict_proba(X_synthetic)\n\n        synthetic_probabilities[col] = pd.DataFrame(index=synthetic_range,data=class_probabilities)\n    \n    if not disable_plot:   \n        plt.figure(figsize=(20,190))\n        PLOT_COLUMNS = 4\n\n        mean_probability = clf.predict_proba(x_mean)[0]\n\n        for (i, key) in enumerate(synthetic_probabilities):\n            if key in cols_non_binary or show_key is False:\n                ax1 = plt.subplot2grid((len(cols), PLOT_COLUMNS), (int(i/PLOT_COLUMNS),i%PLOT_COLUMNS))\n                synthetic_probabilities[key][1].plot(ax=ax1, color='black', linewidth=1.2)\n                ax1.set_title(key)\n                ax1.axhline(y=mean_probability[1], linestyle='dashed', linewidth=0.8, color='red')\n\n    return synthetic_probabilities\n\n# Initialisation\nX_stats = pd.DataFrame(X_sm).describe()\nX_stats.columns = features_w_enc_role.drop(columns=['Concussed']).columns\nX_concussed = features_w_enc_role[features_w_enc_role['Concussed'] == 1].drop(columns=['Concussed'])\ncols = X_stats.columns\n\n# Calculate sensitivity around each concussion example\nconcussed_synth_probs = []\n\nfor i in range(X_concussed.shape[0]):\n    x_mean = pd.DataFrame(X_concussed.iloc[i,:]).transpose()\n    x_max = pd.DataFrame(X_stats.loc['max',:]).transpose()\n    x_min = pd.DataFrame(X_stats.loc['min',:]).transpose()\n\n    synth_probs = platt_generate_graphs(\n        X_stats.columns,['YardLineDist','ScoreDifference','Role','Temperature','Velocity','Play_Duration'], disable_plot=True)\n    concussed_synth_probs.append(synth_probs)\n\n# Compared to an average example (Maybe compare to avg noncussed be more meaningful)\nx_mean = pd.DataFrame(X_stats.loc['mean',:]).transpose()\nmean_probability = clf_init.predict_proba(x_mean)[0]\naxes = []\n\n# Sum over all sensitivity distributions\nsummed_synthetic_probabilities = concussed_synth_probs[0] \n\nfor i in range(1,len(concussed_synth_probs)):\n    for key in summed_synthetic_probabilities:\n        summed_synthetic_probabilities[key] = summed_synthetic_probabilities[key] + concussed_synth_probs[i][key]\n\n# Calculate an average agreement of feature sensitivity\navg_synthetic_probabilities = {}\n\nfor key in summed_synthetic_probabilities:     \n    avg_synthetic_probabilities[key] = summed_synthetic_probabilities[key] / len(concussed_synth_probs) \n    \n# Recombine all binary role features into single categorical\nbinary_role_features = [col for col in avg_synthetic_probabilities if col not in ['YardLineDist','ScoreDifference','Role','Temperature','Velocity','Play_Duration']]\nrole_recombined = {}\n\n# We calculate the relative increase / decrease in probability of concussions as this is more visual\nfor key_nb in binary_role_features:\n    role_recombined[key_nb] = [avg_synthetic_probabilities[key_nb][1][1] - avg_synthetic_probabilities[key_nb][1][0]]\n    \nrole_probabilities = pd.DataFrame(role_recombined).transpose().sort_values(by=0)\n\naxes = {}\n\n# Plot the average sensitivity\nplt.figure(figsize=(20,20))\n# plt.suptitle('SVM Model Sensitivity Analysis By Feature')\nPLOT_COLUMNS = 5\n\nfor (i, key) in enumerate(avg_synthetic_probabilities):\n    if key not in binary_role_features:\n        if len(axes) < i + 1:\n            ax1 = plt.subplot2grid((3, PLOT_COLUMNS), (int(i/(PLOT_COLUMNS-2)),2 + i%(PLOT_COLUMNS-2)))\n            axes[key] = ax1\n        else:\n            ax1 = axes[key]\n\n        avg_synthetic_probabilities[key][1].plot(ax=ax1, color='black', linewidth=1.2)\n        ax1.set_title(key)\n        ax1.axhline(y=mean_probability[1], linestyle='dashed', linewidth=0.8, color='red')\n        ax1.minorticks_on()\n    \naxr = plt.subplot2grid((3, PLOT_COLUMNS), (0,0), colspan=2, rowspan=2)\naxr.set_title('Increase in concussion probability by Role')\nrole_probabilities.plot(kind='barh', ax=axr)\n\ncritical_ranges = {\n    'YardLineDist': 0.2,\n    'ScoreDifference': 0.15,\n    'Temperature': 0.15,\n    'Velocity': 0.55,\n    'Play_Duration': 0.15,\n}\n\n# Annotations \nfor (i, key) in enumerate(avg_synthetic_probabilities):\n    if key not in binary_role_features:\n        ax_ = axes[key]\n        critical_values = avg_synthetic_probabilities[key][1] > critical_ranges[key]\n        x_values = avg_synthetic_probabilities[key][critical_values][1].index.tolist()\n        y_values = avg_synthetic_probabilities[key][critical_values][1].tolist()\n        ax_.fill_between(x=x_values,y1=y_values, color='r', alpha=0.3, interpolate=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4fc3999d54aeb6ebd42316043e2a41e31442035c"},"cell_type":"markdown","source":"<div class=\"header2\">Sensitivity Analysis</div>\n<div class=\"para\">\nThe SVM model uses the kernel trick to make predictions over a non linear decision boundary, this means the feature space in which classification occurs is extremely high dimensional (Actually infinite), thus we can't make a direct link back to the original features we used.Instead we train a logistic hypothesis function on top of the SVM which estimates the probability of concussion given the original feature set.<br><br>\n\nThe graphs above represent the probability of concussion for varying values of a given feature (all other features held at a constant value). These probabilities are evaluated locally around the data points for real concussions in order to find the critical regions for concussions in each feature (Highlighted in red).\n        </div>\n<div class=\"header2\">Feature Sensitivity</div>\n<div class=\"flex-columns\">\n    <div class=\"flex-container-large\">\n        <div class=\"header3\">Role</div>\n        <div class=\"para\">\n        Since roles are either played or not played by a given player the role probabilities of conussion have been calculated as the difference between their baseline probability and the probability of concussion when a player takes on a particular role. \n        \n        We learn from the model that in general players who play in role sof punter, punt recievers, guards, full backs and wide recievers have increased likelihoods of recieving a concussion. Playing any other role would actually reduce your liklihood of reciieving a concussion \n        </div>\n    </div>\n    <div class=\"flex-container-large\">\n        <div class=\"header3\">Yard Line Distance</div>\n        <div class=\"para\">\n        Yard Line Distance measures the distance from an endzone that a specific punt play began. According to the SVM model punt play which begins between the 19 - 35 yard range is more likely to result in a conussion compared to any other yard line distance.\n        </div>\n    </div>\n</div>\n<div class=\"flex-columns\">\n    <div class=\"flex-container-small\">\n            <div class=\"header3\">Score Difference</div>\n            <div class=\"para\">\n            Score differences between two teams in the range -14 to 15 points are more likely to involve a concussion compared to any other score range. \n        </div>\n     </div>\n    <div class=\"flex-container-small\">\n        <div class=\"header3\">Pitch Temperature</div>\n        <div class=\"para\">\n        When play temperatures range between 33F to 64F they are more likely to involve a concussion. \n        </div>\n    </div>\n    <div class=\"flex-container-small\">\n        <div class=\"header3\">Velocity</div>\n        <div class=\"para\">\n        Similarly to early analysis the SVM model suggests that players travelling at less than 1.6 m/s are more likely to recieve a concussion during play.\n        </div>\n    </div>\n    <div class=\"flex-container-small\">\n        <div class=\"header3\">Play Duration</div>\n        <div class=\"para\">\n        When a play lasts less than 41 seconds there is greater likelihood of concussion, this does seem consistent with the fact that concussions plays tend to be longer in duration than regular plays.\n        </div>\n    </div>\n</div>\n<div class=\"header2\">Summary</div>\n<div class=\"para\">\n       Given the above critical regions identified we will now proceed to examine video footage which matches the following criteria:\n       <div class=\"list-items\">\n            <li>Yard Line Distance: <b> 19 to 35 Yards</b></li>\n            <li>Score Difference:  <b> -14 to 15 Points</b></li>\n            <li>Temperature:  <b> 33 to 64 F</b></li>\n            <li>Velocity:  <b> 0 to 2.1 m/s</b></li>\n            <li>Play Duration:  <b> 0 to 41 secs</b></li>\n        </div>\n       Identfiying these key concussion incidents will help us form a rule that will aim to reduce concussions that are highly likely according to the SVM model critical feature regions.\n</div>"},{"metadata":{"trusted":false,"_uuid":"8d21c8cef8f385b0fc2fcad7b6c6bb580b02a7c9"},"cell_type":"code","source":"# Apply critical region restrictions\nX_analysis = data_selected_raw[data_selected_raw['Concussed'] == 1]\nyard_line_restrict =(X_analysis['YardLineDist'] >= 19) & (X_analysis['YardLineDist'] <= 35)\nscore_difference_restrict = (X_analysis['ScoreDifference'] >= -14) & (X_analysis['ScoreDifference'] <= 15)\ntemperature_restrict = (X_analysis['Temperature'] <= 64) & (X_analysis['Temperature'] >= 33)\nvelocity_restrict = X_analysis['Velocity'] <= 2.1\nduration_restrict = X_analysis['Play_Duration'] <= 41\n\nrestricted_concussions = X_analysis[yard_line_restrict & score_difference_restrict & temperature_restrict & velocity_restrict & duration_restrict]\n\n# Video Replay Data\nvideo_data_2016 = pd.read_csv('../input/NFL-Punt-Analytics-Competition/video_footage-injury.csv')\nvideo_data_2017 = pd.read_csv('../input/NFL-Punt-Analytics-Competition/video_footage-control.csv')\n\nvideo_data_2016.columns = video_data_2017.columns\nvideo_data = video_data_2016.append(video_data_2017)\n\n# Player Number\nplayer_punt_data = pd.read_csv('../input/NFL-Punt-Analytics-Competition/player_punt_data.csv')\n# Some players have held more than one number / position\nplayer_punt_data_dedupl = player_punt_data.groupby(by='GSISID').agg(' '.join)\nrestricted_concussions_number = restricted_concussions.merge(right=player_punt_data_dedupl, on='GSISID', how='left')\n\n# Video - Selected Data\nvideo_data_restricted = video_data.merge(right=restricted_concussions_number, how='inner', left_on=['season','gamekey','playid'], right_on=['Season_Year','GameKey','PlayID'])\n\n#Video Review\nvideo_review = pd.read_csv('../input/NFL-Punt-Analytics-Competition/video_review.csv')\nvideo_review_restricted = video_review.merge(right=video_data_restricted, how='inner', on=['Season_Year','GameKey','PlayID','GSISID'])\nvideo_review_restricted = video_review_restricted[[c for c in video_review_restricted.columns if c[-2:] != '_y']]\nvideo_review_restricted.columns = video_review_restricted.columns.str.replace('_x', '')\nvideo_review_restricted['Primary_Partner_GSISID'] = video_review_restricted['Primary_Partner_GSISID'].astype('int64')\n\n# Merging Concussion Partner Data\nvideo_review_restricted_partner = video_review_restricted.merge(right=player_punt_data_dedupl, left_on='Primary_Partner_GSISID', right_on='GSISID', how='left', suffixes=('','_partner'))\nvideo_review_restricted_partner = video_review_restricted_partner.merge(right=data_selected_raw, left_on=['Season_Year','GameKey','PlayID','Primary_Partner_GSISID'], right_on=['Season_Year','GameKey','PlayID','GSISID'], how='left', suffixes=('','_partner'))\nvideo_review_restricted_partner[['Season_Year','GameKey','PlayID','GSISID','Number','Number_partner','Primary_Impact_Type','YardLineDist','ScoreDifference','Temperature','Velocity','Velocity_partner','Player_Activity_Derived','Primary_Partner_GSISID','Preview Link','Play_Duration']]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"28322cfe84c439e982a685411f64d3b6a8e99e51"},"cell_type":"markdown","source":"### Game Previews to Analyse\n<a href='https://nfl-vod.cdn.anvato.net/league/5691/18/11/25/284954/284954_75F12432BA90408C92660A696C1A12C8_181125_284954_huber_punt_3200.mp4'>Preview 1: GKey: 231 Play: 1976 Number: 81 (Season 2016) (Red Team) </a><br>\n<a href='http://a.video.nfl.com//films/vodzilla/153247/Punt_by_Tress_Way-QsI21aYF-20181119_160141260_5000k.mp4'>Preview 2: GKey: 280 Play: 2918 Number: 87 (Season 2016) </a><br>\n<a href='http://a.video.nfl.com//films/vodzilla/153272/Haack_42_yard_punt-iP6aZSRU-20181119_165050694_5000k.mp4'>Preview 3: GKey: 448 Play: 2792 Number: 33 (Season 2017) </a><br>\n<a href='http://a.video.nfl.com//films/vodzilla/153291/Palardy_53_yard_punt-XTESVMq9-20181119_170509550_5000k.mp4'>Preview 1: GKey: 567 Play: 1407 Number: 38 / 32 (Season 2017) </a><br>"},{"metadata":{"trusted":false,"_uuid":"7fb3739fa8c4e8ba3ff61410f2a06725934f8d43","_kg_hide-input":true},"cell_type":"code","source":"video_review_restricted_partner","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"ad2f0bf8b5ca4512acd2fbf2ae5ccd310e33997d","_kg_hide-input":true},"cell_type":"code","source":"video_review_restricted_partner['YardLineDist_Concussion'] = [21,15, 41, 14]\nvideo_review_restricted_partner['Play_Duration_Concussion'] = [13,2, 12, 12]\nvideo_review_restricted_partner['Play_Duration'][0] = 14\nvideo_review_restricted_partner['Play_Duration'][1] = 10\nvideo_review_restricted_partner['Play_Duration'][2] = 13\nvideo_review_restricted_partner['Play_Duration'][3] = 12","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d1c69a35b026c6da6240884a5c95db425a2e9f1d","_kg_hide-input":true},"cell_type":"code","source":"axes = {}\n\n# Plot the average sensitivity\nplt.figure(figsize=(20,16))\n# plt.suptitle('SVM Model Sensitivity Analysis By Feature')\nPLOT_COLUMNS = 7\n\nrows = video_review_restricted_partner[['GSISID','PlayID','Season_Year','GameKey']]\nVIDEOS_TO_REPLAY =  rows.shape[0]\n\nfor k in range(VIDEOS_TO_REPLAY):\n    GSISID_v,PLAYID_v,SEASON_v,GAMEKEY_v = rows.iloc[k,:].tolist()\n    \n    GSISID = video_review_restricted_partner['GSISID'] == GSISID_v\n    PLAYID = video_review_restricted_partner['PlayID'] == PLAYID_v\n    SEASON = video_review_restricted_partner['Season_Year'] == SEASON_v\n    GAMEKEY = video_review_restricted_partner['GameKey'] == GAMEKEY_v\n    selected_play = video_review_restricted_partner[GSISID & PLAYID & SEASON & GAMEKEY]\n\n    for (i, key) in enumerate(avg_synthetic_probabilities):\n        if key not in binary_role_features:\n            if '%d_%s' % (k,key) not in axes:\n                ax1 = plt.subplot2grid((VIDEOS_TO_REPLAY*3, PLOT_COLUMNS), (k*2 + int(i/(PLOT_COLUMNS-2)),2 + i%(PLOT_COLUMNS-2)), rowspan=2)\n                axes['%d_%s' % (k,key)] = ax1\n            else:\n                ax1 = axes['%d_%s' % (k,key)]\n\n            avg_synthetic_probabilities[key][1].plot(ax=ax1, color='grey', linewidth='1.2')\n            ax1.set_title(key)\n    #       ax1.axhline(y=mean_probability[1], linestyle='dashed', linewidth='0.8', color='red')\n            ax1.minorticks_on()\n    \n    critical_ranges = {\n        'YardLineDist': 0.18,\n        'ScoreDifference': 0.15,\n        'Temperature': 0.15,\n        'Velocity': 0.50,\n        'Play_Duration': 0.122,\n    }\n\n    axr = plt.subplot2grid((VIDEOS_TO_REPLAY*3, PLOT_COLUMNS), (k*2,0), colspan=2, rowspan=2)\n    \n\n    axr.set_title('SEASON: %d GAME KEY: %d PLAY ID: %d\\n PLAYER ID: %d Role:' %(SEASON_v,GAMEKEY_v,PLAYID_v, GSISID_v))\n\n    filtered_roles = role_probabilities[role_probabilities[0] >= 0]\n    colors = np.array(['k']*filtered_roles.shape[0])\n    colors[filtered_roles.index == selected_play['Role'].values[0]] = 'c'\n    filtered_roles.plot(kind='barh',colors=''.join(colors), ax=axr)\n\n    # Annotations \n    for (i, key) in enumerate(avg_synthetic_probabilities):\n        if key not in binary_role_features:\n            ax_ = axes['%d_%s' % (k,key)]\n            critical_values = avg_synthetic_probabilities[key][1] > critical_ranges[key]\n            x_values = avg_synthetic_probabilities[key][critical_values][1].index.tolist()\n            y_values = avg_synthetic_probabilities[key][critical_values][1].tolist()\n            \n            ax_.fill_between(x=x_values,y1=y_values, color='r', alpha=0.3, interpolate=True)\n            arrow_value = selected_play[key]\n            ax_.annotate('',xy=(arrow_value, 0), xytext=(arrow_value, np.max(y_values)+0.05), arrowprops=dict(headwidth=5.5,width=0.8,facecolor='black'))\n            ax_.set_ylim((0,np.max(y_values)+0.05))\n\n            if key == 'Velocity':\n                arrow_value_partner = selected_play[\"%s_partner\" % key]\n                \n                # Deals with NANS\n                if arrow_value_partner.values[0] > -1:\n                    ax_.annotate('',xy=(arrow_value_partner, 0), xytext=(arrow_value_partner, np.max(y_values)+0.05), arrowprops=dict(headwidth=5.5,width=0.8,color='red'))\n            \n            if key == 'YardLineDist':\n                arrow_value_partner = selected_play[\"YardLineDist_Concussion\"]\n                \n                # Deals with NANS\n                if arrow_value_partner.values[0] > -1:\n                    ax_.annotate('',xy=(arrow_value_partner, 0), xytext=(arrow_value_partner, np.max(y_values)+0.05), arrowprops=dict(headwidth=5.5,width=0.8,color='blue'))\n            \n            if key == 'Play_Duration':\n                arrow_value_concussion = selected_play[\"%s_Concussion\" % key]\n                ax_.annotate('',xy=(arrow_value_concussion, 0), xytext=(arrow_value_concussion, np.max(y_values)+0.05), arrowprops=dict(headwidth=5.5,width=0.8,color='green'))\n            \nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9b5cd8499e7341b22c5bb6723b505db25764fa5c"},"cell_type":"markdown","source":"<div class=\"header2\">General Observations</div>\n<div class=\"para\">\n<div class=\"list-items\">\n<li>In all incidents punt reciever had possesion of the ball and was advancing towards opponents endzone in 3 out of 4 cases</li>\n<li>Punts were kicked at almost the same yard line distance and all concussions occurred within the punt recieving team's half of the field</li>\n<li>Score differences tended to exceed +-10 points</li>\n<li>All concussions involved one player with a high average velocity and the other at almost stationary average velocity</li>\n<li>3 out of 4 concussions involved a helmet to helmet collision</li>\n</div>\n</div>\n<div class=\"header2\">Holes in NGS</div>\n<div class=\"para\">\nI've attempted to obtain more in depth information in terms of the actual velocities of players at the point of impact and their precise yard distances, but there are large holes in the NGS data, thus some data is missing or is approximated using average estimators. Evidence from the videos may not perfectly coincide (but has been manually corrected where possible) but in general approximations have been made where appropriate.\n</div>\n\n<div class=\"header2\">Section 2 Article 7  PLAYERS IN A DEFENCELESS POSTURE of NFL Rules</div>\n<div class=\"para\">\nThis section of the rulebook states: “It is a foul if a player initiates unnecessary contact against a player who is in a defenceless posture”.<br><br>\nThis definition currently includes:\n<div class=\"list-items\">\n<li>Act of Throwing Pass</li>\n<li>Running pass route</li>\n<li>Attempting a catch</li>\n<li>Intended receiver following interception</li>\n<li>Tackled stopped runner</li>\n<li>PR Attempting Field Kick</li>\n<li>…</li>\n</div>\nMy proposition is to extend this definition to cover the rule and condition stated below.\n</div>\n\n<div class=\"header2\">Rule</div>\n<div class=\"para\">\n<div class=\"list-items\">\n<li>Any player moving slowly relative to his opponent should be considered defenceless and must not be contacted above the shoulder.</li>\n</div>\n</div>\n<div class=\"header2\">Condition</div>\n<div class=\"para\">\n<div class=\"list-items\">\n<li>This rule applies if and only if the punt play starts between the 20 - 30 yard lines and  the player is not capable of avoiding or\nwarding off the impending contact of an opponent. </li>\n</div>\n</div>\n<div class=\"header2\">Effect</div>\n<div class=\"para\">\n<div class=\"list-items\">\n<li>Reduces the risk of high velocity helmet to helmet contacts with slow moving players who may not see / be able to avoid the oncoming tackle. Thus reducing the risk dangerous tackles which could lead to concussions with high probability</li>\n</div>\n</div>\n<div class=\"header2\">Note on Critical Regions</div>\n<div class=\"para\">\n<div class=\"list-items\">\nWe can see the critical regions we have selected align well with the concussions that occurred in 2016 / 2017, as a significant proportion of the 37 concussions satisfy at least four of the critical regions, as in the graph below.\n</div>\n</div>\n"},{"metadata":{"trusted":false,"_uuid":"639b54c999226deeaeea800200e96accc860df46","_kg_hide-input":true},"cell_type":"code","source":"X_analysis['YLD_Satisfy'] = yard_line_restrict.replace(to_replace = {True: 1, False: 0})\nX_analysis['Score_Satisfy'] = score_difference_restrict.replace(to_replace = {True: 1, False: 0})\nX_analysis['Temp_Satisfy'] = temperature_restrict.replace(to_replace = {True: 1, False: 0})\nX_analysis['Velocity_Satisfy'] = velocity_restrict.replace(to_replace = {True: 1, False: 0})\nX_analysis['Duration_Satisfy'] = duration_restrict.replace(to_replace = {True: 1, False: 0})\nX_analysis['Satisfy_Count'] = X_analysis['YLD_Satisfy'] + X_analysis['Score_Satisfy'] + X_analysis['Temp_Satisfy'] + X_analysis['Velocity_Satisfy'] + X_analysis['Duration_Satisfy'] \n\nax = X_analysis.groupby(by=['Satisfy_Count']).count()['GSISID'].plot(kind='barh', title='Frequency of the number of critical regions satisfied by concussion examples')\nax.set_ylabel('Number of critical regions satsified')\nax.set_xlabel('Concussion Examples')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ed47cb2666248841f77d2239dac9fc69e6f4f4d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"431c3874e62495463afa4490e4727ee4627e81ac"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}