{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from __future__ import print_function\nfrom ipywidgets import interact, interactive, fixed, interact_manual\nimport ipywidgets as widgets\n\nimport matplotlib.pyplot as plt\n# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:23:46.022810Z","iopub.execute_input":"2023-01-08T14:23:46.023315Z","iopub.status.idle":"2023-01-08T14:23:46.051874Z","shell.execute_reply.started":"2023-01-08T14:23:46.023277Z","shell.execute_reply":"2023-01-08T14:23:46.050674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def box_var(data,field,new_field,low,high):\n    data[new_field] = np.where(data[field] < low, low, data[field])\n    data[new_field] = np.where(data[new_field] > high, high, data[new_field])\n    \nimport xgboost as xgb\nimport shap\n\ndef easy_gbm(train,test,target,bmf,max_depth,base_score,num_round,objective = 'binary:logistic'):\n    params = {\n        'objective': objective,\n        'learning_rate': bmf,\n        'verbosity': 1,\n        'max_depth': max_depth,\n        'base_score': base_score\n    }\n\n    dtrain = xgb.DMatrix(train.drop([target],axis = 1), label=train[target])\n    dtest = xgb.DMatrix(test.drop([target],axis = 1), label=test[target])\n\n    global model_xgb\n    model_xgb = xgb.train(params, dtrain, num_round)\n\n    global out_train\n    out_train = train.copy()\n    out_train['pred'] = model_xgb.predict(dtrain)\n    \n    global out_test\n    out_test = test.copy()\n    out_test['pred'] = model_xgb.predict(dtest)\n    \ndef lift_chart(test_data, act, pred, bins):\n    test_data['records'] = 1\n    test_data['decile'] = (round(test_data.sort_values(by = 'pred')['records'].cumsum()/test_data.shape[0],2)*bins).apply(np.floor)\n    test_data['decile'] = np.where(test_data['decile'] + 1 > bins ,bins,test_data['decile'] + 1)\n    x = test_data.groupby(['decile'], dropna = False).agg({'records': 'sum', act: 'mean', pred: 'mean'}).reset_index()\n    \n    dfg = x\n    fig, ax = plt.subplots(figsize=(12,6))\n    ax2  = ax.twinx()\n    \n    y_min = np.where(dfg[act].min() < dfg[pred].min(),dfg[act].min(),dfg[pred].min())*.95\n    y_max = np.where(dfg[act].max() > dfg[pred].max(),dfg[act].max(),dfg[pred].max())*1.05\n    ax2.set_ylim(y_min,y_max)\n    \n    dfg['records'].plot.bar(stacked=False, ax=ax, alpha=0.6)\n    dfg[act].plot(kind='line', ax=ax2, marker='o', linewidth = 0, legend='act')\n    dfg[pred].plot(kind='line', ax=ax2, marker='o', legend='pred')\n    plt.show()\n    print(x)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:24:03.361140Z","iopub.execute_input":"2023-01-08T14:24:03.361779Z","iopub.status.idle":"2023-01-08T14:24:03.381497Z","shell.execute_reply.started":"2023-01-08T14:24:03.361708Z","shell.execute_reply":"2023-01-08T14:24:03.379668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# MAD 08/12/2022\n## pff data\n\ngames = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2023/games.csv', engine = 'python')\n\npffScoutingData = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2023/pffScoutingData.csv', engine = 'python')\n\nplays = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2023/plays.csv', engine = 'python')\nplays['completion_pct'] = np.where(plays['passResult'] == 'C',True ,False)\nplays['interception_pct'] = np.where(plays['passResult'] == 'IN',True ,False)\nplays['sack_pct'] = np.where(plays['passResult'] == 'S',True ,False)\nplays['scramble_pct'] = np.where(plays['passResult'] == 'R',True ,False)\n\nplayers = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2023/players.csv', engine = 'python')\n\np_g = plays.merge(games, on = ['gameId']) #plays_merge_games\npff_pg = pffScoutingData.merge(p_g, on = ['gameId','playId'])\npff_pg_plyrs = pff_pg.merge(players, on = 'nflId')","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:25:39.405759Z","iopub.execute_input":"2023-01-08T14:25:39.406420Z","iopub.status.idle":"2023-01-08T14:25:43.068010Z","shell.execute_reply.started":"2023-01-08T14:25:39.406375Z","shell.execute_reply":"2023-01-08T14:25:43.066829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## train and val sets\n\npff_df = pff_pg_plyrs[(['gameId', 'playId', 'nflId', 'officialPosition', 'pff_positionLinedUp',\n                        'completion_pct','interception_pct','sack_pct','scramble_pct',\n                        'playResult'])].copy()\n\npff_df['playResult'] = pff_df['playResult'].astype('int8')\n\n######\n\npath = '/kaggle/input/nfl-big-data-bowl-2023/'\nweeks = [i for i in range(9) if i != 0]\n\ndata = pd.DataFrame()\nfor week in weeks:\n    a = pd.read_csv(path + 'week' + str(week) + '.csv', engine = 'python')\n    \n    if week <= 6:\n        a['train'] = True\n    else:\n        a['train'] = False\n    \n    keys = ['gameId', 'playId', 'nflId']\n    b = a.merge(pff_df, on = keys, how = 'left')\n    \n    if a.shape[0] != b.shape[0]: # test left join quality, if passes continue with inner\n        print('error')\n        break\n    else:\n        b = a.merge(pff_df, on = keys)\n    \n    data = pd.concat([data,b], ignore_index = True)\n\ndata = data.reset_index().drop(columns = 'index')\n\ndata.groupby(['train'], dropna = False).agg({'gameId': 'count'}).reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:26:05.828620Z","iopub.execute_input":"2023-01-08T14:26:05.829040Z","iopub.status.idle":"2023-01-08T14:28:28.718568Z","shell.execute_reply.started":"2023-01-08T14:26:05.829008Z","shell.execute_reply":"2023-01-08T14:28:28.717121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# QB only; action time results\nplays_pass = data.loc[data['officialPosition'] == 'QB'].copy()\n\n# filter to events which are either ball snap or pass plays\nplays_pass = plays_pass.loc[plays_pass['event'].isin(['ball_snap','pass_forward','lateral','fumble','fumble_offense_recovered','qb_sack','qb_strip_sack','run'])]\n\n# need to change this to track the maximum time the ball is held by the QB instead of when a forward pass actually occurs\n\nkeys2 = ['gameId','playId']\nbs = plays_pass.loc[plays_pass['event'] == 'ball_snap'][(keys2)].copy().drop_duplicates() # some duplicate id's - reviewed and insignificant\nnon_bs = plays_pass.loc[plays_pass['event'] != 'ball_snap'][(keys2)].copy().drop_duplicates() # some duplicate id's - reviewed and insignificant\n\nplays_pass = plays_pass.merge(bs, on = keys2).merge(non_bs, on = keys2)\n\nfacts = ['completion_pct','interception_pct','playResult']\n\nbs2 = plays_pass.loc[plays_pass['event'] == 'ball_snap'][(keys2 + ['frameId'] + facts)].drop_duplicates()\nbs2.rename(columns = {'frameId': 'ball_snap_frame'}, inplace = True)\n\nnon_bs2 = plays_pass.loc[plays_pass['event'] != 'ball_snap'][(keys2 + ['frameId'])]\nnon_bs2.rename(columns = {'frameId': 'action_end_frame'}, inplace = True)\n\nnon_bs3 = non_bs2.groupby(keys2).agg({'action_end_frame': 'min'}).reset_index()\n\npf3 = non_bs3.merge(bs2, on = keys2)\nprint(pf3.shape[0] - non_bs3.shape[0])\n\npf3['action_time'] = (pf3['action_end_frame'] - pf3['ball_snap_frame'])/10\n\nsacks = data.groupby(keys2).agg({'sack_pct': 'max'}).reset_index()\nscrambles = data.groupby(keys2).agg({'scramble_pct': 'max'}).reset_index()\n\npf4 = pf3.merge(sacks, on = keys2).merge(scrambles, on = keys2)\n\nprint(pf4.shape[0] - pf3.shape[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:29:09.706959Z","iopub.execute_input":"2023-01-08T14:29:09.707978Z","iopub.status.idle":"2023-01-08T14:29:11.436649Z","shell.execute_reply.started":"2023-01-08T14:29:09.707935Z","shell.execute_reply":"2023-01-08T14:29:11.435206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"box_var(pf4,'action_time','action_time',2,4)\n\n# obtain distance to QB for all relevant defensive positions at all frames \npos_qb = data.loc[data['officialPosition'] == 'QB'][(['gameId','playId','frameId','x','y'])].copy()\npos_qb.rename(columns = {'x': 'qb_x', 'y': 'qb_y'}, inplace = True)\n\npos_defense = ['DE','DT','FS','ILB','MLB','NT','OLB','SS','CB','LB','DB']\ndist_qb0 = data.loc[data['officialPosition'].isin(pos_defense)].copy()\n\nkeys3 = ['gameId','playId','frameId']\nkeys4 = keys3 + ['nflId']\n\npos_qb1 = pos_qb.groupby(keys3).agg({'qb_x': 'mean', 'qb_y': 'mean'}).reset_index()\ndist_qb1 = dist_qb0.groupby(keys4).agg({'x': 'mean', 'y': 'mean'}).reset_index()\n\ndist_qb2 = dist_qb1.merge(pos_qb1, on = keys3)\n\ndist_qb2['dist_qb'] = ((dist_qb2['qb_x'] - dist_qb2['x'])**2 + (dist_qb2['qb_y'] - dist_qb2['y'])**2)**0.5","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:29:46.349276Z","iopub.execute_input":"2023-01-08T14:29:46.349738Z","iopub.status.idle":"2023-01-08T14:29:52.324563Z","shell.execute_reply.started":"2023-01-08T14:29:46.349700Z","shell.execute_reply":"2023-01-08T14:29:52.323202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dist_qb3 = dist_qb2.merge(pf4[(['gameId','playId','ball_snap_frame','action_end_frame'])], on = ['gameId','playId'])\n\ndist_qb4 = dist_qb3.loc[dist_qb3['frameId'] >= dist_qb3['ball_snap_frame']]\ndist_qb4 = dist_qb4.loc[dist_qb4['frameId'] <= dist_qb4['action_end_frame']]\n\ndel dist_qb4['action_end_frame']\n\nkeys3 = ['gameId','playId','nflId']\ndist_qb_plyr_min = dist_qb4.groupby(keys3).agg({'dist_qb': 'min'}).reset_index()\n\ndist_qb_plyr_min['prox_rank'] = dist_qb_plyr_min.groupby(['gameId','playId'])['dist_qb'].rank()\n\nprox1 = dist_qb_plyr_min.loc[dist_qb_plyr_min['prox_rank'] == 1][(['gameId','playId','dist_qb'])].copy()\nprox1.rename(columns = {'dist_qb': 'prox1'}, inplace = True)\n\nprox2 = dist_qb_plyr_min.loc[dist_qb_plyr_min['prox_rank'] == 2][(['gameId','playId','dist_qb'])].copy()\nprox2.rename(columns = {'dist_qb': 'prox2'}, inplace = True)\n\nprox3 = dist_qb_plyr_min.loc[dist_qb_plyr_min['prox_rank'] == 3][(['gameId','playId','dist_qb'])].copy()\nprox3.rename(columns = {'dist_qb': 'prox3'}, inplace = True)\n\nkeys = ['gameId','playId']\n\npf5 = pf4.merge(prox1, on = keys).merge(prox2, on = keys).merge(prox3, on = keys)\nprint(pf5.shape[0] - pf4.shape[0])\n\nx = data[(['gameId','train'])].drop_duplicates()\n\npf5 = pf5.merge(x, on = ['gameId'])","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:30:43.219795Z","iopub.execute_input":"2023-01-08T14:30:43.220238Z","iopub.status.idle":"2023-01-08T14:30:45.087223Z","shell.execute_reply.started":"2023-01-08T14:30:43.220201Z","shell.execute_reply":"2023-01-08T14:30:45.085722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Game level stats\n\npf5['prox1_LTE_20'] = np.where(pf5['prox1'] <= 2.0, True, False)\npf5['prox1_LTE_15'] = np.where(pf5['prox1'] <= 1.5, True, False)\npf5['prox1_LTE_10'] = np.where(pf5['prox1'] <= 1.0, True, False)\npf5['prox1_LTE_05'] = np.where(pf5['prox1'] <= 0.5, True, False)\n\npf5['prox2_LTE_20'] = np.where(pf5['prox2'] <= 2.0, True, False)\npf5['prox2_LTE_15'] = np.where(pf5['prox2'] <= 1.5, True, False)\npf5['prox2_LTE_10'] = np.where(pf5['prox2'] <= 1.0, True, False)\npf5['prox2_LTE_05'] = np.where(pf5['prox2'] <= 0.5, True, False)\n\npf5['prox3_LTE_20'] = np.where(pf5['prox3'] <= 2.0, True, False)\npf5['prox3_LTE_15'] = np.where(pf5['prox3'] <= 1.5, True, False)\npf5['prox3_LTE_10'] = np.where(pf5['prox3'] <= 1.0, True, False)\npf5['prox3_LTE_05'] = np.where(pf5['prox3'] <= 0.5, True, False)\n\nfacts = ['completion_pct','interception_pct','playResult','action_time','sack_pct','scramble_pct',\n         'prox1_LTE_20','prox1_LTE_15','prox1_LTE_10','prox1_LTE_05',\n         'prox2_LTE_20','prox2_LTE_15','prox2_LTE_10','prox2_LTE_05',\n         'prox3_LTE_20','prox3_LTE_15','prox3_LTE_10','prox3_LTE_05'\n        ]\n\nagg_dict = {f: 'mean' for f in facts}\n    \ngame_stats_train = pf5.loc[pf5['train'] == True].groupby(['gameId']).agg(agg_dict).reset_index()\ngame_stats_val = pf5.loc[pf5['train'] == False].groupby(['gameId']).agg(agg_dict).reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:31:05.812411Z","iopub.execute_input":"2023-01-08T14:31:05.812817Z","iopub.status.idle":"2023-01-08T14:31:05.855225Z","shell.execute_reply.started":"2023-01-08T14:31:05.812785Z","shell.execute_reply":"2023-01-08T14:31:05.853970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Game Level Models","metadata":{}},{"cell_type":"code","source":"target = 'sack_pct'\nfeatures = ['action_time',\n 'prox1_LTE_20', 'prox1_LTE_15', 'prox1_LTE_10', 'prox1_LTE_05',\n 'prox2_LTE_20', 'prox2_LTE_15', 'prox2_LTE_10', 'prox2_LTE_05',\n 'prox3_LTE_20', 'prox3_LTE_15', 'prox3_LTE_10', 'prox3_LTE_05']\n\ntrain = game_stats_train[(features + [target])].copy()\ntest = game_stats_val[(features + [target])].copy()\n\nbase_score = train[target].mean()\n\neasy_gbm(train,test, target,.05,6,base_score,50)\n\nlift_chart(out_test, target, 'pred', 10)\n\nexplainer = shap.Explainer(model_xgb)\nshap_values = explainer(train.drop([target],axis = 1))\nshap.plots.beeswarm(shap_values, max_display = 50)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:31:46.782384Z","iopub.execute_input":"2023-01-08T14:31:46.782821Z","iopub.status.idle":"2023-01-08T14:31:47.972879Z","shell.execute_reply.started":"2023-01-08T14:31:46.782784Z","shell.execute_reply":"2023-01-08T14:31:47.968648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**play level models with the proper holdout process**","metadata":{}},{"cell_type":"code","source":"plays_small = plays[(['gameId','playId','down','yardsToGo','defendersInBox',\n                      'possessionTeam','yardlineSide','yardlineNumber',\n                      'preSnapHomeScore','preSnapVisitorScore',\n                      'pff_playAction',\n                      'offenseFormation','pff_passCoverage','pff_passCoverageType'])].copy()\n\ngames_small = games[(['gameId','homeTeamAbbr'])].copy()\n\nplays_small2 = plays_small.merge(games_small, on = ['gameId'])\nprint(plays_small2.shape[0] - plays_small.shape[0])\n\nplays_small2['zone_defense'] = np.where(plays_small2['pff_passCoverageType'] == 'Zone', True, False)\ndel plays_small2['pff_passCoverageType']\n\nplays_small2['o_form_shotgun'] = np.where(plays_small2['offenseFormation'] == 'SHOTGUN', True, False)\nplays_small2['o_form_empty_set'] = np.where(plays_small2['offenseFormation'] == 'EMPTY', True, False)\ndel plays_small2['offenseFormation']\n\ndel plays_small2['pff_passCoverage'] # for now until I can codify this\n\nplays_small2['pff_playAction'] = np.where(plays_small2['pff_playAction'] == 1, True, False).astype('bool')\n\nbox_var(plays_small2, 'down', 'down', 1, 4)\nbox_var(plays_small2, 'yardsToGo', 'yardsToGo',1, 10)\nbox_var(plays_small2, 'defendersInBox','defendersInBox', 3, 8)\n\nplays_small2['yds_end_zone'] = np.where(plays_small2['possessionTeam'] == plays_small2['yardlineSide'], #own side\n                                        100 - plays_small2['yardlineNumber'],\n                                        plays_small2['yardlineNumber']\n                                       )\ndel plays_small2['yardlineSide']\ndel plays_small2['yardlineNumber']\n\nplays_small2['score_diff'] = np.where(plays_small2['possessionTeam'] == plays_small2['homeTeamAbbr'], #home team possession\n                                        plays_small2['preSnapHomeScore'] - plays_small2['preSnapVisitorScore'],\n                                        plays_small2['preSnapVisitorScore'] - plays_small2['preSnapHomeScore']\n                                       )\n\ndel plays_small2['possessionTeam']\ndel plays_small2['homeTeamAbbr']\ndel plays_small2['preSnapHomeScore']\ndel plays_small2['preSnapVisitorScore']\n\npf6 = pf5.merge(plays_small2, on = ['gameId','playId'])\n\nprint(pf6.shape[0] - pf5.shape[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:33:12.693613Z","iopub.execute_input":"2023-01-08T14:33:12.694080Z","iopub.status.idle":"2023-01-08T14:33:12.745052Z","shell.execute_reply.started":"2023-01-08T14:33:12.694042Z","shell.execute_reply":"2023-01-08T14:33:12.743684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = 'completion_pct'\nfeatures = ['action_time',\n            'prox1_LTE_20', 'prox1_LTE_15', 'prox1_LTE_10', 'prox1_LTE_05',\n            'prox2_LTE_20', 'prox2_LTE_15', 'prox2_LTE_10', 'prox2_LTE_05',\n            'prox3_LTE_20', 'prox3_LTE_15', 'prox3_LTE_10', 'prox3_LTE_05',\n            'down', 'yardsToGo', 'defendersInBox', 'pff_playAction', 'zone_defense', 'o_form_shotgun', 'o_form_empty_set', 'yds_end_zone', 'score_diff'\n           ]\ntrain = pf6.loc[pf6['train'] == True][(features + [target])].copy()\ntest = pf6.loc[pf6['train'] == False][(features + [target])].copy()\n\nbase_score = train[target].mean()\n\neasy_gbm(train,test,target,.05,6,base_score,50)\n\nlift_chart(out_test, target, 'pred', 10)\n\nexplainer = shap.Explainer(model_xgb)\nshap_values = explainer(train.drop([target],axis = 1))\nshap.plots.beeswarm(shap_values, max_display = 50)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:33:33.337809Z","iopub.execute_input":"2023-01-08T14:33:33.338345Z","iopub.status.idle":"2023-01-08T14:33:37.986460Z","shell.execute_reply.started":"2023-01-08T14:33:33.338306Z","shell.execute_reply":"2023-01-08T14:33:37.985324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = 'playResult'\nfeatures = ['action_time',\n            'prox1_LTE_20', 'prox1_LTE_15', 'prox1_LTE_10', 'prox1_LTE_05',\n            'prox2_LTE_20', 'prox2_LTE_15', 'prox2_LTE_10', 'prox2_LTE_05',\n            'prox3_LTE_20', 'prox3_LTE_15', 'prox3_LTE_10', 'prox3_LTE_05',\n            'down', 'yardsToGo', 'defendersInBox', 'pff_playAction', 'zone_defense', 'o_form_shotgun', 'o_form_empty_set', 'yds_end_zone', 'score_diff'\n           ]\ntrain = pf6.loc[pf6['train'] == True][(features + [target])].copy()\ntest = pf6.loc[pf6['train'] == False][(features + [target])].copy()\n\nbase_score = train[target].mean()\n\neasy_gbm(train,test,target,.05,6,base_score,50,'reg:squarederror')\n\nlift_chart(out_test, target, 'pred', 10)\n\nexplainer = shap.Explainer(model_xgb)\nshap_values = explainer(train.drop([target],axis = 1))\nshap.plots.beeswarm(shap_values, max_display = 50)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:33:54.952238Z","iopub.execute_input":"2023-01-08T14:33:54.953027Z","iopub.status.idle":"2023-01-08T14:33:59.950807Z","shell.execute_reply.started":"2023-01-08T14:33:54.952985Z","shell.execute_reply":"2023-01-08T14:33:59.949408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Figure out number of rushers","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keys3 = ['gameId','playId','frameId']\nkeys4 = keys3 + ['nflId']\n\npos_o_line_names = ['C', 'T', 'G']\npos_o_line = data.loc[data['officialPosition'].isin(pos_o_line_names)][(keys3 + ['x','y'])].copy()\npos_o_line.rename(columns = {'x': 'ol_x', 'y': 'ol_y'}, inplace = True)\n\n# create distance to nearest o_line\na = dist_qb4.merge(pos_o_line, on = keys3)\na['dist_o_line'] = ((a['ol_x'] - a['x'])**2 + (a['ol_y'] - a['y'])**2)**0.5\nb = a.groupby(keys4).agg({'dist_o_line': 'min'}).reset_index()\n\ndist_o_line0 = dist_qb4.merge(b, on = keys4)\nprint(dist_o_line0.shape[0] - dist_qb4.shape[0])\ndel a\ndel b","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:34:29.206017Z","iopub.execute_input":"2023-01-08T14:34:29.207951Z","iopub.status.idle":"2023-01-08T14:34:37.650232Z","shell.execute_reply.started":"2023-01-08T14:34:29.207868Z","shell.execute_reply":"2023-01-08T14:34:37.648984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ol_dist_threshold = 1.5\npct_threshold = .50\nframes_for_chase_pos = 5\nframes_for_chase_time = 1\nmin_chase_prox = 5\n\ndist_o_line = dist_o_line0.copy()\n\ndist_o_line['frameId_past'] = dist_o_line['frameId'] - frames_for_chase_pos\ndist_o_line['frameId_past'] = np.where(dist_o_line['frameId_past'] < dist_o_line['ball_snap_frame'], dist_o_line['ball_snap_frame'], dist_o_line['frameId_past'])\n\npos_qb2 = pos_qb.groupby(['gameId','playId','frameId']).agg({'qb_x': 'mean', 'qb_y': 'mean'}).reset_index()\npos_qb2.rename(columns = {'frameId': 'frameId_past', 'qb_x': 'qb_x_past', 'qb_y': 'qb_y_past'}, inplace = True)\n\nprint(dist_o_line.shape)\ndist_o_line = dist_o_line.merge(pos_qb2, on = ['gameId','playId','frameId_past'])\nprint(dist_o_line.shape)\n\n########\ndist_o_line_next = dist_o_line[(['gameId','playId','frameId','nflId','x','y','ball_snap_frame'])].copy()\ndist_o_line_next.rename(columns = {'x': 'x_next', 'y': 'y_next'}, inplace = True)\n\ndist_o_line_next = dist_o_line_next.loc[dist_o_line_next['frameId'] >= dist_o_line_next['ball_snap_frame'] + frames_for_chase_time]\ndist_o_line_next['frameId'] = dist_o_line_next['frameId'] - frames_for_chase_time\ndel dist_o_line_next['ball_snap_frame']\n\nprint(dist_o_line.shape)\ndist_o_line = dist_o_line.merge(dist_o_line_next, on = ['gameId','playId','frameId','nflId'], how = 'left')\nprint(dist_o_line.shape)\n\ndist_o_line['dist_qb_next'] = ((dist_o_line['qb_x_past'] - dist_o_line['x_next'])**2 + (dist_o_line['qb_y_past'] - dist_o_line['y_next'])**2)**0.5\n\ndist_o_line['go_to_qb'] = np.where((dist_o_line['dist_qb_next'] <=  dist_o_line['dist_qb']) & (dist_o_line['dist_qb_next'] <= min_chase_prox), True, False)\ndist_o_line['close_to_ol'] = np.where(dist_o_line['dist_o_line'] <= ol_dist_threshold, True, False)\ndist_o_line['crit_met'] = np.where((dist_o_line['go_to_qb'] == True) | (dist_o_line['close_to_ol'] == True), True, False)\n\nrushing_df = dist_o_line.groupby(['gameId','playId','nflId']).agg({'crit_met': 'mean'}).reset_index()\nrushing_df = rushing_df.loc[rushing_df['crit_met'] >= pct_threshold]\n\nrushers_cnt_df = rushing_df.groupby(['gameId','playId']).agg({'nflId': 'count'}).reset_index()\nrushers_cnt_df.rename(columns = {'nflId': 'num_rushing'}, inplace = True)\n\nrushers_cnt_df.groupby(['num_rushing']).agg({'playId': 'count'}).reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:34:58.305180Z","iopub.execute_input":"2023-01-08T14:34:58.305582Z","iopub.status.idle":"2023-01-08T14:35:03.330589Z","shell.execute_reply.started":"2023-01-08T14:34:58.305544Z","shell.execute_reply":"2023-01-08T14:35:03.328850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dist_o_line.loc[(dist_o_line['gameId'] == 2021110100)&(dist_o_line['playId'] == 4433)]\n","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:35:31.623793Z","iopub.execute_input":"2023-01-08T14:35:31.624309Z","iopub.status.idle":"2023-01-08T14:35:32.028073Z","shell.execute_reply.started":"2023-01-08T14:35:31.624270Z","shell.execute_reply":"2023-01-08T14:35:32.026777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ### Analyze when min and max action end frame are not the same to evaluate max action end frame treatment\n# max_aef = non_bs2.groupby(keys2).agg({'action_end_frame': 'max'}).reset_index()\n# max_aef.rename(columns = {'action_end_frame': 'max_aef'}, inplace = True)\n# min_aef = non_bs2.groupby(keys2).agg({'action_end_frame': 'min'}).reset_index()\n# min_aef.rename(columns = {'action_end_frame': 'min_aef'}, inplace = True)\n\n# diff_aef = max_aef.merge(min_aef, on = keys2)\n# diff_aef = diff_aef.loc[diff_aef['max_aef'] != diff_aef['min_aef']]\n\n# print(diff_aef.shape[0])\n\n# x1 = plays_pass.merge(diff_aef, on = keys2)[(keys2 + ['frameId','event'])].sort_values(by = keys2 + ['frameId'])\n# x1.loc[x1['event'] != 'ball_snap']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Data work to date\ngames = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2023/games.csv', engine = 'python')\n\npffScoutingData = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2023/pffScoutingData.csv', engine = 'python')\n\nplays = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2023/plays.csv', engine = 'python')\nplays['completion_pct'] = np.where(plays['passResult'] == 'C',True ,False)\nplays['interception_pct'] = np.where(plays['passResult'] == 'IN',True ,False)\nplays['sack_pct'] = np.where(plays['passResult'] == 'S',True ,False)\nplays['scramble_pct'] = np.where(plays['passResult'] == 'R',True ,False)\n\nplayers = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2023/players.csv', engine = 'python')\n\np_g = plays.merge(games, on = ['gameId']) #plays_merge_games\npff_pg = pffScoutingData.merge(p_g, on = ['gameId','playId'])\npff_pg_plyrs = pff_pg.merge(players, on = 'nflId')\n\nprint(pffScoutingData.shape)\nprint(pff_pg_plyrs.shape)\n\npff_pg_plyrs.head()\n\nweek1 = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2023/week1.csv', engine = 'python')","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:36:20.686449Z","iopub.execute_input":"2023-01-08T14:36:20.686944Z","iopub.status.idle":"2023-01-08T14:36:39.725093Z","shell.execute_reply.started":"2023-01-08T14:36:20.686904Z","shell.execute_reply":"2023-01-08T14:36:39.723808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fields = [i for i in pff_pg_plyrs.columns if i not in ['playId','gameId','nflId','completion_pct','playDescription','season','GameDate']]\npositions = ['All'] + pff_pg_plyrs['pff_positionLinedUp'].drop_duplicates().to_list()\npositions.sort()\n    \ndef eda_plot(data,field,position):\n    if position == 'All':\n        x = data.groupby([field], dropna = False).agg({'playId': 'count', 'completion_pct': 'mean'}).reset_index()\n        x.rename(columns = {'playId': 'record_count'}, inplace = True)\n    else:\n        x = data.loc[data['pff_positionLinedUp'] == position].groupby([field], dropna = False).agg({'playId': 'count', 'completion_pct': 'mean'}).reset_index()\n        x.rename(columns = {'playId': 'record_count'}, inplace = True)\n    \n    fig, ax = plt.subplots(figsize=(12,6))\n    ax2  = ax.twinx()\n\n    x['record_count'].plot.bar(stacked=False, ax=ax, alpha=0.6)\n    x['completion_pct'].plot(kind='line', ax=ax2, marker='o', color = 'r', legend = '')\n\n    plt.xticks(ticks = x.index, labels = x[field])\n\n    ax.set(ylabel='Records', title = field)\n    ax2.set(ylabel='Completion %')\n\n    plt.show()\n    \ndef f(field,position):\n    return eda_plot(pff_pg_plyrs,field,position)\n\ninteract(f, field = fields, position = positions)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:36:42.751827Z","iopub.execute_input":"2023-01-08T14:36:42.752651Z","iopub.status.idle":"2023-01-08T14:36:43.126996Z","shell.execute_reply.started":"2023-01-08T14:36:42.752602Z","shell.execute_reply":"2023-01-08T14:36:43.125700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pff_df = pff_pg_plyrs[(['gameId', 'playId', 'nflId', 'officialPosition', 'pff_positionLinedUp',\n                        'completion_pct','interception_pct','sack_pct','scramble_pct',\n                        'playResult'])].copy()\n\npff_df['playResult'] = pff_df['playResult'].astype('int8')\n\nkeys = ['gameId', 'playId', 'nflId']\nweek1_df = week1.merge(pff_df, on = keys)\n\nprint(week1.shape)\nprint(week1_df.shape)\n\n# No double joins\n# missing records appear to be team data - for now going to ignore this\n\nweek1_df.drop(columns = ['time','jerseyNumber','team','playDirection'], inplace = True)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:37:04.583443Z","iopub.execute_input":"2023-01-08T14:37:04.583876Z","iopub.status.idle":"2023-01-08T14:37:05.833073Z","shell.execute_reply.started":"2023-01-08T14:37:04.583842Z","shell.execute_reply":"2023-01-08T14:37:05.831688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# QB only; action time results\nplays_pass = week1_df.loc[week1_df['officialPosition'] == 'QB'].copy()\nprint(plays_pass.shape)\n\n# filter to events which are either ball snap or pass plays\nplays_pass = plays_pass.loc[plays_pass['event'].isin(['ball_snap','pass_forward','lateral','fumble','fumble_offense_recovered','qb_sack','qb_strip_sack','run'])]\n\nprint(plays_pass.shape)\n\n# need to change this to track the maximum time the ball is held by the QB instead of when a forward pass actually occurs\n\nkeys2 = ['gameId','playId']\nbs = plays_pass.loc[plays_pass['event'] == 'ball_snap'][(keys2)].copy().drop_duplicates() # some duplicate id's - reviewed and insignificant\nnon_bs = plays_pass.loc[plays_pass['event'] != 'ball_snap'][(keys2)].copy().drop_duplicates() # some duplicate id's - reviewed and insignificant\n\nplays_pass = plays_pass.merge(bs, on = keys2).merge(non_bs, on = keys2)\n\nprint(plays_pass.shape)\n\nfacts = ['completion_pct','interception_pct','playResult']\n\nbs2 = plays_pass.loc[plays_pass['event'] == 'ball_snap'][(keys2 + ['frameId'] + facts)]\nbs2.rename(columns = {'frameId': 'ball_snap_frame'}, inplace = True)\n\nnon_bs2 = plays_pass.loc[plays_pass['event'] != 'ball_snap'][(keys2 + ['frameId'])]\nnon_bs2.rename(columns = {'frameId': 'action_end_frame'}, inplace = True)\n\nnon_bs3 = non_bs2.groupby(keys2).agg({'action_end_frame': 'max'}).reset_index()\n\npf3 = non_bs3.merge(bs2, on = keys2)\n\nprint(pf3.shape)\n\npf3['action_time'] = (pf3['action_end_frame'] - pf3['ball_snap_frame'])/10\n\nsacks = week1_df.groupby(keys2).agg({'sack_pct': 'max'}).reset_index()\nscrambles = week1_df.groupby(keys2).agg({'scramble_pct': 'max'}).reset_index()\n\npf4 = pf3.merge(sacks, on = keys2).merge(scrambles, on = keys2)\n\nprint(pf3.shape)\nprint(pf4.shape)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:37:32.006790Z","iopub.execute_input":"2023-01-08T14:37:32.007958Z","iopub.status.idle":"2023-01-08T14:37:32.263804Z","shell.execute_reply.started":"2023-01-08T14:37:32.007899Z","shell.execute_reply":"2023-01-08T14:37:32.262580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Item #1; Time to Throw the Pass","metadata":{}},{"cell_type":"code","source":"agg_dict = {'playId': 'count',\n            'completion_pct': 'mean',\n            'interception_pct': 'mean',\n            'sack_pct': 'mean',\n            'scramble_pct': 'mean',\n            'playResult': 'mean'\n           }\n\ndef box_var(data,field,new_field,low,high):\n    data[new_field] = np.where(data[field] < low, low, data[field])\n    data[new_field] = np.where(data[new_field] > high, high, data[new_field])\n\nbox_var(pf4,'action_time','action_time',2,4)\n\npf4.groupby(['action_time']).agg(agg_dict).reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:37:59.341767Z","iopub.execute_input":"2023-01-08T14:37:59.342194Z","iopub.status.idle":"2023-01-08T14:37:59.374073Z","shell.execute_reply.started":"2023-01-08T14:37:59.342161Z","shell.execute_reply":"2023-01-08T14:37:59.372902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# obtain distance to QB for all relevant defensive positions at all frames \npos_qb = week1_df.loc[week1_df['officialPosition'] == 'QB'][(['gameId','playId','frameId','x','y'])].copy()\npos_qb.rename(columns = {'x': 'qb_x', 'y': 'qb_y'}, inplace = True)\n\npos_defense = ['DE','DT','FS','ILB','MLB','NT','OLB','SS']\ndist_qb0 = week1_df.loc[week1_df['officialPosition'].isin(pos_defense)].copy()\n\nkeys3 = ['gameId','playId','frameId']\nkeys4 = keys3 + ['nflId']\n\npos_qb1 = pos_qb.groupby(keys3).agg({'qb_x': 'mean', 'qb_y': 'mean'}).reset_index()\ndist_qb1 = dist_qb0.groupby(keys4).agg({'x': 'mean', 'y': 'mean'}).reset_index()\n\ndist_qb2 = dist_qb1.merge(pos_qb1, on = keys3)\n\nprint(dist_qb0.shape)\nprint(dist_qb1.shape)\nprint(dist_qb2.shape)\n\nprint(pos_qb.shape)\nprint(pos_qb1.shape)\n\ndist_qb2['dist_qb'] = ((dist_qb2['qb_x'] - dist_qb2['x'])**2 + (dist_qb2['qb_y'] - dist_qb2['y'])**2)**0.5","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:38:17.533062Z","iopub.execute_input":"2023-01-08T14:38:17.533442Z","iopub.status.idle":"2023-01-08T14:38:18.120122Z","shell.execute_reply.started":"2023-01-08T14:38:17.533411Z","shell.execute_reply":"2023-01-08T14:38:18.118503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get the ball snap & action end frame; manipulate and then join the preprocessed features to pf4\ndist_qb3 = dist_qb2.merge(pf4[(['gameId','playId','ball_snap_frame','action_end_frame'])], on = ['gameId','playId'])\nprint(dist_qb2.shape)\nprint(dist_qb3.shape)\n\ndist_qb4 = dist_qb3.loc[dist_qb3['frameId'] >= dist_qb3['ball_snap_frame']]\ndist_qb4 = dist_qb4.loc[dist_qb4['frameId'] <= dist_qb4['action_end_frame']]\nprint(dist_qb4.shape)\n\ndist_qb4['frame_from_ball_snap'] = dist_qb4['frameId'] - dist_qb4['ball_snap_frame']\n\nframes_for_stats = [5,10,15,20,25,30,35,40]\n\ndist_qb5 = dist_qb4.loc[dist_qb4['frame_from_ball_snap'].isin(frames_for_stats)]\nprint(dist_qb5.shape)\n\ndist_qb5 = dist_qb5.drop_duplicates()\nprint(dist_qb5.shape)\n\ndist_qb5['prox_rank'] = dist_qb5.groupby(['gameId','playId','frame_from_ball_snap'])['dist_qb'].rank()\n\nprox1 = dist_qb5.loc[dist_qb5['prox_rank'] == 1][(['gameId','playId','frame_from_ball_snap','dist_qb'])].copy()\nprox2 = dist_qb5.loc[dist_qb5['prox_rank'] == 2][(['gameId','playId','frame_from_ball_snap','dist_qb'])].copy()\n\nprox_name = 'prox1_'\nfor i, frame in enumerate(frames_for_stats):\n    a = prox1.loc[prox1['frame_from_ball_snap'] == frame].groupby(['gameId','playId']).agg({'dist_qb': 'mean'}).reset_index() #yes it doesn't really need a mean but just in case\n    a.rename(columns = {'dist_qb': prox_name + 'frame_' + str(frame)}, inplace = True)\n    \n    if i == 0:\n        prox1_full = a\n    else:\n        prox1_full = prox1_full.merge(a, on = ['gameId','playId'], how = 'outer')\n    \n\nprox_name = 'prox2_'\nfor i, frame in enumerate(frames_for_stats):\n    a = prox2.loc[prox2['frame_from_ball_snap'] == frame].groupby(['gameId','playId']).agg({'dist_qb': 'mean'}).reset_index() #yes it doesn't really need a mean but just in case\n    a.rename(columns = {'dist_qb': prox_name + 'frame_' + str(frame)}, inplace = True)\n    \n    if i == 0:\n        prox2_full = a\n    else:\n        prox2_full = prox2_full.merge(a, on = ['gameId','playId'], how = 'outer')","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:38:34.837233Z","iopub.execute_input":"2023-01-08T14:38:34.837706Z","iopub.status.idle":"2023-01-08T14:38:35.150623Z","shell.execute_reply.started":"2023-01-08T14:38:34.837664Z","shell.execute_reply":"2023-01-08T14:38:35.149403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keys = ['gameId','playId']\n\npf5 = pf4.merge(prox1_full, on = keys).merge(prox2_full, on = keys)\npf5.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:38:48.233008Z","iopub.execute_input":"2023-01-08T14:38:48.233436Z","iopub.status.idle":"2023-01-08T14:38:48.270542Z","shell.execute_reply.started":"2023-01-08T14:38:48.233400Z","shell.execute_reply":"2023-01-08T14:38:48.269133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prox_list = [i for i in pf5.columns if 'prox' in i]\nfor prox in prox_list:\n    print(prox + '; ' + str(pf5[prox].min()))","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:38:58.551457Z","iopub.execute_input":"2023-01-08T14:38:58.551877Z","iopub.status.idle":"2023-01-08T14:38:58.563328Z","shell.execute_reply.started":"2023-01-08T14:38:58.551844Z","shell.execute_reply":"2023-01-08T14:38:58.561847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prox_list = [i for i in pf5.columns if 'prox' in i]\nfor p in prox_list:\n    pf5[p] = round(pf5[p],0)\n\nagg_dicts = [{'playId': 'count'},{'completion_pct': 'mean'},{'interception_pct': 'mean'},{'sack_pct': 'mean'},{'scramble_pct': 'mean'},{'playResult': 'mean'}]\n\nprox_agg_end = pd.DataFrame()\nfor dic in agg_dicts:\n    for k in dic:\n        fact_name = k\n\n    prox_agg = pd.DataFrame()\n    for i, p in enumerate(prox_list):\n        x = pf5.groupby([p]).agg(dic).reset_index()\n        x.rename(columns = {p: 'prox', fact_name: p}, inplace = True)\n\n        if i == 0:\n            prox_agg = x\n        else:\n            prox_agg = prox_agg.merge(x, on = 'prox', how = 'outer')\n\n    prox_agg.sort_values(by = 'prox', inplace = True)\n    prox_agg['fact_name'] = fact_name\n    \n    prox_agg_end = pd.concat([prox_agg_end,prox_agg], ignore_index = True)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:39:19.974445Z","iopub.execute_input":"2023-01-08T14:39:19.974839Z","iopub.status.idle":"2023-01-08T14:39:20.504940Z","shell.execute_reply.started":"2023-01-08T14:39:19.974807Z","shell.execute_reply":"2023-01-08T14:39:20.503123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Item #2; Proximity in Yards to the QB for Rank 1 & 2 defensive positions","metadata":{}},{"cell_type":"code","source":"fact_names = []\n\nfor dic in agg_dicts:\n    for k in dic:\n        fact_names.append(k)\n\ndef f(fact_name):\n    return prox_agg_end.loc[prox_agg_end['fact_name'] == fact_name]\n\ninteract(f, fact_name = fact_names)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:39:44.707585Z","iopub.execute_input":"2023-01-08T14:39:44.708027Z","iopub.status.idle":"2023-01-08T14:39:44.790535Z","shell.execute_reply.started":"2023-01-08T14:39:44.707991Z","shell.execute_reply":"2023-01-08T14:39:44.789671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Some quick GBM models to get a starting idea of what's important","metadata":{}},{"cell_type":"code","source":"import xgboost as xgb\nimport shap\n\ndef easy_gbm(train,target,bmf,max_depth,base_score,num_round,objective = 'binary:logistic'):\n    params = {\n        'objective': objective,\n        'learning_rate': bmf,\n        'verbosity': 1,\n        'max_depth': max_depth,\n        'base_score': base_score\n    }\n\n    dtrain = xgb.DMatrix(train.drop([target],axis = 1), label=train[target])\n\n    global model_xgb\n    model_xgb = xgb.train(params, dtrain, num_round)\n\n    global out_train\n    out_train = train.copy()\n    out_train['pred'] = model_xgb.predict(dtrain)\n    \ndef lift_chart(test_data, act, pred, bins):\n    test_data['records'] = 1\n    test_data['decile'] = (round(test_data.sort_values(by = 'pred')['records'].cumsum()/test_data.shape[0],2)*bins).apply(np.floor)\n    test_data['decile'] = np.where(test_data['decile'] + 1 > bins ,bins,test_data['decile'] + 1)\n    x = test_data.groupby(['decile'], dropna = False).agg({'records': 'sum', act: 'mean', pred: 'mean'}).reset_index()\n    \n    dfg = x\n    fig, ax = plt.subplots(figsize=(12,6))\n    ax2  = ax.twinx()\n    \n    y_min = np.where(dfg[act].min() < dfg[pred].min(),dfg[act].min(),dfg[pred].min())*.95\n    y_max = np.where(dfg[act].max() > dfg[pred].max(),dfg[act].max(),dfg[pred].max())*1.05\n    ax2.set_ylim(y_min,y_max)\n    \n    dfg['records'].plot.bar(stacked=False, ax=ax, alpha=0.6)\n    dfg[act].plot(kind='line', ax=ax2, marker='o', linewidth = 0, legend='act')\n    dfg[pred].plot(kind='line', ax=ax2, marker='o', legend='pred')\n    plt.show()\n    print(x)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:40:24.725006Z","iopub.execute_input":"2023-01-08T14:40:24.725549Z","iopub.status.idle":"2023-01-08T14:40:24.740745Z","shell.execute_reply.started":"2023-01-08T14:40:24.725499Z","shell.execute_reply":"2023-01-08T14:40:24.739738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = 'completion_pct'\nfeatures = ['action_time',\n 'prox1_frame_5', 'prox1_frame_10', 'prox1_frame_15', 'prox1_frame_20', 'prox1_frame_25', 'prox1_frame_30', 'prox1_frame_35', 'prox1_frame_40',\n 'prox2_frame_5', 'prox2_frame_10', 'prox2_frame_15', 'prox2_frame_20', 'prox2_frame_25', 'prox2_frame_30', 'prox2_frame_35', 'prox2_frame_40']\ntrain = pf5[(features + [target])].copy()\n\nprox_list = [i for i in train.columns if 'prox' in i]\nfor prox in prox_list:\n    train[prox] = train[prox].fillna(10)\n\nbase_score = train[target].mean()\n\neasy_gbm(train,target,.10,6,base_score,50)\n\nlift_chart(out_train, target, 'pred', 10)\n\nexplainer = shap.Explainer(model_xgb)\nshap_values = explainer(train.drop([target],axis = 1))\nshap.plots.beeswarm(shap_values, max_display = 50)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:40:36.427445Z","iopub.execute_input":"2023-01-08T14:40:36.427859Z","iopub.status.idle":"2023-01-08T14:40:38.057871Z","shell.execute_reply.started":"2023-01-08T14:40:36.427825Z","shell.execute_reply":"2023-01-08T14:40:38.056510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = 'playResult'\nfeatures = ['action_time',\n 'prox1_frame_5', 'prox1_frame_10', 'prox1_frame_15', 'prox1_frame_20', 'prox1_frame_25', 'prox1_frame_30', 'prox1_frame_35', 'prox1_frame_40',\n 'prox2_frame_5', 'prox2_frame_10', 'prox2_frame_15', 'prox2_frame_20', 'prox2_frame_25', 'prox2_frame_30', 'prox2_frame_35', 'prox2_frame_40']\ntrain = pf5[(features + [target])].copy()\n\nprox_list = [i for i in train.columns if 'prox' in i]\nfor prox in prox_list:\n    train[prox] = train[prox].fillna(10)\n\nbase_score = train[target].mean()\n\neasy_gbm(train,target,.10,6,base_score,50,'reg:squarederror')\n\nlift_chart(out_train, target, 'pred', 10)\n\nexplainer = shap.Explainer(model_xgb)\nshap_values = explainer(train.drop([target],axis = 1))\nshap.plots.beeswarm(shap_values, max_display = 50)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:40:51.059700Z","iopub.execute_input":"2023-01-08T14:40:51.060142Z","iopub.status.idle":"2023-01-08T14:40:52.586645Z","shell.execute_reply.started":"2023-01-08T14:40:51.060106Z","shell.execute_reply":"2023-01-08T14:40:52.585263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = 'interception_pct'\nfeatures = ['action_time',\n 'prox1_frame_5', 'prox1_frame_10', 'prox1_frame_15', 'prox1_frame_20', 'prox1_frame_25', 'prox1_frame_30', 'prox1_frame_35', 'prox1_frame_40',\n 'prox2_frame_5', 'prox2_frame_10', 'prox2_frame_15', 'prox2_frame_20', 'prox2_frame_25', 'prox2_frame_30', 'prox2_frame_35', 'prox2_frame_40']\ntrain = pf5[(features + [target])].copy()\n\nprox_list = [i for i in train.columns if 'prox' in i]\nfor prox in prox_list:\n    train[prox] = train[prox].fillna(10)\n\nbase_score = train[target].mean()\n\neasy_gbm(train,target,.10,6,base_score,50)\n\nlift_chart(out_train, target, 'pred', 10)\n\nexplainer = shap.Explainer(model_xgb)\nshap_values = explainer(train.drop([target],axis = 1))\nshap.plots.beeswarm(shap_values, max_display = 50)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:41:05.387711Z","iopub.execute_input":"2023-01-08T14:41:05.388172Z","iopub.status.idle":"2023-01-08T14:41:06.649559Z","shell.execute_reply.started":"2023-01-08T14:41:05.388134Z","shell.execute_reply":"2023-01-08T14:41:06.648314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = 'sack_pct'\nfeatures = ['action_time',\n 'prox1_frame_5', 'prox1_frame_10', 'prox1_frame_15', 'prox1_frame_20', 'prox1_frame_25', 'prox1_frame_30', 'prox1_frame_35', 'prox1_frame_40',\n 'prox2_frame_5', 'prox2_frame_10', 'prox2_frame_15', 'prox2_frame_20', 'prox2_frame_25', 'prox2_frame_30', 'prox2_frame_35', 'prox2_frame_40']\ntrain = pf5[(features + [target])].copy()\n\nprox_list = [i for i in train.columns if 'prox' in i]\nfor prox in prox_list:\n    train[prox] = train[prox].fillna(10)\n\nbase_score = train[target].mean()\n\neasy_gbm(train,target,.10,6,base_score,50)\n\nlift_chart(out_train, target, 'pred', 10)\n\nexplainer = shap.Explainer(model_xgb)\nshap_values = explainer(train.drop([target],axis = 1))\nshap.plots.beeswarm(shap_values, max_display = 50)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:41:19.851701Z","iopub.execute_input":"2023-01-08T14:41:19.852412Z","iopub.status.idle":"2023-01-08T14:41:21.264265Z","shell.execute_reply.started":"2023-01-08T14:41:19.852363Z","shell.execute_reply":"2023-01-08T14:41:21.263244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = 'scramble_pct'\nfeatures = ['action_time',\n 'prox1_frame_5', 'prox1_frame_10', 'prox1_frame_15', 'prox1_frame_20', 'prox1_frame_25', 'prox1_frame_30', 'prox1_frame_35', 'prox1_frame_40',\n 'prox2_frame_5', 'prox2_frame_10', 'prox2_frame_15', 'prox2_frame_20', 'prox2_frame_25', 'prox2_frame_30', 'prox2_frame_35', 'prox2_frame_40']\ntrain = pf5[(features + [target])].copy()\n\nprox_list = [i for i in train.columns if 'prox' in i]\nfor prox in prox_list:\n    train[prox] = train[prox].fillna(10)\n\nbase_score = train[target].mean()\n\neasy_gbm(train,target,.10,6,base_score,50)\n\nlift_chart(out_train, target, 'pred', 10)\n\nexplainer = shap.Explainer(model_xgb)\nshap_values = explainer(train.drop([target],axis = 1))\nshap.plots.beeswarm(shap_values, max_display = 50)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:41:36.098984Z","iopub.execute_input":"2023-01-08T14:41:36.099605Z","iopub.status.idle":"2023-01-08T14:41:37.671826Z","shell.execute_reply.started":"2023-01-08T14:41:36.099568Z","shell.execute_reply":"2023-01-08T14:41:37.670307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for i in [i for i in pff_df.columns]:\n#     print(i)\n#     print(pff_df[i].min())\n#     print(pff_df[i].max())\n#     print(pff_df.loc[pff_df[i].isna() == True].shape[0])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fields = ['pff_hurry','pff_sack', 'passResult', 'playResult','completion_pct']\n\n# for field in fields:\n#     print(pff_pg_plyrs.groupby([field]).agg({'nflId': 'count'}).reset_index())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# MAD; Need to change yardlineSide and yardlineNumber into the range [-50,50] indicating range from own goal line to opponent's goal line based on gameId team names\n# MAD; Need to change gameClock to minutes remaining in game\n# MAD; Need to change preSnapHomeScore & preSnapVisitorScore to team's position in the game - paying special attention to abs value ranges [0],[1,2],[3],[4,6],[7],[8+] \n# MAD; Good idea to minimize dtypes for faster model fits\n# MAD; Classifier models\n    # QB performance given OL performance; QB by name and essentially pass rush handling effectiveness\n    # rank OL performance; Score OL by position & name\n    # these 2 are co-dependent; maybe RL helps here?\n    # Classify all relevant positions?\n    # These classifiers will be depending on formations, as well\n    # This data is small, might need Aaron's ideas on small data models (Bayesian)\n    # This data doesn't tell us much about what the players actually did (specific body actions when pass rushing or pass blocking); curious if some of the other data sources listed on the Kaggle data page can help\n        # NFL is probably more interested in metrics which help predict game outcomes rather than those which predict play outcomes\n        # Even if they are interested in play outcomes, they are likely satisfied with player effectiveness rankings\n        # How would they even get that data?  Would require CNN to analyze body movements based on game replays.  I doubt the other Kaggle participants are doing this.","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"games = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2023/games.csv', engine = 'python')\n\ngames.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:42:22.855738Z","iopub.execute_input":"2023-01-08T14:42:22.856142Z","iopub.status.idle":"2023-01-08T14:42:22.889795Z","shell.execute_reply.started":"2023-01-08T14:42:22.856110Z","shell.execute_reply":"2023-01-08T14:42:22.888932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pffScoutingData = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2023/pffScoutingData.csv', engine = 'python')\n\npffScoutingData.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:42:37.674448Z","iopub.execute_input":"2023-01-08T14:42:37.674912Z","iopub.status.idle":"2023-01-08T14:42:40.054351Z","shell.execute_reply.started":"2023-01-08T14:42:37.674848Z","shell.execute_reply":"2023-01-08T14:42:40.053099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[i for i in pffScoutingData.columns]","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:42:50.484216Z","iopub.execute_input":"2023-01-08T14:42:50.484850Z","iopub.status.idle":"2023-01-08T14:42:50.493402Z","shell.execute_reply.started":"2023-01-08T14:42:50.484808Z","shell.execute_reply":"2023-01-08T14:42:50.492121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg_list = ['pff_hit','pff_hurry','pff_sack','pff_beatenByDefender','pff_hitAllowed','pff_hurryAllowed','pff_sackAllowed']\nagg_dict = {i: 'mean' for i in agg_list}\n\npffScoutingData.groupby(['pff_positionLinedUp']).agg(agg_dict).reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:43:01.760838Z","iopub.execute_input":"2023-01-08T14:43:01.761271Z","iopub.status.idle":"2023-01-08T14:43:01.828927Z","shell.execute_reply.started":"2023-01-08T14:43:01.761241Z","shell.execute_reply":"2023-01-08T14:43:01.827273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plays = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2023/plays.csv', engine = 'python')\n\nplays.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:43:13.565540Z","iopub.execute_input":"2023-01-08T14:43:13.566479Z","iopub.status.idle":"2023-01-08T14:43:13.790282Z","shell.execute_reply.started":"2023-01-08T14:43:13.566435Z","shell.execute_reply":"2023-01-08T14:43:13.789049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[i for i in plays.columns]","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:43:23.371715Z","iopub.execute_input":"2023-01-08T14:43:23.372392Z","iopub.status.idle":"2023-01-08T14:43:23.381551Z","shell.execute_reply.started":"2023-01-08T14:43:23.372355Z","shell.execute_reply":"2023-01-08T14:43:23.380276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plays['playResult'].hist(bins = 10)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:43:34.451321Z","iopub.execute_input":"2023-01-08T14:43:34.451725Z","iopub.status.idle":"2023-01-08T14:43:34.690745Z","shell.execute_reply.started":"2023-01-08T14:43:34.451692Z","shell.execute_reply":"2023-01-08T14:43:34.689858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plays['completion_pct'] = np.where(plays['passResult'] == 'C',True ,False)\n\nplays.groupby(['yardsToGo'], dropna = False).agg({'playId': 'count', 'completion_pct': 'mean'}).reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:44:07.588153Z","iopub.execute_input":"2023-01-08T14:44:07.588550Z","iopub.status.idle":"2023-01-08T14:44:07.611589Z","shell.execute_reply.started":"2023-01-08T14:44:07.588517Z","shell.execute_reply":"2023-01-08T14:44:07.609873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plays.groupby(['down'], dropna = False).agg({'playId': 'count', 'completion_pct': 'mean'}).reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:44:19.565839Z","iopub.execute_input":"2023-01-08T14:44:19.566956Z","iopub.status.idle":"2023-01-08T14:44:19.584234Z","shell.execute_reply.started":"2023-01-08T14:44:19.566873Z","shell.execute_reply":"2023-01-08T14:44:19.582877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p_g = plays.merge(games, on = ['gameId']) #plays_merge_games\nprint(plays.shape)\nprint(p_g.shape)\n\np_g.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:44:29.786493Z","iopub.execute_input":"2023-01-08T14:44:29.787181Z","iopub.status.idle":"2023-01-08T14:44:29.853869Z","shell.execute_reply.started":"2023-01-08T14:44:29.787144Z","shell.execute_reply":"2023-01-08T14:44:29.852595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(p_g.shape)\nprint(p_g[(['gameId','playId'])].drop_duplicates().shape)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:44:46.609013Z","iopub.execute_input":"2023-01-08T14:44:46.609404Z","iopub.status.idle":"2023-01-08T14:44:46.625062Z","shell.execute_reply.started":"2023-01-08T14:44:46.609372Z","shell.execute_reply":"2023-01-08T14:44:46.623363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Next merge pffScoutingData with play result + game info on playId & gameId\n\npff_pg = pffScoutingData.merge(p_g, on = ['gameId','playId'])\nprint(pffScoutingData.shape)\nprint(pff_pg.shape)\n\npff_pg.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:44:57.029425Z","iopub.execute_input":"2023-01-08T14:44:57.030093Z","iopub.status.idle":"2023-01-08T14:44:57.807961Z","shell.execute_reply.started":"2023-01-08T14:44:57.030056Z","shell.execute_reply":"2023-01-08T14:44:57.806806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"players = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2023/players.csv', engine = 'python')\n\nplayers.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:45:17.169226Z","iopub.execute_input":"2023-01-08T14:45:17.170294Z","iopub.status.idle":"2023-01-08T14:45:17.198561Z","shell.execute_reply.started":"2023-01-08T14:45:17.170249Z","shell.execute_reply":"2023-01-08T14:45:17.197536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Next merge players data\n\npff_pg_plyrs = pff_pg.merge(players, on = 'nflId')\n\nprint(pff_pg.shape)\nprint(pff_pg_plyrs.shape)\n\npff_pg_plyrs.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:45:26.514216Z","iopub.execute_input":"2023-01-08T14:45:26.514824Z","iopub.status.idle":"2023-01-08T14:45:26.769159Z","shell.execute_reply.started":"2023-01-08T14:45:26.514790Z","shell.execute_reply":"2023-01-08T14:45:26.767085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pff_pg_plyrs.groupby(['officialPosition','displayName'], dropna = False).agg({'playId': 'count', 'completion_pct': 'mean'}).reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:45:37.671693Z","iopub.execute_input":"2023-01-08T14:45:37.672204Z","iopub.status.idle":"2023-01-08T14:45:37.735472Z","shell.execute_reply.started":"2023-01-08T14:45:37.672164Z","shell.execute_reply":"2023-01-08T14:45:37.734486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# MAD; Add some widgets next for easier visualization\n# MAD; run xgboost without much thought into feature selection for a head start","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"week1 = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2023/week1.csv', engine = 'python')","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:46:40.778954Z","iopub.execute_input":"2023-01-08T14:46:40.779352Z","iopub.status.idle":"2023-01-08T14:46:56.300128Z","shell.execute_reply.started":"2023-01-08T14:46:40.779320Z","shell.execute_reply":"2023-01-08T14:46:56.298612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"week1.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:47:07.305211Z","iopub.execute_input":"2023-01-08T14:47:07.305608Z","iopub.status.idle":"2023-01-08T14:47:07.336253Z","shell.execute_reply.started":"2023-01-08T14:47:07.305573Z","shell.execute_reply":"2023-01-08T14:47:07.334339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"week1.groupby(['event'], dropna = False).agg({'playId': 'count'}).reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T14:47:18.049638Z","iopub.execute_input":"2023-01-08T14:47:18.050150Z","iopub.status.idle":"2023-01-08T14:47:18.156773Z","shell.execute_reply.started":"2023-01-08T14:47:18.050112Z","shell.execute_reply":"2023-01-08T14:47:18.155141Z"},"trusted":true},"execution_count":null,"outputs":[]}]}