{"cells":[{"metadata":{},"cell_type":"markdown","source":"<a id='bg'></a>\n<div class=\"h2\">  Loading Track Data </div>\n\nI started with https://www.kaggle.com/jpmiller/using-track-data-with-small-memory/ and inserted my feature engineering into the chunk process. I did minimal cleaning and changed events into two categories (keep/drop).  I created 3 datasets for analysis:\n* track: 1% of Track Data randomly sampled.  \n* injtrack: 100% of injured play data\n* oneplayer: 100% of one player's data. I chose the player with the most data in the dataset.\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import sys\nimport numpy as np\nimport pandas as pd\npd.options.display.float_format = '{:,.5f}'.format\nfrom tqdm.notebook import tqdm\n\nimport skmem #utility script","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Convert all Events into either 0 (drop) or 1 (keep)"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"\nEventdict = {\"huddle_start_offense\":0,\n\"huddle_break_offense\":0,\n\"line_set\":0,\n\"ball_snap\":1,\n\"pass_forward\":1,\n\"pass_arrived\":1,\n\"pass_outcome_incomplete\":0,\n\"pass_outcome_caught\":0,\n\"first_contact\":1,\n\"out_of_bounds\":0,\n\"man_in_motion\":0,\n\"handoff\":1,\n\"tackle\":0,\n\"penalty_flag\":0,\n\"penalty_accepted\":0,\n\"touchdown\":0,\n\"shift\":0,\n\"qb_kneel\":0,\n\"fumble\":1,\n\"fumble_offense_recovered\":0,\n\"lateral\":1,\n\"penalty_declined\":0,\n\"qb_sack\":0,\n\"pass_shovel\":1,\n\"pass_outcome_touchdown\":0,\n\"run\":1,\n\"pass_outcome_interception\":1,\n\"qb_strip_sack\":1,\n\"two_point_conversion\":1,\n\"pass_tipped\":1,\n\"fumble_defense_recovered\":0,\n\"two_minute_warning\":0,\n\"two_point_play\":1,\n\"snap_direct\":1,\n\"play_action\":1,\n\"qb_spike\":0,\n\"pass_lateral\":1,\n\"touchback\":0,\n\"timeout_tv\":0,\n\"timeout\":0,\n\"kickoff_play\":1,\n\"onside_kick\":1,\n\"kick_received\":1,\n\"safety\":0,\n\"field_goal_attempt\":1,\n\"field_goal\":0,\n\"punt_play\":1,\n\"punt\":1,\n\"punt_land\":1,\n\"fair_catch\":0,\n\"punt_downed\":0,\n\"punt_received\":1,\n\"punt_fake\":1,\n\"kickoff\":1,\n\"kickoff_land\":1,\n\"timeout_away\":0,\n\"punt_muffed\":1,\n\"timeout_booth_review\":0,\n\"field_goal_play\":1,\n\"run_pass_option\":1,\n\"timeout_injury\":0,\n\"kick_recovered\":0,\n\"extra_point_attempt\":1,\n\"extra_point\":0,\n\"field_goal_blocked\":1,\n\"field_goal_missed\":0,\n\"timeout_home\":0,\n\"extra_point_blocked\":0,\n\"extra_point_missed\":0,\n\"punt_blocked\":0,\n\"timeout_quarter\":0,\n\"end_path\":0,\n\"field_goal_fake\":1,\n\"xp_fake\":1,\n\"extra_point_fake\":1,\n\"timeout_halftime\":0,\n\"free_kick\":1,\n\"free_kick_play\":1,\n\"0_kick\":1,\n\"drop_kick\":1,\n\"play_submit\\t\":0}\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Remove unrealistic distances from the dataset.  I removed all plays where the player moved more than 0.5 yards.\n* Calculate rolling statistics for 0.5 and 1,0 seconds.  Football movements take 0.5 to 1.0 seconds to execute\n* Calculated absolute and relative statistics.\n* Set a sampling rate to reduce the memory footprint further.\n* PlayerKey 43483 is the player with the most data."},{"metadata":{"trusted":true},"cell_type":"code","source":"inj = pd.read_csv('../input/nfl-playing-surface-analytics/InjuryRecord.csv')\nid_array = inj.PlayKey.str.split('-', expand=True).to_numpy()\ninj['PlayerKey'] = id_array[:,0]\ninj['GameID'] = id_array[:,1]\ninj['PlayKey'] = id_array[:,2]\ninj = inj.dropna().astype({'PlayerKey': 'int32',\n           'GameID': 'int32',\n           'PlayKey': 'int32'})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"\n\ncsize = 2_000_000 #set this to fit your situation\nsamplerate = 0.01\nchunker = pd.read_csv('../input/nfl-playing-surface-analytics/PlayerTrackData.csv',\n                      chunksize=csize)\ntrack_list = []\nplayer_list = []\ninj_list = []\nmr = skmem.MemReducer()\nmaxdis = 1\ni = 0\nfor chunk in tqdm(chunker, total = int(80_000_000/csize)):\n    chunk['PlayKey'] = chunk.PlayKey.fillna('0-0-0')\n    id_array = chunk.PlayKey.str.split('-', expand=True).to_numpy()\n    chunk['PlayerKey'] = id_array[:,0].astype(int)\n    chunk['GameID'] = id_array[:,1].astype(int)\n    chunk['PlayKey'] = id_array[:,2].astype(int)\n    chunk = chunk.astype({'PlayerKey': 'int32',\n           'GameID': 'int32',\n           'PlayKey': 'int32'})\n    chunk = chunk.replace({\"event\": Eventdict})\n    chunk['event'] = chunk.event.ffill().fillna(0)\n    chunk['dY'] = np.cos(chunk['dir'].shift(1) * (np.pi/180)) \n    chunk['dX'] = np.sin(chunk['dir'].shift(1) * (np.pi/180)) \n    chunk['Pred_X'] = chunk.dX * chunk.dis.shift(1) + chunk.x.shift(1)\n    chunk['Pred_Y'] = chunk.dY *chunk.dis.shift(1) + chunk.y.shift(1)\n    chunk['Acc_X'] = chunk.Pred_X - chunk.x\n    chunk['Acc_Y'] = chunk.Pred_Y - chunk.y\n    chunk['Total_Acc'] = (chunk.Acc_X**2 + chunk.Acc_Y**2)*0.5\n    chunk['Angle'] = chunk['o']-chunk['dir']\n    chunk['Angle'] = np.where(chunk['Angle']>180,360-chunk['Angle'],chunk['Angle'])\n    chunk['Angle'] = np.where(chunk['Angle']<-180,360+chunk['Angle'],chunk['Angle'])\n    chunk = chunk[(abs(chunk.x - chunk.x.shift(1))<maxdis) &  (abs(chunk.y - chunk.y.shift(1))<maxdis)].copy()\n    chunk = chunk.reset_index()\n    for lag in range(1,10):\n        if lag>1:\n            chunk['Delta_Dis_' + str(lag)] = chunk.dis.shift(lag) - chunk.dis.shift(lag-1)\n            chunk['Delta_Dir_' + str(lag)] = chunk.dir.shift(lag) - chunk.dir.shift(lag-1)\n            chunk['Delta_Angle_' + str(lag)] = chunk['Angle'].shift(lag) - chunk['Angle'].shift(lag-1)\n            chunk['Delta_O_' + str(lag)] = chunk.o.shift(lag) - chunk.o.shift(lag-1)\n            chunk['Delta_Total_Acc_' + str(lag)] = chunk.Total_Acc.shift(lag) - chunk.Total_Acc.shift(lag-1)\n            #Adjust change in direction\n            chunk['Delta_Dir_' + str(lag)] = np.where(chunk['Delta_Dir_' + str(lag)]>180,360-chunk['Delta_Dir_' + str(lag)],chunk['Delta_Dir_' + str(lag)])\n            chunk['Delta_Dir_' + str(lag)] = np.where(chunk['Delta_Dir_' + str(lag)]<-180,360+chunk['Delta_Dir_' + str(lag)],chunk['Delta_Dir_' + str(lag)])\n            chunk['Delta_Angle_' + str(lag)] = np.where(chunk['Delta_Angle_' + str(lag)]>180,360-chunk['Delta_Angle_' + str(lag)],chunk['Delta_Angle_' + str(lag)])\n            chunk['Delta_Angle_' + str(lag)] = np.where(chunk['Delta_Angle_' + str(lag)]<-180,360+chunk['Delta_Angle_' + str(lag)],chunk['Delta_Angle_' + str(lag)])\n            chunk['Delta_O_' + str(lag)] = np.where(chunk['Delta_O_' + str(lag)]>180,360-chunk['Delta_O_' + str(lag)],chunk['Delta_O_' + str(lag)])\n            chunk['Delta_O_' + str(lag)] = np.where(chunk['Delta_O_' + str(lag)]<-180,360+chunk['Delta_O_' + str(lag)],chunk['Delta_O_' + str(lag)])\n            #Kludgy\n            chunk['abs_Delta_Dis_'+ str(lag)] = abs(chunk['Delta_Dis_' + str(lag)])\n            chunk['abs_Delta_Dir_'+ str(lag)] = abs(chunk['Delta_Dir_' + str(lag)])\n            chunk['abs_Delta_Angle_'+ str(lag)] = abs(chunk['Delta_Angle_' + str(lag)])\n            chunk['abs_Delta_O_'+ str(lag)] = abs(chunk['Delta_O_' + str(lag)])\n            chunk['abs_Delta_Total_Acc_'+ str(lag)] = abs(chunk['Delta_Total_Acc_' + str(lag)])\n            \n    for lag in [5,10]:\n\n        chunk['Rolling_'+str(lag)+'_Dis_std']=chunk['Delta_Dis_2'].rolling(lag).std().reset_index(drop=True)\n        chunk['Rolling_'+str(lag)+'_Dir_std']=chunk['Delta_Dir_2'].rolling(lag).std().reset_index(drop=True)\n        chunk['Rolling_'+str(lag)+'_Angle_std']=chunk['Delta_Angle_2'].rolling(lag).std().reset_index(drop=True)\n        chunk['Rolling_'+str(lag)+'_O_std']=chunk['Delta_O_2'].rolling(lag).std().reset_index(drop=True)\n        chunk['Rolling_'+str(lag)+'_Total_Acc_std']=chunk['Delta_Total_Acc_2'].rolling(lag).std().reset_index(drop=True)\n        chunk['Rolling_abs_'+str(lag)+'_Dis_mean']=abs(chunk['Delta_Dis_2']).rolling(lag).mean().reset_index(drop=True)\n        chunk['Rolling_abs_'+str(lag)+'_Dir_mean']=abs(chunk['Delta_Dir_2']).rolling(lag).mean().reset_index(drop=True)\n        chunk['Rolling_abs_'+str(lag)+'_Angle_mean']=abs(chunk['Delta_Angle_2']).rolling(lag).mean().reset_index(drop=True)\n        chunk['Rolling_abs_'+str(lag)+'_O_mean']=abs(chunk['Delta_O_2']).rolling(lag).mean().reset_index(drop=True)\n        chunk['Rolling_abs_'+str(lag)+'_Total_Acc_mean']=abs(chunk['Delta_Total_Acc_2']).rolling(lag).mean().reset_index(drop=True)\n    \n        cols = [c for c in chunk.columns if c.lower()[5:] != 'Delta']\n        chunk = chunk[cols]\n\n    chunk = chunk.replace([np.inf,-np.inf],np.nan).fillna(0)  \n    #floaters = chunk.select_dtypes('float').columns.tolist()\n    #chunk = mr.fit_transform(chunk, float_cols=floaters) #float downcast is optional\n    chunk = chunk[chunk['event']==1]\n    chunk= chunk.drop(columns=['event'])\n    injchunk = inj.merge(chunk, on=['PlayerKey','GameID','PlayKey'])\n    player_list.append(chunk[chunk['PlayerKey']==43483])\n    track_list.append(chunk.sample(frac=samplerate))\n    inj_list.append(injchunk)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Combine the chunks and save to parquet"},{"metadata":{"trusted":true},"cell_type":"code","source":"tracks = pd.concat(track_list)\ntracks.to_parquet('track.parq')\nplayer = pd.concat(player_list)\nplayer.to_parquet('oneplayer.parq')\ninjtrack = pd.concat(inj_list)\ninjtrack.to_parquet('injtrack.parq')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}