{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 1. Import Data\nRefer to https://www.kaggle.com/c/NFL-Punt-Analytics-Competition for details of the data resource","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport dask.dataframe as dd","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load raw data\ngame = pd.read_csv('../input/NFL-Punt-Analytics-Competition/game_data.csv')\nplay_info = pd.read_csv('../input/NFL-Punt-Analytics-Competition/play_information.csv',index_col=['GameKey','PlayID'])\nplay_role = pd.read_csv('../input/NFL-Punt-Analytics-Competition/play_player_role_data.csv')\nplayer_punt = pd.read_csv('../input/NFL-Punt-Analytics-Competition/player_punt_data.csv')\nvideo = pd.read_csv('../input/NFL-Punt-Analytics-Competition/video_review.csv',index_col=['GameKey','PlayID'])\n\n# Combine all NGS files to parq\ndef ngs():\n    ndtypes = {'GameKey': 'int16',         \n           'PlayID': 'int16',         \n           'GSISID': 'float32',                \n           'x': 'float32',         \n           'y': 'float32',         \n           'dis': 'float32',\n           'o': 'float32'}\n    nddf = dd.read_csv('../input/NFL-Punt-Analytics-Competition/NGS*', \n                    usecols=[n for n in ndtypes.keys()], dtype=ndtypes)\n    nddf['GSISID'] = nddf.GSISID.fillna(-1).astype('int32')\n    df = nddf.groupby(['GameKey','PlayID','GSISID']).mean()\n    df = df.compute()\n    df.to_parquet('../input/NFL-Punt-Analytics-Competition/NGS.parq')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Process Data\n### 2.1 Classify play by punt types","metadata":{}},{"cell_type":"code","source":"type_reg_mapping = [(\"not_punted\", \"(no play)|(delay of game)|(false start)|blocked|incomplete\"),\n                     (\"out_of_bounds\", \"out of bounds\"), \n                     (\"downed\",\"downed\"),        \n                     (\"touchback\",\"touchback\"),\n                     (\"fair_catch\",\"fair catch\"),\n                     (\"returned\",\"(no gain)|(for.*yard)\")\n                    ]\n\nfor t,r in type_reg_mapping:\n    play_info[t]=play_info[\"PlayDescription\"].str.contains(r,case=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. EDA\n\nShow concussion stats in terms of different aspects.","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.1 Counts per year\n","metadata":{}},{"cell_type":"code","source":"video[\"Season_Year\"].plot.hist()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.2 Turf, Weather, Temperature","metadata":{}},{"cell_type":"code","source":"temp = video.merge(game[['GameKey','Turf','GameWeather','Temperature']],how='left',left_on='GameKey',right_on='GameKey')\nfig=plt.figure(figsize=[20,5])\nfig.add_subplot(131)\ntemp['Temperature'].value_counts().sort_index().plot()\nfig.add_subplot(132)\ntemp['Turf'].str.strip().str.lower().value_counts().plot.pie()\nfig.add_subplot(133)\ntemp['GameWeather'].str.strip().str.lower().value_counts().plot.pie()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.3 Player Position and Role\n","metadata":{}},{"cell_type":"code","source":"fig=plt.figure(figsize=[10,5])\nfig.add_subplot(121)\nvideo.merge(player_punt,left_on='GSISID',right_on='GSISID',how='left')[\"Position\"].value_counts().plot.pie()\nfig.add_subplot(122)\nvideo.merge(play_role,left_on='GSISID',right_on='GSISID',how='left')[\"Role\"].value_counts().plot.pie()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.4 Player and Partner Activity\n","metadata":{}},{"cell_type":"code","source":"playerA = video['Player_Activity_Derived'].value_counts().sort_index()\npartnerA = video['Primary_Partner_Activity_Derived'].value_counts().sort_index()\npd.DataFrame({'Player':playerA,'Partner':partnerA},index=partnerA.index).plot.bar()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.5 Turnover and Friendly Fire","metadata":{}},{"cell_type":"code","source":"fig=plt.figure(figsize=[10,5])\nfig.add_subplot(121)\nvideo[\"Turnover_Related\"].value_counts().plot.pie()\nfig.add_subplot(122)\nvideo[\"Friendly_Fire\"].value_counts().plot.pie()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.6 Primary Impact Type\n","metadata":{}},{"cell_type":"code","source":"video[\"Primary_Impact_Type\"].value_counts().plot.barh()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.7 Punt Type","metadata":{}},{"cell_type":"code","source":"xTick=[\"not_punted\",\"out_of_bounds\",\"downed\",\"touchback\",\"fair_catch\",\"returned\"]\nxidx=range(len(xTick))\nt=[]\nf=[]\n\nfor s in xTick:\n    v = video.merge(play_info,on=['GameKey','PlayID'],how='left')[s].value_counts().values\n    if len(v)>1:\n        t.append(v[1])\n    else:\n        t.append(0)\n    f.append(v[0])\n\np1=plt.bar(xidx,t)\np2=plt.bar(xidx,f,bottom=t)\nplt.xticks(xidx, xTick, rotation='vertical')\nplt.legend((p1[0],p2[0]),('True','False'),bbox_to_anchor=(1,.5))\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.8 Return Yard","metadata":{}},{"cell_type":"code","source":"play_info['yardage'] = play_info['PlayDescription'].str.extract(r'for (\\d.) yard')\nplay_info.loc[play_info['yardage'].isnull(),'yardage']=0\nplay_info.loc[play_info['returned'].isnull(),'yardage']=0\n\nvideo.merge(play_info,on=['GameKey','PlayID'],how='left').loc[~play_info['returned'].isnull(),'yardage'].value_counts().plot.barh()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.9 Punt Type Mapping","metadata":{}},{"cell_type":"code","source":"import holoviews as hv\nhv.extension('bokeh')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getEdges():\n    nodes1=['returned','downed','fair_catch']\n    nodes2=['no_returned','no_downed','no_fair_catch']\n    edges = [(x,y) for x in nodes1 for y in nodes2]\n    result=[]\n    d=video.merge(play_info,on=['GameKey','PlayID'],how='left')[['returned','downed','fair_catch']]\n    for (n1,n2) in edges:\n        n2_col=[x for x in nodes1 if n2.find(x)>-1][0]\n        if n1!=n2_col:\n            result.append([n1,n2,len(d.loc[(d[n1]==True)&(d[n2_col]==True)])])\n        else:\n            result.append([n1,n2,0])\n    return result\n\n\nedges=getEdges()\nsankey=hv.Sankey(edges)\ndisplay(sankey)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Clustering of NGS","metadata":{}},{"cell_type":"code","source":"# Load parq data\nngs = pd.read_parquet('../input/dsfg-ngs/NGS.parq')\nngs2 = ngs.groupby(['GameKey','PlayID','GSISID']).mean()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.1 PCA","metadata":{}},{"cell_type":"code","source":"from sklearn.decomposition import PCA\nfrom sklearn.preprocessing import StandardScaler, Normalizer","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca = PCA(n_components=4)\ntrain = StandardScaler().fit_transform(ngs2.values)\ntrain= Normalizer().fit_transform(train)\npca_result = pca.fit_transform(train)\nprint(pca.explained_variance_ratio_)\nplt.scatter(pca_result[:,0],pca_result[:,1])\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.2 TSNE","metadata":{}},{"cell_type":"code","source":"from sklearn.manifold import TSNE\nnp.random.seed(42)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N=1000\nrndperm = np.random.permutation(ngs2.shape[0])\nngs2_sub = ngs2.loc[rndperm[:N],:].copy()\ntrain = StandardScaler().fit_transform(ngs2_sub.values)\ntrain= Normalizer().fit_transform(train)\ntsne = TSNE(n_components=2, verbose=0, perplexity=40, n_iter=300)\ntsne_results = tsne.fit_transform(train)\nplt.scatter(tsne_results[:,0],tsne_results[:,1])\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.3 TSNE of ngs with consussion","metadata":{}},{"cell_type":"code","source":"df = video.merge(ngs2,on=['GameKey','PlayID','GSISID'],how='left')[['x','y','dis','o']]\nrndperm = np.random.permutation(ngs2.shape[0])\nngs2_sub = ngs2.iloc[rndperm[:N]][['x','y','dis','o']]\ntrain = pd.concat([ngs2_sub,df])\n\ntrain = StandardScaler().fit_transform(train.values)\ntrain= Normalizer().fit_transform(train)\ntsne = TSNE(n_components=2, verbose=0, perplexity=40, n_iter=300)\ntsne_results = tsne.fit_transform(train)\nplt.scatter(tsne_results[:N,0],tsne_results[:N,1],facecolor='blue')\nplt.scatter(tsne_results[N:,0],tsne_results[N:,1],facecolor='red')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}