{"cells":[{"metadata":{"_uuid":"0f350fa13a7af6b94932c8d249a3ac1211cd841f"},"cell_type":"markdown","source":"# Preprocessing\n## Thompson Bliss and Connor Daly\n\nThis notebook takes the provided data and transforms it into an appropriate form for a logistic regression model."},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np # Linear algebra operations\nimport pandas as pd # Data processing\nimport random # Random sampling of data\nimport datetime as dt # Formatting dates and times","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d95504dc2c499c1038f1f8fa983ad182f0e94d9f"},"cell_type":"markdown","source":"## Section 1: Importing Data\nIn this section we import Next Gen Stats (NGS) and game data while removing unnecessary observations."},{"metadata":{"_uuid":"2aaa57225af02e47fd2b9d02ca11470e812e83c7","trusted":true},"cell_type":"code","source":"def loadAndRemove(s):\n    '''\n    In this project, we are interested in plays (and therefore injuries) that occur downfield, the times of which lie between \n    the time of the punt and the time of the play completion.\n    \n    This function reads in a dataset and removes observations in which the time interval between the snap and the punt is\n    small. Thus, we keep observations about plays that occur downfield, and discard observations about plays that occur at\n    the line of scrimmage or after completion of the play.\n    \n    Arguments:\n        s: A file name, entered as a string.\n    \n    Returns:\n        df: A dataframe which is stripped of any play occurring prior to the punt or after play completion.\n    '''    \n    # Remove NAs\n    df = pd.read_csv(s, low_memory=False).dropna(subset=['Season_Year', 'GameKey', 'PlayID', 'GSISID', 'Time', 'x', 'y', 'dis', 'o', 'dir'])\n    \n    # Convert Time variable (originally a string) to a datetime object\n    df.Time = pd.to_datetime(df.Time)\n    # Create GamePlayKey to denote the game ID and play ID of each observation\n    df['GamePlayKey'] = df.GameKey.astype(str) + '_' + df.PlayID.astype(str)\n    # Create seconds variable, which holds the number of elapsed seconds since January 1, 2015 at 12am\n    df['seconds'] = (df['Time'] - dt.datetime(2015,1,1)).dt.total_seconds()\n    \n    # Create punt_seconds column that holds the time at which the ball was punted\n    # We group by GamePlayKey & then merge punt_seconds based on when Event \"punt\" occurs within the play\n    df = pd.merge(df, df.groupby('GamePlayKey').apply(lambda x: x.loc[x.Event == 'punt'].seconds.mean()).dropna().to_frame(name = 'punt_seconds'),  on = ['GamePlayKey'], how ='outer')\n    \n    # Create play_end_seconds column that holds the num of seconds at which the play is considered completed\n    # We group by GamePlayKey and then merge play_end_seconds based on the first time point at which an event occurs that signals the end of the play\n    # We considered \"play ending events\" to be the following:\n        # fair_catch, tackle, safety, touchback, out_of_bounds, punt_downed, touchdown, fumble_defense_recovered, pass_outcome_incomplete\n    df = pd.merge(df, df.groupby('GamePlayKey').apply(lambda x: x.loc[(x.Event == 'fair_catch') | (x.Event == 'tackle') | (x.Event == 'safety') | (x.Event == 'touchback') | (x.Event == 'out_of_bounds') | (x.Event == 'punt_downed') | (x.Event == 'touchdown') | (x.Event == 'fumble_defense_recovered') | (x.Event == 'pass_outcome_incomplete')].seconds.min()).dropna().to_frame(name = 'play_end_seconds'),  on = ['GamePlayKey'], how ='outer')\n    \n    # Keep only observations that occur after the ball has been punted\n    df = df.loc[df.seconds > df.punt_seconds]\n    # Keep only observations that occur before the completion of play\n    df = df.loc[df.seconds < df.play_end_seconds]\n    \n    # Create post_punt_playduration, which denotes the elapsed time (in seconds) between the punt and the completion of play\n    df['post_punt_playduration'] = df.play_end_seconds - df.punt_seconds\n    # Create post_punt_duration, which denotes the elapsed time (in seconds) between the punt and the occurrence of the observation\n    df['post_punt_duration'] = df.seconds - df.punt_seconds\n\n    return df","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5df756885a00f876b6f229e4a47a2fb5559f4721","trusted":true},"cell_type":"code","source":"# Import Next Gen Stats data in a way that saves RAM; load all data into dataframe ngs\nngs = loadAndRemove('../input/NGS-2016-reg-wk13-17.csv')\nngs = ngs.append(loadAndRemove('../input/NGS-2017-reg-wk1-6.csv'))\nngs = ngs.append(loadAndRemove('../input/NGS-2017-reg-wk7-12.csv'))\nngs = ngs.append(loadAndRemove('../input/NGS-2017-pre.csv'))\nngs = ngs.append(loadAndRemove('../input/NGS-2017-reg-wk13-17.csv'))\nngs = ngs.append(loadAndRemove('../input/NGS-2016-pre.csv'))\nngs = ngs.append(loadAndRemove('../input/NGS-2016-reg-wk7-12.csv'))\nngs = ngs.append(loadAndRemove('../input/NGS-2017-post.csv'))\nngs = ngs.append(loadAndRemove('../input/NGS-2016-reg-wk1-6.csv'))\nngs = ngs.drop(['seconds','punt_seconds','play_end_seconds'], axis = 1)\n\n# Import other data sources\ndf_gamedata = pd.read_csv('../input/game_data.csv')\ndf_videoreview = pd.read_csv('../input/video_review.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"97330b05c2a3d4e7566782d8887c3f8795c367e0"},"cell_type":"markdown","source":"## Section 2: Transforming Data\n\nIn this section, we first add keys which will be used to join disparate datasets. Then, we create a function that merges a player's data with that of another player into a single row if the distance between the players' locations is small enough to be considered contact."},{"metadata":{"_uuid":"97765ceb369e889f6f40795cb14a59f1a421c3eb","trusted":true},"cell_type":"code","source":"# Create GamePlayKey based on the unique Game ID and the unique Player ID\ndf_videoreview = df_videoreview.assign(GamePlayKey = df_videoreview.GameKey.astype(str) + '_' + df_videoreview.PlayID.astype(str))\n# Create GamePlayTimeKey, which is a unique ID for every time point for which there exists recorded data\nngs['GamePlayTimeKey'] = ngs['GamePlayKey'].astype(str) + '_' + ngs['Time'].astype(str)\n\n#Removing rows with missign values from videoreview\ndf_videoreview = df_videoreview.dropna()\ndf_videoreview = df_videoreview.reset_index()\ndf_videoreview = df_videoreview.drop(['index'], axis = 1)\n\n#adding features for referencing\ndf_videoreview['PlayPlayerPartnerID_1'] = df_videoreview['PlayID'].astype(str) + '_' + df_videoreview['GSISID'].astype(str) + '_' + df_videoreview['Primary_Partner_GSISID'].astype(str)\n\ndf_videoreview['PlayPlayerPartnerID_2'] = df_videoreview['PlayID'].astype(str) + '_' + df_videoreview['Primary_Partner_GSISID'].astype(str) + '_' + df_videoreview['GSISID'].astype(str)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4df2194379eb63a13397770bd65f67ee335d2026","trusted":true},"cell_type":"code","source":"def createData(df_create, contactDist):\n    '''\n    This function iterates through player data to create a new dataframe. Each row in the new dataframe consists of player data\n    and contact partner data, only if the two players are deemed to have made contact with each other during the play.\n    \n    Args:\n        df_create: A pandas dataframe that has the same features as the ngs dataframe.\n        contactDist: A value (float or integer) that represents the maximum distance two players can be located from each other\n        in order for there to be considered contact between them.\n    \n    Returns:\n        df_output: A pandas dataframe that has double the number of rows as df_create. Features in df_output consist of the same\n        features as df_create, as well as new feature variable names for the contact partner that are analogous to the features\n        for the primary player, represented with a \"_partner\" suffix on the column name. Every row in df_output represents an\n        instance of contact between the player (represented by the first half of the columns) and his primary partner (represented\n        by the second half of the columns).\n    '''\n    # Instantiate the output dataframe, to which we will continually append rows\n    df_output = pd.DataFrame()\n\n    # For every unique combination of Game, Play, and Time in df_create\n    for key in (df_create.GamePlayTimeKey.unique()):\n        # Filter df_create to include only the current key of interest, saving in a new dataframe called df_temp\n        df_temp = df_create.loc[df_create['GamePlayTimeKey'] == key]\n        # For every row in df_temp\n        for i in range(len(df_temp)):\n            # For every subsequent row in df_temp\n            for j in range(i+1, len(df_temp)):\n                # Compute the distance between the locations of player and contact partner\n                # Then determine if this distance is less than contactDist\n                if (((df_temp.iloc[i]['x'] - df_temp.iloc[j]['x']) ** 2 +\n                    (df_temp.iloc[i]['y'] - df_temp.iloc[j]['y']) ** 2) ** 0.5 <= contactDist):\n                    # Merge the df_temp data of the player and the contact partner if they contacted each other into a single row\n                    # Add this new row into a new dataframe called df_temp2\n                    df_temp2 = pd.merge(df_temp.iloc[i].to_frame().transpose(),\n                                       df_temp.iloc[j].to_frame().transpose(),\n                                       suffixes = ['','_partner'], on='GamePlayTimeKey') # Add a suffix to indicate contact partner feature\n                    \n                    # Drop unnecessary features\n                    df_temp2 = df_temp2.drop(['Season_Year_partner', 'GameKey_partner', 'PlayID_partner', 'Time_partner', 'Event_partner', 'GamePlayKey_partner', 'post_punt_playduration_partner', 'post_punt_duration_partner'], axis = 1)\n                    # Append all rows from df_temp2 to df_output. df_temp2 will then be automatically overwritten during the next iteration\n                    df_output = df_output.append(pd.DataFrame(data=df_temp2), ignore_index = True)\n    \n    # Ensure there is some data stored in df_output\n    if len(df_output) > 0:\n        # Convert GameKey identifier to an integer\n        df_output.GameKey = df_output.GameKey.astype(int)\n        # Merge df_output with dataframe df_gamedata based on the GameKey ID\n        df_output = df_output.merge(df_gamedata, on = ['GameKey'], suffixes = ['','_gameData']) \n    \n        # If there are any duplicate rows in df_output, this means that players contacted each other for a considerable amount of time\n        # Drop such observations to avoid redundancy\n        df_output = df_output.drop_duplicates(subset = ['GSISID', 'GSISID_partner', 'GamePlayKey'])\n        \n        # Drop redundant columns, since such columns were created for both the primary player and the contact partner\n        df_output = df_output.drop(['Season_Year_gameData', 'Home_Team', 'Visit_Team', \"HomeTeamCode\",\"VisitTeamCode\", \"Stadium\",\"Game_Site\",\"OutdoorWeather\", \"StadiumType\", \"GameWeather\", \"Turf\"], axis = 1)\n    \n    return df_output","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b6b74c7a4ef9767ef123e04cfc26e352f8d88533"},"cell_type":"markdown","source":"## Section 3: Exporting Data\nIn this section we use the `createData` function as produced in Section 2 to create data and export it for use in the Classification notebook."},{"metadata":{"trusted":true,"_uuid":"86705707a9eab4fc275fbf8f8fe9d67f5a3cb4d5"},"cell_type":"code","source":"# We first take a random sample of data from ngs dataframe\n# Set a random seed to ensure reproducibility of results\nrandom.seed(53)\n\n# Define contactDist, which determines how far apart two players can be for contact to be considered\ncontactDist = 0.5\n# Define the number of GamePlayTimeKeys that are to be taken as a sample from ngs dataframe\nlength = 1500\n# Obtain the sample which consists of 1500 (length) unique keys\nsamp = random.sample(list(ngs.GamePlayTimeKey.unique()), length)\n# Filter dataframe ngs based on the samp criteria as defined above\nngs_sample = ngs.loc[ngs.GamePlayTimeKey.isin(samp)]\n\n# Create the data based on the ngs_sample dataframe and the defined contactDist\ndf = createData(ngs_sample, contactDist)\n\n# Create PlayPlayerPartnerID variable that serves as a key to indicate the two players that make contact with each other\ndf['PlayPlayerPartnerID'] = df['PlayID'].astype(str) + '_' + df['GSISID'].astype(int).astype(str) + '_' + df['GSISID_partner'].astype(int).astype(str)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b711529dca21da4d528a4535e7f04a5084450c51"},"cell_type":"code","source":"# Now we take a sample from plays taking sample from plays in video review\n# This dataset consists of identifiable plays that are associated with concussions\n\n# Filter dataframe ngs based on all GamePlayKeys that are in dataframe df_videoreview\nngs_sample = ngs.loc[ngs.GamePlayKey.isin(df_videoreview.GamePlayKey)]\n# Create the data based on the new ngs_sample dataframe\ndf2 = createData(ngs_sample, contactDist)\n\n# Create PlayPlayerPartnerID variable that serves as a key to indicate the two players that make contact with each other\ndf2['PlayPlayerPartnerID'] = df2['PlayID'].astype(str) + '_' + df2['GSISID'].astype(int).astype(str) + '_' + df2['GSISID_partner'].astype(int).astype(str)\n\n# Remove plays from the created dataframe that were video-reviewed, but were not associated with concussions\n# This is to ensure that we do not oversample from the data\ndf2 = df2.loc[df2.PlayPlayerPartnerID.isin(df_videoreview.PlayPlayerPartnerID_1) | df2.PlayPlayerPartnerID.isin(df_videoreview.PlayPlayerPartnerID_2)]\n\n# Append this newly created dataframe to the dataframe produced from the first run of createData\ndf = df.append(df2)\n\n# Export data to a CSV file which will be used as input for the Classification notebook\ndf.to_csv('x.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d642e42a21dcd48f65a46ba109eb758c50008e5b"},"cell_type":"code","source":"# Create y data (i.e. dependent variable data) and export it as a CSV file\ny = df.PlayPlayerPartnerID.isin(df_videoreview.PlayPlayerPartnerID_1) | df.PlayPlayerPartnerID.isin(df_videoreview.PlayPlayerPartnerID_2)\ny.to_csv('y.csv', index = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}