{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Data Preprocessing\nThis notebook contains data preparation and preprocessing for my 2020 NCAA Analytics report. It includes:\n\n- Preparing the Kaggle Events and Player Datasets\n- Merging with existing data from Sports Reference\n- Creating a dataset where each row represents a player/season combination. Each of these rows contains data about the player for that given season."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pylab as plt\nimport seaborn as sns\nfrom tqdm.notebook import tqdm\n\nimport time\nsns.set(rc={'figure.figsize':(15, 5)})\npalette = sns.color_palette(\"bright\", 10)\n\npd.set_option('max_columns', 500)\npd.set_option('max_rows', 500)\npd.options.display.float_format = '{:.2f}'.format\n# plt.rcParams['figure.figsize'] = (15.0, 5.0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Player Event Data Prep"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"MENS_PBP_DIR = '../input/march-madness-analytics-2020/MPlayByPlay_Stage2'\n\nMPlayers = pd.read_csv(f'{MENS_PBP_DIR}/MPlayers.csv', error_bad_lines=False)\nMTeamSpelling = pd.read_csv('../input/march-madness-analytics-2020/MDataFiles_Stage2/MTeamSpellings.csv',\n                            engine='python')\nmens_events = []\nfor year in [2015, 2016, 2017, 2018, 2019]:\n    mens_events.append(pd.read_csv(f'{MENS_PBP_DIR}/MEvents{year}.csv'))\nMEvents = pd.concat(mens_events)\n\nMTeams = pd.read_csv('../input/march-madness-analytics-2020/MDataFiles_Stage2/MTeams.csv')\nMPlayers = MPlayers.merge(MTeams[['TeamID','TeamName']], on='TeamID', how='left')\n\narea_mapping = {0: np.nan,\n                1: 'under basket',\n                2: 'in the paint',\n                3: 'inside right wing',\n                4: 'inside right',\n                5: 'inside center',\n                6: 'inside left',\n                7: 'inside left wing',\n                8: 'outside right wing',\n                9: 'outside right',\n                10: 'outside center',\n                11: 'outside left',\n                12: 'outside left wing',\n                13: 'backcourt'}\n\nMEvents['Area_Name'] = MEvents['Area'].map(area_mapping)\n\n# Normalize X, Y positions for court dimentions\n# Court is 50 feet wide and 94 feet end to end.\nMEvents['X_'] = (MEvents['X'] * (94/100))\nMEvents['Y_'] = (MEvents['Y'] * (50/100))\n\n# Merge Player name onto events\nMEvents = MEvents.merge(MPlayers,\n              how='left',\n              left_on='EventPlayerID',\n              right_on='PlayerID')\n\n# Only Look at Events with Player assoicated\nMPlayerEvents = MEvents.query('EventPlayerID != 0')\n\n# Create GameId\nMPlayerEvents['GameId'] = \\\n    MPlayerEvents['Season'].astype('str') + '_' + \\\n    MPlayerEvents['DayNum'].astype('str') + '_' + \\\n    MPlayerEvents['WTeamID'].astype('str') + '_' + \\\n    MPlayerEvents['LTeamID'].astype('str')\n\n# Create Unique Player Season Combo\nEventPlayerSeasonCombo = MPlayerEvents[['EventPlayerID','Season']].drop_duplicates().reset_index(drop=True)\n\n# Expand MPlayers to have a row for each player\nMPlayerSeason = MPlayers.merge(EventPlayerSeasonCombo,\n               left_on=['PlayerID'],\n               right_on=['EventPlayerID'],\n              validate='1:m') \\\n    .drop('EventPlayerID', axis=1)\n\n\n# Create Player/Season Feature\nMPlayerSeason['PlayerID_Season'] = MPlayerSeason['PlayerID'].astype('int').astype('str') \\\n    + '_' + MPlayerSeason['Season'].astype('str')\nMPlayerEvents['PlayerID_Season'] = MPlayerEvents['PlayerID'].astype('int').astype('str') \\\n    + '_' + MPlayerEvents['Season'].astype('str')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create Features for Each Player/Season based on Events Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"#################################################\n# Add Features to MPlayers\n# - Each player/season combo has it's own \"stats\" as features\n#################################################\n\nMPlayerSeason['GameCount'] = MPlayerSeason['PlayerID_Season'].map(\n    MPlayerEvents \\\n    .groupby('PlayerID_Season')['GameId'] \\\n    .nunique() \\\n    .to_dict())\n\nevent_count = MPlayerEvents \\\n    .groupby('PlayerID_Season')['EventID'] \\\n    .count() \\\n    .to_dict()\nMPlayerSeason['EventCount'] = MPlayerSeason['PlayerID_Season'] \\\n    .map(event_count) # Number of Events for this player\n\n# Shot Counts\nMPlayerSeason['Miss3Count'] = MPlayerSeason['PlayerID_Season'].map( \n    MPlayerEvents.query('EventType == \"miss3\"') \\\n    .groupby('PlayerID_Season')['EventID'] \\\n    .count() \\\n    .to_dict())\nMPlayerSeason['Made3Count'] = MPlayerSeason['PlayerID_Season'].map( \n    MPlayerEvents.query('EventType == \"made3\"') \\\n    .groupby('PlayerID_Season')['EventID'] \\\n    .count() \\\n    .to_dict())\nMPlayerSeason['Miss2Count'] = MPlayerSeason['PlayerID_Season'].map( \n    MPlayerEvents.query('EventType == \"miss2\"') \\\n    .groupby('PlayerID_Season')['EventID'] \\\n    .count() \\\n    .to_dict())\nMPlayerSeason['Made2Count'] = MPlayerSeason['PlayerID_Season'].map( \n    MPlayerEvents.query('EventType == \"made2\"') \\\n    .groupby('PlayerID_Season')['EventID'] \\\n    .count() \\\n    .to_dict())\nMPlayerSeason['Miss1Count'] = MPlayerSeason['PlayerID_Season'].map( \n    MPlayerEvents.query('EventType == \"miss1\"') \\\n    .groupby('PlayerID_Season')['EventID'] \\\n    .count() \\\n    .to_dict())\nMPlayerSeason['Made1Count'] = MPlayerSeason['PlayerID_Season'].map( \n    MPlayerEvents.query('EventType == \"made1\"') \\\n    .groupby('PlayerID_Season')['EventID'] \\\n    .count() \\\n    .to_dict())\n\n# Shots Totals\nMPlayerSeason['3Count'] = MPlayerSeason['PlayerID_Season'].map( \n    MPlayerEvents.query('EventType == \"miss3\" or EventType == \"miss3\"') \\\n    .groupby('PlayerID_Season')['EventID'] \\\n    .count() \\\n    .to_dict())\nMPlayerSeason['2Count'] = MPlayerSeason['PlayerID_Season'].map( \n    MPlayerEvents.query('EventType == \"miss2\" or EventType == \"made2\"') \\\n    .groupby('PlayerID_Season')['EventID'] \\\n    .count() \\\n    .to_dict())\nMPlayerSeason['1Count'] = MPlayerSeason['PlayerID_Season'].map( \n    MPlayerEvents.query('EventType == \"miss1\" or EventType == \"made1\"') \\\n    .groupby('PlayerID_Season')['EventID'] \\\n    .count() \\\n    .to_dict())\n\nMPlayerSeason['ShotCount'] = MPlayerSeason['PlayerID_Season'].map( \n    MPlayerEvents \\\n    .query('EventType == \"miss3\" or EventType == \"miss3\" or EventType == \"miss2\" or EventType == \"made2\"') \\\n    .groupby('PlayerID_Season')['EventID'] \\\n    .count() \\\n    .to_dict())\n\n# Games Played by player\nMPlayerSeason['GameCount'] = MPlayerSeason['PlayerID_Season'].map(\n    MPlayerEvents \\\n    .groupby('PlayerID_Season')['GameId'] \\\n    .nunique() \\\n    .to_dict())\n\n# Areas where shots were taken\narea_list = ['inside left', 'backcourt', 'under basket', 'outside center',\n       'outside left', 'outside right', 'in the paint', 'inside right',\n       'inside center', 'outside left wing', 'outside right wing',\n       'inside left wing', 'inside right wing']\n\n# Shot Counts by Area\nfor a in area_list:\n    MPlayerSeason['Miss3Count'+a] = MPlayerSeason['PlayerID_Season'].map( \n        MPlayerEvents.query('EventType == \"miss3\" and Area_Name == @a') \\\n        .groupby('PlayerID_Season')['EventID'] \\\n        .count() \\\n        .to_dict())\n    MPlayerSeason['Made3Count'+a] = MPlayerSeason['PlayerID_Season'].map( \n        MPlayerEvents.query('EventType == \"made3\" and Area_Name == @a') \\\n        .groupby('PlayerID_Season')['EventID'] \\\n        .count() \\\n        .to_dict())\n    MPlayerSeason['Miss2Count'+a] = MPlayerSeason['PlayerID_Season'].map( \n        MPlayerEvents.query('EventType == \"miss2\" and Area_Name == @a') \\\n        .groupby('PlayerID_Season')['EventID'] \\\n        .count() \\\n        .to_dict())\n    MPlayerSeason['Made2Count'+a] = MPlayerSeason['PlayerID_Season'].map( \n        MPlayerEvents.query('EventType == \"made2\" and Area_Name == @a') \\\n        .groupby('PlayerID_Season')['EventID'] \\\n        .count() \\\n        .to_dict())\n    MPlayerSeason['ShotCount'+a] = MPlayerSeason['PlayerID_Season'].map( \n        MPlayerEvents.query('Area_Name == @a') \\\n        .query('EventType == \"made2\" or EventType == \"miss2\" or EventType == \"made3\" or EventType == \"miss3\"')\n        .groupby('PlayerID_Season')['EventID'] \\\n        .count() \\\n        .to_dict())\n    \n    \n# Make Everything with respect to percent of games\nstat_cols = ['Miss3Count', 'Made3Count', 'Miss2Count', 'Made2Count', 'Miss1Count',\n               'Made1Count', 'Miss3Countinside left',\n               'Made3Countinside left', 'Miss2Countinside left',\n               'Made2Countinside left', 'Miss3Countbackcourt',\n               'Made3Countbackcourt',\n               'Miss2Countbackcourt', 'Made2Countbackcourt',\n               'Miss3Countunder basket',\n               'Made3Countunder basket', 'Miss2Countunder basket',\n               'Made2Countunder basket', 'Miss3Countoutside center',\n               'Made3Countoutside center', 'Miss2Countoutside center',\n               'Made2Countoutside center', 'Miss3Countoutside left',\n               'Made3Countoutside left', 'Miss2Countoutside left',\n               'Made2Countoutside left', 'Miss3Countoutside right',\n               'Made3Countoutside right', 'Miss2Countoutside right',\n               'Made2Countoutside right', 'Miss3Countin the paint',\n               'Made3Countin the paint', 'Miss2Countin the paint',\n               'Made2Countin the paint', 'Miss3Countinside right',\n               'Made3Countinside right', 'Miss2Countinside right',\n               'Made2Countinside right', 'Miss3Countinside center',\n               'Made3Countinside center', 'Miss2Countinside center',\n               'Made2Countinside center', 'Miss3Countoutside left wing',\n               'Made3Countoutside left wing', 'Miss2Countoutside left wing',\n               'Made2Countoutside left wing', 'Miss3Countoutside right wing',\n               'Made3Countoutside right wing', 'Miss2Countoutside right wing',\n               'Made2Countoutside right wing', 'Miss3Countinside left wing',\n               'Made3Countinside left wing', 'Miss2Countinside left wing',\n               'Made2Countinside left wing', 'Miss3Countinside right wing',\n               'Made3Countinside right wing', 'Miss2Countinside right wing',\n               'Made2Countinside right wing', '3Count', '2Count', '1Count',\n               'ShotCount', 'ShotCountinside left', 'ShotCountbackcourt',\n               'ShotCountunder basket', 'ShotCountoutside center',\n               'ShotCountoutside left', 'ShotCountoutside right',\n               'ShotCountin the paint', 'ShotCountinside right',\n               'ShotCountinside center', 'ShotCountoutside left wing',\n               'ShotCountoutside right wing', 'ShotCountinside left wing',\n               'ShotCountinside right wing']\n\nfor s in tqdm(stat_cols):\n    MPlayerSeason[s+'_per_game'] = MPlayerSeason[s] / MPlayerSeason['GameCount']\n\nMPlayerSeason = MPlayerSeason.fillna(0)\n    \n# Filter Down Player Data to ones with acutal playing time\nMPlayers_filtered = MPlayerSeason.query('GameCount > 5 and EventCount >= 0').copy()\n\n# List of features for players by \"per game\"\nper_game_cols = [c for c in MPlayers_filtered.columns if 'per_game' in c]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Merge Kaggle Data with Sports Reference Data\n- Created in this notebook: https://www.kaggle.com/robikscube/march-madness-2020-sports-reference-com-scrape\n- Available in this dataset: https://www.kaggle.com/robikscube/ncaa-data-20152019-sportsreference"},{"metadata":{"trusted":true},"cell_type":"code","source":"SRRoster = pd.read_csv('../input/ncaa-data-20152019-sportsreference/SRRosters.csv')\nSRPerGameStats = pd.read_csv('../input/ncaa-data-20152019-sportsreference/SRPerGameStats.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SRPerGameStats = SRPerGameStats.drop([20303, 20563]).reset_index(drop=True)\nSRRoster = SRRoster.drop([20303, 20567]).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SRRosterStats = SRRoster.merge(SRPerGameStats, on=['Player','Season','TeamID','TeamName',\n                                   'FirstName','LastName','AdditionalName'],\n               validate='1:1')\n\nMPlayers_filtered = MPlayerSeason.query('GameCount > 5 and EventCount >= 0').copy()\nMPlayers['Player'] = MPlayers['FirstName'] + ' ' + MPlayers['LastName']\nMPlayerSeason['Player'] = MPlayerSeason['FirstName'] + ' ' + MPlayerSeason['LastName']\nMPlayers_filtered['Player'] = MPlayers_filtered['FirstName'] + ' ' + MPlayers_filtered['LastName']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Merge based on fuzzy matching of names\n- Some manual cleaning was required to account for matching inconsistent spelling and abbreviations of names."},{"metadata":{"trusted":true},"cell_type":"code","source":"MPlayers_filtered['Player_lower'] = MPlayers_filtered['Player'] \\\n    .str.lower() \\\n    .str.replace('.','') \\\n    .str.replace('jr','') \\\n    .str.replace('sr','') \\\n    .str.replace('jr.','') \\\n    .str.replace('sr.','') \\\n    .str.replace('iii','') \\\n    .str.replace('swan-ford','swan') \\\n    .str.replace('christopher','chris') \\\n    .str.replace('rakiya','rakia') \\\n    .str.replace('tracey','tracy') \\\n    .str.replace(' ','') \\\n    .str.replace('michael','mike') \\\n    .str.replace('reginald','reginal') \\\n    .str.replace('herbert','herb') \\\n    .str.replace('freeman-daniels', 'daniels') \\\n    .str.replace('allerik', 'al') \\\n    .str.replace('tjmaston', 'terrymaston') \\\n    .str.replace('coreybarnes','jrbarnes') \\\n    .str.replace('lorencristian','loren') \\\n    .str.replace('peewee','darius') \\\n    .str.replace(\"joe'randlecunningham\",\"joe'randletoliver\") \\\n    .str.replace(\"natewells\",\"wellsnate\") \\\n    .str.replace('mrdjangasevic','gasevicmrdjan') \\\n    .str.replace('letrellviser','snoopviser') \\\n    .str.replace('alexennis','ennisalex') \\\n    .str.replace('azariahsykes','sykesazariah') \\\n    .str.replace('stanleydavis','davisstanley') \\\n    .str.replace('byerstyjhai','tyjhaibyers') \\\n    .str.replace('ocheneyoleakuwovo','akuwovoogheneyole') \\\n    .str.replace('baebaedaniels','demarcusdaniels') \\\n    .str.replace('tarekeyi','tk') \\\n    .str.replace('elishaboone','booneelisha') \\\n    .str.replace(\"ja'kwanjones\",'slinkyjones') \\\n    .str.replace('rodneyhawkins','hawkinsrodney') \\\n    .str.replace('patrickkirksey','kenshaykirksey') \\\n    .str.replace('maxhoetzel','maxmontana') \\\n    .str.replace('jahmelbodrick','bodrickjahmel') \\\n    .str.replace('giacomozilli','zilligiacomo') \\\n    .str.replace('cameroncurry','cc')\n\nSRRosterStats['Player_lower'] = SRRosterStats['Player'] \\\n    .str.lower() \\\n    .str.replace('.','') \\\n    .str.replace('jr','') \\\n    .str.replace('sr','') \\\n    .str.replace('jr.','') \\\n    .str.replace('sr.','') \\\n    .str.replace('iii','') \\\n    .str.replace('swan-ford','swan') \\\n    .str.replace('christopher','chris') \\\n    .str.replace('rakiya','rakia') \\\n    .str.replace('tracey','tracy') \\\n    .str.replace(' ','') \\\n    .str.replace('michael','mike') \\\n    .str.replace('reginald','reginal') \\\n    .str.replace('herbert','herb') \\\n    .str.replace('freeman-daniels', 'daniels') \\\n    .str.replace('allerik', 'al') \\\n    .str.replace('tjmaston', 'terrymaston') \\\n    .str.replace('coreybarnes','jrbarnes') \\\n    .str.replace('lorencristian','loren') \\\n    .str.replace('peewee','darius') \\\n    .str.replace(\"joe'randlecunningham\",\"joe'randletoliver\") \\\n    .str.replace(\"natewells\",\"wellsnate\") \\\n    .str.replace('mrdjangasevic','gasevicmrdjan') \\\n    .str.replace('letrellviser','snoopviser') \\\n    .str.replace('alexennis','ennisalex') \\\n    .str.replace('azariahsykes','sykesazariah') \\\n    .str.replace('stanleydavis','davisstanley') \\\n    .str.replace('byerstyjhai','tyjhaibyers') \\\n    .str.replace('ocheneyoleakuwovo','akuwovoogheneyole') \\\n    .str.replace('baebaedaniels','demarcusdaniels') \\\n    .str.replace('tarekeyi','tk') \\\n    .str.replace('elishaboone','booneelisha') \\\n    .str.replace(\"ja'kwanjones\",'slinkyjones') \\\n    .str.replace('rodneyhawkins','hawkinsrodney') \\\n    .str.replace('patrickkirksey','kenshaykirksey') \\\n    .str.replace('maxhoetzel','maxmontana') \\\n    .str.replace('jahmelbodrick','bodrickjahmel') \\\n    .str.replace('giacomozilli','zilligiacomo') \\\n    .str.replace('cameroncurry','cc')\n\nMPlayers_filtered_roster = MPlayers_filtered \\\n        .merge(SRRosterStats,\n               how='left',\n               on=['Player_lower','TeamID','Season'],\n               suffixes=('',f'_{s}'),\n               validate='m:1')\n\n# Remove Positions that are not common and make more general\nMPlayers_filtered_roster['Pos'] = MPlayers_filtered_roster['Pos'].str.replace('PG','G') \\\n    .replace('SF','F') \\\n    .replace('SG','G') \\\n    .replace('PF','F')\n\ncount_not_merged = MPlayers_filtered_roster['Pos'].isna().sum()\ncount_playerseason = len(MPlayers_filtered_roster)\nprint(f'{count_not_merged} of {count_playerseason} not merged')\nprint(f'{count_not_merged*100/count_playerseason:0.4f}% missing')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Fuzzy matching of names round two\n# These remaining ones that don't match use the second closest match\nfrom difflib import get_close_matches\n\n\nplayers_unmatched_df = MPlayers_filtered_roster.loc[MPlayers_filtered_roster['Pos'].isna()][['Player_lower','TeamName','Season']]\n\nfuzzy_match_dict = {}\nfor i, row in players_unmatched_df.iterrows():\n    p = row['Player_lower']\n    s = row['Season']\n    t = row['TeamName']\n    roster_names = SRRosterStats.query('Season == @s and TeamName == @t')['Player_lower'].unique().tolist()\n    try:\n        closest_match = get_close_matches(p, roster_names)[0]\n        # print(p, closest_match)\n        fuzzy_match_dict[p] = closest_match\n    except:\n        pass\n#             print(f'Broke for {p}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Include Fuzzy Matching\nMPlayers_filtered['Player_lower'] = MPlayers_filtered['Player'] \\\n    .str.lower() \\\n    .str.replace('.','') \\\n    .str.replace('jr','') \\\n    .str.replace('sr','') \\\n    .str.replace('jr.','') \\\n    .str.replace('sr.','') \\\n    .str.replace('iii','') \\\n    .str.replace('swan-ford','swan') \\\n    .str.replace('christopher','chris') \\\n    .str.replace('rakiya','rakia') \\\n    .str.replace('tracey','tracy') \\\n    .str.replace(' ','') \\\n    .str.replace('michael','mike') \\\n    .str.replace('reginald','reginal') \\\n    .str.replace('herbert','herb') \\\n    .str.replace('freeman-daniels', 'daniels') \\\n    .str.replace('allerik', 'al') \\\n    .str.replace('tjmaston', 'terrymaston') \\\n    .str.replace('coreybarnes','jrbarnes') \\\n    .str.replace('lorencristian','loren') \\\n    .str.replace('peewee','darius') \\\n    .str.replace(\"joe'randlecunningham\",\"joe'randletoliver\") \\\n    .str.replace(\"natewells\",\"wellsnate\") \\\n    .str.replace('mrdjangasevic','gasevicmrdjan') \\\n    .str.replace('letrellviser','snoopviser') \\\n    .str.replace('alexennis','ennisalex') \\\n    .str.replace('azariahsykes','sykesazariah') \\\n    .str.replace('stanleydavis','davisstanley') \\\n    .str.replace('byerstyjhai','tyjhaibyers') \\\n    .str.replace('ocheneyoleakuwovo','akuwovoogheneyole') \\\n    .str.replace('baebaedaniels','demarcusdaniels') \\\n    .str.replace('tarekeyi','tk') \\\n    .str.replace('elishaboone','booneelisha') \\\n    .str.replace(\"ja'kwanjones\",'slinkyjones') \\\n    .str.replace('rodneyhawkins','hawkinsrodney') \\\n    .str.replace('patrickkirksey','kenshaykirksey') \\\n    .str.replace('maxhoetzel','maxmontana') \\\n    .str.replace('jahmelbodrick','bodrickjahmel') \\\n    .str.replace('giacomozilli','zilligiacomo') \\\n    .str.replace('cameroncurry','cc')\n\nMPlayers_filtered['Player_lower'] = MPlayers_filtered['Player_lower'].replace(fuzzy_match_dict)\n\nSRRosterStats['Player_lower'] = SRRosterStats['Player'] \\\n    .str.lower() \\\n    .str.replace('.','') \\\n    .str.replace('jr','') \\\n    .str.replace('sr','') \\\n    .str.replace('jr.','') \\\n    .str.replace('sr.','') \\\n    .str.replace('iii','') \\\n    .str.replace('swan-ford','swan') \\\n    .str.replace('christopher','chris') \\\n    .str.replace('rakiya','rakia') \\\n    .str.replace('tracey','tracy') \\\n    .str.replace(' ','') \\\n    .str.replace('michael','mike') \\\n    .str.replace('reginald','reginal') \\\n    .str.replace('herbert','herb') \\\n    .str.replace('freeman-daniels', 'daniels') \\\n    .str.replace('allerik', 'al') \\\n    .str.replace('tjmaston', 'terrymaston') \\\n    .str.replace('coreybarnes','jrbarnes') \\\n    .str.replace('lorencristian','loren') \\\n    .str.replace('peewee','darius') \\\n    .str.replace(\"joe'randlecunningham\",\"joe'randletoliver\") \\\n    .str.replace(\"natewells\",\"wellsnate\") \\\n    .str.replace('mrdjangasevic','gasevicmrdjan') \\\n    .str.replace('letrellviser','snoopviser') \\\n    .str.replace('alexennis','ennisalex') \\\n    .str.replace('azariahsykes','sykesazariah') \\\n    .str.replace('stanleydavis','davisstanley') \\\n    .str.replace('byerstyjhai','tyjhaibyers') \\\n    .str.replace('ocheneyoleakuwovo','akuwovoogheneyole') \\\n    .str.replace('baebaedaniels','demarcusdaniels') \\\n    .str.replace('tarekeyi','tk') \\\n    .str.replace('elishaboone','booneelisha') \\\n    .str.replace(\"ja'kwanjones\",'slinkyjones') \\\n    .str.replace('rodneyhawkins','hawkinsrodney') \\\n    .str.replace('patrickkirksey','kenshaykirksey') \\\n    .str.replace('maxhoetzel','maxmontana') \\\n    .str.replace('jahmelbodrick','bodrickjahmel') \\\n    .str.replace('giacomozilli','zilligiacomo') \\\n    .str.replace('cameroncurry','cc')\n\nSRRosterStats['Player_lower'] = SRRosterStats['Player_lower'].replace(fuzzy_match_dict)\n\nMPlayers_filtered_roster = MPlayers_filtered \\\n        .merge(SRRosterStats,\n               how='left',\n               on=['Player_lower','TeamID','Season'],\n               suffixes=('',f'_{s}'),\n               validate='m:1')\n\n# Remove Positions that are not common and make more general\nMPlayers_filtered_roster['Pos'] = MPlayers_filtered_roster['Pos'].str.replace('PG','G') \\\n    .replace('SF','F') \\\n    .replace('SG','G') \\\n    .replace('PF','F')\n\ncount_not_merged = MPlayers_filtered_roster['Pos'].isna().sum()\ncount_playerseason = len(MPlayers_filtered_roster)\nprint(f'{count_not_merged} of {count_playerseason} not merged')\nprint(f'{count_not_merged*100/count_playerseason:0.4f}% missing')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def parse_height(df):\n    df['HeightInches'] = (pd.to_numeric(df['Height'].str.split('-', expand=True)[0]) * 12) + \\\n        (pd.to_numeric(df['Height'].str.split('-', expand=True)[1]))\n\n    df['HeightFeet'] = (pd.to_numeric(df['Height'].str.split('-', expand=True)[0])) + \\\n        (pd.to_numeric(df['Height'].str.split('-', expand=True)[1]) / 12)    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MPlayers_filtered_roster = parse_height(MPlayers_filtered_roster)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Final Dataset and Save Results"},{"metadata":{"trusted":true},"cell_type":"code","source":"MPlayers_filtered_roster.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MPlayers_filtered_roster.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MPlayers_filtered_roster.to_csv('MPlayerSeasonStats.csv',\n                                 index=False)\nMPlayers_filtered_roster.to_parquet('MPlayerSeasonStats.parquet')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}