{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport os\nimport itertools\nimport pandas as pd\nimport datatable as dt\nimport numpy as np\nimport mlb\nimport pickle as pkl\nfrom tqdm import tqdm\nfrom itertools import product\nimport lightgbm as lgb\nfrom fuzzywuzzy import fuzz\nimport re\nimport numba as nb\nfrom numba import njit\nimport xgboost as xgb\n\n@njit\ndef nb_cumsum(arr):\n return arr.cumsum()\n\n@njit\ndef nb_sum(arr):\n return arr.sum()\n\ndef reduce_mem_usage(df, verbose=True):\n numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n start_mem = df.memory_usage().sum() / 1024**2\n for col in df.columns:\n col_type = df[col].dtypes\n if col_type in numerics:\n c_min = df[col].min()\n c_max = df[col].max()\n if str(col_type)[:3] == 'int':\n if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n df[col] = df[col].astype(np.int16)\n elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n df[col] = df[col].astype(np.int32)\n elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n df[col] = df[col].astype(np.int64)\n elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n df[col] = df[col].astype(np.int64)\n else:\n if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n df[col] = df[col].astype(np.float32)\n elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n df[col] = df[col].astype(np.float64)\n else:\n df[col] = df[col].astype(np.float64)\n end_mem = df.memory_usage().sum() / 1024**2\n if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n return df\n\n\ndef unnest(data, name):\n try:\n date_nested_table = data[['date', name]]\n\n date_nested_table = (date_nested_table[\n ~pd.isna(date_nested_table[name])\n ].\n reset_index(drop = True)\n )\n\n daily_dfs_collection = []\n\n for date_index, date_row in date_nested_table.iterrows():\n daily_df = pd.read_json(date_row[name])\n\n daily_df['dailyDataDate'] = date_row['date']\n\n daily_dfs_collection = daily_dfs_collection + [daily_df]\n\n if daily_dfs_collection:\n # Concatenate all daily dfs into single df for each row\n unnested_table = (pd.concat(daily_dfs_collection,\n ignore_index = True).\n # Set and reset index to move 'dailyDataDate' to front of df\n set_index('dailyDataDate').\n reset_index()\n )\n return reduce_mem_usage(unnested_table, False)\n else:\n return pd.DataFrame()\n except Exception as e:\n print(e)\n print(f'unnest failed for {name}. returning empty dataframe')\n return pd.DataFrame()\n\n\ndef get_unnested_data_dict(data, daily_data_nested_df_names):\n df_dict = {}\n for df_name in daily_data_nested_df_names:\n df_dict[df_name] = unnest(data, df_name)\n return df_dict\n\ndef get_unnested_data(data, colnames):\n return (unnest(data, df_name) for df_name in colnames)\n\n\n## Find win expectancy and volatility given inning, out, base, run situation.\n\n## no. of runs that score with HR in diff. base situations\nbaseHr = {1: 1,\n 2: 2,\n 3: 2,\n 4: 3,\n 5: 2,\n 6: 3,\n 7: 3,\n 8: 4\n }\n \ntangoRunExp = {'60': {1: 0.51400000000000001, 2: 0.19400000000000001, 3: 0.14999999999999999, 4: 0.076999999999999999, 5: 0.036999999999999998, 6: 0.017000000000000001, 7: 0.0060000000000000001, 8: 0.0030000000000000001, 9: 0.001, 10: 0.001, 'm': -0.216, 'b': 0.247}, '61': {1: 0.59599999999999997, 2: 0.17599999999999999, 3: 0.13200000000000001, 4: 0.057000000000000002, 5: 0.024, 6: 0.0089999999999999993, 7: 0.0040000000000000001, 8: 0.001, 9: 0.0, 10: 0.0, 'm': -0.11600000000000001, 'b': 0.40600000000000003}, '62': {1: 0.55900000000000005, 2: 0.20599999999999999, 3: 0.158, 4: 0.051999999999999998, 5: 0.017000000000000001, 6: 0.0050000000000000001, 7: 0.002, 8: 0.001, 9: 0.0, 10: 0.0, 'm': -0.19900000000000001, 'b': 0.82799999999999996}, '82': {1: 0.27300000000000002, 2: 0.35499999999999998, 3: 0.17000000000000001, 4: 0.13800000000000001, 5: 0.041000000000000002, 6: 0.014999999999999999, 7: 0.0050000000000000001, 8: 0.001, 9: 0.0, 10: 0.0, 'm': -0.20899999999999999, 'b': 0.78900000000000003}, '80': {1: 0.311, 2: 0.247, 3: 0.17000000000000001, 4: 0.14399999999999999, 5: 0.070999999999999994, 6: 0.031, 7: 0.012999999999999999, 8: 0.0080000000000000002, 9: 0.0030000000000000001, 10: 0.002, 'm': -0.127, 'b': 0.193}, '81': {1: 0.39700000000000002, 2: 0.24399999999999999, 3: 0.151, 4: 0.123, 5: 0.050999999999999997, 6: 0.021000000000000001, 7: 0.0080000000000000002, 8: 0.0030000000000000001, 9: 0.001, 10: 0.0, 'm': -0.14199999999999999, 'b': 0.40200000000000002}, '20': {1: 0.42399999999999999, 2: 0.29899999999999999, 3: 0.14999999999999999, 4: 0.071999999999999995, 5: 0.032000000000000001, 6: 0.012999999999999999, 7: 0.0050000000000000001, 8: 0.002, 9: 0.001, 10: 0.0, 'm': -0.27800000000000002, 'b': 0.71599999999999997}, '21': {1: 0.44400000000000001, 2: 0.32600000000000001, 3: 0.13700000000000001, 4: 0.056000000000000001, 5: 0.021999999999999999, 6: 0.0089999999999999993, 7: 0.0030000000000000001, 8: 0.001, 9: 0.0, 10: 0.0, 'm': -0.26800000000000002, 'b': 0.86299999999999999}, '22': {1: 0.45300000000000001, 2: 0.374, 3: 0.11600000000000001, 4: 0.039, 5: 0.012, 6: 0.0050000000000000001, 7: 0.001, 8: 0.0, 9: 0.0, 10: 0.0, 'm': -0.19400000000000001, 'b': 0.97399999999999998}, '42': {1: 0.49399999999999999, 2: 0.23699999999999999, 3: 0.18099999999999999, 4: 0.059999999999999998, 5: 0.017999999999999999, 6: 0.0070000000000000001, 7: 0.002, 8: 0.001, 9: 0.0, 10: 0.0, 'm': -0.14399999999999999, 'b': 0.84099999999999997}, '40': {1: 0.36199999999999999, 2: 0.25600000000000001, 3: 0.19400000000000001, 4: 0.104, 5: 0.048000000000000001, 6: 0.02, 7: 0.0089999999999999993, 8: 0.0040000000000000001, 9: 0.002, 10: 0.001, 'm': -0.16700000000000001, 'b': 0.45000000000000001}, '41': {1: 0.40100000000000002, 2: 0.25800000000000001, 3: 0.20300000000000001, 4: 0.083000000000000004, 5: 0.034000000000000002, 6: 0.012999999999999999, 7: 0.0050000000000000001, 8: 0.002, 9: 0.001, 10: 0.0, 'm': -0.17399999999999999, 'b': 0.66900000000000004}, '72': {1: 0.185, 2: 0.54800000000000004, 3: 0.16900000000000001, 4: 0.067000000000000004, 5: 0.023, 6: 0.0060000000000000001, 7: 0.002, 8: 0.0, 9: 0.0, 10: 0.0, 'm': -0.095000000000000001, 'b': 0.78500000000000003}, '71': {1: 0.41299999999999998, 2: 0.32800000000000001, 3: 0.13800000000000001, 4: 0.072999999999999995, 5: 0.029000000000000001, 6: 0.010999999999999999, 7: 0.0050000000000000001, 8: 0.002, 9: 0.001, 10: 0.0, 'm': -0.311, 'b': 0.47799999999999998}, '70': {1: 0.315, 2: 0.35599999999999998, 3: 0.16800000000000001, 4: 0.085999999999999993, 5: 0.043999999999999997, 6: 0.017999999999999999, 7: 0.0070000000000000001, 8: 0.0040000000000000001, 9: 0.002, 10: 0.0, 'm': -0.22900000000000001, 'b': 0.26100000000000001}, '11': {1: 0.59999999999999998, 2: 0.24299999999999999, 3: 0.097000000000000003, 4: 0.036999999999999998, 5: 0.014, 6: 0.0060000000000000001, 7: 0.002, 8: 0.001, 9: 0.0, 10: 0.0, 'm': -0.29599999999999999, 'b': 0.98799999999999999}, '12': {1: 0.67300000000000004, 2: 0.222, 3: 0.070999999999999994, 4: 0.023, 5: 0.0070000000000000001, 6: 0.002, 7: 0.001, 8: 0.0, 9: 0.0, 10: 0.0, 'm': -0.16300000000000001, 'b': 1.014}, '32': {1: 0.68600000000000005, 2: 0.20399999999999999, 3: 0.072999999999999995, 4: 0.025000000000000001, 5: 0.0080000000000000002, 6: 0.002, 7: 0.001, 8: 0.0, 9: 0.0, 10: 0.0, 'm': -0.107, 'b': 0.83199999999999996}, '31': {1: 0.59399999999999997, 2: 0.23400000000000001, 3: 0.104, 4: 0.042000000000000003, 5: 0.017000000000000001, 6: 0.0060000000000000001, 7: 0.0030000000000000001, 8: 0.001, 9: 0.0, 10: 0.0, 'm': -0.191, 'b': 0.69299999999999995}, '30': {1: 0.56599999999999995, 2: 0.22600000000000001, 3: 0.114, 4: 0.052999999999999999, 5: 0.023, 6: 0.01, 7: 0.0040000000000000001, 8: 0.002, 9: 0.001, 10: 0.0, 'm': -0.35799999999999998, 'b': 0.55900000000000005}, '51': {1: 0.73699999999999999, 2: 0.152, 3: 0.067000000000000004, 4: 0.027, 5: 0.010999999999999999, 6: 0.0040000000000000001, 7: 0.001, 8: 0.001, 9: 0.0, 10: 0.0, 'm': -0.27000000000000002, 'b': 0.47699999999999998}, '50': {1: 0.65400000000000003, 2: 0.185, 3: 0.088999999999999996, 4: 0.041000000000000002, 5: 0.017999999999999999, 6: 0.0080000000000000002, 7: 0.0030000000000000001, 8: 0.001, 9: 0.001, 10: 0.0, 'm': -0.37, 'b': 0.35499999999999998}, '52': {1: 0.73199999999999998, 2: 0.17699999999999999, 3: 0.059999999999999998, 4: 0.021000000000000001, 5: 0.0070000000000000001, 6: 0.002, 7: 0.001, 8: 0.0, 9: 0.0, 10: 0.0, 'm': -0.047, 'b': 0.76300000000000001}}\n\ndef getRunsInn(rpinn):\n runsinn = {0: 1/((rpinn*.761)+1),\n 1: (rpinn*(0.761**2))/(((rpinn*.761)+1)**2)\n }\n\n for i in range(2, 11):\n v = (rpinn*(0.761**2)*(((rpinn*.761) - 0.761 + 1)**(i-1)))/(((rpinn*.761)+1)**(i+1))\n runsinn[i] = v\n return runsinn\n\n\ndef getRunExp(rpinn, runsinn):\n runExp = {'10': runsinn\n }\n for i in range(0, 3):\n for j in range(1, 9):\n k = str(j) + str(i)\n if k == '10': continue\n runExp[k] = {0: ((tangoRunExp[k]['m']*rpinn) + tangoRunExp[k]['b'])\n }\n for r in range(1, 11):\n runExp[k][r] = ((1 - runExp[k][0])*tangoRunExp[k][r])\n return runExp\n\ndef getInnWinexp(runExp):\n ## Chance of home team winning with zero\n ## outs at the beg. of each inning\n\n innWinexp = {'101': {0: 0.5\n }\n }\n\n for i in range(-25, 0):\n innWinexp['101'][i] = 0\n for i in range(1, 26):\n innWinexp['101'][i] = 1\n\n for i in range(9, 0, -1):\n for j in range(2, 0, -1):\n if j == 2: next = str(i+1) + '1'\n else: next = str(i) + '2'\n this = str(i) + str(j)\n innWinexp[this] = {}\n if j == 2:\n for k in range(-25, 26):\n p = 0\n if i == 9 and k > 0:\n innWinexp[this][k] = 1\n continue\n else: pass\n for m in range(0, 11):\n if k+m > 25: iw = 1\n else: iw = innWinexp[next][k+m]\n p += runExp['10'][m]*iw\n innWinexp[this][k] = p\n else:\n for k in range(-25, 26):\n p = 0\n for m in range(0, 11):\n if k-m < -25: iw = 0\n else: iw = innWinexp[next][k-m]\n p += runExp['10'][m]*iw\n innWinexp[this][k] = p\n return innWinexp\n\n\ndef getWinexp(innWinexp, runExp, inn, half, base, outs, rdiff): \n if inn > 9: inn = 9\n innkey = str(inn) + str(half)\n if outs > 2: outs = 2\n sitkey = str(base) + str(outs)\n if half == 2: next = str(inn+1) + '1'\n else: next = str(inn) + '2'\n if sitkey == '10': ## beginning of half inning\n if rdiff > 25: rdiff = 25\n elif rdiff < -25: rdiff = -25\n else: pass\n Winexp = innWinexp[innkey][rdiff]\n elif half == 1:\n Winexp = 0\n for i in range(10, -1, -1):\n if rdiff-i < -25: iw = 0\n elif rdiff-i > 25: iw = 1\n else: iw = innWinexp[next][rdiff-i]\n Winexp += runExp[sitkey][i]*iw\n else:\n Winexp = 0\n for i in range(0, 11):\n if rdiff-i < -25: iw = 0\n elif rdiff+i > 25: iw = 1\n else: iw = innWinexp[next][rdiff+i]\n Winexp += runExp[sitkey][i]*iw\n return Winexp\n\ndef getVol(innWinexp, runExp, inn, half, base, outs, rdiff):\n ## changes if strikeout:\n if outs == 2:\n outsK = 0\n baseK = 1\n if half == 1:\n halfK = 2\n innK = inn\n else:\n halfK = 1\n innK = inn + 1\n else:\n outsK = outs + 1\n baseK, halfK, innK = base, half, inn\n WinexpK = getWinexp(innWinexp, runExp, innK, halfK, baseK, outsK, rdiff)\n ## changes if homerun\n if half == 1:\n rdiff -= baseHr[base]\n else:\n rdiff += baseHr[base]\n base = 1\n WinexpHr = getWinexp(innWinexp, runExp, inn, half, base, outs, rdiff)\n return (abs(WinexpHr - WinexpK))/0.133\n\ndef rpgToInnWinexp(rpg):\n rpinn = float(rpg)/9 ## r/inn\n runsinn = getRunsInn(rpinn)\n runExp = getRunExp(rpinn, runsinn)\n innWinexp = getInnWinexp(runExp)\n return innWinexp, runExp\n\ndef winnexp_feature(x):\n return getWinexp(innWinexp, runExp, x['inning'], x['halfInning_index'], x['base_state'], x['outs_beg'], x['run_diff'])\n\n##################################################################################################\n## Functions for extracting and matching ejected player names and getting their playerId\n##################################################################################################\n# Need to map names to the players.csv or playerBoxScores playerIds\ndef find_closest_playerName(playerName, players):\n players['fuzz_score'] = [fuzz.WRatio(playerName, x) for x in players['playerName']]\n best_match = players.loc[players['fuzz_score']==players['fuzz_score'].max(), 'playerName'].iloc[0] \n \n return best_match\n\ndef find_playerId(x, players, rosters_players):\n # rosters_players is a merge of the rosters df and the players df on the playerId\n tmp = players[players['playerName']==x['playerName']]\n if tmp.shape[0]==1:\n return tmp['playerId'].iloc[0]\n else:\n # If there are two players with the same name in players, then use the daily roster data to find the player on the matching team\n return rosters_players.loc[(rosters_players['dailyDataDate']==x['dailyDataDate']) & (rosters_players['teamId']==x['teamId']) & (rosters_players['playerName']==x['playerName']), 'playerId'].iloc[0]\n\n##################################################################################################\n\n# Set up win expectancy variables\nrpg = 4.5\ninnWinexp, runExp = rpgToInnWinexp(rpg)\n\ndef game_score_james(x):\n '''\n # • Start with 50 points\n # • Add 1 point for each out recorded (or 3 points per inning)\n # • Add 2 points for each inning completed after the fourth\n # • Add 1 additional point for every strikeout\n # • Remove 2 points for each hit allowed\n # • Remove 4 points for each earned run allowed\n # • Remove 2 points for each unearned run allowed\n # • Remove 1 point for each walk allowed\n '''\n score = 50\n score += x['outsPitching']\n score += 2*(x['inningsPitched'] - 4)\n score += x['strikeOutsPitching']\n score -= 2*x['hitsPitching']\n score -= 4*x['earnedRuns']\n score -= 2*(x['runsPitching'] - x['earnedRuns'])\n score -= (x['baseOnBallsPitching']+x['hitByPitchPitching'])\n# score = 50 + x['outsPitching'] + 2*(x['inningsPitched'] - 4) + x['strikeOutsPitching'] - 2*x['hitsPitching'] - 4*x['earnedRuns'] - 2*(x['runsPitching'] - x['earnedRuns']) - (x['baseOnBallsPitching']+x['hitByPitchPitching'])\n return score\n\n \ndef game_score_tango(x):\n '''\n Game Score formula (updated by Tom Tango)\n # • Start with 40 points\n # • Add 2 points for each out recorded (or 6 points per inning)\n # • Add 1 additional point for every strikeout\n # • Remove 2 points for each walk allowed\n # • Remove 2 points for each hit allowed\n # • Remove 3 points for each run allowed (earned or unearned)\n # • Remove 6 additional points for each home run allowed \n '''\n score = 40\n score += 2*x['outsPitching']\n score += x['strikeOutsPitching']\n score -= 2*(x['baseOnBallsPitching']+x['hitByPitchPitching'])\n score -= 2*x['hitsPitching']\n score -= 3*x['runsPitching']\n score -= 6*x['homeRunsPitching']\n return score\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-20T20:08:15.573524Z","iopub.execute_input":"2021-09-20T20:08:15.573977Z","iopub.status.idle":"2021-09-20T20:08:18.704816Z","shell.execute_reply.started":"2021-09-20T20:08:15.57387Z","shell.execute_reply":"2021-09-20T20:08:18.703704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pitching_features = ['gamesPlayedPitching', 'gamesStartedPitching',\n 'completeGamesPitching', 'shutoutsPitching', 'winsPitching',\n 'lossesPitching', 'flyOutsPitching', 'airOutsPitching',\n 'groundOutsPitching', 'runsPitching', 'doublesPitching',\n 'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching',\n 'baseOnBallsPitching', 'intentionalWalksPitching', 'hitsPitching',\n 'hitByPitchPitching', 'atBatsPitching', 'caughtStealingPitching',\n 'stolenBasesPitching', 'inningsPitched', 'saveOpportunities',\n 'earnedRuns', 'battersFaced', 'outsPitching', 'pitchesThrown', 'balls',\n 'strikes', 'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching',\n 'rbiPitching', 'gamesFinishedPitching', 'inheritedRunners',\n 'inheritedRunnersScored', 'catchersInterferencePitching',\n 'sacBuntsPitching', 'sacFliesPitching', 'saves', 'holds', 'blownSaves',\n 'assists', 'putOuts', 'errors', 'chances']","metadata":{"execution":{"iopub.status.busy":"2021-09-20T20:08:18.706382Z","iopub.execute_input":"2021-09-20T20:08:18.706711Z","iopub.status.idle":"2021-09-20T20:08:18.713264Z","shell.execute_reply.started":"2021-09-20T20:08:18.706677Z","shell.execute_reply":"2021-09-20T20:08:18.712251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = dt.fread(\"../input/mlb-player-digital-engagement-forecasting/train_updated.csv\").to_pandas()\ntest = True\nif test:\n last_date = train['date'].max()\nelse:\n last_date = 20210430\n\neng = unnest(train, 'nextDayPlayerEngagement')\np_box_scores_og = unnest(train, 'playerBoxScores')\nteams = pd.read_csv(\"../input/mlb-player-digital-engagement-forecasting/teams.csv\")\nplayers = pd.read_csv(\"../input/mlb-player-digital-engagement-forecasting/players.csv\")\nawards_history = pd.read_csv(\"../input/mlb-player-digital-engagement-forecasting/awards.csv\")\n##################################################################################################\n## Regex's for extracting and matching ejected player names and getting their playerId\n##################################################################################################\nteam_names = list(teams['teamName'].unique()) + [\"Diamondbacks\"]\nteam_regex = re.compile('|'.join(map(re.escape, team_names)))\nteam_full_names = list(teams['name'].unique()) + list(teams['teamName'].unique())\nteam_full_regex = re.compile('|'.join(map(re.escape, team_full_names)))\ncoaching_names = [\"Assistant Hitting Coach\", \"Manager\", \"Bench Coach\", \"Interim Manager\", \"Hitting Coach\", \"First Base Coach\", \"Pitching Coach\", \"bench caoch\", \"assistant hitting coach\", \"Third Base Coach\", \"catching coach\", \"field coordinator\", \"first base coach\", \"hitting coach\", \"major league coach\", \"manager\", \"pitching coach\", \"third base coach\", \"bench coach\"]\ncoaching_regex = re.compile('|'.join(map(re.escape, coaching_names)))\npositions = ['pitcher','catcher','first baseman','second baseman', 'third baseman','shortstop','left fielder','center fielder','right fielder', 'designated hitter']\npos_regex = re.compile('|'.join(map(re.escape, positions))) \n##################################################################################################\ncolnames = [\n 'games',\n 'rosters',\n 'playerBoxScores',\n 'teamBoxScores',\n 'transactions',\n 'standings',\n 'awards',\n 'events',\n 'playerTwitterFollowers',\n 'teamTwitterFollowers']\n\n\nhitter_history_feats = ['hits','doubles','triples','homeRuns','rbi','totalBases', 'plateAppearances','strikeOuts','baseOnBalls','hitByPitch', 'atBats','sacFlies']\npitcher_history_feats = ['gamesPlayedPitching', 'gamesStartedPitching','inningsPitched', 'pitchesThrown', 'winsPitching', 'runsPitching', 'homeRunsPitching', 'strikeOutsPitching','earnedRuns', 'blownSaves', 'holds']\nfielder_history_feats = ['errors']\nkeep_awards = ['NLPOW', 'ALPOW', 'NLROM', 'ALROM','NLPOM','ALPOM','NLRRELMON','ALRRELMON','ALPITOM','NLPITOM','MLBPLAYOW']\nkeep_annual_awards = ['ALMVP', 'NLMVP', 'ALCY', 'NLCY','ALROY','NLROY','ALPG','NLPG','ALSS','NLSS', 'ALGG','NLGG']\n\n##################################################################################################\n## Mappings\n##################################################################################################\nteam_mapping = teams.set_index('teamName')['id'].to_dict()\nteam_mapping['Diamondbacks'] = 109\n\nplayer_mapping = p_box_scores_og[['playerId','playerName']].drop_duplicates()\n\n##################################################################################################\n\npitchers = players[players['primaryPositionName']==\"Pitcher\"]\nplayers['value'] = 1\nplayer_country_dummies = pd.pivot_table(players, values='value', index=['playerId'], columns=['birthCountry'], aggfunc='sum', fill_value=0).reset_index()\nplayer_country_dummies.columns = player_country_dummies.columns.str.replace(' ','_')\n\nquantile_20 = lambda x: x.quantile(.20)\nquantile_20.__name__ = 'quantile_20'\n\nquantile_80 = lambda x: x.quantile(.80)\nquantile_80.__name__ = 'quantile_80'\n\n# agg_list = ['median','var', quantile_20, quantile_80]\n# player_aggs = eng.groupby('playerId')[['target1','target2','target3','target4']].agg({'target1': agg_list,\n# 'target2': agg_list,\n# 'target3': agg_list,\n# 'target4': agg_list}).round(6)\n# player_aggs.columns = [\"_\".join(x) for x in player_aggs.columns.ravel()]\n# player_aggs = player_aggs.reset_index()\n\n\n# player_medians = eng.groupby('playerId')[['target1','target2','target3','target4']].median().round(6).reset_index()\n# player_medians = player_medians.rename({'target1': 'target1_p_median',\n# 'target2': 'target2_p_median',\n# 'target3': 'target3_p_median',\n# 'target4': 'target4_p_median'}, axis=1)\n\n# player_variances = eng.groupby('playerId')[['target1','target2','target3','target4']].var().round(6).reset_index()\n# player_variances = player_variances.rename({'target1': 'target1_p_var',\n# 'target2': 'target2_p_var',\n# 'target3': 'target3_p_var',\n# 'target4': 'target4_p_var'}, axis=1)\n\n\n\n# game_day_player_means = eng.merge(p_box_scores_og[['dailyDataDate','playerId', 'gamePk']], how='left')\n# game_day_player_means['game_played'] = game_day_player_means['gamePk'].notnull().astype(int)\n# off_day_player_means = game_day_player_means[game_day_player_means['game_played']==0].groupby(['playerId'])[['target1','target2','target3','target4']].mean().round(6).reset_index()\n# off_day_player_means = off_day_player_means.rename({'target1': 'target1_p_mean_off_day',\n# 'target2': 'target2_p_mean_off_day',\n# 'target3': 'target3_p_mean_off_day',\n# 'target4': 'target4_p_mean_off_day'}, axis=1)\n\n# game_day_player_means = game_day_player_means[game_day_player_means['game_played']==1].groupby(['playerId'])[['target1','target2','target3','target4']].mean().round(6).reset_index()\n# game_day_player_means = game_day_player_means.rename({'target1': 'target1_p_mean_game_day',\n# 'target2': 'target2_p_mean_game_day',\n# 'target3': 'target3_p_mean_game_day',\n# 'target4': 'target4_p_mean_game_day'}, axis=1)\n\n# game_day_player_vars = eng.merge(p_box_scores_og[['dailyDataDate','playerId']])\n# game_day_player_vars = game_day_player_vars.groupby('playerId')[['target1','target2','target3','target4']].var().round(6).reset_index()\n# game_day_player_vars = game_day_player_vars.rename({'target1': 'target1_p_var_game_day',\n# 'target2': 'target2_p_var_game_day',\n# 'target3': 'target3_p_var_game_day',\n# 'target4': 'target4_p_var_game_day'}, axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-09-20T20:08:18.715627Z","iopub.execute_input":"2021-09-20T20:08:18.715971Z","iopub.status.idle":"2021-09-20T20:10:19.729555Z","shell.execute_reply.started":"2021-09-20T20:08:18.71594Z","shell.execute_reply":"2021-09-20T20:10:19.728578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eng['ddd_month'] = np.floor(eng.dailyDataDate/100).astype(int)\n\nmonths = eng.ddd_month.unique()\n## add one extra month to grab entire data set's TEs\nmonths = np.append(months, months[months.size-1]+1)\n\nMAX_MONTH = months[months.size-1]\n\nttl_player_medians = []\nttl_player_variances = []\nttl_player_means = []\nttl_roll12_player_medians = []\nttl_roll12_player_variances = []\nttl_roll12_player_means = []\nfor i in range(months.size-1):\n month=months[i+1]\n roll_month = months[0]\n if i >11:\n roll_month = months[i-12]\n \n #print(str(month) + ' roll:' + str(roll_month))\n player_medians = eng[eng.ddd_month=roll_month)].groupby('playerId')[['target1','target2','target3','target4']].median().round(6).reset_index()\n roll12_player_medians = roll12_player_medians.rename({'target1': 'roll12_target1_p_median',\n 'target2': 'roll12_target2_p_median',\n 'target3': 'roll12_target3_p_median',\n 'target4': 'roll12_target4_p_median'}, axis=1)\n roll12_player_medians['ddd_month'] = month\n\n player_variances = eng[eng.ddd_month=roll_month)].groupby('playerId')[['target1','target2','target3','target4']].var().round(6).reset_index()\n roll12_player_variances = roll12_player_variances.rename({'target1': 'roll12_target1_p_var',\n 'target2': 'roll12_target2_p_var',\n 'target3': 'roll12_target3_p_var',\n 'target4': 'roll12_target4_p_var'}, axis=1)\n roll12_player_variances['ddd_month'] = month\n\n player_means = eng[eng.ddd_month=roll_month)].groupby('playerId')[['target1','target2','target3','target4']].mean().round(6).reset_index()\n roll12_player_means = roll12_player_means.rename({'target1': 'roll12_target1_p_mean',\n 'target2': 'roll12_target2_p_mean',\n 'target3': 'roll12_target3_p_mean',\n 'target4': 'roll12_target4_p_mean'}, axis=1)\n roll12_player_means['ddd_month'] = month\n\n ttl_player_medians.append(player_medians)\n ttl_player_variances.append(player_variances)\n ttl_player_means.append(player_means)\n ttl_roll12_player_medians.append(roll12_player_medians)\n ttl_roll12_player_variances.append(roll12_player_variances)\n ttl_roll12_player_means.append(roll12_player_means)\n \n\ndt_player_medians = pd.concat(ttl_player_medians)\ndt_player_variances = pd.concat(ttl_player_variances)\ndt_player_means = pd.concat(ttl_player_means)\ndt_roll12_player_medians = pd.concat(ttl_roll12_player_medians)\ndt_roll12_player_variances = pd.concat(ttl_roll12_player_variances)\ndt_roll12_player_means = pd.concat(ttl_roll12_player_means)\n\ndt_player_aggregations = dt_player_medians.merge(dt_player_variances,how=\"left\",on=['playerId','ddd_month'])\ndt_player_aggregations = dt_player_aggregations.merge(dt_player_means,how=\"left\",on=['playerId','ddd_month'])\ndt_player_aggregations = dt_player_aggregations.merge(dt_roll12_player_medians,how=\"left\",on=['playerId','ddd_month'])\ndt_player_aggregations = dt_player_aggregations.merge(dt_roll12_player_variances,how=\"left\",on=['playerId','ddd_month'])\ndt_player_aggregations = dt_player_aggregations.merge(dt_roll12_player_means,how=\"left\",on=['playerId','ddd_month'])","metadata":{"execution":{"iopub.status.busy":"2021-09-20T20:10:19.731706Z","iopub.execute_input":"2021-09-20T20:10:19.732138Z","iopub.status.idle":"2021-09-20T20:11:01.62931Z","shell.execute_reply.started":"2021-09-20T20:10:19.732094Z","shell.execute_reply":"2021-09-20T20:11:01.628306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"played_game = p_box_scores_og.groupby(['playerId','dailyDataDate'])['gamePk'].count().reset_index()\nplayed_game = played_game.rename({'gamePk': 'played_game'}, axis=1)\n\neng_box = eng.merge(played_game,how='left', on=['dailyDataDate','playerId'])\neng_box['played_game'] = eng_box['played_game'].fillna(0).clip(upper=1)\n\nttl_player_gameday_medians = []\nttl_player_gameday_variances = []\nttl_player_gameday_means = []\nttl_roll12_player_gameday_medians = []\nttl_roll12_player_gameday_variances = []\nttl_roll12_player_gameday_means = []\nfor i in range(months.size-1):\n month=months[i+1]\n roll_month = months[0]\n if i >11:\n roll_month = months[i-12]\n \n #print(str(month) + ' roll:' + str(roll_month))\n player_gameday_medians = eng_box[eng_box.ddd_month=roll_month)].groupby(['playerId','played_game'])[['target1','target2','target3','target4']].median().round(6).reset_index()\n roll12_player_gameday_medians = roll12_player_gameday_medians.rename({'target1': 'roll12_target1_p_gameday_median',\n 'target2': 'roll12_target2_p_gameday_median',\n 'target3': 'roll12_target3_p_gameday_median',\n 'target4': 'roll12_target4_p_gameday_median'}, axis=1)\n roll12_player_gameday_medians['ddd_month'] = month\n\n player_gameday_variances = eng_box[eng_box.ddd_month=roll_month)].groupby(['playerId','played_game'])[['target1','target2','target3','target4']].var().round(6).reset_index()\n roll12_player_gameday_variances = roll12_player_gameday_variances.rename({'target1': 'roll12_target1_p_gameday_var',\n 'target2': 'roll12_target2_p_gameday_var',\n 'target3': 'roll12_target3_p_gameday_var',\n 'target4': 'roll12_target4_p_gameday_var'}, axis=1)\n roll12_player_gameday_variances['ddd_month'] = month\n\n player_gameday_means = eng_box[eng_box.ddd_month=roll_month)].groupby(['playerId','played_game'])[['target1','target2','target3','target4']].mean().round(6).reset_index()\n roll12_player_gameday_means = roll12_player_gameday_means.rename({'target1': 'roll12_target1_p_gameday_mean',\n 'target2': 'roll12_target2_p_gameday_mean',\n 'target3': 'roll12_target3_p_gameday_mean',\n 'target4': 'roll12_target4_p_gameday_mean'}, axis=1)\n roll12_player_gameday_means['ddd_month'] = month\n\n ttl_player_gameday_medians.append(player_gameday_medians)\n ttl_player_gameday_variances.append(player_gameday_variances)\n ttl_player_gameday_means.append(player_gameday_means)\n ttl_roll12_player_gameday_medians.append(roll12_player_gameday_medians)\n ttl_roll12_player_gameday_variances.append(roll12_player_gameday_variances)\n ttl_roll12_player_gameday_means.append(roll12_player_gameday_means)\n \n\ndt_player_gameday_medians = pd.concat(ttl_player_gameday_medians)\ndt_player_gameday_variances = pd.concat(ttl_player_gameday_variances)\ndt_player_gameday_means = pd.concat(ttl_player_gameday_means)\ndt_roll12_player_gameday_medians = pd.concat(ttl_roll12_player_gameday_medians)\ndt_roll12_player_gameday_variances = pd.concat(ttl_roll12_player_gameday_variances)\ndt_roll12_player_gameday_means = pd.concat(ttl_roll12_player_gameday_means)\n\ndt_player_game_aggregations = dt_player_gameday_medians.merge(dt_player_gameday_variances,how=\"left\",on=['playerId','ddd_month','played_game'])\ndt_player_game_aggregations = dt_player_game_aggregations.merge(dt_player_gameday_means,how=\"left\",on=['playerId','ddd_month','played_game'])\ndt_player_game_aggregations = dt_player_game_aggregations.merge(dt_roll12_player_gameday_medians,how=\"left\",on=['playerId','ddd_month','played_game'])\ndt_player_game_aggregations = dt_player_game_aggregations.merge(dt_roll12_player_gameday_variances,how=\"left\",on=['playerId','ddd_month','played_game'])\ndt_player_game_aggregations = dt_player_game_aggregations.merge(dt_roll12_player_gameday_means,how=\"left\",on=['playerId','ddd_month','played_game'])","metadata":{"execution":{"iopub.status.busy":"2021-09-20T20:11:01.630996Z","iopub.execute_input":"2021-09-20T20:11:01.631424Z","iopub.status.idle":"2021-09-20T20:11:56.922407Z","shell.execute_reply.started":"2021-09-20T20:11:01.631382Z","shell.execute_reply":"2021-09-20T20:11:56.921633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"position_freq = p_box_scores_og['positionType'].fillna(-999).value_counts(normalize=True).to_dict()\n\nposition_target_agg = eng.merge(p_box_scores_og[['dailyDataDate','playerId','gamePk','gameTimeUTC','positionType']], how='left')\ndh_games = position_target_agg[position_target_agg[['dailyDataDate','playerId']].duplicated(keep=False)].sort_values('gameTimeUTC')[['dailyDataDate','playerId','gamePk']].reset_index(drop=True)\ndh_last_game = dh_games[dh_games[['dailyDataDate','playerId']].duplicated(keep='first')] #games to remove\nposition_target_agg = position_target_agg[~(position_target_agg['playerId'].isin(dh_last_game['playerId']) & position_target_agg['gamePk'].isin(dh_last_game['gamePk']))]\nposition_freq = position_target_agg['positionType'].fillna(-999).value_counts(normalize=True).to_dict()\nposition_target_agg = position_target_agg.groupby('positionType')[['target1','target2','target3','target4']].agg({'target1': ['median','var'],\n 'target2': ['median','var'],\n 'target3': ['median','var'],\n 'target4': ['median','var']}).round(6)\nposition_target_agg.columns = [\"_\".join(x + ('position',)) for x in position_target_agg.columns.ravel()]\nposition_target_agg = position_target_agg.reset_index()","metadata":{"execution":{"iopub.status.busy":"2021-09-20T20:11:56.923679Z","iopub.execute_input":"2021-09-20T20:11:56.924362Z","iopub.status.idle":"2021-09-20T20:12:01.480347Z","shell.execute_reply.started":"2021-09-20T20:11:56.924231Z","shell.execute_reply":"2021-09-20T20:12:01.477327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"last_day = train[train['date']==last_date]\neng, games, rosters, p_box_scores, t_box_scores, transactions, standings, awards, events, p_twitter, t_twitter = get_unnested_data(last_day, ['nextDayPlayerEngagement'] + colnames)\n# eng_lag = eng.copy()\n# eng_lag = eng[['playerId','target1','target2','target3','target4']].copy()\n# eng_lag = eng_lag.rename({'target1': 'target1_lag',\n# 'target2': 'target2_lag',\n# 'target3': 'target3_lag',\n# 'target4': 'target4_lag'}, axis=1)\ntry:\n if not p_box_scores.empty:\n t_tmp = eng.merge(p_box_scores[['dailyDataDate','playerId','positionCode','pitchesThrown']], how='left', on=['dailyDataDate', 'playerId'])\n t_tmp['position_player_pitching'] = ((t_tmp['positionCode']>1) & (t_tmp['pitchesThrown']>0)).astype(int)\n prior_day_pos_player_pitching = t_tmp.loc[t_tmp['position_player_pitching']==1, ['playerId','position_player_pitching']].fillna(0)\n else:\n eng['position_player_pitching'] = 0\n prior_data_pos_player_pitching = eng[['playerId','position_player_pitching']]\nexcept Exception as e:\n print(e)\n eng['position_player_pitching'] = 0\n prior_data_pos_player_pitching = eng[['playerId','position_player_pitching']]","metadata":{"execution":{"iopub.status.busy":"2021-09-20T20:12:01.481793Z","iopub.execute_input":"2021-09-20T20:12:01.482269Z","iopub.status.idle":"2021-09-20T20:12:02.757216Z","shell.execute_reply.started":"2021-09-20T20:12:01.482235Z","shell.execute_reply":"2021-09-20T20:12:02.755727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Instead of using specific date, find latest date with twitter data available\nlast_twitter_date = train.loc[train['playerTwitterFollowers'].notnull(), 'date'].max()\nsecond_last_twitter_date = train.loc[train['playerTwitterFollowers'].notnull(), 'date'].iloc[-2]\nlast_twitter_update = train[train['date']==last_twitter_date]\nsecond_last_twitter_update = train[train['date']==second_last_twitter_date]\np_twitter,_ = get_unnested_data(last_twitter_update, ['playerTwitterFollowers', 'teamTwitterFollowers'])\np_twitter_recent = p_twitter.copy()\np_twitter_second_last,_ = get_unnested_data(second_last_twitter_update, ['playerTwitterFollowers', 'teamTwitterFollowers'])\n\np_twitter_recent = p_twitter_recent.set_index(\"playerId\")\np_twitter_second_last = p_twitter_second_last.set_index(\"playerId\")\np_twitter_delta = (p_twitter_recent['numberOfFollowers'] - p_twitter_second_last['numberOfFollowers']).reset_index().rename(columns={'numberOfFollowers': 'numberOfFollowers_delta'})\np_twitter_recent = p_twitter_recent.reset_index()","metadata":{"execution":{"iopub.status.busy":"2021-09-20T20:12:02.759913Z","iopub.execute_input":"2021-09-20T20:12:02.76054Z","iopub.status.idle":"2021-09-20T20:12:02.926911Z","shell.execute_reply.started":"2021-09-20T20:12:02.76045Z","shell.execute_reply":"2021-09-20T20:12:02.926109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# games = unnest(train, 'games')\n# schedule_21 = pd.read_csv(\"../input/mlbdata/schedule_2021.csv\")\n# schedule_21['gameDate'] = pd.to_datetime(schedule_21['gameDate'])\n# games['gameDate'] = pd.to_datetime(games['gameDate'])\n# games = games.sort_values('gameDate')\n# schedule = pd.concat([games[['dailyDataDate', 'homeId', 'gameDate']].rename({'homeId': 'teamId'}, axis=1),\n# games[['dailyDataDate', 'awayId', 'gameDate']].rename({'awayId': 'teamId'}, axis=1)])\n# schedule = schedule[schedule['dailyDataDate']<20210401]\n# schedule = pd.concat([schedule, schedule_21[['dailyDataDate','teamId','gameDate']]])\n# schedule['gameDate'] = pd.to_datetime(schedule['gameDate'])\n\n# all_dates = pd.DataFrame(list(itertools.product(pd.date_range(start=\"2018-01-01\", end=\"2021-12-31\"), schedule['teamId'].unique())), columns=['gameDate', 'teamId'])\n# all_dates = all_dates.merge(schedule, how='outer', on=['gameDate','teamId'])\n# all_dates = all_dates.sort_values(['teamId','gameDate']).drop_duplicates()\n# all_dates['dailyDataDate_lead'] = all_dates.groupby('teamId')['dailyDataDate'].shift(1)\n# all_dates = all_dates[all_dates['dailyDataDate_lead'].notnull()].reset_index(drop=True)\n# all_dates['nextDayGame'] = 1","metadata":{"execution":{"iopub.status.busy":"2021-09-20T20:12:02.928326Z","iopub.execute_input":"2021-09-20T20:12:02.928911Z","iopub.status.idle":"2021-09-20T20:12:02.933179Z","shell.execute_reply.started":"2021-09-20T20:12:02.928849Z","shell.execute_reply":"2021-09-20T20:12:02.932388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"awards_dict = awards_history[awards_history['awardId'].isin(keep_awards + keep_annual_awards)].groupby(['playerId','awardId']).size().to_dict()\nawards_dict_tmp = {n: grp.to_dict('list') for n, grp in awards_history.loc[awards_history['awardId'].isin(keep_awards + keep_annual_awards), ['awardId','playerId']].groupby('playerId')}\n\nawards_dict = {}\nfor k,v in awards_dict_tmp.items():\n if not k in awards_dict:\n awards_dict[k] = {}\n counts = np.unique(v['awardId'], return_counts=True)\n for feat, value in zip(counts[0], counts[1]):\n awards_dict[k][feat] = value\n \n\nhitter_history_dict = {}\nfielder_history_dict = {}\npitcher_history_dict = {}\nfor i, data in tqdm(train[train['date']<=last_date].iterrows()):\n try:\n data = data.to_frame().T\n daily_data_date = data['date'].iloc[0]\n season = int(str(daily_data_date)[:4])\n p_box_scores, games, rosters, awards = get_unnested_data(data, ['playerBoxScores', 'games', 'rosters', 'awards'])\n\n if rosters.empty:\n rosters = prior_day_rosters\n\n prior_day_rosters = rosters.copy()\n\n if not games.empty:\n games_filtered = games.loc[games['gameType'].isin([\"R\", \"F\",\"D\",\"L\",\"W\",\"C\",\"P\"]) & ~games['detailedGameState'].isin([\"Postponed\"])]\n if not games_filtered.empty:\n schedule_day = pd.concat([games_filtered[['dailyDataDate', 'gamePk','homeId', 'gameDate', 'gameTimeUTC', 'homeWinner']].rename({'homeId': 'teamId', 'homeWinner': 'winner'}, axis=1),\n games_filtered[['dailyDataDate', 'gamePk','awayId', 'gameDate', 'gameTimeUTC','awayWinner']].rename({'awayId': 'teamId', 'awayWinner': 'winner'}, axis=1)])\n\n schedule_day = schedule_day.sort_values('gameTimeUTC')\n\n if not schedule_day.empty and not p_box_scores.empty:\n game_rosters = schedule_day.merge(rosters, how='left', on=['gameDate','teamId'])\n game_rosters = game_rosters[game_rosters['playerId'].notnull()] #missing roster for Nationals 20200910\n game_rosters['playerId'] = game_rosters['playerId'].astype(int)\n p_box_scores = p_box_scores.sort_values(\"gameTimeUTC\")\n p_box_scores['gameDate'] = pd.to_datetime(p_box_scores['gameDate'])\n p_box_scores['season'] = p_box_scores['gameDate'].dt.year\n player_history_daily = game_rosters.merge(p_box_scores, how='left', on=['gamePk', 'playerId']) \n player_history_daily['gameTimeUTC_y'] = player_history_daily['gameTimeUTC_y'].fillna(player_history_daily['gameTimeUTC_x'])\n # NOTE: dailyDataDate==2020918 gamePk==631122 Start time of 2020-09-18T03:33:00Z is not accurate; that would imply the game started the day before at ~11:30PM local time\n player_history_daily = player_history_daily.sort_values(['playerId','gameTimeUTC_y']) # SORT BY gameTimeUTC from p_box_scores. `gameTimeUTC` is not accurate from the `games` data\n player_history_daily[hitter_history_feats] = player_history_daily[hitter_history_feats].fillna(0)\n\n\n\n hitter_history_tmp = {n: grp.to_dict('list') for n, grp in player_history_daily[hitter_history_feats + ['season', 'playerId']].groupby('playerId')}\n for k,v in hitter_history_tmp.items():\n if not k in hitter_history_dict:\n hitter_history_dict[k] = v\n else:\n for feat in hitter_history_feats + ['season']:\n hitter_history_dict[k][feat].extend(v[feat])\n # For hitters, only use games they played in. Pitchers need off days filled in because it's important to account for rest/off days\n # Fill in days with 0 if hitter isn't in daily box scores\n # for k,v in hitter_history_dict.items():\n # if not k in hitter_history_tmp:\n # for feat in hitter_history_feats + ['season']:\n # hitter_history_dict[k][feat].append(season if feat=='season' else 0.0)\n fielder_history_tmp = {n: grp.to_dict('list') for n, grp in player_history_daily[fielder_history_feats + ['season', 'playerId']].groupby('playerId')}\n for k,v in fielder_history_tmp.items():\n if not k in fielder_history_dict:\n fielder_history_dict[k] = v\n else:\n for feat in fielder_history_feats + ['season']:\n fielder_history_dict[k][feat].extend(v[feat])\n\n\n pitcher_history_tmp = {n: grp.to_dict('list') for n, grp in p_box_scores.loc[p_box_scores['positionName']=='Pitcher', pitcher_history_feats + ['season', 'playerId']].groupby('playerId')}\n for k,v in pitcher_history_tmp.items():\n if not k in pitcher_history_dict:\n pitcher_history_dict[k] = v\n else:\n for feat in pitcher_history_feats + ['season']:\n pitcher_history_dict[k][feat].extend(v[feat])\n # Fill in days with 0 if pitcher isn't in daily box scores\n for k,v in pitcher_history_dict.items():\n if not k in pitcher_history_tmp:\n for feat in pitcher_history_feats + ['season']:\n pitcher_history_dict[k][feat].append(season if feat=='season' else 0.0)\n except Exception as e:\n # If fails, just move on to the next day\n print(f\"history dicts loop failed: {e}\")\n pass\n \n try:\n if not awards.empty:\n \n awards_filtered = awards[awards['awardId'].isin(keep_awards + keep_annual_awards)].reset_index(drop=True)\n\n # Update awards counts\n awards_dict_tmp = {n: grp.to_dict('list') for n, grp in awards_filtered[['awardId','playerId']].groupby('playerId')}\n for k,v in awards_dict_tmp.items():\n try:\n if not k in awards_dict:\n awards_dict[k] = {}\n counts = np.unique(v['awardId'], return_counts=True)\n for feat, value in zip(counts[0], counts[1]):\n if feat in awards_dict[k]:\n awards_dict[k][feat] += value\n else:\n awards_dict[k][feat] = value\n except:\n # If fails, move on to the next one\n pass\n except Exception as e:\n # If fails, don't worry about updating dict\n print(e)\n pass","metadata":{"execution":{"iopub.status.busy":"2021-09-20T20:12:02.93438Z","iopub.execute_input":"2021-09-20T20:12:02.935085Z","iopub.status.idle":"2021-09-20T20:22:29.309691Z","shell.execute_reply.started":"2021-09-20T20:12:02.935049Z","shell.execute_reply":"2021-09-20T20:22:29.308537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"games_og = unnest(train, 'games')\nschedule_og = pd.concat([games_og.loc[games_og['gameType'].isin([\"R\", \"F\",\"D\",\"L\",\"W\",\"C\",\"P\"]) & ~games_og['detailedGameState'].isin([\"Postponed\"]),['dailyDataDate', 'gamePk','homeId', 'gameDate', 'gameTimeUTC', 'homeWinner']].rename({'homeId': 'teamId', 'homeWinner': 'winner'}, axis=1),\n games_og.loc[games_og['gameType'].isin([\"R\", \"F\",\"D\",\"L\",\"W\",\"C\",\"P\"]) & ~games_og['detailedGameState'].isin([\"Postponed\"]), ['dailyDataDate', 'gamePk','awayId', 'gameDate', 'gameTimeUTC','awayWinner']].rename({'awayId': 'teamId', 'awayWinner': 'winner'}, axis=1)])\n\nschedule_og = schedule_og.sort_values('gameTimeUTC')\nschedule_og = schedule_og[schedule_og['dailyDataDate']<=last_date]\nschedule_og['gameDate'] = pd.to_datetime(schedule_og['gameDate'])\n \nteam_win_history = {}\nteam_win_dict = schedule_og.groupby(\"teamId\")['winner'].apply(list).to_dict()\nfor k,v in team_win_dict.items():\n if not k in team_win_history:\n team_win_history[k] = v\n else:\n team_win_history[k].extend(v)\n\nwin_streaks = {k: v[::-1].index(0) if 0 in v else len(v) for k, v in team_win_history.items()}","metadata":{"execution":{"iopub.status.busy":"2021-09-20T20:22:29.311127Z","iopub.execute_input":"2021-09-20T20:22:29.311433Z","iopub.status.idle":"2021-09-20T20:22:43.113477Z","shell.execute_reply.started":"2021-09-20T20:22:29.311404Z","shell.execute_reply":"2021-09-20T20:22:43.112556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Load models\nlgb_target1 = lgb.Booster(model_file = \"../input/d/brandenkmurray/mlbmodels/lgb_target1_v30_full.txt\")\nlgb_target2 = lgb.Booster(model_file = \"../input/d/brandenkmurray/mlbmodels/lgb_target2_v30_full.txt\")\nlgb_target3 = lgb.Booster(model_file = \"../input/d/brandenkmurray/mlbmodels/lgb_target3_v30_full.txt\")\nlgb_target4 = lgb.Booster(model_file = \"../input/d/brandenkmurray/mlbmodels/lgb_target4_v30_full.txt\")\n\nlgb_dblsqrt_target1 = lgb.Booster(model_file = \"../input/d/brandenkmurray/mlbmodels/lgb_target1_dblsqrt_full.txt\")\nlgb_dblsqrt_target2 = lgb.Booster(model_file = \"../input/d/brandenkmurray/mlbmodels/lgb_target2_dblsqrt_full.txt\")\nlgb_dblsqrt_target3 = lgb.Booster(model_file = \"../input/d/brandenkmurray/mlbmodels/lgb_target3_dblsqrt_full.txt\")\nlgb_dblsqrt_target4 = lgb.Booster(model_file = \"../input/d/brandenkmurray/mlbmodels/lgb_target4_dblsqrt_full.txt\")\n\nlgb_bfa_target1 = lgb.Booster(model_file = \"../input/d/brandenkmurray/mlbmodels/lgb_target1_v30_bfa_full.txt\")\nlgb_bfa_target2 = lgb.Booster(model_file = \"../input/d/brandenkmurray/mlbmodels/lgb_target2_v30_bfa_full.txt\")\nlgb_bfa_target3 = lgb.Booster(model_file = \"../input/d/brandenkmurray/mlbmodels/lgb_target3_v30_bfa_full.txt\")\nlgb_bfa_target4 = lgb.Booster(model_file = \"../input/d/brandenkmurray/mlbmodels/lgb_target4_v30_bfa_full.txt\")\n\nxgb_target1 = xgb.Booster()\nxgb_target2 = xgb.Booster()\nxgb_target3 = xgb.Booster()\nxgb_target4 = xgb.Booster()\nxgb_target1.load_model(\"../input/d/brandenkmurray/mlbmodels/xgb_target1_v30_full.txt\")\nxgb_target2.load_model(\"../input/d/brandenkmurray/mlbmodels/xgb_target2_v30_full.txt\")\nxgb_target3.load_model(\"../input/d/brandenkmurray/mlbmodels/xgb_target3_v30_full.txt\")\nxgb_target4.load_model(\"../input/d/brandenkmurray/mlbmodels/xgb_target4_v30_full.txt\")\n\nlgb_john_target1 = lgb.Booster(model_file = \"../input/mlb-models-and-files/lgb_target1_dubs_tripsX_all.txt\")\nlgb_john_target2 = lgb.Booster(model_file = \"../input/mlb-models-and-files/lgb_target2_dubs_tripsX_all.txt\")\nlgb_john_target3 = lgb.Booster(model_file = \"../input/mlb-models-and-files/lgb_target3_dubs_tripsX_all.txt\")\nlgb_john_target4 = lgb.Booster(model_file = \"../input/mlb-models-and-files/lgb_target4_dubs_tripsX_all.txt\")\n\nlgb_dart_target1 = lgb.Booster(model_file = \"../input/dart-model/lgb_target1_dubs_trips_dart_full_data.txt\")\nlgb_dart_target2 = lgb.Booster(model_file = \"../input/dart-model/lgb_target2_dubs_trips_dart_full_data.txt\")\nlgb_dart_target3 = lgb.Booster(model_file = \"../input/dart-model/lgb_target3_dubs_trips_dart_full_data.txt\")\nlgb_dart_target4 = lgb.Booster(model_file = \"../input/dart-model/lgb_target4_dubs_trips_dart_full_data.txt\")","metadata":{"execution":{"iopub.status.busy":"2021-09-20T20:22:43.114949Z","iopub.execute_input":"2021-09-20T20:22:43.115251Z","iopub.status.idle":"2021-09-20T20:23:29.869433Z","shell.execute_reply.started":"2021-09-20T20:22:43.115223Z","shell.execute_reply":"2021-09-20T20:23:29.868591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"yesterday = pd.DataFrame()\nt = []\nsub_list = []\nenv = mlb.make_env() # initialize the environment\niter_test = env.iter_test() # iterator which loops over each date in test set\nfor i, (data, sub) in enumerate(iter_test):\n# for i, (i2, data) in enumerate(train[(train['date']>=20210501) & (train['date']<=20210731)].iloc[1:].iterrows()):\n\n ### REMOVE below\n# data = data.to_frame().T\n# sub = unnest(data, 'nextDayPlayerEngagement')\n# sub = sub.rename(columns={'target1': 'target1_act', 'target2': 'target2_act', 'target3': 'target3_act', 'target4': 'target4_act'})\n# sub['date_playerId'] = pd.to_datetime(sub['engagementMetricsDate']).dt.strftime(\"%Y%m%d\") + \"_\" + sub['playerId'].astype(str)\n ### REMOVE above\n ### UNCOMMENT BELOW\n sub = sub.reset_index()\n sub = sub.rename({'date': 'dailyDataDate'}, axis=1)\n sub['playerId'] = sub['date_playerId'].apply(lambda x: int(x.split(\"_\")[1]))\n \n data = data.reset_index()\n data = data.rename({'index': 'date'}, axis=1)\n ### UNCOMMENT ABOVE\n try:\n season = int(str(data['date'].iloc[0])[:4])\n except:\n season = 2021.0\n\n try:\n games, rosters, p_box_scores, t_box_scores, transactions, standings, awards, events, p_twitter, t_twitter = get_unnested_data(data, colnames)\n\n eng_shape = sub.shape\n t_tmp = sub.copy()\n\n if rosters.empty:\n rosters = prior_day_rosters\n\n prior_day_rosters = rosters.copy()\n\n if not p_twitter.empty:\n # Get twitter follower delta if not the first month\n if not p_twitter_recent.empty:\n p_twitter = p_twitter.set_index(\"playerId\")\n p_twitter_recent = p_twitter_recent.set_index(\"playerId\")\n p_twitter_delta = (p_twitter['numberOfFollowers'] - p_twitter_recent['numberOfFollowers']).reset_index().rename(columns={'numberOfFollowers': 'numberOfFollowers_delta'})\n p_twitter = p_twitter.reset_index()\n p_twitter_recent = p_twitter\n\n\n if not games.empty:\n schedule_daily = pd.concat([games.loc[games['gameType'].isin([\"R\", \"F\",\"D\",\"L\",\"W\",\"C\",\"P\"]) & ~games['detailedGameState'].isin([\"Postponed\"]),['dailyDataDate', 'gamePk','homeId', 'gameDate', 'gameTimeUTC', 'homeWinner']].rename({'homeId': 'teamId', 'homeWinner': 'winner'}, axis=1),\n games.loc[games['gameType'].isin([\"R\", \"F\",\"D\",\"L\",\"W\",\"C\",\"P\"]) & ~games['detailedGameState'].isin([\"Postponed\"]), ['dailyDataDate', 'gamePk','awayId', 'gameDate', 'gameTimeUTC','awayWinner']].rename({'awayId': 'teamId', 'awayWinner': 'winner'}, axis=1)])\n\n schedule_daily = schedule_daily.sort_values('gameTimeUTC')\n team_win_dict = schedule_daily.groupby(\"teamId\")['winner'].apply(list).to_dict()\n for k,v in team_win_dict.items():\n if not k in team_win_history:\n team_win_history[k] = v\n else:\n team_win_history[k].extend(v)\n\n win_streaks = {k: v[::-1].index(0) if 0 in v else len(v) for k, v in team_win_history.items()}\n\n if not schedule_daily.empty and not p_box_scores.empty: \n game_rosters = schedule_daily.merge(rosters, how='left', on=['gameDate','teamId'])\n game_rosters = game_rosters[game_rosters['playerId'].notnull()] #missing roster for Nationals 20200910\n game_rosters['playerId'] = game_rosters['playerId'].astype(int)\n p_box_scores = p_box_scores.sort_values(\"gameTimeUTC\")\n p_box_scores['gameDate'] = pd.to_datetime(p_box_scores['gameDate'])\n p_box_scores['season'] = p_box_scores['gameDate'].dt.year\n player_history_daily = game_rosters.merge(p_box_scores, how='left', on=['gamePk', 'playerId']) \n player_history_daily['gameTimeUTC_y'] = player_history_daily['gameTimeUTC_y'].fillna(player_history_daily['gameTimeUTC_x'])\n # NOTE: dailyDataDate==2020918 gamePk==631122 Start time of 2020-09-18T03:33:00Z is not accurate; that would imply the game started the day before at ~11:30PM local time\n player_history_daily = player_history_daily.sort_values(['playerId','gameTimeUTC_y']) # SORT BY gameTimeUTC from p_box_scores. `gameTimeUTC` is not accurate from the `games` data\n player_history_daily[hitter_history_feats] = player_history_daily[hitter_history_feats].fillna(0)\n\n\n hitter_history_tmp = {n: grp.to_dict('list') for n, grp in player_history_daily[hitter_history_feats + ['season', 'playerId']].groupby('playerId')}\n for k,v in hitter_history_tmp.items():\n if not k in hitter_history_dict:\n hitter_history_dict[k] = v\n else:\n for feat in hitter_history_feats + ['season']:\n hitter_history_dict[k][feat].extend(v[feat])\n # For hitters, only use games they played in. Pitchers need off days filled in because it's important to account for rest/off days\n # Fill in days with 0 if hitter isn't in daily box scores\n # for k,v in hitter_history_dict.items():\n # if not k in hitter_history_tmp:\n # for feat in hitter_history_feats + ['season']:\n # hitter_history_dict[k][feat].append(season if feat=='season' else 0.0)\n fielder_history_tmp = {n: grp.to_dict('list') for n, grp in player_history_daily[fielder_history_feats + ['season', 'playerId']].groupby('playerId')}\n for k,v in fielder_history_tmp.items():\n if not k in fielder_history_dict:\n fielder_history_dict[k] = v\n else:\n for feat in fielder_history_feats + ['season']:\n fielder_history_dict[k][feat].extend(v[feat])\n\n pitcher_history_tmp = {n: grp.to_dict('list') for n, grp in p_box_scores.loc[p_box_scores['positionName']=='Pitcher', pitcher_history_feats + ['season', 'playerId']].groupby('playerId')}\n for k,v in pitcher_history_tmp.items():\n if not k in pitcher_history_dict:\n pitcher_history_dict[k] = v\n else:\n for feat in pitcher_history_feats + ['season']:\n pitcher_history_dict[k][feat].extend(v[feat])\n # Fill in days with 0 if pitcher isn't in daily box scores\n for k,v in pitcher_history_dict.items():\n if not k in pitcher_history_tmp:\n for feat in pitcher_history_feats + ['season']:\n pitcher_history_dict[k][feat].append(season if feat=='season' else 0.0)\n\n days_of_history = list(range(2,21)) #[2,3,4,5,7,10,20] #also could be games_of_history depending how its used\n max_days_of_history = np.max(days_of_history)\n hitting_history_features = {}\n pitching_history_features = {}\n fielding_history_features = {}\n\n for k, v in hitter_history_dict.items():\n # only need to include players in the current eng\n hitting_history_features[k] = {} \n hitting_history_features[k]['hit_streak'] = v['hits'][::-1].index(0) if 0 in v['hits'] else len(v['hits']) \n for feat in hitter_history_feats: \n d = hitter_history_dict[k][feat]\n hitting_history_features[k][f'{feat}_season'] = sum([f for seas, f in zip(hitter_history_dict[k]['season'], d) if seas==season])\n if feat not in ['sacFlies','atBats']:\n d_padded = np.zeros(max_days_of_history)\n d_padded[:np.minimum(max_days_of_history, len(d))] = d[-np.minimum(max_days_of_history, len(d)):][::-1]\n d_cumsum = nb_cumsum(d_padded)\n for day in days_of_history:\n hitting_history_features[k][f'{feat}_last{day}'] = d_cumsum[day-1]\n # hitting_history_features[k][f'{feat}_{day-1}_games_ago'] = d_padded[day-1]\n\n hitting_history_df = pd.DataFrame.from_dict(hitting_history_features, orient='index').reset_index().rename({'index': 'playerId'}, axis=1)\n if 'homeRuns_season' in hitting_history_df.columns:\n hitting_history_df['homeRuns_rank'] = hitting_history_df['homeRuns_season'].rank(method='min', ascending=False)\n hitting_history_df['BA'] = hitting_history_df['hits_season'] / hitting_history_df['atBats_season']\n hitting_history_df['OBP'] = hitting_history_df[['hits_season','baseOnBalls_season', 'hitByPitch_season']].sum(axis=1) / hitting_history_df[['atBats_season','baseOnBalls_season', 'hitByPitch_season', 'sacFlies_season']].sum(axis=1)\n hitting_history_df['SLG'] = ((hitting_history_df['hits_season'] - hitting_history_df[['doubles_season','triples_season','homeRuns_season']].sum(axis=1)) + 2*hitting_history_df['doubles_season'] + 3*hitting_history_df['triples_season'] + 4*hitting_history_df['homeRuns_season'])/ hitting_history_df['atBats_season']\n\n for k, v in fielder_history_dict.items():\n # only need to include players in the current eng\n fielding_history_features[k] = {} \n for feat in fielder_history_feats: \n d = fielder_history_dict[k][feat]\n d_padded = np.zeros(max_days_of_history)\n d_padded[:np.minimum(max_days_of_history, len(d))] = d[-np.minimum(max_days_of_history, len(d)):][::-1]\n # d_padded = np.pad(d[-days_of_history:], (np.maximum(0, days_of_history-len(d)+1), 0))[::-1]\n d_cumsum = nb_cumsum(d_padded)\n for day in days_of_history:\n # fielding_history_features[k][f'{feat}_last{day}'] = d_cumsum[day-1]\n fielding_history_features[k][f'{feat}_{day-1}_games_ago'] = d_padded[day-1]\n\n fielding_history_df = pd.DataFrame.from_dict(fielding_history_features, orient='index').reset_index().rename({'index': 'playerId'}, axis=1)\n\n for k, v in pitcher_history_dict.items():\n # only need to include players in the current eng\n pitching_history_features[k] = {} \n season_starts = [starts for seas, starts in zip(pitcher_history_dict[k]['season'], pitcher_history_dict[k]['gamesStartedPitching']) if seas==season]\n season_played = [played for seas, played in zip(pitcher_history_dict[k]['season'], pitcher_history_dict[k]['gamesPlayedPitching']) if seas==season]\n pitching_history_features[k]['season_starts_to_date'] = sum(season_starts)\n pitching_history_features[k]['days_since_last_start'] = season_starts[::-1].index(1.0) if 1 in season_starts else len(season_starts)\n pitching_history_features[k]['days_since_last_played'] = season_played[::-1].index(1.0) if 1 in season_played else len(season_played)\n for feat in ['gamesPlayedPitching', 'gamesStartedPitching','inningsPitched', 'pitchesThrown', 'winsPitching', 'runsPitching', 'homeRunsPitching', 'strikeOutsPitching','earnedRuns', 'blownSaves', 'holds']: \n d = pitcher_history_dict[k][feat]\n pitching_history_features[k][f'{feat}_season'] = sum([f for seas, f in zip(pitcher_history_dict[k]['season'], d) if seas==season])\n d_padded = np.pad(d, (np.maximum(0, max_days_of_history-len(d)), 0))[::-1]\n d_cumsum = nb_cumsum(d_padded)\n for day in days_of_history:\n pitching_history_features[k][f'{feat}_last{day}'] = d_cumsum[day-1]\n pitching_history_features[k][f'{feat}_{day-1}_games_ago'] = d_padded[day-1]\n\n pitching_history_df = pd.DataFrame.from_dict(pitching_history_features, orient='index').reset_index().rename({'index': 'playerId'}, axis=1)\n\n\n if not p_twitter.empty:\n p_twitter_recent = p_twitter\n\n # How to handle doubleheaders? Taking stats from first game for now\n if not p_box_scores.empty and not t_box_scores.empty:\n t_tmp = t_tmp.merge(p_box_scores, how='left', on=['dailyDataDate', 'playerId'])\n dh_games = t_tmp[t_tmp[['dailyDataDate','playerId']].duplicated(keep=False)].sort_values('gameTimeUTC')[['dailyDataDate','playerId','gamePk']].reset_index(drop=True)\n dh_last_game = dh_games[dh_games[['dailyDataDate','playerId']].duplicated(keep='first')] #games to remove\n t_tmp = t_tmp[~(t_tmp['playerId'].isin(dh_last_game['playerId']) & t_tmp['gamePk'].isin(dh_last_game['gamePk']))]\n\n t_tmp['game_score_james'] = game_score_james(t_tmp)\n t_tmp['game_score_tango'] = game_score_tango(t_tmp)\n t_tmp['position_player_pitching'] = ((t_tmp['positionCode']>1) & (t_tmp['pitchesThrown']>0)).astype(int)\n t_tmp['pitcher_hit_home_run'] = ((t_tmp['positionCode']==1) & (t_tmp['homeRuns'] > 0)).astype(int)\n # t_tmp['pos_player_pitched_prior_day'] = 0\n # if not prior_day_pos_player_pitching.empty:\n # t_tmp['pos_player_pitched_prior_day'] = t_tmp['playerId'].map(dict(zip(prior_day_pos_player_pitching.playerId, prior_day_pos_player_pitching.position_player_pitching)))\n\n t_tmp['no_hitter'] = ((t_tmp['inningsPitched']>=9) & (t_tmp['hitsPitching']==0)).astype(int)\n t_tmp['no_hitter_league'] = t_tmp['no_hitter'].max()\n\n t_tmp['position_player_pitching_league'] = t_tmp['position_player_pitching'].max()\n t_tmp['game_hour'] = (pd.to_datetime(t_tmp['gameTimeUTC']) + pd.Timedelta(hours=-5)).dt.hour\n\n t_tmp = t_tmp.merge(t_box_scores, how='left', on=['gamePk', 'teamId'], suffixes=['','_team_box_score'])\n t_tmp['positionType_freq'] = t_tmp['positionType'].fillna(-999).map(position_freq)\n if 'positionType' in t_tmp.columns:\n t_tmp = t_tmp.merge(position_target_agg, how='left', on='positionType')\n\n if t_tmp.shape[0]!=eng_shape[0]:\n print(\"t_tmp length does not match engagement frame length, check for duplicated data\")\n t_tmp = t_tmp[~t_tmp[['playerId']].duplicated()]\n else:\n if 'teamId' not in t_tmp.columns and not rosters.empty:\n t_tmp = t_tmp.merge(rosters[['playerId','teamId']], how='left', on='playerId')\n else:\n t_tmp['teamId'] = np.nan\n if t_tmp.shape[0]!=eng_shape[0]:\n print(\"teamId: t_tmp length does not match engagement frame length, check for duplicated data\")\n t_tmp = t_tmp[~t_tmp[['playerId']].duplicated()]\n\n # Did player have a walk-off hit/home run?\n if not events.empty:\n events = events.sort_values(['inning','halfInning', 'atBatIndex', 'eventId'], ascending=[True, False, True, True])\n last_play = events.groupby('gamePk').tail(1)\n # filter out top of inning because one game was ended after the top of the inning\n walk_offs = last_play[(last_play['halfInning']=='bottom') & (last_play['rbi']>0)][['dailyDataDate','hitterId', 'pitcherId','rbi', 'event']]\n walk_offs.columns = ['dailyDataDate','hitterId', 'pitcherId','walk_off_rbi', 'walk_off_hr']\n walk_offs['walk_off_hr'] = (walk_offs['walk_off_hr'].isin(['Home Run'])).astype(int)\n t_tmp = t_tmp.merge(walk_offs[['dailyDataDate','hitterId', 'walk_off_hr', 'walk_off_rbi']].rename({'hitterId': 'playerId'}, axis=1), how='left', on=['dailyDataDate','playerId'])\n t_tmp = t_tmp.merge(walk_offs[['dailyDataDate','pitcherId', 'walk_off_hr', 'walk_off_rbi']].rename({'pitcherId': 'playerId'}, axis=1), how='left', on=['dailyDataDate','playerId'], suffixes=[\"\",\"_pitcher\"])\n t_tmp[['walk_off_rbi', 'walk_off_hr', 'walk_off_hr_pitcher','walk_off_rbi_pitcher']] = t_tmp[['walk_off_rbi', 'walk_off_hr', 'walk_off_hr_pitcher','walk_off_rbi_pitcher']].fillna(0)\n t_tmp['walk_off_league'] = t_tmp['walk_off_rbi'].max()\n\n hr_dist = events[events['event']=='Home Run'].groupby('hitterId')['totalDistance'].max().reset_index()\n hr_launchSpeed = events[events['event']=='Home Run'].groupby('hitterId')['launchSpeed'].max().reset_index()\n t_tmp = t_tmp.merge(hr_dist.rename({'hitterId': 'playerId'}, axis=1), how='left', on='playerId')\n t_tmp = t_tmp.merge(hr_launchSpeed.rename({'hitterId': 'playerId'}, axis=1), how='left', on='playerId')\n\n # How long did a starting pitcher go without a hit? (Did they start picking up potential no-hitter hype?)\n starters = events[events['isStarter']==1].reset_index(drop=True)\n starters['hit'] = starters['event'].isin(['Single','Double', 'Triple', 'Home Run']).astype(int)\n starters['hits_cumsum'] = starters.groupby('pitcherId')['hit'].cumsum()\n starters_first_hit_inning = starters[starters['hits_cumsum']==1].groupby('pitcherId').first()[['inning','outs']].reset_index()\n starters_first_hit_inning['inning'] = starters_first_hit_inning['inning'] + starters_first_hit_inning['outs']/10\n starters_first_hit_inning = starters_first_hit_inning.rename({'inning': 'pitcher_first_hit_inning'}, axis=1)\n t_tmp = t_tmp.merge(starters_first_hit_inning[['pitcherId', 'pitcher_first_hit_inning']], how='left', left_on='playerId', right_on='pitcherId')\n\n starters_first_mob_inning = starters[~starters['menOnBase'].isin([None,\"Empty\"])]\n starters_first_mob_inning = starters_first_mob_inning.groupby('pitcherId').first()[['inning','outs']].reset_index()\n starters_first_mob_inning['inning'] = starters_first_mob_inning['inning'] + starters_first_mob_inning['outs']/10\n starters_first_mob_inning = starters_first_mob_inning.rename({'inning': 'pitcher_first_mob_inning'}, axis=1)\n t_tmp = t_tmp.merge(starters_first_mob_inning[['pitcherId', 'pitcher_first_mob_inning']], how='left', left_on='playerId', right_on='pitcherId') \n\n # Pitch features\n nastyFactor_features = events[events['type']=='pitch'].groupby(\"pitcherId\")['nastyFactor'].agg(['mean','median','min','max']).reset_index().rename(columns={f: f'nastyFactor_{f}' for f in ['mean','median','max','min']}).rename(columns={'pitcherId': 'playerId'})\n t_tmp = t_tmp.merge(nastyFactor_features, how='left', on='playerId')\n # Calculate player Win Probability Added\n #need to get assign 100% WPA to winning team to assign WPA scores to correct player/team\n\n player_wpa = pd.Series(dtype=float)\n for gamePk, game in events.groupby('gamePk'):\n game = game.reset_index(drop=True)\n game['run_diff'] = game['homeScore'] - game['awayScore']\n game['halfInning_index'] = game['halfInning'].map({'top':1, 'bottom' :2})\n game['base_state'] = game['menOnBase'].map({None: np.nan, 'Empty': 1, 'Men_On': 2, 'RISP': 3, 'Loaded': 8}) \n game['base_state'] = game['base_state'].ffill().fillna(1).astype(int)\n game['outs_beg'] = np.maximum(game['outs'] - 1, 0)\n game['win_exp'] = game.apply(winnexp_feature, axis=1)\n game['win_exp_lag'] = game['win_exp'].shift(-1)\n game.loc[game.shape[0]-1, 'win_exp_lag'] = 1 if game.loc[game.shape[0]-1, 'homeScore']>game.loc[game.shape[0]-1, 'awayScore'] else 0\n game['win_exp_delta'] = game['win_exp_lag'] - game['win_exp']\n # Increases in the top of the inning are assigned to the pitcher\n # Increases in the bottom of the inning are assigned to the hitter\n pitcher_wpa_top = game.loc[(game['halfInning']=='top') & (game['win_exp_delta']>0),['pitcherId','win_exp_delta']].groupby('pitcherId')['win_exp_delta'].sum()\n hitter_wpa_top = game.loc[(game['halfInning']=='top') & (game['win_exp_delta']>0),['hitterId','win_exp_delta']].groupby('hitterId')['win_exp_delta'].sum()\n hitter_wpa_top = -hitter_wpa_top\n\n pitcher_wpa_bot = game.loc[(game['halfInning']=='bottom') & (game['win_exp_delta']>0),['pitcherId','win_exp_delta']].groupby('pitcherId')['win_exp_delta'].sum()\n hitter_wpa_bot = game.loc[(game['halfInning']=='bottom') & (game['win_exp_delta']>0),['hitterId','win_exp_delta']].groupby('hitterId')['win_exp_delta'].sum()\n pitcher_wpa_bot = -pitcher_wpa_bot\n\n player_wpa = player_wpa.add(pitcher_wpa_top, fill_value=0)\n player_wpa = player_wpa.add(hitter_wpa_top, fill_value=0)\n player_wpa = player_wpa.add(pitcher_wpa_bot, fill_value=0)\n player_wpa = player_wpa.add(hitter_wpa_bot, fill_value=0)\n\n player_wpa = player_wpa.reset_index()\n player_wpa = player_wpa.rename({\"index\": \"playerId\", 0: \"wpa\"}, axis=1)\n\n t_tmp = t_tmp.merge(player_wpa, how='left', on='playerId')\n t_tmp['wpa_daily_max'] = t_tmp['wpa'].max()\n t_tmp['wpa_rank'] = t_tmp['wpa'].rank(method='min', ascending=False)\n\n # get ejections\n ejections = events.loc[events['event']==\"Ejection\", ['dailyDataDate','description']].reset_index(drop=True)\n if not ejections.empty:\n ejections['description'] = [x.split(\" ejected by\")[0] for x in ejections['description']]\n # Get team; needed for coach_ejected feature\n ejections['teamName'] = [team_regex.findall(x)[0] if team_regex.findall(x) else None for x in ejections['description']] # else None to account for names not spelled in a way that matches the regex\n ejections['teamId'] = ejections['teamName'].map(team_mapping) \n ejections['coach_ejected'] = [1 if coaching_regex.search(x) else 0 for x in ejections['description']] \n ejections['player_ejected'] = 1 - ejections['coach_ejected'] \n # Get player name\n ejections['playerName'] = [team_full_regex.sub(\"\", ' '.join(x.split())) for x in ejections['description']]\n ejections['playerName'] = [coaching_regex.sub(\"\", ' '.join(x.split())) for x in ejections['playerName']]\n ejections['playerName'] = [pos_regex.sub(\"\", ' '.join(x.split())).strip() for x in ejections['playerName']]\n # If there is no match for a player use fuzzywuzzy to find the closest match\n ejections.loc[(ejections['player_ejected']==1), 'playerName'] = ejections.loc[(ejections['player_ejected']==1), 'playerName'].apply(lambda x: find_closest_playerName(x, players))\n ejections.loc[(ejections['player_ejected']==1), 'playerId'] = ejections.loc[(ejections['player_ejected']==1)].apply(lambda x: find_playerId(x, players, rosters), axis=1)\n\n t_tmp = t_tmp.merge(ejections.groupby('teamId')['coach_ejected'].sum().reset_index(), how='left', on='teamId')\n t_tmp['coach_ejected'] = t_tmp['coach_ejected'].fillna(0)\n t_tmp = t_tmp.merge(ejections.loc[ejections['player_ejected']==1,['playerId','player_ejected']], how='left', on='playerId')\n t_tmp['player_ejected'] = t_tmp['player_ejected'].fillna(0)\n else:\n t_tmp['coach_ejected'] = 0\n t_tmp['player_ejected'] = 0\n\n if not rosters.empty:\n # if 'teamId' not in t_tmp.columns:\n # t_tmp = t_tmp.merge(rosters[['playerId','teamId']], how='left', on='playerId')\n # t_tmp = t_tmp.merge(all_dates[['dailyDataDate_lead','teamId','nextDayGame']], how='left', left_on=['dailyDataDate', 'teamId'], right_on=['dailyDataDate_lead','teamId'])\n # t_tmp['nextDayGame'] = t_tmp['nextDayGame'].fillna(0) \n roster_dummies = pd.concat([rosters[['dailyDataDate','playerId']], pd.get_dummies(rosters['statusCode'])], axis=1)\n roster_dummies = roster_dummies.groupby([\"dailyDataDate\", \"playerId\"]).sum().reset_index()\n for col in ['A', 'BRV', 'D10', 'D60', 'D7', 'DEC','FME', 'PL', 'RES', 'RM', 'SU']:\n if col not in roster_dummies.columns:\n roster_dummies[col] = 0\n t_tmp = t_tmp.merge(roster_dummies, how='left', on=['dailyDataDate','playerId'])\n else: \n t_tmp[['A', 'BRV', 'D10', 'D60', 'D7', 'DEC','FME', 'PL', 'RES', 'RM', 'SU']] = 0\n t_tmp['nextDayGame'] = 0 # There should be a better way to handle this. Don't want to miss this just because rosters is missing \n\n if not transactions.empty:\n transactions_dummies = pd.concat([transactions[['dailyDataDate','playerId']], pd.get_dummies(transactions['typeCode'])], axis=1)\n transactions_dummies = transactions_dummies.groupby([\"dailyDataDate\", \"playerId\"]).sum().reset_index()\n for col in ['ASG', 'CLW', 'CU', 'DES', 'DFA', 'NUM','OPT', 'OUT', 'REL', 'RET', 'RTN', 'SC', 'SE', 'SFA', 'SGN', 'TR']:\n if col not in transactions_dummies.columns:\n transactions_dummies[col] = 0\n t_tmp = t_tmp.merge(transactions_dummies, how='left', on=['dailyDataDate','playerId'])\n else:\n t_tmp[['ASG', 'CLW', 'CU', 'DES', 'DFA', 'NUM','OPT', 'OUT', 'REL', 'RET', 'RTN', 'SC', 'SE', 'SFA', 'SGN', 'TR']] = 0\n\n if not awards.empty:\n awards_filtered = awards[awards['awardId'].isin(keep_awards + keep_annual_awards)].reset_index(drop=True)\n\n # Update awards counts\n awards_dict_tmp = {n: grp.to_dict('list') for n, grp in awards_filtered[['awardId','playerId']].groupby('playerId')}\n for k,v in awards_dict_tmp.items():\n if not k in awards_dict:\n awards_dict[k] = {}\n counts = np.unique(v['awardId'], return_counts=True)\n for feat, value in zip(counts[0], counts[1]):\n if feat in awards_dict[k]:\n awards_dict[k][feat] += value\n else:\n awards_dict[k][feat] = value\n\n awards_filtered = awards[awards['awardId'].isin(keep_awards)].reset_index(drop=True)\n if not awards_filtered.empty:\n\n awards_dummies = pd.concat([awards_filtered[['dailyDataDate','playerId']], pd.get_dummies(awards_filtered['awardId'])], axis=1)\n awards_dummies = awards_dummies.groupby([\"dailyDataDate\", \"playerId\"]).sum().reset_index()\n for col in keep_awards:\n if col not in awards_dummies.columns:\n awards_dummies[col] = 0\n t_tmp = t_tmp.merge(awards_dummies, how='left', on=['dailyDataDate','playerId'])\n else:\n t_tmp[keep_awards] = 0 \n else:\n t_tmp[keep_awards] = 0\n\n if t_tmp.shape[0]!=eng_shape[0]:\n print(\"awards: t_tmp length does not match engagement frame length, check for duplicated data\")\n t_tmp = t_tmp[~t_tmp[['playerId']].duplicated()]\n\n awards_df = pd.DataFrame.from_dict(awards_dict, orient='index').fillna(0)\n awards_df.columns = [f'{x}_career' for x in awards_df.columns]\n t_tmp = t_tmp.merge(awards_df.reset_index().rename(columns={'index': 'playerId'}), how='left', on='playerId')\n t_tmp[awards_df.columns] = t_tmp[awards_df.columns].fillna(0)\n\n if not standings.empty:\n standings = standings.replace(\"-\",0.0)\n object_cols = standings.select_dtypes(exclude=['float']).columns\n standings[object_cols] = standings[object_cols].apply(pd.to_numeric, downcast='float', errors='coerce')\n bool_cols = standings.select_dtypes(include=['boolean']).columns\n standings[bool_cols] = standings[bool_cols].astype(int) \n t_tmp = t_tmp.merge(standings, how='left', on=['teamId'], suffixes=['','_team_standings'])\n t_tmp['team_games_played'] = t_tmp['wins'] + t_tmp['losses']\n if t_tmp.shape[0]!=eng_shape[0]:\n print(\"standings: t_tmp length does not match engagement frame length, check for duplicated data\")\n t_tmp = t_tmp[~t_tmp[['playerId']].duplicated()]\n\n\n if len(win_streaks) > 0:\n t_tmp['team_win_streak'] = t_tmp['teamId'].map(win_streaks)\n if not hitting_history_df.empty:\n t_tmp = t_tmp.merge(hitting_history_df, how='left', on='playerId')\n t_tmp['hr_rank'] = t_tmp['homeRuns_season'].rank(ascending=False)\n if t_tmp.shape[0]!=eng_shape[0]:\n print(\"hitting_history_df: t_tmp length does not match engagement frame length, check for duplicated data\")\n t_tmp = t_tmp[~t_tmp[['playerId']].duplicated()]\n\n if not pitching_history_df.empty:\n t_tmp = t_tmp.merge(pitching_history_df, how='left', on='playerId')\n # Calculate ERA\n # there are no more standings after season end so team_games_played is no longer known\n if 'team_games_played' in t_tmp.columns:\n t_tmp['era'] = 9 * (t_tmp['earnedRuns_season']/ t_tmp['inningsPitched_season'])\n t_tmp['era_rank'] = t_tmp.loc[t_tmp['inningsPitched_season'] >= t_tmp['team_games_played'], 'era'].rank(method='min')\n if t_tmp.shape[0]!=eng_shape[0]:\n print(\"pitching_history_df: t_tmp length does not match engagement frame length, check for duplicated data\")\n t_tmp = t_tmp[~t_tmp[['playerId']].duplicated()]\n if not fielding_history_df.empty:\n t_tmp = t_tmp.merge(fielding_history_df, how='left', on='playerId')\n if t_tmp.shape[0]!=eng_shape[0]:\n print(\"fielding_history_df: t_tmp length does not match engagement frame length, check for duplicated data\")\n t_tmp = t_tmp[~t_tmp[['playerId']].duplicated()]\n\n\n player_countries = ['Aruba', 'Australia', 'Bahamas', 'Brazil', 'Canada',\n 'China', 'Colombia', 'Cuba', 'Curacao', 'Dominican Republic', 'Germany',\n 'Honduras', 'Japan', 'Lithuania', 'Mexico', 'Netherlands', 'Nicaragua',\n 'Northern Ireland', 'Panama', 'Peru', 'Puerto Rico', 'Saudi Arabia',\n 'South Africa', 'South Korea', 'Taiwan', 'U.S. Virgin Islands', 'USA',\n 'Venezuela']\n t_tmp = t_tmp.merge(player_country_dummies, how='left', on='playerId')\n\n # Add games features\n # if not games.empty and not p_box_scores.empty:\n # games['dayNight'] = games['dayNight'].map({'day': 0, 'night': 1})\n # games['homeWinner'] = games['homeWinner'].fillna(-1).astype(float)\n # t_tmp = t_tmp.merge(games[['gamePk', 'dayNight','homeWinPct','awayWinPct','homeScore','awayScore','homeWinner']], how='left', on='gamePk')\n # Add Twitter features\n if not p_twitter_recent.empty:\n t_tmp = t_tmp.merge(p_twitter_recent[['playerId','numberOfFollowers']], how='left', on=['playerId'])\n\n if not p_twitter_delta.empty:\n t_tmp = t_tmp.merge(p_twitter_delta, how='left', on=['playerId'])\n\n ### TRAILING AGGREGATION MERGES\n t_tmp['ddd_month'] = np.floor(t_tmp.dailyDataDate/100).clip(upper=MAX_MONTH).astype(int)\n t_tmp = t_tmp.merge(dt_player_aggregations,how=\"left\",on=['playerId','ddd_month'])\n\n if 'gamePk' in t_tmp.columns:\n t_tmp['played_game'] = t_tmp['gamePk'].notnull().astype(float)\n else:\n t_tmp['played_game'] = 0.0\n\n t_tmp = t_tmp.merge(dt_player_game_aggregations,how=\"left\",on=['playerId','ddd_month','played_game'])\n\n # t_tmp = t_tmp.merge(recent_player_means, how='left', on='playerId')\n t_tmp['monthday'] = t_tmp['dailyDataDate'].astype(str).str[4:].astype(int)\n t_tmp['dayofweek'] = pd.to_datetime(t_tmp['date_playerId'].str.split(\"_\", expand=True)[0]).dt.dayofweek\n t_tmp['data_dayofmonth'] = t_tmp['dailyDataDate'].astype(str).str[6:].astype(int)\n t_tmp['eng_dayofmonth'] = pd.to_datetime(t_tmp['date_playerId'].str.split(\"_\", expand=True)[0]).dt.day\n \n # Fill season values with 2021\n t_tmp['season'] = season\n \n# t.append(t_tmp)\n\n\n use_cols = lgb_target1.feature_name()\n # Add any missing columns so that it does not crash\n missing_cols = [col for col in use_cols if col not in t_tmp.columns]\n for col in missing_cols:\n print(f'lgb_target1 missing: {col}')\n t_tmp[col] = np.nan\n\n sub['target1_v30'] = np.clip(lgb_target1.predict(t_tmp[use_cols]), 0, 100)\n sub['target2_v30'] = np.clip(lgb_target2.predict(t_tmp[use_cols]), 0, 100)\n sub['target3_v30'] = np.clip(lgb_target3.predict(t_tmp[use_cols]), 0, 100)\n sub['target4_v30'] = np.clip(lgb_target4.predict(t_tmp[use_cols]), 0, 100)\n \n use_cols = lgb_bfa_target1.feature_name()\n # Add any missing columns so that it does not crash\n missing_cols = [col for col in use_cols if col not in t_tmp.columns]\n for col in missing_cols:\n print(f'lgb_bfa_target1 missing: {col}')\n t_tmp[col] = np.nan\n \n sub['target1_v30_bfa'] = np.clip(lgb_bfa_target1.predict(t_tmp[use_cols]), 0, 100)\n sub['target2_v30_bfa'] = np.clip(lgb_bfa_target2.predict(t_tmp[use_cols]), 0, 100)\n sub['target3_v30_bfa'] = np.clip(lgb_bfa_target3.predict(t_tmp[use_cols]), 0, 100)\n sub['target4_v30_bfa'] = np.clip(lgb_bfa_target4.predict(t_tmp[use_cols]), 0, 100)\n\n use_cols = lgb_dblsqrt_target1.feature_name()\n # Add any missing columns so that it does not crash\n missing_cols = [col for col in use_cols if col not in t_tmp.columns]\n for col in missing_cols:\n print(f'lgb_dblsqrt_target1 missing: {col}')\n t_tmp[col] = np.nan\n \n sub['target1_v30_dblsqrt'] = np.clip(lgb_dblsqrt_target1.predict(t_tmp[use_cols])**2**2, 0, 100)\n sub['target2_v30_dblsqrt'] = np.clip(lgb_dblsqrt_target2.predict(t_tmp[use_cols])**2**2, 0, 100)\n sub['target3_v30_dblsqrt'] = np.clip(lgb_dblsqrt_target3.predict(t_tmp[use_cols])**2**2, 0, 100)\n sub['target4_v30_dblsqrt'] = np.clip(lgb_dblsqrt_target4.predict(t_tmp[use_cols])**2**2, 0, 100)\n\n dart_use_cols = lgb_dart_target1.feature_name()\n # Add any missing columns so that it does not crash\n missing_cols = [col for col in dart_use_cols if col not in t_tmp.columns]\n for col in missing_cols:\n print(f'lgb_dart_target1 missing: {col}')\n t_tmp[col] = np.nan\n sub['target1_dart'] = np.clip(lgb_dart_target1.predict(t_tmp[dart_use_cols]), 0, 100)\n sub['target2_dart'] = np.clip(lgb_dart_target2.predict(t_tmp[dart_use_cols]), 0, 100)\n sub['target3_dart'] = np.clip(lgb_dart_target3.predict(t_tmp[dart_use_cols]), 0, 100)\n sub['target4_dart'] = np.clip(lgb_dart_target4.predict(t_tmp[dart_use_cols]), 0, 100)\n\n with open(\"../input/d/brandenkmurray/mlbmodels/xgb_v30_use_cols.txt\") as f:\n xgb_use_cols = [x.rstrip() for x in f.readlines()]\n# xgb_use_cols = xgb_target1.feature_names\n missing_cols = [col for col in xgb_use_cols if col not in t_tmp.columns]\n missing_cols_filled = [x.replace(\" \", \"_\") for x in missing_cols]\n t_tmp = t_tmp.rename(columns={k: v for k,v in zip(missing_cols_filled, missing_cols)})\n for col in missing_cols:\n print(f\"{col} is missing for XGB model. Adding and filling with NaN\")\n t_tmp[col] = np.nan\n\n sub['target1_xgb'] = np.clip(xgb_target1.predict(xgb.DMatrix(t_tmp[xgb_use_cols].fillna(-99999).replace(np.inf, -99999)))**2**2, 0, 100)\n sub['target2_xgb'] = np.clip(xgb_target2.predict(xgb.DMatrix(t_tmp[xgb_use_cols].fillna(-99999).replace(np.inf, -99999)))**2**2, 0, 100)\n sub['target3_xgb'] = np.clip(xgb_target3.predict(xgb.DMatrix(t_tmp[xgb_use_cols].fillna(-99999).replace(np.inf, -99999)))**2**2, 0, 100)\n sub['target4_xgb'] = np.clip(xgb_target4.predict(xgb.DMatrix(t_tmp[xgb_use_cols].fillna(-99999).replace(np.inf, -99999)))**2**2, 0, 100)\n\n\n\n correlates = ['hitBatsmen',\n 'no_hitter',\n 'home_team_box_score',\n 'hitBatsmen_team_box_score',\n 'season_team_standings',\n 'sportGamesBack',\n 'nlWins',\n 'nlLosses',\n 'errors_1_games_ago']\n\n t_tmp = t_tmp.drop(columns=correlates)\n use_cols = lgb_john_target1.feature_name()\n missing_cols = [col for col in use_cols if col not in t_tmp.columns]\n for col in missing_cols:\n print(col + \" missing for John's model\")\n t_tmp[col] = np.nan\n\n sub['target1_john'] = np.clip(lgb_john_target1.predict(t_tmp[use_cols]), 0, 100)\n sub['target2_john'] = np.clip(lgb_john_target2.predict(t_tmp[use_cols]), 0, 100)\n sub['target3_john'] = np.clip(lgb_john_target3.predict(t_tmp[use_cols]), 0, 100)\n sub['target4_john'] = np.clip(lgb_john_target4.predict(t_tmp[use_cols]), 0, 100) \n\n sub['target1'] = (sub['target1_john']*0.4) + ((sub['target1_v30_bfa']*0.2 + sub['target1_v30']*0.1 + sub['target1_v30_dblsqrt']*0.7)*0.1) + (sub['target1_xgb']*0.1) + (sub['target1_dart']*0.4)\n sub['target2'] = (sub['target2_john']*0.4) + ((sub['target2_v30_bfa']*0.2 + sub['target2_v30']*0.1 + sub['target2_v30_dblsqrt']*0.7)*0.1) + (sub['target2_xgb']*0.1) + (sub['target2_dart']*0.4)\n sub['target3'] = (sub['target3_john']*0.4) + ((sub['target3_v30_bfa']*0.2 + sub['target3_v30']*0.1 + sub['target3_v30_dblsqrt']*0.7)*0.1) + (sub['target3_xgb']*0.1) + (sub['target3_dart']*0.4)\n sub['target4'] = (sub['target4_john']*0.4) + ((sub['target4_v30_bfa']*0.2 + sub['target4_v30']*0.1 + sub['target4_v30_dblsqrt']*0.7)*0.1) + (sub['target4_xgb']*0.1) + (sub['target4_dart']*0.4)\n \n except Exception as e:\n #If all else fails try to use player means\n print(f'Main loop failed: {e}')\n try:\n print(\"Using player rolling12 means\")\n sub['ddd_month'] = np.floor(sub.dailyDataDate/100).clip(upper=MAX_MONTH).astype(int)\n sub = sub.drop(['target1','target2','target3','target4'], axis=1)\n sub = sub.merge(dt_player_game_aggregations[['playerId','ddd_month','roll12_target1_p_gameday_median', 'roll12_target2_p_gameday_median',\n 'roll12_target3_p_gameday_median', 'roll12_target4_p_gameday_median']],how=\"left\",on=['playerId','ddd_month'])\n sub = sub.rename({k:v for k,v in zip(['roll12_target1_p_gameday_median', 'roll12_target2_p_gameday_median',\n 'roll12_target3_p_gameday_median', 'roll12_target4_p_gameday_median'], ['target1','target2','target3','target4'])}, axis=1)\n except Exception as e:\n print(e)\n # If player medians fail, use overall medians\n print(\"Player medians failed. Using overall medians\")\n sub['target1'] = 0.001046\n sub['target2'] = 0.521472\n sub['target3'] = 0.001735\n sub['target4'] = 0.226034\n \n # Do a final check to ensure there are no duplicate players that will cause a scoring error\n sub = sub[~sub[['playerId']].duplicated()]\n \n# sub_list.append(sub)\n env.predict(sub[['date_playerId','target1','target2','target3','target4']])\n\n# eng_lag = sub[['playerId','target1','target2','target3','target4']].copy()\n# eng_lag = eng_lag.rename({'target1': 'target1_lag',\n# 'target2': 'target2_lag',\n# 'target3': 'target3_lag',\n# 'target4': 'target4_lag'}, axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-09-20T20:35:08.403553Z","iopub.execute_input":"2021-09-20T20:35:08.404022Z","iopub.status.idle":"2021-09-20T20:38:48.753512Z","shell.execute_reply.started":"2021-09-20T20:35:08.403986Z","shell.execute_reply":"2021-09-20T20:38:48.752256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sub_all = pd.concat(sub_list)\n# include_players = players[players['playerForTestSetAndFuturePreds']==1]['playerId'].tolist()\n# sub_all = sub_all[sub_all['playerId'].isin(include_players)]\n\n# print(\"v30\")\n# may_mae_list = []\n# june_mae_list = []\n# for target in ['target1', 'target2', 'target3', 'target4']:\n# may_mae_list.append(np.mean(np.abs(sub_all.loc[(sub_all['dailyDataDate']>=20210501) & (sub_all['dailyDataDate']<=20210531), target + '_v30'] - sub_all.loc[(sub_all['dailyDataDate']>=20210501) & (sub_all['dailyDataDate']<=20210531), target + '_act'])))\n# june_mae_list.append(np.mean(np.abs(sub_all.loc[(sub_all['dailyDataDate']>=20210601) & (sub_all['dailyDataDate']<=20210630), target + '_v30'] - sub_all.loc[(sub_all['dailyDataDate']>=20210601) & (sub_all['dailyDataDate']<=20210630), target + '_act'])))\n# # print(may_mae_list)\n# print(f\"May MAE: {np.mean(may_mae_list)}\")\n# print(f\"June MAE: {np.mean(june_mae_list)}\")\n\n# print(\"v30_bfa\")\n# may_mae_list = []\n# june_mae_list = []\n# for target in ['target1', 'target2', 'target3', 'target4']:\n# may_mae_list.append(np.mean(np.abs(sub_all.loc[(sub_all['dailyDataDate']>=20210501) & (sub_all['dailyDataDate']<=20210531), target + '_v30_bfa'] - sub_all.loc[(sub_all['dailyDataDate']>=20210501) & (sub_all['dailyDataDate']<=20210531), target + '_act'])))\n# june_mae_list.append(np.mean(np.abs(sub_all.loc[(sub_all['dailyDataDate']>=20210601) & (sub_all['dailyDataDate']<=20210630), target + '_v30_bfa'] - sub_all.loc[(sub_all['dailyDataDate']>=20210601) & (sub_all['dailyDataDate']<=20210630), target + '_act'])))\n# # print(may_mae_list)\n# print(f\"May MAE: {np.mean(may_mae_list)}\")\n# print(f\"June MAE: {np.mean(june_mae_list)}\")\n\n# print(\"v30_dblsqrt\")\n# may_mae_list = []\n# june_mae_list = []\n# for target in ['target1', 'target2', 'target3', 'target4']:\n# may_mae_list.append(np.mean(np.abs(sub_all.loc[(sub_all['dailyDataDate']>=20210501) & (sub_all['dailyDataDate']<=20210531), target + '_v30_dblsqrt'] - sub_all.loc[(sub_all['dailyDataDate']>=20210501) & (sub_all['dailyDataDate']<=20210531), target + '_act'])))\n# june_mae_list.append(np.mean(np.abs(sub_all.loc[(sub_all['dailyDataDate']>=20210601) & (sub_all['dailyDataDate']<=20210630), target + '_v30_dblsqrt'] - sub_all.loc[(sub_all['dailyDataDate']>=20210601) & (sub_all['dailyDataDate']<=20210630), target + '_act'])))\n# # print(may_mae_list)\n# print(f\"May MAE: {np.mean(may_mae_list)}\")\n# print(f\"June MAE: {np.mean(june_mae_list)}\")\n\n# print(\"john\")\n# may_mae_list = []\n# june_mae_list = []\n# for target in ['target1', 'target2', 'target3', 'target4']:\n# may_mae_list.append(np.mean(np.abs(sub_all.loc[(sub_all['dailyDataDate']>=20210501) & (sub_all['dailyDataDate']<=20210531), target + '_john'] - sub_all.loc[(sub_all['dailyDataDate']>=20210501) & (sub_all['dailyDataDate']<=20210531), target + '_act'])))\n# june_mae_list.append(np.mean(np.abs(sub_all.loc[(sub_all['dailyDataDate']>=20210601) & (sub_all['dailyDataDate']<=20210630), target + '_john'] - sub_all.loc[(sub_all['dailyDataDate']>=20210601) & (sub_all['dailyDataDate']<=20210630), target + '_act'])))\n# # print(may_mae_list)\n# print(f\"May MAE: {np.mean(may_mae_list)}\")\n# print(f\"June MAE: {np.mean(june_mae_list)}\")\n\n# print(\"xgb\")\n# may_mae_list = []\n# june_mae_list = []\n# for target in ['target1', 'target2', 'target3', 'target4']:\n# may_mae_list.append(np.mean(np.abs(sub_all.loc[(sub_all['dailyDataDate']>=20210501) & (sub_all['dailyDataDate']<=20210531), target + '_xgb'] - sub_all.loc[(sub_all['dailyDataDate']>=20210501) & (sub_all['dailyDataDate']<=20210531), target + '_act'])))\n# june_mae_list.append(np.mean(np.abs(sub_all.loc[(sub_all['dailyDataDate']>=20210601) & (sub_all['dailyDataDate']<=20210630), target + '_xgb'] - sub_all.loc[(sub_all['dailyDataDate']>=20210601) & (sub_all['dailyDataDate']<=20210630), target + '_act'])))\n# # print(may_mae_list)\n# print(f\"May MAE: {np.mean(may_mae_list)}\")\n# print(f\"June MAE: {np.mean(june_mae_list)}\")\n\n# print(\"dart\")\n# may_mae_list = []\n# june_mae_list = []\n# for target in ['target1', 'target2', 'target3', 'target4']:\n# may_mae_list.append(np.mean(np.abs(sub_all.loc[(sub_all['dailyDataDate']>=20210501) & (sub_all['dailyDataDate']<=20210531), target + '_dart'] - sub_all.loc[(sub_all['dailyDataDate']>=20210501) & (sub_all['dailyDataDate']<=20210531), target + '_act'])))\n# june_mae_list.append(np.mean(np.abs(sub_all.loc[(sub_all['dailyDataDate']>=20210601) & (sub_all['dailyDataDate']<=20210630), target + '_dart'] - sub_all.loc[(sub_all['dailyDataDate']>=20210601) & (sub_all['dailyDataDate']<=20210630), target + '_act'])))\n# # print(may_mae_list)\n# print(f\"May MAE: {np.mean(may_mae_list)}\")\n# print(f\"June MAE: {np.mean(june_mae_list)}\")\n\n# print(\"blend\")\n# may_mae_list = []\n# june_mae_list = []\n# for target in ['target1', 'target2', 'target3', 'target4']:\n# may_mae_list.append(np.mean(np.abs(sub_all.loc[(sub_all['dailyDataDate']>=20210501) & (sub_all['dailyDataDate']<=20210531), target] - sub_all.loc[(sub_all['dailyDataDate']>=20210501) & (sub_all['dailyDataDate']<=20210531), target + '_act'])))\n# june_mae_list.append(np.mean(np.abs(sub_all.loc[(sub_all['dailyDataDate']>=20210601) & (sub_all['dailyDataDate']<=20210630), target] - sub_all.loc[(sub_all['dailyDataDate']>=20210601) & (sub_all['dailyDataDate']<=20210630), target + '_act'])))\n# # print(may_mae_list)\n# print(f\"May MAE: {np.mean(may_mae_list)}\")\n# print(f\"June MAE: {np.mean(june_mae_list)}\")","metadata":{"execution":{"iopub.status.busy":"2021-07-30T23:22:47.736076Z","iopub.status.idle":"2021-07-30T23:22:47.736599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# t_df = pd.concat(t)\n# t_df.to_csv(\"./train_features.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-30T23:22:47.737395Z","iopub.status.idle":"2021-07-30T23:22:47.737898Z"},"trusted":true},"execution_count":null,"outputs":[]}]}