{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook uses lightGBM to make predictions.\n\nWe use the following features\n* playerId\n* position\n* teamId(rosters)\n* status(rosters)\n* playerBoxScores\n\nand the date 20200401~20200431 as the validation data.\n\nBut I think there is room for improvement.  \nIf you have better ways, I would appreciate it if you could comment on it.\n\nこのnotebookではlightGBMを使って予測します。\n\n特徴量は以下のものを使用しています。\n* playerId\n* position\n* teamId(rosters)\n* status(rosters)\n* playerBoxScores\n\n20200401~20200431を日時をvalidation dataとしていますが、一考の余地がありそうです。  \nもし良さそうな方法があればコメントしていただけると幸いです。","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"https://www.kaggle.com/columbia2131/mlb-lightgbm-starter-dataset-code-en-ja","metadata":{}},{"cell_type":"markdown","source":"## About Dataset","metadata":{}},{"cell_type":"markdown","source":"Train.csv is stored as a csv file with each column as follows.  \n\ntrain.csvを以下のようにして各カラムをcsvファイルとして保管しています。","metadata":{}},{"cell_type":"code","source":"\"\"\"\n%capture\n\n!pip install pandarallel \n\nimport gc\n\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\n\nfrom pandarallel import pandarallel\npandarallel.initialize()\n\nBASE_DIR = Path('../input/mlb-player-digital-engagement-forecasting')\ntrain = pd.read_csv(BASE_DIR / 'train.csv')\n\nnull = np.nan\ntrue = True\nfalse = False\n\nfor col in train.columns:\n\n    if col == 'date': continue\n\n    _index = train[col].notnull()\n    train.loc[_index, col] = train.loc[_index, col].parallel_apply(lambda x: eval(x))\n\n    outputs = []\n    for index, date, record in train.loc[_index, ['date', col]].itertuples():\n        _df = pd.DataFrame(record)\n        _df['index'] = index\n        _df['date'] = date\n        outputs.append(_df)\n\n    outputs = pd.concat(outputs).reset_index(drop=True)\n\n    outputs.to_csv(f'{col}_train.csv', index=False)\n    outputs.to_pickle(f'{col}_train.pkl')\n\n    del outputs\n    del train[col]\n    gc.collect()\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:41:12.015994Z","iopub.execute_input":"2021-07-26T06:41:12.017049Z","iopub.status.idle":"2021-07-26T06:43:42.396294Z","shell.execute_reply.started":"2021-07-26T06:41:12.016995Z","shell.execute_reply":"2021-07-26T06:43:42.395142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{"execution":{"iopub.status.busy":"2021-06-16T09:14:33.869464Z","iopub.execute_input":"2021-06-16T09:14:33.869905Z","iopub.status.idle":"2021-06-16T09:14:33.874766Z","shell.execute_reply.started":"2021-06-16T09:14:33.869879Z","shell.execute_reply":"2021-06-16T09:14:33.873097Z"}}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom sklearn.metrics import mean_absolute_error\nfrom datetime import timedelta\nfrom functools import reduce\nfrom tqdm import tqdm\nimport lightgbm as lgbm\nimport mlb\nimport pickle\nfrom datetime import datetime as dt\nimport copy\nfrom sklearn.decomposition import PCA\nimport math\nimport gc","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:32:18.7374Z","iopub.execute_input":"2021-07-26T06:32:18.73798Z","iopub.status.idle":"2021-07-26T06:32:20.812678Z","shell.execute_reply.started":"2021-07-26T06:32:18.73786Z","shell.execute_reply":"2021-07-26T06:32:20.811561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_DIR = Path('../input/c/mlb-player-digital-engagement-forecasting')\nTRAIN_DIR = Path('../input/mlb-pdef-train-dataset')","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:32:20.814338Z","iopub.execute_input":"2021-07-26T06:32:20.81473Z","iopub.status.idle":"2021-07-26T06:32:20.81944Z","shell.execute_reply.started":"2021-07-26T06:32:20.81469Z","shell.execute_reply":"2021-07-26T06:32:20.81825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def unpack_json(json_str):\n    return np.nan if pd.isna(json_str) else pd.read_json(json_str)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:32:20.821474Z","iopub.execute_input":"2021-07-26T06:32:20.822097Z","iopub.status.idle":"2021-07-26T06:32:20.82964Z","shell.execute_reply.started":"2021-07-26T06:32:20.822037Z","shell.execute_reply":"2021-07-26T06:32:20.828931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:32:20.831341Z","iopub.execute_input":"2021-07-26T06:32:20.832011Z","iopub.status.idle":"2021-07-26T06:33:46.18475Z","shell.execute_reply.started":"2021-07-26T06:32:20.831969Z","shell.execute_reply":"2021-07-26T06:33:46.183942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:55:07.356446Z","iopub.execute_input":"2021-07-26T06:55:07.357026Z","iopub.status.idle":"2021-07-26T06:55:07.363854Z","shell.execute_reply.started":"2021-07-26T06:55:07.356974Z","shell.execute_reply":"2021-07-26T06:55:07.363055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:37:30.053201Z","iopub.execute_input":"2021-07-26T06:37:30.053654Z","iopub.status.idle":"2021-07-26T06:37:30.080045Z","shell.execute_reply.started":"2021-07-26T06:37:30.053625Z","shell.execute_reply":"2021-07-26T06:37:30.079148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"players = pd.read_csv(BASE_DIR / 'players.csv')\n\nrosters = pd.read_pickle(TRAIN_DIR / 'rosters_train.pkl')\ntargets = pd.read_pickle(TRAIN_DIR / 'nextDayPlayerEngagement_train.pkl')\nscores1 = pd.read_pickle(TRAIN_DIR / 'playerBoxScores_train.pkl')\nscores = scores1.groupby(['playerId','date']).sum().reset_index()\ntwitter = pd.read_pickle(TRAIN_DIR / \"playerTwitterFollowers_train.pkl\")\ngames = pd.read_pickle(TRAIN_DIR / 'games_train.pkl')\nevents = pd.read_pickle(TRAIN_DIR / 'events_train.pkl')\nstandings = pd.read_pickle(TRAIN_DIR / 'standings_train.pkl')\nteamtwitter = pd.read_pickle(TRAIN_DIR / 'teamTwitterFollowers_train.pkl')\ntransaction = pd.read_pickle(TRAIN_DIR / 'teamTwitterFollowers_train.pkl')\nawards = pd.read_csv(BASE_DIR / 'awards.csv')\nseasons = pd.read_csv(BASE_DIR / 'seasons.csv')\nteams = pd.read_csv(BASE_DIR / 'teams.csv')\nplayer_target_stats = pd.read_csv(\"../input/old-pts/player_target_stats.csv\")\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:20:21.034274Z","iopub.execute_input":"2021-07-26T06:20:21.034814Z","iopub.status.idle":"2021-07-26T06:20:37.968705Z","shell.execute_reply.started":"2021-07-26T06:20:21.034749Z","shell.execute_reply":"2021-07-26T06:20:37.967681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def unpack_json(json_str):\n    return np.nan if pd.isna(json_str) else pd.read_json(json_str)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:20:37.970177Z","iopub.execute_input":"2021-07-26T06:20:37.970459Z","iopub.status.idle":"2021-07-26T06:20:37.97484Z","shell.execute_reply.started":"2021-07-26T06:20:37.970432Z","shell.execute_reply":"2021-07-26T06:20:37.973734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"games = pd.read_pickle(TRAIN_DIR / 'games_train.pkl')\ngames[games.duplicated(subset=[\"gamePk\"], keep=False)]\n#games.index[games[\"detailedGameState\"] == \"Postponed\"].shape\n#games.drop(games.loc[games['detailedGameState']=='Postponed'].index, inplace=True)\n#games[\"detailedGameState\"].unique()\n#games[games.duplicated(subset=[\"gamePk\"], keep=False)]\n#games['detailedGameState']=='Postponed']だとスコアはキロクされていない\ngames","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:20:37.976122Z","iopub.execute_input":"2021-07-26T06:20:37.976419Z","iopub.status.idle":"2021-07-26T06:20:38.069938Z","shell.execute_reply.started":"2021-07-26T06:20:37.976391Z","shell.execute_reply":"2021-07-26T06:20:38.068878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"awards2 = awards.groupby(\"playerId\").count()\nawards2 = awards2.reset_index()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:20:38.07123Z","iopub.execute_input":"2021-07-26T06:20:38.071542Z","iopub.status.idle":"2021-07-26T06:20:38.087943Z","shell.execute_reply.started":"2021-07-26T06:20:38.071511Z","shell.execute_reply":"2021-07-26T06:20:38.086857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"teamtwitter[\"teamnumberOfFollowers\"] = teamtwitter[\"numberOfFollowers\"]\nteamtwi = teamtwitter.groupby(\"teamId\").mean()[\"teamnumberOfFollowers\"].reset_index()\nteamtwi","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:20:38.090459Z","iopub.execute_input":"2021-07-26T06:20:38.090752Z","iopub.status.idle":"2021-07-26T06:20:38.112163Z","shell.execute_reply.started":"2021-07-26T06:20:38.090724Z","shell.execute_reply":"2021-07-26T06:20:38.111134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets_cols = ['playerId', 'target1', 'target2', 'target3', 'target4', 'date']\nplayers_cols = ['playerId', 'primaryPositionName']\nrosters_cols = ['playerId', 'teamId', 'status', 'date']\nyesterday_targets_cols = [\"date\",\"playerId\",\"yest_target1\",\"yest_target2\",\"yest_target3\",\"yest_target4\"]\nscores_cols = ['playerId', 'battingOrder', 'gamesPlayedBatting', 'flyOuts',\n       'groundOuts', 'runsScored', 'doubles', 'triples', 'homeRuns',\n       'strikeOuts', 'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch',\n       'atBats', 'caughtStealing', 'stolenBases', 'groundIntoDoublePlay',\n       'groundIntoTriplePlay', 'plateAppearances', 'totalBases', 'rbi',\n       'leftOnBase', 'sacBunts', 'sacFlies', 'catchersInterference',\n       'pickoffs', 'gamesPlayedPitching', 'gamesStartedPitching',\n       'completeGamesPitching', 'shutoutsPitching', 'winsPitching',\n       'lossesPitching', 'flyOutsPitching', 'airOutsPitching',\n       'groundOutsPitching', 'runsPitching', 'doublesPitching',\n       'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching',\n       'baseOnBallsPitching', 'intentionalWalksPitching', 'hitsPitching',\n       'hitByPitchPitching', 'atBatsPitching', 'caughtStealingPitching',\n       'stolenBasesPitching', 'inningsPitched', 'saveOpportunities',\n       'earnedRuns', 'battersFaced', 'outsPitching', 'pitchesThrown', 'balls',\n       'strikes', 'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching',\n       'rbiPitching', 'gamesFinishedPitching', 'inheritedRunners',\n       'inheritedRunnersScored', 'catchersInterferencePitching',\n       'sacBuntsPitching', 'sacFliesPitching', 'saves', 'holds', 'blownSaves',\n       'assists', 'putOuts', 'errors', 'chances', 'date',\"gamePk\"]\ngames_cols = [\"gamePk\",\"homeId\",\"dayNight\",\"seriesDescription\",\"gamesInSeries\",\"homeWinner\",\"awayWinner\",\"homeScore\",\"awayScore\",\"gameType\",\n              \"gameDate\"]\nplayertwitter_cols = [\"playerId\",\"numberOfFollowers\"]\nawards_cols = [\"playerId\",\"awardName\"]\nstandings_cols = [\"date\",\"teamId\",\"divisionRank\",\"divisionLeader\",\"wildCardLeader\",\"leagueRank\",\"divisionId\",\"gameDate\"]\nteamtwitter_cols = [\"teamId\",\"teamnumberOfFollowers\"]\ntargets[\"engagementMetricsDate\"] = targets[\"engagementMetricsDate\"].str.replace('-', '')\nyesterday_targets = targets.drop('date', axis=1)\nyesterday_targets = yesterday_targets.rename(columns={'engagementMetricsDate':'date', 'target1': 'yest_target1','target2': 'yest_target2','target3': 'yest_target3','target4': 'yest_target4'})\nyesterday_targets[\"date\"] = yesterday_targets[\"date\"].astype(int)\nyesterday_targets[\"date\"]","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:20:38.116041Z","iopub.execute_input":"2021-07-26T06:20:38.116362Z","iopub.status.idle":"2021-07-26T06:20:40.633211Z","shell.execute_reply.started":"2021-07-26T06:20:38.11633Z","shell.execute_reply":"2021-07-26T06:20:40.632211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_cols = ['label_playerId', 'label_primaryPositionName', 'label_teamId',\n       'label_status', 'battingOrder', 'gamesPlayedBatting', 'flyOuts',\n       'groundOuts', 'runsScored','homeRuns',\n       'strikeOuts', 'baseOnBalls',  'hits', 'hitByPitch',\n       'atBats', 'stolenBases', \n       'plateAppearances', 'totalBases', 'rbi',\n       'leftOnBase', 'sacFlies',\n       'gamesPlayedPitching', 'gamesStartedPitching',\n       'completeGamesPitching','winsPitching',\n       'lossesPitching', 'flyOutsPitching', 'airOutsPitching',\n       'groundOutsPitching', 'runsPitching', 'doublesPitching',\n       'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching',\n       'baseOnBallsPitching',  'hitsPitching',\n       'hitByPitchPitching', 'atBatsPitching', 'caughtStealingPitching',\n       'stolenBasesPitching', 'inningsPitched', 'saveOpportunities',\n       'earnedRuns', 'battersFaced', 'outsPitching', 'pitchesThrown', 'balls',\n       'strikes', 'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching',\n       'rbiPitching', 'gamesFinishedPitching', 'inheritedRunners',\n       'inheritedRunnersScored',\n       'sacFliesPitching', 'saves', 'holds',\n       'assists', 'putOuts', 'errors', 'chances','target1_mean',\n 'target1_median',\n 'target1_std',\n 'target1_min',\n 'target1_max',\n 'target1_prob',\n 'target2_mean',\n 'target2_median',\n 'target2_std',\n 'target2_min',\n 'target2_max',\n 'target2_prob',\n 'target3_mean',\n 'target3_median',\n 'target3_std',\n 'target3_min',\n 'target3_max',\n 'target3_prob',\n 'target4_mean',\n 'target4_median',\n 'target4_std',\n 'target4_min',\n 'target4_max',\n 'target4_prob',\"divisionId\",\"teamnumberOfFollowers\",\"preseasonhits\",\"preseasonhomerun\"]\nfeature_cols2 = ['label_playerId', 'label_primaryPositionName', 'label_teamId',\n       'label_status', 'battingOrder', 'gamesPlayedBatting', 'flyOuts',\n       'groundOuts', 'runsScored',# 'doubles', 'triples', 'homeRuns',\n       'baseOnBalls', 'hits',\n      # 'atBats', 'caughtStealing', 'stolenBases', 'groundIntoDoublePlay',\n       'plateAppearances', 'totalBases', 'rbi',\n      # 'leftOnBase', 'sacBunts', 'sacFlies', 'catchersInterference',\n       'pickoffs', 'gamesPlayedPitching', 'gamesStartedPitching',\n       'winsPitching',\n       'lossesPitching',# 'flyOutsPitching', 'airOutsPitching',\n       'runsPitching', \n       'strikeOutsPitching',\n       #'hitsPitching',\n       'hitByPitchPitching',  'caughtStealingPitching',\n       'stolenBasesPitching', 'inningsPitched',\n        'battersFaced', \n        'balks', 'pickoffsPitching',\n        'inheritedRunners',\n        \n       #'sacBuntsPitching', 'sacFliesPitching', 'saves', 'holds', 'blownSaves',\n       'putOuts','chances','target1_mean',\n 'target1_median',\n 'target1_std',\n 'target1_min',\n 'target1_max',\n 'target1_prob',\n 'target2_mean',\n 'target2_median',\n 'target2_std',\n 'target2_min',\n 'target2_max',\n 'target2_prob',\n 'target3_mean',\n 'target3_median',\n 'target3_std',\n 'target3_min',\n 'target3_max',\n 'target3_prob',\n 'target4_mean',\n 'target4_median',\n 'target4_std',\n 'target4_min',\n 'target4_max',\n 'target4_prob',\n    'target1',\"divisionId\",\"teamnumberOfFollowers\",\"preseasonhits\",\"preseasonhomerun\"]\nfeature_cols3 = ['label_playerId', 'label_primaryPositionName', 'label_teamId',\n       'label_status', #'gamesPlayedBatting', 'flyOuts',\n        'homeRuns',\n      # 'strikeOuts', 'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch',\n     # 'atBats', 'caughtStealing', 'stolenBases', 'groundIntoDoublePlay',\n        'totalBases', 'rbi',\n      # 'leftOnBase', 'sacBunts', 'sacFlies', 'catchersInterference',\n        'gamesStartedPitching',\n      # 'completeGamesPitching', 'shutoutsPitching', 'winsPitching',\n       'lossesPitching', \n      # 'groundOutsPitching', 'runsPitching', 'doublesPitching',\n      # 'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching',\n       #'baseOnBallsPitching', 'intentionalWalksPitching', 'hitsPitching',\n       #'hitByPitchPitching', 'atBatsPitching', 'caughtStealingPitching',\n       'inningsPitched', \n        'battersFaced', 'pitchesThrown', \n      # 'strikes', 'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching',\n      # 'rbiPitching', 'gamesFinishedPitching', 'inheritedRunners',\n       #'inheritedRunnersScored', 'catchersInterferencePitching',\n       #'sacBuntsPitching', 'sacFliesPitching', 'saves', 'holds', 'blownSaves',\n      # 'assists', 'putOuts', 'errors', 'chances',\n      'target1_mean',\n 'target1_median',\n 'target1_std',\n 'target1_min',\n 'target1_max',\n 'target1_prob',\n 'target2_mean',\n 'target2_median',\n 'target2_std',\n 'target2_min',\n 'target2_max',\n 'target2_prob',\n 'target3_mean',\n 'target3_median',\n 'target3_std',\n 'target3_min',\n 'target3_max',\n 'target3_prob',\n 'target4_mean',\n 'target4_median',\n 'target4_std',\n 'target4_min',\n 'target4_max',\n 'target4_prob',\n    \"divisionId\",\"teamnumberOfFollowers\",\"target2\",\"preseasonhits\",\"preseasonhomerun\"]\nfeature_cols4 = ['label_playerId', 'label_primaryPositionName', 'label_teamId',\n       'label_status', 'battingOrder', 'gamesPlayedBatting', 'flyOuts',\n        'runsScored','triples','groundIntoDoublePlay',\n       'strikeOuts', 'baseOnBalls','hits', \n       'atBats', 'caughtStealing',\n       'plateAppearances', 'totalBases', 'rbi',\n       'leftOnBase', 'sacBunts', 'sacFlies',\n        'gamesPlayedPitching',\n        'winsPitching',\n       'airOutsPitching',\n       'runsPitching', 'doublesPitching',\n       'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching',\n        'intentionalWalksPitching', 'hitsPitching',\n       'hitByPitchPitching', 'atBatsPitching', \n        'inningsPitched', 'saveOpportunities',\n       'battersFaced', 'outsPitching', 'pitchesThrown', 'balls',\n       'strikes','wildPitches', \n       'rbiPitching', 'gamesFinishedPitching', 'inheritedRunners',\n       'inheritedRunnersScored', \n       'sacBuntsPitching', 'holds', 'blownSaves',\n       'assists', 'putOuts', 'errors', 'chances','target1_mean',\n 'target1_median',\n 'target1_std',\n 'target1_min',\n 'target1_max',\n 'target1_prob',\n 'target2_mean',\n 'target2_median',\n 'target2_std',\n 'target2_min',\n 'target2_max',\n 'target2_prob',\n 'target3_mean',\n 'target3_median',\n 'target3_std',\n 'target3_min',\n 'target3_max',\n 'target3_prob',\n 'target4_mean',\n 'target4_median',\n 'target4_std',\n 'target4_min',\n 'target4_max',\n 'target4_prob',\n 'target1', \"divisionId\",\"target2\",\"target3\",\"teamnumberOfFollowers\",\"preseasonhits\",\"preseasonhomerun\"]","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:20:40.635918Z","iopub.execute_input":"2021-07-26T06:20:40.636423Z","iopub.status.idle":"2021-07-26T06:20:40.659092Z","shell.execute_reply.started":"2021-07-26T06:20:40.636374Z","shell.execute_reply":"2021-07-26T06:20:40.658003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = targets[targets_cols].merge(players[players_cols], on=['playerId'], how='left')\ntrain = train.merge(rosters[rosters_cols], on=['playerId', 'date'], how='left')\ntrain = train.merge(scores[scores_cols], on=['playerId', 'date'], how='left')\ntrain = train.merge(games[games_cols], on=[\"gamePk\"], how=\"left\")\ntrain = train.merge(standings[standings_cols], on=['date',\"teamId\"], how='left')\ntrain = train.merge(teamtwi[teamtwitter_cols], on=['teamId'], how='left')\ntrain = train.merge(player_target_stats, how='inner', left_on=[\"playerId\"],right_on=[\"playerId\"])","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:20:40.660536Z","iopub.execute_input":"2021-07-26T06:20:40.660884Z","iopub.status.idle":"2021-07-26T06:20:58.018368Z","shell.execute_reply.started":"2021-07-26T06:20:40.660852Z","shell.execute_reply":"2021-07-26T06:20:58.017493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# label encoding\nplayer2num = {c: i for i, c in enumerate(train['playerId'].unique())}\nposition2num = {c: i for i, c in enumerate(train['primaryPositionName'].unique())}\nteamid2num = {c: i for i, c in enumerate(train['teamId'].unique())}\nstatus2num = {c: i for i, c in enumerate(train['status'].unique())}\ndaynight2num = {c: i for i, c in enumerate(train['dayNight'].unique())}\nseriesDescription2num = {c: i for i, c in enumerate(train['seriesDescription'].unique())}\ngameType2num = {c: i for i, c in enumerate(train['gameType'].unique())}\ntrain['label_playerId'] = train['playerId'].map(player2num)\ntrain['label_primaryPositionName'] = train['primaryPositionName'].map(position2num)\ntrain['label_teamId'] = train['teamId'].map(teamid2num)\ntrain['label_status'] = train['status'].map(status2num)\ntrain[\"label_daynight\"] = train['dayNight'].map(daynight2num)\ntrain[\"label_seriesDescription\"] = train[\"seriesDescription\"].map(seriesDescription2num)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:20:58.019658Z","iopub.execute_input":"2021-07-26T06:20:58.020188Z","iopub.status.idle":"2021-07-26T06:21:00.775261Z","shell.execute_reply.started":"2021-07-26T06:20:58.020152Z","shell.execute_reply":"2021-07-26T06:21:00.774227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"twitter = twitter.groupby(\"playerId\").mean().reset_index()\ntwitter\n","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:21:00.776452Z","iopub.execute_input":"2021-07-26T06:21:00.776815Z","iopub.status.idle":"2021-07-26T06:21:00.805154Z","shell.execute_reply.started":"2021-07-26T06:21:00.776786Z","shell.execute_reply":"2021-07-26T06:21:00.804102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_2017post = train[train[\"date\"]<=20180328]\ntrain_2018reg = train[(train[\"date\"]>=20180329)&(train[\"date\"]<=20181001)]\ntrain_2018post = train[(train[\"date\"]>=20181002)&(train[\"date\"]<=20190319)]\ntrain_2019reg = train[(train[\"date\"]>=20190320)&(train[\"date\"]<=20190929)]\ntrain_2019post = train[(train[\"date\"]>=20190930)&(train[\"date\"]<=20200722)]\ntrain_2020reg = train[(train[\"date\"]>=20200723)&(train[\"date\"]<=20200927)]\ntrain_2020post = train[(train[\"date\"]>=20200928)&(train[\"date\"]<=20210331)]\ntrain_2021reg = train[(train[\"date\"]>=20210401)&(train[\"date\"]<=20211003)]\n","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:21:00.806365Z","iopub.execute_input":"2021-07-26T06:21:00.806665Z","iopub.status.idle":"2021-07-26T06:21:05.09216Z","shell.execute_reply.started":"2021-07-26T06:21:00.806639Z","shell.execute_reply":"2021-07-26T06:21:05.091181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#make feature\nhitscol = [\"playerId\",\"preseasonhits\"]\nhits_2018 = train_2018reg.groupby(\"playerId\").sum()[\"hits\"].reset_index()\nhits_2018[\"preseasonhits\"] = hits_2018[\"hits\"]\ntrain_2018post = train_2018post.merge(hits_2018[hitscol], on=['playerId'], how='left')\ntrain_2019reg = train_2019reg.merge(hits_2018[hitscol], on=['playerId'], how='left')\n\nhits2019col = [\"playerId\",\"hits2019\"]\nhits_2019 = train_2019reg.groupby(\"playerId\").sum()[\"hits\"].reset_index()\nhits_2019[\"preseasonhits\"] = hits_2019[\"hits\"]\ntrain_2019post = train_2019post.merge(hits_2019[hitscol], on=['playerId'], how='left')\ntrain_2020reg = train_2020reg.merge(hits_2019[hitscol], on=['playerId'], how='left')\n\nhits2020col = [\"playerId\",\"hits2020\"]\nhits_2020 = train_2020reg.groupby(\"playerId\").sum()[\"hits\"].reset_index()\nhits_2020[\"preseasonhits\"] = hits_2020[\"hits\"]\ntrain_2020post = train_2020post.merge(hits_2020[hitscol], on=['playerId'], how='left')\ntrain_2021reg = train_2021reg.merge(hits_2020[hitscol], on=['playerId'], how='left')\ndel hits_2018,hits_2019\ngc.collect()\n\n\ngamescol = [\"playerId\",\"preseasonhomerun\"]\ngame2018 = train_2018reg.groupby(\"playerId\").sum()[\"homeRuns\"].reset_index()\ngame2018[\"preseasonhomerun\"] = game2018[\"homeRuns\"]\ntrain_2018post = train_2018post.merge(game2018[gamescol], on=['playerId'], how='left')\ntrain_2019reg = train_2019reg.merge(game2018[gamescol], on=['playerId'], how='left')\n\ngame2019 = train_2019reg.groupby(\"playerId\").sum()[\"homeRuns\"].reset_index()\ngame2019[\"preseasonhomerun\"] = game2019[\"homeRuns\"]\ntrain_2019post = train_2019post.merge(game2019[gamescol], on=['playerId'], how='left')\ntrain_2020reg = train_2020reg.merge(game2019[gamescol], on=['playerId'], how='left')\n\ngame2020 = train_2020reg.groupby(\"playerId\").sum()[\"homeRuns\"].reset_index()\ngame2020[\"preseasonhomerun\"] = game2020[\"homeRuns\"]\ntrain_2020post = train_2020post.merge(game2020[gamescol], on=['playerId'], how='left')\ntrain_2021reg = train_2021reg.merge(game2020[gamescol], on=['playerId'], how='left')\n\ndel game2018,game2019\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:21:05.093706Z","iopub.execute_input":"2021-07-26T06:21:05.094033Z","iopub.status.idle":"2021-07-26T06:21:19.914857Z","shell.execute_reply.started":"2021-07-26T06:21:05.094003Z","shell.execute_reply":"2021-07-26T06:21:19.913685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.concat([train_2017post,train_2018reg,train_2018post,train_2019reg,train_2019post,train_2020reg,train_2020post,train_2021reg])\ntrain[\"preseasonhits\"]\ndel train_2017post,train_2018post,train_2019post, train_2020post, train_2018reg, train_2019reg, train_2020reg\ngc.collect","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:21:19.916037Z","iopub.execute_input":"2021-07-26T06:21:19.916312Z","iopub.status.idle":"2021-07-26T06:21:23.975972Z","shell.execute_reply.started":"2021-07-26T06:21:19.916285Z","shell.execute_reply":"2021-07-26T06:21:23.974853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain_y = train[['target1', 'target2', 'target3', 'target4']]\n\n\n_index = (train['date'] < 20210701)\ngc.collect()\n\n\nx_train = train.loc[_index].reset_index(drop=True)\ny_train = train_y.loc[_index].reset_index(drop=True)\nx_valid = train.loc[~_index].reset_index(drop=True)\ny_valid = train_y.loc[~_index].reset_index(drop=True)\n\n\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:21:23.976991Z","iopub.execute_input":"2021-07-26T06:21:23.977289Z","iopub.status.idle":"2021-07-26T06:21:29.573689Z","shell.execute_reply.started":"2021-07-26T06:21:23.977263Z","shell.execute_reply":"2021-07-26T06:21:29.572655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\ndef fit_lgbm(x_train, y_train, x_valid, y_valid, params: dict=None, verbose=100):\n    oof_pred = np.zeros(len(y_valid), dtype=np.float32)\n    #model = lgbm.LGBMRegressor(**params)\n    best_params, tuning_history = {}, []\n    \n    lgb_train = lgbm.Dataset(x_train, y_train)\n    lgb_eval = lgbm.Dataset(x_valid, y_valid, reference=lgb_train)\n    model = lgbm.train(params, \n        lgb_train, valid_sets=lgb_eval,  \n        early_stopping_rounds=100,\n        num_boost_round=10000,\n        verbose_eval=100,\n        #verbose=verbose,               \n        )\n    oof_pred = model.predict(x_valid)\n    score = mean_absolute_error(oof_pred, y_valid)\n    print('mae:', score)\n    return oof_pred, model, score\n\n\n\nimport pickle\n\n\n\nparams1 = {'objective':'mae',\n           'reg_alpha': 0.14947461820098767, \n           'reg_lambda': 0.10185644384043743, \n           'n_estimators': 3633,\n           'learning_rate': 0.08046301304430488,\n           'num_leaves': 674, \n           'feature_fraction': 0.9101240539122566,\n           'bagging_fraction': 0.9884451442950513,\n           'bagging_freq': 8,\n           #'random_state': 1234,\n           'min_child_samples': 51,\n           }\n\nparams2 = {\n 'objective':'mae',\n 'reg_alpha': 0.1,\n 'reg_lambda': 0.1, \n 'n_estimators': 5000,\n 'learning_rate': 0.1,\n 'random_state': 42,\n \"num_leaves\": 22,\n \n}\n\nparams4 = {'objective':'mae',\n           'reg_alpha': 0.016468100279441976,\n           'reg_lambda': 0.09128335764019105, \n           'n_estimators': 9868, \n           'learning_rate': 0.10528150510326864,\n           'num_leaves': 157,\n           'feature_fraction': 0.5419185713426886,\n           'bagging_fraction': 0.2637405128936662, \n           'bagging_freq': 19, \n           #'random_state': 1234,\n           'min_child_samples': 71,\n           }\n\n\nparams3 = {\n 'objective':'mae',\n 'reg_alpha': 0.1,\n 'reg_lambda': 0.1, \n 'n_estimators': 10000,\n 'learning_rate': 0.1,\n 'random_state': 42,\n \"num_leaves\": 100,\n \n}\n\noof1, model1, score1 = fit_lgbm(\n    x_train[feature_cols], y_train['target1'],\n    x_valid[feature_cols], y_valid['target1'],\n    params1\n)\n\nfile = '/content/drive/MyDrive/mlb/script/output/model1.pkl'\npickle.dump(model1, open(file, 'wb'))\n\noof2, model2, score2 = fit_lgbm(\n    x_train[feature_cols2], y_train['target2'],\n    x_valid[feature_cols2], y_valid['target2'],\n    params2\n)\n\nfile = '/content/drive/MyDrive/mlb/script/output/model2.pkl'\npickle.dump(model2, open(file, 'wb'))\n\noof3, model3, score3 = fit_lgbm(\n    x_train[feature_cols3], y_train['target3'],\n    x_valid[feature_cols3], y_valid['target3'],\n    params3\n)\n\nfile = '/content/drive/MyDrive/mlb/script/output/model3.pkl'\npickle.dump(model3, open(file, 'wb'))\n\n\noof4, model4, score4 = fit_lgbm(\n    x_train[feature_cols4], y_train['target4'],\n    x_valid[feature_cols4], y_valid['target4'],\n    params4\n)\n\n\nfile = '/content/drive/MyDrive/mlb/script/output/model4.pkl'\npickle.dump(model4, open(file, 'wb'))\nscore = (score1+score2+score3+score4) / 4\nprint(f'score: {score}')\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:21:29.574828Z","iopub.execute_input":"2021-07-26T06:21:29.575114Z","iopub.status.idle":"2021-07-26T06:21:38.01992Z","shell.execute_reply.started":"2021-07-26T06:21:29.575086Z","shell.execute_reply":"2021-07-26T06:21:38.017928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model1 = pickle.load(open('../input/mymodels/model1.pkl', 'rb'))\nmodel2 = pickle.load(open('../input/mymodels/model2.pkl', 'rb'))\nmodel3 = pickle.load(open('../input/mymodels/model3.pkl', 'rb'))\nmodel4 = pickle.load(open('../input/mymodels/model4.pkl', 'rb'))","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:22:49.005442Z","iopub.execute_input":"2021-07-26T06:22:49.006094Z","iopub.status.idle":"2021-07-26T06:22:49.025407Z","shell.execute_reply.started":"2021-07-26T06:22:49.006055Z","shell.execute_reply":"2021-07-26T06:22:49.024281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"players_cols = ['playerId', 'primaryPositionName']\nrosters_cols = ['playerId', 'teamId', 'status']\nscores_cols = ['playerId', 'battingOrder', 'gamesPlayedBatting', 'flyOuts',\n       'groundOuts', 'runsScored', 'doubles', 'triples', 'homeRuns',\n       'strikeOuts', 'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch',\n       'atBats', 'caughtStealing', 'stolenBases', 'groundIntoDoublePlay',\n       'groundIntoTriplePlay', 'plateAppearances', 'totalBases', 'rbi',\n       'leftOnBase', 'sacBunts', 'sacFlies', 'catchersInterference',\n       'pickoffs', 'gamesPlayedPitching', 'gamesStartedPitching',\n       'completeGamesPitching', 'shutoutsPitching', 'winsPitching',\n       'lossesPitching', 'flyOutsPitching', 'airOutsPitching',\n       'groundOutsPitching', 'runsPitching', 'doublesPitching',\n       'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching',\n       'baseOnBallsPitching', 'intentionalWalksPitching', 'hitsPitching',\n       'hitByPitchPitching', 'atBatsPitching', 'caughtStealingPitching',\n       'stolenBasesPitching', 'inningsPitched', 'saveOpportunities',\n       'earnedRuns', 'battersFaced', 'outsPitching', 'pitchesThrown', 'balls',\n       'strikes', 'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching',\n       'rbiPitching', 'gamesFinishedPitching', 'inheritedRunners',\n       'inheritedRunnersScored', 'catchersInterferencePitching',\n       'sacBuntsPitching', 'sacFliesPitching', 'saves', 'holds', 'blownSaves',\n       'assists', 'putOuts', 'errors', 'chances']\nstandings_cols = [\"teamId\",\"divisionRank\",\"divisionLeader\",\"wildCardLeader\",\"leagueRank\",\"divisionId\",\"gameDate\"]\nyesterday_targets_cols = [\"playerId\",\"yest_target1\",\"yest_target2\",\"yest_target3\",\"yest_target4\"]\nnull = np.nan\ntrue = True\nfalse = False\nhitscol = [\"playerId\",\"preseasonhits\"]\n","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:21:38.020981Z","iopub.status.idle":"2021-07-26T06:21:38.021545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom datetime import timedelta\nfrom tqdm import tqdm\nimport gc\nfrom functools import reduce\nfrom sklearn.model_selection import StratifiedKFold\ndef make_df(df, col, bool_in=False):\n    tp = df.loc[ ~df[col].isnull() ,[col]].copy()\n    df.drop(col, axis=1, inplace=True)\n    \n    tp[col] = tp[col].str.replace(\"null\",'\"\"')\n    if bool_in:\n        tp[col] = tp[col].str.replace(\"false\",'\"False\"')\n        tp[col] = tp[col].str.replace(\"true\",'\"True\"')\n    tp[col] = tp[col].apply(lambda x: eval(x) )\n    a = tp[col].sum()\n    gc.collect()\n    return pd.DataFrame(a)\n#===============\nROOT_DIR = \"../input/c/mlb-player-digital-engagement-forecasting\"\n#UTILITY FUNCTIONS\n#=======================#\ndef flatten(df, col):\n    du = (df.pivot(index=\"playerId\", columns=\"EvalDate\", \n               values=col).add_prefix(f\"{col}_\").\n      rename_axis(None, axis=1).reset_index())\n    return du\n#============================#\ndef reducer(left, right):\n    return left.merge(right, on=\"playerId\")\n#========================\nTGTCOLS = [\"target1\",\"target2\",\"target3\",\"target4\"]\ndef train_lag(df, lag=1):\n    dp = df[[\"playerId\",\"EvalDate\"]+TGTCOLS].copy()\n    dp[\"EvalDate\"]  =dp[\"EvalDate\"] + timedelta(days=lag) \n    df = df.merge(dp, on=[\"playerId\", \"EvalDate\"], suffixes=[\"\",f\"_{lag}\"], how=\"left\")\n    return df\n#=================================\ndef test_lag(sub):\n    sub[\"playerId\"] = sub[\"date_playerId\"].apply(lambda s: int(  s.split(\"_\")[1]  ) )\n    assert sub.date.nunique() == 1\n    dte = sub[\"date\"].unique()[0]\n    \n    eval_dt = pd.to_datetime(dte, format=\"%Y%m%d\")\n    dtes = [eval_dt + timedelta(days=-k) for k in LAGS]\n    mp_dtes = {eval_dt + timedelta(days=-k):k for k in LAGS}\n    \n    sl = LAST.loc[LAST.EvalDate.between(dtes[-1], dtes[0]), [\"EvalDate\",\"playerId\"]+TGTCOLS].copy()\n    sl[\"EvalDate\"] = sl[\"EvalDate\"].map(mp_dtes)\n    du = [flatten(sl, col) for col in TGTCOLS]\n    du = reduce(reducer, du)\n    return du, eval_dt\n    \n\n \n \n\ntr = pd.read_csv(f\"{ROOT_DIR}/train.csv\")\ntr = pd.read_csv(\"../input/mlb-data/target.csv\")\nprint(tr.shape)\ngc.collect()\n\ntr[\"EvalDate\"] = pd.to_datetime(tr[\"EvalDate\"])\ntr[\"EvalDate\"] = tr[\"EvalDate\"] + timedelta(days=-1)\ntr[\"EvalYear\"] = tr[\"EvalDate\"].dt.year\nMED_DF = tr.groupby([\"playerId\",\"EvalYear\"])[TGTCOLS].median().reset_index()\nMEDCOLS = [\"tgt1_med\",\"tgt2_med\", \"tgt3_med\", \"tgt4_med\"]\nMED_DF.columns = [\"playerId\",\"EvalYear\"] + MEDCOLS\nMED_DF.head()\nMAX_LAG = 20\nLAGS = list(range(1, MAX_LAG + 1))\nFECOLS = [f\"{col}_{lag}\" for lag in reversed(LAGS) for col in TGTCOLS]\n\n\nfor lag in tqdm(LAGS):\n    tr = train_lag(tr, lag=lag)\n    gc.collect()\n#===========\ntr = tr.sort_values(by=[\"playerId\", \"EvalDate\"])\nprint(tr.shape)\ntr = tr.dropna()\nprint(tr.shape)\ntr = tr.merge(MED_DF, on=[\"playerId\",\"EvalYear\"])\ngc.collect()\n\nX = tr[FECOLS+MEDCOLS].values\ny = tr[TGTCOLS].values\ncl = tr[\"playerId\"].values\n\nNFOLDS = 10\nskf = StratifiedKFold(n_splits=NFOLDS)\nfolds = skf.split(X, cl)\nfolds = list(folds)\nX.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:21:38.022522Z","iopub.status.idle":"2021-07-26T06:21:38.023137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.models as M\nfrom sklearn.metrics import mean_absolute_error, mean_squared_error\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping\ndef make_model(n_in):\n    inp = L.Input(name=\"inputs\", shape=(n_in,))\n    nh = 50\n    x = L.Dense(nh, activation=\"relu\", name=\"d1\")(inp)\n    x = L.Dense(nh, activation=\"relu\", name=\"d2\")(x)\n    #x = L.Dense(nh, activation=\"relu\", name=\"d3\")(x)\n    preds = L.Dense(4, activation=\"linear\", name=\"preds\")(x)\n    \n    model = M.Model(inp, preds, name=\"ANN\")\n    model.compile(loss=\"mean_absolute_error\", optimizer=\"adam\")\n    return model\nnet = make_model(X.shape[1])\nprint(net.summary())","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:21:38.024064Z","iopub.status.idle":"2021-07-26T06:21:38.024654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof = np.zeros(y.shape)\nnets = []\nEPOCHS  = 10\nfor idx in range(NFOLDS):\n    print(\"FOLD:\", idx)\n    tr_idx, val_idx = folds[idx]\n    ckpt = ModelCheckpoint(f\"w{idx}.h5\", monitor='val_loss', verbose=1, save_best_only=True,mode='min')\n    reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,patience=3, min_lr=0.0005)\n    es = EarlyStopping(monitor='val_loss', patience=6)\n    reg = make_model(X.shape[1])\n    reg.fit(X[tr_idx], y[tr_idx], epochs=EPOCHS, batch_size=30_000, \n            validation_data=(X[val_idx], y[val_idx]),\n            verbose=1, callbacks=[ckpt, reduce_lr, es])\n    reg.load_weights(f\"w{idx}.h5\")\n    oof[val_idx] = reg.predict(X[val_idx], batch_size=50_000, verbose=1)\n    nets.append(reg)\n    gc.collect()\n    #\n#","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:21:38.025572Z","iopub.status.idle":"2021-07-26T06:21:38.026008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mae = mean_absolute_error(y, oof)\nmse = mean_squared_error(y, oof, squared=False)\nprint(\"mae:\", mae)\nprint(\"mse:\", mse)\n\n# Historical information to use in prediction time\nbound_dt = pd.to_datetime(\"2021-01-01\")\nLAST = tr.loc[tr.EvalDate>bound_dt].copy()\nLAST_MED_DF = MED_DF.loc[MED_DF.EvalYear==2021].copy()\nLAST_MED_DF.drop(\"EvalYear\", axis=1, inplace=True)\ndel tr\nLAST.shape, LAST_MED_DF.shape, MED_DF.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:21:38.026859Z","iopub.status.idle":"2021-07-26T06:21:38.027301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def unpack_json(json_str):\n    return np.nan if pd.isna(json_str) else pd.read_json(json_str)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:21:38.028173Z","iopub.status.idle":"2021-07-26T06:21:38.028603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"players_cols = ['playerId', 'primaryPositionName']\nrosters_cols = ['playerId', 'teamId', 'status']\nscores_cols = ['playerId', 'battingOrder', 'gamesPlayedBatting', 'flyOuts',\n       'groundOuts', 'runsScored', 'doubles', 'triples', 'homeRuns',\n       'strikeOuts', 'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch',\n       'atBats', 'caughtStealing', 'stolenBases', 'groundIntoDoublePlay',\n       'groundIntoTriplePlay', 'plateAppearances', 'totalBases', 'rbi',\n       'leftOnBase', 'sacBunts', 'sacFlies', 'catchersInterference',\n       'pickoffs', 'gamesPlayedPitching', 'gamesStartedPitching',\n       'completeGamesPitching', 'shutoutsPitching', 'winsPitching',\n       'lossesPitching', 'flyOutsPitching', 'airOutsPitching',\n       'groundOutsPitching', 'runsPitching', 'doublesPitching',\n       'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching',\n       'baseOnBallsPitching', 'intentionalWalksPitching', 'hitsPitching',\n       'hitByPitchPitching', 'atBatsPitching', 'caughtStealingPitching',\n       'stolenBasesPitching', 'inningsPitched', 'saveOpportunities',\n       'earnedRuns', 'battersFaced', 'outsPitching', 'pitchesThrown', 'balls',\n       'strikes', 'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching',\n       'rbiPitching', 'gamesFinishedPitching', 'inheritedRunners',\n       'inheritedRunnersScored', 'catchersInterferencePitching',\n       'sacBuntsPitching', 'sacFliesPitching', 'saves', 'holds', 'blownSaves',\n       'assists', 'putOuts', 'errors', 'chances']\n\nnull = np.nan\ntrue = True\nfalse = False\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:21:38.029652Z","iopub.status.idle":"2021-07-26T06:21:38.030064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"players_cols = ['playerId', 'primaryPositionName']\nrosters_cols = ['playerId', 'teamId', 'status']\nscores_cols = ['playerId', 'battingOrder', 'gamesPlayedBatting', 'flyOuts',\n       'groundOuts', 'runsScored', 'doubles', 'triples', 'homeRuns',\n       'strikeOuts', 'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch',\n       'atBats', 'caughtStealing', 'stolenBases', 'groundIntoDoublePlay',\n       'groundIntoTriplePlay', 'plateAppearances', 'totalBases', 'rbi',\n       'leftOnBase', 'sacBunts', 'sacFlies', 'catchersInterference',\n       'pickoffs', 'gamesPlayedPitching', 'gamesStartedPitching',\n       'completeGamesPitching', 'shutoutsPitching', 'winsPitching',\n       'lossesPitching', 'flyOutsPitching', 'airOutsPitching',\n       'groundOutsPitching', 'runsPitching', 'doublesPitching',\n       'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching',\n       'baseOnBallsPitching', 'intentionalWalksPitching', 'hitsPitching',\n       'hitByPitchPitching', 'atBatsPitching', 'caughtStealingPitching',\n       'stolenBasesPitching', 'inningsPitched', 'saveOpportunities',\n       'earnedRuns', 'battersFaced', 'outsPitching', 'pitchesThrown', 'balls',\n       'strikes', 'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching',\n       'rbiPitching', 'gamesFinishedPitching', 'inheritedRunners',\n       'inheritedRunnersScored', 'catchersInterferencePitching',\n       'sacBuntsPitching', 'sacFliesPitching', 'saves', 'holds', 'blownSaves',\n       'assists', 'putOuts', 'errors', 'chances',\"gamePk\"]\nplayertwitter_cols = [\"playerId\",\"numberOfFollowers\",\"year_months\"]\nnull = np.nan\ntrue = True\nfalse = False\n\n\n\nenv = mlb.make_env() # initialize the environment\niter_test = env.iter_test() # iterator which loops over each date in test set\n\nfor (test_df, sample_prediction_df) in iter_test: # make predictions here\n    \n    sub = copy.deepcopy(sample_prediction_df.reset_index())\n    sample_prediction_df = copy.deepcopy(sample_prediction_df.reset_index(drop=True))\n    \n    \n    # creat dataset\n    sample_prediction_df['playerId'] = sample_prediction_df['date_playerId']\\\n                                        .map(lambda x: int(x.split('_')[1]))\n    # Dealing with missing values\n    if test_df['rosters'].iloc[0] == test_df['rosters'].iloc[0]:\n        test_rosters = pd.DataFrame(eval(test_df['rosters'].iloc[0]))\n    else:\n        test_rosters = pd.DataFrame({'playerId': sample_prediction_df['playerId']})\n        for col in rosters.columns:\n            if col == 'playerId': continue\n            test_rosters[col] = np.nan\n            \n    if test_df['playerBoxScores'].iloc[0] == test_df['playerBoxScores'].iloc[0]:\n        test_scores = pd.DataFrame(eval(test_df['playerBoxScores'].iloc[0]))\n    else:\n        test_scores = pd.DataFrame({'playerId': sample_prediction_df['playerId']})\n        for col in scores.columns:\n            if col == 'playerId': continue\n            test_scores[col] = np.nan\n    \n    test_games = unpack_json(test_df[\"games\"].iloc[0])\n    test_standings = unpack_json(test_df[\"standings\"].iloc[0])\n    \n            \n    test_scores = test_scores.groupby('playerId').sum().reset_index()\n    test = sample_prediction_df[['playerId']].copy()\n    test = test.merge(players[players_cols], on='playerId', how='left')\n    test = test.merge(test_rosters[rosters_cols], on='playerId', how='left')\n    test = test.merge(test_scores[scores_cols], on='playerId', how='left')\n    test = test.merge(test_games[games_cols], on=\"gamePk\", how=\"left\")\n    test = test.merge(awards2[awards_cols], on=['playerId'], how='left')\n    test = test.merge(player_target_stats, how='inner', left_on=[\"playerId\"],right_on=[\"playerId\"])\n    test = test.merge(test_standings[standings_cols], on=[\"teamId\"], how='left')\n    test = test.merge(teamtwi[teamtwitter_cols], on=['teamId'], how='left')\n    test = test.merge(hits_2020[hitscol], on=['playerId'], how='left')\n    test = test.merge(game2020[gamescol], on=['playerId'], how='left')\n    test[\"strdate\"] = sample_prediction_df[\"date_playerId\"].astype(str)\n    test[\"date1\"] = test[\"strdate\"].str[0:8]\n    twitter[\"strdate\"] = twitter[\"date\"].astype(str)\n    twitter[\"year_months\"] = twitter[\"strdate\"].str[0:6].astype(int)\n    test[\"year_months\"] = test[\"strdate\"].str[0:6].astype(int)\n    test = test.merge(twitter[playertwitter_cols], on=['playerId'], how='left')\n    #test[\"hitpergame\"] = test[\"preseasonhits\"] / test[\"preseasonpt\"]\n\n    \n    #add feature\n    test[\"strmonth\"] = sample_prediction_df[\"date_playerId\"].astype(str)\n    test[\"hasTwitterAccount\"] = test.playerId.isin(twitter.playerId)\n    \n    test[\"date2\"] = pd.to_datetime(test[\"date1\"].astype(str))\n    test[\"month\"] = test[\"strmonth\"].str[5].median()\n    test[\"youbi\"] = test[\"date2\"].dt.dayofweek\n    test[\"allstar\"] = 0\n    test[\"regularseason\"] = 1\n\n\n    \n    #label encoding\n    test['label_playerId'] = test['playerId'].map(player2num)\n    test['label_primaryPositionName'] = test['primaryPositionName'].map(position2num)\n    test['label_teamId'] = test['teamId'].map(teamid2num)\n    test['label_status'] = test['status'].map(status2num)\n    test[\"label_daynight\"] = test['dayNight'].map(daynight2num)\n    test[\"label_seriesDescription\"] = test[\"seriesDescription\"].map(seriesDescription2num)\n    test[\"gameType\"] = test[\"gameType\"].map(gameType2num)\n    \n    display(test)\n    \n    \n    \n    #test_X2 = test[feature_cols]\n    #test_X3 = test[feature_cols]\n    #test_X4 = test[feature_cols]\n    \n    # predict\n    test_X = test[feature_cols]\n    pred1 = model1.predict(test_X)\n    #pred5 = model5.predict(test_X)\n    #pred9 = model9.predict(test_X)\n    test['target1'] = np.clip(pred1,0,100)\n    test_X = test[feature_cols2]\n    pred2 = model2.predict(test_X)\n    #pred6 = model6.predict(test_X)\n    #pred10 = model10.predict(test_X)\n    test['target2'] = np.clip(pred2,0,100)\n    test_X = test[feature_cols3]\n    pred3 = model3.predict(test_X)\n    #pred7 = model7.predict(test_X)\n    #pred11 = model11.predict(test_X)\n    test['target3'] = np.clip(pred3,0,100)\n    \n    test_X = test[feature_cols4]\n    pred4 = model4.predict(test_X)\n    #pred8 = model8.predict(test_X)\n    #pred12 = model12.predict(test_X)\n \n    \n    # merge submission\n    sample_prediction_df['target1'] = np.clip(pred1,0,100) #+ 0.2 * np.clip(pred5,0,100) + 0.2 * np.clip(pred9,0,100) \n    sample_prediction_df['target2'] = np.clip(pred2,0,100) #+ 0.2 * np.clip(pred6,0,100) + 0.2 * np.clip(pred10,0,100) \n    sample_prediction_df['target3'] = np.clip(pred3,0,100) #+ 0.2 * np.clip(pred7,0,100) + 0.2 * np.clip(pred11,0,100) \n    sample_prediction_df['target4'] = np.clip(pred4,0,100) #+ 0.2 * np.clip(pred8,0,100) + 0.2 * np.clip(pred12,0,100) \n    sample_prediction_df = sample_prediction_df.fillna(0.)\n    yesterday_targets_test = sample_prediction_df\n    yesterday_targets_test = yesterday_targets_test.rename(columns={'target1': 'yest_target1','target2': 'yest_target2','target3': 'yest_target3','target4': 'yest_target4'})\n    del sample_prediction_df['playerId']\n    #env.predict(sample_prediction_df)\n    \n    # TF summit\n    # Features computation at Evaluation Date\n    sub_fe, eval_dt = test_lag(sub)\n    sub_fe = sub_fe.merge(LAST_MED_DF, on=\"playerId\", how=\"left\")\n    sub_fe = sub_fe.fillna(0.)\n\n    _preds = 0.\n    for reg in nets:\n        _preds += reg.predict(sub_fe[FECOLS + MEDCOLS]) / NFOLDS\n    sub_fe[TGTCOLS] = np.clip(_preds, 0, 100)\n    sub.drop([\"date\"]+TGTCOLS, axis=1, inplace=True)\n    sub = sub.merge(sub_fe[[\"playerId\"]+TGTCOLS], on=\"playerId\", how=\"left\")\n    sub.drop(\"playerId\", axis=1, inplace=True)\n    sub = sub.fillna(0.)\n    # Blending\n    blend = pd.concat(\n        [sub[['date_playerId']],\n        (0.35*sub.drop('date_playerId', axis=1) + 0.65*sample_prediction_df.drop('date_playerId', axis=1))],\n        axis=1\n    )\n    env.predict(blend)\n    # Update Available information\n    sub_fe[\"EvalDate\"] = eval_dt\n    #sub_fe.drop(MEDCOLS, axis=1, inplace=True)\n    LAST = LAST.append(sub_fe)\n    LAST = LAST.drop_duplicates(subset=[\"EvalDate\",\"playerId\"], keep=\"last\")","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:21:38.031204Z","iopub.status.idle":"2021-07-26T06:21:38.03161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[\"strmonth\"].astype(str)\ntest[\"month\"] = test[\"strmonth\"].str[5].median()\ntest[\"month\"]\n","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:21:38.032599Z","iopub.status.idle":"2021-07-26T06:21:38.033156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def unpack_json(json_str):\n    return np.nan if pd.isna(json_str) else pd.read_json(json_str)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:21:38.033971Z","iopub.status.idle":"2021-07-26T06:21:38.034401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unpack_json(test_df[\"games\"].iloc[0])","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:21:38.035499Z","iopub.status.idle":"2021-07-26T06:21:38.035983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.concat(\n    [sub[['date_playerId']],\n    (sub.drop('date_playerId', axis=1) + sample_prediction_df.drop('date_playerId', axis=1)) / 2],\n    axis=1\n)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:21:38.036871Z","iopub.status.idle":"2021-07-26T06:21:38.03727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.dtypes","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:21:38.038085Z","iopub.status.idle":"2021-07-26T06:21:38.038485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_prediction_df.dtypes","metadata":{"execution":{"iopub.status.busy":"2021-07-26T06:21:38.039372Z","iopub.status.idle":"2021-07-26T06:21:38.039826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}