{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Description of the steps\n\n### 1- Rosters (gameDate, playerId, teamId)\n* statusCode: 11 values\n* status : 16 values\n\n### 2- Games (gamePk, gameDate)\nsince some games are postponed or arrested, we have to choose the last line of the game\n* gameType : 8\n* codedGameState : 6\n* detailedGameState : 7\n* isTie : 2\n* seriesDescription : 8\n\n### 3-playerBoxScores (playerId, gameDate,gamePk)\n* jerseyNum: 100\n* positionCode: 12\n* positionName: 8\n* positionType: 6\n\n\n### 4-teamBoxScores (teamId, index,gamePk)\n* No categorical variable\n\n### 5- events\n\n### 6- transactions\n\n### 7- awards\n\n### 8- standings\n\n### 9- playerTwitterFollowers\n\n### 10- teamTwiterFollowers\nOk","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-21T07:05:26.917263Z","iopub.execute_input":"2021-06-21T07:05:26.917649Z","iopub.status.idle":"2021-06-21T07:05:26.92379Z","shell.execute_reply.started":"2021-06-21T07:05:26.917554Z","shell.execute_reply":"2021-06-21T07:05:26.922427Z"}}},{"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df\n#","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:16:14.130491Z","iopub.execute_input":"2021-06-30T06:16:14.130995Z","iopub.status.idle":"2021-06-30T06:16:14.152859Z","shell.execute_reply.started":"2021-06-30T06:16:14.130875Z","shell.execute_reply":"2021-06-30T06:16:14.151562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#==============ROSTERS========================================\nstatusCode = {'A':1, 'D60':2, 'RM':3, 'D10':4, 'D7':5, 'PL':6, 'SU':7, \n              'FME':8, 'BRV':9, 'RES':10,'DEC':11}\nstatus = {'Active':1, 'Injured 60-Day':2, 'Reassigned to Minors':3,\n       'Injured 10-Day':4, 'Injured 7-Day':5, 'Paternity List':6,\n       'Suspended # days':7, 'Family Medical Emergency':8, 'Bereavement List':9,\n       'Reserve List (Minors)':10, 'Deceased':11, '60-day IL':12, '10-day IL':13,\n       'Reassigned':14, 'Paternity':15, 'Suspended':16}\n#============== GAME =================================#\ngameType = {'E':1, 'S':2, 'R':3, 'A':4, 'F':5, 'D':6, 'L':7, 'W':8}\ncodedGameState = {'F':1, 'C':2, 'D':3, 'S':4, 'U':5, 'O':6}\ndetailedGameState = {'Final':1, 'Cancelled':2, 'Completed Early':3, 'Postponed':4, \n                     'Scheduled':5,'Suspended':6, 'Game Over':7}\nisTie = {False:1, True:2}\nseriesDescription = {'Exhibition':1, 'Spring Training':2, 'Regular Season':3, 'All-Star Game':4,\n                     'Wild Card Game':5, 'Division Series':6, 'League Championship Series':7,\n                     'World Series':8}\n#===============================PLAYER SCORES============================\npositionName = {'Catcher':1,'Designated Hitter':2,'First Base':3,'Outfielder':4,\n                'Pinch Hitter':5,'Pinch Runner':6,'Pitcher':7, 'Second Base':8,\n                'Shortstop':9,'Third Base':10}\npositionType = {'Catcher':1, 'Hitter':2, 'Infielder':3, 'Outfielder':4, 'Pitcher':5, 'Runner':6}\n#=================================GAMES========================\ndoubleHeader = {\"N\":1, \"Y\":2, \"S\":3}\ndayNight = {\"day\":1, \"night\":2}\n#=================================================","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:16:14.154785Z","iopub.execute_input":"2021-06-30T06:16:14.155533Z","iopub.status.idle":"2021-06-30T06:16:14.168501Z","shell.execute_reply.started":"2021-06-30T06:16:14.155494Z","shell.execute_reply":"2021-06-30T06:16:14.167461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SCORES = ['home', 'flyOuts', 'groundOuts', 'runsScored', 'doubles', 'triples', 'homeRuns', 'strikeOuts', \n          'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch', 'atBats', 'caughtStealing', 'stolenBases', \n          'groundIntoDoublePlay', 'groundIntoTriplePlay', 'plateAppearances', 'totalBases', 'rbi', 'leftOnBase', \n          'sacBunts', 'sacFlies', 'catchersInterference', 'pickoffs', 'airOutsPitching', 'groundOutsPitching', \n          'runsPitching', 'doublesPitching', 'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching', \n          'baseOnBallsPitching', 'intentionalWalksPitching', 'hitsPitching', 'hitByPitchPitching', 'atBatsPitching', \n          'caughtStealingPitching', 'stolenBasesPitching', 'inningsPitched', 'earnedRuns', 'battersFaced', 'outsPitching', \n          'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching', 'rbiPitching', 'inheritedRunners', 'inheritedRunnersScored', \n          'catchersInterferencePitching', 'sacBuntsPitching', 'sacFliesPitching']\nren_player = {col:col+\"_p\" for col in SCORES}\nren_team = {col:col+\"_t\" for col in SCORES}","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:16:14.170080Z","iopub.execute_input":"2021-06-30T06:16:14.170526Z","iopub.status.idle":"2021-06-30T06:16:14.187444Z","shell.execute_reply.started":"2021-06-30T06:16:14.170494Z","shell.execute_reply":"2021-06-30T06:16:14.186247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_index = \"index\"\n#============================\nrosters_id = [time_index, 'playerId']\nrosters_cat = [\"statusCode\",\"status\"]\n#=========================\npscores_id = [\"playerId\",\"index\", 'teamId'] #, \"gamePk\"\npscores_cat = [\"jerseyNum\",\"positionCode\",\"positionName\",\"positionType\"]\npscores_num = ['home', \n               'gamesPlayedBatting', 'flyOuts', 'groundOuts', 'runsScored', 'doubles', 'triples', \n               'homeRuns', 'strikeOuts', 'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch', \n               'atBats', 'caughtStealing', 'stolenBases', 'groundIntoDoublePlay', 'groundIntoTriplePlay', \n               'plateAppearances', 'totalBases', 'rbi', 'leftOnBase', 'sacBunts', 'sacFlies', \n               'catchersInterference', 'pickoffs', 'gamesPlayedPitching', 'gamesStartedPitching', \n               'completeGamesPitching', 'shutoutsPitching', 'winsPitching', 'lossesPitching', 'flyOutsPitching', \n               'airOutsPitching', 'groundOutsPitching', \n               'runsPitching', 'doublesPitching', 'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching', \n               'baseOnBallsPitching', 'intentionalWalksPitching', 'hitsPitching', 'hitByPitchPitching', \n               'atBatsPitching', 'caughtStealingPitching', 'stolenBasesPitching', 'inningsPitched', \n               'saveOpportunities', 'earnedRuns', 'battersFaced', 'outsPitching', 'pitchesThrown', \n               'balls', 'strikes', 'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching', 'rbiPitching', \n               'gamesFinishedPitching', 'inheritedRunners', 'inheritedRunnersScored', 'catchersInterferencePitching', \n               'sacBuntsPitching', 'sacFliesPitching', 'saves', 'holds', 'blownSaves', 'assists', \n               'putOuts', 'errors', 'chances']\npscores_num = [col+'_p' if (col in SCORES) else col for col in pscores_num]\n#==============\ntscores_id = [\"teamId\", \"index\",\"gamePk\"]\ntscores_num = ['home', 'flyOuts',\n       'groundOuts', 'runsScored', 'doubles', 'triples', 'homeRuns',\n       'strikeOuts', 'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch',\n       'atBats', 'caughtStealing', 'stolenBases', 'groundIntoDoublePlay',\n       'groundIntoTriplePlay', 'plateAppearances', 'totalBases', 'rbi',\n       'leftOnBase', 'sacBunts', 'sacFlies', 'catchersInterference',\n       'pickoffs', 'airOutsPitching', 'groundOutsPitching', 'runsPitching',\n       'doublesPitching', 'triplesPitching', 'homeRunsPitching',\n       'strikeOutsPitching', 'baseOnBallsPitching', 'intentionalWalksPitching',\n       'hitsPitching', 'hitByPitchPitching', 'atBatsPitching',\n       'caughtStealingPitching', 'stolenBasesPitching', 'inningsPitched',\n       'earnedRuns', 'battersFaced', 'outsPitching', 'hitBatsmen', 'balks',\n       'wildPitches', 'pickoffsPitching', 'rbiPitching', 'inheritedRunners',\n       'inheritedRunnersScored', 'catchersInterferencePitching',\n       'sacBuntsPitching', 'sacFliesPitching']\ntscores_num = [col+'_t' if (col in SCORES) else col for col in tscores_num]\n#===========================\ngames_cat = [\"gameType\",\"codedGameState\",\"detailedGameState\",\"isTie\",\"seriesDescription\",\n             \"doubleHeader\", \"dayNight\"]\ngames_id = ['gamePk','homeId', 'awayId']\ngames_num = ['gameNumber',  'scheduledInnings', 'gamesInSeries', 'homeWins', 'homeLosses', \n             'homeWinPct', 'homeWinner', 'homeScore', 'awayWins', 'awayLosses', 'awayWinPct', \n             'awayWinner', 'awayScore']\n# season (cat) / 'resumeDate', 'resumedFrom' (continuity)\n#===========================","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:16:14.189532Z","iopub.execute_input":"2021-06-30T06:16:14.190053Z","iopub.status.idle":"2021-06-30T06:16:14.207214Z","shell.execute_reply.started":"2021-06-30T06:16:14.189996Z","shell.execute_reply":"2021-06-30T06:16:14.205710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#============================\ndef process_rosters(df):\n    df[\"statusCode\"] = df[\"statusCode\"].map(statusCode)\n    df[\"status\"] = df[\"status\"].map(status)\n    return df\n#=================\ndef process_pscores(df):\n    # deal with battingOrder\n    df[\"positionName\"] = df[\"positionName\"].map(positionName)\n    df[\"positionType\"] = df[\"positionType\"].map(positionType)\n    df[\"jerseyNum\"] = df[\"jerseyNum\"].fillna(100)\n    df[\"positionCode\"] = df[\"positionCode\"].fillna(0)\n    return df\n#======================================\ndef process_games(df):\n    #add season\n    df[\"gameType\"] = df[\"gameType\"].map(gameType)\n    df[\"codedGameState\"] = df[\"codedGameState\"].map(codedGameState)\n    df[\"detailedGameState\"] = df[\"detailedGameState\"].map(detailedGameState)\n    df[\"isTie\"] = df[\"isTie\"].map(isTie)\n    df[\"seriesDescription\"] = df[\"seriesDescription\"].map(seriesDescription)\n    df[\"doubleHeader\"] = df[\"doubleHeader\"].map(doubleHeader)\n    df[\"dayNight\"] = df[\"dayNight\"].map(dayNight)\n    return df\n#====================================\n\n#========================","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:16:14.250644Z","iopub.execute_input":"2021-06-30T06:16:14.251001Z","iopub.status.idle":"2021-06-30T06:16:14.260406Z","shell.execute_reply.started":"2021-06-30T06:16:14.250969Z","shell.execute_reply":"2021-06-30T06:16:14.259125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nimport os\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:16:14.262129Z","iopub.execute_input":"2021-06-30T06:16:14.262462Z","iopub.status.idle":"2021-06-30T06:16:14.278089Z","shell.execute_reply.started":"2021-06-30T06:16:14.262433Z","shell.execute_reply":"2021-06-30T06:16:14.277310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_ROOT = \"../input/mlb-pdef-train-dataset\"","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:16:14.279650Z","iopub.execute_input":"2021-06-30T06:16:14.280102Z","iopub.status.idle":"2021-06-30T06:16:14.291123Z","shell.execute_reply.started":"2021-06-30T06:16:14.280068Z","shell.execute_reply":"2021-06-30T06:16:14.289966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#os.listdir(DATA_ROOT)","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:16:14.292891Z","iopub.execute_input":"2021-06-30T06:16:14.293387Z","iopub.status.idle":"2021-06-30T06:16:14.302718Z","shell.execute_reply.started":"2021-06-30T06:16:14.293352Z","shell.execute_reply":"2021-06-30T06:16:14.301956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = True","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:16:14.303944Z","iopub.execute_input":"2021-06-30T06:16:14.304438Z","iopub.status.idle":"2021-06-30T06:16:14.315083Z","shell.execute_reply.started":"2021-06-30T06:16:14.304404Z","shell.execute_reply":"2021-06-30T06:16:14.313910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntgt = pd.read_csv(f\"{DATA_ROOT}/nextDayPlayerEngagement_train.csv\")\ntgt.drop(\"engagementMetricsDate\", axis=1, inplace=True)\ntgt[\"date\"] = pd.to_datetime(tgt[\"date\"], format='%Y%m%d')\ntgt = tgt.rename(columns={\"date\":\"EvalDate\"})\n\nif DEBUG:\n    print(tgt.shape)\n    tgt = tgt.loc[tgt[time_index]>=1000].reset_index(drop=True)\n    print(tgt.shape)\n#\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:16:14.316624Z","iopub.execute_input":"2021-06-30T06:16:14.316952Z","iopub.status.idle":"2021-06-30T06:16:20.146816Z","shell.execute_reply.started":"2021-06-30T06:16:14.316906Z","shell.execute_reply":"2021-06-30T06:16:20.145987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tgt.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:16:20.147884Z","iopub.execute_input":"2021-06-30T06:16:20.148333Z","iopub.status.idle":"2021-06-30T06:16:20.169307Z","shell.execute_reply.started":"2021-06-30T06:16:20.148303Z","shell.execute_reply":"2021-06-30T06:16:20.168110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tgt[\"index\"].max()","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:16:20.170729Z","iopub.execute_input":"2021-06-30T06:16:20.171029Z","iopub.status.idle":"2021-06-30T06:16:20.177787Z","shell.execute_reply.started":"2021-06-30T06:16:20.170999Z","shell.execute_reply":"2021-06-30T06:16:20.176696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nrosters = pd.read_csv(f\"{DATA_ROOT}/rosters_train.csv\", usecols=rosters_cat + rosters_id)\nrosters = process_rosters(rosters)","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:16:20.180832Z","iopub.execute_input":"2021-06-30T06:16:20.181161Z","iopub.status.idle":"2021-06-30T06:16:22.037430Z","shell.execute_reply.started":"2021-06-30T06:16:20.181131Z","shell.execute_reply":"2021-06-30T06:16:22.036230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\npscores = pd.read_csv(f\"{DATA_ROOT}/playerBoxScores_train.csv\")\npscores = pscores.rename(columns=ren_player)\npscores = pscores[pscores_id+pscores_cat+pscores_num]\npscores = process_pscores(pscores)\n#================\nNO_DOUBLON = True\nif NO_DOUBLON:\n    print(pscores.shape)\n    pscores = pscores.drop_duplicates(subset=rosters_id, keep=\"last\")\n    print(pscores.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:16:22.039278Z","iopub.execute_input":"2021-06-30T06:16:22.039567Z","iopub.status.idle":"2021-06-30T06:16:24.375823Z","shell.execute_reply.started":"2021-06-30T06:16:22.039539Z","shell.execute_reply":"2021-06-30T06:16:24.374850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tscores = pd.read_csv(f\"{DATA_ROOT}/teamBoxScores_train.csv\")\ntscores = tscores.rename(columns=ren_team)\ntscores = tscores[tscores_id + tscores_num]","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:16:24.377156Z","iopub.execute_input":"2021-06-30T06:16:24.377450Z","iopub.status.idle":"2021-06-30T06:16:24.486699Z","shell.execute_reply.started":"2021-06-30T06:16:24.377420Z","shell.execute_reply":"2021-06-30T06:16:24.485585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(tscores.shape)\ntscores  = tscores.drop_duplicates(subset=[\"teamId\", \"index\"], keep=\"last\")\nprint(tscores.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:16:24.488579Z","iopub.execute_input":"2021-06-30T06:16:24.488998Z","iopub.status.idle":"2021-06-30T06:16:24.504646Z","shell.execute_reply.started":"2021-06-30T06:16:24.488952Z","shell.execute_reply":"2021-06-30T06:16:24.503331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"games = pd.read_csv(f\"{DATA_ROOT}/games_train.csv\", usecols=games_id + games_cat + games_num)\ngames = process_games(games)\ngames = games.drop_duplicates(subset=\"gamePk\", keep=\"last\")","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:16:24.505776Z","iopub.execute_input":"2021-06-30T06:16:24.506073Z","iopub.status.idle":"2021-06-30T06:16:24.585708Z","shell.execute_reply.started":"2021-06-30T06:16:24.506044Z","shell.execute_reply":"2021-06-30T06:16:24.584856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## BUILD DATASET...","metadata":{}},{"cell_type":"code","source":"%%time\n#======\ntgt = tgt.merge(rosters, on=[\"playerId\",\"index\"], how=\"left\")\ntgt = tgt.merge(pscores, on=[\"playerId\", \"index\"],  how=\"left\")\ntgt = tgt.merge(tscores, on=[\"teamId\", \"index\"],  how=\"left\")#, suffixes=[\"_p\", \"_t\"]\ntgt = tgt.merge(games, on=\"gamePk\", how=\"left\")\nprint(tgt.shape)\n#====\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:16:24.587127Z","iopub.execute_input":"2021-06-30T06:16:24.587744Z","iopub.status.idle":"2021-06-30T06:16:27.218791Z","shell.execute_reply.started":"2021-06-30T06:16:24.587697Z","shell.execute_reply":"2021-06-30T06:16:27.217589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## REDUCING MEMORY USAGE","metadata":{}},{"cell_type":"code","source":"IDCOLS = list( set( rosters_id + pscores_id + tscores_id + games_id ) ) \nIDCOLS = IDCOLS + [\"EvalDate\"]\nCATCOLS = rosters_cat + pscores_cat  + games_cat\nTGTCOLS = [\"target1\",\"target2\",\"target3\",\"target4\"]\nNUMCOLS = pscores_num + tscores_num + games_num ","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:16:27.220568Z","iopub.execute_input":"2021-06-30T06:16:27.221194Z","iopub.status.idle":"2021-06-30T06:16:27.227722Z","shell.execute_reply.started":"2021-06-30T06:16:27.221147Z","shell.execute_reply":"2021-06-30T06:16:27.226421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TOFLOAT = [\"awayWinPct\", \"homeWinPct\"]\nTOINT = [col for col in NUMCOLS + CATCOLS if col not in TOFLOAT]","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:16:27.229190Z","iopub.execute_input":"2021-06-30T06:16:27.229511Z","iopub.status.idle":"2021-06-30T06:16:27.241931Z","shell.execute_reply.started":"2021-06-30T06:16:27.229471Z","shell.execute_reply":"2021-06-30T06:16:27.241088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndef reduce_df(df, train=True):\n    df[\"jerseyNum\"] = df[\"jerseyNum\"].fillna(100) #100\n    # positionCode #12\n    df[NUMCOLS + CATCOLS] = df[NUMCOLS + CATCOLS].fillna(0.)\n    df[TOINT] = df[TOINT].astype(int)\n    if train:\n        lab = df[TGTCOLS].copy()\n        df.drop(TGTCOLS, axis=1, inplace=True)\n        df = reduce_mem_usage(df)\n        df = df.join(lab)\n    else:\n        df = reduce_mem_usage(df)\n    gc.collect()\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:16:27.243638Z","iopub.execute_input":"2021-06-30T06:16:27.244088Z","iopub.status.idle":"2021-06-30T06:16:27.259860Z","shell.execute_reply.started":"2021-06-30T06:16:27.244035Z","shell.execute_reply":"2021-06-30T06:16:27.258040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntgt  = reduce_df(tgt, train=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:16:27.261379Z","iopub.execute_input":"2021-06-30T06:16:27.262012Z","iopub.status.idle":"2021-06-30T06:17:12.703169Z","shell.execute_reply.started":"2021-06-30T06:16:27.261966Z","shell.execute_reply":"2021-06-30T06:17:12.701759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tgt.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:17:12.704556Z","iopub.execute_input":"2021-06-30T06:17:12.704845Z","iopub.status.idle":"2021-06-30T06:17:12.733100Z","shell.execute_reply.started":"2021-06-30T06:17:12.704817Z","shell.execute_reply":"2021-06-30T06:17:12.731744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:17:12.734303Z","iopub.execute_input":"2021-06-30T06:17:12.734595Z","iopub.status.idle":"2021-06-30T06:17:13.767173Z","shell.execute_reply.started":"2021-06-30T06:17:12.734567Z","shell.execute_reply":"2021-06-30T06:17:13.765912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.models as M\nfrom sklearn.metrics import mean_absolute_error, mean_squared_error\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:17:13.769881Z","iopub.execute_input":"2021-06-30T06:17:13.770307Z","iopub.status.idle":"2021-06-30T06:17:20.012734Z","shell.execute_reply.started":"2021-06-30T06:17:13.770275Z","shell.execute_reply":"2021-06-30T06:17:20.011464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_model(n_in):\n    inp = L.Input(name=\"inputs\", shape=(n_in,))\n    nh = 50\n    x = L.Dense(nh, activation=\"relu\", name=\"d1\")(inp)\n    x = L.Dense(nh, activation=\"relu\", name=\"d2\")(x)\n    #x = L.Dense(nh, activation=\"relu\", name=\"d3\")(x)\n    preds = L.Dense(4, activation=\"linear\", name=\"preds\")(x)\n    \n    model = M.Model(inp, preds, name=\"ANN\")\n    model.compile(loss=\"mean_absolute_error\", optimizer=\"adam\")\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:17:20.014574Z","iopub.execute_input":"2021-06-30T06:17:20.015015Z","iopub.status.idle":"2021-06-30T06:17:20.022702Z","shell.execute_reply.started":"2021-06-30T06:17:20.014967Z","shell.execute_reply":"2021-06-30T06:17:20.021159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = tgt[NUMCOLS].values\ny = tgt[TGTCOLS].values","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:17:20.024720Z","iopub.execute_input":"2021-06-30T06:17:20.025215Z","iopub.status.idle":"2021-06-30T06:17:20.595315Z","shell.execute_reply.started":"2021-06-30T06:17:20.025167Z","shell.execute_reply":"2021-06-30T06:17:20.593849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nsc  = StandardScaler()\nX = sc.fit_transform(X)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:17:20.597105Z","iopub.execute_input":"2021-06-30T06:17:20.597555Z","iopub.status.idle":"2021-06-30T06:17:28.531416Z","shell.execute_reply.started":"2021-06-30T06:17:20.597511Z","shell.execute_reply":"2021-06-30T06:17:28.530107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net = make_model(X.shape[1])\nprint(net.summary())","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:17:28.533317Z","iopub.execute_input":"2021-06-30T06:17:28.533759Z","iopub.status.idle":"2021-06-30T06:17:28.645811Z","shell.execute_reply.started":"2021-06-30T06:17:28.533714Z","shell.execute_reply":"2021-06-30T06:17:28.644806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.mean()","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:17:28.647355Z","iopub.execute_input":"2021-06-30T06:17:28.647952Z","iopub.status.idle":"2021-06-30T06:17:28.842598Z","shell.execute_reply.started":"2021-06-30T06:17:28.647883Z","shell.execute_reply":"2021-06-30T06:17:28.841526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net.fit(X, y, batch_size=30_000, epochs=10)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:17:28.846886Z","iopub.execute_input":"2021-06-30T06:17:28.847365Z","iopub.status.idle":"2021-06-30T06:17:37.819135Z","shell.execute_reply.started":"2021-06-30T06:17:28.847325Z","shell.execute_reply":"2021-06-30T06:17:37.817979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## INFERENCE","metadata":{}},{"cell_type":"code","source":"null = np.nan\ntrue = True\nfalse = False","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:17:37.820912Z","iopub.execute_input":"2021-06-30T06:17:37.821239Z","iopub.status.idle":"2021-06-30T06:17:37.826338Z","shell.execute_reply.started":"2021-06-30T06:17:37.821207Z","shell.execute_reply":"2021-06-30T06:17:37.825085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"START_DATE = pd.to_datetime(\"2018-01-01\")","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:17:37.827953Z","iopub.execute_input":"2021-06-30T06:17:37.828590Z","iopub.status.idle":"2021-06-30T06:17:37.838323Z","shell.execute_reply.started":"2021-06-30T06:17:37.828547Z","shell.execute_reply":"2021-06-30T06:17:37.837477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#LAST_DF = tgt.loc[tgt[time_index]>850].copy().reset_index(drop=True)\n#PLAYERS = pd.DataFrame(tgt.playerId.unique(), columns=[\"playerId\"])\n#LAST_DF.shape, tgt.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:17:37.839954Z","iopub.execute_input":"2021-06-30T06:17:37.840378Z","iopub.status.idle":"2021-06-30T06:17:37.852499Z","shell.execute_reply.started":"2021-06-30T06:17:37.840335Z","shell.execute_reply":"2021-06-30T06:17:37.851253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mlb\nenv = mlb.make_env() # initialize the environment\niter_test = env.iter_test() # iterator which loops over each date in test se","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:17:37.853752Z","iopub.execute_input":"2021-06-30T06:17:37.854201Z","iopub.status.idle":"2021-06-30T06:17:37.888963Z","shell.execute_reply.started":"2021-06-30T06:17:37.854159Z","shell.execute_reply":"2021-06-30T06:17:37.887946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nFE, SUB = [], []\nfor (test_df, sample_prediction_df) in iter_test: \n    FE.append(test_df)\n    SUB.append(sample_prediction_df)\n    env.predict(sample_prediction_df)\n#===========\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:17:37.890211Z","iopub.execute_input":"2021-06-30T06:17:37.890495Z","iopub.status.idle":"2021-06-30T06:17:37.896838Z","shell.execute_reply.started":"2021-06-30T06:17:37.890468Z","shell.execute_reply":"2021-06-30T06:17:37.895784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_test(hid_df, hid_sub):\n    hid_tgt = hid_sub[[\"date\",\"playerId\"]].copy()\n    #================== PART ONE ========================#\n    # adding rosters\n    colname = \"rosters\"\n    \"\"\"#consider input as null\n    rosters = pd.DataFrame(columns=rosters_cat + rosters_id)\n    rosters[\"playerId\"] = hid_tgt[\"playerId\"]\n    \"\"\"\n    if hid_df[colname].iloc[0] ==  hid_df[colname].iloc[0]:\n        rosters = pd.DataFrame(eval(hid_df[colname].iloc[0]))\n        rosters[\"index\"] = (pd.to_datetime(rosters[\"gameDate\"]) - START_DATE).dt.days\n        rosters = rosters[rosters_cat + rosters_id]\n    else:\n        rosters = pd.DataFrame(columns=rosters_cat + rosters_id)\n        rosters[\"playerId\"] = hid_tgt[\"playerId\"]\n    rosters = process_rosters(rosters)\n    # adding player scores\n    colname = \"playerBoxScores\"\n    #consider input as null\n    pscores = pd.DataFrame(columns = pscores_id + pscores_cat + pscores_num)\n    pscores = pscores.rename(columns=ren_player)\n    pscores[\"playerId\"] = hid_tgt[\"playerId\"]\n    \"\"\"\n    if hid_df[colname].iloc[0] ==  hid_df[colname].iloc[0]:## An error occurs in this block.\n        pscores = pd.DataFrame(eval(hid_df[colname].iloc[0]))\n        pscores = pscores.rename(columns=ren_player)\n        pscores[\"index\"] = (pd.to_datetime(pscores[\"gameDate\"]) - START_DATE).dt.days\n        pscores = pscores[pscores_id + pscores_cat + pscores_num]\n    else:\n        pscores = pd.DataFrame(columns = pscores_id + pscores_cat + pscores_num)\n        pscores = pscores.rename(columns=ren_player)\n        pscores[\"playerId\"] = hid_tgt[\"playerId\"]\n    \"\"\"\n    pscores = process_pscores(pscores)\n    pscores = pscores.drop_duplicates(subset=\"playerId\", keep=\"last\")\n    # Merging\n    hid_tgt = hid_tgt.merge(rosters, on=\"playerId\", how=\"left\")\n    hid_tgt = hid_tgt.merge(pscores, on=\"playerId\", how=\"left\")\n    #===================PART TWO=============================\n    colname = \"teamBoxScores\"\n    \"\"\"#consider input as null\n    tscores = pd.DataFrame(columns = tscores_id  + tscores_num)\n    tscores = tscores.rename(columns=ren_team)\n    tscores[\"teamId\"] = list(hid_tgt[\"teamId\"].unique())\n    \"\"\"\n    if hid_df[colname].iloc[0] ==  hid_df[colname].iloc[0]:\n        tscores = pd.DataFrame(eval(hid_df[colname].iloc[0]))\n        tscores = tscores.rename(columns=ren_team)\n        tscores[\"index\"] = (pd.to_datetime(tscores[\"gameDate\"]) - START_DATE).dt.days\n        tscores = tscores[tscores_id  + tscores_num]\n    else:\n        tscores = pd.DataFrame(columns = tscores_id  + tscores_num)\n        tscores = tscores.rename(columns=ren_team)\n        tscores[\"teamId\"] = list(hid_tgt[\"teamId\"].unique())\n        \n    tscores  = tscores.drop_duplicates(subset=\"teamId\", keep=\"last\")\n    hid_tgt = hid_tgt.merge(tscores, on=\"teamId\", how=\"left\")\n    #==================PART THREE ============================\n    colname = \"games\"\n    \"\"\"#consider input as null\n    games = pd.DataFrame(columns = games_id  + games_cat + games_num )\n    games[\"gamePk\"] = list(hid_tgt[\"gamePk\"].unique())\n    \"\"\"\n    if hid_df[colname].iloc[0] ==  hid_df[colname].iloc[0]:\n        games = pd.DataFrame(eval(hid_df[colname].iloc[0]))\n        games[\"index\"] = (pd.to_datetime(games[\"gameDate\"]) - START_DATE).dt.days\n        games = games[games_id  + games_cat + games_num]\n    else:\n        games = pd.DataFrame(columns = games_id  + games_cat + games_num )\n        games[\"gamePk\"] = list(hid_tgt[\"gamePk\"].unique())\n    games = process_games(games)\n    games = games.drop_duplicates(subset=\"gamePk\", keep=\"last\")\n    hid_tgt = hid_tgt.merge(games, on=\"gamePk\", how=\"left\")\n    #================================================\n    hid_tgt  = reduce_df(hid_tgt, train=False)\n    return hid_tgt","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:17:37.898515Z","iopub.execute_input":"2021-06-30T06:17:37.898943Z","iopub.status.idle":"2021-06-30T06:17:37.919075Z","shell.execute_reply.started":"2021-06-30T06:17:37.898883Z","shell.execute_reply":"2021-06-30T06:17:37.917797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nidx = 4\nhid_df = FE[idx].copy()\nhid_sub = SUB[idx].copy()\nprint(hid_sub.shape[0])\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:17:37.920591Z","iopub.execute_input":"2021-06-30T06:17:37.921032Z","iopub.status.idle":"2021-06-30T06:17:37.937807Z","shell.execute_reply.started":"2021-06-30T06:17:37.920989Z","shell.execute_reply":"2021-06-30T06:17:37.936555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nhid_sub = hid_sub.reset_index()\nhid_sub[\"playerId\"] = hid_sub[\"date_playerId\"].apply(lambda x: int( x.split(\"_\")[1] ))\nhid_sub.drop(TGTCOLS, axis=1, inplace=True)\n\nhid_tgt = prepare_test(hid_df, hid_sub)\nXe = hid_tgt[NUMCOLS].values\nXe = sc.transform(Xe)\n    \npe = net.predict(Xe)\npe = np.clip(pe, 0, 100)\n    \nhid_sub[TGTCOLS] = pe\nsub = hid_sub[[\"date_playerId\"] + TGTCOLS].copy()\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:17:37.939419Z","iopub.execute_input":"2021-06-30T06:17:37.939793Z","iopub.status.idle":"2021-06-30T06:17:37.949400Z","shell.execute_reply.started":"2021-06-30T06:17:37.939704Z","shell.execute_reply":"2021-06-30T06:17:37.948084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfor (hid_df, hid_sub) in iter_test: \n    hid_sub = hid_sub.reset_index()\n    hid_sub[\"playerId\"] = hid_sub[\"date_playerId\"].apply(lambda x: int( x.split(\"_\")[1] ))\n    hid_sub.drop(TGTCOLS, axis=1, inplace=True)\n    \n    hid_tgt = prepare_test(hid_df, hid_sub)\n    assert hid_tgt.shape[0] == hid_sub.shape[0]\n    Xe = hid_tgt[NUMCOLS].values\n    Xe = sc.transform(Xe)\n    \n    #pe = net.predict(Xe)\n    #pe = np.clip(pe, 0, 100)\n    \n    hid_sub[TGTCOLS] = 0.\n    sub = hid_sub[[\"date_playerId\"] + TGTCOLS].copy()\n    sub = sub.fillna(0.)\n    env.predict(sub)\n    gc.collect()\n#===========","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:17:37.950907Z","iopub.execute_input":"2021-06-30T06:17:37.951295Z","iopub.status.idle":"2021-06-30T06:17:44.001003Z","shell.execute_reply.started":"2021-06-30T06:17:37.951263Z","shell.execute_reply":"2021-06-30T06:17:43.999720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.tail()","metadata":{"execution":{"iopub.status.busy":"2021-06-30T06:17:44.002717Z","iopub.execute_input":"2021-06-30T06:17:44.003133Z","iopub.status.idle":"2021-06-30T06:17:44.021638Z","shell.execute_reply.started":"2021-06-30T06:17:44.003093Z","shell.execute_reply":"2021-06-30T06:17:44.020679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}