{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Description of the steps\n\n### 1- Rosters (gameDate, playerId, teamId)\n* statusCode: 11 values\n* status : 16 values\n\n### 2- Games (gamePk, gameDate)\nsince some games are postponed or arrested, we have to choose the last line of the game\n* gameType : 8\n* codedGameState : 6\n* detailedGameState : 7\n* isTie : 2\n* seriesDescription : 8\n\n### 3-playerBoxScores (playerId, gameDate,gamePk)\n* jerseyNum: 100\n* positionCode: 12\n* positionName: 8\n* positionType: 6\n\n\n### 4-teamBoxScores (teamId, index,gamePk)\n* No categorical variable\n\n### 5- events\n\n### 6- transactions\n\n### 7- awards\n\n### 8- standings\n\n### 9- playerTwitterFollowers\n\n### 10- teamTwiterFollowers\nOk","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-21T07:05:26.917263Z","iopub.execute_input":"2021-06-21T07:05:26.917649Z","iopub.status.idle":"2021-06-21T07:05:26.92379Z","shell.execute_reply.started":"2021-06-21T07:05:26.917554Z","shell.execute_reply":"2021-06-21T07:05:26.922427Z"}}},{"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df\n#","metadata":{"execution":{"iopub.status.busy":"2021-06-29T12:29:32.691028Z","iopub.execute_input":"2021-06-29T12:29:32.691817Z","iopub.status.idle":"2021-06-29T12:29:32.713434Z","shell.execute_reply.started":"2021-06-29T12:29:32.691694Z","shell.execute_reply":"2021-06-29T12:29:32.712443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#==============ROSTERS========================================\nstatusCode = {'A':1, 'D60':2, 'RM':3, 'D10':4, 'D7':5, 'PL':6, 'SU':7, \n              'FME':8, 'BRV':9, 'RES':10,'DEC':11}\nstatus = {'Active':1, 'Injured 60-Day':2, 'Reassigned to Minors':3,\n       'Injured 10-Day':4, 'Injured 7-Day':5, 'Paternity List':6,\n       'Suspended # days':7, 'Family Medical Emergency':8, 'Bereavement List':9,\n       'Reserve List (Minors)':10, 'Deceased':11, '60-day IL':12, '10-day IL':13,\n       'Reassigned':14, 'Paternity':15, 'Suspended':16}\n#============== GAME =================================#\ngameType = {'E':1, 'S':2, 'R':3, 'A':4, 'F':5, 'D':6, 'L':7, 'W':8}\ncodedGameState = {'F':1, 'C':2, 'D':3, 'S':4, 'U':5, 'O':6}\ndetailedGameState = {'Final':1, 'Cancelled':2, 'Completed Early':3, 'Postponed':4, \n                     'Scheduled':5,'Suspended':6, 'Game Over':7}\nisTie = {False:1, True:2}\nseriesDescription = {'Exhibition':1, 'Spring Training':2, 'Regular Season':3, 'All-Star Game':4,\n                     'Wild Card Game':5, 'Division Series':6, 'League Championship Series':7,\n                     'World Series':8}\n#===============================PLAYER SCORES============================\npositionName = {'Catcher':1,'Designated Hitter':2,'First Base':3,'Outfielder':4,\n                'Pinch Hitter':5,'Pinch Runner':6,'Pitcher':7, 'Second Base':8,\n                'Shortstop':9,'Third Base':10}\npositionType = {'Catcher':1, 'Hitter':2, 'Infielder':3, 'Outfielder':4, 'Pitcher':5, 'Runner':6}\n#=================================GAMES========================\ndoubleHeader = {\"N\":1, \"Y\":2, \"S\":3}\ndayNight = {\"day\":1, \"night\":2}\n#=================================================","metadata":{"execution":{"iopub.status.busy":"2021-06-29T12:29:32.714999Z","iopub.execute_input":"2021-06-29T12:29:32.715598Z","iopub.status.idle":"2021-06-29T12:29:32.728789Z","shell.execute_reply.started":"2021-06-29T12:29:32.715535Z","shell.execute_reply":"2021-06-29T12:29:32.727920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SCORES = ['home', 'flyOuts', 'groundOuts', 'runsScored', 'doubles', 'triples', 'homeRuns', 'strikeOuts', \n          'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch', 'atBats', 'caughtStealing', 'stolenBases', \n          'groundIntoDoublePlay', 'groundIntoTriplePlay', 'plateAppearances', 'totalBases', 'rbi', 'leftOnBase', \n          'sacBunts', 'sacFlies', 'catchersInterference', 'pickoffs', 'airOutsPitching', 'groundOutsPitching', \n          'runsPitching', 'doublesPitching', 'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching', \n          'baseOnBallsPitching', 'intentionalWalksPitching', 'hitsPitching', 'hitByPitchPitching', 'atBatsPitching', \n          'caughtStealingPitching', 'stolenBasesPitching', 'inningsPitched', 'earnedRuns', 'battersFaced', 'outsPitching', \n          'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching', 'rbiPitching', 'inheritedRunners', 'inheritedRunnersScored', \n          'catchersInterferencePitching', 'sacBuntsPitching', 'sacFliesPitching']\nren_player = {col:col+\"_p\" for col in SCORES}\nren_team = {col:col+\"_t\" for col in SCORES}","metadata":{"execution":{"iopub.status.busy":"2021-06-29T12:29:32.733082Z","iopub.execute_input":"2021-06-29T12:29:32.733519Z","iopub.status.idle":"2021-06-29T12:29:32.744759Z","shell.execute_reply.started":"2021-06-29T12:29:32.733488Z","shell.execute_reply":"2021-06-29T12:29:32.743656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_index = \"index\"\n#============================\nrosters_id = [time_index, 'playerId']\nrosters_cat = [\"statusCode\",\"status\"]\n#=========================\npscores_id = [\"playerId\",\"index\", 'teamId'] #, \"gamePk\"\npscores_cat = [\"jerseyNum\",\"positionCode\",\"positionName\",\"positionType\"]\npscores_num = ['home', \n               'gamesPlayedBatting', 'flyOuts', 'groundOuts', 'runsScored', 'doubles', 'triples', \n               'homeRuns', 'strikeOuts', 'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch', \n               'atBats', 'caughtStealing', 'stolenBases', 'groundIntoDoublePlay', 'groundIntoTriplePlay', \n               'plateAppearances', 'totalBases', 'rbi', 'leftOnBase', 'sacBunts', 'sacFlies', \n               'catchersInterference', 'pickoffs', 'gamesPlayedPitching', 'gamesStartedPitching', \n               'completeGamesPitching', 'shutoutsPitching', 'winsPitching', 'lossesPitching', 'flyOutsPitching', \n               'airOutsPitching', 'groundOutsPitching', \n               'runsPitching', 'doublesPitching', 'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching', \n               'baseOnBallsPitching', 'intentionalWalksPitching', 'hitsPitching', 'hitByPitchPitching', \n               'atBatsPitching', 'caughtStealingPitching', 'stolenBasesPitching', 'inningsPitched', \n               'saveOpportunities', 'earnedRuns', 'battersFaced', 'outsPitching', 'pitchesThrown', \n               'balls', 'strikes', 'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching', 'rbiPitching', \n               'gamesFinishedPitching', 'inheritedRunners', 'inheritedRunnersScored', 'catchersInterferencePitching', \n               'sacBuntsPitching', 'sacFliesPitching', 'saves', 'holds', 'blownSaves', 'assists', \n               'putOuts', 'errors', 'chances']\npscores_num = [col+'_p' if (col in SCORES) else col for col in pscores_num]\n#==============\ntscores_id = [\"teamId\", \"index\",\"gamePk\"]\ntscores_num = ['home', 'flyOuts',\n       'groundOuts', 'runsScored', 'doubles', 'triples', 'homeRuns',\n       'strikeOuts', 'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch',\n       'atBats', 'caughtStealing', 'stolenBases', 'groundIntoDoublePlay',\n       'groundIntoTriplePlay', 'plateAppearances', 'totalBases', 'rbi',\n       'leftOnBase', 'sacBunts', 'sacFlies', 'catchersInterference',\n       'pickoffs', 'airOutsPitching', 'groundOutsPitching', 'runsPitching',\n       'doublesPitching', 'triplesPitching', 'homeRunsPitching',\n       'strikeOutsPitching', 'baseOnBallsPitching', 'intentionalWalksPitching',\n       'hitsPitching', 'hitByPitchPitching', 'atBatsPitching',\n       'caughtStealingPitching', 'stolenBasesPitching', 'inningsPitched',\n       'earnedRuns', 'battersFaced', 'outsPitching', 'hitBatsmen', 'balks',\n       'wildPitches', 'pickoffsPitching', 'rbiPitching', 'inheritedRunners',\n       'inheritedRunnersScored', 'catchersInterferencePitching',\n       'sacBuntsPitching', 'sacFliesPitching']\ntscores_num = [col+'_t' if (col in SCORES) else col for col in tscores_num]\n#===========================\ngames_cat = [\"gameType\",\"codedGameState\",\"detailedGameState\",\"isTie\",\"seriesDescription\",\n             \"doubleHeader\", \"dayNight\"]\ngames_id = ['gamePk','homeId', 'awayId']\ngames_num = ['gameNumber',  'scheduledInnings', 'gamesInSeries', 'homeWins', 'homeLosses', \n             'homeWinPct', 'homeWinner', 'homeScore', 'awayWins', 'awayLosses', 'awayWinPct', \n             'awayWinner', 'awayScore']\n# season (cat) / 'resumeDate', 'resumedFrom' (continuity)\n#===========================","metadata":{"execution":{"iopub.status.busy":"2021-06-29T12:29:32.746358Z","iopub.execute_input":"2021-06-29T12:29:32.746648Z","iopub.status.idle":"2021-06-29T12:29:32.762824Z","shell.execute_reply.started":"2021-06-29T12:29:32.746615Z","shell.execute_reply":"2021-06-29T12:29:32.761871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#============================\ndef process_rosters(df):\n    df[\"statusCode\"] = df[\"statusCode\"].map(statusCode)\n    df[\"status\"] = df[\"status\"].map(status)\n    return df\n#=================\ndef process_pscores(df):\n    # deal with battingOrder\n    df[\"positionName\"] = df[\"positionName\"].map(positionName)\n    df[\"positionType\"] = df[\"positionType\"].map(positionType)\n    df[\"jerseyNum\"] = df[\"jerseyNum\"].fillna(100)\n    df[\"positionCode\"] = df[\"positionCode\"].fillna(0)\n    return df\n#======================================\ndef process_games(df):\n    #add season\n    df[\"gameType\"] = df[\"gameType\"].map(gameType)\n    df[\"codedGameState\"] = df[\"codedGameState\"].map(codedGameState)\n    df[\"detailedGameState\"] = df[\"detailedGameState\"].map(detailedGameState)\n    df[\"isTie\"] = df[\"isTie\"].map(isTie)\n    df[\"seriesDescription\"] = df[\"seriesDescription\"].map(seriesDescription)\n    df[\"doubleHeader\"] = df[\"doubleHeader\"].map(doubleHeader)\n    df[\"dayNight\"] = df[\"dayNight\"].map(dayNight)\n    return df\n#====================================\n\n#========================","metadata":{"execution":{"iopub.status.busy":"2021-06-29T12:29:47.155621Z","iopub.execute_input":"2021-06-29T12:29:47.156112Z","iopub.status.idle":"2021-06-29T12:29:47.164030Z","shell.execute_reply.started":"2021-06-29T12:29:47.156080Z","shell.execute_reply":"2021-06-29T12:29:47.163165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nimport os\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2021-06-29T12:29:47.948999Z","iopub.execute_input":"2021-06-29T12:29:47.949344Z","iopub.status.idle":"2021-06-29T12:29:47.953874Z","shell.execute_reply.started":"2021-06-29T12:29:47.949315Z","shell.execute_reply":"2021-06-29T12:29:47.952876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_ROOT = \"../input/mlb-pdef-train-dataset\"","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:51:34.96158Z","iopub.execute_input":"2021-06-29T09:51:34.962077Z","iopub.status.idle":"2021-06-29T09:51:34.965209Z","shell.execute_reply.started":"2021-06-29T09:51:34.962034Z","shell.execute_reply":"2021-06-29T09:51:34.964549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#os.listdir(DATA_ROOT)","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:51:35.182898Z","iopub.execute_input":"2021-06-29T09:51:35.183485Z","iopub.status.idle":"2021-06-29T09:51:35.186407Z","shell.execute_reply.started":"2021-06-29T09:51:35.183452Z","shell.execute_reply":"2021-06-29T09:51:35.185664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = True","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:51:35.619788Z","iopub.execute_input":"2021-06-29T09:51:35.620411Z","iopub.status.idle":"2021-06-29T09:51:35.624023Z","shell.execute_reply.started":"2021-06-29T09:51:35.620372Z","shell.execute_reply":"2021-06-29T09:51:35.623031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntgt = pd.read_csv(f\"{DATA_ROOT}/nextDayPlayerEngagement_train.csv\")\ntgt.drop(\"engagementMetricsDate\", axis=1, inplace=True)\ntgt[\"date\"] = pd.to_datetime(tgt[\"date\"], format='%Y%m%d')\ntgt = tgt.rename(columns={\"date\":\"EvalDate\"})\n\nif DEBUG:\n    print(tgt.shape)\n    tgt = tgt.loc[tgt[time_index]>=1000].reset_index(drop=True)\n    print(tgt.shape)\n#\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:51:36.081024Z","iopub.execute_input":"2021-06-29T09:51:36.081387Z","iopub.status.idle":"2021-06-29T09:51:41.436109Z","shell.execute_reply.started":"2021-06-29T09:51:36.081356Z","shell.execute_reply":"2021-06-29T09:51:41.435173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tgt.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:51:41.43748Z","iopub.execute_input":"2021-06-29T09:51:41.437772Z","iopub.status.idle":"2021-06-29T09:51:41.45591Z","shell.execute_reply.started":"2021-06-29T09:51:41.437745Z","shell.execute_reply":"2021-06-29T09:51:41.455184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tgt[\"index\"].max()","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:51:41.457668Z","iopub.execute_input":"2021-06-29T09:51:41.45823Z","iopub.status.idle":"2021-06-29T09:51:41.46514Z","shell.execute_reply.started":"2021-06-29T09:51:41.458182Z","shell.execute_reply":"2021-06-29T09:51:41.464272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nrosters = pd.read_csv(f\"{DATA_ROOT}/rosters_train.csv\", usecols=rosters_cat + rosters_id)\nrosters = process_rosters(rosters)","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:51:41.466604Z","iopub.execute_input":"2021-06-29T09:51:41.466854Z","iopub.status.idle":"2021-06-29T09:51:43.2031Z","shell.execute_reply.started":"2021-06-29T09:51:41.466831Z","shell.execute_reply":"2021-06-29T09:51:43.201639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\npscores = pd.read_csv(f\"{DATA_ROOT}/playerBoxScores_train.csv\")\npscores = pscores.rename(columns=ren_player)\npscores = pscores[pscores_id+pscores_cat+pscores_num]\npscores = process_pscores(pscores)\n#================\nNO_DOUBLON = True\nif NO_DOUBLON:\n    print(pscores.shape)\n    pscores = pscores.drop_duplicates(subset=rosters_id, keep=\"last\")\n    print(pscores.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:51:43.204231Z","iopub.execute_input":"2021-06-29T09:51:43.20451Z","iopub.status.idle":"2021-06-29T09:51:45.272675Z","shell.execute_reply.started":"2021-06-29T09:51:43.204477Z","shell.execute_reply":"2021-06-29T09:51:45.272041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tscores = pd.read_csv(f\"{DATA_ROOT}/teamBoxScores_train.csv\")\ntscores = tscores.rename(columns=ren_team)\ntscores = tscores[tscores_id + tscores_num]","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:51:45.273654Z","iopub.execute_input":"2021-06-29T09:51:45.274075Z","iopub.status.idle":"2021-06-29T09:51:45.384773Z","shell.execute_reply.started":"2021-06-29T09:51:45.274035Z","shell.execute_reply":"2021-06-29T09:51:45.383839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(tscores.shape)\ntscores  = tscores.drop_duplicates(subset=[\"teamId\", \"index\"], keep=\"last\")\nprint(tscores.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:51:45.386073Z","iopub.execute_input":"2021-06-29T09:51:45.386349Z","iopub.status.idle":"2021-06-29T09:51:45.397791Z","shell.execute_reply.started":"2021-06-29T09:51:45.386323Z","shell.execute_reply":"2021-06-29T09:51:45.396813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"games = pd.read_csv(f\"{DATA_ROOT}/games_train.csv\", usecols=games_id + games_cat + games_num)\ngames = process_games(games)\ngames = games.drop_duplicates(subset=\"gamePk\", keep=\"last\")","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:51:45.401192Z","iopub.execute_input":"2021-06-29T09:51:45.401466Z","iopub.status.idle":"2021-06-29T09:51:45.481201Z","shell.execute_reply.started":"2021-06-29T09:51:45.401435Z","shell.execute_reply":"2021-06-29T09:51:45.480454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## BUILD DATASET...","metadata":{}},{"cell_type":"code","source":"%%time\n#======\ntgt = tgt.merge(rosters, on=[\"playerId\",\"index\"], how=\"left\")\ntgt = tgt.merge(pscores, on=[\"playerId\", \"index\"],  how=\"left\")\ntgt = tgt.merge(tscores, on=[\"teamId\", \"index\"],  how=\"left\")#, suffixes=[\"_p\", \"_t\"]\ntgt = tgt.merge(games, on=\"gamePk\", how=\"left\")\nprint(tgt.shape)\n#====\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:51:45.482612Z","iopub.execute_input":"2021-06-29T09:51:45.483065Z","iopub.status.idle":"2021-06-29T09:51:47.449838Z","shell.execute_reply.started":"2021-06-29T09:51:45.483025Z","shell.execute_reply":"2021-06-29T09:51:47.448908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## REDUCING MEMORY USAGE","metadata":{}},{"cell_type":"code","source":"IDCOLS = list( set( rosters_id + pscores_id + tscores_id + games_id ) ) \nIDCOLS = IDCOLS + [\"EvalDate\"]\nCATCOLS = rosters_cat + pscores_cat  + games_cat\nTGTCOLS = [\"target1\",\"target2\",\"target3\",\"target4\"]\nNUMCOLS = pscores_num + tscores_num + games_num ","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:51:47.450975Z","iopub.execute_input":"2021-06-29T09:51:47.451253Z","iopub.status.idle":"2021-06-29T09:51:47.456661Z","shell.execute_reply.started":"2021-06-29T09:51:47.451228Z","shell.execute_reply":"2021-06-29T09:51:47.455588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TOFLOAT = [\"awayWinPct\", \"homeWinPct\"]\nTOINT = [col for col in NUMCOLS + CATCOLS if col not in TOFLOAT]","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:51:47.457842Z","iopub.execute_input":"2021-06-29T09:51:47.458179Z","iopub.status.idle":"2021-06-29T09:51:47.467908Z","shell.execute_reply.started":"2021-06-29T09:51:47.458152Z","shell.execute_reply":"2021-06-29T09:51:47.467064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndef reduce_df(df, train=True):\n    df[\"jerseyNum\"] = df[\"jerseyNum\"].fillna(100) #100\n    # positionCode #12\n    df[NUMCOLS + CATCOLS] = df[NUMCOLS + CATCOLS].fillna(0.)\n    df[TOINT] = df[TOINT].astype(int)\n    if train:\n        lab = df[TGTCOLS].copy()\n        df.drop(TGTCOLS, axis=1, inplace=True)\n        df = reduce_mem_usage(df)\n        df = df.join(lab)\n    else:\n        df = reduce_mem_usage(df)\n    gc.collect()\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:51:47.469382Z","iopub.execute_input":"2021-06-29T09:51:47.46967Z","iopub.status.idle":"2021-06-29T09:51:47.480545Z","shell.execute_reply.started":"2021-06-29T09:51:47.469643Z","shell.execute_reply":"2021-06-29T09:51:47.479455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntgt  = reduce_df(tgt, train=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:51:47.481981Z","iopub.execute_input":"2021-06-29T09:51:47.482466Z","iopub.status.idle":"2021-06-29T09:52:15.945863Z","shell.execute_reply.started":"2021-06-29T09:51:47.482416Z","shell.execute_reply":"2021-06-29T09:52:15.945016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tgt.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:52:15.947309Z","iopub.execute_input":"2021-06-29T09:52:15.947694Z","iopub.status.idle":"2021-06-29T09:52:15.972582Z","shell.execute_reply.started":"2021-06-29T09:52:15.947652Z","shell.execute_reply":"2021-06-29T09:52:15.971733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:52:15.973684Z","iopub.execute_input":"2021-06-29T09:52:15.973944Z","iopub.status.idle":"2021-06-29T09:52:16.749802Z","shell.execute_reply.started":"2021-06-29T09:52:15.973918Z","shell.execute_reply":"2021-06-29T09:52:16.749007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.models as M\nfrom sklearn.metrics import mean_absolute_error, mean_squared_error\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:52:16.75087Z","iopub.execute_input":"2021-06-29T09:52:16.751313Z","iopub.status.idle":"2021-06-29T09:52:22.000986Z","shell.execute_reply.started":"2021-06-29T09:52:16.75127Z","shell.execute_reply":"2021-06-29T09:52:22.000246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_model(n_in):\n    inp = L.Input(name=\"inputs\", shape=(n_in,))\n    nh = 50\n    x = L.Dense(nh, activation=\"relu\", name=\"d1\")(inp)\n    x = L.Dense(nh, activation=\"relu\", name=\"d2\")(x)\n    #x = L.Dense(nh, activation=\"relu\", name=\"d3\")(x)\n    preds = L.Dense(4, activation=\"linear\", name=\"preds\")(x)\n    \n    model = M.Model(inp, preds, name=\"ANN\")\n    model.compile(loss=\"mean_absolute_error\", optimizer=\"adam\")\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:52:22.002432Z","iopub.execute_input":"2021-06-29T09:52:22.003013Z","iopub.status.idle":"2021-06-29T09:52:22.011154Z","shell.execute_reply.started":"2021-06-29T09:52:22.002968Z","shell.execute_reply":"2021-06-29T09:52:22.01007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = tgt[NUMCOLS].values\ny = tgt[TGTCOLS].values","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:52:22.012188Z","iopub.execute_input":"2021-06-29T09:52:22.012454Z","iopub.status.idle":"2021-06-29T09:52:22.516615Z","shell.execute_reply.started":"2021-06-29T09:52:22.01243Z","shell.execute_reply":"2021-06-29T09:52:22.515626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nsc  = StandardScaler()\nX = sc.fit_transform(X)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:52:22.517715Z","iopub.execute_input":"2021-06-29T09:52:22.517974Z","iopub.status.idle":"2021-06-29T09:52:29.908908Z","shell.execute_reply.started":"2021-06-29T09:52:22.517949Z","shell.execute_reply":"2021-06-29T09:52:29.907996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net = make_model(X.shape[1])\nprint(net.summary())","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:52:29.910423Z","iopub.execute_input":"2021-06-29T09:52:29.910813Z","iopub.status.idle":"2021-06-29T09:52:30.004126Z","shell.execute_reply.started":"2021-06-29T09:52:29.910772Z","shell.execute_reply":"2021-06-29T09:52:30.003125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.mean()","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:52:30.005362Z","iopub.execute_input":"2021-06-29T09:52:30.00564Z","iopub.status.idle":"2021-06-29T09:52:30.196317Z","shell.execute_reply.started":"2021-06-29T09:52:30.005614Z","shell.execute_reply":"2021-06-29T09:52:30.195476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net.fit(X, y, batch_size=30_000, epochs=10)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:52:30.198252Z","iopub.execute_input":"2021-06-29T09:52:30.198514Z","iopub.status.idle":"2021-06-29T09:52:36.928157Z","shell.execute_reply.started":"2021-06-29T09:52:30.198488Z","shell.execute_reply":"2021-06-29T09:52:36.927319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## INFERENCE","metadata":{}},{"cell_type":"code","source":"null = np.nan\ntrue = True\nfalse = False","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:52:36.929422Z","iopub.execute_input":"2021-06-29T09:52:36.929654Z","iopub.status.idle":"2021-06-29T09:52:36.933275Z","shell.execute_reply.started":"2021-06-29T09:52:36.929631Z","shell.execute_reply":"2021-06-29T09:52:36.93232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"START_DATE = pd.to_datetime(\"2018-01-01\")","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:52:36.934362Z","iopub.execute_input":"2021-06-29T09:52:36.934629Z","iopub.status.idle":"2021-06-29T09:52:36.94491Z","shell.execute_reply.started":"2021-06-29T09:52:36.934603Z","shell.execute_reply":"2021-06-29T09:52:36.944354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#LAST_DF = tgt.loc[tgt[time_index]>850].copy().reset_index(drop=True)\n#PLAYERS = pd.DataFrame(tgt.playerId.unique(), columns=[\"playerId\"])\n#LAST_DF.shape, tgt.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:39:14.800557Z","iopub.execute_input":"2021-06-29T09:39:14.800862Z","iopub.status.idle":"2021-06-29T09:39:14.816315Z","shell.execute_reply.started":"2021-06-29T09:39:14.800833Z","shell.execute_reply":"2021-06-29T09:39:14.815356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mlb\nenv = mlb.make_env() # initialize the environment\niter_test = env.iter_test() # iterator which loops over each date in test se","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:52:36.945982Z","iopub.execute_input":"2021-06-29T09:52:36.94655Z","iopub.status.idle":"2021-06-29T09:52:36.985819Z","shell.execute_reply.started":"2021-06-29T09:52:36.946522Z","shell.execute_reply":"2021-06-29T09:52:36.984783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nFE, SUB = [], []\nfor (test_df, sample_prediction_df) in iter_test: \n    FE.append(test_df)\n    SUB.append(sample_prediction_df)\n    env.predict(sample_prediction_df)\n#===========\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:53:16.674516Z","iopub.execute_input":"2021-06-29T09:53:16.674889Z","iopub.status.idle":"2021-06-29T09:53:18.271617Z","shell.execute_reply.started":"2021-06-29T09:53:16.674854Z","shell.execute_reply":"2021-06-29T09:53:18.270622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_test(hid_df, hid_sub):\n    hid_tgt = hid_sub[[\"date\",\"playerId\"]].copy()\n    #================== PART ONE ========================#\n    # adding rosters\n    colname = \"rosters\"\n    if hid_df[colname].iloc[0] ==  hid_df[colname].iloc[0]:\n        rosters = pd.DataFrame(eval(hid_df[colname].iloc[0]))\n        rosters[\"index\"] = (pd.to_datetime(rosters[\"gameDate\"]) - START_DATE).dt.days\n        rosters = rosters[rosters_cat + rosters_id]\n    else:\n        rosters = pd.DataFrame({\"playerId\":hid_tgt[\"playerId\"]})\n        for _col in rosters_cat :\n            rosters[_col] = np.nan\n        # end for\n    rosters = process_rosters(rosters)\n    # adding player scores\n    colname = \"playerBoxScores\"\n    if hid_df[colname].iloc[0] ==  hid_df[colname].iloc[0]:\n        pscores = pd.DataFrame(eval(hid_df[colname].iloc[0]))\n        pscores = pscores.rename(columns=ren_player)\n        pscores[\"index\"] = (pd.to_datetime(pscores[\"gameDate\"]) - START_DATE).dt.days\n        pscores = pscores[pscores_id + pscores_cat + pscores_num]\n    else:\n        pscores = pd.DataFrame({\"playerId\":hid_tgt[\"playerId\"]})\n        for _col in pscores_cat + pscores_num:\n            pscores[_col] = np.nan\n        #\n    pscores = process_pscores(pscores)\n    pscores = pscores.drop_duplicates(subset=\"playerId\", keep=\"last\")\n    # Merging\n    hid_tgt = hid_tgt.merge(rosters, on=\"playerId\", how=\"left\")\n    hid_tgt = hid_tgt.merge(pscores, on=\"playerId\", how=\"left\")\n    #===================PART TWO=============================\n    colname = \"teamBoxScores\"\n    if hid_df[colname].iloc[0] ==  hid_df[colname].iloc[0]:\n        tscores = pd.DataFrame(eval(hid_df[colname].iloc[0]))\n        tscores = tscores.rename(columns=ren_team)\n        tscores[\"index\"] = (pd.to_datetime(tscores[\"gameDate\"]) - START_DATE).dt.days\n        tscores = tscores[tscores_id  + tscores_num]\n    else:\n        tscores = pd.DataFrame({\"teamId\":list(hid_tgt[\"teamId\"].unique())})\n        for _col in tscores_num:\n            tscores[_col] = np.nan\n    tscores  = tscores.drop_duplicates(subset=\"teamId\", keep=\"last\")\n    hid_tgt = hid_tgt.merge(tscores, on=\"teamId\", how=\"left\")\n    #==================PART THREE ============================\n    colname = \"games\"\n    if hid_df[colname].iloc[0] ==  hid_df[colname].iloc[0]:\n        games = pd.DataFrame(eval(hid_df[colname].iloc[0]))\n        games[\"index\"] = (pd.to_datetime(games[\"gameDate\"]) - START_DATE).dt.days\n        games = games[games_id  + games_cat + games_num]\n    else:\n        games = pd.DataFrame({\"gamePk\": list(hid_tgt[\"gamePk\"].unique()) })\n        for _col in games_cat + games_num:\n            games[_col] = np.nan\n    games = process_games(games)\n    games = games.drop_duplicates(subset=\"gamePk\", keep=\"last\")\n    hid_tgt = hid_tgt.merge(games, on=\"gamePk\", how=\"left\")\n    #================================================\n    hid_tgt  = reduce_df(hid_tgt, train=False)\n    return hid_tgt","metadata":{"execution":{"iopub.status.busy":"2021-06-29T10:05:09.269902Z","iopub.execute_input":"2021-06-29T10:05:09.270474Z","iopub.status.idle":"2021-06-29T10:05:09.289001Z","shell.execute_reply.started":"2021-06-29T10:05:09.270436Z","shell.execute_reply":"2021-06-29T10:05:09.288067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nidx = 4\nhid_df = FE[idx].copy()\nhid_sub = SUB[idx].copy()\nprint(hid_sub.shape[0])\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-06-29T10:19:18.830486Z","iopub.execute_input":"2021-06-29T10:19:18.831092Z","iopub.status.idle":"2021-06-29T10:19:18.83718Z","shell.execute_reply.started":"2021-06-29T10:19:18.831025Z","shell.execute_reply":"2021-06-29T10:19:18.836266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nhid_sub = hid_sub.reset_index()\nhid_sub[\"playerId\"] = hid_sub[\"date_playerId\"].apply(lambda x: int( x.split(\"_\")[1] ))\nhid_sub.drop(TGTCOLS, axis=1, inplace=True)\n\nhid_tgt = prepare_test(hid_df, hid_sub)\nXe = hid_tgt[NUMCOLS].values\nXe = sc.transform(Xe)\n    \npe = net.predict(Xe)\npe = np.clip(pe, 0, 100)\n    \nhid_sub[TGTCOLS] = pe\nsub = hid_sub[[\"date_playerId\"] + TGTCOLS].copy()\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-06-29T10:19:20.233082Z","iopub.execute_input":"2021-06-29T10:19:20.233644Z","iopub.status.idle":"2021-06-29T10:19:20.848224Z","shell.execute_reply.started":"2021-06-29T10:19:20.233597Z","shell.execute_reply":"2021-06-29T10:19:20.847422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfor (hid_df, hid_sub) in iter_test: \n    hid_sub = hid_sub.reset_index()\n    hid_sub[\"playerId\"] = hid_sub[\"date_playerId\"].apply(lambda x: int( x.split(\"_\")[1] ))\n    hid_sub.drop(TGTCOLS, axis=1, inplace=True)\n    \n    hid_tgt = prepare_test(hid_df, hid_sub)\n    assert hid_tgt.shape[0] == hid_sub.shape[0]\n    Xe = hid_tgt[NUMCOLS].values\n    Xe = sc.transform(Xe)\n    \n    pe = net.predict(Xe)\n    pe = np.clip(pe, 0, 100)\n    \n    hid_sub[TGTCOLS] = pe\n    sub = hid_sub[[\"date_playerId\"] + TGTCOLS].copy()\n    sub = sub.fillna(0.)\n    env.predict(sub)\n#===========","metadata":{"execution":{"iopub.status.busy":"2021-06-29T09:39:14.901798Z","iopub.execute_input":"2021-06-29T09:39:14.902178Z","iopub.status.idle":"2021-06-29T09:39:20.079451Z","shell.execute_reply.started":"2021-06-29T09:39:14.90215Z","shell.execute_reply":"2021-06-29T09:39:20.078146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.tail()","metadata":{"execution":{"iopub.status.busy":"2021-06-29T10:24:59.314556Z","iopub.execute_input":"2021-06-29T10:24:59.315001Z","iopub.status.idle":"2021-06-29T10:24:59.327425Z","shell.execute_reply.started":"2021-06-29T10:24:59.314964Z","shell.execute_reply":"2021-06-29T10:24:59.326493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}