{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom sklearn.metrics import mean_absolute_error\nfrom datetime import timedelta\nfrom functools import reduce\nfrom tqdm import tqdm\nimport lightgbm as lgbm\nimport mlb\nimport gc\n#import riiideducation\n\npd.options.display.max_rows = 200\npd.options.display.max_columns = 100\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-07T17:00:52.751562Z","iopub.execute_input":"2021-08-07T17:00:52.752035Z","iopub.status.idle":"2021-08-07T17:00:55.893177Z","shell.execute_reply.started":"2021-08-07T17:00:52.751933Z","shell.execute_reply":"2021-08-07T17:00:55.892391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2\n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-07T17:00:55.895954Z","iopub.execute_input":"2021-08-07T17:00:55.896201Z","iopub.status.idle":"2021-08-07T17:00:55.909788Z","shell.execute_reply.started":"2021-08-07T17:00:55.896177Z","shell.execute_reply":"2021-08-07T17:00:55.908828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_DIR = Path('../input/mlb-player-digital-engagement-forecasting')\nTRAIN_DIR = Path('../input/mlb-pdef-train-dataset')","metadata":{"execution":{"iopub.status.busy":"2021-08-07T17:00:55.913437Z","iopub.execute_input":"2021-08-07T17:00:55.913703Z","iopub.status.idle":"2021-08-07T17:00:55.920704Z","shell.execute_reply.started":"2021-08-07T17:00:55.913679Z","shell.execute_reply":"2021-08-07T17:00:55.919879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets_cols = [\n    'playerId', \n    'target1', \n    'target2', \n    'target3', \n    'target4', \n    'date'\n]\n\nplayers_cols = [\n    'playerId', \n    'primaryPositionName'\n]\n\nteams_cols = [\n    'id', \n#     'name', \n#     'teamName', \n#     'teamCode', \n#     'shortName', \n#     'abbreviation', \n#     'locationName', \n    'leagueId', \n#     'leagueName', \n    'divisionId', \n#     'divisionName', \n#     'venueId', \n#     'venueName'\n]\n\nrosters_cols = [\n    'playerId', \n    'teamId', \n    'status', \n    'date'\n]\n\nscores_cols = [\n    'playerId', \n    'battingOrder', \n    'gamesPlayedBatting', \n    'flyOuts',\n    'groundOuts', \n    'runsScored', \n    'doubles', \n    'triples', \n    'homeRuns',\n    'strikeOuts', \n    'baseOnBalls', \n    'intentionalWalks', \n    'hits', \n    'hitByPitch',\n    'atBats', \n    'caughtStealing', \n    'stolenBases', \n    'groundIntoDoublePlay',\n    'groundIntoTriplePlay', \n    'plateAppearances', \n    'totalBases', \n    'rbi',\n    'leftOnBase', \n    'sacBunts', \n    'sacFlies', \n    'catchersInterference',\n    'pickoffs', \n    'gamesPlayedPitching', \n    'gamesStartedPitching',\n    'completeGamesPitching', \n    'shutoutsPitching', \n    'winsPitching',\n    'lossesPitching', \n    'flyOutsPitching', \n    'airOutsPitching',\n    'groundOutsPitching', \n    'runsPitching', \n    'doublesPitching',\n    'triplesPitching', \n    'homeRunsPitching', \n    'strikeOutsPitching',\n    'baseOnBallsPitching', \n    'intentionalWalksPitching', \n    'hitsPitching',\n    'hitByPitchPitching', \n    'atBatsPitching', \n    'caughtStealingPitching',\n    'stolenBasesPitching', \n    'inningsPitched', \n    'saveOpportunities',\n    'earnedRuns', \n    'battersFaced', \n    'outsPitching', \n    'pitchesThrown', \n    'balls',\n    'strikes', \n    'hitBatsmen', \n    'balks', \n    'wildPitches', \n    'pickoffsPitching',\n    'rbiPitching', \n    'gamesFinishedPitching', \n    'inheritedRunners',\n    'inheritedRunnersScored', \n    'catchersInterferencePitching',\n    'sacBuntsPitching', \n    'sacFliesPitching', \n    'saves', \n    'holds', \n    'blownSaves',\n    'assists', \n    'putOuts', \n    'errors', \n    'chances', \n    'date'\n]\n\nawards_cols = [\n    'date', \n    'playerId',\n    'awardId'\n]\n\nplayerTwitterFollowers_cols = [\n    'playerId', \n    'numberOfFollowers'\n]\n\nteamTwitterFollowers_cols = [\n    'teamId', \n    'numberOfFollowers'\n]\n\nstandings_cols = [\n    'teamId', \n#     'wildCardRank', \n    'wins', \n    'losses', \n#     'divisionChamp', \n#     'divisionLeader', \n#     'wildCardLeader', \n    'lastTenWins',\n    'lastTenLosses',\n    'date'\n]\n\nfeature_cols = [\n    'label_playerId', \n    'label_primaryPositionName', \n    'label_teamId',\n    'label_status',\n    'playerId', \n    'battingOrder', \n    'gamesPlayedBatting', \n    'flyOuts',\n    'groundOuts', \n    'runsScored', \n    'doubles', \n    'triples', \n    'homeRuns',\n    'strikeOuts', \n    'baseOnBalls', \n    'intentionalWalks', \n    'hits', \n    'hitByPitch',\n    'atBats', \n    'caughtStealing', \n    'stolenBases', \n    'groundIntoDoublePlay',\n    'groundIntoTriplePlay', \n    'plateAppearances', \n    'totalBases', \n    'rbi',\n    'leftOnBase', \n    'sacBunts', \n    'sacFlies', \n    'catchersInterference',\n    'pickoffs', \n    'gamesPlayedPitching', \n    'gamesStartedPitching',\n    'completeGamesPitching', \n    'shutoutsPitching', \n    'winsPitching',\n    'lossesPitching', \n    'flyOutsPitching', \n    'airOutsPitching',\n    'groundOutsPitching', \n    'runsPitching', \n    'doublesPitching',\n    'triplesPitching', \n    'homeRunsPitching', \n    'strikeOutsPitching',\n    'baseOnBallsPitching', \n    'intentionalWalksPitching', \n    'hitsPitching',\n    'hitByPitchPitching', \n    'atBatsPitching', \n    'caughtStealingPitching',\n    'stolenBasesPitching', \n    'inningsPitched', \n    'saveOpportunities',\n    'earnedRuns', \n    'battersFaced', \n    'outsPitching', \n    'pitchesThrown', \n    'balls',\n    'strikes', \n    'hitBatsmen', \n    'balks', \n    'wildPitches', \n    'pickoffsPitching',\n    'rbiPitching', \n    'gamesFinishedPitching', \n    'inheritedRunners',\n    'inheritedRunnersScored', \n    'catchersInterferencePitching',\n    'sacBuntsPitching', \n    'sacFliesPitching', \n    'saves', \n    'holds', \n    'blownSaves',\n    'assists', \n    'putOuts', \n    'errors', \n    'chances', \n    'target1_mean',\n    'target1_median',\n    'target1_std',\n    'target1_min',\n    'target1_max',\n    'target1_prob',\n    'target2_mean',\n    'target2_median',\n    'target2_std',\n    'target2_min',\n    'target2_max',\n    'target2_prob',\n    'target3_mean',\n    'target3_median',\n    'target3_std',\n    'target3_min',\n    'target3_max',\n    'target3_prob',\n    'target4_mean',\n    'target4_median',\n    'target4_std',\n    'target4_min',\n    'target4_max',\n    'target4_prob',\n    'awardId_count',\n    'playernumberOfFollowers',               \n    'teamnumberOfFollowers',\n    'label_leagueId',\n    'label_divisionId',\n    'wins', \n    'losses', \n    'lastTenWins',\n    'lastTenLosses'\n]","metadata":{"execution":{"iopub.status.busy":"2021-08-07T17:00:55.924348Z","iopub.execute_input":"2021-08-07T17:00:55.924646Z","iopub.status.idle":"2021-08-07T17:00:55.941463Z","shell.execute_reply.started":"2021-08-07T17:00:55.924620Z","shell.execute_reply":"2021-08-07T17:00:55.940650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"players = pd.read_csv(BASE_DIR / 'players.csv', usecols = players_cols)\nplayers = reduce_mem_usage(players)\n\n\nteams = pd.read_csv(BASE_DIR / 'teams.csv', usecols = teams_cols)\nteams = teams.rename(columns = {'id':'teamId'})\nteams = reduce_mem_usage(teams)\n\n\nrosters = pd.read_csv(TRAIN_DIR / 'rosters_train.csv', usecols = rosters_cols)\nrosters = reduce_mem_usage(rosters)\n\n\ntargets = pd.read_csv(TRAIN_DIR / 'nextDayPlayerEngagement_train.csv', usecols = targets_cols)\ntargets = reduce_mem_usage(targets)\n\n\nscores = pd.read_csv(TRAIN_DIR / 'playerBoxScores_train.csv', usecols = scores_cols)\nscores = scores.groupby(['playerId', 'date']).sum().reset_index()\nscores = reduce_mem_usage(scores)\n\n\nawards = pd.read_csv(TRAIN_DIR / 'awards_train.csv', usecols = awards_cols)\n# awards = awards.groupby(['playerId', 'date']).count().reset_index()\n\n\nawards_count = awards[['playerId', 'awardId']].groupby('playerId').count().reset_index()\nawards_count = awards_count.rename(columns = {'awardId':'awardId_count'})\nawards_count = reduce_mem_usage(awards_count)\n\n\nplayerTwitterFollowers = pd.read_csv(TRAIN_DIR / 'playerTwitterFollowers_train.csv', usecols = playerTwitterFollowers_cols)\nplayerTwitterFollowers = playerTwitterFollowers.groupby('playerId').sum().reset_index()\nplayerTwitterFollowers = playerTwitterFollowers.rename(columns = {'numberOfFollowers':'playernumberOfFollowers'})\nplayerTwitterFollowers = reduce_mem_usage(playerTwitterFollowers)\n\n\nteamTwitterFollowers = pd.read_csv(TRAIN_DIR / 'teamTwitterFollowers_train.csv', usecols = teamTwitterFollowers_cols)\nteamTwitterFollowers = teamTwitterFollowers.groupby('teamId').sum().reset_index()\nteamTwitterFollowers = teamTwitterFollowers.rename(columns = {'numberOfFollowers':'teamnumberOfFollowers'})\nteamTwitterFollowers = reduce_mem_usage(teamTwitterFollowers)\n\n\nstandings = pd.read_csv(TRAIN_DIR / 'standings_train.csv', usecols = standings_cols)\nstandings = reduce_mem_usage(standings)\n\ngc.collect()\n","metadata":{"execution":{"iopub.status.busy":"2021-08-07T17:00:55.942972Z","iopub.execute_input":"2021-08-07T17:00:55.943360Z","iopub.status.idle":"2021-08-07T17:01:05.557610Z","shell.execute_reply.started":"2021-08-07T17:00:55.943327Z","shell.execute_reply":"2021-08-07T17:01:05.556769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"player_target_stats = pd.read_csv(\"../input/player-target-stats/player_target_stats.csv\")\ndata_names=player_target_stats.columns.values.tolist()\ndata_names","metadata":{"execution":{"iopub.status.busy":"2021-08-07T17:01:05.559062Z","iopub.execute_input":"2021-08-07T17:01:05.559431Z","iopub.status.idle":"2021-08-07T17:01:05.595134Z","shell.execute_reply.started":"2021-08-07T17:01:05.559393Z","shell.execute_reply":"2021-08-07T17:01:05.594410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# creat dataset\n\ntrain = targets.copy()[targets_cols]\n\nprint(targets[targets_cols].shape)\n\ntrain = train.merge(\n    players, \n    on=['playerId'], \n    how='left'\n)\ngc.collect()\n\nprint(train.shape, 'after_players')\nprint('--------------------------------------')\n\ntrain = train.merge(\n    rosters, \n    on=['playerId', 'date'], \n    how='left'\n)\ngc.collect()\n\nprint(train.shape, 'after_rosters')\nprint('--------------------------------------')\n\ntrain = train.merge(\n    scores, \n    on=['playerId', 'date'], \n    how='left'\n)\ngc.collect()\n\nprint(train.shape, 'after_scores')\nprint('--------------------------------------')\n\ntrain = train.merge(\n    player_target_stats, \n    how='inner', \n    on= \"playerId\",\n)\ngc.collect()\n\nprint(train.shape, 'after_player_target_stats')\n\n\nprint('--------------------------------------')\n\ntrain = train.merge(\n    teams,\n    on = 'teamId',\n    how='left'\n)\n# del rosters\ngc.collect()\n\nprint(train.shape, 'after_teams')\nprint('--------------------------------------')\n\ntrain = train.merge(\n    awards_count,\n    on = 'playerId',\n    how = 'left'\n)\n\ntrain['awardId_count'] = train['awardId_count'].fillna(0)\n\nprint(train.shape, 'after_awards_count')\nprint('--------------------------------------')\n\ntrain = train.merge(\n    playerTwitterFollowers, \n    how = 'left', \n    on = 'playerId'\n)\ngc.collect()\n\nprint(train.shape, 'after_playerTwitter')\nprint('--------------------------------------')\n\n\ntrain = train.merge(\n    teamTwitterFollowers, \n    how = 'left', \n    on = 'teamId'\n)\ngc.collect()\n\nprint(train.shape, 'after_taemTwitter')\nprint('--------------------------------------')\n\ntrain = train.merge(\n    standings, \n    how = 'left', \n    on = ['teamId', 'date']\n)\ngc.collect()\n\nprint(train.shape, 'after_standings')\nprint('--------------------------------------')\n\n\n# label encoding\nplayer2num = {c: i for i, c in enumerate(train['playerId'].unique())}\nposition2num = {c: i for i, c in enumerate(train['primaryPositionName'].unique())}\nteamid2num = {c: i for i, c in enumerate(train['teamId'].unique())}\nstatus2num = {c: i for i, c in enumerate(train['status'].unique())}\nleagueId2num = {c: i for i, c in enumerate(train['leagueId'].unique())}\ndivisionId2num = {c: i for i, c in enumerate(train['divisionId'].unique())}\n\n\ntrain['label_playerId'] = train['playerId'].map(player2num)\ntrain['label_primaryPositionName'] = train['primaryPositionName'].map(position2num)\ntrain['label_teamId'] = train['teamId'].map(teamid2num)\ntrain['label_status'] = train['status'].map(status2num)\ntrain['label_leagueId'] = train['leagueId'].map(leagueId2num)\ntrain['label_divisionId'] = train['divisionId'].map(divisionId2num)","metadata":{"execution":{"iopub.status.busy":"2021-08-07T17:01:05.596370Z","iopub.execute_input":"2021-08-07T17:01:05.596744Z","iopub.status.idle":"2021-08-07T17:01:25.515535Z","shell.execute_reply.started":"2021-08-07T17:01:05.596707Z","shell.execute_reply":"2021-08-07T17:01:25.514658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2021-08-07T17:01:25.517743Z","iopub.execute_input":"2021-08-07T17:01:25.518077Z","iopub.status.idle":"2021-08-07T17:01:25.540998Z","shell.execute_reply.started":"2021-08-07T17:01:25.518051Z","shell.execute_reply":"2021-08-07T17:01:25.539936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train.shape)\ntrain.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-08-07T17:01:25.542853Z","iopub.execute_input":"2021-08-07T17:01:25.543468Z","iopub.status.idle":"2021-08-07T17:01:27.300977Z","shell.execute_reply.started":"2021-08-07T17:01:25.543422Z","shell.execute_reply":"2021-08-07T17:01:27.300094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X = train[feature_cols]\ntrain_y = train[['target1', 'target2', 'target3', 'target4']]\n\n_index = (train['date'] < 20210401)\nx_train = train_X.loc[_index].reset_index(drop=True)\ny_train = train_y.loc[_index].reset_index(drop=True)\nx_valid = train_X.loc[~_index].reset_index(drop=True)\ny_valid = train_y.loc[~_index].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2021-08-07T17:01:27.302395Z","iopub.execute_input":"2021-08-07T17:01:27.302929Z","iopub.status.idle":"2021-08-07T17:01:30.638577Z","shell.execute_reply.started":"2021-08-07T17:01:27.302890Z","shell.execute_reply":"2021-08-07T17:01:30.637697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X = train[feature_cols]\ntrain_y = train[['target1', 'target2', 'target3', 'target4']]\n\n_index = (train['date'] < 20210401)\nx_train = train_X.loc[_index].reset_index(drop=True)\ny_train = train_y.loc[_index].reset_index(drop=True)\nx_valid = train_X.loc[~_index].reset_index(drop=True)\ny_valid = train_y.loc[~_index].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2021-08-07T17:01:30.639841Z","iopub.execute_input":"2021-08-07T17:01:30.640345Z","iopub.status.idle":"2021-08-07T17:01:33.166162Z","shell.execute_reply.started":"2021-08-07T17:01:30.640307Z","shell.execute_reply":"2021-08-07T17:01:33.165232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit_lgbm(x_train, y_train, x_valid, y_valid, params: dict=None, verbose=100):\n    oof_pred = np.zeros(len(y_valid), dtype=np.float32)\n    model = lgbm.LGBMRegressor(**params)\n    model.fit(x_train, y_train, \n        eval_set=[(x_valid, y_valid)],  \n        early_stopping_rounds=verbose, \n        verbose=verbose)\n    oof_pred = model.predict(x_valid)\n    score = mean_absolute_error(oof_pred, y_valid)\n    print('mae:', score)\n    return oof_pred, model, score\n\n\"\"\"\n# training lightgbm before param\nparams = {\n 'objective':'mae',\n 'reg_alpha': 0.1,\n 'reg_lambda': 0.1, \n 'n_estimators': 100000,\n 'learning_rate': 0.1,\n 'random_state': 42,\n}\n\"\"\"\n\nparams1 = {'objective': 'mae', 'metric': 'l1', 'feature_pre_filter': False, 'lambda_l1': 3.485822021802935e-08, 'lambda_l2': 4.230468117096112e-06, 'num_leaves': 253, 'feature_fraction': 0.8, 'bagging_fraction': 0.550250698524785, 'bagging_freq': 1, 'min_child_samples': 20, 'num_iterations': 10000, 'early_stopping_round': 100}\n\nparams2 = {'objective': 'mae', 'metric': 'l1', 'feature_pre_filter': False, 'lambda_l1': 3.731605225849285, 'lambda_l2': 0.02803980626777797, 'num_leaves': 8, 'feature_fraction': 0.5, 'bagging_fraction': 0.5262728428461787, 'bagging_freq': 3, 'min_child_samples': 20, 'num_iterations': 10000, 'early_stopping_round': 100}\n\nparams3 = {'objective': 'mae', 'metric': 'l1', 'feature_pre_filter': False, 'lambda_l1': 7.654830305013684, 'lambda_l2': 4.14748542765967e-07, 'num_leaves': 252, 'feature_fraction': 0.7200000000000001, 'bagging_fraction': 1.0, 'bagging_freq': 0, 'min_child_samples': 20, 'num_iterations': 10000, 'early_stopping_round': 100}\n\nparams4 = {'objective': 'mae', 'metric': 'l1', 'feature_pre_filter': False, 'lambda_l1': 9.486880706514734e-08, 'lambda_l2': 0.005143767850872896, 'num_leaves': 246, 'feature_fraction': 0.5479999999999999, 'bagging_fraction': 0.5238463354446826, 'bagging_freq': 5, 'min_child_samples': 20, 'num_iterations': 10000, 'early_stopping_round': 100}\n\n\n\noof1, model1, score1 = fit_lgbm(\n    x_train, y_train['target1'],\n    x_valid, y_valid['target1'],\n    params1\n)\noof2, model2, score2 = fit_lgbm(\n    x_train, y_train['target2'],\n    x_valid, y_valid['target2'],\n    params2\n)\noof3, model3, score3 = fit_lgbm(\n    x_train, y_train['target3'],\n    x_valid, y_valid['target3'],\n    params3\n)\noof4, model4, score4 = fit_lgbm(\n    x_train, y_train['target4'],\n    x_valid, y_valid['target4'],\n    params4\n)\n\nscore = (score1+score2+score3+score4) / 4\nprint(f'score: {score}')\n","metadata":{"execution":{"iopub.status.busy":"2021-08-07T17:01:33.167578Z","iopub.execute_input":"2021-08-07T17:01:33.168060Z","iopub.status.idle":"2021-08-07T17:12:17.710315Z","shell.execute_reply.started":"2021-08-07T17:01:33.168022Z","shell.execute_reply":"2021-08-07T17:12:17.709459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rosters_cols.remove('date')\nscores_cols.remove('date')\nstandings_cols = [\n   'teamId', \n   'wins', \n   'losses', \n   'lastTenWins',\n   'lastTenLosses'\n]\n\nnull = np.nan\ntrue = True\nfalse = False\n\nenv_subha = mlb.make_env() # initialize the environment\n#env_subha = riiideducation.make_env() # initialize the environment\niter_test = env_subha.iter_test() # iterator which loops over each date in test set\n#iter_test = 1000\n\nfor (test_df, sample_prediction_df) in iter_test: # make predictions here\n    \n    sample_prediction_df = sample_prediction_df.reset_index(drop=True)\n    \n    # creat dataset\n    sample_prediction_df['playerId'] = sample_prediction_df['date_playerId']\\\n                                        .map(lambda x: int(x.split('_')[1]))\n    # Dealing with missing values\n    if test_df['rosters'].iloc[0] == test_df['rosters'].iloc[0]:\n        test_rosters = pd.DataFrame(eval(test_df['rosters'].iloc[0]))\n    else:\n        test_rosters = pd.DataFrame({'playerId': sample_prediction_df['playerId']})\n        for col in rosters.columns:\n            if col == 'playerId': continue\n            test_rosters[col] = np.nan\n            \n    if test_df['playerBoxScores'].iloc[0] == test_df['playerBoxScores'].iloc[0]:\n        test_scores = pd.DataFrame(eval(test_df['playerBoxScores'].iloc[0]))\n    else:\n        test_scores = pd.DataFrame({'playerId': sample_prediction_df['playerId']})\n        for col in scores.columns:\n            if col == 'playerId': continue\n            test_scores[col] = np.nan\n            \n    if test_df['standings'].iloc[0] == test_df['standings'].iloc[0]:\n        test_standings = pd.DataFrame(eval(test_df['standings'].iloc[0]))\n    else:\n        test_standings = pd.DataFrame({'playerId': sample_prediction_df['playerId']})\n        for col in standings.columns:\n            if col == 'playerId': continue\n            test_scores[col] = np.nan\n            \n            \n    test_scores = test_scores.groupby('playerId').sum().reset_index()\n    test = sample_prediction_df[['playerId']].copy()\n    test = test.merge(players[players_cols], on='playerId', how='left')\n    test = test.merge(test_rosters[rosters_cols], on='playerId', how='left')\n    test = test.merge(test_scores[scores_cols], on='playerId', how='left')\n    test = test.merge(player_target_stats, how='inner', left_on=[\"playerId\"],right_on=[\"playerId\"])\n    test = test.merge(awards_count, how = 'left', on = 'playerId')\n    test = test.merge(teams, how = 'left', on = 'teamId')\n    test['awardId_count'] = test['awardId_count'].fillna(0)\n    test = test.merge(playerTwitterFollowers, how = 'left', on ='playerId')\n    test = test.merge(teamTwitterFollowers, how = 'left', on ='teamId')\n    test = test.merge(test_standings[standings_cols], how = 'left', on = 'teamId')\n\n    \n\n    test['label_playerId'] = test['playerId'].map(player2num)\n    test['label_primaryPositionName'] = test['primaryPositionName'].map(position2num)\n    test['label_teamId'] = test['teamId'].map(teamid2num)\n    test['label_status'] = test['status'].map(status2num)\n    test['label_leagueId'] = test['leagueId'].map(leagueId2num)\n    test['label_divisionId'] = test['divisionId'].map(divisionId2num)\n    \n    test_X = test[feature_cols]\n    \n    # predict\n    pred1 = model1.predict(test_X)\n    pred2 = model2.predict(test_X)\n    pred3 = model3.predict(test_X)\n    pred4 = model4.predict(test_X)\n    \n    # merge submission\n    sample_prediction_df['target1'] = np.clip(pred1, 0, 100)\n    sample_prediction_df['target2'] = np.clip(pred2, 0, 100)\n    sample_prediction_df['target3'] = np.clip(pred3, 0, 100)\n    sample_prediction_df['target4'] = np.clip(pred4, 0, 100)\n    sample_prediction_df = sample_prediction_df.fillna(0.)\n    del sample_prediction_df['playerId']\n    \n    env_subha.predict(sample_prediction_df)\n","metadata":{"execution":{"iopub.status.busy":"2021-08-07T17:12:17.714187Z","iopub.execute_input":"2021-08-07T17:12:17.716195Z","iopub.status.idle":"2021-08-07T17:12:21.076020Z","shell.execute_reply.started":"2021-08-07T17:12:17.716157Z","shell.execute_reply":"2021-08-07T17:12:21.075193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_prediction_df","metadata":{"execution":{"iopub.status.busy":"2021-08-07T17:12:21.077223Z","iopub.execute_input":"2021-08-07T17:12:21.077593Z","iopub.status.idle":"2021-08-07T17:12:21.100836Z","shell.execute_reply.started":"2021-08-07T17:12:21.077556Z","shell.execute_reply":"2021-08-07T17:12:21.099914Z"},"trusted":true},"execution_count":null,"outputs":[]}]}