{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Training","metadata":{"_uuid":"ea4d85e1-21d7-4e7a-a8e7-36e5f53f4612","_cell_guid":"8d708767-e50d-4684-82a1-790feb5c0c1f","execution":{"iopub.status.busy":"2021-06-16T09:14:33.869464Z","iopub.execute_input":"2021-06-16T09:14:33.869905Z","iopub.status.idle":"2021-06-16T09:14:33.874766Z","shell.execute_reply.started":"2021-06-16T09:14:33.869879Z","shell.execute_reply":"2021-06-16T09:14:33.873097Z"},"trusted":true}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom sklearn.metrics import mean_absolute_error\nfrom datetime import timedelta\nfrom functools import reduce\nfrom tqdm import tqdm\nimport lightgbm as lgbm\nfrom catboost import CatBoostRegressor\nimport mlb\nimport os\nimport pickle\nimport gc\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import TensorDataset, DataLoader,Dataset\nfrom torch.autograd import Variable\nimport torch.optim as optim\n\nimport random\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"2139878b-da24-41e3-bb59-76b60a1e16ef","_cell_guid":"3181642b-6cb0-424d-b5be-a0e6a5cb0457","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T18:55:24.090154Z","iopub.execute_input":"2021-07-31T18:55:24.090553Z","iopub.status.idle":"2021-07-31T18:55:28.374341Z","shell.execute_reply.started":"2021-07-31T18:55:24.090470Z","shell.execute_reply":"2021-07-31T18:55:28.373475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_random_seed(random_seed):\n    random.seed(random_seed)\n    np.random.seed(random_seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(random_seed)\n\n    torch.manual_seed(random_seed)\n    torch.cuda.manual_seed(random_seed)\n    torch.cuda.manual_seed_all(random_seed)\n\n    torch.backends.cudnn.deterministic = True","metadata":{"execution":{"iopub.status.busy":"2021-07-31T18:55:28.375701Z","iopub.execute_input":"2021-07-31T18:55:28.376016Z","iopub.status.idle":"2021-07-31T18:55:28.384047Z","shell.execute_reply.started":"2021-07-31T18:55:28.375981Z","shell.execute_reply":"2021-07-31T18:55:28.383152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_DIR = Path('../input/mlb-player-digital-engagement-forecasting')\nTRAIN_DIR = Path('../input/mlb-df-files')","metadata":{"_uuid":"3540a2ce-95e1-416f-9892-bcfa92cf6047","_cell_guid":"5f9dc680-5158-4bf2-856f-d43b6aa620de","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T18:55:28.386243Z","iopub.execute_input":"2021-07-31T18:55:28.386522Z","iopub.status.idle":"2021-07-31T18:55:28.391048Z","shell.execute_reply.started":"2021-07-31T18:55:28.386496Z","shell.execute_reply":"2021-07-31T18:55:28.390082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"players = pd.read_csv(BASE_DIR / 'players.csv')\n\nrosters = pd.read_pickle(TRAIN_DIR / 'rosters_train.pkl')\ntargets = pd.read_pickle(TRAIN_DIR / 'nextDayPlayerEngagement_train.pkl')\nscores = pd.read_pickle(TRAIN_DIR / 'playerBoxScores_train.pkl')\nscores = scores.groupby(['playerId', 'date']).sum().reset_index()","metadata":{"_uuid":"38e056cf-cf8f-45f0-9ad4-cda6d11917a2","_cell_guid":"e67cda79-ae71-474a-861b-2b437b94a0a8","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T18:55:29.473303Z","iopub.execute_input":"2021-07-31T18:55:29.473694Z","iopub.status.idle":"2021-07-31T18:55:33.488210Z","shell.execute_reply.started":"2021-07-31T18:55:29.473664Z","shell.execute_reply":"2021-07-31T18:55:33.487414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets_cols = ['playerId', 'target1', 'target2', 'target3', 'target4', 'date']\nplayers_cols = ['playerId', 'primaryPositionName']\nrosters_cols = ['playerId', 'teamId', 'status', 'date']\nscores_cols = ['playerId', 'battingOrder', 'gamesPlayedBatting', 'flyOuts',\n       'groundOuts', 'runsScored', 'doubles', 'triples', 'homeRuns',\n       'strikeOuts', 'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch',\n       'atBats', 'caughtStealing', 'stolenBases', 'groundIntoDoublePlay',\n       'groundIntoTriplePlay', 'plateAppearances', 'totalBases', 'rbi',\n       'leftOnBase', 'sacBunts', 'sacFlies', 'catchersInterference',\n       'pickoffs', 'gamesPlayedPitching', 'gamesStartedPitching',\n       'completeGamesPitching', 'shutoutsPitching', 'winsPitching',\n       'lossesPitching', 'flyOutsPitching', 'airOutsPitching',\n       'groundOutsPitching', 'runsPitching', 'doublesPitching',\n       'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching',\n       'baseOnBallsPitching', 'intentionalWalksPitching', 'hitsPitching',\n       'hitByPitchPitching', 'atBatsPitching', 'caughtStealingPitching',\n       'stolenBasesPitching', 'inningsPitched', 'saveOpportunities',\n       'earnedRuns', 'battersFaced', 'outsPitching', 'pitchesThrown', 'balls',\n       'strikes', 'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching',\n       'rbiPitching', 'gamesFinishedPitching', 'inheritedRunners',\n       'inheritedRunnersScored', 'catchersInterferencePitching',\n       'sacBuntsPitching', 'sacFliesPitching', 'saves', 'holds', 'blownSaves',\n       'assists', 'putOuts', 'errors', 'chances', 'date']\n\nfeature_cols = ['label_playerId', 'label_primaryPositionName', 'label_teamId',\n       'label_status', 'battingOrder', 'gamesPlayedBatting', 'flyOuts',\n       'groundOuts', 'runsScored', 'doubles', 'triples', 'homeRuns',\n       'strikeOuts', 'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch',\n       'atBats', 'caughtStealing', 'stolenBases', 'groundIntoDoublePlay',\n       'groundIntoTriplePlay', 'plateAppearances', 'totalBases', 'rbi',\n       'leftOnBase', 'sacBunts', 'sacFlies', 'catchersInterference',\n       'pickoffs', 'gamesPlayedPitching', 'gamesStartedPitching',\n       'completeGamesPitching', 'shutoutsPitching', 'winsPitching',\n       'lossesPitching', 'flyOutsPitching', 'airOutsPitching',\n       'groundOutsPitching', 'runsPitching', 'doublesPitching',\n       'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching',\n       'baseOnBallsPitching', 'intentionalWalksPitching', 'hitsPitching',\n       'hitByPitchPitching', 'atBatsPitching', 'caughtStealingPitching',\n       'stolenBasesPitching', 'inningsPitched', 'saveOpportunities',\n       'earnedRuns', 'battersFaced', 'outsPitching', 'pitchesThrown', 'balls',\n       'strikes', 'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching',\n       'rbiPitching', 'gamesFinishedPitching', 'inheritedRunners',\n       'inheritedRunnersScored', 'catchersInterferencePitching',\n       'sacBuntsPitching', 'sacFliesPitching', 'saves', 'holds', 'blownSaves',\n       'assists', 'putOuts', 'errors', 'chances', 'target1_mean', 'target1_median',\n        'target1_min','target1_max', 'target1_prob', 'target2_mean',\n                'target2_median','target2_min','target2_max','target2_prob',\n        'target3_mean','target3_median','target3_min',\n        'target3_max','target3_prob','target4_mean','target4_median',\n        'target4_min','target4_max','target4_prob']","metadata":{"_uuid":"046564c8-2d25-4540-9a94-9e629e263a22","_cell_guid":"39e17ba0-96e5-438c-94bd-6c184905c35f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T18:55:33.490345Z","iopub.execute_input":"2021-07-31T18:55:33.490939Z","iopub.status.idle":"2021-07-31T18:55:33.502843Z","shell.execute_reply.started":"2021-07-31T18:55:33.490900Z","shell.execute_reply":"2021-07-31T18:55:33.502081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"player_target_stats  = pd.read_pickle(TRAIN_DIR / 'player_target_stats.pkl')\nprint(player_target_stats.columns)","metadata":{"execution":{"iopub.status.busy":"2021-07-31T18:55:33.504378Z","iopub.execute_input":"2021-07-31T18:55:33.504794Z","iopub.status.idle":"2021-07-31T18:55:33.541486Z","shell.execute_reply.started":"2021-07-31T18:55:33.504754Z","shell.execute_reply":"2021-07-31T18:55:33.540648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets['year'] = pd.DatetimeIndex(targets['engagementMetricsDate']).year","metadata":{"execution":{"iopub.status.busy":"2021-07-31T18:55:35.221346Z","iopub.execute_input":"2021-07-31T18:55:35.221698Z","iopub.status.idle":"2021-07-31T18:55:36.102995Z","shell.execute_reply.started":"2021-07-31T18:55:35.221668Z","shell.execute_reply":"2021-07-31T18:55:36.102150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"player_target_stats['year'] = player_target_stats['year'].astype('int')","metadata":{"execution":{"iopub.status.busy":"2021-07-31T18:55:37.010158Z","iopub.execute_input":"2021-07-31T18:55:37.010504Z","iopub.status.idle":"2021-07-31T18:55:37.015583Z","shell.execute_reply.started":"2021-07-31T18:55:37.010472Z","shell.execute_reply":"2021-07-31T18:55:37.014648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"player_target_stats['year'].head()","metadata":{"execution":{"iopub.status.busy":"2021-07-31T18:55:38.576231Z","iopub.execute_input":"2021-07-31T18:55:38.576603Z","iopub.status.idle":"2021-07-31T18:55:38.584015Z","shell.execute_reply.started":"2021-07-31T18:55:38.576559Z","shell.execute_reply":"2021-07-31T18:55:38.583144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# creat dataset\ntrain = targets[targets_cols+['year']].merge(players[players_cols], on=['playerId'], how='left')\ntrain = train.merge(rosters[rosters_cols], on=['playerId', 'date'], how='left')\ntrain = train.merge(scores[scores_cols], on=['playerId', 'date'], how='left')\ntrain = train.merge(player_target_stats, on=['playerId', 'year'], how='left')\n\n\n# label encoding\nplayer2num = {c: i for i, c in enumerate(train['playerId'].unique())}\nposition2num = {c: i for i, c in enumerate(train['primaryPositionName'].unique())}\nteamid2num = {c: i for i, c in enumerate(train['teamId'].unique())}\nstatus2num = {c: i for i, c in enumerate(train['status'].unique())}\ntrain['label_playerId'] = train['playerId'].map(player2num)\ntrain['label_primaryPositionName'] = train['primaryPositionName'].map(position2num)\ntrain['label_teamId'] = train['teamId'].map(teamid2num)\ntrain['label_status'] = train['status'].map(status2num)","metadata":{"_uuid":"6cb3c4b2-a69b-4ce5-9d70-6633354c11ea","_cell_guid":"69624101-e45d-442a-bbb4-fb5514840e21","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T18:55:40.061627Z","iopub.execute_input":"2021-07-31T18:55:40.061951Z","iopub.status.idle":"2021-07-31T18:55:46.182858Z","shell.execute_reply.started":"2021-07-31T18:55:40.061919Z","shell.execute_reply":"2021-07-31T18:55:46.181867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_lag_features(dt, target, lags_list):\n    lag_columns = []\n    tmp_columns = []\n    for lag, window in tqdm(lags_list, leave=False, desc='Lag features'):\n\n        if window == 1:\n            lag_col = f\"{target}_lag_{lag}\"\n            print(lag_col, end=', ')\n            dt[lag_col] = dt[[\"playerId\", target]].groupby(\"playerId\")[target].shift(lag).astype('float32')\n            lag_columns.append(lag_col)\n            \n        elif window > 1:\n            lag_col = f\"{target}_lag_{lag}\"\n            if lag_col not in dt.columns:\n                dt[lag_col] = dt[[\"playerId\", target]].groupby(\"playerId\")[target].shift(lag).astype('float32')\n                tmp_columns.append(lag_col)\n\n            rmean_col = f\"{target}_rmean_{lag}_{window}\"\n            lag_columns.append(rmean_col)\n            print(rmean_col, end=', ')\n            dt[rmean_col] = dt[[\"playerId\", lag_col]].groupby(\"playerId\")[lag_col]. \\\n                transform(lambda x: x.rolling(window).mean()).astype('float32')\n            \n            rstd_col = f\"{target}_rstd_{lag}_{window}\"\n            lag_columns.append(rstd_col)\n            print(rstd_col, end=', ')\n            dt[rstd_col] = dt[[\"playerId\", lag_col]].groupby(\"playerId\")[lag_col]. \\\n                transform(lambda x: x.rolling(window).std()).astype('float32')\n            \n            rmin_col = f\"{target}_rmin_{lag}_{window}\"\n            lag_columns.append(rmin_col)\n            print(rmin_col, end=', ')\n            dt[rmin_col] = dt[[\"playerId\", lag_col]].groupby(\"playerId\")[lag_col]. \\\n                transform(lambda x: x.rolling(window).min()).astype('float32')\n            \n            rmax_col = f\"{target}_rmax_{lag}_{window}\"\n            lag_columns.append(rmax_col)\n            print(rmax_col, end=', ')\n            dt[rmax_col] = dt[[\"playerId\", lag_col]].groupby(\"playerId\")[lag_col]. \\\n                transform(lambda x: x.rolling(window).max()).astype('float32')\n\n    print('dropping tmp cols:', tmp_columns)\n    dt.drop(tmp_columns, axis=1, inplace=True)\n    return dt, lag_columns","metadata":{"execution":{"iopub.status.busy":"2021-07-31T18:55:46.184229Z","iopub.execute_input":"2021-07-31T18:55:46.184559Z","iopub.status.idle":"2021-07-31T18:55:46.198305Z","shell.execute_reply.started":"2021-07-31T18:55:46.184524Z","shell.execute_reply":"2021-07-31T18:55:46.197140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lags_list = [[1, 30], [7, 30], [14, 30]]\n# lag_cols = []\n# target_cols = ['target1', 'target2', 'target3', 'target4']\n# for target in target_cols:\n#     train, lag_columns = create_lag_features(train, target, lags_list)\n#     lag_cols = lag_columns+lag_cols","metadata":{"execution":{"iopub.status.busy":"2021-07-27T07:17:50.596483Z","iopub.execute_input":"2021-07-27T07:17:50.596867Z","iopub.status.idle":"2021-07-27T07:17:50.611497Z","shell.execute_reply.started":"2021-07-27T07:17:50.59683Z","shell.execute_reply":"2021-07-27T07:17:50.610727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-31T18:55:46.200501Z","iopub.execute_input":"2021-07-31T18:55:46.201140Z","iopub.status.idle":"2021-07-31T18:55:46.212194Z","shell.execute_reply.started":"2021-07-31T18:55:46.201097Z","shell.execute_reply":"2021-07-31T18:55:46.210927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X = train[feature_cols]\ntrain_y = train[['target1', 'target2', 'target3', 'target4']]\n\n_index = (train['date'] < 20210401)\nx_train1 = train_X.loc[_index].reset_index(drop=True)\ny_train1 = train_y.loc[_index].reset_index(drop=True)\nx_valid1 = train_X.loc[~_index].reset_index(drop=True)\ny_valid1 = train_y.loc[~_index].reset_index(drop=True)","metadata":{"_uuid":"ddc79e19-ca6c-40d2-bb71-cf5df58549ce","_cell_guid":"ffc2b454-9e48-44f8-b4fa-d2d87ea5e89d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T18:55:47.577546Z","iopub.execute_input":"2021-07-31T18:55:47.577937Z","iopub.status.idle":"2021-07-31T18:55:52.081523Z","shell.execute_reply.started":"2021-07-31T18:55:47.577902Z","shell.execute_reply":"2021-07-31T18:55:52.080456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_X, train_y\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-07-31T18:55:52.082912Z","iopub.execute_input":"2021-07-31T18:55:52.083232Z","iopub.status.idle":"2021-07-31T18:55:52.220218Z","shell.execute_reply.started":"2021-07-31T18:55:52.083195Z","shell.execute_reply":"2021-07-31T18:55:52.219341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit_lgbm(x_train, y_train, x_valid, y_valid, target, model_path, params: dict=None, verbose=100):\n    oof_pred = np.zeros(len(y_valid), dtype=np.float32)\n    \n    if os.path.isfile(f'{model_path}/model_lgb_{target}.pkl'):\n        with open(f'{model_path}/model_lgb_{target}.pkl', 'rb') as fin:\n            model = pickle.load(fin)\n    else:\n    \n        model = lgbm.LGBMRegressor(**params)\n        model.fit(x_train, y_train, \n            eval_set=[(x_valid, y_valid)],  \n            early_stopping_rounds=verbose, \n            verbose=verbose)\n        \n        with open(f'model_lgb_{target}.pkl', 'wb') as handle:\n            pickle.dump(model, handle, protocol=pickle.HIGHEST_PROTOCOL)\n    \n    oof_pred = model.predict(x_valid)\n    score = mean_absolute_error(oof_pred, y_valid)\n    print('mae:', score)\n    return oof_pred, model, score\n\n\n# training lightgbm\n\nparams = {\n 'objective':'mae',\n 'reg_alpha': 0.1,\n 'reg_lambda': 0.1, \n 'n_estimators': 2000,\n 'learning_rate': 0.02,\n 'random_state': 42,\n \"num_leaves\": 100\n}\n\nmodel_path = '../input/mlb-final-models'\n\nlgb_models = {20210301: [], 20210401: [], 20210501:[], 20210601:[], 20210701:[]}\n\nfor cut_date in [20210301, 20210401, 20210501, 20210601, 20210701]:\n    oof1, model1, score1 = fit_lgbm(\n        x_train1, y_train1['target1'],\n        x_valid1, y_valid1['target1'],\n        f'target1_{cut_date}', model_path, params\n     )\n    \n    lgb_models[cut_date].append(model1)\n    \n    oof2, model2, score2 = fit_lgbm(\n        x_train1, y_train1['target2'],\n        x_valid1, y_valid1['target2'],\n        f'target2_{cut_date}', model_path, params\n    )\n    \n    lgb_models[cut_date].append(model2)\n    \n    oof3, model3, score3 = fit_lgbm(\n        x_train1, y_train1['target3'],\n        x_valid1, y_valid1['target3'],\n       f'target3_{cut_date}', model_path, params\n    )\n    \n    lgb_models[cut_date].append(model3)\n    \n    oof4, model4, score4 = fit_lgbm(\n        x_train1, y_train1['target4'],\n        x_valid1, y_valid1['target4'],\n        f'target4_{cut_date}', model_path, params\n    )\n    \n    lgb_models[cut_date].append(model4)\n    \n    score = (score1+score2+score3+score4) / 4\n    print(f'score: {score}')","metadata":{"_uuid":"c616b4b9-961c-4fcc-9643-391e8225a43c","_cell_guid":"50c7c4dd-9076-4e0a-98bc-bb2e56f0ecb4","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T18:55:56.001904Z","iopub.execute_input":"2021-07-31T18:55:56.002246Z","iopub.status.idle":"2021-07-31T18:58:43.701835Z","shell.execute_reply.started":"2021-07-31T18:55:56.002214Z","shell.execute_reply":"2021-07-31T18:58:43.700691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del oof1, oof2, oof3, oof4\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-07-31T18:58:43.703721Z","iopub.execute_input":"2021-07-31T18:58:43.704362Z","iopub.status.idle":"2021-07-31T18:58:43.838612Z","shell.execute_reply.started":"2021-07-31T18:58:43.704313Z","shell.execute_reply":"2021-07-31T18:58:43.837590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Neural Net","metadata":{}},{"cell_type":"code","source":"def make_2Dinput(dt, cont_cols, cat_cols):\n    input = {\"rnn\": dt[cont_cols].to_numpy()}\n    for i, v in enumerate(cat_cols):\n        input[v] = dt[[v]].to_numpy()\n    return input\n\nclass MLBLoader:\n\n    def __init__(self, X, y, shuffle=True, batch_size=1000, cat_cols=[]):\n        self.X_cont = X[\"rnn\"]\n        try:\n            self.X_cat = np.concatenate([X[k] for k in cat_cols], axis=1)\n        except:\n            self.X_cat = np.concatenate([np.expand_dims(X[k], axis=1) for k in cat_cols], axis=1)\n        self.y = y\n\n        self.shuffle = shuffle\n        self.batch_size = batch_size\n        self.n_conts = self.X_cont.shape[1]\n        self.len = self.X_cont.shape[0]\n        n_batches, remainder = divmod(self.len, self.batch_size)\n\n        if remainder > 0:\n            n_batches += 1\n        self.n_batches = n_batches\n        self.remainder = remainder  # for debugging\n\n        self.idxes = np.array([i for i in range(self.len)])\n\n    def __iter__(self):\n        self.i = 0\n        if self.shuffle:\n            ridxes = self.idxes\n            np.random.shuffle(ridxes)\n            self.X_cat = self.X_cat[[ridxes]]\n            self.X_cont = self.X_cont[[ridxes]]\n            if self.y is not None:\n                self.y = self.y[[ridxes]]\n\n        return self\n\n    def __next__(self):\n        if self.i >= self.len:\n            raise StopIteration\n\n        if self.y is not None:\n            y = torch.FloatTensor(self.y[self.i:self.i + self.batch_size].astype(np.float32))\n\n        else:\n            y = None\n\n        xcont = torch.FloatTensor(self.X_cont[self.i:self.i + self.batch_size])\n        xcat = torch.LongTensor(self.X_cat[self.i:self.i + self.batch_size])\n\n        batch = (xcont, xcat, y)\n        self.i += self.batch_size\n        return batch\n\n    def __len__(self):\n        return self.n_batches\n    \n    \nDEVICE = ('cuda' if torch.cuda.is_available() else 'cpu')\n\n#XAVIER INITILIZATION\ndef init_weights(m):\n    if type(m) == nn.Linear:\n        torch.nn.init.xavier_uniform(m.weight)\n        m.bias.data.fill_(0.01)\n\n####### Simple MLP model for 2D input ############################\n\nclass MLB_NN(nn.Module):\n    def __init__(self, emb_dims, n_cont, hidden_dim, device=DEVICE):\n        super().__init__()\n        self.device = device\n\n        # Embedding layers\n        self.emb_layers = nn.ModuleList([nn.Embedding(x, y) for x, y in emb_dims])\n        n_embs = sum([y for x, y in emb_dims])\n\n        n_embs = n_embs\n        n_cont = n_cont\n        inp_dim = n_cont + n_embs\n\n        # HIDDEN LAYERS\n        self.fc0 = nn.Linear(inp_dim, hidden_dim)\n        #self.drop0 = nn.Dropout(0.1)\n        self.fc1 = nn.Linear(hidden_dim, int(hidden_dim))\n        #self.drop1 = nn.Dropout(0.1)\n        self.fc2 = nn.Linear(int(hidden_dim), int(hidden_dim))\n        self.fc3 = nn.Linear(int(hidden_dim), 4)\n\n\n        # apply initilizations\n        self.fc0.apply(init_weights)\n        self.fc1.apply(init_weights)\n        self.fc2.apply(init_weights)\n        self.fc3.apply(init_weights)\n\n\n    # train embedding layers and concat cat and cont variables\n    def encode_and_combine_data(self, cont_data, cat_data):\n        xcat = [el(cat_data[:, k]) for k, el in enumerate(self.emb_layers)]\n        xcat = torch.cat(xcat, 1)\n        x = torch.cat([xcat, cont_data], 1)\n        return x\n\n    def forward(self, cont_data, cat_data):\n        cont_data = cont_data.to(self.device)\n        cat_data = cat_data.to(self.device)\n        x = self.encode_and_combine_data(cont_data, cat_data)\n\n        hz = F.leaky_relu(self.fc0(x))\n        #hz = self.drop0(hz)\n        hz = F.leaky_relu(self.fc1(hz))\n        #hz = self.drop1(hz)\n        hz = F.leaky_relu(self.fc2(hz))\n        out = self.fc3(hz)\n\n        return out\n    \ndef training_nn(x_train, y_train, x_valid, y_valid, cont_cols, cat_cols, \n                model_nn, model_path, epoch=50, patience=5, hidden_dim=128, device='cpu'):\n\n    uniques = {}\n    for i, v in enumerate(cat_cols):\n        uniques[v] = len(np.unique(list(x_train[v].unique())+list(x_valid[v].unique())))\n\n\n    pred_val = []\n    true_y = []\n\n    \n    #Make input for pytorch loader because we have categorical and continues features\n    x_train = make_2Dinput(x_train, cont_cols=cont_cols, cat_cols=cat_cols)\n    x_valid = make_2Dinput(x_valid, cont_cols=cont_cols, cat_cols=cat_cols)\n\n    #Make loader for pytorch\n    train_loader = MLBLoader(x_train, y_train.values, cat_cols=cat_cols, batch_size=1024, shuffle=True)\n    val_loader = MLBLoader(x_valid, y_valid.values, cat_cols=cat_cols, batch_size=1024, shuffle=False)\n    \n     ## make embedding dimensions\n    \n    dims = [32, 32, 32, 32, 32, 32, 32]\n    emb_dims = [(uniques[col], y) for col, y in zip(cat_cols, dims)]\n\n    # number of continues variables\n    n_cont = train_loader.n_conts\n    \n    if os.path.isfile(model_path):\n    \n        #best_model = model_nn(emb_dims=emb_dims, n_cont=n_cont, hidden_dim=hidden_dim).to(device)\n        best_model = torch.load(model_path, map_location=device).to(device)\n        print('done load model')\n        # Validation phase for single epoch\n        phase='Valid'\n        with torch.no_grad():\n            best_model.eval()\n            y_true = []\n            y_pred = []\n\n            for i, (X_cont, X_cat, y) in enumerate(tqdm(val_loader)):\n                out = best_model(X_cont.to(device), X_cat.to(device))                \n                y_pred.append(out) \n                y_true.append(y)\n            \n            print('done predict')\n            y_pred = torch.cat(y_pred, dim=0).detach().cpu().numpy()\n            y_true = torch.cat(y_true, dim=0).detach().cpu().numpy()\n\n            best_rmse = 0\n            for c in range(4):\n                best_rmse += mean_absolute_error(y_true[:, c], y_pred[:, c])/4\n                print(f'target{c}', mean_absolute_error(y_true[:, c], y_pred[:, c]))\n\n            print(f\" Val MAE: {best_rmse:.4f} \")\n    else:\n        \n        #neural network model\n        model = model_nn(emb_dims=emb_dims, n_cont=n_cont, hidden_dim=hidden_dim).to(device)\n        \n        #loss function\n        criterion = nn.L1Loss()\n\n        #adam optimizer has been used for training\n        optimizer = torch.optim.Adam(model.parameters(), lr=0.001, weight_decay=1e-5)\n\n        #learning rate scheduler\n        scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,\n                                                  max_lr=1e-2, epochs=epoch, steps_per_epoch=len(train_loader))\n\n\n        best_rmse=np.inf\n        counter=0\n        for ep in range(epoch):\n            train_loss, val_loss = 0, 0\n\n            #training phase for single epoch\n            model.train()\n            for i, (X_cont, X_cat, y) in enumerate(tqdm(train_loader)):\n\n                optimizer.zero_grad()\n                out= model(X_cont.to(device), X_cat.to(device))\n                \n                loss = 0\n                \n                for c in range(4):\n                    loss = loss + criterion(out[:, c], y[:, c].to(device))\n                    \n                loss.backward()\n\n                optimizer.step()\n                scheduler.step()\n\n                with torch.no_grad():\n                    train_loss += loss.item() / len(train_loader)\n\n            # Validation phase for single epoch\n            phase='Valid'\n            with torch.no_grad():\n                model.eval()\n                y_true = []\n                y_pred = []\n                \n                for i, (X_cont, X_cat, y) in enumerate(tqdm(val_loader)):\n                    out = model(X_cont.to(device), X_cat.to(device))\n                    y_pred.append(out) \n                    y_true.append(y)\n                \n                y_pred = torch.cat(y_pred, dim=0).detach().cpu().numpy()\n                y_true = torch.cat(y_true, dim=0).detach().cpu().numpy()\n                \n                rmse = 0\n                for c in range(4):\n                    rmse += mean_absolute_error(y_true[:, c], y_pred[:, c])/4\n                    \n                print(f\"[{phase}] Epoch: {ep} | Tain loss: {train_loss:.4f} | Val MAE: {rmse:.4f} \")\n\n                if best_rmse > rmse:\n                    best_rmse = rmse\n                    best_model = model\n                    torch.save(best_model, model_path)\n                    counter = 0\n                else:\n                    counter = counter + 1\n\n            #early stopping \n            if counter>=patience:\n                print(\"Early stopping\")\n                break\n\n    return best_model, best_rmse\n","metadata":{"execution":{"iopub.status.busy":"2021-07-31T18:58:58.216238Z","iopub.execute_input":"2021-07-31T18:58:58.216881Z","iopub.status.idle":"2021-07-31T18:58:58.260771Z","shell.execute_reply.started":"2021-07-31T18:58:58.216839Z","shell.execute_reply":"2021-07-31T18:58:58.259801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols = ['label_playerId', 'label_primaryPositionName', \n            'label_teamId', 'label_status']\n\ncont_cols = feature_cols\ncont_cols = [f for f in cont_cols if f not in cat_cols]\n\nfor col in cont_cols:\n    x_train1[col] = x_train1[col].fillna(0)\n    x_valid1[col] = x_valid1[col].fillna(0)\n\n    \nmodel_nets = []\nfor cut_date in [20210301, 20210401, 20210501, 20210601, 20210701]:\n    model_net, score_nn = training_nn(x_train1, y_train1, x_valid1, y_valid1, cont_cols, \n                                      cat_cols, MLB_NN, model_path=f'../input/mlb-final-models/model_nn_{cut_date}.pth', \n                                      epoch=50, patience=5, hidden_dim=256, device=DEVICE)\n\n    model_nets.append(model_net)","metadata":{"execution":{"iopub.status.busy":"2021-07-31T18:59:07.425496Z","iopub.execute_input":"2021-07-31T18:59:07.425825Z","iopub.status.idle":"2021-07-31T18:59:16.429296Z","shell.execute_reply.started":"2021-07-31T18:59:07.425797Z","shell.execute_reply":"2021-07-31T18:59:16.428483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-07-31T18:59:19.102794Z","iopub.execute_input":"2021-07-31T18:59:19.103118Z","iopub.status.idle":"2021-07-31T18:59:19.231328Z","shell.execute_reply.started":"2021-07-31T18:59:19.103088Z","shell.execute_reply":"2021-07-31T18:59:19.230541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del x_train1, y_train1, x_valid1, y_valid1\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-07-31T18:59:21.087099Z","iopub.execute_input":"2021-07-31T18:59:21.087448Z","iopub.status.idle":"2021-07-31T18:59:21.218371Z","shell.execute_reply.started":"2021-07-31T18:59:21.087395Z","shell.execute_reply":"2021-07-31T18:59:21.217510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.loc[train['date']>20210101]","metadata":{"execution":{"iopub.status.busy":"2021-07-31T18:59:23.002963Z","iopub.execute_input":"2021-07-31T18:59:23.003287Z","iopub.status.idle":"2021-07-31T18:59:23.171260Z","shell.execute_reply.started":"2021-07-31T18:59:23.003257Z","shell.execute_reply":"2021-07-31T18:59:23.170438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-07-31T18:59:24.752624Z","iopub.execute_input":"2021-07-31T18:59:24.752962Z","iopub.status.idle":"2021-07-31T18:59:24.873424Z","shell.execute_reply.started":"2021-07-31T18:59:24.752930Z","shell.execute_reply":"2021-07-31T18:59:24.872584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"players_cols = ['playerId', 'primaryPositionName']\nrosters_cols = ['playerId', 'teamId', 'status']\nscores_cols = ['playerId', 'battingOrder', 'gamesPlayedBatting', 'flyOuts',\n       'groundOuts', 'runsScored', 'doubles', 'triples', 'homeRuns',\n       'strikeOuts', 'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch',\n       'atBats', 'caughtStealing', 'stolenBases', 'groundIntoDoublePlay',\n       'groundIntoTriplePlay', 'plateAppearances', 'totalBases', 'rbi',\n       'leftOnBase', 'sacBunts', 'sacFlies', 'catchersInterference',\n       'pickoffs', 'gamesPlayedPitching', 'gamesStartedPitching',\n       'completeGamesPitching', 'shutoutsPitching', 'winsPitching',\n       'lossesPitching', 'flyOutsPitching', 'airOutsPitching',\n       'groundOutsPitching', 'runsPitching', 'doublesPitching',\n       'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching',\n       'baseOnBallsPitching', 'intentionalWalksPitching', 'hitsPitching',\n       'hitByPitchPitching', 'atBatsPitching', 'caughtStealingPitching',\n       'stolenBasesPitching', 'inningsPitched', 'saveOpportunities',\n       'earnedRuns', 'battersFaced', 'outsPitching', 'pitchesThrown', 'balls',\n       'strikes', 'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching',\n       'rbiPitching', 'gamesFinishedPitching', 'inheritedRunners',\n       'inheritedRunnersScored', 'catchersInterferencePitching',\n       'sacBuntsPitching', 'sacFliesPitching', 'saves', 'holds', 'blownSaves',\n       'assists', 'putOuts', 'errors', 'chances']","metadata":{"execution":{"iopub.status.busy":"2021-07-31T18:59:26.663822Z","iopub.execute_input":"2021-07-31T18:59:26.664159Z","iopub.status.idle":"2021-07-31T18:59:26.670873Z","shell.execute_reply.started":"2021-07-31T18:59:26.664128Z","shell.execute_reply":"2021-07-31T18:59:26.669943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"player_target_stats=player_target_stats.loc[player_target_stats['year']==2021]","metadata":{"execution":{"iopub.status.busy":"2021-07-31T18:59:28.746160Z","iopub.execute_input":"2021-07-31T18:59:28.746502Z","iopub.status.idle":"2021-07-31T18:59:28.752484Z","shell.execute_reply.started":"2021-07-31T18:59:28.746469Z","shell.execute_reply":"2021-07-31T18:59:28.751378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null = np.nan\ntrue = True\nfalse = False\n\nweights = {20210301: 0.15, 20210401: 0.15, 20210501:0.2, 20210601:0.25, 20210701:0.25}\ncut_dates = [20210301, 20210401, 20210501, 20210601, 20210701]","metadata":{"execution":{"iopub.status.busy":"2021-07-31T19:01:49.744192Z","iopub.execute_input":"2021-07-31T19:01:49.744549Z","iopub.status.idle":"2021-07-31T19:01:49.749607Z","shell.execute_reply.started":"2021-07-31T19:01:49.744516Z","shell.execute_reply":"2021-07-31T19:01:49.748523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import copy\n\nenv = mlb.make_env() # initialize the environment\niter_test = env.iter_test() # iterator which loops over each date in test set\n\nfor (test_df, sample_prediction_df) in iter_test: # make predictions here\n    \n    sub = copy.deepcopy(sample_prediction_df.reset_index())\n    sample_prediction_df = copy.deepcopy(sample_prediction_df.reset_index(drop=True))\n    \n    # LGBM summit\n    # creat dataset\n    sample_prediction_df['playerId'] = sample_prediction_df['date_playerId']\\\n                                        .map(lambda x: int(x.split('_')[1]))\n    # Dealing with missing values\n    if test_df['rosters'].iloc[0] == test_df['rosters'].iloc[0]:\n        test_rosters = pd.DataFrame(eval(test_df['rosters'].iloc[0]))\n    else:\n        test_rosters = pd.DataFrame({'playerId': sample_prediction_df['playerId']})\n        for col in rosters.columns:\n            if col == 'playerId': continue\n            test_rosters[col] = np.nan\n            \n    if test_df['playerBoxScores'].iloc[0] == test_df['playerBoxScores'].iloc[0]:\n        test_scores = pd.DataFrame(eval(test_df['playerBoxScores'].iloc[0]))\n    else:\n        test_scores = pd.DataFrame({'playerId': sample_prediction_df['playerId']})\n        for col in scores.columns:\n            if col == 'playerId': continue\n            test_scores[col] = np.nan\n    \n    test_scores = test_scores.groupby('playerId').sum().reset_index()\n    test = sample_prediction_df[['playerId', 'target1', 'target2', 'target3', 'target4']].copy()\n    test = test.merge(players[players_cols], on='playerId', how='left')\n    test = test.merge(test_rosters[rosters_cols], on='playerId', how='left')\n    test = test.merge(test_scores[scores_cols], on='playerId', how='left')\n    test = test.merge(player_target_stats, on='playerId', how='left')\n  \n    \n    test['label_playerId'] = test['playerId'].map(player2num)\n    test['label_primaryPositionName'] = test['primaryPositionName'].map(position2num)\n    test['label_teamId'] = test['teamId'].map(teamid2num)\n    test['label_status'] = test['status'].map(status2num)\n   \n\n    test_X = test[feature_cols]\n    \n    pred1, pred2, pred3, pred4 = 0, 0, 0, 0\n    for cut_date in cut_dates:\n        pred1 += weights[cut_date]*lgb_models[cut_date][0].predict(test_X)\n        pred2 += weights[cut_date]*lgb_models[cut_date][1].predict(test_X)\n        pred3 += weights[cut_date]*lgb_models[cut_date][2].predict(test_X)\n        pred4 += weights[cut_date]*lgb_models[cut_date][3].predict(test_X)\n    \n    \n    test_X=test_X.fillna(0)\n    \n    xcont = torch.FloatTensor(test_X[cont_cols].values)\n    xcat = torch.LongTensor(test_X[cat_cols].values)\n    \n    pred_nn1, pred_nn2, pred_nn3, pred_nn4 = 0, 0, 0, 0\n    \n    for cute_date, model_net in zip(cut_dates, model_nets): \n        pred_nn = model_net(xcont.to(DEVICE), xcat.to(DEVICE))\n        pred_nn = pred_nn.detach().cpu().numpy()\n        \n        pred_nn1 += pred_nn[:, 0]*weights[cut_date]\n        pred_nn2 += pred_nn[:, 1]*weights[cut_date]\n        pred_nn3 += pred_nn[:, 2]*weights[cut_date]\n        pred_nn4 += pred_nn[:, 3]*weights[cut_date]\n\n    \n    # merge submission\n    sample_prediction_df['target1'] = np.clip(0.50*pred1 + 0.50*pred_nn1, 0, 100)\n    sample_prediction_df['target2'] = np.clip(0.50*pred2 + 0.50*pred_nn2, 0, 100)\n    sample_prediction_df['target3'] = np.clip(0.50*pred3 + 0.50*pred_nn3, 0, 100)\n    sample_prediction_df['target4'] = np.clip(0.50*pred4 + 0.50*pred_nn4, 0, 100)\n\n    sample_prediction_df = sample_prediction_df.fillna(0.)\n    del sample_prediction_df['playerId']\n    \n    env.predict(sample_prediction_df)\n","metadata":{"_uuid":"bc76106c-5572-4d75-ae68-28381f7406b7","_cell_guid":"c3148c3f-68b2-46f6-b8d6-3284e68b507a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T19:02:21.952422Z","iopub.execute_input":"2021-07-31T19:02:21.952778Z","iopub.status.idle":"2021-07-31T19:02:23.969087Z","shell.execute_reply.started":"2021-07-31T19:02:21.952745Z","shell.execute_reply":"2021-07-31T19:02:23.967015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_prediction_df","metadata":{"_uuid":"9267b421-3cbb-4b28-a16e-e62ba8da58e4","_cell_guid":"cd1f93f4-37e0-47d0-b05a-0df03d07ce15","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T18:54:03.667817Z","iopub.execute_input":"2021-07-31T18:54:03.668136Z","iopub.status.idle":"2021-07-31T18:54:03.685044Z","shell.execute_reply.started":"2021-07-31T18:54:03.668106Z","shell.execute_reply":"2021-07-31T18:54:03.684037Z"},"trusted":true},"execution_count":null,"outputs":[]}]}