{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Credit to @columbia2131 - I started with his notebook and then added an external data set with descriptive statistics of the targets for each player and also added unique params for each target model","metadata":{"_uuid":"10e9724f-0d14-48a8-a9fa-9059dbac793e","_cell_guid":"1b819f40-c669-47e4-8943-f7267d3bf422","trusted":true}},{"cell_type":"markdown","source":"## About Dataset","metadata":{"_uuid":"f62e37b3-f14a-4a1e-bfe6-8611ad37e5f6","_cell_guid":"8ec126c8-1a3c-48e7-88e3-59b8587a37d5","trusted":true}},{"cell_type":"markdown","source":"## Training","metadata":{"_uuid":"1b9ba20f-020a-4a05-8eb1-af469826de52","_cell_guid":"7af6ba65-a041-4bae-b9d4-e64aa3c27d40","execution":{"iopub.status.busy":"2021-06-16T09:14:33.869464Z","iopub.execute_input":"2021-06-16T09:14:33.869905Z","iopub.status.idle":"2021-06-16T09:14:33.874766Z","shell.execute_reply.started":"2021-06-16T09:14:33.869879Z","shell.execute_reply":"2021-06-16T09:14:33.873097Z"},"trusted":true}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom sklearn.metrics import mean_absolute_error\nfrom datetime import timedelta\nfrom functools import reduce\nfrom tqdm import tqdm\nimport lightgbm as lgbm\nimport mlb\nimport pickle","metadata":{"_uuid":"c5d8c7f0-5100-4355-bfb6-596a9ce078c0","_cell_guid":"360f5ee2-bb4a-4799-9af3-f6f5416bc263","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T17:05:24.056188Z","iopub.execute_input":"2021-07-31T17:05:24.056680Z","iopub.status.idle":"2021-07-31T17:05:25.258549Z","shell.execute_reply.started":"2021-07-31T17:05:24.056564Z","shell.execute_reply":"2021-07-31T17:05:25.257254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport gc\nfrom plotly import express as px\nimport plotly.offline as pyo\n\nfrom matplotlib import pyplot as plt\n\nimport lightgbm as lgbm\n\n\nfrom pathlib import Path\nfrom sklearn.metrics import mean_absolute_error\nfrom datetime import timedelta\nfrom functools import reduce\nfrom tqdm import tqdm\nfrom datetime import datetime\n\nimport os\nfrom time import time\nimport json\nimport pickle\nfrom tqdm.notebook import tqdm\n\nimport keras\nimport tensorflow as tf\nfrom keras.layers import Dense, Concatenate, Dropout, Input, Lambda\nfrom keras.models import Model\n\nfrom sklearn.model_selection import train_test_split\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau\n\nimport catboost\n\n# from tqdm._tqdm_notebook import tqdm_notebook\n# tqdm_notebook.pandas()","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:05:25.260503Z","iopub.execute_input":"2021-07-31T17:05:25.260892Z","iopub.status.idle":"2021-07-31T17:05:27.757000Z","shell.execute_reply.started":"2021-07-31T17:05:25.260852Z","shell.execute_reply":"2021-07-31T17:05:27.755883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport sys\nimport warnings\nfrom joblib import Parallel, delayed\nfrom pathlib import Path\n\nimport ipywidgets as widgets\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split\nfrom statsmodels.tsa.deterministic import (CalendarFourier,\n CalendarSeasonality,\n CalendarTimeTrend,\n DeterministicProcess)\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom keras.layers.experimental.preprocessing import StringLookup\n\nwarnings.simplefilter(\"ignore\")\n\n# Set Matplotlib defaults\nplt.style.use(\"seaborn-whitegrid\")\nplt.rc(\"figure\", autolayout=True, figsize=(11, 5))\nplt.rc(\n \"axes\",\n labelweight=\"bold\",\n labelsize=\"large\",\n titleweight=\"bold\",\n titlesize=14,\n titlepad=10,\n)\nplot_params = dict(\n color=\"0.75\",\n style=\".-\",\n markeredgecolor=\"0.25\",\n markerfacecolor=\"0.25\",\n legend=False,\n)","metadata":{"_uuid":"3bc4320e-4fed-4b65-9cad-5321e997b704","_cell_guid":"11a3090e-70db-48d2-a9ee-92ab0d0cdc0e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T17:05:27.759098Z","iopub.execute_input":"2021-07-31T17:05:27.759422Z","iopub.status.idle":"2021-07-31T17:05:27.784005Z","shell.execute_reply.started":"2021-07-31T17:05:27.759393Z","shell.execute_reply":"2021-07-31T17:05:27.782942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# BASE_DIR = Path('../input/mlb-player-digital-engagement-forecasting')\n# TRAIN_DIR = Path('../input/mlb-pdef-train-dataset')","metadata":{"_uuid":"5c0b8c8c-c9bc-4056-a1f8-0aa3282239cc","_cell_guid":"d4943d35-9130-4f86-b350-057785731d73","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T17:05:27.785735Z","iopub.execute_input":"2021-07-31T17:05:27.786041Z","iopub.status.idle":"2021-07-31T17:05:27.789656Z","shell.execute_reply.started":"2021-07-31T17:05:27.786011Z","shell.execute_reply":"2021-07-31T17:05:27.788746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# players = pd.read_csv(BASE_DIR / 'players.csv')\n\n# rosters = pd.read_pickle(TRAIN_DIR / 'rosters_train.pkl')\n# targets = pd.read_pickle(TRAIN_DIR / 'nextDayPlayerEngagement_train.pkl')\n# followers = pd.read_pickle(TRAIN_DIR / 'playerTwitterFollowers_train.pkl')\n# team_followers = pd.read_pickle(TRAIN_DIR / 'teamTwitterFollowers_train.pkl')\n# team_followers = team_followers.rename(columns={'numberOfFollowers': 'teamFollowers'})\n# scores = pd.read_pickle(TRAIN_DIR / 'playerBoxScores_train.pkl')\n# scores = scores.groupby(['playerId', 'date']).sum().reset_index()","metadata":{"_uuid":"44fc9252-f9fe-4986-a827-c0bef24aaa9f","_cell_guid":"46a8ddb2-050a-4912-bd2f-bfe66a0c0cce","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T17:05:27.791135Z","iopub.execute_input":"2021-07-31T17:05:27.791437Z","iopub.status.idle":"2021-07-31T17:05:27.805292Z","shell.execute_reply.started":"2021-07-31T17:05:27.791407Z","shell.execute_reply":"2021-07-31T17:05:27.804153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# targets_cols = ['playerId', 'target1', 'target2', 'target3', 'target4', 'date']\n# players_cols = ['playerId', 'primaryPositionName']\n# rosters_cols = ['playerId', 'teamId', 'status', 'date']\n# followers_cols = ['playerId', 'numberOfFollowers', 'date']\n# teamfollowers_cols = ['teamId', 'teamFollowers', 'date']\n# scores_cols = ['playerId', 'battingOrder', 'gamesPlayedBatting', 'flyOuts',\n# 'groundOuts', 'runsScored', 'doubles', 'triples', 'homeRuns',\n# 'strikeOuts', 'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch',\n# 'atBats', 'caughtStealing', 'stolenBases', 'groundIntoDoublePlay',\n# 'groundIntoTriplePlay', 'plateAppearances', 'totalBases', 'rbi',\n# 'leftOnBase', 'sacBunts', 'sacFlies', 'catchersInterference',\n# 'pickoffs', 'gamesPlayedPitching', 'gamesStartedPitching',\n# 'completeGamesPitching', 'shutoutsPitching', 'winsPitching',\n# 'lossesPitching', 'flyOutsPitching', 'airOutsPitching',\n# 'groundOutsPitching', 'runsPitching', 'doublesPitching',\n# 'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching',\n# 'baseOnBallsPitching', 'intentionalWalksPitching', 'hitsPitching',\n# 'hitByPitchPitching', 'atBatsPitching', 'caughtStealingPitching',\n# 'stolenBasesPitching', 'inningsPitched', 'saveOpportunities',\n# 'earnedRuns', 'battersFaced', 'outsPitching', 'pitchesThrown', 'balls',\n# 'strikes', 'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching',\n# 'rbiPitching', 'gamesFinishedPitching', 'inheritedRunners',\n# 'inheritedRunnersScored', 'catchersInterferencePitching',\n# 'sacBuntsPitching', 'sacFliesPitching', 'saves', 'holds', 'blownSaves',\n# 'assists', 'putOuts', 'errors', 'chances', 'date']\n\n# feature_cols = ['label_playerId', 'label_primaryPositionName', 'label_teamId',\n# 'label_status', 'battingOrder', 'gamesPlayedBatting', 'flyOuts',\n# 'groundOuts', 'runsScored', 'doubles', 'triples', 'homeRuns',\n# 'strikeOuts', 'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch',\n# 'atBats', 'caughtStealing', 'stolenBases', 'groundIntoDoublePlay',\n# 'groundIntoTriplePlay', 'plateAppearances', 'totalBases', 'rbi',\n# 'leftOnBase', 'sacBunts', 'sacFlies', 'catchersInterference',\n# 'pickoffs', 'gamesPlayedPitching', 'gamesStartedPitching',\n# 'completeGamesPitching', 'shutoutsPitching', 'winsPitching',\n# 'lossesPitching', 'flyOutsPitching', 'airOutsPitching',\n# 'groundOutsPitching', 'runsPitching', 'doublesPitching',\n# 'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching',\n# 'baseOnBallsPitching', 'intentionalWalksPitching', 'hitsPitching',\n# 'hitByPitchPitching', 'atBatsPitching', 'caughtStealingPitching',\n# 'stolenBasesPitching', 'inningsPitched', 'saveOpportunities',\n# 'earnedRuns', 'battersFaced', 'outsPitching', 'pitchesThrown', 'balls',\n# 'strikes', 'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching',\n# 'rbiPitching', 'gamesFinishedPitching', 'inheritedRunners',\n# 'inheritedRunnersScored', 'catchersInterferencePitching',\n# 'sacBuntsPitching', 'sacFliesPitching', 'saves', 'holds', 'blownSaves',\n# 'assists', 'putOuts', 'errors', 'chances','target1_mean',\n# 'target1_median',\n# 'target1_std',\n# 'target1_min',\n# 'target1_max',\n# 'target1_prob','target2_mean',\n# 'target2_median',\n# 'target2_std',\n# 'target2_min',\n# 'target2_max',\n# 'target2_prob','target3_mean',\n# 'target3_median',\n# 'target3_std',\n# 'target3_min',\n# 'target3_max',\n# 'target3_prob','target4_mean',\n# 'target4_median',\n# 'target4_std',\n# 'target4_min',\n# 'target4_max',\n# 'target4_prob']#'numberOfFollowers','teamFollowers'","metadata":{"_uuid":"be11f562-6ab4-47e4-ab84-84fd1bbd721d","_cell_guid":"4ad5876b-9c5c-4011-841b-d6d456434a4d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T17:05:27.806735Z","iopub.execute_input":"2021-07-31T17:05:27.807156Z","iopub.status.idle":"2021-07-31T17:05:27.824311Z","shell.execute_reply.started":"2021-07-31T17:05:27.807125Z","shell.execute_reply":"2021-07-31T17:05:27.823499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# player_target_stats = pd.read_csv(\"../input/player-target-stats/player_target_stats.csv\")\n# data_names=player_target_stats.columns.values.tolist()\n# data_names","metadata":{"_uuid":"b80f71bb-ad15-4fb8-b5e4-501782cf81e5","_cell_guid":"bf8a3fa5-a50b-4f44-a348-eebb9856d1f7","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T17:05:27.825455Z","iopub.execute_input":"2021-07-31T17:05:27.825823Z","iopub.status.idle":"2021-07-31T17:05:27.842047Z","shell.execute_reply.started":"2021-07-31T17:05:27.825788Z","shell.execute_reply":"2021-07-31T17:05:27.840765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # creat dataset\n# train = targets[targets_cols].merge(players[players_cols], on=['playerId'], how='left')\n# train = train.merge(rosters[rosters_cols], on=['playerId', 'date'], how='left')\n# train = train.merge(scores[scores_cols], on=['playerId', 'date'], how='left')\n# train = train.merge(followers[followers_cols], on=['playerId', 'date'], how='left')\n# train = train.merge(team_followers[teamfollowers_cols], on=['teamId', 'date'], how='left')\n# train = train.merge(player_target_stats, how='inner', left_on=[\"playerId\"],right_on=[\"playerId\"])\n\n# # label encoding\n# player2num = {c: i for i, c in enumerate(train['playerId'].unique())}\n# position2num = {c: i for i, c in enumerate(train['primaryPositionName'].unique())}\n# teamid2num = {c: i for i, c in enumerate(train['teamId'].unique())}\n# status2num = {c: i for i, c in enumerate(train['status'].unique())}\n# train['label_playerId'] = train['playerId'].map(player2num)\n# train['label_primaryPositionName'] = train['primaryPositionName'].map(position2num)\n# train['label_teamId'] = train['teamId'].map(teamid2num)\n# train['label_status'] = train['status'].map(status2num)","metadata":{"_uuid":"78a99df4-585a-4fe5-90a5-dd2134b49a67","_cell_guid":"630bd8f2-a558-4574-8ec7-fbcc2602e5de","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T17:05:27.845985Z","iopub.execute_input":"2021-07-31T17:05:27.846538Z","iopub.status.idle":"2021-07-31T17:05:27.853492Z","shell.execute_reply.started":"2021-07-31T17:05:27.846480Z","shell.execute_reply":"2021-07-31T17:05:27.852665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train","metadata":{"_uuid":"e426b0a2-55a5-4b27-90d6-6e03145fdabc","_cell_guid":"a1136064-5d4d-45aa-a446-f2fe68b04b0a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T17:05:27.855428Z","iopub.execute_input":"2021-07-31T17:05:27.855935Z","iopub.status.idle":"2021-07-31T17:05:27.871403Z","shell.execute_reply.started":"2021-07-31T17:05:27.855884Z","shell.execute_reply":"2021-07-31T17:05:27.870545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_X = train[feature_cols]\n# train_y = train[['target1', 'target2', 'target3', 'target4']]\n\n# _index = (train['date'] < 20210401)\n# x_train = train_X.loc[_index].reset_index(drop=True)\n# y_train = train_y.loc[_index].reset_index(drop=True)\n# x_valid = train_X.loc[~_index].reset_index(drop=True)\n# y_valid = train_y.loc[~_index].reset_index(drop=True)","metadata":{"_uuid":"cef57d06-b6be-477c-9d6f-3bf5cf9ac228","_cell_guid":"61fbb325-944a-448a-b849-73827e42823a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T17:05:27.872784Z","iopub.execute_input":"2021-07-31T17:05:27.873299Z","iopub.status.idle":"2021-07-31T17:05:27.885500Z","shell.execute_reply.started":"2021-07-31T17:05:27.873250Z","shell.execute_reply":"2021-07-31T17:05:27.884522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def fit_lgbm(x_train, y_train, x_valid, y_valid, params: dict=None, verbose=100):\n# oof_pred = np.zeros(len(y_valid), dtype=np.float32)\n# model = lgbm.LGBMRegressor(**params)\n# model.fit(x_train, y_train, \n# eval_set=[(x_valid, y_valid)], \n# early_stopping_rounds=verbose, \n# verbose=verbose)\n# oof_pred = model.predict(x_valid)\n# score = mean_absolute_error(oof_pred, y_valid)\n# print('mae:', score)\n# return oof_pred, model, score\n\n\n# # training lightgbm\n# params1 = {\n# 'objective':'mae',\n# 'reg_alpha': 0.1,\n# 'reg_lambda': 0.1, \n# 'n_estimators': 10000,\n# 'learning_rate': 0.1,\n# 'random_state': 42,\n# \"num_leaves\": 100\n# }\n\n# params2 = {\n# 'objective':'mae',\n# 'reg_alpha': 0.1,\n# 'reg_lambda': 0.1, \n# 'n_estimators': 80,\n# 'learning_rate': 0.1,\n# 'random_state': 42,\n# \"num_leaves\": 22\n# }\n\n# params4 = {\n# 'objective':'mae',\n# 'reg_alpha': 0.1,\n# 'reg_lambda': 0.1, \n# 'n_estimators': 10000,\n# 'learning_rate': 0.1,\n# 'random_state': 42,\n# \"num_leaves\": 100\n# }\n\n\n# params = {\n# 'objective':'mae',\n# 'reg_alpha': 0.1,\n# 'reg_lambda': 0.1, \n# 'n_estimators': 10000,\n# 'learning_rate': 0.1,\n# 'random_state': 42,\n# \"num_leaves\": 100\n# }\n\n\n# oof1, model1, score1 = fit_lgbm(\n# x_train, y_train['target1'],\n# x_valid, y_valid['target1'],\n# params1\n# )\n\n# oof2, model2, score2 = fit_lgbm(\n# x_train, y_train['target2'],\n# x_valid, y_valid['target2'],\n# params2\n# )\n\n# oof3, model3, score3 = fit_lgbm(\n# x_train, y_train['target3'],\n# x_valid, y_valid['target3'],\n# params\n# )\n\n# oof4, model4, score4 = fit_lgbm(\n# x_train, y_train['target4'],\n# x_valid, y_valid['target4'],\n# params4\n# )\n\n# score = (score1+score2+score3+score4) / 4\n# print(f'score: {score}')","metadata":{"_uuid":"58fca32e-a4cd-4c8d-b555-2e9f8ee39ff2","_cell_guid":"ac2f2b7e-6e2f-4d88-8347-a4f36b1bb297","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T17:05:27.887047Z","iopub.execute_input":"2021-07-31T17:05:27.887550Z","iopub.status.idle":"2021-07-31T17:05:27.900017Z","shell.execute_reply.started":"2021-07-31T17:05:27.887506Z","shell.execute_reply":"2021-07-31T17:05:27.899031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import matplotlib.pyplot as plt\n# import seaborn as sns\n\n# feature_imp = pd.DataFrame(sorted(zip(model1.feature_importances_,x_train[feature_cols])), columns=['Value','Feature'])\n\n# plt.figure(figsize=(15, 10))\n# sns.barplot(x=\"Value\", y=\"Feature\", data=feature_imp.sort_values(by=\"Value\", ascending=False))\n# plt.title('LightGBM Features (avg over folds)')\n# plt.tight_layout()\n# plt.show()\n# #plt.savefig('lgbm_importances-01.png')\n\n# print (feature_imp['Feature'].tolist())\n\n# feature_imp.tail(20)","metadata":{"_uuid":"c4307836-4083-4f99-98fc-a9541fe25c24","_cell_guid":"84fdad96-1918-488a-a39f-19381b15c11a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T17:05:27.901609Z","iopub.execute_input":"2021-07-31T17:05:27.902279Z","iopub.status.idle":"2021-07-31T17:05:27.917787Z","shell.execute_reply.started":"2021-07-31T17:05:27.902226Z","shell.execute_reply":"2021-07-31T17:05:27.916705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_le(path):\n with open(path, 'rb') as p:\n return pickle.load(p)","metadata":{"_uuid":"53e1f199-3bbe-4fdf-9a16-dd26240e19d2","_cell_guid":"7418fc4e-04c3-4931-8e0c-ca58fe33b977","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T17:05:27.919155Z","iopub.execute_input":"2021-07-31T17:05:27.919468Z","iopub.status.idle":"2021-07-31T17:05:27.935988Z","shell.execute_reply.started":"2021-07-31T17:05:27.919438Z","shell.execute_reply":"2021-07-31T17:05:27.934862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n start_mem = df.memory_usage().sum() / 1024**2\n for col in df.columns:\n col_type = df[col].dtypes\n if col_type in numerics:\n c_min = df[col].min()\n c_max = df[col].max()\n if str(col_type)[:3] == 'int':\n if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n df[col] = df[col].astype(np.int8)\n elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n df[col] = df[col].astype(np.int16)\n elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n df[col] = df[col].astype(np.int32)\n elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n df[col] = df[col].astype(np.int64)\n else:\n if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n df[col] = df[col].astype(np.float16)\n elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n df[col] = df[col].astype(np.float32)\n else:\n df[col] = df[col].astype(np.float64)\n end_mem = df.memory_usage().sum() / 1024**2\n if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n return df","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:05:27.937438Z","iopub.execute_input":"2021-07-31T17:05:27.937781Z","iopub.status.idle":"2021-07-31T17:05:27.953176Z","shell.execute_reply.started":"2021-07-31T17:05:27.937748Z","shell.execute_reply":"2021-07-31T17:05:27.952274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_START = '2021-08-01'\n# TEST_END = '2021-06-01'","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:05:27.954621Z","iopub.execute_input":"2021-07-31T17:05:27.955210Z","iopub.status.idle":"2021-07-31T17:05:27.969926Z","shell.execute_reply.started":"2021-07-31T17:05:27.955162Z","shell.execute_reply":"2021-07-31T17:05:27.968966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Data","metadata":{"_uuid":"419cb980-6676-498f-9076-6f999124dbef","_cell_guid":"03d4ce2f-823a-449e-82e5-df3337b3ae15","trusted":true}},{"cell_type":"code","source":"# # Helper function to unpack json found in daily data\n# def unpack_json(json_str):\n# return pd.DataFrame() if pd.isna(json_str) else pd.read_json(json_str)\n\n\n# def unpack_data(data, dfs=None, n_jobs=-1):\n# if dfs is not None:\n# data = data.loc[:, dfs]\n# unnested_dfs = {}\n# for name, column in data.iteritems():\n# daily_dfs = Parallel(n_jobs=n_jobs)(\n# delayed(unpack_json)(item) for date, item in column.iteritems())\n# df = pd.concat(daily_dfs)\n# unnested_dfs[name] = df\n# return unnested_dfs","metadata":{"_uuid":"05d7363b-0142-44dd-be8c-3af912f050a5","_cell_guid":"a2d3bd6c-be98-4477-bbad-245e774f5ec4","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T17:05:27.971384Z","iopub.execute_input":"2021-07-31T17:05:27.972001Z","iopub.status.idle":"2021-07-31T17:05:27.982331Z","shell.execute_reply.started":"2021-07-31T17:05:27.971956Z","shell.execute_reply":"2021-07-31T17:05:27.981343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data_dir = Path('../input/mlb-player-digital-engagement-forecasting/')\n\n# df_names = ['seasons', 'teams', 'players', 'awards']\n\n# for name in df_names:\n# globals()[name] = pd.read_csv(data_dir / f\"{name}.csv\")\n\n# # kaggle_data_tabs = widgets.Tab()\n# # # Add Output widgets for each pandas DF as tabs' children\n# # kaggle_data_tabs.children = list([widgets.Output() for df_name in df_names])\n\n# # for index in range(0, len(df_names)):\n# # # Rename tab bar titles to df names\n# # kaggle_data_tabs.set_title(index, df_names[index])\n \n# # # Display corresponding table output for this tab name\n# # with kaggle_data_tabs.children[index]:\n# # display(eval(df_names[index]))\n\n# # display(kaggle_data_tabs)","metadata":{"_uuid":"dc1c4a3b-1d2d-4a61-a8f7-61d9959a3919","_cell_guid":"2f2b983f-4d76-4d8f-8641-4f9b0b2ad954","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T17:05:27.984700Z","iopub.execute_input":"2021-07-31T17:05:27.985057Z","iopub.status.idle":"2021-07-31T17:05:27.997712Z","shell.execute_reply.started":"2021-07-31T17:05:27.985022Z","shell.execute_reply":"2021-07-31T17:05:27.996657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# # Define dataframes to load from training set\n# dfs = [\n# 'nextDayPlayerEngagement', # targets\n# 'playerBoxScores', # features\n# # Other dataframes available for features:\n# # 'games',\n# 'rosters',\n# # 'teamBoxScores',\n# # 'transactions',\n# # 'standings',\n# # 'awards',\n# # 'events',\n# # 'playerTwitterFollowers',\n# # 'teamTwitterFollowers',\n# ]\n\n# # Read training data\n# training = pd.read_csv(\n# data_dir / 'train.csv',\n# usecols=['date'] + dfs,\n# )\n\n# # Convert training data date field to datetime type\n# # training['date'] = pd.to_datetime(training['date'], format=\"%Y%m%d\")\n# # training = training.set_index('date').to_period('D')\n# print(training.info())","metadata":{"_uuid":"3683a846-9589-4865-a142-50ded990f3d5","_cell_guid":"33578d73-6aa4-4fa4-bd1a-a4100f809542","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T17:05:27.998833Z","iopub.execute_input":"2021-07-31T17:05:27.999392Z","iopub.status.idle":"2021-07-31T17:05:28.009195Z","shell.execute_reply.started":"2021-07-31T17:05:27.999358Z","shell.execute_reply":"2021-07-31T17:05:28.008170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %time\n# # Unpack nested dataframes and store in dictionary `training_dfs`\n# training_dfs = unpack_data(training, dfs=dfs)\n# print('\\n', training_dfs.keys())","metadata":{"_uuid":"d1636528-23ca-4ed8-9804-35d2c6738124","_cell_guid":"cc3ff1b6-3d88-4ee0-b0ce-037d2dc5319a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T17:05:28.010530Z","iopub.execute_input":"2021-07-31T17:05:28.010868Z","iopub.status.idle":"2021-07-31T17:05:28.021488Z","shell.execute_reply.started":"2021-07-31T17:05:28.010839Z","shell.execute_reply":"2021-07-31T17:05:28.020314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# example_sample_submission = pd.read_csv('../input/mlb-player-digital-engagement-forecasting/example_sample_submission.csv')\n# example_sample_submission","metadata":{"_uuid":"c4a64557-6872-4f80-ac4e-10c18251e12f","_cell_guid":"adec3226-6cee-4d4f-9d80-022df3019e2b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T17:05:28.023065Z","iopub.execute_input":"2021-07-31T17:05:28.023492Z","iopub.status.idle":"2021-07-31T17:05:28.037615Z","shell.execute_reply.started":"2021-07-31T17:05:28.023447Z","shell.execute_reply":"2021-07-31T17:05:28.036790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# example_test = pd.read_csv('../input/mlb-player-digital-engagement-forecasting/example_test.csv')\n# example_test","metadata":{"_uuid":"9ad49603-5429-40c9-9048-ef4bd0241631","_cell_guid":"d96474f2-b35a-43b1-bfcb-96fe4b285df1","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T17:05:28.038811Z","iopub.execute_input":"2021-07-31T17:05:28.039226Z","iopub.status.idle":"2021-07-31T17:05:28.049051Z","shell.execute_reply.started":"2021-07-31T17:05:28.039195Z","shell.execute_reply":"2021-07-31T17:05:28.047931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Load Data\n\n# targets = pd.read_pickle('../input/mlbunnesteddataset/train_nextDayPlayerEngagement.pickle')\n# targets['dailyDataDate'] = pd.to_datetime(targets['dailyDataDate'], format='%Y%m%d')\n# targets = reduce_mem_usage(targets)\n# display(targets)\n\n# # games = pd.read_pickle('../input/mlbunnesteddataset/train_games.pickle')\n# # games['dailyDataDate'] = pd.to_datetime(games['dailyDataDate'], format='%Y%m%d')\n# # games = reduce_mem_usage(games)\n# # games\n\n# rosters = pd.read_pickle('../input/mlbunnesteddataset/train_rosters.pickle')\n# rosters['dailyDataDate'] = pd.to_datetime(rosters['dailyDataDate'], format='%Y%m%d')\n# rosters = reduce_mem_usage(rosters)\n# display(rosters)\n\n# scores = pd.read_pickle('../input/mlbunnesteddataset/train_playerBoxScores.pickle')\n# scores['dailyDataDate'] = pd.to_datetime(scores['dailyDataDate'], format='%Y%m%d')\n# scores = reduce_mem_usage(scores)\n# display(scores)\n\n# # teamBoxScores_df = pd.read_pickle('/content/mlb-player-digital-engagement-forecasting_unnested/train_teamBoxScores.pickle')\n# # train_teamBoxScores_df\n\n# transactions = pd.read_pickle('../input/mlbunnesteddataset/train_transactions.pickle')\n# transactions['dailyDataDate'] = pd.to_datetime(transactions['dailyDataDate'], format='%Y%m%d')\n# transactions = reduce_mem_usage(transactions)\n# display(transactions)\n\n# standings = pd.read_pickle('../input/mlbunnesteddataset/train_standings.pickle')\n# standings['dailyDataDate'] = pd.to_datetime(standings['dailyDataDate'], format='%Y%m%d')\n# standings = reduce_mem_usage(standings)\n# display(standings)\n\n# awards = pd.read_pickle('../input/mlbunnesteddataset/train_awards.pickle')\n# awards['dailyDataDate'] = pd.to_datetime(awards['dailyDataDate'], format='%Y%m%d')\n# awards = reduce_mem_usage(awards)\n# awards\n\n# # events = pd.read_pickle('/content/mlb-player-digital-engagement-forecasting_unnested/train_events.pickle')\n# # events\n\n# playerTwitterFollowers = pd.read_pickle('../input/mlbunnesteddataset/train_playerTwitterFollowers.pickle')\n# playerTwitterFollowers['dailyDataDate'] = pd.to_datetime(playerTwitterFollowers['dailyDataDate'], format='%Y%m%d')\n# playerTwitterFollowers = reduce_mem_usage(playerTwitterFollowers)\n# display(playerTwitterFollowers)\n\n# teamTwitterFollowers = pd.read_pickle('../input/mlbunnesteddataset/train_teamTwitterFollowers.pickle')\n# teamTwitterFollowers['dailyDataDate'] = pd.to_datetime(teamTwitterFollowers['dailyDataDate'], format='%Y%m%d')\n# teamTwitterFollowers = reduce_mem_usage(teamTwitterFollowers)\n# display(teamTwitterFollowers)\n\n# players = pd.read_pickle('../input/mlbunnesteddataset/players.pickle')\n# players = reduce_mem_usage(players)\n# display(players)\n\n# teams = pd.read_pickle('../input/mlbunnesteddataset/teams.pickle')\n# teams = reduce_mem_usage(teams)\n# display(teams)\n\n# awards_pre2018 = pd.read_csv('../input/mlb-player-digital-engagement-forecasting/awards.csv')\n# awards_pre2018 = reduce_mem_usage(awards_pre2018)\n# awards_pre2018\n","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:05:28.050164Z","iopub.execute_input":"2021-07-31T17:05:28.050446Z","iopub.status.idle":"2021-07-31T17:05:28.061523Z","shell.execute_reply.started":"2021-07-31T17:05:28.050419Z","shell.execute_reply":"2021-07-31T17:05:28.060189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Nested Data","metadata":{}},{"cell_type":"code","source":"%%time\n\n# Start with input file path\ninput_file_path = Path('../input/mlb-player-digital-engagement-forecasting')\n\n# Create table with list of CSV files to be read in, w/ corresponding df name\n# This does include large 'train' data set (read in separately)\ncsv_and_df_names = pd.DataFrame(data = {\n 'csv_name': [\n # 'seasons', \n 'teams', \n 'players', \n 'awards',\n # 'example_test', \n # 'example_sample_submission'\n ],\n 'df_name': [\n # 'seasons', \n 'teams', \n 'players', \n 'awards_pre2018',\n # 'example_test', \n # 'example_sample_submission'\n ] \n })\n\nfor index, row in csv_and_df_names.iterrows():\n \n csv_name = row['csv_name']\n df_name = row['df_name']\n \n # Read from CSV and create df with specified name in environment\n globals()[df_name] = pd.read_csv(input_file_path / f\"{csv_name}.csv\")\n\n\ndef unpack_json_dailyDataDate(date_row, df_name):\n daily_df = pd.read_json(date_row[df_name])\n daily_df['dailyDataDate'] = date_row['date']\n return daily_df\n\ntrain = pd.read_csv(input_file_path / 'train_updated.csv')\n\n# Convert training data date field to pandas datetime type\ntrain['date'] = pd.to_datetime(train['date'], format = \"%Y%m%d\")\n\ntrain = train[train['date'] < TEST_START] \n\ndisplay(train.info())\n\n# Get names of all \"nested\" data frames in daily training set\ndaily_data_nested_df_names = pd.DataFrame(data = {\n 'df_name': [\n 'nextDayPlayerEngagement', \n # 'games', \n 'rosters',\n 'playerBoxScores', \n # 'teamBoxScores', \n 'transactions', \n 'standings',\n 'awards', \n # 'events', \n 'playerTwitterFollowers', \n 'teamTwitterFollowers'\n ],\n 'df_rename': [\n 'targets', \n # 'games', \n 'rosters',\n 'scores', \n # 'teamBoxScores', \n 'transactions', \n 'standings',\n 'awards', \n # 'events', \n 'playerTwitterFollowers', \n 'teamTwitterFollowers'\n ]\n })\n\nfor index, row in daily_data_nested_df_names.iterrows():\n df_name = row['df_name']\n df_rename = row['df_rename'] \n\n date_nested_table = train[['date', df_name]]\n\n date_nested_table = (date_nested_table[\n ~pd.isna(date_nested_table[df_name])\n ].\n reset_index(drop = True)\n )\n \n daily_dfs_collection = Parallel(n_jobs=-1)(\n delayed(unpack_json_dailyDataDate)(item, df_name) for date, item in date_nested_table.iterrows())\n \n # for date_index, date_row in date_nested_table.iterrows():\n # daily_df = pd.read_json(date_row[df_name])\n \n # daily_df['dailyDataDate'] = date_row['date']\n \n # daily_dfs_collection = daily_dfs_collection + [daily_df]\n\n # Concatenate all daily dfs into single df for each row\n unnested_table = (pd.concat(daily_dfs_collection,\n ignore_index = True).\n # Set and reset index to move 'dailyDataDate' to front of df\n set_index('dailyDataDate').\n reset_index()\n )\n \n # Creates 1 pandas df per unnested df from daily data read in, with same name\n # Also reduce memory usage\n globals()[df_rename] = reduce_mem_usage(unnested_table) \n \n # Clean up tables and collection of daily data frames for this df\n del(date_nested_table, daily_dfs_collection, unnested_table)\n\ndel train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:05:28.066030Z","iopub.execute_input":"2021-07-31T17:05:28.066610Z","iopub.status.idle":"2021-07-31T17:07:28.115607Z","shell.execute_reply.started":"2021-07-31T17:05:28.066575Z","shell.execute_reply":"2021-07-31T17:07:28.114550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"code","source":"TRAIN_VAL_SPLIT_POINT = '2021-08-01'\nLAG_MONTH = 1","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:07:28.118823Z","iopub.execute_input":"2021-07-31T17:07:28.119427Z","iopub.status.idle":"2021-07-31T17:07:28.124409Z","shell.execute_reply.started":"2021-07-31T17:07:28.119381Z","shell.execute_reply":"2021-07-31T17:07:28.123487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calculate_time_lag(df, time_lag):\n dt_lag = df['dailyDataDate'] - pd.DateOffset(months=time_lag)\n df['year_lag'] = dt_lag.dt.year\n df['month_lag'] = dt_lag.dt.month\n\n return df\n","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:07:28.125792Z","iopub.execute_input":"2021-07-31T17:07:28.126070Z","iopub.status.idle":"2021-07-31T17:07:28.137960Z","shell.execute_reply.started":"2021-07-31T17:07:28.126042Z","shell.execute_reply":"2021-07-31T17:07:28.136785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Transactions\n\n# Xem EDA Transaction ở trên\n\ntypeCode_compact = {key:key if key in ['SFA', 'TR'] else 'OTH' for key in transactions['typeCode']}\ntransactions['typeCode_compact'] = transactions['typeCode'].map(typeCode_compact)\ntransactions_compact = pd.get_dummies(transactions[['dailyDataDate', 'playerId', 'typeCode_compact']], columns=['typeCode_compact']).groupby(['dailyDataDate', 'playerId']).sum().reset_index()\ntransactions_compact","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:07:28.139238Z","iopub.execute_input":"2021-07-31T17:07:28.139779Z","iopub.status.idle":"2021-07-31T17:07:28.253610Z","shell.execute_reply.started":"2021-07-31T17:07:28.139736Z","shell.execute_reply":"2021-07-31T17:07:28.252713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"player_awards_all = (pd.concat([\n # Filter awards from daily df to only those involving tracked players\n awards[np.isin(awards['playerId'], players['playerId'])],\n # Add daily data date to pre-2018 awards_df\n awards_pre2018.assign(\n dailyDataDate = pd.to_datetime(awards_pre2018['awardDate'], \n format = '%Y-%m-%d')\n )], \n ignore_index = True\n ).\n sort_values(['awardDate'], ascending = False, ignore_index = True)\n )\n\ndisplay(player_awards_all)\n\nselected_awards = pd.DataFrame(data = {\n 'awardId': ['ALAS', 'NLAS', 'ALMVP', 'NLMVP', 'ALCY', 'NLCY'],\n 'awardCategory': ['AllStar', 'AllStar', 'MVP', 'MVP', 'CyYoung', 'CyYoung']\n })\n\nplayer_selected_awards = pd.merge(\n player_awards_all,\n selected_awards,\n on = 'awardId',\n # Inner join to limit player awards to only selected ones\n how = 'inner'\n )\n\ndisplay(player_selected_awards)\n\nselected_award_categories_in_data = (player_selected_awards['awardCategory'].\n unique())\n\nplayer_selected_awards_by_date = (player_selected_awards.\n # Add count for use when pivoting\n assign(count = 1).\n pivot_table(\n index = ['dailyDataDate', 'playerId', 'playerName'],\n columns = 'awardCategory',\n values = 'count',\n # NA can be turned to 0 since it means player didn't get that award that day\n fill_value = 0\n ).\n reset_index()\n )\n\n# Add cumulative 'to date' sums for each award category\nfor award_category in selected_award_categories_in_data:\n player_selected_awards_by_date[('toDate' + award_category + 's')] = (\n player_selected_awards_by_date.\n groupby(['playerId', 'playerName'])[award_category].cumsum()\n )\n\n# Prepare for time-based merging by dropping non-\"to date\" fields\nplayer_selected_awards_by_date.drop(selected_award_categories_in_data,\n axis = 1, inplace = True)\n\ndisplay(player_selected_awards_by_date)\n\nplayer_selected_awards_by_date = reduce_mem_usage(player_selected_awards_by_date)\n\n# Get list of awards fields to be added (and fill in NAs for post-merge)\nawards_fields = (player_selected_awards_by_date.\n drop(['dailyDataDate', 'playerId', 'playerName'], axis = 1).\n columns.values.tolist())\n\n# Merge in selected player awards received from latest award date before given date\ntargets_awards = (pd.merge_asof(\n targets,\n player_selected_awards_by_date.drop(['playerName'], axis = 1),\n # \"merge\" on date by player, looking backward (only use award dates up to daily date)\n on = ['dailyDataDate'],\n by = ['playerId'],\n direction = 'backward'\n ).\n # NAs on awards fields can be turned to 0 (player had no awards of that type to date)\n fillna({field: 0 for field in awards_fields})\n )\n\ntargets_awards","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:07:28.255163Z","iopub.execute_input":"2021-07-31T17:07:28.255483Z","iopub.status.idle":"2021-07-31T17:07:29.812245Z","shell.execute_reply.started":"2021-07-31T17:07:28.255452Z","shell.execute_reply":"2021-07-31T17:07:29.811147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Scores\n\nscores\n\n# Có một số cầu thủ chơi 2 trận 1 ngày\n\nscores[scores.groupby(['playerId', 'dailyDataDate'])['gamePk'].transform('count') > 1].sort_values(by=['playerId', 'dailyDataDate'])\n\n# ### mark game\n# scores['hasGame'] = 1\n\nscores = scores.groupby(['playerId', 'dailyDataDate']).sum().reset_index()","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:07:29.813719Z","iopub.execute_input":"2021-07-31T17:07:29.814060Z","iopub.status.idle":"2021-07-31T17:07:30.948101Z","shell.execute_reply.started":"2021-07-31T17:07:29.814021Z","shell.execute_reply":"2021-07-31T17:07:30.947015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Targets\n\n# Tính thống kê của target tháng trước để làm feature cho tháng sau\n\ntargets\n\ntargets_copy = targets.copy()\n\ndt_stats = pd.to_datetime(targets['dailyDataDate'], format='%Y%m%d')\n\ntargets_copy['year_stats'] = dt_stats.dt.year\ntargets_copy['month_stats'] = dt_stats.dt.month\n\ntargets_stats = targets_copy.groupby(['playerId', 'year_stats', 'month_stats'])['target1', 'target2', 'target3', 'target4'].agg(['mean', 'median', 'min', 'max', 'std'])\ntargets_stats.columns = ['_'.join(c) for c in targets_stats.columns]\ntargets_stats.reset_index(inplace=True)\n\ndisplay(targets_stats)\n\ndel targets_copy\ngc.collect()\n\nLAG_MONTH = 1\n\ntargets = calculate_time_lag(targets, LAG_MONTH)\ntargets\n\n# display(dt_lag[('2021-02-01'<=dt_lag) & (dt_lag<'2021-03-01')])\n# display(targets.iloc[2380455:2444345+1])\n\nstats = pd.merge(targets[['dailyDataDate', 'playerId', 'year_lag', 'month_lag']], targets_stats,\n left_on=['playerId', 'year_lag', 'month_lag'], right_on=['playerId', 'year_stats', 'month_stats'])\n\nstats.drop(columns=['year_lag', 'month_lag'], inplace=True)\n\ndisplay(stats)\n\ndict_agg = {'target1_mean':'mean', \n 'target2_mean':'mean',\n 'target3_mean':'mean', \n 'target4_mean':'mean',\n # 'target1_min':'min', \n # 'target2_min':'min', \n # 'target3_min':'min', \n # 'target4_min':'min',\n # 'target1_max':'max', \n # 'target2_max':'max', \n # 'target3_max':'max', \n # 'target4_max':'max', \n }\n\ntargets_stats_expanding = targets_stats.set_index(['year_stats', 'month_stats']).groupby('playerId').expanding().agg(dict_agg)\ntargets_stats_expanding.columns = ['expand_' + c for c in targets_stats_expanding.columns]\ntargets_stats_expanding.reset_index(inplace=True)\n\ntargets_stats_expanding\n\nstats_expanding = pd.merge(targets[['dailyDataDate', 'playerId', 'year_lag', 'month_lag']], targets_stats_expanding,\n left_on=['playerId', 'year_lag', 'month_lag'], right_on=['playerId', 'year_stats', 'month_stats'])\n\nstats_expanding.drop(columns=['year_lag', 'month_lag'], inplace=True)\n\ndisplay(stats_expanding)\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:07:30.949785Z","iopub.execute_input":"2021-07-31T17:07:30.950198Z","iopub.status.idle":"2021-07-31T17:07:37.929692Z","shell.execute_reply.started":"2021-07-31T17:07:30.950160Z","shell.execute_reply":"2021-07-31T17:07:37.928566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"playerTwitterFollowers_prev = playerTwitterFollowers[['playerId', 'dailyDataDate', 'numberOfFollowers']].copy()\n\ndt_playerTwitterFollowers = pd.to_datetime(playerTwitterFollowers_prev['dailyDataDate'], format='%Y%m%d')\n\nplayerTwitterFollowers_prev['year_stats'] = dt_playerTwitterFollowers.dt.year\nplayerTwitterFollowers_prev['month_stats'] = dt_playerTwitterFollowers.dt.month\nplayerTwitterFollowers_prev.rename(columns = {'numberOfFollowers':'playernumberOfFollowers_prev'}, inplace=True)\ndisplay(playerTwitterFollowers_prev)\n\n\nteamTwitterFollowers_prev = teamTwitterFollowers[['teamId', 'dailyDataDate', 'numberOfFollowers']].copy()\n\ndt_teamTwitterFollowers = pd.to_datetime(teamTwitterFollowers_prev['dailyDataDate'], format='%Y%m%d')\nteamTwitterFollowers_prev['year_stats'] = dt_teamTwitterFollowers.dt.year\nteamTwitterFollowers_prev['month_stats'] = dt_teamTwitterFollowers.dt.month\nteamTwitterFollowers_prev.rename(columns = {'numberOfFollowers':'teamnumberOfFollowers_prev'}, inplace=True)\ndisplay(teamTwitterFollowers_prev)\n\nplayerTwitterFollowers_cum = playerTwitterFollowers_prev.set_index(['dailyDataDate', 'year_stats', 'month_stats'])\\\n .groupby('playerId').expanding()\\\n .agg({'playernumberOfFollowers_prev':['mean']})\nplayerTwitterFollowers_cum.columns = ['_'.join(c) for c in playerTwitterFollowers_cum.columns]\nplayerTwitterFollowers_cum.reset_index(inplace=True)\nplayerTwitterFollowers_cum\n\nteamTwitterFollowers_cum = teamTwitterFollowers_prev.set_index(['dailyDataDate', 'year_stats', 'month_stats'])\\\n .groupby('teamId').expanding()\\\n .agg({'teamnumberOfFollowers_prev':['mean']})\nteamTwitterFollowers_cum.columns = ['_'.join(c) for c in teamTwitterFollowers_cum.columns]\nteamTwitterFollowers_cum.reset_index(inplace=True)\nteamTwitterFollowers_cum","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:07:37.931022Z","iopub.execute_input":"2021-07-31T17:07:37.931317Z","iopub.status.idle":"2021-07-31T17:07:38.102489Z","shell.execute_reply.started":"2021-07-31T17:07:37.931287Z","shell.execute_reply":"2021-07-31T17:07:38.097099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets_cols = [\n 'playerId', \n 'target1', \n 'target2', \n 'target3', \n 'target4', \n 'dailyDataDate',\n # 'day',\n # 'weekday'\n]\n\nplayers_cols = [\n 'playerId', \n 'primaryPositionName'\n]\n\nteams_cols = [\n 'teamId', \n# 'name', \n# 'teamName', \n# 'teamCode', \n# 'shortName', \n# 'abbreviation', \n# 'locationName', \n 'leagueId', \n# 'leagueName', \n 'divisionId', \n# 'divisionName', \n# 'venueId', \n# 'venueName'\n]\n\nrosters_cols = [\n 'playerId', \n 'teamId', \n 'status', \n 'dailyDataDate'\n]\n\nscores_cols = [\n 'playerId',\n # 'hasGame',\n 'battingOrder', \n 'gamesPlayedBatting', \n 'flyOuts',\n 'groundOuts', \n 'runsScored', \n 'doubles', \n 'triples', \n 'homeRuns',\n 'strikeOuts', \n 'baseOnBalls', \n 'intentionalWalks', \n 'hits', \n 'hitByPitch',\n 'atBats', \n 'caughtStealing', \n 'stolenBases', \n 'groundIntoDoublePlay',\n 'groundIntoTriplePlay', \n 'plateAppearances', \n 'totalBases', \n 'rbi',\n 'leftOnBase', \n 'sacBunts', \n 'sacFlies', \n 'catchersInterference',\n 'pickoffs', \n 'gamesPlayedPitching', \n 'gamesStartedPitching',\n 'completeGamesPitching', \n 'shutoutsPitching', \n 'winsPitching',\n 'lossesPitching', \n 'flyOutsPitching', \n 'airOutsPitching',\n 'groundOutsPitching', \n 'runsPitching', \n 'doublesPitching',\n 'triplesPitching', \n 'homeRunsPitching', \n 'strikeOutsPitching',\n 'baseOnBallsPitching', \n 'intentionalWalksPitching', \n 'hitsPitching',\n 'hitByPitchPitching', \n 'atBatsPitching', \n 'caughtStealingPitching',\n 'stolenBasesPitching', \n 'inningsPitched', \n 'saveOpportunities',\n 'earnedRuns', \n 'battersFaced', \n 'outsPitching', \n 'pitchesThrown', \n 'balls',\n 'strikes', \n 'hitBatsmen', \n 'balks', \n 'wildPitches', \n 'pickoffsPitching',\n 'rbiPitching', \n 'gamesFinishedPitching', \n 'inheritedRunners',\n 'inheritedRunnersScored', \n 'catchersInterferencePitching',\n 'sacBuntsPitching', \n 'sacFliesPitching', \n 'saves', \n 'holds', \n 'blownSaves',\n 'assists', \n 'putOuts', \n 'errors', \n 'chances', \n 'dailyDataDate',\n # 'isWin',\n # 'isLoss',\n # 'isTie',\n # 'gameType_A',\n # 'gameType_D',\n # 'gameType_E',\n # 'gameType_F',\n # 'gameType_L',\n # 'gameType_R',\n # 'gameType_S',\n # 'gameType_W', \n]\n\nawards_cols = [\n 'dailyDataDate', \n 'playerId',\n 'awardId'\n]\n\ntargets_awards_cols = ['dailyDataDate',\n 'playerId',\n 'toDateMVPs',\n # 'toDateCyYoungs',\n 'toDateAllStars'\n ]\n\nplayerTwitterFollowers_cols = [\n 'playerId', \n 'numberOfFollowers'\n]\n\nteamTwitterFollowers_cols = [\n 'teamId', \n 'numberOfFollowers'\n]\n\nstandings_cols = [\n 'teamId', \n# 'wildCardRank', \n 'wins', \n 'losses', \n# 'divisionChamp', \n# 'divisionLeader', \n# 'wildCardLeader', \n 'lastTenWins',\n 'lastTenLosses',\n 'dailyDataDate'\n]\n\nstats_col = ['dailyDataDate', 'playerId', 'target1_mean',\n 'target1_median', 'target1_min', 'target1_max', 'target1_std',\n 'target2_mean', 'target2_median', 'target2_min', 'target2_max',\n 'target2_std', 'target3_mean', 'target3_median', 'target3_min',\n 'target3_max', 'target3_std', 'target4_mean', 'target4_median',\n 'target4_min', 'target4_max', 'target4_std']\n\nstats_expanding_col = ['dailyDataDate', 'playerId',\n 'expand_target1_mean', 'expand_target2_mean', 'expand_target3_mean', 'expand_target4_mean', \n # 'expand_target1_min', 'expand_target2_min', 'expand_target3_min', 'expand_target4_min', \n # 'expand_target1_max', 'expand_target2_max', 'expand_target3_max', 'expand_target4_max'\n ]\n\nplayerTwitterFollowers_prev_cols = ['playerId',\t'playernumberOfFollowers_prev',\t'year_stats', 'month_stats']\nteamTwitterFollowers_prev_cols = ['teamId',\t'teamnumberOfFollowers_prev',\t'year_stats', 'month_stats']\n\nplayerTwitterFollowers_prev_cum_cols = ['playerId',\t'playernumberOfFollowers_prev_mean',\t'year_stats', 'month_stats']\nteamTwitterFollowers_prev_cum_cols = ['teamId',\t'teamnumberOfFollowers_prev_mean',\t'year_stats', 'month_stats']\n\ntransactions_cols = ['dailyDataDate', 'playerId', 'toTeamId', 'typeCode']\ntransactions_compact_fillna_cols = ['typeCode_compact_OTH','typeCode_compact_SFA','typeCode_compact_TR']\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:07:38.104464Z","iopub.execute_input":"2021-07-31T17:07:38.104793Z","iopub.status.idle":"2021-07-31T17:07:38.123851Z","shell.execute_reply.started":"2021-07-31T17:07:38.104759Z","shell.execute_reply":"2021-07-31T17:07:38.123016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create dataset\n\ntrain = targets.copy()[targets_cols]\ntrain = calculate_time_lag(train, time_lag=LAG_MONTH)\n\nprint(targets[targets_cols].shape)\n\n# train = train.merge(datetime_feat, on=['dailyDataDate'], how='left')\n# gc.collect()\n\n# print(train.shape, 'after_datetime')\n# print('--------------------------------------')\n\ntrain = train.merge(players[players_cols], on=['playerId'], how='left')\ngc.collect()\n\nprint(train.shape, 'after_players')\nprint('--------------------------------------')\n\ntrain = train.merge(rosters[rosters_cols], on=['playerId', 'dailyDataDate'], how='left')\ngc.collect()\n\nprint(train.shape, 'after_rosters')\nprint('--------------------------------------')\n\ntrain = train.merge(scores[scores_cols], on=['playerId', 'dailyDataDate'], how='left')\ngc.collect()\n\nprint(train.shape, 'after_scores')\nprint('--------------------------------------')\n\nteams = teams.rename(columns = {'id':'teamId'})\ntrain = train.merge(teams[teams_cols], on = 'teamId', how='left')\n\nprint(train.shape, 'after_teams')\nprint('--------------------------------------')\n\ntrain = train.merge(standings[standings_cols], on = ['teamId', 'dailyDataDate'], how = 'left')\ngc.collect()\n\nprint(train.shape, 'after_standings')\nprint('--------------------------------------')\n\ntrain = train.merge(stats[stats_col], on=['playerId', 'dailyDataDate'], how='left')\nprint(train.shape, 'after_stats')\nprint('--------------------------------------')\n\ntrain = train.merge(stats_expanding[stats_expanding_col], on=['playerId', 'dailyDataDate'], how='left')\nprint(train.shape, 'after_expanding_stats')\nprint('--------------------------------------')\n\ntrain = calculate_time_lag(train, time_lag=LAG_MONTH)\n\n# train = train.merge(awards_count_prev, \n# left_on=['playerId', 'year_lag', 'month_lag'], \n# right_on=['playerId', 'year_stats', 'month_stats'], how='left')\n\n# train.drop(columns=['year_lag', 'month_lag', 'year_stats', 'month_stats'], inplace=True)\n# print(train.shape, 'after_stats_awards')\n# print('--------------------------------------')\n\ntrain = train.merge(playerTwitterFollowers_cum[playerTwitterFollowers_prev_cum_cols],\n left_on=['playerId', 'year_lag', 'month_lag'],\n right_on=['playerId', 'year_stats', 'month_stats'], how='left')\n\ntrain.drop(columns=['year_stats', 'month_stats'], inplace=True)\nprint(train.shape, 'after_stats_playerfollowers')\nprint('--------------------------------------')\n\ntrain = train.merge(teamTwitterFollowers_cum[teamTwitterFollowers_prev_cum_cols],\n left_on=['teamId', 'year_lag', 'month_lag'], \n right_on=['teamId', 'year_stats', 'month_stats'], how='left')\n\n\n\ntrain.drop(columns=['year_stats', 'month_stats'], inplace=True)\nprint(train.shape, 'after_stats_teamfollowers')\nprint('--------------------------------------')\n\ntrain = train.merge(transactions_compact, on = ['playerId', 'dailyDataDate'], how = 'left')\n## nếu không fillna, khi test gặp trường hợp full 0 mà train không có (trong ngày không có bất kỳ transaction nào)\ntrain[transactions_compact_fillna_cols] = train[transactions_compact_fillna_cols].fillna(0)\ngc.collect()\n\n\nprint(train.shape, 'after_transactions')\nprint('--------------------------------------')\n\ntrain = train.merge(targets_awards[targets_awards_cols], on = ['playerId', 'dailyDataDate'], how = 'left')\ngc.collect()\n\nprint(train.shape, 'after_awards')\nprint('--------------------------------------')\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:07:38.125767Z","iopub.execute_input":"2021-07-31T17:07:38.126166Z","iopub.status.idle":"2021-07-31T17:08:59.212011Z","shell.execute_reply.started":"2021-07-31T17:07:38.126129Z","shell.execute_reply":"2021-07-31T17:08:59.210872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['day'] = train['dailyDataDate'].dt.day\ntrain['weekday'] = train['dailyDataDate'].dt.weekday\n","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:08:59.213354Z","iopub.execute_input":"2021-07-31T17:08:59.213656Z","iopub.status.idle":"2021-07-31T17:08:59.768803Z","shell.execute_reply.started":"2021-07-31T17:08:59.213613Z","shell.execute_reply":"2021-07-31T17:08:59.767724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# label encoding\nplayer2num = {c: i for i, c in enumerate(train['playerId'].unique())}\nposition2num = {c: i for i, c in enumerate(train['primaryPositionName'].unique())}\nteamid2num = {c: i for i, c in enumerate(train['teamId'].unique())}\nstatus2num = {c: i for i, c in enumerate(train['status'].unique())}\nleagueId2num = {c: i for i, c in enumerate(train['leagueId'].unique())}\ndivisionId2num = {c: i for i, c in enumerate(train['divisionId'].unique())}\n\ntrain['label_playerId'] = train['playerId'].map(player2num)\ntrain['label_primaryPositionName'] = train['primaryPositionName'].map(position2num)\ntrain['label_teamId'] = train['teamId'].map(teamid2num)\ntrain['label_status'] = train['status'].map(status2num)\ntrain['label_leagueId'] = train['leagueId'].map(leagueId2num)\ntrain['label_divisionId'] = train['divisionId'].map(divisionId2num)","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:08:59.770546Z","iopub.execute_input":"2021-07-31T17:08:59.771014Z","iopub.status.idle":"2021-07-31T17:09:01.599416Z","shell.execute_reply.started":"2021-07-31T17:08:59.770971Z","shell.execute_reply":"2021-07-31T17:09:01.598261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_cols = [c for c in train.columns if c not in ['playerId', 'target1', 'target2', 'target3', 'target4',\n 'dailyDataDate', 'year_lag', 'month_lag',\n 'primaryPositionName', 'teamId', 'status', 'leagueId', 'divisionId', 'typeCode', 'toTeamId']]\n\ncategorical_features = [\n # 'label_status_lag1', 'label_status_lag2', 'label_status_lag3', 'label_status_lag4', 'label_status_lag5', \n\n 'label_playerId', \n 'label_primaryPositionName', \n 'label_teamId', \n # 'label_toTeamId',\n 'label_status',\n 'label_leagueId',\n 'label_divisionId',\n # 'label_typeCode'\n ]","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:09:01.601296Z","iopub.execute_input":"2021-07-31T17:09:01.601736Z","iopub.status.idle":"2021-07-31T17:09:01.610050Z","shell.execute_reply.started":"2021-07-31T17:09:01.601689Z","shell.execute_reply":"2021-07-31T17:09:01.608849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X = train[feature_cols]\ndisplay(train_X)\n\ntrain_y = train[['target1', 'target2', 'target3', 'target4']]\ndisplay(train_y)\n","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:09:01.611618Z","iopub.execute_input":"2021-07-31T17:09:01.612065Z","iopub.status.idle":"2021-07-31T17:09:04.608212Z","shell.execute_reply.started":"2021-07-31T17:09:01.612019Z","shell.execute_reply":"2021-07-31T17:09:04.607071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_PLAYER_IDS = players[players['playerForTestSetAndFuturePreds'].fillna(False)]['playerId'].reset_index(drop=True)\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:09:04.609855Z","iopub.execute_input":"2021-07-31T17:09:04.610292Z","iopub.status.idle":"2021-07-31T17:09:04.618593Z","shell.execute_reply.started":"2021-07-31T17:09:04.610248Z","shell.execute_reply":"2021-07-31T17:09:04.617514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = train_X[train['dailyDataDate'] < TRAIN_VAL_SPLIT_POINT].reset_index(drop=True)\ny_train = train_y[train['dailyDataDate'] < TRAIN_VAL_SPLIT_POINT].reset_index(drop=True)\nx_val = train_X[(train['dailyDataDate'] >= TRAIN_VAL_SPLIT_POINT)&(train['dailyDataDate'] < TEST_START)].reset_index(drop=True)\ny_val = train_y[(train['dailyDataDate'] >= TRAIN_VAL_SPLIT_POINT)&(train['dailyDataDate'] < TEST_START)].reset_index(drop=True)\n\ndisplay(x_train) \ndisplay(y_train)\ndisplay(x_val) \ndisplay(y_val)","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:09:04.620101Z","iopub.execute_input":"2021-07-31T17:09:04.620674Z","iopub.status.idle":"2021-07-31T17:09:08.706528Z","shell.execute_reply.started":"2021-07-31T17:09:04.620612Z","shell.execute_reply":"2021-07-31T17:09:08.705342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load model","metadata":{"_uuid":"e60cda2f-bfec-4ee0-bad3-2dd31d3bf01f","_cell_guid":"d3ae1cee-7d6a-4edb-9b38-b53147702f75","trusted":true}},{"cell_type":"code","source":"model1 = lgbm.Booster(model_file='../input/mlb-finalv3/lgbm_finalv3_target1_0730_0036.txt')\nmodel2 = lgbm.Booster(model_file='../input/mlb-finalv3/lgbm_finalv3_target2_0730_0036.txt')\nmodel3 = lgbm.Booster(model_file='../input/mlb-finalv3/lgbm_finalv3_target3_0730_0036.txt')\nmodel4 = lgbm.Booster(model_file='../input/mlb-finalv3/lgbm_finalv3_target4_0730_0036.txt')","metadata":{"_uuid":"248b22e4-072a-4d45-879f-c5e71ffb34a1","_cell_guid":"6f3d48b3-e4e3-4bcb-91b9-500999dc0ea8","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T17:09:08.708038Z","iopub.execute_input":"2021-07-31T17:09:08.708335Z","iopub.status.idle":"2021-07-31T17:09:08.866963Z","shell.execute_reply.started":"2021-07-31T17:09:08.708304Z","shell.execute_reply":"2021-07-31T17:09:08.865992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_model1 = catboost.CatBoostRegressor().load_model('../input/mlb-finalv3/cat_final_v3_target1_0730_0228.json', format='json')\ncat_model2 = catboost.CatBoostRegressor().load_model('../input/mlb-finalv3/cat_final_v3_target2_0730_0228.json', format='json')\ncat_model3 = catboost.CatBoostRegressor().load_model('../input/mlb-finalv3/cat_final_v3_target3_0730_0228.json', format='json')\ncat_model4 = catboost.CatBoostRegressor().load_model('../input/mlb-finalv3/cat_final_v3_target4_0730_0228.json', format='json')","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:09:08.868479Z","iopub.execute_input":"2021-07-31T17:09:08.869054Z","iopub.status.idle":"2021-07-31T17:09:09.322405Z","shell.execute_reply.started":"2021-07-31T17:09:08.869013Z","shell.execute_reply":"2021-07-31T17:09:09.321495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model1_ft_tgt1 = lgbm.Booster(model_file='../input/mlb-finalv3/lgbm_final_ft_tgt1_v3_target1_0730_0059.txt')\nmodel2_ft_tgt1 = lgbm.Booster(model_file='../input/mlb-finalv3/lgbm_final_ft_tgt1_v3_target2_0730_0059.txt')\nmodel3_ft_tgt1 = lgbm.Booster(model_file='../input/mlb-finalv3/lgbm_final_ft_tgt1_v3_target3_0730_0059.txt')\nmodel4_ft_tgt1 = lgbm.Booster(model_file='../input/mlb-finalv3/lgbm_final_ft_tgt1_v3_target4_0730_0059.txt')","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:09:09.323906Z","iopub.execute_input":"2021-07-31T17:09:09.324230Z","iopub.status.idle":"2021-07-31T17:09:09.482792Z","shell.execute_reply.started":"2021-07-31T17:09:09.324198Z","shell.execute_reply":"2021-07-31T17:09:09.481843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # score1 = mean_absolute_error(model1.predict(x_val), y_val['target1'])\n# score2 = mean_absolute_error(model2.predict(x_val), y_val['target2'])\n# score3 = mean_absolute_error(model3.predict(x_val), y_val['target3'])\n# score4 = mean_absolute_error(model4.predict(x_val), y_val['target4'])\n\n# score = (score1+score2+score3+score4) / 4\n\n# print({'score1': score1, 'score2': score2, 'score3': score3, 'score4': score4, 'score': score})\n\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:09:09.484225Z","iopub.execute_input":"2021-07-31T17:09:09.484823Z","iopub.status.idle":"2021-07-31T17:09:09.910748Z","shell.execute_reply.started":"2021-07-31T17:09:09.484784Z","shell.execute_reply":"2021-07-31T17:09:09.909082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# score1 = mean_absolute_error(cat_model1.predict(x_val.fillna(-1)), y_val['target1'])\n# score2 = mean_absolute_error(cat_model2.predict(x_val.fillna(-1)), y_val['target2'])\n# score3 = mean_absolute_error(cat_model3.predict(x_val.fillna(-1)), y_val['target3'])\n# score4 = mean_absolute_error(cat_model4.predict(x_val.fillna(-1)), y_val['target4'])\n\n# score = (score1+score2+score3+score4) / 4\n\n# print({'score1': score1, 'score2': score2, 'score3': score3, 'score4': score4, 'score': score})\n\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:09:09.911923Z","iopub.status.idle":"2021-07-31T17:09:09.912623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# player2num = load_le('../input/mlb-labelencoding/player2num.pickle')\n# position2num = load_le('../input/mlb-labelencoding/position2num.pickle')\n# status2num = load_le('../input/mlb-labelencoding/status2num.pickle')\n# teamid2num = load_le('../input/mlb-labelencoding/teamid2num.pickle')","metadata":{"_uuid":"ade0fd51-9b81-45d7-9c9a-fb18a510d202","_cell_guid":"1ef36780-e973-41c1-b832-9288c8d8a8c5","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T17:09:09.913709Z","iopub.status.idle":"2021-07-31T17:09:09.914335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Modelling\n\n# def fit_lgbm(x_train, y_train, x_val, y_val, params: dict=None, categorical_feature=None, verbose=100):\n# oof_pred = np.zeros(len(y_val), dtype=np.float32)\n# model = lgb.LGBMRegressor(**params)\n# model.fit(x_train, y_train, \n# eval_set=[(x_val, y_val)],\n# categorical_feature=categorical_feature,\n# early_stopping_rounds=verbose, \n# verbose=verbose)\n# oof_pred = model.predict(x_val)\n# score = mean_absolute_error(oof_pred, y_val)\n# print('mae:', score)\n# return oof_pred, model, score\n\n# # training lightgbm\n# params1 = {\n# 'objective':'mae',\n# 'reg_alpha': 0.1,\n# 'reg_lambda': 0.1, \n# 'n_estimators': 10000,\n# 'learning_rate': 0.1,\n# 'random_state': 42,\n# \"num_leaves\": 100\n# }\n\n# params2 = {\n# 'objective':'mae',\n# 'reg_alpha': 0.1,\n# 'reg_lambda': 0.1, \n# 'n_estimators': 80,\n# 'learning_rate': 0.1,\n# 'random_state': 42,\n# \"num_leaves\": 22\n# }\n\n# params3 = {\n# 'objective':'mae',\n# 'reg_alpha': 0.1,\n# 'reg_lambda': 0.1, \n# 'n_estimators': 10000,\n# 'learning_rate': 0.1,\n# 'random_state': 42,\n# \"num_leaves\": 100\n# }\n\n# params4 = {\n# 'objective':'mae',\n# 'reg_alpha': 0.1,\n# 'reg_lambda': 0.1, \n# 'n_estimators': 10000,\n# 'learning_rate': 0.1,\n# 'random_state': 42,\n# \"num_leaves\": 100\n# }\n\n\n# oof1, model1, score1 = fit_lgbm(\n# x_train, y_train['target1'],\n# x_val, y_val['target1'],\n# params1,\n# categorical_feature=categorical_features\n# )\n\n# oof2, model2, score2 = fit_lgbm(\n# x_train, y_train['target2'],\n# x_val, y_val['target2'],\n# params2,\n# categorical_feature=categorical_features\n# )\n\n# oof3, model3, score3 = fit_lgbm(\n# x_train, y_train['target3'],\n# x_val, y_val['target3'],\n# params3,\n# categorical_feature=categorical_features\n# )\n\n# oof4, model4, score4 = fit_lgbm(\n# x_train, y_train['target4'],\n# x_val, y_val['target4'],\n# params4,\n# categorical_feature=categorical_features\n# )\n\n# score = (score1+score2+score3+score4) / 4\n# print(f'score: {score}')\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:09:09.915332Z","iopub.status.idle":"2021-07-31T17:09:09.915957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig_name1 = 'feat_imp_1'\n\n# feature_imp1 = pd.DataFrame(sorted(zip(model1.feature_importances_, x_train[feature_cols])), \n# columns=['Value','Feature'])\n\n# # plt.figure(figsize=(20, 15))\n# # sns.barplot(x=\"Value\", y=\"Feature\", data=feature_imp1.sort_values(by=\"Value\", ascending=False))\n# # plt.title('LightGBM Features (avg over folds)')\n# # plt.tight_layout()\n# # plt.show()\n\n# # plt.savefig(os.path.join('/content/drive/Shareddrives/MLB Player Digital Engagement Forecasting/Report', fig_name1))\n\n# # print (feature_imp['Feature'].tolist())\n# # feature_imp.tail(20)\n\n# fig = px.bar(feature_imp1.sort_values(by=\"Value\", ascending=True), x='Value', y='Feature', orientation='h')\n# fig.show()\n\n# hist = {'score1':[score1], 'score2':[score2], 'score3':[score3], 'score4':[score4], 'score':[score]}\n\n# # log(hist, 'baseline_lgbm', '/content/drive/Shareddrives/MLB Player Digital Engagement Forecasting/Log')\n\n# # LGBM_PATH = '/content/drive/Shareddrives/MLB Player Digital Engagement Forecasting/weights/lgbm_models'\n\n# # submit_lgb_model(model1, 'baseline_target1', LGBM_PATH)\n# # submit_lgb_model(model2, 'baseline_target2', LGBM_PATH)\n# # submit_lgb_model(model3, 'baseline_target3', LGBM_PATH)\n# # submit_lgb_model(model4, 'baseline_target4', LGBM_PATH)\n","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:09:09.917027Z","iopub.status.idle":"2021-07-31T17:09:09.917661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference","metadata":{"_uuid":"68e3c939-0436-4536-b441-df77f03e3b1f","_cell_guid":"be3effcf-c08c-42ed-8407-9df2dd290ff5","trusted":true}},{"cell_type":"code","source":"targets_cols = [\n 'playerId', \n 'target1', \n 'target2', \n 'target3', \n 'target4', \n 'dailyDataDate'\n]\n\nplayers_cols = [\n 'playerId', \n 'primaryPositionName'\n]\n\nteams_cols = [\n 'teamId', \n# 'name', \n# 'teamName', \n# 'teamCode', \n# 'shortName', \n# 'abbreviation', \n# 'locationName', \n 'leagueId', \n# 'leagueName', \n 'divisionId', \n# 'divisionName', \n# 'venueId', \n# 'venueName'\n]\n\nrosters_cols = [\n 'playerId', \n 'teamId', \n 'status', \n # 'dailyDataDate'\n]\n\nscores_cols = [\n 'playerId',\n # 'hasGame',\n 'battingOrder', \n 'gamesPlayedBatting', \n 'flyOuts',\n 'groundOuts', \n 'runsScored', \n 'doubles', \n 'triples', \n 'homeRuns',\n 'strikeOuts', \n 'baseOnBalls', \n 'intentionalWalks', \n 'hits', \n 'hitByPitch',\n 'atBats', \n 'caughtStealing', \n 'stolenBases', \n 'groundIntoDoublePlay',\n 'groundIntoTriplePlay', \n 'plateAppearances', \n 'totalBases', \n 'rbi',\n 'leftOnBase', \n 'sacBunts', \n 'sacFlies', \n 'catchersInterference',\n 'pickoffs', \n 'gamesPlayedPitching', \n 'gamesStartedPitching',\n 'completeGamesPitching', \n 'shutoutsPitching', \n 'winsPitching',\n 'lossesPitching', \n 'flyOutsPitching', \n 'airOutsPitching',\n 'groundOutsPitching', \n 'runsPitching', \n 'doublesPitching',\n 'triplesPitching', \n 'homeRunsPitching', \n 'strikeOutsPitching',\n 'baseOnBallsPitching', \n 'intentionalWalksPitching', \n 'hitsPitching',\n 'hitByPitchPitching', \n 'atBatsPitching', \n 'caughtStealingPitching',\n 'stolenBasesPitching', \n 'inningsPitched', \n 'saveOpportunities',\n 'earnedRuns', \n 'battersFaced', \n 'outsPitching', \n 'pitchesThrown', \n 'balls',\n 'strikes', \n 'hitBatsmen', \n 'balks', \n 'wildPitches', \n 'pickoffsPitching',\n 'rbiPitching', \n 'gamesFinishedPitching', \n 'inheritedRunners',\n 'inheritedRunnersScored', \n 'catchersInterferencePitching',\n 'sacBuntsPitching', \n 'sacFliesPitching', \n 'saves', \n 'holds', \n 'blownSaves',\n 'assists', \n 'putOuts', \n 'errors', \n 'chances', \n # 'dailyDataDate'\n]\n\nawards_cols = [\n 'dailyDataDate', \n 'playerId',\n 'awardId'\n]\n\nplayerTwitterFollowers_cols = [\n 'playerId', \n 'numberOfFollowers'\n]\n\nteamTwitterFollowers_cols = [\n 'teamId', \n 'numberOfFollowers'\n]\n\ntargets_awards_cols = [\n # 'dailyDataDate',\n 'playerId',\n 'toDateMVPs',\n # 'toDateCyYoungs',\n 'toDateAllStars']\n\nstandings_cols = [\n 'teamId', \n# 'wildCardRank', \n 'wins', \n 'losses', \n# 'divisionChamp', \n# 'divisionLeader', \n# 'wildCardLeader', \n 'lastTenWins',\n 'lastTenLosses',\n # 'dailyDataDate'\n]\n\nstats_col = ['playerId', 'target1_mean',\n 'target1_median', 'target1_min', 'target1_max', 'target1_std',\n 'target2_mean', 'target2_median', 'target2_min', 'target2_max',\n 'target2_std', 'target3_mean', 'target3_median', 'target3_min',\n 'target3_max', 'target3_std', 'target4_mean', 'target4_median',\n 'target4_min', 'target4_max', 'target4_std']\n\nstats_expanding_col = ['playerId',\n 'expand_target1_mean', 'expand_target2_mean', 'expand_target3_mean', 'expand_target4_mean']\n\n\n# playerTwitterFollowers_prev_cols = ['playerId',\t'playernumberOfFollowers_prev',\t'year_stats', 'month_stats']\n# teamTwitterFollowers_prev_cols = ['teamId',\t'teamnumberOfFollowers_prev',\t'year_stats', 'month_stats']\n\nplayerTwitterFollowers_prev_cum_cols = ['playerId',\t'playernumberOfFollowers_prev_mean',\t'year_stats', 'month_stats']\nteamTwitterFollowers_prev_cum_cols = ['teamId',\t'teamnumberOfFollowers_prev_mean',\t'year_stats', 'month_stats']\n\n\ntransactions_cols = ['dailyDataDate', 'playerId', 'toTeamId', 'typeCode']","metadata":{"_uuid":"4223a8a8-0b50-40a0-87d8-91adcd189f14","_cell_guid":"e82a0707-d440-485e-9826-19a55da3aeae","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-31T17:09:09.918659Z","iopub.status.idle":"2021-07-31T17:09:09.919263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_cols = [c for c in train.columns if c not in ['playerId', 'target1', 'target2', 'target3', 'target4',\n 'dailyDataDate', 'year_lag', 'month_lag',\n 'primaryPositionName', 'teamId', 'status', 'leagueId', 'divisionId']]\n\ncategorical_features = [\n # 'label_status_lag1', 'label_status_lag2', 'label_status_lag3', 'label_status_lag4', 'label_status_lag5', \n 'label_playerId', 'label_primaryPositionName', 'label_teamId', 'label_status',\n 'label_leagueId', 'label_divisionId']","metadata":{"execution":{"iopub.status.busy":"2021-07-31T17:09:09.920200Z","iopub.status.idle":"2021-07-31T17:09:09.920829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_targets_awards = targets_awards[targets_awards['dailyDataDate']