{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport warnings\nwarnings.simplefilter(\"ignore\")\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-17T06:56:59.492531Z","iopub.execute_input":"2021-06-17T06:56:59.493178Z","iopub.status.idle":"2021-06-17T06:56:59.529905Z","shell.execute_reply.started":"2021-06-17T06:56:59.493056Z","shell.execute_reply":"2021-06-17T06:56:59.528600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nimport tqdm\n\nbase_path = (\"/kaggle/input/mlb-player-digital-engagement-forecasting/\")\npath = (\"/kaggle/input/mlb-unnested/\")\npath2 = (\"/kaggle/input/mlb-pdef-train-dataset/\")\n\n# players = pd.read_pickle(path + \"players.pickle\")\n# teams   = pd.read_pickle(path + \"teams.pickle\")\n# seasons = pd.read_pickle(path + \"seasons.pickle\")\n# awards  = pd.read_pickle(path + \"awards.pickle\")\n# sample_sub = pd.read_pickle(path + \"example_sample_submission.pickle\")\n\nplayers = pd.read_csv(base_path + \"players.csv\")\nteams   = pd.read_csv(base_path + \"teams.csv\")\nseasons = pd.read_csv(base_path + \"seasons.csv\")\nawards  = pd.read_csv(base_path + \"awards.csv\")\nsample_sub = pd.read_csv(base_path + \"example_sample_submission.csv\")\n\n# train_awards = pd.read_pickle(path + \"train_awards.pickle\")\n# train_events = pd.read_pickle(path + \"train_events.pickle\")\n# train_games  = pd.read_pickle(path + \"train_games.pickle\")\n# train_rosters = pd.read_pickle(path + \"train_rosters.pickle\")\n# train_standings = pd.read_pickle(path + \"train_standings.pickle\")\n# train_transactions = pd.read_pickle(path + \"train_transactions.pickle\")\n# train_engagements  = pd.read_pickle(path + \"train_nextDayPlayerEngagement.pickle\")\n# train_playerboxscores = pd.read_pickle(path + \"train_playerBoxScores.pickle\")\n# train_playertwitters  = pd.read_pickle(path + \"train_playerTwitterFollowers.pickle\")\n# train_teamboxscores = pd.read_pickle(path + \"train_teamBoxScores.pickle\")\n# train_teamtwitters  = pd.read_pickle(path + \"train_teamTwitterFollowers.pickle\")\n\ntrain_awards = pd.read_pickle(path2 + \"awards_train.pkl\")\ntrain_events = pd.read_pickle(path2 + \"events_train.pkl\")\ntrain_games  = pd.read_pickle(path2 + \"games_train.pkl\")\ntrain_rosters = pd.read_pickle(path2 + \"rosters_train.pkl\")\ntrain_standings = pd.read_pickle(path2 + \"standings_train.pkl\")\ntrain_transactions = pd.read_pickle(path2 + \"transactions_train.pkl\")\ntrain_engagements  = pd.read_pickle(path2 + \"nextDayPlayerEngagement_train.pkl\")\ntrain_playerboxscores = pd.read_pickle(path2 + \"playerBoxScores_train.pkl\")\ntrain_playertwitters  = pd.read_pickle(path2 + \"playerTwitterFollowers_train.pkl\")\ntrain_teamboxscores = pd.read_pickle(path2 + \"teamBoxScores_train.pkl\")\ntrain_teamtwitters  = pd.read_pickle(path2 + \"teamTwitterFollowers_train.pkl\")\n\n# test_awards = pd.read_pickle(path + \"example_test_awards.pickle\")\n# test_events = pd.read_pickle(path + \"example_test_events.pickle\")\n# test_games  = pd.read_pickle(path + \"example_test_games.pickle\")\n# test_rosters = pd.read_pickle(path + \"example_test_rosters.pickle\")\n# test_standings = pd.read_pickle(path + \"example_test_standings.pickle\")\n# test_transactions = pd.read_pickle(path + \"example_test_transactions.pickle\")\n# test_playerboxscores = pd.read_pickle(path + \"example_test_playerBoxScores.pickle\")\n# test_teamboxscores = pd.read_pickle(path + \"example_test_teamBoxScores.pickle\")\nplayers","metadata":{"execution":{"iopub.status.busy":"2021-06-17T06:57:00.283034Z","iopub.execute_input":"2021-06-17T06:57:00.283741Z","iopub.status.idle":"2021-06-17T06:57:18.685518Z","shell.execute_reply.started":"2021-06-17T06:57:00.283692Z","shell.execute_reply":"2021-06-17T06:57:18.684510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events","metadata":{"execution":{"iopub.status.busy":"2021-06-17T06:57:18.687658Z","iopub.execute_input":"2021-06-17T06:57:18.687995Z","iopub.status.idle":"2021-06-17T06:57:20.721433Z","shell.execute_reply.started":"2021-06-17T06:57:18.687963Z","shell.execute_reply":"2021-06-17T06:57:20.720203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_awards[train_awards[\"playerId\"] == 598264]","metadata":{"execution":{"iopub.status.busy":"2021-06-17T06:57:20.722771Z","iopub.execute_input":"2021-06-17T06:57:20.723151Z","iopub.status.idle":"2021-06-17T06:57:20.753873Z","shell.execute_reply.started":"2021-06-17T06:57:20.723118Z","shell.execute_reply":"2021-06-17T06:57:20.752741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_engagements.describe()","metadata":{"execution":{"iopub.status.busy":"2021-06-17T06:57:20.755364Z","iopub.execute_input":"2021-06-17T06:57:20.755784Z","iopub.status.idle":"2021-06-17T06:57:21.369699Z","shell.execute_reply.started":"2021-06-17T06:57:20.755750Z","shell.execute_reply":"2021-06-17T06:57:21.368687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_games","metadata":{"execution":{"iopub.status.busy":"2021-06-17T06:57:21.372785Z","iopub.execute_input":"2021-06-17T06:57:21.373192Z","iopub.status.idle":"2021-06-17T06:57:21.427855Z","shell.execute_reply.started":"2021-06-17T06:57:21.373159Z","shell.execute_reply":"2021-06-17T06:57:21.426805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_rosters","metadata":{"execution":{"iopub.status.busy":"2021-06-17T06:57:21.431350Z","iopub.execute_input":"2021-06-17T06:57:21.431708Z","iopub.status.idle":"2021-06-17T06:57:21.454423Z","shell.execute_reply.started":"2021-06-17T06:57:21.431676Z","shell.execute_reply":"2021-06-17T06:57:21.453027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_standings","metadata":{"execution":{"iopub.status.busy":"2021-06-17T06:57:21.456200Z","iopub.execute_input":"2021-06-17T06:57:21.456673Z","iopub.status.idle":"2021-06-17T06:57:21.512936Z","shell.execute_reply.started":"2021-06-17T06:57:21.456627Z","shell.execute_reply":"2021-06-17T06:57:21.511690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transactions","metadata":{"execution":{"iopub.status.busy":"2021-06-17T06:57:21.514461Z","iopub.execute_input":"2021-06-17T06:57:21.514841Z","iopub.status.idle":"2021-06-17T06:57:21.554895Z","shell.execute_reply.started":"2021-06-17T06:57:21.514807Z","shell.execute_reply":"2021-06-17T06:57:21.553724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_teamtwitters","metadata":{"execution":{"iopub.status.busy":"2021-06-17T06:57:21.556683Z","iopub.execute_input":"2021-06-17T06:57:21.557110Z","iopub.status.idle":"2021-06-17T06:57:21.579177Z","shell.execute_reply.started":"2021-06-17T06:57:21.557065Z","shell.execute_reply":"2021-06-17T06:57:21.578195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_playertwitters","metadata":{"execution":{"iopub.status.busy":"2021-06-17T06:57:21.580524Z","iopub.execute_input":"2021-06-17T06:57:21.580816Z","iopub.status.idle":"2021-06-17T06:57:21.609459Z","shell.execute_reply.started":"2021-06-17T06:57:21.580788Z","shell.execute_reply":"2021-06-17T06:57:21.608349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\ntrain_playertwitters[\"date_new\"] = train_playertwitters[\"date\"].dt.strftime(\"%Y%m%d\").astype(int)\ntrain_teamtwitters[\"date_new\"] = train_teamtwitters[\"date\"].dt.strftime(\"%Y%m%d\").astype(int)\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-06-17T06:57:21.610825Z","iopub.execute_input":"2021-06-17T06:57:21.611116Z","iopub.status.idle":"2021-06-17T06:57:21.622449Z","shell.execute_reply.started":"2021-06-17T06:57:21.611089Z","shell.execute_reply":"2021-06-17T06:57:21.621286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\ncnt = 0\ncnt2 = 0\nfor i in range(train_playertwitters.shape[0]):\n    if(train_playertwitters.loc[i,\"dailyDataDate\"] == train_playertwitters.loc[i,\"date_new\"]):\n        cnt += 1\n\nfor i in range(train_teamtwitters.shape[0]):\n    if(train_teamtwitters.loc[i,\"dailyDataDate\"] == train_teamtwitters.loc[i,\"date_new\"]):\n        cnt2 += 1\nprint(cnt) # equal to player twitter data length -> not need to date\nprint(cnt2) # equal to team twitter data length -> not need to date\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-06-17T06:57:21.624043Z","iopub.execute_input":"2021-06-17T06:57:21.624390Z","iopub.status.idle":"2021-06-17T06:57:21.634073Z","shell.execute_reply.started":"2021-06-17T06:57:21.624347Z","shell.execute_reply":"2021-06-17T06:57:21.633153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_teamboxscores","metadata":{"execution":{"iopub.status.busy":"2021-06-17T06:57:21.635148Z","iopub.execute_input":"2021-06-17T06:57:21.635452Z","iopub.status.idle":"2021-06-17T06:57:21.675040Z","shell.execute_reply.started":"2021-06-17T06:57:21.635423Z","shell.execute_reply":"2021-06-17T06:57:21.673086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = train_playerboxscores\nb = train_playerboxscores.drop_duplicates([\"playerId\",\"gameDate\"])\nprint(a.shape,b.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-17T06:57:21.676575Z","iopub.execute_input":"2021-06-17T06:57:21.676884Z","iopub.status.idle":"2021-06-17T06:57:21.867160Z","shell.execute_reply.started":"2021-06-17T06:57:21.676855Z","shell.execute_reply":"2021-06-17T06:57:21.866194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_engagements","metadata":{"execution":{"iopub.status.busy":"2021-06-17T06:57:21.868456Z","iopub.execute_input":"2021-06-17T06:57:21.868751Z","iopub.status.idle":"2021-06-17T06:57:21.889717Z","shell.execute_reply.started":"2021-06-17T06:57:21.868723Z","shell.execute_reply":"2021-06-17T06:57:21.888717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# merge train_rosters\ntrain1 = pd.merge(train_engagements.drop(\"index\",axis=1),train_rosters.drop([\"index\",\"status\",\"gameDate\"],axis=1),on=[\"date\",\"playerId\"],how=\"left\")\ntrain1","metadata":{"execution":{"iopub.status.busy":"2021-06-17T06:57:21.891145Z","iopub.execute_input":"2021-06-17T06:57:21.891494Z","iopub.status.idle":"2021-06-17T06:57:23.027790Z","shell.execute_reply.started":"2021-06-17T06:57:21.891463Z","shell.execute_reply":"2021-06-17T06:57:23.026543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# merge twitter data\ntrain2_1 = pd.merge(train1,train_playertwitters.drop([\"index\",\"playerName\",\"accountName\",\"twitterHandle\"],axis=1),on=[\"date\",\"playerId\"],how=\"left\")\ntrain2 = pd.merge(train2_1,train_teamtwitters.drop([\"index\",\"teamName\",\"accountName\",\"twitterHandle\"],axis=1),on=[\"date\",\"teamId\"],how=\"left\")\ntrain2","metadata":{"execution":{"iopub.status.busy":"2021-06-17T06:57:23.029501Z","iopub.execute_input":"2021-06-17T06:57:23.029924Z","iopub.status.idle":"2021-06-17T06:57:24.674305Z","shell.execute_reply.started":"2021-06-17T06:57:23.029879Z","shell.execute_reply":"2021-06-17T06:57:24.673096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train2 = train2.rename(columns={\"numberOfFollowers_x\":\"player_followers\",\"numberOfFollowers_y\":\"team_followers\"})\ntrain2","metadata":{"execution":{"iopub.status.busy":"2021-06-17T06:57:24.675831Z","iopub.execute_input":"2021-06-17T06:57:24.676184Z","iopub.status.idle":"2021-06-17T06:57:24.858357Z","shell.execute_reply.started":"2021-06-17T06:57:24.676148Z","shell.execute_reply":"2021-06-17T06:57:24.857290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train2.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-06-17T06:57:24.859985Z","iopub.execute_input":"2021-06-17T06:57:24.860435Z","iopub.status.idle":"2021-06-17T06:57:25.308982Z","shell.execute_reply.started":"2021-06-17T06:57:24.860367Z","shell.execute_reply":"2021-06-17T06:57:25.307826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\nle = LabelEncoder()\nle.fit(train2[\"statusCode\"].values.reshape((-1,1)))\ntrain2[\"statusCode\"] = le.transform(train2[\"statusCode\"].values.reshape((-1,1)))\ntrain2","metadata":{"execution":{"iopub.status.busy":"2021-06-17T06:57:25.310626Z","iopub.execute_input":"2021-06-17T06:57:25.311062Z","iopub.status.idle":"2021-06-17T06:57:27.355433Z","shell.execute_reply.started":"2021-06-17T06:57:25.311016Z","shell.execute_reply":"2021-06-17T06:57:27.354115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train2.groupby([\"date\",\"playerId\"])[\"statusCode\"].nunique().sort_values()","metadata":{"execution":{"iopub.status.busy":"2021-06-17T06:57:27.357141Z","iopub.execute_input":"2021-06-17T06:57:27.357602Z","iopub.status.idle":"2021-06-17T06:57:28.749706Z","shell.execute_reply.started":"2021-06-17T06:57:27.357554Z","shell.execute_reply":"2021-06-17T06:57:28.748712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import KFold,GroupKFold\nfrom lightgbm import LGBMRegressor\nfrom sklearn.multioutput import MultiOutputRegressor\nfrom sklearn.metrics import mean_absolute_error\n# from sklearn.externals import joblib\nimport pickle\n\nX = train2.drop([\"date\",\"engagementMetricsDate\",\"target1\",\"target2\",\"target3\",\"target4\"],axis=1)\ny = train2.loc[:,[\"target1\",\"target2\",\"target3\",\"target4\"]]\n\nlgb_params = { 'objective': 'mean_absolute_error',\n               'boosting_type': 'gbdt',\n               'n_estimators': 10,\n               'learning_rate': 0.01,\n               'num_leaves': 90,\n               'colsample_bytree': 0.4,\n               'subsample': 0.6,\n               'subsample_freq': 2,\n               'bagging_seed': 42,\n               'reg_alpha': 8,\n               'reg_lambda': 2,\n               'random_state': 42,\n               'n_jobs': -1\n             }\n\n# oof = np.zeros((train2.shape[0],4))\n# predict = np.zeros((test.shape[0],))\nN_SPLITS = 10\n\nvalid_predict = []\nvalid_idxes = []\nper_mae = []\nmodels = []\n\n# lgbm = LGBMRegressor(**lgb_params)\n# lgbm_model = MultiOutputRegressor(lgbm)\nkf = KFold(n_splits=N_SPLITS,shuffle=True,random_state=42)\nfor i in range(4):\n    for fn,(tr_idx,val_idx) in enumerate(kf.split(X,y)):\n        train_x,valid_x = X.iloc[tr_idx],X.iloc[val_idx]\n        train_y,valid_y = y.iloc[tr_idx,-4+i],y.iloc[val_idx,-4+i]\n\n        lgbm = LGBMRegressor(**lgb_params)\n        lgbm.fit(train_x,train_y,eval_set=[(valid_x,valid_y)],early_stopping_rounds=30,verbose=0)\n    #     pre_test_lati = lr_lati.predict(test.drop([\"prev_lat\",\"prev_lng\"],axis=1))\n    #     pre_test_long = lr_long.predict(test.drop([\"prev_lat\",\"prev_lng\"],axis=1))\n    #     predict_lati += pre_test_lati / N_SPLITS\n    #     predict_long += pre_test_long / N_SPLITS\n        \n        models.append(lgbm)\n#         joblib.dump(lgbm,f\"target{i+1} - Fold{fn+1}.learn\")\n        pre = lgbm.predict(valid_x)\n#         val_pre = np.array(oof[val_idx])\n#         valid_predict.append(oof[val_idx])\n        valid_idxes.append(val_idx)\n#         with open(f\"target{i+1} - Fold{fn+1}.pickle\", mode='wb') as fp:\n#             pickle.dump(lgbm,fp)\n        \n        \n        print(\"Fold\",str(fn+1),\"：\",str(mean_absolute_error(valid_y,pre)))\n#         print(\"Fold\",str(fn+1),\"：\",str(mean_absolute_error(valid_y,pre2)))\n        per_mae.append(mean_absolute_error(valid_y,pre))\n    print(\"target\",str(i+1),\"MAE：\",np.mean(per_mae[i*10:(i+1)*10]))\nprint(\"Mean Score：\",np.mean(per_mae))","metadata":{"execution":{"iopub.status.busy":"2021-06-17T06:57:28.751181Z","iopub.execute_input":"2021-06-17T06:57:28.751537Z","iopub.status.idle":"2021-06-17T07:00:12.181350Z","shell.execute_reply.started":"2021-06-17T06:57:28.751506Z","shell.execute_reply":"2021-06-17T07:00:12.180412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models","metadata":{"execution":{"iopub.status.busy":"2021-06-17T07:00:12.182799Z","iopub.execute_input":"2021-06-17T07:00:12.183542Z","iopub.status.idle":"2021-06-17T07:00:12.226366Z","shell.execute_reply.started":"2021-06-17T07:00:12.183495Z","shell.execute_reply":"2021-06-17T07:00:12.225352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X","metadata":{"execution":{"iopub.status.busy":"2021-06-17T07:00:12.227632Z","iopub.execute_input":"2021-06-17T07:00:12.227927Z","iopub.status.idle":"2021-06-17T07:00:12.248199Z","shell.execute_reply.started":"2021-06-17T07:00:12.227898Z","shell.execute_reply":"2021-06-17T07:00:12.246944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def unpack_json(json_str):\n    return np.nan if pd.isna(json_str) else pd.read_json(json_str)","metadata":{"execution":{"iopub.status.busy":"2021-06-17T07:00:12.250082Z","iopub.execute_input":"2021-06-17T07:00:12.250573Z","iopub.status.idle":"2021-06-17T07:00:12.261333Z","shell.execute_reply.started":"2021-06-17T07:00:12.250528Z","shell.execute_reply":"2021-06-17T07:00:12.260281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_test = pd.read_csv(\"/kaggle/input/mlb-player-digital-engagement-forecasting/example_test.csv\")\nunpack_json(example_test[\"rosters\"].iloc[4])","metadata":{"execution":{"iopub.status.busy":"2021-06-17T07:00:12.262727Z","iopub.execute_input":"2021-06-17T07:00:12.263318Z","iopub.status.idle":"2021-06-17T07:00:12.938825Z","shell.execute_reply.started":"2021-06-17T07:00:12.263278Z","shell.execute_reply":"2021-06-17T07:00:12.937648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_rosters.columns","metadata":{"execution":{"iopub.status.busy":"2021-06-17T07:00:12.940534Z","iopub.execute_input":"2021-06-17T07:00:12.940859Z","iopub.status.idle":"2021-06-17T07:00:12.947447Z","shell.execute_reply.started":"2021-06-17T07:00:12.940827Z","shell.execute_reply":"2021-06-17T07:00:12.946237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mlb\nimport json\nfrom tqdm.auto import tqdm\nenv = mlb.make_env() # initialize the environment\niter_test = env.iter_test() # iterator which loops over each date in test set\n\nfor (test_df, sample_prediction_df) in tqdm(iter_test):\n#     print(test_df.shape)\n#     print(sample_prediction_df.shape)\n    sample_prediction_df = sample_prediction_df.reset_index()\n    sample_prediction_df['playerId'] = sample_prediction_df['date_playerId'].map(lambda x: int(x.split('_')[1]))\n#     test_rosters = test_df['rosters'].fillna('[]').apply(lambda x:json.loads(x))\n#     test_rosters = list(np.concatenate(test_rosters.values))\n#     test_rosters = pd.DataFrame(test_rosters)\n#     print(test_rosters)\n\n     # Dealing with missing values\n    if test_df['rosters'].iloc[0] == test_df['rosters'].iloc[0]:\n        test_rosters = pd.DataFrame(eval(test_df['rosters'].iloc[0]))\n#         test_rosters['gameDate'] = test_rosters.gameDate.apply(lambda x:x.replace('-','')).astype(int)\n    else:\n        test_rosters = pd.DataFrame({'playerId':sample_prediction_df['playerId']})\n        for col in train_rosters.columns:\n            if col == 'playerId': continue\n            test_rosters[col] = np.nan\n            \n#     test_rosters = test_rosters.rename(columns={'gameDate':'date'})\n    \n    if test_df['playerTwitterFollowers'].iloc[0] == test_df['playerTwitterFollowers'].iloc[0]:\n        test_playertwitters = pd.DataFrame(eval(test_df['playerTwitterFollowers'].iloc[0]))\n#         test_rosters['gameDate'] = test_rosters.gameDate.apply(lambda x:x.replace('-','')).astype(int)\n    else:\n        test_playertwitters = pd.DataFrame({'playerId':sample_prediction_df['playerId']})\n        for col in train_playertwitters.columns:\n            if col == 'playerId': continue\n            test_playertwitters[col] = np.nan\n    \n    test_playertwitters = test_playertwitters.rename(columns={\"numberOfFollowers\":\"player_followers\"})\n    \n    test = sample_prediction_df[['playerId']].copy()\n    df = pd.merge(test,test_rosters[[\"playerId\",\"teamId\",\"statusCode\"]],on=\"playerId\",how='left')\n    df = pd.merge(df,test_playertwitters[[\"playerId\",\"player_followers\"]],on=\"playerId\",how=\"left\")\n    \n    if test_df['teamTwitterFollowers'].iloc[0] == test_df['teamTwitterFollowers'].iloc[0]:\n        test_teamtwitters = pd.DataFrame(eval(test_df['teamTwitterFollowers'].iloc[0]))\n        test_teamtwitters = test_teamtwitters.rename(columns={\"numberOfFollowers\":\"team_followers\"})\n        df = pd.merge(df,test_teamtwitters[[\"teamId\",\"team_followers\"]],on=\"teamId\",how=\"left\")\n#         test_rosters['gameDate'] = test_rosters.gameDate.apply(lambda x:x.replace('-','')).astype(int)\n    else:\n        test_teamtwitters = pd.DataFrame({'playerId':sample_prediction_df['playerId']})\n        test_teamtwitters[\"team_followers\"] = np.nan\n        df = pd.concat([df,test_teamtwitters[\"team_followers\"]],axis=1)\n#         for col in train_teamtwitters.columns[1:]:\n#             if col == 'pId': continue\n#             test_teamtwitters[col] = np.nan\n    \n#     test = sample_prediction_df[['playerId']].copy()\n#     print(test)\n#     df = pd.merge(test,test_rosters[[\"playerId\",\"teamId\",\"statusCode\"]],on=\"playerId\",how='left')\n#     df = pd.merge(df,test_playertwitters[[\"playerId\",\"numberOfFollowers\"]],on=\"playerId\",how=\"left\")\n#     df = pd.merge(df,test_teamtwitters[[\"teamId\",\"numberOfFollowers\"]],on=\"teamId\",how=\"left\")\n#     first_day = [20210501,20210601,20210701,20210801,20210901]\n#     df[\"player_followers\"] = np.nan\n#     df[\"team_followers\"] = np.nan\n    \n#     predicted = pd.DataFrame(index=df.date)\n\n    df[\"statusCode\"] = le.transform(df[\"statusCode\"].values.reshape(-1,1))\n    predict1,predict2,predict3,predict4 = np.zeros((df.shape[0],)),np.zeros((df.shape[0],)),np.zeros((df.shape[0],)),np.zeros((df.shape[0],))\n    predict = [predict1,predict2,predict3,predict4]\n    targets = [\"target1\",\"target2\",\"target3\",\"target4\"]\n    for i in range(4):\n        for j in range(10):\n#             with open(f\"target{i+1} - Fold{j+1}.pickle\", mode='rb') as fp:\n#                 lgbm = pickle.load(fp)\n#                 pre_test = lgbm.predict(df.drop(\"date_playerId\",axis=1))\n#                 predict[i] += pre_test / 10\n            model = models[i*10+j]\n#             print(i*10+j)\n            pre_test = model.predict(df)\n            predict[i] += pre_test / 10\n        pre = np.clip(predict[i],0,100)\n#         print(predict[i])\n#         print(len(predict[i]))\n        sample_prediction_df[targets[i]] = pre\n#         print(df)\n    \n    sample_prediction_df = sample_prediction_df.fillna(0.)\n    sample_prediction_df = sample_prediction_df.set_index(\"date\")\n    del sample_prediction_df['playerId']\n#     print(sample_prediction_df)\n    \n#     for i in range(len(predict)):\n#         predict[i] = pd.DataFrame(predict[i],columns=[f\"target{i+1}\"])\n#         predicted[f\"target{i+1}\"] = predict[i]\n    \n#     predicted['date_playerId'] = sample_prediction_df['date_playerId'].values\n#     predicted = predicted.iloc[:,[4,0,1,2,3]]\n#     predicted.index = sample_prediction_df.reset_index().date\n#     predicted.columns = sample_prediction_df.columns\n#     predicted = predicted.iloc[:,[4,0,1,2,3]]\n#     print(predicted)\n#     print(predicted.shape,sample_prediction_df.shape)\n#     assert df.shape[0]==sample_prediction_df.shape[0]\n\n    env.predict(sample_prediction_df)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-17T07:00:12.948841Z","iopub.execute_input":"2021-06-17T07:00:12.949163Z","iopub.status.idle":"2021-06-17T07:00:15.170688Z","shell.execute_reply.started":"2021-06-17T07:00:12.949135Z","shell.execute_reply":"2021-06-17T07:00:15.169539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2021-06-17T06:29:23.175931Z","iopub.execute_input":"2021-06-17T06:29:23.176271Z","iopub.status.idle":"2021-06-17T06:29:23.19686Z","shell.execute_reply.started":"2021-06-17T06:29:23.176238Z","shell.execute_reply":"2021-06-17T06:29:23.195909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i*10+j","metadata":{"execution":{"iopub.status.busy":"2021-06-15T01:34:44.902399Z","iopub.execute_input":"2021-06-15T01:34:44.90275Z","iopub.status.idle":"2021-06-15T01:34:44.909383Z","shell.execute_reply.started":"2021-06-15T01:34:44.90272Z","shell.execute_reply":"2021-06-15T01:34:44.908005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_test = pd.read_csv(\"/kaggle/input/mlb-player-digital-engagement-forecasting/example_test.csv\")\nexample_test.loc[0,\"rosters\"]","metadata":{"execution":{"iopub.status.busy":"2021-06-15T07:59:09.805126Z","iopub.execute_input":"2021-06-15T07:59:09.805488Z","iopub.status.idle":"2021-06-15T07:59:10.144332Z","shell.execute_reply.started":"2021-06-15T07:59:09.805458Z","shell.execute_reply":"2021-06-15T07:59:10.143574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"b = [0.2,-1,128,45.3,21]\nprint(np.clip(b,0,100))","metadata":{"execution":{"iopub.status.busy":"2021-06-15T02:31:23.901463Z","iopub.execute_input":"2021-06-15T02:31:23.901891Z","iopub.status.idle":"2021-06-15T02:31:23.911488Z","shell.execute_reply.started":"2021-06-15T02:31:23.901859Z","shell.execute_reply":"2021-06-15T02:31:23.909982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.loc[:,\"rosters\"]","metadata":{"execution":{"iopub.status.busy":"2021-06-15T05:39:10.19684Z","iopub.execute_input":"2021-06-15T05:39:10.197196Z","iopub.status.idle":"2021-06-15T05:39:10.204393Z","shell.execute_reply.started":"2021-06-15T05:39:10.197168Z","shell.execute_reply":"2021-06-15T05:39:10.20358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_prediction_df","metadata":{"execution":{"iopub.status.busy":"2021-06-17T07:00:29.409686Z","iopub.execute_input":"2021-06-17T07:00:29.410053Z","iopub.status.idle":"2021-06-17T07:00:29.428958Z","shell.execute_reply.started":"2021-06-17T07:00:29.410024Z","shell.execute_reply":"2021-06-17T07:00:29.427491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_prediction_df.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-16T02:30:26.769569Z","iopub.execute_input":"2021-06-16T02:30:26.770157Z","iopub.status.idle":"2021-06-16T02:30:26.791585Z","shell.execute_reply.started":"2021-06-16T02:30:26.770121Z","shell.execute_reply":"2021-06-16T02:30:26.789719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-16T01:57:38.280328Z","iopub.execute_input":"2021-06-16T01:57:38.280992Z","iopub.status.idle":"2021-06-16T01:57:38.306559Z","shell.execute_reply.started":"2021-06-16T01:57:38.280941Z","shell.execute_reply":"2021-06-16T01:57:38.305563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_prediction_df","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:45:15.893401Z","iopub.execute_input":"2021-06-15T06:45:15.893793Z","iopub.status.idle":"2021-06-15T06:45:15.914006Z","shell.execute_reply.started":"2021-06-15T06:45:15.893762Z","shell.execute_reply":"2021-06-15T06:45:15.912897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:45:00.808079Z","iopub.execute_input":"2021-06-15T06:45:00.808521Z","iopub.status.idle":"2021-06-15T06:45:00.827874Z","shell.execute_reply.started":"2021-06-15T06:45:00.808484Z","shell.execute_reply":"2021-06-15T06:45:00.826811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}